StasonJatham
e8f7d6bfdf
Single stateless Go process that consumes CT logs via embedded (patched) CertSpotter internals, classifies cloud/database/storage DNS identifiers in-process with a zero-allocation classifier, enriches via round-robin DNS resolvers with CNAME following, and batch-writes useful findings to PostgreSQL. - third_party/certspotter: 2-hunk IdentifierSink patch, no stdout/JSON/files - internal/classify: AWS/Azure/GCP/Firebase/Cloudflare R2/DigitalOcean/Supabase/OCI - internal/dns: sharded bounded TTL cache, multi-resolver load balancing + failover - internal/pipeline: bounded queues, dedup, direct-priority load shedding - internal/store: pgx batch writer, Postgres CT checkpoints (no cert contents) - internal/probe: optional non-destructive reachability checks (off by default) - Docker: distroless non-root, read-only rootfs, no app volume, resource limits - tests enforce 0 allocs/op on the classifier and sink hot path
130 lines
3.4 KiB
Go
130 lines
3.4 KiB
Go
// Copyright (C) 2020, 2023 Opsmate, Inc.
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla
|
|
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
|
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
//
|
|
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
|
// See the Mozilla Public License for details.
|
|
|
|
package loglist
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
var UserAgent = ""
|
|
|
|
// maxResponseBytes bounds the size of a log list we are willing to read into memory.
|
|
const maxResponseBytes = 16 << 20 // 16 MiB
|
|
|
|
// httpClient is used for fetching log lists.
|
|
var httpClient = &http.Client{
|
|
Timeout: 60 * time.Second,
|
|
}
|
|
|
|
type ModificationToken struct {
|
|
etag string
|
|
modified time.Time
|
|
}
|
|
|
|
var ErrNotModified = errors.New("loglist has not been modified")
|
|
|
|
func newModificationToken(response *http.Response) *ModificationToken {
|
|
token := &ModificationToken{
|
|
etag: response.Header.Get("ETag"),
|
|
}
|
|
if t, err := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified")); err == nil {
|
|
token.modified = t
|
|
}
|
|
return token
|
|
}
|
|
|
|
func (token *ModificationToken) setRequestHeaders(request *http.Request) {
|
|
if token.etag != "" {
|
|
request.Header.Set("If-None-Match", token.etag)
|
|
} else if !token.modified.IsZero() {
|
|
request.Header.Set("If-Modified-Since", token.modified.Format(http.TimeFormat))
|
|
}
|
|
}
|
|
|
|
func Load(ctx context.Context, urlOrFile string) (*List, error) {
|
|
list, _, err := LoadIfModified(ctx, urlOrFile, nil)
|
|
return list, err
|
|
}
|
|
|
|
func LoadIfModified(ctx context.Context, urlOrFile string, token *ModificationToken) (*List, *ModificationToken, error) {
|
|
if strings.HasPrefix(urlOrFile, "https://") {
|
|
return FetchIfModified(ctx, urlOrFile, token)
|
|
} else {
|
|
list, err := ReadFile(urlOrFile)
|
|
return list, nil, err
|
|
}
|
|
}
|
|
|
|
func Fetch(ctx context.Context, url string) (*List, error) {
|
|
list, _, err := FetchIfModified(ctx, url, nil)
|
|
return list, err
|
|
}
|
|
|
|
func FetchIfModified(ctx context.Context, url string, token *ModificationToken) (*List, *ModificationToken, error) {
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
request.Header.Set("User-Agent", UserAgent)
|
|
if token != nil {
|
|
token.setRequestHeaders(request)
|
|
}
|
|
response, err := httpClient.Do(request)
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
content, err := io.ReadAll(io.LimitReader(response.Body, maxResponseBytes+1))
|
|
response.Body.Close()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
if len(content) > maxResponseBytes {
|
|
return nil, nil, fmt.Errorf("%s: response body exceeds maximum allowed size of %d bytes", url, maxResponseBytes)
|
|
}
|
|
if token != nil && response.StatusCode == http.StatusNotModified {
|
|
return nil, nil, ErrNotModified
|
|
}
|
|
if response.StatusCode != 200 {
|
|
return nil, nil, fmt.Errorf("%s: %s", url, response.Status)
|
|
}
|
|
list, err := Unmarshal(content)
|
|
if err != nil {
|
|
return nil, nil, fmt.Errorf("error parsing %s: %w", url, err)
|
|
}
|
|
return list, newModificationToken(response), err
|
|
}
|
|
|
|
func ReadFile(filename string) (*List, error) {
|
|
content, err := os.ReadFile(filename)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return Unmarshal(content)
|
|
}
|
|
|
|
func Unmarshal(jsonBytes []byte) (*List, error) {
|
|
list := new(List)
|
|
if err := json.Unmarshal(jsonBytes, list); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := list.Validate(); err != nil {
|
|
return nil, fmt.Errorf("invalid log list: %s", err)
|
|
}
|
|
return list, nil
|
|
}
|