feat:移除了弹窗,服务器添加sls
This commit is contained in:
24
apps/telemetry-worker/README.md
Normal file
24
apps/telemetry-worker/README.md
Normal file
@ -0,0 +1,24 @@
|
||||
# Axolotl telemetry worker
|
||||
|
||||
This Worker accepts only opted-in launcher heartbeat batches. It stores anonymous usage metadata in D1; error reports and error context are not accepted or persisted.
|
||||
|
||||
## Provisioning
|
||||
|
||||
```sh
|
||||
pnpm exec wrangler d1 create axolotl-telemetry
|
||||
pnpm exec wrangler secret put INSTALLATION_HMAC_SECRET
|
||||
pnpm exec wrangler d1 migrations apply axolotl-telemetry --remote
|
||||
```
|
||||
|
||||
Replace `database_id` in `wrangler.toml` and keep the Worker on the Free plan. The production custom domain is declared in `wrangler.toml`. Configure Cloudflare usage notifications at 50%, 75%, and 90% for Workers and D1. Do not enable Workers Paid.
|
||||
|
||||
Clients that still have queued error reports receive `400 error_reporting_disabled`; the payload is not logged or stored. Upgrade clients should discard their local error outbox.
|
||||
|
||||
## Storage design
|
||||
|
||||
D1 stays within the Free plan's daily limits (5M rows read / 100k rows written) by avoiding per-event aggregation triggers:
|
||||
|
||||
- `daily_active` inserts feed `wau_seen` / `mau_seen` / `daily_active_dims` through a single lightweight trigger for O(1) WAU/MAU and distribution reads.
|
||||
- A nightly cron (`runMaintenance`) refreshes usage totals, rolls the active windows, and applies usage-data retention. Legacy error tables remain in the schema for migration compatibility but are no longer written.
|
||||
|
||||
Schema changes are forward-only migrations in `migrations/`; never edit an applied migration.
|
||||
219
apps/telemetry-worker/migrations/0001_init.sql
Normal file
219
apps/telemetry-worker/migrations/0001_init.sql
Normal file
@ -0,0 +1,219 @@
|
||||
CREATE TABLE installations (
|
||||
installation_hash TEXT PRIMARY KEY,
|
||||
first_seen_at INTEGER NOT NULL,
|
||||
last_seen_at INTEGER NOT NULL,
|
||||
first_seen_day TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
arch TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE daily_active (
|
||||
day TEXT NOT NULL,
|
||||
installation_hash TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
PRIMARY KEY (day, installation_hash),
|
||||
FOREIGN KEY (installation_hash) REFERENCES installations (installation_hash)
|
||||
);
|
||||
|
||||
CREATE TABLE error_reports (
|
||||
event_id TEXT PRIMARY KEY,
|
||||
installation_hash TEXT NOT NULL,
|
||||
day TEXT NOT NULL,
|
||||
occurred_at TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
error_type TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
object_key TEXT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
FOREIGN KEY (installation_hash) REFERENCES installations (installation_hash)
|
||||
);
|
||||
|
||||
CREATE INDEX error_reports_day ON error_reports (day);
|
||||
CREATE INDEX error_reports_fingerprint ON error_reports (fingerprint, day);
|
||||
|
||||
CREATE TABLE error_groups (
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
first_seen_day TEXT NOT NULL,
|
||||
last_seen_day TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
installation_count INTEGER NOT NULL,
|
||||
latest_error_type TEXT NOT NULL,
|
||||
latest_message TEXT NOT NULL,
|
||||
sample_object_key TEXT NULL,
|
||||
PRIMARY KEY (fingerprint, app_version)
|
||||
);
|
||||
|
||||
CREATE TABLE error_daily (
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
installation_count INTEGER NOT NULL,
|
||||
PRIMARY KEY (day, fingerprint, app_version)
|
||||
);
|
||||
|
||||
CREATE TABLE accepted_batches (
|
||||
batch_id TEXT PRIMARY KEY,
|
||||
installation_hash TEXT NOT NULL,
|
||||
accepted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE daily_totals (
|
||||
day TEXT PRIMARY KEY,
|
||||
new_installations INTEGER NOT NULL DEFAULT 0,
|
||||
active_installations INTEGER NOT NULL DEFAULT 0,
|
||||
error_occurrences INTEGER NOT NULL DEFAULT 0,
|
||||
distinct_error_groups INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE error_context_budget (
|
||||
day TEXT PRIMARY KEY,
|
||||
object_count INTEGER NOT NULL CHECK (object_count <= 2000)
|
||||
);
|
||||
|
||||
CREATE TABLE error_context_samples (
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
sample_count INTEGER NOT NULL CHECK (sample_count <= 3),
|
||||
PRIMARY KEY (day, fingerprint, app_version)
|
||||
);
|
||||
|
||||
CREATE TABLE error_context_reservations (
|
||||
event_id TEXT PRIMARY KEY,
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TRIGGER installations_daily_total
|
||||
AFTER INSERT ON installations
|
||||
BEGIN
|
||||
INSERT INTO daily_totals (day, new_installations)
|
||||
VALUES (NEW.first_seen_day, 1)
|
||||
ON CONFLICT (day) DO UPDATE
|
||||
SET new_installations = new_installations + 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER daily_active_total
|
||||
AFTER INSERT ON daily_active
|
||||
BEGIN
|
||||
INSERT INTO daily_totals (day, active_installations)
|
||||
VALUES (NEW.day, 1)
|
||||
ON CONFLICT (day) DO UPDATE
|
||||
SET active_installations = active_installations + 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER error_report_aggregates
|
||||
AFTER INSERT ON error_reports
|
||||
BEGIN
|
||||
INSERT INTO error_groups
|
||||
(
|
||||
fingerprint,
|
||||
app_version,
|
||||
first_seen_day,
|
||||
last_seen_day,
|
||||
occurrence_count,
|
||||
installation_count,
|
||||
latest_error_type,
|
||||
latest_message,
|
||||
sample_object_key
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
NEW.fingerprint,
|
||||
NEW.app_version,
|
||||
NEW.day,
|
||||
NEW.day,
|
||||
NEW.occurrence_count,
|
||||
1,
|
||||
NEW.error_type,
|
||||
NEW.message,
|
||||
NEW.object_key
|
||||
)
|
||||
ON CONFLICT (fingerprint, app_version) DO UPDATE
|
||||
SET
|
||||
last_seen_day = excluded.last_seen_day,
|
||||
occurrence_count = occurrence_count + excluded.occurrence_count,
|
||||
installation_count = installation_count + CASE
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM error_reports AS previous
|
||||
WHERE
|
||||
previous.fingerprint = NEW.fingerprint
|
||||
AND previous.app_version = NEW.app_version
|
||||
AND previous.installation_hash = NEW.installation_hash
|
||||
AND previous.event_id != NEW.event_id
|
||||
) THEN 1
|
||||
ELSE 0
|
||||
END,
|
||||
latest_error_type = excluded.latest_error_type,
|
||||
latest_message = excluded.latest_message,
|
||||
sample_object_key = COALESCE(error_groups.sample_object_key, excluded.sample_object_key);
|
||||
|
||||
INSERT INTO error_daily
|
||||
(day, fingerprint, app_version, occurrence_count, installation_count)
|
||||
VALUES (NEW.day, NEW.fingerprint, NEW.app_version, NEW.occurrence_count, 1)
|
||||
ON CONFLICT (day, fingerprint, app_version) DO UPDATE
|
||||
SET
|
||||
occurrence_count = occurrence_count + excluded.occurrence_count,
|
||||
installation_count = installation_count + CASE
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM error_reports AS previous
|
||||
WHERE
|
||||
previous.day = NEW.day
|
||||
AND previous.fingerprint = NEW.fingerprint
|
||||
AND previous.app_version = NEW.app_version
|
||||
AND previous.installation_hash = NEW.installation_hash
|
||||
AND previous.event_id != NEW.event_id
|
||||
) THEN 1
|
||||
ELSE 0
|
||||
END;
|
||||
|
||||
INSERT INTO daily_totals (day, error_occurrences, distinct_error_groups)
|
||||
VALUES
|
||||
(
|
||||
NEW.day,
|
||||
NEW.occurrence_count,
|
||||
CASE
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM error_reports AS previous
|
||||
WHERE
|
||||
previous.day = NEW.day
|
||||
AND previous.fingerprint = NEW.fingerprint
|
||||
AND previous.app_version = NEW.app_version
|
||||
AND previous.event_id != NEW.event_id
|
||||
) THEN 1
|
||||
ELSE 0
|
||||
END
|
||||
)
|
||||
ON CONFLICT (day) DO UPDATE
|
||||
SET
|
||||
error_occurrences = error_occurrences + excluded.error_occurrences,
|
||||
distinct_error_groups = distinct_error_groups + excluded.distinct_error_groups;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER error_context_reservation_budget
|
||||
AFTER INSERT ON error_context_reservations
|
||||
BEGIN
|
||||
INSERT INTO error_context_budget (day, object_count)
|
||||
VALUES (NEW.day, 1)
|
||||
ON CONFLICT (day) DO UPDATE SET object_count = object_count + 1;
|
||||
|
||||
INSERT INTO error_context_samples (day, fingerprint, app_version, sample_count)
|
||||
VALUES (NEW.day, NEW.fingerprint, NEW.app_version, 1)
|
||||
ON CONFLICT (day, fingerprint, app_version) DO UPDATE
|
||||
SET sample_count = sample_count + 1;
|
||||
END;
|
||||
155
apps/telemetry-worker/migrations/0002_optimize_usage.sql
Normal file
155
apps/telemetry-worker/migrations/0002_optimize_usage.sql
Normal file
@ -0,0 +1,155 @@
|
||||
-- 0002: eliminate per-event aggregation write/read amplification and add missing indexes.
|
||||
--
|
||||
-- The previous design ran three aggregate UPSERTs (error_groups / error_daily /
|
||||
-- daily_totals) inside an AFTER INSERT trigger for every error report, each with
|
||||
-- a NOT EXISTS subquery that scanned the whole fingerprint history. That single
|
||||
-- INSERT statement accounted for ~8.2M rows read and ~147k rows written per day,
|
||||
-- far beyond the Workers Free daily limits (5M rows read / 100k rows written).
|
||||
--
|
||||
-- This migration:
|
||||
-- 1. Drops the per-event aggregation triggers; counts now flow through a
|
||||
-- realtime error_daily upsert in the worker and nightly cron rollups.
|
||||
-- 2. Rebuilds error_daily and accepted_batches as WITHOUT ROWID so every
|
||||
-- upsert/insert costs one written row instead of two.
|
||||
-- 3. Adds seen/dimension tables for O(1) distinct-install tracking.
|
||||
-- 4. Swaps the error_reports index so filter queries become point lookups.
|
||||
-- 5. Backfills the new tables from existing data.
|
||||
|
||||
DROP TRIGGER error_report_aggregates;
|
||||
DROP TRIGGER installations_daily_total;
|
||||
DROP TRIGGER daily_active_total;
|
||||
|
||||
ALTER TABLE error_daily RENAME TO error_daily_legacy;
|
||||
CREATE TABLE error_daily (
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
installation_count INTEGER NOT NULL,
|
||||
PRIMARY KEY (day, fingerprint, app_version)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
INSERT INTO error_daily (day, fingerprint, app_version, occurrence_count, installation_count)
|
||||
SELECT day, fingerprint, app_version, occurrence_count, installation_count
|
||||
FROM error_daily_legacy;
|
||||
DROP TABLE error_daily_legacy;
|
||||
|
||||
ALTER TABLE accepted_batches RENAME TO accepted_batches_legacy;
|
||||
CREATE TABLE accepted_batches (
|
||||
batch_id TEXT PRIMARY KEY,
|
||||
installation_hash TEXT NOT NULL,
|
||||
accepted_at INTEGER NOT NULL
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
INSERT INTO accepted_batches (batch_id, installation_hash, accepted_at)
|
||||
SELECT batch_id, installation_hash, accepted_at FROM accepted_batches_legacy;
|
||||
DROP TABLE accepted_batches_legacy;
|
||||
|
||||
CREATE TABLE error_daily_installations (
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
installation_hash TEXT NOT NULL,
|
||||
PRIMARY KEY (day, fingerprint, app_version, installation_hash)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE error_group_installations (
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
installation_hash TEXT NOT NULL,
|
||||
first_seen_day TEXT NOT NULL,
|
||||
PRIMARY KEY (fingerprint, app_version, installation_hash)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE wau_seen (
|
||||
installation_hash TEXT PRIMARY KEY
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE mau_seen (
|
||||
installation_hash TEXT PRIMARY KEY
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE daily_active_dims (
|
||||
dimension TEXT NOT NULL,
|
||||
day TEXT NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
install_count INTEGER NOT NULL,
|
||||
PRIMARY KEY (dimension, day, label)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE platforms (
|
||||
platform TEXT PRIMARY KEY
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE error_range_stats (
|
||||
range_days INTEGER NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
installation_count INTEGER NOT NULL,
|
||||
latest_error_type TEXT NOT NULL,
|
||||
latest_message TEXT NOT NULL,
|
||||
PRIMARY KEY (range_days, fingerprint, app_version)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
DROP INDEX error_reports_fingerprint;
|
||||
CREATE INDEX error_reports_group_day ON error_reports (fingerprint, app_version, day);
|
||||
CREATE INDEX installations_first_seen_day ON installations (first_seen_day);
|
||||
CREATE INDEX error_context_reservations_day ON error_context_reservations (day);
|
||||
CREATE INDEX error_context_reservations_fingerprint_created ON error_context_reservations (
|
||||
fingerprint,
|
||||
created_at
|
||||
);
|
||||
CREATE INDEX error_context_reservations_created_at ON error_context_reservations (created_at);
|
||||
CREATE INDEX error_groups_last_seen_day ON error_groups (last_seen_day);
|
||||
CREATE INDEX accepted_batches_accepted_at ON accepted_batches (accepted_at);
|
||||
CREATE INDEX daily_active_installation ON daily_active (installation_hash, day);
|
||||
|
||||
CREATE TRIGGER daily_active_rollups
|
||||
AFTER INSERT ON daily_active
|
||||
BEGIN
|
||||
INSERT OR IGNORE INTO wau_seen (installation_hash) VALUES (NEW.installation_hash);
|
||||
INSERT OR IGNORE INTO mau_seen (installation_hash) VALUES (NEW.installation_hash);
|
||||
INSERT INTO daily_active_dims (dimension, day, label, install_count)
|
||||
VALUES ('version', NEW.day, NEW.app_version, 1)
|
||||
ON CONFLICT (dimension, day, label) DO UPDATE SET install_count = install_count + 1;
|
||||
INSERT INTO daily_active_dims (dimension, day, label, install_count)
|
||||
VALUES ('platform', NEW.day, NEW.platform, 1)
|
||||
ON CONFLICT (dimension, day, label) DO UPDATE SET install_count = install_count + 1;
|
||||
INSERT INTO daily_active_dims (dimension, day, label, install_count)
|
||||
VALUES ('arch', NEW.day, NEW.arch, 1)
|
||||
ON CONFLICT (dimension, day, label) DO UPDATE SET install_count = install_count + 1;
|
||||
END;
|
||||
|
||||
INSERT OR IGNORE INTO wau_seen (installation_hash)
|
||||
SELECT DISTINCT installation_hash FROM daily_active WHERE day >= date('now', '-6 days');
|
||||
INSERT OR IGNORE INTO mau_seen (installation_hash)
|
||||
SELECT DISTINCT installation_hash FROM daily_active WHERE day >= date('now', '-29 days');
|
||||
INSERT OR IGNORE INTO platforms (platform)
|
||||
SELECT DISTINCT platform FROM error_reports WHERE day >= date('now', '-30 days');
|
||||
INSERT OR IGNORE INTO error_group_installations
|
||||
(fingerprint, app_version, installation_hash, first_seen_day)
|
||||
SELECT fingerprint, app_version, installation_hash, MIN(day)
|
||||
FROM error_reports
|
||||
GROUP BY fingerprint, app_version, installation_hash;
|
||||
INSERT OR IGNORE INTO daily_active_dims (dimension, day, label, install_count)
|
||||
SELECT 'version', day, app_version, COUNT(DISTINCT installation_hash)
|
||||
FROM daily_active
|
||||
GROUP BY day, app_version;
|
||||
INSERT OR IGNORE INTO daily_active_dims (dimension, day, label, install_count)
|
||||
SELECT 'platform', day, platform, COUNT(DISTINCT installation_hash)
|
||||
FROM daily_active
|
||||
GROUP BY day, platform;
|
||||
INSERT OR IGNORE INTO daily_active_dims (dimension, day, label, install_count)
|
||||
SELECT 'arch', day, arch, COUNT(DISTINCT installation_hash)
|
||||
FROM daily_active
|
||||
GROUP BY day, arch;
|
||||
@ -0,0 +1,44 @@
|
||||
-- 0003: precompute display fields on error_daily so dashboard list queries stay pure aggregations.
|
||||
--
|
||||
-- The errors list live branch previously resolved per-group correlated subqueries
|
||||
-- (distinct-install count, latest type/message, sample existence) on every page
|
||||
-- load, costing ~145k rows read per execution. These are now maintained by the
|
||||
-- worker on every upsert and only read back, cutting a list execution to ~15k rows.
|
||||
|
||||
ALTER TABLE error_daily ADD COLUMN latest_error_type TEXT NOT NULL DEFAULT 'Unknown';
|
||||
ALTER TABLE error_daily ADD COLUMN latest_message TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE error_daily ADD COLUMN has_sample INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE error_range_stats ADD COLUMN has_sample INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
-- Backfill exact per-day distinct installs for reports ingested before the
|
||||
-- error_daily_installations table existed.
|
||||
INSERT OR IGNORE INTO error_daily_installations (day, fingerprint, app_version, installation_hash)
|
||||
SELECT day, fingerprint, app_version, installation_hash
|
||||
FROM error_reports
|
||||
WHERE day >= date('now', '-1 days');
|
||||
|
||||
-- Backfill display fields for existing rows from the best available sources.
|
||||
UPDATE error_daily
|
||||
SET
|
||||
latest_error_type = (
|
||||
SELECT er.error_type
|
||||
FROM error_reports AS er
|
||||
WHERE er.fingerprint = error_daily.fingerprint AND er.app_version = error_daily.app_version
|
||||
ORDER BY er.occurred_at DESC, er.event_id DESC
|
||||
LIMIT 1
|
||||
),
|
||||
latest_message = (
|
||||
SELECT er.message
|
||||
FROM error_reports AS er
|
||||
WHERE er.fingerprint = error_daily.fingerprint AND er.app_version = error_daily.app_version
|
||||
ORDER BY er.occurred_at DESC, er.event_id DESC
|
||||
LIMIT 1
|
||||
),
|
||||
has_sample = CASE
|
||||
WHEN EXISTS (
|
||||
SELECT 1
|
||||
FROM error_context_reservations AS r
|
||||
WHERE r.fingerprint = error_daily.fingerprint AND r.app_version = error_daily.app_version
|
||||
) THEN 1
|
||||
ELSE 0
|
||||
END;
|
||||
23
apps/telemetry-worker/migrations/0004_ingestion_limits.sql
Normal file
23
apps/telemetry-worker/migrations/0004_ingestion_limits.sql
Normal file
@ -0,0 +1,23 @@
|
||||
CREATE TABLE ingestion_daily (
|
||||
day TEXT NOT NULL,
|
||||
installation_hash TEXT NOT NULL,
|
||||
accepted_batches INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (day, installation_hash)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE ingestion_global_daily (
|
||||
day TEXT PRIMARY KEY,
|
||||
accepted_batches INTEGER NOT NULL DEFAULT 0
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
INSERT INTO ingestion_daily (day, installation_hash, accepted_batches)
|
||||
SELECT date(accepted_at, 'unixepoch'), installation_hash, COUNT(*)
|
||||
FROM accepted_batches
|
||||
GROUP BY date(accepted_at, 'unixepoch'), installation_hash;
|
||||
|
||||
INSERT INTO ingestion_global_daily (day, accepted_batches)
|
||||
SELECT date(accepted_at, 'unixepoch'), COUNT(*)
|
||||
FROM accepted_batches
|
||||
GROUP BY date(accepted_at, 'unixepoch');
|
||||
@ -0,0 +1,24 @@
|
||||
-- 0005: stop retaining legacy error telemetry.
|
||||
--
|
||||
-- Error events are no longer accepted by the worker. Keep the historical
|
||||
-- tables in place so older database snapshots and dashboard migrations remain
|
||||
-- readable, but remove all previously collected error payloads and reset the
|
||||
-- error counters exposed through daily_totals.
|
||||
|
||||
DELETE FROM error_reports;
|
||||
DELETE FROM error_daily_installations;
|
||||
DELETE FROM error_group_installations;
|
||||
DELETE FROM error_daily;
|
||||
DELETE FROM error_range_stats;
|
||||
DELETE FROM error_groups;
|
||||
DELETE FROM error_context_reservations;
|
||||
DELETE FROM error_context_samples;
|
||||
DELETE FROM error_context_budget;
|
||||
-- Platforms are repopulated from heartbeat batches and may previously have
|
||||
-- been introduced only by an error report.
|
||||
DELETE FROM platforms;
|
||||
|
||||
UPDATE daily_totals
|
||||
SET
|
||||
error_occurrences = 0,
|
||||
distinct_error_groups = 0;
|
||||
25
apps/telemetry-worker/package.json
Normal file
25
apps/telemetry-worker/package.json
Normal file
@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "@axolotl/telemetry-worker",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"configure:rate-limit": "node scripts/configure-rate-limit.mjs",
|
||||
"dev": "wrangler dev",
|
||||
"deploy": "wrangler deploy",
|
||||
"lint": "prettier --check . && tsc --noEmit",
|
||||
"fix": "prettier --write .",
|
||||
"test": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"hono": "^4.9.2",
|
||||
"zod": "^3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@cloudflare/vitest-pool-workers": "^0.8.69",
|
||||
"@cloudflare/workers-types": "^5.20260801.1",
|
||||
"@types/node": "^24",
|
||||
"typescript": "^5.9.2",
|
||||
"vitest": "^3.2.4",
|
||||
"wrangler": "^4.28.1"
|
||||
}
|
||||
}
|
||||
81
apps/telemetry-worker/scripts/configure-rate-limit.mjs
Normal file
81
apps/telemetry-worker/scripts/configure-rate-limit.mjs
Normal file
@ -0,0 +1,81 @@
|
||||
const apiToken = process.env.CF_API_TOKEN
|
||||
const zoneId = process.env.CF_ZONE_ID
|
||||
|
||||
if (!apiToken || !zoneId) {
|
||||
throw new Error('CF_API_TOKEN and CF_ZONE_ID are required')
|
||||
}
|
||||
|
||||
const apiBase = `https://api.cloudflare.com/client/v4/zones/${zoneId}/rulesets`
|
||||
const ruleDescription = 'Axolotl telemetry batch ingress limit'
|
||||
const telemetryRule = {
|
||||
action: 'block',
|
||||
description: ruleDescription,
|
||||
enabled: true,
|
||||
expression:
|
||||
'http.host eq "telemetry.axlmc.org" and http.request.method eq "POST" and http.request.uri.path eq "/v1/batch"',
|
||||
action_parameters: {
|
||||
ratelimit: {
|
||||
characteristics: ['ip.src'],
|
||||
period: 60,
|
||||
requests_per_period: 2,
|
||||
mitigation_timeout: 60,
|
||||
requests_to_origin: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
async function request(path = '', options = {}) {
|
||||
const response = await fetch(`${apiBase}${path}`, {
|
||||
...options,
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiToken}`,
|
||||
'Content-Type': 'application/json',
|
||||
...(options.headers ?? {}),
|
||||
},
|
||||
})
|
||||
const payload = await response.json()
|
||||
if (!response.ok || !payload.success) {
|
||||
throw new Error(`Cloudflare API request failed: ${JSON.stringify(payload.errors ?? payload)}`)
|
||||
}
|
||||
return payload.result
|
||||
}
|
||||
|
||||
function editableRule(rule) {
|
||||
const { action, action_parameters, description, enabled, expression, id, logging } = rule
|
||||
return { action, action_parameters, description, enabled, expression, id, logging }
|
||||
}
|
||||
|
||||
const rulesets = await request('?phase=http_ratelimit')
|
||||
const rateLimitRuleset = rulesets.find((ruleset) => ruleset.phase === 'http_ratelimit')
|
||||
|
||||
if (!rateLimitRuleset) {
|
||||
await request('', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
kind: 'zone',
|
||||
name: 'Axolotl rate limits',
|
||||
phase: 'http_ratelimit',
|
||||
rules: [telemetryRule],
|
||||
}),
|
||||
})
|
||||
console.log('Created the Axolotl telemetry batch rate-limit rule')
|
||||
} else {
|
||||
const existing = await request(`/${rateLimitRuleset.id}`)
|
||||
const rules = existing.rules.map(editableRule)
|
||||
const index = rules.findIndex((rule) => rule.description === ruleDescription)
|
||||
if (index >= 0) {
|
||||
rules[index] = { ...telemetryRule, id: rules[index].id }
|
||||
} else {
|
||||
rules.push(telemetryRule)
|
||||
}
|
||||
await request(`/${rateLimitRuleset.id}`, {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify({
|
||||
name: existing.name,
|
||||
kind: existing.kind,
|
||||
phase: existing.phase,
|
||||
rules,
|
||||
}),
|
||||
})
|
||||
console.log('Updated the Axolotl telemetry batch rate-limit rule')
|
||||
}
|
||||
378
apps/telemetry-worker/src/index.ts
Normal file
378
apps/telemetry-worker/src/index.ts
Normal file
@ -0,0 +1,378 @@
|
||||
import { Hono } from 'hono'
|
||||
|
||||
import { redact, truncateUtf8 } from './redact'
|
||||
import { batchSchema, type TelemetryBatch } from './schema'
|
||||
|
||||
const MAX_REQUEST_BYTES = 64 * 1024
|
||||
const HARD_MAX_BATCHES_PER_INSTALLATION_PER_DAY = 25
|
||||
const HARD_MAX_ACCEPTED_BATCHES_PER_DAY = 100_000
|
||||
|
||||
export interface Bindings {
|
||||
DB: D1Database
|
||||
INSTALLATION_HMAC_SECRET: string
|
||||
INGEST_ENABLED?: string
|
||||
MAX_BATCHES_PER_INSTALLATION_PER_DAY?: string
|
||||
MAX_ACCEPTED_BATCHES_PER_DAY?: string
|
||||
}
|
||||
|
||||
type Variables = { requestId: string }
|
||||
type IngestionReservation = 'reserved' | 'duplicate' | 'installation_limit' | 'global_limit'
|
||||
|
||||
const app = new Hono<{ Bindings: Bindings; Variables: Variables }>()
|
||||
|
||||
app.use('*', async (context, next) => {
|
||||
context.set('requestId', crypto.randomUUID())
|
||||
await next()
|
||||
context.header('Cache-Control', 'no-store')
|
||||
})
|
||||
|
||||
app.get('/health', (context) =>
|
||||
context.json({ status: 'ok', schema_version: 1 }, 200, { 'Cache-Control': 'no-store' }),
|
||||
)
|
||||
|
||||
app.post('/v1/batch', async (context) => {
|
||||
const contentLength = Number(context.req.header('content-length') ?? '0')
|
||||
if (Number.isFinite(contentLength) && contentLength > MAX_REQUEST_BYTES) {
|
||||
return context.json({ error: 'request_too_large' }, 413)
|
||||
}
|
||||
|
||||
const raw = await context.req.arrayBuffer()
|
||||
if (raw.byteLength > MAX_REQUEST_BYTES) return context.json({ error: 'request_too_large' }, 413)
|
||||
|
||||
let input: unknown
|
||||
try {
|
||||
input = JSON.parse(new TextDecoder().decode(raw))
|
||||
} catch {
|
||||
return context.json({ error: 'invalid_json' }, 400)
|
||||
}
|
||||
|
||||
const parsed = batchSchema.safeParse(input)
|
||||
if (!parsed.success) {
|
||||
// Do not accept or log legacy error payloads after error reporting was removed.
|
||||
if (containsLegacyErrorEvent(input))
|
||||
return context.json({ error: 'error_reporting_disabled' }, 400)
|
||||
return context.json(
|
||||
{
|
||||
error: 'invalid_batch',
|
||||
issues: parsed.error.issues.map((issue) => ({
|
||||
path: issue.path.join('.'),
|
||||
code: issue.code,
|
||||
})),
|
||||
},
|
||||
400,
|
||||
)
|
||||
}
|
||||
if (context.env.INGEST_ENABLED === 'false') {
|
||||
return context.json({ error: 'ingest_disabled' }, 503, { 'Retry-After': '60' })
|
||||
}
|
||||
if (!context.env.INSTALLATION_HMAC_SECRET || context.env.INSTALLATION_HMAC_SECRET.length < 32) {
|
||||
return context.json({ error: 'service_unavailable' }, 503)
|
||||
}
|
||||
|
||||
try {
|
||||
const accepted = await context.env.DB.prepare(
|
||||
'SELECT 1 FROM accepted_batches WHERE batch_id = ? LIMIT 1',
|
||||
)
|
||||
.bind(parsed.data.batch_id)
|
||||
.first()
|
||||
if (accepted) return context.json({ accepted: true, duplicate: true })
|
||||
|
||||
const installationHash = await hmacInstallationId(
|
||||
context.env.INSTALLATION_HMAC_SECRET,
|
||||
parsed.data.installation_id,
|
||||
)
|
||||
const limits = ingestionLimits(context.env)
|
||||
const reservation = await reserveIngestion(
|
||||
context.env.DB,
|
||||
parsed.data.batch_id,
|
||||
installationHash,
|
||||
limits,
|
||||
)
|
||||
if (reservation === 'duplicate') return context.json({ accepted: true, duplicate: true })
|
||||
if (reservation === 'installation_limit') {
|
||||
return context.json({ error: 'installation_batch_limit' }, 429, {
|
||||
'Retry-After': retryAfterSeconds().toString(),
|
||||
})
|
||||
}
|
||||
if (reservation === 'global_limit') {
|
||||
console.error('Telemetry ingestion budget exhausted', { limit: limits.global })
|
||||
return context.json({ error: 'global_batch_limit' }, 429, {
|
||||
'Retry-After': retryAfterSeconds().toString(),
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
await persistBatch(context.env.DB, sanitizeBatch(parsed.data), installationHash)
|
||||
} catch (error) {
|
||||
await rollbackIngestion(context.env.DB, parsed.data.batch_id, installationHash)
|
||||
throw error
|
||||
}
|
||||
return context.json({ accepted: true, duplicate: false })
|
||||
} catch (error) {
|
||||
console.error('Telemetry ingestion failed', {
|
||||
requestId: context.get('requestId'),
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
return context.json({ error: 'temporarily_unavailable' }, 503)
|
||||
}
|
||||
})
|
||||
|
||||
app.notFound((context) => context.json({ error: 'not_found' }, 404))
|
||||
app.onError((error, context) => {
|
||||
console.error('Unhandled telemetry worker error', {
|
||||
requestId: context.get('requestId'),
|
||||
error: error.message,
|
||||
})
|
||||
return context.json({ error: 'temporarily_unavailable' }, 503)
|
||||
})
|
||||
|
||||
function containsLegacyErrorEvent(input: unknown): boolean {
|
||||
if (!input || typeof input !== 'object') return false
|
||||
const events = (input as { events?: unknown }).events
|
||||
return (
|
||||
Array.isArray(events) &&
|
||||
events.some(
|
||||
(event) =>
|
||||
Boolean(event) &&
|
||||
typeof event === 'object' &&
|
||||
(event as { type?: unknown }).type === 'error',
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
async function hmacInstallationId(secret: string, installationId: string): Promise<string> {
|
||||
const key = await crypto.subtle.importKey(
|
||||
'raw',
|
||||
new TextEncoder().encode(secret),
|
||||
{ name: 'HMAC', hash: 'SHA-256' },
|
||||
false,
|
||||
['sign'],
|
||||
)
|
||||
const signature = await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(installationId))
|
||||
return [...new Uint8Array(signature)].map((byte) => byte.toString(16).padStart(2, '0')).join('')
|
||||
}
|
||||
|
||||
function ingestionLimits(env: Bindings): { installation: number; global: number } {
|
||||
return {
|
||||
installation: Math.min(
|
||||
positiveInteger(
|
||||
env.MAX_BATCHES_PER_INSTALLATION_PER_DAY,
|
||||
HARD_MAX_BATCHES_PER_INSTALLATION_PER_DAY,
|
||||
),
|
||||
HARD_MAX_BATCHES_PER_INSTALLATION_PER_DAY,
|
||||
),
|
||||
global: Math.min(
|
||||
positiveInteger(env.MAX_ACCEPTED_BATCHES_PER_DAY, HARD_MAX_ACCEPTED_BATCHES_PER_DAY),
|
||||
HARD_MAX_ACCEPTED_BATCHES_PER_DAY,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
async function reserveIngestion(
|
||||
db: D1Database,
|
||||
batchId: string,
|
||||
installationHash: string,
|
||||
limits: { installation: number; global: number },
|
||||
): Promise<IngestionReservation> {
|
||||
const day = utcDay()
|
||||
const batch = await db
|
||||
.prepare(
|
||||
'INSERT OR IGNORE INTO accepted_batches (batch_id, installation_hash, accepted_at) VALUES (?, ?, unixepoch())',
|
||||
)
|
||||
.bind(batchId, installationHash)
|
||||
.run()
|
||||
if (batch.meta.changes !== 1) return 'duplicate'
|
||||
|
||||
const installation = await db
|
||||
.prepare(
|
||||
`INSERT INTO ingestion_daily (day, installation_hash, accepted_batches)
|
||||
VALUES (?, ?, 1)
|
||||
ON CONFLICT (day, installation_hash) DO UPDATE
|
||||
SET accepted_batches = accepted_batches + 1
|
||||
WHERE accepted_batches < ?`,
|
||||
)
|
||||
.bind(day, installationHash, limits.installation)
|
||||
.run()
|
||||
if (installation.meta.changes !== 1) {
|
||||
await db.prepare('DELETE FROM accepted_batches WHERE batch_id = ?').bind(batchId).run()
|
||||
return 'installation_limit'
|
||||
}
|
||||
|
||||
const global = await db
|
||||
.prepare(
|
||||
`INSERT INTO ingestion_global_daily (day, accepted_batches)
|
||||
VALUES (?, 1)
|
||||
ON CONFLICT (day) DO UPDATE
|
||||
SET accepted_batches = accepted_batches + 1
|
||||
WHERE accepted_batches < ?`,
|
||||
)
|
||||
.bind(day, limits.global)
|
||||
.run()
|
||||
if (global.meta.changes !== 1) {
|
||||
await db.batch([
|
||||
db
|
||||
.prepare(
|
||||
'UPDATE ingestion_daily SET accepted_batches = accepted_batches - 1 WHERE day = ? AND installation_hash = ?',
|
||||
)
|
||||
.bind(day, installationHash),
|
||||
db.prepare('DELETE FROM accepted_batches WHERE batch_id = ?').bind(batchId),
|
||||
])
|
||||
return 'global_limit'
|
||||
}
|
||||
|
||||
const currentGlobal = await db
|
||||
.prepare('SELECT accepted_batches FROM ingestion_global_daily WHERE day = ?')
|
||||
.bind(day)
|
||||
.first<{ accepted_batches: number }>()
|
||||
if (currentGlobal) warnIngestionThresholds(currentGlobal.accepted_batches, limits.global)
|
||||
return 'reserved'
|
||||
}
|
||||
|
||||
async function rollbackIngestion(
|
||||
db: D1Database,
|
||||
batchId: string,
|
||||
installationHash: string,
|
||||
): Promise<void> {
|
||||
const day = utcDay()
|
||||
await db.batch([
|
||||
db
|
||||
.prepare('DELETE FROM accepted_batches WHERE batch_id = ? AND installation_hash = ?')
|
||||
.bind(batchId, installationHash),
|
||||
db
|
||||
.prepare(
|
||||
'UPDATE ingestion_daily SET accepted_batches = accepted_batches - 1 WHERE day = ? AND installation_hash = ?',
|
||||
)
|
||||
.bind(day, installationHash),
|
||||
db
|
||||
.prepare(
|
||||
'UPDATE ingestion_global_daily SET accepted_batches = accepted_batches - 1 WHERE day = ?',
|
||||
)
|
||||
.bind(day),
|
||||
])
|
||||
}
|
||||
|
||||
function sanitizeBatch(batch: TelemetryBatch): TelemetryBatch {
|
||||
return {
|
||||
...batch,
|
||||
app: {
|
||||
...batch.app,
|
||||
version: truncateUtf8(redact(batch.app.version), 64),
|
||||
platform: truncateUtf8(redact(batch.app.platform), 32),
|
||||
arch: truncateUtf8(redact(batch.app.arch), 32),
|
||||
},
|
||||
events: batch.events,
|
||||
}
|
||||
}
|
||||
|
||||
async function persistBatch(
|
||||
db: D1Database,
|
||||
batch: TelemetryBatch,
|
||||
installationHash: string,
|
||||
): Promise<void> {
|
||||
const acceptedDay = utcDay()
|
||||
const statements: D1PreparedStatement[] = [
|
||||
db
|
||||
.prepare(
|
||||
`INSERT OR IGNORE INTO installations (
|
||||
installation_hash, first_seen_at, last_seen_at,
|
||||
first_seen_day, app_version, platform, arch
|
||||
) VALUES (?, unixepoch(), unixepoch(), ?, ?, ?, ?)`,
|
||||
)
|
||||
.bind(installationHash, acceptedDay, batch.app.version, batch.app.platform, batch.app.arch),
|
||||
db.prepare('INSERT OR IGNORE INTO platforms (platform) VALUES (?)').bind(batch.app.platform),
|
||||
]
|
||||
|
||||
for (const event of batch.events) {
|
||||
statements.push(
|
||||
db
|
||||
.prepare(
|
||||
'INSERT OR IGNORE INTO daily_active (day, installation_hash, app_version, platform, arch) VALUES (?, ?, ?, ?, ?)',
|
||||
)
|
||||
.bind(event.day, installationHash, batch.app.version, batch.app.platform, batch.app.arch),
|
||||
)
|
||||
}
|
||||
await db.batch(statements)
|
||||
}
|
||||
|
||||
async function runMaintenance(db: D1Database): Promise<void> {
|
||||
const now = new Date()
|
||||
const yesterday = daysAgo(now, 1)
|
||||
const dayMinus = (days: number) => daysAgo(now, days)
|
||||
|
||||
// Usage-only maintenance. Error aggregate/R2 work was intentionally removed.
|
||||
await db.batch([
|
||||
db
|
||||
.prepare(
|
||||
`INSERT INTO daily_totals (day, new_installations, active_installations)
|
||||
VALUES (?,
|
||||
(SELECT COUNT(*) FROM installations WHERE first_seen_day = ?),
|
||||
(SELECT COUNT(*) FROM daily_active WHERE day = ?))
|
||||
ON CONFLICT (day) DO UPDATE SET
|
||||
new_installations = excluded.new_installations,
|
||||
active_installations = excluded.active_installations`,
|
||||
)
|
||||
.bind(yesterday, yesterday, yesterday),
|
||||
db
|
||||
.prepare(
|
||||
`DELETE FROM wau_seen WHERE NOT EXISTS (
|
||||
SELECT 1 FROM daily_active da
|
||||
WHERE da.installation_hash = wau_seen.installation_hash AND da.day >= ?
|
||||
)`,
|
||||
)
|
||||
.bind(dayMinus(6)),
|
||||
db
|
||||
.prepare(
|
||||
`DELETE FROM mau_seen WHERE NOT EXISTS (
|
||||
SELECT 1 FROM daily_active da
|
||||
WHERE da.installation_hash = mau_seen.installation_hash AND da.day >= ?
|
||||
)`,
|
||||
)
|
||||
.bind(dayMinus(29)),
|
||||
db.prepare("DELETE FROM daily_active WHERE day < date('now', '-35 days')"),
|
||||
db.prepare("DELETE FROM accepted_batches WHERE accepted_at < unixepoch('now', '-8 days')"),
|
||||
db.prepare("DELETE FROM ingestion_daily WHERE day < date('now', '-8 days')"),
|
||||
db.prepare("DELETE FROM ingestion_global_daily WHERE day < date('now', '-8 days')"),
|
||||
db.prepare("DELETE FROM daily_active_dims WHERE day < date('now', '-365 days')"),
|
||||
])
|
||||
}
|
||||
|
||||
function daysAgo(now: Date, days: number): string {
|
||||
const date = new Date(now)
|
||||
date.setUTCDate(date.getUTCDate() - days)
|
||||
return date.toISOString().slice(0, 10)
|
||||
}
|
||||
|
||||
function positiveInteger(value: string | undefined, fallback: number): number {
|
||||
const parsed = Number(value)
|
||||
return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : fallback
|
||||
}
|
||||
|
||||
function utcDay(): string {
|
||||
return new Date().toISOString().slice(0, 10)
|
||||
}
|
||||
|
||||
function warnIngestionThresholds(count: number, limit: number): void {
|
||||
for (const ratio of [0.8, 0.9, 0.95]) {
|
||||
if (count === Math.ceil(limit * ratio)) {
|
||||
console.warn('Telemetry ingestion budget threshold reached', {
|
||||
count,
|
||||
limit,
|
||||
percent: ratio * 100,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function retryAfterSeconds(now = new Date()): number {
|
||||
const nextDay = Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate() + 1)
|
||||
return Math.max(1, Math.ceil((nextDay - now.getTime()) / 1_000))
|
||||
}
|
||||
|
||||
export { app, hmacInstallationId, runMaintenance, sanitizeBatch }
|
||||
|
||||
export default {
|
||||
fetch: app.fetch,
|
||||
async scheduled(_controller: ScheduledController, env: Bindings, context: ExecutionContext) {
|
||||
context.waitUntil(runMaintenance(env.DB))
|
||||
},
|
||||
} satisfies ExportedHandler<Bindings>
|
||||
35
apps/telemetry-worker/src/redact.ts
Normal file
35
apps/telemetry-worker/src/redact.ts
Normal file
@ -0,0 +1,35 @@
|
||||
const encoder = new TextEncoder()
|
||||
const decoder = new TextDecoder()
|
||||
|
||||
const replacements: Array<[RegExp, string]> = [
|
||||
[/\bbearer\s+[a-z0-9._~+/=-]+/gi, 'Bearer <redacted>'],
|
||||
[
|
||||
/\b(authorization|x-api-key|api[_-]?key|access[_-]?token|refresh[_-]?token|client[_-]?secret|token)\b\s*[:=]\s*[^\s,;]+/gi,
|
||||
'$1=<redacted>',
|
||||
],
|
||||
[
|
||||
/([?&](?:token|access_token|refresh_token|api_key|key|code|secret|session|signature)=)[^&#\s]+/gi,
|
||||
'$1<redacted>',
|
||||
],
|
||||
[/\b[a-z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-z0-9.-]+\.[a-z]{2,}\b/gi, '<email>'],
|
||||
[/\b[a-z]:\\users\\[^\\/\s]+/gi, '<home>'],
|
||||
[/(?:\/home|\/users)\/[^/\s]+\//gi, '<home>/'],
|
||||
[/\b[0-9a-f]{8}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{4}-?[0-9a-f]{12}\b/gi, '<uuid>'],
|
||||
]
|
||||
|
||||
export function redact(input: string): string {
|
||||
return replacements.reduce(
|
||||
(value, [pattern, replacement]) => value.replace(pattern, replacement),
|
||||
input.replaceAll('\0', ''),
|
||||
)
|
||||
}
|
||||
|
||||
export function truncateUtf8(input: string, maxBytes: number): string {
|
||||
const bytes = encoder.encode(input)
|
||||
if (bytes.byteLength <= maxBytes) return input
|
||||
return decoder.decode(bytes.slice(0, maxBytes)).replace(/\uFFFD$/, '')
|
||||
}
|
||||
|
||||
export function byteLength(input: string): number {
|
||||
return encoder.encode(input).byteLength
|
||||
}
|
||||
34
apps/telemetry-worker/src/schema.ts
Normal file
34
apps/telemetry-worker/src/schema.ts
Normal file
@ -0,0 +1,34 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
const uuid = z.string().regex(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i)
|
||||
const timestamp = z.string().datetime({ offset: true })
|
||||
|
||||
export const heartbeatEventSchema = z
|
||||
.object({
|
||||
type: z.literal('heartbeat'),
|
||||
event_id: uuid,
|
||||
occurred_at: timestamp,
|
||||
day: z.string().regex(/^\d{4}-\d{2}-\d{2}$/),
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const telemetryEventSchema = heartbeatEventSchema
|
||||
|
||||
export const batchSchema = z
|
||||
.object({
|
||||
schema_version: z.literal(1),
|
||||
batch_id: uuid,
|
||||
installation_id: uuid,
|
||||
app: z
|
||||
.object({
|
||||
version: z.string().min(1).max(64),
|
||||
environment: z.enum(['production', 'development']),
|
||||
platform: z.string().min(1).max(32),
|
||||
arch: z.string().min(1).max(32),
|
||||
})
|
||||
.strict(),
|
||||
events: z.array(telemetryEventSchema).min(1).max(10),
|
||||
})
|
||||
.strict()
|
||||
|
||||
export type TelemetryBatch = z.infer<typeof batchSchema>
|
||||
13
apps/telemetry-worker/test/setup.ts
Normal file
13
apps/telemetry-worker/test/setup.ts
Normal file
@ -0,0 +1,13 @@
|
||||
import { applyD1Migrations, env, type D1Migration } from 'cloudflare:test'
|
||||
import { beforeAll } from 'vitest'
|
||||
|
||||
declare module 'cloudflare:test' {
|
||||
interface ProvidedEnv {
|
||||
DB: D1Database
|
||||
TEST_MIGRATIONS: D1Migration[]
|
||||
}
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
await applyD1Migrations(env.DB, env.TEST_MIGRATIONS)
|
||||
})
|
||||
210
apps/telemetry-worker/test/worker.test.ts
Normal file
210
apps/telemetry-worker/test/worker.test.ts
Normal file
@ -0,0 +1,210 @@
|
||||
import { env, SELF } from 'cloudflare:test'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { runMaintenance, type Bindings } from '../src'
|
||||
import { batchSchema } from '../src/schema'
|
||||
|
||||
declare module 'cloudflare:test' {
|
||||
interface ProvidedEnv extends Bindings {}
|
||||
}
|
||||
|
||||
const installationId = '018f6ee8-4cb1-7db3-8a8d-8df96f122d85'
|
||||
|
||||
function batch(
|
||||
batchId: string,
|
||||
events: Array<Record<string, unknown>>,
|
||||
clientInstallationId = installationId,
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
schema_version: 1,
|
||||
batch_id: batchId,
|
||||
installation_id: clientInstallationId,
|
||||
app: {
|
||||
version: '1.7.1',
|
||||
environment: 'production',
|
||||
platform: 'windows',
|
||||
arch: 'x86_64',
|
||||
},
|
||||
events,
|
||||
}
|
||||
}
|
||||
|
||||
function heartbeat(eventId: string, day = new Date().toISOString().slice(0, 10)) {
|
||||
return { type: 'heartbeat', event_id: eventId, occurred_at: `${day}T12:00:00.000Z`, day }
|
||||
}
|
||||
|
||||
async function post(payload: unknown): Promise<Response> {
|
||||
return await SELF.fetch('https://telemetry.example/v1/batch', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
}
|
||||
|
||||
describe('telemetry worker', () => {
|
||||
it('serves health without storage access', async () => {
|
||||
const response = await SELF.fetch('https://telemetry.example/health')
|
||||
expect(response.status).toBe(200)
|
||||
expect(await response.json()).toEqual({ status: 'ok', schema_version: 1 })
|
||||
})
|
||||
|
||||
it('accepts strict heartbeat batches and rejects unknown structures', async () => {
|
||||
const invalid = batch('11111111-1111-4111-8111-111111111111', [
|
||||
{ ...heartbeat('21111111-1111-4111-8111-111111111111'), unknown: true },
|
||||
])
|
||||
expect(batchSchema.safeParse(invalid).success).toBe(false)
|
||||
expect((await post(invalid)).status).toBe(400)
|
||||
|
||||
const oversized = await SELF.fetch('https://telemetry.example/v1/batch', {
|
||||
method: 'POST',
|
||||
body: 'x'.repeat(65 * 1024),
|
||||
})
|
||||
expect(oversized.status).toBe(413)
|
||||
})
|
||||
|
||||
it('rejects legacy error batches without charging quota or storing payloads', async () => {
|
||||
const payload = batch('12111111-1111-4111-8111-111111111111', [
|
||||
{
|
||||
type: 'error',
|
||||
event_id: '13111111-1111-4111-8111-111111111111',
|
||||
occurred_at: new Date().toISOString(),
|
||||
fingerprint: 'a'.repeat(64),
|
||||
occurrence_count: 1,
|
||||
error_type: 'legacy_error',
|
||||
message: 'must not be retained',
|
||||
},
|
||||
])
|
||||
const response = await post(payload)
|
||||
expect(response.status).toBe(400)
|
||||
expect(await response.json()).toEqual({ error: 'error_reporting_disabled' })
|
||||
|
||||
const stored = await env.DB.prepare(
|
||||
'SELECT COUNT(*) AS count FROM accepted_batches WHERE batch_id = ?',
|
||||
)
|
||||
.bind('12111111-1111-4111-8111-111111111111')
|
||||
.first<{ count: number }>()
|
||||
expect(stored?.count).toBe(0)
|
||||
})
|
||||
|
||||
it('enforces the daily installation batch cap without charging duplicates', async () => {
|
||||
const cappedInstallation = '018f6ee8-4cb1-7db3-8a8d-8df96f122d99'
|
||||
const day = new Date().toISOString().slice(0, 10)
|
||||
const payloadFor = (index: number) =>
|
||||
batch(
|
||||
`60000000-0000-4000-8000-${index.toString().padStart(12, '0')}`,
|
||||
[heartbeat(`70000000-0000-4000-8000-${index.toString().padStart(12, '0')}`, day)],
|
||||
cappedInstallation,
|
||||
)
|
||||
|
||||
for (let index = 0; index < 25; index++)
|
||||
expect((await post(payloadFor(index))).status).toBe(200)
|
||||
expect((await post(payloadFor(0))).status).toBe(200)
|
||||
|
||||
const rejected = await post(payloadFor(25))
|
||||
expect(rejected.status).toBe(429)
|
||||
expect(rejected.headers.get('Retry-After')).not.toBeNull()
|
||||
})
|
||||
|
||||
it('opens the global ingestion circuit breaker at 100,000 accepted batches', async () => {
|
||||
const day = new Date().toISOString().slice(0, 10)
|
||||
await env.DB.prepare(
|
||||
`INSERT INTO ingestion_global_daily (day, accepted_batches)
|
||||
VALUES (?, 99999)
|
||||
ON CONFLICT (day) DO UPDATE SET accepted_batches = 99999`,
|
||||
)
|
||||
.bind(day)
|
||||
.run()
|
||||
|
||||
const first = batch(
|
||||
'80000000-0000-4000-8000-000000000001',
|
||||
[heartbeat('81000000-0000-4000-8000-000000000001', day)],
|
||||
'018f6ee8-4cb1-7db3-8a8d-8df96f122d98',
|
||||
)
|
||||
expect((await post(first)).status).toBe(200)
|
||||
const rejected = await post({
|
||||
...first,
|
||||
batch_id: '80000000-0000-4000-8000-000000000002',
|
||||
events: [heartbeat('81000000-0000-4000-8000-000000000002', day)],
|
||||
})
|
||||
expect(rejected.status).toBe(429)
|
||||
})
|
||||
|
||||
it('hashes installations and keeps heartbeat batches idempotent', async () => {
|
||||
const payload = batch('31111111-1111-4111-8111-111111111111', [
|
||||
heartbeat('41111111-1111-4111-8111-111111111111', '2026-08-14'),
|
||||
])
|
||||
expect((await post(payload)).status).toBe(200)
|
||||
expect((await post(payload)).status).toBe(200)
|
||||
|
||||
const installation = await env.DB.prepare('SELECT installation_hash FROM installations').first<{
|
||||
installation_hash: string
|
||||
}>()
|
||||
expect(installation?.installation_hash).toMatch(/^[0-9a-f]{64}$/)
|
||||
expect(installation?.installation_hash).not.toBe(installationId)
|
||||
const active = await env.DB.prepare('SELECT COUNT(*) AS count FROM daily_active').first<{
|
||||
count: number
|
||||
}>()
|
||||
expect(active?.count).toBe(1)
|
||||
})
|
||||
|
||||
it('keeps offline heartbeat dates for DAU, WAU, and MAU queries', async () => {
|
||||
const day = (daysAgo: number) =>
|
||||
new Date(Date.now() - daysAgo * 24 * 60 * 60 * 1_000).toISOString().slice(0, 10)
|
||||
const samples = [
|
||||
[
|
||||
'32111111-1111-4111-8111-111111111111',
|
||||
'42111111-1111-4111-8111-111111111111',
|
||||
installationId.replace(/85$/, '81'),
|
||||
0,
|
||||
],
|
||||
[
|
||||
'33111111-1111-4111-8111-111111111111',
|
||||
'43111111-1111-4111-8111-111111111111',
|
||||
installationId.replace(/85$/, '82'),
|
||||
6,
|
||||
],
|
||||
[
|
||||
'34111111-1111-4111-8111-111111111111',
|
||||
'44111111-1111-4111-8111-111111111111',
|
||||
installationId.replace(/85$/, '83'),
|
||||
29,
|
||||
],
|
||||
] as const
|
||||
for (const [batchId, eventId, clientId, offset] of samples) {
|
||||
const heartbeatDay = day(offset)
|
||||
expect(
|
||||
(await post(batch(batchId, [heartbeat(eventId, heartbeatDay)], clientId))).status,
|
||||
).toBe(200)
|
||||
}
|
||||
|
||||
const counts = await env.DB.prepare(
|
||||
`SELECT
|
||||
COUNT(DISTINCT CASE WHEN day = ? THEN installation_hash END) AS dau,
|
||||
COUNT(DISTINCT CASE WHEN day >= date(?, '-6 days') THEN installation_hash END) AS wau,
|
||||
COUNT(DISTINCT CASE WHEN day >= date(?, '-29 days') THEN installation_hash END) AS mau
|
||||
FROM daily_active`,
|
||||
)
|
||||
.bind(day(0), day(0), day(0))
|
||||
.first<{ dau: number; wau: number; mau: number }>()
|
||||
expect(counts).toEqual({ dau: 1, wau: 2, mau: 3 })
|
||||
})
|
||||
|
||||
it('refreshes usage totals without rebuilding error aggregates', async () => {
|
||||
const yesterday = new Date(Date.now() - 24 * 60 * 60 * 1_000).toISOString().slice(0, 10)
|
||||
const response = await post(
|
||||
batch('a2111111-1111-4111-8111-111111111111', [
|
||||
heartbeat('a3111111-1111-4111-8111-111111111111', yesterday),
|
||||
]),
|
||||
)
|
||||
expect(response.status).toBe(200)
|
||||
await runMaintenance(env.DB)
|
||||
const totals = await env.DB.prepare(
|
||||
'SELECT new_installations, active_installations, error_occurrences, distinct_error_groups FROM daily_totals WHERE day = ?',
|
||||
)
|
||||
.bind(yesterday)
|
||||
.first<Record<string, number>>()
|
||||
expect(totals?.active_installations).toBeGreaterThanOrEqual(1)
|
||||
expect(totals?.error_occurrences).toBe(0)
|
||||
expect(totals?.distinct_error_groups).toBe(0)
|
||||
})
|
||||
})
|
||||
13
apps/telemetry-worker/tsconfig.json
Normal file
13
apps/telemetry-worker/tsconfig.json
Normal file
@ -0,0 +1,13 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"lib": ["ES2022", "WebWorker"],
|
||||
"types": ["@cloudflare/workers-types", "@cloudflare/vitest-pool-workers"],
|
||||
"strict": true,
|
||||
"noEmit": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": ["src/**/*.ts", "test/**/*.ts", "vitest.config.ts"]
|
||||
}
|
||||
26
apps/telemetry-worker/vitest.config.ts
Normal file
26
apps/telemetry-worker/vitest.config.ts
Normal file
@ -0,0 +1,26 @@
|
||||
import { defineWorkersConfig, readD1Migrations } from '@cloudflare/vitest-pool-workers/config'
|
||||
|
||||
const migrations = await readD1Migrations('./migrations')
|
||||
|
||||
export default defineWorkersConfig({
|
||||
test: {
|
||||
setupFiles: ['./test/setup.ts'],
|
||||
poolOptions: {
|
||||
workers: {
|
||||
main: './src/index.ts',
|
||||
singleWorker: true,
|
||||
miniflare: {
|
||||
compatibilityDate: '2025-08-13',
|
||||
d1Databases: ['DB'],
|
||||
bindings: {
|
||||
INSTALLATION_HMAC_SECRET: 'test-only-secret-that-is-longer-than-thirty-two-bytes',
|
||||
INGEST_ENABLED: 'true',
|
||||
MAX_BATCHES_PER_INSTALLATION_PER_DAY: '25',
|
||||
MAX_ACCEPTED_BATCHES_PER_DAY: '100000',
|
||||
TEST_MIGRATIONS: migrations,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
23
apps/telemetry-worker/wrangler.toml
Normal file
23
apps/telemetry-worker/wrangler.toml
Normal file
@ -0,0 +1,23 @@
|
||||
name = "axolotl-telemetry"
|
||||
main = "src/index.ts"
|
||||
compatibility_date = "2025-08-13"
|
||||
workers_dev = false
|
||||
routes = [{ pattern = "telemetry.axlmc.org", custom_domain = true }]
|
||||
|
||||
[vars]
|
||||
INGEST_ENABLED = "true"
|
||||
MAX_BATCHES_PER_INSTALLATION_PER_DAY = "25"
|
||||
MAX_ACCEPTED_BATCHES_PER_DAY = "100000"
|
||||
|
||||
[[d1_databases]]
|
||||
binding = "DB"
|
||||
database_name = "axolotl-telemetry"
|
||||
database_id = "8692e2b1-6156-4b0b-94f5-6e7204c05dff"
|
||||
migrations_dir = "migrations"
|
||||
|
||||
[triggers]
|
||||
crons = ["17 0 * * *"]
|
||||
|
||||
[observability]
|
||||
enabled = true
|
||||
head_sampling_rate = 0.01
|
||||
Reference in New Issue
Block a user