feat：移除了弹窗，服务器添加sls
This commit is contained in:
219
apps/telemetry-worker/migrations/0001_init.sql
Normal file
219
apps/telemetry-worker/migrations/0001_init.sql
Normal file
@ -0,0 +1,219 @@
|
||||
CREATE TABLE installations (
|
||||
installation_hash TEXT PRIMARY KEY,
|
||||
first_seen_at INTEGER NOT NULL,
|
||||
last_seen_at INTEGER NOT NULL,
|
||||
first_seen_day TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
arch TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE daily_active (
|
||||
day TEXT NOT NULL,
|
||||
installation_hash TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
PRIMARY KEY (day, installation_hash),
|
||||
FOREIGN KEY (installation_hash) REFERENCES installations (installation_hash)
|
||||
);
|
||||
|
||||
CREATE TABLE error_reports (
|
||||
event_id TEXT PRIMARY KEY,
|
||||
installation_hash TEXT NOT NULL,
|
||||
day TEXT NOT NULL,
|
||||
occurred_at TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
error_type TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
object_key TEXT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
FOREIGN KEY (installation_hash) REFERENCES installations (installation_hash)
|
||||
);
|
||||
|
||||
CREATE INDEX error_reports_day ON error_reports (day);
|
||||
CREATE INDEX error_reports_fingerprint ON error_reports (fingerprint, day);
|
||||
|
||||
CREATE TABLE error_groups (
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
first_seen_day TEXT NOT NULL,
|
||||
last_seen_day TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
installation_count INTEGER NOT NULL,
|
||||
latest_error_type TEXT NOT NULL,
|
||||
latest_message TEXT NOT NULL,
|
||||
sample_object_key TEXT NULL,
|
||||
PRIMARY KEY (fingerprint, app_version)
|
||||
);
|
||||
|
||||
CREATE TABLE error_daily (
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
installation_count INTEGER NOT NULL,
|
||||
PRIMARY KEY (day, fingerprint, app_version)
|
||||
);
|
||||
|
||||
CREATE TABLE accepted_batches (
|
||||
batch_id TEXT PRIMARY KEY,
|
||||
installation_hash TEXT NOT NULL,
|
||||
accepted_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE daily_totals (
|
||||
day TEXT PRIMARY KEY,
|
||||
new_installations INTEGER NOT NULL DEFAULT 0,
|
||||
active_installations INTEGER NOT NULL DEFAULT 0,
|
||||
error_occurrences INTEGER NOT NULL DEFAULT 0,
|
||||
distinct_error_groups INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE error_context_budget (
|
||||
day TEXT PRIMARY KEY,
|
||||
object_count INTEGER NOT NULL CHECK (object_count <= 2000)
|
||||
);
|
||||
|
||||
CREATE TABLE error_context_samples (
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
sample_count INTEGER NOT NULL CHECK (sample_count <= 3),
|
||||
PRIMARY KEY (day, fingerprint, app_version)
|
||||
);
|
||||
|
||||
CREATE TABLE error_context_reservations (
|
||||
event_id TEXT PRIMARY KEY,
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TRIGGER installations_daily_total
|
||||
AFTER INSERT ON installations
|
||||
BEGIN
|
||||
INSERT INTO daily_totals (day, new_installations)
|
||||
VALUES (NEW.first_seen_day, 1)
|
||||
ON CONFLICT (day) DO UPDATE
|
||||
SET new_installations = new_installations + 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER daily_active_total
|
||||
AFTER INSERT ON daily_active
|
||||
BEGIN
|
||||
INSERT INTO daily_totals (day, active_installations)
|
||||
VALUES (NEW.day, 1)
|
||||
ON CONFLICT (day) DO UPDATE
|
||||
SET active_installations = active_installations + 1;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER error_report_aggregates
|
||||
AFTER INSERT ON error_reports
|
||||
BEGIN
|
||||
INSERT INTO error_groups
|
||||
(
|
||||
fingerprint,
|
||||
app_version,
|
||||
first_seen_day,
|
||||
last_seen_day,
|
||||
occurrence_count,
|
||||
installation_count,
|
||||
latest_error_type,
|
||||
latest_message,
|
||||
sample_object_key
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
NEW.fingerprint,
|
||||
NEW.app_version,
|
||||
NEW.day,
|
||||
NEW.day,
|
||||
NEW.occurrence_count,
|
||||
1,
|
||||
NEW.error_type,
|
||||
NEW.message,
|
||||
NEW.object_key
|
||||
)
|
||||
ON CONFLICT (fingerprint, app_version) DO UPDATE
|
||||
SET
|
||||
last_seen_day = excluded.last_seen_day,
|
||||
occurrence_count = occurrence_count + excluded.occurrence_count,
|
||||
installation_count = installation_count + CASE
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM error_reports AS previous
|
||||
WHERE
|
||||
previous.fingerprint = NEW.fingerprint
|
||||
AND previous.app_version = NEW.app_version
|
||||
AND previous.installation_hash = NEW.installation_hash
|
||||
AND previous.event_id != NEW.event_id
|
||||
) THEN 1
|
||||
ELSE 0
|
||||
END,
|
||||
latest_error_type = excluded.latest_error_type,
|
||||
latest_message = excluded.latest_message,
|
||||
sample_object_key = COALESCE(error_groups.sample_object_key, excluded.sample_object_key);
|
||||
|
||||
INSERT INTO error_daily
|
||||
(day, fingerprint, app_version, occurrence_count, installation_count)
|
||||
VALUES (NEW.day, NEW.fingerprint, NEW.app_version, NEW.occurrence_count, 1)
|
||||
ON CONFLICT (day, fingerprint, app_version) DO UPDATE
|
||||
SET
|
||||
occurrence_count = occurrence_count + excluded.occurrence_count,
|
||||
installation_count = installation_count + CASE
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM error_reports AS previous
|
||||
WHERE
|
||||
previous.day = NEW.day
|
||||
AND previous.fingerprint = NEW.fingerprint
|
||||
AND previous.app_version = NEW.app_version
|
||||
AND previous.installation_hash = NEW.installation_hash
|
||||
AND previous.event_id != NEW.event_id
|
||||
) THEN 1
|
||||
ELSE 0
|
||||
END;
|
||||
|
||||
INSERT INTO daily_totals (day, error_occurrences, distinct_error_groups)
|
||||
VALUES
|
||||
(
|
||||
NEW.day,
|
||||
NEW.occurrence_count,
|
||||
CASE
|
||||
WHEN NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM error_reports AS previous
|
||||
WHERE
|
||||
previous.day = NEW.day
|
||||
AND previous.fingerprint = NEW.fingerprint
|
||||
AND previous.app_version = NEW.app_version
|
||||
AND previous.event_id != NEW.event_id
|
||||
) THEN 1
|
||||
ELSE 0
|
||||
END
|
||||
)
|
||||
ON CONFLICT (day) DO UPDATE
|
||||
SET
|
||||
error_occurrences = error_occurrences + excluded.error_occurrences,
|
||||
distinct_error_groups = distinct_error_groups + excluded.distinct_error_groups;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER error_context_reservation_budget
|
||||
AFTER INSERT ON error_context_reservations
|
||||
BEGIN
|
||||
INSERT INTO error_context_budget (day, object_count)
|
||||
VALUES (NEW.day, 1)
|
||||
ON CONFLICT (day) DO UPDATE SET object_count = object_count + 1;
|
||||
|
||||
INSERT INTO error_context_samples (day, fingerprint, app_version, sample_count)
|
||||
VALUES (NEW.day, NEW.fingerprint, NEW.app_version, 1)
|
||||
ON CONFLICT (day, fingerprint, app_version) DO UPDATE
|
||||
SET sample_count = sample_count + 1;
|
||||
END;
|
||||
155
apps/telemetry-worker/migrations/0002_optimize_usage.sql
Normal file
155
apps/telemetry-worker/migrations/0002_optimize_usage.sql
Normal file
@ -0,0 +1,155 @@
|
||||
-- 0002: eliminate per-event aggregation write/read amplification and add missing indexes.
|
||||
--
|
||||
-- The previous design ran three aggregate UPSERTs (error_groups / error_daily /
|
||||
-- daily_totals) inside an AFTER INSERT trigger for every error report, each with
|
||||
-- a NOT EXISTS subquery that scanned the whole fingerprint history. That single
|
||||
-- INSERT statement accounted for ~8.2M rows read and ~147k rows written per day,
|
||||
-- far beyond the Workers Free daily limits (5M rows read / 100k rows written).
|
||||
--
|
||||
-- This migration:
|
||||
-- 1. Drops the per-event aggregation triggers; counts now flow through a
|
||||
-- realtime error_daily upsert in the worker and nightly cron rollups.
|
||||
-- 2. Rebuilds error_daily and accepted_batches as WITHOUT ROWID so every
|
||||
-- upsert/insert costs one written row instead of two.
|
||||
-- 3. Adds seen/dimension tables for O(1) distinct-install tracking.
|
||||
-- 4. Swaps the error_reports index so filter queries become point lookups.
|
||||
-- 5. Backfills the new tables from existing data.
|
||||
|
||||
DROP TRIGGER error_report_aggregates;
|
||||
DROP TRIGGER installations_daily_total;
|
||||
DROP TRIGGER daily_active_total;
|
||||
|
||||
ALTER TABLE error_daily RENAME TO error_daily_legacy;
|
||||
CREATE TABLE error_daily (
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
installation_count INTEGER NOT NULL,
|
||||
PRIMARY KEY (day, fingerprint, app_version)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
INSERT INTO error_daily (day, fingerprint, app_version, occurrence_count, installation_count)
|
||||
SELECT day, fingerprint, app_version, occurrence_count, installation_count
|
||||
FROM error_daily_legacy;
|
||||
DROP TABLE error_daily_legacy;
|
||||
|
||||
ALTER TABLE accepted_batches RENAME TO accepted_batches_legacy;
|
||||
CREATE TABLE accepted_batches (
|
||||
batch_id TEXT PRIMARY KEY,
|
||||
installation_hash TEXT NOT NULL,
|
||||
accepted_at INTEGER NOT NULL
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
INSERT INTO accepted_batches (batch_id, installation_hash, accepted_at)
|
||||
SELECT batch_id, installation_hash, accepted_at FROM accepted_batches_legacy;
|
||||
DROP TABLE accepted_batches_legacy;
|
||||
|
||||
CREATE TABLE error_daily_installations (
|
||||
day TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
installation_hash TEXT NOT NULL,
|
||||
PRIMARY KEY (day, fingerprint, app_version, installation_hash)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE error_group_installations (
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
installation_hash TEXT NOT NULL,
|
||||
first_seen_day TEXT NOT NULL,
|
||||
PRIMARY KEY (fingerprint, app_version, installation_hash)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE wau_seen (
|
||||
installation_hash TEXT PRIMARY KEY
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE mau_seen (
|
||||
installation_hash TEXT PRIMARY KEY
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE daily_active_dims (
|
||||
dimension TEXT NOT NULL,
|
||||
day TEXT NOT NULL,
|
||||
label TEXT NOT NULL,
|
||||
install_count INTEGER NOT NULL,
|
||||
PRIMARY KEY (dimension, day, label)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE platforms (
|
||||
platform TEXT PRIMARY KEY
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE error_range_stats (
|
||||
range_days INTEGER NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
app_version TEXT NOT NULL,
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
occurrence_count INTEGER NOT NULL,
|
||||
installation_count INTEGER NOT NULL,
|
||||
latest_error_type TEXT NOT NULL,
|
||||
latest_message TEXT NOT NULL,
|
||||
PRIMARY KEY (range_days, fingerprint, app_version)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
DROP INDEX error_reports_fingerprint;
|
||||
CREATE INDEX error_reports_group_day ON error_reports (fingerprint, app_version, day);
|
||||
CREATE INDEX installations_first_seen_day ON installations (first_seen_day);
|
||||
CREATE INDEX error_context_reservations_day ON error_context_reservations (day);
|
||||
CREATE INDEX error_context_reservations_fingerprint_created ON error_context_reservations (
|
||||
fingerprint,
|
||||
created_at
|
||||
);
|
||||
CREATE INDEX error_context_reservations_created_at ON error_context_reservations (created_at);
|
||||
CREATE INDEX error_groups_last_seen_day ON error_groups (last_seen_day);
|
||||
CREATE INDEX accepted_batches_accepted_at ON accepted_batches (accepted_at);
|
||||
CREATE INDEX daily_active_installation ON daily_active (installation_hash, day);
|
||||
|
||||
CREATE TRIGGER daily_active_rollups
|
||||
AFTER INSERT ON daily_active
|
||||
BEGIN
|
||||
INSERT OR IGNORE INTO wau_seen (installation_hash) VALUES (NEW.installation_hash);
|
||||
INSERT OR IGNORE INTO mau_seen (installation_hash) VALUES (NEW.installation_hash);
|
||||
INSERT INTO daily_active_dims (dimension, day, label, install_count)
|
||||
VALUES ('version', NEW.day, NEW.app_version, 1)
|
||||
ON CONFLICT (dimension, day, label) DO UPDATE SET install_count = install_count + 1;
|
||||
INSERT INTO daily_active_dims (dimension, day, label, install_count)
|
||||
VALUES ('platform', NEW.day, NEW.platform, 1)
|
||||
ON CONFLICT (dimension, day, label) DO UPDATE SET install_count = install_count + 1;
|
||||
INSERT INTO daily_active_dims (dimension, day, label, install_count)
|
||||
VALUES ('arch', NEW.day, NEW.arch, 1)
|
||||
ON CONFLICT (dimension, day, label) DO UPDATE SET install_count = install_count + 1;
|
||||
END;
|
||||
|
||||
INSERT OR IGNORE INTO wau_seen (installation_hash)
|
||||
SELECT DISTINCT installation_hash FROM daily_active WHERE day >= date('now', '-6 days');
|
||||
INSERT OR IGNORE INTO mau_seen (installation_hash)
|
||||
SELECT DISTINCT installation_hash FROM daily_active WHERE day >= date('now', '-29 days');
|
||||
INSERT OR IGNORE INTO platforms (platform)
|
||||
SELECT DISTINCT platform FROM error_reports WHERE day >= date('now', '-30 days');
|
||||
INSERT OR IGNORE INTO error_group_installations
|
||||
(fingerprint, app_version, installation_hash, first_seen_day)
|
||||
SELECT fingerprint, app_version, installation_hash, MIN(day)
|
||||
FROM error_reports
|
||||
GROUP BY fingerprint, app_version, installation_hash;
|
||||
INSERT OR IGNORE INTO daily_active_dims (dimension, day, label, install_count)
|
||||
SELECT 'version', day, app_version, COUNT(DISTINCT installation_hash)
|
||||
FROM daily_active
|
||||
GROUP BY day, app_version;
|
||||
INSERT OR IGNORE INTO daily_active_dims (dimension, day, label, install_count)
|
||||
SELECT 'platform', day, platform, COUNT(DISTINCT installation_hash)
|
||||
FROM daily_active
|
||||
GROUP BY day, platform;
|
||||
INSERT OR IGNORE INTO daily_active_dims (dimension, day, label, install_count)
|
||||
SELECT 'arch', day, arch, COUNT(DISTINCT installation_hash)
|
||||
FROM daily_active
|
||||
GROUP BY day, arch;
|
||||
@ -0,0 +1,44 @@
|
||||
-- 0003: precompute display fields on error_daily so dashboard list queries stay pure aggregations.
|
||||
--
|
||||
-- The errors list live branch previously resolved per-group correlated subqueries
|
||||
-- (distinct-install count, latest type/message, sample existence) on every page
|
||||
-- load, costing ~145k rows read per execution. These are now maintained by the
|
||||
-- worker on every upsert and only read back, cutting a list execution to ~15k rows.
|
||||
|
||||
ALTER TABLE error_daily ADD COLUMN latest_error_type TEXT NOT NULL DEFAULT 'Unknown';
|
||||
ALTER TABLE error_daily ADD COLUMN latest_message TEXT NOT NULL DEFAULT '';
|
||||
ALTER TABLE error_daily ADD COLUMN has_sample INTEGER NOT NULL DEFAULT 0;
|
||||
ALTER TABLE error_range_stats ADD COLUMN has_sample INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
-- Backfill exact per-day distinct installs for reports ingested before the
|
||||
-- error_daily_installations table existed.
|
||||
INSERT OR IGNORE INTO error_daily_installations (day, fingerprint, app_version, installation_hash)
|
||||
SELECT day, fingerprint, app_version, installation_hash
|
||||
FROM error_reports
|
||||
WHERE day >= date('now', '-1 days');
|
||||
|
||||
-- Backfill display fields for existing rows from the best available sources.
|
||||
UPDATE error_daily
|
||||
SET
|
||||
latest_error_type = (
|
||||
SELECT er.error_type
|
||||
FROM error_reports AS er
|
||||
WHERE er.fingerprint = error_daily.fingerprint AND er.app_version = error_daily.app_version
|
||||
ORDER BY er.occurred_at DESC, er.event_id DESC
|
||||
LIMIT 1
|
||||
),
|
||||
latest_message = (
|
||||
SELECT er.message
|
||||
FROM error_reports AS er
|
||||
WHERE er.fingerprint = error_daily.fingerprint AND er.app_version = error_daily.app_version
|
||||
ORDER BY er.occurred_at DESC, er.event_id DESC
|
||||
LIMIT 1
|
||||
),
|
||||
has_sample = CASE
|
||||
WHEN EXISTS (
|
||||
SELECT 1
|
||||
FROM error_context_reservations AS r
|
||||
WHERE r.fingerprint = error_daily.fingerprint AND r.app_version = error_daily.app_version
|
||||
) THEN 1
|
||||
ELSE 0
|
||||
END;
|
||||
23
apps/telemetry-worker/migrations/0004_ingestion_limits.sql
Normal file
23
apps/telemetry-worker/migrations/0004_ingestion_limits.sql
Normal file
@ -0,0 +1,23 @@
|
||||
CREATE TABLE ingestion_daily (
|
||||
day TEXT NOT NULL,
|
||||
installation_hash TEXT NOT NULL,
|
||||
accepted_batches INTEGER NOT NULL DEFAULT 0,
|
||||
PRIMARY KEY (day, installation_hash)
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
CREATE TABLE ingestion_global_daily (
|
||||
day TEXT PRIMARY KEY,
|
||||
accepted_batches INTEGER NOT NULL DEFAULT 0
|
||||
)
|
||||
WITHOUT ROWID;
|
||||
|
||||
INSERT INTO ingestion_daily (day, installation_hash, accepted_batches)
|
||||
SELECT date(accepted_at, 'unixepoch'), installation_hash, COUNT(*)
|
||||
FROM accepted_batches
|
||||
GROUP BY date(accepted_at, 'unixepoch'), installation_hash;
|
||||
|
||||
INSERT INTO ingestion_global_daily (day, accepted_batches)
|
||||
SELECT date(accepted_at, 'unixepoch'), COUNT(*)
|
||||
FROM accepted_batches
|
||||
GROUP BY date(accepted_at, 'unixepoch');
|
||||
@ -0,0 +1,24 @@
|
||||
-- 0005: stop retaining legacy error telemetry.
|
||||
--
|
||||
-- Error events are no longer accepted by the worker. Keep the historical
|
||||
-- tables in place so older database snapshots and dashboard migrations remain
|
||||
-- readable, but remove all previously collected error payloads and reset the
|
||||
-- error counters exposed through daily_totals.
|
||||
|
||||
DELETE FROM error_reports;
|
||||
DELETE FROM error_daily_installations;
|
||||
DELETE FROM error_group_installations;
|
||||
DELETE FROM error_daily;
|
||||
DELETE FROM error_range_stats;
|
||||
DELETE FROM error_groups;
|
||||
DELETE FROM error_context_reservations;
|
||||
DELETE FROM error_context_samples;
|
||||
DELETE FROM error_context_budget;
|
||||
-- Platforms are repopulated from heartbeat batches and may previously have
|
||||
-- been introduced only by an error report.
|
||||
DELETE FROM platforms;
|
||||
|
||||
UPDATE daily_totals
|
||||
SET
|
||||
error_occurrences = 0,
|
||||
distinct_error_groups = 0;
|
||||
Reference in New Issue
Block a user