Skip to content

dp3.history_management.telemetry

TelemetryConfig

Bases: BaseModel

Configuration of periodic telemetry collection.

Telemetry

Telemetry(db: EntityDatabase, platform_config: PlatformConfig, registrar: CallbackRegistrar)
Source code in dp3/history_management/telemetry.py
def __init__(
    self, db: EntityDatabase, platform_config: PlatformConfig, registrar: CallbackRegistrar
) -> None:
    self.log = logging.getLogger("Telemetry")

    self.db = db
    self.model_spec = platform_config.model_spec
    self.config = TelemetryConfig.model_validate(platform_config.config.get("telemetry", {}))
    self.cache_col = self.db.get_module_cache("Telemetry")

    self.local_cache = {}
    self.local_cache_lock = threading.Lock()
    self.attribute_bson_size_lock = threading.Lock()

    # Schedule master document aggregation
    registrar.register_task_hook("on_task_start", self.note_latest_src_timestamp)
    mod = 30
    proc_i = platform_config.process_index
    n_proc = platform_config.num_processes
    spread_proc_index = proc_i * (mod // n_proc) if n_proc < mod else proc_i
    seconds = ",".join(f"{int(i)}" for i in range(60) if int(i - spread_proc_index) % mod == 0)
    registrar.scheduler_register(
        self.sync_to_db, second=seconds, minute="*", hour="*", misfire_grace_time=10
    )

    if platform_config.process_index == 0:
        registrar.scheduler_register(
            self.collect_attribute_bson_sizes,
            **self.config.attribute_bson_size_schedule.model_dump(),
        )
        configured_entities = set(self.model_spec.entities)
        try:
            cached_entities = set(
                self.cache_col.distinct(
                    "entity_type", {"telemetry_type": ATTRIBUTE_BSON_SIZES_TYPE}
                )
            )
        except Exception:
            cached_entities = set()
            self.log.exception(
                "Failed to check the attribute BSON-size cache; scheduling an initial sweep."
            )
        if not configured_entities.issubset(cached_entities):
            registrar.scheduler_register_once(
                self.collect_attribute_bson_sizes, misfire_grace_time=300
            )
    else:
        self.log.debug(
            "Attribute BSON-size collection is disabled in this worker to avoid duplicate "
            "full collection scans."
        )

note_latest_src_timestamp

note_latest_src_timestamp(task: DataPointTask)

Note the latest timestamp of each source in the task

Source code in dp3/history_management/telemetry.py
def note_latest_src_timestamp(self, task: DataPointTask):
    """Note the latest timestamp of each source in the task"""
    latest_timestamps = {}
    for dp in task.data_points:
        has_timestamp = isinstance(dp, DataPointObservationsBase | DataPointTimeseriesBase)
        if dp.src is None or not has_timestamp:
            continue
        latest_timestamp = dp.t2 or dp.t1
        latest_timestamps[dp.src] = latest_timestamp

    if not latest_timestamps:
        return

    with self.local_cache_lock:
        self.local_cache.update(latest_timestamps)

sync_to_db

sync_to_db()

Sync local timestamp cache to database.

Source code in dp3/history_management/telemetry.py
def sync_to_db(self):
    """Sync local timestamp cache to database."""
    with self.local_cache_lock:
        synced_cache = self.local_cache
        self.local_cache = {}

    updates = [
        UpdateOne(
            {"_id": src},
            [{"$set": {"src_t": {"$max": ["$src_t", latest_timestamp]}}}],
        )
        for src, latest_timestamp in synced_cache.items()
    ]

    if not updates:
        return

    try:
        start = time.time()
        res = self.cache_col.bulk_write(updates, ordered=False)
        end = time.time()
        self.log.debug(
            "Updating %s src_timestamp records: %s matched %s modified in %.4fs",
            len(updates),
            res.matched_count,
            res.modified_count,
            (end - start),
        )
        if len(updates) != res.matched_count:
            upserts = [
                UpdateOne(
                    {"_id": src},
                    [{"$set": {"_id": src, "src_t": {"$max": ["$src_t", latest_timestamp]}}}],
                    upsert=True,
                )
                for src, latest_timestamp in synced_cache.items()
            ]
            start = time.time()
            res = self.cache_col.bulk_write(upserts, ordered=False)
            end = time.time()
            self.log.debug(
                "Upserting %s src_timestamp records: %s matched %s modified in %.4fs",
                len(upserts),
                res.matched_count,
                res.modified_count,
                (end - start),
            )
    except Exception as e:
        self.log.error("Error updating src_timestamp records: %s", e)

collect_attribute_bson_sizes

collect_attribute_bson_sizes()

Calculate and atomically publish attribute size statistics per entity type.

Source code in dp3/history_management/telemetry.py
def collect_attribute_bson_sizes(self):
    """Calculate and atomically publish attribute size statistics per entity type."""
    with self.attribute_bson_size_lock:
        for entity_type in self.model_spec.entities:
            start = time.monotonic()
            try:
                attributes = self.db.get_attribute_bson_size_stats(entity_type)
                duration = time.monotonic() - start
                calculated_at = datetime.now(UTC)
                record = {
                    "_id": {
                        "telemetry_type": ATTRIBUTE_BSON_SIZES_TYPE,
                        "entity_type": entity_type,
                    },
                    "telemetry_type": ATTRIBUTE_BSON_SIZES_TYPE,
                    "entity_type": entity_type,
                    "calculated_at": calculated_at,
                    "duration_s": duration,
                    "attributes": attributes,
                }
                self.cache_col.replace_one({"_id": record["_id"]}, record, upsert=True)
                self.log.info(
                    "Calculated attribute BSON sizes for %s in %.3fs", entity_type, duration
                )
            except Exception:
                self.log.exception(
                    "Failed to calculate attribute BSON sizes for %s; retaining the previous "
                    "cached result.",
                    entity_type,
                )

        self.cache_col.delete_many(
            {
                "telemetry_type": ATTRIBUTE_BSON_SIZES_TYPE,
                "entity_type": {"$nin": list(self.model_spec.entities)},
            }
        )

TelemetryReader

TelemetryReader(db: EntityDatabase, app_name: str, num_processes: int, rabbit_config: dict)

Reader of telemetry data.

Used by API. Not contained inside Telemetry class due to usage of CallbackRegistrar and all of its requirements.

Source code in dp3/history_management/telemetry.py
def __init__(
    self,
    db: EntityDatabase,
    app_name: str,
    num_processes: int,
    rabbit_config: dict,
) -> None:
    self.db = db
    self.app_name = app_name
    self.num_processes = num_processes
    self.rabbit_config = rabbit_config or {}
    self.cache_col = self.db.get_module_cache("Telemetry")

get_sources_validity

get_sources_validity() -> dict[str, datetime]

Return timestamps (datetimes) of current validity of all sources.

Source code in dp3/history_management/telemetry.py
def get_sources_validity(self) -> dict[str, datetime]:
    """Return timestamps (datetimes) of current validity of all sources."""
    src_data = self.cache_col.find({"src_t": {"$exists": True}}).sort([("_id", ASCENDING)])
    return {src["_id"]: src["src_t"] for src in src_data}

get_sources_age

get_sources_age(unit: str = 'minutes') -> dict[str, int]

Return ages of sources relative to now in requested units.

Source code in dp3/history_management/telemetry.py
def get_sources_age(self, unit: str = "minutes") -> dict[str, int]:
    """Return ages of sources relative to now in requested units."""
    now = datetime.now(UTC)
    divider = 60 if unit == "minutes" else 1
    return {
        source: int((now - timestamp).total_seconds() / divider)
        for source, timestamp in self.get_sources_validity().items()
    }

get_entities_per_attr

get_entities_per_attr() -> dict[str, dict[str, int]]

Return counts of entities with data present for each configured attribute.

Source code in dp3/history_management/telemetry.py
def get_entities_per_attr(self) -> dict[str, dict[str, int]]:
    """Return counts of entities with data present for each configured attribute."""
    return self.db.count_entities_per_attr()

get_attribute_bson_sizes

get_attribute_bson_sizes() -> dict[str, dict]

Return the latest cached attribute BSON-size statistics.

Source code in dp3/history_management/telemetry.py
def get_attribute_bson_sizes(self) -> dict[str, dict]:
    """Return the latest cached attribute BSON-size statistics."""
    records = self.cache_col.find({"telemetry_type": ATTRIBUTE_BSON_SIZES_TYPE}).sort(
        [("entity_type", ASCENDING)]
    )
    return {
        record["entity_type"]: {
            "calculated_at": record["calculated_at"],
            "duration_s": record["duration_s"],
            "attributes": record["attributes"],
        }
        for record in records
    }

get_snapshot_summary

get_snapshot_summary() -> dict

Return summary of latest snapshot activity.

Source code in dp3/history_management/telemetry.py
def get_snapshot_summary(self) -> dict:
    """Return summary of latest snapshot activity."""
    now = datetime.now(UTC)
    latest_started = next(self.db.find_metadata(module="SnapShooter").limit(1), None)
    latest_finished = next(
        self.db.find_metadata(
            module="SnapShooter",
            extra_filter={"workers_finished": self.num_processes, "linked_finished": True},
        ).limit(1),
        None,
    )

    summary = {
        "latest_age": None,
        "finished_age": None,
        "entities": None,
        "total_s": None,
    }
    if latest_started is not None:
        summary["latest_age"] = (now - latest_started["#time_created"]).total_seconds()
    if latest_finished is not None:
        summary["finished_age"] = (now - latest_finished["#time_created"]).total_seconds()
        summary["entities"] = latest_finished.get("entities")
        summary["total_s"] = (
            latest_finished["#last_update"] - latest_finished["#time_created"]
        ).total_seconds()
    return summary

get_metadata

get_metadata(module: str = None, date_from: datetime = None, date_to: datetime = None, newest_first: bool = True, skip: int = 0, limit: int = 0) -> list[dict]

Return filtered metadata documents.

Source code in dp3/history_management/telemetry.py
def get_metadata(
    self,
    module: str = None,
    date_from: datetime = None,
    date_to: datetime = None,
    newest_first: bool = True,
    skip: int = 0,
    limit: int = 0,
) -> list[dict]:
    """Return filtered metadata documents."""
    cursor = self.db.find_metadata(module, date_from, date_to, newest_first)
    if skip:
        cursor = cursor.skip(skip)
    if limit:
        cursor = cursor.limit(limit)
    return list(cursor)

get_rabbitmq_queues

get_rabbitmq_queues() -> dict[str, list[dict]]

Return RabbitMQ queue telemetry for this application.

Source code in dp3/history_management/telemetry.py
def get_rabbitmq_queues(self) -> dict[str, list[dict]]:
    """Return RabbitMQ queue telemetry for this application."""
    host = self.rabbit_config.get("host", "localhost")
    port = int(self.rabbit_config.get("management_port", 15672))
    username = self.rabbit_config.get("username", "guest")
    password = self.rabbit_config.get("password", "guest")
    response = requests.get(
        f"http://{host}:{port}/api/queues",
        auth=(username, password),
        timeout=5,
    )
    response.raise_for_status()

    queues = []
    app_prefix = f"{self.app_name}-worker-"
    for queue in response.json():
        queue_name = queue.get("name", "")
        if not queue_name.startswith(app_prefix):
            continue

        short_name = queue_name[len(app_prefix) :]
        alias = short_name if len(short_name) > 2 else f"{short_name}-main"
        queue_data = {"name": queue_name, "queue": alias}
        for key, alias_key in self.exported_queue_keys.items():
            queue_data[alias_key] = queue.get(key, 0)

        message_stats = queue.get("message_stats", {})
        for key, alias_key in self.exported_message_stats_keys.items():
            queue_data[alias_key] = message_stats.get(key, {}).get("rate", 0)
        queues.append(queue_data)

    queues.sort(key=lambda item: item["queue"])
    return {"queues": queues}