Skip to content

CAN events and recording manifests

Reviewed manifests retain explicit roles, hashes, clock units and provenance. The event data module preserves every row and batches independent chronological streams. See event architecture.

recordings

Explicit, content-checked recording splits; usable without torch.

audit_catalog

audit_catalog(raw_root: str | Path, catalog: dict) -> dict

Inventory all requested datasets without assigning trusted split roles.

Source code in graphids/data/recordings.py
def audit_catalog(raw_root: str | Path, catalog: dict) -> dict:
    """Inventory all requested datasets without assigning trusted split roles."""
    result = {"version": 1, "datasets": {}, "duplicates": [], "fit_ready": False}
    hashes: dict[str, str] = {}
    for dataset in catalog:
        root = Path(raw_root) / dataset
        files = []
        errors = []
        for path in sorted(root.rglob("*.csv")):
            digest = file_hash(path)
            key = f"{dataset}/{path.relative_to(root)}"
            if digest in hashes:
                result["duplicates"].append([hashes[digest], key])
            else:
                hashes[digest] = key
            with path.open(newline="") as handle:
                reader = csv.DictReader(handle)
                fields = reader.fieldnames or []
                count, bad_clock, bad_payload, positives = 0, 0, 0, 0
                first, last = None, None
                for raw in reader:
                    count += 1
                    try:
                        t = float(raw["timestamp"])
                        if not math.isfinite(t) or (last is not None and t < last):
                            bad_clock += 1
                        first = t if first is None else first
                        last = t
                    except (KeyError, ValueError):
                        bad_clock += 1
                    try:
                        payload = raw.get("data_field", raw.get("payload", ""))
                        parsed = bytes.fromhex(payload)
                        if (
                            len(payload) % 2
                            or len(parsed) > 8
                            or len(parsed) * 2 != len(payload)
                        ):
                            bad_payload += 1
                    except ValueError:
                        bad_payload += 1
                    positives += raw.get("attack") == "1"
            files.append(
                {
                    "path": key,
                    "sha256": digest,
                    "columns": fields,
                    "rows": count,
                    "first_timestamp": first,
                    "last_timestamp": last,
                    "bad_clock": bad_clock,
                    "bad_payload": bad_payload,
                    "positive_rows": positives,
                }
            )
        if not files:
            errors.append("raw files unavailable")
        errors.append(
            "review vehicle/session independence, clock units, payload validity and label provenance before assigning roles"
        )
        result["datasets"][dataset] = {"files": files, "blocked_reasons": errors}
    return result

iter_rows

iter_rows(rec: Recording)

Yield every row in physical order; validate the entire recording on exhaustion.

Source code in graphids/data/recordings.py
def iter_rows(rec: Recording):
    """Yield every row in physical order; validate the entire recording on exhaustion."""
    count, first = 0, None
    last = -math.inf
    factor = {"seconds": 1.0, "milliseconds": 0.001, "microseconds": 0.000001}[
        rec.timestamp_unit
    ]
    with Path(rec.path).open(newline="") as handle:
        reader = csv.DictReader(handle)
        for line, raw in enumerate(reader, 2):
            try:
                timestamp = float(raw["timestamp"]) * factor
                arb_id = raw.get("arbitration_id", raw.get("arb_id"))
                payload = raw.get("data_field", raw.get("payload"))
                if not arb_id or payload is None:
                    raise ValueError("missing ID or payload")
                payload = payload.strip()
                if len(payload) % 2 or len(payload) > 16:
                    raise ValueError("invalid classic-CAN hex payload length")
                data = bytes.fromhex(payload)
                if len(data) * 2 != len(payload):
                    raise ValueError("whitespace inside payload")
                if not math.isfinite(timestamp) or timestamp < last:
                    raise ValueError("non-finite or decreasing clock")
                label = None
                if rec.labels_verified and raw.get("attack") not in (None, ""):
                    value = int(raw["attack"])
                    if value not in (0, 1):
                        raise ValueError("attack label must be binary")
                    label = value
                if rec.verified_normal:
                    if label == 1:
                        raise ValueError("positive label in verified-normal recording")
                    label = 0
                count += 1
                if first is None:
                    first = timestamp
                yield {
                    "time": timestamp,
                    "id": arb_id.strip(),
                    "data": data,
                    "label": label,
                }
                last = timestamp
            except (KeyError, ValueError) as exc:
                raise ValueError(f"{rec.key}:{line}: {exc}") from exc
    if count < 2 or last <= first:
        raise ValueError(f"{rec.key}: insufficient time exposure")

read_rows

read_rows(rec: Recording) -> list[dict]

Read one ordered recording; never infer normality or labels from names.

Source code in graphids/data/recordings.py
def read_rows(rec: Recording) -> list[dict]:
    """Read one ordered recording; never infer normality or labels from names."""
    return list(iter_rows(rec))

events

Lossless frame caches and chronological source-only event chunks.

Caches preserve byte values, lengths, numeric CAN IDs, FP64 recording clocks, row order (including ties), and unknown labels. Chunks never merge or drop rows.

CANEventDataModule

CANEventDataModule(*, manifest_path: str | Path, datasets: list[str], cache_root: str | Path, representation_cfg: CANEventRepresentationConfig | None = None, chunk_events: int = defaults.CHUNK_EVENTS, stream_batch_size: int = defaults.STREAM_BATCH_SIZE, effective_batch_events: int = defaults.EFFECTIVE_BATCH_EVENTS, max_microbatch_events: int = defaults.MAX_MICROBATCH_EVENTS, num_workers: int = defaults.NUM_WORKERS, pin_memory: bool = defaults.PIN_MEMORY, prefetch_factor: int = defaults.PREFETCH_FACTOR, persistent_workers: bool = defaults.PERSISTENT_WORKERS, cuda_prefetch: bool = defaults.CUDA_PREFETCH)

Bases: LightningDataModule

Explicit source-only learning path with optional causal observation context.

Continuous chronological training and evaluation on one device. Each optimizer window declares independent objective target totals.

Source code in graphids/data/events.py
def __init__(
    self,
    *,
    manifest_path: str | Path,
    datasets: list[str],
    cache_root: str | Path,
    representation_cfg: CANEventRepresentationConfig | None = None,
    chunk_events: int = defaults.CHUNK_EVENTS,
    stream_batch_size: int = defaults.STREAM_BATCH_SIZE,
    effective_batch_events: int = defaults.EFFECTIVE_BATCH_EVENTS,
    max_microbatch_events: int = defaults.MAX_MICROBATCH_EVENTS,
    num_workers: int = defaults.NUM_WORKERS,
    pin_memory: bool = defaults.PIN_MEMORY,
    prefetch_factor: int = defaults.PREFETCH_FACTOR,
    persistent_workers: bool = defaults.PERSISTENT_WORKERS,
    cuda_prefetch: bool = defaults.CUDA_PREFETCH,
):
    super().__init__()
    cfg = CANEventDataConfig(
        manifest_path=str(manifest_path),
        datasets=datasets,
        cache_root=str(cache_root),
        representation_cfg=representation_cfg or CANEventRepresentationConfig(),
        chunk_events=chunk_events,
        stream_batch_size=stream_batch_size,
        effective_batch_events=effective_batch_events,
        max_microbatch_events=max_microbatch_events,
        num_workers=num_workers,
        pin_memory=pin_memory,
        prefetch_factor=prefetch_factor,
        persistent_workers=persistent_workers,
        cuda_prefetch=cuda_prefetch,
    )
    self.manifest_path = cfg.manifest_path
    self.datasets = cfg.datasets
    self.cache_root = Path(cfg.cache_root)
    self.representation_cfg = cfg.representation_cfg
    self.chunk_events = cfg.chunk_events
    self.stream_batch_size = cfg.stream_batch_size
    self.effective_batch_events = cfg.effective_batch_events
    self.max_microbatch_events = cfg.max_microbatch_events
    self.num_workers = cfg.num_workers
    self.pin_memory = cfg.pin_memory
    self.prefetch_factor = cfg.prefetch_factor
    self.persistent_workers = cfg.persistent_workers
    self.cuda_prefetch = cfg.cuda_prefetch
    self.train_sampler = None  # disable Lightning's automatic padded DDP sampler
    self.event_recordings = {}

EventLocalContext dataclass

EventLocalContext(gaps: Tensor, seen: Tensor)

Bases: _TensorFields

Preceding same-ID timing; first-seen rows have seen=False.

EventStreamWindows

EventStreamWindows(recordings, chunk_events, streams=defaults.STREAM_BATCH_SIZE, effective_events=defaults.EFFECTIVE_BATCH_EVENTS)

Bases: Dataset

Schedule independent live recordings; group tails by exact length.

Replacement starts only after a slot's recording ends. No fake frames, warmup replay or padding. Metadata only is retained by the scheduler.

Source code in graphids/data/events.py
def __init__(
    self,
    recordings,
    chunk_events,
    streams=defaults.STREAM_BATCH_SIZE,
    effective_events=defaults.EFFECTIVE_BATCH_EVENTS,
):
    self.recordings, self.windows = recordings, []
    pending = iter(range(len(recordings)))
    active = [
        (i, 0)
        for i in [next(pending, None) for _ in range(streams)]
        if i is not None
    ]
    groups, events = [], 0
    while active:
        by_length, following = {}, []
        for i, start in active:
            stop = min(start + chunk_events, recordings[i].count)
            if stop <= start:
                raise ValueError("empty recording cannot form an event stream")
            by_length.setdefault(stop - start, []).append((i, start, stop))
            if stop < recordings[i].count:
                following.append((i, stop))
            else:
                replacement = next(pending, None)
                if replacement is not None:
                    following.append((replacement, 0))
        for group in by_length.values():
            length = group[0][2] - group[0][1]
            if length > effective_events:
                raise ValueError(
                    "effective window must cover one chronological chunk"
                )
            while group:
                fitting = (effective_events - events) // length
                if not fitting:
                    self.windows.append(groups)
                    groups, events = [], 0
                    continue
                part, group = group[:fitting], group[fitting:]
                groups.append(part)
                events += len(part) * length
                if events == effective_events:
                    self.windows.append(groups)
                    groups, events = [], 0
        active = following
    if groups:
        self.windows.append(groups)

EventTrainingLoader

EventTrainingLoader(*args, cuda_prefetch=False, **kwargs)

Bases: DataLoader

Stage one future optimizer window in the main process only.

Source code in graphids/data/events.py
def __init__(self, *args, cuda_prefetch=False, **kwargs):
    super().__init__(*args, **kwargs)
    self.cuda_prefetch = cuda_prefetch

EventTrainingWindow dataclass

EventTrainingWindow(groups: list[list[CANEventBatch]], classification_targets: int, payload_targets: int)

One optimizer step, with CPU target totals for exact accumulation.

materialize_events

materialize_events(rec: Recording, root: Path)

Two bounded-memory CSV passes; publish only fully written caches.

Source code in graphids/data/events.py
def materialize_events(rec: Recording, root: Path):
    """Two bounded-memory CSV passes; publish only fully written caches."""
    if file_hash(Path(rec.path)) != rec.sha256:
        raise ValueError(f"recording hash changed: {rec.key}")
    digest = recording_digest(rec)
    root.mkdir(parents=True, exist_ok=True)
    destination = root / f"event-v1-{digest}"
    with FileLock(str(destination) + ".lock"):
        if (destination / "metadata.json").exists():
            return destination
        count = sum(1 for _ in iter_rows(rec))
        staging = Path(tempfile.mkdtemp(prefix=".events-", dir=root))
        try:
            arrays = {
                name: np.lib.format.open_memmap(
                    staging / f"{name}.npy",
                    mode="w+",
                    dtype=dtype,
                    shape=(count, *tail),
                )
                for name, (dtype, tail) in COLUMNS.items()
            }
            for i, row in enumerate(iter_rows(rec)):
                try:
                    numeric_id = int(row["id"], 16)
                except ValueError as exc:
                    raise ValueError(f"{rec.key}: invalid hexadecimal CAN ID") from exc
                if not 0 <= numeric_id < 2**29:
                    raise ValueError(f"{rec.key}: CAN ID exceeds 29 bits")
                arrays["times"][i] = row["time"]
                arrays["ids"][i] = numeric_id
                arrays["payload"][i] = 0
                arrays["payload"][i, : len(row["data"])] = np.frombuffer(
                    row["data"], np.uint8
                )
                arrays["lengths"][i] = len(row["data"])
                arrays["labels"][i] = -1 if row["label"] is None else row["label"]
            # Detect concurrent source changes across either parsing pass.
            if file_hash(Path(rec.path)) != rec.sha256:
                raise ValueError(
                    f"recording hash changed during cache build: {rec.key}"
                )
            for array in arrays.values():
                array.flush()
            metadata = dict(
                schema_version=1, count=count, recording=asdict(rec), digest=digest
            )
            (staging / "metadata.json").write_text(
                json.dumps(metadata, indent=2) + "\n"
            )
            if destination.exists():
                shutil.rmtree(destination)  # only an unpublished, incomplete cache
            staging.rename(destination)
        finally:
            if staging.exists():
                shutil.rmtree(staging)
    return destination

config

Continuous event stream batching and pooled CAN data configuration.

CANEventDataConfig

Bases: BaseModel

Data settings; representation schema 2 enables causal per-ID context.

context

Causal observation indices and training-only timing statistics.

fit_timing_statistics

fit_timing_statistics(recordings, *, min_support=32)

Fit only verified-normal train-query rows, never selection/target traffic.

Deterministic evenly spaced rows cap sampling at 65,536 per recording. ID support counts refer to these sampled, observed predecessor pairs. Rows without a preceding bus/ID observation are excluded from fitting.

Source code in graphids/data/context.py
def fit_timing_statistics(recordings, *, min_support=32):
    """Fit only verified-normal train-query rows, never selection/target traffic.

    Deterministic evenly spaced rows cap sampling at 65,536 per recording.
    ID support counts refer to these sampled, observed predecessor pairs.
    Rows without a preceding bus/ID observation are excluded from fitting.
    """
    bus, gaps, ids = [], [], []
    for events in recordings:
        rec = events.recording
        if rec.role != "train_query" or not rec.verified_normal:
            continue
        times, event_ids = events.arrays["times"], events.arrays["ids"]
        rows = np.arange(1, events.count, max(1, (events.count + 65535) // 65536))
        if not len(rows):
            continue
        bus.append(times[rows] - times[rows - 1])
        previous = events.previous_rows
        if previous is None:
            previous = previous_rows(event_ids)
        rows = rows[previous[rows] >= 0]
        gaps.append(times[rows] - times[previous[rows]])
        ids.append(event_ids[rows])
    if not bus:
        raise ValueError("timing normalization requires verified-normal training pairs")
    bus = np.concatenate(bus)
    gaps = np.concatenate(gaps)
    ids = np.concatenate(ids)
    # A training cohort with no repeated IDs can still normalize bus timing.
    statistics = [
        _timing_statistics(bus),
        _timing_statistics(gaps if len(gaps) else bus),
    ]
    supported = []
    for key in np.unique(ids):
        selected = gaps[ids == key]
        if len(selected) >= min_support:
            supported.append(key)
            statistics.append(_timing_statistics(selected))
    return np.asarray(supported, dtype=np.int64), np.asarray(
        statistics, dtype=np.float64
    )

previous_rows

previous_rows(ids)

Previous physical row of each ID, independent of chunk boundaries.

Source code in graphids/data/context.py
def previous_rows(ids):
    """Previous physical row of each ID, independent of chunk boundaries."""
    order = np.argsort(ids, kind="stable")
    same = ids[order[1:]] == ids[order[:-1]]
    previous = np.full(len(ids), -1, dtype=np.int64)
    previous[order[1:][same]] = order[:-1][same]
    return previous