"""CAR (Content Addressable aRchive) v1 reader and writer.
Standard library only. Format per IPLD CAR v1 specification:
- Header: DAG-CBOR map `{"version": 1, "roots": [CIDLink, ...]}` prefixed with unsigned varint length.
- Blocks: Sequence of `varint(len(cid) + len(data)) || binary_cid || data`.
"""
from __future__ import annotations
import io
from typing import BinaryIO, Iterable, Iterator, Mapping, Optional, Sequence
from ..conformance.errors import FleetmeshError
from ..identity.cid import CIDLink
from ..wire import varint
from ..wire.cbor import CBORError, dag_cbor_decode, dag_cbor_encode
__all__ = [
'CarError',
'read_car',
'write_car',
'CarReader',
'CarWriter',
]
[docs]class CarError(FleetmeshError):
"""Base error for CAR file processing."""
def _read_varint_stream(stream: BinaryIO) -> Optional[int]:
"""The next varint, None at a clean end of stream (wire/varint, D3)."""
try:
return varint.read_stream(stream)
except ValueError as e:
raise CarError('%s in CAR stream' % e) from None
MAX_CAR_HEADER_LEN = 1024 * 1024 # 1 MB
MAX_CAR_BLOCK_LEN = 16 * 1024 * 1024 # 16 MB
[docs]class CarReader:
"""Sequential reader for CAR v1 archive streams."""
[docs] def __init__(self, stream: BinaryIO):
self.stream = stream
self.roots: list[CIDLink] = []
self.version: int = 1
self._read_header()
def _read_header(self) -> None:
header_len = _read_varint_stream(self.stream)
if header_len is None:
raise CarError("empty CAR archive")
if header_len > MAX_CAR_HEADER_LEN:
raise CarError(f"CAR header too large: {header_len} bytes (max {MAX_CAR_HEADER_LEN})")
header_data = self.stream.read(header_len)
if len(header_data) != header_len:
raise CarError(f"truncated CAR header: expected {header_len}, got {len(header_data)}")
try:
hdr = dag_cbor_decode(header_data)
except CBORError as e:
raise CarError(f"invalid DAG-CBOR header: {e}") from e
if not isinstance(hdr, dict) or hdr.get('version') != 1:
raise CarError(f"unsupported CAR version: {hdr.get('version') if isinstance(hdr, dict) else hdr}")
raw_roots = hdr.get('roots', [])
self.roots = [r if isinstance(r, CIDLink) else CIDLink.from_str(str(r)) for r in raw_roots]
self.version = hdr['version']
def __iter__(self) -> Iterator[tuple[CIDLink, bytes]]:
"""Yield each (CIDLink, block_bytes) in the CAR archive."""
while True:
block_len = _read_varint_stream(self.stream)
if block_len is None:
break
if block_len > MAX_CAR_BLOCK_LEN:
raise CarError(f"CAR block too large: {block_len} bytes (max {MAX_CAR_BLOCK_LEN})")
payload = self.stream.read(block_len)
if len(payload) != block_len:
raise CarError(f"truncated CAR block: expected {block_len}, got {len(payload)}")
# Parse CID from beginning of block
try:
# CIDLink.from_bytes requires binary CID bytes.
# To find length of CID: CIDv1 is version(1) + codec(varint) + hash_fn(varint) + hash_len(varint) + digest
cid = CIDLink.from_bytes(payload)
cid_len = len(cid.raw_bytes)
data = payload[cid_len:]
yield cid, data
except Exception as e:
raise CarError(f"corrupt block in CAR archive: {e}") from e
[docs]class CarWriter:
"""Sequential writer for CAR v1 archive streams."""
[docs] def __init__(self, stream: BinaryIO, roots: Sequence[CIDLink] | CIDLink):
self.stream = stream
self.roots = [roots] if isinstance(roots, CIDLink) else list(roots)
self._write_header()
def _write_header(self) -> None:
hdr = {'version': 1, 'roots': self.roots}
hdr_bytes = dag_cbor_encode(hdr)
self.stream.write(varint.write(len(hdr_bytes)))
self.stream.write(hdr_bytes)
[docs] def write_block(self, cid: CIDLink, data: bytes) -> None:
"""Write a single block to the CAR stream."""
cid_bytes = cid.raw_bytes
total_len = len(cid_bytes) + len(data)
self.stream.write(varint.write(total_len))
self.stream.write(cid_bytes)
self.stream.write(data)
[docs]def write_car(
roots: Sequence[CIDLink] | CIDLink,
blocks: Mapping[CIDLink, bytes] | Iterable[tuple[CIDLink, bytes]],
) -> bytes:
"""Serialize roots and blocks into a CAR v1 byte string."""
buf = io.BytesIO()
writer = CarWriter(buf, roots)
items = blocks.items() if isinstance(blocks, Mapping) else blocks
for cid, data in items:
writer.write_block(cid, data)
return buf.getvalue()
[docs]def read_car(car_bytes: bytes) -> tuple[list[CIDLink], dict[CIDLink, bytes]]:
"""Parse CAR v1 byte string into (roots, {cid: block_bytes})."""
stream = io.BytesIO(car_bytes)
reader = CarReader(stream)
blocks: dict[CIDLink, bytes] = {}
for cid, data in reader:
blocks[cid] = data
return reader.roots, blocks