"""Content-Addressed Storage (CAS) for blobs and IPFS CIDv1 interoperability.
Spec reference: spec.html section "Content Addressing & CIDs" (lines 3470-3476) and "Blobs" (lines 3768-3776).
- Blossom blobs are SHA-256 addressed.
- Maps directly to IPFS Content Identifiers (CIDs):
multibase 'b' (RFC 4648 Base32 lowercase), CIDv1 (0x01), raw binary codec (0x55), multihash sha2-256 (0x12, 0x20).
- Standard CAS on-disk sharded directory layout: `blobs/data/<sha256[:2]>/<sha256>`
(fanout depth 1). The depth is configurable (0..8 two-hex-char levels) so an
operator may point the store at a tree another CAS already lays out the same
way (magpie: `<data>/objects/<ab>/<cd>/<sha>`, depth 2) - see docs/sidecars.md.
The depth a directory was created with is recorded in `blobs.sqlite` and a
store opened with a different depth refuses rather than silently 404 every blob.
- SQLite metadata index in `blobs.sqlite` tracking size, content-type, uploader attribution, and upload timestamps.
"""
from __future__ import annotations
import hashlib
import os
import re
import tempfile
import time
from typing import Optional, TypedDict
from fleetmesh_core.identity.cid import cid_to_sha256, sha256_to_cid
from fleetmesh_node.compartment.schema import Column, Index, Migration, Table
from fleetmesh_node.storage import SqliteStore
HEX64_RE = re.compile(r'^[0-9a-f]{64}$')
__all__ = ['BlobDescriptor', 'BlobMetadataStore', 'BlobStore', 'cid_to_sha256', 'sha256_to_cid']
[docs]class BlobDescriptor(TypedDict, total=False):
"""Metadata descriptor for a stored or indexed blob."""
sha256: str
size: int
type: str
uploaded: int
uploaded_by: Optional[str]
cid: str
pinned: bool
accessed_at: Optional[int]
class LayoutMismatch(RuntimeError):
"""A blob directory was created with a different fanout depth than it is being opened with."""
MAX_FANOUT = 8
DEFAULT_FANOUT = 1
[docs]class BlobStore:
"""Content-Addressed Storage engine for SHA-256 blobs and artifacts."""
[docs] def __init__(
self,
directory: str,
max_bytes: Optional[int] = None,
max_storage_bytes: Optional[int] = None,
fanout: int = DEFAULT_FANOUT,
data_dir: Optional[str] = None,
) -> None:
"""``directory`` holds ``blobs.sqlite``; blobs live under ``data_dir``
(default ``<directory>/data``) at ``fanout`` two-hex-char levels.
``fanout``/``data_dir`` exist so a node may share a tree with another
CAS that uses the same sha256 layout. Shared disk is not a shared
catalogue - each index knows only what it wrote - and the other writer's
file modes/ownership must let this process read and unlink.
"""
if not isinstance(fanout, int) or isinstance(fanout, bool) or not 0 <= fanout <= MAX_FANOUT:
raise ValueError(f'blob fanout must be an integer 0..{MAX_FANOUT}, got {fanout!r}')
self.dir: str = os.path.realpath(directory)
self.data_dir: str = os.path.realpath(data_dir) if data_dir else os.path.join(self.dir, 'data')
self.db_path: str = os.path.join(self.dir, 'blobs.sqlite')
self.max_bytes: Optional[int] = max_bytes
self.max_storage_bytes: Optional[int] = max_storage_bytes
self.fanout: int = fanout
os.makedirs(self.dir, exist_ok=True)
os.makedirs(self.data_dir, exist_ok=True)
self.db: BlobMetadataStore = BlobMetadataStore(self.db_path)
recorded = self.db.meta_set_if_absent('fanout', str(fanout))
if recorded != str(fanout):
self.db.close()
raise LayoutMismatch(
f'{self.db.safe_path} was created with fanout {recorded}; opening with {fanout} '
f'would hide every stored blob. Keep the recorded depth or point at a new directory.')
recorded_dir = self.db.meta_set_if_absent('data_dir', self.data_dir)
if recorded_dir != self.data_dir:
self.db.close()
raise LayoutMismatch(
f'{self.db.safe_path} indexes blobs under {recorded_dir}, not {self.data_dir}. '
f'Keep the recorded data_dir or point at a new directory.')
def _blob_path(self, sha256_hex: str) -> str:
h = sha256_hex.lower()
levels = [h[2 * i:2 * i + 2] for i in range(self.fanout)]
return os.path.join(self.data_dir, *levels, h)
[docs] def put(
self,
data: bytes,
content_type: str = 'application/octet-stream',
uploaded_by: Optional[str] = None,
now: Optional[int] = None,
) -> BlobDescriptor:
"""Store a blob by content hash. Returns normative descriptor."""
if not isinstance(data, (bytes, bytearray)):
raise TypeError("blob data must be bytes")
size = len(data)
if self.max_bytes is not None and size > self.max_bytes:
raise ValueError(f"blob size {size} exceeds maximum limit {self.max_bytes}")
h = hashlib.sha256(data).hexdigest().lower()
now_ts = int(now if now is not None else time.time())
# Enforce quota watermark with automatic LRU eviction
if self.max_storage_bytes is not None and not self.has(h):
cur_total = self.total_bytes()
if cur_total + size > self.max_storage_bytes:
needed = (cur_total + size) - self.max_storage_bytes
self.evict_lru(needed_bytes=needed)
if self.total_bytes() + size > self.max_storage_bytes:
raise ValueError(
f"CAS quota exceeded: cannot store {size} bytes (storage cap {self.max_storage_bytes} bytes)"
)
dest = self._blob_path(h)
os.makedirs(os.path.dirname(dest), exist_ok=True)
# Write atomically via tempfile in same filesystem directory
if not os.path.exists(dest):
with tempfile.NamedTemporaryFile(dir=os.path.dirname(dest), delete=False) as tf:
tf.write(data)
tmp_name = tf.name
os.replace(tmp_name, dest)
# Record metadata preserving pinned state if existing
self.db.execute('''
INSERT INTO blobs (sha256, size, type, uploaded_by, created_at, accessed_at, pinned)
VALUES (?, ?, ?, ?, ?, ?, COALESCE((SELECT pinned FROM blobs WHERE sha256 = ?), 0))
ON CONFLICT(sha256) DO UPDATE SET
size=excluded.size,
type=excluded.type,
accessed_at=excluded.accessed_at
''', (h, size, content_type, uploaded_by, now_ts, now_ts, h))
self.db.commit()
return {
'sha256': h,
'size': size,
'type': content_type,
'uploaded': now_ts,
'uploaded_by': uploaded_by,
'cid': sha256_to_cid(h),
}
[docs] def get(self, sha256_hex: str) -> Optional[bytes]:
"""Read full blob bytes, verifying SHA-256 integrity."""
h = sha256_hex.lower()
if not HEX64_RE.match(h):
return None
path = self._blob_path(h)
if not os.path.exists(path):
return None
try:
with open(path, 'rb') as f:
data = f.read()
# Verify integrity
if hashlib.sha256(data).hexdigest().lower() != h:
os.unlink(path)
return None
try:
self.db.execute('UPDATE blobs SET accessed_at = ? WHERE sha256 = ?', (int(time.time()), h))
self.db.commit()
except Exception:
pass
return data
except OSError:
return None
[docs] def has(self, sha256_hex: str) -> bool:
"""Check if blob exists on disk and is indexed."""
h = sha256_hex.lower()
if not HEX64_RE.match(h):
return False
return os.path.exists(self._blob_path(h))
[docs] def stat(self, sha256_hex: str) -> Optional[BlobDescriptor]:
"""Retrieve metadata descriptor without loading blob content."""
h = sha256_hex.lower()
if not HEX64_RE.match(h):
return None
row = self.db._one('''
SELECT size, type, uploaded_by, created_at, pinned, accessed_at FROM blobs WHERE sha256 = ?
''', (h,))
if not row:
if not os.path.exists(self._blob_path(h)):
return None
try:
sz = os.path.getsize(self._blob_path(h))
return {
'sha256': h,
'size': sz,
'type': 'application/octet-stream',
'uploaded': int(os.path.getmtime(self._blob_path(h))),
'uploaded_by': None,
'cid': sha256_to_cid(h),
'pinned': False,
}
except OSError:
return None
size, ctype, uploaded_by, created_at, pinned, accessed_at = row
return {
'sha256': h,
'size': size,
'type': ctype,
'uploaded': created_at,
'uploaded_by': uploaded_by,
'cid': sha256_to_cid(h),
'pinned': bool(pinned),
'accessed_at': accessed_at,
}
[docs] def delete(self, sha256_hex: str) -> bool:
"""Delete blob from disk and metadata index.
The digest is validated before it touches the filesystem: get/has
already refuse anything but 64 hex chars, and delete is the one
path here that unlinks, so '../..' must never reach _blob_path.
"""
h = sha256_hex.lower()
if not HEX64_RE.match(h):
return False
path = self._blob_path(h)
deleted = False
if os.path.exists(path):
try:
os.unlink(path)
deleted = True
except OSError:
pass
cur = self.db.execute('DELETE FROM blobs WHERE sha256 = ?', (h,))
self.db.commit()
if cur.rowcount > 0:
deleted = True
return deleted
[docs] def list_by(
self,
uploaded_by: Optional[str] = None,
limit: int = 100,
offset: int = 0,
) -> list[BlobDescriptor]:
"""List stored blobs matching uploader pubkey."""
if uploaded_by:
rows = self.db._all('''
SELECT sha256, size, type, uploaded_by, created_at
FROM blobs WHERE uploaded_by = ?
ORDER BY created_at DESC LIMIT ? OFFSET ?
''', (uploaded_by, limit, offset))
else:
rows = self.db._all('''
SELECT sha256, size, type, uploaded_by, created_at
FROM blobs ORDER BY created_at DESC LIMIT ? OFFSET ?
''', (limit, offset))
out: list[BlobDescriptor] = []
for h, sz, ctype, uploader, ts in rows:
out.append({
'sha256': h,
'size': sz,
'type': ctype,
'uploaded': ts,
'uploaded_by': uploader,
'cid': sha256_to_cid(h),
})
return out
[docs] def total_bytes(self) -> int:
"""Total size in bytes of all indexed blobs."""
row = self.db._one('SELECT SUM(size) FROM blobs')
return int((row and row[0]) or 0)
[docs] def count(self) -> int:
"""Number of indexed blobs."""
row = self.db._one('SELECT COUNT(*) FROM blobs')
return int((row and row[0]) or 0)
[docs] def pin(self, sha256_hex: str) -> bool:
"""Pin a blob to prevent LRU eviction."""
h = sha256_hex.lower()
if not HEX64_RE.match(h) or not self.has(h):
return False
cur = self.db.execute('UPDATE blobs SET pinned = 1 WHERE sha256 = ?', (h,))
self.db.commit()
return cur.rowcount > 0
[docs] def unpin(self, sha256_hex: str) -> bool:
"""Unpin a blob, allowing LRU eviction."""
h = sha256_hex.lower()
if not HEX64_RE.match(h):
return False
cur = self.db.execute('UPDATE blobs SET pinned = 0 WHERE sha256 = ?', (h,))
self.db.commit()
return cur.rowcount > 0
[docs] def is_pinned(self, sha256_hex: str) -> bool:
"""Check if blob is pinned against LRU eviction."""
h = sha256_hex.lower()
row = self.db._one('SELECT pinned FROM blobs WHERE sha256 = ?', (h,))
return bool(row and row[0])
[docs] def evict_lru(self, needed_bytes: int = 0) -> int:
"""Evict oldest unpinned blobs by accessed_at timestamp to free needed_bytes.
Returns total bytes freed.
"""
rows = self.db._all(
'SELECT sha256, size FROM blobs WHERE pinned = 0 '
'ORDER BY COALESCE(accessed_at, created_at) ASC, created_at ASC'
)
freed = 0
for h, sz in rows:
if needed_bytes > 0 and freed >= needed_bytes:
break
if self.delete(h):
freed += sz
return freed
[docs] def close(self) -> None:
"""Safely close underlying metadata database."""
self.db.close()