Source code for fleetmesh_node.protocols.nostr.blobstore

"""Content-Addressed Storage (CAS) for blobs and IPFS CIDv1 interoperability.

Spec reference: spec.html section "Content Addressing & CIDs" (lines 3470-3476) and "Blobs" (lines 3768-3776).
- Blossom blobs are SHA-256 addressed.
- Maps directly to IPFS Content Identifiers (CIDs):
    multibase 'b' (RFC 4648 Base32 lowercase), CIDv1 (0x01), raw binary codec (0x55), multihash sha2-256 (0x12, 0x20).
- Standard CAS on-disk sharded directory layout: `blobs/data/<sha256[:2]>/<sha256>`
  (fanout depth 1). The depth is configurable (0..8 two-hex-char levels) so an
  operator may point the store at a tree another CAS already lays out the same
  way (magpie: `<data>/objects/<ab>/<cd>/<sha>`, depth 2) - see docs/sidecars.md.
  The depth a directory was created with is recorded in `blobs.sqlite` and a
  store opened with a different depth refuses rather than silently 404 every blob.
- SQLite metadata index in `blobs.sqlite` tracking size, content-type, uploader attribution, and upload timestamps.
"""
from __future__ import annotations

import hashlib
import os
import re
import tempfile
import time
from typing import Optional, TypedDict

from fleetmesh_core.identity.cid import cid_to_sha256, sha256_to_cid
from fleetmesh_node.compartment.schema import Column, Index, Migration, Table
from fleetmesh_node.storage import SqliteStore

HEX64_RE = re.compile(r'^[0-9a-f]{64}$')

__all__ = ['BlobDescriptor', 'BlobMetadataStore', 'BlobStore', 'cid_to_sha256', 'sha256_to_cid']


[docs]class BlobDescriptor(TypedDict, total=False): """Metadata descriptor for a stored or indexed blob.""" sha256: str size: int type: str uploaded: int uploaded_by: Optional[str] cid: str pinned: bool accessed_at: Optional[int]
[docs]class BlobMetadataStore(SqliteStore): """Thread-safe SQLite metadata index for Content-Addressed Storage.""" SCHEMA = ( Table('blobs', ( Column('sha256', 'TEXT', primary_key=True), Column('size', 'INTEGER', not_null=True), Column('type', 'TEXT', not_null=True), Column('uploaded_by', 'TEXT'), Column('created_at', 'INTEGER', not_null=True), Column('accessed_at', 'INTEGER'), Column('pinned', 'INTEGER', default=0), ), indices=( Index('idx_blobs_uploaded', ('uploaded_by',)), )), Table('meta', ( Column('key', 'TEXT', primary_key=True), Column('value', 'TEXT', not_null=True), )), ) # A blobs table from before the LRU existed lacks both columns; the index # over them can only be built once they are there, so it is the last step. MIGRATIONS = ( Migration(1, up='ALTER TABLE blobs ADD COLUMN accessed_at INTEGER', name='blobs_accessed_at'), Migration(2, up='ALTER TABLE blobs ADD COLUMN pinned INTEGER DEFAULT 0', name='blobs_pinned'), Migration(3, up='CREATE INDEX IF NOT EXISTS idx_blobs_lru ON blobs(pinned, accessed_at)', name='idx_blobs_lru'), )
[docs] def meta_get(self, key: str) -> Optional[str]: row = self._one('SELECT value FROM meta WHERE key = ?', (key,)) return None if row is None else str(row[0])
[docs] def meta_set_if_absent(self, key: str, value: str) -> str: """Record ``value`` under ``key`` unless one is already there; return the stored one.""" self.execute('INSERT OR IGNORE INTO meta(key, value) VALUES (?, ?)', (key, value)) self.commit() got = self.meta_get(key) return value if got is None else got
class LayoutMismatch(RuntimeError): """A blob directory was created with a different fanout depth than it is being opened with.""" MAX_FANOUT = 8 DEFAULT_FANOUT = 1
[docs]class BlobStore: """Content-Addressed Storage engine for SHA-256 blobs and artifacts."""
[docs] def __init__( self, directory: str, max_bytes: Optional[int] = None, max_storage_bytes: Optional[int] = None, fanout: int = DEFAULT_FANOUT, data_dir: Optional[str] = None, ) -> None: """``directory`` holds ``blobs.sqlite``; blobs live under ``data_dir`` (default ``<directory>/data``) at ``fanout`` two-hex-char levels. ``fanout``/``data_dir`` exist so a node may share a tree with another CAS that uses the same sha256 layout. Shared disk is not a shared catalogue - each index knows only what it wrote - and the other writer's file modes/ownership must let this process read and unlink. """ if not isinstance(fanout, int) or isinstance(fanout, bool) or not 0 <= fanout <= MAX_FANOUT: raise ValueError(f'blob fanout must be an integer 0..{MAX_FANOUT}, got {fanout!r}') self.dir: str = os.path.realpath(directory) self.data_dir: str = os.path.realpath(data_dir) if data_dir else os.path.join(self.dir, 'data') self.db_path: str = os.path.join(self.dir, 'blobs.sqlite') self.max_bytes: Optional[int] = max_bytes self.max_storage_bytes: Optional[int] = max_storage_bytes self.fanout: int = fanout os.makedirs(self.dir, exist_ok=True) os.makedirs(self.data_dir, exist_ok=True) self.db: BlobMetadataStore = BlobMetadataStore(self.db_path) recorded = self.db.meta_set_if_absent('fanout', str(fanout)) if recorded != str(fanout): self.db.close() raise LayoutMismatch( f'{self.db.safe_path} was created with fanout {recorded}; opening with {fanout} ' f'would hide every stored blob. Keep the recorded depth or point at a new directory.') recorded_dir = self.db.meta_set_if_absent('data_dir', self.data_dir) if recorded_dir != self.data_dir: self.db.close() raise LayoutMismatch( f'{self.db.safe_path} indexes blobs under {recorded_dir}, not {self.data_dir}. ' f'Keep the recorded data_dir or point at a new directory.')
def _blob_path(self, sha256_hex: str) -> str: h = sha256_hex.lower() levels = [h[2 * i:2 * i + 2] for i in range(self.fanout)] return os.path.join(self.data_dir, *levels, h)
[docs] def put( self, data: bytes, content_type: str = 'application/octet-stream', uploaded_by: Optional[str] = None, now: Optional[int] = None, ) -> BlobDescriptor: """Store a blob by content hash. Returns normative descriptor.""" if not isinstance(data, (bytes, bytearray)): raise TypeError("blob data must be bytes") size = len(data) if self.max_bytes is not None and size > self.max_bytes: raise ValueError(f"blob size {size} exceeds maximum limit {self.max_bytes}") h = hashlib.sha256(data).hexdigest().lower() now_ts = int(now if now is not None else time.time()) # Enforce quota watermark with automatic LRU eviction if self.max_storage_bytes is not None and not self.has(h): cur_total = self.total_bytes() if cur_total + size > self.max_storage_bytes: needed = (cur_total + size) - self.max_storage_bytes self.evict_lru(needed_bytes=needed) if self.total_bytes() + size > self.max_storage_bytes: raise ValueError( f"CAS quota exceeded: cannot store {size} bytes (storage cap {self.max_storage_bytes} bytes)" ) dest = self._blob_path(h) os.makedirs(os.path.dirname(dest), exist_ok=True) # Write atomically via tempfile in same filesystem directory if not os.path.exists(dest): with tempfile.NamedTemporaryFile(dir=os.path.dirname(dest), delete=False) as tf: tf.write(data) tmp_name = tf.name os.replace(tmp_name, dest) # Record metadata preserving pinned state if existing self.db.execute(''' INSERT INTO blobs (sha256, size, type, uploaded_by, created_at, accessed_at, pinned) VALUES (?, ?, ?, ?, ?, ?, COALESCE((SELECT pinned FROM blobs WHERE sha256 = ?), 0)) ON CONFLICT(sha256) DO UPDATE SET size=excluded.size, type=excluded.type, accessed_at=excluded.accessed_at ''', (h, size, content_type, uploaded_by, now_ts, now_ts, h)) self.db.commit() return { 'sha256': h, 'size': size, 'type': content_type, 'uploaded': now_ts, 'uploaded_by': uploaded_by, 'cid': sha256_to_cid(h), }
[docs] def get(self, sha256_hex: str) -> Optional[bytes]: """Read full blob bytes, verifying SHA-256 integrity.""" h = sha256_hex.lower() if not HEX64_RE.match(h): return None path = self._blob_path(h) if not os.path.exists(path): return None try: with open(path, 'rb') as f: data = f.read() # Verify integrity if hashlib.sha256(data).hexdigest().lower() != h: os.unlink(path) return None try: self.db.execute('UPDATE blobs SET accessed_at = ? WHERE sha256 = ?', (int(time.time()), h)) self.db.commit() except Exception: pass return data except OSError: return None
[docs] def has(self, sha256_hex: str) -> bool: """Check if blob exists on disk and is indexed.""" h = sha256_hex.lower() if not HEX64_RE.match(h): return False return os.path.exists(self._blob_path(h))
[docs] def stat(self, sha256_hex: str) -> Optional[BlobDescriptor]: """Retrieve metadata descriptor without loading blob content.""" h = sha256_hex.lower() if not HEX64_RE.match(h): return None row = self.db._one(''' SELECT size, type, uploaded_by, created_at, pinned, accessed_at FROM blobs WHERE sha256 = ? ''', (h,)) if not row: if not os.path.exists(self._blob_path(h)): return None try: sz = os.path.getsize(self._blob_path(h)) return { 'sha256': h, 'size': sz, 'type': 'application/octet-stream', 'uploaded': int(os.path.getmtime(self._blob_path(h))), 'uploaded_by': None, 'cid': sha256_to_cid(h), 'pinned': False, } except OSError: return None size, ctype, uploaded_by, created_at, pinned, accessed_at = row return { 'sha256': h, 'size': size, 'type': ctype, 'uploaded': created_at, 'uploaded_by': uploaded_by, 'cid': sha256_to_cid(h), 'pinned': bool(pinned), 'accessed_at': accessed_at, }
[docs] def delete(self, sha256_hex: str) -> bool: """Delete blob from disk and metadata index. The digest is validated before it touches the filesystem: get/has already refuse anything but 64 hex chars, and delete is the one path here that unlinks, so '../..' must never reach _blob_path. """ h = sha256_hex.lower() if not HEX64_RE.match(h): return False path = self._blob_path(h) deleted = False if os.path.exists(path): try: os.unlink(path) deleted = True except OSError: pass cur = self.db.execute('DELETE FROM blobs WHERE sha256 = ?', (h,)) self.db.commit() if cur.rowcount > 0: deleted = True return deleted
[docs] def list_by( self, uploaded_by: Optional[str] = None, limit: int = 100, offset: int = 0, ) -> list[BlobDescriptor]: """List stored blobs matching uploader pubkey.""" if uploaded_by: rows = self.db._all(''' SELECT sha256, size, type, uploaded_by, created_at FROM blobs WHERE uploaded_by = ? ORDER BY created_at DESC LIMIT ? OFFSET ? ''', (uploaded_by, limit, offset)) else: rows = self.db._all(''' SELECT sha256, size, type, uploaded_by, created_at FROM blobs ORDER BY created_at DESC LIMIT ? OFFSET ? ''', (limit, offset)) out: list[BlobDescriptor] = [] for h, sz, ctype, uploader, ts in rows: out.append({ 'sha256': h, 'size': sz, 'type': ctype, 'uploaded': ts, 'uploaded_by': uploader, 'cid': sha256_to_cid(h), }) return out
[docs] def total_bytes(self) -> int: """Total size in bytes of all indexed blobs.""" row = self.db._one('SELECT SUM(size) FROM blobs') return int((row and row[0]) or 0)
[docs] def count(self) -> int: """Number of indexed blobs.""" row = self.db._one('SELECT COUNT(*) FROM blobs') return int((row and row[0]) or 0)
[docs] def pin(self, sha256_hex: str) -> bool: """Pin a blob to prevent LRU eviction.""" h = sha256_hex.lower() if not HEX64_RE.match(h) or not self.has(h): return False cur = self.db.execute('UPDATE blobs SET pinned = 1 WHERE sha256 = ?', (h,)) self.db.commit() return cur.rowcount > 0
[docs] def unpin(self, sha256_hex: str) -> bool: """Unpin a blob, allowing LRU eviction.""" h = sha256_hex.lower() if not HEX64_RE.match(h): return False cur = self.db.execute('UPDATE blobs SET pinned = 0 WHERE sha256 = ?', (h,)) self.db.commit() return cur.rowcount > 0
[docs] def is_pinned(self, sha256_hex: str) -> bool: """Check if blob is pinned against LRU eviction.""" h = sha256_hex.lower() row = self.db._one('SELECT pinned FROM blobs WHERE sha256 = ?', (h,)) return bool(row and row[0])
[docs] def evict_lru(self, needed_bytes: int = 0) -> int: """Evict oldest unpinned blobs by accessed_at timestamp to free needed_bytes. Returns total bytes freed. """ rows = self.db._all( 'SELECT sha256, size FROM blobs WHERE pinned = 0 ' 'ORDER BY COALESCE(accessed_at, created_at) ASC, created_at ASC' ) freed = 0 for h, sz in rows: if needed_bytes > 0 and freed >= needed_bytes: break if self.delete(h): freed += sz return freed
[docs] def close(self) -> None: """Safely close underlying metadata database.""" self.db.close()