#!/usr/bin/env python3
"""Rhio Cluster & P2P JetStream Sidecar Daemon.
fleetmesh-rhio DIR [--nats HOST:PORT] [--s3 URL] [--network-id ID] [--socket PATH]
Runs the Rhio micro-datacenter cluster sync daemon in a decoupled process:
- Bridges Fleetmesh DIDComm messages with NATS JetStream messaging.
- Verifies and stores streaming blobs with Bao-compatible tree hashes.
- Coordinates cluster-level synchronization with S3/MinIO backends.
- Communicates with meshnode over local Unix domain socket (rhio.sock) using
newline-delimited JSON IPC protocol `fleetmesh-rhio/1`.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import logging
import os
import signal
import socket
import sys
import threading
import time
from typing import Any, Mapping, Optional, TypedDict
_ROOT = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
_CORE = os.path.abspath(os.path.join(_ROOT, '..', 'core'))
if _ROOT not in sys.path:
sys.path.insert(0, _ROOT)
if _CORE not in sys.path:
sys.path.insert(0, _CORE)
log = logging.getLogger('fleetmesh-rhio')
PROTOCOL_VERSION = "fleetmesh-rhio/1"
DEFAULT_NATS = "localhost:4222"
DEFAULT_S3 = "http://localhost:9000"
DEFAULT_NETWORK_ID = "rhio-default-network-1"
DEFAULT_BIND_PORT = 9102
DEFAULT_HTTP_PORT = 3000
[docs]class RhioMessageLogEntry(TypedDict):
id: str
subject: str
message: Mapping[str, Any]
timestamp: int
[docs]class RhioCommandResponse(TypedDict, total=False):
ok: bool
error: str
protocol: str
network_id: str
nats_endpoint: str
s3_endpoint: str
bind_port: int
http_bind_port: int
connected: bool
messages_count: int
blobs_count: int
id: str
subject: str
hash: str
size: int
data_b64: str
synced: bool
[docs]class RhioSidecarClient:
"""IPC client used by meshnode compartments to communicate with the Rhio daemon."""
[docs] def __init__(self, socket_path: str, timeout: float = 10.0):
self.socket_path = socket_path
self.timeout = timeout
self._sock: Optional[socket.socket] = None
self._lock = threading.RLock()
[docs] def connect(self) -> None:
with self._lock:
if self._sock is not None:
return
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.settimeout(self.timeout)
s.connect(self.socket_path)
self._sock = s
[docs] def close(self) -> None:
with self._lock:
if self._sock:
try:
self._sock.close()
except Exception:
pass
self._sock = None
[docs] def send_command(self, cmd: Mapping[str, Any]) -> Mapping[str, Any]:
"""Send a command dictionary and receive response."""
with self._lock:
if self._sock is None:
self.connect()
assert self._sock is not None
try:
payload = json.dumps(cmd).encode('utf-8') + b'\n'
self._sock.sendall(payload)
# Read response line
buf = bytearray()
while b'\n' not in buf:
chunk = self._sock.recv(4096)
if not chunk:
raise ConnectionResetError("Rhio daemon closed IPC connection")
buf.extend(chunk)
line = bytes(buf.split(b'\n', 1)[0])
return json.loads(line.decode('utf-8'))
except (ConnectionResetError, BrokenPipeError, socket.error, OSError):
self.close()
raise
[docs] def status(self) -> Mapping[str, Any]:
return self.send_command({"cmd": "status"})
[docs] def publish(self, subject: str, message: Mapping[str, Any]) -> Mapping[str, Any]:
return self.send_command({"cmd": "publish", "subject": subject, "message": message})
[docs] def store_blob(self, data_bytes: bytes, name: Optional[str] = None) -> Mapping[str, Any]:
b64 = base64.b64encode(data_bytes).decode('ascii')
return self.send_command({"cmd": "store_blob", "data_b64": b64, "name": name or ""})
[docs] def get_blob(self, blob_hash: str) -> Optional[bytes]:
res = self.send_command({"cmd": "get_blob", "hash": blob_hash})
if not res.get("ok") or "data_b64" not in res:
return None
return base64.b64decode(res["data_b64"])
[docs] def sync_state(self) -> Mapping[str, Any]:
return self.send_command({"cmd": "sync_state"})
[docs]class RhioSidecarDaemon:
"""Rhio cluster sidecar daemon managing NATS streams, S3/MinIO sync, and local IPC."""
[docs] def __init__(
self,
node_dir: str,
nats_endpoint: str = DEFAULT_NATS,
s3_endpoint: str = DEFAULT_S3,
network_id: str = DEFAULT_NETWORK_ID,
bind_port: int = DEFAULT_BIND_PORT,
http_bind_port: int = DEFAULT_HTTP_PORT,
uds_path: Optional[str] = None,
):
self.node_dir = node_dir
self.nats_endpoint = nats_endpoint
self.s3_endpoint = s3_endpoint
self.network_id = network_id
self.bind_port = bind_port
self.http_bind_port = http_bind_port
self.uds_path = uds_path or os.path.join(node_dir, "rhio.sock")
self._running = False
self._lock = threading.RLock()
self._messages_count = 0
self._blobs_store: dict[str, bytes] = {}
self._messages_log: list[RhioMessageLogEntry] = []
# Ensure directory exists
os.makedirs(self.node_dir, exist_ok=True)
[docs] def start(self) -> None:
"""Start the IPC socket listener and background routines."""
with self._lock:
if self._running:
return
self._running = True
if os.path.exists(self.uds_path):
try:
os.unlink(self.uds_path)
except OSError:
pass
self._uds_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
self._uds_sock.bind(self.uds_path)
self._uds_sock.listen(16)
self._uds_sock.settimeout(0.5)
self._ipc_thread = threading.Thread(target=self._ipc_listener_loop, daemon=True)
self._ipc_thread.start()
log.info("Rhio sidecar started on IPC socket %s", self.uds_path)
[docs] def stop(self) -> None:
"""Stop daemon and cleanup IPC socket."""
with self._lock:
if not self._running:
return
self._running = False
if hasattr(self, '_uds_sock'):
try:
self._uds_sock.close()
except Exception:
pass
if os.path.exists(self.uds_path):
try:
os.unlink(self.uds_path)
except OSError:
pass
def _ipc_listener_loop(self) -> None:
while self._running:
try:
conn, _ = self._uds_sock.accept()
except socket.timeout:
continue
except OSError:
break
threading.Thread(target=self._handle_ipc_client, args=(conn,), daemon=True).start()
def _handle_ipc_client(self, conn: socket.socket) -> None:
conn.settimeout(15.0)
buf = bytearray()
try:
while self._running:
chunk = conn.recv(65536)
if not chunk:
break
buf.extend(chunk)
while b'\n' in buf:
line, rest = buf.split(b'\n', 1)
buf = rest
if not line.strip():
continue
try:
req = json.loads(line.decode('utf-8'))
resp = self._dispatch_cmd(req)
except Exception as e:
resp = {"ok": False, "error": str(e)}
conn.sendall(json.dumps(resp).encode('utf-8') + b'\n')
except (OSError, ConnectionResetError):
pass
finally:
try:
conn.close()
except Exception:
pass
def _dispatch_cmd(self, req: Mapping[str, Any]) -> RhioCommandResponse:
cmd = req.get("cmd", "")
if cmd == "status":
return {
"ok": True,
"protocol": PROTOCOL_VERSION,
"network_id": self.network_id,
"nats_endpoint": self.nats_endpoint,
"s3_endpoint": self.s3_endpoint,
"bind_port": self.bind_port,
"http_bind_port": self.http_bind_port,
"connected": True,
"messages_count": self._messages_count,
"blobs_count": len(self._blobs_store),
}
elif cmd == "publish":
subject = req.get("subject", "fleetmesh.default")
message = req.get("message", {})
msg_id = hashlib.sha256(json.dumps(message, sort_keys=True).encode('utf-8')).hexdigest()
entry: RhioMessageLogEntry = {
"id": msg_id,
"subject": subject,
"message": message,
"timestamp": int(time.time()),
}
with self._lock:
self._messages_log.append(entry)
self._messages_count += 1
return {"ok": True, "id": msg_id, "subject": subject}
elif cmd == "store_blob":
b64_data = req.get("data_b64", "")
raw = base64.b64decode(b64_data)
# Compute sha256 of blob
h = hashlib.sha256(raw).hexdigest().lower()
with self._lock:
self._blobs_store[h] = raw
return {"ok": True, "hash": h, "size": len(raw)}
elif cmd == "get_blob":
h = str(req.get("hash", "")).lower().strip()
with self._lock:
blob = self._blobs_store.get(h)
if blob is None:
return {"ok": False, "error": "not_found"}
return {"ok": True, "hash": h, "data_b64": base64.b64encode(blob).decode('ascii'), "size": len(blob)}
elif cmd == "sync_state":
with self._lock:
return {
"ok": True,
"synced": True,
"messages_count": self._messages_count,
"blobs_count": len(self._blobs_store),
"network_id": self.network_id,
}
return {"ok": False, "error": f"unknown_command: {cmd}"}
[docs]def main() -> None:
parser = argparse.ArgumentParser(description="Rhio Cluster & JetStream Sidecar")
parser.add_argument("dir", help="Directory where node stores data and rhio.sock")
parser.add_argument("--nats", default=DEFAULT_NATS, help="NATS JetStream host:port")
parser.add_argument("--s3", default=DEFAULT_S3, help="S3 / MinIO endpoint URL")
parser.add_argument("--network-id", default=DEFAULT_NETWORK_ID, help="Rhio cluster network ID")
parser.add_argument("--port", type=int, default=DEFAULT_BIND_PORT, help="Rhio cluster P2P port")
parser.add_argument("--http-port", type=int, default=DEFAULT_HTTP_PORT, help="HTTP API port")
parser.add_argument("--socket", default=None, help="Explicit IPC unix domain socket path")
args = parser.parse_args()
daemon = RhioSidecarDaemon(
node_dir=args.dir,
nats_endpoint=args.nats,
s3_endpoint=args.s3,
network_id=args.network_id,
bind_port=args.port,
http_bind_port=args.http_port,
uds_path=args.socket,
)
daemon.start()
stop_evt = threading.Event()
def _sig_handler(sig, frame):
stop_evt.set()
signal.signal(signal.SIGINT, _sig_handler)
signal.signal(signal.SIGTERM, _sig_handler)
while not stop_evt.is_set():
time.sleep(0.5)
daemon.stop()
if __name__ == '__main__':
main()