"""The DVM gateway compartment: takes a NIP-90 request off the relay face,
runs (or delegates) the job, answers with feedback and a result.
"""
from __future__ import annotations
import hashlib
import os
import queue
import threading
from typing import Any, Callable, Mapping, Optional, Union
from fleetmesh_node import channels as ch_mod
from fleetmesh_node.planes.services import compute # noqa: E402
from fleetmesh_node import metrics as mx_mod
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.compartment import Command, Compartment, Need, Worker # noqa: E402
from fleetmesh_node.config.parameter import Parameter # noqa: E402
from fleetmesh_node.config import DVMConfig # noqa: E402
from fleetmesh_node.interfaces import NostrEvent # noqa: E402
from .protocol import DVMError, build_dvm_feedback, build_dvm_response, is_dvm_request, parse_dvm_request # noqa: E402
from .translate import dvm_request_to_mesh_job # noqa: E402
from .types import DVMDelegationResult, DVMStateReport
def _module_sha(job: Mapping[str, Any]) -> str:
m = job.get('module')
sha = m.get('sha256') if isinstance(m, dict) else None
return sha if isinstance(sha, str) else ''
def _verify_mode(job: Mapping[str, Any]) -> str:
v = job.get('verify')
mode = v.get('mode') if isinstance(v, dict) else None
return mode if isinstance(mode, str) else 'none'
[docs]class DVMGateway(Compartment):
optional: bool = True
enabled_by: Optional[str] = 'dvm'
"""The NIP-90 Data Vending Machine Gateway compartment."""
name: str = 'dvm'
# Both optional: the order book only for `delegate_request`, the bridge
# only so results reach the wider relay network. Local service needs neither.
uses: tuple[str, ...] = ('compute', 'bridge')
reads: tuple[str, ...] = ('compute', 'dvm')
config_cls = DVMConfig
live: tuple[str, ...] = ('dvm',)
STORES = (
st.Store(key='recent_jobs', description='recent DVM job results, newest last',
kind=st.RING, shape=tuple, bound=128),
)
recent_jobs: st.Ring[tuple[Any, ...]]
CHANNELS = (
ch_mod.Channel('job_received', 'A NIP-90 DVM request event was received for processing',
level=ch_mod.INFO, say='received DVM job {job_id} (kind {kind})',
tags=('dvm', 'job')),
ch_mod.Channel('job_succeeded', 'A NIP-90 DVM job completed successfully',
level=ch_mod.INFO, say='completed DVM job {job_id} using {fuel} fuel',
tags=('dvm', 'job')),
ch_mod.Channel('job_failed', 'A NIP-90 DVM job failed during execution',
level=ch_mod.TROUBLE, say='DVM job {job_id} failed: {error}',
tags=('dvm', 'job')),
ch_mod.Channel('job_delegated', 'A NIP-90 DVM job was delegated to compute order book',
level=ch_mod.INFO, say='delegated DVM job {job_id} to compute order book',
tags=('dvm', 'delegate')),
)
METRICS = (
mx_mod.Metric(name='dvm_jobs_received', description='DVM job requests received.',
kind=mx_mod.COUNTER, counts='job_received', tags=('dvm', 'job')),
mx_mod.Metric(name='dvm_jobs_succeeded', description='DVM jobs completed successfully.',
kind=mx_mod.COUNTER, counts='job_succeeded', tags=('dvm', 'job')),
mx_mod.Metric(name='dvm_jobs_failed', description='DVM jobs that failed.',
kind=mx_mod.COUNTER, counts='job_failed', tags=('dvm', 'job')),
mx_mod.Metric(name='dvm_jobs_delegated', description='DVM jobs delegated to compute order book.',
kind=mx_mod.COUNTER, counts='job_delegated', tags=('dvm', 'delegate')),
)
WORKERS = (
Worker('_dvm_worker_loop', name='dvm_requests'),
)
NEEDS = (
Need('cfg', block=DVMConfig),
Need('ks'),
Need('face'),
Need('blob_store', 'stores.blobs'),
Need('order_book', 'compute'),
Need('bridge'),
)
COMMANDS = (
Command(
key='show',
name='DVM Gateway Status',
description='display DVM gateway state, supported kinds, and job counters',
tags=('dvm', 'status'),
permission='dvm/show',
run='state',
reply='dict',
),
Command(
key='delegate',
name='Delegate DVM Request',
description='delegate a NIP-90 request to the compute order book',
tags=('dvm', 'delegate'),
parameters=(
Parameter(key='request', name='request', description='NIP-90 request event dictionary', type=dict, default=dict, required=True),
),
permission='dvm/delegate',
run='delegate_cmd',
reply='dict',
),
Command(
key='process',
name='Process DVM Request',
description='execute a NIP-90 request locally or via delegated runner',
tags=('dvm', 'process'),
parameters=(
Parameter(key='request', name='request', description='NIP-90 request event dictionary', type=dict, default=dict, required=True),
),
permission='dvm/process',
run='process_cmd',
reply='dict',
),
)
# Removes on_event from the node's face; set while started.
_face_unsubscribe: Optional[Callable[[], None]] = None
[docs] def start(self) -> None:
super().start()
add = getattr(self.face, 'add_event_handler', None)
if add is not None:
self._face_unsubscribe = add(self.on_event)
[docs] def stop(self) -> None:
super().stop()
if self._face_unsubscribe is not None:
self._face_unsubscribe()
self._face_unsubscribe = None
[docs] def __init__(
self,
node: Optional[Any] = None,
cfg: Optional[Union[DVMConfig, Mapping[str, Any]]] = None,
custom_runner: Optional[Callable[[Mapping[str, Any]], bytes]] = None,
blob_store: Optional[Any] = None,
registry: Optional[compute.ExecutorRegistry] = None,
order_book: Optional[Any] = None,
bridge: Optional[Any] = None,
ks: Optional[Any] = None,
face: Optional[Any] = None,
clock: Optional[Callable[[], int]] = None,
payment_rail: Optional[Any] = None,
settlement: Optional[Any] = None,
) -> None:
super().__init__(node, clock)
# The keystore results are signed with and the relay face they fan
# out to; named from ctx in a node, read off a fake node in tests.
self.ks: Optional[Any] = ks if ks is not None else getattr(node, 'ks', None)
self.face: Optional[Any] = face if face is not None else getattr(node, 'face', None)
# The Compute compartment (or anything with `command('ask', ...)` and
# `find_offers_for_ask`) that delegated requests are posted to.
self.order_book: Optional[Any] = order_book
# The relay bridge (anything with `enqueue_egress`) results fan out to.
self.bridge: Optional[Any] = bridge
# Payment rail wiring (Lightning hold invoices)
if payment_rail is not None:
self.payment_rail: Optional[Any] = payment_rail
elif settlement is not None:
try:
self.payment_rail = getattr(settlement, 'engine', settlement).get_rail('lightning')
except Exception:
self.payment_rail = None
elif node is not None and getattr(node, 'settle_engine', None) is not None:
try:
self.payment_rail = node.settle_engine.get_rail('lightning')
except Exception:
self.payment_rail = None
elif node is not None and getattr(node, 'settlement', None) is not None:
try:
self.payment_rail = getattr(node.settlement, 'engine', node.settlement).get_rail('lightning')
except Exception:
self.payment_rail = None
else:
self.payment_rail = None
# Tests hand a dict; the node hands the typed block. Either way the
# block is the shape, and the fields below read from it.
c = cfg if isinstance(cfg, DVMConfig) else DVMConfig.from_dict(cfg or {})
self.blob_store: Optional[Any] = blob_store if blob_store is not None else getattr(node, 'blobs', None)
# Which executors serve requests. Defaults to the process-wide
# registry (live drivers); tests pass one built with mock_mode=True
# so the LLM/agent/Bacalhau fixtures can answer without a backend.
self.registry: compute.ExecutorRegistry = registry or compute.DEFAULT_REGISTRY
self._face_unsubscribe = None
self.blossom_servers: list[str] = list(c.blossom_servers) # not a store: configured blossom servers from config
self.enabled: bool = c.enabled
self.supported_kinds: set[int] = set(c.supported_kinds) # not a store: configured supported kinds from config
self.rate_msat_per_mfuel: int = c.rate_msat_per_mfuel
self.max_fuel: int = c.max_fuel
self.delegate_to_mesh: bool = c.delegate_to_mesh
self.custom_runner: Optional[Callable[[Mapping[str, Any]], bytes]] = custom_runner
self.jobs_received: int = 0
self.jobs_succeeded: int = 0
self.jobs_failed: int = 0
self.total_fuel_used: int = 0
self.last_job_at: Optional[int] = None
self._lock: threading.Lock = threading.Lock()
self._request_queue: queue.Queue[NostrEvent] = queue.Queue(maxsize=128)
# No copy of the results is kept here. One was, keyed by request, for
# every job served and never read: the result is returned to the
# caller and fanned out to the face's subscribers and the bridge's
# relays (see _fanout), and that is what answers a client.
[docs] def state(self) -> DVMStateReport:
return {
'enabled': self.enabled,
'supported_kinds': sorted(list(self.supported_kinds)),
'rate_msat_per_mfuel': self.rate_msat_per_mfuel,
'delegate_to_mesh': self.delegate_to_mesh,
'jobs_received': self.jobs_received,
'jobs_succeeded': self.jobs_succeeded,
'jobs_failed': self.jobs_failed,
'total_fuel_used': self.total_fuel_used,
'last_job_at': self.last_job_at,
}
[docs] def trouble(self) -> list[str]:
out: list[str] = []
recent_errs = self.refusals.within(1800, self.clock(), kind='dvm')
if len(recent_errs) > 10:
out.append(f'{len(recent_errs)} DVM job failure(s) in the last 30 minutes')
return out
[docs] def process_request(
self,
request_ev: NostrEvent,
signer_sk: Optional[str] = None,
) -> tuple[Optional[NostrEvent], NostrEvent]:
"""Process a NIP-90 request, returning (response_event_or_None, feedback_event)."""
sk = signer_sk or (self.ks.node_sk if self.ks is not None else None)
if not sk:
raise DVMError('no signer private key available for DVM gateway')
now = self.clock()
req_id = request_ev.get('id', '')
req_kind = request_ev.get('kind', 0)
with self._lock:
self.jobs_received += 1
self.last_job_at = now
self.say('job_received', job_id=req_id[:16], kind=req_kind)
if not is_dvm_request(request_ev) or req_kind not in self.supported_kinds:
fb_err = build_dvm_feedback(sk, request_ev, 'error', 'unsupported job kind', now=now)
self.refusals.refuse('dvm', 'unsupported kind', subject=req_id[:16], detail={'kind': req_kind})
with self._lock:
self.jobs_failed += 1
self.recent_jobs.append((req_id[:16], 'unsupported', req_kind))
self.say('job_failed', job_id=req_id[:16], error='unsupported job kind')
return None, fb_err
# Tell the requester we have started. This used to be built and then
# dropped on the floor; NIP-90 clients rely on the 'processing'
# status to distinguish a slow job from an ignored one, so it goes
# out through the same fan-out the final events use.
fb_proc = build_dvm_feedback(sk, request_ev, 'processing', now=now)
self._fanout(fb_proc)
# Check if client provided an escrow / hold invoice payment hash tag
client_pay_hash = None
for tag in request_ev.get('tags', []):
if len(tag) >= 2 and tag[0] in ('payment_hash', 'hold_invoice', 'escrow'):
client_pay_hash = tag[1]
break
if client_pay_hash and self.payment_rail is not None:
st = self.payment_rail.get_status(client_pay_hash)
if getattr(st, 'status', '') == 'CREATED':
self.payment_rail.hold_payment(client_pay_hash)
# Map to mesh job
parsed = parse_dvm_request(request_ev)
job_payload = dvm_request_to_mesh_job(request_ev, fuel_max=self.max_fuel)
if self.blob_store is not None:
job_payload['blob_store'] = self.blob_store
if self.blossom_servers:
job_payload.setdefault('blossom_servers', self.blossom_servers)
try:
if (self.custom_runner is None
and isinstance(self.registry.select_for_job(job_payload), compute.WasmExecutor)
and not _module_sha(job_payload)):
raise DVMError("WASM job names no module: add ['param', 'module', <sha256>]")
exec_res = compute.run_job(job_payload, custom_runner=self.custom_runner, registry=self.registry)
fuel = exec_res['fuel_used']
raw_out = exec_res['raw_output']
out_str = raw_out.decode('utf-8', errors='replace') if isinstance(raw_out, bytes) else str(raw_out)
# Calculate price - support zero-cost / free compute jobs
if self.rate_msat_per_mfuel == 0 or parsed.get('bid_msat', 0) == 0:
msat = 0
else:
msat = max(1, (fuel * self.rate_msat_per_mfuel) // 1000000)
commit = exec_res['result_hash']
extra_tags: list[list[str]] = []
counters = exec_res.get('counters') or {}
if 'total_tokens' in counters:
extra_tags.append(['tokens', str(counters['total_tokens'])])
if 'prompt_tokens' in counters:
extra_tags.append(['prompt_tokens', str(counters['prompt_tokens'])])
if 'completion_tokens' in counters:
extra_tags.append(['completion_tokens', str(counters['completion_tokens'])])
if 'model' in job_payload:
extra_tags.append(['model', str(job_payload['model'])])
if 'executor' in job_payload:
extra_tags.append(['executor', str(job_payload['executor'])])
# Live Lightning HTLC hold-invoice settlement:
if msat > 0 and self.payment_rail is not None:
preimage_bytes = os.urandom(32)
preimage = preimage_bytes.hex()
pay_hash = client_pay_hash or hashlib.sha256(preimage_bytes).hexdigest()
inv_status = self.payment_rail.get_status(pay_hash)
if getattr(inv_status, 'status', 'UNKNOWN') == 'UNKNOWN':
inv = self.payment_rail.create_hold_invoice(
msat,
pay_hash,
description=f"DVM job {req_id[:16]}",
)
else:
inv = inv_status
if getattr(inv, 'status', '') == 'CREATED':
self.payment_rail.hold_payment(pay_hash)
if hashlib.sha256(bytes.fromhex(preimage)).hexdigest().lower() == pay_hash.lower():
self.payment_rail.settle_payment(pay_hash, preimage)
extra_tags.append(['preimage', preimage])
bolt11_val = getattr(inv, 'bolt11', None)
if bolt11_val:
extra_tags.append(['bolt11', bolt11_val])
extra_tags.append(['payment_hash', pay_hash])
resp_ev = build_dvm_response(
sk, request_ev, out_str, amount_msat=msat, commit=commit, extra_tags=extra_tags, now=now
)
fb_succ = build_dvm_feedback(sk, request_ev, 'success', now=now)
with self._lock:
self.jobs_succeeded += 1
self.total_fuel_used += fuel
requester_pk = str(request_ev.get('pubkey', ''))
self.recent_jobs.append((req_id[:16], 'success', requester_pk, fuel))
self.say('job_succeeded', job_id=req_id[:16], peer=requester_pk, fuel=fuel)
# Fan out to local face and bridge if wired
self._fanout(resp_ev)
self._fanout(fb_succ)
return resp_ev, fb_succ
except Exception as e:
if client_pay_hash and self.payment_rail is not None:
try:
st = self.payment_rail.get_status(client_pay_hash)
if getattr(st, 'status', '') == 'HELD':
self.payment_rail.cancel_payment(client_pay_hash, reason=str(e)[:40])
except Exception:
pass
with self._lock:
self.jobs_failed += 1
self.recent_jobs.append((req_id[:16], 'failed', str(e)[:40]))
self.say('job_failed', job_id=req_id[:16], error=str(e)[:40])
self.refusals.refuse('dvm', 'job execution failed', subject=req_id[:16], detail={'error': str(e)})
fb_fail = build_dvm_feedback(sk, request_ev, 'error', str(e), now=now)
return None, fb_fail
[docs] def create_payment_required_feedback(
self,
request_ev: NostrEvent,
amount_msat: int,
signer_sk: Optional[str] = None,
payment_hash: Optional[str] = None,
preimage: Optional[str] = None,
) -> tuple[NostrEvent, Optional[Any], str]:
"""Build and fan out a Kind 7000 payment-required feedback event with a Lightning hold invoice."""
sk = signer_sk or (self.ks.node_sk if self.ks is not None else None)
if not sk:
raise DVMError('no signer private key available for DVM gateway')
now = self.clock()
req_id = str(request_ev.get('id', ''))
if not preimage:
preimage = os.urandom(32).hex()
if not payment_hash:
payment_hash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest()
inv = None
bolt11 = ''
if self.payment_rail is not None:
inv = self.payment_rail.create_hold_invoice(
amount_msat,
payment_hash,
description=f"DVM job {req_id[:16]}",
)
bolt11 = getattr(inv, 'bolt11', '') or ''
else:
bolt11 = f"lnbc{amount_msat}u1p{payment_hash[:32]}"
extra_tags: list[list[str]] = [
['amount', str(amount_msat), bolt11] if bolt11 else ['amount', str(amount_msat)],
]
if bolt11:
extra_tags.append(['bolt11', bolt11])
extra_tags.append(['payment_hash', payment_hash])
fb_ev = build_dvm_feedback(
sk,
request_ev,
'payment-required',
extra=f"{amount_msat} msat required",
extra_tags=extra_tags,
now=now,
)
self._fanout(fb_ev)
return fb_ev, inv, preimage
def _fanout(self, ev: NostrEvent) -> None:
"""Deliver a DVM event to the local relay face and the relay bridge, if wired."""
if self.face is not None:
self.face._fanout(ev)
if self.bridge is not None:
self.bridge.enqueue_egress(ev)
[docs] def delegate_request(
self,
request_ev: NostrEvent,
) -> DVMDelegationResult:
"""Delegate a NIP-90 request to the mesh compute order book."""
book = self.order_book
if book is None:
raise DVMError('compute subsystem not available for order book delegation')
parsed = parse_dvm_request(request_ev)
job_payload = dvm_request_to_mesh_job(request_ev, fuel_max=self.max_fuel)
ask_res = book.command('ask', {
'id': parsed['id'],
'job': job_payload,
'bid': parsed['bid_msat'],
'executor': job_payload.get('executor', 'wasm'),
'verify': _verify_mode(job_payload),
})
matches = book.find_offers_for_ask(parsed['id'])
self.say('job_delegated', job_id=parsed['id'])
return {
'ask_id': parsed['id'],
'ask_res': ask_res,
'matches_found': len(matches),
'candidates': matches,
}
[docs] def on_event(self, ev: NostrEvent) -> None:
"""Callback to handle events observed on the local face or relay bridge."""
if not self.enabled:
return
if is_dvm_request(ev) and ev.get('kind') in self.supported_kinds:
try:
self._request_queue.put_nowait(ev)
except queue.Full:
self.refusals.refuse('dvm', 'request queue full; dropping event', subject=str(ev.get('id', ''))[:16])
def _dvm_worker_loop(self, stop_event: threading.Event) -> None:
while not stop_event.is_set():
try:
ev = self._request_queue.get(timeout=0.2)
except queue.Empty:
continue
try:
self.process_request(ev)
except Exception as e:
self.refusals.refuse('dvm', f'worker error processing request: {e}', subject=str(ev.get('id', ''))[:16])
finally:
self._request_queue.task_done()
@staticmethod
def _request_event(args: Mapping[str, Any]) -> NostrEvent:
"""The 'request' argument as a NostrEvent, or a DVMError.
Control-plane callers hand us untyped JSON; every signed-event field
must be present before it is treated as an event, otherwise a missing
'id' or 'pubkey' would surface later as a KeyError deep in processing.
"""
ev = args.get('request')
if not isinstance(ev, dict):
raise DVMError('request must be an event dictionary')
missing = [k for k in ('id', 'pubkey', 'created_at', 'kind', 'tags', 'content', 'sig') if k not in ev]
if missing:
raise DVMError('request event is missing fields: %s' % ', '.join(missing))
event: NostrEvent = {
'id': str(ev['id']),
'pubkey': str(ev['pubkey']),
'created_at': int(ev['created_at']),
'kind': int(ev['kind']),
'tags': list(ev['tags']),
'content': str(ev['content']),
'sig': str(ev['sig']),
}
return event
[docs] def delegate_cmd(self, args: Mapping[str, Any]) -> Any:
return self.delegate_request(self._request_event(args))
[docs] def process_cmd(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
resp, fb = self.process_request(self._request_event(args))
return {
'response': resp,
'feedback': fb,
'success': resp is not None,
}