"""The Compute subsystem: node-side lifecycle, offer publication and job handling.
Owns its wire half: `compute/wire/` (ComputeWire, ComputeSession, the
compute/1.0 and /1.1 messages) is constructed here, and wire imports
executors/jobs from this package - one direction, no cycle.
"""
from __future__ import annotations
import os
from typing import Any, Callable, Mapping, Optional
from fleetmesh_core.crypto import crypto
from fleetmesh_core.wire.interfaces import NostrEvent
from fleetmesh_node import channels as ch_mod, compartment as compartment_mod, metrics as mx_mod
from fleetmesh_node.compartment import stores as st
from fleetmesh_node.compartment import Command, Need
from fleetmesh_node.config.parameter import Parameter
from fleetmesh_node.config import ComputeConfig
from .common import ComputeError, ASK_KIND
from ..compute_wire import ComputeWire
from ..compute_wire.messages import PIURI as COMPUTE_PIURI, PIURI_V11 as COMPUTE_PIURI_V11
from .market import (
build_ask,
build_offer,
match_ask_to_offers,
match_offer_to_asks,
parse_ask,
parse_offer,
validate_offer,
)
from .executors import ExecutorRegistry
from .jobs import build_receipt, run_job
from .load import ComputeLoadMonitor
# -------------------------------------------------------------------------
# Compute Compartment
# -------------------------------------------------------------------------
[docs]class Compute(compartment_mod.Compartment):
optional: bool = True
enabled_by: Optional[str] = 'compute'
"""The compute/1.0 compartment of meshnode."""
name = 'compute'
piuris: tuple[str, ...] = (COMPUTE_PIURI, COMPUTE_PIURI_V11)
reads = ('compute', 'executors', 'relays')
config_cls = ComputeConfig
NEEDS = (
Need('ks', 'ks'),
Need('blobs', 'stores.blobs'),
)
COMMANDS = (
Command(
key='show',
name='Compute Status',
description='display active compute offers, asks, and receipts',
tags=('compute', 'status'),
permission='compute/show',
run='state',
reply='dict',
),
Command(
key='offer',
name='Publish Offer',
description='publish a kind 11803 compute offer',
tags=('compute', 'offer'),
parameters=(
Parameter(key='engine', name='engine', description='execution engine', type=str, default='', required=True),
Parameter(key='msat_per_mfuel', name='msat_per_mfuel', description='price in msat per megabytes fuel', type=object, default=0, required=True),
),
permission='compute/offer',
run='_cmd_offer',
reply='dict',
),
Command(
key='ask',
name='Publish Ask',
description='publish a kind 30810 confidential compute ask',
tags=('compute', 'ask'),
parameters=(
Parameter(key='id', name='id', description='ask id', type=str, default='', required=True),
Parameter(key='job', name='job', description='job payload', type=dict, default=dict, required=True),
Parameter(key='bid', name='bid', description='bid amount', type=object, default=0, required=True),
Parameter(key='executor', name='executor', description='executor type', type=str, default='wasm', required=False),
Parameter(key='verify', name='verify', description='verification mode', type=str, default='none', required=False),
),
permission='compute/ask',
run='_cmd_ask',
reply='dict',
),
Command(
key='run',
name='Run Job',
description='execute a job payload and return signed kind 30811 receipt',
tags=('compute', 'run'),
parameters=(
Parameter(key='job', name='job', description='job payload', type=dict, default=dict, required=True),
Parameter(key='buyer', name='buyer', description='buyer public key', type=str, default='', required=True),
Parameter(key='preimage', name='preimage', description='payment preimage', type=str, default=None, required=False),
Parameter(key='paid_msat', name='paid_msat', description='paid amount in millisatoshis', type=int, default=None, required=False),
Parameter(key='payment_hash', name='payment_hash', description='payment hash', type=str, default=None, required=False),
),
permission='compute/run',
run='_cmd_run',
reply='dict',
),
Command(
key='submit',
name='Submit or Reflect Job',
description='execute a job locally when CPU headroom permits or reflect to a less-loaded mesh peer',
tags=('compute', 'submit', 'reflection'),
parameters=(
Parameter(key='job', name='job', description='job payload', type=dict, default=dict, required=True),
Parameter(key='buyer', name='buyer', description='buyer public key', type=str, default='', required=True),
Parameter(key='bid', name='bid', description='maximum bid in msat per mfuel', type=int, default=0, required=False),
Parameter(key='force_reflect', name='force_reflect', description='force reflection to a remote peer if available', type=bool, default=False, required=False),
),
permission='compute/submit',
run='_cmd_submit',
reply='dict',
),
Command(
key='load',
name='Compute Load Snapshot',
description='display live host CPU utilization, active worker slots, and reflection status',
tags=('compute', 'load', 'metrics'),
permission='compute/load',
run='_cmd_load',
reply='dict',
),
Command(
key='receipts',
name='Job Receipts',
description='list compute job receipts held',
tags=('compute', 'receipts'),
permission='compute/receipts',
run='receipts_cmd',
reply='list',
),
Command(
key='book',
name='Order Book',
description='display active order book offers and asks',
tags=('compute', 'book'),
permission='compute/book',
run='book_cmd',
reply='dict',
),
Command(
key='match',
name='Match Orders',
description='find matching counterparties in order book',
tags=('compute', 'match'),
parameters=(
Parameter(key='ask', name='ask', description='ask id or event', type=object, default=None, required=False),
Parameter(key='offer', name='offer', description='offer id or event', type=object, default=None, required=False),
),
permission='compute/match',
run='match_cmd',
reply='list',
),
Command(
key='index_offer',
name='Index Offer',
description='index an external kind 11803 offer',
tags=('compute', 'offer'),
parameters=(
Parameter(key='offer', name='offer', description='kind 11803 offer to index', type=object, default=dict, required=True),
),
permission='compute/index_offer',
run='index_offer_cmd',
reply='bool',
),
Command(
key='index_ask',
name='Index Ask',
description='index an external kind 30810 ask',
tags=('compute', 'ask'),
parameters=(
Parameter(key='ask', name='ask', description='kind 30810 ask to index', type=object, default=dict, required=True),
),
permission='compute/index_ask',
run='index_ask_cmd',
reply='bool',
),
Command(
key='models',
name='Compute Models',
description='list available AI/LLM models and inference features',
tags=('compute', 'models'),
permission='compute/models',
run='_cmd_models',
reply='dict',
),
Command(
key='stream',
name='Stream Inference',
description='run a streaming AI/LLM inference query and return token deltas',
tags=('compute', 'stream'),
parameters=(
Parameter(key='prompt', name='prompt', description='prompt text', type=str, default='', required=True),
Parameter(key='model', name='model', description='model identifier', type=str, default=None, required=False),
),
permission='compute/stream',
run='_cmd_stream',
reply='dict',
),
Command(
key='link_provider',
name='Link AI Provider',
description='link an external or local AI/LLM inference provider to this meshnode',
tags=('compute', 'models'),
parameters=(
Parameter(key='provider_id', name='provider_id', description='provider preset id (openrouter, claude, openai, gemini, copilot, routstr, payperquery, ollama, vllm)', type=str, default='openrouter', required=True),
Parameter(key='endpoint', name='endpoint', description='custom base endpoint URL', type=str, default=None, required=False),
Parameter(key='default_model', name='default_model', description='default model ID', type=str, default=None, required=False),
Parameter(key='api_key', name='api_key', description='operator API key or bearer secret', type=str, default=None, required=False),
Parameter(key='cashu_token', name='cashu_token', description='operator Cashu token for Routstr/PPQ', type=str, default=None, required=False),
Parameter(key='rate_msat_per_token', name='rate_msat_per_token', description='msat rate per token', type=int, default=None, required=False),
Parameter(key='enabled', name='enabled', description='whether provider is active', type=bool, default=True, required=False),
Parameter(key='set_default', name='set_default', description='set as primary provider', type=bool, default=True, required=False),
),
permission='compute/link_provider',
run='_cmd_link_provider',
reply='dict',
),
)
STORES = (
st.Store(
key='offers',
description='offers published by this node, newest last',
bound=1024,
kind=st.RING,
),
st.Store(
key='asks',
description='active asks placed by this node, by ask_id',
bound=1024,
evict=st.LRU,
kind=st.TABLE,
keyed_by='ask',
),
st.Store(
key='order_book_offers',
description='indexed offers from mesh peers, by offer_id',
bound=4096,
evict=st.LRU,
kind=st.TABLE,
keyed_by='offer',
),
st.Store(
key='order_book_asks',
description='indexed asks from mesh peers, by ask_id',
bound=4096,
evict=st.LRU,
kind=st.TABLE,
keyed_by='ask',
),
st.Store(
key='receipts_held',
description='job completion receipts held by this node, newest last',
bound=1024,
kind=st.RING,
),
)
CHANNELS = (
ch_mod.Channel('job_executed', 'A compute task execution completed and receipt signed',
level=ch_mod.NOTE, say='compute job {job_id} executed for buyer {buyer}',
tags=('compute', 'execution')),
ch_mod.Channel('job_reflected', 'A compute task was reflected to a less-loaded mesh peer',
level=ch_mod.INFO, say='compute job {job_id} reflected to peer {peer} ({reason})',
tags=('compute', 'reflection')),
ch_mod.Channel('offer_published', 'A compute engine capability offer was published',
level=ch_mod.NOTE, say='compute offer published for engine {engine}',
tags=('compute', 'market')),
ch_mod.Channel('ask_published', 'A confidential compute ask was published to the mesh',
level=ch_mod.NOTE, say='compute ask {ask_id} published bid {bid}',
tags=('compute', 'market')),
)
METRICS = (
mx_mod.Metric(name='compute_jobs_executed', description='Compute task executions completed.',
kind=mx_mod.COUNTER, counts='job_executed', tags=('compute', 'execution')),
mx_mod.Metric(name='compute_jobs_reflected', description='Compute jobs reflected to less-loaded mesh peers.',
kind=mx_mod.COUNTER, counts='job_reflected', tags=('compute', 'reflection')),
mx_mod.Metric(name='compute_offers_published', description='Compute engine offers published.',
kind=mx_mod.COUNTER, counts='offer_published', tags=('compute', 'market')),
mx_mod.Metric(name='compute_asks_published', description='Compute asks published to mesh.',
kind=mx_mod.COUNTER, counts='ask_published', tags=('compute', 'market')),
)
[docs] def __init__(self, node: Any = None, registry: Optional[ExecutorRegistry] = None,
ks: Optional[Any] = None, blobs: Optional[Any] = None,
clock: Optional[Callable[[], int]] = None,
load_monitor: Optional[ComputeLoadMonitor] = None,
remote_dispatcher: Optional[Callable[[Mapping[str, Any], Mapping[str, Any]], Mapping[str, Any]]] = None) -> None:
super().__init__(node, clock)
# What this compartment takes from the node, named: the keystore that
# signs offers/asks/receipts and the blob store a job reads from.
# Suites that pass a fake node and nothing else still get them off it.
self.ks: Optional[Any] = ks if ks is not None else getattr(node, 'ks', None)
self.blobs: Optional[Any] = blobs if blobs is not None else getattr(node, 'blobs', None)
self.offers: st.Ring[NostrEvent]
self.asks: st.Table[str, NostrEvent]
self.order_book_offers: st.Table[str, NostrEvent]
self.order_book_asks: st.Table[str, NostrEvent]
self.receipts_held: st.Ring[Mapping[str, Any]]
self.jobs_executed: int = 0
self.load_monitor: ComputeLoadMonitor = load_monitor or ComputeLoadMonitor()
self.remote_dispatcher: Optional[Callable[[Mapping[str, Any], Mapping[str, Any]], Mapping[str, Any]]] = remote_dispatcher
# Operators may enable fixture executors through cfg['compute']
# ['mock_mode'] (for demos on a box with no LLM backend); tests pass
# a registry directly. Neither happens by default.
if registry is None:
node_dir = getattr(node, 'dir', None)
registry = ExecutorRegistry(blob_store=self.blobs,
mock_mode=self.settings().mock_mode,
media_cache_dir=os.path.join(node_dir, 'media-cache') if node_dir else None)
self.registry: ExecutorRegistry = registry
self.compute_wire = getattr(node, 'compute_wire', None) or ComputeWire(node=node, ks=self.ks)
[docs] def handle(self, inbound: Any) -> Optional[tuple[Any, ...]]:
return self.compute_wire.handle(inbound)
[docs] def index_offer(self, offer: NostrEvent) -> bool:
probs = validate_offer(offer)
if probs:
raise ComputeError(f"invalid offer event: {', '.join(probs)}")
self.order_book_offers[offer['id']] = offer
return True
[docs] def index_ask(self, ask: NostrEvent) -> bool:
if ask.get('kind') != ASK_KIND:
raise ComputeError("not a kind 30810 ask")
ask_id = parse_ask(ask)['ask_id']
self.order_book_asks[ask_id or ask['id']] = ask
return True
[docs] def find_offers_for_ask(
self,
ask: NostrEvent | Mapping[str, Any] | str,
trust_weights: Optional[Mapping[str, float]] = None,
) -> list[Mapping[str, Any]]:
ask_ev = self.order_book_asks.get(ask) if isinstance(ask, str) else ask
if not ask_ev:
if isinstance(ask, str) and ask in self.asks:
ask_ev = self.asks[ask]
else:
raise ComputeError(f"ask not found: {ask}")
return match_ask_to_offers(ask_ev, list(self.order_book_offers.values()), now=self.clock(), trust_weights=trust_weights)
[docs] def find_asks_for_offer(
self,
offer: NostrEvent | Mapping[str, Any] | str,
) -> list[Mapping[str, Any]]:
off_ev = self.order_book_offers.get(offer) if isinstance(offer, str) else offer
if not off_ev:
if isinstance(offer, str):
for o in self.offers:
if o.get('id') == offer:
off_ev = o
break
if not off_ev:
raise ComputeError(f"offer not found: {offer}")
return match_offer_to_asks(off_ev, list(self.order_book_asks.values()), now=self.clock())
[docs] def state(self) -> Mapping[str, Any]:
snap = self.load_monitor.snapshot()
return {
'offers_published': len(self.offers),
'active_asks': len(self.asks),
'indexed_offers': len(self.order_book_offers),
'indexed_asks': len(self.order_book_asks),
'receipts_held': len(self.receipts_held),
'jobs_executed': self.jobs_executed,
'jobs_reflected': snap['jobs_reflected'],
'total_fuel_used': snap['total_fuel_used'],
'cpu_utilization_ratio': snap['cpu_utilization_ratio'],
'cpu_load_pct': snap['cpu_load_pct'],
'active_jobs': snap['active_jobs'],
'max_concurrent_jobs': snap['max_concurrent_jobs'],
'free_slots': snap['free_slots'],
'available_engines': self.registry.runnable_engines(),
'registered_engines': list(self.registry.executors.keys()),
'active_sessions': len(self.compute_wire.sessions),
}
[docs] def trouble(self) -> list[str]:
out: list[str] = []
recent_errs = self.refusals.within(self.TROUBLE_WINDOW, self.clock(), kind='compute')
if recent_errs:
out.append('%d compute failure(s) in the last 30 minutes' % len(recent_errs))
return out
[docs] def receipts_cmd(self) -> list[Mapping[str, Any]]:
return list(self.receipts_held)
[docs] def book_cmd(self) -> Mapping[str, Any]:
return {
'offers_count': len(self.order_book_offers),
'asks_count': len(self.order_book_asks),
'offers': [parse_offer(o) for o in self.order_book_offers.values()],
'asks': [parse_ask(a) for a in self.order_book_asks.values()],
}
[docs] def index_offer_cmd(self, args: Mapping[str, Any]) -> bool:
return self.index_offer(args['offer'])
[docs] def index_ask_cmd(self, args: Mapping[str, Any]) -> bool:
return self.index_ask(args['ask'])
[docs] def match_cmd(self, args: Mapping[str, Any]) -> list[Any]:
if args.get('ask') is not None:
return self.find_offers_for_ask(args['ask'])
if args.get('offer') is not None:
return self.find_asks_for_offer(args['offer'])
raise ComputeError("match requires 'ask' or 'offer' argument")
def _require_keystore(self) -> Any:
"""The node keystore, or a ComputeError when this subsystem is detached."""
if self.ks is None:
raise ComputeError('no node keystore available')
return self.ks
def _cmd_load(self) -> Mapping[str, Any]:
return self.load_monitor.snapshot()
def _cmd_offer(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
ks = self._require_keystore()
engine = args['engine']
msat = str(args['msat_per_mfuel'])
runtimes = None
executor_obj = self.registry.get(engine)
if executor_obj and executor_obj.wasi_profile_digest:
runtimes = [[engine, '1.0', executor_obj.wasi_profile_digest]]
limits = self.load_monitor.offer_limits()
now_ts = self.clock()
ev = build_offer(
ks,
executors=[[engine, 'msat_per_mfuel', msat]],
limits=limits,
runtimes=runtimes,
now=now_ts,
expiration=now_ts + 86400,
)
self.offers.append(ev)
self.order_book_offers[ev['id']] = ev
self.say('offer_published', engine=engine)
return {'id': ev['id'], 'kind': ev['kind'], 'tags': ev['tags']}
def _cmd_ask(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
ks = self._require_keystore()
ask_id = args['id']
job = args['job']
bid = int(args['bid'])
now_ts = self.clock()
ev = build_ask(
buyer_sk=ks.node_sk,
ask_id=ask_id,
job_payload=job,
executor=args.get('executor', str(job.get('executor', 'wasm'))),
bid_currency='msat_per_mfuel',
bid_price=bid,
deadline=now_ts + 3600,
fuel_max=int(job.get('fuel_max', 2500000000)),
verify_mode=args.get('verify', 'none'),
now=now_ts,
expiration=now_ts + 7200,
)
self.asks[ask_id] = ev
self.order_book_asks[ask_id] = ev
self.say('ask_published', ask_id=str(ask_id)[:16], bid=bid)
return {'id': ev['id'], 'kind': ev['kind'], 'commit': ev['tags'][1][1]}
def _cmd_run(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
ks = self._require_keystore()
job = dict(args['job'])
if 'blob_store' not in job and self.blobs is not None:
job['blob_store'] = self.blobs
buyer_pk = args['buyer']
with self.load_monitor.job_slot():
try:
res = run_job(job, registry=self.registry)
except Exception as e:
self.refusals.refuse('compute', 'execution failed', detail={'error': str(e)})
raise ComputeError('job execution failed: %s' % (e,))
self.load_monitor.record_fuel(int(res.get('fuel_used') or 0))
paid_msat = int(job.get('paid_msat') or args.get('paid_msat') or 0)
payment_hash = job.get('payment_hash') or args.get('payment_hash') or ('0' * 64)
blinding = crypto.sha256(os.urandom(32))
receipt_payload = {
'ask': job.get('ask', '0' * 64),
'module': (job.get('module') or {}).get('sha256', '0' * 64),
'input': (job.get('input') or {}).get('sha256', '0' * 64),
'result': res['result_hash'],
'result_cid': res.get('result_cid'),
'fuel_used': res['fuel_used'],
'paid_msat': paid_msat,
'payment_hash': payment_hash,
'executor': {
'engine': res.get('engine', 'wasm'),
'wasi_profile_digest': res.get('wasi_profile_digest', '0' * 64),
},
'verify': job.get('verify', {'mode': 'none'}),
'blinding': blinding,
}
if args.get('preimage'):
receipt_payload['preimage'] = args['preimage']
ev = build_receipt(ks.node_sk, buyer_pk, receipt_payload, blinding)
self.receipts_held.append({'receipt': ev, 'payload': receipt_payload, 'buyer': str(buyer_pk)})
self.jobs_executed += 1
self.say(
'job_executed',
job_id=str(job.get('ask', 'job'))[:16],
buyer=str(buyer_pk),
fuel=int(res['fuel_used']),
)
return {
'result_hash': res['result_hash'],
'result_cid': res.get('result_cid'),
'raw_output': res.get('raw_output'),
'counters': res.get('counters', {}),
'receipt_id': ev['id'],
'receipt': ev,
'fuel_used': res['fuel_used'],
'paid_msat': paid_msat,
'payment_hash': payment_hash,
'worker_pubkey': ks.node_pk,
'reflected': False,
}
[docs] def submit_or_reflect(
self,
job: Mapping[str, Any],
buyer_pk: str,
*,
bid_msat: int = 0,
force_reflect: bool = False,
remote_dispatcher: Optional[Callable[[Mapping[str, Any], Mapping[str, Any]], Mapping[str, Any]]] = None,
) -> Mapping[str, Any]:
"""Execute a job locally if CPU/slot headroom permits, or reflect it to the least-loaded mesh peer."""
ks = self._require_keystore()
should_reflect, reason = self.load_monitor.should_reflect_job()
if force_reflect:
should_reflect, reason = True, 'forced_reflection'
if should_reflect:
ask_id = str(job.get('ask') or crypto.sha256(os.urandom(32)))
exec_engine = str(job.get('executor') or job.get('engine') or 'wasm')
v_obj = job.get('verify')
v_mode = str(v_obj.get('mode', 'delivery')) if isinstance(v_obj, Mapping) else 'delivery'
if v_mode not in ('none', 'delivery', 'sample', 'dual', 'proof', 'schema-bounds', 'tee-attestation', 'quasi-deterministic'):
v_mode = 'delivery'
job_with_ask = dict(job, ask=ask_id)
self._cmd_ask({
'id': ask_id,
'job': job_with_ask,
'bid': bid_msat,
'executor': exec_engine,
'verify': v_mode,
})
candidates = [
m for m in self.find_offers_for_ask(ask_id)
if m.get('seller_pubkey') != ks.node_pk
]
if candidates:
best = candidates[0]
peer_pk = str(best['seller_pubkey'])
self.load_monitor.record_reflection()
self.say('job_reflected', job_id=ask_id[:16], peer=peer_pk[:16], reason=reason)
if hasattr(self.compute_wire, 'open_buyer_session'):
self.compute_wire.open_buyer_session(ask_id, peer_pk, job_with_ask)
dispatcher = remote_dispatcher or self.remote_dispatcher
if callable(dispatcher):
remote_res = dict(dispatcher(best, job_with_ask))
remote_res.setdefault('reflected', True)
remote_res.setdefault('reflect_reason', reason)
remote_res.setdefault('worker_pubkey', peer_pk)
remote_res.setdefault('offer_id', best['offer_id'])
remote_res.setdefault('ask_id', ask_id)
return remote_res
return {
'reflected': True,
'reflect_reason': reason,
'ask_id': ask_id,
'offer_id': best['offer_id'],
'worker_pubkey': peer_pk,
'candidate': best,
'candidates': candidates,
}
self.asks.pop(ask_id, None)
self.order_book_asks.pop(ask_id, None)
res = dict(self._cmd_run({'job': job, 'buyer': buyer_pk}))
res['reflected'] = False
res['reflect_reason'] = reason
return res
def _cmd_submit(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
return self.submit_or_reflect(
args['job'],
args['buyer'],
bid_msat=int(args.get('bid') or 0),
force_reflect=bool(args.get('force_reflect', False)),
)
def _cmd_models(self) -> Mapping[str, Any]:
llm_exec = self.registry.get('llm')
router = getattr(self.registry, 'provider_router', None)
active_prov = router.get_provider() if router is not None else None
default_m = (
active_prov.default_model
if active_prov is not None
else getattr(llm_exec, 'default_model', 'meta-llama/Llama-3.3-70B-Instruct')
)
endpoint = (
active_prov.endpoint
if active_prov is not None
else getattr(llm_exec, 'endpoint', 'http://127.0.0.1:8000/v1')
)
avail: list[str] = [
default_m,
'deepseek-ai/DeepSeek-R1',
'meta-llama/Llama-3.1-8B-Instruct',
]
if router is not None:
for p in router.list_public_providers():
for m in p.get('models', []):
if m not in avail:
avail.append(m)
return {
'default_model': default_m,
'endpoint': endpoint,
'default_provider': router.default_provider if router is not None else 'vllm',
'providers': router.list_public_providers() if router is not None else [],
'available_models': avail,
'features': ['streaming', 'tool-calling', 'grammar-json', 'schema-bounds', 'multi-provider'],
}
def _cmd_link_provider(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
from .executors.providers import LinkedProviderConfig
cfg = LinkedProviderConfig.from_preset(
str(args.get('provider_id') or 'openrouter'),
endpoint=args.get('endpoint'),
default_model=args.get('default_model'),
api_key=args.get('api_key'),
cashu_token=args.get('cashu_token'),
rate_msat_per_token=args.get('rate_msat_per_token'),
enabled=bool(args.get('enabled', True)),
)
self.registry.configure_llm_provider(cfg, set_default=bool(args.get('set_default', True)))
return cfg.to_public_dict()
def _cmd_stream(self, args: Mapping[str, Any]) -> Mapping[str, Any]:
prompt = args['prompt']
model = args.get('model')
job = {
'executor': 'llm',
'prompt': prompt,
'stream': True,
}
if model:
job['model'] = model
with self.load_monitor.job_slot():
try:
res = run_job(job, registry=self.registry)
except Exception as e:
self.refusals.refuse('compute', 'stream execution failed', detail={'error': str(e)})
raise ComputeError('stream execution failed: %s' % (e,))
self.load_monitor.record_fuel(int(res.get('fuel_used') or 0))
raw = res['raw_output']
out_text = raw.decode('utf-8', errors='replace') if isinstance(raw, bytes) else str(raw)
return {
'model': model or getattr(self.registry.get('llm'), 'default_model', 'meta-llama/Llama-3.3-70B-Instruct'),
'output': out_text,
'result_hash': res['result_hash'],
'fuel_used': res['fuel_used'],
'counters': res.get('counters', {}),
'verification_mode': res.get('verification_mode', 'delivery'),
}