Source code for fleetmesh_node.planes.services.compute.executors.nondet

"""Non-deterministic executors: LLM inference, Bacalhau jobs and agent runs.

These call out over HTTP (or run in mock mode) and are settled on evidence
rather than reproduction; see docs/non-deterministic-compute.md.
"""
from __future__ import annotations

import json
import time
from typing import Any, Mapping, Optional

from fleetmesh_core.crypto import crypto
from fleetmesh_node.transport import netfetch
from ..common import ComputeError, LLM_MAX_RESPONSE_BYTES
from ..sandbox import ComputeLimits
from .base import ComputeExecutionResult, ComputeExecutor, ComputeQuote
from .providers import (
    PROVIDER_PRESETS,
    LinkedProviderConfig,
    OperatorProviderRouter,
    ProviderPreset,
)

[docs]class LlmExecutor(ComputeExecutor): """Non-deterministic AI/LLM inference driver with grammar-guided decoding. Supports OpenAI-compatible and Anthropic HTTP endpoints (OpenRouter, Claude, OpenAI, Gemini, Copilot, Routstr, PayPerQuery, vLLM, Ollama, llama-server) with CFG grammar constraints and fallback offline simulation. """ engine_id: str = 'llm' wasi_profile_digest: str = ''
[docs] def __init__( self, endpoint: str = 'http://127.0.0.1:8000/v1', default_model: str = 'meta-llama/Llama-3.3-70B-Instruct', timeout_seconds: float = 30.0, mock_mode: bool = False, provider_router: Optional[OperatorProviderRouter] = None, api_key: str = '', cashu_token: str = '', provider_id: str = 'vllm', rate_msat_per_token: int = 12, ) -> None: self.endpoint = endpoint.rstrip('/') self.default_model = default_model self.timeout_seconds = timeout_seconds self.mock_mode = mock_mode self.api_key = api_key self.cashu_token = cashu_token self.provider_id = provider_id self.rate_msat_per_token = int(rate_msat_per_token) self.provider_router: OperatorProviderRouter = provider_router or OperatorProviderRouter()
def _resolve_provider(self, job_payload: Optional[Mapping[str, Any]] = None) -> LinkedProviderConfig: if job_payload is not None: resolved = self.provider_router.resolve_for_job(job_payload) if resolved is not None: return resolved else: resolved = self.provider_router.get_provider() if resolved is not None: return resolved return LinkedProviderConfig( provider_id=self.provider_id, label=self.provider_id.upper(), endpoint=self.endpoint, default_model=self.default_model, models=(self.default_model,), api_key=self.api_key, cashu_token=self.cashu_token, auth_scheme='bearer' if self.api_key else ('cashu' if self.cashu_token else 'none'), wire_format='anthropic_messages' if self.provider_id in ('claude', 'anthropic') else 'openai', rate_msat_per_token=self.rate_msat_per_token, enabled=True, )
[docs] def is_live_runnable(self) -> bool: """Check whether LLM inference can execute on this node.""" if self.mock_mode: return True if self.provider_router.runnable_providers(): return True return bool(self.endpoint and self.endpoint.startswith(('http://', 'https://')))
[docs] def quote( self, module_spec: Mapping[str, Any], input_spec: Mapping[str, Any], limits: ComputeLimits, ) -> ComputeQuote: merged_spec = dict(module_spec) merged_spec.update(input_spec) provider = self._resolve_provider(merged_spec) rate = provider.rate_msat_per_token if provider else self.rate_msat_per_token max_tokens = int(input_spec.get('max_tokens', limits.max_tokens or 2048)) prompt_est = len(str(input_spec.get('prompt', ''))) // 4 fuel_est = (prompt_est + max_tokens) * rate return ComputeQuote( fuel_estimate=min(fuel_est, limits.max_fuel), rate_unit='msat_per_token', rate_amount=rate, engine_id=self.engine_id, )
[docs] def execute( self, job_payload: Mapping[str, Any], fuel_cap: int, ) -> ComputeExecutionResult: provider = self._resolve_provider(job_payload) model = job_payload.get('model') or provider.default_model or self.default_model prompt = job_payload.get('prompt') messages = job_payload.get('messages') sampling = job_payload.get('sampling') or {} grammar = job_payload.get('grammar') or {} stream = bool(job_payload.get('stream', False)) # The backend endpoint is operator configuration only. An earlier # version let ``job_payload['endpoint']`` override it, which handed a # requester a POST-anywhere primitive against the node's own network # (the default endpoint is loopback, so allow_private must be on). verif_mode = self.verification_mode(job_payload) if not messages and prompt is not None: messages = [{'role': 'user', 'content': str(prompt)}] elif not messages: messages = [{'role': 'user', 'content': ''}] max_tokens = int(sampling.get('max_tokens', 512)) temperature = float(sampling.get('temperature', 0.7)) top_p = float(sampling.get('top_p', 1.0)) seed = sampling.get('seed') url, req_body = provider.format_request( messages=messages, model=model, max_tokens=max_tokens, temperature=temperature, top_p=top_p, seed=int(seed) if seed is not None else None, grammar=grammar, stream=stream, ) raw_output_text: str = '' prompt_tokens = max(1, sum(len(str(m.get('content', ''))) // 4 for m in messages)) completion_tokens = 0 gpu_seconds = 0.05 if self.mock_mode: # Deterministic stand-in for tests and demos. It is only reachable # when the operator constructed the executor with mock_mode=True; # a live node never falls back to it. if '_mock_completion' in job_payload and job_payload['_mock_completion'] is not None: raw_output_text = str(job_payload['_mock_completion']) elif grammar and (grammar.get('type') == 'json_schema' or 'json_schema' in grammar or grammar.get('response_format') == 'json_object'): raw_output_text = json.dumps({ 'status': 'completed', 'model': model, 'verified': True, 'result': 'synthesized-conforming-payload', }, sort_keys=True) else: user_msg = messages[-1].get('content', '') raw_output_text = f"Inference completion for [{model}]: {user_msg[:64]}" completion_tokens = max(1, len(raw_output_text) // 4) gpu_seconds = 0.025 else: # A backend failure is a failed job, full stop. data_bytes = json.dumps(req_body).encode('utf-8') headers = provider.build_headers( per_query_cashu_token=job_payload.get('_operator_cashu_token'), ) start_t = time.time() try: body, _ = netfetch.fetch( url, max_bytes=LLM_MAX_RESPONSE_BYTES, timeout=self.timeout_seconds, method='POST', data=data_bytes, headers=headers, # The endpoint is operator-configured (loopback vLLM/Ollama or # operator-linked cloud/eCash provider), never requester-chosen. allow_private=True, ) resp_data = json.loads(body.decode('utf-8')) except netfetch.FetchError as e: raise ComputeError(f"LLM backend request failed: {e}") except (ValueError, UnicodeDecodeError) as e: raise ComputeError(f"LLM backend returned malformed JSON: {e}") if not isinstance(resp_data, dict): raise ComputeError("LLM backend returned a non-object response") try: raw_output_text, prompt_tokens, completion_tokens = provider.parse_response( resp_data, default_prompt_tokens=prompt_tokens, ) except ValueError as e: raise ComputeError(str(e)) gpu_seconds = round(time.time() - start_t, 3) total_tokens = prompt_tokens + completion_tokens rate_msat = provider.rate_msat_per_token or 12 fuel_used = total_tokens * rate_msat if fuel_used > fuel_cap: raise ComputeError(f"LLM generation exceeded fuel cap ({fuel_used} > {fuel_cap})") on_token = job_payload.get('_on_token') if callable(on_token) and raw_output_text: # Emit token chunks to the streaming callback chunk_size = 18 for idx in range(0, len(raw_output_text), chunk_size): try: on_token(raw_output_text[idx:idx + chunk_size]) except Exception: break raw_bytes = raw_output_text.encode('utf-8') result_hash = crypto.sha256(raw_bytes) counters = { 'prompt_tokens': prompt_tokens, 'completion_tokens': completion_tokens, 'total_tokens': total_tokens, 'gpu_seconds': gpu_seconds, 'rate_msat_per_token': rate_msat, } return ComputeExecutionResult( result_hash=result_hash, raw_output=raw_bytes, fuel_used=fuel_used, engine_id=self.engine_id, wasi_profile_digest=f"llm:{model}", counters=counters, verification_mode=verif_mode, )
from .bacalhau import BacalhauExecutor from .agent import ( AgentExecutor, AgentExecutionContext, AgentToolProtocol, AgentToolResult, AgentDecision, AgentBackendProtocol, CalculatorTool, Sha256Tool, RegexSearchTool, FetchBlobTool, SubAskTool, FinishTool, ScriptedAgentBackend, LlmAgentBackend, ) __all__ = [ 'LlmExecutor', 'BacalhauExecutor', 'AgentExecutor', 'AgentExecutionContext', 'AgentToolProtocol', 'AgentToolResult', 'AgentDecision', 'AgentBackendProtocol', 'CalculatorTool', 'Sha256Tool', 'RegexSearchTool', 'FetchBlobTool', 'SubAskTool', 'FinishTool', 'ScriptedAgentBackend', 'LlmAgentBackend', 'PROVIDER_PRESETS', 'ProviderPreset', 'LinkedProviderConfig', 'OperatorProviderRouter', ]