railtracks.llm

 1from . import retries
 2from ._exceptions import (
 3    ProviderAuthenticationError,
 4    ProviderError,
 5    ProviderRateLimitError,
 6    ProviderTimeoutError,
 7    RetryError,
 8)
 9from .content import ToolCall, ToolCalls, ToolResponse
10from .history import MessageHistory
11from .message import AssistantMessage, Message, SystemMessage, ToolMessage, UserMessage
12from .model import ModelBase
13from .models import (
14    AnthropicLLM,
15    AppleFMLLM,
16    AzureAILLM,
17    EchoLLM,
18    GeminiLLM,
19    HuggingFaceLLM,
20    OllamaLLM,
21    OpenAICompatibleProvider,
22    OpenAILLM,
23    PortKeyLLM,
24)
25from .models._model_exception_base import (
26    AttachmentNotSupportedError,
27    FunctionCallingNotSupportedError,
28    ModelError,
29    ModelNotFoundError,
30    MutuallyExclusiveHyperparametersError,
31    UnsupportedHyperparameterError,
32)
33from .providers import ModelProvider
34from .response import Response
35from .tools import (
36    ArrayParameter,
37    ObjectParameter,
38    Parameter,
39    RefParameter,
40    Tool,
41    ToolCreationError,
42    UnionParameter,
43)
44
45__all__ = [
46    "ModelBase",
47    "ProviderError",
48    "ProviderTimeoutError",
49    "ProviderRateLimitError",
50    "ProviderAuthenticationError",
51    "RetryError",
52    "ToolCreationError",
53    "ModelError",
54    "ModelNotFoundError",
55    "FunctionCallingNotSupportedError",
56    "AttachmentNotSupportedError",
57    "UnsupportedHyperparameterError",
58    "MutuallyExclusiveHyperparametersError",
59    "ToolCall",
60    "ToolCalls",
61    "ToolResponse",
62    "UserMessage",
63    "SystemMessage",
64    "AssistantMessage",
65    "Message",
66    "ToolMessage",
67    "MessageHistory",
68    "ModelProvider",
69    "Tool",
70    "AnthropicLLM",
71    "AppleFMLLM",
72    "AzureAILLM",
73    "EchoLLM",
74    "HuggingFaceLLM",
75    "OpenAILLM",
76    "GeminiLLM",
77    "OllamaLLM",
78    "PortKeyLLM",
79    "OpenAICompatibleProvider",
80    # Parameter types
81    "Parameter",
82    "UnionParameter",
83    "ArrayParameter",
84    "ObjectParameter",
85    "RefParameter",
86    "retries",
87    "Response",
88]
class ModelBase(abc.ABC):
 25class ModelBase(ABC):
 26    """
 27    A simple base that represents the behavior of a model that can be used for chat, structured interactions, and streaming.
 28    """
 29
 30    def __init__(
 31        self,
 32        retry_approach: RetryApproach | None = None,
 33    ):
 34        self.retry_approach = retry_approach
 35        self.id = str(uuid4())
 36
 37    @abstractmethod
 38    def model_name(self) -> str:
 39        """
 40        Returns the name of the model being used.
 41
 42        It can be treated as unique identifier for the model when paired with the `model_type`.
 43        """
 44        pass
 45
 46    @abstractmethod
 47    def model_provider(self) -> ModelProvider:
 48        """The name of the provider of this model (The Company that owns the model)"""
 49        pass
 50
 51    @classmethod
 52    @abstractmethod
 53    def model_gateway(cls) -> ModelProvider:
 54        """
 55        Gets the API distrubutor of the model. Note nessecarily the same as the model itself.
 56
 57        E.g. if you are calling openai LLM through Azure AI foundry
 58        """
 59        pass
 60
 61    def chat(self, messages: MessageHistory) -> Response:
 62        """Chat with the model using the provided messages."""
 63        return self._chat(messages)
 64
 65    async def achat(self, messages: MessageHistory) -> Response:
 66        """Asynchronous chat with the model using the provided messages."""
 67        return await self._achat(messages)
 68
 69    def structured(self, messages: MessageHistory, schema: Type[BaseModel]) -> Response:
 70        """Structured interaction with the model using the provided messages and output_schema."""
 71        return self._structured(messages, schema)
 72
 73    async def astructured(
 74        self, messages: MessageHistory, schema: Type[BaseModel]
 75    ) -> Response:
 76        """Asynchronous structured interaction with the model using the provided messages and output_schema."""
 77        return await self._astructured(messages, schema)
 78
 79    def chat_with_tools(self, messages: MessageHistory, tools: List[Tool]) -> Response:
 80        """Chat with the model using the provided messages and tools."""
 81        return self._chat_with_tools(messages, tools)
 82
 83    async def achat_with_tools(
 84        self, messages: MessageHistory, tools: List[Tool]
 85    ) -> Response:
 86        """Asynchronous chat with the model using the provided messages and tools."""
 87        return await self._achat_with_tools(messages, tools)
 88
 89    # ================ START Streaming (per-call) LLM calls ===============
 90    # These methods request a streamed response for a single call. They are the model-level
 91    # building blocks of railtracks streaming (see `rt.astream` at the framework level).
 92
 93    async def astream_chat(
 94        self, messages: MessageHistory
 95    ) -> AsyncGenerator[str | Response, None]:
 96        """
 97        Chat with the model, streaming the response.
 98
 99        Returns an async generator that yields `str` token chunks as they arrive, followed by a
100        single final `Response` object containing the complete message (and usage info).
101
102        ```python
103        async for item in model.astream_chat(MessageHistory([UserMessage("hi")])):
104            if isinstance(item, str):
105                ...  # token chunk
106            else:
107                final = item  # the terminal Response
108        ```
109
110        Args:
111            messages: The conversation so far, as a `MessageHistory`.
112
113        Yields:
114            str | Response: `str` token chunks, then one final complete `Response`.
115        """
116        async for item in self._astream_chat(messages):
117            yield item
118
119    async def astream_chat_with_tools(
120        self, messages: MessageHistory, tools: List[Tool]
121    ) -> AsyncGenerator[str | Response, None]:
122        """
123        Chat with the model using tools, streaming the response.
124
125        Yields `str` content chunks as they arrive, followed by a single final `Response`. The
126        final `Response` contains either the complete assistant text or the requested tool
127        calls (tool-call deltas are accumulated internally and are not yielded as chunks).
128
129        Args:
130            messages: The conversation so far, as a `MessageHistory`.
131            tools: The tools to make available to the model.
132
133        Yields:
134            str | Response: `str` content chunks, then one final complete `Response`.
135        """
136        async for item in self._astream_chat_with_tools(messages, tools):
137            yield item
138
139    async def astream_structured(
140        self, messages: MessageHistory, schema: Type[BaseModel]
141    ) -> AsyncGenerator[str | Response, None]:
142        """
143        Structured interaction with the model, streaming the response.
144
145        Yields the raw (JSON) `str` chunks as they arrive, followed by a single final
146        `Response` whose message content is the parsed `schema` instance.
147
148        Note the chunks are unvalidated JSON fragments; validation only happens once the stream
149        completes, so a schema mismatch surfaces at the end of the stream.
150
151        Args:
152            messages: The conversation so far, as a `MessageHistory`.
153            schema: The pydantic model the response must conform to.
154
155        Yields:
156            str | Response: raw JSON `str` chunks, then one final complete `Response`.
157        """
158        async for item in self._astream_structured(messages, schema):
159            yield item
160
161    def supports_streamed_tool_calling(self) -> bool:
162        """Whether `astream_chat_with_tools` can stream on this model.
163
164        A model that routes tool calls through an API without incremental support should
165        return False; `rt.astream` then runs the call buffered (with a warning) rather than
166        failing, so the final result is unaffected and only the incremental chunks are lost.
167
168        Returns:
169            bool: True by default — subclasses narrow this where they can tell.
170        """
171        return True
172
173    # ================ END Streaming (per-call) LLM calls ===============
174
175    @abstractmethod
176    def _chat(self, messages: MessageHistory) -> Response:
177        pass
178
179    @abstractmethod
180    def _structured(
181        self, messages: MessageHistory, schema: Type[BaseModel]
182    ) -> Response:
183        pass
184
185    @abstractmethod
186    def _chat_with_tools(self, messages: MessageHistory, tools: List[Tool]) -> Response:
187        pass
188
189    # Note: the _astream_* methods are deliberately NOT abstract so that existing ModelBase
190    # subclasses keep working; subclasses that support streaming should override them with
191    # async generator implementations yielding `str` chunks followed by a final `Response`.
192
193    def _astream_chat(
194        self, messages: MessageHistory
195    ) -> AsyncGenerator[str | Response, None]:
196        raise NotImplementedError(
197            f"{type(self).__name__} does not support streamed chat calls."
198        )
199
200    def _astream_chat_with_tools(
201        self, messages: MessageHistory, tools: List[Tool]
202    ) -> AsyncGenerator[str | Response, None]:
203        raise NotImplementedError(
204            f"{type(self).__name__} does not support streamed tool-calling calls."
205        )
206
207    def _astream_structured(
208        self, messages: MessageHistory, schema: Type[BaseModel]
209    ) -> AsyncGenerator[str | Response, None]:
210        raise NotImplementedError(
211            f"{type(self).__name__} does not support streamed structured calls."
212        )
213
214    @abstractmethod
215    async def _achat(self, messages: MessageHistory) -> Response:
216        pass
217
218    @abstractmethod
219    async def _astructured(
220        self,
221        messages: MessageHistory,
222        schema: Type[BaseModel],
223    ) -> Response:
224        pass
225
226    @abstractmethod
227    async def _achat_with_tools(
228        self, messages: MessageHistory, tools: List[Tool]
229    ) -> Response:
230        pass

A simple base that represents the behavior of a model that can be used for chat, structured interactions, and streaming.

retry_approach
id
@abstractmethod
def model_name(self) -> str:
37    @abstractmethod
38    def model_name(self) -> str:
39        """
40        Returns the name of the model being used.
41
42        It can be treated as unique identifier for the model when paired with the `model_type`.
43        """
44        pass

Returns the name of the model being used.

It can be treated as unique identifier for the model when paired with the model_type.

@abstractmethod
def model_provider(self) -> ModelProvider:
46    @abstractmethod
47    def model_provider(self) -> ModelProvider:
48        """The name of the provider of this model (The Company that owns the model)"""
49        pass

The name of the provider of this model (The Company that owns the model)

@classmethod
@abstractmethod
def model_gateway(cls) -> ModelProvider:
51    @classmethod
52    @abstractmethod
53    def model_gateway(cls) -> ModelProvider:
54        """
55        Gets the API distrubutor of the model. Note nessecarily the same as the model itself.
56
57        E.g. if you are calling openai LLM through Azure AI foundry
58        """
59        pass

Gets the API distrubutor of the model. Note nessecarily the same as the model itself.

E.g. if you are calling openai LLM through Azure AI foundry

def chat( self, messages: MessageHistory) -> Response:
61    def chat(self, messages: MessageHistory) -> Response:
62        """Chat with the model using the provided messages."""
63        return self._chat(messages)

Chat with the model using the provided messages.

async def achat( self, messages: MessageHistory) -> Response:
65    async def achat(self, messages: MessageHistory) -> Response:
66        """Asynchronous chat with the model using the provided messages."""
67        return await self._achat(messages)

Asynchronous chat with the model using the provided messages.

def structured( self, messages: MessageHistory, schema: Type[pydantic.main.BaseModel]) -> Response:
69    def structured(self, messages: MessageHistory, schema: Type[BaseModel]) -> Response:
70        """Structured interaction with the model using the provided messages and output_schema."""
71        return self._structured(messages, schema)

Structured interaction with the model using the provided messages and output_schema.

async def astructured( self, messages: MessageHistory, schema: Type[pydantic.main.BaseModel]) -> Response:
73    async def astructured(
74        self, messages: MessageHistory, schema: Type[BaseModel]
75    ) -> Response:
76        """Asynchronous structured interaction with the model using the provided messages and output_schema."""
77        return await self._astructured(messages, schema)

Asynchronous structured interaction with the model using the provided messages and output_schema.

def chat_with_tools( self, messages: MessageHistory, tools: List[Tool]) -> Response:
79    def chat_with_tools(self, messages: MessageHistory, tools: List[Tool]) -> Response:
80        """Chat with the model using the provided messages and tools."""
81        return self._chat_with_tools(messages, tools)

Chat with the model using the provided messages and tools.

async def achat_with_tools( self, messages: MessageHistory, tools: List[Tool]) -> Response:
83    async def achat_with_tools(
84        self, messages: MessageHistory, tools: List[Tool]
85    ) -> Response:
86        """Asynchronous chat with the model using the provided messages and tools."""
87        return await self._achat_with_tools(messages, tools)

Asynchronous chat with the model using the provided messages and tools.

async def astream_chat( self, messages: MessageHistory) -> AsyncGenerator[str | Response, NoneType]:
 93    async def astream_chat(
 94        self, messages: MessageHistory
 95    ) -> AsyncGenerator[str | Response, None]:
 96        """
 97        Chat with the model, streaming the response.
 98
 99        Returns an async generator that yields `str` token chunks as they arrive, followed by a
100        single final `Response` object containing the complete message (and usage info).
101
102        ```python
103        async for item in model.astream_chat(MessageHistory([UserMessage("hi")])):
104            if isinstance(item, str):
105                ...  # token chunk
106            else:
107                final = item  # the terminal Response
108        ```
109
110        Args:
111            messages: The conversation so far, as a `MessageHistory`.
112
113        Yields:
114            str | Response: `str` token chunks, then one final complete `Response`.
115        """
116        async for item in self._astream_chat(messages):
117            yield item

Chat with the model, streaming the response.

Returns an async generator that yields str token chunks as they arrive, followed by a single final Response object containing the complete message (and usage info).

async for item in model.astream_chat(MessageHistory([UserMessage("hi")])):
    if isinstance(item, str):
        ...  # token chunk
    else:
        final = item  # the terminal Response
Arguments:
Yields:

str | Response: str token chunks, then one final complete Response.

async def astream_chat_with_tools( self, messages: MessageHistory, tools: List[Tool]) -> AsyncGenerator[str | Response, NoneType]:
119    async def astream_chat_with_tools(
120        self, messages: MessageHistory, tools: List[Tool]
121    ) -> AsyncGenerator[str | Response, None]:
122        """
123        Chat with the model using tools, streaming the response.
124
125        Yields `str` content chunks as they arrive, followed by a single final `Response`. The
126        final `Response` contains either the complete assistant text or the requested tool
127        calls (tool-call deltas are accumulated internally and are not yielded as chunks).
128
129        Args:
130            messages: The conversation so far, as a `MessageHistory`.
131            tools: The tools to make available to the model.
132
133        Yields:
134            str | Response: `str` content chunks, then one final complete `Response`.
135        """
136        async for item in self._astream_chat_with_tools(messages, tools):
137            yield item

Chat with the model using tools, streaming the response.

Yields str content chunks as they arrive, followed by a single final Response. The final Response contains either the complete assistant text or the requested tool calls (tool-call deltas are accumulated internally and are not yielded as chunks).

Arguments:
  • messages: The conversation so far, as a MessageHistory.
  • tools: The tools to make available to the model.
Yields:

str | Response: str content chunks, then one final complete Response.

async def astream_structured( self, messages: MessageHistory, schema: Type[pydantic.main.BaseModel]) -> AsyncGenerator[str | Response, NoneType]:
139    async def astream_structured(
140        self, messages: MessageHistory, schema: Type[BaseModel]
141    ) -> AsyncGenerator[str | Response, None]:
142        """
143        Structured interaction with the model, streaming the response.
144
145        Yields the raw (JSON) `str` chunks as they arrive, followed by a single final
146        `Response` whose message content is the parsed `schema` instance.
147
148        Note the chunks are unvalidated JSON fragments; validation only happens once the stream
149        completes, so a schema mismatch surfaces at the end of the stream.
150
151        Args:
152            messages: The conversation so far, as a `MessageHistory`.
153            schema: The pydantic model the response must conform to.
154
155        Yields:
156            str | Response: raw JSON `str` chunks, then one final complete `Response`.
157        """
158        async for item in self._astream_structured(messages, schema):
159            yield item

Structured interaction with the model, streaming the response.

Yields the raw (JSON) str chunks as they arrive, followed by a single final Response whose message content is the parsed schema instance.

Note the chunks are unvalidated JSON fragments; validation only happens once the stream completes, so a schema mismatch surfaces at the end of the stream.

Arguments:
  • messages: The conversation so far, as a MessageHistory.
  • schema: The pydantic model the response must conform to.
Yields:

str | Response: raw JSON str chunks, then one final complete Response.

def supports_streamed_tool_calling(self) -> bool:
161    def supports_streamed_tool_calling(self) -> bool:
162        """Whether `astream_chat_with_tools` can stream on this model.
163
164        A model that routes tool calls through an API without incremental support should
165        return False; `rt.astream` then runs the call buffered (with a warning) rather than
166        failing, so the final result is unaffected and only the incremental chunks are lost.
167
168        Returns:
169            bool: True by default — subclasses narrow this where they can tell.
170        """
171        return True

Whether astream_chat_with_tools can stream on this model.

A model that routes tool calls through an API without incremental support should return False; rt.astream then runs the call buffered (with a warning) rather than failing, so the final result is unaffected and only the incremental chunks are lost.

Returns:

bool: True by default — subclasses narrow this where they can tell.

class ProviderError(railtracks.llm._exceptions._ColoredError, builtins.Exception):
30class ProviderError(_ColoredError, Exception):
31    """
32    Base class for failures that happen while talking to a model provider.
33
34    Covers the model misbehaving, an unknown model, and exhausted retries. Errors about
35    *defining* a tool are a separate root -- see
36    :class:`railtracks.llm.tools.tool.ToolCreationError`.
37
38    Raised only by direct model calls. Inside a node these surface as
39    :class:`railtracks.exceptions.LLMError`.
40    """

Base class for failures that happen while talking to a model provider.

Covers the model misbehaving, an unknown model, and exhausted retries. Errors about defining a tool are a separate root -- see railtracks.llm.tools.tool.ToolCreationError.

Raised only by direct model calls. Inside a node these surface as railtracks.exceptions.LLMError.

class ProviderTimeoutError(railtracks.llm.ProviderError):
43class ProviderTimeoutError(ProviderError):
44    """The provider did not answer in time."""

The provider did not answer in time.

class ProviderRateLimitError(railtracks.llm.ProviderError):
47class ProviderRateLimitError(ProviderError):
48    """The provider rejected the call for rate/quota reasons."""

The provider rejected the call for rate/quota reasons.

class ProviderAuthenticationError(railtracks.llm.ProviderError):
51class ProviderAuthenticationError(ProviderError):
52    """The provider rejected the credentials. Retrying will not help."""

The provider rejected the credentials. Retrying will not help.

class RetryError(railtracks.llm.ProviderError):
55class RetryError(ProviderError):
56    """
57    Raised when an error occurs during an LLM call that is being retried.
58    """
59
60    def __init__(
61        self,
62        retry_method: str,
63        message: str,
64        notes: list[str],
65        exception_list: list[Exception],
66    ):
67        full_message = (
68            f"LLM call failed after retries from {retry_method} retry: {message}"
69        )
70        self.message = message
71        self.notes = notes
72        self.exception_list = exception_list
73        super().__init__(full_message)

Raised when an error occurs during an LLM call that is being retried.

RetryError( retry_method: str, message: str, notes: list[str], exception_list: list[Exception])
60    def __init__(
61        self,
62        retry_method: str,
63        message: str,
64        notes: list[str],
65        exception_list: list[Exception],
66    ):
67        full_message = (
68            f"LLM call failed after retries from {retry_method} retry: {message}"
69        )
70        self.message = message
71        self.notes = notes
72        self.exception_list = exception_list
73        super().__init__(full_message)
message
notes
exception_list
class ToolCreationError(railtracks.llm._exceptions._ColoredError, builtins.Exception):
349class ToolCreationError(_ColoredError, Exception):
350    """Exception raised when a tool cannot be created.
351
352    A separate root from `ProviderError`: a malformed tool is a bug in the caller's
353    code, not a runtime failure of a provider, and it ends the run.
354    """
355
356    def __init__(self, message, notes=None):
357        super().__init__(message)
358        self.notes = notes or []
359
360    def __str__(self):
361        base = super().__str__()
362        if self.notes:
363            notes_str = (
364                "\n"
365                + self._color("Tips to debug:\n", self.GREEN)
366                + "\n".join(self._color(f"- {note}", self.GREEN) for note in self.notes)
367            )
368            return f"\n{self._color(base, self.RED)}{notes_str}"
369        return self._color(base, self.RED)

Exception raised when a tool cannot be created.

A separate root from ProviderError: a malformed tool is a bug in the caller's code, not a runtime failure of a provider, and it ends the run.

ToolCreationError(message, notes=None)
356    def __init__(self, message, notes=None):
357        super().__init__(message)
358        self.notes = notes or []
notes
class ModelError(railtracks.llm.ProviderError):
 6class ModelError(ProviderError):
 7    """
 8    Any Large Language Model (LLM) error.
 9    """
10
11    def __init__(
12        self,
13        reason: str,
14        message_history: MessageHistory = None,
15    ):
16        self.reason = reason
17        self.message_history = message_history
18
19        message = f"{self._color('Failure reason: ', self.BOLD_RED)}{self._color(reason, self.RED)}"
20        super().__init__(message)
21
22    def __str__(self):
23        base = super().__str__()
24        if not self.message_history:
25            return self._color(base, self.RED)
26
27        try:
28            count = len(self.message_history)
29        except TypeError:
30            count = None
31
32        summary = (
33            f"{count} message(s) redacted; "
34            "call err.format_verbose() to render or read err.message_history"
35            if count is not None
36            else "message history redacted; "
37            "call err.format_verbose() to render or read err.message_history"
38        )
39        detail = self._color("Message History: ", self.BOLD_GREEN) + self._color(
40            summary, self.GREEN
41        )
42        notes_str = "\n" + self._color("Details:\n", self.BOLD_GREEN) + f"  {detail}"
43        return f"\n{self._color(base, self.RED)}{notes_str}"
44
45    def format_verbose(self) -> str:
46        """Render the exception with the full input ``MessageHistory`` embedded."""
47        base = super().__str__()
48        if not self.message_history:
49            return self._color(base, self.RED)
50
51        mh_str = str(self.message_history)
52        indented_mh = "\n".join("    " + line for line in mh_str.splitlines())
53        detail = self._color("Message History:\n", self.BOLD_GREEN) + self._color(
54            indented_mh, self.GREEN
55        )
56        notes_str = "\n" + self._color("Details:\n", self.BOLD_GREEN) + f"  {detail}"
57        return f"\n{self._color(base, self.RED)}{notes_str}"

Any Large Language Model (LLM) error.

ModelError( reason: str, message_history: MessageHistory = None)
11    def __init__(
12        self,
13        reason: str,
14        message_history: MessageHistory = None,
15    ):
16        self.reason = reason
17        self.message_history = message_history
18
19        message = f"{self._color('Failure reason: ', self.BOLD_RED)}{self._color(reason, self.RED)}"
20        super().__init__(message)
reason
message_history
def format_verbose(self) -> str:
45    def format_verbose(self) -> str:
46        """Render the exception with the full input ``MessageHistory`` embedded."""
47        base = super().__str__()
48        if not self.message_history:
49            return self._color(base, self.RED)
50
51        mh_str = str(self.message_history)
52        indented_mh = "\n".join("    " + line for line in mh_str.splitlines())
53        detail = self._color("Message History:\n", self.BOLD_GREEN) + self._color(
54            indented_mh, self.GREEN
55        )
56        notes_str = "\n" + self._color("Details:\n", self.BOLD_GREEN) + f"  {detail}"
57        return f"\n{self._color(base, self.RED)}{notes_str}"

Render the exception with the full input MessageHistory embedded.

class ModelNotFoundError(railtracks.llm.ProviderError):
60class ModelNotFoundError(ProviderError):
61    def __init__(self, reason: str, notes: list[str] = None):
62        self.reason = reason
63        self.notes = notes or []
64        super().__init__(reason)
65
66    def __str__(self):
67        base = super().__str__()
68        if self.notes:
69            notes_str = (
70                "\n"
71                + self._color("Tips to debug:\n", self.GREEN)
72                + "\n".join(self._color(f"- {note}", self.GREEN) for note in self.notes)
73            )
74            return f"\n{self._color(base, self.RED)}{notes_str}"
75        return self._color(base, self.RED)

Base class for failures that happen while talking to a model provider.

Covers the model misbehaving, an unknown model, and exhausted retries. Errors about defining a tool are a separate root -- see railtracks.llm.tools.tool.ToolCreationError.

Raised only by direct model calls. Inside a node these surface as railtracks.exceptions.LLMError.

ModelNotFoundError(reason: str, notes: list[str] = None)
61    def __init__(self, reason: str, notes: list[str] = None):
62        self.reason = reason
63        self.notes = notes or []
64        super().__init__(reason)
reason
notes
class FunctionCallingNotSupportedError(railtracks.llm.ModelError):
78class FunctionCallingNotSupportedError(ModelError):
79    """Error raised when a model does not support function calling."""
80
81    def __init__(self, model_name: str):
82        super().__init__(
83            reason=f"Model {model_name} does not support function calling. Chat with tools is not supported."
84        )

Error raised when a model does not support function calling.

FunctionCallingNotSupportedError(model_name: str)
81    def __init__(self, model_name: str):
82        super().__init__(
83            reason=f"Model {model_name} does not support function calling. Chat with tools is not supported."
84        )
class AttachmentNotSupportedError(railtracks.llm.ModelError, builtins.ValueError):
87class AttachmentNotSupportedError(ModelError, ValueError):
88    """Error raised when a model cannot accept an attachment's modality.
89
90    Also a `ValueError`, matching the base wrapper's PDF-support rejection.
91    """
92
93    def __init__(self, model_name: str, mime_type: str | None, reason: str):
94        super().__init__(
95            reason=(
96                f"Model {model_name} does not accept "
97                f"{mime_type or 'this'} attachments: {reason}"
98            )
99        )

Error raised when a model cannot accept an attachment's modality.

Also a ValueError, matching the base wrapper's PDF-support rejection.

AttachmentNotSupportedError(model_name: str, mime_type: str | None, reason: str)
93    def __init__(self, model_name: str, mime_type: str | None, reason: str):
94        super().__init__(
95            reason=(
96                f"Model {model_name} does not accept "
97                f"{mime_type or 'this'} attachments: {reason}"
98            )
99        )
class UnsupportedHyperparameterError(railtracks.llm.ModelError):
102class UnsupportedHyperparameterError(ModelError):
103    """Error raised when a model does not support a given common LLM hyperparameter."""
104
105    def __init__(self, model_name: str, hyperparameter: str, value):
106        super().__init__(
107            reason=(
108                f"Model {model_name} does not support '{hyperparameter}' "
109                f"(got {hyperparameter}={value!r})."
110            )
111        )

Error raised when a model does not support a given common LLM hyperparameter.

UnsupportedHyperparameterError(model_name: str, hyperparameter: str, value)
105    def __init__(self, model_name: str, hyperparameter: str, value):
106        super().__init__(
107            reason=(
108                f"Model {model_name} does not support '{hyperparameter}' "
109                f"(got {hyperparameter}={value!r})."
110            )
111        )
class MutuallyExclusiveHyperparametersError(railtracks.llm.ModelError):
114class MutuallyExclusiveHyperparametersError(ModelError):
115    """Error raised when two or more common hyperparameters cannot be combined for
116    this model."""
117
118    def __init__(self, model_name: str, hyperparameters: list[str], values: dict):
119        joined = " and ".join(f"'{p}'" for p in hyperparameters)
120        super().__init__(
121            reason=(
122                f"Model {model_name} does not support specifying {joined} together "
123                f"(got {values!r}). Use only one."
124            )
125        )

Error raised when two or more common hyperparameters cannot be combined for this model.

MutuallyExclusiveHyperparametersError(model_name: str, hyperparameters: list[str], values: dict)
118    def __init__(self, model_name: str, hyperparameters: list[str], values: dict):
119        joined = " and ".join(f"'{p}'" for p in hyperparameters)
120        super().__init__(
121            reason=(
122                f"Model {model_name} does not support specifying {joined} together "
123                f"(got {values!r}). Use only one."
124            )
125        )
class ToolCall(pydantic.main.BaseModel):
22class ToolCall(BaseModel):
23    """
24    A simple model object that represents a tool call.
25
26    This simple model represents a moment when a tool is called.
27    """
28
29    identifier: str = Field(description="The identifier attatched to this tool call.")
30    name: str = Field(description="The name of the tool being called.")
31    arguments: Dict[str, Any] = Field(
32        description="The arguments provided as input to the tool."
33    )
34
35    def __str__(self):
36        return f"{self.name}({self.arguments})"

A simple model object that represents a tool call.

This simple model represents a moment when a tool is called.

identifier: str
name: str
arguments: Dict[str, Any]
model_config: ClassVar[pydantic.config.ConfigDict] = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

class ToolCalls(typing.List[railtracks.llm.content.ToolCall]):
39class ToolCalls(List[ToolCall]):
40    """
41    An assistant turn that calls tools: the calls themselves, plus any text the
42    model spoke alongside them.
43
44    Models routinely answer with both at once — "I'll look that up for you."
45    followed by a call to `search`. Holding that text here rather than in a
46    separate field on the message keeps the model's whole response inside
47    `content`.
48
49    This is a `list[ToolCall]`, so anything that already treats tool-call
50    content as a list — iteration, indexing, `isinstance(content, list)` —
51    keeps working, and `AssistantMessage` accepts a plain list as before.
52    """
53
54    def __init__(
55        self, tool_calls: Iterable[ToolCall] = (), text: str | None = None
56    ) -> None:
57        super().__init__(tool_calls)
58        self.text = text
59
60    def __repr__(self):
61        if self.text is None:
62            return f"ToolCalls({list.__repr__(self)})"
63        return f"ToolCalls({list.__repr__(self)}, text={self.text!r})"

An assistant turn that calls tools: the calls themselves, plus any text the model spoke alongside them.

Models routinely answer with both at once — "I'll look that up for you." followed by a call to search. Holding that text here rather than in a separate field on the message keeps the model's whole response inside content.

This is a list[ToolCall], so anything that already treats tool-call content as a list — iteration, indexing, isinstance(content, list) — keeps working, and AssistantMessage accepts a plain list as before.

ToolCalls( tool_calls: Iterable[ToolCall] = (), text: str | None = None)
54    def __init__(
55        self, tool_calls: Iterable[ToolCall] = (), text: str | None = None
56    ) -> None:
57        super().__init__(tool_calls)
58        self.text = text
text
class ToolResponse(pydantic.main.BaseModel):
66class ToolResponse(BaseModel):
67    """
68    A simple model object that represents a tool response.
69
70    This simple model should be used when adding a response to a tool.
71    """
72
73    identifier: str = Field(
74        description="The identifier attached to this tool response. This should match the identifier of the tool call."
75    )
76    name: str = Field(description="The name of the tool that generated this response.")
77    result: AnyStr = Field(description="The result of the tool call.")
78
79    def __str__(self):
80        return f"{self.name} -> {self.result}"

A simple model object that represents a tool response.

This simple model should be used when adding a response to a tool.

identifier: str
name: str
result: ~AnyStr
model_config: ClassVar[pydantic.config.ConfigDict] = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

class UserMessage(railtracks.llm.message._StringOnlyContent[ForwardRef(<Role.user: 'user'>)]):
309class UserMessage(_StringOnlyContent[Role.user]):
310    """
311    Note that we only support string input
312
313    Args:
314        content: The content of the user message.
315        attachment: The file attachment(s) for the user message. Can be a single string or a list of strings,
316                    containing file paths, URLs, or data URIs. Defaults to None.
317        trust_urls: Allow in-process fetch for URL attachments. Defaults to False.
318                    When False, `.pdf` URLs and URLs whose extension names an
319                    audio, video, text or Office document type raise, and other
320                    unknown-extension URLs are handed to the provider unprobed
321                    (as `image_url`). When True, `.pdf` URLs are downloaded and
322                    embedded as base64, and every other URL without an image
323                    extension is HEAD-probed; one with a non-image extension
324                    raises unless the probe reports an image or PDF.
325                    Set True only when every URL in `attachment` is
326                    developer-controlled — end-user-supplied URLs are an SSRF
327                    surface once their bytes are fetched in-process.
328        attachment_timeout: Per-request timeout in seconds for the HEAD probe
329                    and the in-process PDF download. Defaults to 10. Only
330                    applies when `trust_urls=True`. Raise this for large PDFs
331                    over slow links.
332        attachment_max_mb: Largest PDF accepted from a URL, in megabytes.
333                    Must be positive and finite. Defaults to 32. Only applies
334                    when `trust_urls=True`.
335
336    Raises:
337        ValueError: If an attachment is rejected, for the reasons listed on
338            `Attachment.__init__`.
339    """
340
341    def __init__(
342        self,
343        content: str | None = None,
344        attachment: str | list[str] | None = None,
345        trust_urls: bool = False,
346        attachment_timeout: float = 10.0,
347        attachment_max_mb: float = DEFAULT_MAX_DOWNLOAD_MB,
348    ):
349        if attachment is not None:
350            if isinstance(attachment, list):
351                self.attachment = [
352                    Attachment(
353                        att,
354                        trust_urls=trust_urls,
355                        attachment_timeout=attachment_timeout,
356                        attachment_max_mb=attachment_max_mb,
357                    )
358                    for att in attachment
359                ]
360            else:
361                self.attachment = [
362                    Attachment(
363                        attachment,
364                        trust_urls=trust_urls,
365                        attachment_timeout=attachment_timeout,
366                        attachment_max_mb=attachment_max_mb,
367                    )
368                ]
369
370            if content is None:
371                logger.warning(
372                    "UserMessage initialized without content, setting to empty string."
373                )
374                content = ""
375        else:
376            self.attachment = None
377
378        if content is None:
379            raise ValueError(
380                "UserMessage must have content if no attachment is provided."
381            )
382        super().__init__(content=content, role=Role.user)
383
384    def encode(self):
385        if self.attachment is not None:
386            attachment = [
387                {
388                    "modality": msg_attachment.modality,
389                    "type": msg_attachment.type,
390                    "info": msg_attachment.url,
391                }
392                for msg_attachment in self.attachment
393            ]
394
395            return {
396                "role": self.role.value,
397                "content": [{"type": "text", "text": self.content}, *attachment],
398            }
399
400        return super().encode()

Note that we only support string input

Arguments:
  • content: The content of the user message.
  • attachment: The file attachment(s) for the user message. Can be a single string or a list of strings, containing file paths, URLs, or data URIs. Defaults to None.
  • trust_urls: Allow in-process fetch for URL attachments. Defaults to False. When False, .pdf URLs and URLs whose extension names an audio, video, text or Office document type raise, and other unknown-extension URLs are handed to the provider unprobed (as image_url). When True, .pdf URLs are downloaded and embedded as base64, and every other URL without an image extension is HEAD-probed; one with a non-image extension raises unless the probe reports an image or PDF. Set True only when every URL in attachment is developer-controlled — end-user-supplied URLs are an SSRF surface once their bytes are fetched in-process.
  • attachment_timeout: Per-request timeout in seconds for the HEAD probe and the in-process PDF download. Defaults to 10. Only applies when trust_urls=True. Raise this for large PDFs over slow links.
  • attachment_max_mb: Largest PDF accepted from a URL, in megabytes. Must be positive and finite. Defaults to 32. Only applies when trust_urls=True.
Raises:
  • ValueError: If an attachment is rejected, for the reasons listed on Attachment.__init__.
UserMessage( content: str | None = None, attachment: str | list[str] | None = None, trust_urls: bool = False, attachment_timeout: float = 10.0, attachment_max_mb: float = 32.0)
341    def __init__(
342        self,
343        content: str | None = None,
344        attachment: str | list[str] | None = None,
345        trust_urls: bool = False,
346        attachment_timeout: float = 10.0,
347        attachment_max_mb: float = DEFAULT_MAX_DOWNLOAD_MB,
348    ):
349        if attachment is not None:
350            if isinstance(attachment, list):
351                self.attachment = [
352                    Attachment(
353                        att,
354                        trust_urls=trust_urls,
355                        attachment_timeout=attachment_timeout,
356                        attachment_max_mb=attachment_max_mb,
357                    )
358                    for att in attachment
359                ]
360            else:
361                self.attachment = [
362                    Attachment(
363                        attachment,
364                        trust_urls=trust_urls,
365                        attachment_timeout=attachment_timeout,
366                        attachment_max_mb=attachment_max_mb,
367                    )
368                ]
369
370            if content is None:
371                logger.warning(
372                    "UserMessage initialized without content, setting to empty string."
373                )
374                content = ""
375        else:
376            self.attachment = None
377
378        if content is None:
379            raise ValueError(
380                "UserMessage must have content if no attachment is provided."
381            )
382        super().__init__(content=content, role=Role.user)

A simple class that represents a message that an LLM can read.

Arguments:
  • content: The content of the message. It can take on any of the following types:
    • str: A simple string message.
    • ToolCalls: Tool calls plus any text the model spoke alongside them.
    • List[ToolCall]: A list of tool calls.
    • ToolResponse: A tool response.
    • BaseModel: A custom base model object.
    • Stream: A stream object with a final_message and a generator.
  • role: The role of the message (assistant, user, system, tool, etc.).
def encode(self):
384    def encode(self):
385        if self.attachment is not None:
386            attachment = [
387                {
388                    "modality": msg_attachment.modality,
389                    "type": msg_attachment.type,
390                    "info": msg_attachment.url,
391                }
392                for msg_attachment in self.attachment
393            ]
394
395            return {
396                "role": self.role.value,
397                "content": [{"type": "text", "text": self.content}, *attachment],
398            }
399
400        return super().encode()
class SystemMessage(railtracks.llm.message._StringOnlyContent[ForwardRef(<Role.system: 'system'>)]):
403class SystemMessage(_StringOnlyContent[Role.system]):
404    """
405    A simple class that represents a system message.
406
407    Args:
408        content (str): The content of the system message.
409    """
410
411    def __init__(self, content: str):
412        super().__init__(content=content, role=Role.system)

A simple class that represents a system message.

Arguments:
  • content (str): The content of the system message.
SystemMessage(content: str)
411    def __init__(self, content: str):
412        super().__init__(content=content, role=Role.system)

A simple class that represents a message that an LLM can read.

Arguments:
  • content: The content of the message. It can take on any of the following types:
    • str: A simple string message.
    • ToolCalls: Tool calls plus any text the model spoke alongside them.
    • List[ToolCall]: A list of tool calls.
    • ToolResponse: A tool response.
    • BaseModel: A custom base model object.
    • Stream: A stream object with a final_message and a generator.
  • role: The role of the message (assistant, user, system, tool, etc.).
class AssistantMessage(railtracks.llm.Message[~_T, ForwardRef()], typing.Generic[~_T]):
415class AssistantMessage(Message[_T, Role.assistant], Generic[_T]):
416    """
417    A simple class that represents a message from the assistant.
418
419    Args:
420        content (_T): The content of the assistant message. A tool-calling turn is a
421            `ToolCalls`, which holds both the calls and any text the model spoke
422            alongside them; a plain `list[ToolCall]` is accepted and normalized to one.
423    """
424
425    def __init__(self, content: _T):
426        # Normalizing here means a tool-calling turn is always a ToolCalls, so
427        # nothing downstream has to handle both shapes, while callers that pass a
428        # plain list of tool calls keep working.
429        if isinstance(content, list) and not isinstance(content, ToolCalls):
430            content = cast(_T, ToolCalls(content))
431
432        super().__init__(content=content, role=Role.assistant)
433
434        # Metadata that lives outside `content`. Any attribute added below must also
435        # be copied in `copy_metadata_from`, or rebuild paths will silently drop it.
436        #
437        # Optionally stores the raw litellm message object so providers that
438        # attach extra metadata (e.g. Gemini thought_signature) can round-trip
439        # it back without any manual reconstruction.
440        self.raw_litellm_message: Any | None = None
441
442        # Reasoning/"thinking" the model surfaced alongside its answer, when the
443        # provider returns it (Anthropic, DeepSeek, Gemini, OpenAI in part).
444        self.reasoning_content: str | None = None  # human-readable text
445        self.thinking_blocks: list[dict[str, Any]] | None = None
446
447    def copy_metadata_from(self, other: Message) -> "AssistantMessage":
448        """Copy the provider metadata that lives outside `content` from `other`.
449
450        Rebuild paths (e.g. the tool loop, which must deep-copy `tool_calls`) construct a
451        fresh message from `content` alone; this carries over everything else so the
452        rebuild stays lossless. New metadata attributes must be added here. Read with
453        `getattr` so a plain `Message` source (which lacks these) is a no-op. Returns self
454        for chaining.
455        """
456        self.raw_litellm_message = getattr(other, "raw_litellm_message", None)
457        self.reasoning_content = getattr(other, "reasoning_content", None)
458        self.thinking_blocks = getattr(other, "thinking_blocks", None)
459        return self
460
461    def encode(self):
462        encoded = super().encode()
463
464        # A ToolCalls encodes as a bare array, which would drop the text the model
465        # spoke with its calls; surface it the way providers put it on the wire.
466        if isinstance(self.content, ToolCalls) and self.content.text is not None:
467            encoded["text"] = self.content.text
468
469        # Surface reasoning in the serialized message so it is visible in the run
470        # graph/session data, not just on the live object.
471        if self.reasoning_content is not None:
472            encoded["reasoning_content"] = self.reasoning_content
473        if self.thinking_blocks is not None:
474            encoded["thinking_blocks"] = self.thinking_blocks
475
476        return encoded

A simple class that represents a message from the assistant.

Arguments:
  • content (_T): The content of the assistant message. A tool-calling turn is a ToolCalls, which holds both the calls and any text the model spoke alongside them; a plain list[ToolCall] is accepted and normalized to one.
AssistantMessage(content: ~_T)
425    def __init__(self, content: _T):
426        # Normalizing here means a tool-calling turn is always a ToolCalls, so
427        # nothing downstream has to handle both shapes, while callers that pass a
428        # plain list of tool calls keep working.
429        if isinstance(content, list) and not isinstance(content, ToolCalls):
430            content = cast(_T, ToolCalls(content))
431
432        super().__init__(content=content, role=Role.assistant)
433
434        # Metadata that lives outside `content`. Any attribute added below must also
435        # be copied in `copy_metadata_from`, or rebuild paths will silently drop it.
436        #
437        # Optionally stores the raw litellm message object so providers that
438        # attach extra metadata (e.g. Gemini thought_signature) can round-trip
439        # it back without any manual reconstruction.
440        self.raw_litellm_message: Any | None = None
441
442        # Reasoning/"thinking" the model surfaced alongside its answer, when the
443        # provider returns it (Anthropic, DeepSeek, Gemini, OpenAI in part).
444        self.reasoning_content: str | None = None  # human-readable text
445        self.thinking_blocks: list[dict[str, Any]] | None = None

A simple class that represents a message that an LLM can read.

Arguments:
  • content: The content of the message. It can take on any of the following types:
    • str: A simple string message.
    • ToolCalls: Tool calls plus any text the model spoke alongside them.
    • List[ToolCall]: A list of tool calls.
    • ToolResponse: A tool response.
    • BaseModel: A custom base model object.
    • Stream: A stream object with a final_message and a generator.
  • role: The role of the message (assistant, user, system, tool, etc.).
raw_litellm_message: typing.Any | None
reasoning_content: str | None
thinking_blocks: list[dict[str, typing.Any]] | None
def copy_metadata_from( self, other: Message) -> AssistantMessage:
447    def copy_metadata_from(self, other: Message) -> "AssistantMessage":
448        """Copy the provider metadata that lives outside `content` from `other`.
449
450        Rebuild paths (e.g. the tool loop, which must deep-copy `tool_calls`) construct a
451        fresh message from `content` alone; this carries over everything else so the
452        rebuild stays lossless. New metadata attributes must be added here. Read with
453        `getattr` so a plain `Message` source (which lacks these) is a no-op. Returns self
454        for chaining.
455        """
456        self.raw_litellm_message = getattr(other, "raw_litellm_message", None)
457        self.reasoning_content = getattr(other, "reasoning_content", None)
458        self.thinking_blocks = getattr(other, "thinking_blocks", None)
459        return self

Copy the provider metadata that lives outside content from other.

Rebuild paths (e.g. the tool loop, which must deep-copy tool_calls) construct a fresh message from content alone; this carries over everything else so the rebuild stays lossless. New metadata attributes must be added here. Read with getattr so a plain Message source (which lacks these) is a no-op. Returns self for chaining.

def encode(self):
461    def encode(self):
462        encoded = super().encode()
463
464        # A ToolCalls encodes as a bare array, which would drop the text the model
465        # spoke with its calls; surface it the way providers put it on the wire.
466        if isinstance(self.content, ToolCalls) and self.content.text is not None:
467            encoded["text"] = self.content.text
468
469        # Surface reasoning in the serialized message so it is visible in the run
470        # graph/session data, not just on the live object.
471        if self.reasoning_content is not None:
472            encoded["reasoning_content"] = self.reasoning_content
473        if self.thinking_blocks is not None:
474            encoded["thinking_blocks"] = self.thinking_blocks
475
476        return encoded
class Message(typing.Generic[~_T, ~_TRole]):
226class Message(Generic[_T, _TRole]):
227    """
228    A base class that represents a message that an LLM can read.
229
230    Note the content may take on a variety of allowable types.
231    """
232
233    def __init__(
234        self,
235        content: _T,
236        role: _TRole,
237    ):
238        """
239        A simple class that represents a message that an LLM can read.
240
241        Args:
242            content: The content of the message. It can take on any of the following types:
243                - str: A simple string message.
244                - ToolCalls: Tool calls plus any text the model spoke alongside them.
245                - List[ToolCall]: A list of tool calls.
246                - ToolResponse: A tool response.
247                - BaseModel: A custom base model object.
248                - Stream: A stream object with a final_message and a generator.
249            role: The role of the message (assistant, user, system, tool, etc.).
250        """
251        assert isinstance(role, Role)
252        self.validate_content(content)
253        self._content = content
254        self._role = role
255
256    @classmethod
257    def validate_content(cls, content: _T):
258        pass
259
260    @property
261    def content(self) -> _T:
262        """Collects the content of the message."""
263        return self._content
264
265    @property
266    def role(self) -> _TRole:
267        """Collects the role of the message."""
268        return self._role
269
270    def __str__(self):
271        return f"{self.role.value}: {self.content}"
272
273    def __repr__(self):
274        return str(self)
275
276    def encode(self) -> dict[str, Any]:
277        return {
278            "role": self.role.value,
279            "content": self.content,
280        }
281
282    @property
283    def tool_calls(self):
284        """Gets the tool calls attached to this message, if any. If there are none return and empty list."""
285        tools: list[ToolCall] = []
286        if isinstance(self.content, list):
287            tools.extend(deepcopy(self.content))
288
289        return tools

A base class that represents a message that an LLM can read.

Note the content may take on a variety of allowable types.

Message(content: ~_T, role: ~_TRole)
233    def __init__(
234        self,
235        content: _T,
236        role: _TRole,
237    ):
238        """
239        A simple class that represents a message that an LLM can read.
240
241        Args:
242            content: The content of the message. It can take on any of the following types:
243                - str: A simple string message.
244                - ToolCalls: Tool calls plus any text the model spoke alongside them.
245                - List[ToolCall]: A list of tool calls.
246                - ToolResponse: A tool response.
247                - BaseModel: A custom base model object.
248                - Stream: A stream object with a final_message and a generator.
249            role: The role of the message (assistant, user, system, tool, etc.).
250        """
251        assert isinstance(role, Role)
252        self.validate_content(content)
253        self._content = content
254        self._role = role

A simple class that represents a message that an LLM can read.

Arguments:
  • content: The content of the message. It can take on any of the following types:
    • str: A simple string message.
    • ToolCalls: Tool calls plus any text the model spoke alongside them.
    • List[ToolCall]: A list of tool calls.
    • ToolResponse: A tool response.
    • BaseModel: A custom base model object.
    • Stream: A stream object with a final_message and a generator.
  • role: The role of the message (assistant, user, system, tool, etc.).
@classmethod
def validate_content(cls, content: ~_T):
256    @classmethod
257    def validate_content(cls, content: _T):
258        pass
content: ~_T
260    @property
261    def content(self) -> _T:
262        """Collects the content of the message."""
263        return self._content

Collects the content of the message.

role: ~_TRole
265    @property
266    def role(self) -> _TRole:
267        """Collects the role of the message."""
268        return self._role

Collects the role of the message.

def encode(self) -> dict[str, typing.Any]:
276    def encode(self) -> dict[str, Any]:
277        return {
278            "role": self.role.value,
279            "content": self.content,
280        }
tool_calls
282    @property
283    def tool_calls(self):
284        """Gets the tool calls attached to this message, if any. If there are none return and empty list."""
285        tools: list[ToolCall] = []
286        if isinstance(self.content, list):
287            tools.extend(deepcopy(self.content))
288
289        return tools

Gets the tool calls attached to this message, if any. If there are none return and empty list.

480class ToolMessage(Message[ToolResponse, Role.tool]):
481    """
482    A simple class that represents a message that is a tool call answer.
483
484    Args:
485        content (ToolResponse): The tool response content for the message.
486    """
487
488    def __init__(self, content: ToolResponse):
489        if not isinstance(content, ToolResponse):
490            raise TypeError(
491                f"A {self.__class__.__name__} needs a ToolResponse but got {type(content)}. Check the invoke function of the OutputLessToolCallLLM node. That is the only place to return a ToolMessage."
492            )
493        super().__init__(content=content, role=Role.tool)

A simple class that represents a message that is a tool call answer.

Arguments:
  • content (ToolResponse): The tool response content for the message.
ToolMessage(content: ToolResponse)
488    def __init__(self, content: ToolResponse):
489        if not isinstance(content, ToolResponse):
490            raise TypeError(
491                f"A {self.__class__.__name__} needs a ToolResponse but got {type(content)}. Check the invoke function of the OutputLessToolCallLLM node. That is the only place to return a ToolMessage."
492            )
493        super().__init__(content=content, role=Role.tool)

A simple class that represents a message that an LLM can read.

Arguments:
  • content: The content of the message. It can take on any of the following types:
    • str: A simple string message.
    • ToolCalls: Tool calls plus any text the model spoke alongside them.
    • List[ToolCall]: A list of tool calls.
    • ToolResponse: A tool response.
    • BaseModel: A custom base model object.
    • Stream: A stream object with a final_message and a generator.
  • role: The role of the message (assistant, user, system, tool, etc.).
class MessageHistory(typing.List[railtracks.llm.message.Message]):
 9class MessageHistory(List[Message]):
10    """
11    A basic object that represents a history of messages. The object has all the same capability as a list such as
12    `.remove()`, `.append()`, etc.
13    """
14
15    def __str__(self):
16        return "\n".join([str(message) for message in self])
17
18    def removed_system_messages(self) -> MessageHistory:
19        """
20        Returns a new MessageHistory object with all SystemMessages removed.
21        """
22        return MessageHistory([msg for msg in self if msg.role != Role.system])

A basic object that represents a history of messages. The object has all the same capability as a list such as .remove(), .append(), etc.

def removed_system_messages(self) -> MessageHistory:
18    def removed_system_messages(self) -> MessageHistory:
19        """
20        Returns a new MessageHistory object with all SystemMessages removed.
21        """
22        return MessageHistory([msg for msg in self if msg.role != Role.system])

Returns a new MessageHistory object with all SystemMessages removed.

class ModelProvider(builtins.str, enum.Enum):
13class ModelProvider(str, Enum):
14    """
15    Enum of supported LLM model providers for RailTracks.
16
17    Attributes:
18        OPENAI: OpenAI models (e.g., GPT-3, GPT-4).
19        ANTHROPIC: Anthropic models (e.g., Claude).
20        GEMINI: Google Gemini models.
21        HUGGINGFACE: HuggingFace-hosted models.
22        AZUREAI: Azure OpenAI Service models.
23        OLLAMA: Ollama local LLMs.
24        APPLE_FM: Apple on-device Foundation Model (macOS 26+ Apple Silicon).
25    """
26
27    OPENAI = "OpenAI"
28    ANTHROPIC = "Anthropic"
29    GEMINI = "Vertex_AI"
30    HUGGINGFACE = "HuggingFace"
31    AZUREAI = "AzureAI"
32    OLLAMA = "Ollama"
33    PORTKEY = "PortKey"
34    APPLE_FM = "apple"
35    UNKNOWN = "Unknown"

Enum of supported LLM model providers for RailTracks.

Attributes:
  • OPENAI: OpenAI models (e.g., GPT-3, GPT-4).
  • ANTHROPIC: Anthropic models (e.g., Claude).
  • GEMINI: Google Gemini models.
  • HUGGINGFACE: HuggingFace-hosted models.
  • AZUREAI: Azure OpenAI Service models.
  • OLLAMA: Ollama local LLMs.
  • APPLE_FM: Apple on-device Foundation Model (macOS 26+ Apple Silicon).
OPENAI = <ModelProvider.OPENAI: 'OpenAI'>
ANTHROPIC = <ModelProvider.ANTHROPIC: 'Anthropic'>
GEMINI = <ModelProvider.GEMINI: 'Vertex_AI'>
HUGGINGFACE = <ModelProvider.HUGGINGFACE: 'HuggingFace'>
AZUREAI = <ModelProvider.AZUREAI: 'AzureAI'>
OLLAMA = <ModelProvider.OLLAMA: 'Ollama'>
PORTKEY = <ModelProvider.PORTKEY: 'PortKey'>
APPLE_FM = <ModelProvider.APPLE_FM: 'apple'>
UNKNOWN = <ModelProvider.UNKNOWN: 'Unknown'>
class Tool:
112class Tool:
113    """
114    A quasi-immutable class designed to represent a single Tool object.
115    You pass in key details (name, description, and required parameters).
116    """
117
118    def __init__(
119        self,
120        name: str,
121        detail: str,
122        parameters: Iterable[Parameter] | Dict[str, Any] | None = None,
123    ):
124        """
125        Creates a new Tool instance.
126
127        Args:
128            name: The name of the tool.
129            detail: A detailed description of the tool.
130            parameters: Parameters attached to this tool; a set or list of Parameter objects, or a dict.
131        """
132        parameters = _validate_tool_params(parameters, Parameter)
133
134        if (
135            isinstance(parameters, dict) and len(parameters) > 0
136        ):  # if parameters is a JSON-output_schema, convert into Parameter objects
137            props = parameters["properties"]
138            required_fields = list(parameters.get("required", []))
139            if not props and required_fields:
140                raise ToolCreationError(
141                    f"Tool {name!r}: schema declares required fields "
142                    f"{required_fields} but has no 'properties' block.",
143                    notes=[
144                        "Add a 'properties' entry describing each required field.",
145                        "A schema with no parameters should omit 'required' entirely.",
146                    ],
147                )
148            param_objs: List[Parameter] = []
149            for param_name, prop in props.items():
150                try:
151                    param_objs.append(
152                        parse_json_schema_to_parameter(
153                            param_name, prop, param_name in required_fields
154                        )
155                    )
156                except ValueError as e:
157                    raise ToolCreationError(
158                        f"Tool {name!r}: failed to parse schema for parameter '{param_name}': {e}",
159                        notes=[
160                            "Check that the parameter's 'type' is a valid JSON schema type."
161                        ],
162                    ) from e
163            parameters = param_objs
164
165        self._name = name
166        self._detail = detail
167        self._parameters = parameters
168
169    @property
170    def name(self) -> str:
171        """Get the name of the tool."""
172        return self._name
173
174    @property
175    def detail(self) -> str:
176        """Returns the detailed description for this tool."""
177        return self._detail
178
179    @property
180    def parameters(self) -> List[Parameter] | None:
181        """Gets the parameters attached to this tool (if any)."""
182        return self._parameters
183
184    def __str__(self) -> str:
185        """String representation of the tool."""
186        if self._parameters:
187            params_str = "{" + ", ".join(str(p) for p in self._parameters) + "}"
188        return f"Tool(name={self._name}, detail={self._detail}, parameters={params_str if self._parameters else 'None'})"
189
190    def encode(self):
191        return {
192            "name": self._name,
193            "detail": self._detail,
194            "parameters": self._parameters,
195        }
196
197    @classmethod
198    def from_function(
199        cls,
200        func: Callable[..., Any],
201        /,
202        *,
203        name: str | None = None,
204        details: str | None = None,
205        params: Type[BaseModel] | Dict[str, Any] | List[Parameter] | None = None,
206    ) -> Self:
207        """
208        Creates a Tool from a Python callable.
209        Uses the function's docstring and type annotations to extract details and parameter info.
210
211        KEY NOTE: No checking is done to ensure that the inserted params match the function signature
212
213        Args:
214            func: The function to create a tool from.
215            name: Optional name for the tool. If not provided, uses the function's name. Either
216                is passed through `to_tool_name` so providers accept it.
217            details: Optional detailed description for the tool. If not provided, extracts from the function's docstring.
218            params: Optional parameters for the tool. If not provided, infers from the function's signature and docstring.
219
220        Returns:
221            A Tool instance representing the function.
222        """
223        # TODO: add set verification to ensure that the params match the function signature
224        # Check if the function is a method in a class
225        in_class = bool(func.__qualname__ and "." in func.__qualname__)
226
227        # Parse the docstring to get parameter descriptions
228        arg_descriptions = parse_docstring_args(func.__doc__ or "")
229
230        try:
231            # Get the function signature
232            signature = inspect.signature(func)
233        except ValueError:
234            raise ToolCreationError(
235                message="Cannot convert kwargs for builtin functions.",
236                notes=[
237                    "Please use a cutom made function.",
238                    "Eg.- \ndef my_custom_function(a: int, b: str):\n    pass",
239                ],
240            )
241
242        function_name = to_tool_name(name if name is not None else func.__name__)
243
244        docstring = func.__doc__.strip() if func.__doc__ else ""
245
246        if params is not None:
247            parameters = params
248        else:
249            resolved_types = resolve_type_hints(func, signature)
250
251            # Check for multiple parameter sections (warning)
252            # Only need to do this if we need to.
253            if count_parameter_sections(docstring) > 1:
254                warnings.warn(
255                    "Multiple parameter sections found in the docstring. Only one "
256                    "is used, in priority order: Google, NumPy, reST."
257                )
258            unparsed_fields = find_unparsed_rest_param_fields(docstring)
259            if unparsed_fields:
260                warnings.warn(
261                    f"Could not parse reST parameter fields {unparsed_fields!r} in the "
262                    f"docstring of '{function_name}'; their descriptions are ignored."
263                )
264            # Create parameter handlers
265            handlers: List[ParameterHandler] = [
266                PydanticModelHandler(),
267                SequenceParameterHandler(),
268                UnionParameterHandler(),
269                LiteralParameterHandler(),
270                DefaultParameterHandler(),
271            ]
272
273            parameters: List[Parameter] = []
274
275            for param in signature.parameters.values():
276                # Skip 'self' parameter for class methods
277                if in_class and (param.name == "self" or param.name == "cls"):
278                    continue
279
280                description = arg_descriptions.get(param.name, "")
281
282                # Check if the parameter is required
283                required = param.default == inspect.Parameter.empty
284
285                annotation = (
286                    resolved_types.get(param.name, param.annotation)
287                    if isinstance(param.annotation, str)
288                    else param.annotation
289                )
290                handler = next(h for h in handlers if h.can_handle(annotation))
291
292                param_obj = handler.create_parameter(
293                    param.name, annotation, description, required
294                )
295
296                parameters.append(param_obj)
297
298        if details is not None:
299            main_description = details
300        else:
301            main_description = extract_main_description(docstring)
302
303        tool_info = Tool(
304            name=function_name,
305            detail=main_description,
306            parameters=parameters,
307        )
308        return tool_info
309
310    @classmethod
311    def from_mcp(cls, tool) -> Self:
312        """
313        Creates a Tool from an MCP tool object.
314
315        Args:
316            tool: The MCP tool to create a Tool from.
317
318        Returns:
319            A Tool instance representing the MCP tool.
320        """
321        input_schema = getattr(tool, "inputSchema", None)
322        if not input_schema or input_schema["type"] != "object":
323            raise ToolCreationError(
324                message="The inputSchema for an MCP Tool must be 'object'. ",
325                notes=[
326                    "If an MCP tool has a different output_schema, create a GitHub issue and support will be added."
327                ],
328            )
329
330        properties = input_schema.get("properties", {})
331        required_fields = set(input_schema.get("required", []))
332        param_objs = set()
333        for name, prop in properties.items():
334            required = name in required_fields
335            try:
336                param_objs.add(parse_json_schema_to_parameter(name, prop, required))
337            except Exception as e:
338                warnings.warn(
339                    f"Tool {tool.name!r}: failed to parse schema for parameter '{name}': {e}. Falling back to basic object."
340                )
341                # Fallback to a basic object parameter if parsing fails (e.g. invalid type string)
342                param_objs.add(
343                    Parameter(name=name, param_type="object", required=required)
344                )
345
346        return cls(name=tool.name, detail=tool.description, parameters=param_objs)

A quasi-immutable class designed to represent a single Tool object. You pass in key details (name, description, and required parameters).

Tool( name: str, detail: str, parameters: Union[Iterable[Parameter], Dict[str, Any], NoneType] = None)
118    def __init__(
119        self,
120        name: str,
121        detail: str,
122        parameters: Iterable[Parameter] | Dict[str, Any] | None = None,
123    ):
124        """
125        Creates a new Tool instance.
126
127        Args:
128            name: The name of the tool.
129            detail: A detailed description of the tool.
130            parameters: Parameters attached to this tool; a set or list of Parameter objects, or a dict.
131        """
132        parameters = _validate_tool_params(parameters, Parameter)
133
134        if (
135            isinstance(parameters, dict) and len(parameters) > 0
136        ):  # if parameters is a JSON-output_schema, convert into Parameter objects
137            props = parameters["properties"]
138            required_fields = list(parameters.get("required", []))
139            if not props and required_fields:
140                raise ToolCreationError(
141                    f"Tool {name!r}: schema declares required fields "
142                    f"{required_fields} but has no 'properties' block.",
143                    notes=[
144                        "Add a 'properties' entry describing each required field.",
145                        "A schema with no parameters should omit 'required' entirely.",
146                    ],
147                )
148            param_objs: List[Parameter] = []
149            for param_name, prop in props.items():
150                try:
151                    param_objs.append(
152                        parse_json_schema_to_parameter(
153                            param_name, prop, param_name in required_fields
154                        )
155                    )
156                except ValueError as e:
157                    raise ToolCreationError(
158                        f"Tool {name!r}: failed to parse schema for parameter '{param_name}': {e}",
159                        notes=[
160                            "Check that the parameter's 'type' is a valid JSON schema type."
161                        ],
162                    ) from e
163            parameters = param_objs
164
165        self._name = name
166        self._detail = detail
167        self._parameters = parameters

Creates a new Tool instance.

Arguments:
  • name: The name of the tool.
  • detail: A detailed description of the tool.
  • parameters: Parameters attached to this tool; a set or list of Parameter objects, or a dict.
name: str
169    @property
170    def name(self) -> str:
171        """Get the name of the tool."""
172        return self._name

Get the name of the tool.

detail: str
174    @property
175    def detail(self) -> str:
176        """Returns the detailed description for this tool."""
177        return self._detail

Returns the detailed description for this tool.

parameters: Optional[List[Parameter]]
179    @property
180    def parameters(self) -> List[Parameter] | None:
181        """Gets the parameters attached to this tool (if any)."""
182        return self._parameters

Gets the parameters attached to this tool (if any).

def encode(self):
190    def encode(self):
191        return {
192            "name": self._name,
193            "detail": self._detail,
194            "parameters": self._parameters,
195        }
@classmethod
def from_function( cls, func: Callable[..., Any], /, *, name: str | None = None, details: str | None = None, params: Union[Type[pydantic.main.BaseModel], Dict[str, Any], List[Parameter], NoneType] = None) -> Self:
197    @classmethod
198    def from_function(
199        cls,
200        func: Callable[..., Any],
201        /,
202        *,
203        name: str | None = None,
204        details: str | None = None,
205        params: Type[BaseModel] | Dict[str, Any] | List[Parameter] | None = None,
206    ) -> Self:
207        """
208        Creates a Tool from a Python callable.
209        Uses the function's docstring and type annotations to extract details and parameter info.
210
211        KEY NOTE: No checking is done to ensure that the inserted params match the function signature
212
213        Args:
214            func: The function to create a tool from.
215            name: Optional name for the tool. If not provided, uses the function's name. Either
216                is passed through `to_tool_name` so providers accept it.
217            details: Optional detailed description for the tool. If not provided, extracts from the function's docstring.
218            params: Optional parameters for the tool. If not provided, infers from the function's signature and docstring.
219
220        Returns:
221            A Tool instance representing the function.
222        """
223        # TODO: add set verification to ensure that the params match the function signature
224        # Check if the function is a method in a class
225        in_class = bool(func.__qualname__ and "." in func.__qualname__)
226
227        # Parse the docstring to get parameter descriptions
228        arg_descriptions = parse_docstring_args(func.__doc__ or "")
229
230        try:
231            # Get the function signature
232            signature = inspect.signature(func)
233        except ValueError:
234            raise ToolCreationError(
235                message="Cannot convert kwargs for builtin functions.",
236                notes=[
237                    "Please use a cutom made function.",
238                    "Eg.- \ndef my_custom_function(a: int, b: str):\n    pass",
239                ],
240            )
241
242        function_name = to_tool_name(name if name is not None else func.__name__)
243
244        docstring = func.__doc__.strip() if func.__doc__ else ""
245
246        if params is not None:
247            parameters = params
248        else:
249            resolved_types = resolve_type_hints(func, signature)
250
251            # Check for multiple parameter sections (warning)
252            # Only need to do this if we need to.
253            if count_parameter_sections(docstring) > 1:
254                warnings.warn(
255                    "Multiple parameter sections found in the docstring. Only one "
256                    "is used, in priority order: Google, NumPy, reST."
257                )
258            unparsed_fields = find_unparsed_rest_param_fields(docstring)
259            if unparsed_fields:
260                warnings.warn(
261                    f"Could not parse reST parameter fields {unparsed_fields!r} in the "
262                    f"docstring of '{function_name}'; their descriptions are ignored."
263                )
264            # Create parameter handlers
265            handlers: List[ParameterHandler] = [
266                PydanticModelHandler(),
267                SequenceParameterHandler(),
268                UnionParameterHandler(),
269                LiteralParameterHandler(),
270                DefaultParameterHandler(),
271            ]
272
273            parameters: List[Parameter] = []
274
275            for param in signature.parameters.values():
276                # Skip 'self' parameter for class methods
277                if in_class and (param.name == "self" or param.name == "cls"):
278                    continue
279
280                description = arg_descriptions.get(param.name, "")
281
282                # Check if the parameter is required
283                required = param.default == inspect.Parameter.empty
284
285                annotation = (
286                    resolved_types.get(param.name, param.annotation)
287                    if isinstance(param.annotation, str)
288                    else param.annotation
289                )
290                handler = next(h for h in handlers if h.can_handle(annotation))
291
292                param_obj = handler.create_parameter(
293                    param.name, annotation, description, required
294                )
295
296                parameters.append(param_obj)
297
298        if details is not None:
299            main_description = details
300        else:
301            main_description = extract_main_description(docstring)
302
303        tool_info = Tool(
304            name=function_name,
305            detail=main_description,
306            parameters=parameters,
307        )
308        return tool_info

Creates a Tool from a Python callable. Uses the function's docstring and type annotations to extract details and parameter info.

KEY NOTE: No checking is done to ensure that the inserted params match the function signature

Arguments:
  • func: The function to create a tool from.
  • name: Optional name for the tool. If not provided, uses the function's name. Either is passed through to_tool_name so providers accept it.
  • details: Optional detailed description for the tool. If not provided, extracts from the function's docstring.
  • params: Optional parameters for the tool. If not provided, infers from the function's signature and docstring.
Returns:

A Tool instance representing the function.

@classmethod
def from_mcp(cls, tool) -> Self:
310    @classmethod
311    def from_mcp(cls, tool) -> Self:
312        """
313        Creates a Tool from an MCP tool object.
314
315        Args:
316            tool: The MCP tool to create a Tool from.
317
318        Returns:
319            A Tool instance representing the MCP tool.
320        """
321        input_schema = getattr(tool, "inputSchema", None)
322        if not input_schema or input_schema["type"] != "object":
323            raise ToolCreationError(
324                message="The inputSchema for an MCP Tool must be 'object'. ",
325                notes=[
326                    "If an MCP tool has a different output_schema, create a GitHub issue and support will be added."
327                ],
328            )
329
330        properties = input_schema.get("properties", {})
331        required_fields = set(input_schema.get("required", []))
332        param_objs = set()
333        for name, prop in properties.items():
334            required = name in required_fields
335            try:
336                param_objs.add(parse_json_schema_to_parameter(name, prop, required))
337            except Exception as e:
338                warnings.warn(
339                    f"Tool {tool.name!r}: failed to parse schema for parameter '{name}': {e}. Falling back to basic object."
340                )
341                # Fallback to a basic object parameter if parsing fails (e.g. invalid type string)
342                param_objs.add(
343                    Parameter(name=name, param_type="object", required=required)
344                )
345
346        return cls(name=tool.name, detail=tool.description, parameters=param_objs)

Creates a Tool from an MCP tool object.

Arguments:
  • tool: The MCP tool to create a Tool from.
Returns:

A Tool instance representing the MCP tool.

class AnthropicLLM(railtracks.llm.models.api_providers._provider_wrapper.ProviderLLMWrapper):
6class AnthropicLLM(ProviderLLMWrapper):
7    @classmethod
8    def model_gateway(cls) -> ModelProvider:
9        return ModelProvider.ANTHROPIC

A large base class that wraps around a litellm model.

Note that the model object should be interacted with via the methods provided in the wrapper class:

Each individual API should implement the required abstract_methods in order to allow users to interact with a model of that type.

@classmethod
def model_gateway(cls) -> ModelProvider:
7    @classmethod
8    def model_gateway(cls) -> ModelProvider:
9        return ModelProvider.ANTHROPIC

Gets the API distrubutor of the model. Note nessecarily the same as the model itself.

E.g. if you are calling openai LLM through Azure AI foundry

class AppleFMLLM(railtracks.llm.ModelBase):
111class AppleFMLLM(ModelBase):
112    """Apple's on-device system language model.
113
114    Requires macOS 26+ on Apple Silicon with Apple Intelligence enabled. The
115    SDK is installed via `pip install railtracks[apple]`. Inference is
116    serialized at the hardware level and runs entirely on-device.
117
118    See the module docstring for the deliberate scope: chat, structured
119    output, and streaming chat are supported; tool calling and streaming
120    structured output raise `NotImplementedError`.
121
122    Only the `general` on-device preset is used. Apple also ships a
123    `content_tagging` preset tuned for classification-shaped outputs, but
124    classification is not an agent-loop shape and doesn't fit railtracks'
125    purpose. Users who need it should drop down to `apple_fm_sdk`
126    directly.
127
128    Args:
129        temperature (float | None, optional): Sampling temperature.
130            Passed through to `fm.GenerationOptions`. None uses the
131            SDK default.
132        maximum_response_tokens (int | None, optional): Hard cap on
133            the number of tokens the model may produce.
134        sampling_seed (int | None, optional): If set, sampling runs
135            in `SamplingMode.random(seed=...)` for reproducible
136            outputs across runs.
137        guardrails (bool, optional): When True (default) uses Apple's
138            `SystemLanguageModelGuardrails.DEFAULT`. When False,
139            switches to `PERMISSIVE_CONTENT_TRANSFORMATIONS` if the
140            SDK exposes it. Safety refusals still raise
141            `AppleFMSafetyRefusalError` in either mode.
142        retry_approach (RetryApproach | None, optional): Retry
143            strategy passed to `ModelBase` for transient failures.
144        **kwargs: Rejected. Any non-None value here raises
145            `UnsupportedHyperparameterError` — Apple's SDK does not
146            accept `top_p`, penalties, or stop tokens, so silently
147            accepting them would mislead callers.
148
149    Raises:
150        ImportError: `apple_fm_sdk` is not installed. Install with
151            `pip install railtracks[apple]`.
152        AppleFMUnavailableError: The on-device model is unavailable
153            on this machine — unsupported OS/hardware, Apple
154            Intelligence disabled, or assets not yet downloaded.
155        UnsupportedHyperparameterError: An unknown kwarg was passed
156            with a non-None value.
157    """
158
159    def __init__(
160        self,
161        *,
162        temperature: float | None = None,
163        maximum_response_tokens: int | None = None,
164        sampling_seed: int | None = None,
165        guardrails: bool = True,
166        retry_approach: RetryApproach | None = None,
167        **kwargs: Any,
168    ):
169        try:
170            import apple_fm_sdk as fm
171        except ImportError as e:
172            raise ImportError(
173                "The `apple_fm_sdk` package is required to use AppleFMLLM. "
174                "Install with `pip install railtracks[apple]`. Note that the "
175                "package itself only runs on macOS 26+ Apple Silicon."
176            ) from e
177
178        for k, v in kwargs.items():
179            if v is not None:
180                raise UnsupportedHyperparameterError(
181                    model_name="apple-fm", hyperparameter=k, value=v
182                )
183
184        super().__init__(retry_approach=retry_approach)
185
186        self._fm = fm
187        self._guardrails = guardrails
188        self._temperature = temperature
189        self._maximum_response_tokens = maximum_response_tokens
190        self._sampling_seed = sampling_seed
191
192        self._model_handle = self._make_model_handle()
193        available, reason = self._model_handle.is_available()
194        if not available:
195            raise AppleFMUnavailableError(
196                f"Apple Foundation Model is not available on this device: {reason}"
197            )
198
199    def _make_model_handle(self):
200        """Build the `fm.SystemLanguageModel` handle used to check
201        availability and to seed sessions. Uses `getattr` for the enum
202        classes so an older SDK build without one of them still loads.
203        Always uses the `GENERAL` preset; the `content_tagging` preset
204        is deliberately out of scope for this agentic framework.
205        """
206        fm = self._fm
207        kwargs: dict[str, Any] = {}
208        cases = getattr(fm, "SystemLanguageModelUseCase", None)
209        if cases is not None:
210            kwargs["use_case"] = cases.GENERAL
211        rails = getattr(fm, "SystemLanguageModelGuardrails", None)
212        if rails is not None:
213            kwargs["guardrails"] = (
214                rails.DEFAULT
215                if self._guardrails
216                else getattr(rails, "PERMISSIVE_CONTENT_TRANSFORMATIONS", rails.DEFAULT)
217            )
218        return fm.SystemLanguageModel(**kwargs)
219
220    def _build_options(self) -> Any | None:
221        """Assemble an `fm.GenerationOptions` from the stored knobs, or
222        `None` if none are set. Returning `None` lets `respond()` be
223        called without the `options=` kwarg, which matters when the
224        installed SDK is older than expected and doesn't have
225        `GenerationOptions` at all.
226        """
227        fm = self._fm
228        opts_cls = getattr(fm, "GenerationOptions", None)
229        if opts_cls is None:
230            return None
231
232        sampling = None
233        if self._sampling_seed is not None:
234            sampling_cls = getattr(fm, "SamplingMode", None)
235            if sampling_cls is not None:
236                sampling = sampling_cls.random(seed=self._sampling_seed)
237
238        kwargs: dict[str, Any] = {}
239        if sampling is not None:
240            kwargs["sampling"] = sampling
241        if self._temperature is not None:
242            kwargs["temperature"] = self._temperature
243        if self._maximum_response_tokens is not None:
244            kwargs["maximum_response_tokens"] = self._maximum_response_tokens
245        return opts_cls(**kwargs) if kwargs else None
246
247    def model_name(self) -> str:
248        """Return `"apple-fm"`. There is no preset suffix: the class
249        always uses Apple's `GENERAL` on-device preset.
250        """
251        return "apple-fm"
252
253    def model_provider(self) -> ModelProvider:
254        """Return `ModelProvider.APPLE_FM`. See the enum's docstring."""
255        return ModelProvider.APPLE_FM
256
257    @classmethod
258    def model_gateway(cls) -> ModelProvider:
259        """Return `ModelProvider.APPLE_FM`. Same as `model_provider`
260        because on-device Apple has no separate gateway routing.
261        """
262        return ModelProvider.APPLE_FM
263
264    def _reject_attachments(self, messages: MessageHistory) -> None:
265        """Fail loudly if the message history carries any attachment.
266
267        Apple exposes `fm.ImageAttachment`, but current SDK builds (0.2.x,
268        compiled against macOS 26 SDKs) raise `ImagePromptError` at call
269        time — the on-device model does not process images yet. Silently
270        dropping the attachment means the model hallucinates a description
271        of an image it never saw, which is the worst possible failure
272        mode. Reject up front instead.
273        """
274        for msg in messages:
275            if isinstance(msg, UserMessage) and getattr(msg, "attachment", None):
276                raise NotImplementedError(
277                    "AppleFMLLM does not support UserMessage attachments. "
278                    "Apple's on-device SDK exposes fm.ImageAttachment but "
279                    "current SDK builds do not process images (they require "
280                    "macOS 27 SDKs that are not yet released), so silently "
281                    "dropping the attachment would cause the model to "
282                    "hallucinate a description of an image it never saw. "
283                    "Send text-only prompts, or use another provider "
284                    "(OpenAI, Anthropic) for multimodal input."
285                )
286
287    def _split_history(self, messages: MessageHistory) -> tuple[str, list, str]:
288        """Peel the history into (instructions, prior_turns, final_prompt).
289
290        Apple's session takes `instructions=` once at construction; multi-turn
291        context is either implicit (session reuse) or reconstructed via
292        `Transcript.from_dict`. We build a fresh session per call, so we hand
293        prior turns to `from_transcript` when there are any.
294        """
295        self._reject_attachments(messages)
296
297        instructions_parts: list[str] = []
298        prior: list = []
299        final_prompt: str | None = None
300
301        for msg in messages:
302            if isinstance(msg, SystemMessage):
303                instructions_parts.append(str(msg.content))
304            elif isinstance(msg, UserMessage):
305                if final_prompt is not None:
306                    prior.append({"role": "user", "contents": [final_prompt]})
307                final_prompt = str(msg.content)
308            elif isinstance(msg, AssistantMessage):
309                prior.append({"role": "response", "contents": [str(msg.content)]})
310            else:
311                prior.append({"role": "user", "contents": [str(msg.content)]})
312
313        if final_prompt is None:
314            raise ModelError(
315                reason="AppleFMLLM requires at least one UserMessage in the history.",
316                message_history=messages,
317            )
318
319        instructions = "\n\n".join(p for p in instructions_parts if p)
320        return instructions, prior, final_prompt
321
322    def _make_session(self, messages: MessageHistory) -> tuple[Any, str]:
323        """Build a fresh `fm.LanguageModelSession` for this turn and
324        return it along with the final-user prompt string.
325
326        Two paths:
327          - No prior turns: plain `LanguageModelSession(instructions=...)`.
328          - Prior turns: reconstruct an `fm.Transcript` via
329            `Transcript.from_dict` and seed the session with
330            `from_transcript`. If `from_dict` isn't available or the
331            reconstruction raises (SDK-version shape drift), fall back
332            to a plain single-turn session and log a warning — better
333            than hard-failing multi-turn calls when the SDK changes.
334        """
335        fm = self._fm
336        instructions, prior, final_prompt = self._split_history(messages)
337
338        if not prior:
339            session = fm.LanguageModelSession(instructions=instructions or None)
340            return session, final_prompt
341
342        transcript_entries: list[dict] = []
343        if instructions:
344            transcript_entries.append(
345                {"role": "instructions", "contents": [instructions]}
346            )
347        transcript_entries.extend(prior)
348
349        transcript_cls = getattr(fm, "Transcript", None)
350        from_dict = (
351            getattr(transcript_cls, "from_dict", None)
352            if transcript_cls is not None
353            else None
354        )
355        from_transcript = getattr(fm.LanguageModelSession, "from_transcript", None)
356        if from_dict is not None and from_transcript is not None:
357            try:
358                transcript = from_dict({"entries": transcript_entries})
359                return from_transcript(transcript), final_prompt
360            except Exception as e:  # pragma: no cover - depends on installed SDK shape
361                logger.warning(
362                    "AppleFMLLM: Transcript.from_dict path failed (%s); "
363                    "falling back to a single-turn session for this call.",
364                    e,
365                )
366
367        session = fm.LanguageModelSession(instructions=instructions or None)
368        return session, final_prompt
369
370    def extract_message_info(self, latency: float | None = None) -> MessageInfo:
371        """Populate MessageInfo from what Apple gives us — which is nothing.
372
373        Named to mirror `LiteLLMWrapper.extract_message_info` so the two files
374        read the same. `total_cost` is left `None` to match how other
375        providers report missing usage; on-device inference is genuinely
376        free but reporting `0.0` would silently conflate "free" with
377        "unknown" in mixed-provider aggregations.
378        """
379        return MessageInfo(
380            input_tokens=None,
381            output_tokens=None,
382            latency=latency,
383            model_name=self.model_name(),
384            total_cost=None,
385            system_fingerprint=None,
386        )
387
388    def _prepare_response(self, content: Any, latency: float) -> Response:
389        """Wrap raw model output into a `Response`. Mirrors the
390        `_prepare_response` helper in `LiteLLMWrapper` so both providers
391        return the same shape to the framework.
392        """
393        return Response(
394            AssistantMessage(content),
395            self.extract_message_info(latency=latency),
396        )
397
398    def _translate_fm_error(
399        self, e: BaseException, messages: MessageHistory
400    ) -> ModelError:
401        """Map an `fm.FoundationModelsError` subclass to the closest
402        railtracks-native error. Safety refusals become
403        `AppleFMSafetyRefusalError`, missing-asset failures become
404        `AppleFMUnavailableError`, everything else falls back to a
405        generic `ModelError`.
406        """
407        fm = self._fm
408        safety = tuple(
409            cls
410            for name in ("GuardrailViolationError", "RefusalError")
411            if (cls := getattr(fm, name, None)) is not None
412        )
413        if safety and isinstance(e, safety):
414            return AppleFMSafetyRefusalError(
415                reason=f"Apple Foundation Model refused the request: {e}",
416                message_history=messages,
417            )
418        unavailable = getattr(fm, "AssetsUnavailableError", None)
419        if unavailable is not None and isinstance(e, unavailable):
420            return AppleFMUnavailableError(
421                f"Apple Foundation Model assets unavailable: {e}"
422            )
423        return ModelError(reason=str(e), message_history=messages)
424
425    def _run_sync(self, coro):
426        """Bridge a sync-API call to Apple's async-only SDK. Uses
427        `asyncio.run` when no loop is running; refuses if one is,
428        because nesting an event loop inside another silently corrupts
429        state — the caller has to switch to the async variants.
430        """
431        try:
432            asyncio.get_running_loop()
433        except RuntimeError:
434            return asyncio.run(coro)
435        coro.close()
436        raise ModelError(
437            reason=(
438                "AppleFMLLM sync API cannot be called from inside a running "
439                "event loop. Use achat / astructured / astream_chat instead."
440            )
441        )
442
443    def _chat(self, messages: MessageHistory) -> Response:
444        return self._run_sync(self._achat(messages))
445
446    def _structured(
447        self, messages: MessageHistory, schema: Type[BaseModel]
448    ) -> Response:
449        return self._run_sync(self._astructured(messages, schema))
450
451    def _chat_with_tools(self, messages: MessageHistory, tools: List[Tool]) -> Response:
452        raise NotImplementedError(
453            "AppleFMLLM does not support tool calling. Apple's on-device SDK "
454            "drives the tool loop internally and provides no intent-only "
455            "interception hook, so it does not fit railtracks' external "
456            "orchestration contract. Use OpenAI/Anthropic/Ollama for "
457            "tool-driven flows."
458        )
459
460    async def _achat(self, messages: MessageHistory) -> Response:
461        session, prompt = self._make_session(messages)
462        opts = self._build_options()
463        start = time.time()
464        try:
465            if opts is not None:
466                text = await session.respond(prompt, options=opts)
467            else:
468                text = await session.respond(prompt)
469        except self._fm.FoundationModelsError as e:
470            raise self._translate_fm_error(e, messages) from e
471        return self._prepare_response(str(text), latency=time.time() - start)
472
473    async def _astructured(
474        self, messages: MessageHistory, schema: Type[BaseModel]
475    ) -> Response:
476        session, prompt = self._make_session(messages)
477        opts = self._build_options()
478        json_schema = _normalize_schema_for_apple(schema.model_json_schema())
479        start = time.time()
480        try:
481            if opts is not None:
482                result = await session.respond(
483                    prompt, json_schema=json_schema, options=opts
484                )
485            else:
486                result = await session.respond(prompt, json_schema=json_schema)
487        except self._fm.FoundationModelsError as e:
488            raise self._translate_fm_error(e, messages) from e
489
490        parsed = self._parse_structured(result, schema, messages)
491        return self._prepare_response(parsed, latency=time.time() - start)
492
493    def _parse_structured(
494        self,
495        result: Any,
496        schema: Type[BaseModel],
497        messages: MessageHistory,
498    ) -> BaseModel:
499        """Convert Apple's `GeneratedContent` (or a raw string in some
500        SDK versions) into an instance of `schema`. Raises `ModelError` if
501        the payload doesn't validate — treated as a model failure the
502        caller should retry or surface, not a bug in the schema.
503        """
504        to_json = getattr(result, "to_json", None)
505        raw: str = str(to_json()) if callable(to_json) else str(result)
506        try:
507            return schema.model_validate_json(raw)
508        except Exception as e:
509            raise ModelError(
510                reason=(
511                    f"AppleFMLLM structured output did not match schema "
512                    f"{schema.__name__}: {e}"
513                ),
514                message_history=messages,
515            ) from e
516
517    async def _achat_with_tools(
518        self, messages: MessageHistory, tools: List[Tool]
519    ) -> Response:
520        raise NotImplementedError(
521            "AppleFMLLM does not support tool calling. Apple's on-device SDK "
522            "drives the tool loop internally and provides no intent-only "
523            "interception hook, so it does not fit railtracks' external "
524            "orchestration contract. Use OpenAI/Anthropic/Ollama for "
525            "tool-driven flows."
526        )
527
528    async def _astream_chat(
529        self, messages: MessageHistory
530    ) -> AsyncGenerator[str | Response, None]:
531        session, prompt = self._make_session(messages)
532        opts = self._build_options()
533        prev = ""
534        start = time.time()
535        try:
536            if opts is not None:
537                stream = session.stream_response(prompt, options=opts)
538            else:
539                stream = session.stream_response(prompt)
540            async for snapshot in stream:
541                text = str(snapshot)
542                delta = text[len(prev) :]
543                prev = text
544                if delta:
545                    yield delta
546        except self._fm.FoundationModelsError as e:
547            raise self._translate_fm_error(e, messages) from e
548
549        yield self._prepare_response(prev, latency=time.time() - start)
550
551    async def _astream_chat_with_tools(
552        self, messages: MessageHistory, tools: List[Tool]
553    ) -> AsyncGenerator[str | Response, None]:
554        raise NotImplementedError(
555            "AppleFMLLM does not support tool calling; streaming with tools "
556            "is therefore also unavailable."
557        )
558        yield  # pragma: no cover - marks this as an async generator
559
560    async def _astream_structured(
561        self,
562        messages: MessageHistory,
563        schema: Type[BaseModel],
564    ) -> AsyncGenerator[str | Response, None]:
565        raise NotImplementedError(
566            "AppleFMLLM does not support streaming structured output. "
567            "Apple's `stream_response` does not accept guided generation. "
568            "Use `astructured` for buffered structured output."
569        )
570        yield  # pragma: no cover - marks this as an async generator

Apple's on-device system language model.

Requires macOS 26+ on Apple Silicon with Apple Intelligence enabled. The SDK is installed via pip install railtracks[apple]. Inference is serialized at the hardware level and runs entirely on-device.

See the module docstring for the deliberate scope: chat, structured output, and streaming chat are supported; tool calling and streaming structured output raise NotImplementedError.

Only the general on-device preset is used. Apple also ships a content_tagging preset tuned for classification-shaped outputs, but classification is not an agent-loop shape and doesn't fit railtracks' purpose. Users who need it should drop down to apple_fm_sdk directly.

Arguments:
  • temperature (float | None, optional): Sampling temperature. Passed through to fm.GenerationOptions. None uses the SDK default.
  • maximum_response_tokens (int | None, optional): Hard cap on the number of tokens the model may produce.
  • sampling_seed (int | None, optional): If set, sampling runs in SamplingMode.random(seed=...) for reproducible outputs across runs.
  • guardrails (bool, optional): When True (default) uses Apple's SystemLanguageModelGuardrails.DEFAULT. When False, switches to PERMISSIVE_CONTENT_TRANSFORMATIONS if the SDK exposes it. Safety refusals still raise AppleFMSafetyRefusalError in either mode.
  • retry_approach (RetryApproach | None, optional): Retry strategy passed to ModelBase for transient failures.
  • **kwargs: Rejected. Any non-None value here raises UnsupportedHyperparameterError — Apple's SDK does not accept top_p, penalties, or stop tokens, so silently accepting them would mislead callers.
Raises:
  • ImportError: apple_fm_sdk is not installed. Install with pip install railtracks[apple].
  • AppleFMUnavailableError: The on-device model is unavailable on this machine — unsupported OS/hardware, Apple Intelligence disabled, or assets not yet downloaded.
  • UnsupportedHyperparameterError: An unknown kwarg was passed with a non-None value.
AppleFMLLM( *, temperature: float | None = None, maximum_response_tokens: int | None = None, sampling_seed: int | None = None, guardrails: bool = True, retry_approach: railtracks.llm.retries.RetryApproach | None = None, **kwargs: Any)
159    def __init__(
160        self,
161        *,
162        temperature: float | None = None,
163        maximum_response_tokens: int | None = None,
164        sampling_seed: int | None = None,
165        guardrails: bool = True,
166        retry_approach: RetryApproach | None = None,
167        **kwargs: Any,
168    ):
169        try:
170            import apple_fm_sdk as fm
171        except ImportError as e:
172            raise ImportError(
173                "The `apple_fm_sdk` package is required to use AppleFMLLM. "
174                "Install with `pip install railtracks[apple]`. Note that the "
175                "package itself only runs on macOS 26+ Apple Silicon."
176            ) from e
177
178        for k, v in kwargs.items():
179            if v is not None:
180                raise UnsupportedHyperparameterError(
181                    model_name="apple-fm", hyperparameter=k, value=v
182                )
183
184        super().__init__(retry_approach=retry_approach)
185
186        self._fm = fm
187        self._guardrails = guardrails
188        self._temperature = temperature
189        self._maximum_response_tokens = maximum_response_tokens
190        self._sampling_seed = sampling_seed
191
192        self._model_handle = self._make_model_handle()
193        available, reason = self._model_handle.is_available()
194        if not available:
195            raise AppleFMUnavailableError(
196                f"Apple Foundation Model is not available on this device: {reason}"
197            )
def model_name(self) -> str:
247    def model_name(self) -> str:
248        """Return `"apple-fm"`. There is no preset suffix: the class
249        always uses Apple's `GENERAL` on-device preset.
250        """
251        return "apple-fm"

Return "apple-fm". There is no preset suffix: the class always uses Apple's GENERAL on-device preset.

def model_provider(self) -> ModelProvider:
253    def model_provider(self) -> ModelProvider:
254        """Return `ModelProvider.APPLE_FM`. See the enum's docstring."""
255        return ModelProvider.APPLE_FM

Return ModelProvider.APPLE_FM. See the enum's docstring.

@classmethod
def model_gateway(cls) -> ModelProvider:
257    @classmethod
258    def model_gateway(cls) -> ModelProvider:
259        """Return `ModelProvider.APPLE_FM`. Same as `model_provider`
260        because on-device Apple has no separate gateway routing.
261        """
262        return ModelProvider.APPLE_FM

Return ModelProvider.APPLE_FM. Same as model_provider because on-device Apple has no separate gateway routing.

def extract_message_info( self, latency: float | None = None) -> railtracks.llm.response.MessageInfo:
370    def extract_message_info(self, latency: float | None = None) -> MessageInfo:
371        """Populate MessageInfo from what Apple gives us — which is nothing.
372
373        Named to mirror `LiteLLMWrapper.extract_message_info` so the two files
374        read the same. `total_cost` is left `None` to match how other
375        providers report missing usage; on-device inference is genuinely
376        free but reporting `0.0` would silently conflate "free" with
377        "unknown" in mixed-provider aggregations.
378        """
379        return MessageInfo(
380            input_tokens=None,
381            output_tokens=None,
382            latency=latency,
383            model_name=self.model_name(),
384            total_cost=None,
385            system_fingerprint=None,
386        )

Populate MessageInfo from what Apple gives us — which is nothing.

Named to mirror LiteLLMWrapper.extract_message_info so the two files read the same. total_cost is left None to match how other providers report missing usage; on-device inference is genuinely free but reporting 0.0 would silently conflate "free" with "unknown" in mixed-provider aggregations.

class AzureAILLM(railtracks.llm.models._litellm_wrapper.LiteLLMWrapper):
 23class AzureAILLM(LiteLLMWrapper):
 24    """Azure Foundry LLM wrapper.
 25
 26    Accepts either litellm prefix:
 27    - ``azure/<deployment>`` — Azure OpenAI Service route; the string after the
 28      slash is the user-chosen deployment name and can be anything.
 29    - ``azure_ai/<model>`` — Azure AI Foundry model-inference route; the string
 30      after the slash is a model identifier from Foundry's catalog.
 31
 32    The model string is forwarded verbatim to litellm — no client-side validation
 33    is done against a static catalog, since deployment names are user-defined and
 34    can't be known ahead of time.
 35    """
 36
 37    @classmethod
 38    def model_gateway(cls):
 39        return ModelProvider.AZUREAI
 40
 41    def model_provider(self) -> ModelProvider:
 42        return self.model_gateway()
 43
 44    def __init__(
 45        self,
 46        model_name: str,
 47        *,
 48        temperature: float | None = None,
 49        top_p: float | None = None,
 50        max_tokens: int | None = None,
 51        frequency_penalty: float | None = None,
 52        presence_penalty: float | None = None,
 53        reasoning_effort: Literal["none", "minimal", "low", "medium", "high"]
 54        | None = None,
 55        service_tier: str | None = None,
 56        verbosity: Literal["low", "medium", "high"] | None = None,
 57        retry_approach: RetryApproach | None = None,
 58        **kwargs,
 59    ):
 60        """Initialize an Azure AI LLM instance.
 61
 62        Args:
 63            model_name (str): Full litellm model string, e.g. ``azure/my-deployment``
 64                or ``azure_ai/deepseek-r1``. See the class docstring for the
 65                difference between the two prefixes.
 66            temperature (float | None, optional): Sampling temperature for generation (e.g. 0.0–2.0).
 67                If None, the provider default is used.
 68            top_p (float | None, optional): Nucleus sampling threshold.
 69            max_tokens (int | None, optional): Maximum tokens to generate.
 70            frequency_penalty (float | None, optional): Penalizes tokens by how often
 71                they've already appeared.
 72            presence_penalty (float | None, optional): Penalizes tokens that have
 73                already appeared at all.
 74            reasoning_effort (Literal["none", "minimal", "low", "medium", "high"] | None, optional):
 75                Requested reasoning effort for reasoning-capable models.
 76            service_tier (str | None, optional): Requested service tier. Provider-specific.
 77            verbosity (Literal["low", "medium", "high"] | None, optional): Requested
 78                output verbosity for models that support it.
 79            retry_approach (RetryApproach | None, optional): Retry strategy for transient
 80                failures.
 81            **kwargs: Additional arguments passed to the parent LiteLLMWrapper.
 82
 83        Raises:
 84            AzureAIError: If the specified model is not available or if there are issues with the Azure AI service.
 85        """
 86        if kwargs.get("stream"):
 87            warn_pending_change(
 88                "Constructing a model with `stream=True`",
 89                change="is removed",
 90                instead="rt.astream(agent, ...) to stream an agent run",
 91                detail=(
 92                    "Streaming becomes async in 1.5.0: per-call model methods "
 93                    "(astream_chat, astream_chat_with_tools, astream_structured) "
 94                    "replace the streamed return value of chat()."
 95                ),
 96            )
 97
 98        super().__init__(
 99            model_name,
100            temperature=temperature,
101            top_p=top_p,
102            max_tokens=max_tokens,
103            frequency_penalty=frequency_penalty,
104            presence_penalty=presence_penalty,
105            reasoning_effort=reasoning_effort,
106            service_tier=service_tier,
107            verbosity=verbosity,
108            retry_approach=retry_approach,
109            **kwargs,
110        )
111        self.logger = logger
112
113    def chat(self, messages: MessageHistory):
114        try:
115            return super().chat(messages)
116        except InternalServerError as e:
117            raise AzureAIError(
118                reason=f"Azure AI LLM error while processing the request: {e}"
119            ) from e
120
121    def chat_with_tools(self, messages: MessageHistory, tools: List[Tool]):
122        try:
123            return super().chat_with_tools(messages, tools)
124        except InternalServerError as e:
125            raise AzureAIError(
126                reason=f"Azure AI LLM error while processing the request: {e}"
127            ) from e

Azure Foundry LLM wrapper.

Accepts either litellm prefix:

  • azure/<deployment> — Azure OpenAI Service route; the string after the slash is the user-chosen deployment name and can be anything.
  • azure_ai/<model> — Azure AI Foundry model-inference route; the string after the slash is a model identifier from Foundry's catalog.

The model string is forwarded verbatim to litellm — no client-side validation is done against a static catalog, since deployment names are user-defined and can't be known ahead of time.

AzureAILLM( model_name: str, *, temperature: float | None = None, top_p: float | None = None, max_tokens: int | None = None, frequency_penalty: float | None = None, presence_penalty: float | None = None, reasoning_effort: Optional[Literal['none', 'minimal', 'low', 'medium', 'high']] = None, service_tier: str | None = None, verbosity: Optional[Literal['low', 'medium', 'high']] = None, retry_approach: railtracks.llm.retries.RetryApproach | None = None, **kwargs)
 44    def __init__(
 45        self,
 46        model_name: str,
 47        *,
 48        temperature: float | None = None,
 49        top_p: float | None = None,
 50        max_tokens: int | None = None,
 51        frequency_penalty: float | None = None,
 52        presence_penalty: float | None = None,
 53        reasoning_effort: Literal["none", "minimal", "low", "medium", "high"]
 54        | None = None,
 55        service_tier: str | None = None,
 56        verbosity: Literal["low", "medium", "high"] | None = None,
 57        retry_approach: RetryApproach | None = None,
 58        **kwargs,
 59    ):
 60        """Initialize an Azure AI LLM instance.
 61
 62        Args:
 63            model_name (str): Full litellm model string, e.g. ``azure/my-deployment``
 64                or ``azure_ai/deepseek-r1``. See the class docstring for the
 65                difference between the two prefixes.
 66            temperature (float | None, optional): Sampling temperature for generation (e.g. 0.0–2.0).
 67                If None, the provider default is used.
 68            top_p (float | None, optional): Nucleus sampling threshold.
 69            max_tokens (int | None, optional): Maximum tokens to generate.
 70            frequency_penalty (float | None, optional): Penalizes tokens by how often
 71                they've already appeared.
 72            presence_penalty (float | None, optional): Penalizes tokens that have
 73                already appeared at all.
 74            reasoning_effort (Literal["none", "minimal", "low", "medium", "high"] | None, optional):
 75                Requested reasoning effort for reasoning-capable models.
 76            service_tier (str | None, optional): Requested service tier. Provider-specific.
 77            verbosity (Literal["low", "medium", "high"] | None, optional): Requested
 78                output verbosity for models that support it.
 79            retry_approach (RetryApproach | None, optional): Retry strategy for transient
 80                failures.
 81            **kwargs: Additional arguments passed to the parent LiteLLMWrapper.
 82
 83        Raises:
 84            AzureAIError: If the specified model is not available or if there are issues with the Azure AI service.
 85        """
 86        if kwargs.get("stream"):
 87            warn_pending_change(
 88                "Constructing a model with `stream=True`",
 89                change="is removed",
 90                instead="rt.astream(agent, ...) to stream an agent run",
 91                detail=(
 92                    "Streaming becomes async in 1.5.0: per-call model methods "
 93                    "(astream_chat, astream_chat_with_tools, astream_structured) "
 94                    "replace the streamed return value of chat()."
 95                ),
 96            )
 97
 98        super().__init__(
 99            model_name,
100            temperature=temperature,
101            top_p=top_p,
102            max_tokens=max_tokens,
103            frequency_penalty=frequency_penalty,
104            presence_penalty=presence_penalty,
105            reasoning_effort=reasoning_effort,
106            service_tier=service_tier,
107            verbosity=verbosity,
108            retry_approach=retry_approach,
109            **kwargs,
110        )
111        self.logger = logger

Initialize an Azure AI LLM instance.

Arguments:
  • model_name (str): Full litellm model string, e.g. azure/my-deployment or azure_ai/deepseek-r1. See the class docstring for the difference between the two prefixes.
  • temperature (float | None, optional): Sampling temperature for generation (e.g. 0.0–2.0). If None, the provider default is used.
  • top_p (float | None, optional): Nucleus sampling threshold.
  • max_tokens (int | None, optional): Maximum tokens to generate.
  • frequency_penalty (float | None, optional): Penalizes tokens by how often they've already appeared.
  • presence_penalty (float | None, optional): Penalizes tokens that have already appeared at all.
  • reasoning_effort (Literal["none", "minimal", "low", "medium", "high"] | None, optional): Requested reasoning effort for reasoning-capable models.
  • service_tier (str | None, optional): Requested service tier. Provider-specific.
  • verbosity (Literal["low", "medium", "high"] | None, optional): Requested output verbosity for models that support it.
  • retry_approach (RetryApproach | None, optional): Retry strategy for transient failures.
  • **kwargs: Additional arguments passed to the parent LiteLLMWrapper.
Raises:
  • AzureAIError: If the specified model is not available or if there are issues with the Azure AI service.
@classmethod
def model_gateway(cls):
37    @classmethod
38    def model_gateway(cls):
39        return ModelProvider.AZUREAI

Gets the API distrubutor of the model. Note nessecarily the same as the model itself.

E.g. if you are calling openai LLM through Azure AI foundry

def model_provider(self) -> ModelProvider:
41    def model_provider(self) -> ModelProvider:
42        return self.model_gateway()

The name of the provider of this model (The Company that owns the model)

logger
def chat(self, messages: MessageHistory):
113    def chat(self, messages: MessageHistory):
114        try:
115            return super().chat(messages)
116        except InternalServerError as e:
117            raise AzureAIError(
118                reason=f"Azure AI LLM error while processing the request: {e}"
119            ) from e

Chat with the model using the provided messages.

def chat_with_tools( self, messages: MessageHistory, tools: List[Tool]):
121    def chat_with_tools(self, messages: MessageHistory, tools: List[Tool]):
122        try:
123            return super().chat_with_tools(messages, tools)
124        except InternalServerError as e:
125            raise AzureAIError(
126                reason=f"Azure AI LLM error while processing the request: {e}"
127            ) from e

Chat with the model using the provided messages and tools.

class EchoLLM(railtracks.llm.ModelBase):
 46class EchoLLM(ModelBase):
 47    """Offline model that returns the message history it was given.
 48
 49    Each chat call returns one assistant message. The text is every received
 50    message, in order, one per line in its ``str()`` form (``role: content``),
 51    followed by ``This is an echo of the test model.`` The model never requests
 52    a tool call. A streamed chat call yields that same text as five slices,
 53    one per second, and then the full response. ``structured`` and
 54    ``astructured`` raise ``NotImplementedError``.
 55
 56    Creating or calling this model does not need an API key or network access.
 57    The reported usage is zero tokens, zero cost, and zero latency, and the
 58    model name is ``echo``.
 59    """
 60
 61    def model_name(self) -> str:
 62        """Return ``"echo"``.
 63
 64        Returns:
 65            str: The name ``"echo"``.
 66        """
 67        return "echo"
 68
 69    def model_provider(self) -> ModelProvider:
 70        """Return `ModelProvider.UNKNOWN`.
 71
 72        The echo model is not served by a provider.
 73
 74        Returns:
 75            ModelProvider: `ModelProvider.UNKNOWN`.
 76        """
 77        return ModelProvider.UNKNOWN
 78
 79    @classmethod
 80    def model_gateway(cls) -> ModelProvider:
 81        """Return `ModelProvider.UNKNOWN`.
 82
 83        The echo model is not reached through a gateway.
 84
 85        Returns:
 86            ModelProvider: `ModelProvider.UNKNOWN`.
 87        """
 88        return ModelProvider.UNKNOWN
 89
 90    def _message_info(self) -> MessageInfo:
 91        return MessageInfo(
 92            input_tokens=0,
 93            output_tokens=0,
 94            latency=0.0,
 95            model_name=self.model_name(),
 96            total_cost=0.0,
 97        )
 98
 99    def _echo(self, messages: MessageHistory) -> Response:
100        lines = [str(message) for message in messages]
101        lines.append(_ECHO_LINE)
102        return Response(
103            message=AssistantMessage("\n".join(lines)),
104            message_info=self._message_info(),
105        )
106
107    def _chat(self, messages: MessageHistory) -> Response:
108        """Return the echoed history as an assistant message.
109
110        Args:
111            messages: The conversation the caller sent.
112
113        Returns:
114            Response: One assistant message containing the compacted history.
115        """
116        return self._echo(messages)
117
118    def _chat_with_tools(self, messages: MessageHistory, tools: List[Tool]) -> Response:
119        """Return the echoed history and do not request a tool call.
120
121        Args:
122            messages: The conversation the caller sent.
123            tools: Tools offered to the model. None of them is requested.
124
125        Returns:
126            Response: The same echo `_chat` returns for `messages`.
127        """
128        return self._echo(messages)
129
130    def _structured(
131        self, messages: MessageHistory, schema: Type[BaseModel]
132    ) -> Response:
133        """Reject structured output.
134
135        Args:
136            messages: The conversation the caller sent.
137            schema: The pydantic model requested for the response.
138
139        Raises:
140            NotImplementedError: EchoLLM does not support structured output.
141        """
142        raise NotImplementedError("EchoLLM does not support structured output.")
143
144    async def _achat(self, messages: MessageHistory) -> Response:
145        """Asynchronous form of `_chat`.
146
147        Args:
148            messages: The conversation the caller sent.
149
150        Returns:
151            Response: The same response `_chat` returns.
152        """
153        return self._chat(messages)
154
155    async def _achat_with_tools(
156        self, messages: MessageHistory, tools: List[Tool]
157    ) -> Response:
158        """Asynchronous form of `_chat_with_tools`.
159
160        Args:
161            messages: The conversation the caller sent.
162            tools: Tools offered to the model. None of them is requested.
163
164        Returns:
165            Response: The same response `_chat_with_tools` returns.
166        """
167        return self._chat_with_tools(messages, tools)
168
169    async def _astructured(
170        self, messages: MessageHistory, schema: Type[BaseModel]
171    ) -> Response:
172        """Asynchronous form of `_structured`.
173
174        Args:
175            messages: The conversation the caller sent.
176            schema: The pydantic model requested for the response.
177
178        Raises:
179            NotImplementedError: EchoLLM does not support structured output.
180        """
181        return self._structured(messages, schema)
182
183    async def _astream_chat(
184        self, messages: MessageHistory
185    ) -> AsyncGenerator[str | Response, None]:
186        """Stream the echoed history over five seconds.
187
188        The text `_chat` returns is split into five slices and one is yielded
189        each second. Earlier slices take any leftover characters, so the
190        lengths differ by at most one. A text shorter than five characters
191        still yields five slices; the later ones are empty. Joining the slices
192        reproduces the response text. The last item is that same response, and
193        the reported usage stays at zero.
194
195        Args:
196            messages: The conversation the caller sent.
197
198        Yields:
199            str | Response: Five text slices, then the `Response` `_chat` returns.
200        """
201        response = self._echo(messages)
202        for chunk in _split_evenly(response.text, _STREAM_CHUNKS):
203            await asyncio.sleep(_STREAM_SECONDS / _STREAM_CHUNKS)
204            yield chunk
205        yield response
206
207    async def _astream_chat_with_tools(
208        self, messages: MessageHistory, tools: List[Tool]
209    ) -> AsyncGenerator[str | Response, None]:
210        """Stream the echoed history and do not request a tool call.
211
212        Args:
213            messages: The conversation the caller sent.
214            tools: Tools offered to the model. None of them is requested.
215
216        Yields:
217            str | Response: The same slices and final `Response` `_astream_chat` yields.
218        """
219        async for item in self._astream_chat(messages):
220            yield item

Offline model that returns the message history it was given.

Each chat call returns one assistant message. The text is every received message, in order, one per line in its str() form (role: content), followed by This is an echo of the test model. The model never requests a tool call. A streamed chat call yields that same text as five slices, one per second, and then the full response. structured and astructured raise NotImplementedError.

Creating or calling this model does not need an API key or network access. The reported usage is zero tokens, zero cost, and zero latency, and the model name is echo.

def model_name(self) -> str:
61    def model_name(self) -> str:
62        """Return ``"echo"``.
63
64        Returns:
65            str: The name ``"echo"``.
66        """
67        return "echo"

Return "echo".

Returns:

str: The name "echo".

def model_provider(self) -> ModelProvider:
69    def model_provider(self) -> ModelProvider:
70        """Return `ModelProvider.UNKNOWN`.
71
72        The echo model is not served by a provider.
73
74        Returns:
75            ModelProvider: `ModelProvider.UNKNOWN`.
76        """
77        return ModelProvider.UNKNOWN

Return ModelProvider.UNKNOWN.

The echo model is not served by a provider.

Returns:

ModelProvider: ModelProvider.UNKNOWN.

@classmethod
def model_gateway(cls) -> ModelProvider:
79    @classmethod
80    def model_gateway(cls) -> ModelProvider:
81        """Return `ModelProvider.UNKNOWN`.
82
83        The echo model is not reached through a gateway.
84
85        Returns:
86            ModelProvider: `ModelProvider.UNKNOWN`.
87        """
88        return ModelProvider.UNKNOWN

Return ModelProvider.UNKNOWN.

The echo model is not reached through a gateway.

Returns:

ModelProvider: ModelProvider.UNKNOWN.

class HuggingFaceLLM(railtracks.llm.models.api_providers._provider_wrapper.ProviderLLMWrapper):
 7class HuggingFaceLLM(ProviderLLMWrapper):
 8    def _pre_init_provider_check(self, model_name):
 9        """called by __init__ before the super call in ProviderLLMWrapper"""
10        # for huggingface models there is no good way of using `get_llm_provider` to check if the model is valid.
11        # so we are just goinog to add `huggingface/` to the model name in case it is not there.
12        # if the model name happens to be invalid, the error will be generated at runtime during `litellm.completion`. See `_litellm_wrapper.py`
13        if model_name.startswith(self.model_provider().lower()):
14            model_name = "/".join(model_name.split("/")[1:])
15        try:
16            assert len(model_name.split("/")) == 3, "Invalid model name"
17        except AssertionError as e:
18            raise ModelNotFoundError(
19                reason=e.args[0],
20                notes=[
21                    "Model name must be of the format `huggingface/<provider>/<hf_org_or_user>/<hf_model>` or `<provider>/<hf_org_or_user>/<hf_model>`",
22                    "We only support the huggingface Serverless Inference Provider Models.",
23                    "Provider List: https://docs.litellm.ai/docs/providers",
24                ],
25            )
26        return model_name
27
28    def model_provider(self) -> ModelProvider:
29        # TODO implement logic for all the possible providers attached the hugging face.
30        return ModelProvider.HUGGINGFACE
31
32    def _validate_tool_calling_support(self):
33        # NOTE: special exception case for huggingface
34        # Due to the wide range of huggingface models, `litellm.supports_function_calling` isn't always accurate.
35        # so we are just going to skip the check and the error (if any) will be generated at runtime during `litellm.completion`.
36        pass
37
38    @classmethod
39    def model_gateway(cls):
40        return ModelProvider.HUGGINGFACE

A large base class that wraps around a litellm model.

Note that the model object should be interacted with via the methods provided in the wrapper class:

Each individual API should implement the required abstract_methods in order to allow users to interact with a model of that type.

def model_provider(self) -> ModelProvider:
28    def model_provider(self) -> ModelProvider:
29        # TODO implement logic for all the possible providers attached the hugging face.
30        return ModelProvider.HUGGINGFACE

Returns the name of the provider

@classmethod
def model_gateway(cls):
38    @classmethod
39    def model_gateway(cls):
40        return ModelProvider.HUGGINGFACE

Gets the API distrubutor of the model. Note nessecarily the same as the model itself.

E.g. if you are calling openai LLM through Azure AI foundry

class OpenAILLM(railtracks.llm.models.api_providers._provider_wrapper.ProviderLLMWrapper):
 6class OpenAILLM(ProviderLLMWrapper):
 7    """
 8    A wrapper that provides access to the OPENAI API.
 9    """
10
11    @classmethod
12    def model_gateway(cls):
13        return ModelProvider.OPENAI

A wrapper that provides access to the OPENAI API.

@classmethod
def model_gateway(cls):
11    @classmethod
12    def model_gateway(cls):
13        return ModelProvider.OPENAI

Gets the API distrubutor of the model. Note nessecarily the same as the model itself.

E.g. if you are calling openai LLM through Azure AI foundry

class GeminiLLM(railtracks.llm.models.api_providers._provider_wrapper.ProviderLLMWrapper):
 6class GeminiLLM(ProviderLLMWrapper):
 7    def full_model_name(self, model_name: str) -> str:
 8        # for gemini models through litellm, we need 'gemini/{model_name}' format, but we do this after the checks in ProiLLMWrapper init
 9        return f"gemini/{model_name}"
10
11    @classmethod
12    def model_gateway(cls):
13        return ModelProvider.GEMINI  # litellm uses this for the provider for Gemini, we are using this in the checks in _provider_wrapper.py

A large base class that wraps around a litellm model.

Note that the model object should be interacted with via the methods provided in the wrapper class:

Each individual API should implement the required abstract_methods in order to allow users to interact with a model of that type.

def full_model_name(self, model_name: str) -> str:
7    def full_model_name(self, model_name: str) -> str:
8        # for gemini models through litellm, we need 'gemini/{model_name}' format, but we do this after the checks in ProiLLMWrapper init
9        return f"gemini/{model_name}"

After the provider is checked, this method is called to get the full model name

@classmethod
def model_gateway(cls):
11    @classmethod
12    def model_gateway(cls):
13        return ModelProvider.GEMINI  # litellm uses this for the provider for Gemini, we are using this in the checks in _provider_wrapper.py

Gets the API distrubutor of the model. Note nessecarily the same as the model itself.

E.g. if you are calling openai LLM through Azure AI foundry

class OllamaLLM(railtracks.llm.models._litellm_wrapper.LiteLLMWrapper):
 30class OllamaLLM(LiteLLMWrapper):
 31    def __init__(
 32        self,
 33        model_name: str,
 34        domain: Literal["default", "auto", "custom"] = "default",
 35        custom_domain: str | None = None,
 36        temperature: float | None = None,
 37        top_p: float | None = None,
 38        max_tokens: int | None = None,
 39        frequency_penalty: float | None = None,
 40        presence_penalty: float | None = None,
 41        reasoning_effort: Literal["none", "minimal", "low", "medium", "high"]
 42        | None = None,
 43        service_tier: str | None = None,
 44        verbosity: Literal["low", "medium", "high"] | None = None,
 45        retry_approach: RetryApproach | None = None,
 46        **kwargs,
 47    ):
 48        """Initialize an Ollama LLM instance.
 49
 50        Args:
 51            model_name (str): Name of the Ollama model to use.
 52            domain (Literal["default", "auto", "custom"], optional): The domain configuration mode.
 53                - "default": Uses the default localhost domain (http://localhost:11434)
 54                - "auto": Uses the OLLAMA_HOST environment variable, raises OllamaError if not set
 55                - "custom": Uses the provided custom_domain parameter, raises OllamaError if not provided
 56                Defaults to "default".
 57            custom_domain (str | None, optional): Custom domain URL to use when domain is set to "custom".
 58                Must be provided if domain="custom". Defaults to None.
 59            temperature (float | None, optional): Sampling temperature for generation (e.g. 0.0–2.0).
 60                If None, the provider default is used.
 61            top_p (float | None, optional): Nucleus sampling threshold.
 62            max_tokens (int | None, optional): Maximum tokens to generate.
 63            frequency_penalty (float | None, optional): Penalizes tokens by how often
 64                they've already appeared.
 65            presence_penalty (float | None, optional): Penalizes tokens that have
 66                already appeared at all.
 67            reasoning_effort (Literal["none", "minimal", "low", "medium", "high"] | None, optional):
 68                Requested reasoning effort for reasoning-capable models.
 69            service_tier (str | None, optional): Requested service tier. Provider-specific.
 70            verbosity (Literal["low", "medium", "high"] | None, optional): Requested
 71                output verbosity for models that support it.
 72            retry_approach (RetryApproach | None, optional): Retry strategy for transient
 73                failures.
 74            **kwargs: Additional arguments passed to the parent LiteLLMWrapper.
 75
 76        Raises:
 77            OllamaError: If:
 78                - domain is "auto" and OLLAMA_HOST environment variable is not set
 79                - domain is "custom" and custom_domain is not provided
 80                - specified model is not available on the server
 81            RequestException: If connection to Ollama server fails
 82        """
 83
 84        # litellm's `ollama` provider drops `tools`; only `ollama_chat` sends them (#1457).
 85        bare_model_name = model_name.removeprefix("ollama_chat/").removeprefix(
 86            "ollama/"
 87        )
 88        if not model_name.startswith(CHAT_PREFIX):
 89            logger.warning(
 90                f"Routing model name '{model_name}' as '{CHAT_PREFIX}{bare_model_name}' for Ollama"
 91            )
 92        model_name = f"{CHAT_PREFIX}{bare_model_name}"
 93        self._capability_model_name = f"ollama/{bare_model_name}"
 94        super().__init__(
 95            model_name=model_name,
 96            temperature=temperature,
 97            top_p=top_p,
 98            max_tokens=max_tokens,
 99            frequency_penalty=frequency_penalty,
100            presence_penalty=presence_penalty,
101            reasoning_effort=reasoning_effort,
102            service_tier=service_tier,
103            verbosity=verbosity,
104            retry_approach=retry_approach,
105            **kwargs,
106        )
107
108        match domain:
109            case "default":
110                self.domain = DEFAULT_DOMAIN
111            case "auto":
112                domain_from_env = os.getenv("OLLAMA_HOST")
113                if domain_from_env is None:
114                    raise OllamaError("OLLAMA_HOST environment variable not set")
115                self.domain = domain_from_env
116            case "custom":
117                if custom_domain is None:
118                    raise OllamaError(
119                        "Custom domain must be provided when domain is set to 'custom'"
120                    )
121                self.domain = custom_domain
122
123        self._run_check(
124            "api/tags"
125        )  # This will crash the workflow if Ollama is not setup properly
126
127    def _run_check(self, endpoint: str):
128        url = f"{self.domain}/{endpoint.lstrip('/')}"
129        try:
130            response = requests.get(url)
131            response.raise_for_status()
132
133            models = response.json()
134
135            model_names = {model["name"] for model in models["models"]}
136
137            model_name = self.model_name().rsplit("/", 1)[
138                -1
139            ]  # extract the model name if the provider is also included
140
141            if model_name not in model_names:
142                error_msg = f"{self.model_name()} not available on server {self.domain}. Avaiable models are: {model_names}"
143                logger.error(error_msg)
144                raise OllamaError(error_msg)
145
146        except OllamaError as e:
147            logger.error(e)
148            raise
149
150        except requests.exceptions.RequestException as e:
151            logger.error(e)
152            raise
153
154    def _to_litellm_attachment_content(
155        self, text: str, attachments: List[Attachment]
156    ) -> List[Dict[str, Any]]:
157        """Reject document attachments: litellm's Ollama transform drops them (#1628).
158
159        Raises:
160            AttachmentNotSupportedError: If any attachment is a document (e.g. a PDF).
161        """
162        document = next((a for a in attachments if a.modality == "document"), None)
163        if document is not None:
164            raise AttachmentNotSupportedError(
165                self._model_name,
166                document.mime_type,
167                "litellm's Ollama transform keeps only text and image parts. "
168                "Render the document pages to images first, or use a model that "
169                "supports documents.",
170            )
171        return super()._to_litellm_attachment_content(text, attachments)
172
173    def chat_with_tools(self, messages, tools):
174        # litellm's capability catalog is keyed on `ollama/`, not `ollama_chat/`.
175        if not supports_function_calling(model=self._capability_model_name):
176            raise FunctionCallingNotSupportedError(self._model_name)
177
178        return super().chat_with_tools(messages, tools)
179
180    @classmethod
181    def model_gateway(cls):
182        return ModelProvider.OLLAMA
183
184    def model_provider(self) -> ModelProvider:
185        """Returns the name of the provider"""
186        return self.model_gateway()

A large base class that wraps around a litellm model.

Note that the model object should be interacted with via the methods provided in the wrapper class:

Each individual API should implement the required abstract_methods in order to allow users to interact with a model of that type.

OllamaLLM( model_name: str, domain: Literal['default', 'auto', 'custom'] = 'default', custom_domain: str | None = None, temperature: float | None = None, top_p: float | None = None, max_tokens: int | None = None, frequency_penalty: float | None = None, presence_penalty: float | None = None, reasoning_effort: Optional[Literal['none', 'minimal', 'low', 'medium', 'high']] = None, service_tier: str | None = None, verbosity: Optional[Literal['low', 'medium', 'high']] = None, retry_approach: railtracks.llm.retries.RetryApproach | None = None, **kwargs)
 31    def __init__(
 32        self,
 33        model_name: str,
 34        domain: Literal["default", "auto", "custom"] = "default",
 35        custom_domain: str | None = None,
 36        temperature: float | None = None,
 37        top_p: float | None = None,
 38        max_tokens: int | None = None,
 39        frequency_penalty: float | None = None,
 40        presence_penalty: float | None = None,
 41        reasoning_effort: Literal["none", "minimal", "low", "medium", "high"]
 42        | None = None,
 43        service_tier: str | None = None,
 44        verbosity: Literal["low", "medium", "high"] | None = None,
 45        retry_approach: RetryApproach | None = None,
 46        **kwargs,
 47    ):
 48        """Initialize an Ollama LLM instance.
 49
 50        Args:
 51            model_name (str): Name of the Ollama model to use.
 52            domain (Literal["default", "auto", "custom"], optional): The domain configuration mode.
 53                - "default": Uses the default localhost domain (http://localhost:11434)
 54                - "auto": Uses the OLLAMA_HOST environment variable, raises OllamaError if not set
 55                - "custom": Uses the provided custom_domain parameter, raises OllamaError if not provided
 56                Defaults to "default".
 57            custom_domain (str | None, optional): Custom domain URL to use when domain is set to "custom".
 58                Must be provided if domain="custom". Defaults to None.
 59            temperature (float | None, optional): Sampling temperature for generation (e.g. 0.0–2.0).
 60                If None, the provider default is used.
 61            top_p (float | None, optional): Nucleus sampling threshold.
 62            max_tokens (int | None, optional): Maximum tokens to generate.
 63            frequency_penalty (float | None, optional): Penalizes tokens by how often
 64                they've already appeared.
 65            presence_penalty (float | None, optional): Penalizes tokens that have
 66                already appeared at all.
 67            reasoning_effort (Literal["none", "minimal", "low", "medium", "high"] | None, optional):
 68                Requested reasoning effort for reasoning-capable models.
 69            service_tier (str | None, optional): Requested service tier. Provider-specific.
 70            verbosity (Literal["low", "medium", "high"] | None, optional): Requested
 71                output verbosity for models that support it.
 72            retry_approach (RetryApproach | None, optional): Retry strategy for transient
 73                failures.
 74            **kwargs: Additional arguments passed to the parent LiteLLMWrapper.
 75
 76        Raises:
 77            OllamaError: If:
 78                - domain is "auto" and OLLAMA_HOST environment variable is not set
 79                - domain is "custom" and custom_domain is not provided
 80                - specified model is not available on the server
 81            RequestException: If connection to Ollama server fails
 82        """
 83
 84        # litellm's `ollama` provider drops `tools`; only `ollama_chat` sends them (#1457).
 85        bare_model_name = model_name.removeprefix("ollama_chat/").removeprefix(
 86            "ollama/"
 87        )
 88        if not model_name.startswith(CHAT_PREFIX):
 89            logger.warning(
 90                f"Routing model name '{model_name}' as '{CHAT_PREFIX}{bare_model_name}' for Ollama"
 91            )
 92        model_name = f"{CHAT_PREFIX}{bare_model_name}"
 93        self._capability_model_name = f"ollama/{bare_model_name}"
 94        super().__init__(
 95            model_name=model_name,
 96            temperature=temperature,
 97            top_p=top_p,
 98            max_tokens=max_tokens,
 99            frequency_penalty=frequency_penalty,
100            presence_penalty=presence_penalty,
101            reasoning_effort=reasoning_effort,
102            service_tier=service_tier,
103            verbosity=verbosity,
104            retry_approach=retry_approach,
105            **kwargs,
106        )
107
108        match domain:
109            case "default":
110                self.domain = DEFAULT_DOMAIN
111            case "auto":
112                domain_from_env = os.getenv("OLLAMA_HOST")
113                if domain_from_env is None:
114                    raise OllamaError("OLLAMA_HOST environment variable not set")
115                self.domain = domain_from_env
116            case "custom":
117                if custom_domain is None:
118                    raise OllamaError(
119                        "Custom domain must be provided when domain is set to 'custom'"
120                    )
121                self.domain = custom_domain
122
123        self._run_check(
124            "api/tags"
125        )  # This will crash the workflow if Ollama is not setup properly

Initialize an Ollama LLM instance.

Arguments:
  • model_name (str): Name of the Ollama model to use.
  • domain (Literal["default", "auto", "custom"], optional): The domain configuration mode.
    • "default": Uses the default localhost domain (http://localhost:11434)
    • "auto": Uses the OLLAMA_HOST environment variable, raises OllamaError if not set
    • "custom": Uses the provided custom_domain parameter, raises OllamaError if not provided Defaults to "default".
  • custom_domain (str | None, optional): Custom domain URL to use when domain is set to "custom". Must be provided if domain="custom". Defaults to None.
  • temperature (float | None, optional): Sampling temperature for generation (e.g. 0.0–2.0). If None, the provider default is used.
  • top_p (float | None, optional): Nucleus sampling threshold.
  • max_tokens (int | None, optional): Maximum tokens to generate.
  • frequency_penalty (float | None, optional): Penalizes tokens by how often they've already appeared.
  • presence_penalty (float | None, optional): Penalizes tokens that have already appeared at all.
  • reasoning_effort (Literal["none", "minimal", "low", "medium", "high"] | None, optional): Requested reasoning effort for reasoning-capable models.
  • service_tier (str | None, optional): Requested service tier. Provider-specific.
  • verbosity (Literal["low", "medium", "high"] | None, optional): Requested output verbosity for models that support it.
  • retry_approach (RetryApproach | None, optional): Retry strategy for transient failures.
  • **kwargs: Additional arguments passed to the parent LiteLLMWrapper.
Raises:
  • OllamaError: If:
    • domain is "auto" and OLLAMA_HOST environment variable is not set
    • domain is "custom" and custom_domain is not provided
    • specified model is not available on the server
  • RequestException: If connection to Ollama server fails
def chat_with_tools(self, messages, tools):
173    def chat_with_tools(self, messages, tools):
174        # litellm's capability catalog is keyed on `ollama/`, not `ollama_chat/`.
175        if not supports_function_calling(model=self._capability_model_name):
176            raise FunctionCallingNotSupportedError(self._model_name)
177
178        return super().chat_with_tools(messages, tools)

Chat with the model using the provided messages and tools.

@classmethod
def model_gateway(cls):
180    @classmethod
181    def model_gateway(cls):
182        return ModelProvider.OLLAMA

Gets the API distrubutor of the model. Note nessecarily the same as the model itself.

E.g. if you are calling openai LLM through Azure AI foundry

def model_provider(self) -> ModelProvider:
184    def model_provider(self) -> ModelProvider:
185        """Returns the name of the provider"""
186        return self.model_gateway()

Returns the name of the provider

class PortKeyLLM(railtracks.llm.OpenAICompatibleProvider):
12class PortKeyLLM(OpenAICompatibleProvider):
13    def __init__(
14        self,
15        model_name: str,
16        *,
17        api_key: str | None = None,
18        temperature: float | None = None,
19        top_p: float | None = None,
20        max_tokens: int | None = None,
21        frequency_penalty: float | None = None,
22        presence_penalty: float | None = None,
23        reasoning_effort: Literal["none", "minimal", "low", "medium", "high"]
24        | None = None,
25        service_tier: str | None = None,
26        verbosity: Literal["low", "medium", "high"] | None = None,
27        retry_approach: RetryApproach | None = None,
28        **kwargs: Any,
29    ):
30        try:
31            from portkey_ai import Portkey
32        except ImportError:
33            raise ImportError(
34                "Could not import portkey_ai package. Use railtracks[portkey]"
35            )
36
37        if api_key is None:
38            try:
39                api_key = os.environ["PORTKEY_API_KEY"]
40            except KeyError:
41                raise KeyError("Please set your PORTKEY_API_KEY in your .env file.")
42
43        portkey = Portkey(api_key=api_key)
44
45        super().__init__(
46            model_name,
47            api_base=portkey.base_url,
48            api_key=portkey.api_key,
49            temperature=temperature,
50            top_p=top_p,
51            max_tokens=max_tokens,
52            frequency_penalty=frequency_penalty,
53            presence_penalty=presence_penalty,
54            reasoning_effort=reasoning_effort,
55            service_tier=service_tier,
56            verbosity=verbosity,
57            retry_approach=retry_approach,
58            **kwargs,
59        )
60
61    @classmethod
62    def model_gateway(cls):
63        return ModelProvider.PORTKEY
64
65    def model_provider(self):
66        # TODO: Implement specialized logic to determine the model provider
67        return ModelProvider.PORTKEY

A large base class that wraps around a litellm model.

Note that the model object should be interacted with via the methods provided in the wrapper class:

Each individual API should implement the required abstract_methods in order to allow users to interact with a model of that type.

PortKeyLLM( model_name: str, *, api_key: str | None = None, temperature: float | None = None, top_p: float | None = None, max_tokens: int | None = None, frequency_penalty: float | None = None, presence_penalty: float | None = None, reasoning_effort: Optional[Literal['none', 'minimal', 'low', 'medium', 'high']] = None, service_tier: str | None = None, verbosity: Optional[Literal['low', 'medium', 'high']] = None, retry_approach: railtracks.llm.retries.RetryApproach | None = None, **kwargs: Any)
13    def __init__(
14        self,
15        model_name: str,
16        *,
17        api_key: str | None = None,
18        temperature: float | None = None,
19        top_p: float | None = None,
20        max_tokens: int | None = None,
21        frequency_penalty: float | None = None,
22        presence_penalty: float | None = None,
23        reasoning_effort: Literal["none", "minimal", "low", "medium", "high"]
24        | None = None,
25        service_tier: str | None = None,
26        verbosity: Literal["low", "medium", "high"] | None = None,
27        retry_approach: RetryApproach | None = None,
28        **kwargs: Any,
29    ):
30        try:
31            from portkey_ai import Portkey
32        except ImportError:
33            raise ImportError(
34                "Could not import portkey_ai package. Use railtracks[portkey]"
35            )
36
37        if api_key is None:
38            try:
39                api_key = os.environ["PORTKEY_API_KEY"]
40            except KeyError:
41                raise KeyError("Please set your PORTKEY_API_KEY in your .env file.")
42
43        portkey = Portkey(api_key=api_key)
44
45        super().__init__(
46            model_name,
47            api_base=portkey.base_url,
48            api_key=portkey.api_key,
49            temperature=temperature,
50            top_p=top_p,
51            max_tokens=max_tokens,
52            frequency_penalty=frequency_penalty,
53            presence_penalty=presence_penalty,
54            reasoning_effort=reasoning_effort,
55            service_tier=service_tier,
56            verbosity=verbosity,
57            retry_approach=retry_approach,
58            **kwargs,
59        )

Initialize an OpenAI-compatible gateway LLM instance (e.g. via PortKey).

See ProviderLLMWrapper.__init__ for the full per-hyperparameter description of the common hyperparameters below (top_p, max_tokens, frequency_penalty, presence_penalty, reasoning_effort, service_tier, verbosity).

Note:

Gateway-style providers can't be reliably introspected by litellm, so neither per-model hyperparameter support nor mutual-exclusion checks run here (see _validate_common_hyperparameter_support override below) — every hyperparameter, valid or not, is passed straight through and any error surfaces from the gateway or upstream provider directly.

@classmethod
def model_gateway(cls):
61    @classmethod
62    def model_gateway(cls):
63        return ModelProvider.PORTKEY

Gets the API distrubutor of the model. Note nessecarily the same as the model itself.

E.g. if you are calling openai LLM through Azure AI foundry

def model_provider(self):
65    def model_provider(self):
66        # TODO: Implement specialized logic to determine the model provider
67        return ModelProvider.PORTKEY

Returns the name of the provider

class OpenAICompatibleProvider(railtracks.llm.models.api_providers._provider_wrapper.ProviderLLMWrapper, abc.ABC):
10class OpenAICompatibleProvider(ProviderLLMWrapper, ABC):
11    def __init__(
12        self,
13        model_name: str,
14        *,
15        api_base: str,
16        api_key: str,
17        temperature: float | None = None,
18        top_p: float | None = None,
19        max_tokens: int | None = None,
20        frequency_penalty: float | None = None,
21        presence_penalty: float | None = None,
22        reasoning_effort: Literal["none", "minimal", "low", "medium", "high"]
23        | None = None,
24        service_tier: str | None = None,
25        verbosity: Literal["low", "medium", "high"] | None = None,
26        retry_approach: RetryApproach | None = None,
27        **kwargs: Any,
28    ):
29        """Initialize an OpenAI-compatible gateway LLM instance (e.g. via PortKey).
30
31        See `ProviderLLMWrapper.__init__` for the full per-hyperparameter description
32        of the common hyperparameters below (`top_p`, `max_tokens`, `frequency_penalty`,
33        `presence_penalty`, `reasoning_effort`, `service_tier`, `verbosity`).
34
35        Note:
36            Gateway-style providers can't be reliably introspected by litellm, so
37            neither per-model hyperparameter support nor mutual-exclusion checks run
38            here (see `_validate_common_hyperparameter_support` override below) —
39            every hyperparameter, valid or not, is passed straight through and any
40            error surfaces from the gateway or upstream provider directly.
41        """
42        # litellm needs to be told this is an OpenAI-compatible endpoint to route the
43        # call at all. Forced as a kwarg here instead of baked into the model name
44        # string, so the true model name survives for telemetry/visualization.
45        kwargs["custom_llm_provider"] = "openai"
46        super().__init__(
47            model_name,
48            api_base=api_base,
49            api_key=api_key,
50            temperature=temperature,
51            top_p=top_p,
52            max_tokens=max_tokens,
53            frequency_penalty=frequency_penalty,
54            presence_penalty=presence_penalty,
55            reasoning_effort=reasoning_effort,
56            service_tier=service_tier,
57            verbosity=verbosity,
58            retry_approach=retry_approach,
59            **kwargs,
60        )
61
62    def full_model_name(self, model_name: str) -> str:
63        return model_name
64
65    @classmethod
66    def model_gateway(cls) -> ModelProvider:
67        return ModelProvider.UNKNOWN
68
69    def _pre_init_provider_check(self, model_name: str):
70        # For OpenAI compatible providers, we skip the provider check since there is no way to do it.
71        return model_name
72
73    def _validate_tool_calling_support(self):
74        # For OpenAI compatible providers, we skip the tool calling support check since there is no way to do it.
75        return
76
77    def _validate_common_hyperparameter_support(self) -> None:
78        # For OpenAI compatible providers, litellm can't reliably introspect
79        # gateway-style providers, so we skip the common hyperparameter support check.
80        return

A large base class that wraps around a litellm model.

Note that the model object should be interacted with via the methods provided in the wrapper class:

Each individual API should implement the required abstract_methods in order to allow users to interact with a model of that type.

OpenAICompatibleProvider( model_name: str, *, api_base: str, api_key: str, temperature: float | None = None, top_p: float | None = None, max_tokens: int | None = None, frequency_penalty: float | None = None, presence_penalty: float | None = None, reasoning_effort: Optional[Literal['none', 'minimal', 'low', 'medium', 'high']] = None, service_tier: str | None = None, verbosity: Optional[Literal['low', 'medium', 'high']] = None, retry_approach: railtracks.llm.retries.RetryApproach | None = None, **kwargs: Any)
11    def __init__(
12        self,
13        model_name: str,
14        *,
15        api_base: str,
16        api_key: str,
17        temperature: float | None = None,
18        top_p: float | None = None,
19        max_tokens: int | None = None,
20        frequency_penalty: float | None = None,
21        presence_penalty: float | None = None,
22        reasoning_effort: Literal["none", "minimal", "low", "medium", "high"]
23        | None = None,
24        service_tier: str | None = None,
25        verbosity: Literal["low", "medium", "high"] | None = None,
26        retry_approach: RetryApproach | None = None,
27        **kwargs: Any,
28    ):
29        """Initialize an OpenAI-compatible gateway LLM instance (e.g. via PortKey).
30
31        See `ProviderLLMWrapper.__init__` for the full per-hyperparameter description
32        of the common hyperparameters below (`top_p`, `max_tokens`, `frequency_penalty`,
33        `presence_penalty`, `reasoning_effort`, `service_tier`, `verbosity`).
34
35        Note:
36            Gateway-style providers can't be reliably introspected by litellm, so
37            neither per-model hyperparameter support nor mutual-exclusion checks run
38            here (see `_validate_common_hyperparameter_support` override below) —
39            every hyperparameter, valid or not, is passed straight through and any
40            error surfaces from the gateway or upstream provider directly.
41        """
42        # litellm needs to be told this is an OpenAI-compatible endpoint to route the
43        # call at all. Forced as a kwarg here instead of baked into the model name
44        # string, so the true model name survives for telemetry/visualization.
45        kwargs["custom_llm_provider"] = "openai"
46        super().__init__(
47            model_name,
48            api_base=api_base,
49            api_key=api_key,
50            temperature=temperature,
51            top_p=top_p,
52            max_tokens=max_tokens,
53            frequency_penalty=frequency_penalty,
54            presence_penalty=presence_penalty,
55            reasoning_effort=reasoning_effort,
56            service_tier=service_tier,
57            verbosity=verbosity,
58            retry_approach=retry_approach,
59            **kwargs,
60        )

Initialize an OpenAI-compatible gateway LLM instance (e.g. via PortKey).

See ProviderLLMWrapper.__init__ for the full per-hyperparameter description of the common hyperparameters below (top_p, max_tokens, frequency_penalty, presence_penalty, reasoning_effort, service_tier, verbosity).

Note:

Gateway-style providers can't be reliably introspected by litellm, so neither per-model hyperparameter support nor mutual-exclusion checks run here (see _validate_common_hyperparameter_support override below) — every hyperparameter, valid or not, is passed straight through and any error surfaces from the gateway or upstream provider directly.

def full_model_name(self, model_name: str) -> str:
62    def full_model_name(self, model_name: str) -> str:
63        return model_name

After the provider is checked, this method is called to get the full model name

@classmethod
def model_gateway(cls) -> ModelProvider:
65    @classmethod
66    def model_gateway(cls) -> ModelProvider:
67        return ModelProvider.UNKNOWN

Gets the API distrubutor of the model. Note nessecarily the same as the model itself.

E.g. if you are calling openai LLM through Azure AI foundry

class Parameter(abc.ABC):
 64class Parameter(ABC):
 65    """
 66    Abstract Base Parameter class with default simple parameter behavior.
 67    """
 68
 69    param_type: Optional[Union[str, List[str]]] = None  # class var for default type
 70
 71    def __init__(
 72        self,
 73        name: str,
 74        description: Optional[str] = None,
 75        required: bool = True,
 76        default: Any = None,
 77        enum: Optional[List[Any]] = None,
 78        default_present: bool = False,
 79        param_type: ParameterTypeInput = None,
 80    ):
 81        """
 82        Initialize a Parameter instance.
 83
 84        Args:
 85            name (str): Name of the parameter.
 86            description (Optional[str]): Description of the parameter.
 87            required (bool): Whether the parameter is required.
 88            default (Any): Default value for the parameter.
 89            enum (Optional[List[Any]]): Allowed values for the parameter.
 90            default_present (bool): Whether a default value is explicitly set.
 91            param_type: JSON schema type string (e.g. ``\"string\"``), :class:`ParameterType`,
 92                a Python builtin type (e.g. ``str`` → ``\"string\"``), or a list for unions.
 93        """
 94        self.name = name
 95        self.description = description or ""
 96        self.required = required
 97        self.default = default
 98        self.enum = enum
 99        self.default_present = default_present
100        if param_type is not None:
101            if isinstance(param_type, list):
102                normalized = []
103                for pt in param_type:
104                    try:
105                        normalized.append(_normalize_param_type_scalar(pt))
106                    except ValueError as exc:
107                        raise ValueError(
108                            f"Invalid param_type for parameter '{name}': {exc}"
109                        ) from exc
110                self.param_type = normalized
111            else:
112                try:
113                    self.param_type = _normalize_param_type_scalar(param_type)
114                except ValueError as exc:
115                    raise ValueError(
116                        f"Invalid param_type for parameter '{name}': {exc}"
117                    ) from exc
118        elif hasattr(self, "param_type") and self.param_type is None:
119            self.param_type = None
120
121    def encode(self):
122        return self.to_json_schema()
123
124    def to_json_schema(self) -> Dict[str, Any]:
125        # Base dictionary with type and optional description
126        schema_dict: Dict[str, Any] = {
127            "type": self.param_type.value
128            if isinstance(self.param_type, ParameterType)
129            else self.param_type
130        }
131        if self.description:
132            schema_dict["description"] = self.description
133
134        # Handle enum
135        if self.enum:
136            schema_dict["enum"] = self.enum
137
138        # Handle default
139        # default can be None, 0, False; None means optional parameter
140        if self.default_present:
141            schema_dict["default"] = self.default
142        elif isinstance(self.param_type, list) and "null" in self.param_type:
143            schema_dict["default"] = None
144
145        return schema_dict
146
147    def __repr__(self) -> str:
148        return (
149            f"Parameter(name={self.name!r}, param_type={self.param_type!r}, "
150            f"description={self.description!r}, required={self.required!r}, "
151            f"default={self.default!r}, enum={self.enum!r})"
152        )

Abstract Base Parameter class with default simple parameter behavior.

Parameter( name: str, description: Optional[str] = None, required: bool = True, default: Any = None, enum: Optional[List[Any]] = None, default_present: bool = False, param_type: Union[str, railtracks.llm.tools.parameters._base.ParameterType, type, List[Union[str, railtracks.llm.tools.parameters._base.ParameterType, type]], NoneType] = None)
 71    def __init__(
 72        self,
 73        name: str,
 74        description: Optional[str] = None,
 75        required: bool = True,
 76        default: Any = None,
 77        enum: Optional[List[Any]] = None,
 78        default_present: bool = False,
 79        param_type: ParameterTypeInput = None,
 80    ):
 81        """
 82        Initialize a Parameter instance.
 83
 84        Args:
 85            name (str): Name of the parameter.
 86            description (Optional[str]): Description of the parameter.
 87            required (bool): Whether the parameter is required.
 88            default (Any): Default value for the parameter.
 89            enum (Optional[List[Any]]): Allowed values for the parameter.
 90            default_present (bool): Whether a default value is explicitly set.
 91            param_type: JSON schema type string (e.g. ``\"string\"``), :class:`ParameterType`,
 92                a Python builtin type (e.g. ``str`` → ``\"string\"``), or a list for unions.
 93        """
 94        self.name = name
 95        self.description = description or ""
 96        self.required = required
 97        self.default = default
 98        self.enum = enum
 99        self.default_present = default_present
100        if param_type is not None:
101            if isinstance(param_type, list):
102                normalized = []
103                for pt in param_type:
104                    try:
105                        normalized.append(_normalize_param_type_scalar(pt))
106                    except ValueError as exc:
107                        raise ValueError(
108                            f"Invalid param_type for parameter '{name}': {exc}"
109                        ) from exc
110                self.param_type = normalized
111            else:
112                try:
113                    self.param_type = _normalize_param_type_scalar(param_type)
114                except ValueError as exc:
115                    raise ValueError(
116                        f"Invalid param_type for parameter '{name}': {exc}"
117                    ) from exc
118        elif hasattr(self, "param_type") and self.param_type is None:
119            self.param_type = None

Initialize a Parameter instance.

Arguments:
  • name (str): Name of the parameter.
  • description (Optional[str]): Description of the parameter.
  • required (bool): Whether the parameter is required.
  • default (Any): Default value for the parameter.
  • enum (Optional[List[Any]]): Allowed values for the parameter.
  • default_present (bool): Whether a default value is explicitly set.
  • param_type: JSON schema type string (e.g. "string"), ParameterType, a Python builtin type (e.g. str → "string"), or a list for unions.
param_type: Union[str, List[str], NoneType] = None
name
description
required
default
enum
default_present
def encode(self):
121    def encode(self):
122        return self.to_json_schema()
def to_json_schema(self) -> Dict[str, Any]:
124    def to_json_schema(self) -> Dict[str, Any]:
125        # Base dictionary with type and optional description
126        schema_dict: Dict[str, Any] = {
127            "type": self.param_type.value
128            if isinstance(self.param_type, ParameterType)
129            else self.param_type
130        }
131        if self.description:
132            schema_dict["description"] = self.description
133
134        # Handle enum
135        if self.enum:
136            schema_dict["enum"] = self.enum
137
138        # Handle default
139        # default can be None, 0, False; None means optional parameter
140        if self.default_present:
141            schema_dict["default"] = self.default
142        elif isinstance(self.param_type, list) and "null" in self.param_type:
143            schema_dict["default"] = None
144
145        return schema_dict
class UnionParameter(railtracks.llm.Parameter):
 9class UnionParameter(Parameter):
10    """Parameter representing a union type."""
11
12    param_type: List[str]
13
14    def __init__(
15        self,
16        name: str,
17        options: List[Parameter],
18        description: Optional[str] = None,
19        required: bool = True,
20        default: Any = None,
21        enum: Optional[list] = None,
22        default_present: bool = False,
23    ):
24        """Initialize a UnionParameter instance.
25
26        Args:
27            name (str): Name of the parameter.
28            options (List[Parameter]): List of Parameter instances representing the union types.
29            description (Optional[str]): Description of the parameter.
30            required (bool): Whether the parameter is required.
31            default (Any): Default value for the parameter.
32            enum (Optional[list]): Allowed values for the parameter.
33            default_present (bool): Whether a default value is explicitly set.
34        """
35        super().__init__(name, description, required, default, enum, default_present)
36        self.options = options
37        for opt in options:
38            if isinstance(opt, UnionParameter):
39                raise TypeError(
40                    "UnionParameter cannot contain another UnionParameter in its options"
41                )
42
43        # param_type here is the list of inner types as strings, e.g. ["string", "null"]
44        # flatten and deduplicate types (order does not matter for schema)
45        flattened_types = []
46        for opt in options:
47            pt = opt.param_type
48            if hasattr(pt, "value"):
49                pt = pt.__getattribute__("value")
50            if isinstance(pt, list):
51                flattened_types.extend(p for p in pt if p is not None)
52            elif pt is not None:
53                flattened_types.append(pt)
54
55        # Deduplicate while preserving order
56        self.param_type = list(set(flattened_types))
57
58    def to_json_schema(self) -> Dict[str, Any]:
59        """Convert the union parameter to a JSON schema representation."""
60        schema = {
61            "anyOf": [opt.to_json_schema() for opt in self.options],
62        }
63
64        if self.description:
65            schema["description"] = self.description  # type: ignore
66
67        if self.default_present:
68            schema["default"] = self.default
69
70        return schema
71
72    def __repr__(self) -> str:
73        """Return a string representation of the UnionParameter."""
74        return (
75            f"UnionParameter(name={self.name!r}, options={self.options!r}, "
76            f"description={self.description!r}, required={self.required!r}, default={self.default!r})"
77        )

Parameter representing a union type.

UnionParameter( name: str, options: List[Parameter], description: Optional[str] = None, required: bool = True, default: Any = None, enum: Optional[list] = None, default_present: bool = False)
14    def __init__(
15        self,
16        name: str,
17        options: List[Parameter],
18        description: Optional[str] = None,
19        required: bool = True,
20        default: Any = None,
21        enum: Optional[list] = None,
22        default_present: bool = False,
23    ):
24        """Initialize a UnionParameter instance.
25
26        Args:
27            name (str): Name of the parameter.
28            options (List[Parameter]): List of Parameter instances representing the union types.
29            description (Optional[str]): Description of the parameter.
30            required (bool): Whether the parameter is required.
31            default (Any): Default value for the parameter.
32            enum (Optional[list]): Allowed values for the parameter.
33            default_present (bool): Whether a default value is explicitly set.
34        """
35        super().__init__(name, description, required, default, enum, default_present)
36        self.options = options
37        for opt in options:
38            if isinstance(opt, UnionParameter):
39                raise TypeError(
40                    "UnionParameter cannot contain another UnionParameter in its options"
41                )
42
43        # param_type here is the list of inner types as strings, e.g. ["string", "null"]
44        # flatten and deduplicate types (order does not matter for schema)
45        flattened_types = []
46        for opt in options:
47            pt = opt.param_type
48            if hasattr(pt, "value"):
49                pt = pt.__getattribute__("value")
50            if isinstance(pt, list):
51                flattened_types.extend(p for p in pt if p is not None)
52            elif pt is not None:
53                flattened_types.append(pt)
54
55        # Deduplicate while preserving order
56        self.param_type = list(set(flattened_types))

Initialize a UnionParameter instance.

Arguments:
  • name (str): Name of the parameter.
  • options (List[Parameter]): List of Parameter instances representing the union types.
  • description (Optional[str]): Description of the parameter.
  • required (bool): Whether the parameter is required.
  • default (Any): Default value for the parameter.
  • enum (Optional[list]): Allowed values for the parameter.
  • default_present (bool): Whether a default value is explicitly set.
param_type: List[str] = None
options
def to_json_schema(self) -> Dict[str, Any]:
58    def to_json_schema(self) -> Dict[str, Any]:
59        """Convert the union parameter to a JSON schema representation."""
60        schema = {
61            "anyOf": [opt.to_json_schema() for opt in self.options],
62        }
63
64        if self.description:
65            schema["description"] = self.description  # type: ignore
66
67        if self.default_present:
68            schema["default"] = self.default
69
70        return schema

Convert the union parameter to a JSON schema representation.

class ArrayParameter(railtracks.llm.Parameter):
 9class ArrayParameter(Parameter):
10    """Parameter representing an array type."""
11
12    param_type: ParameterType = ParameterType.ARRAY
13
14    def __init__(
15        self,
16        name: str,
17        items: Parameter,
18        description: Optional[str] = None,
19        required: bool = True,
20        default: Any = None,
21        max_items: Optional[int] = None,
22        additional_properties: bool = False,
23    ):
24        """Initialize an ArrayParameter instance.
25
26        Args:
27            name (str): Name of the parameter.
28            items (Parameter): Parameter instance describing the type of array elements.
29            description (Optional[str]): Description of the parameter.
30            required (bool): Whether the parameter is required.
31            default (Any): Default value for the parameter.
32            max_items (Optional[int]): Maximum number of items allowed in the array.
33            additional_properties (bool): Whether additional properties are allowed (relevant if items are objects).
34        """
35        super().__init__(name, description, required, default)
36        self.items = items
37        self.max_items = max_items
38        self.additional_properties = (
39            additional_properties  # might be relevant if items is object type
40        )
41
42    def to_json_schema(self) -> Dict[str, Any]:
43        """Convert the array parameter to a JSON schema representation."""
44        # Base property for items inside the array
45        items_schema = self.items.to_json_schema()
46
47        schema = {
48            "type": "array",
49            "items": items_schema,
50        }
51        if self.description:
52            schema["description"] = self.description
53
54        if self.max_items is not None:
55            schema["maxItems"] = self.max_items
56
57        # Set defaults and enum if present at the array level
58        if self.default is not None:
59            schema["default"] = self.default
60
61        # Note: enum on arrays is uncommon but if you want to support:
62        if self.enum:
63            schema["enum"] = self.enum
64
65        return schema
66
67    def __repr__(self) -> str:
68        """Return a string representation of the ArrayParameter."""
69        return (
70            f"ArrayParameter(name={self.name!r}, items={self.items!r}, "
71            f"description={self.description!r}, required={self.required!r}, "
72            f"default={self.default!r}, max_items={self.max_items!r}, "
73            f"additional_properties={self.additional_properties!r})"
74        )

Parameter representing an array type.

ArrayParameter( name: str, items: Parameter, description: Optional[str] = None, required: bool = True, default: Any = None, max_items: Optional[int] = None, additional_properties: bool = False)
14    def __init__(
15        self,
16        name: str,
17        items: Parameter,
18        description: Optional[str] = None,
19        required: bool = True,
20        default: Any = None,
21        max_items: Optional[int] = None,
22        additional_properties: bool = False,
23    ):
24        """Initialize an ArrayParameter instance.
25
26        Args:
27            name (str): Name of the parameter.
28            items (Parameter): Parameter instance describing the type of array elements.
29            description (Optional[str]): Description of the parameter.
30            required (bool): Whether the parameter is required.
31            default (Any): Default value for the parameter.
32            max_items (Optional[int]): Maximum number of items allowed in the array.
33            additional_properties (bool): Whether additional properties are allowed (relevant if items are objects).
34        """
35        super().__init__(name, description, required, default)
36        self.items = items
37        self.max_items = max_items
38        self.additional_properties = (
39            additional_properties  # might be relevant if items is object type
40        )

Initialize an ArrayParameter instance.

Arguments:
  • name (str): Name of the parameter.
  • items (Parameter): Parameter instance describing the type of array elements.
  • description (Optional[str]): Description of the parameter.
  • required (bool): Whether the parameter is required.
  • default (Any): Default value for the parameter.
  • max_items (Optional[int]): Maximum number of items allowed in the array.
  • additional_properties (bool): Whether additional properties are allowed (relevant if items are objects).
param_type: railtracks.llm.tools.parameters._base.ParameterType = <ParameterType.ARRAY: 'array'>
items
max_items
additional_properties
def to_json_schema(self) -> Dict[str, Any]:
42    def to_json_schema(self) -> Dict[str, Any]:
43        """Convert the array parameter to a JSON schema representation."""
44        # Base property for items inside the array
45        items_schema = self.items.to_json_schema()
46
47        schema = {
48            "type": "array",
49            "items": items_schema,
50        }
51        if self.description:
52            schema["description"] = self.description
53
54        if self.max_items is not None:
55            schema["maxItems"] = self.max_items
56
57        # Set defaults and enum if present at the array level
58        if self.default is not None:
59            schema["default"] = self.default
60
61        # Note: enum on arrays is uncommon but if you want to support:
62        if self.enum:
63            schema["enum"] = self.enum
64
65        return schema

Convert the array parameter to a JSON schema representation.

class ObjectParameter(railtracks.llm.Parameter):
 9class ObjectParameter(Parameter):
10    """Parameter representing an object type."""
11
12    param_type: ParameterType = ParameterType.OBJECT
13
14    def __init__(
15        self,
16        name: str,
17        properties: list[Parameter],
18        description: Optional[str] = None,
19        required: bool = True,
20        additional_properties: bool = False,
21        default: Any = None,
22    ):
23        """Initialize an ObjectParameter instance.
24
25        Args:
26            name (str): Name of the parameter.
27            properties (list[Parameter]): List of Parameter instances describing object properties.
28            description (Optional[str]): Description of the parameter.
29            required (bool): Whether the parameter is required.
30            additional_properties (bool): Whether additional properties are allowed.
31            default (Any): Default value for the parameter.
32        """
33        super().__init__(name, description, required, default)
34        self.properties = properties
35        self.additional_properties = additional_properties
36
37    def to_json_schema(self) -> Dict[str, Any]:
38        """Convert the object parameter to a JSON schema representation."""
39        schema = {
40            "type": "object",
41            "properties": {},
42            "additionalProperties": self.additional_properties,
43        }
44
45        if self.description:
46            schema["description"] = self.description
47
48        required_props = []
49        for prop in self.properties:
50            schema["properties"][prop.name] = prop.to_json_schema()
51            if prop.required:
52                required_props.append(prop.name)
53
54        if required_props:
55            schema["required"] = required_props
56
57        if self.default is not None:
58            schema["default"] = self.default
59
60        if self.enum:
61            schema["enum"] = self.enum
62
63        return schema
64
65    def __repr__(self) -> str:
66        """Return a string representation of the ObjectParameter."""
67        return (
68            f"ObjectParameter(name={self.name!r}, properties={self.properties!r}, "
69            f"description={self.description!r}, required={self.required!r}, "
70            f"additional_properties={self.additional_properties!r}, default={self.default!r})"
71        )

Parameter representing an object type.

ObjectParameter( name: str, properties: list[Parameter], description: Optional[str] = None, required: bool = True, additional_properties: bool = False, default: Any = None)
14    def __init__(
15        self,
16        name: str,
17        properties: list[Parameter],
18        description: Optional[str] = None,
19        required: bool = True,
20        additional_properties: bool = False,
21        default: Any = None,
22    ):
23        """Initialize an ObjectParameter instance.
24
25        Args:
26            name (str): Name of the parameter.
27            properties (list[Parameter]): List of Parameter instances describing object properties.
28            description (Optional[str]): Description of the parameter.
29            required (bool): Whether the parameter is required.
30            additional_properties (bool): Whether additional properties are allowed.
31            default (Any): Default value for the parameter.
32        """
33        super().__init__(name, description, required, default)
34        self.properties = properties
35        self.additional_properties = additional_properties

Initialize an ObjectParameter instance.

Arguments:
  • name (str): Name of the parameter.
  • properties (list[Parameter]): List of Parameter instances describing object properties.
  • description (Optional[str]): Description of the parameter.
  • required (bool): Whether the parameter is required.
  • additional_properties (bool): Whether additional properties are allowed.
  • default (Any): Default value for the parameter.
param_type: railtracks.llm.tools.parameters._base.ParameterType = <ParameterType.OBJECT: 'object'>
properties
additional_properties
def to_json_schema(self) -> Dict[str, Any]:
37    def to_json_schema(self) -> Dict[str, Any]:
38        """Convert the object parameter to a JSON schema representation."""
39        schema = {
40            "type": "object",
41            "properties": {},
42            "additionalProperties": self.additional_properties,
43        }
44
45        if self.description:
46            schema["description"] = self.description
47
48        required_props = []
49        for prop in self.properties:
50            schema["properties"][prop.name] = prop.to_json_schema()
51            if prop.required:
52                required_props.append(prop.name)
53
54        if required_props:
55            schema["required"] = required_props
56
57        if self.default is not None:
58            schema["default"] = self.default
59
60        if self.enum:
61            schema["enum"] = self.enum
62
63        return schema

Convert the object parameter to a JSON schema representation.

class RefParameter(railtracks.llm.Parameter):
 9class RefParameter(Parameter):
10    """Parameter representing a reference type."""
11
12    param_type: str = "object"  # referenced schemas are always 'object' type
13
14    def __init__(
15        self,
16        name: str,
17        ref_path: str,
18        description: Optional[str] = None,
19        required: bool = True,
20        default: Any = None,
21    ):
22        """Initialize a RefParameter instance.
23
24        Args:
25            name (str): Name of the parameter.
26            ref_path (str): Reference path to the schema definition.
27            description (Optional[str]): Description of the parameter.
28            required (bool): Whether the parameter is required.
29            default (Any): Default value for the parameter.
30        """
31        super().__init__(name, description, required, default)
32        self.ref_path = ref_path
33
34    def to_json_schema(self) -> Dict[str, Any]:
35        """Convert the reference parameter to a JSON schema representation."""
36        schema = {"$ref": self.ref_path}
37        if self.description:
38            schema["description"] = self.description
39
40        if self.default is not None:
41            schema["default"] = self.default
42
43        if self.enum:
44            schema["enum"] = self.enum
45
46        return schema
47
48    def __repr__(self) -> str:
49        """Return a string representation of the RefParameter."""
50        return (
51            f"RefParameter(name={self.name!r}, ref_path={self.ref_path!r}, "
52            f"description={self.description!r}, required={self.required!r}, default={self.default!r})"
53        )

Parameter representing a reference type.

RefParameter( name: str, ref_path: str, description: Optional[str] = None, required: bool = True, default: Any = None)
14    def __init__(
15        self,
16        name: str,
17        ref_path: str,
18        description: Optional[str] = None,
19        required: bool = True,
20        default: Any = None,
21    ):
22        """Initialize a RefParameter instance.
23
24        Args:
25            name (str): Name of the parameter.
26            ref_path (str): Reference path to the schema definition.
27            description (Optional[str]): Description of the parameter.
28            required (bool): Whether the parameter is required.
29            default (Any): Default value for the parameter.
30        """
31        super().__init__(name, description, required, default)
32        self.ref_path = ref_path

Initialize a RefParameter instance.

Arguments:
  • name (str): Name of the parameter.
  • ref_path (str): Reference path to the schema definition.
  • description (Optional[str]): Description of the parameter.
  • required (bool): Whether the parameter is required.
  • default (Any): Default value for the parameter.
param_type: str = 'object'
ref_path
def to_json_schema(self) -> Dict[str, Any]:
34    def to_json_schema(self) -> Dict[str, Any]:
35        """Convert the reference parameter to a JSON schema representation."""
36        schema = {"$ref": self.ref_path}
37        if self.description:
38            schema["description"] = self.description
39
40        if self.default is not None:
41            schema["default"] = self.default
42
43        if self.enum:
44            schema["enum"] = self.enum
45
46        return schema

Convert the reference parameter to a JSON schema representation.

class Response:
 64class Response:
 65    """
 66    A simple object that represents a response from a model. It includes specific detail about the returned message
 67    and any other additional information from the model.
 68    """
 69
 70    def __init__(
 71        self,
 72        message: Message[_T, Literal[Role.assistant]],
 73        message_info: MessageInfo = MessageInfo(),
 74    ):
 75        """
 76        Creates a new instance of a response object.
 77
 78        Args:
 79            message: The message that was returned as part of this.
 80            streamer: A generator that streams the response as a collection of chunked strings.
 81            message_info: Additional information about the message, such as input/output tokens and latency.
 82        """
 83        if message is not None and not isinstance(message, Message):
 84            raise TypeError(f"message must be of type Message, got {type(message)}")
 85        self._message = message
 86        self._message_info = message_info
 87
 88    @property
 89    def message(self):
 90        """
 91        Gets the message that was returned as part of this response.
 92
 93        If none exists, this will return None.
 94        """
 95        return self._message
 96
 97    @property
 98    def message_info(self) -> MessageInfo:
 99        """
100        Gets the message info that was returned as part of this response.
101
102        This object contains additional information about the message, such as input/output tokens and latency.
103        """
104        return self._message_info
105
106    @property
107    def text(self) -> str:
108        """
109        The plain-text content of the response message.
110
111        This is a typed convenience accessor for the common case where the model returned
112        text (e.g. the final `Response` of `astream_chat`).
113
114        Returns:
115            str: The message's text content.
116
117        Raises:
118            TypeError: If the message content is not a string (e.g. tool calls or a
119                structured/pydantic output). Access `message.content` directly for those.
120        """
121        content = self._message.content
122        if not isinstance(content, str):
123            raise TypeError(
124                f"Response content is not text; it is {type(content).__name__}. "
125                "Access `response.message.content` directly for non-text content."
126            )
127        return content
128
129    @property
130    def reasoning(self) -> str | None:
131        """
132        The model's reasoning/"thinking" text for this response, if the provider
133        returned any.
134
135        This is a convenience accessor for the human-readable reasoning; it is None
136        when the message carries no reasoning (a non-reasoning model, or one that
137        was not asked to expose its thinking). The structured, signed form is on
138        `response.message.thinking_blocks`.
139
140        Returns:
141            str | None: The reasoning text, or None if there is none.
142        """
143        return getattr(self._message, "reasoning_content", None)
144
145    def __str__(self):
146        if self._message is not None:
147            return str(self._message)
148        else:
149            return "Response(<no-data>)"
150
151    def __repr__(self):
152        return f"Response(message={self._message}, message_info={self._message_info})"
153
154    def encode(self):
155        # TODO: implement a more expansive serialization of the message info and message content, if needed
156        return {"message": self._message}

A simple object that represents a response from a model. It includes specific detail about the returned message and any other additional information from the model.

Response( message: Message[~_T, typing.Literal[<Role.assistant: 'assistant'>]], message_info: railtracks.llm.response.MessageInfo = MessageInfo(input_tokens=None, output_tokens=None, latency=None, model_name=None, total_cost=None, system_fingerprint=None))
70    def __init__(
71        self,
72        message: Message[_T, Literal[Role.assistant]],
73        message_info: MessageInfo = MessageInfo(),
74    ):
75        """
76        Creates a new instance of a response object.
77
78        Args:
79            message: The message that was returned as part of this.
80            streamer: A generator that streams the response as a collection of chunked strings.
81            message_info: Additional information about the message, such as input/output tokens and latency.
82        """
83        if message is not None and not isinstance(message, Message):
84            raise TypeError(f"message must be of type Message, got {type(message)}")
85        self._message = message
86        self._message_info = message_info

Creates a new instance of a response object.

Arguments:
  • message: The message that was returned as part of this.
  • streamer: A generator that streams the response as a collection of chunked strings.
  • message_info: Additional information about the message, such as input/output tokens and latency.
message
88    @property
89    def message(self):
90        """
91        Gets the message that was returned as part of this response.
92
93        If none exists, this will return None.
94        """
95        return self._message

Gets the message that was returned as part of this response.

If none exists, this will return None.

message_info: railtracks.llm.response.MessageInfo
 97    @property
 98    def message_info(self) -> MessageInfo:
 99        """
100        Gets the message info that was returned as part of this response.
101
102        This object contains additional information about the message, such as input/output tokens and latency.
103        """
104        return self._message_info

Gets the message info that was returned as part of this response.

This object contains additional information about the message, such as input/output tokens and latency.

text: str
106    @property
107    def text(self) -> str:
108        """
109        The plain-text content of the response message.
110
111        This is a typed convenience accessor for the common case where the model returned
112        text (e.g. the final `Response` of `astream_chat`).
113
114        Returns:
115            str: The message's text content.
116
117        Raises:
118            TypeError: If the message content is not a string (e.g. tool calls or a
119                structured/pydantic output). Access `message.content` directly for those.
120        """
121        content = self._message.content
122        if not isinstance(content, str):
123            raise TypeError(
124                f"Response content is not text; it is {type(content).__name__}. "
125                "Access `response.message.content` directly for non-text content."
126            )
127        return content

The plain-text content of the response message.

This is a typed convenience accessor for the common case where the model returned text (e.g. the final Response of astream_chat).

Returns:

str: The message's text content.

Raises:
  • TypeError: If the message content is not a string (e.g. tool calls or a structured/pydantic output). Access message.content directly for those.
reasoning: str | None
129    @property
130    def reasoning(self) -> str | None:
131        """
132        The model's reasoning/"thinking" text for this response, if the provider
133        returned any.
134
135        This is a convenience accessor for the human-readable reasoning; it is None
136        when the message carries no reasoning (a non-reasoning model, or one that
137        was not asked to expose its thinking). The structured, signed form is on
138        `response.message.thinking_blocks`.
139
140        Returns:
141            str | None: The reasoning text, or None if there is none.
142        """
143        return getattr(self._message, "reasoning_content", None)

The model's reasoning/"thinking" text for this response, if the provider returned any.

This is a convenience accessor for the human-readable reasoning; it is None when the message carries no reasoning (a non-reasoning model, or one that was not asked to expose its thinking). The structured, signed form is on response.message.thinking_blocks.

Returns:

str | None: The reasoning text, or None if there is none.

def encode(self):
154    def encode(self):
155        # TODO: implement a more expansive serialization of the message info and message content, if needed
156        return {"message": self._message}