railtracks.retrieval

Retrieval subsystem.

The runtime (RetrievalRuntime) orchestrates loading, chunking, embedding, storage, and retrieval. The ~railtracks.retrieval.stores.Store protocol is the storage contract; ~railtracks.retrieval.stores.VectorStore is the canonical implementation. The rest of the pipeline is provided by railtracks.retrieval.loaders, railtracks.retrieval.chunking, and railtracks.retrieval.embedding.

 1"""Retrieval subsystem.
 2
 3The runtime (:class:`RetrievalRuntime`) orchestrates loading, chunking,
 4embedding, storage, and retrieval. The :class:`~railtracks.retrieval.stores.Store`
 5protocol is the storage contract; :class:`~railtracks.retrieval.stores.VectorStore`
 6is the canonical implementation. The rest of the pipeline is provided
 7by :mod:`railtracks.retrieval.loaders`, :mod:`railtracks.retrieval.chunking`,
 8and :mod:`railtracks.retrieval.embedding`.
 9"""
10
11from .content_extraction import (
12    ContentExtractor,
13    JsonExtractor,
14    ProseExtractor,
15    StrExtractor,
16)
17from .embedding.models import EmbeddingFailure
18from .errors import EmbeddingModelMismatchError
19from .loaders.models import LoadFailure
20from .models import (
21    Chunk,
22    Document,
23    DocumentType,
24    EmbeddedChunk,
25    OCRResult,
26    RetrievalResult,
27    RetrievedChunk,
28)
29from .runtime import (
30    BatchIngested,
31    DocumentFailed,
32    DocumentSkipped,
33    IngestionStage,
34    IngestionStats,
35    RetrievalRuntime,
36)
37from .stores import Store, StoreEntry, StoreQuery, StoreScope, VectorStore
38
39__all__ = [
40    "BatchIngested",
41    "Chunk",
42    "ContentExtractor",
43    "Document",
44    "DocumentFailed",
45    "DocumentSkipped",
46    "DocumentType",
47    "EmbeddedChunk",
48    "EmbeddingFailure",
49    "EmbeddingModelMismatchError",
50    "IngestionStage",
51    "IngestionStats",
52    "JsonExtractor",
53    "LoadFailure",
54    "OCRResult",
55    "ProseExtractor",
56    "RetrievalResult",
57    "RetrievalRuntime",
58    "RetrievedChunk",
59    "Store",
60    "StoreEntry",
61    "StoreQuery",
62    "StoreScope",
63    "StrExtractor",
64    "VectorStore",
65]
@dataclass
class BatchIngested:
30@dataclass
31class BatchIngested:
32    """A batch of chunks that finished embedding and was written to the store.
33
34    ``batch_index`` is **per-document**: it starts at 0 for each document and
35    counts that document's batches (both successful and failed) in order. It
36    is not a run-global counter — to track overall progress, count events or
37    read ``IngestionStats``.
38
39    ``metrics`` carries the per-batch usage and timing reported by the
40    embedder (tokens, dollar cost, latency, vector count). Use it to track
41    per-ingest cost without having to wrap the embedder.
42    """
43
44    document_id: UUID
45    embedded_chunks: list[EmbeddedChunk]
46    batch_index: int
47    metrics: EmbeddingMetrics | None = None

A batch of chunks that finished embedding and was written to the store.

batch_index is per-document: it starts at 0 for each document and counts that document's batches (both successful and failed) in order. It is not a run-global counter — to track overall progress, count events or read IngestionStats.

metrics carries the per-batch usage and timing reported by the embedder (tokens, dollar cost, latency, vector count). Use it to track per-ingest cost without having to wrap the embedder.

BatchIngested( document_id: uuid.UUID, embedded_chunks: list[EmbeddedChunk], batch_index: int, metrics: railtracks.retrieval.embedding.models.EmbeddingMetrics | None = None)
document_id: uuid.UUID
embedded_chunks: list[EmbeddedChunk]
batch_index: int
metrics: railtracks.retrieval.embedding.models.EmbeddingMetrics | None = None
@dataclass
class Chunk:
74@dataclass
75class Chunk:
76    content: str
77    document_id: UUID
78    id: UUID = field(default_factory=uuid4)
79    index: int = 0
80    parent_chunk_id: UUID | None = None
81    offsets: tuple[int, int] | None = None
82    metadata: dict[str, Any] = field(default_factory=dict)
Chunk( content: str, document_id: uuid.UUID, id: uuid.UUID = <factory>, index: int = 0, parent_chunk_id: uuid.UUID | None = None, offsets: tuple[int, int] | None = None, metadata: dict[str, typing.Any] = <factory>)
content: str
document_id: uuid.UUID
id: uuid.UUID
index: int = 0
parent_chunk_id: uuid.UUID | None = None
offsets: tuple[int, int] | None = None
metadata: dict[str, typing.Any]
class ContentExtractor(typing.Protocol):
10class ContentExtractor(Protocol):
11    """Convert one loader field value into searchable text."""
12
13    def __call__(self, value: Any, /) -> str: ...

Convert one loader field value into searchable text.

ContentExtractor(*args, **kwargs)
1953def _no_init_or_replace_init(self, *args, **kwargs):
1954    cls = type(self)
1955
1956    if cls._is_protocol:
1957        raise TypeError('Protocols cannot be instantiated')
1958
1959    # Already using a custom `__init__`. No need to calculate correct
1960    # `__init__` to call. This can lead to RecursionError. See bpo-45121.
1961    if cls.__init__ is not _no_init_or_replace_init:
1962        return
1963
1964    # Initially, `__init__` of a protocol subclass is set to `_no_init_or_replace_init`.
1965    # The first instantiation of the subclass will call `_no_init_or_replace_init` which
1966    # searches for a proper new `__init__` in the MRO. The new `__init__`
1967    # replaces the subclass' old `__init__` (ie `_no_init_or_replace_init`). Subsequent
1968    # instantiation of the protocol subclass will thus use the new
1969    # `__init__` and no longer call `_no_init_or_replace_init`.
1970    for base in cls.__mro__:
1971        init = base.__dict__.get('__init__', _no_init_or_replace_init)
1972        if init is not _no_init_or_replace_init:
1973            cls.__init__ = init
1974            break
1975    else:
1976        # should not happen
1977        cls.__init__ = object.__init__
1978
1979    cls.__init__(self, *args, **kwargs)
@dataclass
class Document:
24@dataclass
25class Document:
26    """A unit of source content produced by a loader.
27
28    Attributes:
29        content: The decoded textual content of the document. Always a string;
30            binary loaders are responsible for their own decoding.
31        type: A :class:`DocumentType` describing the content format. Cloud
32            loaders infer it from the object's file extension; structured
33            loaders (CSV, SQL) set it explicitly.
34        id: Unique identifier. If not provided and ``source`` is set, derived
35            deterministically from ``source`` via UUID5 (RFC 4122 URL
36            namespace) so the same source yields the same id across processes
37            — required for the runtime's upsert (``delete_where`` on
38            ``document_id``) to find and clear the prior version when content
39            changes. Sourceless documents get a random UUID4 (no stable
40            identity → no upsert semantics).
41        source: The natural identifier of where this document came from —
42            a URI (``s3://bucket/key``, ``gs://bucket/name``, ``https://...``),
43            a file path, or a relational id. Cloud loaders always set this;
44            user-constructed documents may leave it ``None``.
45
46            Writers that derive a storage key (when no ``key_fn`` is supplied)
47            look here first; the cloud writers also strip their own URI prefix
48            so that "load from S3, write back to S3" produces a clean key
49            rather than a nested URI.
50        content_hash: SHA-256 of ``content``. Computed by the runtime at
51            ingest time; loaders should leave this ``None``. Used by
52            staleness-detection to skip re-embedding unchanged documents.
53        metadata: Arbitrary provider-specific or user-attached key-value data.
54            Loaders use this to expose details like ``bucket``, ``key``,
55            ``page``, ``row_index``, etc.
56    """
57
58    content: str
59    type: DocumentType = DocumentType.TEXT
60    id: UUID = _UNSET_DOCUMENT_ID
61    source: str | None = None
62    content_hash: str | None = None
63    metadata: dict[str, Any] = field(default_factory=dict)
64
65    def __post_init__(self) -> None:
66        if self.id == _UNSET_DOCUMENT_ID:
67            self.id = (
68                uuid5(NAMESPACE_URL, self.source)
69                if self.source is not None
70                else uuid4()
71            )

A unit of source content produced by a loader.

Attributes:
  • content: The decoded textual content of the document. Always a string; binary loaders are responsible for their own decoding.
  • type: A DocumentType describing the content format. Cloud loaders infer it from the object's file extension; structured loaders (CSV, SQL) set it explicitly.
  • id: Unique identifier. If not provided and source is set, derived deterministically from source via UUID5 (RFC 4122 URL namespace) so the same source yields the same id across processes — required for the runtime's upsert (delete_where on document_id) to find and clear the prior version when content changes. Sourceless documents get a random UUID4 (no stable identity → no upsert semantics).
  • source: The natural identifier of where this document came from — a URI (s3://bucket/key, gs://bucket/name, https://...), a file path, or a relational id. Cloud loaders always set this; user-constructed documents may leave it None.

    Writers that derive a storage key (when no key_fn is supplied) look here first; the cloud writers also strip their own URI prefix so that "load from S3, write back to S3" produces a clean key rather than a nested URI.

  • content_hash: SHA-256 of content. Computed by the runtime at ingest time; loaders should leave this None. Used by staleness-detection to skip re-embedding unchanged documents.
  • metadata: Arbitrary provider-specific or user-attached key-value data. Loaders use this to expose details like bucket, key, page, row_index, etc.
Document( content: str, type: DocumentType = <DocumentType.TEXT: 'text'>, id: uuid.UUID = UUID('00000000-0000-0000-0000-000000000000'), source: str | None = None, content_hash: str | None = None, metadata: dict[str, typing.Any] = <factory>)
content: str
type: DocumentType = <DocumentType.TEXT: 'text'>
id: uuid.UUID = UUID('00000000-0000-0000-0000-000000000000')
source: str | None = None
content_hash: str | None = None
metadata: dict[str, typing.Any]
@dataclass
class DocumentFailed:
50@dataclass
51class DocumentFailed:
52    """A document that could not be fully ingested.
53
54    ``stage`` says where it failed:
55
56    - ``"load"``: the loader could not read the source, so there is no
57      document and ``document_id`` is ``None``.
58    - ``"chunk"``: the chunker raised; nothing was written.
59    - ``"embed"``: at least one embedding batch failed or a chunk exceeded
60      ``max_tokens``. Successful batches *are* written, so the document is
61      now partial.
62    - ``"store"``: a store read or write raised. Batches written before the
63      error stay in the store, so the document may be partial.
64
65    Callers may want to retry, delete, or accept a partial document.
66    """
67
68    document_id: UUID | None
69    source: str | None
70    errors: list[Exception]
71    stage: IngestionStage

A document that could not be fully ingested.

stage says where it failed:

  • "load": the loader could not read the source, so there is no document and document_id is None.
  • "chunk": the chunker raised; nothing was written.
  • "embed": at least one embedding batch failed or a chunk exceeded max_tokens. Successful batches are written, so the document is now partial.
  • "store": a store read or write raised. Batches written before the error stay in the store, so the document may be partial.

Callers may want to retry, delete, or accept a partial document.

DocumentFailed( document_id: uuid.UUID | None, source: str | None, errors: list[Exception], stage: Literal['load', 'chunk', 'embed', 'store'])
document_id: uuid.UUID | None
source: str | None
errors: list[Exception]
stage: Literal['load', 'chunk', 'embed', 'store']
@dataclass
class DocumentSkipped:
74@dataclass
75class DocumentSkipped:
76    """A document skipped during ingest because the store already has an
77    entry with the same ``source_path`` and ``content_hash``."""
78
79    document_id: UUID
80    source: str | None
81    reason: str = "unchanged"

A document skipped during ingest because the store already has an entry with the same source_path and content_hash.

DocumentSkipped( document_id: uuid.UUID, source: str | None, reason: str = 'unchanged')
document_id: uuid.UUID
source: str | None
reason: str = 'unchanged'
class DocumentType(builtins.str, enum.Enum):
15class DocumentType(str, Enum):
16    TEXT = "text"
17    MARKDOWN = "markdown"
18    PDF = "pdf"
19    CSV = "csv"
20    JSON = "json"
21    JSONL = "jsonl"

str(object='') -> str str(bytes_or_buffer[, encoding[, errors]]) -> str

Create a new string object from the given object. If encoding or errors is specified, then the object must expose a data buffer that will be decoded using the given encoding and error handler. Otherwise, returns the result of object.__str__() (if defined) or repr(object). encoding defaults to sys.getdefaultencoding(). errors defaults to 'strict'.

TEXT = <DocumentType.TEXT: 'text'>
MARKDOWN = <DocumentType.MARKDOWN: 'markdown'>
PDF = <DocumentType.PDF: 'pdf'>
CSV = <DocumentType.CSV: 'csv'>
JSON = <DocumentType.JSON: 'json'>
JSONL = <DocumentType.JSONL: 'jsonl'>
@dataclass
class EmbeddedChunk:
85@dataclass
86class EmbeddedChunk:
87    chunk: Chunk
88    vector: list[float]
89    embedding_model: str
90    embedding_version: str | None = None
EmbeddedChunk( chunk: Chunk, vector: list[float], embedding_model: str, embedding_version: str | None = None)
chunk: Chunk
vector: list[float]
embedding_model: str
embedding_version: str | None = None
@dataclass
class EmbeddingFailure:
72@dataclass
73class EmbeddingFailure:
74    """A failed batch embedding attempt.
75
76    Attributes:
77        chunks: Source chunks that could not be embedded.
78        errors: Exceptions raised during embedding.
79    """
80
81    chunks: list[Chunk]
82    errors: list[Exception]

A failed batch embedding attempt.

Attributes:
  • chunks: Source chunks that could not be embedded.
  • errors: Exceptions raised during embedding.
EmbeddingFailure( chunks: list[Chunk], errors: list[Exception])
chunks: list[Chunk]
errors: list[Exception]
class EmbeddingModelMismatchError(builtins.RuntimeError):
 7class EmbeddingModelMismatchError(RuntimeError):
 8    """Raised when the runtime's embedder model differs from the store's.
 9
10    Mixing vectors from different embedding models silently produces
11    meaningless similarity scores, so the runtime fails loudly before
12    issuing the search.
13    """

Raised when the runtime's embedder model differs from the store's.

Mixing vectors from different embedding models silently produces meaningless similarity scores, so the runtime fails loudly before issuing the search.

IngestionStage = typing.Literal['load', 'chunk', 'embed', 'store']
@dataclass
class IngestionStats:
 84@dataclass
 85class IngestionStats:
 86    """Summary of a complete ingest run.
 87
 88    ``documents_loaded`` counts documents the loader produced; sources the
 89    loader could not read count only toward ``documents_failed``.
 90
 91    ``total_metrics`` accumulates per-batch ``EmbeddingMetrics`` (tokens,
 92    dollar cost, latency, vector count) across every successful batch in
 93    the run, so callers can read a single total for billing/observability.
 94    """
 95
 96    documents_loaded: int = 0
 97    documents_failed: int = 0
 98    documents_skipped: int = 0
 99    chunks_created: int = 0
100    chunks_embedded: int = 0
101    batches_failed: int = 0
102    batch_failures: list[EmbeddingFailure] = field(default_factory=list)
103    failed_documents: list[DocumentFailed] = field(default_factory=list)
104    total_metrics: EmbeddingMetrics = field(default_factory=EmbeddingMetrics)

Summary of a complete ingest run.

documents_loaded counts documents the loader produced; sources the loader could not read count only toward documents_failed.

total_metrics accumulates per-batch EmbeddingMetrics (tokens, dollar cost, latency, vector count) across every successful batch in the run, so callers can read a single total for billing/observability.

IngestionStats( documents_loaded: int = 0, documents_failed: int = 0, documents_skipped: int = 0, chunks_created: int = 0, chunks_embedded: int = 0, batches_failed: int = 0, batch_failures: list[EmbeddingFailure] = <factory>, failed_documents: list[DocumentFailed] = <factory>, total_metrics: railtracks.retrieval.embedding.models.EmbeddingMetrics = <factory>)
documents_loaded: int = 0
documents_failed: int = 0
documents_skipped: int = 0
chunks_created: int = 0
chunks_embedded: int = 0
batches_failed: int = 0
batch_failures: list[EmbeddingFailure]
failed_documents: list[DocumentFailed]
total_metrics: railtracks.retrieval.embedding.models.EmbeddingMetrics
class JsonExtractor:
23class JsonExtractor:
24    """Render values as valid JSON while preserving Unicode text."""
25
26    def __call__(self, value: Any) -> str:
27        return json.dumps(value, ensure_ascii=False)

Render values as valid JSON while preserving Unicode text.

@dataclass
class LoadFailure:
 7@dataclass
 8class LoadFailure:
 9    """A source that a loader could not read.
10
11    Yielded by ``BaseDocumentLoader.astream()`` in place of the documents a
12    source would have produced, so one unreadable file or object does not end
13    the stream. Documents the source yielded before it failed are kept; the
14    rest of that source is skipped.
15
16    Attributes:
17        source: The file path, object URI, or other identifier of the source,
18            in the same form the loader uses for ``Document.source``.
19        error: The exception raised while reading the source.
20    """
21
22    source: str
23    error: Exception

A source that a loader could not read.

Yielded by BaseDocumentLoader.astream() in place of the documents a source would have produced, so one unreadable file or object does not end the stream. Documents the source yielded before it failed are kept; the rest of that source is skipped.

Attributes:
  • source: The file path, object URI, or other identifier of the source, in the same form the loader uses for Document.source.
  • error: The exception raised while reading the source.
LoadFailure(source: str, error: Exception)
source: str
error: Exception
@dataclass
class OCRResult:
110@dataclass
111class OCRResult:
112    """Structured output from an OCR engine.
113
114    Attributes:
115        markdown: Full page text as markdown. Tables and headings are
116            represented in markdown syntax where the engine supports it.
117        json_result: Raw structured layout data from the engine, if available.
118            Schema is engine-specific; ``None`` when the engine does not
119            return structured layout data. For GLM-OCR this is the value of
120            ``PipelineResult.to_dict()["json_result"]``, which contains
121            bounding boxes, tables, and reading order.
122    """
123
124    markdown: str
125    json_result: dict[str, Any] | list[Any] | None = None
126
127    def to_text(self) -> str:
128        return self.markdown

Structured output from an OCR engine.

Attributes:
  • markdown: Full page text as markdown. Tables and headings are represented in markdown syntax where the engine supports it.
  • json_result: Raw structured layout data from the engine, if available. Schema is engine-specific; None when the engine does not return structured layout data. For GLM-OCR this is the value of PipelineResult.to_dict()["json_result"], which contains bounding boxes, tables, and reading order.
OCRResult( markdown: str, json_result: dict[str, typing.Any] | list[typing.Any] | None = None)
markdown: str
json_result: dict[str, typing.Any] | list[typing.Any] | None = None
def to_text(self) -> str:
127    def to_text(self) -> str:
128        return self.markdown
class ProseExtractor:
30class ProseExtractor:
31    """Flatten nested JSON-like values into readable, labelled text."""
32
33    def __call__(self, value: Any) -> str:
34        return self._render(value, nested=False)
35
36    def _render(self, value: Any, *, nested: bool = True) -> str:
37        if isinstance(value, dict):
38            rendered = "; ".join(
39                f"{key}: {self._render(item)}" for key, item in value.items()
40            )
41            return f"{{{rendered}}}" if nested else rendered
42        if isinstance(value, (list, tuple)):
43            return f"[{', '.join(self._render(item) for item in value)}]"
44        if value is None:
45            return "null"
46        if isinstance(value, bool):
47            return "true" if value else "false"
48        return str(value)

Flatten nested JSON-like values into readable, labelled text.

@dataclass
class RetrievalResult:
102@dataclass
103class RetrievalResult:
104    query: str
105    chunks: list[RetrievedChunk]
106    total_candidates: int | None = None
107    metadata: dict[str, Any] = field(default_factory=dict)
RetrievalResult( query: str, chunks: list[RetrievedChunk], total_candidates: int | None = None, metadata: dict[str, typing.Any] = <factory>)
query: str
chunks: list[RetrievedChunk]
total_candidates: int | None = None
metadata: dict[str, typing.Any]
class RetrievalRuntime:
131class RetrievalRuntime:
132    """Orchestrates loading, chunking, embedding, storage, and retrieval.
133
134    The runtime captures *how* to process documents (chunker + embedder +
135    store); the loader passed to :meth:`ingest` decides *what* to
136    process. A single runtime can ingest from multiple sources, mix
137    chunking strategies via separate runtimes against the same store,
138    and update existing documents by re-ingesting them. Multi-tenant
139    callers share one runtime and pass ``scope`` per :meth:`ingest` or
140    :meth:`retrieve` call.
141
142    Args:
143        chunker: Splits documents into chunks.
144        embedder: Embeds chunk text into vectors.
145        store: Receives written ``StoreEntry``s and serves similarity search.
146        batch_size: Items per embedding batch. Falls back to
147            ``embedder.default_batch_size`` when omitted; raises
148            ``ValueError`` at construction if neither is set.
149        on_ingest: Synchronous callback invoked with each ``IngestionEvent``
150            as it is yielded. Wrap in ``asyncio.create_task`` for async logging.
151        on_retrieve: Synchronous callback invoked with the query string and
152            the ``RetrievalResult`` after each retrieve call.
153        max_tokens: When set, chunks whose token count exceeds this limit
154            are dropped before embedding and reported via
155            ``EmbeddingFailure`` rather than being sent to the provider.
156            Requires ``tokenizer`` (defaults to ``TiktokenTokenizer``).
157        tokenizer: Tokenizer used to enforce ``max_tokens``. Defaults to
158            ``TiktokenTokenizer`` lazily when ``max_tokens`` is set.
159        on_error: What to do when one document fails to load, chunk, or
160            store. ``"skip"`` (default) reports it as a ``DocumentFailed``
161            and moves on to the next document. ``"raise"`` re-raises the
162            error and ends the run. Failed embedding batches are always
163            reported, not raised. ``EmbeddingModelMismatchError`` and errors
164            raised out of the loader's own stream always propagate, since
165            the run cannot usefully continue past them.
166
167    Raises:
168        ValueError: If ``on_error`` is not ``"skip"`` or ``"raise"``.
169    """
170
171    def __init__(
172        self,
173        chunker: Chunker,
174        embedder: Embedding,
175        store: Store,
176        *,
177        batch_size: int | None = None,
178        on_ingest: Callable[
179            [BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped], None
180        ]
181        | None = None,
182        on_retrieve: Callable[[str, RetrievalResult], None] | None = None,
183        max_tokens: int | None = None,
184        tokenizer: Tokenizer | None = None,
185        on_error: Literal["skip", "raise"] = "skip",
186    ) -> None:
187        if on_error not in ("skip", "raise"):
188            raise ValueError(f"on_error must be 'skip' or 'raise', got {on_error!r}")
189        self._chunker = chunker
190        self._embedder = embedder
191        self._store = store
192        self._batch_size = self._resolve_batch_size(batch_size, embedder)
193        self._on_ingest = on_ingest
194        self._on_retrieve = on_retrieve
195        self._max_tokens = max_tokens
196        self._on_error = on_error
197        if max_tokens is not None and tokenizer is None:
198            from .chunking.tokenization import TiktokenTokenizer
199
200            tokenizer = TiktokenTokenizer()
201        self._tokenizer = tokenizer
202        # Captured on the first successful embedded batch and checked at
203        # retrieve time; survives process restarts by lazy-seeding from
204        # an existing store entry on the first ingest/retrieve.
205        self._captured_model: str | None = None
206        self._seed_attempted: bool = False
207
208    @property
209    def store(self) -> Store:
210        return self._store
211
212    @property
213    def embedder(self) -> Embedding:
214        return self._embedder
215
216    @property
217    def chunker(self) -> Chunker:
218        return self._chunker
219
220    @property
221    def batch_size(self) -> int:
222        return self._batch_size
223
224    @property
225    def max_tokens(self) -> int | None:
226        return self._max_tokens
227
228    @property
229    def on_error(self) -> Literal["skip", "raise"]:
230        return self._on_error
231
232    @staticmethod
233    def _resolve_batch_size(batch_size: int | None, embedder: Embedding) -> int:
234        bs = batch_size if batch_size is not None else embedder.default_batch_size
235        if bs is None:
236            raise ValueError(
237                f"{type(embedder).__name__} does not declare a "
238                "default_batch_size. Pass batch_size= to RetrievalRuntime "
239                "or set default_batch_size on the embedder class."
240            )
241        return bs
242
243    async def ingest(
244        self,
245        loader: BaseDocumentLoader,
246        *,
247        scope: StoreScope | None = None,
248    ) -> AsyncGenerator[
249        BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped, None
250    ]:
251        """Stream loader → chunker → embedder → store, yielding per-batch events.
252
253        Args:
254            loader: Source of ``Document`` objects to ingest.
255            scope: Tag written onto every ``StoreEntry`` produced by this
256                call. Single-tenant callers can leave this ``None``.
257
258        Yields:
259            ``BatchIngested`` after each successful batch finishes writing,
260            ``EmbeddingFailure`` for any failed batch, ``DocumentSkipped``
261            for an unchanged document, and ``DocumentFailed`` once for each
262            document that could not be fully ingested (see its ``stage``).
263            Successful batches for a partially-failed document are still
264            written; ``DocumentFailed`` signals the partial state.
265
266        Raises:
267            Exception: Any per-document error when ``on_error="raise"``.
268            EmbeddingModelMismatchError: When the embedder's model differs
269                from the one the store was built with.
270        """
271        stats = IngestionStats()
272        async for event in self._ingest_with_stats(loader, stats, scope):
273            if self._on_ingest is not None:
274                self._on_ingest(event)
275            yield event
276
277    async def ingest_all(
278        self,
279        loader: BaseDocumentLoader,
280        *,
281        scope: StoreScope | None = None,
282    ) -> IngestionStats:
283        """Drain `ingest` and return aggregate counts."""
284        stats = IngestionStats()
285        async for event in self._ingest_with_stats(loader, stats, scope):
286            if self._on_ingest is not None:
287                self._on_ingest(event)
288        return stats
289
290    async def _ingest_with_stats(
291        self,
292        loader: BaseDocumentLoader,
293        stats: IngestionStats,
294        scope: StoreScope | None,
295    ) -> AsyncGenerator[
296        BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped, None
297    ]:
298        async with aclosing(loader.astream()) as stream:
299            async for item in stream:
300                if isinstance(item, LoadFailure):
301                    if self._on_error == "raise":
302                        raise item.error
303                    yield self._record_document_failed(
304                        None, item.source, [item.error], "load", stats
305                    )
306                    continue
307                async for event in self._ingest_document(item, stats, scope):
308                    yield event
309
310    async def _ingest_document(
311        self, doc: Document, stats: IngestionStats, scope: StoreScope | None
312    ) -> AsyncGenerator[
313        BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped, None
314    ]:
315        stats.documents_loaded += 1
316        progress = _DocumentProgress()
317        try:
318            async for event in self._process_document(doc, stats, progress, scope):
319                yield event
320        except EmbeddingModelMismatchError:
321            raise
322        except Exception as exc:
323            if self._on_error == "raise":
324                raise
325            progress.errors.append(exc)
326        if progress.errors:
327            yield self._record_document_failed(
328                doc.id, doc.source, progress.errors, progress.stage, stats
329            )
330
331    async def _process_document(
332        self,
333        doc: Document,
334        stats: IngestionStats,
335        progress: _DocumentProgress,
336        scope: StoreScope | None,
337    ) -> AsyncGenerator[
338        BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped, None
339    ]:
340        doc.content_hash = _content_hash(doc.content)
341
342        await self._ensure_captured_model_seeded()
343
344        if await self._is_complete_duplicate(doc, scope):
345            stats.documents_skipped += 1
346            yield DocumentSkipped(document_id=doc.id, source=doc.source)
347            return
348
349        progress.stage = "chunk"
350        chunks = await self._chunker.achunk(doc)
351        stats.chunks_created += len(chunks)
352        if not chunks:
353            return
354
355        self._stamp_staleness_metadata(doc, chunks)
356
357        # Token-size guard: drop oversized chunks before embedding to avoid
358        # provider 4xx errors. Each oversize chunk surfaces as an
359        # EmbeddingFailure carried into the document's accumulated errors.
360        progress.stage = "embed"
361        chunks, failures = self._split_oversized(chunks, stats)
362        for failure in failures:
363            progress.errors.extend(failure.errors)
364            yield failure
365        if not chunks:
366            return
367
368        # Stamp the final (post-token-guard) chunk count onto every chunk so a
369        # later staleness check can tell a complete document from a
370        # partially-written one. Every chunk carries the same total, so reading
371        # any one persisted chunk reveals how many were expected.
372        for chunk in chunks:
373            chunk.metadata["doc_chunk_count"] = len(chunks)
374
375        async for event in self._embed_and_store(doc, chunks, stats, progress, scope):
376            yield event
377
378    async def _is_complete_duplicate(
379        self, doc: Document, scope: StoreScope | None
380    ) -> bool:
381        """Whether the store already holds a *complete* copy of ``doc``.
382
383        Skip re-embedding only when as many chunks are present as the last
384        write expected. A partially-written document (some chunks present after
385        an interrupted ingest) has fewer than expected and is re-ingested rather
386        than left broken. find() is metadata-only (no vector search) and only
387        fetches a single entry; the presence check is a count() so no payloads
388        are transferred and large documents never require a single oversized
389        read (Chroma Cloud caps the per-get ``limit`` value).
390        """
391        if doc.source is None:
392            return False
393        stale_filters = {
394            "source_path": doc.source,
395            "content_hash": doc.content_hash,
396            **(scope.to_payload_filters() if scope is not None else {}),
397        }
398        existing = await self._store.find(stale_filters, limit=1)
399        if not existing:
400            return False
401        expected = existing[0].chunk_metadata.get("doc_chunk_count")
402        if expected is None:
403            # Not written by this runtime (every ingest stamps the count):
404            # completeness can't be verified, so re-ingest.
405            return False
406        return await self._store.count(stale_filters) >= expected
407
408    @staticmethod
409    def _stamp_staleness_metadata(doc: Document, chunks: list[Chunk]) -> None:
410        """Inject staleness-detection metadata into every chunk so future
411        `find` calls can identify whether this document has changed."""
412        for chunk in chunks:
413            if doc.source is not None:
414                chunk.metadata.setdefault("source_path", doc.source)
415            if doc.content_hash is not None:
416                chunk.metadata.setdefault("content_hash", doc.content_hash)
417
418    def _split_oversized(
419        self, chunks: list[Chunk], stats: IngestionStats
420    ) -> tuple[list[Chunk], list[EmbeddingFailure]]:
421        """Partition chunks into embeddable ones and per-chunk failures.
422
423        Returns ``(ok_chunks, failures)``; each oversize chunk becomes a
424        single-chunk ``EmbeddingFailure`` and is recorded in ``stats``.
425        """
426        if self._max_tokens is None or self._tokenizer is None:
427            return chunks, []
428        ok_chunks: list[Chunk] = []
429        failures: list[EmbeddingFailure] = []
430        for chunk in chunks:
431            tokens = self._tokenizer.count(chunk.content)
432            if tokens > self._max_tokens:
433                err = ValueError(
434                    f"chunk {chunk.id} has {tokens} tokens "
435                    f"(>{self._max_tokens}); dropped before embedding"
436                )
437                stats.batches_failed += 1
438                failure = EmbeddingFailure(chunks=[chunk], errors=[err])
439                stats.batch_failures.append(failure)
440                failures.append(failure)
441            else:
442                ok_chunks.append(chunk)
443        return ok_chunks, failures
444
445    async def _embed_and_store(
446        self,
447        doc: Document,
448        chunks: list[Chunk],
449        stats: IngestionStats,
450        progress: _DocumentProgress,
451        scope: StoreScope | None,
452    ) -> AsyncGenerator[
453        BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped, None
454    ]:
455        # batch_index is per-document: it counts batches (successful and
456        # failed) within this document and resets for the next one.
457        batch_index = 0
458        delete_done = False
459        async for batch in self._embedder.astream_batches(
460            chunks, batch_size=self._batch_size
461        ):
462            if isinstance(batch, EmbeddingResult):
463                # Check model BEFORE delete_where / write — a mismatch here
464                # must not corrupt the store by clearing prior chunks first.
465                self._check_model(batch.metrics.model)
466                progress.stage = "store"
467                if not delete_done:
468                    await self._store.delete_where(
469                        {
470                            "document_id": str(doc.id),
471                            **(scope.to_payload_filters() if scope is not None else {}),
472                        }
473                    )
474                    delete_done = True
475                for embedded in batch.chunks:
476                    self._capture_model(embedded)
477                    entry = StoreEntry.from_chunk(embedded, scope=scope)
478                    await self._store.write(entry)
479                progress.stage = "embed"
480                stats.chunks_embedded += len(batch.chunks)
481                stats.total_metrics = stats.total_metrics + batch.metrics
482                yield BatchIngested(
483                    document_id=doc.id,
484                    embedded_chunks=batch.chunks,
485                    batch_index=batch_index,
486                    metrics=batch.metrics,
487                )
488            else:
489                progress.errors.extend(batch.errors)
490                stats.batches_failed += 1
491                stats.batch_failures.append(batch)
492                yield batch
493            batch_index += 1
494
495    def _capture_model(self, embedded: EmbeddedChunk) -> None:
496        """Record the embedding model from the first successful chunk so later
497        retrieve() calls can enforce model consistency."""
498        if self._captured_model is None and embedded.embedding_model:
499            self._captured_model = embedded.embedding_model
500            logger.info(
501                "RetrievalRuntime captured embedding model %r "
502                "from first successful batch; subsequent retrieve() "
503                "calls will enforce this model.",
504                self._captured_model,
505            )
506
507    async def _ensure_captured_model_seeded(self) -> None:
508        """Lazily seed ``_captured_model`` from an existing store entry so the
509        guard survives across process restarts. ``StoreEntry.embedding_model``
510        is recorded on every persisted entry, so a single ``find`` call is
511        enough — no schema change required. Runs at most once per runtime;
512        a miss against an empty store sets ``_seed_attempted`` so we don't
513        re-query on every doc."""
514        if self._captured_model is not None or self._seed_attempted:
515            return
516        self._seed_attempted = True
517        existing = await self._store.find({}, limit=1)
518        if existing and existing[0].embedding_model:
519            self._captured_model = existing[0].embedding_model
520            logger.info(
521                "RetrievalRuntime seeded captured embedding model %r from "
522                "an existing store entry; mismatched embedders will raise.",
523                self._captured_model,
524            )
525
526    def _check_model(self, embed_model: str | None) -> None:
527        """Raise if ``embed_model`` disagrees with the captured model."""
528        if (
529            self._captured_model is not None
530            and embed_model
531            and embed_model != self._captured_model
532        ):
533            raise EmbeddingModelMismatchError(
534                f"Embedder produced vectors with model {embed_model!r} but "
535                f"store was built with {self._captured_model!r}. Similarity "
536                "scores across models are meaningless; rebuild the store "
537                "with the correct embedder or switch embedders."
538            )
539
540    @staticmethod
541    def _record_document_failed(
542        document_id: UUID | None,
543        source: str | None,
544        errors: list[Exception],
545        stage: IngestionStage,
546        stats: IngestionStats,
547    ) -> DocumentFailed:
548        failed = DocumentFailed(
549            document_id=document_id,
550            source=source,
551            errors=errors,
552            stage=stage,
553        )
554        stats.documents_failed += 1
555        stats.failed_documents.append(failed)
556        logger.warning("Could not ingest %s (stage %r): %r", source, stage, errors[-1])
557        return failed
558
559    async def delete_document(self, document_id: UUID) -> None:
560        """Remove all chunks for a document from the store.
561
562        Convenience wrapper around ``store.delete_where({"document_id": ...})``
563        so callers don't need to know the metadata key.
564        """
565        await self._store.delete_where({"document_id": str(document_id)})
566
567    async def retrieve(
568        self,
569        query: str,
570        top_k: int = 5,
571        metadata_filters: dict[str, Any] | None = None,
572        scope: StoreScope | None = None,
573    ) -> RetrievalResult:
574        """Embed ``query`` and return the top ``top_k`` matches from the store.
575
576        Args:
577            query: The text to embed and search with.
578            top_k: Maximum number of results.
579            metadata_filters: Additional equality filters on chunk metadata.
580            scope: Restricts the search to entries written with the same
581                scope. Leave ``None`` to search across all scopes.
582
583        Raises:
584            EmbeddingModelMismatchError: When the embedder reports a model
585                different from the one captured on first ingest.
586        """
587        await self._ensure_captured_model_seeded()
588        text_result = await self._embedder.aembed([query])
589        self._check_model(text_result.metrics.model)
590
591        store_query = StoreQuery(
592            text=query,
593            scope=scope,
594            embedding=text_result.vectors[0],
595            top_k=top_k,
596            metadata_filters=metadata_filters,
597        )
598        store_hits = await self._store.read(store_query)
599        chunks = [
600            RetrievedChunk(
601                chunk=_entry_to_chunk(hit.entry),
602                score=hit.score,
603                rank=hit.rank,
604                source_retriever=hit.source_retriever,
605                rerank_score=hit.rerank_score,
606            )
607            for hit in store_hits
608        ]
609        result = RetrievalResult(query=query, chunks=chunks)
610        if self._on_retrieve is not None:
611            self._on_retrieve(query, result)
612        return result

Orchestrates loading, chunking, embedding, storage, and retrieval.

The runtime captures how to process documents (chunker + embedder + store); the loader passed to ingest() decides what to process. A single runtime can ingest from multiple sources, mix chunking strategies via separate runtimes against the same store, and update existing documents by re-ingesting them. Multi-tenant callers share one runtime and pass scope per ingest() or retrieve() call.

Arguments:
  • chunker: Splits documents into chunks.
  • embedder: Embeds chunk text into vectors.
  • store: Receives written StoreEntrys and serves similarity search.
  • batch_size: Items per embedding batch. Falls back to embedder.default_batch_size when omitted; raises ValueError at construction if neither is set.
  • on_ingest: Synchronous callback invoked with each IngestionEvent as it is yielded. Wrap in asyncio.create_task for async logging.
  • on_retrieve: Synchronous callback invoked with the query string and the RetrievalResult after each retrieve call.
  • max_tokens: When set, chunks whose token count exceeds this limit are dropped before embedding and reported via EmbeddingFailure rather than being sent to the provider. Requires tokenizer (defaults to TiktokenTokenizer).
  • tokenizer: Tokenizer used to enforce max_tokens. Defaults to TiktokenTokenizer lazily when max_tokens is set.
  • on_error: What to do when one document fails to load, chunk, or store. "skip" (default) reports it as a DocumentFailed and moves on to the next document. "raise" re-raises the error and ends the run. Failed embedding batches are always reported, not raised. EmbeddingModelMismatchError and errors raised out of the loader's own stream always propagate, since the run cannot usefully continue past them.
Raises:
  • ValueError: If on_error is not "skip" or "raise".
RetrievalRuntime( chunker: railtracks.retrieval.chunking.base.Chunker, embedder: railtracks.retrieval.embedding.base.Embedding, store: Store, *, batch_size: int | None = None, on_ingest: Optional[Callable[[BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped], NoneType]] = None, on_retrieve: Optional[Callable[[str, RetrievalResult], NoneType]] = None, max_tokens: int | None = None, tokenizer: railtracks.retrieval.chunking.tokenization.Tokenizer | None = None, on_error: Literal['skip', 'raise'] = 'skip')
171    def __init__(
172        self,
173        chunker: Chunker,
174        embedder: Embedding,
175        store: Store,
176        *,
177        batch_size: int | None = None,
178        on_ingest: Callable[
179            [BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped], None
180        ]
181        | None = None,
182        on_retrieve: Callable[[str, RetrievalResult], None] | None = None,
183        max_tokens: int | None = None,
184        tokenizer: Tokenizer | None = None,
185        on_error: Literal["skip", "raise"] = "skip",
186    ) -> None:
187        if on_error not in ("skip", "raise"):
188            raise ValueError(f"on_error must be 'skip' or 'raise', got {on_error!r}")
189        self._chunker = chunker
190        self._embedder = embedder
191        self._store = store
192        self._batch_size = self._resolve_batch_size(batch_size, embedder)
193        self._on_ingest = on_ingest
194        self._on_retrieve = on_retrieve
195        self._max_tokens = max_tokens
196        self._on_error = on_error
197        if max_tokens is not None and tokenizer is None:
198            from .chunking.tokenization import TiktokenTokenizer
199
200            tokenizer = TiktokenTokenizer()
201        self._tokenizer = tokenizer
202        # Captured on the first successful embedded batch and checked at
203        # retrieve time; survives process restarts by lazy-seeding from
204        # an existing store entry on the first ingest/retrieve.
205        self._captured_model: str | None = None
206        self._seed_attempted: bool = False
store: Store
208    @property
209    def store(self) -> Store:
210        return self._store
embedder: railtracks.retrieval.embedding.base.Embedding
212    @property
213    def embedder(self) -> Embedding:
214        return self._embedder
chunker: railtracks.retrieval.chunking.base.Chunker
216    @property
217    def chunker(self) -> Chunker:
218        return self._chunker
batch_size: int
220    @property
221    def batch_size(self) -> int:
222        return self._batch_size
max_tokens: int | None
224    @property
225    def max_tokens(self) -> int | None:
226        return self._max_tokens
on_error: Literal['skip', 'raise']
228    @property
229    def on_error(self) -> Literal["skip", "raise"]:
230        return self._on_error
async def ingest( self, loader: railtracks.retrieval.loaders.base.BaseDocumentLoader, *, scope: StoreScope | None = None) -> AsyncGenerator[BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped, None]:
243    async def ingest(
244        self,
245        loader: BaseDocumentLoader,
246        *,
247        scope: StoreScope | None = None,
248    ) -> AsyncGenerator[
249        BatchIngested | EmbeddingFailure | DocumentFailed | DocumentSkipped, None
250    ]:
251        """Stream loader → chunker → embedder → store, yielding per-batch events.
252
253        Args:
254            loader: Source of ``Document`` objects to ingest.
255            scope: Tag written onto every ``StoreEntry`` produced by this
256                call. Single-tenant callers can leave this ``None``.
257
258        Yields:
259            ``BatchIngested`` after each successful batch finishes writing,
260            ``EmbeddingFailure`` for any failed batch, ``DocumentSkipped``
261            for an unchanged document, and ``DocumentFailed`` once for each
262            document that could not be fully ingested (see its ``stage``).
263            Successful batches for a partially-failed document are still
264            written; ``DocumentFailed`` signals the partial state.
265
266        Raises:
267            Exception: Any per-document error when ``on_error="raise"``.
268            EmbeddingModelMismatchError: When the embedder's model differs
269                from the one the store was built with.
270        """
271        stats = IngestionStats()
272        async for event in self._ingest_with_stats(loader, stats, scope):
273            if self._on_ingest is not None:
274                self._on_ingest(event)
275            yield event

Stream loader → chunker → embedder → store, yielding per-batch events.

Arguments:
  • loader: Source of Document objects to ingest.
  • scope: Tag written onto every StoreEntry produced by this call. Single-tenant callers can leave this None.
Yields:

BatchIngested after each successful batch finishes writing, EmbeddingFailure for any failed batch, DocumentSkipped for an unchanged document, and DocumentFailed once for each document that could not be fully ingested (see its stage). Successful batches for a partially-failed document are still written; DocumentFailed signals the partial state.

Raises:
  • Exception: Any per-document error when on_error="raise".
  • EmbeddingModelMismatchError: When the embedder's model differs from the one the store was built with.
async def ingest_all( self, loader: railtracks.retrieval.loaders.base.BaseDocumentLoader, *, scope: StoreScope | None = None) -> IngestionStats:
277    async def ingest_all(
278        self,
279        loader: BaseDocumentLoader,
280        *,
281        scope: StoreScope | None = None,
282    ) -> IngestionStats:
283        """Drain `ingest` and return aggregate counts."""
284        stats = IngestionStats()
285        async for event in self._ingest_with_stats(loader, stats, scope):
286            if self._on_ingest is not None:
287                self._on_ingest(event)
288        return stats

Drain ingest and return aggregate counts.

async def delete_document(self, document_id: uuid.UUID) -> None:
559    async def delete_document(self, document_id: UUID) -> None:
560        """Remove all chunks for a document from the store.
561
562        Convenience wrapper around ``store.delete_where({"document_id": ...})``
563        so callers don't need to know the metadata key.
564        """
565        await self._store.delete_where({"document_id": str(document_id)})

Remove all chunks for a document from the store.

Convenience wrapper around store.delete_where({"document_id": ...}) so callers don't need to know the metadata key.

async def retrieve( self, query: str, top_k: int = 5, metadata_filters: dict[str, typing.Any] | None = None, scope: StoreScope | None = None) -> RetrievalResult:
567    async def retrieve(
568        self,
569        query: str,
570        top_k: int = 5,
571        metadata_filters: dict[str, Any] | None = None,
572        scope: StoreScope | None = None,
573    ) -> RetrievalResult:
574        """Embed ``query`` and return the top ``top_k`` matches from the store.
575
576        Args:
577            query: The text to embed and search with.
578            top_k: Maximum number of results.
579            metadata_filters: Additional equality filters on chunk metadata.
580            scope: Restricts the search to entries written with the same
581                scope. Leave ``None`` to search across all scopes.
582
583        Raises:
584            EmbeddingModelMismatchError: When the embedder reports a model
585                different from the one captured on first ingest.
586        """
587        await self._ensure_captured_model_seeded()
588        text_result = await self._embedder.aembed([query])
589        self._check_model(text_result.metrics.model)
590
591        store_query = StoreQuery(
592            text=query,
593            scope=scope,
594            embedding=text_result.vectors[0],
595            top_k=top_k,
596            metadata_filters=metadata_filters,
597        )
598        store_hits = await self._store.read(store_query)
599        chunks = [
600            RetrievedChunk(
601                chunk=_entry_to_chunk(hit.entry),
602                score=hit.score,
603                rank=hit.rank,
604                source_retriever=hit.source_retriever,
605                rerank_score=hit.rerank_score,
606            )
607            for hit in store_hits
608        ]
609        result = RetrievalResult(query=query, chunks=chunks)
610        if self._on_retrieve is not None:
611            self._on_retrieve(query, result)
612        return result

Embed query and return the top top_k matches from the store.

Arguments:
  • query: The text to embed and search with.
  • top_k: Maximum number of results.
  • metadata_filters: Additional equality filters on chunk metadata.
  • scope: Restricts the search to entries written with the same scope. Leave None to search across all scopes.
Raises:
  • EmbeddingModelMismatchError: When the embedder reports a model different from the one captured on first ingest.
@dataclass
class RetrievedChunk:
93@dataclass
94class RetrievedChunk:
95    chunk: Chunk
96    score: float
97    rank: int
98    source_retriever: str | None = None
99    rerank_score: float | None = None
RetrievedChunk( chunk: Chunk, score: float, rank: int, source_retriever: str | None = None, rerank_score: float | None = None)
chunk: Chunk
score: float
rank: int
source_retriever: str | None = None
rerank_score: float | None = None
@runtime_checkable
class Store(typing.Protocol):
10@runtime_checkable
11class Store(Protocol):
12    async def write(self, entry: StoreEntry) -> str: ...
13    async def read(self, query: StoreQuery) -> list[RetrievedStoreEntry]: ...
14    async def delete(self, id: UUID) -> None: ...
15    async def clear(self, scope: StoreScope) -> None: ...
16    async def delete_where(self, filters: dict[str, Any]) -> None: ...
17    async def find(
18        self, filters: dict[str, Any], limit: int = 1
19    ) -> list[StoreEntry]: ...
20    async def count(self, filters: dict[str, Any] | None = None) -> int: ...

Base class for protocol classes.

Protocol classes are defined as::

class Proto(Protocol):
    def meth(self) -> int:
        ...

Such classes are primarily used with static type checkers that recognize structural subtyping (static duck-typing).

For example::

class C:
    def meth(self) -> int:
        return 0

def func(x: Proto) -> int:
    return x.meth()

func(C())  # Passes static type check

See PEP 544 for details. Protocol classes decorated with @typing.runtime_checkable act as simple-minded runtime protocols that check only the presence of given attributes, ignoring their type signatures. Protocol classes can be generic, they are defined as::

class GenProto(Protocol[T]):
    def meth(self) -> T:
        ...
Store(*args, **kwargs)
1953def _no_init_or_replace_init(self, *args, **kwargs):
1954    cls = type(self)
1955
1956    if cls._is_protocol:
1957        raise TypeError('Protocols cannot be instantiated')
1958
1959    # Already using a custom `__init__`. No need to calculate correct
1960    # `__init__` to call. This can lead to RecursionError. See bpo-45121.
1961    if cls.__init__ is not _no_init_or_replace_init:
1962        return
1963
1964    # Initially, `__init__` of a protocol subclass is set to `_no_init_or_replace_init`.
1965    # The first instantiation of the subclass will call `_no_init_or_replace_init` which
1966    # searches for a proper new `__init__` in the MRO. The new `__init__`
1967    # replaces the subclass' old `__init__` (ie `_no_init_or_replace_init`). Subsequent
1968    # instantiation of the protocol subclass will thus use the new
1969    # `__init__` and no longer call `_no_init_or_replace_init`.
1970    for base in cls.__mro__:
1971        init = base.__dict__.get('__init__', _no_init_or_replace_init)
1972        if init is not _no_init_or_replace_init:
1973            cls.__init__ = init
1974            break
1975    else:
1976        # should not happen
1977        cls.__init__ = object.__init__
1978
1979    cls.__init__(self, *args, **kwargs)
async def write(self, entry: StoreEntry) -> str:
12    async def write(self, entry: StoreEntry) -> str: ...
async def read( self, query: StoreQuery) -> list[railtracks.retrieval.stores.models.RetrievedStoreEntry]:
13    async def read(self, query: StoreQuery) -> list[RetrievedStoreEntry]: ...
async def delete(self, id: uuid.UUID) -> None:
14    async def delete(self, id: UUID) -> None: ...
async def clear(self, scope: StoreScope) -> None:
15    async def clear(self, scope: StoreScope) -> None: ...
async def delete_where(self, filters: dict[str, typing.Any]) -> None:
16    async def delete_where(self, filters: dict[str, Any]) -> None: ...
async def find( self, filters: dict[str, typing.Any], limit: int = 1) -> list[StoreEntry]:
17    async def find(
18        self, filters: dict[str, Any], limit: int = 1
19    ) -> list[StoreEntry]: ...
async def count(self, filters: dict[str, typing.Any] | None = None) -> int:
20    async def count(self, filters: dict[str, Any] | None = None) -> int: ...
@dataclass
class StoreEntry:
 44@dataclass
 45class StoreEntry:
 46    # Required fields
 47    id: UUID
 48    content: str
 49    vector: list[float] | None
 50    embedding_model: str
 51    chunk_id: UUID
 52    document_id: UUID
 53    # Optional enrichment fields
 54    abstract: str | None = None
 55    summary: str | None = None
 56    scope: StoreScope | None = None
 57    # Optional chunk provenance
 58    chunk_index: int = 0
 59    parent_chunk_id: UUID | None = None
 60    chunk_offsets: tuple[int, int] | None = None
 61    chunk_metadata: dict = field(default_factory=dict)
 62    # Optional embedding provenance
 63    embedding_version: str | None = None
 64    # Optional store metadata
 65    entities: list[Entity] | None = None
 66    valid_from: datetime | None = None
 67    valid_until: datetime | None = None
 68    created_at: datetime = field(default_factory=lambda: datetime.now(tz=timezone.utc))
 69
 70    @classmethod
 71    def from_chunk(
 72        cls,
 73        embedded_chunk: EmbeddedChunk,
 74        *,
 75        scope: StoreScope | None = None,
 76        abstract: str | None = None,
 77        summary: str | None = None,
 78        entities: list[Entity] | None = None,
 79        valid_from: datetime | None = None,
 80        valid_until: datetime | None = None,
 81    ) -> StoreEntry:
 82        chunk = embedded_chunk.chunk
 83        return cls(
 84            id=chunk.id,
 85            content=chunk.content,
 86            vector=embedded_chunk.vector,
 87            embedding_model=embedded_chunk.embedding_model,
 88            embedding_version=embedded_chunk.embedding_version,
 89            chunk_id=chunk.id,
 90            document_id=chunk.document_id,
 91            chunk_index=chunk.index,
 92            parent_chunk_id=chunk.parent_chunk_id,
 93            chunk_offsets=chunk.offsets,
 94            chunk_metadata=chunk.metadata,
 95            scope=scope,
 96            abstract=abstract,
 97            summary=summary,
 98            entities=entities,
 99            valid_from=valid_from,
100            valid_until=valid_until,
101        )
StoreEntry( id: uuid.UUID, content: str, vector: list[float] | None, embedding_model: str, chunk_id: uuid.UUID, document_id: uuid.UUID, abstract: str | None = None, summary: str | None = None, scope: StoreScope | None = None, chunk_index: int = 0, parent_chunk_id: uuid.UUID | None = None, chunk_offsets: tuple[int, int] | None = None, chunk_metadata: dict = <factory>, embedding_version: str | None = None, entities: list[railtracks.retrieval.stores.models.Entity] | None = None, valid_from: datetime.datetime | None = None, valid_until: datetime.datetime | None = None, created_at: datetime.datetime = <factory>)
id: uuid.UUID
content: str
vector: list[float] | None
embedding_model: str
chunk_id: uuid.UUID
document_id: uuid.UUID
abstract: str | None = None
summary: str | None = None
scope: StoreScope | None = None
chunk_index: int = 0
parent_chunk_id: uuid.UUID | None = None
chunk_offsets: tuple[int, int] | None = None
chunk_metadata: dict
embedding_version: str | None = None
entities: list[railtracks.retrieval.stores.models.Entity] | None = None
valid_from: datetime.datetime | None = None
valid_until: datetime.datetime | None = None
created_at: datetime.datetime
@classmethod
def from_chunk( cls, embedded_chunk: EmbeddedChunk, *, scope: StoreScope | None = None, abstract: str | None = None, summary: str | None = None, entities: list[railtracks.retrieval.stores.models.Entity] | None = None, valid_from: datetime.datetime | None = None, valid_until: datetime.datetime | None = None) -> StoreEntry:
 70    @classmethod
 71    def from_chunk(
 72        cls,
 73        embedded_chunk: EmbeddedChunk,
 74        *,
 75        scope: StoreScope | None = None,
 76        abstract: str | None = None,
 77        summary: str | None = None,
 78        entities: list[Entity] | None = None,
 79        valid_from: datetime | None = None,
 80        valid_until: datetime | None = None,
 81    ) -> StoreEntry:
 82        chunk = embedded_chunk.chunk
 83        return cls(
 84            id=chunk.id,
 85            content=chunk.content,
 86            vector=embedded_chunk.vector,
 87            embedding_model=embedded_chunk.embedding_model,
 88            embedding_version=embedded_chunk.embedding_version,
 89            chunk_id=chunk.id,
 90            document_id=chunk.document_id,
 91            chunk_index=chunk.index,
 92            parent_chunk_id=chunk.parent_chunk_id,
 93            chunk_offsets=chunk.offsets,
 94            chunk_metadata=chunk.metadata,
 95            scope=scope,
 96            abstract=abstract,
 97            summary=summary,
 98            entities=entities,
 99            valid_from=valid_from,
100            valid_until=valid_until,
101        )
@dataclass
class StoreQuery:
113@dataclass
114class StoreQuery:
115    text: str
116    scope: StoreScope | None = None
117    embedding: list[float] | None = None
118    top_k: int = 10
119    metadata_filters: dict[str, Any] | None = None
StoreQuery( text: str, scope: StoreScope | None = None, embedding: list[float] | None = None, top_k: int = 10, metadata_filters: dict[str, typing.Any] | None = None)
text: str
scope: StoreScope | None = None
embedding: list[float] | None = None
top_k: int = 10
metadata_filters: dict[str, typing.Any] | None = None
@dataclass(frozen=True)
class StoreScope:
21@dataclass(frozen=True)
22class StoreScope:
23    """Equality-filter namespace for store entries.
24
25    Each entry in ``labels`` becomes a mandatory equality filter on every
26    write and read. The retrieval module is agnostic about what dimensions
27    you scope by — pick whichever axes fit your tenancy model::
28
29        StoreScope(labels={"user_id": "alice"})  # SaaS tenancy
30        StoreScope(labels={"organization": "acme", "environment": "prod"})  # B2B
31        StoreScope(labels={"agent_id": "docs-bot", "session_id": "s1"})  # agent context
32        StoreScope(labels={"account_id": 42, "is_prod": True})  # non-string scalars
33
34    The ``scope_`` prefix applied in :meth:`to_payload_filters` avoids key
35    collisions in flat payload dicts that also carry content fields.
36    """
37
38    labels: Mapping[str, Any] = field(default_factory=dict)
39
40    def to_payload_filters(self) -> dict[str, Any]:
41        return {f"scope_{k}": v for k, v in self.labels.items()}

Equality-filter namespace for store entries.

Each entry in labels becomes a mandatory equality filter on every write and read. The retrieval module is agnostic about what dimensions you scope by — pick whichever axes fit your tenancy model::

StoreScope(labels={"user_id": "alice"})  # SaaS tenancy
StoreScope(labels={"organization": "acme", "environment": "prod"})  # B2B
StoreScope(labels={"agent_id": "docs-bot", "session_id": "s1"})  # agent context
StoreScope(labels={"account_id": 42, "is_prod": True})  # non-string scalars

The scope_ prefix applied in to_payload_filters() avoids key collisions in flat payload dicts that also carry content fields.

StoreScope(labels: Mapping[str, typing.Any] = <factory>)
labels: Mapping[str, typing.Any]
def to_payload_filters(self) -> dict[str, typing.Any]:
40    def to_payload_filters(self) -> dict[str, Any]:
41        return {f"scope_{k}": v for k, v in self.labels.items()}
class StrExtractor:
16class StrExtractor:
17    """Render values with Python's built-in string conversion."""
18
19    def __call__(self, value: Any) -> str:
20        return str(value)

Render values with Python's built-in string conversion.

class VectorStore:
237class VectorStore:
238    """Cosine similarity search over StoreEntry vectors.
239
240    Satisfies the Store protocol. Does not inherit from any base class.
241    """
242
243    def __init__(self, backend: VectorBackend) -> None:
244        self._backend = backend
245
246    async def write(self, entry: StoreEntry) -> str:
247        if entry.vector is None:
248            raise ValueError(
249                f"VectorStore.write requires entry.vector to be set "
250                f"(entry_id={entry.id}); embed the chunk before writing."
251            )
252        await self._backend.upsert(
253            str(entry.id), entry.vector, _entry_to_payload(entry)
254        )
255        return str(entry.id)
256
257    async def read(self, query: StoreQuery) -> list[RetrievedStoreEntry]:
258        if query.embedding is None:
259            raise ValueError(
260                "VectorStore.read requires query.embedding to be set; "
261                "caller must supply a pre-computed embedding."
262            )
263
264        filters: dict[str, Any] = (
265            query.scope.to_payload_filters() if query.scope is not None else {}
266        )
267        if query.metadata_filters:
268            filters.update(query.metadata_filters)
269
270        raw_hits = await self._backend.search(query.embedding, query.top_k, filters)
271
272        results: list[RetrievedStoreEntry] = []
273        for rank, (hit_id, score, payload) in enumerate(raw_hits):
274            entry = _payload_to_entry(hit_id, payload)
275            results.append(
276                RetrievedStoreEntry(
277                    entry=entry,
278                    score=score,
279                    rank=rank,
280                    source_retriever="dense",
281                )
282            )
283        return results
284
285    async def delete(self, id: UUID) -> None:
286        await self._backend.delete(str(id))
287
288    async def clear(self, scope: StoreScope) -> None:
289        await self._backend.delete_where(scope.to_payload_filters())
290
291    async def delete_where(self, filters: dict[str, Any]) -> None:
292        await self._backend.delete_where(filters)
293
294    async def find(self, filters: dict[str, Any], limit: int = 1) -> list[StoreEntry]:
295        raw_hits = await self._backend.list_where(filters, limit)
296        return [_payload_to_entry(hit_id, payload) for hit_id, payload in raw_hits]
297
298    async def count(self, filters: dict[str, Any] | None = None) -> int:
299        return await self._backend.count(filters or {})
300
301    async def nearest_neighbors(
302        self,
303        embedding: list[float],
304        k: int,
305        scope: StoreScope | None = None,
306    ) -> list[RetrievedStoreEntry]:
307        filters = scope.to_payload_filters() if scope is not None else {}
308        raw_hits = await self._backend.search(embedding, k, filters)
309
310        results: list[RetrievedStoreEntry] = []
311        for rank, (hit_id, score, payload) in enumerate(raw_hits):
312            entry = _payload_to_entry(hit_id, payload)
313            results.append(
314                RetrievedStoreEntry(
315                    entry=entry,
316                    score=score,
317                    rank=rank,
318                    source_retriever="dense",
319                )
320            )
321        return results

Cosine similarity search over StoreEntry vectors.

Satisfies the Store protocol. Does not inherit from any base class.

VectorStore(backend: railtracks.retrieval.stores.vector.base.VectorBackend)
243    def __init__(self, backend: VectorBackend) -> None:
244        self._backend = backend
async def write(self, entry: StoreEntry) -> str:
246    async def write(self, entry: StoreEntry) -> str:
247        if entry.vector is None:
248            raise ValueError(
249                f"VectorStore.write requires entry.vector to be set "
250                f"(entry_id={entry.id}); embed the chunk before writing."
251            )
252        await self._backend.upsert(
253            str(entry.id), entry.vector, _entry_to_payload(entry)
254        )
255        return str(entry.id)
async def read( self, query: StoreQuery) -> list[railtracks.retrieval.stores.models.RetrievedStoreEntry]:
257    async def read(self, query: StoreQuery) -> list[RetrievedStoreEntry]:
258        if query.embedding is None:
259            raise ValueError(
260                "VectorStore.read requires query.embedding to be set; "
261                "caller must supply a pre-computed embedding."
262            )
263
264        filters: dict[str, Any] = (
265            query.scope.to_payload_filters() if query.scope is not None else {}
266        )
267        if query.metadata_filters:
268            filters.update(query.metadata_filters)
269
270        raw_hits = await self._backend.search(query.embedding, query.top_k, filters)
271
272        results: list[RetrievedStoreEntry] = []
273        for rank, (hit_id, score, payload) in enumerate(raw_hits):
274            entry = _payload_to_entry(hit_id, payload)
275            results.append(
276                RetrievedStoreEntry(
277                    entry=entry,
278                    score=score,
279                    rank=rank,
280                    source_retriever="dense",
281                )
282            )
283        return results
async def delete(self, id: uuid.UUID) -> None:
285    async def delete(self, id: UUID) -> None:
286        await self._backend.delete(str(id))
async def clear(self, scope: StoreScope) -> None:
288    async def clear(self, scope: StoreScope) -> None:
289        await self._backend.delete_where(scope.to_payload_filters())
async def delete_where(self, filters: dict[str, typing.Any]) -> None:
291    async def delete_where(self, filters: dict[str, Any]) -> None:
292        await self._backend.delete_where(filters)
async def find( self, filters: dict[str, typing.Any], limit: int = 1) -> list[StoreEntry]:
294    async def find(self, filters: dict[str, Any], limit: int = 1) -> list[StoreEntry]:
295        raw_hits = await self._backend.list_where(filters, limit)
296        return [_payload_to_entry(hit_id, payload) for hit_id, payload in raw_hits]
async def count(self, filters: dict[str, typing.Any] | None = None) -> int:
298    async def count(self, filters: dict[str, Any] | None = None) -> int:
299        return await self._backend.count(filters or {})
async def nearest_neighbors( self, embedding: list[float], k: int, scope: StoreScope | None = None) -> list[railtracks.retrieval.stores.models.RetrievedStoreEntry]:
301    async def nearest_neighbors(
302        self,
303        embedding: list[float],
304        k: int,
305        scope: StoreScope | None = None,
306    ) -> list[RetrievedStoreEntry]:
307        filters = scope.to_payload_filters() if scope is not None else {}
308        raw_hits = await self._backend.search(embedding, k, filters)
309
310        results: list[RetrievedStoreEntry] = []
311        for rank, (hit_id, score, payload) in enumerate(raw_hits):
312            entry = _payload_to_entry(hit_id, payload)
313            results.append(
314                RetrievedStoreEntry(
315                    entry=entry,
316                    score=score,
317                    rank=rank,
318                    source_retriever="dense",
319                )
320            )
321        return results