Loaders

nlql.loaders —— 文档加载器：Loader Protocol + 按扩展名分派（.txt / .md / .docx / .pdf）。register_loader 可注册自定义格式。DOCX/PDF 需 nlql[loaders]。

loaders

Document loaders. Importing this package registers the built-in txt/md/docx/pdf loaders.

__all__ module-attribute

__all__ = ['Loader', 'load_documents', 'register_loader', 'loader_for', 'TextLoader', 'DocxLoader', 'PdfLoader']

Loader

Bases: Protocol

Reads a source path into one or more documents.

load

load(source: str | Path, *, metadata: dict[str, Any] | None = None) -> Iterable[Document]

源代码位于： src/nlql/loaders/base.py

def load(self, source: str | Path, *, metadata: dict[str, Any] | None = None) -> Iterable[Document]:
    ...

DocxLoader

Loads a .docx file's paragraph text as a single document.

load

load(source: str | Path, *, metadata: dict[str, Any] | None = None) -> Iterable[Document]

源代码位于： src/nlql/loaders/docx_loader.py

def load(
    self, source: str | Path, *, metadata: dict[str, Any] | None = None
) -> Iterable[Document]:
    try:
        import docx
    except ImportError as e:  # pragma: no cover - exercised only without the extra
        raise NLQLError("DocxLoader requires python-docx: pip install python-nlql[loaders]") from e
    path = Path(source)
    document = docx.Document(str(path))
    text = "\n".join(p.text for p in document.paragraphs if p.text and p.text.strip())
    yield Document.from_text(
        text, id=path.stem, metadata={**(metadata or {}), "source": str(path)}
    )

PdfLoader

PdfLoader(*, by_page: bool = False, extract_images: bool = False)

Loads a .pdf file's extracted text (and optionally its images).

源代码位于： src/nlql/loaders/pdf_loader.py

def __init__(self, *, by_page: bool = False, extract_images: bool = False) -> None:
    self._by_page = by_page
    self._extract_images = extract_images

load

load(source: str | Path, *, metadata: dict[str, Any] | None = None) -> Iterable[Document]

源代码位于： src/nlql/loaders/pdf_loader.py

def load(
    self, source: str | Path, *, metadata: dict[str, Any] | None = None
) -> Iterable[Document]:
    try:
        from pypdf import PdfReader
    except ImportError as e:  # pragma: no cover - exercised only without the extra
        raise NLQLError("PdfLoader requires pypdf: pip install python-nlql[loaders]") from e
    path = Path(source)
    reader = PdfReader(str(path))
    base = {**(metadata or {}), "source": str(path)}

    if self._extract_images:
        yield self._with_images(reader, path, base)
    elif self._by_page:
        for i, page in enumerate(reader.pages, start=1):
            text = page.extract_text() or ""
            if text.strip():
                yield Document.from_text(text, id=f"{path.stem}#p{i}", metadata={**base, "page": i})
    else:
        text = "\n".join((page.extract_text() or "") for page in reader.pages)
        yield Document.from_text(text, id=path.stem, metadata={**base, "pages": len(reader.pages)})

TextLoader

Loads a UTF-8 text file as a single document.

load

load(source: str | Path, *, metadata: dict[str, Any] | None = None) -> Iterable[Document]

源代码位于： src/nlql/loaders/text.py

def load(
    self, source: str | Path, *, metadata: dict[str, Any] | None = None
) -> Iterable[Document]:
    path = Path(source)
    text = path.read_text(encoding="utf-8")
    yield Document.from_text(
        text, id=path.stem, metadata={**(metadata or {}), "source": str(path)}
    )

load_documents

load_documents(source: str | Path, *, loader: Loader | None = None, metadata: dict[str, Any] | None = None) -> list[Document]

Load a file into documents, picking a loader by extension unless one is given.

源代码位于： src/nlql/loaders/base.py

def load_documents(
    source: str | Path,
    *,
    loader: Loader | None = None,
    metadata: dict[str, Any] | None = None,
) -> list[Document]:
    """Load a file into documents, picking a loader by extension unless one is given."""
    path = Path(source)
    chosen = loader if loader is not None else _LOADERS.get(path.suffix.lower())
    if chosen is None:
        raise NLQLError(
            f"no loader registered for {path.suffix!r}; pass loader= or register_loader(...)"
        )
    return list(chosen.load(path, metadata=metadata))

loader_for

loader_for(extension: str) -> Loader | None

源代码位于： src/nlql/loaders/base.py

def loader_for(extension: str) -> Loader | None:
    return _LOADERS.get(extension.lower())

register_loader

register_loader(loader: Loader, *extensions: str) -> None

Register loader for one or more file extensions (e.g. ".pdf").

源代码位于： src/nlql/loaders/base.py

def register_loader(loader: Loader, *extensions: str) -> None:
    """Register ``loader`` for one or more file extensions (e.g. ``".pdf"``)."""
    for ext in extensions:
        _LOADERS[ext.lower()] = loader