"""Generates reST source files for autosummary.
Usable as a library or script to generate automatic RST source files for
items referred to in autosummary:: directives.
Each generated RST file contains a single auto*:: directive which
extracts the docstring of the referred item.
Example Makefile rule::
generate:
sphinx-autogen -o source/generated source/*.rst
"""
from __future__ import annotations
import argparse
import importlib
import inspect
import locale
import pkgutil
import pydoc
import re
import sys
from pathlib import Path
from typing import TYPE_CHECKING, NamedTuple
from jinja2 import TemplateNotFound
from jinja2.sandbox import SandboxedEnvironment
import sphinx.locale
from sphinx import __display_version__, package_dir
from sphinx.builders import Builder
from sphinx.config import Config
from sphinx.errors import PycodeError
from sphinx.ext.autodoc._dynamic._importer import _import_module
from sphinx.ext.autodoc._dynamic._member_finder import _filter_enum_dict, unmangle
from sphinx.ext.autodoc._dynamic._mock import ismock, undecorate
from sphinx.ext.autodoc._sentinels import INSTANCE_ATTR, SLOTS_ATTR
from sphinx.ext.autosummary import (
ImportExceptionGroup,
_get_documenter,
import_by_name,
import_ivar_by_name,
)
from sphinx.locale import __
from sphinx.pycode import ModuleAnalyzer
from sphinx.registry import SphinxComponentRegistry
from sphinx.util import logging, rst
from sphinx.util._pathlib import _StrPath
from sphinx.util.inspect import (
getall,
getannotations,
getmro,
getslots,
isenumclass,
safe_getattr,
)
from sphinx.util.osutil import ensuredir
from sphinx.util.template import SphinxTemplateLoader
if TYPE_CHECKING:
import os
from collections.abc import Sequence, Set
from gettext import NullTranslations
from typing import Any
from sphinx.application import Sphinx
from sphinx.events import EventManager
from sphinx.ext.autodoc._property_types import _AutodocObjType
logger = logging.getLogger(__name__)
class _DummyEvents:
def emit_firstresult(self, *args: Any) -> None:
pass
class DummyApplication:
"""Dummy Application class for sphinx-autogen command."""
def __init__(self, translator: NullTranslations) -> None:
self.config = Config()
self.events = _DummyEvents()
self.registry = SphinxComponentRegistry()
self.messagelog: list[str] = []
self.srcdir = _StrPath('/')
self.translator = translator
self.verbosity = 0
self._warncount = 0
self._exception_on_warning = False
self.config.add('autosummary_context', {}, 'env', ())
self.config.add('autosummary_filename_map', {}, 'env', ())
self.config.add('autosummary_ignore_module_all', True, 'env', bool)
def emit_firstresult(self, *args: Any) -> None:
pass
class AutosummaryEntry(NamedTuple):
name: str
path: str | None
template: str
recursive: bool
def _underline(title: str, line: str = '=') -> str:
if '\n' in title:
msg = 'Can only underline single lines'
raise ValueError(msg)
return title + '\n' + line * len(title)
class AutosummaryRenderer:
"""A helper class for rendering."""
def __init__(self, app: Sphinx) -> None:
if isinstance(app, Builder):
msg = 'Expected a Sphinx application object!'
raise TypeError(msg)
system_templates_path = [
package_dir.joinpath('ext', 'autosummary', 'templates')
]
loader = SphinxTemplateLoader(
app.srcdir, app.config.templates_path, system_templates_path
)
self.env = SandboxedEnvironment(loader=loader)
self.env.filters['escape'] = rst.escape
self.env.filters['e'] = rst.escape
self.env.filters['underline'] = _underline
if app.translator:
self.env.add_extension('jinja2.ext.i18n')
# ``install_gettext_translations`` is injected by the ``jinja2.ext.i18n`` extension
self.env.install_gettext_translations(app.translator) # type: ignore[attr-defined]
def render(self, template_name: str, context: dict[str, Any]) -> str:
"""Render a template file."""
try:
template = self.env.get_template(template_name)
except TemplateNotFound:
try:
# objtype is given as template_name
template = self.env.get_template('autosummary/%s.rst' % template_name)
except TemplateNotFound:
# fallback to base.rst
template = self.env.get_template('autosummary/base.rst')
return template.render(context)
def _split_full_qualified_name(name: str) -> tuple[str | None, str]:
"""Split full qualified name to a pair of modname and qualname.
A qualname is an abbreviation for "Qualified name" introduced at PEP-3155
(https://peps.python.org/pep-3155/). It is a dotted path name
from the module top-level.
A "full" qualified name means a string containing both module name and
qualified name.
.. note:: This function actually imports the module to check its existence.
Therefore you need to mock 3rd party modules if needed before
calling this function.
"""
parts = name.split('.')
for i, _part in enumerate(parts, 1):
try:
modname = '.'.join(parts[:i])
importlib.import_module(modname)
except ImportError:
if parts[: i - 1]:
return '.'.join(parts[: i - 1]), '.'.join(parts[i - 1 :])
else:
return None, '.'.join(parts)
except IndexError:
pass
return name, ''
# -- Generating output ---------------------------------------------------------
class ModuleScanner:
def __init__(
self,
obj: Any,
*,
config: Config,
events: EventManager,
) -> None:
self.config = config
self.events = events
self.object = obj
def get_object_type(self, name: str, value: Any) -> str:
return _get_documenter(value, self.object)
def is_skipped(self, name: str, value: Any, obj_type: _AutodocObjType) -> bool:
try:
return self.events.emit_firstresult(
'autodoc-skip-member', obj_type, name, value, False, {}
)
except Exception as exc:
logger.warning(
__(
'autosummary: failed to determine %r to be documented, '
'the following exception was raised:\n%s'
),
name,
exc,
type='autosummary',
)
return False
def scan(self, imported_members: bool) -> list[str]:
members = []
try:
analyzer = ModuleAnalyzer.for_module(self.object.__name__)
attr_docs = analyzer.find_attr_docs()
except PycodeError:
attr_docs = {}
for name in members_of(self.object, config=self.config):
try:
value = safe_getattr(self.object, name)
except AttributeError:
value = None
objtype = _get_documenter(value, self.object)
if self.is_skipped(name, value, objtype):
continue
try:
if ('', name) in attr_docs:
imported = False
elif inspect.ismodule(value): # NoQA: SIM114
imported = True
elif safe_getattr(value, '__module__') != self.object.__name__:
imported = True
else:
imported = False
except AttributeError:
imported = False
respect_module_all = not self.config.autosummary_ignore_module_all
if (
# list all members up
imported_members
# list not-imported members
or imported is False
# list members that have __all__ set
or (respect_module_all and '__all__' in dir(self.object))
):
members.append(name)
return members
def members_of(obj: Any, *, config: Config) -> Sequence[str]:
"""Get the members of ``obj``, possibly ignoring the ``__all__`` module attribute
Follows the ``config.autosummary_ignore_module_all`` setting.
"""
if config.autosummary_ignore_module_all:
return dir(obj)
else:
if (obj___all__ := getall(obj)) is not None:
# return __all__, even if empty.
return obj___all__
# if __all__ is not set, return dir(obj)
return dir(obj)
def generate_autosummary_content(
name: str,
obj: Any,
parent: Any,
template: AutosummaryRenderer,
template_name: str,
imported_members: bool,
recursive: bool,
context: dict[str, Any],
modname: str | None = None,
qualname: str | None = None,
*,
config: Config,
events: EventManager,
) -> str:
obj_type = _get_documenter(obj, parent)
ns: dict[str, Any] = {}
ns.update(context)
if obj_type == 'module':
scanner = ModuleScanner(obj, config=config, events=events)
ns['members'] = scanner.scan(imported_members)
respect_module_all = not config.autosummary_ignore_module_all
imported_members = imported_members or (
'__all__' in dir(obj) and respect_module_all
)
ns['functions'], ns['all_functions'] = _get_members(
obj_type,
obj,
{'function'},
config=config,
events=events,
imported=imported_members,
)
ns['classes'], ns['all_classes'] = _get_members(
obj_type,
obj,
{'class'},
config=config,
events=events,
imported=imported_members,
)
ns['exceptions'], ns['all_exceptions'] = _get_members(
obj_type,
obj,
{'exception'},
config=config,
events=events,
imported=imported_members,
)
ns['attributes'], ns['all_attributes'] = _get_module_attrs(name, ns['members'])
ispackage = hasattr(obj, '__path__')
if ispackage and recursive:
# Use members that are not modules as skip list, because it would then mean
# that module was overwritten in the package namespace
skip = (
ns['all_functions']
+ ns['all_classes']
+ ns['all_exceptions']
+ ns['all_attributes']
)
# If respect_module_all and module has a __all__ attribute, first get
# modules that were explicitly imported. Next, find the rest with the
# get_modules method, but only put in "public" modules that are in the
# __all__ list
#
# Otherwise, use get_modules method normally
if respect_module_all and '__all__' in dir(obj):
imported_modules, all_imported_modules = _get_members(
obj_type,
obj,
{'module'},
config=config,
events=events,
imported=True,
)
skip += all_imported_modules
public_members = getall(obj)
else:
imported_modules, all_imported_modules = [], []
public_members = None
modules, all_modules = _get_modules(
obj, skip=skip, name=name, public_members=public_members
)
ns['modules'] = imported_modules + modules
ns['all_modules'] = all_imported_modules + all_modules
elif obj_type == 'class':
ns['members'] = dir(obj)
ns['inherited_members'] = set(dir(obj)) - set(obj.__dict__.keys())
ns['methods'], ns['all_methods'] = _get_members(
obj_type,
obj,
{'method'},
config=config,
events=events,
include_public={'__init__'},
)
ns['attributes'], ns['all_attributes'] = _get_members(
obj_type,
obj,
{'attribute', 'property'},
config=config,
events=events,
)
if modname is None or qualname is None:
modname, qualname = _split_full_qualified_name(name)
if obj_type in {'method', 'attribute', 'property'}:
ns['class'] = qualname.rsplit('.', 1)[0]
if obj_type == 'class':
shortname = qualname
else:
shortname = qualname.rsplit('.', 1)[-1]
ns['fullname'] = name
ns['module'] = modname
ns['objname'] = qualname
ns['name'] = shortname
ns['objtype'] = obj_type
ns['underline'] = len(name) * '='
if template_name:
return template.render(template_name, ns)
else:
return template.render(obj_type, ns)
def _skip_member(
obj: Any, name: str, obj_type: _AutodocObjType, *, events: EventManager
) -> bool:
try:
return events.emit_firstresult(
'autodoc-skip-member', obj_type, name, obj, False, {}
)
except Exception as exc:
logger.warning(
__(
'autosummary: failed to determine %r to be documented, '
'the following exception was raised:\n%s'
),
name,
exc,
type='autosummary',
)
return False
def _get_class_members(obj: Any) -> dict[str, Any]:
"""Get members and attributes of target class."""
# TODO: Simplify
# the members directly defined in the class
obj_dict = safe_getattr(obj, '__dict__', {})
members_simpler: dict[str, Any] = {}
# enum members
if isenumclass(obj):
for name, defining_class, value in _filter_enum_dict(
obj, safe_getattr, obj_dict
):
# the order of occurrence of *name* matches obj's MRO,
# allowing inherited attributes to be shadowed correctly
if unmangled := unmangle(defining_class, name):
members_simpler[unmangled] = value
# members in __slots__
try:
subject___slots__ = getslots(obj)
if subject___slots__:
for name in subject___slots__:
members_simpler[name] = SLOTS_ATTR
except (TypeError, ValueError):
pass
# other members
for name in dir(obj):
try:
value = safe_getattr(obj, name)
if ismock(value):
value = undecorate(value)
unmangled = unmangle(obj, name)
if unmangled and unmangled not in members_simpler:
members_simpler[unmangled] = value
except AttributeError:
continue
try:
for cls in getmro(obj):
try:
modname = safe_getattr(cls, '__module__')
qualname = safe_getattr(cls, '__qualname__')
except AttributeError:
qualname = None
analyzer = None
else:
try:
analyzer = ModuleAnalyzer.for_module(modname)
analyzer.analyze()
except PycodeError:
analyzer = None
# annotation only member (ex. attr: int)
for name in getannotations(cls):
unmangled = unmangle(cls, name)
if unmangled and unmangled not in members_simpler:
members_simpler[unmangled] = INSTANCE_ATTR
# append or complete instance attributes (cf. self.attr1) if analyzer knows
if analyzer:
for ns, name in analyzer.attr_docs:
if ns == qualname and name not in members_simpler:
# otherwise unknown instance attribute
members_simpler[name] = INSTANCE_ATTR
except AttributeError:
pass
return members_simpler
def _get_module_members(obj: Any, *, config: Config) -> dict[str, Any]:
members = {}
for name in members_of(obj, config=config):
try:
members[name] = safe_getattr(obj, name)
except AttributeError:
continue
return members
def _get_all_members(
obj_type: _AutodocObjType, obj: Any, *, config: Config
) -> dict[str, Any]:
if obj_type == 'module':
return _get_module_members(obj, config=config)
elif obj_type == 'class':
return _get_class_members(obj)
return {}
def _get_members(
obj_type: _AutodocObjType,
obj: Any,
types: set[str],
*,
config: Config,
events: EventManager,
include_public: Set[str] = frozenset(),
imported: bool = True,
) -> tuple[list[str], list[str]]:
items: list[str] = []
public: list[str] = []
all_members = _get_all_members(obj_type, obj, config=config)
for name, value in all_members.items():
obj_type = _get_documenter(value, obj)
if obj_type in types:
# skip imported members if expected
if imported or getattr(value, '__module__', None) == obj.__name__:
skipped = _skip_member(value, name, obj_type, events=events)
if skipped is True:
pass
elif skipped is False:
# show the member forcedly
items.append(name)
public.append(name)
else:
items.append(name)
if name in include_public or not name.startswith('_'):
# considers member as public
public.append(name)
return public, items
def _get_module_attrs(name: str, members: Any) -> tuple[list[str], list[str]]:
"""Find module attributes with docstrings."""
attrs, public = [], []
try:
analyzer = ModuleAnalyzer.for_module(name)
attr_docs = analyzer.find_attr_docs()
for namespace, attr_name in attr_docs:
if not namespace and attr_name in members:
attrs.append(attr_name)
if not attr_name.startswith('_'):
public.append(attr_name)
except PycodeError:
pass # give up if ModuleAnalyzer fails to parse code
return public, attrs
def _get_modules(
obj: Any,
*,
skip: Sequence[str],
name: str,
public_members: Sequence[str] | None = None,
) -> tuple[list[str], list[str]]:
items: list[str] = []
public: list[str] = []
for _, modname, _ispkg in pkgutil.iter_modules(obj.__path__):
if modname in skip:
# module was overwritten in __init__.py, so not accessible
continue
fullname = f'{name}.{modname}'
try:
module = _import_module(fullname)
except ImportError:
pass
else:
if module and hasattr(module, '__sphinx_mock__'):
continue
items.append(modname)
if public_members is not None:
if modname in public_members:
public.append(modname)
else:
if not modname.startswith('_'):
public.append(modname)
return public, items
def generate_autosummary_docs(
sources: list[str],
output_dir: str | os.PathLike[str] | None = None,
suffix: str = '.rst',
base_path: str | os.PathLike[str] | None = None,
imported_members: bool = False,
app: Sphinx | None = None,
overwrite: bool = True,
encoding: str = 'utf-8',
) -> list[Path]:
"""Generate autosummary documentation for the given sources.
:returns: list of generated files (both new and existing ones)
"""
assert app is not None, 'app is required'
showed_sources = sorted(sources)
if len(showed_sources) > 20:
showed_sources = [*showed_sources[:10], '...', *showed_sources[-10:]]
logger.info(
__('[autosummary] generating autosummary for: %s'), ', '.join(showed_sources)
)
if output_dir:
logger.info(__('[autosummary] writing to %s'), output_dir)
if base_path is not None:
base_path = Path(base_path)
source_paths = [base_path / filename for filename in sources]
else:
source_paths = list(map(Path, sources))
template = AutosummaryRenderer(app)
# read
items = find_autosummary_in_files(source_paths)
# keep track of new files
new_files: list[Path] = []
all_files: list[Path] = []
filename_map = app.config.autosummary_filename_map
# write
for entry in sorted(set(items), key=str):
if entry.path is None:
# The corresponding autosummary:: directive did not have
# a :toctree: option
continue
path = output_dir or Path(entry.path).resolve()
ensuredir(path)
try:
name, obj, parent, modname = import_by_name(entry.name)
qualname = name.replace(modname + '.', '')
except ImportExceptionGroup as exc:
try:
# try to import as an instance attribute
name, obj, parent, modname = import_ivar_by_name(entry.name)
qualname = name.replace(modname + '.', '')
except ImportError as exc2:
if exc2.__cause__:
exceptions: list[BaseException] = [*exc.exceptions, exc2.__cause__]
else:
exceptions = [*exc.exceptions, exc2]
errors = list({f'* {type(e).__name__}: {e}' for e in exceptions})
logger.warning(
__('[autosummary] failed to import %s.\nPossible hints:\n%s'),
entry.name,
'\n'.join(errors),
)
continue
context: dict[str, Any] = {**app.config.autosummary_context}
content = generate_autosummary_content(
name,
obj,
parent,
template,
entry.template,
imported_members,
entry.recursive,
context,
modname,
qualname,
config=app.config,
events=app.events,
)
file_path = Path(path, filename_map.get(name, name) + suffix)
all_files.append(file_path)
if file_path.is_file():
with file_path.open(encoding=encoding) as f:
old_content = f.read()
if content == old_content:
continue
if overwrite: # content has changed
with file_path.open('w', encoding=encoding) as f:
f.write(content)
new_files.append(file_path)
else:
with open(file_path, 'w', encoding=encoding) as f:
f.write(content)
new_files.append(file_path)
# descend recursively to new files
if new_files:
all_files.extend(
generate_autosummary_docs(
[str(f) for f in new_files],
output_dir=output_dir,
suffix=suffix,
base_path=base_path,
imported_members=imported_members,
app=app,
overwrite=overwrite,
)
)
return all_files
# -- Finding documented entries in files ---------------------------------------
def find_autosummary_in_files(
filenames: Sequence[str | os.PathLike[str]],
) -> list[AutosummaryEntry]:
"""Find out what items are documented in source/*.rst.
See `find_autosummary_in_lines`.
"""
documented: list[AutosummaryEntry] = []
for filename in filenames:
with open(filename, encoding='utf-8', errors='ignore') as f:
lines = f.read().splitlines()
documented.extend(find_autosummary_in_lines(lines, filename=filename))
return documented
def find_autosummary_in_docstring(
name: str,
filename: str | os.PathLike[str] | None = None,
) -> list[AutosummaryEntry]:
"""Find out what items are documented in the given object's docstring.
See `find_autosummary_in_lines`.
"""
try:
_real_name, obj, _parent, _modname = import_by_name(name)
lines = pydoc.getdoc(obj).splitlines()
return find_autosummary_in_lines(lines, module=name, filename=filename)
except AttributeError:
pass
except ImportExceptionGroup as exc:
errors = '\n'.join({f'* {type(e).__name__}: {e}' for e in exc.exceptions})
logger.warning(f'Failed to import {name}.\nPossible hints:\n{errors}') # NoQA: G004
except SystemExit:
logger.warning(
"Failed to import '%s'; the module executes module level "
'statement and it might call sys.exit().',
name,
)
return []
def find_autosummary_in_lines(
lines: list[str],
module: str | None = None,
filename: str | os.PathLike[str] | None = None,
) -> list[AutosummaryEntry]:
"""Find out what items appear in autosummary:: directives in the
given lines.
Returns a list of (name, toctree, template) where *name* is a name
of an object and *toctree* the :toctree: path of the corresponding
autosummary directive (relative to the root of the file name), and
*template* the value of the :template: option. *toctree* and
*template* ``None`` if the directive does not have the
corresponding options set.
"""
autosummary_re = re.compile(r'^(\s*)\.\.\s+autosummary::\s*')
automodule_re = re.compile(r'^\s*\.\.\s+automodule::\s*([A-Za-z0-9_.]+)\s*$')
module_re = re.compile(r'^\s*\.\.\s+(current)?module::\s*([a-zA-Z0-9_.]+)\s*$')
autosummary_item_re = re.compile(r'^\s+(~?[_a-zA-Z][a-zA-Z0-9_.]*)\s*.*?')
recursive_arg_re = re.compile(r'^\s+:recursive:\s*$')
toctree_arg_re = re.compile(r'^\s+:toctree:\s*(.*?)\s*$')
template_arg_re = re.compile(r'^\s+:template:\s*(.*?)\s*$')
documented: list[AutosummaryEntry] = []
recursive = False
toctree: str | None = None
template = ''
current_module = module
in_autosummary = False
base_indent = ''
for line in lines:
if in_autosummary:
m = recursive_arg_re.match(line)
if m:
recursive = True
continue
m = toctree_arg_re.match(line)
if m:
toctree = m.group(1)
if filename:
toctree = str(Path(filename).parent / toctree)
continue
m = template_arg_re.match(line)
if m:
template = m.group(1).strip()
continue
if line.strip().startswith(':'):
continue # skip options
m = autosummary_item_re.match(line)
if m:
name = m.group(1).strip().removeprefix('~')
if current_module and not name.startswith(current_module + '.'):
name = f'{current_module}.{name}'
documented.append(AutosummaryEntry(name, toctree, template, recursive))
continue
if not line.strip() or line.startswith(base_indent + ' '):
continue
in_autosummary = False
m = autosummary_re.match(line)
if m:
in_autosummary = True
base_indent = m.group(1)
recursive = False
toctree = None
template = ''
continue
m = automodule_re.search(line)
if m:
current_module = m.group(1).strip()
# recurse into the automodule docstring
documented.extend(
find_autosummary_in_docstring(current_module, filename=filename)
)
continue
m = module_re.match(line)
if m:
current_module = m.group(2)
continue
return documented
def get_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
usage='%(prog)s [OPTIONS] ...',
epilog=__('For more information, visit .'),
description=__("""
Generate ReStructuredText using autosummary directives.
sphinx-autogen is a frontend to sphinx.ext.autosummary.generate. It generates
the reStructuredText files from the autosummary directives contained in the
given input files.
The format of the autosummary directive is documented in the
``sphinx.ext.autosummary`` Python module and can be read using::
pydoc sphinx.ext.autosummary
"""),
)
parser.add_argument(
'--version',
action='version',
dest='show_version',
version='%%(prog)s %s' % __display_version__,
)
parser.add_argument(
'source_file', nargs='+', help=__('source files to generate rST files for')
)
parser.add_argument(
'-o',
'--output-dir',
action='store',
dest='output_dir',
help=__('directory to place all output in'),
)
parser.add_argument(
'-s',
'--suffix',
action='store',
dest='suffix',
default='rst',
help=__('default suffix for files (default: %(default)s)'),
)
parser.add_argument(
'-t',
'--templates',
action='store',
dest='templates',
default=None,
help=__('custom template directory (default: %(default)s)'),
)
parser.add_argument(
'-i',
'--imported-members',
action='store_true',
dest='imported_members',
default=False,
help=__('document imported members (default: %(default)s)'),
)
parser.add_argument(
'-a',
'--respect-module-all',
action='store_true',
dest='respect_module_all',
default=False,
help=__(
'document exactly the members in module __all__ attribute. '
'(default: %(default)s)'
),
)
parser.add_argument(
'--remove-old',
action='store_true',
dest='remove_old',
default=False,
help=__(
'Remove existing files in the output directory that were not generated'
),
)
return parser
def main(argv: Sequence[str] = (), /) -> None:
locale.setlocale(locale.LC_ALL, '')
sphinx.locale.init_console()
app = DummyApplication(sphinx.locale.get_translator())
logging.setup(app, sys.stdout, sys.stderr) # type: ignore[arg-type]
args = get_parser().parse_args(argv or sys.argv[1:])
if args.templates:
app.config.templates_path.append(str(Path(args.templates).resolve()))
app.config.autosummary_ignore_module_all = not args.respect_module_all
written_files = generate_autosummary_docs(
args.source_file,
args.output_dir,
'.' + args.suffix,
imported_members=args.imported_members,
app=app, # type: ignore[arg-type]
)
if args.remove_old:
for existing in Path(args.output_dir).glob(f'**/*.{args.suffix}'):
if existing not in written_files:
try:
existing.unlink()
except OSError as exc:
logger.warning(
__('Failed to remove %s: %s'),
existing,
exc.strerror,
type='autosummary',
)
if __name__ == '__main__':
main(sys.argv[1:])