[v17,5/5] dts: add API doc generation

diff --git a/buildtools/call-sphinx-build.py b/buildtools/call-sphinx-build.py
index 623e7363ee..154e9f907b 100755
--- a/buildtools/call-sphinx-build.py
+++ b/buildtools/call-sphinx-build.py
@@ -15,6 +15,8 @@ 
 
 # set the version in environment for sphinx to pick up
 os.environ['DPDK_VERSION'] = version
+if 'dts' in src:
+    os.environ['DTS_BUILD'] = "y"
 
 sphinx_cmd = [sphinx] + extra_args
 
diff --git a/buildtools/get-dts-runtime-deps.py b/buildtools/get-dts-runtime-deps.py
new file mode 100755
index 0000000000..68244480a3
--- /dev/null
+++ b/buildtools/get-dts-runtime-deps.py
@@ -0,0 +1,84 @@ 
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 PANTHEON.tech s.r.o.
+#
+
+"""Utilities for DTS dependencies.
+
+The module can be used as an executable script,
+which verifies that the running Python version meets the version requirement of DTS.
+The script exits with the standard exit codes in this mode (0 is success, 1 is failure).
+
+The module also contains a function, get_missing_imports,
+which looks for runtime dependencies in the DTS pyproject.toml file
+and returns a list of module names used in an import statement (import packages) that are missing.
+This function is not used when the module is run as a script and is available to be imported.
+"""
+
+import configparser
+import importlib.metadata
+import importlib.util
+import os.path
+import platform
+
+from packaging.version import Version
+
+_VERSION_COMPARISON_CHARS = '^<>='
+_EXTRA_DEPS = {'invoke': '>=1.3', 'paramiko': '>=2.4'}
+_DPDK_ROOT = os.path.dirname(os.path.dirname(__file__))
+_DTS_DEP_FILE_PATH = os.path.join(_DPDK_ROOT, 'dts', 'pyproject.toml')
+
+
+def _get_dependencies(cfg_file_path):
+    cfg = configparser.ConfigParser()
+    with open(cfg_file_path) as f:
+        dts_deps_file_str = f.read()
+        dts_deps_file_str = dts_deps_file_str.replace("\n]", "]")
+        cfg.read_string(dts_deps_file_str)
+
+    deps_section = cfg['tool.poetry.dependencies']
+    return {dep: deps_section[dep].strip('"\'') for dep in deps_section}
+
+
+def get_missing_imports():
+    """Get missing DTS import packages from third party libraries.
+
+    Scan the DTS pyproject.toml file for dependencies and find those that are not installed.
+    The dependencies in pyproject.toml are listed by their distribution package names,
+    but the function finds the associated import packages - those used in import statements.
+
+    The function is not used when the module is run as a script. It should be imported.
+
+    Returns:
+        A list of missing import packages.
+    """
+    missing_imports = []
+    req_deps = _get_dependencies(_DTS_DEP_FILE_PATH)
+    req_deps.pop('python')
+
+    for req_dep, req_ver in (req_deps | _EXTRA_DEPS).items():
+        try:
+            req_ver = Version(req_ver.strip(_VERSION_COMPARISON_CHARS))
+            found_dep_ver = Version(importlib.metadata.version(req_dep))
+            if found_dep_ver < req_ver:
+                print(
+                    f'The version "{found_dep_ver}" of package "{req_dep}" '
+                    f'is lower than required "{req_ver}".'
+                )
+        except importlib.metadata.PackageNotFoundError:
+            print(f'Package "{req_dep}" not found.')
+            missing_imports.append(req_dep.lower().replace('-', '_'))
+
+    return missing_imports
+
+
+if __name__ == '__main__':
+    python_version = _get_dependencies(_DTS_DEP_FILE_PATH).pop('python')
+    if python_version:
+        sys_ver = Version(platform.python_version())
+        req_ver = Version(python_version.strip(_VERSION_COMPARISON_CHARS))
+        if sys_ver < req_ver:
+            print(
+                f'The available Python version "{sys_ver}" is lower than required "{req_ver}".'
+            )
+            exit(1)
diff --git a/buildtools/meson.build b/buildtools/meson.build
index 3adf34e1a8..6b938d767c 100644
--- a/buildtools/meson.build
+++ b/buildtools/meson.build
@@ -24,6 +24,7 @@  get_numa_count_cmd = py3 + files('get-numa-count.py')
 get_test_suites_cmd = py3 + files('get-test-suites.py')
 has_hugepages_cmd = py3 + files('has-hugepages.py')
 cmdline_gen_cmd = py3 + files('dpdk-cmdline-gen.py')
+get_dts_runtime_deps = py3 + files('get-dts-runtime-deps.py')
 
 # install any build tools that end-users might want also
 install_data([
diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md
index f9f0300126..ab223bcdf7 100644
--- a/doc/api/doxy-api-index.md
+++ b/doc/api/doxy-api-index.md
@@ -245,3 +245,6 @@  The public API headers are grouped by topics:
   [experimental APIs](@ref rte_compat.h),
   [ABI versioning](@ref rte_function_versioning.h),
   [version](@ref rte_version.h)
+
+- **tests**:
+  [**DTS**](@dts_api_main_page)
diff --git a/doc/api/doxy-api.conf.in b/doc/api/doxy-api.conf.in
index a8823c046f..c94f02d411 100644
--- a/doc/api/doxy-api.conf.in
+++ b/doc/api/doxy-api.conf.in
@@ -124,6 +124,8 @@  SEARCHENGINE            = YES
 SORT_MEMBER_DOCS        = NO
 SOURCE_BROWSER          = YES
 
+ALIASES                 = "dts_api_main_page=@DTS_API_MAIN_PAGE@"
+
 EXAMPLE_PATH            = @TOPDIR@/examples
 EXAMPLE_PATTERNS        = *.c
 EXAMPLE_RECURSIVE       = YES
diff --git a/doc/api/dts/custom.css b/doc/api/dts/custom.css
new file mode 120000
index 0000000000..3c9480c4a0
--- /dev/null
+++ b/doc/api/dts/custom.css
@@ -0,0 +1 @@ 
+../../guides/custom.css
\ No newline at end of file
diff --git a/doc/api/dts/meson.build b/doc/api/dts/meson.build
new file mode 100644
index 0000000000..f338eb69bf
--- /dev/null
+++ b/doc/api/dts/meson.build
@@ -0,0 +1,31 @@ 
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2023 PANTHEON.tech s.r.o.
+
+sphinx = find_program('sphinx-build', required: get_option('enable_docs'))
+if not sphinx.found()
+    subdir_done()
+endif
+
+python_ver_satisfied = run_command(get_dts_runtime_deps, check: false).returncode()
+if python_ver_satisfied != 0
+    subdir_done()
+endif
+
+cdata.set('DTS_API_MAIN_PAGE', join_paths('..', 'dts', 'html', 'index.html'))
+
+extra_sphinx_args = ['-E', '-c', join_paths(doc_source_dir, 'guides')]
+if get_option('werror')
+    extra_sphinx_args += '-W'
+endif
+
+htmldir = join_paths(get_option('datadir'), 'doc', 'dpdk', 'dts')
+dts_api_html = custom_target('dts_api_html',
+        output: 'html',
+        command: [sphinx_wrapper, sphinx, meson.project_version(),
+            meson.current_source_dir(), meson.current_build_dir(), extra_sphinx_args],
+        build_by_default: get_option('enable_docs'),
+        install: get_option('enable_docs'),
+        install_dir: htmldir)
+
+doc_targets += dts_api_html
+doc_target_names += 'DTS_API_HTML'
diff --git a/doc/api/meson.build b/doc/api/meson.build
index 5b50692df9..71b861e42b 100644
--- a/doc/api/meson.build
+++ b/doc/api/meson.build
@@ -1,6 +1,11 @@ 
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
 
+# initialize common Doxygen configuration
+cdata = configuration_data()
+
+subdir('dts')
+
 doxygen = find_program('doxygen', required: get_option('enable_docs'))
 
 if not doxygen.found()
@@ -30,7 +35,6 @@  example = custom_target('examples.dox',
         build_by_default: get_option('enable_docs'))
 
 # set up common Doxygen configuration
-cdata = configuration_data()
 cdata.set('VERSION', meson.project_version())
 cdata.set('API_EXAMPLES', join_paths(dpdk_build_root, 'doc', 'api', 'examples.dox'))
 cdata.set('OUTPUT', join_paths(dpdk_build_root, 'doc', 'api'))
diff --git a/doc/guides/conf.py b/doc/guides/conf.py
index 0f7ff5282d..d7f3030838 100644
--- a/doc/guides/conf.py
+++ b/doc/guides/conf.py
@@ -10,7 +10,7 @@ 
 from os.path import basename
 from os.path import dirname
 from os.path import join as path_join
-from sys import argv, stderr
+from sys import argv, stderr, path
 
 import configparser
 
@@ -58,6 +58,48 @@ 
              ("tools/devbind", "dpdk-devbind",
               "check device status and bind/unbind them from drivers", "", 8)]
 
+# DTS API docs additional configuration
+if environ.get('DTS_BUILD'):
+    extensions = ['sphinx.ext.napoleon', 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
+    # Napoleon enables the Google format of Python doscstrings.
+    napoleon_numpy_docstring = False
+    napoleon_attr_annotations = True
+    napoleon_preprocess_types = True
+
+    # Autodoc pulls documentation from code.
+    autodoc_default_options = {
+        'members': True,
+        'member-order': 'bysource',
+        'show-inheritance': True,
+    }
+    autodoc_class_signature = 'separated'
+    autodoc_typehints = 'both'
+    autodoc_typehints_format = 'short'
+    autodoc_typehints_description_target = 'documented'
+
+    # Intersphinx allows linking to external projects, such as Python docs.
+    intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
+
+    # DTS docstring options.
+    add_module_names = False
+    toc_object_entries = True
+    toc_object_entries_show_parents = 'hide'
+    # DTS Sidebar config.
+    html_theme_options = {
+        'collapse_navigation': False,
+        'navigation_depth': -1,  # unlimited depth
+    }
+
+    # Add path to DTS sources so that Sphinx can find them.
+    dpdk_root = dirname(dirname(dirname(__file__)))
+    path.append(path_join(dpdk_root, 'dts'))
+
+    # Get missing DTS dependencies. Add path to buildtools to find the get_missing_imports function.
+    path.append(path_join(dpdk_root, 'buildtools'))
+    import importlib
+    # Ignore missing imports from DTS dependencies.
+    autodoc_mock_imports = importlib.import_module('get-dts-runtime-deps').get_missing_imports()
+
 
 # ####### :numref: fallback ########
 # The following hook functions add some simple handling for the :numref:
diff --git a/doc/guides/contributing/documentation.rst b/doc/guides/contributing/documentation.rst
index 68454ae0d5..7b287ce631 100644
--- a/doc/guides/contributing/documentation.rst
+++ b/doc/guides/contributing/documentation.rst
@@ -133,6 +133,8 @@  added to by the developer.
 Building the Documentation
 --------------------------
 
+.. _doc_dependencies:
+
 Dependencies
 ~~~~~~~~~~~~
 
diff --git a/doc/guides/contributing/patches.rst b/doc/guides/contributing/patches.rst
index 04c66bebc4..6629928bee 100644
--- a/doc/guides/contributing/patches.rst
+++ b/doc/guides/contributing/patches.rst
@@ -499,6 +499,10 @@  The script usage is::
 For both of the above scripts, the -n option is used to specify a number of commits from HEAD,
 and the -r option allows the user specify a ``git log`` range.
 
+Additionally, when contributing to the DTS tool, patches should also be checked using
+the ``dts-check-format.sh`` script in the ``devtools`` directory of the DPDK repo.
+To run the script, extra :ref:`Python dependencies <dts_deps>` are needed.
+
 .. _contrib_check_compilation:
 
 Checking Compilation
diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst
index 515b15e4d8..9e8929f567 100644
--- a/doc/guides/tools/dts.rst
+++ b/doc/guides/tools/dts.rst
@@ -54,6 +54,7 @@  DTS uses Poetry as its Python dependency management.
 Python build/development and runtime environments are the same and DTS development environment,
 DTS runtime environment or just plain DTS environment are used interchangeably.
 
+.. _dts_deps:
 
 Setting up DTS environment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -291,8 +292,15 @@  When adding code to the DTS framework, pay attention to the rest of the code
 and try not to divert much from it.
 The :ref:`DTS developer tools <dts_dev_tools>` will issue warnings
 when some of the basics are not met.
+You should also build the :ref:`API documentation <building_api_docs>`
+to address any issues found during the build.
 
-The code must be properly documented with docstrings.
+The API documentation, which is a helpful reference when developing, may be accessed
+in the code directly or generated with the :ref:`API docs build steps <building_api_docs>`.
+When adding new files or modifying the directory structure,
+the corresponding changes must be made to DTS api doc sources in ``doc/api/dts``.
+
+Speaking of which, the code must be properly documented with docstrings.
 The style must conform to the `Google style
 <https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings>`_.
 See an example of the style `here
@@ -427,6 +435,35 @@  the DTS code check and format script.
 Refer to the script for usage: ``devtools/dts-check-format.sh -h``.
 
 
+.. _building_api_docs:
+
+Building DTS API docs
+---------------------
+
+The documentation is built using the standard DPDK build system.
+See :doc:`../linux_gsg/build_dpdk` for more details on compiling DPDK with meson.
+
+The :ref:`doc build dependencies <doc_dependencies>` may be installed with Poetry:
+
+.. code-block:: console
+
+   poetry install --no-root --only docs
+   poetry install --no-root --with docs  # an alternative that will also install DTS dependencies
+   poetry shell
+
+After executing the meson command, build the documentation with:
+
+.. code-block:: console
+
+   ninja -C build doc
+
+The output is generated in ``build/doc/api/dts/html``.
+
+.. note::
+
+   Make sure to fix any Sphinx warnings when adding or updating docstrings.
+
+
 Configuration Schema
 --------------------
 
diff --git a/doc/meson.build b/doc/meson.build
index 6f74706aa2..1e0cfa4127 100644
--- a/doc/meson.build
+++ b/doc/meson.build
@@ -1,6 +1,7 @@ 
 # SPDX-License-Identifier: BSD-3-Clause
 # Copyright(c) 2018 Luca Boccassi <bluca@debian.org>
 
+doc_source_dir = meson.current_source_dir()
 doc_targets = []
 doc_target_names = []
 subdir('api')