buildroot/utils/generate-cyclonedx

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-or-later

# This script converts the output of the show-info make target
# to CycloneDX format.
#
# Example usage:
# $ make show-info | utils/generate-cyclonedx > sbom.json


import argparse
import bz2
import gzip
import json
import os
from pathlib import Path
from typing import Iterator
import urllib.parse
import urllib.request
import subprocess
import sys
import re


CYCLONEDX_VERSION = "1.6"
SPDX_SCHEMA_URL = f"https://raw.githubusercontent.com/CycloneDX/specification/{CYCLONEDX_VERSION}/schema/spdx.schema.json"

brpath = Path(__file__).parent.parent

cyclonedxpath = Path(os.getenv("BR2_DL_DIR", brpath / "dl")) / "cyclonedx"
SPDX_SCHEMA_PATH = cyclonedxpath / f"spdx-{CYCLONEDX_VERSION}.schema.json"

BR2_VERSION_FULL = (
    subprocess.check_output(
        ["make", "--no-print-directory", "-C", brpath, "print-version"]
    )
    .decode()
    .strip()
)

# Set of vulnerabilities that were addressed by a patch present in buildroot
# tree. This set is used to set the analysis of the ignored CVEs to
# 'resolved_with_pedigree'.
VULN_WITH_PEDIGREE = set()

SPDX_LICENSES = []

if not SPDX_SCHEMA_PATH.exists():
    # Download the CycloneDX SPDX schema JSON, and cache it locally
    cyclonedxpath.mkdir(parents=True, exist_ok=True)
    urllib.request.urlretrieve(SPDX_SCHEMA_URL, SPDX_SCHEMA_PATH)

try:
    with SPDX_SCHEMA_PATH.open() as f:
        SPDX_LICENSES = json.load(f).get("enum", [])
except json.JSONDecodeError:
    # In case of error the license will just not be matched to the SPDX names
    # but the SBOM generation still work.
    print(f"Failed to load the SPDX licenses file: {SPDX_SCHEMA_PATH}", file=sys.stderr)


def split_top_level_comma(subj):
    """Split a string at comma's, but do not split at comma's in between parentheses.

    Args:
        subj (str): String to be split.

    Returns:
        list: A list of substrings
    """
    counter = 0
    substring = ""

    for char in subj:
        if char == "," and counter == 0:
            yield substring
            substring = ""
        else:
            if char == "(":
                counter += 1
            elif char == ")":
                counter -= 1
            substring += char

    yield substring


def cyclonedx_license(lic):
    """Given the name of a license, create an individual entry in
    CycloneDX format. In CycloneDX, the 'id' keyword is used for
    names that are recognized as SPDX License abbreviations. All other
    license names are placed under the 'name' keyword.

    Args:
        lic (str): Name of the license

    Returns:
        dict: An entry for the license in CycloneDX format.
    """
    key = "id" if lic in SPDX_LICENSES else "name"
    return {
        key: lic,
    }


def cyclonedx_licenses(lic_list):
    """Create a licenses list formatted for a CycloneDX component

    Args:
        lic_list (str): A comma separated list of license names.

    Returns:
        dict: A dictionary with license information for the component,
        in CycloneDX format.
    """
    return {
        "licenses": [
            {"license": cyclonedx_license(lic.strip())} for lic in split_top_level_comma(lic_list)
        ]
    }


def extract_cves_from_header(header: str) -> list[str]:
    """Extract CVE identifiers from the patch header.

    Args:
        header (str): Content of the header of a patch.

    Returns:
        list: Array of CVE identifier present in a patch header passed as
        argument.
    """
    PATCH_CVE_HEADER = "CVE: "
    return [
        line.partition(PATCH_CVE_HEADER)[2].strip()
        for line in header.splitlines()
        if line.startswith(PATCH_CVE_HEADER)
    ]


def patch_retrieve_header(content: str) -> str:
    """Read the content of a patch and split the header from the content.

    Args:
        content (str): Patch content.

    Returns:
        str: Patch header content.
    """
    DIFF_LINE_REGEX = re.compile(r"^diff\s+(?:--git|-[-\w]+)\s+(\S+)\s+(\S+)$")
    INDEX_LINE_REGEX = re.compile(r"^Index:\s+(\S+)$")

    lines = content.split('\n')

    header = []
    for i, line in enumerate(lines):
        if DIFF_LINE_REGEX.match(line):
            # diff --git a/configure.ac b/configure.ac
            # index 1234..1234 100644
            # --- a/configure.ac
            # +++ b/configure.ac
            break
        elif INDEX_LINE_REGEX.match(line):
            # Index: <filename>
            # --- <filename>
            # +++ <filename>
            if i < len(lines) - 2 and lines[i + 1].startswith("===") and lines[i + 2].startswith("---"):
                break
        elif line.startswith("---"):
            # Some patches don't have a 'diff' tag just the --- +++ tuple.
            # Check next line is starting with '+++'
            # ex: package/berkeleydb/0001-cwd-db_config.patch
            if i < len(lines) - 2 and lines[i + 1].startswith("+++") and lines[i + 2].startswith("@@"):
                break
        else:
            header.append(line)

    return '\n'.join(header)


def read_patch_file(patch_path: Path) -> str:
    """Read the content of a patch file, handling compression.

    Args:
        patch_path (Path): Patch path.

    Returns:
        str: Patch content.
    """
    if patch_path.suffix == ".gz":
        f = gzip.open(patch_path, mode="rt")
    elif patch_path.suffix == ".bz":
        f = bz2.open(patch_path, mode="rt")
    else:
        f = open(patch_path)

    content = f.read()
    f.close()
    return content


def cyclonedx_patches(patch_list: list[str]):
    """Translate a list of patches from the show-info JSON to a list of
    patches in CycloneDX format.

    Args:
        patch_list (list): Array of patch relative paths for a given component.

    Returns:
        dict: Patch information in CycloneDX format.
    """
    patch_contents = []
    for patch in patch_list:
        patch_path = brpath / patch
        if patch_path.exists():
            try:
                content = read_patch_file(patch_path)
            except Exception:
                # If the patch can't be read it won't be added to
                # the resulting SBOM.
                print(f"Failed to handle patch: {patch}", file=sys.stderr)
                continue

            header = patch_retrieve_header(content)

            issue = {}
            cves = extract_cves_from_header(header)
            if cves:
                VULN_WITH_PEDIGREE.update(cves)
                issue = {
                    "resolves": [
                        {
                            "type": "security",
                            "name": cve
                        } for cve in cves
                    ]
                }

            patch_contents.append({
                "diff": {
                    "text": {
                        "content": content
                    }
                },
                **issue
            })
        else:
            # If the patch is not a file it's a tarball or diff url passed
            # through the `<pkg-name>_PATCH` variable.
            patch_contents.append({
                "diff": {
                    "url": patch
                }
            })

    return {
        "pedigree": {
            "patches": [{
                "type": "unofficial",
                **content
            } for content in patch_contents]
        },
    }


def parse_uris(uris: list[str]) -> Iterator[tuple[list[str], str]]:
    """Parse download URIs into (schemes, url) tuples.

    Splits the Buildroot URI format "scheme[|scheme]+url" and yields all
    Buildroot schemes with the stripped URL, excluding
    sources.buildroot.net mirrors.

    Args:
        uris (list): Array of URI strings from the show-info output.
    Yields:
        tuple[list[str], str]: (schemes, url) for each usable URI.
    """
    for uri in uris:
        scheme, _, stripped_uri = uri.partition("+")
        if stripped_uri:
            parsed = urllib.parse.urlparse(stripped_uri)
            if parsed.hostname != "sources.buildroot.net":
                yield scheme.split("|"), stripped_uri


def cyclonedx_external_refs(comp):
    """Create CycloneDX external references for a component.

    Args:
        comp (dict): The component information from the show-info output.
    Returns:
        dict: External reference information in CycloneDX format, or empty dict
    """
    SOURCE_DIST_SCHEMES = {"http", "https"}

    refs = []
    for download in comp.get("downloads", []):
        source = download.get("source")
        for schemes, uri in parse_uris(download.get("uris", [])):
            if set(schemes) & SOURCE_DIST_SCHEMES and source:
                refs.append({
                    "type": "source-distribution",
                    "url": f"{uri}/{source}",
                })
    if refs:
        return {"externalReferences": refs}
    return {}


def cyclonedx_component(name, comp):
    """Translate a component from the show-info output, to a component entry in CycloneDX format.

    Args:
        name (str): Key used for the package in the show-info output.
        comp (dict): Data about the package as a Python dictionary.

    Returns:
        dict: Component information in CycloneDX format.
    """
    return {
        "bom-ref": name,
        "type": "library",
        **({
            "name": comp["name"],
        } if "name" in comp else {}),
        **({
            "version": comp["version"],
            **(cyclonedx_licenses(comp["licenses"]) if "licenses" in comp else {}),
        } if not comp["virtual"] else {}),
        **({
            "cpe": comp["cpe-id"],
        } if "cpe-id" in comp else {}),
        **cyclonedx_external_refs(comp),
        **(cyclonedx_patches(comp["patches"]) if comp.get("patches") else {}),
        "properties": [{
            "name": "BR_TYPE",
            "value": comp["type"],
        }],
    }


def cyclonedx_dependency(ref, depends):
    """Create JSON for dependency relationships between components.

    Args:
        ref (str): reference to a component bom-ref.
        depends (list): array of component bom-ref identifier to create the dependencies.

    Returns:
        dict: Dependency information in CycloneDX format.
    """
    return {
        "ref": ref,
        "dependsOn": sorted(depends),
    }


def cyclonedx_vulnerabilities(show_info_dict):
    """Create a JSON list of vulnerabilities ignored by buildroot and associate
    the component for which they are solved.

    Args:
        show_info_dict (dict): The JSON output of the show-info
            command, parsed into a Python dictionary.

    Returns:
        list: Solved vulnerabilities list in CycloneDX format.
    """
    cves = {}

    for name, comp in show_info_dict.items():
        for cve in comp.get('ignore_cves', []):
            cves.setdefault(cve, []).append(name)

    return [{
        "id": cve,
        "source": {
            "name": "NVD",
            "url": "https://nvd.nist.gov/vuln/detail/" + cve
        },
        "analysis": {
            "state": "resolved_with_pedigree" if cve in VULN_WITH_PEDIGREE else "in_triage",
            "detail": f"The CVE '{cve}' has been marked as ignored by Buildroot"
        },
        "affects": [
            {"ref": bomref} for bomref in components
        ]
    } for cve, components in cves.items()]


def br2_virtual_is_provided_by(ref, show_info_dict) -> list:
    """Retrieve the list of packages that provide a virtual package.

    Args:
        ref (str): The identifier of the virtual package.
        show_info_dict (dict): The JSON output of the show-info
            command, parsed into a Python dictionary.

    Returns:
        list: package list that provides the virtual package.
    """
    return [
        name
        for name, comp in show_info_dict.items()
        if "provides" in comp and ref in comp["provides"]
    ]


def br2_parse_deps(ref, show_info_dict, virtual=False) -> list:
    """This function will collect all dependencies from the show-info output.

    The dependency on virtual package will collect the final dependency without
    including the virtual one.

    Args:
        ref (str): The identifier of the package for which the dependencies have
            to be looked up.
        show_info_dict (dict): The JSON output of the show-info
            command, parsed into a Python dictionary.

    Returns:
        list: A list of dependencies of the 'ref' package.
    """
    deps = set()

    for dep in show_info_dict.get(ref, {}).get("dependencies", []):
        if not virtual and show_info_dict.get(dep, {}).get("virtual"):
            deps.update(br2_virtual_is_provided_by(dep, show_info_dict))
        else:
            deps.add(dep)

    return list(deps)


def br2_root_deps(show_info_dict, virtual=False) -> list:
    """Retrieve the list of direct dependencies of the root component.

    This function returns all components that are not a dependency of any
    other component.

    Args:
        show_info_dict (dict): The JSON output of the show-info command.
        virtual (bool): Whether to resolve virtual dependencies to their providers.

    Returns:
        list: List of direct dependencies of the root component.
    """
    indirect = set()
    for ref in show_info_dict:
        indirect.update(br2_parse_deps(ref, show_info_dict, virtual))
    return [ref for ref in show_info_dict if ref not in indirect]


def main():
    parser = argparse.ArgumentParser(
            description='''Create a CycloneDX SBoM for the Buildroot configuration.
                Example usage: make show-info | utils/generate-cyclonedx > sbom.json
            '''
        )
    parser.add_argument("-i", "--in-file", nargs="?", type=argparse.FileType("r"),
                        default=(None if sys.stdin.isatty() else sys.stdin))
    parser.add_argument("-o", "--out-file", nargs="?", type=argparse.FileType("w"),
                        default=sys.stdout)
    parser.add_argument("--virtual", default=False, action='store_true',
                        help="This option includes virtual packages to the CycloneDX output")
    parser.add_argument("--project-name", type=str, default="buildroot",
                        help="Specify the project name to use in the SBOM metadata (default:'buildroot')")
    parser.add_argument("--project-version", type=str, default=f"{BR2_VERSION_FULL}",
                        help="Specify the project version to use in the SBOM metadata (default: builroot version)")

    args = parser.parse_args()

    if args.in_file is None:
        parser.print_help()
        sys.exit(1)

    show_info_dict = json.load(args.in_file)

    # Remove rootfs and virtual packages if not explicitly included
    # from the cli arguments
    filtered_show_info_dict = {k: v for k, v in show_info_dict.items()
                               if ("rootfs" not in v["type"]) and (args.virtual or v["virtual"] is False)}

    cyclonedx_dict = {
        "bomFormat": "CycloneDX",
        "$schema": f"http://cyclonedx.org/schema/bom-{CYCLONEDX_VERSION}.schema.json",
        "specVersion": f"{CYCLONEDX_VERSION}",
        "metadata": {
            "component": {
                "bom-ref": args.project_name,
                "name": args.project_name,
                "version": args.project_version,
                "type": "firmware",
            },
            "tools": {
                "components": [
                    {
                        "type": "application",
                        "name": "Buildroot generate-cyclonedx",
                        "version": f"{BR2_VERSION_FULL}",
                        "licenses": [
                            {
                                "license": {
                                    "id": "GPL-2.0"
                                }
                            }
                        ]
                    }
                ],
            }
        },
        "components": [
            cyclonedx_component(name, comp) for name, comp in filtered_show_info_dict.items()
        ],
        "dependencies": [
            cyclonedx_dependency(args.project_name, br2_root_deps(filtered_show_info_dict, args.virtual)),
            *[cyclonedx_dependency(ref, br2_parse_deps(ref, show_info_dict, args.virtual))
              for ref in filtered_show_info_dict],
        ],
        "vulnerabilities": cyclonedx_vulnerabilities(show_info_dict),
    }

    args.out_file.write(json.dumps(cyclonedx_dict, indent=2))
    args.out_file.write('\n')


if __name__ == "__main__":
    main()