"""Attach generally useful information, not specific to any package.

Copyright (C) 2009 Canonical Ltd.
Authors: Matt Zimmerman <mdz@canonical.com>,
         Brian Murray <brian@ubuntu.com>

This program is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2 of the License, or (at your
option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
the full text of the license.
"""

import os
import re
import subprocess
import sys
import time
import urllib.error
from glob import glob
from urllib.parse import urljoin
from urllib.request import urlopen

import apport.hookutils
import apport.packaging
import problem_report
from apport import unicode_gettext as _


def add_info(report, ui):
    add_release_info(report)

    add_kernel_info(report)

    add_cloud_info(report)

    add_proposed_info(report)

    # collect a condensed version of /proc/cpuinfo
    apport.hookutils.attach_file(report, "/proc/cpuinfo", "ProcCpuinfo")
    short_cpuinfo = []
    for item in reversed(report.get("ProcCpuinfo", "").split("\n")):
        short_cpuinfo.append(item)
        if item.startswith("processor\t:"):
            break
    short_cpuinfo = reversed(short_cpuinfo)
    report["ProcCpuinfoMinimal"] = "\n".join(short_cpuinfo)
    report.pop("ProcCpuinfo")

    hook_errors = [k for k in report.keys() if k.startswith("HookError_")]
    if hook_errors:
        add_tag(report, "apport-hook-error")

    # locally installed python versions can cause a multitude of errors
    if (
        report.get("ProblemType") == "Package"
        or "python" in report.get("InterpreterPath", "")
        or "python" in report.get("ExecutablePath", "")
    ):
        for python in ("python", "python3"):
            add_python_details("%sDetails" % python.title(), python, report)

    try:
        report["ApportVersion"] = apport.packaging.get_version("apport")
    except ValueError:
        # might happen on local installs
        pass

    # We want to know if people have modified apport's crashdb.conf in case
    # crashes are reported to Launchpad when they shouldn't be e.g. for a
    # non-development release.
    apport.hookutils.attach_conffiles(report, "apport", ui=ui)

    # Should the system have been rebooted?
    apport.hookutils.attach_file_if_exists(
        report, "/var/run/reboot-required.pkgs", "RebootRequiredPkgs"
    )

    casper_md5check = "casper-md5check.json"
    if "LiveMediaBuild" in report:
        apport.hookutils.attach_casper_md5check(
            report, "/run/%s" % casper_md5check
        )
    else:
        apport.hookutils.attach_casper_md5check(
            report, "/var/log/installer/%s" % casper_md5check
        )

    if report.get("ProblemType") == "Package":
        # every error report regarding a package should have package manager
        # version information
        apport.hookutils.attach_related_packages(report, ["dpkg", "apt"])
        check_for_disk_error(report)
        # check to see if the real root on a persistent media is full
        if "LiveMediaBuild" in report:
            st = os.statvfs("/cdrom")
            free_mb = st.f_bavail * st.f_frsize / 1000000
            if free_mb < 10:
                report["UnreportableReason"] = (
                    f"Your system partition has less than {free_mb} MB"
                    f" of free space available, which leads to problems"
                    f" using applications and installing updates."
                    f" Please free some space."
                )

    match_error_messages(report)

    # these attachments will not exist if ProblemType is Bug as the package
    # hook runs after the general hook
    for attachment in ["DpkgTerminalLog", "VarLogDistupgradeApttermlog"]:
        if attachment in report:
            log_file = get_attachment_contents(report, attachment)
            untrimmed_dpkg_log = log_file
            check_attachment_for_errors(report, attachment)
            trimmed_log = get_attachment_contents(report, attachment)
            trimmed_log = trimmed_log.split("\n")
            lines = []
            for line in untrimmed_dpkg_log.splitlines():
                if line not in trimmed_log:
                    lines.append(str(line))
                elif line in trimmed_log:
                    trimmed_log.remove(line)
            dpkg_log_without_error = "\n".join(lines)

    # crash reports from live system installer often expose target mount
    for f in ("ExecutablePath", "InterpreterPath"):
        if f in report and report[f].startswith("/target/"):
            report[f] = report[f][7:]

    # Allow filing update-manager bugs with obsolete packages
    if report.get("Package", "").startswith("update-manager"):
        os.environ["APPORT_IGNORE_OBSOLETE_PACKAGES"] = "1"

    # file bugs against OEM project for modified packages
    if "Package" in report:
        v = report["Package"].split()[1]
        oem_project = get_oem_project(report)
        if oem_project and ("common" in v or oem_project in v):
            report["CrashDB"] = "canonical-oem"

    if "Package" in report:
        package = report["Package"].split()[0]
        if package:
            apport.hookutils.attach_conffiles(report, package, ui=ui)

        # do not file bugs against "upgrade-system" if it is not installed
        # (LP#404727)
        if (
            package == "upgrade-system"
            and "not installed" in report["Package"]
        ):
            report["UnreportableReason"] = (
                "You do not have the upgrade-system package installed."
                " Please report package upgrade failures against the package"
                " that failed to install, or against upgrade-manager."
            )

    if "Package" in report:
        package = report["Package"].split()[0]
        if package:
            apport.hookutils.attach_upstart_overrides(report, package)
            apport.hookutils.attach_upstart_logs(report, package)

    # build a duplicate signature tag for package reports
    if report.get("ProblemType") == "Package":

        if "DpkgTerminalLog" in report:
            # this was previously trimmed in check_attachment_for_errors
            termlog = report["DpkgTerminalLog"]
        elif "VarLogDistupgradeApttermlog" in report:
            termlog = get_attachment_contents(
                report, "VarLogDistupgradeApttermlog"
            )
        else:
            termlog = None
        if termlog:
            (package, version) = report["Package"].split(None, 1)
            # for packages that run update-grub include /etc/default/grub
            UPDATE_BOOT = [
                "friendly-recovery",
                "linux",
                "memtest86+",
                "plymouth",
                "ubuntu-meta",
                "virtualbox-ose",
            ]
            ug_failure = (
                r"/etc/kernel/post(inst|rm)\.d/"
                r"zz-update-grub exited with return code [1-9]+"
            )
            mkconfig_failure = (
                r"/usr/sbin/grub-mkconfig.*/etc/default/grub: Syntax error"
            )
            if re.search(ug_failure, termlog) or re.search(
                mkconfig_failure, termlog
            ):
                if report["SourcePackage"] in UPDATE_BOOT:
                    apport.hookutils.attach_default_grub(
                        report, "EtcDefaultGrub"
                    )
            dupe_sig = ""
            dupe_sig_created = False
            # messages we expect to see from a package manager (LP: #1692127)
            pkg_mngr_msgs = re.compile(
                r"^(Authenticating|De-configuring|Examining|Installing"
                r"|Preparing|Processing triggers|Purging|Removing|Replaced"
                r"|Replacing|Setting up|Unpacking|Would remove).*\.\.\.\s*$"
            )
            for line in termlog.split("\n"):
                if pkg_mngr_msgs.search(line):
                    dupe_sig = "%s\n" % line
                    dupe_sig_created = True
                    continue
                dupe_sig += "%s\n" % line
                # this doesn't catch 'dpkg-divert: error' LP: #1581399
                if "dpkg: error" in dupe_sig and line.startswith(" "):
                    if "trying to overwrite" in line:
                        conflict_pkg = re.search("in package (.*) ", line)
                        if (
                            conflict_pkg
                            and not apport.packaging.is_distro_package(
                                conflict_pkg.group(1)
                            )
                        ):
                            report["UnreportableReason"] = _(
                                "An Ubuntu package has a file conflict with a "
                                "package that is not a genuine Ubuntu package."
                            )
                        add_tag(report, "package-conflict")
                    if dupe_sig_created:
                        # the duplicate signature should be the first failure
                        report["DuplicateSignature"] = "package:%s:%s\n%s" % (
                            package,
                            version,
                            dupe_sig,
                        )
                        break
                if dupe_sig:
                    if dpkg_log_without_error.find(dupe_sig) != -1:
                        report["UnreportableReason"] = _(
                            "You have already encountered this package"
                            " installation failure."
                        )


def match_error_messages(report):
    # There are enough of these now that it is probably worth refactoring...
    # -mdz
    if report.get("ProblemType") == "Package":
        if (
            "failed to install/upgrade: corrupted filesystem tarfile"
            in report.get("Title", "")
        ):
            report["UnreportableReason"] = (
                "This failure was caused by a corrupted package download"
                " or file system corruption."
            )

        if "is already installed and configured" in report.get(
            "ErrorMessage", ""
        ):
            report["SourcePackage"] = "dpkg"


def check_attachment_for_errors(report, attachment):
    if report.get("ProblemType") == "Package":
        wrong_grub_msg = _(
            """\
Your system was initially configured with grub version 2, but you have\
 removed it from your system in favor of grub 1 without configuring it.\
 To ensure your bootloader configuration is updated whenever a new kernel\
 is available, open a terminal and run:

      sudo apt-get install grub-pc
"""
        )

        trim_dpkg_log(report)
        log_file = get_attachment_contents(report, attachment)

        grub_hook_failure = "DpkgTerminalLog" in report and bool(
            re.search(
                r"^Not creating /boot/grub/menu.lst as you wish",
                report["DpkgTerminalLog"],
                re.MULTILINE,
            )
        )

        if report["Package"] not in ["grub", "grub2"]:
            # linux-image postinst emits this when update-grub fails
            # https://wiki.ubuntu.com/KernelTeam/DebuggingUpdateErrors
            grub_errors = [
                r"^User postinst hook script \[.*update-grub\]"
                r" exited with value",
                r"^run-parts: /etc/kernel/post(inst|rm).d"
                r"/zz-update-grub exited with return code [1-9]+",
                r"^/usr/sbin/grub-probe: error",
            ]

            for grub_error in grub_errors:
                if attachment in report and re.search(
                    grub_error, log_file, re.MULTILINE
                ):
                    # File these reports on the grub package instead
                    grub_package = apport.packaging.get_file_package(
                        "/usr/sbin/update-grub"
                    )
                    if (
                        grub_package is None
                        or grub_package == "grub"
                        and "grub-probe" not in log_file
                    ):
                        report["SourcePackage"] = "grub"
                        if (
                            os.path.exists("/boot/grub/grub.cfg")
                            and grub_hook_failure
                        ):
                            report["UnreportableReason"] = wrong_grub_msg
                    else:
                        report["SourcePackage"] = "grub2"

        if report["Package"] != "initramfs-tools":
            # update-initramfs emits this when it fails, usually invoked
            # from the linux-image postinst
            # https://wiki.ubuntu.com/KernelTeam/DebuggingUpdateErrors
            if attachment in report and re.search(
                r"^update-initramfs: failed for ", log_file, re.MULTILINE
            ):
                # File these reports on the initramfs-tools package instead
                report["SourcePackage"] = "initramfs-tools"

        if (
            report["Package"].startswith("linux-image-")
            and attachment in report
        ):
            # /etc/kernel/*.d failures from kernel package postinst
            m = re.search(
                r"^run-parts: (/etc/kernel/\S+\.d/\S+)"
                r" exited with return code \d+",
                log_file,
                re.MULTILINE,
            )
            if m:
                path = m.group(1)
                package = apport.packaging.get_file_package(path)
                if package:
                    report["SourcePackage"] = package
                    report["ErrorMessage"] = m.group(0)
                    if package == "grub-pc" and grub_hook_failure:
                        report["UnreportableReason"] = wrong_grub_msg
                else:
                    report["UnreportableReason"] = (
                        "This failure was caused by a program"
                        " which did not originate from Ubuntu"
                    )

        error_message = report.get("ErrorMessage")
        corrupt_package = (
            "This failure was caused by a corrupted package download"
            " or file system corruption."
        )
        out_of_memory = (
            "This failure was caused by the system running out of memory."
        )

        if (
            "failed to install/upgrade: corrupted filesystem tarfile"
            in report.get("Title", "")
        ):
            report["UnreportableReason"] = corrupt_package

        if "dependency problems - leaving unconfigured" in error_message:
            report["UnreportableReason"] = (
                "This failure is a followup error from a previous"
                " package install failure."
            )

        if "failed to allocate memory" in error_message:
            report["UnreportableReason"] = out_of_memory

        if "cannot access archive" in error_message:
            report["UnreportableReason"] = corrupt_package

        if re.search(
            r"(failed to read|failed in write|short read) on buffer copy",
            error_message,
        ):
            report["UnreportableReason"] = corrupt_package

        if re.search(
            r"(failed to read|failed to write|failed to seek"
            r"|unexpected end of file or stream)",
            error_message,
        ):
            report["UnreportableReason"] = corrupt_package

        if re.search(
            r"(--fsys-tarfile|dpkg-deb --control)"
            r" returned error exit status 2",
            error_message,
        ):
            report["UnreportableReason"] = corrupt_package

        if attachment in report and re.search(
            r"dpkg-deb: error.*is not a debian format archive",
            log_file,
            re.MULTILINE,
        ):
            report["UnreportableReason"] = corrupt_package

        if "is already installed and configured" in report.get(
            "ErrorMessage", ""
        ):
            # there is insufficient information in the data currently gathered
            # so gather more data
            report["SourcePackage"] = "dpkg"
            report["AptdaemonVersion"] = apport.packaging.get_version(
                "aptdaemon"
            )
            apport.hookutils.attach_file_if_exists(
                report, "/var/log/dpkg.log", "DpkgLog"
            )
            apport.hookutils.attach_file_if_exists(
                report, "/var/log/apt/term.log", "AptTermLog"
            )
            # gather filenames in /var/crash to see if there is one for dpkg
            reports = glob("/var/crash/*")
            if reports:
                report["CrashReports"] = apport.hookutils.command_output(
                    ["stat", "-c", "%a:%u:%g:%s:%y:%x:%n"] + reports
                )
            add_tag(report, "already-installed")


def check_for_disk_error(report):
    devs_to_check = []
    if "Dmesg.txt" not in report and "CurrentDmesg.txt" not in report:
        return
    if "Df.txt" not in report:
        return
    df = report["Df.txt"]
    device_error = False
    for line in df:
        line = line.strip("\n")
        if (
            line.endswith("/")
            or line.endswith("/usr")
            or line.endswith("/var")
        ):
            # without manipulation it'd look like /dev/sda1
            device = line.split(" ")[0].strip("0123456789")
            device = device.replace("/dev/", "")
            devs_to_check.append(device)
    dmesg = report.get("CurrentDmesg.txt", report["Dmesg.txt"])
    for line in dmesg:
        line = line.strip("\n")
        if "I/O error" in line:
            # no device in this line
            if "journal commit I/O error" in line:
                continue
            for dev in devs_to_check:
                if re.search(dev, line):
                    error_device = dev
                    device_error = True
                    break
    if device_error:
        report["UnreportableReason"] = (
            "This failure was caused by a hardware error on /dev/%s"
            % error_device
        )


def add_kernel_info(report):
    # This includes the Ubuntu packaged kernel version
    apport.hookutils.attach_file_if_exists(
        report, "/proc/version_signature", "ProcVersionSignature"
    )


def add_release_info(report):
    # https://bugs.launchpad.net/bugs/364649
    media = "/var/log/installer/media-info"
    apport.hookutils.attach_file_if_exists(report, media, "InstallationMedia")
    # Preinstalled Raspberry Pi images include a build date breadcrumb
    apport.hookutils.attach_file_if_exists(
        report, "/.disk/info", "ImageMediaBuild"
    )
    if "ImageMediaBuild" in report:
        add_tag(report, "%s-image" % report["Architecture"])
        try:
            with open("/proc/device-tree/compatible", "rb") as f:
                is_a_pi = any(
                    vendor == "raspberrypi"
                    for s in f.read().split(b"\0")
                    if s
                    for vendor, model in (s.decode("ascii").split(",", 1),)
                )
        except FileNotFoundError:
            is_a_pi = False
        if is_a_pi:
            add_tag(report, "raspi-image")

    # if we are running from a live system, add the build timestamp
    apport.hookutils.attach_file_if_exists(
        report, "/cdrom/.disk/info", "LiveMediaBuild"
    )
    if os.path.exists("/cdrom/.disk/info"):
        report["CasperVersion"] = apport.packaging.get_version("casper")

    # https://wiki.ubuntu.com/FoundationsTeam/Specs/OemTrackingId
    apport.hookutils.attach_file_if_exists(
        report, "/var/lib/ubuntu_dist_channel", "DistributionChannelDescriptor"
    )

    release_codename = apport.hookutils.command_output(
        ["lsb_release", "-sc"], stderr=None
    )
    if release_codename.startswith("Error"):
        release_codename = None
    else:
        add_tag(report, release_codename)

    if os.path.exists(media):
        mtime = os.stat(media).st_mtime
        human_mtime = time.strftime("%Y-%m-%d", time.gmtime(mtime))
        delta = time.time() - mtime
        report["InstallationDate"] = "Installed on %s (%d days ago)" % (
            human_mtime,
            delta / 86400,
        )

    log = "/var/log/dist-upgrade/main.log"
    if os.path.exists(log):
        mtime = os.stat(log).st_mtime
        human_mtime = time.strftime("%Y-%m-%d", time.gmtime(mtime))
        delta = time.time() - mtime

        # Would be nice if this also showed which release was originally
        # installed
        report["UpgradeStatus"] = "Upgraded to %s on %s (%d days ago)" % (
            release_codename,
            human_mtime,
            delta / 86400,
        )
    else:
        report[
            "UpgradeStatus"
        ] = "No upgrade log present (probably fresh install)"


def add_proposed_info(report):
    """Tag if package comes from -proposed."""
    if "Package" not in report:
        return
    try:
        (package, version) = report["Package"].split()[:2]
    except ValueError:
        print("WARNING: malformed Package field: " + report["Package"])
        return

    apt_cache = subprocess.run(
        ["apt-cache", "showpkg", package],
        check=False,
        stdout=subprocess.PIPE,
        text=True,
    )
    if apt_cache.returncode != 0:
        print("WARNING: apt-cache showpkg %s failed" % package)
        return

    found_proposed = False
    found_updates = False
    found_security = False
    for line in apt_cache.stdout.splitlines():
        if line.startswith(version + " ("):
            if "-proposed_" in line:
                found_proposed = True
            if "-updates_" in line:
                found_updates = True
            if "-security" in line:
                found_security = True

    if found_proposed and not found_updates and not found_security:
        add_tag(report, "package-from-proposed")


def add_cloud_info(report):
    # EC2 and Ubuntu Enterprise Cloud instances
    ec2_instance = False
    for pkg in ("ec2-init", "cloud-init"):
        try:
            if apport.packaging.get_version(pkg):
                ec2_instance = True
                break
        except ValueError:
            pass
    if ec2_instance:
        metadata_url = "http://169.254.169.254/latest/meta-data/"
        ami_id_url = urljoin(metadata_url, "ami-id")

        try:
            with urlopen(ami_id_url, timeout=5) as url:
                ami = url.read()
        except urllib.error.URLError:
            ami = None

        if ami and ami.startswith(b"ami"):
            add_tag(report, "ec2-images")
            fields = {
                "Ec2AMIManifest": "ami-manifest-path",
                "Ec2Kernel": "kernel-id",
                "Ec2Ramdisk": "ramdisk-id",
                "Ec2InstanceType": "instance-type",
                "Ec2AvailabilityZone": "placement/availability-zone",
            }

            report["Ec2AMI"] = ami
            for key, value in fields.items():
                try:
                    with urlopen(
                        urljoin(metadata_url, value), timeout=5
                    ) as url:
                        report[key] = url.read()
                except urllib.error.URLError:
                    report[key] = "unavailable"
        else:
            add_tag(report, "uec-images")


def add_tag(report, tag):
    report.setdefault("Tags", "")
    if tag in report["Tags"].split():
        return
    report["Tags"] += " " + tag


def get_oem_project(report):
    """Determine OEM project name from Distribution Channel Descriptor.

    Return None if it cannot be determined or does not exist.
    """
    dcd = report.get("DistributionChannelDescriptor", None)
    if dcd and dcd.startswith("canonical-oem-"):
        return dcd.split("-")[2]
    return None


def trim_dpkg_log(report):
    """Trim DpkgTerminalLog to the most recent installation session."""
    if "DpkgTerminalLog" not in report:
        return
    if not report["DpkgTerminalLog"].strip():
        report[
            "UnreportableReason"
        ] = "/var/log/apt/term.log does not contain any data"
        return
    lines = []
    dpkg_log = report["DpkgTerminalLog"]
    if isinstance(dpkg_log, bytes):
        trim_re = re.compile(b"^\\(.* ... \\d+ .*\\)$")
        start_re = re.compile(b"^Log started:")
    else:
        trim_re = re.compile("^\\(.* ... \\d+ .*\\)$")
        start_re = re.compile("^Log started:")
    for line in dpkg_log.splitlines():
        if start_re.match(line) or trim_re.match(line):
            lines = []
            continue
        lines.append(line)
    # If trimming the log file fails, return the whole log file.
    if not lines:
        return
    if isinstance(lines[0], str):
        report["DpkgTerminalLog"] = "\n".join(lines)
    else:
        report["DpkgTerminalLog"] = "\n".join(
            [str(line.decode("UTF-8", "replace")) for line in lines]
        )


def get_attachment_contents(report, attachment):
    if isinstance(report[attachment], problem_report.CompressedValue):
        contents = report[attachment].get_value().decode("UTF-8")
    else:
        contents = report[attachment]
    return contents


def add_python_details(key, python, report):
    """Add comma separated details about which python is being used"""
    python_path = apport.hookutils.command_output(["which", python])
    if python_path.startswith("Error: "):
        report[key] = "N/A"
        return
    python_link = apport.hookutils.command_output(
        ["readlink", "-f", python_path]
    )
    python_pkg = apport.fileutils.find_file_package(python_path)
    if python_pkg:
        python_pkg_version = apport.packaging.get_version(python_pkg)
    python_version = apport.hookutils.command_output(
        [python_link, "--version"]
    )
    data = "%s, %s" % (python_link, python_version)
    if python_pkg:
        data += ", %s, %s" % (python_pkg, python_pkg_version)
    else:
        data += ", unpackaged"
    report[key] = data


def main():
    # for testing: update report file given on command line
    if len(sys.argv) != 2:
        sys.stderr.write(
            "Usage for testing this hook: %s <report file>\n" % sys.argv[0]
        )
        sys.exit(1)

    report_file = sys.argv[1]

    report = apport.Report()
    with open(report_file, "rb") as f:
        report.load(f)
    report_keys = set(report.keys())

    new_report = report.copy()
    add_info(new_report, None)

    new_report_keys = set(new_report.keys())

    # Show differences
    # N.B. Some differences will exist if the report file is not from your
    # system because the hook runs against your local system.
    changed = 0
    for key in sorted(report_keys | new_report_keys):
        if key in new_report_keys and key not in report_keys:
            print("+%s: %s" % (key, new_report[key]))
            changed += 1
        elif key in report_keys and key not in new_report_keys:
            print("-%s: (deleted)" % key)
            changed += 1
        elif key in report_keys and key in new_report_keys:
            if report[key] != new_report[key]:
                print("~%s: (changed)" % key)
                changed += 1
    print("%d items changed" % changed)


if __name__ == "__main__":
    main()
