apt: include apt preferences (e.g. pinning) when selecting packages (#78327)

Fixes #77969

changelogs/fragments/77969-apt-preferences.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - apt - fix package selection to include /etc/apt/preferences(.d) (https://github.com/ansible/ansible/issues/77969)

lib/ansible/modules/apt.py

@@ -210,6 +210,8 @@ notes:
      (If you typo C(foo) as C(fo) apt-get would install packages that have "fo" in their name with a warning and a prompt for the user.
      Since we don't have warnings and prompts before installing we disallow this.Use an explicit fnmatch pattern if you want wildcarding)
    - When used with a C(loop:) each package will be processed individually, it is much more efficient to pass the list directly to the I(name) option.
+   - When C(default_release) is used, an implicit priority of 990 is used. This is the same behavior as C(apt-get -t).
+   - When an exact version is specified, an implicit priority of 1001 is used.
 '''
 
 EXAMPLES = '''
@@ -485,12 +487,17 @@ def package_version_compare(version, other_version):
 
 def package_best_match(pkgname, version_cmp, version, release, cache):
     policy = apt_pkg.Policy(cache)
+
+    policy.read_pinfile(apt_pkg.config.find_file("Dir::Etc::preferences"))
+    policy.read_pindir(apt_pkg.config.find_file("Dir::Etc::preferencesparts"))
+
     if release:
         # 990 is the priority used in `apt-get -t`
         policy.create_pin('Release', pkgname, release, 990)
     if version_cmp == "=":
-        # You can't pin to a minimum version, only equality with a glob
-        policy.create_pin('Version', pkgname, version, 991)
+        # Installing a specific version from command line overrides all pinning
+        # We don't mimmic this exactly, but instead set a priority which is higher than all APT built-in pin priorities.
+        policy.create_pin('Version', pkgname, version, 1001)
     pkg = cache[pkgname]
     pkgver = policy.get_candidate_ver(pkg)
     if not pkgver:
@@ -503,6 +510,14 @@ def package_best_match(pkgname, version_cmp, version, release, cache):
 
 
 def package_status(m, pkgname, version_cmp, version, default_release, cache, state):
+    """
+    :return: A tuple of (installed, installed_version, version_installable, has_files). *installed* indicates whether
+    the package (regardless of version) is installed. *installed_version* indicates whether the installed package
+    matches the provided version criteria. *version_installable* provides the latest matching version that can be
+    installed. In the case of virtual packages where we can't determine an applicable match, True is returned.
+    *has_files* indicates whether the package has files on the filesystem (even if not installed, meaning a purge is
+    required).
+    """
     try:
         # get the package from the cache, as well as the
         # low-level apt_pkg.Package object which contains
@@ -527,15 +542,15 @@ def package_status(m, pkgname, version_cmp, version, default_release, cache, sta
 
                     # Otherwise return nothing so apt will sort out
                     # what package to satisfy this with
-                    return False, False, None, False
+                    return False, False, True, False
 
                 m.fail_json(msg="No package matching '%s' is available" % pkgname)
             except AttributeError:
                 # python-apt version too old to detect virtual packages
                 # mark as not installed and let apt-get install deal with it
-                return False, False, None, False
+                return False, False, True, False
         else:
-            return False, False, False, False
+            return False, False, None, False
     try:
         has_files = len(pkg.installed_files) > 0
     except UnicodeDecodeError:
@@ -565,13 +580,16 @@ def package_status(m, pkgname, version_cmp, version, default_release, cache, sta
         if version_cmp == "=":
             # check if the version is matched as well
             version_is_installed = fnmatch.fnmatch(installed_version, version)
+            if version_best and installed_version != version_best and fnmatch.fnmatch(version_best, version):
+                version_installable = version_best
         elif version_cmp == ">=":
             version_is_installed = apt_pkg.version_compare(installed_version, version) >= 0
+            if version_best and installed_version != version_best and apt_pkg.version_compare(version_best, version) >= 0:
+                version_installable = version_best
         else:
             version_is_installed = True
-
-        if installed_version != version_best:
-            version_installable = version_best
+            if version_best and installed_version != version_best:
+                version_installable = version_best
     else:
         version_installable = version_best
 
@@ -692,23 +710,27 @@ def install(m, pkgspec, cache, upgrade=False, default_release=None,
         name, version_cmp, version = package_split(package)
         package_names.append(name)
         installed, installed_version, version_installable, has_files = package_status(m, name, version_cmp, version, default_release, cache, state='install')
-        if (not installed and not only_upgrade) or (installed and not installed_version) or (upgrade and version_installable):
-            if version_installable or version:
-                pkg_list.append("'%s=%s'" % (name, version_installable or version))
+
+        if (not installed_version and not version_installable) or (not installed and only_upgrade):
+            status = False
+            data = dict(msg="no available installation candidate for %s" % package)
+            return (status, data)
+
+        if version_installable and ((not installed and not only_upgrade) or upgrade or not installed_version):
+            if version_installable is not True:
+                pkg_list.append("'%s=%s'" % (name, version_installable))
+            elif version:
+                pkg_list.append("'%s=%s'" % (name, version))
             else:
                 pkg_list.append("'%s'" % name)
         elif installed_version and version_installable and version_cmp == "=":
             # This happens when the package is installed, a newer version is
             # available, and the version is a wildcard that matches both
             #
-            # We do not apply the upgrade flag because we cannot specify both
-            # a version and state=latest.  (This behaviour mirrors how apt
-            # treats a version with wildcard in the package)
-            #
             # This is legacy behavior, and isn't documented (in fact it does
             # things documentations says it shouldn't). It should not be relied
             # upon.
-            pkg_list.append("'%s=%s'" % (name, version_installable))
+            pkg_list.append("'%s=%s'" % (name, version))
     packages = ' '.join(pkg_list)
 
     if packages:

test/integration/targets/apt/tasks/repo.yml

@@ -40,6 +40,17 @@
         state: absent
         allow_unauthenticated: yes
 
+- name: Try to install non-existent version
+  apt:
+    name: foo=99
+    state: present
+  ignore_errors: true
+  register: apt_result
+
+- name: Check if install failed
+  assert:
+    that:
+      - apt_result is failed
 
 # https://github.com/ansible/ansible/issues/30638
 - block:
@@ -56,6 +67,7 @@
     assert:
       that:
         - "apt_result is not changed"
+        - "apt_result is failed"
 
   - apt:
       name: foo=1.0.0
@@ -122,12 +134,58 @@
         - item.item.3 is none or "Inst foo [1.0.0] (" + item.item.3 + " localhost [all])" in item.stdout_lines
     loop: '{{ apt_result.results }}'
 
+  - name: Pin foo=1.0.0
+    copy:
+      content: |-
+        Package: foo
+        Pin: version 1.0.0
+        Pin-Priority: 1000
+      dest: /etc/apt/preferences.d/foo
+
+  - name: Run pinning version test matrix
+    apt:
+      name: foo{{ item.0 }}
+      default_release: '{{ item.1 }}'
+      state: '{{ item.2 | ternary("latest","present") }}'
+    check_mode: true
+    ignore_errors: true
+    register: apt_result
+    loop:
+      # [filter, release, state_latest, expected] # expected=null for no change. expected=False to assert an error
+      - ["", null, false, null]
+      - ["", null, true, null]
+      - ["=1.0.0", null, false, null]
+      - ["=1.0.0", null, true, null]
+      - ["=1.0.1", null, false, "1.0.1"]
+      #- ["=1.0.*", null, false, null] # legacy behavior. should not upgrade without state=latest
+      - ["=1.0.*", null, true, "1.0.1"]
+      - [">=1.0.0", null, false, null]
+      - [">=1.0.0", null, true, null]
+      - [">=1.0.1", null, false, False]
+      - ["", "testing", false, null]
+      - ["", "testing", true, null]
+      - ["=2.0.0", null, false, "2.0.0"]
+      - [">=2.0.0", "testing", false, False]
+
+  - name: Validate pinning version test matrix
+    assert:
+      that:
+        - (item.item.3 != False) or (item.item.3 == False and item is failed)
+        - (item.item.3 is string) == (item.stdout is defined)
+        - item.item.3 is not string or "Inst foo [1.0.0] (" + item.item.3 + " localhost [all])" in item.stdout_lines
+    loop: '{{ apt_result.results }}'
+
   always:
     - name: Uninstall foo
       apt:
         name: foo
         state: absent
 
+    - name: Unpin foo
+      file:
+        path: /etc/apt/preferences.d/foo
+        state: absent
+
 # https://github.com/ansible/ansible/issues/35900
 - block:
   - name: Disable ubuntu repos so system packages are not upgraded and do not change testing env