Merge branch 'drop_known_apks' into 'master'

Drop stats/known_apks.txt

See merge request fdroid/fdroidserver!1547

MANIFEST.in

@@ -860,7 +860,6 @@ include tests/source-files/ut.ewh.audiometrytest/settings.gradle
 include tests/source-files/yuriykulikov/AlarmClock/gradle/wrapper/gradle-wrapper.properties
 include tests/source-files/Zillode/syncthing-silk/build.gradle
 include tests/SpeedoMeterApp.main_1.apk
-include tests/stats/known_apks.txt
 include tests/testcommon.py
 include tests/test-gradlew-fdroid
 include tests/triple-t-2/build/org.piwigo.android/app/build.gradle

examples/config.yml

@@ -401,11 +401,6 @@
 # configured to allow push access (e.g. ssh key, username/password, etc)
 # binary_transparency_remote: git@gitlab.com:fdroid/binary-transparency-log.git
 
-# If you want to keep the "added" and "last updated" dates for each
-# app and APK in your repo, enable this.  The name comes from an old
-# system for tracking statistics that is no longer included.
-# update_stats: true
-
 # Set this to true to always use a build server. This saves specifying the
 # --server option on dedicated secure build server hosts.
 # build_server_always: true

fdroidserver/common.py

@@ -162,7 +162,6 @@ default_config = {
     'make_current_version_link': False,
     'current_version_name_source': 'Name',
     'deploy_process_logs': False,
-    'update_stats': False,
     'repo_maxage': 0,
     'build_server_always': False,
     'keystore': 'keystore.p12',
@@ -2555,42 +2554,19 @@ class KnownApks:
         this is parsed as a list from the end to allow the filename to
         have any combo of spaces.
         """
-        self.path = os.path.join('stats', 'known_apks.txt')
         self.apks = {}
-        if os.path.isfile(self.path):
+        for part in ('repo', 'archive'):
-            with open(self.path, 'r', encoding='utf-8') as f:
+            path = os.path.join(part, 'index-v2.json')
-                for line in f:
+            if os.path.isfile(path):
-                    t = line.rstrip().split(' ')
+                with open(path, 'r', encoding='utf-8') as f:
-                    if len(t) == 2:
+                    index = json.load(f)
-                        self.apks[t[0]] = (t[1], None)
+                    for appid, data in index["packages"].items():
-                    else:
+                        for version in data["versions"].values():
-                        appid = t[-2]
+                            filename = version["file"]["name"][1:]
-                        date = datetime.strptime(t[-1], '%Y-%m-%d')
+                            date = datetime.fromtimestamp(version["added"] // 1000, tz=timezone.utc)
-                        filename = line[0:line.rfind(appid) - 1]
+                            self.apks[filename] = date
-                        self.apks[filename] = (appid, date)
-                        check_system_clock(date, self.path)
-        self.changed = False
 
-    def writeifchanged(self):
+    def recordapk(self, apkName, default_date=None):
-        if not self.changed:
-            return
-
-        if not os.path.exists('stats'):
-            os.mkdir('stats')
-
-        lst = []
-        for apk, app in self.apks.items():
-            appid, added = app
-            line = apk + ' ' + appid
-            if added:
-                line += ' ' + added.strftime('%Y-%m-%d')
-            lst.append(line)
-
-        with open(self.path, 'w') as f:
-            for line in sorted(lst, key=natural_key):
-                f.write(line + '\n')
-
-    def recordapk(self, apkName, app, default_date=None):
         """
         Record an APK (if it's new, otherwise does nothing).
 
@@ -2601,37 +2577,9 @@ class KnownApks:
         """
         if apkName not in self.apks:
             if default_date is None:
-                default_date = datetime.utcnow()
+                default_date = datetime.now(timezone.utc)
-            self.apks[apkName] = (app, default_date)
+            self.apks[apkName] = default_date
-            self.changed = True
+        return self.apks[apkName]
-        _ignored, added = self.apks[apkName]
-        return added
-
-    def getapp(self, apkname):
-        """Look up information - given the 'apkname'.
-
-        Returns (app id, date added/None).
-        Or returns None for an unknown apk.
-        """
-        if apkname in self.apks:
-            return self.apks[apkname]
-        return None
-
-    def getlatest(self, num):
-        """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first."""
-        apps = {}
-        for apk, app in self.apks.items():
-            appid, added = app
-            if added:
-                if appid in apps:
-                    if apps[appid] > added:
-                        apps[appid] = added
-                else:
-                    apps[appid] = added
-        sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
-        lst = [app for app, _ignored in sortedapps]
-        lst.reverse()
-        return lst
 
 
 def get_file_extension(filename):

fdroidserver/index.py

@@ -77,7 +77,7 @@ def make(apps, apks, repodir, archive):
         sortedapps[appid] = apps[appid]
 
     repodict = collections.OrderedDict()
-    repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
+    repodict['timestamp'] = datetime.now(timezone.utc)
     repodict['version'] = METADATA_VERSION
 
     if common.config['repo_maxage'] != 0:

fdroidserver/nightly.py

@@ -367,7 +367,7 @@ You can use it with the [F-Droid](https://f-droid.org/) Android app.
 
 Last updated: {date}'''.format(repo_git_base=repo_git_base,
                                repo_url=repo_url,
-                               date=datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC'))
+                               date=datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC'))
         with open(readme_path, 'w') as fp:
             fp.write(readme)
         mirror_git_repo.git.add(all=True)
@@ -422,7 +422,6 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
             'keypass': PASSWORD,
             'keydname': DISTINGUISHED_NAME,
             'make_current_version_link': False,
-            'update_stats': True,
         }
         with open('config.yml', 'w') as fp:
             yaml.dump(config, fp, default_flow_style=False)
@@ -496,7 +495,9 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base,
         common.local_rsync(
             options, [repo_basedir + '/metadata/'], git_mirror_metadatadir + '/'
         )
-        common.local_rsync(options, [repo_basedir + '/stats/'], git_mirror_statsdir + '/')
+        stats = repo_basedir + '/stats/'
+        if os.path.exists(stats):
+            common.local_rsync(options, [stats], git_mirror_statsdir + '/')
         mirror_git_repo.git.add(all=True)
         mirror_git_repo.index.commit("update app metadata")
 

fdroidserver/scanner.py

@@ -29,7 +29,7 @@ import urllib.request
 import zipfile
 from argparse import ArgumentParser
 from dataclasses import dataclass, field, fields
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from enum import IntEnum
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -330,7 +330,7 @@ def get_embedded_classes(apkfile, depth=0):
 
 def _datetime_now():
     """Get datetime.now(), using this funciton allows mocking it for testing."""
-    return datetime.utcnow()
+    return datetime.now(timezone.utc)
 
 
 def _scanner_cachedir():
@@ -389,7 +389,7 @@ class SignatureDataController:
         last_updated = self.data.get("last_updated", None)
         if last_updated:
             try:
-                last_updated = datetime.fromtimestamp(last_updated)
+                last_updated = datetime.fromtimestamp(last_updated, timezone.utc)
             except ValueError as e:
                 raise SignatureDataMalformedException() from e
             except TypeError as e:

fdroidserver/update.py

@@ -362,7 +362,7 @@ def get_cache():
             if not isinstance(v['antiFeatures'], dict):
                 v['antiFeatures'] = {k: {} for k in sorted(v['antiFeatures'])}
         if 'added' in v:
-            v['added'] = datetime.fromtimestamp(v['added'])
+            v['added'] = datetime.fromtimestamp(v['added'], tz=timezone.utc)
 
     return apkcache
 
@@ -690,7 +690,7 @@ def scan_repo_for_ipas(apkcache, repodir, knownapks):
             apkcache[ipa_name] = ipa
             cachechanged = True
 
-        added = knownapks.recordapk(ipa_name, ipa['packageName'])
+        added = knownapks.recordapk(ipa_name)
         if added:
             ipa['added'] = added
 
@@ -1549,8 +1549,9 @@ def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False):
             default_date_param = None
 
         # Record in knownapks, getting the added date at the same time..
-        added = knownapks.recordapk(repo_file['apkName'], repo_file['packageName'],
+        added = knownapks.recordapk(
-                                    default_date=default_date_param)
+            repo_file['apkName'], default_date=default_date_param
+        )
         if added:
             repo_file['added'] = added
 
@@ -2001,13 +2002,12 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
         fill_missing_icon_densities(empty_densities, iconfilename, apk, repodir)
 
         if use_date_from_apk:
-            default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime)
+            default_date_param = datetime.fromtimestamp(os.stat(apkfile).st_mtime, tz=timezone.utc)
         else:
             default_date_param = None
 
         # Record in known apks, getting the added date at the same time..
-        added = knownapks.recordapk(apk['apkName'], apk['packageName'],
+        added = knownapks.recordapk(apk['apkName'], default_date=default_date_param)
-                                    default_date=default_date_param)
         if added:
             apk['added'] = added
 
@@ -2420,12 +2420,12 @@ def create_metadata_from_template(apk):
 def read_added_date_from_all_apks(apps, apks):
     """No summary.
 
-    Added dates come from the stats/known_apks.txt file but are
+    Added dates come from the repo/index-v2.json file but are
     read when scanning apks and thus need to be applied form apk
     level to app level for _all_ apps and not only from non-archived
     ones
 
-    TODO: read the added dates directly from known_apks.txt instead of
+    TODO: read the added dates directly from index-v2.json instead of
           going through apks that way it also works for for repos that
           don't keep an archive of apks.
     """
@@ -2772,10 +2772,6 @@ def main():
         from . import btlog
         btlog.make_binary_transparency_log(repodirs)
 
-    if config['update_stats']:
-        # Update known apks info...
-        knownapks.writeifchanged()
-
     status_update_json(apps, apks + archapks)
 
     logging.info(_("Finished"))

tests/common.TestCase

@@ -22,6 +22,7 @@ import textwrap
 import yaml
 import gzip
 from argparse import ArgumentParser
+from datetime import datetime, timezone
 from zipfile import BadZipFile, ZipFile
 from unittest import mock
 from pathlib import Path
@@ -1898,7 +1899,6 @@ class CommonTest(unittest.TestCase):
         self.assertFalse(os.path.exists('config.yml'))
         self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config()
-        self.assertFalse(config.get('update_stats'))
         self.assertIsNotNone(config.get('char_limits'))
 
     def test_with_zero_size_config(self):
@@ -1908,7 +1908,6 @@ class CommonTest(unittest.TestCase):
         self.assertTrue(os.path.exists('config.yml'))
         self.assertFalse(os.path.exists('config.py'))
         config = fdroidserver.common.read_config()
-        self.assertFalse(config.get('update_stats'))
         self.assertIsNotNone(config.get('char_limits'))
 
     def test_with_config_yml(self):
@@ -2937,6 +2936,37 @@ class CommonTest(unittest.TestCase):
             [{'url': s}], fdroidserver.common.parse_mirrors_config(mirrors)
         )
 
+    def test_KnownApks_recordapk(self):
+        """Test that added dates are being fetched from the index.
+
+        There are more related tests in tests/run-tests.
+
+        """
+        now = datetime.now(timezone.utc)
+        knownapks = fdroidserver.common.KnownApks()
+        for apkName in knownapks.apks:
+            knownapks.recordapk(apkName, default_date=now)
+        for added in knownapks.apks.values():
+            self.assertNotEqual(added, now)
+
+    def test_KnownApks_recordapk_new(self):
+        """Test that new added dates work, and are not replaced later.
+
+        There are more related tests in tests/run-tests.
+
+        """
+        now = datetime.now(timezone.utc)
+        knownapks = fdroidserver.common.KnownApks()
+        fake_apk = 'fake.apk'
+        knownapks.recordapk(fake_apk, default_date=now)
+        for apk, added in knownapks.apks.items():
+            if apk == fake_apk:
+                self.assertEqual(added, now)
+            else:
+                self.assertNotEqual(added, now)
+        knownapks.recordapk(fake_apk, default_date=datetime.now(timezone.utc))
+        self.assertEqual(knownapks.apks[fake_apk], now)
+
 
 APKS_WITH_JAR_SIGNATURES = (
     (

tests/config.py

@@ -25,8 +25,6 @@ mirrors = (
     'https://foo.bar/fdroid',
 )
 
-update_stats = True
-
 install_list = 'org.adaway'
 uninstall_list = ('com.android.vending', 'com.facebook.orca', )
 

tests/nightly.TestCase

@@ -274,7 +274,6 @@ class NightlyTest(unittest.TestCase):
             'repo_name': 'f-droid/test-nightly',
             'repo_url': mirror_url + '/repo',
             'servergitmirrors': [{"url": git_url}],
-            'update_stats': True,
         }
         with open('config.yml') as fp:
             config = yaml.safe_load(fp)
@@ -347,7 +346,6 @@ class NightlyTest(unittest.TestCase):
             'repo_name': 'fdroid/test-nightly',
             'repo_url': 'https://gitlab.com/fdroid/test-nightly/-/raw/master/fdroid/repo',
             'servergitmirrors': [{"url": 'git@gitlab.com:fdroid/test-nightly'}],
-            'update_stats': True,
         }
         with open('config.yml') as fp:
             config = yaml.safe_load(fp)

tests/run-tests

@@ -273,7 +273,6 @@ cp -a \
    $WORKSPACE/tests/config \
    $WORKSPACE/tests/metadata \
    $WORKSPACE/tests/repo \
-   $WORKSPACE/tests/stats \
    $REPOROOT/
 cp -a $WORKSPACE/tests/gnupghome $GNUPGHOME
 chmod 0700 $GNUPGHOME
@@ -645,14 +644,13 @@ printf '\narchive_older: 3\n' >> config.yml
 mkdir -p {repo,archive,metadata,stats}
 cp $WORKSPACE/tests/repo/com.politedroid_5.apk archive
 cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo
+cp $WORKSPACE/tests/repo/index-v2.json repo
 cp $WORKSPACE/tests/metadata/com.politedroid.yml metadata
 #TODO: the timestamp of the oldest apk in the file should be used, even if that
 # doesn't exist anymore
-echo "com.politedroid_4.apk com.politedroid 2016-01-01" > stats/known_apks.txt
-echo "com.politedroid_5.apk com.politedroid 2017-01-01" >> stats/known_apks.txt
-echo "com.politedroid_6.apk com.politedroid 2018-01-01" >> stats/known_apks.txt
 $sed -i -e 's/ArchivePolicy:.*/ArchivePolicy: 1 versions/' metadata/com.politedroid.yml
-timestamp=1483228800  # $(date -u --date=2017-01-01 +%s)000
+timestamp=1498176000000  # $(date -u --date=2017-01-01 +%s)000
+pwd
 
 $fdroid update --pretty --nosign
 grep -F "\"added\": $timestamp" repo/index-v1.json
@@ -1078,7 +1076,7 @@ GIT_REMOTE=`create_test_dir`
 GNUPGHOME=$REPOROOT/gnupghome
 cd $REPOROOT
 fdroid_init_with_prebuilt_keystore
-cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/
+cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $REPOROOT/
 echo "binary_transparency_remote: $GIT_REMOTE" >> config.yml
 $fdroid update --verbose
 $fdroid deploy --verbose
@@ -1224,7 +1222,7 @@ $git config receive.denyCurrentBranch updateInstead
 cd $OFFLINE_ROOT
 fdroid_init_with_prebuilt_keystore
 printf '\narchive_older: 3\n' >> config.yml
-cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $OFFLINE_ROOT/
+cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $OFFLINE_ROOT/
 mkdir $OFFLINE_ROOT/unsigned
 cp $WORKSPACE/tests/urzip-release-unsigned.apk $OFFLINE_ROOT/unsigned
 

tests/scanner.TestCase

@@ -14,7 +14,7 @@ import unittest
 import uuid
 import zipfile
 from dataclasses import asdict
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from unittest import mock
 
 if sys.version_info >= (3, 11):
@@ -592,7 +592,7 @@ class Test_SignatureDataController(unittest.TestCase):
         sdc = fdroidserver.scanner.SignatureDataController(
             'nnn', 'fff.yml', 'https://example.com/test.json'
         )
-        sdc.data['last_updated'] = datetime.utcnow().timestamp()
+        sdc.data['last_updated'] = datetime.now(timezone.utc).timestamp()
         sdc.check_last_updated()
 
     def test_check_last_updated_exception_cache_outdated(self):
@@ -600,7 +600,9 @@ class Test_SignatureDataController(unittest.TestCase):
             'nnn', 'fff.yml', 'https://example.com/test.json'
         )
         sdc.cache_duration = timedelta(days=7)
-        sdc.data['last_updated'] = (datetime.utcnow() - timedelta(days=30)).timestamp()
+        sdc.data['last_updated'] = (
+            datetime.now(timezone.utc) - timedelta(days=30)
+        ).timestamp()
         with self.assertRaises(fdroidserver.scanner.SignatureDataOutdatedException):
             sdc.check_last_updated()
 

tests/stats/known_apks.txt

@@ -1,18 +0,0 @@
-com.example.test.helloworld_1.apk com.example.test.helloworld 2023-02-17
-com.politedroid_3.apk com.politedroid 2017-06-23
-com.politedroid_4.apk com.politedroid 2017-06-23
-com.politedroid_5.apk com.politedroid 2017-06-23
-com.politedroid_6.apk com.politedroid 2017-06-23
-duplicate.permisssions_9999999.apk duplicate.permisssions 2017-12-22
-fake.ota.update_1234.zip fake.ota.update 2016-03-10
-info.zwanenburg.caffeinetile_4.apk info.zwanenburg.caffeinetile 2018-10-10
-no.min.target.sdk_987.apk no.min.target.sdk 2018-10-10
-obb.main.oldversion_1444412523.apk obb.main.oldversion 2013-12-31
-obb.main.twoversions_1101613.apk obb.main.twoversions 2015-10-12
-obb.main.twoversions_1101615.apk obb.main.twoversions 2016-01-01
-obb.main.twoversions_1101617.apk obb.main.twoversions 2016-06-20
-obb.mainpatch.current_1619.apk obb.mainpatch.current 2016-04-23
-obb.mainpatch.current_1619_another-release-key.apk obb.mainpatch.current 2017-06-01
-souch.smsbypass_9.apk souch.smsbypass 2018-04-26
-urzip-; Рахма́, [rɐxˈmanʲɪnəf] سيرجي_رخمانينوف 谢·.apk info.guardianproject.urzip 2016-06-23
-v1.v2.sig_1020.apk v1.v2.sig 2023-02-17

tests/update.TestCase

@@ -697,14 +697,10 @@ class UpdateTest(unittest.TestCase):
 
         os.chdir(self.testdir)
         os.mkdir('repo')
-        os.mkdir('stats')
-        with open(os.path.join('stats', 'known_apks.txt'), 'w') as fp:
-            fp.write('se.manyver_30.apk se.manyver 2018-10-10\n')
         filename = 'Norway_bouvet_europe_2.obf.zip'
         shutil.copy(os.path.join(self.basedir, filename), 'repo')
         knownapks = fdroidserver.common.KnownApks()
         files, fcachechanged = fdroidserver.update.scan_repo_files(dict(), 'repo', knownapks, False)
-        knownapks.writeifchanged()
         self.assertTrue(fcachechanged)
 
         info = files[0]
@@ -2052,12 +2048,10 @@ class TestScanRepoForIpas(unittest.TestCase):
 
             knownapks.recordapk.call_count = 2
             self.assertTrue(
-                unittest.mock.call('abc.Def_123.ipa', 'abc')
+                unittest.mock.call('abc.Def_123.ipa') in knownapks.recordapk.mock_calls
-                in knownapks.recordapk.mock_calls
             )
             self.assertTrue(
-                unittest.mock.call('xyz.XXX_123.ipa', 'xyz')
+                unittest.mock.call('xyz.XXX_123.ipa') in knownapks.recordapk.mock_calls
-                in knownapks.recordapk.mock_calls
             )