Merge branch '427-yml-rewritemeta-produces-unreadble-output-for-multiline-scripts' into 'master'

Resolve "yml rewritemeta produces unreadble output for multiline scripts"

Closes #427

See merge request fdroid/fdroidserver!564

fdroidserver/metadata.py

@@ -1091,10 +1091,23 @@ def parse_yaml_metadata(mf, app):
                             _("Unrecognised build flag '{build_flag}' "
                               "in '{path}'").format(build_flag=build_flag,
                                                     path=mf.name))
+        post_parse_yaml_metadata(yamldata)
         app.update(yamldata)
     return app
 
 
+def post_parse_yaml_metadata(yamldata):
+    """transform yaml metadata to our internal data format"""
+    for build in yamldata.get('Builds', []):
+        for flag in build.keys():
+            _flagtype = flagtype(flag)
+
+            # concatenate script flags into a single string if they are stored as list
+            if _flagtype is TYPE_SCRIPT:
+                if isinstance(build[flag], list):
+                    build[flag] = ' && '.join(build[flag])
+
+
 def write_yaml(mf, app):
     """Write metadata in yaml format.
 
@@ -1146,10 +1159,17 @@ def write_yaml(mf, app):
             else:
                 return str(value)
         elif typ is TYPE_SCRIPT:
-            if len(value) > 50:
-                return ruamel.yaml.scalarstring.preserve_literal(value)
+            if type(value) == list:
+                if len(value) == 1:
+                    return value[0]
+                else:
+                    return value
             else:
-                return value
+                script_lines = value.split(' && ')
+                if len(script_lines) > 1:
+                    return script_lines
+                else:
+                    return value
         else:
             return value
 
@@ -1185,8 +1205,8 @@ def write_yaml(mf, app):
         for build in app.builds:
             b = ruamel.yaml.comments.CommentedMap()
             for field in build_flags:
-                if hasattr(build, field) and getattr(build, field):
-                    value = getattr(build, field)
+                value = getattr(build, field)
+                if hasattr(build, field) and value:
                     if field == 'gradle' and value == ['off']:
                         value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
                     if field in ('maven', 'buildozer'):

tests/metadata-rewrite-yml/app.with.special.build.params.yml

@@ -38,8 +38,9 @@ Builds:
       - FacebookSDK@sdk-version-3.0.1
     rm:
       - libs/appbrain-sdk-android.jar
-    prebuild: |-
-        sed -i 's@\(reference.1=\).*@\1$$FacebookSDK$$@' project.properties && sed -i 's/Class\[\]/Class\<?\>\[\]/g' $$FacebookSDK$$/src/com/facebook/model/GraphObject.java
+    prebuild:
+      - sed -i 's@\(reference.1=\).*@\1$$FacebookSDK$$@' project.properties
+      - sed -i 's/Class\[\]/Class\<?\>\[\]/g' $$FacebookSDK$$/src/com/facebook/model/GraphObject.java
 
   - versionName: 1.2.2
     versionCode: 42
@@ -52,8 +53,10 @@ Builds:
       - libs/appbrain-sdk-android.jar
     extlibs:
       - android/android-support-v4.jar
-    prebuild: |-
-        mv libs/android-support-v4.jar $$FacebookSDK$$/libs/ && sed -i 's@\(reference.1=\).*@\1$$FacebookSDK$$@' project.properties && sed -i 's/Class\[\]/Class\<?\>\[\]/g'          $$FacebookSDK$$/src/com/facebook/model/GraphObject.java
+    prebuild:
+      - mv libs/android-support-v4.jar $$FacebookSDK$$/libs/
+      - sed -i 's@\(reference.1=\).*@\1$$FacebookSDK$$@' project.properties
+      - sed -i 's/Class\[\]/Class\<?\>\[\]/g'          $$FacebookSDK$$/src/com/facebook/model/GraphObject.java
 
   - versionName: 2.1.1
     versionCode: 48

tests/metadata.TestCase

@@ -209,6 +209,258 @@ class MetadataTest(unittest.TestCase):
             UpdateCheckMode: Tags
             """))
 
+    def test_parse_yaml_metadata_prebuild_list(self):
+        mf = io.StringIO(textwrap.dedent("""\
+            AutoName: F-Droid
+            RepoType: git
+            Builds:
+                - versionCode: 1
+                  versionName: v0.1.0
+                  sudo:
+                    - apt-get update
+                    - apt-get install -y whatever
+                    - sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
+                  init:
+                    - bash generate_some_file.sh
+                    - sed -i -e 'g/what/ever/' /some/file
+                  prebuild:
+                    - npm something
+                    - echo 'important setting' >> /a/file
+                  build:
+                    - ./gradlew someSpecialTask
+                    - sed -i 'd/that wrong config/' gradle.properties
+                    - ./gradlew compile
+            """))
+        mf.name = 'mock_filename.yaml'
+        mf.seek(0)
+        result = {}
+        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
+            fdroidserver.metadata.parse_yaml_metadata(mf, result)
+        self.maxDiff = None
+        self.assertDictEqual(result, {'AutoName': 'F-Droid',
+                                      'RepoType': 'git',
+                                      'Builds': [{'versionCode': 1,
+                                                  'versionName': 'v0.1.0',
+                                                  'sudo': "apt-get update && "
+                                                          "apt-get install -y whatever && "
+                                                          "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
+                                                  'init': "bash generate_some_file.sh && "
+                                                          "sed -i -e 'g/what/ever/' /some/file",
+                                                  'prebuild': "npm something && echo 'important setting' >> /a/file",
+                                                  'build': "./gradlew someSpecialTask && "
+                                                           "sed -i 'd/that wrong config/' gradle.properties && "
+                                                           "./gradlew compile"}]})
+
+    def test_parse_yaml_metadata_prebuild_strings(self):
+        mf = io.StringIO(textwrap.dedent("""\
+            AutoName: F-Droid
+            RepoType: git
+            Builds:
+                - versionCode: 1
+                  versionName: v0.1.0
+                  sudo: |-
+                    apt-get update && apt-get install -y whatever && sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
+                  init: bash generate_some_file.sh && sed -i -e 'g/what/ever/' /some/file
+                  prebuild: npm something && echo 'important setting' >> /a/file
+                  build: |-
+                    ./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile
+            """))
+        mf.name = 'mock_filename.yaml'
+        mf.seek(0)
+        result = {}
+        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
+            fdroidserver.metadata.parse_yaml_metadata(mf, result)
+        self.maxDiff = None
+        self.assertDictEqual(result, {'AutoName': 'F-Droid',
+                                      'RepoType': 'git',
+                                      'Builds': [{'versionCode': 1,
+                                                  'versionName': 'v0.1.0',
+                                                  'sudo': "apt-get update && "
+                                                          "apt-get install -y whatever && "
+                                                          "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
+                                                  'init': "bash generate_some_file.sh && "
+                                                          "sed -i -e 'g/what/ever/' /some/file",
+                                                  'prebuild': "npm something && echo 'important setting' >> /a/file",
+                                                  'build': "./gradlew someSpecialTask && "
+                                                           "sed -i 'd/that wrong config/' gradle.properties && "
+                                                           "./gradlew compile"}]})
+
+    def test_parse_yaml_metadata_prebuild_string(self):
+        mf = io.StringIO(textwrap.dedent("""\
+            AutoName: F-Droid
+            RepoType: git
+            Builds:
+                - versionCode: 1
+                  versionName: v0.1.0
+                  prebuild: |-
+                    a && b && sed -i 's,a,b,'
+            """))
+        mf.name = 'mock_filename.yaml'
+        mf.seek(0)
+        result = {}
+        with mock.patch('fdroidserver.metadata.warnings_action', 'error'):
+            fdroidserver.metadata.parse_yaml_metadata(mf, result)
+        self.assertDictEqual(result, {'AutoName': 'F-Droid',
+                                      'RepoType': 'git',
+                                      'Builds': [{'versionCode': 1,
+                                                  'versionName': 'v0.1.0',
+                                                  'prebuild': "a && b && "
+                                                              "sed -i 's,a,b,'"}]})
+
+    def test_write_yaml_1_line_scripts_as_string(self):
+        mf = io.StringIO()
+        app = fdroidserver.metadata.App()
+        app.Categories = ['None']
+        app.builds = []
+        build = fdroidserver.metadata.Build()
+        build.versionCode = 102030
+        build.versionName = 'v1.2.3'
+        build.sudo = "chmod +rwx /opt"
+        build.init = "sed -i -e 'g/what/ever/' /some/file"
+        build.prebuild = "sed -i 'd/that wrong config/' gradle.properties"
+        build.build = "./gradlew compile"
+        app.builds.append(build)
+        fdroidserver.metadata.write_yaml(mf, app)
+        mf.seek(0)
+        self.assertEqual(mf.read(), textwrap.dedent("""\
+            Categories:
+              - None
+            License: Unknown
+
+            Builds:
+              - versionName: v1.2.3
+                versionCode: 102030
+                sudo: chmod +rwx /opt
+                init: sed -i -e 'g/what/ever/' /some/file
+                prebuild: sed -i 'd/that wrong config/' gradle.properties
+                build: ./gradlew compile
+
+            AutoUpdateMode: None
+            UpdateCheckMode: None
+            """))
+
+    def test_write_yaml_1_line_scripts_as_list(self):
+        mf = io.StringIO()
+        app = fdroidserver.metadata.App()
+        app.Categories = ['None']
+        app.builds = []
+        build = fdroidserver.metadata.Build()
+        build.versionCode = 102030
+        build.versionName = 'v1.2.3'
+        build.sudo = ["chmod +rwx /opt"]
+        build.init = ["sed -i -e 'g/what/ever/' /some/file"]
+        build.prebuild = ["sed -i 'd/that wrong config/' gradle.properties"]
+        build.build = ["./gradlew compile"]
+        app.builds.append(build)
+        fdroidserver.metadata.write_yaml(mf, app)
+        mf.seek(0)
+        self.assertEqual(mf.read(), textwrap.dedent("""\
+            Categories:
+              - None
+            License: Unknown
+
+            Builds:
+              - versionName: v1.2.3
+                versionCode: 102030
+                sudo: chmod +rwx /opt
+                init: sed -i -e 'g/what/ever/' /some/file
+                prebuild: sed -i 'd/that wrong config/' gradle.properties
+                build: ./gradlew compile
+
+            AutoUpdateMode: None
+            UpdateCheckMode: None
+            """))
+
+    def test_write_yaml_multiline_scripts_from_list(self):
+        mf = io.StringIO()
+        app = fdroidserver.metadata.App()
+        app.Categories = ['None']
+        app.builds = []
+        build = fdroidserver.metadata.Build()
+        build.versionCode = 102030
+        build.versionName = 'v1.2.3'
+        build.sudo = ["apt-get update",
+                      "apt-get install -y whatever",
+                      "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml"]
+        build.init = ["bash generate_some_file.sh",
+                      "sed -i -e 'g/what/ever/' /some/file"]
+        build.prebuild = ["npm something",
+                          "echo 'important setting' >> /a/file"]
+        build.build = ["./gradlew someSpecialTask",
+                       "sed -i 'd/that wrong config/' gradle.properties",
+                       "./gradlew compile"]
+        app.builds.append(build)
+        fdroidserver.metadata.write_yaml(mf, app)
+        mf.seek(0)
+        self.assertEqual(mf.read(), textwrap.dedent("""\
+            Categories:
+              - None
+            License: Unknown
+
+            Builds:
+              - versionName: v1.2.3
+                versionCode: 102030
+                sudo:
+                  - apt-get update
+                  - apt-get install -y whatever
+                  - sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
+                init:
+                  - bash generate_some_file.sh
+                  - sed -i -e 'g/what/ever/' /some/file
+                prebuild:
+                  - npm something
+                  - echo 'important setting' >> /a/file
+                build:
+                  - ./gradlew someSpecialTask
+                  - sed -i 'd/that wrong config/' gradle.properties
+                  - ./gradlew compile
+
+            AutoUpdateMode: None
+            UpdateCheckMode: None
+            """))
+
+    def test_write_yaml_multiline_scripts_from_string(self):
+        mf = io.StringIO()
+        app = fdroidserver.metadata.App()
+        app.Categories = ['None']
+        app.builds = []
+        build = fdroidserver.metadata.Build()
+        build.versionCode = 102030
+        build.versionName = 'v1.2.3'
+        build.sudo = "apt-get update && apt-get install -y whatever && sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml"
+        build.init = "bash generate_some_file.sh && sed -i -e 'g/what/ever/' /some/file"
+        build.prebuild = "npm something && echo 'important setting' >> /a/file"
+        build.build = "./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile"
+        app.builds.append(build)
+        fdroidserver.metadata.write_yaml(mf, app)
+        mf.seek(0)
+        self.assertEqual(mf.read(), textwrap.dedent("""\
+            Categories:
+              - None
+            License: Unknown
+
+            Builds:
+              - versionName: v1.2.3
+                versionCode: 102030
+                sudo:
+                  - apt-get update
+                  - apt-get install -y whatever
+                  - sed -i -e 's/<that attr="bad"/<that attr="good"/' ~/.whatever/config.xml
+                init:
+                  - bash generate_some_file.sh
+                  - sed -i -e 'g/what/ever/' /some/file
+                prebuild:
+                  - npm something
+                  - echo 'important setting' >> /a/file
+                build:
+                  - ./gradlew someSpecialTask
+                  - sed -i 'd/that wrong config/' gradle.properties
+                  - ./gradlew compile
+
+            AutoUpdateMode: None
+            UpdateCheckMode: None
+            """))
+
 
 if __name__ == "__main__":
     os.chdir(os.path.dirname(__file__))