Run shell scripts with -e (Closes: #1035)

Make sudo, init prebuild, build and Prepare fields lists and only
concatenate them with '; ' before execution. This allows arbitrary
commands inside the fileds (even && and ';') as we don't need to split
the commands again for rewritemeta.

tests/common.TestCase

@@ -375,7 +375,7 @@ class CommonTest(unittest.TestCase):
         build = fdroidserver.metadata.Build()
         build.commit = 'master'
         build.gradle = ['yes']
-        build.prebuild = 'test -d $$FakeSrcLib$$/testdirshouldexist'  # actual test condition
+        build.prebuild = ['test -d $$FakeSrcLib$$/testdirshouldexist']  # actual test condition
         build.srclibs = [srclibname + '@1.2.3']
         build.subdir = subdir
         build.versionCode = 0xCAFE

tests/dump_internal_metadata_format.py

@@ -40,7 +40,17 @@ import fdroidserver.metadata  # noqa
 
 def _build_yaml_representer(dumper, data):
     """Create a YAML representation of a Build instance."""
-    return dumper.represent_dict(data)
+    # internal representation of keys were switched
+    # to lists instead of strings concatenated by &&
+    # https://gitlab.com/fdroid/fdroidserver/merge_requests/1185
+    output = {}
+    for k, v in data.items():
+        if k in ("build", "init", "prebuild", "sudo"):
+            output[k] = " && ".join(v)
+        else:
+            output[k] = v
+
+    return dumper.represent_dict(output)
 
 
 parser = ArgumentParser()

tests/metadata.TestCase

@@ -504,15 +504,24 @@ class MetadataTest(unittest.TestCase):
                     {
                         'versionCode': 1,
                         'versionName': 'v0.1.0',
-                        'sudo': "apt-get update && "
-                        "apt-get install -y whatever && "
-                        "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
-                        'init': "bash generate_some_file.sh && "
-                        "sed -i -e 'g/what/ever/' /some/file",
-                        'prebuild': "npm something && echo 'important setting' >> /a/file",
-                        'build': "./gradlew someSpecialTask && "
-                        "sed -i 'd/that wrong config/' gradle.properties && "
-                        "./gradlew compile",
+                        'sudo': [
+                            "apt-get update",
+                            "apt-get install -y whatever",
+                            "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
+                        ],
+                        'init': [
+                            "bash generate_some_file.sh",
+                            "sed -i -e 'g/what/ever/' /some/file",
+                        ],
+                        'prebuild': [
+                            "npm something",
+                            "echo 'important setting' >> /a/file",
+                        ],
+                        'build': [
+                            "./gradlew someSpecialTask",
+                            "sed -i 'd/that wrong config/' gradle.properties",
+                            "./gradlew compile",
+                        ],
                     }
                 ],
             },
@@ -551,15 +560,23 @@ class MetadataTest(unittest.TestCase):
                     {
                         'versionCode': 1,
                         'versionName': 'v0.1.0',
-                        'sudo': "apt-get update && "
-                        "apt-get install -y whatever && "
-                        "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
-                        'init': "bash generate_some_file.sh && "
-                        "sed -i -e 'g/what/ever/' /some/file",
-                        'prebuild': "npm something && echo 'important setting' >> /a/file",
-                        'build': "./gradlew someSpecialTask && "
-                        "sed -i 'd/that wrong config/' gradle.properties && "
-                        "./gradlew compile",
+                        'sudo': [
+                            "apt-get update && "
+                            "apt-get install -y whatever && "
+                            "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml"
+                        ],
+                        'init': [
+                            "bash generate_some_file.sh && "
+                            "sed -i -e 'g/what/ever/' /some/file"
+                        ],
+                        'prebuild': [
+                            "npm something && echo 'important setting' >> /a/file"
+                        ],
+                        'build': [
+                            "./gradlew someSpecialTask && "
+                            "sed -i 'd/that wrong config/' gradle.properties && "
+                            "./gradlew compile"
+                        ],
                     }
                 ],
             },
@@ -593,7 +610,7 @@ class MetadataTest(unittest.TestCase):
                     {
                         'versionCode': 1,
                         'versionName': 'v0.1.0',
-                        'prebuild': "a && b && " "sed -i 's,a,b,'",
+                        'prebuild': ["a && b && " "sed -i 's,a,b,'"],
                     }
                 ],
             },
@@ -630,10 +647,10 @@ class MetadataTest(unittest.TestCase):
         build = fdroidserver.metadata.Build()
         build.versionCode = 102030
         build.versionName = 'v1.2.3'
-        build.sudo = "chmod +rwx /opt"
-        build.init = "sed -i -e 'g/what/ever/' /some/file"
-        build.prebuild = "sed -i 'd/that wrong config/' gradle.properties"
-        build.build = "./gradlew compile"
+        build.sudo = ["chmod +rwx /opt"]
+        build.init = ["sed -i -e 'g/what/ever/' /some/file"]
+        build.prebuild = ["sed -i 'd/that wrong config/' gradle.properties"]
+        build.build = ["./gradlew compile"]
         app['Builds'].append(build)
         fdroidserver.metadata.write_yaml(mf, app)
         mf.seek(0)
@@ -762,10 +779,21 @@ class MetadataTest(unittest.TestCase):
         build = fdroidserver.metadata.Build()
         build.versionCode = 102030
         build.versionName = 'v1.2.3'
-        build.sudo = "apt-get update && apt-get install -y whatever && sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml"
-        build.init = "bash generate_some_file.sh && sed -i -e 'g/what/ever/' /some/file"
-        build.prebuild = "npm something && echo 'important setting' >> /a/file"
-        build.build = "./gradlew someSpecialTask && sed -i 'd/that wrong config/' gradle.properties && ./gradlew compile"
+        build.sudo = [
+            "apt-get update",
+            "apt-get install -y whatever",
+            "sed -i -e 's/<that attr=\"bad\"/<that attr=\"good\"/' ~/.whatever/config.xml",
+        ]
+        build.init = [
+            "bash generate_some_file.sh",
+            "sed -i -e 'g/what/ever/' /some/file",
+        ]
+        build.prebuild = ["npm something", "echo 'important setting' >> /a/file"]
+        build.build = [
+            "./gradlew someSpecialTask",
+            "sed -i 'd/that wrong config/' gradle.properties",
+            "./gradlew compile",
+        ]
         app['Builds'].append(build)
         fdroidserver.metadata.write_yaml(mf, app)
         mf.seek(0)
@@ -914,7 +942,7 @@ class MetadataTest(unittest.TestCase):
                     'Repo': 'https://git.host/repo.git',
                     'RepoType': 'git',
                     'Subdir': [''],
-                    'Prepare': '',
+                    'Prepare': [],
                 },
                 srclib,
             )
@@ -943,9 +971,11 @@ class MetadataTest(unittest.TestCase):
                     'Repo': 'https://github.com/cketti/ckChangeLog',
                     'RepoType': 'git',
                     'Subdir': ['library', 'ckChangeLog/src/main'],
-                    'Prepare': "[ -f project.properties ] || echo 'source.dir=java' > "
-                    "ant.properties && echo -e "
-                    "'android.library=true\\ntarget=android-19' > project.properties",
+                    'Prepare': [
+                        "[ -f project.properties ] || echo 'source.dir=java' > "
+                        "ant.properties && echo -e "
+                        "'android.library=true\\ntarget=android-19' > project.properties"
+                    ],
                 },
             )
 
@@ -993,8 +1023,10 @@ class MetadataTest(unittest.TestCase):
                         'You take the red pill—you stay in Wonderland',
                         'and I show you how deep the rabbit-hole goes.',
                     ],
-                    'Prepare': 'There is a difference between knowing the path '
-                    'and walking the path.',
+                    'Prepare': [
+                        'There is a difference between knowing the path '
+                        'and walking the path.'
+                    ],
                 }
             },
         )
@@ -1014,14 +1046,10 @@ class MetadataTest(unittest.TestCase):
 
                     Subdir:
                     Prepare:
-                     - The Matrix is a system, Neo.
-                     - That system is our enemy.
-                     - But when you're inside, you look around, what do you see?
-                     - Businessmen, teachers, lawyers, carpenters.
-                     - The very minds of the people we are trying to save.
-                     - But until we do, these people are still a part of that system and that makes them our enemy.
-                     - You have to understand, most of these people are not ready to be unplugged.
-                     - And many of them are so inert, so hopelessly dependent on the system that they will fight to protect it.
+                     - Many
+                     - invalid
+                     - commands
+                     - here.
                     '''
                     )
                 )
@@ -1034,14 +1062,12 @@ class MetadataTest(unittest.TestCase):
                     'RepoType': 'git',
                     'Repo': 'https://git.host/repo.git',
                     'Subdir': [''],
-                    'Prepare': 'The Matrix is a system, Neo. && '
-                    'That system is our enemy. && '
-                    'But when you\'re inside, you look around, what do you see? && '
-                    'Businessmen, teachers, lawyers, carpenters. && '
-                    'The very minds of the people we are trying to save. && '
-                    'But until we do, these people are still a part of that system and that makes them our enemy. && '
-                    'You have to understand, most of these people are not ready to be unplugged. && '
-                    'And many of them are so inert, so hopelessly dependent on the system that they will fight to protect it.',
+                    'Prepare': [
+                        'Many',
+                        'invalid',
+                        'commands',
+                        'here.',
+                    ],
                 }
             },
         )
@@ -1081,7 +1107,7 @@ class MetadataTest(unittest.TestCase):
                     'RepoType': 'git',
                     'Repo': 'https://git.host/repo.git',
                     'Subdir': [''],
-                    'Prepare': '',
+                    'Prepare': [],
                 },
                 'simple': {
                     'RepoType': 'git',

tests/metadata/dump/com.politedroid.yaml

@@ -144,7 +144,8 @@ Builds:
   srclibs: []
   subdir: null
   submodules: false
-  sudo: echo 'this is just a test'
+  sudo:
+  - echo 'this is just a test'
   target: null
   timeout: null
   versionCode: 6

tests/metadata/dump/org.adaway.yaml

@@ -281,7 +281,8 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
   rm: []
   scandelete: []
   scanignore: []
@@ -316,7 +317,8 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
   rm: []
   scandelete: []
   scanignore: []
@@ -351,7 +353,8 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
   rm: []
   scandelete: []
   scanignore: []
@@ -386,7 +389,8 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
   rm: []
   scandelete: []
   scanignore: []
@@ -421,7 +425,9 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/*
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
+  - rm -rf libs/armeabi/*
   rm: []
   scandelete: []
   scanignore: []
@@ -455,8 +461,10 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/*
-    && rm libs/android-support-v4.jar
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
+  - rm -rf libs/armeabi/*
+  - rm libs/android-support-v4.jar
   rm: []
   scandelete: []
   scanignore: []
@@ -490,7 +498,9 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/*
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
+  - rm -rf libs/armeabi/*
   rm: []
   scandelete: []
   scanignore: []
@@ -524,8 +534,10 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/*
-    && android update project -p ../org_donations
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
+  - rm -rf libs/armeabi/*
+  - android update project -p ../org_donations
   rm: []
   scandelete: []
   scanignore: []
@@ -559,8 +571,10 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/*
-    && android update project -p ../org_donations
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
+  - rm -rf libs/armeabi/*
+  - android update project -p ../org_donations
   rm: []
   scandelete: []
   scanignore: []
@@ -594,8 +608,10 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/*
-    && android update project -p ../org_donations
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
+  - rm -rf libs/armeabi/*
+  - android update project -p ../org_donations
   rm: []
   scandelete: []
   scanignore: []
@@ -629,8 +645,10 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: android update project -p ../com_actionbarsherlock && rm -rf libs/armeabi/*
-    && android update project -p ../org_donations
+  prebuild:
+  - android update project -p ../com_actionbarsherlock
+  - rm -rf libs/armeabi/*
+  - android update project -p ../org_donations
   rm: []
   scandelete: []
   scanignore: []
@@ -662,8 +680,9 @@ Builds:
   forceversion: false
   gradle: []
   gradleprops: []
-  init: rm android-libs/Donations/custom_rules.xml && git clone https://github.com/dschuermann/HtmlSpanner
-    android-libs/HtmlSpanner
+  init:
+  - rm android-libs/Donations/custom_rules.xml
+  - git clone https://github.com/dschuermann/HtmlSpanner android-libs/HtmlSpanner
   maven: false
   ndk: null
   novcheck: false
@@ -671,12 +690,13 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: rm -rf ../update_zip libs/root-commands-1.2.jar libs/htmlspanner-0.2-fork.jar
-    && cp -f libs/htmlcleaner-2.2.jar android-libs/HtmlSpanner/htmlspanner/libs/ &&
-    echo "android.library.reference.3=$$RootCommands$$" >> project.properties && echo
-    "android.library.reference.4=android-libs/HtmlSpanner/htmlspanner" >> project.properties
-    && find . -type f -print0 | xargs -0 sed -i 's/org.rootcommands/org.sufficientlysecure.rootcommands/g'
-    && cp android-libs/Donations/ant-templates/other/DonationsConfig.java android-libs/Donations/src/org/donations/
+  prebuild:
+  - rm -rf ../update_zip libs/root-commands-1.2.jar libs/htmlspanner-0.2-fork.jar
+  - cp -f libs/htmlcleaner-2.2.jar android-libs/HtmlSpanner/htmlspanner/libs/
+  - echo "android.library.reference.3=$$RootCommands$$" >> project.properties
+  - echo "android.library.reference.4=android-libs/HtmlSpanner/htmlspanner" >> project.properties
+  - find . -type f -print0 | xargs -0 sed -i 's/org.rootcommands/org.sufficientlysecure.rootcommands/g'
+  - cp android-libs/Donations/ant-templates/other/DonationsConfig.java android-libs/Donations/src/org/donations/
   rm: []
   scandelete: []
   scanignore: []
@@ -709,8 +729,9 @@ Builds:
   forceversion: false
   gradle: []
   gradleprops: []
-  init: rm android-libs/Donations/custom_rules.xml && git clone https://github.com/dschuermann/HtmlSpanner
-    android-libs/HtmlSpanner
+  init:
+  - rm android-libs/Donations/custom_rules.xml
+  - git clone https://github.com/dschuermann/HtmlSpanner android-libs/HtmlSpanner
   maven: false
   ndk: null
   novcheck: false
@@ -718,12 +739,13 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: rm -rf ../update_zip libs/root-commands-1.2.jar libs/htmlspanner-0.2-fork.jar
-    && cp -f libs/htmlcleaner-2.2.jar android-libs/HtmlSpanner/htmlspanner/libs/ &&
-    echo "android.library.reference.3=$$RootCommands$$" >> project.properties && echo
-    "android.library.reference.4=android-libs/HtmlSpanner/htmlspanner" >> project.properties
-    && find . -type f -print0 | xargs -0 sed -i 's/org.rootcommands/org.sufficientlysecure.rootcommands/g'
-    && cp android-libs/Donations/ant-templates/other/DonationsConfig.java android-libs/Donations/src/org/donations/
+  prebuild:
+  - rm -rf ../update_zip libs/root-commands-1.2.jar libs/htmlspanner-0.2-fork.jar
+  - cp -f libs/htmlcleaner-2.2.jar android-libs/HtmlSpanner/htmlspanner/libs/
+  - echo "android.library.reference.3=$$RootCommands$$" >> project.properties
+  - echo "android.library.reference.4=android-libs/HtmlSpanner/htmlspanner" >> project.properties
+  - find . -type f -print0 | xargs -0 sed -i 's/org.rootcommands/org.sufficientlysecure.rootcommands/g'
+  - cp android-libs/Donations/ant-templates/other/DonationsConfig.java android-libs/Donations/src/org/donations/
   rm: []
   scandelete: []
   scanignore: []

tests/metadata/dump/org.smssecure.smssecure.yaml

@@ -32,13 +32,27 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd
-    && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar &&
-    sed -i -e '20,22d' build.gradle && pushd $$PreferenceFragment$$ && gradle uploadArchives
-    && popd && sed -i -e '/5470f5872514a6226fa1fc6f4e000991f38805691c534cf0bd2778911fc773ad/d'
-    build.gradle && mkdir smil && pushd smil && wget -c http://www.w3.org/TR/smil-boston-dom/java-binding.zip
-    && unzip java-binding.zip && popd && cp -fR smil/java/org src/ && rm -fR smil
-    && sed -i -e '/org.w3c.smil/d' build.gradle && cp -fR $$AospMms$$/src/org src/
+  prebuild:
+  - touch signing.properties
+  - pushd $$GradleWitness$$
+  - gradle jar
+  - popd
+  - cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar
+  - sed -i -e '20,22d' build.gradle
+  - pushd $$PreferenceFragment$$
+  - gradle uploadArchives
+  - popd
+  - sed -i -e '/5470f5872514a6226fa1fc6f4e000991f38805691c534cf0bd2778911fc773ad/d'
+    build.gradle
+  - mkdir smil
+  - pushd smil
+  - wget -c http://www.w3.org/TR/smil-boston-dom/java-binding.zip
+  - unzip java-binding.zip
+  - popd
+  - cp -fR smil/java/org src/
+  - rm -fR smil
+  - sed -i -e '/org.w3c.smil/d' build.gradle
+  - cp -fR $$AospMms$$/src/org src/
   rm:
   - libs/*
   scandelete: []
@@ -78,8 +92,12 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: touch signing.properties && pushd $$GradleWitness$$ && gradle jar && popd
-    && cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar
+  prebuild:
+  - touch signing.properties
+  - pushd $$GradleWitness$$
+  - gradle jar
+  - popd
+  - cp $$GradleWitness$$/build/libs/GradleWitness.jar libs/gradle-witness.jar
   rm:
   - libs/*.jar
   scandelete: []
@@ -116,8 +134,11 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: touch signing.properties && ./build-witness.sh && rm -rf libs/gradle-witness/build
-    && echo "org.gradle.jvmargs=-Xms512m -Xmx512m -XX:MaxPermSize=512m" >> gradle.properties
+  prebuild:
+  - touch signing.properties
+  - ./build-witness.sh
+  - rm -rf libs/gradle-witness/build
+  - echo "org.gradle.jvmargs=-Xms512m -Xmx512m -XX:MaxPermSize=512m" >> gradle.properties
   rm:
   - libs/*.jar
   scandelete: []
@@ -153,8 +174,11 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: touch signing.properties && ./build-witness.sh && rm -rf libs/gradle-witness/build
-    && echo "org.gradle.jvmargs=-Xms512m -Xmx512m -XX:MaxPermSize=512m" >> gradle.properties
+  prebuild:
+  - touch signing.properties
+  - ./build-witness.sh
+  - rm -rf libs/gradle-witness/build
+  - echo "org.gradle.jvmargs=-Xms512m -Xmx512m -XX:MaxPermSize=512m" >> gradle.properties
   rm:
   - libs/*.jar
   scandelete: []
@@ -190,7 +214,10 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build
+  prebuild:
+  - touch signing.properties
+  - ./scripts/build-witness.sh
+  - rm -rf libs/gradle-witness/build
   rm:
   - libs/*.jar
   scandelete: []
@@ -226,7 +253,10 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build
+  prebuild:
+  - touch signing.properties
+  - ./scripts/build-witness.sh
+  - rm -rf libs/gradle-witness/build
   rm:
   - libs/*.jar
   scandelete: []
@@ -262,7 +292,10 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build
+  prebuild:
+  - touch signing.properties
+  - ./scripts/build-witness.sh
+  - rm -rf libs/gradle-witness/build
   rm:
   - libs/*.jar
   scandelete: []
@@ -298,7 +331,10 @@ Builds:
   output: null
   patch: []
   preassemble: []
-  prebuild: touch signing.properties && ./scripts/build-witness.sh && rm -rf libs/gradle-witness/build
+  prebuild:
+  - touch signing.properties
+  - ./scripts/build-witness.sh
+  - rm -rf libs/gradle-witness/build
   rm:
   - libs/*.jar
   scandelete: []