diff --git a/tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.cc b/tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.cc index d6fb3934b2f..354420d2488 100644 --- a/tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.cc +++ b/tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.cc @@ -71,21 +71,21 @@ class BenchmarkListenerAdapter : public tflite::benchmark::BenchmarkListener { void OnBenchmarkStart( const tflite::benchmark::BenchmarkParams& params) override { if (on_benchmark_start_fn_ != nullptr) { - on_benchmark_start_fn_(); + on_benchmark_start_fn_(user_data_); } } void OnSingleRunStart(tflite::benchmark::RunType runType) override { if (on_single_run_start_fn_ != nullptr) { - on_single_run_start_fn_(runType == tflite::benchmark::WARMUP - ? TfLiteBenchmarkWarmup - : TfLiteBenchmarkRegular); + on_single_run_start_fn_(user_data_, runType == tflite::benchmark::WARMUP + ? TfLiteBenchmarkWarmup + : TfLiteBenchmarkRegular); } } void OnSingleRunEnd() override { if (on_single_run_end_fn_ != nullptr) { - on_single_run_end_fn_(); + on_single_run_end_fn_(user_data_); } } @@ -93,17 +93,22 @@ class BenchmarkListenerAdapter : public tflite::benchmark::BenchmarkListener { const tflite::benchmark::BenchmarkResults& results) override { if (on_benchmark_end_fn_ != nullptr) { TfLiteBenchmarkResults* wrapper = new TfLiteBenchmarkResults{&results}; - on_benchmark_end_fn_(wrapper); + on_benchmark_end_fn_(user_data_, wrapper); delete wrapper; } } + // Keep the user_data pointer provided when setting the callbacks. + void* user_data_; + // Function pointers set by the TfLiteBenchmarkListenerSetCallbacks call. // Only non-null callbacks will be actually called. - void (*on_benchmark_start_fn_)(); - void (*on_single_run_start_fn_)(TfLiteBenchmarkRunType runType); - void (*on_single_run_end_fn_)(); - void (*on_benchmark_end_fn_)(TfLiteBenchmarkResults* results); + void (*on_benchmark_start_fn_)(void* user_data); + void (*on_single_run_start_fn_)(void* user_data, + TfLiteBenchmarkRunType runType); + void (*on_single_run_end_fn_)(void* user_data); + void (*on_benchmark_end_fn_)(void* user_data, + TfLiteBenchmarkResults* results); }; struct TfLiteBenchmarkListener { @@ -121,10 +126,14 @@ void TfLiteBenchmarkListenerDelete(TfLiteBenchmarkListener* listener) { } void TfLiteBenchmarkListenerSetCallbacks( - TfLiteBenchmarkListener* listener, void (*on_benchmark_start_fn)(), - void (*on_single_run_start_fn)(TfLiteBenchmarkRunType runType), - void (*on_single_run_end_fn)(), - void (*on_benchmark_end_fn)(TfLiteBenchmarkResults* results)) { + TfLiteBenchmarkListener* listener, void* user_data, + void (*on_benchmark_start_fn)(void* user_data), + void (*on_single_run_start_fn)(void* user_data, + TfLiteBenchmarkRunType runType), + void (*on_single_run_end_fn)(void* user_data), + void (*on_benchmark_end_fn)(void* user_data, + TfLiteBenchmarkResults* results)) { + listener->adapter->user_data_ = user_data; listener->adapter->on_benchmark_start_fn_ = on_benchmark_start_fn; listener->adapter->on_single_run_start_fn_ = on_single_run_start_fn; listener->adapter->on_single_run_end_fn_ = on_single_run_end_fn; diff --git a/tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.h b/tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.h index a642a4f2785..956996a729c 100644 --- a/tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.h +++ b/tensorflow/lite/tools/benchmark/experimental/c/benchmark_c_api.h @@ -82,7 +82,8 @@ extern TfLiteBenchmarkListener* TfLiteBenchmarkListenerCreate(); extern void TfLiteBenchmarkListenerDelete(TfLiteBenchmarkListener* listener); // Sets the listener callbacks. Only non-null callback functions will be called -// when the following events occur. +// when the following events occur. The user_data pointer provided by the caller +// will also be forwarded as a parameter of each callback function. // // - on_benchmark_start: Called before the (outer) inference loop begins. Note // that this is called *after* the interpreter has been initialized, but @@ -95,10 +96,13 @@ extern void TfLiteBenchmarkListenerDelete(TfLiteBenchmarkListener* listener); // only valid during the callback function execution, and will be destroyed // afterwards. extern void TfLiteBenchmarkListenerSetCallbacks( - TfLiteBenchmarkListener* listener, void (*on_benchmark_start_fn)(), - void (*on_single_run_start_fn)(TfLiteBenchmarkRunType runType), - void (*on_single_run_end_fn)(), - void (*on_benchmark_end_fn)(TfLiteBenchmarkResults* results)); + TfLiteBenchmarkListener* listener, void* user_data, + void (*on_benchmark_start_fn)(void* user_data), + void (*on_single_run_start_fn)(void* user_data, + TfLiteBenchmarkRunType runType), + void (*on_single_run_end_fn)(void* user_data), + void (*on_benchmark_end_fn)(void* user_data, + TfLiteBenchmarkResults* results)); // ----------------------------------------------------------------------------- // C APIs corresponding to tflite::benchmark::BenchmarkTfLiteModel type. diff --git a/tensorflow/lite/tools/benchmark/ios/README.md b/tensorflow/lite/tools/benchmark/ios/README.md index 5c772ac3fca..90fd28bbd6e 100644 --- a/tensorflow/lite/tools/benchmark/ios/README.md +++ b/tensorflow/lite/tools/benchmark/ios/README.md @@ -4,52 +4,43 @@ An iOS app to benchmark TFLite models. -The app reads benchmark parameters from a JSON file named `benchmark_params.json` -in its `benchmark_data` directory. Any downloaded models for benchmarking should -also be placed in `benchmark_data` directory. +The app reads benchmark parameters from a JSON file named +`benchmark_params.json` in its `benchmark_data` directory. Any downloaded models +for benchmarking should also be placed in `benchmark_data` directory. The JSON file specifies the name of the model file and other benchmarking parameters like inputs to the model, type of inputs, number of iterations, number of threads. The default values in the JSON file are for the -Mobilenet_1.0_224 model -([paper](https://arxiv.org/pdf/1704.04861.pdf), -[tflite&pb](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz)) +Mobilenet_1.0_224 model ([paper][mobilenet-paper], +[tflite&pb][mobilenet-model]). -## To build/install/run +## Building / running the app -- Follow instructions at - [iOS build for TFLite](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/g3doc/guide/build_ios.md) - to build TFLite. +* Follow the [iOS build instructions][build-ios] to configure the Bazel + workspace and `.bazelrc` file correctly. -Running +* Run `build_benchmark_framework.sh` script to build the benchmark framework. + This script will build the benchmark framework for iOS and put it under + `TFLiteBenchmark/TFLiteBenchmark/Frameworks` directory. -```bash -tensorflow/lite/tools/make/build_ios_universal_lib.sh -``` +* If you want more detailed profiling, run the build script with `-p` option: + `build_benchmark_framework.sh -p`. -will also build `tensorflow/lite/tools/make/gen/lib/benchmark-lib.a` . +* Modify `benchmark_params.json` change the `input_layer`, `input_layer_shape` + and other benchmark parameters. -- Now copy the downloaded model file to `benchmark_data` directory. +* Change `Build Phases -> Copy Bundle Resources` and add the model file to the + resources that need to be copied. -- Modify `benchmark_params.json` change the `input_layer`, `input_layer_shape` -and other benchmark parameters. +* Ensure that `Build Phases -> Link Binary With Library` contains the + `Accelerate framework` and `TensorFlowLiteBenchmarkC.framework`. -- Change `Build Phases -> Copy Bundle Resources` and add the model file to the -resources that need to be copied. +* Now try running the app. The app has a single button that runs the benchmark + on the model and displays results in a text view below. You can also see the + console output section in your Xcode to see more detailed benchmark + information. -- Ensure that `Build Phases -> Link Binary With Library` contains the -`Accelerate framework` and `tensorflow/lite/tools/make/gen/lib/benchmark-lib.a`. -- Now try running the app. The app has a single button that runs the benchmark - on the model and displays results in a text view below. - -## Profiling - -If you want detailed profiling, use the following command: - -```bash -tensorflow/lite/tools/make/build_ios_universal_lib.sh -p -``` - -Then following the same steps above and run the benchmark app. You will see the -detailed profiling results in the outputs. +[build-ios]: https://tensorflow.org/lite/guide/build_ios +[mobilenet-model]: https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_2018_02_22/mobilenet_v1_1.0_224.tgz +[mobilenet-paper]: https://arxiv.org/pdf/1704.04861.pdf diff --git a/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj index a5f5bfbbdaf..056ca6953ec 100644 --- a/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj +++ b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark.xcodeproj/project.pbxproj @@ -8,7 +8,6 @@ /* Begin PBXBuildFile section */ 6FE7579A20D59CE500F01636 /* benchmark_params.json in Resources */ = {isa = PBXBuildFile; fileRef = 6FE7579920D59CE500F01636 /* benchmark_params.json */; }; - 6FE7579D20D5A5E000F01636 /* benchmark-lib.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 6FE7579C20D5A5E000F01636 /* benchmark-lib.a */; }; 6FE7579F20D5A6A700F01636 /* Accelerate.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 6FE7579E20D5A6A700F01636 /* Accelerate.framework */; }; 6FE757A120D5AB8100F01636 /* mobilenet_v1_1.0_224.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 6FE757A020D5AB8000F01636 /* mobilenet_v1_1.0_224.tflite */; }; 6FE93FFD20D592D8008C9FE4 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 6FE93FFC20D592D8008C9FE4 /* AppDelegate.m */; }; @@ -16,8 +15,24 @@ 6FE9400320D592D8008C9FE4 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 6FE9400120D592D8008C9FE4 /* Main.storyboard */; }; 6FE9400520D592DA008C9FE4 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 6FE9400420D592DA008C9FE4 /* Assets.xcassets */; }; 6FE9400B20D592DA008C9FE4 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 6FE9400A20D592DA008C9FE4 /* main.m */; }; + DC4D465D2373ECF400397CBD /* TensorFlowLiteBenchmarkC.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = DC4D465C2373ECF300397CBD /* TensorFlowLiteBenchmarkC.framework */; }; + DC4D465E2373ECF400397CBD /* TensorFlowLiteBenchmarkC.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = DC4D465C2373ECF300397CBD /* TensorFlowLiteBenchmarkC.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; /* End PBXBuildFile section */ +/* Begin PBXCopyFilesBuildPhase section */ + DC4D465F2373ECF400397CBD /* Embed Frameworks */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 8; + dstPath = ""; + dstSubfolderSpec = 10; + files = ( + DC4D465E2373ECF400397CBD /* TensorFlowLiteBenchmarkC.framework in Embed Frameworks */, + ); + name = "Embed Frameworks"; + runOnlyForDeploymentPostprocessing = 1; + }; +/* End PBXCopyFilesBuildPhase section */ + /* Begin PBXFileReference section */ 6FE7579920D59CE500F01636 /* benchmark_params.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = benchmark_params.json; sourceTree = ""; }; 6FE7579C20D5A5E000F01636 /* benchmark-lib.a */ = {isa = PBXFileReference; lastKnownFileType = archive.ar; name = "benchmark-lib.a"; path = "$SRCROOT/../../../../../../tensorflow/lite/tools/make/gen/lib/benchmark-lib.a"; sourceTree = ""; }; @@ -32,6 +47,7 @@ 6FE9400420D592DA008C9FE4 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 6FE9400920D592DA008C9FE4 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 6FE9400A20D592DA008C9FE4 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; + DC4D465C2373ECF300397CBD /* TensorFlowLiteBenchmarkC.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = TensorFlowLiteBenchmarkC.framework; path = TFLiteBenchmark/Frameworks/TensorFlowLiteBenchmarkC.framework; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -40,7 +56,7 @@ buildActionMask = 2147483647; files = ( 6FE7579F20D5A6A700F01636 /* Accelerate.framework in Frameworks */, - 6FE7579D20D5A5E000F01636 /* benchmark-lib.a in Frameworks */, + DC4D465D2373ECF400397CBD /* TensorFlowLiteBenchmarkC.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -59,6 +75,7 @@ 6FE7579B20D5A5E000F01636 /* Frameworks */ = { isa = PBXGroup; children = ( + DC4D465C2373ECF300397CBD /* TensorFlowLiteBenchmarkC.framework */, 6FE7579E20D5A6A700F01636 /* Accelerate.framework */, 6FE7579C20D5A5E000F01636 /* benchmark-lib.a */, ); @@ -108,6 +125,7 @@ 6FE93FF420D592D8008C9FE4 /* Sources */, 6FE93FF520D592D8008C9FE4 /* Frameworks */, 6FE93FF620D592D8008C9FE4 /* Resources */, + DC4D465F2373ECF400397CBD /* Embed Frameworks */, ); buildRules = ( ); @@ -308,6 +326,10 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; + FRAMEWORK_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)/TFLiteBenchmark/Frameworks", + ); "HEADER_SEARCH_PATHS[arch=*]" = ( $SRCROOT/../../../../../../, $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/eigen, @@ -334,6 +356,10 @@ buildSettings = { ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; + FRAMEWORK_SEARCH_PATHS = ( + "$(inherited)", + "$(PROJECT_DIR)/TFLiteBenchmark/Frameworks", + ); "HEADER_SEARCH_PATHS[arch=*]" = ( $SRCROOT/../../../../../../, $SRCROOT/../../../../../../tensorflow/lite/tools/make/downloads/eigen, diff --git a/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/BenchmarkViewController.mm b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/BenchmarkViewController.mm index 590c215f515..bb24040014b 100644 --- a/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/BenchmarkViewController.mm +++ b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/BenchmarkViewController.mm @@ -18,8 +18,8 @@ #import #import #import -#import "tensorflow/lite/tools/benchmark/benchmark_tflite_model.h" -#import "tensorflow/lite/tools/benchmark/logging.h" + +#import namespace { NSString* FilePathForResourceName(NSString* filename) { @@ -64,42 +64,57 @@ std::vector StringVecToCharPtrVec(const std::vector& str_vec return charptr_vec; } -class ResultsListener : public tflite::benchmark::BenchmarkListener { +class ResultsListener { public: - void OnBenchmarkEnd(const tflite::benchmark::BenchmarkResults& results) override; + void OnBenchmarkEnd(TfLiteBenchmarkResults* results); std::string Results() { return results_; } private: std::string results_; }; -void OutputMicrosecondsStatToStream(const tensorflow::Stat& time_us, +void OutputMicrosecondsStatToStream(const TfLiteBenchmarkInt64Stat& time_us, const std::string& prefix, std::ostringstream* stream) { - *stream << prefix << "Num runs: " << time_us.count() << "\n"; + *stream << prefix << "Num runs: " << time_us.count << "\n"; - *stream << prefix << "Average: " << time_us.avg() / 1e3 << " ms\n"; - *stream << prefix << "Min: " << time_us.min() / 1e3 << " ms \n"; - *stream << prefix << "Max: " << time_us.max() / 1e3 << " ms \n"; - *stream << prefix << "Std deviation: " << time_us.std_deviation() / 1e3 << " ms\n"; + *stream << prefix << "Average: " << time_us.avg / 1e3 << " ms\n"; + *stream << prefix << "Min: " << time_us.min / 1e3 << " ms \n"; + *stream << prefix << "Max: " << time_us.max / 1e3 << " ms \n"; + *stream << prefix << "Std deviation: " << time_us.std_deviation / 1e3 << " ms\n"; } -void ResultsListener::OnBenchmarkEnd(const tflite::benchmark::BenchmarkResults& results) { +void ResultsListener::OnBenchmarkEnd(TfLiteBenchmarkResults* results) { std::ostringstream stream; const std::string prefix = " - "; + + TfLiteBenchmarkInt64Stat inference = TfLiteBenchmarkResultsGetInferenceTimeMicroseconds(results); + TfLiteBenchmarkInt64Stat warmup = TfLiteBenchmarkResultsGetWarmupTimeMicroseconds(results); + stream << "Startup latency: "; - stream << results.startup_latency_us() / 1e3 << " ms\n"; + stream << TfLiteBenchmarkResultsGetStartupLatencyMicroseconds(results) / 1e3 << " ms\n"; stream << "\nInference:\n"; - OutputMicrosecondsStatToStream(results.inference_time_us(), prefix, &stream); + OutputMicrosecondsStatToStream(inference, prefix, &stream); stream << "\nWarmup:\n"; - OutputMicrosecondsStatToStream(results.warmup_time_us(), prefix, &stream); + OutputMicrosecondsStatToStream(warmup, prefix, &stream); results_ = stream.str(); } +void OnBenchmarkEnd(void* user_data, TfLiteBenchmarkResults* results) { + if (user_data != nullptr) { + reinterpret_cast(user_data)->OnBenchmarkEnd(results); + } +} + std::string RunBenchmark() { - ResultsListener listener; - tflite::benchmark::BenchmarkTfLiteModel benchmark; - benchmark.AddListener(&listener); + ResultsListener results_listener; + TfLiteBenchmarkTfLiteModel* benchmark = TfLiteBenchmarkTfLiteModelCreate(); + + TfLiteBenchmarkListener* listener = TfLiteBenchmarkListenerCreate(); + TfLiteBenchmarkListenerSetCallbacks(listener, &results_listener, nullptr, nullptr, nullptr, + OnBenchmarkEnd); + + TfLiteBenchmarkTfLiteModelAddListener(benchmark, listener); // TODO(shashishekhar): Passing arguments like this is brittle, refactor the BenchmarkParams // so that it contains arguments for BenchmarkTfLiteModel and set parameters using BenchmarkParams std::vector command_line_params; @@ -109,8 +124,15 @@ std::string RunBenchmark() { ReadCommandLineParameters(&command_line_params); std::vector argv = StringVecToCharPtrVec(command_line_params); int argc = static_cast(argv.size()); - benchmark.Run(argc, argv.data()); - return listener.Results(); + + TfLiteBenchmarkTfLiteModelRunWithArgs(benchmark, argc, argv.data()); + + std::string results = results_listener.Results(); + + TfLiteBenchmarkListenerDelete(listener); + TfLiteBenchmarkTfLiteModelDelete(benchmark); + + return results; } } // namespace diff --git a/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/Frameworks/.gitignore b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/Frameworks/.gitignore new file mode 100644 index 00000000000..63d8bd7965e --- /dev/null +++ b/tensorflow/lite/tools/benchmark/ios/TFLiteBenchmark/TFLiteBenchmark/Frameworks/.gitignore @@ -0,0 +1 @@ +*.framework/ diff --git a/tensorflow/lite/tools/benchmark/ios/build_benchmark_framework.sh b/tensorflow/lite/tools/benchmark/ios/build_benchmark_framework.sh new file mode 100755 index 00000000000..5c74158723d --- /dev/null +++ b/tensorflow/lite/tools/benchmark/ios/build_benchmark_framework.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e +set -x + +WORKSPACE_ROOT=$(bazel info workspace) +BENCHMARK_DIR=tensorflow/lite/tools/benchmark +DEST_DIR="${BENCHMARK_DIR}/ios/TFLiteBenchmark/TFLiteBenchmark/Frameworks" +FRAMEWORK_TARGET=TensorFlowLiteBenchmarkC_framework + +usage() { + echo "Usage: $(basename "$0") [-p]" + echo "-p enable profiling" + exit 1 +} + +PROFILING_ARGS="" +while getopts "p" opt_name; do + case "$opt_name" in + p) PROFILING_ARGS='--copt=-DGEMMLOWP_PROFILING';; + *) usage;; + esac +done +shift $(($OPTIND - 1)) + +pushd "${WORKSPACE_ROOT}" + +# Build the framework. +bazel build --config=ios_fat -c opt ${PROFILING_ARGS} \ + "//${BENCHMARK_DIR}/experimental/ios:${FRAMEWORK_TARGET}" + +# Copy the framework into the destination and unzip. +mkdir -p "${DEST_DIR}" +cp -f "bazel-bin/${BENCHMARK_DIR}/experimental/ios/${FRAMEWORK_TARGET}.zip" \ + "${DEST_DIR}" +pushd "${DEST_DIR}" +unzip -o "${FRAMEWORK_TARGET}.zip" +rm -f "${FRAMEWORK_TARGET}.zip" + +popd +popd