From 2f9335a19722165cb3c3b26319e34ddecd9305e0 Mon Sep 17 00:00:00 2001 From: bbbboom Date: Sat, 7 Mar 2020 16:38:08 +0800 Subject: [PATCH 0001/1017] Fix a go binding install error on windows platform If running *go generate github.com/tensorflow/tensorflow/tensorflow/go/op* on windows platform, it well show > \go_path/src/github.com/tensorflow/tensorflow: warning: directory does not exist. Cannot convert path "\go_path/src/github.com/tensorflow/tensorflow/tensorflow/core/framework/*.proto" to or from Windows style ..\genop\main.go:17: running "bash": exit status 1 ..\github.com\tensorflow\tensorflow\tensorflow\go\op\generate.go:17: running "go": exit status 1 So this commit will automatically convert windows style slashes. --- tensorflow/go/genop/generate.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tensorflow/go/genop/generate.sh b/tensorflow/go/genop/generate.sh index 18bee11da5a..9df20c47a5e 100644 --- a/tensorflow/go/genop/generate.sh +++ b/tensorflow/go/genop/generate.sh @@ -24,6 +24,15 @@ then GOPATH=$(go env GOPATH) fi +# Check if it is running in git-bash +ls -la / | grep git-bash.exe +if [ $? -e "0" ] +then + GOPATH=${GOPATH//\\/\/} + GOPATH=/${GOPATH//:/} +fi + + cd $(dirname $0) for g in $(echo "${GOPATH//:/ }"); do TF_DIR="${g}/src/github.com/tensorflow/tensorflow" From d61528e84f5627cc8ecea151ddfe55321f3e303c Mon Sep 17 00:00:00 2001 From: bbbboom Date: Thu, 12 Mar 2020 20:39:46 +0800 Subject: [PATCH 0002/1017] C needs to distinguish between platforms Command needs to distinguish between platforms. Maybe it would be better. --- tensorflow/go/genop/generate.go | 5 + tensorflow/go/genop/generate.sh | 139 +++++++++++++-------------- tensorflow/go/genop/generate.win.go | 5 + tensorflow/go/genop/main.go | 142 ++++++++++++++-------------- 4 files changed, 148 insertions(+), 143 deletions(-) create mode 100644 tensorflow/go/genop/generate.go create mode 100644 tensorflow/go/genop/generate.win.go diff --git a/tensorflow/go/genop/generate.go b/tensorflow/go/genop/generate.go new file mode 100644 index 00000000000..72ec6f552c2 --- /dev/null +++ b/tensorflow/go/genop/generate.go @@ -0,0 +1,5 @@ +// +build !windows + +//go:generate bash generate.sh + +package main diff --git a/tensorflow/go/genop/generate.sh b/tensorflow/go/genop/generate.sh index 9df20c47a5e..6346f12687d 100644 --- a/tensorflow/go/genop/generate.sh +++ b/tensorflow/go/genop/generate.sh @@ -1,71 +1,68 @@ -#!/usr/bin/env bash -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -set -e - -go get github.com/golang/protobuf/proto -go get github.com/golang/protobuf/protoc-gen-go - -if [ -z "${GOPATH}" ] -then - GOPATH=$(go env GOPATH) -fi - -# Check if it is running in git-bash -ls -la / | grep git-bash.exe -if [ $? -e "0" ] -then - GOPATH=${GOPATH//\\/\/} - GOPATH=/${GOPATH//:/} -fi - - -cd $(dirname $0) -for g in $(echo "${GOPATH//:/ }"); do - TF_DIR="${g}/src/github.com/tensorflow/tensorflow" - PROTOC="${TF_DIR}/bazel-out/host/bin/external/protobuf/protoc" - if [ -x "${PROTOC}" ]; then - break - fi -done - -if [ ! -x "${PROTOC}" ] -then - set +e - PATH_PROTOC=$(which protoc) - if [ ! -x "${PATH_PROTOC}" ] - then - echo "Protocol buffer compiler protoc not found in PATH or in ${PROTOC}" - echo "Perhaps build it using:" - echo "bazel build --config opt @com_google_protobuf//:protoc" - exit 1 - fi - PROTOC=$PATH_PROTOC - set -e -fi - -# Ensure that protoc-gen-go is available in $PATH -# Since ${PROTOC} will require it. -export PATH=$PATH:${GOPATH}/bin -mkdir -p ../vendor -for FILE in ${TF_DIR}/tensorflow/core/framework/*.proto \ - ${TF_DIR}/tensorflow/core/protobuf/*.proto \ - ${TF_DIR}/tensorflow/stream_executor/*.proto; do - ${PROTOC} \ - -I ${TF_DIR} \ - --go_out=../vendor \ - $FILE -done +#!/usr/bin/env bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +go get github.com/golang/protobuf/proto +go get github.com/golang/protobuf/protoc-gen-go + +if [ -z "${GOPATH}" ] +then + GOPATH=$(go env GOPATH) +fi + +# change GOPATH style +if [ $1 == "win" ]; then + GOPATH=${GOPATH//\\/\/} + GOPATH=/${GOPATH//:/} +fi + +cd $(dirname $0) +for g in $(echo "${GOPATH//:/ }"); do + TF_DIR="${g}/src/github.com/tensorflow/tensorflow" + PROTOC="${TF_DIR}/bazel-out/host/bin/external/protobuf/protoc" + if [ -x "${PROTOC}" ]; then + break + fi +done + +if [ ! -x "${PROTOC}" ] +then + set +e + PATH_PROTOC=$(which protoc) + if [ ! -x "${PATH_PROTOC}" ] + then + echo "Protocol buffer compiler protoc not found in PATH or in ${PROTOC}" + echo "Perhaps build it using:" + echo "bazel build --config opt @com_google_protobuf//:protoc" + exit 1 + fi + PROTOC=$PATH_PROTOC + set -e +fi + +# Ensure that protoc-gen-go is available in $PATH +# Since ${PROTOC} will require it. +export PATH=$PATH:${GOPATH}/bin +mkdir -p ../vendor +for FILE in ${TF_DIR}/tensorflow/core/framework/*.proto \ + ${TF_DIR}/tensorflow/core/protobuf/*.proto \ + ${TF_DIR}/tensorflow/stream_executor/*.proto; do + ${PROTOC} \ + -I ${TF_DIR} \ + --go_out=../vendor \ + $FILE +done diff --git a/tensorflow/go/genop/generate.win.go b/tensorflow/go/genop/generate.win.go new file mode 100644 index 00000000000..23b27fffc2a --- /dev/null +++ b/tensorflow/go/genop/generate.win.go @@ -0,0 +1,5 @@ +// +build windows + +//go:generate bash generate.sh win + +package main diff --git a/tensorflow/go/genop/main.go b/tensorflow/go/genop/main.go index 4a53084ed13..83a361176f0 100644 --- a/tensorflow/go/genop/main.go +++ b/tensorflow/go/genop/main.go @@ -1,72 +1,70 @@ -/* -Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -//go:generate bash generate.sh - -// Command genop generates a Go source file with functions for TensorFlow ops. -package main - -import ( - "bytes" - "flag" - "go/format" - "io/ioutil" - "log" - "os" - "path/filepath" - "strings" - - "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal" -) - -func main() { - var ( - filename = flag.String("outfile", "", "File to write generated source code to.") - header = flag.String("header", "", "Path to a file whose contents will be copied into the generated file. Can be empty") - apiDefDirs = flag.String("api_def_dirs", "", "Comma-separated directories containing api_def_*.pbtxt files.") - buf bytes.Buffer - ) - flag.Parse() - if *filename == "" { - log.Fatal("-outfile must be set") - } - if *header != "" { - hdr, err := ioutil.ReadFile(*header) - if err != nil { - log.Fatalf("Unable to read %s: %v", *header, err) - } - buf.Write(hdr) - buf.WriteString("\n\n") - } - os.MkdirAll(filepath.Dir(*filename), 0755) - - apiDefDirsList := []string{} - if len(*apiDefDirs) > 0 { - apiDefDirsList = strings.Split(*apiDefDirs, ",") - } - - if err := internal.GenerateFunctionsForRegisteredOps( - &buf, apiDefDirsList); err != nil { - log.Fatal(err) - } - formatted, err := format.Source(buf.Bytes()) - if err != nil { - log.Fatalf("Failed to generate valid source? 'go fmt' failed: %v", err) - } - if err := ioutil.WriteFile(*filename, formatted, 0644); err != nil { - log.Fatalf("Failed to write to %q: %v", *filename, err) - } -} +/* +Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command genop generates a Go source file with functions for TensorFlow ops. +package main + +import ( + "bytes" + "flag" + "go/format" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + + "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal" +) + +func main() { + var ( + filename = flag.String("outfile", "", "File to write generated source code to.") + header = flag.String("header", "", "Path to a file whose contents will be copied into the generated file. Can be empty") + apiDefDirs = flag.String("api_def_dirs", "", "Comma-separated directories containing api_def_*.pbtxt files.") + buf bytes.Buffer + ) + flag.Parse() + if *filename == "" { + log.Fatal("-outfile must be set") + } + if *header != "" { + hdr, err := ioutil.ReadFile(*header) + if err != nil { + log.Fatalf("Unable to read %s: %v", *header, err) + } + buf.Write(hdr) + buf.WriteString("\n\n") + } + os.MkdirAll(filepath.Dir(*filename), 0755) + + apiDefDirsList := []string{} + if len(*apiDefDirs) > 0 { + apiDefDirsList = strings.Split(*apiDefDirs, ",") + } + + if err := internal.GenerateFunctionsForRegisteredOps( + &buf, apiDefDirsList); err != nil { + log.Fatal(err) + } + formatted, err := format.Source(buf.Bytes()) + if err != nil { + log.Fatalf("Failed to generate valid source? 'go fmt' failed: %v", err) + } + if err := ioutil.WriteFile(*filename, formatted, 0644); err != nil { + log.Fatalf("Failed to write to %q: %v", *filename, err) + } +} From e75080ce47a30f0c331446f674551e098fa09f58 Mon Sep 17 00:00:00 2001 From: bbbboom Date: Thu, 12 Mar 2020 20:45:26 +0800 Subject: [PATCH 0003/1017] change CR-LF to LF --- tensorflow/go/genop/generate.go | 10 +- tensorflow/go/genop/generate.sh | 136 +++++++++++++-------------- tensorflow/go/genop/generate.win.go | 10 +- tensorflow/go/genop/main.go | 140 ++++++++++++++-------------- 4 files changed, 148 insertions(+), 148 deletions(-) diff --git a/tensorflow/go/genop/generate.go b/tensorflow/go/genop/generate.go index 72ec6f552c2..fcf1d65d594 100644 --- a/tensorflow/go/genop/generate.go +++ b/tensorflow/go/genop/generate.go @@ -1,5 +1,5 @@ -// +build !windows - -//go:generate bash generate.sh - -package main +// +build !windows + +//go:generate bash generate.sh + +package main diff --git a/tensorflow/go/genop/generate.sh b/tensorflow/go/genop/generate.sh index 6346f12687d..54541106f13 100644 --- a/tensorflow/go/genop/generate.sh +++ b/tensorflow/go/genop/generate.sh @@ -1,68 +1,68 @@ -#!/usr/bin/env bash -# Copyright 2016 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -set -e - -go get github.com/golang/protobuf/proto -go get github.com/golang/protobuf/protoc-gen-go - -if [ -z "${GOPATH}" ] -then - GOPATH=$(go env GOPATH) -fi - -# change GOPATH style -if [ $1 == "win" ]; then - GOPATH=${GOPATH//\\/\/} - GOPATH=/${GOPATH//:/} -fi - -cd $(dirname $0) -for g in $(echo "${GOPATH//:/ }"); do - TF_DIR="${g}/src/github.com/tensorflow/tensorflow" - PROTOC="${TF_DIR}/bazel-out/host/bin/external/protobuf/protoc" - if [ -x "${PROTOC}" ]; then - break - fi -done - -if [ ! -x "${PROTOC}" ] -then - set +e - PATH_PROTOC=$(which protoc) - if [ ! -x "${PATH_PROTOC}" ] - then - echo "Protocol buffer compiler protoc not found in PATH or in ${PROTOC}" - echo "Perhaps build it using:" - echo "bazel build --config opt @com_google_protobuf//:protoc" - exit 1 - fi - PROTOC=$PATH_PROTOC - set -e -fi - -# Ensure that protoc-gen-go is available in $PATH -# Since ${PROTOC} will require it. -export PATH=$PATH:${GOPATH}/bin -mkdir -p ../vendor -for FILE in ${TF_DIR}/tensorflow/core/framework/*.proto \ - ${TF_DIR}/tensorflow/core/protobuf/*.proto \ - ${TF_DIR}/tensorflow/stream_executor/*.proto; do - ${PROTOC} \ - -I ${TF_DIR} \ - --go_out=../vendor \ - $FILE -done +#!/usr/bin/env bash +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +go get github.com/golang/protobuf/proto +go get github.com/golang/protobuf/protoc-gen-go + +if [ -z "${GOPATH}" ] +then + GOPATH=$(go env GOPATH) +fi + +# change GOPATH style +if [ $1 == "win" ]; then + GOPATH=${GOPATH//\\/\/} + GOPATH=/${GOPATH//:/} +fi + +cd $(dirname $0) +for g in $(echo "${GOPATH//:/ }"); do + TF_DIR="${g}/src/github.com/tensorflow/tensorflow" + PROTOC="${TF_DIR}/bazel-out/host/bin/external/protobuf/protoc" + if [ -x "${PROTOC}" ]; then + break + fi +done + +if [ ! -x "${PROTOC}" ] +then + set +e + PATH_PROTOC=$(which protoc) + if [ ! -x "${PATH_PROTOC}" ] + then + echo "Protocol buffer compiler protoc not found in PATH or in ${PROTOC}" + echo "Perhaps build it using:" + echo "bazel build --config opt @com_google_protobuf//:protoc" + exit 1 + fi + PROTOC=$PATH_PROTOC + set -e +fi + +# Ensure that protoc-gen-go is available in $PATH +# Since ${PROTOC} will require it. +export PATH=$PATH:${GOPATH}/bin +mkdir -p ../vendor +for FILE in ${TF_DIR}/tensorflow/core/framework/*.proto \ + ${TF_DIR}/tensorflow/core/protobuf/*.proto \ + ${TF_DIR}/tensorflow/stream_executor/*.proto; do + ${PROTOC} \ + -I ${TF_DIR} \ + --go_out=../vendor \ + $FILE +done diff --git a/tensorflow/go/genop/generate.win.go b/tensorflow/go/genop/generate.win.go index 23b27fffc2a..3eff6ab5e7e 100644 --- a/tensorflow/go/genop/generate.win.go +++ b/tensorflow/go/genop/generate.win.go @@ -1,5 +1,5 @@ -// +build windows - -//go:generate bash generate.sh win - -package main +// +build windows + +//go:generate bash generate.sh win + +package main diff --git a/tensorflow/go/genop/main.go b/tensorflow/go/genop/main.go index 83a361176f0..87c1d27c3b5 100644 --- a/tensorflow/go/genop/main.go +++ b/tensorflow/go/genop/main.go @@ -1,70 +1,70 @@ -/* -Copyright 2016 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Command genop generates a Go source file with functions for TensorFlow ops. -package main - -import ( - "bytes" - "flag" - "go/format" - "io/ioutil" - "log" - "os" - "path/filepath" - "strings" - - "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal" -) - -func main() { - var ( - filename = flag.String("outfile", "", "File to write generated source code to.") - header = flag.String("header", "", "Path to a file whose contents will be copied into the generated file. Can be empty") - apiDefDirs = flag.String("api_def_dirs", "", "Comma-separated directories containing api_def_*.pbtxt files.") - buf bytes.Buffer - ) - flag.Parse() - if *filename == "" { - log.Fatal("-outfile must be set") - } - if *header != "" { - hdr, err := ioutil.ReadFile(*header) - if err != nil { - log.Fatalf("Unable to read %s: %v", *header, err) - } - buf.Write(hdr) - buf.WriteString("\n\n") - } - os.MkdirAll(filepath.Dir(*filename), 0755) - - apiDefDirsList := []string{} - if len(*apiDefDirs) > 0 { - apiDefDirsList = strings.Split(*apiDefDirs, ",") - } - - if err := internal.GenerateFunctionsForRegisteredOps( - &buf, apiDefDirsList); err != nil { - log.Fatal(err) - } - formatted, err := format.Source(buf.Bytes()) - if err != nil { - log.Fatalf("Failed to generate valid source? 'go fmt' failed: %v", err) - } - if err := ioutil.WriteFile(*filename, formatted, 0644); err != nil { - log.Fatalf("Failed to write to %q: %v", *filename, err) - } -} +/* +Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Command genop generates a Go source file with functions for TensorFlow ops. +package main + +import ( + "bytes" + "flag" + "go/format" + "io/ioutil" + "log" + "os" + "path/filepath" + "strings" + + "github.com/tensorflow/tensorflow/tensorflow/go/genop/internal" +) + +func main() { + var ( + filename = flag.String("outfile", "", "File to write generated source code to.") + header = flag.String("header", "", "Path to a file whose contents will be copied into the generated file. Can be empty") + apiDefDirs = flag.String("api_def_dirs", "", "Comma-separated directories containing api_def_*.pbtxt files.") + buf bytes.Buffer + ) + flag.Parse() + if *filename == "" { + log.Fatal("-outfile must be set") + } + if *header != "" { + hdr, err := ioutil.ReadFile(*header) + if err != nil { + log.Fatalf("Unable to read %s: %v", *header, err) + } + buf.Write(hdr) + buf.WriteString("\n\n") + } + os.MkdirAll(filepath.Dir(*filename), 0755) + + apiDefDirsList := []string{} + if len(*apiDefDirs) > 0 { + apiDefDirsList = strings.Split(*apiDefDirs, ",") + } + + if err := internal.GenerateFunctionsForRegisteredOps( + &buf, apiDefDirsList); err != nil { + log.Fatal(err) + } + formatted, err := format.Source(buf.Bytes()) + if err != nil { + log.Fatalf("Failed to generate valid source? 'go fmt' failed: %v", err) + } + if err := ioutil.WriteFile(*filename, formatted, 0644); err != nil { + log.Fatalf("Failed to write to %q: %v", *filename, err) + } +} From efe0812a463aa3dcbac53d2d52c623d60d607634 Mon Sep 17 00:00:00 2001 From: Shawn Presser Date: Tue, 9 Jun 2020 05:18:52 -0700 Subject: [PATCH 0004/1017] Make http://metadata.google.internal configurable The TPU client library has a hardcoded dependency on `http://metadata.google.internal`. We happen to need to redirect this URL to a different VM. Since the URL is hardcoded, we're forced to use a fragile code patch against our version of Tensorflow, which isn't ideal, or rely on `/etc/hosts` to forward `metadata.google.internal`, which causes unexpected global side effects to the user's VM. (For example, GCE uses `metadata.google.internal` to distribute SSH keys to GCE VMs, which breaks when we reroute `metadata.google.internal` using `/etc/hosts`.) oauth2client solves this by making `http://metadata.google.internal` configurable via the `GCE_METADATA_IP` environment variable. The final url becomes `'http://' + os.getenv('GCE_METADATA_IP', '169.254.169.254')`: https://github.com/googleapis/oauth2client/blob/50d20532a748f18e53f7d24ccbe6647132c979a9/oauth2client/client.py#L111 Following oauth2client's lead, this PR makes `http://metadata.google.internal` configurable for Tensorflow users via `GCE_METADATA_IP`: ```py _GCE_METADATA_URL_ENV_VARIABLE = 'GCE_METADATA_IP' # ... def _gce_metadata_endpoint(): return 'http://' + os.environ.get( _GCE_METADATA_URL_ENV_VARIABLE, 'metadata.google.internal') ``` `GCE_METADATA_IP` might seem like an awkward name. After all, `metadata.google.internal` is a URL, not an IP address. But it's probably best to match oauth2client's naming convention. That way users won't need to worry about setting two slightly-different variable names to configure both oauth2client and Tensorflow. --- tensorflow/python/tpu/client/client.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/tpu/client/client.py b/tensorflow/python/tpu/client/client.py index bc693cbef68..8f48298345e 100644 --- a/tensorflow/python/tpu/client/client.py +++ b/tensorflow/python/tpu/client/client.py @@ -38,7 +38,7 @@ _GKE_ENV_VARIABLE = 'KUBE_GOOGLE_CLOUD_TPU_ENDPOINTS' _ENDPOINTS_SEPARATOR = ',' _DEFAULT_ENV_VARIABLE = 'TPU_NAME' _DISCOVERY_SERVICE_URL_ENV_VARIABLE = 'TPU_API_DISCOVERY_URL' -_GCE_METADATA_ENDPOINT = 'http://metadata.google.internal' +_GCE_METADATA_URL_ENV_VARIABLE = 'GCE_METADATA_IP' _DEFAULT_ENDPOINT_PORT = '8470' @@ -46,9 +46,15 @@ def _environment_discovery_url(): return os.environ.get(_DISCOVERY_SERVICE_URL_ENV_VARIABLE) +def _gce_metadata_endpoint(): + return 'http://' + os.environ.get( + _GCE_METADATA_URL_ENV_VARIABLE, + 'metadata.google.internal') + + def _request_compute_metadata(path): req = request.Request( - '%s/computeMetadata/v1/%s' % (_GCE_METADATA_ENDPOINT, path), + '%s/computeMetadata/v1/%s' % (_gce_metadata_endpoint(), path), headers={'Metadata-Flavor': 'Google'}) resp = request.urlopen(req) return _as_text(resp.read()) From 9134fbb13794865a45288d2e722ad47c362e0ae4 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 18 Jun 2020 23:13:11 +0000 Subject: [PATCH 0005/1017] summary_op needs tstring C API sync --- tensorflow/c/kernels/diff.patch | 0 tensorflow/c/kernels/ops/summary.cc | 70 ++++++++++ tensorflow/c/kernels/summary_op.cc | 171 ++++++++++++++++++++++++ tensorflow/c/kernels/summary_op_test.cc | 96 +++++++++++++ 4 files changed, 337 insertions(+) create mode 100644 tensorflow/c/kernels/diff.patch create mode 100644 tensorflow/c/kernels/ops/summary.cc create mode 100644 tensorflow/c/kernels/summary_op.cc create mode 100644 tensorflow/c/kernels/summary_op_test.cc diff --git a/tensorflow/c/kernels/diff.patch b/tensorflow/c/kernels/diff.patch new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc new file mode 100644 index 00000000000..550a663d006 --- /dev/null +++ b/tensorflow/c/kernels/ops/summary.cc @@ -0,0 +1,70 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "tensorflow/c/ops.h" +#include "tensorflow/core/framework/selective_registration.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/macros.h" + + +static void TF_ScalarSummary_shape_inference_fn(TF_ShapeInferenceContext* ctx, + TF_Status* status) { + TF_ShapeHandle* result = TF_NewShapeHandle(); + // TODO: what to do in the case of unknown input shape? + if (TF_GetCode(status) == TF_OK && + !TF_ShapeInferenceContextRankKnown(ctx, result)) { + TF_ShapeInferenceContextSetUnknownShape(ctx, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while setting unknown shape function"; + TF_DeleteShapeHandle(result); + return; + } + // make shape handle a scalar value (empty shape) + if (TF_GetCode(status) == TF_OK) { + TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while setting shape function"; + } + TF_DeleteShapeHandle(result); +} + +void Register_ScalarSummaryOp() { + TF_Status* status = TF_NewStatus(); + + TF_OpDefinitionBuilder* op_builder = TF_NewOpDefinitionBuilder("SummaryScalar"); + TF_OpDefinitionBuilderAddInput(op_builder, "tags: string"); + TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); + TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); + TF_OpDefinitionBuilderAddAttr( + op_builder, + "T: realnumbertype"); + TF_OpDefinitionBuilderSetShapeInferenceFunction(op_builder, + &TF_ScalarSummary_shape_inference_fn); + + TF_RegisterOpDefinition(op_builder, status); + CHECK_EQ(TF_GetCode(status), TF_OK) + << "TF_ScalarSummary op registration failed: " << TF_Message(status); + TF_DeleteStatus(status); +} + +TF_ATTRIBUTE_UNUSED static bool SummaryScalarOpRegistered = []() { + if (SHOULD_REGISTER_OP("SummaryScalar")) { + Register_ScalarSummaryOp(); + } + return true; +}(); diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc new file mode 100644 index 00000000000..3a78d321d75 --- /dev/null +++ b/tensorflow/c/kernels/summary_op.cc @@ -0,0 +1,171 @@ + +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/c/kernels.h" +#include "tensorflow/c/ops.h" +#include "tensorflow/c/tf_tensor.h" +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/selective_registration.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/framework/summary.pb.h" +#include "tensorflow/core/platform/protobuf.h" +#include "tensorflow/core/framework/register_types.h" + +#include "tensorflow/core/framework/types.h" + +// BitcastOp implements a bitcast kernel, creating an output tensor that shares +// the same data buffer as the input but with a different shape and/or data +// type. Its inputs are: +// +// * the input tensor +// * an attribute named "T" containing the TF_DataType of the input tensor +// * an attribute named "type" containing the TF_DataType of the output tensor +// +// Given an input tensor of shape [...], if the input DataType "T" is larger +// than the output DataType "type", then the shape changes from [...] +// to [..., sizeof(T)/sizeof(type)]. +// +// If "T" is smaller than "type", the operator requires that the rightmost +// dimension be equal to sizeof(type)/sizeof(T). The shape then goes from +// [..., sizeof(type)/sizeof(T)] to [...]. +// +// Bitcast is implemented as a low-level cast, so machines with different endian +// orderings will give different results. + +static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { + // TODO: replace with a void* pointer type later + int a = 4; + return static_cast(&a); +} + +static void SummaryScalarOp_Delete(void* kernel) { + return; +} + +bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ + if (TF_NumDims(tensor1) != TF_NumDims(tensor2)){ + return false; + } + for(int d = 0; d < TF_NumDims(tensor1); d++){ + if (TF_Dim(tensor1, d) != TF_Dim(tensor2, d)){ + return false; + } + } + return true; +} + +template +static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { + TF_Tensor* tags; + TF_Tensor* values; + TF_Status* status = TF_NewStatus(); + TF_GetInput(ctx, 0, &tags, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while getting input"; + if (TF_GetCode(status) == TF_OK){ + TF_GetInput(ctx, 1, &values, status); + } + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while getting input"; + if (TF_GetCode(status) == TF_OK) { + if (!IsSameSize(tags, values)) { + std::ostringstream err; + err << "tags and values not the same shape: "; + TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); + } + } + + tensorflow::Summary s; + if (TF_GetCode(status) == TF_OK) { + auto Ttags_array = static_cast(TF_TensorData(tags)); + auto values_array = static_cast(TF_TensorData(values)); + for (int i = 0; i < TF_TensorElementCount(tags); ++i){ + tensorflow::Summary::Value* v = s.add_value(); + TF_TString_Init(Ttags_array[i]); + v->set_tag(TF_TString_GetDataPointer(Ttags_array[i]), TF_TString_GetSize(Ttags_array[i])); + v->set_simple_value(float(values_array[i])); + } + + + // TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), 0, 0) + + // TF_Tensor* output = TF_AllocateTensor(k->output_data_type, dims, 0, + // TF_DataTypeSize(k->output_data_type)); + // if (TF_GetCode(status) == TF_OK) { + // TF_SetOutput(ctx, 0, output, status); + // } + // TF_DeleteTensor(output); + } + + // if (TF_GetCode(status) != TF_OK) { + // TF_OpKernelContext_Failure(ctx, status); + // } + // TF_DeleteStatus(status); + // TF_DeleteTensor(tags); +} + +template +void RegisterSummaryScalarOpKernel() { + TF_Status* status = TF_NewStatus(); + { + auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_CPU, + &SummaryScalarOp_Create, &SummaryScalarOp_Compute, + &SummaryScalarOp_Delete); + TF_KernelBuilder_TypeConstraint(builder, "T", static_cast(tensorflow::DataTypeToEnum::v()), status); + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while adding type constraint"; + TF_RegisterKernelBuilder("SummaryScalar", builder, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while registering Summary Scalar kernel"; + } +// template +// #if GOOGLE_CUDA +// { +// auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_GPU, +// &SummaryScalarOp_Create, &SummaryScalarOp_Compute, +// &SummaryScalarOp_Delete); +// TF_RegisterKernelBuilder("SummaryScalar", builder, status); +// CHECK_EQ(TF_OK, TF_GetCode(status)) +// << "Error while registering CUDA SummaryScalar kernel"; +// } +// #endif + + TF_DeleteStatus(status); +} + +// A dummy static variable initialized by a lambda whose side-effect is to +// register the bitcast kernel. + + +TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { + if (SHOULD_REGISTER_OP_KERNEL("SummaryScalar")) { + RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); + } + return true; +}(); + diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc new file mode 100644 index 00000000000..fd6199abd6c --- /dev/null +++ b/tensorflow/c/kernels/summary_op_test.cc @@ -0,0 +1,96 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/attr_value_util.h" +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/platform/test.h" + +#include +#include +#include +namespace tensorflow { +namespace { + +class DummyDevice : public DeviceBase { + public: + explicit DummyDevice(Env* env) : DeviceBase(env) {} + Allocator* GetAllocator(AllocatorAttributes /*attr*/) override { + return cpu_allocator(); + } +}; + +void TestScalarSummaryOp(Tensor* tags, Tensor* values, error::Code expected_code) { + Status status; + NodeDef def; + def.set_op("SummaryScalar"); + + def.set_device(DEVICE_CPU); + + AttrValue valuesTypeAttr; + SetAttrValue(values->dtype(), &valuesTypeAttr); + (*def.mutable_attr())["T"] = valuesTypeAttr; + + def.add_input( + strings::StrCat("input1: ", DataTypeString(tags->dtype()))); + def.add_input( + strings::StrCat("input2: ", DataTypeString(values->dtype()))); + + std::unique_ptr kernel = + CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status); + ASSERT_TRUE(status.ok()) << status.ToString(); + OpKernelContext::Params params; + DummyDevice dummy_device(nullptr); + params.device = &dummy_device; + params.op_kernel = kernel.get(); + gtl::InlinedVector inputs; + inputs.emplace_back(tags); + inputs.emplace_back(values); + params.inputs = &inputs; + OpKernelContext ctx(¶ms, 1); + kernel->Compute(&ctx); + + ASSERT_EQ(expected_code, ctx.status().code()); + if (expected_code == error::OK) { + ASSERT_EQ(true, false) + << ctx.mutable_output(0)->shape().DebugString(); + } +} + +TEST(ScalarSummaryOpTest, Test) { + int vectorSize = 2; + Tensor tags(DT_STRING, {vectorSize}); + Tensor values(DT_FLOAT, {vectorSize}); + for (int i = 0; i < vectorSize; ++i){ + values.vec()(i) = static_cast(i); + } + tags.vec()(0) = "tag 1"; + tags.vec()(1) = "tag 2"; + TestScalarSummaryOp(&tags, &values, error::INVALID_ARGUMENT); +} + + +PartialTensorShape S(std::initializer_list dims) { + return PartialTensorShape(dims); +} + + + +} // namespace +} // namespace tensorflow From 6c82e7b9ab9b9f21508e2c0c1efe1209b16d0b83 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 18 Jun 2020 23:21:30 +0000 Subject: [PATCH 0006/1017] fixed comments --- ...-summary_op-needs-tstring-C-API-sync.patch | 377 ++++++++++++++++++ tensorflow/c/kernels/summary_op.cc | 21 +- 2 files changed, 379 insertions(+), 19 deletions(-) create mode 100644 tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch diff --git a/tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch b/tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch new file mode 100644 index 00000000000..856f4a554c3 --- /dev/null +++ b/tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch @@ -0,0 +1,377 @@ +From 9134fbb13794865a45288d2e722ad47c362e0ae4 Mon Sep 17 00:00:00 2001 +From: Daniel Nguyen +Date: Thu, 18 Jun 2020 23:13:11 +0000 +Subject: [PATCH] summary_op needs tstring C API sync + +--- + tensorflow/c/kernels/diff.patch | 0 + tensorflow/c/kernels/ops/summary.cc | 70 ++++++++++ + tensorflow/c/kernels/summary_op.cc | 171 ++++++++++++++++++++++++ + tensorflow/c/kernels/summary_op_test.cc | 96 +++++++++++++ + 4 files changed, 337 insertions(+) + create mode 100644 tensorflow/c/kernels/diff.patch + create mode 100644 tensorflow/c/kernels/ops/summary.cc + create mode 100644 tensorflow/c/kernels/summary_op.cc + create mode 100644 tensorflow/c/kernels/summary_op_test.cc + +diff --git a/tensorflow/c/kernels/diff.patch b/tensorflow/c/kernels/diff.patch +new file mode 100644 +index 0000000000..e69de29bb2 +diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc +new file mode 100644 +index 0000000000..550a663d00 +--- /dev/null ++++ b/tensorflow/c/kernels/ops/summary.cc +@@ -0,0 +1,70 @@ ++/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. ++ ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ ++ http://www.apache.org/licenses/LICENSE-2.0 ++ ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include ++#include ++ ++#include "tensorflow/c/ops.h" ++#include "tensorflow/core/framework/selective_registration.h" ++#include "tensorflow/core/platform/logging.h" ++#include "tensorflow/core/platform/macros.h" ++ ++ ++static void TF_ScalarSummary_shape_inference_fn(TF_ShapeInferenceContext* ctx, ++ TF_Status* status) { ++ TF_ShapeHandle* result = TF_NewShapeHandle(); ++ // TODO: what to do in the case of unknown input shape? ++ if (TF_GetCode(status) == TF_OK && ++ !TF_ShapeInferenceContextRankKnown(ctx, result)) { ++ TF_ShapeInferenceContextSetUnknownShape(ctx, status); ++ CHECK_EQ(TF_OK, TF_GetCode(status)) ++ << "Error while setting unknown shape function"; ++ TF_DeleteShapeHandle(result); ++ return; ++ } ++ // make shape handle a scalar value (empty shape) ++ if (TF_GetCode(status) == TF_OK) { ++ TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); ++ CHECK_EQ(TF_OK, TF_GetCode(status)) ++ << "Error while setting shape function"; ++ } ++ TF_DeleteShapeHandle(result); ++} ++ ++void Register_ScalarSummaryOp() { ++ TF_Status* status = TF_NewStatus(); ++ ++ TF_OpDefinitionBuilder* op_builder = TF_NewOpDefinitionBuilder("SummaryScalar"); ++ TF_OpDefinitionBuilderAddInput(op_builder, "tags: string"); ++ TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); ++ TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); ++ TF_OpDefinitionBuilderAddAttr( ++ op_builder, ++ "T: realnumbertype"); ++ TF_OpDefinitionBuilderSetShapeInferenceFunction(op_builder, ++ &TF_ScalarSummary_shape_inference_fn); ++ ++ TF_RegisterOpDefinition(op_builder, status); ++ CHECK_EQ(TF_GetCode(status), TF_OK) ++ << "TF_ScalarSummary op registration failed: " << TF_Message(status); ++ TF_DeleteStatus(status); ++} ++ ++TF_ATTRIBUTE_UNUSED static bool SummaryScalarOpRegistered = []() { ++ if (SHOULD_REGISTER_OP("SummaryScalar")) { ++ Register_ScalarSummaryOp(); ++ } ++ return true; ++}(); +diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc +new file mode 100644 +index 0000000000..3a78d321d7 +--- /dev/null ++++ b/tensorflow/c/kernels/summary_op.cc +@@ -0,0 +1,171 @@ ++ ++/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. ++ ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ ++ http://www.apache.org/licenses/LICENSE-2.0 ++ ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include ++ ++#include "tensorflow/c/kernels.h" ++#include "tensorflow/c/ops.h" ++#include "tensorflow/c/tf_tensor.h" ++#include "tensorflow/core/framework/common_shape_fns.h" ++#include "tensorflow/core/framework/op.h" ++#include "tensorflow/core/framework/selective_registration.h" ++#include "tensorflow/core/framework/shape_inference.h" ++#include "tensorflow/core/platform/macros.h" ++#include "tensorflow/core/framework/summary.pb.h" ++#include "tensorflow/core/platform/protobuf.h" ++#include "tensorflow/core/framework/register_types.h" ++ ++#include "tensorflow/core/framework/types.h" ++ ++// BitcastOp implements a bitcast kernel, creating an output tensor that shares ++// the same data buffer as the input but with a different shape and/or data ++// type. Its inputs are: ++// ++// * the input tensor ++// * an attribute named "T" containing the TF_DataType of the input tensor ++// * an attribute named "type" containing the TF_DataType of the output tensor ++// ++// Given an input tensor of shape [...], if the input DataType "T" is larger ++// than the output DataType "type", then the shape changes from [...] ++// to [..., sizeof(T)/sizeof(type)]. ++// ++// If "T" is smaller than "type", the operator requires that the rightmost ++// dimension be equal to sizeof(type)/sizeof(T). The shape then goes from ++// [..., sizeof(type)/sizeof(T)] to [...]. ++// ++// Bitcast is implemented as a low-level cast, so machines with different endian ++// orderings will give different results. ++ ++static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { ++ // TODO: replace with a void* pointer type later ++ int a = 4; ++ return static_cast(&a); ++} ++ ++static void SummaryScalarOp_Delete(void* kernel) { ++ return; ++} ++ ++bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ ++ if (TF_NumDims(tensor1) != TF_NumDims(tensor2)){ ++ return false; ++ } ++ for(int d = 0; d < TF_NumDims(tensor1); d++){ ++ if (TF_Dim(tensor1, d) != TF_Dim(tensor2, d)){ ++ return false; ++ } ++ } ++ return true; ++} ++ ++template ++static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { ++ TF_Tensor* tags; ++ TF_Tensor* values; ++ TF_Status* status = TF_NewStatus(); ++ TF_GetInput(ctx, 0, &tags, status); ++ CHECK_EQ(TF_OK, TF_GetCode(status)) ++ << "Error while getting input"; ++ if (TF_GetCode(status) == TF_OK){ ++ TF_GetInput(ctx, 1, &values, status); ++ } ++ CHECK_EQ(TF_OK, TF_GetCode(status)) ++ << "Error while getting input"; ++ if (TF_GetCode(status) == TF_OK) { ++ if (!IsSameSize(tags, values)) { ++ std::ostringstream err; ++ err << "tags and values not the same shape: "; ++ TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); ++ } ++ } ++ ++ tensorflow::Summary s; ++ if (TF_GetCode(status) == TF_OK) { ++ auto Ttags_array = static_cast(TF_TensorData(tags)); ++ auto values_array = static_cast(TF_TensorData(values)); ++ for (int i = 0; i < TF_TensorElementCount(tags); ++i){ ++ tensorflow::Summary::Value* v = s.add_value(); ++ TF_TString_Init(Ttags_array[i]); ++ v->set_tag(TF_TString_GetDataPointer(Ttags_array[i]), TF_TString_GetSize(Ttags_array[i])); ++ v->set_simple_value(float(values_array[i])); ++ } ++ ++ ++ // TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), 0, 0) ++ ++ // TF_Tensor* output = TF_AllocateTensor(k->output_data_type, dims, 0, ++ // TF_DataTypeSize(k->output_data_type)); ++ // if (TF_GetCode(status) == TF_OK) { ++ // TF_SetOutput(ctx, 0, output, status); ++ // } ++ // TF_DeleteTensor(output); ++ } ++ ++ // if (TF_GetCode(status) != TF_OK) { ++ // TF_OpKernelContext_Failure(ctx, status); ++ // } ++ // TF_DeleteStatus(status); ++ // TF_DeleteTensor(tags); ++} ++ ++template ++void RegisterSummaryScalarOpKernel() { ++ TF_Status* status = TF_NewStatus(); ++ { ++ auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_CPU, ++ &SummaryScalarOp_Create, &SummaryScalarOp_Compute, ++ &SummaryScalarOp_Delete); ++ TF_KernelBuilder_TypeConstraint(builder, "T", static_cast(tensorflow::DataTypeToEnum::v()), status); ++ CHECK_EQ(TF_OK, TF_GetCode(status)) ++ << "Error while adding type constraint"; ++ TF_RegisterKernelBuilder("SummaryScalar", builder, status); ++ CHECK_EQ(TF_OK, TF_GetCode(status)) ++ << "Error while registering Summary Scalar kernel"; ++ } ++// template ++// #if GOOGLE_CUDA ++// { ++// auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_GPU, ++// &SummaryScalarOp_Create, &SummaryScalarOp_Compute, ++// &SummaryScalarOp_Delete); ++// TF_RegisterKernelBuilder("SummaryScalar", builder, status); ++// CHECK_EQ(TF_OK, TF_GetCode(status)) ++// << "Error while registering CUDA SummaryScalar kernel"; ++// } ++// #endif ++ ++ TF_DeleteStatus(status); ++} ++ ++// A dummy static variable initialized by a lambda whose side-effect is to ++// register the bitcast kernel. ++ ++ ++TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { ++ if (SHOULD_REGISTER_OP_KERNEL("SummaryScalar")) { ++ RegisterSummaryScalarOpKernel(); ++ RegisterSummaryScalarOpKernel(); ++ RegisterSummaryScalarOpKernel(); ++ RegisterSummaryScalarOpKernel(); ++ RegisterSummaryScalarOpKernel(); ++ RegisterSummaryScalarOpKernel(); ++ RegisterSummaryScalarOpKernel(); ++ RegisterSummaryScalarOpKernel(); ++ RegisterSummaryScalarOpKernel(); ++ } ++ return true; ++}(); ++ +diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc +new file mode 100644 +index 0000000000..fd6199abd6 +--- /dev/null ++++ b/tensorflow/c/kernels/summary_op_test.cc +@@ -0,0 +1,96 @@ ++/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. ++ ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ ++ http://www.apache.org/licenses/LICENSE-2.0 ++ ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include "tensorflow/core/framework/attr_value.pb.h" ++#include "tensorflow/core/framework/attr_value_util.h" ++#include "tensorflow/core/framework/fake_input.h" ++#include "tensorflow/core/framework/node_def.pb.h" ++#include "tensorflow/core/framework/node_def_builder.h" ++#include "tensorflow/core/framework/op_kernel.h" ++#include "tensorflow/core/framework/shape_inference.h" ++#include "tensorflow/core/platform/test.h" ++ ++#include ++#include ++#include ++namespace tensorflow { ++namespace { ++ ++class DummyDevice : public DeviceBase { ++ public: ++ explicit DummyDevice(Env* env) : DeviceBase(env) {} ++ Allocator* GetAllocator(AllocatorAttributes /*attr*/) override { ++ return cpu_allocator(); ++ } ++}; ++ ++void TestScalarSummaryOp(Tensor* tags, Tensor* values, error::Code expected_code) { ++ Status status; ++ NodeDef def; ++ def.set_op("SummaryScalar"); ++ ++ def.set_device(DEVICE_CPU); ++ ++ AttrValue valuesTypeAttr; ++ SetAttrValue(values->dtype(), &valuesTypeAttr); ++ (*def.mutable_attr())["T"] = valuesTypeAttr; ++ ++ def.add_input( ++ strings::StrCat("input1: ", DataTypeString(tags->dtype()))); ++ def.add_input( ++ strings::StrCat("input2: ", DataTypeString(values->dtype()))); ++ ++ std::unique_ptr kernel = ++ CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status); ++ ASSERT_TRUE(status.ok()) << status.ToString(); ++ OpKernelContext::Params params; ++ DummyDevice dummy_device(nullptr); ++ params.device = &dummy_device; ++ params.op_kernel = kernel.get(); ++ gtl::InlinedVector inputs; ++ inputs.emplace_back(tags); ++ inputs.emplace_back(values); ++ params.inputs = &inputs; ++ OpKernelContext ctx(¶ms, 1); ++ kernel->Compute(&ctx); ++ ++ ASSERT_EQ(expected_code, ctx.status().code()); ++ if (expected_code == error::OK) { ++ ASSERT_EQ(true, false) ++ << ctx.mutable_output(0)->shape().DebugString(); ++ } ++} ++ ++TEST(ScalarSummaryOpTest, Test) { ++ int vectorSize = 2; ++ Tensor tags(DT_STRING, {vectorSize}); ++ Tensor values(DT_FLOAT, {vectorSize}); ++ for (int i = 0; i < vectorSize; ++i){ ++ values.vec()(i) = static_cast(i); ++ } ++ tags.vec()(0) = "tag 1"; ++ tags.vec()(1) = "tag 2"; ++ TestScalarSummaryOp(&tags, &values, error::INVALID_ARGUMENT); ++} ++ ++ ++PartialTensorShape S(std::initializer_list dims) { ++ return PartialTensorShape(dims); ++} ++ ++ ++ ++} // namespace ++} // namespace tensorflow +-- +2.27.0.111.gc72c7da667-goog + diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 3a78d321d75..6921eb4fdaa 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -30,24 +30,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" -// BitcastOp implements a bitcast kernel, creating an output tensor that shares -// the same data buffer as the input but with a different shape and/or data -// type. Its inputs are: -// -// * the input tensor -// * an attribute named "T" containing the TF_DataType of the input tensor -// * an attribute named "type" containing the TF_DataType of the output tensor -// -// Given an input tensor of shape [...], if the input DataType "T" is larger -// than the output DataType "type", then the shape changes from [...] -// to [..., sizeof(T)/sizeof(type)]. -// -// If "T" is smaller than "type", the operator requires that the rightmost -// dimension be equal to sizeof(type)/sizeof(T). The shape then goes from -// [..., sizeof(type)/sizeof(T)] to [...]. -// -// Bitcast is implemented as a low-level cast, so machines with different endian -// orderings will give different results. +// TODO: Copy over Summary Scalar Op Doc static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { // TODO: replace with a void* pointer type later @@ -91,7 +74,7 @@ static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); } } - + // Copy tag and string data into summary protobuf tensorflow::Summary s; if (TF_GetCode(status) == TF_OK) { auto Ttags_array = static_cast(TF_TensorData(tags)); From 43442a2c7f449fb979c26e543976a087fddad96c Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 22 Jun 2020 16:43:58 +0000 Subject: [PATCH 0007/1017] initial map ops set up --- tensorflow/core/BUILD | 2 + tensorflow/core/kernels/map_kernels.cc | 52 ++++++++++++++++++ tensorflow/core/ops/map_ops.cc | 33 ++++++++++++ tensorflow/python/BUILD | 14 +++++ tensorflow/python/kernel_tests/BUILD | 24 +++++++++ .../python/kernel_tests/map_ops_test.py | 39 ++++++++++++++ tensorflow/python/ops/map_ops.py | 53 +++++++++++++++++++ 7 files changed, 217 insertions(+) create mode 100644 tensorflow/core/kernels/map_kernels.cc create mode 100644 tensorflow/core/ops/map_ops.cc create mode 100644 tensorflow/python/kernel_tests/map_ops_test.py create mode 100644 tensorflow/python/ops/map_ops.py diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 2b16801f6ed..66e7061642c 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -628,6 +628,7 @@ tf_gen_op_libs( "io_ops", "linalg_ops", "list_ops", + "map_ops", "lookup_ops", "logging_ops", "manip_ops", @@ -859,6 +860,7 @@ cc_library( ":io_ops_op_lib", ":linalg_ops_op_lib", ":list_ops_op_lib", + ":map_ops_op_lib", ":logging_ops_op_lib", ":lookup_ops_op_lib", ":manip_ops_op_lib", diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc new file mode 100644 index 00000000000..bd6d880aec5 --- /dev/null +++ b/tensorflow/core/kernels/map_kernels.cc @@ -0,0 +1,52 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/op_kernel.h" +#include +using namespace std; + +namespace tensorflow { + +class ZeroOutOp : public OpKernel { + public: + explicit ZeroOutOp(OpKernelConstruction* c) : OpKernel(c) {} + + void Compute(OpKernelContext* c) override { + cout << "Hello World - Op" << endl; + // Grab the input tensor + const Tensor& input_tensor = c->input(0); + auto input = input_tensor.flat(); + + // Create an output tensor + Tensor* output_tensor = NULL; + OP_REQUIRES_OK(context, context->allocate_output(0, input_tensor.shape(), + &output_tensor)); + auto output_flat = output_tensor->flat(); + + // Set all but the first element of the output tensor to 0 + const int N = input.size(); + for (int i=1; i 0) output_flat(0) = input(0); + } +}; + +REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), + ZeroOutOp); + +} // namespace tensorflow diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc new file mode 100644 index 00000000000..bdad6d389b0 --- /dev/null +++ b/tensorflow/core/ops/map_ops.cc @@ -0,0 +1,33 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/shape_inference.h" + +namespace tensorflow { +namespace { + +REGISTER_OP("ZeroOut") + .Input("to_zero: int32") + .Output("zeroed: int32") + .SetShapeFn([](shape_inference::InferenceContext* c) { + //c->set_output(0, c->Scalar()); + c->set_output(0, c->input(0)); + return Status::OK(); + }); + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index e71900b430f..3e59e61ae88 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3000,6 +3000,10 @@ tf_gen_op_wrapper_private_py( name = "list_ops_gen", ) +tf_gen_op_wrapper_private_py( + name = "map_ops_gen", +) + tf_gen_op_wrapper_private_py( name = "script_ops_gen", ) @@ -4175,6 +4179,16 @@ py_library( ], ) +py_library( + name = "map_ops", + srcs = ["ops/map_ops.py"], + srcs_version = "PY2AND3", + deps = [ + ":array_ops", + ":map_ops_gen", + ], +) + py_library( name = "nn", srcs = [ diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index a04c874c9d6..d8c8e3dc2a8 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -142,6 +142,30 @@ cuda_py_test( ], ) +cuda_py_test( + name = "map_ops_test", + size = "small", + srcs = ["map_ops_test.py"], + grpc_enabled = True, + tags = [ + "noasan", # TODO(b/155406705): flaky + ], + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:framework_test_lib", + "//tensorflow/python:gradients_impl", + "//tensorflow/python:map_ops", + "//tensorflow/python:math_ops", + "//tensorflow/python:tensor_shape", + "//tensorflow/python/eager:context", + "//tensorflow/python/eager:def_function", + "//third_party/py/numpy", + "@absl_py//absl/testing:parameterized", + ], +) + cuda_py_test( name = "benchmark_test", size = "small", diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py new file mode 100644 index 00000000000..28cadf0a6df --- /dev/null +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -0,0 +1,39 @@ +# Copyright 2018 The Sonnet Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Tests for zero_out ops.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy as np + +from tensorflow.python.platform import test +#try: +# from tensorflow_zero_out.python.ops.zero_out_ops import zero_out +#except ImportError: +# from zero_out_ops import zero_out +from tensorflow.python.ops import map_ops + +class ZeroOutTest(test.TestCase): + + def testZeroOut(self): + print("Hello World - Test") + with self.test_session(): + self.assertAllClose( + zero_out([[1, 2], [3, 4]]), np.array([[1, 0], [0, 0]])) + + +if __name__ == '__main__': + test.main() \ No newline at end of file diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py new file mode 100644 index 00000000000..93f4cc0bdc8 --- /dev/null +++ b/tensorflow/python/ops/map_ops.py @@ -0,0 +1,53 @@ +# Copyright 2018 The Sonnet Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Use zero_out ops in python.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.framework import load_library +from tensorflow.python.platform import resource_loader +# go/tf-wildcard-import +# pylint: disable=wildcard-import +from tensorflow.python.ops import gen_map_ops +from tensorflow.python.ops.gen_map_ops import * + +#zero_out_ops = load_library.load_op_library( +# resource_loader.get_path_to_datafile('_zero_out_ops.so')) +#zero_out = zero_out_ops.zero_out + +def zero_out(to_zero): + print("Hello World - PythonS Op") + return gen_map_ops.zero_out(to_zero) + +@ops.RegisterGradient("ZeroOut") +def _zero_out_grad(op, grad): + """The gradients for `zero_out`. + + Args: + op: The `zero_out` `Operation` that we are differentiating, which we can use + to find the inputs and outputs of the original op. + grad: Gradient with respect to the output of the `zero_out` op. + + Returns: + Gradients with respect to the input of `zero_out`. + """ + to_zero = op.inputs[0] + shape = array_ops.shape(to_zero) + index = array_ops.zeros_like(shape) + first_grad = array_ops.reshape(grad, [-1])[0] + to_zero_grad = sparse_ops.sparse_to_dense([index], shape, first_grad, 0) + return [to_zero_grad] # List of one Tensor, since we have one input From f8943f369b079fbb833ebbfefcf66a5454c98a32 Mon Sep 17 00:00:00 2001 From: "902449@58880@bigcat_chen@ASIC" Date: Tue, 23 Jun 2020 14:06:45 +0800 Subject: [PATCH 0008/1017] remove unuse comment --- tensorflow/lite/micro/himax_we1_evb/debug_log.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorflow/lite/micro/himax_we1_evb/debug_log.cc b/tensorflow/lite/micro/himax_we1_evb/debug_log.cc index 36ac3f3fa03..0cd26874646 100644 --- a/tensorflow/lite/micro/himax_we1_evb/debug_log.cc +++ b/tensorflow/lite/micro/himax_we1_evb/debug_log.cc @@ -13,10 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// Implementation for the DebugLog() function that prints to the UART on the -// SparkFun Edge microcontroller. The same should work for other targets using -// the Ambiq Apollo 3. - #include "tensorflow/lite/micro/debug_log.h" #include "hx_drv_tflm.h" From b619d2cfdcb2a1875333243bb7b8bb49ec95ad10 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 24 Jun 2020 14:51:28 +0000 Subject: [PATCH 0009/1017] cc build file --- tensorflow/cc/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD index e1fad8e697a..8602bfafff8 100644 --- a/tensorflow/cc/BUILD +++ b/tensorflow/cc/BUILD @@ -558,6 +558,7 @@ tf_gen_op_wrappers_cc( "io_ops", "linalg_ops", "list_ops", + "map_ops", "logging_ops", "lookup_ops", "manip_ops", From fc15ab8a358a2b14d683671fbb1de403ed77c6b8 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 24 Jun 2020 16:58:44 +0000 Subject: [PATCH 0010/1017] initial TensorMap class --- tensorflow/core/kernels/tensor_map.cc | 146 +++++++++++++++++++++++ tensorflow/core/kernels/tensor_map.h | 159 ++++++++++++++++++++++++++ 2 files changed, 305 insertions(+) create mode 100644 tensorflow/core/kernels/tensor_map.cc create mode 100644 tensorflow/core/kernels/tensor_map.h diff --git a/tensorflow/core/kernels/tensor_map.cc b/tensorflow/core/kernels/tensor_map.cc new file mode 100644 index 00000000000..a0b2c7a9f7a --- /dev/null +++ b/tensorflow/core/kernels/tensor_map.cc @@ -0,0 +1,146 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/kernels/tensor_map.h" + +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/variant_op_registry.h" +#include "tensorflow/core/lib/core/coding.h" + +namespace tensorflow { + +TensorMap::~TensorMap() { + if (tensors_) tensors_->Unref(); +} + +void TensorMap::Encode(VariantTensorData* data) const { + data->set_type_name(TypeName()); + + std::map::iterator map_it = tensors().begin(); + size_t i = 0; + std::vector invalid_indices; + while (map_it != tensors().end()) { + Tensor k = map_it->first; + Tensor v = map_it->second; + // k should also not be DT_RESOURCE or DT_VARIANT + if(k.dtype != DT_INVALID && v.dtype != DT_INVALID) { + *data->add_tensors() = k; + *data->add_tensors() = v; + // not sure if this is the correct order + } + else { + invalid_indices.push_back(i); + } + } + /* + for (size_t i = 0; i < tensors().size(); i++) { + if (tensors().at(i).dtype() != DT_INVALID) { + *data->add_tensors() = tensors().at(i); + } else { + invalid_indices.push_back(i); + } + }*/ + string metadata; + // TODO(b/118838800): Add a proto for storing the metadata. + // Metadata format: + // + core::PutVarint64(&metadata, static_cast(invalid_indices.size())); + for (size_t i : invalid_indices) { + core::PutVarint64(&metadata, static_cast(i)); + } + core::PutVarint64(&metadata, static_cast(element_dtype)); + core::PutVarint64(&metadata, static_cast(max_num_elements)); + TensorShapeProto element_shape_proto; + element_shape.AsProto(&element_shape_proto); + element_shape_proto.AppendToString(&metadata); + data->set_metadata(metadata); +} + +static Status TensorMapDeviceCopy( + const TensorMap& from, TensorMap* to, + const UnaryVariantOpRegistry::AsyncTensorDeviceCopyFn& copy) { + to->element_shape = from.element_shape; + to->element_dtype = from.element_dtype; + to->max_num_elements = from.max_num_elements; + //to->tensors().reserve(from.tensors().size()); + for (const std::pair& p : from.tensors()) { + to->tensors().emplace(p); //why was it emplace t.dtype? + if (t.dtype() != DT_INVALID) { + //TF_RETURN_IF_ERROR(copy(p, &to->tensors().back())); + } + } + return Status::OK(); +} + +#define REGISTER_LIST_COPY(DIRECTION) \ + INTERNAL_REGISTER_UNARY_VARIANT_DEVICE_COPY_FUNCTION(TensorMap, DIRECTION, \ + TensorMapDeviceCopy) + +REGISTER_LIST_COPY(VariantDeviceCopyDirection::HOST_TO_DEVICE); +REGISTER_LIST_COPY(VariantDeviceCopyDirection::DEVICE_TO_HOST); +REGISTER_LIST_COPY(VariantDeviceCopyDirection::DEVICE_TO_DEVICE); + +REGISTER_UNARY_VARIANT_DECODE_FUNCTION(TensorMap, TensorMap::kTypeName); + +bool TensorMap::Decode(const VariantTensorData& data) { + // TODO(srbs): Change the signature to Decode(VariantTensorData data) so + // that we do not have to copy each tensor individually below. This would + // require changing VariantTensorData::tensors() as well. + string metadata; + data.get_metadata(&metadata); + uint64 scratch; + StringPiece iter(metadata); + std::vector invalid_indices; + core::GetVarint64(&iter, &scratch); + size_t num_invalid_tensors = static_cast(scratch); + invalid_indices.resize(num_invalid_tensors); + for (size_t i = 0; i < num_invalid_tensors; i++) { + core::GetVarint64(&iter, &scratch); + invalid_indices[i] = static_cast(scratch); + } + + size_t total_num_tensors = data.tensors().size()/2 + num_invalid_tensors; + //tensors().reserve(total_num_tensors); + std::vector::iterator invalid_indices_it = invalid_indices.begin(); + std::vector::const_iterator tensors_it = data.tensors().begin(); + for (size_t i = 0; i < total_num_tensors; i++) { + if (invalid_indices_it != invalid_indices.end() && + *invalid_indices_it == i) { + //no need to do invalid indices for a map + //tensors().emplace(Tensor(DT_INVALID),Tensor(DT_INVALID)); + invalid_indices_it++; + } else if (tensors_it != data.tensors().end()) { + // should assert that tensors_it + 1 is also not the end + tensors().emplace(*tensors_it,*++tensors_it); + tensors_it++; + } else { + // VariantTensorData is corrupted. + return false; + } + } + + core::GetVarint64(&iter, &scratch); + element_dtype = static_cast(scratch); + core::GetVarint64(&iter, &scratch); + max_num_elements = static_cast(scratch); + TensorShapeProto element_shape_proto; + element_shape_proto.ParseFromString(string(iter.data(), iter.size())); + element_shape = PartialTensorShape(element_shape_proto); + return true; +} + +const char TensorMap::kTypeName[] = "tensorflow::TensorMap"; + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h new file mode 100644 index 00000000000..d8726bbecb4 --- /dev/null +++ b/tensorflow/core/kernels/tensor_map.h @@ -0,0 +1,159 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_KERNELS_TENSOR_LIST_H_ +#define TENSORFLOW_CORE_KERNELS_TENSOR_LIST_H_ + +#include + +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/variant.h" +#include "tensorflow/core/framework/variant_tensor_data.h" +#include "tensorflow/core/lib/core/refcount.h" + +namespace tensorflow { + +// Variant compatible type for a map of tensors. This is mutable but instances +// should never be mutated after stored in a variant tensor. +// +// **NOTE**: TensorMap stores a refcounted container of tf::Tensor objects, +// which are accessible via TensorMap::tensors(). Because it is refcounted, +// straight copies of the form: +// +// TensorMap b = a; +// b.tensors().insert(k,v); // WARNING: This modifies a.tensors(). +// +// Do not create a true copy of the underlying container - but instead increment +// a reference count. Modifying b.tensors() modifies a.tensors(). In this way, +// TensorList should be considered similar to the tf::Tensor object. +// +// In order to get a copy of the underlying list, use the Copy method: +// +// TensorList b = a.Copy(); +// b.tensors().push_back(t); // This does not modify a.tensors(). +// +// Note that this is not a deep copy: the memory locations of the underlying +// tensors will still point to the same locations of the corresponding tensors +// in the original. To truly perform a deep copy, Device and Type-specific +// code needs to be applied to the underlying tensors as usual. +// +// The most important implication of RefCounted TLs is that OpKernels +// wishing to reuse TensorList inputs as outputs via context->forward_input() +// need to perform an additional check on the refcount of the TensorList, +// to ensure aliasing can be performed safely. For example: +// +// bool can_alias = false; +// auto fw = c->forward_input(..., DT_VARIANT, {}, ...); +// if (fw && fw->dtype() == DT_VARIANT && fw->NumElements() == 1) { +// auto* tl = fw->scalar()().get(); +// if (tl && tl->RefCountIsOne()) { +// can_alias = true; +// } +// } +// +class TensorMap { + public: + TensorMap() : tensors_(new Tensors) {} + ~TensorMap(); + + TensorMap(const TensorMap& other) + : element_shape(other.element_shape), + element_dtype(other.element_dtype), + max_num_elements(other.max_num_elements), + tensors_(other.tensors_) { + tensors_->Ref(); + } + + TensorMap(TensorMap&& rhs) + : element_shape(std::move(rhs.element_shape)), + element_dtype(rhs.element_dtype), + max_num_elements(rhs.max_num_elements), + tensors_(rhs.tensors_) { + rhs.tensors_ = nullptr; + } + + TensorMap& operator=(const TensorMap& rhs) { + if (this == &rhs) return *this; + element_shape = rhs.element_shape; + element_dtype = rhs.element_dtype; + max_num_elements = rhs.max_num_elements; + tensors_->Unref(); + tensors_ = rhs.tensors_; + tensors_->Ref(); + return *this; + } + + TensorMap& operator=(TensorMap&& rhs) { + if (this == &rhs) return *this; + element_shape = rhs.element_shape; + element_dtype = rhs.element_dtype; + max_num_elements = rhs.max_num_elements; + std::swap(tensors_, rhs.tensors_); + return *this; + } + + static const char kTypeName[]; + + string TypeName() const { return kTypeName; } + + void Encode(VariantTensorData* data) const; + + bool Decode(const VariantTensorData& data); + + // TODO(apassos) fill this out + string DebugString() const { return "TensorMap"; } + + PartialTensorShape element_shape; + + DataType element_dtype; + + // The maximum allowed size of `tensors`. Defaults to -1 meaning that the size + // of `tensors` is unbounded. + int max_num_elements = -1; + + // Access to the underlying tensor container. + std::map& tensors() { return tensors_->values_; } + const std::map& tensors() const { return tensors_->values_; } + + // Get a new TensorList containing a copy of the underlying tensor container. + TensorMap Copy() const { + TensorMap out; + out.element_shape = element_shape; + out.element_dtype = element_dtype; + out.max_num_elements = max_num_elements; + // This performs a copy of the std::map. + out.tensors_->values_ = tensors_->values_; + return out; + } + + // Is this TensorMap the only one with a reference to the underlying + // container? + bool RefCountIsOne() const { return tensors_->RefCountIsOne(); } + + private: + class Tensors : public core::RefCounted { + public: + std::map values_; + }; + Tensors* tensors_; +}; + +#if defined(PLATFORM_GOOGLE) +// TODO(ebrevdo): Identify why Variant inline size is smaller on mobile devices. +static_assert(Variant::CanInlineType(), + "Must be able to inline TensorMap into a Variant"); +#endif +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_TENSOR_LIST_H_ From 26612305bb8c8af70f8845d2d3bb65e96bf3dbe8 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 24 Jun 2020 17:01:49 +0000 Subject: [PATCH 0011/1017] build and test updates --- tensorflow/core/BUILD | 1 + tensorflow/core/kernels/BUILD | 39 +++++++++++++++++++ tensorflow/core/kernels/map_kernels.cc | 36 ++++++++++++++++- tensorflow/core/ops/map_ops.cc | 20 ++++++++++ .../python/kernel_tests/map_ops_test.py | 21 +++++++++- tensorflow/python/ops/map_ops.py | 16 +++++++- 6 files changed, 129 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 66e7061642c..d6e44bb36aa 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1120,6 +1120,7 @@ cc_library( # these also dynamically loading. "//tensorflow/core/kernels:dataset_ops", # Depends on grappler "//tensorflow/core/kernels:list_kernels", # Depends on variant_op_registry.h + "//tensorflow/core/kernels:map_kernels", ], ) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index fd6a8ab1cf6..5139dd95e5d 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2937,6 +2937,45 @@ tf_kernel_library( "//third_party/eigen3", ], ) +cc_library( + name = "tensor_map", + srcs = ["tensor_map.cc"], + hdrs = ["tensor_map.h"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/framework:tensor_shape_proto_cc", + "//tensorflow/core/lib/core:refcount", + ], +) + +tf_kernel_library( + name = "map_kernels", + srcs = ["map_kernels.cc"], + deps = [ + ":concat_lib", + ":fill_functor", + ":tensor_map", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//third_party/eigen3", + ], +) + +tf_cc_tests( + name = "tensor_map_test", + size = "small", + srcs = [ + "tensor_map_test.cc" + ], + deps = [ + ":tensor_map", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "@com_google_absl//absl/strings", + ], +) tf_kernel_library( name = "fact_op", diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index bd6d880aec5..17793cdc0aa 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -14,11 +14,45 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/kernels/tensor_map.h" #include using namespace std; namespace tensorflow { +/*class EmptyTensorMap : public OpKernel { + public: + explicit EmptyTensorMap(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &element_dtype_)); + } + + void Compute(OpKernelContext* ctx) override { + const Tensor& max_num_elements_t = ctx->input(1); + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(max_num_elements_t.shape()), + errors::InvalidArgument( + "max_num_elements expected to be a scalar ", + "but got shape: ", max_num_elements_t.shape().DebugString())); + Tensor* result; + AllocatorAttributes attr; + attr.set_on_host(true); + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape{}, &result, attr)); + TensorMap empty; + empty.element_dtype = element_dtype_; + empty.max_num_elements = max_num_elements_t.scalar()(); + PartialTensorShape element_shape; + OP_REQUIRES_OK(ctx, TensorShapeFromTensor(ctx->input(0), &element_shape)); + empty.element_shape = element_shape; + result->scalar()() = std::move(empty); + } + + private: + DataType element_dtype_; +}; + +REGISTER_KERNEL_BUILDER(Name("EmptyTensorMap").Device(DEVICE_CPU), + EmptyTensorMap);*/ + class ZeroOutOp : public OpKernel { public: explicit ZeroOutOp(OpKernelConstruction* c) : OpKernel(c) {} @@ -31,7 +65,7 @@ class ZeroOutOp : public OpKernel { // Create an output tensor Tensor* output_tensor = NULL; - OP_REQUIRES_OK(context, context->allocate_output(0, input_tensor.shape(), + OP_REQUIRES_OK(c, c->allocate_output(0, input_tensor.shape(), &output_tensor)); auto output_flat = output_tensor->flat(); diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index bdad6d389b0..59c20c6d75f 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -20,6 +20,26 @@ limitations under the License. namespace tensorflow { namespace { + +REGISTER_OP("EmptyTensorMap") + .Input("element_shape: shape_type") + .Input("max_num_elements: int32") + .Output("handle: variant") + .Attr("element_dtype: type") + .Attr("shape_type: {int32, int64}") + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->Scalar()); + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle element_shape; + TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape( + 0, &element_shape)); + c->set_output_handle_shapes_and_types( + 0, std::vector{ + {element_shape, element_dtype}}); + return Status::OK(); + }); + REGISTER_OP("ZeroOut") .Input("to_zero: int32") .Output("zeroed: int32") diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 28cadf0a6df..7b9654886f3 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -20,19 +20,36 @@ from __future__ import print_function import numpy as np from tensorflow.python.platform import test +from absl.testing import parameterized +from tensorflow.python.framework import test_util + #try: # from tensorflow_zero_out.python.ops.zero_out_ops import zero_out #except ImportError: # from zero_out_ops import zero_out from tensorflow.python.ops import map_ops -class ZeroOutTest(test.TestCase): +class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): + """ + @parameterized.named_parameters(("NoMaxNumElements", None), + ("WithMaxNumElements", 2)) + @test_util.run_deprecated_v1 + def testEraseFromEmptyTensorMapFails(self, max_num_elements): + m = map_ops.empty_tensor_map( + element_dtype=dtypes.float32, + element_shape=[], + max_num_elements=max_num_elements) + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "Trying to erase from an empty map"): + m = map_ops.tensor_map_erase(l, element_dtype=dtypes.float32) + self.evaluate(l) + """ def testZeroOut(self): print("Hello World - Test") with self.test_session(): self.assertAllClose( - zero_out([[1, 2], [3, 4]]), np.array([[1, 0], [0, 0]])) + map_ops.zero_out([[1, 2], [3, 4]]), np.array([[1, 0], [0, 0]])) if __name__ == '__main__': diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 93f4cc0bdc8..4abd7f3f998 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -22,6 +22,7 @@ from tensorflow.python.framework import load_library from tensorflow.python.platform import resource_loader # go/tf-wildcard-import # pylint: disable=wildcard-import +from tensorflow.python.framework import ops from tensorflow.python.ops import gen_map_ops from tensorflow.python.ops.gen_map_ops import * @@ -29,8 +30,21 @@ from tensorflow.python.ops.gen_map_ops import * # resource_loader.get_path_to_datafile('_zero_out_ops.so')) #zero_out = zero_out_ops.zero_out +def empty_tensor_map(element_shape, + element_dtype, + max_num_elements=None, + name=None): + if max_num_elements is None: + max_num_elements = -1 + + return gen_map_ops.empty_tensor_map( + element_shape=_build_element_shape(element_shape), + element_dtype=element_dtype, + max_num_elements=max_num_elements, + name=name) + def zero_out(to_zero): - print("Hello World - PythonS Op") + print("Hello World - Python Op") return gen_map_ops.zero_out(to_zero) @ops.RegisterGradient("ZeroOut") From 477470d094b2e96eec8aecae91f9af699946ecb1 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 24 Jun 2020 18:35:02 +0000 Subject: [PATCH 0012/1017] finished test file --- tensorflow/c/kernels.cc | 27 ++++++ tensorflow/c/kernels.h | 4 + tensorflow/c/kernels/BUILD | 39 ++++++++ tensorflow/c/kernels/summary_op.cc | 56 ++++++------ tensorflow/c/kernels/summary_op_test.cc | 113 ++++++++++++++++++++---- tensorflow/c/tf_tensor.cc | 99 ++++----------------- tensorflow/c/tf_tensor.h | 5 ++ tensorflow/c/tf_tensor_internal.h | 3 + 8 files changed, 220 insertions(+), 126 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index a0ed0d9f245..e1ece820ab7 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -26,6 +26,9 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/lib/gtl/array_slice.h" + // This file forms the basis of a stable ABI for third-party kernel // implementations. It is crucial that changes to this file are made cautiously // and with a focus on maintaining both source and binary compatibility. @@ -260,3 +263,27 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, } return result; } + + +TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, + int64_t* dims, int num_dims, TF_Status* Status, TF_Tensor* tf_tensor_temp){ + auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); + // convert inputs to compatible types for API call + // tensorflow::DataType enum_of_dtype = tensorflow::EnumToDataType::v(); + // temp_tensor = Tensor(dtype, shape); + // tensorflow::TensorShape s(dimensions); + tensorflow::TensorShape shape; + for(int i = 0; i < num_dims; ++i){ + shape.AddDim(dims[i]); + } + tensorflow::Status allocation_status; + tensorflow::Tensor tensor_temp; + TF_TensorToTensor(tf_tensor_temp, &tensor_temp); + allocation_status = cc_ctx->allocate_temp(static_cast(dtype), shape, &tensor_temp); + tf_tensor_temp = TF_TensorFromTensor(tensor_temp, &allocation_status); + + + + + // Status allocation_status = cc_ctx->allocate_temp() +} diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index 084717c1d9e..9fcfdbeddc2 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -190,6 +190,10 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int64_t* dims, int num_dims, size_t len, TF_Status* status); +TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, + int64_t* dims, int num_dims, TF_Status* Status); + + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 770352c62c1..3ce53309841 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -24,6 +24,21 @@ tf_kernel_library( ], ) +tf_kernel_library( + name = "summary_op", + prefix = "summary_op", + deps = [ + "//tensorflow/c:kernels", + "//tensorflow/c:ops", + "//tensorflow/c:tf_datatype", + "//tensorflow/c:tf_status", + "//tensorflow/c:tf_tensor", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + ], +) + + tf_gen_op_libs( op_lib_names = ["bitcast"], deps = [ @@ -35,6 +50,17 @@ tf_gen_op_libs( ], ) +tf_gen_op_libs( + op_lib_names = ["summary"], + deps = [ + "//tensorflow/c:ops", + "//tensorflow/c:tf_datatype", + "//tensorflow/c:tf_status", + "//tensorflow/c:tf_tensor", + "//tensorflow/core:lib", + ], +) + tf_cc_test( name = "bitcast_op_test", srcs = ["bitcast_op_test.cc"], @@ -48,6 +74,19 @@ tf_cc_test( ], ) +tf_cc_test( + name = "summary_op_test", + srcs = ["summary_op_test.cc"], + deps = [ + ":summary_op", + ":summary_op_lib", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) # Changes to the Android srcs here should be replicated in # tensorflow/contrib/makefile/tf_op_files.txt. # diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 6921eb4fdaa..002de6fb6e8 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -29,13 +29,14 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/types.h" +#include // TODO: Copy over Summary Scalar Op Doc static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { // TODO: replace with a void* pointer type later - int a = 4; - return static_cast(&a); + void* ptr; + return ptr; } static void SummaryScalarOp_Delete(void* kernel) { @@ -60,48 +61,46 @@ static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { TF_Tensor* values; TF_Status* status = TF_NewStatus(); TF_GetInput(ctx, 0, &tags, status); - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while getting input"; if (TF_GetCode(status) == TF_OK){ TF_GetInput(ctx, 1, &values, status); } - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while getting input"; + if (TF_GetCode(status) == TF_OK) { if (!IsSameSize(tags, values)) { std::ostringstream err; - err << "tags and values not the same shape: "; + err << "tags and values not the same shape: " << TF_ShapeDebugString(tags) + << " != " << TF_ShapeDebugString(values); TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); } } + // Copy tag and string data into summary protobuf tensorflow::Summary s; if (TF_GetCode(status) == TF_OK) { - auto Ttags_array = static_cast(TF_TensorData(tags)); + // Convert tags and values tensor to array to access elements by index + auto tags_array = static_cast(TF_TensorData(tags)); auto values_array = static_cast(TF_TensorData(values)); for (int i = 0; i < TF_TensorElementCount(tags); ++i){ tensorflow::Summary::Value* v = s.add_value(); - TF_TString_Init(Ttags_array[i]); - v->set_tag(TF_TString_GetDataPointer(Ttags_array[i]), TF_TString_GetSize(Ttags_array[i])); + v->set_tag(TF_TString_GetDataPointer(&tags_array[i]), + TF_TString_GetSize(&tags_array[i])); v->set_simple_value(float(values_array[i])); } - - - // TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), 0, 0) - - // TF_Tensor* output = TF_AllocateTensor(k->output_data_type, dims, 0, - // TF_DataTypeSize(k->output_data_type)); - // if (TF_GetCode(status) == TF_OK) { - // TF_SetOutput(ctx, 0, output, status); - // } - // TF_DeleteTensor(output); + TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, + TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, + sizeof(TF_TString), status); + if (TF_GetCode(status) == TF_OK){ + SerializeToTString(s, static_cast + (TF_TensorData(summary_tensor))); + } + TF_DeleteTensor(summary_tensor); } - // if (TF_GetCode(status) != TF_OK) { - // TF_OpKernelContext_Failure(ctx, status); - // } - // TF_DeleteStatus(status); - // TF_DeleteTensor(tags); + if (TF_GetCode(status) != TF_OK) { + TF_OpKernelContext_Failure(ctx, status); + } + TF_DeleteStatus(status); + TF_DeleteTensor(tags); } template @@ -114,15 +113,14 @@ void RegisterSummaryScalarOpKernel() { TF_KernelBuilder_TypeConstraint(builder, "T", static_cast(tensorflow::DataTypeToEnum::v()), status); CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while adding type constraint"; - TF_RegisterKernelBuilder("SummaryScalar", builder, status); + TF_RegisterKernelBuilder("SummaryScalarOp", builder, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while registering Summary Scalar kernel"; } -// template // #if GOOGLE_CUDA // { // auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_GPU, -// &SummaryScalarOp_Create, &SummaryScalarOp_Compute, +// &SummaryScalarOp_Create, &SummaryScalarOp_Compute, // &SummaryScalarOp_Delete); // TF_RegisterKernelBuilder("SummaryScalar", builder, status); // CHECK_EQ(TF_OK, TF_GetCode(status)) @@ -138,7 +136,7 @@ void RegisterSummaryScalarOpKernel() { TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { - if (SHOULD_REGISTER_OP_KERNEL("SummaryScalar")) { + if (SHOULD_REGISTER_OP_KERNEL("SummaryScalarOp")) { RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc index fd6199abd6c..afc818fb7b5 100644 --- a/tensorflow/c/kernels/summary_op_test.cc +++ b/tensorflow/c/kernels/summary_op_test.cc @@ -22,6 +22,11 @@ limitations under the License. #include "tensorflow/core/framework/shape_inference.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/core/framework/summary.pb.h" +#include "tensorflow/core/platform/protobuf.h" +#include "tensorflow/c/tf_tensor.h" +#include "tensorflow/c/tf_tensor_internal.h" + #include #include #include @@ -36,17 +41,25 @@ class DummyDevice : public DeviceBase { } }; -void TestScalarSummaryOp(Tensor* tags, Tensor* values, error::Code expected_code) { +// Helper for comparing ouput and expected output +static void EXPECT_SummaryMatches(const Summary& actual, + const string& expected_str) { + Summary expected; + (protobuf::TextFormat::ParseFromString(expected_str, &expected)); + EXPECT_EQ(expected.DebugString(), actual.DebugString()); +} + + +void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, + error::Code expected_code) { + // initialize node used to fetch OpKernel Status status; NodeDef def; def.set_op("SummaryScalar"); - def.set_device(DEVICE_CPU); - AttrValue valuesTypeAttr; SetAttrValue(values->dtype(), &valuesTypeAttr); (*def.mutable_attr())["T"] = valuesTypeAttr; - def.add_input( strings::StrCat("input1: ", DataTypeString(tags->dtype()))); def.add_input( @@ -55,6 +68,8 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, error::Code expected_code std::unique_ptr kernel = CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status); ASSERT_TRUE(status.ok()) << status.ToString(); + + // initialize OpKernel parameters OpKernelContext::Params params; DummyDevice dummy_device(nullptr); params.device = &dummy_device; @@ -64,27 +79,93 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, error::Code expected_code inputs.emplace_back(values); params.inputs = &inputs; OpKernelContext ctx(¶ms, 1); + AllocatorAttributes alloc_attrs; + std::vector output_alloc_attrs({alloc_attrs}); + params.output_attr_array = output_alloc_attrs.data(); kernel->Compute(&ctx); - ASSERT_EQ(expected_code, ctx.status().code()); - if (expected_code == error::OK) { - ASSERT_EQ(true, false) - << ctx.mutable_output(0)->shape().DebugString(); + if (expected_code == error::OK){ + Summary summary; + ParseProtoUnlimited(&summary, ctx.mutable_output(0)->scalar()()); + EXPECT_SummaryMatches(summary, expected_summary); } } -TEST(ScalarSummaryOpTest, Test) { - int vectorSize = 2; +TEST(ScalarSummaryOpTest, SimpleFloat) { + int vectorSize = 3; Tensor tags(DT_STRING, {vectorSize}); Tensor values(DT_FLOAT, {vectorSize}); - for (int i = 0; i < vectorSize; ++i){ - values.vec()(i) = static_cast(i); - } - tags.vec()(0) = "tag 1"; - tags.vec()(1) = "tag 2"; - TestScalarSummaryOp(&tags, &values, error::INVALID_ARGUMENT); + tags.vec()(0) = "tag1"; + tags.vec()(1) = "tag2"; + tags.vec()(2) = "tag3"; + values.vec()(0) = 1.0f; + values.vec()(1) = -0.73f; + values.vec()(2) = 10000.0f; + TestScalarSummaryOp(&tags, &values, R"( + value { tag: 'tag1' simple_value: 1.0 } + value { tag: 'tag2' simple_value: -0.73} + value { tag: 'tag3' simple_value: 10000.0})", error::OK); } +TEST(ScalarSummaryOpTest, SimpleDouble) { + int vectorSize = 3; + Tensor tags(DT_STRING, {vectorSize}); + Tensor values(DT_DOUBLE, {vectorSize}); + tags.vec()(0) = "tag1"; + tags.vec()(1) = "tag2"; + tags.vec()(2) = "tag3"; + values.vec()(0) = 1.0; + values.vec()(1) = -0.73; + values.vec()(2) = 10000.0; + TestScalarSummaryOp(&tags, &values, R"( + value { tag: 'tag1' simple_value: 1.0 } + value { tag: 'tag2' simple_value: -0.73} + value { tag: 'tag3' simple_value: 10000.0})", error::OK); +} + +TEST(ScalarSummaryOpTest, SimpleHalf) { + int vectorSize = 3; + Tensor tags(DT_STRING, {vectorSize}); + Tensor values(DT_HALF, {vectorSize}); + tags.vec()(0) = "tag1"; + tags.vec()(1) = "tag2"; + tags.vec()(2) = "tag3"; + values.vec()(0) = static_cast(1.0); + values.vec()(1) = static_cast(-2.0); + values.vec()(2) = static_cast(10000.0); + TestScalarSummaryOp(&tags, &values, R"( + value { tag: 'tag1' simple_value: 1.0 } + value { tag: 'tag2' simple_value: -2.0} + value { tag: 'tag3' simple_value: 10000.0})", error::OK); +} + +TEST(ScalarSummaryOpTest, Error_WrongDimsTags) { + int vectorSize = 3; + Tensor tags(DT_STRING, {2, 1}); + Tensor values(DT_FLOAT, {2}); + tags.matrix()(0, 0) = "tag1"; + tags.matrix()(1, 0) = "tag2"; + values.vec()(0) = 1.0f; + values.vec()(1) = -2.0f; + TestScalarSummaryOp(&tags, &values, R"()", error::INVALID_ARGUMENT); +} + +TEST(ScalarSummaryOpTest, Error_WrongValuesTags) { + Tensor tags(DT_STRING, {2}); + Tensor values(DT_FLOAT, {2, 1}); + tags.vec()(0) = "tag1"; + tags.vec()(1) = "tag2"; + values.matrix()(0, 0) = 1.0f; + values.matrix()(1, 0) = -2.0f; + TestScalarSummaryOp(&tags, &values, R"()", error::INVALID_ARGUMENT); +} + +TEST(ScalarSummaryOpTest, IsRegistered){ + const OpRegistrationData* reg; + TF_CHECK_OK(OpRegistry::Global()->LookUp("SummaryScalar", ®)); +} + + PartialTensorShape S(std::initializer_list dims) { return PartialTensorShape(dims); diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc index 7e4d3bb4932..948a8700690 100644 --- a/tensorflow/c/tf_tensor.cc +++ b/tensorflow/c/tf_tensor.cc @@ -28,6 +28,8 @@ limitations under the License. #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/lib/core/coding.h" #include "tensorflow/core/platform/casts.h" +#include +#include using tensorflow::Status; using tensorflow::Tensor; @@ -180,6 +182,11 @@ void TF_TensorBitcastFrom(const TF_Tensor* from, TF_DataType type, Set_TF_Status_from_Status(status, cc_status); } +std::string TF_ShapeDebugString(const TF_Tensor* t){ + return tensorflow::down_cast(t->tensor) + ->ShapeDebugString(); +} + namespace tensorflow { void TensorInterface::Release() { delete this; } @@ -225,6 +232,10 @@ Status TensorInterface::BitcastFrom(const TensorInterface& from, DataType type, return tensor_.BitcastFrom(from.tensor_, type, s); } +std::string TensorInterface::ShapeDebugString() const { + return tensor_.shape().DebugString(); +} + } // namespace tensorflow // -------------------------------------------------------------------------- @@ -307,6 +318,7 @@ static TF_Tensor* EmptyTensor(TF_DataType dtype, namespace tensorflow { // Non-static for testing. + TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, Status* status) { *status = tensorflow::Status::OK(); if (!src.IsInitialized()) { @@ -334,58 +346,13 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, Status* status) { std::memcpy(TF_TensorData(t), str.c_str(), str.size()); return t; } - if (src.dtype() != tensorflow::DT_STRING) { - Tensor tensor; - if (!tensor.CopyFrom(src, src.shape())) { - return nullptr; - } - return new TF_Tensor{new tensorflow::TensorInterface(tensor)}; - } - // DT_STRING tensors require a copying since TF_Tensor.buffer expects a flatly - // encoded sequence of strings. - // Compute bytes needed for encoding. - size_t size = 0; - const auto& srcarray = src.flat(); - for (int i = 0; i < srcarray.size(); ++i) { - const string& s = srcarray(i); - // uint64 starting_offset, TF_StringEncode-d string. - size += sizeof(tensorflow::uint64) + TF_StringEncodedSize(s.size()); - } - - // Encode all strings. - char* base = new char[size]; - char* data_start = base + sizeof(tensorflow::uint64) * srcarray.size(); - char* dst = data_start; // Where next string is encoded. - size_t dst_len = size - static_cast(data_start - base); - tensorflow::uint64* offsets = reinterpret_cast(base); - for (int i = 0; i < srcarray.size(); ++i) { - *offsets = (dst - data_start); - offsets++; - const string& s = srcarray(i); - const size_t consumed = TF_StringEncodedSize(s.size()); - StringEncode(s.data(), s.size(), dst); - dst += consumed; - dst_len -= consumed; - } - if (dst != base + size) { - *status = InvalidArgument( - "invalid string tensor encoding (decoded ", (dst - base), - " bytes, but the tensor is encoded in ", size, " bytes"); - delete[] base; + Tensor tensor; + if (!tensor.CopyFrom(src, src.shape())) { return nullptr; } + return new TF_Tensor{new tensorflow::TensorInterface(tensor)}; - auto dims = src.shape().dim_sizes(); - std::vector dimvec(dims.size()); - for (size_t i = 0; i < dims.size(); ++i) { - dimvec[i] = dims[i]; - } - static_assert(sizeof(int64_t) == sizeof(tensorflow::int64), - "64-bit int types should match in size"); - return TF_NewTensor(TF_STRING, - reinterpret_cast(dimvec.data()), - dimvec.size(), base, size, DeleteArray, base); } Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst) { @@ -409,44 +376,14 @@ Status TensorInterface::ToTensor(tensorflow::Tensor* dst) const { } return Status::OK(); } - if (tensor_.dtype() != DT_STRING) { - *dst = tensor_; - return Status::OK(); - } - // TF_STRING tensors require copying since Tensor class expects a sequence of - // string objects. - const tensorflow::int64 num_elements = tensor_.NumElements(); - const char* input = reinterpret_cast(Data()); - const size_t src_size = ByteSize(); - if (static_cast(src_size / sizeof(tensorflow::uint64)) < - num_elements) { - return InvalidArgument( - "Malformed TF_STRING tensor; too short to hold number of elements"); - } - const char* data_start = input + sizeof(tensorflow::uint64) * num_elements; - const char* limit = input + src_size; - - *dst = tensorflow::Tensor(tensor_.dtype(), tensor_.shape()); - auto dstarray = dst->flat(); - for (tensorflow::int64 i = 0; i < num_elements; ++i) { - tensorflow::uint64 offset = - reinterpret_cast(input)[i]; - if (static_cast(offset) >= (limit - data_start)) { - return InvalidArgument("Malformed TF_STRING tensor; element ", i, - " out of range"); - } - size_t len; - const char* p; - const char* srcp = data_start + offset; - Status status = TF_StringDecode_Impl(srcp, limit - srcp, &p, &len); - if (!status.ok()) return status; - dstarray(i).assign(p, len); - } + *dst = tensor_; return Status::OK(); } + bool TensorInterface::IsAligned() const { return tensor_.IsAligned(); } } // namespace tensorflow bool TF_TensorIsAligned(const TF_Tensor* t) { return t->tensor->IsAligned(); } + diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index 7ed4a9f754e..3da4fef0f13 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -22,6 +22,9 @@ limitations under the License. #include "tensorflow/c/tf_datatype.h" #include "tensorflow/c/tf_status.h" +#include +#include + // Macro to control visibility of exported symbols in the shared library (.so, // .dylib, .dll). // This duplicates the TF_EXPORT macro definition in @@ -179,6 +182,8 @@ TF_CAPI_EXPORT extern size_t TF_StringEncodedSize(size_t len); // Returns bool iff this tensor is aligned. TF_CAPI_EXPORT extern bool TF_TensorIsAligned(const TF_Tensor*); +TF_CAPI_EXPORT extern std::string TF_ShapeDebugString(const TF_Tensor*); + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/tf_tensor_internal.h b/tensorflow/c/tf_tensor_internal.h index 7a896dc5d11..b3f44c71245 100644 --- a/tensorflow/c/tf_tensor_internal.h +++ b/tensorflow/c/tf_tensor_internal.h @@ -24,6 +24,8 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/platform/casts.h" +#include +#include // Internal structures used by the C API. These are likely to change and should // not be depended on. @@ -104,6 +106,7 @@ class TensorInterface : public AbstractTensorInterface { void* Data() const override; bool IsAligned() const override; bool CanMove() const override; + std::string ShapeDebugString() const; Status ToTensor(tensorflow::Tensor* dst) const; Status BitcastFrom(const TensorInterface& from, DataType type, From 07c5adfcfb7b7f62e8aeed9ec40535e9dda10976 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 26 Jun 2020 06:24:53 +0000 Subject: [PATCH 0013/1017] working absl hashable tensor --- tensorflow/core/framework/tensor.h | 30 ++++++++++++- tensorflow/core/kernels/tensor_map.cc | 65 +++++++-------------------- tensorflow/core/kernels/tensor_map.h | 14 ++++-- 3 files changed, 54 insertions(+), 55 deletions(-) diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h index 28eab3ab1e0..33c2338aeb4 100644 --- a/tensorflow/core/framework/tensor.h +++ b/tensorflow/core/framework/tensor.h @@ -319,6 +319,34 @@ class Tensor { return true; } + /// Hashable + // Equality operator. Needed for absl hashing. + friend bool operator==(const Tensor& lhs, const Tensor& rhs) { + return lhs.shape() == rhs.shape() && lhs.dtype() == rhs.dtype(); + } + + // Comparison operator. Needed for absl hashing. + /*friend bool operator<(const Tensor& lhs, const Tensor& rhs) { + return lhs.NumElements() < rhs.NumElements(); + }*/ + + // AbslHashValue() function, needed for absl hashing. + template + friend H AbslHashValue(H h, const Tensor& k) { + //int temp = k.NumElements(); + + uint8* d = (uint8*)(k.buf_->data()); + std::cout << "buffer " << d << std::endl; + size_t s = k.buf_->size(); + std::vector vec; + + for (int i=0; i < s; i++) { + vec.push_back(d[i]); + } + + return H::combine(std::move(h), vec); + } + /// \brief Slice this tensor along the 1st dimension. /// I.e., the returned tensor satisfies @@ -648,7 +676,7 @@ class Tensor { // buffer is one. bool RefCountIsOne() const; - private: + protected: void CheckType(DataType expected_dtype) const; void CheckTypeAndIsAligned(DataType expected_dtype) const; void CheckIsAlignedAndSingleElement() const; diff --git a/tensorflow/core/kernels/tensor_map.cc b/tensorflow/core/kernels/tensor_map.cc index a0b2c7a9f7a..3a20708933e 100644 --- a/tensorflow/core/kernels/tensor_map.cc +++ b/tensorflow/core/kernels/tensor_map.cc @@ -28,38 +28,20 @@ TensorMap::~TensorMap() { void TensorMap::Encode(VariantTensorData* data) const { data->set_type_name(TypeName()); - std::map::iterator map_it = tensors().begin(); - size_t i = 0; - std::vector invalid_indices; + absl::flat_hash_map::const_iterator map_it = tensors().begin(); while (map_it != tensors().end()) { Tensor k = map_it->first; Tensor v = map_it->second; - // k should also not be DT_RESOURCE or DT_VARIANT - if(k.dtype != DT_INVALID && v.dtype != DT_INVALID) { + // TODO: k should also not be DT_RESOURCE or DT_VARIANT + if(k.dtype() != DT_INVALID && v.dtype() != DT_INVALID) { *data->add_tensors() = k; *data->add_tensors() = v; - // not sure if this is the correct order - } - else { - invalid_indices.push_back(i); } } - /* - for (size_t i = 0; i < tensors().size(); i++) { - if (tensors().at(i).dtype() != DT_INVALID) { - *data->add_tensors() = tensors().at(i); - } else { - invalid_indices.push_back(i); - } - }*/ string metadata; // TODO(b/118838800): Add a proto for storing the metadata. // Metadata format: - // - core::PutVarint64(&metadata, static_cast(invalid_indices.size())); - for (size_t i : invalid_indices) { - core::PutVarint64(&metadata, static_cast(i)); - } + // core::PutVarint64(&metadata, static_cast(element_dtype)); core::PutVarint64(&metadata, static_cast(max_num_elements)); TensorShapeProto element_shape_proto; @@ -74,12 +56,11 @@ static Status TensorMapDeviceCopy( to->element_shape = from.element_shape; to->element_dtype = from.element_dtype; to->max_num_elements = from.max_num_elements; - //to->tensors().reserve(from.tensors().size()); for (const std::pair& p : from.tensors()) { - to->tensors().emplace(p); //why was it emplace t.dtype? - if (t.dtype() != DT_INVALID) { + to->tensors().emplace(p); //TODO: check valid dtype + //if (t.dtype() != DT_INVALID) { //TF_RETURN_IF_ERROR(copy(p, &to->tensors().back())); - } + //} } return Status::OK(); } @@ -102,33 +83,17 @@ bool TensorMap::Decode(const VariantTensorData& data) { data.get_metadata(&metadata); uint64 scratch; StringPiece iter(metadata); - std::vector invalid_indices; - core::GetVarint64(&iter, &scratch); - size_t num_invalid_tensors = static_cast(scratch); - invalid_indices.resize(num_invalid_tensors); - for (size_t i = 0; i < num_invalid_tensors; i++) { - core::GetVarint64(&iter, &scratch); - invalid_indices[i] = static_cast(scratch); - } - size_t total_num_tensors = data.tensors().size()/2 + num_invalid_tensors; - //tensors().reserve(total_num_tensors); - std::vector::iterator invalid_indices_it = invalid_indices.begin(); std::vector::const_iterator tensors_it = data.tensors().begin(); - for (size_t i = 0; i < total_num_tensors; i++) { - if (invalid_indices_it != invalid_indices.end() && - *invalid_indices_it == i) { - //no need to do invalid indices for a map - //tensors().emplace(Tensor(DT_INVALID),Tensor(DT_INVALID)); - invalid_indices_it++; - } else if (tensors_it != data.tensors().end()) { - // should assert that tensors_it + 1 is also not the end - tensors().emplace(*tensors_it,*++tensors_it); - tensors_it++; - } else { - // VariantTensorData is corrupted. + while (tensors_it != data.tensors().end()) + { + // should assert that tensors_it + 1 is also not the end + /*if (*tensors_it + 1 == data.tensors().end()) { return false; - } + }*/ + + tensors().emplace(*tensors_it,*++tensors_it); + tensors_it++; } core::GetVarint64(&iter, &scratch); diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index d8726bbecb4..17b2f63f2c9 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/variant.h" #include "tensorflow/core/framework/variant_tensor_data.h" #include "tensorflow/core/lib/core/refcount.h" +#include "absl/container/flat_hash_map.h" namespace tensorflow { @@ -123,8 +124,12 @@ class TensorMap { int max_num_elements = -1; // Access to the underlying tensor container. - std::map& tensors() { return tensors_->values_; } - const std::map& tensors() const { return tensors_->values_; } + absl::flat_hash_map& tensors() { return tensors_->values_; } + const absl::flat_hash_map& tensors() const { return tensors_->values_; } + + // Access to shape and element dtype + PartialTensorShape& shape() { return element_shape; } + DataType dtype() { return element_dtype; } // Get a new TensorList containing a copy of the underlying tensor container. TensorMap Copy() const { @@ -132,7 +137,7 @@ class TensorMap { out.element_shape = element_shape; out.element_dtype = element_dtype; out.max_num_elements = max_num_elements; - // This performs a copy of the std::map. + // This performs a copy of the absl::hashmap. out.tensors_->values_ = tensors_->values_; return out; } @@ -144,7 +149,8 @@ class TensorMap { private: class Tensors : public core::RefCounted { public: - std::map values_; + //std::unordered_map values_; + absl::flat_hash_map values_; }; Tensors* tensors_; }; From 5d8e3a41c57497588e8b22941cd480a5300c15d2 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 26 Jun 2020 07:16:28 +0000 Subject: [PATCH 0014/1017] tensor map tests --- tensorflow/core/kernels/tensor_map.h | 13 ++++++ tensorflow/core/kernels/tensor_map_test.cc | 54 ++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 tensorflow/core/kernels/tensor_map_test.cc diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index 17b2f63f2c9..2e8ebcd219d 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -142,6 +142,19 @@ class TensorMap { return out; } + bool insert(Tensor key, Tensor value) { + tensors_->values_.try_emplace(key, value); + return true; + } + + /*Tensor& lookup(Tensor key) { + return tensors_->values_.find(key); + }*/ + + bool erase(Tensor key) { + return tensors_->values_.erase(key); + } + // Is this TensorMap the only one with a reference to the underlying // container? bool RefCountIsOne() const { return tensors_->RefCountIsOne(); } diff --git a/tensorflow/core/kernels/tensor_map_test.cc b/tensorflow/core/kernels/tensor_map_test.cc new file mode 100644 index 00000000000..49f963bf950 --- /dev/null +++ b/tensorflow/core/kernels/tensor_map_test.cc @@ -0,0 +1,54 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/kernels/tensor_map.h" +#include "tensorflow/core/framework/tensor.h" +#include "absl/container/flat_hash_map.h" + +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/platform/test_benchmark.h" + +namespace tensorflow { + +namespace { + +TEST(TensorMapTest, Empty) { + TensorMap tm; + EXPECT_EQ(tm.tensors().size(), 0); + EXPECT_EQ(tm.tensors().begin(), tm.tensors().end()); +} + +TEST(TensorMap, Copy) { + TensorMap tm; + TensorMap tmc = tm.Copy(); + EXPECT_EQ(tm.dtype(),tmc.dtype()); + EXPECT_EQ(tm.tensors(),tmc.tensors()); +} + +TEST(TensorMap, Insert) { + EXPECT_EQ(1,1); + TensorMap tm; + Tensor k = Tensor(DT_INT64, TensorShape({1,1})); + Tensor v = Tensor(DT_INT64, TensorShape({2,3})); + tm.insert(k,v); + absl::flat_hash_map am; + am.try_emplace(k,v); + EXPECT_EQ(tm.tensors(), am); +} + +//TODO(kattian): test Lookup, Erase + +} // namespace + +} // namespace tensorflow From f040810ceb0caee9d8028c79865181e402238f89 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 26 Jun 2020 20:12:39 +0000 Subject: [PATCH 0015/1017] added allocated_temp function and tests --- tensorflow/c/kernels.cc | 30 +++---- tensorflow/c/kernels.h | 2 +- tensorflow/c/kernels_test.cc | 166 +++++++++++++++++++++++++++-------- 3 files changed, 145 insertions(+), 53 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index e1ece820ab7..864fd916b8b 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -264,26 +264,24 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, return result; } - +/* num_dims must equal the array size of dims */ TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, - int64_t* dims, int num_dims, TF_Status* Status, TF_Tensor* tf_tensor_temp){ + int64_t* dims, int num_dims, TF_Status* status){ auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); - // convert inputs to compatible types for API call - // tensorflow::DataType enum_of_dtype = tensorflow::EnumToDataType::v(); - // temp_tensor = Tensor(dtype, shape); - // tensorflow::TensorShape s(dimensions); + TF_SetStatus(status, TF_OK, ""); tensorflow::TensorShape shape; for(int i = 0; i < num_dims; ++i){ shape.AddDim(dims[i]); } - tensorflow::Status allocation_status; - tensorflow::Tensor tensor_temp; - TF_TensorToTensor(tf_tensor_temp, &tensor_temp); - allocation_status = cc_ctx->allocate_temp(static_cast(dtype), shape, &tensor_temp); - tf_tensor_temp = TF_TensorFromTensor(tensor_temp, &allocation_status); - - - - - // Status allocation_status = cc_ctx->allocate_temp() + tensorflow::Status s; + tensorflow::Tensor tensor_temp; + TF_Tensor* tf_tensor_temp; + s = cc_ctx->allocate_temp(static_cast(dtype), shape, &tensor_temp); + if (s.ok()){ + tf_tensor_temp = TF_TensorFromTensor(tensor_temp, &s); + } + if (s.ok()){ + ::tensorflow::Set_TF_Status_from_Status(status, s); + return tf_tensor_temp; + } } diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index 9fcfdbeddc2..9ae4f4b182d 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -191,7 +191,7 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, size_t len, TF_Status* status); TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, - int64_t* dims, int num_dims, TF_Status* Status); + int64_t* dims, int num_dims, TF_Status* status); #ifdef __cplusplus diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index 423302741de..738c1e12c80 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -360,6 +360,17 @@ class DeviceKernelOpTest : public OpsTestBase { #endif }; +// Helper function for tests that validates that the tensor has +// shape and type corresponding to dims and dtype. +void validate_tensor(TF_Tensor* tensor, int64_t* dims, int64_t num_dims, + TF_DataType dtype); + +// Helper function for tests that copies data of length +// tensor_size_bytes from values to tensor +template +void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, + TF_OpKernelContext* ctx); + REGISTER_OP("AllocateOutputOp1").Output("output1: float"); TEST_F(DeviceKernelOpTest, TestAllocateOutputSizeOne) { @@ -371,22 +382,11 @@ TEST_F(DeviceKernelOpTest, TestAllocateOutputSizeOne) { TF_Tensor* output = TF_AllocateOutput( /*context=*/ctx, /*index=*/0, /*dtype=*/TF_FLOAT, /*dims=*/&dim, /*num_dims=*/1, /*len=*/tensor_size_bytes, s); - EXPECT_EQ(TF_OK, TF_GetCode(s)); - EXPECT_EQ(TF_FLOAT, TF_TensorType(output)); - EXPECT_EQ(1, TF_NumDims(output)); - EXPECT_EQ(1, TF_Dim(output, 0)); - + validate_tensor(output, &dim, 1, TF_FLOAT); + // Set output to 3 - float* data = reinterpret_cast(TF_TensorData(output)); - float value = 3.0f; -#if GOOGLE_CUDA - OpKernelContext* cc_ctx = reinterpret_cast(ctx); - cc_ctx->eigen_gpu_device().memcpyHostToDevice(data, &value, - tensor_size_bytes); -#else - *data = value; -#endif - + float values[1] = {3.0f}; + set_tensor_data(output, values, tensor_size_bytes, ctx); TF_DeleteStatus(s); TF_DeleteTensor(output); }; @@ -409,12 +409,8 @@ TEST_F(DeviceKernelOpTest, TestAllocateEmptyOutput) { TF_Tensor* output = TF_AllocateOutput( /*context=*/ctx, /*index=*/0, /*dtype=*/TF_FLOAT, /*dims=*/&dim, /*num_dims=*/1, /*len=*/0, s); - EXPECT_EQ(TF_OK, TF_GetCode(s)); - EXPECT_EQ(TF_FLOAT, TF_TensorType(output)); - EXPECT_EQ(1, TF_NumDims(output)); - EXPECT_EQ(0, TF_Dim(output, 0)); - + validate_tensor(output, &dim, 1, TF_FLOAT); TF_DeleteStatus(s); TF_DeleteTensor(output); }; @@ -434,27 +430,16 @@ TEST_F(DeviceKernelOpTest, TestAllocateOutputSize2x3) { TF_Status* s = TF_NewStatus(); // Allocate 2x3 output int64_t dim[2] = {2, 3}; - size_t tensor_size_bytes = 6 * TF_DataTypeSize(TF_FLOAT); + size_t tensor_size_bytes = TF_DataTypeSize(TF_FLOAT) * 6; TF_Tensor* output = TF_AllocateOutput( /*context=*/ctx, /*index=*/0, /*dtype=*/TF_FLOAT, /*dims=*/dim, /*num_dims=*/2, /*len=*/tensor_size_bytes, s); EXPECT_EQ(TF_OK, TF_GetCode(s)); - EXPECT_EQ(TF_FLOAT, TF_TensorType(output)); - EXPECT_EQ(2, TF_NumDims(output)); - EXPECT_EQ(2, TF_Dim(output, 0)); - EXPECT_EQ(3, TF_Dim(output, 1)); + validate_tensor(output, dim, 2, TF_FLOAT); // Set output to [1 2 3 4 5 6] - void* data = TF_TensorData(output); - float value[6] = {1, 2, 3, 4, 5, 6}; -#if GOOGLE_CUDA - OpKernelContext* cc_ctx = reinterpret_cast(ctx); - cc_ctx->eigen_gpu_device().memcpyHostToDevice(data, value, - tensor_size_bytes); -#else - memcpy(data, value, tensor_size_bytes); -#endif - + float values[6] = {1, 2, 3, 4, 5, 6}; + set_tensor_data(output, values, tensor_size_bytes, ctx); TF_DeleteStatus(s); TF_DeleteTensor(output); }; @@ -466,4 +451,113 @@ TEST_F(DeviceKernelOpTest, TestAllocateOutputSize2x3) { EXPECT_EQ("Tensor", output->DebugString(100)); } -} // namespace tensorflow + +REGISTER_OP("AllocateTempOp1").Output("output1: float"); + +TEST_F(DeviceKernelOpTest, TestAllocateTempSizeOne) { + auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { + // Allocate output + TF_Status* s = TF_NewStatus(); + int64_t dim = 1; + TF_Tensor* output = TF_AllocateTemp( + /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, + /*num_dims=*/1, s); + size_t tensor_size_bytes = TF_DataTypeSize(TF_FLOAT); + EXPECT_EQ(TF_OK, TF_GetCode(s)); + validate_tensor(output, &dim, 1, TF_FLOAT); + + // Set output to 3 + float values[1] = {3.0f}; + set_tensor_data(output, values, tensor_size_bytes, ctx); + TF_SetOutput(ctx, 0, output, s); + TF_DeleteStatus(s); + TF_DeleteTensor(output); + }; + + SetupOp("AllocateTempOp1", "AllocateTemp1", my_compute_func); + + TF_ASSERT_OK(RunOpKernel()); + Tensor* output = GetOutput(0); + EXPECT_EQ("Tensor", + output->DebugString(100)); +} + +REGISTER_OP("AllocateTempOp0").Output("output1: float"); + +TEST_F(DeviceKernelOpTest, TestAllocateTempEmpty) { + auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { + TF_Status* s = TF_NewStatus(); + // Allocate empty output + int64_t dim = 0; + TF_Tensor* output = TF_AllocateTemp( + /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, + /*num_dims=*/1, s); + EXPECT_EQ(TF_OK, TF_GetCode(s)); + validate_tensor(output, &dim, 1, TF_FLOAT); + TF_SetOutput(ctx, 0, output, s); + TF_DeleteStatus(s); + TF_DeleteTensor(output); + }; + + SetupOp("AllocateTempOp0", "AllocateTemp0", my_compute_func); + + TF_ASSERT_OK(RunOpKernel()); + Tensor* output = GetOutput(0); + EXPECT_EQ("Tensor", + output->DebugString(100)); +} + +REGISTER_OP("AllocateTempOp2x3").Output("output1: float"); + +TEST_F(DeviceKernelOpTest, TestAllocateTempSize2x3) { + auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { + TF_Status* s = TF_NewStatus(); + size_t tensor_size_bytes = 6 * TF_DataTypeSize(TF_FLOAT); + // Allocate 2x3 output + int64_t dim[2] = {2, 3}; + TF_Tensor* output = TF_AllocateTemp( + /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/dim, + /*num_dims=*/2, s); + EXPECT_EQ(TF_OK, TF_GetCode(s)); + validate_tensor(output, dim, 2, TF_FLOAT); + + // Set output to [1 2 3 4 5 6] + void* data = TF_TensorData(output); + float values[6] = {1, 2, 3, 4, 5, 6}; + set_tensor_data(output, values, tensor_size_bytes, ctx); + TF_SetOutput(ctx, 0, output, s); + TF_DeleteStatus(s); + TF_DeleteTensor(output); + }; + + SetupOp("AllocateTempOp2x3", "AllocateTempOp2x3", my_compute_func); + + TF_ASSERT_OK(RunOpKernel()); + Tensor* output = GetOutput(0); + EXPECT_EQ("Tensor", + output->DebugString(100)); +} + +void validate_tensor(TF_Tensor* tensor, int64_t* dims, int64_t num_dims, + TF_DataType dtype){ + EXPECT_EQ(TF_FLOAT, TF_TensorType(tensor)); + EXPECT_EQ(num_dims, TF_NumDims(tensor)); + for(int i = 0; i < num_dims; ++i){ + EXPECT_EQ(dims[i], TF_Dim(tensor, i)); + } +} + +template +void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, + TF_OpKernelContext* ctx){ + T* data = reinterpret_cast(TF_TensorData(tensor)); +#if GOOGLE_CUDA + OpKernelContext* cc_ctx = reinterpret_cast(ctx); + cc_ctx->eigen_gpu_device().memcpyHostToDevice(data, values, + tensor_size_bytes); +#else + memcpy(data, values, tensor_size_bytes); +#endif +} + +} // namespace tensorflow \ No newline at end of file From 7fd7e0a3776a3bb5e5ee7d5bec99e498f19834a8 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 26 Jun 2020 22:00:05 +0000 Subject: [PATCH 0016/1017] added comments to allocate_temp --- tensorflow/c/kernels.cc | 1 - tensorflow/c/kernels.h | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 864fd916b8b..80b5234b52d 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -264,7 +264,6 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, return result; } -/* num_dims must equal the array size of dims */ TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, int64_t* dims, int num_dims, TF_Status* status){ auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index 9ae4f4b182d..e450511da3a 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -190,6 +190,11 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int64_t* dims, int num_dims, size_t len, TF_Status* status); +// Allocates a temporary Tensor of the specified type and shape. The +// Tensor must not be used after kernel construction is +// complete. + +// num_dims must equal the size of array dims TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, int64_t* dims, int num_dims, TF_Status* status); From b2c450ae75cfc07aee0adadf116f150c35cae3b5 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 26 Jun 2020 22:32:02 +0000 Subject: [PATCH 0017/1017] completed priority --- tensorflow/c/kernels.cc | 5 +++++ tensorflow/c/kernels.h | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 80b5234b52d..905219c6e16 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -100,6 +100,11 @@ void TF_KernelBuilder_HostMemory(TF_KernelBuilder* kernel_builder, kernel_builder->cc_builder->HostMemory(arg_name); } +void TF_KernelBuilder_Priority(TF_KernelBuilder* kernel_builder, + int32_t priority_number){ + kernel_builder->cc_builder->Priority(priority_number); +} + namespace tensorflow { namespace { diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index e450511da3a..b245dd8a7fc 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -107,6 +107,10 @@ TF_CAPI_EXPORT extern void TF_KernelBuilder_TypeConstraint( TF_CAPI_EXPORT extern void TF_KernelBuilder_HostMemory( TF_KernelBuilder* kernel_builder, const char* arg_name); +// Specify a priority number for this kernel. +TF_CAPI_EXPORT extern void TF_KernelBuilder_Priority( + TF_KernelBuilder* kernel_builder, int32_t priority_number); + // Register the given kernel builder with the TensorFlow runtime. If // registration fails, the given status will be populated. // From 412da53d657511bc5fafee4d6cfa34e61fc2d069 Mon Sep 17 00:00:00 2001 From: Patrik Laurell Date: Wed, 17 Jun 2020 16:43:22 +0200 Subject: [PATCH 0018/1017] Support float32->int16 and int16->int16 quantization in TFLu --- tensorflow/lite/micro/kernels/quantize.cc | 19 +++++- .../lite/micro/kernels/quantize_test.cc | 60 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/tensorflow/lite/micro/kernels/quantize.cc b/tensorflow/lite/micro/kernels/quantize.cc index b58a1cb368e..efaf2e583cd 100644 --- a/tensorflow/lite/micro/kernels/quantize.cc +++ b/tensorflow/lite/micro/kernels/quantize.cc @@ -66,11 +66,13 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE(context, input->type == kTfLiteFloat32 || input->type == kTfLiteInt16 || input->type == kTfLiteInt8); - TF_LITE_ENSURE(context, - output->type == kTfLiteUInt8 || output->type == kTfLiteInt8); + TF_LITE_ENSURE(context, output->type == kTfLiteUInt8 || + output->type == kTfLiteInt8 || + output->type == kTfLiteInt16); if ((input->type == kTfLiteInt16 || input->type == kTfLiteInt8) && - output->type == kTfLiteInt8) { + output->type == kTfLiteInt8 || + (input->type == kTfLiteInt16 && output->type == kTfLiteInt16)) { double effective_scale = static_cast(input->params.scale / output->params.scale); @@ -103,6 +105,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { op_params, GetTensorShape(input), GetTensorData(input), GetTensorShape(output), GetTensorData(output)); break; + case kTfLiteInt16: + reference_ops::AffineQuantize( + op_params, GetTensorShape(input), GetTensorData(input), + GetTensorShape(output), GetTensorData(output)); + return kTfLiteOk; default: TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", TfLiteTypeGetName(input->type), @@ -118,6 +125,12 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { data->output_shift, input->params.zero_point, output->params.zero_point, GetTensorData(output)); break; + case kTfLiteInt16: + reference_ops::Requantize( + GetTensorData(input), size, data->output_multiplier, + data->output_shift, input->params.zero_point, + output->params.zero_point, GetTensorData(output)); + return kTfLiteOk; default: TF_LITE_KERNEL_LOG(context, "Input %s, output %s not supported.", TfLiteTypeGetName(input->type), diff --git a/tensorflow/lite/micro/kernels/quantize_test.cc b/tensorflow/lite/micro/kernels/quantize_test.cc index b6f885d09e7..8e097429ca0 100644 --- a/tensorflow/lite/micro/kernels/quantize_test.cc +++ b/tensorflow/lite/micro/kernels/quantize_test.cc @@ -198,6 +198,32 @@ TF_LITE_MICRO_TEST(QuantizeOpTestInt8NoScale) { dims, values, dims, values, values_quantized, scale, zero_point, output); } +TF_LITE_MICRO_TEST(QuantizeOpTestInt16) { + const int length = 10; + const int dims[] = {2, 2, 5}; + const float values[] = {-63.5, -63, -62.5, -62, -61.5, + 62, 62.5, 63, 63.5, 64}; + const float scale = 0.5; + const int zero_point = -1; + int16_t output[length]; + int16_t values_quantized[length]; + tflite::testing::TestQuantizeFloat( + dims, values, dims, values, values_quantized, scale, zero_point, output); +} + +TF_LITE_MICRO_TEST(QuantizeOpTestInt16NoScale) { + const int length = 10; + const int dims[] = {2, 2, 5}; + const float values[] = {-128, -127, -126, -125, -124, + 123, 124, 125, 126, 127}; + const float scale = 1.0; + const int zero_point = 0; + int16_t output[length]; + int16_t values_quantized[length]; + tflite::testing::TestQuantizeFloat( + dims, values, dims, values, values_quantized, scale, zero_point, output); +} + TF_LITE_MICRO_TEST(QuantizeOpTestInt16toInt8) { const int length = 10; const int dims[] = {2, 2, 5}; @@ -215,6 +241,40 @@ TF_LITE_MICRO_TEST(QuantizeOpTestInt16toInt8) { output_zero_point, output_quantized); } +TF_LITE_MICRO_TEST(QuantizeOpTestInt16toInt16) { + const int length = 10; + const int dims[] = {2, 2, 5}; + const float values[] = {-64, -62, -60, -58, -56, 54, 56, 58, 60, 62}; + const float input_scale = 2.f; + const int input_zero_point = 0; + const float output_scale = 0.5; + const int output_zero_point = 32; + int16_t output_quantized[length]; + int16_t values_quantized[length]; + int16_t input_quantized[length]; + tflite::testing::TestRequantize(dims, values, input_quantized, input_scale, + input_zero_point, dims, values, + values_quantized, output_scale, + output_zero_point, output_quantized); +} + +TF_LITE_MICRO_TEST(QuantizeOpTestInt16toInt16NoZeroPoint) { + const int length = 10; + const int dims[] = {2, 2, 5}; + const float values[] = {-32, -31, -30, -29, -28, 27, 28, 29, 30, 31}; + const float input_scale = 1.f; + const int input_zero_point = 0; + const float output_scale = 0.5; + const int output_zero_point = 0; + int16_t output_quantized[length]; + int16_t values_quantized[length]; + int16_t input_quantized[length]; + tflite::testing::TestRequantize(dims, values, input_quantized, input_scale, + input_zero_point, dims, values, + values_quantized, output_scale, + output_zero_point, output_quantized); +} + TF_LITE_MICRO_TEST(QuantizeOpTestInt8toInt8) { const int length = 10; const int dims[] = {2, 2, 5}; From ce47a396ff795bdb6cf48eb53dbcba46cb51fa7d Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 30 Jun 2020 04:12:11 +0000 Subject: [PATCH 0019/1017] TensorKey class and TensorMap tests --- tensorflow/core/BUILD | 1 + tensorflow/core/framework/BUILD | 70 ++++++++++++++ tensorflow/core/framework/tensor.h | 30 +----- tensorflow/core/framework/tensor_key.h | 64 ++++++++++++ tensorflow/core/kernels/BUILD | 1 + tensorflow/core/kernels/tensor_map.cc | 13 +-- tensorflow/core/kernels/tensor_map.h | 31 ++++-- tensorflow/core/kernels/tensor_map_test.cc | 107 ++++++++++++++++++--- 8 files changed, 262 insertions(+), 55 deletions(-) create mode 100644 tensorflow/core/framework/tensor_key.h diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index d6e44bb36aa..cf5c1c4faed 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -494,6 +494,7 @@ tf_cuda_library( "//tensorflow/core/framework:shared_ptr_variant.h", "//tensorflow/core/framework:stats_aggregator.h", "//tensorflow/core/framework:tensor.h", + "//tensorflow/core/framework:tensor_key.h", "//tensorflow/core/framework:tensor_shape.h", "//tensorflow/core/framework:tensor_slice.h", "//tensorflow/core/framework:tensor_types.h", diff --git a/tensorflow/core/framework/BUILD b/tensorflow/core/framework/BUILD index 52f15dcb5c2..40f355cc082 100644 --- a/tensorflow/core/framework/BUILD +++ b/tensorflow/core/framework/BUILD @@ -211,6 +211,7 @@ filegroup( "shared_ptr_variant.h", "stats_aggregator.h", "tensor.h", + "tensor_key.h", "tensor_reference.h", "tensor_shape.h", "tensor_slice.h", @@ -762,6 +763,75 @@ tf_cuda_library( alwayslink = 1, ) +tf_cuda_library( + name = "tensor_key", + srcs = [ + "log_memory.cc", + "tensor.cc", + "typed_allocator.cc", + "types.cc", + "variant.cc", + "variant_op_registry.cc", + "variant_tensor_data.cc", + ], + hdrs = [ + "log_memory.h", + "register_types.h", + "tensor.h", + "tensor_key.h", + "typed_allocator.h", + "types.h", + "variant.h", + "variant_encode_decode.h", + "variant_op_registry.h", + "variant_tensor_data.h", + ], + visibility = [ + "//tensorflow/core:__pkg__", + "//tensorflow/core/util:__pkg__", + ], + deps = [ + ":allocation_description_proto_cc", + ":allocator", + ":bfloat16", + ":log_memory_proto_cc", + ":numeric_types", + ":resource_handle", + ":resource_handle_proto_cc", + ":tensor_description_proto_cc", + ":tensor_proto_cc", + ":tensor_shape", + ":tensor_types", + ":type_index", + ":type_traits", + ":types_proto_cc", + "//tensorflow/core/lib/core:coding", + "//tensorflow/core/lib/core:errors", + "//tensorflow/core/lib/core:refcount", + "//tensorflow/core/lib/core:status", + "//tensorflow/core/lib/core:stringpiece", + "//tensorflow/core/lib/gtl:array_slice", + "//tensorflow/core/lib/gtl:flatmap", + "//tensorflow/core/lib/gtl:inlined_vector", + "//tensorflow/core/lib/hash", + "//tensorflow/core/lib/strings:str_util", + "//tensorflow/core/lib/strings:strcat", + "//tensorflow/core/platform:abi", + "//tensorflow/core/platform:logging", + "//tensorflow/core/platform:macros", + "//tensorflow/core/platform:platform_port", + "//tensorflow/core/platform:protobuf", + "//tensorflow/core/platform:strcat", + "//tensorflow/core/platform:tensor_coding", + "//tensorflow/core/platform:types", + "//tensorflow/core/public:version", + "//third_party/eigen3", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + ], + alwayslink = 1, +) + cc_library( name = "shape_inference", srcs = ["shape_inference.cc"], diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h index 33c2338aeb4..28eab3ab1e0 100644 --- a/tensorflow/core/framework/tensor.h +++ b/tensorflow/core/framework/tensor.h @@ -319,34 +319,6 @@ class Tensor { return true; } - /// Hashable - // Equality operator. Needed for absl hashing. - friend bool operator==(const Tensor& lhs, const Tensor& rhs) { - return lhs.shape() == rhs.shape() && lhs.dtype() == rhs.dtype(); - } - - // Comparison operator. Needed for absl hashing. - /*friend bool operator<(const Tensor& lhs, const Tensor& rhs) { - return lhs.NumElements() < rhs.NumElements(); - }*/ - - // AbslHashValue() function, needed for absl hashing. - template - friend H AbslHashValue(H h, const Tensor& k) { - //int temp = k.NumElements(); - - uint8* d = (uint8*)(k.buf_->data()); - std::cout << "buffer " << d << std::endl; - size_t s = k.buf_->size(); - std::vector vec; - - for (int i=0; i < s; i++) { - vec.push_back(d[i]); - } - - return H::combine(std::move(h), vec); - } - /// \brief Slice this tensor along the 1st dimension. /// I.e., the returned tensor satisfies @@ -676,7 +648,7 @@ class Tensor { // buffer is one. bool RefCountIsOne() const; - protected: + private: void CheckType(DataType expected_dtype) const; void CheckTypeAndIsAligned(DataType expected_dtype) const; void CheckIsAlignedAndSingleElement() const; diff --git a/tensorflow/core/framework/tensor_key.h b/tensorflow/core/framework/tensor_key.h new file mode 100644 index 00000000000..8eff58b2dda --- /dev/null +++ b/tensorflow/core/framework/tensor_key.h @@ -0,0 +1,64 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/tensor.h" + +namespace tensorflow { + +class TensorKey : public Tensor { + public: + using Tensor::Tensor; + + TensorKey(const Tensor& t) : Tensor(t) {} + + // Equality operator. Needed for absl hashing. + friend bool operator==(const TensorKey& t1, const TensorKey& t2) { + if (t1.dtype() != t2.dtype() || t1.shape() != t2.shape()) { + return false; + } + if (DataTypeCanUseMemcpy(t1.dtype())) { + return t1.tensor_data() == t2.tensor_data(); + } + if (t1.dtype() == DT_STRING) { + const auto s1 = t1.unaligned_flat(); + const auto s2 = t2.unaligned_flat(); + for (int64 i = 0, n = t1.NumElements(); i < n; ++i) { + if (TF_PREDICT_FALSE(s1(i) != s2(i))) { + return false; + } + } + return true; + } + return false; + } + + friend bool operator!=(const TensorKey& t1, const TensorKey& t2) { + return !(t1==t2); + } + + // AbslHashValue() function, needed for absl hashing. + template + friend H AbslHashValue(H h, const TensorKey& k) { + uint8* d = (uint8*)(k.data()); + size_t s = k.AllocatedBytes(); + std::vector vec; + for (int i=0; i < s; i++) { + vec.push_back(d[i]); + } + return H::combine(std::move(h), s); + } +}; + +} //namespace tensorflow \ No newline at end of file diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 5139dd95e5d..eba435c6b25 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2971,6 +2971,7 @@ tf_cc_tests( ], deps = [ ":tensor_map", + "//tensorflow/core/framework:tensor_testutil", "//tensorflow/core:test", "//tensorflow/core:test_main", "@com_google_absl//absl/strings", diff --git a/tensorflow/core/kernels/tensor_map.cc b/tensorflow/core/kernels/tensor_map.cc index 3a20708933e..6245b01cf13 100644 --- a/tensorflow/core/kernels/tensor_map.cc +++ b/tensorflow/core/kernels/tensor_map.cc @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/tensor_map.h" +#include "tensorflow/core/kernels/tensor_map.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/framework/variant_op_registry.h" @@ -28,7 +28,7 @@ TensorMap::~TensorMap() { void TensorMap::Encode(VariantTensorData* data) const { data->set_type_name(TypeName()); - absl::flat_hash_map::const_iterator map_it = tensors().begin(); + absl::flat_hash_map::const_iterator map_it = tensors().begin(); while (map_it != tensors().end()) { Tensor k = map_it->first; Tensor v = map_it->second; @@ -56,7 +56,7 @@ static Status TensorMapDeviceCopy( to->element_shape = from.element_shape; to->element_dtype = from.element_dtype; to->max_num_elements = from.max_num_elements; - for (const std::pair& p : from.tensors()) { + for (const std::pair& p : from.tensors()) { to->tensors().emplace(p); //TODO: check valid dtype //if (t.dtype() != DT_INVALID) { //TF_RETURN_IF_ERROR(copy(p, &to->tensors().back())); @@ -85,14 +85,15 @@ bool TensorMap::Decode(const VariantTensorData& data) { StringPiece iter(metadata); std::vector::const_iterator tensors_it = data.tensors().begin(); + while (tensors_it != data.tensors().end()) { // should assert that tensors_it + 1 is also not the end - /*if (*tensors_it + 1 == data.tensors().end()) { + /*if (*std::next(tensors_it) == data.tensors().end()) { return false; }*/ - - tensors().emplace(*tensors_it,*++tensors_it); + TensorKey k = TensorKey(*tensors_it); // copy inefficient? + tensors().emplace(k,*++tensors_it); tensors_it++; } diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index 2e8ebcd219d..4e23fd59e51 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -18,6 +18,7 @@ limitations under the License. #include #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_key.h" #include "tensorflow/core/framework/variant.h" #include "tensorflow/core/framework/variant_tensor_data.h" #include "tensorflow/core/lib/core/refcount.h" @@ -124,8 +125,8 @@ class TensorMap { int max_num_elements = -1; // Access to the underlying tensor container. - absl::flat_hash_map& tensors() { return tensors_->values_; } - const absl::flat_hash_map& tensors() const { return tensors_->values_; } + absl::flat_hash_map& tensors() { return tensors_->values_; } + const absl::flat_hash_map& tensors() const { return tensors_->values_; } // Access to shape and element dtype PartialTensorShape& shape() { return element_shape; } @@ -142,19 +143,31 @@ class TensorMap { return out; } - bool insert(Tensor key, Tensor value) { - tensors_->values_.try_emplace(key, value); - return true; + // Insert key and value if the key does not already exist. + // Returns true if the insertion happens. + bool insert(TensorKey key, Tensor value) { + auto r = tensors_->values_.try_emplace(key, value); + return r.second; } - /*Tensor& lookup(Tensor key) { + // Lookup given key. Returns iterator to found key or end. + absl::flat_hash_map::iterator find(TensorKey key) { return tensors_->values_.find(key); - }*/ + } - bool erase(Tensor key) { + Tensor& operator[](TensorKey& k) { + return tensors_->values_[k]; + } + // Removes element with given key. Return size of removed element. + size_t erase(TensorKey key) { return tensors_->values_.erase(key); } + // Size returns the number of elements in the map + size_t size() { + return tensors_->values_.size(); + } + // Is this TensorMap the only one with a reference to the underlying // container? bool RefCountIsOne() const { return tensors_->RefCountIsOne(); } @@ -163,7 +176,7 @@ class TensorMap { class Tensors : public core::RefCounted { public: //std::unordered_map values_; - absl::flat_hash_map values_; + absl::flat_hash_map values_; }; Tensors* tensors_; }; diff --git a/tensorflow/core/kernels/tensor_map_test.cc b/tensorflow/core/kernels/tensor_map_test.cc index 49f963bf950..16726b780de 100644 --- a/tensorflow/core/kernels/tensor_map_test.cc +++ b/tensorflow/core/kernels/tensor_map_test.cc @@ -12,9 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ + #include "tensorflow/core/kernels/tensor_map.h" #include "tensorflow/core/framework/tensor.h" #include "absl/container/flat_hash_map.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/framework/variant.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" @@ -29,26 +32,108 @@ TEST(TensorMapTest, Empty) { EXPECT_EQ(tm.tensors().begin(), tm.tensors().end()); } -TEST(TensorMap, Copy) { - TensorMap tm; - TensorMap tmc = tm.Copy(); - EXPECT_EQ(tm.dtype(),tmc.dtype()); - EXPECT_EQ(tm.tensors(),tmc.tensors()); +TEST(TensorKeyTest, Equal) { + TensorKey k1 = Tensor(15); + TensorKey k2 = Tensor(15); + EXPECT_EQ(k1,k2); + + TensorKey k3 = Tensor(15); + TensorKey k4 = Tensor(37); + EXPECT_NE(k3,k4); } -TEST(TensorMap, Insert) { +TEST(TensorMapTest, Insert) { EXPECT_EQ(1,1); TensorMap tm; - Tensor k = Tensor(DT_INT64, TensorShape({1,1})); - Tensor v = Tensor(DT_INT64, TensorShape({2,3})); + TensorKey k = Tensor(11); + Tensor v = Tensor(22); tm.insert(k,v); - absl::flat_hash_map am; + absl::flat_hash_map am; am.try_emplace(k,v); - EXPECT_EQ(tm.tensors(), am); + + absl::flat_hash_map::iterator map_it = tm.tensors().begin(); + EXPECT_EQ(map_it->first, k); + test::ExpectTensorEqual(map_it->second, v); + map_it++; + EXPECT_EQ(map_it, tm.tensors().end()); } -//TODO(kattian): test Lookup, Erase +TEST(TensorMapTest, Lookup) { + TensorMap tm; + TensorKey k = Tensor(11); + Tensor v = Tensor(22); + tm.insert(k,v); + absl::flat_hash_map::iterator map_it = tm.find(k); + Tensor f = map_it->second; + EXPECT_EQ(map_it->first, k); + test::ExpectTensorEqual(f, v); +} + +TEST(TensorMapTest, Erase) { + TensorMap tm; + TensorKey k = Tensor(11); + Tensor v = Tensor(22); + tm.insert(k,v); + tm.erase(k); + EXPECT_EQ(tm.find(k), tm.tensors().end()); +} + +TEST(TensorMapTest, SameKeyInsert) { + TensorMap tm; + TensorKey k = Tensor(11); + Tensor v1 = Tensor(22); + Tensor v2 = Tensor(23); + bool b1 = tm.insert(k,v1); + bool b2 = tm.insert(k,v2); + EXPECT_EQ(b1, true); + EXPECT_EQ(b2, false); + absl::flat_hash_map::iterator map_it = tm.find(k); + EXPECT_EQ(map_it->first, k); + test::ExpectTensorEqual(map_it->second, v1); +} + +TEST(TensorMapTest, Replace) { + TensorMap tm; + TensorKey k = Tensor(11); + Tensor v1 = Tensor(22); + Tensor v2 = Tensor(23); + tm[k] = v2; + + absl::flat_hash_map::iterator map_it = tm.find(k); + EXPECT_EQ(map_it->first, k); + test::ExpectTensorEqual(map_it->second, v2); +} + +TEST(TensorMapTest, Copy) { + TensorMap tm; + TensorKey k = Tensor(11); + Tensor v = Tensor(22); + tm.insert(k,v); + TensorMap tmc = tm.Copy(); + EXPECT_EQ(tm.dtype(), tmc.dtype()); + EXPECT_EQ(tm.size(), tmc.size()); + EXPECT_NE(tm.find(k), tm.tensors().end()); + EXPECT_NE(tmc.find(k), tmc.tensors().end()); + EXPECT_EQ(tm.find(k)->first, tmc.find(k)->first); + test::ExpectTensorEqual(tm.find(k)->second, tmc.find(k)->second); +} + +/*TEST(TensorMapTest, EncodeDecode) { + TensorMap tm; + TensorKey k = Tensor(11); + Tensor v = Tensor(22); + tm.insert(k,v); + VariantTensorData* data; + TensorMap tmc = Decode(Encode(data)); + + EXPECT_EQ(tm.dtype(), tmc.dtype()); + EXPECT_EQ(tm.size(), tmc.size()); + EXPECT_NE(tm.find(k), tm.tensors().end()); + EXPECT_NE(tmc.find(k), tmc.tensors().end()); + EXPECT_EQ(tm.find(k)->first, tmc.find(k)->first); + test::ExpectTensorEqual(tm.find(k)->second, tmc.find(k)->second); +}*/ } // namespace } // namespace tensorflow From 0d5104e550d306259d09b920fddba76a70814f68 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 30 Jun 2020 07:22:28 +0000 Subject: [PATCH 0020/1017] EncodeDecode Test working --- tensorflow/core/kernels/tensor_map.cc | 1 + tensorflow/core/kernels/tensor_map_test.cc | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/kernels/tensor_map.cc b/tensorflow/core/kernels/tensor_map.cc index 6245b01cf13..abeaf92390e 100644 --- a/tensorflow/core/kernels/tensor_map.cc +++ b/tensorflow/core/kernels/tensor_map.cc @@ -37,6 +37,7 @@ void TensorMap::Encode(VariantTensorData* data) const { *data->add_tensors() = k; *data->add_tensors() = v; } + map_it++; } string metadata; // TODO(b/118838800): Add a proto for storing the metadata. diff --git a/tensorflow/core/kernels/tensor_map_test.cc b/tensorflow/core/kernels/tensor_map_test.cc index 16726b780de..b93171b4f70 100644 --- a/tensorflow/core/kernels/tensor_map_test.cc +++ b/tensorflow/core/kernels/tensor_map_test.cc @@ -119,21 +119,23 @@ TEST(TensorMapTest, Copy) { test::ExpectTensorEqual(tm.find(k)->second, tmc.find(k)->second); } -/*TEST(TensorMapTest, EncodeDecode) { +TEST(TensorMapTest, EncodeDecode) { TensorMap tm; TensorKey k = Tensor(11); Tensor v = Tensor(22); tm.insert(k,v); - VariantTensorData* data; - TensorMap tmc = Decode(Encode(data)); - + VariantTensorData data; + tm.Encode(&data); + TensorMap tmc; + tmc.Decode(data); + EXPECT_EQ(tm.dtype(), tmc.dtype()); EXPECT_EQ(tm.size(), tmc.size()); EXPECT_NE(tm.find(k), tm.tensors().end()); EXPECT_NE(tmc.find(k), tmc.tensors().end()); EXPECT_EQ(tm.find(k)->first, tmc.find(k)->first); test::ExpectTensorEqual(tm.find(k)->second, tmc.find(k)->second); -}*/ +} } // namespace } // namespace tensorflow From 9c424e45882e4fed1097c518053c3e721db6af1d Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 30 Jun 2020 18:40:25 +0000 Subject: [PATCH 0021/1017] build updates --- tensorflow/core/BUILD | 4 +- tensorflow/core/framework/BUILD | 69 --------------------------------- tensorflow/python/BUILD | 3 ++ 3 files changed, 5 insertions(+), 71 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index cf5c1c4faed..5b8eac9759c 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -494,7 +494,6 @@ tf_cuda_library( "//tensorflow/core/framework:shared_ptr_variant.h", "//tensorflow/core/framework:stats_aggregator.h", "//tensorflow/core/framework:tensor.h", - "//tensorflow/core/framework:tensor_key.h", "//tensorflow/core/framework:tensor_shape.h", "//tensorflow/core/framework:tensor_slice.h", "//tensorflow/core/framework:tensor_types.h", @@ -1122,7 +1121,8 @@ cc_library( "//tensorflow/core/kernels:dataset_ops", # Depends on grappler "//tensorflow/core/kernels:list_kernels", # Depends on variant_op_registry.h "//tensorflow/core/kernels:map_kernels", - ], + "//tensorflow/core/kernels:tensor_map", + ], ) tf_cuda_library( diff --git a/tensorflow/core/framework/BUILD b/tensorflow/core/framework/BUILD index 40f355cc082..e09022d5235 100644 --- a/tensorflow/core/framework/BUILD +++ b/tensorflow/core/framework/BUILD @@ -763,75 +763,6 @@ tf_cuda_library( alwayslink = 1, ) -tf_cuda_library( - name = "tensor_key", - srcs = [ - "log_memory.cc", - "tensor.cc", - "typed_allocator.cc", - "types.cc", - "variant.cc", - "variant_op_registry.cc", - "variant_tensor_data.cc", - ], - hdrs = [ - "log_memory.h", - "register_types.h", - "tensor.h", - "tensor_key.h", - "typed_allocator.h", - "types.h", - "variant.h", - "variant_encode_decode.h", - "variant_op_registry.h", - "variant_tensor_data.h", - ], - visibility = [ - "//tensorflow/core:__pkg__", - "//tensorflow/core/util:__pkg__", - ], - deps = [ - ":allocation_description_proto_cc", - ":allocator", - ":bfloat16", - ":log_memory_proto_cc", - ":numeric_types", - ":resource_handle", - ":resource_handle_proto_cc", - ":tensor_description_proto_cc", - ":tensor_proto_cc", - ":tensor_shape", - ":tensor_types", - ":type_index", - ":type_traits", - ":types_proto_cc", - "//tensorflow/core/lib/core:coding", - "//tensorflow/core/lib/core:errors", - "//tensorflow/core/lib/core:refcount", - "//tensorflow/core/lib/core:status", - "//tensorflow/core/lib/core:stringpiece", - "//tensorflow/core/lib/gtl:array_slice", - "//tensorflow/core/lib/gtl:flatmap", - "//tensorflow/core/lib/gtl:inlined_vector", - "//tensorflow/core/lib/hash", - "//tensorflow/core/lib/strings:str_util", - "//tensorflow/core/lib/strings:strcat", - "//tensorflow/core/platform:abi", - "//tensorflow/core/platform:logging", - "//tensorflow/core/platform:macros", - "//tensorflow/core/platform:platform_port", - "//tensorflow/core/platform:protobuf", - "//tensorflow/core/platform:strcat", - "//tensorflow/core/platform:tensor_coding", - "//tensorflow/core/platform:types", - "//tensorflow/core/public:version", - "//third_party/eigen3", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - ], - alwayslink = 1, -) - cc_library( name = "shape_inference", srcs = ["shape_inference.cc"], diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 3e59e61ae88..3c465252007 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -168,6 +168,7 @@ py_library( ":kernels", ":lib", ":list_ops", + ":map_ops", ":manip_ops", ":map_fn", ":math_ops", @@ -3521,6 +3522,7 @@ py_library( ":functional_ops_gen", ":gradients_util", ":list_ops", + ":map_ops", ":pywrap_tf_session", ":tensor_array_ops", ":tensor_shape", @@ -5002,6 +5004,7 @@ cuda_py_test( ":gradients", ":init_ops", ":list_ops", + ":map_ops", ":math_grad", ":math_ops", ":nn_grad", From 19632ba23d5478680c7951d0d20da270f0fa4162 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 30 Jun 2020 18:56:30 +0000 Subject: [PATCH 0022/1017] import guards --- tensorflow/core/BUILD | 2 +- tensorflow/core/framework/tensor_key.h | 6 +++++- tensorflow/core/kernels/tensor_map.h | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 5b8eac9759c..87d71d01935 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1122,7 +1122,7 @@ cc_library( "//tensorflow/core/kernels:list_kernels", # Depends on variant_op_registry.h "//tensorflow/core/kernels:map_kernels", "//tensorflow/core/kernels:tensor_map", - ], + ], ) tf_cuda_library( diff --git a/tensorflow/core/framework/tensor_key.h b/tensorflow/core/framework/tensor_key.h index 8eff58b2dda..14875de5918 100644 --- a/tensorflow/core/framework/tensor_key.h +++ b/tensorflow/core/framework/tensor_key.h @@ -12,6 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#ifndef TENSORFLOW_CORE_FRAMEWORK_TENSOR_KEY_H_ +#define TENSORFLOW_CORE_FRAMEWORK_TENSOR_KEY_H_ #include "tensorflow/core/framework/tensor.h" @@ -61,4 +63,6 @@ class TensorKey : public Tensor { } }; -} //namespace tensorflow \ No newline at end of file +} //namespace tensorflow + +#endif \ No newline at end of file diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index 4e23fd59e51..7da8283c655 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_TENSOR_LIST_H_ -#define TENSORFLOW_CORE_KERNELS_TENSOR_LIST_H_ +#ifndef TENSORFLOW_CORE_KERNELS_TENSOR_MAP_H_ +#define TENSORFLOW_CORE_KERNELS_TENSOR_MAP_H_ #include From b5bafaf4e77267085d1040ffbf78bf01ea31b6e1 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 30 Jun 2020 23:20:45 +0000 Subject: [PATCH 0023/1017] moved from tf_tstring to tstring for readability and fixed formatting --- ...-summary_op-needs-tstring-C-API-sync.patch | 377 ------------------ tensorflow/c/kernels/diff.patch | 0 tensorflow/c/kernels/ops/summary.cc | 30 +- tensorflow/c/kernels/summary_op.cc | 100 +++-- tensorflow/c/kernels/summary_op_test.cc | 30 +- tensorflow/core/kernels/summary_op.cc | 2 +- 6 files changed, 84 insertions(+), 455 deletions(-) delete mode 100644 tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch delete mode 100644 tensorflow/c/kernels/diff.patch diff --git a/tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch b/tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch deleted file mode 100644 index 856f4a554c3..00000000000 --- a/tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch +++ /dev/null @@ -1,377 +0,0 @@ -From 9134fbb13794865a45288d2e722ad47c362e0ae4 Mon Sep 17 00:00:00 2001 -From: Daniel Nguyen -Date: Thu, 18 Jun 2020 23:13:11 +0000 -Subject: [PATCH] summary_op needs tstring C API sync - ---- - tensorflow/c/kernels/diff.patch | 0 - tensorflow/c/kernels/ops/summary.cc | 70 ++++++++++ - tensorflow/c/kernels/summary_op.cc | 171 ++++++++++++++++++++++++ - tensorflow/c/kernels/summary_op_test.cc | 96 +++++++++++++ - 4 files changed, 337 insertions(+) - create mode 100644 tensorflow/c/kernels/diff.patch - create mode 100644 tensorflow/c/kernels/ops/summary.cc - create mode 100644 tensorflow/c/kernels/summary_op.cc - create mode 100644 tensorflow/c/kernels/summary_op_test.cc - -diff --git a/tensorflow/c/kernels/diff.patch b/tensorflow/c/kernels/diff.patch -new file mode 100644 -index 0000000000..e69de29bb2 -diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc -new file mode 100644 -index 0000000000..550a663d00 ---- /dev/null -+++ b/tensorflow/c/kernels/ops/summary.cc -@@ -0,0 +1,70 @@ -+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. -+ -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ -+ http://www.apache.org/licenses/LICENSE-2.0 -+ -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+#include -+#include -+ -+#include "tensorflow/c/ops.h" -+#include "tensorflow/core/framework/selective_registration.h" -+#include "tensorflow/core/platform/logging.h" -+#include "tensorflow/core/platform/macros.h" -+ -+ -+static void TF_ScalarSummary_shape_inference_fn(TF_ShapeInferenceContext* ctx, -+ TF_Status* status) { -+ TF_ShapeHandle* result = TF_NewShapeHandle(); -+ // TODO: what to do in the case of unknown input shape? -+ if (TF_GetCode(status) == TF_OK && -+ !TF_ShapeInferenceContextRankKnown(ctx, result)) { -+ TF_ShapeInferenceContextSetUnknownShape(ctx, status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while setting unknown shape function"; -+ TF_DeleteShapeHandle(result); -+ return; -+ } -+ // make shape handle a scalar value (empty shape) -+ if (TF_GetCode(status) == TF_OK) { -+ TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while setting shape function"; -+ } -+ TF_DeleteShapeHandle(result); -+} -+ -+void Register_ScalarSummaryOp() { -+ TF_Status* status = TF_NewStatus(); -+ -+ TF_OpDefinitionBuilder* op_builder = TF_NewOpDefinitionBuilder("SummaryScalar"); -+ TF_OpDefinitionBuilderAddInput(op_builder, "tags: string"); -+ TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); -+ TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); -+ TF_OpDefinitionBuilderAddAttr( -+ op_builder, -+ "T: realnumbertype"); -+ TF_OpDefinitionBuilderSetShapeInferenceFunction(op_builder, -+ &TF_ScalarSummary_shape_inference_fn); -+ -+ TF_RegisterOpDefinition(op_builder, status); -+ CHECK_EQ(TF_GetCode(status), TF_OK) -+ << "TF_ScalarSummary op registration failed: " << TF_Message(status); -+ TF_DeleteStatus(status); -+} -+ -+TF_ATTRIBUTE_UNUSED static bool SummaryScalarOpRegistered = []() { -+ if (SHOULD_REGISTER_OP("SummaryScalar")) { -+ Register_ScalarSummaryOp(); -+ } -+ return true; -+}(); -diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc -new file mode 100644 -index 0000000000..3a78d321d7 ---- /dev/null -+++ b/tensorflow/c/kernels/summary_op.cc -@@ -0,0 +1,171 @@ -+ -+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. -+ -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ -+ http://www.apache.org/licenses/LICENSE-2.0 -+ -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+#include -+ -+#include "tensorflow/c/kernels.h" -+#include "tensorflow/c/ops.h" -+#include "tensorflow/c/tf_tensor.h" -+#include "tensorflow/core/framework/common_shape_fns.h" -+#include "tensorflow/core/framework/op.h" -+#include "tensorflow/core/framework/selective_registration.h" -+#include "tensorflow/core/framework/shape_inference.h" -+#include "tensorflow/core/platform/macros.h" -+#include "tensorflow/core/framework/summary.pb.h" -+#include "tensorflow/core/platform/protobuf.h" -+#include "tensorflow/core/framework/register_types.h" -+ -+#include "tensorflow/core/framework/types.h" -+ -+// BitcastOp implements a bitcast kernel, creating an output tensor that shares -+// the same data buffer as the input but with a different shape and/or data -+// type. Its inputs are: -+// -+// * the input tensor -+// * an attribute named "T" containing the TF_DataType of the input tensor -+// * an attribute named "type" containing the TF_DataType of the output tensor -+// -+// Given an input tensor of shape [...], if the input DataType "T" is larger -+// than the output DataType "type", then the shape changes from [...] -+// to [..., sizeof(T)/sizeof(type)]. -+// -+// If "T" is smaller than "type", the operator requires that the rightmost -+// dimension be equal to sizeof(type)/sizeof(T). The shape then goes from -+// [..., sizeof(type)/sizeof(T)] to [...]. -+// -+// Bitcast is implemented as a low-level cast, so machines with different endian -+// orderings will give different results. -+ -+static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { -+ // TODO: replace with a void* pointer type later -+ int a = 4; -+ return static_cast(&a); -+} -+ -+static void SummaryScalarOp_Delete(void* kernel) { -+ return; -+} -+ -+bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ -+ if (TF_NumDims(tensor1) != TF_NumDims(tensor2)){ -+ return false; -+ } -+ for(int d = 0; d < TF_NumDims(tensor1); d++){ -+ if (TF_Dim(tensor1, d) != TF_Dim(tensor2, d)){ -+ return false; -+ } -+ } -+ return true; -+} -+ -+template -+static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { -+ TF_Tensor* tags; -+ TF_Tensor* values; -+ TF_Status* status = TF_NewStatus(); -+ TF_GetInput(ctx, 0, &tags, status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while getting input"; -+ if (TF_GetCode(status) == TF_OK){ -+ TF_GetInput(ctx, 1, &values, status); -+ } -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while getting input"; -+ if (TF_GetCode(status) == TF_OK) { -+ if (!IsSameSize(tags, values)) { -+ std::ostringstream err; -+ err << "tags and values not the same shape: "; -+ TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); -+ } -+ } -+ -+ tensorflow::Summary s; -+ if (TF_GetCode(status) == TF_OK) { -+ auto Ttags_array = static_cast(TF_TensorData(tags)); -+ auto values_array = static_cast(TF_TensorData(values)); -+ for (int i = 0; i < TF_TensorElementCount(tags); ++i){ -+ tensorflow::Summary::Value* v = s.add_value(); -+ TF_TString_Init(Ttags_array[i]); -+ v->set_tag(TF_TString_GetDataPointer(Ttags_array[i]), TF_TString_GetSize(Ttags_array[i])); -+ v->set_simple_value(float(values_array[i])); -+ } -+ -+ -+ // TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), 0, 0) -+ -+ // TF_Tensor* output = TF_AllocateTensor(k->output_data_type, dims, 0, -+ // TF_DataTypeSize(k->output_data_type)); -+ // if (TF_GetCode(status) == TF_OK) { -+ // TF_SetOutput(ctx, 0, output, status); -+ // } -+ // TF_DeleteTensor(output); -+ } -+ -+ // if (TF_GetCode(status) != TF_OK) { -+ // TF_OpKernelContext_Failure(ctx, status); -+ // } -+ // TF_DeleteStatus(status); -+ // TF_DeleteTensor(tags); -+} -+ -+template -+void RegisterSummaryScalarOpKernel() { -+ TF_Status* status = TF_NewStatus(); -+ { -+ auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_CPU, -+ &SummaryScalarOp_Create, &SummaryScalarOp_Compute, -+ &SummaryScalarOp_Delete); -+ TF_KernelBuilder_TypeConstraint(builder, "T", static_cast(tensorflow::DataTypeToEnum::v()), status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while adding type constraint"; -+ TF_RegisterKernelBuilder("SummaryScalar", builder, status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while registering Summary Scalar kernel"; -+ } -+// template -+// #if GOOGLE_CUDA -+// { -+// auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_GPU, -+// &SummaryScalarOp_Create, &SummaryScalarOp_Compute, -+// &SummaryScalarOp_Delete); -+// TF_RegisterKernelBuilder("SummaryScalar", builder, status); -+// CHECK_EQ(TF_OK, TF_GetCode(status)) -+// << "Error while registering CUDA SummaryScalar kernel"; -+// } -+// #endif -+ -+ TF_DeleteStatus(status); -+} -+ -+// A dummy static variable initialized by a lambda whose side-effect is to -+// register the bitcast kernel. -+ -+ -+TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { -+ if (SHOULD_REGISTER_OP_KERNEL("SummaryScalar")) { -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ } -+ return true; -+}(); -+ -diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc -new file mode 100644 -index 0000000000..fd6199abd6 ---- /dev/null -+++ b/tensorflow/c/kernels/summary_op_test.cc -@@ -0,0 +1,96 @@ -+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. -+ -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ -+ http://www.apache.org/licenses/LICENSE-2.0 -+ -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+#include "tensorflow/core/framework/attr_value.pb.h" -+#include "tensorflow/core/framework/attr_value_util.h" -+#include "tensorflow/core/framework/fake_input.h" -+#include "tensorflow/core/framework/node_def.pb.h" -+#include "tensorflow/core/framework/node_def_builder.h" -+#include "tensorflow/core/framework/op_kernel.h" -+#include "tensorflow/core/framework/shape_inference.h" -+#include "tensorflow/core/platform/test.h" -+ -+#include -+#include -+#include -+namespace tensorflow { -+namespace { -+ -+class DummyDevice : public DeviceBase { -+ public: -+ explicit DummyDevice(Env* env) : DeviceBase(env) {} -+ Allocator* GetAllocator(AllocatorAttributes /*attr*/) override { -+ return cpu_allocator(); -+ } -+}; -+ -+void TestScalarSummaryOp(Tensor* tags, Tensor* values, error::Code expected_code) { -+ Status status; -+ NodeDef def; -+ def.set_op("SummaryScalar"); -+ -+ def.set_device(DEVICE_CPU); -+ -+ AttrValue valuesTypeAttr; -+ SetAttrValue(values->dtype(), &valuesTypeAttr); -+ (*def.mutable_attr())["T"] = valuesTypeAttr; -+ -+ def.add_input( -+ strings::StrCat("input1: ", DataTypeString(tags->dtype()))); -+ def.add_input( -+ strings::StrCat("input2: ", DataTypeString(values->dtype()))); -+ -+ std::unique_ptr kernel = -+ CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status); -+ ASSERT_TRUE(status.ok()) << status.ToString(); -+ OpKernelContext::Params params; -+ DummyDevice dummy_device(nullptr); -+ params.device = &dummy_device; -+ params.op_kernel = kernel.get(); -+ gtl::InlinedVector inputs; -+ inputs.emplace_back(tags); -+ inputs.emplace_back(values); -+ params.inputs = &inputs; -+ OpKernelContext ctx(¶ms, 1); -+ kernel->Compute(&ctx); -+ -+ ASSERT_EQ(expected_code, ctx.status().code()); -+ if (expected_code == error::OK) { -+ ASSERT_EQ(true, false) -+ << ctx.mutable_output(0)->shape().DebugString(); -+ } -+} -+ -+TEST(ScalarSummaryOpTest, Test) { -+ int vectorSize = 2; -+ Tensor tags(DT_STRING, {vectorSize}); -+ Tensor values(DT_FLOAT, {vectorSize}); -+ for (int i = 0; i < vectorSize; ++i){ -+ values.vec()(i) = static_cast(i); -+ } -+ tags.vec()(0) = "tag 1"; -+ tags.vec()(1) = "tag 2"; -+ TestScalarSummaryOp(&tags, &values, error::INVALID_ARGUMENT); -+} -+ -+ -+PartialTensorShape S(std::initializer_list dims) { -+ return PartialTensorShape(dims); -+} -+ -+ -+ -+} // namespace -+} // namespace tensorflow --- -2.27.0.111.gc72c7da667-goog - diff --git a/tensorflow/c/kernels/diff.patch b/tensorflow/c/kernels/diff.patch deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc index 550a663d006..857ff6f29fa 100644 --- a/tensorflow/c/kernels/ops/summary.cc +++ b/tensorflow/c/kernels/ops/summary.cc @@ -13,32 +13,27 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include -#include - #include "tensorflow/c/ops.h" #include "tensorflow/core/framework/selective_registration.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" - -static void TF_ScalarSummary_shape_inference_fn(TF_ShapeInferenceContext* ctx, - TF_Status* status) { +static void scalar_summary_shape_inference_fn(TF_ShapeInferenceContext* ctx, + TF_Status* status) { TF_ShapeHandle* result = TF_NewShapeHandle(); - // TODO: what to do in the case of unknown input shape? if (TF_GetCode(status) == TF_OK && !TF_ShapeInferenceContextRankKnown(ctx, result)) { TF_ShapeInferenceContextSetUnknownShape(ctx, status); CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while setting unknown shape function"; + << "Error while setting unknown shape function"; TF_DeleteShapeHandle(result); return; } - // make shape handle a scalar value (empty shape) + // Make shape handle a scalar value (empty shape) if (TF_GetCode(status) == TF_OK) { TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while setting shape function"; + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while setting shape function"; } TF_DeleteShapeHandle(result); } @@ -46,19 +41,18 @@ static void TF_ScalarSummary_shape_inference_fn(TF_ShapeInferenceContext* ctx, void Register_ScalarSummaryOp() { TF_Status* status = TF_NewStatus(); - TF_OpDefinitionBuilder* op_builder = TF_NewOpDefinitionBuilder("SummaryScalar"); + TF_OpDefinitionBuilder* op_builder = + TF_NewOpDefinitionBuilder("SummaryScalar"); TF_OpDefinitionBuilderAddInput(op_builder, "tags: string"); - TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); + TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); - TF_OpDefinitionBuilderAddAttr( - op_builder, - "T: realnumbertype"); + TF_OpDefinitionBuilderAddAttr(op_builder, "T: realnumbertype"); TF_OpDefinitionBuilderSetShapeInferenceFunction(op_builder, - &TF_ScalarSummary_shape_inference_fn); + &scalar_summary_shape_inference_fn); TF_RegisterOpDefinition(op_builder, status); CHECK_EQ(TF_GetCode(status), TF_OK) - << "TF_ScalarSummary op registration failed: " << TF_Message(status); + << "ScalarSummary op registration failed: " << TF_Message(status); TF_DeleteStatus(status); } diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 002de6fb6e8..d2220670d74 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -27,14 +27,9 @@ limitations under the License. #include "tensorflow/core/framework/summary.pb.h" #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/framework/register_types.h" - #include "tensorflow/core/framework/types.h" -#include - -// TODO: Copy over Summary Scalar Op Doc static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { - // TODO: replace with a void* pointer type later void* ptr; return ptr; } @@ -43,17 +38,9 @@ static void SummaryScalarOp_Delete(void* kernel) { return; } -bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ - if (TF_NumDims(tensor1) != TF_NumDims(tensor2)){ - return false; - } - for(int d = 0; d < TF_NumDims(tensor1); d++){ - if (TF_Dim(tensor1, d) != TF_Dim(tensor2, d)){ - return false; - } - } - return true; -} +// Helper functions for compute method +bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2); +static tensorflow::string SingleTag(TF_Tensor* tags); template static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { @@ -61,35 +48,34 @@ static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { TF_Tensor* values; TF_Status* status = TF_NewStatus(); TF_GetInput(ctx, 0, &tags, status); - if (TF_GetCode(status) == TF_OK){ + if (TF_GetCode(status) == TF_OK) { TF_GetInput(ctx, 1, &values, status); } - if (TF_GetCode(status) == TF_OK) { if (!IsSameSize(tags, values)) { std::ostringstream err; - err << "tags and values not the same shape: " << TF_ShapeDebugString(tags) - << " != " << TF_ShapeDebugString(values); + err << "tags and values not the same shape: " + << TF_ShapeDebugString(tags) << " != " << TF_ShapeDebugString(values) + << SingleTag(tags); TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); } } - - // Copy tag and string data into summary protobuf + // Copy tag and string data into summary protobuf tensorflow::Summary s; if (TF_GetCode(status) == TF_OK) { // Convert tags and values tensor to array to access elements by index - auto tags_array = static_cast(TF_TensorData(tags)); + auto tags_array = static_cast(TF_TensorData(tags)); auto values_array = static_cast(TF_TensorData(values)); - for (int i = 0; i < TF_TensorElementCount(tags); ++i){ + // Copy tags and values into summary protobuf + for (int i = 0; i < TF_TensorElementCount(tags); ++i) { tensorflow::Summary::Value* v = s.add_value(); - v->set_tag(TF_TString_GetDataPointer(&tags_array[i]), - TF_TString_GetSize(&tags_array[i])); + v->set_tag(tags_array[i].data(), tags_array[i].size()); v->set_simple_value(float(values_array[i])); } TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, sizeof(TF_TString), status); - if (TF_GetCode(status) == TF_OK){ + if (TF_GetCode(status) == TF_OK) { SerializeToTString(s, static_cast (TF_TensorData(summary_tensor))); } @@ -101,40 +87,68 @@ static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { } TF_DeleteStatus(status); TF_DeleteTensor(tags); + TF_DeleteTensor(values); +} + +bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ + if (TF_NumDims(tensor1) != TF_NumDims(tensor2)) { + return false; + } + for (int d = 0; d < TF_NumDims(tensor1); d++) { + if (TF_Dim(tensor1, d) != TF_Dim(tensor2, d)) { + return false; + } + } + return true; +} + +static tensorflow::string SingleTag(TF_Tensor* tags){ + if (TF_TensorElementCount(tags) == 1) { + const char* single_tag = static_cast( + TF_TensorData(tags))->c_str(); + return tensorflow::strings::StrCat(" (tag '", single_tag, "')"); + } + else { + return ""; + } } template void RegisterSummaryScalarOpKernel() { TF_Status* status = TF_NewStatus(); { - auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_CPU, - &SummaryScalarOp_Create, &SummaryScalarOp_Compute, + auto* builder = TF_NewKernelBuilder("SummaryScalar", + tensorflow::DEVICE_CPU, + &SummaryScalarOp_Create, + &SummaryScalarOp_Compute, &SummaryScalarOp_Delete); - TF_KernelBuilder_TypeConstraint(builder, "T", static_cast(tensorflow::DataTypeToEnum::v()), status); + TF_KernelBuilder_TypeConstraint(builder, "T", + static_cast(tensorflow::DataTypeToEnum::v()), status); CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while adding type constraint"; TF_RegisterKernelBuilder("SummaryScalarOp", builder, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while registering Summary Scalar kernel"; } -// #if GOOGLE_CUDA -// { -// auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_GPU, -// &SummaryScalarOp_Create, &SummaryScalarOp_Compute, -// &SummaryScalarOp_Delete); -// TF_RegisterKernelBuilder("SummaryScalar", builder, status); -// CHECK_EQ(TF_OK, TF_GetCode(status)) -// << "Error while registering CUDA SummaryScalar kernel"; -// } -// #endif + +#if GOOGLE_CUDA + { + auto* builder = TF_NewKernelBuilder("SummaryScalar", + tensorflow::DEVICE_GPU, + &SummaryScalarOp_Create, + &SummaryScalarOp_Compute, + &SummaryScalarOp_Delete); + TF_RegisterKernelBuilder("SummaryScalar", builder, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while registering CUDA SummaryScalar kernel"; + } +#endif TF_DeleteStatus(status); } // A dummy static variable initialized by a lambda whose side-effect is to -// register the bitcast kernel. - - +// register the bitcast kernel. TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { if (SHOULD_REGISTER_OP_KERNEL("SummaryScalarOp")) { RegisterSummaryScalarOpKernel(); diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc index afc818fb7b5..5cf84453f80 100644 --- a/tensorflow/c/kernels/summary_op_test.cc +++ b/tensorflow/c/kernels/summary_op_test.cc @@ -21,15 +21,11 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/shape_inference.h" #include "tensorflow/core/platform/test.h" - #include "tensorflow/core/framework/summary.pb.h" #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/c/tf_tensor.h" #include "tensorflow/c/tf_tensor_internal.h" -#include -#include -#include namespace tensorflow { namespace { @@ -52,7 +48,7 @@ static void EXPECT_SummaryMatches(const Summary& actual, void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, error::Code expected_code) { - // initialize node used to fetch OpKernel + // Initialize node used to fetch OpKernel Status status; NodeDef def; def.set_op("SummaryScalar"); @@ -66,10 +62,11 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, strings::StrCat("input2: ", DataTypeString(values->dtype()))); std::unique_ptr kernel = - CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status); + CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, + nullptr, def, 1, &status); ASSERT_TRUE(status.ok()) << status.ToString(); - // initialize OpKernel parameters + // Initialize OpKernel parameters OpKernelContext::Params params; DummyDevice dummy_device(nullptr); params.device = &dummy_device; @@ -88,7 +85,7 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, Summary summary; ParseProtoUnlimited(&summary, ctx.mutable_output(0)->scalar()()); EXPECT_SummaryMatches(summary, expected_summary); - } + } } TEST(ScalarSummaryOpTest, SimpleFloat) { @@ -160,18 +157,19 @@ TEST(ScalarSummaryOpTest, Error_WrongValuesTags) { TestScalarSummaryOp(&tags, &values, R"()", error::INVALID_ARGUMENT); } +TEST(ScalarSummaryOpTest, Error_WrongWithSingleTag) { + Tensor tags(DT_STRING, {1}); + Tensor values(DT_FLOAT, {2, 1}); + tags.vec()(0) = "tag1"; + values.matrix()(0, 0) = 1.0f; + values.matrix()(1, 0) = -2.0f; + TestScalarSummaryOp(&tags, &values, R"()", error::INVALID_ARGUMENT); +} + TEST(ScalarSummaryOpTest, IsRegistered){ const OpRegistrationData* reg; TF_CHECK_OK(OpRegistry::Global()->LookUp("SummaryScalar", ®)); } - - -PartialTensorShape S(std::initializer_list dims) { - return PartialTensorShape(dims); -} - - - } // namespace } // namespace tensorflow diff --git a/tensorflow/core/kernels/summary_op.cc b/tensorflow/core/kernels/summary_op.cc index f4c91fc9ff1..64e8347dfc4 100644 --- a/tensorflow/core/kernels/summary_op.cc +++ b/tensorflow/core/kernels/summary_op.cc @@ -39,7 +39,7 @@ class SummaryScalarOp : public OpKernel { void Compute(OpKernelContext* c) override { const Tensor& tags = c->input(0); const Tensor& values = c->input(1); - + string tag = SingleTag(tags); OP_REQUIRES( c, tags.IsSameSize(values) || (TensorShapeUtils::IsScalar(tags.shape()) && From 2f861a79758545f3a6357814c79d7dcace920a3b Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 30 Jun 2020 23:51:04 +0000 Subject: [PATCH 0024/1017] merge with master --- tensorflow/c/kernels.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index da4d3414842..dd685583a4f 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -194,10 +194,6 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int64_t* dims, int num_dims, size_t len, TF_Status* status); -<<<<<<< HEAD -TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, - int64_t* dims, int num_dims, TF_Status* Status); -======= // Allocates a temporary Tensor of the specified type and shape. Devices // such as GPUs that enqueue Ops for lazy execution may retain references // to the temporary tensors after the Op's Compute method has run. @@ -205,8 +201,6 @@ TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF // num_dims must equal the size of array dims TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, int64_t* dims, int num_dims, TF_Status* status); ->>>>>>> e2187cd137f929eee06bd82a067564c4cdac2fa3 - #ifdef __cplusplus } /* end extern "C" */ From 1a5a6c3f8199d7a18f1e1a2b86c5638f38928fe4 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 1 Jul 2020 00:00:05 +0000 Subject: [PATCH 0025/1017] fixed indentation errors --- tensorflow/c/kernels.cc | 5 +++-- tensorflow/c/kernels.h | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 905219c6e16..525e19513e1 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -270,7 +270,7 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, } TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, - int64_t* dims, int num_dims, TF_Status* status){ + int64_t* dims, int num_dims, TF_Status* status){ auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); TF_SetStatus(status, TF_OK, ""); tensorflow::TensorShape shape; @@ -280,7 +280,8 @@ TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, tensorflow::Status s; tensorflow::Tensor tensor_temp; TF_Tensor* tf_tensor_temp; - s = cc_ctx->allocate_temp(static_cast(dtype), shape, &tensor_temp); + s = cc_ctx->allocate_temp(static_cast(dtype), shape, + &tensor_temp); if (s.ok()){ tf_tensor_temp = TF_TensorFromTensor(tensor_temp, &s); } diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index b245dd8a7fc..5c8f36cde9c 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -199,8 +199,11 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, // complete. // num_dims must equal the size of array dims -TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, - int64_t* dims, int num_dims, TF_Status* status); +TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, + TF_DataType dtype, + int64_t* dims, + int num_dims, + TF_Status* status); #ifdef __cplusplus From 15068b90e5709ae8a9d6fd0aba8ab9e78205f479 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 1 Jul 2020 06:16:13 +0000 Subject: [PATCH 0026/1017] latest --- tensorflow/core/framework/tensor_key.h | 2 +- tensorflow/core/kernels/BUILD | 1 + tensorflow/core/kernels/map_kernels.cc | 83 ++----------------------- tensorflow/core/kernels/map_kernels.h | 84 ++++++++++++++++++++++++++ tensorflow/core/kernels/tensor_map.h | 6 +- tensorflow/core/ops/map_ops.cc | 13 ++-- tensorflow/python/ops/map_ops.py | 9 ++- 7 files changed, 103 insertions(+), 95 deletions(-) create mode 100644 tensorflow/core/kernels/map_kernels.h diff --git a/tensorflow/core/framework/tensor_key.h b/tensorflow/core/framework/tensor_key.h index 14875de5918..aa6fe35181a 100644 --- a/tensorflow/core/framework/tensor_key.h +++ b/tensorflow/core/framework/tensor_key.h @@ -65,4 +65,4 @@ class TensorKey : public Tensor { } //namespace tensorflow -#endif \ No newline at end of file +#endif diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index eba435c6b25..85c2b9d175b 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2953,6 +2953,7 @@ cc_library( tf_kernel_library( name = "map_kernels", srcs = ["map_kernels.cc"], + hdrs = ["map_kernels.h"], deps = [ ":concat_lib", ":fill_functor", diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index 17793cdc0aa..91dfbe8c384 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -1,86 +1,11 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - +#include "tensorflow/core/kernels/map_kernels.h" #include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/kernels/tensor_map.h" -#include -using namespace std; +#include "tensorflow/core/framework/types.h" namespace tensorflow { - -/*class EmptyTensorMap : public OpKernel { - public: - explicit EmptyTensorMap(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &element_dtype_)); - } - - void Compute(OpKernelContext* ctx) override { - const Tensor& max_num_elements_t = ctx->input(1); - OP_REQUIRES( - ctx, TensorShapeUtils::IsScalar(max_num_elements_t.shape()), - errors::InvalidArgument( - "max_num_elements expected to be a scalar ", - "but got shape: ", max_num_elements_t.shape().DebugString())); - Tensor* result; - AllocatorAttributes attr; - attr.set_on_host(true); - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape{}, &result, attr)); - TensorMap empty; - empty.element_dtype = element_dtype_; - empty.max_num_elements = max_num_elements_t.scalar()(); - PartialTensorShape element_shape; - OP_REQUIRES_OK(ctx, TensorShapeFromTensor(ctx->input(0), &element_shape)); - empty.element_shape = element_shape; - result->scalar()() = std::move(empty); - } - - private: - DataType element_dtype_; -}; - REGISTER_KERNEL_BUILDER(Name("EmptyTensorMap").Device(DEVICE_CPU), - EmptyTensorMap);*/ - -class ZeroOutOp : public OpKernel { - public: - explicit ZeroOutOp(OpKernelConstruction* c) : OpKernel(c) {} - - void Compute(OpKernelContext* c) override { - cout << "Hello World - Op" << endl; - // Grab the input tensor - const Tensor& input_tensor = c->input(0); - auto input = input_tensor.flat(); - - // Create an output tensor - Tensor* output_tensor = NULL; - OP_REQUIRES_OK(c, c->allocate_output(0, input_tensor.shape(), - &output_tensor)); - auto output_flat = output_tensor->flat(); - - // Set all but the first element of the output tensor to 0 - const int N = input.size(); - for (int i=1; i 0) output_flat(0) = input(0); - } -}; + EmptyTensorMap); REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp); - -} // namespace tensorflow +} \ No newline at end of file diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h new file mode 100644 index 00000000000..b10eb8dedfb --- /dev/null +++ b/tensorflow/core/kernels/map_kernels.h @@ -0,0 +1,84 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/kernels/tensor_map.h" +#include +using namespace std; + +namespace tensorflow { + +class EmptyTensorMap : public OpKernel { + public: + explicit EmptyTensorMap(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &element_dtype_)); + } + + void Compute(OpKernelContext* ctx) override { + const Tensor& max_num_elements_t = ctx->input(1); + OP_REQUIRES( + ctx, TensorShapeUtils::IsScalar(max_num_elements_t.shape()), + errors::InvalidArgument( + "max_num_elements expected to be a scalar ", + "but got shape: ", max_num_elements_t.shape().DebugString())); + Tensor* result; + AllocatorAttributes attr; + attr.set_on_host(true); + g(ctx, ctx->allocate_output(0, TensorShape{}, &result, attr)); + TensorMap empty; + empty.element_dtype = element_dtype_; + empty.max_num_elements = max_num_elements_t.scalar()(); + PartialTensorShape element_shape; + //OP_REQUIRES_OK(ctx, TensorShapeFromTensor(ctx->input(0), &element_shape)); + empty.element_shape = element_shape; + result->scalar()() = std::move(empty); + } + + private: + DataType element_dtype_; +}; + + + +class ZeroOutOp : public OpKernel { + public: + explicit ZeroOutOp(OpKernelConstruction* c) : OpKernel(c) {} + + void Compute(OpKernelContext* c) override { + cout << "Hello World - Op" << endl; + // Grab the input tensor + const Tensor& input_tensor = c->input(0); + auto input = input_tensor.flat(); + + // Create an output tensor + Tensor* output_tensor = NULL; + OP_REQUIRES_OK(c, c->allocate_output(0, input_tensor.shape(), + &output_tensor)); + auto output_flat = output_tensor->flat(); + + // Set all but the first element of the output tensor to 0 + const int N = input.size(); + for (int i=1; i 0) output_flat(0) = input(0); + } +}; + + + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index 7da8283c655..c5993ec9300 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -40,10 +40,10 @@ namespace tensorflow { // a reference count. Modifying b.tensors() modifies a.tensors(). In this way, // TensorList should be considered similar to the tf::Tensor object. // -// In order to get a copy of the underlying list, use the Copy method: +// In order to get a copy of the underlying map, use the Copy method: // // TensorList b = a.Copy(); -// b.tensors().push_back(t); // This does not modify a.tensors(). +// b.tensors().insert(k, v); // This does not modify a.tensors(). // // Note that this is not a deep copy: the memory locations of the underlying // tensors will still point to the same locations of the corresponding tensors @@ -188,4 +188,4 @@ static_assert(Variant::CanInlineType(), #endif } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_TENSOR_LIST_H_ +#endif // TENSORFLOW_CORE_KERNELS_TENSOR_MAP_H_ diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index 59c20c6d75f..f1d7b291a70 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -20,23 +20,22 @@ limitations under the License. namespace tensorflow { namespace { - REGISTER_OP("EmptyTensorMap") - .Input("element_shape: shape_type") - .Input("max_num_elements: int32") +// .Input("element_shape: shape_type") +// .Input("max_num_elements: int32") .Output("handle: variant") - .Attr("element_dtype: type") - .Attr("shape_type: {int32, int64}") +// .Attr("element_dtype: type") +// .Attr("shape_type: {int32, int64}") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - DataType element_dtype; + /*DataType element_dtype; TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); shape_inference::ShapeHandle element_shape; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape( 0, &element_shape)); c->set_output_handle_shapes_and_types( 0, std::vector{ - {element_shape, element_dtype}}); + {element_shape, element_dtype}});*/ return Status::OK(); }); diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 4abd7f3f998..03acaa8fb72 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -37,11 +37,10 @@ def empty_tensor_map(element_shape, if max_num_elements is None: max_num_elements = -1 - return gen_map_ops.empty_tensor_map( - element_shape=_build_element_shape(element_shape), - element_dtype=element_dtype, - max_num_elements=max_num_elements, - name=name) + return gen_map_ops.empty_tensor_map(element_shape, + element_dtype, + max_num_elements, + name) def zero_out(to_zero): print("Hello World - Python Op") From f8933ade5959aa8d63cdfea31fb068323e564a7c Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 1 Jul 2020 06:32:58 +0000 Subject: [PATCH 0027/1017] latest --- tensorflow/core/kernels/map_kernels.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index b10eb8dedfb..1ed7c663a57 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -36,7 +36,7 @@ class EmptyTensorMap : public OpKernel { Tensor* result; AllocatorAttributes attr; attr.set_on_host(true); - g(ctx, ctx->allocate_output(0, TensorShape{}, &result, attr)); + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape{}, &result, attr)); TensorMap empty; empty.element_dtype = element_dtype_; empty.max_num_elements = max_num_elements_t.scalar()(); From 618ab4c18524f1b683155f30331172747c924e17 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 1 Jul 2020 23:51:24 +0000 Subject: [PATCH 0028/1017] rever to old --- tensorflow/c/kernels.cc | 4 +- tensorflow/c/kernels.h | 1 + tensorflow/c/kernels/summary_op.cc | 10 +-- tensorflow/c/kernels/summary_op_test.cc | 13 ++- tensorflow/c/tf_tensor.cc | 104 ++++++++++++++++++++---- tensorflow/c/tf_tensor.h | 5 -- tensorflow/c/tf_tensor_internal.h | 3 - 7 files changed, 96 insertions(+), 44 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index fc65895c8d5..505cb40f13c 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -27,9 +27,6 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/lib/gtl/array_slice.h" - // This file forms the basis of a stable ABI for third-party kernel // implementations. It is crucial that changes to this file are made cautiously // and with a focus on maintaining both source and binary compatibility. @@ -293,3 +290,4 @@ TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, } return tf_tensor_temp; } + diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index dd685583a4f..8ed3488988d 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -202,6 +202,7 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, int64_t* dims, int num_dims, TF_Status* status); + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index d2220670d74..23fd437af78 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -81,10 +81,6 @@ static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { } TF_DeleteTensor(summary_tensor); } - - if (TF_GetCode(status) != TF_OK) { - TF_OpKernelContext_Failure(ctx, status); - } TF_DeleteStatus(status); TF_DeleteTensor(tags); TF_DeleteTensor(values); @@ -126,11 +122,10 @@ void RegisterSummaryScalarOpKernel() { static_cast(tensorflow::DataTypeToEnum::v()), status); CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while adding type constraint"; - TF_RegisterKernelBuilder("SummaryScalarOp", builder, status); + TF_RegisterKernelBuilder("SummaryScalar", builder, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while registering Summary Scalar kernel"; } - #if GOOGLE_CUDA { auto* builder = TF_NewKernelBuilder("SummaryScalar", @@ -143,14 +138,13 @@ void RegisterSummaryScalarOpKernel() { << "Error while registering CUDA SummaryScalar kernel"; } #endif - TF_DeleteStatus(status); } // A dummy static variable initialized by a lambda whose side-effect is to // register the bitcast kernel. TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { - if (SHOULD_REGISTER_OP_KERNEL("SummaryScalarOp")) { + if (SHOULD_REGISTER_OP_KERNEL("SummaryScalar")) { RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc index 5cf84453f80..d8dbf622a55 100644 --- a/tensorflow/c/kernels/summary_op_test.cc +++ b/tensorflow/c/kernels/summary_op_test.cc @@ -52,10 +52,13 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, Status status; NodeDef def; def.set_op("SummaryScalar"); + def.set_device(DEVICE_CPU); + AttrValue valuesTypeAttr; SetAttrValue(values->dtype(), &valuesTypeAttr); (*def.mutable_attr())["T"] = valuesTypeAttr; + def.add_input( strings::StrCat("input1: ", DataTypeString(tags->dtype()))); def.add_input( @@ -65,8 +68,6 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status); ASSERT_TRUE(status.ok()) << status.ToString(); - - // Initialize OpKernel parameters OpKernelContext::Params params; DummyDevice dummy_device(nullptr); params.device = &dummy_device; @@ -76,10 +77,8 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, inputs.emplace_back(values); params.inputs = &inputs; OpKernelContext ctx(¶ms, 1); - AllocatorAttributes alloc_attrs; - std::vector output_alloc_attrs({alloc_attrs}); - params.output_attr_array = output_alloc_attrs.data(); kernel->Compute(&ctx); + ASSERT_EQ(expected_code, ctx.status().code()); if (expected_code == error::OK){ Summary summary; @@ -88,8 +87,8 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, } } -TEST(ScalarSummaryOpTest, SimpleFloat) { - int vectorSize = 3; +TEST(ScalarSummaryOpTest, Test) { + int vectorSize = 2; Tensor tags(DT_STRING, {vectorSize}); Tensor values(DT_FLOAT, {vectorSize}); tags.vec()(0) = "tag1"; diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc index 39f0176c0bf..34c91fc23dc 100644 --- a/tensorflow/c/tf_tensor.cc +++ b/tensorflow/c/tf_tensor.cc @@ -28,8 +28,6 @@ limitations under the License. #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/lib/core/coding.h" #include "tensorflow/core/platform/casts.h" -#include -#include using tensorflow::Status; using tensorflow::Tensor; @@ -182,11 +180,6 @@ void TF_TensorBitcastFrom(const TF_Tensor* from, TF_DataType type, Set_TF_Status_from_Status(status, cc_status); } -std::string TF_ShapeDebugString(const TF_Tensor* t){ - return tensorflow::down_cast(t->tensor) - ->ShapeDebugString(); -} - namespace tensorflow { void TensorInterface::Release() { delete this; } @@ -232,10 +225,6 @@ Status TensorInterface::BitcastFrom(const TensorInterface& from, DataType type, return tensor_.BitcastFrom(from.tensor_, type, s); } -std::string TensorInterface::ShapeDebugString() const { - return tensor_.shape().DebugString(); -} - } // namespace tensorflow // -------------------------------------------------------------------------- @@ -267,7 +256,6 @@ static TF_Tensor* EmptyTensor(TF_DataType dtype, namespace tensorflow { // Non-static for testing. - TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, Status* status) { *status = tensorflow::Status::OK(); if (!src.IsInitialized()) { @@ -295,12 +283,62 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, Status* status) { std::memcpy(TF_TensorData(t), str.c_str(), str.size()); return t; } + if (src.dtype() != tensorflow::DT_STRING) { + Tensor tensor; + if (!tensor.CopyFrom(src, src.shape())) { + return nullptr; + } + return new TF_Tensor{new tensorflow::TensorInterface(tensor)}; + } + // DT_STRING tensors require a copying since TF_Tensor.buffer expects a flatly + // encoded sequence of strings. - Tensor tensor; - if (!tensor.CopyFrom(src, src.shape())) { + // Compute bytes needed for encoding. + size_t size = 0; + const auto& srcarray = src.flat(); + for (int i = 0; i < srcarray.size(); ++i) { + const string& s = srcarray(i); + // uint64 starting_offset, TF_StringEncode-d string. + size += sizeof(tensorflow::uint64) + TF_StringEncodedSize(s.size()); + } + + // Encode all strings. + char* base = new char[size]; + char* data_start = base + sizeof(tensorflow::uint64) * srcarray.size(); + char* dst = data_start; // Where next string is encoded. + size_t dst_len = size - static_cast(data_start - base); + tensorflow::uint64* offsets = reinterpret_cast(base); + for (int i = 0; i < srcarray.size(); ++i) { + *offsets = (dst - data_start); + offsets++; + const string& s = srcarray(i); + const size_t consumed = TF_StringEncodedSize(s.size()); + StringEncode(s.data(), s.size(), dst); + dst += consumed; + dst_len -= consumed; + } + if (dst != base + size) { + *status = InvalidArgument( + "invalid string tensor encoding (decoded ", (dst - base), + " bytes, but the tensor is encoded in ", size, " bytes"); + delete[] base; return nullptr; } - return new TF_Tensor{new tensorflow::TensorInterface(tensor)}; +// <<<<<<< HEAD +// return new TF_Tensor{new tensorflow::TensorInterface(tensor)}; +// ======= + + auto dims = src.shape().dim_sizes(); + std::vector dimvec(dims.size()); + for (size_t i = 0; i < dims.size(); ++i) { + dimvec[i] = dims[i]; + } + static_assert(sizeof(int64_t) == sizeof(tensorflow::int64), + "64-bit int types should match in size"); + return TF_NewTensor(TF_STRING, + reinterpret_cast(dimvec.data()), + dimvec.size(), base, size, DeleteArray, base); +// >>>>>>> parent of 477470d094... finished test file } Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst) { @@ -324,14 +362,44 @@ Status TensorInterface::ToTensor(tensorflow::Tensor* dst) const { } return Status::OK(); } - *dst = tensor_; + if (tensor_.dtype() != DT_STRING) { + *dst = tensor_; + return Status::OK(); + } + // TF_STRING tensors require copying since Tensor class expects a sequence of + // string objects. + const tensorflow::int64 num_elements = tensor_.NumElements(); + const char* input = reinterpret_cast(Data()); + const size_t src_size = ByteSize(); + if (static_cast(src_size / sizeof(tensorflow::uint64)) < + num_elements) { + return InvalidArgument( + "Malformed TF_STRING tensor; too short to hold number of elements"); + } + const char* data_start = input + sizeof(tensorflow::uint64) * num_elements; + const char* limit = input + src_size; + + *dst = tensorflow::Tensor(tensor_.dtype(), tensor_.shape()); + auto dstarray = dst->flat(); + for (tensorflow::int64 i = 0; i < num_elements; ++i) { + tensorflow::uint64 offset = + reinterpret_cast(input)[i]; + if (static_cast(offset) >= (limit - data_start)) { + return InvalidArgument("Malformed TF_STRING tensor; element ", i, + " out of range"); + } + size_t len; + const char* p; + const char* srcp = data_start + offset; + Status status = TF_StringDecode_Impl(srcp, limit - srcp, &p, &len); + if (!status.ok()) return status; + dstarray(i).assign(p, len); + } return Status::OK(); } - bool TensorInterface::IsAligned() const { return tensor_.IsAligned(); } } // namespace tensorflow bool TF_TensorIsAligned(const TF_Tensor* t) { return t->tensor->IsAligned(); } - diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index f788c0828a8..acdf053e63a 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -22,9 +22,6 @@ limitations under the License. #include "tensorflow/c/tf_datatype.h" #include "tensorflow/c/tf_status.h" -#include -#include - // Macro to control visibility of exported symbols in the shared library (.so, // .dylib, .dll). // This duplicates the TF_EXPORT macro definition in @@ -154,8 +151,6 @@ TF_CAPI_EXPORT extern void TF_TensorBitcastFrom(const TF_Tensor* from, // Returns bool iff this tensor is aligned. TF_CAPI_EXPORT extern bool TF_TensorIsAligned(const TF_Tensor*); -TF_CAPI_EXPORT extern std::string TF_ShapeDebugString(const TF_Tensor*); - #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/tf_tensor_internal.h b/tensorflow/c/tf_tensor_internal.h index b3f44c71245..7a896dc5d11 100644 --- a/tensorflow/c/tf_tensor_internal.h +++ b/tensorflow/c/tf_tensor_internal.h @@ -24,8 +24,6 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/platform/casts.h" -#include -#include // Internal structures used by the C API. These are likely to change and should // not be depended on. @@ -106,7 +104,6 @@ class TensorInterface : public AbstractTensorInterface { void* Data() const override; bool IsAligned() const override; bool CanMove() const override; - std::string ShapeDebugString() const; Status ToTensor(tensorflow::Tensor* dst) const; Status BitcastFrom(const TensorInterface& from, DataType type, From 9a326299119bb12b177960f9c1a407663a8d7223 Mon Sep 17 00:00:00 2001 From: qhduan Date: Thu, 2 Jul 2020 05:07:19 +0000 Subject: [PATCH 0029/1017] Fix the function name in debugging.md --- tensorflow/python/autograph/g3doc/reference/debugging.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/autograph/g3doc/reference/debugging.md b/tensorflow/python/autograph/g3doc/reference/debugging.md index 2c2a96cec86..fb75846b14b 100644 --- a/tensorflow/python/autograph/g3doc/reference/debugging.md +++ b/tensorflow/python/autograph/g3doc/reference/debugging.md @@ -21,10 +21,10 @@ Note: Python debugging can only be used to step through the code during graph construction time (or tracing time in the case of `tf.function`). To debug TensorFlow execution, use Eager execution. -### Debugging `tf.function`: `tf.config.experimental_execute_functions_eagerly` +### Debugging `tf.function`: `tf.config.experimental_run_functions_eagerly` When using `@tf.function`, you can temporarily toggle graph execution -by using `tf.config.experimental_execute_functions_eagerly`. This will +by using `tf.config.experimental_run_functions_eagerly`. This will effectively run the annotated code eagerly, without transformation. Since AutoGraph has semantics consistent with Eager, it's an effective way to debug the code step-by-step. @@ -58,7 +58,7 @@ f(1) 14 ... ``` -Adding a call to `tf.config.experimental_execute_functions_eagerly` before +Adding a call to `tf.config.experimental_run_functions_eagerly` before executing the function will land the debugger in the original code instead: ``` From d69076f6e6575886055845125003472e18882916 Mon Sep 17 00:00:00 2001 From: qhduan Date: Thu, 2 Jul 2020 05:07:19 +0000 Subject: [PATCH 0030/1017] Fix the function name in debugging.md --- tensorflow/python/autograph/g3doc/reference/debugging.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/autograph/g3doc/reference/debugging.md b/tensorflow/python/autograph/g3doc/reference/debugging.md index 2c2a96cec86..fb75846b14b 100644 --- a/tensorflow/python/autograph/g3doc/reference/debugging.md +++ b/tensorflow/python/autograph/g3doc/reference/debugging.md @@ -21,10 +21,10 @@ Note: Python debugging can only be used to step through the code during graph construction time (or tracing time in the case of `tf.function`). To debug TensorFlow execution, use Eager execution. -### Debugging `tf.function`: `tf.config.experimental_execute_functions_eagerly` +### Debugging `tf.function`: `tf.config.experimental_run_functions_eagerly` When using `@tf.function`, you can temporarily toggle graph execution -by using `tf.config.experimental_execute_functions_eagerly`. This will +by using `tf.config.experimental_run_functions_eagerly`. This will effectively run the annotated code eagerly, without transformation. Since AutoGraph has semantics consistent with Eager, it's an effective way to debug the code step-by-step. @@ -58,7 +58,7 @@ f(1) 14 ... ``` -Adding a call to `tf.config.experimental_execute_functions_eagerly` before +Adding a call to `tf.config.experimental_run_functions_eagerly` before executing the function will land the debugger in the original code instead: ``` From 0b501041f648289277e9e04529f7f1a8bb36c03f Mon Sep 17 00:00:00 2001 From: ShengYang1 Date: Fri, 3 Jul 2020 08:53:53 +0800 Subject: [PATCH 0031/1017] Cmp with cast --- tensorflow/core/framework/tensor_testutil.cc | 3 + tensorflow/core/grappler/op_types.cc | 6 ++ tensorflow/core/grappler/op_types.h | 1 + .../core/grappler/optimizers/remapper.cc | 87 +++++++++++++++++++ .../core/grappler/optimizers/remapper_test.cc | 59 +++++++++++++ .../core/grappler/utils/grappler_test.h | 2 +- .../core/kernels/cwise_op_equal_to_1.cc | 2 + tensorflow/core/kernels/cwise_op_greater.cc | 2 + .../core/kernels/cwise_op_greater_equal.cc | 2 + tensorflow/core/kernels/cwise_op_less.cc | 2 + .../core/kernels/cwise_op_less_equal.cc | 2 + .../core/kernels/cwise_op_not_equal_to_1.cc | 2 + tensorflow/core/kernels/cwise_ops.h | 28 +++++- tensorflow/core/kernels/cwise_ops_test.cc | 74 ++++++++-------- tensorflow/core/ops/math_ops.cc | 43 +++++++++ 15 files changed, 278 insertions(+), 37 deletions(-) diff --git a/tensorflow/core/framework/tensor_testutil.cc b/tensorflow/core/framework/tensor_testutil.cc index 313451d6b83..bee7beccb13 100644 --- a/tensorflow/core/framework/tensor_testutil.cc +++ b/tensorflow/core/framework/tensor_testutil.cc @@ -60,6 +60,9 @@ void ExpectClose(const Tensor& x, const Tensor& y, double atol, double rtol) { case DT_HALF: ExpectClose(x, y, atol, rtol); break; + case DT_BFLOAT16: + ExpectClose(x, y, atol, rtol); + break; case DT_FLOAT: ExpectClose(x, y, atol, rtol); break; diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc index efd23b6005e..f0207d24063 100644 --- a/tensorflow/core/grappler/op_types.cc +++ b/tensorflow/core/grappler/op_types.cc @@ -142,6 +142,12 @@ bool IsCollective(const NodeDef& node) { node.op() == "CollectiveBcastRecv"; } +bool IsComparison(const NodeDef& node) { + return node.op() == "Equal" || node.op() == "NotEqual" || + node.op() == "GreaterEqual" || node.op() == "Greater" || + node.op() == "LessEqual" || node.op() == "Less"; +} + bool IsComplex(const NodeDef& node) { return node.op() == "Complex"; } bool IsComplexAbs(const NodeDef& node) { return node.op() == "ComplexAbs"; } diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h index 59fc68daba5..c1d738d6714 100644 --- a/tensorflow/core/grappler/op_types.h +++ b/tensorflow/core/grappler/op_types.h @@ -51,6 +51,7 @@ bool IsBroadcastTo(const NodeDef& node); bool IsCast(const NodeDef& node); bool IsCheckNumerics(const NodeDef& node); bool IsCollective(const NodeDef& node); +bool IsComparison(const NodeDef& node); bool IsComplex(const NodeDef& node); bool IsComplexAbs(const NodeDef& node); bool IsConcat(const NodeDef& node); diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc index 44e6174970e..86a855792b2 100644 --- a/tensorflow/core/grappler/optimizers/remapper.cc +++ b/tensorflow/core/grappler/optimizers/remapper.cc @@ -87,6 +87,15 @@ struct FusedBatchNorm { int fused_batch_norm = kMissingIndex; }; +// Comparison op with cast +struct ComparisonWithCast { + ComparisonWithCast() = default; + + int comparison = kMissingIndex; + int cast = kMissingIndex; + string fused_op = "_"; +}; + // FusedBatchNorm[$is_training] with fused side input and/or activation. struct FusedBatchNormEx { FusedBatchNormEx() = default; @@ -914,6 +923,41 @@ bool FindFusedBatchNormEx(const RemapperContext& ctx, int node_index, return false; } +bool FindComparisonWithCast(const RemapperContext& ctx, int node_index, + ComparisonWithCast* matched) { + const auto* node_view = ctx.graph_view.GetNode(node_index); + const auto* node_def = node_view->node(); + + if (!IsCast(*node_def) || HasControlFaninOrFanout(*node_view)) return false; + + if (node_view->NumRegularFanins() != 1) return false; + const auto& regular_fanin_0 = node_view->GetRegularFanin(0); + const auto* comparison = regular_fanin_0.node_view(); + const auto* comparison_node_def = comparison->node(); + if (!IsComparison(*comparison_node_def) || + HasControlFaninOrFanout(*comparison)) + return false; + + DataType comparator_dtype = GetDataTypeFromAttr(*comparison_node_def, "T"); + DataType src_dtype = GetDataTypeFromAttr(*node_def, "SrcT"); + DataType dst_dtype = GetDataTypeFromAttr(*node_def, "DstT"); + + if ((comparator_dtype != DT_FLOAT) && (comparator_dtype != DT_BFLOAT16)) + return false; + if ((comparator_dtype != dst_dtype) || (src_dtype != DT_BOOL)) return false; + + // Check that only one node consumes the 0-th output of a comparison. + if (!HasAtMostOneDataFanoutAtPort0(*comparison) || + IsInPreserveSet(ctx, comparison_node_def)) + return false; + + matched->cast = node_index; + matched->comparison = regular_fanin_0.node_index(); + matched->fused_op = + matched->fused_op + comparison_node_def->op() + "WithCast"; + return true; +} + void CopyConv2DAttributes(const NodeDef& conv2d, NodeDef* fused_conv2d) { DCHECK(IsConv2D(conv2d)) << "Input node must be a Conv2D"; @@ -1365,6 +1409,40 @@ Status AddFusedBatchNormExNode(RemapperContext* ctx, return Status::OK(); } +Status AddComparisonWithCastNode(RemapperContext* ctx, + const ComparisonWithCast& matched, + std::vector* invalidated_nodes, + std::vector* nodes_to_delete) { + const GraphDef* graph = ctx->graph_view.graph(); + const NodeDef& comparison = graph->node(matched.comparison); + const NodeDef& cast = graph->node(matched.cast); + + VLOG(2) << "Fuse " << cast.op() << " with comparison:" + << " cast=" << cast.name() << " invalidated=" + << " comparison=" << comparison.name(); + + // Replace Comparison and Cast with ComparisonWithCast. + NodeDef fused_op; + fused_op.set_op(matched.fused_op); + fused_op.set_name(cast.name()); + fused_op.set_device(comparison.device()); + + fused_op.add_input(comparison.input(0)); + fused_op.add_input(comparison.input(1)); + (*fused_op.mutable_attr())["T"] = comparison.attr().at("T"); + + utils::Mutation* mutation = ctx->graph_view.GetMutationBuilder(); + Status status; + mutation->AddNode(std::move(fused_op), &status); + TF_RETURN_IF_ERROR(status); + TF_RETURN_IF_ERROR(mutation->Apply()); + + (*nodes_to_delete)[matched.comparison] = true; + (*invalidated_nodes)[matched.cast] = true; + + return Status::OK(); +} + Status AddBatchNormNodes(RemapperContext* ctx, const FusedBatchNorm& matched) { const GraphDef* graph = ctx->graph_view.graph(); const NodeDef& fused_node = graph->node(matched.fused_batch_norm); @@ -1829,6 +1907,15 @@ Status Remapper::Optimize(Cluster* cluster, const GrapplerItem& item, TF_RETURN_IF_ERROR(AddBatchNormNodes(&ctx, fused_batch_norm)); continue; } + + // Remap Comparison+Cast into the ComparisonWithCast. + ComparisonWithCast comparison_with_cast; + if (allow_non_differentiable_rewrites && + FindComparisonWithCast(ctx, i, &comparison_with_cast)) { + TF_RETURN_IF_ERROR(AddComparisonWithCastNode( + &ctx, comparison_with_cast, &invalidated_nodes, &nodes_to_delete)); + continue; + } } // Remove invalidated nodes. diff --git a/tensorflow/core/grappler/optimizers/remapper_test.cc b/tensorflow/core/grappler/optimizers/remapper_test.cc index 9d734801916..eac6b291af4 100644 --- a/tensorflow/core/grappler/optimizers/remapper_test.cc +++ b/tensorflow/core/grappler/optimizers/remapper_test.cc @@ -925,5 +925,64 @@ TEST_F(RemapperTest, FuseConv2DWithSqueezeAndBias) { } #endif +#define REGISTER_TEST_ALL_TYPES(TEST) \ + REGISTER_TEST(TEST, DT_FLOAT); \ + REGISTER_TEST(TEST, DT_BFLOAT16); + +#define REGISTER_TEST(CMP, TYPE) \ + TEST_F(RemapperTest, Fuse##CMP##WithCast_##TYPE) { \ + using ::tensorflow::ops::Placeholder; \ + for (bool is_training : {true, false}) { \ + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); \ + const int num_channels = 24; \ + TensorShape channel_shape({num_channels}); \ + TensorShape empty_shape({0}); \ + auto x = Placeholder(s.WithOpName("x"), TYPE, \ + ops::Placeholder::Shape({2, 8, 8, num_channels})); \ + auto y = Placeholder(s.WithOpName("y"), TYPE, \ + ops::Placeholder::Shape({2, 8, 8, num_channels})); \ + float epsilon = 0.1f; \ + auto comparator = ops::CMP(s.WithOpName("cmp_op"), x, y); \ + auto cast = ops::Cast(s.WithOpName("cast"), comparator.z, TYPE); \ + auto fetch = ops::Identity(s.WithOpName("fetch"), cast); \ + auto input1_t = GenerateRandomTensor({2, 8, 8, num_channels}); \ + auto input2_t = GenerateRandomTensor({2, 8, 8, num_channels}); \ + GrapplerItem item; \ + item.fetch = {"fetch"}; \ + item.feed = {{"x", input1_t}, {"y", input2_t}}; \ + TF_ASSERT_OK(s.ToGraphDef(&item.graph)); \ + for (int i = 0; i < item.graph.node_size(); ++i) { \ + item.graph.mutable_node(i)->set_device("/device:CPU:0"); \ + } \ + Remapper optimizer(RewriterConfig::AGGRESSIVE); \ + GraphDef output; \ + TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output)); \ + int found = 0; \ + for (const NodeDef& node : output.node()) { \ + if (node.name() == "cast") { \ + EXPECT_EQ(node.op(), "_" #CMP "WithCast"); \ + ASSERT_EQ(node.input_size(), 2); \ + EXPECT_EQ(node.input(0), "x"); \ + EXPECT_EQ(node.input(1), "y"); \ + found++; \ + } \ + } \ + EXPECT_EQ(found, 1); \ + auto tensors_expected = \ + EvaluateNodes(item.graph, item.fetch, item.feed); \ + ASSERT_EQ(tensors_expected.size(), 1); \ + auto tensors = EvaluateNodes(output, item.fetch, item.feed); \ + ASSERT_EQ(tensors.size(), 1); \ + test::ExpectClose(tensors[0], tensors_expected[0], 1e-2, 1e-2); \ + } \ + } +REGISTER_TEST_ALL_TYPES(GreaterEqual) +REGISTER_TEST_ALL_TYPES(Greater) +REGISTER_TEST_ALL_TYPES(LessEqual) +REGISTER_TEST_ALL_TYPES(Less) +REGISTER_TEST_ALL_TYPES(Equal) +REGISTER_TEST_ALL_TYPES(NotEqual) +#undef REGISTER_TEST + } // namespace grappler } // namespace tensorflow diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h index 7ac70356f2c..c996c8bbe3a 100644 --- a/tensorflow/core/grappler/utils/grappler_test.h +++ b/tensorflow/core/grappler/utils/grappler_test.h @@ -85,7 +85,7 @@ class GrapplerTest : public ::testing::Test { typedef typename EnumToDataType::Type T; Tensor tensor(DTYPE, shape); for (auto i = 0; i < tensor.NumElements(); i++) - tensor.flat()(i) = i + random::New64() % 10; + tensor.flat()(i) = static_cast(i + random::New64() % 10); return tensor; } diff --git a/tensorflow/core/kernels/cwise_op_equal_to_1.cc b/tensorflow/core/kernels/cwise_op_equal_to_1.cc index 64cd784af73..86da7525685 100644 --- a/tensorflow/core/kernels/cwise_op_equal_to_1.cc +++ b/tensorflow/core/kernels/cwise_op_equal_to_1.cc @@ -19,6 +19,8 @@ namespace tensorflow { REGISTER7(BinaryOp, CPU, "Equal", functor::equal_to, float, Eigen::half, double, uint8, int8, int16, bfloat16); REGISTER3(BinaryOp, CPU, "Equal", functor::equal_to, uint16, uint32, uint64); +REGISTER2(BinaryOp, CPU, "_EqualWithCast", functor::equal_to_with_cast, float, + bfloat16); REGISTER_KERNEL_BUILDER( Name("ApproximateEqual").Device(DEVICE_CPU).TypeConstraint("T"), ApproximateEqualOp); diff --git a/tensorflow/core/kernels/cwise_op_greater.cc b/tensorflow/core/kernels/cwise_op_greater.cc index d70233dc55c..e905f13f6c6 100644 --- a/tensorflow/core/kernels/cwise_op_greater.cc +++ b/tensorflow/core/kernels/cwise_op_greater.cc @@ -18,6 +18,8 @@ limitations under the License. namespace tensorflow { REGISTER9(BinaryOp, CPU, "Greater", functor::greater, float, Eigen::half, double, int32, int64, uint8, int8, int16, bfloat16); +REGISTER2(BinaryOp, CPU, "_GreaterWithCast", functor::greater_with_cast, float, + bfloat16); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER7(BinaryOp, GPU, "Greater", functor::greater, float, Eigen::half, double, int64, uint8, int8, int16); diff --git a/tensorflow/core/kernels/cwise_op_greater_equal.cc b/tensorflow/core/kernels/cwise_op_greater_equal.cc index 7f6b788eb2e..8390035b86b 100644 --- a/tensorflow/core/kernels/cwise_op_greater_equal.cc +++ b/tensorflow/core/kernels/cwise_op_greater_equal.cc @@ -18,6 +18,8 @@ limitations under the License. namespace tensorflow { REGISTER9(BinaryOp, CPU, "GreaterEqual", functor::greater_equal, float, Eigen::half, double, int32, int64, uint8, int8, int16, bfloat16); +REGISTER2(BinaryOp, CPU, "_GreaterEqualWithCast", + functor::greater_equal_with_cast, float, bfloat16); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER7(BinaryOp, GPU, "GreaterEqual", functor::greater_equal, float, Eigen::half, double, int64, uint8, int8, int16); diff --git a/tensorflow/core/kernels/cwise_op_less.cc b/tensorflow/core/kernels/cwise_op_less.cc index 062a029f069..55f165128d8 100644 --- a/tensorflow/core/kernels/cwise_op_less.cc +++ b/tensorflow/core/kernels/cwise_op_less.cc @@ -19,6 +19,8 @@ namespace tensorflow { REGISTER5(BinaryOp, CPU, "Less", functor::less, float, Eigen::half, double, bfloat16, int32); REGISTER4(BinaryOp, CPU, "Less", functor::less, int64, uint8, int8, int16); +REGISTER2(BinaryOp, CPU, "_LessWithCast", functor::less_with_cast, float, + bfloat16); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER7(BinaryOp, GPU, "Less", functor::less, float, Eigen::half, double, diff --git a/tensorflow/core/kernels/cwise_op_less_equal.cc b/tensorflow/core/kernels/cwise_op_less_equal.cc index 43af03878e9..2961742f5f4 100644 --- a/tensorflow/core/kernels/cwise_op_less_equal.cc +++ b/tensorflow/core/kernels/cwise_op_less_equal.cc @@ -20,6 +20,8 @@ REGISTER5(BinaryOp, CPU, "LessEqual", functor::less_equal, float, Eigen::half, bfloat16, double, int32); REGISTER4(BinaryOp, CPU, "LessEqual", functor::less_equal, int64, uint8, int8, int16); +REGISTER2(BinaryOp, CPU, "_LessEqualWithCast", functor::less_equal_with_cast, + float, bfloat16); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER7(BinaryOp, GPU, "LessEqual", functor::less_equal, float, Eigen::half, diff --git a/tensorflow/core/kernels/cwise_op_not_equal_to_1.cc b/tensorflow/core/kernels/cwise_op_not_equal_to_1.cc index 4de69edd21d..68a996c97b6 100644 --- a/tensorflow/core/kernels/cwise_op_not_equal_to_1.cc +++ b/tensorflow/core/kernels/cwise_op_not_equal_to_1.cc @@ -20,6 +20,8 @@ REGISTER7(BinaryOp, CPU, "NotEqual", functor::not_equal_to, float, Eigen::half, double, uint8, int8, int16, bfloat16); REGISTER3(BinaryOp, CPU, "NotEqual", functor::not_equal_to, uint16, uint32, uint64); +REGISTER2(BinaryOp, CPU, "_NotEqualWithCast", functor::not_equal_to_with_cast, + float, bfloat16); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER4(BinaryOp, GPU, "NotEqual", functor::not_equal_to, float, Eigen::half, double, uint8); diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h index 88651d7bfdc..58c2323999b 100644 --- a/tensorflow/core/kernels/cwise_ops.h +++ b/tensorflow/core/kernels/cwise_ops.h @@ -21,10 +21,10 @@ limitations under the License. #include #include -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/bounds_check.h" #include "tensorflow/core/framework/numeric_types.h" #include "tensorflow/core/framework/tensor_types.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" namespace Eigen { namespace internal { @@ -1141,6 +1141,32 @@ struct equal_to : base, bool> {}; template struct not_equal_to : base, bool> {}; +template +struct less_with_cast : base> {}; + +template +struct less_equal_with_cast : base> {}; + +template +struct greater_with_cast : base> {}; + +template +struct greater_equal_with_cast + : base> {}; + +template +struct equal_to_with_cast : base> {}; + +template +struct not_equal_to_with_cast + : base> {}; + struct logical_and : base {}; struct logical_or : base {}; diff --git a/tensorflow/core/kernels/cwise_ops_test.cc b/tensorflow/core/kernels/cwise_ops_test.cc index bc77a119f0a..4fee16fa759 100644 --- a/tensorflow/core/kernels/cwise_ops_test.cc +++ b/tensorflow/core/kernels/cwise_ops_test.cc @@ -96,62 +96,66 @@ BM_UNARY(gpu, Round, float, DT_FLOAT); #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM // data func scalar. -Graph* BinaryScalar(int num, const string& func) { +template +Graph* BinaryScalar(int num, const string& func, DataType dtype) { Graph* g = new Graph(OpRegistry::Global()); - Tensor lhs(DT_FLOAT, TensorShape({64, 64, num / (64 * 64)})); - lhs.flat().setRandom(); - Tensor rhs(DT_FLOAT, TensorShape({})); - rhs.flat().setRandom(); + Tensor lhs(dtype, TensorShape({64, 64, num / (64 * 64)})); + lhs.flat().setRandom(); + Tensor rhs(dtype, TensorShape({})); + rhs.flat().setRandom(); test::graph::Binary(g, func, test::graph::Constant(g, lhs), test::graph::Constant(g, rhs)); return g; } -#define BM_BINARY_SCALAR(DEVICE, FUNC) \ - void BM_##DEVICE##_##FUNC##_scalar(int iters, int num) { \ - const int64 tot = static_cast(iters) * num; \ - testing::UseRealTime(); \ - testing::ItemsProcessed(tot); \ - testing::BytesProcessed(tot * sizeof(float)); \ - test::Benchmark(#DEVICE, BinaryScalar(num, #FUNC)).Run(iters); \ - } \ - BENCHMARK(BM_##DEVICE##_##FUNC##_scalar) \ - ->Arg(1 << 12) /* must >= 4096 */ \ - ->Arg(1 << 13) \ - ->Arg(1 << 14) \ - ->Arg((1 << 15) - (1 << 13)) \ - ->Arg(1 << 15) \ - ->Arg((1 << 15) + (1 << 14)) \ - ->Arg(1 << 16) \ - ->Arg((1 << 17) - (1 << 15)) \ - ->Arg(1 << 17) \ - ->Arg((1 << 17) + (1 << 16)) \ - ->Arg(1 << 18) \ - ->Arg(1 << 19) \ +#define BM_BINARY_SCALAR(DEVICE, FUNC, T, TYPE) \ + void BM_##DEVICE##_##FUNC##_scalar##_##TYPE(int iters, int num) { \ + const int64 tot = static_cast(iters) * num; \ + testing::UseRealTime(); \ + testing::ItemsProcessed(tot); \ + testing::BytesProcessed(tot * sizeof(T)); \ + test::Benchmark(#DEVICE, BinaryScalar(num, #FUNC, TYPE)).Run(iters); \ + } \ + BENCHMARK(BM_##DEVICE##_##FUNC##_scalar##_##TYPE) \ + ->Arg(1 << 12) /* must >= 4096 */ \ + ->Arg(1 << 13) \ + ->Arg(1 << 14) \ + ->Arg((1 << 15) - (1 << 13)) \ + ->Arg(1 << 15) \ + ->Arg((1 << 15) + (1 << 14)) \ + ->Arg(1 << 16) \ + ->Arg((1 << 17) - (1 << 15)) \ + ->Arg(1 << 17) \ + ->Arg((1 << 17) + (1 << 16)) \ + ->Arg(1 << 18) \ + ->Arg(1 << 19) \ ->Arg(1 << 20); -BM_BINARY_SCALAR(cpu, Less); +BM_BINARY_SCALAR(cpu, Less, float, DT_FLOAT); +BM_BINARY_SCALAR(cpu, Less, bfloat16, DT_BFLOAT16); +BM_BINARY_SCALAR(cpu, _LessWithCast, float, DT_FLOAT); +BM_BINARY_SCALAR(cpu, _LessWithCast, bfloat16, DT_BFLOAT16); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -BM_BINARY_SCALAR(gpu, Less); +BM_BINARY_SCALAR(gpu, Less, float, DT_FLOAT); #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM #ifdef TENSORFLOW_USE_SYCL -BM_BINARY_SCALAR(sycl, Less); +BM_BINARY_SCALAR(sycl, Less, float, DT_FLOAT); #endif // TENSORFLOW_USE_SYCL -BM_BINARY_SCALAR(cpu, Add); +BM_BINARY_SCALAR(cpu, Add, float, DT_FLOAT); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -BM_BINARY_SCALAR(gpu, Add); +BM_BINARY_SCALAR(gpu, Add, float, DT_FLOAT); #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM #ifdef TENSORFLOW_USE_SYCL -BM_BINARY_SCALAR(sycl, Add); +BM_BINARY_SCALAR(sycl, Add, float, DT_FLOAT); #endif // TENSORFLOW_USE_SYCL -BM_BINARY_SCALAR(cpu, DivNoNan); +BM_BINARY_SCALAR(cpu, DivNoNan, float, DT_FLOAT); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -BM_BINARY_SCALAR(gpu, DivNoNan); +BM_BINARY_SCALAR(gpu, DivNoNan, float, DT_FLOAT); #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM #ifdef TENSORFLOW_USE_SYCL -BM_BINARY_SCALAR(sycl, DivNoNan); +BM_BINARY_SCALAR(sycl, DivNoNan, float, DT_FLOAT); #endif // TENSORFLOW_USE_SYCL #undef BM_BINARY_SCALAR diff --git a/tensorflow/core/ops/math_ops.cc b/tensorflow/core/ops/math_ops.cc index 2a70f420260..2817e0a50eb 100644 --- a/tensorflow/core/ops/math_ops.cc +++ b/tensorflow/core/ops/math_ops.cc @@ -700,6 +700,23 @@ REGISTER_OP("GreaterEqual").COMPARISON(); #undef COMPARISON +#define COMPARISON_WITH_CAST() \ + Input("x: T") \ + .Input("y: T") \ + .Output("z: T") \ + .Attr("T: {float, bfloat16}") \ + .SetShapeFn(shape_inference::BroadcastBinaryOpShapeFn) + +REGISTER_OP("_LessWithCast").COMPARISON_WITH_CAST(); + +REGISTER_OP("_LessEqualWithCast").COMPARISON_WITH_CAST(); + +REGISTER_OP("_GreaterWithCast").COMPARISON_WITH_CAST(); + +REGISTER_OP("_GreaterEqualWithCast").COMPARISON_WITH_CAST(); + +#undef COMPARISON_WITH_CAST + // -------------------------------------------------------------------------- #define EQUALITY_COMPARISON() \ @@ -731,6 +748,32 @@ REGISTER_OP("NotEqual").EQUALITY_COMPARISON(); #undef EQUALITY_COMPARISON +#define EQUALITY_COMPARISON_WITH_CAST() \ + Input("x: T") \ + .Input("y: T") \ + .Output("z: T") \ + .SetIsCommutative() \ + .Attr("T: {bfloat16, float}") \ + .Attr("incompatible_shape_error: bool = true") \ + .SetShapeFn([](InferenceContext* c) { \ + ShapeHandle x = c->input(0); \ + ShapeHandle y = c->input(1); \ + ShapeHandle output; \ + bool incompatible_shape_error; \ + TF_RETURN_IF_ERROR(c->GetAttr("incompatible_shape_error", \ + &incompatible_shape_error)); \ + TF_RETURN_IF_ERROR(BroadcastBinaryOpOutputShapeFnHelper( \ + c, x, y, incompatible_shape_error, &output)); \ + c->set_output(0, output); \ + return Status::OK(); \ + }) + +REGISTER_OP("_EqualWithCast").EQUALITY_COMPARISON_WITH_CAST(); + +REGISTER_OP("_NotEqualWithCast").EQUALITY_COMPARISON_WITH_CAST(); + +#undef EQUALITY_COMPARISON_WITH_CAST + REGISTER_OP("ApproximateEqual") .Input("x: T") .Input("y: T") From 71be44917ac675ecb951fa3e5be115dae0e5aef8 Mon Sep 17 00:00:00 2001 From: MichelBr Date: Sat, 4 Jul 2020 23:48:44 +0200 Subject: [PATCH 0032/1017] Update output_handler.cc fix incorrectly documented led-color order --- .../micro/examples/hello_world/sparkfun_edge/output_handler.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/lite/micro/examples/hello_world/sparkfun_edge/output_handler.cc b/tensorflow/lite/micro/examples/hello_world/sparkfun_edge/output_handler.cc index c9f23dc2de0..2e727095a5c 100644 --- a/tensorflow/lite/micro/examples/hello_world/sparkfun_edge/output_handler.cc +++ b/tensorflow/lite/micro/examples/hello_world/sparkfun_edge/output_handler.cc @@ -22,7 +22,7 @@ This function uses the device's LEDs to visually indicate the current y value. The y value is in the range -1 <= y <= 1. The LEDs (red, green, blue, and yellow) are physically lined up in the following order: - [ R G B Y ] + [ R B G Y ] The following table represents how we will light the LEDs for different values: From f206eedcf843e9176923a24de5aab9577b25d510 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 6 Jul 2020 20:40:18 +0000 Subject: [PATCH 0033/1017] summary --- tensorflow/c/kernels/summary_op.cc | 31 +++++-- tensorflow/c/kernels/summary_op_test.cc | 11 ++- tensorflow/c/tf_tensor.cc | 103 ++++-------------------- tensorflow/c/tf_tensor.h | 3 + tensorflow/c/tf_tensor_internal.h | 2 + 5 files changed, 52 insertions(+), 98 deletions(-) diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 23fd437af78..c3d7fa84aaa 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/types.h" +#include static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { void* ptr; @@ -42,6 +43,16 @@ static void SummaryScalarOp_Delete(void* kernel) { bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2); static tensorflow::string SingleTag(TF_Tensor* tags); +template +float get_float_value(T* element){ + return static_cast(*element); +} + +template<> +float get_float_value(Eigen::half* element){ + return Eigen::half_impl::half_to_float(*element); +} + template static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { TF_Tensor* tags; @@ -68,19 +79,25 @@ static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { auto values_array = static_cast(TF_TensorData(values)); // Copy tags and values into summary protobuf for (int i = 0; i < TF_TensorElementCount(tags); ++i) { - tensorflow::Summary::Value* v = s.add_value(); - v->set_tag(tags_array[i].data(), tags_array[i].size()); - v->set_simple_value(float(values_array[i])); + tensorflow::Summary::Value* v = s.add_value(); + const tensorflow::tstring& Ttags_i = tags_array[i]; + v->set_tag(Ttags_i.data(), Ttags_i.size()); + v->set_simple_value(get_float_value(&values_array[i])); } - TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, + TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0 TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, - sizeof(TF_TString), status); + sizeof(tensorflow::tstring), status); if (TF_GetCode(status) == TF_OK) { - SerializeToTString(s, static_cast - (TF_TensorData(summary_tensor))); + tensorflow::tstring summary_tstring; + SerializeToTString(s, &summary_tstring); + *(TF_TensorData(summary_tensor)) = &summary_tstring; + TF_SetOutput(ctx, 0, summary_tensor, status); } TF_DeleteTensor(summary_tensor); } + if (TF_GetCode(status) != TF_OK) { + TF_OpKernelContext_Failure(ctx, status); + } TF_DeleteStatus(status); TF_DeleteTensor(tags); TF_DeleteTensor(values); diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc index d8dbf622a55..7438693b430 100644 --- a/tensorflow/c/kernels/summary_op_test.cc +++ b/tensorflow/c/kernels/summary_op_test.cc @@ -72,6 +72,8 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, DummyDevice dummy_device(nullptr); params.device = &dummy_device; params.op_kernel = kernel.get(); + AllocatorAttributes alloc_attrs; + params.output_attr_array = &alloc_attrs; gtl::InlinedVector inputs; inputs.emplace_back(tags); inputs.emplace_back(values); @@ -84,11 +86,12 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, Summary summary; ParseProtoUnlimited(&summary, ctx.mutable_output(0)->scalar()()); EXPECT_SummaryMatches(summary, expected_summary); + } } TEST(ScalarSummaryOpTest, Test) { - int vectorSize = 2; + int vectorSize = 3; Tensor tags(DT_STRING, {vectorSize}); Tensor values(DT_FLOAT, {vectorSize}); tags.vec()(0) = "tag1"; @@ -126,9 +129,9 @@ TEST(ScalarSummaryOpTest, SimpleHalf) { tags.vec()(0) = "tag1"; tags.vec()(1) = "tag2"; tags.vec()(2) = "tag3"; - values.vec()(0) = static_cast(1.0); - values.vec()(1) = static_cast(-2.0); - values.vec()(2) = static_cast(10000.0); + values.vec()(0) = Eigen::half(1.0); + values.vec()(1) = Eigen::half(-2.0); + values.vec()(2) = Eigen::half(10000.0); TestScalarSummaryOp(&tags, &values, R"( value { tag: 'tag1' simple_value: 1.0 } value { tag: 'tag2' simple_value: -2.0} diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc index 34c91fc23dc..5cfd495933c 100644 --- a/tensorflow/c/tf_tensor.cc +++ b/tensorflow/c/tf_tensor.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/lib/core/coding.h" #include "tensorflow/core/platform/casts.h" +#include using tensorflow::Status; using tensorflow::Tensor; @@ -180,6 +181,11 @@ void TF_TensorBitcastFrom(const TF_Tensor* from, TF_DataType type, Set_TF_Status_from_Status(status, cc_status); } +std::string TF_ShapeDebugString(const TF_Tensor* t){ + return tensorflow::down_cast(t->tensor) + ->ShapeDebugString(); +} + namespace tensorflow { void TensorInterface::Release() { delete this; } @@ -225,6 +231,10 @@ Status TensorInterface::BitcastFrom(const TensorInterface& from, DataType type, return tensor_.BitcastFrom(from.tensor_, type, s); } +std::string TensorInterface::ShapeDebugString() const { + return tensor_.shape().DebugString(); +} + } // namespace tensorflow // -------------------------------------------------------------------------- @@ -283,62 +293,11 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, Status* status) { std::memcpy(TF_TensorData(t), str.c_str(), str.size()); return t; } - if (src.dtype() != tensorflow::DT_STRING) { - Tensor tensor; - if (!tensor.CopyFrom(src, src.shape())) { - return nullptr; - } - return new TF_Tensor{new tensorflow::TensorInterface(tensor)}; - } - // DT_STRING tensors require a copying since TF_Tensor.buffer expects a flatly - // encoded sequence of strings. - - // Compute bytes needed for encoding. - size_t size = 0; - const auto& srcarray = src.flat(); - for (int i = 0; i < srcarray.size(); ++i) { - const string& s = srcarray(i); - // uint64 starting_offset, TF_StringEncode-d string. - size += sizeof(tensorflow::uint64) + TF_StringEncodedSize(s.size()); - } - - // Encode all strings. - char* base = new char[size]; - char* data_start = base + sizeof(tensorflow::uint64) * srcarray.size(); - char* dst = data_start; // Where next string is encoded. - size_t dst_len = size - static_cast(data_start - base); - tensorflow::uint64* offsets = reinterpret_cast(base); - for (int i = 0; i < srcarray.size(); ++i) { - *offsets = (dst - data_start); - offsets++; - const string& s = srcarray(i); - const size_t consumed = TF_StringEncodedSize(s.size()); - StringEncode(s.data(), s.size(), dst); - dst += consumed; - dst_len -= consumed; - } - if (dst != base + size) { - *status = InvalidArgument( - "invalid string tensor encoding (decoded ", (dst - base), - " bytes, but the tensor is encoded in ", size, " bytes"); - delete[] base; + Tensor tensor; + if (!tensor.CopyFrom(src, src.shape())) { return nullptr; } -// <<<<<<< HEAD -// return new TF_Tensor{new tensorflow::TensorInterface(tensor)}; -// ======= - - auto dims = src.shape().dim_sizes(); - std::vector dimvec(dims.size()); - for (size_t i = 0; i < dims.size(); ++i) { - dimvec[i] = dims[i]; - } - static_assert(sizeof(int64_t) == sizeof(tensorflow::int64), - "64-bit int types should match in size"); - return TF_NewTensor(TF_STRING, - reinterpret_cast(dimvec.data()), - dimvec.size(), base, size, DeleteArray, base); -// >>>>>>> parent of 477470d094... finished test file + return new TF_Tensor{new tensorflow::TensorInterface(tensor)}; } Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst) { @@ -362,39 +321,7 @@ Status TensorInterface::ToTensor(tensorflow::Tensor* dst) const { } return Status::OK(); } - if (tensor_.dtype() != DT_STRING) { - *dst = tensor_; - return Status::OK(); - } - // TF_STRING tensors require copying since Tensor class expects a sequence of - // string objects. - const tensorflow::int64 num_elements = tensor_.NumElements(); - const char* input = reinterpret_cast(Data()); - const size_t src_size = ByteSize(); - if (static_cast(src_size / sizeof(tensorflow::uint64)) < - num_elements) { - return InvalidArgument( - "Malformed TF_STRING tensor; too short to hold number of elements"); - } - const char* data_start = input + sizeof(tensorflow::uint64) * num_elements; - const char* limit = input + src_size; - - *dst = tensorflow::Tensor(tensor_.dtype(), tensor_.shape()); - auto dstarray = dst->flat(); - for (tensorflow::int64 i = 0; i < num_elements; ++i) { - tensorflow::uint64 offset = - reinterpret_cast(input)[i]; - if (static_cast(offset) >= (limit - data_start)) { - return InvalidArgument("Malformed TF_STRING tensor; element ", i, - " out of range"); - } - size_t len; - const char* p; - const char* srcp = data_start + offset; - Status status = TF_StringDecode_Impl(srcp, limit - srcp, &p, &len); - if (!status.ok()) return status; - dstarray(i).assign(p, len); - } + *dst = tensor_; return Status::OK(); } @@ -403,3 +330,5 @@ bool TensorInterface::IsAligned() const { return tensor_.IsAligned(); } } // namespace tensorflow bool TF_TensorIsAligned(const TF_Tensor* t) { return t->tensor->IsAligned(); } + + diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index acdf053e63a..e4953b53e43 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/c/tf_datatype.h" #include "tensorflow/c/tf_status.h" +#include // Macro to control visibility of exported symbols in the shared library (.so, // .dylib, .dll). @@ -151,6 +152,8 @@ TF_CAPI_EXPORT extern void TF_TensorBitcastFrom(const TF_Tensor* from, // Returns bool iff this tensor is aligned. TF_CAPI_EXPORT extern bool TF_TensorIsAligned(const TF_Tensor*); +TF_CAPI_EXPORT extern std::string TF_ShapeDebugString(const TF_Tensor*); + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/tf_tensor_internal.h b/tensorflow/c/tf_tensor_internal.h index 7a896dc5d11..036559da838 100644 --- a/tensorflow/c/tf_tensor_internal.h +++ b/tensorflow/c/tf_tensor_internal.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_C_TF_TENSOR_INTERNAL_H_ #include +#include #include "tensorflow/c/tensor_interface.h" #include "tensorflow/c/tf_datatype.h" @@ -104,6 +105,7 @@ class TensorInterface : public AbstractTensorInterface { void* Data() const override; bool IsAligned() const override; bool CanMove() const override; + std::string ShapeDebugString() const; Status ToTensor(tensorflow::Tensor* dst) const; Status BitcastFrom(const TensorInterface& from, DataType type, From ea6b83192106bdc145624c73193802cc872ed5d8 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 6 Jul 2020 22:30:56 +0000 Subject: [PATCH 0034/1017] tests passing for summary op, added shape debug string --- tensorflow/c/kernels/summary_op.cc | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index c3d7fa84aaa..cad7bd646ed 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -43,16 +43,6 @@ static void SummaryScalarOp_Delete(void* kernel) { bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2); static tensorflow::string SingleTag(TF_Tensor* tags); -template -float get_float_value(T* element){ - return static_cast(*element); -} - -template<> -float get_float_value(Eigen::half* element){ - return Eigen::half_impl::half_to_float(*element); -} - template static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { TF_Tensor* tags; @@ -82,18 +72,21 @@ static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { tensorflow::Summary::Value* v = s.add_value(); const tensorflow::tstring& Ttags_i = tags_array[i]; v->set_tag(Ttags_i.data(), Ttags_i.size()); - v->set_simple_value(get_float_value(&values_array[i])); + v->set_simple_value(float(values_array[i])); } - TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0 + TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, sizeof(tensorflow::tstring), status); if (TF_GetCode(status) == TF_OK) { tensorflow::tstring summary_tstring; SerializeToTString(s, &summary_tstring); - *(TF_TensorData(summary_tensor)) = &summary_tstring; - TF_SetOutput(ctx, 0, summary_tensor, status); + TF_TString* output_tf_tstring = reinterpret_cast(TF_TensorData(summary_tensor)); + TF_TString_Init(output_tf_tstring); + tensorflow::tstring* output_tstring = reinterpret_cast(output_tf_tstring); + *output_tstring = summary_tstring; // may want to use std::move } TF_DeleteTensor(summary_tensor); + } if (TF_GetCode(status) != TF_OK) { TF_OpKernelContext_Failure(ctx, status); From e0170c9a25948857bbfe69555ac7d06d06586d7f Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 7 Jul 2020 03:42:07 +0000 Subject: [PATCH 0035/1017] working op: include variant_encode_decode.h in map_kernels.h --- tensorflow/core/kernels/map_kernels.cc | 16 ++++++++++++++++ tensorflow/core/kernels/map_kernels.h | 12 +++++++++--- tensorflow/core/ops/map_ops.cc | 12 ++++++------ tensorflow/python/kernel_tests/map_ops_test.py | 16 +++++++++++++--- 4 files changed, 44 insertions(+), 12 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index 91dfbe8c384..7d5d2732bb6 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -1,6 +1,22 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ #include "tensorflow/core/kernels/map_kernels.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/types.h" +//#include "tensorflow/core/framework/variant.h" +//#include "tensorflow/core/framework/variant_op_registry.h" namespace tensorflow { REGISTER_KERNEL_BUILDER(Name("EmptyTensorMap").Device(DEVICE_CPU), diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 1ed7c663a57..88028b13dea 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -1,4 +1,4 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,9 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#ifndef TENSORFLOW_CORE_KERNELS_MAP_KERNELS_H_ +#define TENSORFLOW_CORE_KERNELS_MAP_KERNELS_H_ #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/kernels/tensor_map.h" +//#include "tensorflow/core/framework/variant.h" +//#include "tensorflow/core/framework/variant_op_registry.h" +#include "tensorflow/core/framework/variant_encode_decode.h" + #include using namespace std; @@ -79,6 +85,6 @@ class ZeroOutOp : public OpKernel { } }; - - } // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_MAP_KERNELS_H_ diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index f1d7b291a70..d8ecb5ff0ed 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -21,21 +21,21 @@ namespace tensorflow { namespace { REGISTER_OP("EmptyTensorMap") -// .Input("element_shape: shape_type") -// .Input("max_num_elements: int32") + .Input("element_shape: shape_type") + .Input("max_num_elements: int32") .Output("handle: variant") -// .Attr("element_dtype: type") -// .Attr("shape_type: {int32, int64}") + .Attr("element_dtype: type") + .Attr("shape_type: {int32, int64}") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - /*DataType element_dtype; + DataType element_dtype; TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); shape_inference::ShapeHandle element_shape; TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape( 0, &element_shape)); c->set_output_handle_shapes_and_types( 0, std::vector{ - {element_shape, element_dtype}});*/ + {element_shape, element_dtype}}); return Status::OK(); }); diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 7b9654886f3..a8364c19d8d 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -23,18 +23,27 @@ from tensorflow.python.platform import test from absl.testing import parameterized from tensorflow.python.framework import test_util +from tensorflow.python.client import session +from tensorflow.python.eager import backprop +from tensorflow.python.eager import context +from tensorflow.python.eager import def_function +from tensorflow.python.eager import function +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes + #try: # from tensorflow_zero_out.python.ops.zero_out_ops import zero_out #except ImportError: # from zero_out_ops import zero_out from tensorflow.python.ops import map_ops +@test_util.run_all_in_graph_and_eager_modes class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): - """ @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 2)) @test_util.run_deprecated_v1 def testEraseFromEmptyTensorMapFails(self, max_num_elements): + print("hello world testErase") m = map_ops.empty_tensor_map( element_dtype=dtypes.float32, element_shape=[], @@ -43,14 +52,15 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): "Trying to erase from an empty map"): m = map_ops.tensor_map_erase(l, element_dtype=dtypes.float32) self.evaluate(l) - """ + def testZeroOut(self): - print("Hello World - Test") + print("hello world testZeroOut") with self.test_session(): self.assertAllClose( map_ops.zero_out([[1, 2], [3, 4]]), np.array([[1, 0], [0, 0]])) if __name__ == '__main__': + print("hihihi") test.main() \ No newline at end of file From 38071c95b282ecfa5737a99fcb7256043074ff2c Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 7 Jul 2020 07:04:31 +0000 Subject: [PATCH 0036/1017] working EmptyTensorMap op --- tensorflow/core/kernels/map_kernels.cc | 2 - tensorflow/core/kernels/map_kernels.h | 29 +++---- tensorflow/core/ops/map_ops.cc | 81 +++++++++++++++++-- .../python/kernel_tests/map_ops_test.py | 12 +-- tensorflow/python/ops/map_ops.py | 13 +-- 5 files changed, 97 insertions(+), 40 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index 7d5d2732bb6..12c932eb83e 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -15,8 +15,6 @@ limitations under the License. #include "tensorflow/core/kernels/map_kernels.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/types.h" -//#include "tensorflow/core/framework/variant.h" -//#include "tensorflow/core/framework/variant_op_registry.h" namespace tensorflow { REGISTER_KERNEL_BUILDER(Name("EmptyTensorMap").Device(DEVICE_CPU), diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 88028b13dea..1b344718261 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -17,8 +17,6 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/kernels/tensor_map.h" -//#include "tensorflow/core/framework/variant.h" -//#include "tensorflow/core/framework/variant_op_registry.h" #include "tensorflow/core/framework/variant_encode_decode.h" #include @@ -28,27 +26,14 @@ namespace tensorflow { class EmptyTensorMap : public OpKernel { public: - explicit EmptyTensorMap(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("element_dtype", &element_dtype_)); - } + explicit EmptyTensorMap(OpKernelConstruction* ctx) : OpKernel(ctx) {} void Compute(OpKernelContext* ctx) override { - const Tensor& max_num_elements_t = ctx->input(1); - OP_REQUIRES( - ctx, TensorShapeUtils::IsScalar(max_num_elements_t.shape()), - errors::InvalidArgument( - "max_num_elements expected to be a scalar ", - "but got shape: ", max_num_elements_t.shape().DebugString())); Tensor* result; AllocatorAttributes attr; attr.set_on_host(true); OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape{}, &result, attr)); TensorMap empty; - empty.element_dtype = element_dtype_; - empty.max_num_elements = max_num_elements_t.scalar()(); - PartialTensorShape element_shape; - //OP_REQUIRES_OK(ctx, TensorShapeFromTensor(ctx->input(0), &element_shape)); - empty.element_shape = element_shape; result->scalar()() = std::move(empty); } @@ -56,7 +41,19 @@ class EmptyTensorMap : public OpKernel { DataType element_dtype_; }; +class TensorMapSize : public OpKernel { + public: + explicit TensorMapSize(OpKernelConstruction* c) : OpKernel(c) {} + ~TEnsorMapSize() override {} + void Compute(OpKernelContext* c) override { + const TensorMap* m = nullptr; + OP_REQUIRES_OK(c, GetInputList(c, 0, &m)); + Tensor* result; + OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result)); + result->scalar()() = m->tensors().size(); + } +}; class ZeroOutOp : public OpKernel { public: diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index d8ecb5ff0ed..ab2fdef9127 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -21,24 +21,93 @@ namespace tensorflow { namespace { REGISTER_OP("EmptyTensorMap") - .Input("element_shape: shape_type") - .Input("max_num_elements: int32") .Output("handle: variant") + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->Scalar()); + return Status::OK(); + }); + +REGISTER_OP("TensorMapInsert") + .Input("input_handle: variant") + .Input("key: element_dtype") + .Input("value: element_dtype") + .Output("output_handle: variant") .Attr("element_dtype: type") - .Attr("shape_type: {int32, int64}") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); DataType element_dtype; TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); - shape_inference::ShapeHandle element_shape; - TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensorTreatScalarAsUnknownShape( - 0, &element_shape)); + shape_inference::ShapeHandle element_shape = c->UnknownShape(); + + auto* handle_data = c->input_handle_shapes_and_types(0); + if (handle_data != nullptr && handle_data->size() > 1) { + return errors::InvalidArgument( + "Trying to push to list with wrong variant data."); + } + if (IsValidTensorListHandleData(handle_data)) { + const shape_inference::ShapeAndType& list_shape_type = + (*handle_data)[0]; + if (list_shape_type.dtype != element_dtype) { + return errors::InvalidArgument( + "Trying to push to list with wrong element dtype. List has type ", + DataTypeString(list_shape_type.dtype), + " but trying to push element with type ", + DataTypeString(element_dtype)); + } + shape_inference::ShapeHandle ignored; + TF_RETURN_IF_ERROR( + c->Merge(element_shape, list_shape_type.shape, &ignored)); + element_shape = list_shape_type.shape; + } c->set_output_handle_shapes_and_types( 0, std::vector{ {element_shape, element_dtype}}); return Status::OK(); }); +REGISTER_OP("TensorMapSize") + .Input("input_handle: variant") + .Output("size: int32") + .SetShapeFn(shape_inference::ScalarShape); + +/*REGISTER_OP("TensorMapErase") + .Input("input_handle: variant") + .Input("element_shape: int32") + .Output("output_handle: variant") + .Output("tensor: element_dtype") + .Attr("element_dtype: type") + .SetShapeFn([](shape_inference::InferenceContext* c) { + DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle tensor_shape = c->UnknownShape(); + auto* handle_data = c->input_handle_shapes_and_types(0); + if (handle_data != nullptr && handle_data->size() > 1) { + return errors::InvalidArgument( + "Trying to read from list with invalid variant data."); + } + if (IsValidTensorListHandleData(handle_data)) { + const shape_inference::ShapeAndType& list_shape_type = + (*handle_data)[0]; + if (list_shape_type.dtype != element_dtype) { + return errors::InvalidArgument( + "Trying to read from list with wrong element dtype. List has " + "type ", + DataTypeString(list_shape_type.dtype), + " but trying to push element with type ", + DataTypeString(element_dtype)); + } + shape_inference::ShapeHandle ignored; + TF_RETURN_IF_ERROR( + c->Merge(tensor_shape, list_shape_type.shape, &ignored)); + c->set_output_handle_shapes_and_types(0, *handle_data); + tensor_shape = list_shape_type.shape; + } + c->set_output(1, tensor_shape); + c->set_output(0, c->Scalar()); + return Status::OK(); + });*/ + + REGISTER_OP("ZeroOut") .Input("to_zero: int32") .Output("zeroed: int32") diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index a8364c19d8d..726c97a639b 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -39,20 +39,22 @@ from tensorflow.python.ops import map_ops @test_util.run_all_in_graph_and_eager_modes class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): + def testEmptyTensorMap(self): + m = map_ops.empty_tensor_map() + print("empty tensor map created") + + ''' @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 2)) @test_util.run_deprecated_v1 def testEraseFromEmptyTensorMapFails(self, max_num_elements): print("hello world testErase") - m = map_ops.empty_tensor_map( - element_dtype=dtypes.float32, - element_shape=[], - max_num_elements=max_num_elements) + m = map_ops.empty_tensor_map() with self.assertRaisesRegexp(errors.InvalidArgumentError, "Trying to erase from an empty map"): m = map_ops.tensor_map_erase(l, element_dtype=dtypes.float32) self.evaluate(l) - + ''' def testZeroOut(self): print("hello world testZeroOut") diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 03acaa8fb72..21c58fb773d 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -30,17 +30,8 @@ from tensorflow.python.ops.gen_map_ops import * # resource_loader.get_path_to_datafile('_zero_out_ops.so')) #zero_out = zero_out_ops.zero_out -def empty_tensor_map(element_shape, - element_dtype, - max_num_elements=None, - name=None): - if max_num_elements is None: - max_num_elements = -1 - - return gen_map_ops.empty_tensor_map(element_shape, - element_dtype, - max_num_elements, - name) +def empty_tensor_map(): + return gen_map_ops.empty_tensor_map() def zero_out(to_zero): print("Hello World - Python Op") From bed91800b9a1fb7aaef05232576749addb9b0e5d Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 8 Jul 2020 06:10:53 +0000 Subject: [PATCH 0037/1017] working TensorMapInsert op --- tensorflow/core/kernels/map_kernels.cc | 8 ++ tensorflow/core/kernels/map_kernels.h | 115 +++++++++++++++++- tensorflow/core/ops/map_ops.cc | 34 +++--- .../python/kernel_tests/map_ops_test.py | 16 ++- tensorflow/python/ops/map_ops.py | 10 +- 5 files changed, 164 insertions(+), 19 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index 12c932eb83e..cb749c72f7b 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -15,11 +15,19 @@ limitations under the License. #include "tensorflow/core/kernels/map_kernels.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/variant_encode_decode.h" namespace tensorflow { + + REGISTER_KERNEL_BUILDER(Name("EmptyTensorMap").Device(DEVICE_CPU), EmptyTensorMap); +REGISTER_KERNEL_BUILDER(Name("TensorMapSize").Device(DEVICE_CPU), + TensorMapSize); + + + REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp); } \ No newline at end of file diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 1b344718261..41c1a18a728 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -24,11 +24,65 @@ using namespace std; namespace tensorflow { +Status GetInputMap(OpKernelContext* c, int index, const TensorMap** map) { + if (!TensorShapeUtils::IsScalar(c->input(index).shape())) { + return errors::InvalidArgument("Input list must be a scalar saw: ", + c->input(index).shape().DebugString()); + } + const TensorMap* m = c->input(index).scalar()().get(); + if (m == nullptr) { + return errors::InvalidArgument( + "Input handle is not a map. Saw: '", + c->input(index).scalar()().DebugString(), "'"); + } + *map = m; + return Status::OK(); +} + +Status ForwardInputOrCreateNewMap(OpKernelContext* c, int32 input_index, + int32 output_index, + const TensorMap& input_map, + TensorMap** output_map) { + // Attempt to forward the input tensor to the output if possible. + std::unique_ptr maybe_output = c->forward_input( + input_index, output_index, DT_VARIANT, TensorShape{}, + c->input_memory_type(input_index), AllocatorAttributes()); + Tensor* output_tensor; + if (maybe_output != nullptr && maybe_output->dtype() == DT_VARIANT && + maybe_output->NumElements() == 1) { + output_tensor = maybe_output.get(); + TensorMap* tmp_out = output_tensor->scalar()().get(); + if (tmp_out == nullptr) { + return errors::InvalidArgument( + "Expected input ", input_index, " to be a TensorMap but saw ", + output_tensor->scalar()().TypeName()); + } + if (tmp_out->RefCountIsOne()) { + // Woohoo, forwarding succeeded! + c->set_output(output_index, *output_tensor); + *output_map = tmp_out; + return Status::OK(); + } + } + + // If forwarding is not possible allocate a new output tensor and copy + // the `input_list` to it. + AllocatorAttributes attr; + attr.set_on_host(true); + TF_RETURN_IF_ERROR( + c->allocate_output(output_index, {}, &output_tensor, attr)); + output_tensor->scalar()() = input_map.Copy(); + + *output_map = output_tensor->scalar()().get(); + return Status::OK(); +} + class EmptyTensorMap : public OpKernel { public: explicit EmptyTensorMap(OpKernelConstruction* ctx) : OpKernel(ctx) {} void Compute(OpKernelContext* ctx) override { + std::cout << "hello EmptyTensorMap map_kernels.h" << std::endl; Tensor* result; AllocatorAttributes attr; attr.set_on_host(true); @@ -44,17 +98,74 @@ class EmptyTensorMap : public OpKernel { class TensorMapSize : public OpKernel { public: explicit TensorMapSize(OpKernelConstruction* c) : OpKernel(c) {} - ~TEnsorMapSize() override {} + ~TensorMapSize() override {} void Compute(OpKernelContext* c) override { const TensorMap* m = nullptr; - OP_REQUIRES_OK(c, GetInputList(c, 0, &m)); + OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); Tensor* result; OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result)); result->scalar()() = m->tensors().size(); } }; +class TensorMapInsert : public OpKernel { + public: + explicit TensorMapInsert(OpKernelConstruction* c) : OpKernel(c) { + //OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); + } + ~TensorMapInsert() override {} + + void Compute(OpKernelContext* c) override { + std::cout << "hello TensorMapInsert kernel" << std::endl; + const Tensor& temp_key = c->input(1); + const TensorKey key = TensorKey(temp_key); + std::cout << "got key" << std::endl; + const Tensor& value = c->input(2); + std::cout << "got value" << std::endl; + /*OP_REQUIRES(c, element_dtype_ == value.dtype(), + errors::InvalidArgument("Invalid data types; list elements ", + DataTypeString(element_dtype_), + " but tried to append ", + DataTypeString(value.dtype())));*/ + + const TensorMap* m = nullptr; + OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); + std::cout << "got map" << std::endl; + /*OP_REQUIRES(c, m->element_shape.IsCompatibleWith(input.shape()), + errors::InvalidArgument( + "Tried to append a map with incompatible shape to a " + "list. Op element shape: ", + input.shape().DebugString(), + " list shape: ", m->element_shape.DebugString()));*/ + /*OP_REQUIRES(c, element_dtype_ == m->element_dtype, + errors::InvalidArgument("Invalid data types; op elements ", + DataTypeString(element_dtype_), + " but list elements ", + DataTypeString(l->element_dtype))); + + if (l->max_num_elements != -1) { + OP_REQUIRES( + c, l->tensors().size() < l->max_num_elements, + errors::InvalidArgument("Tried to push item into a full list", + " list size: ", l->tensors().size(), + " max_num_elements: ", l->max_num_elements)); + }*/ + + TensorMap* output_map = nullptr; + OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); + std::cout << "create output" << std::endl; + output_map->insert(key, value); + std::cout << "inserted" << std::endl; + } + + private: + DataType element_dtype_; +}; + +REGISTER_KERNEL_BUILDER(Name("TensorMapInsert").Device(DEVICE_CPU), + TensorMapInsert); + class ZeroOutOp : public OpKernel { public: explicit ZeroOutOp(OpKernelConstruction* c) : OpKernel(c) {} diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index ab2fdef9127..463a2ea102b 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -20,6 +20,13 @@ limitations under the License. namespace tensorflow { namespace { +bool IsValidTensorMapHandleData( + const std::vector* handle_data) { + std::cout << "is valid tensor map handle data " << handle_data->size() << std::endl; + return true; + //return handle_data != nullptr && handle_data->size() == 1; +} + REGISTER_OP("EmptyTensorMap") .Output("handle: variant") .SetShapeFn([](shape_inference::InferenceContext* c) { @@ -27,6 +34,11 @@ REGISTER_OP("EmptyTensorMap") return Status::OK(); }); +REGISTER_OP("TensorMapSize") + .Input("input_handle: variant") + .Output("size: int32") + .SetShapeFn(shape_inference::ScalarShape); + REGISTER_OP("TensorMapInsert") .Input("input_handle: variant") .Input("key: element_dtype") @@ -35,18 +47,17 @@ REGISTER_OP("TensorMapInsert") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - DataType element_dtype; + /*DataType element_dtype; TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); - shape_inference::ShapeHandle element_shape = c->UnknownShape(); + shape_inference::ShapeHandle element_shape = c->UnknownShape();*/ - auto* handle_data = c->input_handle_shapes_and_types(0); + /*auto* handle_data = c->input_handle_shapes_and_types(0); if (handle_data != nullptr && handle_data->size() > 1) { return errors::InvalidArgument( "Trying to push to list with wrong variant data."); } - if (IsValidTensorListHandleData(handle_data)) { - const shape_inference::ShapeAndType& list_shape_type = - (*handle_data)[0]; + if (IsValidTensorMapHandleData(handle_data)) { + const shape_inference::ShapeAndType& map_shape_type = (*handle_data)[0]; if (list_shape_type.dtype != element_dtype) { return errors::InvalidArgument( "Trying to push to list with wrong element dtype. List has type ", @@ -56,20 +67,15 @@ REGISTER_OP("TensorMapInsert") } shape_inference::ShapeHandle ignored; TF_RETURN_IF_ERROR( - c->Merge(element_shape, list_shape_type.shape, &ignored)); - element_shape = list_shape_type.shape; + c->Merge(element_shape, map_shape_type.shape, &ignored)); + element_shape = map_shape_type.shape; } c->set_output_handle_shapes_and_types( 0, std::vector{ - {element_shape, element_dtype}}); + {element_shape, element_dtype}});*/ return Status::OK(); }); -REGISTER_OP("TensorMapSize") - .Input("input_handle: variant") - .Output("size: int32") - .SetShapeFn(shape_inference::ScalarShape); - /*REGISTER_OP("TensorMapErase") .Input("input_handle: variant") .Input("element_shape: int32") diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 726c97a639b..6cd6d7d611d 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -41,8 +41,20 @@ from tensorflow.python.ops import map_ops class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testEmptyTensorMap(self): m = map_ops.empty_tensor_map() - print("empty tensor map created") - + print("test EmptyTensorMap") + + def testTensorMapSize(self): + m = map_ops.empty_tensor_map() + s = map_ops.tensor_map_size(m) + print("size: ", s) + + def testTensorMapInsert(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + print("test TensorMapInsert") + ''' @parameterized.named_parameters(("NoMaxNumElements", None), ("WithMaxNumElements", 2)) diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 21c58fb773d..61493be6b71 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -31,10 +31,18 @@ from tensorflow.python.ops.gen_map_ops import * #zero_out = zero_out_ops.zero_out def empty_tensor_map(): + print("hello gen_map_ops.empty_tensor_map") return gen_map_ops.empty_tensor_map() +def tensor_map_size(input_handle): + print("hello gen_map_ops.tensor_map_size") + return gen_map_ops.tensor_map_size(input_handle) + +def tensor_map_insert(input_handle, key, value): + print("hello gen_map_ops.tensor_map_insert") + return gen_map_ops.tensor_map_insert(input_handle, key, value) + def zero_out(to_zero): - print("Hello World - Python Op") return gen_map_ops.zero_out(to_zero) @ops.RegisterGradient("ZeroOut") From 05b4262f6e1a2edbd0b013fcc0f9f44091a9476a Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 8 Jul 2020 18:45:39 +0000 Subject: [PATCH 0038/1017] refactored inputs to kernel as its own struct --- tensorflow/c/kernels/summary_op.cc | 110 ++++++++++++++---------- tensorflow/c/kernels/summary_op_test.cc | 7 +- 2 files changed, 69 insertions(+), 48 deletions(-) diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index cad7bd646ed..ab44a8baae9 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -30,6 +30,33 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include +// Struct that stores the status and TF_Tensor inputs to the opkernel. +// Used to delete tensor and status in its destructor upon kernel return. +typedef struct Params{ + TF_Tensor* tags; + TF_Tensor* values; + TF_Status* status; + Params(TF_OpKernelContext* ctx) { + status = TF_NewStatus(); + TF_GetInput(ctx, 0, &tags, status); + if (TF_GetCode(status) == TF_OK){ + TF_GetInput(ctx, 1, &values, status); + } + else{ + values = nullptr; + } + }; + ~Params(){ + TF_DeleteStatus(status); + TF_DeleteTensor(tags); + // edge case if params fails to initialize + if (values != nullptr){ + TF_DeleteTensor(values); + } + } +}; + +// dummy functions used for kernel registration static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { void* ptr; return ptr; @@ -45,55 +72,48 @@ static tensorflow::string SingleTag(TF_Tensor* tags); template static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { - TF_Tensor* tags; - TF_Tensor* values; - TF_Status* status = TF_NewStatus(); - TF_GetInput(ctx, 0, &tags, status); - if (TF_GetCode(status) == TF_OK) { - TF_GetInput(ctx, 1, &values, status); - } - if (TF_GetCode(status) == TF_OK) { - if (!IsSameSize(tags, values)) { - std::ostringstream err; - err << "tags and values not the same shape: " - << TF_ShapeDebugString(tags) << " != " << TF_ShapeDebugString(values) - << SingleTag(tags); - TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); - } + Params params(ctx); + if (TF_GetCode(params.status) != TF_OK){ + TF_OpKernelContext_Failure(ctx, params.status); + return; + } + if (!IsSameSize(params.tags, params.values)) { + std::ostringstream err; + err << "tags and values not the same shape: " + << TF_ShapeDebugString(params.tags) << " != " + << TF_ShapeDebugString(params.values) + << SingleTag(params.tags); + TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); + } + if (TF_GetCode(params.status) != TF_OK){ + TF_OpKernelContext_Failure(ctx, params.status); + return; } - // Copy tag and string data into summary protobuf - tensorflow::Summary s; - if (TF_GetCode(status) == TF_OK) { - // Convert tags and values tensor to array to access elements by index - auto tags_array = static_cast(TF_TensorData(tags)); - auto values_array = static_cast(TF_TensorData(values)); - // Copy tags and values into summary protobuf - for (int i = 0; i < TF_TensorElementCount(tags); ++i) { - tensorflow::Summary::Value* v = s.add_value(); - const tensorflow::tstring& Ttags_i = tags_array[i]; - v->set_tag(Ttags_i.data(), Ttags_i.size()); - v->set_simple_value(float(values_array[i])); - } - TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, - TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, - sizeof(tensorflow::tstring), status); - if (TF_GetCode(status) == TF_OK) { - tensorflow::tstring summary_tstring; - SerializeToTString(s, &summary_tstring); - TF_TString* output_tf_tstring = reinterpret_cast(TF_TensorData(summary_tensor)); - TF_TString_Init(output_tf_tstring); - tensorflow::tstring* output_tstring = reinterpret_cast(output_tf_tstring); - *output_tstring = summary_tstring; // may want to use std::move - } - TF_DeleteTensor(summary_tensor); + // Convert tags and values tensor to array to access elements by index + tensorflow::Summary s; + auto tags_array = static_cast( + TF_TensorData(params.tags)); + auto values_array = static_cast(TF_TensorData(params.values)); + // Copy tags and values into summary protobuf + for (int i = 0; i < TF_TensorElementCount(params.tags); ++i) { + tensorflow::Summary::Value* v = s.add_value(); + const tensorflow::tstring& Ttags_i = tags_array[i]; + v->set_tag(Ttags_i.data(), Ttags_i.size()); + v->set_simple_value(float(values_array[i])); } - if (TF_GetCode(status) != TF_OK) { - TF_OpKernelContext_Failure(ctx, status); + TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, + TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, + sizeof(tensorflow::tstring), params.status); + if (TF_GetCode(params.status) != TF_OK){ + TF_DeleteTensor(summary_tensor); + TF_OpKernelContext_Failure(ctx, params.status); + return; } - TF_DeleteStatus(status); - TF_DeleteTensor(tags); - TF_DeleteTensor(values); + tensorflow::tstring* output_tstring = reinterpret_cast( + TF_TensorData(summary_tensor)); + SerializeToTString(s, output_tstring); + TF_DeleteTensor(summary_tensor); } bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc index 7438693b430..42f7a7ff3b3 100644 --- a/tensorflow/c/kernels/summary_op_test.cc +++ b/tensorflow/c/kernels/summary_op_test.cc @@ -80,7 +80,6 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, params.inputs = &inputs; OpKernelContext ctx(¶ms, 1); kernel->Compute(&ctx); - ASSERT_EQ(expected_code, ctx.status().code()); if (expected_code == error::OK){ Summary summary; @@ -90,7 +89,7 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, } } -TEST(ScalarSummaryOpTest, Test) { +TEST(ScalarSummaryOpTest, SimpleFloat) { int vectorSize = 3; Tensor tags(DT_STRING, {vectorSize}); Tensor values(DT_FLOAT, {vectorSize}); @@ -104,7 +103,7 @@ TEST(ScalarSummaryOpTest, Test) { value { tag: 'tag1' simple_value: 1.0 } value { tag: 'tag2' simple_value: -0.73} value { tag: 'tag3' simple_value: 10000.0})", error::OK); -} +} TEST(ScalarSummaryOpTest, SimpleDouble) { int vectorSize = 3; @@ -122,6 +121,7 @@ TEST(ScalarSummaryOpTest, SimpleDouble) { value { tag: 'tag3' simple_value: 10000.0})", error::OK); } + TEST(ScalarSummaryOpTest, SimpleHalf) { int vectorSize = 3; Tensor tags(DT_STRING, {vectorSize}); @@ -168,6 +168,7 @@ TEST(ScalarSummaryOpTest, Error_WrongWithSingleTag) { TestScalarSummaryOp(&tags, &values, R"()", error::INVALID_ARGUMENT); } + TEST(ScalarSummaryOpTest, IsRegistered){ const OpRegistrationData* reg; TF_CHECK_OK(OpRegistry::Global()->LookUp("SummaryScalar", ®)); From 9660f6708e7349f73213e2a742a987eb68ca7cb4 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 8 Jul 2020 20:06:53 +0000 Subject: [PATCH 0039/1017] removed diffs for clean PR --- tensorflow/c/kernels.cc | 31 ----- tensorflow/c/kernels.h | 13 -- tensorflow/c/kernels/summary_op.cc | 1 - tensorflow/c/kernels_test.cc | 166 ++++++-------------------- tensorflow/c/tf_tensor.cc | 5 +- tensorflow/core/kernels/summary_op.cc | 2 +- 6 files changed, 39 insertions(+), 179 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 1bd12353031..3021a38e888 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -25,7 +25,6 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/platform/types.h" -#include "tensorflow/core/framework/tensor_shape.h" // This file forms the basis of a stable ABI for third-party kernel // implementations. It is crucial that changes to this file are made cautiously @@ -98,11 +97,6 @@ void TF_KernelBuilder_HostMemory(TF_KernelBuilder* kernel_builder, kernel_builder->cc_builder->HostMemory(arg_name); } -void TF_KernelBuilder_Priority(TF_KernelBuilder* kernel_builder, - int32_t priority_number){ - kernel_builder->cc_builder->Priority(priority_number); -} - namespace tensorflow { namespace { @@ -273,28 +267,3 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, } return tf_tensor; } - -TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, - int64_t* dims, int num_dims, TF_Status* status){ - auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); - TF_SetStatus(status, TF_OK, ""); - tensorflow::TensorShape shape; - for(int i = 0; i < num_dims; ++i){ - shape.AddDim(dims[i]); - } - tensorflow::Status s; - tensorflow::Tensor tensor_temp; - TF_Tensor* tf_tensor_temp; - s = cc_ctx->allocate_temp(static_cast(dtype), shape, &tensor_temp); - if (!s.ok()){ - ::tensorflow::Set_TF_Status_from_Status(status, s); - return nullptr; - } - tf_tensor_temp = TF_TensorFromTensor(tensor_temp, &s); - if (!s.ok()){ - ::tensorflow::Set_TF_Status_from_Status(status, s); - return nullptr; - } - return tf_tensor_temp; -} - diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index 8ed3488988d..084717c1d9e 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -107,10 +107,6 @@ TF_CAPI_EXPORT extern void TF_KernelBuilder_TypeConstraint( TF_CAPI_EXPORT extern void TF_KernelBuilder_HostMemory( TF_KernelBuilder* kernel_builder, const char* arg_name); -// Specify a priority number for this kernel. -TF_CAPI_EXPORT extern void TF_KernelBuilder_Priority( - TF_KernelBuilder* kernel_builder, int32_t priority_number); - // Register the given kernel builder with the TensorFlow runtime. If // registration fails, the given status will be populated. // @@ -194,15 +190,6 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int64_t* dims, int num_dims, size_t len, TF_Status* status); -// Allocates a temporary Tensor of the specified type and shape. Devices -// such as GPUs that enqueue Ops for lazy execution may retain references -// to the temporary tensors after the Op's Compute method has run. - -// num_dims must equal the size of array dims -TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, - TF_DataType dtype, int64_t* dims, int num_dims, TF_Status* status); - - #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index ab44a8baae9..9d28c0797ff 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -28,7 +28,6 @@ limitations under the License. #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/types.h" -#include // Struct that stores the status and TF_Tensor inputs to the opkernel. // Used to delete tensor and status in its destructor upon kernel return. diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index 738c1e12c80..423302741de 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -360,17 +360,6 @@ class DeviceKernelOpTest : public OpsTestBase { #endif }; -// Helper function for tests that validates that the tensor has -// shape and type corresponding to dims and dtype. -void validate_tensor(TF_Tensor* tensor, int64_t* dims, int64_t num_dims, - TF_DataType dtype); - -// Helper function for tests that copies data of length -// tensor_size_bytes from values to tensor -template -void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, - TF_OpKernelContext* ctx); - REGISTER_OP("AllocateOutputOp1").Output("output1: float"); TEST_F(DeviceKernelOpTest, TestAllocateOutputSizeOne) { @@ -382,11 +371,22 @@ TEST_F(DeviceKernelOpTest, TestAllocateOutputSizeOne) { TF_Tensor* output = TF_AllocateOutput( /*context=*/ctx, /*index=*/0, /*dtype=*/TF_FLOAT, /*dims=*/&dim, /*num_dims=*/1, /*len=*/tensor_size_bytes, s); - validate_tensor(output, &dim, 1, TF_FLOAT); - + EXPECT_EQ(TF_OK, TF_GetCode(s)); + EXPECT_EQ(TF_FLOAT, TF_TensorType(output)); + EXPECT_EQ(1, TF_NumDims(output)); + EXPECT_EQ(1, TF_Dim(output, 0)); + // Set output to 3 - float values[1] = {3.0f}; - set_tensor_data(output, values, tensor_size_bytes, ctx); + float* data = reinterpret_cast(TF_TensorData(output)); + float value = 3.0f; +#if GOOGLE_CUDA + OpKernelContext* cc_ctx = reinterpret_cast(ctx); + cc_ctx->eigen_gpu_device().memcpyHostToDevice(data, &value, + tensor_size_bytes); +#else + *data = value; +#endif + TF_DeleteStatus(s); TF_DeleteTensor(output); }; @@ -409,8 +409,12 @@ TEST_F(DeviceKernelOpTest, TestAllocateEmptyOutput) { TF_Tensor* output = TF_AllocateOutput( /*context=*/ctx, /*index=*/0, /*dtype=*/TF_FLOAT, /*dims=*/&dim, /*num_dims=*/1, /*len=*/0, s); + EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, &dim, 1, TF_FLOAT); + EXPECT_EQ(TF_FLOAT, TF_TensorType(output)); + EXPECT_EQ(1, TF_NumDims(output)); + EXPECT_EQ(0, TF_Dim(output, 0)); + TF_DeleteStatus(s); TF_DeleteTensor(output); }; @@ -430,16 +434,27 @@ TEST_F(DeviceKernelOpTest, TestAllocateOutputSize2x3) { TF_Status* s = TF_NewStatus(); // Allocate 2x3 output int64_t dim[2] = {2, 3}; - size_t tensor_size_bytes = TF_DataTypeSize(TF_FLOAT) * 6; + size_t tensor_size_bytes = 6 * TF_DataTypeSize(TF_FLOAT); TF_Tensor* output = TF_AllocateOutput( /*context=*/ctx, /*index=*/0, /*dtype=*/TF_FLOAT, /*dims=*/dim, /*num_dims=*/2, /*len=*/tensor_size_bytes, s); EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, dim, 2, TF_FLOAT); + EXPECT_EQ(TF_FLOAT, TF_TensorType(output)); + EXPECT_EQ(2, TF_NumDims(output)); + EXPECT_EQ(2, TF_Dim(output, 0)); + EXPECT_EQ(3, TF_Dim(output, 1)); // Set output to [1 2 3 4 5 6] - float values[6] = {1, 2, 3, 4, 5, 6}; - set_tensor_data(output, values, tensor_size_bytes, ctx); + void* data = TF_TensorData(output); + float value[6] = {1, 2, 3, 4, 5, 6}; +#if GOOGLE_CUDA + OpKernelContext* cc_ctx = reinterpret_cast(ctx); + cc_ctx->eigen_gpu_device().memcpyHostToDevice(data, value, + tensor_size_bytes); +#else + memcpy(data, value, tensor_size_bytes); +#endif + TF_DeleteStatus(s); TF_DeleteTensor(output); }; @@ -451,113 +466,4 @@ TEST_F(DeviceKernelOpTest, TestAllocateOutputSize2x3) { EXPECT_EQ("Tensor", output->DebugString(100)); } - -REGISTER_OP("AllocateTempOp1").Output("output1: float"); - -TEST_F(DeviceKernelOpTest, TestAllocateTempSizeOne) { - auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { - // Allocate output - TF_Status* s = TF_NewStatus(); - int64_t dim = 1; - TF_Tensor* output = TF_AllocateTemp( - /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, s); - size_t tensor_size_bytes = TF_DataTypeSize(TF_FLOAT); - EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, &dim, 1, TF_FLOAT); - - // Set output to 3 - float values[1] = {3.0f}; - set_tensor_data(output, values, tensor_size_bytes, ctx); - TF_SetOutput(ctx, 0, output, s); - TF_DeleteStatus(s); - TF_DeleteTensor(output); - }; - - SetupOp("AllocateTempOp1", "AllocateTemp1", my_compute_func); - - TF_ASSERT_OK(RunOpKernel()); - Tensor* output = GetOutput(0); - EXPECT_EQ("Tensor", - output->DebugString(100)); -} - -REGISTER_OP("AllocateTempOp0").Output("output1: float"); - -TEST_F(DeviceKernelOpTest, TestAllocateTempEmpty) { - auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { - TF_Status* s = TF_NewStatus(); - // Allocate empty output - int64_t dim = 0; - TF_Tensor* output = TF_AllocateTemp( - /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, s); - EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, &dim, 1, TF_FLOAT); - TF_SetOutput(ctx, 0, output, s); - TF_DeleteStatus(s); - TF_DeleteTensor(output); - }; - - SetupOp("AllocateTempOp0", "AllocateTemp0", my_compute_func); - - TF_ASSERT_OK(RunOpKernel()); - Tensor* output = GetOutput(0); - EXPECT_EQ("Tensor", - output->DebugString(100)); -} - -REGISTER_OP("AllocateTempOp2x3").Output("output1: float"); - -TEST_F(DeviceKernelOpTest, TestAllocateTempSize2x3) { - auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { - TF_Status* s = TF_NewStatus(); - size_t tensor_size_bytes = 6 * TF_DataTypeSize(TF_FLOAT); - // Allocate 2x3 output - int64_t dim[2] = {2, 3}; - TF_Tensor* output = TF_AllocateTemp( - /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/dim, - /*num_dims=*/2, s); - EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, dim, 2, TF_FLOAT); - - // Set output to [1 2 3 4 5 6] - void* data = TF_TensorData(output); - float values[6] = {1, 2, 3, 4, 5, 6}; - set_tensor_data(output, values, tensor_size_bytes, ctx); - TF_SetOutput(ctx, 0, output, s); - TF_DeleteStatus(s); - TF_DeleteTensor(output); - }; - - SetupOp("AllocateTempOp2x3", "AllocateTempOp2x3", my_compute_func); - - TF_ASSERT_OK(RunOpKernel()); - Tensor* output = GetOutput(0); - EXPECT_EQ("Tensor", - output->DebugString(100)); -} - -void validate_tensor(TF_Tensor* tensor, int64_t* dims, int64_t num_dims, - TF_DataType dtype){ - EXPECT_EQ(TF_FLOAT, TF_TensorType(tensor)); - EXPECT_EQ(num_dims, TF_NumDims(tensor)); - for(int i = 0; i < num_dims; ++i){ - EXPECT_EQ(dims[i], TF_Dim(tensor, i)); - } -} - -template -void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, - TF_OpKernelContext* ctx){ - T* data = reinterpret_cast(TF_TensorData(tensor)); -#if GOOGLE_CUDA - OpKernelContext* cc_ctx = reinterpret_cast(ctx); - cc_ctx->eigen_gpu_device().memcpyHostToDevice(data, values, - tensor_size_bytes); -#else - memcpy(data, values, tensor_size_bytes); -#endif -} - -} // namespace tensorflow \ No newline at end of file +} // namespace tensorflow diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc index 5cfd495933c..aa65cb7c927 100644 --- a/tensorflow/c/tf_tensor.cc +++ b/tensorflow/c/tf_tensor.cc @@ -293,6 +293,7 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, Status* status) { std::memcpy(TF_TensorData(t), str.c_str(), str.size()); return t; } + Tensor tensor; if (!tensor.CopyFrom(src, src.shape())) { return nullptr; @@ -329,6 +330,4 @@ bool TensorInterface::IsAligned() const { return tensor_.IsAligned(); } } // namespace tensorflow -bool TF_TensorIsAligned(const TF_Tensor* t) { return t->tensor->IsAligned(); } - - +bool TF_TensorIsAligned(const TF_Tensor* t) { return t->tensor->IsAligned(); } \ No newline at end of file diff --git a/tensorflow/core/kernels/summary_op.cc b/tensorflow/core/kernels/summary_op.cc index 64e8347dfc4..f4c91fc9ff1 100644 --- a/tensorflow/core/kernels/summary_op.cc +++ b/tensorflow/core/kernels/summary_op.cc @@ -39,7 +39,7 @@ class SummaryScalarOp : public OpKernel { void Compute(OpKernelContext* c) override { const Tensor& tags = c->input(0); const Tensor& values = c->input(1); - string tag = SingleTag(tags); + OP_REQUIRES( c, tags.IsSameSize(values) || (TensorShapeUtils::IsScalar(tags.shape()) && From 0d7c20446047dfcebd44a2f143a32b27c652e9b7 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 9 Jul 2020 20:36:38 +0000 Subject: [PATCH 0040/1017] removed GPU support --- tensorflow/c/kernels/BUILD | 15 +++++++++++++++ tensorflow/c/kernels/summary_op.cc | 12 ------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index e34e3a55e79..309dc7e221b 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -87,6 +87,21 @@ tf_cc_test( "//tensorflow/core:testlib", ], ) + +tf_cc_test( + name = "summary_op_benchmark_test", + srcs = ["summary_op_benchmark_test.cc"], + deps = [ + "summary_op", + "summary_op_lib", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) # Changes to the Android srcs here should be replicated in # tensorflow/contrib/makefile/tf_op_files.txt. # diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 9d28c0797ff..cd2509247da 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -155,18 +155,6 @@ void RegisterSummaryScalarOpKernel() { CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while registering Summary Scalar kernel"; } -#if GOOGLE_CUDA - { - auto* builder = TF_NewKernelBuilder("SummaryScalar", - tensorflow::DEVICE_GPU, - &SummaryScalarOp_Create, - &SummaryScalarOp_Compute, - &SummaryScalarOp_Delete); - TF_RegisterKernelBuilder("SummaryScalar", builder, status); - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while registering CUDA SummaryScalar kernel"; - } -#endif TF_DeleteStatus(status); } From 609fe7990f8311191c6bffc1ee47034443ac319a Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 10 Jul 2020 01:25:03 +0000 Subject: [PATCH 0041/1017] insert, lookup, erase, replace, and initial gradients --- tensorflow/core/kernels/map_kernels.cc | 4 + tensorflow/core/kernels/map_kernels.h | 139 +++++++++++++++++- tensorflow/core/kernels/tensor_map.h | 12 +- tensorflow/core/ops/map_ops.cc | 88 ++++++++++- .../python/kernel_tests/list_ops_test.py | 12 +- .../python/kernel_tests/map_ops_test.py | 65 +++++++- tensorflow/python/ops/map_ops.py | 27 ++++ 7 files changed, 326 insertions(+), 21 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index cb749c72f7b..eba2d99a75b 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -26,7 +26,11 @@ REGISTER_KERNEL_BUILDER(Name("EmptyTensorMap").Device(DEVICE_CPU), REGISTER_KERNEL_BUILDER(Name("TensorMapSize").Device(DEVICE_CPU), TensorMapSize); +REGISTER_KERNEL_BUILDER(Name("TensorMapLookup").Device(DEVICE_CPU), + TensorMapLookup); +REGISTER_KERNEL_BUILDER(Name("TensorMapErase").Device(DEVICE_CPU), + TensorMapErase); REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), ZeroOutOp); diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 41c1a18a728..f67b8b6e10a 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -112,7 +112,7 @@ class TensorMapSize : public OpKernel { class TensorMapInsert : public OpKernel { public: explicit TensorMapInsert(OpKernelConstruction* c) : OpKernel(c) { - //OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); + OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); } ~TensorMapInsert() override {} @@ -151,7 +151,6 @@ class TensorMapInsert : public OpKernel { " list size: ", l->tensors().size(), " max_num_elements: ", l->max_num_elements)); }*/ - TensorMap* output_map = nullptr; OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); std::cout << "create output" << std::endl; @@ -166,6 +165,142 @@ class TensorMapInsert : public OpKernel { REGISTER_KERNEL_BUILDER(Name("TensorMapInsert").Device(DEVICE_CPU), TensorMapInsert); +class TensorMapLookup : public OpKernel { + public: + explicit TensorMapLookup(OpKernelConstruction* c) : OpKernel(c) { + OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); + } + ~TensorMapLookup() override {} + + void Compute(OpKernelContext* c) override { + std::cout << "hello TensorMapInsert kernel" << std::endl; + const Tensor& temp_key = c->input(1); + const TensorKey key = TensorKey(temp_key); + std::cout << "got key" << std::endl; + const TensorMap* m = nullptr; + OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); + std::cout << "got map" << std::endl; + c->set_output(0, m->tensors().find(key)->second); + std::cout << "finished" << std::endl; + } + + private: + DataType element_dtype_; +}; + +class TensorMapErase : public OpKernel { + public: + explicit TensorMapErase(OpKernelConstruction* c) : OpKernel(c) { + OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); + } + + void Compute(OpKernelContext* c) override { + std::cout << "hello TensorMapErase op" << std::endl; + const TensorMap* m = nullptr; + OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); + const Tensor& temp_key = c->input(1); + const TensorKey key = TensorKey(temp_key); + /*OP_REQUIRES(c, element_dtype_ == l->element_dtype, + errors::InvalidArgument("Invalid data types; op elements ", + DataTypeString(element_dtype_), + " but list elements ", + DataTypeString(l->element_dtype)));*/ + + OP_REQUIRES(c, !m->tensors().empty(), + errors::InvalidArgument("Trying to erase from an empty map.")); + + OP_REQUIRES(c, m->tensors().find(key) != m->tensors().end(), + errors::InvalidArgument("Trying to erase non-existent item.")); + + const Tensor& t = m->tensors().find(key)->second; + c->set_output(1, t); + /*if (t.dtype() != DT_INVALID) { + c->set_output(1, t); + } else { + PartialTensorShape partial_element_shape; + OP_REQUIRES_OK( + c, GetElementShapeFromInput(c, *l, 1, &partial_element_shape)); + TensorShape element_shape; + OP_REQUIRES( + c, partial_element_shape.AsTensorShape(&element_shape), + errors::InvalidArgument("Trying to read an uninitialized tensor but ", + "element_shape is not fully defined.", + partial_element_shape.DebugString())); + Tensor* result; + AllocatorAttributes attr; + if (element_dtype_ == DT_VARIANT) { + attr.set_on_host(true); + } + OP_REQUIRES_OK(c, c->allocate_output(1, element_shape, &result, attr)); + functor::SetZeroFunctor()(c->eigen_device(), + result->flat()); + }*/ + + TensorMap* output_map = nullptr; + OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); + output_map->tensors().erase(key); + } + + private: + DataType element_dtype_; +}; + +class TensorMapReplace : public OpKernel { + public: + explicit TensorMapReplace(OpKernelConstruction* c) : OpKernel(c) { + OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); + } + ~TensorMapReplace() override {} + + void Compute(OpKernelContext* c) override { + std::cout << "hello TensorMapReplace kernel" << std::endl; + const Tensor& temp_key = c->input(1); + const TensorKey key = TensorKey(temp_key); + std::cout << "got key" << std::endl; + const Tensor& value = c->input(2); + std::cout << "got value" << std::endl; + /*OP_REQUIRES(c, element_dtype_ == value.dtype(), + errors::InvalidArgument("Invalid data types; list elements ", + DataTypeString(element_dtype_), + " but tried to append ", + DataTypeString(value.dtype())));*/ + + const TensorMap* m = nullptr; + OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); + std::cout << "got map" << std::endl; + /*OP_REQUIRES(c, m->element_shape.IsCompatibleWith(input.shape()), + errors::InvalidArgument( + "Tried to append a map with incompatible shape to a " + "list. Op element shape: ", + input.shape().DebugString(), + " list shape: ", m->element_shape.DebugString()));*/ + /*OP_REQUIRES(c, element_dtype_ == m->element_dtype, + errors::InvalidArgument("Invalid data types; op elements ", + DataTypeString(element_dtype_), + " but list elements ", + DataTypeString(l->element_dtype))); + + if (l->max_num_elements != -1) { + OP_REQUIRES( + c, l->tensors().size() < l->max_num_elements, + errors::InvalidArgument("Tried to push item into a full list", + " list size: ", l->tensors().size(), + " max_num_elements: ", l->max_num_elements)); + }*/ + TensorMap* output_map = nullptr; + OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); + std::cout << "create output" << std::endl; + output_map->replace(key,value); + std::cout << "inserted" << std::endl; + } + + private: + DataType element_dtype_; +}; + +REGISTER_KERNEL_BUILDER(Name("TensorMapReplace").Device(DEVICE_CPU), + TensorMapReplace); + class ZeroOutOp : public OpKernel { public: explicit ZeroOutOp(OpKernelConstruction* c) : OpKernel(c) {} diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index c5993ec9300..a5d44550c98 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -145,7 +145,7 @@ class TensorMap { // Insert key and value if the key does not already exist. // Returns true if the insertion happens. - bool insert(TensorKey key, Tensor value) { + bool insert(const TensorKey& key, const Tensor& value) { auto r = tensors_->values_.try_emplace(key, value); return r.second; } @@ -155,9 +155,19 @@ class TensorMap { return tensors_->values_.find(key); } + Tensor& lookup(TensorKey key) { + return tensors_->values_.find(key)->second; + } + Tensor& operator[](TensorKey& k) { return tensors_->values_[k]; } + + bool replace(const TensorKey& k, const Tensor& v) { + tensors_->values_[k] = v; + return true; + } + // Removes element with given key. Return size of removed element. size_t erase(TensorKey key) { return tensors_->values_.erase(key); diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index 463a2ea102b..09183e715ea 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -76,14 +76,50 @@ REGISTER_OP("TensorMapInsert") return Status::OK(); }); -/*REGISTER_OP("TensorMapErase") +REGISTER_OP("TensorMapLookup") .Input("input_handle: variant") - .Input("element_shape: int32") + .Input("key: element_dtype") + .Output("value: element_dtype") + .Attr("element_dtype: type") + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->Scalar()); + /*DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle element_shape = c->UnknownShape();*/ + + /*auto* handle_data = c->input_handle_shapes_and_types(0); + if (handle_data != nullptr && handle_data->size() > 1) { + return errors::InvalidArgument( + "Trying to push to list with wrong variant data."); + } + if (IsValidTensorMapHandleData(handle_data)) { + const shape_inference::ShapeAndType& map_shape_type = (*handle_data)[0]; + if (list_shape_type.dtype != element_dtype) { + return errors::InvalidArgument( + "Trying to push to list with wrong element dtype. List has type ", + DataTypeString(list_shape_type.dtype), + " but trying to push element with type ", + DataTypeString(element_dtype)); + } + shape_inference::ShapeHandle ignored; + TF_RETURN_IF_ERROR( + c->Merge(element_shape, map_shape_type.shape, &ignored)); + element_shape = map_shape_type.shape; + } + c->set_output_handle_shapes_and_types( + 0, std::vector{ + {element_shape, element_dtype}});*/ + return Status::OK(); + }); + +REGISTER_OP("TensorMapErase") + .Input("input_handle: variant") + .Input("key: element_dtype") .Output("output_handle: variant") .Output("tensor: element_dtype") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { - DataType element_dtype; + /*DataType element_dtype; TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); shape_inference::ShapeHandle tensor_shape = c->UnknownShape(); auto* handle_data = c->input_handle_shapes_and_types(0); @@ -107,11 +143,49 @@ REGISTER_OP("TensorMapInsert") c->Merge(tensor_shape, list_shape_type.shape, &ignored)); c->set_output_handle_shapes_and_types(0, *handle_data); tensor_shape = list_shape_type.shape; - } - c->set_output(1, tensor_shape); - c->set_output(0, c->Scalar()); + }*/ + c->set_output(1, c->Scalar()); // removed element + c->set_output(0, c->Scalar()); // map return Status::OK(); - });*/ + }); + +REGISTER_OP("TensorMapReplace") + .Input("input_handle: variant") + .Input("key: element_dtype") + .Input("value: element_dtype") + .Output("output_handle: variant") + .Attr("element_dtype: type") + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->Scalar()); + /*DataType element_dtype; + TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); + shape_inference::ShapeHandle element_shape = c->UnknownShape();*/ + + /*auto* handle_data = c->input_handle_shapes_and_types(0); + if (handle_data != nullptr && handle_data->size() > 1) { + return errors::InvalidArgument( + "Trying to push to list with wrong variant data."); + } + if (IsValidTensorMapHandleData(handle_data)) { + const shape_inference::ShapeAndType& map_shape_type = (*handle_data)[0]; + if (list_shape_type.dtype != element_dtype) { + return errors::InvalidArgument( + "Trying to push to list with wrong element dtype. List has type ", + DataTypeString(list_shape_type.dtype), + " but trying to push element with type ", + DataTypeString(element_dtype)); + } + shape_inference::ShapeHandle ignored; + TF_RETURN_IF_ERROR( + c->Merge(element_shape, map_shape_type.shape, &ignored)); + element_shape = map_shape_type.shape; + } + c->set_output_handle_shapes_and_types( + 0, std::vector{ + {element_shape, element_dtype}});*/ + return Status::OK(); + }); + REGISTER_OP("ZeroOut") diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py index 53ebdd3ab88..4d7f2beb00b 100644 --- a/tensorflow/python/kernel_tests/list_ops_test.py +++ b/tensorflow/python/kernel_tests/list_ops_test.py @@ -48,7 +48,7 @@ from tensorflow.python.platform import test @test_util.run_all_in_graph_and_eager_modes class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): - + ''' def _testPushPop(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, @@ -130,7 +130,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): _, e = gen_list_ops.tensor_list_pop_back( l, element_dtype=dtypes.float32, element_shape=[1, 3]) self.evaluate(e) - + ''' def testPushGetGrad(self): with backprop.GradientTape() as tape: l = list_ops.empty_tensor_list( @@ -150,7 +150,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): dt0, dt1 = tape.gradient(t1, [c0, c1]) self.assertAllEqual(self.evaluate(dt1), [1.0, 1.0]) self.assertEqual(self.evaluate(dt0), 0.0) - + ''' def _testStack(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, @@ -888,7 +888,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l_worker = array_ops.identity(l_ps) l_worker = list_ops.tensor_list_push_back(l_worker, 3.0) self.evaluate(l_worker) - + ''' def testPushPopGradients(self): with backprop.GradientTape() as tape: l = list_ops.empty_tensor_list( @@ -925,7 +925,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): grad_c, grad_c2 = tape.gradient(y, [c, c2]) self.assertAllEqual(self.evaluate(grad_c), [0.0, 4.0]) self.assertAllEqual(self.evaluate(grad_c2), 6.0) - + ''' @test_util.run_deprecated_v1 def testSetOutOfBounds(self): c = constant_op.constant([1.0, 2.0]) @@ -1664,7 +1664,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): upper, constant_op.constant([0, 1, 2]), dtype=dtypes.string) self.assertAllEqual(f(), [b"A", b"B", b"C"]) - + ''' def testPopBackGrad(self): # https://github.com/tensorflow/tensorflow/issues/37230 diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 6cd6d7d611d..85ca558af4f 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -17,7 +17,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import numpy as np +#import numpy as np from tensorflow.python.platform import test from absl.testing import parameterized @@ -39,7 +39,7 @@ from tensorflow.python.ops import map_ops @test_util.run_all_in_graph_and_eager_modes class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): - def testEmptyTensorMap(self): + '''def testEmptyTensorMap(self): m = map_ops.empty_tensor_map() print("test EmptyTensorMap") @@ -47,13 +47,68 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): m = map_ops.empty_tensor_map() s = map_ops.tensor_map_size(m) print("size: ", s) + self.assertAllClose(s, 0) def testTensorMapInsert(self): + #with self.test_session(): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + s = map_ops.tensor_map_size(m) + self.assertAllClose(s, 1) + print("test TensorMapInsert") + + def testTensorMapLookup(self): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k, v) - print("test TensorMapInsert") + l = map_ops.tensor_map_lookup(m, k) + print("lookup: ", l) + self.assertAllClose(l, v)''' + + def testTensorMapReplace(self): + #with self.test_session(): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + s = map_ops.tensor_map_size(m) + self.assertAllClose(s, 1) + + v2 = constant_op.constant(3.0) + m = map_ops.tensor_map_replace(m, k, v2) + l = map_ops.tensor_map_lookup(m, k) + self.assertAllClose(l, v2) + print("test TensorMapReplace") + + def testTensorMapErase(self): + print("python erase") + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + s = map_ops.tensor_map_size(m) + self.assertAllClose(s, 1) + m, e = map_ops.tensor_map_erase(m, k) + s = map_ops.tensor_map_size(m) + print("erase: ", e) + self.assertAllClose(s, 0) + self.assertAllClose(e, v) + + def testInsertLookupGrad(self): + with backprop.GradientTape() as tape: + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + tape.watch(v) + m = map_ops.tensor_map_insert(m, k, v) + l = map_ops.tensor_map_lookup(m, k) + l *= 5 + #print("gradient", tape.gradient(l, v), 2.0) + + ''' @parameterized.named_parameters(("NoMaxNumElements", None), @@ -68,11 +123,11 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.evaluate(l) ''' - def testZeroOut(self): + '''def testZeroOut(self): print("hello world testZeroOut") with self.test_session(): self.assertAllClose( - map_ops.zero_out([[1, 2], [3, 4]]), np.array([[1, 0], [0, 0]])) + map_ops.zero_out([[1, 2], [3, 4]]), np.array([[1, 0], [0, 0]]))''' if __name__ == '__main__': diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 61493be6b71..5cb045b5406 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -30,6 +30,8 @@ from tensorflow.python.ops.gen_map_ops import * # resource_loader.get_path_to_datafile('_zero_out_ops.so')) #zero_out = zero_out_ops.zero_out +ops.NotDifferentiable("EmptyTensorMap") + def empty_tensor_map(): print("hello gen_map_ops.empty_tensor_map") return gen_map_ops.empty_tensor_map() @@ -42,6 +44,31 @@ def tensor_map_insert(input_handle, key, value): print("hello gen_map_ops.tensor_map_insert") return gen_map_ops.tensor_map_insert(input_handle, key, value) +def tensor_map_lookup(input_handle, key): + return gen_map_ops.tensor_map_lookup(input_handle, key) + +def tensor_map_erase(input_handle, key): + return gen_map_ops.tensor_map_erase(input_handle, key) + +def tensor_map_replace(input_handle, key, value): + return gen_map_ops.tensor_map_replace(input_handle, key, value) + +@ops.RegisterGradient("TensorMapLookup") +def LookupGrad(op, dval): + map_grad = None + key_grad = None + key = op.inputs[1] + value_grad = tensor_map_lookup(dmap, key) + return map_grad, key_grad + +@ops.RegisterGradient("TensorMapInsert") +def InsertGrad(op, dmap): + map_grad, _ = gen_map_ops.tensor_map_erase(dmap, key) + key_grad = None + key = op.inputs[1] + value_grad = tensor_map_lookup(dmap, key) + return map_grad, key_grad, value_grad + def zero_out(to_zero): return gen_map_ops.zero_out(to_zero) From 6e89b83f8643943c38dd413d1ebcdab28a715be5 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 10 Jul 2020 20:26:57 +0000 Subject: [PATCH 0042/1017] working gradient! --- tensorflow/core/kernels/map_kernels.h | 40 +++++++++++++++++++ tensorflow/core/kernels/tensor_map.h | 23 +++++++++++ tensorflow/core/kernels/tensor_map_test.cc | 24 +++++++++++ tensorflow/core/ops/map_ops.cc | 9 +++++ .../python/kernel_tests/map_ops_test.py | 4 +- tensorflow/python/ops/list_ops.py | 11 +++++ tensorflow/python/ops/map_ops.py | 14 +++++-- 7 files changed, 120 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index f67b8b6e10a..78040359026 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -109,6 +109,46 @@ class TensorMapSize : public OpKernel { } }; +class TensorMapZeros : public OpKernel { + public: + explicit TensorMapZeros(OpKernelConstruction* c) : OpKernel(c) { + //OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); + } + ~TensorMapZeros() override {} + + void Compute(OpKernelContext* c) override { + std::cout << "hello TensorMapInsert kernel" << std::endl; + const Tensor& temp_key = c->input(1); + const TensorKey key = TensorKey(temp_key); + std::cout << "got key" << std::endl; + const Tensor& value = c->input(2); + std::cout << "got value" << std::endl; + + const TensorMap* m = nullptr; + OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); + std::cout << "got map" << std::endl; + //TensorMap output_map; + //OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); + //std::cout << "create output" << std::endl; + //output_map = m->Zeros(); + //c->set_output(0, &&output_map); + //std::cout << "inserted" << std::endl; + + Tensor* result; + AllocatorAttributes attr; + attr.set_on_host(true); + OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr)); + TensorMap output_map = m->Zeros(); + result->scalar()() = std::move(output_map); + } + + private: + DataType element_dtype_; +}; + +REGISTER_KERNEL_BUILDER(Name("TensorMapZeros").Device(DEVICE_CPU), + TensorMapZeros); + class TensorMapInsert : public OpKernel { public: explicit TensorMapInsert(OpKernelConstruction* c) : OpKernel(c) { diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index a5d44550c98..7ab792b4813 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -143,6 +143,29 @@ class TensorMap { return out; } + TensorMap Zeros() const { + TensorMap out; + out.element_shape = element_shape; + out.element_dtype = element_dtype; + out.max_num_elements = max_num_elements; + // This performs a copy of the absl::hashmap. + absl::flat_hash_map::iterator it = tensors_->values_.begin(); + while(it != tensors_->values_.end()) { + out.tensors_->values_.try_emplace(it->first, Tensor(0)); + it++; + } + return out; + } + std::vector keys() { + std::vector keys(tensors_->values_.size()); + absl::flat_hash_map::iterator it = tensors_->values_.begin(); + while(it != tensors_->values_.end()) { + keys.push_back((Tensor)it->first); + it++; + } + return keys; + } + // Insert key and value if the key does not already exist. // Returns true if the insertion happens. bool insert(const TensorKey& key, const Tensor& value) { diff --git a/tensorflow/core/kernels/tensor_map_test.cc b/tensorflow/core/kernels/tensor_map_test.cc index b93171b4f70..5774a605bbf 100644 --- a/tensorflow/core/kernels/tensor_map_test.cc +++ b/tensorflow/core/kernels/tensor_map_test.cc @@ -136,6 +136,30 @@ TEST(TensorMapTest, EncodeDecode) { EXPECT_EQ(tm.find(k)->first, tmc.find(k)->first); test::ExpectTensorEqual(tm.find(k)->second, tmc.find(k)->second); } + +TEST(TensorMapTest, Keys) { + TensorMap tm; + TensorKey k = Tensor(11); + TensorKey k2 = Tensor(12); + Tensor v = Tensor(22); + tm.insert(k,v); + tm.insert(k2,v); + std::vector keys = tm.keys(); + EXPECT_EQ(1,1); + Tensor t = Tensor(11); + //std::cout << "keys: " << keys[0] << std::endl; + //test::ExpectTensorEqual(keys[0], t); + //test::ExpectTensorEqual(keys[1], k2); +} + +TEST(TensorMapTest, Zeros) { + TensorMap tm; + TensorKey k = Tensor(11); + Tensor v = Tensor(22); + tm.insert(k,v); + TensorMap z = tm.Zeros(); + test::ExpectTensorEqual(z.find(k)->second,Tensor(0)); +} } // namespace } // namespace tensorflow diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index 09183e715ea..d3711755d9e 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -39,6 +39,15 @@ REGISTER_OP("TensorMapSize") .Output("size: int32") .SetShapeFn(shape_inference::ScalarShape); +REGISTER_OP("TensorMapZeros") + .Input("input_handle: variant") + .Output("output_handle: variant") + //.Attr("element_dtype: type") + .SetShapeFn([](shape_inference::InferenceContext* c) { + c->set_output(0, c->Scalar()); + return Status::OK(); + }); + REGISTER_OP("TensorMapInsert") .Input("input_handle: variant") .Input("key: element_dtype") diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 85ca558af4f..9384571dc2b 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -106,7 +106,9 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): m = map_ops.tensor_map_insert(m, k, v) l = map_ops.tensor_map_lookup(m, k) l *= 5 - #print("gradient", tape.gradient(l, v), 2.0) + g= tape.gradient(l,v) + print("gradient",g) + self.assertAllClose(g, 5.0) diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py index 3e7c116ec97..ccd4e6b0494 100644 --- a/tensorflow/python/ops/list_ops.py +++ b/tensorflow/python/ops/list_ops.py @@ -248,6 +248,7 @@ def _TensorListFromTensorGrad(op, dlist): @ops.RegisterGradient("TensorListGetItem") def _TensorListGetItemGrad(op, ditem): """Gradient for TensorListGetItem.""" + print("---GetItemGrad---") list_size = gen_list_ops.tensor_list_length(op.inputs[0]) list_grad = gen_list_ops.tensor_list_set_item( gen_list_ops.tensor_list_reserve( @@ -256,14 +257,21 @@ def _TensorListGetItemGrad(op, ditem): list_size, element_dtype=ditem.dtype), index=op.inputs[1], item=ditem) + print("op inputs", op.inputs) + print("ditem", ditem) + print("list_grad", list_grad) index_grad = None element_shape_grad = None + print("------") return list_grad, index_grad, element_shape_grad @ops.RegisterGradient("TensorListSetItem") def _TensorListSetItemGrad(op, dlist): """Gradient function for TensorListSetItem.""" + print("---SetItemGrad---") + print("op inputs", op.inputs) + print("dlist", dlist) _, index, item = op.inputs list_grad = gen_list_ops.tensor_list_set_item( dlist, index=index, item=array_ops.zeros_like(item)) @@ -273,6 +281,9 @@ def _TensorListSetItemGrad(op, dlist): index, element_shape=array_ops.shape(item), element_dtype=item.dtype) + print("list_grad", list_grad) + print("value_grad", element_grad) + print("------") return list_grad, index_grad, element_grad diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 5cb045b5406..44894a8b6d9 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -25,6 +25,8 @@ from tensorflow.python.platform import resource_loader from tensorflow.python.framework import ops from tensorflow.python.ops import gen_map_ops from tensorflow.python.ops.gen_map_ops import * +from tensorflow.python.framework import constant_op + #zero_out_ops = load_library.load_op_library( # resource_loader.get_path_to_datafile('_zero_out_ops.so')) @@ -53,20 +55,24 @@ def tensor_map_erase(input_handle, key): def tensor_map_replace(input_handle, key, value): return gen_map_ops.tensor_map_replace(input_handle, key, value) + @ops.RegisterGradient("TensorMapLookup") def LookupGrad(op, dval): - map_grad = None - key_grad = None + # map grad should be a map that is 0 everywhere except 1 @key k + m, k = op.inputs + #m = gen_map_ops.tensor_map_zeros(m) + map_grad = tensor_map_replace(m, k, dval) key = op.inputs[1] - value_grad = tensor_map_lookup(dmap, key) + key_grad = None return map_grad, key_grad @ops.RegisterGradient("TensorMapInsert") def InsertGrad(op, dmap): + _, key, val = op.inputs map_grad, _ = gen_map_ops.tensor_map_erase(dmap, key) key_grad = None - key = op.inputs[1] value_grad = tensor_map_lookup(dmap, key) + #value_grad = constant_op.constant(1.0) return map_grad, key_grad, value_grad def zero_out(to_zero): From e3504b3b421a3614f88e917e9472ba1f71b03dd5 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 10 Jul 2020 21:23:36 +0000 Subject: [PATCH 0043/1017] changed BUILD file --- tensorflow/c/kernels/BUILD | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 770352c62c1..45102ad6160 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -48,6 +48,31 @@ tf_cc_test( ], ) +tf_kernel_library( + name = "histogram_summary_op", + prefix = "histogram_summary_op", + deps = [ + "//tensorflow/c:kernels", + "//tensorflow/c:ops", + "//tensorflow/c:tf_datatype", + "//tensorflow/c:tf_status", + "//tensorflow/c:tf_tensor", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + ], +) + +tf_gen_op_libs( + op_lib_names = ["histogram"], + deps = [ + "//tensorflow/c:ops", + "//tensorflow/c:tf_datatype", + "//tensorflow/c:tf_status", + "//tensorflow/c:tf_tensor", + "//tensorflow/core:lib", + ], +) + # Changes to the Android srcs here should be replicated in # tensorflow/contrib/makefile/tf_op_files.txt. # From eebe5c6cf4849d749b2c7ced9d0fe529c166331b Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 10 Jul 2020 21:54:17 +0000 Subject: [PATCH 0044/1017] cleaned includes and BUILD file --- tensorflow/c/kernels/BUILD | 26 +------------------------ tensorflow/c/kernels/ops/summary.cc | 3 +-- tensorflow/c/kernels/summary_op.cc | 19 +++++++----------- tensorflow/c/kernels/summary_op_test.cc | 8 +------- tensorflow/c/tf_tensor.cc | 2 +- 5 files changed, 11 insertions(+), 47 deletions(-) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 309dc7e221b..b713b27f5dc 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -29,12 +29,8 @@ tf_kernel_library( prefix = "summary_op", deps = [ "//tensorflow/c:kernels", - "//tensorflow/c:ops", - "//tensorflow/c:tf_datatype", - "//tensorflow/c:tf_status", "//tensorflow/c:tf_tensor", "//tensorflow/core:framework", - "//tensorflow/core:lib", ], ) @@ -54,9 +50,6 @@ tf_gen_op_libs( op_lib_names = ["summary"], deps = [ "//tensorflow/c:ops", - "//tensorflow/c:tf_datatype", - "//tensorflow/c:tf_status", - "//tensorflow/c:tf_tensor", "//tensorflow/core:lib", ], ) @@ -80,28 +73,11 @@ tf_cc_test( deps = [ ":summary_op", ":summary_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", "//tensorflow/core:test_main", - "//tensorflow/core:testlib", + "//tensorflow/core:testlib" ], ) -tf_cc_test( - name = "summary_op_benchmark_test", - srcs = ["summary_op_benchmark_test.cc"], - deps = [ - "summary_op", - "summary_op_lib", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) # Changes to the Android srcs here should be replicated in # tensorflow/contrib/makefile/tf_op_files.txt. # diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc index 857ff6f29fa..355d73396b6 100644 --- a/tensorflow/c/kernels/ops/summary.cc +++ b/tensorflow/c/kernels/ops/summary.cc @@ -1,4 +1,4 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,7 +16,6 @@ limitations under the License. #include "tensorflow/c/ops.h" #include "tensorflow/core/framework/selective_registration.h" #include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/macros.h" static void scalar_summary_shape_inference_fn(TF_ShapeInferenceContext* ctx, TF_Status* status) { diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index cd2509247da..18aa897bfa9 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -1,5 +1,5 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,16 +17,9 @@ limitations under the License. #include #include "tensorflow/c/kernels.h" -#include "tensorflow/c/ops.h" #include "tensorflow/c/tf_tensor.h" -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/selective_registration.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/platform/macros.h" #include "tensorflow/core/framework/summary.pb.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/types.h" // Struct that stores the status and TF_Tensor inputs to the opkernel. @@ -41,7 +34,7 @@ typedef struct Params{ if (TF_GetCode(status) == TF_OK){ TF_GetInput(ctx, 1, &values, status); } - else{ + else { values = nullptr; } }; @@ -57,8 +50,7 @@ typedef struct Params{ // dummy functions used for kernel registration static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { - void* ptr; - return ptr; + return nullptr; } static void SummaryScalarOp_Delete(void* kernel) { @@ -162,10 +154,13 @@ void RegisterSummaryScalarOpKernel() { // register the bitcast kernel. TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { if (SHOULD_REGISTER_OP_KERNEL("SummaryScalar")) { - RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); + RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); RegisterSummaryScalarOpKernel(); diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc index 42f7a7ff3b3..722373d36ce 100644 --- a/tensorflow/c/kernels/summary_op_test.cc +++ b/tensorflow/c/kernels/summary_op_test.cc @@ -1,4 +1,4 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,16 +15,10 @@ limitations under the License. #include "tensorflow/core/framework/attr_value.pb.h" #include "tensorflow/core/framework/attr_value_util.h" -#include "tensorflow/core/framework/fake_input.h" #include "tensorflow/core/framework/node_def.pb.h" -#include "tensorflow/core/framework/node_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/framework/summary.pb.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/c/tf_tensor.h" -#include "tensorflow/c/tf_tensor_internal.h" namespace tensorflow { namespace { diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc index aa65cb7c927..b4b8c772341 100644 --- a/tensorflow/c/tf_tensor.cc +++ b/tensorflow/c/tf_tensor.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include +#include #include "tensorflow/c/tf_status.h" #include "tensorflow/c/tf_status_helper.h" @@ -28,7 +29,6 @@ limitations under the License. #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/lib/core/coding.h" #include "tensorflow/core/platform/casts.h" -#include using tensorflow::Status; using tensorflow::Tensor; From fd8eb855c17fe8a76841feb469e3f849b9fc2432 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Sat, 11 Jul 2020 02:59:50 +0000 Subject: [PATCH 0045/1017] clean working first gradient --- tensorflow/core/kernels/map_kernels.cc | 7 +- tensorflow/core/kernels/map_kernels.h | 187 ++---------------- tensorflow/core/kernels/tensor_map.h | 23 --- tensorflow/core/kernels/tensor_map_test.cc | 23 --- tensorflow/core/ops/map_ops.cc | 76 +------ .../python/kernel_tests/list_ops_test.py | 12 +- .../python/kernel_tests/map_ops_test.py | 84 +++----- tensorflow/python/ops/map_ops.py | 30 +-- 8 files changed, 51 insertions(+), 391 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index eba2d99a75b..7d45d3942e1 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -29,9 +29,12 @@ REGISTER_KERNEL_BUILDER(Name("TensorMapSize").Device(DEVICE_CPU), REGISTER_KERNEL_BUILDER(Name("TensorMapLookup").Device(DEVICE_CPU), TensorMapLookup); +REGISTER_KERNEL_BUILDER(Name("TensorMapInsert").Device(DEVICE_CPU), + TensorMapInsert); + REGISTER_KERNEL_BUILDER(Name("TensorMapErase").Device(DEVICE_CPU), TensorMapErase); -REGISTER_KERNEL_BUILDER(Name("ZeroOut").Device(DEVICE_CPU), - ZeroOutOp); +REGISTER_KERNEL_BUILDER(Name("TensorMapReplace").Device(DEVICE_CPU), + TensorMapReplace); } \ No newline at end of file diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 78040359026..33282a75e0a 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -24,9 +24,10 @@ using namespace std; namespace tensorflow { + Status GetInputMap(OpKernelContext* c, int index, const TensorMap** map) { if (!TensorShapeUtils::IsScalar(c->input(index).shape())) { - return errors::InvalidArgument("Input list must be a scalar saw: ", + return errors::InvalidArgument("Input map must be a scalar saw: ", c->input(index).shape().DebugString()); } const TensorMap* m = c->input(index).scalar()().get(); @@ -39,6 +40,7 @@ Status GetInputMap(OpKernelContext* c, int index, const TensorMap** map) { return Status::OK(); } + Status ForwardInputOrCreateNewMap(OpKernelContext* c, int32 input_index, int32 output_index, const TensorMap& input_map, @@ -77,24 +79,22 @@ Status ForwardInputOrCreateNewMap(OpKernelContext* c, int32 input_index, return Status::OK(); } + class EmptyTensorMap : public OpKernel { public: - explicit EmptyTensorMap(OpKernelConstruction* ctx) : OpKernel(ctx) {} + explicit EmptyTensorMap(OpKernelConstruction* c) : OpKernel(c) {} - void Compute(OpKernelContext* ctx) override { - std::cout << "hello EmptyTensorMap map_kernels.h" << std::endl; + void Compute(OpKernelContext* c) override { Tensor* result; AllocatorAttributes attr; attr.set_on_host(true); - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, TensorShape{}, &result, attr)); + OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr)); TensorMap empty; result->scalar()() = std::move(empty); } - - private: - DataType element_dtype_; }; + class TensorMapSize : public OpKernel { public: explicit TensorMapSize(OpKernelConstruction* c) : OpKernel(c) {} @@ -109,45 +109,6 @@ class TensorMapSize : public OpKernel { } }; -class TensorMapZeros : public OpKernel { - public: - explicit TensorMapZeros(OpKernelConstruction* c) : OpKernel(c) { - //OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); - } - ~TensorMapZeros() override {} - - void Compute(OpKernelContext* c) override { - std::cout << "hello TensorMapInsert kernel" << std::endl; - const Tensor& temp_key = c->input(1); - const TensorKey key = TensorKey(temp_key); - std::cout << "got key" << std::endl; - const Tensor& value = c->input(2); - std::cout << "got value" << std::endl; - - const TensorMap* m = nullptr; - OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); - std::cout << "got map" << std::endl; - //TensorMap output_map; - //OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); - //std::cout << "create output" << std::endl; - //output_map = m->Zeros(); - //c->set_output(0, &&output_map); - //std::cout << "inserted" << std::endl; - - Tensor* result; - AllocatorAttributes attr; - attr.set_on_host(true); - OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result, attr)); - TensorMap output_map = m->Zeros(); - result->scalar()() = std::move(output_map); - } - - private: - DataType element_dtype_; -}; - -REGISTER_KERNEL_BUILDER(Name("TensorMapZeros").Device(DEVICE_CPU), - TensorMapZeros); class TensorMapInsert : public OpKernel { public: @@ -157,53 +118,20 @@ class TensorMapInsert : public OpKernel { ~TensorMapInsert() override {} void Compute(OpKernelContext* c) override { - std::cout << "hello TensorMapInsert kernel" << std::endl; - const Tensor& temp_key = c->input(1); - const TensorKey key = TensorKey(temp_key); - std::cout << "got key" << std::endl; + const TensorKey& key = c->input(1); const Tensor& value = c->input(2); - std::cout << "got value" << std::endl; - /*OP_REQUIRES(c, element_dtype_ == value.dtype(), - errors::InvalidArgument("Invalid data types; list elements ", - DataTypeString(element_dtype_), - " but tried to append ", - DataTypeString(value.dtype())));*/ - const TensorMap* m = nullptr; OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); - std::cout << "got map" << std::endl; - /*OP_REQUIRES(c, m->element_shape.IsCompatibleWith(input.shape()), - errors::InvalidArgument( - "Tried to append a map with incompatible shape to a " - "list. Op element shape: ", - input.shape().DebugString(), - " list shape: ", m->element_shape.DebugString()));*/ - /*OP_REQUIRES(c, element_dtype_ == m->element_dtype, - errors::InvalidArgument("Invalid data types; op elements ", - DataTypeString(element_dtype_), - " but list elements ", - DataTypeString(l->element_dtype))); - if (l->max_num_elements != -1) { - OP_REQUIRES( - c, l->tensors().size() < l->max_num_elements, - errors::InvalidArgument("Tried to push item into a full list", - " list size: ", l->tensors().size(), - " max_num_elements: ", l->max_num_elements)); - }*/ TensorMap* output_map = nullptr; OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); - std::cout << "create output" << std::endl; output_map->insert(key, value); - std::cout << "inserted" << std::endl; } private: DataType element_dtype_; }; -REGISTER_KERNEL_BUILDER(Name("TensorMapInsert").Device(DEVICE_CPU), - TensorMapInsert); class TensorMapLookup : public OpKernel { public: @@ -213,21 +141,17 @@ class TensorMapLookup : public OpKernel { ~TensorMapLookup() override {} void Compute(OpKernelContext* c) override { - std::cout << "hello TensorMapInsert kernel" << std::endl; - const Tensor& temp_key = c->input(1); - const TensorKey key = TensorKey(temp_key); - std::cout << "got key" << std::endl; + const TensorKey& key = c->input(1); const TensorMap* m = nullptr; OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); - std::cout << "got map" << std::endl; c->set_output(0, m->tensors().find(key)->second); - std::cout << "finished" << std::endl; } private: DataType element_dtype_; }; + class TensorMapErase : public OpKernel { public: explicit TensorMapErase(OpKernelConstruction* c) : OpKernel(c) { @@ -240,11 +164,6 @@ class TensorMapErase : public OpKernel { OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); const Tensor& temp_key = c->input(1); const TensorKey key = TensorKey(temp_key); - /*OP_REQUIRES(c, element_dtype_ == l->element_dtype, - errors::InvalidArgument("Invalid data types; op elements ", - DataTypeString(element_dtype_), - " but list elements ", - DataTypeString(l->element_dtype)));*/ OP_REQUIRES(c, !m->tensors().empty(), errors::InvalidArgument("Trying to erase from an empty map.")); @@ -254,27 +173,6 @@ class TensorMapErase : public OpKernel { const Tensor& t = m->tensors().find(key)->second; c->set_output(1, t); - /*if (t.dtype() != DT_INVALID) { - c->set_output(1, t); - } else { - PartialTensorShape partial_element_shape; - OP_REQUIRES_OK( - c, GetElementShapeFromInput(c, *l, 1, &partial_element_shape)); - TensorShape element_shape; - OP_REQUIRES( - c, partial_element_shape.AsTensorShape(&element_shape), - errors::InvalidArgument("Trying to read an uninitialized tensor but ", - "element_shape is not fully defined.", - partial_element_shape.DebugString())); - Tensor* result; - AllocatorAttributes attr; - if (element_dtype_ == DT_VARIANT) { - attr.set_on_host(true); - } - OP_REQUIRES_OK(c, c->allocate_output(1, element_shape, &result, attr)); - functor::SetZeroFunctor()(c->eigen_device(), - result->flat()); - }*/ TensorMap* output_map = nullptr; OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); @@ -293,81 +191,20 @@ class TensorMapReplace : public OpKernel { ~TensorMapReplace() override {} void Compute(OpKernelContext* c) override { - std::cout << "hello TensorMapReplace kernel" << std::endl; - const Tensor& temp_key = c->input(1); - const TensorKey key = TensorKey(temp_key); - std::cout << "got key" << std::endl; + const TensorKey& key = c->input(1); const Tensor& value = c->input(2); - std::cout << "got value" << std::endl; - /*OP_REQUIRES(c, element_dtype_ == value.dtype(), - errors::InvalidArgument("Invalid data types; list elements ", - DataTypeString(element_dtype_), - " but tried to append ", - DataTypeString(value.dtype())));*/ - const TensorMap* m = nullptr; OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); - std::cout << "got map" << std::endl; - /*OP_REQUIRES(c, m->element_shape.IsCompatibleWith(input.shape()), - errors::InvalidArgument( - "Tried to append a map with incompatible shape to a " - "list. Op element shape: ", - input.shape().DebugString(), - " list shape: ", m->element_shape.DebugString()));*/ - /*OP_REQUIRES(c, element_dtype_ == m->element_dtype, - errors::InvalidArgument("Invalid data types; op elements ", - DataTypeString(element_dtype_), - " but list elements ", - DataTypeString(l->element_dtype))); - if (l->max_num_elements != -1) { - OP_REQUIRES( - c, l->tensors().size() < l->max_num_elements, - errors::InvalidArgument("Tried to push item into a full list", - " list size: ", l->tensors().size(), - " max_num_elements: ", l->max_num_elements)); - }*/ TensorMap* output_map = nullptr; OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); - std::cout << "create output" << std::endl; output_map->replace(key,value); - std::cout << "inserted" << std::endl; } private: DataType element_dtype_; }; -REGISTER_KERNEL_BUILDER(Name("TensorMapReplace").Device(DEVICE_CPU), - TensorMapReplace); - -class ZeroOutOp : public OpKernel { - public: - explicit ZeroOutOp(OpKernelConstruction* c) : OpKernel(c) {} - - void Compute(OpKernelContext* c) override { - cout << "Hello World - Op" << endl; - // Grab the input tensor - const Tensor& input_tensor = c->input(0); - auto input = input_tensor.flat(); - - // Create an output tensor - Tensor* output_tensor = NULL; - OP_REQUIRES_OK(c, c->allocate_output(0, input_tensor.shape(), - &output_tensor)); - auto output_flat = output_tensor->flat(); - - // Set all but the first element of the output tensor to 0 - const int N = input.size(); - for (int i=1; i 0) output_flat(0) = input(0); - } -}; - } // namespace tensorflow #endif // TENSORFLOW_CORE_KERNELS_MAP_KERNELS_H_ diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index 7ab792b4813..a5d44550c98 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -143,29 +143,6 @@ class TensorMap { return out; } - TensorMap Zeros() const { - TensorMap out; - out.element_shape = element_shape; - out.element_dtype = element_dtype; - out.max_num_elements = max_num_elements; - // This performs a copy of the absl::hashmap. - absl::flat_hash_map::iterator it = tensors_->values_.begin(); - while(it != tensors_->values_.end()) { - out.tensors_->values_.try_emplace(it->first, Tensor(0)); - it++; - } - return out; - } - std::vector keys() { - std::vector keys(tensors_->values_.size()); - absl::flat_hash_map::iterator it = tensors_->values_.begin(); - while(it != tensors_->values_.end()) { - keys.push_back((Tensor)it->first); - it++; - } - return keys; - } - // Insert key and value if the key does not already exist. // Returns true if the insertion happens. bool insert(const TensorKey& key, const Tensor& value) { diff --git a/tensorflow/core/kernels/tensor_map_test.cc b/tensorflow/core/kernels/tensor_map_test.cc index 5774a605bbf..294aa07c963 100644 --- a/tensorflow/core/kernels/tensor_map_test.cc +++ b/tensorflow/core/kernels/tensor_map_test.cc @@ -137,29 +137,6 @@ TEST(TensorMapTest, EncodeDecode) { test::ExpectTensorEqual(tm.find(k)->second, tmc.find(k)->second); } -TEST(TensorMapTest, Keys) { - TensorMap tm; - TensorKey k = Tensor(11); - TensorKey k2 = Tensor(12); - Tensor v = Tensor(22); - tm.insert(k,v); - tm.insert(k2,v); - std::vector keys = tm.keys(); - EXPECT_EQ(1,1); - Tensor t = Tensor(11); - //std::cout << "keys: " << keys[0] << std::endl; - //test::ExpectTensorEqual(keys[0], t); - //test::ExpectTensorEqual(keys[1], k2); -} - -TEST(TensorMapTest, Zeros) { - TensorMap tm; - TensorKey k = Tensor(11); - Tensor v = Tensor(22); - tm.insert(k,v); - TensorMap z = tm.Zeros(); - test::ExpectTensorEqual(z.find(k)->second,Tensor(0)); -} } // namespace } // namespace tensorflow diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index d3711755d9e..8949e3f1923 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -39,15 +39,6 @@ REGISTER_OP("TensorMapSize") .Output("size: int32") .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("TensorMapZeros") - .Input("input_handle: variant") - .Output("output_handle: variant") - //.Attr("element_dtype: type") - .SetShapeFn([](shape_inference::InferenceContext* c) { - c->set_output(0, c->Scalar()); - return Status::OK(); - }); - REGISTER_OP("TensorMapInsert") .Input("input_handle: variant") .Input("key: element_dtype") @@ -56,32 +47,6 @@ REGISTER_OP("TensorMapInsert") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - /*DataType element_dtype; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); - shape_inference::ShapeHandle element_shape = c->UnknownShape();*/ - - /*auto* handle_data = c->input_handle_shapes_and_types(0); - if (handle_data != nullptr && handle_data->size() > 1) { - return errors::InvalidArgument( - "Trying to push to list with wrong variant data."); - } - if (IsValidTensorMapHandleData(handle_data)) { - const shape_inference::ShapeAndType& map_shape_type = (*handle_data)[0]; - if (list_shape_type.dtype != element_dtype) { - return errors::InvalidArgument( - "Trying to push to list with wrong element dtype. List has type ", - DataTypeString(list_shape_type.dtype), - " but trying to push element with type ", - DataTypeString(element_dtype)); - } - shape_inference::ShapeHandle ignored; - TF_RETURN_IF_ERROR( - c->Merge(element_shape, map_shape_type.shape, &ignored)); - element_shape = map_shape_type.shape; - } - c->set_output_handle_shapes_and_types( - 0, std::vector{ - {element_shape, element_dtype}});*/ return Status::OK(); }); @@ -92,32 +57,6 @@ REGISTER_OP("TensorMapLookup") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - /*DataType element_dtype; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); - shape_inference::ShapeHandle element_shape = c->UnknownShape();*/ - - /*auto* handle_data = c->input_handle_shapes_and_types(0); - if (handle_data != nullptr && handle_data->size() > 1) { - return errors::InvalidArgument( - "Trying to push to list with wrong variant data."); - } - if (IsValidTensorMapHandleData(handle_data)) { - const shape_inference::ShapeAndType& map_shape_type = (*handle_data)[0]; - if (list_shape_type.dtype != element_dtype) { - return errors::InvalidArgument( - "Trying to push to list with wrong element dtype. List has type ", - DataTypeString(list_shape_type.dtype), - " but trying to push element with type ", - DataTypeString(element_dtype)); - } - shape_inference::ShapeHandle ignored; - TF_RETURN_IF_ERROR( - c->Merge(element_shape, map_shape_type.shape, &ignored)); - element_shape = map_shape_type.shape; - } - c->set_output_handle_shapes_and_types( - 0, std::vector{ - {element_shape, element_dtype}});*/ return Status::OK(); }); @@ -128,9 +67,9 @@ REGISTER_OP("TensorMapErase") .Output("tensor: element_dtype") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { - /*DataType element_dtype; + DataType element_dtype; TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); - shape_inference::ShapeHandle tensor_shape = c->UnknownShape(); + /*shape_inference::ShapeHandle tensor_shape = c->UnknownShape(); auto* handle_data = c->input_handle_shapes_and_types(0); if (handle_data != nullptr && handle_data->size() > 1) { return errors::InvalidArgument( @@ -195,16 +134,5 @@ REGISTER_OP("TensorMapReplace") return Status::OK(); }); - - -REGISTER_OP("ZeroOut") - .Input("to_zero: int32") - .Output("zeroed: int32") - .SetShapeFn([](shape_inference::InferenceContext* c) { - //c->set_output(0, c->Scalar()); - c->set_output(0, c->input(0)); - return Status::OK(); - }); - } // namespace } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py index 4d7f2beb00b..7ffc4d3889d 100644 --- a/tensorflow/python/kernel_tests/list_ops_test.py +++ b/tensorflow/python/kernel_tests/list_ops_test.py @@ -48,7 +48,7 @@ from tensorflow.python.platform import test @test_util.run_all_in_graph_and_eager_modes class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): - ''' + def _testPushPop(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, @@ -130,7 +130,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): _, e = gen_list_ops.tensor_list_pop_back( l, element_dtype=dtypes.float32, element_shape=[1, 3]) self.evaluate(e) - ''' + def testPushGetGrad(self): with backprop.GradientTape() as tape: l = list_ops.empty_tensor_list( @@ -150,7 +150,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): dt0, dt1 = tape.gradient(t1, [c0, c1]) self.assertAllEqual(self.evaluate(dt1), [1.0, 1.0]) self.assertEqual(self.evaluate(dt0), 0.0) - ''' + def _testStack(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, @@ -888,7 +888,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l_worker = array_ops.identity(l_ps) l_worker = list_ops.tensor_list_push_back(l_worker, 3.0) self.evaluate(l_worker) - ''' + def testPushPopGradients(self): with backprop.GradientTape() as tape: l = list_ops.empty_tensor_list( @@ -925,7 +925,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): grad_c, grad_c2 = tape.gradient(y, [c, c2]) self.assertAllEqual(self.evaluate(grad_c), [0.0, 4.0]) self.assertAllEqual(self.evaluate(grad_c2), 6.0) - ''' + @test_util.run_deprecated_v1 def testSetOutOfBounds(self): c = constant_op.constant([1.0, 2.0]) @@ -1664,7 +1664,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): upper, constant_op.constant([0, 1, 2]), dtype=dtypes.string) self.assertAllEqual(f(), [b"A", b"B", b"C"]) - ''' + def testPopBackGrad(self): # https://github.com/tensorflow/tensorflow/issues/37230 diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 9384571dc2b..dc4e5b97fc3 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -18,7 +18,6 @@ from __future__ import division from __future__ import print_function #import numpy as np - from tensorflow.python.platform import test from absl.testing import parameterized from tensorflow.python.framework import test_util @@ -30,34 +29,26 @@ from tensorflow.python.eager import def_function from tensorflow.python.eager import function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes - -#try: -# from tensorflow_zero_out.python.ops.zero_out_ops import zero_out -#except ImportError: -# from zero_out_ops import zero_out from tensorflow.python.ops import map_ops @test_util.run_all_in_graph_and_eager_modes class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): - '''def testEmptyTensorMap(self): + + def testEmptyTensorMap(self): m = map_ops.empty_tensor_map() - print("test EmptyTensorMap") def testTensorMapSize(self): m = map_ops.empty_tensor_map() s = map_ops.tensor_map_size(m) - print("size: ", s) self.assertAllClose(s, 0) def testTensorMapInsert(self): - #with self.test_session(): - m = map_ops.empty_tensor_map() - k = constant_op.constant(1.0) - v = constant_op.constant(2.0) - m = map_ops.tensor_map_insert(m, k, v) - s = map_ops.tensor_map_size(m) - self.assertAllClose(s, 1) - print("test TensorMapInsert") + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + s = map_ops.tensor_map_size(m) + self.assertAllClose(s, 1) def testTensorMapLookup(self): m = map_ops.empty_tensor_map() @@ -65,35 +56,31 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k, v) l = map_ops.tensor_map_lookup(m, k) - print("lookup: ", l) - self.assertAllClose(l, v)''' + self.assertAllClose(l, v) def testTensorMapReplace(self): - #with self.test_session(): - m = map_ops.empty_tensor_map() - k = constant_op.constant(1.0) - v = constant_op.constant(2.0) - m = map_ops.tensor_map_insert(m, k, v) - s = map_ops.tensor_map_size(m) - self.assertAllClose(s, 1) - - v2 = constant_op.constant(3.0) - m = map_ops.tensor_map_replace(m, k, v2) - l = map_ops.tensor_map_lookup(m, k) - self.assertAllClose(l, v2) - print("test TensorMapReplace") - - def testTensorMapErase(self): - print("python erase") m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k, v) s = map_ops.tensor_map_size(m) self.assertAllClose(s, 1) + + v2 = constant_op.constant(3.0) + m = map_ops.tensor_map_replace(m, k, v2) + l = map_ops.tensor_map_lookup(m, k) + self.assertAllClose(l, v2) + + def testTensorMapErase(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + s = map_ops.tensor_map_size(m) + self.assertAllClose(s, 1) + m, e = map_ops.tensor_map_erase(m, k) s = map_ops.tensor_map_size(m) - print("erase: ", e) self.assertAllClose(s, 0) self.assertAllClose(e, v) @@ -106,32 +93,9 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): m = map_ops.tensor_map_insert(m, k, v) l = map_ops.tensor_map_lookup(m, k) l *= 5 - g= tape.gradient(l,v) - print("gradient",g) + g = tape.gradient(l,v) self.assertAllClose(g, 5.0) - - - ''' - @parameterized.named_parameters(("NoMaxNumElements", None), - ("WithMaxNumElements", 2)) - @test_util.run_deprecated_v1 - def testEraseFromEmptyTensorMapFails(self, max_num_elements): - print("hello world testErase") - m = map_ops.empty_tensor_map() - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Trying to erase from an empty map"): - m = map_ops.tensor_map_erase(l, element_dtype=dtypes.float32) - self.evaluate(l) - ''' - - '''def testZeroOut(self): - print("hello world testZeroOut") - with self.test_session(): - self.assertAllClose( - map_ops.zero_out([[1, 2], [3, 4]]), np.array([[1, 0], [0, 0]]))''' - if __name__ == '__main__': - print("hihihi") test.main() \ No newline at end of file diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 44894a8b6d9..29f0751d91f 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -35,15 +35,12 @@ from tensorflow.python.framework import constant_op ops.NotDifferentiable("EmptyTensorMap") def empty_tensor_map(): - print("hello gen_map_ops.empty_tensor_map") return gen_map_ops.empty_tensor_map() def tensor_map_size(input_handle): - print("hello gen_map_ops.tensor_map_size") return gen_map_ops.tensor_map_size(input_handle) def tensor_map_insert(input_handle, key, value): - print("hello gen_map_ops.tensor_map_insert") return gen_map_ops.tensor_map_insert(input_handle, key, value) def tensor_map_lookup(input_handle, key): @@ -55,13 +52,13 @@ def tensor_map_erase(input_handle, key): def tensor_map_replace(input_handle, key, value): return gen_map_ops.tensor_map_replace(input_handle, key, value) - @ops.RegisterGradient("TensorMapLookup") def LookupGrad(op, dval): # map grad should be a map that is 0 everywhere except 1 @key k m, k = op.inputs #m = gen_map_ops.tensor_map_zeros(m) - map_grad = tensor_map_replace(m, k, dval) + map_grad = empty_tensor_map() + map_grad = tensor_map_insert(map_grad, k, dval) key = op.inputs[1] key_grad = None return map_grad, key_grad @@ -72,27 +69,4 @@ def InsertGrad(op, dmap): map_grad, _ = gen_map_ops.tensor_map_erase(dmap, key) key_grad = None value_grad = tensor_map_lookup(dmap, key) - #value_grad = constant_op.constant(1.0) return map_grad, key_grad, value_grad - -def zero_out(to_zero): - return gen_map_ops.zero_out(to_zero) - -@ops.RegisterGradient("ZeroOut") -def _zero_out_grad(op, grad): - """The gradients for `zero_out`. - - Args: - op: The `zero_out` `Operation` that we are differentiating, which we can use - to find the inputs and outputs of the original op. - grad: Gradient with respect to the output of the `zero_out` op. - - Returns: - Gradients with respect to the input of `zero_out`. - """ - to_zero = op.inputs[0] - shape = array_ops.shape(to_zero) - index = array_ops.zeros_like(shape) - first_grad = array_ops.reshape(grad, [-1])[0] - to_zero_grad = sparse_ops.sparse_to_dense([index], shape, first_grad, 0) - return [to_zero_grad] # List of one Tensor, since we have one input From 06336eefdcdc420b6d486dd8b9cb72b3d452614a Mon Sep 17 00:00:00 2001 From: Steenu Johnson Date: Sun, 5 Jul 2020 13:17:06 +0530 Subject: [PATCH 0046/1017] Adding log_warning option in ignore_errors that controls whether error is logged. Signed-off-by: Steenu Johnson --- .../experimental/ignore_errors_dataset_op.cc | 39 ++++++++++++++++--- .../core/ops/experimental_dataset_ops.cc | 8 ++++ .../kernel_tests/ignore_errors_test.py | 18 +++++++++ .../python/data/experimental/ops/error_ops.py | 23 +++++++---- 4 files changed, 75 insertions(+), 13 deletions(-) diff --git a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc index e177fe27d18..567753a97cd 100644 --- a/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/ignore_errors_dataset_op.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/core/framework/dataset.h" #include "tensorflow/core/framework/partial_tensor_shape.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/logging.h" namespace tensorflow { namespace data { @@ -24,18 +25,28 @@ namespace { class IgnoreErrorsDatasetOp : public UnaryDatasetOpKernel { public: explicit IgnoreErrorsDatasetOp(OpKernelConstruction* ctx) - : UnaryDatasetOpKernel(ctx) {} + : UnaryDatasetOpKernel(ctx), + op_version_(ctx->def().op() == "IgnoreErrorsDatasetV2" ? 2 : 1) {} void MakeDataset(OpKernelContext* ctx, DatasetBase* input, DatasetBase** output) override { - *output = new Dataset(ctx, input); + bool log_warning = false; + if (op_version_ > 1) { + OP_REQUIRES_OK( + ctx, ParseScalarArgument(ctx, "log_warning", &log_warning)); + } + *output = new Dataset(ctx, input, log_warning, op_version_); } private: class Dataset : public DatasetBase { public: - explicit Dataset(OpKernelContext* ctx, const DatasetBase* input) - : DatasetBase(DatasetContext(ctx)), input_(input) { + explicit Dataset(OpKernelContext* ctx, const DatasetBase* input, + bool log_warning, int op_version) + : DatasetBase(DatasetContext(ctx)), + input_(input), + log_warning_(log_warning), + op_version_(op_version) { input_->Ref(); } @@ -69,8 +80,17 @@ class IgnoreErrorsDatasetOp : public UnaryDatasetOpKernel { DatasetGraphDefBuilder* b, Node** output) const override { Node* input_graph_node = nullptr; + Node* log_warning = nullptr; TF_RETURN_IF_ERROR(b->AddInputDataset(ctx, input_, &input_graph_node)); - TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph_node}, output)); + TF_RETURN_IF_ERROR(b->AddScalar(log_warning_, &log_warning)); + if (op_version_ > 1) { + TF_RETURN_IF_ERROR(b->AddDataset(this, + {std::make_pair(0, input_graph_node), + std::make_pair(1, log_warning)}, + {}, {}, output)); + } else { + TF_RETURN_IF_ERROR(b->AddDataset(this, {input_graph_node}, output)); + } return Status::OK(); } @@ -97,6 +117,10 @@ class IgnoreErrorsDatasetOp : public UnaryDatasetOpKernel { } s = input_impl_->GetNext(ctx, out_tensors, end_of_sequence); while (!s.ok() && !errors::IsCancelled(s)) { + if (dataset()->log_warning_) { + LOG(WARNING) << "Error raised with error message " + << s.error_message(); + } out_tensors->clear(); s = input_impl_->GetNext(ctx, out_tensors, end_of_sequence); } @@ -142,11 +166,16 @@ class IgnoreErrorsDatasetOp : public UnaryDatasetOpKernel { }; const DatasetBase* const input_; + const bool log_warning_; + const int op_version_; }; + const int op_version_; }; REGISTER_KERNEL_BUILDER(Name("IgnoreErrorsDataset").Device(DEVICE_CPU), IgnoreErrorsDatasetOp); +REGISTER_KERNEL_BUILDER(Name("IgnoreErrorsDatasetV2").Device(DEVICE_CPU), + IgnoreErrorsDatasetOp); REGISTER_KERNEL_BUILDER( Name("ExperimentalIgnoreErrorsDataset").Device(DEVICE_CPU), IgnoreErrorsDatasetOp); diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc index 5e869a2f0be..c6d49ac2e8a 100644 --- a/tensorflow/core/ops/experimental_dataset_ops.cc +++ b/tensorflow/core/ops/experimental_dataset_ops.cc @@ -441,6 +441,14 @@ REGISTER_OP("ExperimentalIgnoreErrorsDataset") .Attr("output_shapes: list(shape) >= 1") .SetShapeFn(shape_inference::ScalarShape); +REGISTER_OP("IgnoreErrorsDatasetV2") + .Input("input_dataset: variant") + .Input("log_warning : bool") + .Output("handle: variant") + .Attr("output_types: list(type) >= 1") + .Attr("output_shapes: list(shape) >= 1") + .SetShapeFn(shape_inference::ScalarShape); + REGISTER_OP("IteratorGetDevice") .Input("resource: resource") .Output("device: string") diff --git a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py index 5ed72767425..c3b9ab6bd07 100644 --- a/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/ignore_errors_test.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function import os +import sys from absl.testing import parameterized import numpy as np @@ -54,6 +55,23 @@ class IgnoreErrorsTest(test_base.DatasetTestBase, parameterized.TestCase): with self.assertRaises(errors.OutOfRangeError): self.evaluate(get_next()) + @combinations.generate(test_base.default_test_combinations()) + def testIgnoreError_withLogWarning(self): + components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) + dataset = ( + dataset_ops.Dataset.from_tensor_slices(components) + .map(lambda x: array_ops.check_numerics(x, "message")).apply( + error_ops.ignore_errors(log_warning=True))) + get_next = self.getNext(dataset) + for x in [1., 2., 3.]: + self.assertEqual(x, self.evaluate(get_next())) + with self.captureWritesToStream(sys.stderr) as logged: + self.assertEqual(5., self.evaluate(get_next())) + expected = "Tensor had NaN values" + self.assertIn((expected), logged.contents()) + with self.assertRaises(errors.OutOfRangeError): + self.evaluate(get_next()) + @combinations.generate(test_base.default_test_combinations()) def testParallelMapIgnoreError(self): components = np.array([1., 2., 3., np.nan, 5.]).astype(np.float32) diff --git a/tensorflow/python/data/experimental/ops/error_ops.py b/tensorflow/python/data/experimental/ops/error_ops.py index 23937bb76f8..14b8802d920 100644 --- a/tensorflow/python/data/experimental/ops/error_ops.py +++ b/tensorflow/python/data/experimental/ops/error_ops.py @@ -20,10 +20,10 @@ from __future__ import print_function from tensorflow.python.data.ops import dataset_ops from tensorflow.python.ops import gen_experimental_dataset_ops from tensorflow.python.util.tf_export import tf_export - +from tensorflow.python.compat import compat @tf_export("data.experimental.ignore_errors") -def ignore_errors(): +def ignore_errors(log_warning=False): """Creates a `Dataset` from another `Dataset` and silently ignores any errors. Use this transformation to produce a dataset that contains the same elements @@ -48,7 +48,7 @@ def ignore_errors(): """ def _apply_fn(dataset): - return _IgnoreErrorsDataset(dataset) + return _IgnoreErrorsDataset(dataset, log_warning) return _apply_fn @@ -56,11 +56,18 @@ def ignore_errors(): class _IgnoreErrorsDataset(dataset_ops.UnaryUnchangedStructureDataset): """A `Dataset` that silently ignores errors when computing its input.""" - def __init__(self, input_dataset): + def __init__(self, input_dataset, log_warning): """See `Dataset.ignore_errors()` for details.""" self._input_dataset = input_dataset - variant_tensor = ( - gen_experimental_dataset_ops.ignore_errors_dataset( - self._input_dataset._variant_tensor, # pylint: disable=protected-access - **self._flat_structure)) + if compat.forward_compatible(2020, 7, 26) or log_warning: + variant_tensor = ( + gen_experimental_dataset_ops.ignore_errors_dataset_v2( + self._input_dataset._variant_tensor, # pylint: disable=protected-access + log_warning, + **self._flat_structure)) + else: + variant_tensor = ( + gen_experimental_dataset_ops.ignore_errors_dataset( + self._input_dataset._variant_tensor, # pylint: disable=protected-access + **self._flat_structure)) super(_IgnoreErrorsDataset, self).__init__(input_dataset, variant_tensor) From 4904a46f623fce0b2e675a40fc4adfc98a3ad009 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 13 Jul 2020 17:47:03 +0000 Subject: [PATCH 0047/1017] cleanup --- tensorflow/core/kernels/tensor_map.cc | 4 +- tensorflow/core/kernels/tensor_map.h | 5 +- tensorflow/core/ops/map_ops.cc | 52 +------------------ .../python/kernel_tests/list_ops_test.py | 12 ++--- .../python/kernel_tests/map_ops_test.py | 2 +- tensorflow/python/ops/map_ops.py | 2 - 6 files changed, 12 insertions(+), 65 deletions(-) diff --git a/tensorflow/core/kernels/tensor_map.cc b/tensorflow/core/kernels/tensor_map.cc index abeaf92390e..cfba3892650 100644 --- a/tensorflow/core/kernels/tensor_map.cc +++ b/tensorflow/core/kernels/tensor_map.cc @@ -90,9 +90,9 @@ bool TensorMap::Decode(const VariantTensorData& data) { while (tensors_it != data.tensors().end()) { // should assert that tensors_it + 1 is also not the end - /*if (*std::next(tensors_it) == data.tensors().end()) { + if (std::next(tensors_it) == data.tensors().end()) { return false; - }*/ + } TensorKey k = TensorKey(*tensors_it); // copy inefficient? tensors().emplace(k,*++tensors_it); tensors_it++; diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index a5d44550c98..633c7db8668 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -58,7 +58,7 @@ namespace tensorflow { // bool can_alias = false; // auto fw = c->forward_input(..., DT_VARIANT, {}, ...); // if (fw && fw->dtype() == DT_VARIANT && fw->NumElements() == 1) { -// auto* tl = fw->scalar()().get(); +// auto* tl = fw->scalar()().get(); // if (tl && tl->RefCountIsOne()) { // can_alias = true; // } @@ -132,7 +132,7 @@ class TensorMap { PartialTensorShape& shape() { return element_shape; } DataType dtype() { return element_dtype; } - // Get a new TensorList containing a copy of the underlying tensor container. + // Get a new TensorMap containing a copy of the underlying tensor container. TensorMap Copy() const { TensorMap out; out.element_shape = element_shape; @@ -185,7 +185,6 @@ class TensorMap { private: class Tensors : public core::RefCounted { public: - //std::unordered_map values_; absl::flat_hash_map values_; }; Tensors* tensors_; diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index 8949e3f1923..445180c34ef 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -23,8 +23,7 @@ namespace { bool IsValidTensorMapHandleData( const std::vector* handle_data) { std::cout << "is valid tensor map handle data " << handle_data->size() << std::endl; - return true; - //return handle_data != nullptr && handle_data->size() == 1; + return handle_data != nullptr && handle_data->size() == 1; } REGISTER_OP("EmptyTensorMap") @@ -69,29 +68,6 @@ REGISTER_OP("TensorMapErase") .SetShapeFn([](shape_inference::InferenceContext* c) { DataType element_dtype; TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); - /*shape_inference::ShapeHandle tensor_shape = c->UnknownShape(); - auto* handle_data = c->input_handle_shapes_and_types(0); - if (handle_data != nullptr && handle_data->size() > 1) { - return errors::InvalidArgument( - "Trying to read from list with invalid variant data."); - } - if (IsValidTensorListHandleData(handle_data)) { - const shape_inference::ShapeAndType& list_shape_type = - (*handle_data)[0]; - if (list_shape_type.dtype != element_dtype) { - return errors::InvalidArgument( - "Trying to read from list with wrong element dtype. List has " - "type ", - DataTypeString(list_shape_type.dtype), - " but trying to push element with type ", - DataTypeString(element_dtype)); - } - shape_inference::ShapeHandle ignored; - TF_RETURN_IF_ERROR( - c->Merge(tensor_shape, list_shape_type.shape, &ignored)); - c->set_output_handle_shapes_and_types(0, *handle_data); - tensor_shape = list_shape_type.shape; - }*/ c->set_output(1, c->Scalar()); // removed element c->set_output(0, c->Scalar()); // map return Status::OK(); @@ -105,32 +81,6 @@ REGISTER_OP("TensorMapReplace") .Attr("element_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); - /*DataType element_dtype; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); - shape_inference::ShapeHandle element_shape = c->UnknownShape();*/ - - /*auto* handle_data = c->input_handle_shapes_and_types(0); - if (handle_data != nullptr && handle_data->size() > 1) { - return errors::InvalidArgument( - "Trying to push to list with wrong variant data."); - } - if (IsValidTensorMapHandleData(handle_data)) { - const shape_inference::ShapeAndType& map_shape_type = (*handle_data)[0]; - if (list_shape_type.dtype != element_dtype) { - return errors::InvalidArgument( - "Trying to push to list with wrong element dtype. List has type ", - DataTypeString(list_shape_type.dtype), - " but trying to push element with type ", - DataTypeString(element_dtype)); - } - shape_inference::ShapeHandle ignored; - TF_RETURN_IF_ERROR( - c->Merge(element_shape, map_shape_type.shape, &ignored)); - element_shape = map_shape_type.shape; - } - c->set_output_handle_shapes_and_types( - 0, std::vector{ - {element_shape, element_dtype}});*/ return Status::OK(); }); diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py index 7ffc4d3889d..53ebdd3ab88 100644 --- a/tensorflow/python/kernel_tests/list_ops_test.py +++ b/tensorflow/python/kernel_tests/list_ops_test.py @@ -48,7 +48,7 @@ from tensorflow.python.platform import test @test_util.run_all_in_graph_and_eager_modes class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): - + def _testPushPop(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, @@ -130,7 +130,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): _, e = gen_list_ops.tensor_list_pop_back( l, element_dtype=dtypes.float32, element_shape=[1, 3]) self.evaluate(e) - + def testPushGetGrad(self): with backprop.GradientTape() as tape: l = list_ops.empty_tensor_list( @@ -150,7 +150,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): dt0, dt1 = tape.gradient(t1, [c0, c1]) self.assertAllEqual(self.evaluate(dt1), [1.0, 1.0]) self.assertEqual(self.evaluate(dt0), 0.0) - + def _testStack(self, max_num_elements): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, @@ -888,7 +888,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l_worker = array_ops.identity(l_ps) l_worker = list_ops.tensor_list_push_back(l_worker, 3.0) self.evaluate(l_worker) - + def testPushPopGradients(self): with backprop.GradientTape() as tape: l = list_ops.empty_tensor_list( @@ -925,7 +925,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): grad_c, grad_c2 = tape.gradient(y, [c, c2]) self.assertAllEqual(self.evaluate(grad_c), [0.0, 4.0]) self.assertAllEqual(self.evaluate(grad_c2), 6.0) - + @test_util.run_deprecated_v1 def testSetOutOfBounds(self): c = constant_op.constant([1.0, 2.0]) @@ -1664,7 +1664,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): upper, constant_op.constant([0, 1, 2]), dtype=dtypes.string) self.assertAllEqual(f(), [b"A", b"B", b"C"]) - + def testPopBackGrad(self): # https://github.com/tensorflow/tensorflow/issues/37230 diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index dc4e5b97fc3..46f8f21d104 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -93,7 +93,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): m = map_ops.tensor_map_insert(m, k, v) l = map_ops.tensor_map_lookup(m, k) l *= 5 - g = tape.gradient(l,v) + g = tape.gradient(l, v) self.assertAllClose(g, 5.0) diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 29f0751d91f..5d8d2b88f2f 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -54,9 +54,7 @@ def tensor_map_replace(input_handle, key, value): @ops.RegisterGradient("TensorMapLookup") def LookupGrad(op, dval): - # map grad should be a map that is 0 everywhere except 1 @key k m, k = op.inputs - #m = gen_map_ops.tensor_map_zeros(m) map_grad = empty_tensor_map() map_grad = tensor_map_insert(map_grad, k, dval) key = op.inputs[1] From 8f3cbfa0ad628624b3d2caf0203a03b2014d9b36 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 13 Jul 2020 17:51:41 +0000 Subject: [PATCH 0048/1017] restore list_ops_test --- .../python/kernel_tests/list_ops_test.py | 132 +++++++++--------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py index 53ebdd3ab88..ce20cf489e6 100644 --- a/tensorflow/python/kernel_tests/list_ops_test.py +++ b/tensorflow/python/kernel_tests/list_ops_test.py @@ -78,8 +78,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[], max_num_elements=1) l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0)) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Tried to push item into a full list"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Tried to push item into a full list"): l = list_ops.tensor_list_push_back(l, 2.) self.evaluate(l) @@ -91,8 +91,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): element_dtype=dtypes.float32, element_shape=[], max_num_elements=max_num_elements) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Trying to pop from an empty list"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Trying to pop from an empty list"): l = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32) self.evaluate(l) @@ -115,7 +115,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testPopUninitializedTensorWithInvalidElementShapeFails(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Trying to read an uninitialized tensor but " "element_shape is not fully defined"): @@ -124,7 +124,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"): _, e = gen_list_ops.tensor_list_pop_back( @@ -191,8 +191,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should raise an error when the element tensors do not all have the same # shape. - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Incompatible ranks during merge: 0 vs. 1"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Incompatible ranks during merge: 0 vs. 1"): l = list_ops.tensor_list_push_back(l, constant_op.constant([3.0, 4.0])) t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32) self.evaluate(t) @@ -213,7 +213,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should raise an error when the element tensors do not all have the same # shape. - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"Incompatible shapes during merge: \[1\] vs. \[2\]"): l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0, 3.0])) @@ -234,8 +234,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should not be able to stack empty lists with partially defined # element_shape. - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "non-fully-defined"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[None, 2], @@ -244,8 +244,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.evaluate(t) # Should not be able to stack empty lists with undefined element_shape. - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "non-fully-defined"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=None, @@ -285,10 +285,10 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testStackReservedListWithNoElementsAndPartialElementShapeFails(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Tried to stack list which only contains " - "uninitialized tensors and has a " - "non-fully-defined element_shape: "): + with self.assertRaisesRegex( + errors.InvalidArgumentError, "Tried to stack list which only contains " + "uninitialized tensors and has a " + "non-fully-defined element_shape: "): t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32) self.evaluate(t) @@ -341,8 +341,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should raise an error when the requested tensors do not all have the same # shape. - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Incompatible ranks during merge: 0 vs. 1"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Incompatible ranks during merge: 0 vs. 1"): t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32) self.evaluate(t) @@ -366,7 +366,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should raise an error when the requested tensors do not all have the same # shape. - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"Incompatible shapes during merge: \[1\] vs. \[2\]"): t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32) @@ -387,8 +387,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should not be able to gather from empty lists with partially defined # element_shape. - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "non-fully-defined"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[None, 2], @@ -398,8 +398,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should not be able to gather from empty lists with undefined # element_shape. - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "non-fully-defined"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=None, @@ -455,7 +455,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testGatherReservedListWithNoElementsAndPartialElementShapeFails(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Tried to gather uninitialized tensors from a" " list with non-fully-defined element_shape"): @@ -485,7 +485,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testScatterFailsWhenIndexLargerThanNumElements(self): c0 = constant_op.constant([1.0, 2.0]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "TensorListScatter: Trying to scatter at index 3 in list with size 3"): l = gen_list_ops.tensor_list_scatter_v2( @@ -494,7 +494,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testScatterFailsWithInvalidNumElements(self): c0 = constant_op.constant([1.0, 2.0]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "TensorListScatter expects num_elements >= -1, found: -2"): l = gen_list_ops.tensor_list_scatter_v2( @@ -503,7 +503,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testScatterWithInvalidRowsInInputTensorFails(self): c0 = constant_op.constant([1.0, 2.0]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Invalid number of rows in input tensor. Expected: 3 Actual: 2"): l = list_ops.tensor_list_scatter(c0, [1, 0, 2], []) @@ -511,7 +511,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testScatterWithNegativeIndicesFails(self): c0 = constant_op.constant([1.0, 2.0]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Indices in TensorListScatter must all be non-negative."): l = list_ops.tensor_list_scatter(c0, [-1, -2], element_shape=[]) @@ -658,7 +658,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testGetUninitializedTensorWithInvalidElementShapeFails(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Trying to read an uninitialized tensor but " "element_shape is not fully defined"): @@ -676,7 +676,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): error_type = errors.InvalidArgumentError else: error_type = ValueError - with self.assertRaisesRegexp(error_type, r"shapes"): + with self.assertRaisesRegex(error_type, r"shapes"): e0 = gen_list_ops.tensor_list_get_item( l, 0, element_dtype=dtypes.float32, element_shape=[1, 3]) self.evaluate(e0) @@ -699,7 +699,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testSetOnEmptyListWithMaxNumElementsFails(self): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[], max_num_elements=3) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Trying to modify element 0 in a list with 0 elements."): l = list_ops.tensor_list_set_item(l, 0, 1.) @@ -882,8 +882,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with ops.device("/job:ps"): l_ps = array_ops.identity(l) l_ps = list_ops.tensor_list_push_back(l_ps, 2.) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "Tried to push item into a full list"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Tried to push item into a full list"): with ops.device("/job:worker"): l_worker = array_ops.identity(l_ps) l_worker = list_ops.tensor_list_push_back(l_worker, 3.0) @@ -943,8 +943,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # at graph building time. l = list_ops.tensor_list_set_item(l, 0, ph) l_0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "incompatible shape"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "incompatible shape"): sess.run(l_0, {ph: [3.0]}) def testResourceVariableScatterGather(self): @@ -1021,7 +1021,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): "element shapes are not identical at index 0") else: expected_error = (ValueError, "Shapes must be equal rank") - with self.assertRaisesRegexp(*expected_error): + with self.assertRaisesRegex(*expected_error): l_batch_of_vec_tls = array_ops.stack( [list_ops.tensor_list_from_tensor([[1.0]], element_shape=[1])] * 2) self.evaluate( @@ -1033,7 +1033,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): r"input_b\[0\].dtype != element_dtype.") else: expected_error = (ValueError, "input_b.type != element_dtype") - with self.assertRaisesRegexp(*expected_error): + with self.assertRaisesRegex(*expected_error): l_batch_of_int_tls = array_ops.stack( [list_ops.tensor_list_from_tensor([1], element_shape=[])] * 2) self.evaluate( @@ -1073,8 +1073,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.assertRaises((errors.InvalidArgumentError, ValueError)): self.evaluate(list_ops.tensor_list_push_back_batch(l_batch, [])) - with self.assertRaisesRegexp(errors.InvalidArgumentError, - "incompatible shape to a list at index 0"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + "incompatible shape to a list at index 0"): self.evaluate( list_ops.tensor_list_push_back_batch(l_batch, [[3.0], [4.0]])) @@ -1082,7 +1082,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): expected_error = (errors.InvalidArgumentError, "Invalid data type") else: expected_error = (ValueError, "wrong element dtype") - with self.assertRaisesRegexp(*expected_error): + with self.assertRaisesRegex(*expected_error): self.evaluate(list_ops.tensor_list_push_back_batch(l_batch, [3, 4])) def testZerosLike(self): @@ -1246,7 +1246,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): element_shape=[], element_dtype=dtypes.float32, num_elements=2) l2 = list_ops.tensor_list_reserve( element_shape=[], element_dtype=dtypes.float32, num_elements=3) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Trying to add two lists of tensors with different lengths"): l = math_ops.add_n([l1, l2]) @@ -1268,7 +1268,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): element_dtype=dtypes.float32, num_elements=3) l = math_ops.add_n([l1, l2]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Trying to add two lists of tensors with incompatible element shapes" ): @@ -1314,7 +1314,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): element_dtype=dtypes.float32, element_shape=None) l = list_ops.tensor_list_push_back(l, [[0., 1.]]) l = list_ops.tensor_list_push_back(l, [[2.], [4.]]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"Incompatible shapes during merge: " r"\[2\] vs. \[1\]"): t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) @@ -1333,7 +1333,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testConcatEmptyListWithUnknownElementShapeFails(self): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=None) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "All except the first dimension must be fully" " defined when concating an empty tensor list"): @@ -1343,7 +1343,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testConcatEmptyListWithPartiallyDefinedElementShapeFails(self): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[2, None]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "All except the first dimension must be fully" " defined when concating an empty tensor list"): @@ -1354,7 +1354,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=tensor_shape.TensorShape([])) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Concat requires elements to be at least vectors, " "found scalars instead"): @@ -1365,14 +1365,14 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=None) l1 = list_ops.tensor_list_push_back(l, 1.) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Concat saw a scalar shape at index 0" " but requires at least vectors"): t = list_ops.tensor_list_concat(l1, element_dtype=dtypes.float32) self.evaluate(t) l1 = list_ops.tensor_list_push_back(l, [1.]) l1 = list_ops.tensor_list_push_back(l1, 2.) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "Concat saw a scalar shape at index 1" " but requires at least vectors"): t = list_ops.tensor_list_concat(l1, element_dtype=dtypes.float32) @@ -1420,7 +1420,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testConcatWithUninitializedTensorsFailsIfNoElementShape(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"Trying to concat list with only uninitialized tensors " r"but element_shape_except_first_dim_ is not fully defined"): @@ -1430,7 +1430,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testConcatWithUninitializedTensorsFailsIfNoInputLengths(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"List contains uninitialized tensor at index 0" r" but leading_dims has only 0 elements."): @@ -1467,7 +1467,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.cached_session(): tensor = array_ops.placeholder(dtype=dtypes.float32) l = list_ops.tensor_list_split(tensor, element_shape=None, lengths=[1]) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"Tensor must be at least a vector, but saw shape: \[\]"): l.eval({tensor: 1}) @@ -1479,24 +1479,24 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=lengths) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"Expected lengths to be a vector, received shape: \[\]"): l.eval({lengths: 1}) def testSplitWithInvalidLengthsFails(self): - with self.assertRaisesRegexp(errors.InvalidArgumentError, - r"Invalid value in lengths: -1"): + with self.assertRaisesRegex(errors.InvalidArgumentError, + r"Invalid value in lengths: -1"): l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[1, -1]) self.evaluate(l) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"Attempting to slice \[0, 3\] from tensor with length 2"): l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[3]) self.evaluate(l) - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"Unused values in tensor. Length of tensor: 2 Values used: 1"): l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[1]) @@ -1504,11 +1504,11 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @test_util.run_deprecated_v1 def testSkipEagerSplitWithScalarElementShapeFails(self): - with self.assertRaisesRegexp(ValueError, - r"Shapes must be equal rank, but are 1 and 0"): + with self.assertRaisesRegex(ValueError, + r"Shapes must be equal rank, but are 1 and 0"): l = list_ops.tensor_list_split([1., 2.], element_shape=[], lengths=[1, 1]) with self.cached_session(): - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"TensorListSplit requires element_shape to be at least of rank 1, " r"but saw: \[\]"): @@ -1520,7 +1520,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testEagerOnlySplitWithScalarElementShapeFails(self): if context.executing_eagerly(): - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"TensorListSplit requires element_shape to be at least of rank 1, " r"but saw: \[\]"): @@ -1528,14 +1528,14 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @test_util.run_deprecated_v1 def testSkipEagerSplitWithIncompatibleTensorShapeAndElementShapeFails(self): - with self.assertRaisesRegexp(ValueError, - r"Shapes must be equal rank, but are 2 and 1"): + with self.assertRaisesRegex(ValueError, + r"Shapes must be equal rank, but are 2 and 1"): l = list_ops.tensor_list_split([[1.], [2.]], element_shape=[1], lengths=[1, 1]) with self.cached_session(): - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"tensor shape \[2,1\] is not compatible with element_shape \[1\]"): element_shape = array_ops.placeholder(dtype=dtypes.int32) @@ -1546,7 +1546,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testEagerOnlySplitWithIncompatibleTensorShapeAndElementShapeFails(self): if context.executing_eagerly(): - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, r"tensor shape \[2,1\] is not compatible with element_shape \[1\]"): list_ops.tensor_list_split([[1.], [2.]], @@ -1576,7 +1576,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): [1., 2.]) def testResizeWithInvalidSizeFails(self): - with self.assertRaisesRegexp( + with self.assertRaisesRegex( errors.InvalidArgumentError, "TensorListSlice expects size to be non-negative"): l = list_ops.tensor_list_from_tensor([1., 2., 3.], element_shape=[]) From cec1c9c22e6bae059d7888d0e2902e11b53df3be Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 13 Jul 2020 19:40:17 +0000 Subject: [PATCH 0049/1017] added TF_AllocatorAttributes --- tensorflow/c/tf_tensor.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index acdf053e63a..ce08a8d2678 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -45,6 +45,16 @@ limitations under the License. extern "C" { #endif +// Allocator Attributes used for tensor allocation +struct TF_AllocatorAttributes { + size_t struct_size; + unsigned char on_host; +} TF_AllocatorAttributes; + + +#define TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE \ + TF_OFFSET_OF_END(TF_AllocatorAttributes, on_host) + // -------------------------------------------------------------------------- // TF_Tensor holds a multi-dimensional array of elements of a single data type. // For all types other than TF_STRING, the data buffer stores elements From 9e1e8b58541e1d34e70339de37a167c8c7662e45 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 13 Jul 2020 19:43:23 +0000 Subject: [PATCH 0050/1017] updated file path to upstream/master for mergeability --- tensorflow/c/kernels.cc | 30 ------- tensorflow/c/kernels.h | 15 ---- tensorflow/c/kernels_test.cc | 166 ++++++++--------------------------- 3 files changed, 36 insertions(+), 175 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index a8da95c6eac..3021a38e888 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -25,7 +25,6 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/platform/types.h" -#include "tensorflow/core/framework/tensor_shape.h" // This file forms the basis of a stable ABI for third-party kernel // implementations. It is crucial that changes to this file are made cautiously @@ -98,11 +97,6 @@ void TF_KernelBuilder_HostMemory(TF_KernelBuilder* kernel_builder, kernel_builder->cc_builder->HostMemory(arg_name); } -void TF_KernelBuilder_Priority(TF_KernelBuilder* kernel_builder, - int32_t priority_number){ - kernel_builder->cc_builder->Priority(priority_number); -} - namespace tensorflow { namespace { @@ -273,27 +267,3 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, } return tf_tensor; } - -TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, - int64_t* dims, int num_dims, TF_Status* status){ - auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); - TF_SetStatus(status, TF_OK, ""); - tensorflow::TensorShape shape; - for(int i = 0; i < num_dims; ++i){ - shape.AddDim(dims[i]); - } - tensorflow::Status s; - tensorflow::Tensor tensor; - TF_Tensor* tf_tensor; - s = cc_ctx->allocate_temp(static_cast(dtype), shape, &tensor); - if (!s.ok()){ - ::tensorflow::Set_TF_Status_from_Status(status, s); - return nullptr; - } - tf_tensor = TF_TensorFromTensor(tensor, &s); - if (!s.ok()){ - ::tensorflow::Set_TF_Status_from_Status(status, s); - return nullptr; - } - return tf_tensor; -} diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index 1891ce31a23..084717c1d9e 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -107,10 +107,6 @@ TF_CAPI_EXPORT extern void TF_KernelBuilder_TypeConstraint( TF_CAPI_EXPORT extern void TF_KernelBuilder_HostMemory( TF_KernelBuilder* kernel_builder, const char* arg_name); -// Specify a priority number for this kernel. -TF_CAPI_EXPORT extern void TF_KernelBuilder_Priority( - TF_KernelBuilder* kernel_builder, int32_t priority_number); - // Register the given kernel builder with the TensorFlow runtime. If // registration fails, the given status will be populated. // @@ -194,17 +190,6 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int64_t* dims, int num_dims, size_t len, TF_Status* status); -// Allocates a temporary Tensor of the specified type and shape. Devices -// such as GPUs that enqueue Ops for lazy execution may retain references -// to the temporary tensors after the Op's Compute method has run. - -// num_dims must equal the size of array dims -TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, - TF_DataType dtype, - int64_t* dims, int num_dims, - TF_Status* status); - - #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index 738c1e12c80..423302741de 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -360,17 +360,6 @@ class DeviceKernelOpTest : public OpsTestBase { #endif }; -// Helper function for tests that validates that the tensor has -// shape and type corresponding to dims and dtype. -void validate_tensor(TF_Tensor* tensor, int64_t* dims, int64_t num_dims, - TF_DataType dtype); - -// Helper function for tests that copies data of length -// tensor_size_bytes from values to tensor -template -void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, - TF_OpKernelContext* ctx); - REGISTER_OP("AllocateOutputOp1").Output("output1: float"); TEST_F(DeviceKernelOpTest, TestAllocateOutputSizeOne) { @@ -382,11 +371,22 @@ TEST_F(DeviceKernelOpTest, TestAllocateOutputSizeOne) { TF_Tensor* output = TF_AllocateOutput( /*context=*/ctx, /*index=*/0, /*dtype=*/TF_FLOAT, /*dims=*/&dim, /*num_dims=*/1, /*len=*/tensor_size_bytes, s); - validate_tensor(output, &dim, 1, TF_FLOAT); - + EXPECT_EQ(TF_OK, TF_GetCode(s)); + EXPECT_EQ(TF_FLOAT, TF_TensorType(output)); + EXPECT_EQ(1, TF_NumDims(output)); + EXPECT_EQ(1, TF_Dim(output, 0)); + // Set output to 3 - float values[1] = {3.0f}; - set_tensor_data(output, values, tensor_size_bytes, ctx); + float* data = reinterpret_cast(TF_TensorData(output)); + float value = 3.0f; +#if GOOGLE_CUDA + OpKernelContext* cc_ctx = reinterpret_cast(ctx); + cc_ctx->eigen_gpu_device().memcpyHostToDevice(data, &value, + tensor_size_bytes); +#else + *data = value; +#endif + TF_DeleteStatus(s); TF_DeleteTensor(output); }; @@ -409,8 +409,12 @@ TEST_F(DeviceKernelOpTest, TestAllocateEmptyOutput) { TF_Tensor* output = TF_AllocateOutput( /*context=*/ctx, /*index=*/0, /*dtype=*/TF_FLOAT, /*dims=*/&dim, /*num_dims=*/1, /*len=*/0, s); + EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, &dim, 1, TF_FLOAT); + EXPECT_EQ(TF_FLOAT, TF_TensorType(output)); + EXPECT_EQ(1, TF_NumDims(output)); + EXPECT_EQ(0, TF_Dim(output, 0)); + TF_DeleteStatus(s); TF_DeleteTensor(output); }; @@ -430,16 +434,27 @@ TEST_F(DeviceKernelOpTest, TestAllocateOutputSize2x3) { TF_Status* s = TF_NewStatus(); // Allocate 2x3 output int64_t dim[2] = {2, 3}; - size_t tensor_size_bytes = TF_DataTypeSize(TF_FLOAT) * 6; + size_t tensor_size_bytes = 6 * TF_DataTypeSize(TF_FLOAT); TF_Tensor* output = TF_AllocateOutput( /*context=*/ctx, /*index=*/0, /*dtype=*/TF_FLOAT, /*dims=*/dim, /*num_dims=*/2, /*len=*/tensor_size_bytes, s); EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, dim, 2, TF_FLOAT); + EXPECT_EQ(TF_FLOAT, TF_TensorType(output)); + EXPECT_EQ(2, TF_NumDims(output)); + EXPECT_EQ(2, TF_Dim(output, 0)); + EXPECT_EQ(3, TF_Dim(output, 1)); // Set output to [1 2 3 4 5 6] - float values[6] = {1, 2, 3, 4, 5, 6}; - set_tensor_data(output, values, tensor_size_bytes, ctx); + void* data = TF_TensorData(output); + float value[6] = {1, 2, 3, 4, 5, 6}; +#if GOOGLE_CUDA + OpKernelContext* cc_ctx = reinterpret_cast(ctx); + cc_ctx->eigen_gpu_device().memcpyHostToDevice(data, value, + tensor_size_bytes); +#else + memcpy(data, value, tensor_size_bytes); +#endif + TF_DeleteStatus(s); TF_DeleteTensor(output); }; @@ -451,113 +466,4 @@ TEST_F(DeviceKernelOpTest, TestAllocateOutputSize2x3) { EXPECT_EQ("Tensor", output->DebugString(100)); } - -REGISTER_OP("AllocateTempOp1").Output("output1: float"); - -TEST_F(DeviceKernelOpTest, TestAllocateTempSizeOne) { - auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { - // Allocate output - TF_Status* s = TF_NewStatus(); - int64_t dim = 1; - TF_Tensor* output = TF_AllocateTemp( - /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, s); - size_t tensor_size_bytes = TF_DataTypeSize(TF_FLOAT); - EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, &dim, 1, TF_FLOAT); - - // Set output to 3 - float values[1] = {3.0f}; - set_tensor_data(output, values, tensor_size_bytes, ctx); - TF_SetOutput(ctx, 0, output, s); - TF_DeleteStatus(s); - TF_DeleteTensor(output); - }; - - SetupOp("AllocateTempOp1", "AllocateTemp1", my_compute_func); - - TF_ASSERT_OK(RunOpKernel()); - Tensor* output = GetOutput(0); - EXPECT_EQ("Tensor", - output->DebugString(100)); -} - -REGISTER_OP("AllocateTempOp0").Output("output1: float"); - -TEST_F(DeviceKernelOpTest, TestAllocateTempEmpty) { - auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { - TF_Status* s = TF_NewStatus(); - // Allocate empty output - int64_t dim = 0; - TF_Tensor* output = TF_AllocateTemp( - /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, s); - EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, &dim, 1, TF_FLOAT); - TF_SetOutput(ctx, 0, output, s); - TF_DeleteStatus(s); - TF_DeleteTensor(output); - }; - - SetupOp("AllocateTempOp0", "AllocateTemp0", my_compute_func); - - TF_ASSERT_OK(RunOpKernel()); - Tensor* output = GetOutput(0); - EXPECT_EQ("Tensor", - output->DebugString(100)); -} - -REGISTER_OP("AllocateTempOp2x3").Output("output1: float"); - -TEST_F(DeviceKernelOpTest, TestAllocateTempSize2x3) { - auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { - TF_Status* s = TF_NewStatus(); - size_t tensor_size_bytes = 6 * TF_DataTypeSize(TF_FLOAT); - // Allocate 2x3 output - int64_t dim[2] = {2, 3}; - TF_Tensor* output = TF_AllocateTemp( - /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/dim, - /*num_dims=*/2, s); - EXPECT_EQ(TF_OK, TF_GetCode(s)); - validate_tensor(output, dim, 2, TF_FLOAT); - - // Set output to [1 2 3 4 5 6] - void* data = TF_TensorData(output); - float values[6] = {1, 2, 3, 4, 5, 6}; - set_tensor_data(output, values, tensor_size_bytes, ctx); - TF_SetOutput(ctx, 0, output, s); - TF_DeleteStatus(s); - TF_DeleteTensor(output); - }; - - SetupOp("AllocateTempOp2x3", "AllocateTempOp2x3", my_compute_func); - - TF_ASSERT_OK(RunOpKernel()); - Tensor* output = GetOutput(0); - EXPECT_EQ("Tensor", - output->DebugString(100)); -} - -void validate_tensor(TF_Tensor* tensor, int64_t* dims, int64_t num_dims, - TF_DataType dtype){ - EXPECT_EQ(TF_FLOAT, TF_TensorType(tensor)); - EXPECT_EQ(num_dims, TF_NumDims(tensor)); - for(int i = 0; i < num_dims; ++i){ - EXPECT_EQ(dims[i], TF_Dim(tensor, i)); - } -} - -template -void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, - TF_OpKernelContext* ctx){ - T* data = reinterpret_cast(TF_TensorData(tensor)); -#if GOOGLE_CUDA - OpKernelContext* cc_ctx = reinterpret_cast(ctx); - cc_ctx->eigen_gpu_device().memcpyHostToDevice(data, values, - tensor_size_bytes); -#else - memcpy(data, values, tensor_size_bytes); -#endif -} - -} // namespace tensorflow \ No newline at end of file +} // namespace tensorflow From b8ec531156f669a343d45309e12ed900b1f9ebc8 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 13 Jul 2020 21:24:42 +0000 Subject: [PATCH 0051/1017] added TF_OFFSET_OF_END and typedef to struct --- tensorflow/c/tf_tensor.h | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index ce08a8d2678..dc659db1f7c 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -45,8 +45,15 @@ limitations under the License. extern "C" { #endif -// Allocator Attributes used for tensor allocation -struct TF_AllocatorAttributes { +// Macro used to calculate struct size for maintaining ABI stability across +// different struct implementations. +#ifndef TF_OFFSET_OF_END +#define TF_OFFSET_OF_END(TYPE, MEMBER) (offsetof(TYPE, MEMBER) + \ + sizeof(((TYPE *)0)->MEMBER)) +#endif // TF_OFFSET_OF_END + +// Allocator Attributes used for tensor allocation. +typedef struct TF_AllocatorAttributes { size_t struct_size; unsigned char on_host; } TF_AllocatorAttributes; From 749d3eb240e941769864e345314acf46575c9569 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 13 Jul 2020 21:42:04 +0000 Subject: [PATCH 0052/1017] changed naming to ScalarSummary and removed previous registration in core --- tensorflow/c/kernels/BUILD | 7 ++-- tensorflow/c/kernels/ops/summary.cc | 4 +-- tensorflow/c/kernels/summary_op.cc | 12 ++----- tensorflow/c/kernels/summary_op_test.cc | 4 +-- tensorflow/core/kernels/summary_op.cc | 44 ------------------------- tensorflow/core/ops/logging_ops.cc | 7 ---- 6 files changed, 12 insertions(+), 66 deletions(-) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index b713b27f5dc..7e103514645 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -88,12 +88,15 @@ tf_cc_test( filegroup( name = "android_all_op_kernels", srcs = [ - "bitcast_op.cc", "summary_op.cc" + "bitcast_op.cc", + "summary_op.cc" ], ) # LINT.ThenChange(//tensorflow/contrib/makefile/tf_op_files.txt) filegroup( name = "android_all_ops", - srcs = ["ops/bitcast.cc", "ops/summary.cc"], + srcs = ["ops/bitcast.cc", + "ops/summary.cc" + ], ) diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc index 355d73396b6..be39cd0f530 100644 --- a/tensorflow/c/kernels/ops/summary.cc +++ b/tensorflow/c/kernels/ops/summary.cc @@ -41,7 +41,7 @@ void Register_ScalarSummaryOp() { TF_Status* status = TF_NewStatus(); TF_OpDefinitionBuilder* op_builder = - TF_NewOpDefinitionBuilder("SummaryScalar"); + TF_NewOpDefinitionBuilder("ScalarSummary"); TF_OpDefinitionBuilderAddInput(op_builder, "tags: string"); TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); @@ -56,7 +56,7 @@ void Register_ScalarSummaryOp() { } TF_ATTRIBUTE_UNUSED static bool SummaryScalarOpRegistered = []() { - if (SHOULD_REGISTER_OP("SummaryScalar")) { + if (SHOULD_REGISTER_OP("ScalarSummary")) { Register_ScalarSummaryOp(); } return true; diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 18aa897bfa9..5db4a239905 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -34,17 +34,11 @@ typedef struct Params{ if (TF_GetCode(status) == TF_OK){ TF_GetInput(ctx, 1, &values, status); } - else { - values = nullptr; - } }; ~Params(){ TF_DeleteStatus(status); TF_DeleteTensor(tags); - // edge case if params fails to initialize - if (values != nullptr){ - TF_DeleteTensor(values); - } + TF_DeleteTensor(values); } }; @@ -134,7 +128,7 @@ template void RegisterSummaryScalarOpKernel() { TF_Status* status = TF_NewStatus(); { - auto* builder = TF_NewKernelBuilder("SummaryScalar", + auto* builder = TF_NewKernelBuilder("ScalarSummary", tensorflow::DEVICE_CPU, &SummaryScalarOp_Create, &SummaryScalarOp_Compute, @@ -143,7 +137,7 @@ void RegisterSummaryScalarOpKernel() { static_cast(tensorflow::DataTypeToEnum::v()), status); CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while adding type constraint"; - TF_RegisterKernelBuilder("SummaryScalar", builder, status); + TF_RegisterKernelBuilder("ScalarSummary", builder, status); CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while registering Summary Scalar kernel"; } diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc index 722373d36ce..ad5fafe5530 100644 --- a/tensorflow/c/kernels/summary_op_test.cc +++ b/tensorflow/c/kernels/summary_op_test.cc @@ -45,7 +45,7 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, // Initialize node used to fetch OpKernel Status status; NodeDef def; - def.set_op("SummaryScalar"); + def.set_op("ScalarSummary"); def.set_device(DEVICE_CPU); @@ -165,7 +165,7 @@ TEST(ScalarSummaryOpTest, Error_WrongWithSingleTag) { TEST(ScalarSummaryOpTest, IsRegistered){ const OpRegistrationData* reg; - TF_CHECK_OK(OpRegistry::Global()->LookUp("SummaryScalar", ®)); + TF_CHECK_OK(OpRegistry::Global()->LookUp("ScalarSummary", ®)); } } // namespace diff --git a/tensorflow/core/kernels/summary_op.cc b/tensorflow/core/kernels/summary_op.cc index f4c91fc9ff1..22d1a21a889 100644 --- a/tensorflow/core/kernels/summary_op.cc +++ b/tensorflow/core/kernels/summary_op.cc @@ -31,47 +31,6 @@ limitations under the License. namespace tensorflow { -template -class SummaryScalarOp : public OpKernel { - public: - explicit SummaryScalarOp(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* c) override { - const Tensor& tags = c->input(0); - const Tensor& values = c->input(1); - - OP_REQUIRES( - c, - tags.IsSameSize(values) || (TensorShapeUtils::IsScalar(tags.shape()) && - TensorShapeUtils::IsScalar(values.shape())), - errors::InvalidArgument( - "tags and values not the same shape: ", tags.shape().DebugString(), - " != ", values.shape().DebugString(), SingleTag(tags))); - auto Ttags = tags.flat(); - auto Tvalues = values.flat(); - Summary s; - for (int i = 0; i < Ttags.size(); i++) { - Summary::Value* v = s.add_value(); - const tstring& Ttags_i = Ttags(i); - v->set_tag(Ttags_i.data(), Ttags_i.size()); - v->set_simple_value(float(Tvalues(i))); - } - - Tensor* summary_tensor = nullptr; - OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape({}), &summary_tensor)); - CHECK(SerializeToTString(s, &summary_tensor->scalar()())); - } - - // If there's only one tag, include it in the error message - static string SingleTag(const Tensor& tags) { - if (tags.NumElements() == 1) { - return strings::StrCat(" (tag '", tags.flat()(0), "')"); - } else { - return ""; - } - } -}; - template class SummaryHistoOp : public OpKernel { public: @@ -114,9 +73,6 @@ class SummaryHistoOp : public OpKernel { }; #define REGISTER(T) \ - REGISTER_KERNEL_BUILDER( \ - Name("ScalarSummary").Device(DEVICE_CPU).TypeConstraint("T"), \ - SummaryScalarOp); \ REGISTER_KERNEL_BUILDER( \ Name("HistogramSummary").Device(DEVICE_CPU).TypeConstraint("T"), \ SummaryHistoOp); diff --git a/tensorflow/core/ops/logging_ops.cc b/tensorflow/core/ops/logging_ops.cc index 6489074b546..4d5ba6873a7 100644 --- a/tensorflow/core/ops/logging_ops.cc +++ b/tensorflow/core/ops/logging_ops.cc @@ -87,13 +87,6 @@ REGISTER_OP("TensorSummary") .Attr("display_name: string = ''") .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("ScalarSummary") - .Input("tags: string") - .Input("values: T") - .Output("summary: string") - .Attr("T: realnumbertype") - .SetShapeFn(shape_inference::ScalarShape); - REGISTER_OP("HistogramSummary") .Input("tag: string") .Input("values: T") From 6a572639d38b0889ed0cca6c3ef18849db001d40 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 14 Jul 2020 03:48:07 +0000 Subject: [PATCH 0053/1017] restore list_ops.py --- tensorflow/python/ops/list_ops.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tensorflow/python/ops/list_ops.py b/tensorflow/python/ops/list_ops.py index ccd4e6b0494..3e7c116ec97 100644 --- a/tensorflow/python/ops/list_ops.py +++ b/tensorflow/python/ops/list_ops.py @@ -248,7 +248,6 @@ def _TensorListFromTensorGrad(op, dlist): @ops.RegisterGradient("TensorListGetItem") def _TensorListGetItemGrad(op, ditem): """Gradient for TensorListGetItem.""" - print("---GetItemGrad---") list_size = gen_list_ops.tensor_list_length(op.inputs[0]) list_grad = gen_list_ops.tensor_list_set_item( gen_list_ops.tensor_list_reserve( @@ -257,21 +256,14 @@ def _TensorListGetItemGrad(op, ditem): list_size, element_dtype=ditem.dtype), index=op.inputs[1], item=ditem) - print("op inputs", op.inputs) - print("ditem", ditem) - print("list_grad", list_grad) index_grad = None element_shape_grad = None - print("------") return list_grad, index_grad, element_shape_grad @ops.RegisterGradient("TensorListSetItem") def _TensorListSetItemGrad(op, dlist): """Gradient function for TensorListSetItem.""" - print("---SetItemGrad---") - print("op inputs", op.inputs) - print("dlist", dlist) _, index, item = op.inputs list_grad = gen_list_ops.tensor_list_set_item( dlist, index=index, item=array_ops.zeros_like(item)) @@ -281,9 +273,6 @@ def _TensorListSetItemGrad(op, dlist): index, element_shape=array_ops.shape(item), element_dtype=item.dtype) - print("list_grad", list_grad) - print("value_grad", element_grad) - print("------") return list_grad, index_grad, element_grad From 395380e82d2b49d20b8cb46eaef98fc640a2cb58 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 14 Jul 2020 04:02:58 +0000 Subject: [PATCH 0054/1017] most pr edits --- tensorflow/core/framework/tensor_key.h | 6 ++--- tensorflow/core/kernels/map_kernels.cc | 2 +- tensorflow/core/kernels/map_kernels.h | 7 +++--- tensorflow/core/kernels/tensor_map.cc | 18 +++++---------- tensorflow/core/kernels/tensor_map.h | 17 ++++---------- tensorflow/core/kernels/tensor_map_test.cc | 22 +++++++++---------- tensorflow/core/ops/map_ops.cc | 7 +----- tensorflow/python/kernel_tests/BUILD | 3 --- .../python/kernel_tests/map_ops_test.py | 9 ++++---- tensorflow/python/ops/map_ops.py | 7 +----- 10 files changed, 35 insertions(+), 63 deletions(-) diff --git a/tensorflow/core/framework/tensor_key.h b/tensorflow/core/framework/tensor_key.h index aa6fe35181a..9a64969301f 100644 --- a/tensorflow/core/framework/tensor_key.h +++ b/tensorflow/core/framework/tensor_key.h @@ -47,13 +47,13 @@ class TensorKey : public Tensor { } friend bool operator!=(const TensorKey& t1, const TensorKey& t2) { - return !(t1==t2); + return !(t1 == t2); } - // AbslHashValue() function, needed for absl hashing. + // Needed for absl hash function. template friend H AbslHashValue(H h, const TensorKey& k) { - uint8* d = (uint8*)(k.data()); + const uint8* d = static_cast(k.data()); size_t s = k.AllocatedBytes(); std::vector vec; for (int i=0; i < s; i++) { diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index 7d45d3942e1..45fa86c2bf6 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -37,4 +37,4 @@ REGISTER_KERNEL_BUILDER(Name("TensorMapErase").Device(DEVICE_CPU), REGISTER_KERNEL_BUILDER(Name("TensorMapReplace").Device(DEVICE_CPU), TensorMapReplace); -} \ No newline at end of file +} diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 33282a75e0a..98ce1bfac1b 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -20,14 +20,13 @@ limitations under the License. #include "tensorflow/core/framework/variant_encode_decode.h" #include -using namespace std; namespace tensorflow { Status GetInputMap(OpKernelContext* c, int index, const TensorMap** map) { if (!TensorShapeUtils::IsScalar(c->input(index).shape())) { - return errors::InvalidArgument("Input map must be a scalar saw: ", + return errors::InvalidArgument("Input map must be a scalar. Saw: ", c->input(index).shape().DebugString()); } const TensorMap* m = c->input(index).scalar()().get(); @@ -41,6 +40,7 @@ Status GetInputMap(OpKernelContext* c, int index, const TensorMap** map) { } +//TODO(kattian): change into templated function Status ForwardInputOrCreateNewMap(OpKernelContext* c, int32 input_index, int32 output_index, const TensorMap& input_map, @@ -68,7 +68,7 @@ Status ForwardInputOrCreateNewMap(OpKernelContext* c, int32 input_index, } // If forwarding is not possible allocate a new output tensor and copy - // the `input_list` to it. + // the `input_map` to it. AllocatorAttributes attr; attr.set_on_host(true); TF_RETURN_IF_ERROR( @@ -183,6 +183,7 @@ class TensorMapErase : public OpKernel { DataType element_dtype_; }; + class TensorMapReplace : public OpKernel { public: explicit TensorMapReplace(OpKernelConstruction* c) : OpKernel(c) { diff --git a/tensorflow/core/kernels/tensor_map.cc b/tensorflow/core/kernels/tensor_map.cc index cfba3892650..bcb2abebe01 100644 --- a/tensorflow/core/kernels/tensor_map.cc +++ b/tensorflow/core/kernels/tensor_map.cc @@ -44,7 +44,6 @@ void TensorMap::Encode(VariantTensorData* data) const { // Metadata format: // core::PutVarint64(&metadata, static_cast(element_dtype)); - core::PutVarint64(&metadata, static_cast(max_num_elements)); TensorShapeProto element_shape_proto; element_shape.AsProto(&element_shape_proto); element_shape_proto.AppendToString(&metadata); @@ -56,17 +55,15 @@ static Status TensorMapDeviceCopy( const UnaryVariantOpRegistry::AsyncTensorDeviceCopyFn& copy) { to->element_shape = from.element_shape; to->element_dtype = from.element_dtype; - to->max_num_elements = from.max_num_elements; for (const std::pair& p : from.tensors()) { - to->tensors().emplace(p); //TODO: check valid dtype - //if (t.dtype() != DT_INVALID) { - //TF_RETURN_IF_ERROR(copy(p, &to->tensors().back())); - //} + if (p.first.dtype() != DT_INVALID && p.second.dtype() != DT_INVALID) { + to->tensors().emplace(p.first, p.second); + } } return Status::OK(); } -#define REGISTER_LIST_COPY(DIRECTION) \ +#define REGISTER_LIST_COPY(DIRECTION) \ INTERNAL_REGISTER_UNARY_VARIANT_DEVICE_COPY_FUNCTION(TensorMap, DIRECTION, \ TensorMapDeviceCopy) @@ -89,19 +86,16 @@ bool TensorMap::Decode(const VariantTensorData& data) { while (tensors_it != data.tensors().end()) { - // should assert that tensors_it + 1 is also not the end if (std::next(tensors_it) == data.tensors().end()) { return false; } - TensorKey k = TensorKey(*tensors_it); // copy inefficient? - tensors().emplace(k,*++tensors_it); - tensors_it++; + tensors().emplace(tensors_it[0], tensors_it[1]); + tensors_it += 2; } core::GetVarint64(&iter, &scratch); element_dtype = static_cast(scratch); core::GetVarint64(&iter, &scratch); - max_num_elements = static_cast(scratch); TensorShapeProto element_shape_proto; element_shape_proto.ParseFromString(string(iter.data(), iter.size())); element_shape = PartialTensorShape(element_shape_proto); diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index 633c7db8668..f0fe4ae5f57 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -38,11 +38,11 @@ namespace tensorflow { // // Do not create a true copy of the underlying container - but instead increment // a reference count. Modifying b.tensors() modifies a.tensors(). In this way, -// TensorList should be considered similar to the tf::Tensor object. +// TensorMap should be considered similar to the tf::Tensor object. // // In order to get a copy of the underlying map, use the Copy method: // -// TensorList b = a.Copy(); +// TensorMap b = a.Copy(); // b.tensors().insert(k, v); // This does not modify a.tensors(). // // Note that this is not a deep copy: the memory locations of the underlying @@ -50,8 +50,8 @@ namespace tensorflow { // in the original. To truly perform a deep copy, Device and Type-specific // code needs to be applied to the underlying tensors as usual. // -// The most important implication of RefCounted TLs is that OpKernels -// wishing to reuse TensorList inputs as outputs via context->forward_input() +// The most important implication of RefCounted TensorMaps is that OpKernels +// wishing to reuse TensorMap inputs as outputs via context->forward_input() // need to perform an additional check on the refcount of the TensorList, // to ensure aliasing can be performed safely. For example: // @@ -72,7 +72,6 @@ class TensorMap { TensorMap(const TensorMap& other) : element_shape(other.element_shape), element_dtype(other.element_dtype), - max_num_elements(other.max_num_elements), tensors_(other.tensors_) { tensors_->Ref(); } @@ -80,7 +79,6 @@ class TensorMap { TensorMap(TensorMap&& rhs) : element_shape(std::move(rhs.element_shape)), element_dtype(rhs.element_dtype), - max_num_elements(rhs.max_num_elements), tensors_(rhs.tensors_) { rhs.tensors_ = nullptr; } @@ -89,7 +87,6 @@ class TensorMap { if (this == &rhs) return *this; element_shape = rhs.element_shape; element_dtype = rhs.element_dtype; - max_num_elements = rhs.max_num_elements; tensors_->Unref(); tensors_ = rhs.tensors_; tensors_->Ref(); @@ -100,7 +97,6 @@ class TensorMap { if (this == &rhs) return *this; element_shape = rhs.element_shape; element_dtype = rhs.element_dtype; - max_num_elements = rhs.max_num_elements; std::swap(tensors_, rhs.tensors_); return *this; } @@ -120,10 +116,6 @@ class TensorMap { DataType element_dtype; - // The maximum allowed size of `tensors`. Defaults to -1 meaning that the size - // of `tensors` is unbounded. - int max_num_elements = -1; - // Access to the underlying tensor container. absl::flat_hash_map& tensors() { return tensors_->values_; } const absl::flat_hash_map& tensors() const { return tensors_->values_; } @@ -137,7 +129,6 @@ class TensorMap { TensorMap out; out.element_shape = element_shape; out.element_dtype = element_dtype; - out.max_num_elements = max_num_elements; // This performs a copy of the absl::hashmap. out.tensors_->values_ = tensors_->values_; return out; diff --git a/tensorflow/core/kernels/tensor_map_test.cc b/tensorflow/core/kernels/tensor_map_test.cc index 294aa07c963..1ee175be34d 100644 --- a/tensorflow/core/kernels/tensor_map_test.cc +++ b/tensorflow/core/kernels/tensor_map_test.cc @@ -35,21 +35,21 @@ TEST(TensorMapTest, Empty) { TEST(TensorKeyTest, Equal) { TensorKey k1 = Tensor(15); TensorKey k2 = Tensor(15); - EXPECT_EQ(k1,k2); + EXPECT_EQ(k1, k2); TensorKey k3 = Tensor(15); TensorKey k4 = Tensor(37); - EXPECT_NE(k3,k4); + EXPECT_NE(k3, k4); } TEST(TensorMapTest, Insert) { - EXPECT_EQ(1,1); + EXPECT_EQ(1, 1); TensorMap tm; TensorKey k = Tensor(11); Tensor v = Tensor(22); - tm.insert(k,v); + tm.insert(k, v); absl::flat_hash_map am; - am.try_emplace(k,v); + am.try_emplace(k, v); absl::flat_hash_map::iterator map_it = tm.tensors().begin(); EXPECT_EQ(map_it->first, k); @@ -62,7 +62,7 @@ TEST(TensorMapTest, Lookup) { TensorMap tm; TensorKey k = Tensor(11); Tensor v = Tensor(22); - tm.insert(k,v); + tm.insert(k, v); absl::flat_hash_map::iterator map_it = tm.find(k); Tensor f = map_it->second; @@ -74,7 +74,7 @@ TEST(TensorMapTest, Erase) { TensorMap tm; TensorKey k = Tensor(11); Tensor v = Tensor(22); - tm.insert(k,v); + tm.insert(k, v); tm.erase(k); EXPECT_EQ(tm.find(k), tm.tensors().end()); } @@ -84,8 +84,8 @@ TEST(TensorMapTest, SameKeyInsert) { TensorKey k = Tensor(11); Tensor v1 = Tensor(22); Tensor v2 = Tensor(23); - bool b1 = tm.insert(k,v1); - bool b2 = tm.insert(k,v2); + bool b1 = tm.insert(k, v1); + bool b2 = tm.insert(k, v2); EXPECT_EQ(b1, true); EXPECT_EQ(b2, false); absl::flat_hash_map::iterator map_it = tm.find(k); @@ -109,7 +109,7 @@ TEST(TensorMapTest, Copy) { TensorMap tm; TensorKey k = Tensor(11); Tensor v = Tensor(22); - tm.insert(k,v); + tm.insert(k, v); TensorMap tmc = tm.Copy(); EXPECT_EQ(tm.dtype(), tmc.dtype()); EXPECT_EQ(tm.size(), tmc.size()); @@ -123,7 +123,7 @@ TEST(TensorMapTest, EncodeDecode) { TensorMap tm; TensorKey k = Tensor(11); Tensor v = Tensor(22); - tm.insert(k,v); + tm.insert(k, v); VariantTensorData data; tm.Encode(&data); TensorMap tmc; diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index 445180c34ef..e95dd2486be 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -20,12 +20,7 @@ limitations under the License. namespace tensorflow { namespace { -bool IsValidTensorMapHandleData( - const std::vector* handle_data) { - std::cout << "is valid tensor map handle data " << handle_data->size() << std::endl; - return handle_data != nullptr && handle_data->size() == 1; -} - +//TODO(kttian): Support non-scalar values REGISTER_OP("EmptyTensorMap") .Output("handle: variant") .SetShapeFn([](shape_inference::InferenceContext* c) { diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index d8c8e3dc2a8..55a8feeb053 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -147,9 +147,6 @@ cuda_py_test( size = "small", srcs = ["map_ops_test.py"], grpc_enabled = True, - tags = [ - "noasan", # TODO(b/155406705): flaky - ], deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 46f8f21d104..e95a1ab9bec 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -12,12 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -"""Tests for zero_out ops.""" +"""Tests for TensorMap ops.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function -#import numpy as np from tensorflow.python.platform import test from absl.testing import parameterized from tensorflow.python.framework import test_util @@ -40,7 +39,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testTensorMapSize(self): m = map_ops.empty_tensor_map() s = map_ops.tensor_map_size(m) - self.assertAllClose(s, 0) + self.assertAllEqual(s, 0) def testTensorMapInsert(self): m = map_ops.empty_tensor_map() @@ -48,7 +47,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k, v) s = map_ops.tensor_map_size(m) - self.assertAllClose(s, 1) + self.assertAllEqual(s, 1) def testTensorMapLookup(self): m = map_ops.empty_tensor_map() @@ -98,4 +97,4 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): if __name__ == '__main__': - test.main() \ No newline at end of file + test.main() diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 5d8d2b88f2f..4ea50e114ac 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -"""Use zero_out ops in python.""" +"""Ops to manipulate hashmap of tensors.""" from __future__ import absolute_import from __future__ import division @@ -27,11 +27,6 @@ from tensorflow.python.ops import gen_map_ops from tensorflow.python.ops.gen_map_ops import * from tensorflow.python.framework import constant_op - -#zero_out_ops = load_library.load_op_library( -# resource_loader.get_path_to_datafile('_zero_out_ops.so')) -#zero_out = zero_out_ops.zero_out - ops.NotDifferentiable("EmptyTensorMap") def empty_tensor_map(): From 820519bcf32e13b8229b34816ed65662f939e81b Mon Sep 17 00:00:00 2001 From: "902449@58880@bigcat_chen@ASIC" Date: Tue, 14 Jul 2020 17:42:11 +0800 Subject: [PATCH 0055/1017] TFLM: add HIMAX WE1 EVB to support TFLM example(magic wand and micro speech) --- .../lite/micro/examples/magic_wand/README.md | 133 ++++++++++++++++++ .../himax_we1_evb/accelerometer_handler.cc | 89 ++++++++++++ .../micro/examples/micro_speech/README.md | 100 +++++++++++++ .../himax_we1_evb/audio_provider.cc | 58 ++++++++ .../himax_we1_evb/command_responder.cc | 59 ++++++++ .../images/animation_on_himax_we1_evb.gif | Bin 0 -> 1063997 bytes .../tools/make/third_party_downloads.inc | 4 +- 7 files changed, 441 insertions(+), 2 deletions(-) create mode 100644 tensorflow/lite/micro/examples/magic_wand/himax_we1_evb/accelerometer_handler.cc create mode 100644 tensorflow/lite/micro/examples/micro_speech/himax_we1_evb/audio_provider.cc create mode 100644 tensorflow/lite/micro/examples/micro_speech/himax_we1_evb/command_responder.cc create mode 100644 tensorflow/lite/micro/examples/micro_speech/images/animation_on_himax_we1_evb.gif diff --git a/tensorflow/lite/micro/examples/magic_wand/README.md b/tensorflow/lite/micro/examples/magic_wand/README.md index 0cf3b8e74c3..094e985586f 100644 --- a/tensorflow/lite/micro/examples/magic_wand/README.md +++ b/tensorflow/lite/micro/examples/magic_wand/README.md @@ -12,6 +12,7 @@ then outputs the gesture to the serial port. - [Getting started](#getting-started) - [Deploy to Arduino](#deploy-to-arduino) +- [Deploy to Himax WE1 EVB](#deploy-to-himax-we1-evb) - [Deploy to SparkFun Edge](#deploy-to-sparkfun-edge) - [Run the tests on a development machine](#run-the-tests-on-a-development-machine) - [Train your own model](#train-your-own-model) @@ -140,6 +141,138 @@ SLOPE: * * * * * * * * ``` +## Deploy to Himax WE1 EVB + +The following instructions will help you build and deploy this example to +[HIMAX WE1 EVB](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_board_brief) +board. To undstand more about using this board, please check +[HIMAX WE1 EVB user guide](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide). + +### Initial Setup + +To use the HIMAX WE1 EVB, please make sure following software are installed: + +#### MetaWare Development Toolkit + +See +[Install the Synopsys DesignWare ARC MetaWare Development Toolkit](/tensorflow/lite/micro/tools/make/targets/arc/README.md#install-the-synopsys-designware-arc-metaware-development-toolkit) +section for instructions on toolchain installation. + +#### Make Tool version + +A `'make'` tool is required for deploying Tensorflow Lite Micro applications on +HIMAX WE1 EVB, See +[Check make tool version](/tensorflow/lite/micro/tools/make/targets/arc/README.md#make-tool) +section for proper environment. + +#### Serial Terminal Emulation Application + +There are 2 main purposes for HIMAX WE1 EVB Debug UART port + +- print application output +- burn application to flash by using xmodem send application binary + +You can use any terminal emulation program (like [PuTTY](https://www.putty.org/) +or [minicom](https://linux.die.net/man/1/minicom)). + +### Generate Example Project + +The example project for HIMAX WE1 EVB platform can be generated with the +following command: + +Download related third party data + +``` +make -f tensorflow/lite/micro/tools/make/Makefile TARGET=himax_we1_evb third_party_downloads +``` + +Generate magic wand project + +``` +make -f tensorflow/lite/micro/tools/make/Makefile generate_magic_wand_make_project TARGET=himax_we1_evb +``` + +### Build and Burn Example + +Following the Steps to run magic wand example at HIMAX WE1 EVB platform. + +1. Go to the generated example project directory. + + ``` + cd tensorflow/lite/micro/tools/make/gen/himax_we1_evb_arc/prj/magic_wand/make + ``` + +2. Build the example using + + ``` + make app + ``` + +3. After example build finish, copy ELF file and map file to image generate + tool directory. \ + image generate tool directory located at + `'tensorflow/lite/micro/tools/make/downloads/himax_we1_sdk/image_gen_linux_v3/'` + + ``` + cp magic_wand.elf himax_we1_evb.map ../../../../../downloads/himax_we1_sdk/image_gen_linux_v3/ + ``` + +4. Go to flash image generate tool directory. + + ``` + cd ../../../../../downloads/himax_we1_sdk/image_gen_linux_v3/ + ``` + +5. run image generate tool, generate flash image file. + + * Before running image generate tool, by typing `sudo chmod +x image_gen` + and `sudo chmod +x sign_tool` to make sure it is executable. + + ``` + image_gen -e magic_wand.elf -m himax_we1_evb.map -o out.img + ``` + +6. Download flash image file to HIMAX WE1 EVB by UART: + + * more detail about download image through UART can be found at + [HIMAX WE1 EVB update Flash image](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide#flash-image-update) + +After these steps, press reset button on the HIMAX WE1 EVB, you will see +application output in the serial terminal. Perform following gestures `'Wing'`,`'Ring'`,`'Slope'` and you can see the otuput in serial terminal. + +``` +WING: +* * * + * * * * + * * * * + * * * * + * * * * + * * +``` + +``` +RING: + * + * * + * * + * * + * * + * * + * +``` + +``` +SLOPE: + * + * + * + * + * + * + * + * * * * * * * * +``` + ## Deploy to SparkFun Edge The following instructions will help you build and deploy this sample on the diff --git a/tensorflow/lite/micro/examples/magic_wand/himax_we1_evb/accelerometer_handler.cc b/tensorflow/lite/micro/examples/magic_wand/himax_we1_evb/accelerometer_handler.cc new file mode 100644 index 00000000000..9d83b01be05 --- /dev/null +++ b/tensorflow/lite/micro/examples/magic_wand/himax_we1_evb/accelerometer_handler.cc @@ -0,0 +1,89 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/examples/magic_wand/accelerometer_handler.h" + +#include "hx_drv_tflm.h" + +int begin_index = 0; + +namespace { +// Ring buffer size +constexpr int ring_buffer_size = 600; +// Ring buffer +float save_data[ring_buffer_size] = {0.0}; +// Flag to start detect gesture +bool pending_initial_data = true; +// Available data count in accelerometer FIFO +int available_count = 0; + +} // namespace + +TfLiteStatus SetupAccelerometer(tflite::ErrorReporter* error_reporter) { + if (hx_drv_accelerometer_initial() != HX_DRV_LIB_PASS) { + TF_LITE_REPORT_ERROR(error_reporter, "setup fail"); + return kTfLiteError; + } + + TF_LITE_REPORT_ERROR(error_reporter, "setup done"); + + return kTfLiteOk; +} + +bool ReadAccelerometer(tflite::ErrorReporter* error_reporter, float* input, + int length) { + // Check how many accelerometer data + available_count = hx_drv_accelerometer_available_count(); + + if (available_count == 0) return false; + + for (int i = 0; i < available_count; i++) { + float x, y, z; + hx_drv_accelerometer_receive(&x, &y, &z); + + const float norm_x = -x; + const float norm_y = y; + const float norm_z = z; + + // Save data in milli-g unit + save_data[begin_index++] = norm_x * 1000; + save_data[begin_index++] = norm_y * 1000; + save_data[begin_index++] = norm_z * 1000; + + // If reach end of buffer, return to 0 position + if (begin_index >= ring_buffer_size) begin_index = 0; + } + + // Check if data enough for prediction + if (pending_initial_data && begin_index >= 200) { + pending_initial_data = false; + } + + // Return if we don't have enough data + if (pending_initial_data) { + return false; + } + + // Copy the requested number of bytes to the provided input tensor + for (int i = 0; i < length; ++i) { + int ring_array_index = begin_index + i - length; + if (ring_array_index < 0) { + ring_array_index += ring_buffer_size; + } + input[i] = save_data[ring_array_index]; + } + + return true; +} diff --git a/tensorflow/lite/micro/examples/micro_speech/README.md b/tensorflow/lite/micro/examples/micro_speech/README.md index a4a2f2d3be7..0ee367bd854 100644 --- a/tensorflow/lite/micro/examples/micro_speech/README.md +++ b/tensorflow/lite/micro/examples/micro_speech/README.md @@ -22,6 +22,7 @@ kilobytes of Flash. - [Deploy to SparkFun Edge](#deploy-to-sparkfun-edge) - [Deploy to STM32F746](#deploy-to-STM32F746) - [Deploy to NXP FRDM K66F](#deploy-to-nxp-frdm-k66f) +- [Deploy to HIMAX WE1 EVB](#deploy-to-himax-we1-evb) - [Run on macOS](#run-on-macos) - [Run the tests on a development machine](#run-the-tests-on-a-development-machine) - [Train your own model](#train-your-own-model) @@ -562,6 +563,105 @@ using [ARM Mbed](https://github.com/ARMmbed/mbed-cli). in black color. If there is no output on the serial port, you can connect headphone to headphone port to check if audio loopback path is working. +## Deploy to HIMAX WE1 EVB + +The following instructions will help you build and deploy this example to +[HIMAX WE1 EVB](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_board_brief) +board. To undstand more about using this board, please check +[HIMAX WE1 EVB user guide](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide). + +### Initial Setup + +To use the HIMAX WE1 EVB, please make sure following software are installed: + +#### MetaWare Development Toolkit + +See +[Install the Synopsys DesignWare ARC MetaWare Development Toolkit](/tensorflow/lite/micro/tools/make/targets/arc/README.md#install-the-synopsys-designware-arc-metaware-development-toolkit) +section for instructions on toolchain installation. + +#### Make Tool version + +A `'make'` tool is required for deploying Tensorflow Lite Micro +applications on HIMAX WE1 EVB, See +[Check make tool version](/tensorflow/lite/micro/tools/make/targets/arc/README.md#make-tool) +section for proper environment. + +#### Serial Terminal Emulation Application + +There are 2 main purposes for HIMAX WE1 EVB Debug UART port + +- print application output +- burn application to flash by using xmodem send application binary + +You can use any terminal emulation program (like [PuTTY](https://www.putty.org/) or [minicom](https://linux.die.net/man/1/minicom)). + + +### Generate Example Project + +The example project for HIMAX WE1 EVB platform can be generated with the following +command: + +Download related third party data + +``` +make -f tensorflow/lite/micro/tools/make/Makefile TARGET=himax_we1_evb third_party_downloads +``` + +Generate micro speech project + +``` +make -f tensorflow/lite/micro/tools/make/Makefile generate_micro_speech_make_project TARGET=himax_we1_evb +``` + +### Build and Burn Example + +Following the Steps to run micro speech example at HIMAX WE1 EVB platform. + +1. Go to the generated example project directory. + + ``` + cd tensorflow/lite/micro/tools/make/gen/himax_we1_evb_arc/prj/micro_speech/make + ``` + +2. Build the example using + + ``` + make app + ``` + +3. After example build finish, copy ELF file and map file to image generate tool directory. + image generate tool directory located at `'tensorflow/lite/micro/tools/make/downloads/himax_we1_sdk/image_gen_linux_v3/'` + + ``` + cp micro_speech.elf himax_we1_evb.map ../../../../../downloads/himax_we1_sdk/image_gen_linux_v3/ + ``` + +4. Go to flash image generate tool directory. + + ``` + cd ../../../../../downloads/himax_we1_sdk/image_gen_linux_v3/ + ``` + +5. run image generate tool, generate flash image file. + + * Before running image generate tool, by typing `sudo chmod +x image_gen` + and `sudo chmod +x sign_tool` to make sure it is executable. + + ``` + image_gen -e micro_speech.elf -m himax_we1_evb.map -o out.img + ``` + + +6. Download flash image file to HIMAX WE1 EVB by UART: + + * more detail about download image through UART can be found at [HIMAX WE1 EVB update Flash image](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide#flash-image-update) + +After these steps, press reset button on the HIMAX WE1 EVB, you will see application output in the serial +terminal and lighting LED. + +![Animation on Himax WE1 EVB](images/animation_on_himax_we1_evb.gif) + ## Run on macOS The example contains an audio provider compatible with macOS. If you have access diff --git a/tensorflow/lite/micro/examples/micro_speech/himax_we1_evb/audio_provider.cc b/tensorflow/lite/micro/examples/micro_speech/himax_we1_evb/audio_provider.cc new file mode 100644 index 00000000000..a779b4c039d --- /dev/null +++ b/tensorflow/lite/micro/examples/micro_speech/himax_we1_evb/audio_provider.cc @@ -0,0 +1,58 @@ +/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/examples/micro_speech/audio_provider.h" + +#include "tensorflow/lite/micro/examples/micro_speech/micro_features/micro_model_settings.h" + +#include "hx_drv_tflm.h" + +namespace { +// Feedback silence buffer when beginning start_ms <= 0 +int16_t g_silence[kMaxAudioSampleSize] = {0}; +// Latest time-stamp +int32_t g_latest_audio_timestamp = 0; +// config about audio data size and address +hx_drv_mic_data_config_t mic_config; +// Flag for check if audio is initialize or not +bool g_is_audio_initialized = false; +} // namespace + +TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter, + int start_ms, int duration_ms, + int* audio_samples_size, int16_t** audio_samples) { + if (!g_is_audio_initialized) { + if (hx_drv_mic_initial() != HX_DRV_LIB_PASS) return kTfLiteError; + + hx_drv_mic_on(); + g_is_audio_initialized = true; + } + + if (start_ms > 0) { + hx_drv_mic_capture(&mic_config); + } else { + mic_config.data_size = kMaxAudioSampleSize; + mic_config.data_address = (uint32_t)g_silence; + } + + *audio_samples_size = mic_config.data_size; + *audio_samples = (int16_t*)mic_config.data_address; + return kTfLiteOk; +} + +int32_t LatestAudioTimestamp() { + hx_drv_mic_timestamp_get(&g_latest_audio_timestamp); + return g_latest_audio_timestamp; +} diff --git a/tensorflow/lite/micro/examples/micro_speech/himax_we1_evb/command_responder.cc b/tensorflow/lite/micro/examples/micro_speech/himax_we1_evb/command_responder.cc new file mode 100644 index 00000000000..22e941df959 --- /dev/null +++ b/tensorflow/lite/micro/examples/micro_speech/himax_we1_evb/command_responder.cc @@ -0,0 +1,59 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/micro/examples/micro_speech/command_responder.h" +#include "hx_drv_tflm.h" + +static int32_t last_command_time = 0; +static uint32_t loop = 0; +static bool all_on = 0; + +void RespondToCommand(tflite::ErrorReporter* error_reporter, + int32_t current_time, const char* found_command, + uint8_t score, bool is_new_command) { + loop++; + if (is_new_command) { + TF_LITE_REPORT_ERROR(error_reporter, "Heard %s (%d) @%dms", found_command, + score, current_time); + if (found_command[0] == 'y') { + last_command_time = current_time; + hx_drv_led_off(HX_DRV_LED_RED); + hx_drv_led_on(HX_DRV_LED_GREEN); + } else if (found_command[0] == 'n') { + last_command_time = current_time; + hx_drv_led_off(HX_DRV_LED_GREEN); + hx_drv_led_on(HX_DRV_LED_RED); + } + } + + if (last_command_time != 0) { + if (last_command_time < (current_time - 3000)) { + last_command_time = 0; + hx_drv_led_off(HX_DRV_LED_GREEN); + hx_drv_led_off(HX_DRV_LED_RED); + } + } else { + if ((loop % 10) == 0) { + if (all_on) { + hx_drv_led_on(HX_DRV_LED_RED); + hx_drv_led_on(HX_DRV_LED_GREEN); + } else { + hx_drv_led_off(HX_DRV_LED_RED); + hx_drv_led_off(HX_DRV_LED_GREEN); + } + all_on = !all_on; + } + } +} diff --git a/tensorflow/lite/micro/examples/micro_speech/images/animation_on_himax_we1_evb.gif b/tensorflow/lite/micro/examples/micro_speech/images/animation_on_himax_we1_evb.gif new file mode 100644 index 0000000000000000000000000000000000000000..5897c43d5a20c735f8f6fd086720fce083ffb30c GIT binary patch literal 1063997 zcmZ?wbhEHblwb^D{La9@%frjV$tujtE5XYlBE%~#%quC*%*Q9h%O@$sCoIk<$|oeu zCn6~L!>tgNl7tgEG}s;sT0s-vr| zt*)VAq^fMJqhh41ZKDR{ zYh`U_VXkXvEUP5qZYJqtEb3~Zg8nS>S*KbV(sT?>+5YAWf$&Y=I3Eh9YnQB~|D`;iu zs%P$_Y8dHk?d)dj9%%1m;~Z+~JjAX9PHy9;^P(V>mBLk8W!Lh z6YLWa?B(te7UCNc6A~O15#|>dlp5hx5#x~@8k`axk{uhJ9vzV#8c>_ z?%L|ks?wg)oSFIYn|sV>PEkCz$m+*WiJ!mKfBY~%e_n59Yruwn|DDT%dYY>{Tgs<) zR8H+GS~aC`>++H#vwY62^*gsd@5<_wOXq#Qe)WBEGjHD!v$UL+?2?YsijLBnNimtt z?RCu)>zb>Zr&c%ib+@)o>uQ@lrK7!b`mCulru0o+IgFZ0x2~PNYwgTko91rcIBUnI2~%e-TQYCa=6MTOEm^j8)zaOY zmu}y(ef`=kGnZ{WvAO5)j@6I%b)G%C<@x2gn|7X-E&+X?L671|4;A*62z|6?N zz@Wpxz`y{?CJY?^7@l*~7`HDtXwD(56>~x$ahf^5yi<#WVfR!WG2MMTGnOts+NaoK=rrlDF*i#`Z zFE00;tK@04^_7Tf$Y!m;ptNki`EE;Ed>1|OiCph7%P-T5^>xS+&uJ54m7cElnH{nG z&5}dgxkR-?uC3wpnV~PBN_)rDmf zy*y`|tO|N{dFgWB*|MFJc0D-xFYW)lvh2^Vyr*jfzAt?nwD{?3t%d9MEUQ}N^*%ZD z{q$2pH(EH=Bt&yPCbl)UX|lEOJ6dcz%&X^P<}Sk&X?D|*)pOc}^aYM4$6X~l#gg2X zR4$p-sW9=#EKlXVWkqgNs#0z;=(d7lt{8ua<;(Yw>#IZda=9QiwCa?H7>Pp;1n<%zga z*dfc-BH^WT<3{oXUyYd#B_1~}ILI!Ds(SUt>;ntxv@VNY`zLm1R{33%DJ!!T zD_4~Uos1F*zEE{_ZqRYFrJZYo($ZCDCMBL+I`vwn(X)^g&E0EGn0C5r^{))G4r-_g zT$mf(p*YQrzuWZ0b8fL~i4$Zb_f=1DniqL7a<88|uatYYUX@gidS%MX{VF#rG7nfL zSxvsdvHpv+Z>U4LaM03Cw?dA~n1^1ubNOt{91+>Wdq4g=wf>P*QB>l=t@py>Z*6Gh z_4oL=QO&Jpk4R~Prln}$QO9oaVzakj=5NgoS-5svuFpia_{GZw*Cp|Ay=B_a6E^=W0bvzN|TrLxV3^Yv0`Q4V!~A^+zmzijxN<@QwWsH}UX zwoq=|(d+99Z~roUu;X3s-V@QKJ6VLIyfz+vbot;}i{Q_;d!`6_e2NH(3%|K!{zzj%E6;zJ5h0b@wHcS4w*y$1XW=%<=vY)yfl# zIdk_+Q&yOIs?vGY`7avB4s8_4w0UkQHpAiB5`EFvp$*-a{<67Se7Jd_@E>!@q(?i7 z6?VrYa_`^OUGc75bcar&dST?`|0Qk`7bMvXmwTU?r_iFu<=GKCJ#qT$r113>wM#O^ zW=dW4d9r@ujTf$axaTd4P_nVA>u z`RRuRb@5zGMO6;0zF!_BA3ycv@w7$eeN)rzY#o|-r+iwiuc zpUt$#PL(<;4}|j>&ZcNsw)xi|JHcVr5gIk2B|!Sh0mEmv4s!97cgNgu;?CVAahh$p z{i8L<)8;((mM&4-#US*7{c+IUS68O4GqfsD{J815q-Mr3)21h`k-MGTi;P$L?p1Nu zxS1^HeQ5Rqqio&7icgMSXqI1oM?ch|(K{?|*$(e{!Ca9MC#9DJFl+L#8Gc{FWVd#f zx@v|~Ofu`Oh~@-lsS6HlrYjhElLVNRZ%pMV|H~lwnUQFy8@25%xzRy98_|7 z@&UoV+eiKhn(yRXk~&o&MEwY7NPEcY#sNmCSHXG#x{ng`WH$v)jOPo`U)Pn zJX=5Q^~{|3gwLT9&35>4-MXB-Z}u5ZP8GgN->W-wKCXZ2uDn=T_8rzOiN_f@m%m^$-PPEa zHz$ef@V)me5(>FYCJju+_)wjxF|hZfV~Lud2f9hseJ$Q{*jh&zy> zk-crfbH$1Vf!z!n9z6TJbZ140#`nra`YV5gUaaLlWdEVDIjsEQwN}HKKFTLtbut`- zymgPLoSW_!KIc{F=R*B0|9VCKC5ae5`uoJf$Fi^d%9d$iX_w|NJfdQkd!e~~Un#3r zz`+fw3XFx~4Ga$aRTsn^8klYz;J<8f+;ZQ=-n==7C37Ay%#_~HuqyoSt^08eEI|UP zneVrs7M3WUr_Io$TjMA)=LL(zm9=?(1xdnl7qIR=_<=>>%XFr1b}f2GHn!`U9E^N- z@KV*g?*^(F>*}xldMav~eSlqcf7vP5 z;q0X=HrTxijkGxOFm94h;GuNhiVI=VI}%uBA2hI^%1&iwn9Ib%V0W?AuSSo(mPzD+ z(RA4cMs173FMa#}SuSy87xiGclfq`TL;T-_nt#h2SR(IzsyoiZ$YF7SLA=6AJZ6wef z?{8!~u~FRNn%5+^#5`5zTT7XOk1^U@FHSv{S~)HKT3L(hja(Lok~`OP6WL2Dm(|3& zm7EYS<5gfZW#D|qPr-%Q^mao0>;8ivIX4lQqfsucCyqY0!Zw z3bR^cdBme;6gIw_WD`nX^m2`fHZ@MAA^1941 zqp&U8n{Tnk9X5-5oSGV?)|&90A&{e$SAoIRqBLT9OX&2P@bJ>a?F=p&z0U5vd=r@R z6?na_xB9yCKEJ?w&Vj?gfg!~sExD}kQdw=Bcq{h@2Jr;u?CE7C9O(sW93|ga;~$jG z52*iP&h5J)T_J$Iy~6&Dh^x;c!<58|s*Cbn=|QDTCXNb?Pd`a$ePEI}5hO5Ol3zt| za-*OMN7K>_$=C&~$}78y1-dPhiv*YXXNrbf3xwy2{$tJ;WtMHIwz}T^^lI^|AGuEI zJ%`;}LbkVLXiRZE(GqgKC-g>-cSouB&z9hJMjHms`!`B_j`w?g z%B14{Lt$SQ+0N_og`0G96Dkg}#0uW5=zGqjnV>l9gyigxCa1R~elTL6R;V%cB5O6n ztk?^zKYp+VE?^BUVEM6x({TZ(*#QZ&1v<#R+Da&MT`#Af!zl=THaXVAL9e(~RioQvCe??mw4{lUTi zAUE+i!(~fG0fxM|Tx?<(M3 zvLV9kNCf9et=3Gr*h>k$(~SAc3Ir_^Z=^Qrl+J9rB+&WDXU=j?Ij*L~H(6sYuqFtw zC2VMqzYxdwHO1`0BHdSEw_L+_c+M`FI(xe~^Y-bp%{3VvG#Hf@^u=sT_3EfO^|R&X zx1KX6=bX=J%`J0^ zw@2@ps?y47xgqRR_!OA?9C+{Sy7PGPpX%=ovw90-EuXv_1d&GA_8mq6d1D~ z@Z6iR^yxo_c~>{^-v6-lO~cyz4!q|SnE5_TO>xT&^XUD3wD|Eg)>mR|F$P?3KJe|_ zz#9C)?sB0_Y-;%TDDJPz^ISi$?o8m_Yp^}g9Y;Rg)I?=T2nU=aAQCXK82B4_XQZF78fOmY0V#$$Hv zlNU?{3Y=GNm)tne8^?95dI6*40Y>`+Y&HdP1s#lj`_t z!1{qfHh{G`f&0n=-q2k~B<2R5O94QFE3=3du4;^%$jp4*grZMkP8rd(Y! z$MIym!;7o@53Y(gFq%2APfIv@qu|=9Zyd4@uDLUCUYWpqC4g&U0GCz(^YRZo(-rnT z*%9xS-rnTPoh8oNt}7PQU{|(OWQz++!UOI#3ia%JSvN1>zU&~<*v-;%&AD`A(V9(a zH5+$Kh`BgKRwocGmYUJ9idCpOTaEAYiYV6OFROMJ#4+rZLLzE@2k$u_ zFfeV{UVM#n{sFEv3LNnZJp3JE?imK`{lvO>_WF;<CtEn5NTUYSj{Cjze>glQ4!D}*p z8hIz`1}Y?FFeM&7#--WvVGrXI_mk4z&sW$+GcmXII$m8dvFW4ZxwYT zi`suluHI9oYs~8KaBNz_E~;s|O``MA$?*05b1iCbGCZ2FF28t!OyFFrt&>y^E!A*T zQtcFa7Ca@OQLdI(VClui82(Gj91|4mYTgJudc@i-UVLW0`TVDQepyD|+f&cXzE89< zYE#C_IiL0WaS?ZU@Q_0!O79N|( zlTdI>CVkqE1^nqN7P2vKJ(tzM#I_=5L9^+kCEgqJ_Sw9d8N@a-xl7`Y5ks@cDT~x2 z#d68ZWh{5^{KsbGIcFscGyAzN1}0X!7fqA>>==8eMIPul)OLN+;zMjI+Z39(qiPmD z71leTv8ajP;h`_La?9_v7Ai9$`gT5bnCjH^VnK9r=!+H2U#;FX-B@@{Ca+>4tJsCv zk9ogeVivQrxKq)5!}Q6AQl&1Ig{}+ZDiV(`jGec`X|m3$15Di78m|`JY{_VR)GXn) zsgX@$T0xprG*jdwLH*^O%txf2RH#WuPuq~#A0#vLAd{+$qpN7l5uV^tU(ms9P=h z;n1e4(D`gbQNlg8XUEt4dzdyWeL>(6MkbjE21Ztq8wy+oI~1-=j^cT$#HHLb;ZR%0 zq>4w}foZQ*c=UEKBsTE#9BiF!Yxj2xyJnSx6Q`!fRB6kAsWX(URIgQ>vwFSmH6w?7 z(~EbzrDF;j_gFr1c)vI9Z1v*n-zT*8M>M`Wt>j>kHq%#(t(Nib+(?!!P5ZQ#J=iAN zGOtx>iD*uRVw=>2juxrtCkK>7XYVsn=~TJX(R?Cp!;C}$jo&*GIHM!h9b0;2)kC&Z zy9-`2Fl(t~-kaHcc+SEGRkt_CrlreuA8us26_wC5;d0k0-t-M0H_gu7mm_I-)M=f@7NhK22yo?UTU7JE7rc#{*Uao>#(a9~`}M;6kg0 zjU$)Kiv`!S9(G4MFmi>iZn()bu{&3SDf^hG-<=uGGP@<8Tdb+_TfHK9ZbV_b^}U;p zd_@nM#P%q#?rD;#nG&%x$@HV`w}l?Ey9y?-Zd@c5e&?vp3e^Q&92s34g|51G4<{Dg zbYSayb=Y`bW2>r7bKO;qqw@bYa6B%3ELk+;pz?}~y+I`joGUEOc+dK%7viBLrF`lT z>w^zVGQS+;XxPAMT4BJz!_Xwv|4@RRC1KiTg9fJQg$*nf3Z1`7A4}^kjyD%6K3O#5 zuuRq;`Sbg>7+YSQB(>s&Bd5@YX;3UKFs}X3PD;{2`n9sZ2ajNZGWbPFo&FZDyX$#$)rAm24#!G zs>T%!Y*q{>igvK@^A((mno=dUeUIak`!C!W7F+EPxz?&T{m&%JT@!h8zT`+h{c$XO z|En2kG7lNK8X7KMF=vc-OJGp=;lye$*ydM}6tsJRtNMX}e~wW;YlW+xDsXTaG%yu7 za4(tRCO+wd=9L9(oaX`$3AHh=+98|HRWrNqew?bdyqI1`dvPOIU&XwtTg9FGvXnO7 zYiMocS;Cj_;i$e%_yxZfwS~7=F}X@qu^RL|5P3SJXilvNl- zJ0epp^)e@K{kl-<<^fl>o`SBOQx366bhsSxSjZ%LWU17|0}k><)4CXQ4sa|j;1t#g zXy7@L=n`@4Kb z{yk^kPu(!DiBtKve4ZAgomCta%e>OI{~Vp{KYeJ3!2lp{d&UrZ)~=5UBfT7;Q)i)=LQiDN2X{y*8+a_;X$#f{S`pn_*e}ReC zw|^>5bhMslwa1aqd)gf#RwefD0hZi332Up^LMB8hq-?yUQ=O0ftKj zEsSg*3T6bYc)(*Be2h)Ypn1yt!@S}L9E5`QUCmqYi0N;v!xj0AL`IE=g4-H`%tAEn zH=gsq&bzHNJox3+d9ok)52_vLlUu-~Wue5cWx*ZIc_5MBJUS=&_K~~#rJ5RBX1M2O z9uQdPz?CiFWMSs#TUe0u?#UO2ypR{t0#Q$s&;8rLeqZp=S|N7-leT>)mzhqeUh_aZ zH}v1mlD9XRpEpRygx&Zm-hQb`Y+XaEZq34*?5xV#^Bh==Pbp~VbufOY(!f{k z(<+!?(5!sqq3@CzjAF;adsS)<2nuSq&ib|C_`GBWzGo7aAE!D_H$Jd{bHM^{`8K=u z%o|It-jrxz*K=;*WKgIL@v`Q5rNC+b|9xum-8Kuw-J1@xl?3!0yRdbhvJPhj7k@)@ zxeHT@%jOqX8^tF!UosS^f5j9SA$$HMtG1M4tizNW zmz&Zax1`IxdS~N;h{gp9`=n+tN=#_ZJfNrZf??*5X8q0<*!uywdu6vAx-JMb*uVW7};72T3ddw#tX1rys==% zBbGbT8gUHG6AtX)|0lquFTy1e(DUs zQMH53Zc-!11jbz(T9rE*q%2x@IkbM;%;~wKL9(DNr-IG?N5d=&cFzVbE(cTgNemnZ z>{k?UOD?hYgObYt96U#FDl|W?!c-tW4EnlqgX_fx5rHHS*{8l*2`{9?>bMW8?=hwp3GU$C|l9UQPl8ijo9_LL?GP)|XAE}rbrNr*K;=F*uR$IviE(s%S78%7X^J2@TtR%-EuR zdQ%QJX8~imC42e_36%~;z8jNY`K;nJkeYmxDM5g(;zKLj?8*8I+Zdu)_!lrFa&K+DS0tgXdS;Cq-0jnT&orM%fm-|f>Hg{CJBWm zzYW(awp?u7bnyX~o7;DUBOa6)lb{822A& za&%zgJix$h(ImAY-1a5IiU|yY6LVrxB0>b-P#43oU!Z|?0%Rm_0?{* zXxPkc!OdQwz!S~DEYX(0!PfGqt*nCQ=RvldiWa^AvxEn+0yCKBFXCQ*OVVH>i+~CP zhe3k@16#t0NT!H}1-({mAL4?iTE#CClD1%psbE`S%XFmrR^(c@?K5vl&gzpob9K)V z$Ec5tXAe&m^VpP7Yjn^>*Q}tqJ~+D6<)V24^NSaZ0vb)e9qv0@-Oq1~4%b+yB)}~ps>T1LK1%CV$4avpb>;??W z7q~Fa-Os+u;O*31Um_V8XE5+MFkPA=TsA@eS7Dp`3MPpM{~A(KwAp7dFlV$zaI_gz zFgXjX%z2QMbApv$f`QNZUP47vy1{|}$_qk2G0$5obdQyd@AJI{U+=xW8#mE&zP933 zwi`^$Puo;D5;JnH%2nKY;1PdyhN9ftp+6)$(7mT6>4I5?HA{tr^COo%& z*ub-`@w7z*zd?hp1xv0+Pe4k0=8e|l``Th&G;$_1C~|}{ceI92VC7rf@OJLwfBs+J zM2Ea)<6vYpXmbA19($sR<=-jY9}Rmlr`(T-*JNP2-L{AGYUomd%tJG$%5pF%zBnyz zu*k6Z<%z_Xrwu1)83!^Oo-t+EG1Kkh>}5?+isAwcjk*sq9Y3=8MW`tiq)u$eD*72H zzch8(xu<7YJoqXYzvEReNe%Yo!;u}wi2 zR{b|#kl$di>J!`a4z}ou7Ap=Wfeh9Nj4Rx~JaFh>cnuj`tT?XgVT zNkc>7^snG;TotV6-WhN$U^sowHFitkuWAJj4h8{+=CBJ&Z|j;A&R>(7Ann_6ddD2M zXA%E+%O;EF9pO8uC{wnp@xzI!v4Ur&ZEJe`P2J2uK+{+t_vMpInNyk9@CNHLN=$F-&njWK=qiy(G`p2*S==U-@%GTo5BJj&DnC8 zBN~`*Y*0^NzgNo?{5&=;UqJan#?^@XS0i|(*YK4EGgZ9q(>QJ;en3>^wdkg&pH7_o zbkN4V(cz-szrJ?{`g8YiHZ2rrTKFJrBEt!G#)*17zDIrr${pP?(SYkn<*P5>oIZIm zCg|PZ`@pb&#XN3{FNz+GLK=*m3_L~Ck~S}S<{2)}pw963QIPNa`4*B2D;BNN|JatZ zW8IXCY_Sts6FOSM7#NunN?QIdQUAwa5b-T`LtEC3R>OT|v)frCIhf;DH6&DgT<)J9 zyC!M>vZ{!V*2EvBt?vw;w6?BnZw>$O`rchz-v^D$=N`MgFIOv~x43A#bj|ki$)7$5 z@IP9X@;_ARP|dqTZ$8ZoyS9)0X;$DX@e3`Y3h#OsGxl+xy|9N-a>s*nB8k@>D|!#_ zh`e`(=U<@chtz~`4QH=s{o~ug5byRyWW%ddwG46-lh_g>HoM7hS>bZvvi@HyrjC`} z=0CMm%)Hh!f6a?9$kC|MmTS|CXqHGQiJr2b$)nL~25aJn*Rea;vOlmc=Vyx)XiJn} zi`?)bQ6O}A`!DY28?@MdM=<ndRSjH&Bz#*<5cc$`iTAyf3@r{(q#~&6>ZHru!uraC7cbaMTnw&L8 zN4bQgO)_p&Y&_J~D=HIq#A9btYqx}c?1=>%lTuDiG|#=&qxg8a&rI!*6<;1CHeckI z?~#&y)Y!}|re+YaA|Sz|nNwIoV8K6wMhE5%iJQA}H!RR$WMX0ySs}<7zP0G;uF}c3 zbERWyyQ&5_h!k!i(Z~dbJ&OY{gmrVAsKDyvCmyyefNiEK6FF3NhH!?5@x_EqK5mcyQ zXkwRLv4Dw9!lodUQ8;DkR7Mt`4F`I7Z|rF3{g?cdLo9-iL&1Sj>q{fI=}n25*JCSG zIVBSAa=6_vxISlbui1GYqvPy`A%&ghlS9M0O&#T& z4tKKrB_7OR6m1ENG013Q;L#}+yZBHzF++`;!^_}0bFqWL@*~9|8Vl!^-OhdPT<+MR zevj|z&I^YexHUxro%kX@y4B_+4n6| z5%M=asnQ%ETD8F`$VEhuRV1N>k!@?dOJGZfe8GnU0(bd7Hn5llDJb_R);@g{Zr0VB z+-LSngMmHU<`IwSt&+)SV=5mWRY{mE$aco`ht$lo|4gJ12#$ zU2T?#!GQ+GS&W}0vkn``9Lu~b722q?fhBo4WBsddF1+p%iZOYcY&Np;ck4;X3QSn_I9IDh=VC62Bv*6X2l`E{atXd-ynmFGzFJ|pl$-EgGT?CA# zn7rYNKRWI;peeN36H*mYzeLA~&p#9(*Ws zU;z`O^Mc!XyI;>bnit03GBu2ui9x}UztZE8kLeG!%IonyzlaZA#_S*JF(Y-|?s%3wMq z+;Heq3$MGuuFZ?jKU~97GBxUy?5_ufS}oibJWIAXa;*G*Z>8AA#-lD@LPJ`me}*=? zimqEGXr__K?ylTr>7-v+v#X((<%NlSOHM4GYw{;2V+Pl)Wj3k;VoUx#P~coS zzxoD)XK3USrKrtP*S3^xef>p&LvKT)fZwLo{&Hv8#9YLfD~%>iv^mjayU>yC^s`2O z0Y^a4-mKUiN6ETW_{ahQS>PTZ1VUEGcJWJ zKUO?Xe2GH{d*Q6hQk&Y$?<`l+t3CYRis=AD)IzTC1x+GN3=2eCioy#g&-TywB)sHm z%D={=UY*=8)R-kaIyB4_oJ6b|PYZ22BpmisRX@{^TchIy?>U2I#!7;WtPTre^%f;Y zW*s?`9VM)jo_T+bx(3V)C z6R@dcGKbN(uHJ`mnV9;6+d3IiB(t!zQbzZILRLFS5d3Xg6$2I|$_n}t;A8X6I zZ+CmJYVA#7~r%>XBkGf^#_U_h5 zH|hT`-hA+BwuwCP%#Tq)Is1{wtVb=3ETR$&M;*>cTU)XmklS>MHCLG?OvjoH6u0QpE_a`6wpjM_wEjjzF0(?5UOlGao z4L9)=*u=G@iD#7qo4f{-M26rFflW3^CFh5_ahAU=TMrz+hy+9;z{My?CGOq!&rOQ9YUB zEfq}$9Lh--1oG#{{c~jO629Ga;{;Q*i17gf4rVr!%zm-OUm5wmqBlj@B(jUHT~WM9 z`NWNR?@!zn>=$s+njp4>p)O$Bk}DBit9nx$>ltm#i-C_qM7Tpp;Ejp8 zxun3RBGv8lT?(AE8ytBO1lH(#m`+y?iPhVcz^rU=hcj-1vHs407KbB@7ne3PGRV}1 zoi@BYQ+mD$*FT(cHQIQ2M(_NejVfbF%~{;Ux`w&mUuAIsZ+0!bF3) z^Saa=c8WXd&mq}23CHbocw$u~7IFrxXw|b;yl)ZVz_+Twv0nE$ z<9{y)CSi{2%yITCoIeVpogXx?SS7jZ(}xWSP(i=kOXLXk7Vqev{@-ss@y!Q#B&)M|zct|4dLB981_JA0Sr zL7@Og{sk@@8W_A{&bRS!-_h~#o)UEDA4emv0(XGUgjS8Cv+o9RTs@@U!)DWQSoF$7 zUZ2L(5{@DrPP#1@bZQ)h=Nx1+IjHodfqM>zPsjz|D=xfm82WA5g?t#LYg$z1ddXgJ zloL24FK|df6*L69sEBh^#7RC@1E*4 z=`92AjN?j69k`D0OY0~|?>M;QEYiNwAVQ5emSJam?&W8sPLds z`beWW8KKFlTTyl!ycFQA7 zvrg@BJ>s^(kwq@tcVYBBT}M$JM_*q?{w*wmIga8D2NfF{4}Cqg*Ym{11rIiSNSd|x zh<8dykjp`hhIw;#HX9u}tf5Cg`vL??n53%34#JQsy~n+At|vILZBBl-qDHtxri<R+6+a~Qt09L#+%$26l^uYr+cg(q{%4FL!KqZ>{#wKQ}rWb-sT*E!ei6i=h%UJrRb zciksVf(N8CRr?EWFbHk9wwYB()$RCyM#oaVv*iu$lP0$3{Y=rAq~zXndx~emnwZ`o zGu5iFMiG^RR|N#T0-ov|Q4hPc$b-YR{O=pEDf+nq+7xMpH4k@u5l4og>d2>)` z%|Xd82P7F7uYJ7TdBlPD1jD18)66yutQt3%85kI1`npOR^`0C`TiT-*bI{(V+0x_? zS40Y<1VitcS*~@hJq`@aXVi{Xu+DyPa(y5Jd%`W2E6ddW9JbGBE;eG!BsYBYz1)TrUQxeb z4x_qHMyM#`;k6q4E~y7f8}7y&V9jV?k8xnkaA4$NIQ3S1r_Zd7uU7oq84z_Ml~{^7?^qv95rY-n&CS2 zmB98E8P<+DJvt598BNM}4w-y$;1v-N$Y7dvYeA7idN!lbxf$onGEQyHRr3KE1O#!X&ui z;pB!lHjl-MB-_T1?SxV0lmlX1st)5yy#H*{IvI9R$c^DmHDp>SiCOv9Zxp57g+ zZy7akbTDvv9C#b)z&Yi>-+kK56C611I0%#oP1xEfw}DY$55otZJOQ6Wx(}R;Z#gVW z;qw0X&0)5k^qe+>tqae)H;V91Iqpy(ts!;93W DNr!m`oU0o-kOt92V4Q zV7t?(?BO6~et!QlsIH*t^%A2UR_sjXe$XsoJ5!D49Fkt4<9*{Sdx_(@t^n-~hja=U1tlDLRT@v{G)ioa;;3y} zKUIch%GJj^J{s@rUN-ekljQ1UlG(pjg-VBKPQBG(WcW-df`MO!*>uA}9#;p)?W^zP zv9g@$5HdMz^Qg(h!r8>b*~I0r=A8zP9rvc&HE?`6Xko&v=HsMh!Q`LpWD>)~-QvKZ z@PXw9gW{7zCKk$>TQ{jaSXaD3^NAMwxs;I5$4?XsPgQK0dwXibj=SvMfsJlw3X0Y` z^3E=0_~hjdMvHNQB_vc1P9a&UfP=v-Dqi4|GGCG#ZCJLyL}=FIijY;`~(TPjnzq+#Ya z-leQhXSnu-eBqhN#+Y!xc3vavoJNy5hm})gQ7;lSy0^wdMg#Ar3e2aQS$ zPAWFeMt>T3UNrFiX?XXqLCNE=(hZkWRR;u~G^#Cbuy<@=GGPe%{95!*mcR+663gW! z_6z$jY6|@c{4W|=mX%aFgWZ6;cyeo=+Z|bVo>xZm-xX%QnWpO~sUdD+rF@|BfY_UZ zvD{4xXAXv6I>0A$P-@R(A(tl0e|F9$E1KlLG#X1ZDO)g$Uf}gU) zD^BxV8Ru6zs$XF;h)^~-n%2`%CiOnNui#F^6z(SsmV9f?p6WF4m@+UqI7CS}Y-lyBf;;8D8mH?Wm zb~0u-BvSG~^ag|U3QI=q2Bsrd*#($16AnweBn97Qm|gWjIB4sQqc20lZ?okq-TrH& zv8;h(!<4{DrJB#)A}o3y62awtvT?n?trMwAswGXiZV<>TUvZ? z9N<6V7;*Ey(2C;3svU|fO}Zx-H*v)O^FH8^@;Ha%>Spa7PWnHZ^shLn7cfawF!4kw zYlpm;xu%0z#UavHDyDJH_U8iE)_rW4`~Jbc(A^ujCziMNYcQLBVdM#6GM&L>I>m9@ z?-Ng-RGLVnB?@)UNItmcM5EF_WjPiml?~Qy#@~z?nlv9YvD!P!g&a)U?oju&lJN+G z;*3Kk5eMa7uIt^p^7{vd?9SN56V8WrJ5J=})z4qVEumo0@9@9wYrwykt~^;UuWdN% zY#h94E9*5Q(VlBeoxOr*8d)bZ$-X(je&vY&mjhB7%qj+s>(m>~t(;6*nDiW+6fBr! z&SdZ&Ilyzqp`*b;@PgB(%}yMe)jYqP^cl{r{Pxd@yl1sUc~XEN0}#N**)KZ{w#UuQci|*d+ge$^67*%RL8$<}{fcXs~y6h-h$7OQ;dM(!gFG z#H&;A?%luZ4|{Hy{yNG1pzKk@lIBGmEqd+~zSgQ-k8-a+m;FZM!*QVqg?g2P#$`fx zIO5rRa!mNT{qAV(KJBRB(xf=SNj|1lrol<8W1sY$Q*Z5-D=m6<_-D|~?@am=3ez|5 z)vjQ2D0vi=azcQKpG8CC!h(Y=ZNf5s3I_@pox6n8bx!Q~SIEfND`hLBu;F52bC;N; ziid-e%hIg2Ne?c4I}@`-nwd+dWq||ZWRENJP8L4#xuxlFenaMEw-?PTqw{`Qbi4UR zYKAoD2~Kk9oxL+Nzt5xUnG<6R2QNo~LdvOIQ>EOpzHC^id7@v=w&Kpy!Y6vd3bqXl z6Pa8*6An8z{9tIZ@OUiiRCZ^_$3?8%B1$Sd5+A4aPCb`x5xy|!sd47T>}efwn`U)Q z()6C95~$?LCA!ljXUBq#v)y>^??^6LpqR|YZ@I$Dwd>3BnVUj(^ywUIeRXA$lu?q* z#K&r!!VbP&9gmKlnyUTj>BZ^kM^{~5uB0F*Q8Hn3A~VOgkIp;tC))oz-1+~%v$sR1 zn8t_3b|0gaDjWh2P6}qJ%{(j_ykbR*jE##@f~t^###71kDHFXUGj$YGmd)hYcu*vo zqtHcA?SscN5vwN#trI-w>|~vu{$j$ziRo1t42&FV6$hGqd@e5Imd?4bdgJa#r(Sm` zOxUrSU3Hy`kN07w&kGHYH&y6tIvL?{&`0pbhc~^`qCxx7jscZ=o9inUu+1H7BI z$u(_AWVO!}c*N!q-tq9fMAn5wHiU z5_S`(ZH}xA3zv!L%vpGtQGS-=5ytKcEysz*2@{leT1;8V(ig0@amh@_kO@boStU$h zY&K?SY-U&KS*6rpE4xsc&0`VcA)UVx6AtuyE<35RPA&I=BFCOjr(Wyro4~N>Q5XB; zpk#TAXBF)-D#Z(p#N0d_TTC}3B=?@b7jxJ|D@IP&RBPIdre{`($xfWA6IS?k$Whi9nH=ecAC0#OO#bQ2AxlA`fi%%~W zOTJQKJR;;?HsK*7-;@MrR`*4LhZz69`TXEm49CiaC;fi5NUVRob_&ymgIvx^yoZk% z_-yR!oW|2Ge#$42h0`W`zw!AMA9}W2Tov&sZB}kuZr9xJ@r!y?CK$9%z39f}RlqEB zMu6R1=L2tKMYGHU26juCM7E;``$RY%_ShbA(Cd=+VT~<&wq0$$X}L&p zs>M;wNe_EW4=7shpX|t`H-SlVgCc8vft#elhCPa&3;5a(997Ip*rgGvsXuv(Py$2n z45N%S-i?w{$MY046n8wRQ8A939hN3_r9;inB>CytH|h4*54dq|aAXo%@uAJ`Pa=OK z#}OqlM$Q12MIuLiShXf}c7?bs;y;==N$}eT9_N|{zTyc@yjBO6aG50vF4b6Qd`^Hp zl!b}S^#_xN#Y5J>jz`?@868>m3K)4j5?Hl&G(4Koz{sehRZ=El;Z@K0i+z0(^sHklP+ zE<7@h>{1ulbT%>?NVu5%d$OPsnTPLC)=Z%#3UzwXl2QJ~>XujEq*+fs+ga|` zs>~w5-CS_Y?8hsf*ab<)cJ9#D3!2CkwB(W4`3nUL4XqDso#2t7utc!a^+Utx~iAa2Cew*-t*<#8=4yy%5Dq0&`yzd+mGn~pOw}i2Mlg9xaz7vO3<{b=S zai7F)=kVd{qe-?t?|hOoISz#h2PeuuN$`1LJ(=Udlf)TEJ8x~=BK$8)b2h`Vv~=$s zOLKlDh*<4lBxIURGvWUY~Z{Nq>%m`=cnIF#1Phe=4 z*wM&8;ld$yzk(LwIS)4YoH&pgsMyY}(5=WSal}L^u_N=yLVooEhqW9AES(2_w%47$ zebRrQZubfqhVL`|d6XXZ%-DU9k9|Xkc}Nh0nDPN8V4Tog{V9fx}w;8PujVrR9-3c z39H0~1445)cBD^XVmn*F@?1cHk-@@&^q}@V#oueNKF5khd zDCa12XEp1IbwV}l{4)-)L>&;+OKiF3z!!ByP{xt%mxI(A)&njNR^;)D&soU!;}BaJ z!zz~zEA>{cYIsn+58Tq61(zdfHO^L|% zGjCLhdacCO6%lkmaM}X?oQ3>0Pxux&3YI-p32M-o)}X#+y`sWD-ja8Roy8bMmpFcz zbx`aB1K)=Of<6l+vmWrxQWSdEaOuWtzKm5OI*GjR6iVe11@0*b&0`ecD?GC%K`5u^ zy3Ns*TAJ^ktuvT$aaEu0YKi34Iy&quM_FSUSUsK@UD8U6>XeRMX?oA=W7UN8RZphP zVBl29=RBguw_pPIE(V?h4gwa8JQj_@R*oVE7I5}4i#|KRv1I}0q6hNd9k|z=GKF#lU1vWSVBqcOJaK$wCP zlN%F59fzY>3zOU;C9OqIv>q|2JQP}Wh+~F>)Q;EeF?mcZ4h#kd#M-X|p$e-|0Q+pw6Ndu?AL7_)Pb7`0c@cw)2^Mm_2#*s zPmj#hWnLxAyuzIMloI(DHE>5YibW{4Wi^UbEflL*DCTyMGxVV3j6<0>c)7ndSl@fV zdt(7#lft(X0jw48*`gL0RrT{^Gq6o^;Cmz>n(*LpR5sI*H7s`&BquS5awQ7AXkeCb zV&Xf};gI)tl4_(%g23zK=#tZlJjWGd6!`x*R2n^aapnMvP7{L+1GCWsk$DGsC7Axb zkZ5A$JD{j^falPr7zRhdMW2KMas@h8N={M~@=H(`abUl-fd8EX-<8(?`}`Qz9uhQI zp!4N09zmSgGQcG$YXaTan?nAXaNPjv5D}ya!VM^;n};en@NMN)T~d z&+C?*W>$7grZOP2;Q@P?4(FW%0(%s=4jeG9ZWOOu=uxTYnYmE7OM%Pdz2Ji+&U1zQ zY>5I792%xHaAi2K?g(MEU|@?_z+$j~WzQGIXddoa4x&>QN=7?v%Y&E_O9cymZH$4#L$&~7Y}jxaSFVVOYD+k6!K#f znzc|;z;p4rBrd*(PLGqE%|%u%Qc03~bI{IYwaXO4f1g~Go2SG*U_F;|%SFd@jj5ZU z@VXtn?DZR>H?qi1JmCI6L2wZRm(6GK3tz>9R6{Egg=cLMSaDF~UIN#h1ze97lpjc9 zm2+o}IpA-Qz?#;;EOFK6*-B=M16+BErvey7T-Ni4t!9xr;N#O!m~!C9j1&%=hP+En zc@_k^LqGNPI2jq4}4w;>PicA zN)&`-7V>{s!1rJQ?};$Z4+|uJF)%wnU|jMbJfG3|rBKq8i3hcg={G32h$kPCa+dNH zaSat=3w$v3mAl`*kES_pAGevhpZpm2;}J)n0_Qme0f&d&XTFM892CECQ1(uvtd(Nh zj489{C`vMHn4$6zG+AP!@NJe$#u5iMD}@HD1_ql036fixdk%?_ZC_90H|BIFsL!x>`s{ITzMllA)4~Iin zGV|YQNT0EQ{mcTM2L~i%5*;3vs23d&Jg30wqrjTbz^rqCMTdcnDa7gRdUn2rr~59l zH9a`+XkA|aRPQ;41`~2KW|^tuf)-ygC6OyWG` zd3s-iSXQFAg}B^ox|R1AX1pZ$MTzL;=ULA1JdnkD8poo#9 zw2vZpiOR(L3;FUKr8g*@l>7Hk_Q-YVBHoiZ4en zfxW+hz30Gz9tX}14My7;xY8DiWgV2AS0Uzdkkg`3ERB(CkpkbH_q-Dx@UA+*mX*M2 zb$~T#+TntQe9s&xT> zuX7M=s8qjU#I)zB<0i(Fml$PlEtD)-Af)AZ=FB3-WtM%m1$xSyvxR#O8k{t!J1}`m zm*JGRt36J##wV~GF!auzlJ-c;Y}o?#`W@`yIY*^(j@q2(c(;^mlY{7@2i#E$#iAYx zn>BLpQ<%F$p>5Fu&R+^rvkvkvQ{eu!fUhCJJgk9P<`=8Q16HN4UyZIY$1$V@CwR*~ z;P}<>m}jBT97fTYXz_asWsfMz%ySgBIUqLY;Xl9m3;E;}wY7Hf89fw`a}+K}BjT2KM6bfIbY+mkQ zw#jCl+i%0EX-6*z7G33?7rusTOM}QZ1umqcz+H+?%6-80GMv(wUai@O|1-3j8EK?LOQWSpGaNy-_ zwrvl%>=f9xIV_&Y#-g#B*(8zGA%P=6HhKQmXWPMicGjU){N#OmYz$cR^aDnlb z;~&YggfPzC`Bo1E)EY&2bSABvdELsDrG7!uuAK(oE}ax!eD5TCN{|TqvOO%DLJFfF zZ@wP6rP;zg{7m8gJqMXIxVaW`&ub8_O0>Q6M>t}knB8Qt1rGAwjN;!ISgabnQ>Ptx z{c_>!mGSacY!j@XOpHFqmB9I=;jTuw=#Pe#?@NS!?VjgwfH`m9w|@z;2g>g;&v;Um z%fK>2<(lcNO(wfM|FLjtoW2};U|;s$h3sXl)8-yld&v83fr%m`n@76GJ!3`;5@>Hoth+a(SqmWsO4?K)L%YR9DeLZvSXv~d!mX8}XB3ot( zsI9Ru(Y!uI$|mc|jf=_b{Bll}Ukq+BpX66?m}0^j#MU9P(xJv+>nr8#Ytoi^4>B5= z*}H|jP6Y)jEzP{S)oX%DY4?)j6O|n%7<5`sKQ5?d-}k~-X}Q0qoy&qVuby^IUwyQr z@?uxe6CcIsCYv1rjh1caMYUsY%y4dHpDJgVbK$`ON7i=5Ih+DJ0u-Ftl_&m_>HOH( z%==GTJE@{$BRh+ru8TuP=BI_-(|FyseN^~i!2vd5A-@iWM$SGHW)Ia<4x2dS+hvqInm0dC zYL%Y&B7srv>=L1qO7#ym^_pIOk;H5MJR-gV%(V#fPTKm#$y;}E!KNdUZFh<~cn!B4kmPl{n$Ro~a=?(4clXW>OuX6) zz8w;fu?S%J$H*^|(8%UyVZ_0&xyGP{QI5y3ge zDs(Li4_MwRXqK|^D0@l9$D1;4F&o*s{5puZC^m@)^UaRvD z79Ns4yF}q23tP3r3EdqR44wTQZwQ14xO`M7Ilc#B8k53xv@Nk>In&nU7g7S~K*T_2eucvv*-hQU%E ztptTe4%Isf%{*RHoaP(vi0Kw!W>P%R;CRn1VBwC78p%N#yJMDc@0Je{

JsdFb<@ zals`&^Bpxmcq~?YXlRr@yMe)xMaaZ}MbYwYqN%@qr0tZzN3VB04wg2)x^aR_R;kJ) zJ)KS_u7b69uJskDmWK12X}gLGMZH?OS@}u=o6YKjLa!7!(`62F#B&_ga=FN&B)}Nz zYayp{ZUehjW)kbnhYGw|4ga>v1vIi5YACW>Jy^rHr-7NDgGuyMSOZfG1GB*kW|e!} zn0R;|GRk?(W`9x8vD^?+YL>FeNpiLH z;{PS2vV4jo=S$^Fpw)0;^nw zBY(RAV};c#M$sRJD`4R+-(aab63rGZ^h|{^<>|Zu-?gD1fiVKaBQ!X}fbSy~Vf6;Wv?x%3z zHIJl6&Rx+)Vbi{_a%#WsPMvh1iAiQbL#pS+z|Nny?7J0HHR?Fy{o`s_WY46sS$Q;x zs7+*LHd5qUYT!~@mdvax(j*dL;Hve)z*q9iKL@VT8O#|!w@5o)>}Z*x_)b9Tfd+rX z7Up7(7D2lO=ehTMn=cd5XlL1RtFz4HZ1%hrRae%?P2>x27oU27JILUo$O^afCITxt z+8o>@4=j*u3}Drp_mGWUMx~DPv67qd>Hia2xoJr3AJ0t9c z*Y))cKUj9LICbi_CB^rh*U-H;GcIdMb58nfc8?v06u&jFSnNpTEZ)?|yDgwyFXte? zOPh-p+d^iZDQ+!64}5v6oK4T&V3_+m-0n7E4FaNg6PB^TvkB3>*;s9gUrHu5T zW=Xv+U(?)Y8tzx`1YYnJ*{S1@$93iNosEBg-`Bs<$WxKf<>2BJQ}7_m=f1|hRs{wo ztq+XCEPLI{B3v#znH~}p31oBAbYT3G$ZQ<6mbWk?StvW;Ab&zHW2E>3zR5d|Dkq6} zZQAuCE9TIQpHl+Z_^N$8HvHMparDu&_7?e&=w{W0nUJqn+h9b?E(n2?HPG`0dXhyVrV8a&3Cy>y30qyrEm7b!C}3OJz`Z$vV?rRyp+J`j0_+hV*jG4kBqTG3Fj(3O zq)rVt)e>VCdG0UMkjmE(V?2eWxxsRFtF6$4sP;{MS`5rG4`k&Z1aPYcs$X#y+UghG zB4t?MlvbwkX<79Ch5kpL`s+JbePGv^$bLOon74qz=qkhgr}F<87%~MHFg7M| zyg9(DP{g~-fcM82_FMtRW8uQW3pm}s^XM(&4gShktHAv7c#Tj1!-sP2i3W{=VqS`m zlPsLYk;U*_$qX6Hkjcmme zxMT}hydSW}ZQz`vz_s}RtG|Lw@kN$I4-V@BryzzXvj>vvB$$7xvFcec8_zIXQJBiA zz$o3yX!?PpKS3&fOLg&OX)OWfS*hI#Y0L%}7*$2&62hEbX2>mF=>L9&--CzJJ0|vV zZH-(R7<*lnIi!FqqQGXy2lgmd#uL+PSOWz%Ue>*;tWdLndxrqmDA1Wy;~5z+Cr$tza{gRh;I2e7yl zu;fkPs#jpvPte$-$o%@sj6I2xCQJ*}mq>|+GB70sW-VqibKq!gm?k}4DkOwCxIxR! zfyG;3#@0*br!TTpKVURFU@LFH8Tx^-+Q72TfZccjYuyJ%(+1Y1fvmrdRB);@r(Tl` zTAXh4fFnk31tZoNQn6QkY;DgB|hS|joTq`HA8aJ?4Y-o=WWil&d z{bzDv>dXn;=NIt4e86Flz?r*YR>BpA5ChI_4qW{+N}UuVL_agzFsWSk6b)$<{r{Y~ zNL0+cfT^sN=}^nO@Zj8+CxxvSsJ}bO{NbhW%w5u^4J@GnT)qrb7eCx*Yyf6P$ufMZ1i?~)7bOAL5#PT=ynz+RicXcB0v&A@JJz&qKY^!x*!%O93a zYhYI`_%}Polr?lZ*UVSKb_(SWA2X*P74bIAv1yttw^($einoH;yn`*9m|EuD6P~-! zMZ~7C$jE`YUzOR)p>j@ed{F>PoCEh_E`ED%{zV%&w0MmN^ThWH&6BanjLo0(%Yvdnp4)Ljse{hN1}*n4HCzO8#7TvuGCk zLxx-fj`{?yW(W4V2OJXuxQZIsyBM+r1eg>V7R+#kUGs=FU8W(HTTv{!#q}_!mFa@7noQylAiC}eO7v|{_NcoI;Mmi zjx>#CURvDUCcxpRFx8fu|J{P(_yeq)9k~4+Sn?RSmL{;yYs%Mia0ikiVy7nqc{t>7_}rB9TRrX{N;G_m~hplZ1Kf& zGfh`G97F@NdzN75py~coh zjRIHP2i8ZRV`5om8L*m7W#zdgy)!&5m2F#a@!mge=_?(Xg&i397^+uoWUpi32y0-k zY+#=$5LMkRZSX)?^25Qqrx?FjvT6sgdu(8eJj__>z%Fxutx|v`e*uT)0p*20w(bQ=86jj1qB+)M;L<^Fd1Dq$-jXu=K!OZ18eCiHsJ?M zUI9$|&4u+YF!4TMl2Bk+k-CukIzygqEPq34-!lfL2@F~XmKX)tavBuhWODDnEuVUe zfk}YTl7ZP$fm!!~rS1Y|wF69Q3<`_0Z~CucNqfNNvw)H70CUBIvl0jDj0G9BFDRW) zU|kf-Eb%&NZ<@%#(;_|wqV|Wpa{sm3-F=xDXLw8}VC99?>Te_y)wdseC%M@u+A2?n z+3>@$W3kMh6IfdVSo{^ZwjSV`pKy0xgW8r4tkDAOZ40=n4O|TuB((&k&NvzAQ;;9Z zW~JSbD)!XhAb@>Z0SoH`hNejxZ?`y$2}G$~U}Q?@&MWy?E&}nkV_7V&&~wtw+l4dE;WA0pt^wZ zhbZI1w3Gf^3dYClUN2{8n7}i20#_jebE5*UpknqLbw=HQ((?-Jb66X#1sZKGWZQpW zjuE>3d~q0uuITUd$q93q6D~4sEn)JwIq%;bQLnd}~gaSRp6cqX7S>?K4DvK!10AwT13H|cLP&M zfK@~nyV?T{mJ7Pv=`~#I9xS$dWw@1r{ehK>{+A-#jJ^pBoB^-c6&@7xa$IC!iBT2c zE|{fmclJ!X0N(-z`wKi)Pk9!Uv0a+LV_*<`DerNY0I#|L`zZkqA1U4|57>+i8M&u1 z^*eBF4`8(`EV5eg#AYG0?>Xg$Q(kO;nNuhHus0Cx{QoZLz`Hm1Iz5_Hl6&4=uil%; z_e?7GnbGQt>Fb{5&SSpFAy=lv8fw5%x05wifvd#tdDu%GceV3bfuGk(CfB7ked#i-Q}HG%>-@2=yRSTK|6LEX~48omut5!W7a zZ}_`X@5|zKuQuOf;GM`5`&3t0pm>QMW6plI<8O4+cQBkV7xZ)LMgMbc+OeXhmX9NYI&E`s+B7j&05te znKmJ@O(KWkAe;0O373h={|+pj;Oth()S{mA!=c;wR|kt|jKV=BHvW(u*&Fyi{Nu{m zAiC&{H>)Jm=0J!p1A z18dXc;`0xZHv0x#5>Z&!n-QY%mf=~B@(aGK4;2bcoI(u`8lKNe{<f%RR-v!0+SiAjPnivy1|3b`~muyFDzKAmROvno+#QkFsI z0r{vZ%|mn4ax~R4q}K!-nrm<+tY_hjM+;qqf_cpENTg^?cNI&%aEeWfp);k|US;Lt zzL+f=4mR^Ixv;3;ORjN=NXVlViH7RI2fCS?f3BLep*8$PBd^1U6$`t9m0mJ;S`-{) zZ9DRh@2~?CyIjO!AvvRl#QyZKf(C}0r>uSbpGDleY&I=&-dVHz?{Xa4GJ+Tmq&=0Y zIKasLpyL@IJA(u-6Wgq|b3xt|f6U8TUO!Px3oz{DD{fK`4Azgykc2FE*0;w1rM zy#I=YQhzFNh-RI9$Yj2dEjsvVl0ZBsY$B#(t zNMJRJILNic!c}t1f+BeXanVNtovJL4#ELtbwHykY#EvXr_MCALVSoH!LS(PV%M)w%lWEvPbJ_NG5 zZ*WnN`PlAYa(L1|28FXMYl6L|33geo+H4|GBKyzVi8c7!x0i7n5u9>wYh=ue#e0zVGc$XwFla)3JgqFbr`SKJk|CVdL;I5 zHlxS4Wz6y%FY@gJoHm#Ja8})8JdeLAOrW#EzvfnBXVA%mLf?0uSIv5-`LwB#dr^d* zPT0k+xyIMIu54wqoN!LS)2dm^E3wI@?f~1_6D=YffnBx&PTb)u8hN!E8ch=$m_;=j zxOxIu`9&Q1WmX(ko>0)pQ{uqvsgbOr^peLi<`R>jLuO{e0}d6Phhpa%m>Fd@Fj&o9 z9%#J!X-IDHRkx@5jBEZirCvxlY!oDPoQZXV1FP18M&1|&b0g&ijO;t+BxhY#;9QWs zIx2NVSfDP;yhXQ`MQm^o%qV8z`Vi{qqQKzLW6bA!^I-k2ZRT<&-kXdP)BQ9;*woIX zI5{0rmw8jreqQYnU-^WtCrrQ2*XtE_tzN1mYPvk2RV#=!ROhMm(RY}* zO+PTER34hF{Ne$-d%!&3C07qC*k!bTG&wAJ=0e-~zOWPJ4lGJDUYx(o^yLKm)j5vw zx1VTniLi0?GA1S@+;&uW$n2@%q8Dq`>&pg2NTWj01oKsc` z&o-<-7Up~S->Y!m1RRdhnm1))AHr*%zDaVw(8*-C5-xfV8RR*u>=8* z^d*OwUWb~neeq_Pah>PVGL}f)yT?yOt#dQG(0RZi=;fR%0S6cq6qw{)4(99SJmQY= z_#iIC^NgRTA#~F}-j1WkWL7=A6?|xATB0e(8vzf6H`kX0%c?YTJ4rmTw2f~H~S#3^S5nEfBZOI10HQz{&o)cKdb9Z;ybEOZCLv#M>hYdn8BKl z)2t5(z7Sv+s9C@fAi%1dv5-6J%pq=921cP92An}6Ne82D9Gac0z}&lRk-+r(&6))Z zdp+kQ%N4IE+MKES>9k4$Y9bFV2yk5uQi@y?}g7zQgc{> z4O*QaH0#}9*8RZ{8>G$uq1kptlg3Q4oTEgKqQd6$)Fu1XRf$fDblR*Pl2UESZy?Q~@mu{xz#a8MT zrmWdbya_9LvP76)*~dO!z4rC$4a$Zb0@j=YW<3Io@iQAZDw>(+ux#~KpO?@iS#2gW zYxmB>I=h6M%_J6?F|5tYZaMJ#fY%C^}1>R^` z;=TWCwyA_gqngJC--agr2h1$hTbI5xVSOdC?BWT&Uud5l=WJQgBeb!b|i7qi_SG%7PubKbj&v+{zo+D=OL}8=4{|TJSzh+$~uzret2(a7ttTI~Z<;D@HV z$IJpZ8WNlO?6!f4|Ec(E-lHFxpo-(sN z4SyKE%yCs(%#xPDmMOtrq|ja{(Vj7(?Ml%i6{YP*7B4F4-cuIbc!Ik}q~g@_Cvy%o zFe;yD(lcn%`(YE$;9Y;CRimKE{{pMu1Lo8Tdz}neoGn_cc1%}X!NAwS^Wss1R0reE zHw|(>n&mby@Eu^_wP?yLXj5%qv~*z2I?$@Jfq{3&lCF+@b7uE?HT(Qpq(5z;zH{V) z@Qwd`7I-$iifFL1Y1m|RVo9)lY(l%kh6bSyej$gmyb`8?5mw8RS4L$?HE-D8n{uT0 z$u8fO%W_WzCBF9c=Gm=T!Qv=Afi=ZJQrUI(i8Zr+D9$=#rgZ6m^$BL51h%3M_UaXv zt5*b9d$bn?E7`pbR-LtIiH=vvWTj1{pIYvJ!+~J2cjz! zCuKOKYENKl6yTV6a%RlY>P^k(n}Rbv*sDIU*S=t{tB9z3(O!3hQGCITV>d4=KMq#Y zTG1rrC~OkaTGd#V%=k5=QR>1e4GGpLi|)_N%ziIgd@rT6dyIwl)F2xYO@STPd|x#2Ch++NHVAlx--)@d zdH2$UH@nKGUcbj8CRf1{v*TZ@YQce$N7n<-tYMnDO(Ck;D5Lp>$|7q6W|svlnGrYI zQt#G!B&ptGlv;6Dt&ZzfYu8QNO&`z1YR~f0oaI#(y{KX}qo#qA!b_X*gx0VREMXT8 z#(rqB?O-V{VS6IgGIwi}Rs^%I4y$`Zv!(@;=7eUQ2`uRbtlj}_X$P2P9a_KgwpiQ< zNuR;yaiL+e)Zwo|`U{-`7anTh>1g2bXyo=_Y-=UjTMa20@$lBu&ab6 zmFFcbTXFYBFW1efTsKwy?`~BRe{pwlj=5yVLE95eHIJFjpFU`O%`r-%+pUN#J%BZI z2D9SNhJO*v?i(JfHV7(T(6Bndnm&U?YXYuOkK$+ z72ISRz+~RgJc}>NVMnv$k7oOd7LNiJj};jn4K2Y6uXIgPQU#u*MPxaqWwcFed+*b# zX5pb%z*JVi9y_6#_e6u;p$4HJuY?R5c?DV`9ZvizX?XQPBvnt{X_SBMWWXtkZt zr2V4H@&J>A0_zLTm5vrIPA8bsH?VrFXyiG;AbO&~F5`u3!~LusSC8LtdbzDd*M@aD z8-Hd+Tl$H2886;tEMWU^Aa8lutCWaW9vZC+{McLzT0BlLyPs&be$i~q(V|q4@a1xA zV8CN8j~*U|*FqP<#C9;~FJRSpV6MF3I7=6U&AWQMzF5Ttv8t!u z^n@;|yQRMP$?~e(%~CU%>>^r26V65ow1x_>+TEC_T*$UVq1Dr(MdBN?!vjr^4i>{% zY_S*GqAxJT1RPgh!6@;9VgHqopciYuzrEnA%NkTr?6IOnx1`1EN9HS5^U2GY%_lU^ zbYn_wY`pI6C>+4Z!@|=0Tz^eD7MbE)!K{JC;gaZF{z(%3g%aYOlJoDL*%mANPrONHL8J77#=2MPCns`g zS8?giVpKiAY;Vz|Wzd=#&>9%s8WX|#xH$P!DO>N%W1bJ%GA&wkW-wY#_++cVnscEo zX2n~l9jw7Cn$0^{EPpVYA7BoOUs#-gZ}>b)!RZ7c06tNeZZn|B46W0lb(UL-LEXS?ePL9SOfjH@>(3u z4SMD`@x$@EzS^~~Iom(ntNE3^>6wU!qUQrnZpC1p!UN}cBN}yY2=8b1iD0jBcyv6k zY#ImC7XGFQ0-PtylGK;IIXN%3VqbdiTrTk)5o#68Rt8McGnze5wEk0@!K9PXq}I@( z8qs1`aR2#FWBUzleQg;!99dQhEf4iroNh2V&0x7QyCwKTqen!P$Bafh2iDY%mZ_H; zzD@G}%{%ko+gpG3PM&UA>20jKHCObhKqaF^KMTh~n<>$H608~zS|SBl1a9Q7=gYpL z!xV2}OyJh0 z=K<5SDyC>?Ok}^n!z+15=VU;Ga~Fq%fyLAh2~Eu$VTuW7Z!tKvtc_5%Xt=N-tC>?s z%qQZ)g2t=+SK3sV|j$-g0kOyzQU*$7WZX-J1;JL+|H+Fb>z-E-DNzJ@1`A@ z!6|pAPx4Xu8O%k-&7 z53tx99cW?^Uh=?+#eGX+8)va&zySv7nuDDh0S<=U>bw3t>efiGadZ)Mtx-Isx47V; ztGHo^Lqm9EYr}z<{6muJljlzjyPmGpVZb!6>XYRmmX-V zW=J054UZ`-yL0e_(N_cUxy#NyUFa@b6>ywQ)HWl%S-`PjyEE^TrBU-%-@B;byk_IQ z4bCneYaV(8`6Nv25;MMV(OtezgXx6Q(iKhJ#^+pwI(fw_9yszFela+qP*jrO$Zlx! z$WgA}TvWiMzg zotcq$?&-3O!mbT7)0KQaEK6Vc)Y2(?l0(w5oUJFft&p()a;#->znj$^$=n+gIn<`7 z?aC}EfAaC=^2>}1JIn4b^qN<`X42eM6;DrU9WK{nxyUTqtdr=(?LB4FaR$-q3kRhu zz7!mhoNCd`r8!aMAtSr2LSi#-vyT%yk5q=jVP%;RLrxhPivtJQv@YCrXOpaPNMd}m zPkKG;VU+r9tn(+C8Lhu+25e;Hxc!sw=Ap|6*!tRSp51g@o42*i zam_;(wHb~wyBL_11k#wz6j)V$6|70Oa28!U!tgt(OPj7I z9_5>Ufl+2fViVJg2O^ahyqN#VCCt#1;IEwh!$sZ3&?qY6rEtK5rmuY)7+E7)wPI2j zI1|M7eW+wJF4gqSa$7KK-oiAd$_*{XB960f$taqqUg%w{AgcLziz9!~Z+ZJ!$_E(J z7Bxh!K3uYCv8+_f*~U#N4|I+P3v7Ct!0s4vSpMDw#v=`#+;bkVNGxJ#Qet5eJr~es zyehCOi02W%pwtoNR}LIAV?GPEOB|7|aNxLp$B8>y!AY29W4qpogJSn~w6e_j!N4tX zwB%zG*XazF4OicGl*N|Z41aKe=ca`-o7e`|;?4Z^OfdvnMzU z|69SpEs(>M%E_pBP<;8!yJ!3!3D5ua?Vk#hNRI+j#RL}RHwgGyRtfZa<$7=h4$x!2{k@v&44bz-B3l2I|98+byG1YakeUvwg zP=YAKfgLPUHN;H40+_QboS1I^77u%tDxrJeT2ZGPpEX(8dBp({fcFe`RAu;@?F@qY7xRrk&Au7650j3OE=A9Q~?aBODt z7TWG`RQX(2n6t$rk@|*4u>!+Ry*CHB7hG`F`;*Y=S)p)5!obmUnk2jT4JWB-8IPE! z6mTl6Im(z5xs4-3hOvjwXYsEjCaon4I~WfvkSqxJ7M8a~lrktF z1ut_owFeRAN; zmuTT##K0E&qe=F|jh|Wz6uM##eB-~o!0kamB6rB0Ln7OD9M$zZSQ7KkK~{W40Gk$% zVYBjwME)p;bcJsU?9v_wxwt1e?C~jR30l&`+4!P4UH?Iu+>9h%oyDg%U0-yOZAv3= zQl$4L#RUwY9bH@c0iFfj70yd8P*00WaDZ|Isuoaqd%3So-vyg3K%ay1A}kT}5L zUe$kM_t{S|MPkeAxxbqv%2`NjUF2~+;dXZSLixW|nldIY{Db~_^Bvo|`>r=bz0iyq zEGLX7cKSy$atQgXI(0r-?yJdug-xIO%JYu4IZHg`Ty%j|{|+O6`57jjfQBs!yC!gj z_%Mq$b{tmlXzU7_!;~?pL*Rl!AeYAyCDHDHHB|4@IH+DZ7!k4H?$lOh8wZvxjmkaTF&AWerd(s_@Sb&ZZbXVJ3&SMV zIgT<%8jW=hi3dzETH(a1ae$$LHA6Y&v)&S_Cs3g(MEv zxg6q2VcL_nG)osWu!=AUMKH-7;E3sA4evSn zkLkcsNed^=DXjO`9%W`Zz#J0EW#J@k!obGi(6grV{?vv^a~M<@4oS)|@@QPnIe5A6 z$c*QP3|tEuQulBc?)4PlTF8;{Sh@3j0qY&XsASG92ZhvR{@ys|^VdOg%VXcC_l^Xf zl2tiw^}>1Ck9k)Om@nTwq*^fN17oP15BsV$GG8QHcsN?PH{5ZU^U!fhIRA!&3LXa+ zUu;q@X_9MoJpSUk&WTx@Y!3gHbW(9}61w8RTXTSCjRWtE143^c6;v37SeR5a9QUL+ zu%{eQNIJ+`!@w@kz_Q1cWl96vkwzn#Mz)YfEgL7E8GT$&4sO$wj{Go}MSvm3z)>!t zk4^3N!>#`oZQ6Q}??Y^gMZ*Nw1`Z7drWuX_9KAamoJ=(i^B5ewufZhXz`*>Z^Z8ZJ zb60ME&zO16La1P+oX{dSem4dI2PdN`PK%Gk72jq1cVOlc3HGC%LSkm_c2T^S-zMAh z%@3&M5@nHcHd^Am%-u=qiKB*!^eqSeGsmTB97KAUv?~s&A5hc&;H017DL%o;G{l+f zucOui+23CqMfW(0`W%$wX^Ma9)GW%xJ+pyzh69_*LB$wGp@)v59uAB?44^^0Ka5&` zf`kN`WX>>j9_Z##V2n{Y5WKB#mPG?YLIYP1H=_;%y9ooM%Q1s#t}J^TxHho%aD*{0 zX$V=;z+lqA{cj7yo-+rOG?a$gvKg*vM&3I1ox zq&LG!RziX2ilg3-siton#blUNe4I5y4)RDG6gvDtov+KMz1 z+8Df8(>SL%2)HyEsmz&`duidG5MNJ4n{zh{dxRKrA1*UFY-h%NxiIF0q`Y*@LG6Vx zRW)f<950rCDaj zT~-HCv#gD?)s;r$J^2!ayY=@5N0284sdkE;4^<&|Pv^ z<;Owm8BS^y&8h-SRuN62Gnn-l-s%Q4nnf`4-f1$9XwW+hZ4!laeal7}2^!E#X z=R|s5KD@SH^mV~k9?mHZY|{_$Ivo0)c}>kU{L&WXl7oxy2XQGSG?`prw#%7beCN=^ zW%-Y~6tyQRR=CJN>00J|iYt>ZbIl0`{tb=UI*hVD&RPw68Xbq!3Yg?Y4$66)kPl(z zT6yUA)F%0m#>T5Hf>#(suQWyr9+LBM5q_I2GiX^r2pfb;UMGEq`ae%v&EHDbvx3d~T zb8d9LOpT0xbW*uiGGX2U&X7adN8E~^=H|~hc*$g$&Ai4ricG2w&Ner0ZgX?D|F-y% z*1EK7jiM>+RW|XT4PBgq-dXIv?@BWryS^G-*jNY0hyJ{^BU4<0yN?QNiM{ zi4C*L1SZWnOnN*{Cu%RtTQqHrIQ@LtA-Nj|L?jN$_%w=?G+IbD@?K%t-En$fLc^k} z2JQey-hu-y1r2IP4sfwFa#t|$-Z-FknSot|#e4l1?llbT1qZk*7<~90eajixS1<^l zEZM!|bTCgN+p-g^Gg^5x7T@2Pv<>3+qis^e^RquDONxWuQ)KJL&5%chTv zSJT>j9fepjSFLu~pq*&V!ux?iU_+y-#6k9iMzIE_3yw}2XO0M+aFl<*sA(dsrQ;-O zefZ+tL&j5^G;M_Qr!i^H&{ve1_(&#DNaY|q#{>2h2d;<)7M>5mOHMTLIQs17?&dkj z%5$*5|3HB~L&_EJB4;*s4f7t8h7f^-h;1jL?|7G9H)OLZ=Spbce&Ha^;KZrXs62(q zcwUo9NF>vZO=%Cp4(>Vm+;{7o3jcj~d%pK9el?YWYmS3TkG|Z$3;vwPMHXvpRyW;z zKj@CJbFZKRWH6BOeEY4h?$|D5|` znU(W_v&VsRhnw+(gPbN_*5}#|U6niH>Sg@8UpB+W_D-|?mu7#)tLB}GC;R7|oayzn z%qo%p309`3Z)2&mb?%z1FS z_k;63ZL?_gRCNsE}X97 zLEBy&5_8hic))ar>qx`)dqD>tG(BLdP1yNB%HiCOCjzw(%sCA*_?Vw@n0~Mlt>bQD zYCdIr-iE_URFRQ&Mut9L>c3ep7E7s>c(4g@i<~%r`JaM@<^cgWfrm?l{0tVd3a9K? z*edVy!g*5KuMaEL7qn36IWY3iwNX5TUTMoA=NVD zbd30wy@7YPzKTj*Tl%_aq29if8;x8RJ|7P-EA4xcA{NE6%+*uf%tL8H$STFuHdP(t zCub~j7%nqwzqsMdXL+aNNxxLcPc;#9)|0B!we40UJ#zecMR+@pMMX%^Lq64WjgPYQ z@^;R${1LpJqv>;`C8x&KzGf+-%4NJv5;GR;VKvKTXk-!UaA;IsGE-3?)z4zl95p`; zSHVAGOegKzyndZn!jZSbS;ulNov z4@#d`?Qv(??b;g^cen~yaySVqoQqA2Ee&5`_<8%aQuAW@@CV&&s@E)@HY?2$XcddA zdDto*YZWNe!aJ+s$jlIvKITawFMc%hS>3BR+-v@=<8Z(IKZT|I65kG}PSIl#bTQGj z+mY19WU<4+l1JyjStj?yzK|WW#bdubb27X!^ZfL+-22lOx*kwq;g*@8)E`}C!O{3h zErEe?gB1gd351eP0`&c=%t_JmkCODdGG zSt?v?_gr#_|HP%}`*i_rV%s)ocuLA_wG&aQ{`bRKJ~H!8*o_2MEry5fjsi;0V;J<6 zgbHKJU&g#}bJv&rlEAq%gGq3gL09aK!{Q$r+VyuF;PkU-Ln1m*5XwfWrz<9qpM86z%01Ofjd6sqWrrVu7U>)xb!6wId`mJ(+hdX6SF``sAxsA%ru5}M~fs* z_l|>tb_UJ-B@dWPXB-ktV+gdAVB|LaaX|3J0%pmC15*y|>@D7)Z1U8HeZ{4Rnf}TO zCX17bESq;7IA@X2B+@X+!C-cX&o>ul={tu6MJ)W81Qsx?c*@SGWZ}Rh^MKh>?;s-= zLyJaj0vpGJ^rl}2`&lMPPM*=DB_w&{0E5{AM(#NaWll|K)|~T^LoUU5_MIsg0!tD^ zj=P=a3u2tu@u%@zNl`bKtw0m+tw;V+3GY~)K0J^9z`6dcsNla&*O1)I-#?qA7#R6# zGMpSU7aol&QQ`?`I5cUNK*z}^dZNn%jw&1|Y2-V?Akb`ah)rihBU8=;remRuvZpRE z%E}z%&HLdjWVC_B-C_}&MJ>b6qQ@F{mdzyk=j2M& z4f$Mp7i7GC@|d(399Lx!*czb8aB#~R{zz#D2KED&7zHyJ%0v`QBKC5HOV(+xo3^q! zXo?cw3WrvsRf_CvR>}O!D%e!|4w#GUZ`!w%P?;q5Z;^(Ru*xYl zdyB(j6C#*o*C@1VpGo8o@?cWnnAqZ=JC&{Eg(JJp0!H4H2mC7{oOw6>W|xR~z!bDr zrK&RP%GY(~lkB4TYHHIOZ~q82|FuWcxpb zqlzAdeVGc49x?@t3ey%eNZJUqop{^KWpts@PUj&rpT#+5sRuHF*1rOm>!_+GAM?E9 zap2PKlMS40eet4O7O+m!I#|8YSwnQq1K!Gl1N^tbI=!ASGV@9#I3?(22RJscs%A8@ zcui;$+HinXDZ`N~;6Rheo(FpbEe>lcU2Mrzm@O6h(?6_qm(d=Kh;p|bf zY$u16AA^JfIWuM`%idV^rtkMg{$p#4*;nZ%?Y`Zc&-Y;VJ+FP=Cw{c8x^Q_mv-`0* zb38XTDpe$kuv|CNj`-LeUvP-c?Lw35OMzx73njkX3^z&NiQGX?9*MQgXfvD8a9LzU z0&jW2KYw|z##Wva3%|RsI4Cbx*f!fLf#ZPSt%6y~c6T{MC$$Rk)fo3O`eYwqS>SW< zg5?4x(E|@&=nLO6kau9V@Hi|c=fEJNz$oXyD5GE_)Wj(Dpk6`YS{mn@mW=VT=Rgh?2)%~POWu0=d1l=A>Wb%Y`;1`|2g?D?!n6!YnlD@@4x7s=aA28=g;Nr z&(`yR!z+>5u7P7xi&(@#QNDFz8I8hxiri`sC36x5vKlyL9`ejk5cOgd@;E5q(I}#} zkXwyWbix6%zJGqKX$s5<3Yd zm(-tEB>Pt}PkEP}{zm`hyVVZov@LX8*@6@pR1z4Lsxj`+yCxHG@Lt#Ih6C*P{NBVp zcw;uG9EO~09`N@lFc&nf6?ON$`Yw?OqlZI$2SL|c@HIZ7&((1__7wTi75*FYT##kD8SJuuug$<*#fo@ z2j-9j&TX$bIv5shNnrkG#=tg#fvrMe_X7v^EH>sghrbPZm#i)Z7EMx9f156J^3NTA zW-o?+MhwjFR$qQ|{@=fp|21i~WeI$r9x%$~*vPG6Jk+cxc7ailiSdXVf6an7aSCsO z8d&eFV{vN`S3SUb>w%D7ql1n^p3#BzIt#ec9x&x3N-;5(dnM*?$z)smmhJzhjV#YK z-Y5uaC`uk%$Xd7^6zn~MS}G{;k2Slm(HyBZ}MZcQeb{|-dy0z zA2o${W`cjz9Ab|>R`@qr@t3-NLNBB8g8FSiEO8IsMm@N9PVY@#g9Aqhld1yChQ<71 zih8;UoYNARy%eRC8ATNuInxquG4V_DzGts$;+dt&&U;*9Nu$goMLChpGO;2?QxkL_ z^~%}vyO*1o7#kbU=Qmy{>i)Bky+MJW!IACNfkhz;1-3YdhAcGr%YP?fk@+!+(xx zTmAWBEv;#FZv9^ahABF_%y05*gBCFJh5D-=VEm`lW2NQEw8}s@rXgtNGaD6+hNNCb zey1F%1#eE}F?%1l+;@)U*rk<449o&N*0+wa=rCNovw&Z&LrUpjn73nS(Slp+HtKWw zx6W9gA(d@#^Vmm$V%L0+!Wp7QD;Lan33Q*IpYBl{_3`c7HDBLeD|2rY@hER&JN1B9 z>mc)l1p>w9eCxlz~PsJ z;EV;hW0alODDuaF>)QdTBaJCvzDfRS;QXV&*0X@KM}aHMc!JFW=Bxzvgaxc8yv(Hz zFe{j;>la$99ANf%S!yVltop6=c>10nU;c2UPEtA(sJ`-9TLW8D1B1c?1~G%IBwcGK z*CrVO#-uPd0Y@gg+;m34w}<>#y&9@!FXYL4$;YsX|CxoTR+3=YzZ}6$Eu0As?0F1X zv1{1=UQ*8sFeu*;A!z(*!_&4u59K}_l$Y5Vxjw-CyjX+w&w5S?ImQb2-~ zgVu#dOlCb`mUzk`!pIkuP*Qw-a?z(in*)r0uK3SgyuU~k>6c{rFnOF|6+;ZUC(9Ce~heUE9Qv-wJ#ftRE1651SIZiF$)p*0BrO5YmLFnBC zwjf0*hl7Gs7D}%A7`|$)b85hcA8HZaA({e>c|jtP;Zr7_{;2hi(XCgf@t^L~9L<}P zZcl4){JMm-!o+dwDZ`n>4-q&16Ia4coaYT%uv$j{`!m=MSyl)zx|V7JkNv`@_8+b%KA zXJ(%9VdcE*m6Q1#8yS}SF<8H2$l+Pc|4)HWX|a^pLJ_AIJZTE-e1p9eNQj?(U`c{QUl5~LxubZAV?r9^?)9@*bVdp6Y7odt5Y$o>xTX*`JAq52NFa@Y zIe~#+t%Wm*fq9<9thtX%&RSiWX0ejvsqrKU<}~r;N^eSQo&Oi5O`dfwFl+()jRrQU z1B^+UOe_bO=5_P2Ed-quZQ{Vd_aO9=GLz7OH>b|YPR?UKd@isoAzkDElh1?s;tPF> zUml*~D9OaQV(vj+lL@z&&O2uc9GkRMBTQOw5~I{LMMKr~i_70k{OicSqG2A-j!E;I znFA6`md2c_mtM+vV|#1#WNXVmF%H7d7&w3RaB!UBc1aXvixHJ5FY3THhQs=en zBPZ{3ucIc861(KW;ZlH#C;qIV~HVn^2+mfO$eo0$-44#{#CD(=Ert69&zbC`HQa7iOq&o{o57)iIDBVik)yck!ke#o=p?Tyv@!b4xM zHJ56YQ32h1z@MVJ^xoZJL{EcjsV_)$`xCoWiEmI5#HLT;x<5st*Ce;9;B z1TCn(z%AVUs$GBMewmf~_ZW9Fv##3r{BbUymCD)_^N)Kx%J#%C@P{y5 zVRc|gd(gu7(3Gc4>eYdN{5K9vvtnXYbKsfB#r)xB)%6)6=_af#4xBF*u*S6hGW)_Y zD`851g5WAe(N7VQc^kGYytc4fY0+lS&=U*t|1H~i=72$escZk7Pem*4e0gw7`{wPt z^WNU%Q)pWtF=4PJcNnc&#j#UY_$Qz5W* z70c>CS&zL9idxrK#~Q{kXfJ$oWrJ48k`Ak{jm;|s<&N~8YII=c6I=0Mq2l3yRnD=S zWRy&E`BiM06g2D>M7VH?s4BUq1^#Pl?G&^K+H)ar8w(3}nwRdS1jb|R++ycKg12%^ z-E3@?^!!Al@-{x1B)>lw6y1(X$okCac&U4=N7D7wla;f}&(GgqbK&>X*Y0zx>}(z5 zGSoDhn7c&!WaV{UR4j0M%eIxj@}Pm@!4APtmW7+zIi>{ex3Rkam8EOBzx06%Y+j8g zj(s5}66@lq7!JeUEA>mtZ zOl6TvIMB!*a`i=0cah}+Q8u<#(d*IqM@1c(nbt70sGePMfRV@Oj03x3b%;c&mbBfY z28JF-gM*Ap8w{K{)k4CYxKw*CIL(;4bHl%BOsXyiT#UNU2y|qhk~!#Ny5h&zTbx_Y zs?E;bdsA%%-|nf2GKQB{OzyMoaWcPK^7q4Gf4e_#n(uPIkWHD#!NMCT=g3yOd_o$V zVM>;>z`^B*_#9F`9TDI1=MlHfyB?;a>iRlb-JUkGB*%U@5RrRzh3GE* zPba1^J1vrEWHEYUz@aE4puoW9t}?Mj&@E%3tB_gE1~z`RifRWY0R@AD9GX22Gq)#C z%5>t+UeW5vESMs2kaLmY!c!*Cw-g@{Jf|Vtp>^)gzk@D(+s`ihJb!=Z#@Xi`x)WCz zUevk1V(|nwt(8lr?=!e?S*GOq43_(q-)A^15Sx)OaZD3l0SMeG&M4-r8LaqG*%HlA$cw&xYyWNrZHV%rNv?o z$yA={%!L;J)n1%$nf9^yLYIVd-^KHmQkqM>!&k8j%#eCA!$IiZhbxY|nztU$$qk8l zJgZ7dsj=-uhFwEOld#Lh&cHvem@|DU~c>?+aj+XEI#FGV=m=g>Z{T+Q(Kd z2?y5t7l#E+7%ccJ4sc9N2vb$K*lz#f0F!uw*S-iBmrXuTnOHlnN9+kwULUhmH{l_} z-ZM?>*&P^MmAxGWZWM4y2^?T*&}im*a-oq);$We8#39Xp0bSM_3R=7!$|sc`FtBPI zVALyc4A(JeR*hM}?75&>+GhcS#FR%>r#R*t%_v|>U63TWw7^aCTmifBor8SGG|v^7 z?L2v-=V5y44rbLQj(rtp4$Gab6e<0*-NZJ4F>ew1y@Xw>bv26-O9yxNE1aEu z5;q)WWiB>0YC5s^5r_vKh=c$g}D~ zyW@gIlI|NAPBA4eS3aRAzg%h(lazuZU(^kUe}X$4 zxT7{SiLOgvWsiQ*DC@IXAl6_4=htKGIrm&dnqC|dO`6bZ|HhH8dck3tRR+y+=Ny=& z0y5cT9;|KR>g-tPJw3$Fd@7U2Qr+)&o@X+?Iv`oV#bHNb&CZ2<$X&O<);j5bxBjV)3V3)z}?9Mz36c9!_?fyKb% zj?#<*wvY#p_*X1wGnzD!Cn;i)&@lry)o0P1NgSaox)NOKL>)PkRGe2mGlQ6(Quhj z?q7sbyK73Y8mp0_Mq1+y=Nk*SByKTJ*^tm8xnUw_=V=C^7Xb|C!Y;b$2Tj~LuYFgI z7(>a*x#^K{yLX*ln_B63O3*3tDvuVA1M9?q7N&O$9YO|<%w02<7^OAXhNvhD?P)!z z`75DOO6L)el*1&ICl~fgNhArKlW;Mfk-*67kR<5d(8y-=pU9R?8&^@1@8r zmON?TX8dUOOTY445>uzSnVb-O5WMJ<<)y}ROCnD8`)o;M*K^=^G<&MaaH;5E+EmqO z?oabN_N1@XwrG-mbiiFY<01EKhud1;wzczLao}sva8-Pg;Ctv;ig0&F=f;Af4&59@ zf%CTx9h&g(1Ec){CeaBKng#nlu=5s(X%$5Vb~$X{#p(03G#EuE?l>Y_F`*~TV==!}fSc(QMPujr2@Deg*c3z#v;=)o zf#+1qBaK(TkB^I}PN`>Lh3 zXX7$@lCK1tZVg?2n19W)eu1hNjRL#8wAQtzHzYlsqLcntf~#3qX^Ai@e#R63tDx27`ekZTvYyD zkm}%PVpdxqlNI{qpUc_W)$=}=biX=KsH#>B#~HZka^E)e%dW*bEqExH5}KS}|}1H0XEh0kgME~+yM zT6li^;b-QMuX2~|Efm@N;+eg@$+i41UuW0LJf4wdy~-*fH0ac;z6;GFWf!jcT#RCQ z!4&rJp)3cZR(7hKgqR@9kua?Q=DZ8+&xJY9FW_1=L1a`GMm?t$ zRZ)ii0UMQ`8ZrJ)$^Yr({*P}1vqB0hi$XQOf`Q8=2F3(;F#(2s%X!i_F!pWZFnru9 zU&Q7kz_GWLG5#C7w*kWzcaOcL85IVV-2%K55}2MJbGzceSHn^NG`!CLd&$Lsx^k$Zrw6lhuFCOm z4C6n@oL9iUnUVcmL%Aj+S2}}dK>+95D~*jG*c-PCwQewP+RkY?;=V+;i_F z;S+9Ocdl1378CGn3*ZVk-d>ZzCni}h<;f>qIgwF}|7}9nw+Av8A8Pz;wx2wKv%rDX zF@ZVyIHUFi@A(`Jl5BD;ii|;zWdno7#IFg=Ve$!bvC2OGi<-LOEG^2Un~EN2)OUU2|sn4~(K47!4a(&q{EvV&L#*V0PcY?5<{& zJhAuIjNY6?F^!@=mmSLCj#l5*`dHlyv=kUk4ls%&h%+4s>{7Hjp6p&$!nS+?&(s2@ zz2$5&;WPhvc&I(lKAFJj(a1g{v}o$T30yuGSiC267X}oa6Ue+0!1qcZVM!X_2G1-p z%h?h)XLZitd#Nz#+{KPR5BT?LYECKQoOFUsC$sbKa?@EaH0Qn0RANY+U!ld>p*QP? zH^U4j(*V|n2O<#*xK;)I(*YlQ|}AU@tJ>@L^#0DzH*_$`5nwb6Sy-D(Ld@Q`F}bRu_Wxcs|;& zh^sgSrJ4w^^arr>U0`4rU=(&R5?Ww$T$quIfsspq$tr=(@eMzM=IgU~!*j8|m&lZlBk0ojfx8BGhU zJr^*_ZeXdtpksW2rAUC?cS4Y-f$^F`wps=D_yq>O0<69btiA>-eFIoLH?Wo@Op9t@ zS<=A$qJXdE1EU**=Z=PX6C9Wg3z%n#NB=U*pZj^09f#|8F~&m8suC8TYE4%wqk^B0 z7z8db7%{MHU%(;1OO^8hGlPN6(PjqT2@D(pj7J_bFkFZ}+iG;RfNigN;0i{LOAMT? z*ZkZYxR!pXmer72x`4-7inGXp>s$ff%L7X~EBHRF`p5TFVQKIxjj&a-qq$^*+fDvl z;92&8BjPw)m;>un4`z<;4YwNQ!WK82`q`O~uEpqLwOqqhOq+T8W)^7y1%nGLiIFTt zrpynoZggnpD!ibRZm`nwDvN$G%amEHB@XP(6Y|0wIJ(?9%mOmp6xjPVq-$MZ)J-s2 zad|q+cFTpz`2tl7A5~a-zV6N6D8~OG%5qUEN2ng7GJ~)JTU7#EWdi&DRSXJJj2sIj zG_%~bW;GjfaXty)+S|@}VlhLv0`FCa^z;c_N49b-Wn@k-U{k-nU8=$P)B?V@3pCz! zt^X9Y^uw=pJ7&)Q{GG3R$L#YL*L!^CNqogI(||q4VGf%#%dCGJm~}N-Oa!v^7l^Sr z?n(<^&h^7vOna`dgMjV@<_ZON$qkJ10W7W!IkOM2YPjvr`N=M>;a9G(A}xc(>cS@d z%Pgi7_UNTEdm3=G25`6uOcVOF{jey=1x4QN1)Tj8 zI4c*h&-7-^bzn8P!yJ^b%+Vp5W43$clx53)oANWtANV2=evrvVfZh88<3DKwX5o)fX_@Zf6t=V3L|(v7#|Db|Rxx0psOQ96K7gS58p8bcNlZfwg-B z`@{!qAq6b_35>Tm76z_X{-Sn#-qJ0+3$`p$-Woo!MfE^)Y0ti+!lGIS*v=+!Yfo6i z!2mjRQ$+YAZ@{+FEi)7u7$hEKa22pzWZ+<_VKThxv4Wv(MF8(5g_&XrOqN_+aS>cs z3z&=48Y{3j3vjjys0L49mED%-X~5k4!N_YtuSLX`HRk(V zZfv<9y`@%CFUUpj{}r+4(u_<2OnDobypqJvah+`UG*VMy;#|OFwSdWJ0pro&ZRrA> zJ{Q=Z7O<&aV7EFIoW5b(cM0}O0bG&}9t&hR1Zvn^cu!Bgz;koL!S2`Vd}4PrNnhP+ zv*V}MBt@Mw6Qa*7GvI7laHz_FyUFVr%AGU|F(+kNb)~;SHBIOyFEz=npAF)}4GrKp42X|Wq%~tyGsAaixLG>F3p|i(T1N5>#DvL|y zOC*R3E3&@-eS$L}g)e~N$7}|s1h$C*oDCZ|x-KwjaIG~D;FNgEVYGnxBm=v80OumJ zb$h?7Ra}c!U%+AS6?4iVz56oHl>)wJYx&-3tq+;Y_w>WM9X$s>3rq@sb-Fv^eyHgr z<;XMJ8#p!RoGIOK=JXl1%z|qMYgjVxGIPkWaP7OMYkAge!vj6d>xKf%@(Rpe4-|z` z&dEOH|FD2N<^jt*1uj9n?>G-xb?v2X-B96KbjN%8D&pFFd zzhQaVBVDPtdqb!6{#$1o<@K9k?cUWa;i(CXTk9;Rf9;HY zT#Tv$ynPp#bRV!SZ(vWYcGoR1R%=+KbAf5*8s-g#2m0KY)PtG&(ikNVY+q8y)G#4# zeI?%-&Gl^TD0SgKi>^RBV@ zv2v{ACXdhyR)VV%q#Mk#pO8SAFwPwz%Bcb`|cLbxgR(eKVWeQ*nUocqlkf1 zq@%n!BtvQH`RF^01|QfO7*wk-=<+@8sc#6LbCy|3AX@3w-i6DRJpPHTjXl0~@0RLL zM&{#hH;Njt7|hTMU??+WU{v_W@M@hh*93;xbsU$IIVUij6k^y`F2Z;q)<(_fWY0;p zC=ZYK^BCvUGP%^UvB>9M+3=xyNBdI;zNZ2o1^w3x$-j8vyT0qvy>99I@^|Y8E{NGaMkbvqtSz&03WW}11xF_ zm@@C}b@nURqAM=W@Z`~s<8JewI5_Focy4j6RyA^BHgn(zI>0F2$KG&3QO)4wg;{9@ z4nt(R-{wKM-_8fTtc6gV|3^j7G)i<|O##mZt# ziXNYur@Fmwj^(yjcUBc&dvi_2y~XJ1%I58T^DK*R{Mq_xI`33%hZQnrcbdDV91d!* zSa#{L;e?Y8mzHoUE#sATi0ZNsbUl4J!+%Z1U#ZX~9+O?qDMVTb{$}x>CaaTjB7xDF zZQ5?>b1yC|Z0q5XFgef>oON-&eO{20iNwK#BQNz=$IlPA;MmMAwBGoLfg=-JI}3Nk za*JA4RwkB;KboyvqDz9k{Gu8Z4m2^art8U0)Lh;uvf-o}TalB1BdeQ07Ynyj#=@<| zr&g*7a8Iyql}gW<$R(OSWv12<{*_XKE~!fbpSg$z{_&bs_%B9A{n+L+mCvT_e0Azb z|DIP53i)hb3FvG%*ke?P05ExZh&d zE-t$X&QsGBEFIf#7+Em330FMsHG6DvK)0FOW(PwvlR-hlQc0f1l!+d9CMdFTE~#c% z*tsnDim7y!E>;$XiHswbhk2xgzUO4wj78svn5lwip zfZ<+(fnKw+SmcojuID^FcTL~5SkHRSX5sS}o1`=vcE2rH=g0KEXI-9xyX=dD6EpMz zANETnFf0*qpZ4IhirO89mSppy6{YI2(i$HQFc-cujhL%suy9gPpG9Ju@f6M{El=h- zs;YSFF`k?hWXJSJ7Hu$lkhIh2@Z?8biTcK+@Uh-L9%e5o208q;G7!?$w37x3;%Ii z1Uafsb^PXGIMwb~0NW&siroiRSUibHVw1bm@R*~Q*;93@$G;cKeUb(HN*28n=sj@B z-F$9eh9`el#-*j2Qxg=#Ga{5(x%*ByFeYqhQC^V{s(+9{ZpMNJMwSO05;r)RicB42 z-)cEpPF%oIlq94U;w^e=MVnFDf<}voLy}87T9lOnJ4Kxq3F)tC;r0yW^geJQcdCJ# zrq;wBS)u0klOGlu>Lv0hiCvUP7EO0Ksud*I;n1)`>hb|ru}2HK^b->Ky(5kYg)nj@yE1W0{%~MV zc<`?+$U#Y{F~Lc3_GFo#e;oNLBAR&=JWTwyp0|Ecd|=70!?H^`&jxaQ^jj=o#26&y zk|`9>$PzPwLE?*|fZxnhN=XRJDKk~yy2iha7d3x!17&MZqE$eZZwIoWU)HkU=m&B;L5RjVOM4jbIaAPVM+@IAuM>iOsuVs>y=| z4ICW~O(H6{S@u^suA8@nA9rCls?7RN1Zu80a$1h~kr*%`j?U8&%d zAK3|-j+{wTHcL$1a8&nF=}y&88TtV%veK6fSgRNvh26OhO|F`7^RlHI*Yao{fmus9 zRlYdzw-m6MuG$!NZC#nv)dc~jYGJ$~ZydSXE;JqM+rSbqC5f|`$3@O~L!o_*(4+s1y%*P3MCvc;@-la*`F2vSL0-GXT>LlPN6KdZm%mJ%^oP~1UPaE88B}* zY~-~1(QJI{Kq$|@6$b=sGp@={ddSRMx7Fc4>owMD)wvu3n|g0(xz;a|>NK0PkhiR$ zS@uCfyY-!gyrRcG$bMRo=<V%7*wfwfRA+u+a2y=@< zPT6LoW8r;sO(eD4)Od4$Fl%0Vam#$0n0Wr2V;g7f=CBJ>;#*MQcKkp%SLh!{=Hr&m znuoTx7%w<9bz(-79P`9Rt_fRg?plg0zY`wf-MD@Cr_Yh4ZkAVGL>#xeRN29NshiPw ziy}w(QwmxLt(G!#7KoFJ8muCoFibeky-LU0$Y*ArkqS^j(`PDd;w;U z1dsKw*coFt#rX+pQA!%1=6e&mRnyfZD5rNU~uqO zVd53&3OF`JwyoE6V`Q#&^i7)=-`HndXfFy+?kx^ED%p2|Wk==%U-=Jav#+tJ-?>xG zRxu^+$>R@-{Y@Vvm(FO+)A<0pWnD1uN8@*n3PvVbt=-GkbaDl4kWDaCU_890lR0QY z6B|bZ!ya{M2Dv&R@izxpI43a5RWvl(pP8L^!8=g1B%yKggN3p$GLBnL5oq9aVPI6Y zaO5;_VCG!2fKBTJL!gL3@njt707jV%Mlpi}Vi}BU zk{#qPG|FGFQV?hkm_2F3W@h6Z4x0`i*gX56Y5InVWrybYBq(VlHJ{e9krH6q6U`_d z(I^(vC|=Ph)^kv_gi(9~TX$!Nl}}HoF~0cPki#Etnk&m~A4OOluU_ z7jVw{u=f$Onq)v@j35JJ1_MJz1G5JMg9ihn27}}TM)nhj*%TUB6&fBrcD6NXls9mZ z{K0Hv!8FZ_HP=ya<*8nk0=D1jAY=t^JFm0`ggYT3yYFm=v{RRSDc z$`(zFPe`b3Xi#ln3C>{SOlTCUU=&MW6c#v^^e?56Ipn~~-HoY7tkzy;G;BDg>9}>n zW(S_0F8&bl9haN6EqGFIGm6GA@>LvEtY}n}VB#}4F7lwE)Q6G3f|0M`pkz#=LXM+U z1f#S7lca!?gaKof_q>n?5|$N45(jLioLo`b+{nwIB5|O}{YNuD!xC&Pg|>h$+}H2r$^OFmPBj*tIw_ z3h44A9AODK;$z5RdxSxCLc`M-M)-He7O7&By=GfbEbR|rZUWI2|!$Vx)CT4|3$ z38REe(?e+{5s9;k0*#6S{Fh~zI4zDBKWTU`>8l-MCKNVWS$FbzG#|hpG4Jr*So(+vQI~$lZ7bvdNj1#%GQmX*_(S zG5rYR@g-+YBsQs7Fex{9w9H{tGhk|3VxtssS!V&W=8Q&_6HL+#j8+XSmIh2J z7EERhmo0iPKdWL1TG7Z;(7-pLN#_KU{SRf;h=uMNjKM~n(*+o&FW|JbVX(b&n8ktN zCr_w&M)RMWhZ!;$w05v)u4s&2z{H->FM6Qayn#`)fZ0Tc)$(6&u*Fjb*NzsQ4rcog zOu7%4ogXkeNw7FvXz=1_xx1*@@&FTWL{nBo%k-eO^cjsJr)CMPSU1_CG00=0>koyf zJ*U$*GzbMW3TiNNO6-$d8<^NYZMVd6P+6qo!=z4ld_i11E?MuCPZ|J3V)UIVzIKb?Fqgnk6!?QaLTq_zh{xI(Cj#)-~B+7T}8{4 zVzImzZ7~Phq7N|H{9v{SXnNt%Fw^7Yj1RUyw@kG>Gc}6Sh2ucOY@L1oW~VeqOKdAi zU}jp-;JLKfdtr0lFUA1WAp6JFZ(X&0$OI=>>eU^K#yLm@z0i()? z=H$6^EnhI{o@mm0!Q}Ly+4csr{f0*K56$)mns)27+&v9_s1E<~-X0Y%1cbHjW z!oA6dZB-aJI2uGZG-*#@@_xa{^x+)e21e-_jZzU!DM_ns53m+qXw%bZu+eB0)@a~T zU@_sj18@1Y4fvKxTI`pCCLWfvZfeXSJO~OCsJ1DSbZD0`F z;k$QDxFoAjrKsD1D}l)(5n=)ntR@Fc@*0hw#q$3-P!f2ST1{xt;b@7O(Uv_S#dBL*_Jg+fuNkF3FgqkP zSyr$(8!$;%1OzLzdLCen{?VYy!Qvp%5?#TiU#3nOwu13zOC?bHf^$_dY}1GD`DMiqwpDnA&MCp3F%uzZ->%sZpWVL`LXi{>W53;Z3;9xs@6Bvb-An0a3? z6kKlo7sJr1`=WuzV++TF$s4qy6O;s)7IvnXrEJW7#-P#JeY??dLu<^ARx1a0z84KP z3amkTtX2(;VgW58^P03LJQj^;_L{MiQK7~51CvulGxvjjp$4yh!F}8t8jJ$kf&}~| zHZ){dHE|eOEe>vYqS3yTd!e$&qo5gVJO+%d|6KRoIh`2t;vS#RK^wPZo*OKCUPyOz z-H%;f7PLq(g;{%r*@4N7Hev3z&mIR}ir5tC;&|;HYew28&jvrsCLMzo4FQ%|0rtWh zte$c2vpHUs&uEs~z_jFC!(v_)tyAN~W?PA=suBv? zb}bxln>HQ!*ZelxBDOPO_KpWUZ7-AioM$;SL{DsB^z zQI(e>JBlv5Xaxy%IJe!qw>b0N1%5t;70x%BogMPa653-mO6KolD>%?9@t{X}M+47` zhNpinOKtFL?O~Lh(CnDd=g84w|KiJoU(9w1eYYaFt4?T+6JybJV3IRv7BlGB{%`94 zLl1b$@3A>BM1S8K{crk7PJu?z9}N6I7^E-QmYOhG=DyN;(5Sq?g*V^=cLWpDiA`6u z8>BiKMJ<@*F6@}RquFalBcnnh(}o5y4?zoyuDe>yY7GrMk2pC37=63@lW#Uiav0?@ zFj?-uUL(sWCec`<-*`vvn9S3Qonr3}n&h#%yi4=YIog!$p0LQKVNqU=nPSDkME*Jrn*wJ#r zL;dxKHw}^;=O;gGka!W=@y3SxMuX}PrgwXr_!C^WZ|j)bk-dRQZT^zdUB^mio9J6T zVUT8MC#P|J+p(Jtc~beTl`G*iSxU?F@>Bw4ouE7*kTXVF1W{f zCq6$u;OCcZZ805Exc)+aH|gAG=YFgk9zHPHW;aB3ZGzj`!r71b6H||Xlt#j&H)m!_HcmNu(qVas ziF?_lC4x#U>|HZ-A{H#L{H*3DY#x2artq^%x3EQ&fa))v>1(66>-o-lazb#SPvxsU ziOH>8QxzOD))ahvauUKJ!DqO*IUhVE8vy;!7Z> zOXHlvL!7l9LcYew0-QLNI$t!pNhR%RIm~U5d4kE`{NaX&0gew9Lb%UaEWV=aYwF6O zSm(2pT{_1Y`kXx!n4AYz7$)&(h~4ZjfYKoSJZub?M2K4QyNzj_%^IZ#Ev8 zDQKhM$e8Q8;I`X7#^VhO4=rt%E8<~McVO1GaQ9HnS#p7a(d-9fhhnqKp~KAS2MUjr z&pT(}tXi=|eL|k3;sOTFw3;bh#*=Iw^YC;pxYWeTJm=pA7u5+bCIqr@ioBf9$uV!k zLCy(B3pq_Mcx>j`dM2XVQ@K;+&?O1IhXscN^tKp?@ICGGc(=oKsdSR@b`{l5mB}_t zp&_ML4B4eKV;qn2S>JGI=C)iVAjTymk|P$CB`G;4hC9J)3A53WM(!y=QI=1q1e{ys zr4f|#@Tti3RJNHjXQrkdOcP5v5Sf`YP3+kmwJBi^tcEulj_@qibhypTXJarshfiht z3q~QC0*1W9f74`G`q<1}*sWJ=XkuZpXgDa6v&XSj-XP(@F(oY-MRtY4BLQxz-9L^? zU(U`BaMN^uA>c0GeTIQqW%&oGgSY-|)C#EL`rx~qHJzo%NoM&!7h}H1J;By|TdyV^ z^4KP_?Mbu1jNb*!&*yG0IK1P*vkC8Zc-&3m3ig#fv}kYXi^SF*lWzse2eezJHnHq@ z-?4$kZnNn@W_PvDOlF}M7ut>MVkK2Si#gBi)~lZI=!{CynFJU429`-JtZMx`Kg|;J z*cLg{yJkaC#-h?RM@Dgx1)JCscf~WXZ;ljjoOqD;?`)PPW(J3RvqvWGmOgs?3=E7- z>PsG(Hsni8Xq09UoM^4X$luk`Vl-zV$NweA86{rMc2^SW+~vq5bk5`0hCj)TJsc{^ z9j7?%OetI-tfM5Uw&SqolEl8!4*_y^G3_=LJ9dkV`L z{jzT`$6RQ(Vn}2c5ODk`!pPzNhf!#0!ZKx%g&gi7hXivo6q%T&3$m&_U`e0w%w+aX z4%Y(!jS|;eo;k>sd7w+JYD2R$>zT5Ild|k84;IVMO!YVz&F$5# zq9UQADB2Lys^M^u-LB)&(p5X$G>Q^?J>M{LdNLdpnk3L+dq9D+FoH=iZ9|LQk%t_{ z7Kh{>6=*pxSj4aGEXj5)pwr&s5U-SjlggvA7WW?qg^Ug~FvYxF?rwR2Nms+M%HTrN zSCxe691;lyGS8Z$L!4&zonU5BQfNzCkzj6M(JCFL(ZHf}&7Q#^q$;jekkO(cpJ{^w zgVbyD{d=_cp_alJ$=BS$1>(|(3!$8LAF>L}TIN41rRZ{=8$&S$|W z?68pMS?(hq-x~+mJ1(#UIGohK9KfnMFOkbiMUiu&fr|o;5@NS}bkgYwkEGIK^P5-IOFQ!+=AAa~xZl7JQv*%iPCYli=v^hk>m!z(uZT zLqwy_+39-IPP4pf;1o4cU`#b=<5RiNbZOg5o|8Wg=CfQ_b??vez`Z|t&0lM6iT=i8 z``_`vg0Q8294!hPxNh_^2`p)_TT-_5koHii8dE?TwtD4 zJG;%yhG~)D*?&jmRJ;OLN0?~aHt6xTOmN$<;Xzr-$H2%_i>7TC$z=rqR7BaAE1u=FxEO3%s<>0FD z=)o)AEj-K>1xI+;6lU@1Fbc&sAC&iFY(5luQ0)kL5Z2~hoiJ!>66%^CAVKXid3;w%Z&y1+>2 zKxv+_vbM||X`P2X#*=rR;VZAORnq%t_BpFCh&QNy3TG5UY%*a1uYH^oFefYCpk?nH1Z!{U^D)}G-Xv_ zuak|kc(p-O?SJV`i#LUWtpP`cm>9c)RwSvMyRl2A$Dzw2V*%5gmklyH8{3TkB(iy) zc%sy|p(Xf&qleS3O`G2znrN@V%r|WX3){U|E?JI9#$%d(m)o|9ac)WE<-O6z&mzt! z%y5u}?*zkr29~G$H~!ejQ6Rz6cdUU~!ApVRhH3*#kx6gOvdcP{wLcBdt$b2<<{s#p|9=m2!^%BRTm2X#*Zg+d zoKWu3ckOJB$b?4aBaEU8JW5agD`;;N7J10x`{9e$n!t`2o5cco8BPj(Yrd{=RuYI^ zUaG}0LrcS1WydX{2~CPBjy!Yxc?}p<7C31xXyTjV#9-&-?Rr@LLjzlk*uAIzOd<_T z4;qacHA zLI;Kyz4s=X9?+9LJF)g!0nbDJJmKd*-B^}P=5%Qg3OQuFW}590XWJL^j@)&&ZBVqk za#&(o;`_I9?-oXW;FSN=I>+TIqx2rXH&X;lT&D}{X$W7zz+K=F;laRu;*Q7Cqw^Wg zPY9V=o_K)g&BZmj2iP(kIHxeM&1jf$#^ImfhdFvBhZP?jG7)JKlQ<~(rt!|-aN!&7 z3O&ATD-Q`8II37MUwb>TbIBng9VShggMvDT8Zr+E{&7_Jz_8Ngki5)6uDz4k1E(=_ zG%y`$G}^&fFVesnaY%N7=l#=)LHH^V%_!?)adyWoI;jMR5V|7e~?GHY4?#hm3` zGfS)HAbZTQ%r%^BH3wGKG-^F?GUs5@d*GyK!pLLrkGtxi!xCNZ?*|wzo@GeSIdC`U zfKbWYb*xS-^BmP>Q!`TzFl^{&{c$Gr(cJRZ`%)UsoGr<08V4V+Hh8%nVASz4TYEfz zLxWXLo033>Qo&*y#zUGt%%&d9Y%BtWTkh$uRetpI>BCyF9lXmHuQ_gJsK{+8bzwq- zsFRcNio-Sq55<}mIti&b#i^xnsYqGH*xZm*RGI#IgX7~+6}P-r?-duNzAk+Cr=x$t zQ2~{n^Ctp1bfP|PIUppG;9-))GKb-mje|<<0m(H@S{n{4NCYXixUiKtu$g$LN875XQ_;Vqk>N3Ooz4^5pC%wJfkgVv)eMH{^NCsj=ajlFq?A+VRDw3b%6n&gr zT#traVU*d@*rUNH;1R!|@@Prm^}aJQ3Q6I)M<&V`MzOqz&{@SKKWF->hV!g0Ld!YR zO(z`s($FMk;$1YU=VXan+KDr#3XZYMFxJgw)Hvc;F;O-x!EMSR&N*97&U({x-hzQ~ zhQq~|a{@m(@EmF25|QP(;=ujEx7#wKuwXKW%OTY-jXYBt`D_+3Jz@AKbKsz)fg_uW zL!8?*v)b8FRx1*>#aXj0i`pb7!Zt0Zg&`|~LD<5{c#CuUw43&A3sYyU=sK5W|H66g zwO3u+G9>H{smO>cUO6az$m7kNi*76Lgum#>`u5PHHN+!M{Ll*5^9x$JQs(}&WK`Dk zPK$A1&2dnzx@EeeN%77h`2%bx1CE`V!jo#UC_RQD#U)CHg{gJ!!E;9$i}T#36uHgW z;{U)>`X5Ih*PmInQ<{u9-)Lq`)KxjGYT?As($MtEpP8Y7??EG9M1$e11||*%NfT$A zo9?P_*rpivrm&`2FM7O4qj&kfh?Jd+9Sjs#pK#-x;4pWEl2*Zr_H&2r|2=k)`nRO( z8}s>Z%=RDNod1?{{@JV4Z&~Z>V&r^SWxhy?W`u0Km)NJUdSNeXB+E;Wz0V>qTohpO zKi|>7m65)5snmMIMXWimd1?;Izhu%&VN^^wD0GJ5mvgV-tpGHjwCeZ^90nrC_w;Z;fbC}&mn1_K~MA)6d$AO8(f%^cX^cIKn zUwF9fAToz!zPsgXOjudmJ($W%K4I~SFQb5 z{N){U3b#qh=Vk|H6Z-`xR1%tkEu55}9Gv|#%yH)t-aQO$FBV=2+oEv!fE&A`+txDv zE!#wHIFzg`<5f8*_xGaO8mGMf7mxnAXeQAl{h>ixhB2k2K`^0_XIw-7g4i9El6#4pg21EJ3;seYH4!jo{6_30yDLAaMU^&;8g9?8R8v8KI zZt!RJX*8~24nN0EXY74W~5@yXc4l^>?e7ZV$jx?}`95k-^ zY23nWX7H8Shf!gVqvnG{at5W09tTw(9J1Ww{7;nOkZMb5@7GI_dIuuS8g?*VNYeG) zE}GS3mvJiXvX!>)vg=#lGNmmvIBWeokHz6#Lh=ri2gdEE4}Yxt(eX~_0-H(d^{P(q zvbFZi#yYIQ1$Sn@Wb|I*xV_?ZKnjP4@K^4Vqgm{Z`~i;rhx@JtT#?yxP$uP|#Kwan zA&g=Vn0DH!ftzOw03;U;OB6KoT_{vTx&dvmb*LXv_*lje&<#u0~2e99HhFo@k~ zGP}`aYSYBtV$SB_aA5XP<`M?RiU#?NW^)ZzJsW1W3J1n74e}}nWqg|CO|~+wIiUQY zNi*QEAWxIh3I=n=cHN8845FM291KgCkGsWGnf-lG^X%W(c%_QN*Yx6)3s-~{T20hC z^@UMIz^qN*&$6Jxa*4Cmg~LhnKd!lKwQll`weOovJes%0GC#q#4A;%p)jeDM=Dj^weBq$nmfGH1-C`U5o_GDERB}scjnm!v_kXT+(sF2$ z4iRwfxi9C+q?wbh@Z|pPe~t1QM^zXO6*4mk9XTL#;-H9+D|^BH1Kti5>ltJMn$38a zWJ9bK&oHw7IVot-B%g3lCV^4N$MNykA0ZY@Dltv`OE&A*#?@;t+#wp!n<{MedR_Cb zKTYvgAM01ePG4hX#QSOg=hvl;lM!w)g3yeD;^v?+|DN^W06tF<~PSK|IwDn%WQp9t?FKAG&L`DWMbnKGqJe9 zaIkT0l(f%@E(V>*&77;2x$eG}#k48(x>AFHI8*rUBH=5g*4y>sIqH>9Y-wN=P)WbI z%5$}tu9Ri!lM_>em*?HHss8rv?%{U-`TOeT|NbbfTP2@)dFgk@;InUp4BdMENvQ@k zyZO#COm3MV*wk#^C$1l}V}YQ{WFf8W8)sI2o+7Z&`5eFOERk5N(%0viO>Y)Fa+&<@ z-rN%L{1#6k38@R&78_j_&$f8OoY^DHak7ouAnirMrQ^Pw>|$~U99X9_?M`6giJIiV zDw^@(Km#||hlB8d`#it;V{30U9ayvb!J}2Wo9=AVZd3>k@jfo{`PN5$4nG?~2YZFeHmM^b&OAn`rzF!i zRovRCv-zwAv+kCQC!Q=fxxA#YO){-ZlTFf1sB^QKY2I$$cEgJ5hi&35D-=5vRxD83 z`OoUpjm4HPmS{hj7Lcd-bXtAh&Qu{!^_5#DIGnq&jRqOjb6cV@gtAt!oXy)s)Q0g=COWMlD7cuF9DyLG$$wqdI4-E{=Z5j)j_;U^@ zG%qmOaeztsQO85Z{}WsdR%{KLQ^?3DCBWdZr9mZIn{CT3u11B6LPm`ep=qmswy&5_`aso~2OvTLr`(8TNX{lt@I026B$i-IeoO#XtSY|py1T^aaR z$hgb3mxQmbtX#95@%+!I*B&Lmy*{oNWRmt_v{cAkHd}LbWkuqVMVjhK#Tv_=8=d)c zvB$XA@o~f!)7vg=GhHTV-mIJwclGPUCpj3+c-Ucg?#72#UvjouF|X~Nq0Gszx1f<<<{(Gjgu^^+4||IjFw53i zgl&A%#>gqa5Fx&U;h*e=gT3iKk0pyM0$6(rx-%az$v*he@K9w!6IbL3MotBB296IF zj5Y}#oE!{M3=E8C{<;>iRESTgDp^XKl8`JZ=k|8DfywQb*c@eOmz`o_thz1a5N z=3q46!RaJrBB)`ur_u6UrOVPfv3>JiGKu$GIch97vpfCCp@};fZYks>F#Y6Au@j7B zQ4wQgRz9I9VC=!q#x~jOk7qVp(SpM=Opff9Pqg?79h#I^6mkdKH1q0acKRq8Fs1}L zR&U$Tx-_q_m5JvdgZv4ni~p34`fw&JV3yHnc*G_!g-xJ=K|6v^;oSvxs{@Msxd$2; zSPob)Z1G^okh~sxv>E{a~tm@h&9~P!n%=^h>KjY)FXQur9DJy)xv`lETXJX{YNpKdM zHo5azh?dCYpcFl+n;r)vlcoB%wCPoxI_Tkbh(p$bQCKYSwCkKi!HEZ)75X-`c<4Oh zkiEd56=*qwH`SC`LPJ`1LjtR|45PiigO;fYnCLi>Kts~tNT}^D4NhPo9RNc@vKBn&pC&LrWLdq-&)X6vQ5~P zbH<$BlqFNw968};cjmgl{ZdiIr=N9WB8B2Lt~0Ru6n3d<92C8q&=C^g?8Os%!)f)# z4TfpWi=G&?o1MwHSduWMH~x^joz<%Ti4P6?vl`~*9$Rq9BtVh3IH2#_f?&@xiXBn4l_J~-SzC+63 z7VDAhQ&}6sC)``QOYPOtl0RKZ2Ymz@g%%X?_HA%|+w?p7VOsK{yDL^(_Eg7M&YoEQ z%wzGHq*oZu~_7WtL%OZ z&GSILPs5;}|m4L_V$GcN3K)ktD<$Xv~?A~1#hKq*rZ z^EIxN13p607xI}V&F5YAflY;_ajDh=MwPhGW=9KVe({0^h6fCt78Xq0@*H1zW(jgi z9B8<#Hsiy_E|zI)d@8qOuJoE#uCi;#lho67g==$|g&F?|iELT0b8Bls^XlH5l*Q7_ z@7)frPVCE@@mS`F>$^EG&sdipVP3rNw6WcvoS@$8Y8M*Qn&rMs*y~b!KG3{YB|vsB zw?{Iw^tByqX4@Kjr*D?ot85Z=!zh69OUT(MmTA+NWHu~ZJ?|Dv=`fjqOkibRJU0Y5()%V=w z{ffz%iT6gCv$q6`)|!R9IXrW|EKUDxc_)Ui&DcizdB+Y(Rke>5|1KPU^lia9&$ZE; z`j$U@tmyvbY{9V=vz>~X`{MLZr=H_|s$c)}`h#Pq*Bkaalq4{}bS!PtGLc{`F`c?( z+tkLuT+=Tx3Cy9Z>#Fo7u%s_wmcF&)DAyiUb9Rp6S(+al*w;C5>k9C;kO z4@&dpJ)9u$HLU+@Tz>hnMA>VO-(DzwKjHA$M{73|?-K_-VYhWb0-bx7DT>Se9JUbm_rtnKIdnjP1(6EK=s?S?SkF!xexz1dtuB>`?AVAgD=s?u34Zkj(7rUS+ zwdJ8?)dE$i1I#67r`sHOtER|vm|c9;0w$#twj<{_J~41|Fhm+LD1J!h+LR!W(V&*L zKqPOCh!^7~9S5Zu3IYO;c^D2#iZrM_@hr;mU}SP&;94-3$$>E|BOy=2<58#XhBK_1 z`udAburV+8G;87fw?Jgx!L#Qa8`pf4*&z6JOQH12jbG0+?oe4~DAg==tWoCJ!Jah+ z-!1(3FFA1UEU4b|A@IcOCu<%`hP?Z6>cYizcW!v?Vw`7$zN{*__^0) zi4-~i6WheUewjk6o^8bg!JNcbMhiuDzKl>>p~^HtpujPRYoUn8LOY8G%xa9{Y6~s1 z99XaMF=r`oc5!jPdceWa!(4Je=$rzx#1Y=K1*>huIkH0Lq@vWZM#E!_yYrI2Z!vgc^iZBJ*h+vi<4Ah* zn~i%<1W2En@N-|I^gczIUlXMdB^n=1G~RhoO64Hi8)xS)6Zqm6J>PTwx6-A$KL+(a z3z!2^4}D92HzDC4>m&yLm;(g{2^@Sb$77n<)E-KzEqwDVf&T>q+l6;bOCGFyv4Ah> z0NaNI-ZiV3^%A&#F)&&-90*Ym`t^YE*Rfef3PN6qoEMVWSRT!qq{En1;1TuU@RxTY zdX7Rm4@JKCNd`RJv_w%TqCvZhLvP!pdxs_!X`E-uIL+eGut21Nol9BzN?W6pi1gL8 z#(P3~(*qtRoZoTGL+0-!!$po#uUNP@3p^3m@V!p642y!M~PdJZ(1JX!T-0^ipSzm*c4+ooLUTF1Y(Uor6jZ^Y3l77Uiw3(RB_ zludmiR2tZ3DYNNxIO}AvW&L~bHisd~>H)J$uUhT_UabRc9F6POoFQPS%mubBI`N2-hw2LzQA*(@GpomtBg0y|V-Lm@1Eq#5o4OoY z-|_MC*#7tsa*jdh-gBOW1#BB0cwc8=d!%*W;DL)#i;Eruv)qweTncP53x(to1tvXU zl6k_E#=w0?f$hpBt|W$KRYH#2ga!U3DI^~CV{Q@>oubI6wNOTDp`=j*!!|uO;RXX4 zN4B&BEE5*6l_;?FH1P2(475rR@N#7H5hzW1(ov%Fi6O*&#>ND%hORafc7{ZO6^-xh zrlz}_%Boxmd41{Yr=Xr+Mb1kXIwza@{#$WjorBALL*YFIL4F~V+mM-|-;FErlID04tj=FB$Zy^YcZj@(Qd4FNCRJ$XHR*DfeK$D)_=eACn`-`F!1 z7)56;V4k#qf0aXcO2dIwU-iw_L`^hyZaVb)41?$_M$w#u{Ble=Jq-eE28k0MFi%V1 zuVa1db3l;6kx%BR*c?W-Uk;L5j*>67NIp{dCt1cI@zrUYi^8=#3q%wi^7lPpuUf#z z_fYc2LAjm;ey@dWk6b@JINJH)(djhr1g@+FUJMDm2PLb%RaH+55%C!|EgcYbE5=KhlN7RUU8gS#-GK>rr^LFmHv5!DBF|--c=5K z*BeP`)U_4P# z($&G(a76#pA~sev79&#@pF>x39#)-x7<{!c{qi%JhHttbFUhQMEdTXMKh`uXRM|J= zaprmj;av(B_gn}(;qH30VDAyfx{Dv`j&92_-0Ajvd&|G;*?$YAzA-f1`=;=}*}*fn zaciilzKf~uf?s~AY%>}-6Q1*=9pDv96#UST)u0r5hJo)?0_TjR+YAT!IUe%M92A_y zxJhrdWC5f2oP%NoilX-tSd$V2UOZ4#ax9kL#4ppx=W|eO*+OYIPSHCDbPO8`qdIi8 z+*v*x5bW9^^6SAY76rC74C2ccO7$pk2RL%RdCG8X#=H|rcW0c~`cEq~!QiV4%MyOM z#P_cxs+~pqucq~H7Cfi&Y+-_E7e5}AcK|)mh z%!`ogyeSUMR=TWK4y<0E{`DyE1w0g)bx<-uQGh{Fz~`W3P$Fko0;dY2fZYNvm4lKu z7#s~0_`DdoHz;s#NZ?9P;16KrZ^(PSjn8w|8b+lB{928iT#Na|mNUpWaIrNCFga%B zBy!&BjJ=l0c%#+rQzv7BcCl6|i$cPJBRWh~5<<^@RK4Gk{@z0RyTrGUtN#`;8fPwS zoM8GZnWt4ihH=T^_B|RO46k!X9Cei5bHqTj_*+Y_UX$W;^w!OI?q#{l zf2V@~OvR;BublH|p0jDdQ z^4l?R>%F!zRA5`hz^S!>*@lTN$AWp4f~a4kh>D|-+(D6DQhZzv+p3P-{m07mV1dYo z2RwHi*pK*RDx<9AsOi8s$0jir394o4uIBDivF$vU8U1Tfoa^_JHF}2;aHp zH6m9vL=N(&b+WS*@NqGU?`Ra5@Idfh0$bM&$)FpO{}?PS9pZBrtT17?@`2&4mBMzF zhf~=Y`8>o6CE7pU;nfv!VE0-mR5HCu+fh(yp^%1zs8?fY-_)O8w#~Fka>-oDBc~^dW=>k8==!*P z(oR|T>@P<9|L*uMo_l&){n5tB=Ns!Uh8fSj%2AT%?d!X%_}8wB&V_5FvsXRK+NK;H zpvtVoB=S#;k=yQ%pqL}ylLIqU8ToS*n8O&@)+7iy=?f$^97GoO$}Znal=7@iLRB#zSZR z|9k(RdFuOsv*t@)YHX(@q7eUd0e^_Y@ds&M-VEy)*sgp{VlZJ4 z3P@ZP!oVid$a$*4Z)O66Np_q-0@I$Pn6wYq+Fe+iofvddJkqr9Cpa5UhIM2r^3810NTx?h)*8FJ_KzW?v%_^f{NP5UmSBK})^97Jhc` zleaQ9hEJdi^~QIj?66V5;79A84kv6Y?AKqL7Gh(9)sll9GDBKW1kJK9vm)@f->Z9GDtS7fkQt)L^*DFF0pG zGn46>g)D4*CJl_Ke0Lt6P`x?luv&V~th95PMXSVmP1f!ei#FOMYkb>q$K5a<+ZMnKHj3<=(T$xPl33>Mll`Br1nk7dh)6W#J2^YCAbxIqg zFf?;%%?K#JYjk5S6OWNZD$TJ_$7;mpJVAtDuCh@TDjvGO)Jd^KyIw2&! z;sHa1g^|F)aE>{wjJ+Xd4;)yg9@yAs)Wyiu!DryQn1!KgWr-;o$8MMou-BCK2}?{j3HP zCUnd>%T#i@fwlRQ07q1TifrKlX1xy#)wT{y5+030Cob{?r#$4^zT>D$+lE$887D3U zg(G>JJ~X;NXc94f(ZD5V(8yYG&W*RD=Ou%)nCk&XhG`kiOlK~%OR6juIXl6DnT4T& zwPR@l`-AR-VjCFt#$+-~>SWWp^k`Mu8b>3p1_ur&0b3`ZMO9B0EdIJ}y3zFmY5k`! z2+#kcvd?zr`I<8p@#oeY-1qJ8nIO;E@fTm1%Pvbjo^fkpe95Qn76SjWd>JF1=O$h{ zJdfQ&m)VV@MT_ZUquQ6mob=Be?ys09PH1Uqc3K$Vq>wR%z3M}=TtZji6an6tJ4t-A z3|zS#1)80IJmiwgxMTd{0i)B6Lt?fs7}(`LG_WxQ&NiLpz;vX@Wv__?6DLChXT^m@ zLK+Tixg8A*Ocxm1M18!f8IupLn8LsmB4}ja?1`4><}8oV67-ggP=Fuqa$)VDC+GV*gW|M_G5^p^y(w}nZDaXS`ZHsk$ayOcvOlV*Xv`b)>dEvm`_JM&< zV1dqSrPF;9o0qViP%vT)aA06*I8yOVm62z{B}Tcn^V*CGLQ^}A$m$fZhlCuoeN{3~ zh@-IE%F}sE>7#otv24a45=*w8usG&s{pw_t#h5T zMs^F{4kNbW9d~s35?lF18rXFAxXOkVaEOOJVE%d6xg*M>o%_r~v9mLpIIIm?t)4vm z$K@g6#JFmAxO8K&PJL&?+Y3rI$5bBZF#On^Mqot>~a@R5RMf$mklk_9C@ z`sWm>toM2PDqvIHy&cDlmjyI3g&a_f^zLWUTEOt7=OrV{4hQxb7ueNu%GumI8nqRF zI4r+3$?20ISEz1B+$WA7Vx84&dK?QmnRyh2x9?yvex13k)ZYoNG7Qvq;?fuxhz|eQ8 z*40e~&A&Z#_FtJem5bf1WMZpg%g@WHdZ7##gBfz{5>(U+LfN!7Fhw z1fCTOriuA5K1ed?h`CZO)N-R~&su{{9|L3I_8HAfJ@2yeHzWvYt!WS~OKxS7XxKaH z!hKzhTxRBg28Opn2L&EmE@p6GU=ZL~6PL5Sn4dk(E4)kSh(ZH{>{)H5R#u0o2Q#;F zOgO+~b>fKXl8HR&*|Jh6E!?jEKf#kFzgQy0e*V_P)odv_(Y&1%$MV;0Ov$TI4xZ`s za_81L?{tEf->Ljpo>F#bWAw|2WT|3?M#c>@O#@pkbt9D-7(e?8wysdO`}p!iz#kuP zTjpCDG7jr`Ulep)kYB_dy~Bx>XF+G`Erqp_Gn)3i-r&T)W&wA&#}?jA6FN0l++mB( zSSZ8)u}3?g;jD7aL2j1|4b7@NjN7!I$8XxPf%(OY!}4`A+C<%IIetiO$o2VUzS3l- z*)0X($qA9ycmK29D9yb1Xu;aw>w9yzZjR;6s%XpaxV+-2hx@6Q%bw%~zmj>X7E{)q z`menFM{!KX(X@n*dZGpld@ma2xG=DN$Yc<4TIy7}X^G<}L+`H-L(~>9x(14vJ1WX3 zFvCtn9K@T!wp!N zKQKmbV2^#!RGq+8Y*6fMz^K20Npk`dw}5}JP~@)1i7`)HoziQAn@YMx+dSF*+85R; zEnsRk;5@dWuI5u|{g%4C44${??J4DDZqv%DE$Xu-u;o23yQfzEjk)5(qbina{d zGm0|(0gTEjjC>y&m^Uy^jp*ddVBptCOEHmOc)6HyO5@rsjIxeGUk!x}E-)%4hRZNG zNg1*B7jSIQsM=g%sUuNkw1xd=DM$4LR-K5dY6a(L1=er_*4Pc~;S;(`7jUjjbCsFE zq@UoL^w65ep*Hr0INRq|XZ3#)SJfq04>N2r6YEfA_@%-W=c2syvy%1)j!6?r^Ev`c zBHBwU+K)Q$q%Lp&XkOpxP@j3Azw@#5p@qD?i+Implwqytn0lyu_f?PGPi2m$a>;$j zO!~&?_knfx2F`g26Js8*xCSutt?1-!)DycXFoj*dHjR;)fqC%`N3j-xJTp`I3(Ua* z%#IJ(+rGt3{=jck(%o9XFB)A<1dOF@(N z1}5tVqW4d@MtEdzD>hJION#w6wXx9PD@&5H8^dD_2Emotj0%3?EmpVK88r(yrd{Cq z;L)C9-d~Z~|Diqi=L)ur1O0z2*xLSuWmq)x-dMnUPeI^nVroVCgb%{0Rogp^O1azE z3;Z6i&Qai+cVMRYOU^k5SmOg&q%Tb5s}NlIm_b0mTT;>cr(=kB0gGD!i$MX4&j!Yg zHyPyvm_t9Xc`Xp>nP#!`fV5CKcMHRmZ4y=03R7Yex;J=meRbtJ)}j%fz-(3!s<(kj zYXg(P1ST1W#JY##ifRnkE7POI?`^82C8~Brj;D1~V8Ple>S#O~!#^ z*#(}d0h}LKO#iZC`d1I0BL%rN9l5`LOmDPgvuI%6@u2_w0$!nCvWzQv9X6N0Sy29% zEw!tnCdFq9e4LEl$ zu;~BLY_@>O_QLGo18fBsnD2Qe#_wdCL%ZRwnUf)aN8JzQ75?Fi}uvV{PjlIC) z`+;%BkIXhfMsWv@B>~fqB=E5ATG1V~pg(KDhv^(YRxHS1n2~9~=Ja3z-!GoHs0{PP zDL=#~oGHxf?VRvxTeU#lYg8`9!y@Mz|msBtzW*>_2$|o z6S!NAER9^MTvxCcC$QQE%)Q#lQsBVeU%)wIGZUXf_MXeV+zqaAGkagIoLBu({Jl%3 z>xboj3t0UGSmQ6S#w)PLe_-`>VBw2k$PJM!Udi*1bL9~SPS#x;*}OQ}SCwTi@6SkJ zPi|P5d|^S#hLvAKGg>((bVSN5>6)dOxKRD^Q0#%=X=dE^)`!N!A_B%tZ@qL4OR><8m%TD=$jg_V2L-UboDN& z(;QO{aICdnxqfy3#_W|E-n*R^teoA&{O+ogfYicHW0_kMW=sm&+?lbtN?mh*A;-L* zoD!T{-W=e2`e4uP58S&9xMpwQn*D%fm(HsD>IE!~3hbG}9NU}LxEHXr zC2-$gU=d@*nPb7Z!$IcO2aBG~(tVS;vwtsXo3^wiss365v%G?7@dWN|1{~Ig?3Ewb z!Ub4@5?CW0I9eRoJpz~~UEnb^@MA3Kjgs_#EIBPtYq{^FR{k3-H6K|2P2D|l`sqV6 zy(iWPu*w@Scq=ielym*-+_n7nF6%eDZi#Km*`C{(z+T_UmK3nt;=+R2T}&}n>n~o~ zw3qRy=qX+%FoqOB7g*9M~%vn1vh|x0V?surP=e zF!WZ>+cw#i*CD(A={(;FhctcGG#2c0HCp2uuruxg>yl2@J_oK90i0H6cJ*A~=xzAN z(K~_T%9>n9nIm7!kEAuQX=tSK}1-)eC&7XZU`f;k)`^PvRcVm;)0JI;Smma*)4xJlA=)x&gC~gSm$Q=hY89 z93dRyu^idl3r}rfH=Dq6or_CR_mWa9hhi+pgaD3}2F!9F7>o=!wol-_`+%om&50%l zuFDtr{(UfUpTKCifk`BSA+m*`#EX$Zfx-8Z%I0IscE%i%ci1^qnxkd|$AkwQC%iZ{ z7`X(zC&qnX&6~r}KY{DapEDQEaC8)$z0z|wGnc(AVWn%%5s#kT77a{%T}&+xI3@?2 z`y3T7@bA^i%K`^p-!YcpN@dQ7d2=k9^{33Yn7wl>x!yhC`>^5s+YP*LF7UoFxOsiU zvDp_`@@nMv+>(n*Hq%)tApIo7@B_!)fQyp6oC_bE*gj#Y9*eJA!9{baOG^?sBw{(* z5;zttxVn}_w5d||;((VHyu%{Z1da7{?Kb~)y3=HA>g zg(FUD*mCY(OLJh$|9iGj_j=J>PENnNm{aq0c1yS3%zbm`+=YYnT^i@F1jfwV#Jl^z z%sDS_{CmJZ@8O=DM|=LgnQ&~&0nXVMu6|Z?DB^LLqVBlvlb8Ghj;jW|lkc2pUc`!WaKs}C%?WVqT6urFxfK4%-; zV#Yi1PN7`EmADW;*=u(dt}SD|yzFC%tH%viEgr5@8#s{=4d#oD#zGwZ&{6G zI95L3=~}>XDdyUZGxxJ=U;O8NkfHFv?ax`yIV%f$nKTw8%yi(GS#bSX&AEe-8bVgQ z44iwKPsjuvTlm5GVOy>2|2;SMf4K3Ff&blw^KT0Hru^NLvFF$#%TBHj46IYdUMN;d zR>&)7GSwSA-R^MGYTy3TQ&>zDmbo$f<5W3&yGDWIROQ88XKybxU^jk{7M9>_woiK1 z2ChA^96Jy2y??-KHO!UFS#-Nm*qEf%ns z2C#G1y?iqLx{mjZ)}OoI*l4~lZkWJgf9@l1mdvp|7hZi};J^5KkMV!rsegI*1?&w7 zV40;Mxx#oO!xw=wrj;UF1U?w*-E?@mMBzl&9PWRnN~%fBp#dDueHW+HTnt*zb$adX zB^%gPm6**ESV9vx_AKOXI&&-gKJWVvygLgxCdzQ#{wGzGZduY@{VX!&^n+Z3o-+=bBgep_}u$Pe$CU>v`HT9h!XGN;M};c{uYE@0I8+`x`!g zc))#q1GnlqzWX0|Ul_c0dCkf4w;|I^ZJzJnLY_Agj82ja>{ll6o>TZ5WY4{1LwEH9 z_67rvYxhnl%5ojAU3$5IcO?VY&RC8mAJ~HxSd+9_)t_o)lO2b2<3$^K{Y8AiUdT3%k#-En^2(@EC3vrP9~+mb1~T=&lX{SR39+1l3`Bp!Kmpp*Oi zTWRaE)ixi0Jw3fiFKSw?W14zjtoeyOW=mxZ5*9Htami?CEI7cF&AvW0dV_#;Gb<0D z!I8?gw==bZc6!_m64kn{8?>WfsgfMmmEwS%%U;eZTRT_Jxz$0^$mP@p70)FeqObOO z?J0IR(X{L9Slq<4WMp|OGa5Ay=X9-$i#c&5yeI4b|I{=!BU zp=2@7V`?)y6w@Y#3f)Oz*7c}dHp#j}OH0tCE9h|F(m6$3yepPTIrkWDy49tO%XG0ms?`N~JO9YQj4y<_C zDi)TpGH3Hyi{$6i1I=zO>9{16aiEcrD`P>!)fXiT7_4+P7HqrmqlUp;YmVh}$>0#f zlBtnLlvypDJvonBK6kNH5eY3>=_VdJPdg>}twUf)p#Q?#Cew3oWvWlN6+6^6#g=d5 zLMA2&2Zj9%pKoUz_+zsuRGE=0LrInOXXEpbAce-0PAU3F3QW|^a-2k?wJs(fSv= zSg3L&>$98SQIn86-&|KG^%T0TKO&pRTP$q9-QD)WFZRjD4GXWINZ&H^Sdr@1k8JU^ zzrTL}e01N!CvB_M{QnQ`fO>vfO{s3l`n7L(OG5pPO?rf-(e`9 zrOBbNpz-!&d4cf1E7St)*YO%CbxkoVY@h3Q=8@5=1{Y1Gk20Y=XT&udo5W2nu!KxG zBzRRyP-;zUo5_}inzyP1ds0}|t*=>Hkif_!x}t!!xO1sas;79Al%UELu=daa2Eyz5nXkyrlUPVp*9ItQj{P^OaL>IPt7x`NXL@ zDTFjc=m7MmBnW`_Bu^)%D5$XzWCDQd48Gg!FJxt8a-+qA7^I8z5j4* zC7)x5rj0{3w*rIvC*%Ep?yRqC;W@xw$GR=P3&E~O~>m|3R zM$dWA8hXaxJ=0 z*MZf-@13?Wd1f?At=u3fx$ya(mzM4}6B<|)m=4Z;lcR21^B}~rz3hVT$|R{b743S9 zKJt|;P!?~sXc3kBV8`}^i-G0BKaFclTjba29$;Y8aB5m$(B-G{Mttvv9)p^P2Gf5v z30r42G5IN+a-AcjDRyTz>(K*?LQi?{L|^3UJ#uxbLWUBr*V3k-z>3z527S}h6UBGW z__!{SZ*o!jMQ)?R52vd$s;%jMm}tDSP*YdrAg6=NTFQUA8{2BxQl^#_H5ojxD(U5Qym^AB((5*>2S}LbQt~2eF zU7c(yOxWf*kLEcl@80`P z=ZkE4CcF8(vX1DIMrN7SCvx(oqi?Wv*tfHIddhIF@hxc*b5U7$M9GXxbaDyJtfq@0kVlK7U-^{Cl>GQ{Q4Chx7^u#yji`-ue|v3bQ># z??1OYSoO=7+mBIk#lqfaDoePdPBbZ8vgiyDQ4(ph*eJU#soh#*QEuOlW<{}$9P#2z zH|`o|sBn07WtKdaU9t68Qvc?5ZUF~n@r7$ABwMF2%1+8=l4!U->%wgnf3|h|AGE%? z@++6uM1et$;k@#_7!g~QqfI(Rr|u_|G#fscz2bq^9FCpY&tx0KIlM}<9+hQIlec_S z=r}3Sptr=*_Os-^*^4bKX8h#+B_y?P=L{ECtq+a73Y8{Vw^+pITwrmzUCO|vp#H~$ z=@q;4kAqFE_Z#jl*=SytPcFbO)(yr#MFs>p%6*7qY4ZT4@?RyMw^@L{zyA~UWDCb!TcqKg%OJ%i#nJr>Mp2GxV!J$($34j z3;K$`*G45XNi6>z@Oa-$c6sJXi><~YS6P)a9J!-s9G+Sq#^Ku0$l{Z3#phShr2J)} zz<(VU$DQdNwr3uI`cq;`2W11MOzYpV;aIqCdbi_>1_s6k-3fp7|0ysmXjt{IpW(^= zUlI*2*-geLY!fmMXk2JgYhYxRV3;+zC3pvmS4PXh#bWa|F&i(?JNjE}t$^`5iRL=y zZovtRQUXlxf31r=HF5bV!Q#rzXPb@Pf2gJ{=-#zr&gq-oKVGq2+idZt%67BUTD==g znm^2xHJa3R*s9-Y@Lau@cSi$zMFZ0Yrz;x{b#F3ZR$%b{eCPqQvtbPb&y42u6Kv0m zr#NIVb)9O`+rjEr$dM?_{BAe1%LP-{56yALEOrbUpDOwsHZUq1H1SU0D;$G3> zEW*;Dr143E#VMod)T%}miz6=%uyyQe_59FmmC&sDqLHtHfl+|L_l-T%8~YnsCmCZH zylog*XB=R>(7k{addGsczd*^9cW2f(aBSD)Y^l~c!sNlq~kxC z9gS~3bxD2PF0!OilxIi7XFcIXOlBKYeIpirx};j-*u6?fNK$Fpsh7ODH#T>!wn!5? zr1*#X$Esu5H`ww{oZfS%EpLW}{parON7}MixLtW|Y!|$D_mj<+Z*Sgv!`NoGOv!_` zof7@F1}#fZHfc-ndq{9}#;~{=FdI2Au5@7WDqwMXz-MQ{VtJ;|?g_J1z+y>*CTE=% zmmKzz1Fb$6SZeRMiXLcSzp;q(!`|X62Z0KOtIF(}1mZZVvqtF(1$07Trg8T_{Ar^757v@461jP#)Puy;d`*PI8 zTc*kT-@+AE-dlDYEN$#wJxeiLVej^kNmoxVlqu@Tf6!J?5OAzPt+0TnupmH&iRah> z3$D(JL`!4mYKS$hW4dqItN7fekKG`^E=wm#8(N{YpK zg2%fFC*CTwIPGZtX}~1=w7E}be)^6U9|e{xf0*qXn$0ViH*%cb5W}pxpvhMovC@Zv(vMz)v_eWz7g zY;Q0r-)J_iTI|8$Yjk5@%G9oHmHXvNx61LhXg6~xzi@V0)AlKly(ptC&7w8hpe=ew zOE8Cx=LQx}jq6?x;ocisyec|NE5nzbad60OsaSixYH4^?D2q!&TZ%yB@wFfKc?{;Y1K1#i&ij71zd^U;{WGKke0`)Wl>rpQr@zf>sKnZl>6%EO%<;? z*cN=bE$6}pmalDj85e$XwR4oJIYx)hNWH+LYI5+vO}Rt*0)M&He?}I}V0&_dSyv%S z*MP}=(bl&GrWb4)|Gf>qCM?iwH(|}2hOp1ZEqW*TR)21=ePJ4~gY{Eodr1I$W=31; zp4MQG=->%0+h??RbL=VI%d&Is?VYA=-YXm2_#Wf|2u1o#J1)7Xsd&PfQC;rD(GQ~K?Nm2f# z>b3&z9UN-y%6l_j9Ng=mcJ1?pnWC$U1$emk#tPV8xaiDwDB8i|dIspC8NFakSWTusH2#(tRM{_<-4CLf^$j?PVQp zt9P}eRdjQ=+%ENX^)_e?+VHO>Xu)l-6}NY@MtqbxpOny+6v38e(UvL@bJ#cGP;5+H zD_ec(gL>VzEuPY!Q&?ORT3iewtT*UM2sVA&;#D})U+|jONo!;GTu0vpOFey8Meo`p zdB!%)1e!? zH1b}Aewwk5pMg=p;7EwVEawlV#upfkEt*|duy{nUNC;~L{4o8J-K1?1#s9-X_XJ<7 zS9^H{dzHrRRG-`2dsw}9uzEgty!C9n!?bAc9k+Miz3=tmzPCo3i`WAuvj?dj39NGM z$rGL&@@-4~a6S3NljMp!TUK(q{djWlpLEwA#yMX_nhcy7g-uqiyKv`c4dbsbcg{aO zCFb%_h^y&{XS1ZuUGrw+vP$l>1FaLMvhBRkmSJ)8=GwagPuqUa@^;EjdQh6AF4$Ao zxGZg#e!9|@JEA@EcJ1?B+mn*`Vpg>6*}|3^((2)`_$}kv_~`V8M}c+`EQ!plb`O}t zvYD2=U2j`p8pIJ@<-z`qckd#Z@FE_Y;+W{)x~&d!PrPs3-@TP3x!}c~-WMOQq%fB~ zVQynP{59OG?uqxi>%P~bQ$D7+97qV%>xzt7dX!mleTTnz!BbI*M4`T?vI0!fGg41g zuaeLaH}~GLZ9?>#B(|Lb&#b%K&fM%SzAfe*NN3dlgV{A~y2w z^xsZN>4&b?{7fu)(Uy5bGUk9=><{;#juzj-t-_C&3p;0@yVdvBLgAy}t%ngh?*dj_ z6uMoX$5wFkapS+o_I0m{Tk6DIcXXN==omL2EUzI^ZX@IzMOraMpNJ2ywqNK8}7a=X&D zQZVzH>Ffvu*8W0k>*58ED-GI%6WW3|w59|^H{SdAIK`@U=X=O6p%oo#L-Z7ujId7$5{%MM*LW^@lOXOmWRq5K(q(H4EJ9g?qK8oUr(>XRul=cUMNwOJ9( zat%f;x^Yn{-uYDvl5-308}{ehh-DNlxo>>@{h!!y^YU)X`p)Fc>q#A*4XSOOY6XS% zZO67vNNR{lS-`^n{|S%S#Kjg9mxl-%M)-4IKjm^gX1d=O(I-dG&%F})=+Y7`@gyC6 ziDj$eSLH6KV`DabwpSwX-ZBfhW$P7>6-xVm4YN-=^dk51(~o>dyLWHk-W1uf>6_cr z6{Vp8eDfdAH1KM5v*Zu*V0{s|I6y$keAern8OhZ%*z*(EqIay?=XZPigy^EH$-xoD z|FZVS|DAT*DkmC8BojtX@zFefzOF}c%NtU^~ z|AuJAjVM4vMe()@2$t$D@~_uq^7*x^5W$Vj~|5r6-jv&i?6*cz9261>0$lC z3MCn*FnONy7oXL?oOArkt3w&=g$AK)xsf^>*>nWc3)Nn~{BSI4{nxpSTu&_cYwOrb zX0Vr5uvdGu-!fy(^JV*9_pMO&`u4pJg}znU>DRqyw3MuE@o>m-|4`=T@P=!nz)E+v zr|#2=?b(tF>Ym$wUmO1K+_40nz7I<*CK@JmZTkK|HX`JM|8Mbt-yi*}w_LyTYm5K+ z)Pon+s+vjvtFWDMaOEm3pB-8)^=VyQo4qb4`4TNkIF4==R(~h$n|D1gX>%Fx z>~)#D*4S2;bCjwQy^6m;LxB2AV*>&{Qt)t#O7vtPc2pM{*L~NMrk|lI=WAm|d5++H<9OO19 zv-h6T4!f{pO=ar@QT?zTC*B-touFW*c3U9nuB2o~r#*wu(jd1vn)|cLf~kP&%>U#-i@3hewr^>lB-OdK-AtB9gLh z98{J{HuyF}VoK_?$n@E@-y|owE(?oHoy+NXID13pq_*50i8E)*?fEJuqxRq6@|42! zF3xH*&txY1&AOP&a@X=g?(z$6MOB~YI&uZNTy)gQ44G$sr$X3IZ~L2v?DEwDMIMU% z4Mv=r{gW1PX>MOJse9>kyT#mk*CM`bIc;()r+Bi(je^Z*%`V?b={4PvQp9O;wxe@O zP@>b

RW?l^mj;(m8DjgPdQYCe+=_2KcQ#?4k@-&@zl+OJVUE6*6ltp;2*}tjD zeYQ&5ZY1?OiU}|%{$$}WVqjp<=lH?QS7Zl+ak`xgZkrow^krWjbk(3sbl9iGYlMv$*c*4(Na}a*49$h*45Tl(^NB3RW{a9G1Aqx($%)qR5do#wKmqVG*;8p zG%?mQw$L>(H8!?3HMX%bwKlP~wYD-fw$d{+cGZydFqaH85z01^aIw~LGuI5W(r`94 zcC|I}vN3Tsvv#qy^0Kw|GBXLWH1^U^%QaB*mX+1>VK$GF40h+s^An43({OVz^>Q+E zb+qwzvG(({_4PCj^0p0iw+!?&O!LqvkI+mHwXBFT%knd*OfkyM=CQDJ(=+o`mUnlx zc6PUQ^|N)ib`CXn3NZFe_i=IZ@pATb^$78G3iomk3vh`E_K67gaQ6(44hsqL4M~mg zN{J3hj|$HZ^Q}zr%t?;UObM^ajxI}&D9H`;4#^0QO!E!P$%svfN-57yODWGvt0~T| zEJ;t#s;Dk4E6gvAPmJy}(3x*;&>pGVmSHg=&v1E#$>cck)-;cntcbRf=&2=s%gTb< z$}?K3OWG^T`pR-vRAwv)^S+*+b}ULSd%j4_T8T|PW(#I1oLOUWY=-`V$8ukPNc{Mr zfAxymj5hxj?H-#Z`FA(vbhT7Y?WmkRsd`#l=F;hv>-r0}E-T))(*Nj8pYv-1&P~cc zzb^0j!h$cK1D@VW+x9RjtE8i>qP?tka%NFybXId`ZPSyA=IZ9DRZZPpEv?hKT4zsb zZ||BhbLz|q6J{)(*|~i7)U`AES1<0~vaD;(;^~_gPu;S5%H}1rx2~PNYwe6(o967; zICJl&-bvG!E}pk&<&tGvS1sMWdHMD&+t;nzGIQCsQ=5AZ?_Be0?v!U&mu}j5a_-tg zJC9u1d*<=kvzIU4cz)~Yr{_;!K70NB@4tUvfBgFRw}An+xrhR zu?RnstO)3rXlD}ln&YuiNzPE>XwS(Rg^Q2%#9Jiscz!xPLtn#xSxo1or_=Rf#NIa zeGhlZ-WJKaw8*0USxlEmn#lHo+giDi%h%oA9k<6}=4F?T`}>{ietZyAX5+D|`JmvJ zu_4j9heh76Mx#whqOAM5#CtvTLU{(K@N%B#%$HT)cifI=bIAvWVT_#v=ncy%r zRwXc%*NlVBO}g{jL^qY(D>K_9wq1$J)^F_y%1PW4@|M}CDrQY<&2Ihud(=Y*NR;QK+b9nv2CQ6h!cwF+3TgrCKGr6EmDoe4p zVpewBC6(LWaV>&VeR^N1t;~CC$|^VggrMZz;s>%XlZ)?4{+v_pQxuprIdSeUndw17 zr82AkxvgA&uc~O_^SI0pm4@y$iO)ip2{`Z^b51nc@p4%*XW+4RiPLsV*d@*zd0c4X z(|YkBRisGdQo3;7>$Pp|+f?+rG^XyDytDh(wWI(YudK;S%>t3+dI`vq(*Ew?-&Zut_O<9vv z=(cT*TzX20z}EA65)T~&;#4Mi%Fq1rJZmv;l}7gJRz|i(x}^(+Iw#&*mbYY=tg2?U zzU$Ha!ho{ykn+flSM}H4c1+DbxU494+I6<#laHr7JGyn=;n#`1w{MDXw~fF3ZDQs! zfj>VDba)upA~yX?mY6Fuv*`lcJAo8ovA&59g}VYRB)Xde7dD-WO-xuBb8sfQx&+O`U>8x6e)&z3wa zIBRvVP%65KS@rRT^!WH4Q)BlqwC%aF)NWUjoWQrwHvdmNFksONNLr)5{9xgc*8ANL ze@T4b!TIw|p}9)9M4Genue;~l|4r|ff7VhlSzzKjwN=`&c}2`qDkR^O-VnSLzD<~w z*{V0^SC^O2B!;P~XU|!4$#lPUIXbP%(MQ+Avj0}xcdK1BnLb}6gJ;fhZ~3=`lkIZy z!mo2$Cw42mXfXM^>@hwHHI4UObsQOF(AxlIIq|N_<{-iVc5VU~l=B zIIqqs&HVp?1rb-eFFQOiS!;Flg5$i2k!OC**kRDX;+&*$(KAeW-RX(hE)VW_`X2D~ z`y*+nq2;CSm-A^_yJcpDuh!!Un|AD0voz?5|K~laEa)if+bLYyLPzGl{buQZw=NJ zle~%u)9BgeyCgmmkQR z7j-dQ!kd+)_LmEXDN}B6%Y>F4t*r}uBPNN@-oz@qWn1<+<$oCxUnX$asjSqnnzi74 zkFw+0z0ar1?}=DnV|r%S!vzNV35|00uP&bY6F8x&S8tn^^Wk}aJ5{dv_?GSOT<5;5 zdy>`Pw8UyDE0u3yx-ZsFy{gl#;xEN%^V2FJZj!@=^JTqT7My*bK3O4ZSr$7-@Y6$~ zlM`6nSH&FE9pmST+8_DL?PhDB{#mT(jOldOz%oMeTt{d40IwNk>9OQIs^PE&Z zdCPr^NOddEbJwPwiaX7x=|63AGjsTn`2VxATK>O%-t^nj=^xX=gS=%M{>{1?lc2h# zj9aCnn8V@#BfEzK6SD>btJ{Q*+!|)Hmq#b|UHfdc{f?SioKf5i12uQS{tbP{O?!B( zPR-c6^UY4J(;oBq#2<^Fwl;-Co!4zOvuA6YD{+>)ba|eHaMqQY?{!U5?(m(bBx3*CJlvly_0OFD=z(0 zxoQ{o-sOo`TzBb`O;f6-7Ih{5)1Q$ZC++-1NqpA@SIut|n&qDyU{##Z$P@E{QL?6i zS-#-FMcD(5*4_(V-kIjW$aa8%jVFPLd4U6?nL(3Mjv{Z?4HsQAL9R`!5AsACII$ZD zFmV?wV329}Zg$VHBT8PtXO@uQEIJf6aS?7+!& zyyXw&<@E{;a&FDiKU=i(kOQ0hwU#F;Nt=H2>@rbr4ptYueLrOJ+m4Z80f}rSCGho0S}F&o;mNvOKJT@!cB+wmX(gLMwJLzmchBb*X0v zT7H17=idfarwXSxvv0KMMhS97-$;6sb;C)}DWHML=fT~M7p>N8g~CynK7~g9V$v<0 zGLPrm@52XBH_ zUUI{nf%yQVt^mgs0lo(l_#QO0?A*YvcY#5up){}}UATe4siRi#1EWwv`sMcYa<@{R z2YDhBSko79rb*O2T+TLEpzLWl_f`R}^aW)F;^l1$^=%(gJPb3oY>|r#h~)n)ocF{` z!6i`C)Th)%=byoX7?z6_s)rj&E*0GSY%4ZNFr}dJ+Lp$$2aTT|S3V6j_|L53_fa(9 zfw0?Q)_@Np%3qk%!>V;AR0}%PmZz06U#Q`AD7_J08vdd+#GEmrz4XR+#smdkFAlyV z5;ZR!cpDj*%GzqV9@Jb6FXa(n)G_E?zoTb+M5(9)qih08;&#r&?QB^a*k&{I&6>bD z>jBrZa_)r>INcl=^N%x@p5X0Vz@o)2*W4y^@}tP%g{pxM5)*>hV>f6wU$;4UNZv3Y zsdI%-$yOty&+ca)=`>Ha5N#4nIS^{4W;gXlWywd6%?l^GJ`p~AQR8r;Vag-cP3)|< zF0#Za6dU}=xpa+bt;N4uzJgX(0fzAM$>B35o1g5qs%+&-=-&IINBBc4UjXxj2`vZJ z`3`&ZobX^~N=U!(gW>x38bO9yzv?@I zc`~#wa-!%;i@wW^#wp_UiwX^&I(R)2{xX%>VFI&WX7f(<$$sXg6>bdMBD%NUm>iZ~ zy5~f9g<8wLjMB3ly*w9sHohpi_PpktdW(Dm`yr2>LyE2&Gr!yB{uwbrt;640<@79JXPF_8>T%5)V3>S7V?5 za<-5IqniUq#tycH0el|?_}?3FrA%OXY?c?bfNk9d&c`P?wk`|ERZG(w z$}*R;@VPNtJz$D>z;srO*}F>JNma%I2PVCOn12hI4L_{3XxOB1fXVSYUNN;kSbSQG@4*6If9oXy|Gax{PT&@phTUZU4?w**s{Q=*<1$@mHSduolmSnReGO$EWV0${9^KmA3>I23U zgY{1?utp1b?@ZAzijvtAB9+Hn(eguE*-gT5!Upwgff~nj&K50TZ!+4pl-at3Eh~U~ zT>@{z;xbw3Vin-@V!jfdwq7x z7BQjIi~Jl zm5bp%U~p*tgj1_>4joW9;a$LN#=v6ez;r%LVtMv1Gn-ro^^#59C6N-0;tO)imdyz` zHCroc&t9mHV~{6E9{ zBw$_AQtQtl-n!y8+k(XZN^IEcRFP`La`2M;wly1O%cu)TN-p4W=9OBORDMduhkLyP zx5}P>r&M`(vJzNA9wb~;JG?ylu#$D|)i!3M54-ekZ#oz=_13YiSHJgE9NTk6XLZ7k zJ!Vm}onEg#J?F}C=`H(S_3%w#-kHFA;5l#9o}+%&bMDRIZ3NwPewbIF_l7$Ii^1gN zzYLdV)TS5A&APy~{{dgZs)WCDLLDx!x-H;5lE8h_GH_Hx7OTK>T5XN1@g|i z?NSM7Ugn)^a(0t$c8S!iYH@+hN2{lvyD{tf^CJN~y$Kv`+bpMe|DC*__o#z(YxoT2 ziV5781!@j)9*g3s7L~60i=_Nu2*D9wmdIoUp zYT&&feb=#zcaj6+;$N&24R|jxuv#dv@HsHo2yj#|F!Cr&6%=4n3}9Kvudn_9*eBJ0>&64(~}+ab#XWMJfUZ0h|ZmZ#}8h@@W z30v?R%l@2GkD9X@YaBVeS*Q4n+<$79k}cr7!(tj+Z=eEZt>onz4L{F3N(pAu(uTuHE=*+cILdyMBLbswJn4!^g5Seh)r_H-NP z@;Rq;-ZCm*$hrFM*7E;x%H49qx^*>mbV$U&#kW5wpsPV5&*ZIzpq z>g_A09J8R}3;)FL0fj95K+R*%HbE7>v&^gMIS3X8UQ3+n|=GmDg3nkh0P zeviw_$Ew+~iUvE+2nvPiOi)a%{m>!ke2iPH=F6Vi?soopJ1Sn>s_YIwbI+#we23-a z6VnV1wR8%0yT1`%Z=76yMqxVE1Xayrf-Zc^~zA48KfySVJq-4d75HnMg| z*~>LvX5(m<%el9qv-3eiQ!@v5k@boX4-d1l^T>EiFmQBg;Wtftvchq(tGk$Kg2{vn z2bcsD`0R`(9B}yakiCr<2o(%Uj8vRqx+xX_8EQB`tuODA(|>)K$c z(iOBtP=)a^o5F$yrfnLTy}o%HQzz?gyKs|8~iU-|pL^$2?9AnmrqmA31)W?y{}Ir^nz>#~B`@R>ptNrkUDYXk-!p zQo`6{`-VYTz-H2kYrCFNgBCC*;P61=HK+*?B_Lpfp9w zlTFBe#)^flf4Np1U}TqKP~^(o`_^; zVk5WSJ71?xohue=mh0L`Gzu&f@^n&@-tF;9Lw2T@mzHeer$^oTuRa8>UT0!rv_`Q{ z=8=wEO2sB!?Tu45dFigdlYMF*gRro~X`b%qXDaG4b$Ses6$o!KajKM@;p_2_L5f{G z<^m&!mj|bcxbr@S7O76JK&57}A_10d&N&MXaysO3tZQri|KM=H?UIfp{ua4Jrv?tT z9}n7Xy$+_`%-f{d(jc|WarqRPoD8eEF?JIY8aUMw92h_HG%z#?KRUoRr(_G`hubAH zO1E)JJDp(o%(JM2#UW4i)3NB%EsA?5W-i;Q!1Pf?A)$fqLO`-GXYN7+UY_?y=V}^l zy76dQa^6c9Uej$S7BDxSG*sbind{KSEtOre&rK@-&7$L-_ZwF7X|%mr^?Remw?q!L z#EMTW+wXK`r)aIba=35u#~ZJGO>Gq4h$hBys>zjqS>M`iaO6kR*#|0Y9Dg6x9oXP1 z_V3G*=7pup1g=bI)B84|-AQ4!h>rjx?+k_xcgI6)_7|L0<^(iyJ1`0^b>J}F;Mf~9 zrAg}Uj2O+j2Myde6!_0YgqfAh;L8eWU|Z_pZsjA`8d$Pex?Y=s;efK-VZ#NCoGTg^ zCpb@J2sLcK(!ruIOMywSKu?;1E5OC+d2hKzp41J6R)-^(8Q4rRxU&qJrK}EiS{?K@ z+F{HkP!TSfRFcFy)1p~OLx9O?ONW42;Jrg#O13X$U9|!icpXt{oBB4k$0I?KyZlVk zw5=OhYhNng{Ix}e?Z?}re3b{7CH`IC^y{FK#Bq&dhNl?2bdJ2Q@9XT5%1fM?u;USj zV|OEW+P?&5^(Tv@mn1aF`6x~-&rs&~N@x;dakL74@_?=HkejT*gZT^x0~}sG46|~Z zxKq7{$>H(>wuWbF4O~2riWfKVnsOyJUgOLw5H<*9Hki=4MM5ceUfkUQbP>9Reyn>w~_;GF*P2v4R0Glx+^ zqw$AF(w#e|3(YA|ekR*md8S$Ji=3okSL*qsyN4qpP8hK$2pnX$?qK{GQNUz7$6qh# zQqajQPwaMIP_0Y9a76dr_pUG&W{CqoSPiaxSjISs(Jw`ySt_fLCz0i_Sd>B|Uy8t) zy{3#p87d)G(p~>J9QJSt3eIE`u5#e~>ad74a7LR&$qJ{Gjn5q(aOkkT6A;XGaN-KO z(#FAJ(Ckt3Zhj-{0ftG7Wu6)+Fz{9|si-*#mb)z38p3no`HrQ$Sv9M>5_5Ddye6{q zusAS0;Owt(_~iC*=0af^*<-q9hBhbn7>ej^c2z%95Tqs3&Xj)Os9Axcew(3DmHDBG zeX|a=u8vUUy50TcsL?bPU%k%BIf9WVJ=&HACUNpqY+$at>(XYtBGKS`^HI0^rQODR zPWrNy`P65B+!TG_s`0@!jl7E(I7QtYIhW3GYw&y6llg*4>A?awku-+Rlcr3}9jPl6 z{3J(oltpG+f4`+v)6)vnS7j|)KG;Ddc$w5vkJvx7_ zS;=Avr0KMFes##$_bv7Hcy=7TaYxJ;|@!gljH8_f+XR~t=jBT z7Z}(Rwu`f9cM0!nP}WmvocMLpypTN?ro5RuT{|Xl{lqS>5cwYqJPjrsG3bOl*SQyL7MH>pc|8h? z%mf^mnHc6NJvQE=81Qzxhq+t+zYgUEA}6;^UFb7WeKos)H5)_zx~Hj6EEX_?ywLBm zDireDa8rz(-&OnG98S|I2RW2E_+Ou6=*kuFapTTb@tb()gfyoW*R!uo-?~Ju_y}!H zwvmvpc`oF&t3lEG!|9;-H%x86cfDUVWu^STJ8!z8t|W1r9iAx>bT;1q$Gf^22M=+i zD40lHP~hs{;H~0zfk|z~qK$jprWvOza4<|g#O9;Hn#1MDDBy4?x>2ElgJt0jrxkxQ z-=r{U?)%Xy>z}plmY)L?%ZG+Ts_u-7wmPI&$_5ndn0+^yWv)~HZ{4;i6&99;J* z7{wb7C})c_u=X@u^qG;!%WQslhHJi%?wWrGdHCKtFxn-pGwex{p7cTBklDRG_UGla zCaS!bUc)5(Mm6f91IPVImxP$NO=+n8eDu<Cie$o z9y=KIlA1g^I2)K3K4a%wQxSg>RrXtl{cBxTrjYxBg9 zQYM-;oNpOdeN|#ojTBP5$*8NtsOh**bLA?7EBb~K%{B)lfBZYhZF@mAF=2HA$A&Nk z;e-z?*(sBCU##a;^oaN%sjR`f^OCSs#)4DXR?-|TIS%cC5{!ZjjWPkc1v@$n8P?d$ zXxx8E_pfkE&WkpW1r1yWm}Qk(1w9yCX2=AeFo~Ve;k2SbMxwR2L8rp2J##}dgMjXX zpoaa-{R|2XY%3b%4Cb~m$sVcR9@nCqvXD)hgW2)^;&*aw#-E z>pGTU!)#*FtaG4A;KC-Z4zCFY;x-;Eu^jG;FId=bwtZRYr1zrf(;+!8sh#V8*j9+x zJ-Y3*;?^Y1o7@d&-0R*Pf3~^#EKl=`-SgLGu^#@>8t}rgJ%rg;qUT?d#V%8YM(GYl zxd+bT9E~<}&f1hT%2>GG&1w{2X!KN+^$1|pWLTrQqG7ttB90XewjWLe9XPGga5Clu zBVz*7DJ|C6j5&#m8Pp2c1a@}>9%w6RXkoI@Wth;wuF?3BV;#$i1~vf(o)yf|t9v_d zxT;&*G;mD_VqjrdF^w&OPvga!5&_w}T}`#0IU;o$_!JnGAIR!dSO{@2vKK0G1o%%7 zSZmP05+l)SXDp#MLGIw2fcJ~I9A68*-6)~N*tDR^eUpZklEnP2KN^<(TCyT)U&t!0 z*}G>xV{ShCg!!i?&u>msV@sZjz-H4A`u91vMi;c!Co)NJEbA#@lss|Rw&Wk9-Hf)p z8Emo)jbExg?=E80GGNVr$jrmQl5>JB`v4>R3FhURtlSU!V?5Y16+{^W&ecCMouR|} zQm1u~AR`lpP0o!YYz9-=6Brd6CcTkpuq;-}KDkGlp~b^uaqOG6+#hYRYt9=u2{4{u zVDn)3b%4?9gjwMav62(5QXl#rbIh4gFd<|u0~g03rT{)pjmEEQrahKuFsNXOIl!v= zL4bd$VBA#;SwR!cFHLVNIh6uU7IN)#%bv6~p|NU;{&eM~tFq>wKCa?B}s#0GjxswRsQI(Ck&wVts;I@(q2@A;YEG!0y4op}*n<`@atSc!9B8vyz#v*6lD?zC zmzU*VW=BgM}6xDyy1{!YDfB36Efc1ePwV?mQ72V?O^ zj+$9fY?=(R7nnU)T*^D(TDT%IZ-J6`XVjYs3=#q@;!SIBFKT<3dn@-(T(D+igigbo z3W35TcTZfsfNS*)IVHoeN#2v(q?xuT*!_@_ z)NQ;bS-SIg(u1H)(V<+-`X?rvC`=Am+-mgl;v8nS@BoF9tA`gx?qZw7cx%U>d#>gpffzf^ei>#1<;ia`Q41JcX8Q%QuKi=4T{HH_y z4<(5ix7aHhES1ITCo}Lx$Ufod|Li-*(z=hyqsjROYgPe|g2oKB1g1HkCfE!6JXh3T zd0<}B-}^le`HtsYT>5Ltu8wQ^SD4xAEEgsx?B2@UnaQN8(5Q3acHZAJ;R{;xc-anJ zZ4%UIl3LMtB=pXy-w*gtFxov})?sZF`_a(U!N|wJ$iBdaZ-bcBslIEzH&^bpx!C0D z!0IjXh2zNq^+N}j$8T&9GCEaIB=a?6n#~2n?4KQvS1f-sp;5y?w!|U*L?N4uT!ZXx z2A&_xGC_OFZ?~0bvsdg!hJPZ2em+?R8Xm~jH=+laZ$2+Du$1XasGVNlN?Lot- z2NkzJNocrmszH$9!0kmH&V@3$Cs-YSFo{`2elnQOIKxxw1hbbzdU^+Qdfp9^)6yepvZAjIn^q{JZq@8Rs@6AHVvX9o81z0?t8o~@nfw=Y_cq3MzY;{r`) zKMuC$Bl6)z+TjtM#d{mYRy0Z^C@kb$7C9|Qy|nS_4+a5&*G{5*4;}gz#qsGEFFK&f zcO>uDvcG-YGBVcFULBCkGW^lcP}ts}wU|MWy(EL*rh-X)N5h*B9U-L?7%CX#8W`u< z_L-e(kZy3fRLK5v>OFPu_H(^9Q{N`;(P`k2H{~#}=z<7~C zO8EkluZBhRgBAsI=4lq;su{Z^C1yQ3_RpiIBuF42Myw)8{6(g~fd=_$3)t)h*&O8_{R&DN9y^%jGa5MxCWMtv zcx1uIS)0tZqe0rDB`%;R{ze=Bl{SX(ezyy|voi&*7Bg@N7;$CDiV3WZ6mfg4o5SIL z?yq!fY(U1dh0|k?PIUZcd~gbvhQfM1i}i`DS}KC8Jp7`+dvPzVYTSM_#b@y<*0{Gu zGB4Trm=$(3b*(ueYtMAgl1bU(y5NpRnSgM+&aDeGcYQqkvL~l;;m<^^bdI(J#%Bo) z{2FOrpZc4ZC0kE>w$w%Ru;QE#8K*8f6+d*keC%JM?)6C>pX3haIke|(U=#n>#Lmi= zY|$2~!SJyoDshXLlt4?|kJbu`V?Pjjm zDAdrv&CuxLz!JluX+Jw&;f3+RIXT5SO^)1wyp^4^|J#~9=yS8?t~~NWbh}o~75%Hr zF2?=;DzkK_zDJdwz^qFJj0-EPq)tq>>C~2Pc)IK{4|^}8%#Er{d8wt&d79Hd@f+Oa zwqR^q(7#3h8TWyr#cE=2#nxU@>OUmeA71l4oTZ=bg$ILX{%t8ojoH$f60OD@70EAt z9JrRuS-|L+F(=-j?KrzOL#0g06UIo_gO58HSTFRNE@1tavVej6gE^nYt)~1~{qzPQ zj|M@57MBOD1_4a67X%eoOq4M=BQrrx?ZXB4s1$*AruU6Zk>P=&u|f}%FNt@URP+1_ zJf+X}uXJ|teEo>GOX`JXC;U2J@#~>CH`}5*sTGVXmNyDKn7sCVozR3THU8f=DU8p= z`Zzpfw~6yz*e=%ipy87Fd5((TOP??tay-A2UrcRLbuc4`&6O>Ur`gWUwaCABz%XXV z1jQYBf~sw=zjcdmirgGi`oO?}fsu*X;X~oHyxZ%OkM%J=K4vg`Ta1rZh=#%gXJ$73 z2p5e54`=8^$x2xi7#v7$WU^q7nGo=Z@giRf$3v;hs-7Ym{tNza{9DZYx?e5nc;8o{ z*6v7srv)oom0G$d>?rJHU^F_y9w%p*bfDp2Q>&b91ltxf4)JtKHKX4ji2Be&ax<@r?;KF8Q}a zHhrJw`_L@!xCzr^Wj6N?E*66aCs_@nEua1gI%oDWWAZtZz!U`rR@MLo2Tp+ycL#P^ z3FGbnP6q`?7NHmcM{d)=2M(-)a~K%e6eT8c=%=eZJi_m9Bj_sR<#Ox^PPd+f3=R<$DW=7u z9up2QU-A^cTgo5uoq=gu)bzWhH#{%Ic(9xi@X%zNIla$Cpz-oNr~4X5&gvh@_;T9p z%7jVoe8x47Cs#iD(iCL*YOisxkcdgfnvJSf6GT{LB!s7I6u0o~mD!GL zEG-NidOPMM9uf1>xp-9YiPS?D5wnh|ERjh|kFtscibSf2IP6fpE>L>2b^7G$k6)So zafiy8%}DOlRb!J#vN@)jHZ`#Op#BJOWDsu#5imbss1ViR;Zz{oCKvV8I~cjdb;*?CM3Y`J{p zX#mG9w$9C3)sG4m%T;HeH|I9ov$3~bVu|$af~n@G5_>8nnse{WxGfd8aMIKZheRVS z!q`KUx_QqXl9*i}z^V1Ti&N^c=(+_5*vuw0I=VE8YgxL9#eC#Sj^U6LGiegKXF5&8 z^Q*e?$NAfk=UKUvRJg1hwWRAV3*Mg zhiLHw4!kV^%nB9%6j_8PFfjFeV_-FCnDk*X!ygF)%|)^9R$KwI7$PpPIQBG2aYgo9 z?0MGP($FlOz^vfK#?e=@Kv{|TiIN9M>BsArh z60=sUr#D~pnZr_-B@8+j@=bcFo@Vg4RprG2-)yb360<{$tZ%0I<}Tuu@N}5ffAd*y z(Uo2flY|zNi1%W(Ew0)Lj6wF>9*J!;XkKX4z+^pziKlIW3m@BshIbkZGHz6=_?er^ zPIOr?S@_42OLj@Zf<27?gmy*rYyCXFHSWxDi6cMSv^ft=$SZj)eSZT7Z!aTHKtKYo z)I?XEn1@{v7Zx?2FbLz&n{tLPZxOS~4K?B^N$~w7ais$Zoh;fuheWy2~m@x!4 z&a~S!dCr|FKc}h9eI<~~bZu>4>3MSoCW{6J$qXk2pGMxbU-`o%86TK$yV%>Fv3T0Y zgIc_?j!ayCip0*`X<&a**7G+_QS+bZtkrj|9yFS+*xDo-+5uKT3%H}3@CzEYcr7owF!cxhAM9!HXUU){bsZbMang7O7f&KckT~ZRge1jw@HIn2Cm8>6B=F!oDH6JWmWE#1>H7#7j=GJ$R$3eKsV&?-k?;HPEq_>mU)6nyk|p; zPR^rk`<}Xpbv1O@WGoPw@8HPP<>u)nw)W%Hnmr{<3YTZbv1d%%zK2<9!iO{U513AI zHgZVmb!HxUUVr3^5`dZZyjE#7}qdIUFE(G$D3lrK7|GRsoqS96LJsdKi?IrtsbI z{Kx8_Nj4j|1PI1J7I>#UJp=Jvk`H(DYB9gGoig zNyULlWx^qio=qLBWhex_g|oOPa)V4hr3H5MenaVc;m{$|&dJC>Nrt z@WxSr!%5L0SusFarH4s#f|Fj)QljhkLRT}o4>nh^2RA9$XS1fK&{HZ zLlv4X&MGcuJx=TklG!V!GvqWhmmJXA@_2K?38A3GfC>lOBt^s2gHksF*&m44Np=f1 zG>QL9ewHvpXu?9HP(FbPt?jB$CP;*JOq{{5GWQZE&%q52oPQq5WvTr7n>a&Z_SMFe zv>T3!Wz#(8F&4~g6iztkeQN%nx1BjMS3dr@Kl{v+yd4efQx3{WJ~fbO);M-pazdm3 zD+ciw4XvJzQg;q%Z*x+UU{r8$3h!f5dE%rwqe*X0uigtL(;Vgb8=pokds=fYUC*ak zujgrm8&gA7s$$MlIS!^7jE58&W*Rdxf2eGb5NXo$IczB+VD_YmTZG}WVRy5QRC9=j zSixk&e;<0;@62OQU|>BE^SQENQ9y&?+>BH$N2Ld%yo@U6p2|3y%@_P~;iyeM64PNRB@lg0{0$({rJ z0{vNgIcJrK&AM@4dgp;#8yUD74sc9yG-wbpxYJ}U(`+r^EcRoG|D=PvEE+{}95s5> zr$4<}6UA&+@YpQKS#wI0UeBRLPEIO38cJ^thW%<({Lq-dpemjtD{j&_ujhcx-?kqv z4P`0~r7Elj1(T&bq*(48kWgqcp5m_J;25-EakI~aC2!B(TQbL}w1HuYjP8;Ewu(FV zOJuss4)91e|C5q%RJze*5b7skbuis1{^^R#hFLczY+zMB8|9pKU*N#POO0VXCU?1~ zFmV5QDF5j4m7l_5wVaz=Gk>QFCaEy;u8GpP!n9Aw$#5rwK!wlj6)IUz8TcNw9oKoD zdBTBb%K@GP4`%nxmgi(P6lm6eFn3-ZtFVEiP>!Q6$1A}SM)@C%@=qA$N)Ad`FotP3 z@);cDxp9CarGev$GFym)t(Zf(V1rfADTWJg)I1xQT)1Y5_AU*$bV=lZ_K#leH@>M^ z-NsWJ1sM(qElO@a(&W|>YFIk;)Y&G3J7+uc0u0~!h^eHphUQwA3FuZZY`(^{^;U<- z2TjRK7erZ96288$Jo-;ce1l^a*UGgITV38U@SHf1eedDr+$Xk)V((t4d4&0-rA!om zktnWuQ1S|s0fVz{Mw6gH$Npb@AD=K}aOm*0yv)3F|JF;k(>({oI@0rhPJANg$hUKr z|Ex7a8EXs}nx~ffbHq4sh8$pRIO6m4NZH)TKTp;?NxbyyOJ2#Fb&qW0F39-?{RxW6 zTxh7kd^cf*k?ynCFGWoXFLPG*N0nYk`E_YUWWI?c^XZ8V>N(3Ky$*`ZINck@C-UXQ zGnFN)8za^%4PGlMxUN^Q>!8EssScbgvftdfY!@lmtEl%#X|{MnicwW7U(CVBznBb; zM4u>S;3#nWTgaCFhk^IW0mVgbhX1nia?cz%l;rU6P2ik9k#9PULLLW&85-kF8-*q` zmK=LGP1cd4#EZ$K!9zyo{H-etHA2h=z8uT?{TBGEL~=1%%-f&p?)#Rvd7bDR+r)KR ziN{SJS~Fedw3XuIaQ<hgMX{sM!QlBF7-J`Mh#mjWD1qNR(Zkp6EGnAc$ z!B;gg$XD}#|Jw~rUyAgzxtLsp$}&zB8yxtzudu!J%oMeUI;$oHgxPL3+;P}Sskv0| zp%C8r$s0_*_1>U_#ova2tjtTe4;d zo+w@!t0QI9=+t!1s`K1F1G2Ty%>UrWW2gMh( z9+>1HbcIR(LALIc#;!Vs^fUMQOBm&TI4b1K2y;55e8o{NBw6tZ1ILvE90HNtF%9e) zOokmHn!m5<<}e982;dVGk<4k7;y7%0XWE`phjU);Ofn9>o(Gsh!b^1KcU#7@vmp3GL9&tcr+c z(EN5&-*I{0{b}0%Hkof0$cwpdNtIZaS9H*|+ELuX(P*0I?2bnF>rGNQj=s_&d`}$s zo*Y!3lB)9N;NhvN3LH+6QO=4D4tyO4c#3AT>o89Fx<&iIA>9>>UOz+qrM7AZDX1@L zWcNv&IrID@L*B~^ZmNIODGxY4Yb}3C>7=qNA!Q10OG5Wj!G9a>uF}uJg9__R7Ee5^VqQoPzqg zM|_pyB0J^fJ<}&PNKIKUCBURm;FxtLf$zov;UA5Q#hf%vm{d}lHAN12TQ@6MIP0Bp z@V(5Cx21u-qiD{qLz>MYZ>}{-<}}X9XLM&}HvDo?EvJ!Pw0aMtfR$d=K-wc@}(wjKwi9w&XC zrkrSrJx9v=zbuJJ(CMF+mR?;p>BiCRY(3xiI$S=O#`(ukuDV+GPvMU1j622G<@uGQ zl{E4%JSzX2;lCxLLIabAz#+vsjY1RH_-;7x20S(4U=n_EP~uIa*%VdNCQB6`=2<%? z&g^$!_xP?mW0QJa(wn>{sVR+8Ut$e8m_24P@){q%;CH;T{M12Fs}e=uOD{yqWMWFc zZ2Iue;G)D)_9h3FhQkUu2Q+wW%y^p0uijiH&LIBaklq_6(-TiGTxXJ;uts7@qs}}g z^NK^7KF#_S&H8hi^h5S)3NY!*G;QQ(5`EKTwxp4*rS?WXgHlhEp1>gy6~|w%{w>(_ zTEU`ma$31W^dI9%B6o#j;@{r<8`}A3yi+!^vp5UoxBWsV{NV=Q*qp(x|zl$!yLc^*Kzd)Aw4pJXZH% zT+QEfx4c)MRAY4TC{$X?DXiu>OC?8$)k{P%ZUIN9#m7zoMQ1%O zMx%d6*!lYn9rh(`T;#%HQ~KtHW90KCGpynh_Wk*EiQ8vJ>^_%=pIW`o>)FnJQLxbY zIEQQ^gXFElt!}|GdIb+QDEEfUb8Pw~a_aJlnF{H>EWXS}%oa~v%E5>0WVUT!;fm2ArK;nQy)5Bw3sy+d06bzppnJ%vGw&q4b z65j+_{tSg3jBL&i>@06C?Umk<>B$nY;K70e?Mz-AZ+|U2F0d_jhtStY$Ii}Ouep3n z=BJiBcQv0Z^ZV%3oNHn3AL5fKos!zj#e1K9<|WsRu1T7H7qlJUw{!}(u+LO!F_M4e z-1%KMY>NfUiGTd^vXvc-ZQ?Fg3tNPOHYxY(e+pRHH(g=QNwpbezZR+K?df>btylM| zlY2{BR#)5Rl#WIgnG%iFZR!dyxhA=}U0fy@?A7TsA!wPVD!X8@$01JDSrLaijGQ>t zW^A8O!#XXz#cHTTd3C+^Zui@4iLvXnaKE$Yfzv9Mr*V_QM; z1jbdH7BwuH67*@tp_%3Efs9;^at9Z;$>*F<1@Qg{wtd1?!Bwlub@3Ff*;0^TS7%Ke@+He`@-J$Hte>@ccS` z=+6UY@r)e_>Edn%l?;45-3J)$Hc6&(xlCK2(4t@T!y%nrX~BUe!5)P~9w!D#W|6oP zHy-tva1?fF)g(zQKjETb^pHC*@j1xi((s5Q>PCxh&PNrSye9of1x8k`fJW{M&%9(N9AMtIW#jRfWrixs zlM}g^B~R+8O-h=?!0|4j!@Yo!C0?LKVTS|5F_~oy%om)^OA4C>0vs3`=J4h^Jz(H^ zp&;1K;JUo(Aa~rACLUJ_N5MV?2c~(4#6MWLGP(vHa9^6p<-XyN&?<*k{|shPtqq5+ z@0-jPJVVL!>Bm5!WewZZ1diB#oOr%QLTIAe4fi%`3c-*vQLFtLZ@}C;mZC*6;H*&E2Q(#K$a1db<>}@!rtgvvZ%7q1N$|o3w z6&7;ay>Q@(UD3$rvY>fcz(d}}KaPl`nY4Y-Imqnpa9GV`VP`;w60gSx2X-!l*=!*X zm=!lTaIk!7JW-HY+3Ry}rkbRY*CbEhl%fO5D=zjr87vgJvVcv+NT6}D=>ewF18k;l z4vbsd5*Rk>IB;-%SZ?rUk@&tBtx{s4+;KUMx~?0V^iMT(F=aK+IPB1>*5P@^#APy9 zs052>0AoK--UY7YjwzbQ82k!$xpTg44COq2lV!)!$#&)0>TYch=J#DmV1KGKp;2kW zd4*F?+6`86`W8((qWFp>yf3>+_74N6*90b^f-9{0|4J5i$ND&nUzu?%a`VGG?kbEP zege)B0ux(YR1OJPS!nX+B`{BzmQ=D-;_&NF3>;cZ9tx)$Fljg}6x({KjX{vXQRtfk zv)Ye`g8mskjCZ!EF&6i%>U(nLlfq#&@q|+Tkcvo=5<%8TxnqoC51NXnD%juLu##C& zgMt0ZES5!oAFb>B;HsOokyV*Nm~)XqvwDXiPvC)O>BAc~>a$q$99Lq@3i^G-*sJht zcGZKK3oaa0KO^j?`rAX{Hw|!nIRXeZEbj3eIE<27x0?iIb)JquW zc(gp0ZsTx_>MCk>v|tvTAaRuU(*=D;g+&63Co~E981kxyC@}J7OrGk%z?o%H5<7{( zF>%il53{YF#?yS{J{R1TU=mPZ?mfY#w1k1rBf~+et-zsHn&E+rg*m5VY-;P#i{)=cDf5joX->)tW5BFBGxr5%mjiV}{& zE(cotFJuVF>CWwC5oFi0Q4}(j%1R3P*65LSycAiJ~mT>&BkE;;}dp=xk6Wb7x8(>U8`StQZ0pliufUyre|Ca zojMe=elxfHXIZh8#p=QgWtZr?=Utrh)?E`Q{Q9v?r^T^u!I5Vr2Ut{J9AHat*&;M0 zp;_&RqI>&=Ha(k(wQ)7dGPf7B>6Kh)T-cDnVt(KNPo6@nOoIZOSq39l)`F&ga!VGl z_iC`pu{hh9=^o=-D&eZK!BJdAk;TRHfIv>N!{dpE&aE_C%&4+}$!G!7rv@oDM`<5N zPOpUm4hd!)2N<;!*ozdUcO|4-G1T4DTKCUoT}NAB-9j;khvHWp68j!XbT#(p_=q$) zidZpn>M71_Qxvs$_~{X&T-!sT07vP0iCI?tTxuF3%N~lHYiRN~-;$sjmZdt^<8^Ba z$GnUc>~=})E=p(ko;cQUI2HLnzM`}J5)YS+-cGGf;J_Ntz!mZ@ zm%rn9wbWt8v<9JNic8!a#g8mt)N^2x`M_Z9mMpN?Tdw(y)Zr@*3jDJa_>Zj++2Sbh zfRDxI0yFmkHm3rfKQ6DA@UF9BSm&T1Sl1}turN@HRU%ALB<7$*T7Z7XL$NkSxuiE@ zE{dW#Ya|YBlsIE38lWippz*WSdRfQ!BFhqa{^g#Wk)z;up7R*zTt2;Up>yE|Oc4!B zA07zcDeRxmGpU0itD|J~_GgPvJy@nVPl4-A0>76d=ZOXN+Y-1>IdGp^z;i}{_sj#H zZ4R8r7Ko@M@_bX65lFqV0x0r;=w5M{kI;8+Z^OLry%a-$e*LYna0q&D}l#iiMp5;o5O?XGzJzu2IiJ@wu%QV0qiU; z4O}x41j-Wsi6|{#EIXTWV+rFQ1#t#Np)U+&3J+KW7BlV&ns`i4L! z)SK3#*Wvg{Wxa6IL-wN=WzGxAbtxJLKNPxk@$uNb(hHn;NhZIPN3wn%~J*aC~~3L<%pdPVLM zd5jV*j1qYVdHNLieGW=8J&Y?@(Dm(r0D~j{7Y6z zlGVU@tRYA+QDnovg(5->Op|m)eAjYoEaVhu6!B2x(n>IqnQ*Nr*O#wE=cWqNk>?J{ zS5xMFVorImc8$ukKPt=)49pq~EK?dpb~*AVX|k+QpEfPfg?j<>ItK16M)9zRT-Opr z1^6Z69*Sl(dVWY^uMGI4w@}zZQB*BBCRkB)-ocppj#)KVKC^j<`ziANID0?A)s{_( z^VZ?b%g#LD@#C1cz*AX?W7Ze;MPJzW6hy2GIrC$snn+RNRiC^Y7tYqK?h;W_oTR{R z;KK8Uh12YyM^mGO)wG1XhaOuVaCj9-xh>=@Qs}HWEbw6g=P4!E4yVB8P2t85m}MR? z2P|MpQ{eYXEIYFJ{1gSoe@u(_MI2)^IKXY!!1jZIpY0`+ltYa~1A|?H=F&0-s}E_h z3l7;OFgP6Gu6w|Iz(gbV!K9c4D|C*q)-kZ(InMflfx}6GQ$dSmSAs~3S76$ zY6x595q02!09zvW83l=iDH3&w(w2%6If*Hc7pzuL)Z{rRq|zwDvyd~R;g6a+OMyal z=7Chr165H6YK<6}=P0=U$ro65@SpT2h6!D!EDjC6cb-SIvZv}RC>{{w*J03_B*wl& zT9qv*Vcl6nt| zu9vW@`N)4UEI)-@m+DX&lV}6 zysGk~*$N!@9`NlukiJbp_?C&p9YwiE2W1mBh`zeQp>tZwCs9D=3crjNf7T_|7zU0n z36b9p@D(}mD>(9hx>lvtz>t$5^2veg)<5Q8cSn{<57bXAP-pODKj6S#=fE#vAz0+d z!{E4tXW_*k4nhoyJZ_FchZ3X@B~C486exQrtCGn7Z{f7mU=Y8@Xi*?3@~=^bAyH=1FM*v6GW!yn_a7AUuoT>oATH#{@9~ps#ltnp zMw~j0{9hPYGBm||tON@l3Y5>@3k$z&dOb6HeG08|7#|9 zbAmF{Ca=z;-+EHZt_CRPmd~vSx65$lIJJY*tetb7L-#xfo^uT1Jr2B|GAvp3d9N(s zF=Lc)I>>S2fVi(C|FH*?)n;;T`ffd2so0N;-R*&;91z-;xI63SxqiCXpK!2ov={m z&OzA@MY#zFMdK315*pj|8HGa}MISiIKAo}n;X(PfKkb2v9giz#vpI4-56rd8wY}iX z^(C_8c*K@>%n6^}5_b0Fzw$KWKT=j2UDnK>*Rp>5Q)k7Dr|ccK6?QP}EOg|&@<1%- zps1N6_ZA0U6GdU8U1B{4IFpX;aazDy=fEWv%I@G0xc0ebk7!`(KZU^4->Qn9v3u!> zmn_sRW?ZkjPj*IM$p}VhZ4FbHE67B;Oi2frKZPWTN+{M+wMP8RB zFE-zuAy7W|PsRiGj;19Aj3OM3;`c7+GrrVg>~ZmI*x>o)=1-SRioIENkSF>Ytv&6ioH$@(Zo_hQ7KHH%$b1d?{gm$SiZTq^cjcG|ligA08 z!r8R%=8A`SZwmA@-Y#$b{ebt-*_{eJi;6if9T00^^s+k0W5t;G#)f+egTNnu9w6KX)>@PEu!h}hWR0R};1QZtuHHwrd3Wjw3aafT5@241xi^=m^ zJ(e^DmaGTY^$n}tA5=LU=$W*S^OVQZk_P@$9{gQRd2R}9avN8;Iq*E_7AR1h{$>)B zQi2QfY?qBckBVE1owF00z$n?rD7oRHL_(uj#hkX*g%VFBKEJysf9!92`=2}3`gc8} zx#lK#usgM!TihA=ne#|R{L&lYT1wlPt2p*-U~_onRoZd(hEJZsl2Z{Kt!(qkGgFhy zpR;Eyb>Gpz^Fu*=S%XIpi(X8la1^6{&O?r#27Zr+|5$!9TRqq<__gYcHCqdlR?2~h zRENMq2PP&5(OieRCu^pBO*r^BVacfuwuXd5p;;`G4rqkcvK&%icWO`x*5c`D`o}+u zNzp|?!$5LrhQjab87x6u99M$bMZPkLn7;Jn5^;LSQ<5ousf)?$gC1j(GY6mSx`GD| zNug_6xnqwyrSV7}Nj{cwagyt5;Wf#}yN)(LKOetug5u|c-e2EHuJYE=4|Obkb!CBS zaMracMye-8)Pv@zJbbCrHCfGXo=aoXmlXmU-t`=upS~QQAaAqkT?gaiqunCWI~_hg zQsug_-nYdhlj(`hluf?ZoDzjtID{2_>aL_tn$|7naQTzYMG+l|H3yqp_i`P3*)hpx zsn^Ge2OZnEh4i$duYF|o;5$3j@x+9WiY5Z$LRPr^Zt zVsBk{2}Y0Px(5%K*!5;Cm*lf}@}N~NE@v7u-=?)c8M(8V1R9u^FTKX1)Y^vp(VR5H%3dG>UCtxU;3(`jW4BRqKiaB zhoDoXVvB<_dy0w>m*S2em%4=%4;*K#{c_XY$&u^h_X&m{b)@_4MNc+O)ZQRbd}fp4 z+sz40Kc9R)-@MZ0hLF#$nxx>J)?b&f1SoQSbrDvPdg|Kcv5_NmZ_v*GQT~X`rJ-CN zwXGqMaZ^&mCi?#QaD+eCRPrf*(cgsY{H3q9wwtsDY;@7se&-OU`nDUF3N?228M3Ne zW++<0ZF1sU8lPuHV7qMGj+M>amOCCuE|Nd;-~fZ2u!94KrGpU{hw=u2CKkRkzqD6b zX%w~zDla+MDwvYv`0tQd`h)_vRZ*+CI3!beoV1iy3N1P=*1(W-M55uvg;t4}f{pCr z9YG%&D-|a=aA+*Nvgx#0bcsV_&6hxjfQ>&&3v}v_O*pc^{^OaVi8_~pUJEweRD0p? zb^gX@9<>Kyg(o%l)l6Er)57v&VuuYUtF?%#+*h_P`%Ax*dVUHqI*0xFB(Z*?jnUO> z{q9;lHQl+2{|%0CF*CX;EbS>`zM@n;$yL%_W+RJK!v`Y|kqJ|jICzbE9ysOdR42B} zmfo4OFSS+b)zR}p5fd8uIy{7$xF<{%V{E*jz|(GVSb}RutIM2+VcaGzs~8Tlhjccw zBnvbPh8&gE(^zh%q`+8SGT~pd;FV?8FMl(zIV{kS>UqGk)q~|hm17%I$pI#&h36gk znkM{F&=xzsk55P!0Rw^;MT6WX($_Md&596isw^F>qfz@1sSt87JKUeF-Dc%cenJ~@b+S3I8iU{F%kM^i|_!!4LIIK}* z$gI|pD6r>(ui(6}7DI(3@hXo)3SNyI(iaY}>dt87e$&FNpu)&ew#w)11#^2o4+VB% z35W9!oSB^!oLU!XHZVIiA2N91$R4A>%sC}M>vNC8bmbQeP7j`^*;Q=+SKY zv|t0P1cxJc%!fv)9}TSH6BxO38k+wJPB=R~LixZ+;p9I1-Bq?ro zcx1Oh)l$*PJ8H9rfY9&8wK+czJD%U+{$k#VNx^TJN~Mg%{m(6R)ytdM?bdlnTISc$ zNn)PBtrv?Q9h|7=)N$kBl;?YPIq0zKusTWbEaa9lc*s3z zftyms2WIQD57^YCSryn4xjYOOvQFK?IOo8DR;#N#9Hy&h&JI|ynZsbg6c&jFRxSr- z7m-EW3JuFxR3?~<9AdLn66mj(q1F@`tH81_Gdn|%p=s-`1x)e>m001(mo=d^u7OGXt)=rrs|_ZF|L(Edt3;^RPq^q@f53R2vrLce#2H6e zH8teFG03*e^RT*H+oZN?vrBN` z^4Bml%Vi%95xC{a{+YY?yw6JpZUG0DDu)&!tf*f)y7J#eHECkZs7Z;4gA6j$`CL zv8dJh<4J|&GfYuiB+dw~{pwd+rl-0>b^?ICAk<00q# z)r&-IRm(#UzTm4`K2=zRN7t=LIZB3Yso2&PJI@?loIlfGs$PqtU~&SJ@T3Lpfop`= zohuHB&thQnR8itzrok$w_L51lmXRl5l5b~r0Q0g-4D9lJyb+QQHvfroV5xL);Y+*F z#BHL$B$#lJJH|oznkVhUaW#eZp1w2?seFB_9gbFx2X8 zIAE9=z$}}hD7a%qYnL!%@0el)xO9)V`h)hBgTT-?@U{G#Q*6!XzU1<`nIImQYVS0 z{#~(RTC>_iMxmO5Xt_@YT)}r1u$l-sD}7tQ?%3@pw5&opmFpbq^_~TjA5`|9a%Pb7 z_D*2nDR5xX31AfLcqrhv;K>^ITwlop3$C-Ysj>^SYHSPJaj4LDO7q<+#<=#V^_i`& z8VwU3PJNlcvp~T`sp2Dd#%Y5FK8@wIYie7wlcR6jr346AbLWLFQ4&2HH6cyXU~M`KKrhqrwc=ZG$ijNYSmgC znK_}k!+s%OCr7kmQx=z#$U+Wf1;;XFM=pDxJt7SfjyZc8uJ|4EH~S~2Q+Rfl)PoBQ z4bKD%9N6>>+MGHL)-Xxq}QE z=>oITivF;#$of`QHL-(9p|6dxXQSh#zE;WApA}2hJq5JQB_64jvrlsIG!at!$g^Bk zF;qZ!`2@D$30Cq#6~PNw)_zXeWXiTVG+$*?#m{eUe~vMyA7M7wz^ot8#Qa=@eRE|d zqv5RQHu@Rje9vv%QpJ-LI2r_)k`1_G+u5QD*lr!RZ3tkuPhfv+%wfKvXoq-w*`Xl6 zR_2%qEbAs}Fg{~YFev5`U=%H2lzhN4L4jklBD1ht;^t*Wk_8FR3T3y6GyOZCXmY;9 zN0&j)Y69Z}N5%~oixd(<+!nK(TgaSQYJ5vQOz}x!=K{93f!szSkq3kPR1}hbwCApA z;5n<*>8RqrSs-uv46cI=Q7Vi=zDLUSC#F;iG={badokp5r&jQrTB|ojM}AXEY0>}p zEmd`eDC_ddfTA?FM>YYQ3WPp0SE)4fr-$f#jg4pEnDl^Ac>`iD{U|`R&Y}K;&|tyOs$9aj}`2@FM3!WU{7i=UjI?b$CXX@iPLU{f1E9c z8BJ20EmV>tCe+_h6O>t~b1XoRL8AQOG_Ioz{F*B#{ePgKA!x;YqQTJ7b=_4(g(lZl zan|?`0pZsgqs#(#E!O|#)T9~EDS;oAj$=K|0$mU6Q77k1qLQIAe%o(2<3o$VA74(@Ga7=7a zD>%t$p6+vUA5h*xnhS#nz(Lw!k-O0ZWVlo91yXfoD0yd2u+;(Ft5y z|7K3ot4s+i;9T*5XSqVx`)k5})8?$Oh+Z{AWK(f;m>P?-L!fhl=)Yt7TTgU*zHIV1 zIYs;H6rsz^I!Eg)4zQRdFnbEHRu*u0Ca^>Wa34{ae`LYbZ3W!34^;UGOl!Kpe8P?C z?hdu%PdpWlgg6+q@;zXf#K^#Tfq`!tqe%eABnJDf&wLaYurw`@=`dhZHsF}B)6A>D z;p9SgaT7L=8?53>SnhxHUc23Uw~9~ZR_OmNEc1XS%gASc#2@oQpP#`OBE z>A7nhW{WX$Z!zFr*Dzbnvi!q=*_<4Teour#9hTNjXA2T&Y%*_TH_Z<`=DLYFFn@tk z@-C&IiTeK{KBoRTG}n`}+2>^?|8(ZZl2t;Jnaw9KJ3pB6x{|}YVCn&d`KO(D_Fv#! zTfiM3cy^ zFM#=;Ia90y>!}^9tTwQof2gh0B5ROk^1`{!_(1>99pNSh%*FxC#s_A)EjH?Evh)1F z8n3`s(XcqBfNjAB?%fxX1V1FD9%p>gB7g1JM46p_zn2Sa;pD#L#J#O@_6&>ir>`cR zZxC8}Kq$;WQGX$?(L$@G8jY`nQVTB3*(%Js^JrHzo3c@ma^@>*kDGIE?+kKy!2Cqh z(9d#td+6fADrO&nf2>Ij9MKzC&;8*3a6o5!0^7L=ZdU`=Y6Gs;2`tB5n6#Z&S_Vr_ zUS+pfuvlJ!(PF~_A%}=3>Z{@nJi8t-Z&MF5nGj}mfzjaCqU;@Qq3#-WtZNiLOi?hH zwofX;<^!9t1GDCIJJSPd>kqP4G_cn+uy-bK=!&uJxWK(pDc3|rz_6{%_}Q#A8@RH) zm#kCJb)O|THz8%lgxSxsw!g4i8WP1ie?m$#Lxt#$rSg}S2AWlEDHd7tgC$3SB``5C z?4q@S!xYU2O?y^I&QdhATOi3ko0o-vXIF2qFWBIh8S=Q3`G$(LTkE`_1#8kbu+BPQIC}zT+61Gh-((>m(SbaIj5(!dxZF zyf}y^XH)}=#|2IggM)%j+kIypT=Bp% z_|(DbYOWg(w#z(9i8N_c`IB-gCwkjak)G++8#yXB|6o$vz-;%Rd*6<;b9Nt|)y-^| zaK!Y(6z80X7n#gf1*pxo(=})^%c{& zo_X3WPmq7WqAfq{PlLr^ON?*WD#x{RC+ zt=hVbOOG(jFk=vOxHI>a@mwoLo>L4wuNW#nuv)IY{8N-sL_x0e0&j|g_Pm}2X|oS- z@+?^wz`bq*XJi3Ol*5Wp+pE(JuFgFmw4y-B@5v*6h6vht14bbRhJUO9Ud#=b=AOE} zT$gvELe|y0#>;aVMG`n}A7B^RknlBen^Yz1s-3eU8F&tS;68F;>RbVycM~{03Rr@3 zUxf8?erVwQWLwVpgXdhs{V;`>p$s8Q4LD>j-EUT}_@u^Kc;V%@YiE9H%}x5%<*<|a ziplJg)xY&}BwIKe@KZcOJ%X|!+mkgN8D*N~f7(Y5PvNCYoO5k8Uklaqz!KklvzV+dLHVEaGaMn=6I zn?frJ54f)S!NR^yDQT9{QomjQu07cD<81z-omFqu?BAblcRknicjqIYb2bW|<_eyc z0>_&KSd$GnS`>_=nT^sONXI&`EndLA%z?}7fJtoW$|ZZVraFd9aS7QV!YGu`Dz@N) z*az0j0-RR13`_yV%P+F-RN#nr;A*|VBI;1Xv*FTvyJF#nOyPj{90Bi-+7>eyJToz8 z42!#Cc9(&Lfpy7-_dl<`J<5JtM1bRV0>|fr2~3GsjvnfG9-@=-fqPE@%Qgp|nF2g= z2RS|dC9t|1ylk8ILi^xHJx9ebrI&mAx#l~ZF%>*CORv$N?LpkBGoBC680xe9Qe(-v zuwJj`)dcq`Y60r&_`W#gM{I~w&uj91l=H=$A=5U1neXO9-wW}T0_^hxxR!t5^k2i8 z>cAB8;fvh%uM-(q44&H`W64%Hz;u&cKrgBx!*D`dH@`wmNJik}C8V-YogA7xI0;~cug&dpM1>)=${A@Vr%O)gY6Hrj>z|78E@Z?nB z!iHuxab2CI2hwM&^vLQIa0DtGVdsmqEV*#1@kqGx-W=JFUr$f7PCEKPapR$FUZUE$ z`kBICUrMeD{&z0mU~yP$*VO%CpQc!uW}HgY3KG!|vr#=IqFK1arBnE+=M?i9^K7B}ty14bUSN-#FogLy+`C2latJp|Si_I`~mg98Vg;k6R8{gKP8YUL zCM#8ji>^EZZUv2;f4Ckt3wXM1R1vt7vO{Yxzuf_bX0fyr3tI(ZPCR7SFgVS;--4x( zZIVq01Iz575(S1s+!Kl(G}$UNCbeLz&v3=<9vw;1r1z~VNU~0da=*~H-)0JjvY@9z z;Bo|7I!u-$~e^L&eD0P&34I+YO;twzdEKZJzf|F~dPn^p^R*B^UK~ZGW60@o<6Ej17nT9Up0=N*SJGXw4D` zOJ@>FXJl-szQL-*aFA8TLxD+9;Wn#OL8I7%hunG_9Gr3lI6Pcbn6)M}DJ&4?(7Kb) zBXy9&?b=rsk1LEqOBWnbju2qTP1JS}+~v7lt$gx{P`_wvPX}f$0|q7!hZvy=j16a| zP7w+^z$`6skZsxteXbSDLq#|mne9HTTRFjjMOLDb-7SDk?d3WSIf2Czk1rXR2(7O; zZX74$w~)KvlhG(B;E;@@u&ri_8kg4UdvpF+ET3M>mR3JUSmw-hjr%NbYbHj?|IRXC zZhdip)$oO*h}9<+)?Ow4#Sff0R1#bTt}qDvEA2SM#cbH&Sa2}-Ri^_d3)6Z5lZBk} z1r3!>2iSvZ64}fSG;l2Wus`qugV2=|&4B?T3piK~usR6bm7Q16DDh9FsDJUYCrZ~F zvTrV7w9R2)-H^GC*Y*L6ph5#H*8&%QjYRHL4`#`#=Z&(42JV6pv0+95rv*9=Sb6i@ zkT|)6QMu}?n6$XDCpEPw9%C8-?>%M2Gd@?;4#OFk&6eN@=EyyRAkEyp1?`Gaoy z4jWqqTnt%^6PkGEF_?RaOcn3sILx6}(8zH`L7?j2hI1Ud6xh^j6b0KJI0-2!a;i%( z2{kkvk#kez&=O&iZr;HWbfQ(4k)~YCcB7nl>_TK z*L55=3s_V+?pm5JXfa;lC_Kx7S;}M~o9UE9?hi3@nO6n5UR}Fxx|D#UU|xW%T9(5? z&i>Y}v#~nU(2Q&7U4!VYPg!S*ezJS^k0J3eg2j3O0@%MiK`( zO{A<-)(b!x6rim@~M5~I& zMs}M!>qROh7!_80Z5r6lt~$u$eU3%y%mP+pjhh_O8^T4`{$UjCP+(R*;lQ@(&S`0r!1Z-2O`(7fd*KI8sWAKYlQy?2q7n?Y2-d;*8Uw*@TH98A&^cBubzI}y5Y@4Xqv zZWYZ*w$Ge&u!@_}^4dq{Z6|$a^C+l2vvOp!N?2gg&Fl*# zn7CUUN+mxu*tuWu7Ti$AeDq?&OxdVtGjT^&^NP))lhXIJJC|_Hzi>?bO~sv~IUzeQ z2HzC+c%}A|?N(LZE@zo_tM9u0NEGai_<1V1f%*NRJG&q8>igxMyEntAYOmSzl{`{P z!3Hc!oJtl>l4gnB33r@cZVxyt*k#b_){`XSKJ|l`n*bAU3pu~>!28|S<9B)1KM_mr>B^kcCH&xpBEgX3N9p!Cg zNqMlzqimDdgbVMCHcY;ocA{m^T0hs7FRWSCzfwJ5{eRW1Yu74R^(|aP`2w#jzQ+~x zCwG?ryYKJS{Mdf}g=@F_r5CCC{znft zNo-lL-0H!hjZ-JEn*LFeI4sb1DZ7x{S0?dMt@RPf3k7UT!VV~j3LKO;w1HJ`f+K%C z0}J1m1?&bU+U$xSu9|RtYze%;ntq`zP@s|R0*B;ov;CK?PoGrGJH3&uVIl9y-E#$| zPyW%#rD3G%V5EDZ#mIrFEJ^5Zl=`XH4#p167a!?so!ZWOfPw$RCY}Qf0tXm)7dT4s z*k0rMXDcef*sj&rp0wC3d2g%%>#+vg8`hF4M;4zBkhJ*GWd38v9p;@*m)G5kZh64H z&ud1D=Y*Y)H}8CLd&_L6eJ^HjF`Qx|Ie}69!%oQt#+R2HRSa4T0-B6AuqG<7B?Sm2 z88rJoVD(+Vw0*W!FhlFcS4=z^3>+N{ViJr32PO(WXqHc4bKg6iaXxXJC}_XmU_!=4EM%S7@u~V6WF`*SW#u$I)v1gN31~CF%id%mfy@g2phd z#;}E|>;;V0KN`6=FbHmFcKX4{bAp-OfzeT5w`4%0#SFHjL?+S0%d$UDoA*QP$S$>l zi3jo*&dD|AoVBA&pqJcX9k!i+aRHUx6F`ZgH(t}vHbs8q|`$a}%4 z_OZ>m)s8oJIcAv{x1Dyoa=9Vbgf-u1owA6#O?A^PZO!5u<|#%iOJdyZ6LyxbVetxJ z^Xgzroxqk^vGZZ{nSc6Iw!Htevo6J1BI2;g6cdpIXMqchvK%ai5lkilR&71((E-fL z7nn>}usT_^?3>79_@LSNL$fUdlbC~xSVAMW2gk>ZsbUKlcdTsWabS#?$>>qgX0xEl z_5xeL6ef0ox#`6d6n;(P{2`{OrL(u%VZM*f+zAZa3JttBnEWHmRW7iUSG23XkiW^v zDDYvC@`~2d)q9jVB(ANHM4$VHL`F@tl@jLFdE&L{b zjz`QqW1_;SaD&<5MU(N4*03A@TEZfljW~35Z?wc;Xu0;iqDGYm%{CfMMmt!eZ?yQnXfeIP;uml& zVaNJ?imeVBERGB;h72su9xM!9Egt_ASc4>33shONX0&B4XyY$tb9^!T;g%_Z7uxhs zGIuK+W!+?bdZWWh!9}8<&1A9;u>G;FHgzfbsltAPLHWS}Ee_WD5AAUa8o$1n-?*ni zQlioDQ)_Byvx$T<_l$$w9S8MxYe`&b;F;mb)4?Gk!1&;azo^*wqRf|t zn7V>$?p&;l;n|_Z^P@r7fS;#-{q$l9Hc9ydlEz2$jo<+I{_ zPnGZfSlR59(Ij|4Cwd3#w%Di90<6Ijdb3N~f;w6>PB5xUG)hY_tA9{%lxWs_(ezWe z*})*(!=X_+fl1^A<4#eQuFI^s7uvYlT5Jr=92i<0UvQ|@NqPL^xc={j^la1cqgy?* z7qk^FV=G$LmZK4#eLy-#p*2Q>l~IQ^(vF2inZ>kVPy7#7{~Js^QM@YU-JW)mDw?QZvqFGg_40b1l91{H=d?T3m16P01aNP6n+(4LtT?JZcZ( zd`s6kwJ}fa3J84L?Ap=xJz1-;EKVJ)5+ z?9~_8XDxd-o&u6EXJ=erDkuM8zxV-;FJyDSV$;}|kuTxg9w z(Hb~`#q&gKr6G&M1xAMnElw+L=H6iAxZE6bfz@TgQauNO=9a~=3@ly0SsZ^nv=Erd z_Fq+VMPhqHzto1il0O)Jytu;>(0I(L@rbvL$jgNS=N*_TVlt~1FbOc+I(gJ5;9uBf z7dKZHMzIKeaSaWAg%+DLET#*Z_)au%MRoB9?B}*%6tQT$-uFT(<>rf{FRadG^d3tV zox$kHkZEo7lI1Ut?P(r=U(G2=D;I{Qcz3W>9bkXa_iom_UX`PnH#!*S<$ZqGU?O9% z-f!8Kqcy!61}*wGHZc5Q(UfQjpFB%dkJaS@YivWl%Y>GK8*R=Hn1n7gI}2QjxzSe1 z*kWMO60_i%?TOZ~*HRuCOtB1^%oktWKf>_US@6i`ZF_!oF1Wy%^KjaNjx{U|49-2J zM}BW)>1c>Ju9oA>#lXO&daLPd)b@l6OuihgMmHLkIyH(^FiPxbYzR!a=i02s^@6L2 z;o84r49fc3&#ld!Tsu`Q|C9BxDH$YYu8QfVt!Y| zQk?PbyXj|7vr94$G9#b9G^w}z=_}$oN%QSIO}E=7s&5meHZ}ABi^m6MhXpMgg_vapnE&uKEAqDFiZ;7&usAS;J2EitQ)IR6aQx%f#PG9$t)oHm zQ03{9oE%4XGj=fKH812|**Uv`lk4a7dM0zJ}IkvKR@b!d)wRFY4;>&+GjCDvhUQV zOtxL6H`)96ziC-3?N_u^DttD&UjBoX>F9Sxg$2F!%Xv-2e|Wunxk`4Q#qyP|hehow zSRQJq246^YkhtVru$N7~Ip{!}*sW%+eiyrhCOwNsGBdga9xypCVBET>QS1cc?IR4o zGa4iVl9?D7MJ6^#7#!hlXyDeE{&G(zXJ%cYk%^UuM=R}(|Y8^i=Ue84H7z(IVMeU zSXR_~YO442l|f6F`4)$*y1>J(p!Z;5Ap51oR*FgiCm1z47rGnkg`BWnd~UHkpFzro zfP;rxg~f%YtSC@oY~kWp@aT}4*~oHgs=9Z@f(4(N&&}hXlA!a%vqhn|j$_ue?EdC6 zyjm$Y7PCVjA%B0pKu_1b4-EXCDt0&L&$?VJFaGc@X!rx^)$D;8O6WgnR^@|&< zJ{&3$Q8%cO*Qm4M64m-w(tEh&q;tFA{5_5JAALpo3f~=h$+5{_-o9_q)y4f@+Pp7> zyi0E+wu>8FWI5tz`J-X7za3|2$iG$x*;WxgONK^v71pN$94cN34~|q-JeasZal#ZQ zU(JaokDS@$8xFYg+HVMDXXC38Sj5rj^64-u?}OF=%~GH8MpnruUmaQ0O~h^`3OGz~ zND(x+a3FPJnG(y~Qk9pw2(X&@y!pssdf7#wkwxHus*_a4jD@U1b8{BW zHaITx_?+QK5$7{%Q41y>6OvL1Z=5U`Be1euLxWAO{Ay~uW7)&b_C|+CoFZ9VT6-D} z@l-l&5)gg8_4x&9pCjoL-K3wk->`wEf#{KUmFWA6yX-ja6#YdBco5Wvz z2;OXGB6>2`q$H3-Qo3RTBb)M%hsR|X_b&S9tC?f!$jqT&@{w7fb<5Yo940ReJow}u zJv=l?A>+XjjmngO!#q|I2?yDWofuSE+GVHRWaiAc=`dqL>9okS1TVG7bU_t?2Kglm z4zO4SaVRipwz_-(&2INNOg3)N_O4PT9rv2^%JJ>nEY76vZk^A z!^d{nyz>WJeU9C7apALmxOUAwH_=vS;TVTW&8)YlZ9FREE@RguG$CLSkLJ06&EA@e z7o2F9Yj|_9Rbe8_CU*IT4~ZO-Ne)6C@-;06hd5Gr{xO~4h?SVcEq})Jy8~xvoB$(B zRNBm?hohVX3a1MtL`u#`47%pPRPWK;$b03<#)BePZcKD!7PxaS;#6ivP>kB#jKDjm z=hno}k@WuH(^yn!RkHn7m)mvq)FbFv-wX~w?S`_QVA1Bn6 z3wChGJW`m{Tk(XiE8>XCtww&vm1kA&K05sJheDfWMWULy<+PT91(WwKG++<-aY%XA zf_+LkjJ&f2T6LWo*_8L_2~E*xHCvX*6~vGv&?b?j2O5h1cS9*&m!q| z3uc821spLCoP=&xu$!`pE$7&vz@(Akz-;iLkuj!$;regcY5(LN7_?TVF-)sGz;I{^ z1DnMI&MPw<*|tn<=i)geX|9m`P2m9pSIPp8a-(KRvCMrw29qXQCx;}+2rL(JIA!3} z&BwLCS$65o^9GYI#jjvfTD9UV1D|eDTlIs>%%(!mp&A`BcOFS!g8|2zqt^GlRJpu9sM(RL)U*423LKFbmBRcbL&6kgDOlXjOuYYic$#<7NjIoydQ|9&^_i zbT!78=xkm2vStCx<%{#L_}siF^20wX|3{ZGPlZFc)6}qEqO;w9ZRm(T=j;%Bc7fu) zTc`7GCeAR9T-IC?DF(|8Qjak*WjKhiFtRWj9F#0v;gDP=E@>pD%Xj0$ zQT0^;ZH_#P&M9A*roPkrGRKqywG&sZ)M%Z?v?Q{N)&6g0^-|?kpAKx_>%XGK*lpp~ ze`{q8R&Bm@!ab|=s=dY|d&XlA^uj-Jc-kB~kaYF1jLVH{^DK6(JzcXkTKNmJ+>c)F z28SaY+XA{11r8sU3u_iLOYAnDb5P{wfwu5HO1!ZihqZSJbnAHqz73k-WZJ~Q$a`!7 zi>gi|PjrA2pOpi%euDxF`&kCzV^h~_Oi&URRWRaP^0E1D5YHK*%|Y9}F3i-iec;U} z!5}Goq1kJXB4a?;0S29!s%G;OxR(B4F%>Xu4X$w#-}NApP4B{;cU}s6O?mZ~G#tzL zka22x*D-l@Z_QPKf>T%625^MNb-wiR-`e;q_d%#EN0z&TvmBG&)|G*CvzXO8W}L~% z_WKkzIpUx3=9couOmaen&AKg*#11vITh}DE8Hwx`o2AflHAaM0Ri#BFSkuLUHL$}* z!AYQv!A0@dq{iY_1!koahqq}SVC=W%Tc)PxP6SYs4;?GBXf+^s9e z%yi+dy^WLKs-)Zzp>1mkSS+kqVCXvnTM~mh%hgRO2Ly{pU7$&o+Z?Gs# zU={3Oi3*i+E^9T`N*i`PpxC z!7O2`r_!HE1};kbPO;maY2=NVP=?h%_jdin4Y;fsWdq7Z8^a4!hxfsf#XHP_L(9|4otiv4)0tJ@LY%$YB&%d zz|LhN{_zE)Y5^mgL8H@+0~2Bx_(Gawd14q$WSAKm@*Ta7IXEhBVR&iCaQNXxhLAR9 zl>@9chlCRj?AgmUF_ht@tE9*c2R4>N5|B*&%8%F5|b95r7GN;~`|HE#}Bion~ zpy$&3V``L#0hZ>0+Ld!4$Lb8`CJaNTR1SdNJ?-emkTm5MlkU99AF4&U`W)T#h%A!ZVtTGn$QO9Mb4%(hFeLe89+7GJVb*o+T{4j%7lNU;2Ms`EZGX ze9h52GS?DCb_lry+!sF3=xQXe_5`Cs*Ab7NmUS-94KFW!-`NuOz)7iX{?En<%`T5F zos4Sv+2Xgu!MC7M)}u+d$3d*cNqdKrdI^*IjV1$*!}=>6rFxh&Uook0G=&%VsW==G zf5E_O!l=;0WP0PW90SvHmVY-nmYfq(I3(HAAn}4pPlJ&)qG6iL0r>|rm@^i0d=Pw_ z%D|n#z!B2GVBvmvD}xjx8{-dejx!9r0uCWX(R?)rI0_nHsT|b*bWp$TU?NkKL=S_K zf)fWrlO978+k$%v98H=L&Sn|RU5-v7Qx;4X@MqC*TOz~bH!<94-NTPNmwj@*b$Y1- zuL+YT&jsNJ3#FeZx@5!mBH%OcK>4U)Jq4W?w*;oNq{lH zC!Lp}mD{32{6nM41IJS<7|;Be|DQ8ecZFlhOD4@e*UgSiSv8Jq0S7p0813IV>CItW z#d6(R;*f-bwBigVjfO*J6POi58d&~C1RXr#z$)OtaiM`Dz(In+NxX$oX^oQR2S;&_ zLvkICdJ;}T39OPmOnMECdO6K}PngU;FzWR*xAq-YlsIH9(4?}5S%1Po>n}_SHO+cw zURdWG)cez{*};%Aqn+!?L3JNy3m;~A!N}}{pg9v3Chj=D$gy4Op9qVn*6TQVo?*TvaMx=_ic*-9SC{Co&Iv}GN7o#U z|8w*J=hKzGHx9}@X?m@~D4F4OhQrx_gV~^^NveiPNhgv&@bK&&r~+7aLD?}A;lGk z^$uuQemH2I)2y=Mu*Hi5);-QjHHR&pXh=SAvSc`LzB>*1H3Gm6T%*9a2&AyHkl>nz#N;-gBe0oa)PquD(N4oZqRh?g*OmM}^>Jdk+Nz}L|ztkEb}a*%_iNv@|s zuBVY>%4DGnizGA{g^w`GyqePKdyrQpG44p`o~H1`E7^&ELj0CG?Ee?SI-}u_P826Y z&Vh9dHFF(!S)B9&4$JPjG<~0=Kt#&di7RzleZSQ{QBrZX{^Hf>^Tx~Nu%63dPp?BN z9!~nbQ^i-@Up+DB{sf)UTStO)WCdO{syHwSZ#bbihe@NtY3{?BA{X_-6$5jzqr?>s@fQvfDvrD=jj|34ByKc3dey*d!nlh~k2j!^ z?@j|>03+|615Ni1@KiLgbUCo~FjiY6vtCgC738dt;*edi;GN7FhoZcLJZGP4eSH3C z^?os)1M?bQPGjI^iQ@7pTNdbWmo6~IUZ8*Xwu|q(quR!V&L-g(B1SfDW#fL z%}GtYu1&o+CG=dHa%VMx?zs(FArt^Q5SO#Zfqg~a@yd`arGkHnc^ysjPTRFMa4)&d z-qN6Rx~SOndf8LgIF4hJbUd9(W7ss<^=CiScV>-AJHYYdtw=@#8xIrHyMy6h9GFrZ z<^*tZiTDV{A2_HpE6Kr=E5`U^;7eiWk1yQ}xg$PuCKzf4G%H+q_r|FuuPaj`KAw4E zp{{QL>qTcfmi68u>x-h8l+6~&?vVPz)_Tuny`Kzga1Eop&tcvh4k97XG!&X5A`*Xc z9g=HkHB*_nb>^cTKAVD_n6$*0lw_Lhln(W@FtB+nkeI;8TGGI3a;MKEvGm3!R+DfJ zj{`cS0iD zIK*>dq1xi7vn7vBRcny+DrR$VDASlS)oo)^Oz}aFk2Urmx#t|<4tdG*!%^j5O0&YB z19Em#gO46})tex3IjG>5IP1*ARyUg0zd39tUG_=sefY(X0y=k-k3T7{%W(?t@MCZk zaX2LB(x|L_NMen;);n&MH{OaZr=Ms@c27CYv%-YqNP)0~UO9IWp9q6+R)d7cAr23R zZ!3T3`huyAqhPyW0E1HK zan3soTn&luE_lvPJy3bQ!D;GFHjmAwhrd@pUTw}E#p+S`(U!CB_P4s=!*WO7#cp=2 z;i+;fl3M$~T{l*gm36M|ljdgeZK@tSe)fho**=$5Hj#f`cz?zOo-Zr{8!p_Q>SWf# z^l#qGLz)_oMR=N)_A(~;9@a2%TCJU|z|thKq)hn54Ld0#B^RfiEefn147#rz*lP}0 zaLqe@*J#r9Da<^Qje6gUrmHOo6m;hquujV}|vZkeWRn2^v=m((mL!obSn zFjd7tFsXBn4JXSN2jvwGGB!?Z9*&wT*j4^8F4bt0oRw#B-!DqtZ9%8;v~3KlzL>wg zx3z)YvF6AniDutL#~YRY98{UnZ1ZHZ-Sk^ajFsfCyp&lZZ|wU#`MYNFb>8;J8@cCP zsAs<-e&M>h&OsrcM$w#x&Z&n)Oq}?A539>K$%!y&+BoS=VUp`%C^Ki&3vf1n;Uu?! zOW{YOO2@xO?#<^|UNcDCXgto#>BPx1b@2gqiJw+h4p(n&+ARKJjU3O-^Q$JaZ!K}( zscO0~Gb%Iq`woSUWSCQf>6jLC&X{E5q50=a5{(VY!Sh$w!v+qE~X> z={sf?(4Ef`5pCvHr6)|+uBsE{JVKoqN8$^jr4=e z*Zno~_t~Y)7C4!;Sk|VAp{0RchEb?tu9!?adr70q>?X5_W@C$@&7n`{i(7MCaZvJb z66=_sA z3f_I`{QSIsk>|4;H+N58FK=J>dqvpREKcn(yNI_pJ~FYe^GRAHJx~yOtTS;##>rK> z-G^Btcjm|bSh1OX5-Ydsw3J6Br>Cur%IfP`5Oiwhw#@1&4wpi;1XMDs9atq?ePFzBR8|E!VZPTMn_IwxsZfMI(;I! zAtoUqjGU9YxJAFs=ipRS>393JqDi>v1oKLJsmKk29-j_m&vKhFqqBEb9+#8)+&V6< z6&4Q+xE9%qEo7Nh`ma(;YjwyHFKzE14;HVsJjBq{vExJ+qbwj|k8xE~Da|)jcORqTLJl%aCgG;|!hJ<6g+LsH=6KvTU zT?7Q(4zh_ju_?BQ#~r!Q*e!4*yU*%g#E!Mw9;n<{%gEl4t+*vE0<#K52f4_cW1 zD!;Vv7kcpCEPdHVwT;?aFRINBVrg8mNN&c83(GRHADvV;vv_i0h26i1EZM40jjQI1 zhKeL>>m+PZ?pgmtDtpbQf;WZkGQ~O9S<9cU`p&NLO`?@W`4mU$jKmJfh$+4_HjmN=jcgoB8jKtv z6)Y3omMv6j43avzU}8{+!pf%q3_gmD6ZI?@cl`Ui?hngB$43Gw6Ecs;Efw)d+jQYj zMsCt!bqg~dtvC`pZnfjpOPiMNYuGz$*7B#PnyG61*PPU5l^BOLB?7`$1EWq-4!lEYK?Jt}- zjV>){>Xt6)VCq+k(%5ogk>cNdzwavaTD^KdeR7GJn8S{SRxja0Q}%lZ7ad?zWngS! zVid|~Xk-*xA>|<8;oztz&>one$tt>G_92!+?a3()xdIlP4KAM=xECxCNxR%2YL?K- zvm!z2vP6sKmut+zUfKnwFF7C99b)4@c+k1)xAWs0j{GA3TCT34Aw_N=XKbm~~3m*DPF<1DV!5yg-fww2w6A)DXBfh}!O6N`!i z>pGuCE`<$dw(N5yx7dM{ncritM_qq+u%J=?h=vj6g`rEgl^MFXR z#OwfPwKEqwqgNc{XgI)T+OX3)wU>ciNy1U6Z8@{-g$B0LH%nwcA86s}aA5Inc2UT9 z(BiBo2RyR&`&qDUC07IkvwX)(cDsT`eh2quLLK+(lmZ&1UL^3u zE?{7L6u~UFMIn%ZMSzP%<3MPr0;BFjey&T$>>-fSWnoVtAZlu{4SXgd&p`hc?<^3Y1!R4^jwhgVe zNtU4&@mO6K20?ku)NRD}dP zeycY1FFwJhw4>Nj`p3aV;tY<8#{)PV-juO5COEP7xW(MwA`-OUlcUi0l*bddOH5`< zU$RC81dHupu$tP$P}ncl$kUU_;wHc#q7=xP8N)1;zk`8Uf`O6KX2D9W%2YO^539Ks zG3=LcaO9nFfLZhk1E1pyMjoFHlOm@*5^j+=7IC+bL$oJJ>J-mH2 zV&Yub=&=1E3(uJcOr{eW9ao7la?H7OiP7iw`kYKBwLc1_{t1VK7bv_^Q(4IAGlMb2 zw4ur0A*toa4_AG+g-q)ox=Y<`cxGIwyEbFvhIIuAhaY9^;9FkPyLnI zo!&8qNh@`eRdywKn3gTt_V^RCT>Ar#8LUOxO?S8=mnU-iIj~IrT)?b3L5ZvE1Z%dt zVyEt7CBfYb+W4Ft?u(t+%v!l2NN(Pl78wIYE`iO?yh>Z=EEZgqwz<;c9M@5XkCM7f z^8Y66h|7^W&cCB!SxX*cS!#i2&wF;3Z_>@IOdIUKMfKXgSB+x$JGmjok%86!ha+#@ zg=VQgci2p49OQ{HXcFjQ;LizS44MA4MUx?^44yj zD_sdPb`pngfDVuC21w_R~~=3)iT)6PnQ)BIZ`qNFc!am?aW+MyxJ!pQL}&V;%2 z#9`s00?%hoxixOZ2TIp4&$;hV^zHIrhD#0xm%lbTJh|~Vm}Q-F%s-1n7UcyFtO6Gr z7+rj)vN--z)nbmQ<=AtOU%Z2nOGopQSimMD#RZKLa}s-{0zw`AXC#0Aw1Ckuq1kuo z2{&Fn9^bXczw*Xi4~Xro?JXdi z1Is4{LAHh5D;}_&Q{X>wg!7XEdsYJfCk4K&2K#S|)22B;&RKQ-S^Ca1&#msI+n21~ zwPwTP1&_H_9u7!TWR$qXh=YLAn&eH1Y-Mjb?;L2+J9B-R=9`8l-VG~-)-lT7cqlLT zTyugi!(`q$LILld@!fvsC-PEH#NANDJCM!Efi1&<S4F9#eKIS5=yua!~V zsidiBa!&eX&kv=eKlC10Eqcv-DscOi=ge0GFDzTFkh5&L-U9A-=`80a)Gz2~kYaFn zusWAdp<&apNw?B}M>w!P5?~EtV5@qdBBa6i?E|L|(|#64=5Gm7YHC7j9txD}zBb$B zRN(agS=WEDM4<%>WiK$kRZtiHcXgfof=AJN6xdH?zT5QnaZusWkd11-8yO$>vAS%G zTX29+PLXd0gIZamxI-ehL!)?HZj0I?ku{F_8jOM=+>(hC_}LUCx-anmQ{cDr5^!56 zUZE(+aZtF*QT&qv=a&a;MGFL?92lN0e>h{cS>0i?bAgXD`hUFJWH#@EUUt?lD^3=t zKGr{Lc4cK*h#XZoaq)%RDn=%Sl{b>>uQ{)LkS+gL`C`#xwq5$>4f$*d3~c`b6fd{! zyO*H!f@A-gHYLS{f*Op{RSd7&60W~n@07t5mDg0TW-0%l1wxAsO5HjrWEl9iF8J*N z20fVoQAfVpIYlD*MwT!0)Ni}9{rhB@{NNw^D}~CO1KBGS#hD)R6g0kKP~@AlaK^eL zA`A!F+!k<8TF>mVfO&;OuZsh7ngg?3F54^ypAtpCa||L?j3VD2uuV!h`$mEP#KwRu zXQpKakAE0FHhO23#&02YMaKBtvL)HO61bRixPIJP?zFc(V@JEamI4poLusZ)hJVW$ zXX*c9SrPb6(BU7ua>WA{9ZuGi2G+C#3jPkPZ3+BRi7&Ml@F_6ztq~N`a+F@>BAw7H zvP_Y$f?-z7XH(r2Gfwwbvlc-$MJWwIhPMahgM#E6f-QtMyt8v)%g9m>74*6K{N1sV zqX(u~E>!lpT_PLJz`j61;1dJOyoSgxYyuvN%|8Dg3cGl4@o)(pS+J>yaa+a#<`M;g zDo3GR2|Pa(1jN>HG9Bb(XcS;LDCy29u&;qFpqI_*faxMNORglYpO*sWO<30OSmyLb z?vo$&(hti_%{I&M+JA3(HNz6iDlMid4m=k;7#6uPd5G|eu3Wi^ZJ!bY+ouOB;uP38 zdRY?|u+I6)y5$4AhN85N7UPTsf-HQJij2z2N5MFF`tR6CgEt&W6Oh6d{P^jQ$!MG2fLMjdgq`JAG@HotVs5D`?3l? z-wX$S8#Y4)M;6bFyqp6ciyVbY9?DvMlRd#G&8Nu8kSP7{#6p>4i85AULU{@TGF}T^ z8n})*a5^ktYSUupco~+i&oYboaYOUB7e?PZ*2}z~^zF-{W#4+ft@yCxUZJy~Yw(oy z3|tEsg%}tG9)!RD$bQcuV)gs*pa*>0*0XLq9~rF>#rc*s;sI;L0e-ngcA*C+6%_fF zDe$KRI(=)9Qd_3^SwXSV!Lzb~Jzwplgk4(1s?BhEiHbI;yw!nw=EIzXcQ=T zD7z!^+P)%@o(ArUhuYbV;@c9Xl^of*81uUd1f-5^EpcF&;wGf<@E?0w**A_wOP+nQ zD$Cb1)eFn*%INIZ%j{Zaln|UJ8f>H+sIb8D%yfniPm+xu$jLAG$+Un;PEp9?vL6a+soUt?RC z{lQrvqk$=nL1^Crz6B4M7L+qIDerpm#-?JM<4MDy9y8`2lgzd$>#bH+)cR)#%UJyqBKNM(`5V64;xo&u*DO^H z`5Iwi2a2V(G$vOq7yPk}{l~Q?M)TOYS3Ij96`VaSBlf*eL7hR>-233mUGstyGqrs7 zUbjsBF}*pm?XZc#-g4RZ90D;17`Yz2=t$C+`e12PV9xTdvka{a9sO+(9YnoLRS6;^BjQCd>i&o1RR@eQy5%A%L<&Nhe8ONBM z6gU^@FtarT9#Y^um%wxE0Z*Hzl%nExy|+W9ApFm;#Z~&yVlIjz+Zvcm6u7t!iU=_B z`Y4Ln927QbJm@$I7WxR}H8ANYa4|VnPcXR1v$RI)irJG- zwJcV>s|^${S?sd9a!J7TD8Fi!&{BnK%)O2b{2q+;^VT!WJ7;(FlW>ir^b0pm71aib z2P|(I*drdW#7VFUiLuU^%9^#ny;&hQ;%MTtG69_r!Uu!}%iINoF14(YE?D^TkVN6S zoXfH0#j<`vs*V3Xo3|>ko!hpzaVN_snYN!cYkDo&8V>O7b6|;iF2>X*(RPs8^`Nj# z0&`x1fE%M=lC+4#9pNLB1PtaL%opT2?7*Xt$j{X%Uf?Kn%z=d=QACG}`GbQ%%rUlz zh8-Du%(iu#6roD`L`{vJH_1?tqHU^ygYI)=m!WW!pzU=!l+Q*)c?UsW#hSGxf{i{9T51U(8I7$+wtI`Vg-JM zgPeH|>y#g|=Ny=)``}pcoxbeaovWPt#7nB%#j_HA+h{fz-!Qh~~f>nRteoobAnUjNRXZ<6nd1Wq4E@dw}heH)Y3Ci3^3 z+V#BlUiUAxEwc-LB?vi0w4AQ?{^}_IYvG5l2hV-9IrV+c1Ihmn#HX?H$nA@dDBgIh zI+lUsNCHm}18*H;cgk^Dj>V5ICNR4^;Ng2Hkn&Mjz)^?&;idTd!gh)x6$d#y7Wi;I zbmU-Ulg;9>aT7AikZMxrPiZg@I`!{ijFJIgdlC*VV(I2L&92*!Ia%%OJd@NPDT>a=*cUtX*G265vW#1` zdYg`8vG$DJ6&#aPa8_^w~syX6K%-Yuy*X=+t~t zJR-)Tu(71&GOvP1hk)auWNrnmfDHx;2U^wQXGE+pPz#U~6y3)mB%DlR-aXG9V?t=~=&<`#_kF}cn8XUk<@b6wGcK}LTjuq$}a zP~=iyo3p7+YTb=Xr*vmGn6j!Y<}u=kTCrs6H1*s{p;MaCA^{D2$9f$aCn}~qU{R_3 zqI!b2-r%AG>vh94$N z3rJw*tv(arz);Ge)FNxo;n1uwDZznRr7q+VJBzr&hlaWQd)OSA1VScArSP3eVCNG( zB5+(-bIycDg|?1^-1<8LHnVdq^J(@_VLvNy(b*-yhok@ z_eH4r%%{U{JjQb!f?2d9GJ@N*3nu6W$g-J=1X;A6p2B1HW}~~HcgvxZYOl*Kc^VuN z*xbpkWRuy|m#+BeWdFPc3&phZa~&HQlv$WK_{0Pp8rc*)8j}^9T!ctO~0yKM5 z6b>+}D>NKn@`8a$uE}Mg3!jxr zA+vC!!=ocSfi8)u%#tw&9GIu3dPQf)72Vu8O*Y(ZcIuVrYtN=m@TxAFz3k{VBbMH$ z7hL2d1PTr+*Dy?QU}fSEXjN!62<*~a)+6+kcLrO56YrbPA6P`1{wc7kaHKZNJ!ur- zU~w1cXlOiWlgK8X;h@;?fz{&$ll$cp$K?MNu=Cw}$S0lH%+VfTF7&M-Pu(ZlL(Op# z(?XZXzZ^1~+02C($nZI z)xe^Vg+HZ%^PQxN%(+eL^@I+^XMJ?hwJBtCd7;4HbmQ<521Pb?1I8rd1rBUc2Uz|I zE->V)U`Wb=FIr6=CEz0$=1%Z$x;uR)L0gAGD|S1 zNKI^za!F*u~7c8Mvg1bqsc6qX4ZW~V$BoLq54S>|Jht4Wez_l*Z^?=Q6T-+Rcbz%xN! zA&__P&JV5i(Gz&SE3(LQ995URlxUN^NcTA7MCZGUI{9ZBE#iN%fV1wyL3Xzdti~;m z#O8P$?ONr~%9U||vsvP(;p*PK4eU)TYBQC@&yhL_8TF~)*hVn#lzU+y3c#7 z%daDHH)&tyNJua>>GjQ&NsztSaGp^igOOLup@EU(0sn-G7G9|f;z0(E3_%tBX3Hit z@FW~yn=qqIGbEs~`9~vX-3o91q5~|q-Y(=$7GV7J>=(P%f(1;i6)jxTEVpIOahAF| z!Oi4f*q)?xjl)OJeOMhoRWUi$#aZf_g}Z0Wi~gx99&^v$a+hrrXkH|6)X9=F_vxFJ zYlTiYM2l}qeCn_zW!Vk}j=BxayvG>0a()~VNta;J_i5}6eDg4UnFEV!`RV9;LQ1-O zL|t{aP2jL`VHC=dIV^I*fbIK>?VMsN($fwdY2ZlV3YuV)QqAuhykK9OZ|F7@4S^GyY(f?ZhKRNpZ!%eC(62ldy+CM ztdCwfS7?3XpJM-sRY&Bxn2a{RQ(%!1`8n5~QfqB2>#G}CAN0v4yaMqn*=Eowr=xxjI&A?H#V-^~O*%QRl06XjwRjn@+zpBL~w-p1E( z++5>gGUumc-a`t8BF^;*?9LDJj2ZJkwpm!YR8BT;{@Rv5b+gEVuZ)sK&GG_FmJ`_P zE(makbIljX?YO|PtcAnE$o|Knw12ywM&~keB|C6@TpXm-!0dH^RVjfn@u4`oVXWl` zR@)1)6IhDd+A^*h)*2sozve7$)gEi&8vJIf#NDrP;vX2NwlK3CD5*KYP<4c1Mi3+8 z0>&R(IT|gJ*cb9%aqyYW#MpF!W2FeY76WJg1cv;E`d$N$N&}AG31#OlG^8Kqvufj$ zzQMO-N8^hGzUK?Ngm*Nq@KE^lfxl)l|2KmQyKl*D&)I7fShOQjToNnWk}IdOSK3Sz zKH4EHa>)5|ld#Z&X7`WHO%BWo2UsUA@H+B^gM*PX=mY!w0)fQ}9LWV7EeE21FH*K@ zVv8zdi!ER~Cdd&Wz&dFG_iY8vqzR0b0{=KIgsPeaSStdU#e^L-l4aZ4RF_PQo!%&U z$=LJF)ZpeT3}&UmQx;0y_%78`DCKaSK|Xl6K zj44)TUCU1Ju{>&2$lzD%P*9%Xr#Dey+Jsam29|eMD(oIQPyJlc-|oEFwDPBz|G6KG z6OJ=_KV-6NU~e^G%9CU*{J<7sz~p7X=I6i~vVe&p-E6)AYo!5enAX$v+2X21)6e(}Hh3jZI*RTl%tABr`Ve zQgE@8$5q3LybcUH3z)m6d+Hc4n>q+8t*m>&&d|!%q1>rCa}vX{3A_d~OLQtbZwizZ zCU9+k#-sItC3Px$;e@i88m#^XW_9dS-x%aQm^sDm=M;a=sjoNiZM2xWHDYS3d6T^w z=d=k;v7OBC!bCo>MNOWrIN|#I*`}4+7kZSN6fG2|Nf;>!DKKY+a#${?QJi4uTEG&z zh0*l^qf~)QGP6nZhncOFUVaHPf(_Ub4cG#iIOa;&v?p*hEMQldz^Gur?4!WiB*5;o zKtrI#gg0`Q#$&0bDZ#mm+$WaJx-aVgxYgJ`s{Mg-@V^QU7QqJQ>Hy9O4PiD7lj>fS z@E)jRRAA(IAfr2zA&os-bc5RKV8-SJURnoOzAfinCXf@BVCFc}OHOR*^9g*fC-A*V zSpHzaXLulU7F=^OYQjN zV0KU2IPz}mcE8UJkdCleiCO_=vuU}|9H#+NIXi>;ow zDT6=Qv-_Rpil$lIpE5biCTxn^u*u=VrruX8ess?FVpLkXv`1z`^WP=S=D!(b8kp@B zIJ_Jft7fvXr*h9x;Mz8U(>;Ji`2kCB0oQ~DOv{oILb4N{w=#xR$4Le-R~v9GG2mLj zz~Q@qy@`R-jG@J+VNGdD#-tKe(N4pgtZFmAwO?zp01g)US~;oDT87|d*&ksUv)wvgfz2nXAE} z+Q1;tz`*-p=eKm;`U}jL4hWoc;JtPrMB~BIa|yhf0y$a%%Uou7g-vDb-;nc9GFVtYuSAh$Ul|P#uv8VWxou#b{ek<40DsjRzQ_i)g#m1G4>p`D;4;Z#HdUx;NMM}R zxc_kDR`Crp8x}0;{GcT{fw5=`8rW*?Z%93+=Ll&F-pHGDeZ0O!FQ)7SEOPJHMd zY~?XqC_z25LYl#yH{O7u+JaGGPo0Kk!?_RaZ*OyGc*=4pu=HM#GKYirkoOxTdcgyYI+p~Lj&0+I~ zu75%yl?or`2z`IRzte%MKY^n|sz6hktyq99PJqqnz$S+cE8XJ;}SZ&>q7oYh~zyX2CZ*yn9>hb78pXWrhmc4C=r)t1E~b7tN6UVOuy zL1+SV=>hhq&nL1ZWbt)sauhJ_5X+Igz;Qi*$!)3xzW~GfjZ2rdO`4uGhy8(!fJ2$* zWTrblti2Q1OX=&Q~^zZ*%nW_YS+orO)krz5ET&rr zQAY8CQ?Ji6{5!kH;ldu>*Ly4qE^x`-vPzi0YSwoeCHfk#W%1|`Ea(bm#?n& zVfm};lJBmUdM%eceeR7;x4iYyH<|o96E^)?(_J$mwJL#KSB|ZWf$cQ+v9sA1=Kj4o z|LlqdZ>QQVVCL#&wsLU%m$8M#U4hHvH0Rt6Y_k)%_ZM()OJE6`z|)`=De*e`Ej*S%oPA|YXTp<{U=yZr)_$_q?dHtUrmHQ5#%O3w{Zj$vXi zV5qBLNQss8FNpA1;c5`T$o}Q*%?Yn=HgrA7d|3bXm89Ga$>02eKCidQJqqT$IjLYp z^@2y5C)jo8Z7O4U9GAeJaE3MU4C~yxo4T_epMJN=F^;`#L9h1(mpKhj=5F8&YS|Z( z5WUfe<527_r2rO%1t+Gh(bTKmKaJ-fli3H>pbIRP47(EVF$XHt_-=SMJxSA5)MHi8 z0ZXoX9&@ieDBo_f)yB1UDbu_qw-2(0@iI;PAh-W5k8J?s!JIfo28QR%%oC#@_^WHW zxysJalk19R@?>V@dbez;0f*KEhUy6kcM}euo5EW&{nfgN&u0EzYwM=eteR)@pI=Uc z|NEZTn)@C_|6%_#XV2;EH)mGgJavwBj_xtmzi;N={StUVJlKHAyWp)((c6OoJckn4 zTnm^l?&NMxxNQ7kA5ZW}-=h6fE-n(ipfauIn}Pt7Zb6)QfZkFD&Itk>`3B5M?FTGH zpS!xMEBJWaD1Tm8y|$81ZM~1|hBd(p1ex><;ywOlaIG}qZJltq=>}8f2aeVNj++X+ zx(iAsZOW?aU|?rpJU@|vy?{w+&5jF1oj|6!l<2F9t`?>^ivVrXPM%*rF_ z6v8p_pkpf+v+#|EDeeLbBo*{bb}%etV(F7Hcgna>n9Mv$)pydA2MZcoCi5z}c)Sc+ z>dk3jY3C7nkdedrTG2y|5Dov8nU}o|HcVh}74dNMoGMhJ_|LJKMM%NsK-biRjm)Y7 zDjJ2R+Sj;)SS^DC6&tzI7i&FfRMDMnl6m=+SZUk#7hCq!ehu4grfcvZyOle7bK1E( zJCD9Sx7B%i_mm|rZ90*U9v}Blv#xv7anV@)jEqk4o;mXlv!2M*U){Rr8&fpLgzFAF zR&0}H&65*U>QT6O#4|+OZ}xNQy_a#Kx${iHZ4gQlqp8@!rkMHDfx zTPrU|?2T5un|;L}7*_E2G_$B=*{$y zdGnbc51jd}fBZPZXK;q+v5B5v@mf>8mWt127Wv(mw_V#cVTFL;5{bu?{cS#~ObNJf zBdJA`OVd?4%WGy*|5qKI+fR2D0H^W`u9yk zv-SShmNkYRjqH38lIc?d+k{$p{eC30&y74bGfQQf&m?ypZkEkwEtG?Otk0O7<-6u5 zyvWr2p-rWkwRhavZky#N4vXY$xp-70zUm?CbVs*~$0T#B&Mg-3Ske&L6{_g?MEKjW z6Azf)e^PK2aP%oWB;xU>BUvNZ%aUb+&8r2CiW%uerK^582u*Wjl>Ec5S}d-!Qn~L7 zv#0$Wl>=R@@&N)3t-+l}T(d2NE_~?qs5-G=id&A#9R^`Zjm8h6S7p6*P6VsSuxd^@ zZN)EY)UjA7Xqn(HArA$^y^}nY7P1I=MWp{N^-5b_`>if}zHMU5^uV;49m?w>guHcL z3;jED_Sl~v$p!~F4);w6Bm58`_=Un<8zn4WMtXq%=zd#`$P51R?FBGA9p@? zWXo*f;&EAVpwrQ-G5eB7&IyHPNts-iOm}Zd7SSq!cE^ClLDeUig|93UVm$Gj$tux< zB}Y`~*8~-ZKM(txKD2FTfAQQwcBj8@(NUd5PHi@w6$e=7UFvWXD3I#)Xyo*{(Co(1 zWt~r55*rzWpM@@-H&cn#KjOT}9|aDvu#JNKJ02-- z;Na9<^sZz@aO^8BLpP&o8--4;IC4l}8n>5+lTh1%h>vD%HWPYv%wiOLvs03#6+Ue2 zziuok@P^?ZXV#xH()P|OO8hfDf_)VEIy@3&XBbRz^hnGq)nJvATCm?nfU_~Ix&p z%x5s}ulk@H%X5xJYUM%Uc^}wBk38h%bZB^-aAgsT%mG&&qogpsTv;WG>CpQ&+ z7e@-8P79uGe<;(mo<}8b&2v_3mPG>dJ?@|JYwUILOpQzm>1 zXyjR-!1uYSMPb>4dH>uToLHL@rf>vs9^~kFJ5~JR31&MbN2Y~G9Jrh=crmFo3Pl`M z;n^YTUUpPy<(?-@+b2C>P5aQ|E}+UK5O7f5Zz)g-@K_ztUe2ly!e*TZt>?Kf8~zD@=Dr~$HT&mX9*rrdhxQ|P3KTo z*c=ySr{Cx6{F-|6YLbr~NZIpbZl`H}7H_V`DkJ+}fu`H%X2=&iG)o&e2Aw(SW;Vxg ztN)vYg3C2n6-*X1`hC4p9Ma>#zvx4U?U@Hm8UY7|*Cnq~kKqcMd*xVJ5a(Jhmj_Iy zGa8u#8k)pn!kxcVHZVttC?x)KeZaEiN>^uv>C~vY23Ea8Tn}p^MR{Wx*i=qB$j)nE zVo_4ssg=*xV9~&0I;DYOPq9V_?Fe69Yw<7t&XHt!dEw|ZwH1RqaFsmO(mHKt-&b3Di8l`Len3XO# zu$pX$6a8R0Rg}X&(&0dhEyqi@4uK;kJ{LKH1Cpg~b-0K~IW%l?IlvaY$5|$;pxx$+ zf(u6m15?zI29^oG7{xmnnFItF!q)$by2=<)%8=E-ysE@_Hiv{m=gLnEJUos~5&|u< z83nuCpDY!*){&&fsm_*@&@A&`qB&?16T6p3vrw5LSL~+;i~4$Qxac??;JmV^>HNwB zX7vC^Er2Ec%ZdI4llS98>Q5XgK|6zr~M6KBI(YjTwieTF*MJRH$rzdQXzs zSb&l3o6kFG166(r4-by011#wlR{Hs6Y-Zm+ea>WIVwT|SWL|XOrL9e3SH6cA6NAE6 z27?8!7#SRx;}lpJ)}Y0YDqys+VX|5w z6eLo(kUMtDqDEx~7bT&_ZcT>;M+`c;O;TdGj`}N0PdISE?bCOMJ)(w;YCl>`FNKS! zeOX|8Im3xNZbFmzfdqEL83$ih3N)90Zs4%_aBxq%K#Tf@hsU1cWj))ZgVFjz?~4o;`vqMbFRa))OoNyik~;r2Zs4|N2w-4H*u9#YK~bQEu+&5|AsVk;Wm z6_X?{V*Qc+;%ysCzoVuKkj->;XpCGj^{Q z+wm>16Dx3$Y&hb%WS{l~MkR&DH-{};*D#7}*b8i6eB06CrorOa&@35YEI48RjAXrM zNygt6Gjek@M(t^I>R?gLuxvcQz;%Iv$APb-p+%~o;l#h{#e+rn)2!#P&rh`WcK z!B2@7KR3B8*(dO$Vcs8x84nmZIlTBf82BDE%ui`t62s`d#7;cIMSKCH)QLuw1g2Gc z7zGzFscm5Vu!BK>fpKL|lg17Kn-k1h8yM9km2La$>(i{XWapNUNx0?ZBy z&BhF^jz602&TgEyi*esdf#y;M_Sh3_ zu?0=)FSf{8Fp93UVp-vHdd;rgpAK+N@HtsBXX|DkF@}w&Wvs;~uo!f*I8SI`Sm0>0 zqgU?%v+ROKrw(S-OKgQ2ExZz3aw3d|0nAPetp*zUvnEMtd9;_DXe*as{TeXG>*G4N z9Sste;v6R!j+|`pERs)9ZjIT{%x}SXNxM;e#uZOd|Ia%95)ID2PeUw2{J*p?eoHwo zJEKi|X3wuJEsq5yZyngdwshY-okorXM!`!9SR)zuc3q7LW#HvHHLs*m{s)uAKPhGh zixw9Jr)1j}XNR>e8BGgrHc!3Ox-hWGu%RVzLW{`VHmQt2&(3SjKN|E-w5BPy+B9&p zu40h7(5l1A;>^+CWFejNO0;OBHnRakXfng`gW;?Pu2=0{UG2JCvV=kJLZixsW)%$v zb_E8ej3z;aMiqr-K?X+0A53Z4W)3$R3=LYP6&iRfm>mL`V;ESt7j&`RVBT-SbzHbb zibF`#gIjb$lg0@KMgjMdi-(qcn8tE}!6SoZn^23#g;sqIf3XEuJYpOB=3JQ|vv*19 zVcCkqex`eu-)$7}@V|V!v10c52bGIYf8a5k!Q64U@$VA5NY;QE8UclCLggPRa^492 zw{>l3d}o7HMw3S&lhX~>Pd{1nUbM!)Xw7}VR?5LHmC@8WL&ZaaMO}kQ-J`|%0IPFH zi_3-vLk$+Y2_ha0jmcLUolmp|DO{|sY~tO}ASl4%;=ywF&*pb8IQA6E?4A@ZS~6$7 zvN-dP)pEBE@V=SKUBJNL!DQ~$;&7sYNq~Xj1oPdgEVdkt+#D@d8yNXzSseu$)g)Ne z5)K#yG#fKSa!N4p7EY1=!K8Me*?b1$Jmy0zHy9Wn-1#wqA)kwzWx>LIKW1v4V2HWG zYJH$Vct*(km_{LwM$ri&{UwJd%Gmn#9$wn(ymD%k<&`UP6}N&^UAhBzXjcnApS(|E zLgT+F{~97&IX9gR<=k;K?%-^>SBji78W=ee;|m$ziMAMfu;e{x+x?eS?B4DC0Ct5F zjCMDe1p=5HI9OD7FsgIxaS>qF%U}|C(bPESn%;(3t{)95FPPLXBxdzCyNI&n+-Qwu zXf^a;QZHyq-osGt$sn&aJ?OP4%ZG*6L?2&2tH#W6P>7L%fq~JkfyMd7ahD8V>j_K_ z0xY%y%+3ledIHRj6-~XO3|t9K?^RnBR!q=;z-%NDD-yEGUW0YprA~%|2C0UYoCFia znQR3PEGmE0@|zeWI+&TdnB5=U44lB`uwlJ$z*Auk|9^W!CSAF5N%+BvuOTZ+qkN}2 zXWd{8z8bss-(TiGIlO(d?Ib_wGl*VDJlI*dGn6f&A@*oztZ3r4WCqTRMzsgcjta~! zJEF^fu;%4G%>B`t8_;h2fXU9FCGw7!;EqP62ycN60bD=qV?U~*t+F@ABwal>QxAB`+0I1bHB3zT-}c{BC&k#PQh8-yg2dCpj^ImOs`wXso_ zQGG+BdI2%B`ti zLKvB+IkWm%Jlg2_@A*q*SC-rJzML0&z#wKA&H6B*=%FIN+H^*b=&h`b!m2GIW$i^T z(!`&&IGeyf4W4rgo}pUjSg>x7|HX_gybR%y5tGC$nO{&tN$p*eIv*MDXEMk(~^31#DkVF=z;}h@N0RvzBFXY-89vkAR_9&uEVC4;T+*b5<~# zAG1(9C>IKf}FBb6ayH}_BY4@pbaze@SL7#TJmFjyi>2WZN2XO2E&7I`eB)Fo% zTA+oC!+TjrlUPTKn?S=V;p1@+yTlrsf_^k+Rx5aEJDS&PiF3@b zx)L~N-mY_Y=UY7%92Vbk|F^9Fq&cs|1#X##IREP_S+^`|m9PIw)`B&yi$nGrnN3XW zxqW`gm-QSSS64N4?v@k}-mlFhF_C>jqd+Np-UHUa2dwcDzZSNa#_zaYsK6e>(JXzA zk$nfl=Y*W51%=Z3-R!I0m0#p!+tbjS+9A!r`Y*sE%rKzCQe8aikbcp}B>ry;x9!)~ zud7zQ?#3q}=dHnFUC_vop`a(&e8;DWC1Z`)jOUw@m_!bUGX`V?I#2xI#vm7vmCnF^ z@;sYn155UTCLWLFLA%^4awdOiVE^mgodtX5J>x8B)ZZ{O zSfDj{LtWs7e{cC)g9EDK3|b6cFgX-7@oZpN&e-^4bH}pI{f!=qYp?%$6~@W_U}3Ar z!Yyacs%>BpGntUG>@^=72OAe3ht3Lz1x?L7dKQ)$5FmWt7YE{yCN|Wp5%jIv?xaVwA8Ums|O0v|LX4mb(+1 z^dfg#mpwh@>Lv8RrJpDA)8XTM68dos948x@-Fd`yQfht#G`V(i2zq&RC_FUIxF9IW zlVC8JQz3V`OJ~&HS*EW8HdH?3a%Kydr>0!g`2<&NO zW*1uITps6mgojIPo3+G;1C8kyym|F>GVB&Pv3m2{IM_2dv@rg=xH!JzeG&_=fU-`? z59XEUgoEB}IWB+BhuOiG)1|;6P(bB^1Mfr*A$EnrCxs8yuiXgliZVy(UwJCVW4&U*^tkegSFgv{ zdm7DfulbSKF56Ye$Rg11`mBjZXTyiaw3(S|j4TUkf85Lv^h}V-+BfG@RQ9@!-=1kM zn{mijY3G}w4hgf4k0&%HJZW@fYYhv?U~7q+;K+Jd^_Q!{eoyv!%sOWZ7#T&MusgA` zoC#oNV>Y?KFmryd`7M@Ar2_|6J+=JOP{l5AM1+~|%BuzjP8NZN70eg@EpS)8G$qJW zbIOJ%KAM*!9-rZKwVl9F{ZzG~N#T-0_5+ife$FE87xcca3076O(5!IIg4t7f=?aG9 zq0e@_e%JHyQpIDXt}lk|0?uDvB=_4)c$&}e`c9xAE=Ba^^vMmpX`YESGGR*vBJ9GL zC&WfQQ=K7HAjZg&I!jAx7T27B!y>y5RCeVoJMvBH)a?h8PMfqpaCDVh%D~jA;Kj3? zabYK4dB%CkGYTG>x)uqHjB|f7bec@Kae(RKk{b@pnwL3@n3;`gE3#H+x=mr&!Q#5* z0CURr-;e?#R0)ZntjgK@q@v6@`aF~^2QJcpjZpFPT-uWxC@*(^-w3mW(GwDtRWcpKaw(I?CJp;7toh zg;9r^0i#gZY)7`f$&I#KPVhHL+*DRDWR`YN;_B~kW|cD7$?l?{s9J;Jy7??y?xF|C`P}pF)K#k)?l#7Il zSwZRQf6+=+d zuOykjV7ay2mj?EU0XHTKInJ<+XyUJ5a7ld9@;%S4ITIaD-Yo|9aKJ8#{riuA$(Nh>xS4V{f9!QUg~Nwux&pYm~Uy zbXb)hHFRY1Ffp?z9AFK2(C+S#z-Dm5fw#kKg|y2;k$43MCMz>xPK5=GJRA;83l{Vw zK7Gj9ui>is?gN9kRznVR=J|{p4xJJf9qiX1D6+6z9p*O!9F^e6fCq=m%3UNo`C ztK?{0`3q*58z0(kvOessFk$iGEVi+|v5>D+;84;gmlG#k_>C_|r=&0G+?%=1lPM+J zQRI?2*C*XZE+q*^L8AwZT1ymbm)$t3r;_NU7O+WVf`rT8xXk6YHIJrFJaJfa8RI#d z4H|qR8~E5*47o%z1erAwnrw^nW#?AyfZe3ljUK$flt> zOJK*r&>M{x@1AOw+xKs&d%m$1e{bHDo)1RuCR09It-JDiNqNHI;+qqW&yZ3QIKDVY zR^)MOhpxZ3&DTq*b_#oY3)`CemV~OP7BFe@IEgR&a71E?1G8WQ1B-cumMB{ySKNyx zsl^A}G>gR8Kk+gNe*5jB$n}wPa!-=TvJ;0*&V7BLIVDMesoAXmws!`lz?zRQ2T z@R{l6${mS1%6nqmFS%ZM&0lNq&!s8x6}$bLgNY%FCMC5LsLr+Guj<(1!*a}8?R>=> zW2rwpHzPijMhDi^zB6&7&^2^?wRvT1dj!6f+6nZ?gI33O6w;Z7^KW!OwL-U`68^jT8rl z>KSY|R1zMmtGn2~iaceI7%=6YZ?{md2P4-!fx{eko^*uH%el$&DNo(M?9|0cGO26l z1Z=2%UF7}Kh{b@RGv>v&?Oy^uAH2EdL#y(f9SJgbcRQ`>@$^|FV8H95@a@9$!%rA@ zH_50d@=Dr%^zyHock@R2sVgt~V;eY4f3y5kSm7YEpe}(e`y% zU4P@4?0vC=XGOx{j%uAXJMO$adF?lo$~p(WtOEyFVjc)i-O-@1#*nvUgTgHBDdl_J ziv{x4OTx@Eb1TC%c52kUW{5L8#j0fZtp3Zl2cK57+d8a&R^7Aj3s3y!{JUoYI}X2o zB=Ow1F(c}-gLhA5tC;1nunm%DJQ`T%CZ?>knP%I2Re^ui9459o!cHIS8qC8aj%b!# z>&$aBadGIKW-l1>^x(8(H?DDp90*V!2jw2?=|L^8zWy?%1CZKd^9wCicFhts<=o3=QNA9&qo+0B>1EjG`ydBfIXu} zj48fi2HQ-R2Cg%=nk(hG<}iq*9b;Y5Ai7}cIZr0NfM&&xM%D&~^9c;~Gwv6t$S-=4 z_;}Aj^*8(~e;P$hn1nx^St!F*kaA%~OoNY!0}IQ6j1~_!mWF=}e;8KSFtV8(;NfUs zW;rlx!EqJ|(dkbm990g;S~#~4*I=hq z-iJ00oeas&KYN=qUExY^?D^Dk0QZ6ecy zMY=(8vvh7&90)tkFo)lXpoIBxCzR&?mCVOsY-x z!9kfjQv`l62pIH>oN_oSBQ>L;)4%bgWzoYK89ppq8ua#F7nu=1W5a>M3kaj|qy@ah|PV%YN&jKXIQ z{8Ei)z2hKP;G)3MB=5l3k>bK)(qJ{ojrjwEipwGK8B*(A4@$c@%9=O|Uvbliuy{um`Q$64lr%W46ZyLXorZSrz?vZ!Ot@$CXriroY| zBc5|wcj{Kb0)zzw$h4+X3zu z&nqUz{9EMOP_H*DXwIGV8U|s9`|1mrq%tlwRPiVkrfSWXAanOIdG&LVBOLnEYYO$K~&X&NxtLMia+yeEF73) z9F#elSVIojd!?}6U=UBqU}IoZ+EoO={PrgR%GV76YvJonOJ z`!(wGp1llxmsvS!`TR-g6>g9Dw0x8zoJ!wHPCeuJ=>fxMfrC+9$`zHK)?L$=)On_! zd2{C6Q>8?PtTq46FROD><2m%Rf=R38(CnI9nlBoJo-oMYagfVe?eNJ>ZikSH43m-y zQ~b;(g@yysf!d5I2iQ{>>R%k-xWaJb7y~y?!-{|;<^m-~4hLq3v&sgJiVqx*yD|zt za8wdu(wyKVtZ9Z zbiQ+3vF{$bst0Xb$-nph1Ismei6Ju|J}7^cIX`R7{B?Ts%hrT4wW&xroe*o2|05{z zfLnlvkxzgz`d<2CGc6_6OYC=^r`%co@!J%o(6+P&hh?*vv}&3(7BqeCI<#yalhy&Z zFCPy{s5tUHIRC9PzD;dD+rJs{Y#NO5mW@I$8h*7oDk?N3bur3!H1a)Z&ws(r(c+N0 zh2dY^0ro5691M*TR}Q#3Imk-fHF?6MU~r1P;-JZ%!`4d5^HFLNw@n-0g9DDnpLZv-4zb;esDBjaoElK5PO8fl4%Xf z?hVWZhU_f|UEVXT2yEu-;p(Ui)|7B!6L4U7@UKBJhuJj6S@DOX$`%Kg4g=;RpBNp& z0=myWyuhN*UV1=k^Zr>15B_c9e5XBgRlxzLz=MlkZqD&_HWgvH(qo+AY-Z7H>0xaB z;+p-Pm!|(W2)t*u->}u}vvYialWItliba#Ej+6YKgEBW71vWVL&p8^o;wbl*DT+rA zBwHwotCUH!FiJEyN>n&XsFaDTlnJRg+IGJ>SJ!mvM(3A*pF?|v?#7+?vFWCIUUllVbc6@kewwX?SujM6$gbEo_aj1em5{MIxy_H z=J@?uV{tq`sg( z?KbBxfkWytO`0#9rrqmjTfrdugVEw(vdn?S|3oB|c3ju^&?wa5c&_1q+zUqW4~?oj z8kD=XSQYiK)Etm6VYc#MlApt+Z$zHG#=jbeA@P1hb?;K zj#za}W)!`1;H)p>y608v+z(1TX%K$GAZ*bn@@7?c)LMmrMlKJ=YR*G)9EU_)7=LLs z^7b?;m^iXosByet5U6QTb~~niOFMM-dK2 z5f?_08;l}4iLx7RKMdguls9EiS-|M9$l=9LudZ^TK)r}js`jK`qt z4C?_#m71-l4PPyFy!l>#IQ%WFZvMNY>)tK-zCC(=O?13-ugl@0n_HEh{5v7D;h@YG z#{MU)QCl`A##`7HH;U){Sk2y~Ske-HNJLRU%cqn@q2nNLO#4~Q-)C>`61vmC8(_~N-WOJidn+)=9Esl`6C90n-f>w|CKxW?Q8w%n!a-k zstj5VT-*7a`O4I%w=)`iO=@SpOg|u;vd1i=*)+m=(c;64<~Q>=t+DOd!{7ZpihIYq z_s*s}{FWWB>1%0?*m96hpgwXA>jV!)zL*{2HH>l|=98J5vXRtl;5iwKBnq?}m5*vuuOHRpUlqav4rn&hO{ zI}(%q=SjC-S;;KE?xZ6Zo2*9w|D)y>`AN%O-nzyc(8j{MDB!YvUsNB5y5EY%RyH;V zO*ICFgoFkbX@LhSQ=Ni3)fcr)TB$a(qpNGhLJO5w%lIdL5?HxVphHV*`J77^jw~12 zz||zAW$}Z%5IK-`gXhSo9y0|TdkghJC~juM52 z<`Ay~&de@7p6VIA4hJHa*F5SxJipJW;mGWSU!AMi=d^0Mu8=7B*d&x-lgQ@b^;2v1 zhAk(b9h2UCQru(PwaDcsc06%1I=N@s6YmY4%a&~N(w5Gi(V@Hl2xEug!5=yk9InWm z-C|Msa)x+p$xYRixU!Esw%WQJNML*T>BQpgK}wA&{nATL9Of4_I@VkzoW+{C}Gc+Ppb_Mi)}MH-L}1?hr{fMPB6cn z&cTYay?0eAr{7(fBtVr$BPqeRyM7CB`cgg{u>3^ z`IsB*l~^X(vtiNH7Z=)e%+44(%y54DW&ykH6F=Wc9xQqhi)V$4=8NCH!D?2;=$`fE zQSaUbi}i#SG7DaC;JWm?S!9MmtA&RmUq}bDiqu4QwLdx4;twK=X9Voi;A!U!Jm9PB zFrks(g@Hw{pwV%O7-OKD0$bgL7IB#XhM>p=3^Ug_&f-brbnbBEZ~VZ_W#rK5@urcp zX+|5{+u-(~BaLiIHze8W0=nE6C@?Z*1Tg$#bI@6KgQd_cv4PP|WXf_I1$Bmi157a% z&Z|uH7}bP4E%^<03J35xXf%kMHdwe+rY)BKVv}Tk?Gmr*B4yb(0qy($sfu5IsOyX*Dpp zEIhEpd`IIstqqK8Qw~ftc^k>9wV>$}W4gd4;UmgS7n|5x7(!(_8kn5|CM=fQsv!R3 z06UvO1CvMtlh}y{RtI=6n7`UjhK4OyUQDA?m%TbnJ_%boeV^wy=btP6QgKbpFs;h*ZQsHPK3G<#<|ecNsE z%WTJ;8*P*4%EZLHOyW^mpqV@xx27jL0iGmYXs~GAC?p#qM z%D||>EKuR1vggqw@#e$94;0L-`f_vX>bPWHF8FA7ebKbC3Dac1OlTJtT;-a(J(16G zT62zWV6^ugvF%5@lieFn_vYt3mcIAn*zWsl^YRLqrFTm>b8J|X`p;eSRN$$Vnnjh* zJluZqEHw+|5MIK_w;|v$v-pGqPm~tOeGzNmo^*pvPpPpx=td(0%ZUT&M+DkzpENMY zXB=deI>aJ%C6Rx^492474;@wo3!mDAI4QSTa%JB+E4g%qBa@QAWcCnlM)3)byj!?i zB+eXQH-5m#lW>7~<>3W$Oc|g1*{~?_FD#A;Eflo&4`6g;xE=h^pltKhyE$pfv)|^< zH{8<5vLy3!x7-(x7yVmnRjq2&?zk*lk$EcC#=`8U^sBb-4+Y~Zau||W#C{YpnXh=t z_P~TWq3zxN?bD4q9XGP21eGvE}J6{oMO5)f#)v&wM$ zuA{o*mv9p`u9^VUv6Ky>wj;&p!(^XNCEj>f$p3wUbM;vAD^~yxB z3OF2CrN+N=3)5+j3!nC^*|~%}P{fs4j&+%hzhznin?nQVp9O3uxS5pn9$ispsc{h6 zmB>FyfqlmU?koltt9MKe3|zY$cy=|g#T?+?B|7LIKq3*Y~Fw=K%q)T^yxn$F#MXFl9pE3=PL{C+R9`34z_g1dZ;Y#Iz~ zmj0)=^svZkN*>fXQ{@&I!mwN^Q=p=AN!@zZT@EYw9`t?+Vdi3GS?v{jt4d~7~(2C_-*u9P6P_PX$V$g zrvBhx+*C6vYoI+Bht=xZil-o}yZ$knqo! zZ~HnrE+na!K2fioRJM!P`$yg)l?%Q%PW0(~(45cnF(*)KX_409Mwt_dT9<{z_5Ij2 z9CSPlbb1Z%du;gRXAmftT0M`Mb@R#l^+QZ=}G^#mJs3puk6u+=RPD0pbW?#Q!Dk&8otamC5#Q>uck=VBHt znzL^m%Qa3ntp(;qj53EFTlD|CY;m~RB6ZULTYU5DS{O=Ie9s9Yb^4OICB7V14_sQPo@j8p@I(O14t57=YOxZW&C{ih%>;h1^JLU-?} zQeF#xzYt(9c#xE#z@DSZuh-!CSwZN}kuZ;goNo?DvMqF-q0OhPz_`ivg;0XK=Ry${ zMphXYhEEIvC5=*D%HnSnie{O(Uh>ymCK6NdcH@Rc;fL1oaV-=%lqkFJOCD#A>h(m+ zr&s@f(JZ;tBvO4zW2lZ7ba!Bem`YW@4nA|i#PdSE!H}kC?ioObL^n3*tQQV z9C?Hm1?ZdI@7uWZ(7MyR-l$Z%Ej40D33cGIOXN3k_{TL#pMTP7rW^%eN4DL5bycb-v0o6AE{Aft2In+0xv z5}Kn}TAhrYB@`H6H3+R*y-r7TXOseGUIXV81+F;<`R`p&?qr&~<(kprbCYFczDaL5D8OUPv`M{WbB^6xiR3CS^9R2^QvL^N*GLw=>>8`f#6%`@o} z7t~sFsO^}kw#^ig9baTQBtqUyla<-28aYYU?4z1VoD?i&c^t@|jXM_8R?2 zG~Li_xueLieCpLinFOI?#;Zl<+r=X+s=Wf*jLoz)_?fyUem3&AEZ}6#d%&-7P~@Kj zvypJD5CU=VOS=(ppP*_}2?77H`ESecfhh9@yESBF$3nd;Lm*tc8J$gwz7X zgz1wimh{|Bkh-@*NPNAJ?fGJ@aLMJ~64O#|7U`DXRay9POVX5{!sNoNPZ>LyDk_t6 zEf^zCt*l>WImg+ui)Zb9M_E)O03QoH%p z$=d5nEdA39)EH}ulS9i(`8E039@;B5ir6uRm@XCJV$7QGik(MMT*Z-p5`#t50;Z@3 z?C%cn`Sr1;D6j>w@@F1kTf#2Dr8v=m!Ca+i&$a+2mz9j$1Y75%GRh?|xjc$gekrp>RJ8^w64A6@nl^5`{|b7sj4TxEM~p1 z)%)TSe<`V=nr)$n%e&l9UGo($-Ke#8(uw}t%R?n?YpbTF-jofSsTJ+YZgwrnFtly{ zKkL(4`I`b7K5d(m{w$a|>brm1mD0b$A;K^Dzdc}|vSGEzc24<+g72huIvqIwZ8ejf z0*k`}&V~dQw*!2S4zNvGz*%&jDK2l%5$`Ep4O(%Fa!;{tNS(}})6Kc%8(-Z?{xy!Q zT8dmu3l2|Qw0;$948v}%E^SMS=GNfd^G}E@S!nG0S+uaU zW?O0O(#46A6$iImQLxS*#EFsv0t`{t>()%EJTcMK{P`$*oKaQBAi4cRM+i8JS(TqsvOTdsBP-@9cCxqG5cADP;A zm~Z(cspYd4vqy!qu5jR$4&AlacBZuAUG_#}cE&1>m3;pc(o#?Jy>j6HG=aZrG2gt? zObrbpOBzK!v5PNr6#8&H?1ut#NQlr824)^bA%TPT0SCUUX{uj&XamdaE9&Q`vos3j zI57NKAmaDn-qM5oO6-gtoBl=E9*LfFB~s})ztkH+qlDFlpP7Ub=H|&WB`^pXWT>Uv zJb&J*{(0;3xrx_%>$lBpl&o9+qVY7d^@r{Kb6bqdPcEMKYv$gmhTCQ4)XJP{4E@re z6PwbUVe$R&eTmkY#@`nRJUYP7^-!|*=IPI$XRmDFeAKXuA&RR*g4us>0F$R+&j&bk<@Lr)GZO?O$4yZBqfgqH3}(;L+l94Z(botoMC zC2TA%d}?fD;Su7cWdj;tE((m?_MLdWpB-Ir*5`h$<4WY zw@KRh=zZLNq(@p^&b&`1<-~gl_kBE(pPrm}rDp#v=Y~Klx2RXRM}m>*E3V||>DLx0 zonH|&;SrC~Rb}T^o?ZGc&M_z+ymm>$sNiTr;`TfqNsoAs7Y7_!xi8AP?0DeNu9vms zNWnt2P7zLpHlB%`ifvMQ&I&vaXB}%j+WA^b*(GDaB-Q4>y4idj6BzglSq`|!i7rv# zU?_BWbez-u>=zLxc7+56W(I`?4Z$2L8xolrZH&(HGI2B^q)Fp=AI=Zl5RdP^Q0YUO*!DCER<} zw#d}TGO>KiFLpb7RTKjp7!HaGI5aSGm^Tz07BxwbVw3vkVd$Y*>EYO_5cnaDao&=L zCvWG8&p9AntY&`wPVrLB97n!{hKbGGtThjqG$(Z}Y*yJhVNo+*=YvlioO%x?am&AW zxQJU`(BeV^v-l2&W^RQwTnx-Y8@@EK{FV$24vhD)h}pIr#O-uHag?c_(D>$8_kY}4EsG5J%}Gs%=! zF+csDMm^g7Gws!cV7?hU9&{S83Yn}aIK(2dVZ*GvS(i>66*1nJ@mA@%q|wPetvNiNT^_9f_?O{HGn5b&gzMkV|l2truufNMdM=nyC`n zRM2L+$xuAsv!I~mGIu=R@*PK4t}b*sc=e?d<9F725M>cVA25*k?9 ze;<%5+SncI@k%gp0t5RI0Xf^21pbL0EPO@VxEli>DD`VNh?iaLwdGK#e6r9@MP;Ii zj7S0lbHP-3)ddZl8Vo;8BvnCM#1YVc?hwEcdU~>Mgconp661I}AKO8WcPX)$suE!fKW zhCR1EghA!K`I?my*$mx#0~S>S&&-O=jEo`w z5}22EhJ}_UhuXYNTYbQLrqD#?6&rr_%@${1wIN$jR2=3qsM`EgBJ}f?kv7L@>b6+b{QNZ>Rwo!&-o4>w-lTo!;j)4g>SreHdpA{ITk)px0&T@& zXXnOC0j10#_ZryO#5`T$RXT%r$AO5dj9GbR&&$tzJR3egGLA=qh57m4L$cSS{5_8t z3r=ESox`u#>3nAAyz~W|RTvmMoEwrvssx%B{$%hsWN6~I(qLeI^SMdlL<6ht!ZjOJ zycBtjuhdD+z0j&^wUE6)L+jwgc<&Zb;Yo21_?H&68O8A=OD>;utZC)O-q@SB?_Fx- zE1tn@w&G#3e7W+8fAcP;>Un0w2u({qIa^a}^1%`g#()Nel0#^RXms;_HOW3r<+2W{W|F1N4Rw%v?D`o=*yvyGO^y&tPt zzdTY}w-W4VI>9VD zLxGitN02#a!Xcih00o;T4Qv+wHaIWQ6IxmQ*hz@_zPTb-oSf+&=?AZmvg(&L_UZ&N zOFs;7xA@|~+}_N3us48AzAaX8$PsWf8|U|?XL;K^jP zp;0Lyku&&3t)PiQ0}qD+gKC4PmyCk+nj6e}Fwbf%B>} z@9u>3mIDDcx+4*R$%y_$RKLK9Adzl8Niwvz?M9LEqwx8W&n%w1W&$8LVN=3 z%Nj&a25_F7z|*vlCtIR+TWfUhWv}8FUY5tQ9j>!kPiVjEUg9bqvn3?u$kvjV;vJr5 zeD4<2yMz0BMsR69vJ6J*!TD=3A7bMU z{7cd}Vh%8RHn>I};a+@z+apP0?*ZPu4ZOD=G$k)!HZfqjq7lCP25TuRo7Dssiw5?( z2kZ?A>{AtZ%M@&8Em548W+rv5N_&rX;y`2okAmmKpWYt5e(?`C5!y1V;XLFKf69rZ%R)8Ldk`fxw95BF1HR*&PIngY?we4W*v{hWz{si~ zD`3!&xhVA35+)x8j!FS`z6`E>BaR>e)&>Wzy9T_w47k|a)2%)`UQtM}+Q98mJaxeV z?p+Duv$Q-3DBp1{|FY95oMEk_0%6I5;-%U|-U}F89c;zO&G$ z!;F1F)w*;={-*)kjTj_1u*A-o;bFj<{Gg$lfuq~9KYRw~zf1$BP~(}K1G#3O9=K$}51EsSsaL!fWI(UHhgeKp;2YgRIaG#l(n`EBLo4~Zl!bf2; zW6%fo<_jFn54!j`$}1T-?@r*|)xf=&L8xjO_w5C8E$-br3pm;f3g$C#El^lFwQJ?H zDvoLcCcY1hofEjHit~B2@?I`j(Yb*8?gzfR2^^beuv8c@TWwLCo)#&-iXoJl!K-35 z|5H2000z&586E+w=?Mc;eQNRT zQyjLp7CT5SDZ0SE`X-ypg!ZZ(Ot}UVy`(s&e_9fFs$C*e;vs|T$=2BlGiL{0U;0jg z@7)33GnSo4J$a75yIS+r+{8m}; zb}IM%1pfC7TuCcgBu+@Ln4VbAS>3$FtS)n*M{=4B1@;%;xf^K0$hUyFML);VP@S-Z2=vunFgl&#h0b$5@i2>Ev_ z+3s7rvPsDIMN5@->c9KI`|iUsp53K~9e6$p@INh3ja#+j&oA!72R0leqQ0M4t-vv)Hr<-5~VC%xl~R7%&6iS4Unrd&*MDeZ{=#T|8=bNd6nNw4{L zuvk4gu>Rje1wQWV`cQ_Ns>y18N2QE)y6#Vv5p7`7c)ssGMhW_PVU z`o3)Wc{yOMID_PeKJNvr^B!<7zQED{fMcowXMX@^^`CvG7&tp4SyQzc+#I-eF>ub8 zoK+ybz3aef z{1UVyN=ScJp=$e#WoLGBZWY-1@d5wG3(KBfIQ7VZ@8tx(BL+M-+t|VcQV(6+dGO)R z>`x(VKcvMcu)ny#`@-@c*VQiWt6FR+4oq8DFrU1&$!`uvx&g-vZ;p-!oTl8IpF~V8 z6WCi9aPKY1zw&E$X9Cxq3rBYqaI|S{ZggPw@2J|I#URYU;Nuv{F`;UDtNr#O=Jy(F z_I==+wsK}Q59j;WTc--_4}Z=nZNRiEfa}x+4(nAXdNy$UTg}n0!>;VhK1qkY#NlM= z>UNh0v)JFT&3bdP?E?p=59cHS-W>wGp_WaXIdg2kB}?s`=+x>Y!6W%jfbZWg?nxSl zw(0Dg+I{G;0^j3?Q+GCOm|4IXBCwIIS&8$D!HFr8PuKYFDrCPoflq4gyk!Ttmlbfh zIWXB?Huh`anEYx}#Lr#361e<&I2Qb4;Phj-?m2-u_Q9Tg0c&@3bL?5b8+(SkJ(p`i zK}ExbjO{`f{>ylH%vkN(A;NHBFY^V45Ce|T39K0p`2Jns?N3jTF3j-2_w7c1LdnskRcQ4!y^GDVxSAE#EG%G=GvK=XfG1k|lG5K3JJu#M=&V~Ac{1k#n{B~5 z!IkTTb}rFQV6S<=e)8|h(`zqJazC}XbK?CE$t&haJe&}dd2ju_56cu~b{=)$**;+> zbM>5EYfc}i;eI6>%28~7L`CL(uh0DG!=eH#=?m{)$>Qp3pL%mL_qWdLy4fo`61a97 z?9H*_-SK);_1#UGwp@F>*DjcFUEi8(?}ej#9XJ;IHGQ1IBKAREqDgT_mszLCR`vpI zCWgn+29LMT;jXHA%xZbjG?#0pz~gxWPudN5Cl)*@n*GFS)@`4@f3x=9Wt;M60;|v4 z%IrH4`c*XPwlKr z@19s2upG)gaJ2TVwFP_Cgr_Iop6vW^@*;zfHSfWYTkUrzOni8J_Q_V&;`)x9cii9q zxzG7>o$H_i&yfP2;}7_b3-CAi^BlZz$kjlJ@xJ}`=cdaOKd}36*p2We|5kj z*GtFm@!&4AlQRspj^X zy02QYZ>OGN4vu^Ky6SCl0(+vvzngyFPQ}_T)T!L`a(n2n_LX}q4_{SPZRvQ-z?Z*@ za~8v)ISkzE83YdXiCUNG%sbe4&}^aR=0s;ECT0nn6%!O3kFtqMnFus&d~}>kQQD$F zP*c_{cCo|5e@sWYMKppJr}X_|c_n<-*(kryNVHgFk@NfpCf_eBUnmANXVeLKWn5WT z{(hTh*LS}=+ngtS2;un9z{teHE+CPRz|hdRF@bkMth7P{1F!cfnU#-D&t2iOy=&+1 zquxuG1n<8is64Inow~N9-PEQnFZz{E)QWt4>AbvO=FH58TW+2?W-P6g>-7G{#zSYj zE-edRWA^Z9x2*O0#M=*(Pj1Wg+`0JSA<^9F`B}G{f)8)YjozMj`;e~s_L@7zFV3Iz z?f3JG-BI{x-8`eon!(F`=OnOds|h3?{4~dC>B3Ep4Gf*aiYhbaFK9gao8`^z>-#_c z`^zt9si_*AU?d&zgQ*(vRS4aTEOSXBIu#v(lWuBLo;h7TdUO4>4otu@!1QF&X`VnA&U}faF2$Z16%+1gOXyCx zku-tFL}1AjH?fItPMy@4?CfpvBx7-}#jOu(dKJqunoUeD#^jzg-7C=4ZFy({Gq>x8 z8Ha^*@;19nvQ=HkGCij5cB_;VH}AV0Pj*G>&3P%+`|je)Ba-T}<$G=}n`W}*Ld#@7 zi$KT-+siFFS#uQ^z$11RS`tk%8mD znoTE^Uaw&gU|?YAcodbysbX<)yJ%v^%kL6N7no;E*6H&|?&39KSTe=y-;Pg-r?*^L zV{CS0&5kq1mwkHAih40Haa@m^&1`1>CZMJNR^H=aa|c%05{rjIf0xakDIV)L*X?9u ziA<5o-!|#kQwEE6`&zZCibiy4`b#7>J^XS-@Q$)V!w`85|9q4+0opF)_sSY+$X)To^p(wYcM*K%Ukq ze4MEn{R~DM*u88Hl}>J)d*s&zHp`GV#z|bhGfmv2`qmyx-v942(}@)3*>*E}_`5Dl z{;aZb_KQGe@wQuSrajFvuBFLVHx8crUUY5q8-}+$${!j8PfcKZpOYj!(}A^OPw^@x zkKQ`r84fiZH%yt>8XDps$}_WcFtB+f@ZWrK%p{44%caM0YNZ2*>Le9RLXcmf{AxjQt8?K#XDc*99#`-2wA2@jU&N*uIh+Rz}Up~&eP(P=bgBTw2R zh4!0;$C9iygaz*;1l(D=%w&1uS*Duy=6*&;jzi4%a(dcX?N#2J^}YBv`EwQX-suP5 z^qEGBlzjsE=)1t z_tIz*Ird(Ru<{~y%pQVj;5YP zN1c^Z1C}kfHGMzt;$%0Ew;Ap(6R%j-1;l!N6*>p>c-PLypHbhyJCS zJ2ZX|QBYbl%Ykv%@8Csz4Ox#xvntsyTxN4PDD`lLn`TlIhgJ`xKnep($Gojq%?})4 z$>W$TJ?}${`5(sSjI#>&mnk%@(0n+3*MVc!cP4T_X;HSk?BT)}wsqPuU%$h>+&S?p zU!LR4dG;CJFzN9PSuhj5hx>|QPt1-m=W9NxtepkJECvDov4$jzO?kZssp3SgIJ65>7gGQ&(C2N5VZO1ao#DlL43tTzN{Zl1UwB|Ri6Cr3_2h@^~0HPnYiQzuvzel$uwJiw-zQq-aPhDoGB!dZpyVJBZ~#`LX67U?Pf$V{tE5^(Z-tud!T zQT>jB(}HVmAuIkVa_Tk2uq$vhvaYyxeRD-|+~#>-AM+V7W@$Z$VXp9JTd|5|!{$;p zNd*SRSAQCqWgHmGCbXGMS=i(0b4c)X**Cl0B5$vx@szK*Dv|X^@0e&o^auIbE5x6) zw5lFXODSFWe7NQ;lKMO^+ogU#7rUd)qsy%iE}XN! z*uf*WXx}oALmX^19C0Q;aCim2zTGVYR%yFl2@8{jw zk7o#R*eE?PQ&4E?(AqO`ML!nLIl0I6;LO(#7R&RrmvFSd{&27$p}jDn{h7kfm;g>E29{t07Q+`zdk+fm zG`6@Uuxzlha}{V|yshB!fn}Kn%R&a0SO*pb0mcP47QE^1D(GMf`Y_pJ#^Dsr7WWM; z?h!1`0xaGinC<^K+y7v0&y+p2gV}OMv(7GNvxa8Z6lT{NW{U=9R}m(&3g*n)jOGf$ z-W_d01#D>_j(APj;<>pG`pL))?k-|xPT4codX?3lM9M@A1&0K zM(3qVQpW+lG?GcbIys5DIT3Q*1bJ!sS#~i z4Q*K*o)bk*W;C#6bhM>TXv=!x;cw!=JcorPWt)zri~AcO{Y&|CQ0d7$55absBaX5;QzV~wy?Wr5 zxyNnY9zK?IkyU!H7jv)LAu;Sto}aiYYddyPl$oD(T0PHgXHOK0$$AmUlPgRNNN zaA9_Fm=SVd=A35B4?PzRgs=SA7#`CYamO=qiEHOCvjXcqCbN1c zUKD=jdGLAU_6dvHvIY38joc)*%v8wXo2u3R>deeFOZw~<^<|pqJ$^LveWJ*moihs_ zw8eG^7%4OxZEoZdXf)Q~+I4V&+ly6yxz<`-Sn9PvtkmTn%N%JI2MHFt3?{CCrX3|* zITxmt|7b6pz?Klg_Hw3AN6(q=o)f2zc%-cMINQ>es^K~L&xwmIY{e31`BgoyemOEt z$AP7U<;I*I>j}O_j=Gml%(zzND!wA{;Nix|KN30;Ob%}aNixW)P1!CEX7hIVX;(Im4m)U0<%fC6hr6&7^xU7}i zeRlur8CN?P#T!B;U-*XJF^p1SQe=>vpeVO!%L?m@lhwT~T6@@>7qVqHoI3s3@srQ7 z)RjfLE!-@CB3Xf+ib{mVFD`gcLo>5&|+8u%s_w$<(;w*xTZ~w8f*tu%qXM*NhWw zxi?Oyu%#xj1uPAj{Kdz6Z;0Q~2wv7u;ncI&LoT|`Fmx@LAvX2w1&^B-cK9lEUyXQk z@?iH0nWLfnI!2F{2&)~|^~{(j$|I7Ld+=8G2JLLV^s{dB7MiavnX+W(3F(hvT#htN}hg^UA*4VywEjy6in2o*Zvkazk( z^xth0HflUN$Yg0CXT5=C$(_h)okq!~d?`n5N+O)nzn-(b3SPt5Eqlianwy9A*kmLbJ@j^m`H=D-sw{tBaSjkDlnZ} zo6zwqCWEmI?um-MU>lJ5j&oiIXOP-cg|JRm#c4Y$73Qrtv}TvdR^JQt8`|rZwq#GZ-Kcy2 zM97t0OQXLo_1VjtmXdwLbH=^i}87jONL6 zoAm{IS=kmhFfb_oWZ^JkU|`T;U|?X7U<_eo;P}VzoTJ9LeZfI<4q>gB69S3T%=zV= zS|kj+r|O95?%SEMbn($X1?MnH&rM6FP7GePB;;gKs`nJ#8aW| zWu{rJ+RRBVFZ?y_mTJVF3R!t^x$j&hPou4`L{vjIYXt_SW&6!{ThijY=!s9{dXHIt znO>}~LzZ|>n-Ht?bhXdyh~;mV9NNw$svUA|4X4ixeE}^8A#J`x703C+^?11E#4Pta zczU`h*BPUB<74Lp)!Vjh3RFJcBkx~+L}}8KlUnPO!*?1LZ}Yhwyk^Og+~}qEW?E*y zJL4InFREE{{n*yZsV}ZBESu=%Ioo7a(5uT!m;26^?VPmh!O4GV|L2uue}3gXT_f;) z>D!>iPhV>-T(@Ug)grI=$)WG3pAx#!!l@=9n(HyKt+7p$t%cvwV%uR}Js&f78Ky|H zn~toW(K*REcsVb4&x>d;i$^SXe2Y9C z#eEORnqCgL`ZMI7$eq3Fcgr87s>gA752$5&>YQ`a%-y>7$nD%+cPo@9m|M1o6dyEQe#NJBXZw8rOexiC)BgqMKD|=v zAIJW>!R7JU8ypoH zIkqLL3sf=*vNT(BC?4S!TBbcghC6lI!uC0ZQ#3l|)plugDi&V3xFo{km()_#t2bsJ zSWu^RS@haJu{*QM@0v_mnXOp4sx;_ilt}P}s;hH@j+-s*TpN^@t~xU*@#NB}*D{Ts zg`{ZiUUR~<(_O27WteqPLrvhq+~^L)X>R=8rYD|ri(N~cAS1c2dV@xml5Uz%t2d@(qslU!;9Q9m<7+mTtNga$Lqd^va#fXJh7w$R6JN z@!zTSkEDvC5)W>@7Z!hOLo2Vp$H$FoZZ&&EN)t3KMFWpIc8eFAz5OzOYj()OwcB!i zCbGpZUM{#UiHGYg)2>%aQ%WxJR(_N0_;^a%v~$Pv-z%<9E32KobjB)`Z9bf@mr9Fr zsQU}~KR@|p!{;owr*cPS-7B?)a^sF(Usrhhm*ImQ?{fE^h%Vj9A{^zl@#v$=2hUmr zf41E-MbP6@L`Yot%_Z}{UE62*;GE^-ZV{0!3WaYK*e4|3Z57j4(U@^8(|7gtD?+Ke zFPXej+WR#x1m`f%-+EJ{qJ0_8P|E}(ecjclxbQ0AIBPahaag(?p$!56R`^-Fr z7CkP{j@apm(_bfrudk?Gk|{P*>Z;F^^%HNraNWZ_Z&`$rja402rM0ERijL#Q+ZJZ& z-U+-KJSSaMSCX;X)6?m}vJ1>lKP;$==VB_Va$xoS@*w&6sV9%8Ei&($nr>(7(8N3C z(`tQ9#VLSKY?=S&p+L34A<>?IDbG2UT4X2n6}nxrj62s^!nsN~SxTja#p^^j zOW>_bGs4_1EHUiezB~4v!H-2Ak(DJMZ>bqzy8<>4zrHXs0l3r(pL@` zKD%|0i>JIh=8hA0?kmF8oa`Zy8{PH{cp$?7SVR6fLc+U&w zii|iZy(EBHlZVal`w}L*wX@V!Gn`_QS#L!&CooH0a9}fC!N{8=z^r^@Do6QW2Ei}C z72*@Of?>ccvY{CM$O)8h%k0uKH{W6!xdAi2opvr?C3Ryn-QoNbW zoNHfC5xd+KaKvS9qr&2#lFO412=?7R@=wrwC+CvXsRAMDM>s>;Lslo>)J=}^QWesf zxg=SV(X4vGVYxpa+Wvh};HYWTvnx+vR=jY)a6;rkE{h3GZ}&Z5m8m!=a7?>V(Sw0m z>H`CZg9D3_jUrdz3n!5-hx@!C2bcsq4zQ>hv>cKPlZm~-6#C`Ul$|A-$vn4Xp0AQ{ z&|1>i;PArnRhdI`lGcXCMWSn+WfraI(l}KXcv&{vG00}~#fv%BAJ4|8FiF_m3R4$6 zoh-NCXtjRnCyf`_4-VteO%|TowuU z*duQ4ah`F2SY;{&v0_8@Z~$qYMbII z(Akrr^vi4QV!o;C_cJ`ZA}hHn=X*uS-9Lu??x`Q;9a*z~%;B2K`c?g2-I?=o{Zn`4 z#mchpux?2_&cM0+1)J%v#=g8cNnD5Ty=Re7$YnBVU`l@XY5M*`X)MsE$M2feel8Z3~_&Rx}9gX4vrH+2^G@D@ruJS1!_D`6KjVE%zb& z4~@-XTmhiEAlT%#PHGICl)@IedSlS zObbi9G=Jd{6}#LE&F%Y2S+xQVZctTVEEI2GaNw`HAnwq>bmIX3WsBpM`!4q8%{eTY z^MGNd^oE92;dgJ{k85BF5=hN_zx}kZMDaXrh9=z_N0B)%SR}5j&GRcr5}vz&b@#y! zECOGqGkvpb(L1uSUDxDbb5C$tsTxWbP0k8uFI}<0?p0`{#gT_`lY9aXrSn!?2$SBCz$*Kof&El= zDl@}eCKd*}i>-b&dhE4KA`gtF%Qi4-TO5Ar+yBpUi6gtH2g98dHme=t|0dM@TjszL zdGAx*aUMnvivtYe6;9%58^3L>SK^udfJuGFhfcqWMWS!#xr#rDWvCW*65oBHyy~ON zVkL_{R~bh2g2u$i)`=3OLFzsVdQQqL)*oVL1ZL`f4dOQul#vizP|iC0I9vV!wmHjL zB^a9&G+4t9u$hZi>6S^)TqJ&fBio6M;ttomCb=c%sWRVM$`pKz(dK$_>ao zT3m1BvN)97xt^QIUQ)TNCf2Rwgm@XR0;4Gd=QD=di`RK?KHxn!fnECnL!ww&m`CY- zu?UR^rPtW>cJJkzz?84R>vg@=*PZwI1>SQG90m>yDIRIbWqp^*YU9LPxj!(7 zCopGEFDv0lFHqws`NkUmplp6X{SR|)-wo*s0qpG+_HRU7eHIy}Bvw>ile@BM9kr^L%dJ}@YJV4eG*Z1w@p%^SGiDDdqr;9z8nubEMn z_#^755(s`?AP(UY9T2q??;iagZfe@McBdb0*CM#aSmL zXMZ#~y(RI35&N`4jj0z|s~Kj+USR$4gEepgYiI$>k0qRr3pmXVNSG~XHc4PMbeL@; z!F+3Jto(vl)0x$Wzs-&iYdP|x^r85)yJD@u(`%E|r?@PiBhrw2!>lGYvoEHsFZelw z)&=&9|2E`Y+|GL^g7@wZ4*mzZiN_f(TQUkT-3jlKTw<|Gxs>>IJN2 zUs>D)Sbus<&o|?a-!Oew0q>Fx5oSjsI8SP|X3E80O6Z+t%wJX@Xqk8;wNa;ZX455s z&PP6TmUGH+H7&l$8gqd)L4Yk`Lwo#%IKHnbW)~Lez6!hL8otAGcFEM)+r^o;PoHhB z$>^ZLsI;IjW?QOPN6o39EjPdQoH;q?d`4+_`Vx1K){`e|dmuXUwTn&w}?aY#u+7xA_Y!H&eW*Xc`ErfNx*Nj zBGbc#$BS6w9TvTIdi4uO)e(Atl7DyvvsWkH(7;iH_YC#GFE0n@x!CBwpJ_; zju*e?EIai*_vDV2+wE;`nNz$yde2mqR!++eVV}aMz})A+duJ!_jaPH;JeYg$W=ZHR z9-)SwXD1n+>|%IknU?soR5XC4><4Ge0`B52&1OZh1V@&51MamS>hhnlNDHu5Pv?$U z5ZJZK@Ww>LLy0ntg%Xy{3bw+Dk2WdrB_;BfOUisQnIl-p!RR=DBYU54eD-y&trxiW zGVmxxZ`$j?T@qd{=P-L)`afpjuS`2Lne`uR(wjY7`$NjTue106T-_9!o4u>&WM^&6 z$tAwCYHqaEcypCJ-!bKyXByuJ2HOif&ja}GI&8gtfG_0r(wo1Qp4!HAQnM^Ly)~?& zmNkGea6;Mq4Q2Zn_}^?`H%nmMdCf31fhGC@>y`=Y3)$JEKd>@b@G_Z+Tss^QX(ZcY zWf?uu*lZ)4mw`m40c&$&MUT237rWx>!%S0Vu9Vp2!=JcO;vswR239$1?yVoV_a|&p zvgX;_uxqaYcjdRY@d2ky~OQw zt@G)YtC_9WrmYbXSj(rtnEim~-i)PB|1r$Fx`Frphox^C*4}sEJ*U9T_hD*^TW**~ z@9(3U49DduH21HRcH)ctTIDz59VT^cL^chOIZFdCxy!6F*R+zaTzk8iTF&7GZ%@ zmJf_^587uNaPQl|9lOEv%Vd@I6qY#w+-n!)Ee>G$7ihqJ-GDdBAj6}H#m&H?Z)?Eq zg9oY)iZ^d$Ygatkm91|7c>`Ct-q9;engt7GD`$E?&XlQQo$I)0QTDDq54hG%IHjS( ztzpBh{brM~HjDOw0zI9>w@P>4-r8-lfXQl4EI-4Nie;(y&GrdDU^sqFh0$BVE7BJfwS*h%J{7_c1 zSzPUFDA#Y@BvF%@HO%H8nC7n06V29`s?IKPku}m`(&C$(TNC~rQoh5zN~gU1Bv(=T zt~K5)3JH3*SPx%a8f$brO#48X%!I{y0kf?aR2v>({`ZAJM&QhK@grekM~?U$iQYLi z*JH}b6?4x1-0FFX;f7mJ*fR#!4-B#atj!7BR~GPw?m8MD#d}qNLq35qF_-gf0;k6T zW|I#L+6C+l7ubTn?}<6i=%&CiM}X~>S$W_GZI=d-b{31pQ&@Ztu;-!|=M6DL_ZpA^VaU|!-Xl%vOa#A?A*J^8r@xgN3!KJ+PZShU6Akg5-N z!;Q36TJ?(q4(47Hq1C06FoYOwA3H@t06SFq= zy5Y6!@C*vJnT;+doRlI@G%z=Gc!qFQA*G_%okbQ8?oq_Yp z1l}tFToVJhv;vryf8d#}uan^QSv7iRKvZW$hTv!qwaIaCQXWz@Z zc>(uj2a(2ZmX>SIr5lUZY*MS)xMM=h#mP$)1a?OD9X~m(v2k7?yF%?ou0~do#HIzD zY?~8y?LWZ1bpiLAXRJ{ItiA%Q?@x2@PdK#o1M8YyOil^RzJ8mHUvJ8s$X=_!EW3co z@B(v-0%v~%i@X5)4u*Z94UFN}iXUgDKG&V&$kTpAC&K&el!6(xH+`>Ox2fShz-V$o z?4MQxqeK7$>jMVyhAXlk*h1&<-dezWF@ejlfl)kwx!HjCv>Wf$gtf*K7$*wwo_(myH5aftzTQ>GX~iPmWBe(O9k8^ z4UDoE7~lK-+xoPC?|~(6?F2?|2i~&*%uE3c(UA;t4vQ+dI}!v~`RhDBC$fa4viN>r z+5CZf&4RLo3vnCjc#98Q2;3xkQ1nsGV^N#UJ1h;YxUxH=j!4)oC~(?v@qLbB!jYX~ z*&DeVS(R$vmL6EX)nSv$Z|-*t{O=U_b_uY(`rPRJj#aT|SHOW){>N4s@~jFDV73il zZFtbTG=N$5!n@iA-Wv>@@(rxX?|4cB7;LIi`3+bcZQ}MyU~=m<;{Vqi!)_RGKKtat$XxdguF6L>E^;F(as?xDb}`QUkj-8_~9j1RtDE_lFm zvte({nv(3dY|9;ZEB-xr&-s9XX~Xv7Yn<~BaII0`h+p90?+|m(FktT|*2S~ee>@&n zb-es_ibwE4mOsn175~YcWn5{Tak2YjjIi55;n}Pu8*Z7*6UbZehGC-XgQ)7@xSGl=gnU!ZwWYa?A05xFbvM&1>DZXW+AC|)a*D%=wS$L`pTnkL!6J4Uo6wI}3Lh?LRrXBPNtp2Q zg@_y{lhmA8hXsze*jNk>bd=~xZ!Io5n_B$r1TVjYtX@ooz>T-v=4a29T8FQVwk*4P zDs;A3roXYf495!J2M3&l4Q5-^{!4Q8o-$oyR)>dU(-L-3P19`>org|_ulJv8QG1i& z(S&vR#S>%#=UQ!@q;hDfhNF^dr_i(DDFKagwY&mLFE+;TUsC3npkPoKi%`oGV>0qK#3TGFB!RlsU!mNFRFLv`Bl+3MFo)?F@^jwZtrZ+R5)C z<)k9TQF&})kV;`Pf4V?1TQ`5&j70vdRU40XNKRUCn1kJFMIxugs)FV|$6p!-0v>sd zDP2rl6%EZ&zcijo&R(|g*hHR$f@3o2(|#=APhYW+jd|<2tOh2w6*&uys8AVpG|s(8L{8v+${~-T{q8P5cfIeYureey_DqnGw;q^QpsBr>++ZqLV{k ztZ4pf^{(m0!fP^l6$@F#F3f(+`~4ELn4QI)isl=pPd=0?b+IgTT^Lu9czj{(yd6%H zbygi<;?~x9wdiI`M&qMq3Aas+Y!cH7(xjr9A|DCrFYjbNBK4#~O*(qohQ$6LnVAQf zRAn4pMPrr-3WBX<(>(L+BzmxJmL;ad#%Eww}Tug)QzgyTfs~ns-H9e+E zTLw&>p=70ct>T>3>vgXgIpmvOyxT1uQ_#4_@{z;)y>Vx&7hnHAp|wAv@!e@92ZOYk zzG7^(jCbcovTSMEr?u?CHqn-Otx8Kob1D?uq$YHnbpNUGR%AJnp6KNY} zBnoK!-jTo=9kK4%(j%)LvYpyp@REU9OC|H(%;v*$7B;B5y*V~5U9S6ZBh#&@gr*6X zyH4?@Z}_-rcJ96$Ny8(j6x*eOIX)g1%-Zy@ExGKPvE|mqQnwcz)BN{oLi1^#1SZ}I z4L>^`up02Z5?=e@=#>K(S~YAOxm;c>xSsW}JJNxXD|B_kO{R(6xe`p-$2|S+%y5?3 zE&1GHO_ks36~S{O3fryk-E`zDde9`cM}c)ulT6K&h@DBMA8o%a^pM?EFoAXBBC+s0 zM|D=HF6iRO=;A1J)wO##vFN4)Ti>h0#`79mRc)H5D4FikIP zV5v~({8jo`T5oZ@xk&NJq8W!}vi``Q-?zos^6Dh16)zk)g*Ht461aeoQ=@@#&OZh= zzm|n&VRu;FcYJ8oaCyibGl7>=H`G)hWC4Tp4QIV|jvRqI&(1bkl6WUzsaE*ZV9t$P za~@O$td^DW3)iV|mObOV^Tfg9w(kWwYOV$7{JM~?6gy*4Q`JMRFp(0mtrwKUS8rf# zW17;%d{l%ZMuA0Yh9GBsN1O4#gckb=O?r1{v>7WHFDmk3?r&2F(y~fm>2PG@PtR!k zGcANUgN2{3;8fI%}A4Z$jH^uaPf*cW4v1egTfCdR(ruVzlx-w-3wgR z4+Q*kjPhA4T=i6egUg_Sslb7I$qYB~Ngp(?EMVh27kEghjd|4$*>tX&*?srpRJG;B z^g7y$8@c)_=2hJ)?%bE9wDDd;Ya`DRzJw1)^=-m0__e4lyuFIaRicX3py!eJfr3`; zV-b3bS1e!)5%5;y z29^&EVi7A?|Aid%opWYqA&>U&Is1OH&ghC-T{vbVH=Xfws2e!edZwFmbXxW{ikceVTTrp6Am1v4-RluJz%%G z!XV(@&?u;M(MZXb*{hTDh~Se8d(xj+?z#J;nXRjQhC_w}YweFoS_M+QMGi>}Vlx<3 zb~*Mg){+vq%<)8!OQGT4Dt%ps3)Jka9vw53YeAgw{Y-6Wm>%o@L-v`*ZHmFOaD~eyc!7OCQ z>MYqddDBX!L#fqg4u}5>OtilJQ*olB^*pORj(pzJ?g+6ev40P+tN)KacExfVd+`Z6Q38Xe%NDSkjR;*(dsMq zkf ziTvi#Imx$=+|@7D)Yvk^JvZ}!z&Z!6Yyl?=Ge6(Lf}D3xzBuHCypR@%dYXLh-v;*k zf``@$vHPF2?K`>5bVBu-2im!z|8|zVy~+H%K{_Vv##iz7OHE?y8d`O07T#oMRo0&8 zz+!w#K|`;D@k1>?v*eWqzG|OV!32Y53uMNlNB{T3n zld$|a)p5GPGfHG@%NLUZN; zJ)IW}Gk-MeceW^JFe$ENv&;~B7tF|0(QIqQD%#N?mmrxwfsy+~^Kpk(2M(6>gqe;T z8l*fdb2hMs&S=YD&}J^Nm2<&$rEtvOnQt+h7QyXUn zH83&?^ss$sFx%1kzp&Gbfr;tI5;g_~zSjmy6-^o!m<%@T)lXQayMdAK!Nkv*3|=h@ zv@2TDEn4m`(0RzvwA-_HyJg?5$f^4TnH46BRvbQ9R-`Waz(!t2_wuSurzH<*O7E>( z)z8An)>_co@`E*AfbHUq1v?(G+>zFZV`!dmU%?EHB0pKJTReYJ5zlc4R+SmMZ8aOk zBAUECW_r(ZRp_u@c55ORLnFUJ6aNh+ffuLvC776Z^rXxY`c>VatI-l8!IIzPcB)DG z$Yw>!BRWzod*pWUz5l@&m8pJ2vR~fBT~+AN#nVlu8fPvU+8T8?#RPCX2;xXwa9oFL zW9Q@6`@g2$T|Kkm&mqwQ#&a!=q7xV;J(_}Fu%%C67FDqOrzId1ovU*`~VDT%r zgC|cb2q`HgH5Rt?*X&;Y)5BeBF~5$+p-VTXwJ|lvMa*qvYKs?OJ0a4x^5yEdisL-5 z+@(J>OuXQ;SfFcR1LMt^EV(n<%p(~28W>eF*orgSR9-NYFJyB%@sB~$qiy<&6Q4HC zRmfmqZ(zQ?()Di?bMOMT^oq7L3s#E@>upXjYu=d16u@Y)fQgG+l_#RvtANGw0VCgq z$%|K>+h#C}X-9+dhvw)RY_SpxV;z=;?9$8XXy)=bZ*r2+RiXVz#mp!rcGnf>1r)a0 zN;YswFmZ7(Y??EPt$~36eC6?!hK44cWF!E$Ry%zOEwF+~nxRpALnGe= z#>E023pGwCd}Yqk2wKo#>~x@8ZdkIC^Y$PxK^>{V&kTZ54iLUss;E3y2L*O%EvN_RxpY(`0_h2 z1!uHb1~5u#+>okhaa_T;|3H(Y0~6-~25yTcsSV+_FBw)$U=Xb6@}JZ&p|Fc#gWtc+ z7dGhDF3{wRWxrte!(^$icB@6hW^M~^_6h}_Xa;79wge8gmPc)66+Ax=vgK5?@CBG9 zJct#T!90Hv_xf9s1`}BXR2Vo68VnfN5>7-iMKmnvwPO1a7d+J}evy#01xri?+X`Ez zBh|Me*Sc+=c}sFupVXPFdyY6pePld)c&eDkri5CfgD$#e1mkF_FV>Vr|$X^$-p>+fyaUA(iGvc3G%-R+uT<$ zNj&)1kdmU!K8t}lqcwt~&7gwGSzu+(gQT1jto#xTe9re0Dw@&_4*XYM5c-LE-ddr1 ztZaOr?=ARx@9o{ViJtSd6|b`0U}ApSroxe!k#kk9;?@I?_^UG%KVg+kw!tS-Leho}YA3FA4 zZ4p25;M|ilQP0ktJM%z7VV~#^2EGprd>RpjCX8YRjJAKf#qG8!&R|I8>&d-6dB^F* z1BZqFOt#WG$RPKl`B|piv`%4zjf=fMFl%)*vhTR7nb#WL(Q5GHfYgrW?^{^CI~bW) zoSS3XpuU4O_n*bmtc>u!=J@%vpLbp`kVL#}x~=47LRA)?<%1duXdDwB#o< zE8JjI6?mfgf>C0i`D#^?Vf&EkS`oDP`FL|RbytT>n zK$G9PCR2`w?3|tI51MX0YGwPE8I{4RaU!h#%SG`QK8ruxmjCPY;Pasc6^)`7q7&*E zg*6)APh#NT!N7ZgfhQnMQ?BX2fe6N@iE_s#1-3q7vtale#Ax|prG6&+OkdUC3)|9W zu-Lp{6fJ1jD8m-f&}uN@x$VOSo^6e%Eh6|08gwmKay@ziQra_bv>xBr7W1N!Goe9| zBb2$LHGBdq-{OY1b07cn|N166o%^hQ%*WB)&XtI6=!eklFBzDZ`GLZWm`SYl>177hq`A zeURz+k;N}UO{pMtVnbHZ&p`R5smsniJOO=wWu@ji7P z!}5l%%&=pDtOZ*RBwved3c9fBzwv_n27^_f*rs=|MOU;~aWDyFutspSg)tmV&+Go# z-N3Y>L5YJoN`QS`*NkkBW!g>}8VaX>1#jc3U_JNFfNKH6>2t2JTMBOCWcy*w&%iiUv@P~TTfV`^xQ6zG6JHY;+QKTB0@pH3 ztu5phZwvdNnp)iy77%I9mcty;z-OM%=#|!7IInuPm6U z;&q?KaU1aiqAIUNH$DAy;^e1;HtvlM7ybVAy))3CyN9!Bp+M8Z2VoN#POvjh)Z_6z z@-tBG=#Gg7Tt_Njefj3}$%`>T?*`uohW#t%aa(**^k@{)VB}=rDVmnFdC4=+aCruG zhQE)3eCN-%kW^T)XqEoQwwxX7rd(u;ozR-l(Hh3U$dpjh@^^{)KL&${Z?PNNvTn2* z?kk(!&Kk+V9KWg|q2l9m|Mb{3N&A;oMRc?#{wQsIXYi!8b!B^N_=nf`?%Mi3Xk0$` z*zJ9}S`od)McbuowwF)-^g)3C(Xy2Pp-P8p-W_`LX=d29ee6%O0$+(=Xc1L-*Snap zkNfO}J&ck&9-I?NyzW@hdw56Wy)!)j0!2ThCVXo+dp+wP-wuX&w=W_aUY)9Ckeisq zmJqSoO@7M?mjjpe|5`D1tn4=bsik7(wVwHFUW7r8MwPZ)n_fh-L_$gQl>JN|jaD;Q z6FB7lB-t4g8h(BF z|0{e)fs#NYk#FVmi->SSvpQjy@mG&_2C z%fYU#uV+1o*wFBHR_MV7#)Lx;Y!@aZbTo#qPpMtz@X&Kj_{Nm2OtQ;DH=NoU*nHw| zR`F_e|EPmnc5gao$eo(_CE=1Yr+}uzGVke5ty}_cE;q7p9@_hxPu@C@W2fQ=*6!v= zu2uDyFFgF+7Ir6M$&Ny0u|Ij!bR3<6(!3P*vDNCm^viNK`(8GIUM?>$w@=So8rE(4C_wDA z!R7`Asf-mn%@|olkFfPHu9I}QljSe*U%PN~?%hr)>&YTO)N2G^O3 z9SoKqDGt$CIJfL}?sMmI#}4&-d{1{?IOM>sDH7;%T?rTd7x`{!oy>d z-Y*&!@apnLD0peg{&>y4Z<&gazwt?x<^a*E4NgHWB7&?U2`!9lTjN~7#J7uGZu}vtJqv?AbPtcua4VOg++r9O4wYVRiK2Lzx2$m>8WG+|Jwmde+grF#eXQVa!Yn3Xc4h9*=xXf2dVn zkN5dy6k)LU&8Kkl8##y1n$=e(_fFq<#c8_!zQ1BTxeCqOM33bk^E@bU@JQMLVQH;+ z#=?zTlK#y)wXtPmvxrv)(;?x8L!VlB-3@kaUVQ%H8kUl&QKw{oJt)*_;kMvevc-{O z<@b9l#Wpq`b@>t+(klHkw9!>`-7-NljYM{L<<8q|F3jQ|o;b3fn%crJ%zNPUY2G9ztpkp1=?9uQ6(%&u%7gx zO#umkrk>nq#tT0B7HS>dp!;u`nI?0H&Y}gKav4s-E)`BE{lc1E4ZSQcOypZ~V)4G-;yEi6+~Hj%=r&HS!BEaxGfG%ISHcEB2jG%kDX@A|6|s%oZHtomg;8QYe5) zdZD-5nKN7_`Bxa&GEx~$4qbfA@t}c8WC63+k7nIvjV?P4PR9G(aeb3i-g_yOS$d0f zuX_L0I#$M=RZWvu!dA?nu}gym1gV*%9|mE11Pq z)7|pi#mIA;7GLa*>$0A4DP;Mv;(6jr975O&XI+-s)MkEXxsqP(;r~`l2N0^kX;Adb)d8;Zp|QEkvR51` zyl&8{?QxJR=z@!9&Y@t&ooiLuol*nplR^|PIOI6%25b@&U&HTmuvvj;Vd&ajoc?Rd zTlTzh5$U+Vl78YL$Egk%PLT%;Y46IJ{!O!9z3WiXv~BaAYMT@ZKKu5Xfoa22CP{}T z#Q?+J{W^sbFMQN3Be!?AM!HG=fAQvnPqR(riD!O{3d-4!L}oo|VPp}NU^wbg=riH)4xJ?!{&xMT|GPi= z*ax*TJ!;9>XBFVn(Pc7gjc&M!r@$tzB~3i59N6SFm?Sa;cL;215?dqCsv-w8CnId$5WIj6NJFJp+2-jJ|EZwG_8!2t#% z1NKmjiR;DtWGB5y>W%8j6mO|$GT=~7x*(80KklC+W0&ylt{W$qqD71k7;rGNnPm2h zE&j^L?-ji%!X}YjbnS}bMam~`%zJ<0u3*1_i`E3OB@A@|)0SL`=vvjA;#kjUV_w8u zaYuXiL0+#1jU~bzLIQ70)XgOYHWjIEpYKxOq}|}ilOV80-@|mea!9P+t^{UfgFBpY z6O8qD2DCUFVZ6Asp^-tRHte+F<(bm+O&A|#Wo{H%z{py_utTc3k$cUZgR(aq*d{4- z9X!KUuzSVPw!(c6vlG&MHt%3!^?QA^Q}bGKO4ns3mD4#Uwu`U7eb| ztHn4&MTOty!lt&=W`2)y^-HX;9@H}(T+F$YIdYMz__9;nQGX7}zDYQ4pTiTYBC(J& zU`4B*wc>q?2nW7Z4UYA?#~J^7IWP%xTxX86XW{%&5bgY+fyH9MZ@!2V>c$O@yjcv* zDiVsE5gtu4j~ZGnY8Gxh+Sw|4U4kbuW?5O;Du>IvcN}w=sl$_PL{I6y1$DgxS z`WPJJmr6Xr#$y`6B;1$K++p;alSfj=i1r3G!F^|IPx!W+0ek?6?49ghx?9>hxe4AJO4Nuc@?+=bSAWF z9G!hPh~w%Z{T?=(j>Do?Ci40;o|bSF>2T6*xu8?yC_LvNo5?|?FAdytIDA4b_+D}0 zeZ$ai%P!=@C|%Q{GS^G?f}@*4s1-=k4P zC#Jo|QTNLsJ;p=<>NxuPGV*U>5zKKEZ#byf z(0J(UslA>jE-rYm;X~4_y+^!LI)YpdYBbE7v$NUg&|yW5c0mtEUWrCE4@Ns|ugE^)d&3X-t94kDTTW$zA@E_f9lBuPk zV zJ@03V#v~>8p4(GA6V}A^2AQc=g*A$(9K0$Z;1%#x=ZJdPr9~bbuH}Ch`5*Bucf7!( z(9ybrfnCK_|7mJd( z`waD6Ya6u}-4+QsD5jz%v!+oe&#`8kQ%#*C>l6;J9u{ANsmya(Grp_aOHj27xsUvm<6@$DEm^!@#W2 z5LUv<=)u6$bKs~!!_f@asjmdKx5%(|%<0i-$j)d|zH`XrivzESfItS*tXm6;9MZEH zh0e`5UzTxdYp&Ao9sT88;eQic7KHUy1}z9OQmywnD6(g%$OcByBMpKXjx|ht9$C}a zXIyH2)53G6;YQ6Qk1aW8!a5AB?#({iq<=MX?nR~!k4C{W_oJm41v463qLS5B&NDso8`(1m#lUzXE=f9%7L|?9C&XW;CqwzLDuofyam1`4Lm6w>;jJJ5z2Gl2I(#k zI4sF1bVSvp?qIA`?x%=`rn{^8a^E*@)!pIsGP37RPr$7mNt%p1R+~m%X1Sru^2Wi^ zg_(bW%nF4Yvt%0X#PRg*SbfW=fun zxj4m*M@v*e{M91$dWlscK8!K%-igUL>7H>Cs6Y5-QOk`r4rdLX?)F?Ly1`LY;h@lw zgK`0k%f1~6YG6=rxcBLrq(lkF$DKYNE{=2lHeF2eW}V_1dG|z5%w!YCKIz4*yi3>1 zT<7S!z`V}_FV&V&DWt{+zoe6UM4oOd^)=p4o>EynwWYmA?9 z38pyK99yp8(in8+tOv_M$pej6ml&llG-?K@$=;c2Jm-+~3LWnoXW2^}&vgZ8Z#bk= zz$hr;$g9$LI;T-$a}-By)B340EK{yN-tp0RXZNzHcbX(uFO$svwJKCPJag);4kN>7 zLJn9?m8%hc)jsaO}7@-L8S-%Rvhh zW;GutH47&HWG9mtChiso4uuaaHy9M395S&`&fL05?ZLX@4Vq81*w3Yegg$VuV`p&VZi>0fc0=gK(0(oy{ilR<>C!O^syjxwqD;e7>nBBpRZVX)*| zYxY#9fyb1A$-yB?!eK+xIoXH<@-EI&UJfiA406S+3@03?ZZrO7%+RFy zpo!JqSuW&Y(sqZsua%5P7!+q5GKn}S_i|nD)|KBsFl2YeE}n2cwA*naC$E0~B5nx< zgMNqqbzcMiy>#WtdUW9sY`Jk!WJnMwA|0ro3L{J$KK(qL9G za9pR}Xl~_X%EF}Q;G|%|EORD<_s9XBGY%aM4uTh)Hf?s|(5&Y9<)qJWZsoUsP8#(?&LJKTC;M5D!r@Ez-V)PK~Ie{FJ@2ch}$%)Ym%n-6qP_FS1!?=COJD6Y@F@JbALy2$pXb>Hh#+$UannV zme1T2vZGJuVC$x|DC-ZI>j_TG`9N~tyJL0?$T4ZcoloC{h3^blf zrcasZC7G$Cn6hjp$Hs#q(Hw;?f@&W;o{3mJF=(COIcF#9^z;`K9!^ZJ%3xsRP^&o5 z?BjEB8Mk!Kh1DB(KRWfgLt(;>)$FS4RD8S-GksoYc)Y1XXVb|DkApsfH$J@Kys%2PZa(iWLjl%spNvb9w}2CQS{@kUYQV>zU}OQ^U+EnOeBzDjJw3%1C(H|5dPB zZU0|r%L*kn4X!YiZnb{}3zo2(IBj!eWmvdOL}$*z!;JE?9FH(|S7xKJ{*u- zl+dJ@64A)Usq)}Js{%`g21C-n&{vCR1@%QZv$HCFIM~W8mEf?SMd3jsyDUQ?OTY1~ zoi6E;87mg^amr=730i!5u~_nz65|me_p%8O8TqCpIJ3Gh3OvO4_s!=A$6`2EE$OvuHXP(~R^mN;#K31`U*|NQcJWg_i7cEp+53&pulUfj<>IP{M`^Qi+j6_+ zeve<&qcXvub?QYoF0TS+nKJ_H<~kpEBP*I^9x$+5$|SNKJ=iD0@vz7Ch=X32v>%Vw zjusDxM?BpDZGt=txzt4(xfe^cs%JQM2bLrW7cd`OlX0|>U8I4Tt+B&-+tf9Crm8py z|GX+FxoItf#R0+o9WJslAKJWPm^efN{)I6~C^1;fSS%vDp@CUu0mG%6&+Rxm9QdXL zbB1UvmJDlb*|Y6x^G(Y|l2a{?YEF9CV|qZ*a{pvUF1-m%k{c9R>kHf@6*lZq^jyH# ze&DELR>CfgNKO68TZ9rAf@c_ItnqG?lscZLprN?qL5+%Wcgrvp|dN*WfA|;#7TnRKJYl#H1HKq zXyUaxu!PGjQE;ioLgRA+?4c}7Y_30;G%Oyn26jB+e$VL0s#n0s$$ffRSSY9}|1R z0T!X3%^4QUMe=?yYfTE|dMLYst#ZZzg;|N6!Bd>vav6?jYE7J@yo7;SVg;j22m?pv zn#9UqoeeC{3^-$4)=#Raobhs9d5c=AzjrGmlc?BdTSJAm;x|9zHD3IT#0b{dO0(;bc~)o8Yd9$CoI7~v|c6~M?{vyo9Kpiz`f zkySYt)(9EVIz-VZFVCw{r z4230vovt4mw(r?`g1PHU!$gMf4jwK75~>2a9w!>5@-8r#e)^Zds8o7|(P}}PaoPih zUbO>?N0Jzr#T1^YhA6q5aCFkt3Y@Lxbx1_wL-X5&56l)*9&%VMFjCRl*y4TXkeK0A zM!6-7?VCIf= zINEt@;}+q6S(>vMj-{o0?^v4iD?!9+2P1cS#v!3u1uV*E4sZm;ew3SYvB&kvRFOp- zu6p}E_GU&nOV9jpc76gwv&4=@{s|WjvHKOY2+w)2!RN$*)Ih~{ZiQ||R*54fN{Jnr zM;7v{7dWitFktCC@Uy+{^zD=W`*gck$S{1L>CdC|uxG~ZgDh7Hj=0Tq|HmL2uvpT^ z*n2tYY(DU$6;$Plz?DFD;O%j$$>}m~-$Bh{{JS+|g73^qH4a?<} zesEB-Tw;-%dO>#Vj6-a`t5XDg)IHzC3VAnfT-9RhX(!Lo_&#B44~I|Ltm%;lc`wHu zp7Z6o)W45CA3Yx26r6G)kMW+Okk1209=i>BTzao`%?z9-KX|~C?%;H+UBE>nWMO`Z z1+#eY2Zt*u3z+0s{xHicI33WgV3L?|fJJhKqryDF_Nc2r6PLjEnP zpAsQh&>-y=((4@kNO1WMW<@zip*yQtPplKFVdtN5fFK#vRNI1Vgv)t&63aeK`o!jBV%NnMp zE#nf=|M>0sG@19)su(z@DR5Rfm_1k^bZr6KwjPd-1)SS-xaJseZA&=)is*3e0Qx*rFW12PrV06JTRHDEY5}naAM?-y?-S ze(|Jp6FxBHD>$7~NZKxQh@W|*O4Ms5uC9on1A@~Q@aHV#w|T<1$WgHDsY+0T#pFJ=blHuGG?e_iUZPjEk%KbXQ9xuh!9FXF19m)4=NS%;=I< zT2!ZW>`K#nULUI_q_281Z3Y9ULO$maHNFKCxOXw|9B>e@VC1oA6t;2{Ik14Uk6HBD z0gf#TI2S#T|L(xO<|NlA2d*#&))WV}CoS59080t72#afu;7Aa{hdZP7+LFJ*)szV$z9He%a!$SUqkDA&GSxXu?1r7>L zTG;odo z*`p5JObB2LU72?6)U7wq{d{_4rY`d;S>_ey%%_ydzo>ybs!=RLu`R1ntZJcH#X>Q+ zgPfrUC1)JUyur);t-<=<1Kt}8_?i^Hod{s9c+VEKz^JO9C!2w7k^|o(0nvm9kE61g zj;vw1qaZnnL6j>|=tTpwgcB3rkq(EvzmrrWRT2bVCr6i@R^&OZ7^A@d$Dz{b!HY8o zSag~gWEhx@9*E34$ScA0?}bDYBi{iu~Kr9qL5#Lx`+e& ztp)t=9QdxZ{@>@vu=bFk!2+ExhZ$ET2$dX>Sj_S+>%mGDx1cyK&Sw z809^X`me_twemw+8&`se+j?HN>@>5oV=|QinGFxv!*n?B91z%}z;)n&ado43-9nE_ zMbFHI!d(hn9`6MoBypZA1Uy1cle_=w+|p5WSH_cH#l|{|SPN7`SXci(mLE9;6ytktjTC zi@=J5BKH!w?kwPXw4nSz604j$Ys>+Eg9O&J24;z?KF?M%TO8oZQ#=*GDB`l7KWsIN z)Bzu#hQgEsKW3zG*fiu_YRa={$a}GjL8_g>Y5}`V0*6HdlT`!niU#Hz3c?u&Kxc)v zC?^{;^5-ltEz@9QbJSa+Ak5*&`iepH-@HbtUkp8`oLC%IDcVfUS72b4P-6UcQt;3v zF|UWOQY>41Qp1=G9x&BBX0kiL*0z9oMFX!*1MiIkY|9whV&T=*v~BBd2m2NCeh(xiF(li!E*|%J_@V}4a_ zY&OjAOt0G@^ZgO~&m_)6o~QRUh-D>;TZqd&Xp}W`6iHd($$RkchXwrK7H}YNx6RyWsh8!F5*3z^FXpFSLjTl zkd@f=d5YpW4<#oo;PSGRVmtV+t;O*~M)-tp3}OsSVhxn z*fWKd9FAOSjRM~q1o#qJC(UA8*T7}c$h_sc)B7m`9En_Bj$arYna#3#+}N319x!yY zL`+g;6lvI3k)Ko++vjd^Go&~qwI?Vf=UnBL=G^0SjJN3AgHuZ z_6_3^$2)S%64{g%3bCoNC@HXUG>YUg>RGX7R3)%W?RKtK*!1z5{*edo1eEh@6>3gA zXY(_0-8eBNLZR?=%I4(`W}9r*x&1brns)SpV9{0HdEslgwls)rQ{b{%C>C*0*p89w zzyjekjkX;P98VNno;_f`BVo;z%40o^&1wO&&?e>?(mNNcr9F3MQ&AL^YZM7!6nFaf zP+-dg!7@ehB1Pd>4F_J{X503F%T9r9o5SLXY%Ch9nN1Q|9TGSKWRvIr-pP4~*)E}! z_3lFZ1?)1n*isJoi@38*t7cu3FwyX|B#$ENsYO#4X4}tTXAU~Rst{uz%y=wOM%HK5 zJ`)GVlmy;S3Vbq&0v8x>IsTC>O9PZ?b1o% z#rICKrv!jgPUt1_q+z-szlp6e}p3zirGyT zTi_t?%_#njfyJu9J9XND*Dn{oUKuZM#WunE$;9Y$TnU^{8t!U@i~eX>`MyNx*Y0@^ z2blBrefyUnd!YOt^Nc4|xeP2bRIZuM+GMiJ^B)VR#_7wk2li#}UC3U>I&JP@wTHal z7MLh9vU#L?+%sm>kXfQ~fL$d(@Jz!x%bMkD9c7Ljlv=~UujR;YwL_#J>!lo{c!{FS z5l8Vg4`mA)*(3yxA78`#&P04hik{>yy>`)RrP~Hv$sf&+ojkfAd4UXjUPGJi;#0XM z>$o&xjtjcqo_XrP*DaL?^!#Sed)^?z(a3QiA@@SEZN)g=$XZg5MBeG?tfZ7@h6V2;Wq-?UT+_;#`&M)Uw`NiNC^GSXMhbbnkL2Mlo zD;;VKw!Tu%z9wy%_aLK@nY~-c>r_yn($dVETfHWjly)yUK2h0Wfbf{&WPV!MJ&o6GTaLmeR+bL_E4d94 zOo!Uq-(0o-v+bi|+myE+IVBF47@66)RU=Lm2)3MCY`b#K(np0K793y`7V_(0XyoiO zVfIix<*;QFA^B#&Mpx;sZ{@9Q?Kde7fHP4&m)4I90_n!`RXRW+~Jd>$Xtss@1(6?oV;}x7i>Br*>gV%7&0ZCrB zs|n2_AqNaud3W#Jz{IP);M*Yq8H)ghe~kPh35{%S7DgQWnrjSN80B~jTiE4d7P4~q zx_o3543xOoBDioFSCilpp+eWP@POs5f@UcjkFu9!e7q^+7PGNkF!)X5QIQE66B}7n zrzA9S>)%K?s#3e+QID>V#)JSzPD2AnW{$Hh?Z)Rcg1uyFOD3MydAVwKx1g8KqGqPT z9%)XM9U+JNOt0r$=CwNiVBsOzvr7~XvanS?@rFQ%fXhdPLjEFM)+NCK zfu>hOLqne)=5u@ZU@CV~Z)jMo*Ov=nk%f0zj|di?)e#r=%zPPs&PL+ka>39qik_NF z>lC{*<`x`sy%cXUBT=UQ0_P!a?Gp*hd7Zx=_~#~D;IZ*Am%!}>w-&D0d4Q2oMC8Ch zR&j}s?aHPr3N#q!R9Us~wq02@N3go&W20EpjSEK=Ps%2_h_`qo{t%0lnRHaN^^78` zVsXs`*7cDof`>)JZWt`((MnKgz&P!H34<@Z&4czGqoH#inI-H~&CbVcgVCgP6 zCMnj~6*8eoWcT7$*RqM+{A&&h`D{2KsBp2J=Ln;h<+V$Xkz1igM zqkW6SKmOI9!Sv)NQ+5K27SAC`vl-qB1&m$F2N>9l6%rJtZ0z`F{6$f8@(d=PAckhq zA1+)rn*9q>47N&?B(Ta=IP$j}FjiQ-Vif(sE6Y~c!dK$J$~%F9J?=syzlS5&6~iX0 z+F)1y5{HFhR~p#$Cb;?+GBDfLFt7z&cIK;iuxN#2FIU+I2F{KztojoUN);IDUe$F} z;8Ea^Q9rSgIQ$Bj$?XtySlTVFtPYFv`C61GYuQ({d=ylrY}=@HmyYLj8tnrSLqH{p@f1Ky9I|NBQ_l5`C-zm^-tuGWZMZQ&7h4e z5=R`^6qYba-?+d$|3bo+WtzIymJ^y}?GBi-Rvcj7718eQv#>=w;%ye+QOVj@+n8le zIJj4yzTN$4+v0yZH=I=&cJ;P>S;S}YkU6ATQrseeNy)d7yC{G;eaSk>bO8^h6!Bmd z9|r|4jf1j&f*r0J%_2)@95uZmcxAyWB~I^-Ba%7^jA}ZIggBks&HiQc<}P3st+>!A zIpty#N5_H${ufP$?0yOdUh_zLryA&SuYBQFUdF+(f?c zcJZkPxPuHXimY%uZz8agqs_rh^1uSg#sF5mc@NpxWmNh&A{%^!7cfZPm=aUi!J_%V znmhJ{Ij4Jvi=@{A2JR~dc#H~K%np6L|G!F^-@u?za?-@+nGF*eN)=r7I3Df|_?Mz# zCn4WAXH%GA#+l^&zca#KcwJxL@PlO+i&Lj=TT*=Ac@5orGvl(BH0PwxX7|`}Nby?( zi^Yya&f-mdyxRiW^>PmKyR^Azu`Oignc~(G^uU+5%GvbX4TibDL(U4Q7_eM#`SNe8 z?1aM#`*@f|Eer2@`7z#TG54##WC#yV;gLJSgIHd-mNW#hZU` zo9y<{Cic33!|4?>4;}R8Xs%<5$+9)s>{fX?pf_q`a8!-)F~bG3u0Inn6+imf)$p(+ zU&j>Ye@YrZxvvFq*-I$())qAQBzFnqOR6Yp1#Mw65#S8yNG`YE+NQU`pxJ(oA`@qU zZvCYXJR$$hWqNJerHdx?7-=}L2uxsLd{)pAE%NAz;Jwnw~z04XJz$F~(C^nj0))XEp!yYOp{;(=EmDRBPHANtpO!`MU+912slUF1RbBvB)CYEriR{;dg?S4Y zjIJ`=e=7fvfgw|H0b^qV$D0GZ3PrrT40wNhVb2v{JQgl2ynxgFJCEKX-r%o%wF=BH zkJktVFnlQIo@mf0DCVX3ILX4v&TE5+y+Mv+P)@;9X3GQ>MhP*E4JumK!fO?CxjYgd zDo60&5N>i}HVW{q+Q?QsflIc4#rpwk+y>4$3S65Hu=*>=6klXX^x&{Aa0+6GGJ7Do zPJ;QD8mpcKv+)eG6@{s+3XIaNjHVwr`V*w$w^SEjmevwro|W32kj89qfl*aNE+Ne6 zWrp0+h5qkX_&s=o2#C}pOyLB*R{qDwg}@dBHV)E02JY;as^s=WJQ)82&vyBk@0+*?*WjFNaDrPRvEAHXbm zrN`((;lC@IZW%1b49so<0TBnLyCtv~Jcy7tm|+}X%Ep@V^dV#LCB_Rv%x(@GT&5h} z3Cwj5*a|i?Sw3L;Q^B&dfwg*qTk`~_42xp+$O&908I%?<2tHtcqsCFafVJ^+Zf7Zb z)Kvy00c)oToHG}&&-xd@-rm4{r-83w0n^C>_N&TF!XFsIKXB~)z|pU&Y_BMi>e@M@ zqthW_u49nsx)7DQH#!p=nORRT)rk6Vok)r{7uhA|W%a>PUPWY%YU5QC2agA>z5y)p z8#s4;;NJB>qRxeTb^wb@0ZZNlu6hM#{RE9Yip;OC%-EACX~MKneTkHKC<9YMVAf(L zGY5{=hH28%r9wiOgB!Hm99X;sW^BD=e)=Ly^#ew;1Ge%8oS`2Ws|_sc4A_kqu-1KG zG;LsA8p!(VNCl@lbLut8pvCDn4>1$wgnJ&(pd@GiN)zQlm{<^(RE3+%NCj3$A$ z+6?Tr2E3CUO3y#wx%^?-v<7z7f`7APOj$#xbIp7uY^PBE@G*1xQ4w#$9Gj-ea*IVb zs(34i%{$n#iK%7YJ>j_vT|{gOi;NtY`&F5(94hAo#}@^##5r&;=Hj>K=3lgdbBh9( z%LmB?1v<7V%*Kb9mt72eb47C5#VwrA(_}6%=lo=lEC@8-z&_~$v)lrv-&Z8}eq?;= zD9f74a(SU_`fo<*4UF;zEbbL8`#&d)Rip<_zO;YibH=B35mZ2}yA3R7*l`QI%ljz7S<*@4^NfhCWDYiR=O zyrz6T2d9ABey_LW*D%G(HN@=kVh|5)k^R6nBU;MiB=fTNwFVy;jU8kq8PZC+*Ge|5 zz11i!r{E`0Gi_ENv&;f<_3v!*TiEg&7{wNYmU*spWv)6PQuTqOl0h@OfNS!Tnvhqz z#>?A&KGe)Ei&0CG z(J^87%wLW-j|o>@$`)TdH`8>5!=XfF?@ft;4gVJXSiwJI;sMq@KR9Oxa4&CS?S8QI#wp1fCuaW&jTw%o zY$mYS99X+zB2&Qvrm1V>YNXj41LT4X*eerQtsgLXEMO83V6M1eP*9+ue1tJ*0h7^% zll&Xlat<(hIk1+VViSJA1lC2N%o4AY_NIv(JT2m5AZmZeEB9Zk-QAaoafZiq0#;sF zt^P(bQGNTdcaocpqOJ0Dm<>N1I~L3AIf1n`fW=>dYwH27`3ZODHK=X*z#1*U-nM|N z+Q8LtK~hUl>Wq_-J_Y%qY*yM0sbWw44FcGw6|k^AU}&18@pg-|m_U@;1xBWXUS0-< zyGGY#7#L&>7)2(WWqF{R$-$5|gW(wu14C0PdqNGIA(p2Hu=yK z%{?c^ zDR&s;>|SveGME;&t$fG8JFks*0fXiPjvNJsrF|3F7hKnRAn@@RlaQYP+eY~_+b1kp z$6)n`BsuT$S0(tP*1HK6a^bn*A_wk07r0C8K5l)$>Y~6tb%EPb137^l!xt@SGA!mVIAUd=U--1{_~_r9~9Xc{ebH1Xx9Mv8z4MV7Z{nonFJW?!jWaSB6^|*dJKA=zl4)&FGuJ zz!~t0UEx77FULg&mKaq5?t)qBc4yDD3-B#qu)n}#^^|8p8QY}^JO&2Am+~HW3Gk{5 zu%8m(@R8!Z@_^0Qkdb>DQ@;b(_5fD9!Xm2$Piz)4`<_#7IOWCmmpOI94|@aA&j0U{ z4!nDFuhXMRCAsI__3FKee9xp}pBb&bn7-~=?mXs;9CBq!tf2-Rbvs#O6}U?Lo`=0; z_4{{$<;FjrJ1VTdRQeSD`(IF9c)_)B>+%#n2Sy2nG~)+sD+0JST#Q=%P$MXS^X@v1 zi3Ky69@H((tKr)a6>;q$_lCbK^}Z}#_iFP!2HuH0u}^h{1&WvGG3M-NJN`yDeFwt{ zbAg|UkBpD9oJ(MH7vL?Jz}V%$dy+xGO2ndr!O`(xQ}>j)9gB7R=C~FuiJTvly0ooN%B1YYgTPDeeRB3y42++gc}^QSB%PZQ>DD=8wt4KU2Qv>p=ag|wy>LNs z@r%I4p52!w1UerJUgp2lYwg5jH$DkNyPT;;M+N0{Y8_SzsjdoEHwfsE$YgS#XcV&a zNy&=`2btOV{%L(Gda)p!HCD9Vdd-E|j}A6-OT4JsF+nAC6_2=_h=fvO1B+C@mf=R0 zmHa(Sj4~z}4-S;PWMfg&2{>Ej*nEA1=l*5Bv$ixdExv6cJfXp)g;iL=MBo7f|5h{J z8gl`=Mn`6jeT(_i6=Z((r=}aN51ZZBaiHn51Bl&4N6{Z zvJN7JtCn|3ty;Nq(X3Uil4%nX+az)r4zfuvk#L!){O`ch3C?bnOfBj;KODM^e|4~k z#wZ+AV&f0lk-dTM!#}Q^4Wf(Qc(Y0}ZEj@eOIYxz*Yw_ugN7kiI~3EK+UL7P;OlHtKdmN(fC6j^k!YwMe4v}T`RA%h8(PC( zH1axpSh27xSm`Bmr$xa**0v-6_zpWTvCBmq7Lqe+NbFA!D`;T2dCJ6vv|nksuZ}SwV7)Y1GBr&#-;s|eNP{%O!_B%!GT#kX2LRd zaTX^hLph(Ck1_{%3s)V>mk8IANLjI`=yYtu!d9_)zZ~vLt=OR~WX@xFw97n*@hIEA zoE49z`DDyU76?%=Jle0F6Y;RInSWnGtJH)C2b$$pH8A&C%?NlTX#T7~O~}gbz#*~p ziUJ2C&Q0|S%&IDdhdcQCI+n3>y;!oj@gK*P5M9$t4Qy4W3kuVwnuyO5RLZ>VC4GWX zGG`$(GouLuBkzeT1Lhk%yg0z5mf@^=CSm25+*Jyk8jP)e1x&0F z3s~j1@VnK0ZE(E9Bwi9A#`~{WDD|fThiKNxhfL-RIc_fD;k}e7S#!c!o^2yfnnIG~ zyiYrp?>N9}6p$pOe*B2Ujs#Yth=W{9EL5#JrxfaB_1fVB`L6KUua-9tVotqaBSs1;K1s+fmJV{ zkyUvDXmpQ(O{Rg7<3k{;`vw;UnUC!bCWj~eV^BEDvL@JTnqZgZsy#Q0@*X!ZY-;oq znc=XQL!p65gn`+BK_M$)vlN>PFB5ahEfxWVIUFGYf~g88RGS+Z?{^#s66RoJQG1}x zs=&Z>Rfq9v%~Ne}p+{o>W;1$xTgEKU@gmY4`4aVEnrnQW>&jL(%L(TMJgu6wyb_yi>JG4-J<%e<5!hub z;KUuiqLEjtq0uz4fmu|efvYEgm0!f6UuMN&#HxJhT z+GZ|i;=RczG2KrigiY;Cij&h3b(uE>?dR1V@s&^LdcySUe7#;_*XpH8qNd9OTD5{$ zLv^kTUK8k4)^VIPA;49^D4aVQk*oTDE_xvsS(O?XSbnT>l>2m` z&3%EAcsAcrxg3G!Yi|`rdGZd){VC}1S#d$gY{lU+Rl^R88%{#2el!d6e6(4Vc|b|Q z`0Rx_S@FNFW$ydgJzrmUj(syvY3lwOzbqby2KI^toHY!s;$|B;^_Nuf$7UR2*D`2e zirI1K@XP~DzqPhK%QUE;MS{G>b&tIP9|3kT-kJW7&`h zMp>1Q9a?7+1SZU25lax@NMCY@>2;_H+ZS(!8P|C(En|t)y?gvr)H*k_3!MiXf?m$K z5^#V)L4ir$>fa%A(LiW#i=IL-Qy;0pnEftm#z0RpVL84J0i&K%-)WndJ#VZa$Al5{ZY z#-Z7{3e3IB770wh->g}%u-9`=vRv_sqRrV0Sd{9ppGd+1f|JM-)CXoXSas^JSb~3PPSUlu@Rc>|p_ghxo28L_4 z+piQXSnRtq>hj?oY#(nlz0DSUV3oCNSyMx^=Z02y0f9*u8YO-+velTe9PH%0siU~c zTH(O9YOChl&$|ySZt?nXKqXtL{PZLp&ppM5h015IJ(fJioou0QXuO%E_BQ@pv4ucyD7}#D2GZ{2+ zbuiUi+p8Beed%UeUTmdqVal4_#G9~^CrgC+m3{2v)oWj`-k@yAAz;lZVAdnR7(cUt zqoSF44$D?=^?3Zn9i(P~(M7yvMPWH!Xn|SoIW|Y=1BZDYPoyX|_!eVoKoQH(=7; z%&2mLNo4}#!+%9P?4nsjBbtLM+T1OeEE`y2R;nJ6V_KE%PyYaOFojd zi^E%HwvUdE$>vs%1jBh6oZCJ#2IhEjF*eBvDal72Rou1Q`T(=>X%?>oEnXgM$sTMA zOxVzo5y`rK$ zvY{zbqE&xFvqJ>Slc>!x9rFU5d0y;l;QrBIKBH-OM5ERLX4QZuuZG6OMQk278u&Aq zLT0d~&S>=hHRFw8|3CAT)vG7zuc_KH)v))IL`V2e{f8@@U#~vQCb3KJ0gHaXi5P}f z{tFDe7LB|PtJOX*1%7C%d(13wqk)lu@A-_au3t8sd1IfYv{68yQJ~{UKG(TytA5iN zPC*Yk#i9>PE7@JvD|WbvYs z?mcC}jVHK!L@G`#e=_Gl1EcbZCOv~Dy&pF54BquOS~Uuq{4cQjJz!3qu-D0e#o408 zYR7cN6%2eGJTD$KNOdsoeA6KJqgieP1K$A#UW=y8f;QC#MoS0QtOKno8yI+ZEa~dl zH)nRQSF_KrMf%eg>N`g+2;cb6XMtzKtB3|Gn}$tBCzb@;$0oEpY-kYb;1_Z@%PV0T z7-6+6d1X|VRP%=Yy(vd}pX~Boxh(foP~vM}Z=T(n6)cX@6IfFmB$ZufpI9^NhvKX= zW=fY1Sf60_Nnk7LV6R?rxq3x#wMTnVu#(-|VAWZRmgsnuOjcU9X3@$L{~uqx8!@77QbmXH9}paV^|2by#~G@Gwbl6*0#qMK1JW1>VSqj^J% zX#nH?maE1dtn*LJ$eN+TJ>fKm#pKH!CbpFfvn83nSju~CUVr1nls^kEG9TE-rqRIl zgUPSMLwx~jeMUR)0R|tLMj-=V-V+Q;{~ol|J8xBzXyCXI$bVUF&25i;TFd_ZJ{$k= zlHiB4-e1n%+oPsxaUi-vaZ-jus`dn?MgfkACuhbSt=`mpz9~40A(tyPU>$&6n^8l^6r(vV<{vgrQI%$3Cdmo0iVGOU zH^kbXy<7bH@-m*t)~xMk8ynx>Vpi5*@>{?f_Jif=Tb8hnkiZV+00!2O4wg(A7N?39 zPmfzc9&N!Et?ZLnJUN`AE7;OA+JZ9Jlozsv6tvk#FomvQ3#n+c)nMQ~&~W6z;)RC2 z3|$x3$@cy^8^4eh*gpVsaHMF+2XXsumn5d2~JS%o?Vd+Z3Xj zjWU{Fs4TKJV0Kx+k{NNME%k1#N0RD2MyVBd)#|uzwRYXK-SqKHtoAG~%~@V$(Tgfp zGin+*DZI1^PiPJMz!G-hVC;t`+YXlU61FE&EpxXvX+<#W>ae;uG;3NgX-;U?nZS~6 z!0H{)mUe(y)}i$)Z;Qo^kn|aB9v2!mOCA0iq`%N9aN(f_o{k0{k4A0}M*ay6LB5QV z2O3np8f9O|$OkmC9?>zbXf{el ztR8ws-B`gWEr7lH0=r6BQh8p|vK4o4^m5&t%5_uK|L#^L@fUX&=a@@&9JD>rRP&hW z{ON`kyoH4(&5Cfl7?3w zL{jxsJ$jzq|1$M{N%-;9JDKak?|(^=*w<=o!O8xFF=MsB*}XbkPZIwbzerTt#B98w zS<$S$IwAdfY5Iw@B&iuWm2J;Y#wjUADBYf$bo*$meeT-DCK@_F_2%|AYRt%*de$*+ zMQb2KYs?I{bcOg3gI3!KP1-NIEDta_D6qcZT@r@s zHr&tJarO8Orm$fpvB_^ zv-^o=>le+&94$%(312R^1_nIl^624VcrA1xOl${({sLBw2jvj! z8P%rcoX$&9?~7Gj5UYCXO;6~ex?AdtpDeGs-7GbO$u6QbG~sNdKx?P~tKE%>%7tt@ z6k0tkS|q+PJ3P?z=wLCN#TI*^E&2jeOu%vF6^s%u81`QY33{>i``Zh?x~xG3#U3kK zbW2*ieq_F4HJ`kU*?dCtOgE;~#>VU3j=}+qJRCfn0gXZ(3_A=Pg$f#YA|`WiH1gbF zVDCE-7!l-{Vc>K>%siqX%=Q*dS_Z9|0j+`2tuYa-kBgH(m9q8DJm&eJ zEz_b!X9lC?gip2#tT`9jVphC$+QAyUqS?HI#qtNU`2psj2$m3rR@WJ9mNOd9zifDX zE#OJC<)VYC+#mEWZ(z85q2Z3itV= z+x^ON+a51)f;G^8E3d`j+@NQE6F(fk>#JS+nzQ}Gy_#Ryo1TeyD0)8NykDLinF zH=y#Vk?Y^|5?8D3Xb=g5N_j{h*o%64vH=L)U zMd44{w%?_n?k|j=e)6Ty8zJ)u7mxJgd1WV-HJL;(Yb{_>c)_U1(ImTpQ8DB9$$5nh z0#42fyi$5>&vqytZtZE~j@@x%Aq(FEWwSLu4nAGRIn6j-OqYozxlP(*n-Zf^N;03c zc5#MJrO67f9tpR9&jfC5dLA%Mt73|V#zgiDJiL;3bWR2|ICpVK7+6gGkkHi35vG`M z_7;Ov%i0KKi-rpevYI)C#C#$yENHyCR!GKzLrRH-MUY9$QDld(Xdg-Iw<@l0xirE!{|qD}q1KbA*WE-3rW=PQhAVR2vPxAWq{ z$L)M-R!8ov(_O|h`EJ^g8Ju!=`XnEfzcER2TQ9%A_S}5u`g3i*v(2(s1aJx+aT8Fq z-go(jUV;0MCRR=<84c(5!!mkz-ZOk^yD)_>;@OQ!kJTn=1+APC$b3?%ZOJ1Zy$K$R znAqh64!H4q*nB!6EgN!x@t;s#&4c4ArZ!8PIJKq}ws6VDEI7z2R?u*eRnzOkln{Ok z2hoUv$g9^_1X`YkvP_?P^Z<*!(Sarw;Uy28SlqWHws96a1{`3Jt~uDL5#V6ht-kBe zqi&4^8%GyG*BZrBdW#Dlx{4cyI5dPuwl*Ay$v-5iK6(DsuD%L9k`w>%7Mlh`};Vus`)-td^hvO5P)7=1MmpS$eb(}nJ`RRPD@L~S$Dn*|&j zwmb7aSsFEO^}UM<&TBT_+u-ctvF4#ikWa$IE-~W^7v1IiG?-2(EnU&nZG6r}sFPQ` z;(;T-;TMAw3PmLej_iguj~wOtJs$pJU@KTz;HFuUaiK9#HsOF{w?srjBeR&z2L`r~ zPSG$H^LM-TS?u5KNDhtgeW1)N=;5HrU0|WqtbEeramPH1uz*Ivnu1TPQ|DC#b?eK& z58~F{lHz$z{eR0K`ZtHg(htKJ&)o(m0?snw%)u@QGu1v9pr!B=ZEU(9W zp1u0%%)0Y}3`(7<8!xu4SY)NK>Pucfle$dd@B4u-t-gnx%{qUQnbG>IX23>9j@v)^ zZXUXPfUU3H=GjfhwRv0H9M?Q#QJdi?vx|XANg$2cOo3J9SHYTe3un=#Gn{$T5}M5; z4zkr&xX5oPXqEFhC>We^SoKpw3tvS7vz$SbBGbo4exF7@(+>=6auXPNG8V8bH;&zV z;8TO*%WmD@zb=Mvy0qzf;!(co7Z_z$BsMXfsr+$RVyZifippD--k*z<5ErEEVl)-<}FNPs@%|GEaEu(mW-l#>V@9L z3Zj~iw>a_#{g$_%rF?)fZBawy>cb_Q7RyStoNe5c@<8W!u)wCL3G9v$hvn~0U_8>$ z$vx)*i^L*^CM6ao(Q^T9#;XFmf_NVB3rZbPe&xV1Gv>2kyTlRc3I~qscbvGR6`X`w zHn!`XI4E{+M=Q&W9}L_QM@v37ah=X!*>Lr3M_Fvi&F}{ocy3xavx!Y`Ha-;~wq{FV zoOHwExF3%$_%=LTID3M#@V^xd+yXgFsho_82gR4qynDv)k?{Oq-~OpEiS#HiRZMU( z(L31kmHx{E6FZ+RAH&6|4x3V6s8p`RTeD2G(lGS4MUh?gDP`m=;@G^ zxe5;+9+dj_fbD|i!m4taNcDXptd=@Sk$)InWPd$icj8fs=w~<*`Q3rTYL6pl!-B(d z%NBM!w>Swbx!`JAkjPuT=P?tD$3(8Q3rrF-7}yLhFz$VIbJwaj-MZ$BVh^O0&yW`gfnr-B?t68{+dBcIe7oYUm3QV@QyQpMV zt)6B5Fw=jzVk1Y35@(i5vYChWEj96_D`q)$Sxjnbmi@Bea_NJ`Q|wCnly_&eu2seTXcF%^ z-tOSS$R<5wEz_Z<+kH)G+Y3&8I=kbIhwVZUR&`fjmRAX!nITD&H@)n=IBl7RKi_e_ zV;c{behl0?FYlD#j1TY1Uk9|Cl;r04+U8dtnHCc68M-oL0kdL<1B?Cy9q%_ESasj* z?)s-B!ziM`@GawdVb4)juF661C#R6x{_4Y&s0ODhJ-~$vtg5t2d-ZMZ-7K?y<+@MQJP= zx{c>=7;vVC95S2zhWBq;RPZuabBl6=!?J5m^HipYpJ$rGa@_DEhr58&imDw>QhV66 zKi>#aUglr72_b z!awM*H{Y?XyYG54)Cq?8p2+#h9+y=(bTsw4UB;bd26t6wI31U5`Y0QJ zQupdfOZn4F&q~g6cj0Ih*ulX6p+S`AkeCajdXA`JBcpiDA)Owlj|=7-pKugPIH>G# zNLs;B$fkogqD3&nQApxooy#Gf6b7D@18gY`G9Mg;HC!1DF7B9eSJJ{+#$bWGNwai8 z1FHywPz00Q0gjj+*6^OA|CkOOm9%i;oWgp4?NMfy1I!_jToz8!CJbyG4n1o+?@w)* zG>1Wj;gF;ZBag=AoP(G9j?8#&$iTIrA$1RD;a*Pxu7w;KkCi*m7qH$Dj7sL*a!^Q3 z=I@PTK7Sn~w>L#hRHJ}`#L`LM5ABlAVFg@>bs zd&3=vIS(DDg!6AWsNivM@x>ztUi$>#8HNhcKtC!s42yfp`S);RFq zI3V=KQ9*@Kh=oZ-!*NfF1AEE=g`|V5H4N+m4J>BtXrSp*nj3>@VW`qRXKXQL&~%iNu$o;Yf#NZ)ecKXY8F#zCZqNxR~Z z`T;fV4^H|Sp5hanOhcTx{yJ(ckp2C&QFM=^sLw$;o~HPxPR*iB+%p?kXE?B_98`>9 z6nf|=>fylX!vGq@`@^X9CrC)3N#+bg=Yeh>1;!Ya1Hs$+W?3{aBs6gKa5L&Ku$wS2 zx*Ri@=E}0ifolV64@VgDl7^5a4Gbm?-2b*P>^XBlNrTx~b%hB-htZ9L910Gm|G33l z88{j{K7`Rrny_&k1--bzRN#vzFZ4zff{@t0kBw)@Q$#&7CL;vsG z)VzDhxa6U3iA;dvgilk$rQ6gpc0F`ZS>kNcB0Z^{=frV&jYDiAjN%uVv^O|uduZza zVAAh6B=?2!SE#eflHh-~OnNh%WF-`Mt~l!bm}>gQQA~zO#m8AQ= z!**uOmkVP~NXkpc9MoPIQ&p2z#qrXqLe1HOS>CKs?)cZuG>0Mg>?&gyXO)ZwCWiwY zK5iMkA_BAgWB;7q!1plYg{SGVd8U&NaGr2bSLx%Rbg!Xf*VXCE88`uUDHTFl?&G+$tY_EWhfYpys5JYoDN$tWhmq`Br$-Nz>N z4^HX}q&1H)u3Z+dRUn~xqEY#Qll%@@_L>8JcMd6@aTHToF1X~7*@`9wjwbOZO?r3M zna()WneiYx<|6Z#2HhoxRel__p5dfc(X1-KWEIgQI)hn{;jL~!qgezq@0}*&h(^sj zhqz)6>3(5QoYBPlq)~H?!_Pw(R23RUp6IZ@xWL7c!koe+eZqnJ-xdeHDGn?%8Xg!r zs_r8#tz&8B=ufw6=nb*`z!!K=7E;+dP zeh`;ZLX*h_X1ko}#di)pT$cZ+OHq5GVug$RldfgXr?@itGS{47;NQ@gt-~nmM9X>7dOB6x*C^h#s2;2}95C$R$!@;=O(2c+a% zob>J-5=%KKqjR)tYX_GH1G~T>;|q84fZo zP0BkOIa^#gH7-q!bvvseH0MU=%hbsDM<R_EgO^N}+01L4 zqsXM{;B0f_<~BEX`)`XMX{}4U)+n07US$*i+0ex)=$*yxJC1F~7kM61Ty{u)LX(yR zlja;p;V+ItI*zhO92G1Mo7ga`OkmQS!=%UKbfWgMyhYR2h||xP9g@3oKt$q@j8CIT zNuz~iBkvWK-5sa*B{VFmYTyoVk{0MH`s&7c`<}Yzo$gnRsyfbA zH=6ANj7xl)?Bfo7ux$Fscr~rf*HMTibJc2x4cdv;EW95W1U59PN*rWQXcTK;y5Q)f zaps872}k({jG89GS~^am)`u_NJ!Cw!Nz+C+e;SkK41GnJiH~Fgg;Wl*b39;Aao~z* zVBz@?yyQd^kE73S?rxrgtULz`{0|h^Go)PcE^=mL*D&ufX$TQWh}d=_`i^($bwf6r za;}61?iUWy3{IRHjmlG)jOR6(ghVpk*p&7l?BJf0&waPfsqo)-x95A$;#X4{xaK&h z^ytg|yWr1xTx7AvW_8oe_k->jJ2zWhakjhjwV3Drht}8Sjb3F6w;b2W@OK-x8nXzT z_Tt~ssH}2OFvC&dOL3d%8|4QFH7m-Ro~0=+nA_mmB!A(c_~nB#I~uvu4))3)QoP}? zr6GsM$5Fn<;n1OO<|z#%@Hx9C7* z2ImHcPA_MfoJ$fmif>te{CgpHRFqxG zu$_@VmT=rTJ^eAqp^Wma`)6>yIUx0fku||V-~$8O2?zNt3^&|L=N1ccO>5}AX*8$8 zzoz@Z^R>s)zdVRxk>R@YopVW)#-({`e;th2EtZ&kZaA-OvYOfUhqKtcAhnY4fX>$+ z73EhpY!O}X_-x?(vuaZ8+Z>~>i>VR)6!W}#Zl&glj#j7tt&gU<{T7aamup` zPMhu|FOwv6qhXHPK{<(kPKpwT#4a3E32|Jvb8f&nmIb?F*&H_B^v}63mRUI;IC~s8ceoipILK+@Wqq#g&{er3u3pBk`(-n1Z0|JN ze`)q-ylUR5c(Q-a$(deH%d8UlU!GP@sXB3rZ%OYhje{~iOhQwbOkJAOo2QBiI6j}` zB)_9kD}Y&Rg_BCc!92bul{wt5=QH&(B=kNUQQ32_^u`l*6Gw%d#+-VGy)g`%lN&@_ z8`$RbZJGRcs=Z~;k^hsUD~r6pNB`sT-2Qt1_TMRa-VEFfEkd3zR|sSjd{AI%6&6vN zuwcQNl|n^I2N_)6w#-f|SI`g;WOCwQQ*`n9;qhrPhmev>PeXu{ON*eg(+ms76O(jp z+>gAtpt!83&nRiejGu|idL);6PrtV@`Pn&HuM1ah?tTs$vy9$ZcGg8B_=ZpEiB~(B zR8H_I>SbI=c%8I1dS`y>fl_T}j<{_`KX$EfY~@|ZZu5kt>++J$$x5zfE}AJB6E|m_ z^ODs1+HplW{qnDx4+ck$EsVaL_x7Su#`y)sOBb8myqwxC;dsfaXgo)3fOs+*@2mnL)su=fq~<3 zjc3fL{|xX@{$g0qi;B$LCG2(5)J92_zaTf|%#T!iMfR0Xul*Cjk! z!8J4JRnPJTMu+*VH72^rR=qi?F28n{m0`cl39Z$9n=f5((cS(i>9p>Sdy@j!@5-9e z*tYM-Ek`ahwaNp`0v#PqlLK7@TZQ#HGMu;r7R@{+=`l$|RWwAdu|>)$X~%(P{%-Gv}gH72lkArdDwYy06%j?o#Vx zGjX-mvxq=`vvn1Q98xVaPREE}*&BFw>#L~5wWY6%7V7OwxzWgF;q&nTv(mm7DPmD9 z%UnIx%{-JQgsf6bZBx}Tesablhv71__KO?Ne3o}Qp7cwF{8SS$XFaJpUE6L&(j&*8 zSA@6oSX6`rJ>*k8*Z3$)FK_28%OAnpIhsC4T5@V!?Q51Ys$9m)Br#*b9#*qlhDH{l z4u?kNB{LNTQvECz%~A8ya25O`#&puY&Fj~RB^-G>oCRGO3SB1t^AK67GSl=y#)pXc zO}{+Zxn?GOi!@vm`ik$+@}Ts2)gE`I-LAb+afhpLC5Mx+!nxSQ*wXM7hM%`zD>W~c z4}Z|jrh3idX|vKCfmX4&nuo38u~vaXExfZDj?4@(>0_Q0^5RD`pVhsJ!@cI;Iu7^S z|5I4XFY)bw>J&W|K^GHUyB$ewOcpyFEO~SeoMm!P>o{il_0xt!&xS-&W1HV9?MU1 zNMO04>1@1cVNZyKv!p@^o2A0VcF!e;_)lDlzF!y6Cbn&ZhNq;=Ryz@;>VH3+FA1DWGnfQ-8Fa<&I4u65pDsiv$$0+!;#y~fJtb= zh8E3|2aNZtLw+oB)^O5vx8$GEpdxF~IQu8hxH}`ER_8<9k~xnVcsdR+ z-6}kn9(G_0vk3#!OHKtArWtaq5*k8FH6HVrEZ{MZa1iXZvceF_2bniGQXlKyOU-E#-bjBgUG=@M+2}W+y9|r_qEMS&QI56eV z&fek;$|g^J*jHS7nCY*qV6r%=$g+9YfpZoKO(G4G91LcM_BIBr51i}I ziVFVgbPdVP{Qa{@ih+@@Cd0`wbK%jb5+$C1hC`EP33Qx%q9?jM;HbiZl19EG3t?m2VW7$lie@?Dc-H^|vcR|MMCyz;s!Esdvfvo|W3IRiBT|v zp-e=MOoUO2MrEMVkK zdBDFS!kKr|Z+3}@2TVb0RjMkpu6$i*KFKbcuckJw@%E2U^Iu!_HRi5j5?i4Vw6oKJ z<@)yq#gtR5b{80BZY8v>JSa46L(qxUN(BtN)EumDp7fJ`_dsZx$8l9Qp->~SjU8nn zi>2Oj91&b5*lSy}LBwiCqkoZrq9~iIQ&PslZBvELov6!@{kTK1^VdYtx-S)T!&1Aa z^yZ{o?7BAptI4KGuOt1%_b}Z5xUYQQOV4Jx5Je#c2O;fkXHvsHEE3x)uvzd)0(Z<2 zMz(vhEDG})yFGoHM7IA^II8GT*q5o$=pj?Us4#6ogQSfx+ljZ$Tt*if?Q|Y8^I4o@ zmUVNSc7S68t7=9gi`RrEp$!LEl`6`Fsy% z-}Bn{ed0&kstcEAGrJ#~Gskmdqf$kp2+MUN?TC-v@dbz2+%7b^z7%MdvQXm7&2W?S zoyZ;ZvGBY5ii7fEg>AE~5;zVB-YS@-YZuBCCliD_Vsd%$+(DO{%k!DIJ^>>?HV{HwTML=6y;kdmeDB8r^v1LP% z<{{4v1yL_XA&-Ls9*rVu3%S)8MJF6E>-*=&nx?>2KGDc z&r}owj5;lg&RKFSox-KcbV>brMY4Yt^OSel>2LI3zFX~ZPTNAql`TktK_!7RkP7M;oNiF3^~D=;H~-!VOR7X!IZdrO4lqaAL}YdDAviAJ-#@XZB+FXT-q#ZuR9i=l}gn`CpS(Tb980=>el$j*Z+J#zW1D zViy<%nHZ0_@z*SP6Q}SdsDbs)Iu^GEan%E?w;l-TH9F`x2zVvH4g+C9RG2CaJZU~uu`spq3ToM z3?udUXMso5)jy}~snfgM6lC6&w|^FwX8eOc1O<5=s^@ioh z0%!k(4{<)yKcDG~`)lTD(QGnxNAKO6z zxrMzlj6!Y`_`D8Eg)oZTN|7pB$ho9}Z^Z(h2M(-0nn5|q7A49SCI23$S|nKS6HY6- zbm`1`|0X|HD+T6f=gkGa{83YQXD0YZ%^~*KV}*Z{6@RJQC-gEZFR0%Z#1i-5ZPbH% z=k(s>H8^mDFsUlAY*@@Mrl_Zzz&R~}*-KGMnNd`skuxpf78AcT?|b&DCZ1WU?7YV% zmNd#dQj`%YElqj$K+=#K0`TV}0uwiw?uZI}7;b zI;4~ihIu=N7A?57ZlgY@f9s3|8dBK?H;;W3D0a>FD4Zc`v~t0Gmq7RV`RN|TQ6JyF zUGw$rwKDfc5s&gVwo?yywGJ{*SRk-0QECZ?sE4DdLL&dUt(*=Xl3ELoPD|jucEHY3 zfo)BLz$yiw52b7&3)t8Wa(FZ*xU-3DdGMr4fi>j-yWG+g!-Qu)N;nL}m~9ROT6tOs z`I^UF|6|dh5&zxX%AxGHiB|ooK;;ChYwcXK4$8bbpq0)UoXKmO^lh!#nUJ%W-bN(+ zW3E(vc}|b@+JRR-3mkqq2+ml*ufr%hYoUnI0fAq7)2AeIrYOu_6voEx@07Gg{a;W4 z*XD?$hLMpak)I!GB}`$P^5(O_W%G5S?t)u#ohO?(mNvW;bZ`A)9>>5o<-l&U0~}FP zMSl4m31Jiua+Gv<$g;`J*?xu8IY!xejUs;>xV{~bI?|Z(<(uTM2F^bUY&{D&dlb0B zj3?MEV9rWlPguZu!pmIh0JDObx_+Ux$^mANm!*b+$*SKLjHzf$A%t zwKcFcH83bVU=TCNO47A-k?RB8&xuB7$F11FLV+<;kcs60%Pj}K4b2P(e@G+;GBq$bUaUxuJW#dN zoa592UX3?AT8ey67lhtTU<*=|ayTeBWufG%kKwD5p3P7~Ohx8vp4&&C$F$>Gre+$FECRD@+`>POf)m%~`<4;3$~R!2Rt2$F2>aGj(|y zE3zK&h%L-_WE3w+6zkfx;vfSLU!s^(qS!KZaixWz12~_&FyvV%!1Yk@1;f&m1A_k! z@H%}zSYt7Z`Rc@3hp)&f%oaXd8hfnl|B>fxrq7GMPg=8h1;m@d}ihuA6Cx0UOAc1v5{f9AA|Kfh8&*7{QngAlom^gEfjHj z!IP%I{wIQciI~gsZTUC8hil)G;JYkyY@*EPg`4(GbNRG@y-UO0J!-P|u1TCf>MLhY z7A!Gozs=Uvz?XC2ycxqkmYx=lgoOCn2bLr_^aU|`l{<=;I3}bq?p{B8MQ4=2t_Fde z1VJrDfolq3vlF;PiUiUam=hTI)mk`{7?|ft%$obSa10>c)t-)LZyI>4Bu$;5JiXAZ8Mz_Ce7HNvC?CoxK0Q#4dv zzqtI(#J`UGD;nnU?3gsanK>ZAWNFN)dg-N%H@3G%Pqw!F6XPKKjDho44+qC7ZkI$+ z2S-Dj#YLK2#WxehGaPp_A6wD;@7A>43k2jOgcRN}@A|-;vp|6DAltv+JR%qOO*=ek z#^GoCgge8(mxfO7D0N=DK63It_nL~s=6Z#~FRjw`CbGvUfMP7Pq0N?g!XyRGRSas5 zix~I$G2LQtw10bfLLPJa-+h-D_;eiEeHLVyT-mJcINc>js>@e$ks{v`Uv{_MTS^}+ z6wH>sxqVS=b=}EA!-Hn>uM{|nOm?^zrw1sUYIiur`Qvt@bVFmwozt?>xd|0|511#k zB=7}!b}V4ZSuSv83V+DMGt*OiuZM^@Gzz;Yo>}xjAfYRHi-G`CBcIYFpX>unY#%xJ z&a!A_PEz9(mS4M1`RGMe!L=7>&ED&t@$cSgt2u|?>@&WsoO8u3cTZ9R*Q`XLOE(#P z7&u=g@IPZP>t8Rh%7L%w!TMy)X)z0gyl!27vzqnXJBNt}1eY{&^?c(?iIH^cITE%( z%8PNu>W4fl-riWfFFf=GTXU(#8-){s#dW6z<;#5JJzV6!I0#=6bDYV=8sxyzHhcT# zJEwk|@3=kfbiFcL!x1i<1DqMJ#iQ8I**c!ldceGbUxbNK#K}$I$AS;$jvpoEdE$Zv zW-0J8FXVP=6yZpG`iDVCM2^j@nltDDN6>=m3*5rpuiEuD?w47)e~)n|GwZ5-&mZUV zS*ff|G5@&7qijzM1Ahp^6;=m^vq+T8P$A9C%G%F@XH3yz~T+APCR$ZSF zl5WD<;=uW00c%X_FS9Qkvl6ECCkU=m6#Wz-nYUrf!fOkwl@@LG3_Y}^iZH(Mky&GvAIiLAkgaN zr>A^!Hf8gzN{fOX9-is7fT@s))k{ELf5DH2PfEg5Gy-E;B0ZBdCM<~A=^@#v=sroB z*WksMoe7SeT^tfBHWdO}SFx-Pl=ax#ps00yb*y0wgZ9EVS2k#cEa|ZN+St5OQ0_?Y zsYVB8KCu-K7AhVNSmhkMNk+*umtVz}NkPMIL4*sJh^mr%THwE?)=oi-pgk7?x3REr zr+MjKN?<(3&MkH>BzP;w)Xm0LNzYF-DsSVHN%H%1LDB8Fgsjhuj+eT}dL&&>Jy|)s z{QUg=H5YzAeeFKC%Ffm?E<;VDiMdOpPgY*%Ma2TAw`^PaD-Rkd9_$bdWm&kXonuPi zejBU%Us<}A`%53Vz~;z+x9jmi$srlyL*%i_@rg)R!!8HF7R)jXlDDz!TVJv3MF zC>@tJP!X6O6YLop781Vo##9!$gaeK2Ay;1{br)GK5M^U)6}=vve^k_wnQ09}i|W}G z2N-#b&N#3uR)+OC}VhO#pFKQ9w+m=C4WC0_P6`o1l=+gx(b=qY+&P8 ztEhHh5>PNW$f4QeFmrqIq)aF7>=mt!%z`Nb2RRoRE<9!Od`s~W!E+kI9a`t^{5$Bv zxBcw0&-3?pZk&DIp*wMf;YFS6D;7_1(^|P?`aXjTmt{(x&tSP<`F)1N0|V4 zFLhx*y1Y}QMdIU8jm0mVyww9r9-X|sxZy&h(8MWBYZN#>9pv1|cTUKaRs6&T4|(I9 z07pjYi~^SSygz~`gz7>X8(G!QZB6PQMLzZ>yo(i(!NfM{`E?uu}@3D zp@G@-K;aP~v5*Na@{J(^hn4bN7Bw@Am@klE5p+Ddc>X}<@GU3brT*k>^;-8p51}LRoUB7;6?$Ll)wR|290K}Cl?x-Bn}peM;y}p z7tm#`p`gXvp?p&50RyYX0Y<$7$8a5kX4RMl%$^IHrF|AKNKAQDb&6xY(ToD7)CEa` zOAFjI&lRv6-#N&4O!HiU+0K(UdLE{y?qF73;@DSl=CIt^N|Dk}+f8f(7=!f>+$xt- zG8J+04Ssy`pA^&HX%0VJSYm}PI12r_z#>qSD3+gm^mfHT?)ZWvX4w^v@*#l=0TPD< z4j3);4$Lhc4l#&ZnMW@t1^OkKdDDzlKo zMMPAv+#^fXu+#Ig1iRqWhGTyo9bECt)jjj)ZFjj#8=BT~P6&R`DOJAbvq?!B%F-a*n@OACg&uCNC+1Mf_ zv5>8K$5GuFV`qsEA6N`L?kLSDU<-Nhh=0X`Hls-sd6FU)2^}+VQ+*cAnZyygqAS6r zPSlYzNyT~9GlM9t$=kfbo;7kgy||zJhg~k(>uo*H&nZr~F3$@xT+-;Ak-*b;q1_^f zfu%T=RoCVq_mYNR#ZaK5pCOX3#ulnn_jk{c#+cAjPs zdJ({IF6^S4e$d37^V)aSh%uC`oSPmQw|m#=wW*bkrv#lMukvW|IIvC(XkmKC&>>{t z$lNt!iBVdEZHS7p(4N+Vn!genrF0(gNI6VWd2(T|lthxyISCiz83~NM4oQOU4UKF@ zAKHcI95Ai-V6|Lw@Lr0{V#$*RZpM#hzw|4=B{6lHo5=~m2f>RzSzc;9wd?(m6gYqD(4h(cJ}}xZU=p1$p;@r+13Pblm{w6_V3))8U7S8o zOFk5F?_E;iBED3kBx>iTy`>>X9~Ew6v{5*sx`LsBNrO>z;*KM-6%%^WJQnjy1-O|` zQ8ad*pTIC7fK5TDJKYhxyk$>ldhc(I~LXOKV+gdPCCFDLUzY zCAgY(mA0iUh-kmSIQyF;+nZypdSMS+_18Fwt_V0PBy+K6PqPxA4&za!Z)(X?zv!xS z8y}IIw4hbThmkvs!$sxa1*r~xCT6t-GFhQt{<)m3T|Mt}N%yM*rAjLobh)n|P*Txpr9gch& za~XvmII!CtSNNbuY^Qg-30&&4Q~7ffLfAIfqtYGtR&Nr(xu90}73V9vY1{#=;z`~t336NFX; zaFv&{EiEWsIzdQJEuv+q`GNzS+E-ZH4cOfnI9oSxG&68C7H~`o&~-~-bi3eJ^gv8- zLFJm`VtDDg?*I5UFe{|6vM5yZD;T(3Vqi>g z7ZYIEx11+^17qJd4#UT-@Ia-}nP76fpta*bN?76dJHcPGHSc0ndsFkrN8o#2jLlPMFfozie54Xy zPv~G2O9;rC?b9{;bAi|U2mH#OllmUbUiED0_6d_J53q$_V4mi|{P(!ro|4H)rQUZ1 z!+1rOKbe>)6OiqAlF_ul+H(P;>;{(V3p&OZSc(MLeJ2EY8W^uBWUEzRk6&QmE5Pd8 z!0KzT(l>y`a|3Hh!nCLcmL(0`FADftJ}|m5cSjj z7h^2ctSVvgsn&G0GAj7_h(X{2gAoJE_5~dByHq(JFf$m~9BpRcoxs2$z z6_y6C(g<5MJDN*2xZUK>1)gOeI3kX-g*mWJ^PClluEGmC=>{t;ud?VDvrL)ATH?UoJRvX4fuqZv z!z>`fO@X~{L%P-lM%@IX6_=;8Y`0vfoG(zd@KJ@O=j-15jbi*CqAVAsa)jzJDl-Tx zuvI0nRVJ|SU&Wvx#mKQhLNm)R#Yy6Vj zdws^$1q z)zbdmOI0*?UQp!SUclKufwOV}`%G`vTnAQzJIp}|%N!k|IcB?8PFc3>w<$lP{DCh5 z;Rl&)1lYYlF#eM^U=|Ju-(tYEs33_kiKSSGQ+)xW?RG}704Avk7AqPfV<$376);}@ z#Id7+d*uYhOIO$p8d$qGuupuz7E-{%pTKyFV`1QG&Oe&plMrY2>S*uw>3=XG7b(-#Bwr1G1P@7r%Lg?S+V%BfGpI%~ezu@OmuzQXH z*X{@0yB2U~EZ!}BfyJ2lAE$A^QG*M`Y6q6by*WB#Dx-J-qp<>evjAtSfNJmrR@rTN zo(9a#AB?;f^jbu0S!2G><;Ir#(OYUI^@3dV{$CM$F3rdkz?8R<$ty|x9M{QqPa`!Y zCe8&+RtuPn7BC(S-j*)F>2rboX#tz+1$L`b!RZ^eeV1Ur6u>3v;ITl4L!gGuh4=K- z3p_U`9PEC*&L?(9ll0ZCHamW5O;Xf3Ga>rSG6T+*1&68(*b64?%-+Cugp}_NmpD&;W0iJbd9awr z`$8?_C!?-swcW>#N?u@$Nn|PAz+U%HAX1)z*)pAx@e!LvIJ0|$cyPB>&}^j-k6M;H z7gWDt5ITEYH9#->qq4YEzC?n!up;aG-zPW&QuqQGe#~ZIN?@B9z}c{Yqw4~b2G?5i z08WXg97YS6PcpEp2XHPjTetVSTE(?!^#vUEUNNT}(z`G7Tq)ptwwCXm*7}gSd`~~D z+tG9Iv%sYASEsup?uVLAQjR>cy@6A6&Y98;XHK7C%PhENu!be`E;EN53)jADx|V0n zHayVNylyDKEU&=q^*~W5<(%w8{tpYdV;-=~Q{ZxSU@2?}^xw%5rLdxV0n1OGn~ra5 zrba(OkdA^YmNW7hkM)Na);`|HJn=_x{(vJZA)!#jHG3 zYQSCaA1lW?PENnWm;OB9wmWN@z`P=lSzL}I{sGJ41KhF?x$kb_ zocn=u@dFl@fbHi5IEolJMLNoxLo$@6o{zr6Xz+opfkCzUf-c|Vp8AI1IcJ%r1frE* z?OnKB$>X2c+Sub;_im}~WMn@6cB7~Pi@^-N0ERL{21bQ{46oKHb4_4~UB_`bnR5ce zNg;-9gAW-|wl zpaYEJee4Yv6x9q)UYM0u;81iW)kYv-(S2{j4t_>X1*T)4FDV8vDgJA)S=A$Rc|qaR z2|OWp*q46bx$=RhS%PvzA;bUDVdr#(IV}$C zl^1^h^L=?{4qG_G;hY06=jgE-)IT_B|JjKDRgNr+LqKP?KxJS6%gJY)i$8EGethMU zf5T({YqblEEcxoD4UAF-me1?|NlajT-Ore#$zE!}dUE%_d((@%)h6sQ-CBD7t?TXH zb#vb`d|c4T+{nk{r_d45#KI?`B4F}EVd24cevU1bBD{V)?$i9HZc5>FYh)1=(a1(%Hf~}i)EJ{8%{XsaA^sr(lTCYho~+KLD$olGyK<7{FMq_;xXCv zoI<39;BOZ1X|g&gClVN)*{1E5KKJ6n!nPhB36ldI!C4pQ+vf#2nMfQ=IPy||b^QE* z3y#g~LhFr>7&tPqwX<+nEVrm-Wo2Ti_@mj%CAuWo%P*=y;Xo4uYr3B7M9t-mA{$Ps zu@yN9II_A4bg^(dWh~rUd}^hd0QUszR;l!yiCm)TQ)X%%;a@2w=#si5@R^Hf;2*D9 zh5uq?)Q@dGQ~7M#&R3_7^zV7~ppeh@m4MEMgKaiGhFU__Mn*@g0-TJF&v>H9Yj`Rl zxzBOej|ZFm+#a@yhWjmM?c%bV;5;>5!P2q)hLHthn{dVBUbDv*2XvdcZFVp;GZ_># zES2PGOqu9$XM!Rd=aOoMg`LZSzgW3lTKDG#{-g+^wUmIKS?@c9I=uqp&7v`lWYO%x1avRvE~w8GMv z*=pKW)v3WxoK=rz1s^es&fV(kHZx_{+pw8w3r=?3S@UF9i{9>6JF<_@)! zd{b3*#3he^T2lfZ8Pt6bRpQk7b7OHcuicL%@30**tX)cG3XAyDR~fpp%1*FKZc~2L z@t8xR?}c-pWP$NH7SV(k3mEPt80a-Ci$xxp;Cjx(bJz4;i}kGMY!*I$u}Mm!VfWjD zb$(3md)DPCxXZpcI59&n@L|7H0>cs!_h}D4tEk;!Xh}9NT2ZPVE3NV20CV9h(}=lB z1`8(z^;smg8BgJS((+`UqpFIx9^=VLL3T`kL=GN{=RM4k;uJS`-tE<0nT)vWYKh&qgp|N9S#jEq%I#|6??RxOFtoz z-#g-nPzWPevMUp}v`)J#Rrz` zIxM@C^K2l;N591aMvOsHE}23BjVv(}7$m+Z3i!=DrIeH)z`90J()|L1@U#Uy@_)D- zQ?vzGBmxd{zf;i=xNt!5hD3+;nP)6T4KBtiAK6_)F3;)^Q=cnaqGkdExd<0 z4;S7NXt&TgDwvt#)DYpKb*G^# z=!za^qsEbyY=L&FQyimTiV4UXK~!p z=8C95MSzR^nw{bM-jxbY`H`KV>ByNhWwXT84M%k^mF`sil%XHMA}f8#fVGOzQP`d9 z(B!HKH!oYdaV?ML5ty}vQ{{^Te@g+I>8g!U*VdIuU0o1hsuso@^2U+7?LyPBz6~q^ zQ<6A~d0ga-H#7?GV04kSJF}9cUF8-=oHFQ z>-M_x(d>buPJkn)kOA{{!$wZ4AI-+M4utajTX8_JHsh-Nq=(GBbz2?&vtDDZR-MZs zu&MWkmTUbYsZO&w3wg^5nq?m(v|Ha<$SZp6gY2gTi7rnV1-pJ2bo}<>be&MKL493I zuTHr5S<7!58!~$)i7>Y)=B}m4@;l)n-i_OLfBGC* z>SlT6MZ|HdOO+kWm%15^w;1wq?J%lMxL`NkreR6A*+D-Oi3hWIR1WO>qRQgJFnQ}= zN6ARN<4RLDcJK&XlI;88xZ<<|o7$8_&Qy(N$&kBkW-k9K4vE%%mDT7;bh*PQ*zw}< zT34AQuNxY%Gb~)xI3BLMFpR`a&x<9D zx*r_bL<+k^<|Z)Y>a;5P6v*=~aj?h_a11on(3!iVh2f8-XFy%*q&Y1Nj;b2WIj0x$ zaIr43pPIp@7gf+KvqOPB{Cp$-7P(jQJyKWW!?#Mx<*;uGzVVM|)3U8=^J}xEH(s7= z^T>OT(&9In+PB$E4piUEEcm|n@&dNgY1?{DH%8`aN8hx0@r`}Ph4!NGwREaI&-*LHF+kz+cHlt_^BVWZq#fnBn2_`;+<020l zN_`mlD;W724ob!}D&#myMKDSWFi8qHNfEmgI$X=qkt|?!V#8$BR+;4wnrFLCp0|1%xE)#(W0RxgrW687mI5I zi^qpeTm}r%94w#{J%2PXDKxO`XiU*)wLie^w4mAEgEjpJ^Y=wZB?FiiUEute@NiX< z=R^UoiFdnLqBpP#YBcWEX%u;JT1dhzS>#xX%K@xGMtCjXBlrT!jG(D7N5|KEoDA1@Vz<*hWiPPeE@soy!!pB8Aj*G8w z6g|)&a=_d+>6AqSi${fAQ;m~E!{nD;kzretMqe1FM^HZTEd>l(2dz|3C z(4f-L;@Qw>v$KIogMoKOql5*MLkEk;59Vfroii42I^XGYzIfPn$zcl?HMJAWq8vw_ zWZN?Rkx%+ zN0V$0B%7RBX?*7Bk;cPE8q<$39$#|iRL*iw4l^|-*}Nmu^HUCrH=GrT@V#atHBV7B zXcEK64?$N<+>2|@iU%}`6);LnXq1}4sC=O5O=6RZ1(R}vN6Q>WH3O!mB{oVCmvt5} zYtCp?Il&~|z-ZOLVrjsnV!>qAaM_~w^0O+IpcRcg1r2-?nsiPu+5b>hjacZe!5D1B zIbDEZ`T|Z{8wT4ehglpLe)5EhXEgu0d6*%CL2Cz#=8DGX1x)N2{h|k&%^Mg+3z$uG zSS|nc23tI3aP4T(>0q}1z@+AZoath&h>|vuM2Xx$+;doq;rPF$DvU*LZ+fE{XlDJ)Ea>!+*OMG%k7g* zP716+9vZVVFvr2e}|bBCfu8R*j9ysgQG!oLzDIdChr%FOdrniZD5q1(I^$sl#;aC z_5f?)g*H8n1{;l5VT}eZ1r~#j0A&qhhv*hNi)Q~NZDkxSu~Dmy6_~neq(dKCE_8@x z6}TXr(Iot1zJmg5)&>T#9lm?lgiEseREoMCxDuEw5+NoK!D@2AB(Kr|xTq(WLutLh#)K4OhGQ5)@~1&pUi?_sJOx9weJRu`Fp|u4oXR#w7ir z;oAx?XVWIz8BE+W8trb}-hZ}LTcXK&2g|Y*42~a~oHlY zfX%NtiB)39WDWxsL(bFPv$GgjZ*a5(Rfd24?R|jdjI8J}Loqi|i^hW@ZZaP(Hn=`k z;JtKj?U|OX+g4=>m#^2bNZm$Ay!_oK&@Hwr0^^BPz)Ua|$aILoEj$#5vmxv#6h z61#yd`$p@$ZEUeOShEG%%RV%yG%(vgU{qnaukwRIc|x1=X=p$qre)Z$7GEnGOtN{!eh~hX0I7L85LS=KQK8}G;=@b z7i#e87u?6ap}{DiEl9vmVnaiQRTGDi)#BiWCmQWbxfd#XJPMk@#$&+P`prnRXcOjc`|NSxrHD;QND0bCE_*L1qS66aaJ>$ zbS#?8);cv#Y_^q{sw$zNZP&u_wrSImf6Z^BEn+(pX76~w)AlmC&v}+ZL-fQ3Ru2a8 z2RzqMgUcBfz--P43;ej3gDGI@<#cb6<4WxizYO zZZJK1YReq=j7xiebRXeh5>NW{lUQhgF*U&ZK(;9W$r7j z2aU=LTzCU6a7QpPo!E3myFsd>QPhG-?!u1AJDR;_G%_kAGHqxO^ANPK=(?-Ltk%%L z^N5opfYGou~BViJut`i*zwj>$Z|*eUkzph+I9%eyoWouf_3 z?g@)*8W!c{m?>5qOyoby>A|Rf^j(ju%R4`pcWTlWXBwnBnr#DGV=k~drL)I7{Pdib z5>#>1s^H63zCQjLj2$f}Jk(!*c+()saenf{28kD;9dB&7Z#1a>V0yQ=i9f-0`?ij` z9oZY0)aEZK-F2*Vwu!#g69#F9M*anj>;Xq&wlyw`yZrA_0khnL7*<;*_JAhu2aK#g z{5}T7rtk@dw@>4AJs#{$dGO7{m|=D4hh+<*|JyQ1EMs2q()wltUxIy%DnpHOlUl~B zwg_hK)H0p|=BOzxstFeC6BH7dI5-46=BzA9yQu5E{m}spRo}>eD_zrj3@n#2H!<}H zNNFTYdUIx`WaE^hCmoiDn7Ef+S|X^#!rnDQCt|?@%g<_l!sgLuYzjZSbPHQV38?`2y9?{zpJcQc6;WmZC55fQgmh$mP~Lsw}Ej}8;^uZ!Lv0p zH#75!$wh1kaB^nh_2p{)Rqnyp!y)KqpP_AVD3*gy#N@_}2j(B{3kaDwD9mB}c%RSW z2j`RH{Q{i4EF~F9Ppqa%NSxXcc8A35I`TCB6i5x-`xyJj7Y+A>?a(EWn9Vsq;mnn^e-Cmc!f@nJ1Y1%^z-f z7~uF&A%y#!#o{ZvzNW4migi9q*`;#=7+cgFISyXsN?2-eq|ezyfhme_k7*-w(cf)h zv1P0st_qi*Y&xa2*h6xu#$p#sFTLv##V169Qzjl|m2gm-$!3t@@GN~!;s!~!#ijnSwS7j*Pjk3vRpZV?5rl@X*qBxgs7Gbq8i`3wICIoFx|+ z7|nh#b|^Nx96HRLexUG3`Mh%m&Z-ql)F0%BZ3A~|AFS(1`-Vz?8$ zmM|L~Y2=;~6lM8zO2D~AUK&9;51)!mPi31qb7pGV!8Eay1Cg0Y)5M<5QJWIxz-oA- z;Rw%CO^4ged^QHNbNE!IzhD%SDPYJu{5MU8rH{?rh246^h9(vki-vRhEB{I(X~fMy-G^7SoWOi5U%w!gNaiQI~E>=?YvzYVDZoTRWkItwR zok?(!Z(y0!!m8H4^V2Lbk8P1Ny=yiUWh^R9b7T}3S+I#MaaTM8`{qai$B74d|ITJ< zVrFp2H+y8_Zt0`P&%nUgq`u^lX+yrmghpuw!HL#7jQm|4Ek<(|a{OO%oKfQCYu%cOJ%Ii-5JqLCD5;?t1o(cwbo|1bbs;tex$Px8%I`^JW zQ*ZzEU`h4lo_J-`v~4~OjKUrZ&Zj7_hptGf4EU9N^xr0~LYYTmmuI+$^*msUc6lV& zZP3J~_j%dW(gnV#aOC;*pu^(JyY%me;|tUR zbUYri*>XPMWf3T7xTmn3(J%W3bIgTiD~3dN0RhL4B8(jFe;9?9CM;7HS;*lYa!4>Y zLy?JTx*)5{1D5m&&rD|TrWxe&lEP`l!M z?Y{{fE;^5%Y%*OqwPWFee|8y<9FH2fp360i`}9ev+bD6p<(Y$AnFqSmsx~xBvz{qS zI4R4n@?f$2%v6tq(cE6$Dk>5>ilPlMtr`vo+3h+WEnT(4O`|BW*Ygb{rzgWvp-BQA zwg(hA3nQ2W(>Ao&9eK!MY;j2LQGu58f<^q=&XR1`0y^z24)IDkIH^1;YjOW^P{`;& z15?b)c#a-w9?GC55)M6$$1B7Om1@8VxKu z*X$V#LaO3g1sN?G@|iX`Fi5>N-@jL@fg@om*WY3mSCz+n>N}cMXHD#@IMKZI&y(fZ z^&0((PJNQc53s08HFj9PILMd#&*QKZmsX77iWh>byE7z=8`n$3GIB(6HtlCvcI3kN9!VU|0p5;E`@x5_?z2gE)fWt}s%K@yK^AfqNR1`TU8n`I% z2(s%oIN5jKP}U3D*rD1XF4~>ZwCR7;+D%E~G7LB*ILEP-X~Ea2 zw#3MR~ad>M@_^$HWr7?ha#emF|&l|G3rT5|iP zqevA?#k^hy0f*+Qs|$>T4wU8@D{ITlk=A+GV?2518NTugTP3}ZhVLi0_cXJatO@1G zc^Ss9v#VGldix#I07tI#Z?E&E2y75v@qkt0!rXrxZ33TXsO#+3;MJMtThQX*z$p^X zagx*ILL>hH1~%glOjA|`_Bz=pi&q;o)&7_6w0Kh}*cxzDh>5W)Xho9Bxf{D=dK|he zG8Qn+dD$SNv$4(iPa>P=i6=^Z8(M-dIC?nk+O+xop^5ey%zV>Uu&~{W<&x!yWIU$n zce!nw80VHmUfvsh{4C;(!VCvl_)aj~XJC1{f8&ph90d|Aea9M@6}%J}Zm2e}6q)qa zOnaboV17=};)W&*W!Y~6#WRlTe7yU4-uJXVUH+>!d>J~2qt5)OQ2W#H+{!0)XYPTn z`TzGYH>}+AwAGI>a?NkI%?afmeb>(Bh)ifyKEf!vz@zl!zk>EgVUdR{z8}75tqJUi zu~{sTm*J$qx8~~_XC;By<)vC2Gqg0ERd(DGn$V=E;>a_{pVxp76%wt$mQPa|2vlDBt74SUd&l7(B(~V`xWKNd`p^!ty zYo^&AakhOi@5o(e+Xh9uD~BbfCBAJGJvyJ^{DhF1<%tJ)-dtRhdw?y&fpZE2+l+=8XB_?sewd?Ia#-=f zArp}%F^PkcZyN9X4Hv%QuF&Jlw(^jmfuo8A^R>4VJC__1(qYn+IVh-es3G%!;2%eY z4-6|^4#~?L4HL{sWlN69G- zOdihiB`zv+oK#&L%q(wNCbo17wp%?5D*SSc`&Urx&r5CwypM9lR)>VWIUA?4;+oj4 zX4@;x_V=0{xK!*!mN?2iacoO`ZyS^S^TP3`k!pJsWsY%(Y8>qOb2H2%JbcTYw+jvk z$Vh!>^pECQB(s+FU(8wFHM6v84zkA_%Ur|BR&!uwO{3NWCvy%ay$4Q;CX74=|G29T zIxNxk{(gYr;#r3DoC9}r4hWUZUB~LgGS5+6HZ?Ql0K1QWVLYVS!))ro%*G;6xaFSiTIEMSpFXS= z+rhhR@tWgihKk&lQWqvPh&nkLuQ+T|@KCI2p_7n`Q=D2Fmx`2CjLi*6MV0BVH#j~H zRdLH}^g&RHe>(aX92HRMIe#LMLnrFvmIFc}2_7a%EOQu6**K`w9*|tqq_yF& zf<%yFiwj$c1DlC=S`Gt`j`xMg29?}nsXnu0be75(I4bBg&U9#-5z&@@!ZX@(HoGlD z>OWqG=*X))46`|RFotqDq&*DU$LLYT;J`V@Nkzkn&9hnQ!&%M=Ve(rV_+BtXWwkt9 znB1$$8zm*@E^+*!>b&l??yomqb~|zD^+_K|n?t%w4%?hzw)L1}%k#wc!Ql?EC3X** z?UtNw#%%xL&G~OR=bybw{g$=9E=JCWRpyJNXhz7!dx?Dt zs~7gNMzXy0*!wKV*4BPvTWvnsmQ83eP!J^eE}V+A~5COe|hZ zS~Hkr5)Lq^1iZg<{;y2~SAi%`$$`@aH@J>Cu*W#CYd9KPI89ukEL11WVA-rH!lz}? zY&4~TVUB~*lqRJqPDUoqao>)y)J)SiWMI~55SqTGdEQd1ZM%5=}^*wf6mxz`TN|J1;9hJk%SI%9xXX3ulxEeGTu zIBE&K6Zyixf1y!c!cpvkBY#AbvW)|izyZBRy;rIXat0lI;Y}tnsZ1*xBp)yauVi5O zIq7i4?KfiXizmdr6Ut(0`E2#>sm-&X_mftwJfy>Ura^wrA(IKrcI_V}>NP&^+IW85 z#xL)frPzsIDx;zY1Zirk%#u8fS&R!j`v(C{PZ`o?_>;W@3m2`0RMxWkV$yyf9J z{jp))UXdwRLVWjzSPC2zym3HUrjc`w|M!OF>4}BEcAVDn-}L>D-hJv(_|l zv1|=k-oRDh5GZ|sf#m>$gaf0<0frs@ENc$R|7p~8U=lsx5Doiq9JmiKN^fyE|Am*^@Zll@hUHy(y;n1gP-#fu{LQw3~{U?ueK5SA6 za5kCHY$;JDp`0&ydez#0#b4err*NC3d~SANHnCrDLM5Ro*uqKq$-&t_!yI=W;oZZ~ z_G00cuq_Ig54f>Ax@|4v-?B~QhC|89GG3K~a(^$Xt#QixfAQ#_i)Io{(jOY6Wf)UR z8Uzy>nYN|WnjRElVA5RTXE2oiD?Y%S;J|yKQSrzN zlY+x43zl;`{kpGM;f=J4|k%b%z&dGPMA*75u!N4K;3rJvNZzNPm_ z)?)dytvt=E-aeY-79eS*>A<F0h%TUa#u(E?aBQY^=i?TySUhOGfV{j@v6v2c&R#2!G`+Ihw`p$RFU? zf4J{jz!jN22W3(YN^CqR62d6oa~k%HmQgH2Il(r;>;F+ku{Q^+FC-~A zG-041?H>CbJt&rZ!FNE#_<<4hLo*WiDZ0tZ0zWXg1ei)w5w{t8ifa z(jc#LP{yZ8-efD|nghxYnlu9r3-UB6tza-`Y}dUg%^=Fjz`?MD`M6t5mD%41HP8Ni zjaRBTd`&M-xo|~Tq18mKQ(qWW1kBp>{VWSAESET2T{xUH|Kpm=R_iA3So^-&#G`p@ zOjDP6Q=Tf5)EXCoEk`6~aQD``o-S<_$Z*|kUEQ_O)h)K+?|Iix zN+q|H);QgrfB)xNCoP92=@0?up8ImHOqx0A3Qz9u{?{n4aa4ujP$4su(2)Z&Ck~4E zxUv`AKj7_9v7SLDpxKOvNjAh<@eCvDpObkKt{Sjipq!QD_zhtw1 zZCt(f!X2Uky{W=huh%v2`qLC|^|5|c?DREOM!cW)e}3(HVAcgj6`R9rU7Sq>%vQI5 z{HRoG|Id6|@_NH{*R4|BYmGBv95T7D6`$S^&(S86ec(&C+6I=iG|Nju=Bc|o6;rH!8uXY_lXXdNQVD3&EN?V zD=$6wV3RNklUaD^;lXBZK?9cw9}XN8P~D+Zy5hmX!|i-xG8P$yY<_d>@*i!9yv)`& z)vE4=MpN@bM{ zYQ0@Ao}*s*#Fhp|0hRQNt2|eW=}K9qJ~=TpczNDEo9b`x?jCOUpTDne{_l^%x>fR- zmzREb3_klt$k46lpOk7)vzzZM!{n9;f=$iled78tI~EAKOcv70zHw&d=P3dUozL;h z&Ju~WDt&#P+4N??BbUkV?#(R`&u{S*l90NPZL!g1@obAX%$YsH94Fhj4bomDTsrQ{ z$u1^$z=3r-)9wT&o~TI=@|dP*{VQ^l>FI-Ad0Fzar)c;d-&lgmpQ+a%MvG}$EGggQ5ynda^0 zZ8xl_e%L17vO=*#VZ{Qao&T&p-B@h-Vu|*XX#shPPp8%A?MxN&RA0Geg2TBR>o^mjg=W4!3#C3Izoe~fd=Zl#sB$W0 zoNQ#b_|U+>+@`Uhi9hFnLh}NX9S4}CA9Xxr{6E3fV8zy;IfaajQUVMfTN+fdwb{1p z;%Zd5C}h+qQ7#jtD{(jJ^}3xwM^>-eCB@;$$m-OvvW4kGLyJ__9M7DMM-{pyj7l$U zJmYO({o3b@(ODH=-!ucwXQq}{H7a?G-W&<;m>RxpA-m>^4NbgG-%mW*`9f{(Q?a?l zFD);{l<`%Gx%1yl-YtBNqh+@*Gh^z%#WB+(4)IFpIOsET2i#={4fah^;!`%cv4Htc zn!;BO?@Nx)=cG^2P;lh#n3|xLV7Vf(FS_91GHH7*jRQ^Q%Q$PKqGPHOIK-QR4luEH zu_(AQ%H%IN%J!^F+m(TDg^asgdrA1}%E~p{8PETWdhJp2+w0?cK_+P*MoWdvWwSL` zS5_n*S){3+RIIV=xzU+F7ki9*9Un(*F}>}=Hq&K-=FQ3}fydsSo7Vh7@!3q{m5VLs zXwOX*T9E%T^3TVU)3@+T><~VYxn7Zxb&`Y8jE5a|=Wcv>^(AMk74zEO8OogedJ7u) zWe#%WO*qWM_OQ2j0kdqKMcBqCZH$}(3=!fx82-s_IM|!+^H{RDB7n80pgZ#clk9^Z z4G&c&G;u|qVB}N~XW;l?!Dy45CW0Dfdm1gzRk|##6WcfMC6jp1m7~UTGrQBD z9GbX;;g&*90@F{<6g$C478Nl@X5|x#0>&QvY;2Rg{&;4y6)iX{!{o?r`9zDa(4k3r zMIm>vO*5}qH6pL?Kzf#rY&!xj&Q49V-EH@;3?dxJyg%c~gPT9qZ+=d9EBRq>8z zN^TN}SimAy(5#$ixcgRM@ya1!T<#lzzS-!m6%a@?l|G#k`+9_A@>%duGbt zpR&UDOUr~ddnQJXoCIgFX_Gsjg=mRP4ocCJy6JH+GFhsBOPgNBse>L~hd5*{7=^_G zPrJ@Z6r6a#S)p%3i-*o54%rI~T7i}`cvDT8B{ZaEHzcrX%P`uz&14jMq@dyC!|2H0 z(8;3qAxtagk<66^&SL8}v|4%`Y?9pIq&VksyJkrOtIoj&zPf)!ilPY(vzab58_!DQ z^qg}@Xj(y=@vQ|7CEJ8uIcLo2O<6K^&5;vsc4w{&+%FYXeEL~8CQ>L~<2nPYPhpp; z#zE1$2^}E;&R#sRH=I^)++dj2yy%HRyV;qHizNwDdgBkd+gYvZpZL(QKdWJ0?y&`z zOac^niv#+;EeO^OzOyCWmUoV6-KpR&>%L3Axa(S7n8>AaB)`rx^^o9<2W`dy4>|IF z9F~7nFfSyDVYQgS!LY~f?7u!XaL%=AR{C{;Nw`Fm&2h!SjZ7kp>?f8mauu~0KXYhc zig^$z_Jx6;EwRh}&Le@G6U;KV3>LY6Y2-5KI4nQoK?9S(0{-P4%>2s&{xPe_EaW{G z;bJ9PaF%h&W{-%K={uwhZm}N8K9#jGe8Rn@yVPDSE&0=xbkIkjQD{LCZ{G&jw@tsJ zAEqTQy1QbvWlwdC|&X=#;0;i=1Q+=&JV`xWSGYEZS(x#k zkjRz=JGZtLG_UT>Nm(q-{NC;0>cqah8INU-xW1e7@{D!q5$47FP8-|($qDMcu6Cg@ zty%8NguO1s=L5}aRRU!9a(g5*OJCc;X11-dclu_Ty~-w0H;e)pzl5BPVwpCLNoK>s z)$?w#gzkR(*Gj=&Ri^Qvj|T_;I<0nhfh0b&2M+6=ikY*UB(ORwG_Y`eXq50sE=FwW2)5zt$fJtb1hpb8f=Z5u5 ztnVgk3JF{k-L>V!Q+>}(-mjRPnRsuMIeSa6Xsubuo5M5b%hL4EmUm+K+Kg?apLgt# zR8{*}@$bUnN8c8#^IRLfsc-qi$BOP>&K4Y7G25xAxi3!tbm}?Yr~36juRl0;dc9$< zLrDVjOUKeSEfWdG64R+mwoPpe%r*TIlfWFhx~@uZ0!#V=X6aiyj&ki$HD~82o~8N0 zfqk6=$DYu~l4b>B{x%2LvKBDgFgO}DFk3nBEn~2~lOV9-7~?btrBe?j=Pi`wdB_~F zQ0Pemr;?)79S6ZVj%*E0AFnkrTxg1Uq}r9Ty5!u`yQ`eGURd;L!V*0Z1|}s3My@WF zLu(dHX%O&fl;S%l&5_5k`=B&m-opt3U&H#p#^skEOO(Cl`0a(__Y)3}eYAEn@jh|T z6Lwn{B+$8MnWDJdKi*466o1Zm&$H7}W(vRbvWEh83JqJ>uKK)X^f(*klk3cN>dLBT z2Le=WjSfWp+VJbrd9e$MQd=HMRxMDKI>20VcDl`hw`z(!huOtfEnregVLNh;;}ZiX z2ScP0gW`u|u1yI784YS_3q(s59lp&%gen1|t@q)3C>6VIX?4@M>j2CfBj znH(6iG7|DMJRWuGZaBlLsjt821RL{WPqP-ze+xwB9Xxx^v2o2unGJ$pw-idR-1zlO z;|`T&hEmN^#~Njh9qd_S@ZG|X|B?gu&VuSa9|BLjezNAFWXQW8r!M@wa6x*HqRh1O z@;eiy)D9{Joc#4BfS-F!mPnEFKe0{x>z66C>e*I25X?z@WwcOa=gSDC6{<`V1PUC3 zxE6|dEVQ$Dz^uk7uC~xJ%YpS8A9I!hXBQXus|Oq$JGd0>o)$gaFmZ>}xt@;fo+B${PAWWK_ zGJ!9C(epj$e=A+8`(se=vw%4u_0YHUcM}r+u})&(k2z3akify`ay+JqP3@ti+QK)_ z68K**uw8h^wB*6M7Yq234zPVl;9awdSucU>7XzbZ!+{V5p+^@@div%nJp-(@+^xFv%p z9TAvx%;1a4=buy78|{B6ZDlN@Cv0pR%xnKytmic(y}(R1LD|$NLZyLimNJ`8hqF!wTh_k^Z*v%;tR67C^s40^;MF?7 z#?iQbU7mqbqQI)B8o8;00fqu$7bU$8@`|}{d!))3aX?T>kk)|s^|A)o^#*<4nN z_$_3;*BdWzp!fo7>6AhpZZ(4yFRo-U2q-B^-+CB)exa;PNr=3O)BQ`*4|94>Y}7wH z(Qrwk%o0bbEvp}Io4_sjS^i&Qkl5OEjSqV+cuUt^l{%s*Ki%Ig`HE3?iSg7@x5Ede zCV4*hF!o?fF;Hr_vZ>3V^&KBCkL`~SA?Fx`?mg#8SirX7f%kO=wntj`4Ia1{wYcap zFv}ge#ihU|vrtGbQDD*oCYdKpX$;(V6xgn8;!0vzRwd-PO<3Sxl0xE9KjtPO(J6|2 zS_@^g7D^g5Fl^If6K*h&ab!z7z%pS0TZsZ&PXiy%!a%D80WU{3AA!=OCmkg!pBO^i zXKYOHYUpY+VP{AbSkd_2Zfd%_sjSMCkk^;KehTXORph*Mp>wjS@4pon);YM`Hx%Ae z5abs!Nv>GD=Y9Cmhfk-Sm%bw4c2v>r@IvGLj4}rlWzK9f-rFc$;K_qgQJ}v9-U6}PT_Yvgk)e!lE$gYVm@)uH@r`LFsVJU2?=nnUW&By3 zYzhv{QR$yoh_X#d;9cdwckKc12L?W+gOYD9zh^oqprR;n;3}8i0XC(BY&r^TuNv45 z6s(OG@c&!DZ?;1E%|qs245E)7aHPHEiDKZ$P^!GRh~KjADy>ANtn;krJ8@Gm<>bscgF8Jk_$~L2cGvPT;+5uj%M8OXYSq(~|XBhZS zC2-D2y3KHqpW`9F%t66PjGOdUOBOJS&p9YopeTATfi)>X;Kc()CC6g3( zW!*2c1U5g0&c_PO6C^~{&%6k^&YR-EY^BR;<-qFo>0gfmU%*3=SqCKp6a^R*1$+)l z1|@QaC2*=R3fL{+QaLDjgTc{2fzOMPdxHY^h6Jtz1^xg={)W8Q+xR?ZtzlGJz^~QF z$+ehYY&nCB0~cGP0Fz@@P9o>6&e&_2j5k`{K6NrCXcudhvM3}hIHJQ;B_Z_eN7efs z>F+J1ze{`zx%zJrqjBcK#tEjsl6hJMWEhtmZr`Kv!SFhF#8F4-UDu3EgN;oim;NuW zkH59_>NP1oM{nJX=U$e({C6t&&s1DG^~yPa=4lJ~e<=tk9khOyZWA8N_UQm`1p}MF zhc{LaEVnoC-D}_~yDqR`C%+v7x87?jLj|@~44hgEm~EKYax9ovDTw+til{gW$sH8g zCB?_(u&wII-G8i14;F}gc))YVf&G{-dqo19#WeOlh1SyvaSRXUYaY;B))U*dq(k8W zgAxOyh=X3%pV>IPGPM6nYZ)S!oFZ>X>Ai3eJLq}s%_I5rKUAX&S75S`*31! zzBpfC#KJ$@3#aTzp0zagpH%L}YfHU3mYw}3eeIf*$3eDLs!^UZuXx>@ve}C{tWvR6 zDm!QCz6HE&W)C>tgz%kfUL$fY80UsBm_>M+_2@eGCC9rkfkPNyZ`H#WU z(jh)~!3qgSN77A%dh?J0P;<=}%)gNt~e7>>%VwmyVs~jbH-oC!O zihu38=v=r)I(yZ#tZmBS0jkVOOd|ip7`g5K2#PuKJvlH#m61P3fjNwUZB2rZlfFPw z!=Xe4wr2}O*p~C}S~bfrhkuPj&N2o*Hb=I)LrlLI_>C6ii_C88VC_mW*!GIa!{7kR zqysN~PO{V~T()}3;6FHJb=vGA2#BVWh?CYgJJj}q8eT>iOO9bmh6fGcc1pWc7oCkpXT7x0HT z9Dk7J<;}2;f$hrIBnA@(p@76yAq;FHjhv?%{AMOFm}JKZBrxqsib?x$t=)yS*@;0X z#UoAoeu5L@t;ieiE&g0?JU%g2R_m9(epOBSZ;6oRzf1rAo!)pcNW^GijM2_mqx*F^ znIU&x@B9D$-crVbrJ1dGljC?>M; zN}qFy3embC6tvXqYvE`2K6xu+gO~>a&Zp(9LMQxTWN~J0H;XgKKT%EKo!No8iB zbDy+!!Gk>)8A2CwGD@iUOi*BSYTlS~(k)|w17nX@L)t%&;>+r{W(IXXvbb!Z;K;cufPsb6;=&@Mr4B-81uY$#E-9%8{9|@z;8WS)5YM5o zz=5gJbiwpKP7Q{e{DN~9G&7m5S;)f1XVSo!%6I4C3Dui(4y&c-%t||#S+q*5*JSNp zv1p@Bvc|U!ciau*G2Z(!>8$Z?f$z~pk75q@I^46d5-2$n>7KcCY7u*1@&30D89lhB zmd~qvXL{wHlkAKFCvJl|8&)w%PWdp;gE?gbqi{EiP`ek~zlR+%6%3Qu6$&4G)RwFJ z!gxZd&!wrI({1g=GrE($1RPd5EAXjNaPpK%t~|9fKDqPCe_`yDF?0z?V-ct^ZR{29 zv78pc;Id#kv&57pFXL4%OFZP$mn`Zr&f_RJz{sueVEPmR8HR?0pg7G`zUBX>2a|?DQM2e|@g}vi78Op#JSO$`jy<1$#vs@y|CZ0^*#*ZWx62gU z`Af+zox#QHXMK6$=D8(jwl>EuzLrq!$S5A;u&Q#Q+mW~`#e*AM6e50zpOAO2P`Tp7 zsabMFGW|>en{bf}Q>U~+3PUrO){KDiyGA$WGVvHmv@)`XWL)l1jXd-5jPdpw19rWg zXA%$V?zj=;$}{=SrxQZrD;_XJSQrT$4Ck1`%GeuX_P~K<>Vb`IMqP|d9ef6^i&+?| zHtyaqg~#B63Ui3ig9ZjZi$^{tsSyQi{1g8)UP}>}!f;TkrNO6n&b7(LeBUmv*t6;M zKb_4o#%DI23_D|VLLvC`!ozQMCOFtdbN#x&E#+`cXyv zpCgG|rr5Plx|^6;suL-j7ojX}6v1)rM)0&Xu1gs1=s2up+SuhF@Q|ahz*$>SL#QL; z0gJ*#2KL@0M=q;>-Woef92hMUm{k-S+2kf1U^ZE}Bk|UQBK;|co^m{F)V5g1CwHUy z$%F>RK)VE1nHLW1Z66r;1QzJLRyy4$v3UvG2?Zm@00#z^h9ecxW63b@%A+coZ35#QH)~_!1M>#L)Et0?R zX1d9{9VU}5BxYK_2{(D}aXiCdYM;fL#%`%8=?r|1bD8fQRqiO*#K>|WjeQmaSB&XH zj-nY%K@Tra3|CXJcXQ+jVPv=9?J#01-f>5lFR_(Rq=8L$kE?800f%_l1LmK1ojamD z+PTj>6gxYkiNo5U)#}N^e_S3CPK>K|hf6mW>(qBPyuF}gb4=xd4#N*_CZP{E^tc=j zcT5#h7iC~@z4L&jUc*tw>);wj4h1E}4hPN(hNz+&na4GwZ=OjDyLdHe%T|}RweG@- z(b)-$10N}97U*8(D_Kyoqkm43%6gxtuL3sJ-P>`@cv(OrQ^*0uNbi0otpyBUdR{WJ z>~LV8ae-Yer<~2bqfuM&hr{wqlbk*Ya)s)4#C_uUA=X*VrpK|6lbJ_Rc>4|(#SIvY?6igXS9gO_%JF-rMZ3d;@NvjbE#mn1Vi#q zgQ?FnmolA*Vyq5im=ZNXOju(9%Rk)-j=W47Gf%#8nDy9$E$3ayEqV6H6V7F8Cr(_* z=BA<~Xc7Er(%z4~3k-diYF*t_(EQs&XaALnQ@Pm9N+z}{w*0)Dsu#*|F_?B{PyT+Nn} z6V2OMaV&q`#+1AY<=~l4FL!R8^G+vt`JKvt*)~_eDX+1^GqX(L0=2c@}i0-cnc_IiqRM>kUr) zYZh>adu-v|G@(;-#T~ZjjD<4%AA7U|8qO-$9OQPn(9o>P!?;cRdHkjw8<=0bI4oZ` zqfOMUmg9%ihFqUt<||ERn%z*iSAtcte$j>{{a zdbpo@x$H?^@GF_8YB6Q)ssGB$e-y`L98F93s3&T`!1tnIjtc|ZhfD?$r=?Dno0d3! zGW7oXFhp$uqidjuxuc?t0;5a-Yn=hdLU)eX0QTw)tp74yIl?b6%Prt^OJEjdV44+B zZMA~ajDd-ffyu0ZHQa!O`2%D02KLwoP1Olp#RkRB28{X}m^3FaaSQkd3q|gFoEY=O z)hWF;xT&ODw9S*S{if)^DlH%iwvN-kwrk<~FUY+M+&d0$bkm zvU_Uf-L3;0lavu_e*wn^jjGKRmO2tuMqAi_mU2{2VAYAJs#b80R$vV` zV2$0t9zLPFbOGneG*_7kO!^6~Ne``g9BN~4h_iiebyoi;aaCP{^)SO0GqDa;hF>a7 zaW2YBKPzc};FvU_G_NDDB%-~vqW!1?PwMjakLLBA4)vJ_`a2&>A6m%UyNLJvMH$wL zj;V*rcVG3`{Z!_7Dwo`c%%pFOejiw8Z{VDlFfrx1AbS6VYlKJkwqgSXwxrlEQyU8nzOp1KyD>c0U=UoH z&8XlP-ePr&ol&!ZW7-9t4<79)=KU3!{U6$Mf39H5IMDycg01aeScXM2?~Mh#_Y?%K zCZ<-DPxv65TD85ysFb^ny}<7Q>l_8Hc?V{SzvP^AfHgjVMf$=-z6!yWj~N66yd@RA ze>#SE7qGY$uox7u_-tU@c#}~+fI0L7o7V!7o@o|44@e7@bGI-|*(Om{tuQ4vp?iY| z*H>4rV=Wrt3Cv~%p?Vvbv^FpqOkk36NUVD(uBgT!&*s)>=+F8&vA(dkztokE+#q=*LrhoO|IZ}{Y(~8TYC(SAC z21}i1avxgBou^(n*?@EB0*n3+&1MUjY%k0XKEPIRf%%?iV*E~qZHwo{y^M(K?Df4Q zp20caC4t3v0c-Ut*4PUyz8@H8{K#w*WE6McSQ0S(NCFS*t`*%;3;MGbe3;JhW5t3D zh8dX#Y)%gr@crV6i^?!xobp3_!kNOn-p&cHww1eop0UJgp+5st%!8SlRTIT-&YXRK zbIyi|OFSKHoV4CLF7{((_IV)hTfo+MLB4wfN5lhDw+~GppEAmCP_&m&vaw(kGJ(6*$kNEA%5?>MaRRGtz}%~yECmki{RNyeHZ$=#Wbe7$ z%iZ7_H?#NU%6Ziv#oxPhx_(&hw}918fHnRCYrFz`{0CNF2Nu2vhTIUz;*~uAI9DEV z;AGvkkFw#Yxl(MTC-$VriHOCHjo&4hmomN?2CBU`hLep0h1W`-Hi|xn!EnEAEQ%`FhSZJFrf!VTmnhVJ&N-7HfzB zd&$2C>@5Pi`UYYw0S0Z}NxfequDKgT&UEegI!}7Sa=zCrQ*X2Q73>uG$r`7?-eAS> zqS0#dfxf8$3zm3uN>}faI?XZV0LNPEmFrjcZ_Hk);l0~w!OGcP%bX24->$X@w@EnI*lD1kN7fuqHN-6Mc`(ghwv13$)s-Y7}`$CA_Xw3ho$ zYURJdQuBfJ-_+d`r=LDF(|cl#0IR$KgSQfcN;%iR&RxrI@3MZg>z3H2ob9=t3GDTq zY)JvTEiNpW-Nh7Rwf^FzO?w%Sik{*X&kEXCx@uxds)wy^Y+f9JrcoPJH{m z@5Uz9RSUQlI&dv;IMFVB52)p%x@BIbdD;uPzsBui6%>AN)yF`K2 z$br3*fmz6bach}D0t?Pzj^2iU9K91buB^#*lsWRn{76~@n}#+U->f5EJq)} zPBT6rz1cslqs%-{V`=nal(sJgON+Xdt%%N*1S0k{S&y({5f;s3`a-7*(*I~GjrL?5>~qA z9P#McZPCEQ*TvNGfMasNxzADY0{>pEyex3w^&Mjgu2klXm^a6wS%1oWi`hHJlIz_A zz7HGDzumz5<^t~yAWHX(W0@6=H3_oz(4Y(-D%enBuiR}}X z>aqB`6d6Mi#au4XSa0g&D=M4 z&RsZI-=%T>N?^>)O}x7g%$)P`#=i&r^B(TWd9>%>n+eCZ9N?UN;p%5Khaw(_De8{v zK6%MM;J9kQJNeFu<~3Y99`Ni?;A}BqziGtTC&RU6?d>)N4wc%AD-BrX92jLMurG7q zz52kSONOiM0Q-Ul?sK-mEoQtE?-a@vT!{%RKT&!JdPELq@|CZHAhGXReo~{KPmtwBnICDR{_QikR2N?j57G?S z>-c5(6OB7CY&EfuMs-&94 z92&sk+;?$G&BdVgT&LIGUb2B*Rf*Xwfh9D7W6whFrZcy)@AJO@z`L`6W16Rtk)z2bRPCv*s=t*SU6LaQ60>60l^Af-3;cGYE4Bc7tfJO8G`}PFxz6OrYf}g*2 zk2vqSrm~tX?ZeNMgpWxHN0JMecxN$B(z`zS9Gl{LcCLAaAG$f8`((tux}K*k)1k?? ztyFWul!r4v@m`7EvcKW;hX>r(H*l+-U99LH68BHgs1%U~e$sxOVS^qAb_(+NGBZcvmuT?TqDE@_{{Afi+2+Ro;Q6 zH6f#$M_>U<ngSZsKpbev7f*-Ri|;Yo3I zKbM35K2PTjUg|Za(PbRq_$F-5k{rb9`8!;E1%`V^cnPKERtL5b--5uxWJ)LBoJIi#>wJn*#%XRPE z-~WJxpRIkJLE@1|2RgaGzm>KwTW$04*VEIR^rEKKI;N@j#hRbkW42VrAYldplDrXs5^BAW^OBxbihJu0N!N1JxPI?zO5*izr|1d9L z>=C;0fM?1aiKFuW>Mv|$5lR;GJf=3YLosb)sL-7hW?hfUWs|Hsw6p|Gx`Gb(EuB-u z#k*pelyi^ardwUw>kc?A>@~UJv6#=c!sWHjHnm1=y`5cKyruVk`*d3Rzy;3d21j_D zeKwSF{C+koyF~El5jLX{c0GL%$Ve>Of32~uc0>6D^> zq`*YoEXPSSTI*u+k;PM-3jb;5B+l{T$xA@m<>F7n2+VBVd!xjI+Bw0QeREL>xhcqdaruDtH&(aF9GZY*eE zm71}+fyJWCfq^}YThjk((q-R9cBKQpooO}9eG6l{wEcEGl4%v0;?C=uy3oh*rHQql z`pqlpFZgciiNC(F^Qpj|11FzOaPD!)b@;bRY=SbQ`>7ucj9M0&O9LM?x=)WWU|lMb ztmD%r;VQQBm~4XM&4kX7fQ2eYvOc>B9yJNM^UZa2Qct1V`XjQ5yv4%y+udz1{9>Pc z+_3QaiS#Wqj}@tI{m2$y`}^zn&qw#&{O3B|*X8Du2Q8YyX{GIoeKW-U?Fy5c8Py^Z z7$?cTxY1~79oFk-?Ym7OmAztv>g>#rokw@fk&iVIaWAa966_@&der8{rO%eu3%F;p zSo!h>9-UQo?H0=v$^s(rg}!xn6R6YV@4>o9{b)MKcZ@q+#IWixGogV+f$8AfH#zFI zH4j1@+siKau1u18Q_-%s=p$dr0%h@5ixyG24|Z%%xENS2{L{F`v_*cM?g0ii4X36B z23>wCZ^ZX*=rO2yXfXX(ldyGW6O*69Dc3ndnqqfmvmQOLDD;#EPxM8u-XmA1Dr6|} zdM#}V3an_|XwWx3JyCr3jF0OQ`6d^YU*tAA{BXKDquQG8hl$2J3pI5`4strU%+-!j zH0-SGOj@kz#LD)-j_ZUc<1>?V(VtrCrDh7Ia?Fv;Aq|{aKF$_Xl1@}MBp-Y3;=6Yn#Ym5aM-6Pc8nRC_-2Hu-W#3n{{W_4?yL6%ww|K#U20bPQh7uQLiG%TLld2mS z1QHk|D^fPwIC95{2ucWfFG|f|T%(cVz$oA0qA617^&-lf$FDK)U+GbW5P`OW77YnU zfl0Hj2;G{bqor~>6|(8sCy8 zF&C9}N0iLCG~OKH5cFtZQ*-2Uf6(hO_;mC5z4g5hal}i;c3| zlG?2`7UlN+XjT;4$Pq8jbmOjZh6;yQS7ym$*%e!lCG~G^=N52K7GJn_Lb7!VqwJ(? zCW(gYvo73L@n>7N|3T}UE5CAiO%xd97|tu-ixII^IohOCbn1RWNweXD*()As&EeRY z{Yrq+eG!>c@5t$7eSSA`pXFC87&=-j4Tr-c4b&^ z6Il@IzS^ykp=374Uf+}I3pRB68=Rgz|4J}}Q~^hD1e2ii%xjt(uZkSFYkfZ=(PsbF zY-Quy3J!Qlf?4h0gw01WS3{IwAgAaa+Os%!;w3B#^I^; zVH~a6%C!C+8;*tRrguB8XkcJ$ z(4FvC|DOWGf`(NO`x&0>|0U7jlHFu{!ZsoEfX0O;wFX9335HpdTY`76cxAL4Tr4(k z6SMIGy`#Uy)(RM}lW4AE?iQTDC?&x3{@1$5Qxlh;5-hIVe74!x{fBDWg6>@_=A6FS z{o@tewapfPs%$qqt<}51r1`^4S))m9hpqaZ2G7-dd3Q9hS2QqfaJsVLQ1>PiW(5ZC z&xal`I~&$8@XTmVKf(6Ac#1;?Q`e~`y&bH6g&c{(%j|%wor&@u{NE zVFRPGK@;BvrcWpMVs5nkG+vgpi`D8y!-LQLeH)}!J2HTly)ZN|bnIsQ(J*=OZZ;cB z|9c-NM>=2io>*=+0cxmcBv6=Md|rgRP$f`N|Dh zgB4gkD_BZwS{xNv%9b#fuW7cN(QLkw zWx>O~(@k7P0?jeSEbbL8&LS)gN*bRuSe!DNPOWNGu{iSL09(hdR?iR3Rte3TFBuL2gQ2YhxG zES6{b?4B@N1uT{{XmZwRamisXIne5Jfu;72tLT9S_8W^hKkWUksM2TQ$Ff4o+ojHyVUjoa4N+`ga2Z$A%7OjZ-NGZAltzJ_>DNcO0@`D#)KO7h(|? zdtolLK~TJ)@x<-MxGzUdyk(la|1Df$<-KLc!P3U=)w2|{74~iqnRNB^LYbnT{0D6X z1p&tz)CvoD3JU^cn0SsIu;2;}C`nMuyrS6so4feNoQ_j$u?~H07A!#znzd&zy%*GY zcfr);K;t{yvR)SexfXQ&{(Uo_)gfke0f7r=dG)iVPib*sIJ2VOj zGzzQ;7L;fdGH6TT@J|--{~U5ETt_uelKJ38^W)NrYhoCs7MvI5X=-@gsBpsHH)8Wr zFB#FS%`3CIj|U!8QruhF*~a90Aw?-L?OzuA@eSS_TfH5#m#G~LIByZ!eb{1G&((-j zq3+^I1urhtY-Ef1(05v;#r6i1@{MNGs>L20zD76prA+PGR=HoUbgLY1i*_@I@(X8| zHEo{)*^4sT(kxn|4cel2v;=e5cy3_v)VS{D5bnL9#jB#Tv@(3@83%{lmWs94tCogW zg|fIbw53?I)tR<=N%${KVd0hGS`wiu|6>}zkybr(w`c{URKS(kE&hL=1ZjD!S{9`x zBIPZsxqhWmOS!Lp-c<3bgKfc=+j1^!VENjXmvP}IS35_knqze6jMNJ}swM{y+>|?{ zFYuRJ{byvs47MjXm~|DRbPbr?7j1o8V0yu(@!#9vYr+D}b`#dTX$bpl+@g1aZ}sN} z+ZUz*J6JzewwDC3XJ)je?r9D7hz_36vVBI2H^-jRy(~ND-ri~2=KUhr=S6U`gI97w z+dk8_e?qB~y%JoD}6>s%|Ub-oc^PuDmzn#lgJ}YS%trm?^rtSb&FnZ>)grRnE{J zb;f(YX9dWenjUjiJm2j2aD5= zCfx@Djt`hUCiGog)Lz!nwt81fT17W^%k5HMS8s#Xpbh_8f)?ENT5)?fYs5#H^GOM9 zNfB&m7Hz2lF^7E<4#mdQwX)ThKB(7i+u|wxIfcbFp~b}@!g_%1>bLIUTyOOC#?^=%*R`_!$@l432~-%yRxX#t57(1VJV~y&vt=cx+m9y)|4Dc4VVv_-q{+aU zQP^bFx(j!H)-eA1a_9WhQ(`U;g}9oIcs5Jg+%<1DF015DJJ33DD%;KrZ5b9fZ?3&7 z@U-psEN`dmqz9!*>ViFWjmy$@>8C4gxg**mZ`VHGwLK|`FJ?vCo-J&-A*~(`i{CPy zjgL-mcob+C!IH?#YWILSESqV`+x4~ura>IhRUYi$c=s-n2`}QYDUON$t=sA#_r&|g z{oPwxk_%q!>3#9>N(yt?6XrIy!(YR_>YjMNyY71}I^|=E%YlSIy{^cZrAL_+*LV1f z7d#b}NEGUODl5PwJtOs0^(qM+adYn-+a^S>Nn+b6@XWfq?aa;Y;u~`o&-4D5_K&SZ z!K5&t=hrXpv{wNICt@S-PXFzclz!-H&CkS=7j2n0Bx4S^#r|*)>S*yT+$#Kdxv+Eg zxm$g2EfhWq-g+3J^Dbb;MWNgEd29toA26#W&LjJx_f36}v>=1I z>_Op+>xZ7ca9@^jT(!+ljm>QViyKF3*2L|OyZl)mxVmvZV^hR?YaPCv2FE^Yqras(B* zDub6URJtrB_;9+}+v#FWSGJuue(f3+lvu22VWzMn$>;872Vcv>tEB#gRGyM4mbiN7 zZm!zTw&mMCrU^Z==Vq^Ycyoi2NxD$a;%lM5UIn}m>-V&5Y3X8_T`0%0=^gVao%2>2 z=AWi`DzrE^v_vk}X!UwuKBK)Pqb;RE{KwSy2aI1BpT8fh@WxRt#ra-*+1cA=yb()S z5}52ioGQ!Vth+OTEtO;Xhae}_i^~EF1eqi+6kIN26io?PYmzALljyNO!n<^{*sNUl z4XbWlb4*ftRsB!$qG0SYf!3P=+duB-NSa~$HSO8WIiX)ez9+6L6Uv{#7j4nk+9CP+ zrNIlqr9PP=b6$$PR+|;kEZ1Pvq8k^L;+T^;0g_W2XCk z5q)y>{M;*%k1j3I5>L|6msqwcepT**IyPq0XL}_A?=7>CTee>DSfRB4*D(8}Loae4 zKmEvew0rjk?oE*$o4&a{T~Qhuz&HQ#Oarf0H%tB?57rlfivt9t%xAsMnUP#QgFQci zEqceQeSWvMPlztMnj9Qa{4Z;N{NHJ}y%NHYlwHr+-kIXamgLZuyr8Nu|INoG2`Qf6 z+1XRe>&rzdy(BbKon)D-`)`O=+}INSdrSJ$E6Xw?^WJ)_z0!2rMrz92EiYc~@c2;} zP?3~ZvH05C;tS$3pB~mPtWc713X|tKfALxU%Q?rtygHP@UT6@?mK&+FkxfT1y-@A- z%MZt*)_9kQuW#S$Q0QBgoqpYWMoY=s z77vFU_YY-W4sWHDz6VxnO}*QW0eWFta8 z`2Q9U`2Eqpddu}Yzqa_FPd#{Ht*V*yzY5zK2Uo7r^4X!)Qs1^u{Z)nnyTtv6is@<% z7VPJwt@4=)C})&1BS68aMTA$)<-385c^?1$t;{#`R8CG(z1iz>k}uJs zgyZN|VfA;?zIoU4k~Wv|&R&t3|`yPSCg)!O!t{< zRC@K2s&|~e?O7m5Y-RcapKLP)(Hw`YPSWF4!cf}b&O+P zc&P1^s8;5>C5gq+{>$#oH1Iui<;%6T>1Ud}s+Z~ho3-uP*}abKa_?d+ny;;G@7`gM z=+t^&B#bBi!XxJrInzy7DnH-yDb&7y`A=KlGVcPF#7)lE_5RoW75#c#ozMLI9j6ZE z?LC)yOuSSl`vix2nu>%u-*uX*8C9vu86oYb$0%O)VZ9FhqE_iPHM~D zkvMa<+@7yuGHU+~E>9^u@8YaB^Gs&4->i$dEO#w0%#eBJ zcPfPa^tQiw$Sz+kP~@T5-(bY4**|F!m*(~rle(8qw_D7ucP-+}meVG;a*8Kg+$h+5 z*6i|~lwQ*vDMg$nXFEEl1SL9sof5LFW#-k8;Mj_Xs?uRWCsk7Cnl4hFIK_jrD^Jsj zN9o)@(Y4)|Pg#Wbn*E!a+-Iw_?M70sqnH4L;!hS1BL)Tr9R>yl1_{Ox#&$j)9${Wy zNnS1yAzo<_UTFyqK0YBnJ}Du7AxS=AenAm_AxRNo5ou8o8A(x55lLw=DOo8gF$pm~ zL0(lUP9-r>Whqf*X=yE488u09Q(0+UaSk~(0d-Y&Gc^tsMOkfS84VRNBNZ7lO(`Q2 zL0M@P8Bqge86_=QWi=TM6FDV0MI9AoWo=bu9W7N=Wo<20ZC!0`HBB`mRb^ux6(e13 zD_w0%O;uw=fFB=nQGiw)HD=%AXFEf)6OJgq$wOj)=Z&_JAA7=9? z$zXTBJU_7rHw`xjQ!ghoS4SIf7i&LHTVGGpAaC1HcgsLe!!!?#@(9iJP|J!avn)S@ z$`qsAY#s|sH$5|7WqEg3YiD;`S3g^KYv)jNrvPKmbRQQdA1`N5SC0^1r*JR#umG2s zV4sL!4|mV-=&+C=-;mS@uaxMJ^r-OsFyG1)&z$7w%#`q&?C7%eh?3ke?~sh}$TZ)u zoQ&9%sFd>Tw3PC!w3_1V%98Z-tcvQ=vcmk*_{8Ww1D*Ni2JMl$Z5b95@(h<(m`sin zZ%y-P$%<$ziJn^Gx2!CvtvsWpx}?3btgkF*MP0&1myq(eANnl7DwoPFG9i)Q-y8ld7k+WiFjwxvsxp z>$2i)EB%ko^f|vK;M}DA^Xu}SFD&@-IpFE7v~3TgvPwG2D%#6xCubIQMrSp5);2w< zXs&LaTGiCu)zUhxt9AC2_V%tBGpEj+Fk!~hnVrjLPhC5sfA!+-Ez7#rES|o3@zgD= zr)*v_d+XZSyVlOwwQ0_djWhRd>YX%w>Ed~dRxVk#b=A_{o0o6jvVGmUEi;#GJGHsz z@Xj@_=1zHbb?K&^C+Ds`wDZW7y=NYuJ$w1$jpw(XetQ1&<+In{|Ni^;^~bM|U%q~Q zcIDWhj+_~AxEv|2{)8ysLmhN+Ufi-24uYxlstHn%n z>pc~sd1bZJX1lLDzrMP>*7tCi>}`>(ON%VppT%^Eq={@VxUH2NxqRK--En&yW?pvb zxWC`I?#Bl~Wi}qWnhy$o85Pn9Px`@feQ>(u2 zELrQ@Sa5gV)HheQyWRZ9=E_~H@k3+V#bQpmJKU#sm%cl&e)9G130C!gnIs=IdptbM zqL_A}fm7B+(q)3>mI)41V^soEdCfT3+@w3dO>|Sqy)v^+V%wFdZ2i`bpq#`#A&Ku{$fc*02y8vCC-KlhAWmhHr~J$>&$AZuR%v9fZe?U! zq+7aBsB_}2WqC_>$*O8r>$@J!FAOLP4=In_cvXMxZO7F7gUgCyr(I_&KKXdcv!h$* z9e$nId;6yNcH8*d-zH`*6ZrGfK!=BcEn?HZWQn;lGn+23y%R_g7VDe%P`E45LZZ7l zaM2@^>=lZ1exb z0|OSVfTT6*%MTVFX}#b5@R!8*9h^Vk6q>7qOQbnF|GIm={onL%`DZN^lLaQeQ(L7S zn^(jEcEbz--|i&n!H#f+RAhbK(YubcXs>35C9{u>>qC%%oI z^4#akx_K5|7E6P@gtdPxscH2JIdju1V4suMLY^}T=Bw_Cd}I~7U3)R)>BW<|vjk)| zFL`betiZJB5j%NBh-XwDkh*uYD=kG;Vi!s%J zE6*RB<-Yyj9p0J8wJsRhOZm>Q751+^q8+y}^{}n2pXx0g&V-Yq!iw`&)IBR%^6WzC zx`kn)TefAN zQ~sAB@nr&soytlLt62-)_b5A_-TQpH{GN#QHKu2FJzQX*pU^00|LWqYKY5~@8pfln|%|TALHqS}rlegTrh*Y=oJa=u{skqa8n*P%^H#3JHiT^(EdwFzX-?h(H+wZ8U z#TmuTFi>+B?BCFL+_ZeP(AJKyZoI_)u!PyDg?d5gk3NxX@UUy|ErR;j%ZoMmC! zo-)@$=cKz`XMW0+)?1ou<6JJv7AJU3yxyj9Nra4H?qhsU|`*KHkVyQfu-}sQR^=s zSoNl*YiC`N4eWd_r{9_wF1+^qdaGmdauSnS&xAbOqQ-RBK`pPaYJ+o{#ij|pDdC0l zKF3JBV$x7(*E`8(y5iD5m8*7P?_HjF#dVi1*)*kUYEf6>Km8f$anjCDl*D&kaMk=a zp;`XP0anEcjXW_Q7$s{OnB@x&T$DY~Xzjh=<(+8`jBE!O*mx3{m=`!Oni(`Ho`XEoVffKud026n?0tT6e?`HQLJEG(zz4y+{xfSO+Ir{6T>6MOg2AZ?v zm27UYA3yqd$>S;P#}1ra$6NkTUS6-zAm`RB{j)_o4>_>8Uu$`ylC_H6UJFU!Lk7~j2N zV7p_Nb^o^u9SvQ;nodOz|d>-8G zc+qOjRwx{G=~HOrFDBj6Df4)~{XTr)2y6CIc_#-J)s`5J<4#=%1?raA-WQe>E*HGE zBuMFCWy%BNqyToIbW7_k&KoZ#8yt+-GEwRKGO>5Yet8o^;tNAIm22s8u-Hv!wiIB# z`ix1TpjqJnqu7Cxp!9U#>m@hL8JG_+>I!gN5#W0;f$u>>%gzn#dKVak8cG8z(uErs zoH}X+KQIap{)M@KPQDMjeCR^*egDN0f>>Fv=#d zByQ(S+|HJ@fo(QJ->eCovmS6gE9YMLfYZ%^G51uRl5L_7c~wi8m2sA-Nep%>mo~>Lb1V*oJ-f3)>{0l> z)62`!64){{>b5`NPJ6&u<;K`>fqOp#mrtAOB}ebt2-n@L9e=~5z8Bgkcm|$uo=}?X zV>F?GbGwB6R0X~l4J?Vik|#s^A}5NjwCKCsXq+Nmzo^jgse{)e;V)B}9VRgAWj60r zpX_H|TH(g9Euwqtjmcr@rF%|vSE#k@%P2j|(aUq8XXA^KYtL)Wskg{Cupjd1ITYFB z6Um|bfPt5x_QH*x8|sXqFX!yMIg2HL(QN|T!|R-nBDnYb`_L9+IW27g+sqAYb2o6# zNvMllkSANfme3nk|*i4cde9|wY5BV=W)iXH%gXdrUzf2y>aH`O%<){ zBwM)>YK0asthbz^W5sxO#v&uD#l}^$FLa~}eqdg5f%oME-XkygZZYuPo6zDpgSRz- znMr^_@B_p4or^--Yfe1xVRm4U3Sf)b$#(N4_tOV__YZJq6|hAIu-#wT_Gl;PegQ7` z2aF;GY^YW|;;sn*=l~J1{FIG%FS~8#PqBe`mb)ZO#Ub z(p?fQ=3U*}GrM^VrU-9f*pjuHd&8n_uNE0bmG1r3b2@YJ1qYfd9P#SIPvI$7Xp^3)t3e;Cy_N zb9(}t@`QPNK5*}zpscGNRy0eqIzmSBN`F-pI}=9|lShTaf%&DA)r=;@oZPhG+$Np6 z$$~ePjZa@{v@WroShY00p)7Mb3!fXa)dQx82TW(hm@PJ}oD{uxf2;In zuZo#5+wb`7ZJDz*xfUDDs&Te#-J->~Eo!!z6~k7`EmwAJ7G+@B=iYNqV97g$CHD-L z9G=PBbb*2Q1B0Ga_gPKG>s)h_I@snd;GFF+?T!HdzXNI zT7TbxE$|^r($WCGK-V(MjLJqixn;}kA8IR#MLRpNC=~nXFicGUV1E7c1inUF=ByP- z3z)wwRhqhjtz`pOUIh2^?VPhFutgRmERf7Ka$wRci21jW+3>?ki-t`K2bdhcGp@a{ z$^U!JxtpyQ+1KphTzb)*(d_ghja{3sO<%oHV)f=%t9M6~?mN9G=oM4zgT<${_#Q0a z^|xLU@Xx#F<^*mH2L|2+3_1%Ig-bAmzTSLJvxU`w>F$ZC+aK`#TfoSnJpZc{uRbt3d_}y zUgavjTeo|aP66YJR}5i4ml&ik4z`?L%XY1CCbp8zt+zhpEx3i>fXG{6X{iusWCVIc! z#BH7nm{S(8Z9l;I?@R=F7N0|tlIPdK$I=gI2v(%)@7NqJ9f0jC#<>4 zu+@1Luj8qsiF0@_UtnQ$VA$nZx+0R1cSFs!=_RZa<|Rna+v>pc@d3Bn1=f<+!VV7{ zJPo9y4{#qmuyM_Ms0e*+^h>+`ycQntV;MhC)D8rtJ?z3BMIC$E%UZ2 ztlvX0JBSh4_CBp2jfJowoJKRK~3kptORc5 zIhR)QaBIyyrM`yKT_EqQ+b)%W=4IZwCTBP4W|v6Ksuma6e6)J%xf`>tKR*({)0@E2 zw#{;i_ut9;d5=0sw}#JPu9(1mS)k?+=dqZ7J$qXxFtU7L3`}4*Wthtxz^H4$ELo7w z9l#`?z$oIt)cK(IVLJEz2i%Xhxkea>#RXU|@!?t9Fn6IrUh5malGiB}x3$6)RX1H~ zf8Q9UAePh@W`4M-fhj1H>!uU0k%gu+8}F}!Ya33f$nYGzz_tAX*JJlf%XoRR3s}}@ zZ(1gud(n+~2hZV)VoYl_7O{5?wqYH_2e4+lkJvI zOr#dOoL(YiaIJD0qh|oet_I!<(sv!Zcqch9F8;+j(SY|71FMAs3!ejXjQ~dl10#>Z zR6zkI#Q>It4BLK3bH8j)Nc_M&$5O`U607$F)^`W4|C>4CezXABCw4aPrtL9~`b9Uj zzT8YKeXw4EElK2UMnl)dwu4OS0_wd7eYC_nn?zR1@jOzy!?pGR=b{VG?!V*`Uw3Nz zhC>Frx$Av%HRBF1h^*eJ&TRNW#8`mIB!Rs(fy1QW_7(L-%mLjIuUo^y_n6;W?cDvu zNNbIz)YXd~_tY;ih$dXS9LH$3fqkh1?@fUvM{n>hS-^bv8q0(P-WL=2CVt>p62R;; z;qB!IOy|;TBm9_J0?J-xao?N4+9Bh=>Wj2z0Ne2b?k6634Rt4&ZEoyv6}{o;-LcoZ z=bl-n>&0`-I&2ma0&_Zqb6#@Kys2euA+nCWC4p_hU#=4Zx3)9<-ZlYJ-qx3`T z3%DZ%n2aqhUu3&&x|Z2NfUV*Ji^;!T4iDH{71-v_V!u3rcd-NW4fAU|rE5cex9*-@ zbGE(q?Dbjc>$V!ss`2OA!eFr4xpKDr2WIhRW`zbOK7+TM3U3V<*e4i#x!J&X(}Bz0 zfk82Wxy^ys=@#$J3A|S>u$m}v+&af}&SDDF2F5IgZLfKsM;C;qd=l}Rz~UW{z4B}6 zyMp`K61x^C)Yl|v1%}CPQ_BbvwBfsK7U*!Y^@TW_$1jOq_bG8k$CU)@nLYH5u*Y~_ zQupE6@9=y3ho#8^Y)`jwE}wHs=Pjf1g`BJ3Zk_+OQSLTNQvlcE3oMESjC%h*Ft;w? zU24E)EWmNAf%nP=b}<2l!!nHao?8#;G0I<9VtJck``qe^ZJSj;EU~j~-I~d8divMB zuWKwma4dbmGf{w}GJx5*fl)MokwI|bK~_npe-28aCr`I>8#DP{X5*V<-g%Sdu7q=M z6fYAG8;^#<0%ir@*;ct(souU~$}tNnK0aE~JxMhtMJ@8t%ZSyJXZ1+@ypk=mK+iMB ztgvW{x3FH|G_y#lrI{i#;`g|$e5{%+t7x$EjG$1M&IHBO+7BIq&d0dLYQF5L?QZ9v zx1-|4t;+83Gxuz&&v#f(J~7SUP)ny^xBDCM^~NbDdK5RSo}TG@glqdLA*bUU;wA+T z{xLK;vy00v-7Rq`Z6j-kl)YTzWj2mxxtx0&Iy)aUG&OT@7g?|P@bEAzJCBUV1OrE> z7Jk#TCo3EmySj^+CYVgPaDYibfzQrp!U2arKR(KTdg8>cC%r9mBFoj4jSHPv8bx(# z78pKuYLzTowsbPbwyq79DqTTa1XUOxvnecSVA`gU+3TCPF?F);whITj6&G|UaW38w z(A3SHU$}g#sY&2r4t|@POXs+BMV;zgvLI{GF^#M_8#!0*_`#&TcFQ3zj&&w2o;r#R z7L#=3J312CB_a|S^=w~k;?~=>?22>G_HUPr`0c(udd%b0pxLt_`H|!2=`PzUe0mHH zb)4ZbYGwTQY?`Ufg+>F;F0u+q z=@c+F3nYDDWPh2q<01PyhZPA7Y-$||ZSt=;Tqf9j`|+rmA+171NKJ@CvE9AsqVk@8 zAJb4q1~~^lwjiY-f2ER5GuedfXRKJ*`j>0P0Y-Ky21TySO`g}JQxY$yi8w6NbZvSh zA+mr$Il$!Pw6Ltm&K{KoD>icLz4LYI)VX4@X1T77M5Dk$Ax|eY>D?Z$G-PLbd1=We zetOic|LQ~F>UAb2Mr#!NWFG0rrBrOv)!sN|lb7!LJK3l9F$fDwoaX6%ex{->Q>Vx9 zSb^{+6Q@ea8NMF>7^K+6V=gdqczJNDh&%6NXp!pl3RG$qD-vMY=A5(OAg4nf$GW!G z{|^rL+b-!y;%|{lbZX#W`|+UN*6U!}&Ad&TEe%r39G6d#$;q&q8)G*ip@CB^!GZB3 zPXj}f@S_84b4s=_ez;vSqjVdmw9^TO&peAdSRC?HKOKuM-J-a6V&<})3QQkW6cQTv zE(9bCbLK8I;N^LLbgrh+rW=o@CFi|#;WgcMVgYmGNkbLRmbngH+)~*k``o1R-z+-b zdB0&LpGMo8Rlhe{d`sj|ORV_Bvi(k1c8b=@D~J0gf4uS9*VIPwjc8&Vr(m-VgP z21kB0oqeFf#_{)2-GL3RV*kD@Xhd(gmrLxKNXM3`C048E+82DYUh?p8j6t$`(r zrR%jB7!D}Q9X4FR$ho3%af0(ihET)yD;+EfvlN&F3-qKJxB^_9p7)kpQTSKT zf!7hGwyAGpdpr^(xy#QqP20MGwf3d*&0kwo*nYe{%2#=SS>oUIO}`E*NgUTWW_XIR zOXtY@`o7K{sl3FQ2|FHfICeL3r~OM{R)4ZcdPzc)oR8wf@(g8uuY@Kc7Dub#ClA>A z4!Ox1JebdLFu>u}!!Rqii96MMm>e!IU~71$*1*N{sCaP$uPIkz<2BB_0%3ztW`has z|6BZ||IImJZo7R=rihZjM33jsS9u6tI+!N4_=O|WjSg9Lp##jB0&yRr3b_*woG#mQ zyQyQ_2F~dpkMLwFFmo6sG#Y<+B;C1Vy3m{gkh`B5d}=PbNuy!E(M+3^2BcU1=YIr3rBR{eeVimVU{@XgVo^5hh>bD82wTN znx(P|c@kL;i$y6k@}&rz*=x!ul%W!0CEfLp!(k7Xpx{h4;VK8tuMUe?181~Zl&o+{ z+4$Vy0f!FTI|0F52PdweD{UMs2F)Hd@8&nM9$=WXSmvpL0t0UalZu*?V7beZtsy)Y zp6^)7n^m*AD=|mc!fPTs4~qlC1J3>mhfi(~XD$?$kv*nsW@vMAkD-X(W>@ty1wmRe z?M&$hj+zxH>bDshRhb`}*f;A?>*@$quG`&Djv7r<@zv{`oFf={(xYu@U=k-!#RlfO zyDn|UD-sR9Hy?GoU)pWF=cF%NnNNN8$4$`(t{NX))5yDsfm77Yk#p$`w+6q5J((|< zlpZW_6G>y}JZZ|r+>yFMK~709-G_1GgAHvttETb(OZRYOsPNF>)e2x#^Kf>!S>eLU za$y&zM#Gk8n;hh%(xda&nw4zl4X9FK+L<2~D1G?`Gpm$C1A{_(shr?RY4gNMz6D9s zIqtA@IXUi*E=Ur-+^Wqkb%B9BVY@hsc9-zJ24y{!#))4i%?sIcVal7i)3sv)*H7&7 z3X%V@z|&yDK~Ap^C$&>Krz{a+*4jBGUQcy~@bL=P@aTtJ;WrNTr7$$>7J7EX-dlgL zu+))fk@7J z6N65;bDev^W^pNuk=LWJ$V|Y2nTcVZ(qrQ-iUDu8dzicB|Lag*AaZis)P+71)mO6% zShF$YuX~#M#9{$M$P4{0t3n~a4L8Nu`CYZ|&EYhia*#urga7q8hOS%zA2;r76~Bpx zPDpcFaXtIW^sP(eijUCNWE%Pix~ z+2NTIL1*Lrf4r-kaqtjFih_yM1qH7D4c;nl7nsy$EZVroZJKeq0tds?Lu@`8tT|kc zi~`(B|JH4bQek0f*u%{6fl<8SfO57-18YyiMV}doyv*i@XSn7I>8|;AkcaQR1EXEi zI>VkM=}8|14w>ECV}D*wYof|~=`~EkZ&af$I&j>dbV-PL+mwdd&qpu)T6Jov;gll% zgav0#w_NNBT)sqX+u@rMIoDVXMP;uXIKZ>UI{2rI3nTkYrbaeJLBaGV4?~+GmKld` zXkUIef$cO~3$sH(hdx)+o2CXu4~9mY3Yqj34ZJ5BH#oQ3tZ3k=VBYzAP8Rp};J^mf zfWG*TOdd1ZcF$I-y3AG{ty|f=NlTekV8Ld_4}JVMm^}?zWM-`BTGSwTgUR?p%emis zd<^!!AiJ9u`7nnOI zwD50WJk`a-6Tr0dHJhUXgVYV?rF zvC*N$i?mNOGI7YTo#3@Ou_32}Mg9O^p)vzw$HBAS%`dx@Gv^#Ly3r)KK>b9pEvE*9 zjRi~AfmWOBLsEuaw>D4wC}pBq!}*qR)mJ4Z)kq<=n~b_TjGB)7G*_-NxT0?;(QI=- z^2fh}+_o1~6BAY^aBK)u5Kj2OlASVH_r-cnMURLNlFAyqJ1+@KWh^+AZ6(dolH<@G zD8VSm&?pn2TdhPLiAa3Kq63gMf_=1J~X4{vQ zPI@nzJ{^+tlG?fchi!$3-J{!1D{f8Fyvf~g#=Y*%@n@Tx&+;_C*gb!37VF_3tpP6_ z+e4UrC3^lPS?n@ZXq4_?lzZST&e3Qy=d4Xhql|^?-K<6dhDJ|CS&sllO@=j^D;lQT zEaF(vVEf@j(1Ft$4JTtxFft}Eozh~B&6tz8m_e<8O<;FN;DNT1h889ZU4{t_>>7<9 zIo7ePXkZgy;90>Oy}GyahO4@@O#|12AO;qO71P)f_%vRuDG`vp+tpP2nIlrCflq-^ z`GKrXg@q6YBYU9|M}YqXfwcw=EHM(TcE%EF6XXuQ33$JV%kj0~+l>-Rj7Ed$p4hs-<-EIB9GvJWt_pI~0D$;$np zKgNSSQ$dt5;9UJ9(-}IfFLhe?2r@Ep*yP+e!e%g)J%LfNVbU9k2Fqfl?2~(>8CpCn z7RSD6%l*+7yXL%slK|rh1~w0dUk4byPM8(`5Gy&+D)pi7F~^(<1rtKnGH`JmVhZ5n z)M)&=X4+$k27?Ngm;DKP)o?0gNKHDl+9`EZsz9nGiGOd}CvZ_*RujzZJ&RV?F z{HFNYQg zBbNYk&Ve?Y1q`AEBI!FCe0f>^Wp=a#B{IlYwAgnvEZf@JjK|j6*jvn1JHa6Eqe1LIbC5<0zel3ugcg|tr=b=&7Z~jqu*eDt7+zW{ z!_a5Bn&Hjg{^O0k$A3EH|4@>caf`j8!BSbQeli1JgzOWJ{?ER1EUo*PJer(uux1tT zC}_-3OJJJwX@b45&vQlnl?Ub}{k`AwknecT#ihTN?CQ9te}$Q?&T?UL!tSliotaFk z3XM7!Zs+|y6TYA|kC*Mx)h0oWCaD#TM?&wM`u%|a1f$&pW*yc>u^$ah9gKVojO+_+ z_%?`1o$9;hdvoPpn~P1Z4y@iHUpSr|P(O5VdHlu(A)`|TMKWJArrBID%>LQ&c*XKJ z6B;!PWJ?^QuUbsf_5ZUB0HX zq{9|TKJ|TZG0!wLB0z6P5Yvl>r)*x`JzLJQEwT}NkZPC0$g`kdei{Fhj)sSGk3Owv zc)VkpbL^r6E7LAU*&Z~UdQfrulZ1u~ry2wq4%}YU;an(_dxF*R2a}jZJ7@96gFfPz!_Tyk{J|Z7pq#Yj7S-iJVY(=9)g2F=1Ws%c@)Jq$${$LOg zcIs@uCBod`I$bE&JQYEhA$+?bQLvEW;oD42A6tT8kM3*-J9`Z7P_= zcQm~D&=FEPfuVvyu7Po$ZJ*hx2I&TuONHz&r`}WdZa>#+Gxcra9-Rh$4Th_UvQIJ^ z*c%u)?6ZnH9X&tzxp*{v=DL0Bvc-vQ6IV-I(=-Wt?>I?_-R`4e!ikba{Q+yV$VvT!iB=eG;k6B?yQ`edkvi3~(ESZ!ot_$vHlnDs8>)g69bJxei zFMDzt7yeAtO6O=xV0@O)z^{?^^{KykS+e!CXG>i~4=c|3ka6mwQ}IKm%g6pD>RzAJ z@k#Dro|)&iuQE$_>U&h_3Cy}wz__roO6tU9n@(-%hNsIO^RV|a z%G{{Rl$ToSoToYc6TiVtZVSe?1^rv}pK%{3TC67aR&4DhrT#;b{oys=!&&;-UU)EQ z=HHfb)R--uDbZ@oQIY)O$AN3foCS=I8FS(d+K#hpGgQi?JYkGpb& zDGM05KbZ4b+-k~?)lY8_@@Nn=XmNSaY7oFAdqGff#Y7o{Gcpt8)IMBrk4h0}XL{er z6d4{U8Y}cL`I2~tNj1-}z*G8c|4L^U&)1K5yQE%NcEYdo6~7*ObF(d~lUl*JVtJ#$ zgUM^(*9lFiQse(^lfw8+tdGM(cAGfgh3#UE4;n6+pXaFfz4QsgA;~V62Ne3DZHnqyxRTQQUCi{s%Mvu$~duw<;)5J9M*^TgWJ5$AO1UY`n6D4wnUDXL#t8-stiHXuwc0(M z;lRP^?l%r(8sC^;fZ2>&a$xyT}%GWTd$RA9xyX#!cCj7KJVTyqr{v=dl(ozLh8>y3m)#bq;+ z`W4yOrcbT(l$;S0_l#9w?$HFP%!L;loO!GkxXtD^I^ociWpPE}cAC|Ov}oR)U%$;R zP^>Uu;&kgt$lwrBkz!gb>M`K}^CeI5yQTag-x-*uMNPk3dc*TVj0ejZ0S`^KnbZ4R z1R5{TbGol_?`MnZVX zMsbS|2UrDC9z0-^RhjL`#?r#Tp|@jB;t??~or^~WpGZAq5i#qS$`YBh^eC%Hph%>O zh{F!m>jI@WTc=O1{`i&YA9tvn*^J~)T{Sj|B%5QZX;Tx+B4tG;d%Db@-RgEIQ}4w} z_2YVPQfD1Vf5*Cfp6vdcOdPuP3`~pVM55Zn(q;s@iM^~cSg}pT?E*ufTcB1W%k;eM zD@(8Q8Z?$OiAo@jn~ijl9`%B;_)^$%>> zd)DmMoQ;heGM)}U~R!~M;k&D*47CNQ!!PG~wU75=2gMQm#H zI%Y|~FB1CV_RCmVU*yVK{+i_di1jufv$&2#Vz^b$4A~f?KToIgbpLdfb*gEbHE-+A zN~x?Rb7P`1S3lg4c1rfP8FQx6h6DdHI9l=o+tg;?DB}Khqk2)RV43@ACN@Er1B~p# zCCeu-b639mlAXuoz?REbo(6E-V(Z+jRsE=7v0Qcbd2?>VJsW%5C6-9vE|_Y5DzT?R zqB-}@jN4Li3nxvza7Z-LB8)vmshjuQA&J=q0-RdUyEvsDi>_O6fX!?|qoYfcxR#}x zSj?JS?um%x3h|fnDVQBWF*0^Ob{b73r&lKV}H;xqPLKO_^aHx7z_D!HxsW?jFr* z7Kz;nEQ>{JdDy+X8z>S+d#TUA~h@XgjbD=|C7 z$ogiQZ|)*q2~USv{WqWW7G3G(FiB`JiFhwo+v2L7z!+q|?UC3vgXV=s4NTTkn0VS2 zxbU%EXn3cwAmc`*il4cu>_nFZlZAgQxn!3lEZD>NPiR*}zt+#=TjS0gmpJmHO`G%3 zguIf+()TxT@b)tD1Oz1TN=&m(lLX&S5?4Bq(8;3hb4cWygu0QNPwRg-4A`s0k;s`d+gXW2}Z5A@E7FlbVma!`0Xho*<+!sxr-Hj3`s zdQ`2?fJw0Cpj4Fwiy1>;<4n6vljq!-@^hNn+*bm*OxM=-m7X_eV6tdnkj!vW@M+{- z`;|XTlJSB0wu`;(8H=ZVJgCJR>&V3Qr%3F~od)(7Wj%k>6gB^e&RTuf>OrIFimgqe zQO>nnJsEf3TC_%bNvp04hfhL^Bd1@+q1M>~t!qpc#<>)o-gn88<;cI&#yMS$Q=Z*C za?;Mt@O#Zh*{ta|PhXJW5xsoI(K*;=qVE+G5z(1hrfhCfcQV;FRqAp&DJ+!?NaURP z;e|)^!A`%NCgD&qSC87;T)rIUoL3lJ*Xnq?e>=Ey5|q_N z#nlXXqi>{qykO9gdhuB38Kq5!6WEOcub7-$H#sY~dRDvKdNaA^gLC~pM~T-suo)XP zNxB?#kGQc&v`NHO?My(c(Tqmn8^+spuRLH>eZeFUV&JMAGNIvxz}etgN1n{iwOYAE zS9itSRej8Au1exh9Qc>C(3HZbs>Il%sclj8|zuSFwY-$4l;CNqPnZXC>w z1!t>v1YIxTH87B8{~_M^bZXqiLsBZP9B+J=1~RaHIU98{G%iv=@ru);)PvG762d8s zr%dC;KAecyAti8hR&1uZKZdX$s#R&ws4`Sq_grw#3d~;(7WG8;1nroyHIYj%Mp4&Nfq+B@-Nl z7#xKP9Qkw(irr{3_;XiK=b%W!0TGi!ybBt*w)H&Uc!sB;VWQ+r-U|o#|2XjdIq=NI zQTzd)+>?WH3{C&!Iha%=oKzf`R3;qK=s9FEMacBUp?QhT=CkH$?>+S|=b+FH z2N9M-5(bW9u8eXnj&dQY3U3@0IGhw6k`)7#ReG2-CphW#JT;l3YMR1q+7oVG@_1fq zv-!K5CU2Z#f}Hht2-K?lJ5-_R;;iCg*5ky!Aep^lIzvuFbIAd%Esr-RoDd3145)Ch zO;R*WJt%b}ko|#pon*IQLzDQw=|l_yS`Gn(|~^yGd31;R8s^FJvGN2ME02BCfuRtM9K;-9X#Y?yUp!Uk65 zvr*1z_XQ3tywn)RV{(^!3Iq3_hw_guU->C4R?E4`HS>3>V3GME1!I_oBcH)Ro*M@^QW`j}D6@q)*orxn3pQ8tkCe@g{PSeZlf+BE zzT}m>S@*~`?t+|e(4U}~%!P&u%y$!380kKH{ZiDV@G@s*e^lv(lwX%tMCO}FGM}E< zpq{f#((9nejMKegd?H^?JX2Y+x-nwS(%`kCg6nz(yAC>Bp6bB4BKysq%XX21y^4B| zlxB-Jq!?AT^2Ho{{ENxpNc4$P295%^zlCh+e;9b598g^3X812FFZaxWLrD$~-vrL- z6ZxjoDCBWan4vM=v{7h6W6801(_|evO1zj%8a!lV&fmJiP$R@_;LEYB-*17xN+cJP z#k~Eg?!Iq%o7aiHu}xg3m3Z9rp*7QGPFpEX4(A{Dnv9Q5uqzVoD9aZ-@`8miMR-nk zz_h}?MTt_!-rtdu{dV`(evc;-Cr{qLb@1}Tk3YT&%hfl^hWMnuNMqE_Gg3sN0Vg6L5>Y8*i#%g$uS#!aEje_wk~z5 zfzRc03nrw!3%6ldvL$PV;ECduu{u&ljZRJHtUAx_Q{cQek1wNge%D%u|F1sEU2hb% z^hr_B6XX5FsGcY8a!`Ch>w!rQLRXmdA7txJY3!sg8)835y_lJDUQR2cc$$rbvWnc&LrdD z>v@1FB)n9&q4cehs%)=8cKfA{mhkU+9#3*+mO2EcHQqgOA>|YAeG#>v6Z{!9{yFO2 zX=3hPe)~t0^#>-=9*1o*e4H~5>VG+;_n=Ad1CxGBlP-g^cF0<`9cws_ywF|acrK@r zudY$3fbnTapU@8d%&LfJ2F-6b^&OY@-JhoIZsDXA)N4j!JWs=(nC8Re|l;K0{$fTw6SyAI=&uUoVa z9MWCE==C$yUuvs%kb?S>Ms}aXnKREnGUUCy;HLUVo$`R=v)1yLlujzU5>h5|Hn4<2 z|IZ<@k_N#qPIqteCaSrf9^0Ha4l153pb z_6ZE)FAmCeIG&I?r2n9aslP#}=YYbTCNlvhr58<_ADT4J?5gT%NNY08O~*mI<;|I3ny1fBkAY3bEvlWrW{&erpNufyeoX`Fuy z<*KV?{}k@H&bU*2U7lY_T1g}S!lUxP8U9-`Dl{-@2pm$J(P;>>;rc8~A6Gd8K$CB4aOlA6*e^(EGjgV|#yBd_uC3x3Bd z%TFB?wJK5cz4St)OeUuE%cc+i3@%C>Wp8p&X*jHqb3lW~#*C+_{OZkR;tb*s4(YvN zGClG1!gVIe32P*lH0sP_GOswK>C>!V(X2nGNk3$-rT~+^Ow&ewCeb%dW=k5`T550P zGbr^m=?NSXQE~kB>feG*uN5pBC#RK5ME^0KByv|cCVuN-;X}7J%1(3GQEx5RW4)8V zZfElom&0Z`<*)Yir0~6w7wtKycx0C*!y#Uie{-KWO#NOhWN=X838Trpvj@2kDF-y? zAD`&hTI^@PXPT@+kn_^H+Vyq@js7X=HQk8{W-GDzM!-0BuAqgU`?gK}@kJjbR_BBw5&n5mH7%i_yy z#O&THhchwP73X9cohbkuXdudCB&8%gLUJykZb&AArl{vke@(kZFUT)g+$XI^s6=$fSI zcR}0neM_fs3;Rr!79;sb&Yj*(2o!K_JkThh^5Jr;OWDJt+heMJ z_D+ba`*~D6*Uqw4I%I+~>r{8&k4MB`R=s51`F7LqC)U~_ANm9W$}CejzlVJd~b$D0gsin|#g*MJ@-0lgeFz z0*Xc=(S0r4>(u9$EEV;;a>I#*M{CEXhC?a=4o$2&uitDiJifHre zj!H))Cf{&m6Z1=Wcpx<}$jm7{b=6Alqm_>qMJ=eCWZk~}pN9X&+bbG6UbU;`7-aKE z=PgL;a{RP|u|;xL*21=BSsN7l0~sbh682#Gs4C)@SGZcVRj|&;OWEXv0yEQ^IX`@K z`ICDL^{1vkcx-(649~CAhyFZZ7SGs`kS^|KP|3i@(|v%^Zj)pxm&>#T3N89YKOEB8 zl@=Um66{e(~Prn)t3SgWGK$l=BEkI84zhZb{(gKTC3 z6L}X1bY?zrWbwV|$X9mdsB}%DNbdwD6`zAFE)9=3qHeV4=6qDK$!pS&RA6M~3TWiM z@XSkQ!U5)OTQ(k#S!SrBJUNkzS@NWg+N7jO3>@zgI@}8wS>gp+6m~c;9FtkbzQ{tAk&n@Bs|^Yg__Hfu(lPcipNb`7L=YiEdQy2-R4Cjea0|(2O#uEjZmAyXqW~xaVc}?>4O({B{yy9Z7lfgof zD+}0Ej074dn;u{)J-}w_=D@hMErDU9jspkRhvf!u7K!hB(JCbt${m;EsO!3+N&i$s z7gJXAjKdDCY8{?uOk5^&g-Wo91~B&X~#QA~!$0K_MoYFs)GH^4J(-iH5k~h%wk#e_tCn}53agd8(EbZggF-(G^=+Q@&q1emOi{; zqdto@&v7Njtf1dVjJ*oaW>-C!x!}T4^)teLs=qxHK2B`vpY?!6Eon7p+>ZXKw`ZFF z<7rfOnem)AK4AefqoLDQUJVE1C#=e=Dmx_~ISKDu!0s8KEPeApo7sxOblc~ZQnmBi zOjrCfsIH=BM+;`b2@*$nKV8swR9GaictVqqk0GyW zhyo*T#^k9E44hdOC9#tj9255}@i5!!X*|tG?sLIi2_^vr=H3%*N=q2{JTe@l+6o+M zr5PT`SeSD<#-_F&J>JL>(ZH-|;jAlhv^#FW4DoagW+{`z?!*+Q4f|FwX}KtN&0VA< zeBxw_M&HXh#lm4HoaQ-clq7a(>NuYCpLb|7Yfx-)SD5TsblUsCBj?8({(%JX9$*CZz-Q7o7{K(lCp==x39oEcQSoy(OWz9rx7m-uV zx9%M?D{}nDSK86YttjCb>~f&R|3Zd(5XW)>o;@Df0h+nS*$M1P_Xw$QqSQ}TPEOUE7n_kI<#)S49 z$}}jjnPo6?Wi4p>C%0q)d#?to9E-D!neH*Zr4p_x8yv+|6j@w64+!KWJ3O9v=-f)P z#f&Non2Z)MeQJfj*m!_qlgtFr=H@>Hbqg3ho2rX%C$Wd3UHL3mzZVM&!wgzvh1PA zxrQc>^DPOgVOgqkJzlq_aLmhC!ETqt?xJ*t?}=j#hf|UN<10GbFY$2M_-(nzGxt>j zf1Vn9mjh?bNsbrCIldHdb{*pUw%|!xpEyUOfC;0>wujOy6vg)~d|sl!vSGPq-UBw5 z2PIJsr49@%0S>GI4O}7ra``)sS4$mcOluHYrntn-QT)gPMm+~6nGX!sZpi|Rz2%zU zNFBc7puj&%f&bVFku8n_5BOMYE--T+U~?+q`Q!3>3GX^9hII}Kf_07J4GROMSS7*~ zMPd$0qy^}AJQQnVluLRe=AtN?vqs|3Mu{_qq5+Dc4;nvPt(SFtFS0C==U?v08954m z=Q)pY&gIh!7djViz!cH2^x=U3p2Gh5Jd-*YvN}puZ-2J<)PrS;^Axz=B=CDVa-LXF zzb%3Llmqvv1w3aIc+WiG+2+7`Y=MYMBF`5FfpZBw=N|9~F$(-(5a3c2jCpV|>w%D~ z14~qbP&xx!Km%(O1B(d*Yrq4xe^QB34311n4U7yvCQ1%W3XLLMjpD}|MC2Ya2{ric z=v$?re9+=Bqufeu$pa}84XleMi7ad6Jfa}IEb*<<2Bs%@EFO$9KOWqaad`hODBWto zEuMu(8WxJxDN1D@6pe9|=yQ~?XcX>PC~@Wh&!m^1&J;>?DT+2F8r2s_%uAGA?#5~T zUU=Swncs9DoO8Bi&^vWYnJXy&Yt)Opnxo<0jlHe<=h+&U|kV+$eN;1ESqd>-jl!!J4od!0gh0;G1m?Mr=1sJe6 zH85K&U_R7vgyplo;CayoN4;q+dL53RRMrbOJ!C(6QRcj$T$iG8@I#?n7eC)C?9n)3 zW75r4-!kW1=2w|RtqT-5w3y~KF+?;eL{ucOv$?3`o##}!YGkC+3s~YuBhu`=i3_z`(4*z%r#lWS1jMju_#Yqb61};2rSUAlNdNegkSWQdFd+4#{0f$$Sl-okiB8AS1!vY@` zaGp|f?QjZg-V|>9fLZ1NbHDdu=G4qi`2_y^%IS)h2?IyHvL)Ka_4Z%g=3rx8000B7Od&{dgrA4@-4~^hrC>q%vAl( zrj(cUb2#6~**+=5uE2H6tcI{v9#ID#2(Tq`pHYxVm?BY^C~c`Ik&~G6c)@A~MNOWA zLMn|SJPSE98vdxMvlJ*)XC6q^JWv&Npw@_id5(hnpL~I32meWbVwlin%Hq)Ad*^vX zD|@QGg5m)&ejNs_Nn-3f#D)GGVBO&=v`mqQCsA61k%Om^yTL(-hml=HQToFIffbBA zb_^UIisC|!Lfan7MtoYc<>1AX$9i|YT)ek1doZ-Je&|b=z8gkl0<7gXpU(96G0^ zd=dp@uJFrf@n>CPjbY&Uk`Vdr0AG;KvylJPj#>0!xa>R=grSBy2pKuUArzPmJC1d5=!(0t)Zc~`oF%+y<;Es~u zzV=}D^3b--MD}un4(o+6FHh%#5ZNQwRSxWv>FPOZ5rNlKrd z&2LxW_?JBI*Rpv6WoOgM3yZ?f9*VHYUfR>>(0%WK*}(@qZi?Jyihiw%a(@)%b}SU@ zV3AmoqP-msP#L520S57Vj1~owBL5m?7!qYB{Sw&OAhR#AdH+En z4@g*k$F`j4hrk~KKg5brcjwv z7Pxpm;9{EI8lEV+?`CW44N)t`=#xLA4H<>iK7TUEmo;2JcmBh#W>GX48ch_P=IwHzz1FZSv|o`mHCm>}r5wZu#7baJvjwj#E20&DuHVIdso+ z;5o-2-s8agDZ`RgpZCfF9y3M>r-K|P4v70Y@*jIJS#2igrtj9Xm5Tki*xepz3Vi*i z>7c+aH#11Hl>dmbc*1g-M~X7X9?I@p$Xv1@FTP6p%K@QniMz9It}A5ZUU5+1!YzRf z4njv90vHuJH#BgU9XXiUpmEQv;ODhN@^9IH91vjm%EPcQ?i`fuP?Vc+P&6)4ETOSYpHVo(QS^bM?9&;GA0Cu%`_mq% z*zve>Hk%{Y^T1rYT-yuITwfwfjz?^H$DHulEn#O*{wq&2{v&0j(Php2c`fU=KXq2j zc*@>!TVV&o&O%4dD-XnS4vLyNa&K|qHBl5c+9lR=fHUdX9;XGYbq-u&q3jM0foq>@ z_J{_i{!<7%{jI9#8M~LBc*#QDV#f8V3#Iq)i?dB&;WFjXVFca%cPOE2QiI022JRlE zg6Ka&(Kf9Q!(Ch*T;z3G@?!Jd83N^V|71L1?`T?5z$n7eD1Ps9KI2P0#vT{Xh7F!y zj?SLi8o98bu!b$Yz%#9}?O>>AU4VPP%*ms&ay2_PH%rWWdQ;?q=&83K@3S5HGRGqC zOK6uW)3&e6+L)F^q!_mcDV$CFZmxKU_ohHkD>2D@6DJ8ft&vx1P^QgGB*f~3~35=3`jFKBJN+dLjRm^E?T`2KH z;`6(E^2h$RxBt0gt$)`unrm)?2fI_txy7Abi$ zv(~{>3~L9 zEz2PVcBclFU@e}Wrhojim=s+UGz=t{W+?o=p1~5t#c?H=UF0j1i0MmDE)l1PJSCap zm%5m|KIkzvIdkyIt}A%pkQBP6l{@yRQyP!tk>q0;7bm%{7G9Hlyz6N5^YiihCMbSB z=>7GL&>wQ~HGMS#}Oxfgn%_&icg+o};r|wGXq-ou94wpaa zTolofSaYzsbuZVkmmQOAmU?}hc+jzpTS!kU`r1cU55BWg9ZyX7sAwV}F2p5J5OCm- zgl9*|g$0aG?0h$@N;-BQ55w}OpxmMHs39a^&6a$~f_ zsa{96{H3o7+W1mBCb~#8bO<_CDz-Q{v!|#CaVhTjaj9EK@xXD$+AlZVogBG7exG3Y zQAfJpUi4(sMC}a{#b-7tzTKSA^z+H*^UW(=ZV378s!0moY5jExOMoKRR~KOwsi&@8 z9veAA_Xhn85ao}^TpG&dQQI0488;<0Y@+X<4@dZOO(mc57yV7R&R_a!Yr9Enz(yC1 z?RO4ws&Bh-sZe8QpCPNtWrm^^+$JZ!rSW-I1h&h@?O55&ZMoxt_46Q`4QR_}}0N7c--q!qT21<||6olUya;Wj3-%HGDAg z5ScJViG$au=YdnMPIY3tZ0Vgj`%+t_UL8Fz6fvQZufs#AiF?9SF~-IV3Owx=hb6dn zw7SfB7{+blvWnp#dq`&^OR_+-V8~HfJ&omNN(zkSB@_NN3tm}f{qi>ho5KPPsh$Ts zTRm7FR5`XWl^kGVT6o@puW7;`1#Pk8JFeL|3GpX8c#AbZJfS`7Y5&=`$1=u02ig zuZR$S_h^rbhmUd0gTop{hRkXmi2{2r_zKPoYcW(v60h<&q~O)aA${QhtL}_O?l&#W z3Mz~oWvhJ7UNE=k^H5+HmT)-#z?s=u!KrnDW&^WR^C5#5j_ffC%$!pKv_AJZOjmxv zQ10X~SKy5iv(Ic%jvmdXPYX7%N^m%G$9!m%`q98DK7o-tr=j_u;Dm$X3n~;PImDZm z%06M1wKyTl^+_OXLz3cVheviBR4o;qyrVX22nhXdT$}Uru;ck1?l0z@m=yemsZ`2H z-2dECSG~N6-EN(Sq-A~`UB2RlYtTzZ{;UAy`%4p8oh0T7+h>?Okt5|VC8aPb`e>`tPeHP3*R5(h!+fu;{iO zE0@75#?W<}^y{V`t#I?=mU&#U;fa#}(z3=NK9y@NE4OIp?Z{~2&12}W?_y*rZF1t9 zq0kW{!6dM(!$o*k6thN6BYzD;vt0J!5P@5s?4P-N&-=V&;1+OTsd8u$()!HE-Y}JQ z(n;ZSyJj`I$QtK+?>@kKE31(|Mu91JFN3N4pKBL3D40k(C>jWcG9OgSVza!^C|Ggv zP}~1hS?WO&X- z)+bz*pKy?~QSzZ+4MVNoh69F~0nD-~ih?^faSBrZuZQWE83?h?e_gz!iLF0jr6Cv(mQ(?2g@zLdzkJM0(ob#g>2Hf3=+i7ez$R&Xp+cI2}6*(1^*;h3|h z;fmief3tsbI)!I_Q?imY!{RTDdy6#Cj2dp0^=>T8u;{aLY8-BUo@ zT;h>hIr}6RPZJ@vk37p&6+;Dtmrq~|o?s;}R1v&@W$ovbO{Q#{L-SQORs8(s_U9OL z`VnS>4b1uhP0Y_l*f&>ZG8)c$Zlj+e&iCBLEmb^8fuliyDcOK4ww*1ifbG^{+lByk z`vmsK#vJAwigt*{mmLc7Yh{j^z_M(6BIZ$D>4haC2n43 zBw3K~tWb8FIMcuLi6-Yud~_M)tR^r%aAe$Yu}C2y#BDLlxrNM`rN+0^!xWzsb}nFh z8^~=W5_vGlPemd5M|*)u&d2>69Q-U_`&N0+XVMw9*D6V+ALrFOGLU%G7#z|5(Ak`=W>C0rsQ@ z!P886?UdPUAYtz^}P-(*Fkv z8iH2bCmIYLUDsVzRA_Q-6=#kA5DYdv7%DUNPmIaVCx*5}Ry~>oZ0?ri=c$O=4y}u^xH*L-ei|AD|L^c&ihpDkR zI|Mo>i2ggKzx717=gTIKlT)<6P7%7ytaG%^;sA?T0<)(8Yh?k4X97!90QV7v`9~H^ z-B!Ro`#_bCz_g|d%qQHK?(R@K{=`$^NQi?$E8hc#NsJ7f7Z~`aF`5K$Ok%L#`pidh z0ZY>YnGORsWdn{0JI%Zb98NA|7dK(^xWOvEgysH6@3q^#cdPhhZk2w(D5nvUwEm%E zLqOG@19D=W6Tc>wWlXQnnx4DHVYV0}_Z9>0bq%xCEXzL}n9a$d==Ve@)M07ebhaRY z#wPPdcGLX8W3HQ+1M?RsCGS!SnyCLT;$!NcLvuYjn|)qZ@=s@eELkNqnb~{-v-5)~ zuPZsc3#J}Wn19-dXa5DxwFTT!1~cjyIC~SA|9LQbHh5-VvOg}uRJO^;lA(CsK}N#_ zc7_8CLIsTKF3e2~oW2(%^#YjhnKQ*Yu%6nn%4!4a`G?v{EwTn#CNG@pj1Tnx+!1bK zz-%1AY9N$^8b>T$*=E%MinO_bT`_j|d( z7EbON_{W;mz!ANH_1q8c4+nI%C$OE1;C3}& ztv29loxpP3g-P3arDd?>i;y*2WB~UC51x+~xOXgIi%wvZHrA9V=UB&BZ1V4d{oIS88rGRRW@qbt@LKwW zLE}S$0s}|e2KM@ItYsfqGcJX;Y-;=AA-g`Y`p*rP^%q$z9hl=2%;N=EjT6|L8W?LP zu#2m5*j-?0`oJ!pz)|ynDdq#KLI86`gND@y=HnsE@41;+1(!ySPM>#Z|W zTCgU4 z1M92wsH7cH8YnzVBop1 zV15^4g91m=1(pf{j#INk>;)1X64>P@FqauH>wI8c@Qn516ZUEW=9&O@`3LMZ9~ct? z*pt_suuEXKyujpifywBBGP44U-2;ZH8%-1v80XJnHN%2|5<<`a}K4-hkneCS!Y>(7x6ziBAx2lWpXjc*! zYt9FjotJm$`*cTLT(<8-5c}%0M_w@JH!;sYy}>LY!ZhKC+fwFNmCbGo*bXFc&%VH! zeSve40@v&dT>B=l78-Lc7GO);Dd|wKdHzHDsS`bQQY9JF7}=tV`N|lT1paYMT)-mM zFf(=nvrd9>3J2TtC(Ko%%!|XA1url-ZMbNlz|8YOlF=bi=m3L`0E1@lC5^Kyg#;LQ zFEB7GTxPkjfU!(Uv|#~nfW;?vuG<^V=adS_8VJ}|tzFZ3C2|65_JS)>6Ii1za7Hz- zcwFH0FgPgawB2{s!4(fIgHIi-uI9S&V7tttlt`0Cl|Lz`a-z2#73rC7y^*7G^A9Gq z4a{~Ay7%okJ7@RdS>4Qb2}evnOmWVMc#+9$RlsZ{%9{O{F?z$UwF#%^T;PgH;&yjn zYc$|m!oZ=B;OSssUSBbd>zSwB@&x$@OxkxD$|Osx9k~1y7}h`OV|kEubo)uu158Z+ z7#R2`Fa)(S@E%~;q07kG(5kJ=xbz6a3^N8nhdXm$8PBz1(l7Z*I z2ks*mrp^`Mc{hR6qktti_eEGQ=Z6N)PqyWpKX}eH+z(TD8Oji{)PO_g(*0)jice~+ zg%@6ayLRTM*4(6DT@E{$ugL7IoO|6m=i%L)hxtzDc)ZVT?whBy#mINUo>L#iw|`KL zSK>J!z!iCbH7SAXvVpCw-hy=#jccdYUcSEDrBy@k0c)MW<;Wi^D-C%28;Wz@F)S8i zZ8b2Mz`$AefH}ghM)&NUxl#-oxn42%jEgN9SqtLd{9_2oyUfSHdC7petg?@PtppC%0}0;6|7J<8{Lo#@T6K>ra>CTe4V*_9_-6*Z2rGCIJeTvshtpaIuP&P) z^yBWjFouw254Qi)ZDiEju_?5o@PO;8A1v(al#*sCE%n>=@7jYcKhEYa+FA8h&Hnw_ zcGq)Fe|J9eIcKBbX|CXDDR8_=fHm2Gqea0;n%OArfpn|`+u{Y>%N)4O4w%H2u3WM= zYpP?&6qk?r&ETRrIJR2^(w<{KI z$P^BE&k^wcsBJNW!7~$c#;~|MW_KA_7+9BVc>nY2+oSBaMFco*Cvbc|n81{H<>;Y~ z=OH>NAGr4vuxxYSnJK^{caYQLUjnPU!OOOJFSHMS)N@n}Q+m0#pKHFu8B@VSv-BGM z*&f85I^+59jG;cuFEy5&3+wf2UQKYHq86aOj_->@e#C}2^}HtEM>${288U4HnE7r# z^t}*YDZoBIfNS{&PX9HmsSZpbAHK+K|2mO@#o)R9F_vtF157vB1@xjCG7KlQb@MC4 zgk%IhKHe`WvFOR7l7&so>?{UGA}SpxH9U5Hs^KtbILI(XD8MQpQ^>K2T_DbG!Ow<+ zzHCAgHUS024$SP#1y4=|E^KIK6W7&QdLVtaN{_5g0Y{+15q7>v%aRMH8jpl4@6D0@ z`1SNO>!hO(6gM8)<|V3~tDhC z*InGb{oOsYn(I3Xli7HmSk8E$IVJtnilB71MlGWxmZ|dPe;lk@xWgBz=n3>qU}SP} z5|fYMNl{dK*wbWOS90_7V>dYu+1x9du3c@dYzL~BC>6PJ#_;@KV%Pa2x|WFD+)7TENv zL0f2xk|K+8&Bv!4(KQ#3Oi#R1nBJ-UsEb=qd(}S=*V!RHm9Hh!xr&YSwAc(&XF0BD zcw=x(q&sPo#-$KpW3&4ei}}n9iZ7qN^h&0Q!>mfe$@FT)g|qF~OAa4wv#&n=aQpQ( z?sMjkD^@UCN>wE<;&fr_WU^9axai6w;8xJc`G@Oavw)}DMiqfODLb_G^4lF?XckL5 zv9MJj=EOsG4TICn`z=@s*(TY9FtE%HDp6oK#66+tL6faQV^SN}y$Nm|49pW2G|rcp zP|(0t_~~0TxBQO)CQgNl2hAQ{ISR}o{fiDTa_w5cCN6Bfpn-XH(Gx9}b(7UdN0&|T=~7=-!F1N- zVa8?S^LJ{Rc&8`wHTBM{PEbB?^PT5QzzreW5{m~Sz9AO((<)3ZdDL%6p5U;rA!Uj* zgCy&ODEAAE`)#IhC<}Tj1TGhFXmEJJ%o(8(&BT+mVsU5VqKrd*?kt^$+H9BHNS+m> zAlTR$~9?Z5alKIM@SDsFjm@9FIrCH_q0WiJY+L)!-{yJm6f+zIMQ@q^TXIo<*Y?L55)T(h z&De0b-|>+~s+8e5hSn^BuyiJ&bVkO8>Km*|3hUi6*P)Hc*w1{!NDm< zfWyN@g;{GtlfnXF4y`-+JW>Za+^&6P@wmb$v~J*`%1I*GA2ic~b(C1pQJXD0Ek=gFUx|I_gSY#y{+1&!z z)LyRRkP}!e@%WN~iO~9rFS=GGSnSPfq|idcPOVeM7oU;MzCLnXmg;0lAlztWCFT+D_Y zjs*vUUv)ZgvM{X|Fj>ecU(itLbbvjmCXvnTKm*5;5BmczFbG{a(Hs~cvVeo-0IP$* zUD6)HfJqJWR_jQ)${y9kF5#+G7X<3 zyE6BqDbIK(dd~d1D1cot=6qE_fv}8675n64jC{2n&hqEJUy|CPu*`6QBTtq8;!1&th66a=dNZ8*oV zOMy+jMp3Zsfs>GuBB#0plTbs$5ji(S4lNNT>E<0AK_^;u8Cg1_c{mO{@@YEDweg9g z`b|YYlP3zZVzP@ES2?h*b6v+_vw%gF5AXn59e>vYAdv(#aErb`Jp3g!jKs%1GWRGzSwEw8|))#%H%5CggVTJAt?pMxU17>;~?5wUEt?=hcg zHKVY44^)%@wv6;{iqnw4sq zm*pQgt`J?oq+sLNVI*;oLvBF>+oAw22Nosqi6>a~I2=2SRTR1WUNG`9Y~*&H^N^?g z#lKd)6^*_1Z<^R*PqeCdY-G2&vtFc9f>B|`M-HwSmf1IcF?g+MZD4R=P_=k?E!Ovw zh0L!bP90a(l9d@6k5608!s)^A(x!p!?5cx2-sf1P&MaUx*0{+by&+t5?H@+L4h3fA z6Ao;f?wpo337o&_>Rhgz>@=-A4%;NEGiJ-4iWcQ4muk_tTsS#NZAK*wm z;i8ch$nC$xalwRxha-g*6U&1Ra%NsSAYRg#!1+j|#Txfw(S%qMUtd|SXG z&A}u+VTbxZw-ccY_uiXv>{ij7Wc$oX2dlUlEw6oK-geS=HjjeZGb=|ns{|Hf0VUyp zYfMv4GdxwF?I@h4(agR;f{DAup;YoigPr>YZ@~>^%ttRa%#@9aHWPPbHLut#Iw^fm zyK@QG{0qm_-&EW=niI0~V(?92k5_6h*=|+k?Q)h`xB9N@k3_-Vh@YpD8<^i8y0iNc zufAXIxqCB=s`i>aU&$k-6l}nv#HnQABx#n|op8tL<@SKXf?Wo!ZaqmN?o&UAxd|}w z#xOAZcpOpKrjWQYIC;huhGxE^35`-u6a^|Poa9(Ob_4}53rg%rZ_r2)%JJ4yf7IpB zUXtOPdsC&2-@<|S-%;L1mXrsZJjynSO}OyRXv5^YX(w9tto3tU`NEoI{VUZ2*8f-C zx^}ICRo}uzlrQkg;(J^{KaQB_oNKX~>|>~YIpdU_-w8+lUr*`-DwQM7eR5~{zx)1P z&5!NpU$}O=UwVa%A!807;7< zP3Av#++p77ba~yq=#~fE`@Ckfcuv^)c=OH|x3|o8+V^7i7Q-nfk`oxkKkSrjV0?MG zQN^IeAfU-;18brJTT*~fl0mcY16JP!OxtH$1v9j6e8t3*!NAeMASS^maA2a~gJ$^z zM(zcTY7d&MDq1{dG^%&BWM;7ZKh+$3qAh3#qex)`$BGHl3g%=kcF6f8#(rlRZ=*w= z_nJ8loh&yRcr}>)D~xmxxcVD3z2a!pQ)<)=X!ecRRC&9JcflgP%Qm~BW!J8DlwNbN z;jrUvCENC-%~zaluW7R?nMi7GSaEam-qZhPOrBWnWIsvILc`K|v6FU9i{}cKiWYaT z6)bu+>)&T>c|XadR>!$cr$g++PO$`MaRx>yk0u9&W?q)Ic!jo#4)%JDcAXncejKgF zKUf%=TB07X#!O(bD`*VUY7ASb%3i=|{iBh41B2j(W~U#FJSUjh9T*)2c1s2{TFhWe zN@Nl}ye#|kw0S?oj_gt^n0O$6;hbD!&RIJ;<&_$uOqfhp?1>d&%~NPuEzEfHV?!W! zt8asm?+SC7g-W%|jl36(Y9HI2TkUvrmt&TRaocIPE0-H`O<41N)+vj)+f+B*($*}l zVV+{NvLwdcK4EA18WyhrHm?r0)Cp{v6+0hBpZTXhWy||dJL^)MB_a-+OfeBja2B}0 zD9gcO7{O!`VAa;c9v#4}e1XYy1*?-q%f5*$h7X#JKQ!AiFo`+1h$S>~dvJWrm@2k_ zamUI=9tXyVnT#Fl%t4qeKNdIgD1ng zLu>CPXJd~Gtw9o!H)roXJHfVh(`oxh-jg1ko^*PheFTg9ik6HOY}qF+WZwwLT*0RI z=Fsd@n(t?+9KYjU+rn@1=Xk`-GbSpG3OAS?UNjl+XbrpZuO%#^*@#0|_eM+ng_dit z4GaxfYz@q87?|XLG~PVWsHUK*a6;7jM6t4%|beg#K>VXMuH1(`pD34 zMBJ*&xK(qZz2*R;Qpe8slXlMDb?f16_qrDTr-xUb{NpbBfpO{{M&%DJVHUUjUahYZKpu6Z{SuElwRRzYE!l_eMINV9VKY^Pi?hc-Dos zq)iu zCbbKVoO#oNK6FS6Fv~U?p&2lNg_0>>$vh*eC5#!Q%1Z+CELz%onpWC0cbInEeZE z+WxRQeQ4lb!D#b#zT^zXo3|KDB-;D}m`z^H!6|7B;>8tNPAX(&EJ8XGItG^4yxXtJ$ugC3`}9b;hk_t?{1IQjSgu zjjY}I=y7Az(OaIURz54<_f+}rkCn|%8BKx*bfR~#Zi{^yEx;Nqp*OpvEvTbK;{>Cs zM5D9>v-$@GM~P;=7fnBfn;i_oJscXP6PQF^Fzys(>AK9Cd!db+t;NQ`%z>fB@dbxU zos`Evj_dzkNY6G6Kf2X3dqG>_GPa^+Z8;j@*$1R^6k206SQ&L#BkfpNlvzv*_Qd~S z^}oS%^Av;31X+O-3|az>(JE{j&W;mnr>351SZ&pKE;Zv^Hlsz^J=fBE&)@oYr^WU5 z-IUzX=w#3u)WBmO#-sKi&bM@(QycTtu7JR|&8{77U#_x$d)n@~F4KG3t?9F)ZYVJQ z+`3X=>&iJ*CeJs!zn{dfq0yojz@+EF?7M+AFu*E)L#w|-%N|aL*axgDUvs!TU^aAM zQI}xxG+5xV;nm8X-ra2L@Wy5Z&(y{ynHP;#3m9b$X34#Hq$kICUL{#~Lu*)sk@AU6 z4Q}TfuBr3iU`kqH)AB8%W-~*_KCQK@&p(O1B9+7VeC=DSw0qNP@AYn5>-b`}A;#$j#f8?`6Rm+0SUgX(RvNN6TwrvV(Bib>X6_9(j?2w47g${;EY))m zXl_{?%fQn0o5k_RLkoeKZ2wg?S0uJK^h<5HEBS-r$BR2G0gcC;8jpC}h`d}VaNdEb zA||tH0h0j3t&>N60{(?vc5!oMVHAtd7uV3>S7@<0!(zIiiSI-MS5y~&zwPbzQf|ID`oij5M(?p?(HV@644KwOFIoQb*q-L`_tl(|v~po+igyQF)dBVweeY(? z>s2|Ld831IUf$=24JI-M>;0B(Ia<@JVbG#~V*|q<7EOtk@X51O^;lglu*NpzyG&>) zxY6ePfJx{=v$MdZm>X@Cj4cKhEintO*`8<(doAUW!4%7o$$as}{UZ!todu74-nQpg z=Yk8IIS;2T=vc$jz~J0ddgS*;mX3yq<7zq1Tnr3cs<)cXMr}{Hz~sx(YILJvsZ*m! z1*61{#)iOzd#=rDTraqa7_R+0#-Oaf{oLBz$+c6}@;_N0OTOC{DbOJ`L5^7~p-GwN z-Ggf(lWu+buy)C2Te;>G@N89*KjilFy-N3WJ^-uesM!s;ghnu_w%Fvx3|5$opw)h z#=Vz&`76#bmtK#YEBj7;%4FMBdXv46|C^Sz(tbrtrNU>U>*YUKnT~#ER9Mhkzns@Z z{D;@Om#bv=Su9`adRWx1g5{xxYVd_b2Z>A01$)`#n}ZItiQQ`E>UXhAXwtKIBr~H+ z-~p5K0>-V28pTd9-af+cJEK7|Aeo7QQDkC+guxN+h6Zkp=`Z(ma%R?rJo!0mLFb7{ zIxGzgr>q(y>>1f7?AzD8CQqyB%&8W?3r$OQv$BRYuNK$1dy&aHW|NWFP19`+94i*A z*80R_P$XjU>B-l!{QP^{>x&vZ8ktymc(l^qNIum$KSe!go1&OUm3J?nQw8}stY{q3VII~2C`pTY^A6aaDq{z zbD_JjUdRdS#pf2w^BJUU2sn7CRajhT%8CLd#uhGq1&o{gj%kFPJ!>g5YW09M$n(oY;txRe?5*wly*8NucwtDhRpUl1-4#yeJ zE&N@^b1WJkF|oZ0SiiWz>cgQD5p{zad5t<7E>W$2CB27RPCB;>&fn8m|It^ZukhWG zmmHh?gH8j}d%CDxj zJC;4{Y;SaU#3_=+rM0Kw5KpDUCIQjcTc2N$_BoP1(M|ek`~BM5iY*<=&+ar&YTW;B z@`4RKm$n4ASbQ{DxJmrghv3b2CZZ=}O-ceeB&91hFtRECcz9fPaqpskzM46vj?5ei zCLfsvTDN>X%wh7vz=Kcj(ZfTN6fzzh(Wp!bILu=ek#LZ`*oi@vrCoN~O=ixFn+`K3 zlunCGOYl;QOczuUXpmpB-~fwN5QhSjW~UhL;q#Z+?1m1}D0tfJ{($}crqP2gfW*l=jM`4PiYCLJCJk5os;t7O;x ztW}xhQ9oe{kI9cEENdF;KYVPL%{zav)#unP7Z*P3hili|a}#ZK7LIY4)XaK&+Qy?o z?lN{wLK6ZO@o1h4*zB#jc)^KwxrR3vTNNgjmUndBhUAz#yCaEK#?=O5Dv zj#!CF-128kzdLY-#tAU8M5WDKdN|5Spm4fSLZsx3#Gq>qO!Xekjl5T$Y&w z3WJcdQcEkEu0^p<{Bc5ExnKu}%p-+Ky%kURx+0FK+-l@!TzOXY?xVvme<-wBRwSyK zTTW{!STK3-LId`IABU7@E!d}&!^k^ZpjFqYkxhAzp3oGHR1vW z4jfKGoeoD7PB2*8^(>NZw_sMdP{0xMz)9$41-mJm*m8~y3QQUq4$KB08X0367_R@8 zo%T=efkA6!8pE{80}O|zFtAxX;Jh-!k!{Pwb}pVnlI9A@-xMA&aHTBZC^u@B6wBP_ zV=!r=b#h38jKFdshf@Yl-F#dNoMo5ZJZ~`hQv3=wrBy4=GVtjZwN*d3%)IXGal2oF z(H?GxIx->*awStbRdpV=+yA&!RLtSrtmblP_M5jYR+bCz@0-w}A8>%f`+|#hPGNV} zwE%88h9l}xPn^#&D$QvyXps|8>~!jIoYSDdEXl{%X5hjn(lR4mf7?dxtRIV|^*9=t z^#odMIh+J46PQ+gb2!Fkv4BZ~;{bEp70LFAD~)((C?6HL)fB`NA{{)TDZ;=(May=< zd|m+uw!D7^t;P>JCr&V7mMSpg_>#*o@z0aMIq$?*9^%nuzPFO^(kF4*MU~6zgPI+= zI!^7EJfbL+7d$2({bCls4_ylh9ap(>uQGv`~%51fPxv3pPJm z5NYKxjq~-gHOXcVxmzzdb4*EK%f81XVY+#lZeIY4ONR1>%L+#oz9_UAFPU7jq@hLb z%Yk;s9!7!22`uss4IQ}+NdoB#Ec#^|d7X2br7RT~*rbfRiUSTyHfKyU<8o|Zh@jzZL04mZiO$xQFKZUCT)sH}iqFk^B0v1I@_%$0^Hey5J53Gy zB|6*v*M^ShbIuO2XBQ~$yLCG6X5tLv$YsqXK~CFtT|IdGP~$EWa}J>#N8$Y(ZPt$j z`73gqc(<-#6noOOFE(h^a}(sWZ*bsQJmXQCW1*{_L=(46 zfdg}d!xHluOhT0!M->zrd&NQ)Fgj?o3P=TP(%Grzlj73AlVY&kAoUm{Q-*^G3nL4o z!9mHw6%NT|;*v&Ux_mc2993Tx(B{aq=$!I}Y3e(@FLO*eP&;weN{!ZOOiLoWSndC2 zRxedv_36O&z5Xj&jNKM){kK-uVAbYZC)~3-ui9%ovS&Q@Krj3gho{Y<14&m8%edUQ zHqTo+K{u%BfRJ~nl|#snpCQ3WHuB_Esb z2JxH`+8ngq>%vSO+Xvo!5)6{U7n;5HC^80g9bnLzscJSqfotgx7E=Mk*5Dc^@m&uh z+4L^ldFQ3D*OXU(NyD*>4;iPHcO8>g_tsn$C^&V6Z2(7TT<1$4|E-O`avy}sa%8zX zILk5VZCx2SH;Y-lW5$`BY`;%olOz5aZ*D1n%p@mN*sR;~NbFEUyLC-sn~}(Fu~`Z& zS7St2RaIIJ|RbW;+ad?~N0mgo7&V??IoHQ8{83i9C zN=?}Ci8V%n*X}^s&fU6l%uE;V+S@q!O@1kRSoMCB`<+Iwl%9sEoi)3;Y!cbbeza&V zb7yWZv6oB*lhBTgXx)zlS+dF-YyK zn1iG87KWFG42K_HWC&?vRyn|Gb4WPhz@EKq6GItZx=M=NaA0FOB;nJj`J~aWEwqcZh zFh?hHDs$?6`9JKoJhF``0eUXYKbA)QjC5)$aQw3~RK>5uXy^UtLPl{9CJu>4^@2mE zcksk9Hmj>hnwMARsBl>cG4rkk92HyM+UTi=+fs za=9P_V*~?l&jE&j28MtGilPqp1RWAy%;k6^DLkXeIHTEk#vzTKCcOY=%?FHZCDZ53 z;aS4s>sThV_@)2Hl@FIF$k!abBXccLWQUMTz+a13>NN(w=zgEvN8Vf<~YN^E8q}P6wOz2fTN)C zmC8Z=PY3ne4kj`+N%Sx%DL8R3H0d!ku`Rf#z|o``;cS+{+~w#bGG)PJ0e=<^w zeiOr;);;{VbJ-`?Tc?*g@R~4b@>~#puu%GmqRWPZuFvM2>+~&%yXwKCwDRSoTN$ig zZTBzsPQ7$6>h76Yn*%7B@Rg_NGr}@(r7qjHi20|q=DsMM9{$#4y*zW92Xim z0vse5oWxrgmDVU}esC1`I3(BMs3+khl)x(4!=%^HsF%~s_k_vp1EXF~b8FvWMTtY! z0!=D=nDr+dwEn`RP}8h;=7n|6LA^iCnjH)|GupYH98~vVw(wz=7mUnK2%0ltVd9SS ziyYgf{)w;%+~*8&n7U0Z=UiIuv_sAoqF-ZKTn;cwN6bI>SF!#NW4(@(=NaZ(0(ZS; zq$q_sd3A~Y>YQN2d34Ru_&-Mva6Vn>d*h(olcv`yjFK5nXE>YztK%4w*|ZDdrs3kT_`m!Aa-=)B3lEG)fLAG&Bj_Il#-3IU($!2FC&GZG_4rrT%6<@4ovDeU@}8A>qXS;h?05gLnxeX9=UE!vl#I z4SXGq!WxZoB?mcJn&f&K%&RG#z6W_#6620^?r927ypo;x zC&X{5!~TCEtTP(^=tOZcn~o7K5x8S4(qub_VhZW;^Cy%o=M^@lPqlyEQ@P-qLbC@(L zoaR29DZ=5Te8f>X<*-S|=_9?~A`cv8OAbo(Fy3ur6n%PAqsNi$gQK1blW6WWMHME= ze=Urh8tk**p5xqdfVHK8&x1)kKuvtY0m&B(oGl0SUokKjJ4#&P5P#tyq2kD^(kSb& zK;lNjqgM^QCXBn-^mqdr`R+9E1u*j7InZ?P08d2&OP2#%4`a1OGV2B9UqQ|aDGu2M z3*O0`aVW|=$aD6&*2m|MR__<%IWVu`)O@5vCr;CbBua`Y_jpH~rNypQvG=@!sU4QmNeP`B~v;!PJ z-il;2u<<+0fY_0cP*89n@2G=mk z`yA%I;UE(7Ohch5A|ml8*CDxvRx_2ETW3Dn;j<~&iAhV0NlB*3PU%oj3j>?S0*MKX ztR)SsCU^Qw5=(DvVl@fp@Hn6oEi}=Up}cguZuBJI+eLEto_zS)!z<8WN` zY&ioPtHwXZl2&Hx%A>p)4y#-aPOxB5dwPy}M&c5c1Adw@=Ea{cloTK6(mNm&zreR} zfh#-Dl>^5jniXt9^O9Iee-#7W!oJfobzhC$(s41s6vVlY`ntjzT+3va{7seWcQTQJS$8%juZ$> z=#_I9@rf`9XEjK89OCeBI4;=9-Q=Kq*{nRaVe*fe<*ZM{MYk9IFDVx+e^((>&iPgM z^0xAat}mF{I10841~4dv9_PHnz}1lW?tGZ8lEb*BB`|x+;w9`Sy|`WK51?i-=^xZ<7aPZlkIa^WfS@5 zh4*Jn;Q7KLu;Ie(sZM4+O#kN1Jfx}dScIo(X)j}f?_muSr`6iY3M@?$OUi^_+^~}} zQgU(H*`mPO!JzxffxYH{1=qaeca0`ppTf)|*{JuuXu8^hKu)zp|E;h69PSyh-}o}& z>y~NCh6xD`bxF-~A`Gl74pUVe1d}@F*l@CZaZq02AYWlf?ds`dW9czwUl4$l_bi7gN&q0+L%{EUq+fBc<#8^rG%1fCw z^2WZ;lfP>wU*~Oqypem(g?jcY;uo%~>l_sFX%x+A=$v{;#Kei;_prK*lbi^Xrj3){ z6ehVIhB9+Ty#QzP7fx~uxDZoovNk_Govztzwc1!XjZgPcB#DM^7p~JCDlgX9OQhOxiXy1 zcn--m9G1)Il6+)2FM1{CoxTHyx2mnwt69mwGv`zl7n88cpWu&*=e{m$l;2SwXk&fj zu&qti%Dl!gm}hdPDS{Zu?0C#AFrs|O}@FlF9aEazXf z=jFk+vy;so?|pg0$)0h+uHfC5&d<;L7kNIradY?d_44+0zgL89&EnJ!vx|6p<0BIb zJD;RQ(gOvd$2t=?WSm^3+kKcda%X<*j}@EQC$Vz7PD^=Ia(dd@sI0!81wp4~Zp*Bm z;&3TcOF$*F+M$Jk@le1bm$qF`I;XI6=;iF-Xf)cID=w+mSQk-nU;$$r2dBPEM?<44 zH#@J8*$RP!2lE~X)L1JB{MqsG!RhAP^X@jWa^2pVetX+LfrSmPxh1t+5{*uAGIBGk zD(p~bY;@%0l?zFDq|+yo8)6a?!pJ$Pi(B;Dd=5@Um43HhE1HCxPB5>umx|mV=<(@5 z_AIvxJ9Xn2R zF&^J}PV4m=>7!hyk8MdQSlFrmNkNrGXxj;wDbce;R5t`nyW!AkGpF#Gu=I)(&ePrZ zF}U=rWk@)-t9`k^Ji(T&(M3Se?I4?o6Psd-c-)Z-jokuAviq#=MeJC+?SaaTwT$cy z*$PcgGnErAII*XmYviiap3vH&wf5Z3n@y8NSZxF<(?yuH1P+)r{^MW@V0xq#uyD$c zEfbZPgMAzqP7Dk>@Suh1ukuU#exV2N&C-``RNJV%^`hGBAeP1@i{xgkxUeiE`_V~t zGm9q|R@nWE$dax4)VONCXsAfCwobwp<(~CVq_Wp+DtJ@qE>oOyowfYws_*O?-y~XD zluvP_&PeQ#jF{r;AAD zbbKU`G9mMb+)@#bv`rTdW#lFuR<|(Y(dx7N6|qEU*_(~eZ?8PE)~kQrn+b({IvoM4 zV>dq9YuHxxT&w%8w0*f^yZpWel}=rgov9Bh{&c1b>zYk0YEkpLxLYL4<7BHyKd<4E z#~w_M!2&FwCoF2x-TuOf)9BKIrf%ty4yJy!D2*)_7AgMS_xr9wuhpyf(u7L*t%Fmb*L)vxS)#~y3IBH6QmGhRShF7k`3 z+NKGeQA-@zD-X9Bmwc265?SQqdBIVftC2%?#~N1g07fpOhDMnlMNYYZLlQ?WwCZap zGFN>r>WY5~#MyHMidI|0>HO}Hn9Z?K< zVO!a47_#|29N5wpHL<8Tu&(oIr0>YS+gRAwEEA$)ZmG|~$nL<<9qlA8esSaJju%tJcdy&f ztbFgEhMr}|p}&0#IuD36OUw>%Ry%W{GkV2Aj)ntlrVTr-Q+pZMl_VU6+LklRUT9z| zeX~UN^MMwg4hI(RW*3Ew2Q3a9O#HhY=l|iE>M^(WeZV7Yzn=x$R&qr!Fw1wmWVb75 zO}%i>;eY1M-j|&TNDBrSOmCOG!BG@DlqCkCo(~UMO8?!j2CJz}42At-jXXV>EN%h}B1(arnK8^l`8ybx zB^VevZ5FK5s!U}w`mmaN5yO572S?r+2be{#Fz`9PVC3=HFe!4{BjFZ_V-a@?IYfJs zq)zcHH2yEf!nnL(o&N<--!-Ya3pCCNRo+$GQKaEl>=wqRGU4Ea039)vT@vn{sX0CE zBD;#D%r;qOFv&4IY&0rZBs%B98?`7!o+K7$-jq{^O-dA5c?=c`G~Q^|TQ-q9_DPcX z?gOlPha7qJu6$$vS6y+KXWoQ%cLPV}@3Rl)$qRPKB`5@$IcMhH3%@j{v%lukZ25Yf z4~<{%Zds%BdFXKxWFDA}~jSkx%vhbXFz+^h1(Q%a+BgdReml%C+ug}SJQv0J& z>Ys2(c!9zzHI;>&J~J3YOdFc~9g6%X^}Bd!*1j<_7Dse>n2iU1*m2bBE1z#zCGKgC>C<2L7BN#*pbxTQnJxTHeN)D+xU8 z$qBg8B5I{*lKZJ;PR5K@xqlPnCl&IRop^lX{-d<`)lAb)97;WU!|h|&=D88OF3;Uq zIDcDjs?1F(hlhN*pAHDlxYCs%V<&O=2I%nEj-Z}Ge%lp?XD(LYJngI`IL*IBB1-xq z7so74r5zfgEQ}n#;!K!JPaGC5D)4;flw0Fge4um<^PKw*Mc*#}Ww_*EaQSPa!;>3- zgIU%&$NaNMWKmw=z$$Q|fzicxDvRSkRW0U-T8=#j`NcaJxpXu?i3MyjQe4m|F(PYW0w6PkUOo^a#U>~$S*DrI@`q9YEqP;1= zhw-)F`zLWON0zYVFtB`L5M*1(z2X7eIR*X`M>szzuxBOke^TJfYOw#dIBlBq z7!dW#$tOLK_s>JyaPM@c(+C9jbR$dChzF zcME=BJM%j#;cOZMYgz;6w+H+wntW0Zd4)7stNtbMhd2l*tx4XL$X522^Ui@5y))OB zX})P_;@z-PXdR>Mjfe7b&ow9bGEC;3BNXuN8Q<-9ej+dRMBEKUyaU;s9N01(INmG} z+>^}1_E6Zyk=H~~#Oa~%vn5-2Jo0q{9(-dE_|$NmK~eCXdNK>6n-4Zm(Fr-Lj8h%1}O%I2di`W6dE=in{+GvcZ37$BLUVR2DYjPDnc5J-#&2qFzshy zWd4>QrKTpd=Al5j?rXD6P6bZ?pLP8gOB7nLQ1$}jTLpFDe^=MZFL)HaM}hrR=DSUA z9|sj44cVyXyOHs6AFIp8xCICJKLvg}F9ElO;uVU590!G~9K}B=aDI8fRSO)0W>;30g~(Bb6Bl2|tzu+SSa~D4{+jd32ifv}l`j@8 zX4|E2-jL6hz`*t|K=E?hzIzEuFF5v}X;V^MD5$|GUB&RaE#dmR^-dW~QF%=TYnJl= zSs=9Npwz8{LWY5F>w@1dV9=8Z5Ow6col_)|Z)Ev0PyMz#+rLki$q)Xqzf!2oIgq_V zQJm=^PeJ1=21UL(3uml5BEoQx&20hqr1i`$3z%0p^tw1Or#Udo<+9CU@F`LBJI5eW z#VGRa0o$a6vu_mmPizdxa%Ng)@c4((W21LgY5W#aS7eOOEnAYkD}jqShwI0!ylZObe z=*pFw*!C$guzh;4B2Iygqn9;d0qdNvtXn>?YbZ+VXfe)MAjrZesmQ3TtR%#;u)MF} zdf5Tft$Kq0mnnyy;0?aAfh!$jdqKvB*)VrrGe|11E<3RrZz2hj+bHS`Yf}UA2&3AdtvmwW4+A#N#DLK zTK28y+lmi6?iD%pxB38c-4|>42Z9VI@^O4aCQJimC zBOb6;9N?F0WEXmHQbCb#nF4=GpwqVoDYa#qpA{4<9Xu-=INqF(ykr`6?W+1c1wOHZ zf~&R&eLbbQ=}YvBsd1kl%DppXOM75>SEN`xte9Oa;j~D?XVH@Xp+14jH@Q4uQ{eI5 z*3#nVDDJaRaN80Qk4Axlhq5~ouk9-m>1p7uc&MG-D84OGTFH@}i!r~eKtSrq))EJX zDQ-dv5C5^3m3`w_wB*?*tFnAOQ@yat;pG*swD2ap1w91Q8!aznX<&4vHdK3p?{K zcCtJ3UODi|O+oMj^EI}G*&mz*G8&lD7=-p6;9Ky3X+b$dlk%M>*f zG0AM3vfgTCMXi4(0VkF-aW&Oconk6s5GZQoH(C&W^Yu@i28IKrY@e2Ivuko$yo}W^ zA#(5foWJ3#Ek3indd*VRkgpLIcA!{lOJj1?a={@Aml&mjbZv^cVg3q795JSd%}dVQMlN6i3>#Os2q z(<^4gTFgi-NzC3cWAChGkFM}%9bj^Mz_I0knB~I+BSuaQ2bLKQ0x3@g>>8z~M2O7! zAvW8hEBB!Ag-G@v4+I`LuufZ0#pky*=fR?!2dTG$g32Dtwx(HCaph$^&WL}Nv1ql7 zVRhZF8v!3~SnhZpl5vd5Nr7{b4l`Rr;2{Oha|t}h9`LkjN+~LC*L!;@>Hz--Q*))8 zIrFd9Co-_sJrLaVkY7uI`QBr%1H%6dTU@0NO&Wo@S7GvX@}T7RzUoXOlZlQqeKhOa%A79Al`XVVxf;fUIUYk z0vD5G^#p^9JWFe&u9!XfRLf%3yV^kUlEp5oE0+XZkMgT#2`yE)#@y@3!0*9WKW{z5 zymNL(KMB`3O22U9R8eh^c);?efj!~@OPmC&kQnQnsjOKG+?y40BaS9MD-+Q9Abdbr zu*_XR=u*oX>4JqX4@nfR%efp|UM%Y;q}urJvw5om+qrFf8+WpNl4<*Cv!>UQt>FOQ zJ_nYV=VDB45^V>WT@MP|BrxYC2)Ho{CP|A(+z~!9Nx)$4!F)lU!wx(OiTqrR;suUE z#~fG~5=C^lm_IlO#2jObXxNdl$81}-S;1AyM?LoYCK~i_qkZi8I6gS=^f++4BuextcKGkjI`6l_U zNZ|Bw6o2r|-?uSoZ6bfqsa?-&?{)uD+cLZ0SAvj3M9b-F@2`&XzZQP@dhpywn^WKC zJdphVKztf2kKDfah~ka6s$&^AjwJB(F!0tfcBdSd*xBH_fW?PNNG_2_h*6=cu>1N8x0OMQy?nDn7PIxY+sb`<;kc1~ zwp`x44V9OhlaHI{{0rIDmM3dl{WECQo8{~0pPug6u_xi+B9?A`)9ktpnUmGd&NE5< zk)r5)jD4|Fe_h0`FUz=9tGDSm7HiMgU9t7t9K*$~QYRhtWZf;&Ci;M>gnwZ(ebzF87{5yE?2?w7} zK%Y(IXLj!Sy4HOGj84rb#Uo-Y3L8sWF7qmQbO<;eO6FG33fN$vaG+H!en!L!1GNA- zLD792LLw@To$UMqE)@)nOlfDf$$PJO_}sHuNPX3GhmRQol5u-aE!|ula6uz|NzxUG zP033AvVt$eQ-VG&a_!FlVkl(&`bP2sub+FK2-*MHq2j`$b4DbQ)B5ccVQ#^gACudh zf3{rqHP;nA7-aNk0=t6u3`H*WwK<#Gq}JWIbV_%2gDI=ZVjd%os1-}5PE*gV6gs6D zEfUbccdXZ;aiU_%0~VFaFRCYa>kTeCuwFMzlT4kkby;L~??pAqv`M`)Z>GyzH2h;s z}4h*FnN-eVX91hJ2lM)=5Rq8?>v9pLPd}x@< zzlY6{Ng!l`R0`jj1a>~rBLc^THRnudRA}os$gRI4U^6?%GM{D-751|N7oA-KOpdrX zf4Pvo{(ou8MYsBZq&`pfbzg*<&wM)U#$!C!A(%xwA|tp>yI_KDfGnG-NRUPA=_x#B zZ#KFMdbb=psrI_;lBdBTfz6%lN;a8Yed&siPWI1Ruux1ZKi9F5L79b#gHKGrp^;6& zqcK^r$wlbIlqeyG=gAS%G}Y$ym*&h&nV)$kGke*>ZIZb;({j>t`~RwOn8xy4Xpk?r z`FDYdn=eIxL(@2;AV9M>Md1Lmx zzV>YD1h49%*~^Y@Gh*p|dcj3bLZINFat*@-2UaEyfmVf9gTOA$Wj#Vqd1tT{IPt#u z{DDQJ>7N3t3P)=HT4iNF84ijKA6PwJFu7koaZLVS0XyHlhkVkB z%^d9k=0e{Z^3;8zJ=7d0F)eh7{L3M;nax~yfefFsYR7@C5eszrZ1S8Zb;oo#T$#du zLxHP~!HJ`10%zcsN5va5Tn#J=S@=^LINwRS$ei1>UQg&yeAY)7U7JESmlq2BO*al7 zVNhgKH(*RMUf{qMb%5oc-~vOw3Wj7oX2~lx4Mw6CXOq@_m$;SiazdL*Z_bSGY7X00 znr!Vnn=JL9NsVO@C$j{DiqymgDVIbxjRg)OB@bDGk0f%PD`?Zy2$T~2@sMA#p&|a> z2ae1DCjrYB{~To*6gy*7lB5$RJeHFZWS5w-O3qEw*x}_(MDqrC6 zV80QvZ0*5GUp$OGuKT>Vy8JpKca!#Ij)VkLlV0CUnFQIJ4d)pZG8lQa92yuo9`H}7 zXyKK*ARc7k$PiS~Z?+M4BWC6xc&wjCM zEm*+RTG7Ha&2n4j9A~Mk6WmPxh3!d7*EoFi+=tciQx%g_U7V$^S-5+~yy%~*;xYH^ zEqB>Af#yX5N1ZG=bDzFhxmM_eL$vs&#HS8hQkLyt;HcZs%zKQ1E9b`{k#q?beV@kO zz&8)mmpQPwmY$kResrf^drt6b4bM90(N~Sch87nj#<(SaGz|z3T+`+&k@qoe8 z_Ml{8L_FI8E8Yn2gOa^7oMb+zHym@}WES;c*bq3O$tGlxV2Ego@g_rN2aCl56AIj{ z);Z>8#xNhfdZMf=xF;#I!usfybA{G7{wemKSan36i^*v7I|UXQk)LzzId%C~K0Q1q z!ejF!=D>ZfC6nu3Jz%fd(9E_Zfh)d8>1>Nb%T>FN?7w%ayRZDrB5uDh(C}9iPX~jG zkOCu@hsz`3+^-f&c?@sFo-ANHnk~GHx0cam`<|y0w)pw)uMR%lu)eyHMLsaFxcJ77 z>6iEzm{blqcJiAfFmPr#F!dd0VDnJ0^73cA;CGc{-Lg-@g%{jp1pYK13waPyy`tep zaDc4nAIFDH8mBdu875^-k&wN%;G5|g!MvT}kEP15Z@aU0VsgoZ-?D67UpR1@ZD15`Io@Ki;>rPI4#uymE*D*! zvf$~NbdytuLO#D#3}xEjX?6Cd$%&&E(#%dIGxEK9z{GxK`%B+^hs*K>dzP3!;6LZF zK|W#@d-|F4|5m3sZTR1itf8ged`@!d?|vrOX^ zI#DiG(Re+f@p%E?<86Em$IUe^CUbsD<~^ifDB@h7!0!AY&zLd)W1EGQOXXzq=C5t} zQ#Xq&_{u0*)GRN+WI2Jo?t%b!IM;lE+>Q$z%UU=rjO>3LO8d9_X>=|lSF!`g$HhTP z4a{B#Sd|hO6CaAR8^&6GV70vvJAtLRtu5oKVXg6T_iN75R_(DSuEB4%O5FV#C;owP zY6~;Vfs&dN3{^)MW&|-ZE@1qzm7~!jiG3mO6$hW`OpHwzI97_VYcX)!?Q(Q+XA_abGhCbp zsHzJvax%zHI>f;Efnn)12KjAvo}2AXg>fBNW>RLveA$4@U;<;^1rE^*9G5SM|0u6t z`Glv|koRQ+pW8`3x0zFx&6r|k*0t;eAIqagg$#bB4h7{IetHuXrcFq7Vqkf9rNZu^ z^VH83{q4?+O)Gzj`Jem2IN>;>_d_PT2KH71raVd3!VhdA225TCY<>=`Aq$uo(#_@@ zuvQwdwt2F8KbYPX$r6&l;`M-~jDe%+18b}U*RBQJHyAi15~9657fvXfsln=RU{=RI^^HN^gPBv@eopb{ zocekL-$sk6TO+2nnm5_2aZa1i6x+%CE==SDTh!#~iW9ESpKV&HeW6FWNzp=anuL*( zkOFg7D2L^O8pR2gt_3WiTNqs*FiI7;Br}^df0)@?>E)L&BiMi~(SR+GiDRyWO?v`I z!vc1N35*H`%svXNO#NW2Qs=d8PeFZMK`Fu4rXj#;H7nd<=b-JWdb>2 z31*Hnz2wA}KA*t%dIH~@gyj!rE@xS^{DtP!;FtW9TJyp<=l=}g>`&lie9jun$?{E- zd0$3~hgy$>(Mn|@Mk9vGzss6ut14MVFrQk&A|}A3oWPQ~fOC;RjK>B>xebg;4y+g1 zI70#)%oi|)DKII0Sfq8qSgL_>TNk4-!{QnPt{n~BRSTG{Hn0aSU@UA<4hdy9R$!R! zILq_88mEI)-lbU%x73bL4rceXUA|DtT6FfNOxbN0dukMTuQV|CH!`V%E=^`+10CY= zkHKxR`>}-*qazQ%@|sI8rGDiWK1ei740;<$*MNyOw{&J}OWtb>m^2y~UkkIJW8iCYU_JMNZSo7A>l^r9IVhCu+E_hnqvY&) z??3RrmW*te#lKCX=}>jvP6y8Rt`%hpY{eTk#XZ=h`y(d~l>r>m zyB5WMVAX1664zRM=sSyq!>kL=?LC2#x|0_#UAdM&XZzV1Qp!TJt**vdnKCFgv}ZnI z@KtA&|HHs8AbWoMnzwc799et-rv0>43mF2i|K3LNp#MJ(s|%DUhQT zu*_wKSJ+g>{tY?*M58xW-`e=nayLuRMlo&v-MbZlh6Ki0jr$KbZWZ4!vthxa&JS9W6Bvu8FmBt*ZuWuM%t3P5Ly1ah zTf?Ul4sag4F?}tc=fsEZ!B!r#g%Z>=E2J6RdE*Tjsx255_S9)uHk|vw{`NM9hNmot z0!!}&S)LGfn*c^O7S8$wOd63)`3IOe3z#@3F!=|Ba~g0oEnwmaGrJrh^ya}4@zX~h z&Y8DWd$-&UzCF8l*Bmx)==vuVQmOD^j?ni9{5u`E`V%-hqzW{p*@^|&;sn^74s3GR zup;i{iVLrqFWg>X@qj7o&jr_oBJPgv)dn1H0c=|Xc&cRh-#y@rT);YO0%u+1vdat{ zOd_g+Q<*K5%i_arqPiI^7qI_Xy+}EMadsxN@rE_O#993Xyh|>riGAKCcUYoqcINF} zYbTc3R&7}3n6g_%=r`fA6qcT>9M3*UR7V zYzp3U<71A(cLm-h3Y@?89Ay+QIQ9BG!@sk894_q9eZ9w`-~yNIEvtn2OYUAs&zbKt zfxWGOB{e`)s+N1!0=BISI1g=Qah<@vd;xd50K0}N@3H`{7zak*MJE&mn8ZJnMtC#( zPvYno;IMkoT*<)h^PsTu!)nulI}Nk9=lwo#ZFybuku1-@2X43uYrCJ?;FBrA$)v!* zBJaSUFoD_l1M`QuhohsICUCIFhjUGtU=kfAt79`sz(AAnUdNS>3>}*frPMM_uFhFn zz<17pReS^cln-a?dim;VAC|wmF8S_ysn>GJ)92pkbjw>GeUr(*Ghx%OHQhB6QmYc! zb>-N~7}!p8A3K|UVea3X^UtnW@OG-*0%opWW-ABBe;Hd?+!eSyPIJ!Pz&1O9dw&7< zwgi^23G9~_a3>qMudL-*e1TOgfMH=4<5A&d-+!0xY36AAU^4T!Lrr3h`3A{ZLLQd8 zCfq$dVSVCukKQXnK3A@##?^;h@lOwNZ=Gmyfc^3YcHIlsED{oi7dn;~vfD2(sl33X zWwTy6Qj=}Lq4eAkd4gSc-lj5!$8ibl31?Un&#=zDyQw?t@#%M)9OKx_ z7W8^waGBHaWbOvepq70h3DFykI1a_`QVL*ESa4$68cn_0{nL2$J-M!ECQoKYu6N6p8gOV$V5pvua5v%bxhcFg(_gKN z_-y9iwYF|b&8m4e|M}%K_`mOYt-0?}^dI&=bM~ChesgB^%~R)C=ja||{rhI_-7kR` z#Dfi(ybIpy6umtdz;h^p&9#8};!f`7gv-Vs_VEOt^ex&y<>DgI3o6rUz9|SW=@!I^ z2k0$j;G7`9k#E49)PBHH^tr36x`L0#jq>Mp)oUyH)YkjRZden%K#)n#Al~C&2G>dx z-qs0+n{F^we&A>g;JB&4tGl3N(x$As4hD7x#`6;y*bA7H*6g^z$e16?Sly7_Y+&Y8 z89wpD*~=Gx@6G33B>ktxocC4&@6r4}#`FJ}uJ2m0W6FKbd9N3|ZkOV4S>IiD;Cz(< zdlkcDFa8S=2JENyy*YPp)11F=VlJ@ODzZv7==ojY+91I5VLzw)0>+3ho$Urp@*nmo zZ(y97{qDol}#P9Yo<4?4DTF$>>lnBp$5KvF@^WCz1SCYC-KbEk|8g~`m5 zRDCB+d9a|dWiqdli^t2LrQVzdmUbSI2N^k>uN6Ji2+{CgnR(gkV8a9kR}l|4Ks zivJv&S%eg94s=aD*vPCZprTP|s(p<+h}AMUP_dCKeX-V)Mit%JCYhICiIui(f3amx z?boo~X1WFsvRk>MH>aJuv-9ZNb6cIKcTZX3(xwyn=<#v?H0!!I9T$z&&&cQ$@0m02 zFzbm-{nf2|zA;5}Ot|i_W5qUE);u{ur5=TgM?6Ec{booB~sA z`rY21WE2mY!XnAlBruV&g_TE_YtaFPr6HOgOExsGxZt4bF-gYx-f@26r(Tn_A~)3} z*iVX+GH9Bru)(WoRzwj4yS36oX9jCG0S0EZ6#?3fOB7}}vNGDu=~}|D!K$Hg4PONV z6N^`c0^{0$UIrVFE_qaNOwz-rP;It_R>g)XIrFrR&Q!=&nWVqJqr`2N%0!o-lUYkv zvKbs&7kFk)Owr0Gg5FFInKz&L@xYni`p1tmd|^?&u!WSJ1;X6P#5apQJ!Gnd1RMomwK zjg6fSN}Ck>0=-t6^gDi9u(0vRljIYjtITd`G;+K!{-DAtUD3eEZjr>mG(%;_ihZ%+ zMgj+Cg#}G$<(DaNU@**+U|bi-kf_nhlGd=QX$#BGWUV<83f~#mD|86177CEz*6!7s z!E`mqu2Xrfg+gb`tbgA$G+Xa~ZCPXJ(a6phA(=iUuuZ6i*Y8I{``pN5GqY5t`Al-x z;bz%<)cuX?4 z>fB-hk0lL}U7?DOPlUf6JMn<&{U-%i0Y{(0Ln0o3I+8Vly)0QK*t}ZMsF;ynRJ!Vi zgU~cbM#(??s>R|uE0z1cFnik1Q901XDjy)w&>Gxn#5LPO=)#9ykE#<3rnu#(++h%w z)M)%5dR5k2=R~lI46Ejp(^mYVMjeZVf|d#H67oOb<+(*`d5HLdaX^wa~vaXOI2)k!)~)<8a@!XBlxG5h{Vr0`u(AF*zWa3-EP}cd>C9#n~_*v-Uc{7z* z{Ugqs{88W#3)?8zzvGee1`bZ$Mej;h1joM8GITSVwo&NhiX(>vrg3|DI0>~Ki1=vM zW;3Bz$1Fy{H#;R+TH(XS{_Dn)0&f`pac2EFBW>@jqQpPbBiKigufrojc80+eM~}p; zQVmu)sRjFO1UMVRs#_HD99rEj9pY4_J=Y}>v>f2);wpmW?3XK-{bxnzs6n{-{d8> z)~u$>8}z?jNi5NwGG)TIfJUAL3VffNS`?N&nD@`k!HKmgVG2h8=RuB+w^PM0o?y09 za%5U~#DUA{f)|rYqfo?A6`mcU?qx@XR_=Miw0+V8*0c{T?gFY@0s#l*{gyHv%G6+H z5MU4v`Y>^oQbSC1V?)TBSL=8?KQqQQGRX9>^v#>f#Okx)$ct|Y?G}F?@>lLSEU%;; zc|0u4c$VN1r57*T-EFP6rc)Xi z_Jr#M8!EYTxG(niaO9eII8W2Vk=MrOGp|`4v&_8@Z5DSH@|ABmEURRF^OV&!{(6f~ z@?slP)bw`>_6e-b&@S6}dd)PSlg7NWlPkv3%;&YKBI3QW-R)>p- zltaTNmji6Udz@vm3fgVHD7bKBFfc_OX<(V~i&4CTkx4*+A#DA>sH=<-r3_gO%&SU_ zXLCq6bgulwz{BI%Bq7iun^CaK{mD|1YaK~yoa$^j3C%M9C7OdKF|m7jGz*m}a>agn zu&A%+hKr8V0nRInn$E9GU{()sc?skI~`3u$@jypc~m9I&*;IQ0O z`-AyF_nH({{hbfI`Q)bVFG)41Q1p#C#weiV$D;qZfy3fJ#WCf+kA~BK_FMdDI>3@{VWpp6#%A{I)8|YUCT0oF zPUb}iUfS9ucIA6`F)=87WiVLqijl#AIZlCvVNDdHycxq9<_X!1R}%hlYzQ-5Cuz{I z^|2JA&W>hgrUFJA8z!q2LO~*h3%O&bENWC{a8VL!?ACNxaKxaa+ax81>!`o7^n?Qk z+&+DG*duDlsP?19^isHp+Lr~kmouEW<0dqTA4p&~oN@4Fr9gA}=LQa&4+rP#T=Us14j>iri`UrGZ?Ke^uEYov0u=|@xqF&!!(GQA*u6U;|6YP zh5!bJgx#yT859LtI1U`xlFYzVz_3l4VfuqcHVFnM35Gv285d-=NN-@2woucQU`_Af z7kt3%q`=}J&@AcEAhx2>U4ixWC%(o5&0i&2-5yL2`qaFqse!qnVS`75$OMk`A1!{T zm?b(|f?l*q9B8l!Cr|QjaQ3pD<|(&OlY>fb6BRC`5yN^hd0gI zj=HA}&q&sLmSp^GF(WreW7M8Trw$g?49ms?3|to&cpUgT8d{_Z8czHx zZuqm4k!MGv{f;Sa4vf)?;tVTx1#PxEKCxjPGiXJ+z?)sGlNppgG;>ri>|V8t{{xfw z1P1+%##Px33@exw3K+ONIII`2yvyi_zth0+n$hAxvx^0@(~p*5krtLsA_5L-S&ic9 zE2hle#OA%QMa*NO*hQzMI~0Ezsyd!%&0%QbwrCW8aiULUuUU?>dClR=vz@QrZoGc_ zu-Svdwk^zNKb&I~j<|c+8T^!Z@pF^gl6?X{8s`0BnDKytlf#RzgMsfs!~B%SB{7WN zOYFoWT*MbJN}Xs_Nnl#Fhf#0=liCKx4?7qH7#LUfG->P*usOl3wSiGxg4yr|lj?kI}q1_t3o)A*gtYnU3?4_N8HV6%U~lvU9pd1h0A!_?g(GA;9dQ&}_`m>iDDS?(D{SyBPP)W{d4$(tKh4YK36u0v44&+QpfI z=d)WJZ%Cz2XyVdP&0=8w{mDa2qfscKvA^VSpUTNkDko(f&VLE8R{XhPyjt%W8R7Jl*fn$jpRfpPhtMvV!xI0YMd zAJ}s(U=Y0Uk5TZ0z2JdnCyN&S1I)oQ*jx&j_K4UxOSI;lXq6UdcH(GR>)3L4Q=`V2 z)@KtqJWepS9cV6PV2?e)7F*Dy{$h)q1*7OnE0z^Lr`PP-{pkSb1fP>7bGB~w5o6eR zTE<#@0*gT>i}QpAh6Ro`J9_mVFv~7zbn0MMy~I|i(ZVaiB`3mY7{Khr&}yKeKWmbd zmPdQZiMDbH)~^9`ygsgT+tDClDb8_%;mFAb&m#E*<<^)D&HNUOm$VzjXI$|V_5ZBn zFVW!a`!vKd#Q#eRLU=+NxfHjhVZ`akB zPzGMEQ}aq1<$o|){F7pKuxN2na7wmqadueilF_u_X7kibtqTL23>#VkC$xy%ZIj9f z^z6LW{G&ndL~ELIt4#wp>naAR3#~e=EY2MLO%~EQuSAPBYBL)!geEf_KN!w>;Cj{G z)zz-MB}*9eE;OoKXjai+U{_#Z%4iZ)XjD;X7Gz*_{K1r#ZRT*J!O);pTA_i*g4rQ} zIfj9SdqEf54d(qOT*rl5q&S2$J-9_DG-;e*U=(mKxp-*FhiNPq7(6mqwh6U(Txiwj z@E2Qf#Ur+{Z_bqoGJBVl9+s^*>}R@n`Q1hl5C6-z8!Kj?e^9yj^amcp8O$Ar8~-k` zi)0O$p%GBHCRF~BBIk|Je_Pjv#&m* z|7?Euf@4pi% z>@`@oUFu{gXpm}X$w@F#oXJ+;z@qX;Ex(CDqJx>Ki`o6r&AK7;6m#DkrMJ44wb8e)%j z#)>9xOJ?BAXjFU9?5M!(vLm|u2Wwv5!`vUOxdH9Q518x>S|aaw3GQfAitrZL5Ww}r zK1PH^oVP`ji^b7^#lfNF^2=b}4kibN7ULH;95+06|Ix^Dg5%K4v_NTho;OoZ9|`CG zw?RlUndgkvnp2F8R~s8;8Pzv5%4aaC3S`O#FxgbDVNqacTfxY4qS1q6I@^jy0|iE# z1I$H%6K9_a6q~}p=+UU~v*ll+yraeHsRk3*RV-CMu{)@vDPNK`R)eK)NywknMiBx3 z2{w!(6^v`X#WR`NuH2gXC4`ZAnlr1P#iNa$|DL~8c4fIe@5_0i2Ml6{(X0;>iXJNR zt4(M0h~CP|D6HBdQr2GdB2D~RTdqQT%z-A22h36&jEyH4c)q^)q|MEhh`k9r!zN_D`*(eJm1EJ)P)QFa+heS8H`MkLzR||ME!aCfD)A-BS)Q{$I@4!pjgI88J!BlIg`pt{(!79ZWyE zrG#et)LrdjS+FforrBb_Y3T^Q^bD5sfsJw+PXr%M71_xkSHSk=6oZBki|7gFGizBE z$2Nw&^H>;~I?1QeS$?nViG0`Psh6u6LtOZ)QIJ}p2G>LVzxCu0@5@@t1E$X=IuIXcfQqQ!C~eT;P^@i1WX`l6A|X zR{8p`WGz_Jx;SL7k=ew=p4;b_d|A)YadlNw=Wa>y;QiW65);`cGzyfm=RIHze83tn z@oQmwY5b1cg$nF39L>__7}<9)d``$|T2Lsh-_5@2UHL^$wml8KsU6Y`tp5T$!VCjC zEY-!M4(S(tOyd8xaNB-u{km$^>u!7!a^4y&)&-3W847xW&3Am7STfd#&3L{kiAm&u zIAcIYp!38JZVYk(S?LVyC(pBKHn3zbXyWl$9<-~?Dq`!;OB{b{H`*lJOy9!nd4e_b zhpkx#vy#Jk(T2Y_mWP=C`QiNh%5Cw=-_PQ<&2F#Hg_!R z+~4S-xc2(5S7Ds&4;Hq1EZlPDtl96F&5uo_5 zxug8mjfculoEz9gnHY_>91jp?s+3#Vq5NW^f|{t8C3C?-#s(H4IR%e~2L~qy1%7h) z^<>RNF2`F{DjHQxZWB28<<2b0-0XCbM`OhiO-aq^TfNT6SVp-l@1JAoQuZcdqw}%u zEk+3&a=Dd{M$6@tZ@D|MNiTA@b=lKXu3kb9T>5z;KOH{KC!rtLz;UvX*_}sRC#B{` zK$B}HhoF~7hr&bSj0=L2JP8JqITdo3yL3kFon`tuU_<3YE@!rYd1?v^ci_u7Bg}|OhW_F=P&gF5AM|ilzwpmMjIMA4W!JAi4C&O-$6RS7Bje|Xd zLkr`-i;Lqc-Y2o}3MlKO{9s;rPB`ezmgDm0e3%`4Ib8}I0tHkqIPgy75Moy-d{X#O z{o0*?5GE#v4-O38cUc@c`R4?1s)VWvxNF)vC~}5fbudmgG4(q%RVk78BAZ}xuPAer z{*|Xldu_Q%Z( zLC*xKtbKDnMP;ws`0bhIvKfbbm3F=<>X0z&_;^BN!jncvw$`wK47QfI3689XRe!lE z?Du4!$EE|rcenIc+ znqXCh3(X4WESNo&m#$zq9{Oy@>vugLFI7BN>iS~XF5vv-MRLE*gs1uZuI~g2;!;Fk zPM_Sso93BVBNMh%Ai^$;c|vT|Gu0VF1!9aWsk5}CW^v64I4rX3KxJ3XvLoN5PThVm z>9k4v14mc6r3_4+3SK^V6``K0wrK7yf58kwJR2X%r888Zk&30t#o7`x-0nlvK;;4cB@fxKY0TiTF6$ zIrb)sEH!XdVVk%nvqp)FO@~$KQA0;24-+$s!U5KR2kq_-32X)@9C$m-R!F-n6p2@G zV6rk3=2Td~$iv~lv|vF`;?swm{Ti;C?>;byYc=FBXP(cv;m|2z(ZPQGfg%gbMGiR@ z2EKDUn%6M}oZYNi*!s3oY4wGh^H@@N=l+|j{lelpbEB5s0Y^21z~fZ{oSAc+qQfLyh@J7mA_z?x$&XxChNoA3KJF|&SD$e8w>eL1r8-`ayfCr zh2QvcbV~Y?&b^uIJeg9m9YrpgbA8fnCP=va zjmunaTk~k@#1n@#moc8R*`UEEvVo6{#gI!hLy%b`p~=R$fZ4;riFKFMB9^Ws-<)TT zQkD%Ys!G@#fZZhSg)w(OM zmy{c!9l3IMgO|wXh{Sz;f;J4o{id-K#C-)?YEIV=7np2Vl zm>O%8(0v$rxFX_Pvc#lj{ez?%KRp_S9}Hq$A| z)oUCX*d+Ea@COAn_OTx1*UUJ_J@0~CGhYj%<-U(M^4ifK4T~v|C>JtH+8Vgud zUr5BL1ag)9IlTQK@4Nih3!j;8uH2ERqr4}^{gUgI*Zj2x|6H09U$NW2IhYu-Xi`#3 zf$Cf<{;G~GJ}k$q)y`MEF_!wnb2H)ti^H6znh=MCLOTJb-+c?XUS2+GrgE`6R$&o$ zkid}^E}K@j8BBs7omsq>O=O=xBT1xi!3NE53an}d3;9ACTsFBeCQaj5#C}risBjar z(gwo?9sFDe1XeVz(MWM%sGh-gLnYy{y1I+)tH@IZi2+mY`F0ESdN6X$6FAIq=SfHC zyqudXpYqiG%T8UKB$K*!PQZrR*G1kxjaUpAI%8gZ+x{is^TC^IKC~*&*^wY~cem53 z9#5Y|0tUP;3g0d~Km3Gocaw~YBCn+FM=$@Xc{gvQpStp*KemC>^f${tg%u7m3+fWs zLQdpsR&4Bbd66XETEN6x=G6L&cduZ>1gC#dYZ>=ET2psI>!=1xAVEnxN|-&IMw=M*7Y}z$=(+`cvd7F?x@yTv*XU&lh=MTsjPG0%Q|p?CFX(P z)Ex~9YYcfyHYm)}o>IQoy;vY$y(G*$Gq*BKW2Z*lYlb+pQ>;pc&+5N?d+=#RyRF0e zXVpFXzVO6v&cAymu;cLSM-tDC8#AIVJ9zh0wu)IE3)>)h#-o9CZeq$xn`ySaR~7hI z&0%7jBkc6CuE9J^;)rI+MQ+azNm6YOnpj;L7zGpDnI%8iEi_obC}QBnEc&62?O)JL zcFieD{7oG#A_@zb+yfU_9ANPeOmosI;cJ+6J;ky6ilgSA1H2K;Tr*@D3=XhpG~9U_ z(YN)u*AGWF2PZR*xIm9a(|~4^1+$FzFtF^1p6WaEy3RaD6BmcxY4(C4PY+HzcH8`?@g293r5zQS>L|)EKX?P`r)(qhYy>9qp}H82XB+ejVRUy4vTMyE?&^U z9OG~*bHa6#Xzo7_A_5bx9b}Zb(s-@5(YNv-`yO_ulG&3@XZKuXiH%sSc#euNn40HG$8yM#{8_CVFz2N-fVFLfF=FqsM zpSI#r9p4+ zb&(nIGd3J3%wQ0$X*!v}tjBOnu7mM=Dg*D2W8WGb7U^&~iFL)sUTpY#uxVw3yh|fj z&a#w62iPt6l(UX8Pv|o3^Qu*WN>hr;y(GRunLPK zTj&946L*n6E;4%@AJ`sjUlH(xaf<)0Q;(;*Pc@Bms`Y+a<9$5jd5~R`aRl>o)yGCm z3yt3?ghi>^zc^gc_dN7pma3hFvcNTGr3zJrFB1>{nOL?|^>f2wo)Zq-GEqu}4rwP; z*-O|e<|LnLJ)UaOsBq<^LP#T9jzeM2akeMn*)yEZ$-{y( z2WowJJMJA2ZirAm!YF*^z%SK!);kVz1uhC4P4W(m9Vsp>CJk1T+?YQwsJI*wpCPs0 z^`NwiqpXRe@D<0cB23>F9Dk(A!er89e87R{j-y-zvyPGD?~hS3bDU)^xU3dnxqEk6 z(IzjKCyP4P9N#W5rPxidGvYa?g~zUl!`3I5pI>|s@@xTLQflQV=K0@NRwk)dJ!)3j z@;p3B{8Pzd{wvR;wjJPp@w{SU%)dph4fT4ng67;wuVD~&xUar|Nh;$~Q=WsM&OwC* zOmja>7bE=)=vrmi(>5?Rr}cIJRhmjg%20oE-I!V*m? zA4F9hnB+T7t@tyq#=?O)#zC2*i8bVay;lnB4F>U)3^oQvRgOcfHV#aB_nb8xSbj8` z+&QT1(`>TDnV01N>k&s~o2HE>hqz@fIv(I!^FYnfLs!m0GFnsQAF~xGSUZ14kthCd~;>!V1^eRTNg=Nt>*AfcuBzh93u|O%8ER2w*zl zDr?Hkw8nF??2#RAaeC`cP3Jqu75na?t9sD3mHd0}Kd@Yrml!hh;e+y5ne(&O%wMNB zzidq?Q=5u}(+ROQ`9FdZ54Z(*82JPkqwl3JHq%m4y~KX!dCHyTAHPje3T;bka9B2r zNvozwV?opBu0zY_F=-ue`||OSgo-2IgY(}y@SO*rZjm=G)|*vF{woUvr~6Sk$!e^MMeeRoei{#f#9kdL5q z=yTR33>Pg9Ti!9So^$xcw3+2mZ;r0gd(HOo*eA_5cASY9^5h+Oq<=J=x|hz=@%a6= zHy@=Yaoe=`7iufpjh z*K?3(!7+t5j0z%6nkFAK8krP0niP8)6!sjp-{WlI(8O2M$-2SeA9qZ{k~#R6 z$HgJ>a`M(_1_=fs%a$j+GJaJ!HdlSf@$;%R$*Q2W6h*@$|69b|g<%SpTu8uy)Z>VJ1d#6-I$S z2ly@66rUatnbZHnN8e%csxQ|%&0ZXmFgPgpr18R>TiQqaKZ-J4Iw@qfr&+by*?4<{!Ht(OM@ZjGj&Ue}~R}~y^3Ou;z<>nk; zXHyZDD?P>;&SnNv^&IVf|Z zQDB2(|D27>URSJqXWa9YmVQqH5NBCaJMkZKVXzs&{W;gaI>`_;P@sdiH7^{UJL#U z)e~}b3E0II@?F?)wr16T;gWM)&YX*j7p&&ZK5UtC*s{me^zusECky7K6-RIXYWL@R z{pXd@+!cB|{9^47DMd8xy>nXT%?ZAqR-QKp5}4mx*C+EOcqUn!N!)0Z=y}@K-6&!4 zExFiHqU4~&6GthAL+T6q({6MA5;&wT)1>*rY1+MhwiOJbKNu|zCd(XH{7*zeX~%Vq z4~;?{j^`Q<$h}|`|Inzqqd~cAi&aq%OU(iK5@st8Ciyvx3JgthEeAHUIjUapov&(A zcC%4lgemE16aSwQ7ls3CBo46r$SvD_%4vZ?(LW<+g@zeHc^pZ3#b)V_wP(44(w#fM zG^D2gi;L)1tp_h>sUKTs8tre@ush8pph@as$KDym&Hqg!?>vw6VH3a3a4N~7 zv%OLFMdRT;&!=tQb=aau?ub>#WJb|D2hRF3u6tg!&i$aolLp}@48j(TB5zi8N3B%| zXyo!>tmZr<$8ku+h4GhGBX3Wmf{7!Wg&M~T27#IeWw+x>mPsrO2bF&?vhy&oi!iX4 zG%)ZuFwJq`uSpSRa1`NS6mek`xxpx+lPJ64_QMdqKzUOJl?99riyU73^y(@X3iNv_ zwbg%K`7^f9dTM|4%XkdR&afU}RH@l&+VIs<$D8lgK;Yy6)YQ@7ts2*F?uV z_qrS|y17;9$-fga8xG2BVeEgx8ntC}V!VZIaie(7kJao=iX|=KheQ+ww0ufg6gm#_ z#)J*@XkV7LE!gj>`ia zm;(;DMlmo)9H{YdPv79c;oC1nMK_uukw$kOIqz ze*q3{eGI}k9Gub{BpjIL*B*3tXLk@ONI2-|$S!CXu|YuTxVT~UE(-yM#J0J5RXmM~ zHy*HXN%Aro2uKD!X=YH6WMMF9Xk_5z7cdAA2ySFzW6zOV(V3vuG{Y`m%i_R;=VxcI zcgeLz20dwH=aI8${q=y6Wwok8%#IJtm$u|wS(ACW%y0JQlILsYTDSMj-BbVN&E4JY z{p;*@|N8Op>GtXR_iO7`yLM00nW=oZOXT80rOgyvB`P_@IPv9k)O2e<*jSH0c|Y2 zivlj&_eJ$_sQayGY-M9}&{Sh!NJwa4krsHMGSw-lQ+-j(q?KwjJG#17EVNL0wTyq_ zCxMj<1v<2}me09#;mC5K4O~q^S{4tPq!kL?W^Y)wQB9}dkd?ah)@w=WbL-z|7OmU$ zZr5u4x_`G`udij1UL)x)5ty^lQX)Ht{Y06u?})dCjBF!}I%`8jj3P_|>_JeNL;E z>k5g2k4-`eHi>K=UO%;FZ`gA3*)i$OC&fLsU5i|PV#gCFqmz54J@MY)xopWMFKy}U z86CR&k1%!^9{iy*!QqO`*)0~8FK39ymfTcLi7WfKW2>#pfdsaPpH3{^9;DQm(l5Q_ z#9@9>qXP?=o)(>p44l7{`AL$)hlEFMw%;{edSxOxG^xy@iN|JUdQ~bNnv&tv5XH05lxqcF z`mGBEvnMdUa?YQ*Xw?xBVGjmv4u(As?p7?n*fy*B@Tu~++IpT%@jH5Kf)e(8`Lx>L zu-G=E({0;JdN|CE=mhiI=^U&$+k01~a{B$1H&uV!@6?wQcPL?KY%{NWo!9TC@KM>~ zKFdqTbRL}>GoQPNKGoZyvFOCZHmOenOIpPm4lwfAf6xfc3M#B%kzNROO%#dWXI?(l3A}Yol(R^Noe+IHnUHm`se?4 zbi6phW@XdLSF*y{~ct*fJ4W4$+zyrR@4ig&rT^LyO3K|`kh%pAbDX`T|Xc3nQUBf?Tt*I^9&Z{sn`X4Jy$x;;I?~9dbVHJ@E}+YOfdV5_MgYSf0#6VSf@pQ`xfhsvIR7YK=oUY?Y0_!_#J(CNQl+89YYKLW-ES7%T zDqU^ZC78m%qA`JQkyZnf%fbUo%y%@N)7rqeHs!!XledwqS__&!F{TS#5W3M4eW7 zmJ4UpCmfo-Sb$lsjCu1k&gFci9fw$#En>*;YMB)fspvB=NQqg3fx)>@(L(&+g(=5+ zPseC~y4qb4+9(q8fN7!zt7)5tOSRHRnO|$vg>1Wz)NXS1_)_8ha)F0vpCB)U5S`o@Nwh)gNBRcufA|=$#*&0z-E##h3%(mw$q9tHN7x*X-X z2c!kIED)G@!hu2WfQ`oX^vD}%zOK)8|%es)~^rNX8 z8vd!?ifTHsM6-9c)3@Cgzsz>rxzRRxu1rkK%OoD91)8a|yMuM@gDkC!RZ??L+0HQ9 zzEffc!=hOex=S@KN+~FWvM4N&+vKSb3Oe7i;Nz+spU2$d3J095I_;%I7BI>O%v5qs zV3x93#4eh_5H4rP8(x|$Wme&$IYXhsnW9fT8j_tm`HZQM$S$emG zGslKCssG$HPX(S@saaI{%){*$&r-8c4&fz?d>aBDGmB3+@I+~W+!wJ1?nyV;^pqOA zgKji3u$(xMeng`$h(ERMdHi>cH;+(JP8+=R~}w4$CUB8pACx=|H9&!&_Y3L{{Tiu zhTFjp4azo8y_=J!Jo|0#e8Vk`EK4#!cgub8c+tPLR@JIT?T*W`6`7}EZ7j@wO22CR z{!lQ!B8MS~MeIirllh9LY!6JB6WZSG-#*>=RCryr=mrNq9zk1shevw^%LRi~8JP5* zNQ&=Xd_>&he!SkV{LV zTE^l=!+tdfCINw#JF5)0?>ee0etE~SbL$`Wh%mTbYm?^MkrXtku-2B@ ziL~s)8TD3Y>gu;FfBf&^gXOo3Z3(p_^<=G;Msn@$Tq5_*?Bcjm*rwKDq{ z#qakrn{SY@D7eel$fm)-X6b)=OAm{zrsP4LGgWSZAq>lvG6gC+m(;Ci-Q}=??-7ei z!yc^#3PlcF&JC-$nwaz!aK1Y*Y0&~EBTg2B!+Ze^Y)p#m4h%wj8f^6zFvd6tFfpD; z(@G1ZeLtLgaPC_W^eKUDS0VRIuRSkvCUYHP;&@;) zgCVZMgWpDvbmPMBTi-xR09e5j9KPEq`j zqK(5+i~EfS?kTE83JL#w`L?g4<3f^p=@a$ZNoBiuy?^8_Qn}!J<3yj%2hI6BA9Dh= zmKJFpZj?EZsC8LbT;Gpf!$HT>K&RL6zQ=}7eg=VZsnzqCSr;u3QAnKNa)6UzA>V=n z{80&ETq%1_9Ao6-V2)y7iD+Px58?4@RB%r;-G6}X!vn#0T3lR={6-4wT#W1)2UbQs z;OAlF|8h9^g985?M*+KqPIDFc5Em9}_zr{Ddu7#mg z#rNz%^_wCbEewn7obQ}C>wC@P=plcMeFis|FxHTiyRd971=!%elWps z=M?RwyL6WSVG*)p6k4H>QN_%xcK)JP0$U0L^RzsUHwXBmj-_2%qFuGc>HiV?#AAHl zviaXMFl8(dV7tmF!NAP)P%vbQ)HX+1hD4q@5BY;0@IE?V8PKrDsY65N1)s)3_6-Lm znI1B+I56lf5Ku^z*_X&OCqe3!Lm2OKHK!*62UXjGp1fY;Al{_Pa96qH?*t6Sn?b#sf-3cjmD3zY>Nr(-4%T5imtzKzxcoZA+{HVZLe;Y zS@B7FMI+CuS1eD1?FzMYCTic;(svWtnm#L#)j*YBV4>b)hN?dY&PX*d7=7jY^ng9a zjO)#U)PD*B6ONgeEOhstD&@8C_X`2$f(J<%3hX(m{CW+JpB04u90~I{$ob}gB-=vQ z8QOfx3XGdvUkD|*doC1VVPutYVfe%#P|_&Xr7ZqNp=g$g>m`59Wg;;JZ#QmO6n-SS8_3r!Zw|JBP z)ncuqi82ypGRF?eif#L_!jVU4QGmYL{l1Mm53M`B>y1jK+fpNjlu!pQyF`8yhksm? z^!X>PX39|z`1HVmlY=!Vb-$AX%M=BEk3*w9zG0oF+B}m!aY3y$huV&rYTHZ^+3`h|Ln7qOG+CLQs*#gq%|5C*r;9AV ztS)HYv{)sXA)hIAW3SPVMAHq;mOF|J%covVlt~aOX1rQtzFj=RqS`B<&Dcy^gP*Bu z;%6g&%K}c;ya)UW2SxrlFdHctOyy?|aNtr~nf4_xZHEGT1_S#M1))uhGD?hGc8&sh z3Vc(xyO%QZOTBY+JfP@wfGMUSpzgrM00sfKgMK?sncZn)RB2#zD!wA~T;+{|KwPi@ z!$bZh%WM8INm)oKa>Si&S@TlE=*b_qoO%x`%Kc>`XWD{KyQXK$IE0;eW`B&cbpaEj$^(W7 zC)Gs?0{0fo%~WL-Qeb<;z%0YzxW`qXlZlb(z$}K{DiLRMg&r`iS`@Bf^~^(2S|)O_ z`pg5@8ZFHQ=d(@tClWh*vbAMXXw|k#cJUh-;xA70nq}QB`Y2Xxm?$2Wm>9V{b>~#i z9W57&?-nF3;quVYFSVOrovgjS#L_>#K#j4cI61VulwXsd?V-J5qlg`2i0M)hF2<}0 zuh@AM#Z?^nCoxz=Entdz!2a$4pI;wqiUL~@D}Uwzwk7NWT#6G77|d0Q_G}Aaa#_i^ zO|W%NDx+Kilgpz><+p+!4>{*OaJ`=(rRKr<=)+YBtIdunvrlcWXP^|OnOf1V>}J=J3`5)I|Fb@=mA@&V;nTJ`>Cb|hqrUs6T`B!593uRZ|JwuhDH~Rc zZ0D4JDELllr_+J+-&Ql(DX=&!;A}`>aXY~G=m6W41)N3and0*H9Pysw)u0u(DEAcW zhSbRnI^CRGzVX$aMeA3w#xU&G>f&SM*~2E4JJ)WvR$arP&5o0P zYiR)?xChZ$b znTi%Dsv62B#cq9=$B=(#$=rof(-fBmS6#2Zd&i(+v5(}v33qRJ-jKbLm^gFp$%S&o zv*lXn{=Hkakh>@9^pUA;hxwLIl3G4%F?&=v>k0>6>Cjz!ZD&d=-eqqzW@oJ8SjqQK zAuaVZ-zx|HPZRj77W2(J&D78!vZPVu6TA2_N1+eL!+t0*hlB_nVPNJ_6cRXSA8_Ey znx^`dhc>XxzM_79I!mKajswG=1tNYA?kzpYuf)#ivFTrg?UCp?S0a^;^Gm%EG)h=) z_?by4VQ!v0Qv!pKL55no&GYB2>YukhpPP8Sw|?8qM#;M6FB(rXTYuQzKexrW{N&^8YPelyPOZ$T#?UVfI!J&e|(W#l8U&6-X!l%YY79Jsvj0wjhSv~TZo~)e05VSb6oMA z_3kxNTlUuccIsy9mE4@WcblZ0kKV`aM|z~y<;?qJQck>=aNoxh`RU1tS8Dd(a&8E; za*KL}dn6c{zT!%bo_=kC()krZ6CUvxT~&5&<=LhG;v9qG!E2W^j0%o6ByP{+k@Sf7 zcyYjymHVQs%Z>*Q?Rr^Djub3Z>lEQsXycj4sn{l^=d8fN(58lcC{#=NvYlF|zshBHZMx!)2d|XGQ*%MOfaR zu=z>S6N{TO(jSGGElqisu`P01RMxbJbpN`|zDmmk3!-MKEmWJ6xh!=;16QF*(kT%w z55*I_A}_)iGqcso8H-LOZg6BflNxZR_*`hX1B;l>uY>G-3=_FccfMH2theOBp;o?k zj6zO)S6(D}aMZZ`I?AqYBH+k+Ei$~3W!24ZxWdR| zq568mAuo^1Z4wrbd*pXr3=3ryTyT?ttv4XaH`?%+$70i>*vZMU7v9{M-v5lvEoGOp z*vuzp-MV3Y$$@!kHwV8>-A;bg zxjuWz#5T>H5tBbfJ(Em%74y^YY1E_LKhs`K2fV@kncgBB077opFSEUaf=MdZTY86x1@l_5CoBDJghI~41pBO9}){)qn!GGF;S?9!g(QZ?sF^CEO$BYHn+(PCJqrq2E_288E#Gl;98*?)kKaILn=i3VW3+%R@ zJg>oP;KiVz(7^I03cgB}(y2_P1dl}kzWoM{a2uM%NytGBSa-AJ22<2 z!gBvwt=?igrbz`+Ogu6g4B|ou+4TYv_+MN($nj$Whx3I644n??j8hgEJ1{I@u##w2 zIpDm-qoluoofadv(t@p=Z`gC&Ll{)vo3B|ZkAtOs4%sH02ik-Jn5Kj* zWla9)tTIQjclQdBn6i!BY=YrlA!3~#Zds|u5{;Mnn*!o~E;_T%X1e5pZ$d@}JXw3T zHS+33JYZ2Z@XV~(%*Yt>FM)Y!XIN-ya;VMQwABZ^X9`VJUa{d<-)wOPMve`}w$4{3 z*9dSrGI|_nH+a#&lqk?FKQL^UM1I{Jcq@lbk}83hXvPK z&gA5MT`7P2?mMjyyPyBaTa)~B8LPvG)=yWA1sgQ@U5*I&8)+o+R4F*0KT^Q9v+V)T zx4RDPQykh{cyd@`ESMF}DX_hInfQ-^?}8L}$_FMfk=J!Un5S)!;$%pes_P`272j@p zWdDV}x2iH;r#wA&-@2xC*!@h`+nmhV7HM;grf!Q>&Cg%MVReFW<=q=@=1tmn9xf|5 zp?+r4zIRjgwH0p~FVI#jc6M&O6i~_>a<74XP0Z63UZpd5cN~bQ%9xdB_PqSe$Ft${ zBjb1!SeT#xJtTWQ%HQ*dvEU>I);au&oz7==&P!jgS%rbI!?__zq)MQ9;ZFvCLxv`P zD-8zbH=mm%PBgIUE?l!w#Y>Ub_)4AB+zYL$Rtwn+G_(#*jQ4I46`mCLfPZO0n^7E3 zvgGnf$C_4d?2Wy7``)ESzTz3oW-A^h%a<#k_&4ujs-9;?jL@{?le0CoCLb)}U<_zr zx_0+~;H8ExpA06^c^=F{>-I3p-Ez0ObnwWXR!v|&?6Gir#7i^|0 z4t961Dc@UkuUv`a7Ldx zv17>ucXN}4t5Sr*x2+Ogn-Ih_eT@cF<$@NIDUO_}Yb=i0uKKzrH6}ZHbI|tP?{fR9 zZQIQlq;DLQGuvpn-21Vb^~)orMgG?6GZ#Nfx+U^~LFkM^mv2s!@QMq+)J!IJ`p7h% z3fKJ}>^*7Kax1}(rW4GfGZa{Pcm$b)CLH343Q(|l(!gf%Z-es^J)xD=kDY{=@0%-f z#mSlek$&*{D64*1W3NsSv-HCNcZ)9$%nKaoC*V zP_O!_JqNz_mh+aF2_&+AXWM4lB-@u4(05|4_qK`W?|(Vr(IzvWSynE|Y+0c^<= z*wQDkWd^V)Pw?csB*Z7czN|s?WB}*M2|P^;d9o#Hx3xz1UiK<};bnO&+u=H!^@R4j z?j^3`FL(l$JPfIxP?q z7gGFoS!C7mfOYPQ-Y)`H9(QMOVDy^6mKwkszJnz+q18KpEw+NqTOjnu$BfE?pmPh_ zvNR;JBRHE3IWL)W?%Y~@eR*>60k&J)nFV(;CNHrhg9bZmd%JC&2nX zf;BpTb&ol#cS3(>XKUDwe*OTaf`qoE6WY#9Xk%YF^X!BPlOJ%*dC4(9vex`r@oqNu zq6=&}7uYIiwC4#-+$o?LWm2-AHKyT)_p|F0^`_+XFO*z(nLBF%=a=cZvlX}wKj1sg z>2zlS@4gA8iR~<|4vee{vH}JTnTtYiEn)Iu;HVT}=gZ*AH{u8qU~O>Vx@*9@%Yci$ zJ>BZF;}wMjs}0;9#Zwm?;NF!m|Lacf{_yz=j0|Kyu~fFn>N^X4I?UJ?RIN)_wCA3N@a&Igo4iNuJ3QCiE37o_CXH;Y$vSr;9D7Cf6#k7dWu5KFKDylUaMh673C4 zOB!4+E^v)X;r%X_V`W+==jn82n(8?-zNZR&cMk9_I8ZwK0_R)>u7d}7PiXSpd%*Yf z1NWJkxk=`^ya`N;EPNCuGX{NNZ@$3M{Gf}Eqr8%V^X>%RT@Boe8HB2qao=7b*W%v2 zvw)+$pkO`&*8+u=Q@d79tKz6OVB-70*g1iFsyLrVEAQok6`c#X?|$IBo4~Po21|tj zv(*;G>1mPTs~AF=8N4c1^FOs?3}Em~nBft?nx5d$o3N&rfpdZZM~`Ix%m6mY0}KKm zSe7MlEl&8ic+Sqn)~6QVKE+{sYq5jWlA;Ujt8cQoOlYs#!IWz-(MyVR`llsbb)h40-Kxzqejy5 zvQyGN0_+l(nL`#ZX*sY;7%*EZaNKm@eU-Vgw2gbBdqtbUv?UCj?F}4Lq*hi<<1p;z zF#Vk_AHeWdk@N63&2NMtKPvC#gz?HO;MdF0?is^~UU&ESijaS|lI^~=E1QITU$j(dr~bPSyzf3N}p@pFpO-%kzt@m7HY7*ePX}~p0E92&5*XHRQ3KJM#daj)R zfwQH6W4=O0%Lh*VZjOmkS{D;JraJK6ZQxy;FnxvqwmX>pO*vHiZe)l=<{B{I`0AZ;tL%84>+b8aP|jqR{z;| zih;8;k~LME!Oekd7X#;f$yo)`+q({&=()q*rLeA+;XuKJS?*F3Z!cfsb83la7RNM! zrt06_{5DM&KWzVWEBRr-!7o8OqJ;Ek6{@z+SaxP7=T?E8A0P04ys+%)g;S3l_+C!n zJ7U0dvyClGAobA2od+N8%>ER@_Cs2H0{e>#ye};Oab4}=zN*ER;=r_Z1@p;UoBZZ* zq#JO&@aE`vz-h|O`ANjoGJ(By0r%d5{42k9cP4P%xo~t>0Y{tG=0*o*|BkBdSq#Dq z3_gyL922Uhx7u$nVt%i&X5R+U<{_y9V(gsYs0=Q0H;ILkGqGtog zzttT5I_%26?2~lZOB_y?u5NdEFpK>S+pISy+dgn``fyGX;N2m>8*16KnKQ@sTe8&7 ziB7Ff5khM5JNAp#rOnw2=e7@U|g`E-r% zu0r;k6ZoX&&RceXdszX8n*)>WWn;evj>)e!Mf}{gD}l?ehhxD%22MYQ>z)&sV;}6< z7qE6mH^-g@ys>Av+jF@V6jU@^$k;A);lGTB$Bfmk9U=@D_A*~!2r=LYoxqy$fbZW0 z-u{HeO@R}tZ7wz^w0bCTxjtZg@5q_|d$F?ZrAc?#)pgeuHO$if%a(J2**f85`RtRk z_Au2vIC*mKb_GY1iw37Qom}Cdo3q&@ru}rzp3EK7UvqEUzOaHE76^mPdQYL(J+1XnNw@Z)C-uz1=w#i@LjoZV{P=>1q>XlQY$9~aIDqN*u9#2 z;&$#^3VeG%?B3ITgm+a&l0k3tf{T^BT=Nyq-@b5k_XD{-2e|*Iadp~-&B6i}IRmcC4|t-bFDd;! zv14sAgU-5@ktcH=u-O)@6I{7YXy+2`1ooN->?i-8JiYevB==LBJ15@%ki24!#KQ?O znfKP;`>;$=X6I1{p6wHMGFQ*pwdVAJ8tzxJp&Z5LM^t3q_xj9_J}fH0lD_c%l`O8l z_Ng}~bARi+uA9BGBY|tT!QLDz-W{(uRo~r|Y0I_8d+mY=*Y&Nr_Fg!;*MVcPU(?4a zEMgzzC7Kj>beVOEY-KOdW@308ZSZ*e9PX-`$E=nYO>?*9W!N^(OFCXwd zc))k7fP0n#*Y*PLg9+T*3b>Cny#Mizk9qd{m%VHzo(#+mY8jslc$n{BT&lKU^#^eQ zmZcN;ZZur)V}EtE>&4pakK4UhUisB@BY;c%-4VvTkK0~<+%bFivIEQBU*K-IaBgqH z`Hl;$VhSxXPKvYLE^Ip2a(I$w$OJa;2A25=r+ao>6YM>17RA|b@L*rSwniIP=?8n3 zeqitZz~1*~|E{yQ_teh1^zMnZ0n4G>14nD$T3fJJO?Z0Z?a9s$CoeJxS@RwYxz&Dm z!o-KiXP<0UEw1m#dB^?zpZlCI*SQWV@Ej@NIsSm}xB!2HKhMDnhg=Pm81LI}e{Q-w z@dLa6M$QywzJ6X2{#OSqGG4CyIkju|3@ho1z1IwWZ!)poYM@FSm#O zYG1j>^6*tv)s~LO41D>kIA<{&n!~`oo;e)A2@DO58xwdJ#7ZkP zFz|Yxl3Dra^xPFb+q-uDKI*-6N$~zVg38l6->GX$+D&cR@}ghqM6Jlzm(I)kWzNie zxaH=VW5&`-xlZqIY&>+f>(a9DHD(WwcFS6?PrUsw`Q)}-&z*}O9um!+o}YEQDfsZV z-01Cjw-4#6Z?Cyi{Nnsc-+n*8*d2wB*3C1TtQoxAcTNJUwwgfV!B2CHmM+}n*uc;! ztf(?${({D%zggbgzP|tCzrXx)mYS-;2}S}UQ?sO0m;dvh@QYQd~PQ8D3;wuJ748%Yy*OazupaTA;P=F~}z$Jb94S=tvoPYfF(>dK+S|z$V2XC@`QytE-fp0G@pDp$fiA~;NO(YV?WjB#H@P6 zChf`h@Ng5GK)``J8yPqbtl4xz>Gc{00R{$!jz>{RoGKOtf&_Q6UX(q+0174ZvtBSZ{Hg{l^ zEwOkg^mp0pnc}g2bKOohmdF&T{B4ttJ!Pdn_c16g6>Qf$ZS?5$uNRxVyKRjpcig@llbd`k>_-KkPD_Pu&&0Pcn*DTwwy8|? zESjt${-SsyTa{C*F<+po_}0F_xutD-K`-{*benGIwm^IGqOX5N#3Di-UfFFlr651F zGvev529`ZZ4cTjdZqHxD#|oWaq+`5=Jt6%#{D z&j!|-%!R>oUW+^43FK*=!pE7K(a&JCf!)jIQ0e5xxkr9oV6zN)W1Ph0JJZBXs&DPF zrZzM(MF)nt)ilK{(RlO5aBcn?kIt(vB{=VO2Q119O+3?lkP zfgDl-3{%(wWcJ)(WvDc4aI-p<$j3aPfhXW$oV!Dl*q+0jfj68)wm)c*obX_YuEarG zrVR~p8j75*5uHX;Hu9uBQfR+fcr3|ULs;-mLcpD+%S@Iho@J_OZ|-MwKC)lRsB6@11_|O`mDBNXd@lo82;>YhAg(%xd$Zu#u^vrq%nL3?l~ssIy}O{3 zc5|Ws>8AZGF%6u_8qE7n{Pfc`>73(z_q- zQWsT3O!F9&-&?U=>}cv)bktcnHDK9tThsURE>3pyc$?wwGVzLKO^)C9D!0De1ZDx* z7YwX68yaUwJ>+<7bLd~HxkKak5Cx?*vm6+A{SIEl*O2vCG^>*R!eusxgHjJ?xM?Oe zacK203ZyWwbj;g&)%?H#mOPHh((^vFnEzpH&N!=Zf0;tl3eAVpcO5upeP<%~lNM#m z%N{O#VOysi^YuI2%bgRy^5sb$kwmU&4Q8QD0e&Zbb#&ekbqH5E!0zQTQLww?%;8-u z0s5tB9E&|)I9`1cW%iR{+y2eL5{DAnAD!hqx#8g3PqV%svk*H~b#uq!mfT}9Z%kfE zUpDkpI8^l*k zK`G);Ud0Ae8-}YK4;WQ-8aGZ|x+Z3u11s-_8I0@+S@J~- zd%R4N1y6^uDF4vBW!Tu}vpqT2$-6lB`m6;k76%&nleBI0kDOdo^nf`x|pWSAG34M|ky_i$+D;FLpmkYL?m?p=Mh4F+cW=lQ`r2V+R(tb9*Y-n0HQaQB;`N z;UsZLpz((D(eDS?Z5keO$DeSNICX+m;YXv?!vk!ZDMcNsZBBx$M47&nHBkPK5*Ed%b$8DbX^)a6T zW0ux~80HFpwiT;bHf%0slT=_}eD$Y+S;m2}Y(ksKl!ZN>K8FNPmwmI_E%NqC8c+F} zs}fm%^p1%ZM1PQ4y?~U${pZKWVx5#dx=kh2DJI0uKqx<2$p>Jt}2$K0Ml+bf8V~$A@dqBB{^w zvR&%;bFn+xJi6Ta;KDikiyb_2i}o$^IK;tL!x3llL*D&6(}~FpElcYam>%^z{8TPr z{Os<)B$Tk~*VCH%>0t%&gkkJD650zB+Mg-x zj0xa$Vqgh2U@?5bwD+I@Ph*Q)0?P&~J6C}g#@h-mA6S-YuqpbwKhW*kn@Y;oVv;vT``EWqOZf!Y3#v;7a|_DtDRJD4qJH0$hQHfv~hO<{Je zVYX;sb`@bVt68-8s-vG`XOd_t8S#NsjYwH`w%dwK>e(^J-(;g9D4Tlluw_`chK*C@eO&_1X=<&B)(DI@tQe3EaDJ>PIFZXu@w1ItV4X6YNv3o52LI!t?1-29hG+w2E( zyLF4BLJRLI6Q=+!=NC)@5-tATEWroZiZ-;DzF^B<(Y9@}+)KsQpbo=!71s78EU6tE zJLjC}nBvh{W8K@smKxEP)zFs3;W<&{WJUv9Mn_xfgtn{~9{wf{%yU>+Qnu+>x|n$! z<+w7JtLCW0fyRg{2l6McFq*}DINIoz*JAsCd3Qcd?w_Tc$iruC#s0?f*uRuN2bG@8 z^AK#8IpQdrGez==*Q*C^nS0#U?cr02>0@DHb-ux5Xu)jrqDeY|$w7i^Z5E4}L1X!! zRq7Ee{8}5*Gg%Uq+#MyFtGE_6T$xrM!CsQF`{l{jHk)lh86H6m9w$mXy4QI0&N-2C z;>7lDwsZ#12_l}wJJ^aPPG0tZlAEIEn9%k+Q>~}%S?qFzNuR6ug=U|v!u^% zQD3Hs-s49z-zSR9**UY|L0fExfRRG8(dI@TfktBuu3ZNgxV>2Qmus!Xg{58##7bTM zvCNTXagboK%V6RPXxdT2m2+WQ`H%Lp32X@=Y%gc}bo89*?m2Plh)2q5kFzapsT!V> z|D3qk!d5JCmS5HL>X#$abR1YpSZ>Vev7X>-O4Gx6>(GPe%(TzVg+JtryLF!E-2e$vIh()Aon>=3KOSuc+s?^O%nHN|h-=cOJ8* zC+^f+puc3p1eY7L?Pn~oU(w zqP2(3c_CYN!>QAc9Y6UTON}}9@Y2E84w~!VD5ur(Wo0O-RwBEFmCs0ZaM;V63`5s~ z8DdkOF!#2 zZ=w0>k||5pUb1_n{4ee4sVr5el}FivZ?jI3uweC?V949`BjmCMw;_jO+s^$t z(Z?D(z2xMM$-8$Zw@6vLi2t25^{8xEs&GUpqvVXxsH+ZqbB=aaO_q1v6tiVTaj@i* z7_a*~*qjO-uW_@N?ATPI&|Z>pDSJxT3R}H9vU+KyQL-shHifX4GO$m1z&Ufn@dXbi z%*j}=Zt1j)g%e6xoLvlBGheheyl%~0&>42CHLs)HZYp~ggWLA0(axa|k{4SI9U} z*sv)y;%KA9j8LHy4tb{!ME~73VWY;QgG`nNa@HGImfVS))@hV%%9nD~rX<2C{p&f~ zTV7IsPQ56I%A4{qwI^)SW!4xAOO~CTvULk)|45v>fOFTzX`BytvUg6X+M~ba(LKAj z!Nn2VVYYBdG2$qrqyp2qwFw=+Vxo9rA2K#s&d`Y4y29?l&4ku@S5>Dxt2vjFdhASg zVwJYu8v*u3Z)bY$i@o1_O``9i;-0A33$_7??>Ogma0aP8RS4^3Rh+i7QeobTLu+=K zZ1ufRzoETuX-oEm+l{*SPlR0AwKV$cQlGuNX(`z^JZIc!OKnSuh%wj~;TL-Lh;CqD zu5Vfa%kff{NndqN&1jxHw^?7Xmz8aC0|SHNPZka%1_lNl1_lNO3C0k{c3xgy9xhg4 zULJ8?4nARCVPQULVO|MIc0N8KeqIS7J|RgyVLl;YUI{S~5fLd-5m6}-X-QF0QAsH= zDN!kD8EI)%NiHQZQDtdSB{3;w8EGvUNh4`~2{kTBb9QMZX>nB%Nkd_E6%G|8SuGVA zO;rg~4Ji|SK^a+PDOoisX)9$}B{exUO?g8(1!Xx!9eGt_6=h{@Rb?GbRTX6&Ej1lo zZEZCTHDetWV_j`?O?6{KT`OZnYgaSN0CVkRTX`uTW_@2ytyJz{579zDDR*b> z2saH^M^jHHGgn6&Zx?GnPaAJflR#J7Krg#+5A(24^NdidvM7r@Kb^8PlY|@@3rjaW zQx7RQ7Z+P+4_h~1J5OtuFk44oTkk|yXD4qLCm$~tFIVp%SLa|Krw|{nNN>;RV4sK( z4^NNKP~Xt#un^zCkhBQTl<1K3sEC3vpX}tA%#`q&?C7%e@QUK-f()e8~}yt3HDp!PJ$&PctE42$VymJ_n|mR6dz zhRAe8c}}lzZ^@2qEs5#L_i8QAXsIq~uPpB^%UfLKr9Xi&VS!-&CDE#mAhTnP8+y%W zEYz6(K>7PmgGbL)Cv=Bw==bYv$?k5coZ3-2Yf9DRzRcBA3pcMWT(l+;Vm z4&QJy^2=xc+jr6qEwaig=`5@4D65^6p5GRe+1y^+RNXwavZ=4TrFB|Y+w3VF?VZzS zPMtBOZ|c&Soy%uUTREkF_2TX=%evPrnZ9Mol&!0#Y*{jU>zdiS*3H_vYWkjyb9Zi< zv13c;r0GkS&Reu<$9)}>(*?UxoqpPEt8JyTJ>O8+u36qpWm6kY3K2| zYY!dVdwAQCt9#BozJB%g>5~^O-+cb)-s9U(KEHVS>iO$$uRs6&`|sbk@4r94|M2m{ zrG1CnIQ}ube-zQdy?yEOV)Euoh3lSX-rJM^&AhXCq4x22|17IN zvbGBsFt9}=xCs>A5On%ez|7lY+jc00PatHb!^~Dgw{$5v5vlaWrcaJaa8FIUn7J-D zB+x@)<%y)+?MtWJ_SXLBCgc}uQmE`}`KV=b!11G7kLLT&P?ryQ*?Tp#%EIttOlhR( zRM8NZ4EE6QM|)XSLoUr#pDG;VFjb{IsEKvz^jiyGUaQnm6%U(Oc+ll~?7y=Yr_01V zd3hwh@onaj`A#wmBX|vuJh;&^(Q|@>+|=Aa%Hw_^RhkuMC^LTr4(0dwR^(U(cfFm0di@DqF^Rf>oyGQ{;@qj+-A6 zT10&`X0l98-S#j|w2+6C;51$#pS>o5WUN*ZPD(sh6 zJ~ujorMERY^xE=g;AigJqg$Qn^BQF(W;Ai%daz(&pQ=_QGb`^D_O@xg z6EDb4lwyf;p4{fvev>)7BFam6qlVDo1eG^J|8jSE?pbwIXQ7nv8e>_`IsP4|#j}I# zRaeeCx&PCh9U-nxH}6&J<{wY3NV3el5_-G!9Lv^cmNP@9&0^NQdcnE4@k^!Y)T_bs z{QtP^5`XYc+;kr6r?^tVx=qzB3tG&?*0Xemy;$KWS*9W7vbZwh!hz+1E`OJC=1(a& zvZb@+_Q54}h7~^~=3DLiVRZ6Hh+nGV(tG{~5A7ET2>*5EO~Cuw&`selJ&!+qmhwsd z{3c~N6;02gC`;kzOBS`f5pm>>e<3D5uVBiuqnRE~hs1Wg+cf{*m#2R98*g2`v6Wf+ z%7a*qj-Kl(0%qqQyv|8C_E;*lNm1(G>Z)tphsBRD>}y*4BJZCU^I@?T35)42KAGEZ zHa$2dE}VPwiAB_biEpbm?f4#fH2qp;SLg&)*~rp&mLDFUTkm7G=B-YqRk)_uCKdIa z@e@~OEZof%BBddkE%?&bU#QQo!(_^57lHDC$<5*{ob3#6Rb!hueXjXQC~{VwRj4iy zNi(~mDE-a5s-)v~XN|_}8)rJyloY25U7oJhndn{P>HPFZn5o6KCoUfBQ|_*vqcShS zmA!O}N^tp(kTpt6+r9aH&drz-AZ%Eau6-vnrgX>Uc{aIzWvhz2FXeD9wRoE0@$4a! z!e3|heNRs6H!?ow5I3@CpitqZE3=|cE$rOfwnXJ`YIeH(<&!4_B7Ek*`84C0kKg6Crz$)Du=si$ z7MpzJ28)i0z~cSCLMLu|$>Qd9Pi#e*f%3d!$S>7HAw&vY5n^CLDf>pUi zNhXcG(=n0jsj4J*YR6oL&pjd5!cwc^^n?n^xzX zjGXE=*=Fte+AqejPMe+V7d7UU{uiDbcd%G=o?)ZK#Y@wI*J)09H)9HC1jpiMGaj@Y zGyA3=#j#ARP3-c5uMQb53)YBrTnJiuhf$#M$07fOptbs6)F<41!!6^}eZob`G{9Zs zU*bnO<(Vf;66P|UJS-vCx^1PO*Q)(jr>L#G&gz%z!c(ceuJ?%5sb@y4nKqZM9i6j! z&de*P$_f_p2Tn;=+dFMy!))Gs$<71e%WHJ>_U~0Q+ZnLba+3JsJBmE%va9tDaWHb* zUT6}Uu_68%XD`=}w~qW}4iA{#J8)KO9MwG*FuA|zwSDo8xjVWV^i~Td8Z-zo1?4_y zF4ikG5|X&RnC;TJa{-$a4lZ4lzDC{PweOOI8gIS(+Z3)zRZpFAgUfqc%BE|w+f&xP zl|EgVRU|WQVR{s!Pw>=fGI56jIx{|{GF2{By02yanlpieMUijgQ-^|fK3{<*;bk$Y zav?K0tRFP~n^iucnMwN@oAM4v&Sy5vG7lUMc! z3AI@@hP9Ofl2fS1FIH8V@WuJk>COb76*g( z%yWJ-C_Om9IblMp;f#koX&e<|Yc4PgPkF%fqK$!tuYf`4#v|!g4yk}Y#->)a0etZq z7v2kbN*vDl(W0N=v*(|Nz)Y5aV=Jo#n3w20-gAFd#L+9ld3L$VNBVylyO~Vf>bq~2 z?vkH1iw{>VinlHI*3@2j!y|svlx|xqP5rqBwho{9swOmBKUQEjp5e&pvEXY#f5RO? zj_Moy3koe@)cz?%iPdlmGCfV;+PZrkWZuG9`^$V_AR@a1Sc@Cs4@K5 zo0;u`lir?*9r`Yc?!^^O z(%TXW9lyErt^Ufri)HgVpZr_w<`1NTpQ}C+GS-iOSMoFD<;<*2T7o-|^U8>A+wf<* z-R4X^UVibFC129*XR`;(7hGBVPme|JK?2)O^DS(z^vak;4!o;YKX8wKLHW1&fB)8= zx4zF`|6{Lv153pQR?BS*dGmI}N^iTsI(K=+m*?et7Z|>5&(qtG_dbkK=t4P9K)qlB zqtXTD$O#;?47g`s;GA=zagIaf76dBeHyPvCHDuG$)0DEQ6s`J}=c zC8ySn9y_mRYMSWlZeh=S9J1nDkwl{H1!qB%ZO-uqf`%6beHGX`j#~Y?>TqNWOKMtL zCR=XCwzRx9X6p&1?}{>>mN6u4FG$;7&ibJ4<#C3p_KH{S6*b@6K0R;Cx=~(mqWq(H zJxc&1LqHpU1EWm<*PrWU`QOW)Byg8IF#Qu`VB~3N;4Nscd(LF;kiq-Il;uJPi$SAw z14|kM*Q^JPGY)Xh{?IvJqDgTDXJi4BssW=^17lPG_k#%>4vnc>n|1$fl;8Hlm+ewi zvm%SzhD=ijX1xte{eqIU(-{_&DM}ntoFP~v!xW=uCgmV#;?=;KN`07MT#K zzMxb!fk~}_S;e3)@q1~fMO#&R8J`0~8ao3E0|P4qL+AGL&gTp~56aWs+tbP`zOSg` zabVyn=x3_z|MsGTZD-%da0aH6haI># zJFqD&V3Z7CvNhnoHi0A1z-V(vw@7&*)1q$o1qw}{nD?)!et9Ki_CcFyN#~`@k{B0f zT@5ZUPbjc$V9Cg24J%-cG$`;aU{X?G;z=*(`@oP^F}*y!oh_r`_m1fu~_8X_&y8+0S*dy!~Yd6K5l9<&4&w6L@cy_jXL+4`7Tg;LiJAf5)MtbW%+y<3w;R|NZIM6xg1DB!&ci09-iG(SO z0(f;!OmP;R9tuoi35@(77z7&TzcKGG3vc`My{+^`8~=my+Q8EDDUm2@GlmvkfCBZ=S$yJ&W65=j_x4oEDv&vjyg;2h6v9(A7SH zHR77+wi|QgTGXQgGDX!SI1hCcE-=FI%-JQWRjAE!?hP^jSJSS)L>m_L9? zd?QrmU#i~&Caf$eK z@#$eK;ZhoEU%HPeGwxPr^nW_nKUH9&kZoRbsMtlx{LRcOcFddQ&~^9%_Y`xUcdxki z2XuucFtR;pU}a!>zr8>GdfUhA9lQlI>ns;kls8O}T-g~pLvYrN-digKEBd>4PH26# z;8VtgpDP(%61XI0Zj{=+aqdkX;{*o&2@Jv?GPPUgpD@nn`pzJpz+_nYk8Mr?+ky$a zZx8TIKA?H2gVpQ8vWe}S;Sad>IBCL2^9elR&dYFoZk)9eIR_f_SumREeqXy+AZ?9C`z zUbSj|*(#By8Gj_||J;~0NoxkXXM1tP1R<#j|6gqQzoT!eCc_LZjtdKTuN!pS-LP@; z?2WnU>`VcSd>a81C~r!ysGY&OK%i^E4(>Mx_^KON;~XUH*x9^3OtS9gw5w!Q z5@0=U$?av}XueUt>f)Rr0IP zAFCS#6!L2;7H{~qY0Cz_R}S+^zu5atWc4Uu+cIJD42CJco^@?)U|qSKCCR|iBU`;Z zNOym6VpocU`veJrLraA>Re#;olaM8ON!XT0aoP=Iwh9OKhy>R32kW+cSU1~Y@_w!^ z#f)u135?ad`dJS2wVY-UYG8QB&G_j@gT{uP{0fZP4i(H<4gX%3S8MF=vR>FWdqYR( z3RdX}|FU=dtCX2g^P0gpfb;Q&jl6e2OFnM|?0jj!d&_{g(SSkf!m&vQ88jLgCQj_< zI^MuG;n3V4ohz)lL`zIML|M}exV7(aPElv|RN#6Lz`c8Sl~tg+f2aB;^|=yd5?6jH zm^_d;xq2?&hPA(%%w~i*&lTLuyxEy|BYWB8$AEi*0atnho59U>MXNZ= zs@i^^no(CiW7ca1Q32*RY7Fhyk7)-mGcD+nGGMVTSj5UXqg`_*)0^_P=+2H?4D~(} zKAbq-$T@*QvXNC_d%O0Dtcp582lmMic&;??N@-v0I&nB!`WqKS+HfmF<4|y=&j8e3nAuw^ukq<1kJPVy!oWqhCIA(hswfF4W+%P-k z!nBAB%N7J2Ey~(0HlaV=eR@eoyI}&er2$h<^mJ_j7UuwFb{m#D1rMCNY1V!+$pz|60} z?7V=zEQa57dg*av^t4_qj|xhHR z7jW-;z`eWS_HHi6eP>s1`M`Z=H>byjYwxG`f0tzZzLHT~fVo(KqsoAZUx9&7fw}Yp zN528H_yM+M2E1n#*crGO3fD~Uk!&pc)6dn;Q0xAHTe@D#?s#3z4!+6?QH2MI=?ofm zjJ5(C+b`S{(!D5tm-mVRr%C~%(FT@@7kJKIV7D<~mkMAk|G-}7z*NS`sP}+LDuBu0 zX5;J+n!tF=~yUmJh{tB!s6z;Eoz%{Gj-;##PvT6TVj3zMqY+y9{z|vyC z_k@8{ieZ!V1NO%Tyh|0B>my!G6<}1zX85)C{t>-0@df2mZ|0d6oRc!xv}qlq_5+5k z{}_Y?n3c7UcXiK@;$t-0z&=fZv+4tjO#qXWK-uaa48|W=uAM8p9>C>Yz*wijaan;c zx1D#c?5j$LSEU;mgb!>JP+*F5m^{OP`|t+N+ZHCuhgDzJ$>j;Kh8H|toWZs}hIgv~ z>uGVHL6dA}RBy9!(?Hsfb>R+!|xeBtUi29C{L*Vb*|7QfH; z(%@R;0%c>z%YhTty~(QdY+#%c%fR-4L1F{*vI*?P0-Ti#m@EIiFIyJym~8`N<$tC&5WO%4@b8F#+Srdg!+&y@BMR{y?G$eM)I`*{LC>(SSFfO{aER%WV z+bfgQmM!}lX}ItJBL}-uisfd{;N`LFHr%w)Xc;x$Dv_xSZs^OG8SDyleLd^W!ZQ(Y*5adr73GQt>a)K!(kpJpE)@j z7pL>_tHpj{`83)2Ff%K^3C~aE6H``tOp==z@IhtfO4WPMr*xSeKltY-gM`L{^$yOx z9*>SRzR8%CrgN~lLndX%gGL@HF9RjnUm0mKxmK6hq$jMfJjy%S=cGf^)F>;4Zex>& zDZ)`{8=2JAcAU^$u2gg@iCZ;BlWVeX+Q~;O&Pxi_xK)#$G_6u-os+zJP1l1gosCDd zyg4@4MQFE)-s)O?YR9)<+8eC@J=(NhZ#z%3x4|Kv#ld>ee}Nqhdse2tqueip)&+f5mmV`|oREAzO?8UElUdm_G!>W_ z7z!L1*aZ?AOgrApaN9mX?MT|}%t?Yv_|%#b9GQ9KDj1mM-YiIS;@Z8DO`6;AjIv}_ zRD=V=v@oYfJ;pgpo=(>eI&!E}e5*o|me|x2hq$6z9xf3MPqTd3;=E*`T9-t%R8ik* zwR>KC>$@75*C-vaYU0?OQKJa)u=mZ?H;2IHd))eduHZ-o0MjBTtvC&BHPTPQxoG1XKac*BjMbwzplzj zY|)3-a|}ts`x|aa{&PuaWa4R1)Hjk9AR5bCat>6ug zabTay!D$*YgKvV#W3ej|?v{TxFvwK|1WuUXvSXVdCr^xmc~=&j*)m05JrM)+|DkD(Dzko?3On{MnPJ6e8O(S>ohwzClA%^P?D0(JpHVJeT zuqfVmz;4zN6s?tD#JOh8u- z6od2JuCis0J%_}zt6C*p{u!)by5P=SYH+k_fdI?L3};!b95<5%g4@$<9NBj6bmOc{ z=sNAH6e~RInDUzdE+5I`B9CXZsrW2l4)R&i#BH&#b%)@(AdxIPo(u;eSq6qYLq#FQ zjT7z|x%d4~V3wIWxlO~PuuIcUyH&xnU9YH+Lr!D?qe2D)i;_VDw@M>Vss)!uonyDr z490l#mripx6|_dm@C)bJI1A|nFgbWU7V|3L3)m&Fq%%=fJL=+%V`j%})#tK$DP?z` z)KF+#y6#Qt!YADEZ;X~%Hzax|%$k|FD`43atz+Ct3`cYn4y%PvSuri?!+r5rS6c4( zDhjS}SfCwL%#@nPgCUs8c!~Y?skY% zbDP2{$m77$dVp1L)&~}6H6xE}(n8@oiZ*T(S(xni>)`imMoYOo6jRasSS;|2k*|T#;m#wm zqztDC%RaRzX&Pxq6+96)V)I;R#kp_Bl7myct~LBNGSy3*kQKr#thRr`N&PtAo^b9v ztp_J0iX^M9?mQQ8S7*rfillR% zoiSV5B4OLkT!)tCRF-=cH7Sxd4V=3YuWl7pZD#zJQ=NN8;UKeD0(*4J5=l-ihK*7U z3@kQ$Ok5KfE%n*ma;DsjS+-h0;&Nk}!TUqopE2njZok;bw`C`@&c^&X!V4HxXE+LP zf9!Iw!L7T$d#c#F3fBduftTbsw3q~M9M<({y!_l}mdDGvUdOfwgs7FCpJX1@ zXP=SuQ)pt6U|>4A$iF@Dc?0K$gpHRajxfj>Fv`wRlD#5ugzdmnLD`rFL3z%ja}KKZ zrLZJ2`e!Um_6+Y+FHp$O&URzbo6ziFaX982%MsQO%!~{x{=JD15IDdhcd@4}>t@Vd z2IV)_MZL!C%3=p!w3=$PSjGQ1^lj6IrrFU!Ofq*ESUDrKxHtlulT@L1-2@zraHd24ytA=7zKJSNSTv! zAoOR`sny4DE1?`Okv;=HA#1dvAYc zz0sv{o0~24!#<7)jdm3*HU}E{Uo7d-Y7la0(lKcE*)i962h$x(X8#W?(mR<g&)X zWWaE3M!#RzZgvefkp{;s3l`x$47-^*opv;seXz+2V3OmR9^Ap9(XcS>L?ic%MhWI- z#sH=qWp-x^mgo$z^aBk-H=4IDY-@dGC}|+Zkf6(%vFPW6Y3vm?9|IOn_}8%a{A_{T z7i?J`?Qsl;;}R6JDwf3yv_*8ZCJIbppD@R60ow-0mc*bf%n}V82O3mPFqv&Kesg&4 zq$4a=TbP6oG;npua!g?OP`pLn(dE;kEv~!d^a_@Id%R`SjYj1iOMJYverxag#JN-_ za{jX4`e97XD{gC@PdK$+s@LU#WX$IIaU90+3t9s;b|y-p*k=3Wel{juSdscy^feZ(!y*!JOkP&3)ovWZkoY|7#xAz36OHT& zEkzL`vK38hB-<@7SThSW)aERzopZ+WG^72BUWFo!mep%xKkU7I+IE9yd*TmU?EsU1 zJSP~fcd%Z**lN6iiC>_RZ$m@-jt1ckO;dL@X;-w`9%*^Kd-_8u5#uCXt{)AyD_Wxu zFl|g|67@XdWvsG3!R2Mk63-<|yu7#<)Npe-tP9%Rt?RjK^(ig=pZ5DdaK|$<$G>JW z{K2$JxLL=eNqd3o@1LygHM7<)?zudp=b2MeuH><#tV1j__*1mqI43lxHL~z(G;kF# zoLJ1}ka)On@om+qe6RH!x=^eMyCtBqCukR6B?N~S~54V zYOL_RxM~^Gj5ACZ7~}++-g+3_xOtp0qT#*8)U+L&ZocBUaaz}43TwQAc-#%A&;_y_ zhnjB5YH2Yh`dwBFvm_jlw1$3n(sR+1bVjRqcmvu1a_sC1bp zB{cWSjJKMryoF{4WOs`MtXkUCuvcTLzsu79S-aMH`LEKxvQ|rK)$Jy=hy}(AcuqEi zMAytRo!ArJA?2{@EMsZI#+*ZZi}^WMFi3Yy2s#mPZ4%?P>K0LrhC2>Uyi;2;XS8q_ zG%zsFaRS$|?7qqxS*NjvqQ zOj>3CcK`p+LZNrG_A@YBolyOf;~%w{+3H5qw9O4V3wky-g|N=)@y*clNtO8(pt~cv zH>v344h5EIjTS+ThTjql))QK+DtILoTHI$aDswT}YBVt?^zu(&R_b72mpCT5gOR<1 zo84itphBbcfrjbIZhStQ7$n5FX7EpoVZ3%nI=Z99T7v2A12Og)QG7K##Wq&5NJDL;p=+2;rooX!oRm`{swipG&6HBxK^`v?70+P!5VSndcp(NX%>=P z4Gh!QFc_|Aje60z?uO+*-Hv%LOqRTfys==Gi_!-P#TO^%A3h>6A>!SgV6GEFh8Na5 zr0$ySc|L|K(%IMF`T~>w0_Jry7=DI`R()P)ZjJ_7 zfhLcPX~x~*i8oq3{{}^Jv~};cNc<6`?a{>gC@3+5JuzU*qbTRzpv$vwSt=c1a`Dl3 z4~bnc>#}l&km8OdZ&DgX6ePJ${0pAElR1cq$8dwzy|b#$N0T1By`i_D+44n_)rm$) z4Mx5V41yb)OeQW3*H}H_v>?w7hMNx!X31VqTjleMp@CV#jU&Qo)l-LS8`S57GO#AB zWlGRhQ(|B$ICMObp@xOQY2MBH!%-|94XUedvN1ILI>pG()L9fD;5~`lYOnmJG@(TC`uY>9%+XA7jf~!4@FFz*pDW>yxN` z;?m7KL0!8aK6-U0Tedm6fXg#^qNlTdYwG%MMR(PHHvZV{(oxE&9=2-IPrm~jn0ITW zKd@c8g8Syf+*1?Q?wYi>*;0W?T7gkGVu8`Wb7w0j#+htrdir(W&l4H~cS3#zm`#@n za7kOcv!LPKj+@SQ3``kwSSP$-y>MlF_wFfyYyDK+HnJS_x)!x_ZljIF+DOS0k&5lv z8(JL_n1m!6IS#n*Oqk=O$*4b}C1hora9?<&fQaA*Mqjhdv1JaWKR6sVFlg_vjgV;Q zW^NN%b|-n?j&{2SwKq$4%wTx3rp+*+N$E$E_(YSh*)C=dLKXttZMFL9jFKM~#&-Iy z+FB=RsPX*AM&=+TfBlT3p*#i*KK{G*?bv-Igv zMeo_78CLTe-nBAxh`sn<`lO~RgMG#e&IpE-DYB>LasS&{AhSbTEhTtuUswv;12LY4 zreKK^jVTKYXWFJWwAe~8Nv~+|UDa?dx6_S{QAwjEN`u8D;*Nu5gLH#o(SzvhAGhNk zv_4qXxmCE4cUmj+iw1^{nA*MbA1F8QeYiJ$S_8+9=BO2I1{o8@I43p7u2)KE^37h~ zTNt~!$Dxmh<=-*G8rozinI=>00(g zvZ1A$WkSgU0Wq%5T!#ml1#H07sTgTH+jrb`N<|J zzoSv2g0V9+d!h1kT|@tW@3>cg+RAwO5tEO9{AVT`1s1&xX zI9r#&<=BU$l9Q|-4z!eggo~F=2w-o2 z#KzSq;vp#D$-$yNgOT@w2D^mFy?LR0+Zu#_G^btoxMLxk=KN0;u@AQgn`l?G9u1AT zG9_pFl>&W%&;M>V&HFIDCDr9?)spXVLh6p)^LvDTyD^HdXs|I)-#Ks9s(+tuFbEgK zZ_fVev-06yX8Re<273F19Ezr`XgIM>XpLyDxB3O2?_YK#uT|4oR{l_ob3yn2srQ|i z^|PLcy67jlHg-{O*rJxOMH{c&VlQBlIzDH@gWET68SP7KJ9n~CqNCwX-(1FR419h~ zh6!x(FF8s(icbEuatPW|Ef?&>Cir?8)2Y}y2^Fk@9SrZE2J1Vt#QbP_o^j6DKVRL0 zS?@;G_qO+*Ousk(nCJbeaZ4Jv&GjV*4E$Fc`G>vMs?I;fbmx1#vg_^}&5{QgSEs(5 zKKs^{Wfd$E8K>6GN;zH9IITOS@x{%`P234J!Zk0P(+=}puIO4`5%#~rQ&6HoB2Ub3dndo+_vMb)*akfi2mN}=T?a7}R z{U_Ks*qIbLraWkHY-Hxr?GQ**>g41T($xs@Sa5)uonOe~fyM;J6KpIpJ%2bhH8e9R z$eLJiJZxl;ZJd-jdztQBvz!}6_pUtqtiCPpZrLvt_qz|mm*s?ZJ>jj)*JTsh34mBNIbffVAgJ9UcQ%VI4OmUpTautp(E84}> ze758i9y;3j@QB}~0?kK13I&y%x?}`5B_E$CZB}zUUY&iC#B06gD4e=Mdo9Q(N6|;L}76b~}X^2X-BJ zbBkNfhw~xBVPW-29)`+KyQZpmx`fy?Tx@jYWZP`4z!0?Sz)N|4qZfJxO$^*AA6$&D z_-q#Tb4Z-r=f@o+9MWw6vn52qIVYh}XCsU88NG1EtGuSWQ+`}E$S85RrX7ED(J806 zqef@6*-H`*YAYYR!4UUP-~}tAcwOQ`*M#VTgRH{sg$LWjdSuzyQ(S%ASf)71Y&^_w zcI84O&%B)v4smHzyj&_YFiudnsXDVLjfnceX9yY4X~*mz>Z#6*L=4aK~H zc9DyD!w;>n$kmbLbDrpAddXCGlV=G-6N^YoG$Tvx)h1S9UyiaF!ci*6@}&GkE=Hud zSa?bby1$5y;hl3);Bb1}q-U!Cvf~!Gxo5@QSQtH%`!QSe>@7!k9-XuAz_`bu zh4-gpf_&{QMu$sYzn4zA#JrKOppma>#v`BgH*8KQGz(<$J`8ZoOzqXn<~)2_&+6fm zQqy!9@3mcrOFrCTXS@*ccAHk5q7%DCkN_jAu*-$VJDb-2UYF>}cT**`UsADU%8bZL zRo;2M&r;_vYTY<1YxeS)A7(B*KJ)gRrR9Oo#ggtMvdh{$IPjc9^v~MIEJ8XLCf3}N z>=Hl4WOyppJ1{zdNhmdAma+Prl1ZMQPUcJfYh`7&5olIu5l}p-tTKV6apBE{CPCl- znRZQL*p#GlSf**prJfDEX|f^a*Gzt_Ubm0qvY++72;u$LZv`-ORvqZ|4wZ8mdmOp96>?&rqD}U4c4qjyDw=iqE;rf74IN|`y zJdM??IZFarPM#>Mn=acVaD+i{!vW4s32lKHT9xxYxTu{uu;*E!lknmbhuGK@m_4o} zah}aMCLr)tW6FXh_HDQ3mKkjlXE#{IutBL&)}ydHj`1J2^qF5$%}>ntFN{bQ-}~XH zUfU$TpS-D3OAZ_?7g1v3pYy_n*`k3(Y627Mt)~pUFAs38QIHFtq`+dnfPv+NDI-Tp zfK(z!k1&_1glL^LqBY+&VAxY%RHp}@qT5Lp+sNIopro#FbyBpY6XCRLH7iR@J=0qZ83 zE%W+k?NiY>9$Q0%CEvsh?( zZo9(Yhj$&61hi%DBnq9IaLn{p`Q#06Q#9-rx>@Z?6fP<`D|K}RtGJB9?qyveja)vS zj3Js=LLYsyQk~$#!Y9DM>tM)cHld-2Vd}vwktU8NLmu%CPp4$FlX3r?44MSzq_mt0 zdAY{?gX6R`52=d@#b$S7eDgfQtqi6V&kO8g`w$n&2}cvkLzah zJ21+a9$@AD!Qki+z@&4=Vam=P2gV$Rru}axadfO;Q?humu3WG<&Nbmo@|4W=${uV* z!JitZS_n%?UYc-_Z{FG50}1W7m$;c%Jmt;V_JQTzf_I$zE*c0wJHl!sBgU-yZ-WD; zR|BJyXRQE+)msRRAG`ZXq2?+?u!4BB>U3BRcy+EWVz&p6;6I~YFCswWM3?h ztgAS{PEH?T5z~FKE>^Fboj3No1Dn z_}8K!*U-qmNShayw z&P0*RJEKV=XY1rR)fH;57O<6U4CI~fkj#Ic(UDO@AXK*2{o$)~7hk#iFs;6KYbu`w z17{h7MNHA1#~mKxyKR2xxm#Nu=oMxX?3ti{`o@8o|G0RY_a6z%?Yeca%=YV1tu({= z`=hvhXEd*#yFgm+%tB4~{zDRH9M}vs7}?$Olgi`*7+6273Vm_nd-O z?HnfEHO=;0noT#%7t&!AOmO7aa1=>!6#K%Yn{Y_r%7c>lalFS z-5G}jvd+#}+T57X?8|XlEwuShOtbNug8?gVm+qM-8nw_`^Z*Bg8^3~BXRQO@9fk!D zSUEm4urV?T3o!1sU^JXKXU_%h$3C}CSx%ms;CbM}h3PsyvK0rj&m4?B%eabl(d;)< z*OWG{emY&uhSBgUXY0krbu$^nCLGl0X;Pcu$Umcj^Tq*=1K}q%7*;2U0hejmV-pDvQz;I!xW~;u`jvdAn<>jzfy6pj+Q+J*9W0@n)yzkuWGY&-#2ecd=_2{!@tOm{r)+NKFT8Qt;s8rQq!`Bn?FCF)YnnGtZQP>CD59evrokwv z<0vwxQGWrG+6+e#8%Lvtd1g``jWNk45vryMy(V_a#y6UbZ!~EtBx~MaGMLq*8FEO) zrAbBLkg@<%3>%X&N0U6uzb1K(L&`52<=!xAX7Gk_HmO(~HqAL~rlDwV;jB18;*F{R z%Y#Xxeu_3u4k1V7Ql}lb;poYKhasueyKlxqla7O8D;_$^Etx1F|Jdi)W1mjrzXuvq zPo{UYxh`sOefLoA9HYdZHurs;mW*>(%Pv}#$;kOBM*I$=$`l9oF9*C{G3lOZ5}hNo z&h~1T9pA^$~6o6kCl88`;%G3tC{5PQ?WY?XQ|@`1t^ zC4H_Y1&1a%4kv{hjdD*MWuF|B$vG%1;3%HK$X~)J_~yWquMEOp82ES`dEPL5u620m zdVnM50>_Ta_b$pV;A~)g;%M=yf%RVkL!qFPs=zZd4`;<52S$zd`wv>n&(2J~G&6;7 z>fF1MdUsoOTMzKuao{*`{jOu0y_d7nn$yBlo@JJa%AD|CV#)U833qnZG2S~484ncY zo-LC*bnH+;%Ctu^M+@ekXBD1N)|e2`BX-A8amInbxsG}toEVfG{W}lvKA7paz@>8` zOSxI7V~9hU3PZWg0oDWuk%TVo5+3U0o1(U{0;WKV}EDxFz8%H?(%zsBLm`V3C+*qR_qFC0|;!l2W8P{&}oS_I>^1c%m5^BzoKNY7iv6Ve&Ib2{&u1B+vhFhnp2%}`oc zcF17IA)OdUCM~W0uLqdEIrw`iuuAFpcdl}9Xg@8-yK0*BE?$WYJWTEB#3jDR1bHF~9aJ)0?&kzC%xL+?n0KymW<%rMqRelSi_Y6N zeixj4-;Plv;cCK#MtPAYu>cpbhC@0g>5?4`t^q4{XHI{x@}WQm1GfO9+!IFO4-Ntb zjq(MJJUr6hb&hZri1ik}`QIBBSfnT>z$i3_QTN9og8&7g2!@?k-I)X%JU-~X?`rWd zU|`W;h~v<|WO2aN(;@e%{8^zRxf5GfYQ-?VIbh+zEEwUy!g2PZ7W-DcI{^n@B)^l} zRL8JsRa~6K+wYGgS!}Lye3-?Sk-J;*^(H3sB_==^+@ z!8DgMx2lt6fdkX!XW1-b@@#?%Su@%`9uvNDa+T~NrAfLEnWjDb=f@=8z~H<_<#At= zM8N^?X*1@%dm`|}QN@8-<%5Hq!C?~@W<`gC$`8C$jvQcSIKV7$im~88W+{_efzHA* zFKs=>lUa=?)8^{$aFUwK@Gk4PhlT+2g~*Dh?G6=J)+c@xYgO{AVt=m@!>1xHE}@&C zdAw_%+&serQo*k|o;2_#Jmbqa#wXE!FEH)c!w&8@4B`%+9K4+}HjTUnQ{T9R{(W>v zaYvG%gxf^1mr_+zCpBe14!AW%f}O`Tp3B35E5%4%|IF@v+V|cqoBiR=s+F@=u?9~x za}*BQAbjSatOk>|0MipW2O*UvjUFeB35;Sh4qQxYi95j{?a`!pgG=*E<3HmWPAWZ7 z#vX@M9GvAOCb?=JVUBR{zjQ-j@m;k9rq@OZ{5p<3rA_*O4t+{-l5`)>5e^<*;RpskheIX>j->^GT7~?nX4)Kg4#>qgDGD4?eUPB| z<$-F?0Y;mvn^(;f@L@8xU^eb}>gvvHY;#CWOMAf=Wu@EG*=5_5`Q>k!`9(y|`s_RT zX$2>1K@oGtCf=`#?o5wl9+Zgtd|;iuNr@>h(WFs$#im_0jM5xVh7nE|UHgS(nlw5N zDKa>TPjM)CJF|`hQ^>?cPyEDqqf;66wHepHSNe8--zg4EOGn(HAGt|{-;+PkNfHTp3tmH2;O z^>yv4LPw=ruLJVG3)Nf@zT&9ha)@KaWrhL=;g%-dIfqm(9ON(v)k|ey+tH}WaY!hK zVLPXj;k`o&KFx|H2WyP471=mM7c#^p-;U;IZohCya>aa!)VunBnA9r{W$Zleap&Ay z3x@gs_{`itXnK@Xlowcd%e0g$etSQyViixzmn4RD3`}ecOhuN?Qv9z|_no_;I#13a zQ8DI_s?3(Y@Mhx*NB){du@**gjYcsMCX)?^WCT{!emx*_NAOsxjNU^AMTJd4)6%5n z+8!DBEwSyJ^5;Nge_^q4Mch+)8=^Ly4(>JiyPuOL^D58-h zeS=Zuh{L+f2F8R2VTLA!H;#N$B2Lywh$ws((s7*k{Gc$8lkt;-0(y=HT;aj44BTrR z*i5dZ6*{TyFxC3Sq-MH&>4igj4m;Fi7=;QNstcw#s5tycs9+A+xYpc(sUm#c^IdL& zF3c_0&oWK__qoVj+|t>Up^4Mwkcq%D&LbA@1sJujFe!a`pmL?j_>PnKyaPOQ4$N~* zR#b4(-O!|a!%6o_Q=#u6*)I)MNsN3sj>Znn?>rm09~@PbcS=~ZP1mskH8K2f6|orO111869gEgA6Yv7V&wcW*ZSxw))NhE zGki`awD7qc(%ivV_xqrxK+{!?MmY~>g(*$)JA!`}M04LT|2XBKq(T3R-A?)*3HmP% z=`WCFjdK)p*l4-gt36oK=CjrM;KH^26>DcpRP>$~d&sfC<^bb{^E3ZF)vaijTVlAm z&e7PTdE@h@jn>S13I7hN_B8e-J90c=(i3rVetgLEij(3B+Z#>4w_I{u?E`+ zPpEro-MDzQk@%SdRy%`(e`W^n6`gnVKv&!W0g1;o`i&}g1T}LGu(~v9PGRELXqXpv zK**+1TBT8%he?RXY3ZEC=-sC3dzwBmJh11TB6?>XqsMus84^xBT$k^J$(LF%7wl;N z-CirQ-s?x6=s&Anl1EO7F#Pr~)k|P5dgx@i`Zi<`KhVfhvS53?CS?hUD;Nh>Zf4aA~Ir(^5N(VJIb+ZW+xU?*3?U<(SH^-*(&{0lb zJ=eTLia{;UCo0+Xb69>l+}1CxAG07pOOch8Q$%Bl#{{M`bM(UA+RRiI_7R_HG~49k zuO*x}X85ESHGXP2A+$Any3RpIrz0I4qFNah28^vOYB$A(Q!oZQnmxVEaC*EC${GV^4J+hs|=88(TBN?0$e`K51Cuq-%mpqYhRM&v^5 z0!C&w76p-n3AY&=@9ydq)^|`a|8!q~;|m*~j8o|YJ_U>12Mxa&J+JdIPEyl-vQ3s< z!GwYFVMB-a_5%;SLcEjK8Vap$^bTPX|F_P2deO>d`(p}Dz7Pl#y2T-&n^Ty^zQV3S zp-GNw=TQN*js=~av$B#NFPmM#6?jCc$G~ZYeBF*lHmRflFU^&UX02ir*|4jtTW{T` zuI`!hZaiAOepbgyovj@%++B)Y2~C_br7s-WWYT6hvPtL7IM}8U=wsN<%O%}l#8dR) z@Dytv1;$NGG7H``vih7jJVoP<#nVp5HIgdamrfl}Id5#U^HGn{i49IIH~AF!tl1@2 z@G-m)u>G>@zEo--p1YxooT?gFq?9%-UeFlD zf3Q_Vb;g62sxvG~n3fBA{&}gc)O71$n^cI;Dt!^D1qoU!mh3vEwYu!c$78jbzYLeD z_wYz?X_)CGEmwbcLz&-j9s@Iv^@jzE`K<43Q0}+?bKnNQb=w67W?nA?26m?zH=9{B z>@~JDu*>XlXg~Mq#hG3WKf%^!l`|U3&o7=ip|nj{XU1BF$aXb8CWeM&2c~y3+)M)R z*K8Jf<*n-)9Q?wsE3r*6$6x`IWKo1-6NAEp1vv-3X0izys$G1g2R=a+w=b;=ku$yJ&Ps)~ZV358AA&i?)2!lH0yxqKrE$dvV)uQx(JIUMUNx2RhWuoA zgC^dCoZbqD6t^3+J9#uqJwABUgwKgLE5Uivt?UkyIgY%Z4;tGW6b>t73354z9FjbH z!+9Y`KqGs^0p61wtY^#wIDsTa*?_HHH;Cg|PUq_E+&4bp51+CoszG@Xj95~3~=D=cg;H>18BF29l3zC=| zZafl^xFqoM$d83eEAm`i1WpBXJ=!I)CqYwVMmSf3i*oJSE9KU=nuOzi?N-q2T-GQP z5!123Mycc=|8@gbO}@eo4;CeX28JUF6K1j7vOMBzv~V?6W8{3@=_IoJ#8HR5K;E(t zCKgr(238pZ23Z*;$z>L-B6<_p-ER4!euO+{p}#93DZ8X9XMb zUhO<6@u;~wS@PZd5ROI;rwh&AUxK%?{8GQq6wd7;@Q}rPhoj(!grjY43@n9fLY4mc zF}kTunb?!YGO?rKld$rd!dSP3jof#RPGLE-QB&lA;Jz(I%C@^q)Sk}OJaA!=&2|YR zJ?8)+!8r-+wl0Twtr8kJJrWv)5*oavZ!oay2sq119Fkn3(P~nn*cI6DNaWfJH?y3L zy_QpiH;PL*aI+ocaNLl@C^f-Yzza}S%aej5y!T$dpNK!lQ^z0>0*x!ixc}U zPn~kP12?4K-^j>l>ftfUWDmO2#N4=pjgKe5?;t~iz~cnhNs1pBvoG?p$Tl3X=5FVX zd-JMu)!RnVHjasP|BMT+-eA>ED_nE1mytX7UbCD>Lr2@GkIY-STy(chRn$!gS^QXO zx|Hv~5|#;@UhmRh6dv+OR8;TGL$|Ppg8R!vrtz}+S7sT`6TWqbWscM&4*vtr3Q8YY z`2K{mFVpX(Mm+70dPp#>Of;?*_hx2i!ZVl(bhR>}WXOB)-;FKzo_M2g8uWhsi(8 zXBCPb>CRAOnzh)0#p8kHg||C7^bTgcw6I#FETO33zTqH$Pt%l1mtHRs5?H@4=G0=n zjD?&gl`nnT+y5y#J~LVVRy9SQC9%uqg#xFG`=N{d^IB9q6wSmgI9}ja?1u@XEYlacs{iJQEis;pL>{L+GP!P zgUcKI4=a?dn0e48(cMXT+Cfbp!Olz_M}BuThKp>Me?(0=a7x~yvr4Mry_Wd7`ICMh zvcLF&D~@6DgVx8ck2g55^1m@IXmoZy=z6|OSb<3@>PJI{=7&ZhmPf_9g33EGlHBIm zOjma^nBM6(MM-QY2W!y6$DZZaFMa&xp*D3{=DA`&7LAPh-)~|MiOnkKHkzTp6@H>g z(!`L|DRtgKOd(u6O6 zIC;Yu#Znk0;u5(y)_q-&(NUy$Ho{TV<-KShE4SK0sTU4gVidl69VlLMQ1?jc`GiN# zUhV1@uQ(ngJh|d=!EYT~O~R5H`dW*c`u1eU&vWQ!n#dh@it}5Jv%tnn*Svq<@c!Ml zK$kC*^S}Zp2FCQXMll0H(U^t82LCn+J1pes@)LN*&+fOD#bW{cHwS?~4%{yq_zcq6 zruAeMHHQ@lv^uyknVgXZ4^GvM- zN$CU3Q`0OY7#NFQG)+@8n6bF_+kqn~Ih;}~Qae^LD7=~@m%z7ep_tDBrd5lrGG5%< z#=){8LA=OOG~%Ir1f#^P0MQ0Vv4n@BOnu^Ejs+pBKRfh`@A)7qkjS6&P)cJR&+9cT zS}fvxnm>+p{0K|mRp-Rf(x4Hf#}=T)Iz!3rhXL!6Y`!XY_qas@vlA{&St4k85taAmM}9_Fz~;7z{&D(a&X

#Z{2(N;Flzk+rsoQwj@&a3N-k*R)>2?A>JL*pr8J#;-+|Pa zdjh*oxnA(d)984laffxO-zNTGhxoK4_Z!kvT5kf!jc)dNMm&6q zf;u0?OAcC>H;P^B*I2*c#V3VhObfYRG;n`-AgFTiAAef|i%bHGLxRB>7OT7i=1k6E z8WZg~cF&VP1+s*#kC( z2VQ&*u^rhgd<+YBbZ~b(U?@spabZZVI>4>4ki#p1-R%K$RRX)x0&X=%_JRiPk`k70 z4V-BYSY9v)btJH>Jrr4@C|UZl|YCW-w|gTRXgLQNceIgCPS3E}6r)t2S+ zKRU#cc0uf20}D?l`yL1G2|mozT3Du~ux&{Y`jf!UwNPls0|B*%+}{rHS2eIRJmlMV zKrE|~<6DBzjs&4o3kCEXWu>%Msa+SVU=X^(DD>@sq{2cLuLa)Ri9!XAvK$Y&=O`)` zJY>1R%=BXOsVj@!mu7qo>Gxc%pwRh+lXsd&;zF)jC0t?oTn$^k>K!clpUk7@6rK7x z?+Zh6@u|q4XLhUB<~~TE9qvyQfUJq*3gj3)hhaJO+lG77jcfjN%N9EENrWUl@3%HL#~Nu$VNk zoH1?rc0j0X0n5K+j-mvw76vh11&$dFEIthEEDPOK7YeNFQxv=#ukxX2PCx; z`PVI!*|R|G2BVNyqpZ#&>4+ava}I{SIG7)|LhQvtF@=RPryBVhW=z@E$d{8Sb#Jpw zP9ulG4tu^wtyK%eqQu2FG{jwR-eL7%Zcg}ov1`mBjp8ASq5@{3s}w{ZM2IFVH0dn= z>i0m1;ap$!0cO7kEKUj~tql5VpOW$)POuSM{72`>rtjuEr)y*-xW*}@s#>zf{5!C8 zO7Zy}49`UZ_b%G3R{U-9T48Hxi~rjc1UC3_t!dD(Y!qI1fG6fA&pL$${{!55j|2=7 zYGy3ptay;(v4ACS0ZZHzmb8YF^V3+88d!21*bb<%xjC>O`^5R>fxw)HVmSvDZ#K&A zX_US4Q0T)lnG=pia~}%LSt#>PVU9|o(6)m@5epUf9FSB~EGTs3zVlPy(1L=U3k0|l z*;ZBZ>oE#x9N~_7lvTZ;!@#UStxjC$IgPa?(y&V|@ybgMAUGVr)Id8pxkIOoa zD+^ryH7R*}ePa#y<{BiX`Df`Ki!$de>(w%i(jSXYkuT&}a3H<3fqjF5KwA#?5(W{g zMq#UkQ_UX=8#M|WT;!VMz`0IO(B*)O>W@bG9WH`dEM5zvOwTZ0*@Tyf@r~_$Vc2S7Wk=4v%lHKZXn(0p1{^Jv&42c zivz<#kMMQY4X%z4ytXjRH&!dkZsg&3BDSF+@6pS9-*gvQE3k-7OSGOPe#4@dyU)W> zSS-r1!ub|+!E5<#jM>9yO~f1Kx6IVibEfzu>W*t}6JhFLiF zptyq~r%wZ?!a_-|gG@bXwFN(^HY5>U=w+IeHjYUb!?r#a}k1(WXEEKkS zC>n5AwCbUFMx*eW11!@T1ZO0cSpJ^jnQ*B7CR;%Q`>v@RY*8}=Url#!V9#S<(%l@! z(N$Ncer8U16Z#u#*_mfp= zCu>l{)A$2VYY(s|FeOZR{)~I&gnwNN1ZP`Kv7gEQNZh*lKzi*oPA0{JM;n;eIZQP? zC_3lfOY?_3OB$G`G~7~BY|2--p1j^9na!s2#-#m>6T<9iCve-@9A zo40>;b$-CULP0R?A9%*%rzY?#Fy`ey`@3t_KjCZZdCn;ao^s%M)Gr{^zu7w^rd(n9 zk*$8;AMice|N7uxSHn3wtYn;-AFwejx}a#SlGd}qLFrI0zjB$5V4%~lX+IVY%X{xb(xW=gU?yI zfrIf-bCA}m1j|}3?g)d#kd^DAShGV{h4_W;tzvzxy?*1)eH(YTER~$juV)u>Kww3f zuCB|L49!DFJLC5Fb$!ugy20^4$*o7mQ0d8uetEN+I}02iALSOC6#rwxo5SqFYJO(U z7jIW~PF8hsCsql~6ldV(gtET594UY*E7)7!! zOmbMJ^-$rVA$y;sWyKeTL#=$Wy2b?+^EWCV@A$}TD$v2eaG;T0g=yj~=cJZi4hf}@ z2?C0#ox-w~pTe7(IT&U&v$xd!3!9#1|QsVIJrMEcB)r!;@U@3bq??t zi&^|n*~~7sy&~`+d(fd!ZoN{`gEQkQpFW)~9J9nQLNJpfjYY6ccA-SFor72;_oS@_ zk!iCmKi*7VG_g&6o{vD|t<05kzs<_F{vsgFzh&dKR)uW>PEG$Gl#_-fmggLH3`h(0t^q77#YqmaHu4l zNnqxe5D{Qx(ep59mTQ=?fn6b@hp|&ALgbMve|t~jT6uMrMeSmVDJxp#5)%sB#cV4+ zx^muow7^ANedcZESnD1E#=Y_zRs}2VeDmmxs;mq{Q-A``RmNULA46ABORJ-c?fHIR z<}>i$s?2dud76-D$Dkp1MfYw`R^r>||Z|Y*x;) z2hFpKelf7j^>Ns!o*5uuXd+l4^>%sos-0pk3$2fc9m#RrQP!@~IHTZSBj56X;ttlh zWCaGcvVdYQ;iL&k-C~!g8Zd8&C@9SGwzzYlxm?57oQYHAh(IGRgM~xGI@T!#?3%GB z5}F0LrWiDCEP4OXl{0RJH;b5N=@sX4)dMEXw`Wghh~-o`(-8DRwdaB0t0nX1edjX_ z5qQdL*3iJJy=KV|;cf*H?qqh>@_@%8t4=%;`MBZD3xE6HUzj%@5nA3Ps^i$CdfH%p z&sJ`W=^xU$rvx=69F&_JQatxmW#vuPxQ1FuiNp@hjWhUrt}@(AHCduCbB@E8m{}JW zTV7nQ=)h=N^GbMr#N$~`9~Wd^%RIZVf}vGSgh@#4fCJm7g02*oL`Jp)4z*?q|2l$s zoOqTS9F?19z$Efl1&1V}teqj&BT|4o@DinXhPI zRtsR{YROQM6KrIgQ!(LBM*@p%f-HwiyKCD7ca|s?#}`Qg^M6cmw{zOyIK_>DN%~9! z*A533{Z)?Pm zGGyy|aELodNRaKALswP|r*-v1ruaXPHW@T(re!q6pUq1!mU`2Z{yX{5+&KwbBBTT| zm%I$l4%vLbNzlt|%f*9Nb}eXk(s5*$zTmi=t&owsXMwdu1e2miB72Ytvrsw1U7lB) z+W%Q5aJQAdj^W*FEN3WXSiqv;z{n@S=5)uA&2T~^n_R(s{uZtLu z+9~B%o|aAtX}HMsqSe*Itw_aS)ANOL28X$oW?ho?2@}_CIOe_JrO@v~SN18ZsQLEp zjGSX|_|!!fUz3f0E(>M-Tpq1fuxdl&t%YqDS{-DPxcvoM_~sRKdE_uKt3)uFPFLvg z*r6zLQo~g)V7h{CK_a_#M`!t~ukHm&S(v=VlgDJjcEQG19Oe!OCCfLsxtvK7ySk!HhLMe_ zp>#_E$78OL6vdvhAIcN>isQn9E^@6|^C;=SgQI#CA5VC{S;#T@&w-?us%(=lKAjw6 zH9x*VD&)w5&a~c*H)oZxv4k-#tPfc6^uipg7UoAmnT@BGR+lbdzUlLpv$4TNc*caD z00SjUwG9l-N`+i*bCme=3|cjn3fr5e_~+UxG?qIBY(IN-Lg~kg4c!b2oMq(_xF_FW z=5q*PWOi_1Qs4-zNoq*?^rHA^zbqrGfx^v?DpM47wuSF?_yH`eJFI%bN9R7l285qaNIsjkY~!WMO@JCF@^KSlq05 zMQz`{HD_gBt#0AHaZqHTMT=y|WEMdcN1o;c7U>cJu3&*APGg0Ge0u`cIbJkyytgKi z|CmR>0Y<*V{27N7KRC2Vl)PgUT;RYOBXF+vo$!`FI*oSr-fs-KcC$!z__KNmGz*FSHfj$S5MXGBbE2Qf)OU}q}lTMuRPyM(fUy}ffhK3i9nnx4osSTZx3z$R~+-Ol- zSHQ%b!mxJv3#q#W2biyXQ(#|{^}#*lF0=Foy?1UkyDrOaXmNIN;>$hUm{fZ=M`%gx zzpCbQe=hIy5q_du!h5XY`|`*2=QmoOU≀Oh~6uXhm@ZucEX1Bo4bq4n>7%*Fuqs z7K1m}@>VnO9Qt6u&&+e^D%baE2As@ddp>Y&KEVB80pFVgT($w6bq|=7gBcSYxXu}{ z)hdQpFI3HE3uz5xaZ(NGb}YQVwW=tHS#rC5%d(b+O>tYzi+qJmxm8Tnx7UWLvQ3=8 zs@A}gX28Ac1NW{AH7A#I1{r92Phej%fkWbe*}r>^B{P|2SFmbnChA!iB>gt=^F8j& zXTZ{$z!{goY!(pS70M`eLH1KRlTiZG*W=6<2~1_eiMj^`eJ%(lI+P?HU{U?Rdi4l% zLj&`<7NJ%FHkF2~9t~?%Cl(_Lv<;7ZSu30{IR1yvN$KZdQ!KQ_|^#g~`1Sgk-fX5#h zjV?H;EnsQAAk=JO-yBOlxU?~{EGUw3s^X0l&c$5YoLQ3%xOXmS>bSr>$$>-f z15=O?qe%yM;sMqh2gNfVcowR$y9!KSxTJT<775b{l3zZ@+7+}oTwwZbZ2D9p)MthR zw+YJ|3zn+~Gukz!KS;y}Zf40daPpkMy5a%%z6%CB5?CV*xV9v)$xL8Lx8VFUrR?%W z2l;LF3m;|8G_Je;#hfF+k97lcodZW)0#mLsvvoqmrwon5x{s>+z3r;-`4xt7n?x(Cv6nLvQNLps}Ue;juOXy8ds#`sN6d%B zd=9tIEP@&z1!Xo^No-(LSeQCLxTfy|2m$46CR7r*Rc?U1tDbjjXQMzmA zLfI(Ueddc~tyuCHIGPKXA|1GAP2gO?P@JT|nr5(g@c~A2FD@H}dZ~nE30+q9TH3RX zHC!09HZRH+ESPBBqHXLjyZiu4v;#|#1G~!xfh%oGW4cK*<|ejBrk z0n?3ztY!yTYzmmIRGEtl*2fgEhaFe^eT$jfk-g{uYup3&H52sI4cN{F{NuJ@Fknv* z;T2#KLdFcfU$80wL2Sy1_Rb0Ckxa?4KXqaCAvg&nfU;DAR{-3L)UoDh9#T=l$I&IfN zUuM>wRZJcQTq_=M&M9C`pTIdYfo+}x_u>gGUJR^k+gRRtX*C~W7Swcmd{E1U!I@2P ziu(apo05soM40OYII?fD);(aaHdtpM$XGg&**Snc*O}dN0i&P5_TmP1`G6hf0W8%D zEHMw*l_xMy|G;j&Kvvm-y~TjJM1jNFfW37=NpS&-ssl?F3zMpnzt0B_vk&?T|AM(~ z1q{l$YAtHkb~m)8EpdFAz*^;F{BLV8#W`bq?I?HgKm-UKw1V3X3j&p(xY!3RCPZ)ltFz`>XFx$Lc2f!qtlurJCqoA2pzH!VYuBdoj(kR6*3;F@XQGvF zc5);)2*wBeqUGF1~C zmofOBIi^r%t^aUINtN)`Xf7o~1s2zYb2n|68Vk;|1x$SQgn>VR;p{XQt5REQt%xGW>Igs+F##OGsS~G}4y@2Cf19wXTdxdURpn50iN^tAl@UF`KYo?$myzs>_!enf(MfmKyjZ z7;wyIxH4Peu30&g`T>^83p~pUxRwZTWOXphGstqynDx{U~gN@aJ@fT{u-3|aLYbmd&aImJ5+4|p86K8=U|czn!w;*HF5U2?Of-ag%&(mBzLGTlp*E-PsxI_0v{L_`7sJ41gHjZZ*yST zz^eRfPM$)YfimCxj00SI3@)q*h?%nXwdw;#^8;L4FEBo0hcD5LyrK%3oH`~ zxSI-Y)m~ibc5kKqZKE6qmK=fGjuY706))L0Okeedx&G#K$?eR}A58ZCy}j|(?Jshx z6Zb8=ta#@Nqt4xjvfc&kOB2`wCa~`?;7Aep#}#D2z`TJ`T!A?tfn6~rq3wd8w1eCl z=1B=E%#C);Mh9|ootd0I_*n$7DBj8usAK1!;Jo_YJ(~yjtrED-C~*33_;f3V;aJVf z*>Mjz6&PpAT^G8*ut1MdtN&rigR=q#jAC^URT8+*ZFtDCU`hp-P50d8rk5Cl4cPW5 z+>GL%pX~T1Nry>M;9IB?Q*r?Be>1kY0N(ZuJ$epI&J7&b<5(WezIEr^N|s%3FK4hk z(tEcci^V}d)72@FynI3_J%&TU|y6u@be@bW^<8E(E;j`a_?-hJk~!0_S@ z!=3Za8}b6CE?{46mn8Jz)doMNr4M+v+8G!ci7+>dJofy$uh<~8fjcyTRl|;3vY5?; zNkHDIf#c%BBkW=4Y*Y@ls`d!!oKx7j;D_U3fuOT2mJbgd^%OMobLta0d2GqV$vHhs z8aG|)IBnv1g@u!eh2yNeL*0`*6P4Qq=G#^NxYM{eBS6No;6{X%^2;6><0XErTB7Zj zRb!P{qb5BUo2Ic`Dfh$%hbCUf$x~hkq<94_@r1<3#eHR3oq@%IGYn1598w%JZ*4f(u)xbm=El;PT_sJY_*7P;CG2X< z=HwL$$oQ+^#G2Ns@Mf9;V^hmr1+OWSjy+}I;82*-{NlpG1dbOhbDjA=re2pY*>P^e zt=4RDZGJ_TI*mkw7M1kQIU64tb%tuD$7%fd5%BPskjD-M(VrGKp8TKatKr08|bFQ+N5Xl$Iax=++3VkKL**0F@v>4pn_ zKVW`7?RK+d7}vuj7IihtCw#mb6$e}S!xWT6(&ZEnaq8MtJ?fmWBf+_E(v1owM@Gh! z#k}c^J_1f0#V<6ymTdCv(p;Lu5vY~FauOrEG)@sc4!Ne1zkvfH0Rr zg1HQX;1W)aS&C2koWI>@>Ja2Bco3(|sNukD^=!gJMqWJyMt;vP1`N&IdM6g^2LGFP zV)0Cm9v|PvGj?@nM1sFGD4e^Z;^2GU>h&IDi2Ay%x40)eZ%%2*-BOuzA9zUBR&e{g;BvUp#D+bhvP!P4I=5BAbZD zr3Q|mIf^Zv>K002k(0e9J+^0J%vsEL;Pr#Wy$635JT5T56|ko_({PW$%G|}jKHM%^ z>a^WO#zSDci)2c`A>N>4j~WDX@+?yY1omUDMh+RT z9Wy%hJPtIl%NU$!lyoe*(C8vtazc5g&6|v4BD`@8@tkJ)ki?gZ8a zOpF=@O~NwrIT?ptKO_p~bu>TZxz2W+^(dRA#3J4{4aec>2e@PWg#!;v#mf?0u!v7<+T ziMcVOO-@db)8M6|(DH_+FFuY;Odm2$xFQZL6p=Kx+j`8;GCKZjnE9-QoLxrK&&*@y zGjQPPIKc9+*o0Yf(*#zPISwo?y+x4#G8YnehyVv*CCtWVPt@MSn>F)>Z_8LyJYXex!e@{3p^X;IM;}1u^ zQ<3MSTwK}RzJ!JJSWJDUyV1qFWjU{vIkV6N2X?)#7ET=jMq!nQTxlAIB{xlI5uT91 zS0dr01B<9G{`oxXIzg0g+D2Uvn_jw-S_vN=p}tNf0w?JkS(DOcxvZ<^{x`?;?}u=n;3mN-3yvt8@I<%?TvLBzl4PZu&U|#;_VjjVbm%pR-BD&*C`V${PVy2atBn%zN8-xMXzKna&ON>b4`m*v&5Dm2M6D6;8K zNaT0WYSEoDky)MRA(IY6vjR&Z`?ficM0Aci&iND)%(jQebhRTdv&zdf^Xbo)_rF<} zsZx;coSJVcEYK)4>(-Uvw1Zq88=S<}1!NfaEaFjH5OPIl>w!O6SNHE;Qo>gFwQ-x^ zj{|8}9r%*5zWRSfc&=4=a`=Y7u z*^uG_>)9{QaqcKSth#{FJ~*1|`ZOk?TeDepIDYYjTR2Nr709qCeV8WlBR|~#>wT~2 zT}({hR2#!Tc#F>v5V|yDcC%}dx!D=VtN$CtRFC$e6Bs$CZD0>}U_8F!$5E?m5BbX`G%^P<992J~(B{C= zBm!Ejp4G@9$0MNcDc!)zm%wEE;Q#~U0SCcl>Jy@N^Y34@SOB2(j>mp&oE70(-XFH@2^ z$Pqbvk;KUetDk4{|9w<<{-fUICG>KqOLf)jSAII%uC?1fHkiUBpk{DbC!_FNSwOp- z#(EzHR$~d4DV;5j7A?*iEa8R{ObQHcnzI(&WD>3b9gAmGAuM*G@#<#gtR1Yf5)JAW ztXT`%)OYk;OR(^8>Sd{D5MLrZXVPRht@&3ic3hDDx3_t>F?)i=yon6+DjS4OFlB9K zYBXxTY0N07z$k3MDC^MFTfJ9Xf>l+ck)vTLtCD3**OsLV8N?SbO!|E2kJVJJ4@;*7 z+leGFK4(^&xtrCaf=MG|b;#=GqYDr3XOyvf(QF^kvbee}y@Abvhf8IO-(PcqU;A_W=f;6Y2*P7$r|IS|_kcGA!hM(7?;UQh$@d+M`W4g=wA^uQlUd zc80x{4<+paja`$BZ5b`(t{hy%B*pB|eHgLp<&{0hDpVY4~^|=eHit>IMv1MG>X}`&GM|T_QIs9mb3{hyBH_z_%++iL&oaJ zVcR7w=?3i;GuUfRwAWo|uY1s5_kz88MtkLr6U7qjg#zu4HcC%5bxMO=rkN~?pUqW$ z%4+7DU5Uz!axa>k6 z5;g^f(>uhQS2OJ4xX7fzxLtTlU;umRgT`~3je-)55(^roI~aK^7_}GtYrSsGY;?o+ z!Quw43+}Zcv)CA%&t5t6@U#2HH3}Es9KXDz@mY7{jKz&Fnp<9WI$Pe^X%%8>^Mu)c zLrXzHd*uf9+6U})H`tAIFV{Y3H+ailqj9d=gk32~U8(Bwp-p~;g(uzL%#ohZXw&Oc z^T&6QU3ynWyU3Gpa8Mb$RwC$9i2j_F_ zU^u*|;o~j_;ey@TCzxVBu;*@Ql;~&_)L;~AJtO{Njfw>8ydy0x8yfgd%>3ijXz))Y z=rMDnzRW&xg&-wSgUcmB+c-T+V;N^=w?!u~u_rSp-<%yZ+4q2=jNK3BJ9o}jb6hqy zz1$!gVR*Iu-j+sLj~gwnHzYz^HMe@#&*m3KC7~4AE*4P3j9~ zTRu4D(<^5kv0B4HU-Lv$$qRPR4UN(gjnW#80y`S{cQA@uG)d|zn;127*)$3iLz*|-TElF)L(%=j*~$~_ zhO(CpbuV9)jcB~vE^EP*{j;%Ev`MKdSSiU&oC6r}PSV3_nPCYP34cXtrixak#*2Fk@ee0?Q@~y{U`XR@)pD&S|r~ zz-)h^dDRtW+Z)XK$}AogEx&qM{-la~7IaTFV*UC${vS`mzZ6ZTcmC5BwPkqR7q*dY zs=8g0!0ym+ZP(V@d+y$rj$q`|U=FCT^>t_x42XFc9K+Snp#SA!@z+CL6533MO+?eK zizkF9E&A6ex`A;)E?XsUr>99H}utZz1I!Uy= zI2dnRAre!-7Q@iuk)Y{#qJeiu1J8~I^#|tzJlZxcVaqINOG{u&^GJzqSjp`6_={}< zL+_rnAMsyp+(0->fDtxhBAr5d}j+?6EBJwG?oN4rUZ!>1c^m3%1*dcQM)?*)z1Be z%?ECVyGOKD{QGhJO77ijs!7J8P4WWiiULgvHyRZt#NOP>c>CxCzK>$cRui?#7ReMC zs%>D7mS8$=%^It~tg>K{;Xc+GpW=mmlAStOa(A$KGPI~pnCQ;Y;;^GFYeHMpiDs7# zt!WR|Y%SfO$RO=Iw#U zry6-@B&J_$|nV#bROS9rgQev&3#Q6k6~!Z7(D5iC{^E<^yM+ zYG^RBUpN&1?Tq4GpK0EV@uy5SWz80?Xp~4`lne`(T*l}T;HnWI6Lfo@^&Z7K)+EKW z*aEdj%Ku)g^~IL(z4@n*@LZwlMyb%ERS`#Ge7MwRGzUFsRh zC9OO2i40sB3?dv;t~xYwzTmyep~WM? zoNv?YbECyrf>lLeX+n@rEOVo<15>O6tN(=-SL24&Np}>#hH-tkmgp8P&T#hW?y2G) zheZ}Nil+v-lryTSW?k!P*;&{0=J56Gd)Hk9+A3~jbG*-1e9-7;|48;jN#Obq%6v@4 z+io55E0wI#BwAv)F%=ZON_sU2#vL9%W)i8IO z(c+-MsJxR=<~>u;3ppN*2Eheoa{qSpudeTwqCs>918c|ka`ryX4u;Q#si!2;jvr5I&iFv&4oNtIt9c!5bbfyq;$g~4D_&VEM64J`?)8Wwoj za;{h@?b6F-I{C#e$FlVKTE{ofJ=rahFy)`jg$Bt7P1X`k?F;8K8Z~~dvh^uo4d=LN z?7`Gx)u`goWGKNJzv1FCC#J=%3>Us!Jn=QGi@)U3{vW4Y8MoS%h*^|qgua%jNZhur z@wOgoQlr@9>CHzp=gg08kzd!&-v3i^+0U8c|7P#6^uPae_WGamKU9{maRqM|d#<9S zGA%Zuzr}gSBGrTz4}(|db~LD3$jEjyZSZjQXlPMYXw=tg)V;yzAj%@dz{sS*xKX{y zv!Uh82VT~WM)woxSAQ32?00J@P`JRzz#$}IlJQ^xBQqzjj8}nxV3SLuke=HVhrr|( z&uTXjt|iNbOw(JrY%By6nb-`|&ct{gIyrT9+`fMmXD_j2iL5QzR}#B3sI7;STV5le zz>$%qYq7>8r_fnSnOx!;YZf_tWOD6UrL(1OsblidZi%Gm4GkfyLMEH#-Z)Wt`na#O zUFoY249O)H(FQJsO>5)b?jC^=fXUmUA-3EN?HuveTh*5O?Cz+;@7!np>St=SD==-#Z zTQj9qG|Z&OgPF@@$pWP=!#x+6cy(+xiHYRxJm@Byvt#1)*qSd3uN%F%@;N-wUCEP0 zu;25Tb4tJENeO;O4>cEoV6|0mlQ? zLOurfnMG|H8W@;87c@9?2Nqm(<_icBbd!*DnW#H4L7`K{C}F~J6PF3%jI7#;1~0uG zV-IqwulVzcTX##zBhLW+)WD|oNuMGyTZ);L{}ZpBYgk-rllTO%=%~Mfta?o4bz;SJZO<^z0<_V zrlt_craoI{wWnH;f*`9xlY^75@e&WC6$*_kMI5J=2z+8unIKYhLe7Rks?B(ZM3IM5 zYR@EZ-JKza&ne4`Bm}a~<@<1eS=lDwLe2fP-!D`Nt$e=9O?XC-CtH)tB4@U#Cc%wt z0zM5*%pxHTjeL5s9G6A&(u-O*=TC2R==R7|)5qL8 z7d|vaXj|DWR@fH6ygu6M(CfvqwHlXBOsIP)ZI$qcX-?|PfWBul4)|&P>)9aSUh(XR zY?DcH6KBwji=0|>3oe~hU(qwkwQ;*i@Fd-3FDAMxHguG@D=c>rYW>tCGs#_Kg2bZ3 ziaR`vy(DL;EaJA<5%c)GimXEb1FO>&#SS^|69(+cMH;sEFN&)+OmyO3yDPK$nP!9F z)CrC%S__%geI5&PYut9-q|~@Y^`NT0A@jR?X>B`V6@^m;=5btT6A?6Z(7dG-^Wa!zod1HTVSLGhrv1}cq|~hh6N1+yz25P~s!dRl zziC37(4r642HO_Pz7c2^F=E`K(V-}m6_ae76L>ND#Y?em7IDS~SzT!}9*Hdvn5^LR z&#+_R1VzCPhZb?Yjm&P_PO-+_5at)s=uA1`z#w^o(d3501kR8LlZ6&EFbgG^u}7%Z zzb|Yslnvm>Fkoi)OFhQH(&WSG)5@g0p@GwHVz-fj$(n;A%UG2jH1TUJWjdPEwQWMu zA)$A<`D&L=g?b!WB)&$0)$mhe-1)D*VjGI2Di2KTyRy>q+|J1zAEzvr=}2g)b`msk zPf$9jwOaMYR=&*>KRPKCEMyU$;3&S!`j~UD0#^ZU(IhW z$G6UTB(n8tu#wkBW=S1Kb}NTNLMJ}3$bMPC;%9J(UFpyj_6rLo8#g$}y)ih>tKrLb z`9L!>i@`s6h83q6c_cgpv;rG9o2syKH8L~!alqxu>PJ6i-E9UX3z+vWIdNY4y~F0k7uTd$=-j+gs>r|X0+SG% zVOQXqWbsag!+Pr%+|qfXD0D>GRk(zqEog!he>ultxi<`K1}%y_O*an9eo^2I=5b^Y zH8{W^wt>az#X}~x0ES@ogexo>2~5=+jta?bV6;|I7LeR=fH~zsk9|crBY(!#fU}Pq zlw6eAge>!&`_i`kTgrKW|7gHw=6OP1Tn-FOfZ>$hvh#ViM?fzU%|r3 zv!EvI@A}--Jf}JUj{a?ew&gerE7CdZa`|itRDbmEzFilOOdt*S{ri`Y-R;OpC&T;}yMsL^@ zZ#Ze*Xyol-;JhQPm(pzVgi%4kN%4iF!j}flDGvM$PD)=G#1xu5I~p1)!uihlFSlS6 zzu~}X)6gKpARxwYQNV%4V3Ld{Ilm^xZ3)wcf z@N=BtDLKIY!GUkW0alwvO_gS;w{9N;m^B3$1Fqf=X>j1k^lq#5h&dYC_O`d{$<-UV z!rc-I-4fBMz$9VId2@&^x?O&b@T!0LP8RVg-$y1qbE6Fo->55c}|$H|CJ| zmj?M7M!py)#RmuEL=F}CIPqU-)U9C@^Eo75!l><`sIs9+Ou$KZii6^VL*g+_#uW$U z91ffHFtDW@5Zd7=ucKs^a!>b0Bioay%`3%kH5>?ep|b5o%C;k+aj8dJw_Y#{4c_qc zKxD^^TN7F2o?OtfTkvd4iuFxyn~O>bF%QHu9z5v$$NA>Z0lt!@X%hEd|43r5Vl!ZS zy#I%z%8X`RfkxRDCdt}EYB^47Cfy=Dhji~;R5ClP^r};KW|Qs>M@|Wba84$-xlIx` z7&U(!6w+9rr*KH}$m|~szGe>&FbX(a>^a5a;=+H&M?&F{dCVb|70$XbO;#J2R8|}| zt#QnXJ7nJQ%zVRP2^L?)4NTTCPPvZ`$!%cP7ICu7In24@@S{{_>jcLOJqJ~8s3}cp zw(N0W-O$35;=ujnpi)q?Lc#%7hO7TN1=N&|20OThg+y?@J!`sEpkwOt_PqxJSH&8+ zF!x_Ls%0m=MYl=GqG|7y!#ZIPhzzBrh!a8}fKE}_tr+jr1Rq{(xW6T64A?iU8* z4Tm^yFzL-XAg93OaP1Ih0i*UEMp2PNj-S%lKRn}XS;o=hsA!X@$iXNYb4YFq!~L5L zJX;R1%ZnabVL+P1Z$T~?LxTEnfG zP9_EmqCZ@SnIb&l!$ElqC)EXqZ62uK`^hMqkjj5UVtS71j+-}jl&1b$^-n3IgHxQt zNyYG>T+id!F2^>fg9a}5r$*&K{! zagJmN5(;wKc`jaIB@tJ;+(=@Kkb0H03&Nbqs$ox{(^%% z0te;pIPljTM+^A+t|toL3>Ch0mV@^M1NROU(=P}4 z9vMYjNQGqx-1u0N}L+}-FVE=aNvF&14mE8mIHAt zM;thRJmkn|Xt?p1-KAm68WwMv29AaUD+C?+56olBao8y4ZzOS$rD4`T#b7R$thU?} zetAk2X%XQOljA~#;}y!1xNs7d$>2Y>3Tb8MYTAx;(tm_si;TCq!e|2(ZoxfK~D z4=(j3=>L0ed-0(1jYEa5jBi_*6iP(Cyl~8YdeFU-M{y35u?e$>qv2~TgRlq_=Nk=WQwD~f@FP>#G2UTdS+b60eani_f|pHP0gO`(B`068 zOlLKDJ}>J4%bEih->kf$!OOAP<=G?NH;D%h#4+*l9F#F(o@eA_{Kb(|`RG=SR_4^u zlS$oICAHM+9<}P8d&CmWRU^!}h*fSwy2^)+=mY8#EcE3ym{ePutZjnWKGMm~pgZaNCj zI8^w7p}@z7^Ff2en~P#I92BQC39UHdFX^aS(D+G|fkB`_#rgnO37f>ehucpvoPDc# zzsgHvemF~wm$$crcX^lL!c83gpEy?=-E?4J7Z+xGaMtV1ftj1uiG695&2f-7X!?0E zFIx06bE=F{`k1azq7eHvKMeXJxP@E9Y|&XNwuk zws)NEIdXe0IwntDATfPo*sPC|>OGNlY!o^TaZ;oVXwELfwdy`r?QZ=4W-N_7)hA!fP{_bE`+!?cfrCgvT>NyFX|_$W0gqJz znso~tzvecGO=&XDIjn1PP zA2eHBUUc!3`I^3^)~k!_E}PHapS5ONh6;}idw`GFj@!Er9{V}T=~qz&dY(Dp_RY`=| z?#Y^W_&4w@Q{~!1EKcFL7&3fgr=X2i=jQ=>;KkIVF>peL%#kD~`F!c^| zN&{cQZ`J_jc}-Jz9&mCeJm>1!y&&rm*M-ig-+T7ARJnL^eq!%>GADFgK`ijOIKlPF(=iVlf zhz8aV4K6dCgnt}UHYfvK3t(_i=mP^khr0wrlY`?gAsHsqnyZp3O&WKaG!huaIu0=W zsGFA=V|X<2ob_`Z!}%-jR-JFZ#k}Xsr{`t?wiiBk8?lEsY+CG4d?@atymiBrULFxA zT^1)+2L~q!7Po}GX=x1HEVr~Bnb=RfXZyiudFBAio0lw04ru(7Hr}X`a{(LL_@j`AOAB+A`{%N9QHBe(MQpUmcv%rs>7LzO>LQi0$eE6`3gP(yuR+p!oenE zQ^D}Ck(rZU#b?Ha9UqgA3u*bR2zcoDl*=$>O^4v&&vE;whX1&_AhD^}dmC460Rv+b z%c)Ht!x{`8GIFOi>CI5uq8MZ(TzTx}i494IxI|7*UtTbe$+b&VyTl;r-?g5Zo6|0= zFr9r(=lnsBfEzkKmp?2#(mCzW&c){Z@%D8x&e!-oH@TmZHcVaj<^297Q#=-NE|vb| zd{Ra4g$E0_%BclMrQ{MSho|+P@q?n@ZD5t49$@+l6;%@aF z4M}Z0O%+Z(8dC(Ed(@@`G_~tKt4QKtWV6s|I=~>j=EPw+2O+~dDjmHW+2^z<($s`NjW{!Z}tC@H`v=_Drr)e-Su`^A6>&c-p;Q-?qxr8SowzC9Nn(h1& z5B6p@%}`+FoMHXt!b_usM!~FImrl8OE}EGp5bPtlevjFug2y%;Dvz3#y6TsARB;KJ4aDS!3bC39vdE)@`%qsXrGLPMoX#pL4A>1BU5stKwrF>n^w zT(V-z?x0zM?>S{lZX~c-dt5x+!fm$WK_e4OHAA~i$ppnNRfb9?51y_Lmp&P>1G>hA zg3`Pjk4`b!bW)>mak$rEp6-N`+)6tZFiEDYNMMmPkw|Q5Kkbs(#G_Q;(=4$sp>=}b zjVZiLdsH$Snr>*USiqpDlBp&nt9L?~Uu>Ge)ApAOyc3(Aa`gH=f0zo=;+zL>6^waXw#E>2bHiF68xQ(Sv{Y)R|tB|I=+67MXk0aX(Yx+o^oE zM;w%d|M?kxWEJr}w?m2BKjUOev(2o9ZBkBtna{+6=RIg_S1sAVEM(o2mc+K7OJhNk z;;)3ni@YTbN<#KU3C;p;Uv?a5P~H0Hbfc#H>Non0vB!fFno`YUPa8a~ieT#yWz)4h zead8Wi}$LHOIws~+vGG^YGx#)+?I<_Z09xF!SI<^jUy>eM9YEk+?-0c#?~ih6Bs9@ zEvjhZF=P;QW?wotX=kR{r3vzTeI3`HkSTw)QQhw6rfa{HRW@F&+tnsK>CJJrq~8ax zr?fgmKYSIMu=^m7h6jtK027bU{kaNNddk>Gzu0Tp(c2iJ3o4n(?sXjnTZfhE0DJDP6-N2!2|u$}^|ZvoSc z151wT*FEIcEKuNT-f>hnX&H+>$05F{8(eks61$@xERtMPz-Fy=aXME@0!NbppU5YH zPQ?=qf~6M@{P?24A?NaM0fR`z+1EVDNAJ%~W7&3)XR2DkdY#l`J{cyyhfBl_g(d_h z@@WUBDlBN^f8)SkZNVU}64<%QrHPv_!GTNaz@8I6$NGYHo#C*1&?I!{07v|cgXvih znDy@-=;waWsN!VU7W;-t=B0(Jh|fY^H>WBs{mkRsG7V?!c5Jfjk8kQ;AGn zj~9*X6D*FZcr}y6%G~c4`9(1SjgdOz$86~L5f#IS7@TZVV$zX-mD2u>=PHX zDNHK#TzF1D*!8K%ie=ZXd_Tax>hF!?HP5W?|NNl5>eIoiEV+l`qm?%7-?B!l{&)7| zzhXaD|G#=t;Zs5b_a7HwnUvKW_Ksb)Gn1swHMDHy+t{+qK}jUdutkw?Vu!bg6Su0t z0mfY&%ses;%xW4g;yewE%sdRt-X9JNWh6BpyCQsIo$!me4jot3oqj$%*1c?7T$*a8 z9q!*}B;?5Dq@yTw+2BaK*T$BgtP6QN!khyy2ekioJ*?#SRNihu*ZsN} zsmn8ueQEUHb^q_Zt^?7!8+`q4-4E)2z^2M^kjKx8N!lQhzscgrg)_6J`S&D=%s+8R z!7q{9_QgZzNhcinrX@6~UeMrqCGDd4X@RoR0|j=0!;SI+8`)K7FmS#$cI2NSz$WuR zfv0-IVF8`T5|F4mdG(a-q%H#qX}^}L$ww};K>p97_}%) z;@>aFqR7U`Vf$mD!1M#BWOF*WT|QI_EK0Z_SM`Y1Ud4pHywg0-AX(k;qI9ownR(ib z8h^EN!Sb+q+)jVj&OE(gC5!8ZR%5RK1}2dO23-@+&Ixcxv#@MP5|TJ1pVW{gJ!RJw zvjvX4IR;Hi7Y>TeO>C4sb~#FRg6p1^4F;M`2iSa_94F@n{A2!mBY?e2MOo%?!EueX z+lm*}-YkO25y(|6+dQuk<8_W51T)Kj8AASy!d8SzY6; zh+m>Wg|5gUJ^o1#1?D9bEL*BAw~+tNIleCo_4U8hVNr zyi!SE4Ox-e)65#xpd;m4uqKH!?%mIM{TCftg*L2T`QV}Lx8ri}5?_4!*Ldkje*LRN zIggbBz3-Mj;s2X+=*2X>e|OgZd!#S)a{ade-XymK_BRPaT)CXjyrl9N1^5n%I6V~k zlpruGQJ}6%YLep|GY8IT3_jN~Ip!T;Gh4v2hm|?v0NbYne0&eZ{2X;XK8i3bWDnwD z`LckUCz&D4!E08A{xx53u~m274(m@#Sy;8fXU`LpBQ9(QGHQ;o-#1_|)^jsvdd%$Q zz^(U?jgP@hE15~Cq544}lhnJfQvx`Jk6mS3vp++Dqpsmm!2@Qy21&-HsSyu&`uVw} zScNYQ-4s190h5iK> zu{_CFJ)5r@|4B&Skgesx%pNu_HbtQ&iCTGzB2Fp{{ES>qijpo*X7VI{3FzVQI>5Jz zQ{WJTz=lJGEC+dKJrH1P12b0wi`;^e6z_WsuM%Ip)sbITIxFMu z5e?n9XYWlrW85`?)ht1J>ykSPCPPMW*$SD zPXqI_%Pfl&xJnKj3-RWtQea|H=n!$(w&sM?(!hIl>6ZTK(wXV{WgnOx6;++-Wk0(_ z>hf8`6-$_`64$Yv;4izo>sCFhpKoOqp$dz~Ka* z!-A4&%8V)sj0xw%ix2Q;H9X35<;ia7sY@_2&SU@6AR6_6=XQe4iA;fIN_+MhOD&?`(cJto#r#<34TbhTS(?#c{@mZkUO8S- zjcwXxR*zRgQ*$`JED)HX;39s2=~-~1rNSPw29_oVewBwJJc$Bj2mRSaMbZ?L9UH|0 z9Ft8I1&b27-#G~VQ+X&Tu}~}3SI8@o&%}YD;DO*X2L3jNWwSg&x2Bx*J1w=jqtoQT zJtZ~0R)ci?1^q=~+ukfOOJkmAASgH|k^9vX&o8XZ1`Q%=iatEXR&nbMERtE%7O&wpQ8HstLp!+-k&t*%XAf`KHy|Y`()DLI-<+1G|p{`y5Z9TMRKp4gv)U zY;6skNeN{d{!ZEop~i>*CH!MbugQLf5o^VGr0=r}TYTB&*E8{w;yu zgTauCnMvdU1J3~_w>2y}3p_&j4U;Ft9XDgIYiLs2%(|*a?Ti4c(E(m3mEF&-xHoUv zQ)e={hk@(H1ir&x`DS`e`EntD{uMKAiP*Jj=KO+|)x|;&-^T^^3iW?hO-?XAHzje3 z4|7i%|A7SNHZ|^b4Z>!M!b)d&CLLhXX<(o9Oy`7ycz~kdsRcYHi2_E7g69}I&Nqma zHS!1~@@z>E;!tFt;=n#3p?toB&>IEjHWn_u1O~NsFEj1Ce(hT}J=roXr|9UBdlxKx z=5618DVKF!HupxIl(gn5dj+No|5`YrF8B(yu@pRD(OIB7A%XkF0nV5MeRmAm<{a>j zXvm3nVE0(W*3^(w&cH9x$f{@ApzNs8deMEdqILL2-YA7B3mbwLIPm)HwE30b`8|yP zzPIwngZw|$r)l=`XU~jxoguWb#QV8_;Ya;=#c&@*HwD${-F63jQVwvubKo&J;AEF5 zX2y6uFIVIg1E<~sJ7q_gBM(H*JXag@q*MBt!~paYQ97%~jx#I4FA~ zA@I=R$<~quD-^9yYhPT_@3B`j=Jx@iMGs{P&gIAdRnvf zlKnn@=kz5S*JlJiVDE6?Ki0rJAwhuas<_Nbo|;UNMG1D+A9c<+h+JWMvqw^Z%|KVk zQT&{O;FksL9E}1;7&c@r6xfj~^hu#3;=wyJ1@;{ZMT;`nC+nR2mzm(E>~qOTcj@%I z9BS##i|23Z=?grz(BlEO-oh_j3Jh)yoP3Jnw+^iUO5E18h$m_&+glE=rhl{b0nc zhXNK3Tmg;wGnoAO9A{rWr1g5;?1Pme8ICNR3)$~XTkqc}(y@_UK~dmZXK|7nTakhR z6SJYz`+0n6(nkdo>!zKoxn`obK3G~mS?HBsz+!{6^5a!Ww?9w8AIDMY0<8wpp%=|IP5>WKzQ0B9g|2NH^+61i~_r6@;s_KpYf2# zr@@_rQDBOK2)`mv$AR`;PC7bEweN7$@;7^3=#{FSR+V0RWc!PfMIWzme=*tiy-)9P z$`*Gfr3Y+!52g4RnToC--qav+DP7>m0Rg{*;-4Rwba1muB*=v;oH`$nnAO0R(UhYq z$r_eW#bm+%>a&<#qYj&+%r+azrUQ3(e~nIjr|9I_)Vu5K*F?J-zx&3%XTu$Q!Ws7L zp8fBD_4LY_s+aS9!i|!{CMEVrvTKGj7cJnfaN=ClkbHTeY{0|3H4a?=)-{MtQ{wv7 zAfnXB!?oekZp(H0vBCk0B032SDGCBkcUkUTXYo*2FYm~|;irI`9e+mx$FbDl9{-iOnWk4|H1rJgL|ize4FOuQ?S_cLWhyk0XB|)KD!5=TE(1465Nk8@RX!* z%u--CNnqTF(zpeErYZxC}l;Ddr!k{(>$w7+eE7ONL_N2cB<3hpTX-N$!qj- z%H;*YB8rlFo9=1)o_){#>FML=b$RzM?%wO(P`&@RF~`0Kr*E(Py7k+ixeq-nioPCV z+2PPurNDhoAur$|mjR>54K3liM9q>lB2EjR*gw3aZ_95WD{$(-Ovi&yOAlPCKJf1; z{{j{R2KG&hIGH54tS+Y(dwc49VdtLJJ)^vvW7do97k&Oz-revdaZ+)TN^;-+>x>=` zWMY{awGME;NZ^Y}W4?8cPwqfFi=&WQ0^>Aivj)@j4<`Z+u`5Juc`4j-luJz5D5W4- zt6w7a>ZQ9~Mjf|S2A`bdm^m>`c}icF-KxT7otYNpAD-RZ+^!$LfBK)KV|UCq$NS7S z|C;HrP?7Dtgmw~#tkx8_z(r2(D!mJ*tn3z)ninQvxb+Y_uarkiBg@L;oW_fdN?%RM z6kZ#@FGKa=gqDR19h;pPnOxfRX4;g#oX0rz^#MkqTE3YYi3b~4c`TS_c8PR~EVe5% zu;~c=$9RbOw6Idhg$!OeluldPwuS}mY*LNOS>NJDCYC}e5RS-PG42yvJdyE7GZn6FCmTo z$_osdxb==qPz-YEEjS$HtH~P5BOg$@kWGHav{3O@AEy>pj|?L&_2kl~Q~FzGOgzG7 zWgx;OnbKl(BGy4{=J5%g3!F}K+U69nEBfj*HuHIFEOeHi_~f8df7y$K1~xN}2M3sw zodlvX79=kCcaU|SN)wrMyd z(6d3|U^&x^gf0bXj!8bo7kCUE@`OA#I5^Euo4_m|;INRH)9S+p7P&?fWlnDHErngO z(J@YLic8Lj_tsufIqWW<$D+MPdOwpyZ^MTRi!U|m9lY|u$K<2%TA#qs4ToB^YaC{_ zIpv;s7~qwuoz$teXa!@ahF|GHHt!dKox*|z3a*lK792XMwqn7eP7$$!3(1=8JW6d7 zoP{`-#HKD#;+5?9z;H}4fx)Rmp{Zjcv%*9N!vhTeUM*;3m@>6;X6cHJk_TA>7OLG! zpMSLS)2Vrp;Zie~oBk7RW)q6|-oPU8q^F5lTO@#yFXn{70p_zO6ArL($4odVJn_fD z6Pi6&)IH>lG(H?)$yuQ;AX4WNbW*wc#ex>0hC4=`#+OeNFp5O(+Stfr#uB|%bD2yw zm-3VkXFZhbgM)mGE-onUv((hR&EI5TtMPVo|M4%ETdoC(E!=jmX!S#0xiXGN9LiHN z^138y3U0C4d`avS)`^+WA~HYaQ>WUjl%zIEnGY}AG@ZH>xnhzSELGx~r%dF~*xeI& zS}cL%lgsDUj*UlzYNsf5@Nw%bXqGh=S7P3v~0T27%D!>n}C88uQ_Qb)oZ zS+4C>o6RO%a%*`W-w)4&JTCtxu&D5CXi%T=h|@}dg?ruxMzI-50?Q>@IDH&@(@a#D z`4c+$j5e@1?r;?7&tOwwC^Xx0bCKAj9c`Ru4BKtCDB1)loVcsO(c5z;y`cHDhtVNL zg(I5Ts_b499!r_5Wqqrxb^VrSh}M#Y+=bf3^M$!~u40;cVWEeqe85=_g9FW022A`q z5l(VWgZE9NrXvQDE5t0ThqjaJPR(gN(vaZskb=sdkZ+r9oW!X zHkBzqWV5qE+Xn{qfCirlA|ij{*k*C}xYaIVHtng|wPnN^OW}^vLOiB(gnNK*xYbcVaP^Hi& z*K<&8n}v(9+lOu6b}-u5W;x22aPSts_%QoC2RlD&NneU6M@M(Cz>{l|lV_J|Ye}6D zd+M=dCr|2$G^JevFTO=8X}L9MEpzUXm7CZuw&S75l?z8Tg&L0q-+3fbscC|cXx>@S>}>W}&}v3c#;GHlzMaiXn{)BZ zToX>|8zn+!i;tW-TeNxW+^7EL!d3^EnG_g!cpTVz1YG#UCN!>mn#iDYW2r0KhIPDm zCQ0QzU=Z;sJfqIx#NXf%D#B36VgKMDpI`REe|Pj2FuNW3F#8e5alXZZd>LP^u7C37 zp(X#39mnpr%;o;XnL6b~6aSh6tiqcnw7$`FU1qV=$tzOn>4KuE=jLT=sa|MO__Tmo z!{CroYEQQWpWs0chHjCG7DpVPEA4r3@M&N3iDV@gM>a#3N4?7@xM-?5b_KaOv4h4# z<{j9}zJQG>SxR`WL4lgtM7K%jp8DiG`aaVzvuOFolX*NT8@As5@zQM1N}i0efDR`c z2ZpJf4J_M?@3U|yFyB0VKy%T-#&tm(B$r=s)bKH!ubPs;>YLppSCP>8Mv5h`wj)7~ zZ(@(*j)is-mhBdwxO(|M8^}DiIR5ZKqTkdFDji&hI(`4O-B2ix(ssSRsK?IOWFf!J zQ%#ji>1o9=k1rOsX?>PvhDv-hPBtg{QDf{Yhd_(+6j1kHW41o*aQ>29}Rp zY||YVIEi;UxX4X2=up>L$e9-1#Mc&}Rkrdc6I;U5^aa&3Z(Qh1HE!-r^HW~I?NPMc z>h;TXi<`?Q@~#S&TEL(^_h1i4Li)bBP7YiG28}ESv>3Sym`$fWXwxxN656|~Kx5j3 zrJQRXFzIe!R&Kg*fVurR?=*=PIVHzqDNh>h@|Lu?N>gcysUkbPtSrwW_d^d zL~KmVRhqKk$PT6ny=9>a)th-bPj3n?TobfRNl{_>39Zk}2l@8=bLepV!Og#H0!t)& zVRx6Ai@>q9MH(fNoVUt$o-X}!MDxUi4sVTzRni8I{5}U7850(8_A@XmicP+zD)6ZK z;$7qN>nc*O^UAYLw+d^-Pk#QUu4u)!jXilK%}v*97+9JW8MrnyGqHS7W90L2WD-(f zoM~;KJn_T<)`$j?es*7A2`CH#S%tQ&dQh_mfo- z`)29ilccyz;%0(-*XET>k(-=MkKmYmmyFZW8pMPN5y8N7P7u$*1`|UjwvPDkJXa2Xd+1sIsiMPOEd7WFa z$LW|K+#F5vp8}eH>$%+D%frefS~RbRW6`QrZP&^RN0B&K?UtZhN+J?K}uNDJ7pjCj3`LLrqCTLxDM0fn(tUX1`)a8HGmA4Ps&mjN%5} zRvWy<4LO<&#G6D|y#s>we_`=gV5=_RY*moG{J5g{BC~{mxAz5`gajs)1&q!D%o`Y6 z9A>C3cT`^~9(R34%esxhn-o2b4pjXtmfUSv#Q%X&bpx}}0|wCv%%uVB6%Fj#9~ig_ z{)Nn)#IP-}ZR)kQA5YnCP2joy#N@|P=931z$4ePc9^j~C2wQ3@?=?M0>>88ALf-ab zizfjcQl3d44)A?`Ffsag>HFm!FOPIM9bxqgU~w^EaaLflPGHv0WS*uWBA;Y@D=G4C ziD-yQmtca`GGz@x2PT^d?8leJtj%ECbeMCd0_)lx3}Ob%bCwHgFJM+}U=-cKD78Rs z^A5e7A_jSfIOz+l%L<&T6PVN**j)~|wkU8;WZ+zOsA_|`R34MG(d9zh(%vVNgTFS5 zemg8_rCjv0SbFM5|0iY4W*^umeBhY&fx}IKk?VryiyxXs;cZ+AZEo(SqB9sd8B3NK z@CFnxO8mR1Ej@*?+=F-TBth1r90BQkPnS=WJ~^@dMxD0er0$5>T`M|@71%XqPI9?0 zd*Tjey`9X4mdr+$%!UQb#+-USf>uk7ndT*0+1{MYZop(yz`k??i`WK6&jhyl4O}UT z)>cB@Rtwm_Hrg&OWSqZ=folN+(}j7=7v>2~Xk~3+-WP?7L2~#ssivd9t`UusAue z9Ex1_bvpCc@J@3D<{$1%zrXmOi&VP!)8A#Xp-clyiUHTI1>DybaBnK+nmK_*%z;T< zEP-3Wmw?^dmkixp3=9Vt*c6z{FT}M^;HVa0t7>RcPGGc@Sdep!Y4O9@yB&o$ zQWx+qu&am+X_{tKC1k%nJl(Q6E-gq&;R1tz10(AKyVo-rcnyjzADeO?VER78I9Gx< zU<1qdz$nuVOcy8cEHU#b^5DHtQ2TJBYR>}R$S3HD`Bd zEDg+BT5@6CLUZ=rGbio2`EPcj)H0_7%qgdsQ?=$8MAjP{FrPRnl6T5KB#=pp!J$=v zRZW3);R5d07x?}y;9GxUZlpqE^97zO3LJg{y*d+^Ul=Y*x-^fqfkE*V18V|qJdH40Rhec377KFRqBIPYPM7!YCNP@a8Fxs{5f!Td4 zlkf%h15NR62iSx!FvXSB=Img}nZT&YaA5xyM(GLc^EU`eCTN!KULsk^S9W@qjO4*D z3G3b{b ze8!z_pjE>Wz-WI!;+5mJGZUD%KiA}#W*f4V;qy|4)-?h>e>ArpJfVAl(Zq^FIz{uc zL*0|-2cEjut+{#f2?KwG_$fi3q$=x!UkVP&aqeIZ+EMc1pyC_Wq7STbw|2NLVC`GY zsy^rRE}l7uDp^ubF}pgjPiEj)Z_M?hi7Tyu)$0-076UHj6Kh4q7_6_m^=w#SczYfX z1H*-*yPtM5GqG?ou41%IVCP@JyvB(=&$zExEY>bst>oFiDmJU>nQM-5afz<~c&tH_ zQCEvWfI+4;;P~DM_R0kI(gypFmW*`@6E;V1_7+J`UD<#0Fw-v!M$LfCJr7Ux&0&(V zWz2ox7P`1ia)a-c1-wDwmjc7@G`gO%2x|!Ol-+L-dZ5T*sOD0;6U7U2X;{g&b$`$GZReb{#z82yMgh=N#-Y8 zkAF^;)=(%GzOc8`>f&9+Vum^UYZ>Z7%{X)~F!`JDUPzdAQeXn(&P26`zRxG{?2Wx7 zU3Q6SQQf0gmlpf*&;G@3k-H=Az~v~bLs18IZ0otQ^$m+#?G?ATOI!G#4TToV{M3rv_47=5;eD7|9fnQ))cfT5yS)cAuG z+l7#RhUiO)>^c`>7ynXwxcR|@VoA|eN9x{Q_qucLld=6Q$KKr^kIiJ-Z9082;AqN&roRxWvL9+$Z@QQGz$ zTaAV3Ne8~`SJ?lDKB>A7PV96EWSen3XT)-%ypkFNaVaH}M?}GMQyBHl6 zIC%D^O>1IzJiuJ1z*)3_fx}@_Q=R8~OQ}m|n2&{Wm0iu^u{z)8yLb2Tb194LO4N!3 z3%s=Jt7{M4*m%*{CA4)y0^igChL#6xOVYTOHn7iH`6xhvW1@qh+!a}q?XvEw#dlYK z(ZBxK!=AB+fp_8>W;RJK-3{KG>f|r)`x@i_R&rtZtv!q{oIb*Z%gLX!AA5e=lp=cGzC_*gsAPlye!Y& z>=5B}{PSw!0`{~20yrmJ=;PSHVElkxWHSr@0oJ+;?EVdm0SbZ34Zesho+)E@{-WsW zN6yB{OgC5?UJC3pjy66~s#a6?R@(Bj%S<(;QoNMsd(<0b%{p+y_Jg@V4-ktv^ znaERR`u~RAxAgj_^$!lVX&;LAI+XYSDZ{~~9b%19T{3@)E1PJqVk!e>WG@JymaFym#!!6kt^-^nS5p#C{I#xTlVJ7$}b-N z6Xq`S`&#(RcYF!%; z7*b%s=+q)Bz%xPMiTe4KDQA^ZD-0B!<%9Fs92%8YEMpN7D)93#P;6!5oEof>(Ua8l z__St0T88i`k;M~q&2|`G^|_=xW4T8+-_^nrBe#=%s?KRIXS_?^-Y>~47qeqi!O5=n z%a7Cf=UJD`xjfOi{d7s=)lMCgh=MA?#v-Ax@EZYp1V8!xb7)HIzNwpa)~xPM1-JO| z?|Hu>RxMP}>z>}kraJk^G%L1%ghmCel`8!~aw|0^S(oSotK-4ywJVwyFm2v+NORrUe{;MFo3^N}SkS~jXN5z9!X(31sl{Gd$&H+1 zCk~6qPJ3V|<*4G};BmrcwN9gm%;HOinzEO5T6v5LoCVz2BvV%MbbaX-G(PVld^Y)9 ziLQ`$LRX8Bcg5?q<`3I0GxyuSJJ8%``Ru_VZp#(Y844de9KI;Di3Dv?aFZ>1!J*Xh z;2?*kT%ns(uG!Kwsa&%|X^Z_=ZdAJxyr_b+OJUxGhn>Mn7ZcY{R+;J0$X>Ex$;=eL zmzP9lG;K)e448DXnU%+cHd!zym%fPAt09b6-dK@|%sEcRbY0 zCY3$iB$=jgadE$$$jpi)SBae#XN@%tCkypmpL5y#b{?~yWU|k_7GqqjpkkT_(Rnw$Po;zPF3rw6OX?i8aD{Gl#riay6CXF&JS0N1*Pqj(vZwy@b zdht{wFo-FA{3;@>q{zC}rbOV$6jeP52L_gwDGwskA0K;pbI0-~hBZRm54>)z-_pgk zns=#?%&Oztx;8c)H|bzxmV70!MAF>ofkXS1ZD$f1SxXWe*=5Z*nplr18CPs(mHI4Z zq%(g?k9W}miIhaQh=OLxXqJu4{J+cGPubb2ylWQh7TvkyUxmp;OW~PT^}>Gs)3#rh zyeQkUU|*GDl|mA`(uVJr7u)jMLIRcgCVb&gR1k7vlU?vbW!85qscC}N3J*2-&puCG zB4m>ya6~BR!v#lSIYmx&hKBq<-A9BfLYMU2SS`7%;Sg)yq6W3K2}}|P9N6?WH1TXu zVD^)o)U)N1D94jrmYk2$(JBGTb6EV`3SV-?98;aj<+6ZDPVQ@x6fmNepk@ObDzLW{gQnwOZ*yc@;a`FL@S!DT2LC*q#ERJ6C6;_; z>s;Z$HYx3qrIX-_@)I6b2b;bA^m|@lQ;>~0Fq!$noM%E07@{p1rU-I4FeoG(*lE+i zJ|Q9e)|~>b{Wgk1Z!?c68!+;Tjdhk+YgTC6MmiRe>XP5-Ob!zNjE-#EydC%`oG(v)S!Q#E}`vrhbT{x+rN znfCpkXTn)5|1`2oK5%^E;neG>lfaqR!6LGQ?bK=?2L>(yhp^a}?Yb@x7?T1I@X9&z ze&5MIU3Fob@uY7DR8@o-C+=hr|B%3Lzd@lQFXDh1^8^M41qO!Nx7u7!7#P?T4s57* zT@#l8-Gt+8n^K$I zT@G)(Sioiyqur;V#7sOfy0j=Xx|t+i)G9ST>zsB-%w@j4|%f&rWWi3HSJN5}GCNEMPH`J;?4Ra9`-( z33E2Ht(tz$8=C*NHE{gaeIl5nkRTn<&r$c_Ae)(jj@(Qqfyxuj(h`R4j4IqSnKpDW z@+@FfTa(7j#ITy@*8ycm0Vjc`2`-(jLYT`jqNdmmQ=vqb}g&_M&vC%;%cr#uQ>qTs6H zmpCPj;SmG#0Y?P|!LEP_2`mzB68%bso!3;7rJ7!(aF-}FD@0uAw0z?2>6OGK=5SDC zyTB2yln2dHCc7^7usId0%CfwYTDWU^UbEDl33ts8OzgeKyI$#+$=olS%$I-eeb1id z(#Rk1??9XBr336S22GK>88pJh{=QYLpchu+zHaRm)PDixRJzXaAq~5K>!oC!$S3GD?H

$=Bfx=KS+`1lXxP7M=iS=ibncrhYA>(c^8 zu?r4tCJa@)Vs~GNnfzkcWpKK`s0`s?R0zuQlVVjslYWbg(2@rYoD&*YR8>BPdJ8m5oN#C}E|C#xWnj+wwZZO?_Cu!soUMB+ zLR2butuQ*C!)POU@tpG46%nQxjPgHr&eyhbQ?_zn&8Wa|UC&1pMYU0yJ$3NSlZv^W{C zWb9z&-Jt5I&>B3UMfXLcY(|rVMr(9Oi}Q;n&kN1$3=F&(Y^4Ql(Kp(je41p&!L;GU z>O00d+qE_|Y8(Ci;H+%CM)^ROJqrV40h{NJ=BN#<(j5#N6PTPnG+9qzu{qFSdAQNm zqa{dzRpv&+D#^VX681(bT6F`M)Fv>pc5U#V*%lqaM$OEC- zzgsGz7-cn_?G+lO7cbH(za3N>{L{cMm{V6wSz@P{z7qXml= z%VcMR7S9z^n_r-btso4cqc8nLA zZ7(!AX|(XIVAywam)wg+{fs6qfff&gMyCU;1qE!f5e+FDT6iQFgeCR|3a|=qXke6J z=< z;UlGR?10WL1{Mc*XDjv`#^WYUoGn)DG0n%9xY?f&b|~bK6JYu_iBVGEpmfyMEy0Rq zEZvtyxHnjxt$V#(FtI_Rh_QpC#l3^c!J;+h0Bh`pvoRC+Gd0+&cCZM2U`lw@*ulf1 zW5KA(z~XYE)oBB(heD&ALW=_f%i;@6JQfU+94yfVha57R*k^Y!9cXkCXf{02ESDh7 zctJ*>xxsdh=l)-|EDKH^I&`w2%GcqG=Nb-%55Z0PJI<|4YO?*%G)1zBokLF8p(T1k zTht3i))R(nU78#%nq5|~8a|jE{qUb?OCf*s1xB|8Y;F?k&PF!zMlkRv@tp9`D0s}! zGE?TXVXIAphCIWrdD?E0UmK;@GI}sOFtapeZ1Fi>bLM51gF-;F{0C+miLKM5nq*Hf z2Jdc^C}@n(K6qAQ&efXZSC3pORcT;KU=WF5RF+^eS7=dHU=F;{8Z)EScxh|Q0pEgz zc25mPJq{M78x3kZJPZSvG;LafZZvxzV4kSVVjIzux&{fi-E-6mo8#57nrjS zuw@#ss46t_b1|?sFl_hiJ|yI_UrUE=fv0k{Z?*JNCF!MGJh~-jFzWAajpArdp4iC# zz{$m+C5WM!+k%tlLX*Bkv(t+v))~UQ|BM=K6`EZvTJ&BF_rSxo|yv6}{<^yR<&aksw zI_cwf=18;sLnb8&CdCttGBX$#MmI{VU=WwEKI$nb5^_-Z%%!%YK};Ic^;?;2Iaog` zv&P;SFf(lIVwwJ+N$mlX-NoxR z94*UvSmtrJIJ`LS^1|I|Ma#{TtFj)gW~uOJbBI#*Za9>6%kGS4)no<+30095%=Qc{ zh70x>Bpk9SV6wfzWW(BGdxG0BVKy^IgV2G-e~b;CXE+ypux8PVaF^|1abqy&4KR;M zm2wnj&&Xi5*OoPFn0qg>DPBccscs`z4u^SsBwMe`tZiJp`1+D;t14b5g#$io zq^C@iTB0_W@q);4;UvY@Q1^;6L07NzOKdjU*uWst!B{RBY$UT!+4;uAuFdp@6SZ~;xZfVRMpfS1QpTQD=7BP)QYLh0hW;B*bZC=5p zXLN}{#Ne3FGDb<>1F1Dz7A2pN%RO^?>zT#bZpvSqr(R@I{d!#{!{mZY#Kvw%E!%_3 zPe`w^@cmnHRyi$J$8@?4L(2^d)>x1Bk`JueH&`|Au@-D#(rgkYPhm1ET;> zZqu^uLCZuZoW5PfCFUfyu27AggMo1agBn|l8uAtM{r$UA}YN+E;T3YN;K z2400mo>$Fj9c}GK56lvn>koCZY*=uDbMk+WHiZ?9K_9r2{`uUIHaOT^cUc9)0qF8LiT%DH@c6-;$^8x?+BZ{T$lZwT7BTJW-OH=klsyRC1)Tpghc zOmXknvM$_>Uch?ShrPnDJ!b=(^Mgj~9n4$>YUz_0c!brtqLwM`UbU@o*@_dZm^vOa z?g-hiqv3-xv%?FfuI6P71q?d`t}f5LW$(w3HtBp2Yqyg28nz3bhoual*>|0r| z>}M#G&Won=oJ@{Ce0X;>C>t_3u3%Xm!@yhF#d4sF=Yy#1gcc_S7AKA-8;LCeCz=J< z^5*vTsJ-Gm|5GaCflORoi=ao0)Qg=G28j!b!%w<898NJ=((JY<`M|#vmtALLHJc@x zRC9yAaUJI`*wpOmxI#m*Fz#&OxgZCf2FV-DF%Q~GPP7I)u$H;C1~atAII#9^ZPQrC zYH)zren;cUs%4K^H}>1@JHfQkoNdizo_m*b8W?sobeOQXSg^=y2q@k>X_vOFGW*sU zx1?)R&pVsxC{1Hv+R<#w(Bd?sSz<;M`4KQ|+U~trEm1>wB z;MAx&k%70NQ*{A*-jA~eik#)r#uZTv+Et70{}P;C++3{Mn$pnX9vze9;^6k1absIJ zbIF!;ABT)DfoXY-P62(-W-woPe7*kcvjP@tA(7n$69scu_H1^zDseemN$lVGqRnl= zKUxBRv~VB);4jf)$ z2qTLl1B=*&IR&3WSQ;8Wh2FYm_T;)={29Ga&J2b_igBicr)_bl=Oytz>tV<-%k36E5F=u=t*6@z-eP)^83sZ+0|bHtOh&?r5^T!0hl( zfl(~nh)0-#qeI80xq6_-k(igNOXRx?Ow3s(AJD4}kw`y9tc#G5t z#^3W|WbLd{N{(gBc_?<^;L{UUb)rl;=NP3bf;40qMFO@KJt^@k=aG0NR>W7j!GJ+B zgE?pd8&CXK#~&=l7d}MlH#kiJ| zP{AOS!MrAs#dZa=O+|~t3YPYjYv;zP=|nAKjbPkRc}lqYlw2pv*C)CeO$RqmU^pMq zD45J5%dsx4LL<7PtstQ`HJ~jug3W8jkFI*=Ual`a`IW_swk(}&(*5eMWS^V-jXPEb z`Z8%j=X)x5E;~58|Dd0E5^H{e&RU+e@r~hoTDi}&L|!k5WEXz-pJll^Ys`+h!EZJ( zag6+!;KA${}|YK6&VU1FgP-?2&t$TXe?}WWfzqZ=x|uVEhx#)VzVNkJz$B)(n&KX zrD#s_^*pJH~px{==q9}9Z{5<=L7iS(m zYB@1M(W&Xlj6!F(o-@h;M^-%exw(Z)TG6LMK~d$^1V?$B3kEI%Tm75uE}lzv>ykDu zJHs=p)+>5#^fo@z$f6VdLe@>wa$ZbI=V;?Cep8TW=-@7=VdPMpxJ0qDLmWa?W8BVK#F))R7@zrN#j!&Oh!6%pxicPE4AArXS!6 zU_ICo?j3XBjPc_IpZO-b?kf(n@YLGuV>so&*W)pz2QIovXKqLom##`(*(MnKFV#h$ z;z1#Ma!f%WON!H*3C^iTcM6<&^h%yNvC3DtHD*+XnH~%c%M}S?77{UNV6%L2(MMy` zOvY9|nSu)|_{wUOjw&n=WwY-sAvw}+_Dn}PO?%q|0c9+R>o{-;p?$EbcmTjV}XM^r>TM9@@WzQQiHOqp0#vKi^ zGd{9NzF_1sPBNuNr^Bx&|TnUj}x^N)Fb# zI0y??IP90a@xa$o+A-~g%W|#L;Tv{}Tg6m1WeTska?MMT zmtXH-u(yPZ#+?c69vn(SOA4HW;w(7Ucj$8E8L%o>Y-r>@;lO68;p#hsvDMwfN$5;M zqr!qfP7M`7M%}DCwE_kV!5xbwmz`*2koll6En!LFzZe#Ag>HlPXPF9#q6sX@DFuzJ z8yW<;DjJyUj23YjypEQB>EXODRiQiPX|%oIEw`Y6AZIauCX-DUS;98#;f_#}dbi@Z z@_R-8xvxSdO*#@c@j_FditS@bpHDZGWf<-3r=0ZpIQc%`D|2S^iwe{1R$orn5@6(V zaWL8Xh36tuhA4;00S4xZ2A+MDT`OKbV3Le*{1|qiDY=hVddYzW5(OVv90Pm>${G$; z$$e~5wNVP~xxrvCk%`rM$tjVg4NN{I6PO(%4si-!XjG8+*uwioQK;WTe6vB){ND~N z3``!08j}t(G2Td!D%}v;DdQfsK;ndhNy0H@lY&Nu2MhVR{&6&^{JCJ|Xrj#4b$a?f zr&UrDUip<*SFKYl&OGsQ)0YXC&rVdGQFyuP&f?wo{^pgJzY2a8cuCe#_Eo^)k3HLR ziumFlFbVGvu$SXWtoxe1ls9R@=DEEGjLU9ig>U9MU?<8ufv>`W(c}P=MvrcmOu=gw zqlTtOj$T~LFR+?=T&MIzu})|`N7rCZS6lT z7him%a^jRF1v}2H44AqqwPc#)xey2DQZ9pruPTikK2?_l8X7Y7e6Q{Hbzlqdc&H$A zwCQPRLZxs*Bgrw_p|7_p+^&&5?n@ z;lQSzvx`!JXik2l&JyZ&)oA+4EFJd;P5O7ki#5pa^o5kd1B(-V|FMQ8 z%-QP3*p#>TC#%^54db4-nUZWrIFueR_(ZE57kzjkQZ|5rLnD@vS>W69_FPUW<=skW z;yAmr+8IPVo+t?>GB8Nlo^ul{W?hrmxIQOggH_SSq?ehc`7_dJ?Df|8pE>UZGi{m8F?!z%;mY$D-Id2uWU2lv)kh9 z=c9|h_jS*IdBN;<{<$yVpRUffVUmBL^(KD(RgMdCOk0+lT(r3(bzojoqgus=bH*DQ zc_JE^BZO~X&iUA3ld*t-uVL@!seFyyNn5TRV{>5S+0c;0dx-ado|s@oZtaaik4VO) z=Y{_99%-1*!>Bej;fSV=V3*E&mbuG1{8_n@Ca`U}>%hOlQTD|F2A&6OYg&X2oQ*FW z;mSPFr+D4t;KVGB)=-H{Og`85AG}%O80RA!^jNoNuJ633cb)Be9z2_xz*%7eo;WAnC5n~7cj;5kw{PVXY$n_+`JTz#k`HOa}H^o zICiDr0E0!>M^DDwnnSIuO+qG(@-H0N60R_oChpN_V9apX`m<4v#Yyf(qrw+Pi4zT1 zP0ll)ao8j?LrjI$IK)Xaq)8#8VPor*x)PU10n_*<9ANsvC4Fc%Z%+fyn+64gX2F^( z|12flPcRt&Xk`1cP}70=UQLur$u#Sy4vshc+<91~mL8Lzd938_ty0NJVKVpTxJGF9 zNgd2&RW@k0ywS|jrNH^>AG7_RCI0`G1nf(CrW;dUc(1%r{P&zr0gi)$cc$}Pap1YK zpzbBloE<{iHxKZAVVE7_rhS)z^Nj-ky9Hsj4D&)5B-{=NF$8Oz>k(rkq*Z{ES|Ot-tqL7aQYnRRq}C@P%^z2eRby4%;|ogQtVRZX>%mU z9Fm<8$|7-qk%QqaYckItCC(|A*VVj8=3B6q=K!N_gRspZu^UY)EKK|l4)9EHP_%Hi zoZzf*$I;m0phtsrX5(y+69<*wIM&@{c;pkGk>Q#)Da0+3rK94AN9C0Qx9JCD=Y<`Z z;yo+r@luAlsSd0fPDXPM+dR3+;g=o|_1x~mVY@w=WkOG%Nj{Z7;TS7%P*CO|{{;1O zK2K9$T<48w{!2;%Bn?rt7<+u@Ly5?$9Lll&lCrqmZxkQj!OkrG>U16%wrPP zX_R9*$QskYaN>Y~$PsX5Al*d?5$; z+ERI5C^+{u2)sBb*W##f!^vz)6C+Clvjw9{fupHMv*s6uWH$zt7-s7g&5AmWx)uko z+j`4xR8upGb#QIilrb^%LEv_u%&A^4S++&8^)31|W09s-y7#&Uo|5C17o2B>-786Z zUV2KiOiCx~I;Dn-civK45@N{q}jdd@PEih|wo2w#a`+()!$!TEqUxY{O`})|nuuebU4@S)w z2NhhHM0E5Q-*8lsa5DC2HV$D{N^n%X)5!6Las4$W6PrUE|F*PT*mC7U1_R%h1ATFa zbssS4#W?dR^a%(s2^u&m2sll(l$Z5sQn}%zB69AbMuXFXcE=_2UFNjiZ*{5LBb(O7 zQXKm7L19ZoAZw9Q9;@=1IbXXKGa33jm~D1AKPe0MPkZ;2>*?%MnupH4^#62FdQYR& zfrImy9ACJt(LTxW%FV4NlTo(ipzHxXg%~F#6($9akNnI{a(_0w`DDPqJx}h-v7A48 zBK1zuQKwAa9OUcZyu5-v=t#?XHD}Emj&cG_3NB1~cR=T0@{2ghML38{98$e=P*K3? z!bGTg-NC72ouLa%A14sCB z&f_mR4{LHo=Niih)4;dGQNzVa!{H#C zK({o@g5^090*z;%IdWz=a85bEA;2g%fg$_Cys*24ZBGo1OAhipXyE+Pz9y?XK_P4DTwqB)N~aXN41Jnks7qLF(w zBhQLP9{I*2*ZKvHoEJXBDEpX?9(`acXC7DA})}M`+ zmnsT4iHpcDiSQMA;UMJG%O|l-=*Y{doMy`g1D)&4 z!T-!npT7GP^7@nR=fk1#U-gfFn0M=dV9VB-KF43PHHnxREBQFpPG-`);#7B?seXP_ z-DC4Q|3fjCOKaUX%eq`h>^2tB`O##3;mVChxdO(V=o4%nlVv@G6Kaoc;5;N8;c(w0 zfG^`VYXSrJlZ>XHTx;*jxg|z%Ubx2)ae(p24vrV|m>XU&Okm(Y)4=wl>q2h>+lPrV zn&Mucz1;;hAK!RbAk2}w@L0f%ws&80({7m@_SvZ^eQAznp3kxaTsf6&FBpwmBzL85 zw>-l9WZBB{?hj8h<2e7Ta5;aq+rwB@{Vt68;5@U`KQm6sK5(=;WnSyw)bP1n_sJn$ zj$I4*cP(Vz^{@W7c>}kFrpzJPoCeM#pA~#ASQtBff7u{s!6+xt*xFOU;yNMvr33c{ zzO=@J&2kKE5xuf3hgeG*jeVMpO`26|-aeem(J+lc>HR^)35ULNIaP=_2;Xq3vu2)Z z`Xxk{cW;6clYqlo-}!E=9p1%9RA)zK)d)&m^`63X$?d}Fy#njh?Y8@6yZT34vKRei zd*B!t&pab>`;5t!2WI^&?=3hKy6)+%O}gt3R#xlrywCh2;wX5c(d~Vs+!9m8HCyL@ zFJCx)qu!lEhh8`R-uC|R{X+|GH!Wb_rF-R&3d5lfUqelFE&5xSly@}p@h}RPJl?Zs z^9SB@F1ilM=DuCp^O;y2Ht907X9RjbzAg7~V*aJ*eIc2i*G#5PH!9tm z!gl3^x6zrB?w8XR%Whb3+a!kBq=R|)KJgd!9%e;88?W27v4of^PBssP52nQ`WaX{&CS8JtD;Ow&8 zJ6pOQ9qGLsaZYAW#g`XnSBIa!XFFN#^V7@Qi&X7v|0Eo1k#y(vHO+b;qRI5UTimMp zTZ~r73xP$ROFV9_D&-EpyzEMe=VZ32t3plvn}bG6G&()+_S1oUc} znAwFyEIbnK>}Yw+-RB;!z-U$a`kK1Rj13J%--nKSq%XhFwnL9(_Am=2XA4;8dw# zxa6+7_s#B)yB5D$AY$~kFSwP*tC2PEV)XebnL0)McD=7Q^Up0jCwW|^{L#v(_LJVN zWM9e&hm8!UbQBmEO=c`)*EQoi!fo?yM{{U~QCS-|irvoqdG6XR zKHEJJvqIDC*{Oq1FJVL_VAfMxwNobx8uda zmI~z=3lcfhPRuwXWT3H-MYqhruYtqMU?GcY!36~-X%huTesz|>@JSvYj2zlMR20)^ z1*vf!VR4#O*)nCTQsNOIC5_~FfeYK&;=jJrTd?M#_jI@anZAy?{jJ|Grus|&II%F} z-_Mg9&)cm%Uio!S@u6+yQ4Zc+$-p`k7;T=StY&s(i4%jFOpp9c3=Cqb<;%0 zoRkU8*G?puZxD3KUXmPj^}ta%0|BwNoC5`tHx96heK>UVXt!SVP3}0E1y(&zS(zm& z7+R6>-dnJA(1D)zb-WCM>oeHalys+t&O0YQM}a1$N7AhJ68TMWGK*aJc^dHy*>^dUoIVBGFs61VL<|u#t8>j0nU@YikB3h zM|wnZv9;Db8u+PP^G8VjrO+ZYwVIim54re*0hex?t9EHji8hLg+Xm@zgAaGLPh^Eg(qb!4D&W(i* ztP>vJP`IHe(9h7Kz>?Vh=GjBR$^+}Ixuz^v@2R1 z%&m2iPVQxT(sSbUM8#W{@zLCilmoMb+jI-zA^xvStoU;%Nuini5*# z_zW1BrySrvd7_z*&7eW`#z8jb58V@ttQl=gl%+gul#CZ#>`^vzR=PN&O@4vd<)#NB z%b)0WIPtyR!n5Fk&GSJ1-E#yuvnDK-nQ|>(!+zz&pJ&w%%x!bCV2Me{ut~l$pO4+@ z2%|%kXp=xo1B;5njMLE*u4)BNytZXiVsG9HDH)k(2IiMO@+Q`=6`B4{MdXHoTgPKFe=F~v1^trjFwZ}%<;#AxjCavdVxS!iiNXK z<&W;l{~0scuPl_T%m|OzHG^5^MFO|Q>BS06vh!T;m0awew#3Q7_1^b?A-38J^i56$ zX3o8De);i?`1+TZ=P~ljN?rW%#duNU7pD1AVH@VTs9s9p=5+Gbs@=VTefrFYc}o_r zi+O(R`a!;&yb9+VcOGo$5ZsuP^(|CCple&~65ia(k4NLSYad{qCeZ1{afnm!fMbp^ z1FMot!sduq-Kpo4o4Gnnm|AKN2+Musu(e5IPUI5i{Bod`_eGusGGj>4Y=I-PGkW0&+ zE&rBo*);Wa>bI!FsqED^?@N^>me06xbRC;!)qw{QD?aHhn6Th2)3UeiaXF0X8pqx- zF>pCB-+55+L4lo%f$NV$?4mVHd(t;+xiUpCu)cY^MJ|E)*8!1PijqE#uAA1dOk)tL zU|{E3D6;1O%Z!8_o6fY#ISR~U5aL)UpuqSeB%uG3$NN_gCQ2PMI^R2FRJ#P>1JNQ~@ zoX?Frbh#<%0qyQA-vn0KsXc*4r#<0$m%1yhBC zK$;7GibA5>V)qMc6U82|_at!C6&~+c!0V+bHR+**|3gkCMu9g8LM)05oeKoEBrsGl z2-y|>if|Cvby{+n;&ZNM$zQ5MK8^2RD6s!IG*{1=aoz%J`PvHE;psT{jWpsddkB@-c0mcI-8BG#6b~y0OY2babfPYWV zR;B*RA3BmN4r==*UON;dG{=FZtU;i|fo01Afp-e*U!2%ht^S&`P?~%3V=)J|k^@~) z?bgSR*W6klw4mXeQX~7JlYB1O3rJP>}izT)inLgs(*(b<nSZ9D z>2ybr_7cgihaMBZL~dw`mMByyzIrR}LKB0JH(q-ulQ)GfR$+Z15zG1{6 ze#F5le^P6ehdxh$y5~gwDx+l)S#f6GI`a}Z)jn{FC2(F^#QV>QQH&w^Q-j0zL*+j> znbRJyR2|@+(jegXP{2$}WZD6iPbb8f8cjP6@Xv5y>2Tox_F(%LCw?zC0j7rnQYPlV z4z%+m=xHt0H)gPGIv~Q9e73}ats)`z+EUI13`$=n@}%qWr1mK+X3GlWsC(4Sp|sF= z^ZH94vO|v@-0$@E@}h+Yj`W*`2Ag(&@#tJA(bp){_fW#>QuLa>8Ji9scIdgWBTuNc z_w^No@ z`KR-+&@4e9>_bJzdEjv^&Nur5ImD4TT&$eknMCq32Zhp4$!#9JQzW)* z6}+)cQ0C!ZyHAI@G-n*GYbL8|X=X{|ckT5NkC6Ry4k_4e+`e)wi2ZZaJ_sZF(xDWR#j^^z&71hTgtM+SxM?W>4dKaqQvmePYTd zztv7vtZSbl@nU+e>_I`XL_s^Fyp4K??`W-C_9f=?Hm~FFVtOyfZv7Jbwd@U_h2{Q( z&U%h-mwCnquVD~B&QMu2yWvvNDHnZ>5;Hvoem_QC7R8L@^ZZ{J1fuwbb}VRr=E)eih+3FTag{0fYoB?lN-77C;(Gl^-oi)AuJu%FOd zJoj8SgF~4BhoX+ozwbZKA)&0NZ%j-Q`4s+e~Eez#p$Cca>pU{Q-hS>1wZEA%(PbuXI_U(f9|QD%4%TDuJf?wEBha>&HVa8 zb;rN;JuR1fFTMewyE7zTa@WF%n;W}%ZwfpV zUYNr0*Q;>-N-J@7M)My_WE>b27?vG-smJ2LcBdg@p#!(_!YQ8{1cEI1pCyFpPvhQ` z!2iQVKqZkQY=NG~0X7%^GJ^zm4~JwQ26i8Y`xOl=b=k~13s@p9b}o%%Qh3n+??i^$ z(q$dxjDCueY>v7m4}xbWR{lD$p6!9UaS0>8rxABjOj`qMNyEC;;#_~Ycq1L=ee_XQ zG5YglhtkwU$sap6JT~v2sJQXxj{f@!*Hw1j(72c{S+S7ydj4LKMpdiLtlwj0t!}7D zN~+Ko*ZmR; zm^2uIw*C4xk9#${CbP<0iBbjTdr6#MHcPT721uvrzj0#z^=QwLh(d`*fm;efFG}nK z`&!prW9$05a?g4bPu=;iZ){+Xmh3nF2v|&fTsl$Ia-yWv z5z)#;l0VINDqc^Exc6ym`-v@uPF0MHTiDl_^UrhOTyveF%EhDQNWi0~oGc5gx{vew zCB7G6w#!vuZ%SaV3*=~Xkmyw4@Mth{V_?nww@{#JfykUh=|cp~L-ue>a#H%ziKXB0~3 zGu~2QRaRV)rXo3Qf#R9N3aYCj`}H=QFWbO5djt1wkBNzrYI6?h?&)XWwbA&Fs=Cy~ z`=YAFc?)+s${O6hYIys|@3I>XC&bb=O>+OOcJWG+^!s+DZ@O+00&Wp{JPnqUoQvK& zF_|r3`{Gnxs=zy~fpr}RQ{JWeIGC>DN5=kWGTn8s~i;T(c}MofnWb_j=as2r{R)TvXUPEE<4vQXk_I( zaW&@hR+%dcugd=9{VEp6ZxQEf^St<-Q2$G@1B$wa0{(9hdf>rtHAHZ?BlZLUz8!Gw&C$yi*YT z#=!3JKrCrTiCW^dU3YyI8cZLou~=|p$$yS{KbY9JCGU&iSYO&8c&mC&n8LP5uTnmz z*lx>{pb|NDCJ+7sKqmo;lP zy8q?9a$xh(kGg1Gy^cBY+$#dUWt;y%E6f9iGIYYO1x#oIy zffWqeI&DP)N2-`7Eif=^VD69I$hKgKYPs33FI^n0Rz1!Zd+t?eH?X9nvpiYL6xJa6 z;pLmZ4zIpAuw^%j%t&D2aJ+j{R*>P*w9Oo%N)jd!Nk>@OMdch6J{)Ld;S^AFsc87f z*v771l5yfg!oj8ql43DC3>Z^Raq=52D0x|Q=%m?6C&NpOsy7vc)x$TOn2^ZK?RjRA zt#wt%O{1m9dUUh7LM1n@JTXmicbDi*rM2B+M}Pie>yt9Qw`ZyL^S8He7CpW4ud?{| zwl|SmpYGVGTfS}0&D|OMUtaCrwkAHmbmv6v-D@Lf@Jg8FaJ(uy%*xa2e`t@w%|onw z-h8b>EUw2D?4_05Vge2vZ0O*U(8;q zDF;I(;}dsQop7^~%1hqDJ{u0P<{1dJC|Et2ct&%X&9~qdX%C^^z*9DhdxHHvG`z6l>(q#>0}RfQIcr<5n^zPvUym}Fw^c0B@|efszWY0tUQei>deztL z(wl!5B?8nv+YWllu-r(QJR@=2LE#>rM{Xj4J3h9s341gIGD=t)2<2?pTegge$znnS zlTd&HW8Q|9?kp;2A2fCAwti`FWUMwh?4_t$F>$WA{EI|y<5-aZ4!!Lf8xN{nvIuTu z5uEUVRcWh86NiGWL;wTJ%395XEPN?n4%Dv5Y6#3;d-BSC=D(>5!U2*Y8!j>HY$#yl zmVUs(Dq?^0)FCcmJK$m;R)FXm-3k#+%tU_ zm-Dhh-2Bdti0L~kUv6Dzely3oS57tHnnY@_o!FEV^~`Ng*t}gnz82d5Y4$}ACKnOu zS?RMf)0jB^DJVEH3p)e^@HE;y*U+0>(|C|E+Q;ams#1j^r&7$601w@*1&I!)w|)s= zQo3p({EFE|p~_onTaJ|z?b@TjJ}qc{`joii_-M z^pO0}<;csu<$*SLzrkmo&BsJ8>ukyMwodBQs1rRN5xVAt5>MU2)^JPve>cjbYY%t4 z+xbT9S(4S0C5PWlxOe3BS+ko7!W^7Ej?ZQ!hvX$3Wb;pAyPW(p(!uHlbKexfnLNkY zq?w8j9cy6bRk@(%%66B5^JK?_NgN9CQwkXPS{a&L*`s2N+QiuJ1fGy^>Db`@MKwae zO|dN_;2>lCn?P^9f77o7ozk^fq3+0PxTAq--Lko-0wPcpDV!2scDs|u8tYU zj04$m%#5rI3}&1O4sWy!nm%$o7wC9#MdC|B6Z01a!H5G*9905rvs@Rj#~(Pv$+w`z zaE=3um_?(|w}xiso8%rW|p4;oi-s^BCS`MUX%yQ_!YN@oo3)N zSzs?x^EP`zf}5(-%L4*}k0p*bBy4@s<`pWJIMtw{aTg!+4DsFqeFLL~e$E|>4_x_F z&>yYn$3D3?i)XW;%mW$rd9zwpxcxJo$-IHVcw*QDZiY81GItdPM7K2rb0(-rM%+1Nt)2;;MOvMd&UU`&Y!BwRf|-Ze@*(!v_z6y!oisN!Sv}`8=9`LsS0syIpTHq zSP)0n(J65=rCEvW$oVvK45ZR9DR>9wsSN0;}(}0FX|Rc zU){her1K$!PsNdI-UAmki-(;)9pPM_6|2&W8W^YBC@@`WSyp3E)X1o#puzuOI#bwW zt%bY}8Rjz%i*IRQRt$M0b$tTcyn;aPNiRkJZ3w;cy5JQ@#Lh*+f|C+lo3syd1wBxb zelvrGzo4O!>&yZ+BaK3y6jf=ngKBG<*5tUb&Wv>No1)4+At-d>%)2RF*O*@Vp5*d4 z<>|e_WSefOX1JfFW9~z-e1jXg_u@V@Ri-+}e!TQDdQSB8%{uG!tbE_?<`6y|^(nWt zMe>aKvh3cNnDE^}0`JwW6yq$q&4tq@IEkFGUV75y@@lpl3mKQqY*e*T5-O8ulAX7p z*^VcX)Bi*xqt}8~+dGNu`wue8Dll^TZG0rQqoB<=r?G?e`bUAEDy1uh4Y_?Eu9RZ= zS5h{|aHZ{+1`e|ujN&^EG^>j^vL$h;2&)QQJGE=Zgntr@!5hs_2d1bgas?SMOP(;` zFz!&~dU`akuI|l@&7n>wretJJ++3t2d~;i#=$W^Zv%Q+L5|pKH-8h!NT1HvCvGR=U zp9kF~Az~*i*PUscnP;`8u-jT)anFhgn=1}mKli^dbIHX$^wVIQHSDwm!`i<4J zb(2X%Z*S@EX{QpFb#racS@-zfCASPt@q`qW#al`iOJBR-Dz+%0&HVh$vnw20Z;R^P zGcx!yC2qdHRc}QiZ>EQ{_|*e;dTTbWd{XPkX?uXtzFL9ZQlyc?<$+`6p@NPe zi^qcEo@cWU&0;uRwRggF-cA?!Dn_+~j0?VVHan!3EGx@;o;Fi}RjnYdUyyTK0Ou43 z&g%5W-VGcQ5z*UB#j-CnHnB4sFOA;An%Yuc(caE`y@B`khRWBQEG`~3|JQTaJ6WyL zw#a+?7mNBvvG&5We`&l|CX_95uiEv1`-%haz69Q-?Nu`sSf5=Di791d+ZZRq>g(8` zD*Bvlq5#)K0ZxAd=CTio+Z_1LHEA5v5^D$W`3URB_| zejw|*f&PK1`A;V~eZHpqI;m3jOKRFQu`O44pI)e3tWZ_t#(kxM_wEF~mkW5Wd;H^F zQNUI5jMMi=h^nF_yP(1cS0Sqh>`f2YnkTULTwt#h;LK}aE*5CJ{(xtR1J`5)t{)sF zHv0A!P<(Cz z@74nLm;}zc4T3&X>IF6@9WZ13IlcZ!nU>B+2C)a7X$M-Cf9LEenB4qkS_L~v_*x88%y@XCrcW|lW6@ZJ@OdAh;k zms!?_KlRxJ z7&ZSfwq7{GcN4o-tak7j?GPhO*_eM`Heol@>3cRll@NE}hGyjnIYwIM<25BAzlNj}! z7M7e9%~l_`ve-&inP&=@r*lp_z_HwbBkeh7-uKB4lALD~IJ*{1DLTx4L#TU4DNCSa z)7Fe`{RvC~)0oa($h@_@*6DcGwF6UiEkc|>rPfar|M^pV=5(?2wB9ER^zJU0UNe(( z?g#E`8PmHa^Ht28ULBe4Dd1alMd+2OZ+)AjmBN(DumIKtjDiNtNiTRNJIwm+!L?n1 z_i_ODOf#l`Vg><=D`zb>m@wZVi+Lu;5(i`U4eXl6yq5~Nw-#_MX(--WIs456o>l>l zNo6%X9SmEN)m94T91v5FWe;4dC?vL^Vcv^58;^3FaNsO&pL|s_ZiN8rjjf)OKd{#? zm{N0qJz*l-V@ojyhN-Jvx?j#@D(6b=yp$RKl4sY7ynQF82D8N&A4+we6tmHV&-jO4 z=1b)?!D%lWc=s`+&%MA|>^YrxSLLgM={E{`moF$t3y7O(7{%6L|CzB`l2Pb~D#PlH zs~8nb6s_8pD{!?XEGk^leKvr*u0X#*fRp9t1e2`ArcsM$tz=OWU~nIkvA33nALQsgz;XNl zNBE_*l5Zgs3%FZ8us^)HtmXq-z5x5LQ|mJqq$C>#XBh-LFfg-q&(maJ<~lv^rebZ) zt>uxGyw@03+-{Iad1i5Nt6p{Rir;DyS&9qpT&S$R$+=LvbPKCkvKlssKVg1(a$fnadHM-lYaVQv(>AsCR92K`>bql3Vy`!9sWksC zOG~@xHIXY_Z8zV)2mEg~RNgw^*!gPHwH?#LFSOW{35soSwl5JBRG1+VSkE*;il2dj zA%KC2fq}(MR5r&2D>hq){krUI zcMG{4qh>8};LY~tUE)yta|%cFfsUJ}tWq8Y+}l+e#3ik`kwGS*A$J+~-GX(mx_DlD zarP|Wa9_#MlfV`}fpf)$mi7xg6Amo<`D@v~DYJE_a&Ne`X{Q6j;ir`wOhcD1+`WN& z^}NXeoKqicc(-%;2hI5`$EHqtoL9%VTdsOVLfd?sh~2ZM^RfQoR)5X^)?xa!8@xA9 zEa3jd{rIb;jaihgVfU;b+INDu$OOOPgL-q z+%SRJlk0(ad!G_l)am054s{U+SiBOLye`O|t(vgy=Cm6ET-zsbFK*ySFGyDm+`e`q zgRp{7j?q#bHR)vs8B;DyzH7jL^1(XYt?Ss5 z@I+J^J00)d_ag5~&5@8a@jffPpP7~EY|GalwhNqC zp2?wNe6uQM!qS}+c3qhEvVena8F$+Rj-&u7K32oLLoQm=>N74$>274a`m*!&1^zby z2VZybNbD$=KErXcfveW@vcMdkX#zX9&pEAg=Fsjxhuj$s&;89bKl z(rgPD951Fl+`9Xf0Popb?6P-6rRT6WH{3tjTOg*uvFroK_C4$OoH?}5X9J7&a$_Er zmhh%~xo6Vf-K{t1=rah|*_;^j_iFX6|XuCji)R0Y&kYvTQU8j z$I7_|Jo_82A9}$3`oN1r8~7&On*MaE{Hv{TBD3V!mM)56K`e)KKh@Z#s)=?hLURy8~=NVrn)fvxnw72P>( z{dJ)^mRY0+<{iU;f}<{gh@=SobIJ)`ivG2%8z$ftkMf91Lr>P4+Zjsx@Uj~5&_6rWL z3eQY1fBx{d#-nL_Tcb5N+Zj0J>b7nEH-Uw7G26>q+>(!vlybju;M-fr(XfE=px&FU zxo?I%AOEx^q{kQJ$nhiX*XO`dE{$A_G`*+ixe`K0dBqV)D|iM^G|)8E}T?|XVG*XMd}vRUTBB^CcpJy|)~JA8l2FCn!v)7QuA z8+$(7pm?~6QH;lC!h;8njqIM2xhz&KXkcax;N)O<;NW%-%T9tsmruiI;9$gYTWdUFmWVgYMQdK8-A9 zyS20!#Ud2AG|w0`tA?aR1#G=qQ03X>U$N@}(`}}gYx(Y93OMYq+-A+h;ndi`xbw-3 z>UXT-D~hjJxyLU^4)Rjpdt{Smfsp4TU}&}_el4a}FKiZmS)tUsDM@fn?PXkg+KICX|))0S6}tZTF!8qZ1yH2h;^ zSheUHqv%S-GGGB$t;pN8G^}b(`Xq#Ow3fb}%?IvlSfh z$@^TUH$wvh`;3A%nJmAB*+LEz3>-T(SQJ&=w<-y?^ooT}KJ07n~;>e_@jltG$xGAb5gvpZbEU8%N^fsw;u`y&22vnJCusb{5c{y27{U}IzK+f@!{ zlB2zs2()sX;?;1L>}ua(s;leBUBRHiz{GWcfn6Y>AXKwKV1^@KW<_dIiUFgTT)Ljj z)u#eg9?sm?Hn5rg31HRbXp&Yk?AZI|wSJ{RBd6B}7WKP}q*_k4nrsNUB z$1RR?&R@=Yp{m-!~J|EH>N@Z=96jx#RW|2I4ma`nbB z(<5yMdQu)YDC`W?7VmHs-q66R+o8~C<+g@HFCkB6$AvovPZ}6S6lSn#FdUl@bcVeq zpn!qnfKm6&Lx%$oi|8x0N^ElAa_C?bC=qmtWlP*?_9K9I-sk52PYHS|%YyaWRxIPI zIKU`;<#qfOg)_Q(O}!FqP3%k$gd`N*8qQAmSl%zFk#enL&y%H|3k9M&8})u{o3FEg zL2v@Iw$#CPt3Myj@BIE3w~%eAv4q5?^sW?@U&}6QSQadt^uGD<61|(U?_<1zC;Yf# z^o8B@tIuJ{%PVHBd}J(`m!iC4U5=`o&!^s;5NE083MP>`T&)@|rkr_qB1Gx8p2iF$~&A?oZ_o3TP2PJaOllw5Xj;@s=MYOUulY6J=T({OB?642?T;(1nm zuA8Y5t6#$W-`B3&>2E2qzVp-hpW++yd7tJS7n>i_Z?flf@T$nSQ??s%UOUt_vH!<$ zDUHkylNBHDrKmjKyKBPwnw1xMvMiXzw`U)Fdm*?xYRV#!B@8y=t9$il?<$XHkn*fp)5AVisv=g ziY)ziVeWg2L%LHpw5a5Caldo(RpebM)$q^n=T_FL!zz=_JZtPW^?r>yEcNoi7Sml9 zdHs@;_kQd=!K6D^Ya{Q6DdEBkuCDr)5!|;p>uG@d!6QOkxsO(JC#?wAU^C>{dZ3%% z@lT@k4+RcKp3Ioj=eiaB7&(G`nmA=%h->-@TBrYLJQS61NT}mLvn+=rhr7Tb{=N^) z(ia}Ezg~Bczve?;h59Gfr;8o;W?yKE+~#oq_p$H$+E+A*Td1?utYA?+vZljzN@R7T z7GIH!-<|U5{S!<2rp)bhV6jQx!x0lPzofg&Q|Y0|2rx>Fglf<*1zPn zR~S2 z{^6A334ylbPI>yjx>ypfFW4zRYsc#)mrpDBrE@rRJQ9e|Xxw>9$@S2b$H7xIKBy=Z za`=^by`Q_N<(}rn@B7*9KJ+eNdB~d2>E)vIaau=AU(OzI8MAT>1Oqc$c&O;@#XgGq1+| z&Fl(zVz-JX%hKi(Z$N$3^OOHgR%lP+%db$$;WSZT=B+req`pPw&&;WpT#fo7bDd>x zT-cxy#Llr#Ya#oyUEXo~ZnmFkw0%?C;iB^4Qr+d15&{|dO`Hsh|2j5VuvR6p7k9LI zE!Z72fyL9p;gf{}d(D9)ivvjq=6h+damTRn#I&W~aNy%Y)VK|sl(e}fLExBWT zk^&pI%bqjAEGMH`PXBJc_e1x+;JUe;JIuTp&67o}l0{xrG zTV{R|Sw2(tZZQA132gU`m%3-GhE3cO?YQsXi?)0Z_M!`H`cJm#zu_p_F|}w#hwhXu zOOsU>Y99{2$-n+*#}a3&q62I(7ObngnNHkf5*A?0JvH5B1-D1Sbd~_dvp;#w6IwhT zbnkUo6rj+2E{I9EgUNkQYvuyeGKqHPEv>9Q+oCl3Ii@%`bvyj&bl?tg6_#uLI8 z#>1XkaI}EIF>;El@EX_Th=WNBSduK*yd2n?i&-RmcHDi?`_8hjeX@DyYCiW)zE=@b zx`mk(U2JW#*L^$5>(!{~yJ_#AjVo_fwoU#$|AW!e(3OWm5Ax@|=>PeEqep?gOojdT ziCJMU`@eXx=NarPsS*8iW1r5KS#PaXix!xEvS!_Knn~qFqumn*!w4aPU~c;cCZ$Kr zhn%*#s_fovuzQa}vqr>RxeW}$7nYrx#8#}};Jbz;cmhk1zy@X?Hg*$M_Aesra~%F? zx^kPag{?UmzUO3w30ruLYjOo!w9ioy8@BkI7PkTxcM0Ru)y;`>X1?d-dswyovDdnP zFCvyS2s6nEY?oDNQsy|+^O$MsrzX#ddW&B6Eoo$#bxQBf?R6^}*@8aq4U$^_QOT~P z#*!t<`CagWZU=Y$GaM5iu$8`OtGd%M@d3yC+iVkF>?=}WFTK!aT{AIw1M9{PpZys^ zyE|HJUNF0Pm~D^fE(zjJxXc|B%yiL&k=vrNWs6y60DD;jdtn0W)HN-Pvs+kNe5aYP z`g0!TJmY(Jif3{}+bkaTxqCbxt~n`eaxlq&E$N19(gv0|ojr?USXx%wER}J)FC=m? zyYExtG1r-=K7C{?P;Oe&;T*;> zg>BXn&qsGW=lHM*vs{YyIT~xjwy@;fVxD96GtS-nr8`?wL}tk;*TzQa4~?=rf^$4*oE! z<9pvY+1=e&RdW2uoSjuR&gO5VUUAz`oG|r>Woz&PtE{StC4%C6R>*8w&>awQ#OrGJ zQ6bi>4Q<&E*m4fAZTeSWmZQLaMv6U6;Ur7W)tf0N*>gOdl-ZoRJ-v5al(`xx7upt^ z>MHm5->>}ly{R(r~O{`HEB&5bK-__TD_$!uWM*&C$vqH*n=Q+j8O zEpAwRH=K2o+vN905$9yqmfaL`KN&cgOhasR z&m2C|&+2=|cu8B#j@7{&M~-TBZ~D>gsuQ}agIhLeQAws**($@bU9FOuftf$pDiYd@ z8`_v<3>dXpC&}Dojyb@t!WMSJ+J9k-*N%&ccU_;Xah<)!Q}ORf8QE)bIyUYTY+AV8 z>>|R_w)!~*s&p<6s<2)nbE5HI_TC^x4s*pbo0Sgi&=(DUb;!+k5z8lyoo~C^^7pV= zOf|JKwbHYRet!7+bLYcQlTSl>4#TTE-awes<;ow7FTSXccH*j*Ije)j%}nu%KlSlQxy%T};w6r7!$ z>N)kuour1VY$_&xr*?Zji1OC6I8xoRU-*>BvlKzOhspmqu3dT>DH@h2#&^rLqdBCn zdEs2Y`x)09tor6K&X-}kCwZe$az~@=g{RtA!#>zF6*%9^tDU)FjmXcFQ@+h<`LmJr zZZKQRj<`3qTbBt-M`kXqblulevGeDHN7ZN2dOa3=`6{(!@#P?+sT~SD`+e9;JM0ej z>{}Zs+8Hp_NBU~uWfSvjCe>kD?+>wNUSO?TH>)6lHABN8&Enh^owk%63Bd-DT>BC} z%}DWnpyIKi#eKySFSE=0l3S7!SW;eO^!p_Wiak`)y(^N}5>$6rEG*TogE_S|@%-mK zZOVOpvl?YCBubxn82gSft}beY>%FySc1-08Ulf_7woUVHaJZ(}?SJKW@3CDv-s9mM z-kD`!wmzq9sqwY*tfJD(XCK=4<+Smawosz}bHi7WqRRx0G!3-OsPP9fX zkbV6uz3!Xd+ptyIhuU&Jy!$a>OV^C>10m^|68zjcFH$txLf5kFyZ3ru8tXI@gGe@+ zobG+|kA&qMbz|!bdwAXS+VQeC^WM6;e#l?6BldG73Sp;=jGrN<`ov^lNRQckYwlM6XNHU5aJV(c*4(NtAY*3nYa(bd*g(@-S& z3pLLOwJM9U$n(=FOEXExk+HCJ(=+vul5=sfb@s4z^R@G|b_ug}^tJU)bai&}c5(9Y za`AHY4svx4_HhdF@rv~Jj1KmR2=Var2o3cOjSdU(4Gc+(@JxvgNso#s2=mEKj>$|3 zugQ)sOAoIojxNaX3Xe$h3CqriO^HgWj7rSRPE9SzPASh!uPM%{DoIbzEUzvtE6yv6 zO$=&JlkAMt>&UQ}US>HVTW@KlS!;+)N0jIE3ip=m$kvjWo_w#?@{E@1lJ?5--m<*K zRbKiN7!wu<=3f%6>IgDB#<-!^e8xhJ=?|2@|1@~?Om#wc$cBEu&X(-%mddFem9wT) zP43HFJ+*N2>cT~9!q2P?W%;NA3HzU7%_P>26?a(5ttdh>M%8s(yN$L4*F`3Qn zwN2H{Q!AVLx?5VOb+yf&($U^Iedg2|Q~IVZo!Pm3*0hyV`d2UR-m7w(~yB{`UIw-@pIL&& zmMuEkEq<`&r4SpBgnsOSmN!P9lszZJ1}suJxoPQXz37BPcbqn>omDs6{$R<<*0jmG z_SJcUw?04jU*OTKbk=inr>~mJRu5Az(G?-fJ?2_1y|v`En7Z@Qx>&6ejT`D2d*!xD zhOWA?F=p41xtCM>?(Qzyb>ik}S3f~LS0-^qrz0H`4%_G4=y=G)+QlWJ;!$Q$@Zg|r zqCs|Ag?gfdl>on4ONSwkYmb!nEzh0579X1^9bEQP={9%nr0w~uYb1@;#OD^3UaJw3 z(Y&!Cul`QOYtix+AifA`O_`Xg(*Z~+5bM1q?@;SE8jKLyOZJ+^I!QuqWyW;)Dl zHFQgtk`s|iUu^p1s08=aw2PVRazg?=6jq){%H6(n%587$k8VPKu_lGezLt+#CI=ip zy7g$j{|t5cfS0{jL#r$dKgN_sicS>`amip04S%$kRW;<&T=l8KF%DBz%7dC%r%u1M z@a45i9aZtLiG>GUuE+j6dvUr<%#)W#;v3&)9+~eXvoM0!@W_K3EfYN_ILJ-ST^8vm zUG*X|Ln)~0R_C&r&srto_B?y!Hn*=tC~wo*Ye7BB_8kezH~U!;EdI~>?-t?y{dbqd z1Y|Kz%?|Z*48L0(5V}0J%sF-Dlnr+^ze-2=Hl#kAc8ITf-t@|_*~`UZ6SSwtT>bSd zdS2PZgRHV;oF`aiYCc8INbI=zA)!UoM`I?-)YNSc(?kn7UhbPRiA}ImwAb+Bfut5i zHuw0BC5sNJtG+St-Zs%n+H1w4ThCTmGqJudIP~zz5kKkbJ1sTmmtLP*a@BCRXvB-! zolB!5mT}&zIFKcNZR=&T>!HGadF6AXBUpM{qeHJPe_jP+DCfy-ZtXXjvn!&!gg0sk9ZpbrBlItK zm*<{UM|BoT39m7h<(%W+aaue($X<2jyp#Js-PsZ1>U8s7wQm0L)QTj_%qyX{ThFm< zeP%f`WZEod-K!UziyOaGnohkMJkS4++b;13@5D{#v3`mx6|CD--Ljy?Ol&<%XV{As zj*?{>QZ9=tBQ6|R9_aFS8E5{Kf+JfxOKu-rQfFB4Lt?(wz8^*>kA(Q88ZN!(fAG+L zk$~`DSKb7?uMOQ4{?haK(`PB4~#qS=BkZT*G%{5njge0C8i518C6 z&cfNw@K!apnbYT*pM)Z3s?g=>TAhjB zHJ;8-e}tJ@Y>sPj_xcgEL=TeKO86M9bGAaCZX5aVZq<$mga}IG68)30ihM$8(xMim*K0A3q z@$&2IJnBs5n?sjW&H49giu=1|+go@I)%<4&{tOBgUb-?X`qaYC&239m{-$Q9%U?cu zLLkCt{+mxTj`{dqZhNY-^AC%!$6>L_M{cm_s0b|H|0{Ijrk5-|ZRvfhrKK)4ExVYb znyc*f(v#)wkzi}yO|u!bnk-nATa;wd*gG8)xt^*@a;J98W%%3^Vl6DSDo#K6y7i53 zW~Q@eb@+E03Le(l-(dG8k-KSi&dJEBZj)`+p0E949P6~%$$n8|Ug>|~xp4=JMdukd zT3oy|EqI;gly@_xa7J(}em3Jl%Q3TW`cWLq#M;CzFZk+^;j&DtjbtLMzTa;mIgA%EbMWVOB1CN|9G&6n&v5Wc)dM{oaL zHM5-oOD!jfFTSJ5lPAeGIwZ>81V*!)< zi(cCo-N#WqqRq1Qg4PN^$NvQGG zyT480npE}FDL1&hx20^lCc8al-COC?g;_;1(-x*jG5Q2gohB1^D4;XrQz}#CQlMIXy@}4XcAr)lPVW7lf(K!N!zaal}b)uax`=S|F&%gE(sGB+_`$9 z#42I#%*+t$vu#PMX6?#VpLN3av6xVMrOQr(pIW{%PAdm{mLI>xbf>R!kGcEM@*ogDg2F!3qNwHI_pnmln8f%U#HB{hr}m+djrQ_8%HK z8WfmBe=x9WF*KHhGZ+ajU|?}Dc+WiNH-plH1Dq2kv>MKM$dkrVA-3iMv+$G$OfT9P zSojJUbZ$J7Zsm{)_+xBpRU5z;uW{kMkf+4qoF6Uv2|j!NX$Z_@2{^X0T7Y?p&f`7z zS4AAXBAjQJt9+#Yhq0T<#I3&jX6Y{ZS+n?X)uMRYa&JxTg*QCnH%;lbwbIm|Yhdf} znXhU>v-M*IcHqS zu@Ct)8t!pFU|`>}n@Mm21B)8Nk4?@B@A)ho9(gw~tHvDUnRVlcPK+aWxKEu(+6E>e zi-0Dj#cuvUD)_nTBOzn` z_;)2gLtf6z+N348^Ej`J*tQLSw%cva)Z^tBUs>`c-F`NEuzbOl#sBnJ(XGYV`y6_!pFaoB#K3?Ro3_{PjQfx;L;?Y+$wAwvab(N38U=3#@aOSA2P1 z&Ub;~%l16I4SDaw7=dWGud5D6wy>n8rDd|^W^7B#Yh$*aQ2MSY<7pW~()NP1?d7Zw>Ruja zsA{ix)m~Baz3tQUwyYcF1t-csir2FQFfs(R@i#Eq1aSSiUY7s8>`4N5xdYQbK?X*i zh6dh(_PXax<_;OWKTKIJbg&pSN;j~iF>uX#&^Y4&=j;!i^Cg-TS8zraFsT|aN;NP> z1#mx@z~RuCy0uyN-$wauKYZCPMKvq3xNXQZbzs)pz|=1&Sv#F!L7Ae&A;lSjH8M;w zdS+4%f+k)KtVs@GNe;b%3T%-Hk?IRdRYBL(GpiW%C4MjMv}mhJFXMAyNMmPUVPIfo zVCdXl-uaw?=RtY8dwW`W#rG9;JPr&z1^rBw{oh`6u+1ePdRs$0W;qRMt+U_+UX7Q511wdG72l?Ppya+j$jHbU`xO7uQPoE z=d1;sGYZ&deCP~k;9B>9>#zg&W(PK<1&opbOtuEx*CucT8W?Tv=oTq2WLnhizCfYr z6Z8HR)i1Av%syxnE$O^;SrX&otgFEV<_QJ14J;X%tYHPLkp=~x1x!i`Og!o3d>AYx>+d*pwA|onT)@b6oI$Cx{Oz{9SI-*S-5F#LOcHAV-6C&lIeD=Gw?QRa z_=4Fp4s_1>z@=!x9kzi{B4Ns+0A8IFQ=Ek|ZLT_2GO7Z@<0_~SHjoq$sCe@}-{_DRvLHJgA@5yq$Tjkwcja-s5CQ7dCtz0<&7z1C`eB}r1=RVXc&Ek`|IWzw{ zPenuC$LUiA6e{>Q7Rwqe<_};J-@$2N$+dU^_sa`>+a1_aKCmWFU`=yiGn+Zt^%i%= z0Y;exEL#(JI~;sou_{!3Tq1s5e0ms5xRi$4m+oWAjJwqt{h!YDPZgLbWSiF3xn9rI>6bRE9HJ;j{o-7BvB0bOAUjBF1YSQ(h!Z|_gP-uCf&2XDd5I?DwW?i>=SoJG1TKl08>Mz{oO_eUIDvtG0)y~}OzoEW zCyevCzB7m?Fd0_L_=Ys+lQ#9DSquu8ZW;rz5T2Cy^7MxWm!Ci>~(Ie$p)2&A56}V+LkZX zG&_OSeO39Zo>joWnH*B0ddt+`oJ5vB7--bmuv=})$%A3+FYG<%65a?R4gZs?^ zzUl_nI0p$kb~djMldQWr?J8N71Xzz-a(fv#ns1b^x;Q6DIYh)P#IM_Q`*Bmg6;7WY z&Fz0Q_tNC8G1f}6MA(WSuulJ%$~Nc20p?`_J1;+2 zc`INi-<<=J*1VHGFmNx}_o-;+$LanCLzw90VDrF|ryk;;C;C#GcBkvvFyA8ZI0(QPM z;JszQ+i1Wbb>Y~ggA5uC3==2za~*GBn{a6EkIog=T%si=9ipsh2He_rIH#zydMa=| z2;knmyUHq1-M>?PllokVGKnj{6igmSoLoJZZ^PPOO=dGfoaYMeW!~(}yOBNj)#P`- zCeJnCo@2nhz&LVMn3)!INg1$M z7c64soYAg1lj%))TXbi~ErxoZ2_H@zZ{(c7Alb+&u)SUTL{>$epac8l2Rv6Ac%`&2 zcAYtK>%fW244jXDFmzAU5OG-0rp~DMfk`xhG51zu`h$O5`xbDgIQVEEWsR5+8KlS- zD8Rbdfm_z2a+W&RZh^zAmK$DeI&85giuDt#$TB?+B^$LV)rUM7Z$>HF&JdV5<;VvX zTb_l^EY4xc3>>pPj@o;6ZEl#Ia$#D;g=Gr@juvI@7MswY?moRFqunrp+0uZiCwjWJ z0E=?~GrJ8-odW0LtQ8HK{iPfym_3iZzA;N^!OmW*6BRo;7+#)d%q$ZTVB|}v<9A?+ zV&FV0aPi8810r)bUNPWpZ(!zEV0K=>UYDR@mT=B|Lg|ki40;Xc#0=QvPH=A7z`giD zdf*pj$Iq;Wx=}0MaIcua=^C|sgU)H;7U7lY-Eu95?{6vGGF{>7zn|UveWJBnqR-F1 zJOAU_nTwp~ELxq!z|p+m&>JhB{R_DFJ>cHmaCr@&nLfurAmS^NOoG6UYT3hWGA425f^_eeID{psgwXQ*|5 zz%5-bWp});W(Qwog{Z=V#B>IYI!0Roj_nt23h7=Hzsq~YfK#P_(P#t9#0xxUFRGBN~%jsAuQDzNU`aA=$E{KW~phId$> zr&?DYykpd%ccfck&!5BZoS0*7?VOZYRIZSeDExe8NT{li|JB8sQNC;o8@T?3`W)Ri zf$z-*zFiG$fd*bTH?c+ru%cBfvthjDuI2;g`0vlyptQ4#Wyf|GjPtf;yWwAd8UBfc)`sJ4eSk_dGZUM%ss)G zt~=-U4OdmuNJH7n{t9ei8@RVT;MA>Q^K9UV@$*qWT;&W!bcUEJhQUeKs%}ePC%Z;CsTrDaEi!`T_f61Ky$E?}(F;JB>7m)p)eSN2tiH_48jMt2`Df{I!vBnz*VqTSi=jRF3w(xo(%T=4(mz)vEldNwdliDh7Wz#y@KdD#T^ zVgb&|1 zSj}?dlP8~4x+LS*85Ax`eb%CpwIfgKJ>6WoiBAzO#Yfo1y>Z)P&exfa??9}&W zL*QdJUJ(tg1rrnwIx_7nfA{2O^Y?S}49X7OV68dcop3O5kA&HqryCyWG&o(9GCMO> zJL>JVt*57|cF)j_-Bt7NN0plFyu+jf>Oy_|;;+uzZ^Ae3+S)--PF<@`))cJtoOb4EUfj zbEWFN=To}OjvxH)<3S^jl$U{$?5~V8nOv(& zY|;}}SRUn_>~qqgX=;=eL$|TX!xZ7Dw2e&aYCBG7E>|kLmBg(YqscYdH|^vj7Uv~} zYTT+xPnuRKw9ZLhy{79ymd?f_THYL+>mszjz3_cNfWu^wtj)Z1wyCi>hYk@8ec3Zzrl}XWaCO#8!^e9Z5l|3s{-%;+DLFVzj?6rA z6%5RBZx*CEaqZs7Ce3YlMp-f|D#C$bTA0(L9^;%PPp4}K9XZq~zEvSfOKj?iLtIfU z50?msr&&I1abB`etxKX>s;F$KPlKNh{PJW#Nh)8@QP@|3u1cb@QY zD@lYzN-i*Pm3At*&@6TL#6mU!*$N4z{{C5-egawRR+e%ES#e!=%H319ZKfjI4TZbe z%N8DUy31ndzNd$J%w&^xoj>)qa)y`Ukv>JU1C6ch zzCZadah=^7AnWRHsX9|kt%9NH8lT2PE?{^62ES8bY4geOgmIfmzx{i&bHs!NfI;9Zm|C z*V$K2!R=`_j%+)3x^Y$}be(opiWQ!9O!-X!myhIek;gOIRD2dN2l=dM;fJFi+L6B1?&=7 z(wV5L9d+@>F|%W~>T_AWl(M@|YA7@=UH2w+;S=uoH%7~>8xp+}X3b386|iiI)-moR zh9f!(htki}kSA&=6XJ*jVBjHcRR$XxlLgejR6knvusfX`%2PMH@GYEKK(M zb@2N&qorIP3asAfZ5%i%ZaDBQn9#_rqbRvNo6Tg7U@t#Y6Kf-fxT(wbh<{vp3i+}V zkMQU{VDm6-X7}s3VB%B2C{!{{W?2Qp(vk^V`{yJv@NYc-Sk8UES-6bqZ%G*EEagi$k)K=aOaU&QijunWuIDx>Q3Z94?v3V}E;@mf5$-yaJ*BX8s znd&7@$O>T=R@*<}q<)-lPdN9T)`JrgMUqulcb*HltFvSa_aaGWpDSCI&EUIidUOsy zi<&|MgO3BFWeSHS zD=r%U_H7gMym&_Vh{89+35ve{v(HHSDKs%jFfg55tl)yJ$}C2AQg2kq9~wN|LvC}8rs**%FbSmQ6S z`YJSApJX=rp&=~T7C%9I>4paO4Bl&>4(wRGTQs4CtFwjMpuyRYGfSa`Cqb;gK*y1R zGx!DrbAnjW2G+96jS}5W+!vZR2QuAhP|VU`t35_u$J9{;Zk-(@QL zSaP~`&uIR$;JÎjG=pIL8oY24;!3;nQ)dCX*8_h7L_Eg`FHf7;Ja6MlG-u2F|;VrE{sxm)ivEfl(70bGzl3nT$|DF*R`8n!%d{YG0TEQcn`yFW=^La4Q3x~vI3ao zc%}z;aA-6vOgquYJ)=>AxtTG5DMy*z*@7iHLoEG3gV2rUtqa>)Ul~doh%qGSa%L?0 z`CuA*h0Vu+g%kcY>^(nQAom4ZmPdOW!{N9D#jJ{D@d9lT9j%E1Q`jfWv0K2l!LcPV zXbZDM1IK{|l@m;6n~dKao;&FXi`5n;;R6j^9kLt~7(Ntlk#}_Yv}lX#E;+q|CEp%z z*>s~(dB+kTFRkC&yFPI))rp+H?6-axQ}c@3TIUl^t(WR`c_10Hd43#+ar}bTK#iS= z(ruAJTs##WNr%*rFXT_@>E=4n+`mF0`Jv;4juxIBX8jwOc}_6rI7@S%IGCBxCc9ut z^n=-jdl)1>&$_XTE$c)hyFyD*gotcK(;CTk%L~@b0u8k}i)!bbu{_OazoJ*6NTX%- z+Sm_!Z=bf^;Mtz|!&W=M{ z%MAV$Z8y#d&1sD+yc!K$1q>$^vw0*m3O-O1Flxw_W)D_qWKw8clh`Jcq2Q>{p4M=N zQGwCv0Ewp9!tEcklfjxxlfI zaha7Qheo4;hu^H(T`wwK=1B?7y)xsi<|=QYnE~0|A_1$GHZ|L}ZHNC8n{|Xd(tb+sgG6w~mQ@A-@t>B2F;gaUJ zT=TkB=1}&R{U?)F*}vWY|Fcl&9j*Ng%vL8(_$E}-I0#&Xt9=H zdiy|(eMS^tjgeRblbZo6Pt%kVhqHLx3(L(}CkZ4H|eIj<2@_?fySVK|2&f2bf%Z^xZ>Z7tFe>oFSyRW67J8MiB)`t`q-)C+}nqV&XB}pmp!8 zs`JsL2XAlaEoioUkz{qEQBs4EZv%tih9;AVOT#r*PdF{ebA#dLLxWke7t~hy{9Ua9Z`$;o1iEIiU=!32T`Wbk&p?mT}i@>Z|~NHFl#wf6cXYM;1t^G;CL?uU!GvmFCd#vIlOFIX>J+1|Z-O5j>QRkw{S z2feOE?VQ_aBe6D8@{)G{+B+fsmfrV@q#mgA!Ul}sd?Q0 zb{5F&&{j(cUfUOz!uCLnr=clW;zVQ0!or!h=?yKm5=_!78hlqZoXhQWV`Eg(Xo=Eb zF^Rb2VA&wuU|94ZI{U}%xCgBdR&{O_ZseWT%KV~%p(Cbt@B9bK4SXN&O`q1lv7iwZAWvMbc&%HD51 zVM|PHG+6XbFv=(}3TZGNy654z^6age-rjkUjeWa0U&Nf=mhmUz#g57G$4{=%n#r(h zL5Af#e#W(tDaLObmqogkJ&|l^>1LTwvOqwLt25W(L1qD4u*G3njqST%-Q&`jGOd9@ zz@mY-?;k6lK%;cUqR5qNk6gX3Y5%1A^t;-rcOn{ECF~lev4qy_Y4+TaRd=#e^rN_5 z0<&8Qlf?z`dDcxHvs8YvNy_hNl&E0r49#As{9M=2|KB_A)t|O9UVg;n;~)Q-$wq-i z??$8S3`WB`%fn`iIsF(IZ9mS|WpFw6A*tjf>xTm^Wgof4bN_fWT=o;=7tat~AL*j^ zwt7=!Q2TE7h9=?SWfKC}+aIxUHHvr$3V3p`sLx>Jy`aG^A#!hCDBrdQ;UCRu7e4M- z$fh~}Q$_5}Mzew5J|TyqX)78|tP@%zn(M88!RPyz z9m#9ebe5Gr6ysdb{eS9x=Vkq@C!#L;Nv@4u)El;_C2Y~gE4SDSn52%+negED&09wM z65Gz5Y?SC|xYIY6aT^1lUz1@1Tl`Co(vG5&f2|yXwp7anJFyAAUdD7P_D(_tt6&Gi z`=`PB4lOZ1nx1EzGxpC{_h8n$QT4s;y(iP}%|GUOe`?&4#%*(b$pHiZ)kgkdueGZ4 zPchy39)(u)V2Ndq%Q$!h^y%^``jy);BmA__j4g zNwjBQC^V3|9{a3SXGbE_gA+P;Pb}Z|u6K&B$&Kr;;?_=`$xt)pm*;_%&tfLJUFj@T z|GuEAxbLIH`;Cl0qq@Z`+{k`oaNA2#Sj9*lddwEeBr&N(;M{N?&5rY*LtP1Je;rx?!+CzC@>2N&IFJis6r zw(pcu0RvMUr?6awW6O$mF*TnpIfaLgc0N4fcd0=0(T_qwC8sVK!A;4>CrX>u+*$BY z=@_@ThRg014UhaJtXCdhz~{{FBd(ut=!hRLo7p?P4ZC05+m!lU)^?)TjSYp(N7^|A zxc1amHyrphQG?x1;l+Vn2j1M`mh<6!$Z%L#eUgWv^3$%VDxNMOHVqdW9XZ)H8!Ipb z?K<#Mp5N$&ocghDB<10R!h5Z~7C-?br2MLEX+y86{QE<*lXw=!rVths~obf8J z>F$&tR}C^s9Ik1{A6<0HDekDz8Ey8GgoE13$8Iph{S$b>$|zo!xX?8ry5JzIaC_mw zHnARAHue-(A2*gMPBI%0^P63{5Xm!d=YvCB8Wk^>%9bxEYZs3z5o{A**O;i5yXWP% zXL)-|KSm4ZzY09bV|m8FnMX84;b0M~fWZsbzp_i_yM0bwexU3_;`I59e2Rf_lCv6^ zxU?4JEIKLqI?#l{OL#%U0WR$&EfYL?C$+3_(9H~QVC2_HI3UzwV%+O%{c_6X=3Qnt zeEqI_%m_A~7%?%?U~fY)Z=hY|V&3pWD=cz#B>9{tI+c8x`1#a$HaW@u5&*Xm0 z7Cn2*(Va)6jp2dyCQGlGpF0Q!X)Y6ci0&(M7-Um6{qOLZV@EF$SUk|;qlI< zwZGRTdh*>=N$rGl_kX5clNdH7sT`JRnsTXU18AFJ2x;f(_I6DmAY^<+0LY&chz=IDuw;P9D~>fZi;TvSqC=KbYlDnlZbIneEEo zG{1uvnfWcunr^s0U=)rxz%ox`HEYh2K$epy%Ic=eHVGVI5ZrKpb5lZFV1`!ZybmsF zXAbOnR_G+W_{1SLHU(ynD@mMZGmZ%eeASq;V2OR(t+{1Jo5a}-RxxZ)YLxXT?2cpn z$1Q#4msIl;^Zg4WlEwFaII7n+iSH+Gs??GL2g^m2nE25(A zZNjW9k!@x^Td#yKX%FL~*=l9Gx&znv)lF~__Hbwt%urxDRmouUc&4CEU5mm;Psvjy zQ-s=74pz*5(6Y>^uuYq13s0Iuvbx#RV>%HH%q<&O`4ukqm~kjDF(^dVMJ~*PuyNBxxdhRZ76RiKYwB=*#ZDarEDx?V{Nc$x@RgoZc!FG&0T!a^vQ_sU^+8 zw6N^KzbQ%(T8)v*yxGv91rk7jXVQ_*O&zWcHv z;;5D~n}Le_=41iAZ|4H98#b^`N?`L{uqe%f`KZRJ39Kp+54loruxL$M-s$s!N8s!Y z2f0~^?Vn;xtThxnYTqmtTAtgk@b}?e2PFY*nLCL>=O!F8y;VMW!`l=MyM=C6yAp+q zO3q4MUBN0Yqp*8fS4bn5k0)b@=9SP#pR80T_^|K^Fz`AUvYAb2C}Nm;FiWI~qsfp* zyu;He+3aN8KPQ7G!8s`{r$Sz?G5_E=EzLvfVnVUm-5B3Ik8mr4DaCU{7H<&WabUL7 zi-jywI~26_YdR?=fiSy&S+58TSGNuPud4DiCIs`E3oN<`4v&VrkhoNcz+esW9E7+7Q zUaTt@ERJ(cIFmdjbG@<$TT$?*#;F#I!0FY%D5zt&QTpyV7B9CG_XJg#Bn%oQZMwVSe`5m~ag zIZOND@x=>TwG9jdgl!U;B|HALD9AN5vhTROQt(Fuo5BQ{@?(C?Z&VKQm|QSCc>DpI zvBp83s)7=wIosKdA2145ZD5r%QRMQ@Xp+d;Iyp{th1#nHY$Y25dFMMM^PgvQWYiD{ zm92Gu`0CunSMEMctMA>K%4fmAS;k-yQ*`HXhlluXn;&}a)>a34g_#6d{lb2RypLUckUTr=V3khe>x$v;CH4(+%^5bQlE_9QidIMG_pvzA)(~91?xw zBQm3b@5aGxyaxq7Gzjw?%e!1VZj4djt>oNjG!I+yDb* zPLFKG!R#{!W6v_KVqG-*&D1rejjNwd7qekByvo^nv2oo@2C)eTHF}!VCOGoXXyCkY zfa5^;Nezb82_pHNN0@UQL~@uIb}{Kb;J2RRbVJi|mj+{kL1X_}C;dH5j+O~upB&WW za8hw$Qgm_BbU38xaY%E9li@8VO`e6iJP&m(oc|eHG#lG3Hh#cm8ur*k!oyUG!z}J` z^N)Td-|1!w;^rnErf&|iGce3^JRZA8Zmw>70O!FXC9lb(Q|0ZEV&|K_eXPH zX3Ur%a!gD`>3rZBS(YX?3l_;UOy2t@vObu_t8l#LK=3J>oB0cGT(&sCQV=P|u|RtP zlh&H%%~KnBUFf&I$?uUAaEXPQLk2(7cd+9f%O`N;H|JNJ|4`P6$HIG@uX(s0Osk<#X~PGSa* zL3)fj-x$Q+G%#DG-imyn@I^_Vt4YD3Nshxw;YOp}6Gz!62W4^&$_hA&XE5@YFbcjo z@Z>9l@D~O?9!H)x44-Qq9=aajh`GSARFE?5k<8J8`R7@MXOuN21oVjAaa5dfAaJgu z-UlZJB}f0x1H2DrIxcYOT*y*x7U~${P^Q9AZgYS&!9gUUOS^k^YjiX*d-V_?w1 z8=kY(JQOCXGP36!U_0T!mchW5bCY>Xu*XG}3du`XI2lqH4_t|D*){RDt4TxkgQH!C zxOOtmvMxMm>cia8)KE0Vf$zdWl^agSCd@dbc_BF}fNABu^?X+s1Rj`E>+p}+K`m}c zILi&Koi=KloE%srW|=57F+B^<+?ALlz&P3FSc<{Q)Gf~%PYN8U>C9ZB&85I2KZ{fL z2%Ewz&go2Z6<6KL*IKlS^O#V{LGcR*RlhLk^d8hPSgsbqxGlk=?yXL^+m?I1k3_>%M7M2|{*l|cF#*s-&tN-f(rf&}ZUJ9&II{uxj930wD%ki$7 zCcTT7r6htq@1Txnh5zYi#~EYV6?Km@emG$2;LP8{z_vp;wM(RE#q9e$q1OBEy$W0w zuu^X0qsU`-=N8pC?9R>I^l7o;gF`Ailp;1Rl$LrTeJF8>?=eB1h(ZUI443;kK@2lm z{xRmAYn<88c(*9?o8+SNwvFEfC*QYY6iK+6aG_CNq)9BmMXceFj!C*?2ZL+Cirtyh zAFO;Rkioz$z$o{GQTT&{fI*{tK_d^3^mm;joCRXNg>U}%h6NTWiU}|Z&0*C2amXM* zK`4S@=T&zm!3K{HdhfehJPa6EG#KJI^ey{N^$Rqsx~!57KzWLeCW*3!GvsIFMP&q*kD_u*^$akMU$y#g+Am zAH`ag{HoaBYsB!Wh>J_;CTJe-+9x;9@PJhCYmO%kya~_vGLG>{wBHL%JNB@H`wfG* zgC_@Xr;JS_uffzeE}?%P9a7wpBq-rFQS7Bu)znE%*^dKmO_5;dagFElaNtTYQrADT zd!P2bcgtpfxU*{ItW~VR6U`ijLpBJXIVh{aq%FYoM9x7-rAed5Nn-+|*o*@g(^}$A zFi3keY2M({{L=W(c!rZoPn5C8Ar%K_If+TGnn#!;9Q-fc5LkRyErIE^Q3Ah?V^3+5 z{+~miQXDyVuy=<}JLl(~vUc;kEAHiA+`T#0f0!iQ$8&^(M^`w4fzRQPNr7W&L7-M4 zf2x@_$DIRmF;0pChg2UVD1LdM+H-)>=IZ8E^8|dDj4harJD$3_GaK6+Qq$62@I_ha z_H=gHHf4VKTV{R{k+VMgPJUX!$y!jvoUw`btD-y8Bbf&!;yxc(XKzws%1bn96kf4u zmkpydhm&E1(?!>QA(O06z3ooJAqa!C0=nB0wvCNmBx?qD=^IV3C) zCd|sPK=6RZw_vkmCbc>Llyq~NBqJF6T^l7*9ItUUUyEh@V$;s7U$anTb$G2ZA@ zMtyC@_3xFwo!@tg!{M;$i-QaXjm8rWt8B^AS}Pe>!@zb!tFyMz*n!!^LD{6w%p@m( zeZqmm6K`<5Ihe7T$#{cPWLT4?1GDj&hBq_v`DG4yM2p^3JH~PL^vT-Slh&<`+^gn# z$b@Ukd$soNX=ROm%t|Hx-&cKIyQDKFj{O>|F7lf}kD!3ftSaF%5z(KgBNq5d6 zl?w+s3_|r%8Q69-YH}PB%3;{f>125Ckb+OMV#&c8<7-7W4$*}Samlx%`I*}<9Fkly zUn2Fc{vRgwibEMYk9*uX_tt`8{y#o5_YayLB^Bib7Tz)~<%-|lPpeqP)AA*WVI2b# z8v|33rLz?O>(qVcZm7#lQ_!?DX}PvX27XIy`=f`;mXDGn+Q zKN2dKLpH88cVMarU-x{Mo1hDG%k{HN)Bk-gau>IBHf3nybU9=qu#EGF#d`rp?JG=5 zUmmDjX)?a!BtGu|&zuAE9Fr9loOCxd>E3YCz0y?Zdr0<6Lsb$ZUyh@(L-RY&2JQz( z73G~0)-06X#GSQjixF=Gw`4ny>UO5$BBmt{KVMCd^D3AA!1g0>L;jyMCB9NE`^E$T zM(IbEj=vZ=f6TQ$dW!W#1KSLrlL;++E{8OCFxLG(s438NRijbP!&zZUll+e0p9Rs} zH_SgyIVfq+zhbwOzDI)oi$nSgWLe`J#T+(TZuV*qmbCe7wLZ9TZGXku*%B4K=fxg! zEU-DixZ(WFe@}HQn&p-lZmx4Q_GsStylJB~vtGi#gQ`7^eaVg-518~soSYvYGQHxY zxWe{E)9)?Uo8)pDBR(DY@#tvYeI}(HN@bIEcbm8sEZ_3jp-*6rXw%*;GuH=k&8cK! zPMU4cDDlVSdG)`&W<{*QHo_C?URpOUUTq|P=781C;NYK`!FxsL9X-$$cR)bmagBbX z${j(?oCB;bO`21f_%#~lg&h#GX_QuJl;&X);&EC!r!jiBsrsI#PYe(2d8dfpS;y#c zo@s`J6A#zrJ7Myr7R&`Z+JCp#imdngkth1kYM11ZQz8t%Jxuiyn2R1dS+2g#SaU!; zrs+;RlY`Hsw$s;CV=k=<)pj{LbF*PQ*TYrQGJMv0o)dWZE9{@{?QKpzUY629jZNKb z0tGHDOIkaosr${bsXTO)(^tC7Cxu(vidm4$u8XBy2m`S@!I=ZzUYDMpQ-T22UU&7Q7v(9!8g2ZyLuMuh=mYYUs6 zf<=eG!$YiMwF}lnFt|>Tm1N_IIq|V!QxhlmG!CwTt5dWH2J6wE)}7vT89#wX)c`hZWt zBKJYVZ${7Se2kOSbf0XKWmhm^V0_rn;l2I9L$46;q_u`Zs~f#Tn8g3B^PXO`a@qcv zf|D-pS|MMzqmfN2 zDZoo}<)T@u7)3Vh>gv{8_o=IU=DZt^RFr%dS!M>d(X8IEky zc{2{SX$1Nhw)1jHHyH60eKH;t@5Ck{{1xMT6O({YWY3iqW`2UN}* z+w6SQV{~GJQ_D?01wLzbi4}YdF9dAA?7A;HByeixQ^gf}Vd~sw*|!I@l%^;~2eR^@GSHn-RwOQqihVt`^XHF<>6V{or zmLalTjgN_;A=!cH-3&LA!230uMP7O9x&{Zo@asx!Q_L|~z$94|q1eQrFkwN?L9dx? zf`)1rA2G6S1I*2T9Ha*@rqgEu38D#+3N)IoK{59g?-GQuu>5>*}H{ zAGPGR@0h5g-Xs%rat22kQ^gLxZ+wgdb%_5I90bxJOqb;*fYdCWm3njLxFz9A5cQi1X zENI-_^%)C?4-OPwiYDOuU=<=)x2JAgs7(sXHD z@qyTVA8xeDGQL+$WxOFj+1;Rt_aLXY!Xd@&2JKEB%~FpK9yQ@};>}8Mo^&g_!(@&l zujhlt_6CK+3R!|&4kCvn&)#re$Pv)U9&v#8BnRsmGXc(^4o99ygCmMQ6Iz4soM#g2 z@X3`CU{>F9kj;Aqvtkau$>Dofr3<)TVC2`)V_EZ{^;!G&`3y%0$F;Y_L%(Imo}=fK`*Pu)~8zNuYt@h{A+f?6xeA_!=!- zjnx=AUw1l*EI)D7Auo`(EQE=Lm4Sg(#(+UqMoDs+1*?eO1a>!_Mu7=4SQK;wTGQqv zak3payHG@c#na?4Z&L=FrjB7VLk*X&?`pr_vobZ_u^iiQjI*z0*9D&F4W&{F8;>$% z?3lyuqA+(d10#n=5aU_FhP+oh4@x|0?oO6`H$Q}^cI@vJ!_R zmuR$_lqhxuHarr!_QK69XJfDB6yc5H5)RyK2RR%!BrqQBVGw-7z$P2Rz;)K(s6fQA zE$ki+?8_vMD@?lBW5eRazROdmT<*XP>GwA>GMajLj567S?ldts?qK8N3Gh3}&>-+Q z!F7`22gdA+yezT}2dufRk16G9e0R+=v5`>%v$!lu`|^cRJPJQ5YvJM+*j?4jWPGLdP#tp1f* zhVz7P9b%ayHHpLjfU|z`kY+PhD)@@!xUo}t9Sd?6vhD4{*YV3Fi4Wf!)NY4bK09?!8d zdy{lEeFqV*k7r z6%R!-u?vnD_!T>%_!sebsJ80n1#)C;NMeaQx{*O_!?F`1(hNtJo!KyT0#BL+qvaXR z1_quFEq_#-rTpg}W|($agWcfr2LHnfWh-VLbV+n~Ql55D(?_r~Q^%3tU5())+vOio zQx2Sxx9F^rYIv_Des2Dx--qlke&C8@Sp1;%vFqav4y^odj0+l_oe#R6?-EvEl8XA# zkfHgZQHbSHv96%%lU6y&S*pEdc zqyG1s*h6Bo%DIhZC~$?JXp%HBdWN|Im116&c=8^|1XDBdSNVaNQc9dhG)JvEZi!POpU`EQwMI3x$@oGH|gnsVOkMbGYYlfI)^q z%VEKmZ*NceEf6bf6rVKV%O6hOFh;QyMv1sYE{=6y7i4r4DV~jR6m@wo+Q-VRwovMY z!y0x?T$(GS0Cmz!rIIA9TpHUa|d%&e9D4fYCYBN#zpSfd0)B^sb zHJly`G%FWyvMdx}YUGgn$X51%nXiCl^N#I>)^2`x}C*rXg>_c4Q|+4_JMNAm=|pOZe=F<6u&v;27=_=kaAE`fvNpFvR8f7mw?|5V=m#x84^l(1MrU!2RqB{%}95+7*Ni2+7|A6VaLZ~D6%!85( z8o9L;*oykY)J`c)=iYZ9HRhhcu2ZfTJn}R;9%L&+B}yqJGGA!PX?U#Hrof@MfR{y)Uys4J zz`!7YgHfKlm21H{gDFgJnw;|&Fla5X)8_MaYhX!XU^Y--;%ZGKM2bI?R-$AC_ZDM z*W8DE1`R4P4|e1*idQA&gP2qtTpF?a%HVYra!W|vl9S<0a5?EXqlB*7ID=g&jN?><; zz+9EUuC#z#jgh^efxD!H}n51mMDr>Enq&8$-e5%=DY{YHCLEd zEr?&az%!Cjtgl40CnO=>QFK+pSCKNVO>4f+dm)*#&{0b<-gJ2w!$Pq<#gZebC1*Th z98`EKgf;4fU9U`G3rJvf6Y_JP#Le;XVp+=hLyInP9Md(?Rg=6ZI6Z-5O)Y9iD1YmqRFk<|I;Aw zVu4T-2VV}OP+CIx`E9jj`TUO#v7}uPd)L6i)5*TafqQ}v^RyP0X(?=55`_LFuyZXG z+VMa@?IHKK1N>DD>iUki@E-*8_*nH~BV)vyPUqkvmS1TxVe&OVu=8?FNYgP$Y zSUy+7malpTi~c9`=s88Fe$M;CkX(E!^5>bIE=PViEpcgiq#Wqy8rQ%&YyHoj&Azoq zo+a^BBs}_cE&E^X?~vaW4?Ie~309YJvYFN|QsC|>6E0~K`{%-SWC4$XA*Y1{j|ZbT zLnBK?1K$@0o@ov2DGe+p4J>C&TfQ9-DqFzvFPWn#!K;Nqj8}nUMgxlv13Sw?H`Rp# z>-rSt92DE*z>={=WY+;ntwjEH3uX2!5WB%Bj0k@H-#mwq2&BDmZS!j90#@oYHV%} z?8iQFet94;=b>26LB*SmvU?h3uRIj`uuSHJqtV=lLUR_%yi=H?k|?z8pism@#XSck z)f5W~9l7uP6gae?VCMn>u0*y~mHc{)LK;W7qaI~dFX%8ZD^P0`-Pd5ZYN6N$2HR`R zWsV9gc?v9Nc2vwRDHr_KyPAP_RYGrYV&CkN`CYa7n$F$!`mZ1;M=z2G3{hHP&~ z1_7^w-dh(uzEsXz@89FHj^oM#mw!!4-d^8W1HQQiiD~{>y2ql-dCPjWOr!M2;#1@c zITjp9FKuAopdiqe!@Yz-#Hvx)YT;D#hr&jU!Uh+)COL4f(-U+#;G+7YQGSPuU>1wl zgPf;gR$G9L`uqg5o_lpI7 z>eB3Qwy_&XH@YXVwahHB-Ob{_u+SrXoppn&;{&fP4D*fEin1GdIG%`YXvll?^4>Sy zMb-)|V$%|>XNlji=;iM7a1<7ca;$K^#a!@Oew(B0jfc`(ehTeyPHMff6UU+)#HufLq z`Be@|E>hq$Nfb736pLXNjy)*upvdXdz^SlMlItK-Pvp#ehnfA6Y^zkf{Fe3RC9otp zuq7SX&ZWS>)xfQgsP~Rxh0wtj0uNtBByMqyXmhx|*?xhZ%YrZ~hjpd;Y_Aq+n%~~+ zS;(>_)UDmp>qz>HVg~l218g-jIcl=mTNDI~7a5xcBmu9XHDt(d~ui4}Y18 zh9!zNI7<9ulxSEed&^NY$}!=0=yv9F^Vj{9zp+qu-A;ig3{q_lnoE3`|11!bo36U0 zA=&g)WT-md&S;Jvh84LClOh#Zg={A&&-9Icpt|qhQ&lsS%12FVyZMFYMzxe|Yb-V6 z_!YuFje+}H!ulf&=@|=!tsaU7+!d{QD4x+MyygJQvB=QZKIs>@ErHJ`5~JwzBv%a2z>)OK=Cf z$8?ToOgFa5-q|h6ZcxqUBf5F}0(Kt~=u*>~qRoclK)bKR^z|-0T>Q=UKLUOC}k*8;)WR#WU}vOf~H zZa$D+JB^b`@!-)0=5-EJ4G)UWx%bliAY;aII z)XT42rXv{Wbh1;(RxQs`@eqrfu&q~x&cur?9747V-wYf#s<;c({4rA!oFu$WI37q; zP596%Z=18w-Mh8s;$hYJ9)ZIgJa;!Z`>mNXvD?2|rhtKkwc!FopmU`_6MOIi#;d^x zw;WFHkBpt_6`Z*C(NvuSJjP-c|5G-zi*2t6Jjfn&D3n{TRP^A?xXPzbrwhj{F^mw* zp6HM}Tp!{DxJ*N;}^?I-@Ep!_X9< zz;l(cSJB7NRn*ez=wf@m-aHq_tu=d-VF#FvKCQMngayX#(B(sg%Q z?6;>bQuVEwGZH&l7e1Snv+P0h?4n-`EOUJvHmYX^2pF0OR!F^Fp1o?Pn9D-zBVtE# z9Cwtpt2E9i_}9p{JfOIPH7;3!fvqf{*h@HRLQ=Qb<*5eD8zKq{v%D?tTxc%W@HJ=R zR5>Eh$je~i(6EkmN&&lO?1_YC0j?Fhc=N*F z{`VK=jYou*H;L*vHmROASl_di+hY2MbnYoZO$i6(CWjQyJyltGQ#G!kR#GCdgLC5y z{+_E0H&acPD9oJW@Fix}#l@Bvmn%9jn%2A$o*(gemea=tnb$JUF05c^RTE(nQaj+l z_NkyN#U+uE?SMnAnZm!0U>+x)KMkjkbKg&S#JKDahuEeGt_CvOY@Ype zPC63psy#* z5{#wZ^rZhzJ~Veu!j=drfy^Z@gR?_6A8-=%GTU|&TyCK)u#4;mI>T#rLSXn_ZrI?N*NZgXgDzP39vccabz=`(8wlN zFrU9gD?j-PL#a|72-l&OmZ^41`IV=oQ$iXpa=mDE^>8avG1&Bcp`5{CZlzh5WPQTK zbsLU(FL){R`_Pqr$|`EUy*nf47#u!zk;T_!H(M4aZ}H?YnXp~3@fC-; z!$HaN4Q?)HlEkjAXp>=NV`?bflECqpD7^>$% zsioDW3z%>Eyya|ca1ov{p(nsV$x>|tL$gvLm)jgA{yc+LO{K#2rYZipwhE2qP66A` zUY$_-@nS zvTPAo^h&2-vB*31Pnfs`wz0~Jou0r^;kc{(;Y&aJlWYM@Y(5QwO=aT7_GWt-{C?fO zFF5a3$XC6${$9!YmlGB@D_&9Cw{Oi^nOCb@cyAmOS!mHB*)f?#P{om_Ie|sGM1U(; zAc@mh;UM3hfOU=+4IJ;SN#sA~5paN!?=XMHA;k|4EfOW~7zGzNu*L|St9>WD<&RFI zoxS%PL$2K{QXT%RUINWRClr`iFE(uPZeW}dQLGqv>G^^zb8eRhN#d<999SmROj&%v zEHOIMwIK1L!BfriOZ+Z+iN34N4;3jF|0DKUbwg^mt%v5nvxyS(#cfsw@7obs+x(Mz z=DCsj`CDdF|F|H(i7VA`2-jk%6>4Y`79EMI?>20 zbm_ImQ}>%s7~k4Ge(Wz4vD+@W$N!a^0`rT8YjZCc{^n&oHGzMD827_8R=qim9YPwF z@7)c#CeOa}?g5kYso=DE25QHis)aukmAKui{Qo0=se%*7nhVU)%S6~*ERwCSU!C-8 z%ak??{UH(GqZovWOE8i5@*JOQg54p=My+QAtTg|S^ zvKv~QU7YxG4>u;&-pvtO68o>J`P`q&`+S6-=$7yvYxutWasBy?mgg5T-zXE(X%t#f z9KoyTtUiguu8~7gA=MYKUl!` z<^Y#%0B79;CgotpLO25hy8;nfRO^VvdL16iC@L%JOc?{BRt3SySrZr`%3rD0Rt zR`ViXVN-4uQ}yk&VXABsC$OqDu%sDq@A|;K>q5=R<(xqVn%)!GmrUT0IAHego@2>O zX4w_2TAGP^)&)twP5gY1JM$T^v?g%IB`})>gm;B9N?nlsl+I+7!1VPvvqb_^nQ)@+ zK|!Ajf{6|#i3eCzKd@ds!raine6B^PRe(*UA*)BjTGfd~v4OR70!P_(H&q1#H$wxK zWaTdQnD=cqwnY{G+fr8+G=?7FT3^7ns(`JGo$qM_hlK%W-G}VlM#h=XdCCuX#vEX? z?hwy8V5e)G{#Mzzn~`P3_Nq10jejpLbl70x-E7iuDXw#xxAOMBK#q9MPt{u)S=$xZ zJR4XRFW}xcf$#MMzFiG0;SSvC4JVQ&4v;WNR>B_ZJPM@FLyPHGESS}zE-7YMaqV6_n7_#o&b^3lm>f}*Vhd%Ocv zG=qZ61(vF0a|1zz*~JlyikKffv|PEOOJGI$Mu~3LcNb5P3aF3@qwFJ@(i3jC$O$~z`gH+!Hxvh zNCU1d32ZVGSkf&x|4b>ne9=LETm8aESu>66u75G-2=HUwz+C6R5tqP}tITYj5b^k+ z9?!obA(;>LtR0MM0ak(ruK!A{a*vx;ZMN*yNc?He?EJHRlknW!9d6SLIQcucCp$1H zI52pHi6{y%`BYAol(798mLIS)HMF2Hq=7BeK#V_vSIdG^&x1p#frkJ5^X z8NHV^*!>cE6O^hJo2V_Fu2#Ns`d8t&b=&P3T{2c&jBB6i;QQG7<5cFW6Pc|pEVSCd zZ1td8HcHyRfTMW=OV|YN6(6|Y9^iXfz;SpY=dKT&Y63IazsVSHV7!-@wVuuD%Mr7R zr?m^8Ciy>S5Wi{O8o&|rAu*rB?K6v@#z#S!4OS8x7*!UuEl}3xI-&4*VLjh7OIF3) zWoAqk8<_vag(zKI$y!ptUj2aG%Zs&o0{2w|&PAK2I2+FMooNum7UTbHdE`Q_@P%CB z7Oa&=xH1~J*DMg$4C3uLz#J^jY<{38_Q3qA2H80(({-H}xEk~px=mkNuI5%Sz2;@? zgBMj&;eFo03wMgNUR9Lt+PP3RN_L<5B3UbzJO+;D0;Wg@?pYH!S1=SODX^v)EM9zo z(cFv6MxkCRVOc_#mA#hsY-0@<2CdDDvIPq!TDNE$JIpRWz!L4iQsltyvO(ZV+fsSQ zuuS#bf7_WQ6qubqw5{L9>|(%lVRjEf@^g6GV6gn8dhkf>RaOt_3bT)EpY7*T1ZK-P+fF zEUy3OD(P1XWlu2&Xs=G&wa}NDb!QcmM*-K02b^;XSkotP&P-sN=fJ&q0*e;|E88}f zcV1e}hnNL5-5wv*a$#_06P)6HfYqjC;xiHEIsuOCo2+#Y*sBfJ83;0#PGoitV9#}C zw_L#JC$PP^fn7dehj{=?wE|1b19s&J%+o)xTQ87Rc3^KYU@lSMur^?CT~Jb7z@qBF zlEuQLs^stUfy3;BzQVs?Zd(C^axNPm%kFBfm^}~j>bz7EHgdg*T=B@0wKc)oxs9V$ zVDIDruB8c;h7w9X4qQ7JIBb`0E(%!Se_&ZL8#tNS`hl$6tPKAJ?ClE#QyaMEG#r?5fpeV$_qq+-sS{WN3^>@P zvD;Moe)_cd;S%#g=k=AsN}C=tY~SR}TEN_RK!e?YsaRmQxdMx`0yFCb=Hdy=Mh+~; zre*Ujm@U#E&bMKwfB>^gz-|EsMkfPi1&4KZdzkqG7@ZVEtOXeDOqc{dFg`ai@Q-0| zDr9(mh51|px3$4W>7^-;W^Vjz0slC*6>vxiFe^7R-qPl6eIQ;bxZu(Ww#zFv zU%s)~AX(z&ZV8hQ%&s@8W-crYII-&LL6dzfrpm_`GU!MLYOlVO$b2txt6u|q`vKYsB^K1bV zpFLsV4`4VujZxd6*V2Ho{Ge z=qT14Wp7}Pk#YWGfTr*~U zG@SCdF-cvG*)@TgRe`}Wg~2F*UhYT{3N$ZhC=fRY^<1rRjfvSMO)PQ?hm8!-ETdNpJn|_2fUR1m=xavX>dqPpXi{U5BgGM`YU=#+D-=Dx%S24#ka zB_9~r1gahST?XK|C*DhP-mdbH$US5*B*llYXV}XtbMKefYJN_*VYS+ zkJvc2UWmH9f!)v}!0-aggaYoSf?Ks0SGwI>X@A=&$AKkB;I`ugwsyr!_6^flePOP@ zIbCu)v-1a&y?<|Se0BSa-0H-A3ok3)xx%P(_o1wJ0sGPf_J9fOI}A8d1paXa889$! zU=&wi4oF~EOi5_FASmr1w}yFAf(mn^9kbDaoLpxnrw@J>0W6BQas=ww`6oE9zIV^& z!F{U)t}_ap{u@5sieWfb^Ky3F15O3TnR3^KE-)<6W7O(@Sn}YkfB~ae-9wcG?sFR+ zvMiWV!DZ7uce&{$#$W@sJqkCY_~$1(zDd$yQWW?Ws>GBW!292fEiQn!eM66)1Cw(D z$MratN3(C;Ik%E!*W1e(ERXcwEy!YVP*~;I(DL@(^mWHiZj_RX%6q?c*858bYm@zC z6Y4JiyC-XGz*45bS;fHGSiqg6z;^N=qvQq#Z3C7{1&;O)9Lo#1TNbd$3R<1pYR&x5 zl`-ym>01?M%Lgn~4(!GXEE5GdCnd03O^|L}!1#1dJ3|74)CZ193z%~o*e3;W8YR5E zP;-Wx@0DZy1FmiqK}$SitIlWy z9>`rgSM@F5Yn(1*GKYj!}JSOC^ zLqYVX#f>NbCpvjKtEjVuboQFuXi(yvCgC!{k?m@L@-oh5j#o~LCIy{R=UuqVOLO^( zU0m$4PCZ)e!qYZ1i#ScYaCoNeB8!$Oyh;lknk3%Mc*4my$KdG%&t)@H`1O=D9yILc zVtCLf$g{bzMW*8411AB`PZ166iWMHurlovQaGa3Fb@2fsLkRP`CiVj{P7@TFR;IKu z`4l@avPXHZ-Ev*#M6lnflxznkwuZwzLiT0F4;VR>Zs;~~7$`gudps$69rMd+$}1Wh zr>yQ1HHlcs)~$6cp>?|9g5M9ApHI8pEE&f2Fo{K7&GHE!uSUheR{k&rC6RPF#Y3FB zc2$o$C+tXY?wfR@LdlVlF=a7tI-`$(6G!n2O|K=Je7iK4=5PdR<*%H?$S!$6H;`Sj z;=J}#k#@&=ABqP)1$}7xABZ!-5HVKFAWOkuBbTpp0|3v$LHMXtbK1x zIpUb#H9B|RIM{f<>b9;WGtYs8-GUB34qSRRBU!J7El9OPK`X+sT{LW-aA{X?tU&*z z;PMv_+aw(>9B31Kp{2+sqH(E#BWR9dOQ*VpQds0CH6UW3V!J@vjfJi52@`t@8tr3t+`c=3bpaEjMnRLXjC@YUVb>3df_WXy4|%S$9cMktW+}0Vw@t&c zATX6}YVx9xQ&%lZ9xmW)w`k?mJHY5XK}j&|LBoH(21XfyM}qzeteSp_R)G_mcx+!d z#uj|w@a%BpjFezj;9~6P5ny6&%xII76XY~_sVKC(q3Mf{BNNkyOcSn%0}DkYjqSD` z^RtYOKO1H~YawTs(eyL(nE4DGcsdTS{3|wLmfSRfRb`F?%S!KA%Vs%iJ`G4v@Rs&i zrn7VokDl-J<%bej3%52&OxVz*(Qz<&$;NiQE4cbs8#=%&5*|9 z;Go*$xG3GgIhfn3O=m?Ai+A@eZTp6mB6or^AN>owrCa>->Mx$jN*Wo_wGA_wJw259 zON~1>&Rei!@6oUJMH8H*UT|61gb2@W3Sd$G^C~=-t!#Fegi?qe%T%c=32bH>6Stph z)vLbudET_wcZ8>RFw2=dsRW6C63$ zD>v~v9ANcSP}){8rOjm4!oJEG4$RUk4hxrTV3fVV$kkUVXPS~A{?+zVs?}6s=D6g$ zR(sucPfdw0H_cpeKjZ*|n1!R*35T{IGg00&i>0FH5-teKBxIgdWLTQgsVQ*cQqXdt z^LkU+!eRnd4(KVYX1^)MEVsmz-@Cv``bO5Iv{^3es=5EoG5z}?oF}+RaLR%s$|gJ! zZUWcT->83LuT{n52@0>L}7gQswo(Z$ZYO)G>HLw``V$iQ9kg0>=S*Z)=XuDa!_ch2lE}Vo&La_>qmFYvg#@$h z;W1t9$jhwqGR=JYv*rD7)@7;`q&uhPn+gjw3eCE8B{=OMm&XPtv2_6%#yyL8)E0zX z(b;<7PuA7_yO)%(6@G2pCivq(+EoWWB}LwKd!{O@&2*Mt6v@uHIkByNM_R|U%b%xWIb$%X6GNiVv$UV6+d8=DI$ON$A#WRvnICJmD73 zl2rvVEJ`1yiTub9_y2m|D|#0b(>K+|@DJYNGX#V#&6wTnT4Zi^#_{U^MzK|_;b-?* zJrdl!twmEOurnz_Nz&8&*Ba%C=R~Ggu)TKq7}8|>MpQcbwe$o=&S@LigB=);ulRA) z>e@s8vI&jMK@3OL&nUDxa5Ra47OQ79a>(%r=zB^xu<|7^*?u^{z<9tx@ZY56?4A!E z^j?;3;JYKhm1UC5QJ8i1=NFN?7V&ZZFCR~43QHE-tWaaR*gXEQ`i2Hhp9hTM2O2fk zH{9w>SL9kLle&=qZ&lXx2NntqUyA&$GQ^r4VBKf`bEU_>Q1LZ$4v2U%8fx!g5_`fR zB$UY1IOnBLNN~mThTY4QWDasf&R!&OGQ#TT+5CSW)t&#ScX_FbSv`9M;Jw{8kpwE~l~Hhk?~tf@Ml)i=#!0vj$7Jp#+lxgPZ28MK_s*D?rEM zSyc#&U1+?znK^3*tE@zWx&>?2f;ROXJ=YQ}Je+!2DjLL>2+x@`*-dNyRf`=Lr2p-0 z-fhgDU@>nZ!@SA{p%YA5o0%Gonr|923Mw!P8!*Z`H1$^R)s|pY)oA2sSjwto8Pm08 z=|Tqa1q_ouANpf8mFvUOX~A|P35?H~)n@Ky^{8Od$XFe+y7}nB!}}R!>|Qk62ed4% zZcA@qb6`1A(7|SJWAALT=4jEHaxQW4&AUXSm6Ee%B|8|`YBR29ZVnP)`RB~)=fE1v z&}w*~No7Nm|AJP3iI%httbc`@Z8MHKNi?2Gn8JO4f#-z!0R=|M6O7ggY?2HMc^@?J zGO*O&WU%&V6HZ~8r^Rc{xR;$_ujNBYyFg>tBx74f3%M%?7cogOJ2bf%G~ae^4Sdib z^zTMP%_|upfkwlQL)se6MgojaBNnhq?OVg-eDAb=)tU7V7dLD;r696m!8Bz9DFwAg z6GhJ*tsV(XoW)HURjcO(Z?#^d=DvU>jibG|fxT)$d*uT5ih}lH3-%I;_5u!1r<%5u z3by$TT*s_DMWc1ptL#^AwlALJlDzwf2LlVA1arU*7QY8A*FPWAe!vnX(dxLPH7#SU zg9eM?1<=WedN-Pt9yC3=)a>x0$>svH{^Ch#Cz#bMn5<_o%Vsns%CrO<9JA5j#Mym zsj4My0?RJO2|IqxHuI3NI&#={NlUsxd&La)niK7H7uxF{wAa00ub$CfdE-Q}1bd-C zyQ7WLQ%#-HAeU(-i{fWgT7SDprg^lbb+l!6u=OUiOc#mmuPhD9KQRp5+T z?@a$wEreOuPuSg?dmf#LKH@#fVGdpIsKX)taV-VzwVUizT%oMxk-M5Dxl zM(GYl9t%e81^-&FTQeKouzj$&f$M^MZOAM(2IsR^&OH3=esPV$#W%+o#FmN>W#< zx_oGpUt!@%_cwE-Cp6mh`qVt~Q|DmuU|67Yqs31n!oou1 z4_Q~8A4-PpogZyG<>$fq96J~e?`inBi$SzN#4Y$@XTkcSFKXJD51iPW^WkcP|S7jp_@3zZYFlGO2Y!z)%stQ(0@)WIBcV8o8 zy*EPb0CVtx2B8m|<(*ldm>vz}U^TR6o!`1Lc*ZHc0v^K;5}q2Z4l|mq8CV=HFdNL+ zm!iP3$wF`HBDU2w2ZeLmY%eg|Uua%+h1vE7v%WHmM@7r8UY0+p;+_TFQ;k@^zK;LL zlkhJ^lj)uRv_)+h9`}W9WSgpPmn5(|G+f)Y_4b~-x1}Q(`81dVDr|inngj!49tOv7 zH8kjdxmf)5P?v-@(_s_QwCmyt;Yo}BHHvOvT#(CF$=hiqz&U@nbM4)5FNgNatT(R9 zK2i*eP`ts|V%yZR^{%YOaz(9)g28g%yCYO5b$z?s_(1evg>Y-Ig=>gFYal~w07q-I z0E@H3)kQk2P6{m17OYMZEiVqn+g6Cg6tKlGw0Iwq8nB+yFLD5o50Y!C+$c4*I@hSrpmz+a7PJgv?e_`{1o8j&eZ5981T)&cg_nK;wv1pULK)RwplfsQgg$c1Y zw=&*7I)U$_n6lMGt+GWj1%_%Hn4=|_j$5<=7ImW8WkYM)gEd=AHz+a)yLq(v8#p>GxMI72*_xqQ{YBHu*e2B% zF&Yg_wQWu7TbZOUq_25<;PI(O-WiGM*BW^?Gzx5Dv5qxKF)g-0?UC}o*J^#SC46uGDI`2ssJc-qv}jetkr*E?wHeJp z4;pzRgo6cIcnumgI5Z4TFq<1Z6*8H-_5$-XD;CE9CW8g>ju)EJ4z#AtXffbm32I;o zZeX!AYjrnZ6)d?Yzwwa25rfIi?!|{3{>+p=Rl&d*;mz3K7B1WCytC&?-()U{`JWhP zEz(-Wl@a~Yg)%)%QQQi8UpF-ljoL>Kt(>hnE%@bLb~7T1Kj=S z`meLlN{M@?fbit!0+q{U78qKeV0ivha9S9XiozyGfo5Kge@x~bEh!4D`jdt9A2jK0 zXjWqgvC&|h|FOyG0<+->7uA4dr3!`<-=?z9VEiq6@F}0uEKaQ-6OHX&s$6>PxL_g! zR|bO!$CRrMjhruduX1SdNHFKyH2d6WF_vIe5m=fKq!Y{BDD1!#>%i)Np~cm>VRh0S z#jjypAFd_3g^M$seY$(9xW{3U1&!jVK`!NtYN}b+dRlhYHN81}J^S8u*MPQ)8`&K1 zvlSmS`q@8{{ZJCP{(~|fQ}MQ2hx|&VZ+xiuCeY6N(b}Tb=0(r(K!8e}!hoo2K+C@?DTWR!W&6!b!lN25V-L7CjY9X;$j@>m5LUaVvI z)hU^s!ocFtaEW8p!i~oF3M|@x^R_xOa;|6)ox#A`@x7eAkF$f}^P?Tw3s@IiX;l$u z6yD$>WYHmPz!>m>)qjSQ@q-4AABQ}co%3_0xdgsksA1GM$v)NFC~~1<+cHL}gfA}N z8O0Kknev(EI-4F+lzDe~p<4i3<%M?N{-25xOo|dsvKuP*GnDS%@UxhWYeC0l-u(j4 zW4M<6Xsl3UvX02OTGSkzz;c{BHPEhQvXNu5No%mcw|Xf?nHNlQ3|CU+7YJTp(oJCU zRA^x^n3S`h(Q!jd!m5S^UbdVoR!Y0{a+ywkvCFY6eZJQ5&2vw7OC(JBCv%}e@6v8VVFHFfwomNtk3jSis24 z$t&YkARyS}(kP_oHpL+@xy7^EO@wR7av{_7RxTS00YxS@!?ZIoo`+6OT^+aYU&Yx= zELkFJOZJt-?hI<{;pCRr2qchjwD+G5{|M|wqr&D2&c+k1+gFsWn#tBDyMc+#( zrEGd6Ggga!B6+WgoD0vsxp%b9xQMU&~$5(n7b(Ll*md| z@2Uq8Md_Xj9_G5FZGFh@fsmpNB1twk{n@wUOc{>lf$>!{s zI6b!J%fjnMFRpwJPjpxEWD)H5Jm#FzZ+TLJ-_b+OMIcyh+u@W2ZyOG$yK^KS=JUBx z*r6$&v!RhQ&?WJhLfQnuR=zx!NoO>d`4o3+E^QI?@Msm-#IDfd@aU*YtHQ2!p;m`Y z>?-9i5}f#xIu@;1xKzOLK(&yM!F^^?n}!AkX3qr;&fI|o7oGV6LIm9;BwZ%zPE1hf zR541JaNNXYf;c0qcA~*cZ^zh!oa!t7eB##KQu4?%KtDCGX}wa=hl7D4EgiWb3UQ%H zVTKV$|2^W;>+=@tG1}yCBF}%%)@13@2NmKI>UB1qvHq?wGe6N)$MXo^Kfh_I2Ntvb znRy_lt!=|Jrza0uWLxhvF|w&C1hT2mmRaqo7Nj7^s?g-%VUTJw-XT%sp_JM)$y;}4$l-I!@*)X=taJH39AH+q3Aj*mf9>}RRYEJD zuW}Qf5#-6%)-h z>2`k(7QesH{=!9_U5_KtbMeRN>28acs?{Cq^jEndEb8W}WZ^HRX5lF2;K&_o;vp_> z!>YAp!?IwHW~nPXTs8l_dvMI)4x_+CjiZ_#6WeWgm;|aATr_trV=+-n5}2jYtXI_7 z9hJf?dy{X+Mf#ZSz zi?LiGV{p%5HbIGg=MAcj95D~dA}cuGsYSFI@i_6gS~75H1u!lNIN%)mQkr9hk+KfkH%L67WIQ=v1SU5pZu*0E6TyG<@+qP4zaW{ndg)}--PB<_~o?tY& zp)i3n-r3X#?8cUgu=5%eFkaS4sU2eYGrBk6EM;3{%QD8Ov)EIaEtFPFGBB{y)6Z@{L zv^=+Sa>vIh%VjzeTB@A{P23Zd4r;Ady|I;V^Tdx%3Iz*UgeN$P@A7zIQzF=^GJ{cM z!HGi(vkZIFraVesa>G~io6GU7a~_Foy&7!fwUJp;$C2I2;gHaY4=l1@7O?mk9AZ~G zbcOxGLdnJr4sve{j`M2xvRyvV%*|iSOa$t1Zr!e!F&og$Z3A!siK5HmV((YVy zs7Y+00;}PU4UaMu79YEKfZcG`9g{Z~oJ+oG@Y%g#PMZ4BPg!@HLCFH<{Yy@qmwxZC zdGWf{8AJ^ZFo9Fq9>6@%HyVV70yTNJz=xu!hmW z&INN4MdlnhvT2h5TaZY8O3R7EnhFbo8LR1Y|CdL=aJ<}k8baJ1+?VPIe{dBtSa z(W1>hq0we>0+Yb}241%efdg_rj352#)tOBC8)g_hU`c2&VckP%k5iuP5c|=9xgf>vX@po>DL#DRk<9EKm{ZX94>uy|inq`>Kvpukj-v5Rernu7g< zg$%3)43!~F3~pj?8RS>6aPlmu3H!T#H=AuP;gRw;i&MXfpdxjKZBFf z7X~qfCeMzBhKg{$Gycmh7{zZmaN0C9$S??qFq9~t{H_^eCM8SOay&o&@|^%Gle!)A~q#x;#uCBebbHtv1Ey-f{HYAuI^Uvx<|PSHFwQN^cO zw}g@H#X`;-hjcxd^*S0QCNL>@I9c9sQk>xErFux`kB{X8Cr%yt<(_jd-8sNtsAzJ`%6#!2zP0XdOFMLtga7aDbI7{z=JiI*^Hdnl@G zXc7}}(w*X<_~4LuOp|fNK{-9 zIAk7kNM(hyZcLNa1}2pihfQl7v*He!H#{@na9D!HS8)TAb&ON)qeF5Vn6*WmEOQQX zt~mTCmDxJM@j}l*l^beGQ<^P%99TEB@T53!KRKur)U1$jfR*9uzfJ))rK70 z(6~g5&WyPI4WNMj6bedzNax2or8# zRJ_y3xula*gHeIU>FWzdO%5k+87Jchha?Xik(3b9UK7G8bKonl%e^lSrYoEkHJ(c- zH0AakG!tp^+~maW;jH_G!Fai2#!$|j`p-;kJ| zqq^hfjUA<_|5p7|%IM$}=WtRnJSf-mIJV2N&FP@Rj)OuRhZJla^E!IwPH{4JX_9Vi z{QSmIt|3l@@6cR>Lu@t&qnVtn`Wn3^GKvW}a!hDD%$!1IGq@Pk9~lmoIQjhrVAik~?k z{$a7$oyKid2gL*qir;BW*Eq>|&B!syf4sfVmUZCc>Fze9*zP{^; z!Z$;OZ=L1fJ;A`eL&fyVLB5AaH*bBye^y;}fwRqn!*@d;$iG=`d-ui;!Fkg|;B7h=;UI%!@xl>63A zWkLgQ&OxCU$Hh2;IeQ#9t}J%g;>D8T;C1hS@V5qzHwSoZ68TyjIREHA>N~)9=K%Ye z1AI3gb2J>dU&p}F)3D`09Lo_0&L0msG8!6gJZ5)k*s_MjTc&}d;lK((NB#rz*m4{; ziuoH!9As&j^-nRFizTZq_k>@bQbk%sc*NwmP~rH6&lxYhQ8H>0{=&hZ`sy58r&5TM z#R2BfOOICU(%wH$D^hMnM#+OqeF^&ip4(nLsC?s4p)2Ft7AA!fkuNVCbDth`@8nUO z!(?p2>>+tvc?HiI1+K3gO^Q>RRQ^3^V9aRv8p|Lo!o>MTL)nypp(p&v)OCz^7+999 zV_Dy_BDCOT6ITG^R71(hS1i+6O`gxoI>55#z{NK!Z)osxY<79}i1$t6fdg?&d^`tb zOqk~xIT?R($GcID4+mNpEp(FZ$`UDGo zc?~AjmS*cS&UPWH2`^;#^T@Fmd8V7_?znm2(zMjbTMg3<4*q_1aNV;oxt2q^GEIs* zG;Ta_RJpP~Z$^{CiuEcDjfYOIU1Fkmbru7^kCWyD2L>J&t`m$5+Vac_E}E7OJO(~q z2@UdEzB~^um8>~Z-FRxd-vJGKhhx1TH1a?E(~$2?%lmL~$_MTB4ELEj*nT9kNF3mJ zlC7xlQsBi$WgmvOUmCB3p62_Vc`UUxO0iGPFIZ69IFRi&_nWKSXV!STIjcluS$&<5 z_%TP-!P(}F^93O{d!1{tCmh3+&g^KFW?y8;X{RfttZTrunAhN-jEmFhIZbnI4hp|$ zn|H>~=t-kAgOicZ;hdX}!ZQvPeqbo@@!@>XAo1p+*bE27DNRBvj`&MDsunbU5@lcz zXi%{}z*WK~vG3vbQw(R{YTmE%(wHC4Qsd?A?ciPBWw>w?NB<|z)kZfR7}&*y*&dwr zdUIgrrgdUp8f9}F*~3*1>5Y{|;`T*TR82D9xQXM2v^o{NsjQx`}~-xxOQqojIIWE~ryh0nEHjvrrm z%E=y5a9|YL!N|8i@?(pLgv1t2mL|}VK3@*-eK;WN(PTdFkQh&sN(m$9jUwqw_vF?* zWEW^)7HD8%Xjs(A%T~k1wf+FJNIcv2FZDdJj4UTENDAlrH>_AI$y&p8w)R7o`2Bfe z!8dCP?l%?i&pE&$!Em3;KsVpOXko8U`T>s1pE%^t8d^?RP?uy}9Kl_>mH*6}5;JDy zh*wL~J_l`hp4PsN8jB^g_njDm!)1)AhreM&>nZPL7aWK8I zQLLxoQ>A#YRE*;s?b9|xHw0FImztlhY;bqbAV&B5`b3Oz!ci*^kKr!dA ziU2Q*PSo;@XUtFMa7zTM@*HHC;v{#XL4~7<=Y^xul!BWRJ}h`9%AN7Gr)Tbr$vYWO zF~}?AGk*%;H`RZ>V!QQ)X4?nN7MB-Y{A9kSZ>ja_;=0S`^Y>@1*_NTgBf}oxBevuA z?t{mEPI7t{=&ZsK=|1z2%APRs90xUiM#Y3F8Xo=puekJ7oWzz)(danzZ)=>hE>FWJ z&8QhR7rZhWcuq8^{Xf2Da|x?F1Iv+xD>ymMUYxTcas%sn7tWdnwjUO3#T$-DOy)Uq zKyE{mTn~fmjXQ?A5$Ajz*lsu&-EdrM;H1*gta9UE_QnRDn#!9C9M~45G8UiS^*Ei9 zK2(%(kU^$_p(32~`5mvC15eY|^~pO(tYY!fxmB~_%liMvz5Ng9NLI66dF=Vz_XFcU z4))Kwobh^34oz`wkPl3~!<^E%hMP{r#B(n}8RnZI3n$@i+umeH4$9)2vhX?LsrzSDaC#-=v6^bXJZ+Yadk zjq(SkEtI~Z@y1WRq{+FrNhG3y^+SWpOef(V$CM4qK-U5o92EM%z|Y|>!O-O3_)AEJ z$+YIGq)L;I}{&^`zUYSFr}AA#7URMiPgcuNrJ^KVQ*R*12@YpZAT{d6Ytr6Fj}5D!1CrL z%aQ{c|D=s~ZLSS7IU{uEL@z_jT(imrHw>rh1WB9aN*I-VzQ8-bqpPda*nPd+zM4PZ zO27ONOgSNImeX@*)eSijpc$N3!ogMA;WN`QRfy4OV=5=7kP(YT zM4;oLb}m6amk+P6JF;-F$=FmdJZxm<Tabd^Dh<2nRa?No*u-*b)5ov|gNKaVX-#@Fl(r}a83|V&dwF6*(jhL9)6Q(1I^Mby}*VDXf9K9#4Y z1S=_~XgkVjs!p;#Ah5VweMdu58&6Y(Q;)_J0p}jIDFIFGy3Z<-I2hS1beaw@2(LMD zSk6Jn@QzAH??(1HEecGhbfS$mb9{UB#zZp7g14C?VE1Y!UJvbsEy8IU3{31ylizxB zXiPZ3ct$SaiHPki!IWk@zr=&RnN2ekSUG1{f4T6|D4|g>YuBYyE}n~KrU?Z5NUq;w zcB$a8O^3>(W~J_Xo7Lri{(ED6b@9Y6uNjw3%n`V7^?%A={dUQm8H!5<1m-BRE4|QA z=~6MdcyxN%pN(pQDoYHU#Wk0#*s?olmf(9%*^(OxY}OtZ54Uie?Re10#8S=BZc{Qr zu}hVqlF5UotHY&FM(lvDaiO3z@5ZB3Og5d=C|n%wb(p6+;Uu@xjs;ATDJv3KBuyj| zTiQ>%BsTFV75Fqu>`Q2!Ab4X6FVh~CjE1Hg8Y>nsD5_+t3CZf6Q05n#X7IHAugom!mF7gc)P?XU}Zy;=0&-#vAv z*W~|nn}$W^9(CN$RQPr(pY0I`CEIz-uYEl-~^+1%p2YU9!trQ0?+O_rJ&2`RVbA{5(sjdn16=2hcJ ziWAXtU_3Xc(yg)eiP;3kNok8Jns^Ku1fAKJ&Q02xX?AIX{9a$jwI^iCUu{&k`?=}b zFJ+aDSL=4Q2~T=+oGt11!Rsll4$%)^g(mDi$fM!GVkyAHN_P8?SD_}G!@*d)+y(W1x{$dUdp<}kN_MgybL0T#y%517Pf++&(IVZzJ} z5BPE=F2!?fTXIO^S!dL#GbdBb8w)M<%Qht^vA7vDb4^fSQ$28(N2X1xCpOJEgXth+ z&Vv@oGl_gIJI+j=qAq>hLXDaKgu_v_1&l_X4NR*#9gCJsHh!{T0h^A%5nZ;4eI;+4 zWX}aSscd;Dk~Kq#Bf!J?vc#s9zP?M8o;Mt~Fj;3BN2!0KDTG~>XMqxy9Zxit$ExSDqy)lFK)V$X4iZ|VkD-Mqx^=m(1= z*A%c>Yh9eqm6E{GWWXo#NuX2lM1x@Ig#$mnC~(NR{9C{vQgQY*Px8_GbJJM19pss+ zR>e};-8sM!KjUC})&pk!y9fHYA2g~s8Mei~VUl@i;VR;@kk?HmIcjFftv@DL z+5EOH-#@HD6jf- z@G490q4;Q}4g0sO(W?KQJ^8QLkJbOLo>cgh(7^r2MOY?fHHW=pm+j0XsdEi2TlqG& zEOSs2Ni%Fw&czg-V2`8}1lThMjCE=KC|%wt~~{de8}d#~$2wC)C9zgzc%`X8{VavbFGvtp7q zNaSy_ICA04tZDu|Nh0%498&O0rg8K${{9V)JbXQ`Cj0GSGy3Pi zD4tWuC!ONJ_B7-WtH%OnxgCrak4;-mycRN>=~PY5ZD3k_U6{9|W0IW2%#s?ftNgVw zR(4Vgzg+kz*c>Q*WB)?Cs>Z;(@j`bFyzqbjzRvr?G21yzLTVXKDtQyTgC`^kEd20G zVbU^o4Ua>TXF6DvnG`$HnUwhV%dsf3F>=`cSST?4z$w|B4sMqZl>& zVK47A&of9?H@qm_>s)4@HlxO0tz57?Y#z7M-?cMOZ&=CVx}nwBD}aGXWPw4~#Ith( z9MUW-8+T$-3;4cq`(UC{Up*a!8MV(nEoH2?fiRYRfI;zjKc7%L4vi3j7}y z@V`00_iF*)hX*s69*QklD8{BJ=;A1z!zi%f0C&LxcCJJg9tUQ&qjxs2^fJ9lR8pI- z<)EMUf$`c|pNb7WdJDQe9^8A<#F~;&bHmqI?SP)r1LlGUJNX(ISeVU{`l~(E7^U8w zk$Y!s@?doi)73Nv)r^Loq6M#15?Di4r1mtkMm6Y2xfZNR;*5Lub6)>NhgP8t>sLN_ zX#4HB+`GgVpZ+yoI+9=iDpAg3r9khyrBC?(<{Wx4P4C~G_5U8}3%y+bEr2)4ErI<_ zf)H0O=QA&ynz}ILFL^a~gxswM>qA2iVLOuQ+!Loq)`U5}3<3=7$Vcv!wH;O0qY2y^h7m7#yl*IR7WUAM#f(^3{zZSdLi#N>zz z+kuRlW9;`07>xDYjF}!YdpU6HJ!Io!Fw;tA5^AV^5XdC;?(38QPT^x$+1BjOP~fO* zcvSF!*{(s7acOGA1D<|S2~tHys4(l=ylc`&nwjf+iDXi1`0o}!483IjhQ zmy@ET%afTriC+SGIJ^$P3o#2~QYP$A1f-dPU>*cy2boD&dYc4|7jJdsmzB=SxO+rH_wCtx)6N)oO<*-kklwoFj?Q6yw+8Mx3uU%Av?ndD zp2qx5hCwcnK~9vJH7K1!tqXR@;OCV|%vZSQ#t_P(6>;RWrqF5TEXpy7HnuF3-mxcZu z5J`L{pp+P!vYKH|gCrLtJC_5q-h<9ZywckF?DH?`9|;VR*B4V}n;&pEfhYN{n252$ zVlxeX7NrI3Ru4?ia>WU~?ii+pQ$@j|MDBMELjP1A3Q8>0O7#`;O5`(fU?_MX_>6(S zjbYg=kI=0tC;d)KZSLqaIdD%&O|R7;U4KD;QP{ROOU%-k=NSkJ&Pn8c^~CcFE3-j^ zh?=4gkFizUdIO7O*0cp|DIqx}{%-2QYypSZd5T!?91xP=5nxFa_@!Wd)KO#qXGQW&G|B2MX3)s zSrWNb6qwu8IOja&TB5)?$-yr50N;@&5f4SdZ&Lz{#Y9~gMcf!gEW$L_Ef71@z@5;+ zUf{s)}M_17}h~nTEfUwnC`!;eQGLm=bx8-P2-V_Q}3$;E>QY z?O)gfw$&+p9~Q|fGq8V4;P+rKQOr*z-n}W*GXmfvn%e+TlUnMOzvUex-o(8@K?T>UQ@nY$e(}3Oj{y$t(rN%pk;Nj z(8Kp}fxSZgpH-6+jL%I;oZ`dWlg57_fw@hMdtHODnWC`L8J@A?;!L>fw_%^OD};zt=-E^ z`>tR6mQ7E#Ov@=cdgR^(3!i!0w_nO-U6;+hQ70v>xyoLF>B7Gjj;IU1LTxMs4_I^- z=uSxBesO>^=0M*aL$)~wydxTNq8->h7O^!o3=TNiC5o9bUeC)FImN)Kx4=%>(dEbkk#i5U4z7E*MyTU-f=Edt z&xIy-j)emA9NJhI1*RR~Q)%QokjgwSfiI$D>76C>F1=+GP&+BmcGAf-#PzZMAr~LH z{z}>RMr&5z&1>$He89~0fT8R_+m;1V=?xs;9ta#uI4`tNtgVnQZ-L9dmn@7j4t!xD zi8%?8!de_r2}^SoI3y0r-be^Mw0N?$WWfqW>(km7m-Kt=6^;3QKxolJnSyiq@jv6ks#Z6>=0mry%%c0Xs*dz!8QGSqlYrq;G^ zxE%SL8vNdPFy%A|u6i>`|K`%J4Qw)%Mwy!=>lliQehHsZ60v(Ia%%Ms$Cv$E7PBnfLDg-J-gk zDeM5-69@iJ44jJ+=3GA*aqFRgg#%YWWBv>#e?G_AR}X2uUN`$-rAUS&3+F=iJJZ(t zH;Qy@WLHoWxYk*mm=&cWy)=w6Cr5CW+AT9ap zj5B@R%UQlX;Hz@r_%UxSIB1>4+ice1enqAw# zcFrKzwsNWQ0@kPo=DJlo+JknURNi?vfp?KUuUO*Ba}SkX99&V&cUkA`uG=yKYj?lc zaoyPH-?81VZT3`7Rylv$n5AgO{T;ha8CV(aUVX;U_Dou|YbogD<~0uc&n^(2wn)b$ zlE=+)9V4T_u9-ZKs?KLTYusNxVZrh+Ik+IC4P1@1XeS2PPfd ztP%-w;R>hD2P9@Suw^vms7kViB~&q4@W1*jX4k00<|wnxMzZO^-Q8cK6W=L1c{cU# zI{P)zuEy`avG3V%2cK|;J-cWBJ77J%a;EC#e4lWm_FHS%z6xU}1HoqnuvfTD;_0z-;|fYV);d)HY!6xPc- z@^APl;AY3)k-%{*HTZbpd}Rea?q53Y4nEVK%-4S~Kh@ygsU_d0`S=tp_Po$xq;!Cd zqo2?2fu~k6=aB^WBMm$yDIBvD*i8}`x9~73EC{GcILRBtoRwge70uq$aN9J`>e4om z>OE4I9HpJ=H27!m`bY8_y_|A+L9mFTq~4}`n!ab>bANjJ_<3F4{foQzx;Iqs|82~% z@4@NYE5C02_Gj)x&x)e2hgfzvv{fl^pHs*Sc*teIC~`wfxGqt%WQ~Z^!YB3*FX`Lz z8^{WrIxy4m;M39rm#Po^d&<9n#eji*(;`kL2`;P4sm0!&I$zkiXLZjg@8+2GV*5p( zKb3bkJV~5XoTQT6xBoh$#{-#IW=5?8oG%jiV$zszo#T@`(9Ys0q?W)q&DpHMH2uSg zfJ5vG5nEmgw;bgX6E;dINY?6?h`oC0ZkJKVt(CziCpl(LOjDlHmu0uAuvurOMfrzk zH#fKI$M2v1C+XN7^Ud)-v(3L|IxJLVJ1?P~#38FS#Vv4=leTD z#Lg?_(bCAW@;ImQVx!VmQ!<6u#_!8eeK?_I;X=n|Cq^chHociPr7!0(PJMlVQK*)0 zrbgny238&mrkPzLog$0v$_#8e0{<}{Vm>Xb6mnsMrBe(0JX?E-zcWi%c$XW71e~ob zZoSBFmi)G}^iT`eHr0>`4NOk0(k^PTS_%h`D_iF5u(@&JS)hVgqa0_FSjr)0wmpF= z4vD*~`*?H?n9p47>^8y9Ei5HN_@u8=%!h2plat>}S=p0&tAyp}$HmgFM>~r7ygr|4 z=C{*V)wt}#eX2#+UhhjtqrdV3gC=ghBNG&ZTzU%*2l;BUhVsY0vSn!t7JiGGtKGBVP|glt zhQ?BnkPXqSA`=4GwX1C!4hi&ZP&io5^dg~4L7HQdkMRW_Lx(&ej|~n^^V23U%Lh0t zWahN`uz^Lc(L|Y(n|n)Pmuz&5lbhm_Gvd9qS5ywW%jdCZuaVx*B+=XO;lkoejd};K zJn%93D7@AuFm%JA7VR2`nQczFCmsfPWojpNsx4Z<7^>k{dXUZgMPR3}V1a_G5k+XQDJ&Ly#_3zT>zJ3cTRQ%qoR>QHFvn8>U!(ZTQl!@pMx z8X2Zct(;l9Vx#0i)_{d-x63{Fh`bad(~#M376bjp2zpY^B|AQzX>cVJR2I+XFTGx5@6w; zw}DY?Mv}mCi55;D$KEs(6=wc~4nCs|ERH)I1^P4CR2T})w%lAKHfcv2ry0X`n=Ohq z0SYJXYH;-S+(|EJKJ8(2NKxU4X0|H3*M!GXCTm&WDr;T8tkCv>K|P?sXM%{xpE$Nz+&yl!ix_z`R3!Ib zFJLoW^7BdBib)gJ{_PCVIB}EN&4E44;}8Se0S09)MKf6!CCTrF{|+m>n#_~s@UYov z!WEN}15D-<4)Gd_Bq~%Xw8`}x6x(LuV(j)|+qWHzHnv%g@+BO+g)ctLKF`6<&sx%# zBFfRx9W3zVn&jl!rP^9jC&Zq5EZNDEdLm6}m%xi}kxE)_4O+{bdt~J%wu|j}D01b( zQB9%7W5IVGiBu{aP4W@!3UE+Gt=f=JTupXQ~E}UklErRr_L5_-a7ZGzqzp00cIuz1|A*r} zM?TDc#BrQ&u^?Z@m#gcae0gZee`Lq8yDf9MKXIl`dC|nb<^ZekrU|WYG+mcjEOqjV zRC>CgXzICn*;=X>niM`QVAe1=q?FpzEx{*v(1W2{WTM3p$LC6W9vpny*L)&b$;FY) z(B)C@@(C`QYK~n&E>7&A@sN23_OdTvV@j41o@-E`W;W4n(z&NTIgh^2G|VhozVT!p zPs)a^w|~4e+q04ShuP^GcGd5YsZ}U`B71&r5QQg4zM`#IF+{_VO8NN>{5S{ z7}WH^S=ytpD}X0QAen*XBNy9r#|2K}oenN?(+oP)bry1_MK|%a1!$G6{K>?Y@HBlv z^~@U=I#Z3Cd(-@sS8#h2Ew_68GTq|l@`=2wf~6KPXwN;^!;z4_Z?2OAmw-Vd%Kj}8<>@wE*xNPKh8T%qD4;0@mR`}M!UQvZEfZ@ z=Xm>-k0>u|-{jM?;E-9~(LWIz6LXcOEI6`*X+m#Vs6zE-p3c*of(zFKEmKldSbjq5 zGxI^dJ^vg!9Di{0FPp#;$zIsqW#%GqY;BQ7i6rN(vYn?({~XagF`>g-<6)Jwfg``q zfkwuJ1)Tj1%!*=@uc-<=s=j#Fxcs_`)a$(RY}2j68u62#zo{!)v29~dUP*J)^%@42 zW<>_B4b4m}AJiE6JRF&X6c}e(8z@gaaey`AgRg9yF>^r05#6V4J|9Y~B(E1Ao~Yw9 zKcz)U>dB1_7RMA766F14Rm8qo`u8L$E|a*K;NG=)B~#=k=ig5L8wIwwFJFG4(AxFo zui|XG3CuPVltk1b6jW3kyKEE^1u9NQ#2P;Ovh*>lfRw6G?rV!S7ZE0&o(Ly#r9$p> zpGDkk1x+d}iT}7fOx|-A&p7ONHe}X?9p}$~zWwgcqx9z=n6@rI=i9}0V)lM}&xCA| z6Z4t>?QHgTXky|ma9CdFR_t*)<_9-Nll-TE=HGfQ_xJL!GKm(=>)}|mYT5}0?st#f zx*QEef;XSYJn7h1+_U(PqTc_G&2b7z!exbLKkfR{n$i5L{)My0!jIdYtz0`#Lp3@? zP-Mzvmrz9wHVJ0?0QTNQrVv$TuMI5O2N~5Cq)9KR^*<0L$ikY?z?vk$Dwo0JkznSz zfZ4Nwg=sr`l>&!z0sGkk?yC=Y8WR}Q9RYnp7R8W%UltoHhcm7;vF8(zS@S@hPeJX+Wly08MM4gYw+tC|8ko2n zn0_o3$ekbE_XM7$PfHdyc<-4actwFPg{4&KTf0&k-}420&mK5O zr6)OvxU_`kyu8v8m(Jp(z+x%D@~yqzqmk*$6Q-L+R!vID=Z^{hRnbsW($G*~&Q;)8 zxPaNOm{CTd(Q|{CSOTNCLATWgZ*fD8CIj&%5mxVjp#5K1{1w=$3piU9WG_FiD89%n zA>i$O!6qSrNo4_}vjFo3#ukSeYRetfSBl47pV6{zWAG+LPoo1>KZ_-I8y4|@U{u|} zZ1jLZbOLi}0DDCPyY>eLu7ZCdb0;xu3v8Qut?kECwp$Z;u0JvPv6T6w0q^lr#*+s) zDjC9-n#y}k4-&h^B(adUz1ZSOK!=oP(uV_lpC3$&K3@8MdB@8m9ZpAB{Q_8A3|O2M zSgaG6^)s2LX^6-t8Q)5Z{97U#qS7UpV6{wHL(qZAW&->1r7>$W*ft&JoT-XTu<0_(B@r|JYIwFY*V1FkI!oD&&1mmR9w zU@n!%ByDuL(6+Sq$>iX#&7$89OIj%x{VbNA`qBSM8ME03_6Z+2rhVXWQ()w}p!wp5 zrcrnsS3;YcyQ%05Moz|(Wd^(f1&k8^E^13pVJ!FH-8)H;^(aR`I^Waf6Qxg1Y`;;b ztvIPWVs_Vxj$#FNjhT~NF3g^|gIRAUv!NxkktMTX0kbiuo{yl_Qe&ohiB`5ZC$k$c z*%YuZ-M}KYfzdO8ZGHn+ilViZP`A|r_OFe$iwha&Z(`tDz`%529`l8HLK9k98<;nT zakm9`&jMyMf$85ks^312X=19r7}_g7-P7~L^!ZCAr!`7i zUbg3VV2ppj^K=6;FX zwQB+QwFTUpin(S^U=ed*5*JJ0RxtU=tj#50_x2@2Hx~oL0R}b&=JE@1?Grev1=y+@ znv@e5EhQG@9AjGiF!pXo;f>S<{0rrC{Qc}3UAmG5ry1?%B zOa@+qV#~*-+y|Jx&oIuF;0@To@;xxhbOY1H2|P>8e2P4HFBH^1+^E{KfOk0qE0gEq ziwk&P6Ik981&PL7I9U4mmvzC@zShvueefP{sdv5-lohY@;=>T)eDdtqIIR=sS z#s|H>wy@8qfKz3GovSG$Uju{22V;!_)ny+zdJ;I@HZUA=Z=3jG%}ECV z$&Cy>7kDlT@Lq3@pO(OtBOu2;QE>hO=j@88M-wKl-MM7_Nj{mMOXMy2PZ{OBRiFKe z%cbJNx{?R1J5RFiie!mB#k%+BjxSt0T^iO~?qE*IT4ua~`G40O^M3(Mrk>`03LJ48 z=B}8)Cvcl@`GF}x3pjf&D;JyxRdb;R{T0CAB#_SaK#XYBC(yzlBkH0{i?8 zf|3cErMs6%R`Qjdo+TrB@JqtFHwrnTYjhMOcQox5sxDwJPuNkiU`NcUr9HowPSW0) z%(XKyYv*+BIhF_3n>oyJQ($iqU|aHf!^;M)bOWyN0#+lTrm_p(?F(2c8wCGV6mzf& z*s0Av64=e)DZ^-ffn~A-N4o-tje&XH1y&V>lHv`F778po9xf=Jxp~9(^efh@GI*+| z2`>EjWO|*El-Mleu!lQc7#N>%ryFS1a0D>gACP$Exb4gY=IzflIi}f$Y-RYol%aKv zK+hk|Z3j>29$+-F;*d_!yzEf-gL0fZSc7(y zd^o81hPCJeYuv3Jt_xWERux<8Rv6x%$HTyI;ppzC-ONlZoQ$g&Efd)J7cj4JV$U<~D;A5ji&iUn z_OFV~YI^3HV_aOK>pvcA&}7urVh~`EX$?5OH-fz~fxWcB{-Y&hox+685uCk6(od6Mb`-q-+^;AGn1sZj;>Ldu0J{Q23?5@H&;=lYDzF&5h~!&|uNg zTpBrD=b2WH#t!zkzYZ2Tu*Wm7#|xZR$~oP4nl8T&iiI!i?XF}?V<1U2|RmaFG-hOVp>%9=+&jgKK!$Pv0LQsh&ymO%IZ+mfgRg=u55k7 z;#Pab?d?*R3s-LaT{_WXnaLfVB^SB^CvX}EDX(i#PD_{;I6-h>feF_HM$Q5gCIv>H ztszRU7ybk&i%x7WSyocm;K zKg+Rq_s3&18MoFgVw_XVEMUO&D$P-33NzCOh99a7+#3v~9TK=07>`QL`1NAZ;p5UG za~VW^FN(?@*iberPvGRgX-TurCJ0%HOmNG4a&OBMw|lq!;x0+Cbv$~-7qV|<^@W3{ z@2>0m&8qBkWoyi7XW6?+es^P~?#5+3>z7`ZV8s$+AiK{c9GTYf+(Yqo>A(f z&8t?eS~78S+NK3*LK@rK&;L}`5Ds9Fzfc`hz&N$R{G0&q*$X^I28`ctWG=VlJrlsv z5wmFOG_RFQ9-GHy?o5=neaBW~VS3Vm@A?(?|DjK+ZWLd-S6nq!@ZJ{ZlMQ^^$x%anqSzHrXG80&G1vr)_a2ppe zN+{?T%YE3fS>)nD|lKlZR^>|x-YxQ3Zcl1q1k_oh1e%lp2@_`j807=Ejd z?~w@Kiv{d;1y3s`JdF`JolZZuDiU!he9Z?gG1i z17mF&SR7)kbbvA)^we$(R3!u$$$61GZr~(x>tyEzCf1rgE!|I zJodB*wp{;uYy!{ge4cmb|4AnDRGI$2VfQV){%QS#gKgS}qP-5~{eQ}EaA}8FV^o*S zpP~y79F<(9q%@OSeCN5J{b#`Wq(^FN@N%I6$15!7QSsoxzk`=o1TXiR@QcAIggttx zRYAZ5?T}3&8=XoW#Ijn~1_Xu_7%)1u$O`aG5O|_~er3v8rPK-oMQ8coJT`|$r4`Fq zM1%_bJPZ_DnK-8gt7P;fH9bD9S&)_?d`e{T1YNTohF5(qDbHB$(am?Yu*AsiWS^>Y z+RGX5lDGFuGRwv6*i>+`tNrrhbpCnPC37xMbZ$Rg(s;E~$0VYlO0cm=C@lO&z#hR* ze*YYr(z;6( z#}0=t3T+}mTNK=6i(YUjwLCb;VJTPWCY5WpG)*ek>`>Zb|CJlnt^_Zt;OtVEH{oGt zu+qiE^^;X*IyADEY*;cg#qZ@Mkr_=J5;_AWoor_1G2u9)wR~CBw?@_#t4=iv?XFtj zl70LV*XnI%Ee8xG)H@Q^at1K0Zj!Xyk5}Kp*UADDCrO%KN%6{BCYkAB z^_5AZOv_bBL&Z~VlKLA1*S%gm6$uPtN*}+92rDVFZnY^9crry*Pr`wLrDe*4$n?j@ zUf$fXyoq6r5cdPGTkE%UajoWEDkQV&__nT%O~*|-7?~ws2`rH`H+taEer4O4ghtkq z1V?sRGma+KV@k#qn^~nkiy7(6pVH%9v_K*y(Ji8&Su&bsBQyW+GWSzGSO0ard7SLpZ~P&mnARCwk+6JrC6nq#ICgAyXD2Uyta@)rM?MYI209xoY-U+ z{7{+o-AZbjptZt74gRywQ)dGIo3@GJ{GJzvzMsOmW_TE!(NJL3=vXAZ zMX@hsLbKGZ1Q)h>6Qmq%6nS$poTcw_fAf+vIJjHXS@o3u3FbEr^5zLJ z&Ac>anekLjpVF)o|D3-~sd=V-|L2)-7Rx`4?2->0pLjU+I_e~F=5?@$>|i^!+Q)%` zOTZy4_GP=S%LB%wzyrK;j=bM@@=sS?*k(NG+W}P-VaAC&8N@#%u-k7?sK|>rV8%Rw zfkARs2w=Wb=sO`AMrYmfsYmjeUCiHGdxXECeH3s4DO!j_rg z+a}9a!02SqEOj^GINPSwW_OpvTQ3%{nZ#)KDd;#bOLg345}LrMR+`47d!m6a>OkYe zZ3i+}DzZFpc+f0wV$SZk;f3H80T$DYkj1$k&C)hp$0wFe=NE}~kr!i(d-SuD+h@W( zzMO<+$vX>JjARe8y9wME`gg*d&1|ctpYw+1zika1zjdDo<|rgc2lR8)Jvhi_rl2D? z(@CK6M6n7}eIKF*7l&=J|C%*-^krplO0jXRA=BqfgVc(t-mT z8>1NH-X!wHZDLV=wc)J9_p2M)T^(3^0(@C44y<>3x0+#{WuyC=1x!~O7SAbNNn?1#zk#j3I_ucQ|4nx5Azb!Wm|^8*um z@A0lz`eicr%O>;XpL^f4XSp=;2mCwGW_sxWdyGL-PdaM>*6)WK-hEi+w`=K+KKgf-k5 zC5${$3QQ^*t-4hVrvmg2hKm^-6yss!2$XQ*u{goVSuugfF(r>t%;UQ5aha~t&=fxY z1G!Vf1X>ohHVIygh|l`8fKlv%1Dgp$6|dOc7h)#A*mW74E-uUtG~J@Ws~d25sp;C6 ze&x3=R~xK%F1x_|ZCg2u;e`BNtp~3-ro{f!e%N&&MuVY=V}}%r{0|0>7=!NuXJoGz z|8L;!D(JN2DCG3AxFlq?y+yU)K_uIQXqFZO7RihQEQuV=&!-5mIcPAl7w>SEl?!0^ zbh#1hpWwt_vY|N>kUg`Ib<8`b$ zVXHMcqyjE8@@P1)CS78-5@C86tH`8S70{&LA|tfqK?CQ61{PJ7kD=ZI%@QXZ+KfwN zgjyMxvwm%`JEZ-P=|5-d-ii>F%3Ujrj^{AiNM1ar{B=cyX$GVGkDc?it=yEY+*dOy zFdSII-6WQAfGvhuBcsvOgPCi|ffbim*uN0A%=o7{eG}u%u11Lhqfp`9c@r5pCUgre znznAjG@)BuTniXBHd{+?oG7WmWdDI#?L)K2iDr!(417Nv`8G7X+1yYn-6%PM*`uOG z??7|fj#if!OsWFR4i+s=1}qsnSa~<7Ix4gVPiWD7(I}hIim6Ig5xG*}*PwDo8SQec(2(XdK#uZD!Z(TY~x04B8wjI3Q7{AadBM=&|w z5Y^yd)pL??D-e@f$s_VWX!h@xiYP`|4QG3WMk$6}N&j??vE($$e`s9to00WTqnr1x zCDpswN*p+I960B!d-aQX&22tKg(dtv2L;wJ@OB)On%KZGq5CXD_a4sfGLdfn!h=Fh z-9kSbq!pNKE*$(J%V z!FEWQ#i4^GeM75ifrXGlOXd!?td5rOL(SX_-R=tPK^iSV3@nl_7Brq(C1Jegkkux6 z<4q~Mw%dJi-hXO#ft4NOg=X6eO->pud@C6C-P|SjqESDiiA$ix!=TaW0Bb=3n`}fw z%7zvm2?k+_{ec3k!W$YGB^bK94T@&86)CWrzc?Dq(dt+r`cJO1iT%OahQgk6Hyd~w zn9LdUPFu1%CNzp>FiK@GN^Guxp;So8;F<>9vd=%nr;fO&ME!j@O)d zndP7m&@BIf*+ydPG^r-p6O6&T8zl-FBeV~mm6&t2=J?ekmr7L{m=YL7A{dn=n9LPg zR27&5FSN$YXf5Xy&%yaxQ7VEk( zrEDoZ8z`@Fz@7O(+LANuESFCDxScuDZ2yo+NrFl7M5D|M#)Z+15-S+QC9IEn3W|gr z6h3pQ?Pw5_#&rEwCR+~HkIJmEH(2jpxe;@rHR}Ugx+$|0-j=Q{YcUsYM^W>_mN2^&X{Mj6$l)W1c zW!B;Jq8JfYzmlcZ!p=gw%DHFc1)Pf%+Vlppz$ALL+2UJ z1s|+g^dj74J6PNp%y|RMV^XCYh1oMQnC-P?%^K$3i)@NlQI@K0l$?5T0k^|p9v{ip z>oRK_7caiPB-^Trmr3D(&l>3|)1;QD&1JkGa$Gn`u{G4a;!M!hEBz9ijW#wgh;%TP z3kDlWu=Jdbh(8d~dzCf&2iq5}mg&+BDQh<<-e}_0XjmPkD7Ay%t%60>#L%gN#i@bC zw3)>*p=tdo#t+*54m+AvC76^aY|on1;GY|{B=puyovWYsZVwa9kk?Mm+y&H^2JsZ{=wx(MeGY4o)?)Yc0 zM4&}XW0BgVNvs)-Wm210aOoLcVh}MnCbW!ElJ`Jr&6Y*UXXJ9voZfn7akiWC*XF4g znN+`Cm&q`>AQQ2%+fmE*;PMmFYb<>KR-9E%i`6lmZo|-W!-6%|qrKz~Mm6$IV7yYuAhv>~GOB@Bp^@iRb6Q7RyU_!)1m^lfoh%y`oZy`N-=j@oMPtwh z?xcS{cccxDxpD7Wu>0bHDR-6>o3Q$vk+a?9;k-+JM~iYUpI!x1-Q7loAJ-dr9mN}h zHm(-D?Ay(!nAC3T8!%T#=mJyRJGQI~ccT}u-t}Rx@N3W6z~=m*(Rv3nSAkmkBnBQ~ zb*`voO1oEWD_pkX#44ta$Ba8dHtcBlpv>&>f~l){8AAcX4uPx7b8p%EF{Dj8AH>?N zq`ijig6Cl=1~~&pnHl?57A*T2%B1t6={zTs;}0L+9SzEc42~;UR>v^#R(7!*=;HYx zDm$UYNrAHCm+_W(PPmYEER}ZRk{8z@GQxtbrnDxwLUb6oYovqWixDXBRgY ztG1>zw75sdB)K@a{bt9j;1T&Q=opcfM$ITkwyTz#lE#$3OT>v>17`7zwoI9bo-*hxw$OB2&RK-M_0J zTPK|?x>=N`ldHrKkjubmz!1X7;>f@vc41Dzrx2EghEJilu9-c#t`~nsFO)Nb;n1RG zEB7{Q^~6atFql1Qa=P)>c?UC3$4kWx&5P_Zk9~91coiCDGME$# z#4?<2eVgLXrI5t6Na3do!$h{L{b_squl4uJEe-BvoFTPDZ~_4A1UKKp@sDYvtvWc6Q>!$4BM5At~W8x*!A*>ZlmaezP9uQEy)=y z?h!5K4a^SaP4lgqmM-2Rb%OEtyck(KtCW&s8FL zDl;!f1N#ZBoT#(=e|^iE#Atb<;h(u!Lm+QM)-0~zL#wu$hc4avtx#^MN`3>I!MCiS zTTUsHf>ZDA?Eko>`|-Mivo};Q2xTy@No28I!E95};;@3HedXG@acVkI%UB~AH&mVy zu0AE#$@2AyZbs9=%@Y{TM>Gm1v&eF+ORLa`?r1AWs7(!MOO0Ujn(?Enp1GInOHY1f z@uDqDC!2J?`YYMzCV%6Om4UuYTG08P%ALy&&h9_xC!WNbU!b#=XKj3A_?}ko^DL3q z3nJNt-~DGV{Y)9O^jSDVkfq(JW;iwV42U~RSX@B_A6LY4r(RzH`uRWvFEU` zO6awWV6bN3@NUXdD45XT*u=&uX60~W1LHpiHeN-Bf(HzaOe{hwDh3)08(rB&Wdu4L zmT(J7^0U~i2xt#j;<0qn%tKN_1-C5&kv9Yno1oiMhCsp>X><9jnw+msZ8Lt(mykwf^1#&*blUQp%N|**t}x zudjhkFN`n)V=AY>YxB^%Yc7%J!95`eAc)@4BiLU#K!z?_tHv1S(Iq>y(OzDA( zZqk_>62+yfQdhPK#{Nrn5vX`j$etWi5Xh3^^k#x{s?nVSXCA$hXHKm06>g0gm0_j_ zL&I`Kf|!Lw3>w%hUtILj*ff)|l~1PN!V12!8l|HO%L9_Tbo)hw;j18(o5z3ox;&TyQXAVqs+Yz#zBmgn^T+l+C{h?V6P_8(R2!Yz`e_X3$vR z;Ld5PAjstRMbSf6n)n!cN;95u zwTNP6z~pb5{x1@{#hUFB4ltKZ`E-DRO`^b&&oi~)pv9~+p2|6MqKVDopw4GyYP zOTyb_p1B6MFi6%;5qSLb+=~XmsRb9MyKiQbm^7AeLst?II{naEm+7ZnE7tQGG4PCoXfd2a+Wd9E&I1deg2Op@=NDFi)2mk z75MaFs(*(fSHObuR+5<29AmY zT%t4ByG|^WG!{?4_IEOG_VFp3I@CiHRw}O zQLO)^6rj=2G@r$Qx!$Tbw5wfIYHb3Oum|I3qkztg8O?HEH^`|dbDr#u)JaYcY@0MA zh109XU$Cx0NdA|B-LaB`wJr|Af)x(?C2u_NwUl;Dd*QO2>w%`pHj%S#KT@{TJnS|# zJ#qMko#Iw8l}(w#Ypz`LQsm{=I~eRO;i7S8Lc0ftlF*U@=b$(Xj`bb7TzLkp$`urSnJ0Yh-dV##GE8X06hC`?OOQur^1 zMO>lVp#52C?LpL z%%90*(?yoB4STpFl%(FRIIjF&k$>*1kV%t{#7(@=)Td(mSkmXy4P_ZdJNqdoeLhaU z&-col+5DoyG`rQ8)3pQ`xm+AfwtnHc$dn<cWN8DFPss#k$B09m!WSA9 zBtEwAeo++a_YmJ~kTn0d0}BI_N2120gG`J!5~NBugm%ifM=g*z;b4+*OxdKMk>SBY zey)EUO)7sbSUH*~vvr-GzRziu)Pz@l<<(W|6pJ%YyxjCjw}Uj<&0b(DP-aQI`-wwxlqxCczaI|S_IcoOTrW-sMUny`6p?*ZeoTUp_oxenNg z@=oBZa9}h!z@*Wmn5-!s*YXRjrXCl2{1X+JYN8p~7Bn$7awafHoH!`J zzJP(1|K03L1~p}+i9L#cTH?f0CE5+T7#K>PnAkURFmim@!MJ1t@5vK~nf?g$Zf5E) zJeS0gEBb*s?ZpGeegPL%pF|IDmB*YX1&*5>QEZvdld(jOF}(l3N)S``S66+ZC-skV zlwB4%X!mdUr&)e*^>bVMPs_y@->958Wl6z~Gb;n8u1YPLCV4Kzfw`2+py8`ZBZp7b zC4q*9Og-OgyL}zl0z4in$Q*5Y8k$fkoY2VfS-~u)CFyXr1S4aFl0LJ=f+PGL6PUOJ z^w~pLwEQhtMfSaHD`j(JU~o9FspssX)U6pVl93*665>xp93N^n2^?YipwJ<*^2d!H zXO9IAyCy0aI4hi;am+$#VT0K$C8>!K2L)~%*uZ@CTVI-5z%1TF>f)Z}=bBoI6(8rs zx`=IBW`1?EolqTr?HkEG{ck>hJ@J~ae9q%NB@>#HAE~p1x?MGzzA{V4{Xx^ex&s&g zU)5p_@;iMYrSQPwMBjg`VF`1#x-mB8?fuDW_CUk9=WV7W+Yt_>2Mj*ZD#t}1UWk+p zVBpY*Wn>okw!A%;Q%ZTa(wR8U?yPnO5sxQIf{6?aQnu&Z1dCbMBsQ+kN!VajwDCA+ zfCHb(MIMh03-~HDcwJ{+V3XajPATx(+(ri>k^Z`prB|K`&r#1g;grU(taw}C0eME= ziVAai?(~X7#_KEF%=hfJ`1<+iqVIj(^Iu*tyPbdTOZca&vu&8^PXkyvJQV%uA~WUTkbmW zuW*!oae#s60o$4uVFPF53rDyz5A-Qs_c%B)i=#DE;u4e3_5BBLmN>@w$Ob*u?V0O4 z@9AAgQ=n1sL&K~q{O2w-@HEU>)OzPc?E#LS1Ft8}Nq322 zW$<14l=JOdIR%?u1ItsaHaCQ%=I768G7(`?xWTA!!hw+^Hk|Xor9zJOD^mATI@)<3 zu&c&IGL3;)7g@z=s#vR=s`QYKn zo1YYfo`_zWyVLn;X9DM^X3l*|Ty}H5K43hy(39uh_2UOSdG95p?K#Qu;}OS%2EJDd zULTa>I3qVN#bYsVqwJhR8YhljDLBAj(e=@jF}LPWD{GUGNu&G=2eyPOjHQWtG#VH) z9Jc;!lw)y{d(o)yg;C-}!&Q^>%x4@n$;=Q_VKokM(hO-*$Y|KuI;F0}B#y=X_zAV&qV7^xqrBX7@`l*BC4L^4tR;i`O z(|7VH+za;_tlAh_tlo#GB zFBJbhr&ECApx~Y9JXaiet}Li~$unn%koL_3d|w!5hq!6qW#D|H!2fPRSS`c65C#di z140bJ8Y-vMpEQbi@Lr2iHk)@ySENbEBsltL_xcUT7<3Lq>N@apH0c^Rubk2)u-1h^ z!h!Kl7h}dkfhlrlEDpXFnR}^pF7pv@d55R z;~R<>BAkpvn(UWyW&1FQ{9sJwJS6a=rBkHC@ehlq?Sgkay(OGJ2YQu!+$5At??qpo zIW=>--=`G2lzG}*OH1RHYYQz6R7x$M$uA?)D9F*s`=^0tS5WA`V-2Zi7_`qi@SpML z|G*&l#i8a{di6U7zBdh3_Z&DImPUj*M7TAueQ1zi5snqph>e?}_`yMV%_*0kPg&N_xDMVQ#7etA>-&oWnLxZgTjg2Sh!$`*7H9k7k+B z(`S-T0ABPY(Q7)c^6_IKwl= zfv4pun}*|3ffbEn8Y1(UgmoI_SPrtrG%%bvARzLDA;&@B#Q`>lYbpYVbf+9rWMNWx zO;n2P0p|0lv0Wo)-$veGLLH4$8GSD%@}~o6^L{(!gxNs8Zl)>d~zE zg(2CEK_!OSdPTFMPNS~H!Rxl(vK!UZjA9*J8#ZN341Ex|-6wOZ*Gradk!*d7KFwI9 zsg>@%u7RiIxa9@sSz-4|(w>)|(kzqG3Am&wf8bzw8Rz_44g7Zwgfp$;NqNe5Kwa3N zQLM&MyrNP3PNR5>qeO+y$-FneRkUi(sq;Ny;L~Z8NKihL_o^|$2v{~-r8t{SVb)_gq}b!gIf2P~inHYgX6+{q;jy7%nFshLFbLmivdVEb z{qlnEg14Xzlb{dNvNsKkE(aJzoQ!jvEn8M=o;f(7=$zuei9b9Y97?W6_>c*BR9Y%vKkgl~-!c-uCj~AE9d_xtP(3|Jl=LtZjauZWA7w3@SLbRC|=VjR>HWrt5N#LG1G5y zS`r$c?m9`lcq6RS&-a5-^Tk007bX!M{lzyNRV18@J(`U}n3WP774I~1ykT5_jmgC3 z5XZkQEf=<2xsbuY_vJue++p1ZOnNcSdh{Q{wXqb3zI;&F5)sH+q?E_1d}hwqZpBQ7{tjlF9nMe6!u`|U zJ>_~j`;_LPb1(fr9hBbFD0SfAJSN8%ZfmqpGQ4tgtI1@PZ8<1=Ku;mYNlArCf#V}T zbCcYk4R1af@NdtP`*JMjkDf@qQ*_iRlQ#$XIyf({U=KRda$e0@^M<3G0F#0XlinTB zIhgz+PI3_r;u42c?;KPVaJn#&ZT^g7%s1o&Lk^jmFl&c6DPCbx={ds0aZve3Bj*DK zz6l2;Z+-YzZR9A-@kGzMS*;OU}cZT+w+4|E0)m+3>td??qoa?*pH|PaICo>O8*h zogj0eqKuQG3X_rulM>5E6Z>K%kA(u4wS?lC6c`RE?r3D*>G-~EzRA~j-Mojasu=kS z8s%*cGRHLV?Qqm^anf)&$R^M&&2dP9n6<5;?1ve`(CIvUAAEwY1~L8zcU=x}w6HC^dGB-LE#?~s&d3~Cz~jK%x}tpo1DnW} zwDk@=Pp0!eeSGUZcp>p^K_jJ8=-t&xR zq&2X;a187}Y%!*2 zo5y5X58;H`qZ>F62}d~G_Xyz2xXqft!2Kkn=_l9PyK-)cQJfd;P9zCEE)|;}*$XsoO1&Fh5zgvb_7l)66)||0-P0U+wlVR#m?XV?H>~EcMTf zld=yStxlQO`ZqOvF4ui>NS9;R0{&eKnRorG|83sDZJ{Z1NH(W|^T=lfp9>boPTyZP z$XPJT2{g9$RIs>Ch<@q7y@4;S@nEwY16xF|EXyI*l15{nW@D3Pm72E?=W;YmV^Dg3 zP;tVcuUt+QA`Zegoa(HZr<#5V(dFHnpu{BLu-13J8*7Jm@e$S8ky$l@QdhmFFkN!H zaC)!6I(574e%Y@6(U$B*KiM8Q2F5ebNZdYSvgLtUKg)Xy4u!6JdTW#J`h%6#dOYtl z|A;sWo@jJ?-zc}lRB_GL`QOVIPT#0^=g^_oO~1FjKYahtg4<0C*mvn(Ii$jH=)>1g z6J3k`7AEB#jeIN4}PVyIdR%FnoKx+IgXiM z#-a3JBuhvZJA;#HL9S`cVYxq6!Vj8^KWJF$Fgos@cflursp-2snd;0_olF2IpJ+|rlkAjw8gR;7Th+8VK(Vt-aWawEdI>y#~~%3 zmT4XCu6@#tF3;cyutkyb&7i(j=KDP>iU#H!v2%&6GjQ1gEAb4 z*lswkxy@u^!7O;;V@a$l3&#P0JB{)t&ZZwWs$5|b%<*7(b5P}KSF zTiD%V=X;1SieG7zHE@&_ICydmgJ?VRo2yE>!!IwpQsOz8ZR)C!)!|DYo?0b( zUDwIWET_RJW5R65N{P8iBAi`(juYp_NPJLmYGLQ+iP=%`-LaLOFX;j27nT0GmX0N7 z1Wj1l=iAz=7e2IZENbNx(h}IXg?CoK@s=$y8UYWEs5(?LRL*Q%nAak$@=wf0pg`P- zk==TR>FHeUa+CD_Fbx5{nkHs;ArT9YggZN09&`7($15;emA<~Ft}-4meg>I{|R%~>em1Xxwz=eO~E-mL6k4PmQbBWBt31(-Cl=(Fl zbSx2Zyrqz6wu`6Y0Mnj-MGy2OzX-Int7q^iao9+le8|XSGC^^Jl9Is!)7`I%9N6rS zRHn2D8f-Yw!oekRpv_h-<`o~?a|y*JTe0MaoVFhd5*QdN650f;cfHu$ZJlG7%Jz>@ zq=I3WlekCUPL?^66wga3UgB%bzFe4< zwPxitshkyCr`^h3IrUi7$~?x$OtKai7V3UAi*RC>H8YrgO!CL3?)AH?biOc3zG_&) zZ~N)RVt#w2%9IXP9*zYMIAk);dfk$nv7kw|L*ij0!zmpF21b(^3)ywe_>OSfeA|)S zBI?uGBA_9csWyr6+oOd-b@LS%nT0(Xj!cocVY-4%vdcJc=QFidlOPY(sR_INjCuKk zeGShx{0na8(aKl+8qv@p^)sxu|LWT4s~s}MXMVm=p6K?gKYUt~=(dC2O-^DHIcBH6 zT9s9@aa&Z@#*JdPGk>1Dc8kw;&rxpfL#G;=c`Z&jtmj^kX4AZL)fR5%y;V<6@8C%} zvDm=s&5J#J=1(py?AGmgv9P5=dB%c74z&|A&IlQ3EM(CwGw^HR@G@A)qFQi4fl1m# zfstRGB`|!F#|I;ab`KTB^jSe_oJUxkW>vOK*{YOyL`X>^`CZ_`cDDGh@AMX|dFVae z?SH1Pqi%oe_lv3i(mzft%=q{7R^5q_Nq-q*<`)1+x(AMIPG~ z@6wMqIxKmYWOr(cKl9VyeHDk2CvZg^Gvbb5ylnM!LYoPT?T5|In(Map{=V8Tb4!8U za+_gaKwDAh!xJ2Czt~q7U+~KLD&XlE;M8ut?W#3m+*1q{{}jhs~*+GT$jczH)O36(5pQT0&j z<$vI)@=Jg-+T)Pqg&R%^K8;I)0}hMi8XT2TQD|_tWMH+dVK~hraConeL&uj(2bhc& zG=5l+z@%})fmML>q_5&7#pjV85oQnCf=ra_=Y2`}H^Hj!Pb#nS=f*Y*pPyF>zPy)W z=xKLn^EgobCb5xs%TpDX#?vtdOEoWboa$Jj>ccO$dB)8jZszx-!grK4OXvHhvDr@H z$dqbn72mMzQO=LU2a*)@?cW6EB>fB4V0U0(;1AGYWcbkN?y^`&NY&-Afv_gWpQC!J z2~LW08oASMHi=kBI52oQa75cQ2{g`dZD3k*-yKhBO{`cB+V;^2wkql~kbFIGS0!L8p(w z?767)Z4S>lN~#Ww9R}@+762?cJdaweTlFNzr?45rEbacEJs zImCC#=en$!1Cw~#0k)=umN-5G2IeUT_)nf_=3_HxP`z=Gjrl|O1S4xk8xv(I4;v-p z1s8ji&774k&S;ZgV0O9bfynYFx*bk@Z@2I)cwqB9kbn0a0nV%mi)E%<%h#}9Iq~ON z^#gO;+$>mP5;AO(ugvFTw>rY;5GC3q(9*!7qA=rh^n|NgffKK7*_7Cu_d-fW=9z){ zrH{M`_02c-P0Kc0RmdTnaG>={$j3)a*@wm7A8oICsWyR2^-|l*1U6%ZU{_H+6+H79|fmrl?jYWGED56B@3hF z6gPAH@nCMwXp>$b(3N80EL8cUyYhd=O!g}aB`Y(+BX-STmU)rDZEh&d{IXIPe|#}s)cA#IzEs$Tc`mA# zQn)#tytQg~Z(yH3^I_hS#p`08AG>~#FDI|U`No|G8#)9x=45>f)eq>}7Q2KuxANoB zxb4~pn5PMJdT|`$6g=RVW6Z#+-}H<6;ag5G7mMdTGkwwX6Fa}2POSZV?Y+|Fj#t6X zb4!+J*Z%P|ueaz6>WaGI^lHn$rCT;leVzI(>ToK1_09WIWr^i8ZX8|5rdf61LBxtr zdJ85jc+0fxZF^h}W4gw%cT5ai4$OBRRD4ii=VIXc;}E-O4bz_V&04NZ5e%$vo^Fv# zVE%PLWR{|&kE83RH7wH@gen-=xfY7-IlwX_VaKL3t#Xb6^B9CU778dZ{s;-^|K#!h z)q{yr$BfRb=JR>Dkk3)zR|Ds~22Q?2p$(04OA~>|gdI8Kw}8 zm&?6cSyLK#l^RzR={?ucz5Get?nt5(Q_hP+ic((!{v3KJ`Nx3&sQ(|o#J?pcH8&+W z+o{ahl-PW$YhBOV!}|`tR@%7k+xm6K-U*#tC%tN+*pvfI9tjMh=|y`2HmvdOc=Yb* zJ0<2FD;b`!^7uFky?Vh^;UJLa!k?m$=(gDX!rDZ!2kbow9Cd}qI~MSIDN0RxDB=H* zQ;AXFO@a`MB17i_fh`FPRSZIQg})*k1a_U4T&DP(t6B1ws*q3P`xgrAe-6#nb7q{k zKxi66r%EE{wxuF9GurgY&H^S%|!*gL&`Z_&T} z@6h9_@>mze&{6Mi$W;{M|LkoAsO{$JWk90 zZB$6hQ<$r!aNe2uUOMxP2ZA#iFWsAP>0R@FjYs>{HoZ8*uRQ&sWYA~HDUFiz8YO!g zC3iJVKeOuJp-1^MyEu>Ne0+T>Pj&A@bL*Zrml|dEEtF|kD^q8zYV9MOzmWThu1?CT zPjMU@zBH-lYrGTKkQjE6@spCy3vGVA1)LlQMgBN2Gbr5Fd%(}6$e!lFmZreo(!gGp zz;4#Y@y$VIzmcS0qP?o)d9Oqcw+221NB*<}+`o#sCEVHWCHXNi3jJ|lf2JcKmcW>B zCPU-!9ytZ(X9q+UCnxZ8NRf;cP*9p9FV%@qE=cG?4 z-k5Y*dczc%piMGu|BP=Kaflyru*#p*TIHe76QJ%nQNPM)SwvQxnYYfo1WvUNoMH)_ zmlpBfaTK(F{Vb-jsyHN99TLW z_`f~a{>6#k%T0jkp@5W$`L6@*JPCSQOZANzESnC9uqB@@abT-Rh`qLya{+_Wmx(;- zdOWFp3X9pY!Z_+4b#o{!G~T@a(ueHOV+Z#;y}i6>;ejLlrlG;6-CsO97fSRs3iUmd zu(}kzrfg$a)3|fF#oIu&WZ;1Duylr#Vu6~ z0)B1+2PO%uIQYY40gHzNi`xQrw+9Sf2~SuY+1iW*@(zk9EEG^VDCDzHXiiGP?8o0M z0)iJD5D5|3>D5-L@=(Nsk#%B+pvuZj?nBEC6-<(7;!)*v%l5NvS`bmX<@p=tKMrp% zv%HZ!_?+I5OkcsVVV{uNk>AZ*9=r!19f;WfJQ&1^zn;%xnrPTFMMP4t)9ZfK%jf zn*2g;j|3J5t}UAmN>)A4J+O##5@V;2OJzyqU$Z1Z z=$QVQxB5Y%#$C*oVh-%n9C+?HEUakYo#ViM=K${vbB`a@UN+}toW*Pl_Rhyx= z?~!))%!Ap}xLzE4_y~|q`Mk~R z_`8_i%duO(#C|P%!)IZ+|Ddy;2K&1a!NiULzzcR;lRe_28~S0cXxqi4wh z29|{aY06Aun(bnlOcCrS^cK%Om(Ac%CcvSnqx0`O6JMi%4PV;=H=~7au8~`^uJ^5v z;W)bD*}@qMvrQiHapqjnF{)c?TK6`h?&WleH|F)?ihS$%%zJhJFZuF%&z8enH@%C` zNq^hsEq*gUFeq;7XXy#PH(3f!-&hx~&(Px~n6UHYPBDK*8wveo=XsSJ1)LoDZBB{! zIr6_r@c+Xgw5!21EsbA|QIH|ARJD=4g@ZZi0z=btwkIo7zb{~~YT#gETImBN|V;nJUb z>Zh_A7_;jM*|BbgK)^#41Np|*5cM>(UPq9mK6Zpnk-*@>0E4yZVhj|};lvRxWJlUZ%HBs`%&JB;v`zI=H{JEq5zQT2toi{Wt=1W#A zWWAohSENzZYBTHiSXrwZDw5J_jO-49R{9A=D<&6;Y+#U`F0!`wpZ=U=r(|Uhy#>5Z z2Y08b2;EUw^rb;y19!+@hCP242(T=?n&Xln;=r8rKuB((z@Y@TiUt9-h3o|fxCI>f zs~T83ZtyMm#pmYEJnwbC!~!M_hM;Y~zRlxa&92F;@>Zf$f%#q%=azOLM}-o#UP{_7hX*rO#o4$c>rmK0kk$RZW- zYyJO#pf>^*6CamO6t$cvDRo4&a*^au^PP&<(<1JD+S-0%OQBO0i|GxM?93U3()opu54d=@?aL(Sqz1w4A zqNLiKL%MtV*>`O;zN4xxHSxZvYH{AeosO~wx33!BKJvTlhQkT5v`v%Tf2&=*(j@)9 zUFn;yn}mQ{gdR_WYOiGH9dI?#|vFs`b#d`GkKVRV2zndd( z^WGfjJKL0xB2JAi{EGZ)$166Ut=OIE#QK1VCBaElfuT>9@lEY9p=HOV{ii8A zFz?$dCa{p5ukp>cV>hxbSfW~P_UlU*2dh<& zv&Ei!71|9fDd{Xv)-r`Ph<ge0X@aC!%nk#_lvAAiMhi+_79BcicGAi4 z5~J!(1!48@4JRffGIM*LS!8Qn6>`&P>9HQ&Y_3qrjVn(~Q{3GpdQ)j_x7g92zu5Yu zOz-Vks{Q=!?VCkUul%blzP;^D0|y&Axa2en9ylCiV%?I>?4dAG zv4w+=Ys!I*%L5j=M7haSeBhAMb5T8ZV^`_&BXd31+(=+ZIW5K~+@5DD9e9wDMJQ@d ziQ=Wj4lP$6X@{-gP<+hVqio8-P|5hjomD5?tfcaix3JHKL#%lQLM;kbPbQwxTxRnv zxJBASs5kJG&ElS5f77XoA;$zhD~3kKUJVT|toS-LBI^Kyb7aoi*6ZdKh0NEZ4CZYW zOSC-ZvAFO4j-}TV>Ze}yHM{iY-$jW4bNEU@H;8z_PMd^B@af%9jJRE3z5_v)7)y za-aEcs)BHUWXOg~%sLwi7`de%u&|2Q-#m4QOW01h*XUHf#myfT7_J5jv(SylFM0!^Gtjshfj(-XYj?BUi0RcRXHqSNmCf76`WQ_JPI;pBu zVaTZzGbO-7cWXhS!|APG0+^JpS_r>lwo$0^R@#Ack;^(;^1Q8+IyLGA zIibW;x3D$b(*EC#^61*b9q)F&5qp+o^<>H6cN6X%d41OGW`ZyWXOH8v8Ob4e2?yEy zlh`gN|BQ67dcoW`MQ|q1aW-kD;zP$8n0Zw$sJXJ;W#By7F<}yiLj05h2EJB?CRg^T z7^5~Z_B(+mBwRW+xPMWN5O7m$%Lq8g82={FTkqfWD?z7pEmo*IvKsDaU|P3q?x}$6 z9eTFU8g!ScFnVlT#v;Nc{35>I)4_4aR#6`ZClv)3!x;y+Z535+l+RMw;^wke=;B-L zTT#yUBEC$M7j%jF*q{bWV)W6jF`2eV-{HU{!yF8Z(nl)U~dujFSIc4vCzOP+;y1>`4zyjk@A= ztW(XyJU8}3)2k$z-FYG6r3`wqua&09CLIw!bb&*^^Yzlleh$t}fZ$_^;|&R0pR{>}$|X)UsA$~9 z$2>#4w?N;(XrZ5T$KnH5J{9ywEBdid?#<%aY$)?UhJD_wmKARQOlLB0U@)E-Hi4Vr zjf%`&MFG)m4Z)lVDv}X5W}RVaV9?QJSAM{#J;QNRO_-qI1cjD&&#c0=&WQ?i2)ODB zO!S|w_=xWx53_9R=62mVTA8LB8hMf?G&;DoOyHhzLV@$A>T=a073N=){xU6*k#%%R+)QcK<4(b1tqrdFFOHdATe|MZL}yw1w}B6s z+!sgRV~y?H%>B5X zms*z97!)-!>L_UNKbX!GHd$*SuS16UjKktv8kiMB9!XuFz&5WSkbBZg(SI94ue>gJ z#SyV{k+9&T1lK0*gIqxml%(IxVBs%lXyiJxfXzswkS9e|+U%g(nx-{5F03;nUHqn~ za!&{fojCJuO4l`}m%b;tJWhFfZ!p=WTdEoEXX%*xP%PizM((}14^5S+&aoday^Nj{ zJ$0caXq)bt}ajIs)hoPHZ0iR~z8GtO!3 zV7>lP;HOIIN?}88--j!uSpJoi%`seQ`=x=y>;|Lwjswl=B93fHTq?q<0@qIMnla&@ z1Y__<^V5MTYKmMz2F#Kt3^Gh=h8(}^h=nG-h`DGA@)mM41V?c{8) z=Bxx|>039B<*$}e7H_OPBm3t;cS(rY3Cne78fWHNttsrbR#)7!V#4N%!`9FJFU;I? z{7J(U29x={KE;p3FC1L(c0#+|4Hf?W4bJjR2bq~?9N?e+;vkQYUz1#hBe!jUllZZM zPR|>QL_%d+HC!Ifd}o_3wzEJ|Gpn&X>&_X;`rLWqdVw9@6^$I$0#5vG0&NdMl!S^U z9I{VLkv0yU<|DjPXG3~T&JVr@X0-`Uo6iQY887%NociO4xqMP>1plEu>3 zF1U&->&bJ9`oQ-l5c0*hpt$GR>_f8{PFL-nFrBy4MZStr?I7cV@0`sJ=_Sj` zvYw~S6kt^=i0c>R+!nw&#euUry|H%#heSm5HdC?e3yn?e%*IQj_pqk6lvlL3^ImV@ zy}hCG^(KppN6r8B9QICDtF$fh-u}g+zEP~bFzsI&@0AHb6Lz`HMjcWHao zOa<0wS3_b-8QC_*$*}r5HmHg|XPYR%HBo@m-+;O7L*h0EzH<#6hZJSXLOC~-8CNMV zsVZ=8E8yDcP%Bl&z3YKSX+^@^Z`Qdl*tZ*SHZX8pEasg4fz`%=<+icR#pEDy1qLPs zkD^UUZ;SAp^?l>L&LHcf2H72c;8DiL!cvlo~l|1A0{Sl(7D9J9U@WEBc>H&Mx1GeS~>^&FQD+M_7 z8kma(+O9v~S>nJoS%K>ZN6F2AgeoIui2{!H1dfFU6Bag1SR%k>uFkbEfn!M+w*`Yu zR08+y2C;8HWVRJ>v?>&zo4~uZfITLGvu=Z+&y;$B%}EE$7=KQ$KT@Wp^N~U9L1)^5 zmgV0$dkQ8uf8fjuNY6`SlbOJ%yMb}y1fHV{I3!kd6(uWec+n`!!CYY+tP~O4n%=GV zATzwOVvU*Q%?Z4B1!A6Vu=u5xf6=w4zq$GOBlFCK;#;QhO$+a3O)prnfctep@2w4$ z*M8*hYT%mrfOB3zoWoCjHUUP>e~hgcjxc!(a5gRIt9-x~FoE6Kfyudm+1Maqy8~B8 z0!Ldzzm!_ZY$HyK1cnI)6Zj)1FlWwMY{}8Cz+s#$qmj(5Rl&W{lZ&5|W03;ys{?%7 z1=!3#B>vhuNwYzkN5LdUJ*R~wXGOEs$E_^3l2zuJ!sY3l(++SfH{eKn&YAapvV$b& z*#yq61yhO+v)>Tv-ciaDDA}|%qg#IhQ@}K)a~CpiEw6Pto^|cOR9%Y@=TE8i6UBf2 z6rVX=EIqCF$pXE*3#Ql17STEz~ahTiw!2scgSL%$+5)2n0*7grZMlO z0`9E^TuT~?w^q)6^MI#SfMZfwO-~2ImSnY+f;k7o)MMEL*D4B$EohkcV$Q~+948z& z%iAYk)r?yqzF)Uirv7IyI_^{M+Vjl=^L+i*xZ(8P+%|$;JmHC zyVYQ}c{$fs1>UM6_V^9VDh5SM3T7=a<7f_G>ol0Ka^=?L8~pcgWN&}KbLD}?j~O%D z6}Wax;CmCmyY<83MGZ5P5*R+Q=I|;D1>IU(s3e#uD#TR4c=q45wD|$Tv0j`J8`rh} zUdQS&dBum2yMY{;zu11>TA%rWEi++f)&$+cDweE<4dPFjU!I&-zH6R-0@s=c8|Ji4 ztv!_$rJ4Hfn3LG+jan+rzsu6nE_zMmN>|&>_wND!n+=t>4mftc+H`Hl^zaKUHf4fh z8=UP+1O*jlNCeh1O_1VeU|z;O6!e3GKFhJGlOE^QG47VDUXjo?-zH-B ztm%BLzqr+3^S^bNe(eVD%@Yf_e{nzlYH4E@rE3^}RYZ;D!;)*OUHCS*7=GY*Qoy_I z!UQKTu1#I6Nd@fX8`u*S{3ka|VD{vCAl}}m#1(b=c!NV-!~qts1SYQwvS+I%Y`Zz_ zh5*<03EYbtIMNH!6$7`goyZ`pAe3XYR7XvE*+IsX3zP2}@Sl9JPIv3NH@gn5?U~#- zn^Q)Cv-I^yE&1Jzfi>cfGRQwbR+?bi%G}2Y6Pz&b-RN z`|0GrUAI5*uIoP1`Y^`IV)y^bw3`PUm49;<-{cGVlmCE$kMGuk>npgU|Mc>Sw@NJ7 z!`LCT;?e=ZiBg9n8F)6hFdSge3*gAnS?yF+>{P|oF=5S8fhAlsIoK?@mN+cfeqsN; z(_5J(vzRS8Copu(ik!gKIpN9zzE=snZ40_0tyMh zj+SUn-}iv~WkbvLe-*tIXV3X+NdCCE`Pq&@p5pU-3JfWCe7!;#>kb_IDZOvv-kCo* z_VIO2;FjfDE--u71)l8!CzfY&s2Ja@ikYx<=Y(AsroAlSU|YuBHi084K#GsmFz=9y z*0lPJOH#TU8Lz(Ve0_ocO~Ap|T|5#y%B9b6oNVB#^}H-Fhi96=&h2wf>zp~X`_CbF zhQo7zGtG~_y10A!>z(tOBRKjQIHx%nok`fm&bINE`8DS=P6sSj>`IyT_K|pe;HR&It<}I2IPn_{N4cEU}XHT$co>=m7^~{rx+8h=yzrd5dns=cB?+pds z&9m4gF0ADfux?b5u6liGi=Z^y0tUy6X%DyVekH(r_7=PB9Z~5y?9C1LPxcmwDR30a)c^mljb4LbS^0(Leh#{9ineQWuKZQaSWys2k- z-#@!ozi{K}3O!qnP1jaTzv!`Yt^v>fhU($h?6S#IhH0qjAwPz-;%Qw9Ed3XAP z6O2_2PYV*R6ntPSJ#a;L4x4}NU1OiAw{y>EE?`>po2lXfyNY?oBiXqU(`3&mJTJMo z+^L3l-_qxn95DwtKDb`3$VwEuxubV+`GU58++BJ+hc565{C^RBrsZks!p|F}x%iH) zO=93$tj4;*rQ4T5pn?5@!>htG6U?7K{H^h5+TPY^4bFB3PPw{moBvH<;atr2@)oz` z<0GZqZyfmc)^Ri}U_7YzW^3-7jRzSV{hrKVth#fR^YsGmISpLh>$o{i_V(L-oIBy| z)1~Z7A8?c~yen|{wugs(`s*u2Rd?rlGpFWVU3B}drUQG)2lkQ;&vO5*JG^ECkL{V| zx3<66y7B(qwC5k0EG4SbV#0PSoaVjjz`ML)F8eFaDZjWU|No#K{d2;(7KfKYi>_|= z4fb6zQOka}8tVcz;mcAb2E4Z}@JQ+{;gIC2T60|XourN)=fn>jl?)s$3LK%cXSF*l z)_J$uIBJ6VD((ZT`Q*;>dYOJvUcgYKG-r*o^zn5`>xz$Y7c-`&v$1SHy29Y+jCzH0QSVdZ%K(e-99|MrhA6%_s-VzX9fNLi|oIbtbg`E_ntM6 zz?vBwH-%of>6meKj_2wy-rJ_RcQ@re4!bD5y<1{$rSkN5x6S*Wp33#Po||lzxo}Cv zzf(_EPWBGppYlsc?acJ`@%qM|4>u?tZekSUv6=ATfny`P=VUI6RSO!J83Q;u7#=t{ zUS%&@mU+3fD`ds`CHvZpM6!Z3wzTm{c6~Xqc7gN68*W@%zg<}FT>16Y(e4)ku|a2d ztqfN0ookxeA;74#@qi-}D;I}G1jEmdPg(c`GSXH#9BAbUJhSFZOIMis%>5bbw*BOu zs%|`gQN)1>>6cWk3ho_v$g6%f-_*TnHKUNs2|)>ioF5I2kB=6M>jy2FV0U*{iB1UJL<@a!o}P3l~33HIhI?nEqj%qDyQa> z6HgBK3CXCL{+k(7c~V$&(t`!9+;STd8WspmP|#d0S8!(4i3LRr3~WlPxf)l~j1uo4q2F>E&9! z`_ykU! zVcE3hRV3>gEr-Ul5&{kX7#UVAy2dEFQt`Ot`U_Sm0uBt3-FjP#dY1GnTuEpatl+F{ zndr6B>o|b(>lYM$tmY}w{L&<}nMj`WN9!55y6Fy02-Fj9v7>Pe{ zXlJ_orDZaUWX^`AK6WQnI(gYD9`t!~Oyz`#DEpiL&rZ(+8O!vq7zP7M}C zRrjq*f-SvbArX(ao82jB<~3*8_|IHl`6HL=M8_i%&OHvlM4wL6xVn*5Jf>vl(`n8r z5(m7`E^=jP6yA9COg_V+-ZMUY7uFQ#K5I_hm!i4g!GorSFIF7kx2t46$1%HEc-;l( z$;MyUB*bd3q%R1b;M}MGAmcyxqW?db_jn(!J~AnZL+iz~6?0drEo5NiaM-?xzs{`5 zbWQ46>6<@}-6+`D82fgW!T9jhIC?=P#Cv)|wK$V9x_q7ddrhfuhbvc@(l?*%fetE55Y0${&wSh(b z?jos{ldUEj0(oO@FbP>jFw77-&~QKXb$Rl?Ke3%^hZ(XXRTKi=yLx|&yi}wEwkAz&kam@5c+ku{x#|;WQL$$>_T!l9@u;m}LSli6|M zj=_@#MiGS>Y#I#5CIp>fuL&q%;5cB^ee=-az{4W?O05!`9Jm}h7zIiMU1Hf1cbfeO z;GOrmx&Kpwp31Ud{k9d$_$m%C3SW60e?{Spu3l5G1X~k3(*q$1MYo2t6F!#r3u>fX z>)7*Tspmq0sLn>cAKT{ZEMO3vz^pBGu-)pgMyQHz&kds=0zm zWDZxWhKng@-kk`MdtwJ)7BVnLA9dg~NND7Uc)%d>U;*QqsD@ptt7ouTI4~+)xVG(6 zgXGf%ZMuGiOI@xU5=g$WjPL6q4s(`3-mZpl6}R#ZCl#moYRguMBLN(GGa3Z4IFIVC zImlOP4 zHFtWY($q8@+ol9`dZ~DxRiEo-YQ*Z7F#q?p>vsBEO04hvbpEIK#(dtVImgB3hxD85 zIUT$z^6ixEMx56UwN32*aa>9xv%_S?$9pL%kN57Hu)b#HMV>4RX7TOW$KGBD?v9$W zNMy+bmdk3694-no1X?s)6u&HJw{?l&FWJE?%T>4|^Nx~E%7)Wop9~h)DNYou6?kJI z`ccf0hpF`EsqVdO7bBcKlDJi8xWs9B`wB`B~=zwPA- z&pwGs{XUY$u?r&dFCPrej9kN-lE5TY)WGs@LyxR$@ANsSM;}qSRSoy*czv&7oz`Z zuKxR^-JbW(F^f;etBZP=3%7qbrFcT1?YL8({;w{UgzF1-%Fo*IddcO}3V!Jv4jqpK zA~YIzo>Fo>H05#dRE-ZR3WXegWnS;+E^4`_dGY&xcDoO~3s@eq=5uohK`NSJgU-kUtf0Gs3llby0RB|{?6qtD{ z4lJo}k@+)o>LpjBzQ|l>*&7!&Xauoy?9*Dv{%n_b+`gObXButa)ONV2e7ID1d8LFv zMt&0~gW|uAO%|+GN$kZPZC(p@2Tfq{v~c)j;lN&VAj#rD(t-J28f@G#Y&edDF>*{Wd^w?sSc`}d+P--EsA z0-OGmE&6XbigrvbTG639Wy{iJ)rH!J!*BAh|Jkv`*{bLOTZ{$k>TaeJH<^S57;{fe zcUi&hkuaSlfbr~4Uh{+&j|bg*9To*BG@lD%67FDf-_x48z_d)Foq0xqwLmavP>9FyHu!ZrkrxqM7U~r6_;wrqxH96v7(gKzw3pOtYw&r3M37;Kz zAN0Pn>}#KF-np93y_4@%#FTDfCPf!po9uPpPV#y+YWi;4`)A|Io0V;of6xD5v@~?( zq0oc;c`y2ZKH%t4U@uc)|9xUs*vtMeUhH`W`$}p=|J>N8^JUgsYt^C!rk|`?x145D zdC_S1guyUENFbQozJW>U5%VFZZLTW2cN^^9qtL7oF;{K_gYboArzWu#D>(SBVF{kV z5+tyJ*@unYgq8h^2>Tp|Kbo%GCTwAAPKNI}8DYW}UgMfv!4~avRK$iYKBvX4fW=+H z_;hu1;+&c9Ir$z|ZGY^w?%#`uB@Mz%asu0B6`GVe4)r`{n)<29bE4j&mwihbS!SKm zyK{Tpibl4ek9&ip)_+v8E2*($iE@4yyrA2`UH=Tn#0PApFWRc^bWD7}@%}d3gctjY z6xd5Iv{~0o4Bo)HvBPJ7hS2Vg7MmB$ZXRaaBf3k1xDzgO#{@H7G-2enXl&VHmKnfa z*1%qvz&dqJ3*+n-mKNV>CanIPM>)^<-ksu^9MLw5hkfoI&xdPH3Y#2EGGI%(;hMC8 zB~EA0q8OHz)iz6I-0lmBT+HtKlz7Z_=BZB~84Hw~*0i{-<#E!zu_LRc`NgJVix=8D zzMkS+!`A4r<^Rk>+eM@oXN!hsDooh$&pS-(aPgn>T?YI54xBe$vM)l>=Z}F;VUJe0BgENcfbnPOpW%dGVQ`sSQxEY zZ?5t6UU8QFi^%jZQ=YmU6;WZEwZ!w$9nU#FY{D#;qJ56W+ORDwIk%YSnEi}%_kQWl z))bLha>})_QTjup?2aHEQFA2$|DM|x>;D8PyPV(nr_ZOb+ixaISO8mEL~zRu&CCGH z;9YD*6(+A{^OtYw_`QQa%v^^_7f*eJ!07!yud1} zYGR3?_?{IqTNZQ&L>%$D+I>`rHETm#_5-$@18kfA6`18Hu%D4)Pg6L_l5_QD%1QPd zPbXzIr*2R09T#P;2FiuD#iqK-{XO|aC&(+qZ&41*i!FOvx|!9U@}7Ua;$m~-${Ic` z-E}e>7BT$(j21M8{7CPA1b38{IR9kMy(po-tn17PDh@FvpRjTHTv|bi3+=?&{!{4O&!^ zX;!w%uxwYWq-J2|54MVg_Tq*%W*GxUZPrOLH<@D&u&c0z-LUpw*y6S0V&YxbCu>}1 zuklp;ds0UBTAYrJ`vjX7Za2G#u(YjyPJt?&i-Rhxm&lxG{Fl8qNRh)_@yuqW13UCZ zgI^tT^IgR9Nn_{RuD1L=Y!*{ZtxT=-Y@(kZzW&_#@ZXlHHeda1Y!|RxJ)f^}xWblw z#bRyV&1MoWZ%c3nZsCX#I32R7BBDgG`v7kYpK6QhPww>BH?Eqr7bUbN6@*UN(i&mw zAhot7`GrWBLfgLL$P1=HmrR4Ax46c+o=XVzd-mpLQmtdl@r}dAgXIRAnwj76NCH^yb zZT2OfyX4At$@cCnU%#tQBNw`~T`xVD(BgP+(cPJhOB{Z!c=)T)=~JWR3`UtBu@7bA zR@&a{RyMzK+1RRKzSl(Nw;!4vcF*TZX`57i=ryamMXjmU7XLL9j~Dx_ueNpmwSfQg zs{ZM>JB&=ue5l^V7Ih)pf+Lf`Dr*H>_664WP01G1+HS3Ud~2ty%{taqzXNs`g}9%+ zf1+mMRsmMFIN!1r>=^}TC#QN&J#r_h;VPSoiQlQ+o)4nD^(>B5x9k@_CGspqQ0`&! zKaOjco<@p>C5rLga_wji>1$p%*YAGDH3zG{IgImV*zQT*Xq4R1D0|_l_SLWtHcbW2 z_ws6IZdfDo^W>Cob6WmvWW5{A*0Lk+P3_iY!qSnMODkRX^;GQq`QTCYnY3Py1z)~O zEm?dy$Y^SZ0?&RQ_RB>(1CmzPWV~nedTH$(c?U@}@xxyDkCaG=H zyc-;@DR%o``Q3YLmyY*%IEQy;8JMllDO+lM?L4cf^zzw<_I){R{H3i_t!HNUzWq;3 zN(0h2yJbJN&Mtl7ROYw7N2BxKgf@Sj{(=pfB0g-v8(3TXGNzi`3^?<^bD#Y73KowA zEr+kA9Bke*`CG>EcNzY5Z<1nJl5-!nc&FY>y=3mue75doL|f*g6AvYKyp-9;7<(@> z;a=kvwR?r9PwAzuRQ5S^>v5F#0f&-Uwipif76x~%x-D6D3rgPb=lZ#KFL=eIq~-e7 zu82(_Ja|=sV0SRXhNu&*Q43^WKTEIsruQ~%mG+^woDc7QOxV&jBm6)}dZq+Fx6X?c zjkeIWEc@=g-j~KY&BP#*O(v&%-~1zCIY-^t`obPwH@$Ye?9IHluC5>Q7cI#-JuNmY zE%nw?zuT&O9QI8O3=E1tSvZUs7#MUI7#J8N7(*D_d3kwxxLAdGdBk}+_=I_dh54j~ zc_k#-`S^tRc_oDSge3Wd`GkadCB#HTM5IJTM5RQeB}GL=C8flqM5UxBmwNzv@RV7R{q)hY$Wn`75 zWYwgkt(0Yz)a2AO^?f&>GeWJ(qAc?Kbjs395^`iLEZy`>J*4DZTx^{^Y~6hA zJgr^AY#n`Vy%SxXoxEL~e7sz|T)l%_or8UxLVUa;y*;CYeIi0UJUv1~eM6(eLVN>5 z(jq)lqC?W7A_~HMvXf&nQ^ISqqs!96D~h8FGQ7eg(tN_QGh$PsQYxboGqY1uOR`hS zGt+B|bE-3P+S4REBlS8mET)%PPRQ0ga&->eE}u1R z<&^%_i@Uch>t3^D`j#bAwyv78Wy$QVYi93SH*4pr>3cTL-MMMTjxC*&rY~JOZ_%nH zOSi6Eyl3;`?OS%PTeD^6vaQFqOgge_)q`DaXOC@serNusoyX^{J#=vI;cZ8*?m6@L z`qkT~PhPlu^ZBEDk8eNu{Nm}W=dZuL{`~jvzklDp|Ni{`!^aPo_8o5H_{Z@6jbO#{ z1qYiG`7U|BczW)THLrZtN1t3>05dd6P4t&*XuZfuO%b!6`4)V{mBi*}v3dD_)aP|uY~T+!)B$ArW7 zIX5~UGO>1XiKuv#85BG?Xq#w|omQcqC}AbQZ`RUb$m7~0rG3kD=dZ=bCQ1jF{ZzWm zojYlJ{^}Y@V>R))MWxqjgk&^tY{+Z>By+39d-A-ftv2&_xoUsfp0_zWk*Rd`^>=w| z|7Aa&6~Zm9>(|rw_NB{<$(t_~u6vq!Z%_U=^UmUh+Q;Ahv#kEe+Adtcz!s6#qn615$B%A3n(seDT|VGt@72&M3&W2wrIDgjMMGRN*h9k~?PXOB zxinXOs&I_MRF(3eCf2FbZ!LUztx`u-JZxg&L6_^X|IS{VE)(=f-aym%m~MUl-tzGKOvL+Yw;47|5Zw37B(vFO&bRn|}A!$ujdw=w5%2bKrA{9VSGKc(Qvmd=vf2ba_tR{W5d zZ?*4-(a9qreyN5_@A)4*v|l73{MVH?0q<)=H-*3SJpS}q$|w2ro0R2LG(C%=EQOyh zS=91I#F0Dxg_!uff+@?6W_mas65H`^)BJy5p8C~qymj@)R%Yod4`MYsdakPon4N#{ zIw#%OW2w|8MX7(QtFCb$7C*wUuW9XzynkNIhs9bXET+5oWNyFN^x&AdaPG|~7EuQ# zzOCA{<9p=M^lO=2p%YYPBTL^|et3Lty^qp_t;hJKbRMdCIPh6R?a5qAQ5iasfy1|UQoRJ`Z|v~llkV*B~^3&y_(|wZrS!0UPCqi8G=89LWP&E z%!)p>uyb?U5|zKH+3E6^Po5Bn@R|SS(~M(2ewW*xs_gv3;_Go(Z1RyCEIKLzi}(Ku zow(^GOHW&R-)d>8OHIoz=BVZ>d%g5zd3z+-ns?J|My)0bR^=8YnKbrJ$3(8Ds*>EP z9dj8z_k>srORb92558`F{%WDorZ#kwe~mIeM#hQTAgz;a;n>8o3-a_zZl0l zZFaI>)RYPA!y|tMuEm3hx`+Q*6M#zpK$jLw~SBs2^T5T0C$ali67;ZXPz)gn9FqX zu!LOewv~QftM*@=qPFrnt6#1QPo?&{-Xm70o*A)b+FZJJbk6EIGq0Q~D_F=MI3-za z@3e^xvw8C+I}d~}uhG%lzgNv{XTVa+N#cv|DDtGsuGTxm!N_fUp-E`QhWKloy<9)u zI`Wq}JYah7z*((vRQFiGJq&>+MVl>4B$Sg+JbNaFTlwoB*E z1#D6{xO7$e8g+x$zDp8ny!GyHQ@AEoJ$1?rF7ItAo36=jPg(a?`gCDdk<7G(=~0Y6 z!BeNn#2pIg%=nbbRJl~?zLxoG&IAq?MZS$s9SYj{dm&K&Yh0Nrze$e=DR{4Zx zChccz$~zo6pV=_WJa9PRv15(s27y+&IS*JAcQ|syd}tJu*u|2#!_|!8BWu#OtA3@D z)0Z3#UBJI>n}JKhgavo5o+z<~~cAKhq#fj!Ce>fJKevkjSM4 zZrgGfvRl7rIM=q1agP0mMvevrCea@ZtXd3>CE*N4f(sZ}91Pww&-u-u^xy#JgbA&N zGamA!aa4${xxg$uHlHuW-@WB@4i{OOMccYK3ugZ-nQIZ zQ+weJkN8bfx^1mA_2(McI(+7(<$AvttXVe zE6R9U#*nnVAZ>d&>w~(N#~G^HD_*r%)O>II^t>(WMtQ-B@{i*6ECGxR0d4#Zj5YyW zf3BD1e=mEIz+LXZ^iPn1k*A@7x1hc5Ig`0V2Ja73mJ1y$2944UENKi}vmP|gIKVmk zL+5;nCdCz;kp)bu28>b-j8OsH4<>LpG^TEC*8R6pe%lXUwo6gXiY#s$GEE(r^)@i| z3rf~bXIN0CC~-(}hG2~hQ;eROl!Ks&R|9L3Ls*hSZ=eEOWJ09+f>Kq`HTBFY27QU& zOFJ#vs?y8&92nBr8CVz?SQ!{Px0iQ5XW)5Ip6=eBR$lRaMIDa=15ZIeQ)U0R7aeRn z`#y#WX*z`falO=$t6WB`+`0r#~D z9DxQ#n>)Hi$_tqmb-OQ6X!^vwe?|4nD~Eg84bU8Oz$YKs9fGwoKg07MmbBv1lG)cu9M~M zFFTkx8(Ax7wBDS+d$YW^V*-BwV{`#`-uL=D4jnBwcp4Wlavf(->MVb|E$`K{hIV%b z*#ncr8bG(mn_5m@Y`|?$$rip~_KX9ab3Sk>T5yMLV3bIhvM7L8=fo6ep-h{rj+IR8 z6IGp*Ls(`llaRe8VQrzm{hz95%4ErRAz24S&6SMU;scnJ7?|8LnZhQp?!MV5Wx;J; z$>yQJB$mL)|A9fEVg4KQ{<844Pv6^0U$pT*D6frN(6h4djYWHJ=Y-ak3?)And~}~M zsdEMkXM4ZpB-RfMA_tf!94Nc|VP^A>`h0WVa|_r-3+lKEDufLr1q3Qyd}|Om(8r>{ zD4f8cW-!|@a`NU0+}5+W4R+2>UBGG4$vInKj(Wg++Xr3k6IdgziEg_wN3KOZDj-u- zO@i}KcaxF^zf#ZP8B_0eO1@>Oc{jx@aH_NPL{D1*CY1;2kp`1(UrwHSeezUxuG9%E zVg?NJ)#`s&PH(WB^rd~4vgC?R$(jH9Z%z=tRo;8DobOh7H&-K<ZEbNy2VCJNc+HHV5_l+54Eykf_^Sq@!?FK|yW=Xv*vYkxpjSOO#4g9cUxruWS>B!dy~e?FH3L#YJRVxv~pP% zk0E=V8*8#b<>3dD^P{%qOEt|-V0B+r{%U!}myC8^fyUm9qUBYq=9jG!X`1mzqW;f~ zS(CJ8uzR)_M@$fsn(+U{hW|VIrfM?G(BinTfcLsV$K4GZC(quPo6gP@z{s~@(G4v| zj*jxC^orUUtP2FX7VO}DbAYe9fi=!S!j7HI>%%1LZce*ORwV(}nL|E03c`7k-zdRyVIWw%f7 z-D0qw?E+(ZMBA$s9p5+@)!G}Be>NQcqPy{n()Kwryg z2B8LqcifDhel%!o*vYTJsO?a}oYnB}b$PYM{x0iFYbgp2Pp71Yw$G=LM2{o@7 zi~~3yZ`jCthxcv+?~Q<+FAaEa8Spk5Fi2fEHt8UPMgzmdiTzy18`vfsn){=3g*BIG ziAjejYnlPK_8rbC>a3m$Tn_@cckiyU3RL&+RNtgNSE5Yf$}a_z2NEY&&*j^&_E(eH zj1cF!f_s@aJM(U2&wVxd-LJ`W4Y=nRa4#_6N^f8@xVf%q6=zvh+wW5|>dI%#dd(mz z!2Cvyq5b+X?Eq$`1zl1GEY<~!SUG33YtCePQ{EQc*>Q`Z-eV04mO<>Hu)tLU^ zAJ@JG+$jz|+DBOfyI_*p)-qfSTY00Y>%V%o?V+8W~W@37I9(Of`Fq%S-Zt1 z^ryQ|FUe>(OklP&VCspUt}Vdg9Kg(O!&0Zfxj1V@gJypz#|dW7W3O+_5?Zjc*Xl&Y zjt+*G=NU80LKWZc-tG8`4yO*7qHhQXqY9OGoMiU z;|7CX!#ObnHn|g=TQ+bnK9C;xMcMH)tD$bxiZ|RVCUCk&E#IJXTDV1cWqP+<%i;T5 z3b#yGxccvB_kN#f?Uv~Cv+vITxOV0u=Q)d3Coym|FF5qZif8`f7 z?CLEaxbN)d^w@Ci{q+9tl8oP1GKvc@7b|d788GoHFz_ibmww>rH((Y&z_!eQ_pAat z0~bT#n&~}~jb(rOx!M_O-5+pE*Gt(QudCU?S6Lyd@E|drL8Fe*R)Aysg_}aU7sc=L zUNPWQDPT0(z%uay&)Ey?HU{ic0gUAz*y|ja$~YPI9xzD-Fd5uzoc&>QohnnEAZvE^UjrhRq61mbOVF% zfo%c`Opy+gXBcoF-oSa=!bJJ7>dQL0JOS45f~Si!*w)AJZWUlXE$(`ClK67f=JqA0 zCE23)?Rhrucf)p9flI|^{EW^Dlbn|?Tph>2vAOHox((do_xWBLT#Hn2lFNLu-8o7p2V3Ox2Ehdu{9KsjA&GbYpkb{5!eHwP%Lmx+>3mDaV?cnONmG zGz<=lZSh#fqAO^!_OY%k8_$Lf%6YRiWv!-l986?5%%kKpCuig0bUuEy*e@)fCOaQy zX5}~G`Kf$j%1Vz(auWkSsLWicdhhv^F0JppugcX)Yc_;gvbZD9yWyR2KZ1ONgI4W%;le*fD6PnAF zif$!wtHx+@P4-PY`H01NNue6IYSNRYRSKxT|(`IGQ%G7t1`(@C&pwH^kV^r*);XUWs) z+CfJSb&79QNYWCUdg2gQRLjF9!r^I_4_lmcXLg^~ z6i;B5w$Wl$m}f9?4P%Fsg5`Dgl@ocyYz{cHESSZ*D4@+a<&evMi{`YvqRDbEShyli zFiB+uiue~WiG&F-vd?Mn*05>hj{Xonu_eTC-2p|tCizsvOsWqnvEmd&YfmXB|_16Tsyo zd0gc2j5ZaY1`XlEMQc~U|>-)Xy8_92KJ@F^>%C4IOr{_0B0{a!`E6%GrugNm7w#6B~M z&S=*B^xdpGV z;;d%maZOq%d`Hp7jUo$^{eB(%e$8kpmxluDyf06FCYgB|B~DE%leipKeC*U(EjOMN zf!DKcZ0YpvNisGU^zCtA?(liDQG9L%t3voC3w9d^j*1%&dvOp{qw!LYPs!q)ydNeuiO z&p(!PpD#JZNhX0=eE}I^m!_T6o(7@p1z$kgb@!AT(E>)3q&a*RSOIsvt+nMXo(wxe2 zuc9VJ(x!oPSK`&JqN>e||8lBx&nO&Z_DW!nZdoG9sl~8Ss)2#UhL4GB0;8oqyIanb zn=#8)3rJjUOfz_YX!|oJy~FJn8~L{EWY*c3KSy{0qv{Mt;q8xI4mP-T_jgYfTUX(_ zz%=lZ9ETQ@;Elt&9*viu`^@rqIoIph7J(48(({wdBf8ZBH*8{?(~}x=%USc_l~)Fu zYS*@GYZm8{Xo&GvcwV4y|=P+lw_|69AP`~R8TgiK~SFa=$wP9eJL!7jQ$x5lRd-x)C&~y zv$Ne;^d>YrSR9Ty$8v=A12ZGTihpk+1OyJS$X)Dd%eonJmqGcBby2S|yRz897p5R=Ru23F1pEiR6LCb=Gs%6S|mo%{B6h8HmL&e(BC@{|LcFvqGH z$~TzxTn@H)6eqeq*}6q$UV*I&tErCft%IuB3Pyq63sUCf90>i{bZYf6t5=Cy2FpRa zb$6{5YBma(yl!?+;tSUJ3#`5h&DJNGjeckd3%12i&|bQsfjxuw+NT3M7Vj2KXyNK? z;WlV+Hss7wXyHi^>o3r8WZ(?G!N8m#R~f<-cN6!8=FNdjcN!G4G#GgknAE!0 zW*V$zcVM`b-H^PHpYZ~N`UIAoZrwAQ|17v~?v35B_x5Mj8(kW=x!FQL?BkfwXjj2v zbD)v`#gZPa1|f$g9fM|{9dnI$Fx|0a_W!^ly_3o0M2n$A6H8$y#}5YE9j#FdEX6px z)(Xh!6)>qwOzOUAvQePPqiMZt0JoIDd@c=Bu8z%roVV&soVj0u=WxfaAf9H+30C?W zn4_4PS8ndsJJDpw&^oVEl`Y31mZ8iQ4 zU=iNKu$!6FX-9+E2b-(_COMwz!5thL4GYswG;+^qlwfXV3}DJpW_Pw=iOvv9KhPj_ zqj~GXw$@jMk_KW73A&sai+(;JvEKVLg-Q5816PME#{`BC z#arYZT|O<^;<`&tuVBfy$6GetXjI;@#K%kPxAv}2oJ(~g=P&!MAI8+Y;em=gV9cHtfdiO;ic>|)D0(a5gQQWPN~ThX*evfc86HM2lN zZO)?FIcF?SGup4{RVdPES-m#)!`|DcZ8vzfC;qV24lw!0bAr)&2kYgFt;QRe_yrpI zHZ-*FXb|4eG<8>#c15f0k(SrHr$3YuF;3Fu`q5y!qBZ&e)5e4*QO_e@#wzO*Twb;; z@m#XR%ZqzK4L6s=x}e?Nx}LjMpVHF*X}|vicRVw5{A(t|A55!+n{_;zv=_Mk{>j>2 zGi&|gp35_Oo;fw;N*+tfI>a)AKSkS(b3$`kBMYxa16KjViN$Ol35|je)C7zgvZdLB z6&jfo8rLMY$z&)vDzv9HoMBX8bh^MR8YG%Np^=HBC36F-#tPqytClg%IKy;-K~A9Q zt%uQ#o5vX=8s1w>P1~{Q<|~dHr*#dcu*NHh$K7xWT_DSGs5$t@zcaVE+Y%+Ripe@8BGEM#0}CCQ=DXyD;DYj)R*N|$+3LUXUoc&oX}TWDrLcDG2t zs-;a0do`B&yDa^mwQH@H|0?Y(Yqg|S-ELBgSYW(>=VU`jbj>W&i9O*RQVy%mGL|-M z%sIrjn4fb6gLKD)pc4VtCNW;CZV}aJxZ~i&JGCWqMhk~Q19QU!uN{onZv=RrXnZHo zAi0Ctb3%aN+I=VQ1coqheSFEHsZ zU|u(aQP_f!+oQ?E(sGm1>IufoDgunm8Y0Iktu>`D>?mcJ-qH}D%gA)#_)midUWa4s z91P47j5|vk`kpxSX7Ms*G;jzsa4|6KRIpvM_g=z8+37BBJI`)ESK7d}qhIoXRMCyL z2{&Y%Dww2i@Xe`ZnEt@MX9IKoT;1Z2w{K5w)#r8Q=4g-=X!6LIX51Z~c%#+xZ%`yh zTla2@#2-Q09!;!|f)X>>69cw9igNA^x;*=qrP2W=7ax81kk|#YE-PmUDehSECZ$nC zL6Ymlzu?I`nS+>k3^!=qJFDt^H0iL9nC=#*mUl& zCfqorU&`ue$)LQVMf)|IZi|QTF}A!FYylDse08n8K8e~VF5SEn)V2HJqgQvbWt*c5 zxIB|5dOGX3rmp{1bXVsrzm$bDz3mV?-xan-iz?3nE zb;1kQ3s<&x@17F4)=$-KBg;XrYf(GrHrhz6jg&kQso0*qq17RQNl2oR(Bc z^oj=GRSoBIJKfkAl{8wSG+0a`?l@RBNH-W3J&4Z!aXaoo>w{IDTZJ2Wr?oP_Xkh4w zsogvOfpP=ihkMhfHE`@`j#|-XkTFq=b5e8cdZmOW-|Y3hg|VAE?!H;VDDIKHu$Ws{ z^ZBY>%>AOm%a-g4HMz3)TTj>$QyUEyy%UTw3XDP;jEC-dIIcW{^gvIgg)lZDfk^+s0*)u4PXo8(O+qCX_4?5aa61b$F0j zz!q$ASXN{E?pOD?G^R{zU=Xlq;O+az$|uk$U9l)~<=P`xuWQ;r=|26gcIusohE@r? zhG{IJHG7&pcVyL_>=gYdu9v{EX)L42NdlgBKTpKOxyI~pY_7&}9=7b-v3HT3`Y zj(hc|t&Eo+G5Pq%e`c~#V9~qLC_97Eu+H+Z*hd9u1fM#Q4QCMAt{U=)JAp6dBaMo4uh)xOmxw0QUArY+Q{Z9)bd%94zWH79G&1{?GAo%2?$`uFJugK$Co=IpOND{$)qiknocfQ9fyY9ZxEO~%&b?VFMvu|BlR>2~XacbSHl+z`R z)4Ed{U)-$R#GOzhT=T*??J(cvimv4qVgD;!j+KPTznB%f_TSG74PFJ)@)~S!D%GBm ztexNO!?(`VCA!ziEdXq3)R0bs4DLJDDi$HiJ!! zSMc@B(r@5-~!>f7?}miE46dZjWvI{{)uUeEo&3C zUcf2FGsDT`P}9LhHyRHx2!`!DrBuMc6vrtnSK-*QqFqeQXG>1up`)D-kN90G(0ufx zP*BOKOGa>0^6`n%W;J&fJXAWyEw16Rdqu+|KMCuVhZpcUv-^naCmcHB$IE8+PH)5R z7xy-$ewVeK=yhX5Ve^r84gsz`wbcy=K26kMw^Mj=VAp{+x47keI3F?`7FM6+VW|AH zYpRN;ONdRw#YRU?w#~*03_-gNyp-oRdZA~~#K4{M!NvHB&t_pihs4Q!e%wLAA(F4z(S1v^I%-i|k5SK>9%cZjA z3(DHX<4OeE#Md<@s^#u^`R!TW-qMfJ!uhWPPx4rvF>vM)4N*8)#42F$!u7B0lKF0* zQVek@O&~SiDdr8X#kKRcwD;#t)!y6d+ zbrKE;wU`+9`dYu7a=Cez*$rR6>mD7rB@>{Ll)ETpdY1=ZQ|H zmrQjxd6qCVv52%pGqTiPZDJMnWq4yVUW zdZzj>J8prSdsf_yh0!y)AG1Zz-g0#3(K-7*ZfliOxna=EQ~PUWGRvo@Utg47XI!M#hOn4K0%C8JkY3%`I>;K5IUEVhiiWunmU>2{5t>yIgp@vuW+`b%~yQH&s&mB^6ty%!sU1<(=32EOq{(){V2WW-p)l zVdlc)GjGpXS|0dZEa^@nyR6NF1J5}`|Ezt?BBXO+V$ChdF7Z=LhNoh^1EUj|giC_Y1br%O-U+;Wtyg3>e;}X zCL3aY&E&`Gb^ACj`&sXc5Z-_NRsb_+HQR@-#EcbD2D7ADq!t|5$s3@RHu0$Tx(O^F zYq<4A6sC&_G;$x}Y2)W>rZ*Cw3zd-hdWN7S)Y@KwJe}F zOQdXB>&p*j`IBbMu3~1p@;A-z;6-MB3$vyht`8W6BMz|4(^$=#vm}t^9S1% zM;HV*9N^rP&=#1XRXOj2i`tn3d!7|K2`@fzh>cBw+2cwQ=h=*70s>z(rYu-u-*#(m znb9V3c7s(68)SC@F{(cR25 zr7xi6xaTttvlWwgGYXi+|74pmD@$aXna|cM;Y-@XxM;Ro*{<%uHGXvyT!cLwnglZx zm`+tP*gT#os8iRX@X=H9RLK;fHkE@Fvmdl9Gb(J;=GnrN=8&vz_Vk!eL<4in23CHB zi#=u>3QP$X=Baux_I1!ZZ4^yKfx*_h-9kc0{t& zWC^FY3I&afbAsHsId5u7GcYYId+=|Hk_DsakFo~#kOz#)KNvVZY?>a|=g94n;gonQ z)Ti>~LXLujrsMz#$<(m4c>5KOyjtRiB^M>IiVFDhh)pw)$@4Q${O~ZZ$l0)~S9o=7 zMAOVS8`z^+oY+(}8m;fXY=}6jrOalaBELCVK=0eR!0Uz$tdkPhd>1TAvtT}|acTmq zO2k91)Eg{Xla_b-eBco{d&5C)R$}|7*b-|E#g5uHi-nfwwk!O7c-KKmKwIWcqR_bs z$4qaPPu}o0MZ<2Po7JvF;i8hWQdd{7ipwbMUe*=T$mQe77@~P4^wB3P)d@Z#brXI`^Y2s)yB0t7g+lkKeDPOC~|pd$Z2FG8vA(|a4m~q6j^bxosq+VVe;0ss{^i0(v)Yh+I!W5 zMX!L7Hz$F!VMpul8rR-L%?zo?n_jwG7@A(!Y-i&9xNbJT1EY-T0ao5042})~Ogd*A zrtIu-V9a4?+W&SEN5={_C5sp9$_0z#TocYDPsv=b?7>zP{HbxOg|L+5r3nZ5=AF$w zkkEd6iJN)FQ{J3yA6V`!c*nW#qJi+UBdj(uV$7=lHaKv4H82Y57;coldyd7+?ZiDn z6($LTMoF9QuJ|8GvM()M#ikrcmP=k(;p8W$c14*(_Qe9px{3n~WhYlL=`1h5{P1ph z(UiqU!W5X^?mNJivzFOmM$(zxmqY{VltkvgIU*Uqgh0ZSEN_zPt%PL&Fjk{VjNc@ukGf&0>78QkioIVp?b8HdT|KcN=x}3S;WrK;T zxI>d##zLE@CCv4&JX}PU>}}4{et3NGf>vzj$>ckz|G57X*{7Q#(~XHgOS}WKdDSEfPr;Ig@StY)b9b0Zm<_HaL*}d)y`qkUDIs8rP*}Dd?6i1 z!30Ns4M&j#N3kzVx(SCw-}s2kXyChXa2xMIfe#JBJO}wK8f(uwPW(6F!J&eKa%XPI zojEA?f>G{GqrwwMMHVN8GYN_|PKpjpDguX8e413IG%1-L)}3)kAnWXmrOk~A&AuF` z)k2&9#55bfIT*0=cIlpZqEQR2MGtT=xbZ89b=ErY-C~rgs<>aXeo(C>mn6A?!TX8V^%)!{RjH_4|&3-d=O=;umr_;r37!9v-wq9&p zHS<51*qK+DmA->1o>hEZ69akqw&_zA`&3r1;y`5!ff_vy$_%{j(!JqNcMs*xF-q)d zbKl2l$vAhl?4nhfjGUih#P2YwOmSfUa=_~qlkS-&(K$lvY_E1nPGUYXz2?sS9}D@?WI1`K*(efn$&!qs})5u{RCOR;jllA1HiL(&uVYaA=a_a8kI@DEGus z_Q^q+oP)9gj^Y`N{3VQnZw@^9${_rOfseT8D?O2RLFbaO}8z@1pDi&IZOO zjuxL9SpOw36bd@23OqCOa8~SbVAN>8|Dd(}?9Aj#GgJ7c&b=$Ccehoy^#IQu2aW^R z?>eU0dpRquIW0WpS!S82%n9!$mTXU+aA#*7u zR^b_CjR^rgVs{)BXB-He>!|m^i9yNHzw-d^gPD#CTsjxBl$(V*hB%a|FqGRIU`=ok zN$AopVbZ$9q>k3IS1HIIIv|fu;tuj-V*F_QKdri(iKjI z6vhKrVq11iyzOezQ2pR&*CDQ*jI*o@51RTgcQiE=O>yA6a8Tuj)3FIN4ryLUjtXE} zd2c=6)dhhE=F~d;V|GxBTN2K4Lu;pv+9oFl7KvFV3QbJU!ZUXzCJ8W3wmFt!urhVa zv&NGG2WmPqmuPb-@W{{Nls&?xFpG0K(_F<>xAL_Xt>Qc;RB}-K!a>z93_86BbqtoP zMKEqlaA@5$@4*Cy^t@F(A)V1Xr}M5kusG%jLj;4+45fu-hYWTc(ur|o($ebxdVuMh zgTI#otCWs^=PCz>_S15_tENfs;$Dn07_ojFx|ldFL8uHZboC|}UX!z2A&=LlzkSa0E* z|Gi;>MT%kqj6!o5b$=W(2v87;VAy%pok_64~ToN!C6jXlB?zs<_HJ> zOE&}--&IRsdTo@zujANL+NA&I(5Dnf&K>OCq0`R!xu>k%{O*c-`4@L@j`bfVN%!#_ z;o#8~j$q((IAl`bSXvOMRmh)erpf4Nu%SCB7>9o6o*o)i%cgPWTzZb zJ`g5%T##(vjEi4@0coXyu_ z8Nb-HGwV1og-l%Z#7~SjI+amhn{oYnrEll=o#JpfZ2ICLgF&P5gu^OZvb5Gp#?>&e z-O%c+Z8UaZHgQlk=`%CQ31FXa;PAv79B&S0Y-Tdv;1n6wr0KwHe5T>ejC_8XLmtth zH`R`DoIQQA_VuK7Ya{onxgIj%n(|()y?a_&qaU+UiU0RiU)QcGbX2u*UdW zk&Q!iAwyj9?Pz}H_6vt3SIn14y{rF+NxkAw#?Ip&ch0@FV3_}p&&>UUrbkIdd4Ywu zOiQ`qxA)U3R`IlaNn%*Xz{JMDRAlKa#s4~W-?76=M#m%53QiZ#J%QzOmXreFIzlgk1)VA{t53HyBlpIIPQTU`%KbW@u7) z|Z2YNpGV zUO1%ZutPnDQK+Dyx?qZfio=hD3g(cFYt0>)D#F)2-{mIg!rXHGEYtLVpNrhZEuBpn znmAn!nFuW7JYw-)fKmGjlhT(5Dp#6}?>LFiJHRvNz&yufMFl6_4NbZ?oOG`=75W~M z{nAjC#K@Q9XzbAZ&a;90!BItdr-U^Nr8jYBt=eM58^JBv&ZD}Wskn$~iNnuV6Xd+g zr9ZI!2;7kWCryd3RLj0GL4Z;Ek)`7=M$R8|t&g5!J<-56!{=l|3!lp&%^i$&zYl5( zG+otbl=E;_n9?M_Blu@QH1`ejk5djx8uYK&?WFIKp#S2K{sLLnI7cyujh36e+Jhx+ zK3lC1E?nDRv39mZMeljBha3xR4lr&wKl9&H-HK+pC5D^p9F0AiH$HFLXw9sb@b92% zPh(%QBgX?KJrO78$A?U>I4Q2Mz0vf0%k?I?oW_Vx2Yx&{ns=W`X@^qTWZm5+ZUxJ? zJa*_4m?PS>cgxK6L0oexnV6Gi+cQf1F?nA7Z?9PqYp{**gu0j3jf+7m&P&4NMt4ovS6efO+hIwHJglrn6RT`yvn1pzomdJ1AE>nqIcFYdYosPA>qWsb@@)1e5nOhfbEOZ!^{$5RYlP6VK$}GpX(LHPx6)t3tJ1j?Ua{7|-=^)wB$swVvk$ z9{vjZr+a&wlaH6BbWmecH=96#OUshhj%n(Cb8IRP9p&`ZbIm)X7}WB7qLN)dhvlck zZT-^vF$)5;6j@n0MKqRpOkg@QM=$KH%}ixsAMu$+vrRt!TEcl_hEIx7l}1+I?}-*s+Cb;z}VWtrl(-hA@J}Jt61%VH4zN16J#aXcw$a`Y}nMq$vusOYpcq6 zO~ZvQGf#%NU6%BlVUu{Mg!QtTU-~8m%Yp+3npwDIL@vZGU}R=vQ4mR(aGSC5?yg>8 zeFp{ePxl2lzOeDhIF&x&Q?ST=(D0kl^Ew~nBsJY9+ho}lOc)p+HgtGzKk(2i#5-xN zq0s6^?+_;Of9t%b7p+{jKc?X13xP19TO0zqIfZHLE9@E+n&h~49u-jQSkT!yD=X>o zve^|}fk%{j44hWT*X?LzlS&Hk(pYh38#-r8iXLYR9+1la4 z-KE%-(8MWI`ofV-CT)fzn{?icgKZjtK8EeQT+$6jJVhT4PqF4vVBEwcv*1l5tIvtU zQ#9^aJneK`BdNlD>C^$0^TswiAN3fW*x=N1lTU%qnq6WAAHxd)+b_HBOQrTPK5TIo zX?p0F^kB&giKzxmuM{E_ZI^2a9GH^D!1-rPEG#VYkif*z_kC~JB zR4DfGZWiviNXM3`hC=^d_AiR*xf{C3sj7iRN@>&L1&vYs2U|r{XFPbRI>VxbX}O^1 zpO@-NO}7rVNrm{V(if3hkf60<$*xmctILjjJXV|e%W#=`503A!7u!}65AAW z3>Gj+7DXsFF(^z}kaN&$CYzw4+QmlZmu#1f870QO4Azx}adu?P>2UmQ0EAQ%s!{ zrLyzruV}x8%VxGMxOq=RFGYfpU3dz^6EVjr9LfE1N&-Dk+08aQo?^4eBTYcq&+=%? z?9&>~T*g9)Z7K}98O$9Gj3x^jclS838B5$+s&IA7^1|gocO?B1^iG&OD>!B0SHYsf z;m9KQ!ckzt3I|4}%MDx>3JjBO98+-#XgVmv!1CfHLzxCcqwJeR9?^{*ju97`wE_~^ z_&+i8)@`al1jglSi}E zY@K_DXL)9Z zQ(|Gs=4LM+WvTKR&c-ted-Yr%FzlM?kf2uB>&cO8d-`QVl|q2Ijzx0#-c{)Wt``{j zb@W))JZODb(8|5V*JOkAc@K0#v>7lO9C&C{8*^8BG1J| z;8Z}@qg@hv5;QetgmWdhDA%sNQf_^#NjUD;ZUxQGWsNcsF&!Ihlu8crZ#Q7oMVHUeB%Ok!<3s++`M$XrrP9n=s9CgSGiD-u)8SCoy@?<;St1mRvD+KDxs0n zBcV|!p}|}F1_Qf}fU~T`A;~2gttKUkU4adcM6SJXGt1f7YdJ-Dqqu|vH`_rD#|;UL zM|&6q-!QPr#xQW5H8?5|acm2_hXeaEiQ@{BF80{4II-{Y)G3!ca6|h2jf{+@9v-7i z_MkgW%#AzP_;>>R4l*6!Zinb8M$-sHOqN4bhMrN$h?)yMR(g&Mcssu#gCPyOZomQVVSV$^)CHI;USMi zMfJ`+bPIbZxW7zf8ZWDVWtQPQ;ai7T=15KA@IT)2uujSb}FLaWG#7cGcvJf%W@hPA%5USjcHo`O>Go{hy-aGn3_S zRa4|y61!|(C~&&CAG+8-uSLZ}(M;@u;{|@jjwt>`JRYj8x_N;d85@#V;*M@)5Zkcq zgorf5(Pd{gOr5}!X2EEAMzeu|=R?aM)n+OGxrZ61UDjYXxV*vtutM32nFn1G-JO)D z9n|y@?99}0mA;usb`XnpMZc!L8g z{~P0iMrY@PuIIai6_})=el%ogerOb8c~q<`sJtU1$!(s^bagj_>79O4l*D#&um&xB z>{)*O(#LNeYEzeGo-6ia(a5O({U-L1*sOAHqZtZZ;U}6TO$@nw*Tf6YVPMr*G-2ky zlqpW^u>oH>rz~jE&{$X=|ASe4!Qoj2XBb%)+$dRG%k_ZCD1o`;!O9s53>T8E+Lj&V zSSUGZVV#mABZt1NiUX5G0+T?(dzl35RnEa~0v|LSg6BQ9EO{*WDS^{#p$JQ&l)^%x zWvvWctW0VOOz#}-IUHb+VbF3|u;ts^Q+^A?${NKdP5APMlQ)b}EQL`bE|H64-PZ*f z9Yu;~BOFCt-i!9Ja;q(rdf~7oM&Y~Hf#M|xb&sT;Pk7|))vj*wisM1TlPew<{MND6 zBrKVsueGSDZ%=moJcoX!iQI9gIKSmM3v9e}&HMKa@84|;bonwl4=iwEU`$VI6f+PM zjaevc@Nc8A!$O`eKY?fb?0#!mJQlEja}fCB!2P0u&mfI$T2EF{b69~utAiV}S%R+D zLdh=&7^XFE=}~4%Ss=pqP-N0U-joeDq!<__7BJN`&(ungls>>bHO)eTfwAaC(=;`M z8H;Pb9XOJb!zslgwPO{7!mBxQ34GfYiuoL1TD90JC)F&S1SP-)MvqQi5o)4k|iTpVar8L&@yk5hi#Ujq9`Quo}kFfMzbxs^D z4H`juYyn!VGnCwZ7_c76=Bsjdk6R=#JK@rlC4#1FC)_$Tq3?;uGzZSA2i#}W1^pgy z=?MyFGK$(v6#i%K*budVKWPo8#{$jD1)MAk1(+H+MJz(Z5U|D^bk-=eR2{Tg# z1OK}RoGcGlu5jdMS-^0?S-0PfSt)Vtt8+pN6bv>g2iJYfU}?5KpvBQVLGS0J4|WU| zCCMy*9ti$nV3$ka;CLw5(qJpGAb{gA_qs;ei_JS88OmjAuoOL<&#~!&o4@D|0|m#; z4?+?Pqt-uQdae-a$UXC*jjTIjgChecUYJD zZQ>7hh)+v$-*G@S?$uI(kCvvnf)kgk^(JuK=w_d1#KWg3sPj?0$NFx=q=!7QRLTS@GUSf2;g9p=WgX%aL!-~)0-ye`~?hJ3+%M{eBByYQW%&G z6qvXgSc2N@+#j&XEnq1~;9v4ktUw`5DS<7kqs{5T<}aMQR{1<13Pls%i_ZHXn$V~q zc#e6Q!ov@O@_#!Y6e)_&Sm-tPA)i5mO3Z^DIgH{}2|Fh(;XOJfhR3bctL2H^CXI*| zwt!VX|4ir$C{o*V$bCyff5IcxY%Qyq2lhT{phucPO}aI`G*|ajLz@ zkmDdEbY5hhg3t;@$yZvg$_~tK4$PNYSPT*v=A{^vJz!IK;Kk<<+mX$}$FOim2Y1H< zhN1)(7l!1j1KbJ=IlL0s-5xMkC9o?k;8tT~FKFN{DPj56z?t@d{h<6lSmGD)hjBC@Huk&6=<}7s7 zQj9lU9>%axEKjlINNUL$j~E9P-U?xjI$_ra(W#&FzAz*gpNjl>W~a-MA5KeL zS{^9}`nkq6u+CclvuCq!?U83md=&|geqGD{SNl8Ucf|vbl5c|5Wt?oL^@|j^d&-1M z8pZy(a2;8|V_?W>;lSg;D9+HxQqjQog@I>U1A9sXi%A2^8Pk?;2ZYKNu>4ErC`#~Z zVG!e0;F!_C;={nsvd~R+p}@L6g*gYs_BgO)Y!TUYKvFA_f89cvJqyHcFba7!%IZ9l zj`$%p=V0iIgZXhQ#9k~EQ&=c-s*$f@#*}T1d^w3y_cqJqG;$d1u;+W!TD3qdN?d$H zL)`V|9aazK=7i4|yT%;SC?29HDqtqMNuEhyNzK!7WeZB-?|9;1-P5$>o*S=9?V z49p7D8b$Xt*sWS9wt>O+T63AB0!y9(%b6V&vrEbazxA$W;9Zr_8=TlTyJUVCzx>*U zvwDnzTnpR19&j%>$hjfg+mS)Q>!A161&=S4^Va+KxUA#2vcTnElajaBH`ah}u0dj& zf0pjCD0AMjUMLwVGyxu6t-G8)%>BbQKPWIMXpH> zoa^)iT@JXY{%Dln;UbvD;`Jcs>8AOI~-IVonW$L+pe_RbvHJ9`A? z&E}YLh*gV$`BTmL*H@Npj$U4wq~60&TJvl1rX%cfKUtM_vIaFgjX&_T_5gbVQ^J(z z&$w4k_}8^SaJJPH`M^ApRMjP&xtYHl@Fve_}PA8IIh>(8=f z0ejlbJ6j!YZ(qRT;=p>A^32*jkaTrdQh5C*Tb?h`bIUU!CMa*!&Q&N_ienNZhJR< zY68CkV_yEVzq@Ar6TY^d=bVDzDF>cM{Q^S$o4rF~$`zI$+3NTG0pFwjuMhrpHJr1< zO2(P_0UN`j3yRh%X+0Ypln(XsE0^gA20ESW6tY#zvs669;wEhCRiQKSVhe|mt-?10 z$BioPg0@ycM;tFcXyj+tIHl_?)F1I2aE#2WhQJu&m|cjxb0J zS-CEXH9K@wh+pX5D%RK9>o@M)w{dsNQpxH3dUhcP1XhIU>bh*n&^&asGj5Mx*B4!; z8ypXm+kFwz<7e)iL7pg5VXB3jer0**c}ZYI;u6@R%@xQ6%faB!^{M4;3C7viC_^R(w%7 z)XFETYg|w*pEw94D&EOhs7t+{wuHNHpSFbB`w4bFaR=1lDNua+rb zU}0^zz!2zMDbU0oynyj)@WCyIllvoMr+Nh^u6;CB=Kznfn8p8;&Fo^^D*_L)2OSFK z)+-e~I5V#D>C@@LF-r_11T#6(SOnW-7fK}CIfzAaPuf}#nKsMvLyjhf2bk1ZI8-5dlUPJr9FsxrP}V z*cBpr7(0a`L>{^FxA!Ejl~-q3)Gn5ovZ7TkF`=+s%(mjAE9bpO3tYt2XWnLxweAsM z+$+CfRj|^|H;>M!%E~Y_1t{=bW$ab-F?1ERv^u)jp6~Z%J_G-)N=t$h7@H0C_x<_o zs~hp9;dE5)#ntY*mzH$hofiA;sf$#7Yvzo^PS%CbX5}n<&^){77X!;&ABTnK)IB2sH9CSU5DSW1UjKt{HnGp;>@yib3PXlJ^f?Ipbz{ zvxsSyUU4p0Jz&Cod-imOSWbmA4M8tddmaeBS~73mcRs@qfv3D?4GpZ?YnJ>F?p6@t zPG)B<4|puH>ck_Fj~m{+@VEc{g?ZxEe6LAl8x z#dA+pR^C*NYp9i!NbKO;ID^0ED#OiGlO+l>=QwP?QmexU)9*{^21NV>Eqn@ zQywuc`@)j=kS+h3rCZ(c!{M-BtCN(7VXbcGI&9f~3+HC*KarYqEs|Ov3Y2d+$dPBD(;W1GS?=cS4wDDd`2RUz$B&ABx+mOCX?*Ffv@1-6gQ65 zhe8)Ucfb2B`PAPW=-mZh>vAvSOzva8x+%Du4LW&;BG^ z027-}gJ4scxUs$2UIxEkx9K5J`2SpZI zv`BVLW)W0zG*&pswgF}l% z$vZ~D1rDq+0_SSq32*tM(`aY!{l<`MH;YtXG9b$23~r; zAj_QFk9{#Ni|azUocCI&U7tEylC)L^ZXLOi(aDdYV$)y%EkYPeOBF&+HLEh z`R{C^#C&m^mBIUVMAkO{q3jD+R|pREe;2oE=1qnVQ}41wS%KPRa8tX{JQjn z_Iy4828Oa93~D}$1fotf@(Nvgt?|_T<`c%Zc8?$X3q|aaW z8Ba~%Um(W)FpX7jPGg6VM&)~VL$1lQ@4S1!+4r1{n|36&4Rz}TO5B$h=yNbxq`LO8Lb^V) zBJfT)Q>yzH3_h2Xn66cc{FjJ+Rz!ffJt=0jTW_a1x(y23~QIakh)uNfceTd z1@<*rAKXLkGD~mJd*@cO>$2>I7H1bHzTCr&Nws%#gqFnqt7<;?=kh)u;U~H!yvG{8 zFMnKrexv32h0Hg~gmfB(Ruo6@DmtrA;;?JvP*jL^EflF}F?e$=Z#4tYp$`W9%shv# za($m>z{xDO=L6T~1KbZ5@Vz;}WgEa*_kc+`m@(0T>zn~wtzvlfLe+e>kk&vJC)JQ{ z$HMzttBQh{CAZtRENf}l6t~s9$XD2uTg6m;du^C1+r$a1Y7H!D2Hd+oaPPWMb8Az*HujsC!V*=Yn9OLrLNR7S#`|SC23^G%%lQ5o#4+Q)$TR(Xdu^ zVo_{h?VP|-cHK=?!NASXfF)VEi#_Ijn~iNzh5xqHl?9EV2e{T3u&pX!D`V$-+Q4C9 zz*+YpJGYT>=5wC%1D-Jl*sMFma}L<)8mGTiHtuF*S+Tup&2;16iwhk#n0PmvG+c`7 zoaU{(y)Td>p7T@nRz}u#1vbwHmcIyk=z^2l0+!YbLhS`Y?H5=r z1UNnj`iOjV@|mD$>%boGz!c4(;BtYbD%spXP+@j)#G)eR2M;Y*uILh2QNB^4n>D%d z&9T&jOB*B0f+88GD&9!pT+Fr2nKjvfd*_0tjtk6_960noFa-%Qnsjg{9$>w3P(1U2 zXQ3LqtHAVyOL~`VkuaSg`Q>x0T|tY(1*YG|rcWh8eP%dto3OmGV7YoQqg_+_gG7Ab zW|lkyC(jA2D;{v~yI`;*fi=>AYfA!~%mkKn3(h}N$}V4Ykl$9n@KM%GbZQmfqKW>uRldo>b&nln59 zY~Lh2H+P5I^a4))4(`bgObQMRUST4N0!%)YQza#AzlP-p>`VLR1OQ)-qublo>IBwl`dq$Ux6&K^$ zXFB*k_Wn4P`RYVws|yRQHZWT~sFsbA_AlUQp1=||fqTUV?zac{UKVg1p2)fD1E-q6 zO!jXw#v2&#C1$N>v-)zxtm0|y!lz08&l$vTnzsgU#C%B1=WzSXBB=3EP-cUb#0Ex{ z1#JtIwYg3xJYHDO_so)2F?X36lf?$+e{ms77gw^D6tGu6VE6K3t)9Sr)qr!+<|)pG z^L%F-#IVKqKU*HTkSly4m$(IM>dh z=LN0?y@hVmmzJx!6-=*r8T;Txl~j12cksfUBCS^yrMq@6l#Pr9InN!-YX>^P+6Qf{E5G+QtsE z%MY+bJFpZvu)Ay!xYD*%-Z3mwJ@?;sW(fsm=MQb`w=ugIFx^mASZJ zeM|v+*m1?*x0tyd*^3Ua#ywzPGeJ+?fbCqsKW+;K1NHx>Wi9TPY@8aN&$aWyG0YBDfA-pH!9nB8YVaMi_?mtL@N%x3d{V3gog z#kFgJ%MLY%hUxV$t6sPEwI7S?|G7%~)k4`*%mLb~({?TNWoF%3#pF@Iwc-KioC4PL z37j(%*ycHKFP^~S#lXt8jpdz}R`VfdK~1;E2en)noY@4YxF2A(DVg|8gt<FK%F$57=QIz*4Qi67zsvc>?qF5A4>6gaF6*jpEr6c@0lIcX|nk#0{gSpIuy$_aXcgEyIe=?vLZzXEl8*z|4h9a}rJIWa7Wf}nS-5OvahQal zyKz=i)qPfVQT4tjCrla+IVgUg@n*)t;MWU(M9T(FX10DHD>o~{e*t^@0>RVG$U*eM{u>=LkBfPvA;fLXy|o!uU0z5qri z1rcijMmrNGfe(z&O$_{F7@P_jo?l@;m%wdpuu*zx%A=VZ{~e5u7FoUs}LEZ;Pt!>uJ}1sM>b?N!MbcJVH*y`=q_t?@ayP33tb3D~7FcpEzQD=-hHI_^ z_g)9?um+Zd366+7MS(qVZM|LCy zgv&7g4`SM$z`gd;?u{1n9z`mr^yKgE$Y=CrJzdRwCR+JsCr5IFV0^$oj%@`TQUc7% z&5XCSd0QWdR|+n;bb{^jip`gAY&J-ic)44`x2NNPd++|HAmSSm}6v|Kj(-f)`*yQz~FubAU%~f>MM+-jWZj5)YU}FC+_Y zVChicoxmW)d~4+@+g_HeC&g)M=}MPO8>X9HU|LnuQgCVd-`~~y+3%EWUHI_e!e7!` zKYTs;&nkg=qm}GsMmgUA_O`_=7fZP}PvA&%U`s1t(lC?`aNxR9kPs_y@0YsLGhhD) zYpu*Dlqovp+^V@R@qs~^;bF-K1~!4~$-Q@*3|N*uV7L6h@HY1WTiryV2@LL46K9{> z&UMaNXu*R;a);_d8Db9blq@(a@PT2GAEQ7*fNB8uHV2jstjfRU*tt3F^fKfty10^=h#j;$A>E^lBr^awD#z%rqLyQ$z-?ZuUD_g329Hp+2e z$q~5iIDxHQ@sfSR^i^M&>u*k%+|KO$!DR2>+Z$iq{vx+Jao@trig&Is>fC)O>s`RU zG=V)}0{adFjue4^TtNm5%o`ZR6_^7O*cDR}+Aau6JIJkJo|K@%+-S#abRZ|!naSyc zpG5$R;;kHkI(GgE&a3a;vw3jeDuL^a0;m6mPq$(ij@7)J9ru7!fpMnXb)gFk3-lPZ z`X81&I4fYlC|37SC4u|ghKDQ*rc`j*bkALGdWkXEfNhV$%_#o)$&PQ5beI$czJ)3= zB?s{SH)D$n;BDW~qvycn+`w@?j^)wpTX)W_WZCuhat6yIy>|<;SR52qIX1MseK&pG z@sk^+OT z{R7AH0`8UtEV6=D=eAli|8r%GyI%TMh1v1}OO*q=u>#9P0nSMY>{b(`8y7G>ozu>c zz##R3W6}cV+y?eZ0h~q&FE7-b;pThgSpR_Q-DkcF3@`35+&S;OAunL+0`}E*NkSi9 zZSZ4S`haJvoq?f|2y?T@W6!VqiVZ>=xI+_IHSD-0i`h(=1mv9>I4&+c!X9?cM&(ee zYLAf4Ifb1IemEW$2s+DR`S8$DPeC(3r#_LB$CgZ-oYS+Uanq%a(AYA zHnrSU@R~B|*i!}$4uvVrFD@)h;CR6@*O~ug>U9Z|9p^UOYRwkc=2v8?(?~REQAzKd zv+_yR1`>7Q67Y4b37>(=HsIX}idxWeTs-0*5AvcQc-F z^35@LI>B?<%oKh-C5;CSySW%1Gz#);ZfudM`1im`!1Gf?L%U*y$FpfEUlbfCq;Xw* zz{n87{H}@pK#bD_MW&T0ZA?DJ4vg$k-fOp9mpKvacPb^@fr+i*FprRZS@8o#PNf^V zO&kUaPsAQiieAV3a+>mr#>OeD`$SD5Rw7e2y-bUn9DE-F5%RerTC=J`P+@A z4ne+x2XV@b8V<}>&n7%%>1dGO%qL<&?_M;rs4DMhsw8PGDWY#HdlwBrGGJlX2MfL!w|_ zNAp9T>ukqakFr@xEaGj`uq+5nWt*D3DCE>t%aVr+INL2+IrR=OI!{m%413VWa962K;m=(AfJ9-3|m>VO%f9}G--4kOkT3FUGGY+fl}kO zdwpMh(&e(-N?$me{(iu2ui-SiHh{(a_k_bT-wrxC{&3_w6?sm|#g*OdOIS#c#nflI z8(q9xmh)PfGYd^{VAtzv;nWde6jpi2m8Nl6a?^ws;Ry+RB@#|beu7+qJd1czI+(=v z6|~FrIPh;#ZkAx1$ekIoSn5l_@fk{un;c#o5IM^*lXG4HcW}=ElinqleSCw1uOIQL zHC`-v;Ze$Ft|@BOKV~ze@i;iB_Bbv|H*gN-wrbN^5yax%eM{TEVWr5Opv*`A0&nRS z|GfH(XR?w;Ms#h%OlD6HW&Tp*&W-aH?AUwst9{V~XQ>xl7B(Tmvzr1~l>fX6&t)r{ z-6f$EqQ^2->PiBena0HJ=UVlu?|q&(?e!hu=^e~+CJ*_ROmO7%D_}CtU=rEFz^dU^ z*s4-+fF;=Gs3Myqo5KW0&h^SoybcFgJr$I;RZM9!nYFO5a)tx5^oqm6B^wxJZ!mK8 zRmz#BB#3{t{gi4oRhT(0`L5Mox7|}y;>%4lSKJRdz#wMfD0af3EyzriH_c+H=(&Uo z!ZHb&XB8QirgUlwoVXOUT_J~sqHfyqbD`?rhRyjaP?{))0W!|4e{c;FPi$E4Jj_Lp8fJ1=Z@mTstXwH zgQK~wPh%3gHJeq3;}=i3g|lQ;feee%hiM`|^27bV-uH^$#l-YYwK4pIxA+VJp-VGn zH@g;@o1JmI`oB?Z6>Ip}eO8YIH*agv)CufNicpgDH2<|mdEz;d=@o3RT|S01*}f5# zj(#mYfsu3C2KHbF#^Wn~9JRXkkiTp~BXbbLQS~zlZ4MkwBA~_US&bZWJOcWj(haP9 z2~4&h4lpnta1i`AX*s*+g9p8rr5pI}2ykVYBy$vIo&EVmd)toLDHHI`tR(%Ir@(c-MZ z5^gBLq`=^&Icw2PCgBRuv3OP$!eSR1uWn|}+QBL-(V%X@nzf)!eMir=1Pc$RUY3dm z@g>4@CQWwJnt#<|#|7zsdz*I~vnN>0o5(P)vO(wsQ`TmtMx*AN#*BgrjKT(tvJOqX z)qAxiSXDI|IU1I-Dp|&KZCSdIL3{zjq|b-`SWV^nuyk6mok#-Xb7r-fyIDOdm^3n0 zhpcWsy72IRMj5*o&GrE;i>uqx8`vCJjudpT+1uDVo2)rnw5FU(TzvB`(P*XQY+1<; z#zSK_1X%t#v-&x(#xk@T9%xe8(B!|M)nB3|Z3F9H;bz;6qfQcyXA-7xA7J1) zp?*MtQSt<%bpo3t!$RH%4ZI92^*0%;J=%m*nC5BmS~KotXV`1`P|_~Y*fq)6meE4) z%E3iUQp^rbE(XoFom&GRGzk5>(NObBMo6I1u;Y-nMzfIsFK%G3TF_p( zfW4xiz1V`iM54Wb!_%pzEv15OegoGrD^JmA9rY^v)tl{$=eQ*AKH|Z^!Y9ETaD&C~ zLCf{e$Fv`?L`k$d?r2TRSnHs{Vt4^`GNRs%W~B#Bk1jPkylAqyz^uP`QrZb-^$I5I z8O*X7O^Grs!3M`{GfAn#rR0*<979tY*I1m8i@p_oB&JfrZba zG4KV8-vO4-cUYocu=uCAJA0S~Nwj!QVEH-ayjKKI@P^jkE|O^;ZD}2CnH_As32l=X zv7N{k6jt$aG?-x#$Y2#Xqt<)LtcgobF^C*kb9&#FbvG6)VN+l@y+gctHNzf`i%c4f z+l99T2C$btXgsIcC@9e=v7k}9gOSIAQG3C^*6Y^HMmKC9ENSQc+&mN9O(&-HoZPIkNng*SUeaO=-g=W(+K%g z(i;DRHOj)l!Jg>u@2 zMOa)Tz5i4;2>7bJZoIIt(D*~vRp*D2VSDFC+fMm;a6ZQlhQoUrKJH==F4(Pof+^+$ zd+vrtiH=4=4MwrnGvY7Ss7SERJJRB^p@Hwj%s)bLVU|$7N&F%MGFthF9C~ZE2MCxY6Qz zLrKZ-SXIO{5%HhVi)?CLJkIDGF>c_sXf)CY3D*dT;)pB|VZB_)mSw;i#Ifg0;2u>D zma>D)M~xXJ4Hz{PS~E0SgBa$jZ)mVFU=^`xS+s{WYetLU0S327TY*arV!0P36d2RD zHu5K2lep0+aiUS-0^|M%j2b5zpFX;!Akn1G5Um!`q`qLb<%3f`y>iwOt2G?-HBU5^ zykPg-&?qg@D6P>bu%nTG2cx(}lca8QdJe4s(-!)AGB)+eS%136d?ty$-{t_+@WO0R&& z@PmY>Mytb&W@`o(hYQRGGxnt@uxzr>o4SZ?war1{oHpAF%=Q<$grc5S`A=k9Ik z2u3~)=70)YUxy~afS8BDF#~m&!y*)KFt*q>wQRjBtFc^BYocJV-1qJX)k$67E;l|9 zJy;>!8f@VjBG4Mh&>Fze8ZE%$>~M9F4y%&_OSA>6lSIplgYmW%A~6MQF$^sp37U>4 z8hCd!@a$+%e{ep)qiy37w# zCKg8p7B>MF2cf6Ef-DwlM&FE>qb{`S7&Hp4xGl}lc%tQujVYtUhKt%4T68?5&RscU zD3d76cec=?m#=-X3^-s*!g_V*0g4o(+uxn;1D0 z8ru^ac^*7oEEaa&QNQmtOYAm7p#@LV_A>IG2$oc6K5+J_h6W@1g+uY*&M4mXndZ$H zf68Q2)@;#=Mu`MQ$*^$AWsDvHt{MR{LAUo=?@_E{O;SvYEl_);{O`3|Uu+5An|}%k z&lRd}lnO0c6>%iShf8fnbI^lE-U#7fffinaMhy-P!xPNr22X`d=B~ZKJk5&5F@VWn zLA>LIrnCdCX){_3I9P%jSb`f^EX`Wo4Oj(B?#XXFGFh+PY zHn@e$_B!wEdD1tTOJe>f##xKBR&iy7{?TO2s^05+g2niOl2E}#i5-km6B^U+HfRX2 zN}px%^=RVw!EoWsp`%yN)TP`>6gzv|^cl18hK7`&?P-kC6)n*PzPtK-j~+a{KeEL& zp|#lIZvMOMf`8fBKRH`tAF2N9EVNSM-YFnF`ME&la+w8&)+ZRA{}h}S#-yUK$x)!0 zm*XFkc}Gi%0;~RHA^itUdK;S67(#3`80UX%a=O55c)~?BAX%w`;l#J8>@ygD%N~5n z=QN8`>&HZ6yO%1L9y>0W$iS7sAi^=_szW2^3*M_7T09cW`8LfyH(HD(SXBg;CIsok zGB*l4FvU8s`d?^qHEvj)bVu=P7}tkuiEiQI3}>J2o+|EfSY$z?cxsSKIis3t*0r9N zopnub4qwl{cilCht>Q*D$NOx>2aSIAk7PfT1g`&}%*Rx`?bac`Qt2BXD!vJ{^M16p zXtjCKb3D04J%ig==vEp>t4#vSd|%LXuZ%<^`+){o4Rfa%Ee;Bd$~zfl-ZKTgkmJ#4 z5L{3u_islJ`;I(Tfrc0B7=Cq1W~VT)I5b@1ShaAY@x20z_TRj%&WxNZ8boI>uy%Ye zXYb?eVEFuKhxP*21y@>C1R8}mxCmKv2pcd4d|>sT;bi=vf#b&^4`%25Txl+WFBfVU z^-Z!*^)`xJsMxlQQ7Yk!%Xdbxgk+|C=DE(MhZJSrU0&!Gz*c#o-M9azq6CwoM3d}> z%KZ$b`#1b7X5(7WahZ3&!1EZcWj`7#)R?RzGOiXi2Pd!`=S~f@Yng21m~7G-Eby&f zic#hTlN`g9RQUyh7npPtm^>9)7z`%m>}Pb`(2}sKVS$$|=ZclmF1=i)lV9v|EK8rS zb$s*Olid;tQ~t?ZXpnr+WG&IuzHmOHQRC+-Tb}~faE_bC9!xD(jVcaJh7zpt8!j$$ zVp`nFaN*0v6JNu+_)9MB|8dHdajRX4m_><3=xd3J#BJ*uZ|kupHHuB1-h4!J&iv>W z`E~8={XZ3#{hTTOZ}$F5|NB2@um3s!LuCmYSMYYR=PF7n(_$m~Tby?+QcY;_FnD!t zM}w+`jBH2K1`k({h89(YMt!YD-5ZP!qAWrTj7%Dg8`YaU8(PkM;AQP-bU%@P^>>lR zez%4Kg$s-f96}N%84ngPGIR3Ecohf;Hn}tk>A6jD2uyDAtacOOTC!ZoG`*F}#zH`m ziOn$WOpNEDlT%m6?fX}8_7Y2$$l8*9C9yk$+Il#-6ps`OT2JuPeK6smubisP zq=E+voC7r7nk42fN;@U8Qq{ZaK}1oyr-Da&g~rQ^$^GI=6I28*K6ud7Ce);*YZ#!^ z&dVzytB{az!I807irX!OLvYcNUUA*1B|i>08as)rIUc&`%)iBnUnwNOfrWjB!ZAt5 z$cHC&w)}Y1ZNRr~bD#c-7)95Al4%NwAz=ZQrJ|vMzE7LDHB(we!%TWSn7K@rEKuq) z+;f46SI1_Pm`L8vgKn}pJ0?z#t@*O>y3va(pTiT~l{{Gl`#q02r}SH%l;C&tP;(Io zR@-(sWx?Bq!|CoEiHG@oZWMNCisx)-V z4QyJk6!hU>ph!zcZiqr$Xi}JA#L<6`xb*tG#d?f3Ih@Gz-?KGYy7WPX_=I|$O=qmX zE6mJKbk*@Z!uQW_TIzwttbb-6h-quvFwN=7gBID=J57viY6^jD>a%55d#VK~2(l_P zIXL+mFYz#1q0q=u#Bpkgz$X@!2_i)&3=M>!Ym> zyo#3dVwUAle=dmES#%;|?5kXT2&09(_55Bdn6)ZjN(j6Vx9rdJ1Y{pU6 z^o5MdIgWg`TNK^jpIRtgHD$WppM%BkFSNgKQD@iVNc3F%aeBJj;-zYJ$2$F0ZU~FI zxhh%sOQ~5nia9uP$C`MEi`%ejE!nUv*rQqM$_`h}fA1a~Gq}SjFj3>ErpLr~8y+Ts zDh3zLUCUTZ)RF{dX*BBm6UL+5{E(nux+vI8-ZpK zBgQ=%9g0F(G0Da`ffu7+ycFAJ5oc_W)s;5mk=XKp$qG*Y3_BK1P!#NNXc5=j$n3W5 z6l>fKVSXWv&Xf}l43Z}pO>QVm;0$>%S!h86vrvK=dxUEJ`@#l8*#M3V17>!=)MFeh zO+K7HtxU=r8aVwXb{iR(tT`yMj8*AD6Tik%rlUDs+a@F(5_*@LuXgEFsK=2-;%gLG z4L>!;o&V}9wxLL>^1#HtD=RI}?VQ~4amsR;j)azKCqWbU1f_#ot5t7o<=Z^*qmx3x zLKfi(j^evKUf7ffwyMlv6j^ZMkism(-n1!?l9$}@)%@mieCwP?B3rKp8+mPHmeg@% zw{kcnbm9Yx?3V>Beg=owl@48DzpzlUaf5^08-wG#8oq3o4>U8g82poGSaFJxN5Vru zE3k31sR}DsBLhRyfh`;%O`;qf2VAbqo-mc?0L#)HtfqW|tVTRaA`3g1O1&Hy-S#QW zJm&L^U21~vN{`PPN|UrZ*BojRTd2TlxMRbk428wVE*@YvoOQ?K%?0O@ZyJ1dZOiu~IyFbT04b_K3U7VlIzthau_EuANd zLPwNcg-aOPf+jfemvbDJd&9tH(4xrGbmOq>7X{8>9!CaIg98j=8(5rPJY-@EU$THu#FKyeurJM)& zj|OaJo+sqR<-ous_~QWYp9Sn59?Sws3`f2(D9+8&d>Z91IFUQN@2~%R2PV zIOT5Ym~E7G@zK6cF-Mifk~t?H@;$$FSpL&N{@#L(Ny#x1HWy}euKO)Cy<$)D`5O#n zNnE`By$@Jz?>rJxGB~VZbg*;5oJ5g12aasoB)}FVlAqFY;;^Q|LS~_y1{T!=4xC;I zO}aUZ>=qm?x=$Dw*h^k9S#`8%vrlNWS)9NmFu#FUF0fN5<^X59+#yyj0|s7$#R5+p zPdczIGGX93C@EiXK*u>(fGxy9SwP@`oDbtizj}2hlm3Po1`k-$pDg55%WyJgDQx8W zw?={OxXbNeBcJSC$BO)_=4B>dcP--KKX+L7P}<{^Cp*M`G_cksu$kQZ^k&zNM8VfN zg*IOple4*>@qgAiG)<}7;6~!N+kgM`Wb`ShDLq&yaP>wr>!uH!<;jn@X1rk0owPN| zAs|`mipQ}W&$KSXzkSTHCpbCfK2e}&u7#J+x z*AyvmIwdGDRb=d9o1&&*|6m~ls{uo02or;w*jonq6)c=Q3u?mtuHVh1)Fj8UHa5=!Sf<-I5#SyO;9;OhO51_zEz@3vZxn4_U>Z+p9*T)mMi+%2KdEfLNArBTzNdGklc zz6?I=(sM>T4*#C| zgOggzA>kKYQjJqI&rDSDY1S=aWP7oY^Tr`v4`#iNMu`bb3LZ|DH=GnFIC`lb()r_K z`M`-&M}E2I+)H;3aNJldR?x^jksW;0-?yM0U)$HIYT`$ptOD1<$sm zSl{Hfxu}#7^FTb~!Gq3!oNxXd;44|0CUNiek0kagHUqZD`+qp9%xKmXXq0VXlB_+X zmgA&m(k;SsNcYY~C9}gyuR3LCHtF7QKdP zYxdv(qkzN3o>MF?F8p_VBoq#r#~e~w;jA0eWVL}wWyN9B8po`-L*@<7%r_jCVDVMl zz+@fcl>6wA+y-WC5hu%>!<;J)KT2h`PH?=?b5P}mn$nbJ%N_^T4J|w=4%|-;Dg`ww zBphI6xcaYCKuzgru!Cz@NCem0v!+`GI;Jjf-+Lf%RjiQD#hiCf0W_`xB`14kq!gtXU$ zaLOF`%Ik9Pi-YM3XGM+Y5(-VZeFx1%nmji-v3oe{eqk`)aES8;lir*Iatcfi*A8(O zFlygn6cst-_$iJ3!!yp7WgI<@iZ+Rg9E_qdhvcR(+`rktv*iFw&H+}ACKZQ+Oc@87 zR3pm07`XmuKH_-Q*6{pZhAY>TsoX8CZCg9qWmOrkHQbu%WMZ%&`oo2oDZ&#z9F(_k zQeAM^=7IXXpNz5zsr)x2rst^cxOrnoY3jdK|CBO1IK?@fR16Qw^*oO4a%^)tsIcRp z5XT_}8^^qkp1D(;j9r?f8yi2rag=L_6X82F*WeJF&B16UC#$|juZfIe0*)LL+8t~d z0&XgEWE@oab6Rzd1BcHc&M6G`(+-FWFtQdj%A9fFFF42}a8T}!1AomyzAp{pXAbcE zU=;k|P(0;;Y)K>MiG$*24v2qPEOw`HTh&1^frH|A8q+mSG9Ed=m2iNqpjYfp+w}%3&-522i-e)6z4D*n=pGw9#>w$b4G#dD@T*!lqQvb4;mOV8otId2#YXr zzR^%NWnkzDKQeV4;~fT;CF@w$x2y;)c-h1iz&O=Va`F|+bXJq+^Rf=GtT}M;&B_}Z zyd0Zdo;~7ylX&1j91|bUK^YU~c}7mgUmQ7=k8agyWljw}nbdt%QcJz=QLFB`M=a4? zHNuRGSmidPt9_wjGCb~Or9=J3uHS$)& zbc2JxUmaZcEKIKDkgiOV;tq`)4;)pltk0X#q_ASWN<-tJlWUilC|;e#!0+Rv`M`mJ z$A#+zBZIa)vx1AJr2~(Fk5@v2yp}J|gG(iAj#M|E+U|Ei!`|Uo?+1cabdOxXT9DWn7L`4*q27x90z%Wrk@w{qD3z=r^*<$uZfzfvtZIc zjzwXY&2-P*Tj0u_Gr8j=M^wOO)9(UzRt8(Lay}PvwwS?ed&k+HBe&)1*?u$a$kk`qDkQH4oVZ8khwdm>3!sb@H;+aB;0az$_Baw*5;zPb?$Li3^g# zx&93+)=IM0aGkCFkR^V9o>=hBnu7aH1^jaka7ZxR=Q7aEH!xb*>yv(f(zMUPT1~18Ol_|)TU=JM4HdLeUmI(!D`l@&tL|gf z?#BOb#?r`Bee%T&g$zuy54hzNIEWO)#ZPCMX4@ni@K`0FS+~IPYi@(slqTbx!@4F1 zW#=?0h@>ePG;$^|N_HGfuWS_SY4}tr9xORo=W}lHx|T~vcJv%yC^hX}@8&P{4}N%A zG`!fiaQ&PQ!OYz^?i^6eIjkbU%c2vtJmVSj(>dG{!Kyq58KyYNooGMHfGrujyNAy}G#W zvibb|S!=dssPM?J2l$BXxV`(}v7eKio&`Fqa74P#JfyNGOgzUyjh|65VTy)FKmRK( zJryUhB~vsy4*lC2=d8=q@JTajhRp@9j0T<)4Ql_7ui0F}D$l@jWZ?=D)EUkA1u4n{W|*BUsfbTq5nIGDY$ zfv2YO<^l({1v;Ud9d@n$t>N}+j*7Ihj&xtj=5x+(r=$-RWgKLXX<(=b=X`$0tLDJdv~_**4ic+aymW5WZ1}SN z|8Z~s13Hq`tXCd;KKK2=_>Y7Avo2@6-jhR9TpQ#AQ|~aRH1H+-W({DT*EEIa0Vj9D zbFQA<3$h+@Z8*TSqITD5hufPQcsDdi&pA}_dV=(lgL~$0mtOL{>au0@Wu=E7%Wc}- zYQOKap1-lF%`Lsdv;MY2dO@T7foTh+uV}pSQ!i<9?rjo@Xkh)&;4;%m_{TA2gEG*y z00swzJ}~fexJxiJIXM0jl3_BfxhkpBq;aQ7BY{z@;{d~tx_Ox~hDQ_6SwGh?oWJ63 z)%o^Y%zM6kdTthAd*O4p5qo&Uro|4$hvGiUTQ^MUhxWpQG4aBz}faZA{nmd3!% za!cEhiT%WTwjYd^XAZEudC9WmfW|**<6WC;!%WTy-8s?A&@$Joa={J5sX9T@X1NkZ zC7&NIv=FSoDe&$rSqKLk@w$eQK!+}ZfJ&2y5f=c<&OpHzLMEu-#vbY5cX zx1ON()xjxknqKVdOAEb%*sd;6k(t7gbxmiAN>I~`nN=+6mz9BeW+6$}p>nK}7Yd}dtO@iFZd#5}SIxw{g`LFfcZ;oZ9p;tij+RBX?Sp-VCKJ zia|!gmB(J5*pPIHOXT$Qndg-6c z=hIXconR3)_9<9A<(*IEsVTuqiYeNTa+<1>tPcn*?pEK?kkrQ0RN>U4F-5?+M{P<# zQ@if7iX;w3HVd7m0}R4zP8^nV5Hh@@($Tw-eNKx4(6DA-qM2y|!9J4f_n2KOcx=<5@~By<``%`C`JeyZSYKT{@ylz*WfOA*E?oVe z@>joIGG~V3QUQTEitI`+G*r4&OfDXsUiN3BnxM)O17~r~B`db<4w@zSo>R8uMgp6) z$Hl`f+-5r-G%~SNGql^3Oi=7nWvFEG;OXjc>5~yVple(xD9yX^=oFJpCp8KehkG67 z=}tJwt+ZnSlVr+@1QtmXiNu!n(=LfkJW2&V%@X?(S|4wIN1q_NR znQB6^dMA|m#ikiNZGXAIJF)30N3Y-Wmnj9O_=L?28d+u2=D7(3%fvSA`6PBpWKpLU z=krCC9(OzJLSAncJ@|J|o#{3CKi#Hbk-0}5_cIl~oyuo>#6d~;pP$i3RuSKGJCwNn zGfuWN+ssX`E&8c*2Y<*%jfpJpWqKYOSLk2--_N8-^c4nGg znjpW|*KzF$netZ~)$M+6y7o(1W#iSlU2VdX-W+F3`hDw0i6?vl|4RoWI8qpv|F?&G6iy^|BE@yEuhiB zsC0nEal->9@fr7+=1rI|bHf9^T!~BZ9NU&0l6ck`b?VH?6!XSHOZ~D<$w@432F+X( z6xdV`+~tvJlj@00GtOW-$e8n>MenX7M7BI~?u;i$I-9v880tK$-9Y=MO zma*7#9O9e0!Bsaeu{-*~BFQxcY}Q&Ar*owwa5Ne4iF^|1R6NljSbE{Wk1q-waxVWC zFo;x~ea(}6^#0s5mTd=lrm7XJ*GWC*lVRd}xJ2AgXhL8jpLTGn!h%NrHxB&O77XGl zft{;dnz;EA9Jr(o>^b3ctS@NS84kM#O+t4LaKz6zn4a~3S^w^Ve(nd2Do%!Nv2U1U zURt<{_$=ggQ%R1RS#s--$yGMLtqb{jG~6uCJ(OEL^HTl)%d^(TeRW`SnlLpob*a`y zFGnuF4NX2AEpj>uO*$D&!gC8;)vvtj4opcB$Wvf3mB`ffc+to{!Q!ZjM`K68ghyhN zXB;t8b7WxplFICNqLJ%sL6>Dt0;i9}A%T(&qBaj4*rJ@y9C)F5v{mShS@@ltG%=}- zMxCAiyp?Bo9~N20e5Ot_U8>>$drAR&xt!lXjah3E8xU7w1qSa$8o z_XF&!{@yrV^UV7G&kxG0J{`Qul6xpVT4}@nEo-#ue`inrEB0gc|EnhzJ|#48|8WtP zNmwc6f_8ajO~}VBFQg%p=pltft{2&eOoi z%)`L!{o$}sMpE;!E5aw%3BQQz&~a7W>F2{^-OIMcrKx7x0q+32={z|K%wjRi=8JOh z2yQWGX7=%m`S)Rt^2rA*>hDb57@cMu_Ts8xiM_jk&3#6Kpx(z0&WM}-whGQlLXKQc zI*LM<4UV*XZEX3;x{$Xc%sKFKK>Kgk!%BWnRuiv<%w{@OlXDxG z7GD?UE$NsfCo!|6#_KA7ZH$$j)WRwqJ#@_RgYNhRZQ5+JI(V9lGP0_O7}XKnWxRD@mDJsEDxKW6-2@;h@;u#75a; zm!o7SxbA7$V4&G_fX&y*adK|JKjyzT0@%A$lw}?l9M@R8t$0!WeYuDH|Nq5t9h3I> z%N855%Qu6e^!x1lFXlJ?N>5@q_G|h511=w$byXUh)ivIV_$3Nd=!zWDVZjT4|o;0zhB-GsSHC8*Ir}Tii z;K5G51_l;pv!woN4>d-qcW30@8Jj#-NBTl9*MAG(O>#?Mf0H1@mCO0eODd01fbXD)(?gL@2?Dbc1?sw_COOVAbKsoD z;BzgLW8MKavjr@BSeYXZuzfnf$M;ao&r#RoqX@%7_8=aXFAKPNk{QAryk=$SU-R`A zTXom%u>Q1^g;g7T_B=5;;=*f$IM<1+KYyuJYcqKkYrq%8u5UqpPx&LRmkI@h=k(gri97ol$A~= z+ivieDaZ>s)c=Czq8!&nW!p){c8C9IeEgH=z;=UQ=wEOV%aeT7v-zs=pM>-c*;*dV z>|x_#QxsZ~sFkNE;-tdB&&cJZDCzQKCQss*fF2I71AMDE1r9L?Y&cZNa*%h{0|B;1 zo&)Cugcx}W8aTEwFmola$So*I@xI6KD)Gfz9rs$Gf0JR53uKTJWo8XZ=TPf2j_Pa@TES+N$5wHG|Jwp% zqXl!OC2*e76!K~0m1-5#IV$qVKxv`Al4&dNj|2QVjQrd4r*LfK|GAw1<3YJ^#|7+- zp36=AH}Boc7w1)a*Zr0IX#O%^wfD1e*eB+MB1_K(jzbLsvl9Ms>Lg0}JQPz(6gc%d zKWw6)pN3@3B-tyDvNsfE_bn9jTfkoAD7)|A#3Pes<}sxCG%!EA%(6&e>+K0 zQfQKtoAHVT3`)kClN0h39iB?)2q~{ktIwINY`sa~^+Vfx-h#dFEP{jn$$e(aIN&|W z_^+LUBW|cX=2<9Ipq}gDQOfjiX1z@ z<(4Rx#wc3kD6;0DwAE#yKL#-^-hnA0H1#mLU(z^wP6^AWGKc0T+3i~2_b zL*(_vl-cG798Ta#zAGkTtgzTjgP%od0lU=$(=$1SdI$JFEGU_#%&4Nkm~cM4_yB)a z!=pS`p6rI6x&$NRJoY~gqEQccZYS8B$P`$nv}d2O`&THHYKN0)ZI{ zF5(B6o&_gbD(o?9U}xg}f5^OdJ>r9tb{T;BRACHp?S)YsyK#(^8u|I!zATQ&Q7wHAvTA&|eg`?adOi zH0F5*f`W4rxnDi;{KCp?&>*6w=)+@d6}R5NBAGR90b5E)PKm#pdN5nSA$Fc3);kA; zBzOc^5(R!KSRZxN*#BA4|Fg$S_K=z>E4S%C=L-CD=)K$p(?3O%1^6%iTac&n{C(W^ zDXNdZs{a4#{Yi7aOjl9r15TDiZWRUQHZ{&UPq~&Ta87ctOFh7Mq)Eg>QSjT80An#x z7e)~`MiGlJjdcse4mEHmbg&mVu=_Z$&+!zx#Sl~EAW)FN*4Dt8lu)MO@1(5|YJB)# z!at@&o@4j47?^#s?;1EHbWQsg_JD15O5cY?vdRqX-xBye7!0|XnM58i@El-rTf?HW zz$1j;FnL1UaWnS1h9HCj zFBkIXUoq2`h+V5@&M#y-{FpW8uoY%N)^LecuP_QfUr z9(zS&ejgB8^iZbYTz>pd{{8$bHP?Qdc1vic)%3Ta-o=$I+3(|bPG6#NeMaB|_6`UB zV-3s`5(K!eip#9zsmT;slwfE5QRj?<$Q7nHdn5(e40MGY#m^}Sep$fI(I{|)VMEqJ zfgQO*pAboIzQAJ(JsxoD zE&Rfzz~I)v$)_lO>(IJVM=35x{-y@MHy%tm4T7uQOwzx(v}*&KOr=rgCdoR6;-X)| zXOu+j9*Ue=y~A-PZ}fqG%T+I}=u(vYbYbdGb^-6!UB4CO*G0c*;S)$^U`=`uAFD9E zF<7WQSmkl)N=N3syMMQ+Zf6QR!1ly}{}Th}qJ%lu4@TU2C}82h70{SJgUO%IarV_i zTCdm5K3FM|;mE?dkp0fI_5O__9UIvd6a}tz7ALu}6)6}nF&j#~pU0ObeN-^9ZraJ3 zYbJW@gQfM8g{yO7KU-xpBZx8sY95}fiin%rNoM8|sVie~|6e@VYShtm_ zs?>7V7LLdgR<+`j)4yidHn5#D$hECpYP^6os)4y~)sFU{ohOxd-c8_Lq|YmsxboaX zr56WRRP$ZdIlJq&jKJF6FLqovHu`sL_iLLym6KJ@-!^6`nsI-}Zc_$UhPzjvF|<9C z7VTOJI=Oj`!~U}igr_aiF^S}Hb6m&BD6nfL&!ejI84r1U8r(S;1*Ry7@GJ6k9BAL= zq@%M``wmAff3w$xUa8t?Rq3@yw!bJ@^zj<^7n5z@`}7{CY;k8&dcdalP>PR{sp$IQ zO${QK(gltj5b!%F{`rAP2RExkf?T-5sq+DeSq*F%O*yKPtYHaNOcwmFK8x8k>aaP= zY_pMUI&gRQ*XYD|icX$Qy}QnSO|+}=yKn4!Hr&A{oMF%I+5Zk$Pp_P*dO6=G+$cG0 zQeuxJyJk3Z(E{!YC(cC;$(I+(20YAL@{ar)ehRqR@pmL}97_#8UN~P_L67^Fj=O`;v?ufRAIwiR zxOZyFw`o2;1&ci|bQmceVB_fLvwPsFRm^!L!Tm@BPe}^LECqIx1ja2qj0y_^Y7$QJ z1~F$PSY<`C_cYu#&9l0+O{98{)Fnr0r#cP(8NB|HyhbmlTwV|?qA01i>7J(V+4tO^ zo<4qFmv{f-?!E2})%$-NbL@L?`u57NTfhC8`_QwZ=<6Yt9S&_(3f$)u@&X=m88C|6 z&=Rgo)GS#e;T+ta zx2Mh*cJ5i-Gs?R;X1&;c(dSR)-3?C?Clx2DB=_yV&gk(#CYG5|>j3AA1iqLw=3D3Z z9XeV*VYE5wq zT;$}g(z|fV%5Fiad0`TUTMx1GN_n(2vaCGLX}s8|^wpG1;kEJmGE^TvDt}{ z$)!zircLR~d5lwEA7B)!<(sLIc(8$$$AW2Qmq@3`V!JW}n~uPLjE9&{3oC_O*kI|@ z!amQ|UgGb}5*FU&h9LoGD~nq%@|z{U?JPai!nI8`WI_XzQ>(O#TCA4B!Q;x7IXi4_ zTzD3!Al4|ynIx8Sh?#9qpo&A{?&>}sodf1G7dyL6uyYGb$q+v2s}%Dg+wtV&H&a&j z~@H^L`Q7DJ)o^;3_$1!J(6CD;6y36cH=9kgVCxqtrIRS%`B?p*$rcuS=q);1-+Bm&8tCotOzN zBJ)!|b*jxuNotdn`S8L`)2U05D<+A-QYEf=%0v#0-93S)#S%C^xqNQz*my*!c8XF5 zAGgkeW?5sA1&yhE76~)cv~E|bU(Z*@Uu-#^hqD_FpiMtvcy*+o*3z|=R7#&hnIHH-Y%I-Dcv6RVL*0;)9 z*Kc`-Xf0XDU8r3=Uzls>DyFFy7J8V<2b|?FIM8fmz{IZ;;Uwo&*yVrco$w-yR*gjr zronTXM3{tyV&6EmHBDT|v*1Fjq<~?YdW#dkw}7+Ufeo!?Q)0d3LF`medKcryfgo@}!-3P+QC1iJ!U6q`x}568SQQn69<7inm? zrEuUwi-rj!Pi2CO=A8x2&SsAVt!DIOoI0}U+u6*tITz2&HQ|)LQ6glv_{gcVMVq(I zed=#6Y;}N{Nr8cf$APU!z=cn2LgUJ(i3~b7mb$WSSjT&3l2qOU1`(gaGwK{p{0$zV zA`FEb_74v7`DHKscSmmlv)hpmvmbFB=UXhum+|H5`X^r=TJj&+aqMo(T<%YtsZ(Aw z@vk|+D!gez>l;niWfn`FydssJE-0FMZeF&Q>V+nSPYake3=S!!_H;||2_E!d=oXo1 zam4Yt(w+wgpY}DMNLF%jWHWSm)Vq9wi>8`mSCESnJ7_#)-hsXB3)q;FrG)1i6sVa^ zbenYUsZY+M?=uZEiE4Ijh#swoMqzS&K36$y=Rq*(H5I}+shCiXb)SZF6<*>3TPtC#Py zfy`rz;}0Js`c2)S(!q78)AwK74TbV3ZP)9IdhCo%7V_IX)l|8Zo>m<5_+nw3)@Nx( z4z~j=jyz7~?MGNucnZ7JpCkq~eQ=icDC`R0$q`6qVEM?!Hr;W7lX$0ti`+DW4t1S{ zoN3Wbd~E?*Wh;L&u_ZiBUr;^s#)ZyQu{2HXTDHp}o5bG^R~h%DLtNlkNs)<)#Y|mPETNbKN zy_u);^rqm#H9^ai6cv`A(E7}LkZ;dFhYrUd-2BTXutc&Kc6XV%2pn5mq){Ttd8=&a z>C!((G*3+E@YZ-(C2ioy?{lD$F<}8`KLfL(*yL-f0*|UM-Zd`2t|Ik1uRPmytFT7= zjfQIdLcV}r#pMTG=;KUo#AZQZC=R~ zxykvrlmAA6E$+*gUnsP8efg_6+in80%>*S8wFm_j6~`_cg+zgh(-E(N0D^2YA1Z37cQ0G%n z`*GP*=s}T?1LG}2Mx6#G?gpkGO9gT#2nH+=;9ekS)gYKx!00`JXX(?Dg$>?&rU+h9 z;7egCRr=Phl*adb0pGI+&Qa+}4k9isp*b(Fbi}2zI4Q7L3b1@@ulHzV`tpS7rjb>X zQu6s@!hcmX)RZ(d6qs`rI2JBo_A6$TQE2qsASRZ;C~nYgwZU85kfX^!yh()BJ0NKP z7Z!g7w(0`TRt4G1k1L8VGD`?}dtb0gNMKS~!00T%yn(UBVTRgrNA;EBao1wU&!1^4BG z1fJ_pOnxk7K54*vyp-|e0gg(Bu%)K*Ueklbt}#h0{a6Qhrp zzF*$)@<@l%5mvtd78e5+X9X7P1ZMqA=4l!t@=3h=!W-M*wW6a~fn8(fB$o@bC+=X@+sSNb$!uiFY*@f- z%&F%iXtmUsX`OPWh;3l>OkkVez?GtCZ6(xgwSfI=qwV5C#`&8V zxE3%lU6{vwVV=;0R@MgQ&0*Ya0o?5iSW6r@S_Bva9!zW6z~Zxj*-T*iH;(GJk7Jsc zsxOB2ick0SJTZO#QpstJl9re4`5hSJAMiZgz|1GW!0>>9;lm75bw-{BrZ+2RN-8jD zU(nRtz`%5YiTMIkPXd>;OX!bM##!HHHW~0vm4;n5?r=b9RTu(!i{xB^TB$G-uyEbJCuh z|7IsjEps}+oN|geRcnqxWWBKg^NEupd8hnC0-2;399jif)f8A4F5rHBf$!e}zV#>O zMk+KmU*Nf-z~Lv*t22T5h2f&4OY>M87!+SIuqH4#M{-mfaGXwIWEEgwy%18ifZb{W z^Zd*OyjD_@0aDgi7c8xs?y*DiU5W9o!1Qg;`>rkQ^C{p|SzzaC%E;Hipz*<2qd;}p z2acWuPPYvVhuqsHepqwTK|pdNL(c`CivqmYo8zY?aODWdaZePSzrZ=WBI?nEiEDQ* zS$~pG=I0W5Oa4Gkfa+51pHvQ#LUaW=^-@_Oz3pen*V?;)=NX zq>z;=jDi6SZ=UkFIxsRVuyem`AGfKB)UQ(Q@H z&JLEG35=Qy2lj7al%Bvoe}kZ8f@bONC6bkVWv6G!NFMx>uKpcP4Y~Ow8IjU3-q@f%Rq%bKDfzTLjpayx#D#fh*mBE4+Z! zNT{jof_M7@mdXafe-*_XtO9mwvyTLJGkD4{nqOd>S=-tKR%gWXCx&y%Q)=eP8SBoXWZ!qS~VO2jP?g4UO8?% zGl6;gb4`wEwjo;?J}+fxT_e!*M|0c36S@Z&O{_SiQ#3C-)IE8A;Hi7vnwuw|Fz`o+ zpAz&*sT^!-;+b=(k|p&Nv#SI9 zWCo7)##}#|xY7z(y&iFGG2l`@u~t-!!TP#e&xRF-x99ONFkCpg`)M~b6ALHfDn`o$ zcK!v-Yn<5gjQfhkV(p^UN}m0zVzZi_x#k!bm+1PB#~L&lb+s4-7-U)lj_-|NuS{Ss zZLt4n$ylc_VRHm$Z;|xWmHjslGySq))C|bn^YBF9940AS#@q*Pp^MujH~3yzz#9~P zDKNZFrS~M?o=bCMIzBX5bTpSnPS<&+m7}qP{q3)VMGoxo4D9g&r6%N zz_O#~%HEZe)Orr>dpXHPfxXT^Q*8pvCT6a=4V?2XaPGfwb^Zf=c30ks1tu1!n4}pz zSr5$XJGFbyZ#9kIEIuDtVg!1NEX`FmFoKTl{cudpTkYc1BiS(vFLQgYe6?W4#FpIS ztpzfS%{s>g4#Zt|U>9ZJ%xf_}Gr@H3zeO>*8yH`lWPY;s_~%q<4TWOi3wt}QF5Xov zW|*_TmZ2`xj6?SVlfN17g@jor1tu`=OjLX5`+Ne=-q=ghWtW&1)jfK3X|WIg>|g8_ zxjW(xT#m9j6m?+7ww^0n->|sVUU7T7)aAmJTYr~Mv{+_xhiA!!uD}VL#zD&K8kEx# zrUgzATv%YjHGz?{z=TPG(PwLj(klj@3HKQd7%F;2jXy}ST?pxCh`yA_u5%%F@h`Q9 zn;$$VmK0rer0(r?uRG^H8Qaft?A`tG*i6Q)b&DA16f+ALFuh816q&-z^nu}rDg*Zh zLurQuE(XSTx_+}N`&`)?bJ|(_oJ9*5 zI2<-L)p@?Rl)7|=`B*4d+0`r_tMhHXdv_l{m$JyNM6F1$z)QQny7thGjTenwLR%*! z@J$V1XnDZ4B#mon1N*F%j{+1pCOR0(U6D1}F6*vZe0TL1{p*iC>=}C)cqgu5W|QR7 z-Qc~cPX6+~uQC2_B^QR@>f?JP!uMhUdtJfPiV06+1P(deeX8;E-DTZpX?ds9&+WMV zc4=G|OVxXpDh1B74lK$SY(&3u&JPGoQ(#p~h}!PU%ku2a4iQerKd&Y(U_bjWfOEoy zK8_6x#t+y_V6D5r?%%)|pb)s+;ETxOnKE|gFN&^y&dxdx9tErKoAzaE>w^E#jB-T8l#i9A)N|8LlRORs-g|KMPo z_MvF6LwWz7G8|mmA=VhxCG)4~!UIPoS1BpYq!!3Y&0xzdiG$!CUv@+1|vWpCcB{Nmw1VeT@&uZ5qF^z!iv%Xw5h zc<}GwYVm+#=GS0{gTXbF*`ODoa}19{5YL|o^{Eb%M+d3PnR@a z?bI=eD5w%_ED{O}zY(xU@RQ#^ho-ddo4Q$N&Fbz{aEl-Rp7$$a)j|cm?&(czs*{gQ zvtkQKXjITzsnQ=Lw^C!0rP+qVyg`dNRfUe;$l%i2ysk=1d&7>dQw$rpIv%WEyP|0U z)8okhUEWTu@DSKI`mB*;SS-_1=GG!%C*OzWVMw&mJ7&wp<~dq42T8;fq3>NYEAqH`$^W97-(@4suw^6}m~~nk`L}$~8Nb zw%C8=Mzt%!iz+y~6y{BM*cq&JF>(E5m6;BW>?IqP%uMlnc}ZkO(}sl3fJrBtS$Rx2 z&S))P7WJ)>b;YVvjY7Mt7Pw>|zr?kAn_0^NLkabcgteRj46B>^=1N?= z(fI_=jf5k;|9w1{_5D)}Jm7QU#G*Sr_jQynzuCxn$3x9*QrXi@l4%MT7x&wV%&bUq zmDp)<)>zYUvQXdkIhW0E=P~O^Ci~n=?tk`&rNpXvT`@=C%=BqLQ`^N3DLr#lHBGwY zx%0KMz{E+CrdLwDvX)6^dRTpB(kRn%71B`gRGXy!#=v#27f(e3gP79CuOh-qimY30 zN(7!vQPq=hU|?yP@*pz(@v)aTcPwvWSR=&!!0XofEnQrzd6x>wtUA7}Yh%-KlMY5^ z$yWkPB+ZQ;IJ95cb|#^bwIsojUDk}FiS?M0am8j|(VaW~RhUe)6rO2SFYM<(ZTn@(i?S^X_EjlXDI~EgZTN0^ zu`RDHBv7eu!WRxj1tBLk*#$pTW_`DknkHzi@KA&Q?DNzmLN+M^M}&etTyPYYQ{+@< zXvqK5eMG1tbV=Wh)so8^4zcDfYEWC7z$9_NflY5i6VC<(WXf0ILSTW9rGJ9?jf1>-0!%Y6O<86%>3jZ&PZXY2W{OCY;6cPb0hJ1IH&G zPQ8vg37mNyEFwGDPObKFVBivP2#bB$uIuuEF)8o>ubdlq-PuUvez{usm zz;NOr`}tYSD)RzVf|symX85+rvK25o88l1XO*qcBDYe<%xO?Bw>DaE~u1p;_|I0v03LgY0es_l5qQFlRH{s_Eyvq4{rH1IKUO zCxST&3DN=m9CZ&4vY9F9$jx*Ts65duEn(QssKPyyX+swy&jLoZHEGOD46Av59Z+@@ za1v;m;L_PD)amHcG_ADYfX2os2Dvwhd~ustlwWN)EAjp6hIUs6)}8=g7K;Px-QKNc zSZCSjzGeZ_)sow{_i;5aTQo2T9W>y4@{7fD%A?RF3a%=CiBr-T9x*T zEXymYg}bKbHA~%@aM%36#NKy>_)%>A;-eEH|z_v~3Njr;-s4z!tGI=~)d&=k45 zVVT~Emp=UyE`Pn3u-84|km`(vM(zRy!{oBVaS?Y;ZacZ*P^|F*2AvI!OSqa?*mgLC zm^^4dl%o*KogjUCiR~?e8%cZyXI3*B1Tb+sEL5Mi!ZR*8=0>*EwG(^}j0v?p5gS}K z3psT#nsv)em*jcCAU|OZcSZ>#kCXzFibkt$6~n0jy@TOm1_#A>7&!tZoOmowFmhH* z;BidJV-)kau6tajt28u)kN-gK)G&dTg{@737bD`cJ}qDryWqfP!cfI4cK3yt$uD+Y z2B(V)vja`HDDdhA9A0X=_N8C>t;^L0>z&IkFn`-t&SE$rzgO$QD~>6#|Fj==9f;9j zXyVu*#UlTMfg{G?yTBRQ>&5>Yc)JQZEjbD~y(}&XS#57oEqD;g_8^+2#ehXJ;{Zz{ zNAvS30&ET%jO@iboMq(#*gait#QG;V@s})UQR|80j5Rnar*wdU;YI>qG>5Z7g~M)3 zPA6v81qpjsdcIfs{o{BYt4`QzO%ADm%Zxl44y;L+n5{&Z9>yv%DOLqE>9@!TEqTzu zIiZ0?Rpn!-w?MPR35Pc05*eXZ2Ij0^8|)5gKV~W%5;|2rY z4@bTY4R1C#lu9>BPGI(^Xwf^+oVKIYe zW;8iyv_^NdIKOD}ywJ?fz`&cqR$9;&eWUHkr%7fUOdDRTzGJMjU29XLw$a}Y&dSDX zln->-voJ6guzBuij@rN~-NCRifywDZll24^n*$A&hZ}7@T7ncW zontIHjq)EFm;7dA{nO~?y=zJJF18W}&Kw8MIqP2iVqSBbPf=kBKhHsdH4MBR2c;%9 za7^ev%h0`tv%5^Bo4@d&P*bTCixbOm;SC@m%5Ld4XB!N3;8b zCfyCJ(FQF#0Zlp#EC#b%if*tSQf6`JU`gN5s#;(nq|lPNgDtD0CHzn`H$%6(0(+1~ zOArH#4y4TAE6B{Io z7&|yx+&h>YELvj@u*P0E8#94FQ-i%~2aC`Lri4e09Xu>L7L2M4EG{Qnoi?y~C^X6` zv^X%ZEWW_RW5FQF!4h3?$RVSNeRdbqfkr2RX2TQBatYFm7i0vQ8*Jxz?*C=Wvf$*Q zLnjNWd>y`cuHj(#5Zt7{=D-WBF*90?m$t?n@GVGa_taq2<6u#`(V(`&!!UqJ z)21cpMzi+;=84KIwh>Kg8ZD{~+m>{(7)b1W=^`d`fjR2{Tc!bvszM__7Xxbp!*R!c8cl3u#Sqg!GIqyFyJD30djiH+q1nZvRZpXVE4%UH2N{_MjFLMVZdtT4DqQ3i*es?ofnQ2or^&VO zg-DSH+gDB2;0w*735~_NZcHg#O3wz$YaDQAK9IKL3_HuElRj=|jx^gpWKxn~QasTp zGlOwqbfd%y25|}Nqn?5yAqR!eTxvTS#H2A@zm>_BgY}~_YwQiyyH{?+TxiYuz?N>% z_;3z0PelW-1|v@plR*TNtO0`nGsDI%mgx_g)E+R|UA%6?(XyO}Wgd5n!;9lCFWjA0 zwA?(oD(lf|mI{A1hbU$5hC^Am?9O;rO=e(_P!(ChY|p@AxL}V#!XcXiCfgfKHmohS zC%7FGW;1g%2pwqr$Jo$$hI7FOYZko-ci9dWHwJUw0P~nsDMw-Uj0|RbZCSI1x%VQQ z;#HKTY8xe|UR=QKaG1wOvh}*m+Q!9;uP@2As^Vo*IN-BJddf7ZC2DgSFNho$PEu?Q zb+0%RboENV#Ac(74Gbb3jOBvCMiMMNXCvYdMD$)|&Hlmmg{x({bVJJ84T?9Kcr_YU zM=46};CHKFQ8h7is$g+yU@>iGaZG4he~R&gw!g!UW>pC$`pjru?;e>P05iuh(TVOfJYoZ0vT_vOT!`g!CE<-@g@SmD6H%OsCs0 zwA`>@jrC|R`M{cegH`h$YrzIKoiaAV7mYRt88!qpFbeSGHZ9v8v`lou>DyIYVoqY~ z3f0&-7#KG&sIj#;-e}4YZeZ!?IuaDcmY`-|7WHYbd77760Bh*a7+*#OMwJusLLCe| z&YORn4VIN?Q2fB0A$nRe;pW^EGO|&Pyb~C&6f%gdV5y91;8kekdDWcO(bjJCz$}5e z{!k~&h6N`$C;#_oQ&`a$^np9+pU)j>gJW*oyB6%ecwow%CB-JJK4;`?cX>GPlHbvy zoXe+I!Bls*QQ^n+23|+;hM2^+xiB~)e*YD6!(rT>%!gW1*~^{*em?n zb2hL!KWMbx!OT^lmOhDrM_8RJYMIjRRoe=etvIoYspB!@j*tyI8a^m9JG@}(YF@@r zz_3H$>hj!M_I?a$lgbeugsXyl6Vl$>jLMhj&MV zvLS=x3YOI|47`yHnvst1^H8M`f z|1&gePfa1-Qc4^Bhvu~YoOS(4oytA2((liF99nH23 zElx9|(J@Ib4sO30H@1Z{muyM*ame@*n3mV*6wvo< z2J?l-*Xz$dD`2q}64_laQ80I9&t`|K5|^`;#QvQx+T0fWqb2Z13-|F4{t_)l9xX-! zt$7DnKiy$IDW}L(uuS*w>c`eeCyQ}-qF$Cl?Fd8s~FtRu@u!volQ}8K-rJ>5l zS+@(#pVq+Bz$qKhenp|3egRe~o5t{pN7FaV=8#>B2CP?P`D8-u`R-y>d%~dl_d) zEfJi+#W`PN6ucu#0yeZlP5(DKA-Mli#6C8O(2 zj5Bty5*3K%Whhhf~ zK0RSoC(4v_j!~*2NJExUBw%aNlM=sj9*I|CMSP_j3>YLcn1d#;@x*_1{J~;;;X|Z; zv!g_7@P;&Riw|*;>TdI=Zz-I6*E9jO}%B0}byF2?oZs~r!?%?bV6%0Zd%xe-^Y*#SbRJ1s( zU};~uc5a-SPSi5i2*wSSr-Z9d$#t@PeWIJuba3+ohVv1Pg2^nh9P835G@?7&3KD8l z1KLs}*t}-^=&EP#<@(Z-Us=3p%hJgv-LL*i_PNR5xMO9YFOwE@zNd2MvV*hx5BiBG zvE~=(tmRo7-x$89mHRwPk)hxL zgCi4*kcx_d#==Hdc2OCD4u>V&f|C3!HY)|Yw3(3teOEiYqS!bns|hN&G4G06Jla6?qew2{!Yg#_0*+Rv2AN6 zE_SWIcfd3Gd!CeX$2Rffi;1fC|CqSs zLJlx2bYYY74=yoabankLdSs$2=NvW>W;2&VJ}NykKAm9UV-f5WUK9J^0kdF(I)gVC z$ASZH3SJ72SQuhV9T@^vY8+tV{NtX$ETYoj#H9IW`T?#0)`K14-Z2Nx7(ZU{nQx-& zzTz+oPp!>9hEooFJswkf;G&y!=7vOZ>8jL~ZGy4?Qe6Zp9u%@C#}ovzq&U5q;GAl7 zr@)yA}#jT#+DVArXTHHp>?meKa=BWNhV=DY&qLudGJtsKWAq z)&>?7a^4Uh0;N=2LtST2AjF?y$Sw1kxEjwZ0Br9d} zZ$i6fWz2>az8;%HhnN{O7C5+bnkon~`F&CJkk#f0Jg!`Hq~M}IyQIehH??blodRn& zUht6>KB6#Tz0Q156qYri|T^Itm-Q1%GvhW=3o@ILM@H zu-(D<;+)5RHt&xFZ=C3|Xz7xuBNmd=V`@Gebdy%uT=?l^`OB^BGmFnWJd!-&;{;~` zvoitAb7rP~b4=H$VQZWx>ch~;;v$?Ycri5civ!z@z{xKZ0$mm|FSc@!I-0XEpxImZ zT+1gGk%e=Nw#j<8B%RWXXIw3!SQ#++o2LJZ#BQ->yMzPGWm7&KU|^FdaOCq$EjTE% z?8?UmCgFbeQ0(cA00c!7U7uwNnHh|2+4iL2zopMJc&68$KT3D&i1w zx!l8}cwnvQ2A3;L95>{yJahOPxw~F(+Y8}vNfmn(^t3rFguK zzs;uz{n}&h`!zJcB+@W5~;Ls zkz1tD5!|v!WMYJqieF-v$Bss+lM=Uu<|TAlvnl>eZ>EcRp}zi31Q zbJzox*C`FG0g{3NuR;y_R8$n}e<=lMG&Id;F<`E@>J9B`7nNF@z$EO!_}M6+Gh;@x z+}90qD$1NEyCZdy(*xTk%}C+&s__@BYY>wEWng!#2Xc3pe>uBkh z9?ttx6}n@dM%xSCatjIwau)MvGTC&IC2YeU?g%BRcPox7zgOg+`zmD8q$6P^m8w>^$nOLot zoDx~uz~oahf!Q(Q5U22kMg@tFExcb8h59|jHyb3)|LwrSz~qsrG3g)^F08S|v5%m0x*v)jGxE%o8s+ zeVK6i>_pWWg_o=DEZ%+ZZ(e!%tKe6Gmt-AfUj-cg*t0FCh%fE|lkg4!dpVxOy06(w zd6On=p4)rCxa?L|_-3vHcA~r!_$nM2O%5<=^yp^E6uf3JYG``o=*6}C0;{RV#UB4e z1*V#42DSxFjE$TL3=$^}3a~F=VC8=|yOKdoS!rUA;-8i{@l=U+gDwV!k|!qijU0>| zA9gS<*}!}9#9^jC0==7=`U}q`apa1AU`~7SfU#e|Mb#(K!&~Js=ShL%CPx%o=JRAM zkz)++|F06n)cw^}U+78wqa02etr@hYB)Bo1TUyR0=0FvV2xB%V|kE zTrI)K7@?%kY_Z@7f5!wSE&+Y^5Ed z6A{OUnoR;nm_8_Uh^+i^qsQ4}fy1te3I@&!XJ;I!8&tFfx<}07`cu&cM=Hy4} zETL{!ji#^6(s6&#^snx~#s62eScCjdUq~rDusG58A8S~`oULw*O?i8NvYI{6Fz$Jq zDam$(L+Js7PqfN$(T5izWdj&EG-4T<1->nB&*hX--mP>djvIw|SQTwN&Kcmqr*e_UW5WWz3JqS@nHSh(H>^_%yf(MdK}e*(u4L(z zr^0j8b51y=F)SFR77CixdyZ{pWq<+vcnv}L);MVmWP2j(?3s#R<_XS|`2C!&ElLiqOOoR1wg84DQr z8uosk%GcPPwB_0{HU~zY4Gl@Whja`pgt)6#URI>k9w53k^IC za~8GUIZ=CnqvycuiF49jqF5PxmpDx7d& zc=F~a1)(RRm*(zte%hJ9`Kg(6pAwhdoUacUk1h1%xp)2e z!A{XtJ9 zBxKSk|H6SS;R<7E;vS6##tet8KO5y(oaA0KDtuv-IMHy`)WJ+vWrJqR8_gVD3Y@S0 zG28!H;{R_+z`mqsx-sR2_sR>!f6wU@;5aCFXFAUn2c9bn>R$59*&(ET^8nu$hS?!* z+IJZ^-zf0ETM$;uFfW8b!tH<%L$HR*DfK6fA|AZgVwBD19nuwP5;6&nKH9y0!!ZV( z1ChE8yc|us2F@#|bP23=VUTcOywk;)u~1-&+!>35uSMowDxJ%G#9Q8@iSGjg|Bmx~ zCm6&zoa7!jh>IL_<~!&q*Yfy=;)Mt&2UnR;%U3! z9Zzowr_X_2B_B5lCDVJ+S7%PmobLB2#V%!@HrLY9xaHbHO9Pcs%V+Y-h%^dvH1hsw z;Mo-v`tMjn>KO*@vkv@c{P{mH2!3&>IhJ1ij)Ct@L)AS8&W5ECVGa>)4QwA8Bv^!F z#WZ5$W+;Ae5MFc2<>yn@Ia4F=_Et?fB>SR)Q=^en=b)^|A=w$BED{G8IT+ruCiDDJ z;+%4MUCoPRz6EP}4lwFA2-_SIyV0b=!o>gJ0M7&mMGI%k3C;?49E~jwdNfF9HqQ1q zaZvecFbuWHjfn z&6ArPe(3>G&+R@Ow%emwCiL`~4_J-aF63j`fSF ztXibEs^*ge{}uIrd^gVUOmX08dCI2YxKv<8qnL)sJSJhCMmd&)tT7D?Ck_aRJYmRj z5O{Hb&EcAgz#-i!hZI?u6y7+h++k$%aO$^e*8RcA7jl5FEtThmf^%Pkz>9-&EshE| zoXn;)F|sr;TQI5=IGTDiYkpx!c4JV9VYXh;tfs0t&2JU0nse%WPZ;=g8YL2x z&*Z&ojB*yyb5yW7=<@NOgTe&X4UUfC0UiRD%~mPSW>c8;SPm)nIC4&4vYz5>xq(^x zi9>j7XjtX}z6lJ%cbcqnoK3&H;Je^0Xu~Av!?f&81Eb3UMiD3D9B0dx)tYAxPAEF3 z_;2D5PX~w6Soady0<#vkxhhh&4_LmPoED}~+Ni1I8~#|dr_^;uwE?r$g=XcInzOgP zJa|a6EN;2~tOxQ2Ix;I7rDPZdzGU*=VBqywGyBTVllF79fS{&CFoo1B(}#;3bZ5-;8e>-6*eVAOnZP{D;sL`Q$|4M!CTCu5Ih z;}B-01V_a?jT~{_cTf!I5>~V@rBzO?UM|z+}vt1 z8D(1z${x^Dh;dRFmW7I{?W+!fPrtq0m)k*{#6?}inA~oKVd9*#O~Z9;ug`8Q!D6hFgMF1 zbFby~d3U%E1&BY4Gbmcd(03w*?Z?6CJDF`9^!z$MaD*@CJpPjNuqIb@-obw;?)z67B;Z0 zDKhdoZ0Wt0zy=^q$@;n)B!rr}I|Mb8o5_9@~mj&k#9V5tzY2CdEqmR zvR?|Un4A=CJ}W9PDRnqScQa`!IK^x>jh1Fok~tJ*{n>bVsiJ_BxQP6c2w$NW4njV? zd=lG)-W*{6bC%VHfx9HkZv82REsklX8>J&0&P_bP^MXN)|G{0C0~{@E%WmHLoOp}* z#(^_32Nv)+@V2gKpTNK-vL$W31J9G`yw6`q{5*dD)!Th9`s$X*vltw3dlAn(`ATMw zh}6|B^Jl0W{=q$6Z=Lr%qZw%pY%d%G`wv^pX|`-I(7DbW{LkF<>AO!MuRrO2J{%hV zRsZ;hdAAM-wrrj0bNn@1lZctIl8;mEWG2lkPIcFr>gPArJvOiNKNNGhwAOvItjmSO zZetOhA5GR5uH0ypD`3otKEdWOS=K{1q4wwo&O^cx4);9*_%d#@CNOY6$!Pk?wf3%@ zTVfRFg?kJU2N;j+;CL~Qx#0!F1P1;y4QxNUF7!6AeV8btDem>z+g(ue@r{QC!W_8^ zj|I$Vd-o+b?Uu=5pPic0m*!~Z`7ArYl~c*~g3-7|a#!kh%OlKBmaQ!B{_r$2j`P0? zm-APGvlP}14pY<=C%G!4WG+(pB&QV*tLLv*Fxr9|LT96H*i~M z${do-Y2ZBaS;6Omg|XB3mkn|jjB)~vtvwYit`nkPI&g2`OKUvXEXTkW(JRYxh_$5A z*r(apq*z>})q`UrLWwjp9`^-Ngj)Er|-QG9KEiqMGvvvOW@`ckk>fJeX=ylWYZSN1? zKeXU>(*pKgx>pXVFdX{uHPl4cqQ8Ync}F8352J9&<2`#ef8agmqU(@s?ki_18bRL>g6E(N$04>Gj%#i+nOHCjUierN>&n7$K;TZJyos~vhm9&%m;`e? zSl%2|`6uI~IfaR@LgMXB2i6vLx7hg}B8=i!8f6U}Wd#nNT*Kg*xJc+KUxCqdHTRdy zv)Vkh%!?P>KH8_^u*Bm+l2aRxaL|$y2b2zXwO0BB&MwQnv!(0Nk>1M@=VbO&e0gzp zb@=&vwv*L9KfS!YNY%dfPr|VlNq1gf)2s&~noQ5T#jUEp#b|}R5Lo27#N+0wQtt4} z%dV7oPG+0BDr9x|(ub#3iC))r@-oY5Fv^%P+p$t&ZjuOR7oX$Ac`*_n6r5Vv`FUb? z6nuAVW#>zJ!1+a`f3Brt$r(WtmiGC!_UeTXts9G4Ifb+YHg4ga6>z*|ON>UqgCnXA z)eMz08yDuaNUQu4vk@o|cVc9>o?&`ASG(LKy+2GtK(D5WnO#W4!Xx3%j+V#VeeUrJ zj8>(uuc@od*w9e)y_thwN+)81Vz;RFnawvC1aEEae5?_-Tdi-=u|k=%7k^6>%{wvq zdq~nt)tL#mo-dny??k7%RUKE>3a1{WrPG$q*yTDstz@Cws;m_o-DYLk{Sk2C-?&T5 zImRPWNyl6wvv7jhnIdI=jRhS`L>zA^B%1BwsW`y2=U>qSJ;^TuE$!+VJW3ok5+@%r z@|a9e+@Pdnu)uWpt0D(B`y-VpErJFc4zzG^NgQaiRf~DW$M#%8vB_2}`5~w6hk^tK zhKhtX0qb2aHg{X+7^brQV-%@i*ySYd(YKRjPW3wrPL&FVOYW+B-|YUlYw?=}B1Uif zf?IjK8d(D`MxUROsZ+#n*ZXQS|J=fJlE-DrAFZ5fKk3~{_N5HblV0{(-189NvykBN zTH(^>)H22MQi_-OTC*<~W@W8ec}*&3#nx%Ja#v107PT^u@iCLE#f62sU(F(%*k#QO zrXQ31v8j9g?kb%xjFPV!mhjttda;<_Ua2yrgO!J4!2=GNjI&<1tWNw(QV3X`J z&fED+t<@yRLv?DxZa-sQ{$O9jvkm`(n|ZYI6~9I_bV&US>+Qd~Hu`FZO!1kYFO(;` z{pt^&)+D;^pm&p#*hG%msjpULm2BJ=m9=rB*zL@p=dRu2v)yx)oBPnIhGt%i6AtUS z7o^!V?_9Non|W{5lhZqRQcf&3uzK@i51;vyOAEVoJ6YS7!+fpXBku$f4asMKOI=kQ(O^7N=R2EmO8CB_0t{ z(nx+6xUiiq{_8uv1#2F9Pj~yD>FcQ5-}?Pxs=xG)6ALr`{XDtxyxrR4m0#x+AKE5g z)x#3LJF0WTy@F2rn5M>)RnnU;JrP;^BFUw0_qBgpH%)ZRNtw`m?L>n420^FnCCO1& z4;+;<5D;t2IZz;Z;{dDJheJn?cI#E&QLvWzlE5Z)!GTrJfLYe&AfNsjX3Y`>ag942GFKM3 zSuSa8b}(rcta`yL#Cnm(cE!8&v5U5(v zqE^x!^T0{`T!&Z3yJ_16>I|gszF;%#&Gg}u6K;_-;EeV-BzfV6lY&pDBDn@fWmFUz+$|Yc zEo&G~^9UT?>*LVz<id(*tNgjK&BEvBm4Ywtr5JkJ-Pt@2RKH1VQ#dlET3W?7EPIsmv zG`hPi77|i*Icy-T$?@l?o@#=V;+#hAw3|&L77`8&UJe}5HcbMJGaPx^1jJMu7ILcX zaF%0wc$AyPQK(#@k!Qz)c83=Y0w)EIX!=Yv$}&ji+*s(qI^ppRg&T?j{R}M%EQ#%J zo;?(-Jg`ol<)A>j&x46l8FLTwB?Z54@KQ80J;7xbFpt;9QRo80Q}vIt}1 z$RZXQkA+n+J1mZ77H`n$V=#LzDt(*7bB>a#17n9lyQ0NG{{9YzC0d97?b@AjfNes7 zo2r~ir_+mK1_^^{a(^6JRBaCNUGljuYv#Zto_2t(DWN5f&wzn>$^rh9Cz|=#3>s8# z9Asnu&^^J(n$gBYS<1sk$#}uV9%VCUrHeD#{m|wc~Q8%#*HQ5G-h5{--v> zkIhHHXZKHJPA6}z+T9!2r_X$tw`B3UnCHi?ALPr)t8l(?=fQ>! z!Hqdt-$L~Ry0*nG;mxi5crs%tD zBlDlhk6xg{Kxc)fA zE?UF1Cw;S)D^mmm>zk)r^%ji_8j|{JxPWsMC0XhuU6KS241Dc6-9c_b#yO(Qnx#jD8-cX;*g@$mw-Qq z9!mZ(;6LjB$1m}3$w|#kNzQgEGd3kQ-|AY|^Y-w*gRhk~uKTur-LZE;qi_Iyk3e@lO9U=Kjc(m6nK*$#G=U1xj4V=PeML#?Yyf$oVaak%>`Yi=(VxBB!6C5R>n+6s|kxyqT-A zCW$FsIK{kg#WMCzuisnrFaJAr&`yU*YMJ8!CLV#m8c&EVr#ffdz>aRHqrMVYB7IR=LInWi=Zhh={&8-DO3mU#DHL@Q% z$@ijAO2LucOHoKheHo9_vVR*D(()AMswtdzX1gykFzdezi?6 z&hRTwe<&IBS#nCF(1UBRU^npUP9+`_SCF=gp-?nSBdo z8rI6x8LL|R24exj?Bvg%VD$A&LW>iHV)1U4jwU1a>Er1L_XUvB{?$3c-l4$KS+ zxAh+IGbysCIk2TEu(vd@mnE>9wQ+oNklAk}>6d7)>UiEOk;AQlPr;Et?Ev?$Vr~g{ zwtGo_OpHQ*9N3@f2#6&xCY;I8IJ`$rf%(}1kp&6@9*qL?67*Oc*qh#qbfp^x`|aJc zqBP>2jo1MjmLdgN-r`5u3#~5yI`?pL(UfeP+!ubA`DZGcPIvTZFOlqe=rQq28%7-BM;xs3C$(02=<@`qdrs7^ zGFldq6=&wHGcSQt?E|M+0_UYgy#JgS#Tb%5H8^}fRQ`jLIqd;U)dB7)4FY}-1eT_nW4<)QFMX%|bvFYGphn^cd@`PG@UvDyzJ+;u{_&f7nVauyq zz2k+X3lw=0+^vE;Ztj^-$T#6u%@e&%lV`s>WMOiE@5n`we>x8f%@PE{K6IrhGo&2g zlR3;ktAVqkfxU{MOF(f;6@!4Eo4|od0xJ&wFj>Ij;lSdyfZgo@gIB^67Du)=BZ0hw zA_@xyR1OOHEEJlPk}&)6H;aJa1qVbz1a^A0RjND`@nB?~*deI0GL!qzvO@)vB${|s zIo-1TY?~HDlx}(chWU@f+siC(Bo8@C>KK`x+~(0cMPkcV!5iBIWgh;u`*f&FbLJ7z zb!DH0Zkt+8yd1lJiu5ErlsfzkorBnNIi zM^2w|&KC*-3DZ(p68YCGNf0`wf99=zkf?DNv!$2=`!olhI}Qsg8hGb8u-`erJHwpg zmIFJ}rl(R$MyXjwKVQ{m=43$N* z8!i={a?#f)G1F7v_hZy$QOrm_&;Nx%Ac|jT$Ab1}p8PonB|RKD84?YqZ+G)qD3GGS zRM#C)?Z97_P|lUeufXV8a)5zlp+K54lbB|^SSC{h`w6|pbI)ZnIFt!+DC+3^`_9DI zC}6|aw!qD3p_^;umaOZ2>ti^Mu6VX^#=>lqM|_+)S9FZ(mYUYRji`G$UE+;-y|^OZ zIzIDW-TzCzyxz0rFxO4*;&am9wt0)+%nuBToBCOLg6~b1g3~wF#p^TlcnK!#Jh@ZM zpV3A_f7y9nB}V}#M}C`AB7TniZxa0fFbM5xFilJ2S7Q`pNGw%tWN+bMPP)L*^qlR< z%GB=**sB^im>BuHQc4UHxZ4<*xf+-wk2A(3Og!}5DC~j7HD5-f1je>koIZ*ocN|hb zHAwkg@MG@HOnaqp=5@IA=brkhtOmyHIuC2Uvj6eg%&#w0cl=x5({jld(?xHX)jwv4 z_-ElU$)0Cj6R)a#WWn95-nPN9J45m%cP*T_xv`t~rocntg((bwy$aW_v=UcmH2<+g z#(_bBVcD^ldMplXcN#JlI&dp5obtIrAjpFMSwfipH10hK{6Aa-R1!JD7U+2#U~};= zGe}_fa7gxHVE19TU(vu)m(8rRfF~=%*;j=BQipAb56S z<*x(l*&e7HmoV~s8gVDZv^B7nG^|@K&h>|jH_~C=M;~Psqd!k}C{0b2{IPSxWApxr ziW`6K=)bRUU1jGDjf?q`6$@Ff=kFD1RJGd7`aM?G>V}G>v>GG3L!gy@LeYxJg(4dm zWT%U)?fs`eC)p`k*+XvuuhYTZX(~c@6c&AH5ZJ&S@|R)Hp9KOe3$Nz5B#1aLCp{37 zTPScSfvuuJKy4v=!2xaoNB*h?mW~^I3x4srxiimu-7m3#NrNG1+pll)xL31lGON6m zC{Ci$&Q2bg{39M77DURMf_U-KOpFhfW^eer4vOhCrU~k5v^P#`O|!-;`OwMd!M$p zpV(69RK>Wsg?)`V|2zlIHP;!cTs&Hi1U!1m$+EDj`#8T};(GyRyIckKrUdr7K#n#C ziB1I$j|L+*2G-nv3k9kch|EcpKD2UAxUH^z6m*2xymV+XC2Tu3AE;M2A%FAMU!z4R%Mxk^*<1GbNWyKX~Dw5L{D4sd2 zpt>ruUvIy}`0CU>8LPNS&pq7nZR0w7a#q=KjA-`~UxGy(O^T_W_fVqNH9z zmU1k+%0aOnJ^s%Z`1SAR$lE-58ZK!iE9vp?vUBZ%MpnKPS7R=3mASI;s_akRuVQih z7ID5d&x_v)^}iH5uxN|!%}F!YeqvhNuIA1*<)esGqYJ+xzuNJN&1WliXF9PyU}8yd z5>;U6lVyBUdrWBAacTc)$_~u?_KFECWan!<^L_!#I|adS4D22c#FBQDs3l(8b=Oy+ z!Sulziv>59{O6eWgNc1x^1cX;^`#Ahx2oraDQt`MD&=#UEmo(W(r|wzBbUVkrUeaB z^AvQ_ME0pPO7`eQ*~XbZzAiECphSTDzpodsJ+b|JS+i!N`(NHG2R45Viw%vPC?6?% zLh)JU|4pto6ZL;fUvd1AJok;&ntbj`!NP@{Gjw~GYp!P(SizvJ(^e#Kq>6dc0t2%K z=Kk1?YzvmCmYeT33bKG+KJBM>m@*RC43W z6VnuTcZuFqTH7sl^ye?OJ}J|CdzNZHe|!68(bFscDvNJ#dlR|!>5h%M<=fWW+?}!i z<<;(OYvS`ucTUvay*6?NuY_3+$E%{ltUSH`hxRDkJjBZ9&DSc#;(A=cURud5Cg8xq zh7K+{je-Xb2boy6WHWmxOjK;);NzNdVB_+Dg)UKUG8G>x*nMwy9WMmPF+Eb!xJ%v-T*PaxhdfK5=K&2{$XLyyPwH zv*8eHo`Fz{g4L6WXEc}Dd<$-o_7Lg~JY}=EC)nR~s$$47fzOJek+D}p!wW0EPL0Sq zz~CI2v$plRc|{@f^(cdRTg4JBk9jQayT4=U^@RGVSAEScz4>=hB0$}z4*c#%hzpUW%#}6X%M{zew~pjui>u(A%!D@u12ji{M5U!3hsom9~mBaVXeI1Te6y ztkpco!k6;pK<$dGhQRE#C$HRR{+p^G93UC8;S#gXh5|-z=?5&VBK9{=9pVzU6Ye!S zmG@#{yMmu9kI}`Xt)byD3x6Do&|aqzo{)WTDWAE)J=1q_IWH^3&F}1pn7*^}<<@oP zH*<`8z4o~rK=Xgub6EVs=Sr9!ePNI?>}1p=(Yk@zgDB4Y#!accVPI_Hf6$oo~dRC0RXLa`@eZdq-ZM zHM^N0%)!~?_-saUNM6E0Hvc5H%gH|@9jsn3_e~L;$#a}dnyL8Eu?A*dl?!UFY5S3G~+cH~mV` zDP4;d>W-|2I~thQEt`8PAbW?N?Xw2mr7Dab+m^A2a0$PNulICt+_6>E$H7TO!NqXK z!EIYbl^f-=6t=jztQETWR{K_z^Sy{K)8qx+WYY!3cmB8}#=l85CS5V{!Ld|}X>oNIFKF3%*e{XV8)r?@J7p^ z=_AK;fsPkfB)%jxF@IqYj5yH5Q6<1O%XI;J{DDK9d<$9(=Qyy4Su_fLYiMTfDdg$& zV3GXxz*w%utKO``@uC8QG^YoPk1*rqqdc7xBQ%9nBW~Yk3J6%$AECy~q!V?mYbvAU zKbJ!yry~@YI|F;t15=}}I34R$^Dxhi{m}F(NoIFmh9I*i#1CEI(C>V` z^zolby_pjZvCO+I!|qY;laj`#5ho?^jfv&#&oj$IR}`I{5qDUMU4emtBY{Cd)}V=_ z<^f;-f+ShtgPOK7O;(|Im=!L}>@b)!xuW#TUCw+iZ+dtEp%o`YtCx%VnW_Y6_b5~J7bX!9(XM&1k#En^JSQ;2~blH_3Flx_m+*A`L zC^$i(<=r!@aIJHq0v!Ubx&jmZrz<|<`^Up9+q$`3caB!3>4rw0qzR1kIXPi*r z{HeNJwMd2e*QCEpOC-4^9E_PCOrNf`q3H^nsu0JPBVKop1#x5@of0=wn)SF-uvlw@ ztNx2)X4jUkJ2KH(*8Xka119&y(f3$mJ2!JbZgH9MqHeMD)eWpdIv+y#R2;eHJ#bO8 zc-ZOF5zggVu`11|fpNNx0@J0IWi|7))I4QxkN&6sI&;upuH#1oH3mO`^&MaUv(kSFf zQI$43sJ5nQO^yre%t#l%DXQEPfjp?QDNiL65p57Zww&|8?hWlAM<~|h5 zH@J~|FYZHAWvX-R$4f7x=R{B6tg}wf%JLpK@DUB+r;H%kGVd3Ev$g@Lt_Y zG0vjfTsUollgJtCr6*l3uV%Zkka5|}MpYXnp)!dk*?9|^?RXM7{ZBM9dM#+Ry_3kk z{~)8R0wbs2#z$g13fhcw8ar68e-!wsQo2&uklXj+N-36qC1rCASK5AQ;4r(vD8A!B zv$}{QTN0Owu&ThdQ@dtN_$R>_ywUu0V2YX|SC9d-fX%Q9O`sp zN=D|y%|%MWH@D@9o_RYt+p9S%L0S6Njbr(%Wt7DmE6>ROdC*-FB6h-Z-I>Ojc~)x* zyRFp~_pF$(x#F<(bN>r7_Z)xHFonTney>mQBk>Cd7rdR&Zg)e4zkh?XJkvpD<{1b0 zr@uJJm%+KFEyTYOMwy54cBZEIPF4~-Q)ce$6|Gt2u z=|VH#vA8wT&sRTQ2;^I;Be?5o2lhrD1i@djgv8ZnpYcEXum&SW# zLfInss$CDbuQ>4TOWsm{LZzjd3!pzK#v5qR-hT3UEyn;Pf|OF8h$U z&4KS+1IHmnnX*vM4Q0kv3QVdBoZAYxb~@Bbm2vNSpix?pF!!5v?hE$q2AmBH92bi@ zr+;9zabUS^EORkANL+z|Nx`FNQ_>q_!Jp!R(aWtA1>>0|s!GGsgFckapU{xCoO8hn zX8%dKCj&V9Hk7a0Q7*|5?CF;0?G|kQAw@N#;+z5RRR!Mb2ePgk=pUGx|8$bm=WDvJ zlPYDuq^3<1+j52X>4nP03ROjJ+*cZS?@r)*xq$b&$3NZ`1zaW1IDLPFs47ab3o3kY z6|#E3-t>U2c>;US1@=k-&b$WZVu7~n4|tY1a7|X=`oU3hGa#YLh*_e5qdkFRp}~ZO z4HK3KaG9%fEll8862@)8U=x+VeY-*I+Ygy-1stsk#pfpQZY^MsN#Ly8Am}rtUSM<5 z0W-#*)9a6vY3Y1q5PQ&>cA#bXcg~)I$;}@)^8(WI(%57sFzRk#oH&8!=mHLj6pjQ}udG;OW_fc0?_GhIryDGOspVgE?dflBe*VZjbD{W_ zDSXqyds)*9RxIFtUC?`LL*=y}`MVmpWT>xCms-U6IW3;HS_ zumwzDcXnWME?_n`NZ9Vc)seu_*3d7dRx;a&(;|Ukg24p-$O+7uvld%&v@38JC(CFg zb8A&_Z}jBi=j2$V!29X|-*y2u^ACx?wocM)kmgY^iBZpKVaZw1Z1r&~i>+jpd8TlA zI_IvII&tZO!P`pTHC_jp^Kl%v;NA zosMT+J1|w(BEwU67@9u)>H8VNqe&D{AF}-UtU&YMn)sg9* z0=`98gkG8Y*0)JoDNLyh3t(NqC}_Z(^nz!y!>r#PT-z0RF9&eXG-LWFW)QHra@Jyl z3G*GYm}hbd3LSH+jnAWFk6iAp;YHdF&ka@jDP55zEn;Voc6MTcOOIg+zXt= zp3`}ERlX{iexsmw`GSJ9fVi24QEUzNpBbwq8HIkRGOXUXic!Ht(W-5^0#{qYqQVv3 zX9KwF3iKNUI9Yy9Fv(hM8nt-VN){yn1{VX)r3-4$7jp5<fj^_&5bcf6KryU9nTGMRWzHlN88duwU#3$Xt>wLWt}O0r>amO-!s12bFqJWU2>uG8~wD%RHAS{_-+dyQem?FNaIXBG#y z>Qx7?_^l?9rMTeEh05xioGUi)y-nbMd%*Ek!u0Oul`Ola^PFH*bF^YN@Z>I7CH;|s z^+Ni_>m4??r5O|$i~=}sEAVbLm~CFpwN-()s)#*)1G9=j(UO8$OUyW$1K2tZCahe! zb@>MW{TtcaAMjjxpz&kI%ytE?9TWK81n_SCuy|3!%%lW{Ppmn-%0fZ6))p!WCW;C% z6)>LtcP(vxfN-oAXT-*J?Z4NtdQ4vNA>?i#N9HfKpSRX$eqhT?*qJp!x3G#Ot6_uq z6XusE=aui8r=P&J=D~(JZBuJcWkqSGzB}e5_Ijh1O7risw6u#}6S>mWcJuvv!2f1L z<*fsbov$`s+c7=-LW@nApx6dy`w~Gxg&7io^-L3__!$@&0vMPW7+4ILCpqw45?JzE zgUh>`!{jovT7#Sg1IOmfSr40`6bjkH1>lm__Lt#$Odt zWBIV;+G-cR4K9WsIGz;nZo4qS$%|`K7i&@hd-(?TLxgLnO_bG8joj%^+ zP#1B4#Vdiy>w@grstMa}PP-w%wS5Bj;s%cNf^@~e?Q1782rCHX7%kONlU{a^G3CPK zy9WFxAFR{ey6(-cgKK*xH_qmiQQ$Z$z?mzxGxGrZjx}skxHm9lvphYy(_oL+0_k0^ zEO+g6HaeZKYuf>y6|Xa|GVp#n`ES?l54`KTkF-9Fv9j3xzcTIS0Y~NEoW(c!LjL4G zVBq7swcz>+?&v?geB!MV3-&N}2(7quKyaee;YbFa4K54^81w=-a&%TZRTVo`adk{s zvs7RS*GvvJORgmj3$|a_fA92GX2~pOOU?-l9kU`QuyszjvViYZ0&m*_?n?$)$qkIF z89T*~uHEr4X-DGHO`90^Gi z@3YeTnOT|6#+S9RQh7G#7J;KBn$!0^;C|WAa{XULZ^hYjz8aE0E^dCdBao-~Jf8wX z${k;?5XQO#$9_uho49x84~~6&ofEiaxt0se-gSXzyTFO%nH(y{H>+YMEZsR_*M(^> z3pm)8akow2ND7eRV>Qe>#Qyj(r= zy-?M(tnM3=0Hn3E^0?$s~cc)CK*mSfYk71J+ztek7Wv%lf`p$FWr54zWz30vIa_cyBduZ_0W#b?pSMoezz=CRFX2 z$t$^rEpn^&yqSDICv4-~^D=>J)-2xXE7_L{hS|JhNV#L+-XdMpbbE#(Tj?{-T^IiG z?JB4>Eu8%BN8j=dFMi&gzTgC7Rm0PQgewIf*h&vv(VfHQUwhZsXX@?TGnxyS7X4b=>T`)1=o21maqGZ?GxT;+Vd zfO}2@SNA$@j+4Fpb|2?Xc>8oI`_cy-B@FKh9KP-0VW0l`N>SC_x!%mFc~=+RzN_iL zUh;vxWW%%Ef9npf*}!9aX8En{@3n5ce>d&&>a>`!-3q69?>g`taIVGSrO=|Qn|*_QS4`Bh->t^FKu!3vREYuatqVModP_JYxvJJ2mwhLx zqsKY%14ktTM~ebS=Jp%_Fd8#>P#d7j8OcT%F^&I*j+WY3|)kxsSsxN^kF$*juSQ z{oQTzzNe>heXi#wn`JIsQt|K9la-Ub!}q8B5>h)eeSN&XvFF1LiievR#dvHcJb2*P z$nH6r%VO1n24=>R@In&Y=rap6j#=31kxu>cd&tDXAU_$yORjYz~2OjdOpUpRQZ(7YLBy&Pg!XW2I zgX80)#p3!wOD5Re-Bn^$zv=SL&&SU%_0CiDU3KJDq+>HXf7ooZvkwpFeAJFQvZ8SD z_I%~jwSSJ~7HrF2C8)}&x#Yx?1Aam>YNr2Y##EjZ)|~WUK`Xc1hJ=O%LK74;SIZTg zS#@GT5d#C8(rT{86*3Q)q;lsdx{3tuGdeQC^OTkMI!XSkEJA@x^Nc~WYDh{{ zz}Bk;Ri0h`6}uiV-DY~Zmhb+hfW!XEZPrX2PK^zWJD<#`e#a`lqWFrHd;EgrATRa3 zM<#l6RW1?m)>+xIr}0v>zts0tJK1LEZhG0a=x5a37s~mkZlRx<<>P@vMYE!#_rbRg11Mimp^VF1h}KRf>QE zLu9w!)}o#z{R&qSnguI3D_bTyb4Z#`bpG(S+hgmuzptld-_T^Ao|Pr2E$&eAAgEEu zyqSlQP3VMA(pk5jRSibs4;$W@cF!Ik)W4x4v9f;JI>|te=rcu7y5K&^HRyL*R-!*$zC@zV|K5}suN3o ze>+@dkhkro+pHKBpQZ!MwO?CQw<%spygr|82ZJ*+TfqUJyw7DiE_s3r#EuGbGc+); z&nRe<$?{v6E#xr4z_C+u zY2k|%NBHe3na^>|ZWdm5!FjUr7d8p8+AHY`f+sll=|9N$&%NmX59U4IhpUfFO5)IZ zF>S@%m1+wa7&#oaFXFE=YcgGvdRF@8k7G9qHa5n-UFC2lIof-PKr6>7UJYl-uJ#?K zy1I_s6$}~-Ok4*T*aZ>_LNyx%W;pU?R-_iC7%+;-rR&LDeJW7p;mmz)1Dole09IX& zCTS(Zj=f)A>sJ~ya(ZoGQNO!Ls^w&>$%a7Qm>Wz&RuK#{gbp;^PkmjU{O?a}r`lnL z>_}C{v{MayF2+(^#{{eLqZ=4y7#J->9tqYwXwFM|rfw>6+~PRr{Dn0R%o0Bm0-xKp@0$-V(Yq=8KE^9}!jCIPU)W8*`W%+LykgeMN5+DA zDatF><*2&(eCo{!ah7VXU=o?b)vDoQ%9(d3Lgb#ad-k!n7lON^rYsU!GJ)l?nj?pc!VG~H4Hv~P3)*d6BKS*oFw1fk?#R5O zq?5AYwAd$u#dV4k1#1P~ScrZUbL3$v{duZ;FWbckXOARq)fp~vTHgMp7Rmg!2O5Q2 zgM8Qx7BF&VFmMS9EYoj$dBU?#Vp6}4q;c$mi2Tb3Lo*}Su%;w1NfkA){M*puuHhuO z`)%_4Hs=-nZ6TINYa984E;KDz$-vrYlgT9&%3@QjcwTd@$kKlo=DxQ$q&sy(i%L!x z_d7RVMc$QC4gdUpZe^`HtTNfmv&L>y@7JiqQZFBDG2L~M*DpDF@5jy)OuBQmHu8R$ z5-z;p>Z)%U!F`Lfo(8xdJR-!E`)DvaeDYd+*vsDEO8 zy4Z1V_JyX%Z4UQ;AN#(qeMOVFg*sc!3KrEPYdTz~L{>Ly@fFGV-6@~mKe425%G^!| z7Mt`v95E5|OS;QUe(h|{d>YxNewCT`zoW7Pqf_Z={Yzd;9;wSd-C$y|kCD%BIkU9t zLz5Mk1tFI^&f10cRK54h-y>raslkw`J9_GUBA5JNr5NJE@l&AlzizVUu zf}QfScD!D4`Lu#xI)_8YBY_Bw#+|2>Tn|lo96VLygNi~ShhLf3`?-r+?rC29zMtLh zL+=8XhphRWUM@->r**WbN;q=5nJ_&}`gd6A-vf>#(sMp_zwlMaEZgkNdws%j-}oZ& z%8yDJj9;T3FbU3*zN;_dTQ_shmA@a3cRA}X-pze8^J?7R%&veZcB^=@ENwpV2GmzQ zKl$Heh4v)A{0fyEP7?)Y-iiZD>RV*~%$$13)u=Br*ID+)g$)`(>>T^F7P3FvgGA&`;Z#L1xeuVa%1YgH0^aYviig55zASUfEpK3O=h*BnT) zIFNK;zLy3YcMKa(Ok4U52R@$mbdL5EhJz^;Z9k0Ik~`KXDX?+7>^T$6ax$9b^zY_- zKXl&#1UV>h=`&tZ#ekC~>uGXBZAW#%W5BLPYVFAY6Q`22maC;<7X9-|D`;*r^p~d4t_g;rZ0Se9Mf|!ImnB4cY zW-c%-lW1q&(#qPiElQ)GV~T@Qx5J-K2ksC@?teOLJRxjhJnX3jM++DnBd53uuW?O| zIGD76CCP%#%Ym)Am_@>8$K40L?=1V;C!2S!=5z1ldlfOITbN1F#nvW!-M5pxUX7Z* zoA&x?m=*T2|BDxUp25D78qq&D z_UU|?_10RoXo2Y`Yt}8NnN(gh+C5<~j1UqC=C*HOQhLOE$Z4Ca%I@6;yZ0zGYedYI z+rS`vVcDrkY{d!=zH3;5C$IzwY+&|bV>e-C|02RZ$Kj8rE4K+-*qW2!drn4}u!YyS zCReaU`y3UqVT;dcaVubPmoPqE-JCdQ=6g=QhgI7jd#(HTB4SB{Fq53Xc3FicWsXBV zkC~=^YVw?@x9DZxl17$Ur}XaJUbmu=E$HLkAgT2qmF!AtELozQ-vuw|c5v4}!!hvz zTj`6osyiJMA8@?C%{Jl1z9I$o(hF_YH4}q3ux{+|*`Fb_yQ9VC1+$xn+4hL;k|6Gc z%iJ-+OczZUxh)!7wwPrGu$MKk7bdVyUDLuiyM?92cbW;SKj%@-Gro7HcqT`*&EjF7 zyT|k4nv=pN2a^ogl5V&rZD5Jh*|R8yrDe6vQW>}VLLwKl`#vQebDeqW(?`Yv<)$?) zZfkj*bZ_j)YH5D4>Dc0hwvMl-_|~vBdTjYW^U!t?>BZTi;h72(HvIDr(>h%I=X{sJ zzP@2{V zuF)N^f;Cg4{i;m6@DvtCYu1}z@3ro%|<~e3R46Lu5y1*KG6yC%J5s1!}4Ow zo|bNAwWqx2U$3~>+_t%;6*btiETAm$b#~SRKr9 zx8 zY+*O7{TH@)?YNkD*Y(L7*V$`475|=;k-ZkDW8*%-riI(hE+Q;#tDjS#O6TIB3hN~@ zCmR1{?+sGqFjqXYS?RzIebL}ohunM@v3%0l`L?Sqe-E3*R8uQcD?OX&=ZCL9cRu{L zWvb0re;eBcELYFxYaFhyWnZyan|HIB#LL?foPk?7VgycyY^sPTQS3gz8^fpCqWY6N z{q>EjChbKDtw{x;Q?|55*g8n9ZApG15~k3$uQ>98Y0xFppy(~GajxeQLj9h-xtUb! z*z&kJWv<`-MUL+!^`7lGICmqX^F~IQ4UEz&u4&sg$q6v!%5GP9a;bN6^Oq}q_SN@% zge<)e?ET?lnUQc^vnP1t%**``4lT{LJHT_Hs%Bp$&kdWc{kx_*FMYl5@Rjq08tj%! z+x}X$HvbK1R{Yn>ztt>zxAx!OiNOrqn-i9A_|YZt^R`sywty7@&IT8YI6SKsw0~~! z_0wtn%P)vR$&hJImMa>eI-DE^XIK4<@uY-dl8cCgT!^ zUn?H|YIOS4C^>^s=11&9*|?Rq_qvtMuUs~^YMAdek@@Y1W{2JLxl-CDRUdlI>TXeM zsPz9Nou&treflEYPaWu zC~rNBBh@YYg-?k*OA(ZNnEa39+NGzFqG5?*e79UXnnU`U7tZy&pK;B>s&5YCd>OWT zk~bP9cQndgc&dFh?1N2Hf%Cn*+L;^Hi2OV`<=dQ=KO0%^2D7#7hC1=hNCvkDSeGc+91EY5AwX-nCW5Nr_1wJ+h*j1=z&DjpkJ+*dsDGP}Gl zxg|M)CFMm%zh9!D*h3}VyCQilL3MYY`S--dlTS$5gKHMUhEr+cfV6hii)6{#SnY9^0kkJs!^CommEE>vPJM8ecolDk{Bv z_Mv@WP8)w|D^=^6*}ZT76O+<_^v!PBkFB#yUpSTdt?$w3{5PS^U#GudgQkcNTkr%Qr|4O^vss4eHi zyB`y_bj=7q5R#rL!OyMpB1NMubS=xid$0GUu}(8Fh-8z=>E1X0NLbEMH@3d8hu2N7 z9WQ$`@2#urhx|oLa!yZ+4NFVCwbbvnDj$b^Qv(Bo;!hS1BL)Tr9R>yl1_{OxMh1?5 z49_`gjN2C+H0Kc3ia8;WIL(}2-l;{xuzRYGnC`xv8A}%*?Ne|Lll0uQWa`A=RZBun z7NvSm(OrIGj%BK+?@X_ZpqZYkol|C-)vC>$5k&%#K+8X33%LT%y_`*Vb_Q z%+MFmauCwyJ5+I;Ph5|OYfj8^zk{czi*lVYYBxT1PEfsV+onL}<2~~JtdT*v>_PaBlG5VsKHP?@At(^Me>cX;#UY@f}Rt3GfymYzm zY}w99yB?hUm-c^NS@!2w-qSS#-Hne)=h)8!en_5~8^t z6WbcwG}&7C9WAyU=GF5tbC+R?G`s1@>N#yf`T|Fj_L_Of`ya>uvG(^1^_fUN1|fU7@4?up#ltA4lqL8^Leh)0gPYPnm` z(JK)VfuU2YyibTe4f_1_=-uh=oTgJ_j{Kc6Ip*BcC)a0(@H26&5OJkx!2E~SIWIxuSzOMy)xzHewCXQnFlPBtR~;!SpP-ZH`JkA zIB4mnTOr3~%tNo-xqLQej)?5xy&wObTK`C@C@S&b)_Y;`w>Gr$`g?rbsODC)N2D}C z(^53>sAIQyvDw=%^S5S)EL^)S*JmPI{Nm+;>ymi5-ZJfar8K4F5^v=<$&Qbwq)j__ zJpaAo`n0mz*-K}vQrYIi`Fg3eD2KYgkpJ_OUp9Qsa(gOwRMx#xTPQc~==F7lw|^Nv z*zqoR?}_Nroh-spUK@`-x_t1gMet|aJyQfdK1GDYh2LB<|J${FmJiNZKJFF~*`iSR zR)Kv&;@ws;jTMa<$1;6aU%w)hy8DvJE2X`UW0xE_=6L^yYUPQ=oVk0ZDJx7pRq4Fy z{1=U5hc=32+B`QDo8jZ|{(xV;43cF(xx%cntu6S22 zx9Rdpp9t35rP9xS`S{Pe?ux_B<8qACYg z-!Bi6kDq$-c-kWKzNzVUwhm3aQ$DTM*HoPH__gCa4bGPNZypL%8ypht37GPnW2r@U zQeUClCCj*Toh6*Bgp;LIT3EbJbh8BBx-=uq?ZOhn-tD_%-x>T^%TXOP8qa zVi5Yk{y6CFt1DC28Cn%6e%y3jQZwV2Y10$e$lXrvMaCPrw|78&T@{5r-!GT3-Mgt2^0)v1|lK8F@hr}i(d2<~AxU4l21k`G8>G?IZsL&3AGx zNu4SXqJD%kq&;MH@=e|3C@)nZotaCLB^k}C7aW%R^P%nE7X^-*Mm@Xo1ZKqx2Mi}f z9^|r^(DZiS16G-eg96928x=hmn58~2a5y-yDA_1-1-@_+>2kQw8*+e2u;T!WnnBAU zxiFd78%&{JK26zKqM6KdJLdT+2?wnujSUVj9AA|=G$(0oXj~+^)>&rJiY|>)Wr3Gv zvmJwMHebA$Q~mL5dK#r(eFYC(o~@tudS*_1!spP5 zW;^`2Ze32^H~S1HrwU)b!>qO`o&udc8A`vr)-L9ox_&>yvn#Tat8%_qgxvjO*zcbD zQQna?`^OxvsjOes@70|-AJ;#1S6-|v`wr`t#N!N{%U`gW?rQAIo0G(K_}+UK358rH zlLn^bcb}&3zw=bjJb@`?TM?s~LyKwgp|kn#j?B(9O z51xHqy0fB0<9p>I{gppLFV=D&vj5Q799I7DTC3qqALSFSIvI{Z-nvIr&Q13VpYtm8 zbD{p0f4w6Al0*z2{e5ELW7$`JWy`d%v`h0B9#OH&z0lmguas3Q;NS*T1;#@01_lTI zste){4NNx<@L#q#Zn^JbZ{D24k~t3;W=d~pSQUQv*8R8!mLP%D%=g<*3riHw(`IPW zt#K5Y^MXa<%Gx}?f+XR&3s`p_{Ju)EmmSEI*X%Ovu^Xu50zqqfE2m%jb~ zESEU4i+V8JNnx|vA^vYd&A(+1ERpv<)g9+yP z?$;oG6G0gX!3E{4vyZdoA7GoaoK=FcNkM}(>;Rj&Xq9f6^vp%#_cyYg*eLFB&1;fd zVxB7Vt))!C#~5v{7pERet(=yAt*piMMlOp($(`%DiR>kn%W7iXN=}HE@hUKyGH^a) zsJ(cd_vQoMa}(ILA21|}m4$hf-WQ9|cu;zcEsy5{1D``)!U4{N4P{y8+yMtzV?MA} z%_w_yjH}=qt8)P3-|MW^wmRmR#0%sjN0myp{U{gLndS_Vlt6j`RXGj*@Sz@ej)82h{&C=l0!@t`NZ9 zUSa=6#MNh!VM<~})kXQP^q^8E6Gw%{r=KLWJ}^m~2ojhs$*&?fxlvGsqiJb|Wb6V~ z<&|B<0^OF$MS{!xGeyI#1;TSh|1sx_GRrnpTV3yddbRk~k6b79p2O}fA=_IrG^V(o zXbHLA6MCb^yQ9?mXG?H9qYVS+{Tn4d$9uiRdFvFIQ^a~NY4qOT)^|$0OymQD!Uxv5 z56WgA;M}}{`;7wM?g9=*w)mPEWr-g+=NWJ}F)*d1cRUi{V)`a#Wm0kfp|CHDZ0B|P z!cDrl2^9xfVg+wj^gU4rU>9~N??0|&Xf@YHhWDEKLjYGA+5^F!28?;H_T3TK1L2O@Q^M$Mk$N?)VMUcNOq1*$`oNB!cs#R%@nQ z?4^X>X~z6z1%j4|H&PpQN@q4*66k#7GiN!c99Prgo2)SxSQ7-;5;nBQUx?%Tnqqcg zk?yOoTdv_dJZG0ooxNS0dHeL)=9-KS8jMN{`eL@FdUe#C`q^^xThE!3bIxa!hNmxa z_h>zNvc{)?>*j@$s~7kl+~mF8z#deE0 zG;f?yvM5sEWaLbZN}Z>YZ<7T4HY+kcTzI^QHQr&-Yd7w#1*_g1naH)n5s^l+Jyw>{6OGL_vUwRuWE!wGCsy>R>v6Fwu0G5(W#&qW zT|WGY8zmmH2XA1Nv*zCVfqQ?#CM9d0y$!qe8gQ2`=Uij8*m4)stPM=Y(z`9PH}Bwz z3l7V1bPS-l0Zn>J-dTrVo5rMUQ3XItgc<#+u`t%>eysH~{ z?|)eOreW=U2i|iE%zPiFrnu#XdG!81TKxDL>nky~7z3_1ANY1|U=99Ycezj|HZ}Zv z6!+KVd9EK=cP8-ea!@hQnsjv0G=)mT}mHQ4YHtx1NM-rJ826xkwgLCP4cxICJikm>B_MbO!h3g%?!lYTSP_}ZW z_v1{ND%QDA1Kh%Iee>h_wB9SCJUIX_Qdit zEU8$Qdf#lH@B@b9cNhdOFbI5Dlg8D1k+b*uwmCjKra1mw<1xGT$qS|e1ivWC7Z?7zJ_xB)=d&MnOVbZ{())k8a>f$jj8JF5*Jw` z9VRWl$+D#)upjUx5Km#gvm@;tQRobdO@|} z0p@>S7-R&_To*qQCU)eA&ynbzQ*%A0oLn*I?9Z*9rxB2uOz6RV(YlIrSy_=#%{fkR`J_OW0`n4Ap&UKVBUTHh z>dDVN$n}s-@S#tM!=fz?hg5yI8*Zen(yCt^aA@xb?zI;VS;lTQjNZKYXN=hc#vR+4 zOb;};l`R$7;=+>ffP0NXJ^NnP%?r3MJBT!Pv$R}uF5OtPW|LaY#vK!4E>2#eAh0vC z@A%1Sjg9jH*%fLxay7DwBsMMJWZRsuYySc6tqZu{JY$U#VD%MXeSeyJf5M@yA6VDy zVsc7g_VwFr{CZR7MD|(*X4wTyh8LJy6gc}ESmXuRcQEV=ZD0()R{S_K^||gGN1pa0 zIuYJyrxeVnz3F@Px=ju50Y;MxV*j)n7$pK2SRXKmH(ZhZz!o}(_tpa5iwRtY4UFOe z%*_V8r`>q3Cag7{z&KHW_w0kM56W5;9@M>h#+9|5^}WlEo>LZe?ksaY+;~@Y!9{>; zm&5XduHGRRBidIbFZpz0Ld~O)3!C z0N*}?3V6JuGJ%52+ z-husmz_tDf_b$Ip_2oU{&aKwMl78V6Yg9+>>3%L6l z*nI++YyH|1pE1Zburw5KUMk=YX<(GS!1&(p-`1xEd=D&nYbP*zJMf+jU}g$nh>m2C zb68Zt-H{-`%3tU4IguqSmBsf1%jOTs&u`Zsy{^Rkus^jIaQ#^tXviw=5t@uyoEaOV!jEmhLV}#uf3eRRO z*>KBboW1mo6}&h9Uf!a5da8Etnv9=D-if+_3P~AEiHDDI zX|{aW!}!Ggq_p?*6}Hh#%x%4nS657I`sjFWt-G0CNyo+{)?QhYmQx%~tQ|aj{2Vp~ z3l_1<*o1z(QuuH|tFmXRPQrwbFGS=xnWW~#IxKL!#l~WAprb@jdTVjf+0^1^CwTcK zWc6Yy1a7?THa~l=)H-}^v}M`VQ=zlPGX0I+WjI#&J~-ecY%trR_Fs~#_mt@xvpPH+ zo0hPPYMO47=sa{Xe7*l%i`ttEk0z|kFPxJB6MFPYGz0tK}6~ zda*Hv|B^Dt1O>aAHv*3yv383WpP6qy|LLAzmXY`N)HAd16K#yzl(90opv)=;bgH#HW`O^i8*}D1DW+d`wt=f3BLvqrB z!yN2ZD-t;^Ruwe&IsVcx5b(%rOzC3cs%U7I`lazya`v)?$0qV56daRDpY~$`fBK4r zY|LBFWi>Fdt;ku>Y&vO)_r|<^Hg9GIvCT~GlK5l9&@6JwBK1hIT=H@m%iTNwu^D;J zS;@l8ey)pwiPi2!(_}w8#@=a>2RaV5U7xi05Sz+2g(mK(nuSk=^$ut(YT|cz=*z9# z@_Vg?%8ZD;t9MN|7G9Idt60b?c478o-tU)~#q2EZR5af( zee$7Hsf%Ty>%zE-#N!KN=k0Ksth4F>6Sua;t3@|kG8!K>OSo-nWRsXykR}z)6!}O{ ze|abK5veB?YSPiuHYD~3$;>>+q$=a+DjKsyP)Iy<-k#Spb=V##uvrGIJm!5gr}H6G zG@rMsK&VVHTQmErm<5g6&Up_Qr4&v+GdOzcr_m$*?TtO^R!e?3w5cj|KHE@~aF6ZT z@iqS*rp-!U5O{=@owsp&CQ+A?733?(boYZd3LUax!2$RXeK;@xiPn1aSVmX93X z?~OZKz4-d~39bDRjqgq?IT)nP^c7>PWxP8#l4VQNKCNXBwu!dPYgJkznp2_JCN-g> zMJoEq0VUDd`%F|iRqk{&pGey$c?ai@i>2lqN8<}oJB{WUA+;xgKeZ$91vvc?5NE#kFrPwYN%<=KCVAiIGZOLWV zj4ih=mb$&*nC8Dv6Pi!^Brx$#X!zOjfYpHKmGIgJN3R^X(5hkM$mQ~4!S$?%-H{HA zT%oHQZZb{m&Xr)wKIZ9nXNI%PZpr5sYpVQKuLzzSQP^&M@1`SP(Ss(jJqoOQnq+FG zMC?p5{b>7bp@;0Qf(fh}7m0=6IjXZlbwL+LMi)n+tFGO{iA6UZ*!o@_HlEkms%q0* zcU9x4{J#wxk4qm*7R@-Qyy9YSP)P#k3X3z|vp(vDcqmCJpE|_);KP#4F9$gqHgK9& z7%=cKG)eV8lwfB`n6}xVfoXbS151TM=daSo(t3;I%|(h&7R@*;ll4db{Jt&5mRBc9 zt$5+cDYRkQm%s&#oEi;`bN(@~`L!%G3%kSWzT*RE>m_&01YSq5Fx z?2JWCRS&tsL`uZAUQiNWy@9ojX-XIKQ4x+91s16pf}HgoZN~o+TI?q@>D`^tW~^Ym zsK|%8zfB=X%PN7T!;y_YJ)`Z8C%Ah3|hgBTeQZBUeMi#Vh8F z@ootW3O}4!?FHNXDw2YBFK|^q5b)12%4e-`)l&ryE`tW90tfCTGu*@{ebBtJfQ|E9 z;31(l=2bgn)46J9_uY?E)s`32>u4`-E5d&Y7KsJlenK?E9%3<~4CD|CY~l zWllUO(D875GHHd&$-9?rf1YWQUenNOe&M*?OyMJX2Rtr>ZAcQ^!f`?LnS*>=-a-ZT zpRNUm9aIRA~!R<3LK#WoyZ(EHpV!r{oYEU%kUE1_A>=51#2U6)+5jh&9I z2U|jaA7JC!pe~WFD1Pk*vydIDvt--kO)HrWrB(m0bU1)MC*{Co=w za^5}p;*b~eLRui|Y4W*$8`$p)9$G8J?tjv@@8mMm3Ds*JXy=Ch+gbAVCiC+K>6ow^ zU&Y%mHHocjXw|J*c$1w~S$m!Xi}5K14ZRM=54HTvl2;n|s(o4o6AYS_k395UGJ{d< zSa`2W%>h9{?bcbpHXNUq%)s|d!t&!($LYog7H}?D;4R-~*PeM}$<><@E$n*E4V(-L zwIN>CJg*cu?f<_|O}^V^p}2d~VYZTho?{ob&QsRmtl;8rXihh1&**4bvtrWD#k zC6ttm7FB)PQ*+t;;%cM##O6ze0`;$$0wZM4zhu>xQjB$&a^rGS+T)fq!GCRW5^Zb; zcUY=OIJWZqMHBmVLV-r|uJER+ubWarj_a zk-F#u8+jew%d0k>mOP{>y|->vKMNyUYe8$v57u}Awu?6w?0Cd-M_MC}p?SiA9sK_U zxb#K1Bmx>A6>DW3XjPfODbc|^+k^Glq1K`kZ5BHk9yoePdNitbu-Q#&n?}ZZ<{$icQi;AwB=N=+5c#mWx?**z{TZY%07vKS9OD~MoWwYOMa8vsV3EBw>iPAd1E3| z0HehMCN6GOo``0z0v5{$jC>a+FJ5_Wo53um9SzDKnxkj1#Y!xUbyym*OE0UVnakt6 z$w@|6h4v#AGozH)U00kJP}piK*}x^i#Kpm|Y0f0J1_lm^nQTtpng$m^3_ZL|!y! zzSzlrfI)Rai+e)j=6@3!%vUh6m2~}A;!}Chz&D{`+m9Jrv`=r!;pQx0EVpD&KOv#g z!N_-G@++TJoCZ>pZ!#qauvL6$Wt%-&e_?A_hKd7QSo+-eiX0RECv{_iGw5XzplD-_a`4a+O_Hl)d-#m4eyXEqhk(<L#H4kaZo#tNT8xGkEbHiX;0 zWLPnQL9n9Be^SGQ!Y+mle*ZRK*q~dxK$A0;{es;Olcm1etriWNxh=TaD-?L58JH#7 z5;)je9<`NK@ccZ;mQ&Hf7hsm~AXZ=o^ZZ5J>u*UKOk@#IVc;-mFkoOyI1$Md(XgP` zitR&O@Kme#MMBaREHM>qD{PsLRNsnR>$ZL7Ey-DZQfIF2IpP@gk@4)|sbU_R5^9YO zy6Bn}G}i}5m%3auPhfuWf>A)D$+yFOXRG`9jnUy63zY;o$E@!(%WN{Tl7EC%L`)(DO^g9;{Rft5KAl5$S4@=GxA zIp0gDXi7IY@LzdB=qKiRYlZHyvhjVsx8Uo&w|CWD zUTb(qtHF;0QahTzZ(;TBU}Ro#ZjNb#`VQ9Ie-=x#GTJi*4m{k#D$dX_XX(9!hStO% zS1jBz*b=l`k3HV(p{=6OlAp}1aD!1*;ECo7M(GX5zJ^|qjGB6~Su}R;)Ppw}4;QMd zBr_)j_D{|0|K@SLNmF-_Tq`PN-uP)@Xb`iGhCy1MdX}o`5t>xuydLA{d`0${m{&*!qag zg5hrvqveN{`kCxAeN}%iY)hNLV)KGgw4h<53|mA)tHFflwhtS4wl$u%h~PJ9(6wO6 z_2>ymY0tdTdVF77%!@|Oga$>9Q09)-@CmGZiyPj~ef-b=>znA1*K8b&tOiZaKiXqY zG_m|UrTe2{Pv(^S5%HP~Ot;(ia9#~vDv)_-=2Te@CdC)0#SIo27QZ}^`0}*j1TEu0 zX2Ub43_E7JU7WqFDN0dXfT2dE<5-1EQ<$U1>-lp z8zL_ncnr?%-_gJ|p+RxS`_y?1%Nx2f!;S^A7Hm0?d@Z&q=)$W1#tZTr3|4(&o8G|| zUD0C2!6cBu8o|*P#&9q_ulr|r1JjBIB@X5&0rquWGqOFFX*+3XD4hNkyp5}Z_1rrH zt_2LI&$-5KDg0Hfz`?;Fz|b6aLFsK>lfwCHQWK(H1ivacLk^(Q#iU}k^I`%Z23F5(0~QenlSRr(*>a(1kna*-`|LTf@tYZwC~ zQ$k70-zDn*7z`r5#cpWJy3uO5uWWWZYa|DA{Hlh8ijT|v(__~p?O#?E(b1atqqOy% z!IRe3mF=zJA70 zcrKPrH{jMZ*Pk| z(8MaeOna89ldX+NMS=sz3H4GcWUC7giFqx0-6rXyr(<0atXY- z+{nUtXzy=6dFwomor)h=yPG4qR@Gm=@bGtA*qw+aI|`M>{^U*5adZkw^HSK;BIK=q zRKVHCe(#dW9@a+}T;?)zIWeildF=&9cK1dGCP5dEk1T=;H4IJcvMUxau}Rnzgfa@J zES<{8;7ZXkOQ}-NT3s6 z5wn@a6sTL-^T_PvmgcK{>0j+kHXEmT9flc84Nv(>p$5-Kp1N9((7cu(hkr5-~W?z&MNXvt-s` z1DRu)ccnrbbvCdhFK4WO_05IXT|zM?Z_UZmVEQn zfAU3v*Y2l?;%!5TR2MvSx582 z_*CxhlCpreQM!#H`uj#@%e{q zSW2cwos#|apirxY+k$7w7DtYi-|wvy+t_&2dn#Y74`Z14*tLj>nrAd1k08sn;|L*FA34jyT92*AdV%?}5{&d6Ss54mh%< zA86)On9v|Iw_C^h(!|Iui>97PT*n||!6c{hktgd4tNt4e_Jma*mn0NC)7!Ax%{1Zo z^z18#DyDrrt>?E~^6DQpu}us&1tbKTdUBr`FZk$NsC9UQ?!RSbn#>_OixzasWjG1D zR5+dV3u|^Y^s>A#k#EU~<#SE`}Pt#s?n zw#hKKBVlXs#zEL*N8C59U=~+Rcgu4ZBhPJGe6csK%X-G8kmbjU=ZPr6T@;jGT96`cwhk2nvn;NjRN!16xyO5kH{dH2naWqo;hm)Hp~{m%Qa zZ_+%Gr9sgzR|mMthsNeE%U*G)@VY^(w#PxPpbIXZIfsH7cdk`scS;SYPYO}I;E?02 z8?Z@Cd=0Xq`fO=`Zvvb^{zuj z)3(ifs%=sv`0U$j2Br;9nIs*W6ax%<_v;i&yzo)CjNIPc8tEqe|HYdRKFv0fC!YB+ zDkx_^5}Ea=g^@*6g5jvc8EK1ahuCI4@XOr#okcEWilz8lN4NEh1D&F{GMucR@O6D# z*=TY}q0fZFJ9L&{_}lfT{_p4HH1{J4LP zj9tRFyKbCdiWV_GV8Fr5W|G-2w)iU}zgP682%AK9(X}gz7b%~(G4K6}yMp}!E?N`B zmN3)>Oj~j#qH9%eieo*ajd>As#U1V42YJ07G?oZ=2noC~Q8$+q*i@vteZEV9lXin6 zPlCW2eGk*=$|13OyAqg{4eoHpO)%Eq8PMWzgz@6ihDHXN+OX4xmuE`PH(`8~mAO%5 z0V8Vx!w#wDM(#Ct4$9teV4I}Sb?^*Z!R{4D+Y0wR%uY!2*}Q{^)$jGuPR(n{DP5PD zR8HrZ*eaSk*B?rM7b$5W7;SErz$Cn+Vek24Of2WWDNmSaFn8Y9oW)53QMuW>_esB) zHnDfgKM$3s!SjFR2-$C%baiU-t`_4A6%~G)3!B%8PNBub@`zGPIeGX5oio`0@w=Un=ng8;@xOlW<=`bBEDyj(eZy6(8YCd70DB zm1%K5htYRZYINlW{>d9196eZ^7o1wnaKSa?tXss9ooi?B(mW^>;K;wgWkUmlSIqe~ z9_~9j9^O-e?)>9uq;;c7*W$3T#bJ{dhY||n#S15%42VDP7|$BfkSoD2UK;3hlyf#iwA9;$*$>+6 zmI{7poY=wWta$E#;FN=Z{&Id1bv#{jx447<=Y;+rbNby=y(Yb7;GJZr}lcDxVYfKh7U=z_8#$0=?HQ;sL?QQ&dz3| zLx&YL+66rvc_kXvJQxKn4(j~**EF}#N#swX(Gn-gc}>O-nDlIz^(>kb{g{)~p4kK) z7T}#|zfW0fN>j=_CrukBMGGgnAB=Jv4yN@f32Ph_Q8?J1a9;h3lXecnx0Zvs59XL= zH0w1ma;)%VZn+`gz<+eZNv4*Dj)iQVX6HKRx}D-_l-%nf&*!fDq)G6Abf#*5!3_qX z4c9ia3aPps|Ig@H%6GQB!F|%i_Pn1d8k3aVdu~thOjr}s8)T+h71k)Ca`38vfLFj% zog?aDmlk<&xR(E2_b{Rr5mrpUWX7mP7I^O)_r|3avRP`Q?Bl1LL)iw>ytG z@Sb3JlyjQdhJjV%1~UT#Lrh;+X`|kgLupHU)M5_W+caC69O8;dVU%F#Ju}O-uC>R3 zf%%Nu(F)et4^FNRWMEIY#d2kt+MmPr8O_B;jG3H{vW*8~wAlNv_GCZxQE22ZVTk;7 zB}{m^bXz3kpqPr5%$i1>Jja@CPBnFotW!9= zdRTl7rZUfE(bT=VeocdC%n6-|;(9NI^jG-lx*Uu))6gsG7tCQ)_sIwqWjwrAgWo0f zKxxC>m;(%BPK5G33t%t{bda6VY*N#}x@OKY z-hc< z`OYDeFAlsS0se%G6wGLBiAq*iInNN$5cAO^(uA%5$keBIR!5iwH#~g3sfu+=l}S@CWm=M99R+@c7&X}S>?*{KzmEwYnC1crj`Ro=lnay?7+aJz`(q~VRzgaj+O=v zgVm2qxM!c?_!!V2B5+98hPi~*(eLWTxonI=ceG~zlK3CW?sx4aU&eAJ&Li`@4%Aqj z^$L7bqma5wRBM-EYSpUc#h#4(YYwb_)^t*+ue;J=pWy_aD+ktoa^SsjfbUJ-2U*7_ z^A`A)H1MQ!unRbG8H;IJg4&=FOWx`VM&xt}5$n(nUV%YEOtRd_E7b4d4rlkqKwWhq?V|GqiQwv(RIX0UbP zdG|&U-YLf&3Zylpj<{TDUvsBl<>C}K9xYJ?@mGt~>m^o+_%O!2dnYF2qS zMJ+ehIGi zn{|q7;KjHk*_mG)ihT+!1^Cy7Qo_hqJNAVPg+w^?qlQ7mn9AWpkJ~@Y*o# zKNrZx*4Vp{Mc_{Zdv$;R&S^pnLjRXC&W+=$xRdEI)17+*1NVXkju}ecI}iTjxqe(V z@WC$GaNgaFqH`Flv>5Lft}%YbC79w^b8NYWOJmTPvmPu5B@Z-OU1F5J(5M-pCVOY9 z@ti}_D|EbXoMkU@Jl7SVz2T5f0i&RVBd<#1>6}K1%~2e+P3x!1uuQr7c*jTMo!!f( z-f5Cty-YIu*Q!wI@XV>VI*bgT2}LmQt1z2xILPDb;JAJDojg{SGaW)Ehix7;nOHcR zcsQH59M-(kz_H`rbh`$QF9$75nALoo)GV0%lbuXrn7CUUI21mx++a|Aa>&F&IdkhK zwFm2pH)uZ5Vn3G>68iXwV&SQZEpu;AZP;;_-8-<+?My+@T1Vd5r3@e4MFS49zKfNd z@{iHfIovy^QQ^rU9hoML5=S8x?)`5sN?9}+F{G&o95Oa>*1X^-zu=IrNwek`hgr7w z*hCJ_PYj*QYACTHOSoj7#Ca$Eh{v3{9-FNWC}c}zDwj0O{KmVK_2~@PzK}0GGuapu z4%p6XWS!G!GUu?e%7KWxZ)KeuSUMayeU6@b=$II-rud*yiNQ(5#@Xmk1J8>FzCR7` z{xv9h99FvFa;oZpz>`L`#SQk34NN8sL7!iX-pLX;p;ThIyu^NC|3yuqKY{;6Bg?Xq zDrc}8a2HQ*&2zgW>(2AaX#Tsx%s10?9VIoyO{|m;R2~p}b1;^>N#V@F@Jk2yWDZL0 zc`W48Wckm|*!i6&(WX3-10-e(+Ast;C%hH`v8q<_U}o-5=0N=Nl8Oa>9k z21nC+I?ANphxZlSiI~Fugu#+;t=UtZ1|CxeCI^Qo35N|$=VT)e$h$a8c{#9fFvu0N zGMsRbe!wVqq0!ul$#lvgo*DawqX!vcme5KoejWm`uaBP?oIH^=~+9J0# zUw3tWWi?CX-ELZc#Y^}6QnfV)-qjrtl{uv2b4XE!NoY%p?~MceM;s$=-WOU?oLIF( zk)=uZ1mh--_61zmiL^wa&Kb!E z*PLin`ll?%!lbgny3P2TF+-E)gCwV;yLiI+&~C?xoV@z^i?}5e4Ei1Z*L@B6_tKRo>*ci#XPu3MH*ICTW+d8kjj6L& z@Ju7?WG2}+2iUJ1@&9r_N`qO&z;T^=qq&unDGQUHgOh>YtGFCsDA>(>;()A1lhKVsax0pQJq{;|KePGO$os&_IOHG?50jRM zv;37t-36QEA26Apm~6S{pwOHqlLHO*t_~3m4r&QCVpkg2%Y%4z3f{f@cl}|{4bxvI zxgV50YFN^|h@(Z%eZtpTmFrRN_2;tRh` z6KThF!8f! zXk1uukflvn#!ulu0i$!5kh;!^9sdd$8GEH{g%mbiOl9j0xV4Uo6W!}lcCqB0{9nNpayzKU(d1Z9oFN1n<~+ekF1@pNX6E;K zR6TQIY~kSLC{Rc_b!)1WTh^Bi3pG#l%h^`kd0O~HPgudWfng$(i)X@N$A%vaO%@)H zWu40I?D)8dm0LtfWk=%UwBD)bvMs_F20b;-yqG<$BW}~Iu1T8SQ&a+#T)9Mdn&j+Q zuyM8<&;1?AB?}aj+4wD2c)50cSw3@9$c{dpgRQTwOp-E6l9~8ejZ@gcx2xmP(Nj~k zKRvxTJ^kpa%gdD%}nV#>0a92*acL~|6n2&#SXcqU@?#GrM8 z=bW9a)6-u}csMb=DuaQML#^UKvyackW!%y^7glfF{pi%|4uuIjRA%L4T-GwnF5d49Kt&u zo|nkFkjN%+r$I%?qGrWn5zlD>ifyV79GuuBDpo9HGxvCz%;^!7nKU&pL-PEduVQWoay-J=U7_VT(Kum(@=l8>D_Q!2)iy4f=@>HM$TX{j35?Cg42{j~Dm|-|`fFtu zDzkYkVmzetS7O3}e$QnmRo1EHK2YS?^Xb%Uy?qlH7Cq`>e;kx7Z}F_6T}GvNp^=!I zhhvNBhJ@ta^Y>y7n`p(z>6&Uyo6+>lDlyrKQ+2`$-!9o1OB6Ze%Q%dEe4jZrajNyL zSZE;|6XD2dr4-4;CR#B0p_N3M#Uq{ou|nlZL7tMW-2OI_{i0j1XD=1j==$)0iJ$LK z2Lq?njcF&?lsZ=I@ATs``*1*VQ9_eqN<K2UvuDHfLBY7s>m!Iulw#pd?6lNuM22XKv z%VjvCsWowq@)8DSi4}}8Aq*UuYZ5DebvCd(GvJJISwE?wa>mPb0-Mf8Mgs{K zlYdVZv}7?Taa?@CDE70MrPw5y%|=t}g-D6KsL5f;D;}+!-WPd1UMR3hd~o1r_}E~t z;#|7C;FzrcLJso~C&4+d9N#c(WOA@r%zOKi28+ssR%f5X{2m=C{Et3ucYFs{$!9yu-CC7d1h|_Ej+y;<#S^DbO4+Im3~xq_BF5<9;kR@0-fBc|n%y^)k+VT8$>i~1uf(Xm1*Q$(Ce(} z5ZI+A(I~QGLNl9=0HdMxfvpofG8C2wcDjCO*uH1$3FfXZ4HFr@J9xMVNT>?vdYovK z%Dcc|`srT+qf+S=MymyF#%T{2desgn9!X+g7E^ep8lvQK!qG`nD{!`&*C7#!56y2A zJ}_HMdB|b4z(_@FV~h8lLt=(g8ReERwr}z{pu>0KkjlJ+AuR5b*zFuXe0?;@*5{p1 zawf;2FyY`t`6mfJFRUkXJb02g<7nrtja!8OWogc4IF^>~y<=(4uLKdR9gN)R8Ha>s z6|g9uIlvJV`%!Mr#U9ruQ$-eWxa#fu*qa&QEIsqX+4%_!%@R8r`6paB#O_znB0T58 z2A>lLQUev+xfQwIK=cGY+x&u1*o~QTKckE9Bj{aaD`0 zr=2`Uksc+QvSQvW{oeDrv5Q*g?KJjQ#9LOu^1dF(diap}F% zH8XIU{NMpkx`WfPb^#ZSkcIgr7R=(o9~`cvEMSsj`NJ%);B-K{f=Ob=0T#&_jtcVx z+oP`jRA7DKpc%5UqbPt`)KufZL%tA|heB5ro9{_G*ObaDg+5`GxNtyd&c=@PDNJl< z3s{~DC@?ZuIIvzUVB;4F2>G|DeoBO3L4&kgNUwABBf;f6m=)z5h3>3oJ+V%xhMj-L z0hXu(f_jN9*BtnwjtI&)vi)+9TElw4<-v+PUhz2#*?t^iD`QyYvSFp(%2f>ys&_oO zA>sV~%yOeMDy&`&b#8|bFKd{Zwv0wJqWFlLtbR7V=LyDex_U|3`vU(L+hThmtalY_Bp|lk~qOC@`<#V~cY5 z9;Cp0PJoT+pya;>W*&zpe2*0R_{EdXP58i&ui$h}A!)nJA%5nKDp9YMxVj>O4hT+L zz@M{_-{uM5B1gfprz$}W8q*rox2#uG_{UrF?y$2Mqv#UHFS8DcePH1Ga6r&!p=8zr zzFCSw?;0-Mc+HowN<=4-_nktiT%y1|1)+J20(^yM)+7k!^jx<&x>8H?-LrKDGcK;` z(_Jl*yjn+xo#iNNOarUOGowpdX;Gchu`5mQd3~&!kiP24v>6PX3i+Hz)c6)m;NHc+ zbHG8sf|19fQP|2+q8xZb6TbUA zV38;awmQI@uE4kApn#Qwl7_-|83pEl3q%$%adR}rwjBslaAI;}VyNSA6l-CUTco75 z=!w=N29<|Gs}6C@aFE*Znms0uiN%4z;DA`07vsM-&WvgYE*ddC*ILNs?I`K7UefQN z;HwAx3=8=aK5A+&WG!jn6gVg}X<^@+j{h=@4AQG4moW05@L@7J#kkI4Wx>HGa-O2bY!W^9C>Xw+8Eb4|s1Z;A>L&b|Qea;yqi` z0;8&ao@@rTNe+CE1Vj@aJdVm{I8BfM>-tx{!UVjR7ntc zog7_qT9N0tVvGX+ABRe#2QSVXV9{w}kYQjpdLT0IAg=_|zZVisjC=LL#8w-)ffbKtwu`hTAv!`efF1`Bk)9A;dVAXIWhVlm6R ztOqMq+>&MldVec2{1(J&?Z#2#V3hYj>c1Xq)XEQOZCnW=ZtHp7veV4Uj>%L8WHvlt z57Xhib3kB^0@r~9#?_7DbqhTz6+JT-3U?`RdAt{Vki>beke@A4;DJNKlm@O02i6@S ztQHJx5erxh7O?F3q8QD?JE zPYMbhjR;pSydJ=*fg*!{R{l<)NoD4iDia* z{t*W8a(~u$55(SeT-H()nv@v2((mFS4nIzTH*$$xa*RTLj6$;(N(y)`K9|JB_t5Ea zlC!zUszoYEa&HdWnXGo1V)*Zqi*oanxCgA~Qf|5En65E(6BJ&zqnEvYL-a-#*@*|- z|0f78V&Jm*EPmmuc#vvnMWXPmEdnbJirh=!y0d`m(Sq^=Nvv}2tT6}t4H8(>8ki-n z`aE07Y;k}qPw`X$qln9T{;<_7QU`o|8VXYm{Fsr#VbhRzsVUE*A@9X92B~%is|D;f z2^jUGag>D?wdpfli5nkjz5<4-5DnEZ{v6 z#`$4^W3nlkaA)-nAC1sCz;LsHICz9O!nB5Z*VroM9b+xO8l z$L-@bQ}>e}<9-cYz-Fb;VAa52b09%-D|62QF|LCGT#Vcy3#C}L_@st089ZRBdCX*YfURu- z^NI#un+Dz+2iTS|@P{#E+9Zj9>y?ABnxpiq1fhEiCAWO9&QBByd+<-} z0wcegq7(;XTKXZUs}E%-Fz|oTl43|yuSm6@Va6!N!1&>C=t^e(I}PbG7O&6B?Lx4zTDjurY-=y!0d9$IxIx zZpJKAqaudGF#@b!2mEem`FZF}lh|yS-S)tA8kD}gKTq0FlTE_x3IZygjda+LN_Zo}=^z zrIT|19?BlME?vZXGUtJ0QLfOLL?J7&>+=-Fa~?`gSit3FDaCg1Ut5dgiHz_G-x$Og zn8Y49=qL(>oo12q9NQfz3F@drw^~ds|L<%3T$2s%O999IUQikc_45^ z$;)P`o~yvYo~`=ZS|*2Vi2SKHC8#8Yf&XLN$y;fi^;>$QYbUVxH?a2{IMCz3xuL;m zI|EnRLb0ravhym$Tn=(tG>WA$axGHeyYrrR!UNt_2iUR_Sgj7QCQUnBu#oSW!=@w! z;VEa>k`Azz8QZ2Oa9+#hba*IP;3(>IMCymGOkU%rd5W?sjN%s_3Z5|6zQri6#wdGe zA>Tbm8KXq0e@Dut_AFeqHc{x$1F>fcD>)pw)EWi8H3;w}vQC=CwyuH8q>*{cb*J}J z1UM48yd1wUI5L}M^|-M!xjbO#Xo;Al$|%yXuOgvw)(OFX4E%Ktf(@1GH;kC}Jayc} zc=8gX?5%~8B@2YK9M7Ct#JJ3|&$d8MnRB*q&q0He26YD}Z|O3e@^-byN!IuTmIH>~ z*;CRUX_+lsz+S(DJv`^ARL)VG^BnJ%a&2-DUG#uEYN1%vLt(QzZUQ{B$$UaFw6X6wRpg)^!2OJHRd>mwBQ79*#{iI8Xog36q>^*8WSyk zZ=viFMVWby!ZruQ<~;o8H-90YoT9eYEy*vQ9n3b_taJNqI5q9)1;L`L zyz|1>aBXQ2*`~l{wNNbLps*by*MSAXX&P-i8aSRPxIBBnd`H5XE0xE38k^MuW}!{Y zGo*JeR!e*C%%-9!D%U6yz$otY@1ek!2ZCja;zf$WuNn@#yv?@l0hgTu+ct;A6WLfa zRx_IDh7 zF3h%{!Ok3XfK?&JKA7=XqKvH1s(mI7j427cpA`6H5(O?W-g5jSS(XsSxjWzLfq+`0 z2#?OBbu+J9xw6zRNZPg2;M=8>!i(>nWKRhaVPCd~Wm8CD^yAIfBeyhLxQCx9+`s1_ zlLj}}LhgAD!c~d3cm4=REEKbwEVjTw-kVYU8v~0~gLmq*1Fv5$e7!PW-imF4^^=Lw z=eQC$pETUn2p9d)u=0J0(68O|91bw&?fdpGLH0oTJ?0rts&W}vW~f{!jCI=F!)gzCzb!CPWMuP5_qb=ws3EgN`JIXQj1)b|U3%@J)k?Pw zxRO7bA3J$;LGl6__PmBR-NmPJP1bQ~#2goNzdiHRfv;OC59s;Lp7*>#grkw;Ktk?? zWZQ~`;#CJ9&1T?MaC~g55XYkTB=o^$@kyRiQWvc6C8UP&6hC6xlfgWxK}2C8*OCWZ zCJ(vRDI`z+KC9;dN5=!^Z3j5)9JseJ{O&!#JjsE(#_r#FM+NSJIIbjz7s(5lZ1SIH zpJ%g6;OJrK|7yd`GOf_-fYizZg5MnG%xV0?pv1_+!YkpB!SGPQi9=P!vU$mxpr!Up zN?x8S)d*>b@)$IFT_$n>;*R*q4aOTz1uIa0fc2r*M3VPzB7~N#EBcRc; z?YyXV%#9h&t?X0f40A3#IN-?It~iHNU`K$0GrRJ{e=?mP8=HCmNoyxnbZlg25!7{Y z$jJP(uzMP>+qN8qO{^>({8w@tB$y7hwZFM)|7Y7r#kMJLJ#tDME-^B*ajQn0C=hHp zx7c>&oTZNnKP))FCM@LF!O+OrXTt2EddguFhkUz?l1KCA2THBd6JI1S%AH*zbW*AQ z!KPl*%P*36&7VgEIm?~wU{GM;tN3t2cjuXh!T#1a4qOf}mHX<4F4GUMG_j> z+$@YZ_%+uUv@pu?7`Cv>#Vlmy@OAmfC>SVlu|;sBa8#vs#iJfwAB_nCj+}-Dj?5fqTiT7! zX#{)8)Rs&~29Xokh(|g+0=oDmy|B_nBVLxy);I{=vdSvS*hl9Asgub~vHC z zw!mZKVJ?B&3vMl3vGV{Uqln0XgRJ5bAKR5pR}^S4&Z)9$;cdIJYK~xa$;U>qrW+TI zDxQ>0auILwNck6)39RcQQv?r-hTSk&%A=K_(8!^BN1>U=Yl_o+ z;~g>G0?bT`2O1pjxdklTaZw{VNMm=*67JpdA%dL3%sLN!J~S@4zD+dsw~w@)68Px#j>o~$##c8^aLFoFnWU%F$;4H#_Rh7w0@c!R zelu-XaiOSJOE)WDNno>CeNgC?0%y9+L5_HiqgpN(S(F4ALwzmeRL*T+x5`Xno%v9K zH>=^_R=I#iHbV_XcB=<#`1Uj~^K&qXo(gMVieX?jc)_f4ZyOU2&qGEzkJ;=m3OaWC zF5<3MI4ZZzfK!)2JG}Zt6Ysi&)psk>S?_(EBDym9sP9|@TbYJN9u)yb9-Wu0dLB$# zzZ$sH7dUZpMszqyIZSBLc)-$Ka7^2kxSnoZ48HaSVImR|h7gjAMKapZidd?~bR zO7pr057=}pB)KOo+O+Gz`FgX--ADTtiGTd7KZEJXO{VMw7A>Abl4di!6$%);ln*el z87m|xOxf7+&-ja?=;RqpJV6Z2qCZ@?Y&829q!?_KC`n+It8nCRH(;!=dc`REgIAWV zu!XP0ft7ax1AE+sMt%=Rt}BL3R<*&d{3Q+x!>%;2>rHU=FJxf0tzlpbxa`bV@nF#k z$6l_o4-A|gUs&}g9F!_B)V-?fsKBGZA)|hPA>Yw~QK!R2GwlFNss^*jk{!qN>UMQ! zKVf3=X=srYNoewLQQ|9gaktnb$d^~aEW0hjnN#N`6OV}k6K@5BY>W&0ho34OuAK8V z{UhzdR*BX+%<7WZFiBMLi35k{s?~%bNZ5XlIa2-Oex~QEItkjTp9;u{RBH)HJU}1&Nym%L-5LiS4y1T9Y-W} z5*XEV771}Wx10UT=FMHeELw4)QF6+~CXS8;3H&db4%z(_4!q`(^vJm@+9+(=7gkQ~ z*WIa;4m2^zENDpeycpQ|^Ok+LVyZ?RXS{!04U6oVR5mM*CK0uXtjtD=d`k^nO3RX& zbw!#)A`D!$UKsdFe);FXRXT$?$BP{;GZfzmNIlTtkJ!Ro%+Vrfx8OYYo^SJI zA{y;1J8pHBnVij@x1#FG8o7yl;qBs64{!$=TohU1cHTr_B}bcso8*B7l8phZdh;H# zvCF9RaYQ!w2rpoeyfG!Fu!BYOfi-vR33E>O4i`zU1q|F*4)7Qiw3r?Gc>jNuGQWXA zqvWKC%`+P&GL$N~>Tx{W8}Khh!%jlJZ_cJL!;CY@`G04Gz3{rezTpSUE*7Uw-L|Co zzVjNo_h!arEosh4pUv*EYcy~ zz9Kty9P+rXe7>{s@9+EiHyU{=61p5*oMH+dWcl3JxYw${z@+toQJ7_~ds&3bWhc`^ zf+B%zZki5^e-fFEgVypEW+V$`2OQ*2=w*x)U%)qc$5G`Z5wA_Veq_ZQn(=c=02^Pm zkH>~T8#<0Ynil_WlJ|wao40oEE0s-{&2E*@#P_1EMe|3ZP&5OJ%8vu=K@uj`7K$xe zN{y`^dv>!eeRxpB>GtfqONuxD-Zt6oqfP8}0f*BoW*$1|&Cy)P6q99Zve~WjbU<&^ z#^9(L<70*kW?g?KU@Cs}v#a4@NxqIL%>R@$esW(6;Ifxc>a8tk@Ja3x$d^=6)C$_d zWFo*B(2-njzqL(ogF&k;lXLE#FY2`BGZw=FREM76J_kg-lA@%9cG5cNfe(AQENvDall!+$w?j)-_?P z3%Ml}PP{DS~))j;(t&O%%LqFba43!Ku*G(Ihh-oMcQ z$WwoP2dlgQuBZ>}8WY*C2MhBSFc@8Bxc^lC9|J?C-~z_R1dcZccom9xcNy^h_`;qm zz<4ZNSa<=a`*$9_MZCdZ`DzuIUmmX!3Sjt9&OOnfQBcfF@o|!clbzQF5qpCi$Do{o zr_7cKDvT0h8XHu!u7%es=5l!?K2(n2zaiY@#B3DcTeXp`cmkJf0gLwo*0>Fva}>BX zA7J%YkSV^%lIX!^Dl;7g1bgqYnNIJitXyc3w~9p zUzxEdQPPBIq52Xj@lXb)gutxDOlA%otqs$pr%Q!|Fb6kixjC?S3(VMh$^7(1mg)zL zW(REL4LCzTFjgB_))}xHFJP_vz-ZdQx-^jW*O3ZNb>`G-l0l2pZ60vsDX=enz|+^j z=rUcP=bF{+pa~O}F%*0-nZz)=n1O5M1Xkk)_KFSdF``Ung{=QfE=-*{f&2Ud-j@$J z3=%kVH_S@7!VqG>xy^yAe@3a3Vua{tW*a7z>z<+^jiUdbGZ%@9nHMmXwK5%QnHL_M z`|_l)^#b*GCz(II6rQ!6hw08hw{KW)>7{E_QYDFyIJY5OJ1)Q_Z2onBAI-k#T+Ttm}^%3LbE* zXy9FPfqjVq@68EZJ{Q<)6BtbbZM7NLZ4G!QJCvS(z;pS-vS|(Mss;aM$C$E)PUo8W zO4v@J{NZEf^rIr)hB-D(ljRnRZdCDB5Sw?fWfN1&ynDiP7rKbp6c!mdF!!r6TRBwD z363ucV2N|!Ud+XB&&|JR1Lqb6E|(9I3kq~>Q<#kpF)zCq_~we_vWr_dpQp)OV9xo; zAXyM-yn%hv1!lPgOuw&4?)}L4)KQi-mF4n6+4SFx(i<4%4OrYOTK0croW+{v8qqRm zp_J@~1v5@MI!<8EVPG$1;Alu-ve{5HVFHu0_)^KA>uwg!Vt>exYrs*Tz}4))UiW}w zLI77$1A7-kmVf}0A_JrDg!V}R>^T#5wme|3*uXNqfL+x<_)R;DWg@qYu$R)m#|=`a znd_xk8oK7*x@nllYE*bt)cgVyi$>D(ox9IU&()v3dqT&Qki(It(acMWyW0df{1m3z za`V4iP#k}Nb+ZGvzXMAi1J}|7)_G0&dJawjxBXsk$**CGm1~IEB?MnK%@$EC9Y<60oUXwH6gEbjhDCme5jkffurI9=q~x}1ijh`?0pM3)D41{oEE5D zz|qja-tvH>;sg8tD9%DIMlDH3$AsN8e>vVfCR}wXTYT}{Ow$z(hZ2>&Hzfu({9E*6 z1^MLeu*)1^s}x|#U%;VxfVsgys!xE$h=ING0HawuqgNZ_m9MV6 z6PV5&)6fdwJj=j*qe*w^&myM42^ygQ5bH z6u;guE^Vz_b7Ql4%^AW)S=FoO-cZ0?-Wh2Se8!eeu*eo(UWVS~pIp^NX zf1)Q=?=gF{$y6xOCSc`G@0Etud*xYW;mX*nZROmVC{;DOa%*=rmm5zk!EiUkP9+kuS{UIe!%3h zfJr=nx#EIBL4k(y5yqeeOhy+@@^4_vIl$=Uz*>5WP51$mR{+y~b78#;OuP@6Bor7{ zq%P#X&X8vt%ioaN_l$vQ0)y6pB}M_ZoCd`=ncVwt%ctIAU=m=oWMH;bVAg$Lsk?w# z?EsS+gTmtMoBnH9(jKt+EMVk1z+Ca*ti*vjV?jpk3rgn`SQmvdOT13nn5+u zgo{jDOPD-v&igk<)a$Kgxmx+&^%X1Q;`ScX)p+;&;?jebMGRaD4wv>E;9m29YmEc< z-Ur!8gW1!$QY$YqJ`72%N^9n%MG0C397`VXRc14@N{GGt*ris! zQnYuwwrv02FM zdrrCGlo#7y=F|y4>3Wgji3O|yX!b67R+RNP`5O%hHpbu#I=Xq8~(1;`?7f5tIhWqcqj72 zKGhW#C|;t+n6sbl_#55y9SkST1%4(zGCs<3E`iNmfVW@*W0wQ(Nd^Hc5sMB6N5_Lr z-BadvEY|Ux<65*Na*}{!QwN6{pRZHu(zZS+ld=~N0xz}q$=O#iFn)68Ic?;SbZ$zd zTjz|~=CQ9H%sl*@Q^qm%!Ue^}F9H{Pc3+wh=zJ`Ang3F+wG)%w_#_PNa;6#`6_nGd zbyz8+x++-RAfQ7clgWLeQOMFKB`+QvWM=34r}e4m#e#6wSkZdxH5X<-I@ruD@uF(S z1eMTLJmPX95=xB?EK>bih8tN{^7k+?%9vz4I8gGEjYUl-;B1v+^Ysm$`ryTg`ZD%mwTk9ho`yE#^;Gkonc0lI|0ree_7ANqU}=O~V5Q zCl*0=QP$R@o+!eyfJ zzXMArIJ;FcwW#O(aOgJv)xjbfqi|4(jXz{Z_6EKW|G08Ch%S2L%__;XxsjbOVZozb z(|a=x8irWyaA-O_quQa#TJGgSes`sdJC(LxuYU>@^35-A$fUB!|woMO{r=uGLgS6R8ZFJ{YzgU$R)E-dQzl51Qd z67pz8qM>^5fo|sJpQ|QqXbpeS$m{T7#lo&&rI*Z|76k`c+m8I>JM6&3E*EiFNY1Ds zu|GYmpn>7$DQjQCwyBvqMj39;sX-}mp4lr^*=y=A*&LF|d#5Sw# zT#)y|)p@}jMIOg`l}{D6aJtS~cx7Vok2WIh@3Cq~US)7~<<$P*B${gS=Ty-p8B3w%%WyPMN)3FT;TgB%6 za=0tCVu!MjIgjDdF7qJ9qip+fRy>;KlQAP%AVk6NXuoz&#KXpB{(T9pQWG8=XqH>m zz}#mwBjAys`LhBwAuGEBhs4q=3LK0$H`OaJtEvyeF;{EK}afn&ZVNpy8;hlh~Q@;sBFc zhO_FKgq2@%S1E96Ft+*?FtJ7~V3ps(?^gGnFdCV4}q-i8(b7*KDIlU9G>)#LE$XR znqaSKf?bxY_S`JWd)&aVsnJhlhQneGg$5=O24({Wg{*|lQfw}~Ow1{_SOgU2aD)U1 zrYf9JZEj$^-*F&Fn1hi;?SVF{0t3@k9mcCQPqn>;9*O;%&FJxM8M8dci+uY4r_E(Q zoK^Q2&*N_j6X>k)uesIO8FccX(D$9^RkPk{K5c5`UKF9H6Lzs{uJLuQD_hwtC!7=T zv})G!N^G*JJHU4KM2iSVV3)0c6LJj>uATr^ei4U$nH7hXCloaD zlsGVZY9y;Dz2vcsxx^&skeQkAfI~&+q1d?wW=5F}3|4cO2O6(_8j>4))$OT1 zsTUFs8wCj+XJXypz^b*NkvB%c+(>x=Bm0gy$yt{bI2UBEj!In-7O2ZIZ_%w~5gQx? zGm2TbK7=~DC@?tm81wnwJXrs0o4K5c_a>vnbU%#{HnlS;PEJSEW!@CDpI3XtS3aTZ z3Dd9h^?HR}tCuQ?nl2A$)e2$_)wwQsO`ua*$8pkx09OU0fJViN0}IU}nngZ2bcIY% z5-B!linS{sNgRyzEjVKG&*7R7&s8R8@17{f zWBY=v5||b@=Kp(>J^Ra2elu>Kg-l!m2evIstG#{OL2%cDR{e^F+))Njtk<*~*f|{5 zvETSNCD*LAA#0KUVb+DS3f9~V7f3#0r|7_OQ|1AC^c^N{(+`X(m4_xPzj(mz9x%^$ z$<@OOb{Xv-O%6+*xzKjLFYH9Q1B=p(7w0cCeL2B?b&g~F?I&7XB5Yi}jEM;ew;dH8 zGJ9&c=!INlRcc^h`LW7T?$d!b_XSGg*?dRkas-;My;T(D$vY(Xr=Y`U#RVa=6^F}I z4LdAuI0>!#(JaXG(PmNR0VM_Fvlr%M#s9jNx$kH9e0|+H_RTz{srzgEvUnUC*ee!r z)-be+n{DLOUsA;%n{kL;%bHR*4p+g=aiMgvkmKyh4~)-_bQxsK{MBL z0q5dZ4V)979{eY$b%Z5D_QfW)gyPbRKXBdO7DxzySsY1txiygZX+nkGNwzK8OqPJmcqS2;KCLx8vwB znN<&O1s__OmT1cHM!-Yi&GjY0vMP<-P7+Tyo|;yevaok@)h~m2mT}Dzmlm|h%|6KM z{B4`kA3sjlfJa-Fzn#PV&+0mp_>SsR8`eI`kXl(HfJwjQBGL6 zE5hK2Nx;TWQ=?P%ReJGi&wR|lIYVgQmN%DKGG5NHOWp6!^yK~gUq=|2L=G^>6*#Th z$-t^%@sRsfxz*+0Z&`I47_QlFzf!PZvG2~P%ZGQceZ0~1He2w4Ro1R$O%2VS8(Q53 z1SVZ*l=#iaR%6C;u#@wqj^Zk7g#+8Ft(tQ`?>?}&#p}ZXm29Q*)01>O_Y@x%Dxbaf zSn`bH+&tnuJaQgPmN%FMGMMxPSmPf2YmMjFd*O4F)Et&zgI4DU&3ZSOb$>9#25Iwu zXtrI^q%*^@u&UX82dmbKCi8$sNrsjnjaG*POiwnoW_q+9kyz!rTJ`55$=|1Tdfv9Y zmT-8D)RgNx3~nr7V0$6VWYECX!BlT;uU^pfrJHGav6Z@oDQk8UZ^BBRED`2c_OXvw zuYJ9GgR&uqfHkLpS&sl?{LBW9ie~0HEL*+R=Or{rR-4Jp+P(9z&Mx6*Gl@lJ3~TeU zTMqm_;I)D!c?XMX_NGE@o0HK8MM`YqW40f-u&Bbj$#TI#jTbiY9>-GNv;;l?jXw{}t`9i)ImxXb!4qbGKl!Y+#95!4}=n zICV0EQ4Pb(&HA$q51a1MuUld=clCy=93os24O|LLfj3%~c<=w3Z7N~WsOGW3x1mY@ z0W(YW)}=2^SYL@OyLf^x`AF6-4sV&+J~}!kn_E2+4CiTZZu`s_nB&RC*d!yQBp-ED zao2L|1I)&!S-cLkczLiTd$27qVN+MyqqAy`)+HrXt35&?jp81A#1l@56dV+<@De@2 zDBQrPvS5eqkETcuxAF$|ii-BghNeh~R{aUh4iPL*qBh5L%nNYld9ka3`$vQMjHcZY zjamnoRRfy58X6ZDv3cBR;Ll(RnZcGiqtW}t=}%jz?;N=xeB(c#1)dGBA{wl08a5f7 zSQ2a>o6zpCp+TsFU&!GsuY_q}gw?X-l~Gwz%^UXjrX1;gvdeenvfNWaiLZUVd3I}7 zusBLjU`=t5RCb+xV$G}{inGp`DP1~XeS+C1fvu>6y?Vvv>J`D&9_>ZJN_KC9Rc9?) zqT^LES!vmtMJr4Ee|+(lJJDp@(WEzn_5A17ScYc34yG@^TSGZmLIPNW4m8;wXwv!6 zY`#KC^2MZzZbrF`i4vWR<_#^T0gU@wt{Qu=&ObFHYlaH|c0E&KQTZ2ZGZf*;O$e>r<^kD98*f#?dwNf{2Q+7p->1vn<2oEdYp zdQKD|`J9HhvFA!2@AOk2G-pY6^5X;hWLKq_E)Cgw?AqpZ&+j z&B(u6O(*q^&eKZ_M~-kjX!I;lQFZW>d>tNGvyJ2A8a7MU&^0H`erz;zVDaf-vpUP( zUVEeVLHo74?F$8%BqziwE?^Yj5Nm(-Zt?5O%XlJNv$mgYYfq@vAMgMs%z!;uGz7aI05bX{C0+xzEi{6fxX#&ui%gf<9h@n7m-;FW0LXK3wc zVaod29%G?BHOPnWfD4a8Q((t6-xp2134FeR4FVqFcVez<-n}&8&93sP*YB~2$yKn# z?D*HJT5zD`(e=PHYnWzkQ;2Fd%4mL}vdG$i*<}GsX2gxQ)VsAFNvii4rB>WktK+)W z+I7=*)5kNh+OxbgXL*%HFRED0sA=G&@X{tcp*8FSOW1{ju^*ajJ6Os~*q%tW%-!0g z6~U~l!|LA9tZBidIiXo+0!z99t9L+K+5u)+ht{vWEfzOI(r2)FTxi%Vb@*$L{z9j~ zg@+n=IvRL98o50f`6o04`7%l#Xi)WPlzky1AJE8pM8~+I*(iZobwjgpMw4Vm*q6_& z^$qNfD^!GkNKEx&6wzShSndgvK-V+EtM0QTw&>?&bN<#|cVR@}YO%XM=q*G*Ob zyIYmSU))`sV=mco(Dp=A&10tXrw!^`-aD=4T8!S zG^`G=rq5u}n!s#+ph0y-OZtmudk*K28*HHlO?whEQ&%!d1vi-nFqtx{F(Yg0S;x2)t$_@!F*DrK72-n-T5TsZX}{>QJiz3j!1{u7rK3fQ z(+Q^Z4Xhq38hK7Ih@NP$%Xs11a6fCu)#EpuUT$mAwP9V(#-CZymVV-0#*23u3)ns! z$Xj0aDkb8TheqoHKQ`Bb7LOCm?kAe9Uo;zYv?vuMe7W2j81R_Oqlbs#wa|qyu^kNh z3s^NCm@98M&eFvov_j4Q=IUd2MdA-%(rJA?VM(~yyR$+OjhZH{`R}sly(kFWqf;n$ z&ZJ)w*0ZmBPRvb^ecv(yYG zyNK4%gtL(Xt)T*}b~h#}7qab8X!W#ck@&{!@Icd}gT-(bTkM6l=nG6S0mqeBFiN~& z*ncG?=*8OaZ!h@jvIZ3td#q^DEot%kk@@aK;Drn$|n9RY^$a8~%z3)U|M37^Kfz$mkx9bcE`xqP-uDzji)$!`@HPRyJ|jvranx0=q%3?ZO|&GarsOtlkAVQkS})xmwUeS$vNuQQn;Y~ zYHh5zf09Ihp@ev+8Yb%GNvcnCFAGOp6ws8H|<_KG`a;=3HotS@G6s2W#+(X7dgf%OA|< z2bhB*SV9%Cf6%|Yf#LFnhC33oE?Mwe@0|R|puazU z^|sK29Z#EmAFybg$k%w$q-UUQ_bbb7d%VC2)w9VjAqXht^d?!FzIA8sWo({Mzq)!+<*Sl*nUG>Ut5L_ zN0yaB%R@aDryER8Ggz+7ZVCR-=n)a+F{9DWfi<fgAbj`LeI*FvVL~d6fpOQ*NBM`_hiG z4=?A}We3IF?|F82&cBM@aGr`5g+FQAewTi_zc7CK$(KHFgv=vcJkpQnm7Q4DWD>!w zwSYxI5{iuO6jpZ+o5>4wWp0ccE^o{EPM-;&DQ)l_;eZP zG~;wJT_%>~HffJ-N{mV=$$Zk<#Th=8CM&#pB;5Wz6S%eMdB8NSiYXcz6WK5D@Jim% zIT_I4+{GbbU@`SWLQ^wGm}0`&TMSMuYa^5`8ZIozYUUIY^NF~ypz-QjAsGt}DJ2#b zK_)GehlkUJrv%P+`*`apD@zBLw26$KkynAZeqftT#sY;FCZR~hGpP-h#%Y3zHud-Z zSRP@ypzJrFuP~~G#eJFI&Wj5lxAUo49l5hkcNx#*yJ<&eaLV23lYCVE#w5vYz5M># zbMu|+&$ao^Hp^ZSz$tXZO+e9l-{l{A1@1qZSUII+G@RQH%jn&C&+w`3!W6!UXE!E2 zR-2?1v~o%y^GT()C69RYCU`7jVwV#*;KuJ^^XY`NY{&t|e?oON500yt+AMA2)S6P* z!X+EC;2^75LBl~-O|K7ALijBlL?a3!uU=yjXn7jSGJWdN11$DN2bx%fmppJ{ao>{I z##!tbaDYL&=3u8rfP-PT`mR5Zx-}AP99;xmYZOoEEiQQIDsC9!&=4Nk+HfEy|B$5m zuBR(?7%CrDaahNvUlw4;<#-@-VDTV(-+88Ip&1!($4|?i@T}^wmIo z?y_@F7rM(<1srD+warLx7I19X?#%mSY1F*c_bw_pui1ESgR_grnui`iJ_!@M#EdUo zbeHebU^=0+bVXCQ@i`ZvPG0ef2af!PUkpwt6qO`6vK!hwa+K@$c=(Tjtzcz=n`TMI zg~mYHgaeM<5)lcF%wjel7}!EOMZ;Lk-|g0Cv46KCIW)refikn8hl3_}frV1D@=1@! z9rG;00vZKt3O=z;omUamtuOyRh+B6{isxyym=6nE*$Y}rXJ+J`d%7&6uxrE2bS0k; z%hFdqwRFm!m)jHdr#SPoI$ku!a?bZF9k;=r&=^~X--sm$jC0M zkl4)I?Bm4FBbA|WSXm~-kW)s+;=n;RtqXVE*(7Tmk{I9YlU~pISjpxwOG4wnt-toi z{yaVPw0Ygs-=?6}CIs@Wmx$k^t3uw6XH;o%YW#T9S5t>1ARKBuo%zwxBF z+mYK>qax0_GQ}F6wiM5>ydLv;_UflI>&^=@D0QlCyx6v4k(I`(FM0h;>N17D?+3oL z`W|vN>-A zFXn%82{ZI0_$z1sa8b80G>VFNDID;i>1*ExM%IW{t(X)B&IGZ2A1c|5OErD7+!oB5 zw=j*Vazl%;h~w;AGK%J@7kU>fh-yCG;>aKLTi$+_@&U%QMGcXw50`9OEGyM=wsBL+ z1D)f+0-K&DuscQ^mcKWF@km1__nZeT5{np`lvtQV&jqv@uL|r6;(5d`D0M{nl>^7j zn9qXk5=W#f95}AuapI0va1v(O*sgctpxC_~tt>NsFmOv8E&15QbvlD(!_~JPWw9kU z!yjDWxoP3dCN{y@_*8({nk|KK(hZN}emuJ1+wgGV>kZFiFf{U^BSD zxcAk~U8~-7>zXf$J&;m9Q$C`J(`P|jhf*Pzs6dZ4=K%*soehn9BHpK{KX#gFwvp4W zX5k{{4F~#Oe9~ttFxlSjqLNv)dY1LWO#kJIjT|jXoLMT#W**wN)WnyrnB~-EF{!Cp z_RE6Hr4JTQu`BIU-lhHTN=Tg+cX+@8feEj!Y}(p2?a9MNX5~%Hav_a;5=u?NE`~;N zJvky5YZ9H81F>vd=yiz|SgqlgB}2i;!|9Gls^g|<5!RX*1h=4|muq`sk1 ztiZ5S@6AE(1s7cP{v>pIRwx{iFmUvoCdux7!%1pd#v|q_1)K_Njxy#%ZsW+1VeH}a zS^O)BNo&c%4#oouBntw*h2?D#WerJSU}k95n)jzw|A3fC)QT5VbQd(R=`iT39C*7Y z_q6S--jEs<4c|<=#~zaxrLkz}HlDv>z?mL$$ZYl--oI&4!OL9DEy@iJ%dR=iQ<)-u zo@oxtal?-s?gCCLs&+U@?P1gYd?QGKCGclOis8oPq#Fl9{Y z5V)Wa$mOv_Nwhm)%@lu!bG9;ptRANYJPxoP=wSUP!>HxZ#CG6-NJNv$iVo%z4ysoU zMno*QJGGVB#(`x^qjC>-%mvw=Dc2Y}yl35<8OSqip<1-K!@p=*Eg`-hm2Lu0y22q|vVlIs8IiiM* zjN&zibb6dVE|_n8!ci#Upt8pyX$40in-1QH7QqZhA&G-^E{Awh7OxMRv)NegEgg9Y*?&C&%8tRf6T5lnIiIAVHO!+VbYV>)nD(!z;z3hVu~N10g; zFo#5PSvX0XFtBkr^sMQ;Keb`f90nDJLy|I#JQ|mC4qom%GUK@+1J{Cv)IFSqdp!lX z7II`fR_;7szMRkW8PH*=F4{v zsTR!nz!)m$!@g>b%ooWP9*!374R;*oJan8A&cETHg2%zd7n{^en&es?kH5ICb7Iye zo5Q~)om3p0gswR7)*Rqj?sEnk`A)gFt7_Wu^q|r#Gku9WA%f^XkMjzLcgWEKvBR|Y#5nzZhaFk2vV^h2RaO=NCo3#dq2M9hkX9g8gWxkeHdfT@>%-x5@T=^8;$RL|LSqjg~ksb9a(@ z;;5k_eanIW%yFq22az5o?TSO{2h_AbIO%72icfGd4RPlB>!`Ir_V?FD(LIi$J_qG^ zn&O{2HH$KF&un0w;lQSHP%(y4=%J&ihXbPz185NM52MzfAR&P!nKKNX2fBF_7-Lip z1aIq`WzoQp(7@Hh&8Wk`Zo4zYCJGGl~CZh;;8pys_7d? zF&QQmA7{;wgFF%k#STOX6*^n(VSKh{GUtZ|84Cy2H66E)GISnj;EHKz+TzV+(8`@~ zfLo!l+K7>V%>k(k2e@7|itJ&K*}%xV!BNe_L0ZF!uS9`ApmD(s2mZiDU5i7^9?kzO z|1>hiToIgeKq&-#?ykmR*#oPT1tkx@m^Md4=-d~vwjzy$HU=-&G|njw0xnHPDsyJ# zURt;(#Me{N=G@J~9wCO@hs#V3+nF(6E{r)LDK8y!Pm(_t|);%Zjl!Gc~8dbJ5 z8#gp5ayV(cIAl73NpH_#6%hx<9EHdfKOUCn>}L)d3miHbb=ugbNi^lK?h{5+8z#j$ z&L$oQEpIrD^gpI^$4h#)IgXi_Bjdbe9}f`Ek&ChLc)Fv#J1-RYa5M z3}!usx4Hq1W)aN1cbbeN8a3}6;)*$>`-MSqMicLoM$I)2KM!3{RcI7>qQm~;0vAUL za|)C62?y?fTO9bNIIzrUcwp$Ly5r!U1FuqV-0uA${r!U9Igy^153lVPeO>UChjU5; z+w=px4u^hcUQ;s-zqCcUzU9Tyuhfe?w!o4x_A(vsQziM#mww0w#HpgK{1x;i|3FEsL+9G02$%DdsT zKuVM7o&z!&P24XSMR^#dGn}}-91yi>Q2TO#YtAwiZ$|bV2Sg-VMAjVOy3)8L&7HmH zlmLrk*^&nS9}b)e>Hj!qILNp(Deq|HY;on(xHL7^?W~5-oEx1lQzPRaom8%sOqh3o zGvrYA5x3%}x%o2=UNTu`Gp})uB9p3vv(1g0+uYpkzb$^GwJz;iqi70yl}-F-Ll>u@ zcNV+vIJO;Mo%ILaPzRIoT~V#BO5fk|@?lOB)LiQ3EZ z7EN0tPCs9ENbbe~5s5=GK8+$JjTVxPyjNIucbwjr(6FegfjhvFx8OiaL4(?n16(YP z+!YMGHx8&>W?&a#@m~Lhdkq77!2#|H1|NP$-*N`_6%4{BOLp%#9n90nw(JD!j8+~E zMjh=2P8~<%IZVc18@Zeqm~#Fd%Y7*vd&6knolh)14y^U}@+>Ns27XtoP5F7auyk300ivgsq^)wDKWM@*FJiKTu%LkaESl$eE2@ z!@S3&Aw(b{V%v%6JKm+&4cTnUxe^+DgL_Us z_uV?D!hhf0p6@-2UrlA;n&Y6-qc8XGfvdTfh3`d18#ciT*lph?_tSD=GmZrF1Zi8!+ z{Dp(!mk-M9Xyi^i*eicX@rJ{eh8!LrNBJ6uLx;MVr!=s{cKj3N_k3- zYT4=MqOD8W)V7CswAOeXsK3V@^Ft+I8%Km+Im;UdNt0%)1P94EO{y;#cr*@cCOCw> z2wvY77VD7l!^0qROK!p6q63i`oEscEy_{uoE=kxZzGeOK?}gk^QFbMhLygy)?VlVj zvpr;A70&yA=E;?b8DcDn+D|6RrMQ@|I0sDqr~JiH_=KZ9`ymC3=9-&L@;1TpE1I;P zsA)Jb$qIOgmK<33`4_uHqoBY+!MBYn8B8f62|OQyUd%ZWY~jlJqk*gAbg)h9-h_q; z2T$w>+09mFIqi4K_UqmLaTEQ+c1HeK!g1&H^v4{BGRn8^pTYI!fYcL4)&vKE4-9N4 z9OSn!+;A(MTP(;mt)cg((VPnZn(hP7*B(p%@*svqhU?CE&LvSAm*%PcbuePLSYqfRYG&IX&SLX|)JnnwI$wWOlwaAf*->N3XIU1fWj2{gPgb06;5~4u|Np^GOJ_|L zN0|psrZ=3luI$j7b5MxIDbFrAZMu`ZOp?%zhB;~nSXIfCDoZds@XG+vQ66^TyTJE ziwC2?0fR8fkl(So`y*Iv7`Tsg^ZkfrI~Y}S;b#2cAg76! z^|`h~SLKeldKtg&m(8%Tz0+*}rP-hHs(Gj4$^JPfXL>y?vr6QDd0IK8>clC&CB3&a z4$Am22~A-#b!kp-o+>8b_Uf(2(*3Kc0GWN>-gGCQqYK|?^0$%%ta(Z%P7$EU>{ zLP{<@4FOIrErQBUGb|WSOwzS+Kl0*&;mvJHCb<*1Co%yK;O0}Ii;ZE4iAvXr^RL+?;jJOH%7=#}(!D%fD(q7#umaF#2-d+lxjS=NA+&U2Jmm za%#7T)Amgw2Lv9PwcSxQSn}Y&L&t)Lhdb>x0%~?Ja~_=TJz>J3oh_W_mfIirC#%@R z@m!`VVCPY02WCbNX@i6X29C!$(xx1=2wv51l$HJ~V7IX2LZ^ue&OQo~Ob$~bv=+8- zaL7Dt5p!X15t`dl70@nUm+)u>*UX?-J3_2#6y{MubshW$1tv{v(N zzI4Gwcl)EH)4DtEO$uDUD{D$)+rA&S9J$QYDi1IVbaXUL4s;Q071ry>aN-VFH1n9G z$0Q9^(Ga=D7AdEs9S55EzX_Zbuv%52!l%6KAghR@3#Y4i(6$$c#GLdr9x&bEI?}NH zUeJLDO%IrA6Lvn3ayYl+i9qcGb54T{KIUf}rXQ?C>$sbknok*@x8bl7Rb*tHk)h9* z`ft{Y#Zqb|9&7^KA}7vY{->a!c|gET;NemsKZAv=!YMlzw#xgwaGsR*>%&U*1ua_5 zoQqCXd~@QNTE!*kzG73lORbO1#MM^MA_Do%)>RmCNVUv39V32aZ{Xdnuc8vymcA}p zsJAcWMkAMn&&LDIO8Z`EY* z(k~VAQ%%I2^`z=_ZMzjoj~stq5#G*YQ4tdKkWckoNSg}%}R3wTE*gO9=3|dS_KNV@Xl&DGBd=a zk9ktaiyzH=R`)6n_nLp}INWdlPhly)#J2;gQ}kE_T}*WCb|keiS?qAI+5oeyzK<~(NL={Uf2tMFWU*nus~CJan3ITcu#X2`KhXb3IUc+6w6fX6(- zL9lNJZRHQX^VS<~Zua{5hzu)ia?*e9XJ~%FbnQ+;EqqZ zDF1GTtKb0xE`5na&K)b*^gON7`aV<91wi5fLSu(z?4Hfdy6+Hn>_VlUvcSSroXa+$>O9U%jR7N&RHZhi8M@d zFqj?U^UZ}>`pzLi5et7NfdvdJp0YD4SvWAsJYcreJIKhz(4tYBz{c?)z3JD%ewGQ6 zlV|j32}#~Kz+kq3k$cWUnNt&*HRpWfkV`S1eP_yrz>)-!<8G(yf`y!aG){56`1NaIQZqD)_I{H6%Cl_s=FN21dS`3@69Tg-4@Glz0Lf4o#XR z&~fsKp6K#`qY4K~8u^Ye2sB$9V$<2s$dvPd=~yVE?5PWkvN8vG^L{uB8Es&3w^+nx zQOoeN=&{Bdk*zB(clKS_u=(SsM&`E_W6I`WSBqquR0|hr~hHOz^lx4 z7t8uiepmU#RVL7K;77K|ybnjz?GCawR3?f2TcqJ6ta3`t-r}&>ga{_tH43fTXA=2? zJeU+XCbl@}PGu{3;mEGDfRQ)l0so2!XWmV}*(D+#Fa@nusjAGn@^zj0B)e$7n%cC+ z+do3he{I#*n7fKeY=uJ5&Q1rG>)#s`Q%^zEsQ&OYNT0o0D>}>)QOUCYvU`j`SDb!*KuOzVdx9J)7l16onKV zgtWJvNe%n3NNlUXX2B;3+%ZQO+3v}*D9mf@_Vj5I+5S)AsG>(VZt4^{>F?I;yJ4$2{+N9JsXmWCLegU%cp+1+3Gw z4pwh;)(~CufVZ;X0ROG9POm47%)Al_P6@i%0gerHEEr|Jd4M_Eow`yKndA^F5e-&uicJi63pNE?l0??0#&{9M6r7N)?GBEZ2>+BR+P= z7aU@9yU^tNQlMGNLWwUo!%fn6B6rY}M`A5A+RP?2Tozf8z*}DM&tKlFv6bhzV7Y)v^uU7``ogyiYF$|%?fH8Dy(s8>+9md5!erhzr?0o#?QYy}5cix#k|U0^MHz*gok zMS)33=^;nZK{1^!E~^8Qa~|@09hCH2cvL`p_N1rkVyAhM&hW4-*F4}Wd*tn$Q)^w$ z`D*`I$hYJG+po^ge@_03d+_qbT4q1}`!9OuIpnk2`Exn@v-LdS@JeL1Yv7pFA{KE_ zly99_Mx!vFBDdN@$(%%itOib*hdeVBM7WJ#i&+4Ezro*zdGIQ&9*o>a;96XUVa23YRL=CH3bO$^KQ$Q{H8#ztMmB zZneWXZ3`V&wjc!tl>~;RYK%MduE_)(yw|n5;Q;$Rzc+CY-k42zdykJbZvn4DqmT~+ zlMh24hau;h2mCz>%moi6nG}WM0zzw;*i$yBbM>_~QJwhIu&*oJx-DU$nTmvP3Ez6Q(@* z@y$VK-a|;cr9UC98{pMU&Lj-=+(l{By^j*^A+y5d-tP)tBF#|MxHD ze@$9#Spwgu2aIw#Hgani4>c=_U0@VsVm#u;U$fv%oWh%+2G%?4Slk-KRS&SE!+CmOM?GLU<3cz?|Y)_Db7rjzC36FBbaea%YX6+y(27zx0ToV|$Hz@30@qp*f0kMEa{wE2h(}g+L zJP>4X{Kxsh;c7y{O1TDxs!xG4jMU?w1s+jX|D3X?PVaJ4ka<_${#jg_@elqKFf%nJ zGZ{E=&3Y(i@jyY2he4=eolUEl_=L=a%xN_VZ@nI{syMI&9eATQLHwxE)uu%bP6{)O z81!@%mYFK2{4_Iv!j;a?&T&X#>U3;`HfeK zy8kR>Z&2W8aAbRRU{T0Ifh`WAAqx$DIfxp)YM!Df^kBi_DGS)D7O-U`#N|F5tSZR4OtuhdfX$YG6%tl3{A*q*<-zi6G!JAWg%-#ns_nl)o zc4=i11G4~+^{rzpIt&-@EZ~>xkWxAr=It0-wBXjdjryGatuq#ANM#$`JoZtb*frmy zaE7SS$_4XX0^R55r+XAfeSG_N&DXcr%G?`8Jj&bHPCek&I>fxxmvsU;ku9*&|4 ziTvlbayocOYArlEErIvi0Xs_twlxg`s}y`bl(K~^U}HPT;nA4j&L*JCa-PMx3y+xLe5@#8pse?bLYn*cgi$=mQPSZd%O*Ex`xR2>7-i=*iu`fl`gTC- zNMp*EZ<4e-la$T`s;_+3*1*=(z@YGeLChd4N!QxRwMj;RF)55qz>&!=H=R-N z?IAx_uZF7G3wiQh@-b}Ue`X=7l_VJUFGp}w3ul4@dmckp>>9Sem(=qD49Yh|2pWId z@U-pEL%9zJ*_vz}8zjQJ0L^ z^ang{JR(60Wp5-(uX`v}qA0RRfh%p{w9>|D2N^||G5nbJpmpI9lUWa#C7yDKF!Dtu zloVf|T=Xf><^bcLEB_sKKWe>WbnDe={HOagNAu>S+tV5xzb;{|Fmc>Ex!##I zX8{|7qhLA%_qPKayEcH%)a7Zc$a=sdwlLq3QM@EktZUbbgA6=;iDFKPV$0OUl@@{y z;C%AJkY}L)*F(V<3`v_3>-7C#jm0eHs}pA(z9OeETlj2g?6I={N1n5pJ}>${ zY0csl97`Fq8hB?Z@-sOwCIm7FB`{b#*ln~R?GtnOwo8ojnVDyNSUK-{aA=JqW#|%p`Q+&8c&;lk=Dlp9?HYNEbQ4(T@pnd91U$27in@8-%J$GaNNy&Y(?+CThn$g5Rj7)Qh3L_ z>jQJn0s*#zZ2x}qh+Nz^?eL@-ho9{e?hOB48alnB)Oqdt$jSTMYbp+#>lF&Wv`W{T z$R48rim}j!He2QilN2~tF{n8%V%+D)bc?~!{_W)ndCcj5_g!M(({W_?S&(INWwW;9 zbe9~dE?>z-ihN6a+1++;DSfa|FkAZO_C>MPbtelA51PrpQs5{u+2LNC9-wfl-Qg7H zkK2vX4UHvtPRmB;CRFG>V4l#Dz!&7%v4AOOxxkev{2>p|Oi%H>9wOqx<{W;r z&-k)(&K0-ZJxK{%vl4|a-DLD(;Cz+9|BS(`f4#se2fm^Q>ytI7#Vio=x^?-@YSwe_ z93~zRT++zZ^NlYhM$)b4NZ1A`FUA$CAM&htdt>#!@X!}*&7~S|6ix^h*PRxWFY}T2 zaFPGwAbds4aV8gQkONEG?CqQHoce9Pv%@%0rLue z5hg|vCpUo~3qF`Tew38wi3=8(rNGO)klU$Igd_3k9|j>2IX1Iu&Y%YzK?|xca0_?8 zYS-VmUuNb0J;t5PtgH4tf1Jx_rLs1~{No;vvOO^j{2>fiSREMB9<=a1H03FidUfC* z|BVCFte6%)W5UN|@50Ah=3V^izan-i9p; zuPv-rTC~|S^u&Vvf6F$WIbhIV>e@f&Q_+e$Umo1jzIprZytg;`6xtR@Oj?w{)yO8S z_GCiSLxFY~rKE_&<}P`GK&zLZp7P1rl+Cv)Eed*gc&66^ra~rGF9CV|1wR@-DG5)} z2#jTk^i0y2upnlqhh(Rs`y^>zgBM?RCOCF>aY(4xR0wQc#j-k3)?;sjqSp1*v4$}W z+6&)Y*`O7&q{HfKWAjQuxg))&8XcJV#8y05sCYPFm2>PS870$Peid6L1r56e5iVRJ zs!Hx@f&ZFXI|VI*_FM?u#=^p#=B0Znf$WDdHRuK4lw7U!0;YO{0q-c(z`w|i=$jNzpf zllyFYoXqc*{QYp)-|o+w=DXZ4WK-sGuE9x2KIP$*~^}MC4vwA-YTd(}`)!PKzWOS&ZHoa3~51 zC@`?Ot4wSWbjw)iDr8o(fsJ3SqS}E;K*8W3hh~q%%?$SUkZ^Yvq#Z`wT8z zmMM8YgXMnZ_Zbcg#HM6AF5h&#)P?=%@=lQ!iH}D$7Qb-vRu3q7bn^D%h6{~C6Q?k( zQQ-J=kaHv7IU!e8@e><75@ikh!l# z*&d{=OXAW?`#LT9*DH<2J}m)<24>R(g-3+MLMFJ#H--otR?2f()XXemzCeOS&}Bjc zQ-JD%gKbD_n5wHN1GrhRO_ z&?VvAck#TXl;%?J@Kx*rGo+r(a1i?U;fmv~=B>wbazkPs&#IDAYHT}^Vb_q+B5*o_OX>q!hyB^#bH4c1`Ga*0~`|* z!c-M5w%dO=z$D(_wJ*ZOWs}cSCf1JY5qpA^*T*c?O?b$#_e|4zb_WJmWp77;8wFfa z0tc8HG@7}dTxevHI9MniaY*xDK$o?Kf);Ow@=2u!46GUl81)Jq!*vXrRbv(~doE~} z_F2FnG38O!DUSI@GYXhe7bFQTEpXF3SHNz3=OEuP&2t52J5S!|d6=HMgIRTnV_(IY z!*XXUMM^(yH?a+14Awtzt6WaWRK&$M`0>eqQcQcNIs9;8i50rwDD>w7i$G1HSbp-+ z+Z6}7;|r3QWmh=LhXg7FNE{M4V6@OXM1#wqVF8oYfo2~21&sVP3e3B;U1cT-v|H?9 zG(FE8yT?jhFJaeeT~C`wyAPE5Z(ie98XL*|(!t&R3TJ1Z#0^JTnTw5$nog{I6Xr3S zd}#M7@s~8@c41T!Xkc@3tvs=rfsN$@yUm(~Yz8w9@~pbh?zmu)r27VjQ%s4=l}{)N zxf?VY&oMlkq0uZcbpea2%t8(q5mCW%k1SQgPS3{@?1EDpj{SLbaK$fI_spNS-Q_NA zXj;oTA^1V3RQaCICNX`lY#%rJFPECcB&FcU7j?topWqG$?x+n-qU#b^*`r@H%KB^; zh&7nN`Slok&OH~ArWc1qlP0v}k zpUUL1RQLOx=b4PJ4y^NG@F=(_9#Q7dtjNK@*7x8bgG@rB^{R&Qq7`m3Vh>NaesEx9 z4RGL3Tfo4p^N`OyqfJ$3V~do;Lbm1|M|ES2oh3eeU@`Exqco#{E#$!?{uK+_j3!Ow zNs3q`bj-j_^;tA$5=ZEYt^}7lQAf@s73WpY45G9qZ}SR!*2v}b;(qcUcDZP;xAi$XrdB$h z5_F2Z%A>{Oz&bIYh3Oqbhme6IbJvU|MrjSUAu7s3ds+`_{z_<+(s{%q?7RhHT1An8T@KrKar!(h`B22YcS(hd_)?9MsGXbkmWCXCRJe`NM&XF+3Wf$I z4Mx$4JC4X!Oz27TSj;aK;AT2S(b#!@0>gv=HU*IbEkR!tIrAEhs6J|7QISaG@88j? z9B`23o>r2e!2~&;Uj~D^2Z;rL2f>s?LM(!{U7nOe(q&oPSnAH}@WQBhD=W@1o z^}Np|-LDRmDy>}5<-S_DcG_d5M$R>^47?f*3@izZa!(R@ADn7bda{6lTP9I(b;J@T zHUowkuFN8m7EBsm8(UO&IPz)CWfXegz;1V3;j>zRi|UMm7M>q}_?bE6tK4OK3q|(6 zcxG>JaxMSM*V*+lk7s0Aud+%A4LbFz??SUk*@de<7o%8SFoivQD9gd9m7OXlAtuOj zBup!SIqw4db79W&3%FKI5Ly+$RbI}vw4iwD1R*`Oh?b@13l4Bi1g7W5+^#tA)o|254X^Y6UUD&@Zjo8XNB68(3p9FyLW8$wP5IRE z=K)9M1-4lNEG8Eixx`GlW-tj0n9jK*yUx7q>A~!rt8)As!}t#}=M}JTW@JCtP_D_y zmCoQ<5WxBNN@L>(_Qvf(tsBgnwsV?ITOz3>5><?FfQ!|`-%lTH=ngiaWJn?WcCr@EIbe?yMSS4_=MZno$D2h#RNRt0=NQ> zx7TFwiAmN=dGbkDPGl70f18l??Sah2hZ;Yd?I%y*EO20TOkj>a&Zs@Xdp<{lB%2(I zB4f~F*}z~i@oNHen0$gPkkt)o7*PL1Y1Ec5$M#Bcyvl5)E7&yEcnB6xpyQ^6xPwc%lqcJ?PbP4B zG_ubKEt>jo0+-JP7Vin&g#kt91TwD#@Vyd9SdzxK!81$Da<;_HS)DWZUMfsFcd_Ho z1OC05np286C!Ju^$?W{Q+;r9p&3P|0l^7D|S7>o|=*{}!%`k(>G=R0?fk?yxu9X3e zQ2`t)8@Lu#vab}Fx5R*>c0ulf0FG-a95=<1n-;JZf8g-Gkmzo}F}Z-d`hvmUWR3|N z*b59ed>Gih3ar$f^1~eaoK~cy3c7s!6!m$9)rDX^o{u&x;wnx-sU`v}{Q>NJ7Z}(D z7=;~-gcjHw7iQ#QVB`{DvPxjHyb#S?z~m#qKGDRpU%VM)w8ABCrqk5 zz!rXid720F-{W$7N+u_jdfycc;}u!{WMZOBK(^yaM$-ao&jpOK8(69@=onvMDH34! zoe<<{V7#V~tyY0Oeu06n0IP2UtFOUI-vAcR4Xh;z)1n$!mNam`DBx@P!05){xuap; z1P5lr0_Iub(Z9^{=YC#g$Km>2jImI&s)WU-TGQ3asNm-#27wCTu$wi;b6VB2dRxPp=65(8)JH9z+TuB9KUWi_OhF5q#N z;w*CDI#vZSu*D6hes(6LYcaZ5E!S`r(`MejnMGPa!QcW*VkAqEDf7dt8y(uY3NPrS8?3av z%A#M)GG!KPi35A{guE~ZjxKi&vw#dY1@^uT=~@>UbrXzMT%OLd-EyIFzChK&M-`Ty zuY2=1it&GlvRstP5vs?i%pk15R+Ye3nZUk(6@!8lBgX;>%`A7VS&g zSj^C^zpw*;{qSquj+wJRf9LDo zG5h?*^&X#j5?^u5G+@thn8PN`GV9+4W?fAd6M<~~1!8QDyVAm!bN%oZ)1E8rAfS7J zxk7e_?0WHNXuZcy0A(AGK=YiJ$mWPo(3GP0UT}u z(}X^4KP<{|?u2sr%;}GQ=zd&F{X*fW2}9lj8*T zz6TsL7H}9j7EM=N_*hb0v7cL)i;BApDy6xUlTNO z0ng{fJGz?IzYN&%-fD-;oY~=3OZ#^(RngpeL6LWR0cZaN&dLSsGrd`J9as(SFb5?p zb99L2nC)IUW!bXdru>ZZ2fhe|A7ru-VE6vO_)pq^SvVwoivic7f+WTymSQ1J^#zQ! z+Zn|In4~6HtZ0mkoyaIvzl@k;%U12w9VC~+(KJfutNC69f0^==?g@LP; zzo;Fbw{#2df-Q@bw}ww_Q9aOH+OzMdu&CAnwzCP`+7s4rFn|u-6cIkj8?dc(%M3*Z z28jn5Tm>u_88}#Km<+FatYBzc5x{#%VWwCDlO-2dTm;wE0w!Yy&a($*y-heMS#wa@ zX8n7vNeXXv^sQbxLHf+@nf&K3@JydDsdT~_ojE&at!4=^IGh&MX}X8mnqk*MZD#EY zp?{Z)S-g2k3v$u>e?{!M zG$T_0Q{F}18opV%Eu(pR_I?D(lQNm1v_gy=KN3^-dB z9I7&4FPN}1djs1MP8I_nRtE+a#|z9;GnQTXbM4xk4VG`1Zr%yaoXDuf!0e;Ie)bZ} z90%?<7X-Il;yn3{Roa2&!D1fo3$=`&jJlrHb{{(`d4Vw|k)?D4d)+^QNO=Zk%XCJ@ zM{E}1%qF-9J^iq5N6*2}0+YgDo$ijfA8I;DIr7Z*22Ra6XG%AmIemsL zv*4P+8kWqv%p7toT>Gx+TAnrA@IX)Vx}gBGyaKb=14W^fbFvTlKP=#mdB8GHfy>o_ zrLZB;ecoXn;-urOR;5K~~-Upr$T7o(~GZ{Gzb-3M&T8`x8; z-E|9$)fyJ*Twt2HhIxbGfj&1T^tpU58vn9`(OX^oDo0ee*ddyc?0mTFe!ylX6etQ_k&IsFb_`tyL>?yPA7 z^NKuXaXF6o2P}&ZaLYdAzPp8U?g!4r4_I6Rwx1K=C}Q9g=_qdw$xxblKKc%$!3VYm z2G!~dx_pm&>KlURoMo00h*o;Fcj0m+kAGroV~=m$yQR95k@@)BjiLrD1~c>m7|IM8 z7#038yjrKsHGv^^9mnNl&It@Bg&4M#i!dIDwNW!V*>h4Y%ERORJjOY-OfI!-Eb_Tm zHhiew(f-td@2S8?LI3qa@-JTauJ5{ZuUq=Q{2e~U$kX2p>Ze5SjD355+POnB{FdqV zT+7J4X3M~m)BAvf@AE~S&pdI5O>9_m3^<|;95W7cFDlU9a)7(^DT~1dmQ@1xonl4F5}qozoTOv^cO=Uikgb_vM*6Y~c)ta}KFE*$f>E3SHY)&LGh3ehoaCF1IdFaoU8^j z=5UI%pX^*z>*ZQ~?aaTK#+m091x^hMy%oCd;-!6f3|*_&O24xVTFv@o#w77hl3g{mR)*mIN_wjr6rt7%Xp<7 zqPi>uT~A-m@LyB$S1NRg$7I)Y3Xv9qzgfJe$?Bw>NMLkko3>l}+=~kf+j@8;Ob&De zXI-3cpBLn0B5^R`$V>gz@$&;NI5x8jtv5bm;K;<*&ca=>+@hA1m5HU|k7g^E=#pSB zzo-U<15FI9>3XsgHJ3MvY&faLR^%k$$m%B0#lr2Bv2bhgsg-I1+!L%@rP6aIa*3u- znW=Szf2EY5OX`xqXD*_Ff4pWD{)>@OKeqWy<+Eu!U!6MAzvtD1LO$D90y-NGw%Pa? zY6)2z86B+(a56eR4vWKhtsRFbDLWunKO35sl-OR5(e8em|cdM`4%#>Yk!)B%}IN5b) z&68a%db?lk$UZK&TBN7zAm3v1O;yzqmpuMyO$m5pQ1?AliBsp#jm6Epc0ZE5!*1;*!CL=#>tV7QlHpx3M{7I|cX>p2h4 zUDJ0h*0Y|oS@`_LCMk`E-ERxl`7yokS(m5aF8ku(#0K|l~o3WsQk>vr0#0^fSB2&lMw_1*t6Blq4B?+m8c#ED| z(PoskpwS}YkmS;i7Gzw=dE89A6T;Mu zQc{8d>l#H#_X`Zd(-!c^|KV~>(H3Bl2sp_7PDMlD!U4e>5*^lOp0N})xEQN^WOog@ zJgYCDMS084vmPbItrHit@E+zoTzE^M-9qQ6V18nk=L&82%ECwePUgD~Z#ZZW9MU5a zuzZ)6+cQJA8%-iCS*%Vsm_%1OxN@vs*p->X+;X++n8~RZJeC=ZA}%Km@UT_1h3F^= z>`7>Rd}5(cLxhXgorbQUD|(!b8b?;L1=^`jS#H(QHUF%RrR=|jo=285UA*$w zsB$yAi-RI}qz8A(0tVR~x~`2Dt;Tl}PFYWJV)Jg8YVu$~14oBLlZeV~mi<+Z>*g(C zIU3Yj6tb~HIl_T&X%~x}#c@lUE207w0WR`uc82eJS1LH=M|OgyBWKc-%@R{L9M!#4 zx>NO2hJFBxtn?)V)+$CvVRx=WldC4&ylm;lwLF?fVAc{&l`jtbEd^|*t2RbmTURD^ zbwPlsS{QH08%OT83r)xRHn0RtN#ZQ#agj6L&?vlv(M8twFsI@}HVFwufmH#nLJ0?q zxVP|U_GiWa)i@d4S@B7sQz%QV+w00lvj>Vg0gjwP2F%+H8#%3hG#lSK5X$p!#R0+E zjH~jK9y0URZFTt1dX2SObuNd%rrsM`uJwzgI?d)R`mVJ=WZhdDVujsK4vY!?t zx;$YN?D}EQ@!N~jbwb4k^>r=1I^p7HEx&DS$n2RU!rY>eQ?}XYSa{!D6G<&MHQwAG z%$k>8+%n%LCZ0d%*v47AIqbre_!bnn9X}Ax75c}K`M9OC=ArE^#tRNjotV)i$2_r- zYr+2qYMo8^@k5y!1ARdz66>Si?FV#vp}!>Bglg57kRh9%)< z2mMSW9?arVIk4}GDvJxl%*7A^h~s$ zB~K}CNn}gTPvuk=zNNxEFP1RsesE+HDeMxNo4}B()2ifCAkVwR!6HAvG0;>)XYP&` zhCh~`0d=X9=Cm+4s%kLjoLk~*^&9cSN?<9>}xFQckWcPRZNL{^7w;df71uar864y zbUuJ?Sr^Rv(fFOCf{{s9YxlA>om@d1WE0F37!R-MWDc6p#KzIUut!~*L9R|n{LKLt z&Iyci6%CE{XJ+SJ@D9{0Nobt>V4>`bjN_J51R6M97#Ni;961dfm^s%hVADFm5GZ0$ zJXyzz<@x@4qlrr!)t($4C% znb~-U!=}RrHqZWNn!aIT*`YZ;2}&AC&8M|&qy(7uL^FyXMEidQs>^&Au}VHBUh zxbJqOOp2LQf|=rtMyUmi;sK1J0n3c6G`?Fg&-mFO9ASQ^fsyw{gW8cjY6Z=z9gH@S z+a)u4S9Y-Rdrn+wFfqNL&2ER$-3kU_3ucD`W}Ap6(;5Z#1)Q@!?0v+nCK=EeBgnv* z!N8Ew!0f@m;K9JC!5}$-k^RJBHiZUOg@#9uoo!7TkSVvPIM46B2408dMutf-~4S z6B>mo7{wA8g$0f!{Yz5 zjG{4&d=&>3D;gCgnD`8ii#%v3^hhJkmD#7!6+@jBq`t|VZd1BJul>e zgk^=1!~vTrCs&js_5 zFnX+7ZC}u0+ThGSqmxB}bJC3|VhVN)0t|L63>+2>b}i0~0=hg2M_2-m_!x579$`?O z(D3*&qs;_Hi-wjEhSmpNEUpnO9v?Pw88Aq5uz*hV{L#Rq(7>{zF-4=*{s6brf@XUU z*7P6D-xnQ~3}9Mxf%9L&!&OP169v2`-tA(E-oPrT(YRNqQRKyGAqlr+kz*+?2gD9E zN?Z7>I?T9cH>2SR#tfO}3=?L<6@tPc z0t5ez2B{m(PlcB7aV&Z4af0_kgGxh-XG5dS&ITq82HqKs5*ADj9V{L{n41lD&RD?d ze5cR(;$hn*hb>st)J`yqavXV*ZO^R0&@R!?@t}cQf`NNSuZ2Ua>V^g`i3X+}0v;7C z4!4?Y1zMaBtd&^NsBF+UBlsUDw?+fQ3x>x*);IipeK1e0_Fqg4Zor2&(Q1(R9BWsBa+&#G90Ry6Vy zH1JJm(mBCo|3g_dVxhYRW3UnDbODCx3pj0U7;LW`W^rKn$rCD`(fsG;VTKF_tsN|y zD;lF0FtKO!iymk;Z(tNHU^dZVwfxr`Z1I%AwWCF+gW3KAlkNj%=LgJA5-biE8oW4K z?k;M!Jix>o(UcX@GCim*eMY0msaXOm)=jo(4Dy)h`a>aV&*}6H4MG8pf*OpR68j|A z1}3^Nig_?5zqy$5<$B7IMq!Cdif0>-tP$K@S8+xf7wQCs_4lujlXjcEi@a#?l*NO&>KMcG-8Z;-cxF4A3aiG<~p-Fdx zvf~2gpo+GT9}MOiE%p^G!7G}jHZUp|G&|1dcR$c%SJ85%SS;^FTg-vB=mShPKbS27 znqGJ`%=9=pV~m-2v8^6HWXdnB6ZhP8V&_V`z51z+@B1$g9yBzKSi&p}j<4v9MA=hz7gP3t!#_ zlerrh)Hqm-8@4+zoOg{7W3^}$G-!Od`QpAi*B@TKF38~~=X&gr&KVXThep*1nTodb z1FfY|YXpvPS1sDOC%Ump^lXpxQkR!|pXJ``Zr;&az^L+}IeG3}%NI<#Cz|wLFgZPF zw!Oh@zoF6mL$m#XrrkO%w@)@XDX<24wB}p1^B-t(lxXp+X!HEgz_+8pR6)+-(mf9c zChi>#nk(9NDzrwgVC3H5%kyI0y4;4uz^V6y8SH!h9cEUTaBuQqTNMTljt0>UP1+Nf zyk9UfeK^Orfl+!!qf|swO44fE1FVG?+VnIUY&2ShH5#}SSPVJ>lr@YUqFd}Nn*Eow zm2tGhMy)nhVCt%o4t;33&>@yp;DT^QlkkuE4hpPU8yLiP`0ia3F3IXsDe89MN?@`` zgqT1CtH}YAyhh_^vHX7ylmyA9ausBGxL|3rzuV~=*V479R79=2HFU8QYhe`KFlkUF>!FLZdTyh3JiNCjI~Mx*iqHoxX1R*4;xISg0~IZt=b&SGG_!O;>_8UFRR_W_nO zvZBWf#oR;|^%8?E!UvBlnC%@$}c`_Q1$z-<43QH9~Y$`1zR z3C*4wEFY#e^Ui2;SkSEUqPa=%0)I!d#|vg136+2jX5JSJ1(#d@#W1w$zG&d_*uwE( z@&>Kw1SJ8cg`Fv8DI0U2F=%vl-)?l=&>FL&)yl!0??r=+0&9>St5rjzSU`)&ye91l zk3}Pzy=LrWRA{mNz~ofX%>AHWsKKjWa3A-E2BUzsAOSy#4GkGqO&ms6i-Q}UXtXco zUa0KxC};*7j{#%rKi7SCPA7)ExX0&n(8ev9=LXB37t$SF_hXlr1uYUxVb)$@c3?82 zO_;mwv&Vs#A~r?3I9_|lnvu52v%$}@Nynf?Lx3e#fW7brt7qK%Y>rpuGn%C~FfBRP zu$Wgx`NBPwh}T>f7?fAUS(n^0*;Znzs)T~JT?@zCrcFowHNTCvi0w?6 zz2gB-+souW=UEO7(Gwe3Js89v@OZ!FJ{rNSxuMDG#nYRS&FgONV=PG5)Mzjj=wy^& z(7n*4JtKKu%MQC0ELvg-&NfWhjLumBY_1lZi=7!3H!}!7Fk13K?om{Wb{->-0OS5Q zxp(6-l2m-^Y!5WdebHg()~Ncq!Sv{t~)suP;y#8`A4nB)wa#SA*O|C{>%&;y?Gdu$F2(ckw*|C@f2 zQ=n1w2Lt~P2I&j7r6x?4xv#VyG%7D};SIRJ9l^wOV$&7v2C0rlQ41!y3p*z7X!e@X z$f%IWw4p)FL(sya>#i2FT0;ZRBTkM0M&IuKk^d~G2c!PccRj8y@BCcesYzR$ zX^`q@whd^Fxxng_&K~dZ({ox%P{mEFf-hV7`uJxscC?)EP=Ed5O@k!I`ND}HY{sh@2ONpn*0?P0 z^1nv~%yJK6SZ$fu1Dd=aFtYye`&7`usnocNNrCslIVpyT))F%(H#E8yoGv-f$$p?Y zcn7oWj7EzEvxPZWJUiHI3=Y(|Hl9dttdXr~lS?)Bqh7n64Mr-O!Y3HsK8@4$c(6C+!8Z?MhSjAXmMw_>Z_6OD zjCsLJ>zfUH3HC9n3^mG4Y8kKEBAB^T%XkWyqo%Z|CRngfP)K0n;1KYbv$7=ZqOSM$ zM+Y=ieIx&^bWQIuuw2UA#MC1orI9e{&6$~!jZ==EbXXo@;$C)XiJ%e-d)Ew|hy@EQ zKdbo(n@69qDg5lxEo>1bp!$nv`r7F2dcL!soDf{-Q~7F7Vsb0jR0YS3H3c6Zxps*t zS#(TLNIZ0tQ&`F-;ja=qSZ{=Rz>Lvaluz~UYuCiX)?U}Q-U77eu z(V0zHGQr{82F6WoJQ5}a&(_S`%*-n$7qKD0$(e=Mm#g(xxd&elhoGB%hPJ_>SPniB zlN&c4n18r0AY|g8Fo*HueLjmHoKKGT3vlwXlw>45v6?0!acW24rRV1rJlwM)iY{9_ zGTlF{7-10n>dej`|K6Uvs`2%o&-{>YQw;+r82*iw_!7wJ(m1E^5NEB2kgxHv04GkR z&KHetQb~JS4s%;%o?!Aff4Jdcfa61j5bkppi?8VVn!0i**7+=Dm(B@bY*BOMICzyS zVX489K4%XFrYOEWrj5)+f47Cjma%rYDqMcD>6F%D56Pt(i(M?e^sYw~pAZR7nRt{{ z!a;2&n?Z)dv-CNM8zk8lrzRX^U3zk50~?owqq}(Qn~g_i3fd?*GUmE2xb3!&@p!|+ zLrdG`ig;Mm9hkK(+&xrtmRw+9H2cBWq1fzl=rD8ofx;u@^UfJKt5z&gpO7c1xPXB( zt!7G>@g$qaJUrbCE;X?-&-u5(MRmf934tt}A}{B2a?IOskaNP(LQc~Q9-Dc#o{8x8 zRPIzcbV)+*VZq@5y)6bJd{6s4-tBN*DxGA!T}8E1WwH%ZXh`W5Lw4!R7{{Z0);AoQ zxh+=-h;a#tc!fbS;k$Xx|l;zVY0p}KZX$0jwd@3?Mm2KwCnWB7OfFbYj-!vJPJ~nd~ zcIy=znpjvY8V-u&>~UdkzJwiNPwGa_mAV!m$S12+%(-^2)N64pJ8BD zS^h!l;H`ffwF0WRKKL$YO=l@`l3D)G#hCALPp~!L)~iW}Jhq8!d(td0<97k`^SRp# z4)1vIY{I)89(R+tf_-HVE!tc9BC)l{+iZG}*vz0|R4|`jSVc4fzri8l@QoCtB+; z@^^K#7|mJ8@qfv2Mv0fR-IatocR4Z%o%1-h;ZJg74~L3!$0?3GQwkRd>nKU8?KrHt zB(bmbLx7xJOuJ3Rj-FDNcTqDs1X_<6uo?bn_}6K9qLF{Wie{N9uRHzs9Mtto}Tg4vR1E(!U>$FHj56@p#B)%lUwpMWCSJp2BiQzw8^#F&CPx7!uh91ROt# zFmky6VH8@LuuNHGA%}a&A;H`XMJA@{f~+bJSkfmvGnu`U!}Y*Imi0jn91$m+!uGym zE7xc>(wf-gsU*=0O(JZj*2F4r*b(+t6Zn`_x_=Im7E2rWnmY~( zPBEBiHzkS7FyN5j9LH9s1z)GyGWRjpBsetv%WUTx4+`(L`#;!UAo zYrs(3fdcmMIN&Fe)yubCa@#MX0bqChLZx{ny+h|l>}m!muhj$(9&>L*>Ou~LX)D3 zBhMUvUIRvz1x}g^n)s$TG1xhIyB?PR(7+ZWcJHY_lSl*8gGOTyXOkU{(N7yyS?=kI zo-VNHE$o@c!NRaRa9ZXGHkKGC)hnG$TMo)v9AI1_mw!X3(1GDa@4bnp2lQmmPOQCF z!1ItlPx$#yHV-IORXJ z&T+ZQD80w;%@n~B*XaU#8p2mFa2Gg4crdV^xZ|<(=zNCr6GCQ|Cm!H=b8$`X0k#YW z&M6FRGa6=`arh_rVUAwOVZ{fBOhlT*Bo0cxX}t3{T=<5&LXR)o%0q$%jw%++*WOO- zTyjWAhe=cBprFp7hRg$ke;gG)FsyVrBrkK2YwslXz-i1J4NONGjdn2Bi!^XX9Fkoi zdizcEt5Ru}B@QMLO{rfUC8snnc{t0LxTwr=Qgv}Kv%F=Q*wQW7ZuKmv@XImoUqP`y zFS!};KFSqa9TN8DY@Eu9Yht&WZLc)j-)nZ@Qn3?R;wbmTu`ThvZA|vh3&)>Es_jvf zImRKXaj@sl%`lJf@GW=VE;t|{BlVrpKbmKe%v#oeF=u(#%+jhk$R2Yna}6h3&4HCQ zjam*4cf1C+@G`GC4Dso#&U6{}y>f~g+;;>D@L$RiXPC_b9 zacXH?DpFQ4Ha8>{Ri?k*;P^OH#VxPZd&NbmuM6M(>F8f@R6wQY{E0vgov4pn4hV@P zc$g%y%wafX%;wy|7|P|4_AqE4qem5k1Lqtk6%8jg z&t|0$XE`T?$!}@kd%+Nu)$(v*a<3+Dl$4yi#PNr!^Sal%zutJ+?Zl&lB4RhdacU*ga^rTe6~Skw%)7isUCbMIR>@*P|g<7-hCJ_GmB)c*HNL zJX#WXz3+^SLQ;6{k%=;fQ7kVabXGCR&zXLz;XJF0&~nan(+P*ZG&G5sco$9TIa%VC zcH+#bf@AD5jCFGvHI6t|Oq5MaaGP?7bIz8Nv)=Taw_sqL;c)TgoWM^GJVzS1L}Yoc zIB0e{>RbB^=DS?lqO@& zH<}p}byW_lS~&5uG&H^PXJ%;Nd(g-i(O@{Mfr-OG(!|;3rn~AJwkd|aDXeMMiykl1 z=v}@qB4y`d2Lr{`C)_wEILuw4q*bt@{oG;ue~%rc{w?YH#(e%8v;BuR=fCBgfA%W% zTh{uz7&#wSnJhb=9IEj7SR#q|=8kJKKyM)SFiMM(#?>}r(R@mA!_ z0cMeg2{BJ|zMNq`(V#rXN&b$b%!9_Kd`xmb9OZMmI3GIdTJ{{~N$X7%NcV`6kI<2s zuphBOWJl$c|w6YeWLiC1xH z(*5QrJm*x=qofOK&j>{@v3N0Q&0vyAIKZG1@cz#EzcvkA1)@A92Tm8<;5y>K9^=5S z;b?5(G;xKpP@Ob`WwWXXpO!_l(Ub;;ISxisnv|wE8JRf8eLKcdGfm%+fmx?PX!@Gw zc}uOnO|n+YIO(O4!79eq(r_l?0FR2a>5OKp6^FAbGaROAbS+zP!E3$L>W>!|723~m z3bttKNn=t}(Vb%(FEwM?f=buECHlYaoeN*%(8u#B{DUIjBf z;X|X$2giTgu50-m=DE?pd%}@-MFU&M$-s4IPSqS>e8j{uuV@V`(>7;g+_S^N3jcz{1HvcHV#Yz2lN{CUa2z3 z8FcW4H<`qwGOcKke83pIl7Zppq{9`r--x*{o)Gs=D2u7(v(>w&HqV0IPg=F|kPhdW z2KhOMOeQefwSSbT*Z90^*FPP4D zu>=3!tL5q4+rabRL`;6A`8y~W}D7hZ0|hl>mtmUrd#UM-D% zpBgKF?*!8cL9tW!pFGa_ut_Dr* zMZUhB!BGCM_yBW)1Mh`K#Un3F3J$9*SkATOpu(Sn#y-rl8~mAl8jUNM!_PM?f1Rrkfk~}kTl#HAN7*ah4-_-gJr=$^5x!!ZK*|=ih$W(5 zc7z5T6!meO5Z~DM?LeO&qga5E=lgHonsZmn9+W!LSTre~!L3PVN8``0j@OnlnKdw* zCor3dG|3Az3I1WU+QMwLgjsWq!;B0zpRP`xBMs~!2aRif8n-Z;8GL2-VN}@TsQKWK zoIxq0$3c|`hb;Fv{}W|6q}o#2`}II-l5~Byi)J<1Wt>X8Y^CkH?E2QX zOlb=Z&RYM@V{v$wki5g>fpPok!yoH@biC8Kz-E$qy{gl@Y^^=Bu?}l+!JXMJ8NHV{ zZm&2Ukiy|1{FS@pXcoI8e}H5E;l67DS7i1alu0=#vGJft2&32o`SZ3$3WhtEuWY(h zxJg>(1lt6!|3?|c-W;sHkfh+yr1|2Ial~O0pK^sW3}QE$%x*N9+BC7Zn6r5}9GHES zxrBkSqCq~R*<6EF&xV<;!h!KigS^T?8J{M3ldX(v4k$lp(hN8($kU{>g29}zUH76i zgD58h2g4HP<8CokW`7^lJp1=GUa8{nHN80H!WChKRui>OePL7)Fl*EIvn;5vT;gnX z;c(LYk83Vlt(&}K?fYgEkLIm0OckGLj;_A?#sC{Y38IWJh{L7 zU!%OnQ5A+mh0IJsM-IrGI4I)d%3g5)fVV@%dIp(*W-}fp*$`{RGmNZ%P6}Ey$tN6? zNnjN6aeVysM~DTJN=y^~lFj z`L*kTSr-^pY!0t=aW)Y!TiyQgqf)K?Kl5$L>kZdkw@P)dHO`1}$mG6Oe0oDXN1IIc zfiK-^Gi2WEaeQ4P(=Bu8)eo(|?-^C*GyaX6wMvr1VUFeHmW0Wwp3@8_7Ct+{&g*t= zN|#6n=R{TCCpugr8UE8WgC|I=y!70IO~NQlX5pcS2b;MC4O}LCIB-xvb%##riU$V| zxATd~SY#Bk`OUG*f3zj?GF#tNtGX8&P0b4(nbgF>MOHuGAnP&J@18Ncc*r^>)2@j(X)2TN)SzRMIc5@?0&ZD`lDb@~xUi#fJ`0N`YL${uPQmR4CZoab&lUpVTHZ_~~iR;Jg zSRm*!Sx77U#+j9$rwA-`KF2RROC;8+^!0gW)0+j4TqeJ}H@8GQzr|BXLh3@c#YUIK zvn}2*XZ8qloNVJZNPCfR>9{W^yO`Vo2iECKyAzmrq9!@8ie`K`(7?_0A)$fEpumBV zSLT2NGq>J}hE}0Ck49$EAc-|C98wJ24Xg|ur&q7yb9mjrx+_ZI*xFl72iEL<@Mx9p zraPOo8x?{>ypM}~zV%U`!_P+0!Cs-VP3nkfCH*nzx&`-LRtiVVii%3dIhE6$_Mh{1>`9{ zomQW>GgZh_edU%34(D#H<7_;a(4^4FW3;H#Lc&V1na8AIA*Z=e)%t!9g~YA>Dj^pX zn)&)Hl=_VPlD4w(MNE32%BhrbvXR~5LjwbIo5q4B{+t5}%?nI+9AJ`u)bWt<{{&Zq z6vjemS-onP6o(@tt5d_u z7N!pkEmB!?JaaZ4Rp^#5D!sJvjJJXHYo9YlXH|TC(+o79nOa`esN^wvb0oN9YWT8+ z?3yb!H1RrpKk;Pe3$?jV#pV{jw7e8k##bfg&VMs`x9~ZRmfgP0jH&+?$4rkn#4DlW zpwG-5aF-!8*f&XuPub+g0_Hzy3ST+AFF8J+lRiO1!I8UTYJysV<%-0<=z@dGr0uyh z4m6oB@n_jd>paG^tKDzOqU6o zH!G(E9(#LkTJsCVXETjgF1DPbJvUWoLH^6gKOav{-@-4kL-<7IdPPRoNe)Ic9(LHB zyYb=Gmz=Fu%ximRD0A}bEokJIImnSW;V=)|!`|Wr%(8VBVH=;cF>(qpM2PQT_$RyJ zU~jt5W69!*0M?#@?#u^FvJZYVJXD#`#1(mhkyAmOf#ZV(qfLSbCkI0m0|Vokzph0r z72*@BO4e0A&DMUt%PacLQ~jNPwDtc#>S9)K*uc@UfJN!T0p@!jI?@7^3py$#ELSl~ zW;v)yJu^_Yx+K_N{^y9Vv=Y~Q?>OyWIPjvC9&>`s4jXyOirTM9V|Og}kO>;xlORKysWl}{)N7<=%uu}${+ zVP7D?tumdmID?HTRa#tB(I0w_&Rm% z4Gx_zuVQ#>RhDd@vrgYv#XFuUxk(^m0gG5cvvQu{?puM$3wczJACmhOkToMv`uTzg ztGaf{hlObs^M3N!&-l3PnJIsN$_n2vEfd=8nHV{85}d`RP40Xaq9rmpC`C`|rpLj^ zWU2lwZF&`_4tjVU;*hmq6c!6S?K&q>aN+@Hg}x0f9y*UWWG^si1zOJFO*Lhf(2$nh zkie=f!)Wg|lTqlAf`*e1qa%MqCyUyLFs+zJGFKKji>=$xYUy#XNpgde;+)6rnk5OW zItLs0>i!ieiY7G7X1dU9JS&mYbIu{5X$5V@w-z*%Y!i0noH3_2Wy#bvM^3oeow+V> zzf@H5>1W-TNTGO*>kO69HDL7&;Sy0c z#}x-RGKny6b% z4NL+H_?LGu^Dhhd$E+f=koR1KiS;i%sJt9`7?~pRM#d;+BRMy7u3HO%nQhT+u z}eH&cgHvNu%n3lZg?uylxJ=HOmvnQ56^H_Xz&z6fdFOn;-)XL4h zv>?`O$wRKp2tU~rskqsF{F5#RysuJ}&GhhpEEc)pD!U&;^Zb+pEG%wQxw06T-ugbt%X!w=`xj0-zlHImpIGFP*!2uxu=P|8%qe2pvRfR9l0g?y$-^Ldwj zU{hgfT&neeQ6(<4+0lZTU%a4!;Q>RZg#{D0JjYj_S%RDr2O2J`&G@jfi)GpxpUN$n zE4`+btL)nGB=vM%;o2N#Va9(#B3l;h+}c{uyt+3hWwA8#d$)tD6Z`UJJeE1)`fkq4 zGuEX?m>2ImZEW`^C#d(j+J(lnX1OmD_PP|G4>Ye;36R~(?UBqZeQgJu*|x^s>6>Nt zDw{;zFbZJ&5^^?*W!f|*nGFk9&%4DEy8G>4D+PO1nZ|=Y9vuAZwA$STlK9LXIIMdr zX3lPs!0N2fz{2&RQNkmUFY*Ngo6j3zl??7k{tiZ=iVy8PR~Urq8ZI!bDc}m(@>ph@ zM{8wHBbWCACZXjWvMK?b8`dwezMHHmBydr5*On7c^*uLvzhZJ`;=NJk>@C5fwPqo2 z4$qt~OVdAF-ihIBGq#a_-mybcRqbQNzYB*SeOs{3b8YmdzU2=eE4qIev6g{@~c@^@hC;B?-(g9ZTD^Oe7dfOs6i{HnlM@*YrzF0(0o;AqsqY~{eWjKTI! zg20MnjME&HPCb;Iw@{YnA#=b&p(hQTN{Ui<90cb$vNbe)yw=2Up(*B(YFEnYl5mtywUoLBOX`itn5>M;^!SgVKC?4<`tG4eS3JmtTG? zQTCeSw-<`vPdGgG(b~<#`@}&{*lk^qKhqS-<7|{qt~1xEE32Lz2vD^(IuP}1!>>!{#V#mHZFwkJwLn$s0CUON={5)6 zswwguW*1+zfJrHZ?Z`QfPYj$K43S0*iXW1>HYEsTG^nL55XoC3;>Ea0$3ba^f`Gtd z9)^RGA`NO!Jd1KX7?~UxxE9Q1a$wBLNXXOhc+{!8;S8&$zW$;UY|M*2&00ABEfAS^ z@a#Fq#x);hHVA&*QYgK0iy72SD1?fGCGSkk>?@W|ZJE$0N^4FUHe(p6{B1O*s#5VD-U#8HiXIt?= zFemYq(L#}(FC&yzs4`6uC~yqoS}5YN(9Yrkvl^qg+Cs}L2i9wR%vlPYU0mF+9&m8< zFqa$#@YGH1itu1&-a}Ft#qmGk3qf90_K3!L*LThO-T61I*EZl z=0Jf#0tcVV@t7tywTF^w3*S6T;D5oucHte0q1wkcP-*72|HS4=#>8*y}s1%sva0yEhJWmBIBl?Jw1%4|9v&N>-v zS^plq&0&bLdcf?`tCo9!SL*;9N8|c+c?L>}0;`^CP;tQ;$Qwnvs)eKg=xRS*nprj~$>tXQu zg|adwA@U|p_b*95%;`C?QUB~j!zGC_OB|)PtbV+00=M92`G1K)Vr$biKJ2;REnRn2 z>WHHJbbq(xD@NHR##2k(4j+`7v2eV7s!3D~VxQ zm5}2$VS#^13W-Pkn45$|rzrAiEtJt(C~4HduuYFmxWPcikuB{2%Y+4NB?@dk4SYNc z1FaGSyd2ql1WJ>hbd;!kVhC}cu`$7`p{vb=ogqUSPhJn-wF}D5vFN2d-!%2gH}*^gM$wrIm?tgZU*!;<(r{qa zSADZJQ4@`wn-2Xx!yr0~Q8ecuzZ_FePlEuPLE?l5%+nJ1>sa6V91vu1?&u5(6XgKLx=;f4}>ID~G9Wcl*6)O6g zE+uSa`9h#EG(y$5nZd{{*(4{kBxLEhs&&kk=_s(hYG5}|ur^-6|8D`m*$U}551D^4h(3D2k@l7+ih(0Tsq*3? zmJbR-4i_ZfF=%idV0-aE^1{D|vUUx80gRltI&~xx7*AA`bagN`9MS)@h>cZ^#mJP! z=g`%hhgGK^248JVzx+(5;hXNqOEN1Q%YS{+k2MVoRrU>eoVi{>c$dP(Jr@E`xVs)L z*n7mW?&62KquX)}ce?%F-tzBy_TNIOZww9hzA5}~cJRz?+!|`C?_#RE;Fn)2+l&Uz zgy%eI2YAI21wS-oH7JFiVcbh=b%`DqUgN@ z)}#c17Y`Jb9E;^Q@yj&w`5Y8mwouxQQ}oUO9m9sgs198%ca{$a1iN;K{CY5pMS*P% zgZQ$AQauXX0gjw+o-!PpG4Dju-5Dpg{?iIgF!<`ivV>nQ@%<}_YG={@t7-k41<$EG zTbLjk#vN&D5@k8-u{?)jczNl>Td((?2&gN(B5zx4Jh{~P=k}#P#bu{lHD2B*V7FlP zwl2m0i>JspD6XH{?0HtlckPRob-&CK*!&neA1g3VkPuZr^CILrZ;Au6l`gB51FP4k ze?1C(0S`rH9h3}E6kt#k@Hr?Ml*k#Dz^TG0V7Guv<)GvZ21f%0J}*Y@4GP>F61Wl+ z_yZXE8}eRnc2&d#+eHnCz$?9=4lm>VO(;!eUHWm z!|U7;M;)biT{AKbHa3l1`oFwB{?^i~*QEFyy>&C5ds*)C->Kj~Q*r6kE9d-~r!C)7bkI zT2Cj$F+7;Bc|dPjPi)(g4uuB{N(_u54tjYXf>w(%e|R7`g|Yf)-p*SK`+}vVwM}I7 zrKAk1woPZ0n${>ehk1$b!->85;(UP-3;%2{oU$W%*3#I2Qn?qeE%oMDcJ`a}wQEuy z2iaDsMtRP>;&pS%W-sQjO2t;G?3|_h7Vxr}J>Yl~!gsEDjmT9Ek%Rnco$M?Hd|Zs; zI~oNhJP^E>z}9s`GU$fnKL$%nhxptDD@+)!d|-HMrLbM);Z!z8J`eFiiT00ocy&b_ z*u54Cl}vBab`(@vD5N1F>eX0Uc|bR=r6=8Gz6N`Y#`4x}@AQ+D43ZY{vpp1%`?cjZ zV@7wZ(S*M;43^H!H}$8NZ8NQsTryYk$mxlqnUmHix;`$Sv{TkS`-{>3zdOE*=boNc zf3$J(`NsN-Va9W>a+KtG`}*!G{{ZXQwkd}Ps4^=tiTo2|pa`T|J}hY}Uoo-GhzTh700)hxdp{xuFc%NY3B9NFp)G5uoT zH(HP{GP|vVwJXVB+bbpyg99v+4!ra^$x^3q+3G2a&jQwz1uRPLF(+Dg`R8JF zfbHG^uCV!hdjEN!D8xTqz#rmp{6U(RH^Vvxwkuzg7)%(10uon+FtCX`{51-5=m6?IgebU+m5B6MS2wlj@D52srL4nb!d1J~+w~Pf2j6Gfr zY5zQmFRS008PxsA;8+`c*Sw3_S^VtOL~GyNm@QW?E!f(|R(*40;igRe-L3yX}FItZN= zv~*~?q@*73kJ*`lPi2EcJcq&p2c|~T1=IUDH5hL43(i^4%w)P|AqyLyNdsdl-<^ji zRBz5Xtd^cLEA3on(JHZCleK%rqK!7m8s9eDaW{;|c<;-kv&OpxzDE~5iaFftaL>j{ zpyW)Xd*;%qMeKdW``6LpuMIj90lV@sLknvZ%*6kE7rKBe%kX z=~Dz`7#b3S;xtqFmj9a~n8V#Shsl%ACP++Drq9Ev*K+Ab&7zaVo7B!)R5%s$nAG1p z_I&;sgJ7ThTRxv>7aWt^E>mphFD1Kl1{bfN_2q?|=a!t=+8n$1T0*rWqj-$Ns>+3K zN8+j!4{mT#i1;CXLf*YX<%$!hX2}uB^fLu)!bL7jozey=49#3xGXl!*8r_)7#A77U z%E%s)ak)n|^32CG#@lNQ*!6avNj$8(<3^Ax&*VFwP6&ywc)$>0VI*)coMR3vV{eGr z0|%C=2R61Dbult^@EN!+W?`t>xO>AC9)k-i%ppP#8W{L29{HH0Mij8|PyEw(Ek$4o z!$GN*2A|$J*CreDeY?0~&!*G=bT-QvpV@RW?2OR~h2YN%55Lu!;9wWc_5Z@H$ac*V zFKycirtK95aaY5-r36o=yj2uyICwjRkyDMONyL3eKdZrn2_19JGL@WeU~T>+z!6oT zB3pQXS?>cwwXFk_gh!*$iHkhJDG#}}?>MT`wxQKi#)(To;Yi-54~^~*nnX-rG;qlo zG_qElbK~vkdCA}`=6ZmUVOmBr)0qqHk}8Wu&Q5S(W?^Vx?O2+?{-FDy*an8ZF_{dL zI@z=?JzABv#?gqY!GXg`z}CrUQPq zC;c;r`zz*&6IxoDofZZ-DP&Axulmp|m(UeBMSwTvP7>cN16OWGfoA6)54q$r?ijy# z!02@2keKZY26nj*4Qvd7vrT6?FdZp!*=yp!#L3XWS#e>JkcIPcZ6B-x;?GjjJUO2F~ePG}dSfKM->2#mO<|S+=6pR=H z92i&{j#PY8W#pM~iBYcYyf&kP(A172vN{FqAt48CUzN-g;wbF4@^s!(`skiZESvF% z#FDKiERMNZzq;5T<-DY~NdCf`=_c=Xm`u8mm}&hc+~m2(@eG5heHLpPyQQY2Gw?ai zWxjV*xuakcBg=s__E`*EF{TeWie@kcJ-j?ITusH^&5wTWS3fNS4Z^tp? zWdV&$AqNyAz5AK87BGD2dCADK!-0Lq1$MQZayIvlMs39(4$Ci1a{4656{_12_le_& zSZ6hx9>+pXW*$Z1?K@bEUng|^?KKsj{K4T#!NYd0l!uZR7dSDlvto6!Nfzp#(IO(_ z!>A;c=JwHxXYVP^rGm{849PzYraseL%5)-%u{w}pO4I}~VT}bW|8yrf@-k`6Jo(09 z)?*L0oOdO+^7C@4UMR!GV1^vK1QqpyP&Ta%jFGc=_sASf)iu2^foH{nX<|N%50VTzVy=`6 zwcKdhv(})~$G}*)eMa+A&%3Pr4GBV8YZ^q$l3ST18um`Qa9>v=mzg=Bf#I#tL4n7X ziy0gk7z8-h#N})+=4Vgy3hxp+qR_w~dsdsNmDM5Y!OX236Amz0oj9VpWFk*`wye}i z3%Be4Pw-^PFP2ELpT9M6HCswfG;e3cvHW!#Q}Qa5gJ(Lu+_`nmJDuR=cPjstr<5Jq z82vILS*n6cv4wZjgig&Bci5sc7RvB{?9mQrIICQ9klW=# zL$fLm<2LQ*@tbyRV1Du9uzcN&Hc_`)jvrDRa(#Z8uQZuyc1uBcazf z-;0JhE(~lRG8sgimO52#TH^T0(EIDd5VZx2u7M)vj*2o0j4}bNbp{*@-8o_d*sC|N z{>yab2*1EAw}8_vfmxJ+X;whB)e25C1|~)ZCbI(8a03?R4~)?p*kd0wRVQ#28x%Vm zFzRn$(wxA=E#Mz46uIkhV$2g)r}Wz3rjl;aHcxiH_Jy@d3z(V>IFD_ptNBz~zojlO zgXe8}drEnk+qAN3i~6hyYPS=> zZDIde%27RmRVSjVTERJ5fi>KKHFg7g_=N7#1)M9>TxBLO=_j}*J+$U=sExfL&i1+0 zS^b~HRdos0!wg%@#5zwbTyh^WlfE(fePErvfpcEM#Fz&x zt^tgED>``_^~5d;OktO=O=DzcU|zh#QLIHE&&*W*0&{Qxv*QEywr_EhKkyrsbhj38 zOcvnUEK${@P*web|9xq<;}i`6Z5M$H0_X%~1tc(kXO_g7^0e`wGBxq>a@ zK>r^Lwzhv^85YgFHx}^TQxLeCm|9Uj;e&8$)%FgfQtmeP0>1~Wa}>Dd9hfQpl5@@h z*7yJx=?fG2Dg;+PW)KkYmQ?iq=@{Z&z~WZGVo<>1vw?BrO-A_u=FkspUJFEerdjMf zAT3nR-NGTO`s+Q4Kmfl0<8vF@R`q8fud zn_HuyKkMhj`oiA+QdcGe27ZnL$qSmP!3;*nEE<@1Jm^2afLG|3EaOUEht1`07L-3`OYN%YIBOW(7S0l&z%{#b zqI%cF*$P}@J171V_#t3E)k@4kaN=>n?UI}~McE5Ca1<4=b#GuX{Kyy@z#bvMY^cC& zn7~|`Fy~k%*Ukmp)+|$;G^ey1EOnmAeP|_jo_gVA1J0cbEc!n*n=N3ny)ZlY09(NY z=6jxr@jDr|EuI(mG9s?C*Y}cm2IqX21Qy=~tktVnV=u7yeqfyOBePAAQQU!JNx<|Y z2|TR3R&+-#=+9d4VLHc;6$>&LW@H+$IXzgw_lqYkD#Ltn$`A1gXA1LrJ14x_R_^|J z#uBTA{tQep4`ymsO%%I1bM^twIU6P}@pQ0p(t7K-*pHRj=YhO$0bAn*`R)xI5f4n= zJ~VxN$|%1<(OyEy#)7@bz6Ney}5SD1nyQNOCy&m*A?u=39PmO zbFX%?6gaT=7jVwl%*5xAz2|Z-cY|x(%-)wP=T(0cfA7-i`eC`>0#-i(*7yso@e1tm zA6R`ISok6saziAGSMvPhTzSNSlXcfdHZM;0Rb|=B`!f>QlN(khUs#Z`VddA*j8@JG z9g#9ix+W}pwXpxD?1KZm@59*RFL0`7O(m17s|H-I-4nNK&5~W27RI{RNJ`$| z0!#M+mVoRv3mVq=K47aB;M`%rQFMVhD1bF6VOjBlCG87(&bBP=6Xpu%l4&-txGToz z>p9o#z&g2xCAOf2wXBI+tRV*MCI23JD`$5xzq{%rAhod5SmxG*8IyuGcV=v^QrFyH$T9CHrv&GgHwXBhKG<{n1NSZi zuGt&7WN}1lC9gjur=Yj{xRL z7kCT}{1^**qa^(wOHRwvTJAfkmH!4y%?H+hQ+H3Ce)`Z%?};@6tnvm7-bxHA(@Q#|S&3|FGvA_uktDyl))#igoUlJ`ls3AQ$mj;GeKi=4OttYWD5~ zww*;Rv6H#>Hgd0W;A*xx@$LJ*8=F{HE#O+{z_q~PMDrZ(<~dwTK1|-ep^DWY?A8aq z_ZN7tY>=L!#xZ>|_lpMZ5(QQx2lh$^W+4a0tz`xYEDRzA487I!woP{Bb;$02I?s2) zAx)n(jRpH$jn=pZ?2Nm>x};OJ&w*=20H@WNT|E~#dK><6^iJTovL@G2=ExWGBWVq6 z8rp1pvyOQ6F!wib#ze*IUfI+-&G>-yX8*K~GV?r*rO_V_^L9J!)i}j{^#Wh&8NT0V z_^v+KlemX7=D@^*&S{IC9ON$^&vl-yZous0VD2HndG!MiM+k>_EJrr?!c$w=%_gv1 z=i*Y-y`&V&p%}|CA%J700khl(1|tKG?Gt$KKHzCsbE3(C>+%J@e;-WTComdrU=oR7 zh-_gf@nU39VDP=9viaDuoiT^x9d=HY=BU}gG2sEn2`^3!MlJ#GiE$rT^X4%0PvAQ9 z=gfsO932H`uk@VF%w;c2Sm~N`#G_}oMFSIG7gNgvj>!S%K1an1{Cln1gdFC!#umO_-gXn`p zeheYq7tYAe=3%`hUs$kH^d`%EGmd2qToV$mU5+`Mxi_~=;fT{3ww$}y(j3_G|DG+> zyo;V3eJ}zRZF5>H~`|8LqYi>H>T!cri-+sf296^RyOuC@z z7yo%5WGFmv`*YTF&dS1GCXEFNGaWc)7F>T;bM9cIhL9C61LvOR6EZ=^7JhJk*j6k1 zf6tBmA8!0(;D2}F{F?&4DS!85>^Zi`vXkor1M5_=7mAgV74piNO!Wp&w>zA)+PDAo z6c$s3Wo``rI91Nxu2JAPRe5pO+1pDE*o_~gg(Wzf?UP=$foo4J$Ib(M?;r5m&E(#^ zV6vkh!_Ju6{-<~E-QBUH#+5aJMKfmS;tPCBH*73h$8p5qIYa%`^njgn5?Giwu&+ws zZeDl4eI7@~hx;npAE#+ONHbuwH(;~6u)w@wcd_kDiv{eZ0qmT0FP}`muH!wU_2=$4 zHk$8?8z!*WpZmz0C39@gg;yUK_%FWRWBi|Y>R;Y{0eb@iSY~NRt}x!n@I~N^X{E>( zfe(gyHyxfXQ8>{xhx?zYl4=rjXaI+E-^D347lYPwonCu;$p&^+C1$e(me2%_Jqx*; z&fLnr&-?xZ@6G~_i85Tb|49|4Tb6WJKZ{H`{UFz%Cy{Ya%$W}f{NmBiOZ=XPuibbv zbZ5;27SRLj+Y`9^8aO%&e*V@y;=JdY%4)W>4?j~9J|-m`NiJaGoy9y!@A~9(Y>MmI zx#ksq=;nOxlM(modY-mShbG^)Qq2id9?ty4dnJ0y{)W#V9&lgZz^!_Y@BRng7Y46g zUURbiZOAlJo9FwtkmrpAqmv{9`;`g2=M=sM*>f-1&|Uq2y}^Lv+PxEsvRub&mtHR5 zUCF?;GnQk?2likE)+B9Kc?Xu(gp6(;fdwp$hf3edS_Cx}J>^pM2oO_dV=?h)P&nAg z!XTt$vEhNzaW*MuGl7JKC&ktMTn_&GJe@aqsn?W_qf1mKPf|)+5LEZjctQ87n5hD) zUMa`@0vEeo)R9e2@0qzgWkJ_PLpSz5ndE{T*G4Ay>+5oE#9VYXyL{7UhLP{AmY0`w zcbuR1bdq)MEYm&Lwqyz~*S&Lp{{t3&w)S-fiANqC=;Z$XR@%C3wav$0Pfu^si<(yJ zn5NzrYkp#n*-{yUghk9uTrwIO3l1=4v#(E$-XI{|%*w-OaHO*B?M$tpogR0CM76H# z2JI+VswBsCr8r>cvX`^U*3Q*)Zgr3}ayfND#dC>==&OBRdy1V|);8}}eKBoaB;WTN z3I>J;|1z^X>0RJRXl!8q!@Pj8N9e`_o+)!Aj>`Y5zp#--C|S(&nA*$^#k7f`LU&S_ zbv-JVO|tIL(h@Z33Od}kbWRZ$?}}wo&OL^kZgpv|JK(gi*W`xBVm{jnm)APm)Ec$* zc6M#?mfrjA(`o4g7dW3A9N}^H*-*yu``N7Q62YUB11lc3iiKsY%-MX_BKi6BK(m`m zIxY!i9B5?Z%2?2F^+m}71}j~S1>0`?s9`YInq&D~GC0JrWNPFQWmXGkPtK#3&s{84 zL_$kex`~I*(@qI~>kt?c=)dr`$@JV?nd;MR#SXPivE|#ikcmmcL190`=i3RkQO;_V-? zaE(#oolFV2^17c#C;KkAv7mufYR2XU7K<_m2KF#+N&l-!mwg-Al@9oJrqwX_EsW{X z_S^ADrd4E$JFjc%LLbMMCf0uHH?O3>;Jc|O{`$twrviHpoP0XLxyK>b;omB;3CfJ_ zr+zRnYFTJ54Sdk(K0U^Ob*V_Qj!&C}tJun8vI&ki6FNfz7OEV{`s^ln)FkB2H`mok zJ%w)TkH{wS77N>NcelOpi+%EO!@}z)(znb!R;0T1BU^m!@2}rKAKiEJpX+pAmzz%> zv}g*am9{JP%@FsuD@2>0_KFRvvok|>9^ElVKGsCU zy|C^|u$Oq~QJWW+K3iHZ;GW51<;xp*bXM8DldRLrcNofNX>uqmXuSPcULgGM3bg?H zb-V^jT~o{o+vob7d1SPz!9`Q)qf98z8F9_VCUKJsEFn`430{>Flv>l;X0m0W=B+Bh zo)ngK>uZ)4Brr0Gt|(wF?p&&q>M0&2C8%@&{(FKPaSSXQP4YsL-7eC3oIPCP4FK5?o}isEEP%@u~?&XT7rp8Z>R zW$_t~y-pLHGHwZ;FTON+o?m8ru${NEMvq#@$C+7i?>`({$>-RiY2%R1t-#>^$$0;t zJL~IOcn+}Fv2Kemc_ciuuwB2T&@*q0= zE_}Y{rKP*gga#G`rh{|e8ioSGIGbor^g5#PI^$Drn+!Sr8E!q%BhOnwTdT;~XBirtyb zdi21e&{G~f(HFUTk6fLqkfFruwX`WHu%dONLErTBMDg7-KCVmTn_N_Wk=y9-!|Cdb zYHPY5CK~T7)YKI@$m!rRS35@0u(Pr=X|bjgE87D*t`nY&&rH%qe`=|hnkksdF-J0o zG;n75I9p6fI#Jn>eC)Z4)22zXM-;deBRv`(HL&Sw$Ue<*_xEL%eP7Y`>p)`f(uqpk z;spm9^q3eJN?epB4#uxds%~HqNMMkxNZD-T$Q>slC?Vv%C^ds|jYf(CqkM;prbwaJ zizsg%zsA6SrAHM)1lkH(G$b4aCe6AcbZe51mdfdn>r6W(n1WY-)>51F9=$x_uf4y<2OX%NOTLzF!@*ejzvocTzmlE}rmvsnVP|~IJ-sBpi zdy2F43JZV9mDByL+#d0#?O;|=I_U2a6SjHIqj`?XyZ64+`664M$!U%G%Auk1p6hEuTzr_R-u8~`d*&tbW<@Yd-|EmepK;{OhAz_!_an~4 z?%B}DduBns&mWgJ|DNsQ)VEm3A-%$Z@eVtKw|<3^!fX%G`_JtTR{iqj_G46Bv9R}< z$`bCV6HN-2EII>3ltkJrHp*^GYPZ%{l-u{CSy5~wN4z-Gjl0GfDjZ&2nI(^9S8P3& z)W5l%TfjkCeBs&&$<`^1vXiozBpR;Ix^P>?pKaa#2d!_e{L1AuQDBf`IIny!M#NU- zXp>ISsrv~f&4v$VuXvy}hht~q8^2LZwO8Ef(=P7g$_wmojiEsQ>X`dd2Si<6u+k{f2u>HkubI z@ZVg(rk8cCRk`K?bI^^$6)FFG1D(G-j85SwW3aC-9mE5Qs>1suT< zOoGlcuW4?)Dstei_5FxMoBdm}m5pyJJlL|tiF5sjE1M+t?6FsJQ5O3*Ep1J?)Wr(7 zq>YWeKhh4L7hyM9Fn>v5VZ`Fcq7EjDx(lik?(X}xwDa=sg1+MKwNc4T63c%FJl;2x zU7oqpVym&pRaWH;NABnuho{zuakzFgviPK1@%a@rDSufg@Lz|;ac6pm?U@Il{*;)~ zLD_&Q)B1O8I2Nv(-tD-efq}6>cfw!&e+mo>8dg2*XLz#zmqde0c9Zc5+l0&m8W)<> z8W>q67-mgw3EsitmCoR__Lr<_|MvjV84nw(55pJXi1K z-O<2a(ZIC9>B@#f-J4996&SofA9}#-Y*@p3N~Lvx%liyec;r;0v@4UEbLO?(%aKAqr;xzYC1cv;deR;w2c4?g$zZID{+ z$N*aQ!qC9bv77Nn!{o)g*=#J8H!xhD(ZR&P;Q7y>HTpoytPB=jpO!<(TqY~@4svcb ztX?YIVk0%d=HzER-wZ9SfH`NJ7mBW8PJUwZMM@~GV2A9Sw#$!=?YIwh3v1^cXxp=) zJ99-_`UVl7L#&$)wtf!eD>q~fR$%q4U@5U_aa3R_Tf$tvrrB~vv-yT5%?V7JFB+v@ zFiO8*3}$AZUo2v$(6W4Cv(5yT1rPg9H*pyWG{+RPxL33|i?B2(X?)UPamr{qwW?9Y z;>e2wY#qB=JwG&CB{XZkXymJ4U=(2RePhq`#{NduNyZokZyN^I83!0IH1Jw9#s4(( z{Jl+i!CZ|~99G)4(N8_7MT~Z&ni!5mr<=N5j zSxLqXPjm@2_Ez*PzDgNR9vFcd%4Ys@!r}x}x z%bQ_g|G9hnk+$p=ZdYC#+Xe64{bcjy+ne{^Ft*t(Q}Uo~r$oQ4LCeyUP1+Lt9ugd# zF)XeI%tj83D;-$83Rs*T@Yz|gSf1&#d%|oLuvpTd$yukxC5OG_K&#IMmfAb6q6Zq- zZ!F^cu=l&7N}q)v%L*xPmxd=S4BkEsQ%ZQbPc-n{Xb@g;j`Pmy-wg~L8#|0I`p+!umnA5)}F!iUQpxR1yhp)jqjwJtxq(cl45b5;PGz4iMI+ZPCHtE8ZgN| zZSK>VpT48TM}g(aA7=Z8X7dW>jU1;p#4xKaXfj&BWH@7~!iPp#31)c#Cc~{qSKjFo z&R`V&VJBNKUu*4bI7T19o0NZ=7Sf_k4r1A ziD8sla9)(Bso`~_!U=!hh|NpAWJI$zugvN`9(YViac^a38)l&rgkvkyhUjDVT)ZoS0hq|x{D_jytq)akuBy!-)WT=+Z#;EH=0eW7JG2`8r|5J zGPP@4<$k%+t#Z6A+RYrwFPvT0w0#O>FUn|3vuKSrXp7#_63k)axq-z~A)>VkoHDGdIwDoO)=>?m{e{X}Y z2@5paO<41$A?&kpi{1&o)t?(|Uzi5$VEt6tUJ}5bnbDTIr#09kI(S0M_8BeS9D7Rl zvh19Dd#7oe_lsbk7s1I6Udah<`%K%C8`uuEwiQOS9g=N7D0?U6L3sVu+jd(gG+uSs zW7y@WdL>#!aZL%M>r#JlfhObEUh!YniT`nxOfimeQj~wGx~+hF2Zvg_^4^RW2lqOt zUHg1trs(Qo0UqwXu>!VNIYWEY8Snj`6(DzNddyYvoEKMZC0vvRuJ!*`KQClxtiYI> z%4F=&;(B7u=f`q(94+=7EKWO`bRP&fK4A8k(06fBds#=@>Rl~q72Vt|w@ZCpy$xD} zHvDS|T5#KI#qHg!5g%pFCndBcMX;q=w51Bf9QI8(6dP05%2r?cpkB9ai>LJG6c*Qn z78ipE>kWDmf=%DHcooj{7rf?m(%RTP*U@*uQcvGi(Yy9Y-m%&8W$9`z+l(2C%%PKt zC9a+g4BaTjlkdQOtRwQ*tVzc*)aD$$d9*?(KY_jHbm+z|jl373pJwdiXJ8aCI1-{T z%lU(;@dZX>i)PmqEFKXo62ck*KTN-5H)&f$@&B;UJ;B%N)m~n~UZrt6)#o<%9#-!i ztey`ZZ#^6DFfH19$L-yB?|Xf?@2%11BKCmE>_MtW0;^nm@`NXceA`k#Tu(mnB)Q_w zmX(}tKb{=?C*8G&an4tfCIe?iVUtzsF5LN9!}#mVo%2smiMc!!;%YkL*(_;u*Sy)d ztdcwJK9+W1j3-;7CE=${`pRTm!j%bg(UHg33 z_M{}fm=$e%wy@=fw0bx!e#>|^K03YOQJ`G}OCmF?-2>*ZY^Eh|*V`7D2604Jd9Z)u z-MdI8yokr9I41hHZmWab6Ym@McW-4$E_ku0_r=F6Da>V0nA_M6e+~Dld*c1>y6?5< zl#eMc2NDAHx*}ti9%WWs-{CJ_@KjVHQK;{!tN@erjMP)rt0Z*9&AoSQn-IMwiEXFA zGwbfQGdH`7Z_HUd&--86KeiGDlfrg5E{kKz6`k|{eKNCw{v}N9q zj5**I`@=n`qs6yytMKFH!p_;}ZuPyjQ1~c#>tTe>yMPrJg>Kj9u@xMB-1zUYecf&E z6Y-95FIw9;kL-)yH}yf%f(+)e2Zb-LAA0)2eObnF)iyshHn#;VZXBsu6Sq6|dI{;s zgo!=;8RNK?WykroFW-AT{E(Hn>CRL6&dt#?64O+&+^)1O`PO#l?S;k1^x0A`sJwd^ z8T+hO^U*_lrMXqi=Nn4BZD84A_xwx~i-nbo{g$&YXRzf3 ztZA9?x_H6kN`v;`gtp)fttkP~jraaN4l#N0%e3|2K9<(B3@5%De`BZb74<(*`o?Ep z%Bg1=KIc+6{lq@IwE4fw5me-=3|_iW>9UmI!|7&kr;9aR*>>LewQE#RVzHuynZk}F zpSznKd@T>JlKK}?c}k{O;_98dxoSV#mT&u*CiKXjo4w}Y%?(N>=|Vk=uZ8}474Sl= z-_x?CrHf^Dp&ZMmcg&}B&Rc1if12W{(Bj#GXK$DBMl4}TV6y*ksw{`I?#=|ZRF3H%f}B(@E(3Bv}ZTx zgnkYAp17_|D1QcDv_)TQhve&*1}_Ac`ecgCc`5Q*ZB|6HT!T@IZd_D~cYf7^H$`@A`sVg@MQLaN-~7ij4ZK?2Ect^x zSYHG#4iJzspY=LtMsoEG_WT64=pC!}`Q6?=A-d>la&ScPzpVZ7f2ZB{N(etvc0Ff% zXNn_Rl0#eaf~vy&Hy@WIqQDxIp+P8HZlumeHXXtALbcZ~KOBo%|8*`S*Aol=+B&w98SJGM?A0Fax6D}ceA&L& zeJhl`zJ0Glp>I`o`gQLaEhTGPJREY|Ka_bnyy4m?u+p9Fsr$5Id$y#4y65)a*M|Q) zcPxRY@52&{iG~SXo4!AgjR^VR|64rZ_ecNgE!Xe-+Twpc_27lIs%FyvDr{#QT)9fi zXNOixecL|uR~ZWI689e}rmHnru%DB*%4a^<$j$yv+M@8GoKeb*00pNO5neTy?*=mF zdHnacGT+QoIXOx7X0OXhzC?=>j-y+J)!#|`=3UQA+FZsvdtK(PHMZ4doin!;UYcc^ zeC^B)-L&O1cx_8xO}dgY-Dj>*>D5cB-f{ZAt5i5ID{tzTnYi}Ji9l7ayt_imZ9aK- zb{&0n>!^3n#W=SULWW){5gX>ZWC`8e*nI4qgh|pd2f5A3?7gS7!!GPtQ`tH}R6lIT zi8qH@Cn%Vy-4;kX>^ec#F^+lRp|(?^TAAyXBo;^eFS|F>!1vIVFW1(lpK0={UZ(qR z*0yJ7_d2%Ay^FDEzP7f#dxt@yQ|o<^FrN4ekDN>7OgCMr{Cvx&Q2YMnKW%->ybDwk zH#uL|`(O80^y_hTKJ)W;oH~@Z_gv;N@lu`a6CCPkDiY>=*J-L|RHZ6sgtVWMr)vL{ ztzuDq0ZvKLT>*zWlum58v8cQ1;ZY^!I>jcR-Ui;Zh@`9=2bHCg4Zh8gn36gzGJSUK zH_1t^%fcd4=W;q8&fbtYsV#R$;>_7{d%lXvsQov%Jf-lwi?iCyGnvVLvo7Yc+_k)r zyZnM%QPtRvkC zZZWsswTLfUPMh4yDV}U`qhRw{v&(lJWVGarE~v8*LGh%Wf9(M_HSx(pRLli8%e#6Vgd|`KUp}8 z7#J9I7#J8BBp5>&+xd8Sgn4--dAUS{c%?;nr6oA{_=NcQq=fi|B>9B-1x5IUBt?Wp zq(wz!Bt=Cl(b}()nqhG<%q0U>AQ}TV%r(QTG@MP1U2RRgY)qWZtX*uaylkz# z%uGTojlDF~at+kHWo7kzn9ZXkgWdV^{KO*MG~66ay`0Qk9c{c_to=M~eLYQsylq3> zEdxCb(>yfFBQ(=PEi0nTviuAxQ;c%6c`Pj5^vryf<=tJao!xC+{cPQ>okPu?0*pP= zeO#P;yqrBqs@OsyT_(U{@qPE zT`iSUJ1S>Ss-D)CxpaEvy8eQ#%Zj(H^glY&=lq(0bCdGVugiPBu;9z*fTy?8wmpo> zD(NVzXfLasoLSTvoz>h~+w`QOxw?63Ra19YOY5|**4b0q+q-7WoH}#Dgc(a`b}pYi zb?uD))r-5gEbCgcc>3nWQ@5<1vU$nut!roRT03Lcra3z{&fL4HchdBwi{~v`xn$Ya zRZDkoUcPG{)_&t8B3`|sb^AHP0+`TF(Qm1Fx3bNplY`rzW?_WlD+EW(c@D+2l@+L^??=6GyW zk~5Sz+H-P7;o@UG@fJxuo}W(7(AV%^7SlQD>2$r=xMMAv!D?q_8_d0M=VmLnxW2(o zlb0`By3gqa)|5%U3eK3U7BkJQ_f&}HmDNt0?Y{2(`s(sp-@{$9w?(opEwX5T7Skn? zCbGTYwpMQB@^yE2$L(>LdD*4o{(k4WA0Gsj*?8<~J}CHQY)EwOVUhQ%(P&eW=q&pX zbL?~KLyw6Hwq2S(m-3yus21H9w$kY9X+y*4w$77JPhXk0t?*vSr%C1F_w%c(0$&+t zp5N}gT5SHQD>?q_A|@A4t@^sNWUX&w!QFXN-(1=5cJm*bD|fNR4~=aXi#g@)aG%;; z`tHE`$=AClSk?b!l6=(c@$fKW_i&JA~S5*3+#zcBpj%H@5c2IcomO<(n8 z1geVnS=2SAhUDK2n>~H!x)-dqPCb_Hbw|2#iG zIdA8)U7D?z7k^wgclOfE=2?+-S0+ERI2pg%I^iAHvn3A;&RQKTl!|U*R(-r7JwATN z)Yv@?ZF{aPwcC{>C-CjF&Hob*3|O=RlGdm%KUjFA^?vumUlQMUaQ=K#Xs!}2k>>3D z>+bpXf783=pS4s>7MS=>ZIyOxUJ>(@3duL6Hv}(*Zxd!^w(8CK)#W8LiD9bh*>l!h zGTm=oj!vs`^wIUO?7tQF-D+1&rq372;F)vWTmCKKWV@Wa@avq`iQNh>S`AwiGjeVm zo-jqfZt82M-!&5ZZ*-iV_%?dVbDuBk=2>)EEDiP&*8Z`irqwIt%uTO=eNJ8rdCnx5 zuevMpkyY$=?ZuF%7f<9C;5ct$p)9w{J1O^9Iw1MAfOUnfI?O4o+?NJWS{KRv3e<*37Q_H_3V z2ewVJYnr?lickMB_2B%kmzuw?Jb!GK`}Ti#cxN8hx?p54E6!U{_pD^evkRr`7Ue7~bCi8O;mQu>_mkAtpDl0XtW-WN%qwIKg@AK*Mdm`4?n4a17 zaDjn-LZh7htBa@p1Wu^x)!U}!e0bj9PL*puzGeG6*SRn2o@DhmEwNh4O66Oa?u)fk zuj+KG_)Brx{Ip7lo8)led|B_71!v!qhp0&WIZ|2RYr^JSUY;-g4g}Qr*h)+_h<^;!g8v z`cK>3%p86s{{O73mj7>`H~qGB`p2~JAaB`*f3vQ}B&cpF<5uY?=CC-x$nN35#H_)< z>NcSxw}#p5<@g_QcNp7E6rS?K_mW641%3KSblkRq%`6*XgZ)vWLbGay6w8TaK&cv75vtRKj z{9)j5esM7Hu%zP5gKL;g9zVEgQP8+0)}cAG`2n+Hhr?X%2MpXT2bfhZIErm^cvmFd z$RfLdfpyo}Ty_x!md+PPt-pL=)ti>CopnVvu=BZ`ersa5@Y?h1t&YjdNla!v6Y_A2 z8q-||wYI zMO}&i^k<~UNjpDL65n;fRrA|~X89)vSQRHU^2B^#l&ooBmM=JPQT9NiwfBOTccwWo zvK?Sx<4Is*Uf{rJX3(URqsW_e!$sFjkZaTGgFMj&PV5E(Oxy(v7-Sm0o85Elh?1A| z-a9krR-EhP=&zrqS31TSXwH&Xvbn{6{OIE)kEgI7J8*IxZ}~%cdA&k|oLjT>&lc@G zuYI(x%@$yG+!ZgVhCZ-w#AMW>W+eyPv(4|mEDvj7eD{Wd?T#gr(28BmZ)9p&UFsQvmLFj2`L}`9slw^a>>Dk* zQG#62HWitPoa^&m~>00%;Wj?`|yDytl3NDog7$H zTVgnlJ9Qlts9R!tUszJOT=3eGAfgWZ@idna4=%aM5XV`#NHYE zZld`z$lcEez`rp+^v-7L7vD2*7OCOX%cl0m$S_k zD0>>ty;XoKeL-1)czK&becOi=55tTtTjb&bBKbcH=RGk~a0wJO^(l4H`Dd^ohUH>~ z>fwfxO9l5n+loyROetu*wxzM`LF1>#l}|$r{xhrieH0CNAnbOSHQRjPdven4wv-_DxI?a8F<2ZLTR#((S!!h?Go}+75H8>uq66Qo(%1aoG7}|qVIB}af*2T zqC&%`4qlIhzf5Izn82);*}PMIvY&Zrg&V`Ri0-X7CWob$?m5w2q1Li5qx39CFVBUZ zjW0^BJ+C>Z-XhWQbINLX5nKTAiZL zMpl;66sOJ;?*ED>>D+A2G)iV?eWCD9!EST#Zcn|;JyR~78lUI)|7pJiT!-X9TyDJx$e`5%eoGs+Q=;pwYv4d@40N+Oe z{`Ur4DHB*8o8?6yt*T5K?@#@VuUix%UysM%|lR? zT-mi*l!0ZRd(SGFKkuHK6Sy@T7BnT%TeF-rx(`_~*gvQ~FrD|F0^hv{d@mFBUZ34^`2$M=2g8n?ty?qF`8P0d zGt|D_&XT&FE#)Woqb?4a=>2vRw|OpLPFcXV{Q&2`M;Y8%4@&o3;C?K?woObfZHkP$ zYM5u@@}-G3Rl73ycvdt>WmYWi=zgeHQ(`nhdgm=>j;T9X}LkAR2 zco#66F|ZgqFr5#RSf0Jh%qG`Cy<}5&Nu&g$_=4QBWpe^f&DM(AbD3-L*%!NmZI15o zn7xhTXvFndmu1fG*wGrFu;wnqR_9f`j;D?$&f&d$frZh5VV7s=ibzJ@4K>%Mm#|Kl zmmocFs{_x+2i$HKSW8|DJ3MgkG?0!yzBtYn>gwT;>6!!Etsn-0cIy>)Es)$ctO$M#&&S)H(Bk6F}gr`M}b&$)73ddt37 zJ$w_GcP8*2c+MNO=cu3coO^S48$manALbS4z2VNlVlX-RFT}{RE z$nt?PFoE5aVJ>q3qpkt7WI;N20F!(Iqlg1j=Y!sd>D>Dta6jJW8et$77ht`_hi7fW z+=T{tt#9~BUZ+&t)(TTp-E^t_ePft{SW;h@`QfGprl3r&n@+q&7Mjj%yuS{vZ8)VO z!*lQg*Y*oskKHdVdWn$1waRIXo&g-Y8h9^A-*xQbo#eo{_!sL$ z1KvvvtQHC^d=AVt0vr_#j64cc1qGNC16UR^Z2KL}{jxnF@dNW5OBtU_tlkq?-yOLA zZ{~#i(E?nb*x9_Bw#PW?7v0qQax<~?!FmO@B$2ln4P6)84l=0=sP`W9(Gu%y5?Lw7 z^GNLu*V+S|i!MC7|B_35-Kp&x4jJg?uJ_H=j61v_vU;aFv*8C3V*w_U1oqYh4wHi0 zSJW3V2XsfgZVe0HV}5J3bN3S?tu>lbS1)?pQ@_9_d0JG18x0fFm1WLiz!SS7_%6* zz2{_5uUz4B}7$&<-Eh9+KhVQajpu@@57vgLl zza)Cyr^Fc@R}!pe_Ru@R9^-jQ-G^tt!|&}MmL?0ZJ>AB+e9kGIw~Wdca;|>6b^hB% zx!WvF0bGkOuqYNV>izq`+`52wsR5g@0LQHc-YXl}#RM1*%P`t|Zat*OD1Tvz0kH0uCe&QvGf7YL;;S<0A}L`M$rI92El~~StXtRIVgpm zJl)D|%;bBSjc<;5=S`No63)F*yi7c7JQ@lMm=%0yTjge@di#ng$1JG$_-IM@B-NM{ zwa7;=BUVqI)g$rqO18`bJALACQ`Ld_ByPbdDj*1tzD!aqa+_R}Z-(flV#599LEuDhh?r+4`8>gJ; zQQWM0dZzCYuI;CUoQ`vdn-o0w$I#@=E-t%tx5TBijjSC~_HvDv**Kcza_(*D?0nGB z)Xc$MWWD0U!^5oXJTe{=3>=+W_)XKEtZ-cH>MmxQU^3yt0VV+jK0Bic2OR$V_$dGB zi4(h?^tQ~2ELT@HE_7mP6xFF&VEEXnRkCc^(#ag#x;9vNEr+}~)|s?;>L@l?Owy6>=tyLjh)7`6vwg9NTW{C0E6zRJzg;roxBK?! zF^^M&X3vJ?M~%l}a|vWD~NVv0`ECU#=Ag7}=#5 z6uB}td0vxFNxYmU;;>B9wds+B$N~oC0F#r`!m=VedsG&z*vPH-&ey3^=ZeLe<+?T! zjRFgWJe|~}cYD0jke%t}r6rsA=~1`-s}F&z*O{0Ytx@cgd88wkQn5)_d*hT%Ub^e= zWS`o{AS^6#ny35unTonhogTwu1;U$5oGK+}_)KlXKRDcPyQCwDzeO(5seyy-$AflTuY+kf^EPR=G)OIT zTs}o6C&OxPjNOEU22QmE2gZ*)4Gc}fj}EZSDcQpK;daT4(ruj5PA3>X^DOFMamZ8s zbS%1bi{jpinag%6Fnv@}NNC`@5Rfd)nY++{m*@S_xtd0sZakWnocGd&*L2&71<~nq7OJ$erbCb${v*>u|{f3o%8f|Y@{oZKtEs;YlvEmcU_B&nKDOxM99PXR^ z@y2UkQyaxMqKR>wYI5aY*0**W9Qo07_JIl;$KOYF2R68h{rj?{d12`?fh!Z*^uA4K zcT!j_;v>MwJAwCv`)i)#1ox1~!un?kt05DXW8>RtLR}b{KOBRD??=l_c@bv}hL6 z5MXlJ(jj0Lc<)e`lI=@bSFOMWUPqMLroN5s@ko&5E5aa`k=;VH&0og?q-`#O81@)Bnz?0CfC*xksT_Ah~1{mCNfB?(P( zK8h2|GnDzg5}Jfq9Ib+%JYefPrsU?o{t#a=5&Jt>Kwk0~gPu z;>8WTrd)}Q*EsVEgbhNO4JNezZ}F4bnJCR)Z@amN8CZ^h*(FmdYyRNn|-J7NyY0mm+XxuPLKYhDwN)bk{!)hdo?^ zf-~8Ks~kALIxJ!hoY7`cvcf54<8y}x96D_81O#&(oVbFnv~jQ)G<(#%o8QQKfML>N znWqK{47?RgDr!!GFmYF41A-)3l3WqxR4->gHet0PpoZg)R9YBWv7 zSFdw&j$q_TkG7?ONt`?t8<^|vy0jUuNHqA~eAMlJX}9s7lfGsQxWm%rbw6gV~!BZ)i?g zv~`KVLEdSceT8WaM{aSj@Jb0z3_9V?b?yb5#icMtUXQ{eGXV!?CWd)RkBzq|2E5(w zVeXdyuS0o($jNO}7y3+8U(GIH&Blt3WfYO+!SNych$Z(htqV* zK@MdO{@3Rix^e}4+_X~o~nHz`b-`+l^_`e$vs<>$b} z@}c37sypMNtq$pxvH=AQLHM)8IN%Gn|ftUV1EeP$%` zGMgWs;hHa`yXN0P9=`VujCM)u411EKCw&k&WOi?l{dqa9i7M}<*DwjcQH{Fjz;S=l zB_ZZ*QyOYNAHDQz)v2Y1Q;PHx7MwZVa7W*oYqefix4w$p4a%nk(|`dm$Kni>>67#eLVWYSkO@SbSg z;M{JrqJgJ^dFSsrS=`%$0~=Ta`rVm7eEn%bTlw3tFu% zvb@R2LIZ0MpLbY>o;HQa6~F|5VD`$gC)A&Kc0~ z#G$WWgMnQ^iEmCv{UW9O7i?w4Mu!$J(mu_|#393Wg4gE6hMW!-`2&1~$_$Jh2hVyp zzwA=ZoO8_RMw8$I^%KFioEi)^7A#o@T5YlqNf~zC+C1^2l!;~y=Uc{AUzM0tBZbs% zGV1CuYC7)IT)E2NioT&lv&{j?AO8+=+g?yjOjw=3u^~)BIN<|JcFJVk7wb6{Jt96x zDr@lWyd*4@vEWp;l{807jzfE(1fw8BqfCHq!Hy0?hBY=b8uwq){VUv(^PF3>l|njxUh+< z!)tAh(BbV$xiYUlbNwiP0Fk8V4yxHU=hCU?Ub_qsR7 zpKWeF%hUW~_x!b4tcQQJ2E1@=4`KF|==qmqvCCAUQM!Xs?t!y7N2AT0voA?S~UV2Tp4=oQyfa$e6%%N{cl%V@~2?2DJh< zf!!T}2ii&+T9_fX*9uIkn{4O|m~7+4rqOk+#n z(|EC_L_qd#S5xh0j!2ybJ_SbQ2eLX97D611?1f4k0sa#N)*3Xh#7MN-8B3^5kURJ$ z;Qb;l$Jc^yH%cfmHZ7=f-=v|XBr$*MkA`KxmaK@{7qUug_U@U_n48Z&Vg9Mf^PAJu z*pjCru-Wv3{(a7^(FLvbiA)k4%X&%}B~Ki-E&0c2H=`|Y2AeEHh;vS1>0pYLhu}Nz&lh44or^73{J9PBilbFlAY^XF4=;uV}P6 z%gPj?$*_b)rsEVBM@!HNwk(UenH(&P42`xGEsUXF;u~1>7*01PG<#LFSf60zQQ$Kx zTI9@nTEL<=WoY^z*(ysz{4 zmVlkhv}Q`ms!FZBrthITYw=F=o8oIvS^idRi`c|twD53vLu>eoRvn4PKPOJ8)n2ll za!GzdOUKzp0f9#T1ST&I_MiofTmsBF2ij~FFo+h2r0;0(aSkNTN!C3r}qh?kVnhfc+=Y3i`pLM-pc(G7p&PBq0{gtgMq`~{>t12wG}O~H&`oA zHZ5*o()ci;`r!oo-4jfwcvyVxc%9$RZ;Z!uTx1cShj2C)OpK^iUm9*K?< zT4WBKmJ(=DpJ814i&5Oc@kDYXf5e0Xou{%-ZjU#7v}~_~h>Sxvhtm1Ve;W23Oqn~y z`fnJqF>(rAV6OA8Za6u6~yCDLNAGBz*Bp)sr5>oqdUumQ*or z^RWBzFk{AS%V4FYGNQY7`I^p>4qGJo)c3{3Jk!*O0KFYSOfMRqvUzp)Y&pxe$VTiz zs$B{r&w_sWW&BS%8XnF)`n00q@s4TEv5O9@OuHCmd(d#|LB;J)5*jX?Y7k^NaC=dQ zbD>P`30B7+Okx(1pA4om&hV5v!R+Obp5DQno_9l|psgfh*#*IN_0M9ac}+5YGU~UC z*f%h+b9xtZ&au>HG;d>I^9Yr-Xi4^z)8L5a%We8u%A^)@$X+^n)7<+%?+WNU2=O`! zDKW_ZdpP^}gu-s^nSp(LFLeZ&XKQEr?Tgl9Xu2f9xImNHkAtoGh~9~pjEwcPR|h1s z41e@96t*{LEoKm8FUjDysbCV{(eUO&M@Z=eh6)C`2F7`|eP*W`q#Il=6|%padQaWE z{amlj)VGOybQ<_I7_KJDKFMfcZ(!iC&noV8^!(uG;?eY(>-Me77ALk%TrF`;(rSphdx)d74GI zYQ`=}iCK@1{qyK42@(j15vvFie~~G0ph12b!-1Wjm?~`eA_|^pF#dXA{g7!|aA%f& zb!hk=^`{0#j|EQcx_3`=Hb=Qfzk-s6#|~!sj7E-v31OuZ9$7GQ)+V#`Z~H#S9z*MqC-PVghR;MciKN=5V;5`zzfV8<6pA;q=&}6CJ-9 zADqIap|D=hVtpd3mWtpi55MT|UffHo8n+)!@maizHSVpE%u9AYW`!M1U29Ir+B4m= zWKy=cF1VvnCLr9dbL+y)T^|p>?8#|d_%l%}oue&*@mWFxzed{Er~c+;$=1`JEp-t+ ztT^XG#;J==#Sfh>AN!Z6dwo*JC%J=p4()jx*u?)ev9q!zTeO90FnsKYO57qQCD0Q0 zqqV}IZRSI^3lH0NRw5u*7g^+Ru(x zcwu~SPEK)7lOuN^Z)NA~|F&ii`rNF!E04So-L6%0MgQuui*f(I$}HWf?@^^EFzZqQ zUu2d*V^7BD(y%!xN>JI=1n zP$`r0gfY_f;NuPk)(gF+3t0c9EMVaNV9sZ8t0_NLKfOW7qe0N1#pOY(K>(BN1wq9X z6J-p}$V`w^`*6WMDn+23>3t(pWO$%xtkA>cOX3|S)jYofPwBJ$E1g|DUq9mQl6qm; z3BS%){Ceok&9nVF40!bRi2!x=hJvQicW1_zQG znJn02CImcUyvWzW@lfirs;7vC|AIdp{}wa9?pI4X-uG3gwL4PZX~BwCrIzjqI|@4) z7>$mw$H^Hc9cVb%)GB9N!SYyAhpk`2Hi@tKU~>9Lal?uGAI$v2dPZE|s-Qsa(9KqE zA)|~P2Oc)D@yZ%HTo#C(<aJ~YcaZo>3fna#a} zi^brAuF1v-(FezMM9@GGUTCpK*=j$(4`3GzD3{ z+H2e^Bw~`WW}~Xr1QAvl3E?Rl#VtM@U=>Jt@PJKLWws+5OA7;s-i|qmN5s5zE*=$p zBK43(#H?c~OJvg0qpTuug> zO-(F|logrm=`w$ItJ|SWy%#IhkL$fjopm7n9qaOWviol`ap=}FFfEo7iE0x|n-S&qJv8mDPm?izbNa%~(FJonWkt=KYYm)aP*4uo{;yMzE;Z{8} zWMho}Je|(d{nJ&}sitk#ysbMcrLvaHjfu)!{cuCtDcRd*%$Z6X4*bjDXvqs~Q=5IG zi2K`(>P4-BW$ve$*aTe;FtQ7mET6p0UHR@yb{>-hTP|OD8o+Ujt#h+h^`nBta@E=A z&AAQtZ0v29SR#G9V5<44#GVR?=G;3oZcD{2oHX^qA<;;SF!m6oZr*c;BxV-~aB4m8 z;*@$Ux^BS%HnRzhjxJ5&T9$5NF(0{-V>l$mOqzu5nNHI%IjECyl56G_A(x$3mc~0U zNbm<87uOVhW8imW(jl?KOOH3KG@7*Wu(%F0o6%DTc9jE+oIUN$R}QvSq^}bGm?6C9 z@|89=Wrlg&ZU>A6I}R|rdo-(ABz7mTEEcWhVf&UN*k$y>AzJ)^18++JvqHr`MHb-+ z3`{-W7+4J&CViO9@JGTxb5X3j6<5G4hKLI+jy+9MT#@}2d!F^SG&D;mFe`YmarBie zP*!68;^vjX(Gyp5hPg^t4sZsWEN=^WCVH3oSb|PImYe2~GK>#HCG2?=CIUd34_js ze3M?Prx`qMRe5p1H(Tqh#Ox3w>zir5xr=xuJRN5B-+b0vbfuTWB%#G5;=NdHi>r15 zW03u}M`GIynim>1Fj-Gw;%Qsp!pC-@;hn~Uj2o3Ie&(jK6I~We7XGp1l3kLpU=QOz zpeFSZO%gz@=6{{-`~K&+snuk5RkwtHPKZk=3!UFg+PBup!E<_2n7j=29xPZObGVVmMQpZIO`7TIk27+s+As8-WiwSi&^sr> zph;=ULE-HjnjV%5qwjv(D7tU!QMEn;Cc&D6QdJf#W(g?=>4`v!>rXeL;dp^zs=;=U|(OzE?~{L}zB1 zvbjm!$z!)l8lVEMT(u!KCEX$eX){U8dl-ywNtnzTYdeLfQYghUwpE z{8h#jd2E#cyVZlqT)dZ=MXFZc&uC;#+j(`h#bcpolr|ksU^fc9VsdWX2lCL;>IG;CJ|S)GXbqeGa7|&7;o3T@_XX8#^)xGq@hU=}KMklVlNj<>8qWJvStpn8>nMQ7J zO$*nsx%D07aS(2@*!h6X`^G;Od2g;0iGLP}Wu9OX@7d6zlk;fXzNao?T@4*J84E<_ zJ2*0Rxp{htt^GK)W={!|!sVH9>>1Ox?_pM&@Zn7T1Ey1)jT};Xota0T*B`kd?-G)k z;ZhuPiN$Z3k@ktA`|>{PO}1W+2geFD8YS3q0oaHQFK2eaaYM!pzVp_U119L@Jx zuL}vpDc3l0r8q3<=x0%xqh{!Es5YVW-=Cf>Aq~ti4P0Rd*b@%0Z(!iqz`%Rv0Q(D0 zjwhVG7L9y;2PJrz%nYWwaWFR)oUPgsbiIVvz(Ah;hj`=Dsc{z%NvXJUyzyNc$iVjH zY}ConxJUuTD^8124@$>K2&XijGL0Aea3W%dl)%wh(H(v3@zWiA4hIMxO^DrC=_s*)RY2wn$Bs_E9tLHlDSUT4|FQaKIXwE<5<7p1=jl6a z91@Io8bb^?nyrsG+e~4WOmGxpa1<(VoQiIOvUFC5_i=a9)1A=4L! z<|Q_p&zh&b_td|P!~a%Hmok{Y#IT`Rq(OVi)$l@r)2_2xYogd97{rc+p1m5w9vGpd zlE|LgD3~yVZ_-11HkAvkvM*9xFTU-L{&89`bN=K94eLzj@=Q6vu5#w^lglEGXL?M} zTvdz~Yi$%ah|Ut3BKTl}_|ne*k|O^;o;$vTP3EcF9G-JR1x|NZoi(}^=}%}7QaPwu z#3-(DP%z_=?j9%Yk|r^ogF-hPL|6_<7&wZ#GRnC)%7v&Zym3_Ea8h(gRt!*9>0#2G z;H1~{)MSdPX$rGxPq=x><9Vsg=I?Hrym5*Na@OA=P^peLMSLPpu)j6NzpL%pwx{(_6Op1lHGz0P2&HOpC!x?ny}C)luuwn zYrE=`2@+u)6KC+N%)P|Pb8v$L=bwjiSt`H&CeBcpeYG(q?S`Xb*)-32j0N)=g%b{X zpPK*YZD)?mm5)E}&pz`cZ$|_Bl!J1TPYq<6HI5yYoY3h1ib4EEL#wBw)SW}x+nm%S z7!@3x!uyz1o;YdFXwsY0tM`J*G)H;<#-~xsp4Oa8*Yj!C>vRxsJ{--llIJM-8R7+4R) ze6DO*6wqKeHzQTcQR#swFQdx2r!tOa^98@0`0P5rZSLhvFnYJ7V~tIm5>%|Op%C*m920rFG?4i)2QC!q_KigvgZK5K!4U=&RHd5vu@m%-g)5G zMh32i0~}Kv4H`rY?lf7;G+PTei~U&QKk48ui$;+gM~xo!=}&LgL@}EcJT?n*)|}F$ z*K=r*lamUMhSHmZVZRy`KQtyVsEX&vikmdf>p5Wax9x{ZLzzlLsS2w>!DJ~9DV942 zBovyAr?{&)I0h|P-0U-9$=kE{mdr6KZD5!pqq`)4t>Vu85}EF@13Z$=|D+@wm2NZ{ zg!)NX9ZWZhf4btbVb+Za8(5XkMmeY57dWu+Qezm8$zAR#4BUSn%0Ie%<)^S%E$1fJ z%-^YkNh*xIYoau+Fzpj^GTg}^P~kIsg-X^_2EGSv$90}(o^asVa)9T+gV}wvOFVk&8@p}E#&0=g9po3Al#z11P|K~wV51yL52gs(3wkN%Sq z-{6?VwQ}vlR+o1SJSPrh-+OpD_la$y*t-{M9$`LdDHFwCB#NsZl)S=Zz~HQ#(IjZl zvHus}$0rOK96Ee0FEj7lzx9&sbk6~?j`aMW6Q9UA^6i}EKWmLp#u@{L=BcIr95D`@ zAqQ9+j`%!1QZ_g8&yzJz5-^FBV+eHfYD(XE_nl0XtVpP@27jy9OFD8Q{(I-k7I11eU7P6)PVc>mo zKyi_q;lHfB+%pFbB{@8N6F8?&x z^Bp%fkw)KRn(7Cb)TfAi_h{^W@iJX(fx(xHnXa zE+!YDvW!#31_%D_D{L=4Gezy8&Z*=BPQ(iIyIfM>O8kk zf%D!xzKqKGU27fwzxpV5y;0QCCq+R|jQ10xdY-t;LGcBx2PQcPU18FHkgYqVv8#?D z{mgy-5=OZnjtV(5!ki8%UvZQRNmhKqz;WdOhd?BEOaprclVL}Q=I?8|IZQ$i0{8?) zBy$?2I1U@$nYO3Y;hdK{lZ=C}=K-dW@KW7|(ziycvb_e`?Uy=Q!oTNvJjt0^>JXUL zc=yDGlux|(Mbv&y@MqNc=cs$9iMf0E?H^6nADBdY9Ja~uan3lX|K*V0gC@NXO!_TN zx(v?RA#2%otl>EFLU)bhxtvD6x<;V_#-|~DLOb*`t0JNqG{4={cU<0gf10+xP3D^g z@?x%AQYF^q6&-Y~b`-a8G@9l)yQ9(ldXrR+qp!3G-xCMECkK_Mq^i6*czCL+0*6y% zl(S-k17F7ho}$_8I*e1kZqYt)NOuLJ*UwOYsjb>U3hGN5*?kgc&OHCfkoWR}o9Z8R z$^(wiTFYNjI;reRNSVmlz!C=iKZnFh8U(*M-Mz`1sOEZ(OMS{sP9GHVPJ3Ppsnn zYUeKB-pV)ez{iVMSq})j+_7qg>%48fz4GtA1lvD6r=Y&>5nrXa$WD2A&-956Qd8DT z2{0)XIA)zm;Ja}^_($VnF(*wECY6+CO_4+1*3AkQ&U$AYd@nQPZE0ZdD4MhDkY;np zn`=#yIgNAj8Qoc#4ZmDe%V}g6IZ$!_lMy%X!G}5r6EBsNxF4L^KU-I>R5xyaWfx1! z0gaAk#S{jG3GQYAj2;mUEEPxCCoqV=I4IZQctYxs{(~l_{sy6*0}6AR%mkQ}UNmWb zXwp2htKy^6&NZjS0~$p#ob`JSvSl=ItvK+Ht;a#B$4Q^3DJNQD&ylkJFH0g4bo!^I zrB|0tx^Z+nThI5s4wnz6asDxstFD&)Q@Gc2 za7b}ZqtFC4z8em_0Z&ahn1r7klz7u z_^pS958c`*JI!H7y|rAA^-lh}oy|{N4x8nazuME2!uLjAwCAAWkzJY$hj>l?&3)o9 z^?S9D!9j^9j3)EW9^^ix9MGJ9e4=A(v7h~(X|f7I&TBkl`wcQ`v~v!vOm!&sX`dPB zb?UvH(d?S#DWzfg$It!ya57ph*ulxl!AWpNlex@QjgAJ^3u{<;95`M$C{3_1bzsu_ z!qjx!dd>So>PsB;c@8UtG-@tsGMjTqeGb#=^u5+CkJWt`SMxXB_;1Bl!KCr#kb;Ke zE$`JQ)fgQ-3YC^}3afd}Qppiw^%7BxTfot2@v&1t(OHj+(dge1cK&`thkXef7rC(5 zl)ky)82Nn346FEreSbb(;`W&lyU*p}r&jOtdbYD)6fAT;&LNw~AbIO>t6Q*)UcrM6 z%Do};9GgCgoVt8srb2ozi!ZYgvwN?MOJ>i9fFsw|hHbcas7cjF!Z4}iCEMdICwnIH zigB1o7$hEI;1rTDkT~Gb^zc}hs!zZg1;eLDri-h)t+`Q<#5X~fKSN;$Bb)OBJIk9( zd!=_|da^_;c(C9=JChg3+h5C$3v7$sA@udpv9oj6YcAiC`Kjg3UCk%U{60E0=USNi zhxlYlr=&J>@!n^jdC4`SYm%nl1#QRoEuF$G>@!tbjN~6VcYfCm+hW0T;vc`fY-I;y zo4AYB!WN;RP0IcHp8{6)O;?z6Qf)@ruSIHldpaI<>(#yLYwWBPUL^8QUk+uucnaG2ApY%q;M5 z2Va`>f(C8==!C{&0>(O<8AT#|e2(gQ=QN!?b>xi!r_6((CjL`03e1j|B%ku+ZjgK; zP{6hEK%;=lhs&)lWe<;TkE#0EJ0Y&_=TY%oJIhw-kO|JLQ{8<(9ua?8^^$ex+fBcp zSZjxT=o1JivrOUq9`-emv++VVzoLRjmIsI4v)%7`|LAZ%S|s20LBoMDwc?=zhseJ< zC;Mi^1ikD|%{_F%iM#aEBJQ@5ETztQi@LH_EG(Gd*jA7{fpOKQMGZ@)1bv!uXl6Nk zAS0Kf+`+|d@;N6Ixf~QuDt846C>n`G_qA}ZQ=eb5RMhXv4JQ^JtsR>h4ygnw_#U54Mu&SopJ8MxSdgOaX%?BvF2kTA9n5g>*c9Hp3#vj=IvW-+*ykFk2pXLe z;1`Uk;LHmyZ{y6D@ISTUQHPd|_RV%rVRz6x3wV-a2 zb^G#v8vYw^uW0Ca)vlIfkj*2Vw;-v@@zV~*7Rgyz3)_}uZBXnFWSIC!*n{n(s)%1+ z;cC%V!8#)^Ws?&M%uH+M{P5A`Pwp|)pPK&QvGL_IJiksK`tyKUJYz>fy11J`B?BK% z_W?$`O_HfxF4Gn$wCET8a7brYT5zCAuty=0$B99bStRbnjYmBu9EDw4HAxc7Pq=6p zJ><>{IV-&AgR9^Q114>sM6NIcS2Mehro34SybBDDYD`OzQ{SS+$= zcqhNejRe6xf4h|B8yR^b64nSg98gV~aY4N1$6<{tDN&{$nSA#&DDt=}a+Thg>ejGf zt%?dGhZoB~CZ9zgTFe~|vY81?G(6&n zy3wMW^HIenuSq{rfsvIfpppB+GcTD52bi~Q*?2r=nW2jEHe1L8X2lH-94ucNPZVTU_WInL zsU~UUHObRArRae2ii^EY1`9>5EMQYH5@?)kdVs0)0Gp|s1LM}V1cr?|4jfz`mK(fT zB);!OtCUzMcU+F6uIq*-{ZkEHOj*q{4m-4}b$FgJahc2&D#0Qez}U}|cY!OpV~XZ6 z2ET$`?woHMLphJ%WZAKFvR!$$x?9_W`F&Rs*qIo@xG%r%~Bu#&h2IgayouhE7{~H5`neuqv;r?38@uB)o3{ zyJvv1^vwfpW-AKQZJ$?4)y``(UGdM5%Z}raK(oUU^%BN89xach+c+Gfx{8_|Etmx- zNF3$;bV1)yVUfV%2~9#ihP|4R4<)YX%caf6tiIXiFeJ|$}3x}O>n&+fZlGvrG<9O14-l55?L9xYM zVd|e3_TRlVb7F6Sq=Vwghnz|QOdNuGPWdNJZqZ3OGUr80XW!>0r-Gz*cOPZ(BWGWP zvTbB{STko~kXq!$At#H z15t_7L=`yCmpYyMbA&PC)Tgyq?G9@>1ah0MapY}U&uY++DDKyiB(Z&k(`CIz7L`4> zkImFzvoToz_=MeIuFw_VMSLD|*XmcER7>HXB7VrF=^58Urw+xe-^?xlSypUivAQrr z*(Lh!c^Burb=L$6zkY1fX>qJuaO7FZ0T$I42iVeEwg^p0Xjc27=-z&zP0waxZCs7A z%)R6lc&%s)1bg+mchuCwV>&r+>!WjWO4C4Adr*n@Oa{(b1TgjGpa0LGFrg&sX>a(QQF6m(`%uCLxLH{0Y)ta_98{; zT?y${40ZRk*8Ou?*U=VOw@}RCq4*Vt#J-0TU5))YJ|azyB36u?dWtjK6h$o_etN_x z*Y;2-z)^Z$VwP1umzsvivWFt)8k#)LwL^*g z{n_GE50)v;Q{Z}&!0+YAd168Rwgm1|4&0{}@SIWLJ@bHPn*-;u1tKbmJYN_D&L!}i zd%z>aDDZULFiXUqbk$cD_)Zn|LZjVq;dv8ge$#z$&e@hh@6;`2uAuy{Q7`gpj)s3b)>iO> zH7J2c{t54#7u-5}=7q-UuM)PmdC0e~R}@t`=ccH9)OVo06U9I1J{fMfwIJZB1#Jw%g(0U zSi<;6L7YKR=nF%c!UGn8#f-axCLYrhInJVU`v{{Cug>uzhCL2KDvkUp$^0sg0vQWZ zBH9>q8rYN;O8-z`jyP5oV8G(kz-+OA`B1|Vme2Zv=S3SF^`^DxbvS-fSufo5kp1XI zne&2jU5duR4~1@B{CuylN8^NzNjF!0%batWUu6!pE>PgmVw%^)5YeO%QIWvT=Ax2! zo=4Ti`?Z4eD+aEr&8<9rTcjq1EmGh)w!mV$f=FJYUXi;*9-~AHqeR|8o<0SBpM#Q2 z5910JbbUJ@z~IRLg@OM^1Aoy0)_tFu9TYV56<7=uSp6QbWHoReYX}ld6xr}^p@NSxbVDT;WRtw(bOnmH7z0U zp~sd799~6IZVNe!6gn#o3w&6>c}mH(!zr+NQ@HU1W|;@f0SlPY6!^Uo%Z@BQKShD@ zAJgJ}5yuz}4shEwu>D}*XM4#cO)%WfmD#4Fwuy8 zFezri3Y{aYbqwrxj+h_z?R5;MnNKB zibP$aw56g%PGZXA1*;VlHF*vSsWgi4Eac2+_@k!IQlLrV&iVHakZF?vi@oCMLgBMdC>)rKo@!rDh!O+V3 zoohv@C`WNQ$7F+!mMB5(R$r-oL=CXfxg9 zc&4qJcf9f+*bIq*wZ2o^c=FgPyZS$Oe>gAjuvkDH^= zp#b2YHJO<`WgP_SNsJ4%B4+Jo84L)$VF*~<+&tQW?V?})i9C~EMq-SA=2O;OP! z%3SRtCHC9n5>rb!wdSfMDSdi2zg>ajU-G#W~U%AGbH|hF^1DyZz5?JOe5GhdCWPkW$Z$K%S`Y>r&d19R|T1}B@1dj4$;Vdt5vlHh6wHI(uqs+rBPqV_FiCV%#32a5n9`x#A(-n*u$Jx6509 zKj1xdcBcZ*qGHZV2gDi}y{r!MSTQEPvEiP=An?bZM=X{tPl!e7Z}h(4b!UGIEo@>H z`^(IvFk#XoRRKjI0mVf^jUpwAf+1ah92VsN`zglaV)DFJk0nikCF{X;eZy+^2UQLS zdM54TJms;pq=EmG2Y**no|^)j+{P7d4m=OK1qu|WznR3Ol;FZV+hya=qvF`pD`7Iy}I<~&jn zzw}19meMxnDvmuH*c={tm3ExH;ge^uof=evwRn1({_)RZQgl(!FpylDq44{921^hZ z$CY4qk*`c5rY}9YM4TS-lw^uu>SFTxpvTzc%)uwSuHb<~Qs|mi?%1PFX*`lgl8hkUQA9^#&B5l@yh*ErLB}?3Aw8|=YadxX z_|8ssJTc*;qKSaG5SKtfz=1;&o*g9@7BD)o^WCs2>DW2($(fFSg2~%ncw9c(JXu#{ zipom^r;{A~r(8QOJalvs6i{-zv!YI!MO1co=>&<7vz$73#GDG$ojayW#qF+|xFF$R zV+A`Cr_z&xR8gZg9`&Ln8G>$7lEG&T6@@8qwILxm+RWsEEnK zEy3-ru*jv|Kg3%+n?!f~lcD)(P zCHX9#JZP1R%bCW^w`uK9M(!*ofd=N~ORupgb#^?6%rI*Bz{p|JvvAHoVeg6!EkbS@ z8(Z1k3O3GJqUeS+af9qE31(UVORwKqr^~G zCMkHQ_17gV0g7B-U4&Jnp1O8t8}u_kls_VKX(*RRZEHwm+?3R?iN1e69O2J3 zm3+!y^f%!;f9b2O?Ix`O8(lQE-#NsozU{`PLXDk$hO8==8H!eLo1FNT#^+fP*e)Bl zV`Ve9<&FoEi{y_yIKW^h?BKv*>0rdgp}aw$iG}aXFYQ%U8ij3w%1aKm3Z~>Z{yQX= zKB2&ERn%%O4#^ZACoQFwLW_=zH83O{k!W~vp;aQLU?aPDN6?4HO2r8d92yI+Y&vZg zUE7{?hNHo}WUD&S8H(Nvxk}V|4Xezq?jXO?R&1e}f}j%#3ad zOM8l#uP9Yda+P$K*~lW*@WIGKWWp3B4ql_42Tr*<)rsx0rFZ7+OKp{Ub@aSY#Dqq^ z4iBLw?g>-H7#lAr@U&YTmf+ga>N4kH7`KVbDu#pXA)SpZ$pX!SAxCBPG?tqwDKM6o zO!(I^${GOKkY3hcSyD>yH##ZVzh zyvpN{f>$Gl^o0Ygx-%NN-?T6*s4#Mrt@1g0!Q7tDLxEjb!r}Y_XJ%&wr`83U4a`o> zhYVggvd1Veb504+`rP9%UHJt=xs$_Ofj36XKC?wRdNi9pE!e;+!Qsdq^Py4dM+2+) z1V-+hhUR~Q6Ap?ms8E#T5N}#4`-EB6;)E#ICxNgHNs5~t9@%YBwN!NSj@qmtAoRO& zZO+fbj^}r{znFJoQt%t5QYj;G|8q-S_3|cmyLBFtmicvb`HB~=K`$BkvjUXwFHK-| zl9(rO>&4vjbLa<}g?=g+-!)mCJ$IMPw1TLc=l^l?mn|huAEY z1o|sxs5M2#DzGff%+AnbXxh4K0h9azdA5X1CxL_otU3&gftd%I#atLzC001{Wld;} zYhV(8Yw7&ZYJ*APzkBTVDiP}S6D~T}A26QhEYo8CeLu6jTqT6oX&J!&KHuCxy@Ln$_qcYn<=B`vB{$tVaGA1*Y7+45spbu3gxm zU?Scv2zx3{%t=i8F$0 zzxvgd>8UOdmff+af-6|$-vxrJf9 z_httcfvZj7bq$B4rxnbT;W-;wpKw)v!a>eP$%le947GY24j5(zFw3SW3hr3Z+9k{w zdE%NtJL|#oqE|9+M%)peuQ*X%FmWMQ=z&AZJg2T@*p}boSZa1MGa+|p)g`NIse-2x zJB!|CwJ$QynAv0Wh_RsdJJS{(@qfAw8aslezAb6C)Jfv0e^;!S)~xoBQK+UMTJDnp zSMZ$$tR@1^O5YZ+J9aw?Evt}DB5j84aqx+TD2BOW=?4CuwTg6$q}vCl*Q#F zvXDbr!Ldx)k;~p^k4S@rW6qw2D}KlP&Hl;h6rSBB_22?S!!yAG2R1!}Hm8n*HH_07 zR<7P@e8J{3vn-q02J1UaB38D{`@^OzGI9vn<|N8|?jVCky1=ZoqCe~_vc6SSP3&M& z=xbx_+30wwuT^sOXT?%=PXTRniAQSX?2}wPO@!1w@+?6SwK7lQGf|a~bMeqWa zwVzWqnX+vT%~#n}@$;M8pJUAFN0<#ZFzW|2F+Ue!-&~o=XgKS+jedqW-*X$cRPiJQ zjs^jyWCO0)cDASjwp)j78v@wv6WAXcbC_=^+94iab|}cNl{sbt%esjgjL#Sp42pRK z7)1*hB_FU%P~h0C$SmxZxOth8WI@8SLfLKNO#jX&nw&53(Pfacn!xzLk#WPtB87wy zx5X^y7BXj+8sAb6Q+!g`xq$6$Ah(f78Y3TJ-;YOI2MV%DTKVpeW7lkxjs+ z0-?{$RVvN==^;8_W8)b(COu$O-oToGB!ILvU!r7g#%ND5R>5qbH-=JLJW+21%2iP91|PV3QjVbr~8~-$Z{%><&>f3 zsf(=ACDPkAws!_-v9)NmE$~fRz!GD?rg>aT`2x$AP-(Z#Mp{#HlqYa-9f{a;At_ay z@uHcwiUU(-dws-)GX8@CN}k;75(HOW;GC{eUL4MIbOM*wznPQtDpSG=I9EL2S+3Ca z{+h7gv^gs*qF2oj*;E`IrpDsz5a^sB`tO+j))U>HFPl70PSO54Md&iK&e1xH11x3< z%$@?Ql?5E02`o_o+(#7VA6YPUTLJg%164i()0!?YpKxQkyF=~x6HkRBAr1zud=D5V zF*0yoVBnj^XcE9NiNSvBGatnTEKLh!ItTlpRr&#=oJL5}`iG7U0aber$cc4M{F+#nF}*%(dhQyB*`h?)=fUjR z;F*2N{v1FgG!9`d*OK3t+xy&J^pwdTPfis|~E@ zA8ISL$Qopsyl}2FKG6SjN4SXrvvB~k@qw9ci;cRP>^wiP#w)N@G%QXjU|X<(d-sJT z!4FBP#~Gir$X`1)QD&##@8tqpIJqx5ac`@fJ;S2>>8nZS8-!LK5DGI;)L+PJw9sm) zM&m1?)Pf6hwhFWEJlYk_rfd|XocYSyl%d*QxpuQ?URbI`M_rEz^pml&h&uV z`h%<$4eT`y?41c5x?*fQE^u#D$~93DFl;L`el}~(2Ci)HCF>M)-De5TO-PwBVfM4E z?JumBhD5Q>pODhbP$9Zwsr;p-fo4@(ibdA^V98Nn2}}$OyJ&6TFh%o0)1DQQvlI>O z7D%$sW@i7bR`iRxO?A0WDzohcX0s10Le^}N0o)fncs^d>-m!o!I)P2vSW}{$V;y6$ z$-fKsb1#N!SZD5-ovru5Yv~gPjSmS53>PH-;fH^*f`Cf?QgF-$8y{@ry_jtKjcZ#aXL0oQvTm-G4O}xEmcBW) z?not9g9Gb{l}l|6cLXM^x6VwBTay2Kmi4ul`347=^&OhDKQQTgsGlp|Fl*z^+0`5D z3pV&=hCJ?MzM&%R);cd}!J709tg{Xn&Yr-THi0c8fh{e8dtU)daDhYn0u~Jw=5-IH zDwslJmoop`#^Gz#%v|<>f#<@4`CW_+3LHrnSSkcKPR$On7f5tSV3(i3TxP(m^MQ52 zGuDew*sBGYYXaEiAF$VaU`z;LPhNAvE`i2D5|+ z(}W{#OPOC)HoGlgJCMLV`vPb718GtUP} zMu$Y90}MI>44S={G|sLR5@6uHz`(3>V2!%K8P&kzae>pr;Gm$>cHdbCS3Ix`K6S9Vn(M}c?J|#2B25}q z{-m7BiQaZpq-VPIMvltOKbX`uFxx%o-nZlIoZW|Kbu-%~95MYc#W^S9MJBUV0ke@P zYxZNt=ncEpCY+vgfh#77+ueb!(SU0S1BXI_r-OlceZ@4cXP$P;6XYK-Y2RfilPs-v z;PO*oSpTGto4M7N$?^e9-X6^m9-g%yxLO=}4lZDeT)?gt z%r(Iw$0VS@^s@c#XG|izHG6M|UY*4(?{Hc0!%Bq%64ee2ObjM_YNeLE4UcPJFI#X) zsNpuxhTBKaGH_j3$&qlUxH6vEfKiBn;U8;&7jwg_nXx#KB=)5UU>QK+L@nPbCZ5`IqYP!e ztaSpHBY&){G~n%TD9(Auuvm<>)xcl^183a><_NnQ-LrS*N-=2Udd1u`F1BQ3Er@^f zk0B)QG9LryB?IQN%09jV#*dDStPC8t5;#~7BzPD9n!x@>~bkGt!_7($jk*#1wqkx_5QrqGJQ1Foxnu&}RFN}8p# z)Nj|nYY(>kIGev{XVqIZ`}b$tUC%ZB-TBDpoQ;B~xq_#q!0{#l)?@>Y76l_|W}~zR z(y?GTsg5C2TtYU8FbXBKiY>Sx_JQ@X0H>8L15-fp@{6oH z6*%G@xLPl;h&t5pY`FB^u2{GsQ#jx~N5K1|w#5tv&rHl2!{Y9k-DO~5U|q7|{m-j! zkFwtu5#YF;!14KD0#o9ZqlY@4hv=ky;NDZfvdw{KrT~xJK~9f<39Rl0FWct5&_4K4 z&rvZ<>E+&juK5mUOa%|k(rff*dk}Z(jOW8MhWaeO)L3#Ztk z5gX#v^O}4g<$N(`$g~Y$=DYdO_dv^uH_#%{nxOjIxvNN_#(Ic>qG_?gXi|g zSh5ukFx_Mq(2HuwFr3iV&94v>k`eg$c)z5?q9=<=7B(@nvltkOsC1mv@YwmOhQpxY zAj1@)0IPsZA;%_mfjGMbKN}AEvI$Ap1QZlIFtalkJUJD(u%VevTvuo5f%MrbJ+e9l z9Dxc)*!dzYOD>#hJQA+FH%Ip4*VEIila4-6+<0i4m#B8Gex~r(my)Z3|D6jsSRB^c zHFba3rzuvZ8K)Apf<*MgY*bH)XcjJU=@fqIImLX&{G2-Gr)MWCQ)RBYQzrg4!&h9VeG@mItkD;FVs{D5b}>VL{Uptql!K zTsi`aXM03EX=vh;d9bQkVAHDxZJ{kniY&@CAD?nW*IYa@J@HOqdZ+TEE^a;TRsTF( zXNUMyzLreqDmK#7Vlzyg<+!5ZjlnUI?xal`mqLV%&F)t$<}))WzI^u5E14z^vnmNE z)2kI1&bD7KIef6qzWVgT?bqA5&zV22Sixv1Rh7Jm(}k^*$x4;sqAQPpTR|h|AFhYZ z0-kOgRRr#&?9kfFZ+C#9SuE|u!d8Kp6A#%n3{Erew_qt`n`9Hhz%o0iM1kQD_k^Mc zO|}Y+No`#BCb)GlFi%*}IA3ByK?7Ier*F~R@;?HYI29@$G<$gEC@_okFFL@;wQB*J zxUltt2Ikd8PqbLpZF*$R!p~%P)T7+Rl8MuQ%f};KO4~{{bvXu{WIolokmU@Q=n9X+ zQ{w)yCHiz8T{gj|OMO`d(^->;8JCUE->GThou0_o)H|~}LHWGRcb+c+H-v0UEFOsX zhFIKBt1!9bQNJO1g2TRslqu2-lB^S=+%Gilx0%ABEa<5axLm-Y!QllnXM{#H6Hn5L z#hs0dG7j~*vveM6vt4o{c~+2uU}I;HSD}+D5&AE}N9(q9Hu^fx>QSrm4SSmLfFOCm3Hh=o&%u|hoIwuaZZS6mQo9De# z%y1ACy=DGy$wmEL+aG60JX|0(W5eNo$445eQikUkTC)Vg(wT(P85tX@Z?Gyc9AuU8 zP+$^NxXmh6&?xrcA-CQJ2d5kX4i6U}FTD+~hvN;?j5F&lO`790$I)#<>=!n9t%WFe<~K|`g}0rsGpL^iVn4IE28 z><_%aAavzKb6|kT0uGh~tPTQqW#<(%O8iqP>R-I<1vd2>MZvZQPC`nGoaz!xLJbW^O%o+!+U$u44C<-oembsdMz0v1({yO!n)T8vjX z3eR$2mNHq$W;!L2`$NoJ=2bziSJ$qaE+yb7m=_?cmgTTedBR$@yaJn6qc7V+4CL}_ zxdXL*4vOqzIP&>L#Ini0$9$&UOqRd@fOW>zgZ)KDFJ;&KZnyixxck;$XP!OT!5_XI zTQ+f0W@Wln^QT8wSS_DwR;p!QmVe;5LUaL>f{kN`k;Fj`xdjbuivqYDSd_#ko?zAE zaO^NvQRMP_!N|+7k=uFBL!R~*|627{H1^iNX<~~#(W>IHk=^FbdXY*AMuin0Ik;k2 zX5aY5;I*c;fx(4A)#Bl`Sl>?;GQW;EbzD_TR%U2CK5aD%rw7AJn+CSCs}AybpJS0a zvw+oD<0gmnhH%lfe;5Tj6quDyIIwNHb6VOYaQ>#NbGdS|)3ok5Y?G+Ym@RuMT9l() zszu{+;pFU@HeqVUT8DXLUO5EmpYhz)wC$bjj_M5i40E>0Q#5_gX9v&RmT+btH#5(RrBex6EhV19q-&hAIN`hL0R?#(c&+H3ZFC6AO+umOt_r;>$}q*-Ej!X2lV z+XD^@b{Vv~^(2Y7PyHa~CcwlS!@%t0aYSL8LgLEcbIj8k@g zCmi{IJ*f|0tY1yZD7@#;K(1(z{2-s0lR^THoM}7t0r6@ zTLLezreA0a6li3-z#+NYZ2x8J(+uk zOA`7UrGDzQgRw*N#Yg&Dr?&GRVBr6-iRS==zySu{1&&fYw%54+*@{XqwrlmZCoMKh z-WzMcdaS|rhP9;1k;SJ2BrSe4ng7^vhk2*d<#qR>TOM%l^P17(IbrAH%{yP*-ZI;1 z-;3E>45ye#PGA)Ouv4;u@#W=46@wOofF`32tceP2NdZDh2F<Tl5Wilb3asZlqe**9WS zBkBB{Ay#m&ikPyd@Sd1AGb z{UkjL4NK?6PTDmso-0@?THL)>u;|sSf1kDG{UnoG9p^fo4zUkA#S)yw85pHJnj93G zd0E=x71}B~*y}agb#5^EakLu$U}0!#iF&{qGl9jfpfOCVF>IkKdjX^Mk4Ekd41yb) zoqjO#oM2{mV00ANEg8^gF@r5BkxBIMvh2^(=KT;mvP-RC;(`2yb8?M2XYJ^eS89kd zVKQB@Csu?tPoZVCFyqaS4T0RPz70mcE6imUD%CPK@?J2ieQa}Xwd2iQj#(zgZKvI? zTyDrUVa@khr!3-bQ{8k+TeG-^d5Y1>k{EaUgq`JUSiAz*ygJxYC$ME!?0gt~=AZtQ zE$=_=tV?m0h&XIA#Y7~*S>OVrEC-8W1d~aCRa*~xbO5vR1t!xKtWFj!`zEp&K4>=n z&}_@VBFlj`nD3)AcLD>qLIdv&CjSU?l?yE87451o8N>}*!I(PTHZeEo{qzM>^#16x@Hd-a0$(7)}a5$q)mz5J$jA5LjXM)-Ip zJHO{T(J0U%vBSrss8QuXvyDcR(GJ$=8!f&sT1;=S_yt@`*s*?}VylA&iz5SzAp?uE z2Ma@2i^o3&)*uPi0#(+m8Eu&h+W3pv9AC_SxMfPwvE z$3eZ_S`t?pcxE{Aba03WFg|$VFY3_f8hi7fOXJku$mvU&oG-Lyp17o>GLvPgt>yz| z`=SdaJh$u&wz$q{$#P(?+RngfhV9XsDo+BtjIt%tYW>st7q9$tC! zkGt###;JQ4l|QtES={z}!Q#i!mhfVqk3x&11nZtkmS6@>7YCNDO?(qh@H=RF9sG-qfc#G+- z?d{F2tqBTk(F!as5sJlc`)v!D)Gjn~=1mLw&>=0rC@avY(!jE3=YyaR%ri>Z9=Wu1 zNgr@Ja9;9)mB58fVtjs)3;#9Bcr+~x>QhT->hEf@>1eXO(QNY}(eXyJgFuU7pR`8> zi^qp+`!rcIU(C*wXw`LK_Ajt$`@`z=p@Dk^qs`m-k~0`@-eNG3X!8qTHhD3Zi=px1 z+f6(_7|yGnp6naN{IWixZ2V6-t3aLppF)e6O5`7jnWd#>K_ywC7ShKH2oBAb}$I{aA=fHU=n%3xKos+ z>oRNZg*I-s78?UI2Zk2M7aS^eQXc;}uK#->J=-+==vL3{1#N}P*ov05fntG;TwN>M} z)Qof4j2310Tubjgf9v0!7T4Q%Q*uY6lR;}x1CMq>#GE9X?1Jm2j8eiFZiMvGnmlb#2&?*`Vu0IT>7 zt^N)zdpI3pAF!@`&EfKZ+0cPSU4q5aV1dVmS1TLST@3mpGZ=U_$OkvD3vlY(CKtFSF_V`%y3_JMhOceAa-8=Dn8QyZIPUNl-QV3akO zCHLZyo*d(Om1Nxwtzi*H$|p88xSemfrp|wZDQSgG%eRP{%?us;wAQXZ|0MQ`R1V|w zwQsG`?oF$`*Sl@4+ zQd}7OKJ1E`tDQC5`7VRsD?^FaScMkQE(-(GIEKs@7g}Rav<6OK@jTI5X~^Pmfze?? zi_?mmxi{E2E;q+qV0D?WRL?=6xn*%I154L$7RMhCEd*w={a4jok=WkQFSX&W&H|TWZnRZ0wisBn#4NaGd!jY$wUkE& zQ!GO!^TikUk1%|77CiEK+n!&Y3odZxJe;49xIvOI5tK~RzF)(nc z-fB7IcBYdCS{&?53Yqwy7lS9+I3UAnBNt#6lc8qZu;5N?2^ob%*dxN zP3kRw`ii(t(tJBl)9tp2>f1!A4UIM%JTyI+&c0z%pV5>Y#-pak;_-pmVL{7AA!bu8WXRC)R&C&!W9j2#Sl%?r6# zcFu0#^t=-lWkY&P4+(iZ(7z$`xPyf z3ZIRxm;Yd8I{KYaVL@;Ga$XbhA71ZXu9Dqnv3#ZLVNtsZmWLXu!50!8BrZ7@>}8X0 z4m!{#cB`4I-^DJWNzdYu%#1F92TaZj7`HBJ6g$Cq`v}ADj0VYoWF`hik%l2Sbk%+~oCtu6*^Y3l1FKX~;WMbvv(Mo$G`Bdlp z6!oBa4lO?dPDyv=-#c=UBVBopb?t*MotK}VUzG9b*T=`(^Honz)7i@*$i&jYFJtyE z`&&StNa7={kJWf3{(gfNB&o`vmJ&gM2}X_1h3>|BAt$UCpIa=?XOOZX;NYQFVR4};D+-htTe$cY zJUV1%HnNd7;GGW&8k9A`MU@OK%{v1ok6#P%v+{o)3z4~I%b)D3FnHR^1*M792v z^d4?G>D(?je@|omM_-Y?!gohra%}RKx9?kYb#cF!Ht!1|@6sEI?cxR(S&sNw{%Dx& zZ^s!L@~_oFwpE1BlA)1Zh4rZbhl*FigCkWH4<;^9oG``7S97AtBWE`Gh6Ap=_8UUk z+4yP%7I8GXd^*g^`=B*Iv(%@&kyY}^S4S3g6R}%~0uB=#QUnby97vs5`Am|T$7aGz zro=M0ROO{E0<2~}Z$5IEUUm^^WDz)^>LishVqcSDnFppJ4 z!a??8Ck9oPcG+n+nK?6VI?R|*IxR9S!AmVNT~I}!L4L`C11wfS912XDtu7xTIGytr z9Fg3TdGV9l>^nu=hDTV6Ppi#*(BmfQnq%;Iv7cKNQ^_S)uBoN7il%=lztm_ofs5&2 z!=dHoM+{Gyba)&*QXL(yl3n++R%Mb${e&qzCO?+2tZA(O@UdMs@BG15pJTUNT==XX zu3dA_O|;cnIL2X8GwbbX8;=UP%h)vuO$b=Tqj@f1v$y8r1t;3&8s1!NRhY=KiCwFIjyB&!Q{OQ4cG&I98#XOV4qSBBkyd1R$Zq?Hsw8f zLQ^za&6Xu{1u-NEv`Hi>IjM6xa5xEdIvi0r!C-CIvq-w#f?4500Y}UOC!w1a?51pD z%Q-eEFll5sFdKYmWQ=KGxc*yq+CRAm2CbE84AUwPFdUk~z-IA)^U4fIwk;Fexp)pq znkyuKQ+U9@m9l`N+^AVnEOVca!K8`S$sq|c0?UOQP8m3L^KmV3mR)-Dyusv4@hjMr zR;@V8z^7Z(R{h{I^SZOg?S2VHd$=9y$cQk=l}zbW)p^)%|Kn0oF^6-rn#-ZtZ{D_8 zSuVW4Z$gKDzyS{L3ohC@h22@#0=VTEj;KdHaX!bWG^fF!MNUAm)2YL8PJ;roBp+j& zfeW8V%ZzmWZ5z3>ek_*O<7i~o6KJvJa1y9YU|RLf;TW660wxWP1I%q#B-E>m+eE}>k z8Oj?jD;!n$qR?i%WOB)ph8DRm2ihHb7zG+9u*f?!bmTT938X8q=$CEeb1^am>B)h0GFc|%6l)72a#+CDc$ML`c zLm`V5%tEuo9cDBMq-r=XT9qKb4C-Psg$J{jrU5)W2I$Kx1tXaTv`QrR5 zJ~!`){P54p|Iua4Q{fQqG&Ss(=xq028#uUM+R*q~L< zO_0;R!GUM-j7MpXg|2!MP24gC4$Kh_OU!352~}zwRZwW`6$@Fw=%CRmAQiAlXQ!G^ zic14eiotS&)MJcH84e;Wj4X@>2PF$vI3$;eOB#vk^4<7wRDD%InVeMT4-FjYuZ-XW{nKm&n@*Z2jqN>x#6CL2hXXU`G-=M(4 zewIP_*wpnJ6O_b76^!_nd~CiO#B)YybI^9L3o~_WA9(XgFh~ktX!hEp$QaOdfI(-b zs@ePmuBAU%Oa%;EgKM0`cRh$?)4OozotMI1Q(pZg4aYJ*WSm;wbxdB}TXR*Q;M5hi z0UV)moiBa-w>JLDeGn?kk>&2-EXSm`b!Fh(EN1nN8E0~`{XT_Fj`(N1xuyIulbldt zvu?{Hu|o~*)-{Q3Mk2e#W+}8>jS*p0RcR3k)^u@T4eYQ{a1v-^a8W!qsj;|Kfm!Lq z;cc1+82ha`7rH!h(qu?v6nu~f?QI zk!2;*^uy*q|9Kx2mU_sO9&_v2`_99{A|BdN*NRkBe)ij3FiY6#sq|-(fs4|P*gf_zvXmIJ!kGDX8o2%5fhE(jnNmod^cPNTMlr%aNy`@;CRuneWr+#0~4=^!#kG)JQrex8Vo7rd-k$T3}txf zDk*ZqfsN&mgioX9lSX5oX5J14(HK77lmiSk6XzUZm~zLTl_9>;u(w>~zQ5r8YR|Km z1zbKYbl~5>p{*Fef22{`hEe*#9G%Fi%&GU~|FGNg$Tp?~=(#liSQ_;+(y6Jy@z2gs z6~7Lno%f>)8O1%AI3ya?3l5#$!4t>Wtga$$mT=f4L0Uy8fv=`f*u+uX#8GHT@V_6D zPT~rOH10GkZfRh7c2LoRk@dv^j)nsaCNWNp4T=Fx;%<(b9EW&I<}idDkh^m*aY7t} zfTR?w1M`YNK9_^+77h$9k`i3W<$?^15e&RN2N(hx7y=F`iaOjAbVztHm*b73@Qfzo zjAr8*hctSc^a7YQA270&OrJA{X9J~1sU9ZHS4=7#P2mN8Dh`LlUoh~RFe>ygnclc8$H4TQ z<=;(?CFg_`4oUVjNW5Uu(_mzcXqe`5K>ooD=8VN09|Yf~GH@p_aD+54ShyeF${@wa z#`wdV;|v3@fI~=8G+)gDj)KNlDhKsH9n^0-n8?&5(Zism;Kaeuq{q<2w&0!uN0VlR zvsngnm!p%&lm(Lo{8=>ImdNnKh_weJ+WuIJconGp|Yr>?-b3ypQLg^=pE*lQI zKAUr{)3+e*st1qK%9oRFWw3g+-M`p7_0qwpyJu!?5?~DQN#|u~<+kV$|In!N!12@y z#xpNd5Ub7{HL4mn$hevM^uIlw3# zG5_3O#ri*t^*T%j)g}6?bAl1)(KSco{~SHQ`E;f4je~MenqI3g zN@h5n;czzKU^ZxJlB!`+(uw2`JUqL{>1BhfT!2H)#&IC^?|e&?I!{ z0541Cgs_Ji90#nwO=gKXFvq6zV203?oSQHRYut-buKbeGFj-tA_)yf;UkPP zucmbR9^_R?j62e~rzt$~N_OI(5Wl4k`~QWo&S?0f6UE7pb6_1q&0Ggw7AL)c!?JrW zP2cAz5RvkA;!2%X-*2@~lvJFpzj!tJyzz25tmksr)9a9mhm(HqRPhz}S5M5jKS8JT z){!6`S%DXgDh^D-8%`+BVbZ8@n)`632#1sM5l7{e!zLZ4kMw$rJaCjPIVjP?c(;vF z^yy8F9!Ithj(RFgqPf=;RhT6IwJ>sOu+M&bj&sWa)|Li74<_*dHSq}tBwsLawj9)d z#lT$bC~<{D{Dp&riX*Q|qpZUMi5m@%UN!KVFz#a0;|*x!yVJlIz{q>&K-0YgJQWQr zT@GwLjMWy&tQVAj1vx9EIAj+rcqenlp(yVl&)Mf%AD=&3yw*7K=_B&?1CruyYbmYG@%I3Ve#CcRU z`hUrN~xw*b5c{UYg6w{2|bsl+*wVa zdu~Hk2nB!+#O3U9U|-R9yfP$9so-B?UPsfs({`;5+)Hk=w>0RSE-E&?UiQ>Aj^o%Q z9Z#py7&Z-d{n-!oompej4siT2$)?qK*A2c{Hh%?cOZy>V*E>&ldfk7u4(sOuZRdePaAWxcn^`l2W% zWwS-HJEXp_wcc}C?Z3p!*S8GZiAz)t z_-V$N7k|D`QhcCG?|@MJ0^h;~uIxNl4jhYURzog0`=JMQMVat3!_4_d$tuGn`ZZK-Dac6wxq!8h&Ti0}2=#$9@rp+^*)S{&o zTpUGA4r&)U3hhV|GB_yWz@%}Zf%(Q4R)rRY00%~q1B|m5PGxN5*62K7a(&O+FaF!v z8pC)#GJGu(-kKSe8^P}5|Gnfu%htt=ZLB>GDQ}yEdKeTO4)I)AsJ8g&Y{_F&)fyzd zirE|-$~2}-b=#N}Q+&|lV~zbs?l}jzLtgUya8&u1(yZ|3fSldb;G@S~^(IJM4l4L1 z&N}n3)s5!$Zw}i@mwi%uAAa$pfX?0I<4=m~a-70D{1_ZX91h93G%70}l31gz^^RNR zjkjXU=_eYJ-BV8UtT5p?QXnj$SI%9;C&D0{)ga+70T=S0KHJWsN3Nw#nqu%$T z>1qoCIn@&Vx4!msxM#$E5FtD;XOjU6ZOzNCt!^!f+L3xFP zjExhUhoj~Sc9lPjOEuagXXRPk_lr_@ThM7dZ5zX?FXnIWZEaw8tT}Q?qS<%R@kXUT z2UTV?+dSE9H~rQUVe;V|U%0NWb5O{qQ8cHa zbLt@x6DNM(!|F0law1HcHcom|nB;mG%FG${0-ViXILR&GQuxuR(($j6d-FM#*9;Oj z8jtgGI&tz$U3`FD;-{6B!_`}xHjBSlBgb>|{Hn?9TT9$`s+um$jLHoDzC)p-S`RVs^#7sVnWFU3K~ ztC7uM1w%}78iNZHn~;i&LdBy)tP;y+99bx)e$K=(XpZBoi3*9GTvFas8ZHPP>J(D< zQ}J+|l+q@w9+=RNtk4!A=e3BMP4-|wR>rC8`adMSz_hHt^o%yjpR%~XU#LDeD zE#*1k`Dvif=!1f80>Ewg%x!=+Fy0hP>ZhZY9LLjj9i+IBtZoWjnbm$QST(P(R~ zxTIcVT|~iw1&nPRocb;u4UMkc?7Tu|D+CT6%zGeEW33?YXUE3}r<-rjyW7Odb$e_2 z?QQ=A7B;-*meg`dG&;q}$jz*(utTA-(UFr^E+pZRPM=6_h)GBYBj=Y4>2@#>^RZIczowMt=DU$k8+(pwk4%tVW<8l1yvTIZ6{o&M9&gY z-4HPChC{2(oWf_q(ko6lPj}zP;L@*_A>r7r_T>We1Y5R77Xd-HgKQ#BY>F-7aYrsR zb_*QI?z6fVv19GF2P!w#GO{;hD>OOHR8F|y#GZPtk*iL7LTiiG+H*T^Hcb{`wGpUH z7h%#8IAGTJkAo?I>5*2z!YMztOjKeH_HkS|F)--BgBGU0$}jEvg&w>&OJBB8ZKL+q zi)yojSQ?islAE#O!m^C)M<>3kI zu(IhtgO6h4L_G_}9smBW`@?e3@sU8vgv=vyOGP}=HeEQBk(+c_-NKAVtIzIN#1f%p zZ#F)^z4FLful{v!CKU4NbOfx9-S}v)VO!O6t?s+h_T`H0^7|T8I(1EUraq|n)0r-; zYc{c{Ma}EtZjmgHldU5CyoO62doVc$3$T2iu&7CQ`wJ&dqe}~#x}{4xnEKVCG`3t= zr1*E=@4E`URHvUMF-ea85mob7=MI{xP-3LO#Q7>z zzq0ckd#nMAWX}T5cmZX($SYgokt z7`coZ8fAJEIpqQlNgTP*s;{BQT=BtKHZ4HLH#5uf$z!?idDAw#&N?6P=O6=z)>S5v zJ!@+lojMliCAhoPIEyQFL^0%rZDqG%$maKOU`t!n#G>NBy3VJOOCf?$WQvQb%F7cT zJ_3_irX3Qg{Bgv8;<3ntjw2>Z3Y%xo5GZI`-Py*Gz9ai?V_{#jOo)oPr9KBE zy8}aaw3E2_#f_&sUQ7|+y>3IZ^1Xi=dX^oB{`M{CJRs66F+0Fn?aYPF=oJS!8V<0T zHte)c?PXwBl5iAiTh1(dp@FUR%@Wzq2U>VK99X=YT@*4Nv^a1u@$Yt=|A%L)$K2ZY z0gtTxeim$7$rZuCEZ^~x-L9aK-@$#EP{;i`rGQ4M7YRJE3mDiQMKH^4Q3zyU5#VCc zI1n1Dz^MC>pX<^w`FUp?ay301Ik)`s3hqe~xK+?@{UESQ`VF7Z;t9u0d^}jqFR09F ztakR}I==9TW>edj8)^0h7M2@cDCjtJdA~?$a5*fsZ9}WAl2Gb{a|hR5X4PBKXmgZ- zLwUx+gxN1zB#ka|MgMUXmJVRzTt1QA!J+A&$OH`*RUyHS->Oahi%+mA?I?DX{&8@T zID@0&@c<5oH)U*%2~Mm%ZZWsFhy?BTFPU>3c?z~}ga zk;iAlq{wNHgj*zzMcggq5ba5lI>ocl_`etn?)En+hm!+B**Zu(WqdN=$s30)S?u5l31L1Q%)T= zDN$tQF<2GO;YS%e_?Fjnuh)#4D8RrizT=2Qr+18D(n{TAm0bxQre%w^ zJ^sWj*ZzQG25XUa(;cqJ<%yhr4lI*D7cgs1P~s{(!J6%^*s1$iNpSaqHa;hZ`(h_H zvsP{hlACv?MaDppOJK7zuhP~ziv?GuZLYL9$90t9qogjA{J#l1;&P;p^Y3U_){@6q zmRjK1^PZjMn{+cP(+2x*QN6bBRihaGPHu>CWMK9G;mBKep;_wB9X8V$2YF%)ngn_n z_;Z37L#97%(PT(!c^hZ0B=E2&C*VelsFk8g?x&VH88cet{!NgdRLEO);_;39kJ92- zGfg{jDD~(Kw~t+$=SJ+hJa=Q^{B6CdGB>3h9`faWIv_aXN>_r6oy6fApu=N3f_e)1 zZC4zgxmbbow6l`nH2)TfDCvt_9J4r;c4&yQFmn8gGhr@0aag#h!1I|?ZjD>B17bUC zd&^3)j~r-SzsyPNMnk?Wrr6A0+Of*kX z#H3NcXQ7CTTZGa`j@~T z;vk^3CV5jLTiILAI|o|y&Rk!n`KFQ^Rov zMZtIK$t;YLlhp+_yb(CpARxpjB5;uN%K^tl4gy!wYh_e-DrqX3oRfap^F!(A54{Ig zi(WII3fz9>IrA043(Hn3hXUogugx|&6*&EW*7aX3QE0(J*$a$s z71V|QU0o-?;8FA*1@=>!?>4=C98`ETWTTqzM#jf|tS%el798M{Q{?vTjs z&?p|4+oHBeWQ}9K2BTmIw`Af3el|sk?hE|?6!`7D1l$&iS11Z{92Bl{6#t~a`Q-sy z(E@=e2Zm?MAI?~9R(II!T;Su3{vYo)na%s4mz}lCij&2ukM+-*U0GQcB1aWYTznz7 zijhfS<&EU}YtAblWXu0mzF4%FZI`}zLq1yq1KYm<#mjB`?j$%g z_1o@j|2|nJKlsP~N})36K=uknai)hn1&yy56#3>XoU!hR2*W`(w*}mj)-$^-U|!+S z>*Bzi=D;kM%QlO_r$o{39D_&|qsX@hY?BhszER*mu`wXanQ589;~z$kjow+M@mok; zkug5EY)SU61TN+rt{=CSJMC@H*wJpUrNG1YP@1Wc;oow`S^B?NRs?<%boj@vT=9TK zhm$p>fi>-bg1-Z6TLQmS;!CXsd}y~P=w)*{V7f@nk}HYp=cRyo6P7hRmN~tV`{YNx^usb!v&}NR_TO7x z&9KC>N{eZV1J4ByhDB~n9wNM=D_3q}+o#09_UXZjI0ZJ2Ue<&KtaHAyZu!8jp(w4R z#W-VuAPb+QBBQdhk`T+n^1g!WWd}^R>Iwc|rW|^LH!P!2ZlPD@?C+|c~(h0*tp^)l}#efzR#*|(l=D?aSFSLiJ08a!n^1J?pZAqGZ)2jTBOvfp!v zSp7ac=mFoh^{m^@M@B0|alU1Zc)(h5fM2eWUFg9{1x3DP3j8U7PTv}&)Rt*}R#2>T z@T_d$cym7Tl4;bntLpa@_{0thuG%8>^_1qOFVQci#(jP$_s*0p?SbW8kz(<%Vs^2F z(;^9}qo z__jo8B}aBH#{8}V0jVQfOB@)cxCtpd{KsBa_KjoFl4qZ+%JTJ0^}=$yGCKS9GP{-; zB?RY*1{>)HDlD)(Go9hXlVqa@a`Fp)GA&?|Qxq~dV6LK?ZPLp)Ln|jxfh~c-eO@SA z+=HCUW~|rJLcJITmnrbE=<$6!AhIS=WQMBrt+Oo^2^>?VTrbNlSY+}hZ?f0l_iyw* z7Vdkh=@n6AZJ799$Y-y3F{@dk@U-H13xo1WlO)7^iXHfmB&0_-aL#C$W~PuhEkR(# zfd_vQM0^zeY8Hw)D2ik)?99K|$?nX1<-jL51;G!@*Vqm|oLJ7p)l^e;im8M_ps11GXhHbR z*FSX{7!H)OeOkWFuE}NbGFHEY$i3@x{)Vr%_{{R^HA_`PzD8KsfnupGjmcHZ1%GT~ z|8cE}(L8qU70>EN1!qsoh}hdJ;INE#T-oz&z~%o52HKF2!cIM)5lj#cLMM-oH~k@1S_jL-D$YJm(yQ zVjgT^Yvk;Dz?PGv5Y%-{Vy z_lCOFic>Xb#RHd2|Kavvk&{Cfdjdm@GK-VKSpi4R6$xDbm?mcF=VmD{U`u$w5|Gev z@>`_&XC{^%?C%cnS2b9NBna6p%#lj*_g*Nl;ijb5LbkF4Q~V>9tIzT_G4S2f;>ap4 z^mrokpmdt*^=ZZ*H3KXXuM4hDub2^QF(b7kF?+|1y|b1*y277zfXVFv$Cd+PmJbt* z7&$c@SY|i~q&yX{Ym}Z6Au{KO*ldfg+=IdwBH4dD5P0OkI&DD}pWoJ;2a9qZq}~b& zDtj#3nr2nSm6!22BmPy!qSZ2n)pfsa1bnz*x#M|A#xW)*1&x|ag{zac|p*E z{OSabvIlA_e$QOM8dY_OmnZ#(nxo{phcY)5p#mCSo%tqwV8^~h_6U!{kf`h zCUe(J)+7gxPY)`mEAak%B=+mJn2Vywwgx5>1um|GA_9!OK8hkX2Zc=<4?0c|$hs)z z_>f1)k$sjx2Jt%CGz?_#L;KnGJBrPIwNBGDj z0fV^*^96YhJMbtZ@^dwc7dQ$Xb6{ae6w%>g{@@@GbBrybVMoRuvu)jG1y?N}_1N#5 zXwbinv-_C&y|hzz+iNKjCt>L;9E zTyv`M@a>)NjZzabv=Z3oEEHU#!1%(M!K8t6*+Qi!{`vlr&DuLWFNYR9&x&56xaaAe zX)j~mzWK9Py*Kf@jREJrS{}KC@CE0YFZ;fX_Oa*V_~5|Pj%DCDlEBl$z+1=IopM~3 zWAUSl3Cu1Jc=#R)qaMWRccq#tAu$`hv#X$~_1wLF49XS};WV3i|+=PrWq?*+E zQyR>JPW^kB<81b9huNNT$C-=2JyHwY<-Q=?=9g?kM#&vUqXzyJiqZuSEVvZd8?tzo zHT*dq&nfVZY1ubMi3Kct4oogiw;#)gUlU|DIKcj4XM^to79S=dxkMfzMun=v?&~ky zRt7Eh^34ue%+}v-EBEPz<3{$`a(VMMR9Ujj zS*E1)p~lLDNqfg{gRx+VmZ#LEz8}xxp|o^ zG9D~wVrCcDaoJJu@8H2F9DF(feKwJw*}3QITK5GoIyIjZkBG4-Y%FQH%&XwhA>eo@ znOi|CV1t3efmXHn84)WC)B@xLMfY(CiKsYsvhxeLR4_0yrJdO(@4e#TbI)cW^;Ody zK4u6=#_c_|baQpU1&#D2NmnE`B`fvI3cd_a3HrFmwLAZdp^)|K8_5g2e(rf9WdCP} ziVKg<8IeRz>$g*cxdmf>Om1`j*>c&}Tvzm9kkOwB>ctIq^DFqCpAwaD6Y zI5aCvN^oFSsSA0;&LXbxp5C& zjNmryf(g0-vTUXzK^Cp2r|_7)+2}6l-E!!p+Uv4Qo(6{mHg~cs*<^P0r7J!<**|Z= zLNTrUT*pQRWfmq5J~07@Mm7bH#$?4N7oihVqJ$itCr3=vRGZgdnlm$He&(6X>}3nL zN#^EE%Sq4e|EtDf8q0H`LB8DP-vuUaz7zotP2-G$0L|VMg#*m$3JnKXIaLZ88D`pT zJkGC^a3MfA|4Cq%sd>h(7~z^Z2i^55UNCrSHcfeS#sqZ0?5*8iUoNcQ`)hfbKn~Bx zV@j9i96o1%a@iGsyR&B6EP{g9ToO6kqjv}U+oqn-?QpxOk{2vysC?4FFU%;h^6=G1s6F9fr5j|H4GCRSeZBk zS`}Ii0=qPq^$0!XoxxV%#QWy+2Nsd0e+sNB9I4H6PZ~uySlq=q8X6DUB(jNTI4CxJ zVD)&xGT3w>+IQ}>DXP;;Ebw9qB;FNe%#Hgn+xGJMXe z9S62XEYRh%$#b669n;}(WeWcd1+F>&6>rFJHLxgT;ZJGcd?)E5b8gdm zJ)uMKSsz_=Z3@|3UMTQ4-8g)NL6J?}fHBE-x7?SmvC9l*p7>Qb( zOYLX<_ed7V!`wQ*-_a5>p@Jx_b2;|+n^FwQW^aP&oiY)RR zN7W@SCE8>!(ml>N(fRJ8PX3uji}+tG;H>*_klk$qt8vRCu{j<`yH+{0a%CLgY?e5x zxU6BC!JoC+p3ErT`|3oSq38)m#TAMj+yV#qQj~c_6dukCGFVtQ|H3i# zB>`RPDhhnu7adqvXti3nGzs+{Xq0|y&=vT_h*N~Y@t<7BN+#iy2mJGEB-ze8a(Zbj z7HZ(lNl*{$^fXx{c=gCLWhq0>)Co&e*s2!ZpQd`)a!0U}{Ym6+ zo}{>JMj4C4h8F&q3C*foRE5)SuxQy7u9sf0aqWDG!;y9y#kLA4c`Ur)qH%(O)39Tq zKxKiG+_4L+4;ho{mYz7Oe1XG*{YJ>LwFf7C@i6wd?(^R2^6QA)P1=_^5)w>JdVMox z5@c^SoM%+XVC2pT(c+sDpE_(wS+;|L zqi#bp?=c3hoF9io(j{2*eHwcM-#koT=D^}wemeS|kdp2mQCHn<6F6*K7=?0V4vU;H zVEg`JJExe6^t3}q8aPt8f+iTHRP*}=FWA@Sn>*{Q-`e7(<_}exu20U)xl`RJnbN>! ztk7_jV?qN1O9LZw2LqGD0|rmqgOY_2@oWdIcq6!VlB6q1NXU>Os;$NfW2lzGux5`uJ|6Mvn>uSSM5Hs z|K6$YzVb7Rxc$OF!(UB29Skl)3XEJHE{}wBzgj5eF}x9bvViSqw(v6ET1J=cd!A0% z;^)7=I{0+M`szj&`M|*9;u|}rU*cn6QaR+<$#0Utz?tE|)OVbL%|pS;%b)Rr-&Kxv z%RUJgUT~8U_|tqWDDi^LB?~@R)(aqeh88=NpTb+P}pCADYfJ1~@6aQ(#qn;lOFO zfl;{Sc#Fk~D+i1@7{9K%Ty$;9f~RNFO->yO`TSBblxc&f)!CaSCyri7Gdq#Y$oJ|2 z6Z@6zFMabJF3T6}Sz`Kt|D4AL`G{HU>1WRWTb<&x;eSJp*NO)Af)9!67dY4j*@GI~ zdJVXIE^ypx$hlO;cQb*{GL2X0M7dZ+m&D4pSr#DJoGWL8kHMsc-gITHYl!a0^ zzDxBKN;zC-kWXM=BEYF9z`)#~sxH9D$sjxF5Ch`}hNaUOf)V~Uko*Rm6QERPx$GWeA` z6qIN9=}lCaHX+rCf#uzm3cH8SQ$JVqw>vL3t^6tGf9?n4gyW3f51H&5*jo*l@+4Uc zKd^-uFnJlU`8lwLEMQ_tH=A$3T4})A=E>^)V0u?1OGpBX*8`R^29Bl=tg#MUyB2WY zVBnBQh)xUUYS_ST^3Q>pK~j3=wBTD$V-uL-mcFbN$&Agr6kP1&an*1luLFb50_Lvi zo;n81rVfHiE9+jcGqkdGD0gbkoW!tf0gx@B8!e`8jhNbM-ej-FIc-8yY$x-(Fp&>zQIn@D zPPjgQwrQpIg&yT5MGM7g5=Kfw3d~ud9F_}e6en1^7O;eFVRU`KC{^H+%xu#9VP{{I7z8MX-UnI)HORLzqp&q`DU+ya(zS6&N`l$mq^wNMp|y z-Jte5n6Y_*m(~H6Z_9a?3FL$&m^se$k`r6{d;;I=34Cu7mOq%eoMqAS7n)OpU-D0C z%?sn4|1*HIKY^3+IcqE@%Qs2peHkepYCRH0E0u*9jTkEbE^D5xs$>23<+>BU%(Wmz@+qHk=6xcsRqVvU5v&Ii)#$Hb~JET zEnv3Vz#h1Ov9LioB$VA)fnmDiEYIs|oDNcXmu5NKQae65nBCKM`9djc(b=0aWw%}I zsZrp)(!ku`$fORsG?|eNbcn}42DioTXDXT485r8dmT?;JPPi~h-~top0>+DKtegu3 zZVIsLv@+Qmur@DP#y^|=$wbFj6Xv}Zm>O8Q@#V_pVyowE%HR+7?0#ptqG=ZQr%cYW z37g_JY;w4;srS{2AD#2P7?qYT?UC8g{C7#S`EN#<24;H&4lf7Bs+nx;soZlExVBB; zbPr%re!$XOz%^k3)3T(5knDu#t&CySagqVd)dpNk47e6BaQH4@Z(`syV`%YdSW}vk zF{wmVwA1h=tJ=(O?bn-Ymp7`r!S_h6Im zYi0`vmcCzkKVIs}Z|dPVHSHWnrO^WB2Np~*3s@H#@Ei%?Iby)GP=PBWz|}~qUZR2V zT^FO&1jhIzEpdmfQ$A)^25?C4S`_<%RjZXrTx;>6?<^7yvo1Kd_XJAnPF}opijow&&YvW7H-7G;H#kBc%|8AW4 zbobVZys$a^N44kI2ozK?uxahyIS6saP9;9+uIx(p0XSYEWH~et6n+He4Pakz`0arNW0fLf;?o?{whm zPvGc~D$tZ>D;8jj6JT>Xu*qS=inx<2F1%*GaC?Qt1E#D$7hD&LxI4O68*sPSnZD!2W0TBIN|e*_q778`k_1 zXZ08GF1e&8_IaD!VTrQYnYVYXomggDwPmr$oLM)%7vFGa5SqYTdVu}u^NB18S$v(E z90g1}#BwAra9j^ya+@l_FTk*V2o_@FMq?cDR|F~k2wn86?m5@aQ@nJlu^9k)a&yM|IY4lxUfg} z^&X3Y3tY0dtPETZ>>8@P z%L2G!92k8Uolq2D68}&d;mzzniKAbD!|FkEB?G(9gTl%Wt4#~;G|b+f_xr%L<#o+R zvONDDxZx_S?S5*5Po@MXlL7;ayaR*61ZLw8%pc|+j*eoQz`-6L&NXF%NpzH~j?E+i z15L(z9alawbZkD9Qp+^CI%jDC-#G_X@eS-#KAf%V<*TcGSpMp|}>XhxqolYKf7YV+o^U7n7MkHtsETx zWo%(_SK#tE%{g}i+w27H{RQ0H5?I0}uwP!loowK~vX*1<1y->DhJ{&-M}?Pt|6RJL znWOE4$;{slHHkIm8zg55d06h6aQEybHb~e4P8$%AJ)HpB`J49@;85=&+Bb+kAgXGPAXVYz2K4N33lCio5~m-$0e{Q zoMBBo!#elwrtYlAr{8UIjAJib(CdA{WlqDBxf?ixTK0t`L~k_WI25}}DS$;`!HH>W zH1%rtPviN=WcGnI=mLu+!>+`8%z+9uz8juRPttT1^;p$&z>@2p$J{Fq%D0gJk z&6(9VPn~0(qkD|?@0+=IzXVn-(E_ka`^!8u?&!Gf1*8=8?JGq+^E*pQ?#}jTq{j@TPGZDy1`WWful8mXzf=-A4|EPSJ3io3u9 zNd-NV9SjSZSo&nloiZ*ICNob`^_?{3!Ggw?$-GJ~9xsEIdUG0B+Id7CWaMzZR`gIK zM8kh&=4G#g4HFn#MLgU*rwWxQ{&Q?*5mK-@&^7g7BeSZ2ibkQS_BHMxR?FZ(#YV35 z#ad4qRdi>YWL|zHR@%1x#g;v_U&D5r=^8x9Zsm^NoObTc&ZBS7ZFQdBJ!OeYn@;4T z$H)ECtn1!%Tr^fcBcoHiXU@FCtS2({SGVr@#uUvl;kv_)729N4^W+4TdK4}m@eI-S zn>``tg23{L+cFmy)MScq3QV=>cYAx1Q9Ni0izHW*z(mFtRvulhMF$j?hG=>$+0eY= zf`h8ZBpK&>$N7bydQH}f+*FfbKPgViplPbY2Ct@B5k(B_)=CeZ8LZs|7?{;o1ZX!d zQJCS#%4j#IYYD>!tA@rkd=(5#EM64~jBEdS8EibdS06&t4H%+oqL zQz2VrlK%dV61Q0@6J3H%W-VFCW^ia-;F&owMJt~OdNVy_-hAfA1808gA3x6U8JyvH zY@+8^yw+5&rQ)-hMSl0?ZP#{9SRo*|MB?#ef18ggQvxpBNNUmK(sY&1@|u~{|J6&A zWkQgfp{sz$joZo1Tn;lDH9Z+NHg-BFZBpzD^jc}s@AzrK!p0v@l23%LGP|YG$nnDX zg9@v3MFS(dMG^zk43!-#_Qi%92^^di7Br!iU#7r;!7xvPaa|xoqDCuATEnWQEi6Bi zwdP1Dd}my*&>^^5C_si=yH{%l)72omPUW>03Y{&p{(aNXY`y=rWsRXnBRgM&WcrlA zHlY??zaI(hb0d$<%u<==Gs#_tn`QG^3*}%R>oX>2`L6j1FETZMXj5rs?HzZv+h+NR z!y-9bE*=$$uX@Nj-O=shG0EJjbBhH$mNZ0mg(^Be5&m}U!~>@HpA=jL9DNE8iFo|! zNY)7UvSgWH^J+n(Vn%vV>8c+NLem@>CI9fN7K`hwRPOu2>}fwo3|y(?J}9Q#Vk(9LMt zMxm1{jvNx0#_i?dB-C~w;-guc&4gYZvls>6?383_g%2D1uNzAWykYpqnf2$4w7s*6 z68}t(U>`-k4vz%c83t1vJrc7@HCW}O7VNhX;A{-5Zc)f{Xmz`g&n9xB!B6i|1BZbD zKT$G^#4n`>I}nUbz03LlP@O8?7uSe$xTJy#gW3N(}HK)AIdbX=TXU9 z^PJV1Ws$&qkNao*8hc%Qlb6_9vzjh%(EoNNu|#*ulnLJg8hI8d@O^G-QCRk1-aj`7 zC)TEfDI5Wu2RS<4P8Gj+g4s^Vk!j%(2QH@zUQ8;DLJ>z*cy@@ommL*ax#tPf_DK&| z(>}Dg3#f7l1RRw2Tgr4OQ-hg7fI&Fu!^Ble4KdM;4Iyt{t>f+d%oy9qAk)LrH*YEv ztIvWXFTN$TTl{&*U%BJ3ypne0@vt!CS%OEDUc78~(>c@?HpfNT>G%0Mzoy>2n&e{# zQuaKV+i9Ag#ha_K%Ei=e;;BpOC1(OAheqXN? zhxB;xFZ$47d*%U?M!-Sgb;+yLW4MCmUO844#JQHs($!gEIyI`UfmQDi*Tb4fQQlYvHkFeOvhy04Sd^4@YGt$p3M`zKZloxF z?OdoVCESTwMhPHAA+6Rs0%sN~M!zS!Txk!#-JJWUTrUK^j!yk>RGGWR~T zS=?F3SH9t}tdjN3Q&!jb>n%RXi)~C%)88rBC$Kg{yKLj>HPd`f8uQLhy6mne%*YY< ztw{dxnN*`<%Wd`^O}tJ9%<2bHrGDMIbM4WBM(G+qW~B=btR@@cL_b(g73J`cbU4sr z%kk2!L*R&s&qa>lfMls#9WEkL4h@@J4zLC9ahAy{Xt(*I;KGr?z!Y_)fn~xkM)3|t zCIJD4u=W3#JIIk>fI=?c3Sv|m!OV6~|>PiE<&5MJ( z+YMUeFIaOp?)cbOz9!j%!*WyY59R~iYf@D8cRujulbgD~B-Nlo(KqH8qkxhhi~i#V z4vPa7$CUd%8czS&Z}Fp%&nTf;W5yw=*0YW)6)Kyb-jifD7GPxi=JQV4K$Ty@!-FI0 z086@sm41F1o7uNdpEFsQm?bznnHL>+X={_%mG9xj#Gvq%!C=8FMg|AwI0Y7lHBpT6 zW(;eXCuB2TN%+UHA6Lq&u4TmaMoJE7|deW)ZD_rVyUsTBVwwp!U3BXb8I>c96j`zGL~}9V6?u_`yzwI zenA(<3oEt`(;#Msq|Seh8@R0*0vH$)cCY4UP!wq4IB;M~G6Pcq!!~J#=?@y&Bp8?^ z82-#;T#(fwy@65MLQPYGHNAsh@By=v0*iw{v!q9Z*osDX1=iP}_!Yfg;Yrmu{dw|jPjNPlnc6Fap3D{Xpt&tIPtHz;m=M+o*j+$JEpifFh(niGpyJZwAt$T#D;aupcUx? zZ+5LtW>EUj%u&Ixd(|%f4@}|{81y?DS7kRatYB6sVBqrLuwKCOE~6vwQ2b@6>Ug3xhoOnvqEYWa z#sdaU4llkA2EGRk^HUm^#4vg|hXJU|iYL zq_IQ5<^;3W21a!WX2Tausu!9aUNHTWl9+0Eq1mvbNnhZ!iU70v4<_{uDjpjcf39G3 z^kAvsYu9mM(CgUHVtAq9XJV6v0JDQavoS-f;Z#V1?Q@u35hGU7MfgG z_{HCAN~6F8#^rw+H73mB6l~;uV9&LHLGZ#qM!^sEf(M$NEL!vrFbB_Ib17ijBVyw$ z(VBOnRa&6giKAt$W6Rx5jT&cKpH1NKIKkL~0J9TAtAU38tVvQ@9_=M3+R7zZzXr_l`nb+*M}vf=IL8TwBPSa?i{ulO zTVpmf^II@p(ry%=am7>A|Fe$2M1!;M(-6xL|1T|!-%`%Y&S=w~+4E~l%VR;wTL*Tq zE!{Uyr;#IpQSj0N)<_1vT~}j58F;x)%`0h?|G{MOPm0;WqQynQDcQEg*|u z%~LP6E(~lkY-kCb&?0iTO)4YMv-4W>j|ROHt!c`wHVxdYs~Ds%wCb?3ICJzjSxD!+ z5-r-O&1}FBn#^$gU^wf6>s5PKSG(?(EMd^Q(5P~uSw(|^U4elqqe)PqQAMFykb%+h z2UA+MnZu0+LxWang$5oAW`_Xg7zP&Z1zl`6nD?7-9T#qq;t|IiNShnJ@pXuJ^cN;}K{4d{b zteAcNLFMApA9xIBFn1ho{JX?1k~Lt4MnK`3Q29rSoHs)MZCx80-`OCQ(d1Fc5kC~b$7-FulS|4Z-o)PjsrcubFQFKB` zf63vAGPZubhnMy`ubdiXdF6^+#jPM!m+rtF+SS6(C-0M((D-l4zlO+G&P``SId@!* zJ2+eJl_KYi21bs=_(H~aqAkWAEO`&wcK>A+yLUT3fL-APqumW=fdD244i?oNjOrYF zTm+c)GMEHjG&Rn-rne!M>qmpi3nujoiCMkPE}|?sH(FyES`9sz)C-!D_b`-uGRSL9 z4|*-i@?qgM(Z|=%sxfmM6k=pxU|_UsU~ztN+$F=;dIFP!0E?{vv$H~ro&d9BMN_XR z16M-Rd(~Ej6%+IyFdIq4iiE7P*I?atsgt3gL8_r8C&5H36i^?Ch{3Zs84rZn< zX7@)o11GRKY*;TG@Kl(?|KHw_Nms615`M7aYsiYyDBr2hSvOdNug0$Z_m}xk4sYLV zJIN3F45Akj4|W#r3}uUGh&|dFE1I}1nSnE-QSCvqqXM(bj_C3qta*74bAPnv2DBSL zV6ro4iM-<_xT8@i!dqZN0M`%u7!ek6-WE+R7DodX2Zxr+FN1kIm>d{dj9=Vv-0;}_ zM1|i90o-SffbDs z5=62THxSyL2#Q;JWmHR4@0Ne#0C}v1~UULW|bvwD;Vq>SYkU^etR*# z=Q$s$<{xI}Z?Pmb%8Zfu+rc#q9>TotYujo+ZguFV_|6OB%elS`Mt6y@?DpwUan@8Q%Hz<_9|xI zq!8S?70a|%VwPWI$>7kj^_1R;v6%qt_04Rx9gnU`BskwhsAf? z|1Il3Y0fKgfm`Mw&j0#K)-8)##9|+yYWfLd26s(7c??t zDCh|`-|=Z;$yg&c>Y7iN|Ak(5^PC zh^;>_ar~*>Xp?X=eG9Ya3D(RXwq_a3N)G2m8~)x{9%BCIhx79*x5X=eKg*Yh_~NP_ z8f+Un>&~RBLNjk1vXJVioXKAr*#CNWXThF%&o~Pj^*7867HAFLP#1XN-&_9H;DD+) zgBF7qOb!K2JR2C6GdBL%+_9{4f1`)u+Uvhwg>kY!SlH^ZaLbvqY8x2DOeUl(d(Fqj z!N$eMp|iqaK~pmipMru%fa1UAj`CMG9x6X^ZeSB-Vl>)vJV2PKQf_63@{5TIYNB42 z%moV>8(4(o6g(Op9Gn~!_{rhdlQk2$9B);rXjCz|P2k{{JF_Hnv(rT$jTJ{UB{ipS z^*SSC8RfFPe~zU~*_()s&d0j97$t1T#n>F0_3 zboe-*gnnED$H_)!cOG$_l$sv_O|G3Bf?ggS3J;AlE(l8UBp6KQRLEWK(iydPmg(z& z4V4eMoY?~AsVNwonZfwTsg+y8AcopA*2T5~?cTu4(I_$QgFk z!8qB()bG$#r9|F~Y=X(XqRdhHSDuRTSg%+xJ+6G>)$8%~o<=j=YknlQ%XSqqvIz9M zK5OF9+3=w;ZDyt#Bg=x?A2%}uJrks|_RaYemA!7`w`ZEmW*qWW+WDraL&B`%;|Yxk zPZ}NBTEhY|*jnNyII`tsKX9Aenm`yG)%$(nAev2hj z>A-lSQCm1@nb}3*1#NO$qYUoU-AGkLKlw$7eWQZ6`2P zKUHmLQn;j${lMg=pR-8&1--9pf>jkRG%K95VD?mAx`N?&=(8QK-}QXFRPk7;>x*H# zfb*9Z$^AAHp62trz7r^jOA&oJeR2bDnrC8-OxRL^2)i)m39(VnRA&ekh%vIH&eD>a z#Wg43u*j|hm0dZ@j(n3kb^F1j(GB9;2c=0S}T-eE1o^f9CjDm-zu0;YP zimcU{Zc`X`u()nHz?`yuxd6lEn~t5vQ*4UZ z8JTx{I3cI7K;Xzu<0B1Dyy|lf9A@R%R3#joY0PkddHTOFgTo6o91g?;zAi)9*gCY<#}O_?OLvGr?6a-=4|fGcgS-QdUmbX!u(o=gom*0?uC^xbg4h z|5MHHt@ejwik#%P4M`499x(HnUOA9DT_3j=zUnprozB%nBmC6vth!fgIQq$l?V8j zJY<*q@PO}Ud5 zjx%f{n)vG%ToRwOe9yD14s4S?UYfpdiRAf~=gA8rPceBJG;8!c@?G)ah{Ci07T$^l z9HAD@zh1TQ#!g_WYbZD*;^S!N*qbP_)WB7RZQ`2D8YM0^9ag1B4IP<0Ow23_2Ur6h zw7WYbuo;|i;O#J5A?>nIBwoRR$;wQaQ(*xk4~GNOf(1Q^PaksjYq)B@`@kTs)sVxS zc|PNYL#KpA2mAF0iYzP_IpkOv_|ENUUdI%0cC%_>>)T4D)faBgV@cti`){iD3ybT_ zjaqgG9Mud0k5>tBX3lYz`#hoD^`+o3{Yid3X(6$%7ftN(DmfZg{(@QN#)r0>tPguD zOjvw4i*0OgEaWQ{IFz)><-`dWe&fs0Dd|f(_hzp1WJ<|)6uD&1^+~spOG&~}(C7i9 z))K|qWjBuMsU&)-1#A+TAmQ>iE_1nU&7-LkPaM`<#(2(Vg9e|-20k_xLoU$_L1vAF zCL7}dW)BA^)?HGISh|vYbDlX$SvIh!Mm03a_&nz4FkrBf-_&r*^>w5}&1sK-h6a&2 z3Vb#V3^5iWH|yjQ7*n+0GNnA-ddlfz$ke~rg2a9)vT3N!64-Gt^hV>wyQiAv_WfJx zo^Nc$-#n?BQl4<1-?8IS{b6+25PDv7ADmW*4{{hc7F($6U3Z{!81{~=>7CQ4bIS4Y(-pX*K zQR-k83!9JtYxW0+R!+;?Os6DQuW@8xli0(+9~98o$9j-oGvgfhybE&8d@YQY`##>t zdn0hF@AHCnQAHxFPXuskEMQT6ArYez$W`*^@b-he@A6+Sd}g}2az~<$@}3y?ORiU5 z^Vb^ub7@L^#cu!RU}DIkNl7gQs&lRQt2(y$upF~iJ74j}Sn3bY&4>>y4s)7nLL3qb z?F5*9_buRhdHJZB%Ej(jg+<&!0!Lc7Y+BuBFbRHiX7OG&k$wJ*B$2`e8#KQuu&Nm> zu~s`$?^%!cELd8w?k8@N*pySkbsfBgKKCdIsAKm4wIY>MpjgB2O74 z228o<+bz`V!N@gF;4sIXCmo^la&EGG%2W3*J9TlAOzPS>0UK&x7kU3QVliOojCt{G z`0 z>dK4$*alA1-z@(WRyfEks7qiAIgzhfv9Z_XMUr@H0TXYTQ|m9@y@CxBoc=|vW!&>< zP2CBtqZ%xM94Yq`xC#?Ocozw%ak)mto z8RE=Nu__rptN-%t!KW4Nwhrr`Rrl=s!V|wa|L&QE4XNjx`h%!s<|;N4T%DrR{s zY=h((j|SGci76{>rrGvhRp4JWhly>Du+zu72J zj+%cC@J29m&5&g*+n)%)-@V|P%dyViKe zGLl;l9}NwkBGcxZDlU@1InAQ&^AUy#2|lR>4ezHOV9)3gV~Ves!8X&Sf$Pkz=1O_4 zISgWH$5>Z1h%T6V&XY+mpjok_k+p&0d;&xLjQa&D@{3+1KHhUs{SCj$pGFZACgBff z7RqoHq+D1L)8J#`z`}AMqs7CGrQsjLABGh+jBF+ccsLrESq{uvaGXU#box^XN0kGz z7S60(Oqu~s>!t+opJ`Zkq($VRbf)XQj2RDBO-Y*iHP|VY_o0nLCquIH&)((?IqN^p zGp9x#ni$7-&zb*Gvz^UjV>ZQCGjF;cog&Y0sI>Aef!$8IH7^-htJ|4K5Pb# z$|g)5yiFoEqF5I=EWRPSctHbmjKitS3D-@cx&Jtb2u!$kkWuDJy zW=}Sq-E)jhWa!UJ*&+?pW>0|=49>X!Y4#w}P47@*%eQR`B zq{HPT))gCjvElE*rj-ryE{$9{%Tg8{V7K5?&N{|Cq06+_i}`?q@)IVDm1&wD&0C)| znPi+cxxu)S!NF;Y`{b*_DlCp{p$DW*+(rJl$n0@^V0*BAMZgorDgL`oJ)Y`5)ilnj z*86FV_wkVDL3T~X5zNn39~&_(G=8TL7Nu(c;&4UZ^U!};s&*F20@s|CDpVD|Og#K& zV%bvF&kc)tPB?JOL@5A&qP~4uv_#*`9=F&v4S8 zaY*t5qxg$PjuZz6F|PV24-3v5sP*OTxOYIfAwu~Gqwtvnzf|K{?>NX6xF~Qm$vZH1 zq`0t{G+0e?WB$OP;&MoQhSYl3gVHXJvL=qgR~)yBFnwEa{E;RLlSz~D0SBHtj&c#q zI!2DaKSs&SahAE@vRZ)U?%icYo4j0}Eb3Tue7nGuVmHCgi07OZ9=jqATc2Qle(^!b zvju!fsg<9Y=YLySnWS3vs99yp^YAF~PbG`_uRM?1c7XfE^NNWv{}#D6)a%U(nsX<; zhC$fjzWM?tsfboZo2shD7e3`Lg7YSgBJ3=N^TSDc!~kn61|^&%JcmevSIPXD>tFWmZmFK7Uer zh1+93Egz)_r_#5QQ_nbldcg2m;9yjjaz&-5b=UMIb)Knb-kdr2R4I`mYt6s&%j%re zcn1L!-3i1tg?Zl;seLyu8hJD9F;_vG$%L-D_mn&QCNK^ zZL;D4?jMdDejJoGIm9_3fa!#*tSL9s8qdkHM|QZy>8(38o$nl1?7N4q>OtF9^6$O> zz;aDqV#v&g56WL<&d*vif1Td^vNfShZ7LE@C&b$1{|HJv;1=Lv)&VTESZ&RDk z_HRZ!n+9XNWuwrGhF@)tiV96hU5xS_jeJkq^Ix!Yv^b=0VfYt!fc=U%2ScO8l>@F$ z4zdz=O`b3*7@T6SIB2rxu=SF|;w}OeFDCDpq`gT>{y1df&?K=VMz-g@3a67?&q1CA#}wW$Du^^`ntae`WK!U0QtW9^*mKx^kF$kC z6JJdy>jsB^+%XMH>KwRh9C%+i&PzCKzJl4Tg@NIVyTTPl(}ZRP0S9IQ2R;uc^EHR< z-#97+9Aa$XD9Uu{q>$O3X4PtEcZEZ|9~{kB9Cq_Q#2(?WWLkr=djoTUA$!X~m-kF70-O1I zxH>9>H6@(b1RNM1{A*CmVKz;1R{Y_pvc`fbO#oFR3xMqLnrRo0-0`HmaH*7Wg z>>MB9q#DwsV$r0k<0Sv*pv;X%fenuRbB;!?ILiHHisI1&$rg&@DrFKaj1mow5*3aT zDrMp-WkM>Bw%xDJ)is^E(fQ@y=g?lEyKyIeY`U)QlCA7>P{YJY&WBM!!KwN!li7z( z37)23PKPvXm^6PJWM|1pJ7K_m#X;ePrykF$-wh0m4h(y)Iex#^SlrOS-NGpUfKgsS zQ*}qf&DMf|8@&cVWZXnpOXWOU`jQb1p7ku$njfuw}|& z%N|qH%PVc4ESQ&89KHRk-JkFEpI1h6SLp5Vi?u(b6w$Qz&S{x9C-{0=dEOjIV1931 zpUjuwnPhDyaidY9=V@DaqlCq`|Z1Zo7BD(2a(MC6tE*fn(C?|#R{weB&)7cesr}I}<1r{Z!+L;GrDm&X!&gfkZ@$+b z4u1=)oB!_Ux_3*yZ;zf|6CLl|>vFj0=2oRA|4ztkI4HA)vHuBc)RxVO@fNnljp8{! zRXV;@+oCe=s3t5(|%U-_t~4fgzhx(2G}wtGO(r`U`t^Lo%2HBN24dl zq41mi!YK`97Y^`RI4YPpE)Q^E4mjW%#lReKpvJ>JeS-sshr@qS2SyGD#t4Q!s}2SM z2ga6$5{q-QVwSMHIi-_s{)oZh=ERlvf8|bo`&xgxrtchsDub2-*LFT!z>@mw|HjQvzwD|C%`OQ2|YixV=@OM9t;@AGcFJ^~$4Wpcg`DErMxvlD7w?AJHW`9YzlH=bQ2fi9dIfH}z1y?(b9p~S0 zJTCfZYqz|x52J8^>wl9&3M?o71vs?zF$mvqa7u5GaA2BWd(hpT-9e-v;h>`%xRHsC zJx6LqXM$SO47+?SivtgypPj+pCD$4m^rVrUN6w=4*8@hD)v5|HJ3cU9+LCi+P3Gk? zzuB8hp0Al}-QG8MPyLrScXzk&L^V+o$W_udQ3{+C5EYrt;w~k&6qBm-|X9 z<(9eJQ%rW}k+CRZc=(8wN6v0OFRPb{)kGD~9S#qJY`LbZ1T6UH;mve~%Hgkz+%{d>?sK}+DCOPT#j>KgDdD5*{Rx*pPJL$;9 zChHNv|ERe|e$uj+x32L9w6X9m3b<_F7uCn1?zf_`m5t3oQ;mTkA)$dqTHt}oRHvX$ z^+hd{R;tbH=;~Ur&_dKq6J-jAyp?JM z4sq)r+R)5z&tQ0D!VSORvuC63^aPvU*vG)wBDLs;^91`xCzvM%*6m){{!io21SU(K zDb67~Sqw6^F!3z8;mjpnw84>GZ3cr1KM&W-WwT4}Jv_|Bc|p4)MBI+y;Dp?5Csn7% z9#J^Z$PzW-$<^EqjE@vJf*KlDO^}hbuB?w^&rZoFN`ta#J-WuI%HE zt+p-)64)MoIB*n5ndo z<==GqbIL11*xwkq*ax{q2VY?m;bNX(@Z$n=3x|{gW0QnIK?1{MDTVMRjU^M)q%w;p z9-EozRjG7nN`_NI6wg9at`&Ugw=NXSp1}0VIe+G&RYydGJs7k(81_84Te19N+pOxt zr^@4M>v=ZC@941!O4#$|(`tjmV%v;Pw{0)!;V?U*6U=X?bFkuU?_HJ3>GxOORQ++k zQ(sQpp@gBa&Aje)UcZ~dM`erqEH53?d30{feC{IpRBwmIq7x6>q&^8OX%%ZYz{q3& zK_fIPsIY=XHggXH6Nl`M3kw_E|DJF)^*pqIS*qayli&$QKBEf_JX;dj7&#nR?K z+As;VE>QTVvPQ9kOXWbL4x4u$omdx@UcP)Vp`VVm+aS%z_sjxGw!} z7MWqtYT=>C7t+D3A~lg+?N3g%_=Cve83Fq=c-lDw5BMrOOlahHVPMfKXmng6#u(_P zz*aY*MO-F;At-VI!^|~~vv?9YojV-)8$U2}898)%ylLcYn$gDgHn=_LNF$rl4N11T zfG+n13XDt{0Sy1x9CVi5U@0_9Y+!T~nX=qQL7gGs08@;G^C}ZPMl~T%OMZi$!U22^ z8V%y64HhnyX^W-5*d&=>yTq%yNLltxK>PlGs^XU)DtrE2AS5Pwd4{gpo9^NdO|s7l ztnaM#OfF1PHrvps9hPabSo&?NbhTlZU zqR+e_C1wc*2IoRW3-Ny!rX1@%9i#o}YIj9wqe#dDrimJ?rfnK7)k+^_eyvd#vh6-n zyUEq#ONIB#1s>u}af01ZF&fkMWw3Y5;w;~EC1Q5L$Bp+78ZMT<`ogV|J0|DBB5tvU zm#ka1+q$SGq+2^X`sQ-oeSP-<=ZvI7{HuO9OQkGet6Oj+-{oinn@PeHwx14;P6AA_ zC5-$X1}kC>!-6xd(ta+K#DkQUgoKw#nt2L`ci0$cYQ^kv2OOh59^<8c=Fk-ZpX6Tg5+BWYl!*SbKoha2; z#akA~a%cW7y>;m<>q4H>kEU*D_@{a+s_DcM&EDBg-*#L4GTU+IM%(1MGBGhPlX#RC zXr|8Y4%W2~va~K%NzFZFJHu%EPKg~1i)KyeF4edwrJxYXqOd@2lcz!`=zPn9kE?Qg z9&?K;9B{Jgw3iZDz$hOuQ^_@fS;}S+yJ!YOxSSzxcxkefS%r({424$r3}=B-g`)y8 z8x9E_b+P4OXz=V)jboUy*_FMa!Jp@LqTs~UDuz0OJ69BmGB9c|3sktM?0NJ^y!mkO z0|hgyzTBL;Ixd-)3qIOiUo@?3!Zg`06WWCZSGneHPvmo))|{go80~#WZ2QsfWcS9? zz4D>~}92?f8{&Uwn6?kf;W>Muc54T@DOU*(#gqJY#Z3uYG zEI#4D6Qu=mU&I=?C*5GvQ)=uEy3xqMa^gVx5rH<_Ck+hp83$RV4zWmGN#vg}gR!Xj zLx)wt!lyPNPRebTT-kTdN-kaD$fP7NnLUJ?QG9|U?-uSBi8BY-jUO=bBwS!#d3eDb zQ^x0hHY`f~3yWhy3k9wH0~j3{ZU;X!DBC>sZcdu=?6IE-i^_8H*bY`_&wn1O!^{tTNob>!_~y!lylJ zb}r!#6meyiV_jzBZ<&_B=Fq_TX93#@ZYCwYM^}_tY8-@iCGt;FVBfKTJBxwE>K&5< z1J^DGo?Q)WF$eheC@@Xgz%~1nV$xf{`R>4^MGKgWI9Uu1^93}p zF)6Y;FbM5wu+>|@7~>$o#K;oV$;76>!s00YZNc*#6=?-Vwo9ut=dd#VlWG)tcac$T zJ)?{Qv(p2%V+m|#2d2;HT;GzUW|5+q#Bu%DQWmc@;Xw+GObfLyIm(7jQoZje6Jexg zIPw1lzRo#s-?A0NU0B4h_Mz0i_fk^K-ckj8Zw?5ZSt9DqygecCMOD=m=r{cviQ_Jb*pSxoq+TlljsN)Wlx#p%+(xo<(x zrv$cLh1@f}_PofM%yo!~eMYGB<{z+$4z z$)v&WwNPY>A|Km9Nj+UYzucHLsto@WgrXMk?N;ELu|P2-$A(3b|3?mk$D==d3jY}Q zE!D_4VRoH=Q=AI(p+0^&Me#$5HV#WI?l&H|r>GVwB>eN`+rEyD3rXswPt$2OUoXonFKH9veRS83f9u zR?lN*U9><%A#sAs0ZxX6d>SmUd~0cGVWA|3~Z-kMVuW=6}<`l(9g7?JA=L12fY@!H_9Z+Z<&X5_#r4RYcHyo5?ddS4$z@WE4Kp|0PUn0+(1gTdJVZ6`PoSq0ARBa1- z@_LPvo6>?T4@TKZ!Ls)lW%h`KT-mDnw^PVcPoqujRuKAgB+TO==bHnPYztjyX!9v6Fm7^vA(Y_mxln|KkyXZp;S+;ENuyMkviKW? zqFE-cm;5!CiNqAV-MC><_@Q-tTnj}GCCcvmlE;~&dOgwd>DB*VG)pcuiBw+_`Cb~o z`eDK9g-Kbj-%pv;yYI8#;!XZni?xm>%1D&S96Km0w(Y|TM;@U?0s3b5`!?=8wC?n- zH!78GON|&(LLIp568TLW{&7vx=byBiDMvxz(*p-i4%VR5{Z0-nQxy0;5=FcirKk9F zeMk^l)+jRT0n?-d0)G-%3JwUwooCe0=JHS!$mpE*W`WzEgytxgRwrX;2?fSi4MM9{ zuhS9T8KuCP*T6YNfosk|{(BdcJDKKgc_*-Y-RpHtMH60g>+#0!y^{BFo5jqI{A`_f z6-B;3w@NK=l;RVYa!B^uxw)e5p>^Zygww_{W*eF>UY5PoDC1C(crsDu++-P+ZBj4UoMU%dh^av9%@w0k&3;|ML;;pW5uR4ADhs|Zy3A<~{__~PvKrW)YqNh) z5Kv+in6p4&+If*Ji6V;5ueF0%Zxx}bT}VwGfue5TZmy+%J0O*b@K?kF-WpL#V>CPAo} z@oJIzcJT;{YOjDcV>4|Hex|O8pN;%23piQx9`Gw16#3`CY@}c?m7h7lflF;=+Lyqz z9SZCj4D3e~gf=nCC^2%`ISS+{@J-q7UdqTX_0G}pfTGg@rkI9+x&s#j7zEr7`t3Mn zcBhR|rGe3@_=?PPl{X3kalrx%5BZlYuldI$Wg(@=5qGv_%}WiVCx6^>>ZKpN_G8f2 zW8kX#A#>}i#appSU$H zDKJApNa~?!HS_aF7nvLqShW1?1r~BPbXhK0pzZaLFM@&D&Ox?VAv5SH_m_#BX$wB> znw~A=5O(63{V~qg1x$=84;UhxR2L}-+*>d=Q^+naIWJGY?#Av@{o-&o<$oNbKy%)|O47Rog1r#cyPYzc|rr zmUXx2qgb(FqIg(hV&wAFol`w`v|KE{TadVf%R@)M)NX!tviABCOaJr&HO89aJ<%p zwhQvNUc8*>YGFI^&hqkECr=*SF;!+-X`%R(2h-M0KeH;PEpx{7i<>@4E&UuYeOIX( z^P>eKQi&pu6!_jn2yiu4wW#rbQ0O^wfd9{fBq_(${^yF$u<&O+U~XvO|2L_DE#rXE zQ$zM63Y<|2F0&Qb-)&qkI45US(wupVR8$(8&Hf5*V!Sa=fpL#_o#AFdh6OHMr#I~S z`bT6^7VAU4s)mfKf9=(ov~MhBDq5hZYABl&yY*onL;js5a~DcYQ(PKcb-nuT9fOL+ zK9ct)+`Zv>L-tBy;>@`x7s?gSmTR5+_iouj?w+XAN2azN=372VYWb|i>`~#YD;#*G zLwD`9ohhw&m%Y)Low15zCEq`VwA9mluN?S4P2jIu%s1~eQ$vHul17nF?BdHDg+3e) z`=P)b5+Zbjftg29NZ_D-z=1Don(9{`+Q2gViu(EKER8}r4h(-5i1E zrhgH(N22FkiBvkyFZD*yC}FkXXC|S9xq0$T2@FC88EWY^&!4xdf8P3hZsPUc`fW2C zCF_>IXgtkq{b76m+!o{VlZ)s5nz?tX;dYrhwKAs~L%%fW#HKW7SbRTxU!rxU@%IG+ zj}Gv2J(R4ydHVC`*()13A2sY^h~nyyV0K|(6`IEy^?;e-5G%KsZomvT_PT(BjN`1{DK_&8khDYyt)r6Bx9bR&ok$DG1Z@V-=Nn<>MTMG0CyVpo<*<175shh1=a&zw9ZIX6AdLOqR>5*2KGw+j0 zIq_b?eIHNcrza;~so8(axgpTXE$S8Skzi!{iYqyK`n3g0=T`(xc*J9LRoS_fXP5qq za}0_HuU*nGDmdDZxIK?Y(j(sE#Q{fF?u)W6J03W+>t!uDQm|01Q-o8Yjb|dKVw;qn zvjWemIE$wqDvGw7z!O89p`jE`$dF_T_J&i znL%MeLokQRhD2sY8>6$lOdJgdO_~;n-C^Z7X*i_3bk0=X=w z$mZXRaFeqRmwhIl75P^dVR?JP<|j!{EN;$7e-vW2H052!w#aEwS<@oY{p&XSDlHQ% zh?=RkP;E};veXF;T!kh{r$n?o6i@Jqya;2=%vLLBEIO6A!IAAuYQUZ1bD`l5EMhvp z4zlwxOyoA*`C=in-jWB0TKV2F3OVszd6DG7QRDLKD7(6efFtX*$nZv%b)UA&Gjpk2 zaA0C&=#b)3m)N>6P+YFz3L}q&>gx@MygV+qNmxAYk>7POERxT6u2j->SOrD##EiyH-Of28>i`~v%6~zDt zhJ&I44h_s4<_!ghMNJZ<*rfh>7M1L*v3-LvM(9WjNeq(+F6&fhXj|L+Ob&iXOb>0hc=zmOCcgj}YB4 zqj!64I`@z0XBR@R_dTC=JNZ%P`s^hW+cbAZO#T%0OfuzF%um0kQIB^2OnWsUm~Y08 z2b~72LMCep4zY-A*f1+^)}<3iMT|FQyj6NGX>@W=YmQNo#EL(2n0Xo)I4<6ijCJ6c z=Cj3}qwR?jC#Q*oQm2YSfx)S&7p|+^8UG1cI5f*$l|FQwLu}8hRZL;WS2-|j>eGD~ z^69vJVz6jfM`CLR|7iziog)_*jhdAk{BAJW~ziX6||XdG8E7EEGTHX%pK3S ze85>b+2^ksiWbN73$g3CefJN27GqYkdBV)+F1m>lkVWFkTp*C;RRv+-5 zDKt@e#fD#fv&9)0IW`#EI$xPwBf#m%=y9Ok;6(#dqCm5d+XcpsIm;RDDKfG;ZD`e# zNa|DKa8`0-^#2}lsCr#j15ZlBg{0IinxBg`g5)B%ZA@KJ7aaK{SgwKL0$;bPfB^%` zB%wx@xC8uC6T-YoOQxq<9Gt zXS~39m0W}J92S?-U6(~27F=sNlau##rTp!?@3cPbe*PnGP4d%atPUSqKV2~vY|!9$ zIU?Y1q>;!|rQm%2NCDf*ns^7Pn!>zdYK_cL8@b24XJq|Gs!x-C{U zKYtB})d|LxcW=0vH)-E_xUArW`k6`l-c8llR=jDvKwGic*}3sjKq+&`y$1F*F;7=` zmCoSZaUh~9V^*Ho^YSwv&xX&BjN?&YVSfJiknHs+f6pVvf|D3n=kP0bI-l7&FMYvg z6$Zu*=Y}MaDuL#OKNZ3dzTvdif1sJt$3I$U#@)O-@J>d zdY&0ELer8@&eqhLe6WOrF`$9z+T86nL*t9jKD&M;pV$ir z*@f5=B@ZaDv8!JDv&`wHds@fDjwKV^%}o}rN)Za*wn}tuLJ-sRH5yEn3tCL3IC7@0 zu{dVC>g$@+nC$4yLECq~%k8VSZ8u|(zHv~_Y@_9J@5gG^FOQTK`CF^cT>L2MmdFPN zp)(3yzBx_8D=z#}Gnv@wBhz>)T=#ph_oP+Jtpq!oPB4qkP+;ZZ5o8XUaEK==K*8oo z1DnOa4bDsSgjQBRb`oN~Z?4D{CujOc`oZg?tomh*y*fe6(hmdNExtG~w>L9zm}WTg z)&w+juQ{UGeB{C{CI$uugD*~O91gc+D$Q5~7#Nr*crqDnXjBSF&Rb$8kjVa>ZJTM6Y+qhL--)^2+a{jB z|K)^7o6LM>VSTF!)y5S~!ZFWX)V9?9_bE|&6Z(N+<_D3~r4yN!PAq$+rV(_3WB!G* zrVpHz1{_Tr*n0!mO$At+KCoIbFqtqg=_Rn&A7D>U;JoV0yE`GhM)K>ELH6>S^2 zZ*SniZJKI01o8^%nlEj>=W3_Hn6t^a8y2E;!RNeF+G! zwJ@PRaXaUoHm-#YTnWo_LjzbsTn+voSIc_jyQh&!>VulNFndz~*U|)sZG!AR0vxR$ zSd|l+l^vKp8CYy5vv>xCPix@b!N9d6fV-%TJ2Itb_XX}_9lSR#>-JT2BYaaYlTH?Uzv_ME)Nb%cckyXb7*10QszX(`)+?~OJ(Q5)*Y5;5a4wle_ zR__3|*a|jpfzTfxGb#&$&Mj!m(vZlG;A}GFykyR~b8GSS<;led*lulS7Tn2{m(X+f zdu`jrWNXzF|L_hO&pN#$F_Q)L?`+`y9?|)0dhQ$ru6+k4?K{9dk)v!@0^3{-1~G+( z?aR&Yi5N;GNCz1hRt9j*zrfMqz*@P1eZm3RhJf&-fbcp7|B^J0m;;QS4X%+#xECMb z_DGW0dw_Ru1MjT|P00(GO$?Z>XoT;+!CK17W;KDuqJh2c0eeFN`&0$qG6kDiOBCm& znMqx%5}%Zq{yD(Au}bNm0PFh**60A%J?5<53H_a&tzkR*`2&~=655tdXgf2ZjeX_J zvlAvve!wy3CCB{8TJvYcyV=-_F0kcXV5^+bo+mJIr+{XZNy&cJn1&nP&#q6@o08MN zP;%jA?yLozU#92IR^U4PfbTe`)13vp`zDkowzIf8FtRGh3K%qGE(*Q1gvp12qf&sK zFM})Jh$BdVwZVbwt^w~Z11|RVbgR#fR}>PgHgJ0sPhD_;dso8zuRFQ>!{;wBGLZei zx?6y6w*gnD0Y_&8N6iD4BmoX14vx(`*q1c0%RREI?=1A`Fk@d(wJu$e|7pN>BL>M0 zEU`0Yco?uIKWM0C;OMsO51+yLFVlc2)OhCRK(5&*c_vSo&{wc{-c6o`FF7opF1DDO zT&uud;K086B%9z)X6*?}v^Ovh%`z%?p`_q$k*m1&)vr_-5fs^`r3o+|L&Il#N% zK~Ag82+w3lvsP?OHjlilf?qiSGkr=LGJl;(Q*h zyq61BbS~h&`+@In0>|bVEENXKR$COOr$vgdVhCks@TyqN|J05#fWb3ihDQKvdV)i5 z!kS(N&Itw_J(B%11K1=FFbI5LS(d=HIN{&oIXf3ypIUtT6o>7t#ST(SiY~CPzRBh? zp}lGcQ?9{8FDcIHpOyrkYM02Ac*vl7vURq?%-Mn0m%dZrdv}2MjAiFhPoCp1c^?Vr zvz_KW@_~EN1SF7mN0O(H*idmT3I!X!?2sf^mn>^0K;2F&coj{zf~5zoyvVbf&V=NSJFxri4)Q* zrYF{ORyS`ktIJ&I(a6B4kQ=43CX<1q^#VutgPCSloV^BI6B1g#yx{b{z`*-~d0E4} zxZ7Lr{91eW*Ww<5bxv7J*6ytJ?Aq=VWoz|$-QD9WLjK)Kw)@tuY!dQ)(Nd+I`tLsQ zzWcC@XLspg2cC}t{7(y1<5unX^NaiNfelISp{YSq0uAbi7KT`3e~=A2{{9IVMVJT}cD%qfp>Aj{3%=< z9UE4BRN$MGz;#+;{=@_9CI#Bhe^g~AGwisvHa~mD2TNV*BctDHl^*N;~3zaYr5J-2Q-X(rf-5ELKkrtpE2=fsZ@8K9r%R zYOrhi9#`RdoXxRcdXEPW zlPL?^|CT+w3Z}m7c4ZB*Lfcaz=MH<9!n#_9 z0|gUixl2vFy?lw!sU@CS9Mc4vs(*L$+caJLu>I4m@P0xzOekq zb+wE8suo*{1Jl+O%qMSc@|(kvZou)vo1^0arztn*ClOQ21oqYi+g8y%SaJF2#4F$gm-_&7##OsJaPYQMdR`Mt)PeINLyt(;lS!}YXZ8|%rb{~4I!1uV})SV3* zW)^UU2yA3)R^t3(aAL~j(>1=k3fXT?;FFp=Z`lFvWd$5=4otR}jr|%pCcoMg@pIR% z1TMcGjs^c1IQY>2p`hfAhBWM2a#mcsqCf#9I z*IifCFiZO{Th0Y$>x7f#vro?2!&LL&`k#%~v>o`@+%P z59Ib7;QpV+;pM=#kn$rhrxL?VJauk~% zQIUDy>oY(4u&4k_`ojBHvbg%%r{0{*{jKx5ZuZKK1g_l%dvmOKcf8(IeRor)E!Q6J zwF@R(*SF@{d*SF_2ad&lO&_PQh<%WkXj0tKW!5ROmAycliQ#dy!Q<_7xT|U&vszv> z&E=XY@OYlUlXe5%i3LxJWkiN*{1xN!0Pk1GW$-%zUGc&Pv`NRir1_B zUdG!Kwe(^7GpDHsBWHQPe8BhM0pG0x?pX?4+Y7i4CU9>n;6Bpu{>M8$=GpIG_Oh9H zGB7)+WqdN=VZMKHsoH|oAH)S%mQLWi(Qv(w{ngd37i+UWZuee!ELEQfLr9Ibt8ZNXkO z;pvIDCp$l!yvQJA&3iE9R{Pxv6CWO*eX>=xxV|Ij9ryQt?sL9e=Q^msbEJUh_yfM< z0{jjBJO?ivay3w5yl=n#x#{x65A6OMIa8SV`guk8UmdW>c)9ZD)UMq#tfVXUUNiW; z$;5iI;q8|y|93BQc=dh*Z`%fmI~#bb!r0^AiX15Fo3;1RgX-0OhBe9`j%6AgTBN{r zdk27yC;qSj?P z^A0v1G+U^-InkMkiCMyC#RLV%qimv5CISr`A06jXl(r}k)RZ-gUF`7iAJb885sl!* zDSf|KUJ0LdHp=fa5-k>4-q& zH??WYi+-gOwIW|%Ixp{+IWzO&mYZje8A~hWI=#QK@zB|>OUuI7m_0n&Eo;3#@%F>y zliPAVcP@T-NHlkPe%9@#;KSQ;qqpbXKBTL@z2;8wi}NRa`~Cc4cN9KaH_vFYX7F;~ zISH)VY66J|Kg}^(x^Rmq{_@LNYN`e&7zv0>&5}}G z{?C8HFP2GxY?dj97VtT;2zqF^v`ld3(9BxN)+)7hdf`1q_UZMH49^|1)sawW+;Gh4 z^_s2cyts8YFeD@}Fg=-cnkUeOGhZU9OR;B0#e_TB61o#^Bu(Hk5m++CO>E+uQztbh zJ9}F^$ynTLaqGjHUd6JEW)qW(F}Y_=_X;$1TOOLg%KUla&H0nH^1yTfmN3--H4|1L54oGk6Bg>Yw5;UO zeDdKSoA#W7e^WA#{ZyY5v+5C>v?t%g!%b`g0SE4EWZ*ckX446!*J~IA7#J8j9z`W_ zs#sjyE}Gc!^1DRR1?Cx(b^1J#yLgQlmQ3;bx8qad=`B~*7@Hkgv*V2MWuM-&qFxM4 z9M|J!Gn?7J325oRmG?N<+<{fL#NwgQ-(|CBipToRbvxNuB2%RDw@o_sl)<9izE-WO zq7hx1{t}5z55HUyyrZnp@I$WRz@onrJNg@0woJRaf1Pm00sgq)M<*R-=0tJboFkC= zWMYaUBkLrA*+&glXEbgK(BPiWb=)O3XbIc&IXovDu5RFq@V+ON9dbZN_BpFVBZG#4 z(>mo-CtRi^v}pQ=G(FP%mnN)hAyaZ+zf!21f(ug8;@?ObjtS8(3>H7Y5IHE$(#{;ms?KdWq<{UT6VyzN$-X-~6^ zYiY98jf3aD7hT)@hT$!b@`nb&Qxn+U=OhWwbYQL6Q@l#aqqk0YhC>a<4O1qzhKBfu z^2{t93~U|={5M}5Gf86Na_MoLTIs;ztMswcBSAo@GND=ehQdr&lSP6}0xX+Nc5F}M zJv5!SYMS1jkNxEjn51_zi0Bsua!3gi)Hq0+Fy&FWMlAM=C;o`8pO?hZ|2 zdk%93-f$Ax{-8y2!h#vx ztJ#A_rIdAC_ZF^PxoQIoe?S8hON1!1mL-GNrc-rm7TxM=+ojCL!LX8lf`jb2K(Dz1 zPMpdMnuIFcjy+i8{Nj;x;KA0f z+a~tLdNi>yFSr<0Rv{+!?t)I*&4vD_oA$HBG;k(sFz-9@(@)o=bB_0oLt=dsw0?LT zm^Zb6Cq>P8ro|Dy?zw`Or?)9cT~rY<&0|o0Z^d@8qp4@nQD^1UfMv^VP2bPEIN8nP zZHBwc#4DCHIey=(-1>48m<42CFtFNeXq+MSkmIqmWt^8*K1@;D|-&->6~{)e$SNuwor&B}T9hp>d${n0ZJl<^*Y9vIcTW7umnV5d61k!^n1wb4_?`IG z(RoAEAzbAEyO+yE!S0SThj+0A=$EE(EcSfic=bt?*-wUT`!@$m97U>cFV-Al9OM zdgzD7^K+O51Z_ThoOeoX5MMEoFYCt>0Z)Tgl_$SDd6#{4s$a2G`roXBJRSzl(=PFbTE>PAgRteE7VC=MQXjm_OZeiaArHDg$6&p-#7_M?WU{uv< z+&FdVnwV`4th^g$FtR6P$rml`@iIvkJRQcO{6q7WVPl)m_T*eA@8aC+vlg&e9BAZE z(zek*a&l481LoX}%TK00RTJMg(QRQ}_4UUb;nim@8WnB7*!?7_S!!>DnrYd`{Ma{6 z;*9f;9az}T?Wtg6-Z{ZVQDI_-lf)r`#v9H@zaLtShcv-&|1~w|U;z$9x8iSy~Tbm@E9*R;*&#u(^~?Qh|Z-)t?4t83)F)32i1* z7WR1h91=WT_RVg$$lEJvJmqVyN@V@fJ0@BX{Xu^A3h^f`t*VF9Qi@jy=S(s0?Ax~H zZrdj1sUK(9&K8|66RmgLY=ZW_nYE9lA4=?f;T~oDq~+chZgN1VwI zdH3&3CnhtrEUi~yderamQ@Mcgv%3S6P{OKTYmZ%FJ!1b)c;l>_&XXE>T8=PEbxt>1 z8Tqe?Rin|qz&a(FEiI#M$82k<$vyR_7Ij++bKGg%`+0Zv;~7F6HcAi76cm~|wDwHg zIJsamvyR7NJB^u>53cq4F!^qRxn}kD>xXT0lBejLmYkxr_pEYz(T~M*PVR9%IP>*` z#qvDuB^>RqKO8JbXfI4?f2OcACVjV*2oEE}xsTm@PfZ!5Tb zU|FWYvXFr#)`3MqfN{Z%1#h~$3Od+=K1}wQaX3Y@#eGAIdjyNK0E_nrX8S+R_CJ{0 zGi6WhV78pmth0;RtfAR8h1s=+*`k5jRfNf`f;saxqq%~xcSl=L0bAOKBVH4>crLKk z+QnlwqxtebC8j$Y&F&_yHQ41KE?~oV=Ril%5tBq|B z4lLGA?kg#HFzr!s^Is-yvmeau)-8?-ExfBtoC3I zY_ zGp<#+imwPfc(^h0kA#i{(*uzvi;J$^PN!@=9W9vn%2zM8UNrLs&!N;M=VV1r+aKba zbJ6C#qMqB%V>;F=Ri*^pdCZ!gxKnR|{*nz7TyD&^pRvGxMT=WN@ZTqvyAr24{@A2a zz$B}{AQ;fZYt$OY(O&<9y|Ti2YKxp##hEr8)}W4-AcpWXj@#?=l+@rF>z7rvo) z45L(-6d5EZD9SC`vcme}WOZ+g)*d$Jg>2akr%pe1{N!^iHRjmEO9x*&Xs&;woL0-1 zwe?uOi2w5!Y>m>au@~4berC(R(aNOBCGcyD+kr!jy9GEi!{^=>;8kide8D8Upvgv~ zW#7U!Aw`d}gn-NiEa?kaGBvI^_O>`LZSm+Z?C3e+HRD8E?v2waY^e!s0ZT(BfAR6& z8{&5~f|oT^IQ8uHkc+M}3|$Lmh)q3v!QJlMTL=4dFtj?tqf!fJvZMnqe{gAEDS3BSatDUP_Sx4Kp?lZ@iw8Tuf;Jh}vWp04i3YMNJ zC%B42Qa#$bOnrUdh8&7*JNM^AA8Y9Jl9M|o@7|f*B4zC&{&&*Uqq1SC!V#s6k~2c1 zt~&6|Ioeq@S>AP1%$60!!IDp6yzcK{b1HPa#?4-`V^fJjdr8Kn>?vU@Z1wKQ>ZO%N z$)-%%6vAH0z&_;x=gbYq7d)6SCu707rPDGNPAFk%+{!=})Pqm2?XLWNE^LNk_e~tujg!Uc}e{_^`amuZ_2~ep0G)mSz|0LS$1~H)-9O*BXRBm z&RrX)aX#G1-Z`OakN%cN_w3#V7e{Q5nGn6R^|q_;eb?0Zwx_rE)!yiw8qvA-#;>JU zy55Eu#Koj>9CZ^&23QXtLCUpFYiQCaoWyGg?TFut=VO=)%QaEhW5IpE!h)pH|pL$5preM(&(>CefILErDWgm zoN=QqwJjwg#$aEBU+CE*x`BbYzG($4$4glzebqTNqj~b&W_`h4R<^|r3=E1tSvZUs z7#MUI7#J8N7(*BtIQ}s_=cqAmUvSWzLs%>3gh1jnbAEZJ774@dsXAi1`*vn5U3|1p z!8uIQbJLQk6N6VR2{~Dm>ODnw`H4A}sh++wy)uGkda8C#nQ2z5Hgl583x7?!r5drP zLRMZ}?mJh>(`f4}5!H~*T7f}n*?#lgmbCaTdg2qg-eZgThqiNxYKL4~!|5|aUqH)2NSp6a#c@7yJsz$(G0Xi9o}Mnsb;hXO_}Do?^|o!B z0+o;V$orQcQJVDRq}KZ6@SR4*+k9>ZuUWDrH+t#4nU>k_&UnV?i)z+fKen}U>Wixj z%O-kx&Nf*U^y>1`<-W6JJ16aWaPnW;|9NHEpI>=T*9d%H`Zj3s)7M%H*X>zWwaDvz za_IZ%r-W{_aH>g&=6XzQYi!eGYvFga*mjs#&&SMNhAGnQrX#E8vdWU;3Ud{{N^CH^I;t|Uo-y%;(ao+>7rk4Y*{tUS%a%Zpl z-SP*i>aignIqItAZb3({L_`FJPOb7jA^J4v^V6eur@M2SPK`P8cgp0Lb5EaKpB>5* zaig$9maRp?OXtRodbF$B_Yc4X8r!F477yBW$e47Sw577QOaQ?9Qz6yCzds zW-C^%Dh)arB@%q0>gwE}<7P`c*9N7ftIkYHJh^o0wM?UDAt{=>*PJlzbl2)%8D<^S zP!qT?H@ZV{nj3$&>51puV%HKU$Vl$1p5Qbu@?zv(KX+a!_inu^sT}pnl$ZNeZdPO- zuuQU=e1l{C7ir&6hjQVdrJHVr9G5WjRB*_b&ZvWNG6{C8^oBdMaO#DiP!g~i|6 z(8}xY@o}S?Tg@Jk(gaOQ(ZHjQ-QvY&Z@iEQzUmkX{-;^BJBwCk19 zl#)xlmER;gKAw^`?cDMF_loP&%4%mXov}(~n-AyfrP87t>i$Ch&rg2Y@HxxvsoYUn z_eyP{+_#qGD3WRO+)!+W!?Pv&qOU_6x-b1@bGP_#^FZN0=8{Q| zb`&e@j!ER+zpK0AUAgEEokaD*$jSdp+$1hYvKcP-J~L0DMUTs~BX)Y?^w&w@>nm!P zWQxs{y6W>}{lptDT=#I#TNa^YV^zmhX>BR7qT~4SwuM=`cLJ{l&q-I+m1L~;^mKZ# z>;m)C4-4wzxtNNo99Vt7JV-u%>dE72i_H6`rrX&%H1SUPv|3+NamwS@j`K7)Tjsxc zC{S&1NVF$l%5#pT7THOCg>IKDl7TYC=nZ^pyjK&u$&$;wkTrx#Ps0yG!CU z+j9FyYmTSQdF(A+qPB}c=mY!Xpu4ZGOkHPaRiOBB({)MBjAN!vPh2B+JGmDbuk_uk z;;wNsS;*>Ix`!2?9KFyizx{$H3CvO#9N0`(F!Ck|Fe~4f%2EE8LGa5jM&1Mm7Nr>tEIbJe0y0VB zyG|Suo3KD`lgcIcqlviu_9wF?{s*iG`14U-^|S)56j&&0ly##V+?kbNjwh zR;_@88&nk-3&k549Qdm)h&wbe-8jI1+2Xk6zKgwia}G=9JYbk9y`f=M_}yFg;~H3k z1X45KZ$B+8Q9MtZp-H#KQDn{w7Ktls^ZW{ugy$|`-F@%_i@=xZOyBHU^p0$7*EKm9 z`R?GQs(0TFR5RAqU;Fh`)HM45yXpjmxG#^Ggb!u%2QD#@Zag8nX+x#J+!I_@s)o`< zle5CvOIK{LdledKapYm#B%i=T>AV#e!lZX3u*yDYU_X_e%FHmAiG{)LVyj<`9(yg5 z$OEJ4vJH&d7KdN@_W!e7;>a%Q!Eh&q&1#4EzX>(}mN~FQ-uqN{oQILa;sAqqg_C&N z#&28em3U@9U{c@lq0_Hok?7lbuHsK(8LEYy#CKmPulne+SjnQ#RfbW$pfNGBb)rOR zkh+h8o|7_*^@rFQftk8rgZNDZWh4X_l(Wu0&X#|GZO(F53C1P`4c4#&Z04d>x@FQc z7m45B$aZ3*xWhHCNp6XGs?4{RG6f%Fw7FiKdMveaTKcuJ7S|iOEDj}iuIDDQmsBpR ziFGSEAzsF-z-Y?A`HZ3V;&tAe4|vZ_VAp=YkSJCb=23cIEJEW!={2@Io(l|o4tWU& zI1@IMWtnpa9AJ(4z*;qF1*}mGE_2wM_%ABy{1D%CLwx0R znPWy_TedgfVvRd&7WX(cHA<~D;X6YhM=P%agR4bp#PpWX={4curHR`aTr_%}-Fx{a zFy$-odR=ezb?1G4f%lvPhk*k_ibq;L6y+Bor6?hg#&3C!8k%St%X3)DDDzOlwX zD4QQp|HGWycSE{D0DF6d{TmTipGAf#i4|2B<-5{@N|{U?6&jy@lF<6VByl20V7er~ zis0l%K^2asr5Td33s{v`b`=YBTP7C?F7wY64Yw8u&lUa0oG;2O+fZ$Fz5D6a;#WU% zoz!~{ySIdFZ^_V@;(DSb%FAW zdw*NsDe*Fq4-5()Sm!<{n|*+D^9JrW3Vgc@I2hUDYi5)se&C#Ez}>{al#<@@NPvs! zo0yeJ#r=oEzAUnx*X0X0>E9At?VyjjuroJli5an=dR*&j_#Z%O=M#6GQ1W9miL zYKB>{7g&G%U=3Wr8d|{eV+p6@0#3675@rjUO%j+59cJ4|FyC4lE59JtbY}J8Z?hxB zT8{iEeJDQdu2^gE^x7o#DK5+Bh&1HhFsq5p?29Su3x3X^b%FijzYRGTxAWeK;Jy2U zga1Kp;&Fz{mW%=nd2z>6TfVY1Ey&BXp*u^O4V-<(zU{O^a``##~@c z5MWE#&>nvwj_+%V*@Z>AuflG*hVSs4T{3m{c5&wI(`TD&GCF85DlO=X*_P_nQFH2N z%gt{+XHL#JpHUi~zQo<5_2kJKp8~F%7fP;P;CpbB_jUvCoeL$m19+c1@Mb??NK)^; zoYwYW+PsIy=B2(&i`l@rXao0`Ox`F3G4~0gZVoK53LKj^^v{2gr?7!BilO1(&IPPi zPeNV?ibfg9mtGXxE#lC;aYo6aNP&})Gc_u8o=Uz=67bus$nbY+b9sO;+LB4YPNwjFp*C{P1Y3trg3I(gLj2)43xS1a_@5yfM-6P@+s@p@e0# zf~|1kqfH8YNr}AWk}{u6<_H#YFgniP$lfO$pM9Nc>jmz;3_MEFoAx?zmxPzgIn3Uc z{*PJsE7Q(QX8i}7^k&c2{*ZF->+JnMS2u;`X7B1b*;yNNa*6M(nj393-drWmcTBnF zna1~l!S({r^8miP4qI;@;0t-Z^yaUnr?xSj)GP~5Zw;%cWes2qoKQA@L)ktC{x=)g z%@SC5UNa0$V2OUfx@E%pLUuOk53CFpyi8^y*A7QS8p-xpSw>GZHrvSNWgwAhz}lQx z(W9=%#jd#eFw>NoDwLQ9YG&)TX=_9T*77MZ zW7*HA;GP$TxkPL)$!inqBp8EjH|&83=iNa-|J;(<5?gWU$JIrSM1n?=WN zojqH|VfT(*v3s;qo2I5VZB6CfkotJ`92cEl5r^E1JL4~S_WF3YI^Epkp4s+Ljd{Wc zp3qZk?|$G7y~TUAVe8Fk-t!OG#1GWyFNjZ>#$c{b`+7(ZBWvknN z-oO>Eck~LAX2C+)%9-AeGi9n+=Q=K0l)Y=u1Fm%wPHE_HYuIpWzu9E0&7ysvKu_oJ ztV6xg1%g?Z+Vp-~avwgx37>?gz5WK)3@L^3FSMNp6-s{`u`0SYC_;ZcN z?Aj+Umsa*yM#%$=_6OK(3gQYn7&SjIRiv%aRA3bUz*uwP_*ZxC z-3$vBi&%tyl0H)`WkDl<#n_(kU-L$yJoTYmGOHLW15c*27nq#v0uY z(>@R;Ghwk_z-;RU)rJR{|9xSQ5jb;Q{79JCkt04wqIXWs^_X&U#hkN0w|bspxZ&0l z_Kbn`1A}Y;YjXnkl?A+^yN|+kWXMt%;h|r!0B;-+2jL*b^&|C1-78?dt#0= zx+!qX5ny{|Rv!34+oeIIoyB7D6c*nDtos;F7ye=KHQ;7iBh=vS-4reIw@v%m#7UOU zCk3(;n3uQ;<>+x9v05-yPk!z}u7_-b4}D4;7Hx4jr0T=na3gJ%R{i3DLwi4Puf1@{ zGIq0J^ybY!W6T~f?%2*`dZ5Yu0As#3=d=%OLjM}h#H`J|u6e}I`^Y`FDfilP&qz$U zx@L~!$#{nsSNR`Y6>nfPb6}sAaP&sOwNu|XWFK5}XW+atf%i%P*TeuWtpMibA9$uK z?0d2!-Yvbo$(K7zoV8t7EU3Y*Y^lf=7nXzv+-nr-+4r(;Uci0XL8P&prRAD)>Bgcp zo78GH?wAmBaqVe(&voZG^0Xh(iSRx(wP2a26ZEAQAFq&Kt`=`~wC=tNG`hY>a;fm}B zw$M4ew-)eTOyDwXU=$BvZZ_aO?Z$gGVXg55#)$&FXCG{RP}ZXGpzhT(uB`2>?_GBE zoU*8MXPNWi#=EKuE&^P;9F`w+^$xih(Y`8q$)^(&Y957L5G^eVo3ZAkz#%4`2PY*? zDXyKH*=}^HNq}o}?xy_*`1TnbZ(q=*wu0671M8N6T}sy63JQ}Cy`7ckS}g3!T6BTu zTm!oS1G8ZObFBmK`3vmw4(#UxuJup2clm9qFYgg2ub#)(`d}j6t@6rJ;cHQUP~J1EcH(#`k{zwmvQ3dtk|1 zJAu*Lf%j|xGgAOVbR>hE!=eiAjsyW#{yLA(i7a8MEWRIDHhDLfnQr-r@rn z0yl{s6n&KQSkz|o4ogETuI$dJBNBEC3Y<1ve4nG3aAc=g_D1eTR;Ajvr3aR8b=aix zoBJID|2qY~T>@;cJ~ukQV^!?g6>wmc|FKnuJgb5Om~8`C8y@s74PchN@UFIj_XY!} zd;@FpJD$=22Air>eghUqo4CCac@L*C89k8TSiH;?523<+j0lqihmE@ zb3R~T+OWO&8t422Tx%3K;um=MJH*^G4A}dLb@A-=ACJdX9WQ^K;t_n1<2K^V!?D8m!2u^> zgV`3f|B_t2r%czF)#2gTw1iz$({!6e=b@9~>;30i)ZS!xG+|wS@dTN`xmH^zsT^9W z;i#nADfBFONEint9cJjMOIjKl-R34icq*9p7pDs|$ z*3F+bBauIA)yAV8l9Lu3=3uv4k;rMWs-U^g@t1~yfJa_qN*5DXMMJaHFO8>?vzIMA zHjyWx;FwJMv>yxj(^o8HW8Qi$tAUAaMb3g|(@9IbH|FiLc{4MJZDw+p#2+JuW|31C zsYiPueCjaOsq4jp=;Y8BE1JJry=%I$@S03s#X?rG z3$q{de!s*lW@mAyqWOmDlMkgzT`UV-7sgd29$y$cZ->)lomB^zxV1H2ExOr~(fFuY z!fjI{o5Zw&G^uE&$VYKILgJzG_Pm~{ z!}dUd%`#x+G4G=}oe!C!`MgyHLS>5Cn%P&yENIkr&U?TprEv0@!O>GcjUMT5Z|qUG zTJpo8O;w@u*@mKodu-2+ule^dZC3h%z$1)IG7$`ntRgoQxD0kET$voj^HhmTxo5(m zwvI^^kGKQVUaRov?O;f3;O9BmI@{Ln?-q8=DhDS{O^>P4mH|^|C|Rjqt2k%%dfjVA z4*8}R?{-Va6g2L!eB|(cZ`|4H#n-=2Xzh<^e0N&O!60p>uNYe` zHy!zk9yE#VQDEKEBvUgbVrP=+N84`;J!E$kOkmx(NG$x$QJod43%WQmx;P45b?qKb zEV}8y*7xeL@w~=XRh#Czs~Sh;|83xST>4nDXvRV16&HJhN)k9%Se)^m^-(XxLrF^c z)FIXfAC_c(Impqlfz!0YfPsghNvi*$1UpN@w9N($Ow$V+SSl1cf0aI#)>|BJE>e85 zXvSfgtUvPS_iZt@ygEs0#S2GHp$*f%1TJ9Y)M#Lw^N)eeuVtZG*d12)9UodXTpn`A zOyK3z4K)=AS->ED!&z^gBS+xQv$IW>B;E;Fsueypm~$i7oCj3_t7T>U!gVT~WzRV8 zJaO>2?R!Cvnri_%zb>RJ#m-pNRP~T6Or%6?>jfq8)f-sbn5J|w9~I$`5i;8@h``Z+Pw5$?XIvm;f(=*!sObcNSIrCIdYr~RBeFhE6 z7Kc@hD;n6W7)})JVBzO0I2AReN^JWc$0he)xG^lY+8=VQRd4#ANtU}N^5%TWk$(E) zSor=|Gty)pGIBLET)bk=81I(ApzyCzDpVoVdX6q`S;gI3LTKi*?R)JJ+kwX%L*bGLMU5>qrwWI_tb375`QfTA{>rP%ksJz zwGx{3Y~E%T-*w3~+t}&Ydaxz*_W?Gp4eAo7BcJ!QJ3_2V?B4?{xpNZMR;Gwlb?EWWh`%W%1olw2z zfp%`_znvv-Z!$k`kd6tv@m0M2Qj^%ahF0C0g*Vw*m9^(Nuo$0G(9r8({7}o!EP17Y zuiB?oFu|Z%`N%`xB{LYsj)nKC)Ep2L)NY;iYs2w*$qanYBrHEpb)0T|U;*cX1>W*) zcI}xrmR!9l(Za6h+`!48P#fZ9&GSlu)BgYa)a1Ku7K*z!9cC*D=s9*_>pW#0&I&I6 zhURpG_Kc2}H7h3VT)cbT!`*uyF>&46&Bw&Zd_z#@V}mwFYo-QEp>vapgUzFara;TS z;^3+K9vZ%?=#yP#b5K%7RzgY1Xi?RtJvEojFRnI*hY3+DZt6g+A2)W(@X4UCKeJ!~Hu%yzW?FYNSU zU}E~QgpGlL@3nzaMU%z_CW8%o^%IuqZeZklF!6IHgICJ}?TVIki*uno#fJqI`P-4$WK=57SCT)#Bldt9=Tn7?|(2x zWvU;M?3XujR~0&R@pO}^#+gfownp7eF##M8f;bWv9M|F6*!j5i{;z3wSI=ztb4av+ z@mx!z=mbVdkEY-kZ0Qr2MHTG+X$c5LC#(ya(QV(sxcm^e+yRDZHtlH>8rcpotSo7h zD`?>OF|VVNXV-=n?hg%VJ38NWo=i7r6~8^1v!GG7qLHJh;nfT(iOf#B1541W;Y=5v}qg+Ch7!TjW6|<$2`d+O#Sp3TE;K|bpLP|nl~mg1u$AHVB+Fd<%wwaDqyjEz{q!D^5T`}wi(P~+R>o=p*ea6 zTdc&wScjz{yY#X;nz=mAo1A2HRcJp_F*8bu-F3xz0fnu$k_}uEOk5lco90YnYhd7z zn91hkt!Z$<^97@F0Mmm7I!`8eKThD|Z0+02$i%(Z@%@9-(hRyPzxP}&GOAH>`laEN zd$37ML$>Wwi~0kW)(NcXKU*{RI;B@m3#?$0W@r@O(8xD|aj}5MLX8s&UzxKsf);cb zJ00j&Il<6t!fvypLF7eq=8K)|2N+Z*w74fUZvHo+!F&Z1TS?b{B|eo04SW+Cw*8o~ zMf>!o9B$46#&S#c^b-;)9gKW8CcpAo#c3cl`6g3>09(a}R<_xb^%u4=M6vKMU`XU( zOEhRu-od=;&eoq3Cl_0^vI{iEE@I%bX5q_L;7w)-PGwlRc)zA0gXWIb^c}4tEmzrP zMcI2#Un!WK-LhxpUOp`W=4UHx#-Y5{(MF7Xe6^0ADf6^x<`zWfeM!5M9q0gO@_H>4_B99J;z zKhWgpz{Gigf!m@FBkOK_=LvA0@oJ`#7MnpP@cggUC?L|e1|Wfk;kJ^qJx1agX`4QhV={l_$wNci@Vtk z7?v+^VVt|4eV4)8sk^>JGBD0y;BjEOG)1^nY3p#No`^tsi=Cy`*v>N<4Aho0U`xaL34o2n`=jNC;sPACS{b#W>E2BMA z;K0Kztl|s}bC%vqXlPCRamB(dgDpY3_1NRh9@;7jE&0jJ3O5*41)gZWV3gi)>}%)+ z$*8F(n?+;iPCa;&@o=HKN-}doVE@#-{%;<~OWvppZ*4L?(B!wS$&}+EJ7=f*gQi=L zTG{?(MrE*SoCvG`a#8$+&*Bfa<^MW8_%CJQ=v>Hr! zZu_u-XItZGiwJ&$23-r5T#ufBl=jRUt;hGZ#k^?bOlVN#2xabQ4WGcux47Z$+{gd? zzrKkMdCkVb$ZF8!{G&bgL=(%uQ@TGI_GC`E9}%y~z;wH959ig;r2?6UW=@smU{ZW> zTHIifVe!ioi7!tZPS7$AWHvlw%CKXm+r`<-nxYiN1sEE2A7nayWbunoQz}TE*pOB9 zGf;kM>aufB&$4*%RWN?jyCL$Tfydz7{v8cm6B-nEyic9Su)LuwGwfI(Yr&QS$=70= zf-bE3Z@eJC!C=)Vw&@*g(G@LL983ZktPvb-VGIY;^SXa_H!!VeP~u>Y5@27~H6z<& znYNRLhQjGz!P~ehSkJvP;99_N`kZU*mcn1v3LG2^0u0Sz7nI)CH7T6GCN)9Yx8wAV zId0D){_&Pg7R@`tcTiELY**ul6H{Xa&rI9a^!S^)nSp?&u|V$2Czmp(GOx>9v|>dU z7w0+^CS`{Fb!#46e;BrqgHu3)YoSDE0ju2u;VwRbE)fq#(H9Ky*BJOOF#M8hQhdO` z`67msqvwFfv6Md&UMAUFjtPZqPBXu;cvpbxF@>Wm7Rj%D&6dA|JAFcnfka!xkJnrY zt{hr3*?w5_GcZmSZHqn8mT&MeuAx2Q#McCdwy+AOz_kohYYVx>+rmDmrdBtF1w@*& zsLE^6O;10a zIQi+IjeDcRMZbT2?+o}=b{0x*kx?`dN*OAIsU%okg z@?uQTyTSK?VgHJG+!kLHJsO2H7&#euil!xPUh>Q{T%JLl;qRj$-}&<`Bo$UHTBZN7 zEoaBNDHqvdC$uJXw1zP-G9{F>{9U5{kHH|~TkM9mtQ)O{`^sjwvqo|-$FFKgsQ9?t zKRtF$(*9*t5go0GKT2EQ89ZriUD@6m{^9k#ySBa$8kf&Kc6(p0Rzz=c(RS&Y?d6j{ zeGuS(v@GR+sM4XDcZc45ni+O&AN$j+z*piIT0|Az^)6=Y<34*~52NIc2j@f*uRB)s z9^Mgo?+nksK+zAW3EvvdUeEf+w}T54{|etxpd^rz*DkG4ApLWH z=&KivCJ%J-biTh^A^Nb4QHFs-TtDti<>9nG(U#&HDV2{uES%aFxh7#_QlalO)9f`l zYmAO^2}zq|+^E=isI6C2ChUmE&ZO3E3H{g;3pOUDoSJB!d#gwB@p7M;+94~xJVa`CHttcnOjWFAYw&8f=4r_u!O*Ze+G>X%o`Fncjazapu@<-#3r&rkTZO1(bZk0 zlW*rr$L^}wS^79E_x85f15K>b%d}^yI@#KYR3tbs&5qvPasb#XHZ;7Q6?(9N zG2zey+l2`U9gX4ZQ)-tvJoH=>zA`&e_9Y?32G%tlcEkfS|uR$!DTKZmlKm(oY!7(WOr|5U=nok_{but zP{Yu~F1um@6PtuhK`5he%F?NfEIu0!^zh!;(a`%Z`6-831RsZj1EbcLMsCxa5;L#I zR;Y4HB;4h2yJ2vB&f;FP^FBt$*$qPqJIyDDhIN}h3J^POu(^RjDr3b?Ge%a?BWyj4 z>m(iSWcf=xn87I85*TBU(Zs-`Qz~}xp>Se`8aIcR!FA?h2ZQBDibFIO&Mmv0``o$Q zu|xeH-_xBJ4mogZiUd0GMSgUv%};2|auqpS9_X5#@bH+V_lt%Fyt=#*3SOGBKVGx% zTc#r9Z+udvIY6{(gHw=;h#;#-LJK3?)_9k|mJa!X4+jM9@_lSzF$+>q?oX_J`Y7D2 zt2MdL?3V@ud$!FZ9@ASTlh4LfK0Kl*Yg5QD*~7@^Z%dSKnNC-6a%b@;2FQWaaPHladvf zz;uzhY--r6x%_qvf+fq%PdqqOsk6Y!T`Xt8t1l~8SZ!IgMkF+GzH460+O3j#Gd8*i z7)>#G!xhPw5fs2aZ^<`L{U=`(c=YjK5`S7&8-tf+K&W$0HxpA8M7?<9&V^MHuXT^C{f?M$X~0 zX7!cHz0)^dahk5b@2?n7u0r!R(PR0?JP!&SJd$=mSXwKdv2f#-q<^zcZEV@tEaH{H zbV#`2(5Dt&cY|G<7oUH)hNWa`)G66t4+^zfxGi{=Y;ojR`TgEXv5k#KUA}~dv`YUB zZFCh~w@lDXBaz)*x$`!g3$yr#Cywl=rnWFlIgsS4;ds1>k!Oagl6p;}aNXlp?TCZi zaUB6I^By>Tnm36_>wqI$`hjLng$WHZbGvn{FHMZxvS{jw#B~fZ7EE#~A9=E_uA(~rpX+lvuHu5T!xddONG-(zp!RkLodq<6Zw{$SU%U}Pfo@Ru3O7&R0YJA{Cl9l zxpIE>4F=E9$R$cqo29O8Dckz`ivow4vU`rJh{u*DvjvBECl(x&6bfLHUg+(1<_wof{uKtcj8sOG zLl+-&JZNANS-|Y|qgi)ZqsvZ%lkq-xT;C*>_g)HRmfj*Ath;3*Z)^!8%i0O7hr1R_ zy<}Bj;acHf5Z1ix(Mq@8Y?};&I})}AZybb8cEo+t3TAQDbhkWrG4kA|#TR?yx~ykh z3R!-vc%JwYhYs9jYqvYxnHO;OL%l>m?=1kST&v&+H^=b?5V1LrXjaR#|hqZ2Fr|<1Q}T!7R2f; zN{q}pawa=USSLO60P_~lhGPW_*ouDKiC-Wgab;y+NloM>GlR*q6W%QHn(?4Pl2@QD zu|g+cQ^#Zuqi=lO!uN$;#dmQ8ny5QoOb10Z` z=UP>Er__M@q!7gm4mr-c0h`3c*YLX>Y*ye|7`k>Br~jJrmOXD=L^>|8q@Q@majL_G zQ{(|d+PiY5f77g2?>ba8ZQH!3+9pMU&%V88VA}AMNz$Q7F~G2QzfPgV3m+uek+!&Yh;7ybzs#-QS>!^dSc<=O zbX&hT&?$;5!^!#yU)Q&ljV6~A`b;>yLuUzwzg>Uo|L#vd_Cc*ok6Lo}Sq1oXbeYUr zqZ@AGDX@uaNfXa12R3;PCW#Ee9Ri!0#MTJ3YU~SOQFlocUaXL)8*#ByNrX|f`v!xG zQX(Vwl&g%^cLI%XPMx-8&S~w*%NSy$Hze%P+rc1iaDc(cfIU=W;(GBu*-0;wdZT(W z#ak+x3^&6MDXc6NB1{}<6CYk+Wi@!4Rdqr=Gut{VWUAv-q zk@ATf^WLAhE7&jKqBTKm2}51Lv?W&}x>oh3IMy@Tm=`fu+|k~Bkk{)$V~KEwkiZ)g zb#qC9O+~8P=era*X*W3XBnYh0_b{EV91^RyD}h|SxSt#IGN?1VI* z%{!P_{azpK)V!9Q(sh|h<#djTt)jVe{h{=Ck&+gI(dK3eOu|bV_MSh+#B%hNS6IXIL*|7)51@#n0SJ_g75r4moD@t8(13HK#5 zcNqQVxc7Np@e#h1mpScRnHKkR7=0(DMptg&pS;n*(SyZ#!Ku{@7hFTmxxpww0 z&4WS#j{FN;HZ(AJ#hh>B;l88e;XNhj&OeSuUIp#|oe8ZPM`zy+;<$Q9zlY7HNM7KOg2*8S7bZ;yC#@+=S~r?>Ee;!795#7zD4`%;yl~>lfcW!{@vIRIxf1;1 zrGZXIIcGCOOTAr~{h-Zmso27D zWjK0A#^Vk{Ym~A1CGP?5{{AvjXF&XHXX-=zc6G!lvfQ)QcX(wbwiACS3~6j_wANP zm}Z^Y;d;bvgCmPvxbMQ~d%BLII*z`+jQm?z1aln48xATqG#>hTYOm*siwhoX_>eSf z?-B2mjv$wV8V&R2>})nVbXZZNUC_gkSE5nPgHh1ppw6FvO>+yKME*1yEpd{Z*JS*F zNzaB^&!Rcek2y*0nN84P0p5xB`;@h&G^NaQ(zIbxv~ZI9!6>)kU|OG&u*N|Vg@f%0 z=heSBY3DF}YdM(vV2)`N{6{yOWNK;XSjgsScCK@-+bNz#$-N%( zeD1nWngkC>XR7uW++YyeaBVZIkgD79|BQ~Md}qrW+$T+J&-sL zhxA+yirFX&-eAzH;t=dOq(4DGLWEI%fs@p^O*ED#>oY0vluJ=+%e}$i}%fVL@3|0 z00y%_2iY0TCN&MLYvwHDJ;;5BL0}ET?1)*}F=uA!Ffc1Lgq5%|dN45c95`ywa5Tep z>MMcmEi$Yfb9!_dvNM{L?;JAu;=n5+AdtZ{>(+uIhxBYlp>s3Nmt~yVnyd7CM}Ik2 z_}>JV1!29FK?{P6RO@{XitJe`vVl?bNP}R8V+|9ZN7gj<8JC*hwD6p1xKT67V@uAN zunq&Od$Z3r>0gbUdy%Qbqfzk8{b(sh!HmY1sAP4O^9&IUF&{l5P1yR6OnrK1b%aT9 z!^78`s#v!~=}vpZ!R)}m(9vjVa+pWNfhECVN65LGRjwQlw71l~X6a#IYB_Lp&c9>K z4h&2R49p80cE_FJXldXuSpB$!d-fTQj{yxL0*7>Mm`hk4{jOe|%f={lM{D*kiT|PO ze%D^|Wh__XJTlMgK#j#&ufR7o3aPt9wRRb%R;^lI?8(T#=D_M_O(%u=x+@*_8BXB2 za$xNz2i_Y8_}=7wkac`AZ-H+~15Zi^yMUv5g!0_CLAnbB4ofl$9Z@x@I~XgK`zfNK z>F#R2-1m)Jb$58ZjO@A76L4!sk|yJh)uxe`S#IdEym7E}Vdh^TvqIsn5t^{IQEmgHz#fJVJb3~>hjbq}8Q*eP zmcr%z@0-JHJLx%X23r@NcW)HoopRivKw3lUh|87sHFx?|E>3ad(GpbJPqI)N*5u!&!r;yFC|*Zg3P;I4E@Fpj-gsvTsL%8W_|Y?tQu@ zDN(}lai@=mi{qTXO&62AS*N&0-aXM1GugzkPkJ#c@6z=$*E#wwa4?xLusmU~bU7@j z(ZF`6QQ5;m%KQLhK<1rE4EJ7L5MXhXUJxQ@vq||uqohW|9kF+>I}fUQI2(H$Huhjv z?{_wN;dp&hHiwA=uMN}wbAfDZjlBz51pYLzSNHetoF>E|^nWSi+&Hd^JDDCc-MKd~ za4%@!n4#po^WZ<8>&I0CAMBD1=iSXHI)|}Ji}8Np8sleNf+>zQ$ChijGzOhH>%nqR z@<5~2B}VBBjhX>!vUjE$&p9N$LdW~YS@sggb6o-28xH9dFbYaI@~SkR&S{j`9K})F zw0^1#%ap5+cYHM7*}ZJ)ohHfE%OtaZtqPS6&zyRz!^rTNPy_?N3bW~kgFLPdj@wt? z$zx?X(;;MX*yd4_iG{O?hqH;xVa+=Y96Rn!w`<_|a?rwrSh8=g=y#pKF&J+}_ zb>y90%J9)$G~giXyI9F7{}@f3!@YAF6`mZ@k!jK>aTH?V-v9QZltq&fLz;@fA!8F~ z%?pn53l8a;G;4lwm}Ps9P2}MG#L&5{h7v2XgiGd0oOjZXc+8pWvDxZ?Lbg<K=jf@2j)~D~iVqr< z7@SmWoQ?i8@Vsc?`_u65UxSjzVWk@`r>YJJJZV&0++gq6z+}P@^!c^uoh*S9N+p)d zOY9f+U(^)(6Zl^=vMej9at6Brck$%bJhwZt?mVxI=D#bfD96`B z`d6IhxiZeLbX32>WDucja5Sx_qfF|3cwfPth$-Ap7%chLnmyHN;4x)ja&U-}aM;jv zPB!9zyo;sc~RSY2ZpZAY$Td^u_4^vx-BcuT)IqobAsAuC4pnF!%j~eWANIa8E36?bl#7{ldr-!elyw$#jb2 zw%;e7KB+X3NJ|vzoRNHR&51^(f68(!Oe!0!+l;>%Gc;*FXkxW@mJ2zUwB4cZYbE0m z2E`eNOd<}-y;UD4B4Hrizl29?RK2V$*Z5gh+9Izpx@zt-PeGBFI{=EUS8X9 z*4a3C(^l4NMxs5}m^ym}&or`5W|Dn#fc?r5|1SrmG?-Nk9M`Egnp-)UvM}j6I4M{# z%bdyJJ#v8Oj6+9*gWv_HO`DxKG^=@jIq5T;Tlwvu6URaE&NU}aI!49JJ-t&&qF10n z=JD!#%k`(Z3G)dW@e4A{pUz~ebBM>o$$l2Iipv3pg5As~4#;XW8QnM}x1!0|<8Y$* zGn-$Hybqj=Lk{xrFll)>%U@~KU9d_10h9TO$(DN#3e9OUInZG5>JZW3pq5Z0cBO&6 zJcw7P;N81_*B|!WF#UCs`$5^Gh9%95I9l}FCw#3{xgOwZ+JEz{-EmvCf?C{T^o8OuAClsb{ z-m6`~pO6=AA5j;&V&W;rxco%Wf~4S4QXkvgmg6 zjnoWj&J&#E(mQ)+W`3VX)iWo?77kvH0)>=Qx28(DWqsMOQ1e8;oNdLOr-e`SgcWQX z7$!2gcqSZnZ1};@Wa05x)~W2yj*p92xkZ#zb|gMd>z#To+ai2n&{N~gi`mmU;x^6d znxyGHMI}(ll}mJ|NzRT18)v)m+~1L0vOqDJjo)&GmuuITp0(3@thW**rxix!HG?xV#Pu>bB~wF zoE|}$NmBzeB+u{pdM0}6)G)J3rWS6wiUy{MG7_Hle-*4&+y58ZvOTkT)L zf+g%GPTL$=85S-R(V4UGFr)k|$0Lm06TWu02?14WKKpH98j+c$w>(W5T* z$3e;R7SAf$WmJk68i~1iIJTH>NJ#EIe=p{+iB^o9uBq0v8BNcu5|f=cRVS?Q?UJ3b zM3F0uM3%ee?Oju^5h(3s3s}Y>`<1dhHaZ4F|cLm3R*y zG4R>g*Ex-+UHp_!A`7QY_I~5@D?apWxwtCgQQEBBw%o3{-{Tkcs7x?uoqExY%d3D{ z=8OQlxy}dP$cko}2Mp|%GKp+Q5B7<0JnXSO;-J?h?Z>0Fqs7DF5l?qOn;_3ZE_IPc z?!^+V>KTsRfh9@81SmCJvE+e_>1#N(>e=7K_MkXkeCEz;Nm2b32X>2fiu6oFN*ECBqt9_H4V_eA9A~ zy8H+&|foOK$>`Pm;2pGor5vv1PvuOD#Z+~CM0wBkdX-JeAM zMvfy&VvL*tE{jBt`mkzE=J>2ZcqFiD?`U{5qk)l8gImclM$C<+Md6>sMti3- z&PFQ?In?f46j?sO;olRN>kN!*9?GmRaAaf=VC0y<$Hbm+fJNwMbB4uok-Q(wT9X2~ z9?Gs@tDJE_VOC;i@DwMvT!tf>S`+6eFJWMoSivY0!oZQaCb9BYX9LSK1I`$i^^+

DCBr5jV)=;4>xsy@#gv&HL!48JHybVo%V%XGwU-eHjyM6y(nOJx< zTY$#414m=$8uF%xIEz=keQh$&us1U#S!#vFVaZ7at$J@1#h2`GQp_vtwmNdaf1jn3 zLV#e0oyKC$bcdr#HJa_AM;3B4Mz{)V1u$~gY-AJ)XcT2rWL1%PBvPw!a2wx}HEc2~ z!d!S{9NDEVu<2}MG>~vH`S)Z&OBRC?$Hfj>-Bj~VcERb&x|tKPnky_pft(lbAtou9za zEU}}Jf5L@B?0y9;!gC&M@HufHHBhmgTcKN#RpN+=QesEuk%j!~1rBRD3|Kl3{A{l~ zefy;UKHcsWG7R5m`tv9~?3uCqAj_44BW^R@|1pRLESB`~II->>a|5e@_ad$crsIzp z^gKM2_?LGayS#W|lZ2%byIMozabpG!4~s)W1v^?)!*V&L9~_h{mssScUXUF-;}Dzg z>J$MVb0waTk1M9^CHhz(SkbjHnr$h)AG)TLJ^g2gB5?sE6 zSy9eW=+0`^6YGR(*!gE1V2L^)sF&Ds&4DlKh@gxk+b;*HHLM3*9<0dY6`!+^?Z+Xu zGKN(y8&>MAT-ETPddHI+63*|>EH^r%!s^vf=XUt;vWBT?%eX}JKYn{YP3HZyDhAGJ z3Y=9AW)Bt!U0cAmt%svy0p~Uyt~mx=+Y(Mcc_1`tA^((<0^bt&e}>8qYh zo58@Tkk5HUjc>sO?p+K#2OI<}7S#4lLm8V-|gOfMd%7&P5O8zdLZRImz|O zfh){`HN}A~%7HgD;k(ZR7Kx%@s{_303VbUL3RpQPX(()$QDFYJKx7dUH%DV^+kr3z zCnh&0hB^*Mu@)w|MM_$Wo@hN{PJY~a2dN#e*<0rFKXb9Y7~o5Y|CmCt6C^lu~5wIAZO@7$r*<-Z}4(|Yp}le zfcM4%z9xllCjwY2-m^t5Fskb3$!1`i~v;$X56O)(6Pf{Q*01>_2JtdyLjDCC!*F5q!$o4^p$;g0t<^iFQ z2fS*DLQ53vE-5`J<8}!7q+p?N&g~GR-=;#IMO)hz*tM}UmvAt19bgP|IJY2}Rb>H# zO#{2qzrf#44cBCxSZ1i_A7KzL_h)_gKY=;upS(2dRcuBnr>kBCz71$h`!vI}5lTEhs;b z#46{`8gszkAb~Zlfm!0J&$E@x76-WU6i)>(iny%j4_nP5b->4`p)lpZj~OW(HVt`~ zn({0f@?I=skZNbJTEK3Tz+utAWYxgCqJjB_f^fzG&{?4^%E`uz{5cCu%QV>79QBqc z2y-~HzG4vlH?L9Z7emh}Cl-fQiZ)a86&RQ$lo-FA6g+fE%&}4~?!nav28A}}4tP~oo8W?O2BuH*$?l~aFbx?qdkvn9e6w4N$ z)G#K42TV1Ond}a*wJl&?(ZFldzU zbZ?>LmhaX1i9%ry{)t^+7{wSEKO7ES z$;^MJA$`UI_A?839vqO6NpyHvqF!`B@SFmxj{<8#1GCNn799pQrVyvM>)H7hp6{9glm;T?bPcj ztlJoL4hTM3z-;$`!z1xeQvrX90$4j68L{5aOFLed3C@=@1fwWgCa(b z(msmZB`Oo|FXYQ}l-{6pQtsbF*(2Abi+E4wJdiBP6*`kBWF>Zeo}zfpL&*sXxV$W- z*be?{YjHf05kBD?gBSyo*aHV0MWL|MOcJ5<3>L5|ISNKJgnOnpop0du;T3q*zrm83B6e~deME3LDBOK)`T z1or+0_MQU=dK@@6G#G7X;7VI4mUU2eUWJ&;K~9TCu{1`mMGAa(-t$g)z`N=ITUG+A z)dAL|X@?6I@;!6dl%ya$1Bs9)CA^4Ah zzs^Chp;G;Z5!0Thj++=yUSgEJwNSETfsmHtnKO$Rms$4N7U(H+&KB-DXmHY??!e?N zU4~QMuJ$;|8lS*&z|cE;O4=hWvtvyn+=Ny&FIcjsB^RNbHnP03H4_KAHel@zr9LJCroZv0{ zfa6!gW1fXVa~MTqqQ&nmls%#-GtW`j=789ohyVQMFXWR`)YjU?XY^1&&QZ7^k#A2q z-+>=oO$WG^E$~0_+n?(wTh{fRiDj&8e-=J(5cswz1U`LTqX* zN(yWojUsuBdRD9%RSE1;yPc~QHhsLNf8@bC0pow}vs%C`w266!^v=a;O1J$J+DEyD$(}NAK{3FVs?|o7C6XzGm3v>V6kfOPMvn( z^~;5?SH{a*u}!dkGBNrbR|4mghPxW!qCXl|zAq8_wR@h!0p`4Y-~J`Y9w@)ZJmX1K zE(6O9m20N6Hks`5{Kvwnar$!XfqmI~7qXYJPMdpJ?IG{C1tyA&Y#!+z_ly}eWR|EL zU{?tcJkzkwvS#^ON0}oBrPeU;YdNx8?GP!*dMU>!UZN;-#8G_BL)n5xHVJ{_$Ja2w zGZCMWq9?gauU)iS>9zq^@<;PyCyy>jULeDs*U+ZB_*AaRIxdZvfoc<4BQHik8Ks=SoEHRKDaDC$x}+|g7v+G)G(gn zM@)M%m?t%eC@kb!@_@_aA=f&E^&H^nc)+~v0Ee9e_cn&#y$6^lIdIq5{X6fd zz+Di>mE`auc>$A6{`2hfY<3A8Jq-O{ZJ1f66?z?zT6sY5o5P$rjei)F7+F|&B^)vs z9x6C-sLEJ2FIf|`)P70H%TuKqA$l>Pk3wcCKJYO1EdM3p_4UlPqcJz`Sw3#mh-{fD zpti=sMDzL-DVwY-H!dc#^UFC^elfVke3DgD99i2H=Wq(_2vBflSDyG!rt@QC zGw(lX?WBs1jqEIfx-JeGnV%MRPvdplmZPwVm8FCKN^XM$)1kKZH&^ZdZ2PF#Hs!5H zPKm=MMrJl{)rb=Xf-UD3+pe6m^iko51qaxKh5R}g8aew+m_1ZaIc(yPZXx{ul zsa1O7iv&iwvrB|dD%C&O)N6YAMG~+1^N1j4xsx3X3M_mTA5Q4*Jo7Nv-}=UZ%K@fx zU!D0K3=W2b1RAm~39<88I+Z`>Zo`vsoed34(OFwd!>4%MNWC6ca8~uYVA00|M*`eb zzPiaTclhKeGS?!^J8A0|CvV-w1)GjYw%sY};5FQGK$6$(YC^L}$N@uE-rYMlF!5?H z_;yG@#v*{>A0xj=LL-}-g%Jn8<{E<*MmZkC7IwLqg{&ODE*}{M10^oD2rgX4)g-t? zsL-`6JYadNpjpbsqwFOaA8*RI#cXUB41UviRAhq2#6}j?DG5#7`Zp4es?@G{)T8U8 zF(JT_)6l??nd59ryYV@VU@w{4l8L8vUap$mE$F4QsF|s-N19V*N66tm)9X2xd9BVr zSa?YG>=K28ENs;dCvj!Hxdn$@FU6b8NR+9+ zzRb1j@yRzwu0u9DFRaPy$ zZC6&!5v(rx*eKR?JDKg1$sCLI-RJ)_8~SX?uKb$w)t;9=3Q8wN{x zv=S5=IaKc`H1l{(ahh+uBc@w`nMv_LgX2B7fQ36QY9t3~?2cK&y<0v+kW-jh=b_Js z#s!!B%y-oM;IUZop`lUs>;?u$79kS@7DdaqiKhPck+xF;AHCl3I9S^F>c$B!S*0qI z^mICzxC++Zxz<;pS{lx8rtK;&6!mK9X5}jhY&NS83cXU`OqV&x5zlc{%jF`Ak^p0< zuZ5h-xee@AnMtfOA1d%>HT>Hu7tqLNsG-Pi^$4<+d4c>N04DSD$F&U6-)>Zbdrly^m8wS0*3zooiq# z)6mGHBEZO_^O9B1gGuXG19$oYCr-|Y4ksyx2`w5ASh@?2Ns2Xgg-mD?*}b^cwQM3c z|C)nBJ{t}QDqL*mIl?IZ;scx6j*lKCQ<&JK44jRl0)z#14lqj|IVxAPsaeVZ#KF6Xx}37kAL-NFg>}+l%2q$ z#dAo~Y=*Z&0b`f)0R}c>g#?8u8$13Pe^C^jJcEfRh@n~ZhYOdDX8(c|gRK%J39NDz zj{NNgj1^X|7)5{Z%CZ%<@Rc~Q@=jo2kGs&w@8QUG#jwe$HrSQF#9?9Bl?Hab39kNy z49vDQ3~T|Ho%t#rEL!2%%T@M)fwSWatNw(8QU!*(S9Ki~coaBf)DJM^J327xbhv1y z9bie-U=~@jfB`FF;QUR ztzeLiabf@PQ-#BobH1j3q+Qr5(OQRDT@o86i7GyE;P70P`X{Vx|DJ2C>C04}O)C*Q zBh{MERl37fD50RmZowhRhz$pMewZ|C{S!GP*>-|SGiW1=#1RKJg(XbVH!d*GzmTwH znWnC_<%A|#y91`I6$hAiMYOy7ENsz^c$>v{RI>KfHfEU<4(^qwZ+Cy%w)mgU4QEw` zUA=8z7V%j;WDaST6t_rVQu1x&E(%~yU$RazUBH7WMLd|r$3cNhCGv7=>%;yVGU2O9hl zTbPSES_JJDoaf&2ZN5xIqn%~PtuxLK8=8ipK>IaBI&h&f&0n< z9;1R5vqK;6|F2T!H!x_FoHVg{X2V2=QUzB%j)!{#{-tQxNyzuj*%W4&aV9zc?~Je) zUf0(*{9xI|;?$|zmK5K2UPJfZ%($#2%{l3_**$g~QvBAyVzDEUvv^Y<@3w$;y_|#m zE^RJaYzvurrnt2PJ@DnNayC78gJJIPkh20R1}xWGzWm!NJK?axJ|1RKivx^Zmon0W znkDtRd`)woX}Dj#6L`T_WT%cp9@mx6cQ*e0eP91ZBTq#_mxGH_Ou>UJpZgm3S``?W zv_3Ejv+Q**i*UK@WO_(YB#_Nb(}D3%BC~POTHeBpWTEVUgZv4-jFI9C_$Kc-s+=U^ zwQ1Lnte8VHeohHs(2yC#gBe=H9RcI*D;0ppOVH;?rQ;D_7Y0H zwFM16$z1~Zk}8T?L0gzi1ULgalFRM4w&`s!XttlD$i!KoTYu>TPso3BnO>WA>7ofe zMj8$*0uvY*pA~dOi#&QFc&{|_c=*2M8;UevN(<1NY_xjtPM*hXP$D2(U+dU|-?Dk&w(B!eD7D zkUBNoR7;FmcGjPVqf<_62zt+qlFqS`n4X)!R%Jdl-t5WuY(sD8y+Xscgz ziaXu$l^4Jj^?_YuBK!4VVcr4;qpJ+}pUVGZV8|3)z}T3; z@#X-pLJ{vS1KuBB*mDIKkA({hFW_|l&ZD=8H~1@GtpfAQ<26D73?It5CmJ*gig_tM zPO@;a^V%R{Z;;~{lvD7O*)lw}Eqx0@vmPto{lz#TQu;JvgijoPrpl%pORtlVJX(#;Rw*Y&^qkMPVwd z0;6;*qv;2Z{sgJ`E!D-BrL_c@XQg&0q%j*@U{n>6O9*p%nIX4yq5u08eh(f>@0i%f zwKZ~OVC;2O=8yuehyt4(AK0T<8Ba{FVGR`6cv<(VvO>)Q?i~VLuY|bnPy5GGae*x` zT=>o>rkMqt|0i(%FW^4k!1uC&-Lil)Re?#P$Tm!%e)0u_uj(S@=AFMLdj&NbN|~u_ zQ1ND#=u!?#yujvTxmk6Uh2cHnMpFfi0JcaQ0reFDzP=w==Uh%lV5e zwFTTQ8yuIKD(`;Sw0B{^?nag#_m&k8qa+?kDYY{42QW)s=`p%c`0t9QTLz0U1GAez zK*WLRZV4;~4NO#Y2a&Ez;v>J{i-sP@CSzQ4;(u`aP+Gx+bfEsx^~X!=yXV!>lh@uE<|PSjn2eI zX4Vr-HKIOTCz7JgMRtjKS$%MnR}q<`+IZE(!Q(-zZvadD2F_g{xOY8}sB__-9l+vJ zz>+tCt6qUwKS5)UBJ=AjGxj7(nlLR?Um_(Q%D|Kmn6;S6%z>k|VVd-GsgMxn;07%> z2NrLE8Cx%zpT5Xa{eaQzfUUd%XXppUY6HtU19syDtaTq4O&eI32D1J-Qo*UtoO(?% zXmPsD1CBfe_N5Pa`WhHrrVI33v$`ELVZt(of)6H>7-knUaIKucYTUqHv7tRil*z1+ z^`FUwsWT^VpI^ZH@&SiI0%z`qSqWDdLJT;!IdJvQD0Nbd5dF+-!=!TEQ#7Pe^#60_ zB2h8(0;aN7rb8|B!h>^Po)orTp#JV8^M{whGj~awHn4;SaQQM!UHpK1-vhpVp!4Xs zwtSFqHDI<8yFEQY~If2XP0()%&qe-BxHUqn@0qRQ`XSwTr*z@+bNVke9WAFRK(jb$EInr++xv;HB7N` z4KaJX7{o(cWIwRYh?ep=$-Jz6t-%LIV+UDDhP0CIwUP~MZ#7EGDfkK0Oq&(REVDpd z{X3id7PkBbMzO`9Wu7ZtnX3+nRDIy6WYEkm;F|oTCghc_@$$Bx4|TIQa8x{C%NJnJ zPSC5Jz}~ljL){>F$!USg1sn|x>@5#CDn79PkK!!kV$_mkbWGSi^OxhzW5QLJvc(tA z%`{!%a41pPdsAXy!@os8R`Ac*c>VXLca~9t$L6iKh%(xpZ{)Djb^}X61WSSg_ZkE4 zH40pDA6OrOj)`TNWx#4Sm6hj~^v>|KRJLuw#e4s>rLS~k7It9ZW2j!Wk-d(ABdmeF zvVnc3KvZ?Nw7~;m$qxtbo?`rB$*LW|?y-R>@-Sni1G~%twn_n(`~@7E2bdcSr1}I{ zj2PG}4=|dwGkUc#Uis?EJAvukF%7K%&a({6H=1;p{w!h&oWQYx;jbcN>IaTK29Dfh zy#|LPeF^MB2Lx0WFeoZ8Szcf(f57&B4@=$#HlGb_X#q@EKOYecU=D5Pb~k7YQ#O)Z zz0s0sh0P+vLuPwql5_68{3m*1^&YcFn@oigZ30&A^j>Lby;q*)c&ie#Q-IZm6U?%O z9OY|P%|F1Jcz|`!56;;E+{>F-yB{pQaZ0kriP^tGV}|1?n+Ys72iC5b$W*X^Y3dre z8fo^%0J$Io_R0iS>jz983z)x7soCAzr4y>i8*n}T2 zc?B@-Hy75sz{LB2NkV~PMe0KC>kN6evHT6Gea{$}CNO9nSYi}l%V|)2lgYjRwtVU> z1||VUO9p031!mm`mbwd=)ebPJF(@p~zUjY)CG7#5&jLoS1I!f<&Pp7pGZtjjzMynI zfpt+Rv&8G9y=fu`PmA~%h}s|W%Kg`BclTvtoZ&H@fRz_ktG|&R&2PGD^fVDVSr+IoO%e!|^(4Qg9Hutp28w=LkRHgGjukkk^CI^$%dPeFbt zo0WD$s@PM1g8=qv1uU!&7@8(&yxrm~CJ?1|fsrYpmzROzuF-WF1_l`eMv)0;Ssv(S zaxkRLV0gyEz|fS+o>0SdA(U?dgOvg!uOCC-wg=qPLoPWeK06bn-!9N-yVUq0gX#jt zAEJy4(@y$xDHtEGd%c{YVFJ(830#E?%#8}Xf{NL1)ERXHO3y2>&tYw}7HG7&kZu2g zIY#LA^TlBtx}v|+CnwBdPPoXlwS>vz=DdG%M7`c>maCQTU0<;>E^hBJU5$6YFD^Z3 zS;WAl;BaZr0q!*qxYjsu?|s1C|B$U=0@twhd9v4vec+8Tkbm<3gCl0!~J3d%&AebI*x!${hwdyH}ir45o!`E8j8j z&THdcz@YhnBS(Q@Y2O6)1=qD62z)%oB;+T+wo(4f_6bYYF<3p|5DN4VQ&8xe!2IJ( z%_>gDt_3{l0bEN9SW1_0_$}n4*ST5*tr`K?;d$8E@V^+Y#&&4}kAXq(rM$;o0=()1?56}ce581-1<+N$z=fy?SpV z-!rM$XGW_prmuUJJCFGyhg_KwYp4N7-A>k61+Egm=V32d{r+8Gx$%$Zjtc89l|F_4 z{ufjiUT`hkx;%x?fl)#s&G-S^iU6(+7o%1`)CdaTyt|HLV!=$N2X#yHYWOxpMO=Hx zz2Waly)TQ`z1n<_fp;QL>{DG~f#M~4j5+(+j=#}O-@$OgT;ONoBjcki=Mvc51$YZ4 zFm^fco@5ZP60zuDaCAJ_)IDWx$6_77Ij%)ZA}0wrHg#~Q@%cKXE^X_RGAVoUAn;Oq zpPYRa1LG%Wp3_DSN#~|Sx^>Q&Z65pT!OX+YIb|GEFI-Su{339%XZNKEfzHQ*m-#RC zT01e>jZebRE@!IIQ9(JKT8EWFs;h$44FWnOGMU^b8ig!nxy9`*)%+0aAFZ;7j;c8I;J>TSZa!@wCq8 zWZHzpHi;aDgKW}EBwQvc|2wdBg0ovCQ;T}e4~K5!UmYx>F$xEj*!V+sWN+a6@Q*8J zgXp3+-mH>Ln;Y5r5*9q_HN7|EpkavB4u_`0GpZe$tmR%VR&w7HUBqj;QEDk)hSu;Gjl2#YRxIoaR(i?YX;E;H zwe841zQYbo>~ax@h2)GH68qD`3K|%0p0f7!e-?4?ve~rAd1uY;zsqrG%LrmPkoHun z;s7J}gN|o>>g~uieEwFyS=gX-Mx0zn& zEFSW>Dg`cSZRVQ9!0hg`acRF~-_wUGlm1Cxa9|dXnXrsqoW;q>P|m03qs#%`!d1ue zCBn5NQdaCKIvv}vuvKi{FNeEQD|RRgne!MP?J^HyJj%8&XT_swJ{dET1ws@IkM?Wl zL_BP4=HHjlDmCH3fo8c?4a|L3GXfq7nm;R06SA^9a7Zk@qQJq3b5p$nv#Ltr;SRpO zj%DmzFP3a>{Ks)6MA!6E16!5pg2J?^CgQUMl`?O8NuOYp%vs3H%xJ>E$a~^S!7}Bo ztT|qc0ve90I*FYbFAgxNWjL#zNm%(Mca;LC24kyV0TXM)0#^Ag{BCt$8yxR2iI)V3 z@%}3oO8u$8A)0maA(Qz+j+;w(crPVN)|_ybXWPh=rjR5#@6(RuI}WfK1tbZnA3q|o zBZ1W@;vm-&3s=c43yS0o#6=$obgHsE5-aX#)^aFp5<9Yh*>lDbg$x5$E1N@tcXu3B zxN(7bPsIa9i3iGTNeZmm7aEuiE0X0D99y{$IIwzdVATs~WL2I38r@@HlWAb&_z=kI zzQIL7=3~2q$>B-=7!=O3tO@p-CfH@UYR}E0yvGd;n;QK@W;iV7P-tKhVPH03P{>Ny zEXC%+%fy^=i$y?T4o66UV5-6i)#e7q`yB^@ggF>l)E;QFDljlz)nU9^^Hkeg=#kjJ z*^D0FmNCn7yvVl?aN1n6+QPpf7vuf!&sx&v%yPqc_|1a{d9IB|!sXynyuXf#c1U>4PA z;OYrr(kG9*Ui7U}lurz+g3Z zd7$y?ry;q)SKXfKGp_m9lzJiIuu+iEaVFLc4y;-W8hK+B%#D;6FtYEMlbm%~fpbCj z>ZsHeVS&0V^A_D&7O}xWFr%1->qDrcivoi~k1?O`&4cy7wwcSBcyBUFO!w0WVN*Mk z;^cHhUFJ6R;?h`P@U_7*91D1bsQ&62yj&} z3TRZUIIz$xqFLmVLs!TIC6Qu-rdW%DMm~juOUgW%#CBY2=Fd1J)Z%eSBx^#8n#93a z--071{~WFf@mytc_U?&tJhm^$DuHQnWB$K4*|Wba_`@&9?JFqCtcyazR)0Y$MSLZm! z-+rRSCBnwl%b1vuaNAMgA+x83i(be@R;308mLIDeafxTh@XAMKExY??5a!y$( zJlnATSeWnOf3L!M7c_G{7jQ0q)xbIN>A`=3T1QwiWM6Eui)rHLcW0F^Gdt0tut`ba zBD<@Cs*#zO9>Gl#fc85o6b7;pxOBpr;pacFj~0(0-OMFP|BH)|Fw z?Dd?JELXguXmj=g7UhJ6yCMvZm;`M6G&MS9U!@nX_RPl&oHK;>ZFzH7T!GW7oeZoR77w{!m0Ml@{gzd?f#I6%_A3Po7W?jux_o#C+s7MC zZ?gp-SY_>6*3{7KxuMlvKw#2^Mv32yY&B*q2Rk`$>L{+VRyeS&+NwGC^X>zSTf9CT zP{~#*KRrpub5HSMq4L>lk0sAI&dnpv!z1UxWO;*GAcILyfHm&Hzt(t;y%#<=NzGvi zHfVKz(5!cZS@#D+Y>+nphi2OqO*%6i3#*#lcd%-$Xfh9Ilw@cL(r9%!!1QEOYo+BM4Hj`Ln#;`UoyXC;|170gwl6SDEW^XFgwmBJnP^82rK4$xY3yUhen=BU` z)OcYN?{O^UO-tYfRy~C#+aJt93ayHFnr#z=m=bvS4VZK{Gpd|mQklT`@L$mmyJ!~C zh~}V*Hg^jq%LbO16>QNBjZ-Hx7}YSm+^j#_@UZC~{kkP4b60P;$|1rf(ZHp^6nLX$ ziTD1m*`^W}jcOhnd>fkdA2732Z(aJ*g!PrkvWqA9l85^O7c-h6?ZMSKEP~zn#Joti=427Bx-X^ z$GiY%o)^0sxPLU5&uH2m(WrHRSv8=^tD$jm5u3-22L24DkQr>LGa9{r&3I$j|Ia*S z_3BCbYpS+PHSGN)(Gk8=|KSSf*Q*b+N$k>lz@i^;B8H)r{{jQAMI*1nYPAncfghUc z9y1HvXkcXEdp={U>z55@-q>d;Z4?k_6zDjT&vh=_s^4^mQ_zD>vFHQSN_LlZdCJW8 zH2h)sGRIYEF-uwoTc!kikwSZ+M0>`Bwkt)8RFt+KS-hyEdrw(#;|cB_k&08xpUgSX zz^HtpNzb53?}tr1gLnOnR*ixt{|l^s513OY>~%6=akgl&+A&>m1p{9P&x=P5QXPyt z-!#bmXqMZ+z;}Rw*Pp-i@1_s_8OS(Gt&6(Zn)$H?Yk^Z!W`p%IH z!Z-f&S>W05Dx$&4reTxOi6z1Iu?g)C8ybW<_=OzK@=BNnMp!LNUKy1o)x2SUZ_1J0 zC%b%CF3UX?l=#}$n`gIX1&gEf1lAM>NoCjBC)UjRp*ZV|nbM^L)+d;K64;75*sE7u zu3iyb?a^KotYr5#SasH-B|2Uula-dOS+ugm|Hl_^xf4yc9Zh;OSkHfMjb&)o>tOow zyET-9B_x0~=s=V0fhL^~&E_kVBwtLb=w_75m?+W7Xx`9b8o;=}<*Km<>-3U1_m{Kx_Nb{^9Eh$^ zoRr~^sy%_JQGjFO$(b=nt2Z^DZwk)zV6XbXUi*T*t|FrDMSI;1M)3tVj@`Vh{5V)m zYekcgqp(RxYgJ=eGUL~fMyU&@G$dG~EV@54GyA=0@x9O*dV?kAgOS}2X6FUWRx_H- zS1{XjFxxaR+q__Q_`tkam(_DaOIpEIy93Sk2be7%IH@tPrZco=y=dMXwbS#W%jE{U zHFI)pSzt=sdl|aO4QbgGSE+6;%g6$=Bh5HQP8&u3@uu z4PA56?8ink2Ns_WHmkGj?X@>*AGBY)+rChMNpeE0;sQqT4YBrT?-swlyo@KZHEa9X z#>V%zn3Xk{{1&i={a|_emL;qsB(Q@yfPpongC$dj#i^pj)8kf5H+?)4t3At0bCy?G^rDK@jG6{c3NLNK6I#PQu!LPW z82h2gwu7a-gzbq`%iOI^S`o~;I;`#u&6*ZWniHCJCa|O%uzClyr5#|Fb!h#{+hTDe zBz*>($AyN?Qis0==`VB&TzIH~r=x+#qmkQ#k$*x%kT0X;fd*BtM%foK@&S#kM|6xU znvD{eRW~#nXEaH6gnjwUTHnC#xI#tvhs0DbMiC80o)rzDFIxOxZ0DH^X=*9K=vpPFBtB0OZH&!r83t+Flz^)RO zRGyc#Y{lIhy<9h^a@|z*zq?gQ{KehHIp&fb2W?L@)jVc8fBK;DHODB4Znq+~^Z?e- z8O(}58~#NwyKi``+90TWLBr|*Yx)cptqIKL2O3mYw4}diw&!pTxxp4{(6lEpGj%1S zRB)4N0F!w`^DMq7haJt1Kbq|;T09C^JXU0QG_(XOywWvENfmgK7LnzcmeDq??Y&Q{ znuUj60aIB4d+dZ}-V+UWhZ=-_yb>~KIGtch-@xjzqLJqWgXoC{yNnmE4fnHlTs?ln>E*T-T^rWr zZ2XxOZRsc8WxRNov4HKvfxP8quTmmjd1$mQ@MCi=Xz@6~?0%xz`bD!bM~hNH!k5df zfdP-XJbHK-UJG3a6WhU{zkpTafw}UA<1AeaLMzn#Z>~OeS0w)MC7ssS6PARFy*n!u z(Wq(Cn*T0)-iw03JvxPAH!ixqDc;s>W6)M{fW5u-Uyk^FMzv`AYhYIkFzav|Feg;r0C7Kv}n4i7XvI#>*6 zvBh3!i@v}V6L4I41*60ZhW%GUf?lls{`P{eE^AOhvB!!Q-I5lsADORM%_lEoHlNTu z(~T*$vGKaMqi_Ht4+jrtK%-Cx!w$nnp@IgUh{+rrjXXCP*!xZdMg%!#7&zSzbGy!v zu#drU0ju7R7Ci@_1|yibp_0GwU4Ep=yS8oeV*zvU4_W_H>iF}P0O?n2}cE7URw#N&cU=8%&%4=~r zH|Uw)#1F^s`fAs{=4}6PujW_wre`7^ik=TRxfO$X3J;v)jcC-pA-td2CxX4k;nDHD zvS}PlTlkwM2ymV(OHyC<=H$HCihb$5bGgKKM5t9TTNyA(&uI2M(fUtq29r)klUhTE zYD9}&!TskyjqNwI^|fW_aAa92v^>;fak|0eG=t^J?3UmUjUEwE9y1#499UC3TBcrZ z_%_M=H}A}UZ*TqCJ9)ZgrMI!_)?Cr20+oyw{VW^{ZKg!)Nw8`>Xo(bH5x9}Ro-g}~ z4pY2^l~-xdI_1WByD#l1`|xspU3O5+{hnuc=lrYa4d|m>j$>kWGql< zVG@c|Jd@gBX`CjgXj6ahkL3}T3(9`;`3j?2SlpNS?Yy|~aXX)y)sZ{vbeHi=zMFPr z2B+MeKFLSrZ%mTh*30j&JvZOE{#={yY_seY0h~fd+yoS@_g(&>SK$7miIr1IM#H)N zu#DcF_Y9xfE==Kzcy?pbW3@?IK`W;OGM`jxTk?oUZ-U1nCU!Z218)2tHlI#N%Z40a z{3ldb^WeCOsm;C4}F?K{TQu^6E7fftIJCEYqhR zJ-}jbbfAevc*z4N7WXZQZJfo90S6eQYYuj51UMLWtMB^rs9Ph!#?eL4wMOxj-r|CX zuHuFv4h`Xvtqlia@()R>Po6(D?0ULVhXK>Ps!x`OSX$;3os`=4^1xyKEf2%mB=%0d zm?3$HH$0}W?9RaxMqdrY=Po<f{x#c;Luy_{HFaLQzSABfFu^ zBS*P@kB9#l*a}t_xM`MTTxblGO*r7#EfJB>$Sh{_fq^ZgQ#6dl{M~MS7W;QQl0zeW zA1E^mdN^ou7g#7YE1&du+%eB0ETB=arr;Cn)Oi&_-TLzHgSd6Kq}@2&#&}&}#nn`n4RXjbZb+}xQ60a;il$m*zy3hm7p93W?3U%|1@-JW?47hm~bQ3^`?FEDjuG)4FiiolUaFA&K$L zKI!$WkCkj5vm`YB+xlyN?9bCvPn*|GJ+7*H=V1?<&5nyas+t|5j*M-t2iwJC93CD~ zUtIB~+xi{H;dA|$V25=dhNEN9T9$n>$1-=~q!^aBH%+yq9Rj0G&qjbry7_|%~IvRn7}uZ!WEE^WG= zc$9DY1xA?_iA_u|9*9(4@M8WamoP(5g1>V14;OVCL!+pOm%;%Ln!fgJU}TMG)rv`B z;7kzP_o0%_xKz_O%Wc7|c?;8+DmSzki#X1{C8KDbdZBl*f~e->Esp#_zvb;`DIZ`= zThtJ_`f$mn#j;W@XB#)AJkU8FEU@Wm0=r|xVflL#7>_h`a?g3dBC&{}Nr{C?^jtuj z@v6YCAf8A3f>KA6Upa8hjQK3sE^$P3B0UOB6%$-c^bWTC_^S}OduROOy2mH~ zO0o+cRamO&zmwl7g=xh~l?BT$O^}s-!w}}!pvoK>dOBofuEK+d2c^C}V7p+su&P`p zQhlEYtEEm-+a~^PD z)Y;IuC*pmI`eUb=W*a%}Y8EbH-f*Dr#V38X0+a3SE-IN-t7ln1%=BNb*vQeM#F?d% zZ04bTOHF*~idjxw7L%HqWxp)AT>4<~6uZ(s|$sX*OMc%MZ#5Z&VsH?lg*ObBv{0LIkYpMKQ1D4>X6(EFLq0w zMf`3SM@-T#vO3BriC;U=s(bHaAFJ&HCPR&*M&1wGHcWHkEI8;;aZHu*##GnE_EFv} zLJ6V_2X?Sb)etlF3SiE%aALasTRiMps)X)=YgKVSn#6mKw>!8nvPsWa%XFyec3)H4 z_JUKN&hB{QVY^U-Ro&H>|i{wK(ZjJ6z;(eTZ*d+afJQ5uVeZsYkI2At_3hsFw_Q`=W zU!sL~5d&N7k0#j&>u*avMC1^<#XXA_Jbo~cqax;>6brzr6bbZl9wkeIg zNs->06c;dnc64!dI2up6z`)3}@^<9K0}M=tyrF9rai%l4DugMv^X445%hez_LE->| zdsY95-Df|=6p1ab=l*VzC}$z9b&<#QgxlHO3+4Y>Y08+q@DKXy&3A0;?z`R$^+Gdd zu$(ZS*y$h1$RXsj>eTsUxvwVs6*hh9E6+RH<}C4$bI}D>{X2~OS!4pYXY4uJ~_fm|L-ltjA&)=cqtIA<#p$m(%gz~cbxfezMxGK^Xd zO>74ah(t80tmt4q;h=iuU_``%yHi`4Z5&v(G%ELS$6S!@nR1Px!+X}vxe+O@EDV!a z=Qzq7X*AY3BpxuuXoVB2#sP)~)+mvl{s|6Q9sJ*KoXN4VD^8g#o;y+whhDFY z%)2n}Fza!NCvtwW$7K}`9ZkJ%mvLvA!CloEPRC`NKFY?Q)V+GrQvUSPvy!viT{s#A zb};aNXb|N&B<8}Xo+E15$S7WONTWAg@GsK09#6f%m+td4Od2ki#w*=m9%h{F<2mP(kxxjz$(HZ6u~5SfFq`dHN5BO zKc)jmB`usdr?B2%dz6{w0CPwrmxYtG2?HC4L(iJd`%@bx&0$bsI3y{<$fI#N=iueO zBQu^GGH@+uNZrF(xYtvFYavI*W982C1*~@jqmntd928QM`FrD-&tC`0EsuSl-a8U_ zN>=5#)eGlkKjvLEV7`3!kZQr44~(I5KJ2U3$b6A(;o)fE-f+iZ&O^s3;rts8DtH`R ze6dNrq)D#T@%W4DIwxjrvN`-)(n-a^N$83LZ_NRoH4eNt4hX$*R8V0QVqsFzaNLvP zz@BnIA?YA%4FkJC1Ir#)mMINvM;eV}8reb`wQQVtX7q7AIk-(zI`YF@76FDB14p@p zJ~p-654ZkXv}x->z7Me}77Y_z8#pu=m}WQzaP;nMa5B|6%wuryz6O(k0|WDu&gWM> z&t19wJ!9rM3!#FQazcyT_}v%;9Gr}%I4wRBSA3W4-+`G+B-oF33W=Gy+ePtSew%F1 zH$R}3OO!>**=ULLGIuAbCyp8_(zhJ=&m5PkaS-WY(yln9en3t8gOh%Sr}zXX(-3E_ zzm8f9WPg8c6y4(}>T^(zrz!raQ?n=&_sj;?84heJ2Nh!&g&sPJdN?rpFn|W}{xE9& z2@(=$k~zcBd7zs|fiXtqK=8J{Sr!cp2@PC5+>AO5>?RD1F2@X}xw7nW;M&02!x6^3 zq#2F{2TFCRvmzdAE_g&Y4Q z2Cfxtucj{Mw_(y+5_#!Po5Y!re|P3B379iSvRyRk(EmF(HSZoWE_tY1A`_rE;nUP` z={B{DT@M{pmN=WVNKb0#IdNQG;}Dw&qxc0T?F~-a9-8_;nDjdi$$eq`73!?AB>0~# zlimy`SqTN6D~@_UrkcKS6q8|6@p0A+Imjb%Q0zdIP@%Kc9>!;TCUbshkg;%JUDI*< zC`0Fg2CkTfrY+uF2Cdu)2e=g)tBn}>*Bp?#aDeMYqsSfxnGKA*8ywX<9Hcdz_(~M` z0~#0HaNrMY)U`Oo?9u$s@=qgE%oV{o2b4m<=k978mOZdaSy1xei)nK-gwA~tYb(-7 zXk+kVP2-&6AmGwuq%vn#?xlr$LVP_HZO+{+>=9zfeYnizu$>w6<-(W~lJe3q2elW* zRMn(aalCY@P;>TRIsC_w&BU?xuDJdVCjAeGG(I#c?r_wyaW-8dueIcmipL>imu8tA zcUc`cX5DiVPdTV^rcq@}vvEU{B8QX4i$kUpnDq7>RuOSv%u$F;@#A56&VJ^gvB05| zQKyYpo#LwP8}6<80z_(DH_pTt>6ykp{Cl&ayVnR#zI0_cV*dI9b)iSX&%Y zzH!)Cfzj&A1xuS|mN#pZJN|Vu&0)wryUN(bStX-^$>9Kpk6T8sh`=oW*gvN?@IB0U z;c2>Tp6R3moF^Pq7BnfRToRV!Ex6-<#Pj-*pSemeu2|@_i{*)5u6?bWaL7L8*~iAN ze!e4)7V~#G%@^39{ZwwrnkxyYnR1q6-a2F zXjFdSB)>zJz2<=5okNOe9K}?Y3obciwxUUaqe=Wplir=u~6I<~23b@Jm~iOAao+AH=1U&}4Ff*)C^#@ts2t zm*qd|Qq-QPSm7f7q-&Y;DXvVu%rz$%_%}3W>oCgtIBPZNX>=S?D`1irIVk6GLOz6< zYvrNeQ=8;N8XK>+2wq_jz0w#hcu3C2N$fy_ybrVH0V(+wC%rp|#8M8*=p60Z+QH?) zz%Fpe_(CJE$zho}ue=*h3#2rO?l~Zn(Zv0NQIv;KI>U+U%K=fF2DL8-xaKTV@n&S- zaX>_(MP$tZt}Bg8(%jj5P6@C$mMv-E|KY%ykp7QzhJ%btlk$#6&K6frjZ0Hw-Og$V z&AHL}GBq;((Mjc6$%J_aI71F)A8{*wnwvl4;3bn~HuDS)VARoW;M8$6p2KAPwUNt-fhp(TvD}x!u{Vt7-TB1Q3K{{sc~3@KN1H= z{h&L>&dpX=obB#>E#|rZq4jlnqgR>2Eyr~-{N2W_#w-G-z4&)DDytk6%y3lrQrsr` zM)|=(&5E+7XK9KH<~F!C$zM1qe)*uxjz;dZgT3;H6mK|eX~^O6ag?udICQ9+c}fGz ziw5qP^t~P4+RtWnsg$SWsFt06F50@3O>KLKM{AAOf%<#gF+WrSwsA!Gm9xBYkThww zN^p>z)1>->fk)%8W`aZ5i{SNbVX+PwKRgUFx8xT5EjkdH!MVYq)5}>V=aPht;#<}q z|6a%)6=hd4In;Q)+5XAlGTTG;RpGqUNUbD1p!4-dMfsHtn;kWle3oT#T4s}} z^kl{92Hpdg`u`v7v~<=~ag=%BWO~C%>&gzTIR}MUobv2~)22Jg%OnZiXqcmRP)_2X zlcK~Su?q)PLLAraoEvbCWx=jkwg}eE-WOlSTnu*Ey*HrpnTwlnWaU<2hi$^Qr%tv$ zR8pPUqnbVADBHB%!377nwsPZw$lc9Hv>0Ai;(Bb6#^Lr9~4+xg+-JmELd=6rBIR5K?aw%Ewj_g z6*L3{nVdM-6kU9NczjySA*AHe(-7d~(jutrG{b`N#3WrC_aiSZC@$;iGfJ8<<7eWs z9?7NN)9-Cees)gQ>%x_ryPt!`ETeapopsR&zTs1P;?+(jl@ol5dKniIUMH=M-kG0z zpj6wLBW|0~k6kMqTX`3<+dN_Ey1b-wvXZNri)Kp3#LZdfyd<^0c3e?Tzx=D_gTaww z3!^XRy}f9ZaehJZ(#0k>FQ;~kIBnk~azNmrS=${|gC!3RJajC0c(~JEBcNsnGv~qS z-V-Jq+S$T+Zn^z|f3k{A9M5H{0(Kr{c3@`YkTytIVBmP1BW=n-i{MrLMp@~<0(J{K zE_9lx;OwIy$>cC4LTh0Q2ZzkV7BLqF7ooW=RRQhtbqSAFaLo*Q)w6to(P2Jojfrlu zRc}tJ%dg#KW!P_XLTfeO=1Uh`bhkfBI<33o-lV|wyRxP{$G+Hs(n|C_*B0jpINDtyYz4zh|kx^TLR z2W@+CNX$u3;{nqht|JZG?*$!r(DZ<*Heu%jDTi}Ao(R-FFy}PL;A4KqVfw*Jw2r%p zsri)gc^eKZQAI}985#O~ssCoZSS+Ph;=v}sEpp=g<$nqqng;~j1RgFG@-tY-Dx9)o zVXM5)3+G8`zdo!~U(lk}%(>`P#WyFOsa0Hp?khH>yVUyFOk8dCEFzHKY+Z#Rhg8dq z(=p;#_6FYF`YI}MZRzWxg?jr^ZZvXP_4?*S1@c^vLn&72)kX78M~u5BXHjH9pGH z%iB52@<;G?j;7C%mYf<_`Aida4t46wlsW&;pgqwO3jPq!yk0Bsa~^q+N?B3pj9lc z=3%RNtW}^;3-7FkBQrxx`j{t$y!g?~XLYaQaIg8dj>G--{}h(;OME+^Iz^8~(8Wa8 zZbwoZlf@1POCFsAXPMj+`$Bfi7LWb%%*pV^%=6RNa_>)D=z2hbg+G7QYA1`l~wS{&kBCCDz^aF&UyvtiAT$MTaL5?F3%IvX!q*b`#mEU8e! zW~p$o-E+wy{u7s?@7D#iiEZ1U;VCJz)lNjI`ri*{`N+&WVK)+3wHO|@I|?X0k73YP z5-N->e;M<_&0SyeO9JQ83?{){23@f`4vT+iXxHCyfYZ;Sk;AjY#WWz0D`tw5=(Y|P z6}g8T-VTq1>N{K%EN<1yaO8F~U=o_Jp+&Rg0ptDZkROYjHJmiv zE%|3OsK^>L&i=_W`P#)L=UH4e?#@W4)%g&&WX@v-o{j@dw+hdthaK3$Y{J0wl2d_& zX@(rDgoe;kjmJDD3wX>U90dDz@Rm6-F}9uIn#rKW=wfrCMcJm9Ej6OWWR9aw%%6km zT0Ik5G@dvuw()NBzBnOd#izd2Pn_GWQ=Iut6kUAh{L7A=(R`XKZi3~Rlf|7smxK$Z zZfLsr)`7EO0<+*A2k!Wki}LSgxC$OH;L?{!O)umjPs{=(p`sPdGSe8^9W9bL z-8&8n+8H$SmpouHopDGojUmudf|1+w#{t0?3z#Jn4oo?;v$uGIvdL2)_7#^NX8J2D zm@G~zvTWXU;G9K5lSsoP2ZPxmKHprJrSBXP6tVDU5?H{n;wd|$l7$14%mZdiy@QNg z3@sY932Yn>(wlxA>}Q!EIeA8pmXPF)0}N&h7`f*xlsPq_S#!=u4!IQL*>|Q~2rNku zIqr6vFNkqs$DhV?B}Ls_wgOGOw;uURCA?#G`tUsZ1Lyj)qJsZAT|;s+fB$TfVqoN} z$#8PaTzE99M2RP$;n1X60v#ux=!q^5II3`=9kgFv&zAvT>2jZ8TYn2v=q%AUHw zC@XW2H}8kDkkJMfcZ)@A7PSmNiymvd5!t%pa%bO#4Vy2&cUACzYx_R;ST>XBpOY(9 zH{^5aU6Aqm$z#%Da9ou^U~7OT!@(_Q_#>qq7}yV7Vie3^C=*dIiP*~(E?K9&ZraM` zpeagxD;!#lRw=TxStau?t6)>Ue-R>Z3LuHcKzeO5O!YZfK z>@5z9O^9HUU8B&deI}7V$b(6NV`7Vg?o_st7mn;Y3mAD*9`LV-aOU0gn_VK}0aMUg zm8#0DD__@{PqK^VtEo+Ey!|88{MS}}jk&9s#8xN-?d)`5x&FODG36Ai-33OOTM2C| z4+>4&5OiX-QUSv*H3#dPC;g<~JrJ7aaa@&6DAY)7V@FxYVyU+rM+BD%_S)8L5V4xk z=wBqDD9YyQl$5b>+f<=*C+ad}KkiWM{54Uu?n}kou+;7;y*VityROauYO-n4>qvj` zJq-6h?knH-(z97EL{Uh=K}dVsnbfcki^R4HY!-Zyz#Vghk?o!=i^9CdZcm>kk?sE! zjw*T-_GKzGddL(oDok6@AZa7acH(U_m(hhrJDrEjd=}@Jr5?xxTK@`MuA{1&e9ZHX z$AL?`Pd0G2^~H;BS-?6?>tOXpXARLc4|ppJ4)EU!>-2iU$jmE|;FO@79pKo&s+!Tr z;x(a3Xu|cZvO%8e4fzEd1`i;-I`(VcTr01daoOw+dz{+uh|5ozyDCS7Y4E=#zbbWr5GZ z3ziF*L=QZ8p)Y*PK;D7b!sD=*oCAZ50;8M*ql|)$P!prngL(ypYiXQsVj5WE9I=%{O9DqxCbv^tY!AozyG3locO4zEOJy9SO)En*P|Mfui=Wi$%&DRQell*~yK$ZFt}dB`(ELDY*;$m5`ZN27?^ zLT)ui(Fq64`u_Q`rYSHdC~)#93g{`YB{2NE#=!ref&EVVGZlpZqfX1BbCw)Sr*Nq< zT~dEuk?dc^Jmp<>`WyY1?^ZjU)3(rYWeZYZP)T4|s>ZlO@0v`&!Fye+8xFAF^LrEb z;EmaYxA*v1^A_+bGz$4JF!?a#aTs#0dBESJz+CW9l1Wi0E+Djqi9Kb5I#)m2(FW$83Wq{2DS=?-47htv)Gv19R4=sU9!3uSTspZ{cXC?$v=1e znY|eP88I-wTYdS>`G5aX{@0|{mL>3gdcY``VS#3{T9 zYGA#yj>WA(T=f9!tp`GSjSe~vc}54;>nz|(d%%>FD8&*fX@aS1E8mYY_OBz%_w^dxOI6 z6%Tms91sg=zLO$A6q39IhrLtdwhDsQMH*!$>{;S>O?M_0K7L>hvx* z1(|o{?VrV^8UNr<0W(umGLwM=*Q|$P77rBUco>8l*4ea*iBHH($edP_@Yd@AtBM0l z(1AB<6U2`iU2R(A;G{6ah(S+RVVTK;=3fdzQx=Luy8MDMTxRS$-l>`775n-gwu*HT{^SgzsZl)N`d*=d2@j; zf7BG-nF;<;bBI0mSmEDf#b4_73B8QU3+lH8vBW)i8};DcIlVV|4GtV3OsWbj8y546 zDeCDaa865L_EMBmW)xLuFvqQh|U&H{e94k@LBVcw3RMGJ1N+o;d! z-#TM~hE%q}&0`-0ie2+P3TKEKtz0nQCD46-e!53-)W^4P*L;0@t<1eq#G|~8?bHKa zt%J-H76>d$lv=_e>ftD=kjQ^-E2o2pq}IZt(-L^E9k8=hU|Z85uu8$_Ln&Lx0yegT z93G7c?rb7k9z3a1U`;u|F1IwrFyYyc5)K0~W}5?nR-P6@zUFb)|5!9=#D6!pawz+4 zqE&w?P&vWsT07UQgEFrUXr*%oXY$%6eOqgGCgkj;w-E{dm@8FZp3`H!cHouI0*7A? zf-@HI>oAJWS}0<4K;W0&^eKs)DGIX}g|V^wJ0-1A{})uiwK?LbVPs@UqdvTSm5 zwqGH2j!|}AqsSizu5SmVjx?rx`6l_Rf%A_7Th9W{9tEy2;|Vqkn6nbt6Be+Z@G_S= zz^q`Vu3u=aa)8<6WvQWHvg)_ej^I{F!KkGRqB6Q1(Wm^ty*qC5j@86u8nBPAhGkc92nY8N-ih z4_X%h>&2qRxqLP_!U$wi+6Z4NO0x#B-}@%|#cfN75dW6V1{AGc?wSyg52 zaVs{kP+-gyWMVnMa?62lLo>s{9}>xdObrZ<7c0^u4^%BR=Qy>1SK|$jmLlKN1)+Bn z*n$+L91aRjStz;cWB97I&Zz+(eyBxwhiD2k<^_pFhEJJz`lHr6Mz>y_#(%m`b2M*G zx;?GI@#_-S3KPezlk1&Xa~7~MI0~jSaDO|%v1zD9zB+N%;VW_q zvxU!=#vUvCf8;rv>GPuRlh!O=!LgJvtATfxB0rM@V?rQ@T4iQ|A}Bn;ii4lTs|#e@6vF0kDBbgYZB*=`pVgp1xrlYZ?iQu@Z}shZ^rPC zrKg1>At8SDfh7qJeL;*~<&NSdjtOauyVuWN(HSMMt3e;x{6B7rmp z<^%?QwHD4K2IhGZv*tc7Ics%gn#D?vr^b^cnA60UE4?YLb^c$JHhI>$z_11EHyYTa z4lpKZGO-+Bn%B+8wh(kuw21=)--FOg%1lBB-kdroJ2{W}@VUUUgmjSuOg<0hi!byk zetCF`qa+jKin#}QO(xu8I`5n*aBR|2jWB7!NsLn06b)6^FD`#G@vkHQiiUYSJ0{I< zW)4U&SsHVyUV16xjqR<`ldUcP#5f2)W8nPN!@+Ti+a*!d!O_rWagioh@y$f>49DHf z$5!V1N}boP zkDR>Ey{6)@xn800ORIFfiR>{7pco5nXtQOWFiC-P6@!}NBF24wOt%;u?cZLWkjI?< zci$xjJ{?DPp9NVaS2k-qPIt+X>hhIbq{z3#m)&jmmeL0c1+%4ZZeJ8zU3aq3@SvIe zD+P`slO68G=>ZC-+8s`D{meczjlwR9XBIsWNa#x5q9DN3$fq>PC;I>s+eZ$*vn*Pflhinc<=5_0K6+7A zaP7rev-i4Z{JVGBYR=&|`;0Fu=Uj2i-IJ8SH7il*(oIGm2F_Or{LdK7`qvAra^Nd^ zus&IHTFe3=uUnVjtY$s;&SByK!6l7cJ>U3JVkF&qj)ZNH@?u=E`XSGXw>MVr3lDw4 z)?BLbM&X2DaouS_`7$4Q4;T3_4#HQ&9A|Q|205^_&ECHG&Z*z#J8n-qU9Zg6aD>a| z0B6Q)@hJ9lwvK1C9x$)q7hz%)adH#*vEYNb<3~w(p15FvSqi+&3%Q*dMK}_l{$UUj zkz+Hf<_vnk5wxKC0=IDYt9Jd3`(;+{-(%d#%(`md^T)Y-Rw`>#%s=k&DBBamz#qbJ zh1G!}?LiCQLsOnIsaFU7@!vQw&5DUp&4Fhg7xRakRo7>Pq?@p|IB>pLz#7x~%j^rs ztb{5334*H>ML$JI=55%r@Y=#^rA3=PLr*No|F>-8nF9v>rLO&RJ{7IF^X0)U?VGpn z&U<^4PoZsr#H2+DT#anPYELFKJrrn{QA&zPZ0?d52()_n=_#L_P1$^_(xRY;hi7^% zU@Bx{^%9WRU+|;hlalZhjlfuzNY5mV2@7I&dPsIEx=)hkHF)u5XM$sA7l(w3O@+YL zRV=FmWj*#bC~93_9cviFpuOy4lKlQ$P;@&kA?q`v@la<$bQL(`3E!$T9%7X@q2Rj5qSr%?;=a>?>-^S|xSC+2j{?Z37uz59} zIMVK2qq4)Zsi~syvUs#Yp^HLwMq!6SHBYFkO6?9o56u-kO2?%QR0O8S1bc>tg@kXt zF_lFw;XosM$ki7~-9?rQMA_I{MXyKa9~E_EW?IA0qI!140Y)C9GY;&E)gcn8TGDop z8W?&U4GuCYZ7^`+R0|1n;!^Fo;51|E&JF*jF{!#7a53sWBhZn3O6H)8>53m;Z*gup zt2R4#?@hH8e7mP6${1c+F}csS$I1L|$=?r${q6p|X}-(-LN;X{2MceYoFiN9@(F2d zhACOj0tc5L;&Vv(bVPj1pGVv_?|PVys{boxYGO-u5%O#@RFn{IZn~k;B_>)C(5&2O zvhb)_=#>eMLQ8+F;!&9}#mGa_FrnZPbI``nsXa0$I2N-!b$i;#k{tWtKt%4<6{5TJ zKb@Gy?6gRtk;UkZ0f(ZHfC2-XyUN5CLAQ*Bu0m!t8`${ODykir1QZMoa%lEA%-o(l zDbtBNdqt}wvtWwALC!^n3s0Fm-%@-;@SKKlht|0}{|>tFZ9lv0^Zfmt8)u()=uTW= zcv0v2ip3M$v{o*ezR%#oWto!aGg$6dexKp6Kx|5;k8U>C|2RS$LofC3p6+f}TL*6(iz>!fpqkyG7?~mXKp}LU9 zMppH6Ta&s|Q|CSE4Vn8|l(A3DRmeMeb(^y6Lgygx9;9g^6nZ|U5l@^OV zBvW~&GZ$L?S9@{3W!lH)3tbY zd(SkjXLn$5RrYoixKY3*C2)YLL8F=L$%RHHiGzjW5r;JY1$0?!C}{C^D4$e%z`&|; zfKji&FSv6(>v*&_lX`clQ5>p;ko#L2pG^2nibwQHg(gHWla|P_icMkF$(>zyT zw)5nTo`>nFJD63MIQCVXIV^X!Ql#|Lb`#qG#$f#ex60*|OhsILgCC##C&jdPn!^tl zmRO++jzWJfun5#7isdICyG*rLmFRFCE<7uW)wu zN!)OhmATm1sOiMYH(?&L$%l5Y5`RfUZWl%+fd)1g*UA%{8Q54pu-mLz$YwC(AkV4` z?T!l;NxE-fIK`B>T=|5ekh?*X@f^dm85+$JQx~wP$}Hq?5fK$E_sCK;?DTvr!7ezp z;n<%?2Uq-ZbVY!cJ=%Jy-i|8l8GOi~Jtd{H+X{t51I z;EvkRB)Tqvl|A}JqpZ(nfmnkHoL`Tz=iGAIH{oRv9$QopWH8 z3dm%Wd9b#LtFvRF_w*1y^QlZ8OLf2Bd7jDm>cBc529JV^;t^#I&59fhY<&+7GRPz} zTCZv-FIwRyBlhrw>jwu$)&K|ov;_>jIuH5WGul*jHnvDfEM#lmaa1?P*jeJk2Nna5 zJ4!PO*g_sW;$N|#&1lj@o}`FHLdOi;RG&q2CUJzW=t^*@6LsWFQgL4O%pgi@@;0xq zXN_D=FYYJ*VV8^cdRx!)bBfcg%kzQ^moz$OB=GcIXt&5=U@1;z)wMauy(M=MN5f@C zxqlH#?XD@oYOF?z8flF;oNp}PlDNe@WkW)Xmt2|mf4y+RcTA1E3bO;$ZGIz~bVwBck8=|5tw5Ro; z=C6cCDV;|=QVx?;o?O@~C6Od_PQt}_Mgk+RLz19-LnE8fhj!sP2TbcdSS^$2i&Wk18?W>mB zo{h`sNxl+nx;1q9Vg5DG`UR?9Gz#qU(puM=-jMWkicb1p39e>crEMt-BHAx7&i>}e z_U2ftUf9D{{WVUaD*}!R$z1H&)2zg&!+2Ebn_BYJFS_d7#z*8PEojy8VdM_ua8dbp zL8^nFiCJxdOjhWZe=cWhSI_%g(*5c{snW^?UGA%eYo|R{YUEtw%D}79z`&BgDEB0h z_ra+~r6&s*xMdOrS4S*iVl!Zv;mRx`X~Cr7wXsEYha;cHTt=Y>4(xWv6+Wv4xTwx3 zXyN(sho6~4zRF#;w@_s7i)Z%sCfD-6e4Sk{^LR#<^(w1`(4bSV`Ytq!lwG*$b1{nL z1yk6=hq4@uTG^>`5@LcZN5ZrMnDZ{MKNsdazkqAi1ff*{T;=6#OACsZP7u;li)dMD zzTg0-_7&E419mqC&ejba%?uok1sszCblnmd-7feQJrEOIP`T!~*qgqdb_?)MNML$?%K2)G ze00xxwLqgMC^UF`)|5{je;#mDUSOLgz+!TNkxR^!YX+0Bfa#n|vg^#to*vB3xhluM zF^vBpb6x@aW=8gN4dt4QTI0{IttO>K*DF&GM&7*9dw-GgcQLD2M!iQX^otCQ z90J(SGO#l(R%ZO*|0Bqqc>*KXgnx{S0^`zdu&-FabMslN6bJJPMP?rX&cXwcvI`hy zhEKSC-ML=DSWLjPEr2WFczaC-pO|F5lqa8bmQKQM}JU^HxCJuAVvih;wMf!Tcnv%8vA z^2FX-GkS9p#WafgTy`jjJ6e5L>tl5<&{AMDIlw5AAkK6kuuIYAc(Qw03ET1oJW~sp z_Lj5BgwOov;i2|G`(y&AM4y&uK+Us-VlqPf?#&SX~I#=R8qTb?ke zFJLZzz_VgPkf~EMT4`9{tNKf9~g1b{wwX#TW}Ut4dgWsx@7$j0%1} zVi35%V8p<(eF2C3E>+G4%nSxLN1GXVCopgbFdli#z;Ge@Y^%}L0=B*8fh!m}E-`Sn zUh{Kr;9B~jT2@19=>i^SDb6AXu5$%^FApr~tl;~w>L1@zg{8r(G{RQRj^>gLZa4XJ zfoItVj)>!IVGgWQJ(xMVH{5EJ3tQZ9>St#{x)!60)p8A2F>U7Un^~j<6bvq~Bu26n znKD1Ty3wJXtMGzOy1`1zt1SA(EK_E&mN>9CPsj^%;OKJaFbl|VQ(*7gkgj!sQ8&S8 z#pUTN+btI==L=LVd{klS`MNiMqZt2(D9c5u9HDxQ$_&B^Y*h(tl?m+oS1~9^F>)-B z(9Ckzn$>K`#rY(FYi~Q_iNy@v3cObx($gn!9ofpUl#w~TfKC1OcBuyEQw#XsF3@<_ zwfOat~DhdFH0EVKS?VAj=SF%ihtUm(Wj zxGODuIoA(wG3~j+4g$Itm@5?6B{wk22e7y{}kN!8o=QuFiq&w_QRqa=T0b>&z%0~hwkSkTjae85)&8BOkSvOwE9Po z&C$yY{1ce81K2A!FgZ?O?|Z;8V*!VegJIXfH435b8owm>UY~Jw0rUUc(fhU+s=Pfbg%q&xCotaPSQxlk`HR}|c}utOF4(e2d29H@7S#jIr9JzO3X5tTU^|<@tvz84 z2LtHPO%dUfyaC%vx6DvvV32r_!BxO=k%5DyhRN`%#|nnF6#=}L6lRJgFj;bO#YJ#k zEnqTc;5>U^*4u=Gk~IgVZPvf%nxyb%N8jqD6Qs}Vp2>gy0?+gblS(I?(V4Sz)@qgz zgTrZ2ou+%3tr>PL)MnPc5c+qynDra)r?4D!5wfh10t_9o~i+4+3U@>O? z$7vjJ)Zjv~+JWV9Z;sBG$|zpIXsp2AEWp_+pc*`ZRd!pRrvY>G2P3Zqy%rH$)|l^e zxv}Mb^p;vly&xC8|5wDGOEWSBFy(Dz@=6jv$91yZ(@0H;iE{yy)dD7?1&l|7x1|ek z`dnatTEM1yf!*pE@l#%!!Oz49q?X>}M~r%yHm;b3t&+CC-!ISfw3U9xUeZzEI2f$*AjDZTGRGk{1|b z5?M+&u-E+)h?HkwwoGSae8grE&g|YG9^7peG+XJzqn72)1=VjDgw7sU4baQ}s4Om( zFOeWFtjPNQ_X*B`6utn4AF~;l64)jNa5ik<=(@n9!L`;rfK%crhtUG&lML+Y0i27> z*6sbSR&gy_eF2BPSIjAg^zO?%R|@!^t>t^CwLWAn-_sB4cJv(lEHEkj)#>ht`=O?j zlq1h3K$jNMa0}I0i1~CPO{k1dp zaWSe2@b+C`(tW_Tyn#Kn+FiH6Sgm1^&IP8KYnV3}9_VvpQV(Y8OJkHguzg7(Q^SO~ z^_6^UG}o{3JpI=8#TTm`5<2(g|L}dzz5n$u&l!PbF)I(18n9Odu;&O|W2t6k&b!9q z$I7velhg0;r9Th2?arDeFt5mC7MJ6Qf55W%0JrQz?z>w!=YHT^{D8$JVEZ`%jv@w5 zk&g1_kPM}%=cDg18hl`DU{I~Tpv(8Tr@kS0&RJ$DfoP>ydlxQO^7tpVHum_|y<4g~ z8JUm2-6(3nVlYE5fT7Hgfl=Wf!>e`5ToV{#*Ku4<=A6KAQix$&xd`KdSQ|B?lRYQZ zqC7m_&tsfZ%j8na#v-44Wy6Q+9qmsY_?`-U6!c#&B>&=t@A|Gw_qwI;%irNsj6D6# zpngj9&e*s2r=2@A!*7{x&$W!)Yqkt5IlT`!_M`OFh{*u;i4$ABZsz%k=6_o4#r zEeE(upRyQiU|A(_PhMa}w*%LT2W*z%=k@s>Kb^$rWxz4%z*WNwj7AT70(`h?53r~$ zV9LC=*V(UNi>|mh!;?olj=Rl!;^3rT=m4X5AA7?EMKyzy7iOgu zI22t;wGjwdbl=;sgP)O8f$7-iONs$ZivJpHR`tkSUQqaS0#C>t_N5d|#fK!xqkPIOo92 zIeM%H^$$+ke>UQOl_SgI5YU+|P#GA&a`GAH;t!mPA78oT-|*P~TI~WOOTM~k1EZ9I z<@5S~5)&9-_cP{bvX>gLp4|QK-t^*bwF!Gnx0arN>w3F)-Q2ef9~U$-H}diLDRcxh zvG7T#2$=j(Sa`6VpJPj<2(KTH`!v6)n^HL48d(HIG%^ws8`^j{Wn4T0gq)fhc@=yb z7!(ira3~5*F_1i%!pUkdV-BZC`^nBlwO+2(*UtQ#X`Fd(QQ*|D&|9JFE^f-}6)TG| zDSCWrp6d3#IhNaA-C0$9?aeh6_ZFk4E1S3X&9f}N@n`F&>AX|59ahMg-D&QcayY2L zV%ep~h7(RYTw21Zw2W8UA*#zl(Dn4?4F5G1f2Bf~cuaOZrx0l&_?yLhnygOBi3CPx zwrRVi&%L;?u&swj!sI|laMs27_IW{0CK3k|j=a=g9X~(df@3qg(0bz|298W@?JV3C z%PneIS(#WW{%E#xi7pBD@{4LvIMBqvnyx21QFD2t$cB?@Y(-82j;wA1T`b&A84I@- zpIWIVz&*jbRVqDaBA00Tl$lyb_*Y5^x}+`%eC8q=_{VEj;lCIe^<$gQR6d)w^VO*% z{d-?sMGrw}-8w;eLx* zySVHoI8RMiuyky{VPwJBCS38j*X*&y0o`V9n;i_zOa=uFOC@<4Qzm-cnV`tVxulw5 zVdt{oFIH|BSNkn;k%Pk)$yB$;Z(a{0c?{jDs~@OVeup)iB0ZK!()zKW>3|r9{*k__emD)D_QhTp!dKf zck{V@8J_%I8JCu7PEAk{&xlZFUao3mPpV4oNQUXi-)Q>=bocB&5Hlh1)Zf)BC`M+^Gg` znpzWkWQCgBPkvZvsF%p2Bz95a#UYp2Yg0MpULMu2Q=B9!Fj?iw+H)S^TdpTnDYUA1 zByw-oJ|cKdfHU1ESv1|{s8*0*heN{(smljg#U3r_(oaa__l`Is6vD`r?8?L~`NM%d z;laPQAO|I(#snwD*^^~{{&D21h-l_f@G$Y)dfxg)@qs0~4$Cg(JR8XI(QmPU5o3^) zOQujjBTLK#28l0<0)8`3DJ3Ncu&z;*bicqLJZ%Aw{2wmI6m0<(iGYLL?^HAdE*ucN zA<m1!67{&0n2x3xji#^Dj+vZ# z!DE@hDB^PB01sP5TZoR5z>WYm&7OqD$0rsFHAJ{*-D&6wx}wL~sBvT^TcDlll;u_( zUGvZCSjzre=y_y0)5R-)jVd>@yErIvM|yClEMSn`q3hac(Q14r;gt0hCpPbfsU{B= zG;nk{G>NF(X4zlmxNhDOmZL$fMIjqIlp`GYmUgkoSsb^txgsi15#S=fW@q@mccp?; zeq<+TI&vmW*(@=2!%^K!r8`wWW#|X6$Vy)_V69?w6n5u2G`VWR&C8Z-y1lM^G<%?^6X3`xWWc=L zu#wa1N3-#*1ED;U%5pmq=Qe_A8rEW&! zErxttJB(@*F4#@CX;>0&cF@m6;=wE)l>_^}sIs^)Oy2s}Q8H5RxYCr39XtY;B>R3i zt~jm0rZy#!GgYHmGURTXnajV5L!xzGWi@&dUG6XncDy*e)>S6S>xM?`3=0=Ej)&_m zOyn~(o#rFZw4qtyM#D|(4TZg?6C)L#?6|djbB=(?MrO$a32a3cn{qOxIRX|q@dcPY z7BpjM-mb+g63LdRa!#QAgKDFyq`@J@-vXR#w9*BOeU1v56-cvASZ;l_w1HJ7fWg6A zg^5?7E8y7dxjuX;OwUC7S@M+PmPEGX{8Ub5;ae)q^I{33?gvLUk-{#Kxd{xpI;~1R z1@gR094zt!90N@?bms18VfbU|8BmuxX-*4+qpAjT&gq3bT&#=ir)IF}MHMv5>`-72 zKi|l|Medb+kJQ!p@U4<^IqaK)Z~Wugv~26z{Mu~kjhE-zJo4V7wD?V?_H8zk1J(C3 z3%>8YynyX=+O}TPjgh(9(Kl^gd}E(+p}ie&7(u413h28RY7O#NQlX;hexISJBXDe`a>x1@A!3l7z;|4;IS4$T)5}MWBJxg@IAo z!jaR!fthp70yeD^41ppB#glccSf1~%H=4M#QSHewrt}FdK?fOR0vKg77{v?@h-EOY zNp_IG&?tYwNaWjRcLth*xA;kQQp8s z@&~hx1=BPy)?7!ym8W`D3fO`duw^+gU9)c3wvsEoqbq$ySB3?1sAVTpz|=V(Rta!) zDO)rxJ|UsDp+U8QB{+kPGoewaf>A7iQCQ$u(!Z2O=8yv`cQ>XUv08hX(Xio|rsLKP zn;m$1y7)uHcU*4Pw%|#<%_thf$X9Vtv7%8?f{D-IxX6QsQXfYC3P!$$gOV|g3OSBa z5scCTOp*dl5(bP_-t$5pNLW@FNgS}5a&kp!b0aTszM?GXmm2@Q`gGuljGv}kAvVQ78O#o`*l;_+b6(Hb~75DV9bzd&M;v%Tp=iZkmXp?A}a~mYNb62 zC5#d>O%J7+L?q5C3N$JT@L!f;;j zRO2MkF!`m|iI$LM&MO%CFEH@mXpp+m{8VTOAIFl%9w&G&G^jMRcs4ZJ>}+7tVBnq6 zC}F|m(81#IgSpvY=ZpoM&UgBpFCMmCa@c}JP3;7;D94c}+4jr|4DAvP9S<6~B^bDO z^jbKys%~iDl4xMsA>dKL;&7|UR-ncCz*>nFjmid%GlKtda%(g&ykK}-#G72$<`KZ= zu3$A+Ka(K3fo&48(CiH%akWt|1gnll? zSTLD2T(;=F{H%&4XhkDWK?C1}CY=*Z_CJ(WBNn=AFa{fOP8VR9zJSx#hQapAVHO95 zpFE-B8O?ug9%jg3(AvSGxuP+80TX*hzvzKx^9DxI0%j8(R?C0A!4^*$TsvBHI+*Q0 zFzG&Ec7DL@B*Efvp}~uz$4hD5$~6DX~v-ZD67cqnHO{@|%k(U#_PdX%v>YqpG zipBC?w8b1~i$1_)^MlzUpy`E2!%UBpGd|e<+%nbj%+x4O7mfoBvvv0Uo1M}eEwQa6 zfthJRgXhv_^A${16I!GanmjJ3+8tnxInl)bf!X~6<8;v$J%(oI3rseFjJz7H;j7rP z9NJ3+77Hr{glMqqyzu31Fqyl7L5+jOxM8~k!+F;TF;w+9^ za<0b?>6~HlacESHkf~@(KhRnlwMO6wch#bed!iewM9=m}FLimj_gU_}?&ck>1&k^m znv>_wwS2*(d!kA21(VZ*X4@Og_8S__KQ!APXxgpQa{FYHlLBjyM{B-CJO6K((}#0>8yKZ$G)hG@r6jGkJ-}Lcp-oSt!A7H1SfhbU zfyJOBKv~1sA-cuRqS=2*TNy`7Y}9IF1*WbV>ClIk3msxv1uh6@GztHh@1Ve%wShrw zhwt7s;gYOAm7;D3t^_8FM2HDQu$mk&$!j!z7R&$VKuO>Ys~OCii#E!vaWs;(6I>9{ zee_1R_Y_x^22PGAYywYM8J2x2W)OeTXf>flhodECMqBoT6whsK*$>*@zh;#F!0eFF zWLd%DY``R45fH4<>Un@Q`bUE*2aAJ5OLPSb|B41~52jhAY(WAN_EHQTdzf@@H0l1E z5PbJQ!_{uS1jX6h^A6wJeR9Tv2gznnEK3@gD;k8SF-d=D__o5!*|f=a1{3#;M!Org z_n&RmmT0ox!Ln=xgX4!LXAV{?4F={L!YgDJhg7g-Wi%=;VDoEEVwKo2nZtm^kn?o+ z>?{V>8yqb`mEm81dmmsqBP)8$P|Qu#qVb@Jo6Luc4X%$BcrV>sd!}V8_Z|k>!wWWw z{PEH9Wvnvw{pc~}jY5j!yav{cmuvwp&T=VsG8{^C?&~VB#BN~AzR@~w8(ZuR)@*_H zvJVX^4b1ir7*!bVtNdV4p3v;6!SZ2hGw+NhhXu_lFPfVKFYtFXd%R%Qkx&WfVCH?n zP;j~RUkpR5?u!N(A-A6c> zL{(mj>?peIq7@|6;oNrb-r~%67x?)YRyf~ic6P`wOK6YPD4D;Ht>8eb#DgB?9SuA$ z8lL{SEVaR}wTDr1LbGE+pCd<${fjRTelgo6^xcZwt~#MPPK-s@fl1DwSV359GTWZ2&nfprXL8I~l7v6vi+!0JnCpKNt zZjkC|6t!TIyRc*Oj%KeJjf@J3OdA@+JOnK)y6$Q*t2H$6JmTaCVD#Fn_iKRu_V1XbL$D)_RMuaAEQV@Jyg z5B1j{-ZV&ZoS*!#LE=Sd#~T~&8x5*InBMJe;!kkhzO7?!NA?CLwfRd*cO5I8ZK7}W zgh862k$*uWd%%&HZH>#~F8_N}z%2J5hSipdJ)p_^0VC@VzfT1noJx(Wm=t&)oReah zXe}{wazmqQ!ReCooa_ghgLg2?&S{VNVcCM{|F#Sg%a|9uw7%KEmtY^G%21=+q?YljErOXlwT!2LIciFaYJvs( z1cd}94h{j2IV(%jF6w%3e{?`Y)i?6rO4sxr1Iwk%O-wxkQW^=9-kg~!**N9sNr&Yj zChldImIx}buy@VSiCD0}^0S(suzBoSa#BeYskH zm3#2@a0t5DXJ{K7isj%FF}ZQ$f%%8~0zxJZ3Ue4g-siLU!TIEPzW^sMOG!r36RT+w z5~p?qUV46B!NWZ(qUf@iJ@$c=qs~TVb`OFXbHq|h2g5lp-i7$bi zE{$^v4{_Fd2>BWx3vl98>U`1YCY7|O2vl_V2a|~W7^1E^mkiWY#D2ZtHR|cn@(vh_K;kv zvDn4ZOYeF_@d=UOl!-@KB^=advKeGJJWHRGxIvO_acaUr)}<#`Hn4F?IJ%3+zS($W zrl5_2BV(@Xg4=HU7>_qBJhZf3u84<4-GN!#!renPXUPQyMzbG`9g58^hYmBRA1FLh zKJT1?vuec>^$B^BiVGMx(`u%48Bel#%)`^Y;8GJS^PGPhTvR8#m=MUqDe`hYC&#=E z2RSDkE#x%4;IWx!>zRmdPvuUPLzg7<9u^!9(A#1l!uPb#1zCQ`u(DoSB+-FikAwKxAgpG_hxM)TV?vuo~WIIKs13)8RHVpN+xn96pul zFBpYn3K;SZ|4ox&>0>i@VYgnfp^1gXqT!%O&K}2Bd4q%p$CR{W6xkIDj|8}>cKx1ud)^wI4Cz<8{T#Wf1_XJz>ZM~Xw z$YYzxwkOR3GkzB^KcBn3;P8$I&nCRv;c+*KE7(`|(4xJiFA`gOOuiK;AJA@@+QhQs zea8kCyUnHtncdYoGns{6Txd70iC@M%^sP!Tl(nnGcYhV zsV{kC+K?|Xp;4MaaH6#iBY#&%i_x5g9RHUbXOwt3+g(YhbC)BN&^eD|8~!9W_Hd{u zcbww5Go^5Wu#S?X+K$7TOA`A^KLp6x#kAX0?C2?Vc^5UKL!kAD0h{5EhJT%wCmQ(| ztZ0^*^19Q1&p}~{=&<www=mSp*6i?kOy1 z^vk}%9CM-BiXo9*K)~^%2qTC4A4Z|23Com47IL_U91_gUP-J47F376#fF*sxGn3gn zIb07cWLY2Nz!7o6DQxd6wsMVDBdv)&t|}`;E(EX()UG&R`)@*ri_W7bn@kr@?O3?r zpIycy$D;K7CT^HcDJ?dFCKj=7BD?stwK3tY^v+PRg>YJXkD0Gu7i@G`Cl` zii(7eqG&@*tA@itcDs&8OIPi1(B(?ZXp%sO?EwYO!U!h8v<)qGM;>w* zTO5*mRG{U&U=hEzvn1QKfKGdhL%dQBPAZSeTHJpe6f!!{z!dXxxx3{7CS483DuWA6 zUsV#Ob4Vl<$UJL~4sn{KkNue!mMS{73MXPj}Mgxn^HG2kwkgB*=K}L&)e5MT! z3{tPn_wUte;7FLt^|zSCRpl|C`i^GRSrhvzPBd@*^JIB;y+*&HQ=jDV11zdijUCo6 z4)W#x^EfQUr4?hi;)USq?hFaz#`O}hj2w}iP5T*^9lPDxs-tA<9o1GMzLjH1I-doj zu){*0XSt7fd~Y0J@3_Dc;BZp^asaF5yhJW56-Ca81}+Lbg6z5tPWIh5l=Xr(cBpoU zi*{!;t^4O7X|c3{uesx(;1q+Ic2knL3U5qXa60MQ}GI19buwv+n~qWGQn-bh6iOS9|I##EtgOTFAhv6~x%(u)s-jm4mCo zqX(~ex9~7m6dd7UQ<%l4!zdKnd{EwxvH5gZn8%JlO_mP{oE8p?gscph6&3`xg{nSS zx8L)CzBmJef(bJpUq&NKy}|@D1|?>`ACA&`rB7mumfU{nC{o2zF|U_Fz@fS7>H;I7 z1EqP!%Gxq>q;($l7*F1LhOfNBR!Q%p;rq$$J?)Rs-hRh4z>(|x z+v|KO0vp6vJYbc$F!vuvo51H8>N>kMcy*@v7PL4xaEiopoa8jQ(8zy)fz9{>)09<# zy-qgD;?)LCwg06%E#4Fgwgwy(Vq)wHT9Kr3?#3>e9)~WAj0H?{UN*?+Y-}_BlgQ?I z;)znz7q`h8Cag~-}qxAM}Y)O-?0W}1uq4L8>$T~MJBy9(;g@tn4c50xS`2HS@xSi z@rwk1&cZ@F+d`ub{nASmYs#?}sm1YXUoBY!(aTWjHDDt@*mfSxF#v zd8ro13@r_3l^wT)CNwFkIP%Q#=QUtdS>U9(powpa6N8l~M>jM97j-b@iJah)!(ry+a=19yQ#ga-rri8~%kkIrW} zKOtmhdEx<{Hy79B9$?FG;GDw1Hltz28Hay@ALi(l99DdA$V8+`OyZ#Ao5nkT!-a3S zEA;rXtvn=X;HYB3eC_ST&LxM0beJ?{4hrfVYREhw_{UM<1H(#}L-H~Qx%N(C51hu# z(ZF=1(P#%_y+{LR#39)QqPO2fzbchxS>j+4(UkhtQF2NHlZUf>iHph{Csh{*Gs|0+ zi7nlN?N-l%3cnoV{uLDa^OBnZ@1tC?)gfVT&c>;%xF&Y1+4f4a{k>)fE)_eGC601W z9NQA#+s0)7ym0(!q}m=unPVKH8V7s++zj&w58raz_6j8^~ageM{~wsR{7D-rw?nzcJMA+yym!>p(3}X)P)HR zqE1f6D-PQfJQQnM=p>}#6sMNPr6Oe&V{=1NQDyq;4UUgPRowDgy;oe6`nvGlpN{?o zM+H=R&YuY6(24rE<$#b#f`>^G%N&MNHV!Ja2PD@tX>B;HAQ7b4;=)$qz-Ho|mcziK z<9#8rK_&NCs?RJLoux7cjtV-BGacGyM6{)!@Qk*c&2Gz(`j6KkI`S$H!)(qSjGhE{9aVH1bSo)<1NnZ({9?gEli!YqU&6i{R`){ z*IspP%aE`;q#`4(c;%q)S(*))0?4@k1+I&o5}@N}2o9l2KXD zJ1xe6HOE1<>Xzw>CdE64l>3fe&C zZ_MYvG24H5bN*Y-`Dd?Ezh$kji;?qTmH8qmnh~<`USglZ>V>_mkt{Dg_CAZea8ZE8 z|9nRSS4R5MrBdq+7qRBN=BYU-|B^{Bg;6o#pwJnHlQSIBa@f*R*ir-BR9r9d_(<)M zYc!wRSd?^N%dSS59dAX>9AFk{m=NPBRJQww>^WB^CpP0Y6<$vMQUAIi5`%!b(i42#$9IkARhIcy_ z$Yyi>^U&o=kpK2kaYE93`D1e;Sr)NuIB@RkL?6q>06vB)SH|kUtwui0f@g1P8ji*mP7_xs3)M+8ST?JQ@M&2z8%=3onB!nHrAcXulaYyY z+_z&aHPiGB8JKk%gr=`)p10KM+aznXjFVm(8LVP#Ee&TP4)CZ*o6cyqT5&k5GQ(k- zM%S_x7rfR>t^RmnQK9_|r(lbwo-`&!72P?u@lrFEEvR(uTcZE#-nsBK4t+eI!apdo z|5?c162kdH=HGco&*dK-6+Se|d~p1??YfrFVV)ZeyeAxaS2VD7oD5ud=2XoA#z#yn z^NQB6GTn1J_B69??zIE+KQ-{2VPId7&KMw;+4G!v%K`ZZj#@(RM7}WaUucw*?(icGZ+Uo5e{5K{S7geS5Z}EamI4O_ zZyb=8Y2=*a|Gi;3`-16gNB$+LNH7MTO3`(5IyYs*tThc>EL#JXH*gg=1WF%ZU^&1b z;lLNVIBr{5n*=*9|tBD2krxm(pwzP zf8pgee7MMfVR=_x@72=S_o=b+_f9aK5EMIg|HUF>18(e&Zd=Rv zw`>!+;ZU-&j92BL+~138Yn<}_Up)HfqM1aK^oIs%8OD^72El|zrfn&;rU!)>m^7C- z`H49xq&WWj=%BE|IXswIQRM5}84TtBiVrX+IPhL*R6O#+q~Ng1g5_LW4l4XPXzar* zyTPB?r_s2AIsAOX@+YcG9=toObv*yb(e12$=_mE9Z|OafwOIaaD^K&Pw~r>d1xQ+H zI&dy=8*7F|+J(wp1~b{9b%MYFmo`x0KZ%J9Sz$6PVNrwx!=@bd_Mp`jYX5< z8Qhv=b~OI{>UeD_lUW0^c>=SUNRzxkli(jlt1Zk{OPDp+ILydk^Xcm3Inux$a?rTu zr*R9jnZZ|PA4Y{ej+ze+$r+R~dK^@FaL96x^FL9BL#i#MyK%wQYuLefAxYPF zyJ%LEUB;=j%U0UH%dT&I%apdz;H>rUJQjy{3CTN59vHWuKK!xnN5?yz3v4E-*Q+|c z%huX68|$zJ7u=cslF@sKhPOwez`hS#B?9IXI3rPwNO`0zb8Alv8@hMk0!ytB} z$?QgxsZA4mi#eNz!-3gHnM)WLD;nf8n$0y>^=z2gDjXQUG{~zQl<{ejH`&U#=792p zCe47uf;>%1D;Uff+jTEWGl+6Ba4;-kKJFG%W%l<$&9i@Br$@r8qOTWWi6b&GBId*1bvQpqi)HBNWu-~YMRNz0*0 zIz+&^=f0dPlV(o3!jt>E|24{M993aBRLIOEbmV}{iGw0OuIvT(4|qFNtY?r3Xg1?v zk`1v|Jj2NP=cJ%TlYGKKnFK~5AIHaEe}q^tsl+t#FWIbL8&|KraEE9>Z>q4>>vhe$ z{xro~eXL&>=;GTkzVUj5Me`<_u{KI7lGS*s*D9OhVF zZb_J|>N(9|V&StB?7VL0rgVvPa86Y9eWJr9lHos1GkAi;%1h5Z*d&a?WELKJc(9pU z(7C9=DPb@7SpEC>q-p*;!NSYi-fO~T5s2j=crdcv891gKqdX+ zD$mtox>A;@Pfkn?UY>W)ruy5vyNBET=kKeV|NEn`Zk2rI<)z;pgU`MZGIZyk&gQcg%(`1Ho_Mm{w0CmY!4gFk=3hqNpUzbvN|=aY+?G)&?1#J$1`W+QH5>^qtZ(o&v+YHzxFv}bXLXJH_brv znW^PfjY=M)H%EdyriL$D$ga6!Lldvl_Y+TczEGR{RBUeXOUp|!Wqeg)?)*2CcMG56 zXxZ({%$WLbam@6HL%b3?4*Ja80e2ZfgME{f_>@g%IBUJ|~#vU1IK#`8a;UVD`M_WHP9kV)Ey(NZCE*=)_#l@*Cc7HO&{6>BVeZgl3) z#UA5c$Hx&{OmDle&2*Wdd9!j#;IX&orZvA%d^Xc~r|eaYEs#k{t6hB7C=-hxJcnS&g86AtsRJ?t%Bz${y5 z5w`J38zZLxLxlJahJUgf4)&(|JeDl32w?3g=+1n=B>Uh;!$Xw`O`Jk{U%M_d2@qb_CzhYcJp3s{sc z9ALipp(8CoxuBy`!g3X(WR`=P)H4HRt4o6Y<$vDE{kzd)*S3A*#W&0;>l-J3_F~(6 zn}gAO2d9&iiJ*qro<_@al`c!`#P-d5$t2!$<*2dT%(ac!RwkZ<4Du(OF8)(G>cg3^fLTVP;Srm_6gGhd2JHwwg?AU&tqv&i=N@Qa zU^!sHu*HKRL-Km)jjvPJ-r&&r@+yY6R%OZdIqUR&RlMVwlA8n~7O;pFG%M#B?!Fb6 zypTur_#wGp0a-HwrJpZ|u&Qg9d{~%PG4Cgj{fv*xo|*FZr>yY((lVjVo{5noC&5{4 z+T_k>AzC7ngHrUQZh9PyOqS~3(xz8&>Y#_$Ar4s!Mq#nQ)2?$81t%VGR_NQ%;-T}1 zL-qoLR-ok!-c(a&2@Pr44GFB;GK}_aGZ}>*DQGzPFgo%#bh4;@2-AvrBy(kfv)H-~ zt(G1KnFLyo*3hvgp?%nM0kSS@C7Fzm5A z`>&4;oO7+3m401d5-t&Cb6jz7Ba;Xt`-vrtTtzL$&m0<ZPV}QhiS=+?ygvE*;5^3IeTLH zGmphb_iVXX^CG$OO0C@7OABJnmOSLjjPR39k&2t$$3N+E!22pi*-Q`r$6}EiuCn_v zG|x{tz{27-l`D&ZSuP^sz!J5Ls~42lO6V{OH2iR0&A70`RU?VbA#*jmiog{11Eoww z%-6V54)_Q~U&v>gG@p0b2R0R!#-&;h7**mzn;k8f`Nazw7#=WmT39e~%X56?nI*_6 zaiHO{+KdkyyI7{J@u}RBxzcM|xyr5`Pf}0U6|T);7H0e>B(i0}&aJHl&8vHJQWi@y zzjr&hIphj z+#boy(${vdnQd$AoxWLSud+$h4Wj_YFCk~6Sf)*5lG(6u^}Jgwp}XJywNkKGm1#Wa z|Ow^uuHgd1ngXtnEstfkd9+sMG;(<_U=mv1A*&L=xncbh>$}OCLIM{>cWpWGRNr%x z_bVo6Cf*xm&fXF%T5A^a=J3qRbNsv7-BzvjxXi%yuej?u*kuoqCS>kp2dUT@gzP?Es>(y_En%S3{)#B}PC zZBrWqb4|a*Bru1ruB+0Uz>>azS^CzFqg;Db&DlAMXK8+LU|;9Ju_yGgq*;NOzs&)* ztOd+A430((%vKJ3%NT6$BnYfH#yHJE>C{8Xc?)HE9x?|k6nfIYsiY`%$3bw8BU?k$ z$7@Xt7n))osdlBTE;;w~?kcCP7Z!b*utZOUfl0}Mk*ka4(3%BP8U%bArTETCbL4UC zJ}AwX_i%#1*RcMtarx!P5@oMBetV(#{e;6~AFbU?yiXkTgx%H!33TpRrYJ7=kN46M z#h)|Y^Xzn#nZhrmRSQ(54ltLToo;jBt(qdwVRrFV3z(Er*p8gz_{6};!4PT0p!gw~Yg2+iMuS?~ z0+GBmB3_J}bR3jsCJgOSOBfos8BCI`l>jD$Q5k4K%l8_uw5 z>gz8$!N$DU)2xN_-vW_&2hW~!Y+UnEW`p3@Errr6H-0_SxI<-`p;WWfu|}C=2Yc2S ze7Er9zvRHZv!HsH!Bw4|B-@p>qn%5=VH`7BB}nN-{BOh%b~`bWl)d0mB2?=T5jAa*?su%=R7D{dS zD7{Hgdi_SJr$rApOxz)LuBRir=g10~lZsNu8V!#z?#@g8zQy2)(L;H@U@HO6j3epI zZ#M2d5g>hP!q0t;()$!;eod4w(!lf1pXHcY!}`!EqSo+#R9&h z18g4>c-O3A)=S{}#lUFUa3DlM=+^_rU&m${DF}Hba$ZPgV|g@dk`7~5fk)JX!(ZNs z=s61MJQVrjCmHZ?(-K9Yhz9L04!v!Y?j4#`q;Z}p<1~v$!vc{8b}nVP^{y<*|sEbv6YcUjIeZpq+DM+7DvGx(zN`RA1NM*AO1 zTN%sf2^-r6^V)wF>p9S5@?_PU34C8S{8mbEZkuwYYaRdEe#OKCyb(vISTI;xFEEo$ zP&W06P-$SBrOc+&;jELvmi6z!+Z=`{s|U<3y=u7!c(o3&aWt-9muH}qD6s0OMsBKL zfT2LxMMv&2zq%j(D5CU6UW zmj9O+B(^qPy;jkGV-mbc!OM)#8;fKBNjn~nn8s|I!h1#9C4{Qnm4o2`(3^N{%$ zgXp6N9BFTPq8KvTB| zNmmDB!x8;Yi`ZDzSd2_rd=6dBd02J&Ver+)^vlm=8oud%yd<;2vHaI3{aDkmP-Wkc z$C>LDgm)=i+;buDguCm}g1tu^>n?t%JGw2$aHrev?JfVVXa6mf`o_?3@0-H^W(UvQ z#;u{I`Yxur3x4^fvdw7VOnAp!i~1cR?GEKB(1 z65qd)sCE|Zzna#+S@4|7vxNzwVce0XCQ+8N9?NqmhL@L4y!CqTiGaGoEAqC*#*<5p ze{Ns;Q(Sh+RpaH20(J{lZ|hS0zj%s#gW~$B&7Nmf zlgN3iGxl00ED8w=j_5E|NeDgrQT2XD`g;rM?-JiauKruZXq>sQ zaf0cuWS&+58O9}t+xKXEFucwkanwiZV;%k2$(_Zqm$t_v*K z$#2KNt@ql>P=Rd~1EsUpv1r^;-HuJA!xNI^M?n5 zQy8m%=Iy+-urF9zTH8cMUrNfLYTI;1scDUpbC{R-KAhN_FU}VjvGC9K!YMnFXDyBW zCzX5g+EQ7l$_IwGRtnoy z9!_OrRlgK|YMsB-5f?|$*PY%pbW#rFMU=CwoTazHA*{$lPq-#m#v<%_$**e zS-_&?9&_S`mV!)-kb}0gf<|6ItRlnIPY?J58jru7k$!cdOvAK?=N2wfbgsFaBC|kI z@|dFJqPr)qrbsHW@_1T(xe{k|GmiKAG~eN#>s5qXae3GznKXPCfRWU2~2yEV$wcbYjiivEz(&t>FLbNUj1ugaZTKL($ zPu|MdAm%}U^JzJ&&-o(db8*t9PQIJ29b`QXSSeMTJu_k4gQ#W6$TGF$ng_zvc6JcEK^p?J~u7{!+3_XK?ZQSzlhbd2Y#> ztuF;LTOgu&st&Hp;8JBxhBhP$1W4yh_fL(9rnZ(1oJ8lHI@=U(->4cE@ ziU$l47DfUG!#U=#GWLd;J#b)|dSGLlQ5Pdq2cLoKVity~jk`BY;W4pn-wU z;*pO@YD57W|HMCy*HQ$gFdUR>Y4GWtb8WIQ-?xh^_G~)+PiM1?@tI90!_F9;Pze6K z@bFum2@ZDAT>mfJifq>`@zS=PVA@_`5O+1KTT1X`%3DRThJ&|57&+Bgnnc`p^s^dF zn9wojEK|wp2G-_J0vu5VDzb$KnDss|RNFc*Nq960ow&#oobr%s`;Mb3Z5vuWWt_Ma z6prL=`q1e9ph?8^MFW?dK_hF$IXB*po|g>HVy*`m8Kz}4Go886E~&Cu<_vRifv%n8ZajcQ*KbJ9O^xW8haIH9Gb*=b>blS0N6_NoufatU35Qv`Tp?j-Te zGH~T~6liw-@sLX{mU|^T~(7?tJINNlV1JjWrm%SzqOq>i2oD~-q z328X6<#seMFkN716ZP?`W=uY~VhRIOj3n3(7I+?5tpunk(%CUsHa z_&1qVMpNG!`qwbspZD*_@g#-AHP4;dl-myTx^X^{epC_v=Sbp~DR%9X?j~lI>O{)s zMJS6KMQ~iZ5j<^;>k@`LIu2`@Hg-7(Jme@WaMo7T5bDTyz@l)GfxS1$k<03zx5kbV z2S$qoW)+1-Hn|B0m`xV$NWAr+NPo(qryLI(wJp~1$=zsvGNFMn&@O>h=7j@$+Xn_d zfdx9Rl}`6bY+k~4LcxeJz=46K;Yh_dRYsl(ml);R&TBI&2u2MT;_X6l{*SHF|r&;W1q#q6=V94qi60gz#$&?fcfWL=Z+|kcJ4C|#m>%X;;=SowR-aK zAD4%O6XUAg;nI!8I`y3mZ!aj>98-Cq!|;QfN$A53JuZjC9aDwWMHv`e?>u0s*Km~a zI=F_BLqSQg!-2DcA*$#`=5fvFn`hF(E?!OAvel(+t-G*dbaukxz()$21-e)HN*0vt z=$})hvfk(EtAI^)_jVjJUKY^E6mmc@(z~BYYXQTTo|lX)I~>?&TwqtrDQ9!%8>2WONWad#6-oAsy_;o_p-(FMk$sZh^6g+I_N_i-G zae))#IxAKun`EK>87(3*K8#9IX>K3Ac=n#sTq@Ws!I1pZVCpl?rA#NH7^?#rrbJB; z6V_P3@=teyBQKN2%#&{%WV+~~3}(o&OHffS2xZgSz!*7;caO}$R9({x z6L?lEm?q}K_#nxkBj!rEP|J;`J!=g*eGH6++h;T{^}Nf<-;f}rwWdL|EV-3QqG9i( z3-@(3a+#R}8W`RR9Ta$MxtPI$fkA*{OU9?aaz zG2sA{)rljjOD6K9XUj^Rv~aur{{&B#{9=g|`}tcFSF@$$MDuo59Lry~F(t1;Ie4bi z%bi>2yweF@ey8$Zc}m%#jnOY7lBJ3n8W}gtG!1OE)QwbTVEpVW*t$a9?&Hf70e^hF zZJBRr$T+O$eNoVHL4FZ;^bRLho&}w$w-nY!&S={6dV>@Hng!h99$R=fP3Y8IafdBB zW1$TH#~$r~hO^2w2f1A?G&HO7FmBU+9=~bF2Idzp4$IfgXcKj-<@h1BA=l@Z`AUaqcrp4qXlb!ukX#>x;d6NtD-Hxc8^xAH^{lN7E8M>WLaK@V#i5IT4%tq(48YTfW3MH>%UA_j_?c2atk=!5|~99m}Ui3Tdm+UV_;%rU@|LU4L4w6 z{=gW$fj#y?Q*{DYu|cu30i*r~Cd~;<+yef=LXo>3C&oN+bxN-dZYt>(ZS!RJYhPHa zw1BDEfb-afx|&a=^;_!lGI-vmx2KesxlJppwy4jVz?S#C?4DZrH|B~Dm&a#eKiy^xWK5G7%s!$ zBxS_fU%;_JqiS=7rH(|E(H8cfr5x20Sal+*sui506(9K~7$^2|)-FE9rO zFgreAZ~GQE`2)XENq1`j$7BJn%@S2j3RTq~_}`ayTkd4{p251ag8TRiPUi;t=K?;Eh+ZP)W$-CuPjN*ZVZn#7z9^lGb;Fnw^-d`XVfg_HyH0do~juhn9bmac}F}=}}&7y&M$AkX!3wVWo$uh3wb=X|~W7{8NY+v0g~FC*eQdwnm7XK>DUNnr6^z*@bEHTD9F z?+3;iKQh|{8O0qqmIO>clEA~dYejd|g8r-pAEtBsSg|03VMe9_o6~~@e7|_&qB6`E zr~D9~aHcS?w{ya)ZRPHtXDqQ==+D3u^I)cC)kLwIGiM**oU>u#5>E#kC#|=Ri~U%c zeICgB7O*v5kni5W5%Iv(?L*VYr;PF&6zwIHY%JJ|O#X2$c`$j20!NDhw|@Cj*PCmX zOyF)cvNUq3a$Uh*oWN=uF!yRFOMwG>e*x!=%}jg_*?TVcayPif&Fp=-a$faE@%JvB zt{;~BEnxK%V2!`P8n3_}|AE!lfrT%EAvZ*_cqPw2&Xq?TI9YdXWb@)=Usaa9ygws> zJ-K0J@`VK{8&-Y|&1mJE&=D!Kq-(;mR}1@Z%04*2`#y|4{sO0Z*2KjdxK1tLzG}eb z+C6c*)-2hTX<@93jilraF0gbTU4QDDKXC6d;F`UGYxV<{T~bru8m4u+3rYnrsu!>{DzIk?b8K%~<6gkh zmcV^~fkliJXO0Eu4hNZAA1rz{OZQFY&i=imZQ9b7r21AR#S^%<8E{w|vR8g! z3m0GsN??t2;AnAR_XuE~bb-gvz>l$@H%ij~vE;Not>wOxTKR9V)O=w5H+A>K>8B6P z^qyEFz$$OR;H|`S1i}V=T*D5wqEV%kDMj{ttW3aqqod!28BwuUO|^=>svm z333si1^x*OWp3sOt7h*`VB1;55<8h|ZzK0A2d-wD6W_k?yRnIN)dH@C4qOWyPBhQq zZl1%n(0h!ft)wdw+rV$_D8vY8=xibH8ZdE>U1La$v7yU>0&<+*)Rkz``I> zz|dPgZ`)*7UWe@dr}KO#9Mbez(^#<2)o6`tz|Ob}tV=po`y9Ad1aMlN+0}D_qqpH7 zNACoVD{FEcWsZC?Ka$qKrlHNoH|vO34|9J5XG~PQ?v+ig(~J*DZ}v~?C^OH~SQ`D| zFmJchUX4@SS1<6Tp5gm_hVSZwJ&AibV-8F_=$y9L$wB_&@m%NG>ITd{4(1*LoL4{a zaD;G($8uzIFFdt{-E0EObuKPN-AhWb9E!0V69PC^8ZgU!U@$V^*gk>x?gO5NH7A-J zxGrDd`}e`beFCH51}2dRhR7C%5-&yu1qR{{*fxf6iPu!_iT2_Dav$%v|=egq5y2M?89VTQo57buqO(;Fuh6 z?sHVUz`s{3FAE%aeaBdWE0sAT=FPEa)}J!pV)o9l#r<>ooDW{1sgCaFo-@lnaWUt*-xVDV-^0JR5t{yj7wRpHrZQwZKuxm*)7i-s=lM`a!zhyO& z;aK^Ar)vSnrI>3s&fL$gees|7L59Kuw?Ah+=d3L3WztxXFw=o!X2JDmHRldSY6w~J zGH~u`J|PoyY~cszhi$d8|M%S3|KY|z2L5*!&c7+(oAP&0#-3w~EIYYAFtAP)d!bk< zSs|~S$y9Ifbi2bzt9|=VPhl}tSmws?k5lFB?HUD+QB7bsR?w zo-@>6O%K>PCxL}|1N*83?&fv(+vjm)e7LWo{c)PsgERv+djmGB3k%E}b{E^ev{=Ai z8opI>uT7T|-W25=LxM2c|{kf04Su)4=TzK_?f&b#`J;wier~c*L7qB-V zfMu43CCO{`@HWz@a`<&m?*<_`=3-%x@Ads^|Q#7(+_eDdJ-A; z#GLt%z%L&Cyu|N$_}Yy(LwD9ZU=cmQzCD4vuYsep;OB4MBhGuSsjOy8`|vX*;bT(5 zk>mm<-dW6(^sY}n$ELWRooimAavyjP;P>~Hw| z;Q{yc4cw~d`0jt;ePQt0;|c&PNPtVK{$ z(Niu}j{q@cHWm|)28Dx-EDSmwHXXmZbFL1HjMIG7X^q!f^Qx6ovR@;31_4M>6y{Kumj%n(BvF0cCm@SntNLa+o#3iGlvETqxHv9V2=nVqW&8$3p z21hE}-ph`!qA zwWrvbWo`3r)fdy&Me=>WpS=Xa-*(B=@EiFNluAswxOXn1E@vc}V<=kVq=~kEax&uxN zdrfY5EatPVaCxn>O|4N|Z)evQZ|S|?KAo06aDnr=!4V#3pABUkzn{&@E)hIBIk4hk zt5{gZ%ACz-Es~#44>Y^Eq~nrM#(_pgu8ai@S6`GYV6f8FSg`HJj~WJZtvQy@C4)l@ zOQuF1QD(Jp_T)Tj`P{`)MI^LjrJH!@JnfX=w+?|Jf&L3`n@rEWm8m}6R_svQ6kEQH z3z?WC92E95e7>D=;E&CsP-RB03?)_8pN-E$f)pB0I;H3zDKJqt%W)Eo*1DK{WbqWI z!hf1MiF3So^3u;F9+S#-NnEq>l0x?ytGXjBlee|)St=RRRg*TW@?+4XlLhBEdki#} zi@fnHn0KdbuFlmjE#CeS3)dJW-pQ1ZE3f-`bh7V)8w(m(rDkkyV6iB3U|tXIc$&-@=$KZND9lWLib0xbwQEF7$DHX=3fCe)CHD3%;9r;;(P)d@8W# zz{#f*oO>K{9saEno1o0-e(DDUqn3r{(!d9e?$cuoSeJ?<>-e-uxQeYjCY#`RGodpi zV4=#9tj}(OM@>TRd~;o$)KloT{)lWMZ?Ulbc6Zwgzt|@qH!Qq9eKv0`8eCR=&J}M`xAYJIOk|e21ZY zmL`Y7g2vmA`7r+x4vd+K>{O_=!yc?;?AWysh;9dQi3Wcc|GIYHcfwK z!an2dU+$8Y%rjF<9=2YdtCzU<#Zmn*_WrAD^OELIh-GC;ux8wF%vVmi;l#6&t9G3n6n03Rf1&p#Q zoSNJmxilvyP2sg$+N{6p)PsTpOot@-I2t&k3-^s-S_F*knNyEU^;1JirvNTv$+hQ7W1 zT?bYRzjxZkKZINH6dw_vW!>MV3L6@J(8}YpxdJJkF8chGy zBy648#N?-N%5{#Arr4d?tVa(l3O(h)6Md1Z_sG?$3K>egUQ3&T0xMcK8uU$1PZZxh z4Ld74lNM__v9dj|<2vEV z_{=0-^rx13shNVQ9CIXdNCRhVZj@)q~f)YaB zi&8Tf*Jz|TFv@qhXo?hiy@>MW@oNnHS9(+-M4+vpMMJ_-VA8BBLboRAXsMhIxz4mx zf+=|QXDzi!Uk>r+>2St3{Jb8)%+{A5kt|hxTU11zGq${Z&n1e z^sNqk^BG6ZZ0It*a6jTq?4Avcyk{2F`}}cv^Y7U%PJN4o9MUTs81Jw%cR=> z-MDLeNuuHUtP8hQ{Mpv+f6)5o z%CB5r69oo2hV#nzVnl3JjyCBOow}b;(rox(_KF8ub2xTpKa*_`=kO}cdQ_Gb}0jwg8Cm1 zrdRCFKMppv-fy_KWTSbZ0{_hgY7d)uXbx>D4ET%*Y~9Q zf(@Pi2B#;_zY@$KRlpG(!6fKB^P1+yt0D*PTHlXIwAsHkTiN)w!hKd7>8>|Ba2VE6`x;0lk%5^ z0{?Yb9CxO7*q(U+>Q9L&9h422GOd5dhGXHn>D`Vi8W6%9-qoUUv*)V;}sS%Ja(^PvaK&W1G%JTscpPp~~N zp5lC*|mm>X?B zjh7|uVzqkF@ZfWQ-v+7Gjtrn>FANO~9lIHSG)!K+o6W{jc>}}c868Xv44(fCTB8rN z%*tTl^=Ubj%w@7d?;z)9!|J8NEjCgUY)*dG^UcuG3Yc@od7Uz+PERMW5z}B&=)$>EMRYJ4oi$=Z*21Wq}-#7M5Z|rYmon(w* z@U~%KopFHiLIbZwQ~Xae&)?gW7tGZ-#nHvl;O)_tcAzC`MJG?qQELw_;~B0Jl8*mm zb~L{E)Ft(CyU3D8QJx(QpY?p^^I8g>5^)RWA`c{AxWiWr(W{r-q_r^+9FNp zkm4WiAFGaK-(br-aeB|4w!9e@_Mf}AA8E^8;dbSIR!r<-GFr|c-`$PlJjRxTr z=Q!`I{@uX9v7v)m<5Y@4TapHwk3w769f$0f3i2n+g;>PJUYHAQ5EL(HJaM}*?#odV zZg_~gSO}$ zEx{Z%o*P&^HLiO(gnMsj@v7)7tqfm!#=#-CrDE;%s-@vop)4*9Z7CLQb*61z68=k5 zSa@Z)mPDw^|Cq*aq*c${En2}S6>ueXi~pY|L0TTGmPKiaNO{X@u3xFtQtqptH&wjq zU|aCzwwwzaSiZL9WnB2l)y`3><`^A1BlQB0s>#6vH{}lL3;g9){~1{@gYC%;W?h9S zT>~cfMO)t%m|n1H{P#Bany^5#-Gnu78p1vsx9FYVTm8Ag_JwJ{4%SbV?Ii*1nHg=V zds>4%qJt;2Y@gBM&9SF+FU!ukw|APhdA|tuc@doK;FX-vw$HRJxq7icnm?G^uJo%kPD$rR%lCq?;} zs@n>DPD;9CE0_47i8#tMw7sZ7QWEv_fle10rv$I)WX!Q!-|N%w((;{#@o34IqAwU>3Y zt=`p=R?*Gfa=X;m)!U#oXv4pjpar+RR@~mr8u3x)d{RPNQUqI?MO&&s%wgYzL$NV+ zt!(wB59)Q>ws=Z^PGNCPXmK%!u->32A=vb7i&x=Hf5B^BC#{X$a~*vbEcNtV6}@YZ zEwrF-;!Qv6YA|b31@Wb>=c9XV66#owk-4lGR zUhU-->{S}KQ+;l8?_u@c!RqrP%PP6k4zy03%C_@DTZYBWn``e1JZ<|u%iAeC=|O3dx?oRT z$W<`J@LMAfA?0FS8elC zV{==;;>MAhHF3LRua}UHOqkfipD~VWS$3RX`|`cl!w*@Bo9;Z7@7x?cBQZ@S%k4_r zl5cHy-d(3%nu-FWZc;}DY+VcoOXZmU zA;?Mf;l7nk0(*Bzo+R@GjjfHY?YC!>U`?9Fvq@RsWN`C>Xm; zp!H_J_K*8Hl4jU`O?!58PUzQ=?}_Wmgz{(bMO*Z>c1XT{Y4AdDsZXZJoR=c6)n-LB z%QYCa=*C5*c;{CwNX{*|Z`hx2BbHILpPP(uP1eMHmJ6BsudL4 zw;kIyA*mrIWdRHO|0g_R6Bk=dTpl857~#)-{glh~nCX6BM4uczKle)Hqf1M)#FKRN zC6=vQay_x&udQP% znZaIK!Cvjre#?wC&zJ3c-M2#7>)ZD_6#7Lwdwl<*@%!2{=dZoet-0@-g5oUuPy%P zQx9HPt7<0wuflf5!Ii7De0FHH)VJ+Zf0d!YE^+^%V!B#`1^YQ^t9<5zjoj?-q%8^` z${D512vBfp5#d#H`EDR%p2vTGEA!1fm6MZHZ}z&JK&)=yGn)gvht>W znTczkoCs9)%DXG1+~$*aXV=kJw~l)ET#R!&A!O*K60u>fOP0{hjm^i-Nth%ZbCBDd z%-(xSJM6-aHI=OsMD@dVoOpAnb%KJK+HHZP!>$u#9pji69%?%!s+GBJNn&xd|FU~C z4SWw>`EqS-`k5xL>SenBW^H?RcCTZ*+`AZy=4)%)yLT8QIs@W{DD&UDk2 z%Fnla3bpTF{?pdC%)3A(ag+0Pz5jK8MZX?b=QBTl$Eibkd(UMa6ED@tKEa`$rXpd^ zcb%qcMpdeEMo9Z9d8+nL*(w&*7vPi>-4$@CL+QkZ8;iQD9v)Rvu2XFC>22Umi%814 zaZp(*+2Gp@i7BbmBGYHrev_Q!x-2X*buOpl;p`2WliG53B+i^Ix96*vjM{&L%To%^ zyEv=OJd>I1H|t_9%U#P0xyvuO6;*wn>&O-8a?w#IGi09moeE(;z3p!vvddQs6nQB2 zHyCkh_D@>GrMZ2@r0%8D?G|(EU5og#<+RDIoZ`tAHwreNHM@K#rPp*vN)e~Y*^bUB zL5WUZr-UqPnRzuNIJV-Us&rV;NtM*Ori+v(PVwOE%F}e>Q9AcebZz(LQx@U9X8)!p z_t`3KyOGrEC?>$5_>+ahh=GAYhk=2CL4q-av7L{HN0^sal9x+Fh*w&KS6YIDk57n? zPfCbiNRm&OUr>Z!NK!;tL|RltMp9H%L{eHzN>)lrOhSxLkXKcTQ%OuzSxQt{T3Sn1 zMom)OR90G7oI_4cKwXvHOpQZDQC3@7Mngr+NJYj>Q_9FhP*z$+M$|xAMoCLnSxrX6 zL{3RgQAb5tSzA?EM@v;zSzAk0TUT3KO;gQCRoPfa#Yk7%N>|%bQ`Oi|*V)XT}t z)zQY=#oEu)*4NWC$lEs5-7?VAFwH}wJVG-))UqPVEX&WJGQ}u2o5#Y^P0!3%S>D~% z+S%RK)z8-5+BwwRDZtn>-N(hr$IIE%)g#2$Dcs9FEWjlu*e4>`!`(AHIxHl}HzYN} zDnqDyQJJwI%=>zN+Oa6T?D--wYb7@Im@Sy4aAu9gu^IXc9?O0GA@Spf z{?#jLGur%Dw0mrtGg1JaxF} z?L2a2@0rJE&tATGfqKJ=KVVB4knb1C1ci)zt*VJnTko;Ea$ZtFbx^z@Z^ z+Y0ZMe411)em}puD)5zY=K1Z;tHtJ@x{~9+E@E==)T*yLOV;`}7TldT_05&-Za4q2 zxpEh4{Lt8Tv6xft4)>|urSA@`pM1T0f>r%rCdo(59uE(*D5hO#;FNWdbeUkeWrD-h zSe3w3UNa6hH|frA6WvsDugq+d*mfli zPwT~lRFNW)OX(pcEUgykVIHSHzG-XXv zq1(1Ka_K200$b1PNj!8Ah*O#5DL?be^Q^_ZRT|l=TN&9F>6R`O>YR9MS>BRevZ|WZ z`mRUw3j@l+L&_sJUe#ZF+c7o&;Ig9FY1i3`Pd=XV?C92chhHc5-o7cm-8TOAw~3j{ z1pfRq(BWZVi`eupSz@lt%%%%$?*vkW#rh^b6z&SNkmzm>T=d8!d&Q-UzWP8*uY@xq zk2;M@3mc(6;BwQoCJAasuBz+x$QAz<@<7AZd;I@`Hs(TJLv1 z{3Y>y2j|Z>h2|>Z5^2uPzwVxI|2Mr`{#i@KWPyqA)K+Q7<`pqdsgQhAdPDG1_%>ly zW~<(uUtL~8lNhF|o;_#HCDZ-Z<><63M;~1e%l=z&->r7lWcqxO44yg1z2)B$PPWU* z3%|~3o!G7LqSdfPF(c>3;R#dp>!!YD`duTj|3=5@iEpE)JoovsZk|P##nNCeVeKDF zYFfQQ&fN40*yrT6kmpQ-`Kr4jA6dn2*Io>HdhulLECHF#OP*TzU=V8WUbZF3y$+9MxOaKV~0Tli*u63Mb9whb*Cq0yF9q#>3hJ_ z?~kORhL)GQU(Tm#?UtDpzFLnbY}&C~&C;MJ{-5`xvY?}^Z>Ml+3muvJ_M4^ujVCHz zp?}W47s_(Gyz}Dp34`FqjT;$>k6)`S=~FtAST|8+7XsB~?J zk5q)X^V6fcT8=s_XHRz@abVjdyQax|q4@M4QxDGnda3#Q%Jau&xo`h>hj-?2tqVr> zQob{6h5c)fXvb|#J#1_1r+Q0=GvTDDu;RQGbcQ&e3`&ur?OJRYSx1H zJ<5(}_dcI4zb9gSjp>^*>AqMy^{P&{ioX=6%}=X@xJeEd&X@IWS#b7!`ecQuWm)VT!A}o` zPEKIalW=5Tc1x+&y@OfSB7$e}i=%pBsUr6bUA@;us|Vywb&Q`UYJcP}x10U3X5mc= z>-riaQ=FunPd!ld7eCpP7Pn@H4WpK+6esh7Go{J=GgH(Wx^842=#02gbCA=m&2v)u z=NAV94@)Y}Jh+D076px4VjY?@n;$SMb~w!C ze!#%pa)4Rof}_|rhj&HNjV!VY7+80m&1Dx+VCj5u)cVT@R=sKI+F4g*13RC~>9;0^ z3$H!D-s+gVoWx|-Ga(PRs4?AjP|GW<+TfgKv1vkYN_gSC&oL6Om^2jH^-i*xuDJA1 z<*HrSdzU9(aowd$HchFTTGW;JPk%;woV4>3CGlMsTs6NLnW(G}4If}elH(Yeh1i3b?KFAYo;KXhq zz{Fj!fI+6=yV*U*jwpFa@4Yi~ZpFDyj{f>-dZlBWf#xiEC7WC9$B#Z<@^}jSu>&X9 z@s>Z7m)9#a$hkF3|7_9DLk?{2*IJ&aByIZ5v&%%iIapor_Wh8>lW$#fWmcGm2ZdP)zJ=^^5%kr=W#&>TR*zQ;|39Z=0{6?mh)uo;x zX!!xQo_`xyohqE(%)Zg08zsmUeIw~j)(s~?r+@|~p9gn4UbI@X6$(dP`V<=Zi%GY1 z$~>NLzYiZc!kWEQ-pPSQwIzn*xKr0bfx0EO_k|^e%LT732~s*(nexCmDS%xl-O_rC z^TvzG1_vXyOjP>5OzfSpU*5!!_`;A)6hEn%iT(O z9^{EkU`=1ZnI=*9a5>vtfwHIJ+*<{>(ifB!h?lo1)VF;|@i5HTvPCW~Ad>&HaNZL$ z1(!fkQ=d{7oqq-kVpuL#s2*-8xm0lPv#r=9!IXl=Yg-!29yES>T=_KA;6Jm9-$&7a z2f}WLSpz9ddQfvQyp%_PQOBTn{f?gP5v8IIjIs$ViQ737x3gt!V4Ka*H){gt ztOs1r%DERl;B<3f%s z=R54tbHam}DIxvB4~FaCYXli;{ib(^v@_mb&i2rv@8NRJ^zyQ_1hx!~y6q3R(;hHZ zxiL0e;NH)`< zfrTA_{&sghY8Gjnaw-ZC;OR~R=6>2i|F2ZV{%w} z>7Ena6>2T}GD^>K^zvNj+4!R5+Vh%o>MimO?1wyh4n_9(L~`gpVBlq_y>O%FhB{;D z%Q-u5&SD8*beq8T@H*$C2<|=qKD5PHPD@+BHgf~p+zp&_66zusA}}$Z=5-KQ$_1K$yTm}TA>9D>n*3~STUZR zvB=13v2oSx3mxf#ADEY1;C(rP_s9#rTMT^nCbW3Y;B8G{W)ff!{J^k%=c3T|niJ1^ zm>n3T0@z}9vfX^i{qzCf{R7-t1#Hm)Z1-2TJ=)2+Ux3T~0i#F(+nxZvJqKonD9Q2{ zYi1hDNQOuSD8$H%sMRU@Y-D9AO>ycx;r_39lFrTMOrvCW)`vQqLj`VJUXaePa;gMd zpg|X>Tl{&pS*8KZCIQXL4$O)P&58xhMh(^O-x+Uxo3lZqbeBYnc~|%L%x)foDZ(2V zwq&j5-mqxft3`%UrF(z%oX%W)!GgKof%otazFQlnIC=Hha&a0fFkINdu)A_``8S3j z$=N~pV1DUjHKPeJCpT?4w@IgNvfxc+V!so_p^?)hj0n=GAW{V9g=R`GM7AskrRkE}F-zvS?t72x%_B%d%Tjp#{uEhqk zYMd=ww`ehLi<)g_#jw?K%avW5MHyK3x%b=?Sn^I`$vuN5hiCFOU0~q-z@R79eO8n4 zI@jE!4z_sZrO7ChuVr_(asJm3dKG;3=`8om|y=qfv?e)Icr7I0_HDEm8Py>YuUh+7s35} zJLjwkY>@>C3nX)m9GLV9V*V{;HvF*CqG6N50Vc=qjB9Uf^8a3Q?q=&n_BDGrmtHhy zG&{XWW7pfI5g`%W(kdd1ZGVDV`!z6T3<{jHY-{PXU)Ie}Zlfq{1c zgU*6Q;Svm?uQ#95Y+*HEx_e^k_6L0b7VtG+U`g8GT9VC@$iNagf$iyZ&c~VDsSg-a z4Awunz#1*!y)#9>C`x8eh*TbPMavIuWj6`K2^-X}1!^4AIa{=Vy~$|XQfBKCwyXf| zbp_mq6nNG<98%`tS-XKV%7Mvi0`nh7W(x@-(x%9`tA=?dE?=5xQ?)CDk7q@LRA$BEj_!wQH6=z9q<7w8=9s#J zRW646fWe{l6HcwlIdnkbgm(e683T)<1Jn61iRIb5%xrQU)JryXmqbc1iZ945TQ(=) z)NHM&J(sx_pM9}A*yiXSkJ;Nejz(Ocby?=@jvcM>32W{$Y;|76>v-yD;vC+~7g!h_ z7tb16_^8XC) zlYn(eORYbLccH|qk|{s(*ss}las33a%@>b8LMNCNjw%e<`$`E#cW?=>^EN}BHd(duiX zQ2vA)lWiva`k3Jo!0b}s!xe4Y!8lQ?EmN*nP}4atD}h^i&ZU(++*)%_sjuO57sxy7 zwo4_Td6{>v$=OZ1*(Fl5s>KC1AFZBx?#8U^&yNJ~^d@k$ZL^%>{de+y-lGoEt>H76 zD<*JX7N|MIc`W8%&)(Jvj4U4*0~6Ry8RjwvFzOmGOBSSa2QbMeFp4-Zbw22Qn9jZb z0r%r=t`P=eaRJs#e0bJ2%w1@Z*ZPL95SiJ7;T4J-NpIWV_`P6RE{6r>FEjT(a?3V z?I4r7fO_viA1$%YCXtnLJdf1waIHPSx#+^P`!Bh~*PYtF;gErD?t0%`&A7u0BCB_* zGaG&oF&1DlNnmeH;4mq;eMNl{b3k{*>(;RFJ?6JoJ9j@Z(psY_b@ighJ@pF=q6yb7 z$1$31U|;IMdsAS^(Hp!=7BJtv#xfy+_r(Ohi61zY1TgzdczgK))4BB82tTHlfU;Lv z-1la%cF6dz`XcQaz;?WV`-w+hL){5xn;Sb^MQ=EIckK1m+OSUt?dl|c(xyyo9LDIDE-j-0`5owCS!}s7ujx`u4Q%* zV5_*mV)Ad7!vpqK1-ALK*e_4uUF^Vo!~EJ#>DrLrt-EK}oNcc?dwo{=x~+z@YW%sj zFc_?MuAD9ZfmytnS)qZ6&)_Ym!dn9d_6Y`GZZ`1Ubl|dgU{Fk8Zgb#uy2X2Q0`HXz ztR@N^x6U!0vzWrPfia6=+iTwE(FLI?pG3SSuy_Y#ul!p2uHb&Q#I6Mj^)(4vfnl=S z)G~quZTK#m1v;E;eId@~@k^rDeM+3saV5ceW)Hn1>@l8~)O~pNJN(}MVQI1e+tY2F z%jcZZdCRDLA?ND1Tj#%Rl)KH+6u`Ck0*hh+qu##{%&iM}mm07c3vk?O;JvbeT}*)C zuneQU=hj1djPe(jSl(vXKDWAJ+h)}dOYCf0w`MY&p8j?3>l%v>97`YYOcda#3}7~H zU=$5tWDs0VE8*N5#mmIQ#-pLIfLXzJwpDIc zs<*G0a?FB?kB^pgPg0FZQHy-^GGg`QSv?XzuVl+C(DTeOD=gaLEvy$f%`8%CX{N}G z_&qKwAFF1|DjMuOBPbN6GeI%6_CtrD^D%C*nlF25yW9Ea?WlNhtFk-%%srdx^BtCx zPfRm7)Y2)~?fyo5y>ZHk9>vY7r)T;e;o5#m$muwTxJki-e+*5|?BcRZcS~GK+sN7> zWiQuwnT?}aF6Z8c&dvu7P0bwKMb;}mJUq|VpK^4ZwYzhk+n6_zT_WI^+Or5N|?ZQEB#RVNooQrn^G<9?57cQS_Y7%&ugWu-n z(m5_&QKveWEXZ1POe1T~M$VNxelTgT-Ezo_W1UHhr;cKS#Uvg1j*dijiHHP7J=+(X zxb=1|yW-rl{o5rYe!Fjv9`iUgX!dMKe&qOhy34i-pB{rl9cOrqS{eU6n`UZrp^-)W zO9^9-?HdMV0h>uDlBeVx={&;5ld*wuQqCSpmS$;|i>yLYIt7f)0!bej*)>of>?R0|SGtZ(97Kc35PsgH5wBggJ$$2kbculvRSiszP(olu7Wv)XPw^Vk?J~yfSH;ayU-fvjRr_uIi)$ffK-x4|0 z5-UEjY`@c$ouak!%Hh7rA8)+&HMLQEBbpe;sU}zcWqoV6!I2+LXCJ7rar}K$cVL67 z*uO7JnirNX6Sy*=P4C-;b|;0^B0d6)yfYX&+#L_G*3VGjh6BoShYc4na;|7x zoZvi>A=I$_N(YO=ECnXP0zGL4t^gOO=e^|?c~UnNS{;sDW?(bP;Lb8=ma;n7X?4)s zXooSEKt;G@Qb`i;Op9hA4FM*nEgb@8f%gt|DcQc1b=3-7;B`c)ZR*?D9*+b`?(#ED z)3$D4t$nF{^Vb#?wjXbg@>L#SmiTvl)31X{62~=;8J=S7(mC?JzOS=KDlc(n!j4BA zj@^yiY5x+K)t@YqUXsux=c72WJVTk^E1^k<#nCGG$pf~&LvFGL59Tu*3~+e$FwDwr z;!gD*CWp%l*czUxHE{7fDqh^cYs!__c#Si!K-eIZ*!fphxDBRrW3%p67u zjm94yNq6p;E;Oe=`I&5M<(X!=FLIKGU8(1j?jDYaIAO%1AaIb~x`Xj&L;;iS9Dlu_ zOF<{MJh9t-LA5UZ!V%qf-@C$Cm?aMUU^Te%VHx8jM!yt+W~r<~od?^BF z_L?#ZWvGN$Nq7C@aM;5oC^(Z%xXOX^tHUDJz!_~8B`cg#Ha>TFz@fwTPCzi%!HFyA zN*f1@L9<8AyZMc*2N)(TmU(KRz`$Fvs2(qejzIeDyjf=Lklg^k`cen8e9bv4Oenu1lNoibR9&%}3qtmv$TP zIqAz*=2M^jaZ~hxtHuY{H1aND;1qRpmG^yvJx zW+mHs1FBS*cIJl#N?*Rg%qr#3z@U&`DkpeS+B|WRZ$Z*@jyo(}PL8{y3zCE{w`#LX zU0`5O*e=eZ-6g!QL0M0wapKoW^FsDqnDS=sbnTeH^%J|iLgare@HCikkkjkKN$ph5 zDN96{wRTR4*HfJ#e7u4+Jo+J5_>Dt-DGbfJg`OR;_tqaQEOq2rr2NpsfvMGO#vz3X zQ+ZD{aO4&T3$K*m#Gn)IT<2b}SzHQZz<}Qu~@(m@9OO{u;D3FNp(|Iw$BjE%#c$%F6VjYkT+hBTed`jr;v=** z*+xRX=DCp9t_DT#52u6T-!QfP-t~Ufl$G-T?!4)Wx{}0gc6g>l(AjwZAMfgB96ZF4 zqF^F*L4m7(gSU#?1tzr_i#G0Yn`WG@z`-!}5Sxz%YYvwqqkzMq=thMG4wi*CoL2nJ ze3QbYx$j4-tbf+FTYe5qEFT&Usk$>R+Uk&ADH~9*WA@!-mbp&(zjfQ9R9ILV_AqmN zU=(jSpqwqzz}nMr(Pu^?FSGgK8Ls(4x@-O&22moE|9cKD`5 z&NWs;QQ0d84)Cn84*n_Q!pMG;sgX@lP%!<;!_cOPWyYZ!+Lzx=U^~s$!t7Acq0iOy zrl~>EgQ3x;LMDAh1Mi8(4bJU0D;jtzn0Nl3lf}I~IIw{=pfCO-lgEs<-LsXdF0++K z>sB^z(o$v>Sg@J#Lm&SQW>13_nHej(7BvXoU^2eYa_;vYAA`Lwx#UbOnoR$E zgcClnWT#BleX*WX(Ieu6q_PI@&P&2l84FHjTS;@Y**Aq0v)O)+2yXlVOeKiiYVni#S#^*nT(>bl|i`!^xNv zjEo6Pr?gmOGv*{NW>70&6WHAmc%ZGMp@qppmtjH!yGG+jj&&?68rTFFcvdh+ukP)< z;i_(J)4(+$h=GM+#Wc1AK8+V^N(5x@b~V+0=7`j3;8S2!ejuw;VIjo9$X=+#5#T>T zV68y|ON>OTow0=41i6E60^Tp;a(pfLcB6z6W7C2v_e~mFN)q$8{%BbCYsreJeIcu~ zX78T)jJf&j6Xu_qJij?jjV*a90-H@g=-=nu8ePy@pU5P^v8<;sJKCzzLOvT{G@kMUs7R1jqhI9LD3bcPP= zOP$s|f{aWYHaRzruo+BcPheDRnDj=X!LnE>`{W*Jh87Qt#j$VNa(}eNt~qbuB*1ur zfz5;A*8xVa6J~`!#7a)IN`2^i%rR#|!Gw^t3|t(Cm;(4XH5$LJnf6$s!JvXA<^Zee z2Lb-2f^ka0PSnqBfZmmn032&Cod#Siv6q z??f|C08^Gld!|Dp_licFv#d-JnhZ-=WI9f9akK=TV9T3xwNttU8Z06a=NTt# zkz27KKf_$3o28zalRcm>$T##ohYXi$1G@(UhXO-=vyH}tme>g_cMO}F8kwF&9@402 zy0v?{rbD+&;0fT6PNcxTj zUtX4fnH?=bi45`;E%qG^%l0;e&QwsyU~1%O;7(w4_&fE^iCFm^+9e5!js;DU9E`;u zIcjD_v1u~MUSRfIaVhVBYvGE>yah_$ol$QlFh~frh&Qdhy{PSB?ycNEalx965jqWT zG8i}v?yt;kP+QRwdxN#|WYgjXCXEjhsvl0U-#u~l0z|lxqA)pNajVhGi*uOS!UGgat{z?(xr=QQ*uzM?WXC{-XLZi-w+j)P_gfD2#<7GQ^ zwMkH;Noqyok_-dwrY=31|c$mmo-k<8bOX*L%OvwwCxUa|bmghmYm*%F8J6NPLtat*S( z8F+p$%LMHyzui`%(JK7!!Fii!=UEL!gI)QUJmxgyDYGqPko9PZ`N8IJ;p%58pQ0o2 zPQn+zR6Xf2+}W2nX-O61HV?ZW4>M-WwhUHUDkHjUm#^t8>99qTPkmop%ri}m2+-RR z#Pp)!DVtY!&z7@ni)_Rmq}ru0@+|0=U&jBWqv7G)qfaXu9`Bgu9J}bi%Cw77wg(NT z9#q`^B%$HLsRluY1Gg7-I2X#~o?vzS!6ar8`N?29;|x!!6U<%?>FFKJ>3KIa3ff9C zmR%5RSN|+#n%5-bC!>DLh&`J_0q6vwAuyy$=? z-;umq%l`Ip%g9(ydv!oE%kW1(Lt%S^)?x-h_L2;Kn+hiJ9Sv_jbcB>nV5ne_Yhavb z+h=yFLAt@^QX%`xsrS^q+t2mdOnsZUN2h^bgW+nT?30WJ_67zH`>f(lN6!y_E*?#v zxo+ROY;j`S#MKhlG)=yn zDdh`Hz8V(M4_Xw=nWtHVt7hzyl$iDC*gub+k|2SA7_o{V@fVo_2O8w3F&x>)(@Gc1$SoYSBHl0QGaS+^jP53u6y@1XLFQ$^eZT7cl#+7EX^nI??f)@xduv8Vc+6EY>HoYN-gW^6-oP?!~>ds&V_#6raVbSmWLr z$-HFeV^-MF)V1b>tUc2`OD1KD>w-HPWdg$OI=3#&-1YJB%buLZg+CLu(mC1^7@s9H z@N1-fed=#smTW!k*-{tL!-{i0WSqL_RQ%BC^09x3y4NRle3Cnu=g^+Ffld5h6FVzg zvPD~{2E)gWsKhN|QUWb;KUymc+GajvyYR4WXC=$e8|Odi{@^N@z!%W`>27jew41qJ zqfkQwH$$U~14|5trv2=Ag%`#L=j0UUG&yny@>X`v{%>pcpwG>kyYk2j(d}9_SM;wg zyBPQXtIX1!`W{t!0<$g^FfOdDk~%Tjrc+zG;pwu+JnX%UGB>I+<)xN7=V?y=#BXqu z+k&xeLH`#0XWR#h7ORQ96=}_0t=K zJQ@TIT3jBq8U!%OUJz7VF;T|gjLZZ%wGS8Eqf!Lgncg=tMTQ58#tJ=5z9imZQqA)# z@RUB=ztY*o^YtU%E~yumo$%{?#jl6n+-!^Lq*gGlSl%e`VDj4cbwU%W)cAkfq%b}c z>*Mf{-6qa=VY^u4gN94y=Q%2VFMYyr$npG6elfL0)xnG$HdnSZo@P5Y*CPMk0mGOb z6BKvk397ce{?;wNDROg6=>r1?21X`khYy9*@@}tBKGw(h_?W@$Z81JtAsPw~oSE79 zBV05NJe;8uB`al7U~nM0k;#HRWwdMQ<9%O+ zTDv3loffQURch&;u%ob(fzjv)dz_qM(t(D9O|5da6)cY>b=dkPY?Jt!4<@I76gQl> z|G~^ZtY^gatqKa%4&7|!7Bb4%ao}MS8?UUP!)1ZkSsp!78y#n}y=7|271yfs=Weae~1Dt8Z^^t#(gmIB;;f`;7yc#y2L|xa8jw z+4Oyy??bb^<0edxmD$`oxL6DxoMbhOwtV_0=$zThjLGLr0#g(iSXl!U95@9++#T3u zC5*cRI2{xmS%hK)9Jx&cA2_fI&S79=Q@GB`w3q?i_q zdQ3RLe92S%ZYh7rcLt_uQPc01-tfE-x}9+SJ6dNLi7|o-Xrex4Iq5)O)c~{kYzn)L948-?1*AC%gY9 z6Nhd+1JhzTk*GGYv>AbJVlS%q6c6ZYL^&Cz@ZLV&rSKGVAkc{R3O}o;ABQXJg|=xs2Ni%`74T ziR_zMLKilRH7Fe7aDTIB^ET<235;xw6Piv-g+Hlr5t|ylj#<+0i-f+o{W4b87rC;Q zzb1JgGLnNMd$@0H@aTE>5Y(qU#nMU^AQ0=;+cUu4U;a7W0uSIfg@0%%n-^p6N6V zlY=@LC%I-`5pvmiWof(vg9LxjadA!2HwJ!3CLIzxy!3d(N~1{|4~y$Cvl%^gU{^W7 z$l250eC1$UMfxh?j~T*yE?;S5Q)ZaQ?RLOOu;T!;yGOH{MPhdX%VN=59=2~ef?Y;0 z9HPY!IPkUvFe_C2Q)CgIz`)e=je*smVbX`m41XjHG#ACXTX6-v_l(sbDCZ4thE_`ek8s2Fv$hc9d;%9Cu zJJDsqWZ@r6F4-js3-&Pn6WSHgul4iz*0?jrC64@P)8;%hA+O}I^!*JSyuFM(0RaiT zQWITuVjgxyTv*h6!XS)8Z^{|IyhY3^H`Le{t`uCD^MGgZ3Ra_?oM*P~a^keS6BF6u zqOW*2+heiyB*FKS#FY*tbh2ps91{5^p>E{n6FjFUg~`iM@4VPze&r97WPD)0?P70x#^Pxo z4{GtoIx=znDH1z#r-A)NSl%}VaV|xt_g%7NIr8tcaZXp`lxH`OoV0T@{9dzBHf#FL(-$Oo zL@%FlbPl$e=zGOPM094BDVv+rolLe(mAafx3QHve5;ktefrtyV74)m<@nRUfmOtCIK=2mU1o z+FSyZv>CoF%6^%7-n`C{Tgt2I;z^#@f--j%`U9k#OEx#KXDmEpDq+lA=Cjl7+J=_h zaT&^@eNDo1R!&TodB_!XVNOKF4@{V+ zo74jgEQ-&M-8xXMk!j@i*0gX9n_J&O9tYtji=7YHyl?ztk@x00k@#nkSmp^P@tzGW zIysNF?R)AX*45Bqld(W#zJnuEmz$@T*xHX%Yxa~dDO{cz$DT25`yOVc2_MeXKVUk= z*~lTK*O__bdHs!mstFk8EKy=N}lR};Ji@RBAW>>8Kh4ae7V}8-BWPA z_xz$F&k85;qJox9Qi~4-6>N=orK-Yzn00PZ;_}$O6%q1B8aOonc^_nBZWFt2Tz05W zHtk>J9|7pJc#Op38wlD9q-el|LcyO#hqfzpXn9hfb21lB$ zcQ7kXXyl7=6>6EF#?gGA^}3KioN|p5SBk@uj(!%EIckOuhiVf_|NZIN64JmN)4&yW zfIZ;=`vwM%4Gg?z4zR!A!Xkd%rm#~a_JfedV4&PJUKjf)geyyCPd^`LZ&gm6mZDbskd4<{maNC_OB72V+% z^MzaHR@CG<5jh(Vu-nX)yU{4u6F=R-=Wu}F(S+EIm5ve%SOsLRaO~*h>tRq?&stKe;U8c&5kn%vHr`vDQX$gXk=gDS{6sh%fE@FDdfxF#mTE@=|eIVg0)L4@Uygn^@&E2Er?qg;rp z!W%~g4kty2WW@kwl^!O|2~K)FPfez%nx-(D_Jo_4Jf4@@Z2s=1$s4DbAZPs@0<|jt z4pnHnIIFmr^*FIFNM^5?&XCj4Tyj8b%j3-nCxn6$11cPBlN1e84@%t#WPc!DC)q97 z&?NpZ`B}mYp$Q9(Liq$Hw6?20nII9?F>waJ%G^twJO?*8aQ=BHm!1Fp{_HbP@^&<^PdO+j`P4wBS>xDY$q9}A zuNcH%G_-m;O5Hi6z0FBYf>FW2DZGzK<%yH#j3&J~y?QU0OmmdyZ+sfH>}k!pbUmMD zy`HBLZcGhTsfsyI755|ZDinTIKVN*(V#)Z;7*gZOtZCsv)GR%{*w;wvS<{^an$Hh zpZ@e_O%$_P!DF)^XU!>1dOe30IXS8DXehlo81}1C@k3(*gQ|Frthh*1(!5F6D z$Y*en=f(k!lm?C~%4{JHwqg$Df(=$drx-50QS)qIa^adK+PgI1(j}1t+CO@=-}t6x zbsJA@6l6Fcv?#gxNRwMjsA1{UQ)inD?wsw&3ov}^Bc_tZ8k%ceCZJovu=yI()>|DS zA2cN|T@YnaN%;E0^5{P)@ePhyTr1Z;Y;}3Zz;oh2_PvLfbD!8IioJWG<`L$TmNHTN zMWVRsLCGsj1`N)+8BKx)9s7UreSE@@!J)&~@-p+z{aY{DPWK!T>qyW4Iq`{{Bj3(h z{+dsOKz`^g1Xq<8*HrpU9UJ z&s3JIZj4y7GimWyYE}x=5?ZPY!laM zB_20@Xw7t)(^iU;!}-U(CgYFs-m}QKHnb_jjaZzumpH z-{XnI$&>eQ9lZSTKOcJ(#<%;D$0;G^;lX@ zns|zV$1?}5TaF?Z95`h7Ki_d<6KV84rm22_NqvgQcaO&27cbMr78rcFxM@GgH(a>a3a+5N5mCaK~XQrRGw- zheCWev@aZDk!?2E(IlC1kYmFN_7ulWa?C~_oMN|~txKJ1;B)!hf(dEw!fhCqY{{A- zc%pb^td5jXqf^s4tIl)#6gcnA;Sua0oOq|4x}9kP~f#~O|!FLc*9p37+B~@ZwUeQ6lW<;hjdpkdi@Obm)fcwq@cc} zk=-Y8=FIbt40$gvxT*e8r##^JthM|lrIX68gp`S#4J={M|8q#Jq(ShD)7_iAiE6Ir zxFk4BSX9an>V0w2%}7)F;iU0{Y5r43e=)|L0ewP z2NWML=`Ud1X`^6q^~5U9uXgV8?X7$h4}83MmGywY%N?s`xX#OR)XJa|-I~ z9`RL*i|mw__e`JIAT?#ZlmL@Lfn(O01il*wgnu+H7IV@xVNywH))YD9ZQZP3;jDMY z!S^yl-j)XTj-olc4rw-ryt&pSnbSBYpV6I_+3?FnwVXzFkpmUyKN)fJ9(<^CF!54J ziTlBs{j+uDN_FG*S9YTeM0IiN77$xMJr=|z*~hbGN4yDC09?ObzOJfKk|!&$%QAX`QQ*NOxG*m@k4dYtrm znsTBg_8cke|FR?^L8pIOT6%Taq#H-Kv-N!6>u~vC8s{HFx$0`!KZQH4Gwu{$m*-cK zR?^77@TmN6hX0m~3Jpve0*4gmGzv{%Zv6n=Dm)m}l*n zIJ4h@-Q&CNj7{owNpJF+q^2}VeTg;XVD^~F$ZLH3g5UAV@>2&ztx6PqFTD^clZh$) zvgyM=gNqVJ*_#|x8V)Pu9MIsgG2>|}zj||-ID`0uLwawROiw($aGgnV!WxMsjXLv~ z%qtFQ`ZViTH0#f4(hu3IDZr#J)3lMFN%T#V*^)-Kmf9Qn3`#vsdIE<;R2+Z3`nO=y zYXytO$!X;h(SM95iQE;AiQjry_|UD5veO)P)LYB-SnuSo+u8iY<*->!`KvuWDSU6_ zMSBh^9@(YIaERCB-`pn-Q@>XW861>&!e}z@>_P5B$^p&!$0s_r7W>)nnI@|cvM8O^R)o>Cf?fBf9P4=1DLf*qW!9GnDaG?~j>)#zwo zy|9Lr$ARO8gVF>GQwJu!FHB9xt=GIiq`t&apXabbNTcSGCbKz*)aNj*PTyt!6 zaW#L_jsI3`6-*j$4k>6j-tu03QjO8Uqflunr?8smER`G~Rxc67xCI=Y79TqW6rJ_B z7>)iNVdw8Rbl8`$aghs)P3fB(j*-uo%&>}2*!SntC2pS?vHM&eerokTuV*{^MZrSn z;~cVy43f7Fx4H$(=oLKJpxhfW&#~!~$f?UGW-6rjviLF^F}wH5xMcQx2smEUa~#ja~ey#2N8xWKm99YS9p9XmUBz2@>Q znV(wj+|_)t%)cnU`EMx+ZD*UC?%X-_j}E!ah@_#Yp~< zbLV&6uq_rWC;suv%T{(Uwu!q~Eo>1A+N9jC|0!T)-*kmJC)H+@{aU1^x2NM#w_e?= zPVOyjSzT?LQ#u-1WJ)wvx2Y?<c5#_tuve$kgrH@bs_cTr9)~zpXGI+9FmmEl zo3VXD4ePY<7Q;Ss>e7M}|QugrZ_L!=ly%XZN0 zI@R6x;}P+fRWDh0zTNcuiM4jfhdzOTGRqXs?_pm9IU6r@^D8QdWO;DtJ=^`B_m2+O zqeb#o*$=kMHr>WOTUa^BG3Af(0qso@SA$>@o~0(!mS|k4@pt zyPzs0rL$oHgMF@nilEUs0e-=l3eLRX@;1(V3I9_o9(8EhI4boC&0#0CHT#} z%p+X9dzjCQ#M!-kWFuc+lD|ML?y8xchFH@gJN<@^j&Q}AqtX$H$u}I?#QYK-9!L!g zGIL5#UA2}LmwXkhj z)&|A?K!%Brggw|ks*3pK6|NR-6|6JzQZ_lEz|6E}&JQ15{^TA*{i*2>9vfdi!}II( zp+66p#WQv!q>H;5R5I}KbRS@}+a#IFrym1_mNjfs6F6e~apL(J389H8DW6~z zR#?bw_rifEc10tf%Yx=*0S|c>|2QI&X43XS=ODAY!(la(g`EKzO1vH)9N4)GX0wGn zU{>7Vz`^pR@kBvpWv|b@nQD?oUXwh1Q;H5KuejLjWUx@=$^teOBZ0=rrU#fx53rfK zIWTT*OJLZjwO3>bh=d(m&PE#gx@N031Eh~g`j@V@LO**^@NUK5yv3a+r~|0`M89qZ#Per3k7$juM$xT`RF z_z5^i2uy5oQ8^@FWueKNm%uz>T2jeUiNmiyF>q)tc_^H2z@*`@P;Bd^HU>ckN1<;H z%xXU#3i@aGFy7gs##r35s_)5}PYQ?C#1l&SLnirvsf1WeYCFggR5@VMpk78Va`Pc&FUS7Jb?$Ar4Mh|sLx`}b6klrE9mzTW3R%q z*;Nl_F1T=1{fw}m>TeH)j}x2vXFXt1OIpntx1)dR?V0BPcp8;mW<2MOPguarXy~++ zSHr>h39Is|%1+5gPQv>ZuzLn5OW!=uX11a*-S&B=x$HO&2{bz#Q7>Vf z$fj|>N?wgQJ*X@&INIEEH_MdVcTt$WAJiX8v(m3B08D@r&9 zyBuiozmOpyr#rWoMUY*~Mp4LADk~}ETcbycve1MXZ3+y9SIpLC8g|_LrF8emozLvfePtbX;iAI}nvPO;myNe5up9KSvlNPJLQ?)$XvCLm;>58b{uy^{fUB ziQ;}ONfO&vI9=9jWKr33``AnkHXDQWk5AYg<_cZ$UBu@hcddTqNwpOIDdLA*nx1hz zbm~yd`pw+(pJl~X7OM+0lwG3lo_BH1TX#*M@axAmofgNs1xKEh9AHs>aeytoWsA_1 zgl4rLitgH9AMN^U@uaX-j$GU#ZY%oYu!JWbscSibqmED9*SRaNbGwk(bd?W<0I1K zC}PFPsi!!zO;Ob1;ipH8a%~TV0vx61C1zRmbE#>FEPE(&uA#}}d`p6ASeEKskJqg! z9P=_(u-hfEyC|LEd*WEb;Z)@R_=?WpQpy&<-l2UlHg&Z)l!EU(;9@9DK2qy6hE?nQO|)%<^zMZTe84nZ@K0-QircNDDclx;6JuPWQ(J~ z13ng;3(VXH*qjP@{;VgT~KR>t!9^i!4jz`ImcgMvj8tdCp^;bNTebh0cW=Fhw*h zeRv>%r?7uM&!i59td5e^+n+5y^d3H)A;oF^94Z%g1l<-mPv0nZr)-ZKw) zwmEPfTOgv6$n%9k;9LUFxd%K#i~>Ix1h^ChV;)@0dLZQLz!H@pl+M5w(7+nSz+%F{ z8t{PapH!k0gCmnt10#cviIM}8LZb**qxi7~5xIv%k0?klOMI)ef$2#eiwC33j|Vqp9NxbRO1D~Yi)Z1HhJ|8vic;AJMPnQ# z`Wz)J8ihL+N}M^sGwJ20GlddeilR-4M)d^}^Acs3yK!2-7oIm^<~Q93=bUXB^iJJU z<_gOH8ucQt=4kl0V{HX5Sc4LHY+dPPyCb8d>tXB~v! zar8$i_TPFWZgY_1oPxNQBY%zpXBtEAt^^*7CF){YYz_~i(->Iv7?@kq*(x5e1hBKX zG;qyG5GYIhC!(}~vFvQhjU|kK6vP=6g}yM9DLh~iSj@O9XyP$Fk>e~nw~sLT@ah~d zV%Xy#q|(TrlFYB-D3GxrC8CW%r-4mrq4W<0=7?ie0R}8i4a^n`m=84^Vfm~tcwV%@ zQEys{UWel+mG#0+5801ilsPXb*QID2{7~rD#n1N&do)him~?a1x6C=0`Bmmn>jDK1 zEv9)*3=vHV5fusSY%VH!=Xq3Jyk9FgzhdC3+T6<1w?%4F*dhgJ_<5 zlnDR7{$XLa$QRh z72ubMdnlUG==mXuy)xjF-a=suMNzfjm|#WGc?V1KSSNn&qB_OhCgcRECmYH znFmre4^%}Rs5N3>o}=LYCtqOM!GF@97$$U?vN$yO-gzF;%ATsPpm;!xUxz_!k{J6A zaiKp4Sa-MzEmP#-NtD)LV?lU(ligFZ}b4)gvxkPx!RCeowZlR2#MHcObjH2@%O6ol< zx?cLCBvIf;@BItRiZ;_tj%V7sdB-a+Qc_~#sAA<9WX;aesWcNFCw9h6PjAo}VGht6p!pF{zfEBrEA{8^V+V;DHT zBt(8Yz*pqJui(i4=~|Um14B-N$R`J`TmP7Y-5psbJy1WfK%K#p{eS~|oddswg@N@I>=X`AinG&^Ob9?d6TYxIKcTYFM(yw0+A9$aRyQG0>+u;2QThW z5HC9@RK_TMfI<8oqeX$F$iGGzhD4c3zXWzR$m~mO-hWWY!%}cVg1C?)zsFCm6%W@W z8*%D1@_%7q$dyV)9h zL)3~f`sB}OLq=h>&z}tPWewNQo&Rud>kTfuMsdGHfmIBhzt^_>IxBaP*>s_W{jZta z%?Zj(o4h)Ye(OmsyBeUFTRyiU+%ChFAUr8rD8uWcDDzb0$=}WIw-Kq%?#2kk1jUR~!_$a7$o=gU}I&07ga54Gr97M-FB- zXxuX^_<8M+{9E=P2Lu?t@-QsS__kcUAaUxSBaBu(%yv00u@5#_KG?9np*1#9bizWJ zI|pSu6y+uy6pc$1OK5D``{OW4_y|H{*h|43PBbXhZhUd#IJPn{Jrp0an`R@lL?v(SI;7mHU$7um;odcIxD7%A0;M(V!J)(iB{}cjGf2%5b#_pvj zUb0ZPm~p-8Lg_vH;%pOGxJ-F;7(sXc9ZKk$)S$7hfxAbkAo`C`v`y>7a2Hnx7kOQl zyx4qqhCun;KN%0$JDQdhFp6+Air>4O&-hZ0vB$-;VT0$FqqC>BMlLKUtYJ$p@JwrL zI~Xcj7vSD6bMmOHT+NQn%@XsT-V}Ktdg|@R`)r54%(2M(656H8wC(G%Hl`&JDaP$V z3TM;4n=2mTy(!Ssc)Psy_XFNTXLl;_EGp)_bU>_u(aY)}j}>F$8yoH^3<7`rdBkGb z@`PBF{zmT$UU&Al(84BGvA@hr3KJ$hQWa1X5>Q+u)F@J-C>YZ9$6-PKzn@|(E+)@w z^;psrSh5~m*Eg(oe^BLcpl8xP&Ql&sOB(o3dGL2N<+&-a$!%QW=D_oyTcAL3`kP5i zN(nB^vt2g+JSuK2cFs<00;6OfqvVE*5($lB6?58J7fL*l`26mk{IS37?SJlA>)-W^ z=9-(}!S2*@ZgFSeXU-!P@k?)nYbkAGuHx9Ufz9EOS82!D8$NjkOHM^}w6e`B&rD4+ zf6kt<)O|+-&kqIhWepxZEP64G!cmO+IS)B{8u&dP{$u&cZ1rHb;Mb}%)@&_IS}6x2 zQXK*d9hjILL~|YLo~)VjHR0gjge9js*cuWJg=Vo#I-n6&%W_D8-Kjw(Sc|8p=^y_r zCPfzo4Fk!g84ACzXRriuaa;*z7x~I0V*1jPOT_6RPf4cur7k9~4|k1w? zB!#YN<&Hh-l*S`@B>7mz#YwKKh1Vn>?>gH2{Cxbr35uT&dVhT*xyoBdKh&}G)s+RR z!CBX)7^$8VQ4gA<^6;ff*JL%nc`l7jUseccc-M1qe)@8Jg1pVDcO8t6k9Lbh?{xV1 zNR{ixdfyh4Or|F~Q#Sctb4nCq;Sg5zsk@RoX;PVJ1gpxSwv-bmrjuQILoPn zN6e`}-MM4BRNU^Wi3<`AHde4RaVkA2NEJ0|<54eKk|F3OB^i9iP@(k-zy04Y!a_Eh zmK~hlp%IN;n#&b}j*6H(+!EX#{N;&Dhl{MMQjoub>JedYR>q_Jjx2^pLPC!U9Og6H zbHIsHzrt~vxP!w%Rsox$L@^NuovUHOjvJ01i7sON=-ln}PO-zdIbh;3-3=^iJ_!do zioJE+B^W)D>mEE{V%M9oT$0b?$%9t8xSVOse4Ez(WaQ3b5@=vvzVsT4QfJ45$PA-~ z4~!fpJqzdj6ZWpy&?4lfv9XoitzhGvCCWZhhnDQN+!!r!s@IV%f9b1&HolaOi7pZi z9fD4kiY*S#>?tZjT#7q>TZxm&$3~9Oy+J<%MEN5!mxgkA z)V791#!X2Lo9O%J!x8>mQ^}|NMSl~n^OwHb+HTSsu+c?h`<+9a>f3HyD%9B7XUM8@ znW1O}x5sJ!H0t6)lw@WRJ>iH?e=p6Rvlf?RoHbz&k z^}B2J)O6=6{x>+n#mwlYu(YR$`HE8YBv(mynT;$`4Ihj=L?%p8;@~yvdEk_*Q=Ql@ zTY6{CzSLH!S4YnaMNDYq>+le2;+`;7jIr^80#CceVF|7stuAvOhH;y?tYSFG9@5#! zk}S|H7;;orPh+{6k^*CS$%KE+f>)MVzx>U>=CD9Rs^0;IKxK zA+uUXqQIUDzJl|@S_~DE#H&0GDR?zsym~R`%Me8f(j!?*(#s27tHPXJQUc4 zB^=H_aAtN^aB5wk*}&}7e8}L1BYTVjGv|~5t@!=Gqervp z(}E4G5*&`)F&`SGel)O(PhjNEX=wf@IN_l9f(k`R4)La?vQL<0El!AXeG&-UkfgZT z;gQ`2RZB%D@2Jfh0z$tV*XH~@?09~M`-^!eCI!D?DwQ%4_dmDPRWEO1w_E2SX_;R~ zm#=u?8uXHpKPy1_{?Y_iCy99iw_YrMba0}cQ^$>iQ=aeH<)Fi^!|Ei#vyfZL;34;< z1#U_iADFGrK44RmW>sKIN))QJG*aa)`}RNua-ChFVi(tOCoz%x7*eowJ3RYY^6!(QmK(-;rg1^YQIF6C~ z#G+Q~k0%w9&oD)8kvJo`_N!lQnV#wbVc8vvD!770{%zU%@=&Kpf_b#wwTGOEjfb4` zS1%H=RV@!a_=2x$`BY&M9$mL0^yUHasEt$sd_Dng2@R?!jl%X2d)uf zcdj@jK8u0PQ$>k?nFgzz+Dj(IT1K9LNxq%g0nE!TF|fy!%(ZY z;ecUg0JCh0qTr4dtzE*5ktePRw6h*OFM1{OX2c!g`HBf4fLOPwU1 z`gg^OY0YX68HH*JqUAmra0TC4z-l7ktn_UGyJNSb(6S2YRIYQZ*LxOBeo)za%9%mR z+dF}Qr@($Du;sDb07Q7~|Tb)@Qc9 zYBWrEIQ3-$&jJM(rHYT-8K(^z_%xQ+uBmO!PL959ml7ag&7BvzL`n2))Pyuims{*B zSr&wDm@sLj2CI6#NB5?wo1A5%@|RqB=kojA+n%xysrMz#`|P`do-Pat(vZB9s8wrm zWafnC4*P|CogC4MO<7z{A`3Z`6&%Zy9l7j%_J}k{IOgnWxZ-!r-|U~9PT|>IQV%XL zG&~b5aA4CjXmjc~Si?BYVdd(b#uscpGt07>ZLq$>Bw}UDygzKpA|r>8ZBC-h=MFMx zqzlYSEBeE}BI{dK)x-`ag}ye%o{f%|`dTGde^xA2_Y}}Jmw2RB&OXV-(?m$^BhPYG z#ZUp^*euUXz1G9cW6Z3Ns_RW=< zjE1wG+vsPA^F6n5OBGL2;AjwFN;cq%ZD)%rV7qnLwjqGsK7swQF^Bnvq8;M#Wru?N zTA5=eu&kS?!T5|p!JwE&fKjx7QSt%H1O<-Gip;`piJO-hNfsnLE0o>(CQ&C9%(Vn}e zf#?qO8j+1B%kz9@zwJDiHe2T&2>?pB|#~H8!4sW6}dgPByA6GWrCr-N+{&BV( zW;980woplqm{5O1O;Bc`&anVN28r^A)3}Z@@N2G|^#6f^hM*Pqi3USQ*L7DF6`EXI z#aZJ&1cYB}j4}({wOId`Qlt$Oc^CcygIpWf}99CS$Y1A)6=JSvW9d2r(H>FlT&bEX2UbSI}o(z%j8w zt>7f1dAiTZg)FB6Sxy;hp1Q~?T_U}0V|!m04KIKX0-!0ai&T3Nv1nZObizDK2YT&FsW}lap{L`5qOI8U@W;UO|?EGNL>q-vqf~f}-=AU-r*?)m^ zZ2@yJG$qVN?;{*LacZ8c5FdGLj8y}eIw%Dkv$V#{a!Axg_HY|6Zf{t*)uH4pT3%O zzCmc^0iiGhMg4`mMhmT$YBatQN-elBXR9#l&ZAw?Y|2JK%9*dMJ#Nmuy)($+0rL}0 zLqE&q?V*bctC)QR{;?)Ca71rlJ@*L{}Y+=P@F6J|fl+Wx|7X-E|7{0S+|3>Bg~mdamR8faFvrC4Om50)GSmcYco zu#46P4pTHAH0@a-IZM&dZh<8GY-aY~YDK@8+f-iyh$-?+9_ zau!E#FYD%7*}yf!VdyA`%H8`-ISh>{Ja7SRmdh5*8xFz|&XIWo+nQw4_S>K^a z`va4{hx)nV4YM}xoL#-azF>o2X2|1C<{K)~Zmsiz7OY9%z&h)I;p_>VX%pBo64=rb zxc3#X1Q$58FJRG7VP5x8s)8vrb}93}Z5+N<&CF#F7=M{5FEBY>U^053 z%&fp-_kdyQMiYeu#`$v?IieVUDA&s=a5BEx-}>1rJ>72tX8Yv_+at9a#X2U(t?J@C+Lgq`n)88W=j9#xKHX6lm+d9Mkr&MQP0aI8Z!k-UFikk(wv_o*WwYA?wgU;=voCOFU*KG%z%}~<*S-m?g~nWq z1=!MdN;(v5p8wE(>O@bSR7u7(Mz*M8zA{E7fqxtm7qEym%#5ADtdn4z!ofEE33HVw z^WrdO!3zvd8!j3sF!OwnWOPUrI>4YKz@XWCN#pEFApr*73k=K(msu_>U@VgoZCJn? zVDX8a>-L88Ii&)!1_Jg~Yu9vMiJZWiz2HjJ1lFhvoKX!d9v3)03=Rr9ZTFpZaK!`5 z;8O>ytGRAG*e>%ZCDNo(fzbBKl53ZYj;9zsWoH}b(#Wiyp#aS8j z9>kO!ir-a{w5y^^`|MAnroz~!>D&)X4(+VjyEE_51{;B@$2Ad4y_s8WnJgc$;*%O{;f0squATX*H8<&3m%~ox zD>6GP=U%tYd3ZPHVZPHj9`AFT`{pTaG4h?T=hR2>?H^R*m3R&aa77+qO-kUpY+!4v zw_x2w(LH-- zt`vhtu2;-G<6=uj)`Ivq{}@8@F7q*PUNT@VtL)<|VEpLF$jZQRD}jUcK!SJizgbc% zKXezfR^8)@oG>+V1LqM2{+R(U!U|pl&*l8^;k4GltIH+`{kXd>j3H#%gYExx8yWR> zYznO?Jm9+O2MhZ;rKDL(OZ|5JyY^tqkF)uUc2>PrvwwfK-Su44-<^+q&eN~l&xT9y?TUpPGKB-)a|FCUYFo@;@XW-V zF)Z$m*kM~PTEPAr2WMLCC zJBxvlh)Tyv4Ue6lYB&rU4l+y;3a|>u6mo207l^Z4@U!8dFPo5rO+Z1h12a2w!IM*g z3mcl*#C3I+9!Q_9(j%)=z!9i$gq<(avgE?4#v|d%dvjzzemySqdneJQyr_}{sJgT-O3T~qgmeVSrrnsF*oD@a5?%trN;h-TpumrmiQo>R zmNOn`PDwwtA}F1$QOhWaWvYDn9|x-z?(jt_dIEhD7@1t0#N;D*QWTXQ_B0vSmE8RN z*iFtuHus9AYgbzf+YZUiHtzb#{nPp2;>%|*y^?9-FsqVqGQC=H;cWZ$lEVkv?5j^d+Tfjsmkt|DpqoT)P&qi3?jVXkcDl^hAqg-KIz8Ec{G%M?K12ESWg{w|qR(rL?VN zQaO}#U# z6O_-}eCPQRa6`zp#NvU7Z-~YHv zfy)IP8XR6Qb4F-HGw~#?SlrpTDC1C{J4@%GHrpjPl4k`e2sU;Gc@;XjGEUD`?DBXg z@Z@Y}TZX|Q4)%bP>g)$o9h~P#b-ZMpBGa$QWMANLN{K_OV?k57vp_PBm@$WY)cD9?P*99~D1*iluU+@#6SkWAmqP z&OFsfsB_{l+t&W`w|U+>#S8~Q(Oc&KmR!`|wf%91#KQ$rGd3LVcYLIgDrI<%p*2e& zES*UxosqGj`Ua~K!$DRV4+SPch1;xB1&v}49&+n#aB#{I;P7x!Vb+?^q_9AkL+egH zkJLd9w`*TnJgzVbEnRR#IYNLTH&NR`aF^$Hwera)Lj9txJsp_23>cU^9AbnfFgBc- zIz=ex0JF5jLAGfp^to0n4;A5PWVZXTZsi0A7Fmf#cDDdFwU_HSK#LB37SRSbLTD7e8?3P)TqVxWXXtue9S37qel9W5L1TSDg-=EKKVKOcrv= z7c^8l9bgZtNn|rS(7>_e!~Vbv3_@2! z$iBIR(Kd&HbwlPlUfTyOf(i|+Tnk+IH4?c~J(wk{o;S)G8n_Ea#D*CKoEGRfVCBto zL*nEPM(qj*=CB(LY;Fmy%~=Z)nPpdS^*q1OV{5{{Ov5M1uFU;t$}`@Ho-@BL3Sd`^ zIbT&!AS~li#Xk8MBVTQYv;4X5m!x(mEHhl-$de_|Ecu|M_EBNu@{(IEwj77ptPH5tt$6)RyGF7~j<1mL_K_kZ%1%ax68_sd;QeacBQ50-@;3TA^$f+*D zB-GGwM9xi-Lra87x_Jjj(1})EMwX6f9*zT#e45U3ZG7UWepAuURCjJt3Bb>`WV9sJ?jv1Jn{WmcwJHGg_^h1K$@W~Ex@W%&n=D?}GCDcCr67)c!D zkXz8ewkUwhfkjDt;t5tg4#y5-6-6$;7mU0N8@Zk5JmhJA@vl{HMPqOMn0vx8<2RKqsxM*Yra{DiFTrlC_;YeY{ z#PXnnoSBynh?g`baK2}fesH7J_TEKSZU#{S^9dXZ-xjb)b1+Fy*rEQ%WEH*x1IEz&7+|9%*v6?DuKmVKuI{@8q<{13{TZ(I|`?1G_x;| zVB&6ZD3$!sVCR0pTW~`e^U;e9Gi9Tq&BPs9%_}yGPDpV3$FwTThaR`_vC&ZURiaF$~N;9!C_mDI~58PM&dv zp_#8}LZj3ZMS;o+Cpngn9YF!if)YE@8#Gdca=i7_A9Xpjmt^?n-c)Jhw{YP7ca*o0 zCFQ{;kFrf-6E3_n+A#TU+KHAuYyDhTzOZIl|4Q|M_5W43u3f8O)wggF;;=y5rR+j(Uzx;5wbn-@FBGsX2|J)9DsWKZ&<0k$36A{n z3@m(K7O)$bXtOJRxN5@nu_f>VYx;$@K!HZK3mlTW&GuimK7CR#@AO8thK0N*ch41= zKKVx{mxht5gOTou79$6yvLvCuQR=5&I~Y4OUwovmb!t2B0S5jLn|KZ|2pnMGUEnCi zV|$J3pRK3_W4l&gd(vXFC(DfnUJYjd3M1VE zuKos1uQ(d@lp1vdntdZSRo-smU9d>+vdyk&*|n=3rPmy6IP7>^$+kUd^A%^?YucIRlP6X?*-z55(6Dq~?4(`O;<f2!CyTV*%p;9e#Bku*H z+Q&BMRy*F@<(Oq++;-aS%H@V!6V`m6b;=^{Hq}kHv^9%sn5P)6EQxWqPuN+$hQ%v@ z&8vegbpl&v#moCPj0%5tz6MlhKKShe-AM+Yz~ zUtls_!RlnuvTq`b;e%%556!j=Okxf$VhN4h9vmMtriv|K+_AEe$AK|oCZk6|o6UkI z+Y4*~Q<&HV=B5`U%lVDgVJSGmAaUeT`lLjEQv zqritn$}3t+SMO2gkhuC%RpO4#F74Cdf{YhtpA0Y5;K}gr(As;++1TSkYmkKG&Dnd; zPO$CWblU!r_oPRsC!JnrAHm|jq9tPmTlR?y**5|*SFq{5IW+r}=KEPH$M3k;w(y($ zIUX_djEM@P!VPAJ7fr@HTElMqYYB^JHsa9Lz0neXq2=0Z149EATLUv21}6C*jW-W8 zswt={oDj7>(d@3!YSYl9U%?Sz*lP1)LFNx3!+?z`GZ&uN*f~!{C*n>g+e7UzmOTnq zr?^EJ-hI(A$uKYfz+NuE`1wb})s4Y|5sOTAwECTB@sZFiyJ)U^dwcC_{|n8cybR}> zZu>WL2Qc{VUEACee518Hnn&e~vhP-J%g$zX8%=go%h#`%?JHU`Hn5dNuvaf=5B=L- z8o^%D(93UX_u-VLWQ31rvh#bc6O95L5<7f6iW*fOG}~x28SP+=zR}|QqQ&$Ei(kOC zgdOYmDYiOjusAZX7&5Rpd$2HcwRrqfU=5OBEl_36n$ecIppCzn&GE(Thg+ruUTD)l z$=t1Qly#H!>5UF21s92aHj~LZ!1l+w+SH}!rwaQG2IU6_v^ZGnKeWd!X#DzOe&e17 zNr^_oPpzq;%_b7c+%pbxcO2B)ttD}#foFyzPX~vH0ONxv{-O?zuCX`&xin7gjhw!e z$@xNS=7~#6Dl=J@+G;*fwlBI+!gI^cV2kUFmMjPMsvYe$5x43xZq;09uQ|Y|)UosZ zq@A;O-FkT2y{?7->EV?p|G3M3V4S*#QTanln8j_s7c71pZ3!>-`6#qFO0e#!WC>>A zba7zW+Qc{E1iynui&IC-??SfXy^)S5*m8E<{HLiAo^_!uX;TM>hgj;#)tR>ziZWjo zV(JR2xpT2HhG&Ns&yNOS1Ad+c#@UY=rB^UY3oxn%7#UkIC4FeGkZ7GG%ki0^apn|V zeh${?6D^g-O#&CJCdi)VzOY?y@8^~=jxx+>jePDK_iXA-o`fbRi`Gno z3s0O5smQpq)iO_gB+uvPg+H#QX(I$c|5iDo^{_%r2qkgg-vh& zs=l+8v^cT&Svq-MHzjv8IvKPEHSpMn@u)qB z^DSNH)W$ruDB%vkS4r00&>9wDq#4z$CzM>*P_NfPZ0^ zUEEw*7{wy=#Wghe6@098=0;m3 zV~c@BOU#06wkKM{UQ2moFvT)tGGBag{|LiZXTc+%x9$1Wx!?k4&ckU7I@Yi>FgW*= z9{IhIrK2I@xLS@g7Xt&A>aC`;QQH$PF!^${8r^7E>eMJw!6>n#u^}+wo@=uj*9)#9 zhHL+hF(~VAKesk_a_v;L{7=@$lJB-f3Uo+KkYm}_XB+a+;G~I5SsJ=~<+R$jT!9&x7>FgUO^%+gc zVLWPjEFK@29Tv1~6k?VYVE)6`tjODvE86VB!Q#LW?#RHnPm$HO!|{(_6T{C2wvGnL zLzSmba&jEm&Dg<^*SwH>W#{Y$PRSS6q(o{VQ_@Ip@Cat`pZ3?oSAhYPkzo?(0O8#4od^WDXYc^dq(yN`}Q@j z$Vk({x({$BG53wLb9}6p2`Tdh)d_KmXqL z`l1GpMkZDs9<8)Dl23KcPf-t==g{&a;FNS%{=FjyIntHqSl2%I(s}v$`9&F@etmqr zJzw?oG@ZR1f=nzO{4!?$vVSQwEnaqZo<-6ZmafA`{bw5{38+Rfvrn*YHAv`C=9o0a zVOde{sj1%6R|YLz=35-L>H-hDg5HCLf$WzSTPZ39oM6=GTviqCQ z@M@*pSmfrbraLodE0bD}#D?gFb-$Iqt)4v7C$ley!*Pal3xAjK9E-+BOl+?L)-P_b z`f#X3MBSi9UZc*2OH}J$N$=s7lg{me^Y=8?fAkgUD|~n4CC4UzdHcRaR~PqtY4g4i z@-DrR*e-5xk>!Y=<&TER{&t+9A^%z(WLrh}EEyWvRal=2aHx1CJUCKS@nGTt#R*fK zd^IPUJaT4}Z#dw}Yri3sosF+XU=c^7%csMvyboFfG)sNT8(Afve05||Hxav)DBv)` zAw|&O!hzI@mCq!Zd2A-kWJ)Y^OI2R#BEV|q^X4Om>17vzMizkss!mcFGZwN6&COXf z+u*p&<8y`|MV!y5MJ zA$YT$iRj5#lafFVN$H9WjBLt39v+un+`H(XuV#*^BQuAB$wy{^)-7KTbC|p^@Zghs z^zhImg^UMBG%8a94)a(=BphTfc4AOvX_uXLlbJK)ro)U0rPCtQ61>zR(*;!o8swKO zIKW~R#G$~X+3NBkg3~!~!4b(VnHN8)&AwB_ZFq#G__W&02R&|*t~mye7yG$YF_m0$ z<(gVLt7!U{@=J|Y6S$ZTHXK@Re#G#UNr%V5Bh}IID%o{EYgHzB)K8egWAbAO%bLde z4 zUT~sauHntaR)vWyo7m+WJ|uETCOHUo$k((O9O6jf`NwpEBUWM(xBMB??+%=yaRQ7i zQE4-m9*%MnD4Z^o5Ggq$G3c5DQ@uxXBkz?b8xM+HxiQg^S>VpSh*OyvK{0A`GXn3N zo?8p&r~Q+AV9;8b#xSk&0K=gv z3~Uw;IIqlbWZN>aor~v?q`5-!H-!fbTqz4U%8i;O#WMH#7)+XIog9)NBd}b^;go?> zHy_smXW6AU&l^m>6u*K^Y1N9e41BspZPgDhGp{>)-0qiPw1?ZFj*JL{T*;J9Rh@_J z_CGEa6>~T@tGOJS{pM|pmF2?w`zCbg2OQw=zTl#rQ`nt#Er45&;fQ+F6X$b`N^=?v zTI2*2JDoZl=QJoVOY$+c8MyF?w9H7?-?ouE>&IegJ&s0ZJ%JWm4kv-i1g2Hr9FDPB zEMU^$IKbR?MY4V3N+aGG%0~rmH3jj6NC!`7iZF0c(Xw4IpI5+vE$^Q}tMP-*i4#ni zr3%bAzT`4Y{PQGm&O7myhj?_E@2%v!^hsQHQRVXbpk_y|j#K+3k0=Ty2A;3JccJ;` z8Fl6S2OJ&_Lhbf3rA<5bBs7-Y^o}n(EmY$%!KdTFg3XT>L|S=F<9xkrO|scT?$!&= z98(h5vhOiTm~LLC+ZVv%lA*levcge?FA8nOOD2~rX=suAa-iL@hf$z$0*kyuLq~2y zl0dowi+{)!wY-mNQGg;q?E%K8!@ zKGA|j<&0(5`iiC6iw#=!+ypu88yt8R&v=yPSm>%J(ZnrN;J_T=u*7@@lTf9`Q3Zv@ zUa^n`j1C&D0#X5+batxwq_{Njq!=tWNIk~Ll;I%4!pOpCa8R;vg+p?gxTKMoF5it0 zN7Yvav^nxDI;VVLn)*)f%N$b<)J|NrQloVm(~`(8R{Ou1)k~FEeLApxum6e`W4DD{ z|E-lZShe}q3HPkdtM(d?>=}>;c0W|K+@I2GA=i+&9m6C_H@nGXyq@=azA>x z8yt>sYzyd46gYfXF05J1EV0{o&Owoz2in5-DDlR69M;|?(5>ec_%>*QlW7wJBk!>V zEUG$}MH-k4;^#F+oXORKbXE$;alqK|E)KHV19@x-e76_JKE_ z1cRjTg=Vikii`nW2N-l_s+!GD;9B~F#ZN8^947*ZI=Ne{18f+y|ku99ixT&T>q8TUQ3o&0<#Xm~kd2 z+wW7@ zGt-5;_BKv_lV8dnR=wZkey7nZrKe$PXU%Rdn?yFVA1#{899ns64oQZbV3^FNzQLj} zfmN`Bkvm80*aBYNr#{{X7g<&^O+Re@^Pl%YVX22a=`pvSz3)6MEaIUZb*)H6$o=Sfv8Mr9zJH>8yrja*dLh0pB0Y#+)_ghZa)N|%eV%BeI6fx0g-WYwc%Xh<- z(+7IgdK~X2HkX>IWX~xIJ|Q?z;hv1sNq0- z06Uk7_{SHFss)T}28~WP4orw);0tMz<%wZ1kzr1AuxBsZ#88Hpu96}*9N1V6N%%BsK4~=eY3A)<5RKvEO*z0&GjYxlhADUa zSsCIh4SUN)?)wYgul789S-|DfLI?g09NLNj{6`w4Z5X8=%+ZOQ%A9&%{tvq?k8EQ~ zfSyb9kEKyRBb}NG9RKVLRq^XE+Ic^^kWt)&i9@1Mz2MO49XxT2&FU)BW(kK)5~Njh z68LHwg-smAO&o=m1poUX=_IajNaIe!;+6)MX9pE67+GH&;Al9&U=rih*q|8DB<|*@ z$#ICcWDY~f0l7N|6DPzm2uMn?Ixw#Yl<-;Wk z@-;{A$XrVl*&*Z-a9{X9qpOj?+7pZlT}M26TGqKZH@v*`eP>J911F`n`9B*cG`l>y zbTX>tXN%ts2j7B5S&t^+9tW`&C+!_h>LpC-H<}DM4(qRQl~@j>uy zDg$={14l>$gN6Iytqf9(Y>Yp=InFTf3OIxmMf24h;3#N(rE*aJ(?R{VgNaN{5$5rMI(-Y`u6pn&t$aD@RtBqA+x?5ZQ!gEix_f5UCIQ9( zpLAY^R&I+9@ehqE4;)XeU_A3<{(sI?-4%{0FPSv^TsJ#5Wz{&c1sve0VYGkeq&J6g z70Y#Ni9-?!(uy;fG#U<>O<-0KX<+#m5p?i`1FL`o$At!t00#*MC-D|Wr8P>L9~{L! z4#{;m>Pa{WC9q2NFzGck>g6=^Jz+BYz^K>L+}d|oQR0xbK$FTIX8j2Vt-mlS)HLgz zd10M%Q14H(W(PygjCQUk2i1L;Eqs{e1tYT)g62$En7HHoBFA>Ae2@@!&+%WXG}A1hS@Xy|O)Ew@7bm%f z1Cu%qSWJw6?%*Voqh;2>w5gIw|H@^_2TTej4o)xs@l7#Nn=4&NI0>7I4CLNAYQ`AS;8pk@Ic~417Am@utuX?$w3a5Cb^yl zxt>OjDU*dRERxV*6h6Wz^J+?`??GOb#JD4!dz!)%uVg3w3GrL%u>W5O>x_m!I#HYq zIS1A;)Xa6@WpUCAI4ry8()4|f0ud=+C$7|K_5D`+L`lWj`iobi&l@k7!+I`juKZm#9uf_s5tVfG|DGyu zzB>(k0gSwN4m906z*Et{(&fO`!&q&R%z8ojSCF$pibHn6f_E}!9E$P|@|=CH_3`7ruO>t#<}<2a5@((!aEjbYPZ*Ps1R-NiAAh!NpO;{@+Uuv}|3>*v8u9kn*-ksE0wp z;SkS-g=&kR&XznjRjonNtC-Efp-f}SRJV;uF~tWxKGxWOkFnfj)Lui0Sro^ z$2spXa5W^pyWlxH^+4tI2B)bz**rFz9{yhac(plu6st$!M_bOi+u!Pf56c~S7rWW9 zhNsG{NNVi^cimV~R@S+;Pnw&>x2byU_}LrUWcyrJ*+l+%;r$sCc)qX*Y`AcHs*_m{ z)4zE$4{2&V7U5}H+RK>WdsxH7X|;B;0!x#`k}}~JH|(U0lw6#4wkWW6FzCK=V6Qn~ z!8PyrU870Yr!ezKHtKyZny$7WkW($uf9q>MhkHisH@-~xx@DTOVM0PfT~f202m>pN z!&DUq!KBVPHk>S99F$i$$k;fsc{pmWU|0FWxKyJ}a#o(jeZMGmw*{TX)3!0J`eOd} z-qr?o$C@LTB$|B}9dA_nb5LbQv(1yucGGVyF;&JeJ*+O{Bqze8Y2&0fg-Nc5q0F37FTmOSg_GO@ zE`=YBDjokCxi_Cg0~%&5%Z?>iJaniVaST`KRm{C)6lNwv{82RWZ+t_){0oY->}s^6%zViH^!uHqs9= zU-#F{-)EOHTi|5YVp*FehL#3)8AhRoxneTy>?MsZvzyE!nvE@rHitf)FK*3o#X-r# zNvzjV&V@<7#nJo?lfs+liabsd9*5RiF)-E~Sit+0^#B8RLGsJDTXcoD7uxUs#lGbD zM*kn$otawI4s{O9|56-;yc*dIRxrdQr!lxNu?eZTC{#Q;#453D#*u|$>gP-xgXTES zny8T2$tC4IrQw3$p-v%nKNSzhNhxi@>VXLzOqsV9%lTLBd3mty>|}GtdtcsgvS(bd zD|q*%^YiomMV`-Y+}u5Vy}W(h?-gNNvpBWG>>}RY_{hY<&L?S+^gu!AvChN|87Eii zb{}Sq+?gNyW5s6nNvzzi(^4LloSwEeDyy$&LC~q0+cK-CI9v+V5>Uykc4%Q>JQT3V zrES-f&ME91dO15d8jZH*ic9J>)Syt_@TT(`HT-`@65U}3{+Zb>bdM59xjjNHts3Of`U8yz`$GX-@ zhM0teFmg`n;uifjpMz6TrQhw>iYDQv6U-~^r6M;7dVD&NJAy-Xt<@n*ytKW4JXpNi@(@E)$Bq+SjK_DL(|Wx|`Y6}wV_Q-R z7Ix}?Qcz_P+IGTaO7tud)eQmDZaB2s%qe^(EWP4{^K|!p3@-g@84`}|YF{oePq1Zc zbP*7AJIE&D#HQFH9(UwIW4FMO>^`e|5j)myd!TY-EhBqFwnCHBOyz_NPVA}Y8oBDU zC$zR`tv$E%X47O5RvUrJbP*;kfdgiZ|2UWem>y{bES&OV%S0vSU?0bY69a<|JZNG1 ztNhZwU+BSmv-D*f)i!Exy{I-jh^2AKBDonWE-cH)esogZ%;L#~6?Xq3vSh11HLjX3 z8Y+^kt&^}txo7gD%M|BaXDxra>N~r}H;Gmjoi$IW@;1K~wrj-@_wmM=F35%L^x4&@W zG`h5)sav|FgQ;IFN@L4~MT&p-{l2TvYxU~=^vNY=Vh%eRTD^o1P1)}uTy%g9=dubP#zT|vZcZiMu;6dlE-_DP3IP#19Yt1eSVRR1PBfzG|kjRnv zGN%8Qs=o5E1tmrbOq{Pm^(#ByvBw&)NcJq?j2BRri~QoMwrN6V)DlPb%EN8OB_Ab% zL>BpYUT_rWYUGgJv4&MVfRW3np;4wsky9?#Xwue-1KmXkBFz*|WB`(WzsBUV^(zjkCB?M-)R|*j9EMhHQQh2e!0DO)M%7 ztm}LlxfCK8MW(o@s=Pem;Uh4KW!fQ;${$Azd=)vgEgp+Z=s04sq_BDR41t2C)tzk| z={vITHWv0Z%Y>+yTk3N#vO6$zM>~m&U)*@Q&C+3gA%`5oMs33c4BQwnI5dXc~r zyMTf1Q3SKx7KK0t76C36jRT>f3XHlB`ME9~lb?6SAy?DGk#oy0ui%~}fm;Rb)(--^ zq~GufEuL`9#K(ix{DR7?#%gCjuHy@jXg0NdxshgHU}3r8g@TSlm-ma52A9K9+cvb? zDhZ`NICpT}WmdfvjW$OaIFx5BOql(mMbhXZSM(oOVd(%S&gB!?9UPkeiA>O7Q56#G z_^sO1zxV{3(vD(B=^qCdi8DAV9uMGfcvHsKnBc_P;}&y!i%8IZPmV(0Qyx#;E-{%c zeaRXX5G=NX!D?y~Lt(#IBTr8zi<1X{tgCa2?j<^n*}SiDpT2vKCI?m z#IRq&!I5{y0cO!F41A6+7)xEIel(FquwhbX+CI$T8>AB}Sjy>vJ-l)cz=x`X?L`UZC(wO=Tgc&kV*8 z(}pI0hoqJxKV0?Q7Ba1W=q`1$;hAxz?%Irv8`c#h9DbCsgKv3F_j=8!i2@AV;yaE= zbb7}aCau&>R@s%{VOq9m+v88na_tW|X0R4%H{IciT%O43=fE=ga{;sF1SPJr6Rg?p zik-TTl>~P$XybEoxG#2MGi&9BAh~>4W3qxdb*l^D1qfvsiFd+U81&b6iImK1%8` z$^VjeV@Z`qdV3u{xG5;(QS(FzzunJsg zV07`F%HsG>Rf{>ImSfLBe(?@QE*;HJVgZ|s6c;o~%t`E(3J7)dpOO6e(*j1vgl6BR zC){}Tczo9$|H>P8Js`HTwzsS#`^bUT^~;>Jel&8kXm1MeVSMfP{)ylFES+~U6E=KU z$N6c2ugL{$u?pA`7A8tlIh>``T=i zQ-Ra}XI=lr5``8ll)b?CRzY3(-_>>U3m!%9QD8rn`EJwO$3cZhLpG}UZe)Di$Lg{% zZovUQIYqu13~FVK;tq-24vpe*xh-moMAkUwYcL9ia7!jm;Ac~m=)S=JPl4agOTcZR zc!i=M$3fvLNAXVzoL?TW6)g~ma$tD2{Naq%W_5?n&ILZs=>PFm&*>>rhH{`P=FtGg#P`uo> z?_Pq^3y%F~+LRO*3TiM)S24V9OSt}Sy;BBLR9;iTnx*`I76>gmD0S>8V=M;(L8(F^0Q@`!b_V1Hr@`HcuuM{eC4rH%T6lZ$KQ_%Q|L6L9H!Wrw1 zh%g*vb6dbYX+5*c0_GJCy)F*SX%5VCxooo-d`cAk&M}BoF^YV9z&0u2>>CCC6B`4v zoSBvxJpN(y*yx>A8o!0q6&d4m%a&yCO5kG7;relFxzpbEj2-RvS_(XT52cwJ8U8J2 zoTdMZWkujOL5F|r$`ub-bU0a48d%c~DEK?Dwk7aOCBD>Jz^A~-w?1N#C6flmx9^BN+*unBl5Hv9a0DD2|F#lt0ZWWlB)#%&n~m`fA{ zsvL!OCGh-E5D;6($#jsDp;3V0prkvaz`h2yfL=DI1E!1AEV+`neqIWgH(^=BW0}(% zxlex7OFt|#HQOx1YyZ9F)eK84tF)M=IPhHXU|8hFy{7f8j8|7T8uLm2(s`=Dl#f7D+#eIEblA0UUtBAtDfNhWy+x^c*8OZ zQ@kwo9P7z6%9qd;A>79c@eC&eSvLe~r?aM0o zd@~&QZP*MI99cXw@^TJ*EOHboc_?f3P4)z%G@l|TL!$J*6ANXICCXTZ3FRpW$apPu zY2Z5M!0E7nsZEQW<7HU7KFciT#|_QjUKoAvSTFN_(zh>*mVN8_w&KH%dxg$|uEA5* zGjJ_n6k=c$co6>nBl|svh}G}IgC6j0ThF@fd}Opj6z5ykhzG0{2l(Y0*@YgQR8Ztw zrof*P==7~YN^P0uX9dMd2hYj|jyLBcFPTPNyQ+Rqflut9;HoV`Ur%Xn`V#$OYTW0C za_>yp(jHje6)6@ED`poDlR&r$LV$AO<5Rf{uwZws8ikpza!+-2$W#2d!EqV6I zsw`j6R4**IE2FbtFSBczQ9^K@Xt0rPpuz&nGt(J9JV`csASb`zC({BZIYl9Z1Li8K z*(SY=GqiF771$CO+~Xe2r~k z_6KKyj0UDO2BCci_!c~1T2Ri=q`d3J8=H!4jwcO+dd!%AOfuW1thZWOQR|;cz=`Ed zTun7qr_8-@p7|mnnUh%AcRB-mRjM(==1$72hbMJ#Mcg+h<%+&JP zd)+ei$MojNw!zoE327T#>-_ zk7;6-er}fX0=9$)ECC4(C%;9Se`aFY!T#<5e^rBZNP>{v!W^j-fA56?8*WN^Eo3V@ zFvUMox%w<`69eBpEsm_>LXRgh4@#%0UY}*A*gI?4 zqbvMb2bkO*aBMjsX8AC|h>=snfn|n+K+01AyGH3L5h8Pbh|RX>$~`E2A(H*a1A#{l ztkV`$@%e4dd9WzwLF%obpt8rZt!Y+OTzMIfGvZ%mELtsNSY7w)M!<&~mOGw@WE^91 zQs7*q!_3wYcu0ZsTmsLr2Rv<>Qi_V(_1<2JI>7(I)LiLi&it$Oi43fD4+J+o!T=Qb5Pi%@u1@bfvk&Sjt_Z+9NG6Nh<9F;Sm+~=*TAHsz{TWPJ;C52&(a#HD`rnV z)v{Rit~OA-WUtDr?pP_hyCMh@*+m$^>*i2p<_``6F~`^< z8g^vtG27N{R&dqwQIGw;i3a`KIJ=LT-%C4nx4ouAIdqA6&@Wx)Y5eR(3tn7u5dT-@ z=W|fDs8GPbp}s+MseZ!w#Wkl258vMT-Y7L8Lo0!O&O*T@3XCtD8B7{Dmn~F!;-Bv? z*{r?8^Kxjx^Q`D4ihG{knf5Z~?VCS))q4}a+Zb@}tL2eP2w!lX`LgfJXdin%jt>qz zJr3M1i4uK^-G2^>pE@A+Z@Ef}!b9O>31aDimyPF&tW$o-o^xQH?t^2&clxqxcdl~o6ECT57t7K)8&vRk+PB61_csP`Nd+n^vNter zd(w#+X0l_2C0(Q>-l`>Uh;uZ17J9z6Ha=G6B&4R1MjBMCe`47_!W-6_XqITk;>@tgwhn3jEGlvu#R=fLFRbo;S<_%%Uhg9Gdzb~gAf zVDVuRl1tBt#cciUwsN0dIBsO0EtfZML*?b>8r1>`6Gdh^3p~G`ns?=47?A^Gs5Iq$oNcV_)plUl+0K%Q9}& z>TNoX#o9A=S8P2u$8fRhlx0dvA8M>jI4&;V+-$Sb*V`j^MU)$tg@=;H`Cb7Pr;Y;? z*e}UBESBS}+Opi8o12%(BICh=CT4bV9hV&i{|+8}!ojB#&}S3*nVoySu617kqf_%q z@rW3U!p4%8%e)F69RiMrlDQSM0yY>Z9B5UGpAoUbKrKK{P;?)Mkcf(7Cp*7@O9cZX zQ`(tr^4=>RKKE=EQeQRQ;bVq?WZd3UOE*^sT+m2gl5|C4Q?gROtl-P=l%S7`T)XqX z7z$axzLC7Z>*t;)LiT@lsJQUxoDoUnw0=89m|HOB$K*EWpDmYt&2>c&1{wXCz^>pu zLy=2;ZO*1PsdYClozk7%V9Khpn8%1CYQ>VN)6{b-g-&Tkiv%?A9qV;yoT!-cfJLS9 zi|Ps9dV`A&tk(_GBvU7BT^5<$dr?g?ZBp;do9Xfv4gVO^c;~oDX6r1t5H-8oQ$ae7 zFCc-LxB5(g14Ai?Qj4rTheNZ%qyz_MmAa5e>@4C69~$QJ?_qOf5(t?fmBM!>ft^qE zh`@1S%{dbq71}xua_jF1*v!tc%%|Byh5f9+MQ4`)lOrz9UoK>?|6iJN(XBopsn3&r z-4~(eGoKE-@fgo_2xifa$OvxJE|{PjAj@Vd5@gYOdJ2!(n~m;*-Ythts=Y3|ll}7+EELnq&vk5MP-bD`;1d&YXk=6HXiQdYauGT)B}&NQd2+-w zO|^Obr8zTG=4YPC%wD!|n`Cazw4C(Z{=aG*rm;L18sy7u{#{_==1URa&@|2{2+-_J zQ8>V?uF!CRl~bjlkzuCY#^d}t2^Ru{^PdEEnVM(piV?1vbI@I{;st}JX48~MXG}l` z%--7V_2t6)y}y>13FPp6Jf?JM&f#xX5%_1mx%_WhuJ$iSrzisLX-43^l zDtW;&7g8R{n~O|5*0r|e9H-9F8%f95BzziA?6t1@lsv&J^)XxzM+UJawOF4>iY0 zObcBi|8mG|W-}LFAj9XZ+HqiO!~$JDn>^=9-7y^wSElgaP~fU#aN_8hz!|vZQSpWh zR|AVe7XFk5&Ucb7GUqm}*AqGvpY_p2*QSuo<%I%&(~ZMN7!=vm4H%P*7dWs*9bow< zxWJIFf+1OtS@KFvgORAk*`#&fC2l3WoY1Dyn=|9Pn!~o0CR;ntCQChNQe#=f$t=O3 zA~mr=$|aFaV}XN6$wQXlBZ*w+3feR^0;NQMJmi;bXo$b}fg>}(Nx<^OKSx;x#m*R& zB-8UYvy}!`TfA1l$0?!0_ zg+SiDJ3qA6M^E7SuE-+Kaa3LMQld@vBHiPR6P@oa>g1nkw21%30?xV*2ie^=uo|~K z5}V_3v}=_^D_6z=&Sr_Dipv_N87xW^n8_iel_l6^?6HXR=s{*V9y2zr3kw96o?sR> z3Sd;b!N`0*dvdmhu_Wgfw$$s@y{SBvm!Ixr^S-nD<;jfVy{}HR8H%28R9vCh!7Xrr zFGZP0MB(AQAcKW<^Di7zUlP!zuA;!lebIq+g;uMDOOsIVfkx@K23>()j5tLY9RJC6 ztYi{SdB8uvMw0EkBd3?fVxb1!oCNj2PEV6Xf>)0`Q8`sX4I2>uW zQEaP#lE=auE*d8oI1M`%3RD(2$sN1E`j9cHZt01m$`?31*l&a^TYGTQ7Y}2P>pt(T zF29b*-K2e)BO$@mq}Ml7CPDUQ!+A!93`SlphXzKD2mBK%T6m={hzA)sG6Yrhn=PBr zz>{!*ZNiK;&5(e`<{yombt}C2iw>~ddb^N2S%C4=vtR653l=c7Rlw~^@IO;Yu^B!a1%K33fBwd0<->0!R@Xf>YWezN^<)@?X2`TCB z5p~twHi5&&g;6L+=CH^K1GeukwsVT9NKZR-q=6%aD`H6f%oIBNxk|_;r#tIEbIVLnPurx3-cQ7zXJYev&Jt$ci5zltOiZ{ahpk(h1 zCz%iG4aZzKnMFMqHUv&+vI$uv7$VwYyvdN+!D6w%gaS9Kb&k22G0aD=o+#@I?n%n5 zus(X_T%q-ie~SGlRvnS&VlvwNPJu;6h3M57=uq zG_x&9;EL~2I@{vVa@Fo5`|q9V?khjDh}$m=H2l@X)4|{(q`=7K;qpj0_p60c9>W{4 zCkxn)W(zOltz~rCzUS$LEq?y{tAkHBtgmilkq-KnJ1^3i*+`u&@%0#29FsyJZhBqbH1@?sr_3V z@S*8kV}O&wI|WwN7Y>|e8yJOKj<;B>xN^XlgYoOC%SG3wEO>e*-Q?7vkk2m_Lzy;s zTAjUVa^mQPG_w=QjC`*iFtK0R{?a$!;j(H*k)nnQaRbY`DVqeI6#ldGf6Jyf_j+G+pS`3`|6BzOv>U#}1Dh)V#CzPGL(2#zZ&#H}2 z`Uc;U9gQy%_?|E565i3c!b9QH2mYGL{ND^J?7k(pJ!h{`V9}09aY?LfORk*CUTHH? z_-Kc)$RX#;O~OJ8n%zG(H#smX9AKTe!0X5t4h}}npbzZx3j`J`a3mLSv>b^3y-3-r zi7l#-Ew+H|m>@@h0PCa$+_x1tlO`}$3jE`=5UOexV66yX787>VNS1ADQ(ZDKc6y`a zC1cMwQ-hnYFqoAJPgy8+{^x!$ zPB_l!{gBD7fxXp$DNmBM@B>?j0h5;jo1X(~$O0yYbhG&etd$0=ZJw;&52klTvV4=P7A*EG&X@LZt2Tfk<8e< zOTooX9#;(~@;Wf+EMV@M?x|zIZ0aDWw6g96J3}j5hjORp%t;K(Ch!`}EYYd#yeUvt zn83CD8IRTnmei^2g%iqVYOwknnANdQePfXKVCEFJpHuufr@r36x6xwi)`+RC=1ul$ zoYN*W#db2k3lsUk7BzXg;)Lt-XPZ`PU+7V8QnXN z7Dm?xj8X+I$;>9rA7-{zdif>H2sU6#G++y4;+QL8)1JW5uz+1*0;7TfvyTF6lK{KV z0u6x{6W+*K8jq!#rUd6Ma-Uc>>%OS}<5pw)sP+fS!T%~aSOgoGs{=SEG=$kSOsacP z!h4{OQGt=;fsF1*QnLtihf|=t?FFCQL&nNJ`p1}7e zVfllZ%UKpJf1x=w_$B|O*1Ryz`9A|V`x7`BpR>ktvV4%-s;HCh(PAikG0c-PuW&E?*pGMLP{|vZ~Ge)_%RYc6p=vOg70?hLRGCGWjB9HzZ0b zEMP7zC}El)@sUgRjr#g`>I}M>8JrCag%>z9940SqVA3^UE>GZX_}9SnEuGoglK0vI zCXEKh*TU@Q82FkTSkHZ6oBV?3`UbvN4hki^HdfEtC^>uH`w#rDB_kVV@o&>;I#ivv z(}AzGk*?VCnmn_v59m{H7j`Q`63IR2nT{eqg~Avw(G>0nd>D zo+Ac43l+FB0$h!x>LnT&-*qucO<;^q(h_&rI^|<#WdMitu0^pQShZT2#I+V5`pzQZ zFzbSIdrzRG?&QTwSFYvH*?xA0l(NultE+KVrVNS=?U|1leAOA{|1hu%$e!Q6X08T@ zY6F8n0|W1ao!`=V>n|{0Iv{Y)f%n>h5RC^*&n56`3gl=7EOVLR6*iTze?!hc(ddoU zw>G}C+|3fSQB0eE_wUAuPj_#v$P1gpe^h&ZjX*&a1Dn?FO->i4c605GdzokWfa(1+ zrmMFryb?v0ePuK}z*1Gf=C*-#_6P1G0{m5P_#zwF76!1%J=k!rfXgI{*;JvXA%SsL z4)0Nd68o+=sscMmut7qHHnz*!f$>@ouflZdL|RAx)%viNYDsBT8f z1?+!TFH%lmoSn&RykX5RaaMl;?~+SuVxPCk9hNAYoq2oL+KFYhRa+K|%$arLd+`l- z2B8Ver3cubKA*^vkj2-j$x*||LCby{)`~nQ?H!fY;Hfeg&9QFq?0uE)K zlbP=Lu=Y-1kAKd-)Pc8dI$zl=zRl6g-#hFUmp-@i_3}47n}YY;_?V;cU4eIr0_U$i zM;XNnPQ5ex&YFwkVY*Ky?|L&xSrDYZa|?*^tm@W-SXB)-(>RdOxW~mO?S!Wq`YGpuv(ZtBi@eEQuc$2j(~1-;%ET;?=9nY)2AsAXSB zLi9!>jzh7#lmb{37Mz&2MpLhL|1_R|OlBWggD$XGGVDsc#~i3o+JEgWUeNJhlOh2Xo>W85o{3Gf#|u;IFRf z<|;cwPp&JP$&;Cp>)o=Y1{_)w7^){E+)X%qZVGSB^jGU5KAZV>t*x6!-8`cCb5MMkgmv?;5ugMpoa@%%&v_5voQH9IaaGUmrJRySlf8<;s&hEM!(_VR__d-Hi0N&l%a z=e?D{do=%#@%%rg>$_I$m~x+U-s=Ug+od>M)_0d3IA3MJUd8a(i~mA|0sE*f4>?Rm8*1bE;5@;y=e`79j{jmR&1vWE z>^%DR+*ar5-BXshwCO}XdVJhJ&ARSQ$32~1>c zVdc@~T692RX^5uBk`2u(E;y)qOp{_*1spTQZP$0mAy#cNIVS}H!9S>$(L-ga%*gcSmUOC%mo_P6<{G9}=` zjieS$E=^bIEU%eK{a?K_StbOz8M+F1+_;_G%;hknQPY!QV`Ha-(k8{eK(Cc1{f?g& zENuMoB>6<>DzjS}jT|qGKd7)uS2QrPTO=_s%~08~Vqa{yk-))OVL=mG`DF?m7!30y z7}o_dBxq&2K++QRZPS!<4j!gt2?3LS#0g#u)_wR^Q@FkKC@>r`HAq0ref>)$sG z&DQ&0Th~J~#5%%q*2@K9k&axLG!zwNMWBu|8vRmhYOM z@FG+5hc=aF*4}YvyKR=AI4qL0<>FD1_^OAj(;eL|9+S+iI=5KBV@X3~SE!=n6X9>i zPCQ_G|4G4Bz|p7hkch{hj%1BsFH4pQHm?>mDrTe?m9F~XAT-U9QSuMJYO%P^O69&U z%%1jhR1S2p$_EHEv<7z?am}_6y6~abqw2(hDQ-C`cNl~vH5xyNUX}IMIT5TP!>T#u zv=zUoQO9DTpk;!)ggg`s_fGOqTF4^g6_Nh8)GKXy?YFw@`L>BI(*x6Hb||ll5c1Y} zE%fiq*<*ixBpV#yINUcaxq5AiPF(QW?pN1SjL%*El96SbGv}l0><`s1TPx!Zr%_?|7uV zfrC?b(Yulr!LhHj4Bd>TZ4^4W;>aO^Y202OPC{)5B0ie6*-YrwF^f^~%}z;{R`{^7 z|GKfHz#E2toLPU)NZUKBDDltq2=-Cr>+ndBonbJ=(IYXdRD)GcYQcUR0nWy->K27O zhgP=>`D`L58vOJgHE&y6M(62DT|nasg(zX=Y3!kJ@Sy3)1l#U!j(lVG6!6BjFQ6ij{8apZbFAgpWIaR zT^uQVIxTp%{h>_LdLEU$HP2bCSr!S*_qcz?ud&y~H+hMzHLL0J2K{eW5=(TaOquX4 zppj>R0^jGR7KLRG=KXVXaAIvrn8Fdjd61*y?NsrLCz$P&9GMm#ao}>g;KiiUC=_v2 zg=dGTd)ZN;m3y8rZJ+djHSI%-yMQW}K)^wHzokrvGBubP1Q>*aK1^Jt)DRQh*bwsO z)jHnJ&y2B+3^F||eeBY--H=RRW zVRKxRoqnIM^K0tOt4Tg~AZ5>!xt*r@S-iO#tBmY_1)6T3n;~E9&@64>768YBJ>fdRhDz=n?u-3B9J%Hl z&eQa8MCPIDMMBR^Qsc#*&GrMohv^v@bEY`NeHya zW)$plf3j5MT1S!^r#f3sLbJ?&iRPe5Ozd7B%|d00T(O@XEb8mI;iBVofb+_trt>Qk znAHOux%5nXt*$h%+q^iqyWOBg{(?1!={-qi zV*y6CZ$9s&4OIChJUlp}4zQ$KSn21Nv6+4Q^f{A-iCKcPlX=mBm$o*EUHKkfObiNN z84MP@Vq|b&j#FS^SQEu4Z^p2Oc|tbhm4tsB8^TQ2Ng6b4eJsVOv!j`rsesYOhRJG$ zP>@LBLhjfpiyD;~T$F?wyEPpa95LwVHc5%$I_j@1J>kFsw@=?4_J|rXs{Lp&y%a8@ z_GN+XQ4DbQU0xq-vx!@)i60xjwr9v<8F-dy?}L%ZFbh5WT0 z%pXJr+g&vxjV7<`nZ7}WZR$^l$A$`wf-wrrk}mUqT|3^$Q_#@b9en4}-RV2{Rn{>J zv^22g>A9*rQs}VdND?qRz);~(@O(!10%xrijKM61P0cL~ES4HeJ0hmqDjcwRF~_FE zz|ljWDPt+u3`Xk=$%#ys%>HFb!g6Nb3C8xPjZ6A%KA)VfSio21S7ujspj_ zBr`A-Fl>`%nEs%VO@e_*g5l3h#syg|(i<41Ez~q6SkpWB1s^awDX=&QG)sCkh^=UJ zS73epiLdcM^H+&hw+GXMJ~i)YYG7_?*x=D1GJzxgM~mMnW{HlLpcict2O2Ia?-f(n zD_-C%QPC)Cuvg+n|(*}^rIzMq=jXZh=7AyR-<_OiYc=Cuo=o9I@hA;}#*!z{363%!mFEru5wekL|)2rxS+ zG#fLtI{s+7JG*h-F2;Sc*@PXor*iU>%1K#=^It;z@Yie-h*={37{e>%W9!RKVjoUNOE#27Z7ma!I}z+%wJ;yj^&VS%H~ zj$XY7%(4p_ojRCRFR>MBwD3xB$%!x;1~5A@v>IsW&zdBq<igTP`IC8SVvq(Ncxiw}(GrtAnCGAG>8CN_-{XgsYOEft9J`J%9@&D4o_$}qU z?2I<;nLWR@v^*A+ymep)+tPjWbQ(Dl7zHmaV2xzp+jTW2l!2G))Vz{L`5#Oc|D>25 zELvO?oRV!@oE_G>WHc?f**x`9>%zb$!-kf?2`wUb+oUoAJv*;8|7g%V(VC{*YSX~Y zx{5*SLaPodi!(=mlZAB7E778j+RO$Fp~(!#4~DZIxL&n)b+zkm$r1*=3ymrlnpHFy z*cBLpNQFf?eDR%qa{V0H*#j$vToUeLvMgL%IR*Ky$% zDGnh`4{p&3O&TW{7zNx*E*@I)VH(Q?29FGuZ9**`7h3f>{KXbr@rZ5gn{#D?%-$uX zhh-}c`6Ov7Ih6Kb&nS31FX&+ zEiM}x3^iEnCWv@2G$vnZbUx7j4xz-C)#4{!IJl&ZTDYRv3s}k1K1T# zFxuT<76@Q+;9ybR!Klu$$3=iyFM~>|pNbE7qu zq1DiXNxh&cc@IOmCxg7!^q|+GEFTtL6McOBtQs@NK_Ny41_nmE1{UWR$6YdfttT)! z2(Z`+Fgq)>=m{`8Ry6gBGH@j{y;p5jSTRBW0ke@rtVqZzdkxlYmpT~=8l)OpauQ4w zXR;MIu&De|%Wq*wv+sz&mejs@nC1+&QP|9hS;N>v7(9Fk{LKN8r2>&J1Q`{ z?1(P^!J3!%F!x7mZa};71139zmdHC^f;$?OBD@7Q1aSSZj}c)J=WWsCVsSKJad2q4 z{4$uggUNxR#rVYy#|@9&e>Ae3;5alhEl}E>=grj9N5c94Z4i=7<~d`v<`iS&)y76y zM)eJi@)=C30-3S_Og5EkSQHrARxt9MX!PKi&bFe_K!MTb0CQ2`#M!3;#ilSYdNeBh zZ26Zc?`W}ls=>r{6-(7m><;Q^%9mt~)nMsc67natQAEIhf(@fc1>>4;@l0m6E4QY8 z31MWO=FIA6@o1yxzvnNNU0H6=`*L3B0fU%fH0#5JqKAt7YSS4#qPMa#3ahq=l(iSV zNE3h7maEVnbD&A%0kaebW8(=1p06)HX>;>zV6;AvXg!UC$wh%h>AFP*W1B#ujX;Kj z1T%lf{`j3;s~2=-{^=6g5-u8YH#)fC#Y(E61U2AN6%Orb%p=d4_ z;|hj9KN-C~G_TBN6j;$HA;EOw-~nrmR)qwPtpyHV7X-Hn#q)G9^DuOZO>AH>U@$Y_ zVpdt=wt~UFfhD$s<+m5(d!F;5YW`tn{uWD8qs$nY&%GCU5%Se!FZbi8zf7`?BZUcRMLaV_84p2jkY2U_VORE`JaC%dco=}(P${a z^7;^ih)2i04Gi1{96SYu?<*E;S|B>>RKw&Ag_;)4CNClcJzmC~;yQFt|LWB6<2qT# zzdRDU$#pz&_ml&S{}(g1@G^u)Mobd3WO}iY>xTei2h)#kDWTaubyvGs7HkWYX|`B! zS~`L+J%ig8%iIfaC%XRl)RO^R{Q)p{&slxZ>bpaqM1T6$$?!1}AZJ)=3kKVUqN%~`={ ze(d(_ozd|hTB9=_=BcseX|(4aV3qy%fGI$cgC~J;PI8XGgpbQ+ehg`1;H_A&xJn_T z^Cs7VrmvCmsuwt?zgWPx?_Qnir`@N%$q6OP2YuS@R;us8q{qQ59>A^tH+Pa>li-R5 zYk?Ln4)0|hO=2A_ZUPOfgpbEP>=J8i3i{EQS;1aV!JeDYv~0$?uM=iv?r2`GCC)L! z>Pq08dArWpop1G6a9DiD{ok_wljgh<7r12};{30#WZkl;Rlfc!Sqs**E)LmiWHvFe z=l1y}U)FPUTwT@Fxm!{^c)vE2#6%RbxFvEZjOLg(6L;6J@llZ?a+_qm^ zzph&Kx*MN_oVNyxbwMLThJv18^BtcimW(xGGoEisViGwZ&KQsp=sfX*8-rXxRyqUw z$@6TQ4J_FUns_{x2kmOJirD(|633s~jW!84)3-2to?y-VVQZGbtmJTBwBhfKC96Bo;7Bn^U@F^&G1StM%?kInCLH#=SA(O7XrQ&MyKR?}UH0^ptC!FNmwukePlu25N$AHlaGY#pcIOe-NvZh}(B#_5A?W4Nq43Z+B znO$g+b9tQO5gsnFZPpSW4m74;@aEOi$*^1G#Olp&<6zI=(8Bod;^O#<_em_g0?Ilm zKbTjZ6ApT_<+%JgA7%$%PL~3QKmnBt4!jdNgxD1dpABa1-4>$4^voeduv(`II>F|sVE{c$rx&@(|QYu}ttQQ7M@etV|5 zY{ns9rJZkzIwZ_GKAzB+@TAd^tu-tlgRLcQf+Oo;)nBd(`#ss`G3%TuU}O}1!tTV% zawdS8joIV^!_4`;=C@cfl@1(O_0;l9LlwKg5fNs-E3XQX(v%=i z%_$q6_-I~^czlM_)pi0y^;6Y`CWT81*$+%^`ZpOvhxD?Ts(Bwh_DM| zo)8=LOm&7(ffyr8>MSj(SzL1h4vXwMP}!BU?8rB%Q@0;XI&ISaz|mE1DFaidf)~$n z#)X}HRKc)GS2 zGdR3Z!{IZhB4y=- zjfTGka^4&`CgA+#fgAr`{y)|H-fDk1rpQTt+mPh&6HVi({*yTMW!#%RApRg zJTvXqcGDimRcKAlv^;CXaXy<}Dqm+6iNO=mTBS~8v#spOf&ezuiE=_qgWgEuW4 z6-FIu28=>svmM#`CO6t{Ileke< zypm-8g5}n7UmDma2Hcn|cBSXJeCyRx&NkWzp%K@+^A)Dz){U0@OYH~XXYGdxz7{YU0(_w)1TzmlNJ*DdeOul zuacv25Mrh;>#_aE?V6Jz2ktYEqr zV!)C9W1%x|lY=1h?5zw(8l?_qv9JjVux5X7XyvrL&2&m~^%_S8HiH3Z;xMPFCd473&`yBqci#f8mzR&4sa))iRanFwBygmK%cj+B29w}NXBO{e z6WQm_ND?VrutD>i0;`(ALcWj&mrZVrNz*tMv7gjBD%`}Zw83ye2S3*VffbEwG*TQG zs%Nm>P)T^KuI^&{D)N*;V!)JpzTHB-9*kV`1P*iDdD0O&FXtx9r#yB4vQrl)$)v8G z6R@H7b&>Z^BNhXO&X^b9wtorueDLO)53R~`b|lE$-R-oh$J1w#fB~?x{k8R*I{mt@EVTFUtg1Q8@kQ4cu6&rh9UL=XP7BKOa zIko=c-7DBI!RcSrTE;z(*3_NQI;z1E$dPhCfvYefgm;kut4P8E_8<-??wn5xPPM+6 zb^VQFviHRfo)rm)JF0co?6~vxSD(f8hvJM^S`T zk;HT3#*C=T4&FVLtzwqP!Zt{r@n~S3o0ziFW}0p9RR#W4bC}rX2s?eOYcLO!IHFl{ zk=ye_l2qG+CRUdQM!^JkX2}nB3k?=9iWqn?i+-qM`xi8mU2}>Oe^W<`h{6IU_rL`f z2Uz?A)10(Q_!?$iPjT$N;;8xO0B-~{*9=((g99uY4R@YK^ld%v^}~_P!O4sxF3_XV zG@#jJ!7SrF3@kgMr~1ykt~1Zk#Koa^n!RAi(}UBF-MGdXav*ep^Yfo!FBY{@X^rlDKc%osp29DoYO4YJ|AJ2kl>SA(C~ih0rrd@ zF{b#68Ei9M8o18fYOa*$n!_NLc8qmJgXn^(=RBG80-6;&8d)0{&L=R`&$wTpBERTG z;^RFB)!*=|{Am<1VG{muW}ysMLCS>{F%3Q@4lFDOGFm*`SQ`E@{9#yO!^mcGfQO@j zndQK&1;<$=M5jNMa8x-UYvIhw#iSYFv~EfO|Cxq$M_NQ4N@u#>%b4+C)s&>EUxS@e zc^}$1bTTA6|LkqfkhA{dJacN~p^0&P_ni43HQU)dHfB?NHS?zH(JAr_he|8&lzE;> zc3_)u;J}H*hf0M9lnNDt9UPQv-B?Z>Xqb6OLF6EtK>PI6V|h07=N{nKd())&f{`_6 z*0--cixV2Se)uf@;lpO&sBFU2!P_KqBZ_r_!{QsFix)I7$2gqIoN(PFn){D~h`@wv z2N`9qG+wK1^sPL|zK7kZWcFm!**#Yoxn6L!Ej%Enz^&4X&CoFXY>tOtz%E0^M*tbT9MLJwgVqLMZ7aRT_Y+BhM@6yPXvn*xN0d@;M<*Z}O6S_=$ zy_gR;C_iDcSed5j(Y*CZlS#&DlN*dH862FZxKF++tis~R7J5M1#9ic%i_9L!2et>> zR|Gs^oZ`Rh)Z?k{Q%&QXYQ3MK9Krlt^|2AtLgRM|VNt5~FAi7qJrDht zrD|uPEO5M9A}veE~^Dt?%rKiw8_im$)b)m$F~biDRvX= zjCjsz;jt^?u=NS%=NBJ@JX^q*lv??TdH%PRl}V~qkD67sJP(f&|5UP=|H|{IZ3nnt zJg=A-^KX%BL%rUtpgDKaYZ!zb?yE0glFGQ$l;DgojG9B<-n10fOSiQutbx}2T@fACi#w2EB?%@v2b9H zaZu)HVhuT9@0G%OgF!qcgN=bvmE#brjRTY3J!cIEmLH8KcMdB1G@C4O=4Cm+dc;xL zrfH+eA#RzAjt98bJWz8qIU=}`%lVDAZ%E|snJ-HYiIqAvaPCnEnbK{%fZ2Kt^V~~^ z?boQ!d-gK)U1sH^dcq)o$3ZS@wZkVjxgA0(GE7P; zOz|_D6dDdh2Wm5>9AHmjsDE*Q;|jx#V+`Cp4J!hYm#?%9?UBt?`^Jdt`@OoZh-q z)A`PE#lCy!svfj$CI8<04=mT@C5Fs=_@Mk%=KQQR^VjLkFIyAJ)TScgbV966{*R!< z18xBxMm_(H`!Oj-xr zzI;3+q2kE*;QY7F_%^lqZ2xA&vuQBKTQ&;4X!zCUsHo7C)Ws;@(a86tJ^uweM~g%1 z7KVRu2iUKOb1*bYTsh$CKbCwNVwSC^_nLX#D=8r=r4owm}Vq|;Xt8hBW^&I3`a7^J1qk>42rpX75 zMkWQ0CdHlxg*}Ju_c&WPH1XARvTktr#~stKq|Sl6#)0>RdUoGvloXX3=YaYX}mD!miE#9kD^SMP70arX;!Uvc2_vW`@zwC#bGz^L+lX_OQtm_ zyEiZw7_zq}It-YPd}4G6 z3+O)k@B)iId+7nG&HHC5JovYX^PTq0RRsr}0uL^FxjDzz*;It(N{?}dvzbM+rH8Tg zi);3GUYh>jAn=~qe#2I?&(84yPO2eIDi%$uI!^L`4$9nU6xiU{Kj&!Vilf|LrYIgg zkZhqSu2Lq^!YI+;C{f`kp;9KUQYNJ0XxshjTwT+t8=YVNeGcsvx*K=m$ENG*F4@XH z2Q^Hb1yHty%S7 zxa1s{Gw0&s1*>_p4_l@jw(K!Ay}Z))$%1)l#nIcp+Wq-n|9NFJcZJ>#zgYW2N)b(a z@0^x-bAqp@mFLZY1m^eF^~rn*o=Mhb5;qzpdY-m*H%eH1OD=YlC^;zc#8HajkotoD zwA-A&1P-aoG-VM==1#Q&$nh2g*& zi32P@a?5s~a#~BIY;q`hR|L>%q%e z>c`fZM*CYe>`pTYXp(x^v3Evs^MBLGJI^D1*u<|hoJz9jY;Tl((Rg^z^J&|69k%F^ zJ7U!_nNjr4fwR7h>z-Gwb3Z8Yq(S%zgRn)T$eUH&QEL?f8o4|et2qzJaU2qHVf>}l z$lKGXVB*MTp~mroL7=8V+3mQJWfBX+LFFHe>^uzYA`I*$4GcUEOmiIgYf^+697Q-7 zMO+v~ZZL}IB+72M{V;?tP~Mb5WdWnZB8L}0y}HVU0{xy!ZS|j5{*3Ljp4uP%G9H7n zGpq*~Rcf}HHhi_z@#cH|;qbSxy7}*pu6wuS`}XMhHPP|Ty)K7~Zf;e2^6!MqhJ!L& z82g{FMs3-g7;j-)+$f&&V>Nq|Vo6K*ArVCZEuT^rg^q)~G3{qHf1ka%OXyAmZ-6ah zA_Hs60k#x|&^a#@el&V=916eLFPzd)cHsc8g`FM9 zcsTqQbztOhV2ohsv+7_FaA0g{D6u#SH zs4{3daBb&v<||X5-p*+7HL0EXGW~#X${w?fX444gMT-wFn%~Uhw8pk)4}bUbDDEBa z-aDJ_@LP7grmv+nV#`53f%?cftP?yG`C@j6*D%U?m``SIlH02Ob^G%LVfL4VD>?q1 zap0?QlruQUUvRb4*m3?1$K#@pwsy-4`!EUzxc)aeq`-3GUw}hfAA|4>2dDG~2?wV6 zwFlkZ*&Rd*5)L{#vJ2WpY!Fa7E^b)8%R+!5v2Cti6;Gq$jR!1TlDtd?0+K;bni&)% zSr`l&8W}kG1q=cNf*YCG*mI;-bS9`Z&9KYYvN-VI`PmulU2?6FK~EaldE_iwe?4Gi zS*@xNv*QEvr7byE)?{8T^P9c7dF1To^RjxGSWQ&%+~M#r$d+rm zO2C4D9^On>SdR2am9luAJebn2;L*flSSh6HFCrkSk#b@IV>6eC)|~SJjfz|fYLb&) z??_DcpC{dVWhJxtx|5DvY_c8!{EwPjkAdFvW)KpPA1qJYcxeNlZJ>V7L4TiMtg zG}Raw5)v9%qy-+ROmzzCRA1CGX{FlCj;^j13oTS$E#sf~NnqtdfetOL<#R4wII>)5 z16Pxfmc>IRX@x?!*&CK^RMROqWTh^>^;%N;-1;|~MeBCG+qGK1?%%D~>uXu0*GRfc z1mlf@uo3lq=A_njVw_Uo?Ok{!1zdkBdDQa)daZ-3=EtC5(-U|IZ6~7 znnSz}I5WHSc&cacIvj{xUh}B)@ccffh9k2Res!*5pVO-4xtem*lO!?Ac5`SrxS~}2PrkC^h+-}ahPA!=)eM|r$y%?1LyB# zev;(yA>mP*?Y9i3iJ3|ZS^iCzKc~DRg#C?yi+zw=bnq265iaHl20tz^w{S=~Fg8gT z6eKWAmQo0B(pWMvO)9f!;<1^TUX@CRreruZMDZ*%fvgJI8uyA{hXw#}+Oe5yRIww`BG{Ei-*poBeNKCL!5EVj+)bldil9uBi3I>G#Q zItMGx_TE*goPK}hP1PUwJN4zn9ZDD)+sx};=k>cOd{nl$&+^hSok!=!%;zqmPxW?a zEIRS9P3n`tl2);X1B^WOA2dRjH&;Dr*!wxKs`_>M%Mm%qwbO3G`;Xz%|iBBSn$f zhBeA`Wk7q-5~bx9*)h6{WY%j;XB2T!5}JLQ&FoXC{`tQh9WPF>S=qGmm8@{~_-_WqkGnu zN4Ff3+QrRpuotK5y0?|%|U0`4VFT)#0Ew;ktxe<6x1044lu=7 zIIl9%V^kCJwB$F~DICD(pwS?1+F;>QnYLK^i%pXGwM)FJi2STN>>|p38pZxXiVT+q}9OW zvhctX^Bs-nv^FrVO*t^pT;L(z6erjn6{9h2Uj}=}EY9*xS0ZK?eB5~d zpy6Wqt1sLdxnpu3EaDbxc*(kDyRD0ALb|oHqi-(P-Pd;?aL!0N#J}oyvsB6gwz>sJ z@?DNLu$d%GVf*Re=p?`-Tf)fSVc??1`iM!YM}hsNE=PIp0cn9P3j`*faA43oVB@hz zS@!Y*HkGV^#qZ0GN=}^6CZN=CXgY&Jft(TN_C-O9irAjcHdVaXwIpd#QQze@10zOj zX@*X@tZnn|G90&!)rnGlRlH?!EO+Mb(p#6#vM%H~{b=fjhJUKJqMA-D(d?b=^li7r zFS8wYZnRCFD-#p*GKoiNfoAIL?qFT}AWQ3FmDJo*wlj>j@08fVuxQqV?oy45QVI&8 zED8(cHhC(9g3h-r__!*^=P|dq!T~3%PJ1bl1&s0mGnHHun5AqMv5RIfgv%N7hLdVcktK*V+x!|MS^+nUlCQOt4GND~qaFuKB_C!9% zY0WvhfzjS~#I_&pPIhlR-J75DSo+?NW4rIK&C4rbmfkJl%&}oj>OXhQQ-P;eY8F*K z^Kkpcv(zkJW?6l|=ptGZ>4SKXh0XEPQGc;-uVW$(4QQtmM)aj!a4dli5SK8O0|!@^0a7 zkvMaJ-S`0`Pr?P}m4_G1F=c%2XTzezzpywav{2C6KY-DZ;dbytgR;$2@8+Z_&wiUb z-*8JK%aY8`-Ev<%Ui5FRRkf;7yW_HKMdqnk8w<0a(y!XSKNO6w$YDrg5&KcZWWM4l z+XEBkgtmA4w@)`d6<${@y1{{uN6^;Z;n5z!a=~C#1}42HlH$7;9}%~>AMg3^$3X$# z1&(qH7Hs0Vvp~^ug(K^Uux76hii*rH7#LXtyKIgeFLU*2)--1>(-A`GtA+N8O5Bn3?>thHq|n3cm+W87O}A}#xHM!nUUy8124AOCy! zVEL_dop)E1bk`h|Id_oZrqjYtd3-MvPPFx$IJS0?PFF=mw)700dqVF&9C4Jn*DDjj zD&TNnl^XxfElj69E_~XvX6F*_KoM7FIo4%1{+4M8Yz_^ae-^Nv;AT?NdvryarN%*M zS0eu;1@;{axU(2otllv>FmUa1;Mvu{7IT1aj{?(_4NS8f1l%5+k$S)sbwEH)k!AU* zRb2lLN-l9=p0Pks;321x1A`0$gOozF5CempqtvGbd|ik6J}~fqd%(L(nZMwiRMrDc zk%rj23x%aj4=)ZOpGi+olI;BEG&-V-xfU2QIS?) zWV^IVa}F!xKdDBccNZDe)-%c|Fgrb9JC?v^c3}F9&h;%xY8EM)NgUUYEoJdq6CR|{ z$h1)VlA~@8Km_vV1mnI)p$ z%-a(JPu}V{8NqtW=!5i;Mwxw$S}TKO4mpZ%PZTLR@X;^ew9*1;-w&r|X+N0qn8k#D zx`jXMq6Co}U7Rirock68eM(^4RmeTlYtM_E$y|q+I3C!{V2G>m;J49ZIT0xErXg5~ zk>A0Azwp6IsRq_P1uQ1YoJ<=0UJFIGDDtr#l+@GZ^UIA{qss74K`3eg-);r284DCc za%@->`G4dvcs%;Ur|^$)-%^c?6K2=>H^r$iAL`?mQxrd>XydTd;(p_Sdx~n2Lc%{^ zzU}MixR9h?`b52UQrRwE?;m-KR4(}5IMJu`L32LO$DBZ|rA1nY8)Z%;YF!o<*Y{)B zaM1BI(CIb2@3G;NpFyBpYV|y3)Qv=*2bG0GfzY|;Pkvc=(Mi_}T~Z}H8qYhfr=@jZJ`{iX;<3&SEi=Q}6P z`d;%mIp@>K8@$X9d&PfV(YpNEf9<8-A_v8NMRrexA51XZIYoQvE}i9nScL2tg;pqJ zR53HFoxiA+z?Qn+XV`-O`Xjg4<`hUbe@fhE?Z2mV5Oc@IV*sd~4FfcPc z6bzXnwarnMA(3a!L;j!#ypIl81~lw(>d=sR!Kbm1eZxUXriV-{4h(t=1QZfw_9gPn zNsxNw5XSpl&FP82LDjaPC$HByxhXBk@?eym6fAp>QD%=w$d#?Ce>;UN1!WiI>Xx`z z-P|Iz&M|&YYQQF@9$*M|g=WsM@U9xzQhAn+%FrQm=-+<8U~Z7vT*fsD>+Zx*=y zNobB@X>~GomQY}P)gZKL^*SBVoly#$c@3OX6u9ObM=3sWDTidgotrD_9$GiPPB?8WW459B;$_)O zjWP}ui6;|f&P|q)`6j*Lpa73C(A6ya&*s7YUCtYvyW=d=_1Q7s|%VpEmlcp$Y)C3*lYA7 z(R4$z<&Gl5@~Kx7WfFvn8Lt+ZZx@fSsP+nIGd9!K;AiTZ_}R$cvVfB{?*YHUL6LtB z%ti_ZQ~8+#9Jth0rhN%a+o8ao!N7h*L1+`Bj1nW4oufdW0^gMF?xl?UQtuoc4=6ev zV2Wu7s5@{mfI-0Rpx=&DW_Q{cRT>zbim%8#S9zl#5Em@K@Q{DW@|u54QWjE*9C2q` z*1Xg(dh*9Dr(XKOYd;2UJqE6-A2PSTTD%pT^mUu1d0=OQiZn~P@AWGMvK9t15>g8k z6Q)n9SkiMhLF(QLA@TJ>w&#nr!X=k`OH50>S)^NjS7qVHElE>)3X==7K4t7+s;ErP zwP1`mwX%Mh|;bdkv+fkn&DUSJ_-Lzm@}1=?N@`63vY z?Hpu_6*7aKa(|h~nYQ55uIbq_4q+#r*&pL7KLkAJ@ZhMmWf=fKJ&n}MoV+S`D_#ZiNwyH zY;D;TTD7f`UHnFd_=^+0W?6TOK8h6^CW?n8CPpq#-8t2BN6W?Hy9J3$xIA?9OYP=Y zCu^@SvGh+bP-CnqP7Wi!p1$D|Q}5aTQ1YNemWI3z(uF zu)jON=hw%YqQDl!%Aa|FZ3(*om*PYN26L67J=+49Tvjq}6KtK6$|#q>{Huon%6NZsb~2w(7N`L$CGjK@26XSr>dU(vY7S4 zR_}{T{H3IdYPN+QF7I+bbsPF!1S4#g1hX}vq|Mq}=%7)b<+d1VQ3ci!t>2%=yx7AE` z3M>u_I2#gJ+z#+PI>0t%0cX*9rntO4N4%$aHE6{x%00!pA$2l?PB-V4Z+vwp`PVqI zYAJFtEjT=J(fU=aF$}x4y7(A*_OMCi&b8aERo8H6v*V=Snp=Z+&p#ouWTCO|XVJpa znr)@AOBW|fRvg^csb0~rXS*PO>&457t`@cv?<_B$b@JrF9aCkdl@^Lmc`$A5^fRks z+A?QMzqsj>)Y8uZ(|47sF+W-$B9$oeNP+KNgaB7#Rf`(`2Zf#^2l)RyNRo0~?SHQ5 z3=4nO1LlSX{(qAi*fI_nJvC%MqQDuY;4)i*{oTg(f^%|KCC!<)NJXWg+3c_2CdM1{ z6d3n-*BNdWWLV(Bb$Y|DuYW`)WwAcwt7^!&`qy5aN&Ci9rlJLks)n*jv0ESJG34J_ zGIycWG{vRCRoAQU-Z7|H>?3(^!rdF5H)O9QCeECDa-m%DY`NCCfA5wp3`ubF$N8g7@FQ!8_-G4xA= zPHakZhQ;^8_a$0q8h>9P@aO{BeC{#Seu6Lb0JaHwE#bZTbj zm$0$8@Tswpg-3`ZW5V%BR*!t9Co88g1TD_2XHYR<*sR*b$tGZ6F@Zs=X(gx7mVz)X zKUPtRS3b^xTUA=t_AB{1x0~hOD$cq2FUmOm+^wxUudcFOy?c$+mc2E1Y+IS{%Dz-`KIVSptCOkm(IWI5m>C%Qy| zgQ3vj(Q!`qvtLA**cB2Om>ConGz4>~Y)E8gv@tr%%f!)e(4=XB*d11WlZHdeOXp1G zjn3Yhdd{-&WN3KbIfu<>jBNhB2sb(FaM@?#S&@Hb5tg?nY<`mT#Ny_R^hY6POHS*0l{GCQ-M?%xtxC#-dY+ z8ywlrqz2q6J{KD9z#^ve>mWNH!$fY=oi7$L>n(Y3sFm*>qmUEdl^01K95pV#jkUZqjW#^yvDmaIc5-s;g*P{*_djEEOWEZtHuH&Dw{BQpa$sKC&E&a> z+agmV%f#|6zu4{URZ$FZU^pl$;LyO#Vct-1SkxpzicRXDhoOgNrH5myLg0rq#(7H~ zp1hqWKIedRv6}hyJH<;ia~$~+8YVV#v(`Ld(wx+_uvum2ghkDKoew^7aOypn#4Z2g z;UaE%L5m9w%;Gy7nz?=AI5aNYHS~svSBAq)HjRLV8+bxaJd~bj zqv*j~9&ousVYy?%{Rq(=GkUkzrgQ&@es&@Bdf)R|x04@ruFqaFu}yPl#Nb` z#r*Vp8ue)R&$L$)g863bc+hFUDrB;z;1G+*h7GgwW?edQRK$2=##^Q5l13-@wB{HU zNv!xYhnc5=f#c#G$yf)DX+B%rIoh5madMhSD0QkR6d0VUdf~dto$;TLg+sI4Rp~>= zImGt7TE!H0e3b*kras+=A)k)hCkBg#btJZC@Sk>I);V&4K`z08wO*h_A&H?eYNkqP zQ$d^QCPVRj&w_%M%iQsN%Xb`Ixw_CbsU>mW$8{%bmlZ$$aLM9}sbuZT)3WntpV|9o zsio@MCN`0dzEvSdO_&UP+6Ag_%nJF?(jEAa(_x7tn^l?h{RKJYyR>I6s0)`7NoZhY z|9wENXk&M*$1B0a2@LE<1mtX668I;2u<#Xa<8BOmpwzG7AYOK{*Oo(}^2tIs6_tr5 zG9n2K%mq{BRTngHYBeP zxm{rFn6sSWo+2Zw(}q?(iKIRy4re7dM*r^-hpN|gHSnY~Tu4gYqWQU4BSiVZpVQGdX!*SIXbM`%de_?&m-9 z)+9e&#_I5)_0tt&!3GU}mm>oHMjDAcRSM4Mj})-&YN*K$#kZRt*?*z$t*VUIDNm2xx2|a& zc0begHYanoMcN#rsoP>z^YhnmSe;;8dH05!d6V{?hsz31sGpg%@7+{=ZN;0$3$zuB zot+ym1(Y&}+-qQ86Z3S1SLqDi9S0(+GG^tOJug4=@of0~$T%Ja7Ut)F56NDS^7lMq zEI5gQbq>E`r}LSe^U@b=R$*Z5aBfHvsS;>j_>;lkkfDj+N`ry<&F3bG6Ai4o3)gH^ z@lxb9zEUSO_d=_x)k5|H4XuL{{6Xs^^&zBQ!1f0?x7@8RoxB{e>vinYH+*aMdf$BS@PXOJM3KGb1)J%LgWcV0%J&xCD_7$9`2Opv zujy?JKQzAh?6d1v@`=50kX?u^QSyKS8@uYYKg*nMx~Fwa>{v3v-P~m1suZE{ZL37r zCIm4}U!%cPxuC^liX&(08jEALtG=#DjmeJQ9JGD+yWGBN+jcVs=^F>-%r;ss_kOHq z{qjg@k-xS2%*BtAZi#$g5IUpK<(tzayyC(yHIs>*J~EA`!gap~drw-m+)A*c=>)Ur z3FrqYZ>fPsN|f+v&F zhDN1;M9$zBwSp!J4LlqQ45|&DUNQ>OYmUS`GdgZ*95$yo)T@4K&w;PK<-8?k0*UP3 z*|wQB$@b+1^qrXNy=~(8`(I9Yw8_k87S^|#P;Fe%BpmbHMQuynf1eViH=!RGW_}P! zT{@9z>BO>EY8pWoIObm{Yx=-hX~5C6fxS0?-Bf_J=>w}31Ct2@lU@RQ{Q>s$1kS6@ zyt@<9TMkq-2&DhJR?)VR`}PLzT>}39t_X8Z4B${+!0hmV$v%O-Yy*2+07vBmCf)?a zACnc+gc*3I+ONCr;juAMSb^bxB7>*_bBFXnE@=y6Fm7Y3GoTA zFKZAz8NhjR0#DOIo@|NQZLQJ0m%WN#cv&9HcDT-FJ)!-sdx@)f%$AUpBU?*eig$RL z@x5D6_j-BVodeu%1uUroTniKG6Ss5TY2#Yhz?HB(H#C4H#MR*cakZ>RzIz&(q&}#L z3$r%`a4k)6*e1yCBf!!6fmJ!7S=oWvlYzx{GK*(G__PM@9SmGM0=SFXxFb_~c3z@LjJRZtSSdsQ!cQ29biqL((So`z2?C`r6mrWP78#@g%rPC7Fl&XV4b_7_ltm) z$K4qm7`-O2r3SEu?_dc{X!Q+Yu76|?EF{82|=-h&~EDee52+k%$&P(Q;JGT~J zU!GiifbG_HX2G3Ic?mstzt^^1Otw}{@el8i@vPH35;Iv)|IP;P?-8B9rsvL4;M#X! z(!K-S6FJIeC9uuaU=UMi*uLESo`|7Df^?99VPyc<{0kf%4y=_M*e4v2Z3qZY3J9-b z@GnW@h&jOM+29&^gnRJ;ZjU61y$5*rHt^nh(3HG@*~Eb9ibnYE8?2?QY*rIkEE?GB z97N6<8>^K539!D8V2uu7-DA$`ozUOe*&4Q^ zpFe=9AfavPgtjvi+SpglJUd~+rwPq+H)cts(>Y6G`N@zeze zxOXMY|GJaAKYacIBLmqFth)vHb{lYY8gO(raMV0tNfO{N;^5f4gMCQ@yWAtY`p!b1 z4m0)zRqN6f`JV=CH)4?7z!E!ShKB)b@`HwI299pa{_q)`|1u4jLXBr`4&<7Bl4tUS z34H~N=iTI4_>#lo>0*ni$+ZgX1rF@1PqGQ_WY(UrM0*3%k_Oj{3tXd8c)yF~See$z zc{-h$rh3kd@2LXcoddiJ4wTNmz&Tfe>)-+26PkSY9`HT=zx|0~kCLW_Sd!rYAV`CamdY;GAH<(IeSEGk{I<0E55>mSqWCixd7Wp0jhY^{K_T zPjT4ZTI?XTr04?s>YHpX6WXhGFy$Id^pfJ7{%J|zsdkA>iH8iTCtGJL%$yx~ed#*| zzIO+B&scUI_2fDJlJ}8-KHF*DBOkaIUEo}iz$WLwsFAe1?3A>R0K3Fx=8y$US`Mrd z2F#WU95)?!UuCW=ZR4KkUeRVSZ3zQsdjrQ5sg+gJI1IZvOn;}#2Qa)<(e$|fP-7cEuVssHW+@4FAncy^Z_cHsFa!2h&BHEz|8KfkySAJ~xO9-0~?CD5RL zXklnY6O;c_>pd5lngsZ68gR|h%D6e%wRt*+!UV>bo-5~n;A|=2n6Hr0@_|#on`5Gs z*2P4QsSdn%8+aEd%%8%=(XnC0M+Lq~30$Wo=1)ApZc?EA{6|%0GQ*BrdsCTpLY=p! zF!Xx_ur(`8F*~(2-kS4N0B3;`$Hy6L^8`|@7VMe$U}m25_6Mstilmn~bq8l8swS)X9hEZH>AF8vMzn!R;{p521-v^83TC{j(D$fFN(h^1 zz}3OPxh{Lb!Py-9rT2L7FqyKj{cqW`t6=KOPLAdS+&4e)y)T%0a>l$jrv5v3nccPa z==-we=jDL4;tY}>`n(sg&U?VU_yR}&1CFT%oc#fu)qnP#V&LqIWKGp(aC6|=#lSgV za#n%#_O1gbdhW1yDXgnyI8ZQQmb=u%+sl{uoLb_U#W78wsrq*}zfIG{58FT8N`4q{ z@JrB+C?Wk>g{ti{mYvzjxm952#|Qi$FD!d{;nX7szLyjDju`OVY-0-(NIi6M=fQ_N zvp{Axyg51^aGG*+eiAXY zOki(az`eI1|H`l3oe5lbE*#xez|p3)xzT~yzoTk<7K1PYgO6h*$Aqfst@hiCnBQxx z+4q5O+RB;LJe==eZ=EWzKm0kTv;otu0IpLPIILHl=-I&WZ#7514!g21`y?Iq5{Hwe ztJ_^3%wm7THtWsFwhtVfKAe*Tcy|c!hFUgl=FGAEmMpb%qEoAr1drr90lt5~xF=~G z+NQH}YWJbX3Ve?nPTkqCVP*kmh`>g+W+l!q1}COWK3(IxtC0QX1U{*`^OhapURJ>2 z=D=io+1RgvWAdv_5kGhBO5pPA;aKpGfzyxSy5|Ju*av&|1+3lC&9P?zZ|oWF_FS$7 z1r-e!GPVm{_%Gw(F=MrBhX})kz04OFLJT-UC$MHb;QM!hw?AQVQ{aSZn~Ti}tsV+o zt`8XBJ96g#UaV|;Y0@2bb=`GE4YRcWvgKT0woW)%KKtaXJxnzZPM+MmUBS`hqQR+6 zCs#P==4>{JX+NE_Cv(U2*WBASa2^ienIz56wfc1UCW#e$b{tXQN_1!CQw~*QG)$j; z=G2-p^#Uew0rndWd{-{qSR1`|0RsoC)XGT#9BZ{RcCY51xSji!0^i;byZ3Y-;a!!H zWYF8Z;9?~&*L;QZw=W#s{XlNd0q*~499|9#j0Oz{y%3ptEjet?U)NZ{IS zus6qwcgO2Z)ps{#+H&pjUb|qzb$x5Dy%&z|b>LX+*Yt4;i`WNwi6+GzU1pskTiFY= znHU~N8$8}Vhr6ogF{|Z8(_F5Z0*~hjJZU%JomlXsX!aAQS+{-q{>|EZmu}&2g_H-W4sd&B0?`6C_QA;1DKXaOTFmjgn%LjZ99`M~N;GU(xwY`A*U;_8H z0`4OX?|;1GW1ju~WiOkFCj+yCTE-^>9_ITOm#QsT{XtxSW$6UI8x7a{*k4`kda*Y9 z<96?rSAI3!2;dTbcZ4zTJEGJ(yzfn|Qe>7E_e1bdI0MRE2UJlGeot6`oW&1AK1G;u=oAhzw7MnJ+-qg zy?bJ9z;Y<}z|q>b))wql6P})Ud$RMx$%_m^*1QKpZnfW?F!AB>*(Y07i|adb-f@5b z=RW7lb*_U7JVy$6jz8c#F2LX5&vWp?Ay)$>#{2f$pPMdE{J`$Nku!yvub)?h|J4DD zjF&5aPVL$~!%Dhh?=^$pn@p@X8{U4Y@_+X-hga`6@V0G`xU+$`DvUk;t;m6*zFB)8 zJ*ZyoXIP{B;aH}@p+yQ@M;>r5pRlI-H%I#c?)?w=7B;YWKi~{rz}S&6Gf@BQq?+4j z>b`2pzMXo8IXLd^>#DcK3G9gu|8DwyI~8lYP^WU!%k81R+E?zeJbYDEwWZ@R17H3s z&RGnH<}h%tXAn5lCu&`$Gw)#IL9>OLn-iUxn3yGOR!mTEJjy02Wg^h9@zHTEMQMuy zK}}h+*u@SH|1llq7SRY^oYMD;<(2SRXQTW+Bhg}!Mb7gZn0&vie4!Z7oKYv_m2qWV z`TK32UElrgY;&IQA%x>Y10xd)yMRPO0z*UN#suC4vC;|+47}c_WL7>pJ$HrA_O6}3 zk9sd%61@M8pz^fNck0@bc2k?Syy#atQ7iKGrStNBnKLsVZn=5pn6b1{uG9M)8xNiB zy0k2OjoHJa-Llr}6K_9EKDjN|bLZlRheUIy=V#q+3O>9oH+p;C?L)fi+iUI=zc_!= zx8Kh%c1Pi(b@PlSYX&d(os+<-ttOCo@Y5Wlr3*JXHZXJwE2_+xzo7BxZ6D$Yz;xXaS!ii=c;wOUndj4$Z8UY^_pDrx)H+ zWS?ID$ne}DTOA37#tp}uUa#4D&Wl@j14BXr1Jjd9r+EThIP)cvx)gh6R7|*|EulN% zM$!Zx6M-dD+{7lnIdxKFva`3vlZ?f^7PmgE=~XPtXf`pq7?XR}bgw{Dx8rP9X~hCXRp28TW2Nqh1=Jlw=45OCnmMh1=p zYc`!wdcB50fPsOb<55%+r;5eJ?V^btFTYD9U0|LuS*On*-Up5Ah0 zjj`F0H9O82U-s!eE9%9-#Bn`tHnW-in}C-7TX~Oz%^g@}ODrA={arSDrg*I1T(^^r zB{D@Sf7_&EPZ=!Q?Q7MlDjLzH=`WGk^zh3S!8^(d4L{^64lMdBv7^6{Wy`dy`_~C~ z9N>=&est1dW=<5>%{c;@PbQ`)GO|t*n0?e>bw=Z+01fW>T*qBvgO;#OpTl#q;pzsy z2=9AR*&zpXWS_G-G%{!yIIUAYb;4yzLW`z%}JTZd>EY9k(yXmz>LdeKB?kl3@{Ott{l*6#XK*xdJ_ulZ#l#TPvw^iHb7AnD*W!+M0(n}e z@NuSQ^fMT3VE3{)R64nF?vY;?*epZd7$RWp(dH=u9Oea#9XWPx_;qSUI z`LoK#*)IZ>#oKPRnf5fxxRxec-8gved(pMcZy4V4D1T@WJT-yseNK|_Ob6DAJ;ke( zJbLSdXE@Yw+%RQgYiNjnD9_B&!NBH`z<=|_F_R=FE|(t1sg({azDgfEJrV?jDifNe zZz#-kHCZIsB*3!SWXJY2-b2%QtETDg`Pg6nfJu5cgNS}nAcvFy!xXjvnLRgH87d7M z+^kL|@-a_n;0bsb=kCxXw&yTs;0-5{?GIWcCp=i9D{;`4X+wjYh9akHM5ob|jXY_O z6xweV9!s*;5Ei_X5O8PdGLz+rXPIi+oBJ6ZISw)3%js!nwO4s>*7xGy91JV@CpgHS3-p>R;KZrCph@TxgCdVDhfq_6GuNFDq2>n;PP0AHtl(6kxG=?l z-%F!O}Wffvl?=I-1-CXE@x@kX4Oao`K2J^lX zKmBw~I_G%bI3(6LLFz*rkd3u|I)I}8$(>w;{_f~8dJDPeH z9d%Yt4Oq6^*7W_ni<8|v-e$PFOuS-QljHZj%B?RqfmuNI1p}+ihQ=9E4>=y&9Qv1P z?$G!>L_ulIECbB#|pxgIQ=(fZvH<9i2Bs9l})(uzR^o6zuLeb9fg^fPQHj$70VHj#r;Vnf+wg zwtsW5#G!=tM`t-tZaDb%)2#2uEW}P#-Q2ObCHI)j8VwDiR0>)0;hKBXh=N9%| zP>MK|SFyp=hT$s514dPy#*I^#u8G;^z{nx*zesF{|1%#VHJB+fYh*nx%Z+@1PXE8zzwk31=0)hn;+}8Pm5OS)`}@BQvcwNx;eTwZ@zV zMfE!hP7AKNg{=6e$f?&5!>+*5$hzX%_01KQz(d0F_|9x|k4o8`50CaH9cWYh@!^`YNb2*vY?u1|Txhwf5K*)+6@+ zgg4H*={%{Cr{xHvROfWFm689NST!2$3#?O;+0ruFcFeYxn%q-=YEiePFvp$7y`Ohy zKb|4PVWafGOhKWkLu=2(jgt#DGwXOPw$qq7`QTcw50mdEm}^#Vzkb+8CwYp_X~`){ zd(SGj7yVc~=j0yOgEL=0SS-)eUc%A-`oqD3g!aOO_Gbz^V*)sx7+8W0SPWkK z)7avcz_P*0&Q+j=@wS4?2bN_TEDIS}VjWl%1Q-|GSn#I1tDu7|=)+`>8HZCeTiiFa zxJR%!3$S>9V7C9`Z2yC~JyZ774ra?4%{sf7%^I3rQ;4Tgd6a!17YMS^7rvf{JO54$~eLH~(eQHv7TcZr$Rj(89aQ#3_Kw`2~}J zM2o*SOYi}R;C9?_Qi!-d(!VMfS-nRA*gKlEHQ5WezbV|Yws#2wGbC9a*n%nGdcn9S;( zcv1M7=fUTZ+b1k)%NF3XHgc2LGE*UkZ>m=Nt1~m#Ea|gb)R$?Z_xRDw_lY8NcFrt# z&=%VvV5HD&w7HQ-pwU=^YuCXAZZB5-;*ql2<7`V?s)py}KPN7>uoX+3YmK$?=tS9&yIqF_IG2>d5tN4n*gNGX<|48UaFg*}yvbgBl?R3i4)6s&7uYC1l z>qRqP@El5Aa!yv{wEZEzITvl-E9$xJJf>s4Qe{feoyV-{i97Wc=r7qY!R5wm`xy)D zSG2eV1pj?vxhrv+Rh((lVGZhN36f~# zm>Y0<&zbGsY`rmUeKBo(tQWI5E>19E7l>smUT|eni0{=DUuK_eEdS>0mYVDv;<8q5 z_u2igXI$-I6mJNXeBm2<$1qBTNs&Qvf}-4_Ei0@qPFDA}XzgKhUdWc+aO(79$4@@T zQe%!iymaujgXa1-%4xNHSzC|gi}*i(!PY3v8he55;%Bz}8?8*5TmrwgxE(mexLbfT zGkorC0bZpR!xv1l3z}>+TJ|k$6H@djO9;qZz>>azB~#;yV{eP|(iV>n!;YR4UNcU# z<=!}*!j_uA7O*sA@)sZPy&--_BY0Uug;UR754q?%!_c*0hS=1z7d&oW*x{?teKq3E z$%EZ1WR8aN>li&+BCK{;*E3_DD33@|?!jB#8?>|e($BihTWG$zWXh7Ym+T%X|4Vy% zDofRAppXQ zNlVOx3(jk!TjmCMtzhYya)PTUB-Nv>%hcERZOEb6wsU_@^s$CcFFCnm^6s6Z$|ZoTHspljU7E#cWwo94z@H#_RqLHm5?zYuxN5J2sUl zw3lRD%AOLo!dCB&tX^7alx)hBO(E>14D3@LaL(Lte8GbWb21jJTRJUc;e-+vXBUIk z%onW70+!veH#o1}-s5WKHQ`2QZ%Dw~ zh+>a_zK2)?C;Obe661R~_D-5dpy*sfsne6}970e16*3MKHf#!wINB&NBUI>wL*D5F z(SNs1*r@U7Ad{tmob?8lC3hmHbs8m`@}(TLDT#1O|9Z~$mY39@Q!fgl@}@jY?FpN7 znKj13l4WP7Y~6y{KN9CI;M}!w8t22E?41*;_ULbUbkFW>aB;-;mNqdZ$lqj5x|Dslaq@ zZ9>Pdm?)muhm1{@Gc@A1uCTjsGof|fRn;laYR;vk9y^nrSf#D^Mu2_M+nJvGV(<4} zljwV>xF;(1f^9(JJI;9>oIz?&6~a1M6{qd2RG7Eo(3)K)TYWFoZ)mSu+LArtcBAh7 z6Cqc2Esg%V)MqbmT1xf}&lxw`Qrl7@Vhr|0_=TQ5q8k{P>zh`)l*Oj3+rh(}F=Q%O`*SxQt{Oj=o5T1!So zOH#~4O4>kzOGQyuTUl01MchCpr+S1g-O4ravPEp*`Owz|t)J;}3Ojfl(R?XEy+sV|} z#n#lz*3`wu+RN73#mp+$+BV$E+)rOKPfssOQ##a5u)s$m!cEi7!PLvi%+=Ax$JyG? z(>lo2Hpt5^+}%9L%P7M`Bi+-yEW*4nNUtizxHN~`+|pgo%u7kp*W1R~-P$d{#;w@a z$=W&8%F)NzBh}l*$;;g>#K$Sj(<3_ACnDIx!y_!zDr93+|wLB}Wx;P^vtGv3bv^cLcE+O3B z!KzooaE+FEpR8tEzF23Jet)jfvI?WFM9F2AE|<(b`$7U+vZ5yx`Y$Q*ZOKV*EzfAF zDrv7QpHfpktvGXCQQ?ZPnDhCiuX8G{1tuKVwO-$Cwq&;E(WRD$=a~NdE%xJw-qkDG z8~Xjan=7ZbS5E0HT0N;~^|aU{v%F8O3^=>K;M#(Mw+|z;N;=CbI?8G%XB4!@WHooy zHLt5{`c%<&vAVgsd1_U2Z(Z+#?$(y+oo%zHbhLF%pE+&DqzTiO&gfh|YwG5?(^mI( ztX|Z!X36x;i>Gc`J!Q+1SzFi6-nC}tu1#~cZ=AVpMfao`OI9vjx^?xEotu_z-@0SX zx-D}Sues3E`=Ni*sZG7dcC7lfcHNgnv(6pe{QBaeO}mfHU3X~P(aU?!J-T-7*2z=n zF5h~7>&b`b&t5%y^X>JQKVN_U`}^aQtKV$Z;e%UEyGJ zsgRJ*i46-6TO}xY&GFdyNL)Wbuj=BCi{0-1v37MlH-ndXVl=BzyGd49fKbC#r4h(_QdyZ*d~MqfPr)dLpWwOWO~x;W2&j>E}aU(>=? z#4Oy<=rm31lCIy}teZ|H+*9=O=g#F?wqfE(r+MEl1&Znoz*A+?#CLKL5yu0DxjX<9h-77-Ner}qTa!PO6(Jd)2H*GsP zZCiZV#Mj$YuiVN=x2v?Aqboiqs_lNvyqM?b)%>@s=vGbj7GCDR)q!7Y%kK-S;fw#- zDO&$IS+?fM?q9ZA-zy4r_0pYY8vlRmxhLkug`2bMy=C-rS=1yF?L;IF^K}_so55Vs za4|ui&#j|fmd`HbVcP7*hmskKOC=_{Cv+UV=)R_t@lv+N#3Q#nRcCHv_10NiAzh&Q zaa(lmQ_pbeeDkB~ccT0bhRFEYIewQZ@wF(w8s###J*6mMndnphH>MM>PW79yJ*Di7 z=26#aR_|Em#}rOAaThOrXzE_)e`X?^c%A2^@Pt-LF=jr)6Cav5v}}$k$O)L;SfC&n zl%XLp!*vUc8Hi)hGUz$o)L; zyHoB6|2uxLB_t@`GBsu5EnCaG>tZ|>wW>Huu9S)1;&uH>;Db{4slsk~mQ&U{U!3wv zI5xmrCFaJh_Ghti7YeV(6>mAoD(=L>aU-Gi;Q?lTqc;bdn)*baELb4aB$(myMQ_T4 zwm(L;-K3- zrEDkP(rjTr6>r(Hvudfb!Phbwi??W)M=l9e^WVO7s{a;7?Fl#f%ColJxR-ui=J&;m z%LIx!1dj_(`cjj(%UWi~430iafkQ1!H*cG^h+%fBpO4J3Wg{?1cEmRE)UmoQBHaaEowb)Fxl{4iVe(&W}ztSk!(ac||x6CP7 z(V*k~ho5dQ5+pJfuPtkzP^Gfrp!U>~4_*5E*d}=u>^9lBTy2%g^`0WjLYabt7q28g z%y5)kySiWhX65OfPE+>7t19C*mq9u`D7(6LvLHlbCW%` z*=v2wum6>O=l_Aj#v5Mnq{dw2iU?3!a`(#de1|Jyb!pSv@(*e?GbstY7y8V^S8;){ zp6ek;d%BZ|A4^bQ)Vz*4&nGq2W+n3e4)IJ`a>9RS26z7HhxfCVEPlju-2Lf6u7DPm z{RxKNCEg2_rmxsMH&*wkjaB8Oe_I&lEnc(9<9oMiNz_fRZ5$`{zh;YVoVSU^->cmx zrL)_wt8H0sZMS{iuDkL7j9RR2dit01+-v@@^|-LpiT=#b0iTt=wu&#);OUxjHrqi| z$Yauz2__38?glZfDtq;!ZI9)hhm(SJW-$nOp7~dFP3VjH+*yqm=X*-Y23DQ)3X2o8 zSu6A6|D=`!Yec@-I8-pbjFB`}&tcI!sItg!U&^w@PoMkb%82jW!Qg&S&S9!`9TDM|`Pf$l6}wl|&}OWdX?@2&7tsPGf7x!J3W-fd+kpKds% zt@zqgxZ%M0@0=#n?tPRA*s)=D&7n^kHx5c}o|UR1t1(5R;M64NUK72DTt^|_2`mCp zk{4AP69wvCc#1D6P<|@#*{9+rfBiJE1^X{(#&$SPX!^i$DAUi_$zU#ruQaO%e`Q9( zY^5bRkCe}O9%`Fopg3P3^Rl1Gr;MmiJbnH~D=nsM^k03h>z``ryPdbbg`DTUkl6KX zP4m^?o~~LC3|GHsaIkuGs%XCVjMa{ltwUvAFtArNt>ZnD#Swi+iD$p2cSid%H+CR+1%n#-qfXBeOH*)w2)f&e8~4vQvX6oXPy|kL^kCS~|P6 z%r;&s(%I4$ocpUTNoW13oHa{CRyg!p?^>BA=qY^HN%q}=ht@Y3?cxfU3Olt~bU7M# z*h?4|7EfT7`J=#Y)v>YoqX@J3g$KvX7dZBmCa~zHByu{KHSyVfVB{}oV0P6wl&p85 z(J5rHRH6WL$h}QsMuk^>9^B%;^x#M;Rrzo)Ab4WFm&oTPfPP5Ys0=eh91tWI6; zpR+ocIEgHla8>?wq4hPx!lQN#dVCoVn#8s&U|wqWbz;9xv(lC*R?j>K))d=D&Kw37 z1p_CsV-r}7dlaMwOikfQq_qT`L=HUB# zH>ONm(WJU!!J7x0d@2?dUtrmMOoGE&Ew6FjYG!$x>&Hx9II!pKX>|i#anDhy{O^?m zGhd7XlVAn|i_`^%Df|piHPS85r1KTC>ohR3XdGa%`Oug!{~^zejHBvNf?vGn9OOCt zt$|%DfstG1HvhSA%@WfTWHk!jxNT&9Gi!oooPPanzmgOlmL_(m>Y1@ELXGct-Mp*M zTgk{jwf48~``>~drJG&nv#S66#q;69DVD;2lc!E-x-9ccHELSO)(x$~=L;l)W7y5z zwjQ$$U^%63c~VfK;aRL7!*urr93~r_bq@*lT=*KcSkLYM8O-C z^>dyIH2h@BIoH6ZAAu0KgbXD&y{4A~P6tdFim$+3oBo@O`gZOm0zzoK0bq*r`NT+@ujo}rAB$);k7yUAAfcVcGJn-r~D**}HXD=kcq3aG#MNXT-c zzYC*o#?*#?-;-mszcJe@BpbSA@!eorU6yj;I0N$r2IU!@Gc{Uf?qCqwz#zSXf$Kx# zq8FVTmosR5Z^d0F&pk zI=%E99tTGGf|j-h&S?#784a9n%Q%oFJ&+PI=w7# zS$d9Rs8vJ2dC@HMXAOE67!4IFrIr`UC=^?|Gt7!$m?h3&-qAU$y;DW9+FFBAqjS=- zjz(#Yrg=3g0;Bf>_W1>DZs9e45p4U~SknqPk58}J@_=38 z0i&b=W6}ifs|g(0A6s(@oWB;TRV_@NbipKV6KjTqd+rWB!wLO54ooS>g8D*jz6Lp| zKaosRE-z5$Omdj{O`R=8fbHP1Z0`cr3GAI-%S&V?E$onjFTN}iK6*sE14JExzRG=0b9EC@j{NSBb}3tIJ>65=#rM0tYkUq(915C0}K)$*v~XfxtTEK zwgK-p3*M#dd}lteOE}DXd@_RHU>WZR2Gi#ZC*2sOC$P3HnD@Gp`_2R29S$5bBkL0l z=1*V1={tjai$RXG0^9Zny#FtF{`-86qSV{l#3fQ98YU*$U0`mR*{|=w zl$<^_cC))s5PPD7V!x)~#7OqI1MF-&lf4CU(i+&(FL15%;$G)f;=#_uT2N`|KFj)7 zk)=oVos+ZGPIMWX7jHP;y`j9*zG}_-@10hbOFXT*Od}_2TP`;J*}Yr5nyG-%$ARnO z%%!&rc&}*A)wke1W5BMrYYzVg2EC{>9*1QQC2ja8FbHp8@sjAB-@yC!0N=I??B-io z5*^q)GJ0oB;M`fjCeOeW*}#2IfopGyR?0%w{KC|THlJEemcZh+#>BYhP6@tQGhSbx zp%M}-_9}0J=cUEwQ5-(&)AxaA%jn&CRy6w*>B-bTw=8soUKiySf(td%@r;HTmGmCDs*P`z+?2EMwmK zq3L?T(!0N!^jYLRJrzp4~JtP^2isw-fts$$z&u)fzpq%KhR zyE5CZZ=72iI5QGBcQ&wQcX3~A;NGhu8{M#Cb+S;5sblRlmc~#~kw|4K|{+nxu_3rf_HZgP89DBw%JA8}g z^4angdmOo|`7SW?%{lu`S{on6w|Oaw_mhv^dsR&MUK54d*S;o4=edE4(3ZVr{q2|2+wi)~hIiQwE4 zF>{^Kb%qmf7W=+FdV_niS^A_KvyU#5s7~I)V9UAJ`sAbt%e`zDiew$Q^eVR9`_=SN zpy-wY?>#Brn+G@yFH}n~>_7HO{YLjPeFny;&K-YluX@qIm$E^1NphGx7gx`l6Ke}t z*rgBJ3W>&S5WU)@+hL_w{6PMp(Z+>3r;0)jbvMq4I^4g;Ci3WJ{S~M6CoBCEtWai4 z-Y{?L@6&51a2om6aQxY1^JY_e0?SdhJ$o`5=ZP2ZJHGhjH;Fruj5-sT^%{!ZPc8P3 zIotDe^ZBeLD=dy)d(-(z`k0s1v3(j{F}+ z`!_J3x>hazL2r>XlYRgr%Lc}cs}FR)=G~bP#$_7VP&@C%1kQ^U-0lUelXJL(p1GED zvI?HmU>`CdOnmD<$OcBbv6sM&1oXq0Hy3MdOt@9}tqh0{B?t&b<1oqku*Q|DIvGcyVKx1Lu6z^`ylB+^1)-=TNW(eQo!Qguwm*S={Uxl z;orrCRrF%S*l$m>ZZVwEKCxfrLG!y)3JbI}KDe5FO}bRNLDE=9Fy#Wf{qao(ySXjq zaPMs3elMwXux*Fg+TB|iY7TlAAC$hrQotxPfw|-Y%N9=t#s(> zJb~H$_GYi$TV|WD(Q}_<)VbHTbMN8INmqEf^duQ2Ij8J3pUo1$Ty}utG(*wT+-uhq zn#CtDS}H76*v|2M;c5Q|4pV{sS_O=92`pz{^e}mIhX*`hnGxWiz~&&t;JfJ?vk z@SX{;Uo>#H38c>dx^Tt>?p3_4=?*y$e->R(-y;*i$XLLrcYw9mfXn{F5}pH0We!|t z64+%En9n9$YdyedGi&y3$?DT9XYZX^6#i!O{2!YIChXOc-0HrnQg#A!K@+p6!hNml z)iMWIn-B0@G~m^f;9DBNE^>guRDt94f~5}~c+VW*@kwCrSKz($f%)maB8{$=84EZ! zRB^jqNalF@(Ub8;cmeB_x__KKbu8N(Uhcob)|6428zl63W9c8mpjp{^I}bl>?L4$d zTY$Ck&YN#D5}8+tG(FOq_ef}A>$Y{fcTafzwt;WY2YyaXX*DIb@B^ESy}8#lWCuhZ z|NNF&;sTRd1EXjHbNKP_7Vmb{vFjj-5F#zu#{flICFvL zYy$gv@v3bLmR=6{#huSCpTK_p1CNga!)-}MVF$LeD>&C4I5AI?&Bf6wY`T$iB1`%K z?zI8)=O?hb2Ry#2z?N$ychmHuzW;)g|NQkHrAIg_WZG}gGc9;E>#t+lu^9`r%ofgi zv_$G(;k8|X8M}8L;C?&d-@O7h_RinaBiSt8Y+75u`ZIIOv0V0?2h8OT>=GYXE@bjD z1Zp!ga!I@~neFzt| zq6HT^850jM$;Y(l6l`!<&N7!zM&W{I!m+t&mzG3!f9nvOt7xbcrf@)kxsBg4$KZfN zGw;fR%X`Y^s;!+OVt!@G&4mXRG@f8eJ~wA$i)Q$0?W8R_7bksJ5z+A3^Wf^Gu5fLi z8Ah^}N~@&w^ke63Xk5hFHCa7+p2N(?Dd&XsVxFD(CwOV*Y2iyw^Ef({p12Fh+f{sM zV0_HlEug5-@*$F)f5zRt<=+{T)p;ZgN;__I#{Kc0`}A~o_sin8uywKPQ}92 zShlx9aI>rTcf&0eZ?jxq^K>w=O6gS8KNi%qUMhF@N03&GMVjojxE0I`qpTh*;CyDZ zU_rCgj18wcb!RP5Y*%}>;V_?+#0mulHmeN_`5fF5RVKJGoLtbTC%4j-oiSrY@|4tP zGcuQR@B}zC&N0Yonv%0=X4>(bH51cB9CmG3*vg}2VWg{K`Jzc=ebtw&<8rxzDgv1v z3mX`i#3~dTIgjjM5%$%&`E;7$x)aC{>GpjABFTYX47?R%E*wA@I;_zRZhWbAQ{nXLmg5P`xne zQj=J?{hr)Xp7!HuZxY%>co!ut@MfH5@R-NfWMQhf&!!WHIQCq~S>&%)@b4JQ+}H~m zIX%1%0<6=s4*mGdss2kuq-zED+c(CFX0LmU-RDa%aVWNZU~IphQjusTSa3VUJiy*H z&wN|IcBo}wr0rqRS8;`|8!UEPF1C*Ozh$*`T=_#+@hAhqZ5&K&D-JYqFHm5W30je` zXkx~;pMr}dVtyQ&mb2obBJ-4)2bGznf=@CAs(zqxzh{yoQDrOSg$?vnRaP zZE3Qc#d&;Iz=s8F9u6swJek;43>28yCC*$EF!Iy<(E)Y1O#ca?Hf6aEgF}s^;}K3xYNWHgcIfo$U8jjVJGQ$ZbClt(~>) zt#>UCua3Le753y%-tHCL>$ZGg)#Tu7dp2{SVv8svlgMf&i3BI5C5i%SE{Cyk3fmQaaj#|}L@vX(E-LbT!@ zlXm!prnYrESf!3I%+sw=voVQJBPV{ ziNaBhEX8B`B?<9|HZ$@*yE--1z{x!^vTaY$@eb}E4I&O)M|pD=%+Hyyh>1xcBVvzI zuVab=gVjS7ALmPrdMj2}b#8H!Ee|<9>FL- zp;;|vW%uuijOoEMngll$v;;kB5}$G4km{*|1Drk$0^3&{oA=|(mdOlUGiE3*%dk-H zberqeEX{IApkTwlX4w-AY&rpsxo$hmMC6w99ou;LPS55W`E{RLD=e5xFHDvTzjBsm z$FFnZ%Qm#?l(6&6zrd;=w@}XTK{3P8BnIw94qdJ`iv)!)9F&(U<6Yd)Tx#3pCd)I) zI_ORUThEGQ<5vdFPEY)#5?35>&Mj<~H*hYk)##YI&XJ4f<>^lAO|E=fM1tp@N%1y` zIHJfr?ZgAwOYT9Z*j|4N*8e(5$E(Yxn^j=K6;_Qi>Obc+XmGFK^qDb9tj&N~@>oLK zjH^20GZj)Lk1;fDnkDL=D9|r_t4h)_^O)gLleBAA$x`d@nsOu@ZsfN+!F21&F-C!c z1E!sqH6Ijgo~<8t)qJl(`#*_h$9X$9oa5nt&gJvvEqCmLh2onwuo#};-?Hz5YwFeq zjY?09nf}NeQm#vA4m3z$vQ%h2Vvs4)?6;U-BE!je(?K5fl7!yN3deZWHXO|~ab^~I z!L)ZqV6!xXzErG$1KT@;rCc!$Ox_wt1DqWh7+DnDEFY$XWGHxAed3td;iK~P-0wKe ziR;~8%!spEVaUOyA$pX@VI%XL>mmFpj`I?9d2~bWc5VL>esyYZcayyVBUeNNQ|peS zx(6nRvwb@1AAERLhMRKdhMTlmd9zGl>#Ge0Y=>p`ZO+ZD zZseQTdezcC(9g){;c*}BBC!(*9sd80tQ4Ipn`Ae27eil60^_Bu25ydvRt_&7@;7jN zF=d&^sP^QXnBM^gg;%a!oG)fdJsHksaU{{!6!1`n}gS>$A z+_W~f$Rww$tBzgcG* zoO5=bZ)ah$w5a&CT)nW+p75uvlCNcXYF9|ENw~;0Ywo4|_iuF8+g0qE*op0zfLm+81MX{++Ss%nu%u-; zOZhBl=v#8Jou^}6smX!{UWULua-PfrA{?APybPBoGBk;tx9IwMEP8K;qhZozgBMFK zYKO>mCb0$H6k>_^z+$an$|Uf>lSS=Aqt1+lJZT-PbXMBcd!Ok)kZ90!E&DG67mw2Q zxEz}2JRQkRR1{!Do3~F zfd=&n#?mwPrX|RVCp1Vu=*pbY%yOdP^5izR2$sMPT^T9NZH z*Ak|Nwb~NRZYBFyt+rZi+!U$ZFfF0MXG??q2~!&Z=Flps0F?uJ4NQ6kJr^Z-F9jS( z@pj$D%p~|(VSiGixPZ=`hK8*Ny&grZdHTR%T}ofx@223FeFiW1LYrBWOx$8K_E=ah zonhJR&#~uVG3$xX4NNzbbG~%`N@_?_WZ?VI^eC~7YsPk~1#AT$+@u*KGJiC4{a{?2 z(5APb!IgQ^>H`gK8|2J6JZ&txHk^=ENNnPcXt9!NzI|Yjy)g4z z^~vcX&6#r;Rvb9Ve&8I}f##?STbDO7`R6E{)(|=C;BbD2RLBzp_m{K6a=7F=)HZ7x zvU4=D@7U(oDE;KZsl&gTEN`6Ge{o9k^LhOnOa~{<7cw~(-oU!`OY7$i3lA^s`@BH$ z$WI3T73(i=atl1osje-Mwt_8f0wc!+4t<3csVh2cSIl@EZIYSMW~tC{if6HcLIYcc zhSi1!mK6<74R(e>7u7c~EUMsO6ljo%WV4E3;FRd(Uco4r!FXJdv$Nr>SS^#ohJUST z7E8`v**9_Gmb4wrY#SO_H!$Qo`skf%EBV27{|Li1ABHol8<UGkbh776<%*EW#Tuy(5P+E9KM1t;>5=A$bjFKQ(rAu7O3ZIN26726D8gMsVvHbXiAmPl#lb&fuI<*(N2?z}C_Dvx8}=5`*{w zX7QD+P7*ElFB(`^Fxt*&o_tAN;QGkOu=t1cOjTgA~KeubWKL4xFis?w;tO_4l$>=hI7E1~;B; zVc^hcurp|h=3sW1$i#Mu=@f(7rpG;>Vni-`xbk*Qkjv<#xpVwcJz>8O& zJ=neBOJu<(|J-PGy$ehN42(Vj-odT`uN$VyREK)bxo~~Wo_RN>@T^_G?~V<>s40g+ zgH=SEn}ZAAkLIWq%xnw|G6^lN2O1a!7}|F+b6gN$t5D-`Xt+G7fhmJgd;ue$21Ao1 zXDt^4;|vBSgDsb(IK>VyUyl@b|JAtnYUlR`{_mL#1#g(#3g#|+v~S+7whV@rwk-_o z6%0Zhj1C&-{=J+iVf41k?SfV3i%x-r2Bs70(;i;h!m!o(j(0#Z1J{A(7-r`8b9mUe zLYU9Y+7ac&8?}2+Q{?w2jQgt?g)C;@Ug0|Ds@K|_zV$hMArH@6yQj)Yjic4O<;_)t)r7Rme?jSavySj--Qg(@YM=j;;wF zTECQH=G<7!EY~nQr+=md!_0^|wpvQ39yh2bwEVrnuduS!bw#uMk83BMPP%qv!om%6 zCVUjs*wp5hz{%#oaJ{urD@j)VrIE(LNji!QKXc?ydCw?6cz0reRwu(&?imaQD_CO` znBUjTirvH$U~}r?jV2R^e_qpK53KvhxV`%5Ox`uC_C)$8uF1C6`D-&(p8LXv93F#b zP5FyR9Xn6d^mifb12L1&NSuVagGP`7A4!>)9pqtp>yD~t8 zIfZk9jYHNE-F;T{6pXr%{KWq(uG-x{9 zTlZZuC2;rYcU}@LjR{M=8cgPzfA7^~JD2fS*GihP!TK$O!3L$=13PmL$jA$}zuDL< zb$~;p+UQxZmgI%-Oph)Fk8>ufjFks1?@nPW`Ov1G)uyeXJx!#ci|_6)t)|_wuaqY2 zSh1ktwm2%6@! zL^QX>oY;Eo+8rTxzZoof{2grxyO-Q*eY5WTwHN=G!x}CJHO!M}VAAx+eX(-#w0U2v z*vjWJ%lznY?uqze5Y3j*l(dTRbnvQ@BeI1rCaya$N8OPU3dXe*( z3Rs`~YyYM(ZEHlZK*r<939WCZwMG`SCY)s2n%u1ZA#2H&v}`#ux0ePfuN$v*FlHp( zySd=#ZH?!}%d$)2ME{pOe|uJEK`sx+>210mkJBDCHhgN}JdnOP)nM^d8;%Wvtuv#V z|1OD|`l03CV`e=Dmv2wMMPziYz3%7rLZJQ$qeO0JpP;noS%wXbAG9PrFK^29X!qt) zF8j%>QNYO38L_qE)h)3!t{n{;5jVWFw3?aPjT`bJJ6IzN{_vlh9{}PuziXbk$Fzn^(^j^~<+5FSArQ~NE6C)g{V>-+qjM&k_anQH zU2Q=LE&3hdYz}_+jtJ9Mf&qwwL3mBZ)kAI>DQdj>XX6G z=g}Z^BK_^RPjB7w-rai_$uVtm0<+nHG^eW@lB%anuRS%R)nR1~qu@UQ#UeI7>-06R zo}S`+@}7Zr*%_{7A(5iGE1r9OY?RGl6nnrRI>APxy=YPO7mxZ(=V>|59bbHV+qtN@ zq-CS``>2M#>ysv~$oTI1#HW3wJ=2qm&(^i1J(;#SVUpR)y1P#%{uK1MosuTN>d??6 zHX(d!4MTHlgUMOOOM7+aI>)Z}c$)RSNWZ+5b3#K`8`ttG*7ymnTJsyG&CH6AsCw_l z9A3ccz2Tcm{WtB5CXE@*W)k`FUN@KMY?H4LF>hD1C^x&MVJN1=Ja=x~U$zT-pZ(VR z(U-vW6%e`f~JaD<+Rg=N@h0TiRzNfVp6QH_Dq(Nm(y5+4~k6hs^xm~C1Y zG%#};95~=2JSd>UeQErq$y^kPqv zJD>TM1a{+p4^y14n)v%pK4)^qLh%3#j|fAP_QoSk$6`0_2s*$b)bc?f)z4{VM8f)p z1vA*KOfGTAC3z$r5#9Xc=_%1D2gakqGjcCVisyO=v1U4Pd^{{r!E~{i-)0ZnY#xIX z4o(~@D;OL(DjF^x&)dbQE+cWe(NU&oidFlZ;+U9UX+@=19y0S;9Vxg_e!ujZF2zu~EnOv$BsbF9db?Yf)Q(KVmiBm1Tr|`%$0bPYgW|bDz z+<^0X7lK>m%AOo(Q3@@Dd{ zRVr8=(|KCy$wUQKwSpF%ghkD354hYlnAqvl5zsRI#38mvhZxjUn7HaaTKF53rp)`n z;(UFV*mjK)`F@@T29|^atVJ8x-3*ka^lrsGpHQJB=j$x}AR*Rln<8(C%~Q?u4Rcxj z3YgeT9(e4l?wGmei-5{gR$&E>dD?1b-AoH!F-bEpD&`q-sk=1t253yyscCG}_F&}l zyU--I<3o#@jUw-&#+d?57xpS@{JWI4uF{b^BUoBYpv%0kC`5ME=Q)cNc{5{wFkM~} zY#UIO$nAT;v5f5pqj-aZBFB?i)n%y@l}tV~3QSnQ@;q7Sx5oko-Wdnj*A%er-pM39 z?L`8+yTu{CG)5Mt69;6bDRL;?TFAsH;lQlYu!GOFqs?Sx2lIlZXD10HFxty7Fl$|4 zak#=%d{sj>%rB%@bYZHy*UO3h>xB3+8I;O4RXZ@rIV`;`x`3l?=iElF2`WbBAW!;^BdT5=PEiR-JYA7Y#te;%>+}>D>`zV&PzsvF zZF(ZgQnRzBibI&|!UDd98Ew|K=2qobKWnfp0C2JZma4rdD zmRS-Wvq_6d^v40gQxmqQPWm7`{e!dEri3toK2hO+)jt^wnJ%&gsVJ3|g~r5feJadc z^I2W8=t8r^181g+1kMHlSHYjAy!ku&O1w87GrhxfbF+uCfa(HXtVP7~&f@6}$jG9^bUMe+89mOG!y zIQ%6JeUGa$zWL$}s}jfi^EOX4=d1PB*&mhHbmo+u7}UO1?Q)a2S}QaEkp&7?0URIp zFiZ0+nBE=p#rV3P>S~U~pW6kP7O_7m+a7kINo+93Pb~A$XgSwsBf+$C*-1UsBJZfehETl^pLvWPusBUwRI2F59kyiS z`DHCTN-tL&;T3Xd+_0mH$xyQ2>2p#i@0G5jn=afH-)_>_$?0Y&wc(!w8&6<&bb}*z z1mDe$AsK#y>pwR$Mz(AbjThS~UhHzD-v} zcRpaXzV)!zNGFj~Ex$>4hJZrU9bVx&i$fwa5*qDU7O_S@H)ie%Xyk}l5Vq23YTcr) zMzJjnicj+oUgZg5mS4igVYh-YZ*hT))eT0j_zlepE{?sKXFh9rZtRMRIchfNoXG?x zVU7c=Ev!{G9?T0^x@FI>1Tb(a+>Vvu=~*euVR7cBuLDiAFx2mhuw}&L#{jd*YZ4Sh(kOxjpf6sc2!5Jjp(T@$+3@ zy)DerpPKkm7`M453L6|0l3_f)^-!c_qu`ALydI1S1_yZ!9Lt_K@IE=f|Kh+&14nUz zNiXiki{D}V``1zS2BX5Ag9;2x3I3BqD%H)CtTjnuQW_7|U38@T4{vap|USWtSk zw&Mtc$N}~_j(STJ^*1D1|6%4}iTr4LP{^QBqM^aH?tT>~GYpd!tFqi&?Sd zkWotOE`fx817~p#2Id3?W)+9kHVzy%4L3>zB^?!5=QvdU?K-H@IXR~7W9oy%9f#On z%yX^#*OU^{IXQ;WA?sk;7DuzD=(#450w#_EPdf9zcFw=?fcp+xo`Zvx+r5I2u1gNn zOU^2n1i0J&6D;*OF067~OeQ8}&O!SNO^QW8oSLfe>4jmKK-ro2gQl zob)#+TJ;>{sCpvA;3%9BE%xM)_7Tu4vLINmpcX^8R*E{)VGGN0a|q#*L5vY$}b3`h_1 zh&8^#$SUB#q{mRr65`v^z#MZ%e^Fyn%7HgJ4vaSa(TfhU=`afYP?pm?sk4JgYC)rR zNk-g*quc&@Iw_>K-Jd9uclG4lJDv|OCh#0^=ze#`vPg1~14Dtr@q#_dMGZk}w(TVi zLHlieg{%(Vo7^<1YK_JT4>ko` zMv)wT>5eu{&qMz*Zk*&lu)uuNF}^DY_zcvzuT1GFnpyN?K@o>&zf*f^+oTEaV#QS! zig!$zP{b%((3lbR#8v80oQ50slzHDA9atwEU@hQf-_XFF5b8Fgw>G53^+uZZ0VlN= zPO~pDaePUem38n%W~1bjLwY-!)IRvK@%XVPG;k*zIC?L8ZO8$poVg4>N1A&Typtwr}>2vs|o|-4F}BzXYrT=tO*XQ7R)esC&`qj z7&(c-L05rwjl1LCCjB2x+ANPbJ6c%}G$cKks6Qi+Gl8L=!@+3RA)OyisVOhybQ^_r z9JgN;oL$m(YKcgui{s?VWS+YIq)!dJdU5AhI9Ms2njv{yM&!ecQ1`oP8)o%tU&)#0 zVcU38;lA(%M%M7eLu`3^a}Ih4abEu?a%bu;2N9kLWd#TMO&EC#7J^BRsGS@q-Ejrbe})|3XeHSBkC4@k#4 zHX1bY-f*b7=iq%MqH&JnQqML)mYet1bt?3;r6nB8*uk^;Si(G?n@P76xDPZetT?+> z&YjU)PD9loO=+QBU$(fz2YJQ#9c(i49!*XS2ka9zC`vF+DPzBG;=peJx@=p4;km$F zZ`nTwV-B+SEZ`H7e{uAKUduYQ1p$_tGq_)~ ztXmN!d_SPsprcKpP{@7bO3MRB8w)unFj@R*JZf`LJ}-7)mFSae+YTo@H}4jWx*l5OEIE^w7ih%m7^bS+lKOqGG7VS;vq)4^-L z+IM0_!X8WOIZ9T@=tLZnlweFhd*r<9iCE1WQTzXWJ?iViWYM8#=h3XQKyTXtWsk#h z3nUbBT$$>ghU_+Ews`+$ii1K=lBU4nz((PC1O)E!U+J;vGf@>h)l)Q+IiP5XK*42? z>4Ap6LQA<%C5h zSD2J*8WnV$)!YvAZ17_E5nzyXK%!u)+MQnQk~@-j=88!;*7@Et7N0D+pj7{)slfiy z_7edsPYSg7d^X`f5ToCqcW$die=(*;wLHRB5T#`43lzmZWU z$<1TywF&<&p6+N<r>0`O zaD}6aOB1(BsOYzYx*p|<4;XnJ4ouzTz;~mu^1g$xz@)R+oHTfxH19OZ+As@y+|ZRh z!8u2aBXg_T1z!I6C6agUN+dWg`p6V5TkPb?5qth?OucGz{VvOyI*dImj4{FdYqy_# z$P60;NU3Y!?=n2 zkYkPtG{hu zd-A%TcD`wLj&YIln6*l-RLT z!=y=ei^D#>maIo7l%5g4Y7^Cn*2XUrJq=_QLo+R0OOl=tSU}s zS9U3+H2X6#vCVlE;?AgCu&t==RcW20hzp}=M77WyC&Q9Mnm*2kGn@_A{M*|V+^itr zq~H^*aHgT_cVlXHgI7kw1mk9f1;6hFmF(IaHs^-e+zD(va?5^b%sY3bdg}30%u7m2 zHf&K_A9}b&R-wgNhYp^zjH@e~dHHDf1wHrO zIWIpw(U`t8dO^;^O{|ZF)%~|5TxD`OEuU52iiZWQ@HW7A8IMQS<3DP0f0GN_1`5dbMqb_G2Tp;Srn$4W zWCj=nBpf)<#L5-OV9JxyE@>RpFvGVy?c5#d;Aa7>0oGJ4_W4ggG&Hta@G@w+No92?9=ja)E8&=Q{sqJ3lfODSE|vJ!xA2*SBZttD z;H*QD8X{Rf|1L6g{ySmtRLp+b0wylaISGe4`)~a4aFdkgQR>oY;z;UHsA=d_6E{&w zJS-sh;N)`sCZ((uOJ==l>SkW}>QR)q;i^}wRA*%*X?yNC<)|Z7vOtlIKjX!L*0$)5 zgRK$;3l=_=Es|tvuIpM zmI_*SVwni1#3iiv~ zCK6n5P+z22EUGM$_us7AZTIz8>=08~W?|pCgl)cubB9LMuSAX+dM%Bu;t>LkEoJ3J z4;Te|oldP>`Q~m?xB8Svf3);c_IUk}Zp=tpr9Le}iCwqkM$>AwHCtafNbF8{#Kfg{ zU_ld4$hxQfTy_r}&T%Vf%x~58bab5Xgk@)IkMfxXk##=jE*$C6SvTXrb4DqJg&7#>lb2k2mWBGoQf%7W)T`Y)lTDt~e+JU)Eq1Vn}4);OodG z<>A1jl_1Nd(8%Q>;G%G6gI&@DM*g}NF6w@UoFM^^MM5hMOP7@P`0aTp@_XYUkrfIY zjw*@*6DH5uq*2a&+P9HY`r$tgk6jnqUd~XWaA_@$`7pgFLP=S1%SXP37l-A43v}A?BngzRx`HJ_{BK%$cap|9~mrmdBK3 z8m?MFfr9VTO4d%cSjeev8PMFqz@nIw$k!~;s+$wYt$FW|*p37@lNk?JW_UbaerbcN z;-uxwhAoO=EfI(0r5fA$1QeLck`C~0dfcWt){MO*%L=GFOVf`*1nK<*?Y<4XgpVj=a$&%f&2p^G&1}*__Wz7igQXNHMFCH|N1Z zjz8TD%q$ODf?AjamP|OrBca$~DZ(VM@Wo;FUkTh!J`0#wcf4leo6ul3fk`MJf<@Eq zBI^QP2gYL|zRXGs4(@xm)lWOkfm=+GSO3fbsh%0my4xHWR=t|Rl*jRkS7Y^|ml#ZlmeCzH^m52-342Sv84P7q$Q zK+vTAh^xQHg)(`oT?!5!F(Ov$UR@Gx65haFwq>@s__BXSQ}0gwq8SmXDbBR)?wflW z38z0F5_dVc)V__8C((i{WZwb7o*N4V&j_??6eM>0cRUi_{=v;OFR-_yVj+W2M=+z> zghu<8B$4JBj;z}*w21vl6fEyxVpBTP!Fj^due{1ZWY$N`pq3`VGLCa|R2E#fS~%nA z)o;P3?1{Z@28=QdI}RHCTX2JorJczyq0OY~!&%`Ot2SNnOk`sUtqG0ywg`E%Bl6p(pchBNtqoaarcJVWZP9+ zqe-gkporB%39Ex5ehVeeJ^Zxnjf7d^gDDQ2%XCD08t-3ol)Z9L#^$|D&_b>$3SZYO z5Rpn(UZTLYr(mH(SF}-INz5X4zx3}B4d2b4F#0j{`Z4gdH9hG|iW6hGu%d}u@8u-1 zvrjjz<}X!9nD&7C14I6_100(gxIR4R+MvLF>;R9#MxGT9*jNpW=5n4q|~s23Dp-sauC-p7pZlzpz~*@cF`Gj-@%mb1q1nNfZrxD03z8OUVhY%>vwO z6*xCJ*gGknUD6P>=Ym7RlSh6|9Jd%^>K5!cp{iEVpqBH9)z5*&=o4d|K1+jATt~j< zvg`}99%u=@FJC;y_~)CzI4s^FLhPRgW2k6xbvZC8s5d7%*@xXq4Aj2vfhTk(5O6E8)@Tr8cmC3&JAAAm^4bPW<22gcYt#jr}Y8_W-$e}wU6065~Y?Ln7!%~ z)24|`UI&=G9A>jLN^mTaC@J75d}tfQSor3pOiv=`)CA++hZ126jn6ZhSYMXuImi`i z%)PeZOwK`J113dw6|S^k$9+q8PZR#G_UW7vi?XYO^4$r__Fft@nk*XAFHA7L7-Y12 zhQhOX4XOuRty0f%?mMuo=K+^NqIk(d|8gM_heY-B4>;;3ZT_dgQPjZssLT7s1NIIU zHa!Ivy#tzx3pkimS^oTc6{PooT`o~js8QsSx0DNqMO)hqi3Yw!2Ux!~2rl6~`eXU5 zH~z1DLSIQeU|N#*N~(cbDuE%!A%&xvQHw!Oe*v@A0wya3=BSOVMhy&V3z(A>sfyxsBh`7 z4L|RRR4qu~A*W(h|A6yZ1CNWMzkQ>4QKR^UM)8V9v2zPVqZozz7;Lz^1>YqIiZKdE zJQV!r&}8+1DP{qi#R1l+1=IeuD6k##4bTXgGwq<5$3iitMiI9UEndqQxehRkyk_!v za5Tdqt6U{Kwt-5l{ zm>U)(@+@ZDmLRx~N#Mf*PN_t}k_CJ(9N70T@HZUbo|P!DB|+dqxby)>9ydoB4$W0O z8w7VOkeqdpo2grKyTWp*g<@M8WnVquZ}XDk*#5tv^v;osOxJQh1~hz}^3=`O@qwS? zgRX^g>W#5Cuf{$t6}4y-o*)qynJC;>QWAQZtHq%7uj#iEZMTfe3lFezaD^>W5-Qb8 zVUJN*?5ChCZRRugP}~BAq{}6a0bNU1J>|b|;@7>Mlj)+t5@C)x2lzA&aby&!UtB1E zfl=I!u`Wtcd_x1*J_QlJ3kk0t)OTLhu{xl`_HUu&kpzLfgWS^=h|N0aaYBhr&w)WA zfp5_g=9~orXIArnND!1-IQ74xP{sk~Zwtf@Z5LBo$hYR8)T;%OYZ&?0G|I9q6kO9d zVT0nT8;LV2ck&%+l>P8P@DiiQ;e}!^9AsuO@=r2rQNK|y%_yF?P%LYq)HDX3YuZvw z3quT^?$=;QUG0e7H8Na z_8A8%VuP$^CUDM4IJk(h-k?!TXQB1^2Rxe;(s>lcj~(Fn$HMqtB2oLp1O5*U+$9TG ztr9%y!q^KAu$Hv2#T-aHafr3Z;kD3KwwPT4LL0;kX3Fkrl%95zJ0?Nk$U-qGM(J6O z>^l}pDIFC1#x1>Sq2ND8nW&p<_A0DXW;EEMuxe6~z#avm4+=sh3j|*|Xl_5i`hkJH z;Q(vH0)Z?>gQ!Rmmc~N6RHg&VLNY33gs)rem>T_fr|=yG&N~VMR*hS%7YZ3o7Wxud z{4`Xw%2D)MqHNVenW{wLC#Aw26$L&AMVuDKbEZ5>2#(%#w0PU<=%6U2YKGE;1oo`! z=7r%bP7b@?yr{S(Q87z@_o4?+A8bhuUsO@Vz){DrG;kMZmX_3^1OGVEc8TgJrk{DR zY_qxenFk!p8U$x86f9Z5B%zkLU|0BLt5uaLt4=;jUo7Pj639M-^RLhcF_%WKU5x8i zFO)7xVEV8?kg0P5+imV64E%c<*xuaU`N4r*<)OBXgHXu<<|9fvXAUuUDIATgIeM|o zy-ti>PJ;h{1NScl5u1bCRwN2+TenGL0b|w-mVFmp0~c7ynF|UnlsJ?qb!p+r?}DOM z!NuB+#Scq8t}m3ovQT!3!0DeiB}}FZd5DYt*czYMUm6)6y@jJ>537@c5yyoM>}3v# z!4A$ARg24>t6ceVZf5_*gzdW{m7XQ(Ej{?VA~4E|@fi1qN40tfjiSPYO%j)t{uAQf zqAxI`QBa62H@ZQmV{7>0UHkvfj$n;Bz^uTa^?)yB+T~TB`TwmDT;jo_lISk*jM;{P zrDOp^%@OyQJse#OEFp7_o=rM>nrYQprD^96>3IENE=bT>v@1!XjqQknh|5CZyhbUG zg;Cwgb8j!RGI5yw;Ai3StrCA6WiL6({$f1M7-ei4-f8_^V%wZMTVrG&l?sRK7GgUt zd~D7c{!eE%tGYQnSKNMj3HP=gNlEMvWZ0g~VexT@Ett+C=29NObw2u?<1B^U|0h^l zi!D89y>#}2-AASS1?NtBkggsn<{Q;07MCcV!N}d`z|G=#Xxaf*JB3w>Y^x?(u_ZAi z&X79rZ?e?N;suOM4GdZe$s7rs9F8J9iJHv6<*kyBrp`?akxBIVb3;E?$3o$zznt(*u{4LvT@K5zrj4sfv02v^Q;3&Wz*O!65Q<)CKkuslUNv{vw-dB#UJ9cVs$yd;^eS6{=jxWriyJ#mu`CP`0UN2Zu6zVR(lRxvj~)AyHhdIek-ub(2)C`CPWF1uGXuv40WZ*s(8B>X`1NU-haE z$JrhjmmJtQ+u`4HuNd~O2mALwVlGMG^r_*#63>(IfZ6O1^9ujJvmfYeQK)%z|46Fs zG;4=zjE|bxW#!dgTzL5KFfWg+Rm_P4jO@JX3vF1nTzy4Yxx}oBp4>2OW)qQe@-+@! z6Qb_Fe8o)T@>SQZN>_(TM!RNCwJJSUw(2#zNrrvt%AUE0Jy!-T)mq?JIg3ej)s&UN z3q2Z}rk)kn2%NX&O?Fslb#qae7n^M4 z2Ej(fNQDKgM@2rnX&jxQ)SaIWXSsM-Aaf-vGmpN^$|g?z6JHO7mESyiJ;8@3 zY(|0)!^9bh_8BvqlY2^cy6`0#9Xu-Ef2C-yOIlevKRZ^sst~sX;<|({bz{+6~ zvHY&k{HSo*(#RRR?y@I!1jfq0{kpN0ul~wMNBL46MmPD8D~uc>5?dM^IdyIX9Ijhv z;lXPx5isFEqk`fJ$E|9D!Dlq*axih}Eq^e{%g23Xk+8Pa&&T`$im#NF?EkJQK4WRJ zG~#5J?ku6HAt4(JMMd%+?6?|k)pa#lz+n+9Gr#7B)TS7v3rn3Q`+qumeQINDPwKHw zQ#HwCx$>FLGpwFmI3l;ip!o=Ua7|K|W|$1)XTit`id`B$XFhgl#BO=en600=}vO9}5WGr{!+2Fv)sN>+k-fY73uXU-eh@c~jtI2`J)rl`0j>#)DJUlLx z^kv~T#r0A`ZDMT=3lGbtzEEJ3Tla0DtiZxd567li#zAKcZk=jx8XGLIE~ z*IcmX)LK5r>6nP&E9>O_W?4%YhckCyJ$x|pZGcMbrO66^h5dNko+kN`pk&?=11zcDqw^szMWxG=A;HJCihJvfcq6ZHTE|pbTB`y0SfqUo6nb~$1 zn)r7tVD`~wVy`if;S^cUTra^QVP()P71F>U@SuU!;=;cc`vynu+7E}=4>*_`u5h^S2m7k33t#V}`s2aS>jPl8lWEa2_<+8}l(fz5=$Mbqh`P*%!H*97w{JKqq~_2pS;D{^kilXqv9ZPR$|C+#KWB0m7-)C~q?z|yddyEc;J}x&ii4ry z7DLs`Mtcz^e%%H~ZiNF391aHxpU!09wPk4H3`tsFb6mH2TXT z!=A)-bJsY>Zjn}g`=_w)&e|qn%~>w*pJ;Nd2sy|bQ0N?LlV!8Ug^}x&c7({R=QeT& z7^?I=7?1coxTMB$P~RlOtz-$)x1Ic)?~)r2ZxUt}KXAY-|Ayc8e@R=}x@4#E zmKd-`dvwf`yTM^+FYOv6lFT(XLO{!43*StRDW$PnJa?sB4=DTlMCMb}Q7PS9oG~+) z+-^J* zUdEA;6*_xi^QtD{xEOcqTNnFpzFl1QPV1sykYk`(+!qnaw=dk5JBY6Sa+)`A!$pt9 zA4D4!#dU37I4aCty`**RmR(o(t`yktdSh?yncWJT8IxPQLKa?Q;40H_QJnLE-63GB z*u9H~6eJWm93~v%tx9lHekagu8~BXf>cr~?{ux*A+FUiCp~Y0FvY~-Np@A_WfkjR% zw2@I`GON(H25!9%#vB_S%rHu0w2nM*+!^+T zUC~Dj|GDjREuOWuXt9}gj60ikrhM>%m{!Z{OI>dr6mDRUIBM|le0^Ncao)|b6I9t! z+WE2=vK8*h89d}}@bHt*O5<|YImn@K;M`K_Kn_iYCdr-+%`0|3V2#{XU|#d;K)CZ; z2R?@djEoZwToIbkz?IP`Q2(Q0o|M9MuG0>)bt4M7C){O7))u~1d{xcbT3{u&OV;c@ zS#{>UH+?gWQ`oLOW8i$&#Vmc`vdBCGR^vq*Su*DC?)2m7e5TRBGBHDebLZWpJ?vJ2 z$v!hrnOeF}OTWHuq0AJGf5+C^{8gB0UQ)uff9*us{^cLcj?7GOSY>>K`Q|01r3_Ww zoIC<84Ig_F%MJhCk-h(;=S#n(t>Y zv;`P2aUEXADE?}KbZ5<0e+P{#4YL<}i}9pqaeF)y;xg=DtZ!hBEO+4e_DUeTfAcNg z$OX5hnHm^pgfqOb&okArS>k-%E%|Oqgkb}-xdO*uL;rvYj2Dd*TH5nI2J3n_M6GDx z`&h=eYBQgCV1C>Zu2l>8-Wn-wzbw4%D3@3$>vRXp1=IEOA4CTq)HiX8S-xC3H^6{B zIbDm@Bge5tyR>lH7xDJS!iH_Jwp&@`AFJ9ukg#Qlvr-62-&mxtF5qUsmgU5~g@J2J z0cUtXv19>rtpisJ1Is^$&G9=K!`2;?VrFD{<;ZNbAd&Y0Q-E+br$M&W1&%3?jF|-( zwFH>F+rvMnmx?+tKltMNUszMJ&6!uxOTwk|e=)P=5*e<{3K&g4Fq;T8 z{t4u|q`)fr!-YMlf-^w3YdPbRRIcTdtrS*pt#wY_xr1xk^i(HC5w#)_(NO)#&S_Gj zqB2v{f&^H@AEuq!QN_dFV*8;gSF!qnadp?j!stWQa+@Voo=XI2sM#fCIF+_qHt6hH zSpJBEwL*c*#GN&jforZ{+k6M^`30=f0`hqRTrCrr{g@rfT|;~ht22HH+4;mF_%H*v zKsI*+bMFC;gb5mY0*o>T{xRqrU=Tf^{^X-9b0PEEkBpWMvJW3JwoPXAOOW;6z$$yv zOMMHA`3Ke=k60Q%F#0fXuo!Zz4rDf&Fxj+#S??srpO5lo0Sc^>{KXoWBqMY^1vus< z@Xh_exncv?O6R6k2e@{5;17&Q{udmK2T1e|v>hE*?8XX4bdd=Tot)K}&L69-%Nz9&q4FIi;vay2CgSYf!j}T zg|6&gHG%8-0e0Bv&b#vqM);^0&|s+T%A*b90Q}p0amsPC0voF6Ad`lu4InbsvcwBzIi7r=R$KU z2i7$cJ1dVGOC*%8eaI-u&~U!8!@@=OU!7_Fx`ho3Q<&`(7MVR@X;9z@a^1K$0g^P;yXs$0zelG(IdqjL3PK4p*m)drk7A*_s21(K-?Qa!5_Qr&01 zN;`6On)=g%jEPKzk{+AQ3$>?CmzY&u>zN_VWmGwxx$^qLD)&C~121h8JZIzJ$1_tpDHaxaynz^hJ{mC-{{lu$OIM z?-byu7qDF7z}sjrMQ-8pR(B!gMwZ(_TnwTAtiE%msjm?EJhNL}qjGgYlj}l7i-jy= zOIeaSxlJ-wvPUTgeb8UFeL(?Rdj3s!PUUL%Lt@&li~1RjBz`el6fAsrL-omv=^MM4 zU#2kIOz<;l&~f5siCwUI&nrpoD(*IeH4zuMb`-1$GSDn_VBWvbf#Hr9*bW=9=P0lQFmUugU=jJi*!Y2`=Yyc{5trujDN-Fw8W%Vl z19mNG<_eg}Wwww@(}`2eJAZpc)Ak*^{+{Gy;93y+&%knr0*7?!0-4g48EK+shnfQf z(rpWZaxb`ZDX+@@$aH?In0D&s8y0)-|5&wIQSCr_pXCI4A|roGJ6@=_6DwVy}q@2tAi6Wi`@e+*8@xz0qkrE3|u=I z7zBLk71*K~*eV1#D;(IpFB|(FV6?ixQsb1RkfK?BfYsE2z4!uC)dqIGgdMgFEDaaf zOgFGESjJxRfT{Wbhkyfn`2%J<2XcVC=*f3}X_ z1@>A7rZj<>(;C=5baSp-z31nPbMx;Cb0<L7rcFz zH@Sc_c@Jmu8WFF9N5f8XYHZ?6is4NC!{#-CB|m|SwYo?8Y4rZof9_oFF$HV(s83}} z7hKh_B}k8T?`C!uh0Dil|ESBpJuZ89*?O&z9dGt-oaNWQc=AgVX5R&@2RNDb%;KDR zfy;O+UAL^rT6 zUcf9Az;O0qR>EZlo`!ad07gLvhC>G#m?umSGDzf_If3&71Lw_a8_lwL1Q_@?FqF(? zcsqT<+vyAf0xmWMj64bvIk}7+8w5oju$^(>k!@s}JA3)Fu!?s*EE*G7Ci`AU{j0!PCcb(yDSkT+rEFmD#{@n!E6#cI<_J!77%& zkH<4f@9w?*L}T>{qkXQkCz~xep%u%#;|)uN1IGmuwgivM8yN2HQdsQ%fwen<+x{(U znW@007G{=ji(}Gf9Y~zDRFc{5f;V@;bC1RR^08#udG_~*+NKW1_rNZk?E{#*KX5dEU{1A_`L&l}rRK|jvM+z-UgI=)@OJrx9NP!{ z8;S!LyyUxgjfX*3On~)@1CK}pQ}5YxZk2iKJ6JpfxSl-_S{ZO`g+Smm-UVMDy*@U9 zYf8cVDGt$sTRA-)E=-PPT^7LAcp)(OAzS*i<})vj3Gk-JUt~VzeNoG_`dUV>!GkC3 zvo1Z}cl$}l3HQ~LcR4;?H*vE0hIdYvSt>4Ycwewvv_Wmb1kM=^oDmMSkr!AS6S$We za5Z08vu2qux5~fgK9`x6uJkRbV>GilRq5Q7g+m0usbNo zaM+k`Fnh_c?|#7B`!CjBf4GO?!5juA0oEf1>>?K)oOr@e=f`M%falT#28{^}>+13r z?tb`Q=IGo6zWfHR;|9l`ZFtRi_O+KG=W6S(&mG>ZYOr3>@FvgTW(q^fIles_r`eXY zH?N6RUVHY|s^xo5X+Jr+)Sy9RZ|Tn4kL_R> z7MsAatblud0*AXpn_(`Cw*Xsh0N2tB96J_W4b-~#NH?*nDfHU558V7WrW-L^rZJWr zC}wnE;CaBX`aJ`;0ZY#Yj=+Bf@jVZi84Vc9*G}MDz_9lIefF}1eJoc)n(t-DK6tTL zkkf#xe8EfJe++yKH$*ltcrfr@X?VDP-Hm=*At{H~&jKzyo3P6-pKofzn`ySUX7pMI z*Yiwucr?3!C(8b&fgkIl0Ji8RnCJA8#lV~N-Jb`{IQsH%u`$~`+|q3Gq;3W>6{LssjNLK zRHbbyE-X~_QFL1BWqPaV6Eph~PfHQUTZ^18@yX7)cH&~HU#nQD(BI0U!+geWJYqei z3I`dPxT}S3Zb*JExKe^qj7K8+`n!l-|08_&ey@AzZ zN3)<>)Cvdww7P_bb-M~;6qwkQR&<|GWsH8H&%WrN$Qi~HLJ2(+T{K>FH?qrQG%#{& zI5f`43^<|CeAdAEA&aRlhc2^!l6sv3yYb#G?r}k<9d7Rg5K^o2fPnok>fJTxY})UC`0wf zJ?WJy(rHKg_x+2o>THw9o6)E$ZpUz_+x1_D(iFEj6BbQzFj~RTEICi%X&+yX#FB|_ zOEOdV^)fV+m{caJEMVklsAyv2E2>!3ZOis^akCmjWYRIWS0^6yr+FkguG3&|ZfLcZ z+L+8&G4rFN9@9LT*ZLdhA?17BFnm`zPJd)5znLc+AJdI9bP- z*Q)rkujRxQhh#$fET8eJ-qV;8XkWZdc&gvM7n{50HCKmNJjzjI*{Nbw%4cP^q`9eg z`OO1=zQ~{dOI%sh_XulHLc+{_HXt8oP6Ty;-cdMB2wlQ&l|cMt<84>x7k^ z`Vx#RUG6g`F5(Ylx#`f%qgRkP(ftJn(*(D7D_SRbZ2R!AiGQL@!X5cnCmI|1^bRES zC8;iMY|_6c<=U<~Lot<2T~vUFiThQ93s;ar=97usECrYJ4!!D{vHqY?!GnfQb-f-& zuBsgmH?Sxa@Gw2$zkGoACI8_bUo+#~IVKk;6fB;RB=+@luepM0eEPpH@}f}|4_6qZ z25C*3ob=CocJi&uFW0S>di{6fv0qt%h3a-+KXz0<**t0Tkpn7RC5_xk6NLURNnzW+ zDB8=4fyG*cNx+wM#(cr z1O%4q`)PXBEesM@7n9J-^-LDoaNwkdLIY?rX5qyHZ1*bE&TY_SZa;Z|)2+G5Vxxbp zbFlpHbLDmmExl?rJ)HPklvx7?vt-`PW^x9LYsfvrTTx19vyT~ zN;oxP&Lq1+=}soe6At1R);96BDBR7sz$CN9fjyexh(h0qj*J_R#3E-LU@@D}$X~<2 z<{&N?56KVdZEV2fbSQDvWxY{D~+TP)C*kM23;FL#u`EGZuA^1J0TP?)F&^`gM&{Jbkr1+^3pwW=lx@o)Iv|E?{9@-~=X-XV>Q} z&&`^AO>(Jp&8uKr7E^x5tIl$F60X?DY?xyDppohC=PPVZSMB$Bon)3?u{z?|BzxU| z84LItF0eb)BrscXG%FrUVA&#<$gIH8to|>7J*Z=m$UK7<#d8-Xt6h0eyj8$e;#UHP z(+wr5tsd>BA09M0+AI=W+~A}f65gfyqCw=$i>7yC53KfFXielWXs{P8o4)zaXZ^*p z$@{y+nb=zxdNX4VaIhx$Xw5jtpK9Qodp3aWmfu5;md8D;Zz`5e=nC;`U|HhXT^%%4 zZc?qLgr7#vLw0Y|_PPrjGu6tuyjK1!S>b5gBDbrW!$HPr&!eyFO zkO0@df33PZ3c2%6B#W?qIH16Ak(K|>13uwf2YA(@+Zk+fy7`?OA_Nq~&)<7Gzhz=X zVup@btJg_G%Q}VxDT4+!kpw399d9L$B}6MHA3Tw|K|_ee!Q$o4mBtG>Rk%8&Pdrq* zzQoppsd&1!jL4oV7AF=qNoSnWoD;S@cs9#Z|3;}0hZl;14IVCY-_|^LIr3miVfXC% zn8;8E|EbfO=jrQ48#FubR?yN+WRhc==%YO2;QpOI-q-6JCTKJ89jvoB+|+%O@!a1= zfz*J4*u@t+qDvMF$}MPQ=bF&0FEV3h;|pfv8Hwz7S>2e*c0OlfaB$w(Q`YVGDd%v; z9~Bd?mEHZ74gWrfoobLzPvCSnV;1XcVAYOz$Q2f_Ni?fFT013ySvz`ifvd85$#KpJ zk#_5*?f;WEvGNdSaE8?0<F3_@ ztHAlV(M7&;F5X!a4yx)Lp%TaV+zT*kUy1|`EadaaJ%-mTzU z+U%yu$bREj)={CEsu#>ohFLg?9!qF7Oj*d~b<9ap&#_H~htKzRvGDf^38_}AqIZ-( zuP)rHGiU$i^*a*3wdnriO4OVcG~IIUt0{i7#k|bR|V!xF9tg$V1P4HDZT8|9E zQRAha&-9P!_3e)@1)VfJx9tYo3Wq>r4vDwBT$x0B8d#JSnhg&ysqStuTCjL$vgA8K z-aWUiOLlM4c5XV5toMvbg=XpSe?M1+zg%1Eau1+Xu{I8<-VeGDasias6O$n821M z(bBS0hc$qKQ-Xojz`j9>`-?R9&J24=VY4;SW@*71+Lg+@63q4nMq56!7%gDpJ-{HM z!N|L#QDufMlS#Apil$^K-nTz_iWe^~7G7t+$tXxgPh^3E!E0;xB0aYaB0nqno;66# z-_++XG}Zj`y872PA}Xv23T<&LZSh}R6AYHxHaAM?*#6w*c+J~3Eypd*h09f1%uRqr z`9Mqm<5s0V>;fHIj2T*uG+3e)TAluVSUvkUv+Rn6EENndIvMy9+?QlDa7#2f&S1WA zkm3B#=2Q+=sU3_7Ox#Qhx;rN}a7<{h>|tOoXt4fr_^UMcvgBQxUu^99*v-PhsGZnz zXafVA#3t_*x?2vj#(WT6nZYPLLqTLlllKc2rX{-WF2WNRS)Uhh4M;gqYJ4i?VRDwnVf4fmUOUJqu4PushIf zdxKf#1EYKgqhv+DOahbq3r6_^eAW#t4i!xQq%#<$cQo=8ER%^~icVS39&!8!}*MnvP>Yu7vr@nqb1gU?*6%XlE`HS?HeB1yIV^(wDKKj5Yu3kn9(>< zhf(E6gYk|Q?<+^9=S-fyho?x|b%xB*pd7#1TMigJ_6#iOi`=r(SJBISL35PNs>Pqy z*%q**Gq9ISv{!2QmrM8;F|_CZXj@*=_L<9e-4y+}Obdw}juI0XB{LZRT=8G`h(TIK zY2^ohz6*@%3@kPVO@<3teKxT89%!0yxW)SfOPm3-^8(K3AIv;kEo~%PTr*l+C0N`N zT5eZ%pNwKl%V10T!7M1t5|l80al#@E=Co? z=RYhy_d!|Hd*+mbyPxF*n;$(Xx4}pyf+bp`RX>4`?}+@%8^Hlt%x`V9tWGqatY*nj zXs?{WUcI5c`T=|04)*F5>{S!mD;>@kbuJO)OP2mAVf<6lxIyOh z6>V?66^!aXn(b~JGtOxB=?L{^V2xeC8n>Wz+w0~7LalZS4!KFRI6b)FHp40BrK)05 zOKJvN^lH`(1x;xI*W41CTs5pR1zP!jxHB07>CPJlkrS6vm|A@@b{lo5JfmkYE$BHKxu(zFRx9ArIhzN~whV3g1?*KD*lSL-*Ia0ye3gCb z)%MyO>@^$OJwjLY-|$*lwV>I0hhul6%#=nptx%;?{_8eLE9_{t`@tY^fXSyJbm>yo z*axlt7R|d>u|{uT<&HE^=vZi#!Q%G9QtAhT><%QRBy}@LV&{mMbB)g+|`tRoS1*T~Uw-dBjGzyy3A577E(WJYf*=hr` z!-i&ih8FH`27J0Lt^yYWD_F!8S%p+tlQdeRBG^Q>nMPN$iG+q_X0T;d{CgPmqWPwf zL4HPi^p6If3%W-Qiv(b~I*<=Wicy{}x%EF-i(E8j8=zEvBceK}U7 ztG`@PkPb8lcPU(sG&aqmTMoawdp{v91lo%DleP5rXyX7DMm zs8t95ESc?p_8>#y7Wp5H8VoW%AM^u$tT&pVeQc#q($cL?3b*V9cDX6ESg#1k_j<$?nO#vBgK?zM77BOEgH3ucIq(!heHY5gd1je(m z@UXR5&A5}ijoEBNv&IT$jSMEU1RphtGtvcTq*qMa=4p~~F-W{(nwZ06+fO`YsmfXc zEq_)w#T1-2BRf>^*qgG*}%n*lZ6paP45|xpGLiGrH)e&_}ua_Lto4 ziq~}ydOQidWFXjJmK1KZKv!o6OF)2;`w@oAwv6{$8pSR!2>fV?oAKUzL6gO+wXY+S zOiY6{>mHd-O|qA|dr>s!fEkBlL|fP#SH6%O`>X40I6{_)v=yyrKjn7w^uOobAH4TJ zX%zl{y8Fk!9O)a3U!-zFn+k1b`=?yFDfM8L)DK3hh!)KS4Q3abQ-3r|?qF2@!JxIF zS@j2lS^|@KK(y#vX1yO*?N3b5yAY)(F(Lb<_OX}Efh$;p9&jAbYLPy`uppYHr-euN z(_u}|Z3ism^nx@RW(hUSvgdAKIF_(jrL}>d!`de$&F4>x@(G4Za~XB7UgEW2id&J- z+%viNl$_~7xuU+LlDKelHEQj9WUbK}a3OdxkLu#2oVSmdT|C&zXS@;8|F|#b zqlh}w{+y5J#WI#Tt-r7?F7#2z99>%}2By+g%$Aj(93z^xF|^ng=r}brX$Y_+Eoc+i z%gmFo&+5iffvZht{@qb5e#0!6*(jgU$Z&u`zJrPFLLPTU<*G&HSvVq-;1# z==>X{7nX9xOB+N^_oOXuveRhx+rbp@;rb@I4ZeFA7uz=ZJG8UuG8;KGO*$#Z;$KkY z_MFwek)!^D@A267-pjTf+KZRDMu)r!-2U7-fF-q|eNySY(Ehlt7pEiKKSrE?6Rh=O z-$lU+o&15Zw5W+^1*4`yi&H_9V+320#jksMi8r@xbz@)=TotW40M$F=s`V2sI*UD#pLFbtrOIcDo*z2A>pTgcWwY+lr^wKVYjGO6NKLlP_ z*eP7d*80DKi}`DVHP z_JpkE++pw2NVmc|S6fOoFI>gB>rZ#WI3_(`cJ`J;ZKQfe5 zJSWKqE^+8Q#L^|JS+(GXu-6o6&D?)CHtf84?5v-8-j^LGKL?*zbbtEd;pOEenwOXP zsWLJ?O4ho%%AzLb=)vSwVKR}GS1w3xV~gBmlhjh`D0X79V}(V?f{Vu|2=dDr#QX^4 z5D=cfWL9g|5s&s*@u-p~5g{k|68TNoF3$MealVzClO^SX;H&<2PFb6n2!&gWnU@wR z^F&Oja%{QG<>6kiVS&lpTT8`b8jf@{%}Y0}y3u&-#3E*H9*u?>L2W!@+7WsegHCYI z)i2po)p#iAX^^D4ka5tGhKY;ZJ}CH3vyn7P^_r#@vB^ELT|{$s&Ye5YPX@0Dm|ykj zL*WVi_g5zK2eVmCT@V!GP#PM_pZ0WWi?El(;eRcOM;|n|N>2NFSh(ShATygujX|4W zNKfGrK?Ms(4$b}vPCmwGCLHPzFMUvPT;5k?(+T;6JB{peEfouoDm33Yc$8f>XG4p8 ztIkJ9*}Ou9SyLAoZeU<$-f~SUJN{VOY-XOGmj{@)*(NY5ERIMxZeq)E;FwXB#CHb{ z{s4yqg26KqkEx!wW#`gu-ZrUIjW^bbP1GZ0!bRr3s40P!a;;YaJ$2@q3wfz6c_6o3 zlhK{axa>(w{L0)f;voV5Ws5wFSROS$u;aSq#V#A*X&SmNh_SIbBr?%;;ReyH6^eW* z4?i51naFU-iCtb|qO0Wmnx>QL5i*HKWP414yi86{xb*LYe1ptR4rO11Lnq`X3OJom zXcg&vqI~vGBfDhEkB6Shtv!eOOl1uYIOIg7?NnuE(P=O^6uFJz%BItIrWzh)7l{x!#F-f;qUkDDpt5qQ zN@mp|59z(v?|HpnImL?#RrECMrTHpniX!r*EaXyC}DqI#olIfqijghoO48I7*4b~bb2jbYcj?naCO@TLcdxUp3jY6O^3^JhNj=t4?1-w4XjaL7Ky)z zV7EFIU>-H)AxDS+tMQDBJOze}1eyhsm9q@HQm;6P+!11}&0>_)clpdy`=MFGMv&J? zV>9b~X$J-ih7Pwmg@WEwjtPfdXL3_vVrEp1$Q9afbmG&zq)nU=1xqfn-8WR`;@mKI zZh<%xPk;k+i@yBvhnkj^AlAAr7l_hOJ$On z;*e?Ma8SlWk=NnJkvbd4scMaj*cT`~68PK9&fq$8fyabN{F55ETB^35;$S-MD99DY zV%*X!6Vu9FE^t(AT0&dQ8z$MeFWjw<6|foybo10kv~f&#JR5Z-N%De=c9iGBu;>uu z9^(^?64N%Y7~bJt-#4?FFW@jE6T^e#r3p+T?-sD9*)VpA8HjT;iEYrWh@Mho8!n6pWCW=!hs{uWwB&q#-VmMLoPR-NBrFu zmnJez)MK8UB&d79fn6=2kzZ#azx)dyR+|Y+uV=1g?h(jko6vAhnxlbH^ais^)PV+; z7*Df_SA#eu9NSI(bT;MN6lMwwu)2L|*iu^6kSsi5;(U%KsV0SHg=GOxRhXk+wP0`r-Ech+k z)rM5-=%C}(qT4Q3IyoIUtbAYr^M<5FqVo+7OLEn+w%q9!JTu{#^1Fog`)?cB>HUI1_)=>lLx2JU&x`}&9D;3}91R_=5e{$I3K*p)FbRmfV374PoFfp< zAfW1Yplg51QWl2>mhJ@RH*yztu}UbEi7jB*oPVH0BVfT6qrMQ1nmcVmE{!5h0!J2g z1akFFNl>+_yRz8t!b;{D$J6(TdMe+ZyH9uamnBhU&dT!?Ic7eJp1fL$tKmvcta);| zW_N^(%8vkcHq{B|Tn-#OB<;75w=(r%T;lB8-y9Arr!Wdj*)xlsyU^9|p3c*zz-pT2 z()FLKjWha0qu87U&8is-`CBqv$)@x!c%Qd5c z#bH9y!lfNdqF)&N)u$XdDb=`X$*H+{TC4Lj!tNDayw-ln|11Mn&w^urz67WJd!o<3 z#)+l$zih#B6}=U#BJU52%Q1K6uXtE7&*EjF0JH2LhgWAfSUCC$*a9+`glBcw>XvQf zj_O&+zfB-f_uqjpfgP^}n=_dB>=v*ZTv;S}bqA|NO@F)PmIa3-GbYN~J!mcY>A<+B z*n#ENh8}5w2W;gHPV5E&jEo@-%nlq9g6t3ewOD?$n3vjM<9wY#N$xn4gg^sZo&zI~ zLP%=Iao!XMHiLt#|3VsgEf^UWF`N`+HWhF-G+;LMU^cwrm}%>EcFuvhQ)d5|a^QY% z|AUqN1%h!8OHcLM9{%%%<)6d6Cz(NkD$P%0<$`%60v5>E=i#272`0QvCRYxL2{15LG)Vk$7Qfac@qkgez)>;fu-TIf z>J?0SDd%*TH0ug5w5c*MO=y^Q#zFSN1tFgW?_8bbW1OvSIGSWMFIws>Z{Tb-!AWC* zispu96CXz_4`#&_XA28Qt0m5|F3lEq7)>`g%Xc){ABnXRX;Rq0Y#MOTWJ$9^MuThO z0nP~wLK;!ows%?Y^yuV?oU3@?I_DpQTh7xdEBW;gMNgl5rFZT`)rbA7ZcKTSdr48J z>Awia0oFSQZa&cTbeiirr$wN#HRth*nR2&JFqm)hZR<%kzHvzGM&pwQht#(yYw+9> z{P9Ho&cQ{7&Kf;TF``bw66a(}99VrAc<(qWcQh%Q#4DRH@?LQ=nQ=&R!oe*DOB8P$ z(x_ly?3l&8;Kq><2jv?n!X3`0UmQ(*4hub5BHZI-R-@+2=VT$!BD@b48j{4 znd%yu{v8xIaGVp=D7#>}$PPwf0WlE;5!Mt2E)FJT7AD~WmYpXbomet$*VA;pK7P($ zNd<*mJbT=w44MkgIQl<1*cf)!H?nv2lxCZl=|LZy|HT}Ve8O`z)+i?7+0(-hQ(Mbk z9C@n6w}V08gQK#Ev#>|H0*jO8l7|X9j1n!5Eq5CgrZfrt6<5{>-y+7ey@yGq#mVc^ z@spK}>^Y5(IvjVT9Tc@_WPfphUBr<^q+zOo19ylUy91+mNduq2G0Q3kKAnRd7Z_M< zMA=IicrP%p?Kr@b(7?Op!2hZDS*OS|=t(e4y2fe{eZ=iejamp($^j7hc*Qr=`@v4Bcs{~jLra0T∾JH zv6}1Z`V?jJR|)ZH%C~ng%)59{At8y)qfxensl=>Fv%^V3=O3Q}N0XH=v$4ix-4>0g z-b2b)ni3LLfAo2-apjO~g@a%T+wHDK;RUl8L>yQYcx2lASQQdjTm0Ar8k)c6vI;n` znjBy_;QHc7;sJ+-dwdKm2@cFBCf|%?$TDmwTo9HTaey_#h3$bWE5qIYs#92RG>9-b zm7F`+Ca7ho)4ce|Bb`ao9962E9S!-sy<8l;oHIlUXF3SfXc=DN_dVEn_Lk0S5m(Ct z%s~!|gJe#|99y*^LF0&_Oxrt#+b;R-LF>)KR;;&kl6}%3!opUf+1sD3NhcEXPr{rcv0XQP{&#cmCLDOja$7s9WpXG3Yfb@c zOdfa6H4%m;6CY--1F;%Y8l)@MbN2L{cy;ZB-OMSTU1tMWI2Wur^==WDh#XG|7f*?l zvCiR@E@$7n#jTWMGN@^`KH+>`PX7YeRe=YoW>xdPM(M{_Z4BSh@RG4nT*pbahC_1- z(}bXA^&QL3Om&j~d zC-NK;wrMh%!Zf|@y>7)Jku6hXZFEgrR_w}Q{9t)d?nE=w6H!Nq^nhS2d^?R~x?&Q5m z8ypX4#p^NMXAt_5v1RGSO_4_yFOc44%iyiTz_q|(hpXe~Eze!whzZ>0F<$=0X#SP( zcmEnL95}#vKu5XZwCVL6wU5W^1z1(4G+Uo>w*7F}oP7>2--Q6n3oQvN`0_ZuvMm#T ztR;}+cfOQU13Bw}R1nRR$*ZGurVCoC*qWTwb!)-(_v*dlYoQ^R@01zaH)7cb}@Q zo4jz^Tn7h^H+>9A2joSZtpX0F&6W5rAQBMWz`27_d4Yox3lpmWlc5Xai5CauSI)k# zTkLirPp@c|^3(>-KL?8Y51H;h`2L>%swuC`{69ui8=JgqoN%QvN;ucryTiJCW6b8y zt&hsr^%Y9~%jafHVBk2zd)K7BE2BZ3#qFf4^P1MvCk>s*jz#enJ>--m1>F5I*A2Q7u(jQuQ z8B1%s>#=QVa9v=aeLp?Vxq++W25UltG>eml0%PV?iNgCI^t%PQSeP_-95ms$`DkZj zP``mdcETH@Vtp=O&L?;E4>j=KU{FXgJ!^KoqPW!0b7Mq!;p%e z?sHZ0+DGQ=mDYwc9QMlzQ6Gp=PC7=CQ**bp68M^bQmwYU6wO(*1d9&S)7rN z=a9XClS&7pn_c5S;fTY!Ezh1lVA6ZhAYpSzY|nkB2?tzunB3dO!0n>%h<8q#IG(7$@X=b4`HEA>}38 zZ7McL{tk)a7oE3tt93ZXg7^&a)7w9vYG~bYK;pluz=eZtl?U$3XuMm)q?^#>&~Qki zp@FTWQR9cPa7n{L>w`OPF!Jtj;Jn}<%h5UE&%uW#jT#}%8gm+-mOPYv(I8gQz+&;i zVP)VGvjc38TRjh3cpaDD!2a*|y7q5b_Pbf{{L0wOz;<9q&SZwoe-9?Ge@|XJQR=S2 zu22@mTziQjBpEbc9hr;cw)OFWa?c?uRtm|-EP@%<1 z<%ncVleJvqXNIPpzB~!N!$y|3miW|Yw#3vX-T3 zX0tdiBZxLAHA48w@mj=V!fYl9YdOq9CX5qKl8 zhk^0e=jVP)Y)!fK%@~1kC9$X#=%9#E!@0PB58MSFtMtbPfPK5@W3(WCO@Bz#YBaJ%xVjr z1e2FAvKpEn6rQ>vpvmQvfSBJ6yMuF%aPo*OF;M=ec7nIew#ms>S*+vACXJk120ayZ zi;mi~o9t?Se(~PE32HC9S%i$21x&a9DZEm1viQ5kA|i@8N!O&nDev`@_I;2RB9Ar+R4Z&KeM`lg?*m2$S!B^ z6AOBioE#IGMFJT7`ZGNkmxyIa7`@-)cekN=k+Ym;yYi2JPgbwDn-Otnp=v1imCla2 zMZfqpW`DcAQguPWhlAV>hb$KH1Y|sH6^}Y+@t8{|&+>4G!J>uBBwMXgQaUBqoT9-&U8pCfsvUn>ZO`P76D_v)Oy7giquga|m=0!_qugkDa zIHi~_SM_G&bGf>JrTxohMNLg~n4Y(3>YM3VwVGkGnQBh9>oL4seCVIQ!A$91<0oc| zr~3C z=5zRxp={2cHTi54i%vwJ=i@%@hdYJz3=a00rEU0kL0Chhk;!ZZ=M#~VGoI?Q>PtG7 ziMe%5JjSbX$3VYp+3kv||6ko?pWBA*Zm;p%YC3U)R=5?z?3)ct+nD~UgxPNty>YC} zY@_FfFuMnbWPdR*tvKT=thnT;URfje-o^ubM-tqW0vcMZ4GwW{>j=|zDeO-BcV&^_ zb^&+gE{9oMe-Z=?8xHU&cr!N_X2fkSP=%lKbb5pOzD8ZZ1gZLqpIcBk`ye#_2&TR!VQOcCVl5fQgPznUU)!GcEJo5lXlBT z*1Vki!kHwdJWy4e;m99wf!W=NomEVrC7;Exw`7L0j7LF}iN{5bMOzhl-IF~8WG-oT zN2mx&DYDxkr&tkE=87%BfmmBylJmfUrz#Ng~&}{qX zp98yP0+TXVB0HbVOQY0B;`{Z|)J{!s^X9Xdk>bTXOCaT>vFryPc7<)$;%5@d=Sm83 zu8}yN^Tm-tv_m*=(e~r)6LidbDlaGfdg7tP_K@AS!-;>!hD%eGbh&Ic%#(H!~CA+EkCjd>#QWFM9o)V7egP_Vf=n%R3hZ!_#zCK0Ahbmpo^Z)DXxCyQ->p z@|pwJwNtC(r#h~SYGT%_SSZLkgCWRnB4=#MBGFw7j;h-Ta@m_a(&|ZI5*AZrmHN{t z(v!iWD5c01$dkn1e}kEw<3I!Vkp---hZ=+>7+INAdYEKa9DY>rfbp2$MZ;Rn;8z^X z0jCe}{WI9O+96#}K$6Gc4U=rf4axHyk#7T+88K~W3Hq)I%PohOL3 zXiWH|_?%I+dRmm$j|5&7mMiXd96S2|gq5xlx$-K!Sn}G2TPX*Uwyt`&Nvm0GhUi+k z9}FBT3Ocu#BG}eECqfn(tMJ4-(Rg{Gk?*@=`*3p z#fDkHjJs7qqB7R$c7x<;j}yjs8n(!p+){K@c>1sJmi;%I=8!d0+OOI*ZT(kgmHj=U zput@?jFtVym87o@&8%}m7Zxz`Z~f4~ci;iLf58u-`3th;IRd#HYZOHyUN%p-d4bjX z3L{UP$2|ED3~l;14)T?Q&fEOZo(7uJP-s@<5oD8PND_=*a9D2H>tySvJiH~XP)lR@e0ty@YX?L7k|xy~>MT;e#A=x|Jh`v9BS=E})Qi_#g*Tog~6 z2dMDu6K9q<@nx6Uzm7wnB$}DrHf&&-D8nbrF=;~DhQ^M*KBmhmcPA`6D=u_yMpW?A zjhW#|yxHG({5+ewA#20WJ89gDvM8ZWlZh+s44ruzE?W5PQ zP@$hSa)Gyt0Gp(m(GOR%q-D%=90V91GQ}xBlv8+kz}=Flsn+hwuPU6|0%wHzL#NV`QQI5SaU>QAhqF7NWxW_W9-ish`oBicC1Hg zMIzVAMgfto;NxIw36Al6lG>DWN~`H=#W^3Z2k^M{-DE5Op1a(v_#w*rI-@gydLiHXq5WGz%x-{ip*KX8w)PQyc3X7l(<$P zb0^?e%*m`5iptq}0*r=NSqgvK2?>O+<%n7Goadszzn$wjtQgoE5(M8daGJSonBmAj z?Vy+oW8IdM%m*6Uq!LBe9F)4@F1^T6WRC-Po}z5QLFsjhGDj9lK1<+Vq`+zafTQOC z=b;2nn*~f33)r$4n5DWI*%}y_78_2vI6r8E@d;0Fku;Mtlje9d`)IIvDlKMuvv#YS zfuY3(#$^+k@)$U+9xw_#Fil$gRp!qs$7n zUuJLjy;}I|#RR_Wbqg;IMQnU+72pCTk)(5Sh#kyj{DaN23k6$(5| zj3U_ziBb)G5)b*L6c}t4lv?PUc&#>hz{?s~;4|Tg#|akJnTCl$uZ<#Bv#sn15K~}L zNMo~V;L>@(;KcCNNsX&(@z<2a_vIFM=p`6sFJRT->wVK>)ck<&#sd?HgRfsX2yh&H zz3hRBbEC)-M#W%-pAtnYl$dy@W<4vrEO*a9d96R6`C8=&hH{^v-w_UMAr3F5T(;~r zesStNvscjXV^aho9egqtaK1?p)L>vPdB8n~q1fsH+b#tjp@V!k9$ZxTw@^glpg`V3 zu_wnOdRlKs9TYG)C~$6p&2N&-@eZx$pl>M?YDYv3+P(3#xAq_80P{idx9 zZi#(Q?1kI5hD~5~4Ol4Ez$CW*Zq@~pB`NxU`8~#1@;th{rh$* zSMDI6mIK2x1_6l&oI4oGzoedw%J$$mZ{W5yL^F*wJX=zDnMZ(t@8Up~#H&(@4lGG+ zOmhzKJyQt(?8?xV&gFEF*DQg_q`_0hp-O16v4Mlp)DIJMgxL;+oi1Z%>pRROq`1MO z(OhJa$cl$SUJoV34o(y@;yGH%bNs5(4Q8onUgpm?#IDU(Zl287Y$o;GG()q)`=AN` z*7p|O58_@8Ua^P#^O~71Of#1fb;(H-VQ3VY#VB;FU}@6jJn^EK&kDTfr>-p9 z{A@``6SKHQbJ)u%dQy=K;v}!f9oLUPex7wlNb}>be5N8{u;gILs87mn8_gGzy$j2vFR>({zJh?;yX;cOC;p@g2N8bqfTz9&&$J!2jU^ zf6)PMxrKbc7O~vXwv0<)lv9pROW(TYMAwNF$&)`kG#n)7hbIa+&(ApSbBAZU(*bFH z1z(p&$((-;Oe_tIIScq4bOin^5cY8t`rE*1wcv9Avw*}w*1oS_4_#xuC%7w$f$c?t zq{qVvIt$EM7DmPqsvR^)hAwzTiqMHSsCw;K3l4#u2xjGg;R>FvT_W#@hw z&0yuZKF#okh4qc*eqmL^A5CvXeY+n_Q|yl~Z73>Ppk>b}@>ZWNOOZwIH@!!pPg3c%ZD6t4Nb1pc*_vlHc{Q=G={j+~R5L>}0 z{$>Gxo`Sx3GfTw+-hXlarw#IrA{U%HV8-!Hf#0CPu!`AaF$33)1H5Z~MzvqzJ0`F^ zcCupX>yJ+GHOFM0Q@?sJyW4cA2znL`+Sr%J7~m9X4tQn*=UC9imEBmR`)j`P- zqPvWOn3*r_Vy$>^W6dTJpM(4?i99w7MV2k(=W1m8V!`O7V7|yXNsr<44<0uAW>y~s zx5@_N31Yc#6`0c)`VAkj1sVSn*(9pCc!6oC*K$?fn9C0QGK^9O9AlL&R{T`FQt<8> zSNKbw?<))?_G}liTpApAJi4f}az^tu4!QLl84B$G5;#{J5GYVwqs$^JBF62I$Yb(= z`I_eK9c%wR+Qr}Xu|EpjTNW^1$jpCGSu7mBYdz5a2`xh22;XkT$qx>E8^PUYY9Rg-U>n`yQ3=oIhCQUW_e{w$MfJ}JUv z@`LHwRMz?iZVpABo(KCX7s}sK6hEfG-N(Rfbdbkk0rMM&4Gnt)v}XM~_29tMs*e1H zJPrv=777Bt4sezva85hGRxFXT?;Dw2Ct4!T3rz8aP=M|s5{~^oi?PZRP zhELtBP6njFS-eaD${H^)| z2BQT`MhW~9iGnN=0YS>Vs*}hD`BBo+hlXIw~Yf8*nt>$JPu~^HF)PoB>1UI>!D#{3W;v^^& z`%8qWkVRP6duCMTDVEjY>kJi4aw0`e_0K=l@Mp@)s|)e#v=9$JZzkb1`&IuoLD(FB!_Rfuz<1g>efQ@ioLrJvffx9x_#a$Cg#ic zS81-PV@O>1O1jMLYNA3y|6CRxg9wj=_9?PlOS}xFOAjA9$o^ux`i2P`*cVv(=GB~7 zc=+@Jw@dG4To>-@ouVJ0!gPYS-r({HM$ujuNiY)eN6`EFXXQA{_Ah zL-7GPF@HN*V>UG+7s7l8Wxo_aYY!bu1=%7g0xKH5!D4L zlft!kT_`#!C|$wes<4Pbi9@lD=MlSPcWUMh$@x!$j>(&3FnY)*e!0jl8T-QEAG>m~ ziUT97%>f4{CdH5qx0u#4K9^!XUKrrWENF1yK-TuFX)hKW_;|CSK(xStiCseH=&iYD zEg2eFdCnv-i%4)J9F%KFU_Z>v`qGh$Ye|n1hvo!>q;}T)SqEG=MZP>>lWmxA$wSfm z%PLmIttLrl6>i;psKut6!q6&P#$Y6*d`Y0VUGB;=M|bHPX}8&A+Z~wB`n~({I5^;& z$k&HKNb>!dr3SAb%;tr-Ll= z5{;e{-O`FqN=WB?I4IoM62u|d!t#hcx$Eo02>J9MA5ZGGbu4O@sJrv=qC&03N5_)# z1^*Nrm{b{ytLutBtQOC0x&Xagdo+eoM`#TiASh(v&lf(q;BYL#&U*{D32GF^^L-dq|68Thd&U`$%mUORSvsl5=e=*ijL+RBcvCB*OukU?VYRT0op(xO_d5J5loW4!=LyAzHmyBT&_3Meppdql7^1vGL$==3k^x-~EBq?GN?^Yw=|Mum&BN>40e zVA>PI7d7`Y1CK+aV8agHe~xkj4PQUxC`bt$;FS{W;7MWN)L6o1C}G%o@SJX*n1I7M zhDJuVhy|>Dx{cBbj^<&4$};8!%q9|yoLLRbQVUqu8ZR)6x*)ev##)ix+T=?ArBuUi zPS;rR{fC++wiWi|WzLoO{nWZ=-^aLpn^y#DMFlo`hAc9xI3&O21Eab@ljIqVqpO%6 zwoT`HBp47eRsB~2hqJ-6-M25#J6WK3#qmXgKx=}S+>E8|mS+xfO8szPw+m?G3~^wW zVMx#qk8F&*6%@G9bLmBwt~rYx-)3w{;bN8vn7u-GxgSpkqe4hfBg2VO+l*%~mhsx*YU7)f(E?qC#XeDQ|ojf1~(o!J(~ zRyT*h-IKNY`bwl9vF9XfE43Gi^rXzuI8b(7e|11*uTE9++if|4?n_;J)_y)HHtT1j zYRa43#)KA4BeDG#44On*41&wIyQjaK_fTm2hh{}7Lt~AA1xyk@7#Q!RFk9bo6j+?# zB5(1bS#JkJCWns*v&IB_$E|6B%}UwL`m)zpf3Pv_pXt>w{c{7m*>NV>8IJ4<1+!TN zQV#PSI2U`}lP7IMqP4Ap+gjGulQei(1SC}0q8E2-M=Q&ta zW-#()F4!h>N#c|7sm3Ut-ovtNwi-MA{*~|In|b?=QNzLTC5xJFx<0*|zvR=qbBFY` z|9>l0X^7?cfHPU1tnSz1Ti{;%vCl%)_vO!FnOPZMo<{Cy5CS`P&ZK6m>VrF&*T|+@Z)NV&Nfj zXF<04lN^4D7mUUMLR~IPBzt5djaz4H9@@KN@7YDE7IvyVTdiM5b}Rg|;rVl5a*rz`58vFG`~i(Ti4jh$ zc^gWDCMXGbD@^`$$bnVer;#hTfJwCHz;E3tpZUu&nly42^5*s=G0FaLR`y^FH~Xr< zq55Ixndy@qt|}~_*73@6-oNxrM@6kG3+z@e{q&{enME&OZ$wLIY;VI{UX~`lj{O(s zcP6f}Hx_g$5oXYBXv#2bWXjvXVLpK)Zo2lu4?*juH#AJ(xU#IA*_ChMm&^2I|FJMs+P?0uZyrme)uFjQa#QHDs0h`}~>b;lDeFR!d8`#Sa zIQe{FZ!_SiXJ8WzV37U5T2jnz8^EOZfLWp-bVW)@%_kE#=djf(GMiPrFNsU9nrOd4 zv1DDc3HOH*H};PDOH76VEI${Q<}z?pePH5iVBi;Es+Z_sKWMC;5H2!-S$%op)KC7>qm}&RCjoWPEv+~+N<)Doly&GS|SDlcuNN_g{ zVBRr_qk99p^Meky26ag@0rdig=LZEWgjtVl;9zW!5(!|M7m>YrlF%gvPRp50Js)^t zk2{`uz-GIF<1_=`Lj%5z9f}f{ri7YIec90cX+ifoSG~xW-Le+!F$Y*39D0sd^ymdm z)BPaAw>9;1kyz-)ru3~MABvipl->#9GZ19E(`e7`SRUxn=~gg>7JLd>FIM!8e~t zd$Jk>j{`%+q8Z!@42%K{Ob(1@3>;evxVHyzL|tI1FyQ#|P^i^x*ya{{<^v3Z0gTlLcrGxoFbgoSF)%O)WOH1Qu`3r4 z3FtJMnJBhkvdDsL$phY|AN*p(S)~Lx&Oa#K62zpsfU#^tS)K!LI|Fm~1)dEWe4#FU zFBzu3vYfhjbClF0;kBXj9VRwLzlnSb$<^GO^lhQ zZtz~ofc1iyV}o{xfJ*gDZi!RV=WkFEd(6_}z*TX9=~kkExPz>SK?b)2gQ!F1DZ#|j z4cfH^9L*0{%RjI;JYX?5U^l+tV<^Bn!L)YG!M3VT_FDrp>vznmOe*=;$dpxEA+s&A z?q{+FcL0mr1qQ|j2GIk?{0$6l?F`cn+x-g`M$FH~Rk4F$-44C~(f>T3~0ez~TYZlBflzZZPp}?cuwcnqp!QBDGM!fTfs$ z!+XK>1rxYmT;O~AfH!$Vq0B3mRsr5-2Ub;uWx55-W&w;W3Jm>~lf$>JnPSPF#$^1Z zk@2(^gK$EDxdF4kKy7cHsOSm(A~W1TaTxB~MAMQ{IG-a8uqF*`J{ z8%~&AYAV2!z;I|HgZ~DOB@;L#SeR8A7^1V7q_^-Kxxi-Qz|+aV{KK91!UFcokxUX> z1XT|BED4Zm66LZjVDjSRdv$Zl!wd7?T2AebYOFQqf5fy>k9*}`2Y!dbsoy@7*95Sa zOjv38dusg4mH#EDIXqx?l3ul_s>j4(fnLIbuIEp$V*M2MVPV(zXR~H6P%p5@7Yaz}^tR?9r^w(vYB+z+T@FH_ubnY6_!|0Edm#)9T;9h0oZsmRE;H(WFHFk-bzWCTK(4a0Zz{sq?QoVu4-=XyN zzea}1FN$9r)UqvL&T-)C-XOc-6N5p6u3CevE(5bb0F&(o_Jrk=#SSp5PY_^dWY%zC zloDWVDrmC|=Dsvx#j)-c$9~TfpTYOZlK+hYfB&yd6TJ8*X7N`PY%Xb7X?=S0++XwU zxwkCv+F~KF>QQx%o&dAw>@Dgim}Czy%094iVPL5iU@L9pez9PyV=7xj1M7kWHiZK$ z-2$FS)?;$YIk)2|ktC zdqOH3SBFkLcY0IU%}tkfvtRFKzqxz<%v~#OUT>bywXcF_-=pY#77omY4$OKCCkzFc zoe!{1bYSCS=30AzdtCxsx&ycCBJO{$3j~%+ww4tvVPM(n$gp-B!WB}@g?_P6}MFvyS-}rd3o0JUx&mUnBH+oM=~GQmYFDEu&>(TU;u;P z6(N^39~tgGWD_Z1Ub2CmnXz=KSUu+l2etyH=qv$AF(w`XCf0)jc>;1v9e6G~@Ypsm z&g`DYta@xiMtRboe^a_<@o7Xi2EXL{_J)6lh1Cz9y`LJ`|Hb&-n7w(H7H5e9XN&=R z*_*vpGFvKn7C4?hZnA*+xCDER0Y^f@>KPL_XEm@^FXY~}f%~5|Yl8#N(hV%z85zI* zV381DP_<&%{;*h`cT#Bqd&vhDQ--YkB*DGEEkcD?Ft1HVS&iP@VTf8hI8)BWIDdET4z=G~hutdFzC zUSQMZbo#LJ{_P7>&R($laN+U4)2!7OIO;wy#z`AW&E=F|%iU;cu9bS}?>0G+2kKp( znfiMfcze`k9he(*m`wv1j~rnLVBk=nIl^+|P#J?R+!|U`rukjitURL0p zotjhbP=59U&sX02KV$=L&&(~0<-6UP`=B6~(T{JXYWF*b#;<$W^Ukp6$FSEtV6|Ai zQs~^?32!$$EVv+1_fXvKCXej2FRO0$Z{U#Mz!=uRIXyvT0|U2QEvEuQM5jWzUjT#u zWUt9=4i|PE=$kov&YVT&oQ&q3jExiO{;f+o?0i`2?%Pw#XJ%^H+-aBYbkE)KueE*S zL}~3eCo{Hc@EpjfW?(U6*szr?c8LJN#{;Td-!1HS^( zi^j7r8n2c9dsL>t8YsYV>4Is!1J~69JZBkV+v;9ic*?Vnleb$g%Bjrd{s*4>A9$aB z;AON6SZLhHzV1Oq!3~~w4+Q6}jB_|Ha*u^`?!$YtA6DpWY1?CNtS}?QLB%ad<*9^3 zki)q%0&J228N~_A#s^r+FR)srFqa-+5ic;-owdmN97CzAto5vt-hxuQd!Z|zOi=Gh zxFp^=?b?&Fh(qta)ZpTKPXj4f{gyAK0{ zVdk~uxeWRL7}!5dDvqlQ$zjy5W0q=QGMUMtn!vvG1JBkA{sQtJ|4i{0x$l3@fk&J_ z>c|IPnJ>IY4PGjo&wcuVea^d=*QJlo^tvFjj@4-atHXm&Q3{-f`yVd=;*>>fOb_y&J zXIbh(nS2^pWG>vgxsW}`f$?e!)B7XOK3@Nk$M;;RfT5&MZt}nL0_qo-A2AkQOe(v3 zk!S0Js`>nuC#ShA_U8$3;qw3D;??rYYf1Jxf1XndvSt4BUNCrBExt|)&D+%0-9&qj`;NEtD{i-YPl>_dr z7ryi;u+|o^R|{~I1#sydGc_}qxgwE$!Ue7J56ml-r2aG(G%B#_sk8L9s;{5;hBtEu z!^MV!2iUl!EG$keP-tQ0=h160_;Bz*8?Ta9jDf(RW;T8$sXYf4xpFg@3vq68UM0EN z&0Iod!Gko(DLjEWI-83hG&ZyI^R*NRTse4tfo*M>7gWz=J833|fUst{(PS=Gj|+3e#3fT?BxHAeX}UPyxZp>OLURY7 zq}rJm0Sg@3ktESrAadm! zgNwyOG0zmito2*3MR|x7wg|c_{QMSOpeWcDlh_g{xYqvL4d#C9j3dnJ_wHff*`Ofm z?IZM0v8%=BWSwHPkHN8DQ(cS{3MR1hJ9?~aote34W|5TM(m9i6Wk?h*@^NEgOyRe7 zifrptO{sp*X>RDTQY2_mL+}I_rj3o9K}<6n85zYm76h~|YVKs3U)=p-(aRmp9@c#t zy$P2%Ha*U}JXdT-LPN7z(E&>yw^`2HtZJ4iPH|7$pwK8(ayfxb#OlF9$GCb?fg9{j z777|Wnw>s`vsTYCJ(TI8$STDBt~zak?Ux(P{AK+2l_wtI_iQOVF1*rA&|PKv9l_K4 zZNJ}12x3W_V6Aj;aqR9VU!8=i-X8rPUqAK8_OG>nbtlC)7cw+)%g*|7_+gG!vYS&h z1Lx6cQ=e%l{$rMwSb3SxYQ865m+Q2|C2jK-Cb$UnO?)9+p(&www3CH}gQ1aSfmlZa zyN<(rCKv4)E0*`I(*AX+Q>=htQJd0)EiYKV<*!iW;EHe%Y`CPr&2fayOrx>I#h8(i zp(&(6Lxk1Hp;6{S0%zF_MwV$tYglxSHnR&{U=}E7+TD4eh11HBe}Q3dj)1dQ=LrUB zsf#_X7Zmv`H!kG;x`5s8O+i72^$|Hf!z0IR`HH4&o?{?0ai{n}?YOFsPBJXhcGXGh z7_ADH-Tz~$mAuHC1c#1Bz8C{$S(gsUbRL(M6(3u94HS9IXEdDGX=oA>InN|=!C^&2 zYa^SAg1LCY;VdPQf2_71&Y|TA$1>dt8(9<@7}+=&xbKLz$bC@Y3Y^i%z-FNNPdB0F zOEw3O#ZCs6l1-BZJ~;Z@XzpWoC}QB2Xke6cXpwVJV6fld%u~hCabpUTKy8AX=8TCQ zl5=w;Fs5BvfNo0XSj`*sh z8g&!eG?qN%%&ka|yQOem`k5lX^`3P8Y73XTZ^nXM4R@YCa$3Cm4u@yU&PDt+4a{w3 zQ_3tR^oVZ!p|4*RxbW{HM&5Y>9YGasB4s-oMc58@NT)O~tkpUg^e%#lhl7FHIN^}E zl|uv98-8Y=h9iX#4 z>Z&kb1}#^PhM=&k4Dqb1ncEf|T%~t_p;zVrg9-yf(5D7=feZ)XcOtDpSDHi)>9p#W zdCjs6Sio2!Zo~WwK5JZ4=lOw=~ZE?Bg8ws?xuD*TZ?%Z`AEY!>3zaIjH~V5~Fjo zbhE^R1v_|j^NJ*QU3lsw!ok(6Q6jD-*_O10*{tA!r2VTF%QK21Rwo?T7Tjd|63M{K zxq-pNR-q%X#Ig4LN(O$9k32#DY?ZY86IkjL1-bSKJ{D9|P-S9bXyDx8&Hvi#h^*BK z28M#Is~8L#!W9iz*cBKKar{^n&io;iDL^B+W#T}T~N*Wl| zeH++zlru9P31pB=aOyQ)z^KM^h`&YQhOiUIl8LSx8r?MWJaDv&z>-=}K zH!~V|kKbvQo^g;((u`Taz~Ugc$+;%+6AWzn4U9a;_ApOceDjR01k>!x8$Ow>SjekC z-FR;9?serK9o;8L>hZ>0c%f9V;JGV9)rHUt#e6^Zyy&p|vLMzW=IJR0_6cqh@>bgh=vig^RenW!*D z0mTE4nh(gaJ#5r9Im9Z!*}%^*k(u9MA*b;VXAQd#jnV-JIprOg>gHZxxB1e@W6b8V zW)2sp!ySj$2`3y`Ux;w2d(4pxXmC;RXxO(TBH`MR1#W9P5{2D7k_E*%b{;d=tDDel zbH=*j;EB_3!nUsq{5$qt(F}g40+UPJ;!L?Qk|wuUbu^k5GVl7_(4dsjpw!W@ zfZJ;EZ_5Rf8TltPa`YVV@3gkt(5$|L*y=bDS1}3|7_dxWV2WsoVrX^~VCR!)xKq%y z?Q)ZS2FsMh7XBB_4+5w1bF?@*H1REHa#Ub;HfYg(pd;MS7;u5vVFFt=M|(~}lhuMY zCjpkA9Y(est*$$k32-sK$zbje3=WJ=h9ZaMnGVk9XgHSP zbATb_K$Z^E^2be`oO^Vp((1P?Q8yvX&!vm2~3U+^SC@2gg6A)1sGXYD03R^)9`2rIMOA2 z!s3(k{?I==WI1+D_O94X$N=l=E!hzJ5 z296Adj63sPKYC}BFie_qER$syi%6$@MUy{wv)zf+TAge$6=$+K+Hx;k>Ld6T^?qe zh$fzlCWjLYK^t1n{%GRqm|C{F!O?=*`2&lg27~#*gF7l%(i7Xn8ko{JSS(&SNbhKL zdbxl_ppjvw9ruq${s|Ti7LAWrvN}{WNc}j(Ve0xR`{+8K{a>2@xo(zn%`u@t=IF)rl!ndBhg21q zqYtzys$NoD+m`FWuD*fMDuTsqN2Aq`r7{ssx<8l$HCj9*SQJh$Xl!LJWog+Y%pkRc zIZ%Pc=0`*5Z$_qw1~m!Rm<+Z+j%LXdy==^8zb5X{F=)DG4kPa4EtG}y0Tb;@XQV_@cQXtF=ipc=uX`J!3>0E6^m2F42w_Ai>h zeQ9D6U|{o5aG1f!f1rWSz})&%OO zT@4SXFtFaZSbBGU;|m6Jr54o}%rPI>Vji&8=mzC`v|DRTcIjwhx7e+^V6Wd=ap>nd^$mjqd_Wx$>2njjz+WJ+-9?Y#?sJ+g$@nO z0Szx%wO`6cc22oB*_B~(teL%QLzoeRP)8%*2L}ENj8YdE`F6~bF*wP8f=R!E$%>(c zk0X@v1B29z2HOf2E{_JDgm~EtO(Cxoge+QIXE2^965g82mQ>K}trab|qmkvo)t#LG z7`az4x@E9=s<1geV3gKyPFr!B~aiF^c`O5KDWQ?q=UBeLCZ5Kvd09 zsj}05x=**RYCM|CIKTQ1#|iJfPfi~Y>0;dwIPvNHGjC5bu4s_gXlXst8ol9>;@NQR zbC0riuul2YY;~c-DKOaO;rt~x=Q9X2uxLE)Ji*(Vr^X=RH-jO_(BmHe4$0?dFS~7v zb58T?fA)CNQ#Hrf$jM(l*cR;ibd}Nn(_+a8rf;>GzWbUbUo>znXfR&OV5`6;_F(?> zjE)>D2B90wZZn#l5?TT)TJ#c{jI}JxW;AkeHt}&aGR*9-jbJgih;~SLAW^U&wSl28 z_?dV{W5}OVpEd{l%ib3~(RhD`=*}FzJDywZ?OyBdyKzm(R8KsEF=fdyCX@8NJr^^0 ze9Cy=?2|e7t5WsB5_yRg%s~#zqEE0YoozjS?|Pm?dtL{Nutko~~o6p?FC(dSrqZ~d&e>30uJ-1^qZEM>as&90s}7dHk4fw|TK zOZj#+$`>@1|7jAs@kH`Rll2Z}mEy)ELDif?YV$f~f3`4k5MVE4@JdQxQo4Q9g@ZLn zku9@|b76tNnHe*$g}zhBVG~r?&lwsvA^KUyJw{On`^;}%JiXT<*Bm|Y^^_Fb-0dtp zN3JIRTYUQX*3(K1k51j4|I6^7s&wyjqdHZ|1>EBQ<+;?| z`OcN$k{+DVFFyEv_`v(V#ec=usQx79e;l4tim5k3JJ>EP`K!rMd~J`^^ulv-&TR$f zG}s#$Uof0_z$Bj0V46~-SbtAZ{HC*7%2d&FFYdl|c&g@X_9^*Dqx_AX+#H7D8O+un zB&>Hd#4cji;%JuGW9!0|YO6k>t)js;qf>uFv%SVX4F%S;%1)L9hGiyP3k0}?4H*9& zWf0oHY&)aTDWWYkfW@Vw*`mQ->A?@#hArX@nt3S`_s*K;qN9e+HGKsHee07Q4nIl8uPp$#+)@Kqs8M#yX%f- znH`K8Gnh3_Fft~56!X_&$oS})_pMQZVHJ17zwk$Io^QCzq0x|bUxh)UdK;7B#MbJ< z@W}o!h8~%a&9^wun5LB9n;-J`k(7DG8hyrThGq^4%~lF6GAgW@98I6)xE4ytbL?Oa zifD0LA+}zUHKUUP$6!pw1ws6Y$``!hcQ7Y$5?62?rWj+BtZ13?3|4 z+{`S%Z`1SOKm#iqca}?LaI*^&8)ubH(h&}!je*@#t*b1%M3zYL2v}%%U1Ge*VvxtQ zDY7_F$y!r5p!UN8jj%O}0WTE4^+;^Am6W(z_3VxGt&~#}REw9q5LjeW_-IL>am$HV zIlCI3j|ZDM_~ooC4k+A7cITI9PUML)$+$4rBjLk^k4El70tyxdM-Du6V&fK*>1t_M zz{Hd_Suo7GK%jBqN!Io8yTc9?Ffxk;NX@kR%6V1Yf36klt0ylnFZY}8bGOUvMATaA z#3d}7m{|oDNSo&voOEz-7FILoFi3cGgq2acU*fNTfPy0f_rEf^t0_VWe41`%3ml?c z*03@$hNaAAlAim-AydZj1B0{NT$hW^f>tg<%>O2C@~C&T_~78nXfwefTr9va29+#l=GkOKgJAS>M;V zoML&(WoKA)*Pd1nnUV`rj|s%^eN>He{-U79Z#(0rb3!YRkY`H8j+d;mF*b}x`0aDV zoO$$GwxtUSOki_lS@BoncFyXPY|<|4SGG8c^vw8_QIzlV@t~l4fTD-`vWi17rfn|C z-txgQ>KzIc>**;nt* z2N&rxUj#0@D8?RnFRSJR|4C6gWPgTXE_E6Z@jp z1BWtenN~?H;ND_(d*LS6C$lb!8w7f2E^D~NDH)>B*upEHQQ#s`ClTId-fEJ3S|Q#; zkjrpMixP)O&x9Z^&Dt$l-Wm-&NvGHL7##Fa_TgYSuufN}h@Dg9M1+&lm-CoWP?PUCa+8`|aqg);byUf+c%<2|rJ}`?j1c}+a4~gG<^yYr$rAE@~ z%01EY?PuJlMJ_8e`jyG^>9}n9j>yQ=qDM8$xvSotbmDORGgFF%cgM3^SrgbCp4Pgk z3jAnPSoDn3$fuDrWCkN!T0;xJLL%4fO|GU=A2pp#CI~h%xM(bR&=SycD0d+jtER`r zc2^H&kyXDA$a)kqGX@+eknm`!%QIl-*|LD8o`Hq!!{TE7 z>fylI;n?2UvrcKUbrJ^;10(a6L_vcM4$?0^FbUc+_=C%*2VNP|4G-lDxzw}({O-SNMp{b z%m+;S_JA(sWYn7QFz~j)5#3oAI}A2x2?TJkC@)ap;JDk$u54#y|yE5x(gh!ZSO1#Z+$yiZ@(jPF=cK?kJaZ^^@bgkq5LMI$dD; z7r^bTw`r+fYKqOe?+$_V$ zr*j*6CH}cGIMqN#;liII`xlDxHpm;EWb}E& zFx5tZt@OYudAkjb+#wAt;xFRaHFmW4hCUWsw1HLe$c09ZxCCq60}Z@64UJqW4(xsf zhboK`lKZ(jrroIV$)W79HRbviQihWCL@Jj6lGp zpcK~?tqcs)3fmd}GzfM#u(2@-ux368le)yvF1KQn+r5wPly#K2y8Wux_e=Fuo-}z-R9*XL#oLtT0-=tV)cDjfEO-r-X zi!&=^Ud)^yRym7f&8MmBO0*b47Yj8WWZ+sT;M!z-fsv!&0h7#+SL|()9KkM+#d-tO zBr+y23Z*Y-6H#zr<+zZ|_GcF3x}^`xt&X#ZFFwH6x#NH?;|Dn=odgC(1qTK;hC3WB ze3}Q=UaZz*=;XGU!fC+Z!p+1G!{Cuna@XpS^EQiy$g2+pFC9>4ODJN`n83uPn!sFA znXZ3&-c1dwq=}R7{+Rfv)nwo z_dDiqzJ1B5`Ok${PnqZJQQ(|>`a1j6xSvb)d*he<^4vB5+mYkE^0L8rXC)xS}nVi?yz3=Kg2Ia7I!f_7(4sEH)9VS8J_%1Jt$b_<(y)Wys6OR-Snwvpr)zPngHyGhOzIljV_n^Y1#p zeb_9(rzrR+ZvNYw?`p+u&iMS{kry}-t~oO(qo9H3i_iHL3_+$17ruD@Tfp{(msf|C zt9t{3FpJZb5++fGShj=%>lkH4e>AGkY0`~hmJ~Upez8G+ts6L8UfxqW$s-rQ zEjwdQ+7q7B3onVRVtBBz;a|eSlBfMaS0C50axs`TcgMfS!)85VAS2S@%dvTR=t^y)Bh zzF}1HNMNx$B+PSwAtbu_z?0OC3>ilqxFZ_4H$?E<`Paao;J|*xL7IWJFQ;XD1`Cr4 zlkt`VyfzC=e6DdCuqtajmHEO~AnMvBb0=`Y&A<~!4xCVkh->8$bIsJ@cw=(@RqFxH zDaUG_NY4K%DUhUWelFGgp4{=bk+VbGWC|Kp-msnuy|r|sgXlee?hXd-2@Zk`zBaG; zwKoQ3orw8jdga2+28A0l`sfgaMZYY@Wh4&uaxL*f(#5U3=BLC z+&q4~cO3Z7H1h6X;61}A_rihCf|2)yi=fOzcMX2e8*B_W7)}2DVcGBKETVFN=|F?r z8^(;Sj6McOx?~&*6I-7sm4}4o`3Spj|xW_p85Q57}DSHh(;-9-0|;ha^rsE(=~5WMuMW4SYz@w zp{xi7Hl0QVk6z6`ZUq|cMK3fmW-(crT;RWRP}pVeuZ_>-^X}heJf!jGpwOHdyjL83 zZ5aCuSr}ay40c9w?{Mg1d&as$)}c~?fk%Pm4WrYIBQh>Yycv$l9?ljS&g>>k$|o8E zJC1a%Tg%rRczpypaF(I&QxJqK2=%SgX)AnnSk z^a?jE-$cP4x6h6a93>G>4V^H~9H(v^!FfuYmj|& zaINWKSpf%yEJou3X6v42b{;0>BMxpIE$$IV+y#7mYB+i}ESFXd3`?0}Aa{maF5F#l zacGAF)10M6GeeH7owwV8fpZFjyv7TQGtA547RPWcY;nt2{;GL-UDgUYCl!}}PA4)> z3V1LHHoV}QacLFTvV?66YlK4A{&GlTa1=fuF7V>OYbCxvlPMGS{ z)TFuK&_+4!8&zw~1Dxd<8f8luyU#W#ED2C@aJ(JUA-yHnc*$Y25N7i`P2wUth*|UIeDv_F6y3M(+NoX6 zWhdvI&Qo4~P3P>ToU@mnDm}SyTN-38|W35;Qqj=40u>!|E zpO%XK)6uqPTt#h=iaoXdO9-58Zeo(Fk8lCosN6F?3#{Py5UDHClv*!&*xYL|GYVW>;T_} z7oGDir7eg_D`*toQYc&DDBHlOv2~+(Mx*$F*Wdc|=U(yTd%_@Ea&}ooQ^AuJni}qk z`s_bFnI&E@{FCF^z-DmJB+bvwahC(ElZ~=C ze0VN6MqY8!)j4Ee)5jvgD{io%Y+Y2>HixTuj>0wv6>OY%Qy!(pUh6z@*a^B8cD!0_*(}@KacAl2eaWZajw%E~Z!LA?U^kUhmH!X7c zb}Nd-ykAWH&?s|;^-J4FzAMFFPaU{e)^>@B@y8}Z*)u-sCwtU)o=j7#fHqo}}Sg&&RzKN=M#Fs=;PqVT2hztLOSl7l=eX31*EDepMQYcfIM&wF-THo1m_ zLOKU!8ys6+Hn3l4nEmA+Co{_FB5nQ$bu`u3~1X)nY+ z9?HGGc zlv$I;{)B<|LZjRj23Z4V;|-r=H#q!X^4oWHOM;X+Z^c3Rieh7j?fb7C)G%?f<2}Ss zz>syM!SDXDNs1F0c~}c{*UYs#vOtN|SwO3%ee1!4(tCbBdKSz2R%zoqjm3-QRzg+KT z89A(fd+Y8=Q8ph2#)JmG3k|XxrpiuWez3{e_{VSEyCp@=49r^&tmAWf9M)_a;Vj7G zz$kD)-o%+@_ur}Cc6*)iaMyU_854HP(B+oq<_DsGI+sK)OuOq^b;ck`R4ynW|q zdi^x79Rp`U4QGr4=M4sBkxP3UYfi}ju<+P5FR;qe_=l+X3%e!8EvFrq*H^9Ru98qU z6X1zY-tC#@{VjRNF2yTN8h4mJoA3W@Z-3#pjrQSR3Jky5Z#Ybv&Q<#F0+;d?MqY(R z*)Ioqb84?llxV&hd*$j}UWV=BHx7zdFp4~B5Wmy7+0NNmhq>g(fuh|zy$Tvsjxabp z$h@}Z+Wx?P3m%Dp1I-KF+7_8=2`pq}5fc|PNN7lCWM*gM*&raaod*O;BFBM)iv$OXonPy!%*u2bJ-oQez$>pk$c+`uA zkW;fvvd^z+{H?_sV^Q?@*v{GJ{&CS&UtZmvtSufNTl4eXv&*Z)Bv$(S(}MUl9TSdh+iDFH81tro|(nZ&(7f1J!x>5O~ks4iId5tOHf(U^4}|i zO_LQ5w{nSS-PqUgkg4_F?(`cMlb83;)%UhCx%oMnZ?SC!8!r>nLZzc@qKXbTF7w3b zp18pNa6-U>hH1tLN4{JPYB|ixCQuY0;E=%3Je%{IP4uyx$p3%Od;zTp6w2UaWX_wV zxvY;bp+J3pMN+_WHrr1gD_L9)2pnd2iFmbAK*!=?qkyi$LKmK@ge78`dw#vr-grup zTc+q-P_}O6i-Q~I$-bR=dR^^@o9X&{8JK-~4_r}D@;oZ?c@dw@yM{9p9Hmx%l0Rc{ zc&TJe!OCT)_4nQ>Qi#dfm}(rj<--GJH5N{m2@cOD9P)Ks@*~OAVa|&OP24&g&Ti*2 zT5y2rAD4#2+dlg_2}RiG%KpSB~R&$`M(fq~(l2J3mfT^o}7TxLYJ9AFUpv9)oU z)g({H`Atn54lOKO6%f6AUX&K&3OSBM7j8L=g<8eB8jQ>`5{gHUitX+NatbMs<6YAW#?lP z>vtR}6Xi{^6#g|aT0A(g>$OQ!0}ETkheO@1$0j>8Eob-e5t99I;IOAnUfCi+H-^YX zf@yDF_%_(eZCuc=BW+Ni`C4% zjtD!gIi9F_`G$!$mQwlNtt%SX zjV?6a`p{->bK7@*UGVH%Z+dE4q^>N;Xl{D-a}A5m2|i|v10Crm&652q4l{6EX!!2% zfKecVfo0J#M%j#o{4Fb7O_UtvRtO3-=qa3M(Ynyc_n?tect)dwSRhk;J4YuqdyX$l+%4m|Z86llx!%2?i!d1x8CwFOHNKLcJ!c%8C{KI}V8YNq1$Z zJHC_<7bU)=9h4z{;huIULedOCGO z3b$_Q%H-l6WwQ;8?wyA=O7VxrDZi^|p~}p}uv1RqFvFRtOlK#l zS-f@D+pEV~^sq&{!=s^bRY*Qtw}h+7p?`%(j_8Ta@Z@nfd!or(Ww&^q@ghN+Whbm^ zuj&~T3aoHY6rOoM@R?Eel^Mq-u?JpVH*M?M))nSvYFzv#rv1`a7w4JHn*A>GO8VA? z4)V3L8`(NSW*J@KU}k&358=sZ={YYu@$oO zmn7^lIngLvbb-;DVG*C|2?pjFA3D@KHu1e&%%ItI(>!QHhlrSlo`2D&28BC}0^Ju5 zD}5{Ia9Qx4uR7s~Y)UXwL7~t(yEPjE72N$e-e_)6+2O!x<)C`BNm;rl_D#}hC96$W zf;XL?aLha*hEMW#vCNEC_UI=J?3NYZHZSZw;=D%XA6M5*&lM8JaV`_@+{#UGmR|kV zEqieww`|ISm7*tZ&Rdf+`*zQl2~AICdi)G&ydBWMD9W&8hNMG-Pf;R&)B~pqeSyX| zDelX;GaLjP47$=ynD&H9xv(-AG>F+`@jqO9P-t4&aXudgo-%=unbrZ!L1z{+%{;P% z)kBa=ZGi)WWI?mEkpe^Jp$Uxa2@ISnoV;l^%@Ri*@SFTI_RN^j#Cbodfz{%rvqIP9 zq8izW?bde~g%}nI9=PIWF=6hTf(B=4%{a4{uUy4Wd3}?y{o{G>3fSLO zoN2${wakc3Q#UQga6;CNLkmSiKbbv{eXp9oSlYX8Q`YR|&Ht7qHJl3jw%tv!LE88- zTl%iQz4IY1uRa*g#}tm+!oZ!url$L zDC`$|lE7`!v6N5hLW|=GqqX8M7&QM)IB9)PP2|XiW@#tKUY}scS=uUGq2Cxd)i`p~ z4R$c_D#bGM8zc&z?l`p8)JSjffvGWSOcU3+ZrN-X`;UPIrnFN>-bKs2Kk+W@t=R*M_f`IK z_wH`^GkN}=ce)Pw_jm5xH2;M7Ge^ej=?<^YR*SxycRy&aU_(OF*_jTFUjKL*nf$vO zn9b%*RJy>xF2~>?xMKALbsi>xYJrw{Q3+2at}NuV>Nv+<@}XUA#{)Cn8IE@~t~PLO zIl$DT;d-8VWBi|e*IVs2HW=j_F!Nk_z^1$6ApabLj=&xN_?C1WT4&DbvSZQ#jX&z) z97Viyj+{H}wP2!wf?!wf+bemDQ+1Cn;FVdj_T8pjruA#Jx3lFvFv*%=rd7@6 zdPazksp$jLf`r^R37k<3Tnr1@k`C}SG%#nK4^~o87JI-Hv4D|Hf$2pvwR-=g9 zzlDN13d|V|OjZv#84}qZtrwiNfZ>!Hd(?q#PK*L6>r5pUG7B|IOj{`OPC;beLB1IW z*_N!-nK2Z}B_S zl)zuJfXkjUYRv{l9uM{c2KG7!x!DaOGa4li2?*R+Ao%P6UsOL^#sO}rgUnqCE3F*1 zr!5fVbCvqHC{gCpLZ(Ch0$&(}JPz_rc)<6kTO%%m(Jh5vj8Wj-L_x6yOe_xU?+%C^ zNtBy&fO}dauR-8{tIn8?2^@!A)p93k9b4*R*TdqqfK_IVWPqUPO}=+d>P+ez-mg0= zW?Z0ITqvM)*h?qbI7d;+ue+klee0@+_d>MUCU4Z%@4vUJQTo8APYK*p)foC$7)pI_ z@h?dZFnM*lV!{#;zXAz8^BD{HdpZSYJYajnaFBtKjfsJY?ZCO32b?Vr*ykhek~BoVZ3RTVERmf zF^{3Tz+X9Lg{Ibne|&aEJd%laEDWpM9HrJJa_(x7e3r2O$NI@V!VH!xS&Dl9uUho_ zm|iRsH}kgzf(=F*H}!7Lc`KUS!n9deeC|W*gTgio9VPb(vuGxZwE29LdA0DtiIWKh zpC(+?-pMGtX@`i|iI5LeCa(zFP3l>!8?*#u*>FMSdlV)HSeuSRlBkf%(G% zF}a1DD<%qZ9pwB_B=9T2w6l?ysey406XOQYHj_`STbe~?Oc4CX!2FDvdCLQSE=Q4l z4V*I?dB1qI{W@>aX{^5V(RIIn4gOv(#TEyYXD7-WIq1_YBy&XZXv&kLcTC>@I%hpM zQS#7d>&XWtCn-uQq=~4x8P96G>+?aYwNU%MvEJ5 zw;{;!Rlmoo33bY6S^|D|1f0%M2wadLc*lV`FTwQ70w%VEbrSCa(-Ih25+0s9^K9NK z&b)+XrbNk{MnSs=Y)lX3q`xq|N)Y_g%f_~lPefxHSE7i-LH?32{2#Iw z7w|pVDEhj!Bx;S!Cf)awglt}l_<#PA@NY7s#H2)#bu1q*oz-I9y47HPfaugf-ET>{ z>+N<6En1bRAid!8x-&uL+-E#i1#Aj@`0CVlg9JXYg@PW60y7k7U-MydSyu4nMDd-k;x)Vje;Eu z1!EL6HnkRR=q%Jo;EQ={q~9SZw=mXCQAB65WX-!(Rwq5I4Xs(&uiq;DfA1@k$AUL{ z2h3(Z^a(D!IZgN0tChFX4E>&p*i1YqIe{^;kx@csp?^D9<((HN?o3YF{VC~aQL&_H z@?j&%=^Le2J(SO2n6^T2JBNummu3LR(o%n|Y7xDE)u(b6r!jE-bKp9j^L;{fZk19D%T1?+qpV;s2I8h>RR;N<9G z&{*(@;UKr#frokwjEj_5WZE@~4$Vq=!Ism&Iqv{lRsvhqg46{MIH#1eRU|k~U&Q3G zAb5-4`n+C_-zf}@YhBzf9_GuDypkt#CaK`})p>V9Zm)VP_IgU8xVe9`qC~?(@w$V8 zEen(C0{c|91vr~aZhaUKKBHK-Oj^h^P?uj;Prq-;n$O>IrPwq5kFj$^fA<%9XLh9L z%o!i$nzji#CwzOBRWLr}zrr3R$;g$}%v`iU;M5YquwucYe-8O?8HC&tW%L!MSS{dt zcWh7B`L8n`D2PAgd$T6eph@e8zcM>Zo78btp#-*Ds)Fwpu>3IMYI-mu$ARI*nfAWd zlkY9gD#%UOJJ4`C^L52B77>dLKOW?MyXwPQ-gI!`Erx0D_bE2-pKLQxaf-CLq{9z! z5l6vgmrisox7{kl7AqO_baLR)hXJxzlhQ4dQzc4Txj1~bo7rXQR&RLd#$|pieS1^I zX|Z?P|E%_}JIyPx(APsWw@{Dk(sSiZGxH1vo>foyG7c!ed@wnJful;{XUqcD3DcRK za$;r{@Nu1~ux#M2QLtXOLDM2rnR{lR?V=rC4iPLDc|SP|c9%=0IWYVaYGB&e$}&ON zqw&iA4d)u}Dl_g=kImz){(Be$V?vBzt*l91Y!XYu6WLxZ&@;C= zX{XXK`H1VnUhX!NOIK1d*dmw(<~4HOXpj(6Y^idPN;|M%`XM`^hRGsP%3reHR6Srx zd%*71!0yL9>H9VF{ z^(pDbGD*?4od*LiH*OQZs=aohX$P2{vKO5hQ!&GgYiYoXO=pxL@PIj4urPXB_0^__Z!UL#vNl%R-aU zw4(L0V9{=QVeC2zveL?oGjsFwM2aS7SC#lfas-y2P>j} zZ{-awPQSS7P%q5X&T7$^7>b15`wfup#=1tLMO%C_I3tmVUVMt&TViZ`Bz|X`elb3L%fLANb zYaw%hi^c=PPDcTq2bbMe#P}SE@o3;%#izrt?Hz;IF@qbD0X7m-8YL2BC7T+LPGsEL z{_tAtpDBV1w<@0c6tv@5>hGldJ=5A_k0 zM1DDNxiAV8wRMD_dYEJyo8#~|X93HMgw(tR3!2%OSe6IR*nMWsGb@V*wyZ{lJ9DgN zH*(t?FtRG<{P)gAs%rsf!R9%}evEnNCo4Re$>px>-Xiki08>ZeCI4n#xr0~Nd0lDP zGLQG>!Rbvg9IDKB!gd5Mm{InO#o)NhGX|k)2jA`Ze9JFJvZ+xbWnZ&(#532Q;(4}P zWA7XlbQBa|6pTA9lKCiTW@sQ*5o5?q_tB&Ve9q*p|Gc{tH=vL`12c_7GUUB9vaq~LCn7ZTV+wDJ$ zM3@|za}xHdg`8npcShpfrQOdjaGegpa zwQf-kCcCblY3_5|CUsGRX`y4Yk}H?U%7B!EO&okO86^`A9AsqU5Yq7wP;d}tV~}9+ zXi#8a?lrNtO8B6#+>b*{C_vzVhKRDGVwaiLE`xF`A#JhVN%l7K==q*?PjDOW{ybbdhY>jM&CP z;Tt`5M>`fLw3a4cPult-fYDzpCgC6_ThhM?Q-Y;)6mGJLE39EF6ny}muJPx z5#+}?C84o}XG;K+uzf&7BZEg$L8Cx-Na|FhlNOVGHXc;CEZU(o$)YDheVXcVgRG5P z#bgTp9!*Z{xEL|L_wKQb#oH~PUG+8HYVuHIqKZHz)1=wC6V^?Z_z=~^8(`qbDlEHY zf-^UdN$5e=b(W^V2Y3X|MCWm@>AdX2H{&J)d-*@T>t~GqR|HN}={FJb6pS))3Seyc zx4890K~sQ>@~uCQ9T`~|8r_YzOB9{fEq76H`bL=qQWmhf|WlEIF7Y_J{ zux6x`%Ols*RxvRXBqnw&JY3TJWg#<1!XCTE;x?X3Ma;2(0*%l9 zm#})p%nLg9G<(*qi5HyuyBe66ExI+&&C>9~MP~Ml`x80-`R-P4;!y12-`Xg)=J#Qx z(>lRxHlCD`R=SyD&%HJ(d+kv%nL>e$PTL;evRy1GxJ<8hr8Fb&WQoHcJPdDaXqA}b z)W{j3#w;7qm0~ejk6Ym-lS;(dv_H!vmou!e_hnX52xF`N-U{HI&Vk7d1Kb@gjG3OxT zDn|!4n+c8_V#nq&>p1j$m0T=o%{s}%YE9_sS2KM}%mjrnZ4G zATEt*-kF}sZ7GcN9FDaJ-MN&k@j&FXV;jS?le2<7J}hPBNoW;}XmGIzXm{dq3Od~5 z!mIkVKe1~ef33k0*&_jLsa*%VYA+mOpO(pSYu@-u~@=6Na&kNf{H(P=Qk%K_irz9f_WOwGwOV35T4_p{@i&fuf)b4cZTGJJhBs= z-LL34I(bL^NcL6|-`UiY^jP=$q9uuYRLu)MKCLft?71hl<6_pAg-ms`88oIZX;hA2 zWN~M3)R;a+QiU<6X?Ek%v;a-Zn9OCy@t!(K$+W{`>Oa4r-WBKU z)7&^_nKlW;^iEOqGB-}X)+3N`pjqJnLtElEMt)y`hM5r#Osq#6)Le=ktml58XW5j} z$3J8FH#5~qjJzx6XMF1P*AS5Zab$V3xWFY|1xKDFhQ|vhPmPaI^4gzq*FU!~V2P6P zs=)k$Dee!W6t;f|Q}(>{wtiAhE;H{##%X#lIYJE%t6CLy-kG(KE1aRx_u)~=kg{im z6PguLjzK)#k>Dv5CsdSOY#_9QVd12?IjPK<+k}0&>wjF1PdDVc)f2W@N_sB$&LgJg zyZGGPLHcEof2YE%j*0_F|jGS1k+DmxgJm9%+I$Aw8WrA%CFuFd{eQ|PE>7-M&&P1IhG zNyq*%OfmFKE=iV4Tr6e1Y^9_fW0%j9vRhY<<%&+?VOsh?E6nHP%Yw;MrbypV6lu88 ztiW)v!_Ma-J-s{ZHH&L~b+) zE;C>YeE&@-bcc&%o*$c*j|0Pnsq45E0=Z6FIZ3#v?@~Ush(q-5E?1SX=5^nX?_bdz z#;VnEY<=U~M|m5Zm#Bz_GoE~EmUEC*e7ixDs9qyyNP^P#SvQWTF8a&q^Puc)+J-|d zQ;W6+-(pL!|7W_8mt|syWWYh*Mg>-7CPohR8wa^TXOOZjWR>-hRf%?F(J-hmTb35o*~JmaLw;i)QUaAMHgDS-aq8)mN;r8=Gef>;J~Ed-oRpK z`tqLjt_*euhH4%KHy^`>rwR+4R+L+}X}SCt+HJ}#dFnyCPRk{4kr|H49gbZKyZ^s% z4?ohy;pwq#wJ+}*cf3YI21?HiRNS^v@(~+{c>_~`0*BK@jYULDQ487ch0LU@F{{ z)SpqWze)Ljxj1`^a{PAXBz5t71>7?lIA>4b%x&k~*}(mxfcHfM-@OFBdkx%fWznet zjR%g3OxlH7gQoV_uDy;OmtJ)OO?fL-stqHtm8VcU2a?LB?zPKT9djR*u z1p+ZoIj?4LT>VkN?D&u4U1<8^0``(17Q@MWj|{4|2NW!Pz}~FDwKahEq5+4m0efo! zyQl*LbASm~K+R!uw`1iDN7QW-P24VRW)OM67_5*L+Q6FJP}_K5Lbm}&odQR2K=e*i zHu(Ss(F=^d4qS@`I%IE7^e(qAU|`Q&z^1a(!&;!!=t6dngt~4d?|FvKNi&rHPUpKR zg&@4tk2tAcn&0|rI~#uMV12@&D_l5MZH6o)0Sr5Ugp9-q*2 zfMdb~jtLB$oeZ3f3KQ+Gai$nB`hVis%D~h9VWKwEq|1}oRWduWHZX6EoUOfqSyGkx zqkCzHy0PxdjtL#5vPYu~-Eu5HDZRMBdv5}_+cM6X3|xB~x^6$m1AB2_uR%Q za@rHt1(8eRJTU@RBl+PZ+(D7o*&(!6aGc;6`SEoxw|Z3w$IC4}ulhRiR` zzZ$kD(rYHHWUy0YkS>^cF`TVYfa~7@22MBgl0yufvotxp8fxb{u&RCGIBmeyUck}u zVX?wavC0OHf(vX#2D7`iuxV7z*1DO!ms6IbQNSpe_p5kk;_;~CW(_t;b6q;QXKpB) zxqy4c4ZbTg=UlznwX>Dgf_2^o7bhP@ORt5JCsLif9|p55U|y2Id-g*??}y5FQ`64} za2KCs%u|YcnxaBZLPkEhURsh;K>qm@hVcGVY7?Al?Fa^WW5 z+Y9_}7R*`6A%1H@*VUQa+3l>J%OvNEq@FNUn{=3=?ywij1j!9Y7+3@t^geJrJg~fO zR}U+@Y>NPA{53|887*IChSok{@2l#~i{w~%fg|b>WAp{~TUTS33aow-%5~|$N{bz# zi&llm-0GjKCdA>8`S(V@ld_xPMn;2n)`$z7R~)z&X!7(u;OKs^x&6WB&I25+3bory zSwtMTwtnE~IKbZFu;f_Qnxcd?r*^GT-I=ZabB*=~ChY~2LXLN69Z?$9XPuawDK+<5)VdWCWsf%SUYWUU`UIBJe^!oeMrwD;*6%pPz{?Q( z?xmFE0%?;E9M=?9)Mxc=y}<7MfbokPv(*IVo(U_yN=~m2*jRU9LgR*BzXE2<3C!Lf zSnC~DZe!RqZ$isijTz5gRK5u1h;v}*j?_9Ho}dzLqTnuE?#jHib;5!N{AVA`JTig9 z{`clCg)OJKIZ`&X%}rnt7T`MgYfE7Dnjr73Il c6zu3%sxMB_Rf_-15 z@Vq&lX0({c@dxjO9XWb#dQn}Sor!Z#&z#G>g8S(${xlbZRp(|;2X`r79GI;FP?!ddTL8v zCO6;po^1-6^Bi_9VPG}ez_RK$qm=*~-}7l3s<*v7z&Fo4Y~E-YvU| z;XB8kw=6Qj1)S#He18pi4>YVcy0xb+qpn?mW1hpz83t_EPV*dDz#g!BZ}4pP9AWm> zAICBU_NjNy-g$Ft%!)vV zXx}(zjYP_&$aQZIaPJA2%fci6Vgc`+16^AtM3<#AswPN2OJ0BCXzeA{*a;IYH<&Q6 z6fkxt@LrqH!~Q((_Kx1yD|+l>=BK}IzS_WdB_N$CoMVGDM{B^A&CDE(&v(vC;JU8x z@5qh>j+F4YiU*8ds0rxBe&X5I+(+W?qeM-5T zwmybQa`nSg|93ICI-I_g(0A$w=P8XVdwt?=WN_*G@XFg9UhU&I_iFVbj>8i;stdSxG_dZ; zViQi_Sn9y_>))C4nHSFg`?J@1L$*fbaqA5oS_wD0X6)Xz@is5=w0rv|BZjPIr z$r~7J4Y>L@oG+NLMtSF&Ru2{i*?k&6nV#L=%DVT4mI9N{iyQsCC*E-H-~V&c*X`SG zB=Fu^!26B+Rvz=ZIIUAN9UlE#+rYVY0r&0$-1{2%bmT6n-Q~_UXY)*x65QHSgwrJAr$9!IWhJ96P#JHwkbQ zuDcYrfOD<^tF%DqzrE)(6P767Jbyp>0lVD=?te_m3z$?4n0V#vN*dT}6lD7U@^kT? zsAyXo8dYzQ$t!j5;rzF2i>$YApAenh&N)Ex$?iwtD}PfomubJXRU|KZ>AqiatuD(LNP*!botXVS?#eOVg?ZXdex zl~E99n4NYVK&?B4o#Tdnloa~fQW z1qB1x&oAJ8c7x+g-G+GrSC0DLjW)g7(7=9jPMoL4uAApVc`|oju01nFi|@^Wq~?db zF<0633f^VgEi&}Jr?_hq=LAN11Gd~BWy>$T(mKbj)pzpgzgOI)=MOACmR!wXRB(X( zx6JH~mJe9$Zaj)U-v93f&%Wa^0_-&bAE(PbtUGfdXWHGMR~-`jw$0V;?3p8xxZUAJ z18;R2M|K70f*Y?i-*Io`yQHv|`_i=o?gw65oG;onQF`q}=`AAC8v_{}-(58^WIwxM z`QANW=09KlMRVGf*ewkU&dfQz^K0jd3vYKaRGo?3+Iw+9>5T=vZ#Lv87<0%k@Ya*Q zCvSD{72iGfhJXAHj3E_ei1k(%uQ29G$Eo6SDfYbJQ?AIL0MV_~3wN z@P-9aN*wKP0yZCh*e5Cwx+h?vQ%jeil1s`JkK^6qcJe=adP0nXl->VN`ulHnsrSvb zDQ8p8tqD6D<(qS3OX=g2lcUY^99!8~Uu{nJ&%eqOxv}Wv%cVXu6JFlhx;k7xcCB9K zu8^lESeSSuR1!8Q=rnHTIn-sFZT6;tk^9N9*aJ^znnrJTsTES~nh@_)&nj%>(jmCO zzDh_p>B@{iMYl$-N-mKOqlZpyT!LIKCxjpFKf7h)=4HCE#?kfc7bOG|8XK86@akAJ zD3s;i-j=9tQShL#fj{I}&&ndsDSEN%*Zg|*v|GH`Cv?LM#m}J)bKQGi95^Ywyn3EV zE{BxQCC87tl9(9}7@c0r^i1ot&aNku z0@rOy>S5qWJCNu$#m&X=$#Ekk!z0Jfb1AbJ*@IaHFLie%ge>5oLn{@l`7k^kySieYu(a`G8_yJOboY5iv3th zKHqpefnT7ZfoH;uLp(2e=C-WnH?5rMZ+t9*Nrz2fL9>Y4ryqU`%uZ-?1b>#gRl+Vh z!R?mPvZuj{Ex+CfbE^Ca`>H=_N=)d>X-w=s8^U6wudJLsx8qmmig?vGD`!QO++bi) zF2uZ1)H<;3(ig|AD_Gs0GNcNee+$_Q4$fjGt zX0my=v$V@}cFP@({ACxI6;`oy$S+)H(3)|CeOdwwe?Vg*-(feaBZ9oy9nR91Rpui0m;`e;+%RCxv}jS5VN})rxiDeo$sTns z!7kMsi~`;#Vx~$JcB^$Q7F(QgMB!gUyG%&~Lj=n_rW2_R3?2-OTo;l)yjs#=mT6e2 zd5J-E&IDGM-bS881!hsZ4UL()3`~3-{~DNH7<9RRSlIW#@WfK1rxQM|O$>hPdB564 zpwX#|fg^A-%O9WA%YWXe3x!3ysEB;*chAw4WL?C0qbStDw`;LFlT3oS9p|Zxj+K%- zW@H`xRK&eH$~0{84E33A2N$~uESmRK=ZM0a%xm^<6xVV%N66lJ)#|9@EPZiDtNN=h zc4w1A0?`W02@+-U?EanD`wo zF!EXjG;&4o7jio^h&6^^(7(3QJb@+IiJgIgfhU1U^1=ajJA-Ea1O;Zv1q}>x9~gMn zg$bQ&IHJhc*fCK`MR7)!hx0LZL3^j^N_wJK|G738T>f?8VTkG4m$jdQO~Y=!7u=|I zVU1Sg>e(H>>U>6C!5bfj{BjSO z#CLsQmiT>u-T4C}f5Cxfi8~f-vx-h|$1w78XS3jK0Hy6 zPhkyibJmqv*zx!oqabsmvkaFJ%j8cIN^fW8c_&s0&R{yoZel7r`~LU46F-DxJNI7o z$tz7~3uTtM#?Y=d+sYyTht>u0Pg83@R>w0hfjY zanhg1!5nvR0rSnX?>07ixX8I|ZxdYdaDz-(Y`RMDjHb>-e2G5}$oG9Ym$jr>_Q8sy zGMkN$yPXjf>Tht_*!z~l!y-xW-TWD#zfw_=To+I=G(JsiIRdH7A-R01aecg znNHU@Lc-b}}*m%4sD*k!VGdu`UQyCIRf*}Vatm6@U1UyL-=|GmCC zfA>S)5{a0H9cdQ+CY~=nvkzao8?Y)!bD#F6*jrMw-#N>lXka&*aEagjl}{Q|_p9LL z3wWy%&L~959&ylFBj~rFRDN4ZcUA|p+@lJ&^^RU$mQ7c#JAW@VlKaQlvzXcWrMJJ^ z<%x|v=^BrvgUXMY?+YwVPqBS{q@l`Sk|57g@!dBc*}YTTrM~i@(i59mQRk;z7H{g- zSTNyJ+-HFe(fL32>ht|n^z^KI)nWN-g=c)#=f;PnDH_VZPA#arvfr_1b@pA!gE#67 z_N@OB&+u40`TDnQwQ4Ir#dOtNT-veQ`9ogs3I_J9eeX)!1KxkD+`cU2^=|+FjQ=VQ zY71msn7`qnaSj8!Yxa8k3J?Se%ol=(NDeQ;~l(uKyve;usp2v=3%||?Qv=#}p zFPO1~JLLpNjndv7tr;)cax3)jeB=@4;W;;B(%ByqIWPBIb(GD%#j`(PO~8xp00oxL zBRUlq*d8kH3*=y}n&r9ZvuT3Qz62gOwGgWfj}N_7W?Q+~?#=E)D<0crlsF}Ncp3DV zY01IYQq_0n z$GmK_^WMW%vwoSD$GjVDt;QCbOw&4d@9~0-VF9Kh4xf~ebZ&m#>ud%sBBZw**2TUEvdyV zdC5M_m}3bioOGWYQVa1)+;QyFwJ!vE-`nl z@_V19^6O{+0qIq#3seqoUgG4l$9;`Myx@t&8@zX$%-*tc%c;{AowH56UbpdV+3}^v zAx%m6-tN3vmUUO6||RBuua${ z5ya8zF2MSDbpW^Qu0-Zzb6Qvu6V5JecGEREmc+xB!f-KxV}JYvH}@6Cw3T<~YMU1` z^Lue_))YSXrLa+=fN|{^ueDPQEpyIUY&hJ{&^dRJ{F;MO@!sBt4j*T|X`K5(-qV_k z%Vfu9MlLQ!u8%QCnr#d^SKGKnt zA*D%s`QLcB8MH86WII2Ho!Mxg(wYly0{f;P3Am!O&7HyKz-lx1ABUEh1PYiQOaHS^ z?}%@rL|azLsZZ4xC6-R{a_sOfv{o$Um-%<};^QZQYxk@(t#m0xer<*BFm)D$)xBJUAAbXuj%! zPTs8{9&?Yy&p5VZ&9TIQmWHb#pR2vzOkkFYVe+f|=-bCGV@a*1J)$&953bocIPX^GT->QKlr zIb%iJzg2A+2U@cZv_=a=Eu5?6D|{<+?X@gXR^Q~Pf6K0N|NPQjaF?f3h5ddqdu2x3 zgUhZLpB!}(HA|2-+qg75MLU{rYsj51`{Em}Y~32-6MHec=3?$-=3JBQjX%!j*{swo z=IHE=2wob|?RvH6YDCrSSr!3P{~4dQ+;lanqO*FB@$3kJTnC5v+6nfD+42I8bVQ%n z9C&`y%{vPMPPEQ(>3Mu^)`!z29_<%G?=DD)opzUP_ZPOCp%bz^y5`=!x!~)~j2p45 z7D>b;uxU(ctNzm_Vr;kgvnBW2t#Kw-lAJlylCNYJx761Y%UMt z{O)ejJ*@a~!;~$>dW#x6g8oK#9&ME2y<OHl%j4J1?RS^?F`N3X zgu)vSuH1c)_v3-kw+F)O1o8?5?nSn^SG2j^h?-v=urBmoMQ+QpyYU_wQ(t^lOY~@w zKXxTUn(fuvxTPvdKD-f%YB%0r-DItF*#E2BmjhE87#I|PvTztNFfiyaFfcGkForO; z^Kx?w^YTdYa)}7>NelBziF5Gr3G(wx3h@a`@(c3|iiik{NQ;WfNQ#PyNJ@)J$x2C! zNs93c@u*2~Dv63JONlCrNh?cBYstuHNs5_BNgGITsVK^7E6ZxBh?}ZQ8LM;4$|%W7 zt4T{)DN8G9ODk*3YO6}Bn#rhY%Nxll>d32^s3uOu+YMX1Q8yo6c8|zw`sB3DN80#2Y=$M!q8(Etg+gO>}m{{9cTbi0!=^EO| zDT;fVN%|Oyy2+}B$*LB}s<~QdJDD20*qVCTn!4Cnd)ZpMm{|o|+lE`2`{`@u>FGsj zN{6}$7Wha+xM{jMn0h&xxjNeTI9vO9S_iq>26@?qyPF4j8D)5Aq#H2)~ zlxL@=mS?3^7iVN-l~it0xt$o)&v#miMWZ0cY12 zTw74^_F-gJNoQF_M_KLUjDq%ke%@dU@};N7t_1I(h2cq1s{|#lIUXAyiR&lmRbAY1vD>{r z)~=4{=F%l6C&YOz`?6`N_i0`I(FnoRvpC&(F7O&XTkW(Fj~**Pr*$ z=!>Vndcb14R;$og7w7rUaX7i_Yg*Wfn1ve}ou+AB()F90bnn{&a>`SdjTxx+Uf1rfnyuZHq6P_KP%>k2sl-J0gpPw3 z-Pd$7Udq;(c;uF+>dbAd-a2b5qzhC(Zi~)+>KQJbZ+=w$PL$ul5E(x^$L}&Fz82+I zqg>{;rxXP&6MgFc#&qJK)7cn8K+h?&75nP2KDK&rD{7U;zJXMmd!B*IRUd93ls!{GBhM+*e-e4CR1lIOJZ^27gsl>l`jOdWzUvQ z%-T5hi&CD>q3>bdi+8cG`o!N7xu55Kcgh{%f5#8DgapM~rlw51WovnNU5v+~RuxCd zl`_#=yslpfd{F8>RoE@ha>{z=i&I_+#|C(-#N4>m{wy}`LgDqe;w>jx#hq9_Q&X!Mn^)6;w16Jl0bt_&5hrbJoS!N%f=QdDgZY_tLM+{JwZ`nLsgz;BnzeUuyDpS?aaHxgp=55m! z5ibe0y)zwC$`*^%vb1H$EX{bdTswEmQSsvaI~Ix0J?3uyuY2d+j@#aQc-Jb|I9Dy5 zyvED*=-hy_K{r*Vy5=#;Zcz6Bw|cW5<44t#J7(QciMqhFu=T~Qg{ooU%Y(e%MyDjc z7MrQIa;AL4@4cMrR~iL7n)xgBmN_LW8g#t>@YC%@f<(sRwPo!Ss#G={)Sg=Mp-X=s z+a#}o-6k8CtF2PG-cw{*C{u9o;+5ow8IF=`SNH4RtUSHbY3jaUPt$uX>lWSj-!$?2 z=C5}{Z#$lzdPm*l(-#)qsGD518$|d0Uh-0}{vzD0@nfh{MN=!&xHEo7n>RHCm zlL9`zNVVxS(WsOUlFrm9T{LmVG3A2Wx3khdN%@>@aoh58cCUt*-IfifbpKhuaw_(F zy8r7*pQ7g%`_9QdpRA;1=xu9xZnDQVd##W8^}n+3{6BEmc*6^x)R>E05dmsT?p`^b z?{Gz|E^T^S{z0u~CMALQLZ6xVDlRbAb3No}Pj?dWV+rbun%6Ps`J|@WtVG`5A)YBq zPWbQ4;Lbn&@P5{k#gBN7yFWe170{xxKf%zu#CxIA^c9=u#_Armv8tT(Zwte`#cMWs zeD78*iMr{vjpL;L*KDzk^ER>gd$s$dbawl7wJpo7?Y7U`bvOQ>QH#}0PyceBd(9uV z9v5~x(VzJ_;Iq=#R`F#TJY7@HW;=)qc}#jT!DL~?-5{n_Wv^be?XkS`a8j_&ECvD3 zGyjUN34Jl2JFD^Hd`~Iaz^ap8VR3>sYh_;ipVV?-jmQ@phYF^bF_OmWIV^eyRTlZ} zOIfz~>2sf48S$Mv7~BtvoL~>VEWGE#4E-mrPi7srUii_^!-+33MM=RZ(48gB_QsQA ziQ5$Ay%l~66@KD1H+yx_yRGcx(+#Jz6<>P_Hyk+sozrC6y^k^hJ2uR&IrK^6#zD!= zvr<)LHKu43oSNj^YoZsC>nP+qfkhxn@}f#(qCnjXPw^!M%1;G8`&8WIub(EiVE+Zp z*bc`DO&>T8W%?OA8O-JIm1gzeugpl8t+XWPk@7juLv3>m6z2 z2KI`kb-ZV?IHK<;@$A?1&S+oeX7X`4Z+6vWffKIF8wAMvu26N3Wr|nT`SWB zJ%#T&$-X=A(E0|WU0eZEVW&2WE=S`IdkMqB;t9+$e-zlQIyM%66k!&>@ZgyF0>_@x z1Qy+tL{0~@CO$jR$W{ZhtHz;Zy$g*_A&aFF1(-wbZ4xsoyz2Ad7XPIOM^YItu5_^Q zc$p-{%s6e|l(_GQnEy#WyX`)^YKf!yQT=;tZ09-}gAeSQc(y)2VzSNWv-ux43i|jx zU6p9~JZU#g2)xpF`WU+*+@~;c6uNf8|wQJDh%XrWvwq*hH zQnRlU`*oU?wnVXd<~gvY*fw(JFt8{XIEfvbz-rvX$Q^m2NzlQdk;&zOp+JHIE0+OB zOu=FPy*UfQeM;x+JyCDUUBG;K#L^GHpb}~|q%JT_;b(ZNk#2b=ov)Z(r-6}0 z;{c1zhsK2Y4|!%}9954J{Ng?5AkX1%4eVM8jNCf6`Okf8mYAj>t5NXAZ6ouWSratl z^y_c?m89^nG_gBX&x~ymYJ9)z=3RZ>N=E*vwZDDe{}%Kp-RwG_RsH8Lo(~sJu@wHB zJat0TWtm^9QPV=UZfF%gUmy`2!*1@j^_Xn{%PDoslY$Zr&tm-;rn@iTFxlX&dq}V^ zXNsLnZ-Eng#JvVqmjhq9J8D@Z3f`!!pYv3p;U`nhxdt{3X%<`i!|Dc&oezcWUu^NP zQ#)qxhRfgq6YmU%&AT1jbndjQ576kUee;ib-Mi*;{Ye@+b2&<8$ew6meRM5Kjy3f1 zG`neUdk#t8kydH@<;-=jSWfH6oCW36Tvm5fe5<$FsIt~1toylm$kHUuWl0yFGU**) zat~wCz0NTAd(kX)MrH*@t_78nKZ=9|7$rIyWilFNW>hNNV34`MvN(ZvX*%!C4^1~W z@c1<_iWL-dJ20%9Ub*pm(fdP8+TkocQ!B+h7)2H^dIzu_VJ~rTubI}s>2RHE%LMKP z4XoY{IpPJ3q6KW*FYu;6(7t3Swvbi4K~?Yg743o`z2Zyanr1Ba3}u{5HWf?UO}4tf z6ElR)K>fu>LY5Q#T^MyUrZ)Who*bk7joDry+0ZSE?*`NAvXl$Q z8JIUPD9`AesnIfX2ZPWC2I&80oJI55f=w6ryFPHSMxXy9~P&N-`qGoyg> zFnjBk51f%37)2WxT^rbUG;r4z)FwR?s!GjGdlV}yqW5@_c(D+x(+6h910AmqF&jLv zejJweCS1R|L_;XCU36i4DSP?X>1Bb-(sLX`tr`N(i)NWWYtXyEXsA#rwY*S9q1e)$ zVO9jgEO7?&j?P)_ohp*m)*6f&os*VzG)i+c&0E2+@%z6{m7R^u43p>2Xq>?@*`TwN zO`w|b0)w~#`X@IBnXQ&zw{<3!P>=T&?KIo@Gu!IF&Oo(#eh7`-2` z&o5we3$O8uVB6QmnpVJhe0t562kZ(D7$prDlO}LqP2kA>*qT$|{IyW6YGLZ63nqD+ zSTiKtb9d+&PUzQhU`ja_)E8>=HONW*iDa5`d4W1-lEchz>TD?jYzL2Jdl#@yVDI!= zULrFgM=&90W_k{DK%-hkv&Qj>`r(Z%4b9947^dy$6pLU`@|>i>UaeBuIsL?>={qJb z&*(CfkN$%*c5Vaz)5m$<$Hl zsL3RUxY|a&wm@d~x)joY%#DSU} zVT@D48CVqtQvvQwgwdjl4YA+_v(U`RUdzEq4B)eCO7jkqR>6~oD*){z| zm$bxWCCf>NUUsn@V37F0ex_l{&4ekp4S2U%@GfQNJM)2E!eQRylM(y|%XmL9m_BDX z>BcBMfwgVHyw{c7cOLNWaNw93S)XVyfBFJW-x=In405Cu*tS35{eQvp-{*@iu?N^P zjN~&GrQY5qE|C(^FfqyQ0&~mEetid~@AR! z*1(p2foq)?_d2f<4|XQjf=Wa8S=PUbEIq34oSdz8qRY^{c*F7T4dtEoRcqFN@3gX9 z;%U`o8aY|pap?%m?mOa+WS4qO*!F1=mAdqs1uz6I|Y19rV#bNDwf=tZURI4pZ8 zX~RE(L3jg;mqhRU2Hv*^__keOH{ZgN=)mTY(K}-T=gtB)c?PD)2JU+bTzgZrQWmo2 z7p6wE`P6E%1Qxe7CdM^)O7P8^@%s7|Uck1_fqVCcf81L> za2a@W@0w7PK7n;dTP5p(iMrQUTT4}&ecv41>Arhq^KQxNwZ|Lnveq1bRW*M_@?z>%q+lg>{s`dUkr=}%nc5_Hw$<#JIuY8)qQ;duX|=y)GZE!4Ghn# z)+KUnS!d3W{GGwog3YgjE#d>`j)L{A4_J$@X8L?!b(qF_yq$H%gqp+CSR1RjL&}BM ze73EW)MMdNuk+j($Iilfh&eQQ#{T4~^+rxVm4b7Q>;HYkD)6x^se#>AjV-N#bKL^& z-3*%zUUO~x&86$jomRl=70x2)z_50D_2H;J5t{RYJG%^SZnmAhC2;4Yt67sz-R}0- z)wTHF3kFxI$p=?1v99RaXEE<&8S~Z;P1g&S-u>02_oB(Ya_Qv{Ja!JV3x01_i%iq| zRi*f0od^R{T>)EF72D2&^}P-vb%DCymDzTE0vHz0KR8Vg!Ogg(nk6nma#~`iuBLg3UV7zjCu|#mL`l?mkg5R!ZubaTV zwR`vW-&{Macd!4jiJ80R*fYl2;afD9&z7&)TO`Bof7!dYa&xbJz_sfR z*Diz2+kT&LbEsra$O*PtY_n=h1m~8Bnd^+MGn{y{*!T6(8{Cu4(kI=ReRP>bb@Cnt zTh6`KCnrT%?q$1BB>Ft_% zogr!i$JP&pmoj-TT{xC=o5MbUQCxu8zk&JGwQBJXdW)o)^aB`KHZX2neW3d_@6LoU zF4Mq<+IcS~a9*t7b}wL^oWmXT%(a}8Rq$-zi7R1ZPf8cn#L1myicsFAI3=y;WZrL~ zTXPF9B?PcVe%Q2iHMhM^$?Y4CBo%huR8k3_}FS-6NXVUx|XHUI4t6^DuE{4J4I)g6*NB;#L zh407m&TPA+z-4xULE-^(^8t>Y1kjaA!VEL8;7Wak?Q~yZEG2RUSE+(v^7bC`gdzy8N;f(f){VETd-crq|HFa~a5FT21bae$@g0MFtH% zIs2lA$(uVo-~r2w00#v&_l5@-cRb+j_W@l^%3TqC)grWf`77o7a(ulFcD z!cig9euJKA!K+z+9n+4@SfFLLaMq(GQvV9C?F!7;z4HL~+X?^f6|k{){+=GmX7Ogz z+5*;}nOlzKvgbTtE_Yy;_`q@@lb0b-o0*YI;*H5{w^z<=j%Q|E-FaaFV>`Eu#g>MG zhfduRrWOVg43ij{`7CDH{A5&dXJ-;ExX{U%cz{VhrbVYAQ-EhR>b{S1)yiYx~SFlC@M?C8ehyJ8whdBG#_S>e2HYWZmD>i<@%bZgNap2r=tF`pr-Xw zxw}7tv|=pMWUs}oU|twy^DZCJ?X z;FhQ|!HwbMf<`^Lm9FfJ87q>fq&}OGxtxP1z@c%DK~B?@oJ}*+j_0hIm?q+|Ys11; z9xV$aT@}k0O(N^7zGNMj%N0}+$n;p)z`!I{q0q>AWCx3|ug=Y<(+t<0;NGxXV|Grb z&bkM!;^8Jg|E1UpvBL`E5d|bC8 z+sz1phi3C-Ui6y#+rB-!<4K3|)t&kWEJS(q4+uP-7Q1S*o^||%pXxUH+cMVd{&>XV z@t)WW$HN@_QXLG=OkNC2W~O;kc+ES@$TVc1lakJT}WwOX2_nr%H!HlT^M6 zt$5`gZUeL(t;dKyTot|~*$7fFUUm_x1E4bgjF;+Bt-DB)N zUxJB4vF!t6`}LHHL^HvH+acxw_O5y6+xoRbEdwKM4~xEvD|FppvDj0BZgllFIgIw|LwxJ*Q7Tg?GRHX#PZZsQFd+0$e4j#YC0 zogL9sKj-)fS8JojHt#u#+|oB1lKx42-m*M4&8X=WlQd7O#>JLnCT4|G1Qb*?ufJIk zv^lVm%jD@~zprXMd9Oom`*~>XtZi?-Yk7Ee+{Lc2Cx`NOui##{nF|$L zL>ZYxRx?Q?I4Lbr6i{+h0!@xoh!P zbsS(2yWYTUkSMrr^Fdy>gFM+e%>7Fgj%s8n9@8&Lh(ENMk@wlvsi_7|?un6YdxDO4 zaQ|o!ao{@2o3mhk&V)rwOad7Zdz5+|Qxq7i9;)~_Uux7_vAU{ri<@kD$ni;kSM;1N zND8k|vDKK#BVeC!fN9dMl^h}pb?FYxYB4Lje@|pg51!E^xT&Be=uwmSj01;MPZb>C z^l1>-zT()tA78diX5gAJLvdM#g>t9cT(@RvmO}ys8~!!Ro?u|p32@AH+hHamx18_T z#>01dHs8pv``lV#!CZP_vRwF;vphR~ofBWSp;f1ZooD_9R{gkza)u9z8IC3~a4&M` zaTs$vNcUo_9<=Y|>JoikBw@JhiMdoQI9>`vD4?4y6`dhI6*GW2FT{hjU0u!#V zYMfF3Ij2E`dj+S@j7ef`2F#Mj653{5)e)bmkSckMp=r}BQU64Ne&JhHl8%|j43CuMm`UZ`)C)5ok-~L|951i=v3Jx zyQ#Yv`eG6oFJ(1wb6m7?c=3?Gf#ZuQ%S1-CC+Ecc4lpRZa_!=LF;nVcMjM0H0Tyi| zCz~1M1)S%mwXsDeIbB_StVQ)gL#y!&#dK)_2G%Wn9Wk$s z4*3`Ji0JKUH4ET1IjF`f;5YfrI@92sv-5mA3zMZq#joY+g@yKnKV_ACEz488LUK*Q zMXp(MFXg{~qqE-bVpr^srnU$MN5QTQ$6GX~?o-ZeU}qET%{FmbDZ0Q>vFb<%o5jCG z$@#@jY`+BDS_2+%Uz^m%ruBd&EyG#LXF)^Xl8fy;9qUR>7BuiO1n!aZWEK$N;Pl~T zxIB@eN#wjm*VkjwdpjHrlP(*)SaMN2M6NT5E%2rgOT-5jYXws#fd`%}Y9AVPW-R1M z>sY0;(yre7O#gvIgQjcQe;K%Vl&;6^=(Z13)f2EzP*OXWAToQyYQEo%R~cq|O=R40 zc*^aDZCTM9{WbfJYVqy+&G&f0#+=6oa+R44Iuu_^*vOt{jj?Dp>^zhJI9IBDVwh6xkd0$(&pZ(uDvq06&^Sud-Ne?`M}=1JTN z3_8yLY#CQDyJ)ehFJN-C=*rm8$X&rS`_x>+Y9_V{%{dGzOPLsEGBmjch=1;q$egh= z=Yr9fM}i9<*;%WwN}mv#7|>|xARqLi>3ahMM*;(rMnjPFhE6rQbbW^KB z2MY&d^k@4Rj%L+_1-cVv#S2Yg_L*`{!7QnoX?>s?@2|$Ru1490#>`c`w-5C0Ql6Tb zyfFLmZvGgZf?o&ZnVF3`)Mre8tMa}^e` zK47pZSQPbPC)W!G?Tu|(yBkwEx-Abhs82AKp0PJAK~_AWLHa>g=8R^R6AhOqx4A{I z1b*ns_|Yiz?+0`Flr{%})(4d>QZv?@x@=+H(ZI;Tz_ftTT7WgJqpkg*%*@q?WjCr! zoH#Ehf=y#V!({Jnrb-qstyW8qRsO%0Fg2{zmS}b>*}rPF)oSCWNbQDc2@O758thM) z+6XX*R!Ie@9MEfE(ktk>D8YLv;6RGE>o#U4!Osf&lN!YZbnY}XY(41pC}Pdi2M+5} z`tp7^1;6Ywc)=Ii%$j847Mroh!gA>h%VvL$JqL?fPke4*x}lu&rSn%(Ly{usLiR_A zZCo?9TP>uG3UXW;yi!BCsUsrG}}ZA0se<}C#$_DvL=`LWU@Z38pYiH6CC z1wNdV^{VRNmf+-2nDOlcgY1Qw->Oee7irF%!?5DON%jNhxDGT&UD&$3k;y+t;k1Ux zSqF#nJETIM7`VTj6_&##*P*sq(~zB`k$uNDzeedN7fv1i)ns|&y#9+*lAq7(-(Wg8 zalVkrvG4}gtzTL{Z&-MEVc+KkibsAj@UK{Zd6Qe3>NqJjTMgLDLw`Co>chxa_mX}DU`!1bazrr98V@`iaUc{WNPJ!`>x!Gd@8 z3xlg2Q?50fN}J3mrqCGaVra9$|F??h;~$344ut%B;;-;}TPYL2;e|$Ri{|hZd=V!$ zhDQecuAKU6!NRZ;OacIx5-_#zkR@!xIQQ@6LVrIF9**X0)B^YK#%(2x{I`z0gJ)!0A z4St1{t*$GY<$qi|@pRI)BNG;Gm^0y{pvI;)w**c$2Zrmdjao^v@-K}v4o=chWcZmQ zf69AC`N6vr1GG9BwsOy4Fj&DFqrm*WW>)Mbo&cLu7jHC~IQ;XP7JFdbN5<{dM`!Y` zS+yt9KXFaAwa#Cgsq)+xHstUaJZs8-#H7!_BvZh+crR19M0~hGd_=?JFSg7dzB2GH zXvlK$&5_w96La`o(*xba2H%wdBFrhA3v3*+X0rT9_PZ4CH_6<+Z=%G>4UY3!$wZE-pC?Ob#1!_q3*EuqgjvQvYyd&jX&1H<+9eXN}*Zh00Cfm7;zq(e^ zj1AUr84NZkH-(W~l=lBGpFEg0&ZMw&Z;6OW3{SR_mK}=dZo^ z#~jvhIjCWtL<5tiNA8Q2lc&x5TE$jAms#dVhjUNF4})m7gr=lbjHiQFl^l^Rd@*s| zfjR1q?9nTv<3F@UCY*HkY0!(D$5g=j z^$%G~wxngtnYq0*NO|3Ot%ETm>E6u+M{jF9FJ6{i5-0k<3MlmrboLsmvY%pW{mBRGs~-^-AQN*>`Vv`LYJAf0M89*dg>~lR-y%zV~A(k4t?Wjbbkv ztn>I!Hpd?Go55i*L*`7xb5E`2t?|#g1&`lb+&puS%ocBky7^u6%sr+xT$;ABMJ|`^ z+6#er23|oXH|>YH1{$3+*}Nawee7xrN@&sV2xoKfyC?TRPnSblf<-TA<-$pFk6$-1 z-6+zZ->Tj5iFreVOHRM$bXK1Xem;)|p%dwEzkPb^miO-7yGV{{lM|TD4x~9<-H=p0 zWqR$Y8LbX0YZwLp2`Cn^@mZ&@dG+)Z-;?(YyvxpTEenYh)m`!2>tmyA2BX*m2GI#N z8tp}ks=s*DXF5;IdG7e)+uP1X%_S`xz28SQ^j)7caYe>=*C#&hEA5${Tzs~!CGE+y z%?XptUe?`xGV!ON$L*9f0ak~GCb0?OQ)?KSTN_NyGG5xNJJ&gOy~oq6??w9Mt(+4Y zy4tvwSFy%VXw{nEFl}a5d_>iIH|FpHR__hpRO-KJXEbTdXf~6`kN3K{L}#0Pg@}2( znnk(UEe%64CFZ$vhFb9#cYEC= z#c3z?#O(8Yv)?Q}cgZD5u!VC0E#mKV4CSgHSc<@A~>jSLPSG~<7*VsG5);4*E?H`V=~ zTr(K5!#{0$w91O@ep+~u$;PxCJFiO?AzO>5T+MJ$$jHCGVoC&0x}b)kkVd-1=GXb3 z&hNQwmizhl{>&b$gy#1g{JIPs{8kwi1&xPU+4$ zqfof$2nVwp-v^J6j;uUV4jDHdEM#nKXH~OWkr|wFoRLdO>xIR}6wmoi%px0&y`r`) z@m!v}ds^--)8&E7{kEFr&N9usrDkI+5XOE-Ne-GQ>t^$o+E-4xhG^Q`kf&^UM35KJDW0Na~!LHKBoXpGkX6TphsreX`-Y2eK+@=2h z(bDeX7w5K@tURo0$tz`%bfMtkA?JzjUAZE*oOr;<%+9alkTHMbUUxP@!*k-9^-r4E z#EohUoQ|FnO_un`$e|$O;J|Ftx}brX+u*sw4@m&Cr?j_MmaDZ6`qlMQBpkD zLx?rgiR0s8feNOJ&HOfd*koN#dBP+7s?$WhU7@p#@YMs*p9(~XWYMN_QW=M=}p z{7Ne-z4DNm&+16Qh4TBQ2M)4|h8$pE;lB}*!SObe#gUoUAVJWBFUsUntw;p}lc-xy zA)DHQgioAm={uX_>ed+j z>r(-vc!WXY^#6YrH;Y^NOuMcVp2El~9cyv%h**@3qNj9b&o>u=6}FAf`PUqMkiI}? z&jBWGnH$q?=XEi$-&wMYb9qeRX+d?_lB1;?S;YgYz1RiJjs!55-xEE;cwKzUuLZ05 zx0J7_mTGmGUn7*ra*;vQC*YuDdI)d^IIzb)knS~!cw6CX8t7^qYWmKPHPq}@%H&EjD=Svf;BZ#W4pz>H zgvN=i&rKtyZkDh(P!YQ85hJfcOyEuDl}SDx%FP~1oZ>M&^8dQ^!zB*33KyF!WS!%? zMNvwil&Mf^-s+#7%{+!@wr%EUb2`wpsQ2G8>B7UVzg!Eyx^FkR9PRUpRlK<5BQt+i zK=T`hd_~)|%k5K#@0-f2~r%>X^>cN>3&#u&Ncb=p-y^UVFghuEE4kpN@c* z=_d}cJvzjorozNk@6p2FpfqLP4;JU^yTrC@l*sqQ}(TX7a#eUvDEFfP6-EQm4+RB zt{rVAD?6AMEIm6(Ac4_dhJjh@0*k{HrsAs_vSEH9y`l?K-MwB;>|ZCum&u@1wyD~I zNzP&EZP5iBZ9C^Sa!pV3qo~o{#Z*pkkJ2xkv1qXT3cN|gm@^Feu zNaE>=WHM%GTwJH8z+iuJYJgJEByQ6aQI?vWHB}tKTo)GbEzD@Mwl%jZzxr9zJ?5^- zxlR5KEIQ2pw!vWK0lNrzFP2MNa(vWe;a+cpz>vpS#Np)Yk^)!|y zhb*2Z#M6AhIcVo;#)!CvVB*Jm>f;zubLx9gDdH$2_jLhE8*$+5y zL@#J9)%v%|tKUIpYIR(zpz(qOtSrwCx4R_hvB>uK3cI--iFtW!(~V1Sj3zX?-BFT? zi*TASUwp6jnw2RzQYnhJFSOkGRL0>iap-$omGR9NZ&;N$-k-O5sySb+x6b~kyrwg! z?8Kn#MN4v`Hw76unOS#u!mWiXTkLDm@mfH{Zvsl%q79t$z$Naz<6Srhepe}J{t+9mCH`*sTO%h z6*h$GefZ2{^nk@_%A!(5Kkl$48_zFm*-?7A;s~#hL*s@WRZNDG{Z5~gI(e^j9o=-{ zuK0G7#!gN*L#Yk_9N2gQyQ3Q%xg&o>$hB@~J#*&agvFjK=ay72m)F>oW~bo96T{p* z_iEc|*9V{SdxQ%-|6W;swUO`NlHVpTa~kXYFE>eSV`y1YcAQz>pjMmvQPYirg!5}2 zB{BZtxwqolsRH3M=hvz+F!61=D!TIltM#piy+%5ToND<^!ZQREqVDht*I67AnUTZ|G zR&a6b%{=p2%X4E_RLoJcIp<6!FbQ)UU~OTovhiSEz|t*yh9!W3Q{i^33{THWSq_Uc zH+>zrQXAMM4(VQLl5k<;_;=ujD8tNIDaX=zf;^Wt&Q%B!xNt@wAd=zWfk$t9rWHm` z_q-?LUP8v9h3ru=(H(vY>G zq9I1)K<~}gG)c!JGE*iOEZ8!)@t>=wIOJ@X_w+$Pn!eJ|n9XLRH-3li87 z^5|d#_n9`XmIDi4JSdbozT`_tk%9ZH)F8pKMlp$p&q^Jy*zl&aNbG2x)( z9LA5lN3y^8tlGdeUzdgbgu{Z;tF;|R7(@=R&vDdSqNu+i(fSWF2TSBf+k-*|jS>wF zu5}kAXE1WKG%DP1RA6XQU^%43a%k_F6b&0DjV>pRNlv;qoOBJE{pT&xOgg8tt2Jze zlIfdx6A4vQhh~4HX5AZ2T3*bGEr*O!T6YN~^cy&fb1*O`Ffgk)thRCBuxYqaA}Hyo zz&gjF@^9Bcjn2t2Z68w~B$8lkm<6<%~DRU0mUuaUCa`0g3 z0fw(U>=_H!SO|pJsQP_Y{A%@x1qLKZCx@JInkVmZX6-HJ82PQp+YL*b+mImgSGy01flTr@6(Q#n3 z>5pD?kWGhC=!deL=1H9$Oi~LPwM#PM9vt2F&(ldEwe9{yk-V!X=ic#rcrk(JfJ67Y zGnPe?iyRmV6pk0{Q7&o-QnPI@X$abH>nmh+@ZQ{>^d8AMFB=3q4%$CCq`sp`;t%Tw z85W5)*XkID&_Z9q50M@{iIo~h*j1#CrykP#z@-1=&^y<}kFE<%(-?K%FsWrQi8VAR zG$d&hDJknPeGOd{^Yx(YndnlM=VDJ7gs*f6UvOZ1;lQ?~fv@BMPYFY7CZM=j4g;JS-l=!*+#1gn>!ZM%_5%^0ml@0}N9dtr{AOZZdmiu+3rMjBwx`8I z8h0U;b&Er+-HMqzQd4U*PI#~>*fNUb@Jn~JX?h;|mvQ4H|A7VOn~w2aIlyP2#(iZ< zPtnYx9}9{&MEjlEQ`;s@co!?KvQWHZ%7h|D*@DK5s3)#chvGEcxTnnf=IFpW;Q(s^ zFZ+fD?u1ac8NIb3Ev`4xv=2C`y>Oa+iHYM&+N`XDFESe?pB&QL(WLglmyO4dJ)wa+ z;lR;**=s`%Fy+i;@Hx`#!@~OI0>d2{#yc|WKTL?5=#sGXdNW7Qfq(-%SDH*p8Y9+t zd0lA`J~4Hzhs0_I5f+crl@p#a|8bqFa%{Dc21`N%`<{pvJq^}`sKzB1*Wb#Ld)t(9 z;GtN_Y|p(tVgf;)Z?BoLHS)J4Gi=SUG*k0c?CROqf9Zhotk{3c{>v=%kQ7Z1;Lnx} z{akVI;KV1oCQ;gZUbC#Yw^HOFtA*Nii5Bh+4$KaB=Go3*ov8BprIXr{hYzwGMcfoc znHawC_C#<1!*2W`w%mRogJDSNlrNJP+H(lM45RFEFx(Cmv$U)0=b9Lx}VGKao3AcR7gg zOeiZj$Zx{PTfo3uz{alNQ8m?NqwfKUC5dVRI>$Zz^YijV+YU;;a5|#J&Ee2t*~;z7 z)6l}RF`$6^Lhk8pCLG%fx?OIt_&S^rn{_#&hsolGBU=Ci+l>R))_PgFN~|`}kDS+V z^vJ3o*KWk;FtDaHu&rUglY2lq*0Ir`k@to}%{>S2D-n%z9G7~w39{V0x2{v6pDiun zSjG;X&Bqev`P@vprNDilVPVDDt#a;+-f|kM25CwQ?fSCC9X`k_#_wR0k@skFYB*q@ zut8CRaY`BcbrT1E14k)_LkbMf1@3yw{y89T!=!wLfx+hptAz%m%q*Vs8&r23`X?E4 zkiBOCpNRa6qaXBI)~PKBu+*Hv{hDRniYVdx0nG*-Z3=}#?i*KH9yr=q$T@+@;!opI zn}hOsu`8QYSy)0M&h%Y7>mcjODB|HLQPJqd)Uzef;fL33HjXCs7AJ)#DwpwPBXF^MA2nIef*i|1*ibqB5HJLa|K!-H*d1>#rU zdS!olR_G?=#GXEfN1;*y;v?H7jUh~zy6Dtz-SX$3fvO-2D;*g{SWBS=6=Uq?4YTk(2 z|L^NjUl%5e4m~@MW}O9k+YTsu9F|)kp^)RsRQEJww;{8|`!`b@6nc_01r7&3I`l8F zStO%@Z9^ev1_PJD!N65Yh62sOPO7Fm7*nS^IR9?UYGvHEQEcNc!7Tolr*BM~t0E*I zaF72=k42w}s^F=fqM6JAMN0$^_7sHPLgQNoz1zXkb^lF#f zk-RfkOv16w_m;8vWXT1k`X@~V_LsJw2v~VipvC923IBl@{RX{rTP^yFISri5RGQfC zFc`Kl8~-aP?Bdj5iEvmmEsQOJL3qj`<0(y@vn`B84*%QBD5Sv1wWUHtrBPZYRbfY? z0tb_1OQWoabEmHvSI9S(ACCQvj50}X9$T+X_;>MiN1GyNV`7zZQkTirV;YNAXtAVZ z7yalj+EUVAWOVLv%i8;zCs z9fSoYoxSFy!Q-TPr%~31S=i%-uIvfUIbs}{Th%V`^2aZcymMC~!EwCr^&p z^Iv1?Rh#Q~S|tSy3FcqB{p2He{#q%^mD8IGPP|KEP;OyXPGJy=VKv?0_hR*aWcEIOChD%pNWZW&Z`i2M&*KSMQyK2>l{T~7)2wh zh2}UJmK@UbaW*Qc|*Ei`x3g>r9$gQnt8B7b(8CJj~R{&NC+?tx=+; zkvrzV+|*3&GY--M%$hBZvUeI)tq#iGaFRE0QhdN6n{b)w$IYYD9M~HUEXmv&cK@ug zeX-sPCjRhSk_iVzD;gMn2-N*x*-|mNA^iX9@^ow?wCMc2`9PQc7Z|5CU_`sEjHNxb}w^Q5|e0}JCJmyT2NwiO>%2r{$r z24}e; zPoHUO*3(m>Ys1#7r8_md@pLeWim^y+aA;({WGlWTPNC7{^(Ahf{cXJmOx|4KlX&oM zu@by_YvN}9)3e+^sowX%1z#t&uz=0-Ku0RG;o|JY;bF~1c_J7ZukaKk1UT}( z6*jvs6w%->Bs3=X^TJn;qQnhXy;`L@ zD#H<-3g5sMH z*yejUcW6ZYO5~WK*V5Q39wE@!QdVB{fKjm5>D0=VZ|){_t513KM@uhdkJk_B#*Cy@ z>eC{W*mX;8G_6)!v-OpO#O{PgOk8>g7BumMtb5weW%t109JhkT{8n90N5=_ISa!Dd zD4$snS?6=^!jT@Gbu$h;XOvP{n86|8u!(I$TULlP10(kg2j*pLj2sL6c(Xn*^BF8) zv46nG#^kW+ii1M%Werv#hD7!azK&c{9u7=e39?)Yja(iAE(&)x*deB^6*aajJh zK&Kr~l0fN7ZWW1(93Cx6f)ytu4jX-Fbh?owAbY_{*vi33z_NiwgTX~}Mgn_MO(NGq zfg@~v25k0o4)RRbI4seZ&@%7m8ph~kMk#%RO+4$=Ob-U~&Xv&6NaXt*uvn6JzRb}k zTbm@2289DMVaHk-`q~)sH#aX&b71@*;lLmw&~STMa-%ehlEBI&H^UCWuAm(UIizkl z2vyzaOx}_{@AwbtD!++cs!@!bVG&8{6Ixjsxuz7I`_$~VD#G<_W_GvQ!=!&Amjqf1 zc@H+3K49GCvtXgXoQdlE510aOc}!WR;i?rBDEL0DWbI^&g`Db^0nIH8EQ&dae9Z!_ zx;cT|n)eQg?MQGlnelLChR5UOmo~U6PFl`v*rFKL5^+dgsYGh$8DNZ z4)OVLxGEf)VB_@TAZOhN7l|+I%MV@=V#|Hu$h|0m-6}*OSl@}x;J$E6 zK1;FJbxVWDy+jt?M^_rMgbwgts$dh+C~yjjJkG>$L_{RURUlyMA?tt!5BtD{hxdyv zp0{Fx`lB;}ty&*0bIsl0Bh&kJ2d6a!YNvjH|KtsC$K@?q=QoY6tfz6a~>?@_|wh6%<`ZmsD(*j$%I2Z5{ey`B1{4cUmRxt zmB8)fvw(?p$7?3O2@Pfwn1li%STx-(vM%6tU_2J$%dE8E;J$ZT{j}2@xWyEC_0JrT z>Y3rJyUl@N)vFmyc^t2JHC7*bnX%UDl;pKvU4dLdH4oX|uW`zFTFDb~VARH z)wldq90gu@G6_xkkg5`LP-MI61mPtM1WoFXxcYlsD3iC^rQqNZBVx7g)g{p;;SJnn zTV{)kFZ*XS_3qR!nh}wj;!Ml#zPYE7aQgEhahHQj?b{f65-qqw_8k!Hxv^02j6kbK zL1MRm$0OnGAKXmy0((m;7BUER1T(5lXtZxh5^0{{$hz%9i`bt;!SW6!Hl;HioF_c} z%BvhiW_{EQYH1QI<2W}*Wx;i;g)@#`{T6J>p4jVVz$nwO#?%GDRCa1kEP4iGuFhYYTwGuET8>> z`KjtI?oxqP*`5#o8u?`$pWFUFz+1B5v$TSu;Imv8N;1w$dCKCq+ z$rlX=wPF~V64tXy>~LV4bcJJA*w&1d2OSdsUYP4@+Z-2HO^mXE# zydzZW*W9k+3D0G`UbGyNGKfjXd5nCnZC?#va zKIy#T62FIc-c5VN;<9!n6n69~knd9pKp1!1dud*9Had zV+VK?Hu9`^z{aAee`LX>O%6PJ7VxnA(-bjhsdb(kq*pRwc_AGq5rxO5Hjv^Q@OW|Ap-mfzKBf zb1cmfo^wIsOrmJmLzydyUrJ7JZ5H5OtH8O*!QM&n?2?A4Jr^7jo;>n%;<&{SQ@3Ep z301X<2DO|=tbPtGMxPk#^jR8|;yUs*mt|j=^*~GLEx+*E-E|6&7pSwd9OKJs6mUr3 zF?c9!bkHc{p;~65dS)WeiUw<`#>%QZ&TR}l|F$`B?{VP%z`$>jz-YEoy(ut*t$|Te zVYzj>B798g~t7kqCJYDVUAxy97WxVWrG?&Z!$O&vA%Gd16RrFc#%&I+c?fOCG42d zu*mOt%mnXPvrpga^7(onmAMEleWS5^#(Mtw57?QO?QVLYICC=R8de^cMgf6|VnvGL z6-Cz34|&!xaPDXp*>iw7Z2@zh0{cG(fqxF`UNrE1Tkwyy=NMx{pxxXcQ!NGdT?g1^ zF#MjeP%_7Xflt+R*(NqQMgb`wPrZg!4V{dCvofR@7?@V1)M#2Pb8hf@#-veVHRA!_ zzXP1RIIR~bFpDX$t$obqktns~!0c6@m^Mvh@;boeeqn;~#UP{IGZdc9YfwGlYL$A9bKikwJrB4H62(gv`j-od zI3%i{f51^UY4bk?j-m$6M_t}89~e{MLX9Gq zyro7p^L2wD@(I3laz43qL6Z%T(0n?JaS5ghkQV9$(4k;YXj9LtO z`U{w?7BE>UFh^}#3ApdTL((p5=^4U(fmrL48YqZTNXlq-sI>4mlO8`UjlP8hBh3{p}mYiyFl* zG>TU=ik({^8pSBw$6&+VE%+`$P>fMP;-TO_hbF5BOfd`CEDo?nEtvMNMS<;@Z-7R~ zoM{KeJQj*EHHx@>Xz^Oc$aR2O-C>u?-CC`k0;6{Ol6KGY@dzImB$y z!1iwepGCujx&zDw4Gc*R*E1TNGeOtdN0ct$U*XR!#OdkLD7DN{VAa;JiC38n9=PUS zus2~~mANi+v)SfPo3XL5=#5a$8&ice7(Kl|dpr*nJ(nn|*7$jq0_O=HaT~`+SKcVB z|FV=VjH76BOk0Ce_9NB`f90bl$`3jE&V^{L;8vY2oP58e;{F%*1_ky>>6~kh@_l$9 z*7tzR!BM=7QTEots=|j{=METcdGGz;H8)qI;D-l%?-IC;GP6n<_-CjK?RoHzc?AnU z*8?W02k+x@QX^70`|>0+5|VY;5*PF_H!MixSv{DZjLe>nyYv=2<})QIqM)dQ@7@Jh2>HU#kMrczIwpl<|W0k z{eMI0og){SuH}9VX!tngshh9k13$+HT?^&Z8)I)?jeS}wYSAb>K_V_PQMj+9B=j;@ zi$UpM({Cl(ZW)&s9$@9*3R|QkRH~Q49;2|>PeEDQ%xCVQxCIJHmrEQ2x|Xhb%75R) zuX{Tu(?x?N!W?rB@M#?4$S6|3xKREAqqrSoU6i8uh6b*E3L<VHL{j04Qy7Kk0% zE~d1QZ_PoeR|_Q9F!HZylx15exTbN!2E|o35@%HIM_ecL+PEOtP>tYZ_WC;VZ&#R zcQS1j#hQ)6Po@e#5ETt$j5BW(wJH{NGe0fS-ZgPWeEfMY^Y&=JCeCL%94b-Y`(Crg z9bk!JFE_rF*w|jKDyHhXYq!TN&ag-9GY(Y523gHa;GC0ia1mp@L8F+?LhJJncs41d z^C*fRJHYXeh4H;aqV|Ue{2v;)OBS$NC3w_@u@@X*EootkIgohb5NnacYoV=dF}nnW zHi#L_l-<=RJ?$oUOoG6XgQWK_rqU$@#Z zHTv;R;X4YPcN7Gy8n;+46f&AD^d++RX{czGqv*9n*{X*!Rf)n+N`*Tr3VaTVI4z9l zOnH{-{%3&UBQ9Cp2VQE^M6VwV2yMGu}n*peK+sG^90 zqmE%|;4aQAEvZ8X{&A%364g;mKl5POW^?f~4>*=J2+mq4Sh9dgLM?H@uJFlLt144g zoqUwOSjr{C|#1k^kIP@Q|AP>+uTPO`1drhy}7;fg9E$D zLv0%ep^^j4N0fBV9AfTLI2u`V^kSKNofx~E1pfgC?q3QbHV3z@NEFz%Zj;6W#;h4E z`!2c$F0hg_7Zh43aVSyh(!!J91x2lbi?tn#AC`JtUnqZNq3jZY(?4%Ym`oS)5EuQi zH9oPwG%`GT3rEQwRwo4`jtd*u%N!De9h@zy7MDF&x$@=Q%>IiB+jmDQJxkJCdhmBe zV3ZZ(G42nKYV{5pMTH5QBrYrcC&ax)UtmU~pb%Sbbc0UE*6_u<_Wz$9!5VXbS%E?8 z0bk0r%d0-~|63ur#DhmA(Ouvfvke1F$pVI&BknPKIJy{ELgpMjn{@Ov)2g#d)6O5# z@%qDDkf5_@SCT{<+Ytp3mxaQ4jZz#7qq>#n-d<*9;xPNc&%)zdCH^?dUUHQE#dw-A z%Gfl#)B3x_wmEmU#>hS@6%N@g#CBZx*qk%`pU!Mnb#r*Gxc&4J?rl4glGq=}usxf@ z;^PooFr7urr96P^eDpiVSqi)VPq4HWTYAuX>Ffu)k4pCo&YkoiT|H9FH>y!AE>S## zk-N`^R!y{GOJYczA$8#2WT};*@z>xrTc(}gt-Zu=T2e{-+= zpM`h+D&Fb-BEutpru4Unjid61zh@GS9c`St4f+x$N~>|z?KJ+y>T-a^$zgH)f$e@w z72B3B-Spb=*_%h*=1YUE_8him53mZHyqqsBQf;$BKxCMB6r)%|qPSXf`lbf!CaGNW zxolYrRxXTU|02M#V_%}wG2KbO>Qx<%vpp~_Ik0iI!@uWVG3;Fr_V0hhT#~@)Q^S2F zo+sl0v)Lc!75;x`KhW8tQ1j^ikyP7h)(+PgA2qYf%B#J&@bKYbULIMim=gyW*?H9$ z+OTT5`iih}iCGmrxnbDMCL-nJYaF^JMBRV+ikZgctFBv>t`3uocFmk>ReG#!)oXT> z4ExfRJ#!Cxt_)hLwZN}(7L(|zDJz2)dNejoJu9pcIB(0#Qz~8)Zyg1l=yidd+ zsKY?9iRF&C+PoXJm!9TLR`YK$Vf^&u_~iY8vpD`8e8N6KzN7w0#2FR$6j_H?e-uTz zj!g95nXxs?)m!|<`Q#pxjhoj5PUe%(*m3ORjAQ)r4j-;8W^Q&8}6_rZpT zhdFm>xOBL&%gA*p=sZ$fIE`O^S`EWPm*YaOIR)!J_&F;aY+>3TeEn@=XCIdv$GwD4 ziN~h%G3mr?$WLaQ?6A=4=Atk!HrdDxf{lui3JX||ihOp{I66bALvwiz6PL!aC7b+w zj};s`WoY;Ih<}F?)6#Q>wz*E8L6buox%3uJ2y_hz2~}m4j!6CLA`)9F+R|swa`CW0 z=1Nv(9(|dWO`Q5Cz8(rIzj^d}f)7vFj07Ksi8B)IGiEj?_mu8*;Y%_)cvQasO3_@& z`D;2o)u->NY*VcJ^RPv3Ys)5AcD)@BX7O1td|WOTcSX@N&+dmOa{=$)FqwZ0yaEA^ z?5!q?Sb23z0$dnfb50%1Q+Tm}mBS=r`CXy;QQ@+sku!GPWl!n|jFo@;bz>`E{gsc7 z@})YAZt@{l7&$~Fwlp|$>f8u8T({7|gV$IhV8Ve$1;rJPTh#=E&uGr&VB*qS{$P@q zkNe6ZVQs6QkNE=>Unwiu|6Nmj#?oYI#K|t*Swd4oLN*qPisU`maW&kk>uR!q!y;B@ ze$5T3O)*LrmO4%L|8(^F)W+DJ)MK5dYLdxvymcNT5PSnj~H!GV!c$H9TU*@Wp| z>r!12K}Qx>lLL*b6JIzSlUHbXcw8vy%ffAn>!pO+#M&Ab9+pjgp};1$?%P6HiQBb| z@*;kB99V@-3?8=dn%@X#;xxU|(4sJR#wU4Wse&brY)o7pj!m8)g6Cj9xM8;xnRwywS11#F%iR8*2(+LvX(9mXYRgw_+aMS z0F~HFlNJ67`|-FvP4X2)q=7-; zK?AGBg?}yf4UXKk9}ck}a4CAgT+*0V~gXJMf|6J z&g3pI(C`dMGw-+bn4fgOfiGtj2SdXxhN_p1_99IDx($xp3I`ZC91avdoyov!%h1Fb zlE8H3i-YOnV#a@b1q!nz5;)i;9tkXqeb9DSMs;IHD6{mG2TbA`2OsAsbOZUVKbDMv|Z^p{13J&Eh)u5pgtBCY)PPhsDkwN1jBvs~Uk(d1YW za*#Km&^gp5%Vv!WBiAYI2$5OOZR8FxROx##9`Si_NsZ&6zDekr_=bYZx|Kf5Cw>e{ zIr4>B`pE-@=f*zb9*wJC?Km%R;h`|EjRLFN6eXTjGA&|J51JMhGV*V`$P}3C8Pe%; zyfKEkfkoV44%dT)0;Mk;xV;z}IUF7^=@dAY*j#AbeaxVZZ%sqKO#mZX$r7e-JNY@^ zB{v@4B+M*+;DA~F4ZrRGlD4vS$xh=fF<_1M=$I#WgTu~V+BHZdnQLx@fR@1)zL^|T zN@KTp?n=2HQ1!PoAE5kf%&+s@fSA1A~Kn z3BQ`AUSl=5vNTUg^svNPKn$x0gu^)f4pfhQx`JV`KJ-Rj3Xl}boRpLRZYTiG49s4F81GiySVI~))O0$CdlwHWNGNhROgO|_mEflQPN3N~@EN<+iPsJM zGp^pXxoSQ`i>Xj$Lj!|C17ku0i=0?!BcsM-R-tbV+E~)3Y48(+rx05*9G&RLG=hH%-h4V2oPuwu5a>gsvlpl0bfivwFa7 z?yGXG!m(c#`HETXzVT_Xi_qPSxXX~NEqtr^s+zU6z)EhHtl53C>dbp@`equZuw8q`!1=6;S^B_bk$DEJ#)~$x zWX#>&>BrOgOrwEiVuk|e&bvu_*sTJSeP*6AwRE4Betq3SnJF6oj;*!%t1#8Pq=ak# z+KICL%RiVMnVI0Q%J>NL%}Yv48LGNDc?4P-KK3M*8~(c^d;dqvD`vG9b61r1y%+y= zp@a7cgTTBG&639g+6;Rhawu>#-_KxZ3ou~fI=qfi{M7{M&YG?M4jNY)W-s;@<4Mor z_IM`5W!S-3-@qJM?!fWwl|Xj?=3Bgx3vNp@H89QyXLwreZ{ifQe7SONfB}1Qx)!TPj$?~Fq(c~HW6t26UcQ*fmQT}3wuxnXMk?ka>ga8T+1h0DXidH z>zulC2iLUesZNX{YDFTVq56}Z)1*X2Wu~SD39yDgOgpusiif?$_Cr;!V)X^%>aK@{ z(TA$#HcO~Hmk82OvrEWuDs8oF(Al-H{1FFhg#wp}J8LQf*IdE2`3~Ik3s|KERraKp`W6=R53D;Lu{3^Q^kLv&G2~bs$ZRrUvS|Uc z-bs!>ALYve6j&$ui#0GwM(BD9aLh~KoBM%t#RjgG&P}ThaP9KQ|8$9u=_;RHA&*%i zkLXsV=^rY8I#s2f4E)nHtxScL>9eT)4I7Tj%x6Li^^S^dnQnMIPYc*t6rqe#HnTZ zAk=@UugnD|4z}!lPnh^#vdT`FtTN44$ALM@KvpA!(dq&F)j$q!2j-T43=``wu&ayA z@eY{mbAkQlK{N9WEItN|w&L+%34jh6fc&;dL%u7{JC}i?e zQJ5)gAELl9-+_182hPb1Tu+Pwx1Zn&UD>^A0@w2eTw6?qUpeTRE2d3ez;cA6%20`Q zQTVi@pY`*mRoR|!&k$@W7pvCZ%6#)^?A;dvI6r{eHz#ZNIIz~~tfhDiNG>SoHkz2?`L1$S7<|-k%I;R9V21bhmtZWxb zxFSs_8gQ&#$sDm&J;uC!^G;UIh2~ZctZOEARvtB$NGM(VkWrGM;e2I>g^TRJI@9`f z3mX=uFxx3CGJC+%puiF2%3I}-$1+Kw#azf^<&?Pwy!$`rMQ>45x0wGWvuU?Rw0?uYq$gV&H@$>2G(W+E{m%0Y9*5; zmP?eC*LE(d7Z6}(*ucPNAX8z%QD4BG(H?FQ5Y8hayIM?6)PUL6*!iNN7sHFP2VX*5 zRM!7*c3kz%e)^)xh7)3RJ!P(ZINsvH*7Bq4Y?=Keeun5S zU@j4tl)1ocdV$H_fZg~5v$ZjEY=cg50lUNj*5Ux>hQM&g3v7pt*mD$E0vI^@AFzmg zU~K%r)AK>l_lQe#`4p)RCXEZ6jRCusG;;;a;%Yt|~wIh_%^)O~Z?OS^-;H1KaCv)`$kq9R_Uj37NeNY!l zYS)x+ILQ2LhF|6Py}K_?zCT;X?*e-*15=v7%xMj5AG$f$E#N$w#rh(YV{rrf-o*AP zftqDUO`^Zs2kns9DZ;FBjQz$ghE#6`@q$c_#@gry%oYvIlP)mvE?_R%AX9w6!SVu= zpu!nJ2lkQ=4kZkXb_VSH4ovJ=Q zOb^(uIB;=1vh@9P2C>)_F_lbjlxIFn*HQ~$7eO<>7S z;9{-rk$xJzKlPtGmwQaXnmy`M+0q49HEapeW8J%%okijD@!CJ?vTu*e-d(m{D`dx; zy&Grw^)H_M(uCP}0qX%yraiMbXI|hkp3513f_rBIOJV?fZvc0%0_Vk#UQrXIcN{U< zztEdY#l*hY$IxI~l?4OG2L{m%?28vL3k5KoeVCPSnSrOF-6DWd(1GF5K?ddt6NC&B zxn@q_{J_9@^V&wUY#spy{tXNza~a-FpYV1%gMfgGO#vg1LPSn3BgY0okq2yN9C&0K zndZ)3{w%EGT@Q=K1eVFZ7gGOh^b)x6bIsA1JM-6ibN;LpPFcee8o(M{l6LU)v7EH3 zoDUatc2{LKu$<;Dyr>;};a{+d+H#93r=XoGVgf9QsKaH z!GtZr?=E0&ILp9) z;hMk(K_&&(GXY$FQ<$2|Q@CEca9i*C8^W{F`hxsyt`z~tX66RYt`+|Ia>b9CTntth zJOfxg4Oo{pa5o+Z%vWHQeae>Jx#`f>mFwQF6xO|Mb0EgmqS|1=DvJV*9d9r0;FW&V zaavYtOTY8e1A8aiuAJ=ml%+I)U%A&f4IaE*J|V~U0sn^Lzy&Y)?p@-3>G2ntPkCR|GOfOr(QEMF$@;8IkN4ev(s9Cl_2gZSPuESHY`)>0(`A;53mo1T z>=tcMTQGrhMgwPrLv7>**2ViqxyoI|TzLz;VH-Rs|f$O-zv1c1zGoF3zWyrbO`s;It zH>(<~S2VoIGq{<;kaCW1kH%@XCGE{?VwKmPy|rrjo>SUSPA)ZQ5ZPP0^Y$aV?>l_n z9jv{hd3Lhf>dB6e-`P|$dlg$`i?YQga4aj}o}a+s?$BnK%i=A-RvW;z^a96@MOOp0 z?mf~?tZE9qw(SEq|BdNJ%$8}4B?pQb9T<2XFsy#hz-_?Nvwv*EMXtZ)sW_U*|85^>=ooR;3{A6lJ_42AHxli4GbO(yjL0?u3vYf-&RP< z;q|kC3(qF(^2_I&+VEzY?X4NT*1`2WQym`7F5ro>ziHsdx~PDwQGiw2RW#V(o95}X zHFqz}kxI|z7x1{G5O{epd%v7P#gZG0%xumQ7DYcMLSIfTO;pxmR;Z{1QLue{%&k9v(n~DnyRecnlmU@}qD*D9CzQof~#PQZ5=SzID zbFQ7ZnCjOmRx0$jvgk0Mu^W$APpQH|MkelRp_?0$p9`*(U=-t#h`#&&B;}f&+P?&ZoF{n&KWy6O9ADP)iCH^F+@wT#_nQ0nrvGCxbRNjSw881@aomDA5 z8Z^^-`MSwMH#SZUnOXAc#w<+IcuepyOx3<1VjeufyYO%f*X-wcH(U~f z_g2l;xUq40=H=uBsb=e{j`SM^$NOZ&FE8K!Vk2AMcX{u1EI&W8)lN}PKQbfma4Vnq ztQE-uj{7uPChGp$u#}IU_1MuYs1~)tfj_M-p<&&wf*1uRHl-EaCsY}uALz3$ z`X_RR@q|!9&qNoE7u}8QG8qkw+!_vzGcp5CC^VlnaDK>Qs>`9vtYYGzG^Kcw2CryH zm*Mg4S`FTd{BNvsI65b}Yo<$Q5|``KUX?{2Or0tn$cfQ+V3Pmm{%cqT7iIT|XO*!YSn7IoXQ{aoCv z#t@lw%coG3K=@zU*r`am68-kUq<2ysGy!rUcp-Zxf#CckjjK zu6fPXAr_Bv6j^qv7?tu_nJsB<>Ro>GfZwY(TORv`1bV++$7|~6Yr3TOL9;)*Ooqnp zoL_GiD=v}tG162O&$^M{w!=DMWv9LbBTJY2jERf*16gi5H1p^cBu;dH!ND}a?cIvj z2_D-%JZ$2hD3fqU{?&=bMn1g*34KYbiyNEt?@76~tIkkNWm6Xw;9=r^)!@Pvq>%Y! zA~#FHCA~whx@N3DC{*yEp;KM2hmosl$HNUQ3I#k&Pxvn%;C;z|xX0Jbcz2G;#R&zA zXC#Sz{oHG=U>cwP?~A->l*Pjp2B|?>(+;KWtEFE5-FWO*R$!sJ-PeyD z)lW80ntbGd3Rg)ZchUr*|4UNX_AiR|vSMJd7GV-F=U|Co3FJ1aSimImqJh0AfZfW5 zQD6y&i{h^htgBrTTuFUUdtD#MQ+l^m09uMK&BbsiDvSnv7X^ z@c`Ss3bk_^G@08^9^iCqZnD_uU+Ww!|NC6I-9k&RT1^ip{uX7{zyM`|Gb^wBS#V5q zS)}{q>zq^n-?`A{-*&0~-=aqc-IEedO_(#uu28y@N%Dk)_=UAiye$fMGcGX6Y;jrhvPB)`Nas;}lO{Ef4pp zCY;$4Qom;e%&`ktSQj{fN#xn}Im>giCSQ|WDqZs`*p|hV-|?!m+?|9gHZmKgm_BG^ z`uq6`o6}YMJzgi7rB|$uI5x>%_g}^WzJ?3z4mAnPRvgWW#}Zh!$R#o>a5Ss`OJEP` zSR^vfphfZAg~@7H9u#jCaFzI#z~OX5NouP{yXl7qO^!B;1Q$0rDTjo2slI3sIrF0F zo!A4bJr`ONISd-?Ma!md{_|OXv261GE^#LI7KYx;m;)TF2|ijg4)UiOIOm=XV7uk_ zkfY^s59^zXWfQtW{2Ex6cy?C@O_iHet1029k@Jw<+qAvz!p2OsaxSlxKTB3P+P28; zs^)NzaoY3f>pD4~$mZZ>)5F)s=rb#GoL+6Up>fk0E!~wx?HklSb0ljVR?qm@Y1H$O z<8FzQxRWB+?k>lJwjU44STr;{T{$GcweMf6?v6t4yc5YHtRD_2FkEEizw>}k_|^el zwdi&Ro1AWb=Y|LY1@ZIup3ZNX7?GHvBi8D5($KPwAwkNZflVZV$$iILiDL=T%E<>$ zq;Ajv z1#A+{>W72hv3q^e0Tb}cl;`FK5lf8ubhi_)`WwqItRJtpKv{%>DX792hwcSn5_d@>mHl%N=N7*RNoF9wTu?tjvPzs^!9KOFP(%-zhXO zYV6kI_%N5Tu9rc{aE)H8f+h$#a-Nw z_8Ay7`7PK|rg5mupjrC_Q@{@vV*!>xj#eju)~FS&ZaO|)5B zu!eS}GOq-)y@An|&n!j@n0OB`h-fhK?r2n*q03~_?7gBXS&H}VPoCn%i;IQV*>5rm zQqdDx;9&6D+Pz56ZG*_qO1@_eQu8s97|jL7uN)XrMAtD zQaZLjcR613woS`%OLO6Jl@@aoU{OBM(*L+s=?}X=#};FTRwE6TXoXg%e;-!Q{>?1A zVj)Wf!;4M^z6AFr84cVLO^!2|ZyaPe|Fb!jgH>t=V*(R5(}M2Ki47bR8Z3Jlm2o_8i*4z$UTDdxh?n!>lnML|0}o3eQjwnbG9^f`w^` zuDgry#6{NU1zZDC4wM=n4ccS-)nf;~GuvN%m@+@hI6zCw(!0g= zg==~P+wvB+^a8e|30|vI+EN3SHgWIW@vvXY##Xk(P3*_1tJ#cVE2gPSv>0$$N9||{ zJJ6ym+^j9p?0=xuSYyw^6ASDPG~3=_mifRa-@zzZ(Jzz0B>#d@{s5nK1B*ii(?979 zM(G`mJO#^SBAB8R+A;r|;n@l6IXTb2KQ&Z}yf0 z29G@h3;H6rtn^j%GGEXfWwUDWr**akEa?pFbpnQN;=YWe*h4A?gi_d*f*7Tk^<>2mTIl<;fPs(jDQi))R)@apF z;Nv?Y|MEs~Ko;{`8!f97%_pl_G8Eb?C$Lv?2Ubln2dIfvcg!W2@^Flm1oc3ryjhC$@S zr4*)CpN!o`9lOu}=rNVNQDlAj0*9t~Y=osRV^B$i+5^w%8B7a$jz+HOv$xd>xOC3u zL9;DGTYdq1)du#O6YVt@+9zLSpL(^u_6B>+hIWt8RsA=-R#q)&w%*~`-6%7qkxeU9 z>6HJvP0|WGn(clt2pnMYX$W1qlr{E2tG`9_u2rnj8(6s`4HP;ST4k`fy|9$}!65sC zK`KBXNP>0k(K9Mc-TDfxI~K7;Cp2*?FmPow6s~Mwo6y6Rz{s_ufiI&$tClNmiE7-+ zM&(b9(jOS5UoieVl+vX3qftF!nPo+jrNj*D4b0{{n9U=Yj0~E!ESO4Ou$Qe+o2bGl z?9j+*)OeA#(TE|Kc@7hA!eS#)&E}^)rZevb*=&DS62Vn+!|d+KQ(Ksy6>MIpbE?Nx z%k}|t;*Yk<1?|-b+NZ|eoa}q^*;V$g2(3`B*|ydG&Djh5S+D-T5*eixDwVOt-XLo0 znU=5xtPwwU{I6PeDu*q)LdM`i<2qMa-5p|19L%~eICXC@86>n7q%g_uXrBJNIemd? zTEguFEf$S}CiMqX^jLMu{1VwO+LL?qRt$ zcX#hA7ciF`*veP5S6AG7 z(Hm!at-XIo$5JQ#;8|0@EV>zd$}4Ks!9Po8`=33?P`E|@2crgqjL!%CfFJ9PCTJg9 z>65f{tCPYldx2eU3N6+v0y4Q+ZC0Gg(3RoQXq0bgNqwQPa0c5pQLe4Fsar!IrggNX zukA^%eVA5oI6Z+aEtV~4L+Vyp)*uenO|dQ7L0(p~H@g%tI|#5?3$UntX#R7!DTaa7 z*nmkugYn**M4lHiQXIV64_I~ntzc5&<301C=k$Sjrc+}z^jtOUp4X<{2;P(Az~O$1 z^`7n5m1YL3>}9naS=ovtZcb{w_v~)FY1qw|UmL$aU~KFYwVcL(Ve(_^b&bE?c=@~T zxE?I1wxQAGLp|Sj$t=M~P2!An!5Qfl z)3$kTeCpo}kGZmU;0O8scm<6u$S!Dy?%mS*5?WWds8 z#2T2Ha|igC0f%~utirapfT6}#RGc}9ybkEhYU8` z0}Wg|7<#T8((Q~cx+(NgF2DUHce~Is@oMes$RrcfV9mNmrc;ybrS4u7%{gGk;TX{tHpi7OB**^h zIvb9VB_eG_E80)F-8}v8dG`nJ{ZAT&|DW#u@h?aE2ICj0+|Z^%+u8mpS8hr@SS9s? z(JG=vb3udIh33>B&5}D9m47g3ZD>~g!JwAFWF8PL`j%Pm$5s0i6Z9@b=}An;eyM%z zWpm&P)}RL*$Fo|b4=^l)Etuj~b6G}5b>nSN4&JsHRM(Ks6T=CKdk<&eCi<|5;n*DY#1$?-^Np6Gh z9>&GCP5ut;Y`V-w4o#Cz%CYzt6uCWTwQuC8|KNK(w!QbVZHM;aWv=idZtz1X*=asQgH&C~krv~wqaU@R?a;#t9{snFt7(Bv4w zmSpkko?hb3ZCl+KSOix^>tA5hX9(2)(6DPMqnp7?_YNkG6${t(Gg&7vF=;Tc8GPS$ z;82}s`B(39$?t}Loxh5|9%YTHNK>|8 zIx_vVt9XI?nR!AddiI=KY|`?G<-RZ*Uxsbqt99!}+!HnP1R(X2(*N)DHH# zXV0gwH%%?CoIbs@OCaNBy4DYY7Z!F37qYefui#?-+8}v?(aC@%ESpJzqliP2b6Sq%IcO?(MWYu7e&KiI$@!NR|y*}bBv-;;qSp_x1L-#!io9h1cs>$|twPty44 z)1Oc~zK9WaX6$Qrp-fH`ye$ zlsbx?nCw_#5whUo@d<+bat1L!0yzYP=P#MnnsvmZJytxbq8bxghweznxRoCMH7R7Gvh6Man!86RI3rE^~Rf7i?Hy^7htJ@tB4q9ZmDnO{;D+ z9y_s!nVUzWVMb6JkC=9Z-o>C3+;jCy_Ea?<3VIqOsV-z3w4`C;BDW6;zSC?ZjZ(d) zsYPsZPiz;_oSk#$&hwMOD+1RC&CK%FFctlXa!jVI>e}a>b@tFySI>bvKR2-N071?w` zKH*LyyIf1f!lMe!cMcw9m(AJGBHya>(NQ+9P+`{8MTQ#~n3=a+lgf@imNuK2r|0DX z=54kKj0%e*5{{eLavV5jR3-7Qv*6bz&3sNSSbv zxi4x;V5MB^l|WCOx#mJ%YD*r-E!Sjp=Q1vP(h|Qi_ltN)z<=2y4 znOSri3=TzZW4oP`I?YXuIepT%159EnPF!q0HpR}I4Fbi7nfqsD3bM+FBpxdZHMz3s z^qr}ON7+Rp1P*a#hKXppiWR7=T&j{;b;v_{@AZ3L?^jN7;`$sY%;CTEmG~7uC&sDA z6+8AsJqY5s^eRNQ#6aR|bC{*Xnhk0G9-(bwRbQFKt9gnJv%AMMa_TNfQ4&x~U!ieA zW9b=<8**0n3^|nD%#1iBid~I7_Ec09dYf#YlGvfRy&||*_tOSt7DhpX3k(9qHigV= zrWS_m%uFW#96m7cI4DHDSt@Jr=>)qENSngtp-a;d1^s9VmV6fvPu z(0xXu>wKO6Kj+L(4N=_8CSv1oXffvuhj~jL*Sq|yR9HIUV)dL&`}bY?G?B?q>DS%s zY^#F*Kbd^BN@G%wbyWG8)vCA0piB4aD3yEys^V}K^Z3p;}NYiMmZg3fu0F% zY7NWkBa2T0bFczvDc5nWDTzF(5$8l!Okn&p;egMX)gI@MnD6_- z^LD~Z@6HR?+P5rPm|}OiK+5mU_Q~^J=xbDNi28p|aP`tNS`EIPEOI{{u*SUfYE11; zU{bu|uu8{(JwN-1P)`GE)R#r#FCy5jP6e1pO?k)>BEV`q<04Oi;Ua-%fn?<@!>-gT zP9k@NSZlKwCG}lC^VEK5*02%eHPYD3I$zp>!GfW~ZBC(}_mpG8Vb_`5RG63!l>=POo z%eDG{@T_e7JmvgE)j+9BR{v6&B&IlI+Bh7P@lfP-_;IAp#&N1z<0AG23XcT-HnTIh z&RpOzVG{qO2CkN>t*1DcPCE*6g|Qg7G|R-aa+eDn6`PjO7W0Nl_U#LI>thA1#sS?t zwGnL`(;d%7T}hI>;G!MnxiBm`#JI=!1f#^X4J?LtxYzg1Y~~9%%*e#>AbDv5lgPUT z>}fWPU1A2}+)QE{wDlO~^C&dRRS7T|_2w{gE?{8xGH5=#tx?2$g9DSERRG5dL7UD6 zY^Gj@#{()Bi8MJhy4qjbE5KspYkj#xk$CX$Ryab|i|NxzMbdzDjo|JK#S3XkDl|@G6a^WBs53N{Ab3YC^}8#kyr5TZLD?%N!;=l&UzUOL*wQ+^2F!1 zYMyZ52y|I2*_d&t-OZ58jpq@6x5cH2OcV8(CnpK&9&lh+3uxrmS;#N{!iUvn!qV%R zE17!)a@i&{oRj8gU=+Qb0rwG2N)#t zTb%6w`$IT(wc;VctAA58c0LP!%XYOP)jB%pc(v%Zi8z)CY zhiinx8@2*Q=?P2%A}<(Zy$t6Fgfj@Jx*h1+pR$z2p@F44f%%QxgdB*YdeWISqx99HDo&9A= zRGG8#JVlO~kD@29mf~u-k`rs5T&~$2;iB>*fSpZs!a0`%2MW!*1%A>kdJMc7 z4zgSyTX~M8G72f2y`cD^CG|!llX(VgZd!6`Zl2ca{EV=B zMHjENU-Cc8z}2(h*q<-KY5$(+^RIDYDg7^7uv|rN1*^#W!{Tzxo%t&sR?M?_St!6P z`^Vwc84ebXz5=#@3?|`O9k#k<8@Z!;7V>WsNYwpz;7ee~Yr*CWCO*3btOi#WNnYK- zDpAwlZn#CTNCUG2hlC*egMTfS z-z?^(HrO~{XHb$m&Lknwz?SF0$fFRF+HssW#evP>AnU)723`wB#zhP#1({6+oDB__ zO+AlRsI9VSzii9#LX}6? z)S1z{_qr?3y|R490UjGh6@$a_F_W%b5P5R(Mf1JgyHAy^RR4#B-OgU`!-6hSs0t{`c3``Rmrk!z+y>LOuXTdvH zXZaXss~e6c8O@89I?Ee4TTO7%SfHZ0q1nX8(aM8aF~!-!!qIApv#d+A#T`b|4bJi% zP4-7(twfp>HZYq895h+dtdP;*ns|V70)vo7RJQG1);m2qc_Qa39=Oi=$KaOpbjnJ8 z{X@~y=U(ZZJ5lvv|Ee2Pp5$Iq)M@%J!f}B0&Vic`G(DZ>y3T14Xl%`S{9>lu?Gp^< zn|#}Pl8tX161&m(;WP ziYD>OCXBpSoJ?jMlALgGi@_4b8;3M17#KTdF)z4rB*a1aMv8EUv*{N{6Q9FEPnHPx zIGNR``SLkg2sFuhG%K_?UaoP_WI4p!!ldlOpxn~L_JvVXg7Hq~0ilFPLJWt5?l>si zXk_DPk`6e?dxJrE3Ip4gbWRa}kral_S9x}}9>}Xq4evXz>(y1TH~PWM1cOPbO8;g& zy|O`JzhLX?173Cqmip@+GMP5Xbk(H|KNJo>JP_}7!t7B)+cAe$FDDg)X5ExUx)n}} z51b?`KquF!TQG`GcrIJvZL(*X#*c8>hDOOwM4)WY!6u#r|>IsAJhDN5kMy7uU#SI+i1U1SoST3@IQCL7sL_vf#g@KEM zNtuO7xPWEn$ww!aOxyJ|U9XRy^H)+qAs5dccPWFWf-{c(PYyPQo%N0ET|K4QCT4oj z2j_n=ha{iyT#Yq~NqF}3@Wa&BvKL35YVqx05cuGzY~n2Jk*>hvq`Bmwf)1lZi(|{( zMujO&LVv}THNv-uF>UW*QfYDWx^(f z;>Pa4C|=UQCveQN%7IVkV8;ap78_Ca5(eH23~W0N@FXV4KJ@(g+s43n<0 z8blv)dsCwp!jy7A#D$SjO+>-qFtZu6Lc#)_duL8e$~hPE>PL@*+mal8tDtG6r+B_J zEb4VCmR-Cm;Or^^R)Z9#lw3V)JN}ZDA@g zYtrm+lF<3br@+x<<;!fWFwu8R|4IKJrLsk~Bw^DrZMSK5s7<2QTLg zk;0h{0ySEOSNMGoHlDquvs%Q}@&I#?!{Q*BlQG9uZAj2KVkp!0j^VaTetXb*^RN}` z?VMzvG>EV;8Tvd=>1xttY0|vGbmuD5mZ;Y^9yLm292CoOl&xtLc4-v$a1^=0*s%4$ zokeVHFFIITBv}3_G#vfFx1OozUKks5jKh?oJeH6H3<3vOb@Bw?%*jwOU@>8M+}3t8 z)Ip}~{JRMU-m%=)j%t}4%D|daz#5asopViup~=LDnd?BT#*_x>iuIg5JttmWJ7G6- zif7l^02a;#Yfime#3drfQ^LhlB4w;|c%{qP_ik}3<(Ld=nypVbpO@3Wz;#vNL8@8R zysuIE@l_kccQm|YY!ugVlC9y;oWe9As9Al-vNKbiiv0k5$F!&4OwFq!-F&wHk1^>pgSJd(w7*?SDMm3HGf8TFj_Z=5@u zk|6bzhjq>Y`9v0h+3ZI19p2PEHQF!D$)LRa-vK2ZXU!)r`?fw1j_wT%*l62wiRVz{ z!OTX+g=hJj=9#~HVE#|RD*8k`+ro)FhlFjKOr|hRZ+ov>aY$s#6j>Wx)0P#xau`2Y z9+bP%C}!d)oN;hMhZA3EBI?m3M!2bDIgj4yXov~ZOF?ZB;Iv|g2giT#XrJOihK!W);D zto3(U+xZ>^9q@du`^2wDd->g`YU?I1oHp0Nf#XddL(&0x5ofD_gK2Xmz6*#1L^p8m zU{qeMI( z5!J>f?;0muX^axiwf63?F5eil`E%=|@^yWMlK=9#850;d&hXwfY46HtP-k&F>FT_u z_4G+YC$lR|5*&Km3XU7z| zH!!fr9Pm2-g;hHGfX0VRbB6SX7GB2E+U|O6TN+#!7--*5&vS0z>bSw0&>+p?q@lo= zc~zqD{s;YTK`s_1%^e3#IBq`L*%;JsAdsE##;91I%a`-XUHwB1yf+vWQcTa9U9TuE z_4C{q5nj0ZTtdsdNf+E?ZtKKEPmlXtmAv+m`Ff?b;S2}=njK=RV2JN;JjQuSJ*G*N zW3uPDWDOn0%WjwDOq_ME9Ap+}~2qgWP@Bxx{^pUn@*g_4oAZqjB97V;fvoMT(y&PLM8742mU(; zJnuU2GdSr+6gS2R`QBU;Aah80$#$EH&5^%DqWDGUZQW`e&aogqL;Upi&!-w%cN~!T zuPSiiU|Z#ZJ2M*Z)-dTNG&wXJl4xjPD{0jDVJuwIu+aM8jvI`;I~+JKILLBzPWW^1 zp-H1gNVCSA#-}9@C0{g%RWz_zd~jG9_{8i0+v8Tx!xmo0k9nXWBa30ul@8m{-x6Dst_sN~u0AsiYNusfeCWaQ55F&!DRH`_NnJl-lVk|q#o#KL2w7n5;tk#P$*uarpIof}N7 zD(2HtJRUr7%(=l%Wf7S<7ENU?SBfd)SPTT zzwA(_JyUuVrk4)r;xM-Rn9OP9)!8;PXxhuJDf)E{O`Y<;5}x!+ zTNNy1=CjaP)WG&G=~1)lk`2k6))^X!e71`;7#mOPG%R3Z(^AkiI4rjy;iRhcYMu@I zmwd@?WRuf)>%rkAyq1;Eqkz#w+;Yc)6NmIHdYdf&EjZS+NPl99pOv%6+A?V{XweevR4RZm(2bQ1Ia(x5FWeMLYo+4_n2f&RIO> z(#f+t++nb2;WEiqtCW;Z$u+r&d~UB47Pj&!X(%vph&^y%w@FKQ&?Y~lV3DAP!HVRG z4owLPjBFkaYu4BBzxFmdvSrU1FSb=8KHj>_r?)d5(n?@t<_kN*&@8s=?HVR-j^+n( zT(3A%rL0*hjLj`yFZsMh*vH|aFHg^5g=Qgx4nC%1^NRffAA2P<%?P+>a8x8_g<}(^ zx8zFKS)XpbSjek#D}s5^(%I`WY!gl?rpr~m+4x+pE?{Z@vRP466CI}KZJPRKdRDDw z*lebnlkIv8FBc#B=Wj4my4U!L+2X1GeHKjuX)8Eg`Q6-Bvhr{GCcwh)@aBQir1dJ9 z2@Q-|2Ojbtda#F~*+r^w5wrA+AB$V65;vdOa7@m*_e7JY@S6v0VghfB8Vt2{lK8v| z5`DyDW>`EtK{ZxgpH%!6DgS3`{G|_zEj7IjUFI$i27m0N;@W zH>H4v7Hfk;+}k?BbX^L&)BasqB)DC`UAfC)7T2Ey0mFs^{EH0Qxi2&bmSr%q+HK%a zoA5IJmsP}@j+Dj=e@+{$ZjPPmb(x_%^o+sQ!w&3G4LiklFt7(TMPB!04*{7=n%xm9f>Mg?HqQ#T_KRF-_?5uvt&w52(f5KHcg%tt zd7DnNO?x}tqT-MY*G5Kx2o2$rK5DvE7rTv88o8Ph+zcfq>WS1g2yV-8;o~gqj{37$ z>~01NJJaO`z6%dI%{MSdWH~h3{`u#?ZkfQO%$3N_C-c%M^^y2~y)?B`6WqM{EM}y5 zG0zf6IcY5WfrnjTo3;3vgz~wPLY!+Pj^})FWDxBT&RewoIQs-0^PbAfNxzA~v>Ycpiz;*4^>iDUS>!O;N^(qz$vd&-#vYW^mo3coB z*Mg(!HiBICCXcjw5}1U=6j`PIG>Y_Ouqa9?as~1v@%P_gX6HE2z6A1^I(bel}zSx zrbh*}97hyb0@-y77IMlYFtA50V0SQh$Yv_U^4w}cv)Y9R?4>iD#WMsNFTN=@n(ldb zrpM~QONuiM;*>6>X2i5~7_MP8ZaC=deW1y7)-k4kW(7_P`y3b^c=4O7H}K9)S}bwf zW&ulqAB!}f$_$D&89hI&6M`5c1>IV)mdeKk0@wx*9~K3zi}n$t3xyEoX~{@jQm?aH1HjG!0uo0 zLumejY0k%*Vg6K-B$wZ6j06X!8c{sTjs{*8lt<)HI6KeVTT<}?(V6?p{N zWEqkKqZb^OTlTtHSy;`mSyOhR)8QwITgC3KGUd&F=-Xsa`g-e@(nx#H07garf$gE@N>`em2Rf1gL%KqXSZD~ zlhDYw!=NLL!I8T)p-tE4A*WHrBe6vsN98;OJ65?I7UODUiC_!-wuyO>Ky!eh0^3LS z&DS4_T~%n2w{zr(xuJBklz~b1%>;&?qX`}i!9^24X-ZveXuFrL)N@qjmZVUDgUQ^) zMyJd>OZlcOU{%>MW75qVGouM_!Kv)*_v z^CL?zM7LyP*8dlGmMeU>UBGl_rm4{R69tQIh9Env1uxc1M1Tk2rTxrW;X55=}Ia5OAntxMp)rND6Ggpt%? z54{E!g9TfY(pW-|vq&efL}&ZxH7r!ceU`*R>pKmcRSLXQ zma%)R+&gW(;4_%MG|;{SN; ziN}N^5|JO~NhNHJ5U4ozWUF2SYl3!~UbE+ojsUG>HZDb(SqoX59xys3^z%7RWE3T?70BEP_!V<9>xH6ncAfyE;Z>Hx-*!R*;cGc!mOSUV zDDZFRdJZcF_J#z(cMP0nZX0Ge@=rS`=E7LFSkbQae;B!M5a6jPOAru0uM}+7Jn5w!1Clte_wap zjKfUt6a=Fluq7q1y$bx6)$qtCVb_-jl5z)G)@TbdIdZWaG@R)ukhehOT)W5&Mga*W zUYVB}mlSx_ax>3;5Kv&0nAa$?g6)^t+kLMV{(3QiFMHjyPZIE5g| zbFx6;bH1JL|G9Dm`jj+q=H#&5I4ZUvQKV{tn)yV@V~^NgIf!JO5V+JRy=S4+9!I9- zkL0Hai5E0#u5IKMN)()SnsbE$4-=zEwnCy*1E0i0J}CtTn+2s7`X*khO&;*F1{U~C zc;az_g>|N3V$f@&h}CQ>I|9TMSQOIOtQxp<9xymDe05Ue>RS9YW$}Hv#T|MHM%fEk zb@+PU^cXci;Jfj_MB?D7FSmCEc(F!Fd-l@Gf-aZ z&u6|?If9|wC+K&C16zp0iz$~adyQY5I?wDC^!wNpfk+3Rj0K!;5(G6Em`fgT&tWLG zdcd|zfk)^d-;D>LLRN|Wj z35S@w=gLz2!6k5D}!5N-xGV`wyj|kSX~1aN;NQvt-qUf!DLB_{@*^P zHxAla3RO+raV!UTy$3!m@EaqFh(c)tOYs?akkOvkU zMFc)y@b(U}S{dwfWC@4O1E$TY5$X!e8a-S+3Hf#jTsaJ^ix_M?4hp*{+7&+(D`HI6 z7fUub6F=b~vSk6c)Ixzh1zi8WoywIv$fxDN@Qgt~;sNIlhVn0|C!?}GIL;fmZ4J>( zV-3%i6kg^LAmF<=kR|b|l%fMmQXA8p1ANaE!aut*w54-79pp7jU@~d&lyRsMT5N3K zU^Ml^1RY_v17WAj*xC9HGYKhf@Mts_StPRJVUX8DNwI?yg^YNPmhv3Gs&s={YMPh% z^9`|U^Oc(?^EI1EJvYtJ?C?Hl!oT&sMfZcams8^|esPsCFE3cY$+nQmhJmH;0M{u6 zExm<2s}68DBrs1?5b$zLvQ@M*WMr~`7{Ggw|JgI)jE7=p97GN{aPw&&4`4_#SitS} zkgw|r)1m`O9I7YT&VP7ObjnAxSY^t;bnmGlkA1gzEMQ(FsDG2g$SHwQj5*dMfy+Qa z*0{m1_dqpsBG)-{foTUUWDZCRD|5^{kQ%4&mM)T$y`ER>A^*H)rVG=|r9@qF5=9sq zMP@My9V=LxbU9DFDCV;Q@A;`K%Qin-64JyhZqXd}a*Cc*v6~Rk3FmN7oIP9P(Z1P6<+yf3X1?FW5f-a2$rxXGdH}Evw;MY6I zZ}XkUKv8@LFHhY90j`JK9~SU`c)(wDfLm@M->*e1ceE|z5*X!_(@iqSnmn$ieg}Uks#^uaDvVPbCyLCAstfl8l5E)J%klGo|P@_J9be; zcgn2>KAD3tCmmzwzEXO-@K@QnUq&-nd9F`0ykTK|qq$#L)$m8tTT$Qc2h$Y$<4YTg zN)~9@Gm1RXI&9#0_)G$`NdwoL1tL6&w_o^+{cV{2_W@7WkBA~hrq>SaQi%eG7R=^2 zDDWs#0d{f{z zXfUi|Hd)NTb>jfrrO!*=J*)eiWpu-J(M0AD2H&Yt zD{Un#cbXJ#7Fo$F-Wtikq~PByyIuA2caCEU;#G{YvzTO$F>o6^6k6lJe8p3QCy_^Q z$)epeXPuKQ7cm<(tbf+Pq@loJ;1K`g=m!IZ(yVJI zXNXrk=rxG=<+I>KQk+|oid%J1a)jtEqabGHOS@Pr9^6>7NyO(MKT9Hy%|el73;DSk z*}hmXIw_bha!%4?`22&1&Ayq{N5QSK!FYmL?pp=sG=_e|2W&yc|3o&4DlT4N8tS!N zl{eWs4O*6|WS$d&U+1lIQyhLy0}xMJ$&F#~qI@>a3j6yp2O{Jx7KD z`@aOv6$b@>0{4~$%vbXCxE}H! zQD84o(0O)%&tsv8%vZj;h7Us3AJ3?mNLVMX$w==O?2Vo3vE#ad%z73JhwoYs^#8c- za5@mYutD&UBmV?x<{1Y#elg7c-M|xfTZE-iWI+Sdmyi8_4&5{0w|v#ZGUvjQ!~^Ue z3M?K6EQB1hG+$57ZxEc}=+SwFZPV$AQJbec)K=0vJ9Tc!uRq#X9F(q zV%#cIcgraW!Tfo}XYYT=a(a82BctI{H>*>E&%N2Ka_?95OI_$LKL1?wPuI_QQ;El& zJ30R8EqH9LaDt05?_7h}zeMqvgFF`;@=O}VP9-oi?Gb#VC1ArS@QVKm?;d`Ig2N6D z3|0iwKg!c= z;u|&?@i;RZIWYfAVl-j9weONB^FhuP$$}ruOZ*I z%Gp*coqq@%RQvMLp!w!(*6xH0>!VnMzOvaZZ{GAE;@LAElZB#1j^ZT?uh=_qaV%WM zl=v>kuETlmzuEeV0hRZdCoO-gzJS4K0h3VzzeJ)Si-f?d2mBfd3<~~fu~(CV*gYeT zOFZ1#T};HR~ZGi1}vFj_(*UGd%uZo z(jgyVRo*i;-E$h1&Hd-uR6G28-}y_*^~lxVgu6R?P>Sk>elD(RXMb5^Ul znMW+vvLp52LJz@BuBVDJ0-iVt%EbN>VJc)1*7cqlm3fL~b@)0%1(TdekyHKi4>kOm z@-lhHnMcNLcC|*I{vA9lZXa(~!*J1&o!7dT!L_l7{Q?ggr=mf`UMVM5jt$A-8!jwh zY`nU)(7a;r?t`p1)`xDNcZ!Mm^8HnsYw8#h7rv4%bGw?TkkCJug~uSmBcXkYEY}h* zL+R4PhYqs8*si`|!UpyQmcDs4Cl($)y}<3#yBXJoyLzYS2dFTe;H@{fd_wt*O;E5v zrh!K=zk#EKt8_KPXCBMPU%m(j{Qgip#ZE!3RU|kpc0p68Mp)?8;G~lmLc_IoWQg{} zb()4nB~4rr#;U8+=&m3wlXygRLCU0X?Ohj&P6|p_Ft{o#Vo>5xtmApaF4>)$c|&sk zlb~brCK-$#@`+zAvP;IkF!;x=T&&{2$ZB)Ifr&{mWWz0{wT#cDn2#3*I5G>I8WPwKGqb*Q)T>83EW%9b%02`OI^C~lX# z^32g)`bOGqHraLurn7$Uemo8i_$Ko8q1`Wusp|rNo^WuAPInER7VNV!W2M#zZ*=`7Q61jB8p+~`|L!8UlR;7VS zp=S zX5t_yr`)sSBeR07MF2A=x5L@z7xsV5{E~I*-^;U$n!ZUdIQcB7EUVXLS&+#RO9dA0 zjhC-|n9#_ufI(=DlC(+(BX3ebgWy>Ke&q=Z*d0|AxXviJRz65)H+#}0=oir>JS(6h zx@VDC`2hyD8xK}RhAv<)(`Zy!;Mf&l^H{Q9fz6;{;Sp7)$AbQU?rc9`*vHe`?flK- z)UI=_>uPma6MPyEbas7MXY@=@``4MZ2het^}K<>FkCwnGAm8ZGkw4(vTtT?zzSx8ypNCca~w1ekUEs`bOYuGGSaP(h{HPldg^+@dUQvU0E z-<4W&HA*N7G;Lnu$|`5Ec<&k~vHH6iafy2lutqd6>oF+VRV1|ON-1_ZwKNGf8XQ$( zFyyi>xGXGV%yZ(@1&L!*4hruyU{U<{fpPkw62b0-Bg$@uU6uk0%-$Xm>`VcT+z&ea z%erpO%Q`7#`}2JLp^Z`D;;hmWix`;p#PCJU{mj7Q&?wljgZH1KoIu0Z4><}_0ta}d z1Uq%MD1S4lw1GCfu z*0sh945Kc{ZIrQAWVbfC(tjz{u$$90R($`VripEZJ$adPC4N7(?%DS-Zr|n=!CFy) zjh-Qk%qkAaZ~4HeZqOuoM&sxzriX3QxgH4yL`+rxmB8U_@ND<(%kxeaC|+@Vks#2T zASO3sX}jf_gPc-79N6sw8aYE8*ku?J^ur??BX0!-ZuDGw(WPt7V#l``TT-}~Wddfe z&|U7wlfkGE64c0WqEvawtrM~G6B=XI3JYm?uqdxM$l!1sY$x;d$fWuUu!gg|XGmA#nF(t-ihz=|}82$=XWoMIt>Zb2JW=UDsb7 zP}!?fmHc*FPN4fz*PgYX4~otD*{GWGCbu!6Mbk)Z{{@34krspC^6l>F@8&%e+Ww(g zQOeL*BVYlO#196>dnwG;cN_&4C%DL4d}!9&!H~(}Bf_jP!QOFeT41wMcC)_hHP#<& zO#5egHBA59z;1S&Np^-KyF$TiR)LhmdN>v(SIX?L^X|7MzvRJ{a{I7vg&0vwpLqQk$ zLx)Y(o~#mCx*<;D(Vo2y6*>Z~A0i~}CbE7pTgcyZ!cm^Xu*=>ek;yXQh(g>f1M4%6 z{CXVO^RixsF1{&q+1xye^+QW0TTEzNa%9&T!%{D{51%+2ZZz{SY+$fn$ZlIMdeBK? zLPP$xgEmFoO>#^Jc`|n>a*0@Yh}>C_ZT=*OU*ZL$aez>l%M!`D9))9`Zi>AHJXdbH z$gQ}ym-k&k{)!T>d+oM1Q(XBa@7*yinD{y;{v9GKkW%E-evcP4*8BTr(46Kmdv(x3@S0^SOfKOJ&lRrhJ+3NBz0?K$vUcgkn} zvWzB;oQ1r(JxNTmKb)047{krJDsZTN*m-99WQVH?%cpg`vYhuXJ=0N9>&gPV)k{Bp zDS2km%hwyx5*pjvFqfC5iLYb-#rd6yYwV2$T}p%*bQ_v73>%s9HgK3v;E0>9z3@ZO zy6FuK6F9CcD`$4)TR1uDi38ue34Ee2%GW-PYErN8T9)$Npkn<|ev1ImaehV#P0`zw z@)Qk174y>!^Z$Jcm>n)A^i<^kClSE{M&k!;jT1OD7&3GXnB@yt6a-YH4H$1JMzX7O zr5UmQOMJlQx1f6OC37Ev*3t&{@&isjAK2RrIO-YLL<1OPKd_b*v)cwR={;bUC_zEJ8VTB}A{oW06ON*Frn%7H<3x|23wN@_fHj@ zA}$)s<~BP#hUs#p&jlTc3yhKv808IEQwumF99q7GGHEcdE;Qh3e!$YCz#4SH+|q&B zw1D|*#lL>j1ZJ}j%;py)8=N`H7ck2?Fk4PwluZb3eo$m|A!72Owy@9Yt0rdJJ#XVS zoA|7}Hc&Ze<3{hs7x7gmq%0EL4Fi~WOycO?!0!B@gRMbb(o8_TfZ_Q;0SjT)BO5pv z8>B=6nC3-fZ=NJ{iGkB{CR5J`p4j7#XCAQGZs0i0!1vIAZ)1m|#HA^rCR1NFbbngV zz0OrH@@2QI1$)c^RtJZk;}t!6LDO_Ui12Mq{ahp#y0IyJtH_6F#7#knPY;*9r_<>pHLR9gJ1w- z^#Ps>3@pq73~USx3XZG}z|8s`vnrEH{xvdX)mF%Ci>&*Ztic_?B6op-v4KJKfH8jqgIhbpv`Gvc1&sGz z&ehA5%RRu6bAj!W1DDtVHpd0*mo{*ic=Bi{L|zGFVoPLH|G>Uffw%Dh%aH_jje>bT zntU5yOnIy@^>IRDkJS9nFNGg}D*yg6YHtRAX#jhi1AEMZl`#sO^SBn+87#1Pz_cW4 z!KoWed|P|?uBN7#7=%bI6fj^ZX5jE%Fnz%U?iUyM-ag<>-cTs>iltS6x7mSJRbiQK z0kc^EBZ~q@QHej5D*#XyWRh49!JLDmB)* zZP!@m%p|VByg||1|CaZT#(&HX4eW*!W|x`@@FXxCn#kb4fn&)84ha@!RR)IWEGFqK zJV!3D**NfYGBE#e=e@9i{cXQ`PYHpp>XQA59Kuh>?IRcTK=9I|8nJj$!QJ`n4P3oEvo74dax0bI=oII09#{VuRK1TcFv ztFtsD=q0e%H^j~Jl(m||=p(?PWyTzEfho~7bNR~&ciW}wZ!ox5nYdedpE)>d!$*x> zqNXoCwhJ_VK>^Xaj`Uxi41B|i{tXvpa zss-3e8@XRB*y@&0`|rOY~}$>_5q7>vuAl@v!MgC9>WPk0cPg|tP>sB_?WrY9^hV=z?Sa7?YfBj-|GT_<&v#s z1xpxMwmLGb-Nvvca|e$D^Qq-U0R@al4zn0PNLr-0%vF5J{a3|p)x~bF+J0W1_59Z% zaR;V%T+)%uhqYxU3K;CGb~qTo;CDsHWz9#1yARn!3YeE{U}t75T`E@3`N4s$fGIjl zKvIl}M}Ud-pg^90+)@Xg%MLuY4U99p=P|1u+mKP7^ylA{u33B<(T%|``M$m3-(g|( zgJ^@w0{O>et^#zW)4~%ir zhEj7m<=1jITAFL6Ui!ODPUL}lmuIH_UIyMCby)}I1|4S80LCLn7y=kL)MrjGx1Go- za>&tZNl$fXrt6i>A*UCp1?rtqpBgrM^+`s(fN1Uy%smENN?Ukx96~t-7&^bIX*^>q zZ9RLBr#|s-$J6jS{myH=hKZLIcxR{PlslB4{lN2;_x=yrfZH>3%VPO%cji7Q$Yu27 zTdCUp&Y|(^9`?L5?D;Y5H4j)VR<9H~w|BzZ%?=ALNYp(Px4X$BJMGJ=oBbO&Q(P}#u1EmzB_z!1@?Q0^DN;6K@GGMmGNT?hJR&Ym-8kvS)$c_(Azgt~w0k`6l` zmb&}))bg2`8a8*@r90hocl>K@-#AfP`_0LWtr|QBGO8I^%osLoWs6-Rz&zQLG3P?* z6@`E^0ZdB{8M6!2D_(fUUC6+%!1SW=?2E>0rT-q4DX<0#a9p}zTJOMh^#IRVhS;{c z7Z;xL?BnF^mWy&KbGiS4=l%!YryqD3?E)4WH?psLP*HG$=iLLrc`M@_j*Hx5;hg*M z-t31JI$PTIm>Vn12yswx3sQM1Ara(o?u-DNWI#r70<-Y}mhua%Rw>M-2Ux@ljCE%% zvOdR9>MCnJtE9J})b3vB$|n=ldlD{*cTT(Zq%7jld#`m7b(wOVha5g|JiNdmJAorG z;2+bSFC3y9FYzZZn?GaATfpwaz+jkpEqN|O{yzrx50i@H>OyiD_3N0W8kkIGa;PS- zZ~eft^@6{E{Kr32{6+5jpL5_5=Z`w_fmh}W?@@!73g>g5eqf*T?&Wpq<1@W3h^%9E zTEOb?;8TL-Sh&ZLzxCdGA|tn|-!jIisBdOT<~0x=<#c1{RqMw{9+E4{~6<+QRhy$g_{vf8_B! zS1MpA>64rM@4SHe1?ESLg%^{`?q1~C`k-n)zvankE{pwn0$jNKzqoj{{PJ3oz0RNK z)Pii8|GXCrURDcm&gr{Rz;pc1&dt#WSPj>+CfrzoPV2;k_;s=e*?EHK!1p-$No?l>F z+hveta*$K4f$1fui0eP*R;$H1y;)y8UWcuV@ywNF&CxdSIB9daYh?u5rTV$I2L@h*#OF}Q9foa8+xdDrq6FCkrG|L6mGucj>$sr)DS#C6$i`C=8 z95Hdp6d4KGU0<3m&NnXj5u?!D!6&J9=0(5)hc3dCAR8*-4`;OoB8>K zO^mHqj54dH8F45!&TSETwVc`I0;l5=Ig5vk!d@GiCq9|feTd20Es=AxtHjL$XAg-+ z6&FEwl|YwGXPDI1ZkeL!Hq-W>(!$q#b{{ry_wU(pp?m#4CgDY$M`lfE;XceUxy9o+ zQ?$;;BPAX^eU39Gx=Cd%=~0`NxnkzCnc7SDe3EpVA#ko$DsP9PszV*a5_7klhz2IM zHyPa`4sR0LX1Kmtz`*G+gR`|$+ABlPic4%m!U7hK4S5SV7DRus?30o@!N?{s>566B zy4U9v-f6Y2un|u=(7+(=rs&FLJ;9?z(wb$#fhOs9EG-jVIx3r37=;;5sV{JnFj&Ft zcR_RIJfRKAl8zlS+h!z5G!}?l`NrU4@leb&MKEjq)@xB7VudY&?g~G@MHeUvw#6j2 z1PZRT|8|49-#X(6^ZLDe7q*)mfg^PULm>5&|t(_v5qH3YA<=U=y)=u+TBCUR2-)yOV{2#*SvE58zsJ{4J+l34 z?O)wV@y&${P294xejI+7W0maY6wSbSblTKs8jAmzr6pEg=Chjb$=BsNEpbWPyoCua z0(}!-$W~}dC?4%(Vc}qCWLY5A(ZH_bFrUdqd&Y|8eXF#8UFsAoU|7_qG-1mN)^GVM z6gjve90VIKDR6TfVKdWcY;iGWWMpUxY0wa1HF9W_xsbqFHiMC6n$a2-oukd{0vDJC z3YvCz9%$jTa^zoN*qbBZ?A3XKL0al!kLv|R{>qIDdA}}Tw|i4i&|!TZ6kk%d}l}k~&7Kf@SyrSZXCN@+QHdqmeJhfLYe1Lo%JmrDes( zR$c=|9`hLu=XDyIghb9WiCl145z*SnrlMdjo^UuzN#q}^t%q}HdBU+wx57pig$718 z4hHT!qAhYC6u1IsG%~OmX#UepsQHr3!DF$Lfu&^AWPuNk{x+KX*d2-(xFs4GZnHDgf@*O4>@xy(&KI^oR@y4$Zx$Toxj?`rS6-t zU{}MPr;nT#@4my~*|KvHe@z2(o7t2yiwQlV8-M8QR|PKoyNHo@o?%V%>uTn<1qWB@9bo8{Il!R8 zz!3DQfn6ZOLHM0WYtWS@kwZGIx@BIoECUuWmI%3;ta4;#;&{MlK0`5J>B1Z79?iU* z2~RkW99yusVgrl$hE9Sk*qKMTA2et(_nZ86aFmrBTFtJtW2rO}|J-?EH-{T`s&_7!x?fwLoIz>UQ zeS(h#6%|yOm>3#3cX;!^_BtYKb%KGRVCyOdgNATL0~U4#hC>`bR)sTv2<3RPkd2SQ zf&KKU1}2dN1_6O=fr4WVR&O4%OKTm~U=!>LU}>se)%95Dp8-nFvqG#@`{3+^bCW;4hIgf7&bWalqo!u{m{UXr`nVm zI{TIhiz4s3)?(>T3hC9ig84fxFx#_#Y?r)vaC=iDPf@HvhUAV0=C)~!>_WXS_(ZJO zj1=;mXIjU$hgo^pXE3sv95Z*w;SgwTNY0ix(C8S*FpI@NllRqy4r`Hr3e3C;4$M^t z7Rq~EFnzP~0E@K%gN6e?6PrS6r|JbJLscf1o&<+Hz7GjJ(GrKb6Ap$p>N2jcn#x+e zfSGev?KQRv2PV-B2V-#uCbp6WMs?o?wjJfnj7I_)Bomx^jTbPg@f_lBQ8*%==Wt7r z`4JP7#YzS%2KA||hbEk0cJVs@-R#Yb2HxX$nx$tPWRo;w7BH|l$Zc}2N&Ex@n|=c$ z&#^tslNR4RV=KWl`|^fQW-Au*>Q6VGo4b2m`A0|h36greF&ADa6)bq}%20J7^g=P; zk3BCs?7l3Bb%=R-ih+HCn}mGP1tuwzpWLT6HcG!SXqQh}z!bmUfj!27nO~scADi!s z2EM!n&GrqAY_bOq3hMrm2vt)1*E^ug;dXm6qvZJ~#l7W%MVF9zr1ty6(0Zj#U z9jxiT4lT+&4^P@he41k3z;Y%kOi@7bz@z2^a%>M9bxjVj3UD^?GfZUWH(1DN{KHwp z?n9$=z(G!V2d28Y7uapSH1Zg;xvZJP#p!Uz;dR0ZN7ff2T%w?50}lU=eOEMt->Jal61O;0 zZj7YKEmj?kriIMAJ~uQdWi%*tG%Vn@TKwB`!DL4M35^^*2mCv&?KU*4?_hRV(R}mt zzXq#K4Lk=LI9D`q3OLH%Xm;ArtoEWgFrh_1g2|Af*=|R(y+ms|Lz5&!i`x(8Gail# zCm5|ISj7G`t4XxDpJ3efh&elh?O|e@+5%>e4a{~5t5bKcO8cx8)U_dqY1?`w)y-9m zLInma6Bw8xTA~=5-2~YABpU7%G;O=wWS_w@C9#G7Me~Eesr(!*jt)(H3z{4in4Jw; zbRXylH#7!ZV0M_mmd(+glh9Cj%5N|jBhfS`vZBC7+8c8nA{k2 zg$x*lemIF4F!pKNPjI&XBD^zPWoN{lMv(`NA|8!$9*m|Qjb;(eKfIfjv!^#55^r17 z=>G39gBXJYqm!Y?VR@#5^En!hW%wLm2sx0Y!?gTyljVyh-k(hB8=4IYn)C{q3VE6h zZZsXa%&7lhg87~<`v@j=1!kuMT+s>aZVwtb7+cyTTevS#Ng!1Q=Q;r4ucFzFk#L7Dt?_N+N>9x5{erb z85kHfm~;i2RX;HC8Ej-Y!FZR4*(RciC!@*X1Vhk<*0VpFcsiz*t!{9%V0Qk%VyMAj ze(>Oq3YPT5Hn9e#G!7PvR}RuU8l7G)U=e6!m}$rTqmh4tg@Z-oe ze#$<&&S(FZ=6|l6rW_UNI2x_8Q+c<&^ulJPpNvns8pS3siUl+>@i1nbIej38VZVv$ zKgouNZ%%VeXplL2F+HVWGxH%;1?K1jt%|Cb6xX)pda$c+V6=*0G278-^<$|_M3e3h z=0J@W4+$296AT(#nM+w(HVHFG?O+a6V6pkp5c->uDWXA5f;A?CEs&#G@YD_ET}THF|z`5T(- zPc*1TFloMM)<3`?y_kXVLWBK_=5Jq`m;@NuJQN&eF!CR0;4?5c|6nKbVWwn7zq|v( zG7|xIg~pJRYy}hAQvM|}OK<3LvtZP(W(-x?-+$RFZR-V|ucsz0cIDHx8G<2FUuxs3$eAa;V!(}e(o2(I$dP|&`IT#fWFkcXC_L$J3yMWpH29v%3 z^D7o+M-Dq{j;WjlTT@RRbbiQS@PWx-2eZCH!x|+{2%Kg?$;z7XP-lO497$qJs zu1{h4UFFR&(~c;?wI<3mDBjSj-*d8^HaBETFM8v|*t`19L#bOIGcdvXPxr?oD=Om>g?n@7fS%#30ns$oGMP z{{o}b1xCIdvt$fT@}FSRuVAubXyM}sW&FS(HKW0{f`!YYfhQqe_Ciz0D+M8o7S|b! zCyIo(=CUOfG<$1B%k5}nd2n?n=RZd76^w2fY@RA?jt>~6HJsB{oaVS{A>?pv7pvv2 z)N5a|U4=PLuNT?5Ma44ePNN$`;DwW6f4yU5K3o%FJ+N@|={+|6r`FD25HUsZ*qzg{ z4W(E2{Jpv4aI{b0kCx~ItR)}XE^SS^^tCOoqiyr<=Hr`YKA*v`O1fcUjcW4_M(Gue zCS{Bg0+XeFY!qXdvO&^-=|q!(2D3d!Gf#pxb448IiVd%2{qi0?e);wC%cXG%-O>rS z<9N8c)-yG1F^(7KU}%eA_;R(;JC2EK1*7$g1|E%#JUIRktR1XV{xn-%=x_=Qc6m5|$<6r;0u3x0k2_EB zw&tlZNchcQ2r~4z$G=1J`Ps{E+v1$l{Q93gp7d1BF*b7YR}Z!YyFOiI^#8P2GJ@$_ zZKm(OX2}-~TnieE*D}~Du!%jGKRu%($BIGd2D96YW~YRfz={^VgeGGx3$q!G9Gp#j zT#XDfJ8UCZ%q^lF5*|ntEJ$r&=nH-(p3xZc=hUao0spf1MNc%|pCP(4hwqN(R(reG zy8CWi6Ef8k&tObha*W9&eQ(dj3?82{-Z%SX&i$%XeXvAcVg+-M!?NfTtV(BFkKenV z=g^+l!6Gb@;e2rlL&H)fwWY6j%zXYqW&Sti$-K826c}VbusB8J8dUT-*4-wa7I`4}3i(#Wr_43(t|OiT@U#KECy|62qfYcjx~y{HH42``oBbRdPYI z+m5zqhF1R{ExgYQ`0f`({bFhoD99J zE%q8MEya!YD;PB#HU&uRJ-V*3eC_-loYDO|m^D(`TIaNxzMReDFkk9Iv-^%68%i1C zI0e@1VcfE2@kEA})QmRK8;znDPVM5|Db~Q~#_qi#=Ha$^MYn=pTw@+H5Z#7co+MIPfB?%b$7mVWw@jVXY`8?ejh&YzHjkg@inSH$@w3Lr<7vq zjnEFZ3rqfLaui?NBQ?G7T%2=T!8r}~2F4c*Cmt|~XEd0m6e-r1eiSuvdEUL$+ayIK$=% z-HcoGUhn;Cv~Aka+_ckn9WD26asRcJXp6Fv)3Zv=2~2Cb81(*hvt9VR35<3dSfdSC z18x+A7_i1XFNiT`jmc>7_|fjVqgiGLqs9zojT4NF2_MD$wHPu!dggs=RA5-e-S98` z(VOQR?s8}}q}^9xkf`3qWH_<4x-dMlKa8PACS>z1&NHSd<@e@?{Cy;4Ua>}>ahjo- zLqfBaLW_(FYbHn2XF0Bg67n27n1do(+*XLKmt@UoXxV5v|NK0Wg${=71Q)04pg$%6b`8Uus|bhjbgwH#cw?l8*L>eu2wyJ zBYi97)CASyB`*XP*%Urn5@_6VB390>hUeqKW)6Ni>xu&kcaq)tC7Kg?qD(R_%=Jk4 zaN(noyO4l_MZu8+51rV!#bml#8Wu1yWla_gb1o2QTzHaoef;jQ0|kuCVgXV!t-f+z zRrjB3#ro>W%gf9C=KI|3GCL8q);e(s%O+-4fd$g$IR+;k9Gr#K%sC7a9vxw2l*?OpIYEvzeslK5@vDas0sGEH~HXqO+itixBg_ ziJLs?9W6dM_%hl|a0nL*a1~|Xv}tf)a+lrE5U8aZpcKl)VUWbcVRPU>8{d){4jkc@ z7owQd1HL*d9eHu_kirt1;B(gZH7=)Eo^sh47TvX{)kCJ_!qj5|F?=6YgAtOR~3oaEy9~!VTZV;~Gm19F`i5X$plMnfM* z+mfm7wn0Z7I8ZQVY1ZnB88u$@R&si{b`>9-7M0|^`P)A@2=uEc9b#5vcyx@};7#DW4*953hhDdrvqL)>|DQGp3;!sW$H6Xh^)a)$ z#hDMx;tWAzHt$2?_a42uUwNsKw7POnw0!#+_i2&K3XOhc@_afjTfQSQGPUSY&2sLl zcPE`VT>s3JV&UEK>{iwUHixIRE~)}Q8Wk2j<23SVeJ^$i=GZak1UiLs?|iuLH6kh0Kfr2MQ!STI%u)*m<@rV5w(d zVf(PST`2d6GWP_JhJY>XK`aOV<*Ir(aCSJhclNAPnrxlK!Nb7Fyd_c4V1tA7iw{hK zwhX=^9t@9tQW}}x@M%gPy6)cBG*xONhr5;1$-p;Pl`UTtI-4g+a7q+3I_faXe6V<; z_;{g*tJ0;ao2MnB<}Bpt3+2>PJl%Za#R5;OE$3@1RyfOMm2Li6sK#b=V`fIpvL+4= z-hg#6z07~oHLi-Nn(;Io;1$xCvnule)4n~ROF0>}COizhZE!?)*2NBk4O#*L94yKU z6gc>=o#j&r300VpDC`!X*cPc`@ z;g_bfPt5IYW?rhRd39OKSwO?#?+bUeiBGdWM2Pe{Enq7XXw~F#WaUjbz$E%Xl`X^J zph3kVk=q$9y0a9ytZy(_n{8lLX1T~J()NhkZw0IAw2STAt~NukeTaEqOD!UAtgPLmV2HpD%>0ws^2*Dgl$x06pJ{bv@Jq}H6&0)vbKYf;X_B- zjRpqJ6$j{Jcy^5)Exw_T#TIQ~ zRXlQ`ks~g_TK7N$Z%#uaSBe9>U%{aYqXapg1O|bYqYLG>R5pJ%XqnF~a3D8i(}c9h z6lIPIeuhN{c!VrIvMt%bTq7e8a49Iobww)!!?eP7hCdC0-3@GPOaiQ#55lA_F|^CA z*yMKa<2z*?C9ZD2>h=9nJ(VX-U(S}luy~S^J_}F4zf8+#j{UaR7oV^DS3Li5VE@}H zak+=0`YI<^^Y}L@)|s8|;eXT8EcN2d3Yix(=Z96!;#l)(>bepwhS0@AjRzUH77DmF z8DC)JD0sjm^WznJn35-JN3)(~!99TInWV8L5#kg+i!*Z+REaHm~ z@OADupv(9{j!7qhfldgQ## zq9O9?L%~Z2)Y%e>*fSrCA7dSMq`5Bmt$;A7qNyzXgxLeM?q9xdG z@lv;d$)?8`+w(b3n`I~QFa%%o_;nh>-IeQd1C!fB~J~i&=QvKfe zCBHm(&HuLLjPu$R*QO|N1$;gtV&cHaWANju=#2(8oeZvMi{)aiE1J3g88MuZ6o`Gr z`y-1@#Ol>r>t4Y!h9k-v7gk67&<&=%DVbQgatm}-cuR!GP0Ft zUdU|En9mdDarjJ^z2anfT{ZOW0)mH z4yj*k&|j;_X2Nhcl!5I|5Hm;50i6TYZ@k#=FlO9jWc|aSaHV0_-AUY69C&Xu{Nvwp z;PKvAMzhnoKO9)Txhpxa@?AaRYvN?B+T<4LGTHSA zN&~CQL1PuBt_3rtCBqL|L>^i?^S}?m&j$VSCnJrz4sh=9{`}1lv2z zX7`u{^B$XBi<^J(-m#bWluq)<1#rvGn3MK|=k&r$BC8l4Y;5?Ku(0H5f6&#(b*x+r zrVXAt9GnJ>iZ)&9QyhhI7%Y4bu`7a!l z$xxBKaZrYZo8Q5a|BEc!mjk^z44iKmRXh?{><$U@9AF5EZa(lNbt6N@Q3vjb2JQ_J zJa_&z@FzI1UvZFTVC~Ck*`C3|q{3vpRnkSO;ze)-uDVv{5HNPi!{B7jy5I32EMwK_L zr$TQn-RK~CkDt4PfqR03AcL>XD}L>bL0KnazL;LQaI-<-M$?s&L#iQ+qA~{!CLSzY zaA@(ILnd=>2w!OA4LNYRcgEeT6RsE@;MZ}|)L}Nf!lbF=%%$tZTQTjTjRV7qBTOX> z0v8-LZXP_bp}{L9dYd2vg9`%#PXjlPAMYIp{xgldI~aJ+Fv`7f;Im-lec>V~Gtpgx z-}43=!wp80e}7o^J35P~9AG-oAoqqbV=JSN!I3T*hr-0x=Z>eJN%Az!@aQ|CaIiz< zr7eT;y$7#Yg=B4*OkPONKf2T`OWE#8bIUX5mP@L!RgdSNJoBT%QHEze{~Lz%H$0+I ziWztOJDc3N-}rRRo0O5DC??jJyiF)8f`Lt^QNg2E^N(ABMtjiE}qq2vyMTRrG36t`PhQN*^T`kLRxi(7cIEhSY*l>fx`OITa2Jt|VR|mH! zgo(-Ubfrd!u|_EI|9f?J!R{7^y}HfT1}o;LJ{Aai(NdPRJZwelt$Vg>IOPM5#T|2u z7dWW7R!g*rEo0At)$20SFC0j_vMRm8P0Ke?u*dDQqXS1t1d}b71_Qf>%JB_;z8mdM znXohozi5#C4*xA;a;Lsytf@ z8|NDyft*IM97c&NhjcxheBv5p-yB?PdRSJ#K_QFLxPaNZrTRRhCPrWnYb;g$<`S6m$0;lMO!Y0=D(BWvgFc3|L~!XU5l!r~0`vbe=D zTnk&=GM2w;US5~ALe5FW<)71ujFSQ$Oo9zB_-0&M#kDM98^ao*(6zrD(ij|t4~PrA zIB@w<1K$w_-hxc-JN%p_4ZIVkIyE(EE;zJNPWwjHTJr#Bd4@*W62|Vc4GK#FlpGvy z$8<<<$u(Yb*erzE{7#d&2-6vK)bHk{V{*?YMEmb;|-)vB2kra?fk;EtGQQ zm~kS)C@W&tyg45|JS#=_t-E$=mvh<4d8hM~mtWI4dnxDarKd^{a!#CSRXM^a_}~E7 zidU=n-mX&0<uUu;2^I6Ls@OH~92Z=MYmNh)v@Jgeo zMNQAdx$IG+Y>5Ng7Ew(WCk2i}CR|M-J+T^BoJ?d4%tV?bcn*oZXkfeQWE|qG!sEdF z#a}^6fTci_kEh8jqS?swuttRo^M`{9FB-%p4#|l$cCkW>_9=_F^M>?Vt*m_-cMg?K>A#cr_T9NRZK41S&L2C#x8X(Syh~{dV$up4#kUm7RyfKwFlubwD4x+Me&F@D zKK;2@Jo%n5h?bmPR?$@OWQC@NyP`h(Pfuov7YzU8cs8&Z95hLEFJ005H;`R!!BpKV zOnd${itRWcYvOcHtV!JE0PAF#+1K>3NbrgqY$#h7)wRvxYM!I8 z%|QhlC*G7tsj=5OPh56aNf0Z&k{X-pE|C?Mr0S@7r%vw7@w%M1^Onjj_4ssR!G)ct zWptd38=Nh6G+VIi$2h%McIr)wT)y3kVlnR*Q$IAyoMHXa_L1*O@z+xaE|#@jVq*NU z$x!x;kNU|T^_?eE8X6@U9HsLczi;{|-ohvH8gj}z4)U5zQ26tn-Ih(R;h>PtLD>e!mX{6e7aC@N`Nzr3@`%;>08;^jY=g6I zMH}CP2L1yXK93H_&Uh(5?RHhi`e|`3$gETkgaH75pgK-ZYbD2_vE(#?kfyD6B^QwFmV6d;=s4%z_c`l z!*<-EFpO(u>9I}91v{v3$D%eci>R#W(Zrg}N6wt0`6Ly53?;rh0%U)7kW z91xCiV7YV2ueia-EpL0l5uOKSs3a{5u)v zU~I=I(tdcatcB$b)6-##KRh!%v&dM?-J-?cLg_@~^3#lM=@k;E8&)Z;O?ti~efP;( z3w9`)9J23lQr=Ou=6~~Q;ij(t%&EdoDf~^dzA#GEII!Je5WaD+@3w>R6KzME*YZ0U zH-DG12tB}agMoKJLzw}ih)d&!*9T?Rq_IC?;Jwf&cZEULz}a}iC)o`S|CjvsUEPu( zWzJi1P`;wr*kSwrYX>znFj&ws7iY_#{@q4SqlYg|~py1z-|&tc^YR%^wb)+nGds39mhk-Gnf$u_t?1rhb6PO=tayI_) zTla2BkuwAHmILefoF0cYn?^Vb@;ERG9FRA0X4(CB>bKoqXFS|B-gw4@-7<8!rMdZm z=%3CdkqgtVyvVvzv-{or<|%LA`I%ln&1=WNSy00nixoQiE+zm$K~}^E4r&B)XfBV;*)oKrg?u$-my#ZN|VMNrqAa4Kik`1_-&(o z_?H61Z}uAwlcsZ({=2}Xe1(x$p;7kBLEfC&D-$J}uf|@vI+vGWyZDWR;uVY{Pa4GU zG;X$YHr8P-`Ej6V_fD^Z29+ZW4i7S~t+}>8u-}45BH%#tLbtX>rdk3E8Ck@{#S9V} z5*nG=*?2Yx2rW4+BVU&DV&%t#gH0?lvIQ{>4asL`a`Rcdc%h))G{auF;>b&d7tQSK zJxZooR}MBW^OiTT5NvX}DkL8Dq9NqeER*c>YZ`xR@y1vbJwCQ`wz+>?bk&zvcPDF$ z$H&(EeE00~>hO3w%lfA)1Z`$`Br4f8cuYF7luy=XqLSpKJ1^oFM{UhZy|!m&@$<7Y zxOGn&9A*=-E@R?ka_JIO*0lWh%3#xE#lx*!B3d{0H9TZ$y|+94#>M32{d4ubtxRry zPUc%|TfxT5#I#W9D4VFF!;Q;4F}f!%us@s-u%Ka@al(-=7lT?3v$6>k1qe7KFf`BR z{ALqS9>P!Ou2f0f!9R!8MZ32N(`da-cdKd4snALB;ZdtOfF>`V=UtYx_KC>A!64}(E_B~`XXW96; z^Pa(pvrOCu2NKeGShgzcaAn!~*u?rBN6JKblPraQO^g-~4(xhu($v7h7V+Uwck8jq z4o%D1J$!^@KO8vhX_HsBNYITTa*<%#n-{(fwsIR6_sOKaNNAIu!MT)$w_@D^C5AyF7pdzimo)d&#Syw zm3@5YyN%rQ_xG*&d%~jNyAofUeD}G9TB^$;&h(h;)jihhHLrW}c=68E37kd`)-o?J z3@~pN(mAo9bzbdahU$6M{~|B&TuOb!XUnkhasTJ<8y;s#HZU}?d70mxa{0U1goTWu z+z%Eoh?h(dXf0r1mp#z<=f>eiE}Mg#r5Y}53mkf^PPj9aDY*VqK6H@B&7_IbLxC;Z zX<~<~PLp8w0|sxqB#Y@ER5W^v8JJ5ZG5j_-r|h?JVX)r?kwqmdqLsC-Gy5#rS+aG3 z;NlI(1P@;l-6&&N;=4t>_tGQ2iX+XsYi|eIR;f&Hx~OMR)N4XV(0u6c!=UKEa zH1a)Y7l?5?kPikz7k`3*$x(sP zlGBSL<%LkMiK?<p2xJTJ+1EYKAp^Z}fp>fLZDjIo08kn1| ztZ-v|D*cI7tJ>~yup;kHV>XUe!E;xgXDax>QRv|;{ql!%wZemjbVqHbz-$Kbtiayr z3?C6D34g&D15Mor2bpDZ9ohF?Xyw=8$~4~K!1UrnhpkMC$YlvvHoFcMa{*_mz#YuS z1_p<-cweY8GcoLxQ#j0UW-8O!Nop2vo%QzWu@*gS(eChQXj~PN&(+|8b7@>ba`o@cyB&}P{QtJQ#g~LkU3OZaCyyvSU|AY`U>0CUipg-kP#EMfH!q-N(a=w34`kn~<}a4^ zuG^F~dwKJ}Wl0UE!oF>HQ*4knzRZ@s>u+zqPQo)Tg9YX(X%6SB+!}Za8W?05m}I*g z=FWNcC2|3aQ*mK|))Kb`^)jqXd?gC|#hxT^n{+JYle*C2c*1C{_zMQje-lnx-%}Gg zvY}bp$+6cb*m0J&3Rmbi22M4O+;oE-47^IQ%=`w4f~PwUtu-~$TYO+@j2hF#b*@`B z+r>U*Q8;!$VVX8yVN3Gc6L#M=9=`bIh?}&`=S|f;`xfTz&6bwBaAJ4Q%L|V${7V*i zDz+)@)RA}5GVf2kOM7efz~X(CzudjM8~#k5zvrE$BCO z@8;bP+AG+Q&~$dDL!;L}UPdPW?gnPFc@vc`FtE!pI0&v-Jwcs^NuXMwWnNUmQ;91J zIjuU*v6p;kSKINxOm~LkU5%>^Tw4w>^=P=BXWkh9XW#W!yNwM-`3B59S01qGt~kg) z$Dku{$3MO$9f#JLbGq!9bU@>edN@ZB@0=s&4tp(_XrLh2mHYNe9^+KqqYHRtmaKiZ zDVJ&eTgf>K-|JnF+_&)k5e}nMCls}slkybrPGH{}bb9L}UTLj-)|YEpU-C-p==aTH zuT;$v$1z_cJC_e}z46ayE-Lbjv>d<_lES?7b5 z6qLmtFhwk2WK&>z(adehAduB4;`VQ$V2%QFMgx=815SoSwnys)XDwhjrN$n0V4D-8 zK*~B(iG|EUjS|xqio8=0nRk$H#zD3vD>eBJTv0f{C$Z3yYYn%=2~LSbks}M`_c#dL zZeS`o+rB34^`_+OyZrt)W%zqFFnAs~I4_Y=GT`kL-kS#!r4%;2Taz32DUaD)fcft` zriBlgIA4`JO)zdt_YOM!aSqSDkO`k+CVbLcFTLmCryNH91kQddtis zeTjdb@`E!y(uNLPM!EuT67m@w`4=Vd*DT<&=Zspjfsx09y?}we&OvT=gUF0V$wLAH zHx>v!JHQv!&z5n3Tk0TlSHemwhwW(#1o>R0{w+$BxwMe!kiWne1|g4wd=nn<{pr?- z%V2a%;TK~RcsEf{YylIC1N*xJVn-6?<{aRj*2rrR_}{8CregxfVOO=>Nm|F2y4dxw zcr9R+StA)BD0-9cos&9~`iA%G&WafqXciX=C>{3FNjA<=l=ADY=yKn>>fyZ*ZMMl9 zwe|b&?P`=h@aay*{QF%f!w6ZGm8ek;YBEn{(cZCbuwc))k-o(E6aT z%|b`XeZnl7$s%n&A7x%Gd~o7qLcymA7qxdX%I-L*t*`Sc>YMTpR|1)Q%I{PQ{} zwxV&yhi;Ky$s%V`kp+fS=1zWM2d4j7HusUTwe5TXY($FMV{~?_YzzmrJq50p;0= zGDi;jGz-ZbQ9PRRpm?avgJ8+z4N~*kx8-dJa(vbAv1&q{@|l)^-yH#`a})v>BnaMdV9rZ0 z{jz|GEn%I+yTG&rMwWz!r_MZ^w~8|_p_wUBGN)0{?g1OqLpkX$Os^6Izx1-PE#wo? zn8uYTB5{zvb3za;#d%qTG_ zQDhy<$4h6mShsF9SRWudHBk3klJ0uD-9n32B`QcS_`L2+P&xM*k5vJi0w2CQwcQ|r zPi&!}hoZm?g({s4zDqp}ObrY=3;4b*sGY~ak)y!g(7^rfL2lMV=8j(9jz%eohv73B z1(pT+roCPn(spK%f>&g+2vehA$3npv1&vLug&R5xH4^w@-Wut52+A#tbyF14nJiiJ zZk5$Z4{Jkf7WV76O8?*c%H*-&jotyXnGbz}3vW)-z4dD4tu#Zwry@2J4@yp8Ol)M7 zkXh*8&Q*Em#fdwUlXibfI$BgLX_|c4NOJl{=~WNqGZ?0=5ZumTV$P)*z_GN{U#nU~ z?_c$)oW*GjT>l*SllqzR81~1kXTA2CudacYEm2U8QDEKy)|dviw)0|J7D_s;5zIKi z`s{$*S7!k`pT-ymZnnl>83#BydKfeoJYqP=t#;s{9s}bdB^H@>jiN)dQeLp-G;q#4 zz?PN3R<$5?!2`}IfLn@KC(&pkT|wq`JU9m2Cme=8{_<287Qj)-973G7Z$_m(|nn zTe9Z!w_GXqO#fr-9MRwXh2EJR={a-8N4chLg3bxwo@Et`5BaaKM@cerWi>MwEf6@h zL@=yau;`yd{#yniw?r9zg(+4G_}(4c({=vqj0Xzh5Bc7#i8N@^`r)t4&eA4zTvaH6 z?Ut(Gy9F#ijJTQ}%*b(IIB}-E@Ac$+i?a%H)AbHCoX&h*ag0U8V#ALIx!AD-S$7L{p(KiN-XsC5X~*rM1J-xXiy^g!1bwA8v1n3Z(WdDo4_c; z$(+K#JWnfYQWu-V((pvKR}1vaZBE*$G)z9?y0DkK&E(RRlnk~AW`TK)oHrUIgcMt< z9Hi0?ESP@CPN-qBNR;xItT$B;SkfM_do{58F;DuwP0Q&4(~ktftc4|R?C+V({ERK8 zc;8y${8Ri>iNvHt3BSm`gjxRr(tmwQy0J`Bv~B0Xz{`!>#II_vT_|~A!IN(-{Y}~d zW~b~$XU0^_FymSp@M6;$rgIN#)VQXZ)h=-`V`P8hHEsq#8o`RqUqtKlNo6fo(dU3$W<+v0>+2pTz z%m*h+I9V+bpT5PjS|T8Nso=qisNY+8LyOaIF3aCjoV~j^b4T0kh5Sc)%vm(qZN2RM zuetn~!TIPqpB)3&iU#v7{b$;`8Si!1%=x!KX6A)81~r-QzOy$xU>9KKXnDXXFqvB= zF-&6}V*!J0cA`MZ0>KqU-1qK?Sp{?FEEID|WSZ6>a;9r zAa=~)hGc+^#FR#f1X;iy?JnYQw)bH^PR9AfeU7oJ!3IA?(&R5XxhPd zJ3im?i;-+uNNEl z3)mzuYb7wRuv<~dHbHEr)MEF=7i#&P8b#(ba56RuXeR8?N;t)N)Sss+;7KzhOM>0I zovNEo-qlo^^3I1%u2I1%cyqR+kOPC!5eN1X2HVw%OeG7%W-xX~Dm-4{eQw@T#vQKe zR*M`so-hg=V3AN@n8&~_c7T;hlB?~q-5chp5+$t-S1$ER{bOCyC}pBh#jsE$AmPga z5vCUnf>Cu>cN^Pe+AYc6_dPdGvgq%%(!I|sZC^a(`21$_jqIx9`Bulf=l)EM*e1GF zddop6wxU;@c}v{9PB5nK`1yAG4{^NSoUXP*7q@g)Ar zyN)0wh95Bw=ed;FloD6$Il#=2bYZPql!M8xt7n?~+_p(w)L>fZ*sSErC9*Og?rwcLQ%3J zGaHwfmPPJij$4~oo?l?O`xVd3YY8#CtKR;4ecd4C@Tpl-7klt2Sr+a3m-w2^d|gal z$?Hp^>C^Le?Wy=$^m^O$*%k*XUzRziZ=by_YmuGB+5;laY)dC=rremY@X!+;akXTJ zr1p>nR>sd5eqC%}Vp+aYGN8csfNARu9=$zfCl$<@IV>v8I~g~g5ut=&?(h(`L4wFz8Y^loVYg8#W`hu~7I%PuJn#lDLSojNiwKgSFYwjfUcg40|Q^2!KDtxwhG6?3<4S#1X=iR z9EtPfP&jaDId{d9iHuA7Ud82E@p1(DaZX8SY~k4wz$9!R(9p=>kyOwq&>fOG)##+f zWS@-(6)uZ*C{42HiBO-WdfXsu<5n@5g1<+T6FV+OOz*vWEMxI@%V$@8O}Cmn6q%?Z zP{}lDcJ73AlO;YxHSq=*II;@MZkgcB&0`XJkaeAE3TmCI>JyFmU;G%r%&tpeM7KTQ5CkF?&NJx?=3pYBKqN!WUx!nhm`a^Ne)b$ z?&o&jOkqqH(@qh+c1)FRYvhFkJ|b*8ZZoCX@iekGJKQ>9-|h0q^|Vz?%mj&v9SaYa zG=Evh%#pCiuCcg{=TZ@K?4Lm6v;QTmUNQ56jy=tub!*}Ur~a-6=4Fd+&2zIfyl|13 zJ>&jFj(@(p)tfjJd-%6Dimmy5Sn0G*@S2S$Wu%pErr2|@jmlnoR7|E&V58Hv$G2=3 zOA0R2t6eG0$U9l$@COgW8yi|B<~TKSMyN5%26UxZOxEL8c*&#^aW?JGGRfr(Ycyw^ zJ}|>7c{Z0xg9m%EFpHP~L$8BRQd@`1k~ncr0g(f~NfC$mjs{F%%5db_pzR>G<-uX5 z1_$nmKN=X+9Q9qKsmBe>8^&~yky}oEk;vQA=f{#z@ zOB{RdiS4+U^<^Pb-E0Pp=}Q`wBN$oS85}jHPmxq%%xRk4xHK(5(=sM=nQ{E5mpMk^ zjf@&one_?|USl%t@R<6~FQ|9LIr}s>j#;Kn0x`W)6ur!ildts%Bphf~IKa@B_>Ga@ zSD;~LgaZ@n(FQe_Vh8KF-{)C2rS$R7SpLmSbrK`*iuoCzI{h^S9 zgTty;g`Ib1E#wMkX!Lz}R5GOOS>c3cg_L70F+5uum}Dcoia7-iv4t#Pi&?-d5VL^Q zB6M-uwjD?99n#LM^*SXGDZ9RsGqA(Sqf0>vvhsLA@-!pO;QC4Y4BWqs&+Rhh~^m7SgpjPp8X@@6D>3dIQ(WfvO=?O<3qX>Lv`bLKW- zU+(%Jm*dk7xo-7@EtZm=%f0i6srl}|OWh`~>CQBKIW0rr5wo5xlVlz@S3rW(iMb4| zd20(=RC!B1TC%;^W-)n3%qlo#&)}l)?f{F1$H6$i4UV!U1&w?;2l--ltmCd}V4q<7 zNd7{nu%6P1adTL&)<&RFO9}J?a9N3Iz9OO?~!6Z6iL!)|F8>7k&Mr9>!p3`w* z5=|)+*P3gy|J4*asu{-E9cdG_*JIMLe+*L$J(Ek4*^Q7$7m1DW0 zlX#exKF|vD`S`M6@{}pkHxxx0ZZs<}9PF_3Imy>J!REvWPFHfIFTPl(aU$y4!#@YPEg25&nQzc;ZIG0l%iz!; zCXnD>uXR8wGu^uVm}dVIHwKX#4T8%I*aF{w6AInoBAMsMrsd5TjbsSsY`1VoW2InOzqT!4upPJ)o|ZZHY#WnjC(yD`9YcdI&AA9qZM^Oq&tt&(R*vMF5i`xLcek8sh2 zmag{?`MM>J8i_eJurfF>DY!SV*qOe(XT2+f-GQN+N5Rd(q>=BBx!#ZMvB?Q+Zfd#H8@Lus;NB}xwUC`_enIq=bXLJO?TLZXGXoX3 zZIpb(#$n#T6rjN2bdjU1fVnt;-ERZWH3P2Z5A5|9eEuB{DF|cF5MIDKH{5tW=zOJu z9SPhEr>E~|;B8&NEN8%I6~Gbsl(nLPJ+*;7;Q@=^1(t*YcD)y@^#^#GCrBHn6fvn~ zvZ~oC3AQCx)F>usF)m=dZq8=X&Uz`mHk2VNIDl=DTdkD|BX0xaVh65=&pEm-v}+~R zHJ&KhF^#SDMqSYKB<%%ET`QOhHzoCF)a!3j{$DQ6-l818T{%fz{9Xa~j0Vox6F76* zId?X2zbN2+(ZF{vf$v@ew_90sYCz+GqaqVGtF|O}-!*4%Okgin;Al^0?<`=KJ-}La zfn{p~@0o^zcdA_T3b-$B2;3gPJ#m3R%u~**85~!C6fisf<9HXEzPNzBB#6auGT$SE zs_g*<3m>pID{yTM;Js+T;cLL&TEH&qz`z_}!WB?+*xc<{Il~ck+e8z$OPd))9xw(g zWQ8`cCO6bJ9+=Q=z)`2b5gZV`)09mxs<<-gPUZVEX(SQJ&77*ueuYr8;hjyw0#j;?zSQ`!%5FKmxC zNe@;?k!0Vjx=~15>Rj}Hs5P$Um&)EyH3j@-(1@OK(z}NdPA>OJWp3#7TQGxM< zcxFOGc)w)Zt1ZQ032bQwY=*}t^c>)r@PK0i17{}#XQRSI`)iyj28{lnIJPqIw11eW z%{1xqBzBd|j;syLTO(&{Z(x>GW&Y@18lrBj`?6y~N2%=5C_}d#%TG!#F7V!)!0on- zb0!1V-iEH*4|q>iN?h&a+cAOrN;#(|o0pENWx!4^H5PA)0%o5AX4wmw5)DkU4}vWM zxF22+h;ikZ7r=e^GS9^i6?>XnzPz0FgmppW(l}2IuKEXzycZbD1-P~@;5AC_d$BZc z+XUV>3Ve$i*lQcYu1yJHyO1ICOY^UW?TPf72`d@w6d9xoW?l?uYZT!6cYuM@&Aj9g z1LrJF4zGsVxelyqUpP)1aJ3h3bbMH>uv4tEfurC8Tam%+t}SdDm9w>OX7A;cbO~hP10PKPVSi-%4RO$UU7r(%FH=eZ+7i$Wwl_Px5359N72%2q2!5F zC+~;BEDM;IB=Da7P|*9K^4-+*^8wt&CmHjU;$FGVFICfQ72s$)u&C96`9dlCA_3lO z3EZ~?3(m64cr$^|T7uV3gRA)g>$e*jOq!WWQVA&;ZrmSSrgTn7d%(Gnf%AYtUEK?g zPKH?v1G3UCaHdYkT5^D=Uto!j(VEH$>{%1pl`JQzoUHRXWxuU+_Fm1kS_WL(C;a0n zG+L^sImc+_(z{*tg%i7W7^Gad$@lgG|CR&s}OP1a81J?uHB@?!nntmIiICTB{`eHOKDg+$q-4ZK%o zE}K4qrSzYbqnnZ1owD^i4l(dD#J+neCAmP_K(>fw^bG zim#H>>jO5{9hlI#q1Uf~*>VE2_XpN`hn3qHHqD#Ra#mx;vlo>wLOJ3b7`h|1j)y0x zgqtY13zxexZ*85h-~s>H2Q!aM;IRL_xl3WoX>N{`4Q+E1ScC<*4*uE_SiL65duvW` z$EKYgE&;R8&zimSWJiw%QwY1!{|P*APNx|y=5hSNJ7GtTo||4&S7&G9+|x7XaXr^V2hsGl9$QNcfDtug62GjT}v2P%{H*C`psx1z{dA{+J@?F zFAwm|GY{Jx%>MPnLcR~0PK$TTu44GkvF9y|OmG3Gc{kr*1KtA-tBr2$smrKq7vPxZ zFmr|h+qKg?M;5RL?A{wZn>|OEz4gbjOo4ssowIk|+!}Lx_Dk5QrglFyRYpEz23Ni}xDM9U2(3@inV-3h$cCiJjBkGs92_w|Y%`M~sE5m=ifFCWJ5xWV$@weNg=xw*$k|n$6P_ z)_pp_5Jz5qaEtLx)zv zjjkE{x^5lbFrZK2KKEz1@qXiIQ^}>8ZrNv?#{cZ zTrC$?)J=$caWwS9)D?LzcP*(_ten###fe{tf2~Cah83xu(^Fg+X?o#!seax3{wH zy`iPR}b)J%s*|JF8eu3f;r`vCX8 z20k6ROKNwyv(4E&(Psz2~di;{Jey|Qpa&y9{d zS6@7FmgU;;`U%sl7XH&5SKjVyz0h;#0N;Es@p~T%Zp@h>Ww`mM`n?nGnO-q9PLmlF z1ZvH@x!+FU-d-?enE=O*uGLKf9EIyHg)QKmYrrZk5c+TL`OJhR$~Vv7kAA>zcY*sK zlkx&46$2(-IlGbu_8JA5{=fWOyeBH!)`mva8)WiI-FrCy?b;&i?b|0rXSZ|Cbl}$9 z$E_%P>8-#eE}K&d28>%w)Pz*jl8n@B-bw9zs48`bLFytyS-_JMFX9|^x#oZPxBTeZ z(~AmvdmA>sdCHk|@=jmYMuFRhuKYMW<Z`rhj`?ELqs}Ec%R+iI8(P_p1_r(zIUTduQoKWpPUot zsj=(kxlo?W-Ir_6Owr;Nzz3(aR+Qd15QQm+p_ea_C3$L`! zaclLReERPdcj@^9i;pE&GZ+;dVE-*Md!yw87P}jdqL26gd%?5sc#HshO~A+Lau4gy zT*#SrH|SM|#J+8Fbvt|JNF;7|c+tRHoyL(}!MWhZE6sP@8~H9NtmVFR?ST7%*B0lC zc1@ICJ5hRzi1fxl2FG_-O$^!3Zdkr|&zJenmw(Zmb|rR8!-6w&PVfBMx#GgxoeWiH zBDeNlTu^#r0q>g)ISR%c@(aB6r0>aF-FwA%kG?}3=fWR2^2m!;2FGOfs_(Q`e6# zx451B&z_zTqabDX|C9dyTV3jXb8X7mlyhss&PMs>+}Kk3_~hhh^E}5^Hr7|0)BW?W z@(oMQDBT&(;k*ktRq{HZ;QyZ5cm&*y^hx^ZN z*|>R`Zme;1J^MuofrQ3J<_)|$77Yqzxwp3^s#_F1Xl&pQIo7kXh;xcw?D{poUOnv= zFZK!DFhlWkXv19h-WLZ>3NNpoXOhby{>cADrMJFshL@uPD=gDF1e&8Gr!I%OJiY|mDb972a7{oI&&_mtXh%4 zIB~V-I*TM`#sfyD*D^iRI<2$o$)vz_o057MIMNOzx=nF&F?@2|NXhWX@$+2DEJk)$ zJP!A|$auC09%Jgv+3d9HO^@**JL^v#{)-N_$>z-JTqYKHBE^NB>CvpWrp&io4m6lQ zZYegqQDfcE#J@~|uiJCMDVCjaYi_Ad_TDffuS-+SqoK)4F7k-=yGH^UdrYRdba42{ zS7|7T-l<-{sM|@@QK)~<1QnGD-yWY|;u;i{xil&%Hf>qjs>)?*Y|?i;XV2~1RG`7# zy~JI9|MO*gB-PNsk)pF&hbb*afnf*Rgahmu zhccf{bNj)R%qcrfpxNL+pG-=>nOlVO$q%;*4)a>82+#0xR(Z|bZ^iO*@hK;ljYp-* zwrpe-kJehZbfOFgg98)8t&(CtmXgmm9#7yGXlUS>Fyj!-vYA}+8n{3rEZn5i%xL6rL^p6uwu)vH^Q7Mf5N`%Pnr@F`f?f*yU&KO z80jl3XV2~U)wv>G_07szQ6)DRSX9i`9O_xVL7|XGyGzu8flDspKqKpsS*!14bLyKj zTz|x6zK)|{#nD-5zdpH$7?xaMYEd;hZM50#TZD3}snh>|HNA2b96KgBa9w-jadz4c zMi%}nTc6O{kdzK-Hr?1C0cH8X$oo8~*T)Zq2 z3mO@oNw5mdS2pcXQ0-J>+S}d{k^j=)ac=&VUoKWxyMKRpy?%*n6z_(m(=)b)9%S== zYd@Hn#RRyGM(LWha-R41!jd+EFJO-*BP{C9ATf9 zz``HUn89PuY74PH9UxvMc;>*l5(i$P=@G zNwI-}MIwQb%|~InpoRl8R{%qt*@5X9FB2kaUY-_ulpyjT;r0_3L6ty}B<}hTfe9wT z91b@OSTikJlw}xIwSO*5n0c~ColCGw^#-GW_lcOPQia`WU5mvQXB<)Z*U&Cg(!dbG zGLPv*Y6F7@10&akqz|u_G?--?R%%{i5S=rD)up$QCsBb})NVs#rY-{$U&p@&rWXcX z?jIKRJup16)adDik82Zy-+JD!b`fZF>SEvsoXqmaC-w55H|j!R(Jm??AN$>NbR}6A zao#8jb@1(4tj;8pU~b2GDx+hiNrbZ+|jE3s*By(QjN8YRmXX#HHBsPCLDA1+b zBCq1C>U5@CU|n*Aa*jG**%c;!hYO6nRsoG%5&VVR4h>?B;TQC;tu#+yNp@mqU|`@$ zV3NFWfZfiZnLj~+S#m)GgWLxOo^@eD=NgVE@-=o$lu}Wgk>%lhj9t*)X}Xf0=+%F& z4F;EgU3eH`y7p!5r(n~to9_iTYF${P6}ftLhp#%Hkyr4>have-T=spk_41qZD&$vP z(Eg8C-PDX<+JAku%7i&hz+#6Y-?q1@8@ptW2pxE!d?oaxK(j{+uls`aW^Wh;nkN`o zhMMnoymCmuy+%h-ZdzZV2{XUkLniTEADAV6A7FR>z{p>4pjqOM1>3Bm6WnnOg5N7v zDH(nE$er+@N$So47Mlh~UY`$7)ZBD%VKnRlG@=W#H{9bCYC^X$8gjUFy?F5BA#mpt4c6Be7U z557kher4=3z&gg};gCOV8}Xm+l6v3ew!Cy(#vV z)a-Z8@+TVDjV4^;cYo!R#?<{Pc=-a}s)RENQL;xIbk+#^Ehv@WmeQTo!7TTv!fm~y zSC?hemFv#mOO53IG4?EGc7EyY?{;}&BTu@(bsDVN2Yx-}L|_!Re9U_*5NkG=YQKNUSa>t1zOe%mc(xUtyy zLyfR+UeE<@`H~GHb=$c9t_l}@-g#7WU52;wIro$sO%g{QunPD%a4XmQOto3qF7iUV zWI^(!aOf}>{*?CSMuPEI)gpyzr-^<7EiwZZCkC{%1<#}H5Zq5>~{W;*Sms& zJ!{{)()NJ&-zv8+3wgcU|3Bluii6q$85ibncxar%!0wv8-adI1XXcKjzc>p!4#dqg z+4Pe0^kxf=!X3<;O>!f|onA1PU1C&WV!WBia_WMRV@5Nx2cw!N%SjIHw!*DiADJCE zn*EBIv}Q2OI=z}%u)R`(y-P`&SY^{yA2?N)NPz5dOn z^~5ffgKgbsyA8tj2?A`NjP*W0HYyR8|IBDK?U%@mphk`lA~F*;{+w*+Il)|Mg}JNM z)ZZ3UEt02BKEe0Ni*1I+p7b5vU#uLCMmx0LW@9tq()_f!Y4ZUNOX;On8x|#)Xm+uk zG-Q>ZxVKG^)s2Hy_)3q%3q=PHmYhchod1Mp9H`K7=4jSUSk5oNpnSmANkbr3n!QLu zf06aR1=3b3ENmVeyY)-#^v*cF+U%6F#xC{7zPG__Ne`ShaN8&TXifZ}R(RM**LhuI zNAtm`b(4+c%XfEtudx4lnz6FmS#t4y)5V&uOr5S>#`8AKkDoA=NpVm53Q?z&WqAra z;sT}ZnYS$V*pcV4<5=?%&m65q0__WCY~fBh!BL~McSmc+i?-Yf{W~9dgn4+*&6srd z$3)J{Jy#uNvv2Y24_FiMqB}r=rSphR#Ray9%KHL2SgU4vF8XYm;Il7*$4xE7YQy70 zZSD|X)g@0pd9V)e@M$@VfQyyR!tuVZ17 znc-}GXa5gD&eb3L=TBO4@U>L+o%t~@+w8peaMi3|rsXm3Mq8_~g(lOq&fR;wChy?j zY0ucuy07EJ$(7oNKePn6obk62I?Kp=>`u>>l{`~L+Otov#dWp?T+tSTzoRCKn@=5b4EaZ6sZPc!CN!U-qcCx_HRyb^aD`*eBJ#7~==XLx-s z-0d&uF1mtI>V%hUgY!C{~r>2XL?5 zhv{#(8%OJuBmVvY!REf6|E^3ra6w3E(q8^I9&QFLOc&YC&tYda+NZSUf}6m;=|=*t z=xlRmusN{W%>BoqB_@FarpMC%?9)5qn<&wiRdVW6^+k!LQ@k8IybG-ri}_{#9liMY zN#NQ&>rAVi|8z^)-LU@oaHYqO{Rz@q4k2xCWR9?E8n9nf@r*w2DziY#R@ybjI%69x;ux2%zGRZsd%|dMYt)`=MssD(3k2kevik2h|C^_0!zGWv6P9HS z?UggyE3NDVz$KCY@@LmZ>p1bYlz3(WAQVNEm?CcF`%X4YRKnmuXhueKl}`7 z>)=p`W|W*^Bw4`Nd3K|WbEw>1FV7#Ro*l7%-*o(60E>#K7IR?RU&SMAjO}*Sr`=qn zTee(cSY$F|=>grnJ6c*I^`ANva!k%x(e`guTgHLbtOKpl0#OU+YWWJ^%3OObOO(|& zIqKiCtK2`obQj#^=~Q9ApUhsF(e~i7>%}KWokYzNq|G)i4NuXI=Gz)_=gYqMhAUgQ zhWNx@%&xhZJDEAxWP9U}vw1cvHH$eqdn1CEMs&Mg?YSCJHG7stz|?=nr!6;KjjHIZ z-eWvFLLk?{A-;Bk{b9DefFm8zCpHJ3-*ofNf`AjPb6k2JpPTjJbcsj%h0wbT5@M&_ zW!wFQ?PlnNERU|acW*BEdNboj?5af)F$ruMliI5Pw22toE&goD{dQ}d$(1B$&a~t! z*~KmOdqcLYy_ZmMd%J9#vjm&VgE+stn{*E=e%vr+OR?Uf#*U!B(Va&dWq9wHPcf97 z;jHjh&tM0?)q>vU@VEBv9A=C^%}=Ph*k10`=-uI7qksNw%z_zPJ}z;Xu~b0s>7CQX zCWRm7bA<$y1jO?AwR8L3Wq!=2zAK^d#)B(&ALRXbAoT5l@H&CK0)cyxE$$Uh|Tp zlm-R{#h)x3MhpxLIt&a93=)hXjP1PK+`_y(lDu3ZLVVJ~yi(#Ee0+lZ{E|X^!jk;L z{DLAP!Xnb5qB4@AVj_~#qEfO_(qfWg{6aix5}Zn+qRLXD%3{*W($ZQoGFp;iCQ{M{ z5?m^Zvf9eBS}NkE>Qct)+_ExCveIhOQdY{+O4`!O+OpcJlB#Ahs@n2Ka*8_gY9=bm z%GxT*I-07g%G%m0I=b50>KYnGs>;UND#p6nR=V2e8tTS|y4J?JRwnA28Yad%#uhpz zrp89rrp7i_rZy(lcGi}rCRVzJHgbyMo@SChhN5n=s$sIK1+r?c7TQjx#xAy|Ubdz# zHr8IY)-GmN!Pd6nR_1>Cnt6J9QJT`BZh{3q5)p2iZVskiPG+u-Ha^bQexBAruC_s5 zcH!>kL0(1~9vbPM=4BD)g+Y2%DaNHa+~$_2$VZXrHSVV)k*!9Eee9v&WHpE51M z$&uCB(Pe27<(UED5otbQ*%>h@ktyZbsj1~zY1PFU8Cm7kWu?V=rEv-2_6}CP8is4M z%==_D+w#RaqxAc8jh0mybtOtJvvj#+?%5X-*pd}Jq0oOxiEm3zdTV(`OI1mGW%-nv z@@d7H>xv3jgvFfCFMXX;aV;?6xUTj3ZnGt`HIFW}JUqwr=WnqeKlH9%(caMS*WFw> zwY_pmXVL0OMXRU99+~BRYGuIL^##`!6uf;HnN`wRR?$&bJ2|7EJtnKUtFC!nUDKzE zwu{xx)y-3@ntSVd7j(C_Oz&))J*A_qYx>M-GbT-#wsc14@>x?i&z-iqw`299o;6FR zZ(clg%jzjxmdx6^cJ{6{Gk0y8vwh>tZ7aGb%~-N>>C&yMm+aiMZ2Q(7Yu0UoZ0@>4+m2q|d+yP-Yqw6GI(PZj^IK0o zJb(7;*_&^#zx?_7``_Px|Gs_w@$K{5Z+9*}*n6<9DTL!6!$*!I!RZPIn@fd+d`@gw zc-SgI$!m_s#z*4%33^o*cUs8~EJ*sj$o^wq_A{&O5o?)sV*wjyTXhDN7p zT9@N+&rO}?&BLNMv*dEwm+2X6%WoakN=TK03(tdvuF%Z_eIdAVuZ$!Xi-%O<|w zrh4U8M!H?4&Z)%sphsH>OmG}HM1ThBc)FD~4iUGFWUm&>9ik!U9(ahR{m@Y)RKiiV2`@_cR` z?XrA!DG$?TH$IfiSX?SG(LJH#;6?W}os5^VH6|Xp<*7Pz8>_d@+6w6c)sNevbDw&K zOXr&(RlgJEcQ8c8&(86?Oo^{W`PC?wx$P-M0n0?6`oA%qcy+4ZjO{69XEcwxPP2N) zGC!tps)@UJ=|fZZI{z~h*~IHSFNG(xN{TV_8J_si#Gz$#OhHb-?8X8G!JrHci5a#_ z9=6HUSHziNKqZPBe4IZVA-#mYr2gl?}d@$ymHa!#r|Hpql^orBnU4FltY@(N~_e?Z&oUJDUR)+n%prJO zc+!`eyj|8ZJ7#e7SqdC#VY+$Sv_-^Af^F|i$CR?gBDE}S88S;V9xd0--EvgCc>j(? z;&YF=oB!+HdAH-X_a5H0$~DeaODC`Kay>dX;B3%Mm8q_IjItY){r|1r?8o>~_2iCO zcT}P-FfD9-aciM!SorcF@3+w@iLb?Gs;!(U-|%}cr}~vf!H#DBO1))H$%+OY??3!> zdyycKv3PA+`-Cc$4F|QSmVD^a-^Vt|t6;au#^q|ORIc|FSr*C^9K3iX`C*2mEXgH~I91MK|guSM3JTeZQBy)GN7h zXx6M{CPk*c9GMam5?4)|VV8QA@$;mBk1tYfI!!bx<%6U%HA)vvoN-LK;P&mTv`^uJt95&wYf+scRB3DF!+LF6hj^{gE5vxm^-j;t*tC>kj;JwgiCccUbjP+a(Ioi{m zMEqES`l9A_%y~YksWvN-_jibA%90cQJ2SZRPd~h$wPf)lp5yLM4{`;xsO(QL^e*vU zs5E`W=DD%DM{TSsC;i*PFmLgiO&;I7RZF67dTrx4ssA-wY~#F5EdE~YJ}I5ueqC+L za%;Qo^LE{h|7X-w9otTT&2!1K(%qH985%;(N(yg1)eN;a_Sq*qv+pv_vD7yl=< z99Sdr#m1q6>1B+hv3d@R-a(Z`e*03EEq?mkCs#&%=MDz)90SGq0-2ZnOg?2qed6i!H(F^iWuyPvDOKHuj33m$H&9!80Bu<{X)wDX*S=D0h}d(2|`RJmE~{cYkb8de_p~t!1|HQjyM< zw&2`fbxAtwPvxvxBC^7v*Lv5=G(k_{yH2w24m`BJ!Dts(z*N|&&7#ZExWit;u&{Un zv&6&r!;{@HzkqN!K{hT4m7gW!0f7VC|U19qf^LYsYC(h zkb9fNj0&&%Jh;Vw>A{gy#)~T*EIeK&Nij1{+czcd`yu9klFx3t&#qeHXns`x9vj=a zj>g~vyC$Bk&ySdF^Z9K4$BlwMeot2=8a_|kIZ5?ooAyK1&U4|1S)IDxKWB9?aS~Z9 z;i~-WLhEaWg-7ig^!PF!G>L6lz`WG#>%@MYW~D7rte$xetSPpQoH-0E3IUJaA42d)9U8pEOm{eQu*I22WGw)1t!4^ z1{SFc3{&_So@%69o=N8`X4h$8WYIXlV)LOfVg5s&85u{_qXfTr&pF6*_*(7<8OP!Hffst!LrR0wyApu5-jz*b` zMwuCv3O5*JF0d?4;9Z)|d-Frn%?&($4UA$1#oP`I>!w$3{9g3_5R-N|OV89wF%L$O z1&rPSY)9Bj9NcTBHE=py=h`xXdqD%Mw?mG20i$RE+x82*=?}Co8Hz1r6>m`0JAOsG zAV{zHlDMWBi#3Xn$k2S4cK=%i_Diw7M+i!f^)X4GhXNI%jIM%-q2sw1Gi-1q0WI#zikWH!f$; z_}(c0qe$UL)kbzk!wno~9jXj=@EYvkDSptrCA~>-0Rvw{mp}lE)&VBZWp#S#IXn)G z@&zqz4V=>&*fJV8-IjCCD&Wj0;5^LUy5$3BqE>253C=DrM(H)uP)ILN^BQh*j~zB{&jj;;Ii}_$55+= zfb*hR=Fb}RE-)G@R7x!`lu;|_(DX1u^4Zoqyfp-Mla`=$Z!%?W%DH}I4dFwZzq zwCQ;jUqOy{SX!_oW4I@S@B~Kh2ki3;*xbTv{36))wXvoZa2}stv*iK1!UIN01IDBY z+*cDgvOl)w6gYn^RI6H;I_ZK*-X_)z3HRI`dWIAFbsU&djs^9F+I$UiQhy?urd(d2 z&Y9#e^P4(biU8ZeW7*yXtP|Kfy_T2AOvn*T$eEd*!yM44R?)0+e4>7MBTGXw^8tow zJ37T87?eCGX|PwTRCZ23F=_ga$;&gk%q-{Ze>r=f1Ov|o2F3tJy8|4T7@7?#c=aXb zK0LtFaDmC}-wuYg>Ql|!r|PyBRg@I-7%*BTu*Lr@P8Vq1Ucf!yVOp{QTPnk}Sq?RU z8Egt0a-uqTOFT1j9)w&GvQ{#6R61%h$sw+`QLinK**amy3-hwBm;FLfPOsD06B%Yo zIkDFS)LWa0gm1`>Y+$udpB8bTW=9y~lyC-C1x6MI#ydBQmhG(ECs{4}Vz%0giE}h2 zt^ZzSoHfbr)#8O5T}L`68*z3`f6*l^F5;ITQKi+CHI{NygM8?W=7U08qA-*fYWyd z_ZEX3X$7|J4|xAy@cj4rqD$-nwhSZrj76!pw~0%nL^MoHvb(_CGP7UbfhjqCYV2lr zp&<4|2gQC(!HJRVaR=Dhb|!lZN_WA ztDWdFG%wz8yn91=r+w9$_1`?g9!lEqPhb$&7Bf_vu3=$K0_rWSnO5a z1kY6k%bCMJw1*e4t#jbsz2P7CmJeJ8-rTz;)TB>f-O*ObdSIgN_0`r=)n?x}2Y0&f zUfH}`vU=_DM!T#v$6r;=U(tB#)uaU(#lF^EM>;!=I43g;FdX~UedQMeqXBb+1MkfO z-pdYi?`3sgU%>00Srv7Q!(ao$^Qv`;TwB(eGbDd!aJ69bt6+=xz`3Jfed`0(;;WfH zA6Olxu^w+{oiU;2@HE!OD(;YS;WeLaD<$<;1^U z#~>K59A7LEoU6WSRkz@`>)GojaBuD2z5O@W4(r|PKWt*=t~vIMad!9?&E>P@EA}{Y zSMyz9;7i!P$$X0%XH&`R?sYFF8U5^BAKn$RW|EfX9N+A%Yz>q4XB?gRomnA#TWa<; zx0$?pD|kILdCv;4@H|-bqH7(8z_Ok0`vn{rg$-E$9bwZ+aOqWSyZ5WSp7 zyf+VU7+$EBVAy}`mHLhDW%>+^QJp*f++OvffiGo)>XPIzc`mM=Hz(E>u&_%Xv=tJK z*&uqgNw>pFulRxdL!*rgbxsw99O`bI5p}qKjZNgy&H5`&>rYntCs?7(mb_u!*59Ys zPT(~1t>O5y$>zE`oU zOIBDMz4oT_lk_n!t7H2#x?*~}W?pBA+Q6~(L*b=N-b)vb<=p154`37*VD@idK6R~H z{Da;iX(s&uMwShX8&@Cbe$BfxA&kp3u%UL|iwT?;E4bYYSSROj2R(Bw=VTQ;+jrth znAnrjMKy79XPF|DcPUOu>p7YCTj^)}ZC%Z6uTygS#b%Dm@=+#TE|I3*) z|Hj!%7N3h@u(;0P%fQipfk)x{vAi?eE-7%CU0{%Sz}$R*qbC7$Wg_ndrpAYh zDoU=)F)-z7^)k)ouDy`Vx|Fp+_d(Ku={dl&cmlKg?af}h zx6C$Qqvt-!sB^Dv=ibAaldkY|=}9t5a!%Q6KARcDQv{YEC zu$|*y83Tuj!qff_9Hs*MwF(&J5?Ic@=wb5a4i9+1G9$o2fz7?)!Nna9c>B1wE6C<5 zxe5C|%y4R}y?Ij1y6&0lA%lb&vwh2yb8Q-#;upQjlbUo%pOvlc0hfO7;XM;xzi8lY z6G)x^b>WN&+^cw7(;adi{w%tnzDFj2k+FbL?*MDB0hj%UB|Hb1${e`PB(TdSFrQ7h z)_Q=^X4dT6lGUeI&fYt-DE!Ul`9C%bOxUX>xz&ADrR)Uef+l8Bh5K69t7Q(bHXq=* zXuzu{!M8MkUE~0RsRGC81xp_~@SZuqR7fnyxf0;b{>A%+IeV^wg7A6oj2cR zBr>lOX?mnJ?~%~L)@|!{@1F4bZ3Ew)5B!{((rQX<;RiMudvmXA$PS1+{`oDl#04g^ z21d~Y=JEqP4-Rm+crZ#Nu%Bh%ThhQNae$}j4u`~pYTfJAAG~Lua9eEtqe#@DQ}X0Q z!2(7Hk4df%nuA_1{;4zh`YD#?0Pfxo>?I5={5z_5x--abU@5)8apnTg*#!3U;#J!g zEWI4?i#wlPK7sxG2Ob{i!VYX_S8%R9aAKY&n~S4W*mNW3M3(de+-n2o&re`= z4|sf4fi2fa?xyKQeg6d~|M}}ZN{?_<$h6;}XIk)T)?df8V>1?LnJt|4Xo=Ln!fU$% zGj{Jh!2Nc@zk3C2?47@-N3vPG*|fHR^=IamW4Y`(517jx*d;!&T*%~Q2-Id~9j==$kX5N(rm-m#- zRa-kn#Qe&Vn+p#tXgtA`d~VLh7R~V0+DTh8NcBBJ55=fTxWUE$h3GmK;{l~zgV z>Br97(71@TYqEOuJcpT&Q_czN#XLLnPw>*r)54dW=5cf?J#iP1x2yQj!1$Q8TR>5v zr!1>H*!GdO~85>S@ z>dsoA*sk_$!(l!ti4_VAY*rf<@;SIAs!VWWIJuxvPj00vJ7dO*Hk$K`SbRRl6U7B(<2iB%{x zavs^iBJ8Vk^XW9hbtkwt?ADl_)2XxWL92MU$)FB_&t;lvWLg1m6^v3R^EHpB5S2ftJYgENyC!;+b4UK{Ca(Z|j1X!nM z9s2Q^Q~j5SNY@JPw{MIU&0hByyU&+k;!te+z}S90r6SQxu;6xxd4RoZp82+Z?NH0W zNZZ4rui^?_H(2bpTx=clf6Hp?xblat;!y^I+c=omRvc*JUZB7#6SN{>(Zq~xKLr;_ z#QZokEoa3=Mdm3p4=OWD%S~jHX*uw*OLFlcM)f&?c?}IMmTnW(W>0vn+tOq?i}U!b zfDa4UJRDLUc`~u97$`8YOPs%WJX7i3#aUH}@zW>TKKqc^#y2BDW!9wqU7t?Mc_uCs z5!zOBfRRmzL9yF-Lr3=Xn7m_^oPTFWG}X^Je!|t-sIkp^jv}}8jfSLu5}&s$k4-ab zdc`Eo)2eZ?<(P?C;S>P{Rn6;f76fe$Y~(U|I@#~58c*KqklTJ9T03joTkl#PULAL_ zE9}Xkyxl9f*KPU0s>#9E_H5=t#THRUCXv-l5(!R9OB4mvTn=krpK(ly?<%WE7$Xz$RwMTaa?N_;Q3BADe-2 zt7e0`RiXp;zX`iuhMt$Y_Cv6Lk;t(ilSh1I8Z5>M;rf~jn^Sg8W;1-Vp~3do6GrYE z4T4F_I*qRwh_jtxVxHf&StsG3&6W!+Qs)BL1Z2(%U)|6u@!^q)!&%3yzL!o0tN(p# zu`F4}8<`yOIT zuu2_an5SE#$dh;@RC38~b9k7{snOa2q5FuG@T&*X zOA_J_ZD!fGWcTOM+J(%%(5rwfw8D^zSXX7UKwCmdj!v}+}Yh(cYuL$g}U%I@D2 z8PkJjGzo4hXbF1MBtGN7A=Og_2RMBi1h%g@Ht)xmEt46zX3S7rmSLgX={DD`S(@dL zK*5H8&9WyL*mMFMbKQ2BiO4PIJGSxgou17%^6NggR#-5XUYINwe&sCBj$h}*mu+a( zDPiZCe}Pp$ZlRpvgJOoGNetYJ9J*X>76}SpI4Cby#=E$oxzx7HO_pbpb7tI;uaog){|%hR3Kn_T&}hy>3)lj3a>aYT`M+KC6U zm)wI+vAzBltp9bAj#rmWH><#eE36u4)PK%t(BNLd=`&-JSepT}5qwM23^`rh`1{B?-Nk6^`+$Z8(}~;>;}af@$xJz-DO% zeW_Rh2ex+xOSxhin7lQP1~@x1FtRAPSw2h&$x!gL`ouA@!$;-ox!-Y`6W6=Hm=R~S z!jOYYL-Z(*!$#&g*F*SI9OotI^5}-#?b`k&{OZ)+?k0N!My`kkrq&%tbq`DsXZv*2 zKlt#h3^(P@%QvnDPGn*dnDg?W#FSn$4L51C@@AR9)>j)0*bd9=+nk$Q-N-kw^{S

VuXA(5^b_-rT;dK%P*LB;z^*XCfc42{26+MJxoK@|kx5Qh zS08Ipz0lBVJVP;ET7ZFd3tvafYokN{#XKTgGTC11<()UJ?RlW>u1*4#_^@89UGx4YOC`=hBXg27R+Ys2vt&8hp8 zGaK011beehoK}i1a8#^1(!pl&FHv%Su@l=b0k_tG2i(^twXtbEU`fkxmhxH9(6{7b zJ5R^DQj-M@ybOVR8!M?_de5qAkm=dTJ~QCE*_=paXY&016B0| ztP_;f&LxP<-mseQcjHxt*e z4bmG}3s30stYFs5YU5wgaGiM)w*rHX^FLe070fPLtm+Gx94)#sHZ*cqFwH(S*RYz2 ztwM7SgUV7ShM5dat^wkoyCgDa?991f^yQJ@!bf)2Dy-5cgeC?wS~|!Fy=eO0z`&8f zz@*U-B)wtP;$7Yo?0q_Bem~vRs?fp0!5ID7K8B-NHDQ77gjw-IQ<#0GoKrAMs%Bar zsK)!NF|DgnwxKa|74Pi>y}OjBW+pGpe!QDMMyKG{0eNO-qmDI^296O8t>F(_e{Zxh zQfQVi>5r7K@iu5WQM~C_kn>!HMXV1PYzh`deb~wMf4GmildOeC*^Ynqkx|F`W-%Y_U`wU+2g*LM$nYhJf?6I(1I>WNrpJUI# zV%8I%8<=h==X~k>mDG@=2)dB{QDPg{jO|tn*a|+lNi#@f{%GdaSiEAyn) z2O8Wq$eD9^+E{dLI3cT$*u)*tVkOnecVLsjBc@+b4gY!?8rT^)e`GM!W^tOdU0Wb+1zXw#Mve&_`U)#jS9I8}nDIE;Br~JUQla4#&te6I2DS$1ZW);D}DbdNjf>ADm@wgyoXTw>sS|*1L|60>5mYlt^Z{ox) zX*-zNHZ-tqV90m$(L2>v@`LOC5r%6%3};q1Fr8@N|Ir{F!DRlI;pX8zPjVWr)--Uv zXpU(%h@ZS+-b$X0(nrr)@LsUso&Cb#YR8mo4X4s3Gm0rRM!Fc`iH+fr0lzD!zFM#_>;#j5#I&Q0nmo4qxk9<61X{Bz z*d!ltvMp!~U!LIU!<>Zf6(U0mjOOmpcDF6!e}n zDR;%rGzJzS4+f412BC@uDTbL}H<_dzI8zzjJ<&t!?`5mbrLoSm(ufHOJ7q2{fuzSOo$bwJ) zxzXx+7nlSX7<~e~gIxn&H%yhO4)vUK;rg0A^KMMxS-XDU9UFd8Qx1g&tB5u?2N%8{ z%~30u*%%sR5?WjjG%yM0!BU!h9*hQS}q3084OGY zTP{m+iXC9S9x3krt8wqu&hHKU-!mBs-Y~fp%w70s-@ILI84N9LTNv0Y7=$_)9W>7U zdpS|U=xvwV1*^^%odO9BOefT*J-oDqVXN~U?|@_mt^>_6%*^lS@UU@(FrS&VBg%_6 zYWJR|$nQ@W_g66rSvQ@-9-g;)!6dm`l&h;zn}PA=1=d)H&}5r{ z*C%ElpB%dHtm7}Kt-p&JwmRslJ!xjEkek@B>~hi^NeAbqnH-EAT@yUCeksMwxv`j8 zu3>gg|4a#nnGthrwUkahZctBX`Fn$3VP&i9ie~vA*G@d0bnVE5g&XEf_$a8csm(2c zlg)wQdTXOrlC1noBaMTTbQBqW=E$G&o>6}A?!*ACPKK@AGZ+k3u*N7bzpt4UyNM^j z=G4U-O(qWiyr#t-Soe`}d-c(oylYnNiS$oglWnc@*Ji3b_k|5PJO0`;ch~)8rY&K#`QvItA~qA4g-_J#@s!vX&EfaKbX`% z9NF`L=i?0~r_5&cjOHx0-sD}{p;JU=@XlWGN84km!)=DCdhZ@RwOC_$AtZz|;MobC z`Mll+7RfFL8D$xm*#6CKFsr*-n`Wvr!;N2KqKwZz(=!Ku*c$$5&~&)B?z?14;O^7! zyd+v06P9{4n9Mc*-mA%WF5|DRl{906^;-sm4NAEOcIF(Akr!-#v$0v~0EbAm(X(JJ z$qV6`9$g9^=S)-?D-T-Uox)b~p-nxjO&4@aLXWxQw+G|g#=Xl{!+vGv%s zJ3{V$Gg$KYJK7RYxXFAY*&H(u*t%t*R-bHUNu8qbTD zWtYT>{x5m{_N>l=Tpo_o+jKo1r#)(H_|(98AboMF!Q!bl92*2%XGS&uT@p3*L(9F# z%z6wi-=2Pp$mm>q-OuZVK>ZU&iQLXUL21vk3>z9hXi0ir-jwOl?#-oK_LEtofRUv$ zVr#{#TViQkI~p`1Zg^>FH8ZsvH{?ZjutpaAYh}{day6wvNuxojp!KaA^R%8ou8e-| z3RV^OPZ1uURHnCvTQK=gkvga0wL|IXk{prsnM|5x`7>2#KWn`bIcN6W8(zMw0qfu7 zD?D}xec5Etk)H4USjyv4Uq_?Ziw5gF{*%qI$NXk+Sj><)6Y<r(BP8OuQ{F7Cxf5Qqe19I`rB`x z-n!+zyZ0`VW7^~dX0ronPFFW1RZp2-dum3j!^#>)!G8jZMQnW5>1$p+J;nFrJp=Eu zGhE9;B1LsqJooz8D4W43_JBcjf{jLd(W2@v9`%{d({i3WzWDaGb5V0i%SP|_Q4M|9 zCrwpq*FM;dF`wL7` z7oueg82Mi?Fm2>;bDyDB{Kef~H%W2YNj)+9Jm2g$i%&Vba5+r)=Xh>8b6$qiw&=MA zUoW+NnOl$&7QU150K*+imD%PmCm!7LO8cSuB#F!otZo~ar2?+wx6yzbDrWhV1Z9n;xyQV!NLfUSzT{EyvF5l10eY;we`% z927G0udkRA!ILhiVJM`LF0uJ_{-^VME}P|k{=Gl5$10)uJqN!oLkGWAMnys6VODm2 zISY}@hy@24IJm_uIua&2w6IB<`^6{}E;_=&?8f)O1u`$JSz7w;^Mq{t2ZA(0t=kA`Cd&_ir;BvpMX1TLWGcS2CzWNumn3b7Z zBE#(w8(aHSs zzNb$+Jm%@u*+)+bn-$N?dHAQLdxDw&JdQ*o7mf*6?81KE5L9mCklb&azkjr}yZFVq?IkM@t6K6(StMO3czDQp;(J%Fh%F}` zFfz0A>o{c0-?-PEP0;Y1cxL^RCN^=S8Uv@Jr$mz_J~DDBh&VVf+q5odVCFVBaKJ^# zMa7Xt0&d#%^tSdJO(EmoH$fgFgS8l zG+aEMw~J9-M&fj%qfF5htM)m?F)_c=ib}6MWahIvQgEUCe(8aOtfCFu)wjkjXr&@YX;gM+qx(bcVDlMwH0q6BD1h>kS zJvq?E^+lzzT~I)wfra-%!EvKY5uuMlG`Cq#H3{B2BgEv4!J!jI4{WAR>+)uO*~Z2c zpm6;ki|4VXwA5x-2F5AtAGEqPM*sR$z$hMJ&^Z17pT*7M7CzIi>x8E;vP#EVTs$Hc zWuxdRo!RrvMPP+(<8%HsM<1jw(Ajfr-|}n0YW^+dE2^bhUFO#aC9+&(5cLT-DCj0I+f%z_dy%=u z+M1+eg7t|kj69}O3Y>XAWpcmUdbLWUal7<2k%t}voB}oq0hOn$!U`PowAIYInHIcal4f93%roRtcWLAe(3q@K)7Yl%!N}!zp-F7VhZZ#( zMcze?GX{ZnGcPVXMr6YGnu(X&!mw8`Ni0rD*a~3P|X2$+ty1XRVHlQkz+xLKD z8QTv=@dgJ)jwiFK%TgyQnS5vzn6QB5d9u)Nj|B|8GY+t?DPY^ZlSz2miv)Iei$i>A zj4Vti4#-SXW70yULm*ohHIRuO>(n5!i>~8icD@-GYKq8SU%%!CdcZ7jOSQfXA3-FGk-5VtDquE z)=Xl##mty<>WhD$)V<)pSV{Dp?4ApcyByrdWY%eW)Ecn)ddV)fpg6C=Vh*9fGFOMM`GIqn#HCaVC%Cx!{NI@SvrS%mKjUp1cw`I1r9uKmiutP^x0&_^Ldl^ z3A8k19I~9{H`Th`s$o*ymu@|crO6?SrwQ>iA8-!Zd75!;!i8R-m+SzjkG1?e=E$!T@As0fL0 zoV=jU@6-_BGfAHRWIH3X_jC3G4jj=7T1&P5ZSv}OkeON?*D7ed-~cPjv%~E!33@ED z{k_6&ZbxEX9@})|(i@`*jc#|8q~ao+Cd?P#tG#ArN{&>D;_VA9cRrPI_)8r69#>_2 z^TiuhC64#!ZJuh*SL>~_KPs>3%qcrDsC}#2WVt-P$J?ui0*d7OVtGvV()f7i|hlr*P7JN=V6B3!rHZXHZFn017 zI504tSmvS8a<0!tf@$TllX|K}-cf}Op?V)a^B6r~ahkHIRMC$+Y{|y+%UX7nUamO8 zE9B6)VMi5{p=7_)=cG>FD_uu7UAQa0-K4RT)6Gz7!#@W$p1|(t21oA59}#k`+gZQ_{{mWY79(#o34uPe86gb>tU~v zP9mpTev|MF0fneLyux)BheT#1G}^N)VvT-o%-j{w$Pu$3Y^BrGx8=4hd9D6g*eAe>Z*cBCX)NIZ=B72d>lxc8No}SDGYT*f{X%BF9pEdK$K1X(u61MaJcJT~G zM}ZE$498g#jeI;UR#etIsj^YB7Ufhirzr*MMg|TBHkAgZuCvW40XMeZWr=8r5joI%vo%f9@rcZn$ps6x%x(PV>gl*7kS(s` zplHuLi4M0(wQk=_`12VZdCYjN~J}Wgyu&hx` z;^DJW$166x=`51GFBmvpG)PQ1C^?7mBkz&yFFva_aLw0cVL#!pp!8~O#}NjR1MG7g z^_D2=Z%DNM!_2`F`O)^EkU^tFLxXGG1<4tV94(CsHyjlhniNddnQH0hDoE# zNn?_e?hPkhgJ%DEi!_tY>FjC^TcKq7Cf-Cs)zqQc->6ylMw6Bovtr93qm4zazM=UVr#DJ7(H zatxzG*1@zbj%H2Kb4?-zOdJKCbmo8UoPXm1_Z_x82L~&+dj%m~mmH>-oK-FfaJT&@ zSn6?HSmn5wOiaq0gZ3Ai6sH_KSbBirD-V0d!Zj8GA+{89B>2FZ9>N&_! z^+brlQ8*!5?8zbREl!cLjq(PEHG-T4=O!6@Fef>tYZ*Cf8!?;SX!5s9)%=p8QK6(< z(WFt5uE^r#{q3Oq4M%y7CjYaHC$BWh-Cz_pU{r`dCoxJRe?6;5p#X{qBrqk>nx=h607-1$&f>8iLep+e;dP_S^ajSslDL zwG6X=&)+_-ADCxOZho@VmuFv$MCRSVGvEnNM+sP5No$$=8n|V8jTYkYznrFB02oh9c`MP zhyG>UILUutf%&Fmd{++e8K`kznbK17h*7qn zF(c}UtJI-54L9y7^S(JcuueF@TENS`p@BOg)NMv@ZAgpjjWq28PHHclW?y3B_>wj& z>)?ybM#(3K^ma6Y!VZKT1H z(7?VYqD4=GH6f~T$;I`zvgF=2r5t!DRx;ajZ;zNjkmuWLW^9f8Ey)a9Gc3*2d=d-622@N5hy;J%Q1dYcKy_JVGg8!WyKC&Xr5j_6^s zxZ%haz`%Cnz_qnrR<07O4fG@DH5@&%>c_Pk@i`2vDGh9E*ze>XkdAe1G-%|#;ZSqW z!TU-?;~dAOo^66GH}9?MROn|*OE{LXgJ<)xgn2$UlWr++A81%uadxYmJEOOphN?lD z(n7nwY;lJV@`~|0*kt5Anw%OA*e7gIlwh1v#(v$zf#1MUis6s~!*hYV-m-rV$lEX} zUtwVIIl^k8!6-9}=lllM9f$r&#vEktS->YE|KjKey_R)q3j!=PXK=q}S+^oe_gd|vFzCRG-ekccyV*Umb~x-yD*I7(DBdNK8E z33T}3HJgp2Nxj8M;fado6edjvCN72&wkuENuP|wLF#fh;6ftO2_#G&;ty4^*$aAJh zV%6e#8fo1@tND(3t@-d^n_PkTmA78mpPm)E2|2N+&*4$%6cNATs8VpSTxCt>y+i5? z(&qBUu;{q(xiBi|95%YrB-_GaT;M935Mg3-=vu6dnJNQE!vyUJr-RpgweQ4=ggutl zbCj%*(TO-DDZ!Y2_Q-kH6S0~%qW1s$deqm2$)ZEg&ZAjpf!?+Q${vU17Dy=MxH8o} z4cTqTZ1Mih6bFT#Bu#wWpFTXm6D-AbFh=D=?=!!=?>1n z8?#y&w`~;L_)9R0|K;f$)8?uO2?*TdztUsTXQC>2s;6isb3oA&fr85((*q5Cg``AF zD~{b;c;@6AS$Q5Om5_()zZ{g5XexQ^C|tqrT;ag|;ed?6VdX!L$_a~1t}rRrG%Dyg ztGONK+2FD z`E0^}AV$AI@7z|4{$fr8Co`2MwmS@lEzHLM3JSY8HCQ4X)=UdyOJES5a>#f}Q|D|8 zW0AxE_A&}7Fmi3F5K(EAmPu9E(Wt<|B-zp^E8^VgYsMAwjpc`9emv>0pT+TPw#B}*K560!NF0)hjA14A;{v{_bUr~bs=$YLxwBFQ1rD6wOwhDnp`7KeR$ zEm@CFC_OnKbmM@~okj(bro9HviZzZVXBv-|3bIBxaONCfzta06is^W7=FCl@>{~Mn zr-?@S`<*bdV6t#vdhzGn_dVx~8)6%OHTi#zN|S#GO&OU|Uh!tI|405f?_$h-#rZPKG6iG<}>6XE+&r;Y>r< z@5a>Z2Cs~U3C7I|3x3}VD%rI;Y|ag_xf9rUeE_fk`>Q+4zI4@(d@<8BOXwjhZu@ zHTF2Fw=~@9?%%+|c*v#Wl)P=lhZTa%Y&=3bPESe%4jnva8CO>}^YYQ|3wrLmb6$RWqA`7G z^n#p+n^+$UtNU+BxXR>mT1Ypl>&1nKO>5Zs<*gD56c=-_^T|e?Vc4Q0u-K`7hN+fG z7U#Uc#aX7B2M*-+$ryV%EKGXhqvq3Rnws_Wl<3;9^=j!(&2Bs$Orl~e5*r*EnJ?Lj zFNsrVG1U?*WrHSNJ3zyjwZ@VDlYLi35Cn){U$V51axuO><{$$qXcawNsW%Ew1vN4xT4n~m zTCsr1fUQ!=f~O6GRfzH&1TWFXCxkXF>y&KG+h+yIKap$ z)4|Zd9dA2U{fy7A$-y zS$BY;nXhKU<31U#2Me0m#1w8aa_DSm6m&BYSTebXnv;SFhk)S zqg!`E<3*8%vlrZ?eSKeC5n7D(h;V@!I7Nv=SJayqh0yUe2O%Az3PD*@QNZx<5YPa3jU$H|>WtoM2=MuL09?l&aQNI#7X6Usvwu(mxG`5tL7d>DU>~%V| za^;)5N!{vG9{thMOWEV~L%J~|X_fl42qkvik{eB{)z)l%Ko%zCvbrpiG+|Cob_D(XI zy27QkIOfCjo(Lsn#VsHC8eSZh|1HpI$CD&bx{_N(;v$DfOOjy435ml-9~zx*Bnik~ za1yq1FcPqAV9{W3(VUUMo>Y^_wNT&)Tb}`&{hWh5(=`rD^d+>+`?-cOI+;;Q-(VBZ zIyKXSfxL4iG&BcNu)vHfK1r2R))SdhWyRV%hMbf|3^45hzK;? zUY6V_&7vf*GRe)bL$E7o$3YIM8xBHMH#(EIq|ZD4L%PauVwY+ZBWGAdlKO;JmPW2A zMdv;>yRC|FJ)4=`t@beKpU5SF)C@^QDI{yQvfLk6@mT9Q{P9k5kK&x&}Ah+hdLt;A;+)QRXT$$nVc=@Fbu8NbEGaI%j zhP6Z-l9y_1=Mzw1DoZ-RyXkS8=9EKxJ{+zJhbGuK{W!>3_rXQt3;XheSA^JdUpR6v zN?^ANQOG!;a^OPSHWSu&qBFQJ+>*~y>~-DJAaXB}h4;~whAg22yq7B2gft4Af+CMI zF&q&QiE$MOn0m-MpuxjFaN*(oqKoIPn4tdXOkk_lhs#`ZH#iCV8wR8*D)P+JXq9JU zVbv1zi1vOc=GkJfOT9&#L%)-=?VrgJroLr?ad8@4@A?i+xwCMJR^-jOALa>cP&et| z)W}>Z{_ex&441=VXE(40`XjFX z9v8~wt#&Cmc*KZUt$TGzv`KgaciEQN;^NEx8BM)A^^0aiq^3C2vb%5YX(XKfd`R5o z;8OcGMxI0qu8@5P1bc2Q6g(r)s!@>G?cec8c>4!8)4agml8S{4LLI@3Y7-jmTarYY zXE?HMyU-%`CsDAxgNaS)Ob6!)Prvdi2a#DHHG^831j{(i%~4r!-D=^CqgTHLo3bbN zx*0IaH0(HN^l!lpHkNiKzl1iEst;#{XRO+E#WRtODUd5`L;8V*3df|EIj|WmP!ip~ z>G|Zhom`Fnhvsds;(x@E-WS{wc}5{)nkJK>LYA*gw05Y#oc*flUHgv2tW<7vyWM)M z>~Bh(N9SYda`lWgF{;|Pax=?ke_(#9x{JG1pjEc#!@ov;8OP_g{}1q%Ech&~peXn( z*Tp#RW9QFVzph-|;VO8=ih;?*fkE;`!$GYWMy7=ItP(pM*d|@!*cG-lW931I#J?Bj zy4pB06s^jdyGpivsc!BGmKTi5O9IUu3>qg%o{(C`BGB5Vu+U1fqT%+cX1CWV4$MI> zj)YqqvdT=GWb@jh{lMFwZ3lgwI4AE2)%rEJt9Zh58Lt;Dha~x)srWP&?&xh2VcU|{ zzhcY6DwgX7zUj}eqDtzFpqkLk;H2Ehx3EIKju zf1_tGS)F5$o04;*Y1xryOZa9q3d~5{>(R(~$B$77G~sfqC^_H<%S^Kqfi)-11RgLd zJYd?il5dG3*OEq&MGHiP7#kQ~FbOqqb|^~EP}sDJIY+L6iLal@NP&5hf(S>GRM$Zf ztA!F)2SxlAN}PN6Y1tbIv&08e95|Qhi1swzzvd`=<)DnsdzqkxTvHUju2~==m9D%* zfoo5}LW!}IlW9ao`;Av}m(w7t`#&Tgr6Sv;WNn&T8Zd%P> zs*o`40rv-n{AmX`HZ^d4c+Ry!f&1729)*oOD;}`1DC!?saA}hR&z=Q5EdMlx4lEF4 zTgb7K(2_Ahaakj$k%IKfW~Noi zGR6$7Oo>vr4$D02WzT@3IlvKj>(5_k+A3L70X%6O=jnW&zb$g`rsTB@*llstDa{KQV#6RBffF{eT!)#uRGD{p< z|LL93Y53^$wDm$^c*TphKhI?R5=HwCN=&*iuVu$w_@3##?PA! z&P1#)+~&YlvN~SmlfyQSb4>|5W;87FJ03H^JJ#&e_qu$(-bZCFLQCIh?4GfnfBpk@ zre(XE9w^S7%(;e@$E8s~V4_%&qIgA-b@W4?bqt(4nnm^;U`|`WoTtG4k3ry{!@3s@ zeBT!QW9>P{*brzpH^@{=fqmBjwiyh+XDpP=abVz6HC?ueO^#7O%Ewc$VO2vXlvvGp!1wO}=Ppj`1q#e!3T$g1vw0*+EjciI)hDJ+6PdgY zFnKx5W@(h*SR_$Wz)|?nHi)tC%}be{M9!%R#=Q?E!WJ5zXEd?CEYow4E7X{KZNr(I zgTe+(itH*}X~B;BmhPS={9WzSIU^QjR|n<06O`?}G-fneG^SseV0gOMD)J@v_Pl2PTf%8$9_lpPY9V~2m3M_gD zG!++cFsZWq`S&VF?*Y49qM%Tt$R%$n7Y>WIwi^-+e2WgSerph1!g=(^@>y^EU-^W- zl6t_jB=40}1G7{DLySWTM>C@qgP#5ZW~&8ERtn5f8(EDS7}ORpCnYrH9pLr5!t2$_ zq}9N@j@`}Mfq7G4t&^h2Jq6Bt57_%1A3ErZuHfOEzClPUQOGP&xa*39S8-hb7URG~ zPR1fG2SqQ=mKKw@Uypq_yCY-Aq6F38rpJ0JrI((X*S}{uBhuHi{z6dS(q9{X-V>=> zkiJ7s#j5@R=d%VL7e#;jM)9IX@e7UO6^&x&7KlbM3imPCaCZy7OAr)e6p(l*_|Kuq z>H$;C0yc{StWgW5{cBNRJLVgp5i)1mK{1boVoZ%9ZXa5_mN9Z2U>14J~LdNfKca}-#$HEiNlCW8mAxfkqB7+7Vl%iL_X`O{`>EG&8>l=H?^Aq_@P@6R63 zLq*ReimEk!UZuc!f=ArO@zIqx3hTctWeejdnjF*Cpp^ZHwZdQdsEP7Jj=pmtS}V9! zXA39aFR8fyg}p(6eNsB-nxlLl9*Ffl;Bs&jFJqLwwXmx2A=kMBMqA!{KX}c})hPJk z0pGg>ZllbsQU?AR>Oy-S{9|6h!q4@9N$SD-xSZ686wbap$&7?#9k#>;easCD5_uLg zZc7l{$0YD!0jE@=V95f$7Y^)u82B3waL-B<*peV{Azb=^BafS-42R|_o(+OK7D&!I z$j#KPxm{to)IzZ>jk2#E@V9wMacuwJPC)>-fOW@j=%@IrYZa zn^$9>mWo<53Qv%Ti%b;mD=7)R%++F0`q%VZiMCtD<%I`WIk>_WDG8P8rLe~+EcR1S zmNxU5dnj&!Lek|D$AGS-tDf@TH}UJ<&dGGqV2LotoCACshd45d)GscSzrZMN$5R27nVf(jG@<@U}-a+nZ3&ds}^f;l!rsu#Qk-)cT33JW@ zfitW5KO_iBEu8vaQ7Gd8^S1?Jhqj9;E#zBsQ0mnJ$u*4pYZ_(Q77DIuoUlQ0)s4g% zl{@*4G|GN>Ab5#UU@rzq=$2hm%zzHZp?nd6;In?(h26|gOSE@Q zToE6C-pjl_+OLW8nGT0a)c3yEtZ@feV%W=#FC{j%m#d1Yy6)QTF^eQkf{9|E!FOjJI;Q{}L2JVsttX2sgbz$rU z2Uts5*kTSOo;bu>uQCKypNMMhG&<6#fk_CdV95lBdVEw?r-f)1mVSzvv zqd`=p2uow3T`JRoWg!_AGQ!ubc1(?ayi@p&0_PnC0jtI>)(eGY zZ(dZ~lBk%azkAVxrw_IyhcBuqV&JG_SQ@yCGfPYA(1Cv(X}d&q6w}Wj z9@iJjUs))-MBwz#n-V6|g*?PXe{79U>@STBkKV#jvWL}4!HDC+2KF+C#9#+!i>k$C z&sDB`IXAQaV#4;_kxI{!^p+m{T@e^%#dwVS!=qZggGNzd!X}B!O8*IQZ_yW+(I_ay zmK)um)3G&t@vi;3v5pyXI;5r}u&T*E)?*9`kt;Loev|c*< z!S18d{ep8RJxEuN6!VR06pKp~&tT;4bKquiJT&b9tDV9sMYdHFt=N(n5@$#q_%~T< zW$^+=rUnKrg=CHdP7X&AoTy;B*f3dn8U~zI-9DiWDA5+D)rAs%xc6|2cQMdWhV5>cct=R*t0w*u$ON&(7 ztPl_xCLYBomXIi})||en!MaH**L*Hp)`FD_qu9R)aO~KZD0NJC(yw||hvRGyj7tt| zobB-MxmOH(*Mt50A2F9CaQf76Uy0|*c))D-hk1qn-`NjzwkXs*x_>0qcAB-rHO5EH z?6UG|FD^WMc$k+*)+*-20Y-LS^@TR9TCTn#tXyJNMNe)RHnWLHIr$ogt_e~1U%q0d zarvt2R;8=MB%@t3r&^UBD_ixN-6X@lbY;)n!=5XHmTE2VtDMCox@yYG;DsKIO;gVb zYXr{Q^7535*97^b`Y$gIHXrX3F$n4~P;6qkBd#{@M(w4id6U)rTTB=~Jvlykf8Z>R ze+QqiPmu4Ze-d#<#XUvV;ng2SQLZBs{dZ<;&2se?e{nv!$7JK?HGz})TeB6Do;o)J<9U3kjZtOC0T?#sn6c3mE&F&pxe*(N(Iw7R({%!^Gna)V%_Vx+;61!k=xp`$(3Di$AejX77QPk zi^W}0^vtvS;mKUU`!`JH9|NyIfFpaW$s$%>-I4$oM%SEE2lEtOEMVm@iCBJDXns_< zY-!|-U3b}&Is#+m-+tZL%2$8oqoaJO4x^iV$Q4Eo5s57gj+{C-0uI+LwD90HmI#<| zpix0_h2vH=!QeBRb2*r}^p-!EgQwr0L53zO7?%(6rZs)SsHP&OLvyg z)R2&kg`y&P4|ZG)x9Yl@Ea0$+m6>02Luylu(uJi?ll?y(y*{-uwkP#ir>UA`vRwI0 z=NVQ{E*z0tV$ghqJ-8;ROEXM{@v~s$1jQ~5pEDmjG-9_rXw25z;=s&j^=AXCTnxj+ z71^Cd8#0zV@N95kWYlqRU~e{I`q#QtS47Z}#nt3M!YsUzx{>zH2U6b80Q0 z<8(~K@RfD)ezUBli^G|_uO2>_`8Gf$_R?g9zrubzZcmeZMbaXKBaN0Vofg-1bK?x< z?np`HvjQ%xlG`hS*0Noxb#T*NbVI>aW6^_$2bapKtdf@fk-)w4<;-lm3r+kx7BKs0 zGqKkg$Z(1*XReoEk+3pomI`TL5O~nQYH{ITi+zJ5ckPEm><1jo4Oh5Zvt0D8cy^re zlZ(3or(&41!GlIggC{|%Cl>H_d~FcBlfY)e;G*etQ79{AK{3as15a%h?VPwpML^@C zSexH^yUkPw=I({LdThyMqEnW-aIt(tb5fbCrONTP@Y*soafT!?UHRf*y11C}A76pOY>5O8c8Ny<%VHn2-IY<@ z*b&MsJ>>zDxW>W9ISL)Y0ew;zFElWil`GWrr8s(2Fq`^aTvWwN;^Vi)^i)V0}NIA9*jqP z9$Zr6IH+$DdM3W1;IeL|&+>^MgHn!sVU~XKK;gNukGMzU>Q_6?3tV_8%xj~->NZ7* zXO&EgSk!~2g@ug#+b%K%=6Z&7x*TteVQyd%H<-iqV4*Dx|z&UeX;hc^i`iyt^(mVd)<`@f{EY+bU`cuNdeqdhw2 z$=%?vvzK-a5=rKo8zG=&u!V0X$CT38EuOnlt_PI;eIoNI>Zp|NEzX!3Om06PX?U#v zw0=#`6yB4kCm!S}6Pv1bhQ+|(AYa0-rm5Fh4X!NBQxZKaaUvme`J^|Oj_XxVkon?Z z5T9^Fe%b_C**Ooz_Ebfblu$?xZ$G5;t!&YisHI9 zFB}!-u3pl*cFV4-dshlsY>xG2u~!0r&RRqWoyLkbd# z91ar>@m3|cDZdkFwhes7Zgt{y1OJSxcWthk&(LBjRN2tLpwPgWkia4*7TUVU*W7CaGU$8T&r;Gmqor}R=aO}TI?cpckT@P!mj8ehX365 zxfah_TeR3rJI0;OI#WJ)K}@UV^`)-24hlCgNE|hIc)mWa=Q!`?*a@m^DeZh&4A~0z zm1}zIB;&MbRdT&Lz86BhUOJJAFxJlD=@Elbs*gNtplIK0!GFO z2d)TBXyD3d6sZ5vFi%S1I@f83*}4&h+!O9HBx?)bD!!^_Z7r~p+a+supR79b-kZLe z#wl#qo-uGf>tdEZa9L!Y0ju$%jVu{+cX#^nbUxE)V40Yqz`65o(jIoJz+|79r%Wx~ zr=?$Cw@_w^#=m20ZT>1uH7_aQ+P`+9Z2$5PW=CcwIIJ>0!hG|R(o%-1ZcZM7mWGc# ziRFg>?#SN%(ejE}?Zw;`rG4+ke_iO{eZn9x??bcXv4A$io`)O?9L@JL7}^31n79tF zV-$ZiLAtYMtG|QBm4?}iy~TLav$#E;32_;AFxEFPN0vKqe0wF3-M{%3Z{&j8(o7AE zGr}2O*yow**er3r?v{MFB*L(P*<69+uc3dy1jdU-3N7t=fTeX?b zJTO0Q3D>Fxd~c1EwqF+Bc9csjly$m;<$~$@`46Il59*sZ#VlX0oEu=ko}8}5>XGBv zqFq`z?TdJOV`0O#Slg{E@{d*R9!S_S#91kXq;D+JR~K+IV9Rpi-on7OrGPU$pjfhi zxz>TJg@NUt!{+#%jA82zN-;CCymDkVT9C;5fGI#Yo6{iM>H^1+!c{V%L3+2+iv=q2IO`M;Ria)}IAVyDAo`NbuS1_g|!ADB%98vg`xT~c5b z{o%qMRKXdb+qIl=Nh;U!$yN$0xYjzS?%csOZF;H`qlj9Oh-j$(Wal&~QBj$xX+Z+4 z;SbYJ?Wp2mZ?XMQm8)2N!MM8XVPW*4YPro4D$gZ?G}P=8GMq|VEgN)pEi8Y;!CIlf zW#Z17%D^>Oux-8r_xu7@X#x2>0j`z_%zn%c<*p$7~Af#ry;7 zjz=tw9~gZYI9LohRtGYhOqgt1z^r$Y{kc__RQ_FiEvfNB(acf}xD3QMxbY?B*+6GSd z35*&E>^%+~Q3B4p8N;d{>kXhsw za#7G(R)M)nNUqK)L5_jZ;s7h#g%Ykv(}@NgYgaNyY*mjjZ{NI=m2;uFl>_UViJg^4 zjU^IF*FI#FWN0{F*)Qle%-=`g(=K-3X9Ahurw%e1iA87IpncSQfM(3@>n@# zt^x1<&w0^X6xA)}f5~jxtx>spF`u$W{%QlxoDfzR;LP?L!=7ri*r%TMLuJz22<}#|B&RltYVU>HI`2yybCCnWkSz86z+zv=;f$zHB zUcegez`e78#e;#h*?`NUD!f|BWQpYxW#zSD)O@EOQdSa8%AuxGS~TLgsj zh{&!MlM^*ywl#LXXz0c8qU^z!5Eqs8|C=3GeY2mwXtLo1zmf#@vJLE=0vz=MmP;IX z8x5w&EnME}E~MPZayy8NA@rZscg{5R6(XN!cB^Ytt}bYDU8rcWkVR}MOHwDdNybX{ zDCM9J`pdR2C}2y^zv<4YTHV;A$w6lR+VekKh% zPTVZ93s&!WC8=G--Da>R;sV!>f;B+~nxziR`xlzLI%Tp`#F=Gf;<88~#3f}eFq>XrvNvEi{=jT)%pBXGQ(V9LZ@}Uh!mOIWDDZ)~tbs{bfuYQSccFmp-0H^G^eIm|nRvZ< z{svmDtmewNGIi<%k@eA2O$~Q%iWWBA$oWrUHH)IQVxt0UI@_KT6`PJP-Zbab0*8on zgM{?6Ut>11+xC1^Z+XVNe$}eloW9yMyzd?>;gOA+-X3hnN z4?Sb%;$-IZWa9E<5}PoAQ!?VBQUucjwkr-?9FHu0{~WpMB(yks!P{qflM6VL_i!e! z5%D^BH0&g&#wO0B7|zr`Y+e&s@)NjNt9zuMM(`t=9_K@n-MFS$_SCC%-gd_FcewfRky@EY6u1xQyp=#-HHc znZS}5z}_3c-K)TP@uOGN1nC_|O!hDI=29`SFZMAs*j8o1!0~}WbOZb11G6F5IGaNfMO(JY%sfPsGlL&;o*x6>!Qoz5U2 z;9^t2$fFRElgr4lK~UrY+ZhKQ*+!$t9aMLqA`JGvhRh|KO4OSF8o|`H0I9y zwceaRD}__mu!IJ%2A8BAJbf%Dtt#ik1)behnGGzbxeG69$6ojstYZ25cs!Hz?%vx^ zG*+K5+UGiZve|+YTCvPK-mp|Sa9l89OYpe7f#L2hg~je4Si2Lr?ccJNnF?%bVP^TZ zI3|78fy7x$C7JCmcykwA=RCkzAHeO$z`*LjASlAf?T{pscG^VYdPeJk-)krAyKbSA@L#wlutAVXf%QxPm){ho=JFJ-*Dl=FyZ(mothBx$ zKbvbsz_FRRfwOCcf4*GtV)8XLl>x_A2n0^!UGVkM>thqRrWDMd;t(yk zmDAJV!sJ-iWdU4`7XpJHvZYUJKJ((30B?HyMdnl97qv{QuVwTaJb1D`>(b+Wx1V&J za9=%nm*dlQ6DON*c;|GPrQ!mI_XWE}8`Ks|;GEIG8R1YHd4aVtfqSU|SM!B6YnJ(P ztNeTJbD3%BO5c(?M#BpX+yWm1(i!VF@H8lNFur3j`NL6tfwlhwyMuxZhmGk5vzH9} z?gzZR|6=X+hkF>U_D~ME^^_)i6;zoevIY^crHy~(3rrmt}buk?uYMXj?PWs z%WvR1ZgA|`hS!W|UwavHuD1UA+~Lit2J00KZ}JRorZA+Oe$N-}8W((SV_R?F7CB3~TS-XD>_G$8t5K`CfMHgBN=RISshV7rf;C z$H2#MLu3Pk2LtbwhKK9d-RQR!l5%+cEa1Yk3A_CA`KC6!nPz)yMz3{nJ)P_^~c3;A#|Lm39>kHu$D_I&IC}3v;B>v-t%)E-3_FUd-MvXHc=^1|u_@vxG&_ zj|mZi$B#-`}+rnibdF|#l6v=ni?waEDrpX{7#CoZP?wThJr{jDrI%xCPzBi2)@aFCISyISbx zhUDjhDDLMq4aAcqo;3VPM9Kly_%UijM}(v|he$ve1o8I7vqb_|EQUH@e$O>vtuVbK%^qZJIzlJgXv_VMLNESc!GBr}CyFGE9# zNoAtS0!EI8iY7L`qKZY`wroEaH>)v3CLMEob>cyPnn$AJIt}*bhE{8-jmdl!Ge0`& zG0l^Ct-odltN zej$P0Z`bjf`uUnJ>3z`b&n}apu{-D2o5hMtqld+qIH7Dwhs@R_$SIF+>w8EqOp-r??6IdlIr5dCjEO- zuI;Kb6jRyMMFn`6xL-B6a0Mx3KAFhPQgBJ{(5tQ)>kkSQJZR`t*Xv>As@m~z1B*fd z57QI=%LjO0@*nQ;H8b9wV{&mq!QvT7VqZV^nk$&br~msRFB)a>aD_o?kk+)xN&mcO zC*QjKa@}gF*MB!2`;`?~sBZW5V@LIq&66e{IiSK-(#V}OLFoUI6t?|~qP?sbSgb{u z1k5>DB3J^sjVcx}iM(iFFA89{vSAch!r`L$YXht7iv=9q4Gf%71&v%j3z)1{9Ol{3 zbXdwDNl;he5o=ySvsKR{KFbU@-MRzTs*VR(%Nq{zi!pLYyvfsPlst1pKwzo9pQcyc z!XR;VF$uj~&t#De2Tp1zG=L^!7G6BScCSM1+y+hN_LB!V-I|*$Hu~2(2h0CHS8lh^ z(yLa}!->B|nKdv#S>VjdD}NRo(_9wmKKVN5)c(Lwj5gi{mdOtLGK z?qrfY;UIotZ4+;c!rhDuOfp*>*rOSaDD<7^$hh%HEON#H7PASB{51?*J`IPITnf80 z?mQAp_~GzesDR1Jz=>Oqfl+kMgBI12M1hhHM)_%m?K$Tf9V!%D#q$pIq}n*@r3s#4 zU^}o>UyXs`tn-2R6Gk%*wg~ndRrdMFCOqS~#R7f#=$>O<3btAujXWtIn&nqHv|8vs zV^Q}w;H)X&ZlCp_U)MOr(^t#GeX0p(wuIF083A+b0v6T>MVCB;fjsShAE~G8kzonzQX2o)qaoHNoMI4t0Rt0ve*5Wv4F4P0=q*^ z0<#rIv*NJ?mMwCL%nBUM>i-hhgE|(8%rj_FJa=KT+LZ^zTLoMtekE`?-B6O+>d|ic z;X#w5%_70Y4Nl4-;a#dP8br>#XnH61z-rHh)6`z2)?X}}yuVAFiM@rP zH#6n{2Wx_l){KMvsRquuX9L)7`90)ldECSLrefKIt`NTlmL;Cu)j?C`Ce><6_-W)k zWcN00ue-1@Q>~oKYvs?96^^zoa=WTI9AupKJo>s$&L^@tc-i#uwK4k4${eRxTWx6E zbVf^eWl{SEwa*;M8i&<0K6V=QJmk1r;w0{*$hEu6@u2O;LoyZ(%}!Si32^QE*Q&dt zkUQ^0vIy&k0}2cmS^4ig;1j-efLATLoxvujo8P%1LO?P2D#c&;4x_NDU~6U3{@4 zx@57S+=51St_jWhA~R+-zF;<$wQ8Dpa+1+2+ z@b81zsRsG<1WtD|X0g5oR_%y~Twwv5M6Nk-{>f(8aL}EL&)gxnbT8kXe(oK=3Y?D{UF0k0 z;+-|&psLP6uK6cik7qh|8z(S{OrOW3=;CoD-8!mjsupu6^=Ys0?cv;8rbzK z7@x;T91$zC;JRwL@Y>Q2Hsf~+4U8JQ^*BDvWvuIEP%>Pj*XsD~-3q>?&2E~E>^FX8 z9Tl3Xdco{un1z$*v4mE`l!aVg$DAbf9NScQ_cYJ`bM|ju zza#Nmi|#+JM9o=2(=F$|n&LNG%*&j=l>foj&B{WIobC=z60_P_p2<#(vweNDk-sK^ zg;Tar$!SgpuT9CTOicy162(h9U%2XDILICM=P}p(9j%gm0&UhUhn|$kwn*4CEa8e! zh_L?IAh9j7L1{+=f9Z|G%Fhb)ubzF#7SC|RWELX_ql&^yFaKXmDhUl6HD2oZO#hf( z-~RYg&`HB{+itL}a0oQ!ka)Yxl}WUxfkj!N+3*09>h2b!1&enkOTH82-E-TzWcMa* z=cWV6de4|txRx~WePYq>H%GzescM%&?afHd_;ImKLm`U8&3~!EA3}wB<94(E=vk0}LV>jJ!J4$0ytZ~P(sSD&^0SifS%cL4O@01CQ_VlG ztAA}HqQaV>&=$wi7XQUH!Ch+yq#Z547|@ZdLlj zF3_>Xn4#53gC$y_)#=}d)w6#y%dS|+Qo-<|lYuY6eMv?Gw?vcU4CWgL8P5M~PUT>g z+QFE>#LcvzyK`a#$AkvU9tP%u2J0_}ze;m2OWw8l#m26W-7Fl8+KD}fHZZVBZ1P^A zyX7!z%m>kx8H~a+6hvk;dB0#`TB7UjB0O=C^?3o;fRqEJ#z%wpSU+0j5aiM%$H5ZK z!CLj$f$z-rS0AR#&oU0s(z5h!aed*M-oUoJg)O~+Eop++DwVd>fTc~`dv`qSm$I>y zEpZe3aq4O|qu7dR>Jlvm9M(}gTEY&rC<`}hOEmi*Xf@W@v+%?My93R(H<)EUFv@o@ zN>=pCBrwUpV3a?=XWhW!P{H(1I)hPqMedxrBcaLwo*@w&f*lpSf(;P0^3bw2;{0C^3OiGK2BY75{aQ7^GE{R(|m3yTGW< zz+z+2WVnFUX9J7xfu;$ETf9%O#2GLx~f5gd@k{MJUx>O}L&YL*Oz_R0zD)f?KY zAF$W$V6R@mUNxb;(&2njM`O?{QCsKr-zMo_E^hp_Wa*z0#y=&E8)Qyj(e~zB!KnVD z+3v31r)M~fjkeft{(}N3cGn{f>swyV6q-L;1uV&p) z(3BQ%%`KtHRl_P%pq1~3I}^k7JqH5SUT#(T_jgyKaS->48C{w^&oUPqywqS^z`!$u zL50IWXGc@?i}pGOW~E)1?z~|TIdLh4snsWAw^7IL^FMk_C2tg2U%tShX&xJ4>B|^Y z5~23MGkONof}W$1Yx?YMwE`}kvw6^L%g~lzz+Sb1z2-!F&4u>KSJ|gtZLht-UbCUy zBXm{&4X>3|3!1HWICeM6Olf4(3RODgziyMX!j5LU9}EHqn0y*Smo8rG7BT{$P*_5D1cBU3>J53RAbfLhFu2Y|#l#oC*wF84ZOi z8`viFa3wHu?P%c3Xwa(VN?W2Dx3W?BQ={|;M(G!f{|==zsr_hFPgrJI(PSwx!+Ha= z`3`3D2qq(gW-SY*k{9e{E7T^cFbX>~avC*WWNkEJ2xgwc#GA0#NK~`=X^-j5yFoVF zpOr*#mE179d-Bv4=4S<)7wVkqan-VYz?}G_t#Uzo^?~-Ou{S6C-h6hIy(>a1)N8hF zwSRN=0)N)4|F1+wX@yE{C{xX`%HRaSS0m=g!H z?h8)c8%zcXZ3QVzvOAim|87oSV49Y2J3)&@qo7Is!4$n0O}ZPJtu`<_Y-qM;XyN{5 zz^B{dDsVBdf<;`BRY;XJNuxC?f=y(bX>>K4NN8AQ23uCezlT9Dnr{jja@-5TgTeT6|mt#dbE{kknoTtOu zmD|@{yQlWC-&>tH_Xf7|746j(_g?hInOA;Pk^jM{!64)FK|kQfdZP*2$5#3zE#2y*aLZm`mzzS1^@@N@E>@crXEJnUcr+U2 z8(LCdC@h@8woR04t8MDm(1&RqZRu-!(rX{46&y}aU`vZ-3)+ynRhBh~gLPAEi*}Hg z)$GkK13&g^Uyjul560oqsEsRQPz$eCRoS zV4msJSPeZ_4ZG*HsW*c6Bsp-npJKgd`*o$6!76)MEk{)OIl1DzK#)xEmR;bQ!S*X0+T=Yz<=Q4lGz!a6wFW0i(^2P)CW@v=wa8 z6$`o6vN~wYwSV!z-h;k)=8SlLpG+De_`#Lho#57p5?vd%# zBzvj57e#Xpm~l8pw1v%at4ThWU4Q*Jj;|9js3!F&IcM&bXbyMO%4 zk-owBMJhM6snB+|f6A4cQV&*1{b00;Xwh8IV0NK7^+&Vh4o2l43|bqSRevz3B`}!> zM2o&<*86eQ{=@{m3sHI!6S7}wAA8vxxPmq40mt#I7U=^F3!+(iT6lCn9oF>RcECbT zFG!3ejO^-XD8qt?Dh)*7t=7lIe_s4h;*dHaaj#e=PU#v391kNa{yil{T~&-r*> zEMu9|`U~sgLLY_9(Y2LgU@Be3Y+3oqF`{W3LyK*Jj#EREh5$>_f;NG@%sdJEtZpn7 zxY~5)-yOx`H_URGjq(|d3c|y?B{xbjX{)?a!S9SW+9>Czaj{?T_nvaXP~NW5oG4!CEi&t!do9=4=Fls8aI2ANGMzAGW{JN)?cyrrUHwG5LRnhtv81)$f^*=Q1TFU5V z@Y215iDSjWb^T1%2~12H3~UD9cO5uX=UM*MyWBXbTvM=H?zLgXdvP`m#&>0B#4KK^ z&tTMUYKkX`B;C^PF(21Ts=N6lu%-&b`$a{Om-O?Kz z$73A>qW^Gyt$gMebe`F$TkM{ma-AS1rjxQt;bOp{L!8`VYV%gi5M*`j(=eO&BST5WbCP`E5{J%1 zEM2miRSRwidrgtn%>8#`!_J$>&ia|>ec5sHbMSdZ_opu&US3|Jd3l+iDkI~gWUZ^K zENWto9!y>pCKFkC<$}~Uw#ZF3NiC(0Vkag$R#=2AxOjYmAitbJ%#T100pa;eX0>J= z@o0|~k1BZ*5pserk>7;v;*8H7=UcfsSyCU?yR%_~%x zHFc5U1_oy4E!U*7Y@*#=G%0f-9Y&v~ss^L*~kqCi9oS9)Fnyz96Dl3<&WL6#WkluU! zp4a=8Q=GUy2MTleFMTC`#m|Xx>T$)6Jy8#WI4->kku5QhxY`_MDY0fln!iVAn^@IX zX7OsCqQmU&F^!zM3sRH>)Y4aIoX}W$M&pK@)jdNFWj8Y;4vAt{Bab~56@}g=+ovRU zC~mI^?$!OYL79b7(BJ}tK(S3BGn=V}Av-gZ$v=k=3_K1BQE!&Y8hkpzu9Bd}A~0iS zgY4od46bH@298`RsyFJEb0|ejXcTmx(dar~=l{<+^HW0BcIm2PzlE?Kf z|0)%hPPkY-XVdBYC_d46E;Qvo1U#-%Z)MFi0zBaq&z~Qwj51V5mTeStf zX182;CYqdhG{SBVME<<3uH47(ybb^a+2g?$> zFO2*N87y)o1{^VW4hf`M99GYH$Q!t%SvF3Aftjg*(ZR-9>P&#R@w|&1TpSGgWj`F> zxFm1vuw78b$-#I;D~(Z3hgqO!LL1wi13V59N0|6M4t)A2!_dg$lfWFT09wj*Tx&`q zPin+DkrfjdKTSB`b7r;2`6K50zVN)A@Y1{U!nO7-ix#HXT`rLFd$WD=ychZ!l^de| z-xFND^o&-6Zzqe~j|Z$VFTEO5yAzlcuQ;sIF<{TnJ|fi9z#8>sk@$-UcB@kX=224~ za)b!58qc`MQ((AApjjYUIm@ss^@@|o9U<1*EJjIvm(M)4ADT661bK}#HnYx`c3`kz z=y01;DCj-qm~hy2CN~u(W=7?RT%ip|CqB(f+Qb=Au;e1!eM4m~&JAXOyJR3?cj4w*I%2W31I zc^!Tnsk3pMs@AxOeSyLwfxpe{46ZX5cubhYKdFJMrE2Rb4yMzNf?Q!N#x2bM2j z;T`VveKVW+0uD1WF+50Kn!qITZUK9m4P%#>fjBpl*amGqhWR`SjdE22Oh&yqjGPM? zn7s^|&u(iJG2h_8q-Pbtu|m+Ma{-&Fm*Me%ibWz#4vnt%m-Y&<82MUX?oi~PsriLd zx3*X1Wp&yCIf)&KB4;i%tER7#U*#CiUlY(`^x&iCe6xviQf95@197E3l}9BOwn7YEo!cSQgNgwBd-7o!!AB9u5=PIi@e( zr|aqAHtEE~60OsHCGlJd2JHa`$@~^4`~UtBj$N&INbu_46pfwFg5R=TZAi6_4mw^f zy6s}6lhc92$_ExOZ%A4sI^W>1Bv&nK%bjk)GZT&}ze{Mp|F)6MzJQU}$|3Wg)RP08 z-Y*!0FSRx@1Sl}@%s3#I>nhxzi@((kRj-aAZ+OAXnd%1XZiL zD~tUutYn^XJbj<2r}FK&`*df2SrS#|tUOPVW9Fmi$*ZNf8m{ETnkScQc1O6V{0LxY zQ=M?m<-oy1(tZnhD^nlFCC`*T+_#BdLr+3TH1UerQR((a2<;!J=8Cz@YKZ zsDZB`p>e9zv!zWF6ZrMFHB5_m>voaHgz|W8uYwef(&oXfJEI9V(OK{r1C;I$roLEZ#%N8tG z(ObbP^8T>69CK&>iiZ{REM680Fw6dNcy)$@g`=;4Eg*wQcvgq4ZrMiesGf!V+XNDI z{~h=e*zsDhIfIGMZUL*ol|_9awH{=#dt9 zz*gSi#BLD4$QaVV?7$%*$o}A8i{&?qd8rLH&es`~2*2sE(eIWY1lgrs&H=S^{7 zGdRflFQkFjf{}3%!%0DAQvqi~17=eXX2TnfnYLbM=Ny0L`anh@CTJ%)h^bPBWzv0F`O~L{W%nb|@cbJ~tX_ENTxXYv27h7&>+4W|pEgew}vY7WYMb7c29D6ex+M1)CL zhf%qOQCPr9{)vNP$D!M;Ork7Kx)zNJH=N|JI2zw*RPbreFm$#!A;r2yR#=D0RAr*j z%SPA8hVM2F|2TShD&AZ>SEEqn(KU5u^zOax%5$$QpK*Z4hEc`fuzbvapyqnM}d1OO>B)! zY9&qTUmPcR9`5OxV8YvEa^;Yi00U!1gTxE zdzbZ2k4~P*xrzs_bN(^7gZU=kww`3;8;8VhG(LH7NPUa42G1?QA5Y}(99(4R ztkJ_1BkCk9aZaYhfz^kB_l~1-N0XvSys`-+?-eJL8HXe%9Nc2CMDfNUjS2?Fj#$qLZPb?O$3;uD_BR(PB2S*GzLT(+T6wxW^G z;GnR03HyDNQIJ|noAiSZGsjiXf-$8K$ z$2mcbvI~}r>|hiY5ED@lVNGG+;$Tu{VG=H2*?IEOi6zr^Jx$l^Jw5y|wYBWUk*8XGI~W8$ zI4YYs3wxw1usCThd8nYnDAD5Ba<@@oN|VrEab=D0En-aDdze&OoV+d_KUvwxp40fK z!*NI2K~alF_7?}(MI2c~8m1aJaEG|DJ1~luH1G)=v#fI9(>d62fq}(Fl)Z$3_W}dk zjsrXi4ZK?p{GWQCb&5QLo&>|BYpe#*N8H}jsD&`491w9~WK z6O(ezg}nOFUNDP6#DPVDN2bk>RUv`3#g9#(q4{ertAGQm$pMA~t}l)x z9&l*5$H%~u;J|!h^36zwEW?Jv1!1WX2UsIq*dDmDGTi;II)(K{g9w9D$+?4Vf?9?; z&5Msb(wQX9QKib+(U8yE%f-RVIYXpyrh`C@mf;nC--C^3Z|STSakV_a9OSS#NakeB zu~i!qG>#a`w7p}v?ULUfwB9^y#dbV!j#vJ1?r6`Xj z57#_E^-3)b*DLeme!hv@zx3!~MCWkVx<`l5Tl|L`a`wGj z+)6nngPLaR6VB)5^e=E-6?l+pRyFTylzx2G#_$~tFBu!fb(~~tI5ej)O$cgM-?8k> zR44gAj*14(COx|9eNKuYx6(@-*i#zVWflq7H1I9)=H3v+pyR;(;-Hcb3zJU|qe26- zhXc=z2JVgtnnYt9rR;WF?zFL+CS-qvRSPL9PWD0-145Z z9bo$(&o+ZWG$;E1!*)hJC;l7f&ZZ_AYb3i_kMPN3&(R_zDbx)1sA~R*)m1eM%T1u#jYI250(ezt~837I0|PRoY3LKSK9dgDgzU{18azbtPYc~ z4-d0ML)S^ad+*M1MsN80_RcxW0~-CCZq77$!*YhLx!~4YzbDJ)PTrfe!SR4rydKki z2B9w*Tb5qj6nSLv0_k104BjdXTnij_xH^8`^4#@}n80lw(O3*_o>>t$qT2=b#UN# z)5nl>KwiYzD&Sz+T#4@jA_372oI4nm7dR-fFtHji8M-i@cyUmEq#cmh!^omv~ zPi^4*bD+5Ykm>G&@9+7qn)1rb|6@e8vB|r}30E4UgmbOEJFLq$#%%uF`lx(eU!ml` zd~U`B297hlcTL*6G8)ub+)lbWuW3Df($LB53X=qf9yh~5r|@P)4i)7oc~)1P@f$t&VON*jy|CAA=8{8{h@`Iv9z|k9@~}% z*98XJ_tW#78@M`duqHG}vp8ueFlJtrD7^nczgv)tg-LVAK@*Ofk9IZ&^&1FeC%iE# z*5~r&d~#R+Py_D`289&Uvu4*Tic9@GH%5dPu0EI0GH=oaH<{ZyG11fGK365LePq5~ zX>B;e!M|pQ*eV#}`x}pOo>Gr#66Kiec`jK)hw-x8WjPaP-75!~#Tof{4%rJhsdO;9 z*){$XjySB_^6cpYCcPI85;lj#_S|QhaKL4U$-Qk1+%9^L7Hzr7$OY)aP+)&1X| z_~m7XJ8=H7;EF!LwtGunbit!E2X=*nJPHTXmvhdGsTRB2(;wL&7n80e(#)n4C$huQ z@CM`B*>Cvbw+C14N64*U#Gx)H^VaYDW~*96EMQeLv%rebsC?~o{d z(Ro|9T8DEih|dr|z5Vm4hSnViB>t-kTsYWPdEm~B#=A94x(Q7V4TmHe8rVu2HGUWi zmozN2KDgrsBkv9e&I=B*9Gw&X9DHcfs1eevF{klq$wSE(4Pq4yEEXReRt7#XJHYn1 z)$_20*Kzp`?EikRYyXyIznk^WuZ+zMYzKDaOlH{p_h171_vEz`rS2N+3T5F9W>g4h zW({aG+2JT6(5x`wE_+qw{oDn?X)J6r8WdET6d0HkRGgHSa4YX%yz_Bg*zBc`k6wwA zI%;vC+4ja^TZhL7b@i?OtG506SrgoGDBR9UU3Y!eKK`D?x(=rW6V0WKz5$An>BW%RTJLq=pCUw%m&| z@p`^%y|ev#_m!+JnU4lv=iBFAYOH6Iz4dA($y?e>Ji<~zcHNQA0 z7&NneVKBYH$l?2)z3Ke9e|MgjqRrspJn8N-+(1(M9}K9Yv<$7Wb=Ry#9WyYkf$ zMd#(*GLwFo7$q(7(Tmx0%XIG~*30UKZZlXG9qXC0HfV9jM5(JCfj0ts7#M$je(tx# z)}*UK@`jU180p1i99(4F!p$osl6L0?6RV2(v=omA4;*uD^7GkPOjJ0?thUfeFnI|htD*Tp z;i($}np{2!i22>HJ2>YECy&Sy1Lc2eCwR+jo1AQw#X7ER(#W}G&{I*j=%`J*$*$(- z7w_$xp!TwxMaXzrz;yec!Yef=+s`jM)M?L@9>uAq$HjS=%eW|U)%?1GorgG#?LH=R z8hLfL%?z6MvTKTdT|-l+{I7&3{nAzi3z_*WG!`|my-Rx3th!`FGN*NhMk1f>A`QmI z(>e_cnAo%wbPW#6El4=2D!rO#!~P{-vK!gtG~Rk}cnPm%~i)#v7k4}$uW^x zB!I!MKhuM8iCC6|(fd7qcN>})Im>CbEC2ZSWc7Nx84-sTs)ll3>Fk(W^ow6(_P5(B zRTmU|ILPgA$YK#sK*qyX@u+hakGXX6EDv`WELylsvehajrBiZEZX%!CD}{xvd`cP$ zOdMhl9N2Br5+1b4&nQ?Vs9~@od7?v8f&wF(N5h)+HTDattzre>{2~9Hs?im~viCN*;#OW=$(skCSTQ3&!s@#fTUbJ-fx(wTdQ;O+wRc|&v zm#Yg{+P`d8)YL?W>3N%`zL}m?s~I+%spe$69>dGUhyM8+%#`jmeqy$Gs(+tFlR(-E zPFH?6x0S5?o4yIK@H@PDpfqW{N@hX>qt=0k{D&UwVQ6-dDqO@YJ>$pXma4?fXEq#@ zbM8IS8>0q8ZJi`OuYyD$@t7GF4-a{~Si`8IaUy|=tT5+KJL?gxKl{a;9#Fw+J=7@gf%o8napNzJ`pK73ae-3ai+;Es@(szy|6(|1fg$Lwh7tCNWX}5f2&C9tjoJnHJ z168#dj{E@^nB9%oS;Yie@>vXfOJ*p`coa05cwFRIv{jMUJ=sG*=8|T2go>b)BD>AA z!ma%x7aD#gaC&QGm~HgEpvE1u;6~o2(`?h;PPeEyB*V3lQ6NG?_@s}TZq>zZqm)Lj zrUW-biHUk5wGD#XGF8j0gMUmO`k zJB0HVZ9mRFLC3tO@^aFzCmu>{57})yocL#KxHMHsm&<0uJZUFE4o8t@sml}CL}wVR zP4zg;=MnJoqSwy}rVG+-Pyb-EymL`7JWW^Svty`t$#W)24S}4ntEze@uQ_mCJGDA~ zs^hw-CT6{gg@UXz7=r93a>k}C65X}nsJe|Hm%Yg&t)2uXVKGHksXvV(JsB*DQi@!G zJW2fhH<;Nu4m5BdS-|Rgs6kkQk(EiMhe>wD;YS4z7?1f~G_2JOe#OBYaQXn>KZA{{ z9n$p#BzX+pFv(`zkUY;3`8IHw5z~f7-U$yHdk=A%Z+w`Xlb1Z3`Oim2i4D#oc?QgK zcNo|Vb};fM1vClG>0!6^Xesa$lX1A=FVNJ$B2;&QCrQOcs>FlYd4gDr#)MCb&lyFl zr$uS~NZ?grx#Diev7`S_Sm_#(E3d+fC9iF`m2x0y>#BE~w3@|ch_03U!N9Sipwm0S zNqhmrQT0;~IHMVq1kOBIB7G&3xt!@yK`qA-1(rZ|-GYUjG6@XqQ481|3?8zX3b8!5 zTF|U^;Q@Q;3}^8SfyRq(ijAgw-ks^OI`ER>OoKS3OQ{(#EggnySdAMFI(r{zGM#md z>7QAFlfpg+h6i5!=IRZ+bCVWJ+_qW3QsBoT%_n*H{nbh~^B0YrJ`2B*Ewfhvk;NZdMjnGi=tBo#=Gx}`MI-ZMaw>kNayC5|JB4#!lu53reSuAH2-D4o&FMe($GfC|q(ab}4V zUv`=O>p1jDqM6BU!v>a#GJL`ulP08XXzb|gW4f$zcfzu>;zH+ULHhLoBe&q z&$Fo;vNrtO^L(Y7>FQwKFZ0=LSIZ5oyUL?>QV5q?Mk$v;^hhkS1TIB5fGx@{?)8Ep3D5m5)9ET*_ieJ#hv8}-)$E#-I-}BbpFIaA+C!& zYkw&6EE8~*{BxoGz_o*7!SYAM>JG3;Z`sV8v4Cw>!Xc{#%vuYWZ4$2ibKsUb*mADn zcELljZ44X@3s~zC_-`pN+&Ez*b=X6%fyH3K)}%C+(Bmx92`tgsK6(uc75Z5t7kIk} zut};J{ctr)TE;xbL4e^QQ=IZcIfaKe&iG|K_GRew3{hiSpnZOu&#r`q^V14adzuAV z4zd_AFb5>?ze=AavC#TX180>2@04Zi9xL}wTd#QSuwchS=~D+~GtMcqbo^Sg{@Dvh zxdZ-eTlo3+pW^H1dl`0?|NXy$HAnOXQXBq+BwUp_#{O)9*sBL@$9kkzByz266mU5x z%+@Hvk;s{KP^{nt_lyPM3q&9(&?3;fO@!hj~&7TO$N2jy>6`*T9;fou=39d7~ph zE18W;QD)Xc7N-Y{P6_>d4qrQ7vxK~7*|RZD=K=dIhg6Z1ELjO`I()m54zMiB=I?Oi z4?4`mq$v19OT?{FiYbxJ>){@cMyW3hJQEeB$edNYvEWk7I{_I*iE9NicLILJoXmQm zsGOZAz-V}trSP|%kU;oaj+iCSc`ge4+qs^@ih;c$LGT>|r6n9!lV}S-@nm zfGvxGS*n|nt$~4QvEh`9^Mf`RpYZe+Ni#V!X^uy;j|Q8k(qg7JYqz=?7+PFlTsDy@ zkAc(b0i(bJ)1<{;g$}ShdD7q49XI1J(>n#hs0VCG32d(dzhyN%@=4hB<$5f0#+Tq-Zwo)%@6o)JTQ?s`1+ND0LQ`C%O03GH;OD_ zR18-5DN(dSiHUb=*0Zw9a`z0B*ZT9BuT_p>k9!9|6C3q>Rj3gkT$dvYwIr}cK! zK>>q<0_PUkTo(~?YUDnwBp{XeWme}{iUbt;* z*aTMBfQ3>GOk(TrW?e8@lA`~&kLitrc9ud_Q+FK80bZ|zyhaMlk5mlbqkP-F?CK#Vc#tp(8s3-(3{PYK|ZnDbEjz(H9H zrC)lVcq@zf7H_op7{D5{gd^mE#YPc<&lkMCgRE8t`y5%qVe^1#vucF80<%UBS5HE| zT>@7Q1M4CN8;^s+E{b-=55c?1afE)HZ#yeg&Wz>?I)H0J={GllTa zt_*GITuujh%@UYQ8a!njs)QCB8#owE{V+jCnC(E==`wbU{hGc-HA51R0AeQ(kIAnxVV zxQkz0Wz5S97I3mHWU^sk={vx6N2>>md39^`-a zOgQ7A*ck_r0}kAL+Q$PJk_;AbyFKLVdcw5mKoW=QNw)JJUKE}35iM4k@-N+cYRF^X zEglP)7YXX$?ll@asKL&78<}&Rk&H0SlP}(!$Cd^A4oO>AR(i zG;=9Ymz+cqhDMQDj6%l>mL^@!6EBMStiXGI>dLas&z6KVF^gL? zhrOJlCl$FMPV#!(asBw?=UI1zG(Y~zXL=*f`%;|rgPxdA%r*?1#~cnjC<>dr5kB{T z!%TsBS%RQTqrfSJ0L2YFO*i=U4)WW4=P^(e-@(gMw?KgFA@_#`{2w0h7aic1Tgdlo z5z8HI%eVwaIpz4Y^sQ@7be%|%Jo&>z!$ER>c%p#w{EXv1cX+lt9gx;n@O5dF%=y>A z#L~c+vw+V*N8rx_VIN1KzYUyL3qA)h3rHMf?fd%m&^6Y3g1e#^*j^+^dOVzOZ$#pRMDMstAS7EV9ZI!*txHi-Y)!AcJ7za3|5}&(+qD| zSl?*w7gja=(ezf-xBI~~#s2uxhN6-MTK0@0PqYpjI37Nez--dM^=5$xPvY$t{$hU{ zX8(P_)Ab{wh>_{F1G`kBz@Y`RISvXu$`pCi&}Xwg zvs5hL{TJtd+92O3a>2O+W*pxX_zfBitC&p|GjQEFz`N#WRQnaaV*<-#Co85-Zd|y~ z^Lpv?l6TMQK4%%-a9uQ!IfTJ?s?6%Tq1B7XTSIFS_RmZaiV9h4j)y2~htnfcN# z)`|x=)@%~-ImpkF$YZlmWZ6P~u12;m7K}~`=8K$@^cX(>;9;|GX7y2Ut86fyAeQ@9 zfjNz#-|zujknul}O`?j67np{6Em!4@x$M9%!zgvYF;>}P#ZSd61@E45g}>zazQRyq z&vp^ZrNME>ql-E#XEbl)kXz4@p}_txfpf(Hfda)f$}F-XV%!diJSGpAuW8=ivG(7i zUHn}?4k%6#`=h|UWdZY*JUy<5{6`enOB8gT9pLj=C?fNfudd;PQ1!<%Dkc)viEA>_ zy9Ik=r+VzTZXmOs#lqpc)&u=Nt~;C#1TSn5JmknfL7I8S0ghh`vwt`6#N8HQX%tz| z!1U!~|DQwm4EQZy^{~vjuq5#SyN3dc#{ml=$1Kg)lk*z{XE=IvUSZpGdScY(DG#-k z^v+J5Tk`9V_7w-ED<$vbRQ^q0HTl-LnN};0PVt^BC9pH(&oZgzlOjweKbW3PWvy@E z=1}D6d9bf?q5LgH@nZ_yeGJ@22YDP8Fu!rw(6C29Yu3M04-P!7>d0TnNTQFj~N5l)x{M zD99oq@ah4-MgoI^e_HI-q#$ zn}O?21A`8OfKwuy5krtTqe7mF#Eb_mt?Viu78M1LSonEVbz%|@vmNHOD$lanpr8>r z*Ku)&Dfd-Ifvo{cW*9yaT*BUOVw-fxM_85jj7|5PMrCvVc{bG!Kkwc)FPVRFu}ANw zh7Bc}?aKr#Vk%ZOIfqKRro^1pYHsEci?!@XJ-Ey7oH+vlBPV!nKTmFAi{hQx)h zq|4l{CMqQK&t>5;i10{gpCZe(#LG~+^zfmB>@T*fZXhwDf$5_Oec8j4KANhK4TLUERbp75zKGkDB&ty&G4DW^6{51!U4ZO6i=~JP-_(l z4vSsT)Tt2`dNnxdNL75NXsN1QC*NSDO`Kkg`$&! z(iIG@3X2$&I27x699B^P_ zQViK}i)k(6b1CNIg#nJtf(923WNp8i_F}<-k2f0%L<=04*d=t1-kN*XlA)26=S%{# zhy+K%LAizm_QTAqFCDqKmh>obXihLlYG=)#b-;yF-@6}=g9E;ae0^y5OJeG} zz@H}^oTAfRL#GA%tjxH&SO1#CwBDq(5iT65jsY$qycHJ?@|QAvI>;g~(daqREv@LJ zgmlh_gTjq1K^&4TERWceyS_e*kWc^d@uY5B$D(G5x;q~)D%4thbSx=f@K3>kNtJ=g zndyx0uUV|?V-p&)ot+I{ELrnb?M|MnvqNLSs+C~}7jN*D`s&25!yv2K5KwSX{>Xs? z&Pui$0-Z!I-Ertq@aYieGPYG|U{dIr!`6{eJ>e6Fvd@=YEs6(k8FjEr?s%BRX`FDO zP1fwsMIOT?Hq5*#eL9akl%}f}bv&ElJc-BTX@Z5nzeC}Rg}YufNj!R`vf+ReN5H{z zB5!ma%y_@|+p&lvU1pCo#QIpw4>Dw0W7w|H(;~zn*(#BA zg?r=WYab>wGAv*aTB9Vb(!t1^6wn}eR)Al5!UA?j z6$P#{3a*t8(%H?PbP4)JGzrfN=!ouFBvyWaf$heFRgs|!*vm8;6&5&l1=u{6>{nnj zXjphemFcmd|DQYC4;c3G^maRc^EkEZTf+v0O;bfz zF3?RDxe>W+60?;~6Q|XIX5B@Oyn#9kdA1*5HD$RtSETKbK%z#A{J#TxkC~_$?No49 z=2_V6#PEna_{AZiT?XvdHjntLG+K2-F16mBz>vw~a4tJ+^QmiR()>NL4=@)zp0&y_ z!OXw%-13n_-uwfC96(M+7@lKqL2qPXDs5Tl2C`O4$0RJb^+^u!_traduyQFA{t z@HjLIHtgX2=O`!8@byEEf|S4kUMay2o)iX7jU{Y`5{9h@&*|og2{@c%Xk=uISist+ z+bFHzXdWi0EMs24Y$CzPnbp87wSaZ4@dCrB3vwG}tQFa_;c}1{RRA8fL$Re|fL-JcbFsd6gNuJR-x{B#x+jOo+f&mdz z)qf>$I2%0Mef#pflLd-b9A6{|v?hqj%~;xQdFCLe)DH)CyMRW{5C?V{h6Mfa$i~Q9 zL4g}RmtJ(~nzPvPZN`=qE@qj4*(-FH`|)HjDue_zGMp$?UUKV1to($=n6<(}8Xhdl zD-LovuF>UMc%ixSpI-oztpO9ON<*lNku;a%4n~2-7jJmpIQT2qnQdWgb#n;ZJz1-- zuSEJ0drq>pQhSj|Ps$vP17+9sR|iz~>Qp7a-If#RzSOm6?dOAHvwk+Jro72*OlZ+G z65D^lph={~Ah>+Hd-}V14~4dWXjYUmG}Z`Mz$Ed5f$?4nv-KTEfyD_f@)jSO^>#31 za`=caYfP|r+?p2Htd!lXFMEyk2OHD=nO+UkKR2+O9cPlA;mEE~Fq>5%S+nYZs4H5?3IvZ(2%>(jgWOFq3jcSv9R|F=?=hFFeI zzD%0yQ?)EsFf0FSU{o_$B=S(uMgGuXQ?)0nM3!!dlX$deuS127KNDw@75NOgN$tH_O2Kj3d7uNA|p|m!XSq%3L-#k7E7MlF1em8kZc|b;hvN zi|xZF&W0P!JPaEctQWG|mWv*Al9EoH z!Dt*H)a9~7vaUzrn5Ua!ZvoGhTP|`d?(OA$SCGG=#Oq$Wt<4lye#v`xObaHy&Iviz zxOKMXp}i~io?Vn`VW--&)%ta0x57Ugo<9dB_qa0h@XejcAJE8?7~#a4x1ls>f|7u@ z!sJhf99Y$T8o7cCm_&OH{MMcFnZGQfNh4<=Z*ET#lk5*?We>)1v#$yqsvmZqnLgR! zs>1SV9j`3s{Y%euRMfh%z;5-@PhU!&S@iPtMzn;+_BPDrWohE;*ne?;XW|-rV?mb^ zVFulXrVPVIro0Ut<`X#LrfVRf3?tp5@pu=y>h-h0X1N1(N|fxY~Ilg|hCHUo}& z1~$5OV%ZuaDOOq zWACWH#AF!2@^f)%E(1r^2PVD-27UpidWjD9gU0F!;UW__t{mX;NRaEB%3yY(R9}pB zX##suI+I3%wyFq6(}iZA3(SS$d`}p_32xxid^?di_mglDfgx;tp;1<3|yj`9V}at_Rv6BuO^ zf}0-{8C{5&e5ft#v-+xunRd_HxXmU$E3XYy4%)cUyYWSQ)d?w!1b4##<{gtbx;L;p zKj>gCiDC;T zi!8{NJm78m!7oOfRZ4*4{DaahK}@O(7|S-4_J)Oj)%RGTS2SekN;h2e8Op zU|?)u5Itbb-@xG3&M<8f14jYly_a+KGUaj)aO7NIyX3$nc7V-s0sEy5944MT8VZqD z!kE|+8Pz|qFIC`eJiu}!fnB3uo{uKq#url_D@=Wy(AXn2|MN@X$Dhi-zl_?O!CxA{ z9_PRwvtVV60_Qxg1$G7tEFLf|iCS>#1{2@b9=@xoDJBLXQVRtPSc(}qycbMgFoFBU z1-`crc#}62%DiG}72s`lU{zIErdz;l7Qo1&z|dbgIehDyDVFSMOvYat8Bc352qzSn z8!-C|)Gp)9>@u3T`X@tk(UMAyb#B`=);TkYD==?R^!C5yy`%9TvqJ;B;e^?xrUE<( z42LE%_;27?GJ!*ag;|w>Av%jmdJE5y3v4zHJe>^8KiqjQEMUJJ$t1BwQ00Knk^reD zQ7+p8CNECDS2w3TyfE*r<<#z|##(d!M@%dAxL5vl;CCpT`t3t`O#pkzgq4=Rr^dfr z`CoFH!vkg~=~auWdQ2P^=p`)ZdTy1zQ6%)2o3#Q<{RCF`0JdERxZhsjf7`$vn!uWN zpinv?ZCe0W^8t=30am{Y>Ctq>G~TC?o}r4R^De0&f4%%W0$Dui;wLB4eH_ojLZrw)f;&H9ZFySYh;-GqWHx@ zE!zU-90#uM4YC_PF&H%Hsx`>!GB67SFxhTkPgp)#>;SX+1OawNW(@~MDFN1|f;P)w z?n@I^9P3_j?DstJ8GN5C`QIq;_y5{7!Ha)l7Jo&-=8}e$)~7em{Wag7d&>f^EfxZ+ z9#!|~2{3!k-lBejN%jDv>;o$o29|08w$euK7YnvJrm{seur5eoQ#ioVEfAY1z_B>M z#pf$?_sj!bJEeA>WY95SZ++0({DDQhfzjW9-MoOk@c^570F!;dqTK9R-j|DoXSV0O z$mBNDhGud1~B+t5pr4ck>Tz` zHjx76B^%h88B3Rn)pLGuU@KsX&JvImW8x8DVm&C3Cm^@ff#e2D zlqdc9H>GP9pGI_J@JqgLZ}@juSpDGH`>BEbUyR?4*_&r+ah51>#u%`dz1dqOv!#+} zf#d1pCJUI4OR(1%a3mzGo-u)QRs(DGLhfA~xc^zRHaPGs-N3S)k@4FP76}0cRV#+= z4~x}#CzTekmwaF`Wys1;5)2JJsB^)K_hy@r+O_GB zeZjnv0A_~S4HE*GSqd3B4R$e2VC>i`yVUI{zexSkuxn|)M>AxZLhdlp6PngVZ`F7MT8*Q?$(@M|=ln7!%o2flAL-4Cvn=e;>^-o4qv`Z#Os1vXtyrw=Rd z-@Y*A>;=0I7aspR&02kdqwWJ^oV20TTu%A5+>MszTB(=*Zj%#vpx)(~slS(jw?|#p zfw@74*))Lh$PtDB1`hR^6U=QVa*7;s^jgwW9h&KSWpl{s1!{qMXVj;L&0c+yQ7<5x z`vY^20hiJio*aizP639_uWB04*h*W^-s7oH{M+#~yiULK8n0pEWd+{ZsX65i{%-phAzT2I-4+?S_{rFa@cE59I{JMud?+kl>413K3R*Tguh0g7r@OHDq zf(sIL55?_n^2kp6vg&641`hcRjA0F&(-TxSFmTJ&aw;%HbSjkl1u*ze_L|J*aADVh zzL~S<%vog4$!Olm*f^o?-@2s3&WEM$zCE>kW~PSCop$L?_uL)-TH7~Hl-7Q8GGnU- z&w-3;1{O1h4O`h_mk2OV_GHYtP zM`a4EfdU+tE|}Ika9usXbCw~tt?tEzr#$;OdAsGJoXTA8f8e?Qf%oYLUPilsg~pBS z>mF1T+~9flKycp5IEUjR_gFaRKD;;kVTI0?wms&?3Nu0+RNR78o=QjrIh;Eqz$O`x zQJlbRe1N6=0;^RDbLjyV@d9JrS&OXCF_gN>TF)x!Ehx3S7rOGv1ofVTOX8i=u01J> zIP~6YT|`}`T<0N&4;&9KaL7*J2n_hgbmt3)=*CO@3C!lt*zy*z`!FyVW?oC4%aH$% zf&Igz;<&ny97g>*W~l}ylbIZ<3G7=x@NB){FChQ%&lG==`~K$~c*Oamj(p&i`NDhD z;HAR(+@~Md=e&D)UHbS;uL~mUSe+KIIz0FkrNC*p|Kaj~o6B`LofkZ;xWLhT;ke%i z)}{ySa}^|06IdLTSQ`SkW@>Qw9Vp%Az*YKz{SV7mVKXlO1jhc84D-X;eLtl8FJL#l zz$m$VKI^{HX$4MJ+9<#cM9bw-4mfU8aZCB1{r@#_%mZdI~$)|xu z=EAL;3)zDl7_YW4y+88o0-SUDZWQnw|Fd&*^Z{1G^{fdu zS042LY$AV?bL~TC1r~?fGkYJfepcqalEA&_0q2ea?rj&?ue$PHIpE%U;Y*JKYi$90 zwE#z10GHk|Q!|5^D-zi!T+k~2z`RmP>Q7@qqXMg*I!kY>`ud4)cr$k}Tx>XafQ?(q z!s5gNg%(zR9=!&G4+jsl@hWM>7zi9{X5&|q+H+u$D>s9=5a%Z6Rg#3HHUrT|&m4oLO*w%I#WSJb~RBK>*$tmLckGa)qaZYd67mwFr>tZ}} zWm$8y4LlBEOb83Evr}iw`k%LRRaNCCzfnRCC`nI z4tL8kH@;!+;4^5>GE55bnx&g{#bWxbV?83KY1lLHSs%b_Xij8wygkCLYcDcamxJ1t4A)~O@ zhUSS+CUqZTvUW@4-0UiGv%uLyqEW>~&|M|aWz!iZwY6KOD7wwG{in3>HJ{yw4cz^E zc3kLQzmG|HQRk6a6I!?rb4+gWIL;KUv++oYM^B&QjEQbiSxb7ICXpyvLS#Y39 z`W;KlM3;`rCKg6vhEwVboFoiZF#BE5Tsco@L$ah}$IP}FNfM0(B3HgKxL7PJ1W#~b+Ste$#5A*!kx`6eK|t%G=1!*h#oaF!z1-34Vcnp0A3a?zf# zVtL;x?O&HV#R?b}wJA;5@`CkS{t877t_TOghD!?E97ov9G#Xo6j2RgjnnD^hL|Ba+ z8f7jdaF)$rWSM5PhDGOSGrPbAW`TmH-JJ(oIISG{7Z~>D2snFno?wucy4d4-L6N_5 z<3irA3)t=66clt=ACcoTJaWvIuV~8VIR-KlcZwg>j;s3UB*QXoSDmDe(W+qC{Xdpk z$&0*6aOh~{i!or9b?J~y=W%IS@v)WHK#|9MM#FiXh9)7A^GqTa99BfMHnOQGn2RSI z&QcQj$7<{099o`mEYq#9kwu|_k&T0a`;KUf+y@1&z!{AUYzCVDbQ5a6WOMLX>||gm z*)&<;gQLHV=00|ZA_i`W21YrD7C8q62Kx=pJXH)GH>NNN)F!xT&Y0LCIfqB!rNc6d z83J<|R2W#r7aZV~xx&oda~5=N0HaO;L*6!CN1j)XN)rN@L>4IIh_5=TQ8%GYW649# z+=}$LTMFl;pDFTN?@8ycws5KYW-Qp%aOdeGr^UPPaCo-tT*P0~z}#jwrOaYNkLbo9 z`ubIY3;!-+umTCIaY?;@CZI2f3X6Ap=6IW%y+;b-<~ zI3hXop!~m!Hq8=eBRGv`ZS1jTQD9(CXt?rX1H0Lah3sVvM`Snz>{vpst_t&I&~oKy z2nxH(5YM`rxoyG0ReA>)dSwnUs4y@DeQIDA$Z!yTC(;^prAg$FPOEO2*DT9`1&k#^ zt|qG-nVC2qFq+R$3|PAGM!H8cFK5CN&LhVbEUwtVV!okMjRg*Er!!o5wN5t6 z{YcJ)CF#M%`XCe7e<@gZh6iF*-L(H%mNNu!Bc8 zuSjy&g{Mv;99+E`CE{9=ZAn|0%?ch!+P`YCJfkRLb;5yd!A+(wkqpe78yHM%6*>Y- z9Ba?7WZ?Js$P@I>R!O@*fu&ASkZYgdV?jj)RVF5e2F@Mc{I9)^$XcCXU?|wSiou{E zT+x7qU4h{c$B$Lv%pXEIo-AbJV{l+UeX4;;B!NLdAX}i|ScBD@hwRc?M>W_4y8>96 zs#kSA7W!wv;V+WJCQ-q_TIIlFS(qc(JfYWwVPlJ=OaYsX$I6tu&lb$F>XE#n;3hr8 z;IPAi11yFOjyz=w&tyL|aO9~rWrohaWx}G!yRNlZ`jbL>^{rt3jtk88>>t}DFCN_9 z)W}m5Ymgziqk*|?8Y8<fsAJm;C#vF%}29`+fGY$nId9dbAXS{stHB@Q$? z1~SZIG0^0Fb)myr7v0#CHWVeW*3p^dtX>#L@+Rxe=YoK<^`t-^sxG{eDI+<}R$q=8Z0w}EX(IWyyt zKnBSKr(WX)jA}fG_*)c?i03)nQe=L_#ALCO!HPkBD(j&MCzxHl&VM(1Goyj`_?>3y z83)-U&6ouYEDmy;oNE$4!N8{9z{qoK5A&qOH_zBgFwMTa;gi{lg}nOHjpyd>URVCn z(S3rX9&gNr7fJ;Sp1U$sT?oBU%=cr@iw?Ul3t}B&o}OZ0pWr4TUvz;<%H${a>5Yxj zZw%VyQx-7AuXkXNabV^bX!ytG`=Ws_Z$YztLnE8)frG+&0nHJP2iYtcoP-M;nx&p3 zu$k?N+Q{I88Ztewkn=g$##%wNY=5TR3+;MoF zaKe%Gg$S3r#~jIk1{Vd7hJ8yS60RLt;I^hCQP|BRSx}r~=P`4=x(UrTXRIp@o;dv` zZ2P(}-pqi*zhmDO&ER(`FuBAn&XgM?X>yBIN26&W^RCYg4N4geN*xUgxUCldwp=io zk$*xXN6!KOPHVdj&FVXt9ac2oJpHf1YEuKxfd@Ic&R~0(*rv9C*<%B< zoxHT&0T-AZCa`65wC5x=SuJRD5?~41VPwnE z>bhf@02kw%4Ceko-XsPV;RGf(23;WoMxh^0Vg`(T+V&Hi?Y{``3|HA1ai>w_L8FLA zqnrn$sYjz(MDq{tX65YZO^3wW7B#y6d(0rl;K1l)C~{by>EL{hhGQ8%2N*&QWa%(1 zf81pGqKWq>llq2cgMuc#f~G>AW`i3|M=mq!KbT;?r^`NqNnL^2=>S)B0=wIT1`fuS zHpv#Q4-9-CnxZ<|&L%KDUeLH?5p&^&R<{+b(FaU~EciEnQrXP3nq#uZ1|x0G6MHgG z?_sz-pXaco#FeIcod%8xQnEi7WGmdicTM1Nn3sE^P4fhU^lXM%MlD7RM^qCO7-uj( z5NOFrXe;n&xBAhP=D}y3z~tC4kIR!mh(mx~fRSZ|GN;i#4UdL^BVEELEIvu^5B;-4 zmSg9H&!^U@SWeX59Bp!1?nR^Qi$>*cma>h4|MUf#{wkZKq%;aB97t_x;K*RexHI4N zqjyFL!=x$4GFf)9h;+(VH2HHk+nrdg)yWo9aVD#yE%ySOzQhT~1}61{7PlA7$^}gN z1}#wm9+@|qxmPqh{a|)l(dwwtz_y~nQlQnCt5wyYPv(XSUk8hU0JGtTrq8$JIXjmA z3S{Uwd zw4wFvk0zdusb#Ah94(lgKd=~TFqj`axTAt4J+V!!fhmoH#p0EN^o~ZSmkU?~8X0EV zasO!KpJ3r&(fD{Jt3ySD)Q>|Prmml|kFN9C|E2k#>!vA3MLLc~tL#+XZ7;pBS?MR^ zldeXw35;R^jZ8d@8D~x(h+)`oqWVv=;o+Op91|L3j$TYpY1qtsNL7J3`ar9q>Ltas zZMh!o>KhoXB3R6JG+O;wDihJ9`-3@9qs2plMd1X4##ZK1mX=Mz3{pFo0~J_oel&#s zW@L(JP?KPd$zTiQXqG(D%f@W>YvLXqlZMpO20Xu4F~11jY{js7)qIY}oO~A=9e1=w z?O^@!q(SUOgZ&Ctr;HXi24?<-Ci@c&su4_@FPilaFi0U~NsC?ibS?P~UOIG;Td$(2`D3Hh50R8FJ4J2#(`zoe&b;W?)$ni%1M7{8 zrFZ8yzF;s{YEgZ`9P@!K<^gMsZcx5QyS2t-myRZOi`}XV_Nu;EEAxPXU!ub*qD8TU zNj9TVZNiOCS$Btu7WIS%^@b3q0)NSr7LNyv>;Vm(CJgKvHz%JpVEu5J%lam3M5Nvl zCuR;t#RJS21e-l3wCFBiw!XonFTnhYh1rqA&YEK?XTjFgQwN@@U{mh?l+46!JqlCa@sUI7~7^ZBHG+;W>WT3%p&(X}2pv_zn z$GKv|Ygxa%M~`2Az5H@%Ttc^W!tFR7?ymJr4O@)k#W@(-A{f40ZS;;~;#$FI{i1r!$@gMAaOXDm(qB`*iE7 z#-pi>^Q-T0obcZJy$su zRu?*)0)t&1&R=qKK7&95i^k*56TGc?Y77#7GZ=ylJ?`=EkbHjjvfH*e=QO|mXOAa6 zRdbAuocz^;ZNaWjR~h|3EtZU6`c|9iyRTXDMFZD@2II91whC-w59UwL=*Y2R5W2za zHlx`op(U`QMK7VrSj)m}Mk5Dj6CYP2!^{rb2o`gTXorLc5(NuV8yNb6pNVHQhWt78 zX>-88?0wM_jrV7W?#$u4m4(ne^8nKO?fi! zZ3YDf*$*sE5xE8xeU5ea*3XKYe)rJCt#6&oQl^{U?CObgabr*rm}@Ptly6s~d_hzB zpC+LjPb7adS?^$0DQ-*>RLwc0Hm_s$XA2_-0ro-$ucQPfrQ0`MI9P)e*)pp*7ZwPd znKAQP=sSfRHbI5`oS|V8qMv2lV-$6;&-~`a(|av)&Cvs2Pf4-O-Oj>u=+xc$zYPDWO7}iDs#BF*(CoIOEt;X#|3?e&^8&v61yMg*wDQ{QJZx+}G;jrU zgs3(!Dx7SJNQ$^6&ydg%wCKYJM%6iFKdgvMoUX^qx}j-4Tntu5_^xXYb;+oe+OrD{|;u2 zl(yD6ZKf}0^Ek|xy3p*tW5A@NO z;)CCZ54`VN{8xO9>Q8e1$KffZn0h0$gYCkSznUDy*Y-$FFFY6L+*WW-gS~%HP<@&0#2>!EF6O!g@zT>>_3@ zj%JBHwk}+$w(1kwDjIAvI`ua++iUF8P+(1~>|{w`SZ2btK!8iwfbri^2B8hiwlf-? zBHB^|SX?@qEgI~V9{iAP*dorbc|tei7QNSdzZz|ub~HEbv|UHby<6OWttHx`tmO2p zQgZ^+S}q2?KizB>{%!)J-3HcZ1J-~W1tA8kG0zKP%vobHT0DNVyY6U~*}m|U=?b=V1Mb9t7Uu`eHYa|VJos^&{losL zNB@00b)eRD;_9OlvRRx@lw8o%sB-L?p}y`v^YvXK3M?#1#bpJOrs1WnAK3F5tqfX& zHhlGWXyrftHTpkGu)rm^6>V+;ENTr*L5i(`4NS8>Nj5ZZU2=-!$~>)IznjH4-ac1g zwCec9z|g@}&Tw<*g(Ut3>I`B!0S^u={MW?HCK3>maG-&uor6cm;K72$&CCM)Ha!mx zG_bO9XSrksH@h&gaaQRh9pMn#7}zbNr|gf&)!JiN;x$_wRp)3fkifjkCp@)x15NTv#a6xc(9p+ zU(UMXfWn<*cYcZHM4l*JM2IKBePh5)J&_doLANT=UTD8dh+t}a=-aLce~6^M6I<>T*9)6nN?tc zw0VxfNe2gKVKs9OgM>#%SQ(}JCH@KsC^#~3|0|Qbnj)0Ir|D+4z#+A6oFGG!b;FgVN2b-Cy)Xyqcr{BPnWk9tRo4-USJHWM7e#R6PK88~ek9GKi?H#7ul zsRk&8GI1CrF>%-&IMBwoWQGGrxaEZ?CiQ@?&PqpKTs)+(#3uNh^?i-YDVC>Pc7{cF z?P>LpDY-E9m_Q8QN7XpzFA8e>wli)zC$#bid8SnCc*!aoW5ala-#$monMbc>Te_gY z1U5&O6@N8u=d3=-Chf9*Ws9Ro&x}tQMfpA-4+^>mD0-+bt2h*6+UAn%Egu}C-l1^A zH}SZ}QiDe<3TLKV>JSdFaq?2WHu2CISz`_bVeU?be?f<2d<+blef92qaFIUqMc}fF zV(gJe?m8C^vdh{`Fld*nVqrYsEWr08fW!45gRivtyay-bl+>KK_{Ab7oKOx8OjYjZ zpWqVWp>yu^0cO=xTMmTsJebkY$I-TAs=IB_Q3nnc8wbY72NOQ?9Tg2elVTZYHPa`$ zPo{`NzUIloR2io&6Watb3nW>l$MJoPke|52GjdK$f#dVJ6{j9Bu`g;pa455uX_eFh z?k#4w7jANWGV7waL7<1`vW82Xk|7F>Exhs>1uhbG65(CuttQE*72-VvxeS-IC~=7N zObGJQtlg63tvVOB*f~W`L^#PV`S-%Hk+nhRuyZF{hlrv( z<0Z#4N{bynHOqGEY-V?3a|sA!nLOV`hWmb_^v}fs??#^ z?d9yyPR9SI4Z^}d%H?se%UpfTtZs4U1G6|okeJQ;kodhvZ|+xKY9y_$+!HO|e#U)T zT1vdQ`KVyXxIZCl1#?Go@H~cRah5HG$3HX|0Q@z>h|SMb9{m zd>T1JW-zj)HMH<6By!E(y##2CvosFFfwmR6g1f2ApPP4lb|hwuZRc3W1o~p<~Mwr(uc0Q_ccwG+Q{Kq~0o=}do0jUO zrr50eenL|-d6GFg78dz+b;>S|tH)^ZlmaQOSeU2WpitPc?) zy-o|*$^=?9c^p}J6Amzmeo$r0a5!jCu}I{0MvLw&MK0?b4Ay2Fn3Y*BvWm1l;`Up? zYC7#=`?jl1lFd6D6{ZDp1*ND9R$FkWcsyk0dXs2tS8+&5Po(9ZXNw9q$A#)Q4n1KT z6&b}Mjwo%55Md1oRFSOhU}X5vk#?hjfpf({IlT`Jwg!ipbQzAw%`%L9I=7)$;-4#n zb4|HO|AX#?qBEx$t3=jK$a}${r13J;Nq~vlpiJb!40oFsA1}St6OuEuU1F`%W~g3! zSaQmdl}g_pTt3ARD&%zH4F9(x^^i|J%vw7fF8n#Nf1xOEgS_EMMxRFvQ*9L3N)N1( zx7*Oj9n!!e{vw`TV@Hc`=wq=(8(0;OTxjHoOR&~G(7>D1(8!hI!0uOYsKO{gjwgXZ zpylX7xh<8=-wj&ka|;~E4cRmyEiy%!qk^Ad(E%PIi;rweHZa%72n1XTN^xD$%D^zK zu$|#ggJ5?98yk}VYvzM6sY?v)aw|5u-TU}XSx1Sh+pl_kzf@1Tn z6Yww7@|k15?e)dy>;4tbe;nBVwn|*?p{Tyf$<;jmO^S79r+fI{v@}b-II}|L#mxC( zm9se3e44thM2jJGu~6ed2Cjtyu1&@l7&!_aFvNtz|;sbo0I}YeFevo6*Nnl`9aA06#xWm!Hr+HxQ#cDl< zPHvkioCXXo+)NBH3?2z3cdZ^dZ?kBKy!ue^(gAh0gd+Bg2~1q73Ctyx>H4SV-PEv3 znmGCHkBN_3O?IB@HExm%xFoF5QCS_avW%VgMQ?G? zihb~5{q{Hb-+ZQ{v%>%RPy5GPz#{R5;jYCgezzM3*6A`ZvV|XD_E^xQm~ofKDP;qn zY(Xij)`EbQ@e0h$0UE3wvCeEKIG77sn`EU7n}42RV6b0T#=wy9n6V&*f#JakmrF7Y ztN{<0L5Ge7$H6u{G@tRpLlk%G5$~H_Y5s?B6;)hDRnHd;Z&d5GK$-&?t9ooZWyu^3a z6b6P4kD0QrJ`rJo54iVKhP;ey<(U^U+cW0#gn1l3(`BzXSsuAJ|E}}fhs^?fih_^g z=D)r9u2$UUjL#n)d4Ut*nlpnk3L1F6_?%zC5MFO zJ;J!+u8aw@^p(53Zw~TA9AI^5mY&kU>T=Lng{f=7OlisRgBFp8md-rzL-4afzx>Ha zqpky-JG?)CIl*brAgj@7^CQ9b4zt-kX2HD2X4m58U%YqhfaM z@RGH057lUborw#|F0i&W#m--Y(p&SMa-vcZ>!J;-@no|y$ zW;n`PEaAH%+ic3f>T>r=Ns=hI`LLayJ+LUu;K_)34_1|M~#~YPi$!LN{QYk$iU#jz`)bM&Ev;=$ASM$ zBkv9d-ZPAHFC6$R7;-A}R-%4m8NUVa(Xd=wooC zOU9uvvGuv*>1UEWO*1_DPADAg5P50KV0`bvD^?*{8zz$%lJk!)HOo@Ad(zzU%(>-~ zYHZcx`6ti(sBo0wna}@*A^i=HXp~~c9skZIH|{q+UGpYoBq)lBH70Kp%8FoM(`i)j z=+*q=R-n;d^g<(J7L%391^zn+gwbXRJG99V!(VcobOPFgo2hBIA<8o8hSJ;cSuN%x=P@e4-(+<49M_@>{Nr z(mGBeQyMnh;BY?k*pop#P~_FYEec^`GCW#gP-q4yHh4CO~Nl4WdAsDE>Yuk2;eOgWi>b`Z@{F$a!BFJ zL9vR1;s?5Tw`dxh98$<|{G=++*22d5hDRW$Q7ng1;>sai4=10v2H7_U*P0%d6>w0< zVl*ybw(eH!@MkRaSYeO7PpM$ubP+FWv!5NQgQj`bRy%VfCrOc!wbF{ zmsW8tOW4M+MksXcFNZV+N8tnF0xu3+e$>Esgn_ppllu-oXGsI^gsDzVO_~c1ZIsi# zQMJ}Qz*(N5QMQD!`)q^4k^m(K$J;R-(pz$kmmD?=VK%?hBrd|V$L{cfWz7~mPOG=x zVm+`}zxCMFM+c2FoXr!Mt$PkJ{Na%Ob8yeE(0nrozB^GCU!3{=98^AX;DIbh;F_d{ z8FxEwoN(Q;fK@E;d7<3%+ItJ7TsdZ(h%m~Em^E+CM-R_R(S7T#o!aGGc5>e7Jmuxr zbk1JNIeY1;(u156XIfQ`FbY06z_sGlD!#X?lyZ4A){12?ir2grD{$QNX{p#h9o;LJ z8u)zHatFNK^2$Nt%&cV%&o;c$C~8sDGjT3^)F@lxz_vwHlf_Aa+I0WR(_EYg`*wo5uR9Eg3&BJR8)y;du0pT?a-rBnLv-y>A1(suIY%S8-CPsQc-aFe2!J{&ztkd4)ASw(K+u@+Jcz0 zf=2Nzg|ZcnvJH$HTQ`blG>RX1{jE=b?iEkICk&z`XO~qp6+Bs?so}1u&;HYsS>gr5 zKRKQaYz7BS65UHz^!^QGms>DZ_X^XVe~n^04#=7~ofB&kcR9d1*(i&{hv$N0#Q(vPXUA$&`jhi3UgM{KoH_K8m+6iV93t z_~EGVqfuc3_n?9QK!(qw1F|z-%FlQy`+`ya%0Yz#>H6;( z=gw<+5Ic=e`KXJ|8rPQPE-|G>&fN>6bPjPEhTXXMCXeG_Lu%A%g;oRplamchk3uZ!0n#7sQWL7jv zwS46A$iFbnaNZkMfdYM*^FHUo%u>D0Qq2!_2Ag#!m#O?Y82tZWT7DyY#sPMNgJKsB z3jc6;YUn8Zq@m(WqwI!*ycaYq&NhfJ9Flc#l)p3o7DJPXy&Z=$R#yJ?pydZwz$oI@m4HG9j-a`xp3|U7S{O%u{ zq&Sh0hqXX=&0MP^3zS%$1+;3~w;ntwz31nnXR)krl{UW9SiD$n1?vO`-CqumA`crk z{F$z}!{RXW=|@|aneABq%`|>@@$&Ltt?mDQefg)u*Pzj%|E=S(W9Rc&`GVD2aVPl-1~v}sf@pT-TP~$%97K3_OJAAr%k^%Sk;D48x9*-4 zW%FTROlaV{&>*{Es_X>j2b-LYfBe?HTTF>n^t zaK<=r-e6D`xwN;j=7jtY3y)p%0;?>Ie~5a&uv=o>a@uiuebtKYDhYKn0iO8e-JWUQ z-;#IiQoPcnafj)%`To!L_7{HJXdnKi!0?;>hQp-kT&4dma4BD5t{c?~ur}oN3 ziRP=ZSFXfDB8W#tDr&U2!q3e%xi0|?GNm? z;E@P8(7e#CZIP*#z(PhAF>x`2gocDhW_C874FW<-PRq!b<-Az=G2vhni;QeROhZHR z*_qsY7B5~Xs5i~97p^$+QsG52JA03kY1Wm4&C9&y4J-tkT&@a7Y zjeQLdnOg7dPQP(6d3pa_eQzt1o1c^U7TZ>^@iH+jR65Egs_1azGEa=|i3{uxCj=~L zm}Z=C~=1r1qYb^acM}r z?X#bgpwzCKv4OGQHY1~fncsqesb4~3K}X|jg$?{n@^Xyola#s`7IN^jO<2Go!*+0u zr1JrWL);2l1&RXtX*&}7tgAc}7#RL(u%6f3wIR9BWkzJn0S2)jTN|fYP4aY{-_*3> z(897+0ny9nMQJgvkmE>n;g+*ls8y`1!N@Elp?LJD*zT_Hk{b`K-0miQ`;g$e>PNG< zPbhr+^m@JN6NL%`QIUyX)hAmq-St0W#O$@cR-0@QD}f8mv+mAsQ#u)+cXPjp4#BNSk2t)h_KU|Q(Am^ z8+hWxUap;fZ-dMHLYbl~P44q5?^R_V-}!DM_x$~RYyO_FDEO|#*CyY6ZlRXyvWPQ1 z=6ZFH^?J?go;+T>Gj#%|(SxAD~cc)zbE;eByV<`871q|XP69ifd7}#YGH2%49xRJ}|AZMwD3)=#R z9;*}X3}p(g|CA3MdJzQ=B8Ia*RbfE;A6Hp(2;J^EZM)}FayVhhVKp!7zH92SQZ^) zl+9Sk-?GBhM9EQZg`hx#p2B$+tqYBO4;ndzXEZ8^1v14)P7r+5(Pnxmu!DO_lSl&t zi}H$z9Bw9$*>y5Gx&OtVU|@1oV6^1);z)TR)N7)utXSc{qbas-O#am~+y?U%g z4_mZ5JQ^BTh2*n!OSqaG`d4`5h@RLCPab!(Cz`xfc8lj3FA}s_cEYOms-8iizzPRN z;hFaXpBZIenQ?3qd*Ib|)3&Z{U14sf#>H=9+An=|ah}<%+3zy1q;Fm5AYVJXk*y7V)W`U|^o{p+mi66W`0l44Pdx&4V^{h=^(E`4@d^P`JY=(0$>s(zk*Rmj&d_$IKIA z_#|%^%gktHkAA|yZdvhd^TOUE&TCZuadpk~Tp?i`=Q8omt=t4>>D6!DvKI$(%cd+? zDSG1OyfrzqZ})te(DY=c$Ip<)+W`%Xq6|xBNIEq56eaRUJ#ebf7ifHw;=Y_a!$GjY zpex;kX-}w>3oC;`gP2Vg|HHKhg{GAq=ksCUDH8~pX&t~EbY>yb%p*%!Jp{Sb7C10S z7Bov6DKKOnn!w1Oz`&`($(v@=EOF!kzsWyi&x{F8ocEI&SS?;UD|B5hs*#=8ZhePQ zh+&c7fh%qn6Xw1tXmFO+j5B-r%2n)?*Ei`)6>KK+KJrM^Kc4rlfc;&?nf41_%Z%7G zb<=VTCuH3?v`{qkli35=_p14erM>GmWzAmR{BK!O!>O=u+ual!q>V4LrSJONo3E4b zjLTqwc}kkY`6{;to`MDjSq3KAE{C~uo_&d2z~WR~SfI7UZ9%;ZD-&Od!hW$Q3EU7|mUeRN^$B*IrLDph`i+58jUzYRUQ$TG;y8lmd$ptPgxX>9Z;C2%~#lx{Pu+1w~dD{zB%G1E%SL( zb^HdJ0=~__@f@qQN%mv$hpH_ z3nm&U2zKSZy^_Z`RrlxuUYR9p-)+ieTK`sZ&cgS47bN#Be1C+)=+p^Ct>&aW#k&*O zw+5Zw`iNIrE1&h{TGp4m(mMKmvl#h*EjMF0YW72SJ6qlZldK75TGec>XN35enm#Zs zNXUJYz!}BB#jubq=>T6t19R5-U?l}*u?I{M3mDlHm|iqLz?{** zWc7fPA(8FTdcj!>7*46NM;+Ma#3+!m&QxL{vrwbNw1pz?6h!77gNqfC1`T8!u|4kYGUJVSM2M*3lWRwhe zJB9b=fkY{V4e!?E#(m0THWy(2`;KYhLnh8wKj z)@)$p@nA1tV6StKo82HXqfzpZfWVCfg3k`{MfI~~9N?BZ$lR5%(#m0b+5$m7SE+xC z5@jweWIE(8@P$Fh;~?LJ2Yi3JHR3WD-BS3)7zN%<6ck&)#NxpI?ts{lM7cQ!xTiJp z8U+5g>Wt}_z;W1BEq9XEv866{JuF@eSY_5o1_+AY4 zy>ya|a}=fgx+}Wex2}44FGQPd@8JY4xFobz}fPEecl4zMJ#)`*0e}Gl$&`_ zY{o(nuS7|Yg+`JO*^L%(N;$HhN)~vrfLThRii=U;*8;H|#+z0Nrq2`@^BAfN{FP%? zXlgz9$7g56BbjK&!m!HCQEFWx=dK3HX9??nte@N?%wV~arKtD+sztAl>BTZ}Gk;qk z*kGh_Q}5=Sx1z}{Oq+GZ=RUMPC~ULPQF5O!i)OM&o6kp?R|_AUIGIrJX~ISAos62VWI~#eK8W`s=F>df|Gx^lIrCDUg1i^m{%+Hvaw>;qIaunIuz&WFl_lsBC zuk#k2#_CHSUHALf;P2&9Y;iz&cB0IYgFekdGDj4TraU=%$K?I5bJlYcB@cbJo_tVp zlA@$Snuwa4@vO$XJ|DDN3$^bXOYU4~$8V~`Um7SLD)S&%GI@j4y!LH*8-g5P^?R(E zP^WyRCE#~Q!08-?zy%3{cO01W5=_4=U}8&HC-E*YErF3G;o+$>&*rV-%u8ryN|elL z6tsK5#`I85`U}&m1i>%8Y-|hpL^P&xC5lKK{e zSiqN)@HZoYt>gh)$pOxs1c3+7qIDQt(u&R%1-vH4 zus1Ytzk86I^^m!v*SDilO5$PoOh$oafxc<4SBA8mS)||X0`W=FD3uE0BMRX=h*1TI~b<)Gy(3*w)`mNIc_r5ZDEO?`Lz-;D2pWwoq z({yjWT6rtY(C?{;&BTL}6BrX486{*E`nPjc-g$B2&g7)spOTIi6-$~XA2yPlzEOJB zL-`DbX)6S`bC{TOX$EjCE%n!`7Sa1xeJW>h8Uxoq2mYjfraXrIG3!~cz2>WH;AKk` zlw%Z_cYrmffvxSl*p`KoPHO}+4zNBuAotZ-z|N;J#(|ry@mIzHPL3W1jRlVw4sxp< zc&Nv~xJZdbrd^}x(5#dfY&i{_^A50OC9qX3NL}!Nb4oc|MS|1xMNA$Gg17js&+Fy* zox;$#*2V4OVZI#6D|s?!k_vubop&eX_NupHucs7>oBKB_N;EtauRAE%vM{MGuuo-M zfU~*e)`tP%Gm3S~q=ifab@^rW^!t{q`TQ+ciapc+7&}MwcYmRGW=DF?obgevX`7&P z!nbEx1>-~hE9_B{j9gjG%tZ?XPAw4(D;6yJ=aB!FLC7spMqgoy)dIeE$M$rc|2pG= zg7`zeH)|pdnzVlSE3>n-NgY=eN?^OCD)?>z%MT;2rUx@}92icVY43YI`QGBJg4}ey z0}ZD$UsoJs5wY0t<3aAXt3IsdO$Qg=Vwm=RpJMa=$u<)er%0PiI{Xk9aTHv3=|tCZ z+pR)uv64YgCkGyV7$AE!Dcv$TRidPoi^FHTnO&A{^@fLTT;|8pw>MRs7JIk-&uag= z)4UQ3eLX~T3-!1zJy*^&GtXe)S@nc3cCuU{=AJ>@* z%LeWm1?zPiG%Yffxo7s-F52Pc5W#Yh_mi_=ce!Mm1H(U|2Bv+jEE9x18n5i%aIWF5 zGUGn=*gW3qZ`=Gu3MFrxi*sq;q$c2JZCN;1QNlz*a*E@Wi3_L5KAa+&xG-#|)SiX5 z%CGLdF)o%4EzXooF1D21_)zRp%i@2!RT+obYF|%pu1K%r?r5D6aN*2~&Fox$S0X+b z)LwdS;xzetfdkJ0h1FXZq}C=d%5XBLFfh;4%9_;0Cb2X;k?qw2J#(9rb}9{%kGL-E z%bR{K&ErMBKUL)s?1_>d>mMRCSv;zyKAF>l_m@E>d{3Yv6)dQBa2kc%A?0(FX zzHifVdcgD}K`?7!i5vTSCNn={iz(i>mN@?u|5PF|DN({NvM*uQzku{#pOS7YlN4>+ zc`)#D<2LcD+G`g|9$4_?TT6eFc7WL_d(oLO6*J7ZmIl1obcX5N!x}ZNX=b%c9L!i6 zbNgN@cju>f>su{)yu0CnRm6iCYzMim6xi7k`M>mXb3NoMdBA*P1@}7+{#Om01`j5e zI5Em7uw2=$Vb!#*(}dCMNbc-~`!x>C_`V{53j}knIgDw ztKz9oK|7A6{!Yr@Gp#-Lc;as908SP&hN>p36~0;%*XmTAdBYUbz1nNX=WYpqY({WqI(7-Dl=Jv$AMl%W71(GskLnBe%@~BdcQ0 zfA4Ihx)yL2Y@TE6$C!71vci*@T<*&5Eg~-tFm)te@^9vqJ9u@S*Oi7X^LTF_oZb|} zp~`$GY)9aN8D-B{434`zV-T8l@ZFBjxBOxxn;Ioj_BC5aJahdio@cu?_Rdj3M?nEb z!MM|Q)pOc%f8Ln5Px_MLaqH{F#{B{|3Cvmv%q#3xRI*JFn<=%}ees1_ey2u}ISrhQ zjRKkpJG2r`aUS*OsS0?~%*c{p_im@^rjvIym8QJ&VUufAunOLs?I`5HV06TRy@bJb zbs|&A0*tGd-92aYF<0tZ+m6d2|)u!|jFWs>A-yKMJ{IjTfS zYr~aGy;A>Jmo!S5C{!^l6bVT9azKRXMT1~e-PPU3Hko!yviE(@jgu_;d#!Zu^Ge$n z4>>-+nS3L=>Uh4@@$R`lQzN#CZk670P>QYS6=&WOH?I?nsXKnY-TuQ!gvpUPCtlWo;vg_)Z z=03M=QWrIt7CJU7xpIlD3`jZH#K9+%Q8MAcK}I$XAsr6^1qWd^1_>6A1_cJ@UK4Ap zgbxbK{W!#g0t60dh$uTsKATXKtjNs9C8lMOdzjeMvK2+`%2ld#eeSm?;D8@Dav zqk{rd3$t8qlh1?$ObuMz0wD$h50*8~RJSjCvZEo7|G|{WtZYFWI(SW4jF=Tpcd#+A zs>TF322V0^nra-D*xF+pqxpGi_at7s-arh;1wszR^>6v}18X zYiaWJq^&Ok82!a!5)N{*CHHP1+$!m?tG$7YSFAb1#XFX zB6ry2Sp-xZ7R=;lxzWJaeC5)aqV5*~E-V!e-}`yuOpLn3SzU@wYh02H>eiL3IS`<$ z=iGI%5vGzxTwq)s(DX))Pn<3WYXq8&<;EP5i;r>Pz{$lAD7Os3%P(d5LA zixJa%?;guoyxsEIRbSJsCJ#j>st8mvO`4rMVclej4^d6L0S1n&!m?W?ICJxugdSvF zXK5OIfJfj=bRPGb&dWZ0Gj1}lm;ckde#Yp3Mc_o0eiI>2!6*}_0LGSoi(5|=GzGXQ z-}>{|k&%U=(cO5vMA2#8au)?h4l~;Y#qP@vEfHi|JS&S~mC%H$65rLI?gVmh+FY1C zQ$|s39DDkyr5%G zvuE9!c)_W^tATmhqFeLaEDbMQWMb7C3UI{#_d|C*aB z(cy7A>c<8%Q-Q|%{D~L0{Sk8bFB#xg)0x8bzsX_8KNA6dg*`rOY8w~>;?kJro#~m} zmclsC;aH2%olD6Y4@6EowlPdQIV;%X!%|kBgjT_b1{aHfb|)UEpu;^bysBUO6T24j z*BTs=Jrcl{+I6t2_QE0dX_*{1o-W{yf8i`~D#6vQ!ogf1qmd_K!-hq!{ux{vizS?c zgubaHsQ7bteseN%|Mns$n5W@9qt1s0;W-ZK&z+a@N^IJ=~#Co(r&867EdhYjmmQ5*r{43tZw=aO6p1c)W1()c6P`ul*T!{c{TgmM9so3d}E<;{Gs7 zVf%+LWzS1*>nG*pGV?xUoTm4ZBh=uqs#Rg2i})fdZRe0wasdRX^qquLImO?pawMx?WYL@=s-_Cj;ZWj+wj} z37$f6LPgod20}X+7EYR*lggaAP1u*a{>SC`bVIINJz$!oeZ z4PQ>n5O~C_XUim+$ITUx;B;axLu=mJf)-WYQjeBwFSc1s-Vw73PT4cKD7-tsqTz8c z&ToUGY)L^QU(P|km>ui5YZ}-m*glfK&?&5^boA=_b=UW_1?Y!nU+c-LI2_a3HQ6L< zVXMpo26l%9%);|NuqY%v*v}Tg!1OLEVWFt+fn`z4ZN8qG*J}BrQ|bqU=qd*`qZtSJ zQ&upEPT0_>9@fUFvV&1sNt@?%T$n^t%EYzi+U$Qdg^p^5F?L7VMD6vMbnG9)6hqJC zl4QBW#ZuPGR!Z72cKJLhyLIJQuIMBlrlk+G!hAlyESNlHiu4Udk%k-13JeE3?0io0 zHO_Em&naNHdC9O;WM)@huG zy7utTL2gTiLwn{Mv|Ae_CFe3YG>8c#xYuhPkjhNAZa=2k|HO?!NRqPBq(j4b>oQYqQ9Iz56a%AZ8+32wPx1FYj6J%3yb3 zsOC{{^D%6As<6OmMY(mGmdk&k-KNZvryjKHv|REQnc=A1;n=mX`~M5~@FQIuo*v6q z`|`eVH~jCQ&Ae*A!uiU$!tVGd_7h%l=o@P#Bphkv`(v*6V|#3J0-Kv!?(_z(1rxaU z3REp*=bB#-eI=b$uuXenp!Cc@#cdlUAF*+mH!uY#a5!D$C@Wwt4q*4&z;n%jtN8)~^bt_$s2Np+1UN_I?RYrRnyG(Aas0aMosrov50{TcQ8o0R{T zi?g>V$8T3oQWw8hz&)dZbM^$z+;+~L4cspZcwaQ|-AmxR*TC&o7M&WXTLXA68gTd;u(uYli#jkc2bgdL z)EqW++G(iS_y(zJmi+>E`9fweq=UE>3@ z{{-g67Yd?Xd->I?&VHz}7{WJRzQ$5E0%l+4gEnaaaOdngN^P@d-T#I3_&cn83i< z$-vpDFwy=RXNm!%|0j;E3_R^0CTcTHx;%+pC9@-I1M}9%+1eYJB~_U}x|fEi8|%L8 zn9xxwdo;?>Eywbc(u)hc_a<<=E#sWYz_qub>-GcQQB;7$qiPwj z(@TxTTcUv3r+``ZLZ(Cmlk9_FivaG27X)HlIpzg$AHK|U@k7O)=9Vunr#)d^5VLf9^3$o$g$t6_U0y=KBn z20KLt>4KRT!`T`Ixc(hr;B+%DImEy@OOwN^p?0nVtJ)Wi(*|7a1sok87Ax!&t8CyX zxWHCqFuQ9Dn?~hqt()0~L^jau+BGt+JVKB=A<|PTdXFnA5eyDsmHT`@5ckxNaJf*l- zuJcRPG+PBY+72vgbzr_w%DzZ|_gVt?Ey04bEHmCr;Io$CwbS5ge!%+eMh25+rjk@b zN`@Qv2bU?G6Ve`VE@a?5U{F{0f}@jR*1~|Svzut;bFG#E*Y*kjcnXb{>S@j~TDkOYSAF5ct{ny`7jE*sy}IqY|NrxHg4tudoklb*Dfkl8p?*qrf1Iz1n^{}$bwg_;> zUt{!`(ehJt^Ubs zLL3g6e{b|VDZ3eNWHe}Jjkv&h#er*qCQsi3j_wDW+aGN1JiyVaP`ka9MZ|$?>j#dG z1MD3ROO9o&DN0y#YS$Xoo!RO?*JyuW(q1qrbjNOpW?6)``iPQgfe0ty>{c_Gkm|m6^+?PhctiXXWT-q;{ul{f{`I0uIANUh`H2`b?x3hu(?uFP9oCoFitfA+!5BNI66e{b$m*m9bi zBV|L|+yoY30j`6;wggtM3G&{W6WpyPjqrnuyZuEZw&zsX}MvHkI zfACJ&k)!9P7uD6-nK<|K%(>hvxS!tQe|v!MUcua(n|M1O^6qiqp4Grw_B};9!H(T% zN8F(uehV8fq&m$E?$^D*aZP}Cy93A8hTiP~zR?V9(E;rL;u+YYr?%u}a`Rp9*`}a5 z&tcaR23E5TEUSJqS_!c6J)gFrdfUqbeDlo1b_cV6J+YAQgQnBs-Lk6~zH{t(%OVq8 zz-ivi_t${;K*MUITYKs<>e>Z3<~huqVZe6nG|!O*>;b#?2G3^C5oT}waV%3{pL*x) zoj13}+@Af?yHrUawC@9d_B5WY6L?I6c~1r`)i&FoWgTUy8C3DGe)<-N_KkDaNTghf zT=(_>_nv^cEIi^b7VzFV(6wbkbXhv1YJ%jmj;Omb#ef`q5 zs8c5tmQCS3f-U ze;0$R!|6*2eW!kKp3=Co*C*~q2A94Mue{CSC9gT>M2GIo;aDWVbtNMGti};z;VR}t zj*1B(%mSG%k9QwbzsBvr@U&+0^n`Vv4)8d1uU0SOI6Q%)x`2B}1M8kFHsJ)0r4C%b z{+&6WdExxOKYN`wWNSnox8Bg9m2jhL#=dS{CXU_<-5WT#&+faxp`KpN{j^}4cG$(M z7kIBVEG_8HnJXhc#U{nklY9AG?wM6$ZYo#Hg%x!Z;$9pL{V;V!-pifK1bD+^xo665lu=-e z%hfvie8%611ntwCI0CL+3NPM!fd2(UZ_9=~ofDL*7_K)JaKCWi=D5k3yn(USfUAGQ z`GN^+ly|Oa^DleAtb1>0DKPoGxY5sh;tluy{XZvt-M;Nc0`IK_yx+KQ zgWN;nBaf4V-HiaPL0Ay{~~!NA8l^UG8jiHqSIkPKDbm|43P$t8MpY+1BHoShqG^>UGG{=>8slK@BIx=UdTIOiI$ zN(+Sk+j~AUVTtn1^Y^14u-jeW{>P-efJw!GiC509q=CIgL8kvNKNs(bing_(QS}Cy zyi)fb&VReM$a?$s3DMc@oHHG`HTQ8V%3gXaaEZ(2l!5`{785lg6}2QIHJf)*J0Ge_ z-C>Zr$WRvWPW-Pe1Jb`POK-G*nCl^K(TRpFs;K0t{ z_kro(UKL9gHoXf^X9sexJO1$T0|9^bxCg8?2iOJmZVK0}737^0_N(K;2QgW>?JA|Z zzjyC{6vZ}OpiApM*X{+6Hu=4Ndx3kmLT=iG6rJzF?yZlv)k^O@r@^&YP%wb~`~u!* zH#pAJZI~x;<*4u7Xw$0=4eTf9#CdA$x_K^?Cv*4Z+A~wM_}&~yYJSKYbCq4M;9a)e zB17+cin}&(PGFQbV9Whcw*0~?t#jO3eJ7v(d&OOP{=nj6$<+)-1qaxF%go+r`GCdl z#-r%t{r_I@>^mMKz+MyZak|{Yx-%DYrriyC)giHO+g#nwo;ebU+Z|ps@K&dBWLI!5 zxbaH!9rs4QOA2ebFI_v}e&Dsm`J!DDrPofB-XbEsF_6LW-BlAq_Olz7@7?od{`2Ku zG^bsO-O{k&%$(CZzjm&;@OCFd)tShxy%!gh-dMocg=@!ex@ z_{ZZ0v5|QLuZ~57LRs$ZZHej@ z1rHh<_(P8MtSsW3q8Gb<&97HayTyxrLO0A%{2baa*S+_}fs?|^tLK^Ia!C11p74-Q zDO+h>{l-O$xP+vhW=~Pk#ZsCEtdlg=8s#7&2H3KH#G4tQ{e0NTyTnIXWW`ws*}Ap%*gA~ zRP$(PvXYBDV*T!sK*k=EDJ~rxKJrx>N}_kF*DvaJ5_J^n-!nl)Wx}_|=a;w!MP)9H zN{UTembR*LnHrn)9naZw`!*G5aCfir6cI49Q0jiP?UswMZ$#oXXKlYZ4h`bs4ch;F z*&azXG;pNotkz*li&0?M!8YLld&Z&6XVct%FeP)!P7`Q0IM64P(r@M#;e7JLt%Ae6 z)+)j?yqr~DGxuAuyj*;^Nq(7_yrmo zcqYs^#PgD8Zp&(Z)5@9t#>XOf=>`r)_0?1VN)@Mo!8CG4UT+-@l?dm60R z^6QN-r^=tOulkdw#Du<_#>DQkAuLAv%F5YuJAQSph*y2Ha#mEy4F(n!vo(i$mTyof zJK3E2<_y;#ahb2-gumywu6y{|Ims_69RI63Q0%LQ>;GYHtRwKLw_>!OuwBS z>(8EL2;%AMv5nn!J3!}|Tr(Fhi^PIPhG!D2Li3ePI}}tq)tL6ScSPjB^mm+_f902p z)z$9bA6~Ct;u^)fVd?aYt)U0mydRq$TD0P9xXr;f(_3y0oH-A2-7T5~7fo%=kX;zT zwSdO%@RD0Y`PU}CYyIVOS?>G zx7^{#Uv`06VHHb<{K9nxtr9ByG6feLQ3wd z8)CN07+52;D~11EI3W1th|!C}6i;BV$NiqQwkWiB&WBg^`#(*+wIPr<`T=8{ zmBLf@o`O@_5{~Q&KO8n1H8AqTEMQV>U|^9*U}W=Am@cT{z|0lE5NCE^ddACyh?`~_u>{7kKDByh} zW~x+Sw_4X?vBeoj6#g}|%ak-QM6k?bI+5DI;K9Jibs_1)t0fI)nTC~`ml#CnOkj2C zZRAN*U>3F8(3q*qz{J<_uYu`>L6`f7g?$eUPb@WhI^pBm#NfA{_p4n58lAcrI07fL z{P9V>{O66jP*}8!ipa-)_Z(eG)P@m~`aIu@fqIq9+jwrmzyk`GKaV>{)gzTMHt&Td*(ieBMs=w-DcQ!dB5Us$R zAYs9)wLIIB9H=@wX*9HE?}&R2GYiQnM@Bd=9JBUc1} zA-6+=SY!AF{c9`D6Iha+*cliYcoLW-FC1XEGic^dP+*o^(7+(~fq`dTn9#X~BZ_>D z9TTNg6lY|4I3Hsdw0D}Wq$hgypKF7`S^Fv2H0P7|=$p~$!GZR*A@*&{*+ z9w=W4Jt@%a(ZcJ#V7=KJMuFxD29}}byB)6_5^%54QIwn3S7^e_FZYm1eAfqNiQfm< zoj)-07aVAoxMRUKtLOxG9E0HZid9NRA3ky?JZO@-bAZLB!I9VJ!xQ!R6xQH2XI+_v z9gm+e3Nkl3%WxU7O#URH^mb;RcVd;`45owZCZ?jZ?|;8L@k2<}7{ZMyH6bE??#yC;fRG%y9=7 zFyB1;Zeyc|i=4~$Ho+wiH^_v=rmFB@@ERqCIeoNcDeh-IhOysS5K4n{HzCEj!C@I)s(IWFrAU9Q;>2!@lp4_gO zl|HuOyBeqD&2)Tvsq4psT_#Jn*Jk~?8xpCT-5cOpnHj46#YjW_-|MUMcR%DUk%)QN zk!Im<;`!1u`|zc^0jq*E_i1m6y(KmKowNLj26m$fm-yXZ`J^#*zY1QyfVV2)j6#&` z5eJ<$f_@81<+r7DXLT^kJ*se9@95QK*>vT)^Y>CCxqpm3iMIW_J+YY;b$-fa@uqH#1rt8SeHPdd zo&RI6KHpD8PtUqn9hTpAiy3Y#HvUi}?3)*Kfm^<0gGk*ruD`3oMW1&b)m)e1?R?HX z&-yR%43EW= zuYcQCtG4n}Ojpgtr5(GSKjihUU|`SM_pY=(;QhDC?aM-5@Am)C_^;xiwm`;(`5PV@ z=P!j1!RGfg(V^ZZEo6pfWuOHsnv!>2_~9dY$pv_EfEl?n@+2gh#x5<9&!POmmQrL3__y|M3Y zFk8|Crw!cpi9cEsKd2QRHqv!o*Vxf~FlyaoBl+^(9p5YLf1YNn>~@x1yx(-OrYlpY zYnSo7P4nX?Ol4BslfFXKDP>un!j8B=X?x}^i#>MadF(jWe8e+HYmq?vf*D)5Q%-Qy zDDB7wo^vxMo&7P9^K#EsN7?LKJo^LI1ia`DP+;jiqEm5!?V<9% zKn~WbS)PkNn$^K z>z8SH%)8OnYHXp&G_7;@9b7dvZ zRFU@V6KrvvZ2=d0qBpSR&0sH6XiqiRH(lmzoD8dq$~G09ZL@jYl3Lu7m+aGwIhJt3 zN%zSiwGgkw9mhUh-Zb&krsf%5p9^>U3%ZN0V3a!HCEMV<&gXdP5_8uozxP=xzkc=~ zkY1I#K;`h}B~Ctj+}Ak73!YfK!F#vK>@6#|oH}jMIorhRbsNu?9bbAJ(v*Zxemo(+ zaIV`2mOmyeN3Q(yIQ-yz-w~n9yZ!fPw740xuD`T5>j7I?L3>FB+k{;bK^(2_0<4c$ z2XM>oN@PAZr-dak;q208H(irsNjz*R3>OnP_Qy|fb6;^xTX}b`ws|o#zZd6bP2qE2 z3L7N~7}uWhT06ziGUuGdhQs{~opTq-uQ@0c@9ll)@Nw3g#+wI2DI^~GJe?YLgujjujlMY-EQkt}v|BZ*6 zK?~DGw)1n?nT_@-t-0VPuy6X2fGaxN+!<^RtTuE1acGH2pn&PI^gsLbj`$`@v}Ki? z`c!>UV(Anw#}4m8YsF%InSVzwK7JCocF#J~YUe-QQg%12e?DC4@ne62w3b6i+Z&l9 zteOVw7gao?&%4Sj(6W_wjq&)VBE2xcgJYqI=Bv)wMvFMcrK>NQWc8jf+S?kn=bF)6 znezexd7`ZTJI?>+>Dh3}Bk+V}Swnl}jP^<^yGtgAW~rDhF*e(1Y{r}FKh*zYH^S7x+5xa@lI$x$a! zvjl0gjZ4E*w4?d9hTQqGFTUZ*)~z8vu@|#zF6K^V&NbQI_~UGz%}UK;j?UhQ;H44W zu2*}mMpVt7Wf3s-pYdtSO;@8TI;-~>&yEntb#REUonU{MEid3mNA!u!f#)~fyt5$S zMC%-vp2z2AeK=j>(S9NH?t+BaX?NLne_^{BIw8xWYwq2f3%=gWxDmT*kwi=ao5rNJ z>OXBF#&(N8TXMhM8fS7P$(b`P`AT+iOa0!EEo<*36x`k}+vY66=JFuU@9rku!-^j_ zOxaSbx2Ulr=x=oA(MB2GJLXdiC1*G*ywx+kW+y*q~)<4^MwsxG#dJ2iTD zxYy{Pe;c!4#+Hvu9A+#P(0h94bg@a{hxuF~0VM&kJbvxmes`H4v#IY&D7^9D%H0Qf zKOP8udmy|{Ag@5+USx}VMVs4=sQJ|a>q75U{lB_>IWVPxfx#L8l&k?n literal 0 HcmV?d00001 diff --git a/tensorflow/lite/micro/tools/make/third_party_downloads.inc b/tensorflow/lite/micro/tools/make/third_party_downloads.inc index 8590ace9fda..d4d5c1c73be 100644 --- a/tensorflow/lite/micro/tools/make/third_party_downloads.inc +++ b/tensorflow/lite/micro/tools/make/third_party_downloads.inc @@ -86,7 +86,7 @@ XTENSA_HIFI4_MD5 :="f234764928f9a42901df33a27e118c8b" ETHOSU_URL := "https://git.mlplatform.org/ml/ethos-u/ethos-u-core-driver.git/snapshot/ethos-u-core-driver-bcb5aaa99756f1b5c1295b079ebdd60996bc75a5.tar.gz" ETHOSU_MD5 := "d2073c8d88fc167fd5c46b5dcda58ea1" -HIMAX_WE1_SDK_URL ="https://www.himax.com.tw/we-i/himax_we1_sdk_v02.zip" -HIMAX_WE1_SDK_MD5 ="9a4b2f29b16052764e437b64bdcba816" +HIMAX_WE1_SDK_URL ="https://www.himax.com.tw/we-i/himax_we1_sdk_v03.zip" +HIMAX_WE1_SDK_MD5 ="1cd9b17f3fdb3e9a1dfd1cc356694325" From ef90c47a66ba3bdb438b096c740cc8c83949b049 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 14 Jul 2020 16:56:04 +0000 Subject: [PATCH 0056/1017] added comment to on_host flag --- tensorflow/c/tf_tensor.h | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index dc659db1f7c..fa1bc80138c 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -55,6 +55,7 @@ extern "C" { // Allocator Attributes used for tensor allocation. typedef struct TF_AllocatorAttributes { size_t struct_size; + // Set flag to 0 for CPU allocation, else 1. unsigned char on_host; } TF_AllocatorAttributes; From 165baf2eb9ca13a3d0613369ad0d00c7fe466abd Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 14 Jul 2020 17:11:04 +0000 Subject: [PATCH 0057/1017] fixed comment for on_host bool --- tensorflow/c/tf_tensor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index fa1bc80138c..190e545dcd9 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -1,4 +1,4 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -55,7 +55,7 @@ extern "C" { // Allocator Attributes used for tensor allocation. typedef struct TF_AllocatorAttributes { size_t struct_size; - // Set flag to 0 for CPU allocation, else 1. + // Set boolean to 0 for CPU allocation, else 1. unsigned char on_host; } TF_AllocatorAttributes; From 4c4223eac9a0dae3ee9e901bdaa554a2cdd54279 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 14 Jul 2020 18:50:53 +0000 Subject: [PATCH 0058/1017] fixed Copyright date --- tensorflow/c/tf_tensor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index 190e545dcd9..237b41cbb10 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -1,4 +1,4 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From f2587bf5cc5c50706f24ea6f1dc4fe47d9c45fe1 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 14 Jul 2020 21:03:45 +0000 Subject: [PATCH 0059/1017] added tensor_shape_utils for TF_Tensor DebugString --- tensorflow/c/BUILD | 26 +++++++++ tensorflow/c/kernels/BUILD | 1 + tensorflow/c/kernels/ops/summary.cc | 18 ++---- tensorflow/c/kernels/summary_op.cc | 77 +++++++++++++------------ tensorflow/c/tensor_shape_utils.cc | 38 ++++++++++++ tensorflow/c/tensor_shape_utils.h | 31 ++++++++++ tensorflow/c/tensor_shape_utils_test.cc | 44 ++++++++++++++ tensorflow/c/tf_tensor.cc | 12 +--- tensorflow/c/tf_tensor.h | 3 - tensorflow/c/tf_tensor_internal.h | 2 - 10 files changed, 187 insertions(+), 65 deletions(-) create mode 100644 tensorflow/c/tensor_shape_utils.cc create mode 100644 tensorflow/c/tensor_shape_utils.h create mode 100644 tensorflow/c/tensor_shape_utils_test.cc diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index 410fc22069f..65bad3b5de9 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -513,6 +513,32 @@ tf_cuda_library( alwayslink = 1, ) +tf_cuda_library( + name = "tensor_shape_utils", + srcs = [ + "tensor_shape_utils.cc", + ], + hdrs = [ + "tensor_shape_utils.h", + ], + deps = [ + ":tf_tensor", + ], + copts = tf_copts(), + visibility = ["//visibility:public"], +) + +tf_cc_test( + name = "tensor_shape_utils_test", + srcs = ["tensor_shape_utils_test.cc"], + deps = [ + ":tensor_shape_utils", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + # ----------------------------------------------------------------------------- # Tests diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 7e103514645..77fbd869105 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -30,6 +30,7 @@ tf_kernel_library( deps = [ "//tensorflow/c:kernels", "//tensorflow/c:tf_tensor", + "//tensorflow/c:tensor_shape_utils", "//tensorflow/core:framework", ], ) diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc index be39cd0f530..9cacda36adf 100644 --- a/tensorflow/c/kernels/ops/summary.cc +++ b/tensorflow/c/kernels/ops/summary.cc @@ -19,20 +19,14 @@ limitations under the License. static void scalar_summary_shape_inference_fn(TF_ShapeInferenceContext* ctx, TF_Status* status) { + TF_SetStatus(status, TF_OK, ""); TF_ShapeHandle* result = TF_NewShapeHandle(); - if (TF_GetCode(status) == TF_OK && - !TF_ShapeInferenceContextRankKnown(ctx, result)) { - TF_ShapeInferenceContextSetUnknownShape(ctx, status); - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while setting unknown shape function"; - TF_DeleteShapeHandle(result); - return; - } // Make shape handle a scalar value (empty shape) - if (TF_GetCode(status) == TF_OK) { - TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while setting shape function"; + TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); + if (TF_GetCode(status) != TF_OK) { + std::ostringstream err; + err << "Error in setting output shape inference"; + TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); } TF_DeleteShapeHandle(result); } diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 5db4a239905..e373ef13871 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -18,36 +18,39 @@ limitations under the License. #include "tensorflow/c/kernels.h" #include "tensorflow/c/tf_tensor.h" +#include "tensorflow/c/tensor_shape_utils.h" #include "tensorflow/core/framework/selective_registration.h" #include "tensorflow/core/framework/summary.pb.h" #include "tensorflow/core/framework/types.h" // Struct that stores the status and TF_Tensor inputs to the opkernel. // Used to delete tensor and status in its destructor upon kernel return. -typedef struct Params{ - TF_Tensor* tags; - TF_Tensor* values; - TF_Status* status; - Params(TF_OpKernelContext* ctx) { - status = TF_NewStatus(); - TF_GetInput(ctx, 0, &tags, status); - if (TF_GetCode(status) == TF_OK){ - TF_GetInput(ctx, 1, &values, status); +namespace { + struct Params{ + TF_Tensor* tags; + TF_Tensor* values; + TF_Status* status; + Params(TF_OpKernelContext* ctx) { + status = TF_NewStatus(); + TF_GetInput(ctx, 0, &tags, status); + if (TF_GetCode(status) == TF_OK){ + TF_GetInput(ctx, 1, &values, status); + } + }; + ~Params(){ + TF_DeleteStatus(status); + TF_DeleteTensor(tags); + TF_DeleteTensor(values); } }; - ~Params(){ - TF_DeleteStatus(status); - TF_DeleteTensor(tags); - TF_DeleteTensor(values); - } -}; +} // dummy functions used for kernel registration -static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { +static void* ScalarSummaryOp_Create(TF_OpKernelConstruction* ctx) { return nullptr; } -static void SummaryScalarOp_Delete(void* kernel) { +static void ScalarSummaryOp_Delete(void* kernel) { return; } @@ -56,7 +59,7 @@ bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2); static tensorflow::string SingleTag(TF_Tensor* tags); template -static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { +static void ScalarSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { Params params(ctx); if (TF_GetCode(params.status) != TF_OK){ TF_OpKernelContext_Failure(ctx, params.status); @@ -125,41 +128,41 @@ static tensorflow::string SingleTag(TF_Tensor* tags){ } template -void RegisterSummaryScalarOpKernel() { +void RegisterScalarSummaryOpKernel() { TF_Status* status = TF_NewStatus(); { auto* builder = TF_NewKernelBuilder("ScalarSummary", tensorflow::DEVICE_CPU, - &SummaryScalarOp_Create, - &SummaryScalarOp_Compute, - &SummaryScalarOp_Delete); + &ScalarSummaryOp_Create, + &ScalarSummaryOp_Compute, + &ScalarSummaryOp_Delete); TF_KernelBuilder_TypeConstraint(builder, "T", static_cast(tensorflow::DataTypeToEnum::v()), status); CHECK_EQ(TF_OK, TF_GetCode(status)) << "Error while adding type constraint"; TF_RegisterKernelBuilder("ScalarSummary", builder, status); CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while registering Summary Scalar kernel"; + << "Error while registering Scalar Summmary kernel"; } TF_DeleteStatus(status); } // A dummy static variable initialized by a lambda whose side-effect is to // register the bitcast kernel. -TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { - if (SHOULD_REGISTER_OP_KERNEL("SummaryScalar")) { - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); +TF_ATTRIBUTE_UNUSED static bool IsScalarSummaryOpKernelRegistered = []() { + if (SHOULD_REGISTER_OP_KERNEL("ScalarSummary")) { + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); + RegisterScalarSummaryOpKernel(); } return true; }(); diff --git a/tensorflow/c/tensor_shape_utils.cc b/tensorflow/c/tensor_shape_utils.cc new file mode 100644 index 00000000000..c38eb95724c --- /dev/null +++ b/tensorflow/c/tensor_shape_utils.cc @@ -0,0 +1,38 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/tensor_shape_utils.h" + +#include + +#include "tensorflow/c/tf_tensor.h" +#include "tensorflow/core/platform/str_util.h" +#include "tensorflow/core/platform/strcat.h" +#include "tensorflow/core/platform/logging.h" + +std::string TF_ShapeDebugString(TF_Tensor* tensor) { + // A TF_Tensor cannot have an unknown rank + CHECK_GE(TF_NumDims(tensor), 0); + tensorflow::string s = "["; + for (int i = 0; i < TF_NumDims(tensor); ++i) { + if (i > 0) tensorflow::strings::StrAppend(&s, ","); + int64_t dim = TF_Dim(tensor, i); + // A TF_Tensor cannot have an unknown dimension + CHECK_GE(dim, 0); + tensorflow::strings::StrAppend(&s, dim); + } + tensorflow::strings::StrAppend(&s, "]"); + return s; +} \ No newline at end of file diff --git a/tensorflow/c/tensor_shape_utils.h b/tensorflow/c/tensor_shape_utils.h new file mode 100644 index 00000000000..cde929f3f4e --- /dev/null +++ b/tensorflow/c/tensor_shape_utils.h @@ -0,0 +1,31 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_C_TENSOR_SHAPE_UTILS_H_ +#define TENSORFLOW_C_TENSOR_SHAPE_UTILS_H_ + +#include + +#include "tensorflow/c/tf_tensor.h" + +// The following are utils for the shape of a TF_Tensor type. +// These functions may later be subsumed by the methods for a +// TF_TensorShape type + +// Returns a string representation of the TF_Tensor +std::string TF_ShapeDebugString(TF_Tensor* tensor); + +#endif // TENSORFLOW_C_TENSOR_SHAPE_UTILS_H_ + diff --git a/tensorflow/c/tensor_shape_utils_test.cc b/tensorflow/c/tensor_shape_utils_test.cc new file mode 100644 index 00000000000..ef1fd1e839f --- /dev/null +++ b/tensorflow/c/tensor_shape_utils_test.cc @@ -0,0 +1,44 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/tensor_shape_utils.h" +#include "tensorflow/c/tf_tensor_internal.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/framework/partial_tensor_shape.h" + +namespace tensorflow { + +template +void TestShapeMatch(T shape) { + Tensor tensor(DT_FLOAT, shape); + Status status; + TF_Tensor* tf_tensor = TF_TensorFromTensor(tensor, &status); + ASSERT_TRUE(status.ok()) << status.ToString(); + ASSERT_EQ(tensor.shape().DebugString(), TF_ShapeDebugString(tf_tensor)); +} + +TEST(ShapeDebugString, RegularShape) { + TestShapeMatch(TensorShape({5, 4, 7})); +} + +TEST(ShapeDebugString, ScalarShape) { + TestShapeMatch(TensorShape({})); +} + +} // namespace tensorflow diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc index b4b8c772341..0feb986ce44 100644 --- a/tensorflow/c/tf_tensor.cc +++ b/tensorflow/c/tf_tensor.cc @@ -17,7 +17,6 @@ limitations under the License. #include #include -#include #include "tensorflow/c/tf_status.h" #include "tensorflow/c/tf_status_helper.h" @@ -181,11 +180,6 @@ void TF_TensorBitcastFrom(const TF_Tensor* from, TF_DataType type, Set_TF_Status_from_Status(status, cc_status); } -std::string TF_ShapeDebugString(const TF_Tensor* t){ - return tensorflow::down_cast(t->tensor) - ->ShapeDebugString(); -} - namespace tensorflow { void TensorInterface::Release() { delete this; } @@ -231,10 +225,6 @@ Status TensorInterface::BitcastFrom(const TensorInterface& from, DataType type, return tensor_.BitcastFrom(from.tensor_, type, s); } -std::string TensorInterface::ShapeDebugString() const { - return tensor_.shape().DebugString(); -} - } // namespace tensorflow // -------------------------------------------------------------------------- @@ -330,4 +320,4 @@ bool TensorInterface::IsAligned() const { return tensor_.IsAligned(); } } // namespace tensorflow -bool TF_TensorIsAligned(const TF_Tensor* t) { return t->tensor->IsAligned(); } \ No newline at end of file +bool TF_TensorIsAligned(const TF_Tensor* t) { return t->tensor->IsAligned(); } diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index e4953b53e43..acdf053e63a 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -21,7 +21,6 @@ limitations under the License. #include "tensorflow/c/tf_datatype.h" #include "tensorflow/c/tf_status.h" -#include // Macro to control visibility of exported symbols in the shared library (.so, // .dylib, .dll). @@ -152,8 +151,6 @@ TF_CAPI_EXPORT extern void TF_TensorBitcastFrom(const TF_Tensor* from, // Returns bool iff this tensor is aligned. TF_CAPI_EXPORT extern bool TF_TensorIsAligned(const TF_Tensor*); -TF_CAPI_EXPORT extern std::string TF_ShapeDebugString(const TF_Tensor*); - #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/tf_tensor_internal.h b/tensorflow/c/tf_tensor_internal.h index 036559da838..7a896dc5d11 100644 --- a/tensorflow/c/tf_tensor_internal.h +++ b/tensorflow/c/tf_tensor_internal.h @@ -17,7 +17,6 @@ limitations under the License. #define TENSORFLOW_C_TF_TENSOR_INTERNAL_H_ #include -#include #include "tensorflow/c/tensor_interface.h" #include "tensorflow/c/tf_datatype.h" @@ -105,7 +104,6 @@ class TensorInterface : public AbstractTensorInterface { void* Data() const override; bool IsAligned() const override; bool CanMove() const override; - std::string ShapeDebugString() const; Status ToTensor(tensorflow::Tensor* dst) const; Status BitcastFrom(const TensorInterface& from, DataType type, From 97b946227cfd6a9d1fdc786cedeb09674d7221c4 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 14 Jul 2020 21:35:35 +0000 Subject: [PATCH 0060/1017] added tensor_shape_utils_tests --- tensorflow/c/kernels/summary_op.cc | 6 ++-- tensorflow/c/tf_shape_utils_test.cc | 43 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 tensorflow/c/tf_shape_utils_test.cc diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index e373ef13871..10b46284814 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -26,18 +26,18 @@ limitations under the License. // Struct that stores the status and TF_Tensor inputs to the opkernel. // Used to delete tensor and status in its destructor upon kernel return. namespace { - struct Params{ + struct Params { TF_Tensor* tags; TF_Tensor* values; TF_Status* status; Params(TF_OpKernelContext* ctx) { status = TF_NewStatus(); TF_GetInput(ctx, 0, &tags, status); - if (TF_GetCode(status) == TF_OK){ + if (TF_GetCode(status) == TF_OK) { TF_GetInput(ctx, 1, &values, status); } }; - ~Params(){ + ~Params() { TF_DeleteStatus(status); TF_DeleteTensor(tags); TF_DeleteTensor(values); diff --git a/tensorflow/c/tf_shape_utils_test.cc b/tensorflow/c/tf_shape_utils_test.cc new file mode 100644 index 00000000000..49cf042c5c0 --- /dev/null +++ b/tensorflow/c/tf_shape_utils_test.cc @@ -0,0 +1,43 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/tensor_shape_utils.h" +#include "tensorflow/c/tf_tensor_internal.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { + +void TestShapeMatch(TensorShape shape) { + Tensor tensor(DT_FLOAT, shape); + Status status; + TF_Tensor* tf_tensor = TF_TensorFromTensor(tensor. &status); + ASSERT_EQ(status) + ASSERT_EQ(tensor.shape.DebugString(), TF_ShapeDebugString(tf_tensor)); +} + +TEST(ShapeDebugString, RegularShape) { + TestShapeMatch(TensorShape({5, 4, 7})); +} + +TEST(ShapeDebugString, ShapeWithUnknownDimension) { + TestShapeMatch(TensorShape({5, -1, 7})); +} + + +} // namespace tensorflow From a41366eee035777f38ab1f06e7cba12b1b533a9b Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 15 Jul 2020 00:12:54 +0000 Subject: [PATCH 0061/1017] moved tf_shape_utils under kernels --- tensorflow/c/BUILD | 26 ----------- tensorflow/c/kernels/BUILD | 34 ++++++++++++++- tensorflow/c/kernels/summary_op.cc | 6 +-- .../c/{ => kernels}/tensor_shape_utils.cc | 4 +- .../c/{ => kernels}/tensor_shape_utils.h | 9 +++- .../{ => kernels}/tensor_shape_utils_test.cc | 5 +-- tensorflow/c/tf_shape_utils_test.cc | 43 ------------------- 7 files changed, 47 insertions(+), 80 deletions(-) rename tensorflow/c/{ => kernels}/tensor_shape_utils.cc (92%) rename tensorflow/c/{ => kernels}/tensor_shape_utils.h (81%) rename tensorflow/c/{ => kernels}/tensor_shape_utils_test.cc (88%) delete mode 100644 tensorflow/c/tf_shape_utils_test.cc diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index 65bad3b5de9..410fc22069f 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -513,32 +513,6 @@ tf_cuda_library( alwayslink = 1, ) -tf_cuda_library( - name = "tensor_shape_utils", - srcs = [ - "tensor_shape_utils.cc", - ], - hdrs = [ - "tensor_shape_utils.h", - ], - deps = [ - ":tf_tensor", - ], - copts = tf_copts(), - visibility = ["//visibility:public"], -) - -tf_cc_test( - name = "tensor_shape_utils_test", - srcs = ["tensor_shape_utils_test.cc"], - deps = [ - ":tensor_shape_utils", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - ], -) - # ----------------------------------------------------------------------------- # Tests diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 77fbd869105..e8354a8941d 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -5,6 +5,11 @@ load( "tf_kernel_library", ) +load( + "//tensorflow/core/platform:rules_cc.bzl", + "cc_library" +) + package( default_visibility = ["//visibility:public"], licenses = ["notice"], # Apache 2.0 @@ -28,10 +33,10 @@ tf_kernel_library( name = "summary_op", prefix = "summary_op", deps = [ + "//tensorflow/c/kernels:tensor_shape_utils", "//tensorflow/c:kernels", "//tensorflow/c:tf_tensor", - "//tensorflow/c:tensor_shape_utils", - "//tensorflow/core:framework", + "//tensorflow/core:framework" ], ) @@ -79,6 +84,31 @@ tf_cc_test( ], ) +cc_library( + name = "tensor_shape_utils", + srcs = [ + "tensor_shape_utils.cc", + ], + hdrs = [ + "tensor_shape_utils.h", + ], + deps = [ + "//tensorflow/c:tf_tensor", + ], + visibility = ["//visibility:public"], +) + +tf_cc_test( + name = "tensor_shape_utils_test", + srcs = ["tensor_shape_utils_test.cc"], + deps = [ + ":tensor_shape_utils", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + # Changes to the Android srcs here should be replicated in # tensorflow/contrib/makefile/tf_op_files.txt. # diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 10b46284814..87418c6ccea 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -18,7 +18,7 @@ limitations under the License. #include "tensorflow/c/kernels.h" #include "tensorflow/c/tf_tensor.h" -#include "tensorflow/c/tensor_shape_utils.h" +#include "tensorflow/c/kernels/tensor_shape_utils.h" #include "tensorflow/core/framework/selective_registration.h" #include "tensorflow/core/framework/summary.pb.h" #include "tensorflow/core/framework/types.h" @@ -68,8 +68,8 @@ static void ScalarSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { if (!IsSameSize(params.tags, params.values)) { std::ostringstream err; err << "tags and values not the same shape: " - << TF_ShapeDebugString(params.tags) << " != " - << TF_ShapeDebugString(params.values) + << ShapeDebugString(params.tags) << " != " + << ShapeDebugString(params.values) << SingleTag(params.tags); TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); } diff --git a/tensorflow/c/tensor_shape_utils.cc b/tensorflow/c/kernels/tensor_shape_utils.cc similarity index 92% rename from tensorflow/c/tensor_shape_utils.cc rename to tensorflow/c/kernels/tensor_shape_utils.cc index c38eb95724c..062cdbd049a 100644 --- a/tensorflow/c/tensor_shape_utils.cc +++ b/tensorflow/c/kernels/tensor_shape_utils.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/c/tensor_shape_utils.h" +#include "tensorflow/c/kernels/tensor_shape_utils.h" #include @@ -22,7 +22,7 @@ limitations under the License. #include "tensorflow/core/platform/strcat.h" #include "tensorflow/core/platform/logging.h" -std::string TF_ShapeDebugString(TF_Tensor* tensor) { +std::string ShapeDebugString(TF_Tensor* tensor) { // A TF_Tensor cannot have an unknown rank CHECK_GE(TF_NumDims(tensor), 0); tensorflow::string s = "["; diff --git a/tensorflow/c/tensor_shape_utils.h b/tensorflow/c/kernels/tensor_shape_utils.h similarity index 81% rename from tensorflow/c/tensor_shape_utils.h rename to tensorflow/c/kernels/tensor_shape_utils.h index cde929f3f4e..a62f460998b 100644 --- a/tensorflow/c/tensor_shape_utils.h +++ b/tensorflow/c/kernels/tensor_shape_utils.h @@ -13,6 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +// This file contains shape utilities to be used by kernels and is not part of +// the C API. As such, it is subject to change at any time. + #ifndef TENSORFLOW_C_TENSOR_SHAPE_UTILS_H_ #define TENSORFLOW_C_TENSOR_SHAPE_UTILS_H_ @@ -20,12 +23,16 @@ limitations under the License. #include "tensorflow/c/tf_tensor.h" +namespace tensorflow { + // The following are utils for the shape of a TF_Tensor type. // These functions may later be subsumed by the methods for a // TF_TensorShape type // Returns a string representation of the TF_Tensor -std::string TF_ShapeDebugString(TF_Tensor* tensor); +std::string ShapeDebugString(TF_Tensor* tensor); + +} // namespace tensorflow #endif // TENSORFLOW_C_TENSOR_SHAPE_UTILS_H_ diff --git a/tensorflow/c/tensor_shape_utils_test.cc b/tensorflow/c/kernels/tensor_shape_utils_test.cc similarity index 88% rename from tensorflow/c/tensor_shape_utils_test.cc rename to tensorflow/c/kernels/tensor_shape_utils_test.cc index ef1fd1e839f..25620838437 100644 --- a/tensorflow/c/tensor_shape_utils_test.cc +++ b/tensorflow/c/kernels/tensor_shape_utils_test.cc @@ -13,14 +13,13 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/c/tensor_shape_utils.h" #include "tensorflow/c/tf_tensor_internal.h" +#include "tensorflow/c/kernels/tensor_shape_utils.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/framework/partial_tensor_shape.h" namespace tensorflow { @@ -30,7 +29,7 @@ void TestShapeMatch(T shape) { Status status; TF_Tensor* tf_tensor = TF_TensorFromTensor(tensor, &status); ASSERT_TRUE(status.ok()) << status.ToString(); - ASSERT_EQ(tensor.shape().DebugString(), TF_ShapeDebugString(tf_tensor)); + ASSERT_EQ(tensor.shape().DebugString(), ShapeDebugString(tf_tensor)); } TEST(ShapeDebugString, RegularShape) { diff --git a/tensorflow/c/tf_shape_utils_test.cc b/tensorflow/c/tf_shape_utils_test.cc deleted file mode 100644 index 49cf042c5c0..00000000000 --- a/tensorflow/c/tf_shape_utils_test.cc +++ /dev/null @@ -1,43 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/c/tensor_shape_utils.h" -#include "tensorflow/c/tf_tensor_internal.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/lib/core/status.h" - -namespace tensorflow { - -void TestShapeMatch(TensorShape shape) { - Tensor tensor(DT_FLOAT, shape); - Status status; - TF_Tensor* tf_tensor = TF_TensorFromTensor(tensor. &status); - ASSERT_EQ(status) - ASSERT_EQ(tensor.shape.DebugString(), TF_ShapeDebugString(tf_tensor)); -} - -TEST(ShapeDebugString, RegularShape) { - TestShapeMatch(TensorShape({5, 4, 7})); -} - -TEST(ShapeDebugString, ShapeWithUnknownDimension) { - TestShapeMatch(TensorShape({5, -1, 7})); -} - - -} // namespace tensorflow From a574b291479712e8fe60a9b029a8b30566e31749 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 15 Jul 2020 00:20:37 +0000 Subject: [PATCH 0062/1017] added namespace tensorflow --- tensorflow/c/kernels/summary_op.cc | 4 ++-- tensorflow/c/kernels/tensor_shape_utils.cc | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 87418c6ccea..7500c3046e1 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -68,8 +68,8 @@ static void ScalarSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { if (!IsSameSize(params.tags, params.values)) { std::ostringstream err; err << "tags and values not the same shape: " - << ShapeDebugString(params.tags) << " != " - << ShapeDebugString(params.values) + << tensorflow::ShapeDebugString(params.tags) << " != " + << tensorflow::ShapeDebugString(params.values) << SingleTag(params.tags); TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); } diff --git a/tensorflow/c/kernels/tensor_shape_utils.cc b/tensorflow/c/kernels/tensor_shape_utils.cc index 062cdbd049a..b3cba8cb99f 100644 --- a/tensorflow/c/kernels/tensor_shape_utils.cc +++ b/tensorflow/c/kernels/tensor_shape_utils.cc @@ -22,6 +22,8 @@ limitations under the License. #include "tensorflow/core/platform/strcat.h" #include "tensorflow/core/platform/logging.h" +namespace tensorflow { + std::string ShapeDebugString(TF_Tensor* tensor) { // A TF_Tensor cannot have an unknown rank CHECK_GE(TF_NumDims(tensor), 0); @@ -35,4 +37,5 @@ std::string ShapeDebugString(TF_Tensor* tensor) { } tensorflow::strings::StrAppend(&s, "]"); return s; -} \ No newline at end of file +} +} // namespace tensorflow \ No newline at end of file From d09fd4d5f1a490e8ca0a2c439959757c952625a2 Mon Sep 17 00:00:00 2001 From: ShengYang1 Date: Wed, 15 Jul 2020 13:18:23 +0800 Subject: [PATCH 0063/1017] Refine UT code --- .../core/grappler/optimizers/remapper.cc | 2 +- .../core/grappler/optimizers/remapper_test.cc | 110 +++++++++--------- 2 files changed, 54 insertions(+), 58 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/remapper.cc b/tensorflow/core/grappler/optimizers/remapper.cc index 4785e2a633f..661ad7895c2 100644 --- a/tensorflow/core/grappler/optimizers/remapper.cc +++ b/tensorflow/core/grappler/optimizers/remapper.cc @@ -87,7 +87,7 @@ struct FusedBatchNorm { int fused_batch_norm = kMissingIndex; }; -// Comparison op with cast +// Comparison op followed by a cast, e.g., GreaterEqual + Cast. struct ComparisonWithCast { ComparisonWithCast() = default; diff --git a/tensorflow/core/grappler/optimizers/remapper_test.cc b/tensorflow/core/grappler/optimizers/remapper_test.cc index eac6b291af4..417ecd6dd44 100644 --- a/tensorflow/core/grappler/optimizers/remapper_test.cc +++ b/tensorflow/core/grappler/optimizers/remapper_test.cc @@ -925,64 +925,60 @@ TEST_F(RemapperTest, FuseConv2DWithSqueezeAndBias) { } #endif -#define REGISTER_TEST_ALL_TYPES(TEST) \ - REGISTER_TEST(TEST, DT_FLOAT); \ - REGISTER_TEST(TEST, DT_BFLOAT16); - -#define REGISTER_TEST(CMP, TYPE) \ - TEST_F(RemapperTest, Fuse##CMP##WithCast_##TYPE) { \ - using ::tensorflow::ops::Placeholder; \ - for (bool is_training : {true, false}) { \ - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); \ - const int num_channels = 24; \ - TensorShape channel_shape({num_channels}); \ - TensorShape empty_shape({0}); \ - auto x = Placeholder(s.WithOpName("x"), TYPE, \ - ops::Placeholder::Shape({2, 8, 8, num_channels})); \ - auto y = Placeholder(s.WithOpName("y"), TYPE, \ - ops::Placeholder::Shape({2, 8, 8, num_channels})); \ - float epsilon = 0.1f; \ - auto comparator = ops::CMP(s.WithOpName("cmp_op"), x, y); \ - auto cast = ops::Cast(s.WithOpName("cast"), comparator.z, TYPE); \ - auto fetch = ops::Identity(s.WithOpName("fetch"), cast); \ - auto input1_t = GenerateRandomTensor({2, 8, 8, num_channels}); \ - auto input2_t = GenerateRandomTensor({2, 8, 8, num_channels}); \ - GrapplerItem item; \ - item.fetch = {"fetch"}; \ - item.feed = {{"x", input1_t}, {"y", input2_t}}; \ - TF_ASSERT_OK(s.ToGraphDef(&item.graph)); \ - for (int i = 0; i < item.graph.node_size(); ++i) { \ - item.graph.mutable_node(i)->set_device("/device:CPU:0"); \ - } \ - Remapper optimizer(RewriterConfig::AGGRESSIVE); \ - GraphDef output; \ - TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output)); \ - int found = 0; \ - for (const NodeDef& node : output.node()) { \ - if (node.name() == "cast") { \ - EXPECT_EQ(node.op(), "_" #CMP "WithCast"); \ - ASSERT_EQ(node.input_size(), 2); \ - EXPECT_EQ(node.input(0), "x"); \ - EXPECT_EQ(node.input(1), "y"); \ - found++; \ - } \ - } \ - EXPECT_EQ(found, 1); \ - auto tensors_expected = \ - EvaluateNodes(item.graph, item.fetch, item.feed); \ - ASSERT_EQ(tensors_expected.size(), 1); \ - auto tensors = EvaluateNodes(output, item.fetch, item.feed); \ - ASSERT_EQ(tensors.size(), 1); \ - test::ExpectClose(tensors[0], tensors_expected[0], 1e-2, 1e-2); \ - } \ +class FusedCmpAndCastTest : public GrapplerTest { + protected: + template + void TestFusedCmpAndCast() { + using ::tensorflow::ops::Placeholder; + for (bool is_training : {true, false}) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + const int num_channels = 24; + TensorShape channel_shape({num_channels}); + TensorShape empty_shape({0}); + auto x = Placeholder(s.WithOpName("x"), TYPE, + ops::Placeholder::Shape({2, 8, 8, num_channels})); + auto y = Placeholder(s.WithOpName("y"), TYPE, + ops::Placeholder::Shape({2, 8, 8, num_channels})); + float epsilon = 0.1f; + auto comparator = ops::Equal(s.WithOpName("Equal"), x, y); + auto cast = ops::Cast(s.WithOpName("cast"), comparator.z, TYPE); + auto fetch = ops::Identity(s.WithOpName("fetch"), cast); + auto input1_t = GenerateRandomTensor({2, 8, 8, num_channels}); + auto input2_t = GenerateRandomTensor({2, 8, 8, num_channels}); + GrapplerItem item; + item.fetch = {"fetch"}; + item.feed = {{"x", input1_t}, {"y", input2_t}}; + TF_ASSERT_OK(s.ToGraphDef(&item.graph)); + for (int i = 0; i < item.graph.node_size(); ++i) { + item.graph.mutable_node(i)->set_device("/device:CPU:0"); + } + Remapper optimizer(RewriterConfig::AGGRESSIVE); + GraphDef output; + TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output)); + int found = 0; + for (const NodeDef& node : output.node()) { + if (node.name() == "cast") { + EXPECT_EQ(node.op(), "_EqualWithCast"); + ASSERT_EQ(node.input_size(), 2); + EXPECT_EQ(node.input(0), "x"); + EXPECT_EQ(node.input(1), "y"); + found++; + } + } + EXPECT_EQ(found, 1); + auto tensors_expected = EvaluateNodes(item.graph, item.fetch, item.feed); + ASSERT_EQ(tensors_expected.size(), 1); + auto tensors = EvaluateNodes(output, item.fetch, item.feed); + ASSERT_EQ(tensors.size(), 1); + test::ExpectClose(tensors[0], tensors_expected[0], 1e-2, 1e-2); + } } -REGISTER_TEST_ALL_TYPES(GreaterEqual) -REGISTER_TEST_ALL_TYPES(Greater) -REGISTER_TEST_ALL_TYPES(LessEqual) -REGISTER_TEST_ALL_TYPES(Less) -REGISTER_TEST_ALL_TYPES(Equal) -REGISTER_TEST_ALL_TYPES(NotEqual) -#undef REGISTER_TEST +}; + +TEST_F(FusedCmpAndCastTest, FusedCmpAndCast) { + TestFusedCmpAndCast(); + TestFusedCmpAndCast(); +} } // namespace grappler } // namespace tensorflow From 2191a9d795259b211dfecadb8a9e3a471b1488da Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 15 Jul 2020 06:23:33 +0000 Subject: [PATCH 0064/1017] update map test cases for insert and erase --- tensorflow/core/kernels/map_kernels.h | 8 ++-- .../python/kernel_tests/map_ops_test.py | 41 +++++++++++++++++-- tensorflow/python/ops/map_ops.py | 5 +-- 3 files changed, 44 insertions(+), 10 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 98ce1bfac1b..33a950ee63a 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -144,6 +144,10 @@ class TensorMapLookup : public OpKernel { const TensorKey& key = c->input(1); const TensorMap* m = nullptr; OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); + + OP_REQUIRES(c, m->tensors().find(key) != m->tensors().end(), + errors::InvalidArgument("Trying to lookup non-existent key.")); + c->set_output(0, m->tensors().find(key)->second); } @@ -159,11 +163,9 @@ class TensorMapErase : public OpKernel { } void Compute(OpKernelContext* c) override { - std::cout << "hello TensorMapErase op" << std::endl; const TensorMap* m = nullptr; OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); - const Tensor& temp_key = c->input(1); - const TensorKey key = TensorKey(temp_key); + const TensorKey& key = c->input(1); OP_REQUIRES(c, !m->tensors().empty(), errors::InvalidArgument("Trying to erase from an empty map.")); diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index e95a1ab9bec..d8a075c7b4e 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -1,4 +1,4 @@ -# Copyright 2018 The Sonnet Authors. All Rights Reserved. +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ from tensorflow.python.eager import def_function from tensorflow.python.eager import function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import errors from tensorflow.python.ops import map_ops @test_util.run_all_in_graph_and_eager_modes @@ -56,7 +57,17 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): m = map_ops.tensor_map_insert(m, k, v) l = map_ops.tensor_map_lookup(m, k) self.assertAllClose(l, v) - + + def testTensorMapLookupMissingKeyFails(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Trying to lookup non-existent key."): + l = map_ops.tensor_map_lookup(m, k) + self.evaluate(l) + def testTensorMapReplace(self): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) @@ -76,13 +87,35 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k, v) s = map_ops.tensor_map_size(m) - self.assertAllClose(s, 1) + self.assertAllEqual(s, 1) m, e = map_ops.tensor_map_erase(m, k) s = map_ops.tensor_map_size(m) - self.assertAllClose(s, 0) + self.assertAllEqual(s, 0) self.assertAllClose(e, v) + def testTensorMapEraseFromEmptyMapFails(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Trying to erase from an empty map."): + m, e = map_ops.tensor_map_erase(m, k) + self.evaluate(e) + + def testTensorMapEraseMissingKeyFails(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + k2 = constant_op.constant(2.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k2, v) + + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Trying to erase non-existent item."): + m, e = map_ops.tensor_map_erase(m, k) + self.evaluate(e) + def testInsertLookupGrad(self): with backprop.GradientTape() as tape: m = map_ops.empty_tensor_map() diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 4ea50e114ac..20806e6fd30 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -1,4 +1,4 @@ -# Copyright 2018 The Sonnet Authors. All Rights Reserved. +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -52,14 +52,13 @@ def LookupGrad(op, dval): m, k = op.inputs map_grad = empty_tensor_map() map_grad = tensor_map_insert(map_grad, k, dval) - key = op.inputs[1] key_grad = None return map_grad, key_grad @ops.RegisterGradient("TensorMapInsert") def InsertGrad(op, dmap): _, key, val = op.inputs - map_grad, _ = gen_map_ops.tensor_map_erase(dmap, key) + map_grad = None key_grad = None value_grad = tensor_map_lookup(dmap, key) return map_grad, key_grad, value_grad From 13e3f357196e7fb2d80b7b105800784e5ce0f27a Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 15 Jul 2020 06:34:38 +0000 Subject: [PATCH 0065/1017] test replace missing key fails --- tensorflow/core/kernels/map_kernels.h | 3 +++ tensorflow/python/kernel_tests/map_ops_test.py | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 33a950ee63a..00a6a654b54 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -199,6 +199,9 @@ class TensorMapReplace : public OpKernel { const TensorMap* m = nullptr; OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); + OP_REQUIRES(c, m->tensors().find(key) != m->tensors().end(), + errors::InvalidArgument("Trying to replace non-existent key.")); + TensorMap* output_map = nullptr; OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); output_map->replace(key,value); diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index d8a075c7b4e..a0bfd104a9c 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -81,6 +81,18 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = map_ops.tensor_map_lookup(m, k) self.assertAllClose(l, v2) + def testTensorMapReplaceMissingKeyFails(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + k2 = constant_op.constant(2.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k2, v) + + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Trying to replace non-existent key."): + m = map_ops.tensor_map_replace(m, k, v) + self.evaluate(m) + def testTensorMapErase(self): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) From c4b16f1c258cfd6320bd92eb60de8be0899daf14 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 15 Jul 2020 15:35:45 +0000 Subject: [PATCH 0066/1017] restore list_ops_test.py --- .../python/kernel_tests/list_ops_test.py | 132 +++++++++--------- 1 file changed, 66 insertions(+), 66 deletions(-) diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py index ce20cf489e6..53ebdd3ab88 100644 --- a/tensorflow/python/kernel_tests/list_ops_test.py +++ b/tensorflow/python/kernel_tests/list_ops_test.py @@ -78,8 +78,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[], max_num_elements=1) l = list_ops.tensor_list_push_back(l, constant_op.constant(1.0)) - with self.assertRaisesRegex(errors.InvalidArgumentError, - "Tried to push item into a full list"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "Tried to push item into a full list"): l = list_ops.tensor_list_push_back(l, 2.) self.evaluate(l) @@ -91,8 +91,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): element_dtype=dtypes.float32, element_shape=[], max_num_elements=max_num_elements) - with self.assertRaisesRegex(errors.InvalidArgumentError, - "Trying to pop from an empty list"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "Trying to pop from an empty list"): l = list_ops.tensor_list_pop_back(l, element_dtype=dtypes.float32) self.evaluate(l) @@ -115,7 +115,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testPopUninitializedTensorWithInvalidElementShapeFails(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Trying to read an uninitialized tensor but " "element_shape is not fully defined"): @@ -124,7 +124,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=[None, 2], num_elements=3) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Incompatible shapes during merge: \[1,3\] vs. \[\?,2\]"): _, e = gen_list_ops.tensor_list_pop_back( @@ -191,8 +191,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should raise an error when the element tensors do not all have the same # shape. - with self.assertRaisesRegex(errors.InvalidArgumentError, - "Incompatible ranks during merge: 0 vs. 1"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "Incompatible ranks during merge: 0 vs. 1"): l = list_ops.tensor_list_push_back(l, constant_op.constant([3.0, 4.0])) t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32) self.evaluate(t) @@ -213,7 +213,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should raise an error when the element tensors do not all have the same # shape. - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Incompatible shapes during merge: \[1\] vs. \[2\]"): l = list_ops.tensor_list_push_back(l, constant_op.constant([2.0, 3.0])) @@ -234,8 +234,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should not be able to stack empty lists with partially defined # element_shape. - with self.assertRaisesRegex(errors.InvalidArgumentError, - "non-fully-defined"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[None, 2], @@ -244,8 +244,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.evaluate(t) # Should not be able to stack empty lists with undefined element_shape. - with self.assertRaisesRegex(errors.InvalidArgumentError, - "non-fully-defined"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=None, @@ -285,10 +285,10 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testStackReservedListWithNoElementsAndPartialElementShapeFails(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegex( - errors.InvalidArgumentError, "Tried to stack list which only contains " - "uninitialized tensors and has a " - "non-fully-defined element_shape: "): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "Tried to stack list which only contains " + "uninitialized tensors and has a " + "non-fully-defined element_shape: "): t = list_ops.tensor_list_stack(l, element_dtype=dtypes.float32) self.evaluate(t) @@ -341,8 +341,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should raise an error when the requested tensors do not all have the same # shape. - with self.assertRaisesRegex(errors.InvalidArgumentError, - "Incompatible ranks during merge: 0 vs. 1"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "Incompatible ranks during merge: 0 vs. 1"): t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32) self.evaluate(t) @@ -366,7 +366,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should raise an error when the requested tensors do not all have the same # shape. - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Incompatible shapes during merge: \[1\] vs. \[2\]"): t = list_ops.tensor_list_gather(l, [0, 2], element_dtype=dtypes.float32) @@ -387,8 +387,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should not be able to gather from empty lists with partially defined # element_shape. - with self.assertRaisesRegex(errors.InvalidArgumentError, - "non-fully-defined"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[None, 2], @@ -398,8 +398,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # Should not be able to gather from empty lists with undefined # element_shape. - with self.assertRaisesRegex(errors.InvalidArgumentError, - "non-fully-defined"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "non-fully-defined"): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=None, @@ -455,7 +455,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testGatherReservedListWithNoElementsAndPartialElementShapeFails(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Tried to gather uninitialized tensors from a" " list with non-fully-defined element_shape"): @@ -485,7 +485,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testScatterFailsWhenIndexLargerThanNumElements(self): c0 = constant_op.constant([1.0, 2.0]) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "TensorListScatter: Trying to scatter at index 3 in list with size 3"): l = gen_list_ops.tensor_list_scatter_v2( @@ -494,7 +494,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testScatterFailsWithInvalidNumElements(self): c0 = constant_op.constant([1.0, 2.0]) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "TensorListScatter expects num_elements >= -1, found: -2"): l = gen_list_ops.tensor_list_scatter_v2( @@ -503,7 +503,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testScatterWithInvalidRowsInInputTensorFails(self): c0 = constant_op.constant([1.0, 2.0]) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Invalid number of rows in input tensor. Expected: 3 Actual: 2"): l = list_ops.tensor_list_scatter(c0, [1, 0, 2], []) @@ -511,7 +511,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testScatterWithNegativeIndicesFails(self): c0 = constant_op.constant([1.0, 2.0]) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Indices in TensorListScatter must all be non-negative."): l = list_ops.tensor_list_scatter(c0, [-1, -2], element_shape=[]) @@ -658,7 +658,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testGetUninitializedTensorWithInvalidElementShapeFails(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Trying to read an uninitialized tensor but " "element_shape is not fully defined"): @@ -676,7 +676,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): error_type = errors.InvalidArgumentError else: error_type = ValueError - with self.assertRaisesRegex(error_type, r"shapes"): + with self.assertRaisesRegexp(error_type, r"shapes"): e0 = gen_list_ops.tensor_list_get_item( l, 0, element_dtype=dtypes.float32, element_shape=[1, 3]) self.evaluate(e0) @@ -699,7 +699,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testSetOnEmptyListWithMaxNumElementsFails(self): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[], max_num_elements=3) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Trying to modify element 0 in a list with 0 elements."): l = list_ops.tensor_list_set_item(l, 0, 1.) @@ -882,8 +882,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with ops.device("/job:ps"): l_ps = array_ops.identity(l) l_ps = list_ops.tensor_list_push_back(l_ps, 2.) - with self.assertRaisesRegex(errors.InvalidArgumentError, - "Tried to push item into a full list"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "Tried to push item into a full list"): with ops.device("/job:worker"): l_worker = array_ops.identity(l_ps) l_worker = list_ops.tensor_list_push_back(l_worker, 3.0) @@ -943,8 +943,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): # at graph building time. l = list_ops.tensor_list_set_item(l, 0, ph) l_0 = list_ops.tensor_list_get_item(l, 0, element_dtype=dtypes.float32) - with self.assertRaisesRegex(errors.InvalidArgumentError, - "incompatible shape"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "incompatible shape"): sess.run(l_0, {ph: [3.0]}) def testResourceVariableScatterGather(self): @@ -1021,7 +1021,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): "element shapes are not identical at index 0") else: expected_error = (ValueError, "Shapes must be equal rank") - with self.assertRaisesRegex(*expected_error): + with self.assertRaisesRegexp(*expected_error): l_batch_of_vec_tls = array_ops.stack( [list_ops.tensor_list_from_tensor([[1.0]], element_shape=[1])] * 2) self.evaluate( @@ -1033,7 +1033,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): r"input_b\[0\].dtype != element_dtype.") else: expected_error = (ValueError, "input_b.type != element_dtype") - with self.assertRaisesRegex(*expected_error): + with self.assertRaisesRegexp(*expected_error): l_batch_of_int_tls = array_ops.stack( [list_ops.tensor_list_from_tensor([1], element_shape=[])] * 2) self.evaluate( @@ -1073,8 +1073,8 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.assertRaises((errors.InvalidArgumentError, ValueError)): self.evaluate(list_ops.tensor_list_push_back_batch(l_batch, [])) - with self.assertRaisesRegex(errors.InvalidArgumentError, - "incompatible shape to a list at index 0"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + "incompatible shape to a list at index 0"): self.evaluate( list_ops.tensor_list_push_back_batch(l_batch, [[3.0], [4.0]])) @@ -1082,7 +1082,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): expected_error = (errors.InvalidArgumentError, "Invalid data type") else: expected_error = (ValueError, "wrong element dtype") - with self.assertRaisesRegex(*expected_error): + with self.assertRaisesRegexp(*expected_error): self.evaluate(list_ops.tensor_list_push_back_batch(l_batch, [3, 4])) def testZerosLike(self): @@ -1246,7 +1246,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): element_shape=[], element_dtype=dtypes.float32, num_elements=2) l2 = list_ops.tensor_list_reserve( element_shape=[], element_dtype=dtypes.float32, num_elements=3) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Trying to add two lists of tensors with different lengths"): l = math_ops.add_n([l1, l2]) @@ -1268,7 +1268,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): element_dtype=dtypes.float32, num_elements=3) l = math_ops.add_n([l1, l2]) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Trying to add two lists of tensors with incompatible element shapes" ): @@ -1314,7 +1314,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): element_dtype=dtypes.float32, element_shape=None) l = list_ops.tensor_list_push_back(l, [[0., 1.]]) l = list_ops.tensor_list_push_back(l, [[2.], [4.]]) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Incompatible shapes during merge: " r"\[2\] vs. \[1\]"): t = list_ops.tensor_list_concat(l, element_dtype=dtypes.float32) @@ -1333,7 +1333,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testConcatEmptyListWithUnknownElementShapeFails(self): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=None) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "All except the first dimension must be fully" " defined when concating an empty tensor list"): @@ -1343,7 +1343,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testConcatEmptyListWithPartiallyDefinedElementShapeFails(self): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=[2, None]) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "All except the first dimension must be fully" " defined when concating an empty tensor list"): @@ -1354,7 +1354,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=tensor_shape.TensorShape([])) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Concat requires elements to be at least vectors, " "found scalars instead"): @@ -1365,14 +1365,14 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.empty_tensor_list( element_dtype=dtypes.float32, element_shape=None) l1 = list_ops.tensor_list_push_back(l, 1.) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Concat saw a scalar shape at index 0" " but requires at least vectors"): t = list_ops.tensor_list_concat(l1, element_dtype=dtypes.float32) self.evaluate(t) l1 = list_ops.tensor_list_push_back(l, [1.]) l1 = list_ops.tensor_list_push_back(l1, 2.) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "Concat saw a scalar shape at index 1" " but requires at least vectors"): t = list_ops.tensor_list_concat(l1, element_dtype=dtypes.float32) @@ -1420,7 +1420,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testConcatWithUninitializedTensorsFailsIfNoElementShape(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=None, num_elements=3) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Trying to concat list with only uninitialized tensors " r"but element_shape_except_first_dim_ is not fully defined"): @@ -1430,7 +1430,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testConcatWithUninitializedTensorsFailsIfNoInputLengths(self): l = list_ops.tensor_list_reserve( element_dtype=dtypes.float32, element_shape=[None, 3], num_elements=3) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"List contains uninitialized tensor at index 0" r" but leading_dims has only 0 elements."): @@ -1467,7 +1467,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.cached_session(): tensor = array_ops.placeholder(dtype=dtypes.float32) l = list_ops.tensor_list_split(tensor, element_shape=None, lengths=[1]) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Tensor must be at least a vector, but saw shape: \[\]"): l.eval({tensor: 1}) @@ -1479,24 +1479,24 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=lengths) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Expected lengths to be a vector, received shape: \[\]"): l.eval({lengths: 1}) def testSplitWithInvalidLengthsFails(self): - with self.assertRaisesRegex(errors.InvalidArgumentError, - r"Invalid value in lengths: -1"): + with self.assertRaisesRegexp(errors.InvalidArgumentError, + r"Invalid value in lengths: -1"): l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[1, -1]) self.evaluate(l) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Attempting to slice \[0, 3\] from tensor with length 2"): l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[3]) self.evaluate(l) - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"Unused values in tensor. Length of tensor: 2 Values used: 1"): l = list_ops.tensor_list_split([1., 2.], element_shape=None, lengths=[1]) @@ -1504,11 +1504,11 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @test_util.run_deprecated_v1 def testSkipEagerSplitWithScalarElementShapeFails(self): - with self.assertRaisesRegex(ValueError, - r"Shapes must be equal rank, but are 1 and 0"): + with self.assertRaisesRegexp(ValueError, + r"Shapes must be equal rank, but are 1 and 0"): l = list_ops.tensor_list_split([1., 2.], element_shape=[], lengths=[1, 1]) with self.cached_session(): - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"TensorListSplit requires element_shape to be at least of rank 1, " r"but saw: \[\]"): @@ -1520,7 +1520,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testEagerOnlySplitWithScalarElementShapeFails(self): if context.executing_eagerly(): - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"TensorListSplit requires element_shape to be at least of rank 1, " r"but saw: \[\]"): @@ -1528,14 +1528,14 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @test_util.run_deprecated_v1 def testSkipEagerSplitWithIncompatibleTensorShapeAndElementShapeFails(self): - with self.assertRaisesRegex(ValueError, - r"Shapes must be equal rank, but are 2 and 1"): + with self.assertRaisesRegexp(ValueError, + r"Shapes must be equal rank, but are 2 and 1"): l = list_ops.tensor_list_split([[1.], [2.]], element_shape=[1], lengths=[1, 1]) with self.cached_session(): - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"tensor shape \[2,1\] is not compatible with element_shape \[1\]"): element_shape = array_ops.placeholder(dtype=dtypes.int32) @@ -1546,7 +1546,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testEagerOnlySplitWithIncompatibleTensorShapeAndElementShapeFails(self): if context.executing_eagerly(): - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, r"tensor shape \[2,1\] is not compatible with element_shape \[1\]"): list_ops.tensor_list_split([[1.], [2.]], @@ -1576,7 +1576,7 @@ class ListOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): [1., 2.]) def testResizeWithInvalidSizeFails(self): - with self.assertRaisesRegex( + with self.assertRaisesRegexp( errors.InvalidArgumentError, "TensorListSlice expects size to be non-negative"): l = list_ops.tensor_list_from_tensor([1., 2., 3.], element_shape=[]) From 344c570221846e2ac205862083317f3e01fa63b0 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 15 Jul 2020 19:22:46 +0000 Subject: [PATCH 0067/1017] update TensorMapDeviceCopy --- tensorflow/core/kernels/map_kernels.h | 1 + tensorflow/core/kernels/tensor_map.cc | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 00a6a654b54..9f18e4242d5 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -211,6 +211,7 @@ class TensorMapReplace : public OpKernel { DataType element_dtype_; }; + } // namespace tensorflow #endif // TENSORFLOW_CORE_KERNELS_MAP_KERNELS_H_ diff --git a/tensorflow/core/kernels/tensor_map.cc b/tensorflow/core/kernels/tensor_map.cc index bcb2abebe01..4f694a37b17 100644 --- a/tensorflow/core/kernels/tensor_map.cc +++ b/tensorflow/core/kernels/tensor_map.cc @@ -33,10 +33,10 @@ void TensorMap::Encode(VariantTensorData* data) const { Tensor k = map_it->first; Tensor v = map_it->second; // TODO: k should also not be DT_RESOURCE or DT_VARIANT - if(k.dtype() != DT_INVALID && v.dtype() != DT_INVALID) { - *data->add_tensors() = k; - *data->add_tensors() = v; - } + CHECK_NE(k.dtype(), DT_INVALID); + CHECK_NE(v.dtype(), DT_INVALID); + *data->add_tensors() = k; + *data->add_tensors() = v; map_it++; } string metadata; @@ -56,9 +56,11 @@ static Status TensorMapDeviceCopy( to->element_shape = from.element_shape; to->element_dtype = from.element_dtype; for (const std::pair& p : from.tensors()) { - if (p.first.dtype() != DT_INVALID && p.second.dtype() != DT_INVALID) { - to->tensors().emplace(p.first, p.second); - } + TensorKey to_key(p.first.dtype()); + Tensor to_val(p.second.dtype()); + copy(p.first, &to_key); + copy(p.second, &to_val); + to->tensors().emplace(to_key, to_val); } return Status::OK(); } From f18240b8761942524fb5c8049169f2e99b9440a5 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 15 Jul 2020 19:58:45 +0000 Subject: [PATCH 0068/1017] minor fix --- tensorflow/core/kernels/map_kernels.h | 5 +---- tensorflow/python/kernel_tests/map_ops_test.py | 10 ---------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 9f18e4242d5..1ab6fbd2323 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -167,9 +167,6 @@ class TensorMapErase : public OpKernel { OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); const TensorKey& key = c->input(1); - OP_REQUIRES(c, !m->tensors().empty(), - errors::InvalidArgument("Trying to erase from an empty map.")); - OP_REQUIRES(c, m->tensors().find(key) != m->tensors().end(), errors::InvalidArgument("Trying to erase non-existent item.")); @@ -204,7 +201,7 @@ class TensorMapReplace : public OpKernel { TensorMap* output_map = nullptr; OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); - output_map->replace(key,value); + output_map->replace(key, value); } private: diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index a0bfd104a9c..09df2cca134 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -105,16 +105,6 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): s = map_ops.tensor_map_size(m) self.assertAllEqual(s, 0) self.assertAllClose(e, v) - - def testTensorMapEraseFromEmptyMapFails(self): - m = map_ops.empty_tensor_map() - k = constant_op.constant(1.0) - v = constant_op.constant(2.0) - - with self.assertRaisesRegex(errors.InvalidArgumentError, - "Trying to erase from an empty map."): - m, e = map_ops.tensor_map_erase(m, k) - self.evaluate(e) def testTensorMapEraseMissingKeyFails(self): m = map_ops.empty_tensor_map() From 5a93bb92d4c4472d5d821d5aa2e5b1a8bc4b6b52 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 15 Jul 2020 20:42:32 +0000 Subject: [PATCH 0069/1017] BUILD file and summary.cc style --- tensorflow/c/kernels/BUILD | 17 +++-------------- tensorflow/c/kernels/ops/summary.cc | 5 ++--- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index e8354a8941d..c5c652ab5d7 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -5,11 +5,6 @@ load( "tf_kernel_library", ) -load( - "//tensorflow/core/platform:rules_cc.bzl", - "cc_library" -) - package( default_visibility = ["//visibility:public"], licenses = ["notice"], # Apache 2.0 @@ -86,15 +81,9 @@ tf_cc_test( cc_library( name = "tensor_shape_utils", - srcs = [ - "tensor_shape_utils.cc", - ], - hdrs = [ - "tensor_shape_utils.h", - ], - deps = [ - "//tensorflow/c:tf_tensor", - ], + srcs = ["tensor_shape_utils.cc",], + hdrs = ["tensor_shape_utils.h",], + deps = [ "//tensorflow/c:tf_tensor",], visibility = ["//visibility:public"], ) diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc index 9cacda36adf..98b8b743fa1 100644 --- a/tensorflow/c/kernels/ops/summary.cc +++ b/tensorflow/c/kernels/ops/summary.cc @@ -24,9 +24,8 @@ static void scalar_summary_shape_inference_fn(TF_ShapeInferenceContext* ctx, // Make shape handle a scalar value (empty shape) TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); if (TF_GetCode(status) != TF_OK) { - std::ostringstream err; - err << "Error in setting output shape inference"; - TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); + TF_SetStatus(status, TF_INVALID_ARGUMENT, + "Error in setting output shape inference"); } TF_DeleteShapeHandle(result); } From b9a65d13537196e2d04af9c00b1cde56efab736e Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 15 Jul 2020 21:22:18 +0000 Subject: [PATCH 0070/1017] fixed style errors: --- tensorflow/c/kernels/summary_op.cc | 7 +++---- tensorflow/c/kernels/tensor_shape_utils.cc | 5 ++--- tensorflow/c/kernels/tensor_shape_utils.h | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 7500c3046e1..9ebda7188cb 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -56,6 +56,8 @@ static void ScalarSummaryOp_Delete(void* kernel) { // Helper functions for compute method bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2); +// Returns a string representation of a single tag or empty string if there +// are multiple tags static tensorflow::string SingleTag(TF_Tensor* tags); template @@ -72,12 +74,9 @@ static void ScalarSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { << tensorflow::ShapeDebugString(params.values) << SingleTag(params.tags); TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); - } - if (TF_GetCode(params.status) != TF_OK){ TF_OpKernelContext_Failure(ctx, params.status); return; } - // Convert tags and values tensor to array to access elements by index tensorflow::Summary s; auto tags_array = static_cast( @@ -148,7 +147,7 @@ void RegisterScalarSummaryOpKernel() { } // A dummy static variable initialized by a lambda whose side-effect is to -// register the bitcast kernel. +// register the ScalarSummary kernel. TF_ATTRIBUTE_UNUSED static bool IsScalarSummaryOpKernelRegistered = []() { if (SHOULD_REGISTER_OP_KERNEL("ScalarSummary")) { RegisterScalarSummaryOpKernel(); diff --git a/tensorflow/c/kernels/tensor_shape_utils.cc b/tensorflow/c/kernels/tensor_shape_utils.cc index b3cba8cb99f..6ca138584b7 100644 --- a/tensorflow/c/kernels/tensor_shape_utils.cc +++ b/tensorflow/c/kernels/tensor_shape_utils.cc @@ -18,20 +18,19 @@ limitations under the License. #include #include "tensorflow/c/tf_tensor.h" -#include "tensorflow/core/platform/str_util.h" #include "tensorflow/core/platform/strcat.h" #include "tensorflow/core/platform/logging.h" namespace tensorflow { std::string ShapeDebugString(TF_Tensor* tensor) { - // A TF_Tensor cannot have an unknown rank + // A TF_Tensor cannot have an unknown rank. CHECK_GE(TF_NumDims(tensor), 0); tensorflow::string s = "["; for (int i = 0; i < TF_NumDims(tensor); ++i) { if (i > 0) tensorflow::strings::StrAppend(&s, ","); int64_t dim = TF_Dim(tensor, i); - // A TF_Tensor cannot have an unknown dimension + // A TF_Tensor cannot have an unknown dimension. CHECK_GE(dim, 0); tensorflow::strings::StrAppend(&s, dim); } diff --git a/tensorflow/c/kernels/tensor_shape_utils.h b/tensorflow/c/kernels/tensor_shape_utils.h index a62f460998b..7b48a8939ae 100644 --- a/tensorflow/c/kernels/tensor_shape_utils.h +++ b/tensorflow/c/kernels/tensor_shape_utils.h @@ -27,9 +27,9 @@ namespace tensorflow { // The following are utils for the shape of a TF_Tensor type. // These functions may later be subsumed by the methods for a -// TF_TensorShape type +// TF_TensorShape type. -// Returns a string representation of the TF_Tensor +// Returns a string representation of the TF_Tensor shape. std::string ShapeDebugString(TF_Tensor* tensor); } // namespace tensorflow From ae85ce56b77da3534428415996f5832caa24a248 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 15 Jul 2020 22:29:01 +0000 Subject: [PATCH 0071/1017] update tests --- tensorflow/python/kernel_tests/map_ops_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 09df2cca134..e9355af27ba 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -105,6 +105,16 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): s = map_ops.tensor_map_size(m) self.assertAllEqual(s, 0) self.assertAllClose(e, v) + + def testTensorMapEraseFromEmptyMapFails(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + + with self.assertRaisesRegex(errors.InvalidArgumentError, + "Trying to erase non-existent item."): + m, e = map_ops.tensor_map_erase(m, k) + self.evaluate(e) def testTensorMapEraseMissingKeyFails(self): m = map_ops.empty_tensor_map() From 834873d460bbfea5c645371a2a8790f77a017966 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 15 Jul 2020 22:54:37 +0000 Subject: [PATCH 0072/1017] added HistogramSummary to C API and removed C++ Histogram Summary. All tests pass --- tensorflow/c/kernels/BUILD | 17 ++++-- tensorflow/core/kernels/BUILD | 6 +- tensorflow/core/kernels/summary_op.cc | 82 +++++++++++++-------------- tensorflow/core/ops/logging_ops.cc | 12 ++-- 4 files changed, 63 insertions(+), 54 deletions(-) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 45102ad6160..ee283d06169 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -54,11 +54,8 @@ tf_kernel_library( deps = [ "//tensorflow/c:kernels", "//tensorflow/c:ops", - "//tensorflow/c:tf_datatype", - "//tensorflow/c:tf_status", "//tensorflow/c:tf_tensor", "//tensorflow/core:framework", - "//tensorflow/core:lib", ], ) @@ -66,13 +63,21 @@ tf_gen_op_libs( op_lib_names = ["histogram"], deps = [ "//tensorflow/c:ops", - "//tensorflow/c:tf_datatype", - "//tensorflow/c:tf_status", - "//tensorflow/c:tf_tensor", "//tensorflow/core:lib", ], ) +tf_cc_test( + name = "histogram_summary_op_test", + srcs = ["histogram_summary_op_test.cc"], + deps = [ + ":histogram_op_lib", + ":histogram_summary_op", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib" + ], +) + # Changes to the Android srcs here should be replicated in # tensorflow/contrib/makefile/tf_op_files.txt. # diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 0dee2a48e76..415d44867f0 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -3887,7 +3887,11 @@ tf_kernel_library( tf_kernel_library( name = "summary_op", prefix = "summary_op", - deps = LOGGING_DEPS, + deps = LOGGING_DEPS + + [ + "//tensorflow/c/kernels:histogram_summary_op", + "//tensorflow/c/kernels:histogram_op_lib", + ], ) tf_kernel_library( diff --git a/tensorflow/core/kernels/summary_op.cc b/tensorflow/core/kernels/summary_op.cc index f4c91fc9ff1..b35759cb7bc 100644 --- a/tensorflow/core/kernels/summary_op.cc +++ b/tensorflow/core/kernels/summary_op.cc @@ -72,54 +72,54 @@ class SummaryScalarOp : public OpKernel { } }; -template -class SummaryHistoOp : public OpKernel { - public: - // SummaryHistoOp could be extended to take a list of custom bucket - // boundaries as an option. - explicit SummaryHistoOp(OpKernelConstruction* context) : OpKernel(context) {} +// template +// class SummaryHistoOp : public OpKernel { +// public: +// // SummaryHistoOp could be extended to take a list of custom bucket +// // boundaries as an option. +// explicit SummaryHistoOp(OpKernelConstruction* context) : OpKernel(context) {} - void Compute(OpKernelContext* c) override { - const Tensor& tags = c->input(0); - const Tensor& values = c->input(1); - const auto flat = values.flat(); - OP_REQUIRES(c, TensorShapeUtils::IsScalar(tags.shape()), - errors::InvalidArgument("tags must be scalar")); - // Build histogram of values in "values" tensor - histogram::Histogram histo; - for (int64 i = 0; i < flat.size(); i++) { - const double double_val = static_cast(flat(i)); - if (Eigen::numext::isnan(double_val)) { - c->SetStatus( - errors::InvalidArgument("Nan in summary histogram for: ", name())); - break; - } else if (Eigen::numext::isinf(double_val)) { - c->SetStatus(errors::InvalidArgument( - "Infinity in summary histogram for: ", name())); - break; - } - histo.Add(double_val); - } +// void Compute(OpKernelContext* c) override { +// const Tensor& tags = c->input(0); +// const Tensor& values = c->input(1); +// const auto flat = values.flat(); +// OP_REQUIRES(c, TensorShapeUtils::IsScalar(tags.shape()), +// errors::InvalidArgument("tags must be scalar")); +// // Build histogram of values in "values" tensor +// histogram::Histogram histo; +// for (int64 i = 0; i < flat.size(); i++) { +// const double double_val = static_cast(flat(i)); +// if (Eigen::numext::isnan(double_val)) { +// c->SetStatus( +// errors::InvalidArgument("Nan in summary histogram for: ", name())); +// break; +// } else if (Eigen::numext::isinf(double_val)) { +// c->SetStatus(errors::InvalidArgument( +// "Infinity in summary histogram for: ", name())); +// break; +// } +// histo.Add(double_val); +// } - Summary s; - Summary::Value* v = s.add_value(); - const tstring& tags0 = tags.scalar()(); - v->set_tag(tags0.data(), tags0.size()); - histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */); +// Summary s; +// Summary::Value* v = s.add_value(); +// const tstring& tags0 = tags.scalar()(); +// v->set_tag(tags0.data(), tags0.size()); +// histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */); - Tensor* summary_tensor = nullptr; - OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape({}), &summary_tensor)); - CHECK(SerializeToTString(s, &summary_tensor->scalar()())); - } -}; +// Tensor* summary_tensor = nullptr; +// OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape({}), &summary_tensor)); +// CHECK(SerializeToTString(s, &summary_tensor->scalar()())); +// } +// }; #define REGISTER(T) \ REGISTER_KERNEL_BUILDER( \ Name("ScalarSummary").Device(DEVICE_CPU).TypeConstraint("T"), \ - SummaryScalarOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("HistogramSummary").Device(DEVICE_CPU).TypeConstraint("T"), \ - SummaryHistoOp); + SummaryScalarOp); + // REGISTER_KERNEL_BUILDER( \ + // Name("HistogramSummary").Device(DEVICE_CPU).TypeConstraint("T"), \ + // SummaryHistoOp); TF_CALL_REAL_NUMBER_TYPES(REGISTER) #undef REGISTER diff --git a/tensorflow/core/ops/logging_ops.cc b/tensorflow/core/ops/logging_ops.cc index 6489074b546..e620da86c68 100644 --- a/tensorflow/core/ops/logging_ops.cc +++ b/tensorflow/core/ops/logging_ops.cc @@ -94,12 +94,12 @@ REGISTER_OP("ScalarSummary") .Attr("T: realnumbertype") .SetShapeFn(shape_inference::ScalarShape); -REGISTER_OP("HistogramSummary") - .Input("tag: string") - .Input("values: T") - .Output("summary: string") - .Attr("T: realnumbertype = DT_FLOAT") - .SetShapeFn(shape_inference::ScalarShape); +// REGISTER_OP("HistogramSummary") +// .Input("tag: string") +// .Input("values: T") +// .Output("summary: string") +// .Attr("T: realnumbertype = DT_FLOAT") +// .SetShapeFn(shape_inference::ScalarShape); REGISTER_OP("ImageSummary") .Input("tag: string") From 2e79e5c34953819027585d959d7a7b2b9aa14129 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 15 Jul 2020 23:47:40 +0000 Subject: [PATCH 0073/1017] initialized pointers for params struct and added CHECK for SerializeToTString --- tensorflow/c/kernels/summary_op.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 9ebda7188cb..1bd14eaf9c9 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -30,7 +30,9 @@ namespace { TF_Tensor* tags; TF_Tensor* values; TF_Status* status; - Params(TF_OpKernelContext* ctx) { + Params(TF_OpKernelContext* ctx) : tags(nullptr), + values(nullptr), + status(nullptr) { status = TF_NewStatus(); TF_GetInput(ctx, 0, &tags, status); if (TF_GetCode(status) == TF_OK) { @@ -99,7 +101,7 @@ static void ScalarSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { } tensorflow::tstring* output_tstring = reinterpret_cast( TF_TensorData(summary_tensor)); - SerializeToTString(s, output_tstring); + CHECK(SerializeToTString(s, output_tstring)); TF_DeleteTensor(summary_tensor); } From 1a290148dcc1996be4fe08dcecfe750a481c6bf4 Mon Sep 17 00:00:00 2001 From: "902449@58880@bigcat_chen@ASIC" Date: Thu, 16 Jul 2020 08:57:33 +0800 Subject: [PATCH 0074/1017] TFLM:replace Himax WE1 EVB micro speech example animation with external link --- .../micro/examples/micro_speech/README.md | 2 +- .../images/animation_on_himax_we1_evb.gif | Bin 1063997 -> 0 bytes 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 tensorflow/lite/micro/examples/micro_speech/images/animation_on_himax_we1_evb.gif diff --git a/tensorflow/lite/micro/examples/micro_speech/README.md b/tensorflow/lite/micro/examples/micro_speech/README.md index 0ee367bd854..5b291a4d6cf 100644 --- a/tensorflow/lite/micro/examples/micro_speech/README.md +++ b/tensorflow/lite/micro/examples/micro_speech/README.md @@ -660,7 +660,7 @@ Following the Steps to run micro speech example at HIMAX WE1 EVB platform. After these steps, press reset button on the HIMAX WE1 EVB, you will see application output in the serial terminal and lighting LED. -![Animation on Himax WE1 EVB](images/animation_on_himax_we1_evb.gif) +![Animation on Himax WE1 EVB](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide/images/tflm_example_micro_speech_int8_led.gif) ## Run on macOS diff --git a/tensorflow/lite/micro/examples/micro_speech/images/animation_on_himax_we1_evb.gif b/tensorflow/lite/micro/examples/micro_speech/images/animation_on_himax_we1_evb.gif deleted file mode 100644 index 5897c43d5a20c735f8f6fd086720fce083ffb30c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1063997 zcmZ?wbhEHblwb^D{La9@%frjV$tujtE5XYlBE%~#%quC*%*Q9h%O@$sCoIk<$|oeu zCn6~L!>tgNl7tgEG}s;sT0s-vr| zt*)VAq^fMJqhh41ZKDR{ zYh`U_VXkXvEUP5qZYJqtEb3~Zg8nS>S*KbV(sT?>+5YAWf$&Y=I3Eh9YnQB~|D`;iu zs%P$_Y8dHk?d)dj9%%1m;~Z+~JjAX9PHy9;^P(V>mBLk8W!Lh z6YLWa?B(te7UCNc6A~O15#|>dlp5hx5#x~@8k`axk{uhJ9vzV#8c>_ z?%L|ks?wg)oSFIYn|sV>PEkCz$m+*WiJ!mKfBY~%e_n59Yruwn|DDT%dYY>{Tgs<) zR8H+GS~aC`>++H#vwY62^*gsd@5<_wOXq#Qe)WBEGjHD!v$UL+?2?YsijLBnNimtt z?RCu)>zb>Zr&c%ib+@)o>uQ@lrK7!b`mCulru0o+IgFZ0x2~PNYwgTko91rcIBUnI2~%e-TQYCa=6MTOEm^j8)zaOY zmu}y(ef`=kGnZ{WvAO5)j@6I%b)G%C<@x2gn|7X-E&+X?L671|4;A*62z|6?N zz@Wpxz`y{?CJY?^7@l*~7`HDtXwD(56>~x$ahf^5yi<#WVfR!WG2MMTGnOts+NaoK=rrlDF*i#`Z zFE00;tK@04^_7Tf$Y!m;ptNki`EE;Ed>1|OiCph7%P-T5^>xS+&uJ54m7cElnH{nG z&5}dgxkR-?uC3wpnV~PBN_)rDmf zy*y`|tO|N{dFgWB*|MFJc0D-xFYW)lvh2^Vyr*jfzAt?nwD{?3t%d9MEUQ}N^*%ZD z{q$2pH(EH=Bt&yPCbl)UX|lEOJ6dcz%&X^P<}Sk&X?D|*)pOc}^aYM4$6X~l#gg2X zR4$p-sW9=#EKlXVWkqgNs#0z;=(d7lt{8ua<;(Yw>#IZda=9QiwCa?H7>Pp;1n<%zga z*dfc-BH^WT<3{oXUyYd#B_1~}ILI!Ds(SUt>;ntxv@VNY`zLm1R{33%DJ!!T zD_4~Uos1F*zEE{_ZqRYFrJZYo($ZCDCMBL+I`vwn(X)^g&E0EGn0C5r^{))G4r-_g zT$mf(p*YQrzuWZ0b8fL~i4$Zb_f=1DniqL7a<88|uatYYUX@gidS%MX{VF#rG7nfL zSxvsdvHpv+Z>U4LaM03Cw?dA~n1^1ubNOt{91+>Wdq4g=wf>P*QB>l=t@py>Z*6Gh z_4oL=QO&Jpk4R~Prln}$QO9oaVzakj=5NgoS-5svuFpia_{GZw*Cp|Ay=B_a6E^=W0bvzN|TrLxV3^Yv0`Q4V!~A^+zmzijxN<@QwWsH}UX zwoq=|(d+99Z~roUu;X3s-V@QKJ6VLIyfz+vbot;}i{Q_;d!`6_e2NH(3%|K!{zzj%E6;zJ5h0b@wHcS4w*y$1XW=%<=vY)yfl# zIdk_+Q&yOIs?vGY`7avB4s8_4w0UkQHpAiB5`EFvp$*-a{<67Se7Jd_@E>!@q(?i7 z6?VrYa_`^OUGc75bcar&dST?`|0Qk`7bMvXmwTU?r_iFu<=GKCJ#qT$r113>wM#O^ zW=dW4d9r@ujTf$axaTd4P_nVA>u z`RRuRb@5zGMO6;0zF!_BA3ycv@w7$eeN)rzY#o|-r+iwiuc zpUt$#PL(<;4}|j>&ZcNsw)xi|JHcVr5gIk2B|!Sh0mEmv4s!97cgNgu;?CVAahh$p z{i8L<)8;((mM&4-#US*7{c+IUS68O4GqfsD{J815q-Mr3)21h`k-MGTi;P$L?p1Nu zxS1^HeQ5Rqqio&7icgMSXqI1oM?ch|(K{?|*$(e{!Ca9MC#9DJFl+L#8Gc{FWVd#f zx@v|~Ofu`Oh~@-lsS6HlrYjhElLVNRZ%pMV|H~lwnUQFy8@25%xzRy98_|7 z@&UoV+eiKhn(yRXk~&o&MEwY7NPEcY#sNmCSHXG#x{ng`WH$v)jOPo`U)Pn zJX=5Q^~{|3gwLT9&35>4-MXB-Z}u5ZP8GgN->W-wKCXZ2uDn=T_8rzOiN_f@m%m^$-PPEa zHz$ef@V)me5(>FYCJju+_)wjxF|hZfV~Lud2f9hseJ$Q{*jh&zy> zk-crfbH$1Vf!z!n9z6TJbZ140#`nra`YV5gUaaLlWdEVDIjsEQwN}HKKFTLtbut`- zymgPLoSW_!KIc{F=R*B0|9VCKC5ae5`uoJf$Fi^d%9d$iX_w|NJfdQkd!e~~Un#3r zz`+fw3XFx~4Ga$aRTsn^8klYz;J<8f+;ZQ=-n==7C37Ay%#_~HuqyoSt^08eEI|UP zneVrs7M3WUr_Io$TjMA)=LL(zm9=?(1xdnl7qIR=_<=>>%XFr1b}f2GHn!`U9E^N- z@KV*g?*^(F>*}xldMav~eSlqcf7vP5 z;q0X=HrTxijkGxOFm94h;GuNhiVI=VI}%uBA2hI^%1&iwn9Ib%V0W?AuSSo(mPzD+ z(RA4cMs173FMa#}SuSy87xiGclfq`TL;T-_nt#h2SR(IzsyoiZ$YF7SLA=6AJZ6wef z?{8!~u~FRNn%5+^#5`5zTT7XOk1^U@FHSv{S~)HKT3L(hja(Lok~`OP6WL2Dm(|3& zm7EYS<5gfZW#D|qPr-%Q^mao0>;8ivIX4lQqfsucCyqY0!Zw z3bR^cdBme;6gIw_WD`nX^m2`fHZ@MAA^1941 zqp&U8n{Tnk9X5-5oSGV?)|&90A&{e$SAoIRqBLT9OX&2P@bJ>a?F=p&z0U5vd=r@R z6?na_xB9yCKEJ?w&Vj?gfg!~sExD}kQdw=Bcq{h@2Jr;u?CE7C9O(sW93|ga;~$jG z52*iP&h5J)T_J$Iy~6&Dh^x;c!<58|s*Cbn=|QDTCXNb?Pd`a$ePEI}5hO5Ol3zt| za-*OMN7K>_$=C&~$}78y1-dPhiv*YXXNrbf3xwy2{$tJ;WtMHIwz}T^^lI^|AGuEI zJ%`;}LbkVLXiRZE(GqgKC-g>-cSouB&z9hJMjHms`!`B_j`w?g z%B14{Lt$SQ+0N_og`0G96Dkg}#0uW5=zGqjnV>l9gyigxCa1R~elTL6R;V%cB5O6n ztk?^zKYp+VE?^BUVEM6x({TZ(*#QZ&1v<#R+Da&MT`#Af!zl=THaXVAL9e(~RioQvCe??mw4{lUTi zAUE+i!(~fG0fxM|Tx?<(M3 zvLV9kNCf9et=3Gr*h>k$(~SAc3Ir_^Z=^Qrl+J9rB+&WDXU=j?Ij*L~H(6sYuqFtw zC2VMqzYxdwHO1`0BHdSEw_L+_c+M`FI(xe~^Y-bp%{3VvG#Hf@^u=sT_3EfO^|R&X zx1KX6=bX=J%`J0^ zw@2@ps?y47xgqRR_!OA?9C+{Sy7PGPpX%=ovw90-EuXv_1d&GA_8mq6d1D~ z@Z6iR^yxo_c~>{^-v6-lO~cyz4!q|SnE5_TO>xT&^XUD3wD|Eg)>mR|F$P?3KJe|_ zz#9C)?sB0_Y-;%TDDJPz^ISi$?o8m_Yp^}g9Y;Rg)I?=T2nU=aAQCXK82B4_XQZF78fOmY0V#$$Hv zlNU?{3Y=GNm)tne8^?95dI6*40Y>`+Y&HdP1s#lj`_t z!1{qfHh{G`f&0n=-q2k~B<2R5O94QFE3=3du4;^%$jp4*grZMkP8rd(Y! z$MIym!;7o@53Y(gFq%2APfIv@qu|=9Zyd4@uDLUCUYWpqC4g&U0GCz(^YRZo(-rnT z*%9xS-rnTPoh8oNt}7PQU{|(OWQz++!UOI#3ia%JSvN1>zU&~<*v-;%&AD`A(V9(a zH5+$Kh`BgKRwocGmYUJ9idCpOTaEAYiYV6OFROMJ#4+rZLLzE@2k$u_ zFfeV{UVM#n{sFEv3LNnZJp3JE?imK`{lvO>_WF;<CtEn5NTUYSj{Cjze>glQ4!D}*p z8hIz`1}Y?FFeM&7#--WvVGrXI_mk4z&sW$+GcmXII$m8dvFW4ZxwYT zi`suluHI9oYs~8KaBNz_E~;s|O``MA$?*05b1iCbGCZ2FF28t!OyFFrt&>y^E!A*T zQtcFa7Ca@OQLdI(VClui82(Gj91|4mYTgJudc@i-UVLW0`TVDQepyD|+f&cXzE89< zYE#C_IiL0WaS?ZU@Q_0!O79N|( zlTdI>CVkqE1^nqN7P2vKJ(tzM#I_=5L9^+kCEgqJ_Sw9d8N@a-xl7`Y5ks@cDT~x2 z#d68ZWh{5^{KsbGIcFscGyAzN1}0X!7fqA>>==8eMIPul)OLN+;zMjI+Z39(qiPmD z71leTv8ajP;h`_La?9_v7Ai9$`gT5bnCjH^VnK9r=!+H2U#;FX-B@@{Ca+>4tJsCv zk9ogeVivQrxKq)5!}Q6AQl&1Ig{}+ZDiV(`jGec`X|m3$15Di78m|`JY{_VR)GXn) zsgX@$T0xprG*jdwLH*^O%txf2RH#WuPuq~#A0#vLAd{+$qpN7l5uV^tU(ms9P=h z;n1e4(D`gbQNlg8XUEt4dzdyWeL>(6MkbjE21Ztq8wy+oI~1-=j^cT$#HHLb;ZR%0 zq>4w}foZQ*c=UEKBsTE#9BiF!Yxj2xyJnSx6Q`!fRB6kAsWX(URIgQ>vwFSmH6w?7 z(~EbzrDF;j_gFr1c)vI9Z1v*n-zT*8M>M`Wt>j>kHq%#(t(Nib+(?!!P5ZQ#J=iAN zGOtx>iD*uRVw=>2juxrtCkK>7XYVsn=~TJX(R?Cp!;C}$jo&*GIHM!h9b0;2)kC&Z zy9-`2Fl(t~-kaHcc+SEGRkt_CrlreuA8us26_wC5;d0k0-t-M0H_gu7mm_I-)M=f@7NhK22yo?UTU7JE7rc#{*Uao>#(a9~`}M;6kg0 zjU$)Kiv`!S9(G4MFmi>iZn()bu{&3SDf^hG-<=uGGP@<8Tdb+_TfHK9ZbV_b^}U;p zd_@nM#P%q#?rD;#nG&%x$@HV`w}l?Ey9y?-Zd@c5e&?vp3e^Q&92s34g|51G4<{Dg zbYSayb=Y`bW2>r7bKO;qqw@bYa6B%3ELk+;pz?}~y+I`joGUEOc+dK%7viBLrF`lT z>w^zVGQS+;XxPAMT4BJz!_Xwv|4@RRC1KiTg9fJQg$*nf3Z1`7A4}^kjyD%6K3O#5 zuuRq;`Sbg>7+YSQB(>s&Bd5@YX;3UKFs}X3PD;{2`n9sZ2ajNZGWbPFo&FZDyX$#$)rAm24#!G zs>T%!Y*q{>igvK@^A((mno=dUeUIak`!C!W7F+EPxz?&T{m&%JT@!h8zT`+h{c$XO z|En2kG7lNK8X7KMF=vc-OJGp=;lye$*ydM}6tsJRtNMX}e~wW;YlW+xDsXTaG%yu7 za4(tRCO+wd=9L9(oaX`$3AHh=+98|HRWrNqew?bdyqI1`dvPOIU&XwtTg9FGvXnO7 zYiMocS;Cj_;i$e%_yxZfwS~7=F}X@qu^RL|5P3SJXilvNl- zJ0epp^)e@K{kl-<<^fl>o`SBOQx366bhsSxSjZ%LWU17|0}k><)4CXQ4sa|j;1t#g zXy7@L=n`@4Kb z{yk^kPu(!DiBtKve4ZAgomCta%e>OI{~Vp{KYeJ3!2lp{d&UrZ)~=5UBfT7;Q)i)=LQiDN2X{y*8+a_;X$#f{S`pn_*e}ReC zw|^>5bhMslwa1aqd)gf#RwefD0hZi332Up^LMB8hq-?yUQ=O0ftKj zEsSg*3T6bYc)(*Be2h)Ypn1yt!@S}L9E5`QUCmqYi0N;v!xj0AL`IE=g4-H`%tAEn zH=gsq&bzHNJox3+d9ok)52_vLlUu-~Wue5cWx*ZIc_5MBJUS=&_K~~#rJ5RBX1M2O z9uQdPz?CiFWMSs#TUe0u?#UO2ypR{t0#Q$s&;8rLeqZp=S|N7-leT>)mzhqeUh_aZ zH}v1mlD9XRpEpRygx&Zm-hQb`Y+XaEZq34*?5xV#^Bh==Pbp~VbufOY(!f{k z(<+!?(5!sqq3@CzjAF;adsS)<2nuSq&ib|C_`GBWzGo7aAE!D_H$Jd{bHM^{`8K=u z%o|It-jrxz*K=;*WKgIL@v`Q5rNC+b|9xum-8Kuw-J1@xl?3!0yRdbhvJPhj7k@)@ zxeHT@%jOqX8^tF!UosS^f5j9SA$$HMtG1M4tizNW zmz&Zax1`IxdS~N;h{gp9`=n+tN=#_ZJfNrZf??*5X8q0<*!uywdu6vAx-JMb*uVW7};72T3ddw#tX1rys==% zBbGbT8gUHG6AtX)|0lquFTy1e(DUs zQMH53Zc-!11jbz(T9rE*q%2x@IkbM;%;~wKL9(DNr-IG?N5d=&cFzVbE(cTgNemnZ z>{k?UOD?hYgObYt96U#FDl|W?!c-tW4EnlqgX_fx5rHHS*{8l*2`{9?>bMW8?=hwp3GU$C|l9UQPl8ijo9_LL?GP)|XAE}rbrNr*K;=F*uR$IviE(s%S78%7X^J2@TtR%-EuR zdQ%QJX8~imC42e_36%~;z8jNY`K;nJkeYmxDM5g(;zKLj?8*8I+Zdu)_!lrFa&K+DS0tgXdS;Cq-0jnT&orM%fm-|f>Hg{CJBWm zzYW(awp?u7bnyX~o7;DUBOa6)lb{822A& za&%zgJix$h(ImAY-1a5IiU|yY6LVrxB0>b-P#43oU!Z|?0%Rm_0?{* zXxPkc!OdQwz!S~DEYX(0!PfGqt*nCQ=RvldiWa^AvxEn+0yCKBFXCQ*OVVH>i+~CP zhe3k@16#t0NT!H}1-({mAL4?iTE#CClD1%psbE`S%XFmrR^(c@?K5vl&gzpob9K)V z$Ec5tXAe&m^VpP7Yjn^>*Q}tqJ~+D6<)V24^NSaZ0vb)e9qv0@-Oq1~4%b+yB)}~ps>T1LK1%CV$4avpb>;??W z7q~Fa-Os+u;O*31Um_V8XE5+MFkPA=TsA@eS7Dp`3MPpM{~A(KwAp7dFlV$zaI_gz zFgXjX%z2QMbApv$f`QNZUP47vy1{|}$_qk2G0$5obdQyd@AJI{U+=xW8#mE&zP933 zwi`^$Puo;D5;JnH%2nKY;1PdyhN9ftp+6)$(7mT6>4I5?HA{tr^COo%& z*ub-`@w7z*zd?hp1xv0+Pe4k0=8e|l``Th&G;$_1C~|}{ceI92VC7rf@OJLwfBs+J zM2Ea)<6vYpXmbA19($sR<=-jY9}Rmlr`(T-*JNP2-L{AGYUomd%tJG$%5pF%zBnyz zu*k6Z<%z_Xrwu1)83!^Oo-t+EG1Kkh>}5?+isAwcjk*sq9Y3=8MW`tiq)u$eD*72H zzch8(xu<7YJoqXYzvEReNe%Yo!;u}wi2 zR{b|#kl$di>J!`a4z}ou7Ap=Wfeh9Nj4Rx~JaFh>cnuj`tT?XgVT zNkc>7^snG;TotV6-WhN$U^sowHFitkuWAJj4h8{+=CBJ&Z|j;A&R>(7Ann_6ddD2M zXA%E+%O;EF9pO8uC{wnp@xzI!v4Ur&ZEJe`P2J2uK+{+t_vMpInNyk9@CNHLN=$F-&njWK=qiy(G`p2*S==U-@%GTo5BJj&DnC8 zBN~`*Y*0^NzgNo?{5&=;UqJan#?^@XS0i|(*YK4EGgZ9q(>QJ;en3>^wdkg&pH7_o zbkN4V(cz-szrJ?{`g8YiHZ2rrTKFJrBEt!G#)*17zDIrr${pP?(SYkn<*P5>oIZIm zCg|PZ`@pb&#XN3{FNz+GLK=*m3_L~Ck~S}S<{2)}pw963QIPNa`4*B2D;BNN|JatZ zW8IXCY_Sts6FOSM7#NunN?QIdQUAwa5b-T`LtEC3R>OT|v)frCIhf;DH6&DgT<)J9 zyC!M>vZ{!V*2EvBt?vw;w6?BnZw>$O`rchz-v^D$=N`MgFIOv~x43A#bj|ki$)7$5 z@IP9X@;_ARP|dqTZ$8ZoyS9)0X;$DX@e3`Y3h#OsGxl+xy|9N-a>s*nB8k@>D|!#_ zh`e`(=U<@chtz~`4QH=s{o~ug5byRyWW%ddwG46-lh_g>HoM7hS>bZvvi@HyrjC`} z=0CMm%)Hh!f6a?9$kC|MmTS|CXqHGQiJr2b$)nL~25aJn*Rea;vOlmc=Vyx)XiJn} zi`?)bQ6O}A`!DY28?@MdM=<ndRSjH&Bz#*<5cc$`iTAyf3@r{(q#~&6>ZHru!uraC7cbaMTnw&L8 zN4bQgO)_p&Y&_J~D=HIq#A9btYqx}c?1=>%lTuDiG|#=&qxg8a&rI!*6<;1CHeckI z?~#&y)Y!}|re+YaA|Sz|nNwIoV8K6wMhE5%iJQA}H!RR$WMX0ySs}<7zP0G;uF}c3 zbERWyyQ&5_h!k!i(Z~dbJ&OY{gmrVAsKDyvCmyyefNiEK6FF3NhH!?5@x_EqK5mcyQ zXkwRLv4Dw9!lodUQ8;DkR7Mt`4F`I7Z|rF3{g?cdLo9-iL&1Sj>q{fI=}n25*JCSG zIVBSAa=6_vxISlbui1GYqvPy`A%&ghlS9M0O&#T& z4tKKrB_7OR6m1ENG013Q;L#}+yZBHzF++`;!^_}0bFqWL@*~9|8Vl!^-OhdPT<+MR zevj|z&I^YexHUxro%kX@y4B_+4n6| z5%M=asnQ%ETD8F`$VEhuRV1N>k!@?dOJGZfe8GnU0(bd7Hn5llDJb_R);@g{Zr0VB z+-LSngMmHU<`IwSt&+)SV=5mWRY{mE$aco`ht$lo|4gJ12#$ zU2T?#!GQ+GS&W}0vkn``9Lu~b722q?fhBo4WBsddF1+p%iZOYcY&Np;ck4;X3QSn_I9IDh=VC62Bv*6X2l`E{atXd-ynmFGzFJ|pl$-EgGT?CA# zn7rYNKRWI;peeN36H*mYzeLA~&p#9(*Ws zU;z`O^Mc!XyI;>bnit03GBu2ui9x}UztZE8kLeG!%IonyzlaZA#_S*JF(Y-|?s%3wMq z+;Heq3$MGuuFZ?jKU~97GBxUy?5_ufS}oibJWIAXa;*G*Z>8AA#-lD@LPJ`me}*=? zimqEGXr__K?ylTr>7-v+v#X((<%NlSOHM4GYw{;2V+Pl)Wj3k;VoUx#P~coS zzxoD)XK3USrKrtP*S3^xef>p&LvKT)fZwLo{&Hv8#9YLfD~%>iv^mjayU>yC^s`2O z0Y^a4-mKUiN6ETW_{ahQS>PTZ1VUEGcJWJ zKUO?Xe2GH{d*Q6hQk&Y$?<`l+t3CYRis=AD)IzTC1x+GN3=2eCioy#g&-TywB)sHm z%D={=UY*=8)R-kaIyB4_oJ6b|PYZ22BpmisRX@{^TchIy?>U2I#!7;WtPTre^%f;Y zW*s?`9VM)jo_T+bx(3V)C z6R@dcGKbN(uHJ`mnV9;6+d3IiB(t!zQbzZILRLFS5d3Xg6$2I|$_n}t;A8X6I zZ+CmJYVA#7~r%>XBkGf^#_U_h5 zH|hT`-hA+BwuwCP%#Tq)Is1{wtVb=3ETR$&M;*>cTU)XmklS>MHCLG?OvjoH6u0QpE_a`6wpjM_wEjjzF0(?5UOlGao z4L9)=*u=G@iD#7qo4f{-M26rFflW3^CFh5_ahAU=TMrz+hy+9;z{My?CGOq!&rOQ9YUB zEfq}$9Lh--1oG#{{c~jO629Ga;{;Q*i17gf4rVr!%zm-OUm5wmqBlj@B(jUHT~WM9 z`NWNR?@!zn>=$s+njp4>p)O$Bk}DBit9nx$>ltm#i-C_qM7Tpp;Ejp8 zxun3RBGv8lT?(AE8ytBO1lH(#m`+y?iPhVcz^rU=hcj-1vHs407KbB@7ne3PGRV}1 zoi@BYQ+mD$*FT(cHQIQ2M(_NejVfbF%~{;Ux`w&mUuAIsZ+0!bF3) z^Saa=c8WXd&mq}23CHbocw$u~7IFrxXw|b;yl)ZVz_+Twv0nE$ z<9{y)CSi{2%yITCoIeVpogXx?SS7jZ(}xWSP(i=kOXLXk7Vqev{@-ss@y!Q#B&)M|zct|4dLB981_JA0Sr zL7@Og{sk@@8W_A{&bRS!-_h~#o)UEDA4emv0(XGUgjS8Cv+o9RTs@@U!)DWQSoF$7 zUZ2L(5{@DrPP#1@bZQ)h=Nx1+IjHodfqM>zPsjz|D=xfm82WA5g?t#LYg$z1ddXgJ zloL24FK|df6*L69sEBh^#7RC@1E*4 z=`92AjN?j69k`D0OY0~|?>M;QEYiNwAVQ5emSJam?&W8sPLds z`beWW8KKFlTTyl!ycFQA7 zvrg@BJ>s^(kwq@tcVYBBT}M$JM_*q?{w*wmIga8D2NfF{4}Cqg*Ym{11rIiSNSd|x zh<8dykjp`hhIw;#HX9u}tf5Cg`vL??n53%34#JQsy~n+At|vILZBBl-qDHtxri<R+6+a~Qt09L#+%$26l^uYr+cg(q{%4FL!KqZ>{#wKQ}rWb-sT*E!ei6i=h%UJrRb zciksVf(N8CRr?EWFbHk9wwYB()$RCyM#oaVv*iu$lP0$3{Y=rAq~zXndx~emnwZ`o zGu5iFMiG^RR|N#T0-ov|Q4hPc$b-YR{O=pEDf+nq+7xMpH4k@u5l4og>d2>)` z%|Xd82P7F7uYJ7TdBlPD1jD18)66yutQt3%85kI1`npOR^`0C`TiT-*bI{(V+0x_? zS40Y<1VitcS*~@hJq`@aXVi{Xu+DyPa(y5Jd%`W2E6ddW9JbGBE;eG!BsYBYz1)TrUQxeb z4x_qHMyM#`;k6q4E~y7f8}7y&V9jV?k8xnkaA4$NIQ3S1r_Zd7uU7oq84z_Ml~{^7?^qv95rY-n&CS2 zmB98E8P<+DJvt598BNM}4w-y$;1v-N$Y7dvYeA7idN!lbxf$onGEQyHRr3KE1O#!X&ui z;pB!lHjl-MB-_T1?SxV0lmlX1st)5yy#H*{IvI9R$c^DmHDp>SiCOv9Zxp57g+ zZy7akbTDvv9C#b)z&Yi>-+kK56C611I0%#oP1xEfw}DY$55otZJOQ6Wx(}R;Z#gVW z;qw0X&0)5k^qe+>tqae)H;V91Iqpy(ts!;93W DNr!m`oU0o-kOt92V4Q zV7t?(?BO6~et!QlsIH*t^%A2UR_sjXe$XsoJ5!D49Fkt4<9*{Sdx_(@t^n-~hja=U1tlDLRT@v{G)ioa;;3y} zKUIch%GJj^J{s@rUN-ekljQ1UlG(pjg-VBKPQBG(WcW-df`MO!*>uA}9#;p)?W^zP zv9g@$5HdMz^Qg(h!r8>b*~I0r=A8zP9rvc&HE?`6Xko&v=HsMh!Q`LpWD>)~-QvKZ z@PXw9gW{7zCKk$>TQ{jaSXaD3^NAMwxs;I5$4?XsPgQK0dwXibj=SvMfsJlw3X0Y` z^3E=0_~hjdMvHNQB_vc1P9a&UfP=v-Dqi4|GGCG#ZCJLyL}=FIijY;`~(TPjnzq+#Ya z-leQhXSnu-eBqhN#+Y!xc3vavoJNy5hm})gQ7;lSy0^wdMg#Ar3e2aQS$ zPAWFeMt>T3UNrFiX?XXqLCNE=(hZkWRR;u~G^#Cbuy<@=GGPe%{95!*mcR+663gW! z_6z$jY6|@c{4W|=mX%aFgWZ6;cyeo=+Z|bVo>xZm-xX%QnWpO~sUdD+rF@|BfY_UZ zvD{4xXAXv6I>0A$P-@R(A(tl0e|F9$E1KlLG#X1ZDO)g$Uf}gU) zD^BxV8Ru6zs$XF;h)^~-n%2`%CiOnNui#F^6z(SsmV9f?p6WF4m@+UqI7CS}Y-lyBf;;8D8mH?Wm zb~0u-BvSG~^ag|U3QI=q2Bsrd*#($16AnweBn97Qm|gWjIB4sQqc20lZ?okq-TrH& zv8;h(!<4{DrJB#)A}o3y62awtvT?n?trMwAswGXiZV<>TUvZ? z9N<6V7;*Ey(2C;3svU|fO}Zx-H*v)O^FH8^@;Ha%>Spa7PWnHZ^shLn7cfawF!4kw zYlpm;xu%0z#UavHDyDJH_U8iE)_rW4`~Jbc(A^ujCziMNYcQLBVdM#6GM&L>I>m9@ z?-Ng-RGLVnB?@)UNItmcM5EF_WjPiml?~Qy#@~z?nlv9YvD!P!g&a)U?oju&lJN+G z;*3Kk5eMa7uIt^p^7{vd?9SN56V8WrJ5J=})z4qVEumo0@9@9wYrwykt~^;UuWdN% zY#h94E9*5Q(VlBeoxOr*8d)bZ$-X(je&vY&mjhB7%qj+s>(m>~t(;6*nDiW+6fBr! z&SdZ&Ilyzqp`*b;@PgB(%}yMe)jYqP^cl{r{Pxd@yl1sUc~XEN0}#N**)KZ{w#UuQci|*d+ge$^67*%RL8$<}{fcXs~y6h-h$7OQ;dM(!gFG z#H&;A?%luZ4|{Hy{yNG1pzKk@lIBGmEqd+~zSgQ-k8-a+m;FZM!*QVqg?g2P#$`fx zIO5rRa!mNT{qAV(KJBRB(xf=SNj|1lrol<8W1sY$Q*Z5-D=m6<_-D|~?@am=3ez|5 z)vjQ2D0vi=azcQKpG8CC!h(Y=ZNf5s3I_@pox6n8bx!Q~SIEfND`hLBu;F52bC;N; ziid-e%hIg2Ne?c4I}@`-nwd+dWq||ZWRENJP8L4#xuxlFenaMEw-?PTqw{`Qbi4UR zYKAoD2~Kk9oxL+Nzt5xUnG<6R2QNo~LdvOIQ>EOpzHC^id7@v=w&Kpy!Y6vd3bqXl z6Pa8*6An8z{9tIZ@OUiiRCZ^_$3?8%B1$Sd5+A4aPCb`x5xy|!sd47T>}efwn`U)Q z()6C95~$?LCA!ljXUBq#v)y>^??^6LpqR|YZ@I$Dwd>3BnVUj(^ywUIeRXA$lu?q* z#K&r!!VbP&9gmKlnyUTj>BZ^kM^{~5uB0F*Q8Hn3A~VOgkIp;tC))oz-1+~%v$sR1 zn8t_3b|0gaDjWh2P6}qJ%{(j_ykbR*jE##@f~t^###71kDHFXUGj$YGmd)hYcu*vo zqtHcA?SscN5vwN#trI-w>|~vu{$j$ziRo1t42&FV6$hGqd@e5Imd?4bdgJa#r(Sm` zOxUrSU3Hy`kN07w&kGHYH&y6tIvL?{&`0pbhc~^`qCxx7jscZ=o9inUu+1H7BI z$u(_AWVO!}c*N!q-tq9fMAn5wHiU z5_S`(ZH}xA3zv!L%vpGtQGS-=5ytKcEysz*2@{leT1;8V(ig0@amh@_kO@boStU$h zY&K?SY-U&KS*6rpE4xsc&0`VcA)UVx6AtuyE<35RPA&I=BFCOjr(Wyro4~N>Q5XB; zpk#TAXBF)-D#Z(p#N0d_TTC}3B=?@b7jxJ|D@IP&RBPIdre{`($xfWA6IS?k$Whi9nH=ecAC0#OO#bQ2AxlA`fi%%~W zOTJQKJR;;?HsK*7-;@MrR`*4LhZz69`TXEm49CiaC;fi5NUVRob_&ymgIvx^yoZk% z_-yR!oW|2Ge#$42h0`W`zw!AMA9}W2Tov&sZB}kuZr9xJ@r!y?CK$9%z39f}RlqEB zMu6R1=L2tKMYGHU26juCM7E;``$RY%_ShbA(Cd=+VT~<&wq0$$X}L&p zs>M;wNe_EW4=7shpX|t`H-SlVgCc8vft#elhCPa&3;5a(997Ip*rgGvsXuv(Py$2n z45N%S-i?w{$MY046n8wRQ8A939hN3_r9;inB>CytH|h4*54dq|aAXo%@uAJ`Pa=OK z#}OqlM$Q12MIuLiShXf}c7?bs;y;==N$}eT9_N|{zTyc@yjBO6aG50vF4b6Qd`^Hp zl!b}S^#_xN#Y5J>jz`?@868>m3K)4j5?Hl&G(4Koz{sehRZ=El;Z@K0i+z0(^sHklP+ zE<7@h>{1ulbT%>?NVu5%d$OPsnTPLC)=Z%#3UzwXl2QJ~>XujEq*+fs+ga|` zs>~w5-CS_Y?8hsf*ab<)cJ9#D3!2CkwB(W4`3nUL4XqDso#2t7utc!a^+Utx~iAa2Cew*-t*<#8=4yy%5Dq0&`yzd+mGn~pOw}i2Mlg9xaz7vO3<{b=S zai7F)=kVd{qe-?t?|hOoISz#h2PeuuN$`1LJ(=Udlf)TEJ8x~=BK$8)b2h`Vv~=$s zOLKlDh*<4lBxIURGvWUY~Z{Nq>%m`=cnIF#1Phe=4 z*wM&8;ld$yzk(LwIS)4YoH&pgsMyY}(5=WSal}L^u_N=yLVooEhqW9AES(2_w%47$ zebRrQZubfqhVL`|d6XXZ%-DU9k9|Xkc}Nh0nDPN8V4Tog{V9fx}w;8PujVrR9-3c z39H0~1445)cBD^XVmn*F@?1cHk-@@&^q}@V#oueNKF5khd zDCa12XEp1IbwV}l{4)-)L>&;+OKiF3z!!ByP{xt%mxI(A)&njNR^;)D&soU!;}BaJ z!zz~zEA>{cYIsn+58Tq61(zdfHO^L|% zGjCLhdacCO6%lkmaM}X?oQ3>0Pxux&3YI-p32M-o)}X#+y`sWD-ja8Roy8bMmpFcz zbx`aB1K)=Of<6l+vmWrxQWSdEaOuWtzKm5OI*GjR6iVe11@0*b&0`ecD?GC%K`5u^ zy3Ns*TAJ^ktuvT$aaEu0YKi34Iy&quM_FSUSUsK@UD8U6>XeRMX?oA=W7UN8RZphP zVBl29=RBguw_pPIE(V?h4gwa8JQj_@R*oVE7I5}4i#|KRv1I}0q6hNd9k|z=GKF#lU1vWSVBqcOJaK$wCP zlN%F59fzY>3zOU;C9OqIv>q|2JQP}Wh+~F>)Q;EeF?mcZ4h#kd#M-X|p$e-|0Q+pw6Ndu?AL7_)Pb7`0c@cw)2^Mm_2#*s zPmj#hWnLxAyuzIMloI(DHE>5YibW{4Wi^UbEflL*DCTyMGxVV3j6<0>c)7ndSl@fV zdt(7#lft(X0jw48*`gL0RrT{^Gq6o^;Cmz>n(*LpR5sI*H7s`&BquS5awQ7AXkeCb zV&Xf};gI)tl4_(%g23zK=#tZlJjWGd6!`x*R2n^aapnMvP7{L+1GCWsk$DGsC7Axb zkZ5A$JD{j^falPr7zRhdMW2KMas@h8N={M~@=H(`abUl-fd8EX-<8(?`}`Qz9uhQI zp!4N09zmSgGQcG$YXaTan?nAXaNPjv5D}ya!VM^;n};en@NMN)T~d z&+C?*W>$7grZOP2;Q@P?4(FW%0(%s=4jeG9ZWOOu=uxTYnYmE7OM%Pdz2Ji+&U1zQ zY>5I792%xHaAi2K?g(MEU|@?_z+$j~WzQGIXddoa4x&>QN=7?v%Y&E_O9cymZH$4#L$&~7Y}jxaSFVVOYD+k6!K#f znzc|;z;p4rBrd*(PLGqE%|%u%Qc03~bI{IYwaXO4f1g~Go2SG*U_F;|%SFd@jj5ZU z@VXtn?DZR>H?qi1JmCI6L2wZRm(6GK3tz>9R6{Egg=cLMSaDF~UIN#h1ze97lpjc9 zm2+o}IpA-Qz?#;;EOFK6*-B=M16+BErvey7T-Ni4t!9xr;N#O!m~!C9j1&%=hP+En zc@_k^LqGNPI2jq4}4w;>PicA zN)&`-7V>{s!1rJQ?};$Z4+|uJF)%wnU|jMbJfG3|rBKq8i3hcg={G32h$kPCa+dNH zaSat=3w$v3mAl`*kES_pAGevhpZpm2;}J)n0_Qme0f&d&XTFM892CECQ1(uvtd(Nh zj489{C`vMHn4$6zG+AP!@NJe$#u5iMD}@HD1_ql036fixdk%?_ZC_90H|BIFsL!x>`s{ITzMllA)4~Iin zGV|YQNT0EQ{mcTM2L~i%5*;3vs23d&Jg30wqrjTbz^rqCMTdcnDa7gRdUn2rr~59l zH9a`+XkA|aRPQ;41`~2KW|^tuf)-ygC6OyWG` zd3s-iSXQFAg}B^ox|R1AX1pZ$MTzL;=ULA1JdnkD8poo#9 zw2vZpiOR(L3;FUKr8g*@l>7Hk_Q-YVBHoiZ4en zfxW+hz30Gz9tX}14My7;xY8DiWgV2AS0Uzdkkg`3ERB(CkpkbH_q-Dx@UA+*mX*M2 zb$~T#+TntQe9s&xT> zuX7M=s8qjU#I)zB<0i(Fml$PlEtD)-Af)AZ=FB3-WtM%m1$xSyvxR#O8k{t!J1}`m zm*JGRt36J##wV~GF!auzlJ-c;Y}o?#`W@`yIY*^(j@q2(c(;^mlY{7@2i#E$#iAYx zn>BLpQ<%F$p>5Fu&R+^rvkvkvQ{eu!fUhCJJgk9P<`=8Q16HN4UyZIY$1$V@CwR*~ z;P}<>m}jBT97fTYXz_asWsfMz%ySgBIUqLY;Xl9m3;E;}wY7Hf89fw`a}+K}BjT2KM6bfIbY+mkQ zw#jCl+i%0EX-6*z7G33?7rusTOM}QZ1umqcz+H+?%6-80GMv(wUai@O|1-3j8EK?LOQWSpGaNy-_ zwrvl%>=f9xIV_&Y#-g#B*(8zGA%P=6HhKQmXWPMicGjU){N#OmYz$cR^aDnlb z;~&YggfPzC`Bo1E)EY&2bSABvdELsDrG7!uuAK(oE}ax!eD5TCN{|TqvOO%DLJFfF zZ@wP6rP;zg{7m8gJqMXIxVaW`&ub8_O0>Q6M>t}knB8Qt1rGAwjN;!ISgabnQ>Ptx z{c_>!mGSacY!j@XOpHFqmB9I=;jTuw=#Pe#?@NS!?VjgwfH`m9w|@z;2g>g;&v;Um z%fK>2<(lcNO(wfM|FLjtoW2};U|;s$h3sXl)8-yld&v83fr%m`n@76GJ!3`;5@>Hoth+a(SqmWsO4?K)L%YR9DeLZvSXv~d!mX8}XB3ot( zsI9Ru(Y!uI$|mc|jf=_b{Bll}Ukq+BpX66?m}0^j#MU9P(xJv+>nr8#Ytoi^4>B5= z*}H|jP6Y)jEzP{S)oX%DY4?)j6O|n%7<5`sKQ5?d-}k~-X}Q0qoy&qVuby^IUwyQr z@?uxe6CcIsCYv1rjh1caMYUsY%y4dHpDJgVbK$`ON7i=5Ih+DJ0u-Ftl_&m_>HOH( z%==GTJE@{$BRh+ru8TuP=BI_-(|FyseN^~i!2vd5A-@iWM$SGHW)Ia<4x2dS+hvqInm0dC zYL%Y&B7srv>=L1qO7#ym^_pIOk;H5MJR-gV%(V#fPTKm#$y;}E!KNdUZFh<~cn!B4kmPl{n$Ro~a=?(4clXW>OuX6) zz8w;fu?S%J$H*^|(8%UyVZ_0&xyGP{QI5y3ge zDs(Li4_MwRXqK|^D0@l9$D1;4F&o*s{5puZC^m@)^UaRvD z79Ns4yF}q23tP3r3EdqR44wTQZwQ14xO`M7Ilc#B8k53xv@Nk>In&nU7g7S~K*T_2eucvv*-hQU%E ztptTe4%Isf%{*RHoaP(vi0Kw!W>P%R;CRn1VBwC78p%N#yJMDc@0Je{

JsdFb<@ zals`&^Bpxmcq~?YXlRr@yMe)xMaaZ}MbYwYqN%@qr0tZzN3VB04wg2)x^aR_R;kJ) zJ)KS_u7b69uJskDmWK12X}gLGMZH?OS@}u=o6YKjLa!7!(`62F#B&_ga=FN&B)}Nz zYayp{ZUehjW)kbnhYGw|4ga>v1vIi5YACW>Jy^rHr-7NDgGuyMSOZfG1GB*kW|e!} zn0R;|GRk?(W`9x8vD^?+YL>FeNpiLH z;{PS2vV4jo=S$^Fpw)0;^nw zBY(RAV};c#M$sRJD`4R+-(aab63rGZ^h|{^<>|Zu-?gD1fiVKaBQ!X}fbSy~Vf6;Wv?x%3z zHIJl6&Rx+)Vbi{_a%#WsPMvh1iAiQbL#pS+z|Nny?7J0HHR?Fy{o`s_WY46sS$Q;x zs7+*LHd5qUYT!~@mdvax(j*dL;Hve)z*q9iKL@VT8O#|!w@5o)>}Z*x_)b9Tfd+rX z7Up7(7D2lO=ehTMn=cd5XlL1RtFz4HZ1%hrRae%?P2>x27oU27JILUo$O^afCITxt z+8o>@4=j*u3}Drp_mGWUMx~DPv67qd>Hia2xoJr3AJ0t9c z*Y))cKUj9LICbi_CB^rh*U-H;GcIdMb58nfc8?v06u&jFSnNpTEZ)?|yDgwyFXte? zOPh-p+d^iZDQ+!64}5v6oK4T&V3_+m-0n7E4FaNg6PB^TvkB3>*;s9gUrHu5T zW=Xv+U(?)Y8tzx`1YYnJ*{S1@$93iNosEBg-`Bs<$WxKf<>2BJQ}7_m=f1|hRs{wo ztq+XCEPLI{B3v#znH~}p31oBAbYT3G$ZQ<6mbWk?StvW;Ab&zHW2E>3zR5d|Dkq6} zZQAuCE9TIQpHl+Z_^N$8HvHMparDu&_7?e&=w{W0nUJqn+h9b?E(n2?HPG`0dXhyVrV8a&3Cy>y30qyrEm7b!C}3OJz`Z$vV?rRyp+J`j0_+hV*jG4kBqTG3Fj(3O zq)rVt)e>VCdG0UMkjmE(V?2eWxxsRFtF6$4sP;{MS`5rG4`k&Z1aPYcs$X#y+UghG zB4t?MlvbwkX<79Ch5kpL`s+JbePGv^$bLOon74qz=qkhgr}F<87%~MHFg7M| zyg9(DP{g~-fcM82_FMtRW8uQW3pm}s^XM(&4gShktHAv7c#Tj1!-sP2i3W{=VqS`m zlPsLYk;U*_$qX6Hkjcmme zxMT}hydSW}ZQz`vz_s}RtG|Lw@kN$I4-V@BryzzXvj>vvB$$7xvFcec8_zIXQJBiA zz$o3yX!?PpKS3&fOLg&OX)OWfS*hI#Y0L%}7*$2&62hEbX2>mF=>L9&--CzJJ0|vV zZH-(R7<*lnIi!FqqQGXy2lgmd#uL+PSOWz%Ue>*;tWdLndxrqmDA1Wy;~5z+Cr$tza{gRh;I2e7yl zu;fkPs#jpvPte$-$o%@sj6I2xCQJ*}mq>|+GB70sW-VqibKq!gm?k}4DkOwCxIxR! zfyG;3#@0*br!TTpKVURFU@LFH8Tx^-+Q72TfZccjYuyJ%(+1Y1fvmrdRB);@r(Tl` zTAXh4fFnk31tZoNQn6QkY;DgB|hS|joTq`HA8aJ?4Y-o=WWil&d z{bzDv>dXn;=NIt4e86Flz?r*YR>BpA5ChI_4qW{+N}UuVL_agzFsWSk6b)$<{r{Y~ zNL0+cfT^sN=}^nO@Zj8+CxxvSsJ}bO{NbhW%w5u^4J@GnT)qrb7eCx*Yyf6P$ufMZ1i?~)7bOAL5#PT=ynz+RicXcB0v&A@JJz&qKY^!x*!%O93a zYhYI`_%}Polr?lZ*UVSKb_(SWA2X*P74bIAv1yttw^($einoH;yn`*9m|EuD6P~-! zMZ~7C$jE`YUzOR)p>j@ed{F>PoCEh_E`ED%{zV%&w0MmN^ThWH&6BanjLo0(%Yvdnp4)Ljse{hN1}*n4HCzO8#7TvuGCk zLxx-fj`{?yW(W4V2OJXuxQZIsyBM+r1eg>V7R+#kUGs=FU8W(HTTv{!#q}_!mFa@7noQylAiC}eO7v|{_NcoI;Mmi zjx>#CURvDUCcxpRFx8fu|J{P(_yeq)9k~4+Sn?RSmL{;yYs%Mia0ikiVy7nqc{t>7_}rB9TRrX{N;G_m~hplZ1Kf& zGfh`G97F@NdzN75py~coh zjRIHP2i8ZRV`5om8L*m7W#zdgy)!&5m2F#a@!mge=_?(Xg&i397^+uoWUpi32y0-k zY+#=$5LMkRZSX)?^25Qqrx?FjvT6sgdu(8eJj__>z%Fxutx|v`e*uT)0p*20w(bQ=86jj1qB+)M;L<^Fd1Dq$-jXu=K!OZ18eCiHsJ?M zUI9$|&4u+YF!4TMl2Bk+k-CukIzygqEPq34-!lfL2@F~XmKX)tavBuhWODDnEuVUe zfk}YTl7ZP$fm!!~rS1Y|wF69Q3<`_0Z~CucNqfNNvw)H70CUBIvl0jDj0G9BFDRW) zU|kf-Eb%&NZ<@%#(;_|wqV|Wpa{sm3-F=xDXLw8}VC99?>Te_y)wdseC%M@u+A2?n z+3>@$W3kMh6IfdVSo{^ZwjSV`pKy0xgW8r4tkDAOZ40=n4O|TuB((&k&NvzAQ;;9Z zW~JSbD)!XhAb@>Z0SoH`hNejxZ?`y$2}G$~U}Q?@&MWy?E&}nkV_7V&&~wtw+l4dE;WA0pt^wZ zhbZI1w3Gf^3dYClUN2{8n7}i20#_jebE5*UpknqLbw=HQ((?-Jb66X#1sZKGWZQpW zjuE>3d~q0uuITUd$q93q6D~4sEn)JwIq%;bQLnd}~gaSRp6cqX7S>?K4DvK!10AwT13H|cLP&M zfK@~nyV?T{mJ7Pv=`~#I9xS$dWw@1r{ehK>{+A-#jJ^pBoB^-c6&@7xa$IC!iBT2c zE|{fmclJ!X0N(-z`wKi)Pk9!Uv0a+LV_*<`DerNY0I#|L`zZkqA1U4|57>+i8M&u1 z^*eBF4`8(`EV5eg#AYG0?>Xg$Q(kO;nNuhHus0Cx{QoZLz`Hm1Iz5_Hl6&4=uil%; z_e?7GnbGQt>Fb{5&SSpFAy=lv8fw5%x05wifvd#tdDu%GceV3bfuGk(CfB7ked#i-Q}HG%>-@2=yRSTK|6LEX~48omut5!W7a zZ}_`X@5|zKuQuOf;GM`5`&3t0pm>QMW6plI<8O4+cQBkV7xZ)LMgMbc+OeXhmX9NYI&E`s+B7j&05te znKmJ@O(KWkAe;0O373h={|+pj;Oth()S{mA!=c;wR|kt|jKV=BHvW(u*&Fyi{Nu{m zAiC&{H>)Jm=0J!p1A z18dXc;`0xZHv0x#5>Z&!n-QY%mf=~B@(aGK4;2bcoI(u`8lKNe{<f%RR-v!0+SiAjPnivy1|3b`~muyFDzKAmROvno+#QkFsI z0r{vZ%|mn4ax~R4q}K!-nrm<+tY_hjM+;qqf_cpENTg^?cNI&%aEeWfp);k|US;Lt zzL+f=4mR^Ixv;3;ORjN=NXVlViH7RI2fCS?f3BLep*8$PBd^1U6$`t9m0mJ;S`-{) zZ9DRh@2~?CyIjO!AvvRl#QyZKf(C}0r>uSbpGDleY&I=&-dVHz?{Xa4GJ+Tmq&=0Y zIKasLpyL@IJA(u-6Wgq|b3xt|f6U8TUO!Px3oz{DD{fK`4Azgykc2FE*0;w1rM zy#I=YQhzFNh-RI9$Yj2dEjsvVl0ZBsY$B#(t zNMJRJILNic!c}t1f+BeXanVNtovJL4#ELtbwHykY#EvXr_MCALVSoH!LS(PV%M)w%lWEvPbJ_NG5 zZ*WnN`PlAYa(L1|28FXMYl6L|33geo+H4|GBKyzVi8c7!x0i7n5u9>wYh=ue#e0zVGc$XwFla)3JgqFbr`SKJk|CVdL;I5 zHlxS4Wz6y%FY@gJoHm#Ja8})8JdeLAOrW#EzvfnBXVA%mLf?0uSIv5-`LwB#dr^d* zPT0k+xyIMIu54wqoN!LS)2dm^E3wI@?f~1_6D=YffnBx&PTb)u8hN!E8ch=$m_;=j zxOxIu`9&Q1WmX(ko>0)pQ{uqvsgbOr^peLi<`R>jLuO{e0}d6Phhpa%m>Fd@Fj&o9 z9%#J!X-IDHRkx@5jBEZirCvxlY!oDPoQZXV1FP18M&1|&b0g&ijO;t+BxhY#;9QWs zIx2NVSfDP;yhXQ`MQm^o%qV8z`Vi{qqQKzLW6bA!^I-k2ZRT<&-kXdP)BQ9;*woIX zI5{0rmw8jreqQYnU-^WtCrrQ2*XtE_tzN1mYPvk2RV#=!ROhMm(RY}* zO+PTER34hF{Ne$-d%!&3C07qC*k!bTG&wAJ=0e-~zOWPJ4lGJDUYx(o^yLKm)j5vw zx1VTniLi0?GA1S@+;&uW$n2@%q8Dq`>&pg2NTWj01oKsc` z&o-<-7Up~S->Y!m1RRdhnm1))AHr*%zDaVw(8*-C5-xfV8RR*u>=8* z^d*OwUWb~neeq_Pah>PVGL}f)yT?yOt#dQG(0RZi=;fR%0S6cq6qw{)4(99SJmQY= z_#iIC^NgRTA#~F}-j1WkWL7=A6?|xATB0e(8vzf6H`kX0%c?YTJ4rmTw2f~H~S#3^S5nEfBZOI10HQz{&o)cKdb9Z;ybEOZCLv#M>hYdn8BKl z)2t5(z7Sv+s9C@fAi%1dv5-6J%pq=921cP92An}6Ne82D9Gac0z}&lRk-+r(&6))Z zdp+kQ%N4IE+MKES>9k4$Y9bFV2yk5uQi@y?}g7zQgc{> z4O*QaH0#}9*8RZ{8>G$uq1kptlg3Q4oTEgKqQd6$)Fu1XRf$fDblR*Pl2UESZy?Q~@mu{xz#a8MT zrmWdbya_9LvP76)*~dO!z4rC$4a$Zb0@j=YW<3Io@iQAZDw>(+ux#~KpO?@iS#2gW zYxmB>I=h6M%_J6?F|5tYZaMJ#fY%C^}1>R^` z;=TWCwyA_gqngJC--agr2h1$hTbI5xVSOdC?BWT&Uud5l=WJQgBeb!b|i7qi_SG%7PubKbj&v+{zo+D=OL}8=4{|TJSzh+$~uzret2(a7ttTI~Z<;D@HV z$IJpZ8WNlO?6!f4|Ec(E-lHFxpo-(sN z4SyKE%yCs(%#xPDmMOtrq|ja{(Vj7(?Ml%i6{YP*7B4F4-cuIbc!Ik}q~g@_Cvy%o zFe;yD(lcn%`(YE$;9Y;CRimKE{{pMu1Lo8Tdz}neoGn_cc1%}X!NAwS^Wss1R0reE zHw|(>n&mby@Eu^_wP?yLXj5%qv~*z2I?$@Jfq{3&lCF+@b7uE?HT(Qpq(5z;zH{V) z@Qwd`7I-$iifFL1Y1m|RVo9)lY(l%kh6bSyej$gmyb`8?5mw8RS4L$?HE-D8n{uT0 z$u8fO%W_WzCBF9c=Gm=T!Qv=Afi=ZJQrUI(i8Zr+D9$=#rgZ6m^$BL51h%3M_UaXv zt5*b9d$bn?E7`pbR-LtIiH=vvWTj1{pIYvJ!+~J2cjz! zCuKOKYENKl6yTV6a%RlY>P^k(n}Rbv*sDIU*S=t{tB9z3(O!3hQGCITV>d4=KMq#Y zTG1rrC~OkaTGd#V%=k5=QR>1e4GGpLi|)_N%ziIgd@rT6dyIwl)F2xYO@STPd|x#2Ch++NHVAlx--)@d zdH2$UH@nKGUcbj8CRf1{v*TZ@YQce$N7n<-tYMnDO(Ck;D5Lp>$|7q6W|svlnGrYI zQt#G!B&ptGlv;6Dt&ZzfYu8QNO&`z1YR~f0oaI#(y{KX}qo#qA!b_X*gx0VREMXT8 z#(rqB?O-V{VS6IgGIwi}Rs^%I4y$`Zv!(@;=7eUQ2`uRbtlj}_X$P2P9a_KgwpiQ< zNuR;yaiL+e)Zwo|`U{-`7anTh>1g2bXyo=_Y-=UjTMa20@$lBu&ab6 zmFFcbTXFYBFW1efTsKwy?`~BRe{pwlj=5yVLE95eHIJFjpFU`O%`r-%+pUN#J%BZI z2D9SNhJO*v?i(JfHV7(T(6Bndnm&U?YXYuOkK$+ z72ISRz+~RgJc}>NVMnv$k7oOd7LNiJj};jn4K2Y6uXIgPQU#u*MPxaqWwcFed+*b# zX5pb%z*JVi9y_6#_e6u;p$4HJuY?R5c?DV`9ZvizX?XQPBvnt{X_SBMWWXtkZt zr2V4H@&J>A0_zLTm5vrIPA8bsH?VrFXyiG;AbO&~F5`u3!~LusSC8LtdbzDd*M@aD z8-Hd+Tl$H2886;tEMWU^Aa8lutCWaW9vZC+{McLzT0BlLyPs&be$i~q(V|q4@a1xA zV8CN8j~*U|*FqP<#C9;~FJRSpV6MF3I7=6U&AWQMzF5Ttv8t!u z^n@;|yQRMP$?~e(%~CU%>>^r26V65ow1x_>+TEC_T*$UVq1Dr(MdBN?!vjr^4i>{% zY_S*GqAxJT1RPgh!6@;9VgHqopciYuzrEnA%NkTr?6IOnx1`1EN9HS5^U2GY%_lU^ zbYn_wY`pI6C>+4Z!@|=0Tz^eD7MbE)!K{JC;gaZF{z(%3g%aYOlJoDL*%mANPrONHL8J77#=2MPCns`g zS8?giVpKiAY;Vz|Wzd=#&>9%s8WX|#xH$P!DO>N%W1bJ%GA&wkW-wY#_++cVnscEo zX2n~l9jw7Cn$0^{EPpVYA7BoOUs#-gZ}>b)!RZ7c06tNeZZn|B46W0lb(UL-LEXS?ePL9SOfjH@>(3u z4SMD`@x$@EzS^~~Iom(ntNE3^>6wU!qUQrnZpC1p!UN}cBN}yY2=8b1iD0jBcyv6k zY#ImC7XGFQ0-PtylGK;IIXN%3VqbdiTrTk)5o#68Rt8McGnze5wEk0@!K9PXq}I@( z8qs1`aR2#FWBUzleQg;!99dQhEf4iroNh2V&0x7QyCwKTqen!P$Bafh2iDY%mZ_H; zzD@G}%{%ko+gpG3PM&UA>20jKHCObhKqaF^KMTh~n<>$H608~zS|SBl1a9Q7=gYpL z!xV2}OyJh0 z=K<5SDyC>?Ok}^n!z+15=VU;Ga~Fq%fyLAh2~Eu$VTuW7Z!tKvtc_5%Xt=N-tC>?s z%qQZ)g2t=+SK3sV|j$-g0kOyzQU*$7WZX-J1;JL+|H+Fb>z-E-DNzJ@1`A@ z!6|pAPx4Xu8O%k-&7 z53tx99cW?^Uh=?+#eGX+8)va&zySv7nuDDh0S<=U>bw3t>efiGadZ)Mtx-Isx47V; ztGHo^Lqm9EYr}z<{6muJljlzjyPmGpVZb!6>XYRmmX-V zW=J054UZ`-yL0e_(N_cUxy#NyUFa@b6>ywQ)HWl%S-`PjyEE^TrBU-%-@B;byk_IQ z4bCneYaV(8`6Nv25;MMV(OtezgXx6Q(iKhJ#^+pwI(fw_9yszFela+qP*jrO$Zlx! z$WgA}TvWiMzg zotcq$?&-3O!mbT7)0KQaEK6Vc)Y2(?l0(w5oUJFft&p()a;#->znj$^$=n+gIn<`7 z?aC}EfAaC=^2>}1JIn4b^qN<`X42eM6;DrU9WK{nxyUTqtdr=(?LB4FaR$-q3kRhu zz7!mhoNCd`r8!aMAtSr2LSi#-vyT%yk5q=jVP%;RLrxhPivtJQv@YCrXOpaPNMd}m zPkKG;VU+r9tn(+C8Lhu+25e;Hxc!sw=Ap|6*!tRSp51g@o42*i zam_;(wHb~wyBL_11k#wz6j)V$6|70Oa28!U!tgt(OPj7I z9_5>Ufl+2fViVJg2O^ahyqN#VCCt#1;IEwh!$sZ3&?qY6rEtK5rmuY)7+E7)wPI2j zI1|M7eW+wJF4gqSa$7KK-oiAd$_*{XB960f$taqqUg%w{AgcLziz9!~Z+ZJ!$_E(J z7Bxh!K3uYCv8+_f*~U#N4|I+P3v7Ct!0s4vSpMDw#v=`#+;bkVNGxJ#Qet5eJr~es zyehCOi02W%pwtoNR}LIAV?GPEOB|7|aNxLp$B8>y!AY29W4qpogJSn~w6e_j!N4tX zwB%zG*XazF4OicGl*N|Z41aKe=ca`-o7e`|;?4Z^OfdvnMzU z|69SpEs(>M%E_pBP<;8!yJ!3!3D5ua?Vk#hNRI+j#RL}RHwgGyRtfZa<$7=h4$x!2{k@v&44bz-B3l2I|98+byG1YakeUvwg zP=YAKfgLPUHN;H40+_QboS1I^77u%tDxrJeT2ZGPpEX(8dBp({fcFe`RAu;@?F@qY7xRrk&Au7650j3OE=A9Q~?aBODt z7TWG`RQX(2n6t$rk@|*4u>!+Ry*CHB7hG`F`;*Y=S)p)5!obmUnk2jT4JWB-8IPE! z6mTl6Im(z5xs4-3hOvjwXYsEjCaon4I~WfvkSqxJ7M8a~lrktF z1ut_owFeRAN; zmuTT##K0E&qe=F|jh|Wz6uM##eB-~o!0kamB6rB0Ln7OD9M$zZSQ7KkK~{W40Gk$% zVYBjwME)p;bcJsU?9v_wxwt1e?C~jR30l&`+4!P4UH?Iu+>9h%oyDg%U0-yOZAv3= zQl$4L#RUwY9bH@c0iFfj70yd8P*00WaDZ|Isuoaqd%3So-vyg3K%ay1A}kT}5L zUe$kM_t{S|MPkeAxxbqv%2`NjUF2~+;dXZSLixW|nldIY{Db~_^Bvo|`>r=bz0iyq zEGLX7cKSy$atQgXI(0r-?yJdug-xIO%JYu4IZHg`Ty%j|{|+O6`57jjfQBs!yC!gj z_%Mq$b{tmlXzU7_!;~?pL*Rl!AeYAyCDHDHHB|4@IH+DZ7!k4H?$lOh8wZvxjmkaTF&AWerd(s_@Sb&ZZbXVJ3&SMV zIgT<%8jW=hi3dzETH(a1ae$$LHA6Y&v)&S_Cs3g(MEv zxg6q2VcL_nG)osWu!=AUMKH-7;E3sA4evSn zkLkcsNed^=DXjO`9%W`Zz#J0EW#J@k!obGi(6grV{?vv^a~M<@4oS)|@@QPnIe5A6 z$c*QP3|tEuQulBc?)4PlTF8;{Sh@3j0qY&XsASG92ZhvR{@ys|^VdOg%VXcC_l^Xf zl2tiw^}>1Ck9k)Om@nTwq*^fN17oP15BsV$GG8QHcsN?PH{5ZU^U!fhIRA!&3LXa+ zUu;q@X_9MoJpSUk&WTx@Y!3gHbW(9}61w8RTXTSCjRWtE143^c6;v37SeR5a9QUL+ zu%{eQNIJ+`!@w@kz_Q1cWl96vkwzn#Mz)YfEgL7E8GT$&4sO$wj{Go}MSvm3z)>!t zk4^3N!>#`oZQ6Q}??Y^gMZ*Nw1`Z7drWuX_9KAamoJ=(i^B5ewufZhXz`*>Z^Z8ZJ zb60ME&zO16La1P+oX{dSem4dI2PdN`PK%Gk72jq1cVOlc3HGC%LSkm_c2T^S-zMAh z%@3&M5@nHcHd^Am%-u=qiKB*!^eqSeGsmTB97KAUv?~s&A5hc&;H017DL%o;G{l+f zucOui+23CqMfW(0`W%$wX^Ma9)GW%xJ+pyzh69_*LB$wGp@)v59uAB?44^^0Ka5&` zf`kN`WX>>j9_Z##V2n{Y5WKB#mPG?YLIYP1H=_;%y9ooM%Q1s#t}J^TxHho%aD*{0 zX$V=;z+lqA{cj7yo-+rOG?a$gvKg*vM&3I1ox zq&LG!RziX2ilg3-siton#blUNe4I5y4)RDG6gvDtov+KMz1 z+8Df8(>SL%2)HyEsmz&`duidG5MNJ4n{zh{dxRKrA1*UFY-h%NxiIF0q`Y*@LG6Vx zRW)f<950rCDaj zT~-HCv#gD?)s;r$J^2!ayY=@5N0284sdkE;4^<&|Pv^ z<;Owm8BS^y&8h-SRuN62Gnn-l-s%Q4nnf`4-f1$9XwW+hZ4!laeal7}2^!E#X z=R|s5KD@SH^mV~k9?mHZY|{_$Ivo0)c}>kU{L&WXl7oxy2XQGSG?`prw#%7beCN=^ zW%-Y~6tyQRR=CJN>00J|iYt>ZbIl0`{tb=UI*hVD&RPw68Xbq!3Yg?Y4$66)kPl(z zT6yUA)F%0m#>T5Hf>#(suQWyr9+LBM5q_I2GiX^r2pfb;UMGEq`ae%v&EHDbvx3d~T zb8d9LOpT0xbW*uiGGX2U&X7adN8E~^=H|~hc*$g$&Ai4ricG2w&Ner0ZgX?D|F-y% z*1EK7jiM>+RW|XT4PBgq-dXIv?@BWryS^G-*jNY0hyJ{^BU4<0yN?QNiM{ zi4C*L1SZWnOnN*{Cu%RtTQqHrIQ@LtA-Nj|L?jN$_%w=?G+IbD@?K%t-En$fLc^k} z2JQey-hu-y1r2IP4sfwFa#t|$-Z-FknSot|#e4l1?llbT1qZk*7<~90eajixS1<^l zEZM!|bTCgN+p-g^Gg^5x7T@2Pv<>3+qis^e^RquDONxWuQ)KJL&5%chTv zSJT>j9fepjSFLu~pq*&V!ux?iU_+y-#6k9iMzIE_3yw}2XO0M+aFl<*sA(dsrQ;-O zefZ+tL&j5^G;M_Qr!i^H&{ve1_(&#DNaY|q#{>2h2d;<)7M>5mOHMTLIQs17?&dkj z%5$*5|3HB~L&_EJB4;*s4f7t8h7f^-h;1jL?|7G9H)OLZ=Spbce&Ha^;KZrXs62(q zcwUo9NF>vZO=%Cp4(>Vm+;{7o3jcj~d%pK9el?YWYmS3TkG|Z$3;vwPMHXvpRyW;z zKj@CJbFZKRWH6BOeEY4h?$|D5|` znU(W_v&VsRhnw+(gPbN_*5}#|U6niH>Sg@8UpB+W_D-|?mu7#)tLB}GC;R7|oayzn z%qo%p309`3Z)2&mb?%z1FS z_k;63ZL?_gRCNsE}X97 zLEBy&5_8hic))ar>qx`)dqD>tG(BLdP1yNB%HiCOCjzw(%sCA*_?Vw@n0~Mlt>bQD zYCdIr-iE_URFRQ&Mut9L>c3ep7E7s>c(4g@i<~%r`JaM@<^cgWfrm?l{0tVd3a9K? z*edVy!g*5KuMaEL7qn36IWY3iwNX5TUTMoA=NVD zbd30wy@7YPzKTj*Tl%_aq29if8;x8RJ|7P-EA4xcA{NE6%+*uf%tL8H$STFuHdP(t zCub~j7%nqwzqsMdXL+aNNxxLcPc;#9)|0B!we40UJ#zecMR+@pMMX%^Lq64WjgPYQ z@^;R${1LpJqv>;`C8x&KzGf+-%4NJv5;GR;VKvKTXk-!UaA;IsGE-3?)z4zl95p`; zSHVAGOegKzyndZn!jZSbS;ulNov z4@#d`?Qv(??b;g^cen~yaySVqoQqA2Ee&5`_<8%aQuAW@@CV&&s@E)@HY?2$XcddA zdDto*YZWNe!aJ+s$jlIvKITawFMc%hS>3BR+-v@=<8Z(IKZT|I65kG}PSIl#bTQGj z+mY19WU<4+l1JyjStj?yzK|WW#bdubb27X!^ZfL+-22lOx*kwq;g*@8)E`}C!O{3h zErEe?gB1gd351eP0`&c=%t_JmkCODdGG zSt?v?_gr#_|HP%}`*i_rV%s)ocuLA_wG&aQ{`bRKJ~H!8*o_2MEry5fjsi;0V;J<6 zgbHKJU&g#}bJv&rlEAq%gGq3gL09aK!{Q$r+VyuF;PkU-Ln1m*5XwfWrz<9qpM86z%01Ofjd6sqWrrVu7U>)xb!6wId`mJ(+hdX6SF``sAxsA%ru5}M~fs* z_l|>tb_UJ-B@dWPXB-ktV+gdAVB|LaaX|3J0%pmC15*y|>@D7)Z1U8HeZ{4Rnf}TO zCX17bESq;7IA@X2B+@X+!C-cX&o>ul={tu6MJ)W81Qsx?c*@SGWZ}Rh^MKh>?;s-= zLyJaj0vpGJ^rl}2`&lMPPM*=DB_w&{0E5{AM(#NaWll|K)|~T^LoUU5_MIsg0!tD^ zj=P=a3u2tu@u%@zNl`bKtw0m+tw;V+3GY~)K0J^9z`6dcsNla&*O1)I-#?qA7#R6# zGMpSU7aol&QQ`?`I5cUNK*z}^dZNn%jw&1|Y2-V?Akb`ah)rihBU8=;remRuvZpRE z%E}z%&HLdjWVC_B-C_}&MJ>b6qQ@F{mdzyk=j2M& z4f$Mp7i7GC@|d(399Lx!*czb8aB#~R{zz#D2KED&7zHyJ%0v`QBKC5HOV(+xo3^q! zXo?cw3WrvsRf_CvR>}O!D%e!|4w#GUZ`!w%P?;q5Z;^(Ru*xYl zdyB(j6C#*o*C@1VpGo8o@?cWnnAqZ=JC&{Eg(JJp0!H4H2mC7{oOw6>W|xR~z!bDr zrK&RP%GY(~lkB4TYHHIOZ~q82|FuWcxpb zqlzAdeVGc49x?@t3ey%eNZJUqop{^KWpts@PUj&rpT#+5sRuHF*1rOm>!_+GAM?E9 zap2PKlMS40eet4O7O+m!I#|8YSwnQq1K!Gl1N^tbI=!ASGV@9#I3?(22RJscs%A8@ zcui;$+HinXDZ`N~;6Rheo(FpbEe>lcU2Mrzm@O6h(?6_qm(d=Kh;p|bf zY$u16AA^JfIWuM`%idV^rtkMg{$p#4*;nZ%?Y`Zc&-Y;VJ+FP=Cw{c8x^Q_mv-`0* zb38XTDpe$kuv|CNj`-LeUvP-c?Lw35OMzx73njkX3^z&NiQGX?9*MQgXfvD8a9LzU z0&jW2KYw|z##Wva3%|RsI4Cbx*f!fLf#ZPSt%6y~c6T{MC$$Rk)fo3O`eYwqS>SW< zg5?4x(E|@&=nLO6kau9V@Hi|c=fEJNz$oXyD5GE_)Wj(Dpk6`YS{mn@mW=VT=Rgh?2)%~POWu0=d1l=A>Wb%Y`;1`|2g?D?!n6!YnlD@@4x7s=aA28=g;Nr z&(`yR!z+>5u7P7xi&(@#QNDFz8I8hxiri`sC36x5vKlyL9`ejk5cOgd@;E5q(I}#} zkXwyWbix6%zJGqKX$s5<3Yd zm(-tEB>Pt}PkEP}{zm`hyVVZov@LX8*@6@pR1z4Lsxj`+yCxHG@Lt#Ih6C*P{NBVp zcw;uG9EO~09`N@lFc&nf6?ON$`Yw?OqlZI$2SL|c@HIZ7&((1__7wTi75*FYT##kD8SJuuug$<*#fo@ z2j-9j&TX$bIv5shNnrkG#=tg#fvrMe_X7v^EH>sghrbPZm#i)Z7EMx9f156J^3NTA zW-o?+MhwjFR$qQ|{@=fp|21i~WeI$r9x%$~*vPG6Jk+cxc7ailiSdXVf6an7aSCsO z8d&eFV{vN`S3SUb>w%D7ql1n^p3#BzIt#ec9x&x3N-;5(dnM*?$z)smmhJzhjV#YK z-Y5uaC`uk%$Xd7^6zn~MS}G{;k2Slm(HyBZ}MZcQeb{|-dy0z zA2o${W`cjz9Ab|>R`@qr@t3-NLNBB8g8FSiEO8IsMm@N9PVY@#g9Aqhld1yChQ<71 zih8;UoYNARy%eRC8ATNuInxquG4V_DzGts$;+dt&&U;*9Nu$goMLChpGO;2?QxkL_ z^~%}vyO*1o7#kbU=Qmy{>i)Bky+MJW!IACNfkhz;1-3YdhAcGr%YP?fk@+!+(xx zTmAWBEv;#FZv9^ahABF_%y05*gBCFJh5D-=VEm`lW2NQEw8}s@rXgtNGaD6+hNNCb zey1F%1#eE}F?%1l+;@)U*rk<449o&N*0+wa=rCNovw&Z&LrUpjn73nS(Slp+HtKWw zx6W9gA(d@#^Vmm$V%L0+!Wp7QD;Lan33Q*IpYBl{_3`c7HDBLeD|2rY@hER&JN1B9 z>mc)l1p>w9eCxlz~PsJ z;EV;hW0alODDuaF>)QdTBaJCvzDfRS;QXV&*0X@KM}aHMc!JFW=Bxzvgaxc8yv(Hz zFe{j;>la$99ANf%S!yVltop6=c>10nU;c2UPEtA(sJ`-9TLW8D1B1c?1~G%IBwcGK z*CrVO#-uPd0Y@gg+;m34w}<>#y&9@!FXYL4$;YsX|CxoTR+3=YzZ}6$Eu0As?0F1X zv1{1=UQ*8sFeu*;A!z(*!_&4u59K}_l$Y5Vxjw-CyjX+w&w5S?ImQb2-~ zgVu#dOlCb`mUzk`!pIkuP*Qw-a?z(in*)r0uK3SgyuU~k>6c{rFnOF|6+;ZUC(9Ce~heUE9Qv-wJ#ftRE1651SIZiF$)p*0BrO5YmLFnBC zwjf0*hl7Gs7D}%A7`|$)b85hcA8HZaA({e>c|jtP;Zr7_{;2hi(XCgf@t^L~9L<}P zZcl4){JMm-!o+dwDZ`n>4-q&16Ia4coaYT%uv$j{`!m=MSyl)zx|V7JkNv`@_8+b%KA zXJ(%9VdcE*m6Q1#8yS}SF<8H2$l+Pc|4)HWX|a^pLJ_AIJZTE-e1p9eNQj?(U`c{QUl5~LxubZAV?r9^?)9@*bVdp6Y7odt5Y$o>xTX*`JAq52NFa@Y zIe~#+t%Wm*fq9<9thtX%&RSiWX0ejvsqrKU<}~r;N^eSQo&Oi5O`dfwFl+()jRrQU z1B^+UOe_bO=5_P2Ed-quZQ{Vd_aO9=GLz7OH>b|YPR?UKd@isoAzkDElh1?s;tPF> zUml*~D9OaQV(vj+lL@z&&O2uc9GkRMBTQOw5~I{LMMKr~i_70k{OicSqG2A-j!E;I znFA6`md2c_mtM+vV|#1#WNXVmF%H7d7&w3RaB!UBc1aXvixHJ5FY3THhQs=en zBPZ{3ucIc861(KW;ZlH#C;qIV~HVn^2+mfO$eo0$-44#{#CD(=Ert69&zbC`HQa7iOq&o{o57)iIDBVik)yck!ke#o=p?Tyv@!b4xM zHJ56YQ32h1z@MVJ^xoZJL{EcjsV_)$`xCoWiEmI5#HLT;x<5st*Ce;9;B z1TCn(z%AVUs$GBMewmf~_ZW9Fv##3r{BbUymCD)_^N)Kx%J#%C@P{y5 zVRc|gd(gu7(3Gc4>eYdN{5K9vvtnXYbKsfB#r)xB)%6)6=_af#4xBF*u*S6hGW)_Y zD`851g5WAe(N7VQc^kGYytc4fY0+lS&=U*t|1H~i=72$escZk7Pem*4e0gw7`{wPt z^WNU%Q)pWtF=4PJcNnc&#j#UY_$Qz5W* z70c>CS&zL9idxrK#~Q{kXfJ$oWrJ48k`Ak{jm;|s<&N~8YII=c6I=0Mq2l3yRnD=S zWRy&E`BiM06g2D>M7VH?s4BUq1^#Pl?G&^K+H)ar8w(3}nwRdS1jb|R++ycKg12%^ z-E3@?^!!Al@-{x1B)>lw6y1(X$okCac&U4=N7D7wla;f}&(GgqbK&>X*Y0zx>}(z5 zGSoDhn7c&!WaV{UR4j0M%eIxj@}Pm@!4APtmW7+zIi>{ex3Rkam8EOBzx06%Y+j8g zj(s5}66@lq7!JeUEA>mtZ zOl6TvIMB!*a`i=0cah}+Q8u<#(d*IqM@1c(nbt70sGePMfRV@Oj03x3b%;c&mbBfY z28JF-gM*Ap8w{K{)k4CYxKw*CIL(;4bHl%BOsXyiT#UNU2y|qhk~!#Ny5h&zTbx_Y zs?E;bdsA%%-|nf2GKQB{OzyMoaWcPK^7q4Gf4e_#n(uPIkWHD#!NMCT=g3yOd_o$V zVM>;>z`^B*_#9F`9TDI1=MlHfyB?;a>iRlb-JUkGB*%U@5RrRzh3GE* zPba1^J1vrEWHEYUz@aE4puoW9t}?Mj&@E%3tB_gE1~z`RifRWY0R@AD9GX22Gq)#C z%5>t+UeW5vESMs2kaLmY!c!*Cw-g@{Jf|Vtp>^)gzk@D(+s`ihJb!=Z#@Xi`x)WCz zUevk1V(|nwt(8lr?=!e?S*GOq43_(q-)A^15Sx)OaZD3l0SMeG&M4-r8LaqG*%HlA$cw&xYyWNrZHV%rNv?o z$yA={%!L;J)n1%$nf9^yLYIVd-^KHmQkqM>!&k8j%#eCA!$IiZhbxY|nztU$$qk8l zJgZ7dsj=-uhFwEOld#Lh&cHvem@|DU~c>?+aj+XEI#FGV=m=g>Z{T+Q(Kd z2?y5t7l#E+7%ccJ4sc9N2vb$K*lz#f0F!uw*S-iBmrXuTnOHlnN9+kwULUhmH{l_} z-ZM?>*&P^MmAxGWZWM4y2^?T*&}im*a-oq);$We8#39Xp0bSM_3R=7!$|sc`FtBPI zVALyc4A(JeR*hM}?75&>+GhcS#FR%>r#R*t%_v|>U63TWw7^aCTmifBor8SGG|v^7 z?L2v-=V5y44rbLQj(rtp4$Gab6e<0*-NZJ4F>ew1y@Xw>bv26-O9yxNE1aEu z5;q)WWiB>0YC5s^5r_vKh=c$g}D~ zyW@gIlI|NAPBA4eS3aRAzg%h(lazuZU(^kUe}X$4 zxT7{SiLOgvWsiQ*DC@IXAl6_4=htKGIrm&dnqC|dO`6bZ|HhH8dck3tRR+y+=Ny=& z0y5cT9;|KR>g-tPJw3$Fd@7U2Qr+)&o@X+?Iv`oV#bHNb&CZ2<$X&O<);j5bxBjV)3V3)z}?9Mz36c9!_?fyKb% zj?#<*wvY#p_*X1wGnzD!Cn;i)&@lry)o0P1NgSaox)NOKL>)PkRGe2mGlQ6(Quhj z?q7sbyK73Y8mp0_Mq1+y=Nk*SByKTJ*^tm8xnUw_=V=C^7Xb|C!Y;b$2Tj~LuYFgI z7(>a*x#^K{yLX*ln_B63O3*3tDvuVA1M9?q7N&O$9YO|<%w02<7^OAXhNvhD?P)!z z`75DOO6L)el*1&ICl~fgNhArKlW;Mfk-*67kR<5d(8y-=pU9R?8&^@1@8r zmON?TX8dUOOTY445>uzSnVb-O5WMJ<<)y}ROCnD8`)o;M*K^=^G<&MaaH;5E+EmqO z?oabN_N1@XwrG-mbiiFY<01EKhud1;wzczLao}sva8-Pg;Ctv;ig0&F=f;Af4&59@ zf%CTx9h&g(1Ec){CeaBKng#nlu=5s(X%$5Vb~$X{#p(03G#EuE?l>Y_F`*~TV==!}fSc(QMPujr2@Deg*c3z#v;=)o zf#+1qBaK(TkB^I}PN`>Lh3 zXX7$@lCK1tZVg?2n19W)eu1hNjRL#8wAQtzHzYlsqLcntf~#3qX^Ai@e#R63tDx27`ekZTvYyD zkm}%PVpdxqlNI{qpUc_W)$=}=biX=KsH#>B#~HZka^E)e%dW*bEqExH5}KS}|}1H0XEh0kgME~+yM zT6li^;b-QMuX2~|Efm@N;+eg@$+i41UuW0LJf4wdy~-*fH0ac;z6;GFWf!jcT#RCQ z!4&rJp)3cZR(7hKgqR@9kua?Q=DZ8+&xJY9FW_1=L1a`GMm?t$ zRZ)ii0UMQ`8ZrJ)$^Yr({*P}1vqB0hi$XQOf`Q8=2F3(;F#(2s%X!i_F!pWZFnru9 zU&Q7kz_GWLG5#C7w*kWzcaOcL85IVV-2%K55}2MJbGzceSHn^NG`!CLd&$Lsx^k$Zrw6lhuFCOm z4C6n@oL9iUnUVcmL%Aj+S2}}dK>+95D~*jG*c-PCwQewP+RkY?;=V+;i_F z;S+9Ocdl1378CGn3*ZVk-d>ZzCni}h<;f>qIgwF}|7}9nw+Av8A8Pz;wx2wKv%rDX zF@ZVyIHUFi@A(`Jl5BD;ii|;zWdno7#IFg=Ve$!bvC2OGi<-LOEG^2Un~EN2)OUU2|sn4~(K47!4a(&q{EvV&L#*V0PcY?5<{& zJhAuIjNY6?F^!@=mmSLCj#l5*`dHlyv=kUk4ls%&h%+4s>{7Hjp6p&$!nS+?&(s2@ zz2$5&;WPhvc&I(lKAFJj(a1g{v}o$T30yuGSiC267X}oa6Ue+0!1qcZVM!X_2G1-p z%h?h)XLZitd#Nz#+{KPR5BT?LYECKQoOFUsC$sbKa?@EaH0Qn0RANY+U!ld>p*QP? zH^U4j(*V|n2O<#*xK;)I(*YlQ|}AU@tJ>@L^#0DzH*_$`5nwb6Sy-D(Ld@Q`F}bRu_Wxcs|;& zh^sgSrJ4w^^arr>U0`4rU=(&R5?Ww$T$quIfsspq$tr=(@eMzM=IgU~!*j8|m&lZlBk0ojfx8BGhU zJr^*_ZeXdtpksW2rAUC?cS4Y-f$^F`wps=D_yq>O0<69btiA>-eFIoLH?Wo@Op9t@ zS<=A$qJXdE1EU**=Z=PX6C9Wg3z%n#NB=U*pZj^09f#|8F~&m8suC8TYE4%wqk^B0 z7z8db7%{MHU%(;1OO^8hGlPN6(PjqT2@D(pj7J_bFkFZ}+iG;RfNigN;0i{LOAMT? z*ZkZYxR!pXmer72x`4-7inGXp>s$ff%L7X~EBHRF`p5TFVQKIxjj&a-qq$^*+fDvl z;92&8BjPw)m;>un4`z<;4YwNQ!WK82`q`O~uEpqLwOqqhOq+T8W)^7y1%nGLiIFTt zrpynoZggnpD!ibRZm`nwDvN$G%amEHB@XP(6Y|0wIJ(?9%mOmp6xjPVq-$MZ)J-s2 zad|q+cFTpz`2tl7A5~a-zV6N6D8~OG%5qUEN2ng7GJ~)JTU7#EWdi&DRSXJJj2sIj zG_%~bW;GjfaXty)+S|@}VlhLv0`FCa^z;c_N49b-Wn@k-U{k-nU8=$P)B?V@3pCz! zt^X9Y^uw=pJ7&)Q{GG3R$L#YL*L!^CNqogI(||q4VGf%#%dCGJm~}N-Oa!v^7l^Sr z?n(<^&h^7vOna`dgMjV@<_ZON$qkJ10W7W!IkOM2YPjvr`N=M>;a9G(A}xc(>cS@d z%Pgi7_UNTEdm3=G25`6uOcVOF{jey=1x4QN1)Tj8 zI4c*h&-7-^bzn8P!yJ^b%+Vp5W43$clx53)oANWtANV2=evrvVfZh88<3DKwX5o)fX_@Zf6t=V3L|(v7#|Db|Rxx0psOQ96K7gS58p8bcNlZfwg-B z`@{!qAq6b_35>Tm76z_X{-Sn#-qJ0+3$`p$-Woo!MfE^)Y0ti+!lGIS*v=+!Yfo6i z!2mjRQ$+YAZ@{+FEi)7u7$hEKa22pzWZ+<_VKThxv4Wv(MF8(5g_&XrOqN_+aS>cs z3z&=48Y{3j3vjjys0L49mED%-X~5k4!N_YtuSLX`HRk(V zZfv<9y`@%CFUUpj{}r+4(u_<2OnDobypqJvah+`UG*VMy;#|OFwSdWJ0pro&ZRrA> zJ{Q=Z7O<&aV7EFIoW5b(cM0}O0bG&}9t&hR1Zvn^cu!Bgz;koL!S2`Vd}4PrNnhP+ zv*V}MBt@Mw6Qa*7GvI7laHz_FyUFVr%AGU|F(+kNb)~;SHBIOyFEz=npAF)}4GrKp42X|Wq%~tyGsAaixLG>F3p|i(T1N5>#DvL|y zOC*R3E3&@-eS$L}g)e~N$7}|s1h$C*oDCZ|x-KwjaIG~D;FNgEVYGnxBm=v80OumJ zb$h?7Ra}c!U%+AS6?4iVz56oHl>)wJYx&-3tq+;Y_w>WM9X$s>3rq@sb-Fv^eyHgr z<;XMJ8#p!RoGIOK=JXl1%z|qMYgjVxGIPkWaP7OMYkAge!vj6d>xKf%@(Rpe4-|z` z&dEOH|FD2N<^jt*1uj9n?>G-xb?v2X-B96KbjN%8D&pFFd zzhQaVBVDPtdqb!6{#$1o<@K9k?cUWa;i(CXTk9;Rf9;HY zT#Tv$ynPp#bRV!SZ(vWYcGoR1R%=+KbAf5*8s-g#2m0KY)PtG&(ikNVY+q8y)G#4# zeI?%-&Gl^TD0SgKi>^RBV@ zv2v{ACXdhyR)VV%q#Mk#pO8SAFwPwz%Bcb`|cLbxgR(eKVWeQ*nUocqlkf1 zq@%n!BtvQH`RF^01|QfO7*wk-=<+@8sc#6LbCy|3AX@3w-i6DRJpPHTjXl0~@0RLL zM&{#hH;Njt7|hTMU??+WU{v_W@M@hh*93;xbsU$IIVUij6k^y`F2Z;q)<(_fWY0;p zC=ZYK^BCvUGP%^UvB>9M+3=xyNBdI;zNZ2o1^w3x$-j8vyT0qvy>99I@^|Y8E{NGaMkbvqtSz&03WW}11xF_ zm@@C}b@nURqAM=W@Z`~s<8JewI5_Focy4j6RyA^BHgn(zI>0F2$KG&3QO)4wg;{9@ z4nt(R-{wKM-_8fTtc6gV|3^j7G)i<|O##mZt# ziXNYur@Fmwj^(yjcUBc&dvi_2y~XJ1%I58T^DK*R{Mq_xI`33%hZQnrcbdDV91d!* zSa#{L;e?Y8mzHoUE#sATi0ZNsbUl4J!+%Z1U#ZX~9+O?qDMVTb{$}x>CaaTjB7xDF zZQ5?>b1yC|Z0q5XFgef>oON-&eO{20iNwK#BQNz=$IlPA;MmMAwBGoLfg=-JI}3Nk za*JA4RwkB;KboyvqDz9k{Gu8Z4m2^art8U0)Lh;uvf-o}TalB1BdeQ07Ynyj#=@<| zr&g*7a8Iyql}gW<$R(OSWv12<{*_XKE~!fbpSg$z{_&bs_%B9A{n+L+mCvT_e0Azb z|DIP53i)hb3FvG%*ke?P05ExZh&d zE-t$X&QsGBEFIf#7+Em330FMsHG6DvK)0FOW(PwvlR-hlQc0f1l!+d9CMdFTE~#c% z*tsnDim7y!E>;$XiHswbhk2xgzUO4wj78svn5lwip zfZ<+(fnKw+SmcojuID^FcTL~5SkHRSX5sS}o1`=vcE2rH=g0KEXI-9xyX=dD6EpMz zANETnFf0*qpZ4IhirO89mSppy6{YI2(i$HQFc-cujhL%suy9gPpG9Ju@f6M{El=h- zs;YSFF`k?hWXJSJ7Hu$lkhIh2@Z?8biTcK+@Uh-L9%e5o208q;G7!?$w37x3;%Ii z1Uafsb^PXGIMwb~0NW&siroiRSUibHVw1bm@R*~Q*;93@$G;cKeUb(HN*28n=sj@B z-F$9eh9`el#-*j2Qxg=#Ga{5(x%*ByFeYqhQC^V{s(+9{ZpMNJMwSO05;r)RicB42 z-)cEpPF%oIlq94U;w^e=MVnFDf<}voLy}87T9lOnJ4Kxq3F)tC;r0yW^geJQcdCJ# zrq;wBS)u0klOGlu>Lv0hiCvUP7EO0Ksud*I;n1)`>hb|ru}2HK^b->Ky(5kYg)nj@yE1W0{%~MV zc<`?+$U#Y{F~Lc3_GFo#e;oNLBAR&=JWTwyp0|Ecd|=70!?H^`&jxaQ^jj=o#26&y zk|`9>$PzPwLE?*|fZxnhN=XRJDKk~yy2iha7d3x!17&MZqE$eZZwIoWU)HkU=m&B;L5RjVOM4jbIaAPVM+@IAuM>iOsuVs>y=| z4ICW~O(H6{S@u^suA8@nA9rCls?7RN1Zu80a$1h~kr*%`j?U8&%d zAK3|-j+{wTHcL$1a8&nF=}y&88TtV%veK6fSgRNvh26OhO|F`7^RlHI*Yao{fmus9 zRlYdzw-m6MuG$!NZC#nv)dc~jYGJ$~ZydSXE;JqM+rSbqC5f|`$3@O~L!o_*(4+s1y%*P3MCvc;@-la*`F2vSL0-GXT>LlPN6KdZm%mJ%^oP~1UPaE88B}* zY~-~1(QJI{Kq$|@6$b=sGp@={ddSRMx7Fc4>owMD)wvu3n|g0(xz;a|>NK0PkhiR$ zS@uCfyY-!gyrRcG$bMRo=<V%7*wfwfRA+u+a2y=@< zPT6LoW8r;sO(eD4)Od4$Fl%0Vam#$0n0Wr2V;g7f=CBJ>;#*MQcKkp%SLh!{=Hr&m znuoTx7%w<9bz(-79P`9Rt_fRg?plg0zY`wf-MD@Cr_Yh4ZkAVGL>#xeRN29NshiPw ziy}w(QwmxLt(G!#7KoFJ8muCoFibeky-LU0$Y*ArkqS^j(`PDd;w;U z1dsKw*coFt#rX+pQA!%1=6e&mRnyfZD5rNU~uqO zVd53&3OF`JwyoE6V`Q#&^i7)=-`HndXfFy+?kx^ED%p2|Wk==%U-=Jav#+tJ-?>xG zRxu^+$>R@-{Y@Vvm(FO+)A<0pWnD1uN8@*n3PvVbt=-GkbaDl4kWDaCU_890lR0QY z6B|bZ!ya{M2Dv&R@izxpI43a5RWvl(pP8L^!8=g1B%yKggN3p$GLBnL5oq9aVPI6Y zaO5;_VCG!2fKBTJL!gL3@njt707jV%Mlpi}Vi}BU zk{#qPG|FGFQV?hkm_2F3W@h6Z4x0`i*gX56Y5InVWrybYBq(VlHJ{e9krH6q6U`_d z(I^(vC|=Ph)^kv_gi(9~TX$!Nl}}HoF~0cPki#Etnk&m~A4OOluU_ z7jVw{u=f$Onq)v@j35JJ1_MJz1G5JMg9ihn27}}TM)nhj*%TUB6&fBrcD6NXls9mZ z{K0Hv!8FZ_HP=ya<*8nk0=D1jAY=t^JFm0`ggYT3yYFm=v{RRSDc z$`(zFPe`b3Xi#ln3C>{SOlTCUU=&MW6c#v^^e?56Ipn~~-HoY7tkzy;G;BDg>9}>n zW(S_0F8&bl9haN6EqGFIGm6GA@>LvEtY}n}VB#}4F7lwE)Q6G3f|0M`pkz#=LXM+U z1f#S7lca!?gaKof_q>n?5|$N45(jLioLo`b+{nwIB5|O}{YNuD!xC&Pg|>h$+}H2r$^OFmPBj*tIw_ z3h44A9AODK;$z5RdxSxCLc`M-M)-He7O7&By=GfbEbR|rZUWI2|!$Vx)CT4|3$ z38REe(?e+{5s9;k0*#6S{Fh~zI4zDBKWTU`>8l-MCKNVWS$FbzG#|hpG4Jr*So(+vQI~$lZ7bvdNj1#%GQmX*_(S zG5rYR@g-+YBsQs7Fex{9w9H{tGhk|3VxtssS!V&W=8Q&_6HL+#j8+XSmIh2J z7EERhmo0iPKdWL1TG7Z;(7-pLN#_KU{SRf;h=uMNjKM~n(*+o&FW|JbVX(b&n8ktN zCr_w&M)RMWhZ!;$w05v)u4s&2z{H->FM6Qayn#`)fZ0Tc)$(6&u*Fjb*NzsQ4rcog zOu7%4ogXkeNw7FvXz=1_xx1*@@&FTWL{nBo%k-eO^cjsJr)CMPSU1_CG00=0>koyf zJ*U$*GzbMW3TiNNO6-$d8<^NYZMVd6P+6qo!=z4ld_i11E?MuCPZ|J3V)UIVzIKb?Fqgnk6!?QaLTq_zh{xI(Cj#)-~B+7T}8{4 zVzImzZ7~Phq7N|H{9v{SXnNt%Fw^7Yj1RUyw@kG>Gc}6Sh2ucOY@L1oW~VeqOKdAi zU}jp-;JLKfdtr0lFUA1WAp6JFZ(X&0$OI=>>eU^K#yLm@z0i()? z=H$6^EnhI{o@mm0!Q}Ly+4csr{f0*K56$)mns)27+&v9_s1E<~-X0Y%1cbHjW z!oA6dZB-aJI2uGZG-*#@@_xa{^x+)e21e-_jZzU!DM_ns53m+qXw%bZu+eB0)@a~T zU@_sj18@1Y4fvKxTI`pCCLWfvZfeXSJO~OCsJ1DSbZD0`F z;k$QDxFoAjrKsD1D}l)(5n=)ntR@Fc@*0hw#q$3-P!f2ST1{xt;b@7O(Uv_S#dBL*_Jg+fuNkF3FgqkP zSyr$(8!$;%1OzLzdLCen{?VYy!Qvp%5?#TiU#3nOwu13zOC?bHf^$_dY}1GD`DMiqwpDnA&MCp3F%uzZ->%sZpWVL`LXi{>W53;Z3;9xs@6Bvb-An0a3? z6kKlo7sJr1`=WuzV++TF$s4qy6O;s)7IvnXrEJW7#-P#JeY??dLu<^ARx1a0z84KP z3amkTtX2(;VgW58^P03LJQj^;_L{MiQK7~51CvulGxvjjp$4yh!F}8t8jJ$kf&}~| zHZ){dHE|eOEe>vYqS3yTd!e$&qo5gVJO+%d|6KRoIh`2t;vS#RK^wPZo*OKCUPyOz z-H%;f7PLq(g;{%r*@4N7Hev3z&mIR}ir5tC;&|;HYew28&jvrsCLMzo4FQ%|0rtWh zte$c2vpHUs&uEs~z_jFC!(v_)tyAN~W?PA=suBv? zb}bxln>HQ!*ZelxBDOPO_KpWUZ7-AioM$;SL{DsB^z zQI(e>JBlv5Xaxy%IJe!qw>b0N1%5t;70x%BogMPa653-mO6KolD>%?9@t{X}M+47` zhNpinOKtFL?O~Lh(CnDd=g84w|KiJoU(9w1eYYaFt4?T+6JybJV3IRv7BlGB{%`94 zLl1b$@3A>BM1S8K{crk7PJu?z9}N6I7^E-QmYOhG=DyN;(5Sq?g*V^=cLWpDiA`6u z8>BiKMJ<@*F6@}RquFalBcnnh(}o5y4?zoyuDe>yY7GrMk2pC37=63@lW#Uiav0?@ zFj?-uUL(sWCec`<-*`vvn9S3Qonr3}n&h#%yi4=YIog!$p0LQKVNqU=nPSDkME*Jrn*wJ#r zL;dxKHw}^;=O;gGka!W=@y3SxMuX}PrgwXr_!C^WZ|j)bk-dRQZT^zdUB^mio9J6T zVUT8MC#P|J+p(Jtc~beTl`G*iSxU?F@>Bw4ouE7*kTXVF1W{f zCq6$u;OCcZZ805Exc)+aH|gAG=YFgk9zHPHW;aB3ZGzj`!r71b6H||Xlt#j&H)m!_HcmNu(qVas ziF?_lC4x#U>|HZ-A{H#L{H*3DY#x2artq^%x3EQ&fa))v>1(66>-o-lazb#SPvxsU ziOH>8QxzOD))ahvauUKJ!DqO*IUhVE8vy;!7Z> zOXHlvL!7l9LcYew0-QLNI$t!pNhR%RIm~U5d4kE`{NaX&0gew9Lb%UaEWV=aYwF6O zSm(2pT{_1Y`kXx!n4AYz7$)&(h~4ZjfYKoSJZub?M2K4QyNzj_%^IZ#Ev8 zDQKhM$e8Q8;I`X7#^VhO4=rt%E8<~McVO1GaQ9HnS#p7a(d-9fhhnqKp~KAS2MUjr z&pT(}tXi=|eL|k3;sOTFw3;bh#*=Iw^YC;pxYWeTJm=pA7u5+bCIqr@ioBf9$uV!k zLCy(B3pq_Mcx>j`dM2XVQ@K;+&?O1IhXscN^tKp?@ICGGc(=oKsdSR@b`{l5mB}_t zp&_ML4B4eKV;qn2S>JGI=C)iVAjTymk|P$CB`G;4hC9J)3A53WM(!y=QI=1q1e{ys zr4f|#@Tti3RJNHjXQrkdOcP5v5Sf`YP3+kmwJBi^tcEulj_@qibhypTXJarshfiht z3q~QC0*1W9f74`G`q<1}*sWJ=XkuZpXgDa6v&XSj-XP(@F(oY-MRtY4BLQxz-9L^? zU(U`BaMN^uA>c0GeTIQqW%&oGgSY-|)C#EL`rx~qHJzo%NoM&!7h}H1J;By|TdyV^ z^4KP_?Mbu1jNb*!&*yG0IK1P*vkC8Zc-&3m3ig#fv}kYXi^SF*lWzse2eezJHnHq@ z-?4$kZnNn@W_PvDOlF}M7ut>MVkK2Si#gBi)~lZI=!{CynFJU429`-JtZMx`Kg|;J z*cLg{yJkaC#-h?RM@Dgx1)JCscf~WXZ;ljjoOqD;?`)PPW(J3RvqvWGmOgs?3=E7- z>PsG(Hsni8Xq09UoM^4X$luk`Vl-zV$NweA86{rMc2^SW+~vq5bk5`0hCj)TJsc{^ z9j7?%OetI-tfM5Uw&SqolEl8!4*_y^G3_=LJ9dkV`L z{jzT`$6RQ(Vn}2c5ODk`!pPzNhf!#0!ZKx%g&gi7hXivo6q%T&3$m&_U`e0w%w+aX z4%Y(!jS|;eo;k>sd7w+JYD2R$>zT5Ild|k84;IVMO!YVz&F$5# zq9UQADB2Lys^M^u-LB)&(p5X$G>Q^?J>M{LdNLdpnk3L+dq9D+FoH=iZ9|LQk%t_{ z7Kh{>6=*pxSj4aGEXj5)pwr&s5U-SjlggvA7WW?qg^Ug~FvYxF?rwR2Nms+M%HTrN zSCxe691;lyGS8Z$L!4&zonU5BQfNzCkzj6M(JCFL(ZHf}&7Q#^q$;jekkO(cpJ{^w zgVbyD{d=_cp_alJ$=BS$1>(|(3!$8LAF>L}TIN41rRZ{=8$&S$|W z?68pMS?(hq-x~+mJ1(#UIGohK9KfnMFOkbiMUiu&fr|o;5@NS}bkgYwkEGIK^P5-IOFQ!+=AAa~xZl7JQv*%iPCYli=v^hk>m!z(uZT zLqwy_+39-IPP4pf;1o4cU`#b=<5RiNbZOg5o|8Wg=CfQ_b??vez`Z|t&0lM6iT=i8 z``_`vg0Q8294!hPxNh_^2`p)_TT-_5koHii8dE?TwtD4 zJG;%yhG~)D*?&jmRJ;OLN0?~aHt6xTOmN$<;Xzr-$H2%_i>7TC$z=rqR7BaAE1u=FxEO3%s<>0FD z=)o)AEj-K>1xI+;6lU@1Fbc&sAC&iFY(5luQ0)kL5Z2~hoiJ!>66%^CAVKXid3;w%Z&y1+>2 zKxv+_vbM||X`P2X#*=rR;VZAORnq%t_BpFCh&QNy3TG5UY%*a1uYH^oFefYCpk?nH1Z!{U^D)}G-Xv_ zuak|kc(p-O?SJV`i#LUWtpP`cm>9c)RwSvMyRl2A$Dzw2V*%5gmklyH8{3TkB(iy) zc%sy|p(Xf&qleS3O`G2znrN@V%r|WX3){U|E?JI9#$%d(m)o|9ac)WE<-O6z&mzt! z%y5u}?*zkr29~G$H~!ejQ6Rz6cdUU~!ApVRhH3*#kx6gOvdcP{wLcBdt$b2<<{s#p|9=m2!^%BRTm2X#*Zg+d zoKWu3ckOJB$b?4aBaEU8JW5agD`;;N7J10x`{9e$n!t`2o5cco8BPj(Yrd{=RuYI^ zUaG}0LrcS1WydX{2~CPBjy!Yxc?}p<7C31xXyTjV#9-&-?Rr@LLjzlk*uAIzOd<_T z4;qacHA zLI;Kyz4s=X9?+9LJF)g!0nbDJJmKd*-B^}P=5%Qg3OQuFW}590XWJL^j@)&&ZBVqk za#&(o;`_I9?-oXW;FSN=I>+TIqx2rXH&X;lT&D}{X$W7zz+K=F;laRu;*Q7Cqw^Wg zPY9V=o_K)g&BZmj2iP(kIHxeM&1jf$#^ImfhdFvBhZP?jG7)JKlQ<~(rt!|-aN!&7 z3O&ATD-Q`8II37MUwb>TbIBng9VShggMvDT8Zr+E{&7_Jz_8Ngki5)6uDz4k1E(=_ zG%y`$G}^&fFVesnaY%N7=l#=)LHH^V%_!?)adyWoI;jMR5V|7e~?GHY4?#hm3` zGfS)HAbZTQ%r%^BH3wGKG-^F?GUs5@d*GyK!pLLrkGtxi!xCNZ?*|wzo@GeSIdC`U zfKbWYb*xS-^BmP>Q!`TzFl^{&{c$Gr(cJRZ`%)UsoGr<08V4V+Hh8%nVASz4TYEfz zLxWXLo033>Qo&*y#zUGt%%&d9Y%BtWTkh$uRetpI>BCyF9lXmHuQ_gJsK{+8bzwq- zsFRcNio-Sq55<}mIti&b#i^xnsYqGH*xZm*RGI#IgX7~+6}P-r?-duNzAk+Cr=x$t zQ2~{n^Ctp1bfP|PIUppG;9-))GKb-mje|<<0m(H@S{n{4NCYXixUiKtu$g$LN875XQ_;Vqk>N3Ooz4^5pC%wJfkgVv)eMH{^NCsj=ajlFq?A+VRDw3b%6n&gr zT#traVU*d@*rUNH;1R!|@@Prm^}aJQ3Q6I)M<&V`MzOqz&{@SKKWF->hV!g0Ld!YR zO(z`s($FMk;$1YU=VXan+KDr#3XZYMFxJgw)Hvc;F;O-x!EMSR&N*97&U({x-hzQ~ zhQq~|a{@m(@EmF25|QP(;=ujEx7#wKuwXKW%OTY-jXYBt`D_+3Jz@AKbKsz)fg_uW zL!8?*v)b8FRx1*>#aXj0i`pb7!Zt0Zg&`|~LD<5{c#CuUw43&A3sYyU=sK5W|H66g zwO3u+G9>H{smO>cUO6az$m7kNi*76Lgum#>`u5PHHN+!M{Ll*5^9x$JQs(}&WK`Dk zPK$A1&2dnzx@EeeN%77h`2%bx1CE`V!jo#UC_RQD#U)CHg{gJ!!E;9$i}T#36uHgW z;{U)>`X5Ih*PmInQ<{u9-)Lq`)KxjGYT?As($MtEpP8Y7??EG9M1$e11||*%NfT$A zo9?P_*rpivrm&`2FM7O4qj&kfh?Jd+9Sjs#pK#-x;4pWEl2*Zr_H&2r|2=k)`nRO( z8}s>Z%=RDNod1?{{@JV4Z&~Z>V&r^SWxhy?W`u0Km)NJUdSNeXB+E;Wz0V>qTohpO zKi|>7m65)5snmMIMXWimd1?;Izhu%&VN^^wD0GJ5mvgV-tpGHjwCeZ^90nrC_w;Z;fbC}&mn1_K~MA)6d$AO8(f%^cX^cIKn zUwF9fAToz!zPsgXOjudmJ($W%K4I~SFQb5 z{N){U3b#qh=Vk|H6Z-`xR1%tkEu55}9Gv|#%yH)t-aQO$FBV=2+oEv!fE&A`+txDv zE!#wHIFzg`<5f8*_xGaO8mGMf7mxnAXeQAl{h>ixhB2k2K`^0_XIw-7g4i9El6#4pg21EJ3;seYH4!jo{6_30yDLAaMU^&;8g9?8R8v8KI zZt!RJX*8~24nN0EXY74W~5@yXc4l^>?e7ZV$jx?}`95k-^ zY23nWX7H8Shf!gVqvnG{at5W09tTw(9J1Ww{7;nOkZMb5@7GI_dIuuS8g?*VNYeG) zE}GS3mvJiXvX!>)vg=#lGNmmvIBWeokHz6#Lh=ri2gdEE4}Yxt(eX~_0-H(d^{P(q zvbFZi#yYIQ1$Sn@Wb|I*xV_?ZKnjP4@K^4Vqgm{Z`~i;rhx@JtT#?yxP$uP|#Kwan zA&g=Vn0DH!ftzOw03;U;OB6KoT_{vTx&dvmb*LXv_*lje&<#u0~2e99HhFo@k~ zGP}`aYSYBtV$SB_aA5XP<`M?RiU#?NW^)ZzJsW1W3J1n74e}}nWqg|CO|~+wIiUQY zNi*QEAWxIh3I=n=cHN8845FM291KgCkGsWGnf-lG^X%W(c%_QN*Yx6)3s-~{T20hC z^@UMIz^qN*&$6Jxa*4Cmg~LhnKd!lKwQll`weOovJes%0GC#q#4A;%p)jeDM=Dj^weBq$nmfGH1-C`U5o_GDERB}scjnm!v_kXT+(sF2$ z4iRwfxi9C+q?wbh@Z|pPe~t1QM^zXO6*4mk9XTL#;-H9+D|^BH1Kti5>ltJMn$38a zWJ9bK&oHw7IVot-B%g3lCV^4N$MNykA0ZY@Dltv`OE&A*#?@;t+#wp!n<{MedR_Cb zKTYvgAM01ePG4hX#QSOg=hvl;lM!w)g3yeD;^v?+|DN^W06tF<~PSK|IwDn%WQp9t?FKAG&L`DWMbnKGqJe9 zaIkT0l(f%@E(V>*&77;2x$eG}#k48(x>AFHI8*rUBH=5g*4y>sIqH>9Y-wN=P)WbI z%5$}tu9Ri!lM_>em*?HHss8rv?%{U-`TOeT|NbbfTP2@)dFgk@;InUp4BdMENvQ@k zyZO#COm3MV*wk#^C$1l}V}YQ{WFf8W8)sI2o+7Z&`5eFOERk5N(%0viO>Y)Fa+&<@ z-rN%L{1#6k38@R&78_j_&$f8OoY^DHak7ouAnirMrQ^Pw>|$~U99X9_?M`6giJIiV zDw^@(Km#||hlB8d`#it;V{30U9ayvb!J}2Wo9=AVZd3>k@jfo{`PN5$4nG?~2YZFeHmM^b&OAn`rzF!i zRovRCv-zwAv+kCQC!Q=fxxA#YO){-ZlTFf1sB^QKY2I$$cEgJ5hi&35D-=5vRxD83 z`OoUpjm4HPmS{hj7Lcd-bXtAh&Qu{!^_5#DIGnq&jRqOjb6cV@gtAt!oXy)s)Q0g=COWMlD7cuF9DyLG$$wqdI4-E{=Z5j)j_;U^@ zG%qmOaeztsQO85Z{}WsdR%{KLQ^?3DCBWdZr9mZIn{CT3u11B6LPm`ep=qmswy&5_`aso~2OvTLr`(8TNX{lt@I026B$i-IeoO#XtSY|py1T^aaR z$hgb3mxQmbtX#95@%+!I*B&Lmy*{oNWRmt_v{cAkHd}LbWkuqVMVjhK#Tv_=8=d)c zvB$XA@o~f!)7vg=GhHTV-mIJwclGPUCpj3+c-Ucg?#72#UvjouF|X~Nq0Gszx1f<<<{(Gjgu^^+4||IjFw53i zgl&A%#>gqa5Fx&U;h*e=gT3iKk0pyM0$6(rx-%az$v*he@K9w!6IbL3MotBB296IF zj5Y}#oE!{M3=E8C{<;>iRESTgDp^XKl8`JZ=k|8DfywQb*c@eOmz`o_thz1a5N z=3q46!RaJrBB)`ur_u6UrOVPfv3>JiGKu$GIch97vpfCCp@};fZYks>F#Y6Au@j7B zQ4wQgRz9I9VC=!q#x~jOk7qVp(SpM=Opff9Pqg?79h#I^6mkdKH1q0acKRq8Fs1}L zR&U$Tx-_q_m5JvdgZv4ni~p34`fw&JV3yHnc*G_!g-xJ=K|6v^;oSvxs{@Msxd$2; zSPob)Z1G^okh~sxv>E{a~tm@h&9~P!n%=^h>KjY)FXQur9DJy)xv`lETXJX{YNpKdM zHo5azh?dCYpcFl+n;r)vlcoB%wCPoxI_Tkbh(p$bQCKYSwCkKi!HEZ)75X-`c<4Oh zkiEd56=*qwH`SC`LPJ`1LjtR|45PiigO;fYnCLi>Kts~tNT}^D4NhPo9RNc@vKBn&pC&LrWLdq-&)X6vQ5~P zbH<$BlqFNw968};cjmgl{ZdiIr=N9WB8B2Lt~0Ru6n3d<92C8q&=C^g?8Os%!)f)# z4TfpWi=G&?o1MwHSduWMH~x^joz<%Ti4P6?vl`~*9$Rq9BtVh3IH2#_f?&@xiXBn4l_J~-SzC+63 z7VDAhQ&}6sC)``QOYPOtl0RKZ2Ymz@g%%X?_HA%|+w?p7VOsK{yDL^(_Eg7M&YoEQ z%wzGHq*oZu~_7WtL%OZ z&GSILPs5;}|m4L_V$GcN3K)ktD<$Xv~?A~1#hKq*rZ z^EIxN13p607xI}V&F5YAflY;_ajDh=MwPhGW=9KVe({0^h6fCt78Xq0@*H1zW(jgi z9B8<#Hsiy_E|zI)d@8qOuJoE#uCi;#lho67g==$|g&F?|iELT0b8Bls^XlH5l*Q7_ z@7)frPVCE@@mS`F>$^EG&sdipVP3rNw6WcvoS@$8Y8M*Qn&rMs*y~b!KG3{YB|vsB zw?{Iw^tByqX4@Kjr*D?ot85Z=!zh69OUT(MmTA+NWHu~ZJ?|Dv=`fjqOkibRJU0Y5()%V=w z{ffz%iT6gCv$q6`)|!R9IXrW|EKUDxc_)Ui&DcizdB+Y(Rke>5|1KPU^lia9&$ZE; z`j$U@tmyvbY{9V=vz>~X`{MLZr=H_|s$c)}`h#Pq*Bkaalq4{}bS!PtGLc{`F`c?( z+tkLuT+=Tx3Cy9Z>#Fo7u%s_wmcF&)DAyiUb9Rp6S(+al*w;C5>k9C;kO z4@&dpJ)9u$HLU+@Tz>hnMA>VO-(DzwKjHA$M{73|?-K_-VYhWb0-bx7DT>Se9JUbm_rtnKIdnjP1(6EK=s?S?SkF!xexz1dtuB>`?AVAgD=s?u34Zkj(7rUS+ zwdJ8?)dE$i1I#67r`sHOtER|vm|c9;0w$#twj<{_J~41|Fhm+LD1J!h+LR!W(V&*L zKqPOCh!^7~9S5Zu3IYO;c^D2#iZrM_@hr;mU}SP&;94-3$$>E|BOy=2<58#XhBK_1 z`udAburV+8G;87fw?Jgx!L#Qa8`pf4*&z6JOQH12jbG0+?oe4~DAg==tWoCJ!Jah+ z-!1(3FFA1UEU4b|A@IcOCu<%`hP?Z6>cYizcW!v?Vw`7$zN{*__^0) zi4-~i6WheUewjk6o^8bg!JNcbMhiuDzKl>>p~^HtpujPRYoUn8LOY8G%xa9{Y6~s1 z99XaMF=r`oc5!jPdceWa!(4Je=$rzx#1Y=K1*>huIkH0Lq@vWZM#E!_yYrI2Z!vgc^iZBJ*h+vi<4Ah* zn~i%<1W2En@N-|I^gczIUlXMdB^n=1G~RhoO64Hi8)xS)6Zqm6J>PTwx6-A$KL+(a z3z!2^4}D92HzDC4>m&yLm;(g{2^@Sb$77n<)E-KzEqwDVf&T>q+l6;bOCGFyv4Ah> z0NaNI-ZiV3^%A&#F)&&-90*Ym`t^YE*Rfef3PN6qoEMVWSRT!qq{En1;1TuU@RxTY zdX7Rm4@JKCNd`RJv_w%TqCvZhLvP!pdxs_!X`E-uIL+eGut21Nol9BzN?W6pi1gL8 z#(P3~(*qtRoZoTGL+0-!!$po#uUNP@3p^3m@V!p642y!M~PdJZ(1JX!T-0^ipSzm*c4+ooLUTF1Y(Uor6jZ^Y3l77Uiw3(RB_ zludmiR2tZ3DYNNxIO}AvW&L~bHisd~>H)J$uUhT_UabRc9F6POoFQPS%mubBI`N2-hw2LzQA*(@GpomtBg0y|V-Lm@1Eq#5o4OoY z-|_MC*#7tsa*jdh-gBOW1#BB0cwc8=d!%*W;DL)#i;Eruv)qweTncP53x(to1tvXU zl6k_E#=w0?f$hpBt|W$KRYH#2ga!U3DI^~CV{Q@>oubI6wNOTDp`=j*!!|uO;RXX4 zN4B&BEE5*6l_;?FH1P2(475rR@N#7H5hzW1(ov%Fi6O*&#>ND%hORafc7{ZO6^-xh zrlz}_%Boxmd41{Yr=Xr+Mb1kXIwza@{#$WjorBALL*YFIL4F~V+mM-|-;FErlID04tj=FB$Zy^YcZj@(Qd4FNCRJ$XHR*DfeK$D)_=eACn`-`F!1 z7)56;V4k#qf0aXcO2dIwU-iw_L`^hyZaVb)41?$_M$w#u{Ble=Jq-eE28k0MFi%V1 zuVa1db3l;6kx%BR*c?W-Uk;L5j*>67NIp{dCt1cI@zrUYi^8=#3q%wi^7lPpuUf#z z_fYc2LAjm;ey@dWk6b@JINJH)(djhr1g@+FUJMDm2PLb%RaH+55%C!|EgcYbE5=KhlN7RUU8gS#-GK>rr^LFmHv5!DBF|--c=5K z*BeP`)U_4P# z($&G(a76#pA~sev79&#@pF>x39#)-x7<{!c{qi%JhHttbFUhQMEdTXMKh`uXRM|J= zaprmj;av(B_gn}(;qH30VDAyfx{Dv`j&92_-0Ajvd&|G;*?$YAzA-f1`=;=}*}*fn zaciilzKf~uf?s~AY%>}-6Q1*=9pDv96#UST)u0r5hJo)?0_TjR+YAT!IUe%M92A_y zxJhrdWC5f2oP%NoilX-tSd$V2UOZ4#ax9kL#4ppx=W|eO*+OYIPSHCDbPO8`qdIi8 z+*v*x5bW9^^6SAY76rC74C2ccO7$pk2RL%RdCG8X#=H|rcW0c~`cEq~!QiV4%MyOM z#P_cxs+~pqucq~H7Cfi&Y+-_E7e5}AcK|)mh z%!`ogyeSUMR=TWK4y<0E{`DyE1w0g)bx<-uQGh{Fz~`W3P$Fko0;dY2fZYNvm4lKu z7#s~0_`DdoHz;s#NZ?9P;16KrZ^(PSjn8w|8b+lB{928iT#Na|mNUpWaIrNCFga%B zBy!&BjJ=l0c%#+rQzv7BcCl6|i$cPJBRWh~5<<^@RK4Gk{@z0RyTrGUtN#`;8fPwS zoM8GZnWt4ihH=T^_B|RO46k!X9Cei5bHqTj_*+Y_UX$W;^w!OI?q#{l zf2V@~OvR;BublH|p0jDdQ z^4l?R>%F!zRA5`hz^S!>*@lTN$AWp4f~a4kh>D|-+(D6DQhZzv+p3P-{m07mV1dYo z2RwHi*pK*RDx<9AsOi8s$0jir394o4uIBDivF$vU8U1Tfoa^_JHF}2;aHp zH6m9vL=N(&b+WS*@NqGU?`Ra5@Idfh0$bM&$)FpO{}?PS9pZBrtT17?@`2&4mBMzF zhf~=Y`8>o6CE7pU;nfv!VE0-mR5HCu+fh(yp^%1zs8?fY-_)O8w#~Fka>-oDBc~^dW=>k8==!*P z(oR|T>@P<9|L*uMo_l&){n5tB=Ns!Uh8fSj%2AT%?d!X%_}8wB&V_5FvsXRK+NK;H zpvtVoB=S#;k=yQ%pqL}ylLIqU8ToS*n8O&@)+7iy=?f$^97GoO$}Znal=7@iLRB#zSZR z|9k(RdFuOsv*t@)YHX(@q7eUd0e^_Y@ds&M-VEy)*sgp{VlZJ4 z3P@ZP!oVid$a$*4Z)O66Np_q-0@I$Pn6wYq+Fe+iofvddJkqr9Cpa5UhIM2r^3810NTx?h)*8FJ_KzW?v%_^f{NP5UmSBK})^97Jhc` zleaQ9hEJdi^~QIj?66V5;79A84kv6Y?AKqL7Gh(9)sll9GDBKW1kJK9vm)@f->Z9GDtS7fkQt)L^*DFF0pG zGn46>g)D4*CJl_Ke0Lt6P`x?luv&V~th95PMXSVmP1f!ei#FOMYkb>q$K5a<+ZMnKHj3<=(T$xPl33>Mll`Br1nk7dh)6W#J2^YCAbxIqg zFf?;%%?K#JYjk5S6OWNZD$TJ_$7;mpJVAtDuCh@TDjvGO)Jd^KyIw2&! z;sHa1g^|F)aE>{wjJ+Xd4;)yg9@yAs)Wyiu!DryQn1!KgWr-;o$8MMou-BCK2}?{j3HP zCUnd>%T#i@fwlRQ07q1TifrKlX1xy#)wT{y5+030Cob{?r#$4^zT>D$+lE$887D3U zg(G>JJ~X;NXc94f(ZD5V(8yYG&W*RD=Ou%)nCk&XhG`kiOlK~%OR6juIXl6DnT4T& zwPR@l`-AR-VjCFt#$+-~>SWWp^k`Mu8b>3p1_ur&0b3`ZMO9B0EdIJ}y3zFmY5k`! z2+#kcvd?zr`I<8p@#oeY-1qJ8nIO;E@fTm1%Pvbjo^fkpe95Qn76SjWd>JF1=O$h{ zJdfQ&m)VV@MT_ZUquQ6mob=Be?ys09PH1Uqc3K$Vq>wR%z3M}=TtZji6an6tJ4t-A z3|zS#1)80IJmiwgxMTd{0i)B6Lt?fs7}(`LG_WxQ&NiLpz;vX@Wv__?6DLChXT^m@ zLK+Tixg8A*Ocxm1M18!f8IupLn8LsmB4}ja?1`4><}8oV67-ggP=Fuqa$)VDC+GV*gW|M_G5^p^y(w}nZDaXS`ZHsk$ayOcvOlV*Xv`b)>dEvm`_JM&< zV1dqSrPF;9o0qViP%vT)aA06*I8yOVm62z{B}Tcn^V*CGLQ^}A$m$fZhlCuoeN{3~ zh@-IE%F}sE>7#otv24a45=*w8usG&s{pw_t#h5T zMs^F{4kNbW9d~s35?lF18rXFAxXOkVaEOOJVE%d6xg*M>o%_r~v9mLpIIIm?t)4vm z$K@g6#JFmAxO8K&PJL&?+Y3rI$5bBZF#On^Mqot>~a@R5RMf$mklk_9C@ z`sWm>toM2PDqvIHy&cDlmjyI3g&a_f^zLWUTEOt7=OrV{4hQxb7ueNu%GumI8nqRF zI4r+3$?20ISEz1B+$WA7Vx84&dK?QmnRyh2x9?yvex13k)ZYoNG7Qvq;?fuxhz|eQ8 z*40e~&A&Z#_FtJem5bf1WMZpg%g@WHdZ7##gBfz{5>(U+LfN!7Fhw z1fCTOriuA5K1ed?h`CZO)N-R~&su{{9|L3I_8HAfJ@2yeHzWvYt!WS~OKxS7XxKaH z!hKzhTxRBg28Opn2L&EmE@p6GU=ZL~6PL5Sn4dk(E4)kSh(ZH{>{)H5R#u0o2Q#;F zOgO+~b>fKXl8HR&*|Jh6E!?jEKf#kFzgQy0e*V_P)odv_(Y&1%$MV;0Ov$TI4xZ`s za_81L?{tEf->Ljpo>F#bWAw|2WT|3?M#c>@O#@pkbt9D-7(e?8wysdO`}p!iz#kuP zTjpCDG7jr`Ulep)kYB_dy~Bx>XF+G`Erqp_Gn)3i-r&T)W&wA&#}?jA6FN0l++mB( zSSZ8)u}3?g;jD7aL2j1|4b7@NjN7!I$8XxPf%(OY!}4`A+C<%IIetiO$o2VUzS3l- z*)0X($qA9ycmK29D9yb1Xu;aw>w9yzZjR;6s%XpaxV+-2hx@6Q%bw%~zmj>X7E{)q z`menFM{!KX(X@n*dZGpld@ma2xG=DN$Yc<4TIy7}X^G<}L+`H-L(~>9x(14vJ1WX3 zFvCtn9K@T!wp!N zKQKmbV2^#!RGq+8Y*6fMz^K20Npk`dw}5}JP~@)1i7`)HoziQAn@YMx+dSF*+85R; zEnsRk;5@dWuI5u|{g%4C44${??J4DDZqv%DE$Xu-u;o23yQfzEjk)5(qbina{d zGm0|(0gTEjjC>y&m^Uy^jp*ddVBptCOEHmOc)6HyO5@rsjIxeGUk!x}E-)%4hRZNG zNg1*B7jSIQsM=g%sUuNkw1xd=DM$4LR-K5dY6a(L1=er_*4Pc~;S;(`7jUjjbCsFE zq@UoL^w65ep*Hr0INRq|XZ3#)SJfq04>N2r6YEfA_@%-W=c2syvy%1)j!6?r^Ev`c zBHBwU+K)Q$q%Lp&XkOpxP@j3Azw@#5p@qD?i+Implwqytn0lyu_f?PGPi2m$a>;$j zO!~&?_knfx2F`g26Js8*xCSutt?1-!)DycXFoj*dHjR;)fqC%`N3j-xJTp`I3(Ua* z%#IJ(+rGt3{=jck(%o9XFB)A<1dOF@(N z1}5tVqW4d@MtEdzD>hJION#w6wXx9PD@&5H8^dD_2Emotj0%3?EmpVK88r(yrd{Cq z;L)C9-d~Z~|Diqi=L)ur1O0z2*xLSuWmq)x-dMnUPeI^nVroVCgb%{0Rogp^O1azE z3;Z6i&Qai+cVMRYOU^k5SmOg&q%Tb5s}NlIm_b0mTT;>cr(=kB0gGD!i$MX4&j!Yg zHyPyvm_t9Xc`Xp>nP#!`fV5CKcMHRmZ4y=03R7Yex;J=meRbtJ)}j%fz-(3!s<(kj zYXg(P1ST1W#JY##ifRnkE7POI?`^82C8~Brj;D1~V8Ple>S#O~!#^ z*#(}d0h}LKO#iZC`d1I0BL%rN9l5`LOmDPgvuI%6@u2_w0$!nCvWzQv9X6N0Sy29% zEw!tnCdFq9e4LEl$ zu;~BLY_@>O_QLGo18fBsnD2Qe#_wdCL%ZRwnUf)aN8JzQ75?Fi}uvV{PjlIC) z`+;%BkIXhfMsWv@B>~fqB=E5ATG1V~pg(KDhv^(YRxHS1n2~9~=Ja3z-!GoHs0{PP zDL=#~oGHxf?VRvxTeU#lYg8`9!y@Mz|msBtzW*>_2$|o z6S!NAER9^MTvxCcC$QQE%)Q#lQsBVeU%)wIGZUXf_MXeV+zqaAGkagIoLBu({Jl%3 z>xboj3t0UGSmQ6S#w)PLe_-`>VBw2k$PJM!Udi*1bL9~SPS#x;*}OQ}SCwTi@6SkJ zPi|P5d|^S#hLvAKGg>((bVSN5>6)dOxKRD^Q0#%=X=dE^)`!N!A_B%tZ@qL4OR><8m%TD=$jg_V2L-UboDN& z(;QO{aICdnxqfy3#_W|E-n*R^teoA&{O+ogfYicHW0_kMW=sm&+?lbtN?mh*A;-L* zoD!T{-W=e2`e4uP58S&9xMpwQn*D%fm(HsD>IE!~3hbG}9NU}LxEHXr zC2-$gU=d@*nPb7Z!$IcO2aBG~(tVS;vwtsXo3^wiss365v%G?7@dWN|1{~Ig?3Ewb z!Ub4@5?CW0I9eRoJpz~~UEnb^@MA3Kjgs_#EIBPtYq{^FR{k3-H6K|2P2D|l`sqV6 zy(iWPu*w@Scq=ielym*-+_n7nF6%eDZi#Km*`C{(z+T_UmK3nt;=+R2T}&}n>n~o~ zw3qRy=qX+%FoqOB7g*9M~%vn1vh|x0V?surP=e zF!WZ>+cw#i*CD(A={(;FhctcGG#2c0HCp2uuruxg>yl2@J_oK90i0H6cJ*A~=xzAN z(K~_T%9>n9nIm7!kEAuQX=tSK}1-)eC&7XZU`f;k)`^PvRcVm;)0JI;Smma*)4xJlA=)x&gC~gSm$Q=hY89 z93dRyu^idl3r}rfH=Dq6or_CR_mWa9hhi+pgaD3}2F!9F7>o=!wol-_`+%om&50%l zuFDtr{(UfUpTKCifk`BSA+m*`#EX$Zfx-8Z%I0IscE%i%ci1^qnxkd|$AkwQC%iZ{ z7`X(zC&qnX&6~r}KY{DapEDQEaC8)$z0z|wGnc(AVWn%%5s#kT77a{%T}&+xI3@?2 z`y3T7@bA^i%K`^p-!YcpN@dQ7d2=k9^{33Yn7wl>x!yhC`>^5s+YP*LF7UoFxOsiU zvDp_`@@nMv+>(n*Hq%)tApIo7@B_!)fQyp6oC_bE*gj#Y9*eJA!9{baOG^?sBw{(* z5;zttxVn}_w5d||;((VHyu%{Z1da7{?Kb~)y3=HA>g zg(FUD*mCY(OLJh$|9iGj_j=J>PENnNm{aq0c1yS3%zbm`+=YYnT^i@F1jfwV#Jl^z z%sDS_{CmJZ@8O=DM|=LgnQ&~&0nXVMu6|Z?DB^LLqVBlvlb8Ghj;jW|lkc2pUc`!WaKs}C%?WVqT6urFxfK4%-; zV#Yi1PN7`EmADW;*=u(dt}SD|yzFC%tH%viEgr5@8#s{=4d#oD#zGwZ&{6G zI95L3=~}>XDdyUZGxxJ=U;O8NkfHFv?ax`yIV%f$nKTw8%yi(GS#bSX&AEe-8bVgQ z44iwKPsjuvTlm5GVOy>2|2;SMf4K3Ff&blw^KT0Hru^NLvFF$#%TBHj46IYdUMN;d zR>&)7GSwSA-R^MGYTy3TQ&>zDmbo$f<5W3&yGDWIROQ88XKybxU^jk{7M9>_woiK1 z2ChA^96Jy2y??-KHO!UFS#-Nm*qEf%ns z2C#G1y?iqLx{mjZ)}OoI*l4~lZkWJgf9@l1mdvp|7hZi};J^5KkMV!rsegI*1?&w7 zV40;Mxx#oO!xw=wrj;UF1U?w*-E?@mMBzl&9PWRnN~%fBp#dDueHW+HTnt*zb$adX zB^%gPm6**ESV9vx_AKOXI&&-gKJWVvygLgxCdzQ#{wGzGZduY@{VX!&^n+Z3o-+=bBgep_}u$Pe$CU>v`HT9h!XGN;M};c{uYE@0I8+`x`!g zc))#q1GnlqzWX0|Ul_c0dCkf4w;|I^ZJzJnLY_Agj82ja>{ll6o>TZ5WY4{1LwEH9 z_67rvYxhnl%5ojAU3$5IcO?VY&RC8mAJ~HxSd+9_)t_o)lO2b2<3$^K{Y8AiUdT3%k#-En^2(@EC3vrP9~+mb1~T=&lX{SR39+1l3`Bp!Kmpp*Oi zTWRaE)ixi0Jw3fiFKSw?W14zjtoeyOW=mxZ5*9Htami?CEI7cF&AvW0dV_#;Gb<0D z!I8?gw==bZc6!_m64kn{8?>WfsgfMmmEwS%%U;eZTRT_Jxz$0^$mP@p70)FeqObOO z?J0IR(X{L9Slq<4WMp|OGa5Ay=X9-$i#c&5yeI4b|I{=!BU zp=2@7V`?)y6w@Y#3f)Oz*7c}dHp#j}OH0tCE9h|F(m6$3yepPTIrkWDy49tO%XG0ms?`N~JO9YQj4y<_C zDi)TpGH3Hyi{$6i1I=zO>9{16aiEcrD`P>!)fXiT7_4+P7HqrmqlUp;YmVh}$>0#f zlBtnLlvypDJvonBK6kNH5eY3>=_VdJPdg>}twUf)p#Q?#Cew3oWvWlN6+6^6#g=d5 zLMA2&2Zj9%pKoUz_+zsuRGE=0LrInOXXEpbAce-0PAU3F3QW|^a-2k?wJs(fSv= zSg3L&>$98SQIn86-&|KG^%T0TKO&pRTP$q9-QD)WFZRjD4GXWINZ&H^Sdr@1k8JU^ zzrTL}e01N!CvB_M{QnQ`fO>vfO{s3l`n7L(OG5pPO?rf-(e`9 zrOBbNpz-!&d4cf1E7St)*YO%CbxkoVY@h3Q=8@5=1{Y1Gk20Y=XT&udo5W2nu!KxG zBzRRyP-;zUo5_}inzyP1ds0}|t*=>Hkif_!x}t!!xO1sas;79Al%UELu=daa2Eyz5nXkyrlUPVp*9ItQj{P^OaL>IPt7x`NXL@ zDTFjc=m7MmBnW`_Bu^)%D5$XzWCDQd48Gg!FJxt8a-+qA7^I8z5j4* zC7)x5rj0{3w*rIvC*%Ep?yRqC;W@xw$GR=P3&E~O~>m|3R zM$dWA8hXaxJ=0 z*MZf-@13?Wd1f?At=u3fx$ya(mzM4}6B<|)m=4Z;lcR21^B}~rz3hVT$|R{b743S9 zKJt|;P!?~sXc3kBV8`}^i-G0BKaFclTjba29$;Y8aB5m$(B-G{Mttvv9)p^P2Gf5v z30r42G5IN+a-AcjDRyTz>(K*?LQi?{L|^3UJ#uxbLWUBr*V3k-z>3z527S}h6UBGW z__!{SZ*o!jMQ)?R52vd$s;%jMm}tDSP*YdrAg6=NTFQUA8{2BxQl^#_H5ojxD(U5Qym^AB((5*>2S}LbQt~2eF zU7c(yOxWf*kLEcl@80`P z=ZkE4CcF8(vX1DIMrN7SCvx(oqi?Wv*tfHIddhIF@hxc*b5U7$M9GXxbaDyJtfq@0kVlK7U-^{Cl>GQ{Q4Chx7^u#yji`-ue|v3bQ># z??1OYSoO=7+mBIk#lqfaDoePdPBbZ8vgiyDQ4(ph*eJU#soh#*QEuOlW<{}$9P#2z zH|`o|sBn07WtKdaU9t68Qvc?5ZUF~n@r7$ABwMF2%1+8=l4!U->%wgnf3|h|AGE%? z@++6uM1et$;k@#_7!g~QqfI(Rr|u_|G#fscz2bq^9FCpY&tx0KIlM}<9+hQIlec_S z=r}3Sptr=*_Os-^*^4bKX8h#+B_y?P=L{ECtq+a73Y8{Vw^+pITwrmzUCO|vp#H~$ z=@q;4kAqFE_Z#jl*=SytPcFbO)(yr#MFs>p%6*7qY4ZT4@?RyMw^@L{zyA~UWDCb!TcqKg%OJ%i#nJr>Mp2GxV!J$($34j z3;K$`*G45XNi6>z@Oa-$c6sJXi><~YS6P)a9J!-s9G+Sq#^Ku0$l{Z3#phShr2J)} zz<(VU$DQdNwr3uI`cq;`2W11MOzYpV;aIqCdbi_>1_s6k-3fp7|0ysmXjt{IpW(^= zUlI*2*-geLY!fmMXk2JgYhYxRV3;+zC3pvmS4PXh#bWa|F&i(?JNjE}t$^`5iRL=y zZovtRQUXlxf31r=HF5bV!Q#rzXPb@Pf2gJ{=-#zr&gq-oKVGq2+idZt%67BUTD==g znm^2xHJa3R*s9-Y@Lau@cSi$zMFZ0Yrz;x{b#F3ZR$%b{eCPqQvtbPb&y42u6Kv0m zr#NIVb)9O`+rjEr$dM?_{BAe1%LP-{56yALEOrbUpDOwsHZUq1H1SU0D;$G3> zEW*;Dr143E#VMod)T%}miz6=%uyyQe_59FmmC&sDqLHtHfl+|L_l-T%8~YnsCmCZH zylog*XB=R>(7k{addGsczd*^9cW2f(aBSD)Y^l~c!sNlq~kxC z9gS~3bxD2PF0!OilxIi7XFcIXOlBKYeIpirx};j-*u6?fNK$Fpsh7ODH#T>!wn!5? zr1*#X$Esu5H`ww{oZfS%EpLW}{parON7}MixLtW|Y!|$D_mj<+Z*Sgv!`NoGOv!_` zof7@F1}#fZHfc-ndq{9}#;~{=FdI2Au5@7WDqwMXz-MQ{VtJ;|?g_J1z+y>*CTE=% zmmKzz1Fb$6SZeRMiXLcSzp;q(!`|X62Z0KOtIF(}1mZZVvqtF(1$07Trg8T_{Ar^757v@461jP#)Puy;d`*PI8 zTc*kT-@+AE-dlDYEN$#wJxeiLVej^kNmoxVlqu@Tf6!J?5OAzPt+0TnupmH&iRah> z3$D(JL`!4mYKS$hW4dqItN7fekKG`^E=wm#8(N{YpK zg2%fFC*CTwIPGZtX}~1=w7E}be)^6U9|e{xf0*qXn$0ViH*%cb5W}pxpvhMovC@Zv(vMz)v_eWz7g zY;Q0r-)J_iTI|8$Yjk5@%G9oHmHXvNx61LhXg6~xzi@V0)AlKly(ptC&7w8hpe=ew zOE8Cx=LQx}jq6?x;ocisyec|NE5nzbad60OsaSixYH4^?D2q!&TZ%yB@wFfKc?{;Y1K1#i&ij71zd^U;{WGKke0`)Wl>rpQr@zf>sKnZl>6%EO%<;? z*cN=bE$6}pmalDj85e$XwR4oJIYx)hNWH+LYI5+vO}Rt*0)M&He?}I}V0&_dSyv%S z*MP}=(bl&GrWb4)|Gf>qCM?iwH(|}2hOp1ZEqW*TR)21=ePJ4~gY{Eodr1I$W=31; zp4MQG=->%0+h??RbL=VI%d&Is?VYA=-YXm2_#Wf|2u1o#J1)7Xsd&PfQC;rD(GQ~K?Nm2f# z>b3&z9UN-y%6l_j9Ng=mcJ1?pnWC$U1$emk#tPV8xaiDwDB8i|dIspC8NFakSWTusH2#(tRM{_<-4CLf^$j?PVQp zt9P}eRdjQ=+%ENX^)_e?+VHO>Xu)l-6}NY@MtqbxpOny+6v38e(UvL@bJ#cGP;5+H zD_ec(gL>VzEuPY!Q&?ORT3iewtT*UM2sVA&;#D})U+|jONo!;GTu0vpOFey8Meo`p zdB!%)1e!? zH1b}Aewwk5pMg=p;7EwVEawlV#upfkEt*|duy{nUNC;~L{4o8J-K1?1#s9-X_XJ<7 zS9^H{dzHrRRG-`2dsw}9uzEgty!C9n!?bAc9k+Miz3=tmzPCo3i`WAuvj?dj39NGM z$rGL&@@-4~a6S3NljMp!TUK(q{djWlpLEwA#yMX_nhcy7g-uqiyKv`c4dbsbcg{aO zCFb%_h^y&{XS1ZuUGrw+vP$l>1FaLMvhBRkmSJ)8=GwagPuqUa@^;EjdQh6AF4$Ao zxGZg#e!9|@JEA@EcJ1?B+mn*`Vpg>6*}|3^((2)`_$}kv_~`V8M}c+`EQ!plb`O}t zvYD2=U2j`p8pIJ@<-z`qckd#Z@FE_Y;+W{)x~&d!PrPs3-@TP3x!}c~-WMOQq%fB~ zVQynP{59OG?uqxi>%P~bQ$D7+97qV%>xzt7dX!mleTTnz!BbI*M4`T?vI0!fGg41g zuaeLaH}~GLZ9?>#B(|Lb&#b%K&fM%SzAfe*NN3dlgV{A~y2w z^xsZN>4&b?{7fu)(Uy5bGUk9=><{;#juzj-t-_C&3p;0@yVdvBLgAy}t%ngh?*dj_ z6uMoX$5wFkapS+o_I0m{Tk6DIcXXN==omL2EUzI^ZX@IzMOraMpNJ2ywqNK8}7a=X&D zQZVzH>Ffvu*8W0k>*58ED-GI%6WW3|w59|^H{SdAIK`@U=X=O6p%oo#L-Z7ujId7$5{%MM*LW^@lOXOmWRq5K(q(H4EJ9g?qK8oUr(>XRul=cUMNwOJ9( zat%f;x^Yn{-uYDvl5-308}{ehh-DNlxo>>@{h!!y^YU)X`p)Fc>q#A*4XSOOY6XS% zZO67vNNR{lS-`^n{|S%S#Kjg9mxl-%M)-4IKjm^gX1d=O(I-dG&%F})=+Y7`@gyC6 ziDj$eSLH6KV`DabwpSwX-ZBfhW$P7>6-xVm4YN-=^dk51(~o>dyLWHk-W1uf>6_cr z6{Vp8eDfdAH1KM5v*Zu*V0{s|I6y$keAern8OhZ%*z*(EqIay?=XZPigy^EH$-xoD z|FZVS|DAT*DkmC8BojtX@zFefzOF}c%NtU^~ z|AuJAjVM4vMe()@2$t$D@~_uq^7*x^5W$Vj~|5r6-jv&i?6*cz9261>0$lC z3MCn*FnONy7oXL?oOArkt3w&=g$AK)xsf^>*>nWc3)Nn~{BSI4{nxpSTu&_cYwOrb zX0Vr5uvdGu-!fy(^JV*9_pMO&`u4pJg}znU>DRqyw3MuE@o>m-|4`=T@P=!nz)E+v zr|#2=?b(tF>Ym$wUmO1K+_40nz7I<*CK@JmZTkK|HX`JM|8Mbt-yi*}w_LyTYm5K+ z)Pon+s+vjvtFWDMaOEm3pB-8)^=VyQo4qb4`4TNkIF4==R(~h$n|D1gX>%Fx z>~)#D*4S2;bCjwQy^6m;LxB2AV*>&{Qt)t#O7vtPc2pM{*L~NMrk|lI=WAm|d5++H<9OO19 zv-h6T4!f{pO=ar@QT?zTC*B-touFW*c3U9nuB2o~r#*wu(jd1vn)|cLf~kP&%>U#-i@3hewr^>lB-OdK-AtB9gLh z98{J{HuyF}VoK_?$n@E@-y|owE(?oHoy+NXID13pq_*50i8E)*?fEJuqxRq6@|42! zF3xH*&txY1&AOP&a@X=g?(z$6MOB~YI&uZNTy)gQ44G$sr$X3IZ~L2v?DEwDMIMU% z4Mv=r{gW1PX>MOJse9>kyT#mk*CM`bIc;()r+Bi(je^Z*%`V?b={4PvQp9O;wxe@O zP@>b

RW?l^mj;(m8DjgPdQYCe+=_2KcQ#?4k@-&@zl+OJVUE6*6ltp;2*}tjD zeYQ&5ZY1?OiU}|%{$$}WVqjp<=lH?QS7Zl+ak`xgZkrow^krWjbk(3sbl9iGYlMv$*c*4(Na}a*49$h*45Tl(^NB3RW{a9G1Aqx($%)qR5do#wKmqVG*;8p zG%?mQw$L>(H8!?3HMX%bwKlP~wYD-fw$d{+cGZydFqaH85z01^aIw~LGuI5W(r`94 zcC|I}vN3Tsvv#qy^0Kw|GBXLWH1^U^%QaB*mX+1>VK$GF40h+s^An43({OVz^>Q+E zb+qwzvG(({_4PCj^0p0iw+!?&O!LqvkI+mHwXBFT%knd*OfkyM=CQDJ(=+o`mUnlx zc6PUQ^|N)ib`CXn3NZFe_i=IZ@pATb^$78G3iomk3vh`E_K67gaQ6(44hsqL4M~mg zN{J3hj|$HZ^Q}zr%t?;UObM^ajxI}&D9H`;4#^0QO!E!P$%svfN-57yODWGvt0~T| zEJ;t#s;Dk4E6gvAPmJy}(3x*;&>pGVmSHg=&v1E#$>cck)-;cntcbRf=&2=s%gTb< z$}?K3OWG^T`pR-vRAwv)^S+*+b}ULSd%j4_T8T|PW(#I1oLOUWY=-`V$8ukPNc{Mr zfAxymj5hxj?H-#Z`FA(vbhT7Y?WmkRsd`#l=F;hv>-r0}E-T))(*Nj8pYv-1&P~cc zzb^0j!h$cK1D@VW+x9RjtE8i>qP?tka%NFybXId`ZPSyA=IZ9DRZZPpEv?hKT4zsb zZ||BhbLz|q6J{)(*|~i7)U`AES1<0~vaD;(;^~_gPu;S5%H}1rx2~PNYwe6(o967; zICJl&-bvG!E}pk&<&tGvS1sMWdHMD&+t;nzGIQCsQ=5AZ?_Be0?v!U&mu}j5a_-tg zJC9u1d*<=kvzIU4cz)~Yr{_;!K70NB@4tUvfBgFRw}An+xrhR zu?RnstO)3rXlD}ln&YuiNzPE>XwS(Rg^Q2%#9Jiscz!xPLtn#xSxo1or_=Rf#NIa zeGhlZ-WJKaw8*0USxlEmn#lHo+giDi%h%oA9k<6}=4F?T`}>{ietZyAX5+D|`JmvJ zu_4j9heh76Mx#whqOAM5#CtvTLU{(K@N%B#%$HT)cifI=bIAvWVT_#v=ncy%r zRwXc%*NlVBO}g{jL^qY(D>K_9wq1$J)^F_y%1PW4@|M}CDrQY<&2Ihud(=Y*NR;QK+b9nv2CQ6h!cwF+3TgrCKGr6EmDoe4p zVpewBC6(LWaV>&VeR^N1t;~CC$|^VggrMZz;s>%XlZ)?4{+v_pQxuprIdSeUndw17 zr82AkxvgA&uc~O_^SI0pm4@y$iO)ip2{`Z^b51nc@p4%*XW+4RiPLsV*d@*zd0c4X z(|YkBRisGdQo3;7>$Pp|+f?+rG^XyDytDh(wWI(YudK;S%>t3+dI`vq(*Ew?-&Zut_O<9vv z=(cT*TzX20z}EA65)T~&;#4Mi%Fq1rJZmv;l}7gJRz|i(x}^(+Iw#&*mbYY=tg2?U zzU$Ha!ho{ykn+flSM}H4c1+DbxU494+I6<#laHr7JGyn=;n#`1w{MDXw~fF3ZDQs! zfj>VDba)upA~yX?mY6Fuv*`lcJAo8ovA&59g}VYRB)Xde7dD-WO-xuBb8sfQx&+O`U>8x6e)&z3wa zIBRvVP%65KS@rRT^!WH4Q)BlqwC%aF)NWUjoWQrwHvdmNFksONNLr)5{9xgc*8ANL ze@T4b!TIw|p}9)9M4Genue;~l|4r|ff7VhlSzzKjwN=`&c}2`qDkR^O-VnSLzD<~w z*{V0^SC^O2B!;P~XU|!4$#lPUIXbP%(MQ+Avj0}xcdK1BnLb}6gJ;fhZ~3=`lkIZy z!mo2$Cw42mXfXM^>@hwHHI4UObsQOF(AxlIIq|N_<{-iVc5VU~l=B zIIqqs&HVp?1rb-eFFQOiS!;Flg5$i2k!OC**kRDX;+&*$(KAeW-RX(hE)VW_`X2D~ z`y*+nq2;CSm-A^_yJcpDuh!!Un|AD0voz?5|K~laEa)if+bLYyLPzGl{buQZw=NJ zle~%u)9BgeyCgmmkQR z7j-dQ!kd+)_LmEXDN}B6%Y>F4t*r}uBPNN@-oz@qWn1<+<$oCxUnX$asjSqnnzi74 zkFw+0z0ar1?}=DnV|r%S!vzNV35|00uP&bY6F8x&S8tn^^Wk}aJ5{dv_?GSOT<5;5 zdy>`Pw8UyDE0u3yx-ZsFy{gl#;xEN%^V2FJZj!@=^JTqT7My*bK3O4ZSr$7-@Y6$~ zlM`6nSH&FE9pmST+8_DL?PhDB{#mT(jOldOz%oMeTt{d40IwNk>9OQIs^PE&Z zdCPr^NOddEbJwPwiaX7x=|63AGjsTn`2VxATK>O%-t^nj=^xX=gS=%M{>{1?lc2h# zj9aCnn8V@#BfEzK6SD>btJ{Q*+!|)Hmq#b|UHfdc{f?SioKf5i12uQS{tbP{O?!B( zPR-c6^UY4J(;oBq#2<^Fwl;-Co!4zOvuA6YD{+>)ba|eHaMqQY?{!U5?(m(bBx3*CJlvly_0OFD=z(0 zxoQ{o-sOo`TzBb`O;f6-7Ih{5)1Q$ZC++-1NqpA@SIut|n&qDyU{##Z$P@E{QL?6i zS-#-FMcD(5*4_(V-kIjW$aa8%jVFPLd4U6?nL(3Mjv{Z?4HsQAL9R`!5AsACII$ZD zFmV?wV329}Zg$VHBT8PtXO@uQEIJf6aS?7+!& zyyXw&<@E{;a&FDiKU=i(kOQ0hwU#F;Nt=H2>@rbr4ptYueLrOJ+m4Z80f}rSCGho0S}F&o;mNvOKJT@!cB+wmX(gLMwJLzmchBb*X0v zT7H17=idfarwXSxvv0KMMhS97-$;6sb;C)}DWHML=fT~M7p>N8g~CynK7~g9V$v<0 zGLPrm@52XBH_ zUUI{nf%yQVt^mgs0lo(l_#QO0?A*YvcY#5up){}}UATe4siRi#1EWwv`sMcYa<@{R z2YDhBSko79rb*O2T+TLEpzLWl_f`R}^aW)F;^l1$^=%(gJPb3oY>|r#h~)n)ocF{` z!6i`C)Th)%=byoX7?z6_s)rj&E*0GSY%4ZNFr}dJ+Lp$$2aTT|S3V6j_|L53_fa(9 zfw0?Q)_@Np%3qk%!>V;AR0}%PmZz06U#Q`AD7_J08vdd+#GEmrz4XR+#smdkFAlyV z5;ZR!cpDj*%GzqV9@Jb6FXa(n)G_E?zoTb+M5(9)qih08;&#r&?QB^a*k&{I&6>bD z>jBrZa_)r>INcl=^N%x@p5X0Vz@o)2*W4y^@}tP%g{pxM5)*>hV>f6wU$;4UNZv3Y zsdI%-$yOty&+ca)=`>Ha5N#4nIS^{4W;gXlWywd6%?l^GJ`p~AQR8r;Vag-cP3)|< zF0#Za6dU}=xpa+bt;N4uzJgX(0fzAM$>B35o1g5qs%+&-=-&IINBBc4UjXxj2`vZJ z`3`&ZobX^~N=U!(gW>x38bO9yzv?@I zc`~#wa-!%;i@wW^#wp_UiwX^&I(R)2{xX%>VFI&WX7f(<$$sXg6>bdMBD%NUm>iZ~ zy5~f9g<8wLjMB3ly*w9sHohpi_PpktdW(Dm`yr2>LyE2&Gr!yB{uwbrt;640<@79JXPF_8>T%5)V3>S7V?5 za<-5IqniUq#tycH0el|?_}?3FrA%OXY?c?bfNk9d&c`P?wk`|ERZG(w z$}*R;@VPNtJz$D>z;srO*}F>JNma%I2PVCOn12hI4L_{3XxOB1fXVSYUNN;kSbSQG@4*6If9oXy|Gax{PT&@phTUZU4?w**s{Q=*<1$@mHSduolmSnReGO$EWV0${9^KmA3>I23U zgY{1?utp1b?@ZAzijvtAB9+Hn(eguE*-gT5!Upwgff~nj&K50TZ!+4pl-at3Eh~U~ zT>@{z;xbw3Vin-@V!jfdwq7x z7BQjIi~Jl zm5bp%U~p*tgj1_>4joW9;a$LN#=v6ez;r%LVtMv1Gn-ro^^#59C6N-0;tO)imdyz` zHCroc&t9mHV~{6E9{ zBw$_AQtQtl-n!y8+k(XZN^IEcRFP`La`2M;wly1O%cu)TN-p4W=9OBORDMduhkLyP zx5}P>r&M`(vJzNA9wb~;JG?ylu#$D|)i!3M54-ekZ#oz=_13YiSHJgE9NTk6XLZ7k zJ!Vm}onEg#J?F}C=`H(S_3%w#-kHFA;5l#9o}+%&bMDRIZ3NwPewbIF_l7$Ii^1gN zzYLdV)TS5A&APy~{{dgZs)WCDLLDx!x-H;5lE8h_GH_Hx7OTK>T5XN1@g|i z?NSM7Ugn)^a(0t$c8S!iYH@+hN2{lvyD{tf^CJN~y$Kv`+bpMe|DC*__o#z(YxoT2 ziV5781!@j)9*g3s7L~60i=_Nu2*D9wmdIoUp zYT&&feb=#zcaj6+;$N&24R|jxuv#dv@HsHo2yj#|F!Cr&6%=4n3}9Kvudn_9*eBJ0>&64(~}+ab#XWMJfUZ0h|ZmZ#}8h@@W z30v?R%l@2GkD9X@YaBVeS*Q4n+<$79k}cr7!(tj+Z=eEZt>onz4L{F3N(pAu(uTuHE=*+cILdyMBLbswJn4!^g5Seh)r_H-NP z@;Rq;-ZCm*$hrFM*7E;x%H49qx^*>mbV$U&#kW5wpsPV5&*ZIzpq z>g_A09J8R}3;)FL0fj95K+R*%HbE7>v&^gMIS3X8UQ3+n|=GmDg3nkh0P zeviw_$Ew+~iUvE+2nvPiOi)a%{m>!ke2iPH=F6Vi?soopJ1Sn>s_YIwbI+#we23-a z6VnV1wR8%0yT1`%Z=76yMqxVE1Xayrf-Zc^~zA48KfySVJq-4d75HnMg| z*~>LvX5(m<%el9qv-3eiQ!@v5k@boX4-d1l^T>EiFmQBg;Wtftvchq(tGk$Kg2{vn z2bcsD`0R`(9B}yakiCr<2o(%Uj8vRqx+xX_8EQB`tuODA(|>)K$c z(iOBtP=)a^o5F$yrfnLTy}o%HQzz?gyKs|8~iU-|pL^$2?9AnmrqmA31)W?y{}Ir^nz>#~B`@R>ptNrkUDYXk-!p zQo`6{`-VYTz-H2kYrCFNgBCC*;P61=HK+*?B_Lpfp9w zlTFBe#)^flf4Np1U}TqKP~^(o`_^; zVk5WSJ71?xohue=mh0L`Gzu&f@^n&@-tF;9Lw2T@mzHeer$^oTuRa8>UT0!rv_`Q{ z=8=wEO2sB!?Tu45dFigdlYMF*gRro~X`b%qXDaG4b$Ses6$o!KajKM@;p_2_L5f{G z<^m&!mj|bcxbr@S7O76JK&57}A_10d&N&MXaysO3tZQri|KM=H?UIfp{ua4Jrv?tT z9}n7Xy$+_`%-f{d(jc|WarqRPoD8eEF?JIY8aUMw92h_HG%z#?KRUoRr(_G`hubAH zO1E)JJDp(o%(JM2#UW4i)3NB%EsA?5W-i;Q!1Pf?A)$fqLO`-GXYN7+UY_?y=V}^l zy76dQa^6c9Uej$S7BDxSG*sbind{KSEtOre&rK@-&7$L-_ZwF7X|%mr^?Remw?q!L z#EMTW+wXK`r)aIba=35u#~ZJGO>Gq4h$hBys>zjqS>M`iaO6kR*#|0Y9Dg6x9oXP1 z_V3G*=7pup1g=bI)B84|-AQ4!h>rjx?+k_xcgI6)_7|L0<^(iyJ1`0^b>J}F;Mf~9 zrAg}Uj2O+j2Myde6!_0YgqfAh;L8eWU|Z_pZsjA`8d$Pex?Y=s;efK-VZ#NCoGTg^ zCpb@J2sLcK(!ruIOMywSKu?;1E5OC+d2hKzp41J6R)-^(8Q4rRxU&qJrK}EiS{?K@ z+F{HkP!TSfRFcFy)1p~OLx9O?ONW42;Jrg#O13X$U9|!icpXt{oBB4k$0I?KyZlVk zw5=OhYhNng{Ix}e?Z?}re3b{7CH`IC^y{FK#Bq&dhNl?2bdJ2Q@9XT5%1fM?u;USj zV|OEW+P?&5^(Tv@mn1aF`6x~-&rs&~N@x;dakL74@_?=HkejT*gZT^x0~}sG46|~Z zxKq7{$>H(>wuWbF4O~2riWfKVnsOyJUgOLw5H<*9Hki=4MM5ceUfkUQbP>9Reyn>w~_;GF*P2v4R0Glx+^ zqw$AF(w#e|3(YA|ekR*md8S$Ji=3okSL*qsyN4qpP8hK$2pnX$?qK{GQNUz7$6qh# zQqajQPwaMIP_0Y9a76dr_pUG&W{CqoSPiaxSjISs(Jw`ySt_fLCz0i_Sd>B|Uy8t) zy{3#p87d)G(p~>J9QJSt3eIE`u5#e~>ad74a7LR&$qJ{Gjn5q(aOkkT6A;XGaN-KO z(#FAJ(Ckt3Zhj-{0ftG7Wu6)+Fz{9|si-*#mb)z38p3no`HrQ$Sv9M>5_5Ddye6{q zusAS0;Owt(_~iC*=0af^*<-q9hBhbn7>ej^c2z%95Tqs3&Xj)Os9Axcew(3DmHDBG zeX|a=u8vUUy50TcsL?bPU%k%BIf9WVJ=&HACUNpqY+$at>(XYtBGKS`^HI0^rQODR zPWrNy`P65B+!TG_s`0@!jl7E(I7QtYIhW3GYw&y6llg*4>A?awku-+Rlcr3}9jPl6 z{3J(oltpG+f4`+v)6)vnS7j|)KG;Ddc$w5vkJvx7_ zS;=Avr0KMFes##$_bv7Hcy=7TaYxJ;|@!gljH8_f+XR~t=jBT z7Z}(Rwu`f9cM0!nP}WmvocMLpypTN?ro5RuT{|Xl{lqS>5cwYqJPjrsG3bOl*SQyL7MH>pc|8h? z%mf^mnHc6NJvQE=81Qzxhq+t+zYgUEA}6;^UFb7WeKos)H5)_zx~Hj6EEX_?ywLBm zDireDa8rz(-&OnG98S|I2RW2E_+Ou6=*kuFapTTb@tb()gfyoW*R!uo-?~Ju_y}!H zwvmvpc`oF&t3lEG!|9;-H%x86cfDUVWu^STJ8!z8t|W1r9iAx>bT;1q$Gf^22M=+i zD40lHP~hs{;H~0zfk|z~qK$jprWvOza4<|g#O9;Hn#1MDDBy4?x>2ElgJt0jrxkxQ z-=r{U?)%Xy>z}plmY)L?%ZG+Ts_u-7wmPI&$_5ndn0+^yWv)~HZ{4;i6&99;J* z7{wb7C})c_u=X@u^qG;!%WQslhHJi%?wWrGdHCKtFxn-pGwex{p7cTBklDRG_UGla zCaS!bUc)5(Mm6f91IPVImxP$NO=+n8eDu<Cie$o z9y=KIlA1g^I2)K3K4a%wQxSg>RrXtl{cBxTrjYxBg9 zQYM-;oNpOdeN|#ojTBP5$*8NtsOh**bLA?7EBb~K%{B)lfBZYhZF@mAF=2HA$A&Nk z;e-z?*(sBCU##a;^oaN%sjR`f^OCSs#)4DXR?-|TIS%cC5{!ZjjWPkc1v@$n8P?d$ zXxx8E_pfkE&WkpW1r1yWm}Qk(1w9yCX2=AeFo~Ve;k2SbMxwR2L8rp2J##}dgMjXX zpoaa-{R|2XY%3b%4Cb~m$sVcR9@nCqvXD)hgW2)^;&*aw#-E z>pGTU!)#*FtaG4A;KC-Z4zCFY;x-;Eu^jG;FId=bwtZRYr1zrf(;+!8sh#V8*j9+x zJ-Y3*;?^Y1o7@d&-0R*Pf3~^#EKl=`-SgLGu^#@>8t}rgJ%rg;qUT?d#V%8YM(GYl zxd+bT9E~<}&f1hT%2>GG&1w{2X!KN+^$1|pWLTrQqG7ttB90XewjWLe9XPGga5Clu zBVz*7DJ|C6j5&#m8Pp2c1a@}>9%w6RXkoI@Wth;wuF?3BV;#$i1~vf(o)yf|t9v_d zxT;&*G;mD_VqjrdF^w&OPvga!5&_w}T}`#0IU;o$_!JnGAIR!dSO{@2vKK0G1o%%7 zSZmP05+l)SXDp#MLGIw2fcJ~I9A68*-6)~N*tDR^eUpZklEnP2KN^<(TCyT)U&t!0 z*}G>xV{ShCg!!i?&u>msV@sZjz-H4A`u91vMi;c!Co)NJEbA#@lss|Rw&Wk9-Hf)p z8Emo)jbExg?=E80GGNVr$jrmQl5>JB`v4>R3FhURtlSU!V?5Y16+{^W&ecCMouR|} zQm1u~AR`lpP0o!YYz9-=6Brd6CcTkpuq;-}KDkGlp~b^uaqOG6+#hYRYt9=u2{4{u zVDn)3b%4?9gjwMav62(5QXl#rbIh4gFd<|u0~g03rT{)pjmEEQrahKuFsNXOIl!v= zL4bd$VBA#;SwR!cFHLVNIh6uU7IN)#%bv6~p|NU;{&eM~tFq>wKCa?B}s#0GjxswRsQI(Ck&wVts;I@(q2@A;YEG!0y4op}*n<`@atSc!9B8vyz#v*6lD?zC zmzU*VW=BgM}6xDyy1{!YDfB36Efc1ePwV?mQ72V?O^ zj+$9fY?=(R7nnU)T*^D(TDT%IZ-J6`XVjYs3=#q@;!SIBFKT<3dn@-(T(D+igigbo z3W35TcTZfsfNS*)IVHoeN#2v(q?xuT*!_@_ z)NQ;bS-SIg(u1H)(V<+-`X?rvC`=Am+-mgl;v8nS@BoF9tA`gx?qZw7cx%U>d#>gpffzf^ei>#1<;ia`Q41JcX8Q%QuKi=4T{HH_y z4<(5ix7aHhES1ITCo}Lx$Ufod|Li-*(z=hyqsjROYgPe|g2oKB1g1HkCfE!6JXh3T zd0<}B-}^le`HtsYT>5Ltu8wQ^SD4xAEEgsx?B2@UnaQN8(5Q3acHZAJ;R{;xc-anJ zZ4%UIl3LMtB=pXy-w*gtFxov})?sZF`_a(U!N|wJ$iBdaZ-bcBslIEzH&^bpx!C0D z!0IjXh2zNq^+N}j$8T&9GCEaIB=a?6n#~2n?4KQvS1f-sp;5y?w!|U*L?N4uT!ZXx z2A&_xGC_OFZ?~0bvsdg!hJPZ2em+?R8Xm~jH=+laZ$2+Du$1XasGVNlN?Lot- z2NkzJNocrmszH$9!0kmH&V@3$Cs-YSFo{`2elnQOIKxxw1hbbzdU^+Qdfp9^)6yepvZAjIn^q{JZq@8Rs@6AHVvX9o81z0?t8o~@nfw=Y_cq3MzY;{r`) zKMuC$Bl6)z+TjtM#d{mYRy0Z^C@kb$7C9|Qy|nS_4+a5&*G{5*4;}gz#qsGEFFK&f zcO>uDvcG-YGBVcFULBCkGW^lcP}ts}wU|MWy(EL*rh-X)N5h*B9U-L?7%CX#8W`u< z_L-e(kZy3fRLK5v>OFPu_H(^9Q{N`;(P`k2H{~#}=z<7~C zO8EkluZBhRgBAsI=4lq;su{Z^C1yQ3_RpiIBuF42Myw)8{6(g~fd=_$3)t)h*&O8_{R&DN9y^%jGa5MxCWMtv zcx1uIS)0tZqe0rDB`%;R{ze=Bl{SX(ezyy|voi&*7Bg@N7;$CDiV3WZ6mfg4o5SIL z?yq!fY(U1dh0|k?PIUZcd~gbvhQfM1i}i`DS}KC8Jp7`+dvPzVYTSM_#b@y<*0{Gu zGB4Trm=$(3b*(ueYtMAgl1bU(y5NpRnSgM+&aDeGcYQqkvL~l;;m<^^bdI(J#%Bo) z{2FOrpZc4ZC0kE>w$w%Ru;QE#8K*8f6+d*keC%JM?)6C>pX3haIke|(U=#n>#Lmi= zY|$2~!SJyoDshXLlt4?|kJbu`V?Pjjm zDAdrv&CuxLz!JluX+Jw&;f3+RIXT5SO^)1wyp^4^|J#~9=yS8?t~~NWbh}o~75%Hr zF2?=;DzkK_zDJdwz^qFJj0-EPq)tq>>C~2Pc)IK{4|^}8%#Er{d8wt&d79Hd@f+Oa zwqR^q(7#3h8TWyr#cE=2#nxU@>OUmeA71l4oTZ=bg$ILX{%t8ojoH$f60OD@70EAt z9JrRuS-|L+F(=-j?KrzOL#0g06UIo_gO58HSTFRNE@1tavVej6gE^nYt)~1~{qzPQ zj|M@57MBOD1_4a67X%eoOq4M=BQrrx?ZXB4s1$*AruU6Zk>P=&u|f}%FNt@URP+1_ zJf+X}uXJ|teEo>GOX`JXC;U2J@#~>CH`}5*sTGVXmNyDKn7sCVozR3THU8f=DU8p= z`Zzpfw~6yz*e=%ipy87Fd5((TOP??tay-A2UrcRLbuc4`&6O>Ur`gWUwaCABz%XXV z1jQYBf~sw=zjcdmirgGi`oO?}fsu*X;X~oHyxZ%OkM%J=K4vg`Ta1rZh=#%gXJ$73 z2p5e54`=8^$x2xi7#v7$WU^q7nGo=Z@giRf$3v;hs-7Ym{tNza{9DZYx?e5nc;8o{ z*6v7srv)oom0G$d>?rJHU^F_y9w%p*bfDp2Q>&b91ltxf4)JtKHKX4ji2Be&ax<@r?;KF8Q}a zHhrJw`_L@!xCzr^Wj6N?E*66aCs_@nEua1gI%oDWWAZtZz!U`rR@MLo2Tp+ycL#P^ z3FGbnP6q`?7NHmcM{d)=2M(-)a~K%e6eT8c=%=eZJi_m9Bj_sR<#Ox^PPd+f3=R<$DW=7u z9up2QU-A^cTgo5uoq=gu)bzWhH#{%Ic(9xi@X%zNIla$Cpz-oNr~4X5&gvh@_;T9p z%7jVoe8x47Cs#iD(iCL*YOisxkcdgfnvJSf6GT{LB!s7I6u0o~mD!GL zEG-NidOPMM9uf1>xp-9YiPS?D5wnh|ERjh|kFtscibSf2IP6fpE>L>2b^7G$k6)So zafiy8%}DOlRb!J#vN@)jHZ`#Op#BJOWDsu#5imbss1ViR;Zz{oCKvV8I~cjdb;*?CM3Y`J{p zX#mG9w$9C3)sG4m%T;HeH|I9ov$3~bVu|$af~n@G5_>8nnse{WxGfd8aMIKZheRVS z!q`KUx_QqXl9*i}z^V1Ti&N^c=(+_5*vuw0I=VE8YgxL9#eC#Sj^U6LGiegKXF5&8 z^Q*e?$NAfk=UKUvRJg1hwWRAV3*Mg zhiLHw4!kV^%nB9%6j_8PFfjFeV_-FCnDk*X!ygF)%|)^9R$KwI7$PpPIQBG2aYgo9 z?0MGP($FlOz^vfK#?e=@Kv{|TiIN9M>BsArh z60=sUr#D~pnZr_-B@8+j@=bcFo@Vg4RprG2-)yb360<{$tZ%0I<}Tuu@N}5ffAd*y z(Uo2flY|zNi1%W(Ew0)Lj6wF>9*J!;XkKX4z+^pziKlIW3m@BshIbkZGHz6=_?er^ zPIOr?S@_42OLj@Zf<27?gmy*rYyCXFHSWxDi6cMSv^ft=$SZj)eSZT7Z!aTHKtKYo z)I?XEn1@{v7Zx?2FbLz&n{tLPZxOS~4K?B^N$~w7ais$Zoh;fuheWy2~m@x!4 z&a~S!dCr|FKc}h9eI<~~bZu>4>3MSoCW{6J$qXk2pGMxbU-`o%86TK$yV%>Fv3T0Y zgIc_?j!ayCip0*`X<&a**7G+_QS+bZtkrj|9yFS+*xDo-+5uKT3%H}3@CzEYcr7owF!cxhAM9!HXUU){bsZbMang7O7f&KckT~ZRge1jw@HIn2Cm8>6B=F!oDH6JWmWE#1>H7#7j=GJ$R$3eKsV&?-k?;HPEq_>mU)6nyk|p; zPR^rk`<}Xpbv1O@WGoPw@8HPP<>u)nw)W%Hnmr{<3YTZbv1d%%zK2<9!iO{U513AI zHgZVmb!HxUUVr3^5`dZZyjE#7}qdIUFE(G$D3lrK7|GRsoqS96LJsdKi?IrtsbI z{Kx8_Nj4j|1PI1J7I>#UJp=Jvk`H(DYB9gGoig zNyULlWx^qio=qLBWhex_g|oOPa)V4hr3H5MenaVc;m{$|&dJC>Nrt z@WxSr!%5L0SusFarH4s#f|Fj)QljhkLRT}o4>nh^2RA9$XS1fK&{HZ zLlv4X&MGcuJx=TklG!V!GvqWhmmJXA@_2K?38A3GfC>lOBt^s2gHksF*&m44Np=f1 zG>QL9ewHvpXu?9HP(FbPt?jB$CP;*JOq{{5GWQZE&%q52oPQq5WvTr7n>a&Z_SMFe zv>T3!Wz#(8F&4~g6iztkeQN%nx1BjMS3dr@Kl{v+yd4efQx3{WJ~fbO);M-pazdm3 zD+ciw4XvJzQg;q%Z*x+UU{r8$3h!f5dE%rwqe*X0uigtL(;Vgb8=pokds=fYUC*ak zujgrm8&gA7s$$MlIS!^7jE58&W*Rdxf2eGb5NXo$IczB+VD_YmTZG}WVRy5QRC9=j zSixk&e;<0;@62OQU|>BE^SQENQ9y&?+>BH$N2Ld%yo@U6p2|3y%@_P~;iyeM64PNRB@lg0{0$({rJ z0{vNgIcJrK&AM@4dgp;#8yUD74sc9yG-wbpxYJ}U(`+r^EcRoG|D=PvEE+{}95s5> zr$4<}6UA&+@YpQKS#wI0UeBRLPEIO38cJ^thW%<({Lq-dpemjtD{j&_ujhcx-?kqv z4P`0~r7Elj1(T&bq*(48kWgqcp5m_J;25-EakI~aC2!B(TQbL}w1HuYjP8;Ewu(FV zOJuss4)91e|C5q%RJze*5b7skbuis1{^^R#hFLczY+zMB8|9pKU*N#POO0VXCU?1~ zFmV5QDF5j4m7l_5wVaz=Gk>QFCaEy;u8GpP!n9Aw$#5rwK!wlj6)IUz8TcNw9oKoD zdBTBb%K@GP4`%nxmgi(P6lm6eFn3-ZtFVEiP>!Q6$1A}SM)@C%@=qA$N)Ad`FotP3 z@);cDxp9CarGev$GFym)t(Zf(V1rfADTWJg)I1xQT)1Y5_AU*$bV=lZ_K#leH@>M^ z-NsWJ1sM(qElO@a(&W|>YFIk;)Y&G3J7+uc0u0~!h^eHphUQwA3FuZZY`(^{^;U<- z2TjRK7erZ96288$Jo-;ce1l^a*UGgITV38U@SHf1eedDr+$Xk)V((t4d4&0-rA!om zktnWuQ1S|s0fVz{Mw6gH$Npb@AD=K}aOm*0yv)3F|JF;k(>({oI@0rhPJANg$hUKr z|Ex7a8EXs}nx~ffbHq4sh8$pRIO6m4NZH)TKTp;?NxbyyOJ2#Fb&qW0F39-?{RxW6 zTxh7kd^cf*k?ynCFGWoXFLPG*N0nYk`E_YUWWI?c^XZ8V>N(3Ky$*`ZINck@C-UXQ zGnFN)8za^%4PGlMxUN^Q>!8EssScbgvftdfY!@lmtEl%#X|{MnicwW7U(CVBznBb; zM4u>S;3#nWTgaCFhk^IW0mVgbhX1nia?cz%l;rU6P2ik9k#9PULLLW&85-kF8-*q` zmK=LGP1cd4#EZ$K!9zyo{H-etHA2h=z8uT?{TBGEL~=1%%-f&p?)#Rvd7bDR+r)KR ziN{SJS~Fedw3XuIaQ<hgMX{sM!QlBF7-J`Mh#mjWD1qNR(Zkp6EGnAc$ z!B;gg$XD}#|Jw~rUyAgzxtLsp$}&zB8yxtzudu!J%oMeUI;$oHgxPL3+;P}Sskv0| zp%C8r$s0_*_1>U_#ova2tjtTe4;d zo+w@!t0QI9=+t!1s`K1F1G2Ty%>UrWW2gMh( z9+>1HbcIR(LALIc#;!Vs^fUMQOBm&TI4b1K2y;55e8o{NBw6tZ1ILvE90HNtF%9e) zOokmHn!m5<<}e982;dVGk<4k7;y7%0XWE`phjU);Ofn9>o(Gsh!b^1KcU#7@vmp3GL9&tcr+c z(EN5&-*I{0{b}0%Hkof0$cwpdNtIZaS9H*|+ELuX(P*0I?2bnF>rGNQj=s_&d`}$s zo*Y!3lB)9N;NhvN3LH+6QO=4D4tyO4c#3AT>o89Fx<&iIA>9>>UOz+qrM7AZDX1@L zWcNv&IrID@L*B~^ZmNIODGxY4Yb}3C>7=qNA!Q10OG5Wj!G9a>uF}uJg9__R7Ee5^VqQoPzqg zM|_pyB0J^fJ<}&PNKIKUCBURm;FxtLf$zov;UA5Q#hf%vm{d}lHAN12TQ@6MIP0Bp z@V(5Cx21u-qiD{qLz>MYZ>}{-<}}X9XLM&}HvDo?EvJ!Pw0aMtfR$d=K-wc@}(wjKwi9w&XC zrkrSrJx9v=zbuJJ(CMF+mR?;p>BiCRY(3xiI$S=O#`(ukuDV+GPvMU1j622G<@uGQ zl{E4%JSzX2;lCxLLIabAz#+vsjY1RH_-;7x20S(4U=n_EP~uIa*%VdNCQB6`=2<%? z&g^$!_xP?mW0QJa(wn>{sVR+8Ut$e8m_24P@){q%;CH;T{M12Fs}e=uOD{yqWMWFc zZ2Iue;G)D)_9h3FhQkUu2Q+wW%y^p0uijiH&LIBaklq_6(-TiGTxXJ;uts7@qs}}g z^NK^7KF#_S&H8hi^h5S)3NY!*G;QQ(5`EKTwxp4*rS?WXgHlhEp1>gy6~|w%{w>(_ zTEU`ma$31W^dI9%B6o#j;@{r<8`}A3yi+!^vp5UoxBWsV{NV=Q*qp(x|zl$!yLc^*Kzd)Aw4pJXZH% zT+QEfx4c)MRAY4TC{$X?DXiu>OC?8$)k{P%ZUIN9#m7zoMQ1%O zMx%d6*!lYn9rh(`T;#%HQ~KtHW90KCGpynh_Wk*EiQ8vJ>^_%=pIW`o>)FnJQLxbY zIEQQ^gXFElt!}|GdIb+QDEEfUb8Pw~a_aJlnF{H>EWXS}%oa~v%E5>0WVUT!;fm2ArK;nQy)5Bw3sy+d06bzppnJ%vGw&q4b z65j+_{tSg3jBL&i>@06C?Umk<>B$nY;K70e?Mz-AZ+|U2F0d_jhtStY$Ii}Ouep3n z=BJiBcQv0Z^ZV%3oNHn3AL5fKos!zj#e1K9<|WsRu1T7H7qlJUw{!}(u+LO!F_M4e z-1%KMY>NfUiGTd^vXvc-ZQ?Fg3tNPOHYxY(e+pRHH(g=QNwpbezZR+K?df>btylM| zlY2{BR#)5Rl#WIgnG%iFZR!dyxhA=}U0fy@?A7TsA!wPVD!X8@$01JDSrLaijGQ>t zW^A8O!#XXz#cHTTd3C+^Zui@4iLvXnaKE$Yfzv9Mr*V_QM; z1jbdH7BwuH67*@tp_%3Efs9;^at9Z;$>*F<1@Qg{wtd1?!Bwlub@3Ff*;0^TS7%Ke@+He`@-J$Hte>@ccS` z=+6UY@r)e_>Edn%l?;45-3J)$Hc6&(xlCK2(4t@T!y%nrX~BUe!5)P~9w!D#W|6oP zHy-tva1?fF)g(zQKjETb^pHC*@j1xi((s5Q>PCxh&PNrSye9of1x8k`fJW{M&%9(N9AMtIW#jRfWrixs zlM}g^B~R+8O-h=?!0|4j!@Yo!C0?LKVTS|5F_~oy%om)^OA4C>0vs3`=J4h^Jz(H^ zp&;1K;JUo(Aa~rACLUJ_N5MV?2c~(4#6MWLGP(vHa9^6p<-XyN&?<*k{|shPtqq5+ z@0-jPJVVL!>Bm5!WewZZ1diB#oOr%QLTIAe4fi%`3c-*vQLFtLZ@}C;mZC*6;H*&E2Q(#K$a1db<>}@!rtgvvZ%7q1N$|o3w z6&7;ay>Q@(UD3$rvY>fcz(d}}KaPl`nY4Y-Imqnpa9GV`VP`;w60gSx2X-!l*=!*X zm=!lTaIk!7JW-HY+3Ry}rkbRY*CbEhl%fO5D=zjr87vgJvVcv+NT6}D=>ewF18k;l z4vbsd5*Rk>IB;-%SZ?rUk@&tBtx{s4+;KUMx~?0V^iMT(F=aK+IPB1>*5P@^#APy9 zs052>0AoK--UY7YjwzbQ82k!$xpTg44COq2lV!)!$#&)0>TYch=J#DmV1KGKp;2kW zd4*F?+6`86`W8((qWFp>yf3>+_74N6*90b^f-9{0|4J5i$ND&nUzu?%a`VGG?kbEP zege)B0ux(YR1OJPS!nX+B`{BzmQ=D-;_&NF3>;cZ9tx)$Fljg}6x({KjX{vXQRtfk zv)Ye`g8mskjCZ!EF&6i%>U(nLlfq#&@q|+Tkcvo=5<%8TxnqoC51NXnD%juLu##C& zgMt0ZES5!oAFb>B;HsOokyV*Nm~)XqvwDXiPvC)O>BAc~>a$q$99Lq@3i^G-*sJht zcGZKK3oaa0KO^j?`rAX{Hw|!nIRXeZEbj3eIE<27x0?iIb)JquW zc(gp0ZsTx_>MCk>v|tvTAaRuU(*=D;g+&63Co~E981kxyC@}J7OrGk%z?o%H5<7{( zF>%il53{YF#?yS{J{R1TU=mPZ?mfY#w1k1rBf~+et-zsHn&E+rg*m5VY-;P#i{)=cDf5joX->)tW5BFBGxr5%mjiV}{& zE(cotFJuVF>CWwC5oFi0Q4}(j%1R3P*65LSycAiJ~mT>&BkE;;}dp=xk6Wb7x8(>U8`StQZ0pliufUyre|Ca zojMe=elxfHXIZh8#p=QgWtZr?=Utrh)?E`Q{Q9v?r^T^u!I5Vr2Ut{J9AHat*&;M0 zp;_&RqI>&=Ha(k(wQ)7dGPf7B>6Kh)T-cDnVt(KNPo6@nOoIZOSq39l)`F&ga!VGl z_iC`pu{hh9=^o=-D&eZK!BJdAk;TRHfIv>N!{dpE&aE_C%&4+}$!G!7rv@oDM`<5N zPOpUm4hd!)2N<;!*ozdUcO|4-G1T4DTKCUoT}NAB-9j;khvHWp68j!XbT#(p_=q$) zidZpn>M71_Qxvs$_~{X&T-!sT07vP0iCI?tTxuF3%N~lHYiRN~-;$sjmZdt^<8^Ba z$GnUc>~=})E=p(ko;cQUI2HLnzM`}J5)YS+-cGGf;J_Ntz!mZ@ zm%rn9wbWt8v<9JNic8!a#g8mt)N^2x`M_Z9mMpN?Tdw(y)Zr@*3jDJa_>Zj++2Sbh zfRDxI0yFmkHm3rfKQ6DA@UF9BSm&T1Sl1}turN@HRU%ALB<7$*T7Z7XL$NkSxuiE@ zE{dW#Ya|YBlsIE38lWippz*WSdRfQ!BFhqa{^g#Wk)z;up7R*zTt2;Up>yE|Oc4!B zA07zcDeRxmGpU0itD|J~_GgPvJy@nVPl4-A0>76d=ZOXN+Y-1>IdGp^z;i}{_sj#H zZ4R8r7Ko@M@_bX65lFqV0x0r;=w5M{kI;8+Z^OLry%a-$e*LYna0q&D}l#iiMp5;o5O?XGzJzu2IiJ@wu%QV0qiU; z4O}x41j-Wsi6|{#EIXTWV+rFQ1#t#Np)U+&3J+KW7BlV&ns`i4L! z)SK3#*Wvg{Wxa6IL-wN=WzGxAbtxJLKNPxk@$uNb(hHn;NhZIPN3wn%~J*aC~~3L<%pdPVLM zd5jV*j1qYVdHNLieGW=8J&Y?@(Dm(r0D~j{7Y6z zlGVU@tRYA+QDnovg(5->Op|m)eAjYoEaVhu6!B2x(n>IqnQ*Nr*O#wE=cWqNk>?J{ zS5xMFVorImc8$ukKPt=)49pq~EK?dpb~*AVX|k+QpEfPfg?j<>ItK16M)9zRT-Opr z1^6Z69*Sl(dVWY^uMGI4w@}zZQB*BBCRkB)-ocppj#)KVKC^j<`ziANID0?A)s{_( z^VZ?b%g#LD@#C1cz*AX?W7Ze;MPJzW6hy2GIrC$snn+RNRiC^Y7tYqK?h;W_oTR{R z;KK8Uh12YyM^mGO)wG1XhaOuVaCj9-xh>=@Qs}HWEbw6g=P4!E4yVB8P2t85m}MR? z2P|MpQ{eYXEIYFJ{1gSoe@u(_MI2)^IKXY!!1jZIpY0`+ltYa~1A|?H=F&0-s}E_h z3l7;OFgP6Gu6w|Iz(gbV!K9c4D|C*q)-kZ(InMflfx}6GQ$dSmSAs~3S76$ zY6x595q02!09zvW83l=iDH3&w(w2%6If*Hc7pzuL)Z{rRq|zwDvyd~R;g6a+OMyal z=7Chr165H6YK<6}=P0=U$ro65@SpT2h6!D!EDjC6cb-SIvZv}RC>{{w*J03_B*wl& zT9qv*Vcl6nt| zu9vW@`N)4UEI)-@m+DX&lV}6 zysGk~*$N!@9`NlukiJbp_?C&p9YwiE2W1mBh`zeQp>tZwCs9D=3crjNf7T_|7zU0n z36b9p@D(}mD>(9hx>lvtz>t$5^2veg)<5Q8cSn{<57bXAP-pODKj6S#=fE#vAz0+d z!{E4tXW_*k4nhoyJZ_FchZ3X@B~C486exQrtCGn7Z{f7mU=Y8@Xi*?3@~=^bAyH=1FM*v6GW!yn_a7AUuoT>oATH#{@9~ps#ltnp zMw~j0{9hPYGBm||tON@l3Y5>@3k$z&dOb6HeG08|7#|9 zbAmF{Ca=z;-+EHZt_CRPmd~vSx65$lIJJY*tetb7L-#xfo^uT1Jr2B|GAvp3d9N(s zF=Lc)I>>S2fVi(C|FH*?)n;;T`ffd2so0N;-R*&;91z-;xI63SxqiCXpK!2ov={m z&OzA@MY#zFMdK315*pj|8HGa}MISiIKAo}n;X(PfKkb2v9giz#vpI4-56rd8wY}iX z^(C_8c*K@>%n6^}5_b0Fzw$KWKT=j2UDnK>*Rp>5Q)k7Dr|ccK6?QP}EOg|&@<1%- zps1N6_ZA0U6GdU8U1B{4IFpX;aazDy=fEWv%I@G0xc0ebk7!`(KZU^4->Qn9v3u!> zmn_sRW?ZkjPj*IM$p}VhZ4FbHE67B;Oi2frKZPWTN+{M+wMP8RB zFE-zuAy7W|PsRiGj;19Aj3OM3;`c7+GrrVg>~ZmI*x>o)=1-SRioIENkSF>Ytv&6ioH$@(Zo_hQ7KHH%$b1d?{gm$SiZTq^cjcG|ligA08 z!r8R%=8A`SZwmA@-Y#$b{ebt-*_{eJi;6if9T00^^s+k0W5t;G#)f+egTNnu9w6KX)>@PEu!h}hWR0R};1QZtuHHwrd3Wjw3aafT5@241xi^=m^ zJ(e^DmaGTY^$n}tA5=LU=$W*S^OVQZk_P@$9{gQRd2R}9avN8;Iq*E_7AR1h{$>)B zQi2QfY?qBckBVE1owF00z$n?rD7oRHL_(uj#hkX*g%VFBKEJysf9!92`=2}3`gc8} zx#lK#usgM!TihA=ne#|R{L&lYT1wlPt2p*-U~_onRoZd(hEJZsl2Z{Kt!(qkGgFhy zpR;Eyb>Gpz^Fu*=S%XIpi(X8la1^6{&O?r#27Zr+|5$!9TRqq<__gYcHCqdlR?2~h zRENMq2PP&5(OieRCu^pBO*r^BVacfuwuXd5p;;`G4rqkcvK&%icWO`x*5c`D`o}+u zNzp|?!$5LrhQjab87x6u99M$bMZPkLn7;Jn5^;LSQ<5ousf)?$gC1j(GY6mSx`GD| zNug_6xnqwyrSV7}Nj{cwagyt5;Wf#}yN)(LKOetug5u|c-e2EHuJYE=4|Obkb!CBS zaMracMye-8)Pv@zJbbCrHCfGXo=aoXmlXmU-t`=upS~QQAaAqkT?gaiqunCWI~_hg zQsug_-nYdhlj(`hluf?ZoDzjtID{2_>aL_tn$|7naQTzYMG+l|H3yqp_i`P3*)hpx zsn^Ge2OZnEh4i$duYF|o;5$3j@x+9WiY5Z$LRPr^Zt zVsBk{2}Y0Px(5%K*!5;Cm*lf}@}N~NE@v7u-=?)c8M(8V1R9u^FTKX1)Y^vp(VR5H%3dG>UCtxU;3(`jW4BRqKiaB zhoDoXVvB<_dy0w>m*S2em%4=%4;*K#{c_XY$&u^h_X&m{b)@_4MNc+O)ZQRbd}fp4 z+sz40Kc9R)-@MZ0hLF#$nxx>J)?b&f1SoQSbrDvPdg|Kcv5_NmZ_v*GQT~X`rJ-CN zwXGqMaZ^&mCi?#QaD+eCRPrf*(cgsY{H3q9wwtsDY;@7se&-OU`nDUF3N?228M3Ne zW++<0ZF1sU8lPuHV7qMGj+M>amOCCuE|Nd;-~fZ2u!94KrGpU{hw=u2CKkRkzqD6b zX%w~zDla+MDwvYv`0tQd`h)_vRZ*+CI3!beoV1iy3N1P=*1(W-M55uvg;t4}f{pCr z9YG%&D-|a=aA+*Nvgx#0bcsV_&6hxjfQ>&&3v}v_O*pc^{^OaVi8_~pUJEweRD0p? zb^gX@9<>Kyg(o%l)l6Er)57v&VuuYUtF?%#+*h_P`%Ax*dVUHqI*0xFB(Z*?jnUO> z{q9;lHQl+2{|%0CF*CX;EbS>`zM@n;$yL%_W+RJK!v`Y|kqJ|jICzbE9ysOdR42B} zmfo4OFSS+b)zR}p5fd8uIy{7$xF<{%V{E*jz|(GVSb}RutIM2+VcaGzs~8Tlhjccw zBnvbPh8&gE(^zh%q`+8SGT~pd;FV?8FMl(zIV{kS>UqGk)q~|hm17%I$pI#&h36gk znkM{F&=xzsk55P!0Rw^;MT6WX($_Md&596isw^F>qfz@1sSt87JKUeF-Dc%cenJ~@b+S3I8iU{F%kM^i|_!!4LIIK}* z$gI|pD6r>(ui(6}7DI(3@hXo)3SNyI(iaY}>dt87e$&FNpu)&ew#w)11#^2o4+VB% z35W9!oSB^!oLU!XHZVIiA2N91$R4A>%sC}M>vNC8bmbQeP7j`^*;Q=+SKY zv|t0P1cxJc%!fv)9}TSH6BxO38k+wJPB=R~LixZ+;p9I1-Bq?ro zcx1Oh)l$*PJ8H9rfY9&8wK+czJD%U+{$k#VNx^TJN~Mg%{m(6R)ytdM?bdlnTISc$ zNn)PBtrv?Q9h|7=)N$kBl;?YPIq0zKusTWbEaa9lc*s3z zftyms2WIQD57^YCSryn4xjYOOvQFK?IOo8DR;#N#9Hy&h&JI|ynZsbg6c&jFRxSr- z7m-EW3JuFxR3?~<9AdLn66mj(q1F@`tH81_Gdn|%p=s-`1x)e>m001(mo=d^u7OGXt)=rrs|_ZF|L(Edt3;^RPq^q@f53R2vrLce#2H6e zH8teFG03*e^RT*H+oZN?vrBN` z^4Bml%Vi%95xC{a{+YY?yw6JpZUG0DDu)&!tf*f)y7J#eHECkZs7Z;4gA6j$`CL zv8dJh<4J|&GfYuiB+dw~{pwd+rl-0>b^?ICAk<00q# z)r&-IRm(#UzTm4`K2=zRN7t=LIZB3Yso2&PJI@?loIlfGs$PqtU~&SJ@T3Lpfop`= zohuHB&thQnR8itzrok$w_L51lmXRl5l5b~r0Q0g-4D9lJyb+QQHvfroV5xL);Y+*F z#BHL$B$#lJJH|oznkVhUaW#eZp1w2?seFB_9gbFx2X8 zIAE9=z$}}hD7a%qYnL!%@0el)xO9)V`h)hBgTT-?@U{G#Q*6!XzU1<`nIImQYVS0 z{#~(RTC>_iMxmO5Xt_@YT)}r1u$l-sD}7tQ?%3@pw5&opmFpbq^_~TjA5`|9a%Pb7 z_D*2nDR5xX31AfLcqrhv;K>^ITwlop3$C-Ysj>^SYHSPJaj4LDO7q<+#<=#V^_i`& z8VwU3PJNlcvp~T`sp2Dd#%Y5FK8@wIYie7wlcR6jr346AbLWLFQ4&2HH6cyXU~M`KKrhqrwc=ZG$ijNYSmgC znK_}k!+s%OCr7kmQx=z#$U+Wf1;;XFM=pDxJt7SfjyZc8uJ|4EH~S~2Q+Rfl)PoBQ z4bKD%9N6>>+MGHL)-Xxq}QE z=>oITivF;#$of`QHL-(9p|6dxXQSh#zE;WApA}2hJq5JQB_64jvrlsIG!at!$g^Bk zF;qZ!`2@D$30Cq#6~PNw)_zXeWXiTVG+$*?#m{eUe~vMyA7M7wz^ot8#Qa=@eRE|d zqv5RQHu@Rje9vv%QpJ-LI2r_)k`1_G+u5QD*lr!RZ3tkuPhfv+%wfKvXoq-w*`Xl6 zR_2%qEbAs}Fg{~YFev5`U=%H2lzhN4L4jklBD1ht;^t*Wk_8FR3T3y6GyOZCXmY;9 zN0&j)Y69Z}N5%~oixd(<+!nK(TgaSQYJ5vQOz}x!=K{93f!szSkq3kPR1}hbwCApA z;5n<*>8RqrSs-uv46cI=Q7Vi=zDLUSC#F;iG={badokp5r&jQrTB|ojM}AXEY0>}p zEmd`eDC_ddfTA?FM>YYQ3WPp0SE)4fr-$f#jg4pEnDl^Ac>`iD{U|`R&Y}K;&|tyOs$9aj}`2@FM3!WU{7i=UjI?b$CXX@iPLU{f1E9c z8BJ20EmV>tCe+_h6O>t~b1XoRL8AQOG_Ioz{F*B#{ePgKA!x;YqQTJ7b=_4(g(lZl zan|?`0pZsgqs#(#E!O|#)T9~EDS;oAj$=K|0$mU6Q77k1qLQIAe%o(2<3o$VA74(@Ga7=7a zD>%t$p6+vUA5h*xnhS#nz(Lw!k-O0ZWVlo91yXfoD0yd2u+;(Ft5y z|7K3ot4s+i;9T*5XSqVx`)k5})8?$Oh+Z{AWK(f;m>P?-L!fhl=)Yt7TTgU*zHIV1 zIYs;H6rsz^I!Eg)4zQRdFnbEHRu*u0Ca^>Wa34{ae`LYbZ3W!34^;UGOl!Kpe8P?C z?hdu%PdpWlgg6+q@;zXf#K^#Tfq`!tqe%eABnJDf&wLaYurw`@=`dhZHsF}B)6A>D z;p9SgaT7L=8?53>SnhxHUc23Uw~9~ZR_OmNEc1XS%gASc#2@oQpP#`OBE z>A7nhW{WX$Z!zFr*Dzbnvi!q=*_<4Teour#9hTNjXA2T&Y%*_TH_Z<`=DLYFFn@tk z@-C&IiTeK{KBoRTG}n`}+2>^?|8(ZZl2t;Jnaw9KJ3pB6x{|}YVCn&d`KO(D_Fv#! zTfiM3cy^ zFM#=;Ia90y>!}^9tTwQof2gh0B5ROk^1`{!_(1>99pNSh%*FxC#s_A)EjH?Evh)1F z8n3`s(XcqBfNjAB?%fxX1V1FD9%p>gB7g1JM46p_zn2Sa;pD#L#J#O@_6&>ir>`cR zZxC8}Kq$;WQGX$?(L$@G8jY`nQVTB3*(%Js^JrHzo3c@ma^@>*kDGIE?+kKy!2Cqh z(9d#td+6fADrO&nf2>Ij9MKzC&;8*3a6o5!0^7L=ZdU`=Y6Gs;2`tB5n6#Z&S_Vr_ zUS+pfuvlJ!(PF~_A%}=3>Z{@nJi8t-Z&MF5nGj}mfzjaCqU;@Qq3#-WtZNiLOi?hH zwofX;<^!9t1GDCIJJSPd>kqP4G_cn+uy-bK=!&uJxWK(pDc3|rz_6{%_}Q#A8@RH) zm#kCJb)O|THz8%lgxSxsw!g4i8WP1ie?m$#Lxt#$rSg}S2AWlEDHd7tgC$3SB``5C z?4q@S!xYU2O?y^I&QdhATOi3ko0o-vXIF2qFWBIh8S=Q3`G$(LTkE`_1#8kbu+BPQIC}zT+61Gh-((>m(SbaIj5(!dxZF zyf}y^XH)}=#|2IggM)%j+kIypT=Bp% z_|(DbYOWg(w#z(9i8N_c`IB-gCwkjak)G++8#yXB|6o$vz-;%Rd*6<;b9Nt|)y-^| zaK!Y(6z80X7n#gf1*pxo(=})^%c{& zo_X3WPmq7WqAfq{PlLr^ON?*WD#x{RC+ zt=hVbOOG(jFk=vOxHI>a@mwoLo>L4wuNW#nuv)IY{8N-sL_x0e0&j|g_Pm}2X|oS- z@+?^wz`bq*XJi3Ol*5Wp+pE(JuFgFmw4y-B@5v*6h6vht14bbRhJUO9Ud#=b=AOE} zT$gvELe|y0#>;aVMG`n}A7B^RknlBen^Yz1s-3eU8F&tS;68F;>RbVycM~{03Rr@3 zUxf8?erVwQWLwVpgXdhs{V;`>p$s8Q4LD>j-EUT}_@u^Kc;V%@YiE9H%}x5%<*<|a ziplJg)xY&}BwIKe@KZcOJ%X|!+mkgN8D*N~f7(Y5PvNCYoO5k8Uklaqz!KklvzV+dLHVEaGaMn=6I zn?frJ54f)S!NR^yDQT9{QomjQu07cD<81z-omFqu?BAblcRknicjqIYb2bW|<_eyc z0>_&KSd$GnS`>_=nT^sONXI&`EndLA%z?}7fJtoW$|ZZVraFd9aS7QV!YGu`Dz@N) z*az0j0-RR13`_yV%P+F-RN#nr;A*|VBI;1Xv*FTvyJF#nOyPj{90Bi-+7>eyJToz8 z42!#Cc9(&Lfpy7-_dl<`J<5JtM1bRV0>|fr2~3GsjvnfG9-@=-fqPE@%Qgp|nF2g= z2RS|dC9t|1ylk8ILi^xHJx9ebrI&mAx#l~ZF%>*CORv$N?LpkBGoBC680xe9Qe(-v zuwJj`)dcq`Y60r&_`W#gM{I~w&uj91l=H=$A=5U1neXO9-wW}T0_^hxxR!t5^k2i8 z>cAB8;fvh%uM-(q44&H`W64%Hz;u&cKrgBx!*D`dH@`wmNJik}C8V-YogA7xI0;~cug&dpM1>)=${A@Vr%O)gY6Hrj>z|78E@Z?nB z!iHuxab2CI2hwM&^vLQIa0DtGVdsmqEV*#1@kqGx-W=JFUr$f7PCEKPapR$FUZUE$ z`kBICUrMeD{&z0mU~yP$*VO%CpQc!uW}HgY3KG!|vr#=IqFK1arBnE+=M?i9^K7B}ty14bUSN-#FogLy+`C2latJp|Si_I`~mg98Vg;k6R8{gKP8YUL zCM#8ji>^EZZUv2;f4Ckt3wXM1R1vt7vO{Yxzuf_bX0fyr3tI(ZPCR7SFgVS;--4x( zZIVq01Iz575(S1s+!Kl(G}$UNCbeLz&v3=<9vw;1r1z~VNU~0da=*~H-)0JjvY@9z z;Bo|7I!u-$~e^L&eD0P&34I+YO;twzdEKZJzf|F~dPn^p^R*B^UK~ZGW60@o<6Ej17nT9Up0=N*SJGXw4D` zOJ@>FXJl-szQL-*aFA8TLxD+9;Wn#OL8I7%hunG_9Gr3lI6Pcbn6)M}DJ&4?(7Kb) zBXy9&?b=rsk1LEqOBWnbju2qTP1JS}+~v7lt$gx{P`_wvPX}f$0|q7!hZvy=j16a| zP7w+^z$`6skZsxteXbSDLq#|mne9HTTRFjjMOLDb-7SDk?d3WSIf2Czk1rXR2(7O; zZX74$w~)KvlhG(B;E;@@u&ri_8kg4UdvpF+ET3M>mR3JUSmw-hjr%NbYbHj?|IRXC zZhdip)$oO*h}9<+)?Ow4#Sff0R1#bTt}qDvEA2SM#cbH&Sa2}-Ri^_d3)6Z5lZBk} z1r3!>2iSvZ64}fSG;l2Wus`qugV2=|&4B?T3piK~usR6bm7Q16DDh9FsDJUYCrZ~F zvTrV7w9R2)-H^GC*Y*L6ph5#H*8&%QjYRHL4`#`#=Z&(42JV6pv0+95rv*9=Sb6i@ zkT|)6QMu}?n6$XDCpEPw9%C8-?>%M2Gd@?;4#OFk&6eN@=EyyRAkEyp1?`Gaoy z4jWqqTnt%^6PkGEF_?RaOcn3sILx6}(8zH`L7?j2hI1Ud6xh^j6b0KJI0-2!a;i%( z2{kkvk#kez&=O&iZr;HWbfQ(4k)~YCcB7nl>_TK z*L55=3s_V+?pm5JXfa;lC_Kx7S;}M~o9UE9?hi3@nO6n5UR}Fxx|D#UU|xW%T9(5? z&i>Y}v#~nU(2Q&7U4!VYPg!S*ezJS^k0J3eg2j3O0@%MiK`( zO{A<-)(b!x6rim@~M5~I& zMs}M!>qROh7!_80Z5r6lt~$u$eU3%y%mP+pjhh_O8^T4`{$UjCP+(R*;lQ@(&S`0r!1Z-2O`(7fd*KI8sWAKYlQy?2q7n?Y2-d;*8Uw*@TH98A&^cBubzI}y5Y@4Xqv zZWYZ*w$Ge&u!@_}^4dq{Z6|$a^C+l2vvOp!N?2gg&Fl*# zn7CUUN+mxu*tuWu7Ti$AeDq?&OxdVtGjT^&^NP))lhXIJJC|_Hzi>?bO~sv~IUzeQ z2HzC+c%}A|?N(LZE@zo_tM9u0NEGai_<1V1f%*NRJG&q8>igxMyEntAYOmSzl{`{P z!3Hc!oJtl>l4gnB33r@cZVxyt*k#b_){`XSKJ|l`n*bAU3pu~>!28|S<9B)1KM_mr>B^kcCH&xpBEgX3N9p!Cg zNqMlzqimDdgbVMCHcY;ocA{m^T0hs7FRWSCzfwJ5{eRW1Yu74R^(|aP`2w#jzQ+~x zCwG?ryYKJS{Mdf}g=@F_r5CCC{znft zNo-lL-0H!hjZ-JEn*LFeI4sb1DZ7x{S0?dMt@RPf3k7UT!VV~j3LKO;w1HJ`f+K%C z0}J1m1?&bU+U$xSu9|RtYze%;ntq`zP@s|R0*B;ov;CK?PoGrGJH3&uVIl9y-E#$| zPyW%#rD3G%V5EDZ#mIrFEJ^5Zl=`XH4#p167a!?so!ZWOfPw$RCY}Qf0tXm)7dT4s z*k0rMXDcef*sj&rp0wC3d2g%%>#+vg8`hF4M;4zBkhJ*GWd38v9p;@*m)G5kZh64H z&ud1D=Y*Y)H}8CLd&_L6eJ^HjF`Qx|Ie}69!%oQt#+R2HRSa4T0-B6AuqG<7B?Sm2 z88rJoVD(+Vw0*W!FhlFcS4=z^3>+N{ViJr32PO(WXqHc4bKg6iaXxXJC}_XmU_!=4EM%S7@u~V6WF`*SW#u$I)v1gN31~CF%id%mfy@g2phd z#;}E|>;;V0KN`6=FbHmFcKX4{bAp-OfzeT5w`4%0#SFHjL?+S0%d$UDoA*QP$S$>l zi3jo*&dD|AoVBA&pqJcX9k!i+aRHUx6F`ZgH(t}vHbs8q|`$a}%4 z_OZ>m)s8oJIcAv{x1Dyoa=9Vbgf-u1owA6#O?A^PZO!5u<|#%iOJdyZ6LyxbVetxJ z^Xgzroxqk^vGZZ{nSc6Iw!Htevo6J1BI2;g6cdpIXMqchvK%ai5lkilR&71((E-fL z7nn>}usT_^?3>79_@LSNL$fUdlbC~xSVAMW2gk>ZsbUKlcdTsWabS#?$>>qgX0xEl z_5xeL6ef0ox#`6d6n;(P{2`{OrL(u%VZM*f+zAZa3JttBnEWHmRW7iUSG23XkiW^v zDDYvC@`~2d)q9jVB(ANHM4$VHL`F@tl@jLFdE&L{b zjz`QqW1_;SaD&<5MU(N4*03A@TEZfljW~35Z?wc;Xu0;iqDGYm%{CfMMmt!eZ?yQnXfeIP;uml& zVaNJ?imeVBERGB;h72su9xM!9Egt_ASc4>33shONX0&B4XyY$tb9^!T;g%_Z7uxhs zGIuK+W!+?bdZWWh!9}8<&1A9;u>G;FHgzfbsltAPLHWS}Ee_WD5AAUa8o$1n-?*ni zQlioDQ)_Byvx$T<_l$$w9S8MxYe`&b;F;mb)4?Gk!1&;azo^*wqRf|t zn7V>$?p&;l;n|_Z^P@r7fS;#-{q$l9Hc9ydlEz2$jo<+I{_ zPnGZfSlR59(Ij|4Cwd3#w%Di90<6Ijdb3N~f;w6>PB5xUG)hY_tA9{%lxWs_(ezWe z*})*(!=X_+fl1^A<4#eQuFI^s7uvYlT5Jr=92i<0UvQ|@NqPL^xc={j^la1cqgy?* z7qk^FV=G$LmZK4#eLy-#p*2Q>l~IQ^(vF2inZ>kVPy7#7{~Js^QM@YU-JW)mDw?QZvqFGg_40b1l91{H=d?T3m16P01aNP6n+(4LtT?JZcZ( zd`s6kwJ}fa3J84L?Ap=xJz1-;EKVJ)5+ z?9~_8XDxd-o&u6EXJ=erDkuM8zxV-;FJyDSV$;}|kuTxg9w z(Hb~`#q&gKr6G&M1xAMnElw+L=H6iAxZE6bfz@TgQauNO=9a~=3@ly0SsZ^nv=Erd z_Fq+VMPhqHzto1il0O)Jytu;>(0I(L@rbvL$jgNS=N*_TVlt~1FbOc+I(gJ5;9uBf z7dKZHMzIKeaSaWAg%+DLET#*Z_)au%MRoB9?B}*%6tQT$-uFT(<>rf{FRadG^d3tV zox$kHkZEo7lI1Ut?P(r=U(G2=D;I{Qcz3W>9bkXa_iom_UX`PnH#!*S<$ZqGU?O9% z-f!8Kqcy!61}*wGHZc5Q(UfQjpFB%dkJaS@YivWl%Y>GK8*R=Hn1n7gI}2QjxzSe1 z*kWMO60_i%?TOZ~*HRuCOtB1^%oktWKf>_US@6i`ZF_!oF1Wy%^KjaNjx{U|49-2J zM}BW)>1c>Ju9oA>#lXO&daLPd)b@l6OuihgMmHLkIyH(^FiPxbYzR!a=i02s^@6L2 z;o84r49fc3&#ld!Tsu`Q|C9BxDH$YYu8QfVt!Y| zQk?PbyXj|7vr94$G9#b9G^w}z=_}$oN%QSIO}E=7s&5meHZ}ABi^m6MhXpMgg_vapnE&uKEAqDFiZ;7&usAS;J2EitQ)IR6aQx%f#PG9$t)oHm zQ03{9oE%4XGj=fKH812|**Uv`lk4a7dM0zJ}IkvKR@b!d)wRFY4;>&+GjCDvhUQV zOtxL6H`)96ziC-3?N_u^DttD&UjBoX>F9Sxg$2F!%Xv-2e|Wunxk`4Q#qyP|hehow zSRQJq246^YkhtVru$N7~Ip{!}*sW%+eiyrhCOwNsGBdga9xypCVBET>QS1cc?IR4o zGa4iVl9?D7MJ6^#7#!hlXyDeE{&G(zXJ%cYk%^UuM=R}(|Y8^i=Ue84H7z(IVMeU zSXR_~YO442l|f6F`4)$*y1>J(p!Z;5Ap51oR*FgiCm1z47rGnkg`BWnd~UHkpFzro zfP;rxg~f%YtSC@oY~kWp@aT}4*~oHgs=9Z@f(4(N&&}hXlA!a%vqhn|j$_ue?EdC6 zyjm$Y7PCVjA%B0pKu_1b4-EXCDt0&L&$?VJFaGc@X!rx^)$D;8O6WgnR^@|&< zJ{&3$Q8%cO*Qm4M64m-w(tEh&q;tFA{5_5JAALpo3f~=h$+5{_-o9_q)y4f@+Pp7> zyi0E+wu>8FWI5tz`J-X7za3|2$iG$x*;WxgONK^v71pN$94cN34~|q-JeasZal#ZQ zU(JaokDS@$8xFYg+HVMDXXC38Sj5rj^64-u?}OF=%~GH8MpnruUmaQ0O~h^`3OGz~ zND(x+a3FPJnG(y~Qk9pw2(X&@y!pssdf7#wkwxHus*_a4jD@U1b8{BW zHaITx_?+QK5$7{%Q41y>6OvL1Z=5U`Be1euLxWAO{Ay~uW7)&b_C|+CoFZ9VT6-D} z@l-l&5)gg8_4x&9pCjoL-K3wk->`wEf#{KUmFWA6yX-ja6#YdBco5Wvz z2;OXGB6>2`q$H3-Qo3RTBb)M%hsR|X_b&S9tC?f!$jqT&@{w7fb<5Yo940ReJow}u zJv=l?A>+XjjmngO!#q|I2?yDWofuSE+GVHRWaiAc=`dqL>9okS1TVG7bU_t?2Kglm z4zO4SaVRipwz_-(&2INNOg3)N_O4PT9rv2^%JJ>nEY76vZk^A z!^d{nyz>WJeU9C7apALmxOUAwH_=vS;TVTW&8)YlZ9FREE@RguG$CLSkLJ06&EA@e z7o2F9Yj|_9Rbe8_CU*IT4~ZO-Ne)6C@-;06hd5Gr{xO~4h?SVcEq})Jy8~xvoB$(B zRNBm?hohVX3a1MtL`u#`47%pPRPWK;$b03<#)BePZcKD!7PxaS;#6ivP>kB#jKDjm z=hno}k@WuH(^yn!RkHn7m)mvq)FbFv-wX~w?S`_QVA1Bn6 z3wChGJW`m{Tk(XiE8>XCtww&vm1kA&K05sJheDfWMWULy<+PT91(WwKG++<-aY%XA zf_+LkjJ&f2T6LWo*_8L_2~E*xHCvX*6~vGv&?b?j2O5h1cS9*&m!q| z3uc821spLCoP=&xu$!`pE$7&vz@(Akz-;iLkuj!$;regcY5(LN7_?TVF-)sGz;I{^ z1DnMI&MPw<*|tn<=i)geX|9m`P2m9pSIPp8a-(KRvCMrw29qXQCx;}+2rL(JIA!3} z&BwLCS$65o^9GYI#jjvfTD9UV1D|eDTlIs>%%(!mp&A`BcOFS!g8|2zqt^GlRJpu9sM(RL)U*423LKFbmBRcbL&6kgDOlXjOuYYic$#<7NjIoydQ|9&^_i zbT!78=xkm2vStCx<%{#L_}siF^20wX|3{ZGPlZFc)6}qEqO;w9ZRm(T=j;%Bc7fu) zTc`7GCeAR9T-IC?DF(|8Qjak*WjKhiFtRWj9F#0v;gDP=E@>pD%Xj0$ zQT0^;ZH_#P&M9A*roPkrGRKqywG&sZ)M%Z?v?Q{N)&6g0^-|?kpAKx_>%XGK*lpp~ ze`{q8R&Bm@!ab|=s=dY|d&XlA^uj-Jc-kB~kaYF1jLVH{^DK6(JzcXkTKNmJ+>c)F z28SaY+XA{11r8sU3u_iLOYAnDb5P{wfwu5HO1!ZihqZSJbnAHqz73k-WZJ~Q$a`!7 zi>gi|PjrA2pOpi%euDxF`&kCzV^h~_Oi&URRWRaP^0E1D5YHK*%|Y9}F3i-iec;U} z!5}Goq1kJXB4a?;0S29!s%G;OxR(B4F%>Xu4X$w#-}NApP4B{;cU}s6O?mZ~G#tzL zka22x*D-l@Z_QPKf>T%625^MNb-wiR-`e;q_d%#EN0z&TvmBG&)|G*CvzXO8W}L~% z_WKkzIpUx3=9couOmaen&AKg*#11vITh}DE8Hwx`o2AflHAaM0Ri#BFSkuLUHL$}* z!AYQv!A0@dq{iY_1!koahqq}SVC=W%Tc)PxP6SYs4;?GBXf+^s9e z%yi+dy^WLKs-)Zzp>1mkSS+kqVCXvnTM~mh%hgRO2Ly{pU7$&o+Z?Gs# zU={3Oi3*i+E^9T`N*i`PpxC z!7O2`r_!HE1};kbPO;maY2=NVP=?h%_jdin4Y;fsWdq7Z8^a4!hxfsf#XHP_L(9|4otiv4)0tJ@LY%$YB&%d zz|LhN{_zE)Y5^mgL8H@+0~2Bx_(Gawd14q$WSAKm@*Ta7IXEhBVR&iCaQNXxhLAR9 zl>@9chlCRj?AgmUF_ht@tE9*c2R4>N5|B*&%8%F5|b95r7GN;~`|HE#}Bion~ zpy$&3V``L#0hZ>0+Ld!4$Lb8`CJaNTR1SdNJ?-emkTm5MlkU99AF4&U`W)T#h%A!ZVtTGn$QO9Mb4%(hFeLe89+7GJVb*o+T{4j%7lNU;2Ms`EZGX ze9h52GS?DCb_lry+!sF3=xQXe_5`Cs*Ab7NmUS-94KFW!-`NuOz)7iX{?En<%`T5F zos4Sv+2Xgu!MC7M)}u+d$3d*cNqdKrdI^*IjV1$*!}=>6rFxh&Uook0G=&%VsW==G zf5E_O!l=;0WP0PW90SvHmVY-nmYfq(I3(HAAn}4pPlJ&)qG6iL0r>|rm@^i0d=Pw_ z%D|n#z!B2GVBvmvD}xjx8{-dejx!9r0uCWX(R?)rI0_nHsT|b*bWp$TU?NkKL=S_K zf)fWrlO978+k$%v98H=L&Sn|RU5-v7Qx;4X@MqC*TOz~bH!<94-NTPNmwj@*b$Y1- zuL+YT&jsNJ3#FeZx@5!mBH%OcK>4U)Jq4W?w*;oNq{lH zC!Lp}mD{32{6nM41IJS<7|;Be|DQ8ecZFlhOD4@e*UgSiSv8Jq0S7p0813IV>CItW z#d6(R;*f-bwBigVjfO*J6POi58d&~C1RXr#z$)OtaiM`Dz(In+NxX$oX^oQR2S;&_ zLvkICdJ;}T39OPmOnMECdO6K}PngU;FzWR*xAq-YlsIH9(4?}5S%1Po>n}_SHO+cw zURdWG)cez{*};%Aqn+!?L3JNy3m;~A!N}}{pg9v3Chj=D$gy4Op9qVn*6TQVo?*TvaMx=_ic*-9SC{Co&Iv}GN7o#U z|8w*J=hKzGHx9}@X?m@~D4F4OhQrx_gV~^^NveiPNhgv&@bK&&r~+7aLD?}A;lGk z^$uuQemH2I)2y=Mu*Hi5);-QjHHR&pXh=SAvSc`LzB>*1H3Gm6T%*9a2&AyHkl>nz#N;-gBe0oa)PquD(N4oZqRh?g*OmM}^>Jdk+Nz}L|ztkEb}a*%_iNv@|s zuBVY>%4DGnizGA{g^w`GyqePKdyrQpG44p`o~H1`E7^&ELj0CG?Ee?SI-}u_P826Y z&Vh9dHFF(!S)B9&4$JPjG<~0=Kt#&di7RzleZSQ{QBrZX{^Hf>^Tx~Nu%63dPp?BN z9!~nbQ^i-@Up+DB{sf)UTStO)WCdO{syHwSZ#bbihe@NtY3{?BA{X_-6$5jzqr?>s@fQvfDvrD=jj|34ByKc3dey*d!nlh~k2j!^ z?@j|>03+|615Ni1@KiLgbUCo~FjiY6vtCgC738dt;*edi;GN7FhoZcLJZGP4eSH3C z^?os)1M?bQPGjI^iQ@7pTNdbWmo6~IUZ8*Xwu|q(quR!V&L-g(B1SfDW#fL z%}GtYu1&o+CG=dHa%VMx?zs(FArt^Q5SO#Zfqg~a@yd`arGkHnc^ysjPTRFMa4)&d z-qN6Rx~SOndf8LgIF4hJbUd9(W7ss<^=CiScV>-AJHYYdtw=@#8xIrHyMy6h9GFrZ z<^*tZiTDV{A2_HpE6Kr=E5`U^;7eiWk1yQ}xg$PuCKzf4G%H+q_r|FuuPaj`KAw4E zp{{QL>qTcfmi68u>x-h8l+6~&?vVPz)_Tuny`Kzga1Eop&tcvh4k97XG!&X5A`*Xc z9g=HkHB*_nb>^cTKAVD_n6$*0lw_Lhln(W@FtB+nkeI;8TGGI3a;MKEvGm3!R+DfJ zj{`cS0iD zIK*>dq1xi7vn7vBRcny+DrR$VDASlS)oo)^Oz}aFk2Urmx#t|<4tdG*!%^j5O0&YB z19Em#gO46})tex3IjG>5IP1*ARyUg0zd39tUG_=sefY(X0y=k-k3T7{%W(?t@MCZk zaX2LB(x|L_NMen;);n&MH{OaZr=Ms@c27CYv%-YqNP)0~UO9IWp9q6+R)d7cAr23R zZ!3T3`huyAqhPyW0E1HK zan3soTn&luE_lvPJy3bQ!D;GFHjmAwhrd@pUTw}E#p+S`(U!CB_P4s=!*WO7#cp=2 z;i+;fl3M$~T{l*gm36M|ljdgeZK@tSe)fho**=$5Hj#f`cz?zOo-Zr{8!p_Q>SWf# z^l#qGLz)_oMR=N)_A(~;9@a2%TCJU|z|thKq)hn54Ld0#B^RfiEefn147#rz*lP}0 zaLqe@*J#r9Da<^Qje6gUrmHOo6m;hquujV}|vZkeWRn2^v=m((mL!obSn zFjd7tFsXBn4JXSN2jvwGGB!?Z9*&wT*j4^8F4bt0oRw#B-!DqtZ9%8;v~3KlzL>wg zx3z)YvF6AniDutL#~YRY98{UnZ1ZHZ-Sk^ajFsfCyp&lZZ|wU#`MYNFb>8;J8@cCP zsAs<-e&M>h&OsrcM$w#x&Z&n)Oq}?A539>K$%!y&+BoS=VUp`%C^Ki&3vf1n;Uu?! zOW{YOO2@xO?#<^|UNcDCXgto#>BPx1b@2gqiJw+h4p(n&+ARKJjU3O-^Q$JaZ!K}( zscO0~Gb%Iq`woSUWSCQf>6jLC&X{E5q50=a5{(VY!Sh$w!v+qE~X> z={sf?(4Ef`5pCvHr6)|+uBsE{JVKoqN8$^jr4=e z*Zno~_t~Y)7C4!;Sk|VAp{0RchEb?tu9!?adr70q>?X5_W@C$@&7n`{i(7MCaZvJb z66=_sA z3f_I`{QSIsk>|4;H+N58FK=J>dqvpREKcn(yNI_pJ~FYe^GRAHJx~yOtTS;##>rK> z-G^Btcjm|bSh1OX5-Ydsw3J6Br>Cur%IfP`5Oiwhw#@1&4wpi;1XMDs9atq?ePFzBR8|E!VZPTMn_IwxsZfMI(;I! zAtoUqjGU9YxJAFs=ipRS>393JqDi>v1oKLJsmKk29-j_m&vKhFqqBEb9+#8)+&V6< z6&4Q+xE9%qEo7Nh`ma(;YjwyHFKzE14;HVsJjBq{vExJ+qbwj|k8xE~Da|)jcORqTLJl%aCgG;|!hJ<6g+LsH=6KvTU zT?7Q(4zh_ju_?BQ#~r!Q*e!4*yU*%g#E!Mw9;n<{%gEl4t+*vE0<#K52f4_cW1 zD!;Vv7kcpCEPdHVwT;?aFRINBVrg8mNN&c83(GRHADvV;vv_i0h26i1EZM40jjQI1 zhKeL>>m+PZ?pgmtDtpbQf;WZkGQ~O9S<9cU`p&NLO`?@W`4mU$jKmJfh$+4_HjmN=jcgoB8jKtv z6)Y3omMv6j43avzU}8{+!pf%q3_gmD6ZI?@cl`Ui?hngB$43Gw6Ecs;Efw)d+jQYj zMsCt!bqg~dtvC`pZnfjpOPiMNYuGz$*7B#PnyG61*PPU5l^BOLB?7`$1EWq-4!lEYK?Jt}- zjV>){>Xt6)VCq+k(%5ogk>cNdzwavaTD^KdeR7GJn8S{SRxja0Q}%lZ7ad?zWngS! zVid|~Xk-*xA>|<8;oztz&>one$tt>G_92!+?a3()xdIlP4KAM=xECxCNxR%2YL?K- zvm!z2vP6sKmut+zUfKnwFF7C99b)4@c+k1)xAWs0j{GA3TCT34Aw_N=XKbm~~3m*DPF<1DV!5yg-fww2w6A)DXBfh}!O6N`!i z>pGuCE`<$dw(N5yx7dM{ncritM_qq+u%J=?h=vj6g`rEgl^MFXR z#OwfPwKEqwqgNc{XgI)T+OX3)wU>ciNy1U6Z8@{-g$B0LH%nwcA86s}aA5Inc2UT9 z(BiBo2RyR&`&qDUC07IkvwX)(cDsT`eh2quLLK+(lmZ&1UL^3u zE?{7L6u~UFMIn%ZMSzP%<3MPr0;BFjey&T$>>-fSWnoVtAZlu{4SXgd&p`hc?<^3Y1!R4^jwhgVe zNtU4&@mO6K20?ku)NRD}dP zeycY1FFwJhw4>Nj`p3aV;tY<8#{)PV-juO5COEP7xW(MwA`-OUlcUi0l*bddOH5`< zU$RC81dHupu$tP$P}ncl$kUU_;wHc#q7=xP8N)1;zk`8Uf`O6KX2D9W%2YO^539Ks zG3=LcaO9nFfLZhk1E1pyMjoFHlOm@*5^j+=7IC+bL$oJJ>J-mH2 zV&Yub=&=1E3(uJcOr{eW9ao7la?H7OiP7iw`kYKBwLc1_{t1VK7bv_^Q(4IAGlMb2 zw4ur0A*toa4_AG+g-q)ox=Y<`cxGIwyEbFvhIIuAhaY9^;9FkPyLnI zo!&8qNh@`eRdywKn3gTt_V^RCT>Ar#8LUOxO?S8=mnU-iIj~IrT)?b3L5ZvE1Z%dt zVyEt7CBfYb+W4Ft?u(t+%v!l2NN(Pl78wIYE`iO?yh>Z=EEZgqwz<;c9M@5XkCM7f z^8Y66h|7^W&cCB!SxX*cS!#i2&wF;3Z_>@IOdIUKMfKXgSB+x$JGmjok%86!ha+#@ zg=VQgci2p49OQ{HXcFjQ;LizS44MA4MUx?^44yj zD_sdPb`pngfDVuC21w_R~~=3)iT)6PnQ)BIZ`qNFc!am?aW+MyxJ!pQL}&V;%2 z#9`s00?%hoxixOZ2TIp4&$;hV^zHIrhD#0xm%lbTJh|~Vm}Q-F%s-1n7UcyFtO6Gr z7+rj)vN--z)nbmQ<=AtOU%Z2nOGopQSimMD#RZKLa}s-{0zw`AXC#0Aw1Ckuq1kuo z2{&Fn9^bXczw*Xi4~Xro?JXdi z1Is4{LAHh5D;}_&Q{X>wg!7XEdsYJfCk4K&2K#S|)22B;&RKQ-S^Ca1&#msI+n21~ zwPwTP1&_H_9u7!TWR$qXh=YLAn&eH1Y-Mjb?;L2+J9B-R=9`8l-VG~-)-lT7cqlLT zTyugi!(`q$LILld@!fvsC-PEH#NANDJCM!Efi1&<S4F9#eKIS5=yua!~V zsidiBa!&eX&kv=eKlC10Eqcv-DscOi=ge0GFDzTFkh5&L-U9A-=`80a)Gz2~kYaFn zusWAdp<&apNw?B}M>w!P5?~EtV5@qdBBa6i?E|L|(|#64=5Gm7YHC7j9txD}zBb$B zRN(agS=WEDM4<%>WiK$kRZtiHcXgfof=AJN6xdH?zT5QnaZusWkd11-8yO$>vAS%G zTX29+PLXd0gIZamxI-ehL!)?HZj0I?ku{F_8jOM=+>(hC_}LUCx-anmQ{cDr5^!56 zUZE(+aZtF*QT&qv=a&a;MGFL?92lN0e>h{cS>0i?bAgXD`hUFJWH#@EUUt?lD^3=t zKGr{Lc4cK*h#XZoaq)%RDn=%Sl{b>>uQ{)LkS+gL`C`#xwq5$>4f$*d3~c`b6fd{! zyO*H!f@A-gHYLS{f*Op{RSd7&60W~n@07t5mDg0TW-0%l1wxAsO5HjrWEl9iF8J*N z20fVoQAfVpIYlD*MwT!0)Ni}9{rhB@{NNw^D}~CO1KBGS#hD)R6g0kKP~@AlaK^eL zA`A!F+!k<8TF>mVfO&;OuZsh7ngg?3F54^ypAtpCa||L?j3VD2uuV!h`$mEP#KwRu zXQpKakAE0FHhO23#&02YMaKBtvL)HO61bRixPIJP?zFc(V@JEamI4poLusZ)hJVW$ zXX*c9SrPb6(BU7ua>WA{9ZuGi2G+C#3jPkPZ3+BRi7&Ml@F_6ztq~N`a+F@>BAw7H zvP_Y$f?-z7XH(r2Gfwwbvlc-$MJWwIhPMahgM#E6f-QtMyt8v)%g9m>74*6K{N1sV zqX(u~E>!lpT_PLJz`j61;1dJOyoSgxYyuvN%|8Dg3cGl4@o)(pS+J>yaa+a#<`M;g zDo3GR2|Pa(1jN>HG9Bb(XcS;LDCy29u&;qFpqI_*faxMNORglYpO*sWO<30OSmyLb z?vo$&(hti_%{I&M+JA3(HNz6iDlMid4m=k;7#6uPd5G|eu3Wi^ZJ!bY+ouOB;uP38 zdRY?|u+I6)y5$4AhN85N7UPTsf-HQJij2z2N5MFF`tR6CgEt&W6Oh6d{P^jQ$!MG2fLMjdgq`JAG@HotVs5D`?3l? z-wX$S8#Y4)M;6bFyqp6ciyVbY9?DvMlRd#G&8Nu8kSP7{#6p>4i85AULU{@TGF}T^ z8n})*a5^ktYSUupco~+i&oYboaYOUB7e?PZ*2}z~^zF-{W#4+ft@yCxUZJy~Yw(oy z3|tEsg%}tG9)!RD$bQcuV)gs*pa*>0*0XLq9~rF>#rc*s;sI;L0e-ngcA*C+6%_fF zDe$KRI(=)9Qd_3^SwXSV!Lzb~Jzwplgk4(1s?BhEiHbI;yw!nw=EIzXcQ=T zD7z!^+P)%@o(ArUhuYbV;@c9Xl^of*81uUd1f-5^EpcF&;wGf<@E?0w**A_wOP+nQ zD$Cb1)eFn*%INIZ%j{Zaln|UJ8f>H+sIb8D%yfniPm+xu$jLAG$+Un;PEp9?vL6a+soUt?RC z{lQrvqk$=nL1^Crz6B4M7L+qIDerpm#-?JM<4MDy9y8`2lgzd$>#bH+)cR)#%UJyqBKNM(`5V64;xo&u*DO^H z`5Iwi2a2V(G$vOq7yPk}{l~Q?M)TOYS3Ij96`VaSBlf*eL7hR>-233mUGstyGqrs7 zUbjsBF}*pm?XZc#-g4RZ90D;17`Yz2=t$C+`e12PV9xTdvka{a9sO+(9YnoLRS6;^BjQCd>i&o1RR@eQy5%A%L<&Nhe8ONBM z6gU^@FtarT9#Y^um%wxE0Z*Hzl%nExy|+W9ApFm;#Z~&yVlIjz+Zvcm6u7t!iU=_B z`Y4Ln927QbJm@$I7WxR}H8ANYa4|VnPcXR1v$RI)irJG- zwJcV>s|^${S?sd9a!J7TD8Fi!&{BnK%)O2b{2q+;^VT!WJ7;(FlW>ir^b0pm71aib z2P|(I*drdW#7VFUiLuU^%9^#ny;&hQ;%MTtG69_r!Uu!}%iINoF14(YE?D^TkVN6S zoXfH0#j<`vs*V3Xo3|>ko!hpzaVN_snYN!cYkDo&8V>O7b6|;iF2>X*(RPs8^`Nj# z0&`x1fE%M=lC+4#9pNLB1PtaL%opT2?7*Xt$j{X%Uf?Kn%z=d=QACG}`GbQ%%rUlz zh8-Du%(iu#6roD`L`{vJH_1?tqHU^ygYI)=m!WW!pzU=!l+Q*)c?UsW#hSGxf{i{9T51U(8I7$+wtI`Vg-JM zgPeH|>y#g|=Ny=)``}pcoxbeaovWPt#7nB%#j_HA+h{fz-!Qh~~f>nRteoobAnUjNRXZ<6nd1Wq4E@dw}heH)Y3Ci3^3 z+V#BlUiUAxEwc-LB?vi0w4AQ?{^}_IYvG5l2hV-9IrV+c1Ihmn#HX?H$nA@dDBgIh zI+lUsNCHm}18*H;cgk^Dj>V5ICNR4^;Ng2Hkn&Mjz)^?&;idTd!gh)x6$d#y7Wi;I zbmU-Ulg;9>aT7AikZMxrPiZg@I`!{ijFJIgdlC*VV(I2L&92*!Ia%%OJd@NPDT>a=*cUtX*G265vW#1` zdYg`8vG$DJ6&#aPa8_^w~syX6K%-Yuy*X=+t~t zJR-)Tu(71&GOvP1hk)auWNrnmfDHx;2U^wQXGE+pPz#U~6y3)mB%DlR-aXG9V?t=~=&<`#_kF}cn8XUk<@b6wGcK}LTjuq$}a zP~=iyo3p7+YTb=Xr*vmGn6j!Y<}u=kTCrs6H1*s{p;MaCA^{D2$9f$aCn}~qU{R_3 zqI!b2-r%AG>vh94$N z3rJw*tv(arz);Ge)FNxo;n1uwDZznRr7q+VJBzr&hlaWQd)OSA1VScArSP3eVCNG( zB5+(-bIycDg|?1^-1<8LHnVdq^J(@_VLvNy(b*-yhok@ z_eH4r%%{U{JjQb!f?2d9GJ@N*3nu6W$g-J=1X;A6p2B1HW}~~HcgvxZYOl*Kc^VuN z*xbpkWRuy|m#+BeWdFPc3&phZa~&HQlv$WK_{0Pp8rc*)8j}^9T!ctO~0yKM5 z6b>+}D>NKn@`8a$uE}Mg3!jxr zA+vC!!=ocSfi8)u%#tw&9GIu3dPQf)72Vu8O*Y(ZcIuVrYtN=m@TxAFz3k{VBbMH$ z7hL2d1PTr+*Dy?QU}fSEXjN!62<*~a)+6+kcLrO56YrbPA6P`1{wc7kaHKZNJ!ur- zU~w1cXlOiWlgK8X;h@;?fz{&$ll$cp$K?MNu=Cw}$S0lH%+VfTF7&M-Pu(ZlL(Op# z(?XZXzZ^1~+02C($nZI z)xe^Vg+HZ%^PQxN%(+eL^@I+^XMJ?hwJBtCd7;4HbmQ<521Pb?1I8rd1rBUc2Uz|I zE->V)U`Wb=FIr6=CEz0$=1%Z$x;uR)L0gAGD|S1 zNKI^za!F*u~7c8Mvg1bqsc6qX4ZW~V$BoLq54S>|Jht4Wez_l*Z^?=Q6T-+Rcbz%xN! zA&__P&JV5i(Gz&SE3(LQ995URlxUN^NcTA7MCZGUI{9ZBE#iN%fV1wyL3Xzdti~;m z#O8P$?ONr~%9U||vsvP(;p*PK4eU)TYBQC@&yhL_8TF~)*hVn#lzU+y3c#7 z%daDHH)&tyNJua>>GjQ&NsztSaGp^igOOLup@EU(0sn-G7G9|f;z0(E3_%tBX3Hit z@FW~yn=qqIGbEs~`9~vX-3o91q5~|q-Y(=$7GV7J>=(P%f(1;i6)jxTEVpIOahAF| z!Oi4f*q)?xjl)OJeOMhoRWUi$#aZf_g}Z0Wi~gx99&^v$a+hrrXkH|6)X9=F_vxFJ zYlTiYM2l}qeCn_zW!Vk}j=BxayvG>0a()~VNta;J_i5}6eDg4UnFEV!`RV9;LQ1-O zL|t{aP2jL`VHC=dIV^I*fbIK>?VMsN($fwdY2ZlV3YuV)QqAuhykK9OZ|F7@4S^GyY(f?ZhKRNpZ!%eC(62ldy+CM ztdCwfS7?3XpJM-sRY&Bxn2a{RQ(%!1`8n5~QfqB2>#G}CAN0v4yaMqn*=Eowr=xxjI&A?H#V-^~O*%QRl06XjwRjn@+zpBL~w-p1E( z++5>gGUumc-a`t8BF^;*?9LDJj2ZJkwpm!YR8BT;{@Rv5b+gEVuZ)sK&GG_FmJ`_P zE(makbIljX?YO|PtcAnE$o|Knw12ywM&~keB|C6@TpXm-!0dH^RVjfn@u4`oVXWl` zR@)1)6IhDd+A^*h)*2sozve7$)gEi&8vJIf#NDrP;vX2NwlK3CD5*KYP<4c1Mi3+8 z0>&R(IT|gJ*cb9%aqyYW#MpF!W2FeY76WJg1cv;E`d$N$N&}AG31#OlG^8Kqvufj$ zzQMO-N8^hGzUK?Ngm*Nq@KE^lfxl)l|2KmQyKl*D&)I7fShOQjToNnWk}IdOSK3Sz zKH4EHa>)5|ld#Z&X7`WHO%BWo2UsUA@H+B^gM*PX=mY!w0)fQ}9LWV7EeE21FH*K@ zVv8zdi!ER~Cdd&Wz&dFG_iY8vqzR0b0{=KIgsPeaSStdU#e^L-l4aZ4RF_PQo!%&U z$=LJF)ZpeT3}&UmQx;0y_%78`DCKaSK|Xl6K zj44)TUCU1Ju{>&2$lzD%P*9%Xr#Dey+Jsam29|eMD(oIQPyJlc-|oEFwDPBz|G6KG z6OJ=_KV-6NU~e^G%9CU*{J<7sz~p7X=I6i~vVe&p-E6)AYo!5enAX$v+2X21)6e(}Hh3jZI*RTl%tABr`Ve zQgE@8$5q3LybcUH3z)m6d+Hc4n>q+8t*m>&&d|!%q1>rCa}vX{3A_d~OLQtbZwizZ zCU9+k#-sItC3Px$;e@i88m#^XW_9dS-x%aQm^sDm=M;a=sjoNiZM2xWHDYS3d6T^w z=d=k;v7OBC!bCo>MNOWrIN|#I*`}4+7kZSN6fG2|Nf;>!DKKY+a#${?QJi4uTEG&z zh0*l^qf~)QGP6nZhncOFUVaHPf(_Ub4cG#iIOa;&v?p*hEMQldz^Gur?4!WiB*5;o zKtrI#gg0`Q#$&0bDZ#mm+$WaJx-aVgxYgJ`s{Mg-@V^QU7QqJQ>Hy9O4PiD7lj>fS z@E)jRRAA(IAfr2zA&os-bc5RKV8-SJURnoOzAfinCXf@BVCFc}OHOR*^9g*fC-A*V zSpHzaXLulU7F=^OYQjN zV0KU2IPz}mcE8UJkdCleiCO_=vuU}|9H#+NIXi>;ow zDT6=Qv-_Rpil$lIpE5biCTxn^u*u=VrruX8ess?FVpLkXv`1z`^WP=S=D!(b8kp@B zIJ_Jft7fvXr*h9x;Mz8U(>;Ji`2kCB0oQ~DOv{oILb4N{w=#xR$4Le-R~v9GG2mLj zz~Q@qy@`R-jG@J+VNGdD#-tKe(N4pgtZFmAwO?zp01g)US~;oDT87|d*&ksUv)wvgfz2nXAE} z+Q1;tz`*-p=eKm;`U}jL4hWoc;JtPrMB~BIa|yhf0y$a%%Uou7g-vDb-;nc9GFVtYuSAh$Ul|P#uv8VWxou#b{ek<40DsjRzQ_i)g#m1G4>p`D;4;Z#HdUx;NMM}R zxc_kDR`Crp8x}0;{GcT{fw5=`8rW*?Z%93+=Ll&F-pHGDeZ0O!FQ)7SEOPJHMd zY~?XqC_z25LYl#yH{O7u+JaGGPo0Kk!?_RaZ*OyGc*=4pu=HM#GKYirkoOxTdcgyYI+p~Lj&0+I~ zu75%yl?or`2z`IRzte%MKY^n|sz6hktyq99PJqqnz$S+cE8XJ;}SZ&>q7oYh~zyX2CZ*yn9>hb78pXWrhmc4C=r)t1E~b7tN6UVOuy zL1+SV=>hhq&nL1ZWbt)sauhJ_5X+Igz;Qi*$!)3xzW~GfjZ2rdO`4uGhy8(!fJ2$* zWTrblti2Q1OX=&Q~^zZ*%nW_YS+orO)krz5ET&rr zQAY8CQ?Ji6{5!kH;ldu>*Ly4qE^x`-vPzi0YSwoeCHfk#W%1|`Ea(bm#?n& zVfm};lJBmUdM%eceeR7;x4iYyH<|o96E^)?(_J$mwJL#KSB|ZWf$cQ+v9sA1=Kj4o z|LlqdZ>QQVVCL#&wsLU%m$8M#U4hHvH0Rt6Y_k)%_ZM()OJE6`z|)`=De*e`Ej*S%oPA|YXTp<{U=yZr)_$_q?dHtUrmHQ5#%O3w{Zj$vXi zV5qBLNQss8FNpA1;c5`T$o}Q*%?Yn=HgrA7d|3bXm89Ga$>02eKCidQJqqT$IjLYp z^@2y5C)jo8Z7O4U9GAeJaE3MU4C~yxo4T_epMJN=F^;`#L9h1(mpKhj=5F8&YS|Z( z5WUfe<527_r2rO%1t+Gh(bTKmKaJ-fli3H>pbIRP47(EVF$XHt_-=SMJxSA5)MHi8 z0ZXoX9&@ieDBo_f)yB1UDbu_qw-2(0@iI;PAh-W5k8J?s!JIfo28QR%%oC#@_^WHW zxysJalk19R@?>V@dbez;0f*KEhUy6kcM}euo5EW&{nfgN&u0EzYwM=eteR)@pI=Uc z|NEZTn)@C_|6%_#XV2;EH)mGgJavwBj_xtmzi;N={StUVJlKHAyWp)((c6OoJckn4 zTnm^l?&NMxxNQ7kA5ZW}-=h6fE-n(ipfauIn}Pt7Zb6)QfZkFD&Itk>`3B5M?FTGH zpS!xMEBJWaD1Tm8y|$81ZM~1|hBd(p1ex><;ywOlaIG}qZJltq=>}8f2aeVNj++X+ zx(iAsZOW?aU|?rpJU@|vy?{w+&5jF1oj|6!l<2F9t`?>^ivVrXPM%*rF_ z6v8p_pkpf+v+#|EDeeLbBo*{bb}%etV(F7Hcgna>n9Mv$)pydA2MZcoCi5z}c)Sc+ z>dk3jY3C7nkdedrTG2y|5Dov8nU}o|HcVh}74dNMoGMhJ_|LJKMM%NsK-biRjm)Y7 zDjJ2R+Sj;)SS^DC6&tzI7i&FfRMDMnl6m=+SZUk#7hCq!ehu4grfcvZyOle7bK1E( zJCD9Sx7B%i_mm|rZ90*U9v}Blv#xv7anV@)jEqk4o;mXlv!2M*U){Rr8&fpLgzFAF zR&0}H&65*U>QT6O#4|+OZ}xNQy_a#Kx${iHZ4gQlqp8@!rkMHDfx zTPrU|?2T5un|;L}7*_E2G_$B=*{$y zdGnbc51jd}fBZPZXK;q+v5B5v@mf>8mWt127Wv(mw_V#cVTFL;5{bu?{cS#~ObNJf zBdJA`OVd?4%WGy*|5qKI+fR2D0H^W`u9yk zv-SShmNkYRjqH38lIc?d+k{$p{eC30&y74bGfQQf&m?ypZkEkwEtG?Otk0O7<-6u5 zyvWr2p-rWkwRhavZky#N4vXY$xp-70zUm?CbVs*~$0T#B&Mg-3Ske&L6{_g?MEKjW z6Azf)e^PK2aP%oWB;xU>BUvNZ%aUb+&8r2CiW%uerK^582u*Wjl>Ec5S}d-!Qn~L7 zv#0$Wl>=R@@&N)3t-+l}T(d2NE_~?qs5-G=id&A#9R^`Zjm8h6S7p6*P6VsSuxd^@ zZN)EY)UjA7Xqn(HArA$^y^}nY7P1I=MWp{N^-5b_`>if}zHMU5^uV;49m?w>guHcL z3;jED_Sl~v$p!~F4);w6Bm58`_=Un<8zn4WMtXq%=zd#`$P51R?FBGA9p@? zWXo*f;&EAVpwrQ-G5eB7&IyHPNts-iOm}Zd7SSq!cE^ClLDeUig|93UVm$Gj$tux< zB}Y`~*8~-ZKM(txKD2FTfAQQwcBj8@(NUd5PHi@w6$e=7UFvWXD3I#)Xyo*{(Co(1 zWt~r55*rzWpM@@-H&cn#KjOT}9|aDvu#JNKJ02-- z;Na9<^sZz@aO^8BLpP&o8--4;IC4l}8n>5+lTh1%h>vD%HWPYv%wiOLvs03#6+Ue2 zziuok@P^?ZXV#xH()P|OO8hfDf_)VEIy@3&XBbRz^hnGq)nJvATCm?nfU_~Ix&p z%x5s}ulk@H%X5xJYUM%Uc^}wBk38h%bZB^-aAgsT%mG&&qogpsTv;WG>CpQ&+ z7e@-8P79uGe<;(mo<}8b&2v_3mPG>dJ?@|JYwUILOpQzm>1 zXyjR-!1uYSMPb>4dH>uToLHL@rf>vs9^~kFJ5~JR31&MbN2Y~G9Jrh=crmFo3Pl`M z;n^YTUUpPy<(?-@+b2C>P5aQ|E}+UK5O7f5Zz)g-@K_ztUe2ly!e*TZt>?Kf8~zD@=Dr~$HT&mX9*rrdhxQ|P3KTo z*c=ySr{Cx6{F-|6YLbr~NZIpbZl`H}7H_V`DkJ+}fu`H%X2=&iG)o&e2Aw(SW;Vxg ztN)vYg3C2n6-*X1`hC4p9Ma>#zvx4U?U@Hm8UY7|*Cnq~kKqcMd*xVJ5a(Jhmj_Iy zGa8u#8k)pn!kxcVHZVttC?x)KeZaEiN>^uv>C~vY23Ea8Tn}p^MR{Wx*i=qB$j)nE zVo_4ssg=*xV9~&0I;DYOPq9V_?Fe69Yw<7t&XHt!dEw|ZwH1RqaFsmO(mHKt-&b3Di8l`Len3XO# zu$pX$6a8R0Rg}X&(&0dhEyqi@4uK;kJ{LKH1Cpg~b-0K~IW%l?IlvaY$5|$;pxx$+ zf(u6m15?zI29^oG7{xmnnFItF!q)$by2=<)%8=E-ysE@_Hiv{m=gLnEJUos~5&|u< z83nuCpDY!*){&&fsm_*@&@A&`qB&?16T6p3vrw5LSL~+;i~4$Qxac??;JmV^>HNwB zX7vC^Er2Ec%ZdI4llS98>Q5XgK|6zr~M6KBI(YjTwieTF*MJRH$rzdQXzs zSb&l3o6kFG166(r4-by011#wlR{Hs6Y-Zm+ea>WIVwT|SWL|XOrL9e3SH6cA6NAE6 z27?8!7#SRx;}lpJ)}Y0YDqys+VX|5w z6eLo(kUMtDqDEx~7bT&_ZcT>;M+`c;O;TdGj`}N0PdISE?bCOMJ)(w;YCl>`FNKS! zeOX|8Im3xNZbFmzfdqEL83$ih3N)90Zs4%_aBxq%K#Tf@hsU1cWj))ZgVFjz?~4o;`vqMbFRa))OoNyik~;r2Zs4|N2w-4H*u9#YK~bQEu+&5|AsVk;Wm z6_X?{V*Qc+;%ysCzoVuKkj->;XpCGj^{Q z+wm>16Dx3$Y&hb%WS{l~MkR&DH-{};*D#7}*b8i6eB06CrorOa&@35YEI48RjAXrM zNygt6Gjek@M(t^I>R?gLuxvcQz;%Iv$APb-p+%~o;l#h{#e+rn)2!#P&rh`WcK z!B2@7KR3B8*(dO$Vcs8x84nmZIlTBf82BDE%ui`t62s`d#7;cIMSKCH)QLuw1g2Gc z7zGzFscm5Vu!BK>fpKL|lg17Kn-k1h8yM9km2La$>(i{XWapNUNx0?ZBy z&BhF^jz602&TgEyi*esdf#y;M_Sh3_ zu?0=)FSf{8Fp93UVp-vHdd;rgpAK+N@HtsBXX|DkF@}w&Wvs;~uo!f*I8SI`Sm0>0 zqgU?%v+ROKrw(S-OKgQ2ExZz3aw3d|0nAPetp*zUvnEMtd9;_DXe*as{TeXG>*G4N z9Sste;v6R!j+|`pERs)9ZjIT{%x}SXNxM;e#uZOd|Ia%95)ID2PeUw2{J*p?eoHwo zJEKi|X3wuJEsq5yZyngdwshY-okorXM!`!9SR)zuc3q7LW#HvHHLs*m{s)uAKPhGh zixw9Jr)1j}XNR>e8BGgrHc!3Ox-hWGu%RVzLW{`VHmQt2&(3SjKN|E-w5BPy+B9&p zu40h7(5l1A;>^+CWFejNO0;OBHnRakXfng`gW;?Pu2=0{UG2JCvV=kJLZixsW)%$v zb_E8ej3z;aMiqr-K?X+0A53Z4W)3$R3=LYP6&iRfm>mL`V;ESt7j&`RVBT-SbzHbb zibF`#gIjb$lg0@KMgjMdi-(qcn8tE}!6SoZn^23#g;sqIf3XEuJYpOB=3JQ|vv*19 zVcCkqex`eu-)$7}@V|V!v10c52bGIYf8a5k!Q64U@$VA5NY;QE8UclCLggPRa^492 zw{>l3d}o7HMw3S&lhX~>Pd{1nUbM!)Xw7}VR?5LHmC@8WL&ZaaMO}kQ-J`|%0IPFH zi_3-vLk$+Y2_ha0jmcLUolmp|DO{|sY~tO}ASl4%;=ywF&*pb8IQA6E?4A@ZS~6$7 zvN-dP)pEBE@V=SKUBJNL!DQ~$;&7sYNq~Xj1oPdgEVdkt+#D@d8yNXzSseu$)g)Ne z5)K#yG#fKSa!N4p7EY1=!K8Me*?b1$Jmy0zHy9Wn-1#wqA)kwzWx>LIKW1v4V2HWG zYJH$Vct*(km_{LwM$ri&{UwJd%Gmn#9$wn(ymD%k<&`UP6}N&^UAhBzXjcnApS(|E zLgT+F{~97&IX9gR<=k;K?%-^>SBji78W=ee;|m$ziMAMfu;e{x+x?eS?B4DC0Ct5F zjCMDe1p=5HI9OD7FsgIxaS>qF%U}|C(bPESn%;(3t{)95FPPLXBxdzCyNI&n+-Qwu zXf^a;QZHyq-osGt$sn&aJ?OP4%ZG*6L?2&2tH#W6P>7L%fq~JkfyMd7ahD8V>j_K_ z0xY%y%+3ledIHRj6-~XO3|t9K?^RnBR!q=;z-%NDD-yEGUW0YprA~%|2C0UYoCFia znQR3PEGmE0@|zeWI+&TdnB5=U44lB`uwlJ$z*Auk|9^W!CSAF5N%+BvuOTZ+qkN}2 zXWd{8z8bss-(TiGIlO(d?Ib_wGl*VDJlI*dGn6f&A@*oztZ3r4WCqTRMzsgcjta~! zJEF^fu;%4G%>B`t8_;h2fXU9FCGw7!;EqP62ycN60bD=qV?U~*t+F@ABwal>QxAB`+0I1bHB3zT-}c{BC&k#PQh8-yg2dCpj^ImOs`wXso_ zQGG+BdI2%B`ti zLKvB+IkWm%Jlg2_@A*q*SC-rJzML0&z#wKA&H6B*=%FIN+H^*b=&h`b!m2GIW$i^T z(!`&&IGeyf4W4rgo}pUjSg>x7|HX_gybR%y5tGC$nO{&tN$p*eIv*MDXEMk(~^31#DkVF=z;}h@N0RvzBFXY-89vkAR_9&uEVC4;T+*b5<~# zAG1(9C>IKf}FBb6ayH}_BY4@pbaze@SL7#TJmFjyi>2WZN2XO2E&7I`eB)Fo% zTA+oC!+TjrlUPTKn?S=V;p1@+yTlrsf_^k+Rx5aEJDS&PiF3@b zx)L~N-mY_Y=UY7%92Vbk|F^9Fq&cs|1#X##IREP_S+^`|m9PIw)`B&yi$nGrnN3XW zxqW`gm-QSSS64N4?v@k}-mlFhF_C>jqd+Np-UHUa2dwcDzZSNa#_zaYsK6e>(JXzA zk$nfl=Y*W51%=Z3-R!I0m0#p!+tbjS+9A!r`Y*sE%rKzCQe8aikbcp}B>ry;x9!)~ zud7zQ?#3q}=dHnFUC_vop`a(&e8;DWC1Z`)jOUw@m_!bUGX`V?I#2xI#vm7vmCnF^ z@;sYn155UTCLWLFLA%^4awdOiVE^mgodtX5J>x8B)ZZ{O zSfDj{LtWs7e{cC)g9EDK3|b6cFgX-7@oZpN&e-^4bH}pI{f!=qYp?%$6~@W_U}3Ar z!Yyacs%>BpGntUG>@^=72OAe3ht3Lz1x?L7dKQ)$5FmWt7YE{yCN|Wp5%jIv?xaVwA8Ums|O0v|LX4mb(+1 z^dfg#mpwh@>Lv8RrJpDA)8XTM68dos948x@-Fd`yQfht#G`V(i2zq&RC_FUIxF9IW zlVC8JQz3V`OJ~&HS*EW8HdH?3a%Kydr>0!g`2<&NO zW*1uITps6mgojIPo3+G;1C8kyym|F>GVB&Pv3m2{IM_2dv@rg=xH!JzeG&_=fU-`? z59XEUgoEB}IWB+BhuOiG)1|;6P(bB^1Mfr*A$EnrCxs8yuiXgliZVy(UwJCVW4&U*^tkegSFgv{ zdm7DfulbSKF56Ye$Rg11`mBjZXTyiaw3(S|j4TUkf85Lv^h}V-+BfG@RQ9@!-=1kM zn{mijY3G}w4hgf4k0&%HJZW@fYYhv?U~7q+;K+Jd^_Q!{eoyv!%sOWZ7#T&MusgA` zoC#oNV>Y?KFmryd`7M@Ar2_|6J+=JOP{l5AM1+~|%BuzjP8NZN70eg@EpS)8G$qJW zbIOJ%KAM*!9-rZKwVl9F{ZzG~N#T-0_5+ife$FE87xcca3076O(5!IIg4t7f=?aG9 zq0e@_e%JHyQpIDXt}lk|0?uDvB=_4)c$&}e`c9xAE=Ba^^vMmpX`YESGGR*vBJ9GL zC&WfQQ=K7HAjZg&I!jAx7T27B!y>y5RCeVoJMvBH)a?h8PMfqpaCDVh%D~jA;Kj3? zabYK4dB%CkGYTG>x)uqHjB|f7bec@Kae(RKk{b@pnwL3@n3;`gE3#H+x=mr&!Q#5* z0CURr-;e?#R0)ZntjgK@q@v6@`aF~^2QJcpjZpFPT-uWxC@*(^-w3mW(GwDtRWcpKaw(I?CJp;7toh zg;9r^0i#gZY)7`f$&I#KPVhHL+*DRDWR`YN;_B~kW|cD7$?l?{s9J;Jy7??y?xF|C`P}pF)K#k)?l#7Il zSwZRQf6+=+d zuOykjV7ay2mj?EU0XHTKInJ<+XyUJ5a7ld9@;%S4ITIaD-Yo|9aKJ8#{riuA$(Nh>xS4V{f9!QUg~Nwux&pYm~Uy zbXb)hHFRY1Ffp?z9AFK2(C+S#z-Dm5fw#kKg|y2;k$43MCMz>xPK5=GJRA;83l{Vw zK7Gj9ui>is?gN9kRznVR=J|{p4xJJf9qiX1D6+6z9p*O!9F^e6fCq=m%3UNo`C ztK?{0`3q*58z0(kvOessFk$iGEVi+|v5>D+;84;gmlG#k_>C_|r=&0G+?%=1lPM+J zQRI?2*C*XZE+q*^L8AwZT1ymbm)$t3r;_NU7O+WVf`rT8xXk6YHIJrFJaJfa8RI#d z4H|qR8~E5*47o%z1erAwnrw^nW#?AyfZe3ljUK$flt> zOJK*r&>M{x@1AOw+xKs&d%m$1e{bHDo)1RuCR09It-JDiNqNHI;+qqW&yZ3QIKDVY zR^)MOhpxZ3&DTq*b_#oY3)`CemV~OP7BFe@IEgR&a71E?1G8WQ1B-cumMB{ySKNyx zsl^A}G>gR8Kk+gNe*5jB$n}wPa!-=TvJ;0*&V7BLIVDMesoAXmws!`lz?zRQ2T z@R{l6${mS1%6nqmFS%ZM&0lNq&!s8x6}$bLgNY%FCMC5LsLr+Guj<(1!*a}8?R>=> zW2rwpHzPijMhDi^zB6&7&^2^?wRvT1dj!6f+6nZ?gI33O6w;Z7^KW!OwL-U`68^jT8rl z>KSY|R1zMmtGn2~iaceI7%=6YZ?{md2P4-!fx{eko^*uH%el$&DNo(M?9|0cGO26l z1Z=2%UF7}Kh{b@RGv>v&?Oy^uAH2EdL#y(f9SJgbcRQ`>@$^|FV8H95@a@9$!%rA@ zH_50d@=Dr%^zyHock@R2sVgt~V;eY4f3y5kSm7YEpe}(e`y% zU4P@4?0vC=XGOx{j%uAXJMO$adF?lo$~p(WtOEyFVjc)i-O-@1#*nvUgTgHBDdl_J ziv{x4OTx@Eb1TC%c52kUW{5L8#j0fZtp3Zl2cK57+d8a&R^7Aj3s3y!{JUoYI}X2o zB=Ow1F(c}-gLhA5tC;1nunm%DJQ`T%CZ?>knP%I2Re^ui9459o!cHIS8qC8aj%b!# z>&$aBadGIKW-l1>^x(8(H?DDp90*V!2jw2?=|L^8zWy?%1CZKd^9wCicFhts<=o3=QNA9&qo+0B>1EjG`ydBfIXu} zj48fi2HQ-R2Cg%=nk(hG<}iq*9b;Y5Ai7}cIZr0NfM&&xM%D&~^9c;~Gwv6t$S-=4 z_;}Aj^*8(~e;P$hn1nx^St!F*kaA%~OoNY!0}IQ6j1~_!mWF=}e;8KSFtV8(;NfUs zW;rlx!EqJ|(dkbm990g;S~#~4*I=hq z-iJ00oeas&KYN=qUExY^?D^Dk0QZ6ecy zMY=(8vvh7&90)tkFo)lXpoIBxCzR&?mCVOsY-x z!9kfjQv`l62pIH>oN_oSBQ>L;)4%bgWzoYK89ppq8ua#F7nu=1W5a>M3kaj|qy@ah|PV%YN&jKXIQ z{8Ei)z2hKP;G)3MB=5l3k>bK)(qJ{ojrjwEipwGK8B*(A4@$c@%9=O|Uvbliuy{um`Q$64lr%W46ZyLXorZSrz?vZ!Ot@$CXriroY| zBc5|wcj{Kb0)zzw$h4+X3zu z&nqUz{9EMOP_H*DXwIGV8U|s9`|1mrq%tlwRPiVkrfSWXAanOIdG&LVBOLnEYYO$K~&X&NxtLMia+yeEF73) z9F#elSVIojd!?}6U=UBqU}IoZ+EoO={PrgR%GV76YvJonOJ z`!(wGp1llxmsvS!`TR-g6>g9Dw0x8zoJ!wHPCeuJ=>fxMfrC+9$`zHK)?L$=)On_! zd2{C6Q>8?PtTq46FROD><2m%Rf=R38(CnI9nlBoJo-oMYagfVe?eNJ>ZikSH43m-y zQ~b;(g@yysf!d5I2iQ{>>R%k-xWaJb7y~y?!-{|;<^m-~4hLq3v&sgJiVqx*yD|zt za8wdu(wyKVtZ9Z zbiQ+3vF{$bst0Xb$-nph1Ismei6Ju|J}7^cIX`R7{B?Ts%hrT4wW&xroe*o2|05{z zfLnlvkxzgz`d<2CGc6_6OYC=^r`%co@!J%o(6+P&hh?*vv}&3(7BqeCI<#yalhy&Z zFCPy{s5tUHIRC9PzD;dD+rJs{Y#NO5mW@I$8h*7oDk?N3bur3!H1a)Z&ws(r(c+N0 zh2dY^0ro5691M*TR}Q#3Imk-fHF?6MU~r1P;-JZ%!`4d5^HFLNw@n-0g9DDnpLZv-4zb;esDBjaoElK5PO8fl4%Xf z?hVWZhU_f|UEVXT2yEu-;p(Ui)|7B!6L4U7@UKBJhuJj6S@DOX$`%Kg4g=;RpBNp& z0=myWyuhN*UV1=k^Zr>15B_c9e5XBgRlxzLz=MlkZqD&_HWgvH(qo+AY-Z7H>0xaB z;+p-Pm!|(W2)t*u->}u}vvYialWItliba#Ej+6YKgEBW71vWVL&p8^o;wbl*DT+rA zBwHwotCUH!FiJEyN>n&XsFaDTlnJRg+IGJ>SJ!mvM(3A*pF?|v?#7+?vFWCIUUllVbc6@kewwX?SujM6$gbEo_aj1em5{MIxy_H z=J@?uV{tq`sg( z?KbBxfkWytO`0#9rrqmjTfrdugVEw(vdn?S|3oB|c3ju^&?wa5c&_1q+zUqW4~?oj z8kD=XSQYiK)Etm6VYc#MlApt+Z$zHG#=jbeA@P1hb?;K zj#za}W)!`1;H)p>y608v+z(1TX%K$GAZ*bn@@7?c)LMmrMlKJ=YR*G)9EU_)7=LLs z^7b?;m^iXosByet5U6QTb~~niOFMM-dK2 z5f?_08;l}4iLx7RKMdguls9EiS-|M9$l=9LudZ^TK)r}js`jK`qt z4C?_#m71-l4PPyFy!l>#IQ%WFZvMNY>)tK-zCC(=O?13-ugl@0n_HEh{5v7D;h@YG z#{MU)QCl`A##`7HH;U){Sk2y~Ske-HNJLRU%cqn@q2nNLO#4~Q-)C>`61vmC8(_~N-WOJidn+)=9Esl`6C90n-f>w|CKxW?Q8w%n!a-k zstj5VT-*7a`O4I%w=)`iO=@SpOg|u;vd1i=*)+m=(c;64<~Q>=t+DOd!{7ZpihIYq z_s*s}{FWWB>1%0?*m96hpgwXA>jV!)zL*{2HH>l|=98J5vXRtl;5iwKBnq?}m5*vuuOHRpUlqav4rn&hO{ zI}(%q=SjC-S;;KE?xZ6Zo2*9w|D)y>`AN%O-nzyc(8j{MDB!YvUsNB5y5EY%RyH;V zO*ICFgoFkbX@LhSQ=Ni3)fcr)TB$a(qpNGhLJO5w%lIdL5?HxVphHV*`J77^jw~12 zz||zAW$}Z%5IK-`gXhSo9y0|TdkghJC~juM52 z<`Ay~&de@7p6VIA4hJHa*F5SxJipJW;mGWSU!AMi=d^0Mu8=7B*d&x-lgQ@b^;2v1 zhAk(b9h2UCQru(PwaDcsc06%1I=N@s6YmY4%a&~N(w5Gi(V@Hl2xEug!5=yk9InWm z-C|Msa)x+p$xYRixU!Esw%WQJNML*T>BQpgK}wA&{nATL9Of4_I@VkzoW+{C}Gc+Ppb_Mi)}MH-L}1?hr{fMPB6cn z&cTYay?0eAr{7(fBtVr$BPqeRyM7CB`cgg{u>3^ z`IsB*l~^X(vtiNH7Z=)e%+44(%y54DW&ykH6F=Wc9xQqhi)V$4=8NCH!D?2;=$`fE zQSaUbi}i#SG7DaC;JWm?S!9MmtA&RmUq}bDiqu4QwLdx4;twK=X9Voi;A!U!Jm9PB zFrks(g@Hw{pwV%O7-OKD0$bgL7IB#XhM>p=3^Ug_&f-brbnbBEZ~VZ_W#rK5@urcp zX+|5{+u-(~BaLiIHze8W0=nE6C@?Z*1Tg$#bI@6KgQd_cv4PP|WXf_I1$Bmi157a% z&Z|uH7}bP4E%^<03J35xXf%kMHdwe+rY)BKVv}Tk?Gmr*B4yb(0qy($sfu5IsOyX*Dpp zEIhEpd`IIstqqK8Qw~ftc^k>9wV>$}W4gd4;UmgS7n|5x7(!(_8kn5|CM=fQsv!R3 z06UvO1CvMtlh}y{RtI=6n7`UjhK4OyUQDA?m%TbnJ_%boeV^wy=btP6QgKbpFs;h*ZQsHPK3G<#<|ecNsE z%WTJ;8*P*4%EZLHOyW^mpqV@xx27jL0iGmYXs~GAC?p#qM z%D||>EKuR1vggqw@#e$94;0L-`f_vX>bPWHF8FA7ebKbC3Dac1OlTJtT;-a(J(16G zT62zWV6^ugvF%5@lieFn_vYt3mcIAn*zWsl^YRLqrFTm>b8J|X`p;eSRN$$Vnnjh* zJluZqEHw+|5MIK_w;|v$v-pGqPm~tOeGzNmo^*pvPpPpx=td(0%ZUT&M+DkzpENMY zXB=deI>aJ%C6Rx^492474;@wo3!mDAI4QSTa%JB+E4g%qBa@QAWcCnlM)3)byj!?i zB+eXQH-5m#lW>7~<>3W$Oc|g1*{~?_FD#A;Eflo&4`6g;xE=h^pltKhyE$pfv)|^< zH{8<5vLy3!x7-(x7yVmnRjq2&?zk*lk$EcC#=`8U^sBb-4+Y~Zau||W#C{YpnXh=t z_P~TWq3zxN?bD4q9XGP21eGvE}J6{oMO5)f#)v&wM$ zuA{o*mv9p`u9^VUv6Ky>wj;&p!(^XNCEj>f$p3wUbM;vAD^~yxB z3OF2CrN+N=3)5+j3!nC^*|~%}P{fs4j&+%hzhznin?nQVp9O3uxS5pn9$ispsc{h6 zmB>FyfqlmU?koltt9MKe3|zY$cy=|g#T?+?B|7LIKq3*Y~Fw=K%q)T^yxn$F#MXFl9pE3=PL{C+R9`34z_g1dZ;Y#Iz~ zmj0)=^svZkN*>fXQ{@&I!mwN^Q=p=AN!@zZT@EYw9`t?+Vdi3GS?v{jt4d~7~(2C_-*u9P6P_PX$V$g zrvBhx+*C6vYoI+Bht=xZil-o}yZ$knqo! zZ~HnrE+na!K2fioRJM!P`$yg)l?%Q%PW0(~(45cnF(*)KX_409Mwt_dT9<{z_5Ij2 z9CSPlbb1Z%du;gRXAmftT0M`Mb@R#l^+QZ=}G^#mJs3puk6u+=RPD0pbW?#Q!Dk&8otamC5#Q>uck=VBHt znzL^m%Qa3ntp(;qj53EFTlD|CY;m~RB6ZULTYU5DS{O=Ie9s9Yb^4OICB7V14_sQPo@j8p@I(O14t57=YOxZW&C{ih%>;h1^JLU-?} zQeF#xzYt(9c#xE#z@DSZuh-!CSwZN}kuZ;goNo?DvMqF-q0OhPz_`ivg;0XK=Ry${ zMphXYhEEIvC5=*D%HnSnie{O(Uh>ymCK6NdcH@Rc;fL1oaV-=%lqkFJOCD#A>h(m+ zr&s@f(JZ;tBvO4zW2lZ7ba!Bem`YW@4nA|i#PdSE!H}kC?ioObL^n3*tQQV z9C?Hm1?ZdI@7uWZ(7MyR-l$Z%Ej40D33cGIOXN3k_{TL#pMTP7rW^%eN4DL5bycb-v0o6AE{Aft2In+0xv z5}Kn}TAhrYB@`H6H3+R*y-r7TXOseGUIXV81+F;<`R`p&?qr&~<(kprbCYFczDaL5D8OUPv`M{WbB^6xiR3CS^9R2^QvL^N*GLw=>>8`f#6%`@o} z7t~sFsO^}kw#^ig9baTQBtqUyla<-28aYYU?4z1VoD?i&c^t@|jXM_8R?2 zG~Li_xueLieCpLinFOI?#;Zl<+r=X+s=Wf*jLoz)_?fyUem3&AEZ}6#d%&-7P~@Kj zvypJD5CU=VOS=(ppP*_}2?77H`ESecfhh9@yESBF$3nd;Lm*tc8J$gwz7X zgz1wimh{|Bkh-@*NPNAJ?fGJ@aLMJ~64O#|7U`DXRay9POVX5{!sNoNPZ>LyDk_t6 zEf^zCt*l>WImg+ui)Zb9M_E)O03QoH%p z$=d5nEdA39)EH}ulS9i(`8E039@;B5ir6uRm@XCJV$7QGik(MMT*Z-p5`#t50;Z@3 z?C%cn`Sr1;D6j>w@@F1kTf#2Dr8v=m!Ca+i&$a+2mz9j$1Y75%GRh?|xjc$gekrp>RJ8^w64A6@nl^5`{|b7sj4TxEM~p1 z)%)TSe<`V=nr)$n%e&l9UGo($-Ke#8(uw}t%R?n?YpbTF-jofSsTJ+YZgwrnFtly{ zKkL(4`I`b7K5d(m{w$a|>brm1mD0b$A;K^Dzdc}|vSGEzc24<+g72huIvqIwZ8ejf z0*k`}&V~dQw*!2S4zNvGz*%&jDK2l%5$`Ep4O(%Fa!;{tNS(}})6Kc%8(-Z?{xy!Q zT8dmu3l2|Qw0;$948v}%E^SMS=GNfd^G}E@S!nG0S+uaU zW?O0O(#46A6$iImQLxS*#EFsv0t`{t>()%EJTcMK{P`$*oKaQBAi4cRM+i8JS(TqsvOTdsBP-@9cCxqG5cADP;A zm~Z(cspYd4vqy!qu5jR$4&AlacBZuAUG_#}cE&1>m3;pc(o#?Jy>j6HG=aZrG2gt? zObrbpOBzK!v5PNr6#8&H?1ut#NQlr824)^bA%TPT0SCUUX{uj&XamdaE9&Q`vos3j zI57NKAmaDn-qM5oO6-gtoBl=E9*LfFB~s})ztkH+qlDFlpP7Ub=H|&WB`^pXWT>Uv zJb&J*{(0;3xrx_%>$lBpl&o9+qVY7d^@r{Kb6bqdPcEMKYv$gmhTCQ4)XJP{4E@re z6PwbUVe$R&eTmkY#@`nRJUYP7^-!|*=IPI$XRmDFeAKXuA&RR*g4us>0F$R+&j&bk<@Lr)GZO?O$4yZBqfgqH3}(;L+l94Z(botoMC zC2TA%d}?fD;Su7cWdj;tE((m?_MLdWpB-Ir*5`h$<4WY zw@KRh=zZLNq(@p^&b&`1<-~gl_kBE(pPrm}rDp#v=Y~Klx2RXRM}m>*E3V||>DLx0 zonH|&;SrC~Rb}T^o?ZGc&M_z+ymm>$sNiTr;`TfqNsoAs7Y7_!xi8AP?0DeNu9vms zNWnt2P7zLpHlB%`ifvMQ&I&vaXB}%j+WA^b*(GDaB-Q4>y4idj6BzglSq`|!i7rv# zU?_BWbez-u>=zLxc7+56W(I`?4Z$2L8xolrZH&(HGI2B^q)Fp=AI=Zl5RdP^Q0YUO*!DCER<} zw#d}TGO>KiFLpb7RTKjp7!HaGI5aSGm^Tz07BxwbVw3vkVd$Y*>EYO_5cnaDao&=L zCvWG8&p9AntY&`wPVrLB97n!{hKbGGtThjqG$(Z}Y*yJhVNo+*=YvlioO%x?am&AW zxQJU`(BeV^v-l2&W^RQwTnx-Y8@@EK{FV$24vhD)h}pIr#O-uHag?c_(D>$8_kY}4EsG5J%}Gs%=! zF+csDMm^g7Gws!cV7?hU9&{S83Yn}aIK(2dVZ*GvS(i>66*1nJ@mA@%q|wPetvNiNT^_9f_?O{HGn5b&gzMkV|l2truufNMdM=nyC`n zRM2L+$xuAsv!I~mGIu=R@*PK4t}b*sc=e?d<9F725M>cVA25*k?9 ze;<%5+SncI@k%gp0t5RI0Xf^21pbL0EPO@VxEli>DD`VNh?iaLwdGK#e6r9@MP;Ii zj7S0lbHP-3)ddZl8Vo;8BvnCM#1YVc?hwEcdU~>Mgconp661I}AKO8WcPX)$suE!fKW zhCR1EghA!K`I?my*$mx#0~S>S&&-O=jEo`w z5}22EhJ}_UhuXYNTYbQLrqD#?6&rr_%@${1wIN$jR2=3qsM`EgBJ}f?kv7L@>b6+b{QNZ>Rwo!&-o4>w-lTo!;j)4g>SreHdpA{ITk)px0&T@& zXXnOC0j10#_ZryO#5`T$RXT%r$AO5dj9GbR&&$tzJR3egGLA=qh57m4L$cSS{5_8t z3r=ESox`u#>3nAAyz~W|RTvmMoEwrvssx%B{$%hsWN6~I(qLeI^SMdlL<6ht!ZjOJ zycBtjuhdD+z0j&^wUE6)L+jwgc<&Zb;Yo21_?H&68O8A=OD>;utZC)O-q@SB?_Fx- zE1tn@w&G#3e7W+8fAcP;>Un0w2u({qIa^a}^1%`g#()Nel0#^RXms;_HOW3r<+2W{W|F1N4Rw%v?D`o=*yvyGO^y&tPt zzdTY}w-W4VI>9VD zLxGitN02#a!Xcih00o;T4Qv+wHaIWQ6IxmQ*hz@_zPTb-oSf+&=?AZmvg(&L_UZ&N zOFs;7xA@|~+}_N3us48AzAaX8$PsWf8|U|?XL;K^jP zp;0Lyku&&3t)PiQ0}qD+gKC4PmyCk+nj6e}Fwbf%B>} z@9u>3mIDDcx+4*R$%y_$RKLK9Adzl8Niwvz?M9LEqwx8W&n%w1W&$8LVN=3 z%Nj&a25_F7z|*vlCtIR+TWfUhWv}8FUY5tQ9j>!kPiVjEUg9bqvn3?u$kvjV;vJr5 zeD4<2yMz0BMsR69vJ6J*!TD=3A7bMU z{7cd}Vh%8RHn>I};a+@z+apP0?*ZPu4ZOD=G$k)!HZfqjq7lCP25TuRo7Dssiw5?( z2kZ?A>{AtZ%M@&8Em548W+rv5N_&rX;y`2okAmmKpWYt5e(?`C5!y1V;XLFKf69rZ%R)8Ldk`fxw95BF1HR*&PIngY?we4W*v{hWz{si~ zD`3!&xhVA35+)x8j!FS`z6`E>BaR>e)&>Wzy9T_w47k|a)2%)`UQtM}+Q98mJaxeV z?p+Duv$Q-3DBp1{|FY95oMEk_0%6I5;-%U|-U}F89c;zO&G$ z!;F1F)w*;={-*)kjTj_1u*A-o;bFj<{Gg$lfuq~9KYRw~zf1$BP~(}K1G#3O9=K$}51EsSsaL!fWI(UHhgeKp;2YgRIaG#l(n`EBLo4~Zl!bf2; zW6%fo<_jFn54!j`$}1T-?@r*|)xf=&L8xjO_w5C8E$-br3pm;f3g$C#El^lFwQJ?H zDvoLcCcY1hofEjHit~B2@?I`j(Yb*8?gzfR2^^beuv8c@TWwLCo)#&-iXoJl!K-35 z|5H2000z&586E+w=?Mc;eQNRT zQyjLp7CT5SDZ0SE`X-ypg!ZZ(Ot}UVy`(s&e_9fFs$C*e;vs|T$=2BlGiL{0U;0jg z@7)33GnSo4J$a75yIS+r+{8m}; zb}IM%1pfC7TuCcgBu+@Ln4VbAS>3$FtS)n*M{=4B1@;%;xf^K0$hUyFML);VP@S-Z2=vunFgl&#h0b$5@i2>Ev_ z+3s7rvPsDIMN5@->c9KI`|iUsp53K~9e6$p@INh3ja#+j&oA!72R0leqQ0M4t-vv)Hr<-5~VC%xl~R7%&6iS4Unrd&*MDeZ{=#T|8=bNd6nNw4{L zuvk4gu>Rje1wQWV`cQ_Ns>y18N2QE)y6#Vv5p7`7c)ssGMhW_PVU z`o3)Wc{yOMID_PeKJNvr^B!<7zQED{fMcowXMX@^^`CvG7&tp4SyQzc+#I-eF>ub8 zoK+ybz3aef z{1UVyN=ScJp=$e#WoLGBZWY-1@d5wG3(KBfIQ7VZ@8tx(BL+M-+t|VcQV(6+dGO)R z>`x(VKcvMcu)ny#`@-@c*VQiWt6FR+4oq8DFrU1&$!`uvx&g-vZ;p-!oTl8IpF~V8 z6WCi9aPKY1zw&E$X9Cxq3rBYqaI|S{ZggPw@2J|I#URYU;Nuv{F`;UDtNr#O=Jy(F z_I==+wsK}Q59j;WTc--_4}Z=nZNRiEfa}x+4(nAXdNy$UTg}n0!>;VhK1qkY#NlM= z>UNh0v)JFT&3bdP?E?p=59cHS-W>wGp_WaXIdg2kB}?s`=+x>Y!6W%jfbZWg?nxSl zw(0Dg+I{G;0^j3?Q+GCOm|4IXBCwIIS&8$D!HFr8PuKYFDrCPoflq4gyk!Ttmlbfh zIWXB?Huh`anEYx}#Lr#361e<&I2Qb4;Phj-?m2-u_Q9Tg0c&@3bL?5b8+(SkJ(p`i zK}ExbjO{`f{>ylH%vkN(A;NHBFY^V45Ce|T39K0p`2Jns?N3jTF3j-2_w7c1LdnskRcQ4!y^GDVxSAE#EG%G=GvK=XfG1k|lG5K3JJu#M=&V~Ac{1k#n{B~5 z!IkTTb}rFQV6S<=e)8|h(`zqJazC}XbK?CE$t&haJe&}dd2ju_56cu~b{=)$**;+> zbM>5EYfc}i;eI6>%28~7L`CL(uh0DG!=eH#=?m{)$>Qp3pL%mL_qWdLy4fo`61a97 z?9H*_-SK);_1#UGwp@F>*DjcFUEi8(?}ej#9XJ;IHGQ1IBKAREqDgT_mszLCR`vpI zCWgn+29LMT;jXHA%xZbjG?#0pz~gxWPudN5Cl)*@n*GFS)@`4@f3x=9Wt;M60;|v4 z%IrH4`c*XPwlKr z@19s2upG)gaJ2TVwFP_Cgr_Iop6vW^@*;zfHSfWYTkUrzOni8J_Q_V&;`)x9cii9q zxzG7>o$H_i&yfP2;}7_b3-CAi^BlZz$kjlJ@xJ}`=cdaOKd}36*p2We|5kj z*GtFm@!&4AlQRspj^X zy02QYZ>OGN4vu^Ky6SCl0(+vvzngyFPQ}_T)T!L`a(n2n_LX}q4_{SPZRvQ-z?Z*@ za~8v)ISkzE83YdXiCUNG%sbe4&}^aR=0s;ECT0nn6%!O3kFtqMnFus&d~}>kQQD$F zP*c_{cCo|5e@sWYMKppJr}X_|c_n<-*(kryNVHgFk@NfpCf_eBUnmANXVeLKWn5WT z{(hTh*LS}=+ngtS2;un9z{teHE+CPRz|hdRF@bkMth7P{1F!cfnU#-D&t2iOy=&+1 zquxuG1n<8is64Inow~N9-PEQnFZz{E)QWt4>AbvO=FH58TW+2?W-P6g>-7G{#zSYj zE-edRWA^Z9x2*O0#M=*(Pj1Wg+`0JSA<^9F`B}G{f)8)YjozMj`;e~s_L@7zFV3Iz z?f3JG-BI{x-8`eon!(F`=OnOds|h3?{4~dC>B3Ep4Gf*aiYhbaFK9gao8`^z>-#_c z`^zt9si_*AU?d&zgQ*(vRS4aTEOSXBIu#v(lWuBLo;h7TdUO4>4otu@!1QF&X`VnA&U}faF2$Z16%+1gOXyCx zku-tFL}1AjH?fItPMy@4?CfpvBx7-}#jOu(dKJqunoUeD#^jzg-7C=4ZFy({Gq>x8 z8Ha^*@;19nvQ=HkGCij5cB_;VH}AV0Pj*G>&3P%+`|je)Ba-T}<$G=}n`W}*Ld#@7 zi$KT-+siFFS#uQ^z$11RS`tk%8mD znoTE^Uaw&gU|?YAcodbysbX<)yJ%v^%kL6N7no;E*6H&|?&39KSTe=y-;Pg-r?*^L zV{CS0&5kq1mwkHAih40Haa@m^&1`1>CZMJNR^H=aa|c%05{rjIf0xakDIV)L*X?9u ziA<5o-!|#kQwEE6`&zZCibiy4`b#7>J^XS-@Q$)V!w`85|9q4+0opF)_sSY+$X)To^p(wYcM*K%Ukq ze4MEn{R~DM*u88Hl}>J)d*s&zHp`GV#z|bhGfmv2`qmyx-v942(}@)3*>*E}_`5Dl z{;aZb_KQGe@wQuSrajFvuBFLVHx8crUUY5q8-}+$${!j8PfcKZpOYj!(}A^OPw^@x zkKQ`r84fiZH%yt>8XDps$}_WcFtB+f@ZWrK%p{44%caM0YNZ2*>Le9RLXcmf{AxjQt8?K#XDc*99#`-2wA2@jU&N*uIh+Rz}Up~&eP(P=bgBTw2R zh4!0;$C9iygaz*;1l(D=%w&1uS*Duy=6*&;jzi4%a(dcX?N#2J^}YBv`EwQX-suP5 z^qEGBlzjsE=)1t z_tIz*Ird(Ru<{~y%pQVj;5YP zN1c^Z1C}kfHGMzt;$%0Ew;Ap(6R%j-1;l!N6*>p>c-PLypHbhyJCS zJ2ZX|QBYbl%Ykv%@8Csz4Ox#xvntsyTxN4PDD`lLn`TlIhgJ`xKnep($Gojq%?})4 z$>W$TJ?}${`5(sSjI#>&mnk%@(0n+3*MVc!cP4T_X;HSk?BT)}wsqPuU%$h>+&S?p zU!LR4dG;CJFzN9PSuhj5hx>|QPt1-m=W9NxtepkJECvDov4$jzO?kZssp3SgIJ65>7gGQ&(C2N5VZO1ao#DlL43tTzN{Zl1UwB|Ri6Cr3_2h@^~0HPnYiQzuvzel$uwJiw-zQq-aPhDoGB!dZpyVJBZ~#`LX67U?Pf$V{tE5^(Z-tud!T zQT>jB(}HVmAuIkVa_Tk2uq$vhvaYyxeRD-|+~#>-AM+V7W@$Z$VXp9JTd|5|!{$;p zNd*SRSAQCqWgHmGCbXGMS=i(0b4c)X**Cl0B5$vx@szK*Dv|X^@0e&o^auIbE5x6) zw5lFXODSFWe7NQ;lKMO^+ogU#7rUd)qsy%iE}XN! z*uf*WXx}oALmX^19C0Q;aCim2zTGVYR%yFl2@8{jw zk7o#R*eE?PQ&4E?(AqO`ML!nLIl0I6;LO(#7R&RrmvFSd{&27$p}jDn{h7kfm;g>E29{t07Q+`zdk+fm zG`6@Uuxzlha}{V|yshB!fn}Kn%R&a0SO*pb0mcP47QE^1D(GMf`Y_pJ#^Dsr7WWM; z?h!1`0xaGinC<^K+y7v0&y+p2gV}OMv(7GNvxa8Z6lT{NW{U=9R}m(&3g*n)jOGf$ z-W_d01#D>_j(APj;<>pG`pL))?k-|xPT4codX?3lM9M@A1&0K zM(3qVQpW+lG?GcbIys5DIT3Q*1bJ!sS#~i z4Q*K*o)bk*W;C#6bhM>TXv=!x;cw!=JcorPWt)zri~AcO{Y&|CQ0d7$55absBaX5;QzV~wy?Wr5 zxyNnY9zK?IkyU!H7jv)LAu;Sto}aiYYddyPl$oD(T0PHgXHOK0$$AmUlPgRNNN zaA9_Fm=SVd=A35B4?PzRgs=SA7#`CYamO=qiEHOCvjXcqCbN1c zUKD=jdGLAU_6dvHvIY38joc)*%v8wXo2u3R>deeFOZw~<^<|pqJ$^LveWJ*moihs_ zw8eG^7%4OxZEoZdXf)Q~+I4V&+ly6yxz<`-Sn9PvtkmTn%N%JI2MHFt3?{CCrX3|* zITxmt|7b6pz?Klg_Hw3AN6(q=o)f2zc%-cMINQ>es^K~L&xwmIY{e31`BgoyemOEt z$AP7U<;I*I>j}O_j=Gml%(zzND!wA{;Nix|KN30;Ob%}aNixW)P1!CEX7hIVX;(Im4m)U0<%fC6hr6&7^xU7}i zeRlur8CN?P#T!B;U-*XJF^p1SQe=>vpeVO!%L?m@lhwT~T6@@>7qVqHoI3s3@srQ7 z)RjfLE!-@CB3Xf+ib{mVFD`gcLo>5&|+8u%s_w$<(;w*xTZ~w8f*tu%qXM*NhWw zxi?Oyu%#xj1uPAj{Kdz6Z;0Q~2wv7u;ncI&LoT|`Fmx@LAvX2w1&^B-cK9lEUyXQk z@?iH0nWLfnI!2F{2&)~|^~{(j$|I7Ld+=8G2JLLV^s{dB7MiavnX+W(3F(hvT#htN}hg^UA*4VywEjy6in2o*Zvkazk( z^xth0HflUN$Yg0CXT5=C$(_h)okq!~d?`n5N+O)nzn-(b3SPt5Eqlianwy9A*kmLbJ@j^m`H=D-sw{tBaSjkDlnZ} zo6zwqCWEmI?um-MU>lJ5j&oiIXOP-cg|JRm#c4Y$73Qrtv}TvdR^JQt8`|rZwq#GZ-Kcy2 zM97t0OQXLo_1VjtmXdwLbH=^i}87jONL6 zoAm{IS=kmhFfb_oWZ^JkU|`T;U|?X7U<_eo;P}VzoTJ9LeZfI<4q>gB69S3T%=zV= zS|kj+r|O95?%SEMbn($X1?MnH&rM6FP7GePB;;gKs`nJ#8aW| zWu{rJ+RRBVFZ?y_mTJVF3R!t^x$j&hPou4`L{vjIYXt_SW&6!{ThijY=!s9{dXHIt znO>}~LzZ|>n-Ht?bhXdyh~;mV9NNw$svUA|4X4ixeE}^8A#J`x703C+^?11E#4Pta zczU`h*BPUB<74Lp)!Vjh3RFJcBkx~+L}}8KlUnPO!*?1LZ}Yhwyk^Og+~}qEW?E*y zJL4InFREE{{n*yZsV}ZBESu=%Ioo7a(5uT!m;26^?VPmh!O4GV|L2uue}3gXT_f;) z>D!>iPhV>-T(@Ug)grI=$)WG3pAx#!!l@=9n(HyKt+7p$t%cvwV%uR}Js&f78Ky|H zn~toW(K*REcsVb4&x>d;i$^SXe2Y9C z#eEORnqCgL`ZMI7$eq3Fcgr87s>gA752$5&>YQ`a%-y>7$nD%+cPo@9m|M1o6dyEQe#NJBXZw8rOexiC)BgqMKD|=v zAIJW>!R7JU8ypoH zIkqLL3sf=*vNT(BC?4S!TBbcghC6lI!uC0ZQ#3l|)plugDi&V3xFo{km()_#t2bsJ zSWu^RS@haJu{*QM@0v_mnXOp4sx;_ilt}P}s;hH@j+-s*TpN^@t~xU*@#NB}*D{Ts zg`{ZiUUR~<(_O27WteqPLrvhq+~^L)X>R=8rYD|ri(N~cAS1c2dV@xml5Uz%t2d@(qslU!;9Q9m<7+mTtNga$Lqd^va#fXJh7w$R6JN z@!zTSkEDvC5)W>@7Z!hOLo2Vp$H$FoZZ&&EN)t3KMFWpIc8eFAz5OzOYj()OwcB!i zCbGpZUM{#UiHGYg)2>%aQ%WxJR(_N0_;^a%v~$Pv-z%<9E32KobjB)`Z9bf@mr9Fr zsQU}~KR@|p!{;owr*cPS-7B?)a^sF(Usrhhm*ImQ?{fE^h%Vj9A{^zl@#v$=2hUmr zf41E-MbP6@L`Yot%_Z}{UE62*;GE^-ZV{0!3WaYK*e4|3Z57j4(U@^8(|7gtD?+Ke zFPXej+WR#x1m`f%-+EJ{qJ0_8P|E}(ecjclxbQ0AIBPahaag(?p$!56R`^-Fr z7CkP{j@apm(_bfrudk?Gk|{P*>Z;F^^%HNraNWZ_Z&`$rja402rM0ERijL#Q+ZJZ& z-U+-KJSSaMSCX;X)6?m}vJ1>lKP;$==VB_Va$xoS@*w&6sV9%8Ei&($nr>(7(8N3C z(`tQ9#VLSKY?=S&p+L34A<>?IDbG2UT4X2n6}nxrj62s^!nsN~SxTja#p^^j zOW>_bGs4_1EHUiezB~4v!H-2Ak(DJMZ>bqzy8<>4zrHXs0l3r(pL@` zKD%|0i>JIh=8hA0?kmF8oa`Zy8{PH{cp$?7SVR6fLc+U&w zii|iZy(EBHlZVal`w}L*wX@V!Gn`_QS#L!&CooH0a9}fC!N{8=z^r^@Do6QW2Ei}C z72*@Of?>ccvY{CM$O)8h%k0uKH{W6!xdAi2opvr?C3Ryn-QoNbW zoNHfC5xd+KaKvS9qr&2#lFO412=?7R@=wrwC+CvXsRAMDM>s>;Lslo>)J=}^QWesf zxg=SV(X4vGVYxpa+Wvh};HYWTvnx+vR=jY)a6;rkE{h3GZ}&Z5m8m!=a7?>V(Sw0m z>H`CZg9D3_jUrdz3n!5-hx@!C2bcsq4zQ>hv>cKPlZm~-6#C`Ul$|A-$vn4Xp0AQ{ z&|1>i;PArnRhdI`lGcXCMWSn+WfraI(l}KXcv&{vG00}~#fv%BAJ4|8FiF_m3R4$6 zoh-NCXtjRnCyf`_4-VteO%|TowuU z*duQ4ah`F2SY;{&v0_8@Z~$qYMbII z(Akrr^vi4QV!o;C_cJ`ZA}hHn=X*uS-9Lu??x`Q;9a*z~%;B2K`c?g2-I?=o{Zn`4 z#mchpux?2_&cM0+1)J%v#=g8cNnD5Ty=Re7$YnBVU`l@XY5M*`X)MsE$M2feel8Z3~_&Rx}9gX4vrH+2^G@D@ruJS1!_D`6KjVE%zb& z4~@-XTmhiEAlT%#PHGICl)@IedSlS zObbi9G=Jd{6}#LE&F%Y2S+xQVZctTVEEI2GaNw`HAnwq>bmIX3WsBpM`!4q8%{eTY z^MGNd^oE92;dgJ{k85BF5=hN_zx}kZMDaXrh9=z_N0B)%SR}5j&GRcr5}vz&b@#y! zECOGqGkvpb(L1uSUDxDbb5C$tsTxWbP0k8uFI}<0?p0`{#gT_`lY9aXrSn!?2$SBCz$*Kof&El= zDl@}eCKd*}i>-b&dhE4KA`gtF%Qi4-TO5Ar+yBpUi6gtH2g98dHme=t|0dM@TjszL zdGAx*aUMnvivtYe6;9%58^3L>SK^udfJuGFhfcqWMWS!#xr#rDWvCW*65oBHyy~ON zVkL_{R~bh2g2u$i)`=3OLFzsVdQQqL)*oVL1ZL`f4dOQul#vizP|iC0I9vV!wmHjL zB^a9&G+4t9u$hZi>6S^)TqJ&fBio6M;ttomCb=c%sWRVM$`pKz(dK$_>ao zT3m1BvN)97xt^QIUQ)TNCf2Rwgm@XR0;4Gd=QD=di`RK?KHxn!fnECnL!ww&m`CY- zu?UR^rPtW>cJJkzz?84R>vg@=*PZwI1>SQG90m>yDIRIbWqp^*YU9LPxj!(7 zCopGEFDv0lFHqws`NkUmplp6X{SR|)-wo*s0qpG+_HRU7eHIy}Bvw>ile@BM9kr^L%dJ}@YJV4eG*Z1w@p%^SGiDDdqr;9z8nubEMn z_#^755(s`?AP(UY9T2q??;iagZfe@McBdb0*CM#aSmL zXMZ#~y(RI35&N`4jj0z|s~Kj+USR$4gEepgYiI$>k0qRr3pmXVNSG~XHc4PMbeL@; z!F+3Jto(vl)0x$Wzs-&iYdP|x^r85)yJD@u(`%E|r?@PiBhrw2!>lGYvoEHsFZelw z)&=&9|2E`Y+|GL^g7@wZ4*mzZiN_f(TQUkT-3jlKTw<|Gxs>>IJN2 zUs>D)Sbus<&o|?a-!Oew0q>Fx5oSjsI8SP|X3E80O6Z+t%wJX@Xqk8;wNa;ZX455s z&PP6TmUGH+H7&l$8gqd)L4Yk`Lwo#%IKHnbW)~Lez6!hL8otAGcFEM)+r^o;PoHhB z$>^ZLsI;IjW?QOPN6o39EjPdQoH;q?d`4+_`Vx1K){`e|dmuXUwTn&w}?aY#u+7xA_Y!H&eW*Xc`ErfNx*Nj zBGbc#$BS6w9TvTIdi4uO)e(Atl7DyvvsWkH(7;iH_YC#GFE0n@x!CBwpJ_; zju*e?EIai*_vDV2+wE;`nNz$yde2mqR!++eVV}aMz})A+duJ!_jaPH;JeYg$W=ZHR z9-)SwXD1n+>|%IknU?soR5XC4><4Ge0`B52&1OZh1V@&51MamS>hhnlNDHu5Pv?$U z5ZJZK@Ww>LLy0ntg%Xy{3bw+Dk2WdrB_;BfOUisQnIl-p!RR=DBYU54eD-y&trxiW zGVmxxZ`$j?T@qd{=P-L)`afpjuS`2Lne`uR(wjY7`$NjTue106T-_9!o4u>&WM^&6 z$tAwCYHqaEcypCJ-!bKyXByuJ2HOif&ja}GI&8gtfG_0r(wo1Qp4!HAQnM^Ly)~?& zmNkGea6;Mq4Q2Zn_}^?`H%nmMdCf31fhGC@>y`=Y3)$JEKd>@b@G_Z+Tss^QX(ZcY zWf?uu*lZ)4mw`m40c&$&MUT237rWx>!%S0Vu9Vp2!=JcO;vswR239$1?yVoV_a|&p zvgX;_uxqaYcjdRY@d2ky~OQw zt@G)YtC_9WrmYbXSj(rtnEim~-i)PB|1r$Fx`Frphox^C*4}sEJ*U9T_hD*^TW**~ z@9(3U49DduH21HRcH)ctTIDz59VT^cL^chOIZFdCxy!6F*R+zaTzk8iTF&7GZ%@ zmJf_^587uNaPQl|9lOEv%Vd@I6qY#w+-n!)Ee>G$7ihqJ-GDdBAj6}H#m&H?Z)?Eq zg9oY)iZ^d$Ygatkm91|7c>`Ct-q9;engt7GD`$E?&XlQQo$I)0QTDDq54hG%IHjS( ztzpBh{brM~HjDOw0zI9>w@P>4-r8-lfXQl4EI-4Nie;(y&GrdDU^sqFh0$BVE7BJfwS*h%J{7_c1 zSzPUFDA#Y@BvF%@HO%H8nC7n06V29`s?IKPku}m`(&C$(TNC~rQoh5zN~gU1Bv(=T zt~K5)3JH3*SPx%a8f$brO#48X%!I{y0kf?aR2v>({`ZAJM&QhK@grekM~?U$iQYLi z*JH}b6?4x1-0FFX;f7mJ*fR#!4-B#atj!7BR~GPw?m8MD#d}qNLq35qF_-gf0;k6T zW|I#L+6C+l7ubTn?}<6i=%&CiM}X~>S$W_GZI=d-b{31pQ&@Ztu;-!|=M6DL_ZpA^VaU|!-Xl%vOa#A?A*J^8r@xgN3!KJ+PZShU6Akg5-N z!;Q36TJ?(q4(47Hq1C06FoYOwA3H@t06SFq= zy5Y6!@C*vJnT;+doRlI@G%z=Gc!qFQA*G_%okbQ8?oq_Yp z1l}tFToVJhv;vryf8d#}uan^QSv7iRKvZW$hTv!qwaIaCQXWz@Z zc>(uj2a(2ZmX>SIr5lUZY*MS)xMM=h#mP$)1a?OD9X~m(v2k7?yF%?ou0~do#HIzD zY?~8y?LWZ1bpiLAXRJ{ItiA%Q?@x2@PdK#o1M8YyOil^RzJ8mHUvJ8s$X=_!EW3co z@B(v-0%v~%i@X5)4u*Z94UFN}iXUgDKG&V&$kTpAC&K&el!6(xH+`>Ox2fShz-V$o z?4MQxqeK7$>jMVyhAXlk*h1&<-dezWF@ejlfl)kwx!HjCv>Wf$gtf*K7$*wwo_(myH5aftzTQ>GX~iPmWBe(O9k8^ z4UDoE7~lK-+xoPC?|~(6?F2?|2i~&*%uE3c(UA;t4vQ+dI}!v~`RhDBC$fa4viN>r z+5CZf&4RLo3vnCjc#98Q2;3xkQ1nsGV^N#UJ1h;YxUxH=j!4)oC~(?v@qLbB!jYX~ z*&DeVS(R$vmL6EX)nSv$Z|-*t{O=U_b_uY(`rPRJj#aT|SHOW){>N4s@~jFDV73il zZFtbTG=N$5!n@iA-Wv>@@(rxX?|4cB7;LIi`3+bcZQ}MyU~=m<;{Vqi!)_RGKKtat$XxdguF6L>E^;F(as?xDb}`QUkj-8_~9j1RtDE_lFm zvte({nv(3dY|9;ZEB-xr&-s9XX~Xv7Yn<~BaII0`h+p90?+|m(FktT|*2S~ee>@&n zb-es_ibwE4mOsn175~YcWn5{Tak2YjjIi55;n}Pu8*Z7*6UbZehGC-XgQ)7@xSGl=gnU!ZwWYa?A05xFbvM&1>DZXW+AC|)a*D%=wS$L`pTnkL!6J4Uo6wI}3Lh?LRrXBPNtp2Q zg@_y{lhmA8hXsze*jNk>bd=~xZ!Io5n_B$r1TVjYtX@ooz>T-v=4a29T8FQVwk*4P zDs;A3roXYf495!J2M3&l4Q5-^{!4Q8o-$oyR)>dU(-L-3P19`>org|_ulJv8QG1i& z(S&vR#S>%#=UQ!@q;hDfhNF^dr_i(DDFKagwY&mLFE+;TUsC3npkPoKi%`oGV>0qK#3TGFB!RlsU!mNFRFLv`Bl+3MFo)?F@^jwZtrZ+R5)C z<)k9TQF&})kV;`Pf4V?1TQ`5&j70vdRU40XNKRUCn1kJFMIxugs)FV|$6p!-0v>sd zDP2rl6%EZ&zcijo&R(|g*hHR$f@3o2(|#=APhYW+jd|<2tOh2w6*&uys8AVpG|s(8L{8v+${~-T{q8P5cfIeYureey_DqnGw;q^QpsBr>++ZqLV{k ztZ4pf^{(m0!fP^l6$@F#F3f(+`~4ELn4QI)isl=pPd=0?b+IgTT^Lu9czj{(yd6%H zbygi<;?~x9wdiI`M&qMq3Aas+Y!cH7(xjr9A|DCrFYjbNBK4#~O*(qohQ$6LnVAQf zRAn4pMPrr-3WBX<(>(L+BzmxJmL;ad#%Eww}Tug)QzgyTfs~ns-H9e+E zTLw&>p=70ct>T>3>vgXgIpmvOyxT1uQ_#4_@{z;)y>Vx&7hnHAp|wAv@!e@92ZOYk zzG7^(jCbcovTSMEr?u?CHqn-Otx8Kob1D?uq$YHnbpNUGR%AJnp6KNY} zBnoK!-jTo=9kK4%(j%)LvYpyp@REU9OC|H(%;v*$7B;B5y*V~5U9S6ZBh#&@gr*6X zyH4?@Z}_-rcJ96$Ny8(j6x*eOIX)g1%-Zy@ExGKPvE|mqQnwcz)BN{oLi1^#1SZ}I z4L>^`up02Z5?=e@=#>K(S~YAOxm;c>xSsW}JJNxXD|B_kO{R(6xe`p-$2|S+%y5?3 zE&1GHO_ks36~S{O3fryk-E`zDde9`cM}c)ulT6K&h@DBMA8o%a^pM?EFoAXBBC+s0 zM|D=HF6iRO=;A1J)wO##vFN4)Ti>h0#`79mRc)H5D4FikIP zV5v~({8jo`T5oZ@xk&NJq8W!}vi``Q-?zos^6Dh16)zk)g*Ht461aeoQ=@@#&OZh= zzm|n&VRu;FcYJ8oaCyibGl7>=H`G)hWC4Tp4QIV|jvRqI&(1bkl6WUzsaE*ZV9t$P za~@O$td^DW3)iV|mObOV^Tfg9w(kWwYOV$7{JM~?6gy*4Q`JMRFp(0mtrwKUS8rf# zW17;%d{l%ZMuA0Yh9GBsN1O4#gckb=O?r1{v>7WHFDmk3?r&2F(y~fm>2PG@PtR!k zGcANUgN2{3;8fI%}A4Z$jH^uaPf*cW4v1egTfCdR(ruVzlx-w-3wgR z4+Q*kjPhA4T=i6egUg_Sslb7I$qYB~Ngp(?EMVh27kEghjd|4$*>tX&*?srpRJG;B z^g7y$8@c)_=2hJ)?%bE9wDDd;Ya`DRzJw1)^=-m0__e4lyuFIaRicX3py!eJfr3`; zV-b3bS1e!)5%5;y z29^&EVi7A?|Aid%opWYqA&>U&Is1OH&ghC-T{vbVH=Xfws2e!edZwFmbXxW{ikceVTTrp6Am1v4-RluJz%%G z!XV(@&?u;M(MZXb*{hTDh~Se8d(xj+?z#J;nXRjQhC_w}YweFoS_M+QMGi>}Vlx<3 zb~*Mg){+vq%<)8!OQGT4Dt%ps3)Jka9vw53YeAgw{Y-6Wm>%o@L-v`*ZHmFOaD~eyc!7OCQ z>MYqddDBX!L#fqg4u}5>OtilJQ*olB^*pORj(pzJ?g+6ev40P+tN)KacExfVd+`Z6Q38Xe%NDSkjR;*(dsMq zkf ziTvi#Imx$=+|@7D)Yvk^JvZ}!z&Z!6Yyl?=Ge6(Lf}D3xzBuHCypR@%dYXLh-v;*k zf``@$vHPF2?K`>5bVBu-2im!z|8|zVy~+H%K{_Vv##iz7OHE?y8d`O07T#oMRo0&8 zz+!w#K|`;D@k1>?v*eWqzG|OV!32Y53uMNlNB{T3n zld$|a)p5GPGfHG@%NLUZN; zJ)IW}Gk-MeceW^JFe$ENv&;~B7tF|0(QIqQD%#N?mmrxwfsy+~^Kpk(2M(6>gqe;T z8l*fdb2hMs&S=YD&}J^Nm2<&$rEtvOnQt+h7QyXUn zH83&?^ss$sFx%1kzp&Gbfr;tI5;g_~zSjmy6-^o!m<%@T)lXQayMdAK!Nkv*3|=h@ zv@2TDEn4m`(0RzvwA-_HyJg?5$f^4TnH46BRvbQ9R-`Waz(!t2_wuSurzH<*O7E>( z)z8An)>_co@`E*AfbHUq1v?(G+>zFZV`!dmU%?EHB0pKJTReYJ5zlc4R+SmMZ8aOk zBAUECW_r(ZRp_u@c55ORLnFUJ6aNh+ffuLvC776Z^rXxY`c>VatI-l8!IIzPcB)DG z$Yw>!BRWzod*pWUz5l@&m8pJ2vR~fBT~+AN#nVlu8fPvU+8T8?#RPCX2;xXwa9oFL zW9Q@6`@g2$T|Kkm&mqwQ#&a!=q7xV;J(_}Fu%%C67FDqOrzId1ovU*`~VDT%r zgC|cb2q`HgH5Rt?*X&;Y)5BeBF~5$+p-VTXwJ|lvMa*qvYKs?OJ0a4x^5yEdisL-5 z+@(J>OuXQ;SfFcR1LMt^EV(n<%p(~28W>eF*orgSR9-NYFJyB%@sB~$qiy<&6Q4HC zRmfmqZ(zQ?()Di?bMOMT^oq7L3s#E@>upXjYu=d16u@Y)fQgG+l_#RvtANGw0VCgq z$%|K>+h#C}X-9+dhvw)RY_SpxV;z=;?9$8XXy)=bZ*r2+RiXVz#mp!rcGnf>1r)a0 zN;YswFmZ7(Y??EPt$~36eC6?!hK44cWF!E$Ry%zOEwF+~nxRpALnGe= z#>E023pGwCd}Yqk2wKo#>~x@8ZdkIC^Y$PxK^>{V&kTZ54iLUss;E3y2L*O%EvN_RxpY(`0_h2 z1!uHb1~5u#+>okhaa_T;|3H(Y0~6-~25yTcsSV+_FBw)$U=Xb6@}JZ&p|Fc#gWtc+ z7dGhDF3{wRWxrte!(^$icB@6hW^M~^_6h}_Xa;79wge8gmPc)66+Ax=vgK5?@CBG9 zJct#T!90Hv_xf9s1`}BXR2Vo68VnfN5>7-iMKmnvwPO1a7d+J}evy#01xri?+X`Ez zBh|Me*Sc+=c}sFupVXPFdyY6pePld)c&eDkri5CfgD$#e1mkF_FV>Vr|$X^$-p>+fyaUA(iGvc3G%-R+uT<$ zNj&)1kdmU!K8t}lqcwt~&7gwGSzu+(gQT1jto#xTe9re0Dw@&_4*XYM5c-LE-ddr1 ztZaOr?=ARx@9o{ViJtSd6|b`0U}ApSroxe!k#kk9;?@I?_^UG%KVg+kw!tS-Leho}YA3FA4 zZ4p25;M|ilQP0ktJM%z7VV~#^2EGprd>RpjCX8YRjJAKf#qG8!&R|I8>&d-6dB^F* z1BZqFOt#WG$RPKl`B|piv`%4zjf=fMFl%)*vhTR7nb#WL(Q5GHfYgrW?^{^CI~bW) zoSS3XpuU4O_n*bmtc>u!=J@%vpLbp`kVL#}x~=47LRA)?<%1duXdDwB#o< zE8JjI6?mfgf>C0i`D#^?Vf&EkS`oDP`FL|RbytT>n zK$G9PCR2`w?3|tI51MX0YGwPE8I{4RaU!h#%SG`QK8ruxmjCPY;Pasc6^)`7q7&*E zg*6)APh#NT!N7ZgfhQnMQ?BX2fe6N@iE_s#1-3q7vtale#Ax|prG6&+OkdUC3)|9W zu-Lp{6fJ1jD8m-f&}uN@x$VOSo^6e%Eh6|08gwmKay@ziQra_bv>xBr7W1N!Goe9| zBb2$LHGBdq-{OY1b07cn|N166o%^hQ%*WB)&XtI6=!eklFBzDZ`GLZWm`SYl>177hq`A zeURz+k;N}UO{pMtVnbHZ&p`R5smsniJOO=wWu@ji7P z!}5l%%&=pDtOZ*RBwved3c9fBzwv_n27^_f*rs=|MOU;~aWDyFutspSg)tmV&+Go# z-N3Y>L5YJoN`QS`*NkkBW!g>}8VaX>1#jc3U_JNFfNKH6>2t2JTMBOCWcy*w&%iiUv@P~TTfV`^xQ6zG6JHY;+QKTB0@pH3 ztu5phZwvdNnp)iy77%I9mcty;z-OM%=#|!7IInuPm6U z;&q?KaU1aiqAIUNH$DAy;^e1;HtvlM7ybVAy))3CyN9!Bp+M8Z2VoN#POvjh)Z_6z z@-tBG=#Gg7Tt_Njefj3}$%`>T?*`uohW#t%aa(**^k@{)VB}=rDVmnFdC4=+aCruG zhQE)3eCN-%kW^T)XqEoQwwxX7rd(u;ozR-l(Hh3U$dpjh@^^{)KL&${Z?PNNvTn2* z?kk(!&Kk+V9KWg|q2l9m|Mb{3N&A;oMRc?#{wQsIXYi!8b!B^N_=nf`?%Mi3Xk0$` z*zJ9}S`od)McbuowwF)-^g)3C(Xy2Pp-P8p-W_`LX=d29ee6%O0$+(=Xc1L-*Snap zkNfO}J&ck&9-I?NyzW@hdw56Wy)!)j0!2ThCVXo+dp+wP-wuX&w=W_aUY)9Ckeisq zmJqSoO@7M?mjjpe|5`D1tn4=bsik7(wVwHFUW7r8MwPZ)n_fh-L_$gQl>JN|jaD;Q z6FB7lB-t4g8h(BF z|0{e)fs#NYk#FVmi->SSvpQjy@mG&_2C z%fYU#uV+1o*wFBHR_MV7#)Lx;Y!@aZbTo#qPpMtz@X&Kj_{Nm2OtQ;DH=NoU*nHw| zR`F_e|EPmnc5gao$eo(_CE=1Yr+}uzGVke5ty}_cE;q7p9@_hxPu@C@W2fQ=*6!v= zu2uDyFFgF+7Ir6M$&Ny0u|Ij!bR3<6(!3P*vDNCm^viNK`(8GIUM?>$w@=So8rE(4C_wDA z!R7`Asf-mn%@|olkFfPHu9I}QljSe*U%PN~?%hr)>&YTO)N2G^O3 z9SoKqDGt$CIJfL}?sMmI#}4&-d{1{?IOM>sDH7;%T?rTd7x`{!oy>d z-Y*&!@apnLD0peg{&>y4Z<&gazwt?x<^a*E4NgHWB7&?U2`!9lTjN~7#J7uGZu}vtJqv?AbPtcua4VOg++r9O4wYVRiK2Lzx2$m>8WG+|Jwmde+grF#eXQVa!Yn3Xc4h9*=xXf2dVn zkN5dy6k)LU&8Kkl8##y1n$=e(_fFq<#c8_!zQ1BTxeCqOM33bk^E@bU@JQMLVQH;+ z#=?zTlK#y)wXtPmvxrv)(;?x8L!VlB-3@kaUVQ%H8kUl&QKw{oJt)*_;kMvevc-{O z<@b9l#Wpq`b@>t+(klHkw9!>`-7-NljYM{L<<8q|F3jQ|o;b3fn%crJ%zNPUY2G9ztpkp1=?9uQ6(%&u%7gx zO#umkrk>nq#tT0B7HS>dp!;u`nI?0H&Y}gKav4s-E)`BE{lc1E4ZSQcOypZ~V)4G-;yEi6+~Hj%=r&HS!BEaxGfG%ISHcEB2jG%kDX@A|6|s%oZHtomg;8QYe5) zdZD-5nKN7_`Bxa&GEx~$4qbfA@t}c8WC63+k7nIvjV?P4PR9G(aeb3i-g_yOS$d0f zuX_L0I#$M=RZWvu!dA?nu}gym1gV*%9|mE11Pq z)7|pi#mIA;7GLa*>$0A4DP;Mv;(6jr975O&XI+-s)MkEXxsqP(;r~`l2N0^kX;Adb)d8;Zp|QEkvR51` zyl&8{?QxJR=z@!9&Y@t&ooiLuol*nplR^|PIOI6%25b@&U&HTmuvvj;Vd&ajoc?Rd zTlTzh5$U+Vl78YL$Egk%PLT%;Y46IJ{!O!9z3WiXv~BaAYMT@ZKKu5Xfoa22CP{}T z#Q?+J{W^sbFMQN3Be!?AM!HG=fAQvnPqR(riD!O{3d-4!L}oo|VPp}NU^wbg=riH)4xJ?!{&xMT|GPi= z*ax*TJ!;9>XBFVn(Pc7gjc&M!r@$tzB~3i59N6SFm?Sa;cL;215?dqCsv-w8CnId$5WIj6NJFJp+2-jJ|EZwG_8!2t#% z1NKmjiR;DtWGB5y>W%8j6mO|$GT=~7x*(80KklC+W0&ylt{W$qqD71k7;rGNnPm2h zE&j^L?-ji%!X}YjbnS}bMam~`%zJ<0u3*1_i`E3OB@A@|)0SL`=vvjA;#kjUV_w8u zaYuXiL0+#1jU~bzLIQ70)XgOYHWjIEpYKxOq}|}ilOV80-@|mea!9P+t^{UfgFBpY z6O8qD2DCUFVZ6Asp^-tRHte+F<(bm+O&A|#Wo{H%z{py_utTc3k$cUZgR(aq*d{4- z9X!KUuzSVPw!(c6vlG&MHt%3!^?QA^Q}bGKO4ns3mD4#Uwu`U7eb| ztHn4&MTOty!lt&=W`2)y^-HX;9@H}(T+F$YIdYMz__9;nQGX7}zDYQ4pTiTYBC(J& zU`4B*wc>q?2nW7Z4UYA?#~J^7IWP%xTxX86XW{%&5bgY+fyH9MZ@!2V>c$O@yjcv* zDiVsE5gtu4j~ZGnY8Gxh+Sw|4U4kbuW?5O;Du>IvcN}w=sl$_PL{I6y1$DgxS z`WPJJmr6Xr#$y`6B;1$K++p;alSfj=i1r3G!F^|IPx!W+0ek?6?49ghx?9>hxe4AJO4Nuc@?+=bSAWF z9G!hPh~w%Z{T?=(j>Do?Ci40;o|bSF>2T6*xu8?yC_LvNo5?|?FAdytIDA4b_+D}0 zeZ$ai%P!=@C|%Q{GS^G?f}@*4s1-=k4P zC#Jo|QTNLsJ;p=<>NxuPGV*U>5zKKEZ#byf z(0J(UslA>jE-rYm;X~4_y+^!LI)YpdYBbE7v$NUg&|yW5c0mtEUWrCE4@Ns|ugE^)d&3X-t94kDTTW$zA@E_f9lBuPk zV zJ@03V#v~>8p4(GA6V}A^2AQc=g*A$(9K0$Z;1%#x=ZJdPr9~bbuH}Ch`5*Bucf7!( z(9ybrfnCK_|7mJd( z`waD6Ya6u}-4+QsD5jz%v!+oe&#`8kQ%#*C>l6;J9u{ANsmya(Grp_aOHj27xsUvm<6@$DEm^!@#W2 z5LUv<=)u6$bKs~!!_f@asjmdKx5%(|%<0i-$j)d|zH`XrivzESfItS*tXm6;9MZEH zh0e`5UzTxdYp&Ao9sT88;eQic7KHUy1}z9OQmywnD6(g%$OcByBMpKXjx|ht9$C}a zXIyH2)53G6;YQ6Qk1aW8!a5AB?#({iq<=MX?nR~!k4C{W_oJm41v463qLS5B&NDso8`(1m#lUzXE=f9%7L|?9C&XW;CqwzLDuofyam1`4Lm6w>;jJJ5z2Gl2I(#k zI4sF1bVSvp?qIA`?x%=`rn{^8a^E*@)!pIsGP37RPr$7mNt%p1R+~m%X1Sru^2Wi^ zg_(bW%nF4Yvt%0X#PRg*SbfW=fun zxj4m*M@v*e{M91$dWlscK8!K%-igUL>7H>Cs6Y5-QOk`r4rdLX?)F?Ly1`LY;h@lw zgK`0k%f1~6YG6=rxcBLrq(lkF$DKYNE{=2lHeF2eW}V_1dG|z5%w!YCKIz4*yi3>1 zT<7S!z`V}_FV&V&DWt{+zoe6UM4oOd^)=p4o>EynwWYmA?9 z38pyK99yp8(in8+tOv_M$pej6ml&llG-?K@$=;c2Jm-+~3LWnoXW2^}&vgZ8Z#bk= zz$hr;$g9$LI;T-$a}-By)B340EK{yN-tp0RXZNzHcbX(uFO$svwJKCPJag);4kN>7 zLJn9?m8%hc)jsaO}7@-L8S-%Rvhh zW;GutH47&HWG9mtChiso4uuaaHy9M395S&`&fL05?ZLX@4Vq81*w3Yegg$VuV`p&VZi>0fc0=gK(0(oy{ilR<>C!O^syjxwqD;e7>nBBpRZVX)*| zYxY#9fyb1A$-yB?!eK+xIoXH<@-EI&UJfiA406S+3@03?ZZrO7%+RFy zpo!JqSuW&Y(sqZsua%5P7!+q5GKn}S_i|nD)|KBsFl2YeE}n2cwA*naC$E0~B5nx< zgMNqqbzcMiy>#WtdUW9sY`Jk!WJnMwA|0ro3L{J$KK(qL9G za9pR}Xl~_X%EF}Q;G|%|EORD<_s9XBGY%aM4uTh)Hf?s|(5&Y9<)qJWZsoUsP8#(?&LJKTC;M5D!r@Ez-V)PK~Ie{FJ@2ch}$%)Ym%n-6qP_FS1!?=COJD6Y@F@JbALy2$pXb>Hh#+$UannV zme1T2vZGJuVC$x|DC-ZI>j_TG`9N~tyJL0?$T4ZcoloC{h3^blf zrcasZC7G$Cn6hjp$Hs#q(Hw;?f@&W;o{3mJF=(COIcF#9^z;`K9!^ZJ%3xsRP^&o5 z?BjEB8Mk!Kh1DB(KRWfgLt(;>)$FS4RD8S-GksoYc)Y1XXVb|DkApsfH$J@Kys%2PZa(iWLjl%spNvb9w}2CQS{@kUYQV>zU}OQ^U+EnOeBzDjJw3%1C(H|5dPB zZU0|r%L*kn4X!YiZnb{}3zo2(IBj!eWmvdOL}$*z!;JE?9FH(|S7xKJ{*u- zl+dJ@64A)Usq)}Js{%`g21C-n&{vCR1@%QZv$HCFIM~W8mEf?SMd3jsyDUQ?OTY1~ zoi6E;87mg^amr=730i!5u~_nz65|me_p%8O8TqCpIJ3Gh3OvO4_s!=A$6`2EE$OvuHXP(~R^mN;#K31`U*|NQcJWg_i7cEp+53&pulUfj<>IP{M`^Qi+j6_+ zeve<&qcXvub?QYoF0TS+nKJ_H<~kpEBP*I^9x$+5$|SNKJ=iD0@vz7Ch=X32v>%Vw zjusDxM?BpDZGt=txzt4(xfe^cs%JQM2bLrW7cd`OlX0|>U8I4Tt+B&-+tf9Crm8py z|GX+FxoItf#R0+o9WJslAKJWPm^efN{)I6~C^1;fSS%vDp@CUu0mG%6&+Rxm9QdXL zbB1UvmJDlb*|Y6x^G(Y|l2a{?YEF9CV|qZ*a{pvUF1-m%k{c9R>kHf@6*lZq^jyH# ze&DELR>CfgNKO68TZ9rAf@c_ItnqG?lscZLprN?qL5+%Wcgrvp|dN*WfA|;#7TnRKJYl#H1HKq zXyUaxu!PGjQE;ioLgRA+?4c}7Y_30;G%Oyn26jB+e$VL0s#n0s$$ffRSSY9}|1R z0T!X3%^4QUMe=?yYfTE|dMLYst#ZZzg;|N6!Bd>vav6?jYE7J@yo7;SVg;j22m?pv zn#9UqoeeC{3^-$4)=#Raobhs9d5c=AzjrGmlc?BdTSJAm;x|9zHD3IT#0b{dO0(;bc~)o8Yd9$CoI7~v|c6~M?{vyo9Kpiz`f zkySYt)(9EVIz-VZFVCw{r z4230vovt4mw(r?`g1PHU!$gMf4jwK75~>2a9w!>5@-8r#e)^Zds8o7|(P}}PaoPih zUbO>?N0Jzr#T1^YhA6q5aCFkt3Y@Lxbx1_wL-X5&56l)*9&%VMFjCRl*y4TXkeK0A zM!6-7?VCIf= zINEt@;}+q6S(>vMj-{o0?^v4iD?!9+2P1cS#v!3u1uV*E4sZm;ew3SYvB&kvRFOp- zu6p}E_GU&nOV9jpc76gwv&4=@{s|WjvHKOY2+w)2!RN$*)Ih~{ZiQ||R*54fN{Jnr zM;7v{7dWitFktCC@Uy+{^zD=W`*gck$S{1L>CdC|uxG~ZgDh7Hj=0Tq|HmL2uvpT^ z*n2tYY(DU$6;$Plz?DFD;O%j$$>}m~-$Bh{{JS+|g73^qH4a?<} zesEB-Tw;-%dO>#Vj6-a`t5XDg)IHzC3VAnfT-9RhX(!Lo_&#B44~I|Ltm%;lc`wHu zp7Z6o)W45CA3Yx26r6G)kMW+Okk1209=i>BTzao`%?z9-KX|~C?%;H+UBE>nWMO`Z z1+#eY2Zt*u3z+0s{xHicI33WgV3L?|fJJhKqryDF_Nc2r6PLjEnP zpAsQh&>-y=((4@kNO1WMW<@zip*yQtPplKFVdtN5fFK#vRNI1Vgv)t&63aeK`o!jBV%NnMp zE#nf=|M>0sG@19)su(z@DR5Rfm_1k^bZr6KwjPd-1)SS-xaJseZA&=)is*3e0Qx*rFW12PrV06JTRHDEY5}naAM?-y?-S ze(|Jp6FxBHD>$7~NZKxQh@W|*O4Ms5uC9on1A@~Q@aHV#w|T<1$WgHDsY+0T#pFJ=blHuGG?e_iUZPjEk%KbXQ9xuh!9FXF19m)4=NS%;=I< zT2!ZW>`K#nULUI_q_281Z3Y9ULO$maHNFKCxOXw|9B>e@VC1oA6t;2{Ik14Uk6HBD z0gf#TI2S#T|L(xO<|NlA2d*#&))WV}CoS59080t72#afu;7Aa{hdZP7+LFJ*)szV$z9He%a!$SUqkDA&GSxXu?1r7>L zTG;odo z*`p5JObB2LU72?6)U7wq{d{_4rY`d;S>_ey%%_ydzo>ybs!=RLu`R1ntZJcH#X>Q+ zgPfrUC1)JUyur);t-<=<1Kt}8_?i^Hod{s9c+VEKz^JO9C!2w7k^|o(0nvm9kE61g zj;vw1qaZnnL6j>|=tTpwgcB3rkq(EvzmrrWRT2bVCr6i@R^&OZ7^A@d$Dz{b!HY8o zSag~gWEhx@9*E34$ScA0?}bDYBi{iu~Kr9qL5#Lx`+e& ztp)t=9QdxZ{@>@vu=bFk!2+ExhZ$ET2$dX>Sj_S+>%mGDx1cyK&Sw z809^X`me_twemw+8&`se+j?HN>@>5oV=|QinGFxv!*n?B91z%}z;)n&ado43-9nE_ zMbFHI!d(hn9`6MoBypZA1Uy1cle_=w+|p5WSH_cH#l|{|SPN7`SXci(mLE9;6ytktjTC zi@=J5BKH!w?kwPXw4nSz604j$Ys>+Eg9O&J24;z?KF?M%TO8oZQ#=*GDB`l7KWsIN z)Bzu#hQgEsKW3zG*fiu_YRa={$a}GjL8_g>Y5}`V0*6HdlT`!niU#Hz3c?u&Kxc)v zC?^{;^5-ltEz@9QbJSa+Ak5*&`iepH-@HbtUkp8`oLC%IDcVfUS72b4P-6UcQt;3v zF|UWOQY>41Qp1=G9x&BBX0kiL*0z9oMFX!*1MiIkY|9whV&T=*v~BBd2m2NCeh(xiF(li!E*|%J_@V}4a_ zY&OjAOt0G@^ZgO~&m_)6o~QRUh-D>;TZqd&Xp}W`6iHd($$RkchXwrK7H}YNx6RyWsh8!F5*3z^FXpFSLjTl zkd@f=d5YpW4<#oo;PSGRVmtV+t;O*~M)-tp3}OsSVhxn z*fWKd9FAOSjRM~q1o#qJC(UA8*T7}c$h_sc)B7m`9En_Bj$arYna#3#+}N319x!yY zL`+g;6lvI3k)Ko++vjd^Go&~qwI?Vf=UnBL=G^0SjJN3AgHuZ z_6_3^$2)S%64{g%3bCoNC@HXUG>YUg>RGX7R3)%W?RKtK*!1z5{*edo1eEh@6>3gA zXY(_0-8eBNLZR?=%I4(`W}9r*x&1brns)SpV9{0HdEslgwls)rQ{b{%C>C*0*p89w zzyjekjkX;P98VNno;_f`BVo;z%40o^&1wO&&?e>?(mNNcr9F3MQ&AL^YZM7!6nFaf zP+-dg!7@ehB1Pd>4F_J{X503F%T9r9o5SLXY%Ch9nN1Q|9TGSKWRvIr-pP4~*)E}! z_3lFZ1?)1n*isJoi@38*t7cu3FwyX|B#$ENsYO#4X4}tTXAU~Rst{uz%y=wOM%HK5 zJ`)GVlmy;S3Vbq&0v8x>IsTC>O9PZ?b1o% z#rICKrv!jgPUt1_q+z-szlp6e}p3zirGyT zTi_t?%_#njfyJu9J9XND*Dn{oUKuZM#WunE$;9Y$TnU^{8t!U@i~eX>`MyNx*Y0@^ z2blBrefyUnd!YOt^Nc4|xeP2bRIZuM+GMiJ^B)VR#_7wk2li#}UC3U>I&JP@wTHal z7MLh9vU#L?+%sm>kXfQ~fL$d(@Jz!x%bMkD9c7Ljlv=~UujR;YwL_#J>!lo{c!{FS z5l8Vg4`mA)*(3yxA78`#&P04hik{>yy>`)RrP~Hv$sf&+ojkfAd4UXjUPGJi;#0XM z>$o&xjtjcqo_XrP*DaL?^!#Sed)^?z(a3QiA@@SEZN)g=$XZg5MBeG?tfZ7@h6V2;Wq-?UT+_;#`&M)Uw`NiNC^GSXMhbbnkL2Mlo zD;;VKw!Tu%z9wy%_aLK@nY~-c>r_yn($dVETfHWjly)yUK2h0Wfbf{&WPV!MJ&o6GTaLmeR+bL_E4d94 zOo!Uq-(0o-v+bi|+myE+IVBF47@66)RU=Lm2)3MCY`b#K(np0K793y`7V_(0XyoiO zVfIix<*;QFA^B#&Mpx;sZ{@9Q?Kde7fHP4&m)4I90_n!`RXRW+~Jd>$Xtss@1(6?oV;}x7i>Br*>gV%7&0ZCrB zs|n2_AqNaud3W#Jz{IP);M*Yq8H)ghe~kPh35{%S7DgQWnrjSN80B~jTiE4d7P4~q zx_o3543xOoBDioFSCilpp+eWP@POs5f@UcjkFu9!e7q^+7PGNkF!)X5QIQE66B}7n zrzA9S>)%K?s#3e+QID>V#)JSzPD2AnW{$Hh?Z)Rcg1uyFOD3MydAVwKx1g8KqGqPT z9%)XM9U+JNOt0r$=CwNiVBsOzvr7~XvanS?@rFQ%fXhdPLjEFM)+NCK zfu>hOLqne)=5u@ZU@CV~Z)jMo*Ov=nk%f0zj|di?)e#r=%zPPs&PL+ka>39qik_NF z>lC{*<`x`sy%cXUBT=UQ0_P!a?Gp*hd7Zx=_~#~D;IZ*Am%!}>w-&D0d4Q2oMC8Ch zR&j}s?aHPr3N#q!R9Us~wq02@N3go&W20EpjSEK=Ps%2_h_`qo{t%0lnRHaN^^78` zVsXs`*7cDof`>)JZWt`((MnKgz&P!H34<@Z&4czGqoH#inI-H~&CbVcgVCgP6 zCMnj~6*8eoWcT7$*RqM+{A&&h`D{2KsBp2J=Ln;h<+V$Xkz1igM zqkW6SKmOI9!Sv)NQ+5K27SAC`vl-qB1&m$F2N>9l6%rJtZ0z`F{6$f8@(d=PAckhq zA1+)rn*9q>47N&?B(Ta=IP$j}FjiQ-Vif(sE6Y~c!dK$J$~%F9J?=syzlS5&6~iX0 z+F)1y5{HFhR~p#$Cb;?+GBDfLFt7z&cIK;iuxN#2FIU+I2F{KztojoUN);IDUe$F} z;8Ea^Q9rSgIQ$Bj$?XtySlTVFtPYFv`C61GYuQ({d=ylrY}=@HmyYLj8tnrSLqH{p@f1Ky9I|NBQ_l5`C-zm^-tuGWZMZQ&7h4e z5=R`^6qYba-?+d$|3bo+WtzIymJ^y}?GBi-Rvcj7718eQv#>=w;%ye+QOVj@+n8le zIJj4yzTN$4+v0yZH=I=&cJ;P>S;S}YkU6ATQrseeNy)d7yC{G;eaSk>bO8^h6!Bmd z9|r|4jf1j&f*r0J%_2)@95uZmcxAyWB~I^-Ba%7^jA}ZIggBks&HiQc<}P3st+>!A zIpty#N5_H${ufP$?0yOdUh_zLryA&SuYBQFUdF+(f?c zcJZkPxPuHXimY%uZz8agqs_rh^1uSg#sF5mc@NpxWmNh&A{%^!7cfZPm=aUi!J_%V znmhJ{Ij4Jvi=@{A2JR~dc#H~K%np6L|G!F^-@u?za?-@+nGF*eN)=r7I3Df|_?Mz# zCn4WAXH%GA#+l^&zca#KcwJxL@PlO+i&Lj=TT*=Ac@5orGvl(BH0PwxX7|`}Nby?( zi^Yya&f-mdyxRiW^>PmKyR^Azu`Oignc~(G^uU+5%GvbX4TibDL(U4Q7_eM#`SNe8 z?1aM#`*@f|Eer2@`7z#TG54##WC#yV;gLJSgIHd-mNW#hZU` zo9y<{Cic33!|4?>4;}R8Xs%<5$+9)s>{fX?pf_q`a8!-)F~bG3u0Inn6+imf)$p(+ zU&j>Ye@YrZxvvFq*-I$())qAQBzFnqOR6Yp1#Mw65#S8yNG`YE+NQU`pxJ(oA`@qU zZvCYXJR$$hWqNJerHdx?7-=}L2uxsLd{)pAE%NAz;Jwnw~z04XJz$F~(C^nj0))XEp!yYOp{;(=EmDRBPHANtpO!`MU+912slUF1RbBvB)CYEriR{;dg?S4Y zjIJ`=e=7fvfgw|H0b^qV$D0GZ3PrrT40wNhVb2v{JQgl2ynxgFJCEKX-r%o%wF=BH zkJktVFnlQIo@mf0DCVX3ILX4v&TE5+y+Mv+P)@;9X3GQ>MhP*E4JumK!fO?CxjYgd zDo60&5N>i}HVW{q+Q?QsflIc4#rpwk+y>4$3S65Hu=*>=6klXX^x&{Aa0+6GGJ7Do zPJ;QD8mpcKv+)eG6@{s+3XIaNjHVwr`V*w$w^SEjmevwro|W32kj89qfl*aNE+Ne6 zWrp0+h5qkX_&s=o2#C}pOyLB*R{qDwg}@dBHV)E02JY;as^s=WJQ)82&vyBk@0+*?*WjFNaDrPRvEAHXbm zrN`((;lC@IZW%1b49so<0TBnLyCtv~Jcy7tm|+}X%Ep@V^dV#LCB_Rv%x(@GT&5h} z3Cwj5*a|i?Sw3L;Q^B&dfwg*qTk`~_42xp+$O&908I%?<2tHtcqsCFafVJ^+Zf7Zb z)Kvy00c)oToHG}&&-xd@-rm4{r-83w0n^C>_N&TF!XFsIKXB~)z|pU&Y_BMi>e@M@ zqthW_u49nsx)7DQH#!p=nORRT)rk6Vok)r{7uhA|W%a>PUPWY%YU5QC2agA>z5y)p z8#s4;;NJB>qRxeTb^wb@0ZZNlu6hM#{RE9Yip;OC%-EACX~MKneTkHKC<9YMVAf(L zGY5{=hH28%r9wiOgB!Hm99X;sW^BD=e)=Ly^#ew;1Ge%8oS`2Ws|_sc4A_kqu-1KG zG;LsA8p!(VNCl@lbLut8pvCDn4>1$wgnJ&(pd@GiN)zQlm{<^(RE3+%NCj3$A$ z+6?Tr2E3CUO3y#wx%^?-v<7z7f`7APOj$#xbIp7uY^PBE@G*1xQ4w#$9Gj-ea*IVb zs(34i%{$n#iK%7YJ>j_vT|{gOi;NtY`&F5(94hAo#}@^##5r&;=Hj>K=3lgdbBh9( z%LmB?1v<7V%*Kb9mt72eb47C5#VwrA(_}6%=lo=lEC@8-z&_~$v)lrv-&Z8}eq?;= zD9f74a(SU_`fo<*4UF;zEbbL8`#&d)Rip<_zO;YibH=B35mZ2}yA3R7*l`QI%ljz7S<*@4^NfhCWDYiR=O zyrz6T2d9ABey_LW*D%G(HN@=kVh|5)k^R6nBU;MiB=fTNwFVy;jU8kq8PZC+*Ge|5 zz11i!r{E`0Gi_ENv&;f<_3v!*TiEg&7{wNYmU*spWv)6PQuTqOl0h@OfNS!Tnvhqz z#>?A&KGe)Ei&0CG z(J^87%wLW-j|o>@$`)TdH`8>5!=XfF?@ft;4gVJXSiwJI;sMq@KR9Oxa4&CS?S8QI#wp1fCuaW&jTw%o zY$mYS99X+zB2&Qvrm1V>YNXj41LT4X*eerQtsgLXEMO83V6M1eP*9+ue1tJ*0h7^% zll&Xlat<(hIk1+VViSJA1lC2N%o4AY_NIv(JT2m5AZmZeEB9Zk-QAaoafZiq0#;sF zt^P(bQGNTdcaocpqOJ0Dm<>N1I~L3AIf1n`fW=>dYwH27`3ZODHK=X*z#1*U-nM|N z+Q8LtK~hUl>Wq_-J_Y%qY*yM0sbWw44FcGw6|k^AU}&18@pg-|m_U@;1xBWXUS0-< zyGGY#7#L&>7)2(WWqF{R$-$5|gW(wu14C0PdqNGIA(p2Hu=yK z%{?c^ zDR&s;>|SveGME;&t$fG8JFks*0fXiPjvNJsrF|3F7hKnRAn@@RlaQYP+eY~_+b1kp z$6)n`BsuT$S0(tP*1HK6a^bn*A_wk07r0C8K5l)$>Y~6tb%EPb137^l!xt@SGA!mVIAUd=U--1{_~_r9~9Xc{ebH1Xx9Mv8z4MV7Z{nonFJW?!jWaSB6^|*dJKA=zl4)&FGuJ zz!~t0UEx77FULg&mKaq5?t)qBc4yDD3-B#qu)n}#^^|8p8QY}^JO&2Am+~HW3Gk{5 zu%8m(@R8!Z@_^0Qkdb>DQ@;b(_5fD9!Xm2$Piz)4`<_#7IOWCmmpOI94|@aA&j0U{ z4!nDFuhXMRCAsI__3FKee9xp}pBb&bn7-~=?mXs;9CBq!tf2-Rbvs#O6}U?Lo`=0; z_4{{$<;FjrJ1VTdRQeSD`(IF9c)_)B>+%#n2Sy2nG~)+sD+0JST#Q=%P$MXS^X@v1 zi3Ky69@H((tKr)a6>;q$_lCbK^}Z}#_iFP!2HuH0u}^h{1&WvGG3M-NJN`yDeFwt{ zbAg|UkBpD9oJ(MH7vL?Jz}V%$dy+xGO2ndr!O`(xQ}>j)9gB7R=C~FuiJTvly0ooN%B1YYgTPDeeRB3y42++gc}^QSB%PZQ>DD=8wt4KU2Qv>p=ag|wy>LNs z@r%I4p52!w1UerJUgp2lYwg5jH$DkNyPT;;M+N0{Y8_SzsjdoEHwfsE$YgS#XcV&a zNy&=`2btOV{%L(Gda)p!HCD9Vdd-E|j}A6-OT4JsF+nAC6_2=_h=fvO1B+C@mf=R0 zmHa(Sj4~z}4-S;PWMfg&2{>Ej*nEA1=l*5Bv$ixdExv6cJfXp)g;iL=MBo7f|5h{J z8gl`=Mn`6jeT(_i6=Z((r=}aN51ZZBaiHn51Bl&4N6{Z zvJN7JtCn|3ty;Nq(X3Uil4%nX+az)r4zfuvk#L!){O`ch3C?bnOfBj;KODM^e|4~k z#wZ+AV&f0lk-dTM!#}Q^4Wf(Qc(Y0}ZEj@eOIYxz*Yw_ugN7kiI~3EK+UL7P;OlHtKdmN(fC6j^k!YwMe4v}T`RA%h8(PC( zH1axpSh27xSm`Bmr$xa**0v-6_zpWTvCBmq7Lqe+NbFA!D`;T2dCJ6vv|nksuZ}SwV7)Y1GBr&#-;s|eNP{%O!_B%!GT#kX2LRd zaTX^hLph(Ck1_{%3s)V>mk8IANLjI`=yYtu!d9_)zZ~vLt=OR~WX@xFw97n*@hIEA zoE49z`DDyU76?%=Jle0F6Y;RInSWnGtJH)C2b$$pH8A&C%?NlTX#T7~O~}gbz#*~p ziUJ2C&Q0|S%&IDdhdcQCI+n3>y;!oj@gK*P5M9$t4Qy4W3kuVwnuyO5RLZ>VC4GWX zGG`$(GouLuBkzeT1Lhk%yg0z5mf@^=CSm25+*Jyk8jP)e1x&0F z3s~j1@VnK0ZE(E9Bwi9A#`~{WDD|fThiKNxhfL-RIc_fD;k}e7S#!c!o^2yfnnIG~ zyiYrp?>N9}6p$pOe*B2Ujs#Yth=W{9EL5#JrxfaB_1fVB`L6KUua-9tVotqaBSs1;K1s+fmJV{ zkyUvDXmpQ(O{Rg7<3k{;`vw;UnUC!bCWj~eV^BEDvL@JTnqZgZsy#Q0@*X!ZY-;oq znc=XQL!p65gn`+BK_M$)vlN>PFB5ahEfxWVIUFGYf~g88RGS+Z?{^#s66RoJQG1}x zs=&Z>Rfq9v%~Ne}p+{o>W;1$xTgEKU@gmY4`4aVEnrnQW>&jL(%L(TMJgu6wyb_yi>JG4-J<%e<5!hub z;KUuiqLEjtq0uz4fmu|efvYEgm0!f6UuMN&#HxJhT z+GZ|i;=RczG2KrigiY;Cij&h3b(uE>?dR1V@s&^LdcySUe7#;_*XpH8qNd9OTD5{$ zLv^kTUK8k4)^VIPA;49^D4aVQk*oTDE_xvsS(O?XSbnT>l>2m` z&3%EAcsAcrxg3G!Yi|`rdGZd){VC}1S#d$gY{lU+Rl^R88%{#2el!d6e6(4Vc|b|Q z`0Rx_S@FNFW$ydgJzrmUj(syvY3lwOzbqby2KI^toHY!s;$|B;^_Nuf$7UR2*D`2e zirI1K@XP~DzqPhK%QUE;MS{G>b&tIP9|3kT-kJW7&`h zMp>1Q9a?7+1SZU25lax@NMCY@>2;_H+ZS(!8P|C(En|t)y?gvr)H*k_3!MiXf?m$K z5^#V)L4ir$>fa%A(LiW#i=IL-Qy;0pnEftm#z0RpVL84J0i&K%-)WndJ#VZa$Al5{ZY z#-Z7{3e3IB770wh->g}%u-9`=vRv_sqRrV0Sd{9ppGd+1f|JM-)CXoXSas^JSb~3PPSUlu@Rc>|p_ghxo28L_4 z+piQXSnRtq>hj?oY#(nlz0DSUV3oCNSyMx^=Z02y0f9*u8YO-+velTe9PH%0siU~c zTH(O9YOChl&$|ySZt?nXKqXtL{PZLp&ppM5h015IJ(fJioou0QXuO%E_BQ@pv4ucyD7}#D2GZ{2+ zbuiUi+p8Beed%UeUTmdqVal4_#G9~^CrgC+m3{2v)oWj`-k@yAAz;lZVAdnR7(cUt zqoSF44$D?=^?3Zn9i(P~(M7yvMPWH!Xn|SoIW|Y=1BZDYPoyX|_!eVoKoQH(=7; z%&2mLNo4}#!+%9P?4nsjBbtLM+T1OeEE`y2R;nJ6V_KE%PyYaOFojd zi^E%HwvUdE$>vs%1jBh6oZCJ#2IhEjF*eBvDal72Rou1Q`T(=>X%?>oEnXgM$sTMA zOxVzo5y`rK$ zvY{zbqE&xFvqJ>Slc>!x9rFU5d0y;l;QrBIKBH-OM5ERLX4QZuuZG6OMQk278u&Aq zLT0d~&S>=hHRFw8|3CAT)vG7zuc_KH)v))IL`V2e{f8@@U#~vQCb3KJ0gHaXi5P}f z{tFDe7LB|PtJOX*1%7C%d(13wqk)lu@A-_au3t8sd1IfYv{68yQJ~{UKG(TytA5iN zPC*Yk#i9>PE7@JvD|WbvYs z?mcC}jVHK!L@G`#e=_Gl1EcbZCOv~Dy&pF54BquOS~Uuq{4cQjJz!3qu-D0e#o408 zYR7cN6%2eGJTD$KNOdsoeA6KJqgieP1K$A#UW=y8f;QC#MoS0QtOKno8yI+ZEa~dl zH)nRQSF_KrMf%eg>N`g+2;cb6XMtzKtB3|Gn}$tBCzb@;$0oEpY-kYb;1_Z@%PV0T z7-6+6d1X|VRP%=Yy(vd}pX~Boxh(foP~vM}Z=T(n6)cX@6IfFmB$ZufpI9^NhvKX= zW=fY1Sf60_Nnk7LV6R?rxq3x#wMTnVu#(-|VAWZRmgsnuOjcU9X3@$L{~uqx8!@77QbmXH9}paV^|2by#~G@Gwbl6*0#qMK1JW1>VSqj^J% zX#nH?maE1dtn*LJ$eN+TJ>fKm#pKH!CbpFfvn83nSju~CUVr1nls^kEG9TE-rqRIl zgUPSMLwx~jeMUR)0R|tLMj-=V-V+Q;{~ol|J8xBzXyCXI$bVUF&25i;TFd_ZJ{$k= zlHiB4-e1n%+oPsxaUi-vaZ-jus`dn?MgfkACuhbSt=`mpz9~40A(tyPU>$&6n^8l^6r(vV<{vgrQI%$3Cdmo0iVGOU zH^kbXy<7bH@-m*t)~xMk8ynx>Vpi5*@>{?f_Jif=Tb8hnkiZV+00!2O4wg(A7N?39 zPmfzc9&N!Et?ZLnJUN`AE7;OA+JZ9Jlozsv6tvk#FomvQ3#n+c)nMQ~&~W6z;)RC2 z3|$x3$@cy^8^4eh*gpVsaHMF+2XXsumn5d2~JS%o?Vd+Z3Xj zjWU{Fs4TKJV0Kx+k{NNME%k1#N0RD2MyVBd)#|uzwRYXK-SqKHtoAG~%~@V$(Tgfp zGin+*DZI1^PiPJMz!G-hVC;t`+YXlU61FE&EpxXvX+<#W>ae;uG;3NgX-;U?nZS~6 z!0H{)mUe(y)}i$)Z;Qo^kn|aB9v2!mOCA0iq`%N9aN(f_o{k0{k4A0}M*ay6LB5QV z2O3np8f9O|$OkmC9?>zbXf{el ztR8ws-B`gWEr7lH0=r6BQh8p|vK4o4^m5&t%5_uK|L#^L@fUX&=a@@&9JD>rRP&hW z{ON`kyoH4(&5Cfl7?3w zL{jxsJ$jzq|1$M{N%-;9JDKak?|(^=*w<=o!O8xFF=MsB*}XbkPZIwbzerTt#B98w zS<$S$IwAdfY5Iw@B&iuWm2J;Y#wjUADBYf$bo*$meeT-DCK@_F_2%|AYRt%*de$*+ zMQb2KYs?I{bcOg3gI3!KP1-NIEDta_D6qcZT@r@s zHr&tJarO8Orm$fpvB_^ zv-^o=>le+&94$%(312R^1_nIl^624VcrA1xOl${({sLBw2jvj! z8P%rcoX$&9?~7Gj5UYCXO;6~ex?AdtpDeGs-7GbO$u6QbG~sNdKx?P~tKE%>%7tt@ z6k0tkS|q+PJ3P?z=wLCN#TI*^E&2jeOu%vF6^s%u81`QY33{>i``Zh?x~xG3#U3kK zbW2*ieq_F4HJ`kU*?dCtOgE;~#>VU3j=}+qJRCfn0gXZ(3_A=Pg$f#YA|`WiH1gbF zVDCE-7!l-{Vc>K>%siqX%=Q*dS_Z9|0j+`2tuYa-kBgH(m9q8DJm&eJ zEz_b!X9lC?gip2#tT`9jVphC$+QAyUqS?HI#qtNU`2psj2$m3rR@WJ9mNOd9zifDX zE#OJC<)VYC+#mEWZ(z85q2Z3itV= z+x^ON+a51)f;G^8E3d`j+@NQE6F(fk>#JS+nzQ}Gy_#Ryo1TeyD0)8NykDLinF zH=y#Vk?Y^|5?8D3Xb=g5N_j{h*o%64vH=L)U zMd44{w%?_n?k|j=e)6Ty8zJ)u7mxJgd1WV-HJL;(Yb{_>c)_U1(ImTpQ8DB9$$5nh z0#42fyi$5>&vqytZtZE~j@@x%Aq(FEWwSLu4nAGRIn6j-OqYozxlP(*n-Zf^N;03c zc5#MJrO67f9tpR9&jfC5dLA%Mt73|V#zgiDJiL;3bWR2|ICpVK7+6gGkkHi35vG`M z_7;Ov%i0KKi-rpevYI)C#C#$yENHyCR!GKzLrRH-MUY9$QDld(Xdg-Iw<@l0xirE!{|qD}q1KbA*WE-3rW=PQhAVR2vPxAWq{ z$L)M-R!8ov(_O|h`EJ^g8Ju!=`XnEfzcER2TQ9%A_S}5u`g3i*v(2(s1aJx+aT8Fq z-go(jUV;0MCRR=<84c(5!!mkz-ZOk^yD)_>;@OQ!kJTn=1+APC$b3?%ZOJ1Zy$K$R znAqh64!H4q*nB!6EgN!x@t;s#&4c4ArZ!8PIJKq}ws6VDEI7z2R?u*eRnzOkln{Ok z2hoUv$g9^_1X`YkvP_?P^Z<*!(Sarw;Uy28SlqWHws96a1{`3Jt~uDL5#V6ht-kBe zqi&4^8%GyG*BZrBdW#Dlx{4cyI5dPuwl*Ay$v-5iK6(DsuD%L9k`w>%7Mlh`};Vus`)-td^hvO5P)7=1MmpS$eb(}nJ`RRPD@L~S$Dn*|&j zwmb7aSsFEO^}UM<&TBT_+u-ctvF4#ikWa$IE-~W^7v1IiG?-2(EnU&nZG6r}sFPQ` z;(;T-;TMAw3PmLej_iguj~wOtJs$pJU@KTz;HFuUaiK9#HsOF{w?srjBeR&z2L`r~ zPSG$H^LM-TS?u5KNDhtgeW1)N=;5HrU0|WqtbEeramPH1uz*Ivnu1TPQ|DC#b?eK& z58~F{lHz$z{eR0K`ZtHg(htKJ&)o(m0?snw%)u@QGu1v9pr!B=ZEU(9W zp1u0%%)0Y}3`(7<8!xu4SY)NK>Pucfle$dd@B4u-t-gnx%{qUQnbG>IX23>9j@v)^ zZXUXPfUU3H=GjfhwRv0H9M?Q#QJdi?vx|XANg$2cOo3J9SHYTe3un=#Gn{$T5}M5; z4zkr&xX5oPXqEFhC>We^SoKpw3tvS7vz$SbBGbo4exF7@(+>=6auXPNG8V8bH;&zV z;8TO*%WmD@zb=Mvy0qzf;!(co7Z_z$BsMXfsr+$RVyZifippD--k*z<5ErEEVl)-<}FNPs@%|GEaEu(mW-l#>V@9L z3Zj~iw>a_#{g$_%rF?)fZBawy>cb_Q7RyStoNe5c@<8W!u)wCL3G9v$hvn~0U_8>$ z$vx)*i^L*^CM6ao(Q^T9#;XFmf_NVB3rZbPe&xV1Gv>2kyTlRc3I~qscbvGR6`X`w zHn!`XI4E{+M=Q&W9}L_QM@v37ah=X!*>Lr3M_Fvi&F}{ocy3xavx!Y`Ha-;~wq{FV zoOHwExF3%$_%=LTID3M#@V^xd+yXgFsho_82gR4qynDv)k?{Oq-~OpEiS#HiRZMU( z(L31kmHx{E6FZ+RAH&6|4x3V6s8p`RTeD2G(lGS4MUh?gDP`m=;@G^ zxe5;+9+dj_fbD|i!m4taNcDXptd=@Sk$)InWPd$icj8fs=w~<*`Q3rTYL6pl!-B(d z%NBM!w>Swbx!`JAkjPuT=P?tD$3(8Q3rrF-7}yLhFz$VIbJwaj-MZ$BVh^O0&yW`gfnr-B?t68{+dBcIe7oYUm3QV@QyQpMV zt)6B5Fw=jzVk1Y35@(i5vYChWEj96_D`q)$Sxjnbmi@Bea_NJ`Q|wCnly_&eu2seTXcF%^ z-tOSS$R<5wEz_Z<+kH)G+Y3&8I=kbIhwVZUR&`fjmRAX!nITD&H@)n=IBl7RKi_e_ zV;c{behl0?FYlD#j1TY1Uk9|Cl;r04+U8dtnHCc68M-oL0kdL<1B?Cy9q%_ESasj* z?)s-B!ziM`@GawdVb4)juF661C#R6x{_4Y&s0ODhJ-~$vtg5t2d-ZMZ-7K?y<+@MQJP= zx{c>=7;vVC95S2zhWBq;RPZuabBl6=!?J5m^HipYpJ$rGa@_DEhr58&imDw>QhV66 zKi>#aUglr72_b z!awM*H{Y?XyYG54)Cq?8p2+#h9+y=(bTsw4UB;bd26t6wI31U5`Y0QJ zQupdfOZn4F&q~g6cj0Ih*ulX6p+S`AkeCajdXA`JBcpiDA)Owlj|=7-pKugPIH>G# zNLs;B$fkogqD3&nQApxooy#Gf6b7D@18gY`G9Mg;HC!1DF7B9eSJJ{+#$bWGNwai8 z1FHywPz00Q0gjj+*6^OA|CkOOm9%i;oWgp4?NMfy1I!_jToz8!CJbyG4n1o+?@w)* zG>1Wj;gF;ZBag=AoP(G9j?8#&$iTIrA$1RD;a*Pxu7w;KkCi*m7qH$Dj7sL*a!^Q3 z=I@PTK7Sn~w>L#hRHJ}`#L`LM5ABlAVFg@>bs zd&3=vIS(DDg!6AWsNivM@x>ztUi$>#8HNhcKtC!s42yfp`S);RFq zI3V=KQ9*@Kh=oZ-!*NfF1AEE=g`|V5H4N+m4J>BtXrSp*nj3>@VW`qRXKXQL&~%iNu$o;Yf#NZ)ecKXY8F#zCZqNxR~Z z`T;fV4^H|Sp5hanOhcTx{yJ(ckp2C&QFM=^sLw$;o~HPxPR*iB+%p?kXE?B_98`>9 z6nf|=>fylX!vGq@`@^X9CrC)3N#+bg=Yeh>1;!Ya1Hs$+W?3{aBs6gKa5L&Ku$wS2 zx*Ri@=E}0ifolV64@VgDl7^5a4Gbm?-2b*P>^XBlNrTx~b%hB-htZ9L910Gm|G33l z88{j{K7`Rrny_&k1--bzRN#vzFZ4zff{@t0kBw)@Q$#&7CL;vsG z)VzDhxa6U3iA;dvgilk$rQ6gpc0F`ZS>kNcB0Z^{=frV&jYDiAjN%uVv^O|uduZza zVAAh6B=?2!SE#eflHh-~OnNh%WF-`Mt~l!bm}>gQQA~zO#m8AQ= z!**uOmkVP~NXkpc9MoPIQ&p2z#qrXqLe1HOS>CKs?)cZuG>0Mg>?&gyXO)ZwCWiwY zK5iMkA_BAgWB;7q!1plYg{SGVd8U&NaGr2bSLx%Rbg!Xf*VXCE88`uUDHTFl?&G+$tY_EWhfYpys5JYoDN$tWhmq`Br$-Nz>N z4^HX}q&1H)u3Z+dRUn~xqEY#Qll%@@_L>8JcMd6@aTHToF1X~7*@`9wjwbOZO?r3M zna()WneiYx<|6Z#2HhoxRel__p5dfc(X1-KWEIgQI)hn{;jL~!qgezq@0}*&h(^sj zhqz)6>3(5QoYBPlq)~H?!_Pw(R23RUp6IZ@xWL7c!koe+eZqnJ-xdeHDGn?%8Xg!r zs_r8#tz&8B=ufw6=nb*`z!!K=7E;+dP zeh`;ZLX*h_X1ko}#di)pT$cZ+OHq5GVug$RldfgXr?@itGS{47;NQ@gt-~nmM9X>7dOB6x*C^h#s2;2}95C$R$!@;=O(2c+a% zob>J-5=%KKqjR)tYX_GH1G~T>;|q84fZo zP0BkOIa^#gH7-q!bvvseH0MU=%hbsDM<R_EgO^N}+01L4 zqsXM{;B0f_<~BEX`)`XMX{}4U)+n07US$*i+0ex)=$*yxJC1F~7kM61Ty{u)LX(yR zlja;p;V+ItI*zhO92G1Mo7ga`OkmQS!=%UKbfWgMyhYR2h||xP9g@3oKt$q@j8CIT zNuz~iBkvWK-5sa*B{VFmYTyoVk{0MH`s&7c`<}Yzo$gnRsyfbA zH=6ANj7xl)?Bfo7ux$Fscr~rf*HMTibJc2x4cdv;EW95W1U59PN*rWQXcTK;y5Q)f zaps872}k({jG89GS~^am)`u_NJ!Cw!Nz+C+e;SkK41GnJiH~Fgg;Wl*b39;Aao~z* zVBz@?yyQd^kE73S?rxrgtULz`{0|h^Go)PcE^=mL*D&ufX$TQWh}d=_`i^($bwf6r za;}61?iUWy3{IRHjmlG)jOR6(ghVpk*p&7l?BJf0&waPfsqo)-x95A$;#X4{xaK&h z^ytg|yWr1xTx7AvW_8oe_k->jJ2zWhakjhjwV3Drht}8Sjb3F6w;b2W@OK-x8nXzT z_Tt~ssH}2OFvC&dOL3d%8|4QFH7m-Ro~0=+nA_mmB!A(c_~nB#I~uvu4))3)QoP}? zr6GsM$5Fn<;n1OO<|z#%@Hx9C7* z2ImHcPA_MfoJ$fmif>te{CgpHRFqxG zu$_@VmT=rTJ^eAqp^Wma`)6>yIUx0fku||V-~$8O2?zNt3^&|L=N1ccO>5}AX*8$8 zzoz@Z^R>s)zdVRxk>R@YopVW)#-({`e;th2EtZ&kZaA-OvYOfUhqKtcAhnY4fX>$+ z73EhpY!O}X_-x?(vuaZ8+Z>~>i>VR)6!W}#Zl&glj#j7tt&gU<{T7aamup` zPMhu|FOwv6qhXHPK{<(kPKpwT#4a3E32|Jvb8f&nmIb?F*&H_B^v}63mRUI;IC~s8ceoipILK+@Wqq#g&{er3u3pBk`(-n1Z0|JN ze`)q-ylUR5c(Q-a$(deH%d8UlU!GP@sXB3rZ%OYhje{~iOhQwbOkJAOo2QBiI6j}` zB)_9kD}Y&Rg_BCc!92bul{wt5=QH&(B=kNUQQ32_^u`l*6Gw%d#+-VGy)g`%lN&@_ z8`$RbZJGRcs=Z~;k^hsUD~r6pNB`sT-2Qt1_TMRa-VEFfEkd3zR|sSjd{AI%6&6vN zuwcQNl|n^I2N_)6w#-f|SI`g;WOCwQQ*`n9;qhrPhmev>PeXu{ON*eg(+ms76O(jp z+>gAtpt!83&nRiejGu|idL);6PrtV@`Pn&HuM1ah?tTs$vy9$ZcGg8B_=ZpEiB~(B zR8H_I>SbI=c%8I1dS`y>fl_T}j<{_`KX$EfY~@|ZZu5kt>++J$$x5zfE}AJB6E|m_ z^ODs1+HplW{qnDx4+ck$EsVaL_x7Su#`y)sOBb8myqwxC;dsfaXgo)3fOs+*@2mnL)su=fq~<3 zjc3fL{|xX@{$g0qi;B$LCG2(5)J92_zaTf|%#T!iMfR0Xul*Cjk! z!8J4JRnPJTMu+*VH72^rR=qi?F28n{m0`cl39Z$9n=f5((cS(i>9p>Sdy@j!@5-9e z*tYM-Ek`ahwaNp`0v#PqlLK7@TZQ#HGMu;r7R@{+=`l$|RWwAdu|>)$X~%(P{%-Gv}gH72lkArdDwYy06%j?o#Vx zGjX-mvxq=`vvn1Q98xVaPREE}*&BFw>#L~5wWY6%7V7OwxzWgF;q&nTv(mm7DPmD9 z%UnIx%{-JQgsf6bZBx}Tesablhv71__KO?Ne3o}Qp7cwF{8SS$XFaJpUE6L&(j&*8 zSA@6oSX6`rJ>*k8*Z3$)FK_28%OAnpIhsC4T5@V!?Q51Ys$9m)Br#*b9#*qlhDH{l z4u?kNB{LNTQvECz%~A8ya25O`#&puY&Fj~RB^-G>oCRGO3SB1t^AK67GSl=y#)pXc zO}{+Zxn?GOi!@vm`ik$+@}Ts2)gE`I-LAb+afhpLC5Mx+!nxSQ*wXM7hM%`zD>W~c z4}Z|jrh3idX|vKCfmX4&nuo38u~vaXExfZDj?4@(>0_Q0^5RD`pVhsJ!@cI;Iu7^S z|5I4XFY)bw>J&W|K^GHUyB$ewOcpyFEO~SeoMm!P>o{il_0xt!&xS-&W1HV9?MU1 zNMO04>1@1cVNZyKv!p@^o2A0VcF!e;_)lDlzF!y6Cbn&ZhNq;=Ryz@;>VH3+FA1DWGnfQ-8Fa<&I4u65pDsiv$$0+!;#y~fJtb= zh8E3|2aNZtLw+oB)^O5vx8$GEpdxF~IQu8hxH}`ER_8<9k~xnVcsdR+ z-6}kn9(G_0vk3#!OHKtArWtaq5*k8FH6HVrEZ{MZa1iXZvceF_2bniGQXlKyOU-E#-bjBgUG=@M+2}W+y9|r_qEMS&QI56eV z&fek;$|g^J*jHS7nCY*qV6r%=$g+9YfpZoKO(G4G91LcM_BIBr51i}I ziVFVgbPdVP{Qa{@ih+@@Cd0`wbK%jb5+$C1hC`EP33Qx%q9?jM;HbiZl19EG3t?m2VW7$lie@?Dc-H^|vcR|MMCyz;s!Esdvfvo|W3IRiBT|v zp-e=MOoUO2MrEMVkK zdBDFS!kKr|Z+3}@2TVb0RjMkpu6$i*KFKbcuckJw@%E2U^Iu!_HRi5j5?i4Vw6oKJ z<@)yq#gtR5b{80BZY8v>JSa46L(qxUN(BtN)EumDp7fJ`_dsZx$8l9Qp->~SjU8nn zi>2Oj91&b5*lSy}LBwiCqkoZrq9~iIQ&PslZBvELov6!@{kTK1^VdYtx-S)T!&1Aa z^yZ{o?7BAptI4KGuOt1%_b}Z5xUYQQOV4Jx5Je#c2O;fkXHvsHEE3x)uvzd)0(Z<2 zMz(vhEDG})yFGoHM7IA^II8GT*q5o$=pj?Us4#6ogQSfx+ljZ$Tt*if?Q|Y8^I4o@ zmUVNSc7S68t7=9gi`RrEp$!LEl`6`Fsy% z-}Bn{ed0&kstcEAGrJ#~Gskmdqf$kp2+MUN?TC-v@dbz2+%7b^z7%MdvQXm7&2W?S zoyZ;ZvGBY5ii7fEg>AE~5;zVB-YS@-YZuBCCliD_Vsd%$+(DO{%k!DIJ^>>?HV{HwTML=6y;kdmeDB8r^v1LP% z<{{4v1yL_XA&-Ls9*rVu3%S)8MJF6E>-*=&nx?>2KGDc z&r}owj5;lg&RKFSox-KcbV>brMY4Yt^OSel>2LI3zFX~ZPTNAql`TktK_!7RkP7M;oNiF3^~D=;H~-!VOR7X!IZdrO4lqaAL}YdDAviAJ-#@XZB+FXT-q#ZuR9i=l}gn`CpS(Tb980=>el$j*Z+J#zW1D zViy<%nHZ0_@z*SP6Q}SdsDbs)Iu^GEan%E?w;l-TH9F`x2zVvH4g+C9RG2CaJZU~uu`spq3ToM z3?udUXMso5)jy}~snfgM6lC6&w|^FwX8eOc1O<5=s^@ioh z0%!k(4{<)yKcDG~`)lTD(QGnxNAKO6z zxrMzlj6!Y`_`D8Eg)oZTN|7pB$ho9}Z^Z(h2M(-0nn5|q7A49SCI23$S|nKS6HY6- zbm`1`|0X|HD+T6f=gkGa{83YQXD0YZ%^~*KV}*Z{6@RJQC-gEZFR0%Z#1i-5ZPbH% z=k(s>H8^mDFsUlAY*@@Mrl_Zzz&R~}*-KGMnNd`skuxpf78AcT?|b&DCZ1WU?7YV% zmNd#dQj`%YElqj$K+=#K0`TV}0uwiw?uZI}7;b zI;4~ihIu=N7A?57ZlgY@f9s3|8dBK?H;;W3D0a>FD4Zc`v~t0Gmq7RV`RN|TQ6JyF zUGw$rwKDfc5s&gVwo?yywGJ{*SRk-0QECZ?sE4DdLL&dUt(*=Xl3ELoPD|jucEHY3 zfo)BLz$yiw52b7&3)t8Wa(FZ*xU-3DdGMr4fi>j-yWG+g!-Qu)N;nL}m~9ROT6tOs z`I^UF|6|dh5&zxX%AxGHiB|ooK;;ChYwcXK4$8bbpq0)UoXKmO^lh!#nUJ%W-bN(+ zW3E(vc}|b@+JRR-3mkqq2+ml*ufr%hYoUnI0fAq7)2AeIrYOu_6voEx@07Gg{a;W4 z*XD?$hLMpak)I!GB}`$P^5(O_W%G5S?t)u#ohO?(mNvW;bZ`A)9>>5o<-l&U0~}FP zMSl4m31Jiua+Gv<$g;`J*?xu8IY!xejUs;>xV{~bI?|Z(<(uTM2F^bUY&{D&dlb0B zj3?MEV9rWlPguZu!pmIh0JDObx_+Ux$^mANm!*b+$*SKLjHzf$A%t zwKcFcH83bVU=TCNO47A-k?RB8&xuB7$F11FLV+<;kcs60%Pj}K4b2P(e@G+;GBq$bUaUxuJW#dN zoa592UX3?AT8ey67lhtTU<*=|ayTeBWufG%kKwD5p3P7~Ohx8vp4&&C$F$>Gre+$FECRD@+`>POf)m%~`<4;3$~R!2Rt2$F2>aGj(|y zE3zK&h%L-_WE3w+6zkfx;vfSLU!s^(qS!KZaixWz12~_&FyvV%!1Yk@1;f&m1A_k! z@H%}zSYt7Z`Rc@3hp)&f%oaXd8hfnl|B>fxrq7GMPg=8h1;m@d}ihuA6Cx0UOAc1v5{f9AA|Kfh8&*7{QngAlom^gEfjHj z!IP%I{wIQciI~gsZTUC8hil)G;JYkyY@*EPg`4(GbNRG@y-UO0J!-P|u1TCf>MLhY z7A!Gozs=Uvz?XC2ycxqkmYx=lgoOCn2bLr_^aU|`l{<=;I3}bq?p{B8MQ4=2t_Fde z1VJrDfolq3vlF;PiUiUam=hTI)mk`{7?|ft%$obSa10>c)t-)LZyI>4Bu$;5JiXAZ8Mz_Ce7HNvC?CoxK0Q#4dv zzqtI(#J`UGD;nnU?3gsanK>ZAWNFN)dg-N%H@3G%Pqw!F6XPKKjDho44+qC7ZkI$+ z2S-Dj#YLK2#WxehGaPp_A6wD;@7A>43k2jOgcRN}@A|-;vp|6DAltv+JR%qOO*=ek z#^GoCgge8(mxfO7D0N=DK63It_nL~s=6Z#~FRjw`CbGvUfMP7Pq0N?g!XyRGRSas5 zix~I$G2LQtw10bfLLPJa-+h-D_;eiEeHLVyT-mJcINc>js>@e$ks{v`Uv{_MTS^}+ z6wH>sxqVS=b=}EA!-Hn>uM{|nOm?^zrw1sUYIiur`Qvt@bVFmwozt?>xd|0|511#k zB=7}!b}V4ZSuSv83V+DMGt*OiuZM^@Gzz;Yo>}xjAfYRHi-G`CBcIYFpX>unY#%xJ z&a!A_PEz9(mS4M1`RGMe!L=7>&ED&t@$cSgt2u|?>@&WsoO8u3cTZ9R*Q`XLOE(#P z7&u=g@IPZP>t8Rh%7L%w!TMy)X)z0gyl!27vzqnXJBNt}1eY{&^?c(?iIH^cITE%( z%8PNu>W4fl-riWfFFf=GTXU(#8-){s#dW6z<;#5JJzV6!I0#=6bDYV=8sxyzHhcT# zJEwk|@3=kfbiFcL!x1i<1DqMJ#iQ8I**c!ldceGbUxbNK#K}$I$AS;$jvpoEdE$Zv zW-0J8FXVP=6yZpG`iDVCM2^j@nltDDN6>=m3*5rpuiEuD?w47)e~)n|GwZ5-&mZUV zS*ff|G5@&7qijzM1Ahp^6;=m^vq+T8P$A9C%G%F@XH3yz~T+APCR$ZSF zl5WD<;=uW00c%X_FS9Qkvl6ECCkU=m6#Wz-nYUrf!fOkwl@@LG3_Y}^iZH(Mky&GvAIiLAkgaN zr>A^!Hf8gzN{fOX9-is7fT@s))k{ELf5DH2PfEg5Gy-E;B0ZBdCM<~A=^@#v=sroB z*WksMoe7SeT^tfBHWdO}SFx-Pl=ax#ps00yb*y0wgZ9EVS2k#cEa|ZN+St5OQ0_?Y zsYVB8KCu-K7AhVNSmhkMNk+*umtVz}NkPMIL4*sJh^mr%THwE?)=oi-pgk7?x3REr zr+MjKN?<(3&MkH>BzP;w)Xm0LNzYF-DsSVHN%H%1LDB8Fgsjhuj+eT}dL&&>Jy|)s z{QUg=H5YzAeeFKC%Ffm?E<;VDiMdOpPgY*%Ma2TAw`^PaD-Rkd9_$bdWm&kXonuPi zejBU%Us<}A`%53Vz~;z+x9jmi$srlyL*%i_@rg)R!!8HF7R)jXlDDz!TVJv3MF zC>@tJP!X6O6YLop781Vo##9!$gaeK2Ay;1{br)GK5M^U)6}=vve^k_wnQ09}i|W}G z2N-#b&N#3uR)+OC}VhO#pFKQ9w+m=C4WC0_P6`o1l=+gx(b=qY+&P8 ztEhHh5>PNW$f4QeFmrqIq)aF7>=mt!%z`Nb2RRoRE<9!Od`s~W!E+kI9a`t^{5$Bv zxBcw0&-3?pZk&DIp*wMf;YFS6D;7_1(^|P?`aXjTmt{(x&tSP<`F)1N0|V4 zFLhx*y1Y}QMdIU8jm0mVyww9r9-X|sxZy&h(8MWBYZN#>9pv1|cTUKaRs6&T4|(I9 z07pjYi~^SSygz~`gz7>X8(G!QZB6PQMLzZ>yo(i(!NfM{`E?uu}@3D zp@G@-K;aP~v5*Na@{J(^hn4bN7Bw@Am@klE5p+Ddc>X}<@GU3brT*k>^;-8p51}LRoUB7;6?$Ll)wR|290K}Cl?x-Bn}peM;y}p z7tm#`p`gXvp?p&50RyYX0Y<$7$8a5kX4RMl%$^IHrF|AKNKAQDb&6xY(ToD7)CEa` zOAFjI&lRv6-#N&4O!HiU+0K(UdLE{y?qF73;@DSl=CIt^N|Dk}+f8f(7=!f>+$xt- zG8J+04Ssy`pA^&HX%0VJSYm}PI12r_z#>qSD3+gm^mfHT?)ZWvX4w^v@*#l=0TPD< z4j3);4$Lhc4l#&ZnMW@t1^OkKdDDzlKo zMMPAv+#^fXu+#Ig1iRqWhGTyo9bECt)jjj)ZFjj#8=BT~P6&R`DOJAbvq?!B%F-a*n@OACg&uCNC+1Mf_ zv5>8K$5GuFV`qsEA6N`L?kLSDU<-Nhh=0X`Hls-sd6FU)2^}+VQ+*cAnZyygqAS6r zPSlYzNyT~9GlM9t$=kfbo;7kgy||zJhg~k(>uo*H&nZr~F3$@xT+-;Ak-*b;q1_^f zfu%T=RoCVq_mYNR#ZaK5pCOX3#ulnn_jk{c#+cAjPs zdJ({IF6^S4e$d37^V)aSh%uC`oSPmQw|m#=wW*bkrv#lMukvW|IIvC(XkmKC&>>{t z$lNt!iBVdEZHS7p(4N+Vn!genrF0(gNI6VWd2(T|lthxyISCiz83~NM4oQOU4UKF@ zAKHcI95Ai-V6|Lw@Lr0{V#$*RZpM#hzw|4=B{6lHo5=~m2f>RzSzc;9wd?(m6gYqD(4h(cJ}}xZU=p1$p;@r+13Pblm{w6_V3))8U7S8o zOFk5F?_E;iBED3kBx>iTy`>>X9~Ew6v{5*sx`LsBNrO>z;*KM-6%%^WJQnjy1-O|` zQ8ad*pTIC7fK5TDJKYhxyk$>ldhc(I~LXOKV+gdPCCFDLUzY zCAgY(mA0iUh-kmSIQyF;+nZypdSMS+_18Fwt_V0PBy+K6PqPxA4&za!Z)(X?zv!xS z8y}IIw4hbThmkvs!$sxa1*r~xCT6t-GFhQt{<)m3T|Mt}N%yM*rAjLobh)n|P*Txpr9gch& za~XvmII!CtSNNbuY^Qg-30&&4Q~7ffLfAIfqtYGtR&Nr(xu90}73V9vY1{#=;z`~t336NFX; zaFv&{EiEWsIzdQJEuv+q`GNzS+E-ZH4cOfnI9oSxG&68C7H~`o&~-~-bi3eJ^gv8- zLFJm`VtDDg?*I5UFe{|6vM5yZD;T(3Vqi>g z7ZYIEx11+^17qJd4#UT-@Ia-}nP76fpta*bN?76dJHcPGHSc0ndsFkrN8o#2jLlPMFfozie54Xy zPv~G2O9;rC?b9{;bAi|U2mH#OllmUbUiED0_6d_J53q$_V4mi|{P(!ro|4H)rQUZ1 z!+1rOKbe>)6OiqAlF_ul+H(P;>;{(V3p&OZSc(MLeJ2EY8W^uBWUEzRk6&QmE5Pd8 z!0KzT(l>y`a|3Hh!nCLcmL(0`FADftJ}|m5cSjj z7h^2ctSVvgsn&G0GAj7_h(X{2gAoJE_5~dByHq(JFf$m~9BpRcoxs2$z z6_y6C(g<5MJDN*2xZUK>1)gOeI3kX-g*mWJ^PClluEGmC=>{t;ud?VDvrL)ATH?UoJRvX4fuqZv z!z>`fO@X~{L%P-lM%@IX6_=;8Y`0vfoG(zd@KJ@O=j-15jbi*CqAVAsa)jzJDl-Tx zuvI0nRVJ|SU&Wvx#mKQhLNm)R#Yy6Vj zdws^$1q z)zbdmOI0*?UQp!SUclKufwOV}`%G`vTnAQzJIp}|%N!k|IcB?8PFc3>w<$lP{DCh5 z;Rl&)1lYYlF#eM^U=|Ju-(tYEs33_kiKSSGQ+)xW?RG}704Avk7AqPfV<$376);}@ z#Id7+d*uYhOIO$p8d$qGuupuz7E-{%pTKyFV`1QG&Oe&plMrY2>S*uw>3=XG7b(-#Bwr1G1P@7r%Lg?S+V%BfGpI%~ezu@OmuzQXH z*X{@0yB2U~EZ!}BfyJ2lAE$A^QG*M`Y6q6by*WB#Dx-J-qp<>evjAtSfNJmrR@rTN zo(9a#AB?;f^jbu0S!2G><;Ir#(OYUI^@3dV{$CM$F3rdkz?8R<$ty|x9M{QqPa`!Y zCe8&+RtuPn7BC(S-j*)F>2rboX#tz+1$L`b!RZ^eeV1Ur6u>3v;ITl4L!gGuh4=K- z3p_U`9PEC*&L?(9ll0ZCHamW5O;Xf3Ga>rSG6T+*1&68(*b64?%-+Cugp}_NmpD&;W0iJbd9awr z`$8?_C!?-swcW>#N?u@$Nn|PAz+U%HAX1)z*)pAx@e!LvIJ0|$cyPB>&}^j-k6M;H z7gWDt5ITEYH9#->qq4YEzC?n!up;aG-zPW&QuqQGe#~ZIN?@B9z}c{Yqw4~b2G?5i z08WXg97YS6PcpEp2XHPjTetVSTE(?!^#vUEUNNT}(z`G7Tq)ptwwCXm*7}gSd`~~D z+tG9Iv%sYASEsup?uVLAQjR>cy@6A6&Y98;XHK7C%PhENu!be`E;EN53)jADx|V0n zHayVNylyDKEU&=q^*~W5<(%w8{tpYdV;-=~Q{ZxSU@2?}^xw%5rLdxV0n1OGn~ra5 zrba(OkdA^YmNW7hkM)Na);`|HJn=_x{(vJZA)!#jHG3 zYQSCaA1lW?PENnWm;OB9wmWN@z`P=lSzL}I{sGJ41KhF?x$kb_ zocn=u@dFl@fbHi5IEolJMLNoxLo$@6o{zr6Xz+opfkCzUf-c|Vp8AI1IcJ%r1frE* z?OnKB$>X2c+Sub;_im}~WMn@6cB7~Pi@^-N0ERL{21bQ{46oKHb4_4~UB_`bnR5ce zNg;-9gAW-|wl zpaYEJee4Yv6x9q)UYM0u;81iW)kYv-(S2{j4t_>X1*T)4FDV8vDgJA)S=A$Rc|qaR z2|OWp*q46bx$=RhS%PvzA;bUDVdr#(IV}$C zl^1^h^L=?{4qG_G;hY06=jgE-)IT_B|JjKDRgNr+LqKP?KxJS6%gJY)i$8EGethMU zf5T({YqblEEcxoD4UAF-me1?|NlajT-Ore#$zE!}dUE%_d((@%)h6sQ-CBD7t?TXH zb#vb`d|c4T+{nk{r_d45#KI?`B4F}EVd24cevU1bBD{V)?$i9HZc5>FYh)1=(a1(%Hf~}i)EJ{8%{XsaA^sr(lTCYho~+KLD$olGyK<7{FMq_;xXCv zoI<39;BOZ1X|g&gClVN)*{1E5KKJ6n!nPhB36ldI!C4pQ+vf#2nMfQ=IPy||b^QE* z3y#g~LhFr>7&tPqwX<+nEVrm-Wo2Ti_@mj%CAuWo%P*=y;Xo4uYr3B7M9t-mA{$Ps zu@yN9II_A4bg^(dWh~rUd}^hd0QUszR;l!yiCm)TQ)X%%;a@2w=#si5@R^Hf;2*D9 zh5uq?)Q@dGQ~7M#&R3_7^zV7~ppeh@m4MEMgKaiGhFU__Mn*@g0-TJF&v>H9Yj`Rl zxzBOej|ZFm+#a@yhWjmM?c%bV;5;>5!P2q)hLHthn{dVBUbDv*2XvdcZFVp;GZ_># zES2PGOqu9$XM!Rd=aOoMg`LZSzgW3lTKDG#{-g+^wUmIKS?@c9I=uqp&7v`lWYO%x1avRvE~w8GMv z*=pKW)v3WxoK=rz1s^es&fV(kHZx_{+pw8w3r=?3S@UF9i{9>6JF<_@)! zd{b3*#3he^T2lfZ8Pt6bRpQk7b7OHcuicL%@30**tX)cG3XAyDR~fpp%1*FKZc~2L z@t8xR?}c-pWP$NH7SV(k3mEPt80a-Ci$xxp;Cjx(bJz4;i}kGMY!*I$u}Mm!VfWjD zb$(3md)DPCxXZpcI59&n@L|7H0>cs!_h}D4tEk;!Xh}9NT2ZPVE3NV20CV9h(}=lB z1`8(z^;smg8BgJS((+`UqpFIx9^=VLL3T`kL=GN{=RM4k;uJS`-tE<0nT)vWYKh&qgp|N9S#jEq%I#|6??RxOFtoz z-#g-nPzWPevMUp}v`)J#Rrz` zIxM@C^K2l;N591aMvOsHE}23BjVv(}7$m+Z3i!=DrIeH)z`90J()|L1@U#Uy@_)D- zQ?vzGBmxd{zf;i=xNt!5hD3+;nP)6T4KBtiAK6_)F3;)^Q=cnaqGkdExd<0 z4;S7NXt&TgDwvt#)DYpKb*G^# z=!za^qsEbyY=L&FQyimTiV4UXK~!p z=8C95MSzR^nw{bM-jxbY`H`KV>ByNhWwXT84M%k^mF`sil%XHMA}f8#fVGOzQP`d9 z(B!HKH!oYdaV?ML5ty}vQ{{^Te@g+I>8g!U*VdIuU0o1hsuso@^2U+7?LyPBz6~q^ zQ<6A~d0ga-H#7?GV04kSJF}9cUF8-=oHFQ z>-M_x(d>buPJkn)kOA{{!$wZ4AI-+M4utajTX8_JHsh-Nq=(GBbz2?&vtDDZR-MZs zu&MWkmTUbYsZO&w3wg^5nq?m(v|Ha<$SZp6gY2gTi7rnV1-pJ2bo}<>be&MKL493I zuTHr5S<7!58!~$)i7>Y)=B}m4@;l)n-i_OLfBGC* z>SlT6MZ|HdOO+kWm%15^w;1wq?J%lMxL`NkreR6A*+D-Oi3hWIR1WO>qRQgJFnQ}= zN6ARN<4RLDcJK&XlI;88xZ<<|o7$8_&Qy(N$&kBkW-k9K4vE%%mDT7;bh*PQ*zw}< zT34AQuNxY%Gb~)xI3BLMFpR`a&x<9D zx*r_bL<+k^<|Z)Y>a;5P6v*=~aj?h_a11on(3!iVh2f8-XFy%*q&Y1Nj;b2WIj0x$ zaIr43pPIp@7gf+KvqOPB{Cp$-7P(jQJyKWW!?#Mx<*;uGzVVM|)3U8=^J}xEH(s7= z^T>OT(&9In+PB$E4piUEEcm|n@&dNgY1?{DH%8`aN8hx0@r`}Ph4!NGwREaI&-*LHF+kz+cHlt_^BVWZq#fnBn2_`;+<020l zN_`mlD;W724ob!}D&#myMKDSWFi8qHNfEmgI$X=qkt|?!V#8$BR+;4wnrFLCp0|1%xE)#(W0RxgrW687mI5I zi^qpeTm}r%94w#{J%2PXDKxO`XiU*)wLie^w4mAEgEjpJ^Y=wZB?FiiUEute@NiX< z=R^UoiFdnLqBpP#YBcWEX%u;JT1dhzS>#xX%K@xGMtCjXBlrT!jG(D7N5|KEoDA1@Vz<*hWiPPeE@soy!!pB8Aj*G8w z6g|)&a=_d+>6AqSi${fAQ;m~E!{nD;kzretMqe1FM^HZTEd>l(2dz|3C z(4f-L;@Qw>v$KIogMoKOql5*MLkEk;59Vfroii42I^XGYzIfPn$zcl?HMJAWq8vw_ zWZN?Rkx%+ zN0V$0B%7RBX?*7Bk;cPE8q<$39$#|iRL*iw4l^|-*}Nmu^HUCrH=GrT@V#atHBV7B zXcEK64?$N<+>2|@iU%}`6);LnXq1}4sC=O5O=6RZ1(R}vN6Q>WH3O!mB{oVCmvt5} zYtCp?Il&~|z-ZOLVrjsnV!>qAaM_~w^0O+IpcRcg1r2-?nsiPu+5b>hjacZe!5D1B zIbDEZ`T|Z{8wT4ehglpLe)5EhXEgu0d6*%CL2Cz#=8DGX1x)N2{h|k&%^Mg+3z$uG zSS|nc23tI3aP4T(>0q}1z@+AZoath&h>|vuM2Xx$+;doq;rPF$DvU*LZ+fE{XlDJ)Ea>!+*OMG%k7g* zP716+9vZVVFvr2e}|bBCfu8R*j9ysgQG!oLzDIdChr%FOdrniZD5q1(I^$sl#;aC z_5f?)g*H8n1{;l5VT}eZ1r~#j0A&qhhv*hNi)Q~NZDkxSu~Dmy6_~neq(dKCE_8@x z6}TXr(Iot1zJmg5)&>T#9lm?lgiEseREoMCxDuEw5+NoK!D@2AB(Kr|xTq(WLutLh#)K4OhGQ5)@~1&pUi?_sJOx9weJRu`Fp|u4oXR#w7ir z;oAx?XVWIz8BE+W8trb}-hZ}LTcXK&2g|Y*42~a~oHlY zfX%NtiB)39WDWxsL(bFPv$GgjZ*a5(Rfd24?R|jdjI8J}Loqi|i^hW@ZZaP(Hn=`k z;JtKj?U|OX+g4=>m#^2bNZm$Ay!_oK&@Hwr0^^BPz)Ua|$aILoEj$#5vmxv#6h z61#yd`$p@$ZEUeOShEG%%RV%yG%(vgU{qnaukwRIc|x1=X=p$qre)Z$7GEnGOtN{!eh~hX0I7L85LS=KQK8}G;=@b z7i#e87u?6ap}{DiEl9vmVnaiQRTGDi)#BiWCmQWbxfd#XJPMk@#$&+P`prnRXcOjc`|NSxrHD;QND0bCE_*L1qS66aaJ>$ zbS#?8);cv#Y_^q{sw$zNZP&u_wrSImf6Z^BEn+(pX76~w)AlmC&v}+ZL-fQ3Ru2a8 z2RzqMgUcBfz--P43;ej3gDGI@<#cb6<4WxizYO zZZJK1YReq=j7xiebRXeh5>NW{lUQhgF*U&ZK(;9W$r7j z2aU=LTzCU6a7QpPo!E3myFsd>QPhG-?!u1AJDR;_G%_kAGHqxO^ANPK=(?-Ltk%%L z^N5opfYGou~BViJut`i*zwj>$Z|*eUkzph+I9%eyoWouf_3 z?g@)*8W!c{m?>5qOyoby>A|Rf^j(ju%R4`pcWTlWXBwnBnr#DGV=k~drL)I7{Pdib z5>#>1s^H63zCQjLj2$f}Jk(!*c+()saenf{28kD;9dB&7Z#1a>V0yQ=i9f-0`?ij` z9oZY0)aEZK-F2*Vwu!#g69#F9M*anj>;Xq&wlyw`yZrA_0khnL7*<;*_JAhu2aK#g z{5}T7rtk@dw@>4AJs#{$dGO7{m|=D4hh+<*|JyQ1EMs2q()wltUxIy%DnpHOlUl~B zwg_hK)H0p|=BOzxstFeC6BH7dI5-46=BzA9yQu5E{m}spRo}>eD_zrj3@n#2H!<}H zNNFTYdUIx`WaE^hCmoiDn7Ef+S|X^#!rnDQCt|?@%g<_l!sgLuYzjZSbPHQV38?`2y9?{zpJcQc6;WmZC55fQgmh$mP~Lsw}Ej}8;^uZ!Lv0p zH#75!$wh1kaB^nh_2p{)Rqnyp!y)KqpP_AVD3*gy#N@_}2j(B{3kaDwD9mB}c%RSW z2j`RH{Q{i4EF~F9Ppqa%NSxXcc8A35I`TCB6i5x-`xyJj7Y+A>?a(EWn9Vsq;mnn^e-Cmc!f@nJ1Y1%^z-f z7~uF&A%y#!#o{ZvzNW4migi9q*`;#=7+cgFISyXsN?2-eq|ezyfhme_k7*-w(cf)h zv1P0st_qi*Y&xa2*h6xu#$p#sFTLv##V169Qzjl|m2gm-$!3t@@GN~!;s!~!#ijnSwS7j*Pjk3vRpZV?5rl@X*qBxgs7Gbq8i`3wICIoFx|+ z7|nh#b|^Nx96HRLexUG3`Mh%m&Z-ql)F0%BZ3A~|AFS(1`-Vz?8$ zmM|L~Y2=;~6lM8zO2D~AUK&9;51)!mPi31qb7pGV!8Eay1Cg0Y)5M<5QJWIxz-oA- z;Rw%CO^4ged^QHNbNE!IzhD%SDPYJu{5MU8rH{?rh246^h9(vki-vRhEB{I(X~fMy-G^7SoWOi5U%w!gNaiQI~E>=?YvzYVDZoTRWkItwR zok?(!Z(y0!!m8H4^V2Lbk8P1Ny=yiUWh^R9b7T}3S+I#MaaTM8`{qai$B74d|ITJ< zVrFp2H+y8_Zt0`P&%nUgq`u^lX+yrmghpuw!HL#7jQm|4Ek<(|a{OO%oKfQCYu%cOJ%Ii-5JqLCD5;?t1o(cwbo|1bbs;tex$Px8%I`^JW zQ*ZzEU`h4lo_J-`v~4~OjKUrZ&Zj7_hptGf4EU9N^xr0~LYYTmmuI+$^*msUc6lV& zZP3J~_j%dW(gnV#aOC;*pu^(JyY%me;|tUR zbUYri*>XPMWf3T7xTmn3(J%W3bIgTiD~3dN0RhL4B8(jFe;9?9CM;7HS;*lYa!4>Y zLy?JTx*)5{1D5m&&rD|TrWxe&lEP`l!M z?Y{{fE;^5%Y%*OqwPWFee|8y<9FH2fp360i`}9ev+bD6p<(Y$AnFqSmsx~xBvz{qS zI4R4n@?f$2%v6tq(cE6$Dk>5>ilPlMtr`vo+3h+WEnT(4O`|BW*Ygb{rzgWvp-BQA zwg(hA3nQ2W(>Ao&9eK!MY;j2LQGu58f<^q=&XR1`0y^z24)IDkIH^1;YjOW^P{`;& z15?b)c#a-w9?GC55)M6$$1B7Om1@8VxKu z*X$V#LaO3g1sN?G@|iX`Fi5>N-@jL@fg@om*WY3mSCz+n>N}cMXHD#@IMKZI&y(fZ z^&0((PJNQc53s08HFj9PILMd#&*QKZmsX77iWh>byE7z=8`n$3GIB(6HtlCvcI3kN9!VU|0p5;E`@x5_?z2gE)fWt}s%K@yK^AfqNR1`TU8n`I% z2(s%oIN5jKP}U3D*rD1XF4~>ZwCR7;+D%E~G7LB*ILEP-X~Ea2 zw#3MR~ad>M@_^$HWr7?ha#emF|&l|G3rT5|iP zqevA?#k^hy0f*+Qs|$>T4wU8@D{ITlk=A+GV?2518NTugTP3}ZhVLi0_cXJatO@1G zc^Ss9v#VGldix#I07tI#Z?E&E2y75v@qkt0!rXrxZ33TXsO#+3;MJMtThQX*z$p^X zagx*ILL>hH1~%glOjA|`_Bz=pi&q;o)&7_6w0Kh}*cxzDh>5W)Xho9Bxf{D=dK|he zG8Qn+dD$SNv$4(iPa>P=i6=^Z8(M-dIC?nk+O+xop^5ey%zV>Uu&~{W<&x!yWIU$n zce!nw80VHmUfvsh{4C;(!VCvl_)aj~XJC1{f8&ph90d|Aea9M@6}%J}Zm2e}6q)qa zOnaboV17=};)W&*W!Y~6#WRlTe7yU4-uJXVUH+>!d>J~2qt5)OQ2W#H+{!0)XYPTn z`TzGYH>}+AwAGI>a?NkI%?afmeb>(Bh)ifyKEf!vz@zl!zk>EgVUdR{z8}75tqJUi zu~{sTm*J$qx8~~_XC;By<)vC2Gqg0ERd(DGn$V=E;>a_{pVxp76%wt$mQPa|2vlDBt74SUd&l7(B(~V`xWKNd`p^!ty zYo^&AakhOi@5o(e+Xh9uD~BbfCBAJGJvyJ^{DhF1<%tJ)-dtRhdw?y&fpZE2+l+=8XB_?sewd?Ia#-=f zArp}%F^PkcZyN9X4Hv%QuF&Jlw(^jmfuo8A^R>4VJC__1(qYn+IVh-es3G%!;2%eY z4-6|^4#~?L4HL{sWlN69G- zOdihiB`zv+oK#&L%q(wNCbo17wp%?5D*SSc`&Urx&r5CwypM9lR)>VWIUA?4;+oj4 zX4@;x_V=0{xK!*!mN?2iacoO`ZyS^S^TP3`k!pJsWsY%(Y8>qOb2H2%JbcTYw+jvk z$Vh!>^pECQB(s+FU(8wFHM6v84zkA_%Ur|BR&!uwO{3NWCvy%ay$4Q;CX74=|G29T zIxNxk{(gYr;#r3DoC9}r4hWUZUB~LgGS5+6HZ?Ql0K1QWVLYVS!))ro%*G;6xaFSiTIEMSpFXS= z+rhhR@tWgihKk&lQWqvPh&nkLuQ+T|@KCI2p_7n`Q=D2Fmx`2CjLi*6MV0BVH#j~H zRdLH}^g&RHe>(aX92HRMIe#LMLnrFvmIFc}2_7a%EOQu6**K`w9*|tqq_yF& zf<%yFiwj$c1DlC=S`Gt`j`xMg29?}nsXnu0be75(I4bBg&U9#-5z&@@!ZX@(HoGlD z>OWqG=*X))46`|RFotqDq&*DU$LLYT;J`V@Nkzkn&9hnQ!&%M=Ve(rV_+BtXWwkt9 znB1$$8zm*@E^+*!>b&l??yomqb~|zD^+_K|n?t%w4%?hzw)L1}%k#wc!Ql?EC3X** z?UtNw#%%xL&G~OR=bybw{g$=9E=JCWRpyJNXhz7!dx?Dt zs~7gNMzXy0*!wKV*4BPvTWvnsmQ83eP!J^eE}V+A~5COe|hZ zS~Hkr5)Lq^1iZg<{;y2~SAi%`$$`@aH@J>Cu*W#CYd9KPI89ukEL11WVA-rH!lz}? zY&4~TVUB~*lqRJqPDUoqao>)y)J)SiWMI~55SqTGdEQd1ZM%5=}^*wf6mxz`TN|J1;9hJk%SI%9xXX3ulxEeGTu zIBE&K6Zyixf1y!c!cpvkBY#AbvW)|izyZBRy;rIXat0lI;Y}tnsZ1*xBp)yauVi5O zIq7i4?KfiXizmdr6Ut(0`E2#>sm-&X_mftwJfy>Ura^wrA(IKrcI_V}>NP&^+IW85 z#xL)frPzsIDx;zY1Zirk%#u8fS&R!j`v(C{PZ`o?_>;W@3m2`0RMxWkV$yyf9J z{jp))UXdwRLVWjzSPC2zym3HUrjc`w|M!OF>4}BEcAVDn-}L>D-hJv(_|l zv1|=k-oRDh5GZ|sf#m>$gaf0<0frs@ENc$R|7p~8U=lsx5Doiq9JmiKN^fyE|Am*^@Zll@hUHy(y;n1gP-#fu{LQw3~{U?ueK5SA6 za5kCHY$;JDp`0&ydez#0#b4err*NC3d~SANHnCrDLM5Ro*uqKq$-&t_!yI=W;oZZ~ z_G00cuq_Ig54f>Ax@|4v-?B~QhC|89GG3K~a(^$Xt#QixfAQ#_i)Io{(jOY6Wf)UR z8Uzy>nYN|WnjRElVA5RTXE2oiD?Y%S;J|yKQSrzN zlY+x43zl;`{kpGM;f=J4|k%b%z&dGPMA*75u!N4K;3rJvNZzNPm_ z)?)dytvt=E-aeY-79eS*>A<F0h%TUa#u(E?aBQY^=i?TySUhOGfV{j@v6v2c&R#2!G`+Ihw`p$RFU? zf4J{jz!jN22W3(YN^CqR62d6oa~k%HmQgH2Il(r;>;F+ku{Q^+FC-~A zG-041?H>CbJt&rZ!FNE#_<<4hLo*WiDZ0tZ0zWXg1ei)w5w{t8ifa z(jc#LP{yZ8-efD|nghxYnlu9r3-UB6tza-`Y}dUg%^=Fjz`?MD`M6t5mD%41HP8Ni zjaRBTd`&M-xo|~Tq18mKQ(qWW1kBp>{VWSAESET2T{xUH|Kpm=R_iA3So^-&#G`p@ zOjDP6Q=Tf5)EXCoEk`6~aQD``o-S<_$Z*|kUEQ_O)h)K+?|Iix zN+q|H);QgrfB)xNCoP92=@0?up8ImHOqx0A3Qz9u{?{n4aa4ujP$4su(2)Z&Ck~4E zxUv`AKj7_9v7SLDpxKOvNjAh<@eCvDpObkKt{Sjipq!QD_zhtw1 zZCt(f!X2Uky{W=huh%v2`qLC|^|5|c?DREOM!cW)e}3(HVAcgj6`R9rU7Sq>%vQI5 z{HRoG|Id6|@_NH{*R4|BYmGBv95T7D6`$S^&(S86ec(&C+6I=iG|Nju=Bc|o6;rH!8uXY_lXXdNQVD3&EN?V zD=$6wV3RNklUaD^;lXBZK?9cw9}XN8P~D+Zy5hmX!|i-xG8P$yY<_d>@*i!9yv)`& z)vE4=MpN@bM{ zYQ0@Ao}*s*#Fhp|0hRQNt2|eW=}K9qJ~=TpczNDEo9b`x?jCOUpTDne{_l^%x>fR- zmzREb3_klt$k46lpOk7)vzzZM!{n9;f=$iled78tI~EAKOcv70zHw&d=P3dUozL;h z&Ju~WDt&#P+4N??BbUkV?#(R`&u{S*l90NPZL!g1@obAX%$YsH94Fhj4bomDTsrQ{ z$u1^$z=3r-)9wT&o~TI=@|dP*{VQ^l>FI-Ad0Fzar)c;d-&lgmpQ+a%MvG}$EGggQ5ynda^0 zZ8xl_e%L17vO=*#VZ{Qao&T&p-B@h-Vu|*XX#shPPp8%A?MxN&RA0Geg2TBR>o^mjg=W4!3#C3Izoe~fd=Zl#sB$W0 zoNQ#b_|U+>+@`Uhi9hFnLh}NX9S4}CA9Xxr{6E3fV8zy;IfaajQUVMfTN+fdwb{1p z;%Zd5C}h+qQ7#jtD{(jJ^}3xwM^>-eCB@;$$m-OvvW4kGLyJ__9M7DMM-{pyj7l$U zJmYO({o3b@(ODH=-!ucwXQq}{H7a?G-W&<;m>RxpA-m>^4NbgG-%mW*`9f{(Q?a?l zFD);{l<`%Gx%1yl-YtBNqh+@*Gh^z%#WB+(4)IFpIOsET2i#={4fah^;!`%cv4Htc zn!;BO?@Nx)=cG^2P;lh#n3|xLV7Vf(FS_91GHH7*jRQ^Q%Q$PKqGPHOIK-QR4luEH zu_(AQ%H%IN%J!^F+m(TDg^asgdrA1}%E~p{8PETWdhJp2+w0?cK_+P*MoWdvWwSL` zS5_n*S){3+RIIV=xzU+F7ki9*9Un(*F}>}=Hq&K-=FQ3}fydsSo7Vh7@!3q{m5VLs zXwOX*T9E%T^3TVU)3@+T><~VYxn7Zxb&`Y8jE5a|=Wcv>^(AMk74zEO8OogedJ7u) zWe#%WO*qWM_OQ2j0kdqKMcBqCZH$}(3=!fx82-s_IM|!+^H{RDB7n80pgZ#clk9^Z z4G&c&G;u|qVB}N~XW;l?!Dy45CW0Dfdm1gzRk|##6WcfMC6jp1m7~UTGrQBD z9GbX;;g&*90@F{<6g$C478Nl@X5|x#0>&QvY;2Rg{&;4y6)iX{!{o?r`9zDa(4k3r zMIm>vO*5}qH6pL?Kzf#rY&!xj&Q49V-EH@;3?dxJyg%c~gPT9qZ+=d9EBRq>8z zN^TN}SimAy(5#$ixcgRM@ya1!T<#lzzS-!m6%a@?l|G#k`+9_A@>%duGbt zpR&UDOUr~ddnQJXoCIgFX_Gsjg=mRP4ocCJy6JH+GFhsBOPgNBse>L~hd5*{7=^_G zPrJ@Z6r6a#S)p%3i-*o54%rI~T7i}`cvDT8B{ZaEHzcrX%P`uz&14jMq@dyC!|2H0 z(8;3qAxtagk<66^&SL8}v|4%`Y?9pIq&VksyJkrOtIoj&zPf)!ilPY(vzab58_!DQ z^qg}@Xj(y=@vQ|7CEJ8uIcLo2O<6K^&5;vsc4w{&+%FYXeEL~8CQ>L~<2nPYPhpp; z#zE1$2^}E;&R#sRH=I^)++dj2yy%HRyV;qHizNwDdgBkd+gYvZpZL(QKdWJ0?y&`z zOac^niv#+;EeO^OzOyCWmUoV6-KpR&>%L3Axa(S7n8>AaB)`rx^^o9<2W`dy4>|IF z9F~7nFfSyDVYQgS!LY~f?7u!XaL%=AR{C{;Nw`Fm&2h!SjZ7kp>?f8mauu~0KXYhc zig^$z_Jx6;EwRh}&Le@G6U;KV3>LY6Y2-5KI4nQoK?9S(0{-P4%>2s&{xPe_EaW{G z;bJ9PaF%h&W{-%K={uwhZm}N8K9#jGe8Rn@yVPDSE&0=xbkIkjQD{LCZ{G&jw@tsJ zAEqTQy1QbvWlwdC|&X=#;0;i=1Q+=&JV`xWSGYEZS(x#k zkjRz=JGZtLG_UT>Nm(q-{NC;0>cqah8INU-xW1e7@{D!q5$47FP8-|($qDMcu6Cg@ zty%8NguO1s=L5}aRRU!9a(g5*OJCc;X11-dclu_Ty~-w0H;e)pzl5BPVwpCLNoK>s z)$?w#gzkR(*Gj=&Ri^Qvj|T_;I<0nhfh0b&2M+6=ikY*UB(ORwG_Y`eXq50sE=FwW2)5zt$fJtb1hpb8f=Z5u5 ztnVgk3JF{k-L>V!Q+>}(-mjRPnRsuMIeSa6Xsubuo5M5b%hL4EmUm+K+Kg?apLgt# zR8{*}@$bUnN8c8#^IRLfsc-qi$BOP>&K4Y7G25xAxi3!tbm}?Yr~36juRl0;dc9$< zLrDVjOUKeSEfWdG64R+mwoPpe%r*TIlfWFhx~@uZ0!#V=X6aiyj&ki$HD~82o~8N0 zfqk6=$DYu~l4b>B{x%2LvKBDgFgO}DFk3nBEn~2~lOV9-7~?btrBe?j=Pi`wdB_~F zQ0Pemr;?)79S6ZVj%*E0AFnkrTxg1Uq}r9Ty5!u`yQ`eGURd;L!V*0Z1|}s3My@WF zLu(dHX%O&fl;S%l&5_5k`=B&m-opt3U&H#p#^skEOO(Cl`0a(__Y)3}eYAEn@jh|T z6Lwn{B+$8MnWDJdKi*466o1Zm&$H7}W(vRbvWEh83JqJ>uKK)X^f(*klk3cN>dLBT z2Le=WjSfWp+VJbrd9e$MQd=HMRxMDKI>20VcDl`hw`z(!huOtfEnregVLNh;;}ZiX z2ScP0gW`u|u1yI784YS_3q(s59lp&%gen1|t@q)3C>6VIX?4@M>j2CfBj znH(6iG7|DMJRWuGZaBlLsjt821RL{WPqP-ze+xwB9Xxx^v2o2unGJ$pw-idR-1zlO z;|`T&hEmN^#~Njh9qd_S@ZG|X|B?gu&VuSa9|BLjezNAFWXQW8r!M@wa6x*HqRh1O z@;eiy)D9{Joc#4BfS-F!mPnEFKe0{x>z66C>e*I25X?z@WwcOa=gSDC6{<`V1PUC3 zxE6|dEVQ$Dz^uk7uC~xJ%YpS8A9I!hXBQXus|Oq$JGd0>o)$gaFmZ>}xt@;fo+B${PAWWK_ zGJ!9C(epj$e=A+8`(se=vw%4u_0YHUcM}r+u})&(k2z3akify`ay+JqP3@ti+QK)_ z68K**uw8h^wB*6M7Yq234zPVl;9awdSucU>7XzbZ!+{V5p+^@@div%nJp-(@+^xFv%p z9TAvx%;1a4=buy78|{B6ZDlN@Cv0pR%xnKytmic(y}(R1LD|$NLZyLimNJ`8hqF!wTh_k^Z*v%;tR67C^s40^;MF?7 z#?iQbU7mqbqQI)B8o8;00fqu$7bU$8@`|}{d!))3aX?T>kk)|s^|A)o^#*<4nN z_$_3;*BdWzp!fo7>6AhpZZ(4yFRo-U2q-B^-+CB)exa;PNr=3O)BQ`*4|94>Y}7wH z(Qrwk%o0bbEvp}Io4_sjS^i&Qkl5OEjSqV+cuUt^l{%s*Ki%Ig`HE3?iSg7@x5Ede zCV4*hF!o?fF;Hr_vZ>3V^&KBCkL`~SA?Fx`?mg#8SirX7f%kO=wntj`4Ia1{wYcap zFv}ge#ihU|vrtGbQDD*oCYdKpX$;(V6xgn8;!0vzRwd-PO<3Sxl0xE9KjtPO(J6|2 zS_@^g7D^g5Fl^If6K*h&ab!z7z%pS0TZsZ&PXiy%!a%D80WU{3AA!=OCmkg!pBO^i zXKYOHYUpY+VP{AbSkd_2Zfd%_sjSMCkk^;KehTXORph*Mp>wjS@4pon);YM`Hx%Ae z5abs!Nv>GD=Y9Cmhfk-Sm%bw4c2v>r@IvGLj4}rlWzK9f-rFc$;K_qgQJ}v9-U6}PT_Yvgk)e!lE$gYVm@)uH@r`LFsVJU2?=nnUW&By3 zYzhv{QR$yoh_X#d;9cdwckKc12L?W+gOYD9zh^oqprR;n;3}8i0XC(BY&r^TuNv45 z6s(OG@c&!DZ?;1E%|qs245E)7aHPHEiDKZ$P^!GRh~KjADy>ANtn;krJ8@Gm<>bscgF8Jk_$~L2cGvPT;+5uj%M8OXYSq(~|XBhZS zC2-D2y3KHqpW`9F%t66PjGOdUOBOJS&p9YopeTATfi)>X;Kc()CC6g3( zW!*2c1U5g0&c_PO6C^~{&%6k^&YR-EY^BR;<-qFo>0gfmU%*3=SqCKp6a^R*1$+)l z1|@QaC2*=R3fL{+QaLDjgTc{2fzOMPdxHY^h6Jtz1^xg={)W8Q+xR?ZtzlGJz^~QF z$+ehYY&nCB0~cGP0Fz@@P9o>6&e&_2j5k`{K6NrCXcudhvM3}hIHJQ;B_Z_eN7efs z>F+J1ze{`zx%zJrqjBcK#tEjsl6hJMWEhtmZr`Kv!SFhF#8F4-UDu3EgN;oim;NuW zkH59_>NP1oM{nJX=U$e({C6t&&s1DG^~yPa=4lJ~e<=tk9khOyZWA8N_UQm`1p}MF zhc{LaEVnoC-D}_~yDqR`C%+v7x87?jLj|@~44hgEm~EKYax9ovDTw+til{gW$sH8g zCB?_(u&wII-G8i14;F}gc))YVf&G{-dqo19#WeOlh1SyvaSRXUYaY;B))U*dq(k8W zgAxOyh=X3%pV>IPGPM6nYZ)S!oFZ>X>Ai3eJLq}s%_I5rKUAX&S75S`*31! zzBpfC#KJ$@3#aTzp0zagpH%L}YfHU3mYw}3eeIf*$3eDLs!^UZuXx>@ve}C{tWvR6 zDm!QCz6HE&W)C>tgz%kfUL$fY80UsBm_>M+_2@eGCC9rkfkPNyZ`H#WU z(jh)~!3qgSN77A%dh?J0P;<=}%)gNt~e7>>%VwmyVs~jbH-oC!O zihu38=v=r)I(yZ#tZmBS0jkVOOd|ip7`g5K2#PuKJvlH#m61P3fjNwUZB2rZlfFPw z!=Xe4wr2}O*p~C}S~bfrhkuPj&N2o*Hb=I)LrlLI_>C6ii_C88VC_mW*!GIa!{7kR zqysN~PO{V~T()}3;6FHJb=vGA2#BVWh?CYgJJj}q8eT>iOO9bmh6fGcc1pWc7oCkpXT7x0HT z9Dk7J<;}2;f$hrIBnA@(p@76yAq;FHjhv?%{AMOFm}JKZBrxqsib?x$t=)yS*@;0X z#UoAoeu5L@t;ieiE&g0?JU%g2R_m9(epOBSZ;6oRzf1rAo!)pcNW^GijM2_mqx*F^ znIU&x@B9D$-crVbrJ1dGljC?>M; zN}qFy3embC6tvXqYvE`2K6xu+gO~>a&Zp(9LMQxTWN~J0H;XgKKT%EKo!No8iB zbDy+!!Gk>)8A2CwGD@iUOi*BSYTlS~(k)|w17nX@L)t%&;>+r{W(IXXvbb!Z;K;cufPsb6;=&@Mr4B-81uY$#E-9%8{9|@z;8WS)5YM5o zz=5gJbiwpKP7Q{e{DN~9G&7m5S;)f1XVSo!%6I4C3Dui(4y&c-%t||#S+q*5*JSNp zv1p@Bvc|U!ciau*G2Z(!>8$Z?f$z~pk75q@I^46d5-2$n>7KcCY7u*1@&30D89lhB zmd~qvXL{wHlkAKFCvJl|8&)w%PWdp;gE?gbqi{EiP`ek~zlR+%6%3Qu6$&4G)RwFJ z!gxZd&!wrI({1g=GrE($1RPd5EAXjNaPpK%t~|9fKDqPCe_`yDF?0z?V-ct^ZR{29 zv78pc;Id#kv&57pFXL4%OFZP$mn`Zr&f_RJz{sueVEPmR8HR?0pg7G`zUBX>2a|?DQM2e|@g}vi78Op#JSO$`jy<1$#vs@y|CZ0^*#*ZWx62gU z`Af+zox#QHXMK6$=D8(jwl>EuzLrq!$S5A;u&Q#Q+mW~`#e*AM6e50zpOAO2P`Tp7 zsabMFGW|>en{bf}Q>U~+3PUrO){KDiyGA$WGVvHmv@)`XWL)l1jXd-5jPdpw19rWg zXA%$V?zj=;$}{=SrxQZrD;_XJSQrT$4Ck1`%GeuX_P~K<>Vb`IMqP|d9ef6^i&+?| zHtyaqg~#B63Ui3ig9ZjZi$^{tsSyQi{1g8)UP}>}!f;TkrNO6n&b7(LeBUmv*t6;M zKb_4o#%DI23_D|VLLvC`!ozQMCOFtdbN#x&E#+`cXyv zpCgG|rr5Plx|^6;suL-j7ojX}6v1)rM)0&Xu1gs1=s2up+SuhF@Q|ahz*$>SL#QL; z0gJ*#2KL@0M=q;>-Woef92hMUm{k-S+2kf1U^ZE}Bk|UQBK;|co^m{F)V5g1CwHUy z$%F>RK)VE1nHLW1Z66r;1QzJLRyy4$v3UvG2?Zm@00#z^h9ecxW63b@%A+coZ35#QH)~_!1M>#L)Et0?R zX1d9{9VU}5BxYK_2{(D}aXiCdYM;fL#%`%8=?r|1bD8fQRqiO*#K>|WjeQmaSB&XH zj-nY%K@Tra3|CXJcXQ+jVPv=9?J#01-f>5lFR_(Rq=8L$kE?800f%_l1LmK1ojamD z+PTj>6gxYkiNo5U)#}N^e_S3CPK>K|hf6mW>(qBPyuF}gb4=xd4#N*_CZP{E^tc=j zcT5#h7iC~@z4L&jUc*tw>);wj4h1E}4hPN(hNz+&na4GwZ=OjDyLdHe%T|}RweG@- z(b)-$10N}97U*8(D_Kyoqkm43%6gxtuL3sJ-P>`@cv(OrQ^*0uNbi0otpyBUdR{WJ z>~LV8ae-Yer<~2bqfuM&hr{wqlbk*Ya)s)4#C_uUA=X*VrpK|6lbJ_Rc>4|(#SIvY?6igXS9gO_%JF-rMZ3d;@NvjbE#mn1Vi#q zgQ?FnmolA*Vyq5im=ZNXOju(9%Rk)-j=W47Gf%#8nDy9$E$3ayEqV6H6V7F8Cr(_* z=BA<~Xc7Er(%z4~3k-diYF*t_(EQs&XaALnQ@Pm9N+z}{w*0)Dsu#*|F_?B{PyT+Nn} z6V2OMaV&q`#+1AY<=~l4FL!R8^G+vt`JKvt*)~_eDX+1^GqX(L0=2c@}i0-cnc_IiqRM>kUr) zYZh>adu-v|G@(;-#T~ZjjD<4%AA7U|8qO-$9OQPn(9o>P!?;cRdHkjw8<=0bI4oZ` zqfOMUmg9%ihFqUt<||ERn%z*iSAtcte$j>{{a zdbpo@x$H?^@GF_8YB6Q)ssGB$e-y`L98F93s3&T`!1tnIjtc|ZhfD?$r=?Dno0d3! zGW7oXFhp$uqidjuxuc?t0;5a-Yn=hdLU)eX0QTw)tp74yIl?b6%Prt^OJEjdV44+B zZMA~ajDd-ffyu0ZHQa!O`2%D02KLwoP1Olp#RkRB28{X}m^3FaaSQkd3q|gFoEY=O z)hWF;xT&ODw9S*S{if)^DlH%iwvN-kwrk<~FUY+M+&d0$bkm zvU_Uf-L3;0lavu_e*wn^jjGKRmO2tuMqAi_mU2{2VAYAJs#b80R$vV` zV2$0t9zLPFbOGneG*_7kO!^6~Ne``g9BN~4h_iiebyoi;aaCP{^)SO0GqDa;hF>a7 zaW2YBKPzc};FvU_G_NDDB%-~vqW!1?PwMjakLLBA4)vJ_`a2&>A6m%UyNLJvMH$wL zj;V*rcVG3`{Z!_7Dwo`c%%pFOejiw8Z{VDlFfrx1AbS6VYlKJkwqgSXwxrlEQyU8nzOp1KyD>c0U=UoH z&8XlP-ePr&ol&!ZW7-9t4<79)=KU3!{U6$Mf39H5IMDycg01aeScXM2?~Mh#_Y?%K zCZ<-DPxv65TD85ysFb^ny}<7Q>l_8Hc?V{SzvP^AfHgjVMf$=-z6!yWj~N66yd@RA ze>#SE7qGY$uox7u_-tU@c#}~+fI0L7o7V!7o@o|44@e7@bGI-|*(Om{tuQ4vp?iY| z*H>4rV=Wrt3Cv~%p?Vvbv^FpqOkk36NUVD(uBgT!&*s)>=+F8&vA(dkztokE+#q=*LrhoO|IZ}{Y(~8TYC(SAC z21}i1avxgBou^(n*?@EB0*n3+&1MUjY%k0XKEPIRf%%?iV*E~qZHwo{y^M(K?Df4Q zp20caC4t3v0c-Ut*4PUyz8@H8{K#w*WE6McSQ0S(NCFS*t`*%;3;MGbe3;JhW5t3D zh8dX#Y)%gr@crV6i^?!xobp3_!kNOn-p&cHww1eop0UJgp+5st%!8SlRTIT-&YXRK zbIyi|OFSKHoV4CLF7{((_IV)hTfo+MLB4wfN5lhDw+~GppEAmCP_&m&vaw(kGJ(6*$kNEA%5?>MaRRGtz}%~yECmki{RNyeHZ$=#Wbe7$ z%iZ7_H?#NU%6Ziv#oxPhx_(&hw}918fHnRCYrFz`{0CNF2Nu2vhTIUz;*~uAI9DEV z;AGvkkFw#Yxl(MTC-$VriHOCHjo&4hmomN?2CBU`hLep0h1W`-Hi|xn!EnEAEQ%`FhSZJFrf!VTmnhVJ&N-7HfzB zd&$2C>@5Pi`UYYw0S0Z}NxfequDKgT&UEegI!}7Sa=zCrQ*X2Q73>uG$r`7?-eAS> zqS0#dfxf8$3zm3uN>}faI?XZV0LNPEmFrjcZ_Hk);l0~w!OGcP%bX24->$X@w@EnI*lD1kN7fuqHN-6Mc`(ghwv13$)s-Y7}`$CA_Xw3ho$ zYURJdQuBfJ-_+d`r=LDF(|cl#0IR$KgSQfcN;%iR&RxrI@3MZg>z3H2ob9=t3GDTq zY)JvTEiNpW-Nh7Rwf^FzO?w%Sik{*X&kEXCx@uxds)wy^Y+f9JrcoPJH{m z@5Uz9RSUQlI&dv;IMFVB52)p%x@BIbdD;uPzsBui6%>AN)yF`K2 z$br3*fmz6bach}D0t?Pzj^2iU9K91buB^#*lsWRn{76~@n}#+U->f5EJq)} zPBT6rz1cslqs%-{V`=nal(sJgON+Xdt%%N*1S0k{S&y({5f;s3`a-7*(*I~GjrL?5>~qA z9P#McZPCEQ*TvNGfMasNxzADY0{>pEyex3w^&Mjgu2klXm^a6wS%1oWi`hHJlIz_A zz7HGDzumz5<^t~yAWHX(W0@6=H3_oz(4Y(-D%enBuiR}}X z>aqB`6d6Mi#au4XSa0g&D=M4 z&RsZI-=%T>N?^>)O}x7g%$)P`#=i&r^B(TWd9>%>n+eCZ9N?UN;p%5Khaw(_De8{v zK6%MM;J9kQJNeFu<~3Y99`Ni?;A}BqziGtTC&RU6?d>)N4wc%AD-BrX92jLMurG7q zz52kSONOiM0Q-Ul?sK-mEoQtE?-a@vT!{%RKT&!JdPELq@|CZHAhGXReo~{KPmtwBnICDR{_QikR2N?j57G?S z>-c5(6OB7CY&EfuMs-&94 z92&sk+;?$G&BdVgT&LIGUb2B*Rf*Xwfh9D7W6whFrZcy)@AJO@z`L`6W16Rtk)z2bRPCv*s=t*SU6LaQ60>60l^Af-3;cGYE4Bc7tfJO8G`}PFxz6OrYf}g*2 zk2vqSrm~tX?ZeNMgpWxHN0JMecxN$B(z`zS9Gl{LcCLAaAG$f8`((tux}K*k)1k?? ztyFWul!r4v@m`7EvcKW;hX>r(H*l+-U99LH68BHgs1%U~e$sxOVS^qAb_(+NGBZcvmuT?TqDE@_{{Afi+2+Ro;Q6 zH6f#$M_>U<ngSZsKpbev7f*-Ri|;Yo3I zKbM35K2PTjUg|Za(PbRq_$F-5k{rb9`8!;E1%`V^cnPKERtL5b--5uxWJ)LBoJIi#>wJn*#%XRPE z-~WJxpRIkJLE@1|2RgaGzm>KwTW$04*VEIR^rEKKI;N@j#hRbkW42VrAYldplDrXs5^BAW^OBxbihJu0N!N1JxPI?zO5*izr|1d9L z>=C;0fM?1aiKFuW>Mv|$5lR;GJf=3YLosb)sL-7hW?hfUWs|Hsw6p|Gx`Gb(EuB-u z#k*pelyi^ardwUw>kc?A>@~UJv6#=c!sWHjHnm1=y`5cKyruVk`*d3Rzy;3d21j_D zeKwSF{C+koyF~El5jLX{c0GL%$Ve>Of32~uc0>6D^> zq`*YoEXPSSTI*u+k;PM-3jb;5B+l{T$xA@m<>F7n2+VBVd!xjI+Bw0QeREL>xhcqdaruDtH&(aF9GZY*eE zm71}+fyJWCfq^}YThjk((q-R9cBKQpooO}9eG6l{wEcEGl4%v0;?C=uy3oh*rHQql z`pqlpFZgciiNC(F^Qpj|11FzOaPD!)b@;bRY=SbQ`>7ucj9M0&O9LM?x=)WWU|lMb ztmD%r;VQQBm~4XM&4kX7fQ2eYvOc>B9yJNM^UZa2Qct1V`XjQ5yv4%y+udz1{9>Pc z+_3QaiS#Wqj}@tI{m2$y`}^zn&qw#&{O3B|*X8Du2Q8YyX{GIoeKW-U?Fy5c8Py^Z z7$?cTxY1~79oFk-?Ym7OmAztv>g>#rokw@fk&iVIaWAa966_@&der8{rO%eu3%F;p zSo!h>9-UQo?H0=v$^s(rg}!xn6R6YV@4>o9{b)MKcZ@q+#IWixGogV+f$8AfH#zFI zH4j1@+siKau1u18Q_-%s=p$dr0%h@5ixyG24|Z%%xENS2{L{F`v_*cM?g0ii4X36B z23>wCZ^ZX*=rO2yXfXX(ldyGW6O*69Dc3ndnqqfmvmQOLDD;#EPxM8u-XmA1Dr6|} zdM#}V3an_|XwWx3JyCr3jF0OQ`6d^YU*tAA{BXKDquQG8hl$2J3pI5`4strU%+-!j zH0-SGOj@kz#LD)-j_ZUc<1>?V(VtrCrDh7Ia?Fv;Aq|{aKF$_Xl1@}MBp-Y3;=6Yn#Ym5aM-6Pc8nRC_-2Hu-W#3n{{W_4?yL6%ww|K#U20bPQh7uQLiG%TLld2mS z1QHk|D^fPwIC95{2ucWfFG|f|T%(cVz$oA0qA617^&-lf$FDK)U+GbW5P`OW77YnU zfl0Hj2;G{bqor~>6|(8sCy8 zF&C9}N0iLCG~OKH5cFtZQ*-2Uf6(hO_;mC5z4g5hal}i;c3| zlG?2`7UlN+XjT;4$Pq8jbmOjZh6;yQS7ym$*%e!lCG~G^=N52K7GJn_Lb7!VqwJ(? zCW(gYvo73L@n>7N|3T}UE5CAiO%xd97|tu-ixII^IohOCbn1RWNweXD*()As&EeRY z{Yrq+eG!>c@5t$7eSSA`pXFC87&=-j4Tr-c4b&^ z6Il@IzS^ykp=374Uf+}I3pRB68=Rgz|4J}}Q~^hD1e2ii%xjt(uZkSFYkfZ=(PsbF zY-Quy3J!Qlf?4h0gw01WS3{IwAgAaa+Os%!;w3B#^I^; zVH~a6%C!C+8;*tRrguB8XkcJ$ z(4FvC|DOWGf`(NO`x&0>|0U7jlHFu{!ZsoEfX0O;wFX9335HpdTY`76cxAL4Tr4(k z6SMIGy`#Uy)(RM}lW4AE?iQTDC?&x3{@1$5Qxlh;5-hIVe74!x{fBDWg6>@_=A6FS z{o@tewapfPs%$qqt<}51r1`^4S))m9hpqaZ2G7-dd3Q9hS2QqfaJsVLQ1>PiW(5ZC z&xal`I~&$8@XTmVKf(6Ac#1;?Q`e~`y&bH6g&c{(%j|%wor&@u{NE zVFRPGK@;BvrcWpMVs5nkG+vgpi`D8y!-LQLeH)}!J2HTly)ZN|bnIsQ(J*=OZZ;cB z|9c-NM>=2io>*=+0cxmcBv6=Md|rgRP$f`N|Dh zgB4gkD_BZwS{xNv%9b#fuW7cN(QLkw zWx>O~(@k7P0?jeSEbbL8&LS)gN*bRuSe!DNPOWNGu{iSL09(hdR?iR3Rte3TFBuL2gQ2YhxG zES6{b?4B@N1uT{{XmZwRamisXIne5Jfu;72tLT9S_8W^hKkWUksM2TQ$Ff4o+ojHyVUjoa4N+`ga2Z$A%7OjZ-NGZAltzJ_>DNcO0@`D#)KO7h(|? zdtolLK~TJ)@x<-MxGzUdyk(la|1Df$<-KLc!P3U=)w2|{74~iqnRNB^LYbnT{0D6X z1p&tz)CvoD3JU^cn0SsIu;2;}C`nMuyrS6so4feNoQ_j$u?~H07A!#znzd&zy%*GY zcfr);K;t{yvR)SexfXQ&{(Uo_)gfke0f7r=dG)iVPib*sIJ2VOj zGzzQ;7L;fdGH6TT@J|--{~U5ETt_uelKJ38^W)NrYhoCs7MvI5X=-@gsBpsHH)8Wr zFB#FS%`3CIj|U!8QruhF*~a90Aw?-L?OzuA@eSS_TfH5#m#G~LIByZ!eb{1G&((-j zq3+^I1urhtY-Ef1(05v;#r6i1@{MNGs>L20zD76prA+PGR=HoUbgLY1i*_@I@(X8| zHEo{)*^4sT(kxn|4cel2v;=e5cy3_v)VS{D5bnL9#jB#Tv@(3@83%{lmWs94tCogW zg|fIbw53?I)tR<=N%${KVd0hGS`wiu|6>}zkybr(w`c{URKS(kE&hL=1ZjD!S{9`x zBIPZsxqhWmOS!Lp-c<3bgKfc=+j1^!VENjXmvP}IS35_knqze6jMNJ}swM{y+>|?{ zFYuRJ{byvs47MjXm~|DRbPbr?7j1o8V0yu(@!#9vYr+D}b`#dTX$bpl+@g1aZ}sN} z+ZUz*J6JzewwDC3XJ)je?r9D7hz_36vVBI2H^-jRy(~ND-ri~2=KUhr=S6U`gI97w z+dk8_e?qB~y%JoD}6>s%|Ub-oc^PuDmzn#lgJ}YS%trm?^rtSb&FnZ>)grRnE{J zb;f(YX9dWenjUjiJm2j2aD5= zCfx@Djt`hUCiGog)Lz!nwt81fT17W^%k5HMS8s#Xpbh_8f)?ENT5)?fYs5#H^GOM9 zNfB&m7Hz2lF^7E<4#mdQwX)ThKB(7i+u|wxIfcbFp~b}@!g_%1>bLIUTyOOC#?^=%*R`_!$@l432~-%yRxX#t57(1VJV~y&vt=cx+m9y)|4Dc4VVv_-q{+aU zQP^bFx(j!H)-eA1a_9WhQ(`U;g}9oIcs5Jg+%<1DF015DJJ33DD%;KrZ5b9fZ?3&7 z@U-psEN`dmqz9!*>ViFWjmy$@>8C4gxg**mZ`VHGwLK|`FJ?vCo-J&-A*~(`i{CPy zjgL-mcob+C!IH?#YWILSESqV`+x4~ura>IhRUYi$c=s-n2`}QYDUON$t=sA#_r&|g z{oPwxk_%q!>3#9>N(yt?6XrIy!(YR_>YjMNyY71}I^|=E%YlSIy{^cZrAL_+*LV1f z7d#b}NEGUODl5PwJtOs0^(qM+adYn-+a^S>Nn+b6@XWfq?aa;Y;u~`o&-4D5_K&SZ z!K5&t=hrXpv{wNICt@S-PXFzclz!-H&CkS=7j2n0Bx4S^#r|*)>S*yT+$#Kdxv+Eg zxm$g2EfhWq-g+3J^Dbb;MWNgEd29toA26#W&LjJx_f36}v>=1I z>_Op+>xZ7ca9@^jT(!+ljm>QViyKF3*2L|OyZl)mxVmvZV^hR?YaPCv2FE^Yqras(B* zDub6URJtrB_;9+}+v#FWSGJuue(f3+lvu22VWzMn$>;872Vcv>tEB#gRGyM4mbiN7 zZm!zTw&mMCrU^Z==Vq^Ycyoi2NxD$a;%lM5UIn}m>-V&5Y3X8_T`0%0=^gVao%2>2 z=AWi`DzrE^v_vk}X!UwuKBK)Pqb;RE{KwSy2aI1BpT8fh@WxRt#ra-*+1cA=yb()S z5}52ioGQ!Vth+OTEtO;Xhae}_i^~EF1eqi+6kIN26io?PYmzALljyNO!n<^{*sNUl z4XbWlb4*ftRsB!$qG0SYf!3P=+duB-NSa~$HSO8WIiX)ez9+6L6Uv{#7j4nk+9CP+ zrNIlqr9PP=b6$$PR+|;kEZ1Pvq8k^L;+T^;0g_W2XCk z5q)y>{M;*%k1j3I5>L|6msqwcepT**IyPq0XL}_A?=7>CTee>DSfRB4*D(8}Loae4 zKmEvew0rjk?oE*$o4&a{T~Qhuz&HQ#Oarf0H%tB?57rlfivt9t%xAsMnUP#QgFQci zEqceQeSWvMPlztMnj9Qa{4Z;N{NHJ}y%NHYlwHr+-kIXamgLZuyr8Nu|INoG2`Qf6 z+1XRe>&rzdy(BbKon)D-`)`O=+}INSdrSJ$E6Xw?^WJ)_z0!2rMrz92EiYc~@c2;} zP?3~ZvH05C;tS$3pB~mPtWc713X|tKfALxU%Q?rtygHP@UT6@?mK&+FkxfT1y-@A- z%MZt*)_9kQuW#S$Q0QBgoqpYWMoY=s z77vFU_YY-W4sWHDz6VxnO}*QW0eWFta8 z`2Q9U`2Eqpddu}Yzqa_FPd#{Ht*V*yzY5zK2Uo7r^4X!)Qs1^u{Z)nnyTtv6is@<% z7VPJwt@4=)C})&1BS68aMTA$)<-385c^?1$t;{#`R8CG(z1iz>k}uJs zgyZN|VfA;?zIoU4k~Wv|&R&t3|`yPSCg)!O!t{< zRC@K2s&|~e?O7m5Y-RcapKLP)(Hw`YPSWF4!cf}b&O+P zc&P1^s8;5>C5gq+{>$#oH1Iui<;%6T>1Ud}s+Z~ho3-uP*}abKa_?d+ny;;G@7`gM z=+t^&B#bBi!XxJrInzy7DnH-yDb&7y`A=KlGVcPF#7)lE_5RoW75#c#ozMLI9j6ZE z?LC)yOuSSl`vix2nu>%u-*uX*8C9vu86oYb$0%O)VZ9FhqE_iPHM~D zkvMa<+@7yuGHU+~E>9^u@8YaB^Gs&4->i$dEO#w0%#eBJ zcPfPa^tQiw$Sz+kP~@T5-(bY4**|F!m*(~rle(8qw_D7ucP-+}meVG;a*8Kg+$h+5 z*6i|~lwQ*vDMg$nXFEEl1SL9sof5LFW#-k8;Mj_Xs?uRWCsk7Cnl4hFIK_jrD^Jsj zN9o)@(Y4)|Pg#Wbn*E!a+-Iw_?M70sqnH4L;!hS1BL)Tr9R>yl1_{Ox#&$j)9${Wy zNnS1yAzo<_UTFyqK0YBnJ}Du7AxS=AenAm_AxRNo5ou8o8A(x55lLw=DOo8gF$pm~ zL0(lUP9-r>Whqf*X=yE488u09Q(0+UaSk~(0d-Y&Gc^tsMOkfS84VRNBNZ7lO(`Q2 zL0M@P8Bqge86_=QWi=TM6FDV0MI9AoWo=bu9W7N=Wo<20ZC!0`HBB`mRb^ux6(e13 zD_w0%O;uw=fFB=nQGiw)HD=%AXFEf)6OJgq$wOj)=Z&_JAA7=9? z$zXTBJU_7rHw`xjQ!ghoS4SIf7i&LHTVGGpAaC1HcgsLe!!!?#@(9iJP|J!avn)S@ z$`qsAY#s|sH$5|7WqEg3YiD;`S3g^KYv)jNrvPKmbRQQdA1`N5SC0^1r*JR#umG2s zV4sL!4|mV-=&+C=-;mS@uaxMJ^r-OsFyG1)&z$7w%#`q&?C7%eh?3ke?~sh}$TZ)u zoQ&9%sFd>Tw3PC!w3_1V%98Z-tcvQ=vcmk*_{8Ww1D*Ni2JMl$Z5b95@(h<(m`sin zZ%y-P$%<$ziJn^Gx2!CvtvsWpx}?3btgkF*MP0&1myq(eANnl7DwoPFG9i)Q-y8ld7k+WiFjwxvsxp z>$2i)EB%ko^f|vK;M}DA^Xu}SFD&@-IpFE7v~3TgvPwG2D%#6xCubIQMrSp5);2w< zXs&LaTGiCu)zUhxt9AC2_V%tBGpEj+Fk!~hnVrjLPhC5sfA!+-Ez7#rES|o3@zgD= zr)*v_d+XZSyVlOwwQ0_djWhRd>YX%w>Ed~dRxVk#b=A_{o0o6jvVGmUEi;#GJGHsz z@Xj@_=1zHbb?K&^C+Ds`wDZW7y=NYuJ$w1$jpw(XetQ1&<+In{|Ni^;^~bM|U%q~Q zcIDWhj+_~AxEv|2{)8ysLmhN+Ufi-24uYxlstHn%n z>pc~sd1bZJX1lLDzrMP>*7tCi>}`>(ON%VppT%^Eq={@VxUH2NxqRK--En&yW?pvb zxWC`I?#Bl~Wi}qWnhy$o85Pn9Px`@feQ>(u2 zELrQ@Sa5gV)HheQyWRZ9=E_~H@k3+V#bQpmJKU#sm%cl&e)9G130C!gnIs=IdptbM zqL_A}fm7B+(q)3>mI)41V^soEdCfT3+@w3dO>|Sqy)v^+V%wFdZ2i`bpq#`#A&Ku{$fc*02y8vCC-KlhAWmhHr~J$>&$AZuR%v9fZe?U! zq+7aBsB_}2WqC_>$*O8r>$@J!FAOLP4=In_cvXMxZO7F7gUgCyr(I_&KKXdcv!h$* z9e$nId;6yNcH8*d-zH`*6ZrGfK!=BcEn?HZWQn;lGn+23y%R_g7VDe%P`E45LZZ7l zaM2@^>=lZ1exb z0|OSVfTT6*%MTVFX}#b5@R!8*9h^Vk6q>7qOQbnF|GIm={onL%`DZN^lLaQeQ(L7S zn^(jEcEbz--|i&n!H#f+RAhbK(YubcXs>35C9{u>>qC%%oI z^4#akx_K5|7E6P@gtdPxscH2JIdju1V4suMLY^}T=Bw_Cd}I~7U3)R)>BW<|vjk)| zFL`betiZJB5j%NBh-XwDkh*uYD=kG;Vi!s%J zE6*RB<-Yyj9p0J8wJsRhOZm>Q751+^q8+y}^{}n2pXx0g&V-Yq!iw`&)IBR%^6WzC zx`kn)TefAN zQ~sAB@nr&soytlLt62-)_b5A_-TQpH{GN#QHKu2FJzQX*pU^00|LWqYKY5~@8pfln|%|TALHqS}rlegTrh*Y=oJa=u{skqa8n*P%^H#3JHiT^(EdwFzX-?h(H+wZ8U z#TmuTFi>+B?BCFL+_ZeP(AJKyZoI_)u!PyDg?d5gk3NxX@UUy|ErR;j%ZoMmC! zo-)@$=cKz`XMW0+)?1ou<6JJv7AJU3yxyj9Nra4H?qhsU|`*KHkVyQfu-}sQR^=s zSoNl*YiC`N4eWd_r{9_wF1+^qdaGmdauSnS&xAbOqQ-RBK`pPaYJ+o{#ij|pDdC0l zKF3JBV$x7(*E`8(y5iD5m8*7P?_HjF#dVi1*)*kUYEf6>Km8f$anjCDl*D&kaMk=a zp;`XP0anEcjXW_Q7$s{OnB@x&T$DY~Xzjh=<(+8`jBE!O*mx3{m=`!Oni(`Ho`XEoVffKud026n?0tT6e?`HQLJEG(zz4y+{xfSO+Ir{6T>6MOg2AZ?v zm27UYA3yqd$>S;P#}1ra$6NkTUS6-zAm`RB{j)_o4>_>8Uu$`ylC_H6UJFU!Lk7~j2N zV7p_Nb^o^u9SvQ;nodOz|d>-8G zc+qOjRwx{G=~HOrFDBj6Df4)~{XTr)2y6CIc_#-J)s`5J<4#=%1?raA-WQe>E*HGE zBuMFCWy%BNqyToIbW7_k&KoZ#8yt+-GEwRKGO>5Yet8o^;tNAIm22s8u-Hv!wiIB# z`ix1TpjqJnqu7Cxp!9U#>m@hL8JG_+>I!gN5#W0;f$u>>%gzn#dKVak8cG8z(uErs zoH}X+KQIap{)M@KPQDMjeCR^*egDN0f>>Fv=#d zByQ(S+|HJ@fo(QJ->eCovmS6gE9YMLfYZ%^G51uRl5L_7c~wi8m2sA-Nep%>mo~>Lb1V*oJ-f3)>{0l> z)62`!64){{>b5`NPJ6&u<;K`>fqOp#mrtAOB}ebt2-n@L9e=~5z8Bgkcm|$uo=}?X zV>F?GbGwB6R0X~l4J?Vik|#s^A}5NjwCKCsXq+Nmzo^jgse{)e;V)B}9VRgAWj60r zpX_H|TH(g9Euwqtjmcr@rF%|vSE#k@%P2j|(aUq8XXA^KYtL)Wskg{Cupjd1ITYFB z6Um|bfPt5x_QH*x8|sXqFX!yMIg2HL(QN|T!|R-nBDnYb`_L9+IW27g+sqAYb2o6# zNvMllkSANfme3nk|*i4cde9|wY5BV=W)iXH%gXdrUzf2y>aH`O%<){ zBwM)>YK0asthbz^W5sxO#v&uD#l}^$FLa~}eqdg5f%oME-XkygZZYuPo6zDpgSRz- znMr^_@B_p4or^--Yfe1xVRm4U3Sf)b$#(N4_tOV__YZJq6|hAIu-#wT_Gl;PegQ7` z2aF;GY^YW|;;sn*=l~J1{FIG%FS~8#PqBe`mb)ZO#Ub z(p?fQ=3U*}GrM^VrU-9f*pjuHd&8n_uNE0bmG1r3b2@YJ1qYfd9P#SIPvI$7Xp^3)t3e;Cy_N zb9(}t@`QPNK5*}zpscGNRy0eqIzmSBN`F-pI}=9|lShTaf%&DA)r=;@oZPhG+$Np6 z$$~ePjZa@{v@WroShY00p)7Mb3!fXa)dQx82TW(hm@PJ}oD{uxf2;In zuZo#5+wb`7ZJDz*xfUDDs&Te#-J->~Eo!!z6~k7`EmwAJ7G+@B=iYNqV97g$CHD-L z9G=PBbb*2Q1B0Ga_gPKG>s)h_I@snd;GFF+?T!HdzXNI zT7TbxE$|^r($WCGK-V(MjLJqixn;}kA8IR#MLRpNC=~nXFicGUV1E7c1inUF=ByP- z3z)wwRhqhjtz`pOUIh2^?VPhFutgRmERf7Ka$wRci21jW+3>?ki-t`K2bdhcGp@a{ z$^U!JxtpyQ+1KphTzb)*(d_ghja{3sO<%oHV)f=%t9M6~?mN9G=oM4zgT<${_#Q0a z^|xLU@Xx#F<^*mH2L|2+3_1%Ig-bAmzTSLJvxU`w>F$ZC+aK`#TfoSnJpZc{uRbt3d_}y zUgavjTeo|aP66YJR}5i4ml&ik4z`?L%XY1CCbp8zt+zhpEx3i>fXG{6X{iusWCVIc! z#BH7nm{S(8Z9l;I?@R=F7N0|tlIPdK$I=gI2v(%)@7NqJ9f0jC#<>4 zu+@1Luj8qsiF0@_UtnQ$VA$nZx+0R1cSFs!=_RZa<|Rna+v>pc@d3Bn1=f<+!VV7{ zJPo9y4{#qmuyM_Ms0e*+^h>+`ycQntV;MhC)D8rtJ?z3BMIC$E%UZ2 ztlvX0JBSh4_CBp2jfJowoJKRK~3kptORc5 zIhR)QaBIyyrM`yKT_EqQ+b)%W=4IZwCTBP4W|v6Ksuma6e6)J%xf`>tKR*({)0@E2 zw#{;i_ut9;d5=0sw}#JPu9(1mS)k?+=dqZ7J$qXxFtU7L3`}4*Wthtxz^H4$ELo7w z9l#`?z$oIt)cK(IVLJEz2i%Xhxkea>#RXU|@!?t9Fn6IrUh5malGiB}x3$6)RX1H~ zf8Q9UAePh@W`4M-fhj1H>!uU0k%gu+8}F}!Ya33f$nYGzz_tAX*JJlf%XoRR3s}}@ zZ(1gud(n+~2hZV)VoYl_7O{5?wqYH_2e4+lkJvI zOr#dOoL(YiaIJD0qh|oet_I!<(sv!Zcqch9F8;+j(SY|71FMAs3!ejXjQ~dl10#>Z zR6zkI#Q>It4BLK3bH8j)Nc_M&$5O`U607$F)^`W4|C>4CezXABCw4aPrtL9~`b9Uj zzT8YKeXw4EElK2UMnl)dwu4OS0_wd7eYC_nn?zR1@jOzy!?pGR=b{VG?!V*`Uw3Nz zhC>Frx$Av%HRBF1h^*eJ&TRNW#8`mIB!Rs(fy1QW_7(L-%mLjIuUo^y_n6;W?cDvu zNNbIz)YXd~_tY;ih$dXS9LH$3fqkh1?@fUvM{n>hS-^bv8q0(P-WL=2CVt>p62R;; z;qB!IOy|;TBm9_J0?J-xao?N4+9Bh=>Wj2z0Ne2b?k6634Rt4&ZEoyv6}{o;-LcoZ z=bl-n>&0`-I&2ma0&_Zqb6#@Kys2euA+nCWC4p_hU#=4Zx3)9<-ZlYJ-qx3`T z3%DZ%n2aqhUu3&&x|Z2NfUV*Ji^;!T4iDH{71-v_V!u3rcd-NW4fAU|rE5cex9*-@ zbGE(q?Dbjc>$V!ss`2OA!eFr4xpKDr2WIhRW`zbOK7+TM3U3V<*e4i#x!J&X(}Bz0 zfk82Wxy^ys=@#$J3A|S>u$m}v+&af}&SDDF2F5IgZLfKsM;C;qd=l}Rz~UW{z4B}6 zyMp`K61x^C)Yl|v1%}CPQ_BbvwBfsK7U*!Y^@TW_$1jOq_bG8k$CU)@nLYH5u*Y~_ zQupE6@9=y3ho#8^Y)`jwE}wHs=Pjf1g`BJ3Zk_+OQSLTNQvlcE3oMESjC%h*Ft;w? zU24E)EWmNAf%nP=b}<2l!!nHao?8#;G0I<9VtJck``qe^ZJSj;EU~j~-I~d8divMB zuWKwma4dbmGf{w}GJx5*fl)MokwI|bK~_npe-28aCr`I>8#DP{X5*V<-g%Sdu7q=M z6fYAG8;^#<0%ir@*;ct(souU~$}tNnK0aE~JxMhtMJ@8t%ZSyJXZ1+@ypk=mK+iMB ztgvW{x3FH|G_y#lrI{i#;`g|$e5{%+t7x$EjG$1M&IHBO+7BIq&d0dLYQF5L?QZ9v zx1-|4t;+83Gxuz&&v#f(J~7SUP)ny^xBDCM^~NbDdK5RSo}TG@glqdLA*bUU;wA+T z{xLK;vy00v-7Rq`Z6j-kl)YTzWj2mxxtx0&Iy)aUG&OT@7g?|P@bEAzJCBUV1OrE> z7Jk#TCo3EmySj^+CYVgPaDYibfzQrp!U2arKR(KTdg8>cC%r9mBFoj4jSHPv8bx(# z78pKuYLzTowsbPbwyq79DqTTa1XUOxvnecSVA`gU+3TCPF?F);whITj6&G|UaW38w z(A3SHU$}g#sY&2r4t|@POXs+BMV;zgvLI{GF^#M_8#!0*_`#&TcFQ3zj&&w2o;r#R z7L#=3J312CB_a|S^=w~k;?~=>?22>G_HUPr`0c(udd%b0pxLt_`H|!2=`PzUe0mHH zb)4ZbYGwTQY?`Ufg+>F;F0u+q z=@c+F3nYDDWPh2q<01PyhZPA7Y-$||ZSt=;Tqf9j`|+rmA+171NKJ@CvE9AsqVk@8 zAJb4q1~~^lwjiY-f2ER5GuedfXRKJ*`j>0P0Y-Ky21TySO`g}JQxY$yi8w6NbZvSh zA+mr$Il$!Pw6Ltm&K{KoD>icLz4LYI)VX4@X1T77M5Dk$Ax|eY>D?Z$G-PLbd1=We zetOic|LQ~F>UAb2Mr#!NWFG0rrBrOv)!sN|lb7!LJK3l9F$fDwoaX6%ex{->Q>Vx9 zSb^{+6Q@ea8NMF>7^K+6V=gdqczJNDh&%6NXp!pl3RG$qD-vMY=A5(OAg4nf$GW!G z{|^rL+b-!y;%|{lbZX#W`|+UN*6U!}&Ad&TEe%r39G6d#$;q&q8)G*ip@CB^!GZB3 zPXj}f@S_84b4s=_ez;vSqjVdmw9^TO&peAdSRC?HKOKuM-J-a6V&<})3QQkW6cQTv zE(9bCbLK8I;N^LLbgrh+rW=o@CFi|#;WgcMVgYmGNkbLRmbngH+)~*k``o1R-z+-b zdB0&LpGMo8Rlhe{d`sj|ORV_Bvi(k1c8b=@D~J0gf4uS9*VIPwjc8&Vr(m-VgP z21kB0oqeFf#_{)2-GL3RV*kD@Xhd(gmrLxKNXM3`C048E+82DYUh?p8j6t$`(r zrR%jB7!D}Q9X4FR$ho3%af0(ihET)yD;+EfvlN&F3-qKJxB^_9p7)kpQTSKT zf!7hGwyAGpdpr^(xy#QqP20MGwf3d*&0kwo*nYe{%2#=SS>oUIO}`E*NgUTWW_XIR zOXtY@`o7K{sl3FQ2|FHfICeL3r~OM{R)4ZcdPzc)oR8wf@(g8uuY@Kc7Dub#ClA>A z4!Ox1JebdLFu>u}!!Rqii96MMm>e!IU~71$*1*N{sCaP$uPIkz<2BB_0%3ztW`has z|6BZ||IImJZo7R=rihZjM33jsS9u6tI+!N4_=O|WjSg9Lp##jB0&yRr3b_*woG#mQ zyQyQ_2F~dpkMLwFFmo6sG#Y<+B;C1Vy3m{gkh`B5d}=PbNuy!E(M+3^2BcU1=YIr3rBR{eeVimVU{@XgVo^5hh>bD82wTN znx(P|c@kL;i$y6k@}&rz*=x!ul%W!0CEfLp!(k7Xpx{h4;VK8tuMUe?181~Zl&o+{ z+4$Vy0f!FTI|0F52PdweD{UMs2F)Hd@8&nM9$=WXSmvpL0t0UalZu*?V7beZtsy)Y zp6^)7n^m*AD=|mc!fPTs4~qlC1J3>mhfi(~XD$?$kv*nsW@vMAkD-X(W>@ty1wmRe z?M&$hj+zxH>bDshRhb`}*f;A?>*@$quG`&Djv7r<@zv{`oFf={(xYu@U=k-!#RlfO zyDn|UD-sR9Hy?GoU)pWF=cF%NnNNN8$4$`(t{NX))5yDsfm77Yk#p$`w+6q5J((|< zlpZW_6G>y}JZZ|r+>yFMK~709-G_1GgAHvttETb(OZRYOsPNF>)e2x#^Kf>!S>eLU za$y&zM#Gk8n;hh%(xda&nw4zl4X9FK+L<2~D1G?`Gpm$C1A{_(shr?RY4gNMz6D9s zIqtA@IXUi*E=Ur-+^Wqkb%B9BVY@hsc9-zJ24y{!#))4i%?sIcVal7i)3sv)*H7&7 z3X%V@z|&yDK~Ap^C$&>Krz{a+*4jBGUQcy~@bL=P@aTtJ;WrNTr7$$>7J7EX-dlgL zu+))fk@7J z6N65;bDev^W^pNuk=LWJ$V|Y2nTcVZ(qrQ-iUDu8dzicB|Lag*AaZis)P+71)mO6% zShF$YuX~#M#9{$M$P4{0t3n~a4L8Nu`CYZ|&EYhia*#urga7q8hOS%zA2;r76~Bpx zPDpcFaXtIW^sP(eijUCNWE%Pix~ z+2NTIL1*Lrf4r-kaqtjFih_yM1qH7D4c;nl7nsy$EZVroZJKeq0tds?Lu@`8tT|kc zi~`(B|JH4bQek0f*u%{6fl<8SfO57-18YyiMV}doyv*i@XSn7I>8|;AkcaQR1EXEi zI>VkM=}8|14w>ECV}D*wYof|~=`~EkZ&af$I&j>dbV-PL+mwdd&qpu)T6Jov;gll% zgav0#w_NNBT)sqX+u@rMIoDVXMP;uXIKZ>UI{2rI3nTkYrbaeJLBaGV4?~+GmKld` zXkUIef$cO~3$sH(hdx)+o2CXu4~9mY3Yqj34ZJ5BH#oQ3tZ3k=VBYzAP8Rp};J^mf zfWG*TOdd1ZcF$I-y3AG{ty|f=NlTekV8Ld_4}JVMm^}?zWM-`BTGSwTgUR?p%emis zd<^!!AiJ9u`7nnOI zwD50WJk`a-6Tr0dHJhUXgVYV?rF zvC*N$i?mNOGI7YTo#3@Ou_32}Mg9O^p)vzw$HBAS%`dx@Gv^#Ly3r)KK>b9pEvE*9 zjRi~AfmWOBLsEuaw>D4wC}pBq!}*qR)mJ4Z)kq<=n~b_TjGB)7G*_-NxT0?;(QI=- z^2fh}+_o1~6BAY^aBK)u5Kj2OlASVH_r-cnMURLNlFAyqJ1+@KWh^+AZ6(dolH<@G zD8VSm&?pn2TdhPLiAa3Kq63gMf_=1J~X4{vQ zPI@nzJ{^+tlG?fchi!$3-J{!1D{f8Fyvf~g#=Y*%@n@Tx&+;_C*gb!37VF_3tpP6_ z+e4UrC3^lPS?n@ZXq4_?lzZST&e3Qy=d4Xhql|^?-K<6dhDJ|CS&sllO@=j^D;lQT zEaF(vVEf@j(1Ft$4JTtxFft}Eozh~B&6tz8m_e<8O<;FN;DNT1h889ZU4{t_>>7<9 zIo7ePXkZgy;90>Oy}GyahO4@@O#|12AO;qO71P)f_%vRuDG`vp+tpP2nIlrCflq-^ z`GKrXg@q6YBYU9|M}YqXfwcw=EHM(TcE%EF6XXuQ33$JV%kj0~+l>-Rj7Ed$p4hs-<-EIB9GvJWt_pI~0D$;$np zKgNSSQ$dt5;9UJ9(-}IfFLhe?2r@Ep*yP+e!e%g)J%LfNVbU9k2Fqfl?2~(>8CpCn z7RSD6%l*+7yXL%slK|rh1~w0dUk4byPM8(`5Gy&+D)pi7F~^(<1rtKnGH`JmVhZ5n z)M)&=X4+$k27?Ngm;DKP)o?0gNKHDl+9`EZsz9nGiGOd}CvZ_*RujzZJ&RV?F z{HFNYQg zBbNYk&Ve?Y1q`AEBI!FCe0f>^Wp=a#B{IlYwAgnvEZf@JjK|j6*jvn1JHa6Eqe1LIbC5<0zel3ugcg|tr=b=&7Z~jqu*eDt7+zW{ z!_a5Bn&Hjg{^O0k$A3EH|4@>caf`j8!BSbQeli1JgzOWJ{?ER1EUo*PJer(uux1tT zC}_-3OJJJwX@b45&vQlnl?Ub}{k`AwknecT#ihTN?CQ9te}$Q?&T?UL!tSliotaFk z3XM7!Zs+|y6TYA|kC*Mx)h0oWCaD#TM?&wM`u%|a1f$&pW*yc>u^$ah9gKVojO+_+ z_%?`1o$9;hdvoPpn~P1Z4y@iHUpSr|P(O5VdHlu(A)`|TMKWJArrBID%>LQ&c*XKJ z6B;!PWJ?^QuUbsf_5ZUB0HX zq{9|TKJ|TZG0!wLB0z6P5Yvl>r)*x`JzLJQEwT}NkZPC0$g`kdei{Fhj)sSGk3Owv zc)VkpbL^r6E7LAU*&Z~UdQfrulZ1u~ry2wq4%}YU;an(_dxF*R2a}jZJ7@96gFfPz!_Tyk{J|Z7pq#Yj7S-iJVY(=9)g2F=1Ws%c@)Jq$${$LOg zcIs@uCBod`I$bE&JQYEhA$+?bQLvEW;oD42A6tT8kM3*-J9`Z7P_= zcQm~D&=FEPfuVvyu7Po$ZJ*hx2I&TuONHz&r`}WdZa>#+Gxcra9-Rh$4Th_UvQIJ^ z*c%u)?6ZnH9X&tzxp*{v=DL0Bvc-vQ6IV-I(=-Wt?>I?_-R`4e!ikba{Q+yV$VvT!iB=eG;k6B?yQ`edkvi3~(ESZ!ot_$vHlnDs8>)g69bJxei zFMDzt7yeAtO6O=xV0@O)z^{?^^{KykS+e!CXG>i~4=c|3ka6mwQ}IKm%g6pD>RzAJ z@k#Dro|)&iuQE$_>U&h_3Cy}wz__roO6tU9n@(-%hNsIO^RV|a z%G{{Rl$ToSoToYc6TiVtZVSe?1^rv}pK%{3TC67aR&4DhrT#;b{oys=!&&;-UU)EQ z=HHfb)R--uDbZ@oQIY)O$AN3foCS=I8FS(d+K#hpGgQi?JYkGpb& zDGM05KbZ4b+-k~?)lY8_@@Nn=XmNSaY7oFAdqGff#Y7o{Gcpt8)IMBrk4h0}XL{er z6d4{U8Y}cL`I2~tNj1-}z*G8c|4L^U&)1K5yQE%NcEYdo6~7*ObF(d~lUl*JVtJ#$ zgUM^(*9lFiQse(^lfw8+tdGM(cAGfgh3#UE4;n6+pXaFfz4QsgA;~V62Ne3DZHnqyxRTQQUCi{s%Mvu$~duw<;)5J9M*^TgWJ5$AO1UY`n6D4wnUDXL#t8-stiHXuwc0(M z;lRP^?l%r(8sC^;fZ2>&a$xyT}%GWTd$RA9xyX#!cCj7KJVTyqr{v=dl(ozLh8>y3m)#bq;+ z`W4yOrcbT(l$;S0_l#9w?$HFP%!L;loO!GkxXtD^I^ociWpPE}cAC|Ov}oR)U%$;R zP^>Uu;&kgt$lwrBkz!gb>M`K}^CeI5yQTag-x-*uMNPk3dc*TVj0ejZ0S`^KnbZ4R z1R5{TbGol_?`MnZVX zMsbS|2UrDC9z0-^RhjL`#?r#Tp|@jB;t??~or^~WpGZAq5i#qS$`YBh^eC%Hph%>O zh{F!m>jI@WTc=O1{`i&YA9tvn*^J~)T{Sj|B%5QZX;Tx+B4tG;d%Db@-RgEIQ}4w} z_2YVPQfD1Vf5*Cfp6vdcOdPuP3`~pVM55Zn(q;s@iM^~cSg}pT?E*ufTcB1W%k;eM zD@(8Q8Z?$OiAo@jn~ijl9`%B;_)^$%>> zd)DmMoQ;heGM)}U~R!~M;k&D*47CNQ!!PG~wU75=2gMQm#H zI%Y|~FB1CV_RCmVU*yVK{+i_di1jufv$&2#Vz^b$4A~f?KToIgbpLdfb*gEbHE-+A zN~x?Rb7P`1S3lg4c1rfP8FQx6h6DdHI9l=o+tg;?DB}Khqk2)RV43@ACN@Er1B~p# zCCeu-b639mlAXuoz?REbo(6E-V(Z+jRsE=7v0Qcbd2?>VJsW%5C6-9vE|_Y5DzT?R zqB-}@jN4Li3nxvza7Z-LB8)vmshjuQA&J=q0-RdUyEvsDi>_O6fX!?|qoYfcxR#}x zSj?JS?um%x3h|fnDVQBWF*0^Ob{b73r&lKV}H;xqPLKO_^aHx7z_D!HxsW?jFr* z7Kz;nEQ>{JdDy+X8z>S+d#TUA~h@XgjbD=|C7 z$ogiQZ|)*q2~USv{WqWW7G3G(FiB`JiFhwo+v2L7z!+q|?UC3vgXV=s4NTTkn0VS2 zxbU%EXn3cwAmc`*il4cu>_nFZlZAgQxn!3lEZD>NPiR*}zt+#=TjS0gmpJmHO`G%3 zguIf+()TxT@b)tD1Oz1TN=&m(lLX&S5?4Bq(8;3hb4cWygu0QNPwRg-4A`s0k;s`d+gXW2}Z5A@E7FlbVma!`0Xho*<+!sxr-Hj3`s zdQ`2?fJw0Cpj4Fwiy1>;<4n6vljq!-@^hNn+*bm*OxM=-m7X_eV6tdnkj!vW@M+{- z`;|XTlJSB0wu`;(8H=ZVJgCJR>&V3Qr%3F~od)(7Wj%k>6gB^e&RTuf>OrIFimgqe zQO>nnJsEf3TC_%bNvp04hfhL^Bd1@+q1M>~t!qpc#<>)o-gn88<;cI&#yMS$Q=Z*C za?;Mt@O#Zh*{ta|PhXJW5xsoI(K*;=qVE+G5z(1hrfhCfcQV;FRqAp&DJ+!?NaURP z;e|)^!A`%NCgD&qSC87;T)rIUoL3lJ*Xnq?e>=Ey5|q_N z#nlXXqi>{qykO9gdhuB38Kq5!6WEOcub7-$H#sY~dRDvKdNaA^gLC~pM~T-suo)XP zNxB?#kGQc&v`NHO?My(c(Tqmn8^+spuRLH>eZeFUV&JMAGNIvxz}etgN1n{iwOYAE zS9itSRej8Au1exh9Qc>C(3HZbs>Il%sclj8|zuSFwY-$4l;CNqPnZXC>w z1!t>v1YIxTH87B8{~_M^bZXqiLsBZP9B+J=1~RaHIU98{G%iv=@ru);)PvG762d8s zr%dC;KAecyAti8hR&1uZKZdX$s#R&ws4`Sq_grw#3d~;(7WG8;1nroyHIYj%Mp4&Nfq+B@-Nl z7#xKP9Qkw(irr{3_;XiK=b%W!0TGi!ybBt*w)H&Uc!sB;VWQ+r-U|o#|2XjdIq=NI zQTzd)+>?WH3{C&!Iha%=oKzf`R3;qK=s9FEMacBUp?QhT=CkH$?>+S|=b+FH z2N9M-5(bW9u8eXnj&dQY3U3@0IGhw6k`)7#ReG2-CphW#JT;l3YMR1q+7oVG@_1fq zv-!K5CU2Z#f}Hht2-K?lJ5-_R;;iCg*5ky!Aep^lIzvuFbIAd%Esr-RoDd3145)Ch zO;R*WJt%b}ko|#pon*IQLzDQw=|l_yS`Gn(|~^yGd31;R8s^FJvGN2ME02BCfuRtM9K;-9X#Y?yUp!Uk65 zvr*1z_XQ3tywn)RV{(^!3Iq3_hw_guU->C4R?E4`HS>3>V3GME1!I_oBcH)Ro*M@^QW`j}D6@q)*orxn3pQ8tkCe@g{PSeZlf+BE zzT}m>S@*~`?t+|e(4U}~%!P&u%y$!380kKH{ZiDV@G@s*e^lv(lwX%tMCO}FGM}E< zpq{f#((9nejMKegd?H^?JX2Y+x-nwS(%`kCg6nz(yAC>Bp6bB4BKysq%XX21y^4B| zlxB-Jq!?AT^2Ho{{ENxpNc4$P295%^zlCh+e;9b598g^3X812FFZaxWLrD$~-vrL- z6ZxjoDCBWan4vM=v{7h6W6801(_|evO1zj%8a!lV&fmJiP$R@_;LEYB-*17xN+cJP z#k~Eg?!Iq%o7aiHu}xg3m3Z9rp*7QGPFpEX4(A{Dnv9Q5uqzVoD9aZ-@`8miMR-nk zz_h}?MTt_!-rtdu{dV`(evc;-Cr{qLb@1}Tk3YT&%hfl^hWMnuNMqE_Gg3sN0Vg6L5>Y8*i#%g$uS#!aEje_wk~z5 zfzRc03nrw!3%6ldvL$PV;ECduu{u&ljZRJHtUAx_Q{cQek1wNge%D%u|F1sEU2hb% z^hr_B6XX5FsGcY8a!`Ch>w!rQLRXmdA7txJY3!sg8)835y_lJDUQR2cc$$rbvWnc&LrdD z>v@1FB)n9&q4cehs%)=8cKfA{mhkU+9#3*+mO2EcHQqgOA>|YAeG#>v6Z{!9{yFO2 zX=3hPe)~t0^#>-=9*1o*e4H~5>VG+;_n=Ad1CxGBlP-g^cF0<`9cws_ywF|acrK@r zudY$3fbnTapU@8d%&LfJ2F-6b^&OY@-JhoIZsDXA)N4j!JWs=(nC8Re|l;K0{$fTw6SyAI=&uUoVa z9MWCE==C$yUuvs%kb?S>Ms}aXnKREnGUUCy;HLUVo$`R=v)1yLlujzU5>h5|Hn4<2 z|IZ<@k_N#qPIqteCaSrf9^0Ha4l153pb z_6ZE)FAmCeIG&I?r2n9aslP#}=YYbTCNlvhr58<_ADT4J?5gT%NNY08O~*mI<;|I3ny1fBkAY3bEvlWrW{&erpNufyeoX`Fuy z<*KV?{}k@H&bU*2U7lY_T1g}S!lUxP8U9-`Dl{-@2pm$J(P;>>;rc8~A6Gd8K$CB4aOlA6*e^(EGjgV|#yBd_uC3x3Bd z%TFB?wJK5cz4St)OeUuE%cc+i3@%C>Wp8p&X*jHqb3lW~#*C+_{OZkR;tb*s4(YvN zGClG1!gVIe32P*lH0sP_GOswK>C>!V(X2nGNk3$-rT~+^Ow&ewCeb%dW=k5`T550P zGbr^m=?NSXQE~kB>feG*uN5pBC#RK5ME^0KByv|cCVuN-;X}7J%1(3GQEx5RW4)8V zZfElom&0Z`<*)Yir0~6w7wtKycx0C*!y#Uie{-KWO#NOhWN=X838Trpvj@2kDF-y? zAD`&hTI^@PXPT@+kn_^H+Vyq@js7X=HQk8{W-GDzM!-0BuAqgU`?gK}@kJjbR_BBw5&n5mH7%i_yy z#O&THhchwP73X9cohbkuXdudCB&8%gLUJykZb&AArl{vke@(kZFUT)g+$XI^s6=$fSI zcR}0neM_fs3;Rr!79;sb&Yj*(2o!K_JkThh^5Jr;OWDJt+heMJ z_D+ba`*~D6*Uqw4I%I+~>r{8&k4MB`R=s51`F7LqC)U~_ANm9W$}CejzlVJd~b$D0gsin|#g*MJ@-0lgeFz z0*Xc=(S0r4>(u9$EEV;;a>I#*M{CEXhC?a=4o$2&uitDiJifHre zj!H))Cf{&m6Z1=Wcpx<}$jm7{b=6Alqm_>qMJ=eCWZk~}pN9X&+bbG6UbU;`7-aKE z=PgL;a{RP|u|;xL*21=BSsN7l0~sbh682#Gs4C)@SGZcVRj|&;OWEXv0yEQ^IX`@K z`ICDL^{1vkcx-(649~CAhyFZZ7SGs`kS^|KP|3i@(|v%^Zj)pxm&>#T3N89YKOEB8 zl@=Um66{e(~Prn)t3SgWGK$l=BEkI84zhZb{(gKTC3 z6L}X1bY?zrWbwV|$X9mdsB}%DNbdwD6`zAFE)9=3qHeV4=6qDK$!pS&RA6M~3TWiM z@XSkQ!U5)OTQ(k#S!SrBJUNkzS@NWg+N7jO3>@zgI@}8wS>gp+6m~c;9FtkbzQ{tAk&n@Bs|^Yg__Hfu(lPcipNb`7L=YiEdQy2-R4Cjea0|(2O#uEjZmAyXqW~xaVc}?>4O({B{yy9Z7lfgof zD+}0Ej074dn;u{)J-}w_=D@hMErDU9jspkRhvf!u7K!hB(JCbt${m;EsO!3+N&i$s z7gJXAjKdDCY8{?uOk5^&g-Wo91~B&X~#QA~!$0K_MoYFs)GH^4J(-iH5k~h%wk#e_tCn}53agd8(EbZggF-(G^=+Q@&q1emOi{; zqdto@&v7Njtf1dVjJ*oaW>-C!x!}T4^)teLs=qxHK2B`vpY?!6Eon7p+>ZXKw`ZFF z<7rfOnem)AK4AefqoLDQUJVE1C#=e=Dmx_~ISKDu!0s8KEPeApo7sxOblc~ZQnmBi zOjrCfsIH=BM+;`b2@*$nKV8swR9GaictVqqk0GyW zhyo*T#^k9E44hdOC9#tj9255}@i5!!X*|tG?sLIi2_^vr=H3%*N=q2{JTe@l+6o+M zr5PT`SeSD<#-_F&J>JL>(ZH-|;jAlhv^#FW4DoagW+{`z?!*+Q4f|FwX}KtN&0VA< zeBxw_M&HXh#lm4HoaQ-clq7a(>NuYCpLb|7Yfx-)SD5TsblUsCBj?8({(%JX9$*CZz-Q7o7{K(lCp==x39oEcQSoy(OWz9rx7m-uV zx9%M?D{}nDSK86YttjCb>~f&R|3Zd(5XW)>o;@Df0h+nS*$M1P_Xw$QqSQ}TPEOUE7n_kI<#)S49 z$}}jjnPo6?Wi4p>C%0q)d#?to9E-D!neH*Zr4p_x8yv+|6j@w64+!KWJ3O9v=-f)P z#f&Non2Z)MeQJfj*m!_qlgtFr=H@>Hbqg3ho2rX%C$Wd3UHL3mzZVM&!wgzvh1PA zxrQc>^DPOgVOgqkJzlq_aLmhC!ETqt?xJ*t?}=j#hf|UN<10GbFY$2M_-(nzGxt>j zf1Vn9mjh?bNsbrCIldHdb{*pUw%|!xpEyUOfC;0>wujOy6vg)~d|sl!vSGPq-UBw5 z2PIJsr49@%0S>GI4O}7ra``)sS4$mcOluHYrntn-QT)gPMm+~6nGX!sZpi|Rz2%zU zNFBc7puj&%f&bVFku8n_5BOMYE--T+U~?+q`Q!3>3GX^9hII}Kf_07J4GROMSS7*~ zMPd$0qy^}AJQQnVluLRe=AtN?vqs|3Mu{_qq5+Dc4;nvPt(SFtFS0C==U?v08954m z=Q)pY&gIh!7djViz!cH2^x=U3p2Gh5Jd-*YvN}puZ-2J<)PrS;^Axz=B=CDVa-LXF zzb%3Llmqvv1w3aIc+WiG+2+7`Y=MYMBF`5FfpZBw=N|9~F$(-(5a3c2jCpV|>w%D~ z14~qbP&xx!Km%(O1B(d*Yrq4xe^QB34311n4U7yvCQ1%W3XLLMjpD}|MC2Ya2{ric z=v$?re9+=Bqufeu$pa}84XleMi7ad6Jfa}IEb*<<2Bs%@EFO$9KOWqaad`hODBWto zEuMu(8WxJxDN1D@6pe9|=yQ~?XcX>PC~@Wh&!m^1&J;>?DT+2F8r2s_%uAGA?#5~T zUU=Swncs9DoO8Bi&^vWYnJXy&Yt)Opnxo<0jlHe<=h+&U|kV+$eN;1ESqd>-jl!!J4od!0gh0;G1m?Mr=1sJe6 zH85K&U_R7vgyplo;CayoN4;q+dL53RRMrbOJ!C(6QRcj$T$iG8@I#?n7eC)C?9n)3 zW75r4-!kW1=2w|RtqT-5w3y~KF+?;eL{ucOv$?3`o##}!YGkC+3s~YuBhu`=i3_z`(4*z%r#lWS1jMju_#Yqb61};2rSUAlNdNegkSWQdFd+4#{0f$$Sl-okiB8AS1!vY@` zaGp|f?QjZg-V|>9fLZ1NbHDdu=G4qi`2_y^%IS)h2?IyHvL)Ka_4Z%g=3rx8000B7Od&{dgrA4@-4~^hrC>q%vAl( zrj(cUb2#6~**+=5uE2H6tcI{v9#ID#2(Tq`pHYxVm?BY^C~c`Ik&~G6c)@A~MNOWA zLMn|SJPSE98vdxMvlJ*)XC6q^JWv&Npw@_id5(hnpL~I32meWbVwlin%Hq)Ad*^vX zD|@QGg5m)&ejNs_Nn-3f#D)GGVBO&=v`mqQCsA61k%Om^yTL(-hml=HQToFIffbBA zb_^UIisC|!Lfan7MtoYc<>1AX$9i|YT)ek1doZ-Je&|b=z8gkl0<7gXpU(96G0^ zd=dp@uJFrf@n>CPjbY&Uk`Vdr0AG;KvylJPj#>0!xa>R=grSBy2pKuUArzPmJC1d5=!(0t)Zc~`oF%+y<;Es~u zzV=}D^3b--MD}un4(o+6FHh%#5ZNQwRSxWv>FPOZ5rNlKrd z&2LxW_?JBI*Rpv6WoOgM3yZ?f9*VHYUfR>>(0%WK*}(@qZi?Jyihiw%a(@)%b}SU@ zV3AmoqP-msP#L520S57Vj1~owBL5m?7!qYB{Sw&OAhR#AdH+En z4@g*k$F`j4hrk~KKg5brcjwv z7Pxpm;9{EI8lEV+?`CW44N)t`=#xLA4H<>iK7TUEmo;2JcmBh#W>GX48ch_P=IwHzz1FZSv|o`mHCm>}r5wZu#7baJvjwj#E20&DuHVIdso+ z;5o-2-s8agDZ`RgpZCfF9y3M>r-K|P4v70Y@*jIJS#2igrtj9Xm5Tki*xepz3Vi*i z>7c+aH#11Hl>dmbc*1g-M~X7X9?I@p$Xv1@FTP6p%K@QniMz9It}A5ZUU5+1!YzRf z4njv90vHuJH#BgU9XXiUpmEQv;ODhN@^9IH91vjm%EPcQ?i`fuP?Vc+P&6)4ETOSYpHVo(QS^bM?9&;GA0Cu%`_mq% z*zve>Hk%{Y^T1rYT-yuITwfwfjz?^H$DHulEn#O*{wq&2{v&0j(Php2c`fU=KXq2j zc*@>!TVV&o&O%4dD-XnS4vLyNa&K|qHBl5c+9lR=fHUdX9;XGYbq-u&q3jM0foq>@ z_J{_i{!<7%{jI9#8M~LBc*#QDV#f8V3#Iq)i?dB&;WFjXVFca%cPOE2QiI022JRlE zg6Ka&(Kf9Q!(Ch*T;z3G@?!Jd83N^V|71L1?`T?5z$n7eD1Ps9KI2P0#vT{Xh7F!y zj?SLi8o98bu!b$Yz%#9}?O>>AU4VPP%*ms&ay2_PH%rWWdQ;?q=&83K@3S5HGRGqC zOK6uW)3&e6+L)F^q!_mcDV$CFZmxKU_ohHkD>2D@6DJ8ft&vx1P^QgGB*f~3~35=3`jFKBJN+dLjRm^E?T`2KH z;`6(E^2h$RxBt0gt$)`unrm)?2fI_txy7Abi$ zv(~{>3~L9 zEz2PVcBclFU@e}Wrhojim=s+UGz=t{W+?o=p1~5t#c?H=UF0j1i0MmDE)l1PJSCap zm%5m|KIkzvIdkyIt}A%pkQBP6l{@yRQyP!tk>q0;7bm%{7G9Hlyz6N5^YiihCMbSB z=>7GL&>wQ~HGMS#}Oxfgn%_&icg+o};r|wGXq-ou94wpaa zTolofSaYzsbuZVkmmQOAmU?}hc+jzpTS!kU`r1cU55BWg9ZyX7sAwV}F2p5J5OCm- zgl9*|g$0aG?0h$@N;-BQ55w}OpxmMHs39a^&6a$~f_ zsa{96{H3o7+W1mBCb~#8bO<_CDz-Q{v!|#CaVhTjaj9EK@xXD$+AlZVogBG7exG3Y zQAfJpUi4(sMC}a{#b-7tzTKSA^z+H*^UW(=ZV378s!0moY5jExOMoKRR~KOwsi&@8 z9veAA_Xhn85ao}^TpG&dQQI0488;<0Y@+X<4@dZOO(mc57yV7R&R_a!Yr9Enz(yC1 z?RO4ws&Bh-sZe8QpCPNtWrm^^+$JZ!rSW-I1h&h@?O55&ZMoxt_46Q`4QR_}}0N7c--q!qT21<||6olUya;Wj3-%HGDAg z5ScJViG$au=YdnMPIY3tZ0Vgj`%+t_UL8Fz6fvQZufs#AiF?9SF~-IV3Owx=hb6dn zw7SfB7{+blvWnp#dq`&^OR_+-V8~HfJ&omNN(zkSB@_NN3tm}f{qi>ho5KPPsh$Ts zTRm7FR5`XWl^kGVT6o@puW7;`1#Pk8JFeL|3GpX8c#AbZJfS`7Y5&=`$1=u02ig zuZR$S_h^rbhmUd0gTop{hRkXmi2{2r_zKPoYcW(v60h<&q~O)aA${QhtL}_O?l&#W z3Mz~oWvhJ7UNE=k^H5+HmT)-#z?s=u!KrnDW&^WR^C5#5j_ffC%$!pKv_AJZOjmxv zQ10X~SKy5iv(Ic%jvmdXPYX7%N^m%G$9!m%`q98DK7o-tr=j_u;Dm$X3n~;PImDZm z%06M1wKyTl^+_OXLz3cVheviBR4o;qyrVX22nhXdT$}Uru;ck1?l0z@m=yemsZ`2H z-2dECSG~N6-EN(Sq-A~`UB2RlYtTzZ{;UAy`%4p8oh0T7+h>?Okt5|VC8aPb`e>`tPeHP3*R5(h!+fu;{iO zE0@75#?W<}^y{V`t#I?=mU&#U;fa#}(z3=NK9y@NE4OIp?Z{~2&12}W?_y*rZF1t9 zq0kW{!6dM(!$o*k6thN6BYzD;vt0J!5P@5s?4P-N&-=V&;1+OTsd8u$()!HE-Y}JQ z(n;ZSyJj`I$QtK+?>@kKE31(|Mu91JFN3N4pKBL3D40k(C>jWcG9OgSVza!^C|Ggv zP}~1hS?WO&X- z)+bz*pKy?~QSzZ+4MVNoh69F~0nD-~ih?^faSBrZuZQWE83?h?e_gz!iLF0jr6Cv(mQ(?2g@zLdzkJM0(ob#g>2Hf3=+i7ez$R&Xp+cI2}6*(1^*;h3|h z;fmief3tsbI)!I_Q?imY!{RTDdy6#Cj2dp0^=>T8u;{aLY8-BUo@ zT;h>hIr}6RPZJ@vk37p&6+;Dtmrq~|o?s;}R1v&@W$ovbO{Q#{L-SQORs8(s_U9OL z`VnS>4b1uhP0Y_l*f&>ZG8)c$Zlj+e&iCBLEmb^8fuliyDcOK4ww*1ifbG^{+lByk z`vmsK#vJAwigt*{mmLc7Yh{j^z_M(6BIZ$D>4haC2n43 zBw3K~tWb8FIMcuLi6-Yud~_M)tR^r%aAe$Yu}C2y#BDLlxrNM`rN+0^!xWzsb}nFh z8^~=W5_vGlPemd5M|*)u&d2>69Q-U_`&N0+XVMw9*D6V+ALrFOGLU%G7#z|5(Ak`=W>C0rsQ@ z!P886?UdPUAYtz^}P-(*Fkv z8iH2bCmIYLUDsVzRA_Q-6=#kA5DYdv7%DUNPmIaVCx*5}Ry~>oZ0?ri=c$O=4y}u^xH*L-ei|AD|L^c&ihpDkR zI|Mo>i2ggKzx717=gTIKlT)<6P7%7ytaG%^;sA?T0<)(8Yh?k4X97!90QV7v`9~H^ z-B!Ro`#_bCz_g|d%qQHK?(R@K{=`$^NQi?$E8hc#NsJ7f7Z~`aF`5K$Ok%L#`pidh z0ZY>YnGORsWdn{0JI%Zb98NA|7dK(^xWOvEgysH6@3q^#cdPhhZk2w(D5nvUwEm%E zLqOG@19D=W6Tc>wWlXQnnx4DHVYV0}_Z9>0bq%xCEXzL}n9a$d==Ve@)M07ebhaRY z#wPPdcGLX8W3HQ+1M?RsCGS!SnyCLT;$!NcLvuYjn|)qZ@=s@eELkNqnb~{-v-5)~ zuPZsc3#J}Wn19-dXa5DxwFTT!1~cjyIC~SA|9LQbHh5-VvOg}uRJO^;lA(CsK}N#_ zc7_8CLIsTKF3e2~oW2(%^#YjhnKQ*Yu%6nn%4!4a`G?v{EwTn#CNG@pj1Tnx+!1bK zz-%1AY9N$^8b>T$*=E%MinO_bT`_j|d( z7EbON_{W;mz!ANH_1q8c4+nI%C$OE1;C3}& ztv29loxpP3g-P3arDd?>i;y*2WB~UC51x+~xOXgIi%wvZHrA9V=UB&BZ1V4d{oIS88rGRRW@qbt@LKwW zLE}S$0s}|e2KM@ItYsfqGcJX;Y-;=AA-g`Y`p*rP^%q$z9hl=2%;N=EjT6|L8W?LP zu#2m5*j-?0`oJ!pz)|ynDdq#KLI86`gND@y=HnsE@41;+1(!ySPM>#Z|W zTCgU4 z1M92wsH7cH8YnzVBop1 zV15^4g91m=1(pf{j#INk>;)1X64>P@FqauH>wI8c@Qn516ZUEW=9&O@`3LMZ9~ct? z*pt_suuEXKyujpifywBBGP44U-2;ZH8%-1v80XJnHN%2|5<<`a}K4-hkneCS!Y>(7x6ziBAx2lWpXjc*! zYt9FjotJm$`*cTLT(<8-5c}%0M_w@JH!;sYy}>LY!ZhKC+fwFNmCbGo*bXFc&%VH! zeSve40@v&dT>B=l78-Lc7GO);Dd|wKdHzHDsS`bQQY9JF7}=tV`N|lT1paYMT)-mM zFf(=nvrd9>3J2TtC(Ko%%!|XA1url-ZMbNlz|8YOlF=bi=m3L`0E1@lC5^Kyg#;LQ zFEB7GTxPkjfU!(Uv|#~nfW;?vuG<^V=adS_8VJ}|tzFZ3C2|65_JS)>6Ii1za7Hz- zcwFH0FgPgawB2{s!4(fIgHIi-uI9S&V7tttlt`0Cl|Lz`a-z2#73rC7y^*7G^A9Gq z4a{~Ay7%okJ7@RdS>4Qb2}evnOmWVMc#+9$RlsZ{%9{O{F?z$UwF#%^T;PgH;&yjn zYc$|m!oZ=B;OSssUSBbd>zSwB@&x$@OxkxD$|Osx9k~1y7}h`OV|kEubo)uu158Z+ z7#R2`Fa)(S@E%~;q07kG(5kJ=xbz6a3^N8nhdXm$8PBz1(l7Z*I z2ks*mrp^`Mc{hR6qktti_eEGQ=Z6N)PqyWpKX}eH+z(TD8Oji{)PO_g(*0)jice~+ zg%@6ayLRTM*4(6DT@E{$ugL7IoO|6m=i%L)hxtzDc)ZVT?whBy#mINUo>L#iw|`KL zSK>J!z!iCbH7SAXvVpCw-hy=#jccdYUcSEDrBy@k0c)MW<;Wi^D-C%28;Wz@F)S8i zZ8b2Mz`$AefH}ghM)&NUxl#-oxn42%jEgN9SqtLd{9_2oyUfSHdC7petg?@PtppC%0}0;6|7J<8{Lo#@T6K>ra>CTe4V*_9_-6*Z2rGCIJeTvshtpaIuP&P) z^yBWjFouw254Qi)ZDiEju_?5o@PO;8A1v(al#*sCE%n>=@7jYcKhEYa+FA8h&Hnw_ zcGq)Fe|J9eIcKBbX|CXDDR8_=fHm2Gqea0;n%OArfpn|`+u{Y>%N)4O4w%H2u3WM= zYpP?&6qk?r&ETRrIJR2^(w<{KI z$P^BE&k^wcsBJNW!7~$c#;~|MW_KA_7+9BVc>nY2+oSBaMFco*Cvbc|n81{H<>;Y~ z=OH>NAGr4vuxxYSnJK^{caYQLUjnPU!OOOJFSHMS)N@n}Q+m0#pKHFu8B@VSv-BGM z*&f85I^+59jG;cuFEy5&3+wf2UQKYHq86aOj_->@e#C}2^}HtEM>${288U4HnE7r# z^t}*YDZoBIfNS{&PX9HmsSZpbAHK+K|2mO@#o)R9F_vtF157vB1@xjCG7KlQb@MC4 zgk%IhKHe`WvFOR7l7&so>?{UGA}SpxH9U5Hs^KtbILI(XD8MQpQ^>K2T_DbG!Ow<+ zzHCAgHUS024$SP#1y4=|E^KIK6W7&QdLVtaN{_5g0Y{+15q7>v%aRMH8jpl4@6D0@ z`1SNO>!hO(6gM8)<|V3~tDhC z*InGb{oOsYn(I3Xli7HmSk8E$IVJtnilB71MlGWxmZ|dPe;lk@xWgBz=n3>qU}SP} z5|fYMNl{dK*wbWOS90_7V>dYu+1x9du3c@dYzL~BC>6PJ#_;@KV%Pa2x|WFD+)7TENv zL0f2xk|K+8&Bv!4(KQ#3Oi#R1nBJ-UsEb=qd(}S=*V!RHm9Hh!xr&YSwAc(&XF0BD zcw=x(q&sPo#-$KpW3&4ei}}n9iZ7qN^h&0Q!>mfe$@FT)g|qF~OAa4wv#&n=aQpQ( z?sMjkD^@UCN>wE<;&fr_WU^9axai6w;8xJc`G@Oavw)}DMiqfODLb_G^4lF?XckL5 zv9MJj=EOsG4TICn`z=@s*(TY9FtE%HDp6oK#66+tL6faQV^SN}y$Nm|49pW2G|rcp zP|(0t_~~0TxBQO)CQgNl2hAQ{ISR}o{fiDTa_w5cCN6Bfpn-XH(Gx9}b(7UdN0&|T=~7=-!F1N- zVa8?S^LJ{Rc&8`wHTBM{PEbB?^PT5QzzreW5{m~Sz9AO((<)3ZdDL%6p5U;rA!Uj* zgCy&ODEAAE`)#IhC<}Tj1TGhFXmEJJ%o(8(&BT+mVsU5VqKrd*?kt^$+H9BHNS+m> zAlTR$~9?Z5alKIM@SDsFjm@9FIrCH_q0WiJY+L)!-{yJm6f+zIMQ@q^TXIo<*Y?L55)T(h z&De0b-|>+~s+8e5hSn^BuyiJ&bVkO8>Km*|3hUi6*P)Hc*w1{!NDm< zfWyN@g;{GtlfnXF4y`-+JW>Za+^&6P@wmb$v~J*`%1I*GA2ic~b(C1pQJXD0Ek=gFUx|I_gSY#y{+1&!z z)LyRRkP}!e@%WN~iO~9rFS=GGSnSPfq|idcPOVeM7oU;MzCLnXmg;0lAlztWCFT+D_Y zjs*vUUv)ZgvM{X|Fj>ecU(itLbbvjmCXvnTKm*5;5BmczFbG{a(Hs~cvVeo-0IP$* zUD6)HfJqJWR_jQ)${y9kF5#+G7X<3 zyE6BqDbIK(dd~d1D1cot=6qE_fv}8675n64jC{2n&hqEJUy|CPu*`6QBTtq8;!1&th66a=dNZ8*oV zOMy+jMp3Zsfs>GuBB#0plTbs$5ji(S4lNNT>E<0AK_^;u8Cg1_c{mO{@@YEDweg9g z`b|YYlP3zZVzP@ES2?h*b6v+_vw%gF5AXn59e>vYAdv(#aErb`Jp3g!jKs%1GWRGzSwEw8|))#%H%5CggVTJAt?pMxU17>;~?5wUEt?=hcg zHKVY44^)%@wv6;{iqnw4sq zm*pQgt`J?oq+sLNVI*;oLvBF>+oAw22Nosqi6>a~I2=2SRTR1WUNG`9Y~*&H^N^?g z#lKd)6^*_1Z<^R*PqeCdY-G2&vtFc9f>B|`M-HwSmf1IcF?g+MZD4R=P_=k?E!Ovw zh0L!bP90a(l9d@6k5608!s)^A(x!p!?5cx2-sf1P&MaUx*0{+by&+t5?H@+L4h3fA z6Ao;f?wpo337o&_>Rhgz>@=-A4%;NEGiJ-4iWcQ4muk_tTsS#NZAK*wm z;i8ch$nC$xalwRxha-g*6U&1Ra%NsSAYRg#!1+j|#Txfw(S%qMUtd|SXG z&A}u+VTbxZw-ccY_uiXv>{ij7Wc$oX2dlUlEw6oK-geS=HjjeZGb=|ns{|Hf0VUyp zYfMv4GdxwF?I@h4(agR;f{DAup;YoigPr>YZ@~>^%ttRa%#@9aHWPPbHLut#Iw^fm zyK@QG{0qm_-&EW=niI0~V(?92k5_6h*=|+k?Q)h`xB9N@k3_-Vh@YpD8<^i8y0iNc zufAXIxqCB=s`i>aU&$k-6l}nv#HnQABx#n|op8tL<@SKXf?Wo!ZaqmN?o&UAxd|}w z#xOAZcpOpKrjWQYIC;huhGxE^35`-u6a^|Poa9(Ob_4}53rg%rZ_r2)%JJ4yf7IpB zUXtOPdsC&2-@<|S-%;L1mXrsZJjynSO}OyRXv5^YX(w9tto3tU`NEoI{VUZ2*8f-C zx^}ICRo}uzlrQkg;(J^{KaQB_oNKX~>|>~YIpdU_-w8+lUr*`-DwQM7eR5~{zx)1P z&5!NpU$}O=UwVa%A!807;7< zP3Av#++p77ba~yq=#~fE`@Ckfcuv^)c=OH|x3|o8+V^7i7Q-nfk`oxkKkSrjV0?MG zQN^IeAfU-;18brJTT*~fl0mcY16JP!OxtH$1v9j6e8t3*!NAeMASS^maA2a~gJ$^z zM(zcTY7d&MDq1{dG^%&BWM;7ZKh+$3qAh3#qex)`$BGHl3g%=kcF6f8#(rlRZ=*w= z_nJ8loh&yRcr}>)D~xmxxcVD3z2a!pQ)<)=X!ecRRC&9JcflgP%Qm~BW!J8DlwNbN z;jrUvCENC-%~zaluW7R?nMi7GSaEam-qZhPOrBWnWIsvILc`K|v6FU9i{}cKiWYaT z6)bu+>)&T>c|XadR>!$cr$g++PO$`MaRx>yk0u9&W?q)Ic!jo#4)%JDcAXncejKgF zKUf%=TB07X#!O(bD`*VUY7ASb%3i=|{iBh41B2j(W~U#FJSUjh9T*)2c1s2{TFhWe zN@Nl}ye#|kw0S?oj_gt^n0O$6;hbD!&RIJ;<&_$uOqfhp?1>d&%~NPuEzEfHV?!W! zt8asm?+SC7g-W%|jl36(Y9HI2TkUvrmt&TRaocIPE0-H`O<41N)+vj)+f+B*($*}l zVV+{NvLwdcK4EA18WyhrHm?r0)Cp{v6+0hBpZTXhWy||dJL^)MB_a-+OfeBja2B}0 zD9gcO7{O!`VAa;c9v#4}e1XYy1*?-q%f5*$h7X#JKQ!AiFo`+1h$S>~dvJWrm@2k_ zamUI=9tXyVnT#Fl%t4qeKNdIgD1ng zLu>CPXJd~Gtw9o!H)roXJHfVh(`oxh-jg1ko^*PheFTg9ik6HOY}qF+WZwwLT*0RI z=Fsd@n(t?+9KYjU+rn@1=Xk`-GbSpG3OAS?UNjl+XbrpZuO%#^*@#0|_eM+ng_dit z4GaxfYz@q87?|XLG~PVWsHUK*a6;7jM6t4%|beg#K>VXMuH1(`pD34 zMBJ*&xK(qZz2*R;Qpe8slXlMDb?f16_qrDTr-xUb{NpbBfpO{{M&%DJVHUUjUahYZKpu6Z{SuElwRRzYE!l_eMINV9VKY^Pi?hc-Dos zq)iu zCbbKVoO#oNK6FS6Fv~U?p&2lNg_0>>$vh*eC5#!Q%1Z+CELz%onpWC0cbInEeZE z+WxRQeQ4lb!D#b#zT^zXo3|KDB-;D}m`z^H!6|7B;>8tNPAX(&EJ8XGItG^4yxXtJ$ugC3`}9b;hk_t?{1IQjSgu zjjY}I=y7Az(OaIURz54<_f+}rkCn|%8BKx*bfR~#Zi{^yEx;Nqp*OpvEvTbK;{>Cs zM5D9>v-$@GM~P;=7fnBfn;i_oJscXP6PQF^Fzys(>AK9Cd!db+t;NQ`%z>fB@dbxU zos`Evj_dzkNY6G6Kf2X3dqG>_GPa^+Z8;j@*$1R^6k206SQ&L#BkfpNlvzv*_Qd~S z^}oS%^Av;31X+O-3|az>(JE{j&W;mnr>351SZ&pKE;Zv^Hlsz^J=fBE&)@oYr^WU5 z-IUzX=w#3u)WBmO#-sKi&bM@(QycTtu7JR|&8{77U#_x$d)n@~F4KG3t?9F)ZYVJQ z+`3X=>&iJ*CeJs!zn{dfq0yojz@+EF?7M+AFu*E)L#w|-%N|aL*axgDUvs!TU^aAM zQI}xxG+5xV;nm8X-ra2L@Wy5Z&(y{ynHP;#3m9b$X34#Hq$kICUL{#~Lu*)sk@AU6 z4Q}TfuBr3iU`kqH)AB8%W-~*_KCQK@&p(O1B9+7VeC=DSw0qNP@AYn5>-b`}A;#$j#f8?`6Rm+0SUgX(RvNN6TwrvV(Bib>X6_9(j?2w47g${;EY))m zXl_{?%fQn0o5k_RLkoeKZ2wg?S0uJK^h<5HEBS-r$BR2G0gcC;8jpC}h`d}VaNdEb zA||tH0h0j3t&>N60{(?vc5!oMVHAtd7uV3>S7@<0!(zIiiSI-MS5y~&zwPbzQf|ID`oij5M(?p?(HV@644KwOFIoQb*q-L`_tl(|v~po+igyQF)dBVweeY(? z>s2|Ld831IUf$=24JI-M>;0B(Ia<@JVbG#~V*|q<7EOtk@X51O^;lglu*NpzyG&>) zxY6ePfJx{=v$MdZm>X@Cj4cKhEintO*`8<(doAUW!4%7o$$as}{UZ!todu74-nQpg z=Yk8IIS;2T=vc$jz~J0ddgS*;mX3yq<7zq1Tnr3cs<)cXMr}{Hz~sx(YILJvsZ*m! z1*61{#)iOzd#=rDTraqa7_R+0#-Oaf{oLBz$+c6}@;_N0OTOC{DbOJ`L5^7~p-GwN z-Ggf(lWu+buy)C2Te;>G@N89*KjilFy-N3WJ^-uesM!s;ghnu_w%Fvx3|5$opw)h z#=Vz&`76#bmtK#YEBj7;%4FMBdXv46|C^Sz(tbrtrNU>U>*YUKnT~#ER9Mhkzns@Z z{D;@Om#bv=Su9`adRWx1g5{xxYVd_b2Z>A01$)`#n}ZItiQQ`E>UXhAXwtKIBr~H+ z-~p5K0>-V28pTd9-af+cJEK7|Aeo7QQDkC+guxN+h6Zkp=`Z(ma%R?rJo!0mLFb7{ zIxGzgr>q(y>>1f7?AzD8CQqyB%&8W?3r$OQv$BRYuNK$1dy&aHW|NWFP19`+94i*A z*80R_P$XjU>B-l!{QP^{>x&vZ8ktymc(l^qNIum$KSe!go1&OUm3J?nQw8}stY{q3VII~2C`pTY^A6aaDq{z zbD_JjUdRdS#pf2w^BJUU2sn7CRajhT%8CLd#uhGq1&o{gj%kFPJ!>g5YW09M$n(oY;txRe?5*wly*8NucwtDhRpUl1-4#yeJ zE&N@^b1WJkF|oZ0SiiWz>cgQD5p{zad5t<7E>W$2CB27RPCB;>&fn8m|It^ZukhWG zmmHh?gH8j}d%CDxj zJC;4{Y;SaU#3_=+rM0Kw5KpDUCIQjcTc2N$_BoP1(M|ek`~BM5iY*<=&+ar&YTW;B z@`4RKm$n4ASbQ{DxJmrghv3b2CZZ=}O-ceeB&91hFtRECcz9fPaqpskzM46vj?5ei zCLfsvTDN>X%wh7vz=Kcj(ZfTN6fzzh(Wp!bILu=ek#LZ`*oi@vrCoN~O=ixFn+`K3 zlunCGOYl;QOczuUXpmpB-~fwN5QhSjW~UhL;q#Z+?1m1}D0tfJ{($}crqP2gfW*l=jM`4PiYCLJCJk5os;t7O;x ztW}xhQ9oe{kI9cEENdF;KYVPL%{zav)#unP7Z*P3hili|a}#ZK7LIY4)XaK&+Qy?o z?lN{wLK6ZO@o1h4*zB#jc)^KwxrR3vTNNgjmUndBhUAz#yCaEK#?=O5Dv zj#!CF-128kzdLY-#tAU8M5WDKdN|5Spm4fSLZsx3#Gq>qO!Xekjl5T$Y&w z3WJcdQcEkEu0^p<{Bc5ExnKu}%p-+Ky%kURx+0FK+-l@!TzOXY?xVvme<-wBRwSyK zTTW{!STK3-LId`IABU7@E!d}&!^k^ZpjFqYkxhAzp3oGHR1vW z4jfKGoeoD7PB2*8^(>NZw_sMdP{0xMz)9$41-mJm*m8~y3QQUq4$KB08X0367_R@8 zo%T=efkA6!8pE{80}O|zFtAxX;Jh-!k!{Pwb}pVnlI9A@-xMA&aHTBZC^u@B6wBP_ zV=!r=b#h38jKFdshf@Yl-F#dNoMo5ZJZ~`hQv3=wrBy4=GVtjZwN*d3%)IXGal2oF z(H?GxIx->*awStbRdpV=+yA&!RLtSrtmblP_M5jYR+bCz@0-w}A8>%f`+|#hPGNV} zwE%88h9l}xPn^#&D$QvyXps|8>~!jIoYSDdEXl{%X5hjn(lR4mf7?dxtRIV|^*9=t z^#odMIh+J46PQ+gb2!Fkv4BZ~;{bEp70LFAD~)((C?6HL)fB`NA{{)TDZ;=(May=< zd|m+uw!D7^t;P>JCr&V7mMSpg_>#*o@z0aMIq$?*9^%nuzPFO^(kF4*MU~6zgPI+= zI!^7EJfbL+7d$2({bCls4_ylh9ap(>uQGv`~%51fPxv3pPJm z5NYKxjq~-gHOXcVxmzzdb4*EK%f81XVY+#lZeIY4ONR1>%L+#oz9_UAFPU7jq@hLb z%Yk;s9!7!22`uss4IQ}+NdoB#Ec#^|d7X2br7RT~*rbfRiUSTyHfKyU<8o|Zh@jzZL04mZiO$xQFKZUCT)sH}iqFk^B0v1I@_%$0^Hey5J53Gy zB|6*v*M^ShbIuO2XBQ~$yLCG6X5tLv$YsqXK~CFtT|IdGP~$EWa}J>#N8$Y(ZPt$j z`73gqc(<-#6noOOFE(h^a}(sWZ*bsQJmXQCW1*{_L=(46 zfdg}d!xHluOhT0!M->zrd&NQ)Fgj?o3P=TP(%Grzlj73AlVY&kAoUm{Q-*^G3nL4o z!9mHw6%NT|;*v&Ux_mc2993Tx(B{aq=$!I}Y3e(@FLO*eP&;weN{!ZOOiLoWSndC2 zRxedv_36O&z5Xj&jNKM){kK-uVAbYZC)~3-ui9%ovS&Q@Krj3gho{Y<14&m8%edUQ zHqTo+K{u%BfRJ~nl|#snpCQ3WHuB_Esb z2JxH`+8ngq>%vSO+Xvo!5)6{U7n;5HC^80g9bnLzscJSqfotgx7E=Mk*5Dc^@m&uh z+4L^ldFQ3D*OXU(NyD*>4;iPHcO8>g_tsn$C^&V6Z2(7TT<1$4|E-O`avy}sa%8zX zILk5VZCx2SH;Y-lW5$`BY`;%olOz5aZ*D1n%p@mN*sR;~NbFEUyLC-sn~}(Fu~`Z& zS7St2RaIIJ|RbW;+ad?~N0mgo7&V??IoHQ8{83i9C zN=?}Ci8V%n*X}^s&fU6l%uE;V+S@q!O@1kRSoMCB`<+Iwl%9sEoi)3;Y!cbbeza&V zb7yWZv6oB*lhBTgXx)zlS+dF-YyK zn1iG87KWFG42K_HWC&?vRyn|Gb4WPhz@EKq6GItZx=M=NaA0FOB;nJj`J~aWEwqcZh zFh?hHDs$?6`9JKoJhF``0eUXYKbA)QjC5)$aQw3~RK>5uXy^UtLPl{9CJu>4^@2mE zcksk9Hmj>hnwMARsBl>cG4rkk92HyM+UTi=+fs za=9P_V*~?l&jE&j28MtGilPqp1RWAy%;k6^DLkXeIHTEk#vzTKCcOY=%?FHZCDZ53 z;aS4s>sThV_@)2Hl@FIF$k!abBXccLWQUMTz+a13>NN(w=zgEvN8Vf<~YN^E8q}P6wOz2fTN)C zmC8Z=PY3ne4kj`+N%Sx%DL8R3H0d!ku`Rf#z|o``;cS+{+~w#bGG)PJ0e=<^w zeiOr;);;{VbJ-`?Tc?*g@R~4b@>~#puu%GmqRWPZuFvM2>+~&%yXwKCwDRSoTN$ig zZTBzsPQ7$6>h76Yn*%7B@Rg_NGr}@(r7qjHi20|q=DsMM9{$#4y*zW92Xim z0vse5oWxrgmDVU}esC1`I3(BMs3+khl)x(4!=%^HsF%~s_k_vp1EXF~b8FvWMTtY! z0!=D=nDr+dwEn`RP}8h;=7n|6LA^iCnjH)|GupYH98~vVw(wz=7mUnK2%0ltVd9SS ziyYgf{)w;%+~*8&n7U0Z=UiIuv_sAoqF-ZKTn;cwN6bI>SF!#NW4(@(=NaZ(0(ZS; zq$q_sd3A~Y>YQN2d34Ru_&-Mva6Vn>d*h(olcv`yjFK5nXE>YztK%4w*|ZDdrs3kT_`m!Aa-=)B3lEG)fLAG&Bj_Il#-3IU($!2FC&GZG_4rrT%6<@4ovDeU@}8A>qXS;h?05gLnxeX9=UE!vl#I z4SXGq!WxZoB?mcJn&f&K%&RG#z6W_#6620^?r927ypo;x zC&X{5!~TCEtTP(^=tOZcn~o7K5x8S4(qub_VhZW;^Cy%o=M^@lPqlyEQ@P-qLbC@(L zoaR29DZ=5Te8f>X<*-S|=_9?~A`cv8OAbo(Fy3ur6n%PAqsNi$gQK1blW6WWMHME= ze=Urh8tk**p5xqdfVHK8&x1)kKuvtY0m&B(oGl0SUokKjJ4#&P5P#tyq2kD^(kSb& zK;lNjqgM^QCXBn-^mqdr`R+9E1u*j7InZ?P08d2&OP2#%4`a1OGV2B9UqQ|aDGu2M z3*O0`aVW|=$aD6&*2m|MR__<%IWVu`)O@5vCr;CbBua`Y_jpH~rNypQvG=@!sU4QmNeP`B~v;!PJ z-il;2u<<+0fY_0cP*89n@2G=mk z`yA%I;UE(7Ohch5A|ml8*CDxvRx_2ETW3Dn;j<~&iAhV0NlB*3PU%oj3j>?S0*MKX ztR)SsCU^Qw5=(DvVl@fp@Hn6oEi}=Up}cguZuBJI+eLEto_zS)!z<8WN` zY&ioPtHwXZl2&Hx%A>p)4y#-aPOxB5dwPy}M&c5c1Adw@=Ea{cloTK6(mNm&zreR} zfh#-Dl>^5jniXt9^O9Iee-#7W!oJfobzhC$(s41s6vVlY`ntjzT+3va{7seWcQTQJS$8%juZ$> z=#_I9@rf`9XEjK89OCeBI4;=9-Q=Kq*{nRaVe*fe<*ZM{MYk9IFDVx+e^((>&iPgM z^0xAat}mF{I10841~4dv9_PHnz}1lW?tGZ8lEb*BB`|x+;w9`Sy|`WK51?i-=^xZ<7aPZlkIa^WfS@5 zh4*Jn;Q7KLu;Ie(sZM4+O#kN1Jfx}dScIo(X)j}f?_muSr`6iY3M@?$OUi^_+^~}} zQgU(H*`mPO!JzxffxYH{1=qaeca0`ppTf)|*{JuuXu8^hKu)zp|E;h69PSyh-}o}& z>y~NCh6xD`bxF-~A`Gl74pUVe1d}@F*l@CZaZq02AYWlf?ds`dW9czwUl4$l_bi7gN&q0+L%{EUq+fBc<#8^rG%1fCw z^2WZ;lfP>wU*~Oqypem(g?jcY;uo%~>l_sFX%x+A=$v{;#Kei;_prK*lbi^Xrj3){ z6ehVIhB9+Ty#QzP7fx~uxDZoovNk_Govztzwc1!XjZgPcB#DM^7p~JCDlgX9OQhOxiXy1 zcn--m9G1)Il6+)2FM1{CoxTHyx2mnwt69mwGv`zl7n88cpWu&*=e{m$l;2SwXk&fj zu&qti%Dl!gm}hdPDS{Zu?0C#AFrs|O}@FlF9aEazXf z=jFk+vy;so?|pg0$)0h+uHfC5&d<;L7kNIradY?d_44+0zgL89&EnJ!vx|6p<0BIb zJD;RQ(gOvd$2t=?WSm^3+kKcda%X<*j}@EQC$Vz7PD^=Ia(dd@sI0!81wp4~Zp*Bm z;&3TcOF$*F+M$Jk@le1bm$qF`I;XI6=;iF-Xf)cID=w+mSQk-nU;$$r2dBPEM?<44 zH#@J8*$RP!2lE~X)L1JB{MqsG!RhAP^X@jWa^2pVetX+LfrSmPxh1t+5{*uAGIBGk zD(p~bY;@%0l?zFDq|+yo8)6a?!pJ$Pi(B;Dd=5@Um43HhE1HCxPB5>umx|mV=<(@5 z_AIvxJ9Xn2R zF&^J}PV4m=>7!hyk8MdQSlFrmNkNrGXxj;wDbce;R5t`nyW!AkGpF#Gu=I)(&ePrZ zF}U=rWk@)-t9`k^Ji(T&(M3Se?I4?o6Psd-c-)Z-jokuAviq#=MeJC+?SaaTwT$cy z*$PcgGnErAII*XmYviiap3vH&wf5Z3n@y8NSZxF<(?yuH1P+)r{^MW@V0xq#uyD$c zEfbZPgMAzqP7Dk>@Suh1ukuU#exV2N&C-``RNJV%^`hGBAeP1@i{xgkxUeiE`_V~t zGm9q|R@nWE$dax4)VONCXsAfCwobwp<(~CVq_Wp+DtJ@qE>oOyowfYws_*O?-y~XD zluvP_&PeQ#jF{r;AAD zbbKU`G9mMb+)@#bv`rTdW#lFuR<|(Y(dx7N6|qEU*_(~eZ?8PE)~kQrn+b({IvoM4 zV>dq9YuHxxT&w%8w0*f^yZpWel}=rgov9Bh{&c1b>zYk0YEkpLxLYL4<7BHyKd<4E z#~w_M!2&FwCoF2x-TuOf)9BKIrf%ty4yJy!D2*)_7AgMS_xr9wuhpyf(u7L*t%Fmb*L)vxS)#~y3IBH6QmGhRShF7k`3 z+NKGeQA-@zD-X9Bmwc265?SQqdBIVftC2%?#~N1g07fpOhDMnlMNYYZLlQ?WwCZap zGFN>r>WY5~#MyHMidI|0>HO}Hn9Z?K< zVO!a47_#|29N5wpHL<8Tu&(oIr0>YS+gRAwEEA$)ZmG|~$nL<<9qlA8esSaJju%tJcdy&f ztbFgEhMr}|p}&0#IuD36OUw>%Ry%W{GkV2Aj)ntlrVTr-Q+pZMl_VU6+LklRUT9z| zeX~UN^MMwg4hI(RW*3Ew2Q3a9O#HhY=l|iE>M^(WeZV7Yzn=x$R&qr!Fw1wmWVb75 zO}%i>;eY1M-j|&TNDBrSOmCOG!BG@DlqCkCo(~UMO8?!j2CJz}42At-jXXV>EN%h}B1(arnK8^l`8ybx zB^VevZ5FK5s!U}w`mmaN5yO572S?r+2be{#Fz`9PVC3=HFe!4{BjFZ_V-a@?IYfJs zq)zcHH2yEf!nnL(o&N<--!-Ya3pCCNRo+$GQKaEl>=wqRGU4Ea039)vT@vn{sX0CE zBD;#D%r;qOFv&4IY&0rZBs%B98?`7!o+K7$-jq{^O-dA5c?=c`G~Q^|TQ-q9_DPcX z?gOlPha7qJu6$$vS6y+KXWoQ%cLPV}@3Rl)$qRPKB`5@$IcMhH3%@j{v%lukZ25Yf z4~<{%Zds%BdFXKxWFDA}~jSkx%vhbXFz+^h1(Q%a+BgdReml%C+ug}SJQv0J& z>Ys2(c!9zzHI;>&J~J3YOdFc~9g6%X^}Bd!*1j<_7Dse>n2iU1*m2bBE1z#zCGKgC>C<2L7BN#*pbxTQnJxTHeN)D+xU8 z$qBg8B5I{*lKZJ;PR5K@xqlPnCl&IRop^lX{-d<`)lAb)97;WU!|h|&=D88OF3;Uq zIDcDjs?1F(hlhN*pAHDlxYCs%V<&O=2I%nEj-Z}Ge%lp?XD(LYJngI`IL*IBB1-xq z7so74r5zfgEQ}n#;!K!JPaGC5D)4;flw0Fge4um<^PKw*Mc*#}Ww_*EaQSPa!;>3- zgIU%&$NaNMWKmw=z$$Q|fzicxDvRSkRW0U-T8=#j`NcaJxpXu?i3MyjQe4m|F(PYW0w6PkUOo^a#U>~$S*DrI@`q9YEqP;1= zhw-)F`zLWON0zYVFtB`L5M*1(z2X7eIR*X`M>szzuxBOke^TJfYOw#dIBlBq z7!dW#$tOLK_s>JyaPM@c(+C9jbR$dChzF zcME=BJM%j#;cOZMYgz;6w+H+wntW0Zd4)7stNtbMhd2l*tx4XL$X522^Ui@5y))OB zX})P_;@z-PXdR>Mjfe7b&ow9bGEC;3BNXuN8Q<-9ej+dRMBEKUyaU;s9N01(INmG} z+>^}1_E6Zyk=H~~#Oa~%vn5-2Jo0q{9(-dE_|$NmK~eCXdNK>6n-4Zm(Fr-Lj8h%1}O%I2di`W6dE=in{+GvcZ37$BLUVR2DYjPDnc5J-#&2qFzshy zWd4>QrKTpd=Al5j?rXD6P6bZ?pLP8gOB7nLQ1$}jTLpFDe^=MZFL)HaM}hrR=DSUA z9|sj44cVyXyOHs6AFIp8xCICJKLvg}F9ElO;uVU590!G~9K}B=aDI8fRSO)0W>;30g~(Bb6Bl2|tzu+SSa~D4{+jd32ifv}l`j@8 zX4|E2-jL6hz`*t|K=E?hzIzEuFF5v}X;V^MD5$|GUB&RaE#dmR^-dW~QF%=TYnJl= zSs=9Npwz8{LWY5F>w@1dV9=8Z5Ow6col_)|Z)Ev0PyMz#+rLki$q)Xqzf!2oIgq_V zQJm=^PeJ1=21UL(3uml5BEoQx&20hqr1i`$3z%0p^tw1Or#Udo<+9CU@F`LBJI5eW z#VGRa0o$a6vu_mmPizdxa%Ng)@c4((W21LgY5W#aS7eOOEnAYkD}jqShwI0!ylZObe z=*pFw*!C$guzh;4B2Iygqn9;d0qdNvtXn>?YbZ+VXfe)MAjrZesmQ3TtR%#;u)MF} zdf5Tft$Kq0mnnyy;0?aAfh!$jdqKvB*)VrrGe|11E<3RrZz2hj+bHS`Yf}UA2&3AdtvmwW4+A#N#DLK zTK28y+lmi6?iD%pxB38c-4|>42Z9VI@^O4aCQJimC zBOb6;9N?F0WEXmHQbCb#nF4=GpwqVoDYa#qpA{4<9Xu-=INqF(ykr`6?W+1c1wOHZ zf~&R&eLbbQ=}YvBsd1kl%DppXOM75>SEN`xte9Oa;j~D?XVH@Xp+14jH@Q4uQ{eI5 z*3#nVDDJaRaN80Qk4Axlhq5~ouk9-m>1p7uc&MG-D84OGTFH@}i!r~eKtSrq))EJX zDQ-dv5C5^3m3`w_wB*?*tFnAOQ@yat;pG*swD2ap1w91Q8!aznX<&4vHdK3p?{K zcCtJ3UODi|O+oMj^EI}G*&mz*G8&lD7=-p6;9Ky3X+b$dlk%M>*f zG0AM3vfgTCMXi4(0VkF-aW&Oconk6s5GZQoH(C&W^Yu@i28IKrY@e2Ivuko$yo}W^ zA#(5foWJ3#Ek3indd*VRkgpLIcA!{lOJj1?a={@Aml&mjbZv^cVg3q795JSd%}dVQMlN6i3>#Os2q z(<^4gTFgi-NzC3cWAChGkFM}%9bj^Mz_I0knB~I+BSuaQ2bLKQ0x3@g>>8z~M2O7! zAvW8hEBB!Ag-G@v4+I`LuufZ0#pky*=fR?!2dTG$g32Dtwx(HCaph$^&WL}Nv1ql7 zVRhZF8v!3~SnhZpl5vd5Nr7{b4l`Rr;2{Oha|t}h9`LkjN+~LC*L!;@>Hz--Q*))8 zIrFd9Co-_sJrLaVkY7uI`QBr%1H%6dTU@0NO&Wo@S7GvX@}T7RzUoXOlZlQqeKhOa%A79Al`XVVxf;fUIUYk z0vD5G^#p^9JWFe&u9!XfRLf%3yV^kUlEp5oE0+XZkMgT#2`yE)#@y@3!0*9WKW{z5 zymNL(KMB`3O22U9R8eh^c);?efj!~@OPmC&kQnQnsjOKG+?y40BaS9MD-+Q9Abdbr zu*_XR=u*oX>4JqX4@nfR%efp|UM%Y;q}urJvw5om+qrFf8+WpNl4<*Cv!>UQt>FOQ zJ_nYV=VDB45^V>WT@MP|BrxYC2)Ho{CP|A(+z~!9Nx)$4!F)lU!wx(OiTqrR;suUE z#~fG~5=C^lm_IlO#2jObXxNdl$81}-S;1AyM?LoYCK~i_qkZi8I6gS=^f++4BuextcKGkjI`6l_U zNZ|Bw6o2r|-?uSoZ6bfqsa?-&?{)uD+cLZ0SAvj3M9b-F@2`&XzZQP@dhpywn^WKC zJdphVKztf2kKDfah~ka6s$&^AjwJB(F!0tfcBdSd*xBH_fW?PNNG_2_h*6=cu>1N8x0OMQy?nDn7PIxY+sb`<;kc1~ zwp`x44V9OhlaHI{{0rIDmM3dl{WECQo8{~0pPug6u_xi+B9?A`)9ktpnUmGd&NE5< zk)r5)jD4|Fe_h0`FUz=9tGDSm7HiMgU9t7t9K*$~QYRhtWZf;&Ci;M>gnwZ(ebzF87{5yE?2?w7} zK%Y(IXLj!Sy4HOGj84rb#Uo-Y3L8sWF7qmQbO<;eO6FG33fN$vaG+H!en!L!1GNA- zLD792LLw@To$UMqE)@)nOlfDf$$PJO_}sHuNPX3GhmRQol5u-aE!|ula6uz|NzxUG zP033AvVt$eQ-VG&a_!FlVkl(&`bP2sub+FK2-*MHq2j`$b4DbQ)B5ccVQ#^gACudh zf3{rqHP;nA7-aNk0=t6u3`H*WwK<#Gq}JWIbV_%2gDI=ZVjd%os1-}5PE*gV6gs6D zEfUbccdXZ;aiU_%0~VFaFRCYa>kTeCuwFMzlT4kkby;L~??pAqv`M`)Z>GyzH2h;s z}4h*FnN-eVX91hJ2lM)=5Rq8?>v9pLPd}x@< zzlY6{Ng!l`R0`jj1a>~rBLc^THRnudRA}os$gRI4U^6?%GM{D-751|N7oA-KOpdrX zf4Pvo{(ou8MYsBZq&`pfbzg*<&wM)U#$!C!A(%xwA|tp>yI_KDfGnG-NRUPA=_x#B zZ#KFMdbb=psrI_;lBdBTfz6%lN;a8Yed&siPWI1Ruux1ZKi9F5L79b#gHKGrp^;6& zqcK^r$wlbIlqeyG=gAS%G}Y$ym*&h&nV)$kGke*>ZIZb;({j>t`~RwOn8xy4Xpk?r z`FDYdn=eIxL(@2;AV9M>Md1Lmx zzV>YD1h49%*~^Y@Gh*p|dcj3bLZINFat*@-2UaEyfmVf9gTOA$Wj#Vqd1tT{IPt#u z{DDQJ>7N3t3P)=HT4iNF84ijKA6PwJFu7koaZLVS0XyHlhkVkB z%^d9k=0e{Z^3;8zJ=7d0F)eh7{L3M;nax~yfefFsYR7@C5eszrZ1S8Zb;oo#T$#du zLxHP~!HJ`10%zcsN5va5Tn#J=S@=^LINwRS$ei1>UQg&yeAY)7U7JESmlq2BO*al7 zVNhgKH(*RMUf{qMb%5oc-~vOw3Wj7oX2~lx4Mw6CXOq@_m$;SiazdL*Z_bSGY7X00 znr!Vnn=JL9NsVO@C$j{DiqymgDVIbxjRg)OB@bDGk0f%PD`?Zy2$T~2@sMA#p&|a> z2ae1DCjrYB{~To*6gy*7lB5$RJeHFZWS5w-O3qEw*x}_(MDqrC6 zV80QvZ0*5GUp$OGuKT>Vy8JpKca!#Ij)VkLlV0CUnFQIJ4d)pZG8lQa92yuo9`H}7 zXyKK*ARc7k$PiS~Z?+M4BWC6xc&wjCM zEm*+RTG7Ha&2n4j9A~Mk6WmPxh3!d7*EoFi+=tciQx%g_U7V$^S-5+~yy%~*;xYH^ zEqB>Af#yX5N1ZG=bDzFhxmM_eL$vs&#HS8hQkLyt;HcZs%zKQ1E9b`{k#q?beV@kO zz&8)mmpQPwmY$kResrf^drt6b4bM90(N~Sch87nj#<(SaGz|z3T+`+&k@qoe8 z_Ml{8L_FI8E8Yn2gOa^7oMb+zHym@}WES;c*bq3O$tGlxV2Ego@g_rN2aCl56AIj{ z);Z>8#xNhfdZMf=xF;#I!usfybA{G7{wemKSan36i^*v7I|UXQk)LzzId%C~K0Q1q z!ejF!=D>ZfC6nu3Jz%fd(9E_Zfh)d8>1>Nb%T>FN?7w%ayRZDrB5uDh(C}9iPX~jG zkOCu@hsz`3+^-f&c?@sFo-ANHnk~GHx0cam`<|y0w)pw)uMR%lu)eyHMLsaFxcJ77 z>6iEzm{blqcJiAfFmPr#F!dd0VDnJ0^73cA;CGc{-Lg-@g%{jp1pYK13waPyy`tep zaDc4nAIFDH8mBdu875^-k&wN%;G5|g!MvT}kEP15Z@aU0VsgoZ-?D67UpR1@ZD15`Io@Ki;>rPI4#uymE*D*! zvf$~NbdytuLO#D#3}xEjX?6Cd$%&&E(#%dIGxEK9z{GxK`%B+^hs*K>dzP3!;6LZF zK|W#@d-|F4|5m3sZTR1itf8ged`@!d?|vrOX^ zI#DiG(Re+f@p%E?<86Em$IUe^CUbsD<~^ifDB@h7!0!AY&zLd)W1EGQOXXzq=C5t} zQ#Xq&_{u0*)GRN+WI2Jo?t%b!IM;lE+>Q$z%UU=rjO>3LO8d9_X>=|lSF!`g$HhTP z4a{B#Sd|hO6CaAR8^&6GV70vvJAtLRtu5oKVXg6T_iN75R_(DSuEB4%O5FV#C;owP zY6~;Vfs&dN3{^)MW&|-ZE@1qzm7~!jiG3mO6$hW`OpHwzI97_VYcX)!?Q(Q+XA_abGhCbp zsHzJvax%zHI>f;Efnn)12KjAvo}2AXg>fBNW>RLveA$4@U;<;^1rE^*9G5SM|0u6t z`Glv|koRQ+pW8`3x0zFx&6r|k*0t;eAIqagg$#bB4h7{IetHuXrcFq7Vqkf9rNZu^ z^VH83{q4?+O)Gzj`Jem2IN>;>_d_PT2KH71raVd3!VhdA225TCY<>=`Aq$uo(#_@@ zuvQwdwt2F8KbYPX$r6&l;`M-~jDe%+18b}U*RBQJHyAi15~9657fvXfsln=RU{=RI^^HN^gPBv@eopb{ zocekL-$sk6TO+2nnm5_2aZa1i6x+%CE==SDTh!#~iW9ESpKV&HeW6FWNzp=anuL*( zkOFg7D2L^O8pR2gt_3WiTNqs*FiI7;Br}^df0)@?>E)L&BiMi~(SR+GiDRyWO?v`I z!vc1N35*H`%svXNO#NW2Qs=d8PeFZMK`Fu4rXj#;H7nd<=b-JWdb>2 z31*Hnz2wA}KA*t%dIH~@gyj!rE@xS^{DtP!;FtW9TJyp<=l=}g>`&lie9jun$?{E- zd0$3~hgy$>(Mn|@Mk9vGzss6ut14MVFrQk&A|}A3oWPQ~fOC;RjK>B>xebg;4y+g1 zI70#)%oi|)DKII0Sfq8qSgL_>TNk4-!{QnPt{n~BRSTG{Hn0aSU@UA<4hdy9R$!R! zILq_88mEI)-lbU%x73bL4rceXUA|DtT6FfNOxbN0dukMTuQV|CH!`V%E=^`+10CY= zkHKxR`>}-*qazQ%@|sI8rGDiWK1ei740;<$*MNyOw{&J}OWtb>m^2y~UkkIJW8iCYU_JMNZSo7A>l^r9IVhCu+E_hnqvY&) z??3RrmW*te#lKCX=}>jvP6y8Rt`%hpY{eTk#XZ=h`y(d~l>r>m zyB5WMVAX1664zRM=sSyq!>kL=?LC2#x|0_#UAdM&XZzV1Qp!TJt**vdnKCFgv}ZnI z@KtA&|HHs8AbWoMnzwc799et-rv0>43mF2i|K3LNp#MJ(s|%DUhQT zu*_wKSJ+g>{tY?*M58xW-`e=nayLuRMlo&v-MbZlh6Ki0jr$KbZWZ4!vthxa&JS9W6Bvu8FmBt*ZuWuM%t3P5Ly1ah zTf?Ul4sag4F?}tc=fsEZ!B!r#g%Z>=E2J6RdE*Tjsx255_S9)uHk|vw{`NM9hNmot z0!!}&S)LGfn*c^O7S8$wOd63)`3IOe3z#@3F!=|Ba~g0oEnwmaGrJrh^ya}4@zX~h z&Y8DWd$-&UzCF8l*Bmx)==vuVQmOD^j?ni9{5u`E`V%-hqzW{p*@^|&;sn^74s3GR zup;i{iVLrqFWg>X@qj7o&jr_oBJPgv)dn1H0c=|Xc&cRh-#y@rT);YO0%u+1vdat{ zOd_g+Q<*K5%i_arqPiI^7qI_Xy+}EMadsxN@rE_O#993Xyh|>riGAKCcUYoqcINF} zYbTc3R&7}3n6g_%=r`fA6qcT>9M3*UR7V zYzp3U<71A(cLm-h3Y@?89Ay+QIQ9BG!@sk894_q9eZ9w`-~yNIEvtn2OYUAs&zbKt zfxWGOB{e`)s+N1!0=BISI1g=Qah<@vd;xd50K0}N@3H`{7zak*MJE&mn8ZJnMtC#( zPvYno;IMkoT*<)h^PsTu!)nulI}Nk9=lwo#ZFybuku1-@2X43uYrCJ?;FBrA$)v!* zBJaSUFoD_l1M`QuhohsICUCIFhjUGtU=kfAt79`sz(AAnUdNS>3>}*frPMM_uFhFn zz<17pReS^cln-a?dim;VAC|wmF8S_ysn>GJ)92pkbjw>GeUr(*Ghx%OHQhB6QmYc! zb>-N~7}!p8A3K|UVea3X^UtnW@OG-*0%opWW-ABBe;Hd?+!eSyPIJ!Pz&1O9dw&7< zwgi^23G9~_a3>qMudL-*e1TOgfMH=4<5A&d-+!0xY36AAU^4T!Lrr3h`3A{ZLLQd8 zCfq$dVSVCukKQXnK3A@##?^;h@lOwNZ=Gmyfc^3YcHIlsED{oi7dn;~vfD2(sl33X zWwTy6Qj=}Lq4eAkd4gSc-lj5!$8ibl31?Un&#=zDyQw?t@#%M)9OKx_ z7W8^waGBHaWbOvepq70h3DFykI1a_`QVL*ESa4$68cn_0{nL2$J-M!ECQoKYu6N6p8gOV$V5pvua5v%bxhcFg(_gKN z_-y9iwYF|b&8m4e|M}%K_`mOYt-0?}^dI&=bM~ChesgB^%~R)C=ja||{rhI_-7kR` z#Dfi(ybIpy6umtdz;h^p&9#8};!f`7gv-Vs_VEOt^ex&y<>DgI3o6rUz9|SW=@!I^ z2k0$j;G7`9k#E49)PBHH^tr36x`L0#jq>Mp)oUyH)YkjRZden%K#)n#Al~C&2G>dx z-qs0+n{F^we&A>g;JB&4tGl3N(x$As4hD7x#`6;y*bA7H*6g^z$e16?Sly7_Y+&Y8 z89wpD*~=Gx@6G33B>ktxocC4&@6r4}#`FJ}uJ2m0W6FKbd9N3|ZkOV4S>IiD;Cz(< zdlkcDFa8S=2JENyy*YPp)11F=VlJ@ODzZv7==ojY+91I5VLzw)0>+3ho$Urp@*nmo zZ(y97{qDol}#P9Yo<4?4DTF$>>lnBp$5KvF@^WCz1SCYC-KbEk|8g~`m5 zRDCB+d9a|dWiqdli^t2LrQVzdmUbSI2N^k>uN6Ji2+{CgnR(gkV8a9kR}l|4Ks zivJv&S%eg94s=aD*vPCZprTP|s(p<+h}AMUP_dCKeX-V)Mit%JCYhICiIui(f3amx z?boo~X1WFsvRk>MH>aJuv-9ZNb6cIKcTZX3(xwyn=<#v?H0!!I9T$z&&&cQ$@0m02 zFzbm-{nf2|zA;5}Ot|i_W5qUE);u{ur5=TgM?6Ec{booB~sA z`rY21WE2mY!XnAlBruV&g_TE_YtaFPr6HOgOExsGxZt4bF-gYx-f@26r(Tn_A~)3} z*iVX+GH9Bru)(WoRzwj4yS36oX9jCG0S0EZ6#?3fOB7}}vNGDu=~}|D!K$Hg4PONV z6N^`c0^{0$UIrVFE_qaNOwz-rP;It_R>g)XIrFrR&Q!=&nWVqJqr`2N%0!o-lUYkv zvKbs&7kFk)Owr0Gg5FFInKz&L@xYni`p1tmd|^?&u!WSJ1;X6P#5apQJ!Gnd1RMomwK zjg6fSN}Ck>0=-t6^gDi9u(0vRljIYjtITd`G;+K!{-DAtUD3eEZjr>mG(%;_ihZ%+ zMgj+Cg#}G$<(DaNU@**+U|bi-kf_nhlGd=QX$#BGWUV<83f~#mD|86177CEz*6!7s z!E`mqu2Xrfg+gb`tbgA$G+Xa~ZCPXJ(a6phA(=iUuuZ6i*Y8I{``pN5GqY5t`Al-x z;bz%<)cuX?4 z>fB-hk0lL}U7?DOPlUf6JMn<&{U-%i0Y{(0Ln0o3I+8Vly)0QK*t}ZMsF;ynRJ!Vi zgU~cbM#(??s>R|uE0z1cFnik1Q901XDjy)w&>Gxn#5LPO=)#9ykE#<3rnu#(++h%w z)M)%5dR5k2=R~lI46Ejp(^mYVMjeZVf|d#H67oOb<+(*`d5HLdaX^wa~vaXOI2)k!)~)<8a@!XBlxG5h{Vr0`u(AF*zWa3-EP}cd>C9#n~_*v-Uc{7z* z{Ugqs{88W#3)?8zzvGee1`bZ$Mej;h1joM8GITSVwo&NhiX(>vrg3|DI0>~Ki1=vM zW;3Bz$1Fy{H#;R+TH(XS{_Dn)0&f`pac2EFBW>@jqQpPbBiKigufrojc80+eM~}p; zQVmu)sRjFO1UMVRs#_HD99rEj9pY4_J=Y}>v>f2);wpmW?3XK-{bxnzs6n{-{d8> z)~u$>8}z?jNi5NwGG)TIfJUAL3VffNS`?N&nD@`k!HKmgVG2h8=RuB+w^PM0o?y09 za%5U~#DUA{f)|rYqfo?A6`mcU?qx@XR_=Miw0+V8*0c{T?gFY@0s#l*{gyHv%G6+H z5MU4v`Y>^oQbSC1V?)TBSL=8?KQqQQGRX9>^v#>f#Okx)$ct|Y?G}F?@>lLSEU%;; zc|0u4c$VN1r57*T-EFP6rc)Xi z_Jr#M8!EYTxG(niaO9eII8W2Vk=MrOGp|`4v&_8@Z5DSH@|ABmEURRF^OV&!{(6f~ z@?slP)bw`>_6e-b&@S6}dd)PSlg7NWlPkv3%;&YKBI3QW-R)>p- zltaTNmji6Udz@vm3fgVHD7bKBFfc_OX<(V~i&4CTkx4*+A#DA>sH=<-r3_gO%&SU_ zXLCq6bgulwz{BI%Bq7iun^CaK{mD|1YaK~yoa$^j3C%M9C7OdKF|m7jGz*m}a>agn zu&A%+hKr8V0nRInn$E9GU{()sc?skI~`3u$@jypc~m9I&*;IQ0O z`-AyF_nH({{hbfI`Q)bVFG)41Q1p#C#weiV$D;qZfy3fJ#WCf+kA~BK_FMdDI>3@{VWpp6#%A{I)8|YUCT0oF zPUb}iUfS9ucIA6`F)=87WiVLqijl#AIZlCvVNDdHycxq9<_X!1R}%hlYzQ-5Cuz{I z^|2JA&W>hgrUFJA8z!q2LO~*h3%O&bENWC{a8VL!?ACNxaKxaa+ax81>!`o7^n?Qk z+&+DG*duDlsP?19^isHp+Lr~kmouEW<0dqTA4p&~oN@4Fr9gA}=LQa&4+rP#T=Us14j>iri`UrGZ?Ke^uEYov0u=|@xqF&!!(GQA*u6U;|6YP zh5!bJgx#yT859LtI1U`xlFYzVz_3l4VfuqcHVFnM35Gv285d-=NN-@2woucQU`_Af z7kt3%q`=}J&@AcEAhx2>U4ixWC%(o5&0i&2-5yL2`qaFqse!qnVS`75$OMk`A1!{T zm?b(|f?l*q9B8l!Cr|QjaQ3pD<|(&OlY>fb6BRC`5yN^hd0gI zj=HA}&q&sLmSp^GF(WreW7M8Trw$g?49ms?3|to&cpUgT8d{_Z8czHx zZuqm4k!MGv{f;Sa4vf)?;tVTx1#PxEKCxjPGiXJ+z?)sGlNppgG;>ri>|V8t{{xfw z1P1+%##Px33@exw3K+ONIII`2yvyi_zth0+n$hAxvx^0@(~p*5krtLsA_5L-S&ic9 zE2hle#OA%QMa*NO*hQzMI~0Ezsyd!%&0%QbwrCW8aiULUuUU?>dClR=vz@QrZoGc_ zu-Svdwk^zNKb&I~j<|c+8T^!Z@pF^gl6?X{8s`0BnDKytlf#RzgMsfs!~B%SB{7WN zOYFoWT*MbJN}Xs_Nnl#Fhf#0=liCKx4?7qH7#LUfG->P*usOl3wSiGxg4yr|lj?kI}q1_t3o)A*gtYnU3?4_N8HV6%U~lvU9pd1h0A!_?g(GA;9dQ&}_`m>iDDS?(D{SyBPP)W{d4$(tKh4YK36u0v44&+QpfI z=d)WJZ%Cz2XyVdP&0=8w{mDa2qfscKvA^VSpUTNkDko(f&VLE8R{XhPyjt%W8R7Jl*fn$jpRfpPhtMvV!xI0YMd zAJ}s(U=Y0Uk5TZ0z2JdnCyN&S1I)oQ*jx&j_K4UxOSI;lXq6UdcH(GR>)3L4Q=`V2 z)@KtqJWepS9cV6PV2?e)7F*Dy{$h)q1*7OnE0z^Lr`PP-{pkSb1fP>7bGB~w5o6eR zTE<#@0*gT>i}QpAh6Ro`J9_mVFv~7zbn0MMy~I|i(ZVaiB`3mY7{Khr&}yKeKWmbd zmPdQZiMDbH)~^9`ygsgT+tDClDb8_%;mFAb&m#E*<<^)D&HNUOm$VzjXI$|V_5ZBn zFVW!a`!vKd#Q#eRLU=+NxfHjhVZ`akB zPzGMEQ}aq1<$o|){F7pKuxN2na7wmqadueilF_u_X7kibtqTL23>#VkC$xy%ZIj9f z^z6LW{G&ndL~ELIt4#wp>naAR3#~e=EY2MLO%~EQuSAPBYBL)!geEf_KN!w>;Cj{G z)zz-MB}*9eE;OoKXjai+U{_#Z%4iZ)XjD;X7Gz*_{K1r#ZRT*J!O);pTA_i*g4rQ} zIfj9SdqEf54d(qOT*rl5q&S2$J-9_DG-;e*U=(mKxp-*FhiNPq7(6mqwh6U(Txiwj z@E2Qf#Ur+{Z_bqoGJBVl9+s^*>}R@n`Q1hl5C6-z8!Kj?e^9yj^amcp8O$Ar8~-k` zi)0O$p%GBHCRF~BBIk|Je_Pjv#&m* z|7?Euf@4pi% z>@`@oUFu{gXpm}X$w@F#oXJ+;z@qX;Ex(CDqJx>Ki`o6r&AK7;6m#DkrMJ44wb8e)%j z#)>9xOJ?BAXjFU9?5M!(vLm|u2Wwv5!`vUOxdH9Q518x>S|aaw3GQfAitrZL5Ww}r zK1PH^oVP`ji^b7^#lfNF^2=b}4kibN7ULH;95+06|Ix^Dg5%K4v_NTho;OoZ9|`CG zw?RlUndgkvnp2F8R~s8;8Pzv5%4aaC3S`O#FxgbDVNqacTfxY4qS1q6I@^jy0|iE# z1I$H%6K9_a6q~}p=+UU~v*ll+yraeHsRk3*RV-CMu{)@vDPNK`R)eK)NywknMiBx3 z2{w!(6^v`X#WR`NuH2gXC4`ZAnlr1P#iNa$|DL~8c4fIe@5_0i2Ml6{(X0;>iXJNR zt4(M0h~CP|D6HBdQr2GdB2D~RTdqQT%z-A22h36&jEyH4c)q^)q|MEhh`k9r!zN_D`*(eJm1EJ)P)QFa+heS8H`MkLzR||ME!aCfD)A-BS)Q{$I@4!pjgI88J!BlIg`pt{(!79ZWyE zrG#et)LrdjS+FforrBb_Y3T^Q^bD5sfsJw+PXr%M71_xkSHSk=6oZBki|7gFGizBE z$2Nw&^H>;~I?1QeS$?nViG0`Psh6u6LtOZ)QIJ}p2G>LVzxCu0@5@@t1E$X=IuIXcfQqQ!C~eT;P^@i1WX`l6A|X zR{8p`WGz_Jx;SL7k=ew=p4;b_d|A)YadlNw=Wa>y;QiW65);`cGzyfm=RIHze83tn z@oQmwY5b1cg$nF39L>__7}<9)d``$|T2Lsh-_5@2UHL^$wml8KsU6Y`tp5T$!VCjC zEY-!M4(S(tOyd8xaNB-u{km$^>u!7!a^4y&)&-3W847xW&3Am7STfd#&3L{kiAm&u zIAcIYp!38JZVYk(S?LVyC(pBKHn3zbXyWl$9<-~?Dq`!;OB{b{H`*lJOy9!nd4e_b zhpkx#vy#Jk(T2Y_mWP=C`QiNh%5Cw=-_PQ<&2F#Hg_!R z+~4S-xc2(5S7Ds&4;Hq1EZlPDtl96F&5uo_5 zxug8mjfculoEz9gnHY_>91jp?s+3#Vq5NW^f|{t8C3C?-#s(H4IR%e~2L~qy1%7h) z^<>RNF2`F{DjHQxZWB28<<2b0-0XCbM`OhiO-aq^TfNT6SVp-l@1JAoQuZcdqw}%u zEk+3&a=Dd{M$6@tZ@D|MNiTA@b=lKXu3kb9T>5z;KOH{KC!rtLz;UvX*_}sRC#B{` zK$B}HhoF~7hr&bSj0=L2JP8JqITdo3yL3kFon`tuU_<3YE@!rYd1?v^ci_u7Bg}|OhW_F=P&gF5AM|ilzwpmMjIMA4W!JAi4C&O-$6RS7Bje|Xd zLkr`-i;Lqc-Y2o}3MlKO{9s;rPB`ezmgDm0e3%`4Ib8}I0tHkqIPgy75Moy-d{X#O z{o0*?5GE#v4-O38cUc@c`R4?1s)VWvxNF)vC~}5fbudmgG4(q%RVk78BAZ}xuPAer z{*|Xldu_Q%Z( zLC*xKtbKDnMP;ws`0bhIvKfbbm3F=<>X0z&_;^BN!jncvw$`wK47QfI3689XRe!lE z?Du4!$EE|rcenIc+ znqXCh3(X4WESNo&m#$zq9{Oy@>vugLFI7BN>iS~XF5vv-MRLE*gs1uZuI~g2;!;Fk zPM_Sso93BVBNMh%Ai^$;c|vT|Gu0VF1!9aWsk5}CW^v64I4rX3KxJ3XvLoN5PThVm z>9k4v14mc6r3_4+3SK^V6``K0wrK7yf58kwJR2X%r888Zk&30t#o7`x-0nlvK;;4cB@fxKY0TiTF6$ zIrb)sEH!XdVVk%nvqp)FO@~$KQA0;24-+$s!U5KR2kq_-32X)@9C$m-R!F-n6p2@G zV6rk3=2Td~$iv~lv|vF`;?swm{Ti;C?>;byYc=FBXP(cv;m|2z(ZPQGfg%gbMGiR@ z2EKDUn%6M}oZYNi*!s3oY4wGh^H@@N=l+|j{lelpbEB5s0Y^21z~fZ{oSAc+qQfLyh@J7mA_z?x$&XxChNoA3KJF|&SD$e8w>eL1r8-`ayfCr zh2QvcbV~Y?&b^uIJeg9m9YrpgbA8fnCP=va zjmunaTk~k@#1n@#moc8R*`UEEvVo6{#gI!hLy%b`p~=R$fZ4;riFKFMB9^Ws-<)TT zQkD%Ys!G@#fZZhSg)w(OM zmy{c!9l3IMgO|wXh{Sz;f;J4o{id-K#C-)?YEIV=7np2Vl zm>O%8(0v$rxFX_Pvc#lj{ez?%KRp_S9}Hq$A| z)oUCX*d+Ea@COAn_OTx1*UUJ_J@0~CGhYj%<-U(M^4ifK4T~v|C>JtH+8Vgud zUr5BL1ag)9IlTQK@4Nih3!j;8uH2ERqr4}^{gUgI*Zj2x|6H09U$NW2IhYu-Xi`#3 zf$Cf<{;G~GJ}k$q)y`MEF_!wnb2H)ti^H6znh=MCLOTJb-+c?XUS2+GrgE`6R$&o$ zkid}^E}K@j8BBs7omsq>O=O=xBT1xi!3NE53an}d3;9ACTsFBeCQaj5#C}risBjar z(gwo?9sFDe1XeVz(MWM%sGh-gLnYy{y1I+)tH@IZi2+mY`F0ESdN6X$6FAIq=SfHC zyqudXpYqiG%T8UKB$K*!PQZrR*G1kxjaUpAI%8gZ+x{is^TC^IKC~*&*^wY~cem53 z9#5Y|0tUP;3g0d~Km3Gocaw~YBCn+FM=$@Xc{gvQpStp*KemC>^f${tg%u7m3+fWs zLQdpsR&4Bbd66XETEN6x=G6L&cduZ>1gC#dYZ>=ET2psI>!=1xAVEnxN|-&IMw=M*7Y}z$=(+`cvd7F?x@yTv*XU&lh=MTsjPG0%Q|p?CFX(P z)Ex~9YYcfyHYm)}o>IQoy;vY$y(G*$Gq*BKW2Z*lYlb+pQ>;pc&+5N?d+=#RyRF0e zXVpFXzVO6v&cAymu;cLSM-tDC8#AIVJ9zh0wu)IE3)>)h#-o9CZeq$xn`ySaR~7hI z&0%7jBkc6CuE9J^;)rI+MQ+azNm6YOnpj;L7zGpDnI%8iEi_obC}QBnEc&62?O)JL zcFieD{7oG#A_@zb+yfU_9ANPeOmosI;cJ+6J;ky6ilgSA1H2K;Tr*@D3=XhpG~9U_ z(YN)u*AGWF2PZR*xIm9a(|~4^1+$FzFtF^1p6WaEy3RaD6BmcxY4(C4PY+HzcH8`?@g293r5zQS>L|)EKX?P`r)(qhYy>9qp}H82XB+ejVRUy4vTMyE?&^U z9OG~*bHa6#Xzo7_A_5bx9b}Zb(s-@5(YNv-`yO_ulG&3@XZKuXiH%sSc#euNn40HG$8yM#{8_CVFz2N-fVFLfF=FqsM zpSI#r9p4+ zb&(nIGd3J3%wQ0$X*!v}tjBOnu7mM=Dg*D2W8WGb7U^&~iFL)sUTpY#uxVw3yh|fj z&a#w62iPt6l(UX8Pv|o3^Qu*WN>hr;y(GRunLPK zTj&946L*n6E;4%@AJ`sjUlH(xaf<)0Q;(;*Pc@Bms`Y+a<9$5jd5~R`aRl>o)yGCm z3yt3?ghi>^zc^gc_dN7pma3hFvcNTGr3zJrFB1>{nOL?|^>f2wo)Zq-GEqu}4rwP; z*-O|e<|LnLJ)UaOsBq<^LP#T9jzeM2akeMn*)yEZ$-{y( z2WowJJMJA2ZirAm!YF*^z%SK!);kVz1uhC4P4W(m9Vsp>CJk1T+?YQwsJI*wpCPs0 z^`NwiqpXRe@D<0cB23>F9Dk(A!er89e87R{j-y-zvyPGD?~hS3bDU)^xU3dnxqEk6 z(IzjKCyP4P9N#W5rPxidGvYa?g~zUl!`3I5pI>|s@@xTLQflQV=K0@NRwk)dJ!)3j z@;p3B{8Pzd{wvR;wjJPp@w{SU%)dph4fT4ng67;wuVD~&xUar|Nh;$~Q=WsM&OwC* zOmja>7bE=)=vrmi(>5?Rr}cIJRhmjg%20oE-I!V*m? zA4F9hnB+T7t@tyq#=?O)#zC2*i8bVay;lnB4F>U)3^oQvRgOcfHV#aB_nb8xSbj8` z+&QT1(`>TDnV01N>k&s~o2HE>hqz@fIv(I!^FYnfLs!m0GFnsQAF~xGSUZ14kthCd~;>!V1^eRTNg=Nt>*AfcuBzh93u|O%8ER2w*zl zDr?Hkw8nF??2#RAaeC`cP3Jqu75na?t9sD3mHd0}Kd@Yrml!hh;e+y5ne(&O%wMNB zzidq?Q=5u}(+ROQ`9FdZ54Z(*82JPkqwl3JHq%m4y~KX!dCHyTAHPje3T;bka9B2r zNvozwV?opBu0zY_F=-ue`||OSgo-2IgY(}y@SO*rZjm=G)|*vF{woUvr~6Sk$!e^MMeeRoei{#f#9kdL5q z=yTR33>Pg9Ti!9So^$xcw3+2mZ;r0gd(HOo*eA_5cASY9^5h+Oq<=J=x|hz=@%a6= zHy@=Yaoe=`7iufpjh z*K?3(!7+t5j0z%6nkFAK8krP0niP8)6!sjp-{WlI(8O2M$-2SeA9qZ{k~#R6 z$HgJ>a`M(_1_=fs%a$j+GJaJ!HdlSf@$;%R$*Q2W6h*@$|69b|g<%SpTu8uy)Z>VJ1d#6-I$S z2ly@66rUatnbZHnN8e%csxQ|%&0ZXmFgPgpr18R>TiQqaKZ-J4Iw@qfr&+by*?4<{!Ht(OM@ZjGj&Ue}~R}~y^3Ou;z<>nk; zXHyZDD?P>;&SnNv^&IVf|Z zQDB2(|D27>URSJqXWa9YmVQqH5NBCaJMkZKVXzs&{W;gaI>`_;P@sdiH7^{UJL#U z)e~}b3E0II@?F?)wr16T;gWM)&YX*j7p&&ZK5UtC*s{me^zusECky7K6-RIXYWL@R z{pXd@+!cB|{9^47DMd8xy>nXT%?ZAqR-QKp5}4mx*C+EOcqUn!N!)0Z=y}@K-6&!4 zExFiHqU4~&6GthAL+T6q({6MA5;&wT)1>*rY1+MhwiOJbKNu|zCd(XH{7*zeX~%Vq z4~;?{j^`Q<$h}|`|Inzqqd~cAi&aq%OU(iK5@st8Ciyvx3JgthEeAHUIjUapov&(A zcC%4lgemE16aSwQ7ls3CBo46r$SvD_%4vZ?(LW<+g@zeHc^pZ3#b)V_wP(44(w#fM zG^D2gi;L)1tp_h>sUKTs8tre@ush8pph@as$KDym&Hqg!?>vw6VH3a3a4N~7 zv%OLFMdRT;&!=tQb=aau?ub>#WJb|D2hRF3u6tg!&i$aolLp}@48j(TB5zi8N3B%| zXyo!>tmZr<$8ku+h4GhGBX3Wmf{7!Wg&M~T27#IeWw+x>mPsrO2bF&?vhy&oi!iX4 zG%)ZuFwJq`uSpSRa1`NS6mek`xxpx+lPJ64_QMdqKzUOJl?99riyU73^y(@X3iNv_ zwbg%K`7^f9dTM|4%XkdR&afU}RH@l&+VIs<$D8lgK;Yy6)YQ@7ts2*F?uV z_qrS|y17;9$-fga8xG2BVeEgx8ntC}V!VZIaie(7kJao=iX|=KheQ+ww0ufg6gm#_ z#)J*@XkV7LE!gj>`ia zm;(;DMlmo)9H{YdPv79c;oC1nMK_uukw$kOIqz ze*q3{eGI}k9Gub{BpjIL*B*3tXLk@ONI2-|$S!CXu|YuTxVT~UE(-yM#J0J5RXmM~ zHy*HXN%Aro2uKD!X=YH6WMMF9Xk_5z7cdAA2ySFzW6zOV(V3vuG{Y`m%i_R;=VxcI zcgeLz20dwH=aI8${q=y6Wwok8%#IJtm$u|wS(ACW%y0JQlILsYTDSMj-BbVN&E4JY z{p;*@|N8Op>GtXR_iO7`yLM00nW=oZOXT80rOgyvB`P_@IPv9k)O2e<*jSH0c|Y2 zivlj&_eJ$_sQayGY-M9}&{Sh!NJwa4krsHMGSw-lQ+-j(q?KwjJG#17EVNL0wTyq_ zCxMj<1v<2}me09#;mC5K4O~q^S{4tPq!kL?W^Y)wQB9}dkd?ah)@w=WbL-z|7OmU$ zZr5u4x_`G`udij1UL)x)5ty^lQX)Ht{Y06u?})dCjBF!}I%`8jj3P_|>_JeNL;E z>k5g2k4-`eHi>K=UO%;FZ`gA3*)i$OC&fLsU5i|PV#gCFqmz54J@MY)xopWMFKy}U z86CR&k1%!^9{iy*!QqO`*)0~8FK39ymfTcLi7WfKW2>#pfdsaPpH3{^9;DQm(l5Q_ z#9@9>qXP?=o)(>p44l7{`AL$)hlEFMw%;{edSxOxG^xy@iN|JUdQ~bNnv&tv5XH05lxqcF z`mGBEvnMdUa?YQ*Xw?xBVGjmv4u(As?p7?n*fy*B@Tu~++IpT%@jH5Kf)e(8`Lx>L zu-G=E({0;JdN|CE=mhiI=^U&$+k01~a{B$1H&uV!@6?wQcPL?KY%{NWo!9TC@KM>~ zKFdqTbRL}>GoQPNKGoZyvFOCZHmOenOIpPm4lwfAf6xfc3M#B%kzNROO%#dWXI?(l3A}Yol(R^Noe+IHnUHm`se?4 zbi6phW@XdLSF*y{~ct*fJ4W4$+zyrR@4ig&rT^LyO3K|`kh%pAbDX`T|Xc3nQUBf?Tt*I^9&Z{sn`X4Jy$x;;I?~9dbVHJ@E}+YOfdV5_MgYSf0#6VSf@pQ`xfhsvIR7YK=oUY?Y0_!_#J(CNQl+89YYKLW-ES7%T zDqU^ZC78m%qA`JQkyZnf%fbUo%y%@N)7rqeHs!!XledwqS__&!F{TS#5W3M4eW7 zmJ4UpCmfo-Sb$lsjCu1k&gFci9fw$#En>*;YMB)fspvB=NQqg3fx)>@(L(&+g(=5+ zPseC~y4qb4+9(q8fN7!zt7)5tOSRHRnO|$vg>1Wz)NXS1_)_8ha)F0vpCB)U5S`o@Nwh)gNBRcufA|=$#*&0z-E##h3%(mw$q9tHN7x*X-X z2c!kIED)G@!hu2WfQ`oX^vD}%zOK)8|%es)~^rNX8 z8vd!?ifTHsM6-9c)3@Cgzsz>rxzRRxu1rkK%OoD91)8a|yMuM@gDkC!RZ??L+0HQ9 zzEffc!=hOex=S@KN+~FWvM4N&+vKSb3Oe7i;Nz+spU2$d3J095I_;%I7BI>O%v5qs zV3x93#4eh_5H4rP8(x|$Wme&$IYXhsnW9fT8j_tm`HZQM$S$emG zGslKCssG$HPX(S@saaI{%){*$&r-8c4&fz?d>aBDGmB3+@I+~W+!wJ1?nyV;^pqOA zgKji3u$(xMeng`$h(ERMdHi>cH;+(JP8+=R~}w4$CUB8pACx=|H9&!&_Y3L{{Tiu zhTFjp4azo8y_=J!Jo|0#e8Vk`EK4#!cgub8c+tPLR@JIT?T*W`6`7}EZ7j@wO22CR z{!lQ!B8MS~MeIirllh9LY!6JB6WZSG-#*>=RCryr=mrNq9zk1shevw^%LRi~8JP5* zNQ&=Xd_>&he!SkV{LV zTE^l=!+tdfCINw#JF5)0?>ee0etE~SbL$`Wh%mTbYm?^MkrXtku-2B@ ziL~s)8TD3Y>gu;FfBf&^gXOo3Z3(p_^<=G;Msn@$Tq5_*?Bcjm*rwKDq{ z#qakrn{SY@D7eel$fm)-X6b)=OAm{zrsP4LGgWSZAq>lvG6gC+m(;Ci-Q}=??-7ei z!yc^#3PlcF&JC-$nwaz!aK1Y*Y0&~EBTg2B!+Ze^Y)p#m4h%wj8f^6zFvd6tFfpD; z(@G1ZeLtLgaPC_W^eKUDS0VRIuRSkvCUYHP;&@;) zgCVZMgWpDvbmPMBTi-xR09e5j9KPEq`j zqK(5+i~EfS?kTE83JL#w`L?g4<3f^p=@a$ZNoBiuy?^8_Qn}!J<3yj%2hI6BA9Dh= zmKJFpZj?EZsC8LbT;Gpf!$HT>K&RL6zQ=}7eg=VZsnzqCSr;u3QAnKNa)6UzA>V=n z{80&ETq%1_9Ao6-V2)y7iD+Px58?4@RB%r;-G6}X!vn#0T3lR={6-4wT#W1)2UbQs z;OAlF|8h9^g985?M*+KqPIDFc5Em9}_zr{Ddu7#mg z#rNz%^_wCbEewn7obQ}C>wC@P=plcMeFis|FxHTiyRd971=!%elWps z=M?RwyL6WSVG*)p6k4H>QN_%xcK)JP0$U0L^RzsUHwXBmj-_2%qFuGc>HiV?#AAHl zviaXMFl8(dV7tmF!NAP)P%vbQ)HX+1hD4q@5BY;0@IE?V8PKrDsY65N1)s)3_6-Lm znI1B+I56lf5Ku^z*_X&OCqe3!Lm2OKHK!*62UXjGp1fY;Al{_Pa96qH?*t6Sn?b#sf-3cjmD3zY>Nr(-4%T5imtzKzxcoZA+{HVZLe;Y zS@B7FMI+CuS1eD1?FzMYCTic;(svWtnm#L#)j*YBV4>b)hN?dY&PX*d7=7jY^ng9a zjO)#U)PD*B6ONgeEOhstD&@8C_X`2$f(J<%3hX(m{CW+JpB04u90~I{$ob}gB-=vQ z8QOfx3XGdvUkD|*doC1VVPutYVfe%#P|_&Xr7ZqNp=g$g>m`59Wg;;JZ#QmO6n-SS8_3r!Zw|JBP z)ncuqi82ypGRF?eif#L_!jVU4QGmYL{l1Mm53M`B>y1jK+fpNjlu!pQyF`8yhksm? z^!X>PX39|z`1HVmlY=!Vb-$AX%M=BEk3*w9zG0oF+B}m!aY3y$huV&rYTHZ^+3`h|Ln7qOG+CLQs*#gq%|5C*r;9AV ztS)HYv{)sXA)hIAW3SPVMAHq;mOF|J%covVlt~aOX1rQtzFj=RqS`B<&Dcy^gP*Bu z;%6g&%K}c;ya)UW2SxrlFdHctOyy?|aNtr~nf4_xZHEGT1_S#M1))uhGD?hGc8&sh z3Vc(xyO%QZOTBY+JfP@wfGMUSpzgrM00sfKgMK?sncZn)RB2#zD!wA~T;+{|KwPi@ z!$bZh%WM8INm)oKa>Si&S@TlE=*b_qoO%x`%Kc>`XWD{KyQXK$IE0;eW`B&cbpaEj$^(W7 zC)Gs?0{0fo%~WL-Qeb<;z%0YzxW`qXlZlb(z$}K{DiLRMg&r`iS`@Bf^~^(2S|)O_ z`pg5@8ZFHQ=d(@tClWh*vbAMXXw|k#cJUh-;xA70nq}QB`Y2Xxm?$2Wm>9V{b>~#i z9W57&?-nF3;quVYFSVOrovgjS#L_>#K#j4cI61VulwXsd?V-J5qlg`2i0M)hF2<}0 zuh@AM#Z?^nCoxz=Entdz!2a$4pI;wqiUL~@D}Uwzwk7NWT#6G77|d0Q_G}Aaa#_i^ zO|W%NDx+Kilgpz><+p+!4>{*OaJ`=(rRKr<=)+YBtIdunvrlcWXP^|OnOf1V>}J=J3`5)I|Fb@=mA@&V;nTJ`>Cb|hqrUs6T`B!593uRZ|JwuhDH~Rc zZ0D4JDELllr_+J+-&Ql(DX=&!;A}`>aXY~G=m6W41)N3and0*H9Pysw)u0u(DEAcW zhSbRnI^CRGzVX$aMeA3w#xU&G>f&SM*~2E4JJ)WvR$arP&5o0P zYiR)?xChZ$b znTi%Dsv62B#cq9=$B=(#$=rof(-fBmS6#2Zd&i(+v5(}v33qRJ-jKbLm^gFp$%S&o zv*lXn{=Hkakh>@9^pUA;hxwLIl3G4%F?&=v>k0>6>Cjz!ZD&d=-eqqzW@oJ8SjqQK zAuaVZ-zx|HPZRj77W2(J&D78!vZPVu6TA2_N1+eL!+t0*hlB_nVPNJ_6cRXSA8_Ey znx^`dhc>XxzM_79I!mKajswG=1tNYA?kzpYuf)#ivFTrg?UCp?S0a^;^Gm%EG)h=) z_?by4VQ!v0Qv!pKL55no&GYB2>YukhpPP8Sw|?8qM#;M6FB(rXTYuQzKexrW{N&^8YPelyPOZ$T#?UVfI!J&e|(W#l8U&6-X!l%YY79Jsvj0wjhSv~TZo~)e05VSb6oMA z_3kxNTlUuccIsy9mE4@WcblZ0kKV`aM|z~y<;?qJQck>=aNoxh`RU1tS8Dd(a&8E; za*KL}dn6c{zT!%bo_=kC()krZ6CUvxT~&5&<=LhG;v9qG!E2W^j0%o6ByP{+k@Sf7 zcyYjymHVQs%Z>*Q?Rr^Djub3Z>lEQsXycj4sn{l^=d8fN(58lcC{#=NvYlF|zshBHZMx!)2d|XGQ*%MOfaR zu=z>S6N{TO(jSGGElqisu`P01RMxbJbpN`|zDmmk3!-MKEmWJ6xh!=;16QF*(kT%w z55*I_A}_)iGqcso8H-LOZg6BflNxZR_*`hX1B;l>uY>G-3=_FccfMH2theOBp;o?k zj6zO)S6(D}aMZZ`I?AqYBH+k+Ei$~3W!24ZxWdR| zq568mAuo^1Z4wrbd*pXr3=3ryTyT?ttv4XaH`?%+$70i>*vZMU7v9{M-v5lvEoGOp z*vuzp-MV3Y$$@!kHwV8>-A;bg zxjuWz#5T>H5tBbfJ(Em%74y^YY1E_LKhs`K2fV@kncgBB077opFSEUaf=MdZTY86x1@l_5CoBDJghI~41pBO9}){)qn!GGF;S?9!g(QZ?sF^CEO$BYHn+(PCJqrq2E_288E#Gl;98*?)kKaILn=i3VW3+%R@ zJg>oP;KiVz(7^I03cgB}(y2_P1dl}kzWoM{a2uM%NytGBSa-AJ22<2 z!gBvwt=?igrbz`+Ogu6g4B|ou+4TYv_+MN($nj$Whx3I644n??j8hgEJ1{I@u##w2 zIpDm-qoluoofadv(t@p=Z`gC&Ll{)vo3B|ZkAtOs4%sH02ik-Jn5Kj* zWla9)tTIQjclQdBn6i!BY=YrlA!3~#Zds|u5{;Mnn*!o~E;_T%X1e5pZ$d@}JXw3T zHS+33JYZ2Z@XV~(%*Yt>FM)Y!XIN-ya;VMQwABZ^X9`VJUa{d<-)wOPMve`}w$4{3 z*9dSrGI|_nH+a#&lqk?FKQL^UM1I{Jcq@lbk}83hXvPK z&gA5MT`7P2?mMjyyPyBaTa)~B8LPvG)=yWA1sgQ@U5*I&8)+o+R4F*0KT^Q9v+V)T zx4RDPQykh{cyd@`ESMF}DX_hInfQ-^?}8L}$_FMfk=J!Un5S)!;$%pes_P`272j@p zWdDV}x2iH;r#wA&-@2xC*!@h`+nmhV7HM;grf!Q>&Cg%MVReFW<=q=@=1tmn9xf|5 zp?+r4zIRjgwH0p~FVI#jc6M&O6i~_>a<74XP0Z63UZpd5cN~bQ%9xdB_PqSe$Ft${ zBjb1!SeT#xJtTWQ%HQ*dvEU>I);au&oz7==&P!jgS%rbI!?__zq)MQ9;ZFvCLxv`P zD-8zbH=mm%PBgIUE?l!w#Y>Ub_)4AB+zYL$Rtwn+G_(#*jQ4I46`mCLfPZO0n^7E3 zvgGnf$C_4d?2Wy7``)ESzTz3oW-A^h%a<#k_&4ujs-9;?jL@{?le0CoCLb)}U<_zr zx_0+~;H8ExpA06^c^=F{>-I3p-Ez0ObnwWXR!v|&?6Gir#7i^|0 z4t961Dc@UkuUv`a7Ldx zv17>ucXN}4t5Sr*x2+Ogn-Ih_eT@cF<$@NIDUO_}Yb=i0uKKzrH6}ZHbI|tP?{fR9 zZQIQlq;DLQGuvpn-21Vb^~)orMgG?6GZ#Nfx+U^~LFkM^mv2s!@QMq+)J!IJ`p7h% z3fKJ}>^*7Kax1}(rW4GfGZa{Pcm$b)CLH343Q(|l(!gf%Z-es^J)xD=kDY{=@0%-f z#mSlek$&*{D64*1W3NsSv-HCNcZ)9$%nKaoC*V zP_O!_JqNz_mh+aF2_&+AXWM4lB-@u4(05|4_qK`W?|(Vr(IzvWSynE|Y+0c^<= z*wQDkWd^V)Pw?csB*Z7czN|s?WB}*M2|P^;d9o#Hx3xz1UiK<};bnO&+u=H!^@R4j z?j^3`FL(l$JPfIxP?q z7gGFoS!C7mfOYPQ-Y)`H9(QMOVDy^6mKwkszJnz+q18KpEw+NqTOjnu$BfE?pmPh_ zvNR;JBRHE3IWL)W?%Y~@eR*>60k&J)nFV(;CNHrhg9bZmd%JC&2nX zf;BpTb&ol#cS3(>XKUDwe*OTaf`qoE6WY#9Xk%YF^X!BPlOJ%*dC4(9vex`r@oqNu zq6=&}7uYIiwC4#-+$o?LWm2-AHKyT)_p|F0^`_+XFO*z(nLBF%=a=cZvlX}wKj1sg z>2zlS@4gA8iR~<|4vee{vH}JTnTtYiEn)Iu;HVT}=gZ*AH{u8qU~O>Vx@*9@%Yci$ zJ>BZF;}wMjs}0;9#Zwm?;NF!m|Lacf{_yz=j0|Kyu~fFn>N^X4I?UJ?RIN)_wCA3N@a&Igo4iNuJ3QCiE37o_CXH;Y$vSr;9D7Cf6#k7dWu5KFKDylUaMh673C4 zOB!4+E^v)X;r%X_V`W+==jn82n(8?-zNZR&cMk9_I8ZwK0_R)>u7d}7PiXSpd%*Yf z1NWJkxk=`^ya`N;EPNCuGX{NNZ@$3M{Gf}Eqr8%V^X>%RT@Boe8HB2qao=7b*W%v2 zvw)+$pkO`&*8+u=Q@d79tKz6OVB-70*g1iFsyLrVEAQok6`c#X?|$IBo4~Po21|tj zv(*;G>1mPTs~AF=8N4c1^FOs?3}Em~nBft?nx5d$o3N&rfpdZZM~`Ix%m6mY0}KKm zSe7MlEl&8ic+Sqn)~6QVKE+{sYq5jWlA;Ujt8cQoOlYs#!IWz-(MyVR`llsbb)h40-Kxzqejy5 zvQyGN0_+l(nL`#ZX*sY;7%*EZaNKm@eU-Vgw2gbBdqtbUv?UCj?F}4Lq*hi<<1p;z zF#Vk_AHeWdk@N63&2NMtKPvC#gz?HO;MdF0?is^~UU&ESijaS|lI^~=E1QITU$j(dr~bPSyzf3N}p@pFpO-%kzt@m7HY7*ePX}~p0E92&5*XHRQ3KJM#daj)R zfwQH6W4=O0%Lh*VZjOmkS{D;JraJK6ZQxy;FnxvqwmX>pO*vHiZe)l=<{B{I`0AZ;tL%84>+b8aP|jqR{z;| zih;8;k~LME!Oekd7X#;f$yo)`+q({&=()q*rLeA+;XuKJS?*F3Z!cfsb83la7RNM! zrt06_{5DM&KWzVWEBRr-!7o8OqJ;Ek6{@z+SaxP7=T?E8A0P04ys+%)g;S3l_+C!n zJ7U0dvyClGAobA2od+N8%>ER@_Cs2H0{e>#ye};Oab4}=zN*ER;=r_Z1@p;UoBZZ* zq#JO&@aE`vz-h|O`ANjoGJ(By0r%d5{42k9cP4P%xo~t>0Y{tG=0*o*|BkBdSq#Dq z3_gyL922Uhx7u$nVt%i&X5R+U<{_y9V(gsYs0=Q0H;ILkGqGtog zzttT5I_%26?2~lZOB_y?u5NdEFpK>S+pISy+dgn``fyGX;N2m>8*16KnKQ@sTe8&7 ziB7Ff5khM5JNAp#rOnw2=e7@U|g`E-r% zu0r;k6ZoX&&RceXdszX8n*)>WWn;evj>)e!Mf}{gD}l?ehhxD%22MYQ>z)&sV;}6< z7qE6mH^-g@ys>Av+jF@V6jU@^$k;A);lGTB$Bfmk9U=@D_A*~!2r=LYoxqy$fbZW0 z-u{HeO@R}tZ7wz^w0bCTxjtZg@5q_|d$F?ZrAc?#)pgeuHO$if%a(J2**f85`RtRk z_Au2vIC*mKb_GY1iw37Qom}Cdo3q&@ru}rzp3EK7UvqEUzOaHE76^mPdQYL(J+1XnNw@Z)C-uz1=w#i@LjoZV{P=>1q>XlQY$9~aIDqN*u9#2 z;&$#^3VeG%?B3ITgm+a&l0k3tf{T^BT=Nyq-@b5k_XD{-2e|*Iadp~-&B6i}IRmcC4|t-bFDd;! zv14sAgU-5@ktcH=u-O)@6I{7YXy+2`1ooN->?i-8JiYevB==LBJ15@%ki24!#KQ?O znfKP;`>;$=X6I1{p6wHMGFQ*pwdVAJ8tzxJp&Z5LM^t3q_xj9_J}fH0lD_c%l`O8l z_Ng}~bARi+uA9BGBY|tT!QLDz-W{(uRo~r|Y0I_8d+mY=*Y&Nr_Fg!;*MVcPU(?4a zEMgzzC7Kj>beVOEY-KOdW@308ZSZ*e9PX-`$E=nYO>?*9W!N^(OFCXwd zc))k7fP0n#*Y*PLg9+T*3b>Cny#Mizk9qd{m%VHzo(#+mY8jslc$n{BT&lKU^#^eQ zmZcN;ZZur)V}EtE>&4pakK4UhUisB@BY;c%-4VvTkK0~<+%bFivIEQBU*K-IaBgqH z`Hl;$VhSxXPKvYLE^Ip2a(I$w$OJa;2A25=r+ao>6YM>17RA|b@L*rSwniIP=?8n3 zeqitZz~1*~|E{yQ_teh1^zMnZ0n4G>14nD$T3fJJO?Z0Z?a9s$CoeJxS@RwYxz&Dm z!o-KiXP<0UEw1m#dB^?zpZlCI*SQWV@Ej@NIsSm}xB!2HKhMDnhg=Pm81LI}e{Q-w z@dLa6M$QywzJ6X2{#OSqGG4CyIkju|3@ho1z1IwWZ!)poYM@FSm#O zYG1j>^6*tv)s~LO41D>kIA<{&n!~`oo;e)A2@DO58xwdJ#7ZkP zFz|Yxl3Dra^xPFb+q-uDKI*-6N$~zVg38l6->GX$+D&cR@}ghqM6Jlzm(I)kWzNie zxaH=VW5&`-xlZqIY&>+f>(a9DHD(WwcFS6?PrUsw`Q)}-&z*}O9um!+o}YEQDfsZV z-01Cjw-4#6Z?Cyi{Nnsc-+n*8*d2wB*3C1TtQoxAcTNJUwwgfV!B2CHmM+}n*uc;! ztf(?${({D%zggbgzP|tCzrXx)mYS-;2}S}UQ?sO0m;dvh@QYQd~PQ8D3;wuJ748%Yy*OazupaTA;P=F~}z$Jb94S=tvoPYfF(>dK+S|z$V2XC@`QytE-fp0G@pDp$fiA~;NO(YV?WjB#H@P6 zChf`h@Ng5GK)``J8yPqbtl4xz>Gc{00R{$!jz>{RoGKOtf&_Q6UX(q+0174ZvtBSZ{Hg{l^ zEwOkg^mp0pnc}g2bKOohmdF&T{B4ttJ!Pdn_c16g6>Qf$ZS?5$uNRxVyKRjpcig@llbd`k>_-KkPD_Pu&&0Pcn*DTwwy8|? zESjt${-SsyTa{C*F<+po_}0F_xutD-K`-{*benGIwm^IGqOX5N#3Di-UfFFlr651F zGvev529`ZZ4cTjdZqHxD#|oWaq+`5=Jt6%#{D z&j!|-%!R>oUW+^43FK*=!pE7K(a&JCf!)jIQ0e5xxkr9oV6zN)W1Ph0JJZBXs&DPF zrZzM(MF)nt)ilK{(RlO5aBcn?kIt(vB{=VO2Q119O+3?lkP zfgDl-3{%(wWcJ)(WvDc4aI-p<$j3aPfhXW$oV!Dl*q+0jfj68)wm)c*obX_YuEarG zrVR~p8j75*5uHX;Hu9uBQfR+fcr3|ULs;-mLcpD+%S@Iho@J_OZ|-MwKC)lRsB6@11_|O`mDBNXd@lo82;>YhAg(%xd$Zu#u^vrq%nL3?l~ssIy}O{3 zc5|Ws>8AZGF%6u_8qE7n{Pfc`>73(z_q- zQWsT3O!F9&-&?U=>}cv)bktcnHDK9tThsURE>3pyc$?wwGVzLKO^)C9D!0De1ZDx* z7YwX68yaUwJ>+<7bLd~HxkKak5Cx?*vm6+A{SIEl*O2vCG^>*R!eusxgHjJ?xM?Oe zacK203ZyWwbj;g&)%?H#mOPHh((^vFnEzpH&N!=Zf0;tl3eAVpcO5upeP<%~lNM#m z%N{O#VOysi^YuI2%bgRy^5sb$kwmU&4Q8QD0e&Zbb#&ekbqH5E!0zQTQLww?%;8-u z0s5tB9E&|)I9`1cW%iR{+y2eL5{DAnAD!hqx#8g3PqV%svk*H~b#uq!mfT}9Z%kfE zUpDkpI8^l*k zK`G);Ud0Ae8-}YK4;WQ-8aGZ|x+Z3u11s-_8I0@+S@J~- zd%R4N1y6^uDF4vBW!Tu}vpqT2$-6lB`m6;k76%&nleBI0kDOdo^nf`x|pWSAG34M|ky_i$+D;FLpmkYL?m?p=Mh4F+cW=lQ`r2V+R(tb9*Y-n0HQaQB;`N z;UsZLpz((D(eDS?Z5keO$DeSNICX+m;YXv?!vk!ZDMcNsZBBx$M47&nHBkPK5*Ed%b$8DbX^)a6T zW0ux~80HFpwiT;bHf%0slT=_}eD$Y+S;m2}Y(ksKl!ZN>K8FNPmwmI_E%NqC8c+F} zs}fm%^p1%ZM1PQ4y?~U${pZKWVx5#dx=kh2DJI0uKqx<2$p>Jt}2$K0Ml+bf8V~$A@dqBB{^w zvR&%;bFn+xJi6Ta;KDikiyb_2i}o$^IK;tL!x3llL*D&6(}~FpElcYam>%^z{8TPr z{Os<)B$Tk~*VCH%>0t%&gkkJD650zB+Mg-x zj0xa$Vqgh2U@?5bwD+I@Ph*Q)0?P&~J6C}g#@h-mA6S-YuqpbwKhW*kn@Y;oVv;vT``EWqOZf!Y3#v;7a|_DtDRJD4qJH0$hQHfv~hO<{Je zVYX;sb`@bVt68-8s-vG`XOd_t8S#NsjYwH`w%dwK>e(^J-(;g9D4Tlluw_`chK*C@eO&_1X=<&B)(DI@tQe3EaDJ>PIFZXu@w1ItV4X6YNv3o52LI!t?1-29hG+w2E( zyLF4BLJRLI6Q=+!=NC)@5-tATEWroZiZ-;DzF^B<(Y9@}+)KsQpbo=!71s78EU6tE zJLjC}nBvh{W8K@smKxEP)zFs3;W<&{WJUv9Mn_xfgtn{~9{wf{%yU>+Qnu+>x|n$! z<+w7JtLCW0fyRg{2l6McFq*}DINIoz*JAsCd3Qcd?w_Tc$iruC#s0?f*uRuN2bG@8 z^AK#8IpQdrGez==*Q*C^nS0#U?cr02>0@DHb-ux5Xu)jrqDeY|$w7i^Z5E4}L1X!! zRq7Ee{8}5*Gg%Uq+#MyFtGE_6T$xrM!CsQF`{l{jHk)lh86H6m9w$mXy4QI0&N-2C z;>7lDwsZ#12_l}wJJ^aPPG0tZlAEIEn9%k+Q>~}%S?qFzNuR6ug=U|v!u^% zQD3Hs-s49z-zSR9**UY|L0fExfRRG8(dI@TfktBuu3ZNgxV>2Qmus!Xg{58##7bTM zvCNTXagboK%V6RPXxdT2m2+WQ`H%Lp32X@=Y%gc}bo89*?m2Plh)2q5kFzapsT!V> z|D3qk!d5JCmS5HL>X#$abR1YpSZ>Vev7X>-O4Gx6>(GPe%(TzVg+JtryLF!E-2e$vIh()Aon>=3KOSuc+s?^O%nHN|h-=cOJ8* zC+^f+puc3p1eY7L?Pn~oU(w zqP2(3c_CYN!>QAc9Y6UTON}}9@Y2E84w~!VD5ur(Wo0O-RwBEFmCs0ZaM;V63`5s~ z8DdkOF!#2 zZ=w0>k||5pUb1_n{4ee4sVr5el}FivZ?jI3uweC?V949`BjmCMw;_jO+s^$t z(Z?D(z2xMM$-8$Zw@6vLi2t25^{8xEs&GUpqvVXxsH+ZqbB=aaO_q1v6tiVTaj@i* z7_a*~*qjO-uW_@N?ATPI&|Z>pDSJxT3R}H9vU+KyQL-shHifX4GO$m1z&Ufn@dXbi z%*j}=Zt1j)g%e6xoLvlBGheheyl%~0&>42CHLs)HZYp~ggWLA0(axa|k{4SI9U} z*sv)y;%KA9j8LHy4tb{!ME~73VWY;QgG`nNa@HGImfVS))@hV%%9nD~rX<2C{p&f~ zTV7IsPQ56I%A4{qwI^)SW!4xAOO~CTvULk)|45v>fOFTzX`BytvUg6X+M~ba(LKAj z!Nn2VVYYBdG2$qrqyp2qwFw=+Vxo9rA2K#s&d`Y4y29?l&4ku@S5>Dxt2vjFdhASg zVwJYu8v*u3Z)bY$i@o1_O``9i;-0A33$_7??>Ogma0aP8RS4^3Rh+i7QeobTLu+=K zZ1ufRzoETuX-oEm+l{*SPlR0AwKV$cQlGuNX(`z^JZIc!OKnSuh%wj~;TL-Lh;CqD zu5Vfa%kff{NndqN&1jxHw^?7Xmz8aC0|SHNPZka%1_lNl1_lNO3C0k{c3xgy9xhg4 zULJ8?4nARCVPQULVO|MIc0N8KeqIS7J|RgyVLl;YUI{S~5fLd-5m6}-X-QF0QAsH= zDN!kD8EI)%NiHQZQDtdSB{3;w8EGvUNh4`~2{kTBb9QMZX>nB%Nkd_E6%G|8SuGVA zO;rg~4Ji|SK^a+PDOoisX)9$}B{exUO?g8(1!Xx!9eGt_6=h{@Rb?GbRTX6&Ej1lo zZEZCTHDetWV_j`?O?6{KT`OZnYgaSN0CVkRTX`uTW_@2ytyJz{579zDDR*b> z2saH^M^jHHGgn6&Zx?GnPaAJflR#J7Krg#+5A(24^NdidvM7r@Kb^8PlY|@@3rjaW zQx7RQ7Z+P+4_h~1J5OtuFk44oTkk|yXD4qLCm$~tFIVp%SLa|Krw|{nNN>;RV4sK( z4^NNKP~Xt#un^zCkhBQTl<1K3sEC3vpX}tA%#`q&?C7%e@QUK-f()e8~}yt3HDp!PJ$&PctE42$VymJ_n|mR6dz zhRAe8c}}lzZ^@2qEs5#L_i8QAXsIq~uPpB^%UfLKr9Xi&VS!-&CDE#mAhTnP8+y%W zEYz6(K>7PmgGbL)Cv=Bw==bYv$?k5coZ3-2Yf9DRzRcBA3pcMWT(l+;Vm z4&QJy^2=xc+jr6qEwaig=`5@4D65^6p5GRe+1y^+RNXwavZ=4TrFB|Y+w3VF?VZzS zPMtBOZ|c&Soy%uUTREkF_2TX=%evPrnZ9Mol&!0#Y*{jU>zdiS*3H_vYWkjyb9Zi< zv13c;r0GkS&Reu<$9)}>(*?UxoqpPEt8JyTJ>O8+u36qpWm6kY3K2| zYY!dVdwAQCt9#BozJB%g>5~^O-+cb)-s9U(KEHVS>iO$$uRs6&`|sbk@4r94|M2m{ zrG1CnIQ}ube-zQdy?yEOV)Euoh3lSX-rJM^&AhXCq4x22|17IN zvbGBsFt9}=xCs>A5On%ez|7lY+jc00PatHb!^~Dgw{$5v5vlaWrcaJaa8FIUn7J-D zB+x@)<%y)+?MtWJ_SXLBCgc}uQmE`}`KV=b!11G7kLLT&P?ryQ*?Tp#%EIttOlhR( zRM8NZ4EE6QM|)XSLoUr#pDG;VFjb{IsEKvz^jiyGUaQnm6%U(Oc+ll~?7y=Yr_01V zd3hwh@onaj`A#wmBX|vuJh;&^(Q|@>+|=Aa%Hw_^RhkuMC^LTr4(0dwR^(U(cfFm0di@DqF^Rf>oyGQ{;@qj+-A6 zT10&`X0l98-S#j|w2+6C;51$#pS>o5WUN*ZPD(sh6 zJ~ujorMERY^xE=g;AigJqg$Qn^BQF(W;Ai%daz(&pQ=_QGb`^D_O@xg z6EDb4lwyf;p4{fvev>)7BFam6qlVDo1eG^J|8jSE?pbwIXQ7nv8e>_`IsP4|#j}I# zRaeeCx&PCh9U-nxH}6&J<{wY3NV3el5_-G!9Lv^cmNP@9&0^NQdcnE4@k^!Y)T_bs z{QtP^5`XYc+;kr6r?^tVx=qzB3tG&?*0Xemy;$KWS*9W7vbZwh!hz+1E`OJC=1(a& zvZb@+_Q54}h7~^~=3DLiVRZ6Hh+nGV(tG{~5A7ET2>*5EO~Cuw&`selJ&!+qmhwsd z{3c~N6;02gC`;kzOBS`f5pm>>e<3D5uVBiuqnRE~hs1Wg+cf{*m#2R98*g2`v6Wf+ z%7a*qj-Kl(0%qqQyv|8C_E;*lNm1(G>Z)tphsBRD>}y*4BJZCU^I@?T35)42KAGEZ zHa$2dE}VPwiAB_biEpbm?f4#fH2qp;SLg&)*~rp&mLDFUTkm7G=B-YqRk)_uCKdIa z@e@~OEZof%BBddkE%?&bU#QQo!(_^57lHDC$<5*{ob3#6Rb!hueXjXQC~{VwRj4iy zNi(~mDE-a5s-)v~XN|_}8)rJyloY25U7oJhndn{P>HPFZn5o6KCoUfBQ|_*vqcShS zmA!O}N^tp(kTpt6+r9aH&drz-AZ%Eau6-vnrgX>Uc{aIzWvhz2FXeD9wRoE0@$4a! z!e3|heNRs6H!?ow5I3@CpitqZE3=|cE$rOfwnXJ`YIeH(<&!4_B7Ek*`84C0kKg6Crz$)Du=si$ z7MpzJ28)i0z~cSCLMLu|$>Qd9Pi#e*f%3d!$S>7HAw&vY5n^CLDf>pUi zNhXcG(=n0jsj4J*YR6oL&pjd5!cwc^^n?n^xzX zjGXE=*=Fte+AqejPMe+V7d7UU{uiDbcd%G=o?)ZK#Y@wI*J)09H)9HC1jpiMGaj@Y zGyA3=#j#ARP3-c5uMQb53)YBrTnJiuhf$#M$07fOptbs6)F<41!!6^}eZob`G{9Zs zU*bnO<(Vf;66P|UJS-vCx^1PO*Q)(jr>L#G&gz%z!c(ceuJ?%5sb@y4nKqZM9i6j! z&de*P$_f_p2Tn;=+dFMy!))Gs$<71e%WHJ>_U~0Q+ZnLba+3JsJBmE%va9tDaWHb* zUT6}Uu_68%XD`=}w~qW}4iA{#J8)KO9MwG*FuA|zwSDo8xjVWV^i~Td8Z-zo1?4_y zF4ikG5|X&RnC;TJa{-$a4lZ4lzDC{PweOOI8gIS(+Z3)zRZpFAgUfqc%BE|w+f&xP zl|EgVRU|WQVR{s!Pw>=fGI56jIx{|{GF2{By02yanlpieMUijgQ-^|fK3{<*;bk$Y zav?K0tRFP~n^iucnMwN@oAM4v&Sy5vG7lUMc! z3AI@@hP9Ofl2fS1FIH8V@WuJk>COb76*g( z%yWJ-C_Om9IblMp;f#koX&e<|Yc4PgPkF%fqK$!tuYf`4#v|!g4yk}Y#->)a0etZq z7v2kbN*vDl(W0N=v*(|Nz)Y5aV=Jo#n3w20-gAFd#L+9ld3L$VNBVylyO~Vf>bq~2 z?vkH1iw{>VinlHI*3@2j!y|svlx|xqP5rqBwho{9swOmBKUQEjp5e&pvEXY#f5RO? zj_Moy3koe@)cz?%iPdlmGCfV;+PZrkWZuG9`^$V_AR@a1Sc@Cs4@K5 zo0;u`lir?*9r`Yc?!^^O z(%TXW9lyErt^Ufri)HgVpZr_w<`1NTpQ}C+GS-iOSMoFD<;<*2T7o-|^U8>A+wf<* z-R4X^UVibFC129*XR`;(7hGBVPme|JK?2)O^DS(z^vak;4!o;YKX8wKLHW1&fB)8= zx4zF`|6{Lv153pQR?BS*dGmI}N^iTsI(K=+m*?et7Z|>5&(qtG_dbkK=t4P9K)qlB zqtXTD$O#;?47g`s;GA=zagIaf76dBeHyPvCHDuG$)0DEQ6s`J}=c zC8ySn9y_mRYMSWlZeh=S9J1nDkwl{H1!qB%ZO-uqf`%6beHGX`j#~Y?>TqNWOKMtL zCR=XCwzRx9X6p&1?}{>>mN6u4FG$;7&ibJ4<#C3p_KH{S6*b@6K0R;Cx=~(mqWq(H zJxc&1LqHpU1EWm<*PrWU`QOW)Byg8IF#Qu`VB~3N;4Nscd(LF;kiq-Il;uJPi$SAw z14|kM*Q^JPGY)Xh{?IvJqDgTDXJi4BssW=^17lPG_k#%>4vnc>n|1$fl;8Hlm+ewi zvm%SzhD=ijX1xte{eqIU(-{_&DM}ntoFP~v!xW=uCgmV#;?=;KN`07MT#K zzMxb!fk~}_S;e3)@q1~fMO#&R8J`0~8ao3E0|P4qL+AGL&gTp~56aWs+tbP`zOSg` zabVyn=x3_z|MsGTZD-%da0aH6haI># zJFqD&V3Z7CvNhnoHi0A1z-V(vw@7&*)1q$o1qw}{nD?)!et9Ki_CcFyN#~`@k{B0f zT@5ZUPbjc$V9Cg24J%-cG$`;aU{X?G;z=*(`@oP^F}*y!oh_r`_m1fu~_8X_&y8+0S*dy!~Yd6K5l9<&4&w6L@cy_jXL+4`7Tg;LiJAf5)MtbW%+y<3w;R|NZIM6xg1DB!&ci09-iG(SO z0(f;!OmP;R9tuoi35@(77z7&TzcKGG3vc`My{+^`8~=my+Q8EDDUm2@GlmvkfCBZ=S$yJ&W65=j_x4oEDv&vjyg;2h6v9(A7SH zHR77+wi|QgTGXQgGDX!SI1hCcE-=FI%-JQWRjAE!?hP^jSJSS)L>m_L9? zd?QrmU#i~&Caf$eK z@#$eK;ZhoEU%HPeGwxPr^nW_nKUH9&kZoRbsMtlx{LRcOcFddQ&~^9%_Y`xUcdxki z2XuucFtR;pU}a!>zr8>GdfUhA9lQlI>ns;kls8O}T-g~pLvYrN-digKEBd>4PH26# z;8VtgpDP(%61XI0Zj{=+aqdkX;{*o&2@Jv?GPPUgpD@nn`pzJpz+_nYk8Mr?+ky$a zZx8TIKA?H2gVpQ8vWe}S;Sad>IBCL2^9elR&dYFoZk)9eIR_f_SumREeqXy+AZ?9C`z zUbSj|*(#By8Gj_||J;~0NoxkXXM1tP1R<#j|6gqQzoT!eCc_LZjtdKTuN!pS-LP@; z?2WnU>`VcSd>a81C~r!ysGY&OK%i^E4(>Mx_^KON;~XUH*x9^3OtS9gw5w!Q z5@0=U$?av}XueUt>f)Rr0IP zAFCS#6!L2;7H{~qY0Cz_R}S+^zu5atWc4Uu+cIJD42CJco^@?)U|qSKCCR|iBU`;Z zNOym6VpocU`veJrLraA>Re#;olaM8ON!XT0aoP=Iwh9OKhy>R32kW+cSU1~Y@_w!^ z#f)u135?ad`dJS2wVY-UYG8QB&G_j@gT{uP{0fZP4i(H<4gX%3S8MF=vR>FWdqYR( z3RdX}|FU=dtCX2g^P0gpfb;Q&jl6e2OFnM|?0jj!d&_{g(SSkf!m&vQ88jLgCQj_< zI^MuG;n3V4ohz)lL`zIML|M}exV7(aPElv|RN#6Lz`c8Sl~tg+f2aB;^|=yd5?6jH zm^_d;xq2?&hPA(%%w~i*&lTLuyxEy|BYWB8$AEi*0atnho59U>MXNZ= zs@i^^no(CiW7ca1Q32*RY7Fhyk7)-mGcD+nGGMVTSj5UXqg`_*)0^_P=+2H?4D~(} zKAbq-$T@*QvXNC_d%O0Dtcp582lmMic&;??N@-v0I&nB!`WqKS+HfmF<4|y=&j8e3nAuw^ukq<1kJPVy!oWqhCIA(hswfF4W+%P-k z!nBAB%N7J2Ey~(0HlaV=eR@eoyI}&er2$h<^mJ_j7UuwFb{m#D1rMCNY1V!+$pz|60} z?7V=zEQa57dg*av^t4_qj|xhHR z7jW-;z`eWS_HHi6eP>s1`M`Z=H>byjYwxG`f0tzZzLHT~fVo(KqsoAZUx9&7fw}Yp zN528H_yM+M2E1n#*crGO3fD~Uk!&pc)6dn;Q0xAHTe@D#?s#3z4!+6?QH2MI=?ofm zjJ5(C+b`S{(!D5tm-mVRr%C~%(FT@@7kJKIV7D<~mkMAk|G-}7z*NS`sP}+LDuBu0 zX5;J+n!tF=~yUmJh{tB!s6z;Eoz%{Gj-;##PvT6TVj3zMqY+y9{z|vyC z_k@8{ieZ!V1NO%Tyh|0B>my!G6<}1zX85)C{t>-0@df2mZ|0d6oRc!xv}qlq_5+5k z{}_Y?n3c7UcXiK@;$t-0z&=fZv+4tjO#qXWK-uaa48|W=uAM8p9>C>Yz*wijaan;c zx1D#c?5j$LSEU;mgb!>JP+*F5m^{OP`|t+N+ZHCuhgDzJ$>j;Kh8H|toWZs}hIgv~ z>uGVHL6dA}RBy9!(?Hsfb>R+!|xeBtUi29C{L*Vb*|7QfH; z(%@R;0%c>z%YhTty~(QdY+#%c%fR-4L1F{*vI*?P0-Ti#m@EIiFIyJym~8`N<$tC&5WO%4@b8F#+Srdg!+&y@BMR{y?G$eM)I`*{LC>(SSFfO{aER%WV z+bfgQmM!}lX}ItJBL}-uisfd{;N`LFHr%w)Xc;x$Dv_xSZs^OG8SDyleLd^W!ZQ(Y*5adr73GQt>a)K!(kpJpE)@j z7pL>_tHpj{`83)2Ff%K^3C~aE6H``tOp==z@IhtfO4WPMr*xSeKltY-gM`L{^$yOx z9*>SRzR8%CrgN~lLndX%gGL@HF9RjnUm0mKxmK6hq$jMfJjy%S=cGf^)F>;4Zex>& zDZ)`{8=2JAcAU^$u2gg@iCZ;BlWVeX+Q~;O&Pxi_xK)#$G_6u-os+zJP1l1gosCDd zyg4@4MQFE)-s)O?YR9)<+8eC@J=(NhZ#z%3x4|Kv#ld>ee}Nqhdse2tqueip)&+f5mmV`|oREAzO?8UElUdm_G!>W_ z7z!L1*aZ?AOgrApaN9mX?MT|}%t?Yv_|%#b9GQ9KDj1mM-YiIS;@Z8DO`6;AjIv}_ zRD=V=v@oYfJ;pgpo=(>eI&!E}e5*o|me|x2hq$6z9xf3MPqTd3;=E*`T9-t%R8ik* zwR>KC>$@75*C-vaYU0?OQKJa)u=mZ?H;2IHd))eduHZ-o0MjBTtvC&BHPTPQxoG1XKac*BjMbwzplzj zY|)3-a|}ts`x|aa{&PuaWa4R1)Hjk9AR5bCat>6ug zabTay!D$*YgKvV#W3ej|?v{TxFvwK|1WuUXvSXVdCr^xmc~=&j*)m05JrM)+|DkD(Dzko?3On{MnPJ6e8O(S>ohwzClA%^P?D0(JpHVJeT zuqfVmz;4zN6s?tD#JOh8u- z6od2JuCis0J%_}zt6C*p{u!)by5P=SYH+k_fdI?L3};!b95<5%g4@$<9NBj6bmOc{ z=sNAH6e~RInDUzdE+5I`B9CXZsrW2l4)R&i#BH&#b%)@(AdxIPo(u;eSq6qYLq#FQ zjT7z|x%d4~V3wIWxlO~PuuIcUyH&xnU9YH+Lr!D?qe2D)i;_VDw@M>Vss)!uonyDr z490l#mripx6|_dm@C)bJI1A|nFgbWU7V|3L3)m&Fq%%=fJL=+%V`j%})#tK$DP?z` z)KF+#y6#Qt!YADEZ;X~%Hzax|%$k|FD`43atz+Ct3`cYn4y%PvSuri?!+r5rS6c4( zDhjS}SfCwL%#@nPgCUs8c!~Y?skY% zbDP2{$m77$dVp1L)&~}6H6xE}(n8@oiZ*T(S(xni>)`imMoYOo6jRasSS;|2k*|T#;m#wm zqztDC%RaRzX&Pxq6+96)V)I;R#kp_Bl7myct~LBNGSy3*kQKr#thRr`N&PtAo^b9v ztp_J0iX^M9?mQQ8S7*rfillR% zoiSV5B4OLkT!)tCRF-=cH7Sxd4V=3YuWl7pZD#zJQ=NN8;UKeD0(*4J5=l-ihK*7U z3@kQ$Ok5KfE%n*ma;DsjS+-h0;&Nk}!TUqopE2njZok;bw`C`@&c^&X!V4HxXE+LP zf9!Iw!L7T$d#c#F3fBduftTbsw3q~M9M<({y!_l}mdDGvUdOfwgs7FCpJX1@ zXP=SuQ)pt6U|>4A$iF@Dc?0K$gpHRajxfj>Fv`wRlD#5ugzdmnLD`rFL3z%ja}KKZ zrLZJ2`e!Um_6+Y+FHp$O&URzbo6ziFaX982%MsQO%!~{x{=JD15IDdhcd@4}>t@Vd z2IV)_MZL!C%3=p!w3=$PSjGQ1^lj6IrrFU!Ofq*ESUDrKxHtlulT@L1-2@zraHd24ytA=7zKJSNSTv! zAoOR`sny4DE1?`Okv;=HA#1dvAYc zz0sv{o0~24!#<7)jdm3*HU}E{Uo7d-Y7la0(lKcE*)i962h$x(X8#W?(mR<g&)X zWWaE3M!#RzZgvefkp{;s3l`x$47-^*opv;seXz+2V3OmR9^Ap9(XcS>L?ic%MhWI- z#sH=qWp-x^mgo$z^aBk-H=4IDY-@dGC}|+Zkf6(%vFPW6Y3vm?9|IOn_}8%a{A_{T z7i?J`?Qsl;;}R6JDwf3yv_*8ZCJIbppD@R60ow-0mc*bf%n}V82O3mPFqv&Kesg&4 zq$4a=TbP6oG;npua!g?OP`pLn(dE;kEv~!d^a_@Id%R`SjYj1iOMJYverxag#JN-_ za{jX4`e97XD{gC@PdK$+s@LU#WX$IIaU90+3t9s;b|y-p*k=3Wel{juSdscy^feZ(!y*!JOkP&3)ovWZkoY|7#xAz36OHT& zEkzL`vK38hB-<@7SThSW)aERzopZ+WG^72BUWFo!mep%xKkU7I+IE9yd*TmU?EsU1 zJSP~fcd%Z**lN6iiC>_RZ$m@-jt1ckO;dL@X;-w`9%*^Kd-_8u5#uCXt{)AyD_Wxu zFl|g|67@XdWvsG3!R2Mk63-<|yu7#<)Npe-tP9%Rt?RjK^(ig=pZ5DdaK|$<$G>JW z{K2$JxLL=eNqd3o@1LygHM7<)?zudp=b2MeuH><#tV1j__*1mqI43lxHL~z(G;kF# zoLJ1}ka)On@om+qe6RH!x=^eMyCtBqCukR6B?N~S~54V zYOL_RxM~^Gj5ACZ7~}++-g+3_xOtp0qT#*8)U+L&ZocBUaaz}43TwQAc-#%A&;_y_ zhnjB5YH2Yh`dwBFvm_jlw1$3n(sR+1bVjRqcmvu1a_sC1bp zB{cWSjJKMryoF{4WOs`MtXkUCuvcTLzsu79S-aMH`LEKxvQ|rK)$Jy=hy}(AcuqEi zMAytRo!ArJA?2{@EMsZI#+*ZZi}^WMFi3Yy2s#mPZ4%?P>K0LrhC2>Uyi;2;XS8q_ zG%zsFaRS$|?7qqxS*NjvqQ zOj>3CcK`p+LZNrG_A@YBolyOf;~%w{+3H5qw9O4V3wky-g|N=)@y*clNtO8(pt~cv zH>v344h5EIjTS+ThTjql))QK+DtILoTHI$aDswT}YBVt?^zu(&R_b72mpCT5gOR<1 zo84itphBbcfrjbIZhStQ7$n5FX7EpoVZ3%nI=Z99T7v2A12Og)QG7K##Wq&5NJDL;p=+2;rooX!oRm`{swipG&6HBxK^`v?70+P!5VSndcp(NX%>=P z4Gh!QFc_|Aje60z?uO+*-Hv%LOqRTfys==Gi_!-P#TO^%A3h>6A>!SgV6GEFh8Na5 zr0$ySc|L|K(%IMF`T~>w0_Jry7=DI`R()P)ZjJ_7 zfhLcPX~x~*i8oq3{{}^Jv~};cNc<6`?a{>gC@3+5JuzU*qbTRzpv$vwSt=c1a`Dl3 z4~bnc>#}l&km8OdZ&DgX6ePJ${0pAElR1cq$8dwzy|b#$N0T1By`i_D+44n_)rm$) z4Mx5V41yb)OeQW3*H}H_v>?w7hMNx!X31VqTjleMp@CV#jU&Qo)l-LS8`S57GO#AB zWlGRhQ(|B$ICMObp@xOQY2MBH!%-|94XUedvN1ILI>pG()L9fD;5~`lYOnmJG@(TC`uY>9%+XA7jf~!4@FFz*pDW>yxN` z;?m7KL0!8aK6-U0Tedm6fXg#^qNlTdYwG%MMR(PHHvZV{(oxE&9=2-IPrm~jn0ITW zKd@c8g8Syf+*1?Q?wYi>*;0W?T7gkGVu8`Wb7w0j#+htrdir(W&l4H~cS3#zm`#@n za7kOcv!LPKj+@SQ3``kwSSP$-y>MlF_wFfyYyDK+HnJS_x)!x_ZljIF+DOS0k&5lv z8(JL_n1m!6IS#n*Oqk=O$*4b}C1hora9?<&fQaA*Mqjhdv1JaWKR6sVFlg_vjgV;Q zW^NN%b|-n?j&{2SwKq$4%wTx3rp+*+N$E$E_(YSh*)C=dLKXttZMFL9jFKM~#&-Iy z+FB=RsPX*AM&=+TfBlT3p*#i*KK{G*?bv-Igv zMeo_78CLTe-nBAxh`sn<`lO~RgMG#e&IpE-DYB>LasS&{AhSbTEhTtuUswv;12LY4 zreKK^jVTKYXWFJWwAe~8Nv~+|UDa?dx6_S{QAwjEN`u8D;*Nu5gLH#o(SzvhAGhNk zv_4qXxmCE4cUmj+iw1^{nA*MbA1F8QeYiJ$S_8+9=BO2I1{o8@I43p7u2)KE^37h~ zTNt~!$Dxmh<=-*G8rozinI=>00(g zvZ1A$WkSgU0Wq%5T!#ml1#H07sTgTH+jrb`N<|J zzoSv2g0V9+d!h1kT|@tW@3>cg+RAwO5tEO9{AVT`1s1&xX zI9r#&<=BU$l9Q|-4z!eggo~F=2w-o2 z#KzSq;vp#D$-$yNgOT@w2D^mFy?LR0+Zu#_G^btoxMLxk=KN0;u@AQgn`l?G9u1AT zG9_pFl>&W%&;M>V&HFIDCDr9?)spXVLh6p)^LvDTyD^HdXs|I)-#Ks9s(+tuFbEgK zZ_fVev-06yX8Re<273F19Ezr`XgIM>XpLyDxB3O2?_YK#uT|4oR{l_ob3yn2srQ|i z^|PLcy67jlHg-{O*rJxOMH{c&VlQBlIzDH@gWET68SP7KJ9n~CqNCwX-(1FR419h~ zh6!x(FF8s(icbEuatPW|Ef?&>Cir?8)2Y}y2^Fk@9SrZE2J1Vt#QbP_o^j6DKVRL0 zS?@;G_qO+*Ousk(nCJbeaZ4Jv&GjV*4E$Fc`G>vMs?I;fbmx1#vg_^}&5{QgSEs(5 zKKs^{Wfd$E8K>6GN;zH9IITOS@x{%`P234J!Zk0P(+=}puIO4`5%#~rQ&6HoB2Ub3dndo+_vMb)*akfi2mN}=T?a7}R z{U_Ks*qIbLraWkHY-Hxr?GQ**>g41T($xs@Sa5)uonOe~fyM;J6KpIpJ%2bhH8e9R z$eLJiJZxl;ZJd-jdztQBvz!}6_pUtqtiCPpZrLvt_qz|mm*s?ZJ>jj)*JTsh34mBNIbffVAgJ9UcQ%VI4OmUpTautp(E84}> ze758i9y;3j@QB}~0?kK13I&y%x?}`5B_E$CZB}zUUY&iC#B06gD4e=Mdo9Q(N6|;L}76b~}X^2X-BJ zbBkNfhw~xBVPW-29)`+KyQZpmx`fy?Tx@jYWZP`4z!0?Sz)N|4qZfJxO$^*AA6$&D z_-q#Tb4Z-r=f@o+9MWw6vn52qIVYh}XCsU88NG1EtGuSWQ+`}E$S85RrX7ED(J806 zqef@6*-H`*YAYYR!4UUP-~}tAcwOQ`*M#VTgRH{sg$LWjdSuzyQ(S%ASf)71Y&^_w zcI84O&%B)v4smHzyj&_YFiudnsXDVLjfnceX9yY4X~*mz>Z#6*L=4aK~H zc9DyD!w;>n$kmbLbDrpAddXCGlV=G-6N^YoG$Tvx)h1S9UyiaF!ci*6@}&GkE=Hud zSa?bby1$5y;hl3);Bb1}q-U!Cvf~!Gxo5@QSQtH%`!QSe>@7!k9-XuAz_`bu zh4-gpf_&{QMu$sYzn4zA#JrKOppma>#v`BgH*8KQGz(<$J`8ZoOzqXn<~)2_&+6fm zQqy!9@3mcrOFrCTXS@*ccAHk5q7%DCkN_jAu*-$VJDb-2UYF>}cT**`UsADU%8bZL zRo;2M&r;_vYTY<1YxeS)A7(B*KJ)gRrR9Oo#ggtMvdh{$IPjc9^v~MIEJ8XLCf3}N z>=Hl4WOyppJ1{zdNhmdAma+Prl1ZMQPUcJfYh`7&5olIu5l}p-tTKV6apBE{CPCl- znRZQL*p#GlSf**prJfDEX|f^a*Gzt_Ubm0qvY++72;u$LZv`-ORvqZ|4wZ8mdmOp96>?&rqD}U4c4qjyDw=iqE;rf74IN|`y zJdM??IZFarPM#>Mn=acVaD+i{!vW4s32lKHT9xxYxTu{uu;*E!lknmbhuGK@m_4o} zah}aMCLr)tW6FXh_HDQ3mKkjlXE#{IutBL&)}ydHj`1J2^qF5$%}>ntFN{bQ-}~XH zUfU$TpS-D3OAZ_?7g1v3pYy_n*`k3(Y627Mt)~pUFAs38QIHFtq`+dnfPv+NDI-Tp zfK(z!k1&_1glL^LqBY+&VAxY%RHp}@qT5Lp+sNIopro#FbyBpY6XCRLH7iR@J=0qZ83 zE%W+k?NiY>9$Q0%CEvsh?( zZo9(Yhj$&61hi%DBnq9IaLn{p`Q#06Q#9-rx>@Z?6fP<`D|K}RtGJB9?qyveja)vS zj3Js=LLYsyQk~$#!Y9DM>tM)cHld-2Vd}vwktU8NLmu%CPp4$FlX3r?44MSzq_mt0 zdAY{?gX6R`52=d@#b$S7eDgfQtqi6V&kO8g`w$n&2}cvkLzah zJ21+a9$@AD!Qki+z@&4=Vam=P2gV$Rru}axadfO;Q?humu3WG<&Nbmo@|4W=${uV* z!JitZS_n%?UYc-_Z{FG50}1W7m$;c%Jmt;V_JQTzf_I$zE*c0wJHl!sBgU-yZ-WD; zR|BJyXRQE+)msRRAG`ZXq2?+?u!4BB>U3BRcy+EWVz&p6;6I~YFCswWM3?h ztgAS{PEH?T5z~FKE>^Fboj3No1Dn z_}8K!*U-qmNShayw z&P0*RJEKV=XY1rR)fH;57O<6U4CI~fkj#Ic(UDO@AXK*2{o$)~7hk#iFs;6KYbu`w z17{h7MNHA1#~mKxyKR2xxm#Nu=oMxX?3ti{`o@8o|G0RY_a6z%?Yeca%=YV1tu({= z`=hvhXEd*#yFgm+%tB4~{zDRH9M}vs7}?$Olgi`*7+6273Vm_nd-O z?HnfEHO=;0noT#%7t&!AOmO7aa1=>!6#K%Yn{Y_r%7c>lalFS z-5G}jvd+#}+T57X?8|XlEwuShOtbNug8?gVm+qM-8nw_`^Z*Bg8^3~BXRQO@9fk!D zSUEm4urV?T3o!1sU^JXKXU_%h$3C}CSx%ms;CbM}h3PsyvK0rj&m4?B%eabl(d;)< z*OWG{emY&uhSBgUXY0krbu$^nCLGl0X;Pcu$Umcj^Tq*=1K}q%7*;2U0hejmV-pDvQz;I!xW~;u`jvdAn<>jzfy6pj+Q+J*9W0@n)yzkuWGY&-#2ecd=_2{!@tOm{r)+NKFT8Qt;s8rQq!`Bn?FCF)YnnGtZQP>CD59evrokwv z<0vwxQGWrG+6+e#8%Lvtd1g``jWNk45vryMy(V_a#y6UbZ!~EtBx~MaGMLq*8FEO) zrAbBLkg@<%3>%X&N0U6uzb1K(L&`52<=!xAX7Gk_HmO(~HqAL~rlDwV;jB18;*F{R z%Y#Xxeu_3u4k1V7Ql}lb;poYKhasueyKlxqla7O8D;_$^Etx1F|Jdi)W1mjrzXuvq zPo{UYxh`sOefLoA9HYdZHurs;mW*>(%Pv}#$;kOBM*I$=$`l9oF9*C{G3lOZ5}hNo z&h~1T9pA^$~6o6kCl88`;%G3tC{5PQ?WY?XQ|@`1t^ zC4H_Y1&1a%4kv{hjdD*MWuF|B$vG%1;3%HK$X~)J_~yWquMEOp82ES`dEPL5u620m zdVnM50>_Ta_b$pV;A~)g;%M=yf%RVkL!qFPs=zZd4`;<52S$zd`wv>n&(2J~G&6;7 z>fF1MdUsoOTMzKuao{*`{jOu0y_d7nn$yBlo@JJa%AD|CV#)U833qnZG2S~484ncY zo-LC*bnH+;%Ctu^M+@ekXBD1N)|e2`BX-A8amInbxsG}toEVfG{W}lvKA7paz@>8` zOSxI7V~9hU3PZWg0oDWuk%TVo5+3U0o1(U{0;WKV}EDxFz8%H?(%zsBLm`V3C+*qR_qFC0|;!l2W8P{&}oS_I>^1c%m5^BzoKNY7iv6Ve&Ib2{&u1B+vhFhnp2%}`oc zcF17IA)OdUCM~W0uLqdEIrw`iuuAFpcdl}9Xg@8-yK0*BE?$WYJWTEB#3jDR1bHF~9aJ)0?&kzC%xL+?n0KymW<%rMqRelSi_Y6N zeixj4-;Plv;cCK#MtPAYu>cpbhC@0g>5?4`t^q4{XHI{x@}WQm1GfO9+!IFO4-Ntb zjq(MJJUr6hb&hZri1ik}`QIBBSfnT>z$i3_QTN9og8&7g2!@?k-I)X%JU-~X?`rWd zU|`W;h~v<|WO2aN(;@e%{8^zRxf5GfYQ-?VIbh+zEEwUy!g2PZ7W-DcI{^n@B)^l} zRL8JsRa~6K+wYGgS!}Lye3-?Sk-J;*^(H3sB_==^+@ z!8DgMx2lt6fdkX!XW1-b@@#?%Su@%`9uvNDa+T~NrAfLEnWjDb=f@=8z~H<_<#At= zM8N^?X*1@%dm`|}QN@8-<%5Hq!C?~@W<`gC$`8C$jvQcSIKV7$im~88W+{_efzHA* zFKs=>lUa=?)8^{$aFUwK@Gk4PhlT+2g~*Dh?G6=J)+c@xYgO{AVt=m@!>1xHE}@&C zdAw_%+&serQo*k|o;2_#Jmbqa#wXE!FEH)c!w&8@4B`%+9K4+}HjTUnQ{T9R{(W>v zaYvG%gxf^1mr_+zCpBe14!AW%f}O`Tp3B35E5%4%|IF@v+V|cqoBiR=s+F@=u?9~x za}*BQAbjSatOk>|0MipW2O*UvjUFeB35;Sh4qQxYi95j{?a`!pgG=*E<3HmWPAWZ7 z#vX@M9GvAOCb?=JVUBR{zjQ-j@m;k9rq@OZ{5p<3rA_*O4t+{-l5`)>5e^<*;RpskheIX>j->^GT7~?nX4)Kg4#>qgDGD4?eUPB| z<$-F?0Y;mvn^(;f@L@8xU^eb}>gvvHY;#CWOMAf=Wu@EG*=5_5`Q>k!`9(y|`s_RT zX$2>1K@oGtCf=`#?o5wl9+Zgtd|;iuNr@>h(WFs$#im_0jM5xVh7nE|UHgS(nlw5N zDKa>TPjM)CJF|`hQ^>?cPyEDqqf;66wHepHSNe8--zg4EOGn(HAGt|{-;+PkNfHTp3tmH2;O z^>yv4LPw=ruLJVG3)Nf@zT&9ha)@KaWrhL=;g%-dIfqm(9ON(v)k|ey+tH}WaY!hK zVLPXj;k`o&KFx|H2WyP471=mM7c#^p-;U;IZohCya>aa!)VunBnA9r{W$Zleap&Ay z3x@gs_{`itXnK@Xlowcd%e0g$etSQyViixzmn4RD3`}ecOhuN?Qv9z|_no_;I#13a zQ8DI_s?3(Y@Mhx*NB){du@**gjYcsMCX)?^WCT{!emx*_NAOsxjNU^AMTJd4)6%5n z+8!DBEwSyJ^5;Nge_^q4Mch+)8=^Ly4(>JiyPuOL^D58-h zeS=Zuh{L+f2F8R2VTLA!H;#N$B2Lywh$ws((s7*k{Gc$8lkt;-0(y=HT;aj44BTrR z*i5dZ6*{TyFxC3Sq-MH&>4igj4m;Fi7=;QNstcw#s5tycs9+A+xYpc(sUm#c^IdL& zF3c_0&oWK__qoVj+|t>Up^4Mwkcq%D&LbA@1sJujFe!a`pmL?j_>PnKyaPOQ4$N~* zR#b4(-O!|a!%6o_Q=#u6*)I)MNsN3sj>Znn?>rm09~@PbcS=~ZP1mskH8K2f6|orO111869gEgA6Yv7V&wcW*ZSxw))NhE zGki`awD7qc(%ivV_xqrxK+{!?MmY~>g(*$)JA!`}M04LT|2XBKq(T3R-A?)*3HmP% z=`WCFjdK)p*l4-gt36oK=CjrM;KH^26>DcpRP>$~d&sfC<^bb{^E3ZF)vaijTVlAm z&e7PTdE@h@jn>S13I7hN_B8e-J90c=(i3rVetgLEij(3B+Z#>4w_I{u?E`+ zPpEro-MDzQk@%SdRy%`(e`W^n6`gnVKv&!W0g1;o`i&}g1T}LGu(~v9PGRELXqXpv zK**+1TBT8%he?RXY3ZEC=-sC3dzwBmJh11TB6?>XqsMus84^xBT$k^J$(LF%7wl;N z-CirQ-s?x6=s&Anl1EO7F#Pr~)k|P5dgx@i`Zi<`KhVfhvS53?CS?hUD;Nh>Zf4aA~Ir(^5N(VJIb+ZW+xU?*3?U<(SH^-*(&{0lb zJ=eTLia{;UCo0+Xb69>l+}1CxAG07pOOch8Q$%Bl#{{M`bM(UA+RRiI_7R_HG~49k zuO*x}X85ESHGXP2A+$Any3RpIrz0I4qFNah28^vOYB$A(Q!oZQnmxVEaC*EC${GV^4J+hs|=88(TBN?0$e`K51Cuq-%mpqYhRM&v^5 z0!C&w76p-n3AY&=@9ydq)^|`a|8!q~;|m*~j8o|YJ_U>12Mxa&J+JdIPEyl-vQ3s< z!GwYFVMB-a_5%;SLcEjK8Vap$^bTPX|F_P2deO>d`(p}Dz7Pl#y2T-&n^Ty^zQV3S zp-GNw=TQN*js=~av$B#NFPmM#6?jCc$G~ZYeBF*lHmRflFU^&UX02ir*|4jtTW{T` zuI`!hZaiAOepbgyovj@%++B)Y2~C_br7s-WWYT6hvPtL7IM}8U=wsN<%O%}l#8dR) z@Dytv1;$NGG7H``vih7jJVoP<#nVp5HIgdamrfl}Id5#U^HGn{i49IIH~AF!tl1@2 z@G-m)u>G>@zEo--p1YxooT?gFq?9%-UeFlD zf3Q_Vb;g62sxvG~n3fBA{&}gc)O71$n^cI;Dt!^D1qoU!mh3vEwYu!c$78jbzYLeD z_wYz?X_)CGEmwbcLz&-j9s@Iv^@jzE`K<43Q0}+?bKnNQb=w67W?nA?26m?zH=9{B z>@~JDu*>XlXg~Mq#hG3WKf%^!l`|U3&o7=ip|nj{XU1BF$aXb8CWeM&2c~y3+)M)R z*K8Jf<*n-)9Q?wsE3r*6$6x`IWKo1-6NAEp1vv-3X0izys$G1g2R=a+w=b;=ku$yJ&Ps)~ZV358AA&i?)2!lH0yxqKrE$dvV)uQx(JIUMUNx2RhWuoA zgC^dCoZbqD6t^3+J9#uqJwABUgwKgLE5Uivt?UkyIgY%Z4;tGW6b>t73354z9FjbH z!+9Y`KqGs^0p61wtY^#wIDsTa*?_HHH;Cg|PUq_E+&4bp51+CoszG@Xj95~3~=D=cg;H>18BF29l3zC=| zZafl^xFqoM$d83eEAm`i1WpBXJ=!I)CqYwVMmSf3i*oJSE9KU=nuOzi?N-q2T-GQP z5!123Mycc=|8@gbO}@eo4;CeX28JUF6K1j7vOMBzv~V?6W8{3@=_IoJ#8HR5K;E(t zCKgr(238pZ23Z*;$z>L-B6<_p-ER4!euO+{p}#93DZ8X9XMb zUhO<6@u;~wS@PZd5ROI;rwh&AUxK%?{8GQq6wd7;@Q}rPhoj(!grjY43@n9fLY4mc zF}kTunb?!YGO?rKld$rd!dSP3jof#RPGLE-QB&lA;Jz(I%C@^q)Sk}OJaA!=&2|YR zJ?8)+!8r-+wl0Twtr8kJJrWv)5*oavZ!oay2sq119Fkn3(P~nn*cI6DNaWfJH?y3L zy_QpiH;PL*aI+ocaNLl@C^f-Yzza}S%aej5y!T$dpNK!lQ^z0>0*x!ixc}U zPn~kP12?4K-^j>l>ftfUWDmO2#N4=pjgKe5?;t~iz~cnhNs1pBvoG?p$Tl3X=5FVX zd-JMu)!RnVHjasP|BMT+-eA>ED_nE1mytX7UbCD>Lr2@GkIY-STy(chRn$!gS^QXO zx|Hv~5|#;@UhmRh6dv+OR8;TGL$|Ppg8R!vrtz}+S7sT`6TWqbWscM&4*vtr3Q8YY z`2K{mFVpX(Mm+70dPp#>Of;?*_hx2i!ZVl(bhR>}WXOB)-;FKzo_M2g8uWhsi(8 zXBCPb>CRAOnzh)0#p8kHg||C7^bTgcw6I#FETO33zTqH$Pt%l1mtHRs5?H@4=G0=n zjD?&gl`nnT+y5y#J~LVVRy9SQC9%uqg#xFG`=N{d^IB9q6wSmgI9}ja?1u@XEYlacs{iJQEis;pL>{L+GP!P zgUcKI4=a?dn0e48(cMXT+Cfbp!Olz_M}BuThKp>Me?(0=a7x~yvr4Mry_Wd7`ICMh zvcLF&D~@6DgVx8ck2g55^1m@IXmoZy=z6|OSb<3@>PJI{=7&ZhmPf_9g33EGlHBIm zOjma^nBM6(MM-QY2W!y6$DZZaFMa&xp*D3{=DA`&7LAPh-)~|MiOnkKHkzTp6@H>g z(!`L|DRtgKOd(u6O6 zIC;Yu#Znk0;u5(y)_q-&(NUy$Ho{TV<-KShE4SK0sTU4gVidl69VlLMQ1?jc`GiN# zUhV1@uQ(ngJh|d=!EYT~O~R5H`dW*c`u1eU&vWQ!n#dh@it}5Jv%tnn*Svq<@c!Ml zK$kC*^S}Zp2FCQXMll0H(U^t82LCn+J1pes@)LN*&+fOD#bW{cHwS?~4%{yq_zcq6 zruAeMHHQ@lv^uyknVgXZ4^GvM- zN$CU3Q`0OY7#NFQG)+@8n6bF_+kqn~Ih;}~Qae^LD7=~@m%z7ep_tDBrd5lrGG5%< z#=){8LA=OOG~%Ir1f#^P0MQ0Vv4n@BOnu^Ejs+pBKRfh`@A)7qkjS6&P)cJR&+9cT zS}fvxnm>+p{0K|mRp-Rf(x4Hf#}=T)Iz!3rhXL!6Y`!XY_qas@vlA{&St4k85taAmM}9_Fz~;7z{&D(a&X

#Z{2(N;Flzk+rsoQwj@&a3N-k*R)>2?A>JL*pr8J#;-+|Pa zdjh*oxnA(d)984laffxO-zNTGhxoK4_Z!kvT5kf!jc)dNMm&6q zf;u0?OAcC>H;P^B*I2*c#V3VhObfYRG;n`-AgFTiAAef|i%bHGLxRB>7OT7i=1k6E z8WZg~cF&VP1+s*#kC( z2VQ&*u^rhgd<+YBbZ~b(U?@spabZZVI>4>4ki#p1-R%K$RRX)x0&X=%_JRiPk`k70 z4V-BYSY9v)btJH>Jrr4@C|UZl|YCW-w|gTRXgLQNceIgCPS3E}6r)t2S+ zKRU#cc0uf20}D?l`yL1G2|mozT3Du~ux&{Y`jf!UwNPls0|B*%+}{rHS2eIRJmlMV zKrE|~<6DBzjs&4o3kCEXWu>%Msa+SVU=X^(DD>@sq{2cLuLa)Ri9!XAvK$Y&=O`)` zJY>1R%=BXOsVj@!mu7qo>Gxc%pwRh+lXsd&;zF)jC0t?oTn$^k>K!clpUk7@6rK7x z?+Zh6@u|q4XLhUB<~~TE9qvyQfUJq*3gj3)hhaJO+lG77jcfjN%N9EENrWUl@3%HL#~Nu$VNk zoH1?rc0j0X0n5K+j-mvw76vh11&$dFEIthEEDPOK7YeNFQxv=#ukxX2PCx; z`PVI!*|R|G2BVNyqpZ#&>4+ava}I{SIG7)|LhQvtF@=RPryBVhW=z@E$d{8Sb#Jpw zP9ulG4tu^wtyK%eqQu2FG{jwR-eL7%Zcg}ov1`mBjp8ASq5@{3s}w{ZM2IFVH0dn= z>i0m1;ap$!0cO7kEKUj~tql5VpOW$)POuSM{72`>rtjuEr)y*-xW*}@s#>zf{5!C8 zO7Zy}49`UZ_b%G3R{U-9T48Hxi~rjc1UC3_t!dD(Y!qI1fG6fA&pL$${{!55j|2=7 zYGy3ptay;(v4ACS0ZZHzmb8YF^V3+88d!21*bb<%xjC>O`^5R>fxw)HVmSvDZ#K&A zX_US4Q0T)lnG=pia~}%LSt#>PVU9|o(6)m@5epUf9FSB~EGTs3zVlPy(1L=U3k0|l z*;ZBZ>oE#x9N~_7lvTZ;!@#UStxjC$IgPa?(y&V|@ybgMAUGVr)Id8pxkIOoa zD+^ryH7R*}ePa#y<{BiX`Df`Ki!$de>(w%i(jSXYkuT&}a3H<3fqjF5KwA#?5(W{g zMq#UkQ_UX=8#M|WT;!VMz`0IO(B*)O>W@bG9WH`dEM5zvOwTZ0*@Tyf@r~_$Vc2S7Wk=4v%lHKZXn(0p1{^Jv&42c zivz<#kMMQY4X%z4ytXjRH&!dkZsg&3BDSF+@6pS9-*gvQE3k-7OSGOPe#4@dyU)W> zSS-r1!ub|+!E5<#jM>9yO~f1Kx6IVibEfzu>W*t}6JhFLiF zptyq~r%wZ?!a_-|gG@bXwFN(^HY5>U=w+IeHjYUb!?r#a}k1(WXEEKkS zC>n5AwCbUFMx*eW11!@T1ZO0cSpJ^jnQ*B7CR;%Q`>v@RY*8}=Url#!V9#S<(%l@! z(N$Ncer8U16Z#u#*_mfp= zCu>l{)A$2VYY(s|FeOZR{)~I&gnwNN1ZP`Kv7gEQNZh*lKzi*oPA0{JM;n;eIZQP? zC_3lfOY?_3OB$G`G~7~BY|2--p1j^9na!s2#-#m>6T<9iCve-@9A zo40>;b$-CULP0R?A9%*%rzY?#Fy`ey`@3t_KjCZZdCn;ao^s%M)Gr{^zu7w^rd(n9 zk*$8;AMice|N7uxSHn3wtYn;-AFwejx}a#SlGd}qLFrI0zjB$5V4%~lX+IVY%X{xb(xW=gU?yI zfrIf-bCA}m1j|}3?g)d#kd^DAShGV{h4_W;tzvzxy?*1)eH(YTER~$juV)u>Kww3f zuCB|L49!DFJLC5Fb$!ugy20^4$*o7mQ0d8uetEN+I}02iALSOC6#rwxo5SqFYJO(U z7jIW~PF8hsCsql~6ldV(gtET594UY*E7)7!! zOmbMJ^-$rVA$y;sWyKeTL#=$Wy2b?+^EWCV@A$}TD$v2eaG;T0g=yj~=cJZi4hf}@ z2?C0#ox-w~pTe7(IT&U&v$xd!3!9#1|QsVIJrMEcB)r!;@U@3bq??t zi&^|n*~~7sy&~`+d(fd!ZoN{`gEQkQpFW)~9J9nQLNJpfjYY6ccA-SFor72;_oS@_ zk!iCmKi*7VG_g&6o{vD|t<05kzs<_F{vsgFzh&dKR)uW>PEG$Gl#_-fmggLH3`h(0t^q77#YqmaHu4l zNnqxe5D{Qx(ep59mTQ=?fn6b@hp|&ALgbMve|t~jT6uMrMeSmVDJxp#5)%sB#cV4+ zx^muow7^ANedcZESnD1E#=Y_zRs}2VeDmmxs;mq{Q-A``RmNULA46ABORJ-c?fHIR z<}>i$s?2dud76-D$Dkp1MfYw`R^r>||Z|Y*x;) z2hFpKelf7j^>Ns!o*5uuXd+l4^>%sos-0pk3$2fc9m#RrQP!@~IHTZSBj56X;ttlh zWCaGcvVdYQ;iL&k-C~!g8Zd8&C@9SGwzzYlxm?57oQYHAh(IGRgM~xGI@T!#?3%GB z5}F0LrWiDCEP4OXl{0RJH;b5N=@sX4)dMEXw`Wghh~-o`(-8DRwdaB0t0nX1edjX_ z5qQdL*3iJJy=KV|;cf*H?qqh>@_@%8t4=%;`MBZD3xE6HUzj%@5nA3Ps^i$CdfH%p z&sJ`W=^xU$rvx=69F&_JQatxmW#vuPxQ1FuiNp@hjWhUrt}@(AHCduCbB@E8m{}JW zTV7nQ=)h=N^GbMr#N$~`9~Wd^%RIZVf}vGSgh@#4fCJm7g02*oL`Jp)4z*?q|2l$s zoOqTS9F?19z$Efl1&1V}teqj&BT|4o@DinXhPI zRtsR{YROQM6KrIgQ!(LBM*@p%f-HwiyKCD7ca|s?#}`Qg^M6cmw{zOyIK_>DN%~9! z*A533{Z)?Pm zGGyy|aELodNRaKALswP|r*-v1ruaXPHW@T(re!q6pUq1!mU`2Z{yX{5+&KwbBBTT| zm%I$l4%vLbNzlt|%f*9Nb}eXk(s5*$zTmi=t&owsXMwdu1e2miB72Ytvrsw1U7lB) z+W%Q5aJQAdj^W*FEN3WXSiqv;z{n@S=5)uA&2T~^n_R(s{uZtLu z+9~B%o|aAtX}HMsqSe*Itw_aS)ANOL28X$oW?ho?2@}_CIOe_JrO@v~SN18ZsQLEp zjGSX|_|!!fUz3f0E(>M-Tpq1fuxdl&t%YqDS{-DPxcvoM_~sRKdE_uKt3)uFPFLvg z*r6zLQo~g)V7h{CK_a_#M`!t~ukHm&S(v=VlgDJjcEQG19Oe!OCCfLsxtvK7ySk!HhLMe_ zp>#_E$78OL6vdvhAIcN>isQn9E^@6|^C;=SgQI#CA5VC{S;#T@&w-?us%(=lKAjw6 zH9x*VD&)w5&a~c*H)oZxv4k-#tPfc6^uipg7UoAmnT@BGR+lbdzUlLpv$4TNc*caD z00SjUwG9l-N`+i*bCme=3|cjn3fr5e_~+UxG?qIBY(IN-Lg~kg4c!b2oMq(_xF_FW z=5q*PWOi_1Qs4-zNoq*?^rHA^zbqrGfx^v?DpM47wuSF?_yH`eJFI%bN9R7l285qaNIsjkY~!WMO@JCF@^KSlq05 zMQz`{HD_gBt#0AHaZqHTMT=y|WEMdcN1o;c7U>cJu3&*APGg0Ge0u`cIbJkyytgKi z|CmR>0Y<*V{27N7KRC2Vl)PgUT;RYOBXF+vo$!`FI*oSr-fs-KcC$!z__KNmGz*FSHfj$S5MXGBbE2Qf)OU}q}lTMuRPyM(fUy}ffhK3i9nnx4osSTZx3z$R~+-Ol- zSHQ%b!mxJv3#q#W2biyXQ(#|{^}#*lF0=Foy?1UkyDrOaXmNIN;>$hUm{fZ=M`%gx zzpCbQe=hIy5q_du!h5XY`|`*2=QmoOU≀Oh~6uXhm@ZucEX1Bo4bq4n>7%*Fuqs z7K1m}@>VnO9Qt6u&&+e^D%baE2As@ddp>Y&KEVB80pFVgT($w6bq|=7gBcSYxXu}{ z)hdQpFI3HE3uz5xaZ(NGb}YQVwW=tHS#rC5%d(b+O>tYzi+qJmxm8Tnx7UWLvQ3=8 zs@A}gX28Ac1NW{AH7A#I1{r92Phej%fkWbe*}r>^B{P|2SFmbnChA!iB>gt=^F8j& zXTZ{$z!{goY!(pS70M`eLH1KRlTiZG*W=6<2~1_eiMj^`eJ%(lI+P?HU{U?Rdi4l% zLj&`<7NJ%FHkF2~9t~?%Cl(_Lv<;7ZSu30{IR1yvN$KZdQ!KQ_|^#g~`1Sgk-fX5#h zjV?H;EnsQAAk=JO-yBOlxU?~{EGUw3s^X0l&c$5YoLQ3%xOXmS>bSr>$$>-f z15=O?qe%yM;sMqh2gNfVcowR$y9!KSxTJT<775b{l3zZ@+7+}oTwwZbZ2D9p)MthR zw+YJ|3zn+~Gukz!KS;y}Zf40daPpkMy5a%%z6%CB5?CV*xV9v)$xL8Lx8VFUrR?%W z2l;LF3m;|8G_Je;#hfF+k97lcodZW)0#mLsvvoqmrwon5x{s>+z3r;-`4xt7n?x(Cv6nLvQNLps}Ue;juOXy8ds#`sN6d%B zd=9tIEP@&z1!Xo^No-(LSeQCLxTfy|2m$46CR7r*Rc?U1tDbjjXQMzmA zLfI(Ueddc~tyuCHIGPKXA|1GAP2gO?P@JT|nr5(g@c~A2FD@H}dZ~nE30+q9TH3RX zHC!09HZRH+ESPBBqHXLjyZiu4v;#|#1G~!xfh%oGW4cK*<|ejBrk z0n?3ztY!yTYzmmIRGEtl*2fgEhaFe^eT$jfk-g{uYup3&H52sI4cN{F{NuJ@Fknv* z;T2#KLdFcfU$80wL2Sy1_Rb0Ckxa?4KXqaCAvg&nfU;DAR{-3L)UoDh9#T=l$I&IfN zUuM>wRZJcQTq_=M&M9C`pTIdYfo+}x_u>gGUJR^k+gRRtX*C~W7Swcmd{E1U!I@2P ziu(apo05soM40OYII?fD);(aaHdtpM$XGg&**Snc*O}dN0i&P5_TmP1`G6hf0W8%D zEHMw*l_xMy|G;j&Kvvm-y~TjJM1jNFfW37=NpS&-ssl?F3zMpnzt0B_vk&?T|AM(~ z1q{l$YAtHkb~m)8EpdFAz*^;F{BLV8#W`bq?I?HgKm-UKw1V3X3j&p(xY!3RCPZ)ltFz`>XFx$Lc2f!qtlurJCqoA2pzH!VYuBdoj(kR6*3;F@XQGvF zc5);)2*wBeqUGF1~C zmofOBIi^r%t^aUINtN)`Xf7o~1s2zYb2n|68Vk;|1x$SQgn>VR;p{XQt5REQt%xGW>Igs+F##OGsS~G}4y@2Cf19wXTdxdURpn50iN^tAl@UF`KYo?$myzs>_!enf(MfmKyjZ z7;wyIxH4Peu30&g`T>^83p~pUxRwZTWOXphGstqynDx{U~gN@aJ@fT{u-3|aLYbmd&aImJ5+4|p86K8=U|czn!w;*HF5U2?Of-ag%&(mBzLGTlp*E-PsxI_0v{L_`7sJ41gHjZZ*yST zz^eRfPM$)YfimCxj00SI3@)q*h?%nXwdw;#^8;L4FEBo0hcD5LyrK%3oH`~ zxSI-Y)m~ibc5kKqZKE6qmK=fGjuY706))L0Okeedx&G#K$?eR}A58ZCy}j|(?Jshx z6Zb8=ta#@Nqt4xjvfc&kOB2`wCa~`?;7Aep#}#D2z`TJ`T!A?tfn6~rq3wd8w1eCl z=1B=E%#C);Mh9|ootd0I_*n$7DBj8usAK1!;Jo_YJ(~yjtrED-C~*33_;f3V;aJVf z*>Mjz6&PpAT^G8*ut1MdtN&rigR=q#jAC^URT8+*ZFtDCU`hp-P50d8rk5Cl4cPW5 z+>GL%pX~T1Nry>M;9IB?Q*r?Be>1kY0N(ZuJ$epI&J7&b<5(WezIEr^N|s%3FK4hk z(tEcci^V}d)72@FynI3_J%&TU|y6u@be@bW^<8E(E;j`a_?-hJk~!0_S@ z!=3Za8}b6CE?{46mn8Jz)doMNr4M+v+8G!ci7+>dJofy$uh<~8fjcyTRl|;3vY5?; zNkHDIf#c%BBkW=4Y*Y@ls`d!!oKx7j;D_U3fuOT2mJbgd^%OMobLta0d2GqV$vHhs z8aG|)IBnv1g@u!eh2yNeL*0`*6P4Qq=G#^NxYM{eBS6No;6{X%^2;6><0XErTB7Zj zRb!P{qb5BUo2Ic`Dfh$%hbCUf$x~hkq<94_@r1<3#eHR3oq@%IGYn1598w%JZ*4f(u)xbm=El;PT_sJY_*7P;CG2X< z=HwL$$oQ+^#G2Ns@Mf9;V^hmr1+OWSjy+}I;82*-{NlpG1dbOhbDjA=re2pY*>P^e zt=4RDZGJ_TI*mkw7M1kQIU64tb%tuD$7%fd5%BPskjD-M(VrGKp8TKatKr08|bFQ+N5Xl$Iax=++3VkKL**0F@v>4pn_ zKVW`7?RK+d7}vuj7IihtCw#mb6$e}S!xWT6(&ZEnaq8MtJ?fmWBf+_E(v1owM@Gh! z#k}c^J_1f0#V<6ymTdCv(p;Lu5vY~FauOrEG)@sc4!Ne1zkvfH0Rr zg1HQX;1W)aS&C2koWI>@>Ja2Bco3(|sNukD^=!gJMqWJyMt;vP1`N&IdM6g^2LGFP zV)0Cm9v|PvGj?@nM1sFGD4e^Z;^2GU>h&IDi2Ay%x40)eZ%%2*-BOuzA9zUBR&e{g;BvUp#D+bhvP!P4I=5BAbZD zr3Q|mIf^Zv>K002k(0e9J+^0J%vsEL;Pr#Wy$635JT5T56|ko_({PW$%G|}jKHM%^ z>a^WO#zSDci)2c`A>N>4j~WDX@+?yY1omUDMh+RT z9Wy%hJPtIl%NU$!lyoe*(C8vtazc5g&6|v4BD`@8@tkJ)ki?gZ8a zOpF=@O~NwrIT?ptKO_p~bu>TZxz2W+^(dRA#3J4{4aec>2e@PWg#!;v#mf?0u!v7<+T ziMcVOO-@db)8M6|(DH_+FFuY;Odm2$xFQZL6p=Kx+j`8;GCKZjnE9-QoLxrK&&*@y zGjQPPIKc9+*o0Yf(*#zPISwo?y+x4#G8YnehyVv*CCtWVPt@MSn>F)>Z_8LyJYXex!e@{3p^X;IM;}1u^ zQ<3MSTwK}RzJ!JJSWJDUyV1qFWjU{vIkV6N2X?)#7ET=jMq!nQTxlAIB{xlI5uT91 zS0dr01B<9G{`oxXIzg0g+D2Uvn_jw-S_vN=p}tNf0w?JkS(DOcxvZ<^{x`?;?}u=n;3mN-3yvt8@I<%?TvLBzl4PZu&U|#;_VjjVbm%pR-BD&*C`V${PVy2atBn%zN8-xMXzKna&ON>b4`m*v&5Dm2M6D6;8K zNaT0WYSEoDky)MRA(IY6vjR&Z`?ficM0Aci&iND)%(jQebhRTdv&zdf^Xbo)_rF<} zsZx;coSJVcEYK)4>(-Uvw1Zq88=S<}1!NfaEaFjH5OPIl>w!O6SNHE;Qo>gFwQ-x^ zj{|8}9r%*5zWRSfc&=4=a`=Y7u z*^uG_>)9{QaqcKSth#{FJ~*1|`ZOk?TeDepIDYYjTR2Nr709qCeV8WlBR|~#>wT~2 zT}({hR2#!Tc#F>v5V|yDcC%}dx!D=VtN$CtRFC$e6Bs$CZD0>}U_8F!$5E?m5BbX`G%^P<992J~(B{C= zBm!Ejp4G@9$0MNcDc!)zm%wEE;Q#~U0SCcl>Jy@N^Y34@SOB2(j>mp&oE70(-XFH@2^ z$Pqbvk;KUetDk4{|9w<<{-fUICG>KqOLf)jSAII%uC?1fHkiUBpk{DbC!_FNSwOp- z#(EzHR$~d4DV;5j7A?*iEa8R{ObQHcnzI(&WD>3b9gAmGAuM*G@#<#gtR1Yf5)JAW ztXT`%)OYk;OR(^8>Sd{D5MLrZXVPRht@&3ic3hDDx3_t>F?)i=yon6+DjS4OFlB9K zYBXxTY0N07z$k3MDC^MFTfJ9Xf>l+ck)vTLtCD3**OsLV8N?SbO!|E2kJVJJ4@;*7 z+leGFK4(^&xtrCaf=MG|b;#=GqYDr3XOyvf(QF^kvbee}y@Abvhf8IO-(PcqU;A_W=f;6Y2*P7$r|IS|_kcGA!hM(7?;UQh$@d+M`W4g=wA^uQlUd zc80x{4<+paja`$BZ5b`(t{hy%B*pB|eHgLp<&{0hDpVY4~^|=eHit>IMv1MG>X}`&GM|T_QIs9mb3{hyBH_z_%++iL&oaJ zVcR7w=?3i;GuUfRwAWo|uY1s5_kz88MtkLr6U7qjg#zu4HcC%5bxMO=rkN~?pUqW$ z%4+7DU5Uz!axa>k6 z5;g^f(>uhQS2OJ4xX7fzxLtTlU;umRgT`~3je-)55(^roI~aK^7_}GtYrSsGY;?o+ z!Quw43+}Zcv)CA%&t5t6@U#2HH3}Es9KXDz@mY7{jKz&Fnp<9WI$Pe^X%%8>^Mu)c zLrXzHd*uf9+6U})H`tAIFV{Y3H+ailqj9d=gk32~U8(Bwp-p~;g(uzL%#ohZXw&Oc z^T&6QU3ynWyU3Gpa8Mb$RwC$9i2j_F_ zU^u*|;o~j_;ey@TCzxVBu;*@Ql;~&_)L;~AJtO{Njfw>8ydy0x8yfgd%>3ijXz))Y z=rMDnzRW&xg&-wSgUcmB+c-T+V;N^=w?!u~u_rSp-<%yZ+4q2=jNK3BJ9o}jb6hqy zz1$!gVR*Iu-j+sLj~gwnHzYz^HMe@#&*m3KC7~4AE*4P3j9~ zTRu4D(<^5kv0B4HU-Lv$$qRPR4UN(gjnW#80y`S{cQA@uG)d|zn;127*)$3iLz*|-TElF)L(%=j*~$~_ zhO(CpbuV9)jcB~vE^EP*{j;%Ev`MKdSSiU&oC6r}PSV3_nPCYP34cXtrixak#*2Fk@ee0?Q@~y{U`XR@)pD&S|r~ zz-)h^dDRtW+Z)XK$}AogEx&qM{-la~7IaTFV*UC${vS`mzZ6ZTcmC5BwPkqR7q*dY zs=8g0!0ym+ZP(V@d+y$rj$q`|U=FCT^>t_x42XFc9K+Snp#SA!@z+CL6533MO+?eK zizkF9E&A6ex`A;)E?XsUr>99H}utZz1I!Uy= zI2dnRAre!-7Q@iuk)Y{#qJeiu1J8~I^#|tzJlZxcVaqINOG{u&^GJzqSjp`6_={}< zL+_rnAMsyp+(0->fDtxhBAr5d}j+?6EBJwG?oN4rUZ!>1c^m3%1*dcQM)?*)z1Be z%?ECVyGOKD{QGhJO77ijs!7J8P4WWiiULgvHyRZt#NOP>c>CxCzK>$cRui?#7ReMC zs%>D7mS8$=%^It~tg>K{;Xc+GpW=mmlAStOa(A$KGPI~pnCQ;Y;;^GFYeHMpiDs7# zt!WR|Y%SfO$RO=Iw#U zry6-@B&J_$|nV#bROS9rgQev&3#Q6k6~!Z7(D5iC{^E<^yM+ zYG^RBUpN&1?Tq4GpK0EV@uy5SWz80?Xp~4`lne`(T*l}T;HnWI6Lfo@^&Z7K)+EKW z*aEdj%Ku)g^~IL(z4@n*@LZwlMyb%ERS`#Ge7MwRGzUFsRh zC9OO2i40sB3?dv;t~xYwzTmyep~WM? zoNv?YbECyrf>lLeX+n@rEOVo<15>O6tN(=-SL24&Np}>#hH-tkmgp8P&T#hW?y2G) zheZ}Nil+v-lryTSW?k!P*;&{0=J56Gd)Hk9+A3~jbG*-1e9-7;|48;jN#Obq%6v@4 z+io55E0wI#BwAv)F%=ZON_sU2#vL9%W)i8IO z(c+-MsJxR=<~>u;3ppN*2Eheoa{qSpudeTwqCs>918c|ka`ryX4u;Q#si!2;jvr5I&iFv&4oNtIt9c!5bbfyq;$g~4D_&VEM64J`?)8Wwoj za;{h@?b6F-I{C#e$FlVKTE{ofJ=rahFy)`jg$Bt7P1X`k?F;8K8Z~~dvh^uo4d=LN z?7`Gx)u`goWGKNJzv1FCC#J=%3>Us!Jn=QGi@)U3{vW4Y8MoS%h*^|qgua%jNZhur z@wOgoQlr@9>CHzp=gg08kzd!&-v3i^+0U8c|7P#6^uPae_WGamKU9{maRqM|d#<9S zGA%Zuzr}gSBGrTz4}(|db~LD3$jEjyZSZjQXlPMYXw=tg)V;yzAj%@dz{sS*xKX{y zv!Uh82VT~WM)woxSAQ32?00J@P`JRzz#$}IlJQ^xBQqzjj8}nxV3SLuke=HVhrr|( z&uTXjt|iNbOw(JrY%By6nb-`|&ct{gIyrT9+`fMmXD_j2iL5QzR}#B3sI7;STV5le zz>$%qYq7>8r_fnSnOx!;YZf_tWOD6UrL(1OsblidZi%Gm4GkfyLMEH#-Z)Wt`na#O zUFoY249O)H(FQJsO>5)b?jC^=fXUmUA-3EN?HuveTh*5O?Cz+;@7!np>St=SD==-#Z zTQj9qG|Z&OgPF@@$pWP=!#x+6cy(+xiHYRxJm@Byvt#1)*qSd3uN%F%@;N-wUCEP0 zu;25Tb4tJENeO;O4>cEoV6|0mlQ? zLOurfnMG|H8W@;87c@9?2Nqm(<_icBbd!*DnW#H4L7`K{C}F~J6PF3%jI7#;1~0uG zV-IqwulVzcTX##zBhLW+)WD|oNuMGyTZ);L{}ZpBYgk-rllTO%=%~Mfta?o4bz;SJZO<^z0<_V zrlt_craoI{wWnH;f*`9xlY^75@e&WC6$*_kMI5J=2z+8unIKYhLe7Rks?B(ZM3IM5 zYR@EZ-JKza&ne4`Bm}a~<@<1eS=lDwLe2fP-!D`Nt$e=9O?XC-CtH)tB4@U#Cc%wt z0zM5*%pxHTjeL5s9G6A&(u-O*=TC2R==R7|)5qL8 z7d|vaXj|DWR@fH6ygu6M(CfvqwHlXBOsIP)ZI$qcX-?|PfWBul4)|&P>)9aSUh(XR zY?DcH6KBwji=0|>3oe~hU(qwkwQ;*i@Fd-3FDAMxHguG@D=c>rYW>tCGs#_Kg2bZ3 ziaR`vy(DL;EaJA<5%c)GimXEb1FO>&#SS^|69(+cMH;sEFN&)+OmyO3yDPK$nP!9F z)CrC%S__%geI5&PYut9-q|~@Y^`NT0A@jR?X>B`V6@^m;=5btT6A?6Z(7dG-^Wa!zod1HTVSLGhrv1}cq|~hh6N1+yz25P~s!dRl zziC37(4r642HO_Pz7c2^F=E`K(V-}m6_ae76L>ND#Y?em7IDS~SzT!}9*Hdvn5^LR z&#+_R1VzCPhZb?Yjm&P_PO-+_5at)s=uA1`z#w^o(d3501kR8LlZ6&EFbgG^u}7%Z zzb|Yslnvm>Fkoi)OFhQH(&WSG)5@g0p@GwHVz-fj$(n;A%UG2jH1TUJWjdPEwQWMu zA)$A<`D&L=g?b!WB)&$0)$mhe-1)D*VjGI2Di2KTyRy>q+|J1zAEzvr=}2g)b`msk zPf$9jwOaMYR=&*>KRPKCEMyU$;3&S!`j~UD0#^ZU(IhW z$G6UTB(n8tu#wkBW=S1Kb}NTNLMJ}3$bMPC;%9J(UFpyj_6rLo8#g$}y)ih>tKrLb z`9L!>i@`s6h83q6c_cgpv;rG9o2syKH8L~!alqxu>PJ6i-E9UX3z+vWIdNY4y~F0k7uTd$=-j+gs>r|X0+SG% zVOQXqWbsag!+Pr%+|qfXD0D>GRk(zqEog!he>ultxi<`K1}%y_O*an9eo^2I=5b^Y zH8{W^wt>az#X}~x0ES@ogexo>2~5=+jta?bV6;|I7LeR=fH~zsk9|crBY(!#fU}Pq zlw6eAge>!&`_i`kTgrKW|7gHw=6OP1Tn-FOfZ>$hvh#ViM?fzU%|r3 zv!EvI@A}--Jf}JUj{a?ew&gerE7CdZa`|itRDbmEzFilOOdt*S{ri`Y-R;OpC&T;}yMsL^@ zZ#Ze*Xyol-;JhQPm(pzVgi%4kN%4iF!j}flDGvM$PD)=G#1xu5I~p1)!uihlFSlS6 zzu~}X)6gKpARxwYQNV%4V3Ld{Ilm^xZ3)wcf z@N=BtDLKIY!GUkW0alwvO_gS;w{9N;m^B3$1Fqf=X>j1k^lq#5h&dYC_O`d{$<-UV z!rc-I-4fBMz$9VId2@&^x?O&b@T!0LP8RVg-$y1qbE6Fo->55c}|$H|CJ| zmj?M7M!py)#RmuEL=F}CIPqU-)U9C@^Eo75!l><`sIs9+Ou$KZii6^VL*g+_#uW$U z91ffHFtDW@5Zd7=ucKs^a!>b0Bioay%`3%kH5>?ep|b5o%C;k+aj8dJw_Y#{4c_qc zKxD^^TN7F2o?OtfTkvd4iuFxyn~O>bF%QHu9z5v$$NA>Z0lt!@X%hEd|43r5Vl!ZS zy#I%z%8X`RfkxRDCdt}EYB^47Cfy=Dhji~;R5ClP^r};KW|Qs>M@|Wba84$-xlIx` z7&U(!6w+9rr*KH}$m|~szGe>&FbX(a>^a5a;=+H&M?&F{dCVb|70$XbO;#J2R8|}| zt#QnXJ7nJQ%zVRP2^L?)4NTTCPPvZ`$!%cP7ICu7In24@@S{{_>jcLOJqJ~8s3}cp zw(N0W-O$35;=ujnpi)q?Lc#%7hO7TN1=N&|20OThg+y?@J!`sEpkwOt_PqxJSH&8+ zF!x_Ls%0m=MYl=GqG|7y!#ZIPhzzBrh!a8}fKE}_tr+jr1Rq{(xW6T64A?iU8* z4Tm^yFzL-XAg93OaP1Ih0i*UEMp2PNj-S%lKRn}XS;o=hsA!X@$iXNYb4YFq!~L5L zJX;R1%ZnabVL+P1Z$T~?LxTEnfG zP9_EmqCZ@SnIb&l!$ElqC)EXqZ62uK`^hMqkjj5UVtS71j+-}jl&1b$^-n3IgHxQt zNyYG>T+id!F2^>fg9a}5r$*&K{! zagJmN5(;wKc`jaIB@tJ;+(=@Kkb0H03&Nbqs$ox{(^%% z0te;pIPljTM+^A+t|toL3>Ch0mV@^M1NROU(=P}4 z9vMYjNQGqx-1u0N}L+}-FVE=aNvF&14mE8mIHAt zM;thRJmkn|Xt?p1-KAm68WwMv29AaUD+C?+56olBao8y4ZzOS$rD4`T#b7R$thU?} zetAk2X%XQOljA~#;}y!1xNs7d$>2Y>3Tb8MYTAx;(tm_si;TCq!e|2(ZoxfK~D z4=(j3=>L0ed-0(1jYEa5jBi_*6iP(Cyl~8YdeFU-M{y35u?e$>qv2~TgRlq_=Nk=WQwD~f@FP>#G2UTdS+b60eani_f|pHP0gO`(B`068 zOlLKDJ}>J4%bEih->kf$!OOAP<=G?NH;D%h#4+*l9F#F(o@eA_{Kb(|`RG=SR_4^u zlS$oICAHM+9<}P8d&CmWRU^!}h*fSwy2^)+=mY8#EcE3ym{ePutZjnWKGMm~pgZaNCj zI8^w7p}@z7^Ff2en~P#I92BQC39UHdFX^aS(D+G|fkB`_#rgnO37f>ehucpvoPDc# zzsgHvemF~wm$$crcX^lL!c83gpEy?=-E?4J7Z+xGaMtV1ftj1uiG695&2f-7X!?0E zFIx06bE=F{`k1azq7eHvKMeXJxP@E9Y|&XNwuk zws)NEIdXe0IwntDATfPo*sPC|>OGNlY!o^TaZ;oVXwELfwdy`r?QZ=4W-N_7)hA!fP{_bE`+!?cfrCgvT>NyFX|_$W0gqJz znso~tzvecGO=&XDIjn1PP zA2eHBUUc!3`I^3^)~k!_E}PHapS5ONh6;}idw`GFj@!Er9{V}T=~qz&dY(Dp_RY`=| z?#Y^W_&4w@Q{~!1EKcFL7&3fgr=X2i=jQ=>;KkIVF>peL%#kD~`F!c^| zN&{cQZ`J_jc}-Jz9&mCeJm>1!y&&rm*M-ig-+T7ARJnL^eq!%>GADFgK`ijOIKlPF(=iVlf zhz8aV4K6dCgnt}UHYfvK3t(_i=mP^khr0wrlY`?gAsHsqnyZp3O&WKaG!huaIu0=W zsGFA=V|X<2ob_`Z!}%-jR-JFZ#k}Xsr{`t?wiiBk8?lEsY+CG4d?@atymiBrULFxA zT^1)+2L~q!7Po}GX=x1HEVr~Bnb=RfXZyiudFBAio0lw04ru(7Hr}X`a{(LL_@j`AOAB+A`{%N9QHBe(MQpUmcv%rs>7LzO>LQi0$eE6`3gP(yuR+p!oenE zQ^D}Ck(rZU#b?Ha9UqgA3u*bR2zcoDl*=$>O^4v&&vE;whX1&_AhD^}dmC460Rv+b z%c)Ht!x{`8GIFOi>CI5uq8MZ(TzTx}i494IxI|7*UtTbe$+b&VyTl;r-?g5Zo6|0= zFr9r(=lnsBfEzkKmp?2#(mCzW&c){Z@%D8x&e!-oH@TmZHcVaj<^297Q#=-NE|vb| zd{Ra4g$E0_%BclMrQ{MSho|+P@q?n@ZD5t49$@+l6;%@aF z4M}Z0O%+Z(8dC(Ed(@@`G_~tKt4QKtWV6s|I=~>j=EPw+2O+~dDjmHW+2^z<($s`NjW{!Z}tC@H`v=_Drr)e-Su`^A6>&c-p;Q-?qxr8SowzC9Nn(h1& z5B6p@%}`+FoMHXt!b_usM!~FImrl8OE}EGp5bPtlevjFug2y%;Dvz3#y6TsARB;KJ4aDS!3bC39vdE)@`%qsXrGLPMoX#pL4A>1BU5stKwrF>n^w zT(V-z?x0zM?>S{lZX~c-dt5x+!fm$WK_e4OHAA~i$ppnNRfb9?51y_Lmp&P>1G>hA zg3`Pjk4`b!bW)>mak$rEp6-N`+)6tZFiEDYNMMmPkw|Q5Kkbs(#G_Q;(=4$sp>=}b zjVZiLdsH$Snr>*USiqpDlBp&nt9L?~Uu>Ge)ApAOyc3(Aa`gH=f0zo=;+zL>6^waXw#E>2bHiF68xQ(Sv{Y)R|tB|I=+67MXk0aX(Yx+o^oE zM;w%d|M?kxWEJr}w?m2BKjUOev(2o9ZBkBtna{+6=RIg_S1sAVEM(o2mc+K7OJhNk z;;)3ni@YTbN<#KU3C;p;Uv?a5P~H0Hbfc#H>Non0vB!fFno`YUPa8a~ieT#yWz)4h zead8Wi}$LHOIws~+vGG^YGx#)+?I<_Z09xF!SI<^jUy>eM9YEk+?-0c#?~ih6Bs9@ zEvjhZF=P;QW?wotX=kR{r3vzTeI3`HkSTw)QQhw6rfa{HRW@F&+tnsK>CJJrq~8ax zr?fgmKYSIMu=^m7h6jtK027bU{kaNNddk>Gzu0Tp(c2iJ3o4n(?sXjnTZfhE0DJDP6-N2!2|u$}^|ZvoSc z151wT*FEIcEKuNT-f>hnX&H+>$05F{8(eks61$@xERtMPz-Fy=aXME@0!NbppU5YH zPQ?=qf~6M@{P?24A?NaM0fR`z+1EVDNAJ%~W7&3)XR2DkdY#l`J{cyyhfBl_g(d_h z@@WUBDlBN^f8)SkZNVU}64<%QrHPv_!GTNaz@8I6$NGYHo#C*1&?I!{07v|cgXvih znDy@-=;waWsN!VU7W;-t=B0(Jh|fY^H>WBs{mkRsG7V?!c5Jfjk8kQ;AGn zj~9*X6D*FZcr}y6%G~c4`9(1SjgdOz$86~L5f#IS7@TZVV$zX-mD2u>=PHX zDNHK#TzF1D*!8K%ie=ZXd_Tax>hF!?HP5W?|NNl5>eIoiEV+l`qm?%7-?B!l{&)7| zzhXaD|G#=t;Zs5b_a7HwnUvKW_Ksb)Gn1swHMDHy+t{+qK}jUdutkw?Vu!bg6Su0t z0mfY&%ses;%xW4g;yewE%sdRt-X9JNWh6BpyCQsIo$!me4jot3oqj$%*1c?7T$*a8 z9q!*}B;?5Dq@yTw+2BaK*T$BgtP6QN!khyy2ekioJ*?#SRNihu*ZsN} zsmn8ueQEUHb^q_Zt^?7!8+`q4-4E)2z^2M^kjKx8N!lQhzscgrg)_6J`S&D=%s+8R z!7q{9_QgZzNhcinrX@6~UeMrqCGDd4X@RoR0|j=0!;SI+8`)K7FmS#$cI2NSz$WuR zfv0-IVF8`T5|F4mdG(a-q%H#qX}^}L$ww};K>p97_}%) z;@>aFqR7U`Vf$mD!1M#BWOF*WT|QI_EK0Z_SM`Y1Ud4pHywg0-AX(k;qI9ownR(ib z8h^EN!Sb+q+)jVj&OE(gC5!8ZR%5RK1}2dO23-@+&Ixcxv#@MP5|TJ1pVW{gJ!RJw zvjvX4IR;Hi7Y>TeO>C4sb~#FRg6p1^4F;M`2iSa_94F@n{A2!mBY?e2MOo%?!EueX z+lm*}-YkO25y(|6+dQuk<8_W51T)Kj8AASy!d8SzY6; zh+m>Wg|5gUJ^o1#1?D9bEL*BAw~+tNIleCo_4U8hVNr zyi!SE4Ox-e)65#xpd;m4uqKH!?%mIM{TCftg*L2T`QV}Lx8ri}5?_4!*Ldkje*LRN zIggbBz3-Mj;s2X+=*2X>e|OgZd!#S)a{ade-XymK_BRPaT)CXjyrl9N1^5n%I6V~k zlpruGQJ}6%YLep|GY8IT3_jN~Ip!T;Gh4v2hm|?v0NbYne0&eZ{2X;XK8i3bWDnwD z`LckUCz&D4!E08A{xx53u~m274(m@#Sy;8fXU`LpBQ9(QGHQ;o-#1_|)^jsvdd%$Q zz^(U?jgP@hE15~Cq544}lhnJfQvx`Jk6mS3vp++Dqpsmm!2@Qy21&-HsSyu&`uVw} zScNYQ-4s190h5iK> zu{_CFJ)5r@|4B&Skgesx%pNu_HbtQ&iCTGzB2Fp{{ES>qijpo*X7VI{3FzVQI>5Jz zQ{WJTz=lJGEC+dKJrH1P12b0wi`;^e6z_WsuM%Ip)sbITIxFMu z5e?n9XYWlrW85`?)ht1J>ykSPCPPMW*$SD zPXqI_%Pfl&xJnKj3-RWtQea|H=n!$(w&sM?(!hIl>6ZTK(wXV{WgnOx6;++-Wk0(_ z>hf8`6-$_`64$Yv;4izo>sCFhpKoOqp$dz~Ka* z!-A4&%8V)sj0xw%ix2Q;H9X35<;ia7sY@_2&SU@6AR6_6=XQe4iA;fIN_+MhOD&?`(cJto#r#<34TbhTS(?#c{@mZkUO8S- zjcwXxR*zRgQ*$`JED)HX;39s2=~-~1rNSPw29_oVewBwJJc$Bj2mRSaMbZ?L9UH|0 z9Ft8I1&b27-#G~VQ+X&Tu}~}3SI8@o&%}YD;DO*X2L3jNWwSg&x2Bx*J1w=jqtoQT zJtZ~0R)ci?1^q=~+ukfOOJkmAASgH|k^9vX&o8XZ1`Q%=iatEXR&nbMERtE%7O&wpQ8HstLp!+-k&t*%XAf`KHy|Y`()DLI-<+1G|p{`y5Z9TMRKp4gv)U zY;6skNeN{d{!ZEop~i>*CH!MbugQLf5o^VGr0=r}TYTB&*E8{w;yu zgTauCnMvdU1J3~_w>2y}3p_&j4U;Ft9XDgIYiLs2%(|*a?Ti4c(E(m3mEF&-xHoUv zQ)e={hk@(H1ir&x`DS`e`EntD{uMKAiP*Jj=KO+|)x|;&-^T^^3iW?hO-?XAHzje3 z4|7i%|A7SNHZ|^b4Z>!M!b)d&CLLhXX<(o9Oy`7ycz~kdsRcYHi2_E7g69}I&Nqma zHS!1~@@z>E;!tFt;=n#3p?toB&>IEjHWn_u1O~NsFEj1Ce(hT}J=roXr|9UBdlxKx z=5618DVKF!HupxIl(gn5dj+No|5`YrF8B(yu@pRD(OIB7A%XkF0nV5MeRmAm<{a>j zXvm3nVE0(W*3^(w&cH9x$f{@ApzNs8deMEdqILL2-YA7B3mbwLIPm)HwE30b`8|yP zzPIwngZw|$r)l=`XU~jxoguWb#QV8_;Ya;=#c&@*HwD${-F63jQVwvubKo&J;AEF5 zX2y6uFIVIg1E<~sJ7q_gBM(H*JXag@q*MBt!~paYQ97%~jx#I4FA~ zA@I=R$<~quD-^9yYhPT_@3B`j=Jx@iMGs{P&gIAdRnvf zlKnn@=kz5S*JlJiVDE6?Ki0rJAwhuas<_Nbo|;UNMG1D+A9c<+h+JWMvqw^Z%|KVk zQT&{O;FksL9E}1;7&c@r6xfj~^hu#3;=wyJ1@;{ZMT;`nC+nR2mzm(E>~qOTcj@%I z9BS##i|23Z=?grz(BlEO-oh_j3Jh)yoP3Jnw+^iUO5E18h$m_&+glE=rhl{b0nc zhXNK3Tmg;wGnoAO9A{rWr1g5;?1Pme8ICNR3)$~XTkqc}(y@_UK~dmZXK|7nTakhR z6SJYz`+0n6(nkdo>!zKoxn`obK3G~mS?HBsz+!{6^5a!Ww?9w8AIDMY0<8wpp%=|IP5>WKzQ0B9g|2NH^+61i~_r6@;s_KpYf2# zr@@_rQDBOK2)`mv$AR`;PC7bEweN7$@;7^3=#{FSR+V0RWc!PfMIWzme=*tiy-)9P z$`*Gfr3Y+!52g4RnToC--qav+DP7>m0Rg{*;-4Rwba1muB*=v;oH`$nnAO0R(UhYq z$r_eW#bm+%>a&<#qYj&+%r+azrUQ3(e~nIjr|9I_)Vu5K*F?J-zx&3%XTu$Q!Ws7L zp8fBD_4LY_s+aS9!i|!{CMEVrvTKGj7cJnfaN=ClkbHTeY{0|3H4a?=)-{MtQ{wv7 zAfnXB!?oekZp(H0vBCk0B032SDGCBkcUkUTXYo*2FYm~|;irI`9e+mx$FbDl9{-iOnWk4|H1rJgL|ize4FOuQ?S_cLWhyk0XB|)KD!5=TE(1465Nk8@RX!* z%u--CNnqTF(zpeErYZxC}l;Ddr!k{(>$w7+eE7ONL_N2cB<3hpTX-N$!qj- z%H;*YB8rlFo9=1)o_){#>FML=b$RzM?%wO(P`&@RF~`0Kr*E(Py7k+ixeq-nioPCV z+2PPurNDhoAur$|mjR>54K3liM9q>lB2EjR*gw3aZ_95WD{$(-Ovi&yOAlPCKJf1; z{{j{R2KG&hIGH54tS+Y(dwc49VdtLJJ)^vvW7do97k&Oz-revdaZ+)TN^;-+>x>=` zWMY{awGME;NZ^Y}W4?8cPwqfFi=&WQ0^>Aivj)@j4<`Z+u`5Juc`4j-luJz5D5W4- zt6w7a>ZQ9~Mjf|S2A`bdm^m>`c}icF-KxT7otYNpAD-RZ+^!$LfBK)KV|UCq$NS7S z|C;HrP?7Dtgmw~#tkx8_z(r2(D!mJ*tn3z)ninQvxb+Y_uarkiBg@L;oW_fdN?%RM z6kZ#@FGKa=gqDR19h;pPnOxfRX4;g#oX0rz^#MkqTE3YYi3b~4c`TS_c8PR~EVe5% zu;~c=$9RbOw6Idhg$!OeluldPwuS}mY*LNOS>NJDCYC}e5RS-PG42yvJdyE7GZn6FCmTo z$_osdxb==qPz-YEEjS$HtH~P5BOg$@kWGHav{3O@AEy>pj|?L&_2kl~Q~FzGOgzG7 zWgx;OnbKl(BGy4{=J5%g3!F}K+U69nEBfj*HuHIFEOeHi_~f8df7y$K1~xN}2M3sw zodlvX79=kCcaU|SN)wrMyd z(6d3|U^&x^gf0bXj!8bo7kCUE@`OA#I5^Euo4_m|;INRH)9S+p7P&?fWlnDHErngO z(J@YLic8Lj_tsufIqWW<$D+MPdOwpyZ^MTRi!U|m9lY|u$K<2%TA#qs4ToB^YaC{_ zIpv;s7~qwuoz$teXa!@ahF|GHHt!dKox*|z3a*lK792XMwqn7eP7$$!3(1=8JW6d7 zoP{`-#HKD#;+5?9z;H}4fx)Rmp{Zjcv%*9N!vhTeUM*;3m@>6;X6cHJk_TA>7OLG! zpMSLS)2Vrp;Zie~oBk7RW)q6|-oPU8q^F5lTO@#yFXn{70p_zO6ArL($4odVJn_fD z6Pi6&)IH>lG(H?)$yuQ;AX4WNbW*wc#ex>0hC4=`#+OeNFp5O(+Stfr#uB|%bD2yw zm-3VkXFZhbgM)mGE-onUv((hR&EI5TtMPVo|M4%ETdoC(E!=jmX!S#0xiXGN9LiHN z^138y3U0C4d`avS)`^+WA~HYaQ>WUjl%zIEnGY}AG@ZH>xnhzSELGx~r%dF~*xeI& zS}cL%lgsDUj*UlzYNsf5@Nw%bXqGh=S7P3v~0T27%D!>n}C88uQ_Qb)oZ zS+4C>o6RO%a%*`W-w)4&JTCtxu&D5CXi%T=h|@}dg?ruxMzI-50?Q>@IDH&@(@a#D z`4c+$j5e@1?r;?7&tOwwC^Xx0bCKAj9c`Ru4BKtCDB1)loVcsO(c5z;y`cHDhtVNL zg(I5Ts_b499!r_5Wqqrxb^VrSh}M#Y+=bf3^M$!~u40;cVWEeqe85=_g9FW022A`q z5l(VWgZE9NrXvQDE5t0ThqjaJPR(gN(vaZskb=sdkZ+r9oW!X zHkBzqWV5qE+Xn{qfCirlA|ij{*k*C}xYaIVHtng|wPnN^OW}^vLOiB(gnNK*xYbcVaP^Hi& z*K<&8n}v(9+lOu6b}-u5W;x22aPSts_%QoC2RlD&NneU6M@M(Cz>{l|lV_J|Ye}6D zd+M=dCr|2$G^JevFTO=8X}L9MEpzUXm7CZuw&S75l?z8Tg&L0q-+3fbscC|cXx>@S>}>W}&}v3c#;GHlzMaiXn{)BZ zToX>|8zn+!i;tW-TeNxW+^7EL!d3^EnG_g!cpTVz1YG#UCN!>mn#iDYW2r0KhIPDm zCQ0QzU=Z;sJfqIx#NXf%D#B36VgKMDpI`REe|Pj2FuNW3F#8e5alXZZd>LP^u7C37 zp(X#39mnpr%;o;XnL6b~6aSh6tiqcnw7$`FU1qV=$tzOn>4KuE=jLT=sa|MO__Tmo z!{CroYEQQWpWs0chHjCG7DpVPEA4r3@M&N3iDV@gM>a#3N4?7@xM-?5b_KaOv4h4# z<{j9}zJQG>SxR`WL4lgtM7K%jp8DiG`aaVzvuOFolX*NT8@As5@zQM1N}i0efDR`c z2ZpJf4J_M?@3U|yFyB0VKy%T-#&tm(B$r=s)bKH!ubPs;>YLppSCP>8Mv5h`wj)7~ zZ(@(*j)is-mhBdwxO(|M8^}DiIR5ZKqTkdFDji&hI(`4O-B2ix(ssSRsK?IOWFf!J zQ%#ji>1o9=k1rOsX?>PvhDv-hPBtg{QDf{Yhd_(+6j1kHW41o*aQ>29}Rp zY||YVIEi;UxX4X2=up>L$e9-1#Mc&}Rkrdc6I;U5^aa&3Z(Qh1HE!-r^HW~I?NPMc z>h;TXi<`?Q@~#S&TEL(^_h1i4Li)bBP7YiG28}ESv>3Sym`$fWXwxxN656|~Kx5j3 zrJQRXFzIe!R&Kg*fVurR?=*=PIVHzqDNh>h@|Lu?N>gcysUkbPtSrwW_d^d zL~KmVRhqKk$PT6ny=9>a)th-bPj3n?TobfRNl{_>39Zk}2l@8=bLepV!Og#H0!t)& zVRx6Ai@>q9MH(fNoVUt$o-X}!MDxUi4sVTzRni8I{5}U7850(8_A@XmicP+zD)6ZK z;$7qN>nc*O^UAYLw+d^-Pk#QUu4u)!jXilK%}v*97+9JW8MrnyGqHS7W90L2WD-(f zoM~;KJn_T<)`$j?es*7A2`CH#S%tQ&dQh_mfo- z`)29ilccyz;%0(-*XET>k(-=MkKmYmmyFZW8pMPN5y8N7P7u$*1`|UjwvPDkJXa2Xd+1sIsiMPOEd7WFa z$LW|K+#F5vp8}eH>$%+D%frefS~RbRW6`QrZP&^RN0B&K?UtZhN+J?K}uNDJ7pjCj3`LLrqCTLxDM0fn(tUX1`)a8HGmA4Ps&mjN%5} zRvWy<4LO<&#G6D|y#s>we_`=gV5=_RY*moG{J5g{BC~{mxAz5`gajs)1&q!D%o`Y6 z9A>C3cT`^~9(R34%esxhn-o2b4pjXtmfUSv#Q%X&bpx}}0|wCv%%uVB6%Fj#9~ig_ z{)Nn)#IP-}ZR)kQA5YnCP2joy#N@|P=931z$4ePc9^j~C2wQ3@?=?M0>>88ALf-ab zizfjcQl3d44)A?`Ffsag>HFm!FOPIM9bxqgU~w^EaaLflPGHv0WS*uWBA;Y@D=G4C ziD-yQmtca`GGz@x2PT^d?8leJtj%ECbeMCd0_)lx3}Ob%bCwHgFJM+}U=-cKD78Rs z^A5e7A_jSfIOz+l%L<&T6PVN**j)~|wkU8;WZ+zOsA_|`R34MG(d9zh(%vVNgTFS5 zemg8_rCjv0SbFM5|0iY4W*^umeBhY&fx}IKk?VryiyxXs;cZ+AZEo(SqB9sd8B3NK z@CFnxO8mR1Ej@*?+=F-TBth1r90BQkPnS=WJ~^@dMxD0er0$5>T`M|@71%XqPI9?0 zd*Tjey`9X4mdr+$%!UQb#+-USf>uk7ndT*0+1{MYZop(yz`k??i`WK6&jhyl4O}UT z)>cB@Rtwm_Hrg&OWSqZ=folN+(}j7=7v>2~Xk~3+-WP?7L2~#ssivd9t`UusAue z9Ex1_bvpCc@J@3D<{$1%zrXmOi&VP!)8A#Xp-clyiUHTI1>DybaBnK+nmK_*%z;T< zEP-3Wmw?^dmkixp3=9Vt*c6z{FT}M^;HVa0t7>RcPGGc@Sdep!Y4O9@yB&o$ zQWx+qu&am+X_{tKC1k%nJl(Q6E-gq&;R1tz10(AKyVo-rcnyjzADeO?VER78I9Gx< zU<1qdz$nuVOcy8cEHU#b^5DHtQ2TJBYR>}R$S3HD`Bd zEDg+BT5@6CLUZ=rGbio2`EPcj)H0_7%qgdsQ?=$8MAjP{FrPRnl6T5KB#=pp!J$=v zRZW3);R5d07x?}y;9GxUZlpqE^97zO3LJg{y*d+^Ul=Y*x-^fqfkE*V18V|qJdH40Rhec377KFRqBIPYPM7!YCNP@a8Fxs{5f!Td4 zlkf%h15NR62iSx!FvXSB=Img}nZT&YaA5xyM(GLc^EU`eCTN!KULsk^S9W@qjO4*D z3G3b{b ze8!z_pjE>Wz-WI!;+5mJGZUD%KiA}#W*f4V;qy|4)-?h>e>ArpJfVAl(Zq^FIz{uc zL*0|-2cEjut+{#f2?KwG_$fi3q$=x!UkVP&aqeIZ+EMc1pyC_Wq7STbw|2NLVC`GY zsy^rRE}l7uDp^ubF}pgjPiEj)Z_M?hi7Tyu)$0-076UHj6Kh4q7_6_m^=w#SczYfX z1H*-*yPtM5GqG?ou41%IVCP@JyvB(=&$zExEY>bst>oFiDmJU>nQM-5afz<~c&tH_ zQCEvWfI+4;;P~DM_R0kI(gypFmW*`@6E;V1_7+J`UD<#0Fw-v!M$LfCJr7Ux&0&(V zWz2ox7P`1ia)a-c1-wDwmjc7@G`gO%2x|!Ol-+L-dZ5T*sOD0;6U7U2X;{g&b$`$GZReb{#z82yMgh=N#-Y8 zkAF^;)=(%GzOc8`>f&9+Vum^UYZ>Z7%{X)~F!`JDUPzdAQeXn(&P26`zRxG{?2Wx7 zU3Q6SQQf0gmlpf*&;G@3k-H=Az~v~bLs18IZ0otQ^$m+#?G?ATOI!G#4TToV{M3rv_47=5;eD7|9fnQ))cfT5yS)cAuG z+l7#RhUiO)>^c`>7ynXwxcR|@VoA|eN9x{Q_qucLld=6Q$KKr^kIiJ-Z9082;AqN&roRxWvL9+$Z@QQGz$ zTaAV3Ne8~`SJ?lDKB>A7PV96EWSen3XT)-%ypkFNaVaH}M?}GMQyBHl6 zIC%D^O>1IzJiuJ1z*)3_fx}@_Q=R8~OQ}m|n2&{Wm0iu^u{z)8yLb2Tb194LO4N!3 z3%s=Jt7{M4*m%*{CA4)y0^igChL#6xOVYTOHn7iH`6xhvW1@qh+!a}q?XvEw#dlYK z(ZBxK!=AB+fp_8>W;RJK-3{KG>f|r)`x@i_R&rtZtv!q{oIb*Z%gLX!AA5e=lp=cGzC_*gsAPlye!Y& z>=5B}{PSw!0`{~20yrmJ=;PSHVElkxWHSr@0oJ+;?EVdm0SbZ34Zesho+)E@{-WsW zN6yB{OgC5?UJC3pjy66~s#a6?R@(Bj%S<(;QoNMsd(<0b%{p+y_Jg@V4-ktv^ znaERR`u~RAxAgj_^$!lVX&;LAI+XYSDZ{~~9b%19T{3@)E1PJqVk!e>WG@JymaFym#!!6kt^-^nS5p#C{I#xTlVJ7$}b-N z6Xq`S`&#(RcYF!%; z7*b%s=+q)Bz%xPMiTe4KDQA^ZD-0B!<%9Fs92%8YEMpN7D)93#P;6!5oEof>(Ua8l z__St0T88i`k;M~q&2|`G^|_=xW4T8+-_^nrBe#=%s?KRIXS_?^-Y>~47qeqi!O5=n z%a7Cf=UJD`xjfOi{d7s=)lMCgh=MA?#v-Ax@EZYp1V8!xb7)HIzNwpa)~xPM1-JO| z?|Hu>RxMP}>z>}kraJk^G%L1%ghmCel`8!~aw|0^S(oSotK-4ywJVwyFm2v+NORrUe{;MFo3^N}SkS~jXN5z9!X(31sl{Gd$&H+1 zCk~6qPJ3V|<*4G};BmrcwN9gm%;HOinzEO5T6v5LoCVz2BvV%MbbaX-G(PVld^Y)9 ziLQ`$LRX8Bcg5?q<`3I0GxyuSJJ8%``Ru_VZp#(Y844de9KI;Di3Dv?aFZ>1!J*Xh z;2?*kT%ns(uG!Kwsa&%|X^Z_=ZdAJxyr_b+OJUxGhn>Mn7ZcY{R+;J0$X>Ex$;=eL zmzP9lG;K)e448DXnU%+cHd!zym%fPAt09b6-dK@|%sEcRbY0 zCY3$iB$=jgadE$$$jpi)SBae#XN@%tCkypmpL5y#b{?~yWU|k_7GqqjpkkT_(Rnw$Po;zPF3rw6OX?i8aD{Gl#riay6CXF&JS0N1*Pqj(vZwy@b zdht{wFo-FA{3;@>q{zC}rbOV$6jeP52L_gwDGwskA0K;pbI0-~hBZRm54>)z-_pgk zns=#?%&Oztx;8c)H|bzxmV70!MAF>ofkXS1ZD$f1SxXWe*=5Z*nplr18CPs(mHI4Z zq%(g?k9W}miIhaQh=OLxXqJu4{J+cGPubb2ylWQh7TvkyUxmp;OW~PT^}>Gs)3#rh zyeQkUU|*GDl|mA`(uVJr7u)jMLIRcgCVb&gR1k7vlU?vbW!85qscC}N3J*2-&puCG zB4m>ya6~BR!v#lSIYmx&hKBq<-A9BfLYMU2SS`7%;Sg)yq6W3K2}}|P9N6?WH1TXu zVD^)o)U)N1D94jrmYk2$(JBGTb6EV`3SV-?98;aj<+6ZDPVQ@x6fmNepk@ObDzLW{gQnwOZ*yc@;a`FL@S!DT2LC*q#ERJ6C6;_; z>s;Z$HYx3qrIX-_@)I6b2b;bA^m|@lQ;>~0Fq!$noM%E07@{p1rU-I4FeoG(*lE+i zJ|Q9e)|~>b{Wgk1Z!?c68!+;Tjdhk+YgTC6MmiRe>XP5-Ob!zNjE-#EydC%`oG(v)S!Q#E}`vrhbT{x+rN znfCpkXTn)5|1`2oK5%^E;neG>lfaqR!6LGQ?bK=?2L>(yhp^a}?Yb@x7?T1I@X9&z ze&5MIU3Fob@uY7DR8@o-C+=hr|B%3Lzd@lQFXDh1^8^M41qO!Nx7u7!7#P?T4s57* zT@#l8-Gt+8n^K$I zT@G)(Sioiyqur;V#7sOfy0j=Xx|t+i)G9ST>zsB-%w@j4|%f&rWWi3HSJN5}GCNEMPH`J;?4Ra9`-( z33E2Ht(tz$8=C*NHE{gaeIl5nkRTn<&r$c_Ae)(jj@(Qqfyxuj(h`R4j4IqSnKpDW z@+@FfTa(7j#ITy@*8ycm0Vjc`2`-(jLYT`jqNdmmQ=vqb}g&_M&vC%;%cr#uQ>qTs6H zmpCPj;SmG#0Y?P|!LEP_2`mzB68%bso!3;7rJ7!(aF-}FD@0uAw0z?2>6OGK=5SDC zyTB2yln2dHCc7^7usId0%CfwYTDWU^UbEDl33ts8OzgeKyI$#+$=olS%$I-eeb1id z(#Rk1??9XBr336S22GK>88pJh{=QYLpchu+zHaRm)PDixRJzXaAq~5K>!oC!$S3GD?H

$=Bfx=KS+`1lXxP7M=iS=ibncrhYA>(c^8 zu?r4tCJa@)Vs~GNnfzkcWpKK`s0`s?R0zuQlVVjslYWbg(2@rYoD&*YR8>BPdJ8m5oN#C}E|C#xWnj+wwZZO?_Cu!soUMB+ zLR2butuQ*C!)POU@tpG46%nQxjPgHr&eyhbQ?_zn&8Wa|UC&1pMYU0yJ$3NSlZv^W{C zWb9z&-Jt5I&>B3UMfXLcY(|rVMr(9Oi}Q;n&kN1$3=F&(Y^4Ql(Kp(je41p&!L;GU z>O00d+qE_|Y8(Ci;H+%CM)^ROJqrV40h{NJ=BN#<(j5#N6PTPnG+9qzu{qFSdAQNm zqa{dzRpv&+D#^VX681(bT6F`M)Fv>pc5U#V*%lqaM$OEC- zzgsGz7-cn_?G+lO7cbH(za3N>{L{cMm{V6wSz@P{z7qXml= z%VcMR7S9z^n_r-btso4cqc8nLA zZ7(!AX|(XIVAywam)wg+{fs6qfff&gMyCU;1qE!f5e+FDT6iQFgeCR|3a|=qXke6J z=< z;UlGR?10WL1{Mc*XDjv`#^WYUoGn)DG0n%9xY?f&b|~bK6JYu_iBVGEpmfyMEy0Rq zEZvtyxHnjxt$V#(FtI_Rh_QpC#l3^c!J;+h0Bh`pvoRC+Gd0+&cCZM2U`lw@*ulf1 zW5KA(z~XYE)oBB(heD&ALW=_f%i;@6JQfU+94yfVha57R*k^Y!9cXkCXf{02ESDh7 zctJ*>xxsdh=l)-|EDKH^I&`w2%GcqG=Nb-%55Z0PJI<|4YO?*%G)1zBokLF8p(T1k zTht3i))R(nU78#%nq5|~8a|jE{qUb?OCf*s1xB|8Y;F?k&PF!zMlkRv@tp9`D0s}! zGE?TXVXIAphCIWrdD?E0UmK;@GI}sOFtapeZ1Fi>bLM51gF-;F{0C+miLKM5nq*Hf z2Jdc^C}@n(K6qAQ&efXZSC3pORcT;KU=WF5RF+^eS7=dHU=F;{8Z)EScxh|Q0pEgz zc25mPJq{M78x3kZJPZSvG;LafZZvxzV4kSVVjIzux&{fi-E-6mo8#57nrjS zuw@#ss46t_b1|?sFl_hiJ|yI_UrUE=fv0k{Z?*JNCF!MGJh~-jFzWAajpArdp4iC# zz{$m+C5WM!+k%tlLX*Bkv(t+v))~UQ|BM=K6`EZvTJ&BF_rSxo|yv6}{<^yR<&aksw zI_cwf=18;sLnb8&CdCttGBX$#MmI{VU=WwEKI$nb5^_-Z%%!%YK};Ic^;?;2Iaog` zv&P;SFf(lIVwwJ+N$mlX-NoxR z94*UvSmtrJIJ`LS^1|I|Ma#{TtFj)gW~uOJbBI#*Za9>6%kGS4)no<+30095%=Qc{ zh70x>Bpk9SV6wfzWW(BGdxG0BVKy^IgV2G-e~b;CXE+ypux8PVaF^|1abqy&4KR;M zm2wnj&&Xi5*OoPFn0qg>DPBccscs`z4u^SsBwMe`tZiJp`1+D;t14b5g#$io zq^C@iTB0_W@q);4;UvY@Q1^;6L07NzOKdjU*uWst!B{RBY$UT!+4;uAuFdp@6SZ~;xZfVRMpfS1QpTQD=7BP)QYLh0hW;B*bZC=5p zXLN}{#Ne3FGDb<>1F1Dz7A2pN%RO^?>zT#bZpvSqr(R@I{d!#{!{mZY#Kvw%E!%_3 zPe`w^@cmnHRyi$J$8@?4L(2^d)>x1Bk`JueH&`|Au@-D#(rgkYPhm1ET;> zZqu^uLCZuZoW5PfCFUfyu27AggMo1agBn|l8uAtM{r$UA}YN+E;T3YN;K z2400mo>$Fj9c}GK56lvn>koCZY*=uDbMk+WHiZ?9K_9r2{`uUIHaOT^cUc9)0qF8LiT%DH@c6-;$^8x?+BZ{T$lZwT7BTJW-OH=klsyRC1)Tpghc zOmXknvM$_>Uch?ShrPnDJ!b=(^Mgj~9n4$>YUz_0c!brtqLwM`UbU@o*@_dZm^vOa z?g-hiqv3-xv%?FfuI6P71q?d`t}f5LW$(w3HtBp2Yqyg28nz3bhoual*>|0r| z>}M#G&Won=oJ@{Ce0X;>C>t_3u3%Xm!@yhF#d4sF=Yy#1gcc_S7AKA-8;LCeCz=J< z^5*vTsJ-Gm|5GaCflORoi=ao0)Qg=G28j!b!%w<898NJ=((JY<`M|#vmtALLHJc@x zRC9yAaUJI`*wpOmxI#m*Fz#&OxgZCf2FV-DF%Q~GPP7I)u$H;C1~atAII#9^ZPQrC zYH)zren;cUs%4K^H}>1@JHfQkoNdizo_m*b8W?sobeOQXSg^=y2q@k>X_vOFGW*sU zx1?)R&pVsxC{1Hv+R<#w(Bd?sSz<;M`4KQ|+U~trEm1>wB z;MAx&k%70NQ*{A*-jA~eik#)r#uZTv+Et70{}P;C++3{Mn$pnX9vze9;^6k1absIJ zbIF!;ABT)DfoXY-P62(-W-woPe7*kcvjP@tA(7n$69scu_H1^zDseemN$lVGqRnl= zKUxBRv~VB);4jf)$ z2qTLl1B=*&IR&3WSQ;8Wh2FYm_T;)={29Ga&J2b_igBicr)_bl=Oytz>tV<-%k36E5F=u=t*6@z-eP)^83sZ+0|bHtOh&?r5^T!0hl( zfl(~nh)0-#qeI80xq6_-k(igNOXRx?Ow3s(AJD4}kw`y9tc#G5t z#^3W|WbLd{N{(gBc_?<^;L{UUb)rl;=NP3bf;40qMFO@KJt^@k=aG0NR>W7j!GJ+B zgE?pd8&CXK#~&=l7d}MlH#kiJ| zP{AOS!MrAs#dZa=O+|~t3YPYjYv;zP=|nAKjbPkRc}lqYlw2pv*C)CeO$RqmU^pMq zD45J5%dsx4LL<7PtstQ`HJ~jug3W8jkFI*=Ual`a`IW_swk(}&(*5eMWS^V-jXPEb z`Z8%j=X)x5E;~58|Dd0E5^H{e&RU+e@r~hoTDi}&L|!k5WEXz-pJll^Ys`+h!EZJ( zag6+!;KA${}|YK6&VU1FgP-?2&t$TXe?}WWfzqZ=x|uVEhx#)VzVNkJz$B)(n&KX zrD#s_^*pJH~px{==q9}9Z{5<=L7iS(m zYB@1M(W&Xlj6!F(o-@h;M^-%exw(Z)TG6LMK~d$^1V?$B3kEI%Tm75uE}lzv>ykDu zJHs=p)+>5#^fo@z$f6VdLe@>wa$ZbI=V;?Cep8TW=-@7=VdPMpxJ0qDLmWa?W8BVK#F))R7@zrN#j!&Oh!6%pxicPE4AArXS!6 zU_ICo?j3XBjPc_IpZO-b?kf(n@YLGuV>so&*W)pz2QIovXKqLom##`(*(MnKFV#h$ z;z1#Ma!f%WON!H*3C^iTcM6<&^h%yNvC3DtHD*+XnH~%c%M}S?77{UNV6%L2(MMy` zOvY9|nSu)|_{wUOjw&n=WwY-sAvw}+_Dn}PO?%q|0c9+R>o{-;p?$EbcmTjV}XM^r>TM9@@WzQQiHOqp0#vKi^ zGd{9NzF_1sPBNuNr^Bx&|TnUj}x^N)Fb# zI0y??IP90a@xa$o+A-~g%W|#L;Tv{}Tg6m1WeTska?MMT zmtXH-u(yPZ#+?c69vn(SOA4HW;w(7Ucj$8E8L%o>Y-r>@;lO68;p#hsvDMwfN$5;M zqr!qfP7M`7M%}DCwE_kV!5xbwmz`*2koll6En!LFzZe#Ag>HlPXPF9#q6sX@DFuzJ z8yW<;DjJyUj23YjypEQB>EXODRiQiPX|%oIEw`Y6AZIauCX-DUS;98#;f_#}dbi@Z z@_R-8xvxSdO*#@c@j_FditS@bpHDZGWf<-3r=0ZpIQc%`D|2S^iwe{1R$orn5@6(V zaWL8Xh36tuhA4;00S4xZ2A+MDT`OKbV3Le*{1|qiDY=hVddYzW5(OVv90Pm>${G$; z$$e~5wNVP~xxrvCk%`rM$tjVg4NN{I6PO(%4si-!XjG8+*uwioQK;WTe6vB){ND~N z3``!08j}t(G2Td!D%}v;DdQfsK;ndhNy0H@lY&Nu2MhVR{&6&^{JCJ|Xrj#4b$a?f zr&UrDUip<*SFKYl&OGsQ)0YXC&rVdGQFyuP&f?wo{^pgJzY2a8cuCe#_Eo^)k3HLR ziumFlFbVGvu$SXWtoxe1ls9R@=DEEGjLU9ig>U9MU?<8ufv>`W(c}P=MvrcmOu=gw zqlTtOj$T~LFR+?=T&MIzu})|`N7rCZS6lT z7him%a^jRF1v}2H44AqqwPc#)xey2DQZ9pruPTikK2?_l8X7Y7e6Q{Hbzlqdc&H$A zwCQPRLZxs*Bgrw_p|7_p+^&&5?n@ z;lQSzvx`!JXik2l&JyZ&)oA+4EFJd;P5O7ki#5pa^o5kd1B(-V|FMQ8 z%-QP3*p#>TC#%^54db4-nUZWrIFueR_(ZE57kzjkQZ|5rLnD@vS>W69_FPUW<=skW z;yAmr+8IPVo+t?>GB8Nlo^ul{W?hrmxIQOggH_SSq?ehc`7_dJ?Df|8pE>UZGi{m8F?!z%;mY$D-Id2uWU2lv)kh9 z=c9|h_jS*IdBN;<{<$yVpRUffVUmBL^(KD(RgMdCOk0+lT(r3(bzojoqgus=bH*DQ zc_JE^BZO~X&iUA3ld*t-uVL@!seFyyNn5TRV{>5S+0c;0dx-ado|s@oZtaaik4VO) z=Y{_99%-1*!>Bej;fSV=V3*E&mbuG1{8_n@Ca`U}>%hOlQTD|F2A&6OYg&X2oQ*FW z;mSPFr+D4t;KVGB)=-H{Og`85AG}%O80RA!^jNoNuJ633cb)Be9z2_xz*%7eo;WAnC5n~7cj;5kw{PVXY$n_+`JTz#k`HOa}H^o zICiDr0E0!>M^DDwnnSIuO+qG(@-H0N60R_oChpN_V9apX`m<4v#Yyf(qrw+Pi4zT1 zP0ll)ao8j?LrjI$IK)Xaq)8#8VPor*x)PU10n_*<9ANsvC4Fc%Z%+fyn+64gX2F^( z|12flPcRt&Xk`1cP}70=UQLur$u#Sy4vshc+<91~mL8Lzd938_ty0NJVKVpTxJGF9 zNgd2&RW@k0ywS|jrNH^>AG7_RCI0`G1nf(CrW;dUc(1%r{P&zr0gi)$cc$}Pap1YK zpzbBloE<{iHxKZAVVE7_rhS)z^Nj-ky9Hsj4D&)5B-{=NF$8Oz>k(rkq*Z{ES|Ot-tqL7aQYnRRq}C@P%^z2eRby4%;|ogQtVRZX>%mU z9Fm<8$|7-qk%QqaYckItCC(|A*VVj8=3B6q=K!N_gRspZu^UY)EKK|l4)9EHP_%Hi zoZzf*$I;m0phtsrX5(y+69<*wIM&@{c;pkGk>Q#)Da0+3rK94AN9C0Qx9JCD=Y<`Z z;yo+r@luAlsSd0fPDXPM+dR3+;g=o|_1x~mVY@w=WkOG%Nj{Z7;TS7%P*CO|{{;1O zK2K9$T<48w{!2;%Bn?rt7<+u@Ly5?$9Lll&lCrqmZxkQj!OkrG>U16%wrPP zX_R9*$QskYaN>Y~$PsX5Al*d?5$; z+ERI5C^+{u2)sBb*W##f!^vz)6C+Clvjw9{fupHMv*s6uWH$zt7-s7g&5AmWx)uko z+j`4xR8upGb#QIilrb^%LEv_u%&A^4S++&8^)31|W09s-y7#&Uo|5C17o2B>-786Z zUV2KiOiCx~I;Dn-civK45@N{q}jdd@PEih|wo2w#a`+()!$!TEqUxY{O`})|nuuebU4@S)w z2NhhHM0E5Q-*8lsa5DC2HV$D{N^n%X)5!6Las4$W6PrUE|F*PT*mC7U1_R%h1ATFa zbssS4#W?dR^a%(s2^u&m2sll(l$Z5sQn}%zB69AbMuXFXcE=_2UFNjiZ*{5LBb(O7 zQXKm7L19ZoAZw9Q9;@=1IbXXKGa33jm~D1AKPe0MPkZ;2>*?%MnupH4^#62FdQYR& zfrImy9ACJt(LTxW%FV4NlTo(ipzHxXg%~F#6($9akNnI{a(_0w`DDPqJx}h-v7A48 zBK1zuQKwAa9OUcZyu5-v=t#?XHD}Emj&cG_3NB1~cR=T0@{2ghML38{98$e=P*K3? z!bGTg-NC72ouLa%A14sCB z&f_mR4{LHo=Niih)4;dGQNzVa!{H#C zK({o@g5^090*z;%IdWz=a85bEA;2g%fg$_Cys*24ZBGo1OAhipXyE+Pz9y?XK_P4DTwqB)N~aXN41Jnks7qLF(w zBhQLP9{I*2*ZKvHoEJXBDEpX?9(`acXC7DA})}M`+ zmnsT4iHpcDiSQMA;UMJG%O|l-=*Y{doMy`g1D)&4 z!T-!npT7GP^7@nR=fk1#U-gfFn0M=dV9VB-KF43PHHnxREBQFpPG-`);#7B?seXP_ z-DC4Q|3fjCOKaUX%eq`h>^2tB`O##3;mVChxdO(V=o4%nlVv@G6Kaoc;5;N8;c(w0 zfG^`VYXSrJlZ>XHTx;*jxg|z%Ubx2)ae(p24vrV|m>XU&Okm(Y)4=wl>q2h>+lPrV zn&Mucz1;;hAK!RbAk2}w@L0f%ws&80({7m@_SvZ^eQAznp3kxaTsf6&FBpwmBzL85 zw>-l9WZBB{?hj8h<2e7Ta5;aq+rwB@{Vt68;5@U`KQm6sK5(=;WnSyw)bP1n_sJn$ zj$I4*cP(Vz^{@W7c>}kFrpzJPoCeM#pA~#ASQtBff7u{s!6+xt*xFOU;yNMvr33c{ zzO=@J&2kKE5xuf3hgeG*jeVMpO`26|-aeem(J+lc>HR^)35ULNIaP=_2;Xq3vu2)Z z`Xxk{cW;6clYqlo-}!E=9p1%9RA)zK)d)&m^`63X$?d}Fy#njh?Y8@6yZT34vKRei zd*B!t&pab>`;5t!2WI^&?=3hKy6)+%O}gt3R#xlrywCh2;wX5c(d~Vs+!9m8HCyL@ zFJCx)qu!lEhh8`R-uC|R{X+|GH!Wb_rF-R&3d5lfUqelFE&5xSly@}p@h}RPJl?Zs z^9SB@F1ilM=DuCp^O;y2Ht907X9RjbzAg7~V*aJ*eIc2i*G#5PH!9tm z!gl3^x6zrB?w8XR%Whb3+a!kBq=R|)KJgd!9%e;88?W27v4of^PBssP52nQ`WaX{&CS8JtD;Ow&8 zJ6pOQ9qGLsaZYAW#g`XnSBIa!XFFN#^V7@Qi&X7v|0Eo1k#y(vHO+b;qRI5UTimMp zTZ~r73xP$ROFV9_D&-EpyzEMe=VZ32t3plvn}bG6G&()+_S1oUc} znAwFyEIbnK>}Yw+-RB;!z-U$a`kK1Rj13J%--nKSq%XhFwnL9(_Am=2XA4;8dw# zxa6+7_s#B)yB5D$AY$~kFSwP*tC2PEV)XebnL0)McD=7Q^Up0jCwW|^{L#v(_LJVN zWM9e&hm8!UbQBmEO=c`)*EQoi!fo?yM{{U~QCS-|irvoqdG6XR zKHEJJvqIDC*{Oq1FJVL_VAfMxwNobx8uda zmI~z=3lcfhPRuwXWT3H-MYqhruYtqMU?GcY!36~-X%huTesz|>@JSvYj2zlMR20)^ z1*vf!VR4#O*)nCTQsNOIC5_~FfeYK&;=jJrTd?M#_jI@anZAy?{jJ|Grus|&II%F} z-_Mg9&)cm%Uio!S@u6+yQ4Zc+$-p`k7;T=StY&s(i4%jFOpp9c3=Cqb<;%0 zoRkU8*G?puZxD3KUXmPj^}ta%0|BwNoC5`tHx96heK>UVXt!SVP3}0E1y(&zS(zm& z7+R6>-dnJA(1D)zb-WCM>oeHalys+t&O0YQM}a1$N7AhJ68TMWGK*aJc^dHy*>^dUoIVBGFs61VL<|u#t8>j0nU@YikB3h zM|wnZv9;Db8u+PP^G8VjrO+ZYwVIim54re*0hex?t9EHji8hLg+Xm@zgAaGLPh^Eg(qb!4D&W(i* ztP>vJP`IHe(9h7Kz>?Vh=GjBR$^+}Ixuz^v@2R1 z%&m2iPVQxT(sSbUM8#W{@zLCilmoMb+jI-zA^xvStoU;%Nuini5*# z_zW1BrySrvd7_z*&7eW`#z8jb58V@ttQl=gl%+gul#CZ#>`^vzR=PN&O@4vd<)#NB z%b)0WIPtyR!n5Fk&GSJ1-E#yuvnDK-nQ|>(!+zz&pJ&w%%x!bCV2Me{ut~l$pO4+@ z2%|%kXp=xo1B;5njMLE*u4)BNytZXiVsG9HDH)k(2IiMO@+Q`=6`B4{MdXHoTgPKFe=F~v1^trjFwZ}%<;#AxjCavdVxS!iiNXK z<&W;l{~0scuPl_T%m|OzHG^5^MFO|Q>BS06vh!T;m0awew#3Q7_1^b?A-38J^i56$ zX3o8De);i?`1+TZ=P~ljN?rW%#duNU7pD1AVH@VTs9s9p=5+Gbs@=VTefrFYc}o_r zi+O(R`a!;&yb9+VcOGo$5ZsuP^(|CCple&~65ia(k4NLSYad{qCeZ1{afnm!fMbp^ z1FMot!sduq-Kpo4o4Gnnm|AKN2+Musu(e5IPUI5i{Bod`_eGusGGj>4Y=I-PGkW0&+ zE&rBo*);Wa>bI!FsqED^?@N^>me06xbRC;!)qw{QD?aHhn6Th2)3UeiaXF0X8pqx- zF>pCB-+55+L4lo%f$NV$?4mVHd(t;+xiUpCu)cY^MJ|E)*8!1PijqE#uAA1dOk)tL zU|{E3D6;1O%Z!8_o6fY#ISR~U5aL)UpuqSeB%uG3$NN_gCQ2PMI^R2FRJ#P>1JNQ~@ zoX?Frbh#<%0qyQA-vn0KsXc*4r#<0$m%1yhBC zK$;7GibA5>V)qMc6U82|_at!C6&~+c!0V+bHR+**|3gkCMu9g8LM)05oeKoEBrsGl z2-y|>if|Cvby{+n;&ZNM$zQ5MK8^2RD6s!IG*{1=aoz%J`PvHE;psT{jWpsddkB@-c0mcI-8BG#6b~y0OY2babfPYWV zR;B*RA3BmN4r==*UON;dG{=FZtU;i|fo01Afp-e*U!2%ht^S&`P?~%3V=)J|k^@~) z?bgSR*W6klw4mXeQX~7JlYB1O3rJP>}izT)inLgs(*(b<nSZ9D z>2ybr_7cgihaMBZL~dw`mMByyzIrR}LKB0JH(q-ulQ)GfR$+Z15zG1{6 ze#F5le^P6ehdxh$y5~gwDx+l)S#f6GI`a}Z)jn{FC2(F^#QV>QQH&w^Q-j0zL*+j> znbRJyR2|@+(jegXP{2$}WZD6iPbb8f8cjP6@Xv5y>2Tox_F(%LCw?zC0j7rnQYPlV z4z%+m=xHt0H)gPGIv~Q9e73}ats)`z+EUI13`$=n@}%qWr1mK+X3GlWsC(4Sp|sF= z^ZH94vO|v@-0$@E@}h+Yj`W*`2Ag(&@#tJA(bp){_fW#>QuLa>8Ji9scIdgWBTuNc z_w^No@ z`KR-+&@4e9>_bJzdEjv^&Nur5ImD4TT&$eknMCq32Zhp4$!#9JQzW)* z6}+)cQ0C!ZyHAI@G-n*GYbL8|X=X{|ckT5NkC6Ry4k_4e+`e)wi2ZZaJ_sZF(xDWR#j^^z&71hTgtM+SxM?W>4dKaqQvmePYTd zztv7vtZSbl@nU+e>_I`XL_s^Fyp4K??`W-C_9f=?Hm~FFVtOyfZv7Jbwd@U_h2{Q( z&U%h-mwCnquVD~B&QMu2yWvvNDHnZ>5;Hvoem_QC7R8L@^ZZ{J1fuwbb}VRr=E)eih+3FTag{0fYoB?lN-77C;(Gl^-oi)AuJu%FOd zJoj8SgF~4BhoX+ozwbZKA)&0NZ%j-Q`4s+e~Eez#p$Cca>pU{Q-hS>1wZEA%(PbuXI_U(f9|QD%4%TDuJf?wEBha>&HVa8 zb;rN;JuR1fFTMewyE7zTa@WF%n;W}%ZwfpV zUYNr0*Q;>-N-J@7M)My_WE>b27?vG-smJ2LcBdg@p#!(_!YQ8{1cEI1pCyFpPvhQ` z!2iQVKqZkQY=NG~0X7%^GJ^zm4~JwQ26i8Y`xOl=b=k~13s@p9b}o%%Qh3n+??i^$ z(q$dxjDCueY>v7m4}xbWR{lD$p6!9UaS0>8rxABjOj`qMNyEC;;#_~Ycq1L=ee_XQ zG5YglhtkwU$sap6JT~v2sJQXxj{f@!*Hw1j(72c{S+S7ydj4LKMpdiLtlwj0t!}7D zN~+Ko*ZmR; zm^2uIw*C4xk9#${CbP<0iBbjTdr6#MHcPT721uvrzj0#z^=QwLh(d`*fm;efFG}nK z`&!prW9$05a?g4bPu=;iZ){+Xmh3nF2v|&fTsl$Ia-yWv z5z)#;l0VINDqc^Exc6ym`-v@uPF0MHTiDl_^UrhOTyveF%EhDQNWi0~oGc5gx{vew zCB7G6w#!vuZ%SaV3*=~Xkmyw4@Mth{V_?nww@{#JfykUh=|cp~L-ue>a#H%ziKXB0~3 zGu~2QRaRV)rXo3Qf#R9N3aYCj`}H=QFWbO5djt1wkBNzrYI6?h?&)XWwbA&Fs=Cy~ z`=YAFc?)+s${O6hYIys|@3I>XC&bb=O>+OOcJWG+^!s+DZ@O+00&Wp{JPnqUoQvK& zF_|r3`{Gnxs=zy~fpr}RQ{JWeIGC>DN5=kWGTn8s~i;T(c}MofnWb_j=as2r{R)TvXUPEE<4vQXk_I( zaW&@hR+%dcugd=9{VEp6ZxQEf^St<-Q2$G@1B$wa0{(9hdf>rtHAHZ?BlZLUz8!Gw&C$yi*YT z#=!3JKrCrTiCW^dU3YyI8cZLou~=|p$$yS{KbY9JCGU&iSYO&8c&mC&n8LP5uTnmz z*lx>{pb|NDCJ+7sKqmo;lP zy8q?9a$xh(kGg1Gy^cBY+$#dUWt;y%E6f9iGIYYO1x#oIy zffWqeI&DP)N2-`7Eif=^VD69I$hKgKYPs33FI^n0Rz1!Zd+t?eH?X9nvpiYL6xJa6 z;pLmZ4zIpAuw^%j%t&D2aJ+j{R*>P*w9Oo%N)jd!Nk>@OMdch6J{)Ld;S^AFsc87f z*v771l5yfg!oj8ql43DC3>Z^Raq=52D0x|Q=%m?6C&NpOsy7vc)x$TOn2^ZK?RjRA zt#wt%O{1m9dUUh7LM1n@JTXmicbDi*rM2B+M}Pie>yt9Qw`ZyL^S8He7CpW4ud?{| zwl|SmpYGVGTfS}0&D|OMUtaCrwkAHmbmv6v-D@Lf@Jg8FaJ(uy%*xa2e`t@w%|onw z-h8b>EUw2D?4_05Vge2vZ0O*U(8;q zDF;I(;}dsQop7^~%1hqDJ{u0P<{1dJC|Et2ct&%X&9~qdX%C^^z*9DhdxHHvG`z6l>(q#>0}RfQIcr<5n^zPvUym}Fw^c0B@|efszWY0tUQei>deztL z(wl!5B?8nv+YWllu-r(QJR@=2LE#>rM{Xj4J3h9s341gIGD=t)2<2?pTegge$znnS zlTd&HW8Q|9?kp;2A2fCAwti`FWUMwh?4_t$F>$WA{EI|y<5-aZ4!!Lf8xN{nvIuTu z5uEUVRcWh86NiGWL;wTJ%395XEPN?n4%Dv5Y6#3;d-BSC=D(>5!U2*Y8!j>HY$#yl zmVUs(Dq?^0)FCcmJK$m;R)FXm-3k#+%tU_ zm-Dhh-2Bdti0L~kUv6Dzely3oS57tHnnY@_o!FEV^~`Ng*t}gnz82d5Y4$}ACKnOu zS?RMf)0jB^DJVEH3p)e^@HE;y*U+0>(|C|E+Q;ams#1j^r&7$601w@*1&I!)w|)s= zQo3p({EFE|p~_onTaJ|z?b@TjJ}qc{`joii_-M z^pO0}<;csu<$*SLzrkmo&BsJ8>ukyMwodBQs1rRN5xVAt5>MU2)^JPve>cjbYY%t4 z+xbT9S(4S0C5PWlxOe3BS+ko7!W^7Ej?ZQ!hvX$3Wb;pAyPW(p(!uHlbKexfnLNkY zq?w8j9cy6bRk@(%%66B5^JK?_NgN9CQwkXPS{a&L*`s2N+QiuJ1fGy^>Db`@MKwae zO|dN_;2>lCn?P^9f77o7ozk^fq3+0PxTAq--Lko-0wPcpDV!2scDs|u8tYU zj04$m%#5rI3}&1O4sWy!nm%$o7wC9#MdC|B6Z01a!H5G*9905rvs@Rj#~(Pv$+w`z zaE=3um_?(|w}xiso8%rW|p4;oi-s^BCS`MUX%yQ_!YN@oo3)N zSzs?x^EP`zf}5(-%L4*}k0p*bBy4@s<`pWJIMtw{aTg!+4DsFqeFLL~e$E|>4_x_F z&>yYn$3D3?i)XW;%mW$rd9zwpxcxJo$-IHVcw*QDZiY81GItdPM7K2rb0(-rM%+1Nt)2;;MOvMd&UU`&Y!BwRf|-Ze@*(!v_z6y!oisN!Sv}`8=9`LsS0syIpTHq zSP)0n(J65=rCEvW$oVvK45ZR9DR>9wsSN0;}(}0FX|Rc zU){her1K$!PsNdI-UAmki-(;)9pPM_6|2&W8W^YBC@@`WSyp3E)X1o#puzuOI#bwW zt%bY}8Rjz%i*IRQRt$M0b$tTcyn;aPNiRkJZ3w;cy5JQ@#Lh*+f|C+lo3syd1wBxb zelvrGzo4O!>&yZ+BaK3y6jf=ngKBG<*5tUb&Wv>No1)4+At-d>%)2RF*O*@Vp5*d4 z<>|e_WSefOX1JfFW9~z-e1jXg_u@V@Ri-+}e!TQDdQSB8%{uG!tbE_?<`6y|^(nWt zMe>aKvh3cNnDE^}0`JwW6yq$q&4tq@IEkFGUV75y@@lpl3mKQqY*e*T5-O8ulAX7p z*^VcX)Bi*xqt}8~+dGNu`wue8Dll^TZG0rQqoB<=r?G?e`bUAEDy1uh4Y_?Eu9RZ= zS5h{|aHZ{+1`e|ujN&^EG^>j^vL$h;2&)QQJGE=Zgntr@!5hs_2d1bgas?SMOP(;` zFz!&~dU`akuI|l@&7n>wretJJ++3t2d~;i#=$W^Zv%Q+L5|pKH-8h!NT1HvCvGR=U zp9kF~Az~*i*PUscnP;`8u-jT)anFhgn=1}mKli^dbIHX$^wVIQHSDwm!`i<4J zb(2X%Z*S@EX{QpFb#racS@-zfCASPt@q`qW#al`iOJBR-Dz+%0&HVh$vnw20Z;R^P zGcx!yC2qdHRc}QiZ>EQ{_|*e;dTTbWd{XPkX?uXtzFL9ZQlyc?<$+`6p@NPe zi^qcEo@cWU&0;uRwRggF-cA?!Dn_+~j0?VVHan!3EGx@;o;Fi}RjnYdUyyTK0Ou43 z&g%5W-VGcQ5z*UB#j-CnHnB4sFOA;An%Yuc(caE`y@B`khRWBQEG`~3|JQTaJ6WyL zw#a+?7mNBvvG&5We`&l|CX_95uiEv1`-%haz69Q-?Nu`sSf5=Di791d+ZZRq>g(8` zD*Bvlq5#)K0ZxAd=CTio+Z_1LHEA5v5^D$W`3URB_| zejw|*f&PK1`A;V~eZHpqI;m3jOKRFQu`O44pI)e3tWZ_t#(kxM_wEF~mkW5Wd;H^F zQNUI5jMMi=h^nF_yP(1cS0Sqh>`f2YnkTULTwt#h;LK}aE*5CJ{(xtR1J`5)t{)sF zHv0A!P<(Cz z@74nLm;}zc4T3&X>IF6@9WZ13IlcZ!nU>B+2C)a7X$M-Cf9LEenB4qkS_L~v_*x88%y@XCrcW|lW6@ZJ@OdAh;k zms!?_KlRxJ z7&ZSfwq7{GcN4o-tak7j?GPhO*_eM`Heol@>3cRll@NE}hGyjnIYwIM<25BAzlNj}! z7M7e9%~l_`ve-&inP&=@r*lp_z_HwbBkeh7-uKB4lALD~IJ*{1DLTx4L#TU4DNCSa z)7Fe`{RvC~)0oa($h@_@*6DcGwF6UiEkc|>rPfar|M^pV=5(?2wB9ER^zJU0UNe(( z?g#E`8PmHa^Ht28ULBe4Dd1alMd+2OZ+)AjmBN(DumIKtjDiNtNiTRNJIwm+!L?n1 z_i_ODOf#l`Vg><=D`zb>m@wZVi+Lu;5(i`U4eXl6yq5~Nw-#_MX(--WIs456o>l>l zNo6%X9SmEN)m94T91v5FWe;4dC?vL^Vcv^58;^3FaNsO&pL|s_ZiN8rjjf)OKd{#? zm{N0qJz*l-V@ojyhN-Jvx?j#@D(6b=yp$RKl4sY7ynQF82D8N&A4+we6tmHV&-jO4 z=1b)?!D%lWc=s`+&%MA|>^YrxSLLgM={E{`moF$t3y7O(7{%6L|CzB`l2Pb~D#PlH zs~8nb6s_8pD{!?XEGk^leKvr*u0X#*fRp9t1e2`ArcsM$tz=OWU~nIkvA33nALQsgz;XNl zNBE_*l5Zgs3%FZ8us^)HtmXq-z5x5LQ|mJqq$C>#XBh-LFfg-q&(maJ<~lv^rebZ) zt>uxGyw@03+-{Iad1i5Nt6p{Rir;DyS&9qpT&S$R$+=LvbPKCkvKlssKVg1(a$fnadHM-lYaVQv(>AsCR92K`>bql3Vy`!9sWksC zOG~@xHIXY_Z8zV)2mEg~RNgw^*!gPHwH?#LFSOW{35soSwl5JBRG1+VSkE*;il2dj zA%KC2fq}(MR5r&2D>hq){krUI zcMG{4qh>8};LY~tUE)yta|%cFfsUJ}tWq8Y+}l+e#3ik`kwGS*A$J+~-GX(mx_DlD zarP|Wa9_#MlfV`}fpf)$mi7xg6Amo<`D@v~DYJE_a&Ne`X{Q6j;ir`wOhcD1+`WN& z^}NXeoKqicc(-%;2hI5`$EHqtoL9%VTdsOVLfd?sh~2ZM^RfQoR)5X^)?xa!8@xA9 zEa3jd{rIb;jaihgVfU;b+INDu$OOOPgL-q z+%SRJlk0(ad!G_l)am054s{U+SiBOLye`O|t(vgy=Cm6ET-zsbFK*ySFGyDm+`e`q zgRp{7j?q#bHR)vs8B;DyzH7jL^1(XYt?Ss5 z@I+J^J00)d_ag5~&5@8a@jffPpP7~EY|GalwhNqC zp2?wNe6uQM!qS}+c3qhEvVena8F$+Rj-&u7K32oLLoQm=>N74$>274a`m*!&1^zby z2VZybNbD$=KErXcfveW@vcMdkX#zX9&pEAg=Fsjxhuj$s&;89bKl z(rgPD951Fl+`9Xf0Popb?6P-6rRT6WH{3tjTOg*uvFroK_C4$OoH?}5X9J7&a$_Er zmhh%~xo6Vf-K{t1=rah|*_;^j_iFX6|XuCji)R0Y&kYvTQU8j z$I7_|Jo_82A9}$3`oN1r8~7&On*MaE{Hv{TBD3V!mM)56K`e)KKh@Z#s)=?hLURy8~=NVrn)fvxnw72P>( z{dJ)^mRY0+<{iU;f}<{gh@=SobIJ)`ivG2%8z$ftkMf91Lr>P4+Zjsx@Uj~5&_6rWL z3eQY1fBx{d#-nL_Tcb5N+Zj0J>b7nEH-Uw7G26>q+>(!vlybju;M-fr(XfE=px&FU zxo?I%AOEx^q{kQJ$nhiX*XO`dE{$A_G`*+ixe`K0dBqV)D|iM^G|)8E}T?|XVG*XMd}vRUTBB^CcpJy|)~JA8l2FCn!v)7QuA z8+$(7pm?~6QH;lC!h;8njqIM2xhz&KXkcax;N)O<;NW%-%T9tsmruiI;9$gYTWdUFmWVgYMQdK8-A9 zyS20!#Ud2AG|w0`tA?aR1#G=qQ03X>U$N@}(`}}gYx(Y93OMYq+-A+h;ndi`xbw-3 z>UXT-D~hjJxyLU^4)Rjpdt{Smfsp4TU}&}_el4a}FKiZmS)tUsDM@fn?PXkg+KICX|))0S6}tZTF!8qZ1yH2h;^ zSheUHqv%S-GGGB$t;pN8G^}b(`Xq#Ow3fb}%?IvlSfh z$@^TUH$wvh`;3A%nJmAB*+LEz3>-T(SQJ&=w<-y?^ooT}KJ07n~;>e_@jltG$xGAb5gvpZbEU8%N^fsw;u`y&22vnJCusb{5c{y27{U}IzK+f@!{ zlB2zs2()sX;?;1L>}ua(s;leBUBRHiz{GWcfn6Y>AXKwKV1^@KW<_dIiUFgTT)Ljj z)u#eg9?sm?Hn5rg31HRbXp&Yk?AZI|wSJ{RBd6B}7WKP}q*_k4nrsNUB z$1RR?&R@=Yp{m-!~J|EH>N@Z=96jx#RW|2I4ma`nbB z(<5yMdQu)YDC`W?7VmHs-q66R+o8~C<+g@HFCkB6$AvovPZ}6S6lSn#FdUl@bcVeq zpn!qnfKm6&Lx%$oi|8x0N^ElAa_C?bC=qmtWlP*?_9K9I-sk52PYHS|%YyaWRxIPI zIKU`;<#qfOg)_Q(O}!FqP3%k$gd`N*8qQAmSl%zFk#enL&y%H|3k9M&8})u{o3FEg zL2v@Iw$#CPt3Myj@BIE3w~%eAv4q5?^sW?@U&}6QSQadt^uGD<61|(U?_<1zC;Yf# z^o8B@tIuJ{%PVHBd}J(`m!iC4U5=`o&!^s;5NE083MP>`T&)@|rkr_qB1Gx8p2iF$~&A?oZ_o3TP2PJaOllw5Xj;@s=MYOUulY6J=T({OB?642?T;(1nm zuA8Y5t6#$W-`B3&>2E2qzVp-hpW++yd7tJS7n>i_Z?flf@T$nSQ??s%UOUt_vH!<$ zDUHkylNBHDrKmjKyKBPwnw1xMvMiXzw`U)Fdm*?xYRV#!B@8y=t9$il?<$XHkn*fp)5AVisv=g ziY)ziVeWg2L%LHpw5a5Caldo(RpebM)$q^n=T_FL!zz=_JZtPW^?r>yEcNoi7Sml9 zdHs@;_kQd=!K6D^Ya{Q6DdEBkuCDr)5!|;p>uG@d!6QOkxsO(JC#?wAU^C>{dZ3%% z@lT@k4+RcKp3Ioj=eiaB7&(G`nmA=%h->-@TBrYLJQS61NT}mLvn+=rhr7Tb{=N^) z(ia}Ezg~Bczve?;h59Gfr;8o;W?yKE+~#oq_p$H$+E+A*Td1?utYA?+vZljzN@R7T z7GIH!-<|U5{S!<2rp)bhV6jQx!x0lPzofg&Q|Y0|2rx>Fglf<*1zPn zR~S2 z{^6A334ylbPI>yjx>ypfFW4zRYsc#)mrpDBrE@rRJQ9e|Xxw>9$@S2b$H7xIKBy=Z za`=^by`Q_N<(}rn@B7*9KJ+eNdB~d2>E)vIaau=AU(OzI8MAT>1Oqc$c&O;@#XgGq1+| z&Fl(zVz-JX%hKi(Z$N$3^OOHgR%lP+%db$$;WSZT=B+req`pPw&&;WpT#fo7bDd>x zT-cxy#Llr#Ya#oyUEXo~ZnmFkw0%?C;iB^4Qr+d15&{|dO`Hsh|2j5VuvR6p7k9LI zE!Z72fyL9p;gf{}d(D9)ivvjq=6h+damTRn#I&W~aNy%Y)VK|sl(e}fLExBWT zk^&pI%bqjAEGMH`PXBJc_e1x+;JUe;JIuTp&67o}l0{xrG zTV{R|Sw2(tZZQA132gU`m%3-GhE3cO?YQsXi?)0Z_M!`H`cJm#zu_p_F|}w#hwhXu zOOsU>Y99{2$-n+*#}a3&q62I(7ObngnNHkf5*A?0JvH5B1-D1Sbd~_dvp;#w6IwhT zbnkUo6rj+2E{I9EgUNkQYvuyeGKqHPEv>9Q+oCl3Ii@%`bvyj&bl?tg6_#uLI8 z#>1XkaI}EIF>;El@EX_Th=WNBSduK*yd2n?i&-RmcHDi?`_8hjeX@DyYCiW)zE=@b zx`mk(U2JW#*L^$5>(!{~yJ_#AjVo_fwoU#$|AW!e(3OWm5Ax@|=>PeEqep?gOojdT ziCJMU`@eXx=NarPsS*8iW1r5KS#PaXix!xEvS!_Knn~qFqumn*!w4aPU~c;cCZ$Kr zhn%*#s_fovuzQa}vqr>RxeW}$7nYrx#8#}};Jbz;cmhk1zy@X?Hg*$M_Aesra~%F? zx^kPag{?UmzUO3w30ruLYjOo!w9ioy8@BkI7PkTxcM0Ru)y;`>X1?d-dswyovDdnP zFCvyS2s6nEY?oDNQsy|+^O$MsrzX#ddW&B6Eoo$#bxQBf?R6^}*@8aq4U$^_QOT~P z#*!t<`CagWZU=Y$GaM5iu$8`OtGd%M@d3yC+iVkF>?=}WFTK!aT{AIw1M9{PpZys^ zyE|HJUNF0Pm~D^fE(zjJxXc|B%yiL&k=vrNWs6y60DD;jdtn0W)HN-Pvs+kNe5aYP z`g0!TJmY(Jif3{}+bkaTxqCbxt~n`eaxlq&E$N19(gv0|ojr?USXx%wER}J)FC=m? zyYExtG1r-=K7C{?P;Oe&;T*;> zg>BXn&qsGW=lHM*vs{YyIT~xjwy@;fVxD96GtS-nr8`?wL}tk;*TzQa4~?=rf^$4*oE! z<9pvY+1=e&RdW2uoSjuR&gO5VUUAz`oG|r>Woz&PtE{StC4%C6R>*8w&>awQ#OrGJ zQ6bi>4Q<&E*m4fAZTeSWmZQLaMv6U6;Ur7W)tf0N*>gOdl-ZoRJ-v5al(`xx7upt^ z>MHm5->>}ly{R(r~O{`HEB&5bK-__TD_$!uWM*&C$vqH*n=Q+j8O zEpAwRH=K2o+vN905$9yqmfaL`KN&cgOhasR z&m2C|&+2=|cu8B#j@7{&M~-TBZ~D>gsuQ}agIhLeQAws**($@bU9FOuftf$pDiYd@ z8`_v<3>dXpC&}Dojyb@t!WMSJ+J9k-*N%&ccU_;Xah<)!Q}ORf8QE)bIyUYTY+AV8 z>>|R_w)!~*s&p<6s<2)nbE5HI_TC^x4s*pbo0Sgi&=(DUb;!+k5z8lyoo~C^^7pV= zOf|JKwbHYRet!7+bLYcQlTSl>4#TTE-awes<;ow7FTSXccH*j*Ije)j%}nu%KlSlQxy%T};w6r7!$ z>N)kuour1VY$_&xr*?Zji1OC6I8xoRU-*>BvlKzOhspmqu3dT>DH@h2#&^rLqdBCn zdEs2Y`x)09tor6K&X-}kCwZe$az~@=g{RtA!#>zF6*%9^tDU)FjmXcFQ@+h<`LmJr zZZKQRj<`3qTbBt-M`kXqblulevGeDHN7ZN2dOa3=`6{(!@#P?+sT~SD`+e9;JM0ej z>{}Zs+8Hp_NBU~uWfSvjCe>kD?+>wNUSO?TH>)6lHABN8&Enh^owk%63Bd-DT>BC} z%}DWnpyIKi#eKySFSE=0l3S7!SW;eO^!p_Wiak`)y(^N}5>$6rEG*TogE_S|@%-mK zZOVOpvl?YCBubxn82gSft}beY>%FySc1-08Ulf_7woUVHaJZ(}?SJKW@3CDv-s9mM z-kD`!wmzq9sqwY*tfJD(XCK=4<+Smawosz}bHi7WqRRx0G!3-OsPP9fX zkbV6uz3!Xd+ptyIhuU&Jy!$a>OV^C>10m^|68zjcFH$txLf5kFyZ3ru8tXI@gGe@+ zobG+|kA&qMbz|!bdwAXS+VQeC^WM6;e#l?6BldG73Sp;=jGrN<`ov^lNRQckYwlM6XNHU5aJV(c*4(NtAY*3nYa(bd*g(@-S& z3pLLOwJM9U$n(=FOEXExk+HCJ(=+vul5=sfb@s4z^R@G|b_ug}^tJU)bai&}c5(9Y za`AHY4svx4_HhdF@rv~Jj1KmR2=Var2o3cOjSdU(4Gc+(@JxvgNso#s2=mEKj>$|3 zugQ)sOAoIojxNaX3Xe$h3CqriO^HgWj7rSRPE9SzPASh!uPM%{DoIbzEUzvtE6yv6 zO$=&JlkAMt>&UQ}US>HVTW@KlS!;+)N0jIE3ip=m$kvjWo_w#?@{E@1lJ?5--m<*K zRbKiN7!wu<=3f%6>IgDB#<-!^e8xhJ=?|2@|1@~?Om#wc$cBEu&X(-%mddFem9wT) zP43HFJ+*N2>cT~9!q2P?W%;NA3HzU7%_P>26?a(5ttdh>M%8s(yN$L4*F`3Qn zwN2H{Q!AVLx?5VOb+yf&($U^Iedg2|Q~IVZo!Pm3*0hyV`d2UR-m7w(~yB{`UIw-@pIL&& zmMuEkEq<`&r4SpBgnsOSmN!P9lszZJ1}suJxoPQXz37BPcbqn>omDs6{$R<<*0jmG z_SJcUw?04jU*OTKbk=inr>~mJRu5Az(G?-fJ?2_1y|v`En7Z@Qx>&6ejT`D2d*!xD zhOWA?F=p41xtCM>?(Qzyb>ik}S3f~LS0-^qrz0H`4%_G4=y=G)+QlWJ;!$Q$@Zg|r zqCs|Ag?gfdl>on4ONSwkYmb!nEzh0579X1^9bEQP={9%nr0w~uYb1@;#OD^3UaJw3 z(Y&!Cul`QOYtix+AifA`O_`Xg(*Z~+5bM1q?@;SE8jKLyOZJ+^I!QuqWyW;)Dl zHFQgtk`s|iUu^p1s08=aw2PVRazg?=6jq){%H6(n%587$k8VPKu_lGezLt+#CI=ip zy7g$j{|t5cfS0{jL#r$dKgN_sicS>`amip04S%$kRW;<&T=l8KF%DBz%7dC%r%u1M z@a45i9aZtLiG>GUuE+j6dvUr<%#)W#;v3&)9+~eXvoM0!@W_K3EfYN_ILJ-ST^8vm zUG*X|Ln)~0R_C&r&srto_B?y!Hn*=tC~wo*Ye7BB_8kezH~U!;EdI~>?-t?y{dbqd z1Y|Kz%?|Z*48L0(5V}0J%sF-Dlnr+^ze-2=Hl#kAc8ITf-t@|_*~`UZ6SSwtT>bSd zdS2PZgRHV;oF`aiYCc8INbI=zA)!UoM`I?-)YNSc(?kn7UhbPRiA}ImwAb+Bfut5i zHuw0BC5sNJtG+St-Zs%n+H1w4ThCTmGqJudIP~zz5kKkbJ1sTmmtLP*a@BCRXvB-! zolB!5mT}&zIFKcNZR=&T>!HGadF6AXBUpM{qeHJPe_jP+DCfy-ZtXXjvn!&!gg0sk9ZpbrBlItK zm*<{UM|BoT39m7h<(%W+aaue($X<2jyp#Js-PsZ1>U8s7wQm0L)QTj_%qyX{ThFm< zeP%f`WZEod-K!UziyOaGnohkMJkS4++b;13@5D{#v3`mx6|CD--Ljy?Ol&<%XV{As zj*?{>QZ9=tBQ6|R9_aFS8E5{Kf+JfxOKu-rQfFB4Lt?(wz8^*>kA(Q88ZN!(fAG+L zk$~`DSKb7?uMOQ4{?haK(`PB4~#qS=BkZT*G%{5njge0C8i518C6 z&cfNw@K!apnbYT*pM)Z3s?g=>TAhjB zHJ;8-e}tJ@Y>sPj_xcgEL=TeKO86M9bGAaCZX5aVZq<$mga}IG68)30ihM$8(xMim*K0A3q z@$&2IJnBs5n?sjW&H49giu=1|+go@I)%<4&{tOBgUb-?X`qaYC&239m{-$Q9%U?cu zLLkCt{+mxTj`{dqZhNY-^AC%!$6>L_M{cm_s0b|H|0{Ijrk5-|ZRvfhrKK)4ExVYb znyc*f(v#)wkzi}yO|u!bnk-nATa;wd*gG8)xt^*@a;J98W%%3^Vl6DSDo#K6y7i53 zW~Q@eb@+E03Le(l-(dG8k-KSi&dJEBZj)`+p0E949P6~%$$n8|Ug>|~xp4=JMdukd zT3oy|EqI;gly@_xa7J(}em3Jl%Q3TW`cWLq#M;CzFZk+^;j&DtjbtLMzTa;mIgA%EbMWVOB1CN|9G&6n&v5Wc)dM{oaL zHM5-oOD!jfFTSJ5lPAeGIwZ>81V*!)< zi(cCo-N#WqqRq1Qg4PN^$NvQGG zyT480npE}FDL1&hx20^lCc8al-COC?g;_;1(-x*jG5Q2gohB1^D4;XrQz}#CQlMIXy@}4XcAr)lPVW7lf(K!N!zaal}b)uax`=S|F&%gE(sGB+_`$9 z#42I#%*+t$vu#PMX6?#VpLN3av6xVMrOQr(pIW{%PAdm{mLI>xbf>R!kGcEM@*ogDg2F!3qNwHI_pnmln8f%U#HB{hr}m+djrQ_8%HK z8WfmBe=x9WF*KHhGZ+ajU|?}Dc+WiNH-plH1Dq2kv>MKM$dkrVA-3iMv+$G$OfT9P zSojJUbZ$J7Zsm{)_+xBpRU5z;uW{kMkf+4qoF6Uv2|j!NX$Z_@2{^X0T7Y?p&f`7z zS4AAXBAjQJt9+#Yhq0T<#I3&jX6Y{ZS+n?X)uMRYa&JxTg*QCnH%;lbwbIm|Yhdf} znXhU>v-M*IcHqS zu@Ct)8t!pFU|`>}n@Mm21B)8Nk4?@B@A)ho9(gw~tHvDUnRVlcPK+aWxKEu(+6E>e zi-0Dj#cuvUD)_nTBOzn` z_;)2gLtf6z+N348^Ej`J*tQLSw%cva)Z^tBUs>`c-F`NEuzbOl#sBnJ(XGYV`y6_!pFaoB#K3?Ro3_{PjQfx;L;?Y+$wAwvab(N38U=3#@aOSA2P1 z&Ub;~%l16I4SDaw7=dWGud5D6wy>n8rDd|^W^7B#Yh$*aQ2MSY<7pW~()NP1?d7Zw>Ruja zsA{ix)m~Baz3tQUwyYcF1t-csir2FQFfs(R@i#Eq1aSSiUY7s8>`4N5xdYQbK?X*i zh6dh(_PXax<_;OWKTKIJbg&pSN;j~iF>uX#&^Y4&=j;!i^Cg-TS8zraFsT|aN;NP> z1#mx@z~RuCy0uyN-$wauKYZCPMKvq3xNXQZbzs)pz|=1&Sv#F!L7Ae&A;lSjH8M;w zdS+4%f+k)KtVs@GNe;b%3T%-Hk?IRdRYBL(GpiW%C4MjMv}mhJFXMAyNMmPUVPIfo zVCdXl-uaw?=RtY8dwW`W#rG9;JPr&z1^rBw{oh`6u+1ePdRs$0W;qRMt+U_+UX7Q511wdG72l?Ppya+j$jHbU`xO7uQPoE z=d1;sGYZ&deCP~k;9B>9>#zg&W(PK<1&opbOtuEx*CucT8W?Tv=oTq2WLnhizCfYr z6Z8HR)i1Av%syxnE$O^;SrX&otgFEV<_QJ14J;X%tYHPLkp=~x1x!i`Og!o3d>AYx>+d*pwA|onT)@b6oI$Cx{Oz{9SI-*S-5F#LOcHAV-6C&lIeD=Gw?QRa z_=4Fp4s_1>z@=!x9kzi{B4Ns+0A8IFQ=Ek|ZLT_2GO7Z@<0_~SHjoq$sCe@}-{_DRvLHJgA@5yq$Tjkwcja-s5CQ7dCtz0<&7z1C`eB}r1=RVXc&Ek`|IWzw{ zPenuC$LUiA6e{>Q7Rwqe<_};J-@$2N$+dU^_sa`>+a1_aKCmWFU`=yiGn+Zt^%i%= z0Y;exEL#(JI~;sou_{!3Tq1s5e0ms5xRi$4m+oWAjJwqt{h!YDPZgLbWSiF3xn9rI>6bRE9HJ;j{o-7BvB0bOAUjBF1YSQ(h!Z|_gP-uCf&2XDd5I?DwW?i>=SoJG1TKl08>Mz{oO_eUIDvtG0)y~}OzoEW zCyevCzB7m?Fd0_L_=Ys+lQ#9DSquu8ZW;rz5T2Cy^7MxWm!Ci>~(Ie$p)2&A56}V+LkZX zG&_OSeO39Zo>joWnH*B0ddt+`oJ5vB7--bmuv=})$%A3+FYG<%65a?R4gZs?^ zzUl_nI0p$kb~djMldQWr?J8N71Xzz-a(fv#ns1b^x;Q6DIYh)P#IM_Q`*Bmg6;7WY z&Fz0Q_tNC8G1f}6MA(WSuulJ%$~Nc20p?`_J1;+2 zc`INi-<<=J*1VHGFmNx}_o-;+$LanCLzw90VDrF|ryk;;C;C#GcBkvvFyA8ZI0(QPM z;JszQ+i1Wbb>Y~ggA5uC3==2za~*GBn{a6EkIog=T%si=9ipsh2He_rIH#zydMa=| z2;knmyUHq1-M>?PllokVGKnj{6igmSoLoJZZ^PPOO=dGfoaYMeW!~(}yOBNj)#P`- zCeJnCo@2nhz&LVMn3)!INg1$M z7c64soYAg1lj%))TXbi~ErxoZ2_H@zZ{(c7Alb+&u)SUTL{>$epac8l2Rv6Ac%`&2 zcAYtK>%fW244jXDFmzAU5OG-0rp~DMfk`xhG51zu`h$O5`xbDgIQVEEWsR5+8KlS- zD8Rbdfm_z2a+W&RZh^zAmK$DeI&85giuDt#$TB?+B^$LV)rUM7Z$>HF&JdV5<;VvX zTb_l^EY4xc3>>pPj@o;6ZEl#Ia$#D;g=Gr@juvI@7MswY?moRFqunrp+0uZiCwjWJ z0E=?~GrJ8-odW0LtQ8HK{iPfym_3iZzA;N^!OmW*6BRo;7+#)d%q$ZTVB|}v<9A?+ zV&FV0aPi8810r)bUNPWpZ(!zEV0K=>UYDR@mT=B|Lg|ki40;Xc#0=QvPH=A7z`giD zdf*pj$Iq;Wx=}0MaIcua=^C|sgU)H;7U7lY-Eu95?{6vGGF{>7zn|UveWJBnqR-F1 zJOAU_nTwp~ELxq!z|p+m&>JhB{R_DFJ>cHmaCr@&nLfurAmS^NOoG6UYT3hWGA425f^_eeID{psgwXQ*|5 zz%5-bWp});W(Qwog{Z=V#B>IYI!0Roj_nt23h7=Hzsq~YfK#P_(P#t9#0xxUFRGBN~%jsAuQDzNU`aA=$E{KW~phId$> zr&?DYykpd%ccfck&!5BZoS0*7?VOZYRIZSeDExe8NT{li|JB8sQNC;o8@T?3`W)Ri zf$z-*zFiG$fd*bTH?c+ru%cBfvthjDuI2;g`0vlyptQ4#Wyf|GjPtf;yWwAd8UBfc)`sJ4eSk_dGZUM%ss)G zt~=-U4OdmuNJH7n{t9ei8@RVT;MA>Q^K9UV@$*qWT;&W!bcUEJhQUeKs%}ePC%Z;CsTrDaEi!`T_f61Ky$E?}(F;JB>7m)p)eSN2tiH_48jMt2`Df{I!vBnz*VqTSi=jRF3w(xo(%T=4(mz)vEldNwdliDh7Wz#y@KdD#T^ zVgb&|1 zSj}?dlP8~4x+LS*85Ax`eb%CpwIfgKJ>6WoiBAzO#Yfo1y>Z)P&exfa??9}&W zL*QdJUJ(tg1rrnwIx_7nfA{2O^Y?S}49X7OV68dcop3O5kA&HqryCyWG&o(9GCMO> zJL>JVt*57|cF)j_-Bt7NN0plFyu+jf>Oy_|;;+uzZ^Ae3+S)--PF<@`))cJtoOb4EUfj zbEWFN=To}OjvxH)<3S^jl$U{$?5~V8nOv(& zY|;}}SRUn_>~qqgX=;=eL$|TX!xZ7Dw2e&aYCBG7E>|kLmBg(YqscYdH|^vj7Uv~} zYTT+xPnuRKw9ZLhy{79ymd?f_THYL+>mszjz3_cNfWu^wtj)Z1wyCi>hYk@8ec3Zzrl}XWaCO#8!^e9Z5l|3s{-%;+DLFVzj?6rA z6%5RBZx*CEaqZs7Ce3YlMp-f|D#C$bTA0(L9^;%PPp4}K9XZq~zEvSfOKj?iLtIfU z50?msr&&I1abB`etxKX>s;F$KPlKNh{PJW#Nh)8@QP@|3u1cb@QY zD@lYzN-i*Pm3At*&@6TL#6mU!*$N4z{{C5-egawRR+e%ES#e!=%H319ZKfjI4TZbe z%N8DUy31ndzNd$J%w&^xoj>)qa)y`Ukv>JU1C6ch zzCZadah=^7AnWRHsX9|kt%9NH8lT2PE?{^62ES8bY4geOgmIfmzx{i&bHs!NfI;9Zm|C z*V$K2!R=`_j%+)3x^Y$}be(opiWQ!9O!-X!myhIek;gOIRD2dN2l=dM;fJFi+L6B1?&=7 z(wV5L9d+@>F|%W~>T_AWl(M@|YA7@=UH2w+;S=uoH%7~>8xp+}X3b386|iiI)-moR zh9f!(htki}kSA&=6XJ*jVBjHcRR$XxlLgejR6knvusfX`%2PMH@GYEKK(M zb@2N&qorIP3asAfZ5%i%ZaDBQn9#_rqbRvNo6Tg7U@t#Y6Kf-fxT(wbh<{vp3i+}V zkMQU{VDm6-X7}s3VB%B2C{!{{W?2Qp(vk^V`{yJv@NYc-Sk8UES-6bqZ%G*EEagi$k)K=aOaU&QijunWuIDx>Q3Z94?v3V}E;@mf5$-yaJ*BX8s znd&7@$O>T=R@*<}q<)-lPdN9T)`JrgMUqulcb*HltFvSa_aaGWpDSCI&EUIidUOsy zi<&|MgO3BFWeSHS zD=r%U_H7gMym&_Vh{89+35ve{v(HHSDKs%jFfg55tl)yJ$}C2AQg2kq9~wN|LvC}8rs**%FbSmQ6S z`YJSApJX=rp&=~T7C%9I>4paO4Bl&>4(wRGTQs4CtFwjMpuyRYGfSa`Cqb;gK*y1R zGx!DrbAnjW2G+96jS}5W+!vZR2QuAhP|VU`t35_u$J9{;Zk-(@QL zSaP~`&uIR$;JÎjG=pIL8oY24;!3;nQ)dCX*8_h7L_Eg`FHf7;Ja6MlG-u2F|;VrE{sxm)ivEfl(70bGzl3nT$|DF*R`8n!%d{YG0TEQcn`yFW=^La4Q3x~vI3ao zc%}z;aA-6vOgquYJ)=>AxtTG5DMy*z*@7iHLoEG3gV2rUtqa>)Ul~doh%qGSa%L?0 z`CuA*h0Vu+g%kcY>^(nQAom4ZmPdOW!{N9D#jJ{D@d9lT9j%E1Q`jfWv0K2l!LcPV zXbZDM1IK{|l@m;6n~dKao;&FXi`5n;;R6j^9kLt~7(Ntlk#}_Yv}lX#E;+q|CEp%z z*>s~(dB+kTFRkC&yFPI))rp+H?6-axQ}c@3TIUl^t(WR`c_10Hd43#+ar}bTK#iS= z(ruAJTs##WNr%*rFXT_@>E=4n+`mF0`Jv;4juxIBX8jwOc}_6rI7@S%IGCBxCc9ut z^n=-jdl)1>&$_XTE$c)hyFyD*gotcK(;CTk%L~@b0u8k}i)!bbu{_OazoJ*6NTX%- z+Sm_!Z=bf^;Mtz|!&W=M{ z%MAV$Z8y#d&1sD+yc!K$1q>$^vw0*m3O-O1Flxw_W)D_qWKw8clh`Jcq2Q>{p4M=N zQGwCv0Ewp9!tEcklfjxxlfI zaha7Qheo4;hu^H(T`wwK=1B?7y)xsi<|=QYnE~0|A_1$GHZ|L}ZHNC8n{|Xd(tb+sgG6w~mQ@A-@t>B2F;gaUJ zT=TkB=1}&R{U?)F*}vWY|Fcl&9j*Ng%vL8(_$E}-I0#&Xt9=H zdiy|(eMS^tjgeRblbZo6Pt%kVhqHLx3(L(}CkZ4H|eIj<2@_?fySVK|2&f2bf%Z^xZ>Z7tFe>oFSyRW67J8MiB)`t`q-)C+}nqV&XB}pmp!8 zs`JsL2XAlaEoioUkz{qEQBs4EZv%tih9;AVOT#r*PdF{ebA#dLLxWke7t~hy{9Ua9Z`$;o1iEIiU=!32T`Wbk&p?mT}i@>Z|~NHFl#wf6cXYM;1t^G;CL?uU!GvmFCd#vIlOFIX>J+1|Z-O5j>QRkw{S z2feOE?VQ_aBe6D8@{)G{+B+fsmfrV@q#mgA!Ul}sd?Q0 zb{5F&&{j(cUfUOz!uCLnr=clW;zVQ0!or!h=?yKm5=_!78hlqZoXhQWV`Eg(Xo=Eb zF^Rb2VA&wuU|94ZI{U}%xCgBdR&{O_ZseWT%KV~%p(Cbt@B9bK4SXN&O`q1lv7iwZAWvMbc&%HD51 zVM|PHG+6XbFv=(}3TZGNy654z^6age-rjkUjeWa0U&Nf=mhmUz#g57G$4{=%n#r(h zL5Af#e#W(tDaLObmqogkJ&|l^>1LTwvOqwLt25W(L1qD4u*G3njqST%-Q&`jGOd9@ zz@mY-?;k6lK%;cUqR5qNk6gX3Y5%1A^t;-rcOn{ECF~lev4qy_Y4+TaRd=#e^rN_5 z0<&8Qlf?z`dDcxHvs8YvNy_hNl&E0r49#As{9M=2|KB_A)t|O9UVg;n;~)Q-$wq-i z??$8S3`WB`%fn`iIsF(IZ9mS|WpFw6A*tjf>xTm^Wgof4bN_fWT=o;=7tat~AL*j^ zwt7=!Q2TE7h9=?SWfKC}+aIxUHHvr$3V3p`sLx>Jy`aG^A#!hCDBrdQ;UCRu7e4M- z$fh~}Q$_5}Mzew5J|TyqX)78|tP@%zn(M88!RPyz z9m#9ebe5Gr6ysdb{eS9x=Vkq@C!#L;Nv@4u)El;_C2Y~gE4SDSn52%+negED&09wM z65Gz5Y?SC|xYIY6aT^1lUz1@1Tl`Co(vG5&f2|yXwp7anJFyAAUdD7P_D(_tt6&Gi z`=`PB4lOZ1nx1EzGxpC{_h8n$QT4s;y(iP}%|GUOe`?&4#%*(b$pHiZ)kgkdueGZ4 zPchy39)(u)V2Ndq%Q$!h^y%^``jy);BmA__j4g zNwjBQC^V3|9{a3SXGbE_gA+P;Pb}Z|u6K&B$&Kr;;?_=`$xt)pm*;_%&tfLJUFj@T z|GuEAxbLIH`;Cl0qq@Z`+{k`oaNA2#Sj9*lddwEeBr&N(;M{N?&5rY*LtP1Je;rx?!+CzC@>2N&IFJis6r zw(pcu0RvMUr?6awW6O$mF*TnpIfaLgc0N4fcd0=0(T_qwC8sVK!A;4>CrX>u+*$BY z=@_@ThRg014UhaJtXCdhz~{{FBd(ut=!hRLo7p?P4ZC05+m!lU)^?)TjSYp(N7^|A zxc1amHyrphQG?x1;l+Vn2j1M`mh<6!$Z%L#eUgWv^3$%VDxNMOHVqdW9XZ)H8!Ipb z?K<#Mp5N$&ocghDB<10R!h5Z~7C-?br2MLEX+y86{QE<*lXw=!rVths~obf8J z>F$&tR}C^s9Ik1{A6<0HDekDz8Ey8GgoE13$8Iph{S$b>$|zo!xX?8ry5JzIaC_mw zHnARAHue-(A2*gMPBI%0^P63{5Xm!d=YvCB8Wk^>%9bxEYZs3z5o{A**O;i5yXWP% zXL)-|KSm4ZzY09bV|m8FnMX84;b0M~fWZsbzp_i_yM0bwexU3_;`I59e2Rf_lCv6^ zxU?4JEIKLqI?#l{OL#%U0WR$&EfYL?C$+3_(9H~QVC2_HI3UzwV%+O%{c_6X=3Qnt zeEqI_%m_A~7%?%?U~fY)Z=hY|V&3pWD=cz#B>9{tI+c8x`1#a$HaW@u5&*Xm0 z7Cn2*(Va)6jp2dyCQGlGpF0Q!X)Y6ci0&(M7-Um6{qOLZV@EF$SUk|;qlI< zwZGRTdh*>=N$rGl_kX5clNdH7sT`JRnsTXU18AFJ2x;f(_I6DmAY^<+0LY&chz=IDuw;P9D~>fZi;TvSqC=KbYlDnlZbIneEEo zG{1uvnfWcunr^s0U=)rxz%ox`HEYh2K$epy%Ic=eHVGVI5ZrKpb5lZFV1`!ZybmsF zXAbOnR_G+W_{1SLHU(ynD@mMZGmZ%eeASq;V2OR(t+{1Jo5a}-RxxZ)YLxXT?2cpn z$1Q#4msIl;^Zg4WlEwFaII7n+iSH+Gs??GL2g^m2nE25(A zZNjW9k!@x^Td#yKX%FL~*=l9Gx&znv)lF~__Hbwt%urxDRmouUc&4CEU5mm;Psvjy zQ-s=74pz*5(6Y>^uuYq13s0Iuvbx#RV>%HH%q<&O`4ukqm~kjDF(^dVMJ~*PuyNBxxdhRZ76RiKYwB=*#ZDarEDx?V{Nc$x@RgoZc!FG&0T!a^vQ_sU^+8 zw6N^KzbQ%(T8)v*yxGv91rk7jXVQ_*O&zWcHv z;;5D~n}Le_=41iAZ|4H98#b^`N?`L{uqe%f`KZRJ39Kp+54loruxL$M-s$s!N8s!Y z2f0~^?Vn;xtThxnYTqmtTAtgk@b}?e2PFY*nLCL>=O!F8y;VMW!`l=MyM=C6yAp+q zO3q4MUBN0Yqp*8fS4bn5k0)b@=9SP#pR80T_^|K^Fz`AUvYAb2C}Nm;FiWI~qsfp* zyu;He+3aN8KPQ7G!8s`{r$Sz?G5_E=EzLvfVnVUm-5B3Ik8mr4DaCU{7H<&WabUL7 zi-jywI~26_YdR?=fiSy&S+58TSGNuPud4DiCIs`E3oN<`4v&VrkhoNcz+esW9E7+7Q zUaTt@ERJ(cIFmdjbG@<$TT$?*#;F#I!0FY%D5zt&QTpyV7B9CG_XJg#Bn%oQZMwVSe`5m~ag zIZOND@x=>TwG9jdgl!U;B|HALD9AN5vhTROQt(Fuo5BQ{@?(C?Z&VKQm|QSCc>DpI zvBp83s)7=wIosKdA2145ZD5r%QRMQ@Xp+d;Iyp{th1#nHY$Y25dFMMM^PgvQWYiD{ zm92Gu`0CunSMEMctMA>K%4fmAS;k-yQ*`HXhlluXn;&}a)>a34g_#6d{lb2RypLUckUTr=V3khe>x$v;CH4(+%^5bQlE_9QidIMG_pvzA)(~91?xw zBQm3b@5aGxyaxq7Gzjw?%e!1VZj4djt>oNjG!I+yDb* zPLFKG!R#{!W6v_KVqG-*&D1rejjNwd7qekByvo^nv2oo@2C)eTHF}!VCOGoXXyCkY zfa5^;Nezb82_pHNN0@UQL~@uIb}{Kb;J2RRbVJi|mj+{kL1X_}C;dH5j+O~upB&WW za8hw$Qgm_BbU38xaY%E9li@8VO`e6iJP&m(oc|eHG#lG3Hh#cm8ur*k!oyUG!z}J` z^N)Td-|1!w;^rnErf&|iGce3^JRZA8Zmw>70O!FXC9lb(Q|0ZEV&|K_eXPH zX3Ur%a!gD`>3rZBS(YX?3l_;UOy2t@vObu_t8l#LK=3J>oB0cGT(&sCQV=P|u|RtP zlh&H%%~KnBUFf&I$?uUAaEXPQLk2(7cd+9f%O`N;H|JNJ|4`P6$HIG@uX(s0Osk<#X~PGSa* zL3)fj-x$Q+G%#DG-imyn@I^_Vt4YD3Nshxw;YOp}6Gz!62W4^&$_hA&XE5@YFbcjo z@Z>9l@D~O?9!H)x44-Qq9=aajh`GSARFE?5k<8J8`R7@MXOuN21oVjAaa5dfAaJgu z-UlZJB}f0x1H2DrIxcYOT*y*x7U~${P^Q9AZgYS&!9gUUOS^k^YjiX*d-V_?w1 z8=kY(JQOCXGP36!U_0T!mchW5bCY>Xu*XG}3du`XI2lqH4_t|D*){RDt4TxkgQH!C zxOOtmvMxMm>cia8)KE0Vf$zdWl^agSCd@dbc_BF}fNABu^?X+s1Rj`E>+p}+K`m}c zILi&Koi=KloE%srW|=57F+B^<+?ALlz&P3FSc<{Q)Gf~%PYN8U>C9ZB&85I2KZ{fL z2%Ewz&go2Z6<6KL*IKlS^O#V{LGcR*RlhLk^d8hPSgsbqxGlk=?yXL^+m?I1k3_>%M7M2|{*l|cF#*s-&tN-f(rf&}ZUJ9&II{uxj930wD%ki$7 zCcTT7r6htq@1Txnh5zYi#~EYV6?Km@emG$2;LP8{z_vp;wM(RE#q9e$q1OBEy$W0w zuu^X0qsU`-=N8pC?9R>I^l7o;gF`Ailp;1Rl$LrTeJF8>?=eB1h(ZUI443;kK@2lm z{xRmAYn<88c(*9?o8+SNwvFEfC*QYY6iK+6aG_CNq)9BmMXceFj!C*?2ZL+Cirtyh zAFO;Rkioz$z$o{GQTT&{fI*{tK_d^3^mm;joCRXNg>U}%h6NTWiU}|Z&0*C2amXM* zK`4S@=T&zm!3K{HdhfehJPa6EG#KJI^ey{N^$Rqsx~!57KzWLeCW*3!GvsIFMP&q*kD_u*^$akMU$y#g+Am zAH`ag{HoaBYsB!Wh>J_;CTJe-+9x;9@PJhCYmO%kya~_vGLG>{wBHL%JNB@H`wfG* zgC_@Xr;JS_uffzeE}?%P9a7wpBq-rFQS7Bu)znE%*^dKmO_5;dagFElaNtTYQrADT zd!P2bcgtpfxU*{ItW~VR6U`ijLpBJXIVh{aq%FYoM9x7-rAed5Nn-+|*o*@g(^}$A zFi3keY2M({{L=W(c!rZoPn5C8Ar%K_If+TGnn#!;9Q-fc5LkRyErIE^Q3Ah?V^3+5 z{+~miQXDyVuy=<}JLl(~vUc;kEAHiA+`T#0f0!iQ$8&^(M^`w4fzRQPNr7W&L7-M4 zf2x@_$DIRmF;0pChg2UVD1LdM+H-)>=IZ8E^8|dDj4harJD$3_GaK6+Qq$62@I_ha z_H=gHHf4VKTV{R{k+VMgPJUX!$y!jvoUw`btD-y8Bbf&!;yxc(XKzws%1bn96kf4u zmkpydhm&E1(?!>QA(O06z3ooJAqa!C0=nB0wvCNmBx?qD=^IV3C) zCd|sPK=6RZw_vkmCbc>Llyq~NBqJF6T^l7*9ItUUUyEh@V$;s7U$anTb$G2ZA@ zMtyC@_3xFwo!@tg!{M;$i-QaXjm8rWt8B^AS}Pe>!@zb!tFyMz*n!!^LD{6w%p@m( zeZqmm6K`<5Ihe7T$#{cPWLT4?1GDj&hBq_v`DG4yM2p^3JH~PL^vT-Slh&<`+^gn# z$b@Ukd$soNX=ROm%t|Hx-&cKIyQDKFj{O>|F7lf}kD!3ftSaF%5z(KgBNq5d6 zl?w+s3_|r%8Q69-YH}PB%3;{f>125Ckb+OMV#&c8<7-7W4$*}Samlx%`I*}<9Fkly zUn2Fc{vRgwibEMYk9*uX_tt`8{y#o5_YayLB^Bib7Tz)~<%-|lPpeqP)AA*WVI2b# z8v|33rLz?O>(qVcZm7#lQ_!?DX}PvX27XIy`=f`;mXDGn+Q zKN2dKLpH88cVMarU-x{Mo1hDG%k{HN)Bk-gau>IBHf3nybU9=qu#EGF#d`rp?JG=5 zUmmDjX)?a!BtGu|&zuAE9Fr9loOCxd>E3YCz0y?Zdr0<6Lsb$ZUyh@(L-RY&2JQz( z73G~0)-06X#GSQjixF=Gw`4ny>UO5$BBmt{KVMCd^D3AA!1g0>L;jyMCB9NE`^E$T zM(IbEj=vZ=f6TQ$dW!W#1KSLrlL;++E{8OCFxLG(s438NRijbP!&zZUll+e0p9Rs} zH_SgyIVfq+zhbwOzDI)oi$nSgWLe`J#T+(TZuV*qmbCe7wLZ9TZGXku*%B4K=fxg! zEU-DixZ(WFe@}HQn&p-lZmx4Q_GsStylJB~vtGi#gQ`7^eaVg-518~soSYvYGQHxY zxWe{E)9)?Uo8)pDBR(DY@#tvYeI}(HN@bIEcbm8sEZ_3jp-*6rXw%*;GuH=k&8cK! zPMU4cDDlVSdG)`&W<{*QHo_C?URpOUUTq|P=781C;NYK`!FxsL9X-$$cR)bmagBbX z${j(?oCB;bO`21f_%#~lg&h#GX_QuJl;&X);&EC!r!jiBsrsI#PYe(2d8dfpS;y#c zo@s`J6A#zrJ7Myr7R&`Z+JCp#imdngkth1kYM11ZQz8t%Jxuiyn2R1dS+2g#SaU!; zrs+;RlY`Hsw$s;CV=k=<)pj{LbF*PQ*TYrQGJMv0o)dWZE9{@{?QKpzUY629jZNKb z0tGHDOIkaosr${bsXTO)(^tC7Cxu(vidm4$u8XBy2m`S@!I=ZzUYDMpQ-T22UU&7Q7v(9!8g2ZyLuMuh=mYYUs6 zf<=eG!$YiMwF}lnFt|>Tm1N_IIq|V!QxhlmG!CwTt5dWH2J6wE)}7vT89#wX)c`hZWt zBKJYVZ${7Se2kOSbf0XKWmhm^V0_rn;l2I9L$46;q_u`Zs~f#Tn8g3B^PXO`a@qcv zf|D-pS|MMzqmfN2 zDZoo}<)T@u7)3Vh>gv{8_o=IU=DZt^RFr%dS!M>d(X8IEky zc{2{SX$1Nhw)1jHHyH60eKH;t@5Ck{{1xMT6O({YWY3iqW`2UN}* z+w6SQV{~GJQ_D?01wLzbi4}YdF9dAA?7A;HByeixQ^gf}Vd~sw*|!I@l%^;~2eR^@GSHn-RwOQqihVt`^XHF<>6V{or zmLalTjgN_;A=!cH-3&LA!230uMP7O9x&{Zo@asx!Q_L|~z$94|q1eQrFkwN?L9dx? zf`)1rA2G6S1I*2T9Ha*@rqgEu38D#+3N)IoK{59g?-GQuu>5>*}H{ zAGPGR@0h5g-Xs%rat22kQ^gLxZ+wgdb%_5I90bxJOqb;*fYdCWm3njLxFz9A5cQi1X zENI-_^%)C?4-OPwiYDOuU=<=)x2JAgs7(sXHD z@qyTVA8xeDGQL+$WxOFj+1;Rt_aLXY!Xd@&2JKEB%~FpK9yQ@};>}8Mo^&g_!(@&l zujhlt_6CK+3R!|&4kCvn&)#re$Pv)U9&v#8BnRsmGXc(^4o99ygCmMQ6Iz4soM#g2 z@X3`CU{>F9kj;Aqvtkau$>Dofr3<)TVC2`)V_EZ{^;!G&`3y%0$F;Y_L%(Imo}=fK`*Pu)~8zNuYt@h{A+f?6xeA_!=!- zjnx=AUw1l*EI)D7Auo`(EQE=Lm4Sg(#(+UqMoDs+1*?eO1a>!_Mu7=4SQK;wTGQqv zak3payHG@c#na?4Z&L=FrjB7VLk*X&?`pr_vobZ_u^iiQjI*z0*9D&F4W&{F8;>$% z?3lyuqA+(d10#n=5aU_FhP+oh4@x|0?oO6`H$Q}^cI@vJ!_R zmuR$_lqhxuHarr!_QK69XJfDB6yc5H5)RyK2RR%!BrqQBVGw-7z$P2Rz;)K(s6fQA zE$ki+?8_vMD@?lBW5eRazROdmT<*XP>GwA>GMajLj567S?ldts?qK8N3Gh3}&>-+Q z!F7`22gdA+yezT}2dufRk16G9e0R+=v5`>%v$!lu`|^cRJPJQ5YvJM+*j?4jWPGLdP#tp1f* zhVz7P9b%ayHHpLjfU|z`kY+PhD)@@!xUo}t9Sd?6vhD4{*YV3Fi4Wf!)NY4bK09?!8d zdy{lEeFqV*k7r z6%R!-u?vnD_!T>%_!sebsJ80n1#)C;NMeaQx{*O_!?F`1(hNtJo!KyT0#BL+qvaXR z1_quFEq_#-rTpg}W|($agWcfr2LHnfWh-VLbV+n~Ql55D(?_r~Q^%3tU5())+vOio zQx2Sxx9F^rYIv_Des2Dx--qlke&C8@Sp1;%vFqav4y^odj0+l_oe#R6?-EvEl8XA# zkfHgZQHbSHv96%%lU6y&S*pEdc zqyG1s*h6Bo%DIhZC~$?JXp%HBdWN|Im116&c=8^|1XDBdSNVaNQc9dhG)JvEZi!POpU`EQwMI3x$@oGH|gnsVOkMbGYYlfI)^q z%VEKmZ*NceEf6bf6rVKV%O6hOFh;QyMv1sYE{=6y7i4r4DV~jR6m@wo+Q-VRwovMY z!y0x?T$(GS0Cmz!rIIA9TpHUa|d%&e9D4fYCYBN#zpSfd0)B^sb zHJly`G%FWyvMdx}YUGgn$X51%nXiCl^N#I>)^2`x}C*rXg>_c4Q|+4_JMNAm=|pOZe=F<6u&v;27=_=kaAE`fvNpFvR8f7mw?|5V=m#x84^l(1MrU!2RqB{%}95+7*Ni2+7|A6VaLZ~D6%!85( z8o9L;*oykY)J`c)=iYZ9HRhhcu2ZfTJn}R;9%L&+B}yqJGGA!PX?U#Hrof@MfR{y)Uys4J zz`!7YgHfKlm21H{gDFgJnw;|&Fla5X)8_MaYhX!XU^Y--;%ZGKM2bI?R-$AC_ZDM z*W8DE1`R4P4|e1*idQA&gP2qtTpF?a%HVYra!W|vl9S<0a5?EXqlB*7ID=g&jN?><; zz+9EUuC#z#jgh^efxD!H}n51mMDr>Enq&8$-e5%=DY{YHCLEd zEr?&az%!Cjtgl40CnO=>QFK+pSCKNVO>4f+dm)*#&{0b<-gJ2w!$Pq<#gZebC1*Th z98`EKgf;4fU9U`G3rJvf6Y_JP#Le;XVp+=hLyInP9Md(?Rg=6ZI6Z-5O)Y9iD1YmqRFk<|I;Aw zVu4T-2VV}OP+CIx`E9jj`TUO#v7}uPd)L6i)5*TafqQ}v^RyP0X(?=55`_LFuyZXG z+VMa@?IHKK1N>DD>iUki@E-*8_*nH~BV)vyPUqkvmS1TxVe&OVu=8?FNYgP$Y zSUy+7malpTi~c9`=s88Fe$M;CkX(E!^5>bIE=PViEpcgiq#Wqy8rQ%&YyHoj&Azoq zo+a^BBs}_cE&E^X?~vaW4?Ie~309YJvYFN|QsC|>6E0~K`{%-SWC4$XA*Y1{j|ZbT zLnBK?1K$@0o@ov2DGe+p4J>C&TfQ9-DqFzvFPWn#!K;Nqj8}nUMgxlv13Sw?H`Rp# z>-rSt92DE*z>={=WY+;ntwjEH3uX2!5WB%Bj0k@H-#mwq2&BDmZS!j90#@oYHV%} z?8iQFet94;=b>26LB*SmvU?h3uRIj`uuSHJqtV=lLUR_%yi=H?k|?z8pism@#XSck z)f5W~9l7uP6gae?VCMn>u0*y~mHc{)LK;W7qaI~dFX%8ZD^P0`-Pd5ZYN6N$2HR`R zWsV9gc?v9Nc2vwRDHr_KyPAP_RYGrYV&CkN`CYa7n$F$!`mZ1;M=z2G3{hHP&~ z1_7^w-dh(uzEsXz@89FHj^oM#mw!!4-d^8W1HQQiiD~{>y2ql-dCPjWOr!M2;#1@c zITjp9FKuAopdiqe!@Yz-#Hvx)YT;D#hr&jU!Uh+)COL4f(-U+#;G+7YQGSPuU>1wl zgPf;gR$G9L`uqg5o_lpI7 z>eB3Qwy_&XH@YXVwahHB-Ob{_u+SrXoppn&;{&fP4D*fEin1GdIG%`YXvll?^4>Sy zMb-)|V$%|>XNlji=;iM7a1<7ca;$K^#a!@Oew(B0jfc`(ehTeyPHMff6UU+)#HufLq z`Be@|E>hq$Nfb736pLXNjy)*upvdXdz^SlMlItK-Pvp#ehnfA6Y^zkf{Fe3RC9otp zuq7SX&ZWS>)xfQgsP~Rxh0wtj0uNtBByMqyXmhx|*?xhZ%YrZ~hjpd;Y_Aq+n%~~+ zS;(>_)UDmp>qz>HVg~l218g-jIcl=mTNDI~7a5xcBmu9XHDt(d~ui4}Y18 zh9!zNI7<9ulxSEed&^NY$}!=0=yv9F^Vj{9zp+qu-A;ig3{q_lnoE3`|11!bo36U0 zA=&g)WT-md&S;Jvh84LClOh#Zg={A&&-9Icpt|qhQ&lsS%12FVyZMFYMzxe|Yb-V6 z_!YuFje+}H!ulf&=@|=!tsaU7+!d{QD4x+MyygJQvB=QZKIs>@ErHJ`5~JwzBv%a2z>)OK=Cf z$8?ToOgFa5-q|h6ZcxqUBf5F}0(Kt~=u*>~qRoclK)bKR^z|-0T>Q=UKLUOC}k*8;)WR#WU}vOf~H zZa$D+JB^b`@!-)0=5-EJ4G)UWx%bliAY;aII z)XT42rXv{Wbh1;(RxQs`@eqrfu&q~x&cur?9747V-wYf#s<;c({4rA!oFu$WI37q; zP596%Z=18w-Mh8s;$hYJ9)ZIgJa;!Z`>mNXvD?2|rhtKkwc!FopmU`_6MOIi#;d^x zw;WFHkBpt_6`Z*C(NvuSJjP-c|5G-zi*2t6Jjfn&D3n{TRP^A?xXPzbrwhj{F^mw* zp6HM}Tp!{DxJ*N;}^?I-@Ep!_X9< zz;l(cSJB7NRn*ez=wf@m-aHq_tu=d-VF#FvKCQMngayX#(B(sg%Q z?6;>bQuVEwGZH&l7e1Snv+P0h?4n-`EOUJvHmYX^2pF0OR!F^Fp1o?Pn9D-zBVtE# z9Cwtpt2E9i_}9p{JfOIPH7;3!fvqf{*h@HRLQ=Qb<*5eD8zKq{v%D?tTxc%W@HJ=R zR5>Eh$je~i(6EkmN&&lO?1_YC0j?Fhc=N*F z{`VK=jYou*H;L*vHmROASl_di+hY2MbnYoZO$i6(CWjQyJyltGQ#G!kR#GCdgLC5y z{+_E0H&acPD9oJW@Fix}#l@Bvmn%9jn%2A$o*(gemea=tnb$JUF05c^RTE(nQaj+l z_NkyN#U+uE?SMnAnZm!0U>+x)KMkjkbKg&S#JKDahuEeGt_CvOY@Ype zPC63psy#* z5{#wZ^rZhzJ~Veu!j=drfy^Z@gR?_6A8-=%GTU|&TyCK)u#4;mI>T#rLSXn_ZrI?N*NZgXgDzP39vccabz=`(8wlN zFrU9gD?j-PL#a|72-l&OmZ^41`IV=oQ$iXpa=mDE^>8avG1&Bcp`5{CZlzh5WPQTK zbsLU(FL){R`_Pqr$|`EUy*nf47#u!zk;T_!H(M4aZ}H?YnXp~3@fC-; z!$HaN4Q?)HlEkjAXp>=NV`?bflECqpD7^>$% zsioDW3z%>Eyya|ca1ov{p(nsV$x>|tL$gvLm)jgA{yc+LO{K#2rYZipwhE2qP66A` zUY$_-@nS zvTPAo^h&2-vB*31Pnfs`wz0~Jou0r^;kc{(;Y&aJlWYM@Y(5QwO=aT7_GWt-{C?fO zFF5a3$XC6${$9!YmlGB@D_&9Cw{Oi^nOCb@cyAmOS!mHB*)f?#P{om_Ie|sGM1U(; zAc@mh;UM3hfOU=+4IJ;SN#sA~5paN!?=XMHA;k|4EfOW~7zGzNu*L|St9>WD<&RFI zoxS%PL$2K{QXT%RUINWRClr`iFE(uPZeW}dQLGqv>G^^zb8eRhN#d<999SmROj&%v zEHOIMwIK1L!BfriOZ+Z+iN34N4;3jF|0DKUbwg^mt%v5nvxyS(#cfsw@7obs+x(Mz z=DCsj`CDdF|F|H(i7VA`2-jk%6>4Y`79EMI?>20 zbm_ImQ}>%s7~k4Ge(Wz4vD+@W$N!a^0`rT8YjZCc{^n&oHGzMD827_8R=qim9YPwF z@7)c#CeOa}?g5kYso=DE25QHis)aukmAKui{Qo0=se%*7nhVU)%S6~*ERwCSU!C-8 z%ak??{UH(GqZovWOE8i5@*JOQg54p=My+QAtTg|S^ zvKv~QU7YxG4>u;&-pvtO68o>J`P`q&`+S6-=$7yvYxutWasBy?mgg5T-zXE(X%t#f z9KoyTtUiguu8~7gA=MYKUl!` z<^Y#%0B79;CgotpLO25hy8;nfRO^VvdL16iC@L%JOc?{BRt3SySrZr`%3rD0Rt zR`ViXVN-4uQ}yk&VXABsC$OqDu%sDq@A|;K>q5=R<(xqVn%)!GmrUT0IAHego@2>O zX4w_2TAGP^)&)twP5gY1JM$T^v?g%IB`})>gm;B9N?nlsl+I+7!1VPvvqb_^nQ)@+ zK|!Ajf{6|#i3eCzKd@ds!raine6B^PRe(*UA*)BjTGfd~v4OR70!P_(H&q1#H$wxK zWaTdQnD=cqwnY{G+fr8+G=?7FT3^7ns(`JGo$qM_hlK%W-G}VlM#h=XdCCuX#vEX? z?hwy8V5e)G{#Mzzn~`P3_Nq10jejpLbl70x-E7iuDXw#xxAOMBK#q9MPt{u)S=$xZ zJR4XRFW}xcf$#MMzFiG0;SSvC4JVQ&4v;WNR>B_ZJPM@FLyPHGESS}zE-7YMaqV6_n7_#o&b^3lm>f}*Vhd%Ocv zG=qZ61(vF0a|1zz*~JlyikKffv|PEOOJGI$Mu~3LcNb5P3aF3@qwFJ@(i3jC$O$~z`gH+!Hxvh zNCU1d32ZVGSkf&x|4b>ne9=LETm8aESu>66u75G-2=HUwz+C6R5tqP}tITYj5b^k+ z9?!obA(;>LtR0MM0ak(ruK!A{a*vx;ZMN*yNc?He?EJHRlknW!9d6SLIQcucCp$1H zI52pHi6{y%`BYAol(798mLIS)HMF2Hq=7BeK#V_vSIdG^&x1p#frkJ5^X z8NHV^*!>cE6O^hJo2V_Fu2#Ns`d8t&b=&P3T{2c&jBB6i;QQG7<5cFW6Pc|pEVSCd zZ1td8HcHyRfTMW=OV|YN6(6|Y9^iXfz;SpY=dKT&Y63IazsVSHV7!-@wVuuD%Mr7R zr?m^8Ciy>S5Wi{O8o&|rAu*rB?K6v@#z#S!4OS8x7*!UuEl}3xI-&4*VLjh7OIF3) zWoAqk8<_vag(zKI$y!ptUj2aG%Zs&o0{2w|&PAK2I2+FMooNum7UTbHdE`Q_@P%CB z7Oa&=xH1~J*DMg$4C3uLz#J^jY<{38_Q3qA2H80(({-H}xEk~px=mkNuI5%Sz2;@? zgBMj&;eFo03wMgNUR9Lt+PP3RN_L<5B3UbzJO+;D0;Wg@?pYH!S1=SODX^v)EM9zo z(cFv6MxkCRVOc_#mA#hsY-0@<2CdDDvIPq!TDNE$JIpRWz!L4iQsltyvO(ZV+fsSQ zuuS#bf7_WQ6qubqw5{L9>|(%lVRjEf@^g6GV6gn8dhkf>RaOt_3bT)EpY7*T1ZK-P+fF zEUy3OD(P1XWlu2&Xs=G&wa}NDb!QcmM*-K02b^;XSkotP&P-sN=fJ&q0*e;|E88}f zcV1e}hnNL5-5wv*a$#_06P)6HfYqjC;xiHEIsuOCo2+#Y*sBfJ83;0#PGoitV9#}C zw_L#JC$PP^fn7dehj{=?wE|1b19s&J%+o)xTQ87Rc3^KYU@lSMur^?CT~Jb7z@qBF zlEuQLs^stUfy3;BzQVs?Zd(C^axNPm%kFBfm^}~j>bz7EHgdg*T=B@0wKc)oxs9V$ zVDIDruB8c;h7w9X4qQ7JIBb`0E(%!Se_&ZL8#tNS`hl$6tPKAJ?ClE#QyaMEG#r?5fpeV$_qq+-sS{WN3^>@P zvD;Moe)_cd;S%#g=k=AsN}C=tY~SR}TEN_RK!e?YsaRmQxdMx`0yFCb=Hdy=Mh+~; zre*Ujm@U#E&bMKwfB>^gz-|EsMkfPi1&4KZdzkqG7@ZVEtOXeDOqc{dFg`ai@Q-0| zDr9(mh51|px3$4W>7^-;W^Vjz0slC*6>vxiFe^7R-qPl6eIQ;bxZu(Ww#zFv zU%s)~AX(z&ZV8hQ%&s@8W-crYII-&LL6dzfrpm_`GU!MLYOlVO$b2txt6u|q`vKYsB^K1bV zpFLsV4`4VujZxd6*V2Ho{Ge z=qT14Wp7}Pk#YWGfTr*~U zG@SCdF-cvG*)@TgRe`}Wg~2F*UhYT{3N$ZhC=fRY^<1rRjfvSMO)PQ?hm8!-ETdNpJn|_2fUR1m=xavX>dqPpXi{U5BgGM`YU=#+D-=Dx%S24#ka zB_9~r1gahST?XK|C*DhP-mdbH$US5*B*llYXV}XtbMKefYJN_*VYS+ zkJvc2UWmH9f!)v}!0-aggaYoSf?Ks0SGwI>X@A=&$AKkB;I`ugwsyr!_6^flePOP@ zIbCu)v-1a&y?<|Se0BSa-0H-A3ok3)xx%P(_o1wJ0sGPf_J9fOI}A8d1paXa889$! zU=&wi4oF~EOi5_FASmr1w}yFAf(mn^9kbDaoLpxnrw@J>0W6BQas=ww`6oE9zIV^& z!F{U)t}_ap{u@5sieWfb^Ky3F15O3TnR3^KE-)<6W7O(@Sn}YkfB~ae-9wcG?sFR+ zvMiWV!DZ7uce&{$#$W@sJqkCY_~$1(zDd$yQWW?Ws>GBW!292fEiQn!eM66)1Cw(D z$MratN3(C;Ik%E!*W1e(ERXcwEy!YVP*~;I(DL@(^mWHiZj_RX%6q?c*858bYm@zC z6Y4JiyC-XGz*45bS;fHGSiqg6z;^N=qvQq#Z3C7{1&;O)9Lo#1TNbd$3R<1pYR&x5 zl`-ym>01?M%Lgn~4(!GXEE5GdCnd03O^|L}!1#1dJ3|74)CZ193z%~o*e3;W8YR5E zP;-Wx@0DZy1FmiqK}$SitIlWy z9>`rgSM@F5Yn(1*GKYj!}JSOC^ zLqYVX#f>NbCpvjKtEjVuboQFuXi(yvCgC!{k?m@L@-oh5j#o~LCIy{R=UuqVOLO^( zU0m$4PCZ)e!qYZ1i#ScYaCoNeB8!$Oyh;lknk3%Mc*4my$KdG%&t)@H`1O=D9yILc zVtCLf$g{bzMW*8411AB`PZ166iWMHurlovQaGa3Fb@2fsLkRP`CiVj{P7@TFR;IKu z`4l@avPXHZ-Ev*#M6lnflxznkwuZwzLiT0F4;VR>Zs;~~7$`gudps$69rMd+$}1Wh zr>yQ1HHlcs)~$6cp>?|9g5M9ApHI8pEE&f2Fo{K7&GHE!uSUheR{k&rC6RPF#Y3FB zc2$o$C+tXY?wfR@LdlVlF=a7tI-`$(6G!n2O|K=Je7iK4=5PdR<*%H?$S!$6H;`Sj z;=J}#k#@&=ABqP)1$}7xABZ!-5HVKFAWOkuBbTpp0|3v$LHMXtbK1x zIpUb#H9B|RIM{f<>b9;WGtYs8-GUB34qSRRBU!J7El9OPK`X+sT{LW-aA{X?tU&*z z;PMv_+aw(>9B31Kp{2+sqH(E#BWR9dOQ*VpQds0CH6UW3V!J@vjfJi52@`t@8tr3t+`c=3bpaEjMnRLXjC@YUVb>3df_WXy4|%S$9cMktW+}0Vw@t&c zATX6}YVx9xQ&%lZ9xmW)w`k?mJHY5XK}j&|LBoH(21XfyM}qzeteSp_R)G_mcx+!d z#uj|w@a%BpjFezj;9~6P5ny6&%xII76XY~_sVKC(q3Mf{BNNkyOcSn%0}DkYjqSD` z^RtYOKO1H~YawTs(eyL(nE4DGcsdTS{3|wLmfSRfRb`F?%S!KA%Vs%iJ`G4v@Rs&i zrn7VokDl-J<%bej3%52&OxVz*(Qz<&$;NiQE4cbs8#=%&5*|9 z;Go*$xG3GgIhfn3O=m?Ai+A@eZTp6mB6or^AN>owrCa>->Mx$jN*Wo_wGA_wJw259 zON~1>&Rei!@6oUJMH8H*UT|61gb2@W3Sd$G^C~=-t!#Fegi?qe%T%c=32bH>6Stph z)vLbudET_wcZ8>RFw2=dsRW6C63$ zD>v~v9ANcSP}){8rOjm4!oJEG4$RUk4hxrTV3fVV$kkUVXPS~A{?+zVs?}6s=D6g$ zR(sucPfdw0H_cpeKjZ*|n1!R*35T{IGg00&i>0FH5-teKBxIgdWLTQgsVQ*cQqXdt z^LkU+!eRnd4(KVYX1^)MEVsmz-@Cv``bO5Iv{^3es=5EoG5z}?oF}+RaLR%s$|gJ! zZUWcT->83LuT{n52@0>L}7gQswo(Z$ZYO)G>HLw``V$iQ9kg0>=S*Z)=XuDa!_ch2lE}Vo&La_>qmFYvg#@$h z;W1t9$jhwqGR=JYv*rD7)@7;`q&uhPn+gjw3eCE8B{=OMm&XPtv2_6%#yyL8)E0zX z(b;<7PuA7_yO)%(6@G2pCivq(+EoWWB}LwKd!{O@&2*Mt6v@uHIkByNM_R|U%b%xWIb$%X6GNiVv$UV6+d8=DI$ON$A#WRvnICJmD73 zl2rvVEJ`1yiTub9_y2m|D|#0b(>K+|@DJYNGX#V#&6wTnT4Zi^#_{U^MzK|_;b-?* zJrdl!twmEOurnz_Nz&8&*Ba%C=R~Ggu)TKq7}8|>MpQcbwe$o=&S@LigB=);ulRA) z>e@s8vI&jMK@3OL&nUDxa5Ra47OQ79a>(%r=zB^xu<|7^*?u^{z<9tx@ZY56?4A!E z^j?;3;JYKhm1UC5QJ8i1=NFN?7V&ZZFCR~43QHE-tWaaR*gXEQ`i2Hhp9hTM2O2fk zH{9w>SL9kLle&=qZ&lXx2NntqUyA&$GQ^r4VBKf`bEU_>Q1LZ$4v2U%8fx!g5_`fR zB$UY1IOnBLNN~mThTY4QWDasf&R!&OGQ#TT+5CSW)t&#ScX_FbSv`9M;Jw{8kpwE~l~Hhk?~tf@Ml)i=#!0vj$7Jp#+lxgPZ28MK_s*D?rEM zSyc#&U1+?znK^3*tE@zWx&>?2f;ROXJ=YQ}Je+!2DjLL>2+x@`*-dNyRf`=Lr2p-0 z-fhgDU@>nZ!@SA{p%YA5o0%Gonr|923Mw!P8!*Z`H1$^R)s|pY)oA2sSjwto8Pm08 z=|Tqa1q_ouANpf8mFvUOX~A|P35?H~)n@Ky^{8Od$XFe+y7}nB!}}R!>|Qk62ed4% zZcA@qb6`1A(7|SJWAALT=4jEHaxQW4&AUXSm6Ee%B|8|`YBR29ZVnP)`RB~)=fE1v z&}w*~No7Nm|AJP3iI%httbc`@Z8MHKNi?2Gn8JO4f#-z!0R=|M6O7ggY?2HMc^@?J zGO*O&WU%&V6HZ~8r^Rc{xR;$_ujNBYyFg>tBx74f3%M%?7cogOJ2bf%G~ae^4Sdib z^zTMP%_|upfkwlQL)se6MgojaBNnhq?OVg-eDAb=)tU7V7dLD;r696m!8Bz9DFwAg z6GhJ*tsV(XoW)HURjcO(Z?#^d=DvU>jibG|fxT)$d*uT5ih}lH3-%I;_5u!1r<%5u z3by$TT*s_DMWc1ptL#^AwlALJlDzwf2LlVA1arU*7QY8A*FPWAe!vnX(dxLPH7#SU zg9eM?1<=WedN-Pt9yC3=)a>x0$>svH{^Ch#Cz#bMn5<_o%Vsns%CrO<9JA5j#Mym zsj4My0?RJO2|IqxHuI3NI&#={NlUsxd&La)niK7H7uxF{wAa00ub$CfdE-Q}1bd-C zyQ7WLQ%#-HAeU(-i{fWgT7SDprg^lbb+l!6u=OUiOc#mmuPhD9KQRp5+T z?@a$wEreOuPuSg?dmf#LKH@#fVGdpIsKX)taV-VzwVUizT%oMxk-M5Dxl zM(GYl9t%e81^-&FTQeKouzj$&f$M^MZOAM(2IsR^&OH3=esPV$#W%+o#FmN>W#< zx_oGpUt!@%_cwE-Cp6mh`qVt~Q|DmuU|67Yqs31n!oou1 z4_Q~8A4-PpogZyG<>$fq96J~e?`inBi$SzN#4Y$@XTkcSFKXJD51iPW^WkcP|S7jp_@3zZYFlGO2Y!z)%stQ(0@)WIBcV8o8 zy*EPb0CVtx2B8m|<(*ldm>vz}U^TR6o!`1Lc*ZHc0v^K;5}q2Z4l|mq8CV=HFdNL+ zm!iP3$wF`HBDU2w2ZeLmY%eg|Uua%+h1vE7v%WHmM@7r8UY0+p;+_TFQ;k@^zK;LL zlkhJ^lj)uRv_)+h9`}W9WSgpPmn5(|G+f)Y_4b~-x1}Q(`81dVDr|inngj!49tOv7 zH8kjdxmf)5P?v-@(_s_QwCmyt;Yo}BHHvOvT#(CF$=hiqz&U@nbM4)5FNgNatT(R9 zK2i*eP`ts|V%yZR^{%YOaz(9)g28g%yCYO5b$z?s_(1evg>Y-Ig=>gFYal~w07q-I z0E@H3)kQk2P6{m17OYMZEiVqn+g6Cg6tKlGw0Iwq8nB+yFLD5o50Y!C+$c4*I@hSrpmz+a7PJgv?e_`{1o8j&eZ5981T)&cg_nK;wv1pULK)RwplfsQgg$c1Y zw=&*7I)U$_n6lMGt+GWj1%_%Hn4=|_j$5<=7ImW8WkYM)gEd=AHz+a)yLq(v8#p>GxMI72*_xqQ{YBHu*e2B% zF&Yg_wQWu7TbZOUq_25<;PI(O-WiGM*BW^?Gzx5Dv5qxKF)g-0?UC}o*J^#SC46uGDI`2ssJc-qv}jetkr*E?wHeJp z4;pzRgo6cIcnumgI5Z4TFq<1Z6*8H-_5$-XD;CE9CW8g>ju)EJ4z#AtXffbm32I;o zZeX!AYjrnZ6)d?Yzwwa25rfIi?!|{3{>+p=Rl&d*;mz3K7B1WCytC&?-()U{`JWhP zEz(-Wl@a~Yg)%)%QQQi8UpF-ljoL>Kt(>hnE%@bLb~7T1Kj=S z`meLlN{M@?fbit!0+q{U78qKeV0ivha9S9XiozyGfo5Kge@x~bEh!4D`jdt9A2jK0 zXjWqgvC&|h|FOyG0<+->7uA4dr3!`<-=?z9VEiq6@F}0uEKaQ-6OHX&s$6>PxL_g! zR|bO!$CRrMjhruduX1SdNHFKyH2d6WF_vIe5m=fKq!Y{BDD1!#>%i)Np~cm>VRh0S z#jjypAFd_3g^M$seY$(9xW{3U1&!jVK`!NtYN}b+dRlhYHN81}J^S8u*MPQ)8`&K1 zvlSmS`q@8{{ZJCP{(~|fQ}MQ2hx|&VZ+xiuCeY6N(b}Tb=0(r(K!8e}!hoo2K+C@?DTWR!W&6!b!lN25V-L7CjY9X;$j@>m5LUaVvI z)hU^s!ocFtaEW8p!i~oF3M|@x^R_xOa;|6)ox#A`@x7eAkF$f}^P?Tw3s@IiX;l$u z6yD$>WYHmPz!>m>)qjSQ@q-4AABQ}co%3_0xdgsksA1GM$v)NFC~~1<+cHL}gfA}N z8O0Kknev(EI-4F+lzDe~p<4i3<%M?N{-25xOo|dsvKuP*GnDS%@UxhWYeC0l-u(j4 zW4M<6Xsl3UvX02OTGSkzz;c{BHPEhQvXNu5No%mcw|Xf?nHNlQ3|CU+7YJTp(oJCU zRA^x^n3S`h(Q!jd!m5S^UbdVoR!Y0{a+ywkvCFY6eZJQ5&2vw7OC(JBCv%}e@6v8VVFHFfwomNtk3jSis24 z$t&YkARyS}(kP_oHpL+@xy7^EO@wR7av{_7RxTS00YxS@!?ZIoo`+6OT^+aYU&Yx= zELkFJOZJt-?hI<{;pCRr2qchjwD+G5{|M|wqr&D2&c+k1+gFsWn#tBDyMc+#( zrEGd6Ggga!B6+WgoD0vsxp%b9xQMU&~$5(n7b(Ll*md| z@2Uq8Md_Xj9_G5FZGFh@fsmpNB1twk{n@wUOc{>lf$>!{s zI6b!J%fjnMFRpwJPjpxEWD)H5Jm#FzZ+TLJ-_b+OMIcyh+u@W2ZyOG$yK^KS=JUBx z*r6$&v!RhQ&?WJhLfQnuR=zx!NoO>d`4o3+E^QI?@Msm-#IDfd@aU*YtHQ2!p;m`Y z>?-9i5}f#xIu@;1xKzOLK(&yM!F^^?n}!AkX3qr;&fI|o7oGV6LIm9;BwZ%zPE1hf zR541JaNNXYf;c0qcA~*cZ^zh!oa!t7eB##KQu4?%KtDCGX}wa=hl7D4EgiWb3UQ%H zVTKV$|2^W;>+=@tG1}yCBF}%%)@13@2NmKI>UB1qvHq?wGe6N)$MXo^Kfh_I2Ntvb znRy_lt!=|Jrza0uWLxhvF|w&C1hT2mmRaqo7Nj7^s?g-%VUTJw-XT%sp_JM)$y;}4$l-I!@*)X=taJH39AH+q3Aj*mf9>}RRYEJD zuW}Qf5#-6%)-h z>2`k(7QesH{=!9_U5_KtbMeRN>28acs?{Cq^jEndEb8W}WZ^HRX5lF2;K&_o;vp_> z!>YAp!?IwHW~nPXTs8l_dvMI)4x_+CjiZ_#6WeWgm;|aATr_trV=+-n5}2jYtXI_7 z9hJf?dy{X+Mf#ZSz zi?LiGV{p%5HbIGg=MAcj95D~dA}cuGsYSFI@i_6gS~75H1u!lNIN%)mQkr9hk+KfkH%L67WIQ=v1SU5pZu*0E6TyG<@+qP4zaW{ndg)}--PB<_~o?tY& zp)i3n-r3X#?8cUgu=5%eFkaS4sU2eYGrBk6EM;3{%QD8Ov)EIaEtFPFGBB{y)6Z@{L zv^=+Sa>vIh%VjzeTB@A{P23Zd4r;Ady|I;V^Tdx%3Iz*UgeN$P@A7zIQzF=^GJ{cM z!HGi(vkZIFraVesa>G~io6GU7a~_Foy&7!fwUJp;$C2I2;gHaY4=l1@7O?mk9AZ~G zbcOxGLdnJr4sve{j`M2xvRyvV%*|iSOa$t1Zr!e!F&og$Z3A!siK5HmV((YVy zs7Y+00;}PU4UaMu79YEKfZcG`9g{Z~oJ+oG@Y%g#PMZ4BPg!@HLCFH<{Yy@qmwxZC zdGWf{8AJ^ZFo9Fq9>6@%HyVV70yTNJz=xu!hmW z&INN4MdlnhvT2h5TaZY8O3R7EnhFbo8LR1Y|CdL=aJ<}k8baJ1+?VPIe{dBtSa z(W1>hq0we>0+Yb}241%efdg_rj352#)tOBC8)g_hU`c2&VckP%k5iuP5c|=9xgf>vX@po>DL#DRk<9EKm{ZX94>uy|inq`>Kvpukj-v5Rernu7g< zg$%3)43!~F3~pj?8RS>6aPlmu3H!T#H=AuP;gRw;i&MXfpdxjKZBFf z7X~qfCeMzBhKg{$Gycmh7{zZmaN0C9$S??qFq9~t{H_^eCM8SOay&o&@|^%Gle!)A~q#x;#uCBebbHtv1Ey-f{HYAuI^Uvx<|PSHFwQN^cO zw}g@H#X`;-hjcxd^*S0QCNL>@I9c9sQk>xErFux`kB{X8Cr%yt<(_jd-8sNtsAzJ`%6#!2zP0XdOFMLtga7aDbI7{z=JiI*^Hdnl@G zXc7}}(w*X<_~4LuOp|fNK{-9 zIAk7kNM(hyZcLNa1}2pihfQl7v*He!H#{@na9D!HS8)TAb&ON)qeF5Vn6*WmEOQQX zt~mTCmDxJM@j}l*l^beGQ<^P%99TEB@T53!KRKur)U1$jfR*9uzfJ))rK70 z(6~g5&WyPI4WNMj6bedzNax2or8# zRJ_y3xula*gHeIU>FWzdO%5k+87Jchha?Xik(3b9UK7G8bKonl%e^lSrYoEkHJ(c- zH0AakG!tp^+~maW;jH_G!Fai2#!$|j`p-;kJ| zqq^hfjUA<_|5p7|%IM$}=WtRnJSf-mIJV2N&FP@Rj)OuRhZJla^E!IwPH{4JX_9Vi z{QSmIt|3l@@6cR>Lu@t&qnVtn`Wn3^GKvW}a!hDD%$!1IGq@Pk9~lmoIQjhrVAik~?k z{$a7$oyKid2gL*qir;BW*Eq>|&B!syf4sfVmUZCc>Fze9*zP{^; z!Z$;OZ=L1fJ;A`eL&fyVLB5AaH*bBye^y;}fwRqn!*@d;$iG=`d-ui;!Fkg|;B7h=;UI%!@xl>63A zWkLgQ&OxCU$Hh2;IeQ#9t}J%g;>D8T;C1hS@V5qzHwSoZ68TyjIREHA>N~)9=K%Ye z1AI3gb2J>dU&p}F)3D`09Lo_0&L0msG8!6gJZ5)k*s_MjTc&}d;lK((NB#rz*m4{; ziuoH!9As&j^-nRFizTZq_k>@bQbk%sc*NwmP~rH6&lxYhQ8H>0{=&hZ`sy58r&5TM z#R2BfOOICU(%wH$D^hMnM#+OqeF^&ip4(nLsC?s4p)2Ft7AA!fkuNVCbDth`@8nUO z!(?p2>>+tvc?HiI1+K3gO^Q>RRQ^3^V9aRv8p|Lo!o>MTL)nypp(p&v)OCz^7+999 zV_Dy_BDCOT6ITG^R71(hS1i+6O`gxoI>55#z{NK!Z)osxY<79}i1$t6fdg?&d^`tb zOqk~xIT?R($GcID4+mNpEp(FZ$`UDGo zc?~AjmS*cS&UPWH2`^;#^T@Fmd8V7_?znm2(zMjbTMg3<4*q_1aNV;oxt2q^GEIs* zG;Ta_RJpP~Z$^{CiuEcDjfYOIU1Fkmbru7^kCWyD2L>J&t`m$5+Vac_E}E7OJO(~q z2@UdEzB~^um8>~Z-FRxd-vJGKhhx1TH1a?E(~$2?%lmL~$_MTB4ELEj*nT9kNF3mJ zlC7xlQsBi$WgmvOUmCB3p62_Vc`UUxO0iGPFIZ69IFRi&_nWKSXV!STIjcluS$&<5 z_%TP-!P(}F^93O{d!1{tCmh3+&g^KFW?y8;X{RfttZTrunAhN-jEmFhIZbnI4hp|$ zn|H>~=t-kAgOicZ;hdX}!ZQvPeqbo@@!@>XAo1p+*bE27DNRBvj`&MDsunbU5@lcz zXi%{}z*WK~vG3vbQw(R{YTmE%(wHC4Qsd?A?ciPBWw>w?NB<|z)kZfR7}&*y*&dwr zdUIgrrgdUp8f9}F*~3*1>5Y{|;`T*TR82D9xQXM2v^o{NsjQx`}~-xxOQqojIIWE~ryh0nEHjvrrm z%E=y5a9|YL!N|8i@?(pLgv1t2mL|}VK3@*-eK;WN(PTdFkQh&sN(m$9jUwqw_vF?* zWEW^)7HD8%Xjs(A%T~k1wf+FJNIcv2FZDdJj4UTENDAlrH>_AI$y&p8w)R7o`2Bfe z!8dCP?l%?i&pE&$!Em3;KsVpOXko8U`T>s1pE%^t8d^?RP?uy}9Kl_>mH*6}5;JDy zh*wL~J_l`hp4PsN8jB^g_njDm!)1)AhreM&>nZPL7aWK8I zQLLxoQ>A#YRE*;s?b9|xHw0FImztlhY;bqbAV&B5`b3Oz!ci*^kKr!dA ziU2Q*PSo;@XUtFMa7zTM@*HHC;v{#XL4~7<=Y^xul!BWRJ}h`9%AN7Gr)Tbr$vYWO zF~}?AGk*%;H`RZ>V!QQ)X4?nN7MB-Y{A9kSZ>ja_;=0S`^Y>@1*_NTgBf}oxBevuA z?t{mEPI7t{=&ZsK=|1z2%APRs90xUiM#Y3F8Xo=puekJ7oWzz)(danzZ)=>hE>FWJ z&8QhR7rZhWcuq8^{Xf2Da|x?F1Iv+xD>ymMUYxTcas%sn7tWdnwjUO3#T$-DOy)Uq zKyE{mTn~fmjXQ?A5$Ajz*lsu&-EdrM;H1*gta9UE_QnRDn#!9C9M~45G8UiS^*Ei9 zK2(%(kU^$_p(32~`5mvC15eY|^~pO(tYY!fxmB~_%liMvz5Ng9NLI66dF=Vz_XFcU z4))Kwobh^34oz`wkPl3~!<^E%hMP{r#B(n}8RnZI3n$@i+umeH4$9)2vhX?LsrzSDaC#-=v6^bXJZ+Yadk zjq(SkEtI~Z@y1WRq{+FrNhG3y^+SWpOef(V$CM4qK-U5o92EM%z|Y|>!O-O3_)AEJ z$+YIGq)L;I}{&^`zUYSFr}AA#7URMiPgcuNrJ^KVQ*R*12@YpZAT{d6Ytr6Fj}5D!1CrL z%aQ{c|D=s~ZLSS7IU{uEL@z_jT(imrHw>rh1WB9aN*I-VzQ8-bqpPda*nPd+zM4PZ zO27ONOgSNImeX@*)eSijpc$N3!ogMA;WN`QRfy4OV=5=7kP(YT zM4;oLb}m6amk+P6JF;-F$=FmdJZxm<Tabd^Dh<2nRa?No*u-*b)5ov|gNKaVX-#@Fl(r}a83|V&dwF6*(jhL9)6Q(1I^Mby}*VDXf9K9#4Y z1S=_~XgkVjs!p;#Ah5VweMdu58&6Y(Q;)_J0p}jIDFIFGy3Z<-I2hS1beaw@2(LMD zSk6Jn@QzAH??(1HEecGhbfS$mb9{UB#zZp7g14C?VE1Y!UJvbsEy8IU3{31ylizxB zXiPZ3ct$SaiHPki!IWk@zr=&RnN2ekSUG1{f4T6|D4|g>YuBYyE}n~KrU?Z5NUq;w zcB$a8O^3>(W~J_Xo7Lri{(ED6b@9Y6uNjw3%n`V7^?%A={dUQm8H!5<1m-BRE4|QA z=~6MdcyxN%pN(pQDoYHU#Wk0#*s?olmf(9%*^(OxY}OtZ54Uie?Re10#8S=BZc{Qr zu}hVqlF5UotHY&FM(lvDaiO3z@5ZB3Og5d=C|n%wb(p6+;Uu@xjs;ATDJv3KBuyj| zTiQ>%BsTFV75Fqu>`Q2!Ab4X6FVh~CjE1Hg8Y>nsD5_+t3CZf6Q05n#X7IHAugom!mF7gc)P?XU}Zy;=0&-#vAv z*W~|nn}$W^9(CN$RQPr(pY0I`CEIz-uYEl-~^+1%p2YU9!trQ0?+O_rJ&2`RVbA{5(sjdn16=2hcJ ziWAXtU_3Xc(yg)eiP;3kNok8Jns^Ku1fAKJ&Q02xX?AIX{9a$jwI^iCUu{&k`?=}b zFJ+aDSL=4Q2~T=+oGt11!Rsll4$%)^g(mDi$fM!GVkyAHN_P8?SD_}G!@*d)+y(W1x{$dUdp<}kN_MgybL0T#y%517Pf++&(IVZzJ} z5BPE=F2!?fTXIO^S!dL#GbdBb8w)M<%Qht^vA7vDb4^fSQ$28(N2X1xCpOJEgXth+ z&Vv@oGl_gIJI+j=qAq>hLXDaKgu_v_1&l_X4NR*#9gCJsHh!{T0h^A%5nZ;4eI;+4 zWX}aSscd;Dk~Kq#Bf!J?vc#s9zP?M8o;Mt~Fj;3BN2!0KDTG~>XMqxy9Zxit$ExSDqy)lFK)V$X4iZ|VkD-Mqx^=m(1= z*A%c>Yh9eqm6E{GWWXo#NuX2lM1x@Ig#$mnC~(NR{9C{vQgQY*Px8_GbJJM19pss+ zR>e};-8sM!KjUC})&pk!y9fHYA2g~s8Mei~VUl@i;VR;@kk?HmIcjFftv@DL z+5EOH-#@HD6jf- z@G490q4;Q}4g0sO(W?KQJ^8QLkJbOLo>cgh(7^r2MOY?fHHW=pm+j0XsdEi2TlqG& zEOSs2Ni%Fw&czg-V2`8}1lThMjCE=KC|%wt~~{de8}d#~$2wC)C9zgzc%`X8{VavbFGvtp7q zNaSy_ICA04tZDu|Nh0%498&O0rg8K${{9V)JbXQ`Cj0GSGy3Pi zD4tWuC!ONJ_B7-WtH%OnxgCrak4;-mycRN>=~PY5ZD3k_U6{9|W0IW2%#s?ftNgVw zR(4Vgzg+kz*c>Q*WB)?Cs>Z;(@j`bFyzqbjzRvr?G21yzLTVXKDtQyTgC`^kEd20G zVbU^o4Ua>TXF6DvnG`$HnUwhV%dsf3F>=`cSST?4z$w|B4sMqZl>& zVK47A&of9?H@qm_>s)4@HlxO0tz57?Y#z7M-?cMOZ&=CVx}nwBD}aGXWPw4~#Ith( z9MUW-8+T$-3;4cq`(UC{Up*a!8MV(nEoH2?fiRYRfI;zjKc7%L4vi3j7}y z@V`00_iF*)hX*s69*QklD8{BJ=;A1z!zi%f0C&LxcCJJg9tUQ&qjxs2^fJ9lR8pI- z<)EMUf$`c|pNb7WdJDQe9^8A<#F~;&bHmqI?SP)r1LlGUJNX(ISeVU{`l~(E7^U8w zk$Y!s@?doi)73Nv)r^Loq6M#15?Di4r1mtkMm6Y2xfZNR;*5Lub6)>NhgP8t>sLN_ zX#4HB+`GgVpZ+yoI+9=iDpAg3r9khyrBC?(<{Wx4P4C~G_5U8}3%y+bEr2)4ErI<_ zf)H0O=QA&ynz}ILFL^a~gxswM>qA2iVLOuQ+!Loq)`U5}3<3=7$Vcv!wH;O0qY2y^h7m7#yl*IR7WUAM#f(^3{zZSdLi#N>zz z+kuRlW9;`07>xDYjF}!YdpU6HJ!Io!Fw;tA5^AV^5XdC;?(38QPT^x$+1BjOP~fO* zcvSF!*{(s7acOGA1D<|S2~tHys4(l=ylc`&nwjf+iDXi1`0o}!483IjhQ zmy@ET%afTriC+SGIJ^$P3o#2~QYP$A1f-dPU>*cy2boD&dYc4|7jJdsmzB=SxO+rH_wCtx)6N)oO<*-kklwoFj?Q6yw+8Mx3uU%Av?ndD zp2qx5hCwcnK~9vJH7K1!tqXR@;OCV|%vZSQ#t_P(6>;RWrqF5TEXpy7HnuF3-mxcZu z5J`L{pp+P!vYKH|gCrLtJC_5q-h<9ZywckF?DH?`9|;VR*B4V}n;&pEfhYN{n252$ zVlxeX7NrI3Ru4?ia>WU~?ii+pQ$@j|MDBMELjP1A3Q8>0O7#`;O5`(fU?_MX_>6(S zjbYg=kI=0tC;d)KZSLqaIdD%&O|R7;U4KD;QP{ROOU%-k=NSkJ&Pn8c^~CcFE3-j^ zh?=4gkFizUdIO7O*0cp|DIqx}{%-2QYypSZd5T!?91xP=5nxFa_@!Wd)KO#qXGQW&G|B2MX3)s zSrWNb6qwu8IOja&TB5)?$-yr50N;@&5f4SdZ&Lz{#Y9~gMcf!gEW$L_Ef71@z@5;+ zUf{s)}M_17}h~nTEfUwnC`!;eQGLm=bx8-P2-V_Q}3$;E>QY z?O)gfw$&+p9~Q|fGq8V4;P+rKQOr*z-n}W*GXmfvn%e+TlUnMOzvUex-o(8@K?T>UQ@nY$e(}3Oj{y$t(rN%pk;Nj z(8Kp}fxSZgpH-6+jL%I;oZ`dWlg57_fw@hMdtHODnWC`L8J@A?;!L>fw_%^OD};zt=-E^ z`>tR6mQ7E#Ov@=cdgR^(3!i!0w_nO-U6;+hQ70v>xyoLF>B7Gjj;IU1LTxMs4_I^- z=uSxBesO>^=0M*aL$)~wydxTNq8->h7O^!o3=TNiC5o9bUeC)FImN)Kx4=%>(dEbkk#i5U4z7E*MyTU-f=Edt z&xIy-j)emA9NJhI1*RR~Q)%QokjgwSfiI$D>76C>F1=+GP&+BmcGAf-#PzZMAr~LH z{z}>RMr&5z&1>$He89~0fT8R_+m;1V=?xs;9ta#uI4`tNtgVnQZ-L9dmn@7j4t!xD zi8%?8!de_r2}^SoI3y0r-be^Mw0N?$WWfqW>(km7m-Kt=6^;3QKxolJnSyiq@jv6ks#Z6>=0mry%%c0Xs*dz!8QGSqlYrq;G^ zxE%SL8vNdPFy%A|u6i>`|K`%J4Qw)%Mwy!=>lliQehHsZ60v(Ia%%Ms$Cv$E7PBnfLDg-J-gk zDeM5-69@iJ44jJ+=3GA*aqFRgg#%YWWBv>#e?G_AR}X2uUN`$-rAUS&3+F=iJJZ(t zH;Qy@WLHoWxYk*mm=&cWy)=w6Cr5CW+AT9ap zj5B@R%UQlX;Hz@r_%UxSIB1>4+ice1enqAw# zcFrKzwsNWQ0@kPo=DJlo+JknURNi?vfp?KUuUO*Ba}SkX99&V&cUkA`uG=yKYj?lc zaoyPH-?81VZT3`7Rylv$n5AgO{T;ha8CV(aUVX;U_Dou|YbogD<~0uc&n^(2wn)b$ zlE=+)9V4T_u9-ZKs?KLTYusNxVZrh+Ik+IC4P1@1XeS2PPfd ztP%-w;R>hD2P9@Suw^vms7kViB~&q4@W1*jX4k00<|wnxMzZO^-Q8cK6W=L1c{cU# zI{P)zuEy`avG3V%2cK|;J-cWBJ77J%a;EC#e4lWm_FHS%z6xU}1HoqnuvfTD;_0z-;|fYV);d)HY!6xPc- z@^APl;AY3)k-%{*HTZbpd}Rea?q53Y4nEVK%-4S~Kh@ygsU_d0`S=tp_Po$xq;!Cd zqo2?2fu~k6=aB^WBMm$yDIBvD*i8}`x9~73EC{GcILRBtoRwge70uq$aN9J`>e4om z>OE4I9HpJ=H27!m`bY8_y_|A+L9mFTq~4}`n!ab>bANjJ_<3F4{foQzx;Iqs|82~% z@4@NYE5C02_Gj)x&x)e2hgfzvv{fl^pHs*Sc*teIC~`wfxGqt%WQ~Z^!YB3*FX`Lz z8^{WrIxy4m;M39rm#Po^d&<9n#eji*(;`kL2`;P4sm0!&I$zkiXLZjg@8+2GV*5p( zKb3bkJV~5XoTQT6xBoh$#{-#IW=5?8oG%jiV$zszo#T@`(9Ys0q?W)q&DpHMH2uSg zfJ5vG5nEmgw;bgX6E;dINY?6?h`oC0ZkJKVt(CziCpl(LOjDlHmu0uAuvurOMfrzk zH#fKI$M2v1C+XN7^Ud)-v(3L|IxJLVJ1?P~#38FS#Vv4=leTD z#Lg?_(bCAW@;ImQVx!VmQ!<6u#_!8eeK?_I;X=n|Cq^chHociPr7!0(PJMlVQK*)0 zrbgny238&mrkPzLog$0v$_#8e0{<}{Vm>Xb6mnsMrBe(0JX?E-zcWi%c$XW71e~ob zZoSBFmi)G}^iT`eHr0>`4NOk0(k^PTS_%h`D_iF5u(@&JS)hVgqa0_FSjr)0wmpF= z4vD*~`*?H?n9p47>^8y9Ei5HN_@u8=%!h2plat>}S=p0&tAyp}$HmgFM>~r7ygr|4 z=C{*V)wt}#eX2#+UhhjtqrdV3gC=ghBNG&ZTzU%*2l;BUhVsY0vSn!t7JiGGtKGBVP|glt zhQ?BnkPXqSA`=4GwX1C!4hi&ZP&io5^dg~4L7HQdkMRW_Lx(&ej|~n^^V23U%Lh0t zWahN`uz^Lc(L|Y(n|n)Pmuz&5lbhm_Gvd9qS5ywW%jdCZuaVx*B+=XO;lkoejd};K zJn%93D7@AuFm%JA7VR2`nQczFCmsfPWojpNsx4Z<7^>k{dXUZgMPR3}V1a_G5k+XQDJ&Ly#_3zT>zJ3cTRQ%qoR>QHFvn8>U!(ZTQl!@pMx z8X2Zct(;l9Vx#0i)_{d-x63{Fh`bad(~#M376bjp2zpY^B|AQzX>cVJR2I+XFTGx5@6w; zw}DY?Mv}mCi55;D$KEs(6=wc~4nCs|ERH)I1^P4CR2T})w%lAKHfcv2ry0X`n=Ohq z0SYJXYH;-S+(|EJKJ8(2NKxU4X0|H3*M!GXCTm&WDr;T8tkCv>K|P?sXM%{xpE$Nz+&yl!ix_z`R3!Ib zFJLoW^7BdBib)gJ{_PCVIB}EN&4E44;}8Se0S09)MKf6!CCTrF{|+m>n#_~s@UYov z!WEN}15D-<4)Gd_Bq~%Xw8`}x6x(LuV(j)|+qWHzHnv%g@+BO+g)ctLKF`6<&sx%# zBFfRx9W3zVn&jl!rP^9jC&Zq5EZNDEdLm6}m%xi}kxE)_4O+{bdt~J%wu|j}D01b( zQB9%7W5IVGiBu{aP4W@!3UE+Gt=f=JTupXQ~E}UklErRr_L5_-a7ZGzqzp00cIuz1|A*r} zM?TDc#BrQ&u^?Z@m#gcae0gZee`Lq8yDf9MKXIl`dC|nb<^ZekrU|WYG+mcjEOqjV zRC>CgXzICn*;=X>niM`QVAe1=q?FpzEx{*v(1W2{WTM3p$LC6W9vpny*L)&b$;FY) z(B)C@@(C`QYK~n&E>7&A@sN23_OdTvV@j41o@-E`W;W4n(z&NTIgh^2G|VhozVT!p zPs)a^w|~4e+q04ShuP^GcGd5YsZ}U`B71&r5QQg4zM`#IF+{_VO8NN>{5S{ z7}WH^S=ytpD}X0QAen*XBNy9r#|2K}oenN?(+oP)bry1_MK|%a1!$G6{K>?Y@HBlv z^~@U=I#Z3Cd(-@sS8#h2Ew_68GTq|l@`=2wf~6KPXwN;^!;z4_Z?2OAmw-Vd%Kj}8<>@wE*xNPKh8T%qD4;0@mR`}M!UQvZEfZ@ z=Xm>-k0>u|-{jM?;E-9~(LWIz6LXcOEI6`*X+m#Vs6zE-p3c*of(zFKEmKldSbjq5 zGxI^dJ^vg!9Di{0FPp#;$zIsqW#%GqY;BQ7i6rN(vYn?({~XagF`>g-<6)Jwfg``q zfkwuJ1)Tj1%!*=@uc-<=s=j#Fxcs_`)a$(RY}2j68u62#zo{!)v29~dUP*J)^%@42 zW<>_B4b4m}AJiE6JRF&X6c}e(8z@gaaey`AgRg9yF>^r05#6V4J|9Y~B(E1Ao~Yw9 zKcz)U>dB1_7RMA766F14Rm8qo`u8L$E|a*K;NG=)B~#=k=ig5L8wIwwFJFG4(AxFo zui|XG3CuPVltk1b6jW3kyKEE^1u9NQ#2P;Ovh*>lfRw6G?rV!S7ZE0&o(Ly#r9$p> zpGDkk1x+d}iT}7fOx|-A&p7ONHe}X?9p}$~zWwgcqx9z=n6@rI=i9}0V)lM}&xCA| z6Z4t>?QHgTXky|ma9CdFR_t*)<_9-Nll-TE=HGfQ_xJL!GKm(=>)}|mYT5}0?st#f zx*QEef;XSYJn7h1+_U(PqTc_G&2b7z!exbLKkfR{n$i5L{)My0!jIdYtz0`#Lp3@? zP-Mzvmrz9wHVJ0?0QTNQrVv$TuMI5O2N~5Cq)9KR^*<0L$ikY?z?vk$Dwo0JkznSz zfZ4Nwg=sr`l>&!z0sGkk?yC=Y8WR}Q9RYnp7R8W%UltoHhcm7;vF8(zS@S@hPeJX+Wly08MM4gYw+tC|8ko2n zn0_o3$ekbE_XM7$PfHdyc<-4actwFPg{4&KTf0&k-}420&mK5O zr6)OvxU_`kyu8v8m(Jp(z+x%D@~yqzqmk*$6Q-L+R!vID=Z^{hRnbsW($G*~&Q;)8 zxPaNOm{CTd(Q|{CSOTNCLATWgZ*fD8CIj&%5mxVjp#5K1{1w=$3piU9WG_FiD89%n zA>i$O!6qSrNo4_}vjFo3#ukSeYRetfSBl47pV6{zWAG+LPoo1>KZ_-I8y4|@U{u|} zZ1jLZbOLi}0DDCPyY>eLu7ZCdb0;xu3v8Qut?kECwp$Z;u0JvPv6T6w0q^lr#*+s) zDjC9-n#y}k4-&h^B(adUz1ZSOK!=oP(uV_lpC3$&K3@8MdB@8m9ZpAB{Q_8A3|O2M zSgaG6^)s2LX^6-t8Q)5Z{97U#qS7UpV6{wHL(qZAW&->1r7>$W*ft&JoT-XTu<0_(B@r|JYIwFY*V1FkI!oD&&1mmR9w zU@n!%ByDuL(6+Sq$>iX#&7$89OIj%x{VbNA`qBSM8ME03_6Z+2rhVXWQ()w}p!wp5 zrcrnsS3;YcyQ%05Moz|(Wd^(f1&k8^E^13pVJ!FH-8)H;^(aR`I^Waf6Qxg1Y`;;b ztvIPWVs_Vxj$#FNjhT~NF3g^|gIRAUv!NxkktMTX0kbiuo{yl_Qe&ohiB`5ZC$k$c z*%YuZ-M}KYfzdO8ZGHn+ilViZP`A|r_OFe$iwha&Z(`tDz`%529`l8HLK9k98<;nT zakm9`&jMyMf$85ks^312X=19r7}_g7-P7~L^!ZCAr!`7i zUbg3VV2ppj^K=6;FX zwQB+QwFTUpin(S^U=ed*5*JJ0RxtU=tj#50_x2@2Hx~oL0R}b&=JE@1?Grev1=y+@ znv@e5EhQG@9AjGiF!pXo;f>S<{0rrC{Qc}3UAmG5ry1?%B zOa@+qV#~*-+y|Jx&oIuF;0@To@;xxhbOY1H2|P>8e2P4HFBH^1+^E{KfOk0qE0gEq ziwk&P6Ik981&PL7I9U4mmvzC@zShvueefP{sdv5-lohY@;=>T)eDdtqIIR=sS z#s|H>wy@8qfKz3GovSG$Uju{22V;!_)ny+zdJ;I@HZUA=Z=3jG%}ECV z$&Cy>7kDlT@Lq3@pO(OtBOu2;QE>hO=j@88M-wKl-MM7_Nj{mMOXMy2PZ{OBRiFKe z%cbJNx{?R1J5RFiie!mB#k%+BjxSt0T^iO~?qE*IT4ua~`G40O^M3(Mrk>`03LJ48 z=B}8)Cvcl@`GF}x3pjf&D;JyxRdb;R{T0CAB#_SaK#XYBC(yzlBkH0{i?8 zf|3cErMs6%R`Qjdo+TrB@JqtFHwrnTYjhMOcQox5sxDwJPuNkiU`NcUr9HowPSW0) z%(XKyYv*+BIhF_3n>oyJQ($iqU|aHf!^;M)bOWyN0#+lTrm_p(?F(2c8wCGV6mzf& z*s0Av64=e)DZ^-ffn~A-N4o-tje&XH1y&V>lHv`F778po9xf=Jxp~9(^efh@GI*+| z2`>EjWO|*El-Mleu!lQc7#N>%ryFS1a0D>gACP$Exb4gY=IzflIi}f$Y-RYol%aKv zK+hk|Z3j>29$+-F;*d_!yzEf-gL0fZSc7(y zd^o81hPCJeYuv3Jt_xWERux<8Rv6x%$HTyI;ppzC-ONlZoQ$g&Efd)J7cj4JV$U<~D;A5ji&iUn z_OFV~YI^3HV_aOK>pvcA&}7urVh~`EX$?5OH-fz~fxWcB{-Y&hox+685uCk6(od6Mb`-q-+^;AGn1sZj;>Ldu0J{Q23?5@H&;=lYDzF&5h~!&|uNg zTpBrD=b2WH#t!zkzYZ2Tu*Wm7#|xZR$~oP4nl8T&iiI!i?XF}?V<1U2|RmaFG-hOVp>%9=+&jgKK!$Pv0LQsh&ymO%IZ+mfgRg=u55k7 z;#Pab?d?*R3s-LaT{_WXnaLfVB^SB^CvX}EDX(i#PD_{;I6-h>feF_HM$Q5gCIv>H ztszRU7ybk&i%x7WSyocm;K zKg+Rq_s3&18MoFgVw_XVEMUO&D$P-33NzCOh99a7+#3v~9TK=07>`QL`1NAZ;p5UG za~VW^FN(?@*iberPvGRgX-TurCJ0%HOmNG4a&OBMw|lq!;x0+Cbv$~-7qV|<^@W3{ z@2>0m&8qBkWoyi7XW6?+es^P~?#5+3>z7`ZV8s$+AiK{c9GTYf+(Yqo>A(f z&8t?eS~78S+NK3*LK@rK&;L}`5Ds9Fzfc`hz&N$R{G0&q*$X^I28`ctWG=VlJrlsv z5wmFOG_RFQ9-GHy?o5=neaBW~VS3Vm@A?(?|DjK+ZWLd-S6nq!@ZJ{ZlMQ^^$x%anqSzHrXG80&G1vr)_a2ppe zN+{?T%YE3fS>)nD|lKlZR^>|x-YxQ3Zcl1q1k_oh1e%lp2@_`j807=Ejd z?~w@Kiv{d;1y3s`JdF`JolZZuDiU!he9Z?gG1i z17mF&SR7)kbbvA)^we$(R3!u$$$61GZr~(x>tyEzCf1rgE!|I zJodB*wp{;uYy!{ge4cmb|4AnDRGI$2VfQV){%QS#gKgS}qP-5~{eQ}EaA}8FV^o*S zpP~y79F<(9q%@OSeCN5J{b#`Wq(^FN@N%I6$15!7QSsoxzk`=o1TXiR@QcAIggttx zRYAZ5?T}3&8=XoW#Ijn~1_Xu_7%)1u$O`aG5O|_~er3v8rPK-oMQ8coJT`|$r4`Fq zM1%_bJPZ_DnK-8gt7P;fH9bD9S&)_?d`e{T1YNTohF5(qDbHB$(am?Yu*AsiWS^>Y z+RGX5lDGFuGRwv6*i>+`tNrrhbpCnPC37xMbZ$Rg(s;E~$0VYlO0cm=C@lO&z#hR* ze*YYr(z;6( z#}0=t3T+}mTNK=6i(YUjwLCb;VJTPWCY5WpG)*ek>`>Zb|CJlnt^_Zt;OtVEH{oGt zu+qiE^^;X*IyADEY*;cg#qZ@Mkr_=J5;_AWoor_1G2u9)wR~CBw?@_#t4=iv?XFtj zl70LV*XnI%Ee8xG)H@Q^at1K0Zj!Xyk5}Kp*UADDCrO%KN%6{BCYkAB z^_5AZOv_bBL&Z~VlKLA1*S%gm6$uPtN*}+92rDVFZnY^9crry*Pr`wLrDe*4$n?j@ zUf$fXyoq6r5cdPGTkE%UajoWEDkQV&__nT%O~*|-7?~ws2`rH`H+taEer4O4ghtkq z1V?sRGma+KV@k#qn^~nkiy7(6pVH%9v_K*y(Ji8&Su&bsBQyW+GWSzGSO0ard7SLpZ~P&mnARCwk+6JrC6nq#ICgAyXD2Uyta@)rM?MYI209xoY-U+ z{7{+o-AZbjptZt74gRywQ)dGIo3@GJ{GJzvzMsOmW_TE!(NJL3=vXAZ zMX@hsLbKGZ1Q)h>6Qmq%6nS$poTcw_fAf+vIJjHXS@o3u3FbEr^5zLJ z&Ac>anekLjpVF)o|D3-~sd=V-|L2)-7Rx`4?2->0pLjU+I_e~F=5?@$>|i^!+Q)%` zOTZy4_GP=S%LB%wzyrK;j=bM@@=sS?*k(NG+W}P-VaAC&8N@#%u-k7?sK|>rV8%Rw zfkARs2w=Wb=sO`AMrYmfsYmjeUCiHGdxXECeH3s4DO!j_rg z+a}9a!02SqEOj^GINPSwW_OpvTQ3%{nZ#)KDd;#bOLg345}LrMR+`47d!m6a>OkYe zZ3i+}DzZFpc+f0wV$SZk;f3H80T$DYkj1$k&C)hp$0wFe=NE}~kr!i(d-SuD+h@W( zzMO<+$vX>JjARe8y9wME`gg*d&1|ctpYw+1zika1zjdDo<|rgc2lR8)Jvhi_rl2D? z(@CK6M6n7}eIKF*7l&=J|C%*-^krplO0jXRA=BqfgVc(t-mT z8>1NH-X!wHZDLV=wc)J9_p2M)T^(3^0(@C44y<>3x0+#{WuyC=1x!~O7SAbNNn?1#zk#j3I_ucQ|4nx5Azb!Wm|^8*um z@A0lz`eicr%O>;XpL^f4XSp=;2mCwGW_sxWdyGL-PdaM>*6)WK-hEi+w`=K+KKgf-k5 zC5${$3QQ^*t-4hVrvmg2hKm^-6yss!2$XQ*u{goVSuugfF(r>t%;UQ5aha~t&=fxY z1G!Vf1X>ohHVIygh|l`8fKlv%1Dgp$6|dOc7h)#A*mW74E-uUtG~J@Ws~d25sp;C6 ze&x3=R~xK%F1x_|ZCg2u;e`BNtp~3-ro{f!e%N&&MuVY=V}}%r{0|0>7=!NuXJoGz z|8L;!D(JN2DCG3AxFlq?y+yU)K_uIQXqFZO7RihQEQuV=&!-5mIcPAl7w>SEl?!0^ zbh#1hpWwt_vY|N>kUg`Ib<8`b$ zVXHMcqyjE8@@P1)CS78-5@C86tH`8S70{&LA|tfqK?CQ61{PJ7kD=ZI%@QXZ+KfwN zgjyMxvwm%`JEZ-P=|5-d-ii>F%3Ujrj^{AiNM1ar{B=cyX$GVGkDc?it=yEY+*dOy zFdSII-6WQAfGvhuBcsvOgPCi|ffbim*uN0A%=o7{eG}u%u11Lhqfp`9c@r5pCUgre znznAjG@)BuTniXBHd{+?oG7WmWdDI#?L)K2iDr!(417Nv`8G7X+1yYn-6%PM*`uOG z??7|fj#if!OsWFR4i+s=1}qsnSa~<7Ix4gVPiWD7(I}hIim6Ig5xG*}*PwDo8SQec(2(XdK#uZD!Z(TY~x04B8wjI3Q7{AadBM=&|w z5Y^yd)pL??D-e@f$s_VWX!h@xiYP`|4QG3WMk$6}N&j??vE($$e`s9to00WTqnr1x zCDpswN*p+I960B!d-aQX&22tKg(dtv2L;wJ@OB)On%KZGq5CXD_a4sfGLdfn!h=Fh z-9kSbq!pNKE*$(J%V z!FEWQ#i4^GeM75ifrXGlOXd!?td5rOL(SX_-R=tPK^iSV3@nl_7Brq(C1Jegkkux6 z<4q~Mw%dJi-hXO#ft4NOg=X6eO->pud@C6C-P|SjqESDiiA$ix!=TaW0Bb=3n`}fw z%7zvm2?k+_{ec3k!W$YGB^bK94T@&86)CWrzc?Dq(dt+r`cJO1iT%OahQgk6Hyd~w zn9LdUPFu1%CNzp>FiK@GN^Guxp;So8;F<>9vd=%nr;fO&ME!j@O)d zndP7m&@BIf*+ydPG^r-p6O6&T8zl-FBeV~mm6&t2=J?ekmr7L{m=YL7A{dn=n9LPg zR27&5FSN$YXf5Xy&%yaxQ7VEk( zrEDoZ8z`@Fz@7O(+LANuESFCDxScuDZ2yo+NrFl7M5D|M#)Z+15-S+QC9IEn3W|gr z6h3pQ?Pw5_#&rEwCR+~HkIJmEH(2jpxe;@rHR}Ugx+$|0-j=Q{YcUsYM^W>_mN2^&X{Mj6$l)W1c zW!B;Jq8JfYzmlcZ!p=gw%DHFc1)Pf%+Vlppz$ALL+2UJ z1s|+g^dj74J6PNp%y|RMV^XCYh1oMQnC-P?%^K$3i)@NlQI@K0l$?5T0k^|p9v{ip z>oRK_7caiPB-^Trmr3D(&l>3|)1;QD&1JkGa$Gn`u{G4a;!M!hEBz9ijW#wgh;%TP z3kDlWu=Jdbh(8d~dzCf&2iq5}mg&+BDQh<<-e}_0XjmPkD7Ay%t%60>#L%gN#i@bC zw3)>*p=tdo#t+*54m+AvC76^aY|on1;GY|{B=puyovWYsZVwa9kk?Mm+y&H^2JsZ{=wx(MeGY4o)?)Yc0 zM4&}XW0BgVNvs)-Wm210aOoLcVh}MnCbW!ElJ`Jr&6Y*UXXJ9voZfn7akiWC*XF4g znN+`Cm&q`>AQQ2%+fmE*;PMmFYb<>KR-9E%i`6lmZo|-W!-6%|qrKz~Mm6$IV7yYuAhv>~GOB@Bp^@iRb6Q7RyU_!)1m^lfoh%y`oZy`N-=j@oMPtwh z?xcS{cccxDxpD7Wu>0bHDR-6>o3Q$vk+a?9;k-+JM~iYUpI!x1-Q7loAJ-dr9mN}h zHm(-D?Ay(!nAC3T8!%T#=mJyRJGQI~ccT}u-t}Rx@N3W6z~=m*(Rv3nSAkmkBnBQ~ zb*`voO1oEWD_pkX#44ta$Ba8dHtcBlpv>&>f~l){8AAcX4uPx7b8p%EF{Dj8AH>?N zq`ijig6Cl=1~~&pnHl?57A*T2%B1t6={zTs;}0L+9SzEc42~;UR>v^#R(7!*=;HYx zDm$UYNrAHCm+_W(PPmYEER}ZRk{8z@GQxtbrnDxwLUb6oYovqWixDXBRgY ztG1>zw75sdB)K@a{bt9j;1T&Q=opcfM$ITkwyTz#lE#$3OT>v>17`7zwoI9bo-*hxw$OB2&RK-M_0J zTPK|?x>=N`ldHrKkjubmz!1X7;>f@vc41Dzrx2EghEJilu9-c#t`~nsFO)Nb;n1RG zEB7{Q^~6atFql1Qa=P)>c?UC3$4kWx&5P_Zk9~91coiCDGME$# z#4?<2eVgLXrI5t6Na3do!$h{L{b_squl4uJEe-BvoFTPDZ~_4A1UKKp@sDYvtvWc6Q>!$4BM5At~W8x*!A*>ZlmaezP9uQEy)=y z?h!5K4a^SaP4lgqmM-2Rb%OEtyck(KtCW&s8FL zDl;!f1N#ZBoT#(=e|^iE#Atb<;h(u!Lm+QM)-0~zL#wu$hc4avtx#^MN`3>I!MCiS zTTUsHf>ZDA?Eko>`|-Mivo};Q2xTy@No28I!E95};;@3HedXG@acVkI%UB~AH&mVy zu0AE#$@2AyZbs9=%@Y{TM>Gm1v&eF+ORLa`?r1AWs7(!MOO0Ujn(?Enp1GInOHY1f z@uDqDC!2J?`YYMzCV%6Om4UuYTG08P%ALy&&h9_xC!WNbU!b#=XKj3A_?}ko^DL3q z3nJNt-~DGV{Y)9O^jSDVkfq(JW;iwV42U~RSX@B_A6LY4r(RzH`uRWvFEU` zO6awWV6bN3@NUXdD45XT*u=&uX60~W1LHpiHeN-Bf(HzaOe{hwDh3)08(rB&Wdu4L zmT(J7^0U~i2xt#j;<0qn%tKN_1-C5&kv9Yno1oiMhCsp>X><9jnw+msZ8Lt(mykwf^1#&*blUQp%N|**t}x zudjhkFN`n)V=AY>YxB^%Yc7%J!95`eAc)@4BiLU#K!z?_tHv1S(Iq>y(OzDA( zZqk_>62+yfQdhPK#{Nrn5vX`j$etWi5Xh3^^k#x{s?nVSXCA$hXHKm06>g0gm0_j_ zL&I`Kf|!Lw3>w%hUtILj*ff)|l~1PN!V12!8l|HO%L9_Tbo)hw;j18(o5z3ox;&TyQXAVqs+Yz#zBmgn^T+l+C{h?V6P_8(R2!Yz`e_X3$vR z;Ld5PAjstRMbSf6n)n!cN;95u zwTNP6z~pb5{x1@{#hUFB4ltKZ`E-DRO`^b&&oi~)pv9~+p2|6MqKVDopw4GyYP zOTyb_p1B6MFi6%;5qSLb+=~XmsRb9MyKiQbm^7AeLst?II{naEm+7ZnE7tQGG4PCoXfd2a+Wd9E&I1deg2Op@=NDFi)2mk z75MaFs(*(fSHObuR+5<29AmY zT%t4ByG|^WG!{?4_IEOG_VFp3I@CiHRw}O zQLO)^6rj=2G@r$Qx!$Tbw5wfIYHb3Oum|I3qkztg8O?HEH^`|dbDr#u)JaYcY@0MA zh109XU$Cx0NdA|B-LaB`wJr|Af)x(?C2u_NwUl;Dd*QO2>w%`pHj%S#KT@{TJnS|# zJ#qMko#Iw8l}(w#Ypz`LQsm{=I~eRO;i7S8Lc0ftlF*U@=b$(Xj`bb7TzLkp$`urSnJ0Yh-dV##GE8X06hC`?OOQur^1 zMO>lVp#52C?LpL z%%90*(?yoB4STpFl%(FRIIjF&k$>*1kV%t{#7(@=)Td(mSkmXy4P_ZdJNqdoeLhaU z&-col+5DoyG`rQ8)3pQ`xm+AfwtnHc$dn<cWN8DFPss#k$B09m!WSA9 zBtEwAeo++a_YmJ~kTn0d0}BI_N2120gG`J!5~NBugm%ifM=g*z;b4+*OxdKMk>SBY zey)EUO)7sbSUH*~vvr-GzRziu)Pz@l<<(W|6pJ%YyxjCjw}Uj<&0b(DP-aQI`-wwxlqxCczaI|S_IcoOTrW-sMUny`6p?*ZeoTUp_oxenNg z@=oBZa9}h!z@*Wmn5-!s*YXRjrXCl2{1X+JYN8p~7Bn$7awafHoH!`J zzJP(1|K03L1~p}+i9L#cTH?f0CE5+T7#K>PnAkURFmim@!MJ1t@5vK~nf?g$Zf5E) zJeS0gEBb*s?ZpGeegPL%pF|IDmB*YX1&*5>QEZvdld(jOF}(l3N)S``S66+ZC-skV zlwB4%X!mdUr&)e*^>bVMPs_y@->958Wl6z~Gb;n8u1YPLCV4Kzfw`2+py8`ZBZp7b zC4q*9Og-OgyL}zl0z4in$Q*5Y8k$fkoY2VfS-~u)CFyXr1S4aFl0LJ=f+PGL6PUOJ z^w~pLwEQhtMfSaHD`j(JU~o9FspssX)U6pVl93*665>xp93N^n2^?YipwJ<*^2d!H zXO9IAyCy0aI4hi;am+$#VT0K$C8>!K2L)~%*uZ@CTVI-5z%1TF>f)Z}=bBoI6(8rs zx`=IBW`1?EolqTr?HkEG{ck>hJ@J~ae9q%NB@>#HAE~p1x?MGzzA{V4{Xx^ex&s&g zU)5p_@;iMYrSQPwMBjg`VF`1#x-mB8?fuDW_CUk9=WV7W+Yt_>2Mj*ZD#t}1UWk+p zVBpY*Wn>okw!A%;Q%ZTa(wR8U?yPnO5sxQIf{6?aQnu&Z1dCbMBsQ+kN!VajwDCA+ zfCHb(MIMh03-~HDcwJ{+V3XajPATx(+(ri>k^Z`prB|K`&r#1g;grU(taw}C0eME= ziVAai?(~X7#_KEF%=hfJ`1<+iqVIj(^Iu*tyPbdTOZca&vu&8^PXkyvJQV%uA~WUTkbmW zuW*!oae#s60o$4uVFPF53rDyz5A-Qs_c%B)i=#DE;u4e3_5BBLmN>@w$Ob*u?V0O4 z@9AAgQ=n1sL&K~q{O2w-@HEU>)OzPc?E#LS1Ft8}Nq322 zW$<14l=JOdIR%?u1ItsaHaCQ%=I768G7(`?xWTA!!hw+^Hk|Xor9zJOD^mATI@)<3 zu&c&IGL3;)7g@z=s#vR=s`QYKn zo1YYfo`_zWyVLn;X9DM^X3l*|Ty}H5K43hy(39uh_2UOSdG95p?K#Qu;}OS%2EJDd zULTa>I3qVN#bYsVqwJhR8YhljDLBAj(e=@jF}LPWD{GUGNu&G=2eyPOjHQWtG#VH) z9Jc;!lw)y{d(o)yg;C-}!&Q^>%x4@n$;=Q_VKokM(hO-*$Y|KuI;F0}B#y=X_zAV&qV7^xqrBX7@`l*BC4L^4tR;i`O z(|7VH+za;_tlAh_tlo#GB zFBJbhr&ECApx~Y9JXaiet}Li~$unn%koL_3d|w!5hq!6qW#D|H!2fPRSS`c65C#di z140bJ8Y-vMpEQbi@Lr2iHk)@ySENbEBsltL_xcUT7<3Lq>N@apH0c^Rubk2)u-1h^ z!h!Kl7h}dkfhlrlEDpXFnR}^pF7pv@d55R z;~R<>BAkpvn(UWyW&1FQ{9sJwJS6a=rBkHC@ehlq?Sgkay(OGJ2YQu!+$5At??qpo zIW=>--=`G2lzG}*OH1RHYYQz6R7x$M$uA?)D9F*s`=^0tS5WA`V-2Zi7_`qi@SpML z|G*&l#i8a{di6U7zBdh3_Z&DImPUj*M7TAueQ1zi5snqph>e?}_`yMV%_*0kPg&N_xDMVQ#7etA>-&oWnLxZgTjg2Sh!$`*7H9k7k+B z(`S-T0ABPY(Q7)c^6_IKwl= zfv4pun}*|3ffbEn8Y1(UgmoI_SPrtrG%%bvARzLDA;&@B#Q`>lYbpYVbf+9rWMNWx zO;n2P0p|0lv0Wo)-$veGLLH4$8GSD%@}~o6^L{(!gxNs8Zl)>d~zE zg(2CEK_!OSdPTFMPNS~H!Rxl(vK!UZjA9*J8#ZN341Ex|-6wOZ*Gradk!*d7KFwI9 zsg>@%u7RiIxa9@sSz-4|(w>)|(kzqG3Am&wf8bzw8Rz_44g7Zwgfp$;NqNe5Kwa3N zQLM&MyrNP3PNR5>qeO+y$-FneRkUi(sq;Ny;L~Z8NKihL_o^|$2v{~-r8t{SVb)_gq}b!gIf2P~inHYgX6+{q;jy7%nFshLFbLmivdVEb z{qlnEg14Xzlb{dNvNsKkE(aJzoQ!jvEn8M=o;f(7=$zuei9b9Y97?W6_>c*BR9Y%vKkgl~-!c-uCj~AE9d_xtP(3|Jl=LtZjauZWA7w3@SLbRC|=VjR>HWrt5N#LG1G5y zS`r$c?m9`lcq6RS&-a5-^Tk007bX!M{lzyNRV18@J(`U}n3WP774I~1ykT5_jmgC3 z5XZkQEf=<2xsbuY_vJue++p1ZOnNcSdh{Q{wXqb3zI;&F5)sH+q?E_1d}hwqZpBQ7{tjlF9nMe6!u`|U zJ>_~j`;_LPb1(fr9hBbFD0SfAJSN8%ZfmqpGQ4tgtI1@PZ8<1=Ku;mYNlArCf#V}T zbCcYk4R1af@NdtP`*JMjkDf@qQ*_iRlQ#$XIyf({U=KRda$e0@^M<3G0F#0XlinTB zIhgz+PI3_r;u42c?;KPVaJn#&ZT^g7%s1o&Lk^jmFl&c6DPCbx={ds0aZve3Bj*DK zz6l2;Z+-YzZR9A-@kGzMS*;OU}cZT+w+4|E0)m+3>td??qoa?*pH|PaICo>O8*h zogj0eqKuQG3X_rulM>5E6Z>K%kA(u4wS?lC6c`RE?r3D*>G-~EzRA~j-Mojasu=kS z8s%*cGRHLV?Qqm^anf)&$R^M&&2dP9n6<5;?1ve`(CIvUAAEwY1~L8zcU=x}w6HC^dGB-LE#?~s&d3~Cz~jK%x}tpo1DnW} zwDk@=Pp0!eeSGUZcp>p^K_jJ8=-t&xR zq&2X;a187}Y%!*2 zo5y5X58;H`qZ>F62}d~G_Xyz2xXqft!2Kkn=_l9PyK-)cQJfd;P9zCEE)|;}*$XsoO1&Fh5zgvb_7l)66)||0-P0U+wlVR#m?XV?H>~EcMTf zld=yStxlQO`ZqOvF4ui>NS9;R0{&eKnRorG|83sDZJ{Z1NH(W|^T=lfp9>boPTyZP z$XPJT2{g9$RIs>Ch<@q7y@4;S@nEwY16xF|EXyI*l15{nW@D3Pm72E?=W;YmV^Dg3 zP;tVcuUt+QA`Zegoa(HZr<#5V(dFHnpu{BLu-13J8*7Jm@e$S8ky$l@QdhmFFkN!H zaC)!6I(574e%Y@6(U$B*KiM8Q2F5ebNZdYSvgLtUKg)Xy4u!6JdTW#J`h%6#dOYtl z|A;sWo@jJ?-zc}lRB_GL`QOVIPT#0^=g^_oO~1FjKYahtg4<0C*mvn(Ii$jH=)>1g z6J3k`7AEB#jeIN4}PVyIdR%FnoKx+IgXiM z#-a3JBuhvZJA;#HL9S`cVYxq6!Vj8^KWJF$Fgos@cflursp-2snd;0_olF2IpJ+|rlkAjw8gR;7Th+8VK(Vt-aWawEdI>y#~~%3 zmT4XCu6@#tF3;cyutkyb&7i(j=KDP>iU#H!v2%&6GjQ1gEAb4 z*lswkxy@u^!7O;;V@a$l3&#P0JB{)t&ZZwWs$5|b%<*7(b5P}KSF zTiD%V=X;1SieG7zHE@&_ICydmgJ?VRo2yE>!!IwpQsOz8ZR)C!)!|DYo?0b( zUDwIWET_RJW5R65N{P8iBAi`(juYp_NPJLmYGLQ+iP=%`-LaLOFX;j27nT0GmX0N7 z1Wj1l=iAz=7e2IZENbNx(h}IXg?CoK@s=$y8UYWEs5(?LRL*Q%nAak$@=wf0pg`P- zk==TR>FHeUa+CD_Fbx5{nkHs;ArT9YggZN09&`7($15;emA<~Ft}-4meg>I{|R%~>em1Xxwz=eO~E-mL6k4PmQbBWBt31(-Cl=(Fl zbSx2Zyrqz6wu`6Y0Mnj-MGy2OzX-Int7q^iao9+le8|XSGC^^Jl9Is!)7`I%9N6rS zRHn2D8f-Yw!oekRpv_h-<`o~?a|y*JTe0MaoVFhd5*QdN650f;cfHu$ZJlG7%Jz>@ zq=I3WlekCUPL?^66wga3UgB%bzFe4< zwPxitshkyCr`^h3IrUi7$~?x$OtKai7V3UAi*RC>H8YrgO!CL3?)AH?biOc3zG_&) zZ~N)RVt#w2%9IXP9*zYMIAk);dfk$nv7kw|L*ij0!zmpF21b(^3)ywe_>OSfeA|)S zBI?uGBA_9csWyr6+oOd-b@LS%nT0(Xj!cocVY-4%vdcJc=QFidlOPY(sR_INjCuKk zeGShx{0na8(aKl+8qv@p^)sxu|LWT4s~s}MXMVm=p6K?gKYUt~=(dC2O-^DHIcBH6 zT9s9@aa&Z@#*JdPGk>1Dc8kw;&rxpfL#G;=c`Z&jtmj^kX4AZL)fR5%y;V<6@8C%} zvDm=s&5J#J=1(py?AGmgv9P5=dB%c74z&|A&IlQ3EM(CwGw^HR@G@A)qFQi4fl1m# zfstRGB`|!F#|I;ab`KTB^jSe_oJUxkW>vOK*{YOyL`X>^`CZ_`cDDGh@AMX|dFVae z?SH1Pqi%oe_lv3i(mzft%=q{7R^5q_Nq-q*<`)1+x(AMIPG~ z@6wMqIxKmYWOr(cKl9VyeHDk2CvZg^Gvbb5ylnM!LYoPT?T5|In(Map{=V8Tb4!8U za+_gaKwDAh!xJ2Czt~q7U+~KLD&XlE;M8ut?W#3m+*1q{{}jhs~*+GT$jczH)O36(5pQT0&j z<$vI)@=Jg-+T)Pqg&R%^K8;I)0}hMi8XT2TQD|_tWMH+dVK~hraConeL&uj(2bhc& zG=5l+z@%})fmML>q_5&7#pjV85oQnCf=ra_=Y2`}H^Hj!Pb#nS=f*Y*pPyF>zPy)W z=xKLn^EgobCb5xs%TpDX#?vtdOEoWboa$Jj>ccO$dB)8jZszx-!grK4OXvHhvDr@H z$dqbn72mMzQO=LU2a*)@?cW6EB>fB4V0U0(;1AGYWcbkN?y^`&NY&-Afv_gWpQC!J z2~LW08oASMHi=kBI52oQa75cQ2{g`dZD3k*-yKhBO{`cB+V;^2wkql~kbFIGS0!L8p(w z?767)Z4S>lN~#Ww9R}@+762?cJdaweTlFNzr?45rEbacEJs zImCC#=en$!1Cw~#0k)=umN-5G2IeUT_)nf_=3_HxP`z=Gjrl|O1S4xk8xv(I4;v-p z1s8ji&774k&S;ZgV0O9bfynYFx*bk@Z@2I)cwqB9kbn0a0nV%mi)E%<%h#}9Iq~ON z^#gO;+$>mP5;AO(ugvFTw>rY;5GC3q(9*!7qA=rh^n|NgffKK7*_7Cu_d-fW=9z){ zrH{M`_02c-P0Kc0RmdTnaG>={$j3)a*@wm7A8oICsWyR2^-|l*1U6%ZU{_H+6+H79|fmrl?jYWGED56B@3hF z6gPAH@nCMwXp>$b(3N80EL8cUyYhd=O!g}aB`Y(+BX-STmU)rDZEh&d{IXIPe|#}s)cA#IzEs$Tc`mA# zQn)#tytQg~Z(yH3^I_hS#p`08AG>~#FDI|U`No|G8#)9x=45>f)eq>}7Q2KuxANoB zxb4~pn5PMJdT|`$6g=RVW6Z#+-}H<6;ag5G7mMdTGkwwX6Fa}2POSZV?Y+|Fj#t6X zb4!+J*Z%P|ueaz6>WaGI^lHn$rCT;leVzI(>ToK1_09WIWr^i8ZX8|5rdf61LBxtr zdJ85jc+0fxZF^h}W4gw%cT5ai4$OBRRD4ii=VIXc;}E-O4bz_V&04NZ5e%$vo^Fv# zVE%PLWR{|&kE83RH7wH@gen-=xfY7-IlwX_VaKL3t#Xb6^B9CU778dZ{s;-^|K#!h z)q{yr$BfRb=JR>Dkk3)zR|Ds~22Q?2p$(04OA~>|gdI8Kw}8 zm&?6cSyLK#l^RzR={?ucz5Get?nt5(Q_hP+ic((!{v3KJ`Nx3&sQ(|o#J?pcH8&+W z+o{ahl-PW$YhBOV!}|`tR@%7k+xm6K-U*#tC%tN+*pvfI9tjMh=|y`2HmvdOc=Yb* zJ0<2FD;b`!^7uFky?Vh^;UJLa!k?m$=(gDX!rDZ!2kbow9Cd}qI~MSIDN0RxDB=H* zQ;AXFO@a`MB17i_fh`FPRSZIQg})*k1a_U4T&DP(t6B1ws*q3P`xgrAe-6#nb7q{k zKxi66r%EE{wxuF9GurgY&H^S%|!*gL&`Z_&T} z@6h9_@>mze&{6Mi$W;{M|LkoAsO{$JWk90 zZB$6hQ<$r!aNe2uUOMxP2ZA#iFWsAP>0R@FjYs>{HoZ8*uRQ&sWYA~HDUFiz8YO!g zC3iJVKeOuJp-1^MyEu>Ne0+T>Pj&A@bL*Zrml|dEEtF|kD^q8zYV9MOzmWThu1?CT zPjMU@zBH-lYrGTKkQjE6@spCy3vGVA1)LlQMgBN2Gbr5Fd%(}6$e!lFmZreo(!gGp zz;4#Y@y$VIzmcS0qP?o)d9Oqcw+221NB*<}+`o#sCEVHWCHXNi3jJ|lf2JcKmcW>B zCPU-!9ytZ(X9q+UCnxZ8NRf;cP*9p9FV%@qE=cG?4 z-k5Y*dczc%piMGu|BP=Kaflyru*#p*TIHe76QJ%nQNPM)SwvQxnYYfo1WvUNoMH)_ zmlpBfaTK(F{Vb-jsyHN99TLW z_`f~a{>6#k%T0jkp@5W$`L6@*JPCSQOZANzESnC9uqB@@abT-Rh`qLya{+_Wmx(;- zdOWFp3X9pY!Z_+4b#o{!G~T@a(ueHOV+Z#;y}i6>;ejLlrlG;6-CsO97fSRs3iUmd zu(}kzrfg$a)3|fF#oIu&WZ;1Duylr#Vu6~ z0)B1+2PO%uIQYY40gHzNi`xQrw+9Sf2~SuY+1iW*@(zk9EEG^VDCDzHXiiGP?8o0M z0)iJD5D5|3>D5-L@=(Nsk#%B+pvuZj?nBEC6-<(7;!)*v%l5NvS`bmX<@p=tKMrp% zv%HZ!_?+I5OkcsVVV{uNk>AZ*9=r!19f;WfJQ&1^zn;%xnrPTFMMP4t)9ZfK%jf zn*2g;j|3J5t}UAmN>)A4J+O##5@V;2OJzyqU$Z1Z z=$QVQxB5Y%#$C*oVh-%n9C+?HEUakYo#ViM=K${vbB`a@UN+}toW*Pl_Rhyx= z?~!))%!Ap}xLzE4_y~|q`Mk~R z_`8_i%duO(#C|P%!)IZ+|Ddy;2K&1a!NiULzzcR;lRe_28~S0cXxqi4wh z29|{aY06Aun(bnlOcCrS^cK%Om(Ac%CcvSnqx0`O6JMi%4PV;=H=~7au8~`^uJ^5v z;W)bD*}@qMvrQiHapqjnF{)c?TK6`h?&WleH|F)?ihS$%%zJhJFZuF%&z8enH@%C` zNq^hsEq*gUFeq;7XXy#PH(3f!-&hx~&(Px~n6UHYPBDK*8wveo=XsSJ1)LoDZBB{! zIr6_r@c+Xgw5!21EsbA|QIH|ARJD=4g@ZZi0z=btwkIo7zb{~~YT#gETImBN|V;nJUb z>Zh_A7_;jM*|BbgK)^#41Np|*5cM>(UPq9mK6Zpnk-*@>0E4yZVhj|};lvRxWJlUZ%HBs`%&JB;v`zI=H{JEq5zQT2toi{Wt=1W#A zWWAohSENzZYBTHiSXrwZDw5J_jO-49R{9A=D<&6;Y+#U`F0!`wpZ=U=r(|Uhy#>5Z z2Y08b2;EUw^rb;y19!+@hCP242(T=?n&Xln;=r8rKuB((z@Y@TiUt9-h3o|fxCI>f zs~T83ZtyMm#pmYEJnwbC!~!M_hM;Y~zRlxa&92F;@>Zf$f%#q%=azOLM}-o#UP{_7hX*rO#o4$c>rmK0kk$RZW- zYyJO#pf>^*6CamO6t$cvDRo4&a*^au^PP&<(<1JD+S-0%OQBO0i|GxM?93U3()opu54d=@?aL(Sqz1w4A zqNLiKL%MtV*>`O;zN4xxHSxZvYH{AeosO~wx33!BKJvTlhQkT5v`v%Tf2&=*(j@)9 zUFn;yn}mQ{gdR_WYOiGH9dI?#|vFs`b#d`GkKVRV2zndd( z^WGfjJKL0xB2JAi{EGZ)$166Ut=OIE#QK1VCBaElfuT>9@lEY9p=HOV{ii8A zFz?$dCa{p5ukp>cV>hxbSfW~P_UlU*2dh<& zv&Ei!71|9fDd{Xv)-r`Ph<ge0X@aC!%nk#_lvAAiMhi+_79BcicGAi4 z5~J!(1!48@4JRffGIM*LS!8Qn6>`&P>9HQ&Y_3qrjVn(~Q{3GpdQ)j_x7g92zu5Yu zOz-Vks{Q=!?VCkUul%blzP;^D0|y&Axa2en9ylCiV%?I>?4dAG zv4w+=Ys!I*%L5j=M7haSeBhAMb5T8ZV^`_&BXd31+(=+ZIW5K~+@5DD9e9wDMJQ@d ziQ=Wj4lP$6X@{-gP<+hVqio8-P|5hjomD5?tfcaix3JHKL#%lQLM;kbPbQwxTxRnv zxJBASs5kJG&ElS5f77XoA;$zhD~3kKUJVT|toS-LBI^Kyb7aoi*6ZdKh0NEZ4CZYW zOSC-ZvAFO4j-}TV>Ze}yHM{iY-$jW4bNEU@H;8z_PMd^B@af%9jJRE3z5_v)7)y za-aEcs)BHUWXOg~%sLwi7`de%u&|2Q-#m4QOW01h*XUHf#myfT7_J5jv(SylFM0!^Gtjshfj(-XYj?BUi0RcRXHqSNmCf76`WQ_JPI;pBu zVaTZzGbO-7cWXhS!|APG0+^JpS_r>lwo$0^R@#Ack;^(;^1Q8+IyLGA zIibW;x3D$b(*EC#^61*b9q)F&5qp+o^<>H6cN6X%d41OGW`ZyWXOH8v8Ob4e2?yEy zlh`gN|BQ67dcoW`MQ|q1aW-kD;zP$8n0Zw$sJXJ;W#By7F<}yiLj05h2EJB?CRg^T z7^5~Z_B(+mBwRW+xPMWN5O7m$%Lq8g82={FTkqfWD?z7pEmo*IvKsDaU|P3q?x}$6 z9eTFU8g!ScFnVlT#v;Nc{35>I)4_4aR#6`ZClv)3!x;y+Z535+l+RMw;^wke=;B-L zTT#yUBEC$M7j%jF*q{bWV)W6jF`2eV-{HU{!yF8Z(nl)U~dujFSIc4vCzOP+;y1>`4zyjk@A= ztW(XyJU8}3)2k$z-FYG6r3`wqua&09CLIw!bb&*^^Yzlleh$t}fZ$_^;|&R0pR{>}$|X)UsA$~9 z$2>#4w?N;(XrZ5T$KnH5J{9ywEBdid?#<%aY$)?UhJD_wmKARQOlLB0U@)E-Hi4Vr zjf%`&MFG)m4Z)lVDv}X5W}RVaV9?QJSAM{#J;QNRO_-qI1cjD&&#c0=&WQ?i2)ODB zO!S|w_=xWx53_9R=62mVTA8LB8hMf?G&;DoOyHhzLV@$A>T=a073N=){xU6*k#%%R+)QcK<4(b1tqrdFFOHdATe|MZL}yw1w}B6s z+!sgRV~y?H%>B5X zms*z97!)-!>L_UNKbX!GHd$*SuS16UjKktv8kiMB9!XuFz&5WSkbBZg(SI94ue>gJ z#SyV{k+9&T1lK0*gIqxml%(IxVBs%lXyiJxfXzswkS9e|+U%g(nx-{5F03;nUHqn~ za!&{fojCJuO4l`}m%b;tJWhFfZ!p=WTdEoEXX%*xP%PizM((}14^5S+&aoday^Nj{ zJ$0caXq)bt}ajIs)hoPHZ0iR~z8GtO!3 zV7>lP;HOIIN?}88--j!uSpJoi%`seQ`=x=y>;|Lwjswl=B93fHTq?q<0@qIMnla&@ z1Y__<^V5MTYKmMz2F#Kt3^Gh=h8(}^h=nG-h`DGA@)mM41V?c{8) z=Bxx|>039B<*$}e7H_OPBm3t;cS(rY3Cne78fWHNttsrbR#)7!V#4N%!`9FJFU;I? z{7J(U29x={KE;p3FC1L(c0#+|4Hf?W4bJjR2bq~?9N?e+;vkQYUz1#hBe!jUllZZM zPR|>QL_%d+HC!Ifd}o_3wzEJ|Gpn&X>&_X;`rLWqdVw9@6^$I$0#5vG0&NdMl!S^U z9I{VLkv0yU<|DjPXG3~T&JVr@X0-`Uo6iQY887%NociO4xqMP>1plEu>3 zF1U&->&bJ9`oQ-l5c0*hpt$GR>_f8{PFL-nFrBy4MZStr?I7cV@0`sJ=_Sj` zvYw~S6kt^=i0c>R+!nw&#euUry|H%#heSm5HdC?e3yn?e%*IQj_pqk6lvlL3^ImV@ zy}hCG^(KppN6r8B9QICDtF$fh-u}g+zEP~bFzsI&@0AHb6Lz`HMjcWHao zOa<0wS3_b-8QC_*$*}r5HmHg|XPYR%HBo@m-+;O7L*h0EzH<#6hZJSXLOC~-8CNMV zsVZ=8E8yDcP%Bl&z3YKSX+^@^Z`Qdl*tZ*SHZX8pEasg4fz`%=<+icR#pEDy1qLPs zkD^UUZ;SAp^?l>L&LHcf2H72c;8DiL!cvlo~l|1A0{Sl(7D9J9U@WEBc>H&Mx1GeS~>^&FQD+M_7 z8kma(+O9v~S>nJoS%K>ZN6F2AgeoIui2{!H1dfFU6Bag1SR%k>uFkbEfn!M+w*`Yu zR08+y2C;8HWVRJ>v?>&zo4~uZfITLGvu=Z+&y;$B%}EE$7=KQ$KT@Wp^N~U9L1)^5 zmgV0$dkQ8uf8fjuNY6`SlbOJ%yMb}y1fHV{I3!kd6(uWec+n`!!CYY+tP~O4n%=GV zATzwOVvU*Q%?Z4B1!A6Vu=u5xf6=w4zq$GOBlFCK;#;QhO$+a3O)prnfctep@2w4$ z*M8*hYT%mrfOB3zoWoCjHUUP>e~hgcjxc!(a5gRIt9-x~FoE6Kfyudm+1Maqy8~B8 z0!Ldzzm!_ZY$HyK1cnI)6Zj)1FlWwMY{}8Cz+s#$qmj(5Rl&W{lZ&5|W03;ys{?%7 z1=!3#B>vhuNwYzkN5LdUJ*R~wXGOEs$E_^3l2zuJ!sY3l(++SfH{eKn&YAapvV$b& z*#yq61yhO+v)>Tv-ciaDDA}|%qg#IhQ@}K)a~CpiEw6Pto^|cOR9%Y@=TE8i6UBf2 z6rVX=EIqCF$pXE*3#Ql17STEz~ahTiw!2scgSL%$+5)2n0*7grZMlO z0`9E^TuT~?w^q)6^MI#SfMZfwO-~2ImSnY+f;k7o)MMEL*D4B$EohkcV$Q~+948z& z%iAYk)r?yqzF)Uirv7IyI_^{M+Vjl=^L+i*xZ(8P+%|$;JmHC zyVYQ}c{$fs1>UM6_V^9VDh5SM3T7=a<7f_G>ol0Ka^=?L8~pcgWN&}KbLD}?j~O%D z6}Wax;CmCmyY<83MGZ5P5*R+Q=I|;D1>IU(s3e#uD#TR4c=q45wD|$Tv0j`J8`rh} zUdQS&dBum2yMY{;zu11>TA%rWEi++f)&$+cDweE<4dPFjU!I&-zH6R-0@s=c8|Ji4 ztv!_$rJ4Hfn3LG+jan+rzsu6nE_zMmN>|&>_wND!n+=t>4mftc+H`Hl^zaKUHf4fh z8=UP+1O*jlNCeh1O_1VeU|z;O6!e3GKFhJGlOE^QG47VDUXjo?-zH-B ztm%BLzqr+3^S^bNe(eVD%@Yf_e{nzlYH4E@rE3^}RYZ;D!;)*OUHCS*7=GY*Qoy_I z!UQKTu1#I6Nd@fX8`u*S{3ka|VD{vCAl}}m#1(b=c!NV-!~qts1SYQwvS+I%Y`Zz_ zh5*<03EYbtIMNH!6$7`goyZ`pAe3XYR7XvE*+IsX3zP2}@Sl9JPIv3NH@gn5?U~#- zn^Q)Cv-I^yE&1Jzfi>cfGRQwbR+?bi%G}2Y6Pz&b-RN z`|0GrUAI5*uIoP1`Y^`IV)y^bw3`PUm49;<-{cGVlmCE$kMGuk>npgU|Mc>Sw@NJ7 z!`LCT;?e=ZiBg9n8F)6hFdSge3*gAnS?yF+>{P|oF=5S8fhAlsIoK?@mN+cfeqsN; z(_5J(vzRS8Copu(ik!gKIpN9zzE=snZ40_0tyMh zj+SUn-}iv~WkbvLe-*tIXV3X+NdCCE`Pq&@p5pU-3JfWCe7!;#>kb_IDZOvv-kCo* z_VIO2;FjfDE--u71)l8!CzfY&s2Ja@ikYx<=Y(AsroAlSU|YuBHi084K#GsmFz=9y z*0lPJOH#TU8Lz(Ve0_ocO~Ap|T|5#y%B9b6oNVB#^}H-Fhi96=&h2wf>zp~X`_CbF zhQo7zGtG~_y10A!>z(tOBRKjQIHx%nok`fm&bINE`8DS=P6sSj>`IyT_K|pe;HR&It<}I2IPn_{N4cEU}XHT$co>=m7^~{rx+8h=yzrd5dns=cB?+pds z&9m4gF0ADfux?b5u6liGi=Z^y0tUy6X%DyVekH(r_7=PB9Z~5y?9C1LPxcmwDR30a)c^mljb4LbS^0(Leh#{9ineQWuKZQaSWys2k- z-#@!ozi{K}3O!qnP1jaTzv!`Yt^v>fhU($h?6S#IhH0qjAwPz-;%Qw9Ed3XAP z6O2_2PYV*R6ntPSJ#a;L4x4}NU1OiAw{y>EE?`>po2lXfyNY?oBiXqU(`3&mJTJMo z+^L3l-_qxn95DwtKDb`3$VwEuxubV+`GU58++BJ+hc565{C^RBrsZks!p|F}x%iH) zO=93$tj4;*rQ4T5pn?5@!>htG6U?7K{H^h5+TPY^4bFB3PPw{moBvH<;atr2@)oz` z<0GZqZyfmc)^Ri}U_7YzW^3-7jRzSV{hrKVth#fR^YsGmISpLh>$o{i_V(L-oIBy| z)1~Z7A8?c~yen|{wugs(`s*u2Rd?rlGpFWVU3B}drUQG)2lkQ;&vO5*JG^ECkL{V| zx3<66y7B(qwC5k0EG4SbV#0PSoaVjjz`ML)F8eFaDZjWU|No#K{d2;(7KfKYi>_|= z4fb6zQOka}8tVcz;mcAb2E4Z}@JQ+{;gIC2T60|XourN)=fn>jl?)s$3LK%cXSF*l z)_J$uIBJ6VD((ZT`Q*;>dYOJvUcgYKG-r*o^zn5`>xz$Y7c-`&v$1SHy29Y+jCzH0QSVdZ%K(e-99|MrhA6%_s-VzX9fNLi|oIbtbg`E_ntM6 zz?vBwH-%of>6meKj_2wy-rJ_RcQ@re4!bD5y<1{$rSkN5x6S*Wp33#Po||lzxo}Cv zzf(_EPWBGppYlsc?acJ`@%qM|4>u?tZekSUv6=ATfny`P=VUI6RSO!J83Q;u7#=t{ zUS%&@mU+3fD`ds`CHvZpM6!Z3wzTm{c6~Xqc7gN68*W@%zg<}FT>16Y(e4)ku|a2d ztqfN0ookxeA;74#@qi-}D;I}G1jEmdPg(c`GSXH#9BAbUJhSFZOIMis%>5bbw*BOu zs%|`gQN)1>>6cWk3ho_v$g6%f-_*TnHKUNs2|)>ioF5I2kB=6M>jy2FV0U*{iB1UJL<@a!o}P3l~33HIhI?nEqj%qDyQa> z6HgBK3CXCL{+k(7c~V$&(t`!9+;STd8WspmP|#d0S8!(4i3LRr3~WlPxf)l~j1uo4q2F>E&9! z`_ykU! zVcE3hRV3>gEr-Ul5&{kX7#UVAy2dEFQt`Ot`U_Sm0uBt3-FjP#dY1GnTuEpatl+F{ zndr6B>o|b(>lYM$tmY}w{L&<}nMj`WN9!55y6Fy02-Fj9v7>Pe{ zXlJ_orDZaUWX^`AK6WQnI(gYD9`t!~Oyz`#DEpiL&rZ(+8O!vq7zP7M}C zRrjq*f-SvbArX(ao82jB<~3*8_|IHl`6HL=M8_i%&OHvlM4wL6xVn*5Jf>vl(`n8r z5(m7`E^=jP6yA9COg_V+-ZMUY7uFQ#K5I_hm!i4g!GorSFIF7kx2t46$1%HEc-;l( z$;MyUB*bd3q%R1b;M}MGAmcyxqW?db_jn(!J~AnZL+iz~6?0drEo5NiaM-?xzs{`5 zbWQ46>6<@}-6+`D82fgW!T9jhIC?=P#Cv)|wK$V9x_q7ddrhfuhbvc@(l?*%fetE55Y0${&wSh(b z?jos{ldUEj0(oO@FbP>jFw77-&~QKXb$Rl?Ke3%^hZ(XXRTKi=yLx|&yi}wEwkAz&kam@5c+ku{x#|;WQL$$>_T!l9@u;m}LSli6|M zj=_@#MiGS>Y#I#5CIp>fuL&q%;5cB^ee=-az{4W?O05!`9Jm}h7zIiMU1Hf1cbfeO z;GOrmx&Kpwp31Ud{k9d$_$m%C3SW60e?{Spu3l5G1X~k3(*q$1MYo2t6F!#r3u>fX z>)7*Tspmq0sLn>cAKT{ZEMO3vz^pBGu-)pgMyQHz&kds=0zm zWDZxWhKng@-kk`MdtwJ)7BVnLA9dg~NND7Uc)%d>U;*QqsD@ptt7ouTI4~+)xVG(6 zgXGf%ZMuGiOI@xU5=g$WjPL6q4s(`3-mZpl6}R#ZCl#moYRguMBLN(GGa3Z4IFIVC zImlOP4 zHFtWY($q8@+ol9`dZ~DxRiEo-YQ*Z7F#q?p>vsBEO04hvbpEIK#(dtVImgB3hxD85 zIUT$z^6ixEMx56UwN32*aa>9xv%_S?$9pL%kN57Hu)b#HMV>4RX7TOW$KGBD?v9$W zNMy+bmdk3694-no1X?s)6u&HJw{?l&FWJE?%T>4|^Nx~E%7)Wop9~h)DNYou6?kJI z`ccf0hpF`EsqVdO7bBcKlDJi8xWs9B`wB`B~=zwPA- z&pwGs{XUY$u?r&dFCPrej9kN-lE5TY)WGs@LyxR$@ANsSM;}qSRSoy*czv&7oz`Z zuKxR^-JbW(F^f;etBZP=3%7qbrFcT1?YL8({;w{UgzF1-%Fo*IddcO}3V!Jv4jqpK zA~YIzo>Fo>H05#dRE-ZR3WXegWnS;+E^4`_dGY&xcDoO~3s@eq=5uohK`NSJgU-kUtf0Gs3llby0RB|{?6qtD{ z4lJo}k@+)o>LpjBzQ|l>*&7!&Xauoy?9*Dv{%n_b+`gObXButa)ONV2e7ID1d8LFv zMt&0~gW|uAO%|+GN$kZPZC(p@2Tfq{v~c)j;lN&VAj#rD(t-J28f@G#Y&edDF>*{Wd^w?sSc`}d+P--EsA z0-OGmE&6XbigrvbTG639Wy{iJ)rH!J!*BAh|Jkv`*{bLOTZ{$k>TaeJH<^S57;{fe zcUi&hkuaSlfbr~4Uh{+&j|bg*9To*BG@lD%67FDf-_x48z_d)Foq0xqwLmavP>9FyHu!ZrkrxqM7U~r6_;wrqxH96v7(gKzw3pOtYw&r3M37;Kz zAN0Pn>}#KF-np93y_4@%#FTDfCPf!po9uPpPV#y+YWi;4`)A|Io0V;of6xD5v@~?( zq0oc;c`y2ZKH%t4U@uc)|9xUs*vtMeUhH`W`$}p=|J>N8^JUgsYt^C!rk|`?x145D zdC_S1guyUENFbQozJW>U5%VFZZLTW2cN^^9qtL7oF;{K_gYboArzWu#D>(SBVF{kV z5+tyJ*@unYgq8h^2>Tp|Kbo%GCTwAAPKNI}8DYW}UgMfv!4~avRK$iYKBvX4fW=+H z_;hu1;+&c9Ir$z|ZGY^w?%#`uB@Mz%asu0B6`GVe4)r`{n)<29bE4j&mwihbS!SKm zyK{Tpibl4ek9&ip)_+v8E2*($iE@4yyrA2`UH=Tn#0PApFWRc^bWD7}@%}d3gctjY z6xd5Iv{~0o4Bo)HvBPJ7hS2Vg7MmB$ZXRaaBf3k1xDzgO#{@H7G-2enXl&VHmKnfa z*1%qvz&dqJ3*+n-mKNV>CanIPM>)^<-ksu^9MLw5hkfoI&xdPH3Y#2EGGI%(;hMC8 zB~EA0q8OHz)iz6I-0lmBT+HtKlz7Z_=BZB~84Hw~*0i{-<#E!zu_LRc`NgJVix=8D zzMkS+!`A4r<^Rk>+eM@oXN!hsDooh$&pS-(aPgn>T?YI54xBe$vM)l>=Z}F;VUJe0BgENcfbnPOpW%dGVQ`sSQxEY zZ?5t6UU8QFi^%jZQ=YmU6;WZEwZ!w$9nU#FY{D#;qJ56W+ORDwIk%YSnEi}%_kQWl z))bLha>})_QTjup?2aHEQFA2$|DM|x>;D8PyPV(nr_ZOb+ixaISO8mEL~zRu&CCGH z;9YD*6(+A{^OtYw_`QQa%v^^_7f*eJ!07!yud1} zYGR3?_?{IqTNZQ&L>%$D+I>`rHETm#_5-$@18kfA6`18Hu%D4)Pg6L_l5_QD%1QPd zPbXzIr*2R09T#P;2FiuD#iqK-{XO|aC&(+qZ&41*i!FOvx|!9U@}7Ua;$m~-${Ic` z-E}e>7BT$(j21M8{7CPA1b38{IR9kMy(po-tn17PDh@FvpRjTHTv|bi3+=?&{!{4O&!^ zX;!w%uxwYWq-J2|54MVg_Tq*%W*GxUZPrOLH<@D&u&c0z-LUpw*y6S0V&YxbCu>}1 zuklp;ds0UBTAYrJ`vjX7Za2G#u(YjyPJt?&i-Rhxm&lxG{Fl8qNRh)_@yuqW13UCZ zgI^tT^IgR9Nn_{RuD1L=Y!*{ZtxT=-Y@(kZzW&_#@ZXlHHeda1Y!|RxJ)f^}xWblw z#bRyV&1MoWZ%c3nZsCX#I32R7BBDgG`v7kYpK6QhPww>BH?Eqr7bUbN6@*UN(i&mw zAhot7`GrWBLfgLL$P1=HmrR4Ax46c+o=XVzd-mpLQmtdl@r}dAgXIRAnwj76NCH^yb zZT2OfyX4At$@cCnU%#tQBNw`~T`xVD(BgP+(cPJhOB{Z!c=)T)=~JWR3`UtBu@7bA zR@&a{RyMzK+1RRKzSl(Nw;!4vcF*TZX`57i=ryamMXjmU7XLL9j~Dx_ueNpmwSfQg zs{ZM>JB&=ue5l^V7Ih)pf+Lf`Dr*H>_664WP01G1+HS3Ud~2ty%{taqzXNs`g}9%+ zf1+mMRsmMFIN!1r>=^}TC#QN&J#r_h;VPSoiQlQ+o)4nD^(>B5x9k@_CGspqQ0`&! zKaOjco<@p>C5rLga_wji>1$p%*YAGDH3zG{IgImV*zQT*Xq4R1D0|_l_SLWtHcbW2 z_ws6IZdfDo^W>Cob6WmvWW5{A*0Lk+P3_iY!qSnMODkRX^;GQq`QTCYnY3Py1z)~O zEm?dy$Y^SZ0?&RQ_RB>(1CmzPWV~nedTH$(c?U@}@xxyDkCaG=H zyc-;@DR%o``Q3YLmyY*%IEQy;8JMllDO+lM?L4cf^zzw<_I){R{H3i_t!HNUzWq;3 zN(0h2yJbJN&Mtl7ROYw7N2BxKgf@Sj{(=pfB0g-v8(3TXGNzi`3^?<^bD#Y73KowA zEr+kA9Bke*`CG>EcNzY5Z<1nJl5-!nc&FY>y=3mue75doL|f*g6AvYKyp-9;7<(@> z;a=kvwR?r9PwAzuRQ5S^>v5F#0f&-Uwipif76x~%x-D6D3rgPb=lZ#KFL=eIq~-e7 zu82(_Ja|=sV0SRXhNu&*Q43^WKTEIsruQ~%mG+^woDc7QOxV&jBm6)}dZq+Fx6X?c zjkeIWEc@=g-j~KY&BP#*O(v&%-~1zCIY-^t`obPwH@$Ye?9IHluC5>Q7cI#-JuNmY zE%nw?zuT&O9QI8O3=E1tSvZUs7#MUI7#J8N7(*D_d3kwxxLAdGdBk}+_=I_dh54j~ zc_k#-`S^tRc_oDSge3Wd`GkadCB#HTM5IJTM5RQeB}GL=C8flqM5UxBmwNzv@RV7R{q)hY$Wn`75 zWYwgkt(0Yz)a2AO^?f&>GeWJ(qAc?Kbjs395^`iLEZy`>J*4DZTx^{^Y~6hA zJgr^AY#n`Vy%SxXoxEL~e7sz|T)l%_or8UxLVUa;y*;CYeIi0UJUv1~eM6(eLVN>5 z(jq)lqC?W7A_~HMvXf&nQ^ISqqs!96D~h8FGQ7eg(tN_QGh$PsQYxboGqY1uOR`hS zGt+B|bE-3P+S4REBlS8mET)%PPRQ0ga&->eE}u1R z<&^%_i@Uch>t3^D`j#bAwyv78Wy$QVYi93SH*4pr>3cTL-MMMTjxC*&rY~JOZ_%nH zOSi6Eyl3;`?OS%PTeD^6vaQFqOgge_)q`DaXOC@serNusoyX^{J#=vI;cZ8*?m6@L z`qkT~PhPlu^ZBEDk8eNu{Nm}W=dZuL{`~jvzklDp|Ni{`!^aPo_8o5H_{Z@6jbO#{ z1qYiG`7U|BczW)THLrZtN1t3>05dd6P4t&*XuZfuO%b!6`4)V{mBi*}v3dD_)aP|uY~T+!)B$ArW7 zIX5~UGO>1XiKuv#85BG?Xq#w|omQcqC}AbQZ`RUb$m7~0rG3kD=dZ=bCQ1jF{ZzWm zojYlJ{^}Y@V>R))MWxqjgk&^tY{+Z>By+39d-A-ftv2&_xoUsfp0_zWk*Rd`^>=w| z|7Aa&6~Zm9>(|rw_NB{<$(t_~u6vq!Z%_U=^UmUh+Q;Ahv#kEe+Adtcz!s6#qn615$B%A3n(seDT|VGt@72&M3&W2wrIDgjMMGRN*h9k~?PXOB zxinXOs&I_MRF(3eCf2FbZ!LUztx`u-JZxg&L6_^X|IS{VE)(=f-aym%m~MUl-tzGKOvL+Yw;47|5Zw37B(vFO&bRn|}A!$ujdw=w5%2bKrA{9VSGKc(Qvmd=vf2ba_tR{W5d zZ?*4-(a9qreyN5_@A)4*v|l73{MVH?0q<)=H-*3SJpS}q$|w2ro0R2LG(C%=EQOyh zS=91I#F0Dxg_!uff+@?6W_mas65H`^)BJy5p8C~qymj@)R%Yod4`MYsdakPon4N#{ zIw#%OW2w|8MX7(QtFCb$7C*wUuW9XzynkNIhs9bXET+5oWNyFN^x&AdaPG|~7EuQ# zzOCA{<9p=M^lO=2p%YYPBTL^|et3Lty^qp_t;hJKbRMdCIPh6R?a5qAQ5iasfy1|UQoRJ`Z|v~llkV*B~^3&y_(|wZrS!0UPCqi8G=89LWP&E z%!)p>uyb?U5|zKH+3E6^Po5Bn@R|SS(~M(2ewW*xs_gv3;_Go(Z1RyCEIKLzi}(Ku zow(^GOHW&R-)d>8OHIoz=BVZ>d%g5zd3z+-ns?J|My)0bR^=8YnKbrJ$3(8Ds*>EP z9dj8z_k>srORb92558`F{%WDorZ#kwe~mIeM#hQTAgz;a;n>8o3-a_zZl0l zZFaI>)RYPA!y|tMuEm3hx`+Q*6M#zpK$jLw~SBs2^T5T0C$ali67;ZXPz)gn9FqX zu!LOewv~QftM*@=qPFrnt6#1QPo?&{-Xm70o*A)b+FZJJbk6EIGq0Q~D_F=MI3-za z@3e^xvw8C+I}d~}uhG%lzgNv{XTVa+N#cv|DDtGsuGTxm!N_fUp-E`QhWKloy<9)u zI`Wq}JYah7z*((vRQFiGJq&>+MVl>4B$Sg+JbNaFTlwoB*E z1#D6{xO7$e8g+x$zDp8ny!GyHQ@AEoJ$1?rF7ItAo36=jPg(a?`gCDdk<7G(=~0Y6 z!BeNn#2pIg%=nbbRJl~?zLxoG&IAq?MZS$s9SYj{dm&K&Yh0Nrze$e=DR{4Zx zChccz$~zo6pV=_WJa9PRv15(s27y+&IS*JAcQ|syd}tJu*u|2#!_|!8BWu#OtA3@D z)0Z3#UBJI>n}JKhgavo5o+z<~~cAKhq#fj!Ce>fJKevkjSM4 zZrgGfvRl7rIM=q1agP0mMvevrCea@ZtXd3>CE*N4f(sZ}91Pww&-u-u^xy#JgbA&N zGamA!aa4${xxg$uHlHuW-@WB@4i{OOMccYK3ugZ-nQIZ zQ+weJkN8bfx^1mA_2(McI(+7(<$AvttXVe zE6R9U#*nnVAZ>d&>w~(N#~G^HD_*r%)O>II^t>(WMtQ-B@{i*6ECGxR0d4#Zj5YyW zf3BD1e=mEIz+LXZ^iPn1k*A@7x1hc5Ig`0V2Ja73mJ1y$2944UENKi}vmP|gIKVmk zL+5;nCdCz;kp)bu28>b-j8OsH4<>LpG^TEC*8R6pe%lXUwo6gXiY#s$GEE(r^)@i| z3rf~bXIN0CC~-(}hG2~hQ;eROl!Ks&R|9L3Ls*hSZ=eEOWJ09+f>Kq`HTBFY27QU& zOFJ#vs?y8&92nBr8CVz?SQ!{Px0iQ5XW)5Ip6=eBR$lRaMIDa=15ZIeQ)U0R7aeRn z`#y#WX*z`falO=$t6WB`+`0r#~D z9DxQ#n>)Hi$_tqmb-OQ6X!^vwe?|4nD~Eg84bU8Oz$YKs9fGwoKg07MmbBv1lG)cu9M~M zFFTkx8(Ax7wBDS+d$YW^V*-BwV{`#`-uL=D4jnBwcp4Wlavf(->MVb|E$`K{hIV%b z*#ncr8bG(mn_5m@Y`|?$$rip~_KX9ab3Sk>T5yMLV3bIhvM7L8=fo6ep-h{rj+IR8 z6IGp*Ls(`llaRe8VQrzm{hz95%4ErRAz24S&6SMU;scnJ7?|8LnZhQp?!MV5Wx;J; z$>yQJB$mL)|A9fEVg4KQ{<844Pv6^0U$pT*D6frN(6h4djYWHJ=Y-ak3?)And~}~M zsdEMkXM4ZpB-RfMA_tf!94Nc|VP^A>`h0WVa|_r-3+lKEDufLr1q3Qyd}|Om(8r>{ zD4f8cW-!|@a`NU0+}5+W4R+2>UBGG4$vInKj(Wg++Xr3k6IdgziEg_wN3KOZDj-u- zO@i}KcaxF^zf#ZP8B_0eO1@>Oc{jx@aH_NPL{D1*CY1;2kp`1(UrwHSeezUxuG9%E zVg?NJ)#`s&PH(WB^rd~4vgC?R$(jH9Z%z=tRo;8DobOh7H&-K<ZEbNy2VCJNc+HHV5_l+54Eykf_^Sq@!?FK|yW=Xv*vYkxpjSOO#4g9cUxruWS>B!dy~e?FH3L#YJRVxv~pP% zk0E=V8*8#b<>3dD^P{%qOEt|-V0B+r{%U!}myC8^fyUm9qUBYq=9jG!X`1mzqW;f~ zS(CJ8uzR)_M@$fsn(+U{hW|VIrfM?G(BinTfcLsV$K4GZC(quPo6gP@z{s~@(G4v| zj*jxC^orUUtP2FX7VO}DbAYe9fi=!S!j7HI>%%1LZce*ORwV(}nL|E03c`7k-zdRyVIWw%f7 z-D0qw?E+(ZMBA$s9p5+@)!G}Be>NQcqPy{n()Kwryg z2B8LqcifDhel%!o*vYTJsO?a}oYnB}b$PYM{x0iFYbgp2Pp71Yw$G=LM2{o@7 zi~~3yZ`jCthxcv+?~Q<+FAaEa8Spk5Fi2fEHt8UPMgzmdiTzy18`vfsn){=3g*BIG ziAjejYnlPK_8rbC>a3m$Tn_@cckiyU3RL&+RNtgNSE5Yf$}a_z2NEY&&*j^&_E(eH zj1cF!f_s@aJM(U2&wVxd-LJ`W4Y=nRa4#_6N^f8@xVf%q6=zvh+wW5|>dI%#dd(mz z!2Cvyq5b+X?Eq$`1zl1GEY<~!SUG33YtCePQ{EQc*>Q`Z-eV04mO<>Hu)tLU^ zAJ@JG+$jz|+DBOfyI_*p)-qfSTY00Y>%V%o?V+8W~W@37I9(Of`Fq%S-Zt1 z^ryQ|FUe>(OklP&VCspUt}Vdg9Kg(O!&0Zfxj1V@gJypz#|dW7W3O+_5?Zjc*Xl&Y zjt+*G=NU80LKWZc-tG8`4yO*7qHhQXqY9OGoMiU z;|7CX!#ObnHn|g=TQ+bnK9C;xMcMH)tD$bxiZ|RVCUCk&E#IJXTDV1cWqP+<%i;T5 z3b#yGxccvB_kN#f?Uv~Cv+vITxOV0u=Q)d3Coym|FF5qZif8`f7 z?CLEaxbN)d^w@Ci{q+9tl8oP1GKvc@7b|d788GoHFz_ibmww>rH((Y&z_!eQ_pAat z0~bT#n&~}~jb(rOx!M_O-5+pE*Gt(QudCU?S6Lyd@E|drL8Fe*R)Aysg_}aU7sc=L zUNPWQDPT0(z%uay&)Ey?HU{ic0gUAz*y|ja$~YPI9xzD-Fd5uzoc&>QohnnEAZvE^UjrhRq61mbOVF% zfo%c`Opy+gXBcoF-oSa=!bJJ7>dQL0JOS45f~Si!*w)AJZWUlXE$(`ClK67f=JqA0 zCE23)?Rhrucf)p9flI|^{EW^Dlbn|?Tph>2vAOHox((do_xWBLT#Hn2lFNLu-8o7p2V3Ox2Ehdu{9KsjA&GbYpkb{5!eHwP%Lmx+>3mDaV?cnONmG zGz<=lZSh#fqAO^!_OY%k8_$Lf%6YRiWv!-l986?5%%kKpCuig0bUuEy*e@)fCOaQy zX5}~G`Kf$j%1Vz(auWkSsLWicdhhv^F0JppugcX)Yc_;gvbZD9yWyR2KZ1ONgI4W%;le*fD6PnAF zif$!wtHx+@P4-PY`H01NNue6IYSNRYRSKxT|(`IGQ%G7t1`(@C&pwH^kV^r*);XUWs) z+CfJSb&79QNYWCUdg2gQRLjF9!r^I_4_lmcXLg^~ z6i;B5w$Wl$m}f9?4P%Fsg5`Dgl@ocyYz{cHESSZ*D4@+a<&evMi{`YvqRDbEShyli zFiB+uiue~WiG&F-vd?Mn*05>hj{Xonu_eTC-2p|tCizsvOsWqnvEmd&YfmXB|_16Tsyo zd0gc2j5ZaY1`XlEMQc~U|>-)Xy8_92KJ@F^>%C4IOr{_0B0{a!`E6%GrugNm7w#6B~M z&S=*B^xdpGV z;;d%maZOq%d`Hp7jUo$^{eB(%e$8kpmxluDyf06FCYgB|B~DE%leipKeC*U(EjOMN zf!DKcZ0YpvNisGU^zCtA?(liDQG9L%t3voC3w9d^j*1%&dvOp{qw!LYPs!q)ydNeuiO z&p(!PpD#JZNhX0=eE}I^m!_T6o(7@p1z$kgb@!AT(E>)3q&a*RSOIsvt+nMXo(wxe2 zuc9VJ(x!oPSK`&JqN>e||8lBx&nO&Z_DW!nZdoG9sl~8Ss)2#UhL4GB0;8oqyIanb zn=#8)3rJjUOfz_YX!|oJy~FJn8~L{EWY*c3KSy{0qv{Mt;q8xI4mP-T_jgYfTUX(_ zz%=lZ9ETQ@;Elt&9*viu`^@rqIoIph7J(48(({wdBf8ZBH*8{?(~}x=%USc_l~)Fu zYS*@GYZm8{Xo&GvcwV4y|=P+lw_|69AP`~R8TgiK~SFa=$wP9eJL!7jQ$x5lRd-x)C&~y zv$Ne;^d>YrSR9Ty$8v=A12ZGTihpk+1OyJS$X)Dd%eonJmqGcBby2S|yRz897p5R=Ru23F1pEiR6LCb=Gs%6S|mo%{B6h8HmL&e(BC@{|LcFvqGH z$~TzxTn@H)6eqeq*}6q$UV*I&tErCft%IuB3Pyq63sUCf90>i{bZYf6t5=Cy2FpRa zb$6{5YBma(yl!?+;tSUJ3#`5h&DJNGjeckd3%12i&|bQsfjxuw+NT3M7Vj2KXyNK? z;WlV+Hss7wXyHi^>o3r8WZ(?G!N8m#R~f<-cN6!8=FNdjcN!G4G#GgknAE!0 zW*V$zcVM`b-H^PHpYZ~N`UIAoZrwAQ|17v~?v35B_x5Mj8(kW=x!FQL?BkfwXjj2v zbD)v`#gZPa1|f$g9fM|{9dnI$Fx|0a_W!^ly_3o0M2n$A6H8$y#}5YE9j#FdEX6px z)(Xh!6)>qwOzOUAvQePPqiMZt0JoIDd@c=Bu8z%roVV&soVj0u=WxfaAf9H+30C?W zn4_4PS8ndsJJDpw&^oVEl`Y31mZ8iQ4 zU=iNKu$!6FX-9+E2b-(_COMwz!5thL4GYswG;+^qlwfXV3}DJpW_Pw=iOvv9KhPj_ zqj~GXw$@jMk_KW73A&sai+(;JvEKVLg-Q5816PME#{`BC z#arYZT|O<^;<`&tuVBfy$6GetXjI;@#K%kPxAv}2oJ(~g=P&!MAI8+Y;em=gV9cHtfdiO;ic>|)D0(a5gQQWPN~ThX*evfc86HM2lN zZO)?FIcF?SGup4{RVdPES-m#)!`|DcZ8vzfC;qV24lw!0bAr)&2kYgFt;QRe_yrpI zHZ-*FXb|4eG<8>#c15f0k(SrHr$3YuF;3Fu`q5y!qBZ&e)5e4*QO_e@#wzO*Twb;; z@m#XR%ZqzK4L6s=x}e?Nx}LjMpVHF*X}|vicRVw5{A(t|A55!+n{_;zv=_Mk{>j>2 zGi&|gp35_Oo;fw;N*+tfI>a)AKSkS(b3$`kBMYxa16KjViN$Ol35|je)C7zgvZdLB z6&jfo8rLMY$z&)vDzv9HoMBX8bh^MR8YG%Np^=HBC36F-#tPqytClg%IKy;-K~A9Q zt%uQ#o5vX=8s1w>P1~{Q<|~dHr*#dcu*NHh$K7xWT_DSGs5$t@zcaVE+Y%+Ripe@8BGEM#0}CCQ=DXyD;DYj)R*N|$+3LUXUoc&oX}TWDrLcDG2t zs-;a0do`B&yDa^mwQH@H|0?Y(Yqg|S-ELBgSYW(>=VU`jbj>W&i9O*RQVy%mGL|-M z%sIrjn4fb6gLKD)pc4VtCNW;CZV}aJxZ~i&JGCWqMhk~Q19QU!uN{onZv=RrXnZHo zAi0Ctb3%aN+I=VQ1coqheSFEHsZ zU|u(aQP_f!+oQ?E(sGm1>IufoDgunm8Y0Iktu>`D>?mcJ-qH}D%gA)#_)midUWa4s z91P47j5|vk`kpxSX7Ms*G;jzsa4|6KRIpvM_g=z8+37BBJI`)ESK7d}qhIoXRMCyL z2{&Y%Dww2i@Xe`ZnEt@MX9IKoT;1Z2w{K5w)#r8Q=4g-=X!6LIX51Z~c%#+xZ%`yh zTla2@#2-Q09!;!|f)X>>69cw9igNA^x;*=qrP2W=7ax81kk|#YE-PmUDehSECZ$nC zL6Ymlzu?I`nS+>k3^!=qJFDt^H0iL9nC=#*mUl& zCfqorU&`ue$)LQVMf)|IZi|QTF}A!FYylDse08n8K8e~VF5SEn)V2HJqgQvbWt*c5 zxIB|5dOGX3rmp{1bXVsrzm$bDz3mV?-xan-iz?3nE zb;1kQ3s<&x@17F4)=$-KBg;XrYf(GrHrhz6jg&kQso0*qq17RQNl2oR(Bc z^oj=GRSoBIJKfkAl{8wSG+0a`?l@RBNH-W3J&4Z!aXaoo>w{IDTZJ2Wr?oP_Xkh4w zsogvOfpP=ihkMhfHE`@`j#|-XkTFq=b5e8cdZmOW-|Y3hg|VAE?!H;VDDIKHu$Ws{ z^ZBY>%>AOm%a-g4HMz3)TTj>$QyUEyy%UTw3XDP;jEC-dIIcW{^gvIgg)lZDfk^+s0*)u4PXo8(O+qCX_4?5aa61b$F0j zz!q$ASXN{E?pOD?G^R{zU=Xlq;O+az$|uk$U9l)~<=P`xuWQ;r=|26gcIusohE@r? zhG{IJHG7&pcVyL_>=gYdu9v{EX)L42NdlgBKTpKOxyI~pY_7&}9=7b-v3HT3`Y zj(hc|t&Eo+G5Pq%e`c~#V9~qLC_97Eu+H+Z*hd9u1fM#Q4QCMAt{U=)JAp6dBaMo4uh)xOmxw0QUArY+Q{Z9)bd%94zWH79G&1{?GAo%2?$`uFJugK$Co=IpOND{$)qiknocfQ9fyY9ZxEO~%&b?VFMvu|BlR>2~XacbSHl+z`R z)4Ed{U)-$R#GOzhT=T*??J(cvimv4qVgD;!j+KPTznB%f_TSG74PFJ)@)~S!D%GBm ztexNO!?(`VCA!ziEdXq3)R0bs4DLJDDi$HiJ!! zSMc@B(r@5-~!>f7?}miE46dZjWvI{{)uUeEo&3C zUcf2FGsDT`P}9LhHyRHx2!`!DrBuMc6vrtnSK-*QqFqeQXG>1up`)D-kN90G(0ufx zP*BOKOGa>0^6`n%W;J&fJXAWyEw16Rdqu+|KMCuVhZpcUv-^naCmcHB$IE8+PH)5R z7xy-$ewVeK=yhX5Ve^r84gsz`wbcy=K26kMw^Mj=VAp{+x47keI3F?`7FM6+VW|AH zYpRN;ONdRw#YRU?w#~*03_-gNyp-oRdZA~~#K4{M!NvHB&t_pihs4Q!e%wLAA(F4z(S1v^I%-i|k5SK>9%cZjA z3(DHX<4OeE#Md<@s^#u^`R!TW-qMfJ!uhWPPx4rvF>vM)4N*8)#42F$!u7B0lKF0* zQVek@O&~SiDdr8X#kKRcwD;#t)!y6d+ zbrKE;wU`+9`dYu7a=Cez*$rR6>mD7rB@>{Ll)ETpdY1=ZQ|H zmrQjxd6qCVv52%pGqTiPZDJMnWq4yVUW zdZzj>J8prSdsf_yh0!y)AG1Zz-g0#3(K-7*ZfliOxna=EQ~PUWGRvo@Utg47XI!M#hOn4K0%C8JkY3%`I>;K5IUEVhiiWunmU>2{5t>yIgp@vuW+`b%~yQH&s&mB^6ty%!sU1<(=32EOq{(){V2WW-p)l zVdlc)GjGpXS|0dZEa^@nyR6NF1J5}`|Ezt?BBXO+V$ChdF7Z=LhNoh^1EUj|giC_Y1br%O-U+;Wtyg3>e;}X zCL3aY&E&`Gb^ACj`&sXc5Z-_NRsb_+HQR@-#EcbD2D7ADq!t|5$s3@RHu0$Tx(O^F zYq<4A6sC&_G;$x}Y2)W>rZ*Cw3zd-hdWN7S)Y@KwJe}F zOQdXB>&p*j`IBbMu3~1p@;A-z;6-MB3$vyht`8W6BMz|4(^$=#vm}t^9S1% zM;HV*9N^rP&=#1XRXOj2i`tn3d!7|K2`@fzh>cBw+2cwQ=h=*70s>z(rYu-u-*#(m znb9V3c7s(68)SC@F{(cR25 zr7xi6xaTttvlWwgGYXi+|74pmD@$aXna|cM;Y-@XxM;Ro*{<%uHGXvyT!cLwnglZx zm`+tP*gT#os8iRX@X=H9RLK;fHkE@Fvmdl9Gb(J;=GnrN=8&vz_Vk!eL<4in23CHB zi#=u>3QP$X=Baux_I1!ZZ4^yKfx*_h-9kc0{t& zWC^FY3I&afbAsHsId5u7GcYYId+=|Hk_DsakFo~#kOz#)KNvVZY?>a|=g94n;gonQ z)Ti>~LXLujrsMz#$<(m4c>5KOyjtRiB^M>IiVFDhh)pw)$@4Q${O~ZZ$l0)~S9o=7 zMAOVS8`z^+oY+(}8m;fXY=}6jrOalaBELCVK=0eR!0Uz$tdkPhd>1TAvtT}|acTmq zO2k91)Eg{Xla_b-eBco{d&5C)R$}|7*b-|E#g5uHi-nfwwk!O7c-KKmKwIWcqR_bs z$4qaPPu}o0MZ<2Po7JvF;i8hWQdd{7ipwbMUe*=T$mQe77@~P4^wB3P)d@Z#brXI`^Y2s)yB0t7g+lkKeDPOC~|pd$Z2FG8vA(|a4m~q6j^bxosq+VVe;0ss{^i0(v)Yh+I!W5 zMX!L7Hz$F!VMpul8rR-L%?zo?n_jwG7@A(!Y-i&9xNbJT1EY-T0ao5042})~Ogd*A zrtIu-V9a4?+W&SEN5={_C5sp9$_0z#TocYDPsv=b?7>zP{HbxOg|L+5r3nZ5=AF$w zkkEd6iJN)FQ{J3yA6V`!c*nW#qJi+UBdj(uV$7=lHaKv4H82Y57;coldyd7+?ZiDn z6($LTMoF9QuJ|8GvM()M#ikrcmP=k(;p8W$c14*(_Qe9px{3n~WhYlL=`1h5{P1ph z(UiqU!W5X^?mNJivzFOmM$(zxmqY{VltkvgIU*Uqgh0ZSEN_zPt%PL&Fjk{VjNc@ukGf&0>78QkioIVp?b8HdT|KcN=x}3S;WrK;T zxI>d##zLE@CCv4&JX}PU>}}4{et3NGf>vzj$>ckz|G57X*{7Q#(~XHgOS}WKdDSEfPr;Ig@StY)b9b0Zm<_HaL*}d)y`qkUDIs8rP*}Dd?6i1 z!30Ns4M&j#N3kzVx(SCw-}s2kXyChXa2xMIfe#JBJO}wK8f(uwPW(6F!J&eKa%XPI zojEA?f>G{GqrwwMMHVN8GYN_|PKpjpDguX8e413IG%1-L)}3)kAnWXmrOk~A&AuF` z)k2&9#55bfIT*0=cIlpZqEQR2MGtT=xbZ89b=ErY-C~rgs<>aXeo(C>mn6A?!TX8V^%)!{RjH_4|&3-d=O=;umr_;r37!9v-wq9&p zHS<51*qK+DmA->1o>hEZ69akqw&_zA`&3r1;y`5!ff_vy$_%{j(!JqNcMs*xF-q)d zbKl2l$vAhl?4nhfjGUih#P2YwOmSfUa=_~qlkS-&(K$lvY_E1nPGUYXz2?sS9}D@?WI1`K*(efn$&!qs})5u{RCOR;jllA1HiL(&uVYaA=a_a8kI@DEGus z_Q^q+oP)9gj^Y`N{3VQnZw@^9${_rOfseT8D?O2RLFbaO}8z@1pDi&IZOO zjuxL9SpOw36bd@23OqCOa8~SbVAN>8|Dd(}?9Aj#GgJ7c&b=$Ccehoy^#IQu2aW^R z?>eU0dpRquIW0WpS!S82%n9!$mTXU+aA#*7u zR^b_CjR^rgVs{)BXB-He>!|m^i9yNHzw-d^gPD#CTsjxBl$(V*hB%a|FqGRIU`=ok zN$AopVbZ$9q>k3IS1HIIIv|fu;tuj-V*F_QKdri(iKjI z6vhKrVq11iyzOezQ2pR&*CDQ*jI*o@51RTgcQiE=O>yA6a8Tuj)3FIN4ryLUjtXE} zd2c=6)dhhE=F~d;V|GxBTN2K4Lu;pv+9oFl7KvFV3QbJU!ZUXzCJ8W3wmFt!urhVa zv&NGG2WmPqmuPb-@W{{Nls&?xFpG0K(_F<>xAL_Xt>Qc;RB}-K!a>z93_86BbqtoP zMKEqlaA@5$@4*Cy^t@F(A)V1Xr}M5kusG%jLj;4+45fu-hYWTc(ur|o($ebxdVuMh zgTI#otCWs^=PCz>_S15_tENfs;$Dn07_ojFx|ldFL8uHZboC|}UX!z2A&=LlzkSa0E* z|Gi;>MT%kqj6!o5b$=W(2v87;VAy%pok_64~ToN!C6jXlB?zs<_HJ> zOE&}--&IRsdTo@zujANL+NA&I(5Dnf&K>OCq0`R!xu>k%{O*c-`4@L@j`bfVN%!#_ z;o#8~j$q((IAl`bSXvOMRmh)erpf4Nu%SCB7>9o6o*o)i%cgPWTzZb zJ`g5%T##(vjEi4@0coXyu_ z8Nb-HGwV1og-l%Z#7~SjI+amhn{oYnrEll=o#JpfZ2ICLgF&P5gu^OZvb5Gp#?>&e z-O%c+Z8UaZHgQlk=`%CQ31FXa;PAv79B&S0Y-Tdv;1n6wr0KwHe5T>ejC_8XLmtth zH`R`DoIQQA_VuK7Ya{onxgIj%n(|()y?a_&qaU+UiU0RiU)QcGbX2u*UdW zk&Q!iAwyj9?Pz}H_6vt3SIn14y{rF+NxkAw#?Ip&ch0@FV3_}p&&>UUrbkIdd4Ywu zOiQ`qxA)U3R`IlaNn%*Xz{JMDRAlKa#s4~W-?76=M#m%53QiZ#J%QzOmXreFIzlgk1)VA{t53HyBlpIIPQTU`%KbW@u7) z|Z2YNpGV zUO1%ZutPnDQK+Dyx?qZfio=hD3g(cFYt0>)D#F)2-{mIg!rXHGEYtLVpNrhZEuBpn znmAn!nFuW7JYw-)fKmGjlhT(5Dp#6}?>LFiJHRvNz&yufMFl6_4NbZ?oOG`=75W~M z{nAjC#K@Q9XzbAZ&a;90!BItdr-U^Nr8jYBt=eM58^JBv&ZD}Wskn$~iNnuV6Xd+g zr9ZI!2;7kWCryd3RLj0GL4Z;Ek)`7=M$R8|t&g5!J<-56!{=l|3!lp&%^i$&zYl5( zG+otbl=E;_n9?M_Blu@QH1`ejk5djx8uYK&?WFIKp#S2K{sLLnI7cyujh36e+Jhx+ zK3lC1E?nDRv39mZMeljBha3xR4lr&wKl9&H-HK+pC5D^p9F0AiH$HFLXw9sb@b92% zPh(%QBgX?KJrO78$A?U>I4Q2Mz0vf0%k?I?oW_Vx2Yx&{ns=W`X@^qTWZm5+ZUxJ? zJa*_4m?PS>cgxK6L0oexnV6Gi+cQf1F?nA7Z?9PqYp{**gu0j3jf+7m&P&4NMt4ovS6efO+hIwHJglrn6RT`yvn1pzomdJ1AE>nqIcFYdYosPA>qWsb@@)1e5nOhfbEOZ!^{$5RYlP6VK$}GpX(LHPx6)t3tJ1j?Ua{7|-=^)wB$swVvk$ z9{vjZr+a&wlaH6BbWmecH=96#OUshhj%n(Cb8IRP9p&`ZbIm)X7}WB7qLN)dhvlck zZT-^vF$)5;6j@n0MKqRpOkg@QM=$KH%}ixsAMu$+vrRt!TEcl_hEIx7l}1+I?}-*s+Cb;z}VWtrl(-hA@J}Jt61%VH4zN16J#aXcw$a`Y}nMq$vusOYpcq6 zO~ZvQGf#%NU6%BlVUu{Mg!QtTU-~8m%Yp+3npwDIL@vZGU}R=vQ4mR(aGSC5?yg>8 zeFp{ePxl2lzOeDhIF&x&Q?ST=(D0kl^Ew~nBsJY9+ho}lOc)p+HgtGzKk(2i#5-xN zq0s6^?+_;Of9t%b7p+{jKc?X13xP19TO0zqIfZHLE9@E+n&h~49u-jQSkT!yD=X>o zve^|}fk%{j44hWT*X?LzlS&Hk(pYh38#-r8iXLYR9+1la4 z-KE%-(8MWI`ofV-CT)fzn{?icgKZjtK8EeQT+$6jJVhT4PqF4vVBEwcv*1l5tIvtU zQ#9^aJneK`BdNlD>C^$0^TswiAN3fW*x=N1lTU%qnq6WAAHxd)+b_HBOQrTPK5TIo zX?p0F^kB&giKzxmuM{E_ZI^2a9GH^D!1-rPEG#VYkif*z_kC~JB zR4DfGZWiviNXM3`hC=^d_AiR*xf{C3sj7iRN@>&L1&vYs2U|r{XFPbRI>VxbX}O^1 zpO@-NO}7rVNrm{V(if3hkf60<$*xmctILjjJXV|e%W#=`503A!7u!}65AAW z3>Gj+7DXsFF(^z}kaN&$CYzw4+QmlZmu#1f870QO4Azx}adu?P>2UmQ0EAQ%s!{ zrLyzruV}x8%VxGMxOq=RFGYfpU3dz^6EVjr9LfE1N&-Dk+08aQo?^4eBTYcq&+=%? z?9&>~T*g9)Z7K}98O$9Gj3x^jclS838B5$+s&IA7^1|gocO?B1^iG&OD>!B0SHYsf z;m9KQ!ckzt3I|4}%MDx>3JjBO98+-#XgVmv!1CfHLzxCcqwJeR9?^{*ju97`wE_~^ z_&+i8)@`al1jglSi}E zY@K_DXL)9Z zQ(|Gs=4LM+WvTKR&c-ted-Yr%FzlM?kf2uB>&cO8d-`QVl|q2Ijzx0#-c{)Wt``{j zb@W))JZODb(8|5V*JOkAc@K0#v>7lO9C&C{8*^8BG1J| z;8Z}@qg@hv5;QetgmWdhDA%sNQf_^#NjUD;ZUxQGWsNcsF&!Ihlu8crZ#Q7oMVHUeB%Ok!<3s++`M$XrrP9n=s9CgSGiD-u)8SCoy@?<;St1mRvD+KDxs0n zBcV|!p}|}F1_Qf}fU~T`A;~2gttKUkU4adcM6SJXGt1f7YdJ-Dqqu|vH`_rD#|;UL zM|&6q-!QPr#xQW5H8?5|acm2_hXeaEiQ@{BF80{4II-{Y)G3!ca6|h2jf{+@9v-7i z_MkgW%#AzP_;>>R4l*6!Zinb8M$-sHOqN4bhMrN$h?)yMR(g&Mcssu#gCPyOZomQVVSV$^)CHI;USMi zMfJ`+bPIbZxW7zf8ZWDVWtQPQ;ai7T=15KA@IT)2uujSb}FLaWG#7cGcvJf%W@hPA%5USjcHo`O>Go{hy-aGn3_S zRa4|y61!|(C~&&CAG+8-uSLZ}(M;@u;{|@jjwt>`JRYj8x_N;d85@#V;*M@)5Zkcq zgorf5(Pd{gOr5}!X2EEAMzeu|=R?aM)n+OGxrZ61UDjYXxV*vtutM32nFn1G-JO)D z9n|y@?99}0mA;usb`XnpMZc!L8g z{~P0iMrY@PuIIai6_})=el%ogerOb8c~q<`sJtU1$!(s^bagj_>79O4l*D#&um&xB z>{)*O(#LNeYEzeGo-6ia(a5O({U-L1*sOAHqZtZZ;U}6TO$@nw*Tf6YVPMr*G-2ky zlqpW^u>oH>rz~jE&{$X=|ASe4!Qoj2XBb%)+$dRG%k_ZCD1o`;!O9s53>T8E+Lj&V zSSUGZVV#mABZt1NiUX5G0+T?(dzl35RnEa~0v|LSg6BQ9EO{*WDS^{#p$JQ&l)^%x zWvvWctW0VOOz#}-IUHb+VbF3|u;ts^Q+^A?${NKdP5APMlQ)b}EQL`bE|H64-PZ*f z9Yu;~BOFCt-i!9Ja;q(rdf~7oM&Y~Hf#M|xb&sT;Pk7|))vj*wisM1TlPew<{MND6 zBrKVsueGSDZ%=moJcoX!iQI9gIKSmM3v9e}&HMKa@84|;bonwl4=iwEU`$VI6f+PM zjaevc@Nc8A!$O`eKY?fb?0#!mJQlEja}fCB!2P0u&mfI$T2EF{b69~utAiV}S%R+D zLdh=&7^XFE=}~4%Ss=pqP-N0U-joeDq!<__7BJN`&(ungls>>bHO)eTfwAaC(=;`M z8H;Pb9XOJb!zslgwPO{7!mBxQ34GfYiuoL1TD90JC)F&S1SP-)MvqQi5o)4k|iTpVar8L&@yk5hi#Ujq9`Quo}kFfMzbxs^D z4H`juYyn!VGnCwZ7_c76=Bsjdk6R=#JK@rlC4#1FC)_$Tq3?;uGzZSA2i#}W1^pgy z=?MyFGK$(v6#i%K*budVKWPo8#{$jD1)MAk1(+H+MJz(Z5U|D^bk-=eR2{Tg# z1OK}RoGcGlu5jdMS-^0?S-0PfSt)Vtt8+pN6bv>g2iJYfU}?5KpvBQVLGS0J4|WU| zCCMy*9ti$nV3$ka;CLw5(qJpGAb{gA_qs;ei_JS88OmjAuoOL<&#~!&o4@D|0|m#; z4?+?Pqt-uQdae-a$UXC*jjTIjgChecUYJD zZQ>7hh)+v$-*G@S?$uI(kCvvnf)kgk^(JuK=w_d1#KWg3sPj?0$NFx=q=!7QRLTS@GUSf2;g9p=WgX%aL!-~)0-ye`~?hJ3+%M{eBByYQW%&G z6qvXgSc2N@+#j&XEnq1~;9v4ktUw`5DS<7kqs{5T<}aMQR{1<13Pls%i_ZHXn$V~q zc#e6Q!ov@O@_#!Y6e)_&Sm-tPA)i5mO3Z^DIgH{}2|Fh(;XOJfhR3bctL2H^CXI*| zwt!VX|4ir$C{o*V$bCyff5IcxY%Qyq2lhT{phucPO}aI`G*|ajLz@ zkmDdEbY5hhg3t;@$yZvg$_~tK4$PNYSPT*v=A{^vJz!IK;Kk<<+mX$}$FOim2Y1H< zhN1)(7l!1j1KbJ=IlL0s-5xMkC9o?k;8tT~FKFN{DPj56z?t@d{h<6lSmGD)hjBC@Huk&6=<}7s7 zQj9lU9>%axEKjlINNUL$j~E9P-U?xjI$_ra(W#&FzAz*gpNjl>W~a-MA5KeL zS{^9}`nkq6u+CclvuCq!?U83md=&|geqGD{SNl8Ucf|vbl5c|5Wt?oL^@|j^d&-1M z8pZy(a2;8|V_?W>;lSg;D9+HxQqjQog@I>U1A9sXi%A2^8Pk?;2ZYKNu>4ErC`#~Z zVG!e0;F!_C;={nsvd~R+p}@L6g*gYs_BgO)Y!TUYKvFA_f89cvJqyHcFba7!%IZ9l zj`$%p=V0iIgZXhQ#9k~EQ&=c-s*$f@#*}T1d^w3y_cqJqG;$d1u;+W!TD3qdN?d$H zL)`V|9aazK=7i4|yT%;SC?29HDqtqMNuEhyNzK!7WeZB-?|9;1-P5$>o*S=9?V z49p7D8b$Xt*sWS9wt>O+T63AB0!y9(%b6V&vrEbazxA$W;9Zr_8=TlTyJUVCzx>*U zvwDnzTnpR19&j%>$hjfg+mS)Q>!A161&=S4^Va+KxUA#2vcTnElajaBH`ah}u0dj& zf0pjCD0AMjUMLwVGyxu6t-G8)%>BbQKPWIMXpH> zoa^)iT@JXY{%Dln;UbvD;`Jcs>8AOI~-IVonW$L+pe_RbvHJ9`A? z&E}YLh*gV$`BTmL*H@Npj$U4wq~60&TJvl1rX%cfKUtM_vIaFgjX&_T_5gbVQ^J(z z&$w4k_}8^SaJJPH`M^ApRMjP&xtYHl@Fve_}PA8IIh>(8=f z0ejlbJ6j!YZ(qRT;=p>A^32*jkaTrdQh5C*Tb?h`bIUU!CMa*!&Q&N_ienNZhJR< zY68CkV_yEVzq@Ar6TY^d=bVDzDF>cM{Q^S$o4rF~$`zI$+3NTG0pFwjuMhrpHJr1< zO2(P_0UN`j3yRh%X+0Ypln(XsE0^gA20ESW6tY#zvs669;wEhCRiQKSVhe|mt-?10 z$BioPg0@ycM;tFcXyj+tIHl_?)F1I2aE#2WhQJu&m|cjxb0J zS-CEXH9K@wh+pX5D%RK9>o@M)w{dsNQpxH3dUhcP1XhIU>bh*n&^&asGj5Mx*B4!; z8ypXm+kFwz<7e)iL7pg5VXB3jer0**c}ZYI;u6@R%@xQ6%faB!^{M4;3C7viC_^R(w%7 z)XFETYg|w*pEw94D&EOhs7t+{wuHNHpSFbB`w4bFaR=1lDNua+rb zU}0^zz!2zMDbU0oynyj)@WCyIllvoMr+Nh^u6;CB=Kznfn8p8;&Fo^^D*_L)2OSFK z)+-e~I5V#D>C@@LF-r_11T#6(SOnW-7fK}CIfzAaPuf}#nKsMvLyjhf2bk1ZI8-5dlUPJr9FsxrP}V z*cBpr7(0a`L>{^FxA!Ejl~-q3)Gn5ovZ7TkF`=+s%(mjAE9bpO3tYt2XWnLxweAsM z+$+CfRj|^|H;>M!%E~Y_1t{=bW$ab-F?1ERv^u)jp6~Z%J_G-)N=t$h7@H0C_x<_o zs~hp9;dE5)#ntY*mzH$hofiA;sf$#7Yvzo^PS%CbX5}n<&^){77X!;&ABTnK)IB2sH9CSU5DSW1UjKt{HnGp;>@yib3PXlJ^f?Ipbz{ zvxsSyUU4p0Jz&Cod-imOSWbmA4M8tddmaeBS~73mcRs@qfv3D?4GpZ?YnJ>F?p6@t zPG)B<4|puH>ck_Fj~m{+@VEc{g?ZxEe6LAl8x z#dA+pR^C*NYp9i!NbKO;ID^0ED#OiGlO+l>=QwP?QmexU)9*{^21NV>Eqn@ zQywuc`@)j=kS+h3rCZ(c!{M-BtCN(7VXbcGI&9f~3+HC*KarYqEs|Ov3Y2d+$dPBD(;W1GS?=cS4wDDd`2RUz$B&ABx+mOCX?*Ffv@1-6gQ65 zhe8)Ucfb2B`PAPW=-mZh>vAvSOzva8x+%Du4LW&;BG^ z027-}gJ4scxUs$2UIxEkx9K5J`2SpZI zv`BVLW)W0zG*&pswgF}l% z$vZ~D1rDq+0_SSq32*tM(`aY!{l<`MH;YtXG9b$23~r; zAj_QFk9{#Ni|azUocCI&U7tEylC)L^ZXLOi(aDdYV$)y%EkYPeOBF&+HLEh z`R{C^#C&m^mBIUVMAkO{q3jD+R|pREe;2oE=1qnVQ}41wS%KPRa8tX{JQjn z_Iy4828Oa93~D}$1fotf@(Nvgt?|_T<`c%Zc8?$X3q|aaW z8Ba~%Um(W)FpX7jPGg6VM&)~VL$1lQ@4S1!+4r1{n|36&4Rz}TO5B$h=yNbxq`LO8Lb^V) zBJfT)Q>yzH3_h2Xn66cc{FjJ+Rz!ffJt=0jTW_a1x(y23~QIakh)uNfceTd z1@<*rAKXLkGD~mJd*@cO>$2>I7H1bHzTCr&Nws%#gqFnqt7<;?=kh)u;U~H!yvG{8 zFMnKrexv32h0Hg~gmfB(Ruo6@DmtrA;;?JvP*jL^EflF}F?e$=Z#4tYp$`W9%shv# za($m>z{xDO=L6T~1KbZ5@Vz;}WgEa*_kc+`m@(0T>zn~wtzvlfLe+e>kk&vJC)JQ{ z$HMzttBQh{CAZtRENf}l6t~s9$XD2uTg6m;du^C1+r$a1Y7H!D2Hd+oaPPWMb8Az*HujsC!V*=Yn9OLrLNR7S#`|SC23^G%%lQ5o#4+Q)$TR(Xdu^ zVo_{h?VP|-cHK=?!NASXfF)VEi#_Ijn~iNzh5xqHl?9EV2e{T3u&pX!D`V$-+Q4C9 zz*+YpJGYT>=5wC%1D-Jl*sMFma}L<)8mGTiHtuF*S+Tup&2;16iwhk#n0PmvG+c`7 zoaU{(y)Td>p7T@nRz}u#1vbwHmcIyk=z^2l0+!YbLhS`Y?H5=r z1UNnj`iOjV@|mD$>%boGz!c4(;BtYbD%spXP+@j)#G)eR2M;Y*uILh2QNB^4n>D%d z&9T&jOB*B0f+88GD&9!pT+Fr2nKjvfd*_0tjtk6_960noFa-%Qnsjg{9$>w3P(1U2 zXQ3LqtHAVyOL~`VkuaSg`Q>x0T|tY(1*YG|rcWh8eP%dto3OmGV7YoQqg_+_gG7Ab zW|lkyC(jA2D;{v~yI`;*fi=>AYfA!~%mkKn3(h}N$}V4Ykl$9n@KM%GbZQmfqKW>uRldo>b&nln59 zY~Lh2H+P5I^a4))4(`bgObQMRUST4N0!%)YQza#AzlP-p>`VLR1OQ)-qublo>IBwl`dq$Ux6&K^$ zXFB*k_Wn4P`RYVws|yRQHZWT~sFsbA_AlUQp1=||fqTUV?zac{UKVg1p2)fD1E-q6 zO!jXw#v2&#C1$N>v-)zxtm0|y!lz08&l$vTnzsgU#C%B1=WzSXBB=3EP-cUb#0Ex{ z1#JtIwYg3xJYHDO_so)2F?X36lf?$+e{ms77gw^D6tGu6VE6K3t)9Sr)qr!+<|)pG z^L%F-#IVKqKU*HTkSly4m$(IM>dh z=LN0?y@hVmmzJx!6-=*r8T;Txl~j12cksfUBCS^yrMq@6l#Pr9InN!-YX>^P+6Qf{E5G+QtsE z%MY+bJFpZvu)Ay!xYD*%-Z3mwJ@?;sW(fsm=MQb`w=ugIFx^mASZJ zeM|v+*m1?*x0tyd*^3Ua#ywzPGeJ+?fbCqsKW+;K1NHx>Wi9TPY@8aN&$aWyG0YBDfA-pH!9nB8YVaMi_?mtL@N%x3d{V3gog z#kFgJ%MLY%hUxV$t6sPEwI7S?|G7%~)k4`*%mLb~({?TNWoF%3#pF@Iwc-KioC4PL z37j(%*ycHKFP^~S#lXt8jpdz}R`VfdK~1;E2en)noY@4YxF2A(DVg|8gt<FK%F$57=QIz*4Qi67zsvc>?qF5A4>6gaF6*jpEr6c@0lIcX|nk#0{gSpIuy$_aXcgEyIe=?vLZzXEl8*z|4h9a}rJIWa7Wf}nS-5OvahQal zyKz=i)qPfVQT4tjCrla+IVgUg@n*)t;MWU(M9T(FX10DHD>o~{e*t^@0>RVG$U*eM{u>=LkBfPvA;fLXy|o!uU0z5qri z1rcijMmrNGfe(z&O$_{F7@P_jo?l@;m%wdpuu*zx%A=VZ{~e5u7FoUs}LEZ;Pt!>uJ}1sM>b?N!MbcJVH*y`=q_t?@ayP33tb3D~7FcpEzQD=-hHI_^ z_g)9?um+Zd366+7MS(qVZM|LCy zgv&7g4`SM$z`gd;?u{1n9z`mr^yKgE$Y=CrJzdRwCR+JsCr5IFV0^$oj%@`TQUc7% z&5XCSd0QWdR|+n;bb{^jip`gAY&J-ic)44`x2NNPd++|HAmSSm}6v|Kj(-f)`*yQz~FubAU%~f>MM+-jWZj5)YU}FC+_Y zVChicoxmW)d~4+@+g_HeC&g)M=}MPO8>X9HU|LnuQgCVd-`~~y+3%EWUHI_e!e7!` zKYTs;&nkg=qm}GsMmgUA_O`_=7fZP}PvA&%U`s1t(lC?`aNxR9kPs_y@0YsLGhhD) zYpu*Dlqovp+^V@R@qs~^;bF-K1~!4~$-Q@*3|N*uV7L6h@HY1WTiryV2@LL46K9{> z&UMaNXu*R;a);_d8Db9blq@(a@PT2GAEQ7*fNB8uHV2jstjfRU*tt3F^fKfty10^=h#j;$A>E^lBr^awD#z%rqLyQ$z-?ZuUD_g329Hp+2e z$q~5iIDxHQ@sfSR^i^M&>u*k%+|KO$!DR2>+Z$iq{vx+Jao@trig&Is>fC)O>s`RU zG=V)}0{adFjue4^TtNm5%o`ZR6_^7O*cDR}+Aau6JIJkJo|K@%+-S#abRZ|!naSyc zpG5$R;;kHkI(GgE&a3a;vw3jeDuL^a0;m6mPq$(ij@7)J9ru7!fpMnXb)gFk3-lPZ z`X81&I4fYlC|37SC4u|ghKDQ*rc`j*bkALGdWkXEfNhV$%_#o)$&PQ5beI$czJ)3= zB?s{SH)D$n;BDW~qvycn+`w@?j^)wpTX)W_WZCuhat6yIy>|<;SR52qIX1MseK&pG z@sk^+OT z{R7AH0`8UtEV6=D=eAli|8r%GyI%TMh1v1}OO*q=u>#9P0nSMY>{b(`8y7G>ozu>c zz##R3W6}cV+y?eZ0h~q&FE7-b;pThgSpR_Q-DkcF3@`35+&S;OAunL+0`}E*NkSi9 zZSZ4S`haJvoq?f|2y?T@W6!VqiVZ>=xI+_IHSD-0i`h(=1mv9>I4&+c!X9?cM&(ee zYLAf4Ifb1IemEW$2s+DR`S8$DPeC(3r#_LB$CgZ-oYS+Uanq%a(AYA zHnrSU@R~B|*i!}$4uvVrFD@)h;CR6@*O~ug>U9Z|9p^UOYRwkc=2v8?(?~REQAzKd zv+_yR1`>7Q67Y4b37>(=HsIX}idxWeTs-0*5AvcQc-F z^35@LI>B?<%oKh-C5;CSySW%1Gz#);ZfudM`1im`!1Gf?L%U*y$FpfEUlbfCq;Xw* zz{n87{H}@pK#bD_MW&T0ZA?DJ4vg$k-fOp9mpKvacPb^@fr+i*FprRZS@8o#PNf^V zO&kUaPsAQiieAV3a+>mr#>OeD`$SD5Rw7e2y-bUn9DE-F5%RerTC=J`P+@A z4ne+x2XV@b8V<}>&n7%%>1dGO%qL<&?_M;rs4DMhsw8PGDWY#HdlwBrGGJlX2MfL!w|_ zNAp9T>ukqakFr@xEaGj`uq+5nWt*D3DCE>t%aVr+INL2+IrR=OI!{m%413VWa962K;m=(AfJ9-3|m>VO%f9}G--4kOkT3FUGGY+fl}kO zdwpMh(&e(-N?$me{(iu2ui-SiHh{(a_k_bT-wrxC{&3_w6?sm|#g*OdOIS#c#nflI z8(q9xmh)PfGYd^{VAtzv;nWde6jpi2m8Nl6a?^ws;Ry+RB@#|beu7+qJd1czI+(=v z6|~FrIPh;#ZkAx1$ekIoSn5l_@fk{un;c#o5IM^*lXG4HcW}=ElinqleSCw1uOIQL zHC`-v;Ze$Ft|@BOKV~ze@i;iB_Bbv|H*gN-wrbN^5yax%eM{TEVWr5Opv*`A0&nRS z|GfH(XR?w;Ms#h%OlD6HW&Tp*&W-aH?AUwst9{V~XQ>xl7B(Tmvzr1~l>fX6&t)r{ z-6f$EqQ^2->PiBena0HJ=UVlu?|q&(?e!hu=^e~+CJ*_ROmO7%D_}CtU=rEFz^dU^ z*s4-+fF;=Gs3Myqo5KW0&h^SoybcFgJr$I;RZM9!nYFO5a)tx5^oqm6B^wxJZ!mK8 zRmz#BB#3{t{gi4oRhT(0`L5Mox7|}y;>%4lSKJRdz#wMfD0af3EyzriH_c+H=(&Uo z!ZHb&XB8QirgUlwoVXOUT_J~sqHfyqbD`?rhRyjaP?{))0W!|4e{c;FPi$E4Jj_Lp8fJ1=Z@mTstXwH zgQK~wPh%3gHJeq3;}=i3g|lQ;feee%hiM`|^27bV-uH^$#l-YYwK4pIxA+VJp-VGn zH@g;@o1JmI`oB?Z6>Ip}eO8YIH*agv)CufNicpgDH2<|mdEz;d=@o3RT|S01*}f5# zj(#mYfsu3C2KHbF#^Wn~9JRXkkiTp~BXbbLQS~zlZ4MkwBA~_US&bZWJOcWj(haP9 z2~4&h4lpnta1i`AX*s*+g9p8rr5pI}2ykVYBy$vIo&EVmd)toLDHHI`tR(%Ir@(c-MZ z5^gBLq`=^&Icw2PCgBRuv3OP$!eSR1uWn|}+QBL-(V%X@nzf)!eMir=1Pc$RUY3dm z@g>4@CQWwJnt#<|#|7zsdz*I~vnN>0o5(P)vO(wsQ`TmtMx*AN#*BgrjKT(tvJOqX z)qAxiSXDI|IU1I-Dp|&KZCSdIL3{zjq|b-`SWV^nuyk6mok#-Xb7r-fyIDOdm^3n0 zhpcWsy72IRMj5*o&GrE;i>uqx8`vCJjudpT+1uDVo2)rnw5FU(TzvB`(P*XQY+1<; z#zSK_1X%t#v-&x(#xk@T9%xe8(B!|M)nB3|Z3F9H;bz;6qfQcyXA-7xA7J1) zp?*MtQSt<%bpo3t!$RH%4ZI92^*0%;J=%m*nC5BmS~KotXV`1`P|_~Y*fq)6meE4) z%E3iUQp^rbE(XoFom&GRGzk5>(NObBMo6I1u;Y-nMzfIsFK%G3TF_p( zfW4xiz1V`iM54Wb!_%pzEv15OegoGrD^JmA9rY^v)tl{$=eQ*AKH|Z^!Y9ETaD&C~ zLCf{e$Fv`?L`k$d?r2TRSnHs{Vt4^`GNRs%W~B#Bk1jPkylAqyz^uP`QrZb-^$I5I z8O*X7O^Grs!3M`{GfAn#rR0*<979tY*I1m8i@p_oB&JfrZba zG4KV8-vO4-cUYocu=uCAJA0S~Nwj!QVEH-ayjKKI@P^jkE|O^;ZD}2CnH_As32l=X zv7N{k6jt$aG?-x#$Y2#Xqt<)LtcgobF^C*kb9&#FbvG6)VN+l@y+gctHNzf`i%c4f z+l99T2C$btXgsIcC@9e=v7k}9gOSIAQG3C^*6Y^HMmKC9ENSQc+&mN9O(&-HoZPIkNng*SUeaO=-g=W(+K%g z(i;DRHOj)l!Jg>u@2 zMOa)Tz5i4;2>7bJZoIIt(D*~vRp*D2VSDFC+fMm;a6ZQlhQoUrKJH==F4(Pof+^+$ zd+vrtiH=4=4MwrnGvY7Ss7SERJJRB^p@Hwj%s)bLVU|$7N&F%MGFthF9C~ZE2MCxY6Qz zLrKZ-SXIO{5%HhVi)?CLJkIDGF>c_sXf)CY3D*dT;)pB|VZB_)mSw;i#Ifg0;2u>D zma>D)M~xXJ4Hz{PS~E0SgBa$jZ)mVFU=^`xS+s{WYetLU0S327TY*arV!0P36d2RD zHu5K2lep0+aiUS-0^|M%j2b5zpFX;!Akn1G5Um!`q`qLb<%3f`y>iwOt2G?-HBU5^ zykPg-&?qg@D6P>bu%nTG2cx(}lca8QdJe4s(-!)AGB)+eS%136d?ty$-{t_+@WO0R&& z@PmY>Mytb&W@`o(hYQRGGxnt@uxzr>o4SZ?war1{oHpAF%=Q<$grc5S`A=k9Ik z2u3~)=70)YUxy~afS8BDF#~m&!y*)KFt*q>wQRjBtFc^BYocJV-1qJX)k$67E;l|9 zJy;>!8f@VjBG4Mh&>Fze8ZE%$>~M9F4y%&_OSA>6lSIplgYmW%A~6MQF$^sp37U>4 z8hCd!@a$+%e{ep)qiy37w# zCKg8p7B>MF2cf6Ef-DwlM&FE>qb{`S7&Hp4xGl}lc%tQujVYtUhKt%4T68?5&RscU zD3d76cec=?m#=-X3^-s*!g_V*0g4o(+uxn;1D0 z8ru^ac^*7oEEaa&QNQmtOYAm7p#@LV_A>IG2$oc6K5+J_h6W@1g+uY*&M4mXndZ$H zf68Q2)@;#=Mu`MQ$*^$AWsDvHt{MR{LAUo=?@_E{O;SvYEl_);{O`3|Uu+5An|}%k z&lRd}lnO0c6>%iShf8fnbI^lE-U#7fffinaMhy-P!xPNr22X`d=B~ZKJk5&5F@VWn zLA>LIrnCdCX){_3I9P%jSb`f^EX`Wo4Oj(B?#XXFGFh+PY zHn@e$_B!wEdD1tTOJe>f##xKBR&iy7{?TO2s^05+g2niOl2E}#i5-km6B^U+HfRX2 zN}px%^=RVw!EoWsp`%yN)TP`>6gzv|^cl18hK7`&?P-kC6)n*PzPtK-j~+a{KeEL& zp|#lIZvMOMf`8fBKRH`tAF2N9EVNSM-YFnF`ME&la+w8&)+ZRA{}h}S#-yUK$x)!0 zm*XFkc}Gi%0;~RHA^itUdK;S67(#3`80UX%a=O55c)~?BAX%w`;l#J8>@ygD%N~5n z=QN8`>&HZ6yO%1L9y>0W$iS7sAi^=_szW2^3*M_7T09cW`8LfyH(HD(SXBg;CIsok zGB*l4FvU8s`d?^qHEvj)bVu=P7}tkuiEiQI3}>J2o+|EfSY$z?cxsSKIis3t*0r9N zopnub4qwl{cilCht>Q*D$NOx>2aSIAk7PfT1g`&}%*Rx`?bac`Qt2BXD!vJ{^M16p zXtjCKb3D04J%ig==vEp>t4#vSd|%LXuZ%<^`+){o4Rfa%Ee;Bd$~zfl-ZKTgkmJ#4 z5L{3u_islJ`;I(Tfrc0B7=Cq1W~VT)I5b@1ShaAY@x20z_TRj%&WxNZ8boI>uy%Ye zXYb?eVEFuKhxP*21y@>C1R8}mxCmKv2pcd4d|>sT;bi=vf#b&^4`%25Txl+WFBfVU z^-Z!*^)`xJsMxlQQ7Yk!%Xdbxgk+|C=DE(MhZJSrU0&!Gz*c#o-M9azq6CwoM3d}> z%KZ$b`#1b7X5(7WahZ3&!1EZcWj`7#)R?RzGOiXi2Pd!`=S~f@Yng21m~7G-Eby&f zic#hTlN`g9RQUyh7npPtm^>9)7z`%m>}Pb`(2}sKVS$$|=ZclmF1=i)lV9v|EK8rS zb$s*Olid;tQ~t?ZXpnr+WG&IuzHmOHQRC+-Tb}~faE_bC9!xD(jVcaJh7zpt8!j$$ zVp`nFaN*0v6JNu+_)9MB|8dHdajRX4m_><3=xd3J#BJ*uZ|kupHHuB1-h4!J&iv>W z`E~8={XZ3#{hTTOZ}$F5|NB2@um3s!LuCmYSMYYR=PF7n(_$m~Tby?+QcY;_FnD!t zM}w+`jBH2K1`k({h89(YMt!YD-5ZP!qAWrTj7%Dg8`YaU8(PkM;AQP-bU%@P^>>lR zez%4Kg$s-f96}N%84ngPGIR3Ecohf;Hn}tk>A6jD2uyDAtacOOTC!ZoG`*F}#zH`m ziOn$WOpNEDlT%m6?fX}8_7Y2$$l8*9C9yk$+Il#-6ps`OT2JuPeK6smubisP zq=E+voC7r7nk42fN;@U8Qq{ZaK}1oyr-Da&g~rQ^$^GI=6I28*K6ud7Ce);*YZ#!^ z&dVzytB{az!I807irX!OLvYcNUUA*1B|i>08as)rIUc&`%)iBnUnwNOfrWjB!ZAt5 z$cHC&w)}Y1ZNRr~bD#c-7)95Al4%NwAz=ZQrJ|vMzE7LDHB(we!%TWSn7K@rEKuq) z+;f46SI1_Pm`L8vgKn}pJ0?z#t@*O>y3va(pTiT~l{{Gl`#q02r}SH%l;C&tP;(Io zR@-(sWx?Bq!|CoEiHG@oZWMNCisx)-V z4QyJk6!hU>ph!zcZiqr$Xi}JA#L<6`xb*tG#d?f3Ih@Gz-?KGYy7WPX_=I|$O=qmX zE6mJKbk*@Z!uQW_TIzwttbb-6h-quvFwN=7gBID=J57viY6^jD>a%55d#VK~2(l_P zIXL+mFYz#1q0q=u#Bpkgz$X@!2_i)&3=M>!Ym> zyo#3dVwUAle=dmES#%;|?5kXT2&09(_55Bdn6)ZjN(j6Vx9rdJ1Y{pU6 z^o5MdIgWg`TNK^jpIRtgHD$WppM%BkFSNgKQD@iVNc3F%aeBJj;-zYJ$2$F0ZU~FI zxhh%sOQ~5nia9uP$C`MEi`%ejE!nUv*rQqM$_`h}fA1a~Gq}SjFj3>ErpLr~8y+Ts zDh3zLUCUTZ)RF{dX*BBm6UL+5{E(nux+vI8-ZpK zBgQ=%9g0F(G0Da`ffu7+ycFAJ5oc_W)s;5mk=XKp$qG*Y3_BK1P!#NNXc5=j$n3W5 z6l>fKVSXWv&Xf}l43Z}pO>QVm;0$>%S!h86vrvK=dxUEJ`@#l8*#M3V17>!=)MFeh zO+K7HtxU=r8aVwXb{iR(tT`yMj8*AD6Tik%rlUDs+a@F(5_*@LuXgEFsK=2-;%gLG z4L>!;o&V}9wxLL>^1#HtD=RI}?VQ~4amsR;j)azKCqWbU1f_#ot5t7o<=Z^*qmx3x zLKfi(j^evKUf7ffwyMlv6j^ZMkism(-n1!?l9$}@)%@mieCwP?B3rKp8+mPHmeg@% zw{kcnbm9Yx?3V>Beg=owl@48DzpzlUaf5^08-wG#8oq3o4>U8g82poGSaFJxN5Vru zE3k31sR}DsBLhRyfh`;%O`;qf2VAbqo-mc?0L#)HtfqW|tVTRaA`3g1O1&Hy-S#QW zJm&L^U21~vN{`PPN|UrZ*BojRTd2TlxMRbk428wVE*@YvoOQ?K%?0O@ZyJ1dZOiu~IyFbT04b_K3U7VlIzthau_EuANd zLPwNcg-aOPf+jfemvbDJd&9tH(4xrGbmOq>7X{8>9!CaIg98j=8(5rPJY-@EU$THu#FKyeurJM)& zj|OaJo+sqR<-ous_~QWYp9Sn59?Sws3`f2(D9+8&d>Z91IFUQN@2~%R2PV zIOT5Ym~E7G@zK6cF-Mifk~t?H@;$$FSpL&N{@#L(Ny#x1HWy}euKO)Cy<$)D`5O#n zNnE`By$@Jz?>rJxGB~VZbg*;5oJ5g12aasoB)}FVlAqFY;;^Q|LS~_y1{T!=4xC;I zO}aUZ>=qm?x=$Dw*h^k9S#`8%vrlNWS)9NmFu#FUF0fN5<^X59+#yyj0|s7$#R5+p zPdczIGGX93C@EiXK*u>(fGxy9SwP@`oDbtizj}2hlm3Po1`k-$pDg55%WyJgDQx8W zw?={OxXbNeBcJSC$BO)_=4B>dcP--KKX+L7P}<{^Cp*M`G_cksu$kQZ^k&zNM8VfN zg*IOple4*>@qgAiG)<}7;6~!N+kgM`Wb`ShDLq&yaP>wr>!uH!<;jn@X1rk0owPN| zAs|`mipQ}W&$KSXzkSTHCpbCfK2e}&u7#J+x z*AyvmIwdGDRb=d9o1&&*|6m~ls{uo02or;w*jonq6)c=Q3u?mtuHVh1)Fj8UHa5=!Sf<-I5#SyO;9;OhO51_zEz@3vZxn4_U>Z+p9*T)mMi+%2KdEfLNArBTzNdGklc zz6?I=(sM>T4*#C| zgOggzA>kKYQjJqI&rDSDY1S=aWP7oY^Tr`v4`#iNMu`bb3LZ|DH=GnFIC`lb()r_K z`M`-&M}E2I+)H;3aNJldR?x^jksW;0-?yM0U)$HIYT`$ptOD1<$sm zSl{Hfxu}#7^FTb~!Gq3!oNxXd;44|0CUNiek0kagHUqZD`+qp9%xKmXXq0VXlB_+X zmgA&m(k;SsNcYY~C9}gyuR3LCHtF7QKdP zYxdv(qkzN3o>MF?F8p_VBoq#r#~e~w;jA0eWVL}wWyN9B8po`-L*@<7%r_jCVDVMl zz+@fcl>6wA+y-WC5hu%>!<;J)KT2h`PH?=?b5P}mn$nbJ%N_^T4J|w=4%|-;Dg`ww zBphI6xcaYCKuzgru!Cz@NCem0v!+`GI;Jjf-+Lf%RjiQD#hiCf0W_`xB`14kq!gtXU$ zaLOF`%Ik9Pi-YM3XGM+Y5(-VZeFx1%nmji-v3oe{eqk`)aES8;lir*Iatcfi*A8(O zFlygn6cst-_$iJ3!!yp7WgI<@iZ+Rg9E_qdhvcR(+`rktv*iFw&H+}ACKZQ+Oc@87 zR3pm07`XmuKH_-Q*6{pZhAY>TsoX8CZCg9qWmOrkHQbu%WMZ%&`oo2oDZ&#z9F(_k zQeAM^=7IXXpNz5zsr)x2rst^cxOrnoY3jdK|CBO1IK?@fR16Qw^*oO4a%^)tsIcRp z5XT_}8^^qkp1D(;j9r?f8yi2rag=L_6X82F*WeJF&B16UC#$|juZfIe0*)LL+8t~d z0&XgEWE@oab6Rzd1BcHc&M6G`(+-FWFtQdj%A9fFFF42}a8T}!1AomyzAp{pXAbcE zU=;k|P(0;;Y)K>MiG$*24v2qPEOw`HTh&1^frH|A8q+mSG9Ed=m2iNqpjYfp+w}%3&-522i-e)6z4D*n=pGw9#>w$b4G#dD@T*!lqQvb4;mOV8otId2#YXr zzR^%NWnkzDKQeV4;~fT;CF@w$x2y;)c-h1iz&O=Va`F|+bXJq+^Rf=GtT}M;&B_}Z zyd0Zdo;~7ylX&1j91|bUK^YU~c}7mgUmQ7=k8agyWljw}nbdt%QcJz=QLFB`M=a4? zHNuRGSmidPt9_wjGCb~Or9=J3uHS$)& zbc2JxUmaZcEKIKDkgiOV;tq`)4;)pltk0X#q_ASWN<-tJlWUilC|;e#!0+Rv`M`mJ z$A#+zBZIa)vx1AJr2~(Fk5@v2yp}J|gG(iAj#M|E+U|Ei!`|Uo?+1cabdOxXT9DWn7L`4*q27x90z%Wrk@w{qD3z=r^*<$uZfzfvtZIc zjzwXY&2-P*Tj0u_Gr8j=M^wOO)9(UzRt8(Lay}PvwwS?ed&k+HBe&)1*?u$a$kk`qDkQH4oVZ8khwdm>3!sb@H;+aB;0az$_Baw*5;zPb?$Li3^g# zx&93+)=IM0aGkCFkR^V9o>=hBnu7aH1^jaka7ZxR=Q7aEH!xb*>yv(f(zMUPT1~18Ol_|)TU=JM4HdLeUmI(!D`l@&tL|gf z?#BOb#?r`Bee%T&g$zuy54hzNIEWO)#ZPCMX4@ni@K`0FS+~IPYi@(slqTbx!@4F1 zW#=?0h@>ePG;$^|N_HGfuWS_SY4}tr9xORo=W}lHx|T~vcJv%yC^hX}@8&P{4}N%A zG`!fiaQ&PQ!OYz^?i^6eIjkbU%c2vtJmVSj(>dG{!Kyq58KyYNooGMHfGrujyNAy}G#W zvibb|S!=dssPM?J2l$BXxV`(}v7eKio&`Fqa74P#JfyNGOgzUyjh|65VTy)FKmRK( zJryUhB~vsy4*lC2=d8=q@JTajhRp@9j0T<)4Ql_7ui0F}D$l@jWZ?=D)EUkA1u4n{W|*BUsfbTq5nIGDY$ zfv2YO<^l({1v;Ud9d@n$t>N}+j*7Ihj&xtj=5x+(r=$-RWgKLXX<(=b=X`$0tLDJdv~_**4ic+aymW5WZ1}SN z|8Z~s13Hq`tXCd;KKK2=_>Y7Avo2@6-jhR9TpQ#AQ|~aRH1H+-W({DT*EEIa0Vj9D zbFQA<3$h+@Z8*TSqITD5hufPQcsDdi&pA}_dV=(lgL~$0mtOL{>au0@Wu=E7%Wc}- zYQOKap1-lF%`Lsdv;MY2dO@T7foTh+uV}pSQ!i<9?rjo@Xkh)&;4;%m_{TA2gEG*y z00swzJ}~fexJxiJIXM0jl3_BfxhkpBq;aQ7BY{z@;{d~tx_Ox~hDQ_6SwGh?oWJ63 z)%o^Y%zM6kdTthAd*O4p5qo&Uro|4$hvGiUTQ^MUhxWpQG4aBz}faZA{nmd3!% za!cEhiT%WTwjYd^XAZEudC9WmfW|**<6WC;!%WTy-8s?A&@$Joa={J5sX9T@X1NkZ zC7&NIv=FSoDe&$rSqKLk@w$eQK!+}ZfJ&2y5f=c<&OpHzLMEu-#vbY5cX zx1ON()xjxknqKVdOAEb%*sd;6k(t7gbxmiAN>I~`nN=+6mz9BeW+6$}p>nK}7Yd}dtO@iFZd#5}SIxw{g`LFfcZ;oZ9p;tij+RBX?Sp-VCKJ zia|!gmB(J5*pPIHOXT$Qndg-6c z=hIXconR3)_9<9A<(*IEsVTuqiYeNTa+<1>tPcn*?pEK?kkrQ0RN>U4F-5?+M{P<# zQ@if7iX;w3HVd7m0}R4zP8^nV5Hh@@($Tw-eNKx4(6DA-qM2y|!9J4f_n2KOcx=<5@~By<``%`C`JeyZSYKT{@ylz*WfOA*E?oVe z@>joIGG~V3QUQTEitI`+G*r4&OfDXsUiN3BnxM)O17~r~B`db<4w@zSo>R8uMgp6) z$Hl`f+-5r-G%~SNGql^3Oi=7nWvFEG;OXjc>5~yVple(xD9yX^=oFJpCp8KehkG67 z=}tJwt+ZnSlVr+@1QtmXiNu!n(=LfkJW2&V%@X?(S|4wIN1q_NR znQB6^dMA|m#ikiNZGXAIJF)30N3Y-Wmnj9O_=L?28d+u2=D7(3%fvSA`6PBpWKpLU z=krCC9(OzJLSAncJ@|J|o#{3CKi#Hbk-0}5_cIl~oyuo>#6d~;pP$i3RuSKGJCwNn zGfuWN+ssX`E&8c*2Y<*%jfpJpWqKYOSLk2--_N8-^c4nGg znjpW|*KzF$netZ~)$M+6y7o(1W#iSlU2VdX-W+F3`hDw0i6?vl|4RoWI8qpv|F?&G6iy^|BE@yEuhiB zsC0nEal->9@fr7+=1rI|bHf9^T!~BZ9NU&0l6ck`b?VH?6!XSHOZ~D<$w@432F+X( z6xdV`+~tvJlj@00GtOW-$e8n>MenX7M7BI~?u;i$I-9v880tK$-9Y=MO zma*7#9O9e0!Bsaeu{-*~BFQxcY}Q&Ar*owwa5Ne4iF^|1R6NljSbE{Wk1q-waxVWC zFo;x~ea(}6^#0s5mTd=lrm7XJ*GWC*lVRd}xJ2AgXhL8jpLTGn!h%NrHxB&O77XGl zft{;dnz;EA9Jr(o>^b3ctS@NS84kM#O+t4LaKz6zn4a~3S^w^Ve(nd2Do%!Nv2U1U zURt<{_$=ggQ%R1RS#s--$yGMLtqb{jG~6uCJ(OEL^HTl)%d^(TeRW`SnlLpob*a`y zFGnuF4NX2AEpj>uO*$D&!gC8;)vvtj4opcB$Wvf3mB`ffc+to{!Q!ZjM`K68ghyhN zXB;t8b7WxplFICNqLJ%sL6>Dt0;i9}A%T(&qBaj4*rJ@y9C)F5v{mShS@@ltG%=}- zMxCAiyp?Bo9~N20e5Ot_U8>>$drAR&xt!lXjah3E8xU7w1qSa$8o z_XF&!{@yrV^UV7G&kxG0J{`Qul6xpVT4}@nEo-#ue`inrEB0gc|EnhzJ|#48|8WtP zNmwc6f_8ajO~}VBFQg%p=pltft{2&eOoi z%)`L!{o$}sMpE;!E5aw%3BQQz&~a7W>F2{^-OIMcrKx7x0q+32={z|K%wjRi=8JOh z2yQWGX7=%m`S)Rt^2rA*>hDb57@cMu_Ts8xiM_jk&3#6Kpx(z0&WM}-whGQlLXKQc zI*LM<4UV*XZEX3;x{$Xc%sKFKK>Kgk!%BWnRuiv<%w{@OlXDxG z7GD?UE$NsfCo!|6#_KA7ZH$$j)WRwqJ#@_RgYNhRZQ5+JI(V9lGP0_O7}XKnWxRD@mDJsEDxKW6-2@;h@;u#75a; zm!o7SxbA7$V4&G_fX&y*adK|JKjyzT0@%A$lw}?l9M@R8t$0!WeYuDH|Nq5t9h3I> z%N855%Qu6e^!x1lFXlJ?N>5@q_G|h511=w$byXUh)ivIV_$3Nd=!zWDVZjT4|o;0zhB-GsSHC8*Ir}Tii z;K5G51_l;pv!woN4>d-qcW30@8Jj#-NBTl9*MAG(O>#?Mf0H1@mCO0eODd01fbXD)(?gL@2?Dbc1?sw_COOVAbKsoD z;BzgLW8MKavjr@BSeYXZuzfnf$M;ao&r#RoqX@%7_8=aXFAKPNk{QAryk=$SU-R`A zTXom%u>Q1^g;g7T_B=5;;=*f$IM<1+KYyuJYcqKkYrq%8u5UqpPx&LRmkI@h=k(gri97ol$A~= z+ivieDaZ>s)c=Czq8!&nW!p){c8C9IeEgH=z;=UQ=wEOV%aeT7v-zs=pM>-c*;*dV z>|x_#QxsZ~sFkNE;-tdB&&cJZDCzQKCQss*fF2I71AMDE1r9L?Y&cZNa*%h{0|B;1 zo&)Cugcx}W8aTEwFmola$So*I@xI6KD)Gfz9rs$Gf0JR53uKTJWo8XZ=TPf2j_Pa@TES+N$5wHG|Jwp% zqXl!OC2*e76!K~0m1-5#IV$qVKxv`Al4&dNj|2QVjQrd4r*LfK|GAw1<3YJ^#|7+- zp36=AH}Boc7w1)a*Zr0IX#O%^wfD1e*eB+MB1_K(jzbLsvl9Ms>Lg0}JQPz(6gc%d zKWw6)pN3@3B-tyDvNsfE_bn9jTfkoAD7)|A#3Pes<}sxCG%!EA%(6&e>+K0 zQfQKtoAHVT3`)kClN0h39iB?)2q~{ktIwINY`sa~^+Vfx-h#dFEP{jn$$e(aIN&|W z_^+LUBW|cX=2<9Ipq}gDQOfjiX1z@ z<(4Rx#wc3kD6;0DwAE#yKL#-^-hnA0H1#mLU(z^wP6^AWGKc0T+3i~2_b zL*(_vl-cG798Ta#zAGkTtgzTjgP%od0lU=$(=$1SdI$JFEGU_#%&4Nkm~cM4_yB)a z!=pS`p6rI6x&$NRJoY~gqEQccZYS8B$P`$nv}d2O`&THHYKN0)ZI{ zF5(B6o&_gbD(o?9U}xg}f5^OdJ>r9tb{T;BRACHp?S)YsyK#(^8u|I!zATQ&Q7wHAvTA&|eg`?adOi zH0F5*f`W4rxnDi;{KCp?&>*6w=)+@d6}R5NBAGR90b5E)PKm#pdN5nSA$Fc3);kA; zBzOc^5(R!KSRZxN*#BA4|Fg$S_K=z>E4S%C=L-CD=)K$p(?3O%1^6%iTac&n{C(W^ zDXNdZs{a4#{Yi7aOjl9r15TDiZWRUQHZ{&UPq~&Ta87ctOFh7Mq)Eg>QSjT80An#x z7e)~`MiGlJjdcse4mEHmbg&mVu=_Z$&+!zx#Sl~EAW)FN*4Dt8lu)MO@1(5|YJB)# z!at@&o@4j47?^#s?;1EHbWQsg_JD15O5cY?vdRqX-xBye7!0|XnM58i@El-rTf?HW zz$1j;FnL1UaWnS1h9HCj zFBkIXUoq2`h+V5@&M#y-{FpW8uoY%N)^LecuP_QfUr z9(zS&ejgB8^iZbYTz>pd{{8$bHP?Qdc1vic)%3Ta-o=$I+3(|bPG6#NeMaB|_6`UB zV-3s`5(K!eip#9zsmT;slwfE5QRj?<$Q7nHdn5(e40MGY#m^}Sep$fI(I{|)VMEqJ zfgQO*pAboIzQAJ(JsxoD zE&Rfzz~I)v$)_lO>(IJVM=35x{-y@MHy%tm4T7uQOwzx(v}*&KOr=rgCdoR6;-X)| zXOu+j9*Ue=y~A-PZ}fqG%T+I}=u(vYbYbdGb^-6!UB4CO*G0c*;S)$^U`=`uAFD9E zF<7WQSmkl)N=N3syMMQ+Zf6QR!1ly}{}Th}qJ%lu4@TU2C}82h70{SJgUO%IarV_i zTCdm5K3FM|;mE?dkp0fI_5O__9UIvd6a}tz7ALu}6)6}nF&j#~pU0ObeN-^9ZraJ3 zYbJW@gQfM8g{yO7KU-xpBZx8sY95}fiin%rNoM8|sVie~|6e@VYShtm_ zs?>7V7LLdgR<+`j)4yidHn5#D$hECpYP^6os)4y~)sFU{ohOxd-c8_Lq|YmsxboaX zr56WRRP$ZdIlJq&jKJF6FLqovHu`sL_iLLym6KJ@-!^6`nsI-}Zc_$UhPzjvF|<9C z7VTOJI=Oj`!~U}igr_aiF^S}Hb6m&BD6nfL&!ejI84r1U8r(S;1*Ry7@GJ6k9BAL= zq@%M``wmAff3w$xUa8t?Rq3@yw!bJ@^zj<^7n5z@`}7{CY;k8&dcdalP>PR{sp$IQ zO${QK(gltj5b!%F{`rAP2RExkf?T-5sq+DeSq*F%O*yKPtYHaNOcwmFK8x8k>aaP= zY_pMUI&gRQ*XYD|icX$Qy}QnSO|+}=yKn4!Hr&A{oMF%I+5Zk$Pp_P*dO6=G+$cG0 zQeuxJyJk3Z(E{!YC(cC;$(I+(20YAL@{ar)ehRqR@pmL}97_#8UN~P_L67^Fj=O`;v?ufRAIwiR zxOZyFw`o2;1&ci|bQmceVB_fLvwPsFRm^!L!Tm@BPe}^LECqIx1ja2qj0y_^Y7$QJ z1~F$PSY<`C_cYu#&9l0+O{98{)Fnr0r#cP(8NB|HyhbmlTwV|?qA01i>7J(V+4tO^ zo<4qFmv{f-?!E2})%$-NbL@L?`u57NTfhC8`_QwZ=<6Yt9S&_(3f$)u@&X=m88C|6 z&=Rgo)GS#e;T+ta zx2Mh*cJ5i-Gs?R;X1&;c(dSR)-3?C?Clx2DB=_yV&gk(#CYG5|>j3AA1iqLw=3D3Z z9XeV*VYE5wq zT;$}g(z|fV%5Fiad0`TUTMx1GN_n(2vaCGLX}s8|^wpG1;kEJmGE^TvDt}{ z$)!zircLR~d5lwEA7B)!<(sLIc(8$$$AW2Qmq@3`V!JW}n~uPLjE9&{3oC_O*kI|@ z!amQ|UgGb}5*FU&h9LoGD~nq%@|z{U?JPai!nI8`WI_XzQ>(O#TCA4B!Q;x7IXi4_ zTzD3!Al4|ynIx8Sh?#9qpo&A{?&>}sodf1G7dyL6uyYGb$q+v2s}%Dg+wtV&H&a&j z~@H^L`Q7DJ)o^;3_$1!J(6CD;6y36cH=9kgVCxqtrIRS%`B?p*$rcuS=q);1-+Bm&8tCotOzN zBJ)!|b*jxuNotdn`S8L`)2U05D<+A-QYEf=%0v#0-93S)#S%C^xqNQz*my*!c8XF5 zAGgkeW?5sA1&yhE76~)cv~E|bU(Z*@Uu-#^hqD_FpiMtvcy*+o*3z|=R7#&hnIHH-Y%I-Dcv6RVL*0;)9 z*Kc`-Xf0XDU8r3=Uzls>DyFFy7J8V<2b|?FIM8fmz{IZ;;Uwo&*yVrco$w-yR*gjr zronTXM3{tyV&6EmHBDT|v*1Fjq<~?YdW#dkw}7+Ufeo!?Q)0d3LF`medKcryfgo@}!-3P+QC1iJ!U6q`x}568SQQn69<7inm? zrEuUwi-rj!Pi2CO=A8x2&SsAVt!DIOoI0}U+u6*tITz2&HQ|)LQ6glv_{gcVMVq(I zed=#6Y;}N{Nr8cf$APU!z=cn2LgUJ(i3~b7mb$WSSjT&3l2qOU1`(gaGwK{p{0$zV zA`FEb_74v7`DHKscSmmlv)hpmvmbFB=UXhum+|H5`X^r=TJj&+aqMo(T<%YtsZ(Aw z@vk|+D!gez>l;niWfn`FydssJE-0FMZeF&Q>V+nSPYake3=S!!_H;||2_E!d=oXo1 zam4Yt(w+wgpY}DMNLF%jWHWSm)Vq9wi>8`mSCESnJ7_#)-hsXB3)q;FrG)1i6sVa^ zbenYUsZY+M?=uZEiE4Ijh#swoMqzS&K36$y=Rq*(H5I}+shCiXb)SZF6<*>3TPtC#Py zfy`rz;}0Js`c2)S(!q78)AwK74TbV3ZP)9IdhCo%7V_IX)l|8Zo>m<5_+nw3)@Nx( z4z~j=jyz7~?MGNucnZ7JpCkq~eQ=icDC`R0$q`6qVEM?!Hr;W7lX$0ti`+DW4t1S{ zoN3Wbd~E?*Wh;L&u_ZiBUr;^s#)ZyQu{2HXTDHp}o5bG^R~h%DLtNlkNs)<)#Y|mPETNbKN zy_u);^rqm#H9^ai6cv`A(E7}LkZ;dFhYrUd-2BTXutc&Kc6XV%2pn5mq){Ttd8=&a z>C!((G*3+E@YZ-(C2ioy?{lD$F<}8`KLfL(*yL-f0*|UM-Zd`2t|Ik1uRPmytFT7= zjfQIdLcV}r#pMTG=;KUo#AZQZC=R~ zxykvrlmAA6E$+*gUnsP8efg_6+in80%>*S8wFm_j6~`_cg+zgh(-E(N0D^2YA1Z37cQ0G%n z`*GP*=s}T?1LG}2Mx6#G?gpkGO9gT#2nH+=;9ekS)gYKx!00`JXX(?Dg$>?&rU+h9 z;7egCRr=Phl*adb0pGI+&Qa+}4k9isp*b(Fbi}2zI4Q7L3b1@@ulHzV`tpS7rjb>X zQu6s@!hcmX)RZ(d6qs`rI2JBo_A6$TQE2qsASRZ;C~nYgwZU85kfX^!yh()BJ0NKP z7Z!g7w(0`TRt4G1k1L8VGD`?}dtb0gNMKS~!00T%yn(UBVTRgrNA;EBao1wU&!1^4BG z1fJ_pOnxk7K54*vyp-|e0gg(Bu%)K*Ueklbt}#h0{a6Qhrp zzF*$)@<@l%5mvtd78e5+X9X7P1ZMqA=4l!t@=3h=!W-M*wW6a~fn8(fB$o@bC+=X@+sSNb$!uiFY*@f- z%&F%iXtmUsX`OPWh;3l>OkkVez?GtCZ6(xgwSfI=qwV5C#`&8V zxE3%lU6{vwVV=;0R@MgQ&0*Ya0o?5iSW6r@S_Bva9!zW6z~Zxj*-T*iH;(GJk7Jsc zsxOB2ick0SJTZO#QpstJl9re4`5hSJAMiZgz|1GW!0>>9;lm75bw-{BrZ+2RN-8jD zU(nRtz`%5YiTMIkPXd>;OX!bM##!HHHW~0vm4;n5?r=b9RTu(!i{xB^TB$G-uyEbJCuh z|7IsjEps}+oN|geRcnqxWWBKg^NEupd8hnC0-2;399jif)f8A4F5rHBf$!e}zV#>O zMk+KmU*Nf-z~Lv*t22T5h2f&4OY>M87!+SIuqH4#M{-mfaGXwIWEEgwy%18ifZb{W z^Zd*OyjD_@0aDgi7c8xs?y*DiU5W9o!1Qg;`>rkQ^C{p|SzzaC%E;Hipz*<2qd;}p z2acWuPPYvVhuqsHepqwTK|pdNL(c`CivqmYo8zY?aODWdaZePSzrZ=WBI?nEiEDQ* zS$~pG=I0W5Oa4Gkfa+51pHvQ#LUaW=^-@_Oz3pen*V?;)=NX zq>z;=jDi6SZ=UkFIxsRVuyem`AGfKB)UQ(Q@H z&JLEG35=Qy2lj7al%Bvoe}kZ8f@bONC6bkVWv6G!NFMx>uKpcP4Y~Ow8IjU3-q@f%Rq%bKDfzTLjpayx#D#fh*mBE4+Z! zNT{jof_M7@mdXafe-*_XtO9mwvyTLJGkD4{nqOd>S=-tKR%gWXCx&y%Q)=eP8SBoXWZ!qS~VO2jP?g4UO8?% zGl6;gb4`wEwjo;?J}+fxT_e!*M|0c36S@Z&O{_SiQ#3C-)IE8A;Hi7vnwuw|Fz`o+ zpAz&*sT^!-;+b=(k|p&Nv#SI9 zWCo7)##}#|xY7z(y&iFGG2l`@u~t-!!TP#e&xRF-x99ONFkCpg`)M~b6ALHfDn`o$ zcK!v-Yn<5gjQfhkV(p^UN}m0zVzZi_x#k!bm+1PB#~L&lb+s4-7-U)lj_-|NuS{Ss zZLt4n$ylc_VRHm$Z;|xWmHjslGySq))C|bn^YBF9940AS#@q*Pp^MujH~3yzz#9~P zDKNZFrS~M?o=bCMIzBX5bTpSnPS<&+m7}qP{q3)VMGoxo4D9g&r6%N zz_O#~%HEZe)Orr>dpXHPfxXT^Q*8pvCT6a=4V?2XaPGfwb^Zf=c30ks1tu1!n4}pz zSr5$XJGFbyZ#9kIEIuDtVg!1NEX`FmFoKTl{cudpTkYc1BiS(vFLQgYe6?W4#FpIS ztpzfS%{s>g4#Zt|U>9ZJ%xf_}Gr@H3zeO>*8yH`lWPY;s_~%q<4TWOi3wt}QF5Xov zW|*_TmZ2`xj6?SVlfN17g@jor1tu`=OjLX5`+Ne=-q=ghWtW&1)jfK3X|WIg>|g8_ zxjW(xT#m9j6m?+7ww^0n->|sVUU7T7)aAmJTYr~Mv{+_xhiA!!uD}VL#zD&K8kEx# zrUgzATv%YjHGz?{z=TPG(PwLj(klj@3HKQd7%F;2jXy}ST?pxCh`yA_u5%%F@h`Q9 zn;$$VmK0rer0(r?uRG^H8Qaft?A`tG*i6Q)b&DA16f+ALFuh816q&-z^nu}rDg*Zh zLurQuE(XSTx_+}N`&`)?bJ|(_oJ9*5 zI2<-L)p@?Rl)7|=`B*4d+0`r_tMhHXdv_l{m$JyNM6F1$z)QQny7thGjTenwLR%*! z@J$V1XnDZ4B#mon1N*F%j{+1pCOR0(U6D1}F6*vZe0TL1{p*iC>=}C)cqgu5W|QR7 z-Qc~cPX6+~uQC2_B^QR@>f?JP!uMhUdtJfPiV06+1P(deeX8;E-DTZpX?ds9&+WMV zc4=G|OVxXpDh1B74lK$SY(&3u&JPGoQ(#p~h}!PU%ku2a4iQerKd&Y(U_bjWfOEoy zK8_6x#t+y_V6D5r?%%)|pb)s+;ETxOnKE|gFN&^y&dxdx9tErKoAzaE>w^E#jB-T8l#i9A)N|8LlRORs-g|KMPo z_MvF6LwWz7G8|mmA=VhxCG)4~!UIPoS1BpYq!!3Y&0xzdiG$!CUv@+1|vWpCcB{Nmw1VeT@&uZ5qF^z!iv%Xw5h zc<}GwYVm+#=GS0{gTXbF*`ODoa}19{5YL|o^{Eb%M+d3PnR@a z?bI=eD5w%_ED{O}zY(xU@RQ#^ho-ddo4Q$N&Fbz{aEl-Rp7$$a)j|cm?&(czs*{gQ zvtkQKXjITzsnQ=Lw^C!0rP+qVyg`dNRfUe;$l%i2ysk=1d&7>dQw$rpIv%WEyP|0U z)8okhUEWTu@DSKI`mB*;SS-_1=GG!%C*OzWVMw&mJ7&wp<~dq42T8;fq3>NYEAqH`$^W97-(@4suw^6}m~~nk`L}$~8Nb zw%C8=Mzt%!iz+y~6y{BM*cq&JF>(E5m6;BW>?IqP%uMlnc}ZkO(}sl3fJrBtS$Rx2 z&S))P7WJ)>b;YVvjY7Mt7Pw>|zr?kAn_0^NLkabcgteRj46B>^=1N?= z(fI_=jf5k;|9w1{_5D)}Jm7QU#G*Sr_jQynzuCxn$3x9*QrXi@l4%MT7x&wV%&bUq zmDp)<)>zYUvQXdkIhW0E=P~O^Ci~n=?tk`&rNpXvT`@=C%=BqLQ`^N3DLr#lHBGwY zx%0KMz{E+CrdLwDvX)6^dRTpB(kRn%71B`gRGXy!#=v#27f(e3gP79CuOh-qimY30 zN(7!vQPq=hU|?yP@*pz(@v)aTcPwvWSR=&!!0XofEnQrzd6x>wtUA7}Yh%-KlMY5^ z$yWkPB+ZQ;IJ95cb|#^bwIsojUDk}FiS?M0am8j|(VaW~RhUe)6rO2SFYM<(ZTn@(i?S^X_EjlXDI~EgZTN0^ zu`RDHBv7eu!WRxj1tBLk*#$pTW_`DknkHzi@KA&Q?DNzmLN+M^M}&etTyPYYQ{+@< zXvqK5eMG1tbV=Wh)so8^4zcDfYEWC7z$9_NflY5i6VC<(WXf0ILSTW9rGJ9?jf1>-0!%Y6O<86%>3jZ&PZXY2W{OCY;6cPb0hJ1IH&G zPQ8vg37mNyEFwGDPObKFVBivP2#bB$uIuuEF)8o>ubdlq-PuUvez{usm zz;NOr`}tYSD)RzVf|symX85+rvK25o88l1XO*qcBDYe<%xO?Bw>DaE~u1p;_|I0v03LgY0es_l5qQFlRH{s_Eyvq4{rH1IKUO zCxST&3DN=m9CZ&4vY9F9$jx*Ts65duEn(QssKPyyX+swy&jLoZHEGOD46Av59Z+@@ za1v;m;L_PD)amHcG_ADYfX2os2Dvwhd~ustlwWN)EAjp6hIUs6)}8=g7K;Px-QKNc zSZCSjzGeZ_)sow{_i;5aTQo2T9W>y4@{7fD%A?RF3a%=CiBr-T9x*T zEXymYg}bKbHA~%@aM%36#NKy>_)%>A;-eEH|z_v~3Njr;-s4z!tGI=~)d&=k45 zVVT~Emp=UyE`Pn3u-84|km`(vM(zRy!{oBVaS?Y;ZacZ*P^|F*2AvI!OSqa?*mgLC zm^^4dl%o*KogjUCiR~?e8%cZyXI3*B1Tb+sEL5Mi!ZR*8=0>*EwG(^}j0v?p5gS}K z3psT#nsv)em*jcCAU|OZcSZ>#kCXzFibkt$6~n0jy@TOm1_#A>7&!tZoOmowFmhH* z;BidJV-)kau6tajt28u)kN-gK)G&dTg{@737bD`cJ}qDryWqfP!cfI4cK3yt$uD+Y z2B(V)vja`HDDdhA9A0X=_N8C>t;^L0>z&IkFn`-t&SE$rzgO$QD~>6#|Fj==9f;9j zXyVu*#UlTMfg{G?yTBRQ>&5>Yc)JQZEjbD~y(}&XS#57oEqD;g_8^+2#ehXJ;{Zz{ zNAvS30&ET%jO@iboMq(#*gait#QG;V@s})UQR|80j5Rnar*wdU;YI>qG>5Z7g~M)3 zPA6v81qpjsdcIfs{o{BYt4`QzO%ADm%Zxl44y;L+n5{&Z9>yv%DOLqE>9@!TEqTzu zIiZ0?Rpn!-w?MPR35Pc05*eXZ2Ij0^8|)5gKV~W%5;|2rY z4@bTY4R1C#lu9>BPGI(^Xwf^+oVKIYe zW;8iyv_^NdIKOD}ywJ?fz`&cqR$9;&eWUHkr%7fUOdDRTzGJMjU29XLw$a}Y&dSDX zln->-voJ6guzBuij@rN~-NCRifywDZll24^n*$A&hZ}7@T7ncW zontIHjq)EFm;7dA{nO~?y=zJJF18W}&Kw8MIqP2iVqSBbPf=kBKhHsdH4MBR2c;%9 za7^ev%h0`tv%5^Bo4@d&P*bTCixbOm;SC@m%5Ld4XB!N3;8b zCfyCJ(FQF#0Zlp#EC#b%if*tSQf6`JU`gN5s#;(nq|lPNgDtD0CHzn`H$%6(0(+1~ zOArH#4y4TAE6B{Io z7&|yx+&h>YELvj@u*P0E8#94FQ-i%~2aC`Lri4e09Xu>L7L2M4EG{Qnoi?y~C^X6` zv^X%ZEWW_RW5FQF!4h3?$RVSNeRdbqfkr2RX2TQBatYFm7i0vQ8*Jxz?*C=Wvf$*Q zLnjNWd>y`cuHj(#5Zt7{=D-WBF*90?m$t?n@GVGa_taq2<6u#`(V(`&!!UqJ z)21cpMzi+;=84KIwh>Kg8ZD{~+m>{(7)b1W=^`d`fjR2{Tc!bvszM__7Xxbp!*R!c8cl3u#Sqg!GIqyFyJD30djiH+q1nZvRZpXVE4%UH2N{_MjFLMVZdtT4DqQ3i*es?ofnQ2or^&VO zg-DSH+gDB2;0w*735~_NZcHg#O3wz$YaDQAK9IKL3_HuElRj=|jx^gpWKxn~QasTp zGlOwqbfd%y25|}Nqn?5yAqR!eTxvTS#H2A@zm>_BgY}~_YwQiyyH{?+TxiYuz?N>% z_;3z0PelW-1|v@plR*TNtO0`nGsDI%mgx_g)E+R|UA%6?(XyO}Wgd5n!;9lCFWjA0 zwA?(oD(lf|mI{A1hbU$5hC^Am?9O;rO=e(_P!(ChY|p@AxL}V#!XcXiCfgfKHmohS zC%7FGW;1g%2pwqr$Jo$$hI7FOYZko-ci9dWHwJUw0P~nsDMw-Uj0|RbZCSI1x%VQQ z;#HKTY8xe|UR=QKaG1wOvh}*m+Q!9;uP@2As^Vo*IN-BJddf7ZC2DgSFNho$PEu?Q zb+0%RboENV#Ac(74Gbb3jOBvCMiMMNXCvYdMD$)|&Hlmmg{x({bVJJ84T?9Kcr_YU zM=46};CHKFQ8h7is$g+yU@>iGaZG4he~R&gw!g!UW>pC$`pjru?;e>P05iuh(TVOfJYoZ0vT_vOT!`g!CE<-@g@SmD6H%OsCs0 zwA`>@jrC|R`M{cegH`h$YrzIKoiaAV7mYRt88!qpFbeSGHZ9v8v`lou>DyIYVoqY~ z3f0&-7#KG&sIj#;-e}4YZeZ!?IuaDcmY`-|7WHYbd77760Bh*a7+*#OMwJusLLCe| z&YORn4VIN?Q2fB0A$nRe;pW^EGO|&Pyb~C&6f%gdV5y91;8kekdDWcO(bjJCz$}5e z{!k~&h6N`$C;#_oQ&`a$^np9+pU)j>gJW*oyB6%ecwow%CB-JJK4;`?cX>GPlHbvy zoXe+I!Bls*QQ^n+23|+;hM2^+xiB~)e*YD6!(rT>%!gW1*~^{*em?n zb2hL!KWMbx!OT^lmOhDrM_8RJYMIjRRoe=etvIoYspB!@j*tyI8a^m9JG@}(YF@@r zz_3H$>hj!M_I?a$lgbeugsXyl6Vl$>jLMhj&MV zvLS=x3YOI|47`yHnvst1^H8M`f z|1&gePfa1-Qc4^Bhvu~YoOS(4oytA2((liF99nH23 zElx9|(J@Ib4sO30H@1Z{muyM*ame@*n3mV*6wvo< z2J?l-*Xz$dD`2q}64_laQ80I9&t`|K5|^`;#QvQx+T0fWqb2Z13-|F4{t_)l9xX-! zt$7DnKiy$IDW}L(uuS*w>c`eeCyQ}-qF$Cl?Fd8s~FtRu@u!volQ}8K-rJ>5l zS+@(#pVq+Bz$qKhenp|3egRe~o5t{pN7FaV=8#>B2CP?P`D8-u`R-y>d%~dl_d) zEfJi+#W`PN6ucu#0yeZlP5(DKA-Mli#6C8O(2 zj5Bty5*3K%Whhhf~ zK0RSoC(4v_j!~*2NJExUBw%aNlM=sj9*I|CMSP_j3>YLcn1d#;@x*_1{J~;;;X|Z; zv!g_7@P;&Riw|*;>TdI=Zz-I6*E9jO}%B0}byF2?oZs~r!?%?bV6%0Zd%xe-^Y*#SbRJ1s( zU};~uc5a-SPSi5i2*wSSr-Z9d$#t@PeWIJuba3+ohVv1Pg2^nh9P835G@?7&3KD8l z1KLs}*t}-^=&EP#<@(Z-Us=3p%hJgv-LL*i_PNR5xMO9YFOwE@zNd2MvV*hx5BiBG zvE~=(tmRo7-x$89mHRwPk)hxL zgCi4*kcx_d#==Hdc2OCD4u>V&f|C3!HY)|Yw3(3teOEiYqS!bns|hN&G4G06Jla6?qew2{!Yg#_0*+Rv2AN6 zE_SWIcfd3Gd!CeX$2Rffi;1fC|CqSs zLJlx2bYYY74=yoabankLdSs$2=NvW>W;2&VJ}NykKAm9UV-f5WUK9J^0kdF(I)gVC z$ASZH3SJ72SQuhV9T@^vY8+tV{NtX$ETYoj#H9IW`T?#0)`K14-Z2Nx7(ZU{nQx-& zzTz+oPp!>9hEooFJswkf;G&y!=7vOZ>8jL~ZGy4?Qe6Zp9u%@C#}ovzq&U5q;GAl7 zr@)yA}#jT#+DVArXTHHp>?meKa=BWNhV=DY&qLudGJtsKWAq z)&>?7a^4Uh0;N=2LtST2AjF?y$Sw1kxEjwZ0Br9d} zZ$i6fWz2>az8;%HhnN{O7C5+bnkon~`F&CJkk#f0Jg!`Hq~M}IyQIehH??blodRn& zUht6>KB6#Tz0Q156qYri|T^Itm-Q1%GvhW=3o@ILM@H zu-(D<;+)5RHt&xFZ=C3|Xz7xuBNmd=V`@Gebdy%uT=?l^`OB^BGmFnWJd!-&;{;~` zvoitAb7rP~b4=H$VQZWx>ch~;;v$?Ycri5civ!z@z{xKZ0$mm|FSc@!I-0XEpxImZ zT+1gGk%e=Nw#j<8B%RWXXIw3!SQ#++o2LJZ#BQ->yMzPGWm7&KU|^FdaOCq$EjTE% z?8?UmCgFbeQ0(cA00c!7U7uwNnHh|2+4iL2zopMJc&68$KT3D&i1w zx!l8}cwnvQ2A3;L95>{yJahOPxw~F(+Y8}vNfmn(^t3rFguK zzs;uz{n}&h`!zJcB+@W5~;Ls zkz1tD5!|v!WMYJqieF-v$Bss+lM=Uu<|TAlvnl>eZ>EcRp}zi31Q zbJzox*C`FG0g{3NuR;y_R8$n}e<=lMG&Id;F<`E@>J9B`7nNF@z$EO!_}M6+Gh;@x z+}90qD$1NEyCZdy(*xTk%}C+&s__@BYY>wEWng!#2Xc3pe>uBkh z9?ttx6}n@dM%xSCatjIwau)MvGTC&IC2YeU?g%BRcPox7zgOg+`zmD8q$6P^m8w>^$nOLot zoDx~uz~oahf!Q(Q5U22kMg@tFExcb8h59|jHyb3)|LwrSz~qsrG3g)^F08S|v5%m0x*v)jGxE%o8s+ zeVK6i>_pWWg_o=DEZ%+ZZ(e!%tKe6Gmt-AfUj-cg*t0FCh%fE|lkg4!dpVxOy06(w zd6On=p4)rCxa?L|_-3vHcA~r!_$nM2O%5<=^yp^E6uf3JYG``o=*6}C0;{RV#UB4e z1*V#42DSxFjE$TL3=$^}3a~F=VC8=|yOKdoS!rUA;-8i{@l=U+gDwV!k|!qijU0>| zA9gS<*}!}9#9^jC0==7=`U}q`apa1AU`~7SfU#e|Mb#(K!&~Js=ShL%CPx%o=JRAM zkz)++|F06n)cw^}U+78wqa02etr@hYB)Bo1TUyR0=0FvV2xB%V|kE zTrI)K7@?%kY_Z@7f5!wSE&+Y^5Ed z6A{OUnoR;nm_8_Uh^+i^qsQ4}fy1te3I@&!XJ;I!8&tFfx<}07`cu&cM=Hy4} zETL{!ji#^6(s6&#^snx~#s62eScCjdUq~rDusG58A8S~`oULw*O?i8NvYI{6Fz$Jq zDam$(L+Js7PqfN$(T5izWdj&EG-4T<1->nB&*hX--mP>djvIw|SQTwN&Kcmqr*e_UW5WWz3JqS@nHSh(H>^_%yf(MdK}e*(u4L(z zr^0j8b51y=F)SFR77CixdyZ{pWq<+vcnv}L);MVmWP2j(?3s#R<_XS|`2C!&ElLiqOOoR1wg84DQr z8uosk%GcPPwB_0{HU~zY4Gl@Whja`pgt)6#URI>k9w53k^IC za~8GUIZ=CnqvycuiF49jqF5PxmpDx7d& zc=F~a1)(RRm*(zte%hJ9`Kg(6pAwhdoUacUk1h1%xp)2e z!A{XtJ9 zBxKSk|H6SS;R<7E;vS6##tet8KO5y(oaA0KDtuv-IMHy`)WJ+vWrJqR8_gVD3Y@S0 zG28!H;{R_+z`mqsx-sR2_sR>!f6wU@;5aCFXFAUn2c9bn>R$59*&(ET^8nu$hS?!* z+IJZ^-zf0ETM$;uFfW8b!tH<%L$HR*DfK6fA|AZgVwBD19nuwP5;6&nKH9y0!!ZV( z1ChE8yc|us2F@#|bP23=VUTcOywk;)u~1-&+!>35uSMowDxJ%G#9Q8@iSGjg|Bmx~ zCm6&zoa7!jh>IL_<~!&q*Yfy=;)Mt&2UnR;%U3! z9Zzowr_X_2B_B5lCDVJ+S7%PmobLB2#V%!@HrLY9xaHbHO9Pcs%V+Y-h%^dvH1hsw z;Mo-v`tMjn>KO*@vkv@c{P{mH2!3&>IhJ1ij)Ct@L)AS8&W5ECVGa>)4QwA8Bv^!F z#WZ5$W+;Ae5MFc2<>yn@Ia4F=_Et?fB>SR)Q=^en=b)^|A=w$BED{G8IT+ruCiDDJ z;+%4MUCoPRz6EP}4lwFA2-_SIyV0b=!o>gJ0M7&mMGI%k3C;?49E~jwdNfF9HqQ1q zaZvecFbuWHjfn z&6ArPe(3>G&+R@Ow%emwCiL`~4_J-aF63j`fSF ztXibEs^*ge{}uIrd^gVUOmX08dCI2YxKv<8qnL)sJSJhCMmd&)tT7D?Ck_aRJYmRj z5O{Hb&EcAgz#-i!hZI?u6y7+h++k$%aO$^e*8RcA7jl5FEtThmf^%Pkz>9-&EshE| zoXn;)F|sr;TQI5=IGTDiYkpx!c4JV9VYXh;tfs0t&2JU0nse%WPZ;=g8YL2x z&*Z&ojB*yyb5yW7=<@NOgTe&X4UUfC0UiRD%~mPSW>c8;SPm)nIC4&4vYz5>xq(^x zi9>j7XjtX}z6lJ%cbcqnoK3&H;Je^0Xu~Av!?f&81Eb3UMiD3D9B0dx)tYAxPAEF3 z_;2D5PX~w6Soady0<#vkxhhh&4_LmPoED}~+Ni1I8~#|dr_^;uwE?r$g=XcInzOgP zJa|a6EN;2~tOxQ2Ix;I7rDPZdzGU*=VBqywGyBTVllF79fS{&CFoo1B(}#;3bZ5-;8e>-6*eVAOnZP{D;sL`Q$|4M!CTCu5Ih z;}B-01V_a?jT~{_cTf!I5>~V@rBzO?UM|z+}vt1 z8D(1z${x^Dh;dRFmW7I{?W+!fPrtq0m)k*{#6?}inA~oKVd9*#O~Z9;ug`8Q!D6hFgMF1 zbFby~d3U%E1&BY4Gbmcd(03w*?Z?6CJDF`9^!z$MaD*@CJpPjNuqIb@-obw;?)z67B;Z0 zDKhdoZ0Wt0zy=^q$@;n)B!rr}I|Mb8o5_9@~mj&k#9V5tzY2CdEqmR zvR?|Un4A=CJ}W9PDRnqScQa`!IK^x>jh1Fok~tJ*{n>bVsiJ_BxQP6c2w$NW4njV? zd=lG)-W*{6bC%VHfx9HkZv82REsklX8>J&0&P_bP^MXN)|G{0C0~{@E%WmHLoOp}* z#(^_32Nv)+@V2gKpTNK-vL$W31J9G`yw6`q{5*dD)!Th9`s$X*vltw3dlAn(`ATMw zh}6|B^Jl0W{=q$6Z=Lr%qZw%pY%d%G`wv^pX|`-I(7DbW{LkF<>AO!MuRrO2J{%hV zRsZ;hdAAM-wrrj0bNn@1lZctIl8;mEWG2lkPIcFr>gPArJvOiNKNNGhwAOvItjmSO zZetOhA5GR5uH0ypD`3otKEdWOS=K{1q4wwo&O^cx4);9*_%d#@CNOY6$!Pk?wf3%@ zTVfRFg?kJU2N;j+;CL~Qx#0!F1P1;y4QxNUF7!6AeV8btDem>z+g(ue@r{QC!W_8^ zj|I$Vd-o+b?Uu=5pPic0m*!~Z`7ArYl~c*~g3-7|a#!kh%OlKBmaQ!B{_r$2j`P0? zm-APGvlP}14pY<=C%G!4WG+(pB&QV*tLLv*Fxr9|LT96H*i~M z${do-Y2ZBaS;6Omg|XB3mkn|jjB)~vtvwYit`nkPI&g2`OKUvXEXTkW(JRYxh_$5A z*r(apq*z>})q`UrLWwjp9`^-Ngj)Er|-QG9KEiqMGvvvOW@`ckk>fJeX=ylWYZSN1? zKeXU>(*pKgx>pXVFdX{uHPl4cqQ8Ync}F8352J9&<2`#ef8agmqU(@s?ki_18bRL>g6E(N$04>Gj%#i+nOHCjUierN>&n7$K;TZJyos~vhm9&%m;`e? zSl%2|`6uI~IfaR@LgMXB2i6vLx7hg}B8=i!8f6U}Wd#nNT*Kg*xJc+KUxCqdHTRdy zv)Vkh%!?P>KH8_^u*Bm+l2aRxaL|$y2b2zXwO0BB&MwQnv!(0Nk>1M@=VbO&e0gzp zb@=&vwv*L9KfS!YNY%dfPr|VlNq1gf)2s&~noQ5T#jUEp#b|}R5Lo27#N+0wQtt4} z%dV7oPG+0BDr9x|(ub#3iC))r@-oY5Fv^%P+p$t&ZjuOR7oX$Ac`*_n6r5Vv`FUb? z6nuAVW#>zJ!1+a`f3Brt$r(WtmiGC!_UeTXts9G4Ifb+YHg4ga6>z*|ON>UqgCnXA z)eMz08yDuaNUQu4vk@o|cVc9>o?&`ASG(LKy+2GtK(D5WnO#W4!Xx3%j+V#VeeUrJ zj8>(uuc@od*w9e)y_thwN+)81Vz;RFnawvC1aEEae5?_-Tdi-=u|k=%7k^6>%{wvq zdq~nt)tL#mo-dny??k7%RUKE>3a1{WrPG$q*yTDstz@Cws;m_o-DYLk{Sk2C-?&T5 zImRPWNyl6wvv7jhnIdI=jRhS`L>zA^B%1BwsW`y2=U>qSJ;^TuE$!+VJW3ok5+@%r z@|a9e+@Pdnu)uWpt0D(B`y-VpErJFc4zzG^NgQaiRf~DW$M#%8vB_2}`5~w6hk^tK zhKhtX0qb2aHg{X+7^brQV-%@i*ySYd(YKRjPW3wrPL&FVOYW+B-|YUlYw?=}B1Uif zf?IjK8d(D`MxUROsZ+#n*ZXQS|J=fJlE-DrAFZ5fKk3~{_N5HblV0{(-189NvykBN zTH(^>)H22MQi_-OTC*<~W@W8ec}*&3#nx%Ja#v107PT^u@iCLE#f62sU(F(%*k#QO zrXQ31v8j9g?kb%xjFPV!mhjttda;<_Ua2yrgO!J4!2=GNjI&<1tWNw(QV3X`J z&fED+t<@yRLv?DxZa-sQ{$O9jvkm`(n|ZYI6~9I_bV&US>+Qd~Hu`FZO!1kYFO(;` z{pt^&)+D;^pm&p#*hG%msjpULm2BJ=m9=rB*zL@p=dRu2v)yx)oBPnIhGt%i6AtUS z7o^!V?_9Non|W{5lhZqRQcf&3uzK@i51;vyOAEVoJ6YS7!+fpXBku$f4asMKOI=kQ(O^7N=R2EmO8CB_0t{ z(nx+6xUiiq{_8uv1#2F9Pj~yD>FcQ5-}?Pxs=xG)6ALr`{XDtxyxrR4m0#x+AKE5g z)x#3LJF0WTy@F2rn5M>)RnnU;JrP;^BFUw0_qBgpH%)ZRNtw`m?L>n420^FnCCO1& z4;+;<5D;t2IZz;Z;{dDJheJn?cI#E&QLvWzlE5Z)!GTrJfLYe&AfNsjX3Y`>ag942GFKM3 zSuSa8b}(rcta`yL#Cnm(cE!8&v5U5(v zqE^x!^T0{`T!&Z3yJ_16>I|gszF;%#&Gg}u6K;_-;EeV-BzfV6lY&pDBDn@fWmFUz+$|Yc zEo&G~^9UT?>*LVz<id(*tNgjK&BEvBm4Ywtr5JkJ-Pt@2RKH1VQ#dlET3W?7EPIsmv zG`hPi77|i*Icy-T$?@l?o@#=V;+#hAw3|&L77`8&UJe}5HcbMJGaPx^1jJMu7ILcX zaF%0wc$AyPQK(#@k!Qz)c83=Y0w)EIX!=Yv$}&ji+*s(qI^ppRg&T?j{R}M%EQ#%J zo;?(-Jg`ol<)A>j&x46l8FLTwB?Z54@KQ80J;7xbFpt;9QRo80Q}vIt}1 z$RZXQkA+n+J1mZ77H`n$V=#LzDt(*7bB>a#17n9lyQ0NG{{9YzC0d97?b@AjfNes7 zo2r~ir_+mK1_^^{a(^6JRBaCNUGljuYv#Zto_2t(DWN5f&wzn>$^rh9Cz|=#3>s8# z9Asnu&^^J(n$gBYS<1sk$#}uV9%VCUrHeD#{m|wc~Q8%#*HQ5G-h5{--v> zkIhHHXZKHJPA6}z+T9!2r_X$tw`B3UnCHi?ALPr)t8l(?=fQ>! z!Hqdt-$L~Ry0*nG;mxi5crs%tD zBlDlhk6xg{Kxc)fA zE?UF1Cw;S)D^mmm>zk)r^%ji_8j|{JxPWsMC0XhuU6KS241Dc6-9c_b#yO(Qnx#jD8-cX;*g@$mw-Qq z9!mZ(;6LjB$1m}3$w|#kNzQgEGd3kQ-|AY|^Y-w*gRhk~uKTur-LZE;qi_Iyk3e@lO9U=Kjc(m6nK*$#G=U1xj4V=PeML#?Yyf$oVaak%>`Yi=(VxBB!6C5R>n+6s|kxyqT-A zCW$FsIK{kg#WMCzuisnrFaJAr&`yU*YMJ8!CLV#m8c&EVr#ffdz>aRHqrMVYB7IR=LInWi=Zhh={&8-DO3mU#DHL@Q% z$@ijAO2LucOHoKheHo9_vVR*D(()AMswtdzX1gykFzdezi?6 z&hRTwe<&IBS#nCF(1UBRU^npUP9+`_SCF=gp-?nSBdo z8rI6x8LL|R24exj?Bvg%VD$A&LW>iHV)1U4jwU1a>Er1L_XUvB{?$3c-l4$KS+ zxAh+IGbysCIk2TEu(vd@mnE>9wQ+oNklAk}>6d7)>UiEOk;AQlPr;Et?Ev?$Vr~g{ zwtGo_OpHQ*9N3@f2#6&xCY;I8IJ`$rf%(}1kp&6@9*qL?67*Oc*qh#qbfp^x`|aJc zqBP>2jo1MjmLdgN-r`5u3#~5yI`?pL(UfeP+!ubA`DZGcPIvTZFOlqe=rQq28%7-BM;xs3C$(02=<@`qdrs7^ zGFldq6=&wHGcSQt?E|M+0_UYgy#JgS#Tb%5H8^}fRQ`jLIqd;U)dB7)4FY}-1eT_nW4<)QFMX%|bvFYGphn^cd@`PG@UvDyzJ+;u{_&f7nVauyq zz2k+X3lw=0+^vE;Ztj^-$T#6u%@e&%lV`s>WMOiE@5n`we>x8f%@PE{K6IrhGo&2g zlR3;ktAVqkfxU{MOF(f;6@!4Eo4|od0xJ&wFj>Ij;lSdyfZgo@gIB^67Du)=BZ0hw zA_@xyR1OOHEEJlPk}&)6H;aJa1qVbz1a^A0RjND`@nB?~*deI0GL!qzvO@)vB${|s zIo-1TY?~HDlx}(chWU@f+siC(Bo8@C>KK`x+~(0cMPkcV!5iBIWgh;u`*f&FbLJ7z zb!DH0Zkt+8yd1lJiu5ErlsfzkorBnNIi zM^2w|&KC*-3DZ(p68YCGNf0`wf99=zkf?DNv!$2=`!olhI}Qsg8hGb8u-`erJHwpg zmIFJ}rl(R$MyXjwKVQ{m=43$N* z8!i={a?#f)G1F7v_hZy$QOrm_&;Nx%Ac|jT$Ab1}p8PonB|RKD84?YqZ+G)qD3GGS zRM#C)?Z97_P|lUeufXV8a)5zlp+K54lbB|^SSC{h`w6|pbI)ZnIFt!+DC+3^`_9DI zC}6|aw!qD3p_^;umaOZ2>ti^Mu6VX^#=>lqM|_+)S9FZ(mYUYRji`G$UE+;-y|^OZ zIzIDW-TzCzyxz0rFxO4*;&am9wt0)+%nuBToBCOLg6~b1g3~wF#p^TlcnK!#Jh@ZM zpV3A_f7y9nB}V}#M}C`AB7TniZxa0fFbM5xFilJ2S7Q`pNGw%tWN+bMPP)L*^qlR< z%GB=**sB^im>BuHQc4UHxZ4<*xf+-wk2A(3Og!}5DC~j7HD5-f1je>koIZ*ocN|hb zHAwkg@MG@HOnaqp=5@IA=brkhtOmyHIuC2Uvj6eg%&#w0cl=x5({jld(?xHX)jwv4 z_-ElU$)0Cj6R)a#WWn95-nPN9J45m%cP*T_xv`t~rocntg((bwy$aW_v=UcmH2<+g z#(_bBVcD^ldMplXcN#JlI&dp5obtIrAjpFMSwfipH10hK{6Aa-R1!JD7U+2#U~};= zGe}_fa7gxHVE19TU(vu)m(8rRfF~=%*;j=BQipAb56S z<*x(l*&e7HmoV~s8gVDZv^B7nG^|@K&h>|jH_~C=M;~Psqd!k}C{0b2{IPSxWApxr ziW`6K=)bRUU1jGDjf?q`6$@Ff=kFD1RJGd7`aM?G>V}G>v>GG3L!gy@LeYxJg(4dm zWT%U)?fs`eC)p`k*+XvuuhYTZX(~c@6c&AH5ZJ&S@|R)Hp9KOe3$Nz5B#1aLCp{37 zTPScSfvuuJKy4v=!2xaoNB*h?mW~^I3x4srxiimu-7m3#NrNG1+pll)xL31lGON6m zC{Ci$&Q2bg{39M77DURMf_U-KOpFhfW^eer4vOhCrU~k5v^P#`O|!-;`OwMd!M$p zpV(69RK>Wsg?)`V|2zlIHP;!cTs&Hi1U!1m$+EDj`#8T};(GyRyIckKrUdr7K#n#C ziB1I$j|L+*2G-nv3k9kch|EcpKD2UAxUH^z6m*2xymV+XC2Tu3AE;M2A%FAMU!z4R%Mxk^*<1GbNWyKX~Dw5L{D4sd2 zpt>ruUvIy}`0CU>8LPNS&pq7nZR0w7a#q=KjA-`~UxGy(O^T_W_fVqNH9z zmU1k+%0aOnJ^s%Z`1SAR$lE-58ZK!iE9vp?vUBZ%MpnKPS7R=3mASI;s_akRuVQih z7ID5d&x_v)^}iH5uxN|!%}F!YeqvhNuIA1*<)esGqYJ+xzuNJN&1WliXF9PyU}8yd z5>;U6lVyBUdrWBAacTc)$_~u?_KFECWan!<^L_!#I|adS4D22c#FBQDs3l(8b=Oy+ z!Sulziv>59{O6eWgNc1x^1cX;^`#Ahx2oraDQt`MD&=#UEmo(W(r|wzBbUVkrUeaB z^AvQ_ME0pPO7`eQ*~XbZzAiECphSTDzpodsJ+b|JS+i!N`(NHG2R45Viw%vPC?6?% zLh)JU|4pto6ZL;fUvd1AJok;&ntbj`!NP@{Gjw~GYp!P(SizvJ(^e#Kq>6dc0t2%K z=Kk1?YzvmCmYeT33bKG+KJBM>m@*RC43W z6VnuTcZuFqTH7sl^ye?OJ}J|CdzNZHe|!68(bFscDvNJ#dlR|!>5h%M<=fWW+?}!i z<<;(OYvS`ucTUvay*6?NuY_3+$E%{ltUSH`hxRDkJjBZ9&DSc#;(A=cURud5Cg8xq zh7K+{je-Xb2boy6WHWmxOjK;);NzNdVB_+Dg)UKUG8G>x*nMwy9WMmPF+Eb!xJ%v-T*PaxhdfK5=K&2{$XLyyPwH zv*8eHo`Fz{g4L6WXEc}Dd<$-o_7Lg~JY}=EC)nR~s$$47fzOJek+D}p!wW0EPL0Sq zz~CI2v$plRc|{@f^(cdRTg4JBk9jQayT4=U^@RGVSAEScz4>=hB0$}z4*c#%hzpUW%#}6X%M{zew~pjui>u(A%!D@u12ji{M5U!3hsom9~mBaVXeI1Te6y ztkpco!k6;pK<$dGhQRE#C$HRR{+p^G93UC8;S#gXh5|-z=?5&VBK9{=9pVzU6Ye!S zmG@#{yMmu9kI}`Xt)byD3x6Do&|aqzo{)WTDWAE)J=1q_IWH^3&F}1pn7*^}<<@oP zH*<`8z4o~rK=Xgub6EVs=Sr9!ePNI?>}1p=(Yk@zgDB4Y#!accVPI_Hf6$oo~dRC0RXLa`@eZdq-ZM zHM^N0%)!~?_-saUNM6E0Hvc5H%gH|@9jsn3_e~L;$#a}dnyL8Eu?A*dl?!UFY5S3G~+cH~mV` zDP4;d>W-|2I~thQEt`8PAbW?N?Xw2mr7Dab+m^A2a0$PNulICt+_6>E$H7TO!NqXK z!EIYbl^f-=6t=jztQETWR{K_z^Sy{K)8qx+WYY!3cmB8}#=l85CS5V{!Ld|}X>oNIFKF3%*e{XV8)r?@J7p^ z=_AK;fsPkfB)%jxF@IqYj5yH5Q6<1O%XI;J{DDK9d<$9(=Qyy4Su_fLYiMTfDdg$& zV3GXxz*w%utKO``@uC8QG^YoPk1*rqqdc7xBQ%9nBW~Yk3J6%$AECy~q!V?mYbvAU zKbJ!yry~@YI|F;t15=}}I34R$^Dxhi{m}F(NoIFmh9I*i#1CEI(C>V` z^zolby_pjZvCO+I!|qY;laj`#5ho?^jfv&#&oj$IR}`I{5qDUMU4emtBY{Cd)}V=_ z<^f;-f+ShtgPOK7O;(|Im=!L}>@b)!xuW#TUCw+iZ+dtEp%o`YtCx%VnW_Y6_b5~J7bX!9(XM&1k#En^JSQ;2~blH_3Flx_m+*A`L zC^$i(<=r!@aIJHq0v!Ubx&jmZrz<|<`^Up9+q$`3caB!3>4rw0qzR1kIXPi*r z{HeNJwMd2e*QCEpOC-4^9E_PCOrNf`q3H^nsu0JPBVKop1#x5@of0=wn)SF-uvlw@ ztNx2)X4jUkJ2KH(*8Xka119&y(f3$mJ2!JbZgH9MqHeMD)eWpdIv+y#R2;eHJ#bO8 zc-ZOF5zggVu`11|fpNNx0@J0IWi|7))I4QxkN&6sI&;upuH#1oH3mO`^&MaUv(kSFf zQI$43sJ5nQO^yre%t#l%DXQEPfjp?QDNiL65p57Zww&|8?hWlAM<~|h5 zH@J~|FYZHAWvX-R$4f7x=R{B6tg}wf%JLpK@DUB+r;H%kGVd3Ev$g@Lt_Y zG0vjfTsUollgJtCr6*l3uV%Zkka5|}MpYXnp)!dk*?9|^?RXM7{ZBM9dM#+Ry_3kk z{~)8R0wbs2#z$g13fhcw8ar68e-!wsQo2&uklXj+N-36qC1rCASK5AQ;4r(vD8A!B zv$}{QTN0Owu&ThdQ@dtN_$R>_ywUu0V2YX|SC9d-fX%Q9O`sp zN=D|y%|%MWH@D@9o_RYt+p9S%L0S6Njbr(%Wt7DmE6>ROdC*-FB6h-Z-I>Ojc~)x* zyRFp~_pF$(x#F<(bN>r7_Z)xHFonTney>mQBk>Cd7rdR&Zg)e4zkh?XJkvpD<{1b0 zr@uJJm%+KFEyTYOMwy54cBZEIPF4~-Q)ce$6|Gt2u z=|VH#vA8wT&sRTQ2;^I;Be?5o2lhrD1i@djgv8ZnpYcEXum&SW# zLfInss$CDbuQ>4TOWsm{LZzjd3!pzK#v5qR-hT3UEyn;Pf|OF8h$U z&4KS+1IHmnnX*vM4Q0kv3QVdBoZAYxb~@Bbm2vNSpix?pF!!5v?hE$q2AmBH92bi@ zr+;9zabUS^EORkANL+z|Nx`FNQ_>q_!Jp!R(aWtA1>>0|s!GGsgFckapU{xCoO8hn zX8%dKCj&V9Hk7a0Q7*|5?CF;0?G|kQAw@N#;+z5RRR!Mb2ePgk=pUGx|8$bm=WDvJ zlPYDuq^3<1+j52X>4nP03ROjJ+*cZS?@r)*xq$b&$3NZ`1zaW1IDLPFs47ab3o3kY z6|#E3-t>U2c>;US1@=k-&b$WZVu7~n4|tY1a7|X=`oU3hGa#YLh*_e5qdkFRp}~ZO z4HK3KaG9%fEll8862@)8U=x+VeY-*I+Ygy-1stsk#pfpQZY^MsN#Ly8Am}rtUSM<5 z0W-#*)9a6vY3Y1q5PQ&>cA#bXcg~)I$;}@)^8(WI(%57sFzRk#oH&8!=mHLj6pjQ}udG;OW_fc0?_GhIryDGOspVgE?dflBe*VZjbD{W_ zDSXqyds)*9RxIFtUC?`LL*=y}`MVmpWT>xCms-U6IW3;HS_ zumwzDcXnWME?_n`NZ9Vc)seu_*3d7dRx;a&(;|Ukg24p-$O+7uvld%&v@38JC(CFg zb8A&_Z}jBi=j2$V!29X|-*y2u^ACx?wocM)kmgY^iBZpKVaZw1Z1r&~i>+jpd8TlA zI_IvII&tZO!P`pTHC_jp^Kl%v;NA zosMT+J1|w(BEwU67@9u)>H8VNqe&D{AF}-UtU&YMn)sg9* z0=`98gkG8Y*0)JoDNLyh3t(NqC}_Z(^nz!y!>r#PT-z0RF9&eXG-LWFW)QHra@Jyl z3G*GYm}hbd3LSH+jnAWFk6iAp;YHdF&ka@jDP55zEn;Voc6MTcOOIg+zXt= zp3`}ERlX{iexsmw`GSJ9fVi24QEUzNpBbwq8HIkRGOXUXic!Ht(W-5^0#{qYqQVv3 zX9KwF3iKNUI9Yy9Fv(hM8nt-VN){yn1{VX)r3-4$7jp5<fj^_&5bcf6KryU9nTGMRWzHlN88duwU#3$Xt>wLWt}O0r>amO-!s12bFqJWU2>uG8~wD%RHAS{_-+dyQem?FNaIXBG#y z>Qx7?_^l?9rMTeEh05xioGUi)y-nbMd%*Ek!u0Oul`Ola^PFH*bF^YN@Z>I7CH;|s z^+Ni_>m4??r5O|$i~=}sEAVbLm~CFpwN-()s)#*)1G9=j(UO8$OUyW$1K2tZCahe! zb@>MW{TtcaAMjjxpz&kI%ytE?9TWK81n_SCuy|3!%%lW{Ppmn-%0fZ6))p!WCW;C% z6)>LtcP(vxfN-oAXT-*J?Z4NtdQ4vNA>?i#N9HfKpSRX$eqhT?*qJp!x3G#Ot6_uq z6XusE=aui8r=P&J=D~(JZBuJcWkqSGzB}e5_Ijh1O7risw6u#}6S>mWcJuvv!2f1L z<*fsbov$`s+c7=-LW@nApx6dy`w~Gxg&7io^-L3__!$@&0vMPW7+4ILCpqw45?JzE zgUh>`!{jovT7#Sg1IOmfSr40`6bjkH1>lm__Lt#$Odt zWBIV;+G-cR4K9WsIGz;nZo4qS$%|`K7i&@hd-(?TLxgLnO_bG8joj%^+ zP#1B4#Vdiy>w@grstMa}PP-w%wS5Bj;s%cNf^@~e?Q1782rCHX7%kONlU{a^G3CPK zy9WFxAFR{ey6(-cgKK*xH_qmiQQ$Z$z?mzxGxGrZjx}skxHm9lvphYy(_oL+0_k0^ zEO+g6HaeZKYuf>y6|Xa|GVp#n`ES?l54`KTkF-9Fv9j3xzcTIS0Y~NEoW(c!LjL4G zVBq7swcz>+?&v?geB!MV3-&N}2(7quKyaee;YbFa4K54^81w=-a&%TZRTVo`adk{s zvs7RS*GvvJORgmj3$|a_fA92GX2~pOOU?-l9kU`QuyszjvViYZ0&m*_?n?$)$qkIF z89T*~uHEr4X-DGHO`90^Gi z@3YeTnOT|6#+S9RQh7G#7J;KBn$!0^;C|WAa{XULZ^hYjz8aE0E^dCdBao-~Jf8wX z${k;?5XQO#$9_uho49x84~~6&ofEiaxt0se-gSXzyTFO%nH(y{H>+YMEZsR_*M(^> z3pm)8akow2ND7eRV>Qe>#Qyj(r= zy-?M(tnM3=0Hn3E^0?$s~cc)CK*mSfYk71J+ztek7Wv%lf`p$FWr54zWz30vIa_cyBduZ_0W#b?pSMoezz=CRFX2 z$t$^rEpn^&yqSDICv4-~^D=>J)-2xXE7_L{hS|JhNV#L+-XdMpbbE#(Tj?{-T^IiG z?JB4>Eu8%BN8j=dFMi&gzTgC7Rm0PQgewIf*h&vv(VfHQUwhZsXX@?TGnxyS7X4b=>T`)1=o21maqGZ?GxT;+Vd zfO}2@SNA$@j+4Fpb|2?Xc>8oI`_cy-B@FKh9KP-0VW0l`N>SC_x!%mFc~=+RzN_iL zUh;vxWW%%Ef9npf*}!9aX8En{@3n5ce>d&&>a>`!-3q69?>g`taIVGSrO=|Qn|*_QS4`Bh->t^FKu!3vREYuatqVModP_JYxvJJ2mwhLx zqsKY%14ktTM~ebS=Jp%_Fd8#>P#d7j8OcT%F^&I*j+WY3|)kxsSsxN^kF$*juSQ z{oQTzzNe>heXi#wn`JIsQt|K9la-Ub!}q8B5>h)eeSN&XvFF1LiievR#dvHcJb2*P z$nH6r%VO1n24=>R@In&Y=rap6j#=31kxu>cd&tDXAU_$yORjYz~2OjdOpUpRQZ(7YLBy&Pg!XW2I zgX80)#p3!wOD5Re-Bn^$zv=SL&&SU%_0CiDU3KJDq+>HXf7ooZvkwpFeAJFQvZ8SD z_I%~jwSSJ~7HrF2C8)}&x#Yx?1Aam>YNr2Y##EjZ)|~WUK`Xc1hJ=O%LK74;SIZTg zS#@GT5d#C8(rT{86*3Q)q;lsdx{3tuGdeQC^OTkMI!XSkEJA@x^Nc~WYDh{{ zz}Bk;Ri0h`6}uiV-DY~Zmhb+hfW!XEZPrX2PK^zWJD<#`e#a`lqWFrHd;EgrATRa3 zM<#l6RW1?m)>+xIr}0v>zts0tJK1LEZhG0a=x5a37s~mkZlRx<<>P@vMYE!#_rbRg11Mimp^VF1h}KRf>QE zLu9w!)}o#z{R&qSnguI3D_bTyb4Z#`bpG(S+hgmuzptld-_T^Ao|Pr2E$&eAAgEEu zyqSlQP3VMA(pk5jRSibs4;$W@cF!Ik)W4x4v9f;JI>|te=rcu7y5K&^HRyL*R-!*$zC@zV|K5}suN3o ze>+@dkhkro+pHKBpQZ!MwO?CQw<%spygr|82ZJ*+TfqUJyw7DiE_s3r#EuGbGc+); z&nRe<$?{v6E#xr4z_C+u zY2k|%NBHe3na^>|ZWdm5!FjUr7d8p8+AHY`f+sll=|9N$&%NmX59U4IhpUfFO5)IZ zF>S@%m1+wa7&#oaFXFE=YcgGvdRF@8k7G9qHa5n-UFC2lIof-PKr6>7UJYl-uJ#?K zy1I_s6$}~-Ok4*T*aZ>_LNyx%W;pU?R-_iC7%+;-rR&LDeJW7p;mmz)1Dole09IX& zCTS(Zj=f)A>sJ~ya(ZoGQNO!Ls^w&>$%a7Qm>Wz&RuK#{gbp;^PkmjU{O?a}r`lnL z>_}C{v{MayF2+(^#{{eLqZ=4y7#J->9tqYwXwFM|rfw>6+~PRr{Dn0R%o0Bm0-xKp@0$-V(Yq=8KE^9}!jCIPU)W8*`W%+LykgeMN5+DA zDatF><*2&(eCo{!ah7VXU=o?b)vDoQ%9(d3Lgb#ad-k!n7lON^rYsU!GJ)l?nj?pc!VG~H4Hv~P3)*d6BKS*oFw1fk?#R5O zq?5AYwAd$u#dV4k1#1P~ScrZUbL3$v{duZ;FWbckXOARq)fp~vTHgMp7Rmg!2O5Q2 zgM8Qx7BF&VFmMS9EYoj$dBU?#Vp6}4q;c$mi2Tb3Lo*}Su%;w1NfkA){M*puuHhuO z`)%_4Hs=-nZ6TINYa984E;KDz$-vrYlgT9&%3@QjcwTd@$kKlo=DxQ$q&sy(i%L!x z_d7RVMc$QC4gdUpZe^`HtTNfmv&L>y@7JiqQZFBDG2L~M*DpDF@5jy)OuBQmHu8R$ z5-z;p>Z)%U!F`Lfo(8xdJR-!E`)DvaeDYd+*vsDEO8 zy4Z1V_JyX%Z4UQ;AN#(qeMOVFg*sc!3KrEPYdTz~L{>Ly@fFGV-6@~mKe425%G^!| z7Mt`v95E5|OS;QUe(h|{d>YxNewCT`zoW7Pqf_Z={Yzd;9;wSd-C$y|kCD%BIkU9t zLz5Mk1tFI^&f10cRK54h-y>raslkw`J9_GUBA5JNr5NJE@l&AlzizVUu zf}QfScD!D4`Lu#xI)_8YBY_Bw#+|2>Tn|lo96VLygNi~ShhLf3`?-r+?rC29zMtLh zL+=8XhphRWUM@->r**WbN;q=5nJ_&}`gd6A-vf>#(sMp_zwlMaEZgkNdws%j-}oZ& z%8yDJj9;T3FbU3*zN;_dTQ_shmA@a3cRA}X-pze8^J?7R%&veZcB^=@ENwpV2GmzQ zKl$Heh4v)A{0fyEP7?)Y-iiZD>RV*~%$$13)u=Br*ID+)g$)`(>>T^F7P3FvgGA&`;Z#L1xeuVa%1YgH0^aYviig55zASUfEpK3O=h*BnT) zIFNK;zLy3YcMKa(Ok4U52R@$mbdL5EhJz^;Z9k0Ik~`KXDX?+7>^T$6ax$9b^zY_- zKXl&#1UV>h=`&tZ#ekC~>uGXBZAW#%W5BLPYVFAY6Q`22maC;<7X9-|D`;*r^p~d4t_g;rZ0Se9Mf|!ImnB4cY zW-c%-lW1q&(#qPiElQ)GV~T@Qx5J-K2ksC@?teOLJRxjhJnX3jM++DnBd53uuW?O| zIGD76CCP%#%Ym)Am_@>8$K40L?=1V;C!2S!=5z1ldlfOITbN1F#nvW!-M5pxUX7Z* zoA&x?m=*T2|BDxUp25D78qq&D z_UU|?_10RoXo2Y`Yt}8NnN(gh+C5<~j1UqC=C*HOQhLOE$Z4Ca%I@6;yZ0zGYedYI z+rS`vVcDrkY{d!=zH3;5C$IzwY+&|bV>e-C|02RZ$Kj8rE4K+-*qW2!drn4}u!YyS zCReaU`y3UqVT;dcaVubPmoPqE-JCdQ=6g=QhgI7jd#(HTB4SB{Fq53Xc3FicWsXBV zkC~=^YVw?@x9DZxl17$Ur}XaJUbmu=E$HLkAgT2qmF!AtELozQ-vuw|c5v4}!!hvz zTj`6osyiJMA8@?C%{Jl1z9I$o(hF_YH4}q3ux{+|*`Fb_yQ9VC1+$xn+4hL;k|6Gc z%iJ-+OczZUxh)!7wwPrGu$MKk7bdVyUDLuiyM?92cbW;SKj%@-Gro7HcqT`*&EjF7 zyT|k4nv=pN2a^ogl5V&rZD5Jh*|R8yrDe6vQW>}VLLwKl`#vQebDeqW(?`Yv<)$?) zZfkj*bZ_j)YH5D4>Dc0hwvMl-_|~vBdTjYW^U!t?>BZTi;h72(HvIDr(>h%I=X{sJ zzP@2{V zuF)N^f;Cg4{i;m6@DvtCYu1}z@3ro%|<~e3R46Lu5y1*KG6yC%J5s1!}4Ow zo|bNAwWqx2U$3~>+_t%;6*btiETAm$b#~SRKr9 zx8 zY+*O7{TH@)?YNkD*Y(L7*V$`475|=;k-ZkDW8*%-riI(hE+Q;#tDjS#O6TIB3hN~@ zCmR1{?+sGqFjqXYS?RzIebL}ohunM@v3%0l`L?Sqe-E3*R8uQcD?OX&=ZCL9cRu{L zWvb0re;eBcELYFxYaFhyWnZyan|HIB#LL?foPk?7VgycyY^sPTQS3gz8^fpCqWY6N z{q>EjChbKDtw{x;Q?|55*g8n9ZApG15~k3$uQ>98Y0xFppy(~GajxeQLj9h-xtUb! z*z&kJWv<`-MUL+!^`7lGICmqX^F~IQ4UEz&u4&sg$q6v!%5GP9a;bN6^Oq}q_SN@% zge<)e?ET?lnUQc^vnP1t%**``4lT{LJHT_Hs%Bp$&kdWc{kx_*FMYl5@Rjq08tj%! z+x}X$HvbK1R{Yn>ztt>zxAx!OiNOrqn-i9A_|YZt^R`sywty7@&IT8YI6SKsw0~~! z_0wtn%P)vR$&hJImMa>eI-DE^XIK4<@uY-dl8cCgT!^ zUn?H|YIOS4C^>^s=11&9*|?Rq_qvtMuUs~^YMAdek@@Y1W{2JLxl-CDRUdlI>TXeM zsPz9Nou&treflEYPaWu zC~rNBBh@YYg-?k*OA(ZNnEa39+NGzFqG5?*e79UXnnU`U7tZy&pK;B>s&5YCd>OWT zk~bP9cQndgc&dFh?1N2Hf%Cn*+L;^Hi2OV`<=dQ=KO0%^2D7#7hC1=hNCvkDSeGc+91EY5AwX-nCW5Nr_1wJ+h*j1=z&DjpkJ+*dsDGP}Gl zxg|M)CFMm%zh9!D*h3}VyCQilL3MYY`S--dlTS$5gKHMUhEr+cfV6hii)6{#SnY9^0kkJs!^CommEE>vPJM8ecolDk{Bv z_Mv@WP8)w|D^=^6*}ZT76O+<_^v!PBkFB#yUpSTdt?$w3{5PS^U#GudgQkcNTkr%Qr|4O^vss4eHi zyB`y_bj=7q5R#rL!OyMpB1NMubS=xid$0GUu}(8Fh-8z=>E1X0NLbEMH@3d8hu2N7 z9WQ$`@2#urhx|oLa!yZ+4NFVCwbbvnDj$b^Qv(Bo;!hS1BL)Tr9R>yl1_{OxMh1?5 z49_`gjN2C+H0Kc3ia8;WIL(}2-l;{xuzRYGnC`xv8A}%*?Ne|Lll0uQWa`A=RZBun z7NvSm(OrIGj%BK+?@X_ZpqZYkol|C-)vC>$5k&%#K+8X33%LT%y_`*Vb_Q z%+MFmauCwyJ5+I;Ph5|OYfj8^zk{czi*lVYYBxT1PEfsV+onL}<2~~JtdT*v>_PaBlG5VsKHP?@At(^Me>cX;#UY@f}Rt3GfymYzm zY}w99yB?hUm-c^NS@!2w-qSS#-Hne)=h)8!en_5~8^t z6WbcwG}&7C9WAyU=GF5tbC+R?G`s1@>N#yf`T|Fj_L_Of`ya>uvG(^1^_fUN1|fU7@4?up#ltA4lqL8^Leh)0gPYPnm` z(JK)VfuU2YyibTe4f_1_=-uh=oTgJ_j{Kc6Ip*BcC)a0(@H26&5OJkx!2E~SIWIxuSzOMy)xzHewCXQnFlPBtR~;!SpP-ZH`JkA zIB4mnTOr3~%tNo-xqLQej)?5xy&wObTK`C@C@S&b)_Y;`w>Gr$`g?rbsODC)N2D}C z(^53>sAIQyvDw=%^S5S)EL^)S*JmPI{Nm+;>ymi5-ZJfar8K4F5^v=<$&Qbwq)j__ zJpaAo`n0mz*-K}vQrYIi`Fg3eD2KYgkpJ_OUp9Qsa(gOwRMx#xTPQc~==F7lw|^Nv z*zqoR?}_Nroh-spUK@`-x_t1gMet|aJyQfdK1GDYh2LB<|J${FmJiNZKJFF~*`iSR zR)Kv&;@ws;jTMa<$1;6aU%w)hy8DvJE2X`UW0xE_=6L^yYUPQ=oVk0ZDJx7pRq4Fy z{1=U5hc=32+B`QDo8jZ|{(xV;43cF(xx%cntu6S22 zx9Rdpp9t35rP9xS`S{Pe?ux_B<8qACYg z-!Bi6kDq$-c-kWKzNzVUwhm3aQ$DTM*HoPH__gCa4bGPNZypL%8ypht37GPnW2r@U zQeUClCCj*Toh6*Bgp;LIT3EbJbh8BBx-=uq?ZOhn-tD_%-x>T^%TXOP8qa zVi5Yk{y6CFt1DC28Cn%6e%y3jQZwV2Y10$e$lXrvMaCPrw|78&T@{5r-!GT3-Mgt2^0)v1|lK8F@hr}i(d2<~AxU4l21k`G8>G?IZsL&3AGx zNu4SXqJD%kq&;MH@=e|3C@)nZotaCLB^k}C7aW%R^P%nE7X^-*Mm@Xo1ZKqx2Mi}f z9^|r^(DZiS16G-eg96928x=hmn58~2a5y-yDA_1-1-@_+>2kQw8*+e2u;T!WnnBAU zxiFd78%&{JK26zKqM6KdJLdT+2?wnujSUVj9AA|=G$(0oXj~+^)>&rJiY|>)Wr3Gv zvmJwMHebA$Q~mL5dK#r(eFYC(o~@tudS*_1!spP5 zW;^`2Ze32^H~S1HrwU)b!>qO`o&udc8A`vr)-L9ox_&>yvn#Tat8%_qgxvjO*zcbD zQQna?`^OxvsjOes@70|-AJ;#1S6-|v`wr`t#N!N{%U`gW?rQAIo0G(K_}+UK358rH zlLn^bcb}&3zw=bjJb@`?TM?s~LyKwgp|kn#j?B(9O z51xHqy0fB0<9p>I{gppLFV=D&vj5Q799I7DTC3qqALSFSIvI{Z-nvIr&Q13VpYtm8 zbD{p0f4w6Al0*z2{e5ELW7$`JWy`d%v`h0B9#OH&z0lmguas3Q;NS*T1;#@01_lTI zste){4NNx<@L#q#Zn^JbZ{D24k~t3;W=d~pSQUQv*8R8!mLP%D%=g<*3riHw(`IPW zt#K5Y^MXa<%Gx}?f+XR&3s`p_{Ju)EmmSEI*X%Ovu^Xu50zqqfE2m%jb~ zESEU4i+V8JNnx|vA^vYd&A(+1ERpv<)g9+yP z?$;oG6G0gX!3E{4vyZdoA7GoaoK=FcNkM}(>;Rj&Xq9f6^vp%#_cyYg*eLFB&1;fd zVxB7Vt))!C#~5v{7pERet(=yAt*piMMlOp($(`%DiR>kn%W7iXN=}HE@hUKyGH^a) zsJ(cd_vQoMa}(ILA21|}m4$hf-WQ9|cu;zcEsy5{1D``)!U4{N4P{y8+yMtzV?MA} z%_w_yjH}=qt8)P3-|MW^wmRmR#0%sjN0myp{U{gLndS_Vlt6j`RXGj*@Sz@ej)82h{&C=l0!@t`NZ9 zUSa=6#MNh!VM<~})kXQP^q^8E6Gw%{r=KLWJ}^m~2ojhs$*&?fxlvGsqiJb|Wb6V~ z<&|B<0^OF$MS{!xGeyI#1;TSh|1sx_GRrnpTV3yddbRk~k6b79p2O}fA=_IrG^V(o zXbHLA6MCb^yQ9?mXG?H9qYVS+{Tn4d$9uiRdFvFIQ^a~NY4qOT)^|$0OymQD!Uxv5 z56WgA;M}}{`;7wM?g9=*w)mPEWr-g+=NWJ}F)*d1cRUi{V)`a#Wm0kfp|CHDZ0B|P z!cDrl2^9xfVg+wj^gU4rU>9~N??0|&Xf@YHhWDEKLjYGA+5^F!28?;H_T3TK1L2O@Q^M$Mk$N?)VMUcNOq1*$`oNB!cs#R%@nQ z?4^X>X~z6z1%j4|H&PpQN@q4*66k#7GiN!c99Prgo2)SxSQ7-;5;nBQUx?%Tnqqcg zk?yOoTdv_dJZG0ooxNS0dHeL)=9-KS8jMN{`eL@FdUe#C`q^^xThE!3bIxa!hNmxa z_h>zNvc{)?>*j@$s~7kl+~mF8z#deE0 zG;f?yvM5sEWaLbZN}Z>YZ<7T4HY+kcTzI^QHQr&-Yd7w#1*_g1naH)n5s^l+Jyw>{6OGL_vUwRuWE!wGCsy>R>v6Fwu0G5(W#&qW zT|WGY8zmmH2XA1Nv*zCVfqQ?#CM9d0y$!qe8gQ2`=Uij8*m4)stPM=Y(z`9PH}Bwz z3l7V1bPS-l0Zn>J-dTrVo5rMUQ3XItgc<#+u`t%>eysH~{ z?|)eOreW=U2i|iE%zPiFrnu#XdG!81TKxDL>nky~7z3_1ANY1|U=99Ycezj|HZ}Zv z6!+KVd9EK=cP8-ea!@hQnsjv0G=)mT}mHQ4YHtx1NM-rJ826xkwgLCP4cxICJikm>B_MbO!h3g%?!lYTSP_}ZW z_v1{ND%QDA1Kh%Iee>h_wB9SCJUIX_Qdit zEU8$Qdf#lH@B@b9cNhdOFbI5Dlg8D1k+b*uwmCjKra1mw<1xGT$qS|e1ivWC7Z?7zJ_xB)=d&MnOVbZ{())k8a>f$jj8JF5*Jw` z9VRWl$+D#)upjUx5Km#gvm@;tQRobdO@|} z0p@>S7-R&_To*qQCU)eA&ynbzQ*%A0oLn*I?9Z*9rxB2uOz6RV(YlIrSy_=#%{fkR`J_OW0`n4Ap&UKVBUTHh z>dDVN$n}s-@S#tM!=fz?hg5yI8*Zen(yCt^aA@xb?zI;VS;lTQjNZKYXN=hc#vR+4 zOb;};l`R$7;=+>ffP0NXJ^NnP%?r3MJBT!Pv$R}uF5OtPW|LaY#vK!4E>2#eAh0vC z@A%1Sjg9jH*%fLxay7DwBsMMJWZRsuYySc6tqZu{JY$U#VD%MXeSeyJf5M@yA6VDy zVsc7g_VwFr{CZR7MD|(*X4wTyh8LJy6gc}ESmXuRcQEV=ZD0()R{S_K^||gGN1pa0 zIuYJyrxeVnz3F@Px=ju50Y;MxV*j)n7$pK2SRXKmH(ZhZz!o}(_tpa5iwRtY4UFOe z%*_V8r`>q3Cag7{z&KHW_w0kM56W5;9@M>h#+9|5^}WlEo>LZe?ksaY+;~@Y!9{>; zm&5XduHGRRBidIbFZpz0Ld~O)3!C z0N*}?3V6JuGJ%52+ z-husmz_tDf_b$Ip_2oU{&aKwMl78V6Yg9+>>3%L6l z*nI++YyH|1pE1Zburw5KUMk=YX<(GS!1&(p-`1xEd=D&nYbP*zJMf+jU}g$nh>m2C zb68Zt-H{-`%3tU4IguqSmBsf1%jOTs&u`Zsy{^Rkus^jIaQ#^tXviw=5t@uyoEaOV!jEmhLV}#uf3eRRO z*>KBboW1mo6}&h9Uf!a5da8Etnv9=D-if+_3P~AEiHDDI zX|{aW!}!Ggq_p?*6}Hh#%x%4nS657I`sjFWt-G0CNyo+{)?QhYmQx%~tQ|aj{2Vp~ z3l_1<*o1z(QuuH|tFmXRPQrwbFGS=xnWW~#IxKL!#l~WAprb@jdTVjf+0^1^CwTcK zWc6Yy1a7?THa~l=)H-}^v}M`VQ=zlPGX0I+WjI#&J~-ecY%trR_Fs~#_mt@xvpPH+ zo0hPPYMO47=sa{Xe7*l%i`ttEk0z|kFPxJB6MFPYGz0tK}6~ zda*Hv|B^Dt1O>aAHv*3yv383WpP6qy|LLAzmXY`N)HAd16K#yzl(90opv)=;bgH#HW`O^i8*}D1DW+d`wt=f3BLvqrB z!yN2ZD-t;^Ruwe&IsVcx5b(%rOzC3cs%U7I`lazya`v)?$0qV56daRDpY~$`fBK4r zY|LBFWi>Fdt;ku>Y&vO)_r|<^Hg9GIvCT~GlK5l9&@6JwBK1hIT=H@m%iTNwu^D;J zS;@l8ey)pwiPi2!(_}w8#@=a>2RaV5U7xi05Sz+2g(mK(nuSk=^$ut(YT|cz=*z9# z@_Vg?%8ZD;t9MN|7G9Idt60b?c478o-tU)~#q2EZR5af( zee$7Hsf%Ty>%zE-#N!KN=k0Ksth4F>6Sua;t3@|kG8!K>OSo-nWRsXykR}z)6!}O{ ze|abK5veB?YSPiuHYD~3$;>>+q$=a+DjKsyP)Iy<-k#Spb=V##uvrGIJm!5gr}H6G zG@rMsK&VVHTQmErm<5g6&Up_Qr4&v+GdOzcr_m$*?TtO^R!e?3w5cj|KHE@~aF6ZT z@iqS*rp-!U5O{=@owsp&CQ+A?733?(boYZd3LUax!2$RXeK;@xiPn1aSVmX93X z?~OZKz4-d~39bDRjqgq?IT)nP^c7>PWxP8#l4VQNKCNXBwu!dPYgJkznp2_JCN-g> zMJoEq0VUDd`%F|iRqk{&pGey$c?ai@i>2lqN8<}oJB{WUA+;xgKeZ$91vvc?5NE#kFrPwYN%<=KCVAiIGZOLWV zj4ih=mb$&*nC8Dv6Pi!^Brx$#X!zOjfYpHKmGIgJN3R^X(5hkM$mQ~4!S$?%-H{HA zT%oHQZZb{m&Xr)wKIZ9nXNI%PZpr5sYpVQKuLzzSQP^&M@1`SP(Ss(jJqoOQnq+FG zMC?p5{b>7bp@;0Qf(fh}7m0=6IjXZlbwL+LMi)n+tFGO{iA6UZ*!o@_HlEkms%q0* zcU9x4{J#wxk4qm*7R@-Qyy9YSP)P#k3X3z|vp(vDcqmCJpE|_);KP#4F9$gqHgK9& z7%=cKG)eV8lwfB`n6}xVfoXbS151TM=daSo(t3;I%|(h&7R@*;ll4db{Jt&5mRBc9 zt$5+cDYRkQm%s&#oEi;`bN(@~`L!%G3%kSWzT*RE>m_&01YSq5Fx z?2JWCRS&tsL`uZAUQiNWy@9ojX-XIKQ4x+91s16pf}HgoZN~o+TI?q@>D`^tW~^Ym zsK|%8zfB=X%PN7T!;y_YJ)`Z8C%Ah3|hgBTeQZBUeMi#Vh8F z@ootW3O}4!?FHNXDw2YBFK|^q5b)12%4e-`)l&ryE`tW90tfCTGu*@{ebBtJfQ|E9 z;31(l=2bgn)46J9_uY?E)s`32>u4`-E5d&Y7KsJlenK?E9%3<~4CD|CY~l zWllUO(D875GHHd&$-9?rf1YWQUenNOe&M*?OyMJX2Rtr>ZAcQ^!f`?LnS*>=-a-ZT zpRNUm9aIRA~!R<3LK#WoyZ(EHpV!r{oYEU%kUE1_A>=51#2U6)+5jh&9I z2U|jaA7JC!pe~WFD1Pk*vydIDvt--kO)HrWrB(m0bU1)MC*{Co=w za^5}p;*b~eLRui|Y4W*$8`$p)9$G8J?tjv@@8mMm3Ds*JXy=Ch+gbAVCiC+K>6ow^ zU&Y%mHHocjXw|J*c$1w~S$m!Xi}5K14ZRM=54HTvl2;n|s(o4o6AYS_k395UGJ{d< zSa`2W%>h9{?bcbpHXNUq%)s|d!t&!($LYog7H}?D;4R-~*PeM}$<><@E$n*E4V(-L zwIN>CJg*cu?f<_|O}^V^p}2d~VYZTho?{ob&QsRmtl;8rXihh1&**4bvtrWD#k zC6ttm7FB)PQ*+t;;%cM##O6ze0`;$$0wZM4zhu>xQjB$&a^rGS+T)fq!GCRW5^Zb; zcUY=OIJWZqMHBmVLV-r|uJER+ubWarj_a zk-F#u8+jew%d0k>mOP{>y|->vKMNyUYe8$v57u}Awu?6w?0Cd-M_MC}p?SiA9sK_U zxb#K1Bmx>A6>DW3XjPfODbc|^+k^Glq1K`kZ5BHk9yoePdNitbu-Q#&n?}ZZ<{$icQi;AwB=N=+5c#mWx?**z{TZY%07vKS9OD~MoWwYOMa8vsV3EBw>iPAd1E3| z0HehMCN6GOo``0z0v5{$jC>a+FJ5_Wo53um9SzDKnxkj1#Y!xUbyym*OE0UVnakt6 z$w@|6h4v#AGozH)U00kJP}piK*}x^i#Kpm|Y0f0J1_lm^nQTtpng$m^3_ZL|!y! zzSzlrfI)Rai+e)j=6@3!%vUh6m2~}A;!}Chz&D{`+m9Jrv`=r!;pQx0EVpD&KOv#g z!N_-G@++TJoCZ>pZ!#qauvL6$Wt%-&e_?A_hKd7QSo+-eiX0RECv{_iGw5XzplD-_a`4a+O_Hl)d-#m4eyXEqhk(<L#H4kaZo#tNT8xGkEbHiX;0 zWLPnQL9n9Be^SGQ!Y+mle*ZRK*q~dxK$A0;{es;Olcm1etriWNxh=TaD-?L58JH#7 z5;)je9<`NK@ccZ;mQ&Hf7hsm~AXZ=o^ZZ5J>u*UKOk@#IVc;-mFkoOyI1$Md(XgP` zitR&O@Kme#MMBaREHM>qD{PsLRNsnR>$ZL7Ey-DZQfIF2IpP@gk@4)|sbU_R5^9YO zy6Bn}G}i}5m%3auPhfuWf>A)D$+yFOXRG`9jnUy63zY;o$E@!(%WN{Tl7EC%L`)(DO^g9;{Rft5KAl5$S4@=GxA zIp0gDXi7IY@LzdB=qKiRYlZHyvhjVsx8Uo&w|CWD zUTb(qtHF;0QahTzZ(;TBU}Ro#ZjNb#`VQ9Ie-=x#GTJi*4m{k#D$dX_XX(9!hStO% zS1jBz*b=l`k3HV(p{=6OlAp}1aD!1*;ECo7M(GX5zJ^|qjGB6~Su}R;)Ppw}4;QMd zBr_)j_D{|0|K@SLNmF-_Tq`PN-uP)@Xb`iGhCy1MdX}o`5t>xuydLA{d`0${m{&*!qag zg5hrvqveN{`kCxAeN}%iY)hNLV)KGgw4h<53|mA)tHFflwhtS4wl$u%h~PJ9(6wO6 z_2>ymY0tdTdVF77%!@|Oga$>9Q09)-@CmGZiyPj~ef-b=>znA1*K8b&tOiZaKiXqY zG_m|UrTe2{Pv(^S5%HP~Ot;(ia9#~vDv)_-=2Te@CdC)0#SIo27QZ}^`0}*j1TEu0 zX2Ub43_E7JU7WqFDN0dXfT2dE<5-1EQ<$U1>-lp z8zL_ncnr?%-_gJ|p+RxS`_y?1%Nx2f!;S^A7Hm0?d@Z&q=)$W1#tZTr3|4(&o8G|| zUD0C2!6cBu8o|*P#&9q_ulr|r1JjBIB@X5&0rquWGqOFFX*+3XD4hNkyp5}Z_1rrH zt_2LI&$-5KDg0Hfz`?;Fz|b6aLFsK>lfwCHQWK(H1ivacLk^(Q#iU}k^I`%Z23F5(0~QenlSRr(*>a(1kna*-`|LTf@tYZwC~ zQ$k70-zDn*7z`r5#cpWJy3uO5uWWWZYa|DA{Hlh8ijT|v(__~p?O#?E(b1atqqOy% z!IRe3mF=zJA70 zcrKPrH{jMZ*Pk| z(8MaeOna89ldX+NMS=sz3H4GcWUC7giFqx0-6rXyr(<0atXY- z+{nUtXzy=6dFwomor)h=yPG4qR@Gm=@bGtA*qw+aI|`M>{^U*5adZkw^HSK;BIK=q zRKVHCe(#dW9@a+}T;?)zIWeildF=&9cK1dGCP5dEk1T=;H4IJcvMUxau}Rnzgfa@J zES<{8;7ZXkOQ}-NT3s6 z5wn@a6sTL-^T_PvmgcK{>0j+kHXEmT9flc84Nv(>p$5-Kp1N9((7cu(hkr5-~W?z&MNXvt-s` z1DRu)ccnrbbvCdhFK4WO_05IXT|zM?Z_UZmVEQn zfAU3v*Y2l?;%!5TR2MvSx582 z_*CxhlCpreQM!#H`uj#@%e{q zSW2cwos#|apirxY+k$7w7DtYi-|wvy+t_&2dn#Y74`Z14*tLj>nrAd1k08sn;|L*FA34jyT92*AdV%?}5{&d6Ss54mh%< zA86)On9v|Iw_C^h(!|Iui>97PT*n||!6c{hktgd4tNt4e_Jma*mn0NC)7!Ax%{1Zo z^z18#DyDrrt>?E~^6DQpu}us&1tbKTdUBr`FZk$NsC9UQ?!RSbn#>_OixzasWjG1D zR5+dV3u|^Y^s>A#k#EU~<#SE`}Pt#s?n zw#hKKBVlXs#zEL*N8C59U=~+Rcgu4ZBhPJGe6csK%X-G8kmbjU=ZPr6T@;jGT96`cwhk2nvn;NjRN!16xyO5kH{dH2naWqo;hm)Hp~{m%Qa zZ_+%Gr9sgzR|mMthsNeE%U*G)@VY^(w#PxPpbIXZIfsH7cdk`scS;SYPYO}I;E?02 z8?Z@Cd=0Xq`fO=`Zvvb^{zuj z)3(ifs%=sv`0U$j2Br;9nIs*W6ax%<_v;i&yzo)CjNIPc8tEqe|HYdRKFv0fC!YB+ zDkx_^5}Ea=g^@*6g5jvc8EK1ahuCI4@XOr#okcEWilz8lN4NEh1D&F{GMucR@O6D# z*=TY}q0fZFJ9L&{_}lfT{_p4HH1{J4LP zj9tRFyKbCdiWV_GV8Fr5W|G-2w)iU}zgP682%AK9(X}gz7b%~(G4K6}yMp}!E?N`B zmN3)>Oj~j#qH9%eieo*ajd>As#U1V42YJ07G?oZ=2noC~Q8$+q*i@vteZEV9lXin6 zPlCW2eGk*=$|13OyAqg{4eoHpO)%Eq8PMWzgz@6ihDHXN+OX4xmuE`PH(`8~mAO%5 z0V8Vx!w#wDM(#Ct4$9teV4I}Sb?^*Z!R{4D+Y0wR%uY!2*}Q{^)$jGuPR(n{DP5PD zR8HrZ*eaSk*B?rM7b$5W7;SErz$Cn+Vek24Of2WWDNmSaFn8Y9oW)53QMuW>_esB) zHnDfgKM$3s!SjFR2-$C%baiU-t`_4A6%~G)3!B%8PNBub@`zGPIeGX5oio`0@w=Un=ng8;@xOlW<=`bBEDyj(eZy6(8YCd70DB zm1%K5htYRZYINlW{>d9196eZ^7o1wnaKSa?tXss9ooi?B(mW^>;K;wgWkUmlSIqe~ z9_~9j9^O-e?)>9uq;;c7*W$3T#bJ{dhY||n#S15%42VDP7|$BfkSoD2UK;3hlyf#iwA9;$*$>+6 zmI{7poY=wWta$E#;FN=Z{&Id1bv#{jx447<=Y;+rbNby=y(Yb7;GJZr}lcDxVYfKh7U=z_8#$0=?HQ;sL?QQ&dz3| zLx&YL+66rvc_kXvJQxKn4(j~**EF}#N#swX(Gn-gc}>O-nDlIz^(>kb{g{)~p4kK) z7T}#|zfW0fN>j=_CrukBMGGgnAB=Jv4yN@f32Ph_Q8?J1a9;h3lXecnx0Zvs59XL= zH0w1ma;)%VZn+`gz<+eZNv4*Dj)iQVX6HKRx}D-_l-%nf&*!fDq)G6Abf#*5!3_qX z4c9ia3aPps|Ig@H%6GQB!F|%i_Pn1d8k3aVdu~thOjr}s8)T+h71k)Ca`38vfLFj% zog?aDmlk<&xR(E2_b{Rr5mrpUWX7mP7I^O)_r|3avRP`Q?Bl1LL)iw>ytG z@Sb3JlyjQdhJjV%1~UT#Lrh;+X`|kgLupHU)M5_W+caC69O8;dVU%F#Ju}O-uC>R3 zf%%Nu(F)et4^FNRWMEIY#d2kt+MmPr8O_B;jG3H{vW*8~wAlNv_GCZxQE22ZVTk;7 zB}{m^bXz3kpqPr5%$i1>Jja@CPBnFotW!9= zdRTl7rZUfE(bT=VeocdC%n6-|;(9NI^jG-lx*Uu))6gsG7tCQ)_sIwqWjwrAgWo0f zKxxC>m;(%BPK5G33t%t{bda6VY*N#}x@OKY z-hc< z`OYDeFAlsS0se%G6wGLBiAq*iInNN$5cAO^(uA%5$keBIR!5iwH#~g3sfu+=l}S@CWm=M99R+@c7&X}S>?*{KzmEwYnC1crj`Ro=lnay?7+aJz`(q~VRzgaj+O=v zgVm2qxM!c?_!!V2B5+98hPi~*(eLWTxonI=ceG~zlK3CW?sx4aU&eAJ&Li`@4%Aqj z^$L7bqma5wRBM-EYSpUc#h#4(YYwb_)^t*+ue;J=pWy_aD+ktoa^SsjfbUJ-2U*7_ z^A`A)H1MQ!unRbG8H;IJg4&=FOWx`VM&xt}5$n(nUV%YEOtRd_E7b4d4rlkqKwWhq?V|GqiQwv(RIX0UbP zdG|&U-YLf&3Zylpj<{TDUvsBl<>C}K9xYJ?@mGt~>m^o+_%O!2dnYF2qS zMJ+ehIGi zn{|q7;KjHk*_mG)ihT+!1^Cy7Qo_hqJNAVPg+w^?qlQ7mn9AWpkJ~@Y*o# zKNrZx*4Vp{Mc_{Zdv$;R&S^pnLjRXC&W+=$xRdEI)17+*1NVXkju}ecI}iTjxqe(V z@WC$GaNgaFqH`Flv>5Lft}%YbC79w^b8NYWOJmTPvmPu5B@Z-OU1F5J(5M-pCVOY9 z@ti}_D|EbXoMkU@Jl7SVz2T5f0i&RVBd<#1>6}K1%~2e+P3x!1uuQr7c*jTMo!!f( z-f5Cty-YIu*Q!wI@XV>VI*bgT2}LmQt1z2xILPDb;JAJDojg{SGaW)Ehix7;nOHcR zcsQH59M-(kz_H`rbh`$QF9$75nALoo)GV0%lbuXrn7CUUI21mx++a|Aa>&F&IdkhK zwFm2pH)uZ5Vn3G>68iXwV&SQZEpu;AZP;;_-8-<+?My+@T1Vd5r3@e4MFS49zKfNd z@{iHfIovy^QQ^rU9hoML5=S8x?)`5sN?9}+F{G&o95Oa>*1X^-zu=IrNwek`hgr7w z*hCJ_PYj*QYACTHOSoj7#Ca$Eh{v3{9-FNWC}c}zDwj0O{KmVK_2~@PzK}0GGuapu z4%p6XWS!G!GUu?e%7KWxZ)KeuSUMayeU6@b=$II-rud*yiNQ(5#@Xmk1J8>FzCR7` z{xv9h99FvFa;oZpz>`L`#SQk34NN8sL7!iX-pLX;p;ThIyu^NC|3yuqKY{;6Bg?Xq zDrc}8a2HQ*&2zgW>(2AaX#Tsx%s10?9VIoyO{|m;R2~p}b1;^>N#V@F@Jk2yWDZL0 zc`W48Wckm|*!i6&(WX3-10-e(+Ast;C%hH`v8q<_U}o-5=0N=Nl8Oa>9k z21nC+I?ANphxZlSiI~Fugu#+;t=UtZ1|CxeCI^Qo35N|$=VT)e$h$a8c{#9fFvu0N zGMsRbe!wVqq0!ul$#lvgo*DawqX!vcme5KoejWm`uaBP?oIH^=~+9J0# zUw3tWWi?CX-ELZc#Y^}6QnfV)-qjrtl{uv2b4XE!NoY%p?~MceM;s$=-WOU?oLIF( zk)=uZ1mh--_61zmiL^wa&Kb!E z*PLin`ll?%!lbgny3P2TF+-E)gCwV;yLiI+&~C?xoV@z^i?}5e4Ei1Z*L@B6_tKRo>*ci#XPu3MH*ICTW+d8kjj6L& z@Ju7?WG2}+2iUJ1@&9r_N`qO&z;T^=qq&unDGQUHgOh>YtGFCsDA>(>;()A1lhKVsax0pQJq{;|KePGO$os&_IOHG?50jRM zv;37t-36QEA26Apm~6S{pwOHqlLHO*t_~3m4r&QCVpkg2%Y%4z3f{f@cl}|{4bxvI zxgV50YFN^|h@(Z%eZtpTmFrRN_2;tRh` z6KThF!8f! zXk1uukflvn#!ulu0i$!5kh;!^9sdd$8GEH{g%mbiOl9j0xV4Uo6W!}lcCqB0{9nNpayzKU(d1Z9oFN1n<~+ekF1@pNX6E;K zR6TQIY~kSLC{Rc_b!)1WTh^Bi3pG#l%h^`kd0O~HPgudWfng$(i)X@N$A%vaO%@)H zWu40I?D)8dm0LtfWk=%UwBD)bvMs_F20b;-yqG<$BW}~Iu1T8SQ&a+#T)9Mdn&j+Q zuyM8<&;1?AB?}aj+4wD2c)50cSw3@9$c{dpgRQTwOp-E6l9~8ejZ@gcx2xmP(Nj~k zKRvxTJ^kpa%gdD%}nV#>0a92*acL~|6n2&#SXcqU@?#GrM8 z=bW9a)6-u}csMb=DuaQML#^UKvyackW!%y^7glfF{pi%|4uuIjRA%L4T-GwnF5d49Kt&u zo|nkFkjN%+r$I%?qGrWn5zlD>ifyV79GuuBDpo9HGxvCz%;^!7nKU&pL-PEduVQWoay-J=U7_VT(Kum(@=l8>D_Q!2)iy4f=@>HM$TX{j35?Cg42{j~Dm|-|`fFtu zDzkYkVmzetS7O3}e$QnmRo1EHK2YS?^Xb%Uy?qlH7Cq`>e;kx7Z}F_6T}GvNp^=!I zhhvNBhJ@ta^Y>y7n`p(z>6&Uyo6+>lDlyrKQ+2`$-!9o1OB6Ze%Q%dEe4jZrajNyL zSZE;|6XD2dr4-4;CR#B0p_N3M#Uq{ou|nlZL7tMW-2OI_{i0j1XD=1j==$)0iJ$LK z2Lq?njcF&?lsZ=I@ATs``*1*VQ9_eqN<K2UvuDHfLBY7s>m!Iulw#pd?6lNuM22XKv z%VjvCsWowq@)8DSi4}}8Aq*UuYZ5DebvCd(GvJJISwE?wa>mPb0-Mf8Mgs{K zlYdVZv}7?Taa?@CDE70MrPw5y%|=t}g-D6KsL5f;D;}+!-WPd1UMR3hd~o1r_}E~t z;#|7C;FzrcLJso~C&4+d9N#c(WOA@r%zOKi28+ssR%f5X{2m=C{Et3ucYFs{$!9yu-CC7d1h|_Ej+y;<#S^DbO4+Im3~xq_BF5<9;kR@0-fBc|n%y^)k+VT8$>i~1uf(Xm1*Q$(Ce(} z5ZI+A(I~QGLNl9=0HdMxfvpofG8C2wcDjCO*uH1$3FfXZ4HFr@J9xMVNT>?vdYovK z%Dcc|`srT+qf+S=MymyF#%T{2desgn9!X+g7E^ep8lvQK!qG`nD{!`&*C7#!56y2A zJ}_HMdB|b4z(_@FV~h8lLt=(g8ReERwr}z{pu>0KkjlJ+AuR5b*zFuXe0?;@*5{p1 zawf;2FyY`t`6mfJFRUkXJb02g<7nrtja!8OWogc4IF^>~y<=(4uLKdR9gN)R8Ha>s z6|g9uIlvJV`%!Mr#U9ruQ$-eWxa#fu*qa&QEIsqX+4%_!%@R8r`6paB#O_znB0T58 z2A>lLQUev+xfQwIK=cGY+x&u1*o~QTKckE9Bj{aaD`0 zr=2`Uksc+QvSQvW{oeDrv5Q*g?KJjQ#9LOu^1dF(diap}F% zH8XIU{NMpkx`WfPb^#ZSkcIgr7R=(o9~`cvEMSsj`NJ%);B-K{f=Ob=0T#&_jtcVx z+oP`jRA7DKpc%5UqbPt`)KufZL%tA|heB5ro9{_G*ObaDg+5`GxNtyd&c=@PDNJl< z3s{~DC@?ZuIIvzUVB;4F2>G|DeoBO3L4&kgNUwABBf;f6m=)z5h3>3oJ+V%xhMj-L z0hXu(f_jN9*BtnwjtI&)vi)+9TElw4<-v+PUhz2#*?t^iD`QyYvSFp(%2f>ys&_oO zA>sV~%yOeMDy&`&b#8|bFKd{Zwv0wJqWFlLtbR7V=LyDex_U|3`vU(L+hThmtalY_Bp|lk~qOC@`<#V~cY5 z9;Cp0PJoT+pya;>W*&zpe2*0R_{EdXP58i&ui$h}A!)nJA%5nKDp9YMxVj>O4hT+L zz@M{_-{uM5B1gfprz$}W8q*rox2#uG_{UrF?y$2Mqv#UHFS8DcePH1Ga6r&!p=8zr zzFCSw?;0-Mc+HowN<=4-_nktiT%y1|1)+J20(^yM)+7k!^jx<&x>8H?-LrKDGcK;` z(_Jl*yjn+xo#iNNOarUOGowpdX;Gchu`5mQd3~&!kiP24v>6PX3i+Hz)c6)m;NHc+ zbHG8sf|19fQP|2+q8xZb6TbUA zV38;awmQI@uE4kApn#Qwl7_-|83pEl3q%$%adR}rwjBslaAI;}VyNSA6l-CUTco75 z=!w=N29<|Gs}6C@aFE*Znms0uiN%4z;DA`07vsM-&WvgYE*ddC*ILNs?I`K7UefQN z;HwAx3=8=aK5A+&WG!jn6gVg}X<^@+j{h=@4AQG4moW05@L@7J#kkI4Wx>HGa-O2bY!W^9C>Xw+8Eb4|s1Z;A>L&b|Qea;yqi` z0;8&ao@@rTNe+CE1Vj@aJdVm{I8BfM>-tx{!UVjR7ntc zog7_qT9N0tVvGX+ABRe#2QSVXV9{w}kYQjpdLT0IAg=_|zZVisjC=LL#8w-)ffbKtwu`hTAv!`efF1`Bk)9A;dVAXIWhVlm6R ztOqMq+>&MldVec2{1(J&?Z#2#V3hYj>c1Xq)XEQOZCnW=ZtHp7veV4Uj>%L8WHvlt z57Xhib3kB^0@r~9#?_7DbqhTz6+JT-3U?`RdAt{Vki>beke@A4;DJNKlm@O02i6@S ztQHJx5erxh7O?F3q8QD?JE zPYMbhjR;pSydJ=*fg*!{R{l<)NoD4iDia* z{t*W8a(~u$55(SeT-H()nv@v2((mFS4nIzTH*$$xa*RTLj6$;(N(y)`K9|JB_t5Ea zlC!zUszoYEa&HdWnXGo1V)*Zqi*oanxCgA~Qf|5En65E(6BJ&zqnEvYL-a-#*@*|- z|0f78V&Jm*EPmmuc#vvnMWXPmEdnbJirh=!y0d`m(Sq^=Nvv}2tT6}t4H8(>8ki-n z`aE07Y;k}qPw`X$qln9T{;<_7QU`o|8VXYm{Fsr#VbhRzsVUE*A@9X92B~%is|D;f z2^jUGag>D?wdpfli5nkjz5<4-5DnEZ{v6 z#`$4^W3nlkaA)-nAC1sCz;LsHICz9O!nB5Z*VroM9b+xO8l z$L-@bQ}>e}<9-cYz-Fb;VAa52b09%-D|62QF|LCGT#Vcy3#C}L_@st089ZRBdCX*YfURu- z^NI#un+Dz+2iTS|@P{#E+9Zj9>y?ABnxpiq1fhEiCAWO9&QBByd+<-} z0wcegq7(;XTKXZUs}E%-Fz|oTl43|yuSm6@Va6!N!1&>C=t^e(I}PbG7O&6B?Lx4zTDjurY-=y!0d9$IxIx zZpJKAqaudGF#@b!2mEem`FZF}lh|yS-S)tA8kD}gKTq0FlTE_x3IZygjda+LN_Zo}=^z zrIT|19?BlME?vZXGUtJ0QLfOLL?J7&>+=-Fa~?`gSit3FDaCg1Ut5dgiHz_G-x$Og zn8Y49=qL(>oo12q9NQfz3F@drw^~ds|L<%3T$2s%O999IUQikc_45^ z$;)P`o~yvYo~`=ZS|*2Vi2SKHC8#8Yf&XLN$y;fi^;>$QYbUVxH?a2{IMCz3xuL;m zI|EnRLb0ravhym$Tn=(tG>WA$axGHeyYrrR!UNt_2iUR_Sgj7QCQUnBu#oSW!=@w! z;VEa>k`Azz8QZ2Oa9+#hba*IP;3(>IMCymGOkU%rd5W?sjN%s_3Z5|6zQri6#wdGe zA>Tbm8KXq0e@Dut_AFeqHc{x$1F>fcD>)pw)EWi8H3;w}vQC=CwyuH8q>*{cb*J}J z1UM48yd1wUI5L}M^|-M!xjbO#Xo;Al$|%yXuOgvw)(OFX4E%Ktf(@1GH;kC}Jayc} zc=8gX?5%~8B@2YK9M7Ct#JJ3|&$d8MnRB*q&q0He26YD}Z|O3e@^-byN!IuTmIH>~ z*;CRUX_+lsz+S(DJv`^ARL)VG^BnJ%a&2-DUG#uEYN1%vLt(QzZUQ{B$$UaFw6X6wRpg)^!2OJHRd>mwBQ79*#{iI8Xog36q>^*8WSyk zZ=viFMVWby!ZruQ<~;o8H-90YoT9eYEy*vQ9n3b_taJNqI5q9)1;L`L zyz|1>aBXQ2*`~l{wNNbLps*by*MSAXX&P-i8aSRPxIBBnd`H5XE0xE38k^MuW}!{Y zGo*JeR!e*C%%-9!D%U6yz$otY@1ek!2ZCja;zf$WuNn@#yv?@l0hgTu+ct;A6WLfa zRx_IDh7 zF3h%{!Ok3XfK?&JKA7=XqKvH1s(mI7j427cpA`6H5(O?W-g5jSS(XsSxjWzLfq+`0 z2#?OBbu+J9xw6zRNZPg2;M=8>!i(>nWKRhaVPCd~Wm8CD^yAIfBeyhLxQCx9+`s1_ zlLj}}LhgAD!c~d3cm4=REEKbwEVjTw-kVYU8v~0~gLmq*1Fv5$e7!PW-imF4^^=Lw z=eQC$pETUn2p9d)u=0J0(68O|91bw&?fdpGLH0oTJ?0rts&W}vW~f{!jCI=F!)gzCzb!CPWMuP5_qb=ws3EgN`JIXQj1)b|U3%@J)k?Pw zxRO7bA3J$;LGl6__PmBR-NmPJP1bQ~#2goNzdiHRfv;OC59s;Lp7*>#grkw;Ktk?? zWZQ~`;#CJ9&1T?MaC~g55XYkTB=o^$@kyRiQWvc6C8UP&6hC6xlfgWxK}2C8*OCWZ zCJ(vRDI`z+KC9;dN5=!^Z3j5)9JseJ{O&!#JjsE(#_r#FM+NSJIIbjz7s(5lZ1SIH zpJ%g6;OJrK|7yd`GOf_-fYizZg5MnG%xV0?pv1_+!YkpB!SGPQi9=P!vU$mxpr!Up zN?x8S)d*>b@)$IFT_$n>;*R*q4aOTz1uIa0fc2r*M3VPzB7~N#EBcRc; z?YyXV%#9h&t?X0f40A3#IN-?It~iHNU`K$0GrRJ{e=?mP8=HCmNoyxnbZlg25!7{Y z$jJP(uzMP>+qN8qO{^>({8w@tB$y7hwZFM)|7Y7r#kMJLJ#tDME-^B*ajQn0C=hHp zx7c>&oTZNnKP))FCM@LF!O+OrXTt2EddguFhkUz?l1KCA2THBd6JI1S%AH*zbW*AQ z!KPl*%P*36&7VgEIm?~wU{GM;tN3t2cjuXh!T#1a4qOf}mHX<4F4GUMG_j> z+$@YZ_%+uUv@pu?7`Cv>#Vlmy@OAmfC>SVlu|;sBa8#vs#iJfwAB_nCj+}-Dj?5fqTiT7! zX#{)8)Rs&~29Xokh(|g+0=oDmy|B_nBVLxy);I{=vdSvS*hl9Asgub~vHC z zw!mZKVJ?B&3vMl3vGV{Uqln0XgRJ5bAKR5pR}^S4&Z)9$;cdIJYK~xa$;U>qrW+TI zDxQ>0auILwNck6)39RcQQv?r-hTSk&%A=K_(8!^BN1>U=Yl_o+ z;~g>G0?bT`2O1pjxdklTaZw{VNMm=*67JpdA%dL3%sLN!J~S@4zD+dsw~w@)68Px#j>o~$##c8^aLFoFnWU%F$;4H#_Rh7w0@c!R zelu-XaiOSJOE)WDNno>CeNgC?0%y9+L5_HiqgpN(S(F4ALwzmeRL*T+x5`Xno%v9K zH>=^_R=I#iHbV_XcB=<#`1Uj~^K&qXo(gMVieX?jc)_f4ZyOU2&qGEzkJ;=m3OaWC zF5<3MI4ZZzfK!)2JG}Zt6Ysi&)psk>S?_(EBDym9sP9|@TbYJN9u)yb9-Wu0dLB$# zzZ$sH7dUZpMszqyIZSBLc)-$Ka7^2kxSnoZ48HaSVImR|h7gjAMKapZidd?~bR zO7pr057=}pB)KOo+O+Gz`FgX--ADTtiGTd7KZEJXO{VMw7A>Abl4di!6$%);ln*el z87m|xOxf7+&-ja?=;RqpJV6Z2qCZ@?Y&829q!?_KC`n+It8nCRH(;!=dc`REgIAWV zu!XP0ft7ax1AE+sMt%=Rt}BL3R<*&d{3Q+x!>%;2>rHU=FJxf0tzlpbxa`bV@nF#k z$6l_o4-A|gUs&}g9F!_B)V-?fsKBGZA)|hPA>Yw~QK!R2GwlFNss^*jk{!qN>UMQ! zKVf3=X=srYNoewLQQ|9gaktnb$d^~aEW0hjnN#N`6OV}k6K@5BY>W&0ho34OuAK8V z{UhzdR*BX+%<7WZFiBMLi35k{s?~%bNZ5XlIa2-Oex~QEItkjTp9;u{RBH)HJU}1&Nym%L-5LiS4y1T9Y-W} z5*XEV771}Wx10UT=FMHeELw4)QF6+~CXS8;3H&db4%z(_4!q`(^vJm@+9+(=7gkQ~ z*WIa;4m2^zENDpeycpQ|^Ok+LVyZ?RXS{!04U6oVR5mM*CK0uXtjtD=d`k^nO3RX& zbw!#)A`D!$UKsdFe);FXRXT$?$BP{;GZfzmNIlTtkJ!Ro%+Vrfx8OYYo^SJI zA{y;1J8pHBnVij@x1#FG8o7yl;qBs64{!$=TohU1cHTr_B}bcso8*B7l8phZdh;H# zvCF9RaYQ!w2rpoeyfG!Fu!BYOfi-vR33E>O4i`zU1q|F*4)7Qiw3r?Gc>jNuGQWXA zqvWKC%`+P&GL$N~>Tx{W8}Khh!%jlJZ_cJL!;CY@`G04Gz3{rezTpSUE*7Uw-L|Co zzVjNo_h!arEosh4pUv*EYcy~ zz9Kty9P+rXe7>{s@9+EiHyU{=61p5*oMH+dWcl3JxYw${z@+toQJ7_~ds&3bWhc`^ zf+B%zZki5^e-fFEgVypEW+V$`2OQ*2=w*x)U%)qc$5G`Z5wA_Veq_ZQn(=c=02^Pm zkH>~T8#<0Ynil_WlJ|wao40oEE0s-{&2E*@#P_1EMe|3ZP&5OJ%8vu=K@uj`7K$xe zN{y`^dv>!eeRxpB>GtfqONuxD-Zt6oqfP8}0f*BoW*$1|&Cy)P6q99Zve~WjbU<&^ z#^9(L<70*kW?g?KU@Cs}v#a4@NxqIL%>R@$esW(6;Ifxc>a8tk@Ja3x$d^=6)C$_d zWFo*B(2-njzqL(ogF&k;lXLE#FY2`BGZw=FREM76J_kg-lA@%9cG5cNfe(AQENvDall!+$w?j)-_?P z3%Ml}PP{DS~))j;(t&O%%LqFba43!Ku*G(Ihh-oMcQ z$WwoP2dlgQuBZ>}8WY*C2MhBSFc@8Bxc^lC9|J?C-~z_R1dcZccom9xcNy^h_`;qm zz<4ZNSa<=a`*$9_MZCdZ`DzuIUmmX!3Sjt9&OOnfQBcfF@o|!clbzQF5qpCi$Do{o zr_7cKDvT0h8XHu!u7%es=5l!?K2(n2zaiY@#B3DcTeXp`cmkJf0gLwo*0>Fva}>BX zA7J%YkSV^%lIX!^Dl;7g1bgqYnNIJitXyc3w~9p zUzxEdQPPBIq52Xj@lXb)gutxDOlA%otqs$pr%Q!|Fb6kixjC?S3(VMh$^7(1mg)zL zW(REL4LCzTFjgB_))}xHFJP_vz-ZdQx-^jW*O3ZNb>`G-l0l2pZ60vsDX=enz|+^j z=rUcP=bF{+pa~O}F%*0-nZz)=n1O5M1Xkk)_KFSdF``Ung{=QfE=-*{f&2Ud-j@$J z3=%kVH_S@7!VqG>xy^yAe@3a3Vua{tW*a7z>z<+^jiUdbGZ%@9nHMmXwK5%QnHL_M z`|_l)^#b*GCz(II6rQ!6hw08hw{KW)>7{E_QYDFyIJY5OJ1)Q_Z2onBAI-k#T+Ttm}^%3LbE* zXy9FPfqjVq@68EZJ{Q<)6BtbbZM7NLZ4G!QJCvS(z;pS-vS|(Mss;aM$C$E)PUo8W zO4v@J{NZEf^rIr)hB-D(ljRnRZdCDB5Sw?fWfN1&ynDiP7rKbp6c!mdF!!r6TRBwD z363ucV2N|!Ud+XB&&|JR1Lqb6E|(9I3kq~>Q<#kpF)zCq_~we_vWr_dpQp)OV9xo; zAXyM-yn%hv1!lPgOuw&4?)}L4)KQi-mF4n6+4SFx(i<4%4OrYOTK0croW+{v8qqRm zp_J@~1v5@MI!<8EVPG$1;Alu-ve{5HVFHu0_)^KA>uwg!Vt>exYrs*Tz}4))UiW}w zLI77$1A7-kmVf}0A_JrDg!V}R>^T#5wme|3*uXNqfL+x<_)R;DWg@qYu$R)m#|=`a znd_xk8oK7*x@nllYE*bt)cgVyi$>D(ox9IU&()v3dqT&Qki(It(acMWyW0df{1m3z za`V4iP#k}Nb+ZGvzXMAi1J}|7)_G0&dJawjxBXsk$**CGm1~IEB?MnK%@$EC9Y<60oUXwH6gEbjhDCme5jkffurI9=q~x}1ijh`?0pM3)D41{oEE5D zz|qja-tvH>;sg8tD9%DIMlDH3$AsN8e>vVfCR}wXTYT}{Ow$z(hZ2>&Hzfu({9E*6 z1^MLeu*)1^s}x|#U%;VxfVsgys!xE$h=ING0HawuqgNZ_m9MV6 z6PV5&)6fdwJj=j*qe*w^&myM42^ygQ5bH z6u;guE^Vz_b7Ql4%^AW)S=FoO-cZ0?-Wh2Se8!eeu*eo(UWVS~pIp^NX zf1)Q=?=gF{$y6xOCSc`G@0Etud*xYW;mX*nZROmVC{;DOa%*=rmm5zk!EiUkP9+kuS{UIe!%3h zfJr=nx#EIBL4k(y5yqeeOhy+@@^4_vIl$=Uz*>5WP51$mR{+y~b78#;OuP@6Bor7{ zq%P#X&X8vt%ioaN_l$vQ0)y6pB}M_ZoCd`=ncVwt%ctIAU=m=oWMH;bVAg$Lsk?w# z?EsS+gTmtMoBnH9(jKt+EMVk1z+Ca*ti*vjV?jpk3rgn`SQmvdOT13nn5+u zgo{jDOPD-v&igk<)a$Kgxmx+&^%X1Q;`ScX)p+;&;?jebMGRaD4wv>E;9m29YmEc< z-Ur!8gW1!$QY$YqJ`72%N^9n%MG0C397`VXRc14@N{GGt*ris! zQnYuwwrv02FM zdrrCGlo#7y=F|y4>3Wgji3O|yX!b67R+RNP`5O%hHpbu#I=Xq8~(1;`?7f5tIhWqcqj72 zKGhW#C|;t+n6sbl_#55y9SkST1%4(zGCs<3E`iNmfVW@*W0wQ(Nd^Hc5sMB6N5_Lr z-BadvEY|Ux<65*Na*}{!QwN6{pRZHu(zZS+ld=~N0xz}q$=O#iFn)68Ic?;SbZ$zd zTjz|~=CQ9H%sl*@Q^qm%!Ue^}F9H{Pc3+wh=zJ`Ang3F+wG)%w_#_PNa;6#`6_nGd zbyz8+x++-RAfQ7clgWLeQOMFKB`+QvWM=34r}e4m#e#6wSkZdxH5X<-I@ruD@uF(S z1eMTLJmPX95=xB?EK>bih8tN{^7k+?%9vz4I8gGEjYUl-;B1v+^Ysm$`ryTg`ZD%mwTk9ho`yE#^;Gkonc0lI|0ree_7ANqU}=O~V5Q zCl*0=QP$R@o+!eyfJ zzXMArIJ;FcwW#O(aOgJv)xjbfqi|4(jXz{Z_6EKW|G08Ch%S2L%__;XxsjbOVZozb z(|a=x8irWyaA-O_quQa#TJGgSes`sdJC(LxuYU>@^35-A$fUB!|woMO{r=uGLgS6R8ZFJ{YzgU$R)E-dQzl51Qd z67pz8qM>^5fo|sJpQ|QqXbpeS$m{T7#lo&&rI*Z|76k`c+m8I>JM6&3E*EiFNY1Ds zu|GYmpn>7$DQjQCwyBvqMj39;sX-}mp4lr^*=y=A*&LF|d#5Sw# zT#)y|)p@}jMIOg`l}{D6aJtS~cx7Vok2WIh@3Cq~US)7~<<$P*B${gS=Ty-p8B3w%%WyPMN)3FT;TgB%6 za=0tCVu!MjIgjDdF7qJ9qip+fRy>;KlQAP%AVk6NXuoz&#KXpB{(T9pQWG8=XqH>m zz}#mwBjAys`LhBwAuGEBhs4q=3LK0$H`OaJtEvyeF;{EK}afn&ZVNpy8;hlh~Q@;sBFc zhO_FKgq2@%S1E96Ft+*?FtJ7~V3ps(?^gGnFdCV4}q-i8(b7*KDIlU9G>)#LE$XR znqaSKf?bxY_S`JWd)&aVsnJhlhQneGg$5=O24({Wg{*|lQfw}~Ow1{_SOgU2aD)U1 zrYf9JZEj$^-*F&Fn1hi;?SVF{0t3@k9mcCQPqn>;9*O;%&FJxM8M8dci+uY4r_E(Q zoK^Q2&*N_j6X>k)uesIO8FccX(D$9^RkPk{K5c5`UKF9H6Lzs{uJLuQD_hwtC!7=T zv})G!N^G*JJHU4KM2iSVV3)0c6LJj>uATr^ei4U$nH7hXCloaD zlsGVZY9y;Dz2vcsxx^&skeQkAfI~&+q1d?wW=5F}3|4cO2O6(_8j>4))$OT1 zsTUFs8wCj+XJXypz^b*NkvB%c+(>x=Bm0gy$yt{bI2UBEj!In-7O2ZIZ_%w~5gQx? zGm2TbK7=~DC@?tm81wnwJXrs0o4K5c_a>vnbU%#{HnlS;PEJSEW!@CDpI3XtS3aTZ z3Dd9h^?HR}tCuQ?nl2A$)e2$_)wwQsO`ua*$8pkx09OU0fJViN0}IU}nngZ2bcIY% z5-B!linS{sNgRyzEjVKG&*7R7&s8R8@17{f zWBY=v5||b@=Kp(>J^Ra2elu>Kg-l!m2evIstG#{OL2%cDR{e^F+))Njtk<*~*f|{5 zvETSNCD*LAA#0KUVb+DS3f9~V7f3#0r|7_OQ|1AC^c^N{(+`X(m4_xPzj(mz9x%^$ z$<@OOb{Xv-O%6+*xzKjLFYH9Q1B=p(7w0cCeL2B?b&g~F?I&7XB5Yi}jEM;ew;dH8 zGJ9&c=!INlRcc^h`LW7T?$d!b_XSGg*?dRkas-;My;T(D$vY(Xr=Y`U#RVa=6^F}I z4LdAuI0>!#(JaXG(PmNR0VM_Fvlr%M#s9jNx$kH9e0|+H_RTz{srzgEvUnUC*ee!r z)-be+n{DLOUsA;%n{kL;%bHR*4p+g=aiMgvkmKyh4~)-_bQxsK{MBL z0q5dZ4V)979{eY$b%Z5D_QfW)gyPbRKXBdO7DxzySsY1txiygZX+nkGNwzK8OqPJmcqS2;KCLx8vwB znN<&O1s__OmT1cHM!-Yi&GjY0vMP<-P7+Tyo|;yevaok@)h~m2mT}Dzmlm|h%|6KM z{B4`kA3sjlfJa-Fzn#PV&+0mp_>SsR8`eI`kXl(HfJwjQBGL6 zE5hK2Nx;TWQ=?P%ReJGi&wR|lIYVgQmN%DKGG5NHOWp6!^yK~gUq=|2L=G^>6*#Th z$-t^%@sRsfxz*+0Z&`I47_QlFzf!PZvG2~P%ZGQceZ0~1He2w4Ro1R$O%2VS8(Q53 z1SVZ*l=#iaR%6C;u#@wqj^Zk7g#+8Ft(tQ`?>?}&#p}ZXm29Q*)01>O_Y@x%Dxbaf zSn`bH+&tnuJaQgPmN%FMGMMxPSmPf2YmMjFd*O4F)Et&zgI4DU&3ZSOb$>9#25Iwu zXtrI^q%*^@u&UX82dmbKCi8$sNrsjnjaG*POiwnoW_q+9kyz!rTJ`55$=|1Tdfv9Y zmT-8D)RgNx3~nr7V0$6VWYECX!BlT;uU^pfrJHGav6Z@oDQk8UZ^BBRED`2c_OXvw zuYJ9GgR&uqfHkLpS&sl?{LBW9ie~0HEL*+R=Or{rR-4Jp+P(9z&Mx6*Gl@lJ3~TeU zTMqm_;I)D!c?XMX_NGE@o0HK8MM`YqW40f-u&Bbj$#TI#jTbiY9>-GNv;;l?jXw{}t`9i)ImxXb!4qbGKl!Y+#95!4}=n zICV0EQ4Pb(&HA$q51a1MuUld=clCy=93os24O|LLfj3%~c<=w3Z7N~WsOGW3x1mY@ z0W(YW)}=2^SYL@OyLf^x`AF6-4sV&+J~}!kn_E2+4CiTZZu`s_nB&RC*d!yQBp-ED zao2L|1I)&!S-cLkczLiTd$27qVN+MyqqAy`)+HrXt35&?jp81A#1l@56dV+<@De@2 zDBQrPvS5eqkETcuxAF$|ii-BghNeh~R{aUh4iPL*qBh5L%nNYld9ka3`$vQMjHcZY zjamnoRRfy58X6ZDv3cBR;Ll(RnZcGiqtW}t=}%jz?;N=xeB(c#1)dGBA{wl08a5f7 zSQ2a>o6zpCp+TsFU&!GsuY_q}gw?X-l~Gwz%^UXjrX1;gvdeenvfNWaiLZUVd3I}7 zusBLjU`=t5RCb+xV$G}{inGp`DP1~XeS+C1fvu>6y?Vvv>J`D&9_>ZJN_KC9Rc9?) zqT^LES!vmtMJr4Ee|+(lJJDp@(WEzn_5A17ScYc34yG@^TSGZmLIPNW4m8;wXwv!6 zY`#KC^2MZzZbrF`i4vWR<_#^T0gU@wt{Qu=&ObFHYlaH|c0E&KQTZ2ZGZf*;O$e>r<^kD98*f#?dwNf{2Q+7p->1vn<2oEdYp zdQKD|`J9HhvFA!2@AOk2G-pY6^5X;hWLKq_E)Cgw?AqpZ&+j z&B(u6O(*q^&eKZ_M~-kjX!I;lQFZW>d>tNGvyJ2A8a7MU&^0H`erz;zVDaf-vpUP( zUVEeVLHo74?F$8%BqziwE?^Yj5Nm(-Zt?5O%XlJNv$mgYYfq@vAMgMs%z!;uGz7aI05bX{C0+xzEi{6fxX#&ui%gf<9h@n7m-;FW0LXK3wc zVaod29%G?BHOPnWfD4a8Q((t6-xp2134FeR4FVqFcVez<-n}&8&93sP*YB~2$yKn# z?D*HJT5zD`(e=PHYnWzkQ;2Fd%4mL}vdG$i*<}GsX2gxQ)VsAFNvii4rB>WktK+)W z+I7=*)5kNh+OxbgXL*%HFRED0sA=G&@X{tcp*8FSOW1{ju^*ajJ6Os~*q%tW%-!0g z6~U~l!|LA9tZBidIiXo+0!z99t9L+K+5u)+ht{vWEfzOI(r2)FTxi%Vb@*$L{z9j~ zg@+n=IvRL98o50f`6o04`7%l#Xi)WPlzky1AJE8pM8~+I*(iZobwjgpMw4Vm*q6_& z^$qNfD^!GkNKEx&6wzShSndgvK-V+EtM0QTw&>?&bN<#|cVR@}YO%XM=q*G*Ob zyIYmSU))`sV=mco(Dp=A&10tXrw!^`-aD=4T8!S zG^`G=rq5u}n!s#+ph0y-OZtmudk*K28*HHlO?whEQ&%!d1vi-nFqtx{F(Yg0S;x2)t$_@!F*DrK72-n-T5TsZX}{>QJiz3j!1{u7rK3fQ z(+Q^Z4Xhq38hK7Ih@NP$%Xs11a6fCu)#EpuUT$mAwP9V(#-CZymVV-0#*23u3)ns! z$Xj0aDkb8TheqoHKQ`Bb7LOCm?kAe9Uo;zYv?vuMe7W2j81R_Oqlbs#wa|qyu^kNh z3s^NCm@98M&eFvov_j4Q=IUd2MdA-%(rJA?VM(~yyR$+OjhZH{`R}sly(kFWqf;n$ z&ZJ)w*0ZmBPRvb^ecv(yYG zyNK4%gtL(Xt)T*}b~h#}7qab8X!W#ck@&{!@Icd}gT-(bTkM6l=nG6S0mqeBFiN~& z*ncG?=*8OaZ!h@jvIZ3td#q^DEot%kk@@aK;Drn$|n9RY^$a8~%z3)U|M37^Kfz$mkx9bcE`xqP-uDzji)$!`@HPRyJ|jvranx0=q%3?ZO|&GarsOtlkAVQkS})xmwUeS$vNuQQn;Y~ zYHh5zf09Ihp@ev+8Yb%GNvcnCFAGOp6ws8H|<_KG`a;=3HotS@G6s2W#+(X7dgf%OA|< z2bhB*SV9%Cf6%|Yf#LFnhC33oE?Mwe@0|R|puazU z^|sK29Z#EmAFybg$k%w$q-UUQ_bbb7d%VC2)w9VjAqXht^d?!FzIA8sWo({Mzq)!+<*Sl*nUG>Ut5L_ zN0yaB%R@aDryER8Ggz+7ZVCR-=n)a+F{9DWfi<fgAbj`LeI*FvVL~d6fpOQ*NBM`_hiG z4=?A}We3IF?|F82&cBM@aGr`5g+FQAewTi_zc7CK$(KHFgv=vcJkpQnm7Q4DWD>!w zwSYxI5{iuO6jpZ+o5>4wWp0ccE^o{EPM-;&DQ)l_;eZP zG~;wJT_%>~HffJ-N{mV=$$Zk<#Th=8CM&#pB;5Wz6S%eMdB8NSiYXcz6WK5D@Jim% zIT_I4+{GbbU@`SWLQ^wGm}0`&TMSMuYa^5`8ZIozYUUIY^NF~ypz-QjAsGt}DJ2#b zK_)GehlkUJrv%P+`*`apD@zBLw26$KkynAZeqftT#sY;FCZR~hGpP-h#%Y3zHud-Z zSRP@ypzJrFuP~~G#eJFI&Wj5lxAUo49l5hkcNx#*yJ<&eaLV23lYCVE#w5vYz5M># zbMu|+&$ao^Hp^ZSz$tXZO+e9l-{l{A1@1qZSUII+G@RQH%jn&C&+w`3!W6!UXE!E2 zR-2?1v~o%y^GT()C69RYCU`7jVwV#*;KuJ^^XY`NY{&t|e?oON500yt+AMA2)S6P* z!X+EC;2^75LBl~-O|K7ALijBlL?a3!uU=yjXn7jSGJWdN11$DN2bx%fmppJ{ao>{I z##!tbaDYL&=3u8rfP-PT`mR5Zx-}AP99;xmYZOoEEiQQIDsC9!&=4Nk+HfEy|B$5m zuBR(?7%CrDaahNvUlw4;<#-@-VDTV(-+88Ip&1!($4|?i@T}^wmIo z?y_@F7rM(<1srD+warLx7I19X?#%mSY1F*c_bw_pui1ESgR_grnui`iJ_!@M#EdUo zbeHebU^=0+bVXCQ@i`ZvPG0ef2af!PUkpwt6qO`6vK!hwa+K@$c=(Tjtzcz=n`TMI zg~mYHgaeM<5)lcF%wjel7}!EOMZ;Lk-|g0Cv46KCIW)refikn8hl3_}frV1D@=1@! z9rG;00vZKt3O=z;omUamtuOyRh+B6{isxyym=6nE*$Y}rXJ+J`d%7&6uxrE2bS0k; z%hFdqwRFm!m)jHdr#SPoI$ku!a?bZF9k;=r&=^~X--sm$jC0M zkl4)I?Bm4FBbA|WSXm~-kW)s+;=n;RtqXVE*(7Tmk{I9YlU~pISjpxwOG4wnt-toi z{yaVPw0Ygs-=?6}CIs@Wmx$k^t3uw6XH;o%YW#T9S5t>1ARKBuo%zwxBF z+mYK>qax0_GQ}F6wiM5>ydLv;_UflI>&^=@D0QlCyx6v4k(I`(FM0h;>N17D?+3oL z`W|vN>-A zFXn%82{ZI0_$z1sa8b80G>VFNDID;i>1*ExM%IW{t(X)B&IGZ2A1c|5OErD7+!oB5 zw=j*Vazl%;h~w;AGK%J@7kU>fh-yCG;>aKLTi$+_@&U%QMGcXw50`9OEGyM=wsBL+ z1D)f+0-K&DuscQ^mcKWF@km1__nZeT5{np`lvtQV&jqv@uL|r6;(5d`D0M{nl>^7j zn9qXk5=W#f95}AuapI0va1v(O*sgctpxC_~tt>NsFmOv8E&15QbvlD(!_~JPWw9kU z!yjDWxoP3dCN{y@_*8({nk|KK(hZN}emuJ1+wgGV>kZFiFf{U^BSD zxcAk~U8~-7>zXf$J&;m9Q$C`J(`P|jhf*Pzs6dZ4=K%*soehn9BHpK{KX#gFwvp4W zX5k{{4F~#Oe9~ttFxlSjqLNv)dY1LWO#kJIjT|jXoLMT#W**wN)WnyrnB~-EF{!Cp z_RE6Hr4JTQu`BIU-lhHTN=Tg+cX+@8feEj!Y}(p2?a9MNX5~%Hav_a;5=u?NE`~;N zJvky5YZ9H81F>vd=yiz|SgqlgB}2i;!|9Gls^g|<5!RX*1h=4|muq`sk1 ztiZ5S@6AE(1s7cP{v>pIRwx{iFmUvoCdux7!%1pd#v|q_1)K_Njxy#%ZsW+1VeH}a zS^O)BNo&c%4#oouBntw*h2?D#WerJSU}k95n)jzw|A3fC)QT5VbQd(R=`iT39C*7Y z_q6S--jEs<4c|<=#~zaxrLkz}HlDv>z?mL$$ZYl--oI&4!OL9DEy@iJ%dR=iQ<)-u zo@oxtal?-s?gCCLs&+U@?P1gYd?QGKCGclOis8oPq#Fl9{Y z5V)Wa$mOv_Nwhm)%@lu!bG9;ptRANYJPxoP=wSUP!>HxZ#CG6-NJNv$iVo%z4ysoU zMno*QJGGVB#(`x^qjC>-%mvw=Dc2Y}yl35<8OSqip<1-K!@p=*Eg`-hm2Lu0y22q|vVlIs8IiiM* zjN&zibb6dVE|_n8!ci#Upt8pyX$40in-1QH7QqZhA&G-^E{Awh7OxMRv)NegEgg9Y*?&C&%8tRf6T5lnIiIAVHO!+VbYV>)nD(!z;z3hVu~N10g; zFo#5PSvX0XFtBkr^sMQ;Keb`f90nDJLy|I#JQ|mC4qom%GUK@+1J{Cv)IFSqdp!lX z7II`fR_;7szMRkW8PH*=F4{v zsTR!nz!)m$!@g>b%ooWP9*!374R;*oJan8A&cETHg2%zd7n{^en&es?kH5ICb7Iye zo5Q~)om3p0gswR7)*Rqj?sEnk`A)gFt7_Wu^q|r#Gku9WA%f^XkMjzLcgWEKvBR|Y#5nzZhaFk2vV^h2RaO=NCo3#dq2M9hkX9g8gWxkeHdfT@>%-x5@T=^8;$RL|LSqjg~ksb9a(@ z;;5k_eanIW%yFq22az5o?TSO{2h_AbIO%72icfGd4RPlB>!`Ir_V?FD(LIi$J_qG^ zn&O{2HH$KF&un0w;lQSHP%(y4=%J&ihXbPz185NM52MzfAR&P!nKKNX2fBF_7-Lip z1aIq`WzoQp(7@Hh&8Wk`Zo4zYCJGGl~CZh;;8pys_7d? zF&QQmA7{;wgFF%k#STOX6*^n(VSKh{GUtZ|84Cy2H66E)GISnj;EHKz+TzV+(8`@~ zfLo!l+K7>V%>k(k2e@7|itJ&K*}%xV!BNe_L0ZF!uS9`ApmD(s2mZiDU5i7^9?kzO z|1>hiToIgeKq&-#?ykmR*#oPT1tkx@m^Md4=-d~vwjzy$HU=-&G|njw0xnHPDsyJ# zURt;(#Me{N=G@J~9wCO@hs#V3+nF(6E{r)LDK8y!Pm(_t|);%Zjl!Gc~8dbJ5 z8#gp5ayV(cIAl73NpH_#6%hx<9EHdfKOUCn>}L)d3miHbb=ugbNi^lK?h{5+8z#j$ z&L$oQEpIrD^gpI^$4h#)IgXi_Bjdbe9}f`Ek&ChLc)Fv#J1-RYa5M z3}!usx4Hq1W)aN1cbbeN8a3}6;)*$>`-MSqMicLoM$I)2KM!3{RcI7>qQm~;0vAUL za|)C62?y?fTO9bNIIzrUcwp$Ly5r!U1FuqV-0uA${r!U9Igy^153lVPeO>UChjU5; z+w=px4u^hcUQ;s-zqCcUzU9Tyuhfe?w!o4x_A(vsQziM#mww0w#HpgK{1x;i|3FEsL+9G02$%DdsT zKuVM7o&z!&P24XSMR^#dGn}}-91yi>Q2TO#YtAwiZ$|bV2Sg-VMAjVOy3)8L&7HmH zlmLrk*^&nS9}b)e>Hj!qILNp(Deq|HY;on(xHL7^?W~5-oEx1lQzPRaom8%sOqh3o zGvrYA5x3%}x%o2=UNTu`Gp})uB9p3vv(1g0+uYpkzb$^GwJz;iqi70yl}-F-Ll>u@ zcNV+vIJO;Mo%ILaPzRIoT~V#BO5fk|@?lOB)LiQ3EZ z7EN0tPCs9ENbbe~5s5=GK8+$JjTVxPyjNIucbwjr(6FegfjhvFx8OiaL4(?n16(YP z+!YMGHx8&>W?&a#@m~Lhdkq77!2#|H1|NP$-*N`_6%4{BOLp%#9n90nw(JD!j8+~E zMjh=2P8~<%IZVc18@Zeqm~#Fd%Y7*vd&6knolh)14y^U}@+>Ns27XtoP5F7auyk300ivgsq^)wDKWM@*FJiKTu%LkaESl$eE2@ z!@S3&Aw(b{V%v%6JKm+&4cTnUxe^+DgL_Us z_uV?D!hhf0p6@-2UrlA;n&Y6-qc8XGfvdTfh3`d18#ciT*lph?_tSD=GmZrF1Zi8!+ z{Dp(!mk-M9Xyi^i*eicX@rJ{eh8!LrNBJ6uLx;MVr!=s{cKj3N_k3- zYT4=MqOD8W)V7CswAOeXsK3V@^Ft+I8%Km+Im;UdNt0%)1P94EO{y;#cr*@cCOCw> z2wvY77VD7l!^0qROK!p6q63i`oEscEy_{uoE=kxZzGeOK?}gk^QFbMhLygy)?VlVj zvpr;A70&yA=E;?b8DcDn+D|6RrMQ@|I0sDqr~JiH_=KZ9`ymC3=9-&L@;1TpE1I;P zsA)Jb$qIOgmK<33`4_uHqoBY+!MBYn8B8f62|OQyUd%ZWY~jlJqk*gAbg)h9-h_q; z2T$w>+09mFIqi4K_UqmLaTEQ+c1HeK!g1&H^v4{BGRn8^pTYI!fYcL4)&vKE4-9N4 z9OSn!+;A(MTP(;mt)cg((VPnZn(hP7*B(p%@*svqhU?CE&LvSAm*%PcbuePLSYqfRYG&IX&SLX|)JnnwI$wWOlwaAf*->N3XIU1fWj2{gPgb06;5~4u|Np^GOJ_|L zN0|psrZ=3luI$j7b5MxIDbFrAZMu`ZOp?%zhB;~nSXIfCDoZds@XG+vQ66^TyTJE ziwC2?0fR8fkl(So`y*Iv7`Tsg^ZkfrI~Y}S;b#2cAg76! z^|`h~SLKeldKtg&m(8%Tz0+*}rP-hHs(Gj4$^JPfXL>y?vr6QDd0IK8>clC&CB3&a z4$Am22~A-#b!kp-o+>8b_Uf(2(*3Kc0GWN>-gGCQqYK|?^0$%%ta(Z%P7$EU>{ zLP{<@4FOIrErQBUGb|WSOwzS+Kl0*&;mvJHCb<*1Co%yK;O0}Ii;ZE4iAvXr^RL+?;jJOH%7=#}(!D%fD(q7#umaF#2-d+lxjS=NA+&U2Jmm za%#7T)Amgw2Lv9PwcSxQSn}Y&L&t)Lhdb>x0%~?Ja~_=TJz>J3oh_W_mfIirC#%@R z@m!`VVCPY02WCbNX@i6X29C!$(xx1=2wv51l$HJ~V7IX2LZ^ue&OQo~Ob$~bv=+8- zaL7Dt5p!X15t`dl70@nUm+)u>*UX?-J3_2#6y{MubshW$1tv{v(N zzI4Gwcl)EH)4DtEO$uDUD{D$)+rA&S9J$QYDi1IVbaXUL4s;Q071ry>aN-VFH1n9G z$0Q9^(Ga=D7AdEs9S55EzX_Zbuv%52!l%6KAghR@3#Y4i(6$$c#GLdr9x&bEI?}NH zUeJLDO%IrA6Lvn3ayYl+i9qcGb54T{KIUf}rXQ?C>$sbknok*@x8bl7Rb*tHk)h9* z`ft{Y#Zqb|9&7^KA}7vY{->a!c|gET;NemsKZAv=!YMlzw#xgwaGsR*>%&U*1ua_5 zoQqCXd~@QNTE!*kzG73lORbO1#MM^MA_Do%)>RmCNVUv39V32aZ{Xdnuc8vymcA}p zsJAcWMkAMn&&LDIO8Z`EY* z(k~VAQ%%I2^`z=_ZMzjoj~stq5#G*YQ4tdKkWckoNSg}%}R3wTE*gO9=3|dS_KNV@Xl&DGBd=a zk9ktaiyzH=R`)6n_nLp}INWdlPhly)#J2;gQ}kE_T}*WCb|keiS?qAI+5oeyzK<~(NL={Uf2tMFWU*nus~CJan3ITcu#X2`KhXb3IUc+6w6fX6(- zL9lNJZRHQX^VS<~Zua{5hzu)ia?*e9XJ~%FbnQ+;EqqZ zDF1GTtKb0xE`5na&K)b*^gON7`aV<91wi5fLSu(z?4Hfdy6+Hn>_VlUvcSSroXa+$>O9U%jR7N&RHZhi8M@d zFqj?U^UZ}>`pzLi5et7NfdvdJp0YD4SvWAsJYcreJIKhz(4tYBz{c?)z3JD%ewGQ6 zlV|j32}#~Kz+kq3k$cWUnNt&*HRpWfkV`S1eP_yrz>)-!<8G(yf`y!aG){56`1NaIQZqD)_I{H6%Cl_s=FN21dS`3@69Tg-4@Glz0Lf4o#XR z&~fsKp6K#`qY4K~8u^Ye2sB$9V$<2s$dvPd=~yVE?5PWkvN8vG^L{uB8Es&3w^+nx zQOoeN=&{Bdk*zB(clKS_u=(SsM&`E_W6I`WSBqquR0|hr~hHOz^lx4 z7t8uiepmU#RVL7K;77K|ybnjz?GCawR3?f2TcqJ6ta3`t-r}&>ga{_tH43fTXA=2? zJeU+XCbl@}PGu{3;mEGDfRQ)l0so2!XWmV}*(D+#Fa@nusjAGn@^zj0B)e$7n%cC+ z+do3he{I#*n7fKeY=uJ5&Q1rG>)#s`Q%^zEsQ&OYNT0o0D>}>)QOUCYvU`j`SDb!*KuOzVdx9J)7l16onKV zgtWJvNe%n3NNlUXX2B;3+%ZQO+3v}*D9mf@_Vj5I+5S)AsG>(VZt4^{>F?I;yJ4$2{+N9JsXmWCLegU%cp+1+3Gw z4pwh;)(~CufVZ;X0ROG9POm47%)Al_P6@i%0gerHEEr|Jd4M_Eow`yKndA^F5e-&uicJi63pNE?l0??0#&{9M6r7N)?GBEZ2>+BR+P= z7aU@9yU^tNQlMGNLWwUo!%fn6B6rY}M`A5A+RP?2Tozf8z*}DM&tKlFv6bhzV7Y)v^uU7``ogyiYF$|%?fH8Dy(s8>+9md5!erhzr?0o#?QYy}5cix#k|U0^MHz*gok zMS)33=^;nZK{1^!E~^8Qa~|@09hCH2cvL`p_N1rkVyAhM&hW4-*F4}Wd*tn$Q)^w$ z`D*`I$hYJG+po^ge@_03d+_qbT4q1}`!9OuIpnk2`Exn@v-LdS@JeL1Yv7pFA{KE_ zly99_Mx!vFBDdN@$(%%itOib*hdeVBM7WJ#i&+4Ezro*zdGIQ&9*o>a;96XUVa23YRL=CH3bO$^KQ$Q{H8#ztMmB zZneWXZ3`V&wjc!tl>~;RYK%MduE_)(yw|n5;Q;$Rzc+CY-k42zdykJbZvn4DqmT~+ zlMh24hau;h2mCz>%moi6nG}WM0zzw;*i$yBbM>_~QJwhIu&*oJx-DU$nTmvP3Ez6Q(@* z@y$VK-a|;cr9UC98{pMU&Lj-=+(l{By^j*^A+y5d-tP)tBF#|MxHD ze@$9#Spwgu2aIw#Hgani4>c=_U0@VsVm#u;U$fv%oWh%+2G%?4Slk-KRS&SE!+CmOM?GLU<3cz?|Y)_Db7rjzC36FBbaea%YX6+y(27zx0ToV|$Hz@30@qp*f0kMEa{wE2h(}g+L zJP>4X{Kxsh;c7y{O1TDxs!xG4jMU?w1s+jX|D3X?PVaJ4ka<_${#jg_@elqKFf%nJ zGZ{E=&3Y(i@jyY2he4=eolUEl_=L=a%xN_VZ@nI{syMI&9eATQLHwxE)uu%bP6{)O z81!@%mYFK2{4_Iv!j;a?&T&X#>U3;`HfeK zy8kR>Z&2W8aAbRRU{T0Ifh`WAAqx$DIfxp)YM!Df^kBi_DGS)D7O-U`#N|F5tSZR4OtuhdfX$YG6%tl3{A*q*<-zi6G!JAWg%-#ns_nl)o zc4=i11G4~+^{rzpIt&-@EZ~>xkWxAr=It0-wBXjdjryGatuq#ANM#$`JoZtb*frmy zaE7SS$_4XX0^R55r+XAfeSG_N&DXcr%G?`8Jj&bHPCek&I>fxxmvsU;ku9*&|4 ziTvlbayocOYArlEErIvi0Xs_twlxg`s}y`bl(K~^U}HPT;nA4j&L*JCa-PMx3y+xLe5@#8pse?bLYn*cgi$=mQPSZd%O*Ex`xR2>7-i=*iu`fl`gTC- zNMp*EZ<4e-la$T`s;_+3*1*=(z@YGeLChd4N!QxRwMj;RF)55qz>&!=H=R-N z?IAx_uZF7G3wiQh@-b}Ue`X=7l_VJUFGp}w3ul4@dmckp>>9Sem(=qD49Yh|2pWId z@U-pEL%9zJ*_vz}8zjQJ0L^ z^ang{JR(60Wp5-(uX`v}qA0RRfh%p{w9>|D2N^||G5nbJpmpI9lUWa#C7yDKF!Dtu zloVf|T=Xf><^bcLEB_sKKWe>WbnDe={HOagNAu>S+tV5xzb;{|Fmc>Ex!##I zX8{|7qhLA%_qPKayEcH%)a7Zc$a=sdwlLq3QM@EktZUbbgA6=;iDFKPV$0OUl@@{y z;C%AJkY}L)*F(V<3`v_3>-7C#jm0eHs}pA(z9OeETlj2g?6I={N1n5pJ}>${ zY0csl97`Fq8hB?Z@-sOwCIm7FB`{b#*ln~R?GtnOwo8ojnVDyNSUK-{aA=JqW#|%p`Q+&8c&;lk=Dlp9?HYNEbQ4(T@pnd91U$27in@8-%J$GaNNy&Y(?+CThn$g5Rj7)Qh3L_ z>jQJn0s*#zZ2x}qh+Nz^?eL@-ho9{e?hOB48alnB)Oqdt$jSTMYbp+#>lF&Wv`W{T z$R48rim}j!He2QilN2~tF{n8%V%+D)bc?~!{_W)ndCcj5_g!M(({W_?S&(INWwW;9 zbe9~dE?>z-ihN6a+1++;DSfa|FkAZO_C>MPbtelA51PrpQs5{u+2LNC9-wfl-Qg7H zkK2vX4UHvtPRmB;CRFG>V4l#Dz!&7%v4AOOxxkev{2>p|Oi%H>9wOqx<{W;r z&-k)(&K0-ZJxK{%vl4|a-DLD(;Cz+9|BS(`f4#se2fm^Q>ytI7#Vio=x^?-@YSwe_ z93~zRT++zZ^NlYhM$)b4NZ1A`FUA$CAM&htdt>#!@X!}*&7~S|6ix^h*PRxWFY}T2 zaFPGwAbds4aV8gQkONEG?CqQHoce9Pv%@%0rLue z5hg|vCpUo~3qF`Tew38wi3=8(rNGO)klU$Igd_3k9|j>2IX1Iu&Y%YzK?|xca0_?8 zYS-VmUuNb0J;t5PtgH4tf1Jx_rLs1~{No;vvOO^j{2>fiSREMB9<=a1H03FidUfC* z|BVCFte6%)W5UN|@50Ah=3V^izan-i9p; zuPv-rTC~|S^u&Vvf6F$WIbhIV>e@f&Q_+e$Umo1jzIprZytg;`6xtR@Oj?w{)yO8S z_GCiSLxFY~rKE_&<}P`GK&zLZp7P1rl+Cv)Eed*gc&66^ra~rGF9CV|1wR@-DG5)} z2#jTk^i0y2upnlqhh(Rs`y^>zgBM?RCOCF>aY(4xR0wQc#j-k3)?;sjqSp1*v4$}W z+6&)Y*`O7&q{HfKWAjQuxg))&8XcJV#8y05sCYPFm2>PS870$Peid6L1r56e5iVRJ zs!Hx@f&ZFXI|VI*_FM?u#=^p#=B0Znf$WDdHRuK4lw7U!0;YO{0q-c(z`w|i=$jNzpf zllyFYoXqc*{QYp)-|o+w=DXZ4WK-sGuE9x2KIP$*~^}MC4vwA-YTd(}`)!PKzWOS&ZHoa3~51 zC@`?Ot4wSWbjw)iDr8o(fsJ3SqS}E;K*8W3hh~q%%?$SUkZ^Yvq#Z`wT8z zmMM8YgXMnZ_Zbcg#HM6AF5h&#)P?=%@=lQ!iH}D$7Qb-vRu3q7bn^D%h6{~C6Q?k( zQQ-J=kaHv7IU!e8@e><75@ikh!l# z*&d{=OXAW?`#LT9*DH<2J}m)<24>R(g-3+MLMFJ#H--otR?2f()XXemzCeOS&}Bjc zQ-JD%gKbD_n5wHN1GrhRO_ z&?VvAck#TXl;%?J@Kx*rGo+r(a1i?U;fmv~=B>wbazkPs&#IDAYHT}^Vb_q+B5*o_OX>q!hyB^#bH4c1`Ga*0~`|* z!c-M5w%dO=z$D(_wJ*ZOWs}cSCf1JY5qpA^*T*c?O?b$#_e|4zb_WJmWp77;8wFfa z0tc8HG@7}dTxevHI9MniaY*xDK$o?Kf);Ow@=2u!46GUl81)Jq!*vXrRbv(~doE~} z_F2FnG38O!DUSI@GYXhe7bFQTEpXF3SHNz3=OEuP&2t52J5S!|d6=HMgIRTnV_(IY z!*XXUMM^(yH?a+14Awtzt6WaWRK&$M`0>eqQcQcNIs9;8i50rwDD>w7i$G1HSbp-+ z+Z6}7;|r3QWmh=LhXg7FNE{M4V6@OXM1#wqVF8oYfo2~21&sVP3e3B;U1cT-v|H?9 zG(FE8yT?jhFJaeeT~C`wyAPE5Z(ie98XL*|(!t&R3TJ1Z#0^JTnTw5$nog{I6Xr3S zd}#M7@s~8@c41T!Xkc@3tvs=rfsN$@yUm(~Yz8w9@~pbh?zmu)r27VjQ%s4=l}{)N zxf?VY&oMlkq0uZcbpea2%t8(q5mCW%k1SQgPS3{@?1EDpj{SLbaK$fI_spNS-Q_NA zXj;oTA^1V3RQaCICNX`lY#%rJFPECcB&FcU7j?topWqG$?x+n-qU#b^*`r@H%KB^; zh&7nN`Slok&OH~ArWc1qlP0v}k zpUUL1RQLOx=b4PJ4y^NG@F=(_9#Q7dtjNK@*7x8bgG@rB^{R&Qq7`m3Vh>NaesEx9 z4RGL3Tfo4p^N`OyqfJ$3V~do;Lbm1|M|ES2oh3eeU@`Exqco#{E#$!?{uK+_j3!Ow zNs3q`bj-j_^;tA$5=ZEYt^}7lQAf@s73WpY45G9qZ}SR!*2v}b;(qcUcDZP;xAi$XrdB$h z5_F2Z%A>{Oz&bIYh3Oqbhme6IbJvU|MrjSUAu7s3ds+`_{z_<+(s{%q?7RhHT1An8T@KrKar!(h`B22YcS(hd_)?9MsGXbkmWCXCRJe`NM&XF+3Wf$I z4Mx$4JC4X!Oz27TSj;aK;AT2S(b#!@0>gv=HU*IbEkR!tIrAEhs6J|7QISaG@88j? z9B`23o>r2e!2~&;Uj~D^2Z;rL2f>s?LM(!{U7nOe(q&oPSnAH}@WQBhD=W@1o z^}Np|-LDRmDy>}5<-S_DcG_d5M$R>^47?f*3@izZa!(R@ADn7bda{6lTP9I(b;J@T zHUowkuFN8m7EBsm8(UO&IPz)CWfXegz;1V3;j>zRi|UMm7M>q}_?bE6tK4OK3q|(6 zcxG>JaxMSM*V*+lk7s0Aud+%A4LbFz??SUk*@de<7o%8SFoivQD9gd9m7OXlAtuOj zBup!SIqw4db79W&3%FKI5Ly+$RbI}vw4iwD1R*`Oh?b@13l4Bi1g7W5+^#tA)o|254X^Y6UUD&@Zjo8XNB68(3p9FyLW8$wP5IRE z=K)9M1-4lNEG8Eixx`GlW-tj0n9jK*yUx7q>A~!rt8)As!}t#}=M}JTW@JCtP_D_y zmCoQ<5WxBNN@L>(_Qvf(tsBgnwsV?ITOz3>5><?FfQ!|`-%lTH=ngiaWJn?WcCr@EIbe?yMSS4_=MZno$D2h#RNRt0=NQ> zx7TFwiAmN=dGbkDPGl70f18l??Sah2hZ;Yd?I%y*EO20TOkj>a&Zs@Xdp<{lB%2(I zB4f~F*}z~i@oNHen0$gPkkt)o7*PL1Y1Ec5$M#Bcyvl5)E7&yEcnB6xpyQ^6xPwc%lqcJ?PbP4B zG_ubKEt>jo0+-JP7Vin&g#kt91TwD#@Vyd9SdzxK!81$Da<;_HS)DWZUMfsFcd_Ho z1OC05np286C!Ju^$?W{Q+;r9p&3P|0l^7D|S7>o|=*{}!%`k(>G=R0?fk?yxu9X3e zQ2`t)8@Lu#vab}Fx5R*>c0ulf0FG-a95=<1n-;JZf8g-Gkmzo}F}Z-d`hvmUWR3|N z*b59ed>Gih3ar$f^1~eaoK~cy3c7s!6!m$9)rDX^o{u&x;wnx-sU`v}{Q>NJ7Z}(D z7=;~-gcjHw7iQ#QVB`{DvPxjHyb#S?z~m#qKGDRpU%VM)w8ABCrqk5 zz!rXid720F-{W$7N+u_jdfycc;}u!{WMZOBK(^yaM$-ao&jpOK8(69@=onvMDH34! zoe<<{V7#V~tyY0Oeu06n0IP2UtFOUI-vAcR4Xh;z)1n$!mNam`DBx@P!05){xuap; z1P5lr0_Iub(Z9^{=YC#g$Km>2jImI&s)WU-TGQ3asNm-#27wCTu$wi;b6VB2dRxPp=65(8)JH9z+TuB9KUWi_OhF5q#N z;w*CDI#vZSu*D6hes(6LYcaZ5E!S`r(`MejnMGPa!QcW*VkAqEDf7dt8y(uY3NPrS8?3av z%A#M)GG!KPi35A{guE~ZjxKi&vw#dY1@^uT=~@>UbrXzMT%OLd-EyIFzChK&M-`Ty zuY2=1it&GlvRstP5vs?i%pk15R+Ye3nZUk(6@!8lBgX;>%`A7VS&g zSj^C^zpw*;{qSquj+wJRf9LDo zG5h?*^&X#j5?^u5G+@thn8PN`GV9+4W?fAd6M<~~1!8QDyVAm!bN%oZ)1E8rAfS7J zxk7e_?0WHNXuZcy0A(AGK=YiJ$mWPo(3GP0UT}u z(}X^4KP<{|?u2sr%;}GQ=zd&F{X*fW2}9lj8*T zz6TsL7H}9j7EM=N_*hb0v7cL)i;BApDy6xUlTNO z0ng{fJGz?IzYN&%-fD-;oY~=3OZ#^(RngpeL6LWR0cZaN&dLSsGrd`J9as(SFb5?p zb99L2nC)IUW!bXdru>ZZ2fhe|A7ru-VE6vO_)pq^SvVwoivic7f+WTymSQ1J^#zQ! z+Zn|In4~6HtZ0mkoyaIvzl@k;%U12w9VC~+(KJfutNC69f0^==?g@LP; zzo;Fbw{#2df-Q@bw}ww_Q9aOH+OzMdu&CAnwzCP`+7s4rFn|u-6cIkj8?dc(%M3*Z z28jn5Tm>u_88}#Km<+FatYBzc5x{#%VWwCDlO-2dTm;wE0w!Yy&a($*y-heMS#wa@ zX8n7vNeXXv^sQbxLHf+@nf&K3@JydDsdT~_ojE&at!4=^IGh&MX}X8mnqk*MZD#EY zp?{Z)S-g2k3v$u>e?{!M zG$T_0Q{F}18opV%Eu(pR_I?D(lQNm1v_gy=KN3^-dB z9I7&4FPN}1djs1MP8I_nRtE+a#|z9;GnQTXbM4xk4VG`1Zr%yaoXDuf!0e;Ie)bZ} z90%?<7X-Il;yn3{Roa2&!D1fo3$=`&jJlrHb{{(`d4Vw|k)?D4d)+^QNO=Zk%XCJ@ zM{E}1%qF-9J^iq5N6*2}0+YgDo$ijfA8I;DIr7Z*22Ra6XG%AmIemsL zv*4P+8kWqv%p7toT>Gx+TAnrA@IX)Vx}gBGyaKb=14W^fbFvTlKP=#mdB8GHfy>o_ zrLZB;ecoXn;-urOR;5K~~-Upr$T7o(~GZ{Gzb-3M&T8`x8; z-E|9$)fyJ*Twt2HhIxbGfj&1T^tpU58vn9`(OX^oDo0ee*ddyc?0mTFe!ylX6etQ_k&IsFb_`tyL>?yPA7 z^NKuXaXF6o2P}&ZaLYdAzPp8U?g!4r4_I6Rwx1K=C}Q9g=_qdw$xxblKKc%$!3VYm z2G!~dx_pm&>KlURoMo00h*o;Fcj0m+kAGroV~=m$yQR95k@@)BjiLrD1~c>m7|IM8 z7#038yjrKsHGv^^9mnNl&It@Bg&4M#i!dIDwNW!V*>h4Y%ERORJjOY-OfI!-Eb_Tm zHhiew(f-td@2S8?LI3qa@-JTauJ5{ZuUq=Q{2e~U$kX2p>Ze5SjD355+POnB{FdqV zT+7J4X3M~m)BAvf@AE~S&pdI5O>9_m3^<|;95W7cFDlU9a)7(^DT~1dmQ@1xonl4F5}qozoTOv^cO=Uikgb_vM*6Y~c)ta}KFE*$f>E3SHY)&LGh3ehoaCF1IdFaoU8^j z=5UI%pX^*z>*ZQ~?aaTK#+m091x^hMy%oCd;-!6f3|*_&O24xVTFv@o#w77hl3g{mR)*mIN_wjr6rt7%Xp<7 zqPi>uT~A-m@LyB$S1NRg$7I)Y3Xv9qzgfJe$?Bw>NMLkko3>l}+=~kf+j@8;Ob&De zXI-3cpBLn0B5^R`$V>gz@$&;NI5x8jtv5bm;K;<*&ca=>+@hA1m5HU|k7g^E=#pSB zzo-U<15FI9>3XsgHJ3MvY&faLR^%k$$m%B0#lr2Bv2bhgsg-I1+!L%@rP6aIa*3u- znW=Szf2EY5OX`xqXD*_Ff4pWD{)>@OKeqWy<+Eu!U!6MAzvtD1LO$D90y-NGw%Pa? zY6)2z86B+(a56eR4vWKhtsRFbDLWunKO35sl-OR5(e8em|cdM`4%#>Yk!)B%}IN5b) z&68a%db?lk$UZK&TBN7zAm3v1O;yzqmpuMyO$m5pQ1?AliBsp#jm6Epc0ZE5!*1;*!CL=#>tV7QlHpx3M{7I|cX>p2h4 zUDJ0h*0Y|oS@`_LCMk`E-ERxl`7yokS(m5aF8ku(#0K|l~o3WsQk>vr0#0^fSB2&lMw_1*t6Blq4B?+m8c#ED| z(PoskpwS}YkmS;i7Gzw=dE89A6T;Mu zQc{8d>l#H#_X`Zd(-!c^|KV~>(H3Bl2sp_7PDMlD!U4e>5*^lOp0N})xEQN^WOog@ zJgYCDMS084vmPbItrHit@E+zoTzE^M-9qQ6V18nk=L&82%ECwePUgD~Z#ZZW9MU5a zuzZ)6+cQJA8%-iCS*%Vsm_%1OxN@vs*p->X+;X++n8~RZJeC=ZA}%Km@UT_1h3F^= z>`7>Rd}5(cLxhXgorbQUD|(!b8b?;L1=^`jS#H(QHUF%RrR=|jo=285UA*$w zsB$yAi-RI}qz8A(0tVR~x~`2Dt;Tl}PFYWJV)Jg8YVu$~14oBLlZeV~mi<+Z>*g(C zIU3Yj6tb~HIl_T&X%~x}#c@lUE207w0WR`uc82eJS1LH=M|OgyBWKc-%@R{L9M!#4 zx>NO2hJFBxtn?)V)+$CvVRx=WldC4&ylm;lwLF?fVAc{&l`jtbEd^|*t2RbmTURD^ zbwPlsS{QH08%OT83r)xRHn0RtN#ZQ#agj6L&?vlv(M8twFsI@}HVFwufmH#nLJ0?q zxVP|U_GiWa)i@d4S@B7sQz%QV+w00lvj>Vg0gjwP2F%+H8#%3hG#lSK5X$p!#R0+E zjH~jK9y0URZFTt1dX2SObuNd%rrsM`uJwzgI?d)R`mVJ=WZhdDVujsK4vY!?t zx;$YN?D}EQ@!N~jbwb4k^>r=1I^p7HEx&DS$n2RU!rY>eQ?}XYSa{!D6G<&MHQwAG z%$k>8+%n%LCZ0d%*v47AIqbre_!bnn9X}Ax75c}K`M9OC=ArE^#tRNjotV)i$2_r- zYr+2qYMo8^@k5y!1ARdz66>Si?FV#vp}!>Bglg57kRh9%)< z2mMSW9?arVIk4}GDvJxl%*7A^h~s$ zB~K}CNn}gTPvuk=zNNxEFP1RsesE+HDeMxNo4}B()2ifCAkVwR!6HAvG0;>)XYP&` zhCh~`0d=X9=Cm+4s%kLjoLk~*^&9cSN?<9>}xFQckWcPRZNL{^7w;df71uar864y zbUuJ?Sr^Rv(fFOCf{{s9YxlA>om@d1WE0F37!R-MWDc6p#KzIUut!~*L9R|n{LKLt z&Iyci6%CE{XJ+SJ@D9{0Nobt>V4>`bjN_J51R6M97#Ni;961dfm^s%hVADFm5GZ0$ zJXyzz<@x@4qlrr!)t($4C% znb~-U!=}RrHqZWNn!aIT*`YZ;2}&AC&8M|&qy(7uL^FyXMEidQs>^&Au}VHBUh zxbJqOOp2LQf|=rtMyUmi;sK1J0n3c6G`?Fg&-mFO9ASQ^fsyw{gW8cjY6Z=z9gH@S z+a)u4S9Y-Rdrn+wFfqNL&2ER$-3kU_3ucD`W}Ap6(;5Z#1)Q@!?0v+nCK=EeBgnv* z!N8Ew!0f@m;K9JC!5}$-k^RJBHiZUOg@#9uoo!7TkSVvPIM46B2408dMutf-~4S z6B>mo7{wA8g$0f!{Yz5 zjG{4&d=&>3D;gCgnD`8ii#%v3^hhJkmD#7!6+@jBq`t|VZd1BJul>e zgk^=1!~vTrCs&js_5 zFnX+7ZC}u0+ThGSqmxB}bJC3|VhVN)0t|L63>+2>b}i0~0=hg2M_2-m_!x579$`?O z(D3*&qs;_Hi-wjEhSmpNEUpnO9v?Pw88Aq5uz*hV{L#Rq(7>{zF-4=*{s6brf@XUU z*7P6D-xnQ~3}9Mxf%9L&!&OP169v2`-tA(E-oPrT(YRNqQRKyGAqlr+kz*+?2gD9E zN?Z7>I?T9cH>2SR#tfO}3=?L<6@tPc z0t5ez2B{m(PlcB7aV&Z4af0_kgGxh-XG5dS&ITq82HqKs5*ADj9V{L{n41lD&RD?d ze5cR(;$hn*hb>st)J`yqavXV*ZO^R0&@R!?@t}cQf`NNSuZ2Ua>V^g`i3X+}0v;7C z4!4?Y1zMaBtd&^NsBF+UBlsUDw?+fQ3x>x*);IipeK1e0_Fqg4Zor2&(Q1(R9BWsBa+&#G90Ry6Vy zH1JJm(mBCo|3g_dVxhYRW3UnDbODCx3pj0U7;LW`W^rKn$rCD`(fsG;VTKF_tsN|y zD;lF0FtKO!iymk;Z(tNHU^dZVwfxr`Z1I%AwWCF+gW3KAlkNj%=LgJA5-biE8oW4K z?k;M!Jix>o(UcX@GCim*eMY0msaXOm)=jo(4Dy)h`a>aV&*}6H4MG8pf*OpR68j|A z1}3^Nig_?5zqy$5<$B7IMq!Cdif0>-tP$K@S8+xf7wQCs_4lujlXjcEi@a#?l*NO&>KMcG-8Z;-cxF4A3aiG<~p-Fdx zvf~2gpo+GT9}MOiE%p^G!7G}jHZUp|G&|1dcR$c%SJ85%SS;^FTg-vB=mShPKbS27 znqGJ`%=9=pV~m-2v8^6HWXdnB6ZhP8V&_V`z51z+@B1$g9yBzKSi&p}j<4v9MA=hz7gP3t!#_ zlerrh)Hqm-8@4+zoOg{7W3^}$G-!Od`QpAi*B@TKF38~~=X&gr&KVXThep*1nTodb z1FfY|YXpvPS1sDOC%Ump^lXpxQkR!|pXJ``Zr;&az^L+}IeG3}%NI<#Cz|wLFgZPF zw!Oh@zoF6mL$m#XrrkO%w@)@XDX<24wB}p1^B-t(lxXp+X!HEgz_+8pR6)+-(mf9c zChi>#nk(9NDzrwgVC3H5%kyI0y4;4uz^V6y8SH!h9cEUTaBuQqTNMTljt0>UP1+Nf zyk9UfeK^Orfl+!!qf|swO44fE1FVG?+VnIUY&2ShH5#}SSPVJ>lr@YUqFd}Nn*Eow zm2tGhMy)nhVCt%o4t;33&>@yp;DT^QlkkuE4hpPU8yLiP`0ia3F3IXsDe89MN?@`` zgqT1CtH}YAyhh_^vHX7ylmyA9ausBGxL|3rzuV~=*V479R79=2HFU8QYhe`KFlkUF>!FLZdTyh3JiNCjI~Mx*iqHoxX1R*4;xISg0~IZt=b&SGG_!O;>_8UFRR_W_nO zvZBWf#oR;|^%8?E!UvBlnC%@$}c`_Q1$z-<43QH9~Y$`1zR z3C*4wEFY#e^Ui2;SkSEUqPa=%0)I!d#|vg136+2jX5JSJ1(#d@#W1w$zG&d_*uwE( z@&>Kw1SJ8cg`Fv8DI0U2F=%vl-)?l=&>FL&)yl!0??r=+0&9>St5rjzSU`)&ye91l zk3}Pzy=LrWRA{mNz~ofX%>AHWsKKjWa3A-E2BUzsAOSy#4GkGqO&ms6i-Q}UXtXco zUa0KxC};*7j{#%rKi7SCPA7)ExX0&n(8ev9=LXB37t$SF_hXlr1uYUxVb)$@c3?82 zO_;mwv&Vs#A~r?3I9_|lnvu52v%$}@Nynf?Lx3e#fW7brt7qK%Y>rpuGn%C~FfBRP zu$Wgx`NBPwh}T>f7?fAUS(n^0*;Znzs)T~JT?@zCrcFowHNTCvi0w?6 zz2gB-+souW=UEO7(Gwe3Js89v@OZ!FJ{rNSxuMDG#nYRS&FgONV=PG5)Mzjj=wy^& z(7n*4JtKKu%MQC0ELvg-&NfWhjLumBY_1lZi=7!3H!}!7Fk13K?om{Wb{->-0OS5Q zxp(6-l2m-^Y!5WdebHg()~Ncq!Sv{t~)suP;y#8`A4nB)wa#SA*O|C{>%&;y?Gdu$F2(ckw*|C@f2 zQ=n1w2Lt~P2I&j7r6x?4xv#VyG%7D};SIRJ9l^wOV$&7v2C0rlQ41!y3p*z7X!e@X z$f%IWw4p)FL(sya>#i2FT0;ZRBTkM0M&IuKk^d~G2c!PccRj8y@BCcesYzR$ zX^`q@whd^Fxxng_&K~dZ({ox%P{mEFf-hV7`uJxscC?)EP=Ed5O@k!I`ND}HY{sh@2ONpn*0?P0 z^1nv~%yJK6SZ$fu1Dd=aFtYye`&7`usnocNNrCslIVpyT))F%(H#E8yoGv-f$$p?Y zcn7oWj7EzEvxPZWJUiHI3=Y(|Hl9dttdXr~lS?)Bqh7n64Mr-O!Y3HsK8@4$c(6C+!8Z?MhSjAXmMw_>Z_6OD zjCsLJ>zfUH3HC9n3^mG4Y8kKEBAB^T%XkWyqo%Z|CRngfP)K0n;1KYbv$7=ZqOSM$ zM+Y=ieIx&^bWQIuuw2UA#MC1orI9e{&6$~!jZ==EbXXo@;$C)XiJ%e-d)Ew|hy@EQ zKdbo(n@69qDg5lxEo>1bp!$nv`r7F2dcL!soDf{-Q~7F7Vsb0jR0YS3H3c6Zxps*t zS#(TLNIZ0tQ&`F-;ja=qSZ{=Rz>Lvaluz~UYuCiX)?U}Q-U77eu z(V0zHGQr{82F6WoJQ5}a&(_S`%*-n$7qKD0$(e=Mm#g(xxd&elhoGB%hPJ_>SPniB zlN&c4n18r0AY|g8Fo*HueLjmHoKKGT3vlwXlw>45v6?0!acW24rRV1rJlwM)iY{9_ zGTlF{7-10n>dej`|K6Uvs`2%o&-{>YQw;+r82*iw_!7wJ(m1E^5NEB2kgxHv04GkR z&KHetQb~JS4s%;%o?!Aff4Jdcfa61j5bkppi?8VVn!0i**7+=Dm(B@bY*BOMICzyS zVX489K4%XFrYOEWrj5)+f47Cjma%rYDqMcD>6F%D56Pt(i(M?e^sYw~pAZR7nRt{{ z!a;2&n?Z)dv-CNM8zk8lrzRX^U3zk50~?owqq}(Qn~g_i3fd?*GUmE2xb3!&@p!|+ zLrdG`ig;Mm9hkK(+&xrtmRw+9H2cBWq1fzl=rD8ofx;u@^UfJKt5z&gpO7c1xPXB( zt!7G>@g$qaJUrbCE;X?-&-u5(MRmf934tt}A}{B2a?IOskaNP(LQc~Q9-Dc#o{8x8 zRPIzcbV)+*VZq@5y)6bJd{6s4-tBN*DxGA!T}8E1WwH%ZXh`W5Lw4!R7{{Z0);AoQ zxh+=-h;a#tc!fbS;k$Xx|l;zVY0p}KZX$0jwd@3?Mm2KwCnWB7OfFbYj-!vJPJ~nd~ zcIy=znpjvY8V-u&>~UdkzJwiNPwGa_mAV!m$S12+%(-^2)N64pJ8BD zS^h!l;H`ffwF0WRKKL$YO=l@`l3D)G#hCALPp~!L)~iW}Jhq8!d(td0<97k`^SRp# z4)1vIY{I)89(R+tf_-HVE!tc9BC)l{+iZG}*vz0|R4|`jSVc4fzri8l@QoCtB+; z@^^K#7|mJ8@qfv2Mv0fR-IatocR4Z%o%1-h;ZJg74~L3!$0?3GQwkRd>nKU8?KrHt zB(bmbLx7xJOuJ3Rj-FDNcTqDs1X_<6uo?bn_}6K9qLF{Wie{N9uRHzs9Mtto}Tg4vR1E(!U>$FHj56@p#B)%lUwpMWCSJp2BiQzw8^#F&CPx7!uh91ROt# zFmky6VH8@LuuNHGA%}a&A;H`XMJA@{f~+bJSkfmvGnu`U!}Y*Imi0jn91$m+!uGym zE7xc>(wf-gsU*=0O(JZj*2F4r*b(+t6Zn`_x_=Im7E2rWnmY~( zPBEBiHzkS7FyN5j9LH9s1z)GyGWRjpBsetv%WUTx4+`(L`#;!UAo zYrs(3fdcmMIN&Fe)yubCa@#MX0bqChLZx{ny+h|l>}m!muhj$(9&>L*>Ou~LX)D3 zBhMUvUIRvz1x}g^n)s$TG1xhIyB?PR(7+ZWcJHY_lSl*8gGOTyXOkU{(N7yyS?=kI zo-VNHE$o@c!NRaRa9ZXGHkKGC)hnG$TMo)v9AI1_mw!X3(1GDa@4bnp2lQmmPOQCF z!1ItlPx$#yHV-IORXJ z&T+ZQD80w;%@n~B*XaU#8p2mFa2Gg4crdV^xZ|<(=zNCr6GCQ|Cm!H=b8$`X0k#YW z&M6FRGa6=`arh_rVUAwOVZ{fBOhlT*Bo0cxX}t3{T=<5&LXR)o%0q$%jw%++*WOO- zTyjWAhe=cBprFp7hRg$ke;gG)FsyVrBrkK2YwslXz-i1J4NONGjdn2Bi!^XX9Fkoi zdizcEt5Ru}B@QMLO{rfUC8snnc{t0LxTwr=Qgv}Kv%F=Q*wQW7ZuKmv@XImoUqP`y zFS!};KFSqa9TN8DY@Eu9Yht&WZLc)j-)nZ@Qn3?R;wbmTu`ThvZA|vh3&)>Es_jvf zImRKXaj@sl%`lJf@GW=VE;t|{BlVrpKbmKe%v#oeF=u(#%+jhk$R2Yna}6h3&4HCQ zjam*4cf1C+@G`GC4Dso#&U6{}y>f~g+;;>D@L$RiXPC_b9 zacXH?DpFQ4Ha8>{Ri?k*;P^OH#VxPZd&NbmuM6M(>F8f@R6wQY{E0vgov4pn4hV@P zc$g%y%wafX%;wy|7|P|4_AqE4qem5k1Lqtk6%8jg z&t|0$XE`T?$!}@kd%+Nu)$(v*a<3+Dl$4yi#PNr!^Sal%zutJ+?Zl&lB4RhdacU*ga^rTe6~Skw%)7isUCbMIR>@*P|g<7-hCJ_GmB)c*HNL zJX#WXz3+^SLQ;6{k%=;fQ7kVabXGCR&zXLz;XJF0&~nan(+P*ZG&G5sco$9TIa%VC zcH+#bf@AD5jCFGvHI6t|Oq5MaaGP?7bIz8Nv)=Taw_sqL;c)TgoWM^GJVzS1L}Yoc zIB0e{>RbB^=DS?lqO@& zH<}p}byW_lS~&5uG&H^PXJ%;Nd(g-i(O@{Mfr-OG(!|;3rn~AJwkd|aDXeMMiykl1 z=v}@qB4y`d2Lr{`C)_wEILuw4q*bt@{oG;ue~%rc{w?YH#(e%8v;BuR=fCBgfA%W% zTh{uz7&#wSnJhb=9IEj7SR#q|=8kJKKyM)SFiMM(#?>}r(R@mA!_ z0cMeg2{BJ|zMNq`(V#rXN&b$b%!9_Kd`xmb9OZMmI3GIdTJ{{~N$X7%NcV`6kI<2s zuphBOWJl$c|w6YeWLiC1xH z(*5QrJm*x=qofOK&j>{@v3N0Q&0vyAIKZG1@cz#EzcvkA1)@A92Tm8<;5y>K9^=5S z;b?5(G;xKpP@Ob`WwWXXpO!_l(Ub;;ISxisnv|wE8JRf8eLKcdGfm%+fmx?PX!@Gw zc}uOnO|n+YIO(O4!79eq(r_l?0FR2a>5OKp6^FAbGaROAbS+zP!E3$L>W>!|723~m z3bttKNn=t}(Vb%(FEwM?f=buECHlYaoeN*%(8u#B{DUIjBf z;X|X$2giTgu50-m=DE?pd%}@-MFU&M$-s4IPSqS>e8j{uuV@V`(>7;g+_S^N3jcz{1HvcHV#Yz2lN{CUa2z3 z8FcW4H<`qwGOcKke83pIl7Zppq{9`r--x*{o)Gs=D2u7(v(>w&HqV0IPg=F|kPhdW z2KhOMOeQefwSSbT*Z90^*FPP4D zu>=3!tL5q4+rabRL`;6A`8y~W}D7hZ0|hl>mtmUrd#UM-D% zpBgKF?*!8cL9tW!pFGa_ut_Dr* zMZUhB!BGCM_yBW)1Mh`K#Un3F3J$9*SkATOpu(Sn#y-rl8~mAl8jUNM!_PM?f1Rrkfk~}kTl#HAN7*ah4-_-gJr=$^5x!!ZK*|=ih$W(5 zc7z5T6!meO5Z~DM?LeO&qga5E=lgHonsZmn9+W!LSTre~!L3PVN8``0j@OnlnKdw* zCor3dG|3Az3I1WU+QMwLgjsWq!;B0zpRP`xBMs~!2aRif8n-Z;8GL2-VN}@TsQKWK zoIxq0$3c|`hb;Fv{}W|6q}o#2`}II-l5~Byi)J<1Wt>X8Y^CkH?E2QX zOlb=Z&RYM@V{v$wki5g>fpPok!yoH@biC8Kz-E$qy{gl@Y^^=Bu?}l+!JXMJ8NHV{ zZm&2Ukiy|1{FS@pXcoI8e}H5E;l67DS7i1alu0=#vGJft2&32o`SZ3$3WhtEuWY(h zxJg>(1lt6!|3?|c-W;sHkfh+yr1|2Ial~O0pK^sW3}QE$%x*N9+BC7Zn6r5}9GHES zxrBkSqCq~R*<6EF&xV<;!h!KigS^T?8J{M3ldX(v4k$lp(hN8($kU{>g29}zUH76i zgD58h2g4HP<8CokW`7^lJp1=GUa8{nHN80H!WChKRui>OePL7)Fl*EIvn;5vT;gnX z;c(LYk83Vlt(&}K?fYgEkLIm0OckGLj;_A?#sC{Y38IWJh{L7 zU!%OnQ5A+mh0IJsM-IrGI4I)d%3g5)fVV@%dIp(*W-}fp*$`{RGmNZ%P6}Ey$tN6? zNnjN6aeVysM~DTJN=y^~lFj z`L*kTSr-^pY!0t=aW)Y!TiyQgqf)K?Kl5$L>kZdkw@P)dHO`1}$mG6Oe0oDXN1IIc zfiK-^Gi2WEaeQ4P(=Bu8)eo(|?-^C*GyaX6wMvr1VUFeHmW0Wwp3@8_7Ct+{&g*t= zN|#6n=R{TCCpugr8UE8WgC|I=y!70IO~NQlX5pcS2b;MC4O}LCIB-xvb%##riU$V| zxATd~SY#Bk`OUG*f3zj?GF#tNtGX8&P0b4(nbgF>MOHuGAnP&J@18Ncc*r^>)2@j(X)2TN)SzRMIc5@?0&ZD`lDb@~xUi#fJ`0N`YL${uPQmR4CZoab&lUpVTHZ_~~iR;Jg zSRm*!Sx77U#+j9$rwA-`KF2RROC;8+^!0gW)0+j4TqeJ}H@8GQzr|BXLh3@c#YUIK zvn}2*XZ8qloNVJZNPCfR>9{W^yO`Vo2iECKyAzmrq9!@8ie`K`(7?_0A)$fEpumBV zSLT2NGq>J}hE}0Ck49$EAc-|C98wJ24Xg|ur&q7yb9mjrx+_ZI*xFl72iEL<@Mx9p zraPOo8x?{>ypM}~zV%U`!_P+0!Cs-VP3nkfCH*nzx&`-LRtiVVii%3dIhE6$_Mh{1>`9{ zomQW>GgZh_edU%34(D#H<7_;a(4^4FW3;H#Lc&V1na8AIA*Z=e)%t!9g~YA>Dj^pX zn)&)Hl=_VPlD4w(MNE32%BhrbvXR~5LjwbIo5q4B{+t5}%?nI+9AJ`u)bWt<{{&Zq z6vjemS-onP6o(@tt5d_u z7N!pkEmB!?JaaZ4Rp^#5D!sJvjJJXHYo9YlXH|TC(+o79nOa`esN^wvb0oN9YWT8+ z?3yb!H1RrpKk;Pe3$?jV#pV{jw7e8k##bfg&VMs`x9~ZRmfgP0jH&+?$4rkn#4DlW zpwG-5aF-!8*f&XuPub+g0_Hzy3ST+AFF8J+lRiO1!I8UTYJysV<%-0<=z@dGr0uyh z4m6oB@n_jd>paG^tKDzOqU6o zH!G(E9(#LkTJsCVXETjgF1DPbJvUWoLH^6gKOav{-@-4kL-<7IdPPRoNe)Ic9(LHB zyYb=Gmz=Fu%ximRD0A}bEokJIImnSW;V=)|!`|Wr%(8VBVH=;cF>(qpM2PQT_$RyJ zU~jt5W69!*0M?#@?#u^FvJZYVJXD#`#1(mhkyAmOf#ZV(qfLSbCkI0m0|Vokzph0r z72*@BO4e0A&DMUt%PacLQ~jNPwDtc#>S9)K*uc@UfJN!T0p@!jI?@7^3py$#ELSl~ zW;v)yJu^_Yx+K_N{^y9Vv=Y~Q?>OyWIPjvC9&>`s4jXyOirTM9V|Og}kO>;xlORKysWl}{)N7<=%uu}${+ zVP7D?tumdmID?HTRa#tB(I0w_&Rm% z4Gx_zuVQ#>RhDd@vrgYv#XFuUxk(^m0gG5cvvQu{?puM$3wczJACmhOkToMv`uTzg ztGaf{hlObs^M3N!&-l3PnJIsN$_n2vEfd=8nHV{85}d`RP40Xaq9rmpC`C`|rpLj^ zWU2lwZF&`_4tjVU;*hmq6c!6S?K&q>aN+@Hg}x0f9y*UWWG^si1zOJFO*Lhf(2$nh zkie=f!)Wg|lTqlAf`*e1qa%MqCyUyLFs+zJGFKKji>=$xYUy#XNpgde;+)6rnk5OW zItLs0>i!ieiY7G7X1dU9JS&mYbIu{5X$5V@w-z*%Y!i0noH3_2Wy#bvM^3oeow+V> zzf@H5>1W-TNTGO*>kO69HDL7&;Sy0c z#}x-RGKny6b% z4NL+H_?LGu^Dhhd$E+f=koR1KiS;i%sJt9`7?~pRM#d;+BRMy7u3HO%nQhT+u z}eH&cgHvNu%n3lZg?uylxJ=HOmvnQ56^H_Xz&z6fdFOn;-)XL4h zv>?`O$wRKp2tU~rskqsF{F5#RysuJ}&GhhpEEc)pD!U&;^Zb+pEG%wQxw06T-ugbt%X!w=`xj0-zlHImpIGFP*!2uxu=P|8%qe2pvRfR9l0g?y$-^Ldwj zU{hgfT&neeQ6(<4+0lZTU%a4!;Q>RZg#{D0JjYj_S%RDr2O2J`&G@jfi)GpxpUN$n zE4`+btL)nGB=vM%;o2N#Va9(#B3l;h+}c{uyt+3hWwA8#d$)tD6Z`UJJeE1)`fkq4 zGuEX?m>2ImZEW`^C#d(j+J(lnX1OmD_PP|G4>Ye;36R~(?UBqZeQgJu*|x^s>6>Nt zDw{;zFbZJ&5^^?*W!f|*nGFk9&%4DEy8G>4D+PO1nZ|=Y9vuAZwA$STlK9LXIIMdr zX3lPs!0N2fz{2&RQNkmUFY*Ngo6j3zl??7k{tiZ=iVy8PR~Urq8ZI!bDc}m(@>ph@ zM{8wHBbWCACZXjWvMK?b8`dwezMHHmBydr5*On7c^*uLvzhZJ`;=NJk>@C5fwPqo2 z4$qt~OVdAF-ihIBGq#a_-mybcRqbQNzYB*SeOs{3b8YmdzU2=eE4qIev6g{@~c@^@hC;B?-(g9ZTD^Oe7dfOs6i{HnlM@*YrzF0(0o;AqsqY~{eWjKTI! zg20MnjME&HPCb;Iw@{YnA#=b&p(hQTN{Ui<90cb$vNbe)yw=2Up(*B(YFEnYl5mtywUoLBOX`itn5>M;^!SgVKC?4<`tG4eS3JmtTG? zQTCeSw-<`vPdGgG(b~<#`@}&{*lk^qKhqS-<7|{qt~1xEE32Lz2vD^(IuP}1!>>!{#V#mHZFwkJwLn$s0CUON={5)6 zswwguW*1+zfJrHZ?Z`QfPYj$K43S0*iXW1>HYEsTG^nL55XoC3;>Ea0$3ba^f`Gtd z9)^RGA`NO!Jd1KX7?~UxxE9Q1a$wBLNXXOhc+{!8;S8&$zW$;UY|M*2&00ABEfAS^ z@a#Fq#x);hHVA&*QYgK0iy72SD1?fGCGSkk>?@W|ZJE$0N^4FUHe(p6{B1O*s#5VD-U#8HiXIt?= zFemYq(L#}(FC&yzs4`6uC~yqoS}5YN(9Yrkvl^qg+Cs}L2i9wR%vlPYU0mF+9&m8< zFqa$#@YGH1itu1&-a}Ft#qmGk3qf90_K3!L*LThO-T61I*EZl z=0Jf#0tcVV@t7tywTF^w3*S6T;D5oucHte0q1wkcP-*72|HS4=#>8*y}s1%sva0yEhJWmBIBl?Jw1%4|9v&N>-v zS^plq&0&bLdcf?`tCo9!SL*;9N8|c+c?L>}0;`^CP;tQ;$Qwnvs)eKg=xRS*nprj~$>tXQu zg|adwA@U|p_b*95%;`C?QUB~j!zGC_OB|)PtbV+00=M92`G1K)Vr$biKJ2;REnRn2 z>WHHJbbq(xD@NHR##2k(4j+`7v2eV7s!3D~VxQ zm5}2$VS#^13W-Pkn45$|rzrAiEtJt(C~4HduuYFmxWPcikuB{2%Y+4NB?@dk4SYNc z1FaGSyd2ql1WJ>hbd;!kVhC}cu`$7`p{vb=ogqUSPhJn-wF}D5vFN2d-!%2gH}*^gM$wrIm?tgZU*!;<(r{qa zSADZJQ4@`wn-2Xx!yr0~Q8ecuzZ_FePlEuPLE?l5%+nJ1>sa6V91vu1?&u5(6XgKLx=;f4}>ID~G9Wcl*6)O6g zE+uSa`9h#EG(y$5nZd{{*(4{kBxLEhs&&kk=_s(hYG5}|ur^-6|8D`m*$U}551D^4h(3D2k@l7+ih(0Tsq*3? zmJbR-4i_ZfF=%idV0-aE^1{D|vUUx80gRltI&~xx7*AA`bagN`9MS)@h>cZ^#mJP! z=g`%hhgGK^248JVzx+(5;hXNqOEN1Q%YS{+k2MVoRrU>eoVi{>c$dP(Jr@E`xVs)L z*n7mW?&62KquX)}ce?%F-tzBy_TNIOZww9hzA5}~cJRz?+!|`C?_#RE;Fn)2+l&Uz zgy%eI2YAI21wS-oH7JFiVcbh=b%`DqUgN@ z)}#c17Y`Jb9E;^Q@yj&w`5Y8mwouxQQ}oUO9m9sgs198%ca{$a1iN;K{CY5pMS*P% zgZQ$AQauXX0gjw+o-!PpG4Dju-5Dpg{?iIgF!<`ivV>nQ@%<}_YG={@t7-k41<$EG zTbLjk#vN&D5@k8-u{?)jczNl>Td((?2&gN(B5zx4Jh{~P=k}#P#bu{lHD2B*V7FlP zwl2m0i>JspD6XH{?0HtlckPRob-&CK*!&neA1g3VkPuZr^CILrZ;Au6l`gB51FP4k ze?1C(0S`rH9h3}E6kt#k@Hr?Ml*k#Dz^TG0V7Guv<)GvZ21f%0J}*Y@4GP>F61Wl+ z_yZXE8}eRnc2&d#+eHnCz$?9=4lm>VO(;!eUHWm z!|U7;M;)biT{AKbHa3l1`oFwB{?^i~*QEFyy>&C5ds*)C->Kj~Q*r6kE9d-~r!C)7bkI zT2Cj$F+7;Bc|dPjPi)(g4uuB{N(_u54tjYXf>w(%e|R7`g|Yf)-p*SK`+}vVwM}I7 zrKAk1woPZ0n${>ehk1$b!->85;(UP-3;%2{oU$W%*3#I2Qn?qeE%oMDcJ`a}wQEuy z2iaDsMtRP>;&pS%W-sQjO2t;G?3|_h7Vxr}J>Yl~!gsEDjmT9Ek%Rnco$M?Hd|Zs; zI~oNhJP^E>z}9s`GU$fnKL$%nhxptDD@+)!d|-HMrLbM);Z!z8J`eFiiT00ocy&b_ z*u54Cl}vBab`(@vD5N1F>eX0Uc|bR=r6=8Gz6N`Y#`4x}@AQ+D43ZY{vpp1%`?cjZ zV@7wZ(S*M;43^H!H}$8NZ8NQsTryYk$mxlqnUmHix;`$Sv{TkS`-{>3zdOE*=boNc zf3$J(`NsN-Va9W>a+KtG`}*!G{{ZXQwkd}Ps4^=tiTo2|pa`T|J}hY}Uoo-GhzTh700)hxdp{xuFc%NY3B9NFp)G5uoT zH(HP{GP|vVwJXVB+bbpyg99v+4!ra^$x^3q+3G2a&jQwz1uRPLF(+Dg`R8JF zfbHG^uCV!hdjEN!D8xTqz#rmp{6U(RH^Vvxwkuzg7)%(10uon+FtCX`{51-5=m6?IgebU+m5B6MS2wlj@D52srL4nb!d1J~+w~Pf2j6Gfr zY5zQmFRS008PxsA;8+`c*Sw3_S^VtOL~GyNm@QW?E!f(|R(*40;igRe-L3yX}FItZN= zv~*~?q@*73kJ*`lPi2EcJcq&p2c|~T1=IUDH5hL43(i^4%w)P|AqyLyNdsdl-<^ji zRBz5Xtd^cLEA3on(JHZCleK%rqK!7m8s9eDaW{;|c<;-kv&OpxzDE~5iaFftaL>j{ zpyW)Xd*;%qMeKdW``6LpuMIj90lV@sLknvZ%*6kE7rKBe%kX z=~Dz`7#b3S;xtqFmj9a~n8V#Shsl%ACP++Drq9Ev*K+Ab&7zaVo7B!)R5%s$nAG1p z_I&;sgJ7ThTRxv>7aWt^E>mphFD1Kl1{bfN_2q?|=a!t=+8n$1T0*rWqj-$Ns>+3K zN8+j!4{mT#i1;CXLf*YX<%$!hX2}uB^fLu)!bL7jozey=49#3xGXl!*8r_)7#A77U z%E%s)ak)n|^32CG#@lNQ*!6avNj$8(<3^Ax&*VFwP6&ywc)$>0VI*)coMR3vV{eGr z0|%C=2R61Dbult^@EN!+W?`t>xO>AC9)k-i%ppP#8W{L29{HH0Mij8|PyEw(Ek$4o z!$GN*2A|$J*CreDeY?0~&!*G=bT-QvpV@RW?2OR~h2YN%55Lu!;9wWc_5Z@H$ac*V zFKycirtK95aaY5-r36o=yj2uyICwjRkyDMONyL3eKdZrn2_19JGL@WeU~T>+z!6oT zB3pQXS?>cwwXFk_gh!*$iHkhJDG#}}?>MT`wxQKi#)(To;Yi-54~^~*nnX-rG;qlo zG_qElbK~vkdCA}`=6ZmUVOmBr)0qqHk}8Wu&Q5S(W?^Vx?O2+?{-FDy*an8ZF_{dL zI@z=?JzABv#?gqY!GXg`z}CrUQPq zC;c;r`zz*&6IxoDofZZ-DP&Axulmp|m(UeBMSwTvP7>cN16OWGfoA6)54q$r?ijy# z!02@2keKZY26nj*4Qvd7vrT6?FdZp!*=yp!#L3XWS#e>JkcIPcZ6B-x;?GjjJUO2F~ePG}dSfKM->2#mO<|S+=6pR=H z92i&{j#PY8W#pM~iBYcYyf&kP(A172vN{FqAt48CUzN-g;wbF4@^s!(`skiZESvF% z#FDKiERMNZzq;5T<-DY~NdCf`=_c=Xm`u8mm}&hc+~m2(@eG5heHLpPyQQY2Gw?ai zWxjV*xuakcBg=s__E`*EF{TeWie@kcJ-j?ITusH^&5wTWS3fNS4Z^tp? zWdV&$AqNyAz5AK87BGD2dCADK!-0Lq1$MQZayIvlMs39(4$Ci1a{4656{_12_le_& zSZ6hx9>+pXW*$Z1?K@bEUng|^?KKsj{K4T#!NYd0l!uZR7dSDlvto6!Nfzp#(IO(_ z!>A;c=JwHxXYVP^rGm{849PzYraseL%5)-%u{w}pO4I}~VT}bW|8yrf@-k`6Jo(09 z)?*L0oOdO+^7C@4UMR!GV1^vK1QqpyP&Ta%jFGc=_sASf)iu2^foH{nX<|N%50VTzVy=`6 zwcKdhv(})~$G}*)eMa+A&%3Pr4GBV8YZ^q$l3ST18um`Qa9>v=mzg=Bf#I#tL4n7X ziy0gk7z8-h#N})+=4Vgy3hxp+qR_w~dsdsNmDM5Y!OX236Amz0oj9VpWFk*`wye}i z3%Be4Pw-^PFP2ELpT9M6HCswfG;e3cvHW!#Q}Qa5gJ(Lu+_`nmJDuR=cPjstr<5Jq z82vILS*n6cv4wZjgig&Bci5sc7RvB{?9mQrIICQ9klW=# zL$fLm<2LQ*@tbyRV1Du9uzcN&Hc_`)jvrDRa(#Z8uQZuyc1uBcazf z-;0JhE(~lRG8sgimO52#TH^T0(EIDd5VZx2u7M)vj*2o0j4}bNbp{*@-8o_d*sC|N z{>yab2*1EAw}8_vfmxJ+X;whB)e25C1|~)ZCbI(8a03?R4~)?p*kd0wRVQ#28x%Vm zFzRn$(wxA=E#Mz46uIkhV$2g)r}Wz3rjl;aHcxiH_Jy@d3z(V>IFD_ptNBz~zojlO zgXe8}drEnk+qAN3i~6hyYPS=> zZDIde%27RmRVSjVTERJ5fi>KKHFg7g_=N7#1)M9>TxBLO=_j}*J+$U=sExfL&i1+0 zS^b~HRdos0!wg%@#5zwbTyh^WlfE(fePErvfpcEM#Fz&x zt^tgED>``_^~5d;OktO=O=DzcU|zh#QLIHE&&*W*0&{Qxv*QEywr_EhKkyrsbhj38 zOcvnUEK${@P*web|9xq<;}i`6Z5M$H0_X%~1tc(kXO_g7^0e`wGBxq>a@ zK>r^Lwzhv^85YgFHx}^TQxLeCm|9Uj;e&8$)%FgfQtmeP0>1~Wa}>Dd9hfQpl5@@h z*7yJx=?fG2Dg;+PW)KkYmQ?iq=@{Z&z~WZGVo<>1vw?BrO-A_u=FkspUJFEerdjMf zAT3nR-NGTO`s+Q4Kmfl0<8vF@R`q8fud zn_HuyKkMhj`oiA+QdcGe27ZnL$qSmP!3;*nEE<@1Jm^2afLG|3EaOUEht1`07L-3`OYN%YIBOW(7S0l&z%{#b zqI%cF*$P}@J171V_#t3E)k@4kaN=>n?UI}~McE5Ca1<4=b#GuX{Kyy@z#bvMY^cC& zn7~|`Fy~k%*Ukmp)+|$;G^ey1EOnmAeP|_jo_gVA1J0cbEc!n*n=N3ny)ZlY09(NY z=6jxr@jDr|EuI(mG9s?C*Y}cm2IqX21Qy=~tktVnV=u7yeqfyOBePAAQQU!JNx<|Y z2|TR3R&+-#=+9d4VLHc;6$>&LW@H+$IXzgw_lqYkD#Ltn$`A1gXA1LrJ14x_R_^|J z#uBTA{tQep4`ymsO%%I1bM^twIU6P}@pQ0p(t7K-*pHRj=YhO$0bAn*`R)xI5f4n= zJ~VxN$|%1<(OyEy#)7@bz6Ney}5SD1nyQNOCy&m*A?u=39PmO zbFX%?6gaT=7jVwl%*5xAz2|Z-cY|x(%-)wP=T(0cfA7-i`eC`>0#-i(*7yso@e1tm zA6R`ISok6saziAGSMvPhTzSNSlXcfdHZM;0Rb|=B`!f>QlN(khUs#Z`VddA*j8@JG z9g#9ix+W}pwXpxD?1KZm@59*RFL0`7O(m17s|H-I-4nNK&5~W27RI{RNJ`$| z0!#M+mVoRv3mVq=K47aB;M`%rQFMVhD1bF6VOjBlCG87(&bBP=6Xpu%l4&-txGToz z>p9o#z&g2xCAOf2wXBI+tRV*MCI23JD`$5xzq{%rAhod5SmxG*8IyuGcV=v^QrFyH$T9CHrv&GgHwXBhKG<{n1NSZi zuGt&7WN}1lC9gjur=Yj{xRL z7kCT}{1^**qa^(wOHRwvTJAfkmH!4y%?H+hQ+H3Ce)`Z%?};@6tnvm7-bxHA(@Q#|S&3|FGvA_uktDyl))#igoUlJ`ls3AQ$mj;GeKi=4OttYWD5~ zww*;Rv6H#>Hgd0W;A*xx@$LJ*8=F{HE#O+{z_q~PMDrZ(<~dwTK1|-ep^DWY?A8aq z_ZN7tY>=L!#xZ>|_lpMZ5(QQx2lh$^W+4a0tz`xYEDRzA487I!woP{Bb;$02I?s2) zAx)n(jRpH$jn=pZ?2Nm>x};OJ&w*=20H@WNT|E~#dK><6^iJTovL@G2=ExWGBWVq6 z8rp1pvyOQ6F!wib#ze*IUfI+-&G>-yX8*K~GV?r*rO_V_^L9J!)i}j{^#Wh&8NT0V z_^v+KlemX7=D@^*&S{IC9ON$^&vl-yZous0VD2HndG!MiM+k>_EJrr?!c$w=%_gv1 z=i*Y-y`&V&p%}|CA%J700khl(1|tKG?Gt$KKHzCsbE3(C>+%J@e;-WTComdrU=oR7 zh-_gf@nU39VDP=9viaDuoiT^x9d=HY=BU}gG2sEn2`^3!MlJ#GiE$rT^X4%0PvAQ9 z=gfsO932H`uk@VF%w;c2Sm~N`#G_}oMFSIG7gNgvj>!S%K1an1{Cln1gdFC!#umO_-gXn`p zeheYq7tYAe=3%`hUs$kH^d`%EGmd2qToV$mU5+`Mxi_~=;fT{3ww$}y(j3_G|DG+> zyo;V3eJ}zRZF5>H~`|8LqYi>H>T!cri-+sf296^RyOuC@z z7yo%5WGFmv`*YTF&dS1GCXEFNGaWc)7F>T;bM9cIhL9C61LvOR6EZ=^7JhJk*j6k1 zf6tBmA8!0(;D2}F{F?&4DS!85>^Zi`vXkor1M5_=7mAgV74piNO!Wp&w>zA)+PDAo z6c$s3Wo``rI91Nxu2JAPRe5pO+1pDE*o_~gg(Wzf?UP=$foo4J$Ib(M?;r5m&E(#^ zV6vkh!_Ju6{-<~E-QBUH#+5aJMKfmS;tPCBH*73h$8p5qIYa%`^njgn5?Giwu&+ws zZeDl4eI7@~hx;npAE#+ONHbuwH(;~6u)w@wcd_kDiv{eZ0qmT0FP}`muH!wU_2=$4 zHk$8?8z!*WpZmz0C39@gg;yUK_%FWRWBi|Y>R;Y{0eb@iSY~NRt}x!n@I~N^X{E>( zfe(gyHyxfXQ8>{xhx?zYl4=rjXaI+E-^D347lYPwonCu;$p&^+C1$e(me2%_Jqx*; z&fLnr&-?xZ@6G~_i85Tb|49|4Tb6WJKZ{H`{UFz%Cy{Ya%$W}f{NmBiOZ=XPuibbv zbZ5;27SRLj+Y`9^8aO%&e*V@y;=JdY%4)W>4?j~9J|-m`NiJaGoy9y!@A~9(Y>MmI zx#ksq=;nOxlM(modY-mShbG^)Qq2id9?ty4dnJ0y{)W#V9&lgZz^!_Y@BRng7Y46g zUURbiZOAlJo9FwtkmrpAqmv{9`;`g2=M=sM*>f-1&|Uq2y}^Lv+PxEsvRub&mtHR5 zUCF?;GnQk?2likE)+B9Kc?Xu(gp6(;fdwp$hf3edS_Cx}J>^pM2oO_dV=?h)P&nAg z!XTt$vEhNzaW*MuGl7JKC&ktMTn_&GJe@aqsn?W_qf1mKPf|)+5LEZjctQ87n5hD) zUMa`@0vEeo)R9e2@0qzgWkJ_PLpSz5ndE{T*G4Ay>+5oE#9VYXyL{7UhLP{AmY0`w zcbuR1bdq)MEYm&Lwqyz~*S&Lp{{t3&w)S-fiANqC=;Z$XR@%C3wav$0Pfu^si<(yJ zn5NzrYkp#n*-{yUghk9uTrwIO3l1=4v#(E$-XI{|%*w-OaHO*B?M$tpogR0CM76H# z2JI+VswBsCr8r>cvX`^U*3Q*)Zgr3}ayfND#dC>==&OBRdy1V|);8}}eKBoaB;WTN z3I>J;|1z^X>0RJRXl!8q!@Pj8N9e`_o+)!Aj>`Y5zp#--C|S(&nA*$^#k7f`LU&S_ zbv-JVO|tIL(h@Z33Od}kbWRZ$?}}wo&OL^kZgpv|JK(gi*W`xBVm{jnm)APm)Ec$* zc6M#?mfrjA(`o4g7dW3A9N}^H*-*yu``N7Q62YUB11lc3iiKsY%-MX_BKi6BK(m`m zIxY!i9B5?Z%2?2F^+m}71}j~S1>0`?s9`YInq&D~GC0JrWNPFQWmXGkPtK#3&s{84 zL_$kex`~I*(@qI~>kt?c=)dr`$@JV?nd;MR#SXPivE|#ikcmmcL190`=i3RkQO;_V-? zaE(#oolFV2^17c#C;KkAv7mufYR2XU7K<_m2KF#+N&l-!mwg-Al@9oJrqwX_EsW{X z_S^ADrd4E$JFjc%LLbMMCf0uHH?O3>;Jc|O{`$twrviHpoP0XLxyK>b;omB;3CfJ_ zr+zRnYFTJ54Sdk(K0U^Ob*V_Qj!&C}tJun8vI&ki6FNfz7OEV{`s^ln)FkB2H`mok zJ%w)TkH{wS77N>NcelOpi+%EO!@}z)(znb!R;0T1BU^m!@2}rKAKiEJpX+pAmzz%> zv}g*am9{JP%@FsuD@2>0_KFRvvok|>9^ElVKGsCU zy|C^|u$Oq~QJWW+K3iHZ;GW51<;xp*bXM8DldRLrcNofNX>uqmXuSPcULgGM3bg?H zb-V^jT~o{o+vob7d1SPz!9`Q)qf98z8F9_VCUKJsEFn`430{>Flv>l;X0m0W=B+Bh zo)ngK>uZ)4Brr0Gt|(wF?p&&q>M0&2C8%@&{(FKPaSSXQP4YsL-7eC3oIPCP4FK5?o}isEEP%@u~?&XT7rp8Z>R zW$_t~y-pLHGHwZ;FTON+o?m8ru${NEMvq#@$C+7i?>`({$>-RiY2%R1t-#>^$$0;t zJL~IOcn+}Fv2Kemc_ciuuwB2T&@*q0= zE_}Y{rKP*gga#G`rh{|e8ioSGIGbor^g5#PI^$Drn+!Sr8E!q%BhOnwTdT;~XBirtyb zdi21e&{G~f(HFUTk6fLqkfFruwX`WHu%dONLErTBMDg7-KCVmTn_N_Wk=y9-!|Cdb zYHPY5CK~T7)YKI@$m!rRS35@0u(Pr=X|bjgE87D*t`nY&&rH%qe`=|hnkksdF-J0o zG;n75I9p6fI#Jn>eC)Z4)22zXM-;deBRv`(HL&Sw$Ue<*_xEL%eP7Y`>p)`f(uqpk z;spm9^q3eJN?epB4#uxds%~HqNMMkxNZD-T$Q>slC?Vv%C^ds|jYf(CqkM;prbwaJ zizsg%zsA6SrAHM)1lkH(G$b4aCe6AcbZe51mdfdn>r6W(n1WY-)>51F9=$x_uf4y<2OX%NOTLzF!@*ejzvocTzmlE}rmvsnVP|~IJ-sBpi zdy2F43JZV9mDByL+#d0#?O;|=I_U2a6SjHIqj`?XyZ64+`664M$!U%G%Auk1p6hEuTzr_R-u8~`d*&tbW<@Yd-|EmepK;{OhAz_!_an~4 z?%B}DduBns&mWgJ|DNsQ)VEm3A-%$Z@eVtKw|<3^!fX%G`_JtTR{iqj_G46Bv9R}< z$`bCV6HN-2EII>3ltkJrHp*^GYPZ%{l-u{CSy5~wN4z-Gjl0GfDjZ&2nI(^9S8P3& z)W5l%TfjkCeBs&&$<`^1vXiozBpR;Ix^P>?pKaa#2d!_e{L1AuQDBf`IIny!M#NU- zXp>ISsrv~f&4v$VuXvy}hht~q8^2LZwO8Ef(=P7g$_wmojiEsQ>X`dd2Si<6u+k{f2u>HkubI z@ZVg(rk8cCRk`K?bI^^$6)FFG1D(G-j85SwW3aC-9mE5Qs>1suT< zOoGlcuW4?)Dstei_5FxMoBdm}m5pyJJlL|tiF5sjE1M+t?6FsJQ5O3*Ep1J?)Wr(7 zq>YWeKhh4L7hyM9Fn>v5VZ`Fcq7EjDx(lik?(X}xwDa=sg1+MKwNc4T63c%FJl;2x zU7oqpVym&pRaWH;NABnuho{zuakzFgviPK1@%a@rDSufg@Lz|;ac6pm?U@Il{*;)~ zLD_&Q)B1O8I2Nv(-tD-efq}6>cfw!&e+mo>8dg2*XLz#zmqde0c9Zc5+l0&m8W)<> z8W>q67-mgw3EsitmCoR__Lr<_|MvjV84nw(55pJXi1K z-O<2a(ZIC9>B@#f-J4996&SofA9}#-Y*@p3N~Lvx%liyec;r;0v@4UEbLO?(%aKAqr;xzYC1cv;deR;w2c4?g$zZID{+ z$N*aQ!qC9bv77Nn!{o)g*=#J8H!xhD(ZR&P;Q7y>HTpoytPB=jpO!<(TqY~@4svcb ztX?YIVk0%d=HzER-wZ9SfH`NJ7mBW8PJUwZMM@~GV2A9Sw#$!=?YIwh3v1^cXxp=) zJ99-_`UVl7L#&$)wtf!eD>q~fR$%q4U@5U_aa3R_Tf$tvrrB~vv-yT5%?V7JFB+v@ zFiO8*3}$AZUo2v$(6W4Cv(5yT1rPg9H*pyWG{+RPxL33|i?B2(X?)UPamr{qwW?9Y z;>e2wY#qB=JwG&CB{XZkXymJ4U=(2RePhq`#{NduNyZokZyN^I83!0IH1Jw9#s4(( z{Jl+i!CZ|~99G)4(N8_7MT~Z&ni!5mr<=N5j zSxLqXPjm@2_Ez*PzDgNR9vFcd%4Ys@!r}x}x z%bQ_g|G9hnk+$p=ZdYC#+Xe64{bcjy+ne{^Ft*t(Q}Uo~r$oQ4LCeyUP1+Lt9ugd# zF)XeI%tj83D;-$83Rs*T@Yz|gSf1&#d%|oLuvpTd$yukxC5OG_K&#IMmfAb6q6Zq- zZ!F^cu=l&7N}q)v%L*xPmxd=S4BkEsQ%ZQbPc-n{Xb@g;j`Pmy-wg~L8#|0I`p+!umnA5)}F!iUQpxR1yhp)jqjwJtxq(cl45b5;PGz4iMI+ZPCHtE8ZgN| zZSK>VpT48TM}g(aA7=Z8X7dW>jU1;p#4xKaXfj&BWH@7~!iPp#31)c#Cc~{qSKjFo z&R`V&VJBNKUu*4bI7T19o0NZ=7Sf_k4r1A ziD8sla9)(Bso`~_!U=!hh|NpAWJI$zugvN`9(YViac^a38)l&rgkvkyhUjDVT)ZoS0hq|x{D_jytq)akuBy!-)WT=+Z#;EH=0eW7JG2`8r|5J zGPP@4<$k%+t#Z6A+RYrwFPvT0w0#O>FUn|3vuKSrXp7#_63k)axq-z~A)>VkoHDGdIwDoO)=>?m{e{X}Y z2@5paO<41$A?&kpi{1&o)t?(|Uzi5$VEt6tUJ}5bnbDTIr#09kI(S0M_8BeS9D7Rl zvh19Dd#7oe_lsbk7s1I6Udah<`%K%C8`uuEwiQOS9g=N7D0?U6L3sVu+jd(gG+uSs zW7y@WdL>#!aZL%M>r#JlfhObEUh!YniT`nxOfimeQj~wGx~+hF2Zvg_^4^RW2lqOt zUHg1trs(Qo0UqwXu>!VNIYWEY8Snj`6(DzNddyYvoEKMZC0vvRuJ!*`KQClxtiYI> z%4F=&;(B7u=f`q(94+=7EKWO`bRP&fK4A8k(06fBds#=@>Rl~q72Vt|w@ZCpy$xD} zHvDS|T5#KI#qHg!5g%pFCndBcMX;q=w51Bf9QI8(6dP05%2r?cpkB9ai>LJG6c*Qn z78ipE>kWDmf=%DHcooj{7rf?m(%RTP*U@*uQcvGi(Yy9Y-m%&8W$9`z+l(2C%%PKt zC9a+g4BaTjlkdQOtRwQ*tVzc*)aD$$d9*?(KY_jHbm+z|jl373pJwdiXJ8aCI1-{T z%lU(;@dZX>i)PmqEFKXo62ck*KTN-5H)&f$@&B;UJ;B%N)m~n~UZrt6)#o<%9#-!i ztey`ZZ#^6DFfH19$L-yB?|Xf?@2%11BKCmE>_MtW0;^nm@`NXceA`k#Tu(mnB)Q_w zmX(}tKb{=?C*8G&an4tfCIe?iVUtzsF5LN9!}#mVo%2smiMc!!;%YkL*(_;u*Sy)d ztdcwJK9+W1j3-;7CE=${`pRTm!j%bg(UHg33 z_M{}fm=$e%wy@=fw0bx!e#>|^K03YOQJ`G}OCmF?-2>*ZY^Eh|*V`7D2604Jd9Z)u z-MdI8yokr9I41hHZmWab6Ym@McW-4$E_ku0_r=F6Da>V0nA_M6e+~Dld*c1>y6?5< zl#eMc2NDAHx*}ti9%WWs-{CJ_@KjVHQK;{!tN@erjMP)rt0Z*9&AoSQn-IMwiEXFA zGwbfQGdH`7Z_HUd&--86KeiGDlfrg5E{kKz6`k|{eKNCw{v}N9q zj5**I`@=n`qs6yytMKFH!p_;}ZuPyjQ1~c#>tTe>yMPrJg>Kj9u@xMB-1zUYecf&E z6Y-95FIw9;kL-)yH}yf%f(+)e2Zb-LAA0)2eObnF)iyshHn#;VZXBsu6Sq6|dI{;s zgo!=;8RNK?WykroFW-AT{E(Hn>CRL6&dt#?64O+&+^)1O`PO#l?S;k1^x0A`sJwd^ z8T+hO^U*_lrMXqi=Nn4BZD84A_xwx~i-nbo{g$&YXRzf3 ztZA9?x_H6kN`v;`gtp)fttkP~jraaN4l#N0%e3|2K9<(B3@5%De`BZb74<(*`o?Ep z%Bg1=KIc+6{lq@IwE4fw5me-=3|_iW>9UmI!|7&kr;9aR*>>LewQE#RVzHuynZk}F zpSznKd@T>JlKK}?c}k{O;_98dxoSV#mT&u*CiKXjo4w}Y%?(N>=|Vk=uZ8}474Sl= z-_x?CrHf^Dp&ZMmcg&}B&Rc1if12W{(Bj#GXK$DBMl4}TV6y*ksw{`I?#=|ZRF3H%f}B(@E(3Bv}ZTx zgnkYAp17_|D1QcDv_)TQhve&*1}_Ac`ecgCc`5Q*ZB|6HT!T@IZd_D~cYf7^H$`@A`sVg@MQLaN-~7ij4ZK?2Ect^x zSYHG#4iJzspY=LtMsoEG_WT64=pC!}`Q6?=A-d>la&ScPzpVZ7f2ZB{N(etvc0Ff% zXNn_Rl0#eaf~vy&Hy@WIqQDxIp+P8HZlumeHXXtALbcZ~KOBo%|8*`S*Aol=+B&w98SJGM?A0Fax6D}ceA&L& zeJhl`zJ0Glp>I`o`gQLaEhTGPJREY|Ka_bnyy4m?u+p9Fsr$5Id$y#4y65)a*M|Q) zcPxRY@52&{iG~SXo4!AgjR^VR|64rZ_ecNgE!Xe-+Twpc_27lIs%FyvDr{#QT)9fi zXNOixecL|uR~ZWI689e}rmHnru%DB*%4a^<$j$yv+M@8GoKeb*00pNO5neTy?*=mF zdHnacGT+QoIXOx7X0OXhzC?=>j-y+J)!#|`=3UQA+FZsvdtK(PHMZ4doin!;UYcc^ zeC^B)-L&O1cx_8xO}dgY-Dj>*>D5cB-f{ZAt5i5ID{tzTnYi}Ji9l7ayt_imZ9aK- zb{&0n>!^3n#W=SULWW){5gX>ZWC`8e*nI4qgh|pd2f5A3?7gS7!!GPtQ`tH}R6lIT zi8qH@Cn%Vy-4;kX>^ec#F^+lRp|(?^TAAyXBo;^eFS|F>!1vIVFW1(lpK0={UZ(qR z*0yJ7_d2%Ay^FDEzP7f#dxt@yQ|o<^FrN4ekDN>7OgCMr{Cvx&Q2YMnKW%->ybDwk zH#uL|`(O80^y_hTKJ)W;oH~@Z_gv;N@lu`a6CCPkDiY>=*J-L|RHZ6sgtVWMr)vL{ ztzuDq0ZvKLT>*zWlum58v8cQ1;ZY^!I>jcR-Ui;Zh@`9=2bHCg4Zh8gn36gzGJSUK zH_1t^%fcd4=W;q8&fbtYsV#R$;>_7{d%lXvsQov%Jf-lwi?iCyGnvVLvo7Yc+_k)r zyZnM%QPtRvkC zZZWsswTLfUPMh4yDV}U`qhRw{v&(lJWVGarE~v8*LGh%Wf9(M_HSx(pRLli8%e#6Vgd|`KUp}8 z7#J9I7#J8BBp5>&+xd8Sgn4--dAUS{c%?;nr6oA{_=NcQq=fi|B>9B-1x5IUBt?Wp zq(wz!Bt=Cl(b}()nqhG<%q0U>AQ}TV%r(QTG@MP1U2RRgY)qWZtX*uaylkz# z%uGTojlDF~at+kHWo7kzn9ZXkgWdV^{KO*MG~66ay`0Qk9c{c_to=M~eLYQsylq3> zEdxCb(>yfFBQ(=PEi0nTviuAxQ;c%6c`Pj5^vryf<=tJao!xC+{cPQ>okPu?0*pP= zeO#P;yqrBqs@OsyT_(U{@qPE zT`iSUJ1S>Ss-D)CxpaEvy8eQ#%Zj(H^glY&=lq(0bCdGVugiPBu;9z*fTy?8wmpo> zD(NVzXfLasoLSTvoz>h~+w`QOxw?63Ra19YOY5|**4b0q+q-7WoH}#Dgc(a`b}pYi zb?uD))r-5gEbCgcc>3nWQ@5<1vU$nut!roRT03Lcra3z{&fL4HchdBwi{~v`xn$Ya zRZDkoUcPG{)_&t8B3`|sb^AHP0+`TF(Qm1Fx3bNplY`rzW?_WlD+EW(c@D+2l@+L^??=6GyW zk~5Sz+H-P7;o@UG@fJxuo}W(7(AV%^7SlQD>2$r=xMMAv!D?q_8_d0M=VmLnxW2(o zlb0`By3gqa)|5%U3eK3U7BkJQ_f&}HmDNt0?Y{2(`s(sp-@{$9w?(opEwX5T7Skn? zCbGTYwpMQB@^yE2$L(>LdD*4o{(k4WA0Gsj*?8<~J}CHQY)EwOVUhQ%(P&eW=q&pX zbL?~KLyw6Hwq2S(m-3yus21H9w$kY9X+y*4w$77JPhXk0t?*vSr%C1F_w%c(0$&+t zp5N}gT5SHQD>?q_A|@A4t@^sNWUX&w!QFXN-(1=5cJm*bD|fNR4~=aXi#g@)aG%;; z`tHE`$=AClSk?b!l6=(c@$fKW_i&JA~S5*3+#zcBpj%H@5c2IcomO<(n8 z1geVnS=2SAhUDK2n>~H!x)-dqPCb_Hbw|2#iG zIdA8)U7D?z7k^wgclOfE=2?+-S0+ERI2pg%I^iAHvn3A;&RQKTl!|U*R(-r7JwATN z)Yv@?ZF{aPwcC{>C-CjF&Hob*3|O=RlGdm%KUjFA^?vumUlQMUaQ=K#Xs!}2k>>3D z>+bpXf783=pS4s>7MS=>ZIyOxUJ>(@3duL6Hv}(*Zxd!^w(8CK)#W8LiD9bh*>l!h zGTm=oj!vs`^wIUO?7tQF-D+1&rq372;F)vWTmCKKWV@Wa@avq`iQNh>S`AwiGjeVm zo-jqfZt82M-!&5ZZ*-iV_%?dVbDuBk=2>)EEDiP&*8Z`irqwIt%uTO=eNJ8rdCnx5 zuevMpkyY$=?ZuF%7f<9C;5ct$p)9w{J1O^9Iw1MAfOUnfI?O4o+?NJWS{KRv3e<*37Q_H_3V z2ewVJYnr?lickMB_2B%kmzuw?Jb!GK`}Ti#cxN8hx?p54E6!U{_pD^evkRr`7Ue7~bCi8O;mQu>_mkAtpDl0XtW-WN%qwIKg@AK*Mdm`4?n4a17 zaDjn-LZh7htBa@p1Wu^x)!U}!e0bj9PL*puzGeG6*SRn2o@DhmEwNh4O66Oa?u)fk zuj+KG_)Brx{Ip7lo8)led|B_71!v!qhp0&WIZ|2RYr^JSUY;-g4g}Qr*h)+_h<^;!g8v z`cK>3%p86s{{O73mj7>`H~qGB`p2~JAaB`*f3vQ}B&cpF<5uY?=CC-x$nN35#H_)< z>NcSxw}#p5<@g_QcNp7E6rS?K_mW641%3KSblkRq%`6*XgZ)vWLbGay6w8TaK&cv75vtRKj z{9)j5esM7Hu%zP5gKL;g9zVEgQP8+0)}cAG`2n+Hhr?X%2MpXT2bfhZIErm^cvmFd z$RfLdfpyo}Ty_x!md+PPt-pL=)ti>CopnVvu=BZ`ersa5@Y?h1t&YjdNla!v6Y_A2 z8q-||wYI zMO}&i^k<~UNjpDL65n;fRrA|~X89)vSQRHU^2B^#l&ooBmM=JPQT9NiwfBOTccwWo zvK?Sx<4Is*Uf{rJX3(URqsW_e!$sFjkZaTGgFMj&PV5E(Oxy(v7-Sm0o85Elh?1A| z-a9krR-EhP=&zrqS31TSXwH&Xvbn{6{OIE)kEgI7J8*IxZ}~%cdA&k|oLjT>&lc@G zuYI(x%@$yG+!ZgVhCZ-w#AMW>W+eyPv(4|mEDvj7eD{Wd?T#gr(28BmZ)9p&UFsQvmLFj2`L}`9slw^a>>Dk* zQG#62HWitPoa^&m~>00%;Wj?`|yDytl3NDog7$H zTVgnlJ9Qlts9R!tUszJOT=3eGAfgWZ@idna4=%aM5XV`#NHYE zZld`z$lcEez`rp+^v-7L7vD2*7OCOX%cl0m$S_k zD0>>ty;XoKeL-1)czK&becOi=55tTtTjb&bBKbcH=RGk~a0wJO^(l4H`Dd^ohUH>~ z>fwfxO9l5n+loyROetu*wxzM`LF1>#l}|$r{xhrieH0CNAnbOSHQRjPdven4wv-_DxI?a8F<2ZLTR#((S!!h?Go}+75H8>uq66Qo(%1aoG7}|qVIB}af*2T zqC&%`4qlIhzf5Izn82);*}PMIvY&Zrg&V`Ri0-X7CWob$?m5w2q1Li5qx39CFVBUZ zjW0^BJ+C>Z-XhWQbINLX5nKTAiZL zMpl;66sOJ;?*ED>>D+A2G)iV?eWCD9!EST#Zcn|;JyR~78lUI)|7pJiT!-X9TyDJx$e`5%eoGs+Q=;pwYv4d@40N+Oe z{`Ur4DHB*8o8?6yt*T5K?@#@VuUix%UysM%|lR? zT-mi*l!0ZRd(SGFKkuHK6Sy@T7BnT%TeF-rx(`_~*gvQ~FrD|F0^hv{d@mFBUZ34^`2$M=2g8n?ty?qF`8P0d zGt|D_&XT&FE#)Woqb?4a=>2vRw|OpLPFcXV{Q&2`M;Y8%4@&o3;C?K?woObfZHkP$ zYM5u@@}-G3Rl73ycvdt>WmYWi=zgeHQ(`nhdgm=>j;T9X}LkAR2 zco#66F|ZgqFr5#RSf0Jh%qG`Cy<}5&Nu&g$_=4QBWpe^f&DM(AbD3-L*%!NmZI15o zn7xhTXvFndmu1fG*wGrFu;wnqR_9f`j;D?$&f&d$frZh5VV7s=ibzJ@4K>%Mm#|Kl zmmocFs{_x+2i$HKSW8|DJ3MgkG?0!yzBtYn>gwT;>6!!Etsn-0cIy>)Es)$ctO$M#&&S)H(Bk6F}gr`M}b&$)73ddt37 zJ$w_GcP8*2c+MNO=cu3coO^S48$manALbS4z2VNlVlX-RFT}{RE z$nt?PFoE5aVJ>q3qpkt7WI;N20F!(Iqlg1j=Y!sd>D>Dta6jJW8et$77ht`_hi7fW z+=T{tt#9~BUZ+&t)(TTp-E^t_ePft{SW;h@`QfGprl3r&n@+q&7Mjj%yuS{vZ8)VO z!*lQg*Y*oskKHdVdWn$1waRIXo&g-Y8h9^A-*xQbo#eo{_!sL$ z1KvvvtQHC^d=AVt0vr_#j64cc1qGNC16UR^Z2KL}{jxnF@dNW5OBtU_tlkq?-yOLA zZ{~#i(E?nb*x9_Bw#PW?7v0qQax<~?!FmO@B$2ln4P6)84l=0=sP`W9(Gu%y5?Lw7 z^GNLu*V+S|i!MC7|B_35-Kp&x4jJg?uJ_H=j61v_vU;aFv*8C3V*w_U1oqYh4wHi0 zSJW3V2XsfgZVe0HV}5J3bN3S?tu>lbS1)?pQ@_9_d0JG18x0fFm1WLiz!SS7_%6* zz2{_5uUz4B}7$&<-Eh9+KhVQajpu@@57vgLl zza)Cyr^Fc@R}!pe_Ru@R9^-jQ-G^tt!|&}MmL?0ZJ>AB+e9kGIw~Wdca;|>6b^hB% zx!WvF0bGkOuqYNV>izq`+`52wsR5g@0LQHc-YXl}#RM1*%P`t|Zat*OD1Tvz0kH0uCe&QvGf7YL;;S<0A}L`M$rI92El~~StXtRIVgpm zJl)D|%;bBSjc<;5=S`No63)F*yi7c7JQ@lMm=%0yTjge@di#ng$1JG$_-IM@B-NM{ zwa7;=BUVqI)g$rqO18`bJALACQ`Ld_ByPbdDj*1tzD!aqa+_R}Z-(flV#599LEuDhh?r+4`8>gJ; zQQWM0dZzCYuI;CUoQ`vdn-o0w$I#@=E-t%tx5TBijjSC~_HvDv**Kcza_(*D?0nGB z)Xc$MWWD0U!^5oXJTe{=3>=+W_)XKEtZ-cH>MmxQU^3yt0VV+jK0Bic2OR$V_$dGB zi4(h?^tQ~2ELT@HE_7mP6xFF&VEEXnRkCc^(#ag#x;9vNEr+}~)|s?;>L@l?Owy6>=tyLjh)7`6vwg9NTW{C0E6zRJzg;roxBK?! zF^^M&X3vJ?M~%l}a|vWD~NVv0`ECU#=Ag7}=#5 z6uB}td0vxFNxYmU;;>B9wds+B$N~oC0F#r`!m=VedsG&z*vPH-&ey3^=ZeLe<+?T! zjRFgWJe|~}cYD0jke%t}r6rsA=~1`-s}F&z*O{0Ytx@cgd88wkQn5)_d*hT%Ub^e= zWS`o{AS^6#ny35unTonhogTwu1;U$5oGK+}_)KlXKRDcPyQCwDzeO(5seyy-$AflTuY+kf^EPR=G)OIT zTs}o6C&OxPjNOEU22QmE2gZ*)4Gc}fj}EZSDcQpK;daT4(ruj5PA3>X^DOFMamZ8s zbS%1bi{jpinag%6Fnv@}NNC`@5Rfd)nY++{m*@S_xtd0sZakWnocGd&*L2&71<~nq7OJ$erbCb${v*>u|{f3o%8f|Y@{oZKtEs;YlvEmcU_B&nKDOxM99PXR^ z@y2UkQyaxMqKR>wYI5aY*0**W9Qo07_JIl;$KOYF2R68h{rj?{d12`?fh!Z*^uA4K zcT!j_;v>MwJAwCv`)i)#1ox1~!un?kt05DXW8>RtLR}b{KOBRD??=l_c@bv}hL6 z5MXlJ(jj0Lc<)e`lI=@bSFOMWUPqMLroN5s@ko&5E5aa`k=;VH&0og?q-`#O81@)Bnz?0CfC*xksT_Ah~1{mCNfB?(P( zK8h2|GnDzg5}Jfq9Ib+%JYefPrsU?o{t#a=5&Jt>Kwk0~gPu z;>8WTrd)}Q*EsVEgbhNO4JNezZ}F4bnJCR)Z@amN8CZ^h*(FmdYyRNn|-J7NyY0mm+XxuPLKYhDwN)bk{!)hdo?^ zf-~8Ks~kALIxJ!hoY7`cvcf54<8y}x96D_81O#&(oVbFnv~jQ)G<(#%o8QQKfML>N znWqK{47?RgDr!!GFmYF41A-)3l3WqxR4->gHet0PpoZg)R9YBWv7 zSFdw&j$q_TkG7?ONt`?t8<^|vy0jUuNHqA~eAMlJX}9s7lfGsQxWm%rbw6gV~!BZ)i?g zv~`KVLEdSceT8WaM{aSj@Jb0z3_9V?b?yb5#icMtUXQ{eGXV!?CWd)RkBzq|2E5(w zVeXdyuS0o($jNO}7y3+8U(GIH&Blt3WfYO+!SNych$Z(htqV* zK@MdO{@3Rix^e}4+_X~o~nHz`b-`+l^_`e$vs<>$b} z@}c37sypMNtq$pxvH=AQLHM)8IN%Gn|ftUV1EeP$%` zGMgWs;hHa`yXN0P9=`VujCM)u411EKCw&k&WOi?l{dqa9i7M}<*DwjcQH{Fjz;S=l zB_ZZ*QyOYNAHDQz)v2Y1Q;PHx7MwZVa7W*oYqefix4w$p4a%nk(|`dm$Kni>>67#eLVWYSkO@SbSg z;M{JrqJgJ^dFSsrS=`%$0~=Ta`rVm7eEn%bTlw3tFu% zvb@R2LIZ0MpLbY>o;HQa6~F|5VD`$gC)A&Kc0~ z#G$WWgMnQ^iEmCv{UW9O7i?w4Mu!$J(mu_|#393Wg4gE6hMW!-`2&1~$_$Jh2hVyp zzwA=ZoO8_RMw8$I^%KFioEi)^7A#o@T5YlqNf~zC+C1^2l!;~y=Uc{AUzM0tBZbs% zGV1CuYC7)IT)E2NioT&lv&{j?AO8+=+g?yjOjw=3u^~)BIN<|JcFJVk7wb6{Jt96x zDr@lWyd*4@vEWp;l{807jzfE(1fw8BqfCHq!Hy0?hBY=b8uwq){VUv(^PF3>l|njxUh+< z!)tAh(BbV$xiYUlbNwiP0Fk8V4yxHU=hCU?Ub_qsR7 zpKWeF%hUW~_x!b4tcQQJ2E1@=4`KF|==qmqvCCAUQM!Xs?t!y7N2AT0voA?S~UV2Tp4=oQyfa$e6%%N{cl%V@~2?2DJh< zf!!T}2ii&+T9_fX*9uIkn{4O|m~7+4rqOk+#n z(|EC_L_qd#S5xh0j!2ybJ_SbQ2eLX97D611?1f4k0sa#N)*3Xh#7MN-8B3^5kURJ$ z;Qb;l$Jc^yH%cfmHZ7=f-=v|XBr$*MkA`KxmaK@{7qUug_U@U_n48Z&Vg9Mf^PAJu z*pjCru-Wv3{(a7^(FLvbiA)k4%X&%}B~Ki-E&0c2H=`|Y2AeEHh;vS1>0pYLhu}Nz&lh44or^73{J9PBilbFlAY^XF4=;uV}P6 z%gPj?$*_b)rsEVBM@!HNwk(UenH(&P42`xGEsUXF;u~1>7*01PG<#LFSf60zQQ$Kx zTI9@nTEL<=WoY^z*(ysz{4 zmVlkhv}Q`ms!FZBrthITYw=F=o8oIvS^idRi`c|twD53vLu>eoRvn4PKPOJ8)n2ll za!GzdOUKzp0f9#T1ST&I_MiofTmsBF2ij~FFo+h2r0;0(aSkNTN!C3r}qh?kVnhfc+=Y3i`pLM-pc(G7p&PBq0{gtgMq`~{>t12wG}O~H&`oA zHZ5*o()ci;`r!oo-4jfwcvyVxc%9$RZ;Z!uTx1cShj2C)OpK^iUm9*K?< zT4WBKmJ(=DpJ814i&5Oc@kDYXf5e0Xou{%-ZjU#7v}~_~h>Sxvhtm1Ve;W23Oqn~y z`fnJqF>(rAV6OA8Za6u6~yCDLNAGBz*Bp)sr5>oqdUumQ*or z^RWBzFk{AS%V4FYGNQY7`I^p>4qGJo)c3{3Jk!*O0KFYSOfMRqvUzp)Y&pxe$VTiz zs$B{r&w_sWW&BS%8XnF)`n00q@s4TEv5O9@OuHCmd(d#|LB;J)5*jX?Y7k^NaC=dQ zbD>P`30B7+Okx(1pA4om&hV5v!R+Obp5DQno_9l|psgfh*#*IN_0M9ac}+5YGU~UC z*f%h+b9xtZ&au>HG;d>I^9Yr-Xi4^z)8L5a%We8u%A^)@$X+^n)7<+%?+WNU2=O`! zDKW_ZdpP^}gu-s^nSp(LFLeZ&XKQEr?Tgl9Xu2f9xImNHkAtoGh~9~pjEwcPR|h1s z41e@96t*{LEoKm8FUjDysbCV{(eUO&M@Z=eh6)C`2F7`|eP*W`q#Il=6|%padQaWE z{amlj)VGOybQ<_I7_KJDKFMfcZ(!iC&noV8^!(uG;?eY(>-Me77ALk%TrF`;(rSphdx)d74GI zYQ`=}iCK@1{qyK42@(j15vvFie~~G0ph12b!-1Wjm?~`eA_|^pF#dXA{g7!|aA%f& zb!hk=^`{0#j|EQcx_3`=Hb=Qfzk-s6#|~!sj7E-v31OuZ9$7GQ)+V#`Z~H#S9z*MqC-PVghR;MciKN=5V;5`zzfV8<6pA;q=&}6CJ-9 zADqIap|D=hVtpd3mWtpi55MT|UffHo8n+)!@maizHSVpE%u9AYW`!M1U29Ir+B4m= zWKy=cF1VvnCLr9dbL+y)T^|p>?8#|d_%l%}oue&*@mWFxzed{Er~c+;$=1`JEp-t+ ztT^XG#;J==#Sfh>AN!Z6dwo*JC%J=p4()jx*u?)ev9q!zTeO90FnsKYO57qQCD0Q0 zqqV}IZRSI^3lH0NRw5u*7g^+Ru(x zcwu~SPEK)7lOuN^Z)NA~|F&ii`rNF!E04So-L6%0MgQuui*f(I$}HWf?@^^EFzZqQ zUu2d*V^7BD(y%!xN>JI=1n zP$`r0gfY_f;NuPk)(gF+3t0c9EMVaNV9sZ8t0_NLKfOW7qe0N1#pOY(K>(BN1wq9X z6J-p}$V`w^`*6WMDn+23>3t(pWO$%xtkA>cOX3|S)jYofPwBJ$E1g|DUq9mQl6qm; z3BS%){Ceok&9nVF40!bRi2!x=hJvQicW1_zQG znJn02CImcUyvWzW@lfirs;7vC|AIdp{}wa9?pI4X-uG3gwL4PZX~BwCrIzjqI|@4) z7>$mw$H^Hc9cVb%)GB9N!SYyAhpk`2Hi@tKU~>9Lal?uGAI$v2dPZE|s-Qsa(9KqE zA)|~P2Oc)D@yZ%HTo#C(<aJ~YcaZo>3fna#a} zi^brAuF1v-(FezMM9@GGUTCpK*=j$(4`3GzD3{ z+H2e^Bw~`WW}~Xr1QAvl3E?Rl#VtM@U=>Jt@PJKLWws+5OA7;s-i|qmN5s5zE*=$p zBK43(#H?c~OJvg0qpTuug> zO-(F|logrm=`w$ItJ|SWy%#IhkL$fjopm7n9qaOWviol`ap=}FFfEo7iE0x|n-S&qJv8mDPm?izbNa%~(FJonWkt=KYYm)aP*4uo{;yMzE;Z{8} zWMho}Je|(d{nJ&}sitk#ysbMcrLvaHjfu)!{cuCtDcRd*%$Z6X4*bjDXvqs~Q=5IG zi2K`(>P4-BW$ve$*aTe;FtQ7mET6p0UHR@yb{>-hTP|OD8o+Ujt#h+h^`nBta@E=A z&AAQtZ0v29SR#G9V5<44#GVR?=G;3oZcD{2oHX^qA<;;SF!m6oZr*c;BxV-~aB4m8 z;*@$Ux^BS%HnRzhjxJ5&T9$5NF(0{-V>l$mOqzu5nNHI%IjECyl56G_A(x$3mc~0U zNbm<87uOVhW8imW(jl?KOOH3KG@7*Wu(%F0o6%DTc9jE+oIUN$R}QvSq^}bGm?6C9 z@|89=Wrlg&ZU>A6I}R|rdo-(ABz7mTEEcWhVf&UN*k$y>AzJ)^18++JvqHr`MHb-+ z3`{-W7+4J&CViO9@JGTxb5X3j6<5G4hKLI+jy+9MT#@}2d!F^SG&D;mFe`YmarBie zP*!68;^vjX(Gyp5hPg^t4sZsWEN=^WCVH3oSb|PImYe2~GK>#HCG2?=CIUd34_js ze3M?Prx`qMRe5p1H(Tqh#Ox3w>zir5xr=xuJRN5B-+b0vbfuTWB%#G5;=NdHi>r15 zW03u}M`GIynim>1Fj-Gw;%Qsp!pC-@;hn~Uj2o3Ie&(jK6I~We7XGp1l3kLpU=QOz zpeFSZO%gz@=6{{-`~K&+snuk5RkwtHPKZk=3!UFg+PBup!E<_2n7j=29xPZObGVVmMQpZIO`7TIk27+s+As8-WiwSi&^sr> zph;=ULE-HjnjV%5qwjv(D7tU!QMEn;Cc&D6QdJf#W(g?=>4`v!>rXeL;dp^zs=;=U|(OzE?~{L}zB1 zvbjm!$z!)l8lVEMT(u!KCEX$eX){U8dl-ywNtnzTYdeLfQYghUwpE z{8h#jd2E#cyVZlqT)dZ=MXFZc&uC;#+j(`h#bcpolr|ksU^fc9VsdWX2lCL;>IG;CJ|S)GXbqeGa7|&7;o3T@_XX8#^)xGq@hU=}KMklVlNj<>8qWJvStpn8>nMQ7J zO$*nsx%D07aS(2@*!h6X`^G;Od2g;0iGLP}Wu9OX@7d6zlk;fXzNao?T@4*J84E<_ zJ2*0Rxp{htt^GK)W={!|!sVH9>>1Ox?_pM&@Zn7T1Ey1)jT};Xota0T*B`kd?-G)k z;ZhuPiN$Z3k@ktA`|>{PO}1W+2geFD8YS3q0oaHQFK2eaaYM!pzVp_U119L@Jx zuL}vpDc3l0r8q3<=x0%xqh{!Es5YVW-=Cf>Aq~ti4P0Rd*b@%0Z(!iqz`%Rv0Q(D0 zjwhVG7L9y;2PJrz%nYWwaWFR)oUPgsbiIVvz(Ah;hj`=Dsc{z%NvXJUyzyNc$iVjH zY}ConxJUuTD^8124@$>K2&XijGL0Aea3W%dl)%wh(H(v3@zWiA4hIMxO^DrC=_s*)RY2wn$Bs_E9tLHlDSUT4|FQaKIXwE<5<7p1=jl6a z91@Io8bb^?nyrsG+e~4WOmGxpa1<(VoQiIOvUFC5_i=a9)1A=4L! z<|Q_p&zh&b_td|P!~a%Hmok{Y#IT`Rq(OVi)$l@r)2_2xYogd97{rc+p1m5w9vGpd zlE|LgD3~yVZ_-11HkAvkvM*9xFTU-L{&89`bN=K94eLzj@=Q6vu5#w^lglEGXL?M} zTvdz~Yi$%ah|Ut3BKTl}_|ne*k|O^;o;$vTP3EcF9G-JR1x|NZoi(}^=}%}7QaPwu z#3-(DP%z_=?j9%Yk|r^ogF-hPL|6_<7&wZ#GRnC)%7v&Zym3_Ea8h(gRt!*9>0#2G z;H1~{)MSdPX$rGxPq=x><9Vsg=I?Hrym5*Na@OA=P^peLMSLPpu)j6NzpL%pwx{(_6Op1lHGz0P2&HOpC!x?ny}C)luuwn zYrE=`2@+u)6KC+N%)P|Pb8v$L=bwjiSt`H&CeBcpeYG(q?S`Xb*)-32j0N)=g%b{X zpPK*YZD)?mm5)E}&pz`cZ$|_Bl!J1TPYq<6HI5yYoY3h1ib4EEL#wBw)SW}x+nm%S z7!@3x!uyz1o;YdFXwsY0tM`J*G)H;<#-~xsp4Oa8*Yj!C>vRxsJ{--llIJM-8R7+4R) ze6DO*6wqKeHzQTcQR#swFQdx2r!tOa^98@0`0P5rZSLhvFnYJ7V~tIm5>%|Op%C*m920rFG?4i)2QC!q_KigvgZK5K!4U=&RHd5vu@m%-g)5G zMh32i0~}Kv4H`rY?lf7;G+PTei~U&QKk48ui$;+gM~xo!=}&LgL@}EcJT?n*)|}F$ z*K=r*lamUMhSHmZVZRy`KQtyVsEX&vikmdf>p5Wax9x{ZLzzlLsS2w>!DJ~9DV942 zBovyAr?{&)I0h|P-0U-9$=kE{mdr6KZD5!pqq`)4t>Vu85}EF@13Z$=|D+@wm2NZ{ zg!)NX9ZWZhf4btbVb+Za8(5XkMmeY57dWu+Qezm8$zAR#4BUSn%0Ie%<)^S%E$1fJ z%-^YkNh*xIYoau+Fzpj^GTg}^P~kIsg-X^_2EGSv$90}(o^asVa)9T+gV}wvOFVk&8@p}E#&0=g9po3Al#z11P|K~wV51yL52gs(3wkN%Sq z-{6?VwQ}vlR+o1SJSPrh-+OpD_la$y*t-{M9$`LdDHFwCB#NsZl)S=Zz~HQ#(IjZl zvHus}$0rOK96Ee0FEj7lzx9&sbk6~?j`aMW6Q9UA^6i}EKWmLp#u@{L=BcIr95D`@ zAqQ9+j`%!1QZ_g8&yzJz5-^FBV+eHfYD(XE_nl0XtVpP@27jy9OFD8Q{(I-k7I11eU7P6)PVc>mo zKyi_q;lHfB+%pFbB{@8N6F8?&x z^Bp%fkw)KRn(7Cb)TfAi_h{^W@iJX(fx(xHnXa zE+!YDvW!#31_%D_D{L=4Gezy8&Z*=BPQ(iIyIfM>O8kk zf%D!xzKqKGU27fwzxpV5y;0QCCq+R|jQ10xdY-t;LGcBx2PQcPU18FHkgYqVv8#?D z{mgy-5=OZnjtV(5!ki8%UvZQRNmhKqz;WdOhd?BEOaprclVL}Q=I?8|IZQ$i0{8?) zBy$?2I1U@$nYO3Y;hdK{lZ=C}=K-dW@KW7|(ziycvb_e`?Uy=Q!oTNvJjt0^>JXUL zc=yDGlux|(Mbv&y@MqNc=cs$9iMf0E?H^6nADBdY9Ja~uan3lX|K*V0gC@NXO!_TN zx(v?RA#2%otl>EFLU)bhxtvD6x<;V_#-|~DLOb*`t0JNqG{4={cU<0gf10+xP3D^g z@?x%AQYF^q6&-Y~b`-a8G@9l)yQ9(ldXrR+qp!3G-xCMECkK_Mq^i6*czCL+0*6y% zl(S-k17F7ho}$_8I*e1kZqYt)NOuLJ*UwOYsjb>U3hGN5*?kgc&OHCfkoWR}o9Z8R z$^(wiTFYNjI;reRNSVmlz!C=iKZnFh8U(*M-Mz`1sOEZ(OMS{sP9GHVPJ3Ppsnn zYUeKB-pV)ez{iVMSq})j+_7qg>%48fz4GtA1lvD6r=Y&>5nrXa$WD2A&-956Qd8DT z2{0)XIA)zm;Ja}^_($VnF(*wECY6+CO_4+1*3AkQ&U$AYd@nQPZE0ZdD4MhDkY;np zn`=#yIgNAj8Qoc#4ZmDe%V}g6IZ$!_lMy%X!G}5r6EBsNxF4L^KU-I>R5xyaWfx1! z0gaAk#S{jG3GQYAj2;mUEEPxCCoqV=I4IZQctYxs{(~l_{sy6*0}6AR%mkQ}UNmWb zXwp2htKy^6&NZjS0~$p#ob`JSvSl=ItvK+Ht;a#B$4Q^3DJNQD&ylkJFH0g4bo!^I zrB|0tx^Z+nThI5s4wnz6asDxstFD&)Q@Gc2 za7b}ZqtFC4z8em_0Z&ahn1r7klz7u z_^pS958c`*JI!H7y|rAA^-lh}oy|{N4x8nazuME2!uLjAwCAAWkzJY$hj>l?&3)o9 z^?S9D!9j^9j3)EW9^^ix9MGJ9e4=A(v7h~(X|f7I&TBkl`wcQ`v~v!vOm!&sX`dPB zb?UvH(d?S#DWzfg$It!ya57ph*ulxl!AWpNlex@QjgAJ^3u{<;95`M$C{3_1bzsu_ z!qjx!dd>So>PsB;c@8UtG-@tsGMjTqeGb#=^u5+CkJWt`SMxXB_;1Bl!KCr#kb;Ke zE$`JQ)fgQ-3YC^}3afd}Qppiw^%7BxTfot2@v&1t(OHj+(dge1cK&`thkXef7rC(5 zl)ky)82Nn346FEreSbb(;`W&lyU*p}r&jOtdbYD)6fAT;&LNw~AbIO>t6Q*)UcrM6 z%Do};9GgCgoVt8srb2ozi!ZYgvwN?MOJ>i9fFsw|hHbcas7cjF!Z4}iCEMdICwnIH zigB1o7$hEI;1rTDkT~Gb^zc}hs!zZg1;eLDri-h)t+`Q<#5X~fKSN;$Bb)OBJIk9( zd!=_|da^_;c(C9=JChg3+h5C$3v7$sA@udpv9oj6YcAiC`Kjg3UCk%U{60E0=USNi zhxlYlr=&J>@!n^jdC4`SYm%nl1#QRoEuF$G>@!tbjN~6VcYfCm+hW0T;vc`fY-I;y zo4AYB!WN;RP0IcHp8{6)O;?z6Qf)@ruSIHldpaI<>(#yLYwWBPUL^8QUk+uucnaG2ApY%q;M5 z2Va`>f(C8==!C{&0>(O<8AT#|e2(gQ=QN!?b>xi!r_6((CjL`03e1j|B%ku+ZjgK; zP{6hEK%;=lhs&)lWe<;TkE#0EJ0Y&_=TY%oJIhw-kO|JLQ{8<(9ua?8^^$ex+fBcp zSZjxT=o1JivrOUq9`-emv++VVzoLRjmIsI4v)%7`|LAZ%S|s20LBoMDwc?=zhseJ< zC;Mi^1ikD|%{_F%iM#aEBJQ@5ETztQi@LH_EG(Gd*jA7{fpOKQMGZ@)1bv!uXl6Nk zAS0Kf+`+|d@;N6Ixf~QuDt846C>n`G_qA}ZQ=eb5RMhXv4JQ^JtsR>h4ygnw_#U54Mu&SopJ8MxSdgOaX%?BvF2kTA9n5g>*c9Hp3#vj=IvW-+*ykFk2pXLe z;1`Uk;LHmyZ{y6D@ISTUQHPd|_RV%rVRz6x3wV-a2 zb^G#v8vYw^uW0Ca)vlIfkj*2Vw;-v@@zV~*7Rgyz3)_}uZBXnFWSIC!*n{n(s)%1+ z;cC%V!8#)^Ws?&M%uH+M{P5A`Pwp|)pPK&QvGL_IJiksK`tyKUJYz>fy11J`B?BK% z_W?$`O_HfxF4Gn$wCET8a7brYT5zCAuty=0$B99bStRbnjYmBu9EDw4HAxc7Pq=6p zJ><>{IV-&AgR9^Q114>sM6NIcS2Mehro34SybBDDYD`OzQ{SS+$= zcqhNejRe6xf4h|B8yR^b64nSg98gV~aY4N1$6<{tDN&{$nSA#&DDt=}a+Thg>ejGf zt%?dGhZoB~CZ9zgTFe~|vY81?G(6&n zy3wMW^HIenuSq{rfsvIfpppB+GcTD52bi~Q*?2r=nW2jEHe1L8X2lH-94ucNPZVTU_WInL zsU~UUHObRArRae2ii^EY1`9>5EMQYH5@?)kdVs0)0Gp|s1LM}V1cr?|4jfz`mK(fT zB);!OtCUzMcU+F6uIq*-{ZkEHOj*q{4m-4}b$FgJahc2&D#0Qez}U}|cY!OpV~XZ6 z2ET$`?woHMLphJ%WZAKFvR!$$x?9_W`F&Rs*qIo@xG%r%~Bu#&h2IgayouhE7{~H5`neuqv;r?38@uB)o3{ zyJvv1^vwfpW-AKQZJ$?4)y``(UGdM5%Z}raK(oUU^%BN89xach+c+Gfx{8_|Etmx- zNF3$;bV1)yVUfV%2~9#ihP|4R4<)YX%caf6tiIXiFeJ|$}3x}O>n&+fZlGvrG<9O14-l55?L9xYM zVd|e3_TRlVb7F6Sq=Vwghnz|QOdNuGPWdNJZqZ3OGUr80XW!>0r-Gz*cOPZ(BWGWP zvTbB{STko~kXq!$At#H z15t_7L=`yCmpYyMbA&PC)Tgyq?G9@>1ah0MapY}U&uY++DDKyiB(Z&k(`CIz7L`4> zkImFzvoToz_=MeIuFw_VMSLD|*XmcER7>HXB7VrF=^58Urw+xe-^?xlSypUivAQrr z*(Lh!c^Burb=L$6zkY1fX>qJuaO7FZ0T$I42iVeEwg^p0Xjc27=-z&zP0waxZCs7A z%)R6lc&%s)1bg+mchuCwV>&r+>!WjWO4C4Adr*n@Oa{(b1TgjGpa0LGFrg&sX>a(QQF6m(`%uCLxLH{0Y)ta_98{; zT?y${40ZRk*8Ou?*U=VOw@}RCq4*Vt#J-0TU5))YJ|azyB36u?dWtjK6h$o_etN_x z*Y;2-z)^Z$VwP1umzsvivWFt)8k#)LwL^*g z{n_GE50)v;Q{Z}&!0+YAd168Rwgm1|4&0{}@SIWLJ@bHPn*-;u1tKbmJYN_D&L!}i zd%z>aDDZULFiXUqbk$cD_)Zn|LZjVq;dv8ge$#z$&e@hh@6;`2uAuy{Q7`gpj)s3b)>iO> zH7J2c{t54#7u-5}=7q-UuM)PmdC0e~R}@t`=ccH9)OVo06U9I1J{fMfwIJZB1#Jw%g(0U zSi<;6L7YKR=nF%c!UGn8#f-axCLYrhInJVU`v{{Cug>uzhCL2KDvkUp$^0sg0vQWZ zBH9>q8rYN;O8-z`jyP5oV8G(kz-+OA`B1|Vme2Zv=S3SF^`^DxbvS-fSufo5kp1XI zne&2jU5duR4~1@B{CuylN8^NzNjF!0%batWUu6!pE>PgmVw%^)5YeO%QIWvT=Ax2! zo=4Ti`?Z4eD+aEr&8<9rTcjq1EmGh)w!mV$f=FJYUXi;*9-~AHqeR|8o<0SBpM#Q2 z5910JbbUJ@z~IRLg@OM^1Aoy0)_tFu9TYV56<7=uSp6QbWHoReYX}ld6xr}^p@NSxbVDT;WRtw(bOnmH7z0U zp~sd799~6IZVNe!6gn#o3w&6>c}mH(!zr+NQ@HU1W|;@f0SlPY6!^Uo%Z@BQKShD@ zAJgJ}5yuz}4shEwu>D}*XM4#cO)%WfmD#4Fwuy8 zFezri3Y{aYbqwrxj+h_z?R5;MnNKB zibP$aw56g%PGZXA1*;VlHF*vSsWgi4Eac2+_@k!IQlLrV&iVHakZF?vi@oCMLgBMdC>)rKo@!rDh!O+V3 zoohv@C`WNQ$7F+!mMB5(R$r-oL=CXfxg9 zc&4qJcf9f+*bIq*wZ2o^c=FgPyZS$Oe>gAjuvkDH^= zp#b2YHJO<`WgP_SNsJ4%B4+Jo84L)$VF*~<+&tQW?V?})i9C~EMq-SA=2O;OP! z%3SRtCHC9n5>rb!wdSfMDSdi2zg>ajU-G#W~U%AGbH|hF^1DyZz5?JOe5GhdCWPkW$Z$K%S`Y>r&d19R|T1}B@1dj4$;Vdt5vlHh6wHI(uqs+rBPqV_FiCV%#32a5n9`x#A(-n*u$Jx6509 zKj1xdcBcZ*qGHZV2gDi}y{r!MSTQEPvEiP=An?bZM=X{tPl!e7Z}h(4b!UGIEo@>H z`^(IvFk#XoRRKjI0mVf^jUpwAf+1ah92VsN`zglaV)DFJk0nikCF{X;eZy+^2UQLS zdM54TJms;pq=EmG2Y**no|^)j+{P7d4m=OK1qu|WznR3Ol;FZV+hya=qvF`pD`7Iy}I<~&jn zzw}19meMxnDvmuH*c={tm3ExH;ge^uof=evwRn1({_)RZQgl(!FpylDq44{921^hZ z$CY4qk*`c5rY}9YM4TS-lw^uu>SFTxpvTzc%)uwSuHb<~Qs|mi?%1PFX*`lgl8hkUQA9^#&B5l@yh*ErLB}?3Aw8|=YadxX z_|8ssJTc*;qKSaG5SKtfz=1;&o*g9@7BD)o^WCs2>DW2($(fFSg2~%ncw9c(JXu#{ zipom^r;{A~r(8QOJalvs6i{-zv!YI!MO1co=>&<7vz$73#GDG$ojayW#qF+|xFF$R zV+A`Cr_z&xR8gZg9`&Ln8G>$7lEG&T6@@8qwILxm+RWsEEnK zEy3-ru*jv|Kg3%+n?!f~lcD)(P zCHX9#JZP1R%bCW^w`uK9M(!*ofd=N~ORupgb#^?6%rI*Bz{p|JvvAHoVeg6!EkbS@ z8(Z1k3O3GJqUeS+af9qE31(UVORwKqr^~G zCMkHQ_17gV0g7B-U4&Jnp1O8t8}u_kls_VKX(*RRZEHwm+?3R?iN1e69O2J3 zm3+!y^f%!;f9b2O?Ix`O8(lQE-#NsozU{`PLXDk$hO8==8H!eLo1FNT#^+fP*e)Bl zV`Ve9<&FoEi{y_yIKW^h?BKv*>0rdgp}aw$iG}aXFYQ%U8ij3w%1aKm3Z~>Z{yQX= zKB2&ERn%%O4#^ZACoQFwLW_=zH83O{k!W~vp;aQLU?aPDN6?4HO2r8d92yI+Y&vZg zUE7{?hNHo}WUD&S8H(Nvxk}V|4Xezq?jXO?R&1e}f}j%#3ad zOM8l#uP9Yda+P$K*~lW*@WIGKWWp3B4ql_42Tr*<)rsx0rFZ7+OKp{Ub@aSY#Dqq^ z4iBLw?g>-H7#lAr@U&YTmf+ga>N4kH7`KVbDu#pXA)SpZ$pX!SAxCBPG?tqwDKM6o zO!(I^${GOKkY3hcSyD>yH##ZVzh zyvpN{f>$Gl^o0Ygx-%NN-?T6*s4#Mrt@1g0!Q7tDLxEjb!r}Y_XJ%&wr`83U4a`o> zhYVggvd1Veb504+`rP9%UHJt=xs$_Ofj36XKC?wRdNi9pE!e;+!Qsdq^Py4dM+2+) z1V-+hhUR~Q6Ap?ms8E#T5N}#4`-EB6;)E#ICxNgHNs5~t9@%YBwN!NSj@qmtAoRO& zZO+fbj^}r{znFJoQt%t5QYj;G|8q-S_3|cmyLBFtmicvb`HB~=K`$BkvjUXwFHK-| zl9(rO>&4vjbLa<}g?=g+-!)mCJ$IMPw1TLc=l^l?mn|huAEY z1o|sxs5M2#DzGff%+AnbXxh4K0h9azdA5X1CxL_otU3&gftd%I#atLzC001{Wld;} zYhV(8Yw7&ZYJ*APzkBTVDiP}S6D~T}A26QhEYo8CeLu6jTqT6oX&J!&KHuCxy@Ln$_qcYn<=B`vB{$tVaGA1*Y7+45spbu3gxm zU?Scv2zx3{%t=i8F$0 zzxvgd>8UOdmff+af-6|$-vxrJf9 z_httcfvZj7bq$B4rxnbT;W-;wpKw)v!a>eP$%le947GY24j5(zFw3SW3hr3Z+9k{w zdE%NtJL|#oqE|9+M%)peuQ*X%FmWMQ=z&AZJg2T@*p}boSZa1MGa+|p)g`NIse-2x zJB!|CwJ$QynAv0Wh_RsdJJS{(@qfAw8aslezAb6C)Jfv0e^;!S)~xoBQK+UMTJDnp zSMZ$$tR@1^O5YZ+J9aw?Evt}DB5j84aqx+TD2BOW=?4CuwTg6$q}vCl*Q#F zvXDbr!Ldx)k;~p^k4S@rW6qw2D}KlP&Hl;h6rSBB_22?S!!yAG2R1!}Hm8n*HH_07 zR<7P@e8J{3vn-q02J1UaB38D{`@^OzGI9vn<|N8|?jVCky1=ZoqCe~_vc6SSP3&M& z=xbx_+30wwuT^sOXT?%=PXTRniAQSX?2}wPO@!1w@+?6SwK7lQGf|a~bMeqWa zwVzWqnX+vT%~#n}@$;M8pJUAFN0<#ZFzW|2F+Ue!-&~o=XgKS+jedqW-*X$cRPiJQ zjs^jyWCO0)cDASjwp)j78v@wv6WAXcbC_=^+94iab|}cNl{sbt%esjgjL#Sp42pRK z7)1*hB_FU%P~h0C$SmxZxOth8WI@8SLfLKNO#jX&nw&53(Pfacn!xzLk#WPtB87wy zx5X^y7BXj+8sAb6Q+!g`xq$6$Ah(f78Y3TJ-;YOI2MV%DTKVpeW7lkxjs+ z0-?{$RVvN==^;8_W8)b(COu$O-oToGB!ILvU!r7g#%ND5R>5qbH-=JLJW+21%2iP91|PV3QjVbr~8~-$Z{%><&>f3 zsf(=ACDPkAws!_-v9)NmE$~fRz!GD?rg>aT`2x$AP-(Z#Mp{#HlqYa-9f{a;At_ay z@uHcwiUU(-dws-)GX8@CN}k;75(HOW;GC{eUL4MIbOM*wznPQtDpSG=I9EL2S+3Ca z{+h7gv^gs*qF2oj*;E`IrpDsz5a^sB`tO+j))U>HFPl70PSO54Md&iK&e1xH11x3< z%$@?Ql?5E02`o_o+(#7VA6YPUTLJg%164i()0!?YpKxQkyF=~x6HkRBAr1zud=D5V zF*0yoVBnj^XcE9NiNSvBGatnTEKLh!ItTlpRr&#=oJL5}`iG7U0aber$cc4M{F+#nF}*%(dhQyB*`h?)=fUjR z;F*2N{v1FgG!9`d*OK3t+xy&J^pwdTPfis|~E@ zA8ISL$Qopsyl}2FKG6SjN4SXrvvB~k@qw9ci;cRP>^wiP#w)N@G%QXjU|X<(d-sJT z!4FBP#~Gir$X`1)QD&##@8tqpIJqx5ac`@fJ;S2>>8nZS8-!LK5DGI;)L+PJw9sm) zM&m1?)Pf6hwhFWEJlYk_rfd|XocYSyl%d*QxpuQ?URbI`M_rEz^pml&h&uV z`h%<$4eT`y?41c5x?*fQE^u#D$~93DFl;L`el}~(2Ci)HCF>M)-De5TO-PwBVfM4E z?JumBhD5Q>pODhbP$9Zwsr;p-fo4@(ibdA^V98Nn2}}$OyJ&6TFh%o0)1DQQvlI>O z7D%$sW@i7bR`iRxO?A0WDzohcX0s10Le^}N0o)fncs^d>-m!o!I)P2vSW}{$V;y6$ z$-fKsb1#N!SZD5-ovru5Yv~gPjSmS53>PH-;fH^*f`Cf?QgF-$8y{@ry_jtKjcZ#aXL0oQvTm-G4O}xEmcBW) z?not9g9Gb{l}l|6cLXM^x6VwBTay2Kmi4ul`347=^&OhDKQQTgsGlp|Fl*z^+0`5D z3pV&=hCJ?MzM&%R);cd}!J709tg{Xn&Yr-THi0c8fh{e8dtU)daDhYn0u~Jw=5-IH zDwslJmoop`#^Gz#%v|<>f#<@4`CW_+3LHrnSSkcKPR$On7f5tSV3(i3TxP(m^MQ52 zGuDew*sBGYYXaEiAF$VaU`z;LPhNAvE`i2D5|+ z(}W{#OPOC)HoGlgJCMLV`vPb718GtUP} zMu$Y90}MI>44S={G|sLR5@6uHz`(3>V2!%K8P&kzae>pr;Gm$>cHdbCS3Ix`K6S9Vn(M}c?J|#2B25}q z{-m7BiQaZpq-VPIMvltOKbX`uFxx%o-nZlIoZW|Kbu-%~95MYc#W^S9MJBUV0ke@P zYxZNt=ncEpCY+vgfh#77+ueb!(SU0S1BXI_r-OlceZ@4cXP$P;6XYK-Y2RfilPs-v z;PO*oSpTGto4M7N$?^e9-X6^m9-g%yxLO=}4lZDeT)?gt z%r(Iw$0VS@^s@c#XG|izHG6M|UY*4(?{Hc0!%Bq%64ee2ObjM_YNeLE4UcPJFI#X) zsNpuxhTBKaGH_j3$&qlUxH6vEfKiBn;U8;&7jwg_nXx#KB=)5UU>QK+L@nPbCZ5`IqYP!e ztaSpHBY&){G~n%TD9(Auuvm<>)xcl^183a><_NnQ-LrS*N-=2Udd1u`F1BQ3Er@^f zk0B)QG9LryB?IQN%09jV#*dDStPC8t5;#~7BzPD9n!x@>~bkGt!_7($jk*#1wqkx_5QrqGJQ1Foxnu&}RFN}8p# z)Nj|nYY(>kIGev{XVqIZ`}b$tUC%ZB-TBDpoQ;B~xq_#q!0{#l)?@>Y76l_|W}~zR z(y?GTsg5C2TtYU8FbXBKiY>Sx_JQ@X0H>8L15-fp@{6oH z6*%G@xLPl;h&t5pY`FB^u2{GsQ#jx~N5K1|w#5tv&rHl2!{Y9k-DO~5U|q7|{m-j! zkFwtu5#YF;!14KD0#o9ZqlY@4hv=ky;NDZfvdw{KrT~xJK~9f<39Rl0FWct5&_4K4 z&rvZ<>E+&juK5mUOa%|k(rff*dk}Z(jOW8MhWaeO)L3#Ztk z5gX#v^O}4g<$N(`$g~Y$=DYdO_dv^uH_#%{nxOjIxvNN_#(Ic>qG_?gXi|g zSh5ukFx_Mq(2HuwFr3iV&94v>k`eg$c)z5?q9=<=7B(@nvltkOsC1mv@YwmOhQpxY zAj1@)0IPsZA;%_mfjGMbKN}AEvI$Ap1QZlIFtalkJUJD(u%VevTvuo5f%MrbJ+e9l z9Dxc)*!dzYOD>#hJQA+FH%Ip4*VEIila4-6+<0i4m#B8Gex~r(my)Z3|D6jsSRB^c zHFba3rzuvZ8K)Apf<*MgY*bH)XcjJU=@fqIImLX&{G2-Gr)MWCQ)RBYQzrg4!&h9VeG@mItkD;FVs{D5b}>VL{Uptql!K zTsi`aXM03EX=vh;d9bQkVAHDxZJ{kniY&@CAD?nW*IYa@J@HOqdZ+TEE^a;TRsTF( zXNUMyzLreqDmK#7Vlzyg<+!5ZjlnUI?xal`mqLV%&F)t$<}))WzI^u5E14z^vnmNE z)2kI1&bD7KIef6qzWVgT?bqA5&zV22Sixv1Rh7Jm(}k^*$x4;sqAQPpTR|h|AFhYZ z0-kOgRRr#&?9kfFZ+C#9SuE|u!d8Kp6A#%n3{Erew_qt`n`9Hhz%o0iM1kQD_k^Mc zO|}Y+No`#BCb)GlFi%*}IA3ByK?7Ier*F~R@;?HYI29@$G<$gEC@_okFFL@;wQB*J zxUltt2Ikd8PqbLpZF*$R!p~%P)T7+Rl8MuQ%f};KO4~{{bvXu{WIolokmU@Q=n9X+ zQ{w)yCHiz8T{gj|OMO`d(^->;8JCUE->GThou0_o)H|~}LHWGRcb+c+H-v0UEFOsX zhFIKBt1!9bQNJO1g2TRslqu2-lB^S=+%Gilx0%ABEa<5axLm-Y!QllnXM{#H6Hn5L z#hs0dG7j~*vveM6vt4o{c~+2uU}I;HSD}+D5&AE}N9(q9Hu^fx>QSrm4SSmLfFOCm3Hh=o&%u|hoIwuaZZS6mQo9De# z%y1ACy=DGy$wmEL+aG60JX|0(W5eNo$445eQikUkTC)Vg(wT(P85tX@Z?Gyc9AuU8 zP+$^NxXmh6&?xrcA-CQJ2d5kX4i6U}FTD+~hvN;?j5F&lO`790$I)#<>=!n9t%WFe<~K|`g}0rsGpL^iVn4IE28 z><_%aAavzKb6|kT0uGh~tPTQqW#<(%O8iqP>R-I<1vd2>MZvZQPC`nGoaz!xLJbW^O%o+!+U$u44C<-oembsdMz0v1({yO!n)T8vjX z3eR$2mNHq$W;!L2`$NoJ=2bziSJ$qaE+yb7m=_?cmgTTedBR$@yaJn6qc7V+4CL}_ zxdXL*4vOqzIP&>L#Ini0$9$&UOqRd@fOW>zgZ)KDFJ;&KZnyixxck;$XP!OT!5_XI zTQ+f0W@Wln^QT8wSS_DwR;p!QmVe;5LUaL>f{kN`k;Fj`xdjbuivqYDSd_#ko?zAE zaO^NvQRMP_!N|+7k=uFBL!R~*|627{H1^iNX<~~#(W>IHk=^FbdXY*AMuin0Ik;k2 zX5aY5;I*c;fx(4A)#Bl`Sl>?;GQW;EbzD_TR%U2CK5aD%rw7AJn+CSCs}AybpJS0a zvw+oD<0gmnhH%lfe;5Tj6quDyIIwNHb6VOYaQ>#NbGdS|)3ok5Y?G+Ym@RuMT9l() zszu{+;pFU@HeqVUT8DXLUO5EmpYhz)wC$bjj_M5i40E>0Q#5_gX9v&RmT+btH#5(RrBex6EhV19q-&hAIN`hL0R?#(c&+H3ZFC6AO+umOt_r;>$}q*-Ej!X2lV z+XD^@b{Vv~^(2Y7PyHa~CcwlS!@%t0aYSL8LgLEcbIj8k@g zCmi{IJ*f|0tY1yZD7@#;K(1(z{2-s0lR^THoM}7t0r6@ zTLLezreA0a6li3-z#+NYZ2x8J(+uk zOA`7UrGDzQgRw*N#Yg&Dr?&GRVBr6-iRS==zySu{1&&fYw%54+*@{XqwrlmZCoMKh z-WzMcdaS|rhP9;1k;SJ2BrSe4ng7^vhk2*d<#qR>TOM%l^P17(IbrAH%{yP*-ZI;1 z-;3E>45ye#PGA)Ouv4;u@#W=46@wOofF`32tceP2NdZDh2F<Tl5Wilb3asZlqe**9WS zBkBB{Ay#m&ikPyd@Sd1AGb z{UkjL4NK?6PTDmso-0@?THL)>u;|sSf1kDG{UnoG9p^fo4zUkA#S)yw85pHJnj93G zd0E=x71}B~*y}agb#5^EakLu$U}0!#iF&{qGl9jfpfOCVF>IkKdjX^Mk4Ekd41yb) zoqjO#oM2{mV00ANEg8^gF@r5BkxBIMvh2^(=KT;mvP-RC;(`2yb8?M2XYJ^eS89kd zVKQB@Csu?tPoZVCFyqaS4T0RPz70mcE6imUD%CPK@?J2ieQa}Xwd2iQj#(zgZKvI? zTyDrUVa@khr!3-bQ{8k+TeG-^d5Y1>k{EaUgq`JUSiAz*ygJxYC$ME!?0gt~=AZtQ zE$=_=tV?m0h&XIA#Y7~*S>OVrEC-8W1d~aCRa*~xbO5vR1t!xKtWFj!`zEp&K4>=n z&}_@VBFlj`nD3)AcLD>qLIdv&CjSU?l?yE87451o8N>}*!I(PTHZeEo{qzM>^#16x@Hd-a0$(7)}a5$q)mz5J$jA5LjXM)-Ip zJHO{T(J0U%vBSrss8QuXvyDcR(GJ$=8!f&sT1;=S_yt@`*s*?}VylA&iz5SzAp?uE z2Ma@2i^o3&)*uPi0#(+m8Eu&h+W3pv9AC_SxMfPwvE z$3eZ_S`t?pcxE{Aba03WFg|$VFY3_f8hi7fOXJku$mvU&oG-Lyp17o>GLvPgt>yz| z`=SdaJh$u&wz$q{$#P(?+RngfhV9XsDo+BtjIt%tYW>st7q9$tC! zkGt###;JQ4l|QtES={z}!Q#i!mhfVqk3x&11nZtkmS6@>7YCNDO?(qh@H=RF9sG-qfc#G+- z?d{F2tqBTk(F!as5sJlc`)v!D)Gjn~=1mLw&>=0rC@avY(!jE3=YyaR%ri>Z9=Wu1 zNgr@Ja9;9)mB58fVtjs)3;#9Bcr+~x>QhT->hEf@>1eXO(QNY}(eXyJgFuU7pR`8> zi^qp+`!rcIU(C*wXw`LK_Ajt$`@`z=p@Dk^qs`m-k~0`@-eNG3X!8qTHhD3Zi=px1 z+f6(_7|yGnp6naN{IWixZ2V6-t3aLppF)e6O5`7jnWd#>K_ywC7ShKH2oBAb}$I{aA=fHU=n%3xKos+ z>oRNZg*I-s78?UI2Zk2M7aS^eQXc;}uK#->J=-+==vL3{1#N}P*ov05fntG;TwN>M} z)Qof4j2310Tubjgf9v0!7T4Q%Q*uY6lR;}x1CMq>#GE9X?1Jm2j8eiFZiMvGnmlb#2&?*`Vu0IT>7 zt^N)zdpI3pAF!@`&EfKZ+0cPSU4q5aV1dVmS1TLST@3mpGZ=U_$OkvD3vlY(CKtFSF_V`%y3_JMhOceAa-8=Dn8QyZIPUNl-QV3akO zCHLZyo*d(Om1Nxwtzi*H$|p88xSemfrp|wZDQSgG%eRP{%?us;wAQXZ|0MQ`R1V|w zwQsG`?oF$`*Sl@4+ zQd}7OKJ1E`tDQC5`7VRsD?^FaScMkQE(-(GIEKs@7g}Rav<6OK@jTI5X~^Pmfze?? zi_?mmxi{E2E;q+qV0D?WRL?=6xn*%I154L$7RMhCEd*w={a4jok=WkQFSX&W&H|TWZnRZ0wisBn#4NaGd!jY$wUkE& zQ!GO!^TikUk1%|77CiEK+n!&Y3odZxJe;49xIvOI5tK~RzF)(nc z-fB7IcBYdCS{&?53Yqwy7lS9+I3UAnBNt#6lc8qZu;5N?2^ob%*dxN zP3kRw`ii(t(tJBl)9tp2>f1!A4UIM%JTyI+&c0z%pV5>Y#-pak;_-pmVL{7AA!bu8WXRC)R&C&!W9j2#Sl%?r6# zcFu0#^t=-lWkY&P4+(iZ(7z$`xPyf z3ZIRxm;Yd8I{KYaVL@;Ga$XbhA71ZXu9Dqnv3#ZLVNtsZmWLXu!50!8BrZ7@>}8X0 z4m!{#cB`4I-^DJWNzdYu%#1F92TaZj7`HBJ6g$Cq`v}ADj0VYoWF`hik%l2Sbk%+~oCtu6*^Y3l1FKX~;WMbvv(Mo$G`Bdlp z6!oBa4lO?dPDyv=-#c=UBVBopb?t*MotK}VUzG9b*T=`(^Honz)7i@*$i&jYFJtyE z`&&StNa7={kJWf3{(gfNB&o`vmJ&gM2}X_1h3>|BAt$UCpIa=?XOOZX;NYQFVR4};D+-htTe$cY zJUV1%HnNd7;GGW&8k9A`MU@OK%{v1ok6#P%v+{o)3z4~I%b)D3FnHR^1*M792v z^d4?G>D(?je@|omM_-Y?!gohra%}RKx9?kYb#cF!Ht!1|@6sEI?cxR(S&sNw{%Dx& zZ^s!L@~_oFwpE1BlA)1Zh4rZbhl*FigCkWH4<;^9oG``7S97AtBWE`Gh6Ap=_8UUk z+4yP%7I8GXd^*g^`=B*Iv(%@&kyY}^S4S3g6R}%~0uB=#QUnby97vs5`Am|T$7aGz zro=M0ROO{E0<2~}Z$5IEUUm^^WDz)^>LishVqcSDnFppJ4 z!a??8Ck9oPcG+n+nK?6VI?R|*IxR9S!AmVNT~I}!L4L`C11wfS912XDtu7xTIGytr z9Fg3TdGV9l>^nu=hDTV6Ppi#*(BmfQnq%;Iv7cKNQ^_S)uBoN7il%=lztm_ofs5&2 z!=dHoM+{Gyba)&*QXL(yl3n++R%Mb${e&qzCO?+2tZA(O@UdMs@BG15pJTUNT==XX zu3dA_O|;cnIL2X8GwbbX8;=UP%h)vuO$b=Tqj@f1v$y8r1t;3&8s1!NRhY=KiCwFIjyB&!Q{OQ4cG&I98#XOV4qSBBkyd1R$Zq?Hsw8f zLQ^za&6Xu{1u-NEv`Hi>IjM6xa5xEdIvi0r!C-CIvq-w#f?4500Y}UOC!w1a?51pD z%Q-eEFll5sFdKYmWQ=KGxc*yq+CRAm2CbE84AUwPFdUk~z-IA)^U4fIwk;Fexp)pq znkyuKQ+U9@m9l`N+^AVnEOVca!K8`S$sq|c0?UOQP8m3L^KmV3mR)-Dyusv4@hjMr zR;@V8z^7Z(R{h{I^SZOg?S2VHd$=9y$cQk=l}zbW)p^)%|Kn0oF^6-rn#-ZtZ{D_8 zSuVW4Z$gKDzyS{L3ohC@h22@#0=VTEj;KdHaX!bWG^fF!MNUAm)2YL8PJ;roBp+j& zfeW8V%ZzmWZ5z3>ek_*O<7i~o6KJvJa1y9YU|RLf;TW660wxWP1I%q#B-E>m+eE}>k z8Oj?jD;!n$qR?i%WOB)ph8DRm2ihHb7zG+9u*f?!bmTT938X8q=$CEeb1^am>B)h0GFc|%6l)72a#+CDc$ML`c zLm`V5%tEuo9cDBMq-r=XT9qKb4C-Psg$J{jrU5)W2I$Kx1tXaTv`QrR5 zJ~!`){P54p|Iua4Q{fQqG&Ss(=xq028#uUM+R*q~L< zO_0;R!GUM-j7MpXg|2!MP24gC4$Kh_OU!352~}zwRZwW`6$@Fw=%CRmAQiAlXQ!G^ zic14eiotS&)MJcH84e;Wj4X@>2PF$vI3$;eOB#vk^4<7wRDD%InVeMT4-FjYuZ-XW{nKm&n@*Z2jqN>x#6CL2hXXU`G-=M(4 zewIP_*wpnJ6O_b76^!_nd~CiO#B)YybI^9L3o~_WA9(XgFh~ktX!hEp$QaOdfI(-b zs@ePmuBAU%Oa%;EgKM0`cRh$?)4OozotMI1Q(pZg4aYJ*WSm;wbxdB}TXR*Q;M5hi z0UV)moiBa-w>JLDeGn?kk>&2-EXSm`b!Fh(EN1nN8E0~`{XT_Fj`(N1xuyIulbldt zvu?{Hu|o~*)-{Q3Mk2e#W+}8>jS*p0RcR3k)^u@T4eYQ{a1v-^a8W!qsj;|Kfm!Lq z;cc1+82ha`7rH!h(qu?v6nu~f?QI zk!2;*^uy*q|9Kx2mU_sO9&_v2`_99{A|BdN*NRkBe)ij3FiY6#sq|-(fs4|P*gf_zvXmIJ!kGDX8o2%5fhE(jnNmod^cPNTMlr%aNy`@;CRuneWr+#0~4=^!#kG)JQrex8Vo7rd-k$T3}txf zDk*ZqfsN&mgioX9lSX5oX5J14(HK77lmiSk6XzUZm~zLTl_9>;u(w>~zQ5r8YR|Km z1zbKYbl~5>p{*Fef22{`hEe*#9G%Fi%&GU~|FGNg$Tp?~=(#liSQ_;+(y6Jy@z2gs z6~7Lno%f>)8O1%AI3ya?3l5#$!4t>Wtga$$mT=f4L0Uy8fv=`f*u+uX#8GHT@V_6D zPT~rOH10GkZfRh7c2LoRk@dv^j)nsaCNWNp4T=Fx;%<(b9EW&I<}idDkh^m*aY7t} zfTR?w1M`YNK9_^+77h$9k`i3W<$?^15e&RN2N(hx7y=F`iaOjAbVztHm*b73@Qfzo zjAr8*hctSc^a7YQA270&OrJA{X9J~1sU9ZHS4=7#P2mN8Dh`LlUoh~RFe>ygnclc8$H4TQ z<=;(?CFg_`4oUVjNW5Uu(_mzcXqe`5K>ooD=8VN09|Yf~GH@p_aD+54ShyeF${@wa z#`wdV;|v3@fI~=8G+)gDj)KNlDhKsH9n^0-n8?&5(Zism;Kaeuq{q<2w&0!uN0VlR zvsngnm!p%&lm(Lo{8=>ImdNnKh_weJ+WuIJconGp|Yr>?-b3ypQLg^=pE*lQI zKAUr{)3+e*st1qK%9oRFWw3g+-M`p7_0qwpyJu!?5?~DQN#|u~<+kV$|In!N!12@y z#xpNd5Ub7{HL4mn$hevM^uIlw3# zG5_3O#ri*t^*T%j)g}6?bAl1)(KSco{~SHQ`E;f4je~MenqI3g zN@h5n;czzKU^ZxJlB!`+(uw2`JUqL{>1BhfT!2H)#&IC^?|e&?I!{ z0541Cgs_Ji90#nwO=gKXFvq6zV203?oSQHRYut-buKbeGFj-tA_)yf;UkPP zucmbR9^_R?j62e~rzt$~N_OI(5Wl4k`~QWo&S?0f6UE7pb6_1q&0Ggw7AL)c!?JrW zP2cAz5RvkA;!2%X-*2@~lvJFpzj!tJyzz25tmksr)9a9mhm(HqRPhz}S5M5jKS8JT z){!6`S%DXgDh^D-8%`+BVbZ8@n)`632#1sM5l7{e!zLZ4kMw$rJaCjPIVjP?c(;vF z^yy8F9!Ithj(RFgqPf=;RhT6IwJ>sOu+M&bj&sWa)|Li74<_*dHSq}tBwsLawj9)d z#lT$bC~<{D{Dp&riX*Q|qpZUMi5m@%UN!KVFz#a0;|*x!yVJlIz{q>&K-0YgJQWQr zT@GwLjMWy&tQVAj1vx9EIAj+rcqenlp(yVl&)Mf%AD=&3yw*7K=_B&?1CruyYbmYG@%I3Ve#CcRU z`hUrN~xw*b5c{UYg6w{2|bsl+*wVa zdu~Hk2nB!+#O3U9U|-R9yfP$9so-B?UPsfs({`;5+)Hk=w>0RSE-E&?UiQ>Aj^o%Q z9Z#py7&Z-d{n-!oompej4siT2$)?qK*A2c{Hh%?cOZy>V*E>&ldfk7u4(sOuZRdePaAWxcn^`l2W% zWwS-HJEXp_wcc}C?Z3p!*S8GZiAz)t z_-V$N7k|D`QhcCG?|@MJ0^h;~uIxNl4jhYURzog0`=JMQMVat3!_4_d$tuGn`ZZK-Dac6wxq!8h&Ti0}2=#$9@rp+^*)S{&o zTpUGA4r&)U3hhV|GB_yWz@%}Zf%(Q4R)rRY00%~q1B|m5PGxN5*62K7a(&O+FaF!v z8pC)#GJGu(-kKSe8^P}5|Gnfu%htt=ZLB>GDQ}yEdKeTO4)I)AsJ8g&Y{_F&)fyzd zirE|-$~2}-b=#N}Q+&|lV~zbs?l}jzLtgUya8&u1(yZ|3fSldb;G@S~^(IJM4l4L1 z&N}n3)s5!$Zw}i@mwi%uAAa$pfX?0I<4=m~a-70D{1_ZX91h93G%70}l31gz^^RNR zjkjXU=_eYJ-BV8UtT5p?QXnj$SI%9;C&D0{)ga+70T=S0KHJWsN3Nw#nqu%$T z>1qoCIn@&Vx4!msxM#$E5FtD;XOjU6ZOzNCt!^!f+L3xFP zjExhUhoj~Sc9lPjOEuagXXRPk_lr_@ThM7dZ5zX?FXnIWZEaw8tT}Q?qS<%R@kXUT z2UTV?+dSE9H~rQUVe;V|U%0NWb5O{qQ8cHa zbLt@x6DNM(!|F0law1HcHcom|nB;mG%FG${0-ViXILR&GQuxuR(($j6d-FM#*9;Oj z8jtgGI&tz$U3`FD;-{6B!_`}xHjBSlBgb>|{Hn?9TT9$`s+um$jLHoDzC)p-S`RVs^#7sVnWFU3K~ ztC7uM1w%}78iNZHn~;i&LdBy)tP;y+99bx)e$K=(XpZBoi3*9GTvFas8ZHPP>J(D< zQ}J+|l+q@w9+=RNtk4!A=e3BMP4-|wR>rC8`adMSz_hHt^o%yjpR%~XU#LDeD zE#*1k`Dvif=!1f80>Ewg%x!=+Fy0hP>ZhZY9LLjj9i+IBtZoWjnbm$QST(P(R~ zxTIcVT|~iw1&nPRocb;u4UMkc?7Tu|D+CT6%zGeEW33?YXUE3}r<-rjyW7Odb$e_2 z?QQ=A7B;-*meg`dG&;q}$jz*(utTA-(UFr^E+pZRPM=6_h)GBYBj=Y4>2@#>^RZIczowMt=DU$k8+(pwk4%tVW<8l1yvTIZ6{o&M9&gY z-4HPChC{2(oWf_q(ko6lPj}zP;L@*_A>r7r_T>We1Y5R77Xd-HgKQ#BY>F-7aYrsR zb_*QI?z6fVv19GF2P!w#GO{;hD>OOHR8F|y#GZPtk*iL7LTiiG+H*T^Hcb{`wGpUH z7h%#8IAGTJkAo?I>5*2z!YMztOjKeH_HkS|F)--BgBGU0$}jEvg&w>&OJBB8ZKL+q zi)yojSQ?islAE#O!m^C)M<>3kI zu(IhtgO6h4L_G_}9smBW`@?e3@sU8vgv=vyOGP}=HeEQBk(+c_-NKAVtIzIN#1f%p zZ#F)^z4FLful{v!CKU4NbOfx9-S}v)VO!O6t?s+h_T`H0^7|T8I(1EUraq|n)0r-; zYc{c{Ma}EtZjmgHldU5CyoO62doVc$3$T2iu&7CQ`wJ&dqe}~#x}{4xnEKVCG`3t= zr1*E=@4E`URHvUMF-ea85mob7=MI{xP-3LO#Q7>z zzq0ckd#nMAWX}T5cmZX($SYgokt z7`coZ8fAJEIpqQlNgTP*s;{BQT=BtKHZ4HLH#5uf$z!?idDAw#&N?6P=O6=z)>S5v zJ!@+lojMliCAhoPIEyQFL^0%rZDqG%$maKOU`t!n#G>NBy3VJOOCf?$WQvQb%F7cT zJ_3_irX3Qg{Bgv8;<3ntjw2>Z3Y%xo5GZI`-Py*Gz9ai?V_{#jOo)oPr9KBE zy8}aaw3E2_#f_&sUQ7|+y>3IZ^1Xi=dX^oB{`M{CJRs66F+0Fn?aYPF=oJS!8V<0T zHte)c?PXwBl5iAiTh1(dp@FUR%@Wzq2U>VK99X=YT@*4Nv^a1u@$Yt=|A%L)$K2ZY z0gtTxeim$7$rZuCEZ^~x-L9aK-@$#EP{;i`rGQ4M7YRJE3mDiQMKH^4Q3zyU5#VCc zI1n1Dz^MC>pX<^w`FUp?ay301Ik)`s3hqe~xK+?@{UESQ`VF7Z;t9u0d^}jqFR09F ztakR}I==9TW>edj8)^0h7M2@cDCjtJdA~?$a5*fsZ9}WAl2Gb{a|hR5X4PBKXmgZ- zLwUx+gxN1zB#ka|MgMUXmJVRzTt1QA!J+A&$OH`*RUyHS->Oahi%+mA?I?DX{&8@T zID@0&@c<5oH)U*%2~Mm%ZZWsFhy?BTFPU>3c?z~}ga zk;iAlq{wNHgj*zzMcggq5ba5lI>ocl_`etn?)En+hm!+B**Zu(WqdN=$s30)S?u5l31L1Q%)T= zDN$tQF<2GO;YS%e_?Fjnuh)#4D8RrizT=2Qr+18D(n{TAm0bxQre%w^ zJ^sWj*ZzQG25XUa(;cqJ<%yhr4lI*D7cgs1P~s{(!J6%^*s1$iNpSaqHa;hZ`(h_H zvsP{hlACv?MaDppOJK7zuhP~ziv?GuZLYL9$90t9qogjA{J#l1;&P;p^Y3U_){@6q zmRjK1^PZjMn{+cP(+2x*QN6bBRihaGPHu>CWMK9G;mBKep;_wB9X8V$2YF%)ngn_n z_;Z37L#97%(PT(!c^hZ0B=E2&C*VelsFk8g?x&VH88cet{!NgdRLEO);_;39kJ92- zGfg{jDD~(Kw~t+$=SJ+hJa=Q^{B6CdGB>3h9`faWIv_aXN>_r6oy6fApu=N3f_e)1 zZC4zgxmbbow6l`nH2)TfDCvt_9J4r;c4&yQFmn8gGhr@0aag#h!1I|?ZjD>B17bUC zd&^3)j~r-SzsyPNMnk?Wrr6A0+Of*kX z#H3NcXQ7CTTZGa`j@~T z;vk^3CV5jLTiILAI|o|y&Rk!n`KFQ^Rov zMZtIK$t;YLlhp+_yb(CpARxpjB5;uN%K^tl4gy!wYh_e-DrqX3oRfap^F!(A54{Ig zi(WII3fz9>IrA043(Hn3hXUogugx|&6*&EW*7aX3QE0(J*$a$s z71V|QU0o-?;8FA*1@=>!?>4=C98`ETWTTqzM#jf|tS%el798M{Q{?vTjs z&?p|4+oHBeWQ}9K2BTmIw`Af3el|sk?hE|?6!`7D1l$&iS11Z{92Bl{6#t~a`Q-sy z(E@=e2Zm?MAI?~9R(II!T;Su3{vYo)na%s4mz}lCij&2ukM+-*U0GQcB1aWYTznz7 zijhfS<&EU}YtAblWXu0mzF4%FZI`}zLq1yq1KYm<#mjB`?j$%g z_1o@j|2|nJKlsP~N})36K=uknai)hn1&yy56#3>XoU!hR2*W`(w*}mj)-$^-U|!+S z>*Bzi=D;kM%QlO_r$o{39D_&|qsX@hY?BhszER*mu`wXanQ589;~z$kjow+M@mok; zkug5EY)SU61TN+rt{=CSJMC@H*wJpUrNG1YP@1Wc;oow`S^B?NRs?<%boj@vT=9TK zhm$p>fi>-bg1-Z6TLQmS;!CXsd}y~P=w)*{V7f@nk}HYp=cRyo6P7hRmN~tV`{YNx^usb!v&}NR_TO7x z&9KC>N{eZV1J4ByhDB~n9wNM=D_3q}+o#09_UXZjI0ZJ2Ue<&KtaHAyZu!8jp(w4R z#W-VuAPb+QBBQdhk`T+n^1g!WWd}^R>Iwc|rW|^LH!P!2ZlPD@?C+|c~(h0*tp^)l}#efzR#*|(l=D?aSFSLiJ08a!n^1J?pZAqGZ)2jTBOvfp!v zSp7ac=mFoh^{m^@M@B0|alU1Zc)(h5fM2eWUFg9{1x3DP3j8U7PTv}&)Rt*}R#2>T z@T_d$cym7Tl4;bntLpa@_{0thuG%8>^_1qOFVQci#(jP$_s*0p?SbW8kz(<%Vs^2F z(;^9}qo z__jo8B}aBH#{8}V0jVQfOB@)cxCtpd{KsBa_KjoFl4qZ+%JTJ0^}=$yGCKS9GP{-; zB?RY*1{>)HDlD)(Go9hXlVqa@a`Fp)GA&?|Qxq~dV6LK?ZPLp)Ln|jxfh~c-eO@SA z+=HCUW~|rJLcJITmnrbE=<$6!AhIS=WQMBrt+Oo^2^>?VTrbNlSY+}hZ?f0l_iyw* z7Vdkh=@n6AZJ799$Y-y3F{@dk@U-H13xo1WlO)7^iXHfmB&0_-aL#C$W~PuhEkR(# zfd_vQM0^zeY8Hw)D2ik)?99K|$?nX1<-jL51;G!@*Vqm|oLJ7p)l^e;im8M_ps11GXhHbR z*FSX{7!H)OeOkWFuE}NbGFHEY$i3@x{)Vr%_{{R^HA_`PzD8KsfnupGjmcHZ1%GT~ z|8cE}(L8qU70>EN1!qsoh}hdJ;INE#T-oz&z~%o52HKF2!cIM)5lj#cLMM-oH~k@1S_jL-D$YJm(yQ zVjgT^Yvk;Dz?PGv5Y%-{Vy z_lCOFic>Xb#RHd2|Kavvk&{Cfdjdm@GK-VKSpi4R6$xDbm?mcF=VmD{U`u$w5|Gev z@>`_&XC{^%?C%cnS2b9NBna6p%#lj*_g*Nl;ijb5LbkF4Q~V>9tIzT_G4S2f;>ap4 z^mrokpmdt*^=ZZ*H3KXXuM4hDub2^QF(b7kF?+|1y|b1*y277zfXVFv$Cd+PmJbt* z7&$c@SY|i~q&yX{Ym}Z6Au{KO*ldfg+=IdwBH4dD5P0OkI&DD}pWoJ;2a9qZq}~b& zDtj#3nr2nSm6!22BmPy!qSZ2n)pfsa1bnz*x#M|A#xW)*1&x|ag{zac|p*E z{OSabvIlA_e$QOM8dY_OmnZ#(nxo{phcY)5p#mCSo%tqwV8^~h_6U!{kf`h zCUe(J)+7gxPY)`mEAak%B=+mJn2Vywwgx5>1um|GA_9!OK8hkX2Zc=<4?0c|$hs)z z_>f1)k$sjx2Jt%CGz?_#L;KnGJBrPIwNBGDj z0fV^*^96YhJMbtZ@^dwc7dQ$Xb6{ae6w%>g{@@@GbBrybVMoRuvu)jG1y?N}_1N#5 zXwbinv-_C&y|hzz+iNKjCt>L;9E zTyv`M@a>)NjZzabv=Z3oEEHU#!1%(M!K8t6*+Qi!{`vlr&DuLWFNYR9&x&56xaaAe zX)j~mzWK9Py*Kf@jREJrS{}KC@CE0YFZ;fX_Oa*V_~5|Pj%DCDlEBl$z+1=IopM~3 zWAUSl3Cu1Jc=#R)qaMWRccq#tAu$`hv#X$~_1wLF49XS};WV3i|+=PrWq?*+E zQyR>JPW^kB<81b9huNNT$C-=2JyHwY<-Q=?=9g?kM#&vUqXzyJiqZuSEVvZd8?tzo zHT*dq&nfVZY1ubMi3Kct4oogiw;#)gUlU|DIKcj4XM^to79S=dxkMfzMun=v?&~ky zRt7Eh^34ue%+}v-EBEPz<3{$`a(VMMR9Ujj zS*E1)p~lLDNqfg{gRx+VmZ#LEz8}xxp|o^ zG9D~wVrCcDaoJJu@8H2F9DF(feKwJw*}3QITK5GoIyIjZkBG4-Y%FQH%&XwhA>eo@ znOi|CV1t3efmXHn84)WC)B@xLMfY(CiKsYsvhxeLR4_0yrJdO(@4e#TbI)cW^;Ody zK4u6=#_c_|baQpU1&#D2NmnE`B`fvI3cd_a3HrFmwLAZdp^)|K8_5g2e(rf9WdCP} ziVKg<8IeRz>$g*cxdmf>Om1`j*>c&}Tvzm9kkOwB>ctIq^DFqCpAwaD6Y zI5aCvN^oFSsSA0;&LXbxp5C& zjNmryf(g0-vTUXzK^Cp2r|_7)+2}6l-E!!p+Uv4Qo(6{mHg~cs*<^P0r7J!<**|Z= zLNTrUT*pQRWfmq5J~07@Mm7bH#$?4N7oihVqJ$itCr3=vRGZgdnlm$He&(6X>}3nL zN#^EE%Sq4e|EtDf8q0H`LB8DP-vuUaz7zotP2-G$0L|VMg#*m$3JnKXIaLZ88D`pT zJkGC^a3MfA|4Cq%sd>h(7~z^Z2i^55UNCrSHcfeS#sqZ0?5*8iUoNcQ`)hfbKn~Bx zV@j9i96o1%a@iGsyR&B6EP{g9ToO6kqjv}U+oqn-?QpxOk{2vysC?4FFU%;h^6=G1s6F9fr5j|H4GCRSeZBk zS`}Ii0=qPq^$0!XoxxV%#QWy+2Nsd0e+sNB9I4H6PZ~uySlq=q8X6DUB(jNTI4CxJ zVD)&xGT3w>+IQ}>DXP;;Ebw9qB;FNe%#Hgn+xGJMXe z9S62XEYRh%$#b669n;}(WeWcd1+F>&6>rFJHLxgT;ZJGcd?)E5b8gdm zJ)uMKSsz_=Z3@|3UMTQ4-8g)NL6J?}fHBE-x7?SmvC9l*p7>Qb( zOYLX<_ed7V!`wQ*-_a5>p@Jx_b2;|+n^FwQW^aP&oiY)RR zN7W@SCE8>!(ml>N(fRJ8PX3uji}+tG;H>*_klk$qt8vRCu{j<`yH+{0a%CLgY?e5x zxU6BC!JoC+p3ErT`|3oSq38)m#TAMj+yV#qQj~c_6dukCGFVtQ|H3i# zB>`RPDhhnu7adqvXti3nGzs+{Xq0|y&=vT_h*N~Y@t<7BN+#iy2mJGEB-ze8a(Zbj z7HZ(lNl*{$^fXx{c=gCLWhq0>)Co&e*s2!ZpQd`)a!0U}{Ym6+ zo}{>JMj4C4h8F&q3C*foRE5)SuxQy7u9sf0aqWDG!;y9y#kLA4c`Ur)qH%(O)39Tq zKxKiG+_4L+4;ho{mYz7Oe1XG*{YJ>LwFf7C@i6wd?(^R2^6QA)P1=_^5)w>JdVMox z5@c^SoM%+XVC2pT(c+sDpE_(wS+;|L zqi#bp?=c3hoF9io(j{2*eHwcM-#koT=D^}wemeS|kdp2mQCHn<6F6*K7=?0V4vU;H zVEg`JJExe6^t3}q8aPt8f+iTHRP*}=FWA@Sn>*{Q-`e7(<_}exu20U)xl`RJnbN>! ztk7_jV?qN1O9LZw2LqGD0|rmqgOY_2@oWdIcq6!VlB6q1NXU>Os;$NfW2lzGux5`uJ|6Mvn>uSSM5Hs z|K6$YzVb7Rxc$OF!(UB29Skl)3XEJHE{}wBzgj5eF}x9bvViSqw(v6ET1J=cd!A0% z;^)7=I{0+M`szj&`M|*9;u|}rU*cn6QaR+<$#0Utz?tE|)OVbL%|pS;%b)Rr-&Kxv z%RUJgUT~8U_|tqWDDi^LB?~@R)(aqeh88=NpTb+P}pCADYfJ1~@6aQ(#qn;lOFO zfl;{Sc#Fk~D+i1@7{9K%Ty$;9f~RNFO->yO`TSBblxc&f)!CaSCyri7Gdq#Y$oJ|2 z6Z@6zFMabJF3T6}Sz`Kt|D4AL`G{HU>1WRWTb<&x;eSJp*NO)Af)9!67dY4j*@GI~ zdJVXIE^ypx$hlO;cQb*{GL2X0M7dZ+m&D4pSr#DJoGWL8kHMsc-gITHYl!a0^ zzDxBKN;zC-kWXM=BEYF9z`)#~sxH9D$sjxF5Ch`}hNaUOf)V~Uko*Rm6QERPx$GWeA` z6qIN9=}lCaHX+rCf#uzm3cH8SQ$JVqw>vL3t^6tGf9?n4gyW3f51H&5*jo*l@+4Uc zKd^-uFnJlU`8lwLEMQ_tH=A$3T4})A=E>^)V0u?1OGpBX*8`R^29Bl=tg#MUyB2WY zVBnBQh)xUUYS_ST^3Q>pK~j3=wBTD$V-uL-mcFbN$&Agr6kP1&an*1luLFb50_Lvi zo;n81rVfHiE9+jcGqkdGD0gbkoW!tf0gx@B8!e`8jhNbM-ej-FIc-8yY$x-(Fp&>zQIn@D zPPjgQwrQpIg&yT5MGM7g5=Kfw3d~ud9F_}e6en1^7O;eFVRU`KC{^H+%xu#9VP{{I7z8MX-UnI)HORLzqp&q`DU+ya(zS6&N`l$mq^wNMp|y z-Jte5n6Y_*m(~H6Z_9a?3FL$&m^se$k`r6{d;;I=34Cu7mOq%eoMqAS7n)OpU-D0C z%?sn4|1*HIKY^3+IcqE@%Qs2peHkepYCRH0E0u*9jTkEbE^D5xs$>23<+>BU%(Wmz@+qHk=6xcsRqVvU5v&Ii)#$Hb~JET zEnv3Vz#h1Ov9LioB$VA)fnmDiEYIs|oDNcXmu5NKQae65nBCKM`9djc(b=0aWw%}I zsZrp)(!ku`$fORsG?|eNbcn}42DioTXDXT485r8dmT?;JPPi~h-~top0>+DKtegu3 zZVIsLv@+Qmur@DP#y^|=$wbFj6Xv}Zm>O8Q@#V_pVyowE%HR+7?0#ptqG=ZQr%cYW z37g_JY;w4;srS{2AD#2P7?qYT?UC8g{C7#S`EN#<24;H&4lf7Bs+nx;soZlExVBB; zbPr%re!$XOz%^k3)3T(5knDu#t&CySagqVd)dpNk47e6BaQH4@Z(`syV`%YdSW}vk zF{wmVwA1h=tJ=(O?bn-Ymp7`r!S_h6Im zYi0`vmcCzkKVIs}Z|dPVHSHWnrO^WB2Np~*3s@H#@Ei%?Iby)GP=PBWz|}~qUZR2V zT^FO&1jhIzEpdmfQ$A)^25?C4S`_<%RjZXrTx;>6?<^7yvo1Kd_XJAnPF}opijow&&YvW7H-7G;H#kBc%|8AW4 zbobVZys$a^N44kI2ozK?uxahyIS6saP9;9+uIx(p0XSYEWH~et6n+He4Pakz`0arNW0fLf;?o?{whm zPvGc~D$tZ>D;8jj6JT>Xu*qS=inx<2F1%*GaC?Qt1E#D$7hD&LxI4O68*sPSnZD!2W0TBIN|e*_q778`k_1 zXZ08GF1e&8_IaD!VTrQYnYVYXomggDwPmr$oLM)%7vFGa5SqYTdVu}u^NB18S$v(E z90g1}#BwAra9j^ya+@l_FTk*V2o_@FMq?cDR|F~k2wn86?m5@aQ@nJlu^9k)a&yM|IY4lxUfg} z^&X3Y3tY0dtPETZ>>8@P z%L2G!92k8Uolq2D68}&d;mzzniKAbD!|FkEB?G(9gTl%Wt4#~;G|b+f_xr%L<#o+R zvONDDxZx_S?S5*5Po@MXlL7;ayaR*61ZLw8%pc|+j*eoQz`-6L&NXF%NpzH~j?E+i z15L(z9alawbZkD9Qp+^CI%jDC-#G_X@eS-#KAf%V<*TcGSpMp|}>XhxqolYKf7YV+o^U7n7MkHtsETx zWo%(_SK#tE%{g}i+w27H{RQ0H5?I0}uwP!loowK~vX*1<1y->DhJ{&-M}?Pt|6RJL znWOE4$;{slHHkIm8zg55d06h6aQEybHb~e4P8$%AJ)HpB`J49@;85=&+Bb+kAgXGPAXVYz2K4N33lCio5~m-$0e{Q zoMBBo!#elwrtYlAr{8UIjAJib(CdA{WlqDBxf?ixTK0t`L~k_WI25}}DS$;`!HH>W zH1%rtPviN=WcGnI=mLu+!>+`8%z+9uz8juRPttT1^;p$&z>@2p$J{Fq%D0gJk z&6(9VPn~0(qkD|?@0+=IzXVn-(E_ka`^!8u?&!Gf1*8=8?JGq+^E*pQ?#}jTq{j@TPGZDy1`WWful8mXzf=-A4|EPSJ3io3u9 zNd-NV9SjSZSo&nloiZ*ICNob`^_?{3!Ggw?$-GJ~9xsEIdUG0B+Id7CWaMzZR`gIK zM8kh&=4G#g4HFn#MLgU*rwWxQ{&Q?*5mK-@&^7g7BeSZ2ibkQS_BHMxR?FZ(#YV35 z#ad4qRdi>YWL|zHR@%1x#g;v_U&D5r=^8x9Zsm^NoObTc&ZBS7ZFQdBJ!OeYn@;4T z$H)ECtn1!%Tr^fcBcoHiXU@FCtS2({SGVr@#uUvl;kv_)729N4^W+4TdK4}m@eI-S zn>``tg23{L+cFmy)MScq3QV=>cYAx1Q9Ni0izHW*z(mFtRvulhMF$j?hG=>$+0eY= zf`h8ZBpK&>$N7bydQH}f+*FfbKPgViplPbY2Ct@B5k(B_)=CeZ8LZs|7?{;o1ZX!d zQJCS#%4j#IYYD>!tA@rkd=(5#EM64~jBEdS8EibdS06&t4H%+oqL zQz2VrlK%dV61Q0@6J3H%W-VFCW^ia-;F&owMJt~OdNVy_-hAfA1808gA3x6U8JyvH zY@+8^yw+5&rQ)-hMSl0?ZP#{9SRo*|MB?#ef18ggQvxpBNNUmK(sY&1@|u~{|J6&A zWkQgfp{sz$joZo1Tn;lDH9Z+NHg-BFZBpzD^jc}s@AzrK!p0v@l23%LGP|YG$nnDX zg9@v3MFS(dMG^zk43!-#_Qi%92^^di7Br!iU#7r;!7xvPaa|xoqDCuATEnWQEi6Bi zwdP1Dd}my*&>^^5C_si=yH{%l)72omPUW>03Y{&p{(aNXY`y=rWsRXnBRgM&WcrlA zHlY??zaI(hb0d$<%u<==Gs#_tn`QG^3*}%R>oX>2`L6j1FETZMXj5rs?HzZv+h+NR z!y-9bE*=$$uX@Nj-O=shG0EJjbBhH$mNZ0mg(^Be5&m}U!~>@HpA=jL9DNE8iFo|! zNY)7UvSgWH^J+n(Vn%vV>8c+NLem@>CI9fN7K`hwRPOu2>}fwo3|y(?J}9Q#Vk(9LMt zMxm1{jvNx0#_i?dB-C~w;-guc&4gYZvls>6?383_g%2D1uNzAWykYpqnf2$4w7s*6 z68}t(U>`-k4vz%c83t1vJrc7@HCW}O7VNhX;A{-5Zc)f{Xmz`g&n9xB!B6i|1BZbD zKT$G^#4n`>I}nUbz03LlP@O8?7uSe$xTJy#gW3N(}HK)AIdbX=TXU9 z^PJV1Ws$&qkNao*8hc%Qlb6_9vzjh%(EoNNu|#*ulnLJg8hI8d@O^G-QCRk1-aj`7 zC)TEfDI5Wu2RS<4P8Gj+g4s^Vk!j%(2QH@zUQ8;DLJ>z*cy@@ommL*ax#tPf_DK&| z(>}Dg3#f7l1RRw2Tgr4OQ-hg7fI&Fu!^Ble4KdM;4Iyt{t>f+d%oy9qAk)LrH*YEv ztIvWXFTN$TTl{&*U%BJ3ypne0@vt!CS%OEDUc78~(>c@?HpfNT>G%0Mzoy>2n&e{# zQuaKV+i9Ag#ha_K%Ei=e;;BpOC1(OAheqXN? zhxB;xFZ$47d*%U?M!-Sgb;+yLW4MCmUO844#JQHs($!gEIyI`UfmQDi*Tb4fQQlYvHkFeOvhy04Sd^4@YGt$p3M`zKZloxF z?OdoVCESTwMhPHAA+6Rs0%sN~M!zS!Txk!#-JJWUTrUK^j!yk>RGGWR~T zS=?F3SH9t}tdjN3Q&!jb>n%RXi)~C%)88rBC$Kg{yKLj>HPd`f8uQLhy6mne%*YY< ztw{dxnN*`<%Wd`^O}tJ9%<2bHrGDMIbM4WBM(G+qW~B=btR@@cL_b(g73J`cbU4sr z%kk2!L*R&s&qa>lfMls#9WEkL4h@@J4zLC9ahAy{Xt(*I;KGr?z!Y_)fn~xkM)3|t zCIJD4u=W3#JIIk>fI=?c3Sv|m!OV6~|>PiE<&5MJ( z+YMUeFIaOp?)cbOz9!j%!*WyY59R~iYf@D8cRujulbgD~B-Nlo(KqH8qkxhhi~i#V z4vPa7$CUd%8czS&Z}Fp%&nTf;W5yw=*0YW)6)Kyb-jifD7GPxi=JQV4K$Ty@!-FI0 z086@sm41F1o7uNdpEFsQm?bznnHL>+X={_%mG9xj#Gvq%!C=8FMg|AwI0Y7lHBpT6 zW(;eXCuB2TN%+UHA6Lq&u4TmaMoJE7|deW)ZD_rVyUsTBVwwp!U3BXb8I>c96j`zGL~}9V6?u_`yzwI zenA(<3oEt`(;#Msq|Seh8@R0*0vH$)cCY4UP!wq4IB;M~G6Pcq!!~J#=?@y&Bp8?^ z82-#;T#(fwy@65MLQPYGHNAsh@By=v0*iw{v!q9Z*osDX1=iP}_!Yfg;Yrmu{dw|jPjNPlnc6Fap3D{Xpt&tIPtHz;m=M+o*j+$JEpifFh(niGpyJZwAt$T#D;aupcUx? zZ+5LtW>EUj%u&Ixd(|%f4@}|{81y?DS7kRatYB6sVBqrLuwKCOE~6vwQ2b@6>Ug3xhoOnvqEYWa z#sdaU4llkA2EGRk^HUm^#4vg|hXJU|iYL zq_IQ5<^;3W21a!WX2Tausu!9aUNHTWl9+0Eq1mvbNnhZ!iU70v4<_{uDjpjcf39G3 z^kAvsYu9mM(CgUHVtAq9XJV6v0JDQavoS-f;Z#V1?Q@u35hGU7MfgG z_{HCAN~6F8#^rw+H73mB6l~;uV9&LHLGZ#qM!^sEf(M$NEL!vrFbB_Ib17ijBVyw$ z(VBOnRa&6giKAt$W6Rx5jT&cKpH1NKIKkL~0J9TAtAU38tVvQ@9_=M3+R7zZzXr_l`nb+*M}vf=IL8TwBPSa?i{ulO zTVpmf^II@p(ry%=am7>A|Fe$2M1!;M(-6xL|1T|!-%`%Y&S=w~+4E~l%VR;wTL*Tq zE!{Uyr;#IpQSj0N)<_1vT~}j58F;x)%`0h?|G{MOPm0;WqQynQDcQEg*|u z%~LP6E(~lkY-kCb&?0iTO)4YMv-4W>j|ROHt!c`wHVxdYs~Ds%wCb?3ICJzjSxD!+ z5-r-O&1}FBn#^$gU^wf6>s5PKSG(?(EMd^Q(5P~uSw(|^U4elqqe)PqQAMFykb%+h z2UA+MnZu0+LxWang$5oAW`_Xg7zP&Z1zl`6nD?7-9T#qq;t|IiNShnJ@pXuJ^cN;}K{4d{b zteAcNLFMApA9xIBFn1ho{JX?1k~Lt4MnK`3Q29rSoHs)MZCx80-`OCQ(d1Fc5kC~b$7-FulS|4Z-o)PjsrcubFQFKB` zf63vAGPZubhnMy`ubdiXdF6^+#jPM!m+rtF+SS6(C-0M((D-l4zlO+G&P``SId@!* zJ2+eJl_KYi21bs=_(H~aqAkWAEO`&wcK>A+yLUT3fL-APqumW=fdD244i?oNjOrYF zTm+c)GMEHjG&Rn-rne!M>qmpi3nujoiCMkPE}|?sH(FyES`9sz)C-!D_b`-uGRSL9 z4|*-i@?qgM(Z|=%sxfmM6k=pxU|_UsU~ztN+$F=;dIFP!0E?{vv$H~ro&d9BMN_XR z16M-Rd(~Ej6%+IyFdIq4iiE7P*I?atsgt3gL8_r8C&5H36i^?Ch{3Zs84rZn< zX7@)o11GRKY*;TG@Kl(?|KHw_Nms615`M7aYsiYyDBr2hSvOdNug0$Z_m}xk4sYLV zJIN3F45Akj4|W#r3}uUGh&|dFE1I}1nSnE-QSCvqqXM(bj_C3qta*74bAPnv2DBSL zV6ro4iM-<_xT8@i!dqZN0M`%u7!ek6-WE+R7DodX2Zxr+FN1kIm>d{dj9=Vv-0;}_ zM1|i90o-SffbDs z5=62THxSyL2#Q;JWmHR4@0Ne#0C}v1~UULW|bvwD;Vq>SYkU^etR*# z=Q$s$<{xI}Z?Pmb%8Zfu+rc#q9>TotYujo+ZguFV_|6OB%elS`Mt6y@?DpwUan@8Q%Hz<_9|xI zq!8S?70a|%VwPWI$>7kj^_1R;v6%qt_04Rx9gnU`BskwhsAf? z|1Il3Y0fKgfm`Mw&j0#K)-8)##9|+yYWfLd26s(7c??t zDCh|`-|=Z;$yg&c>Y7iN|Ak(5^PC zh^;>_ar~*>Xp?X=eG9Ya3D(RXwq_a3N)G2m8~)x{9%BCIhx79*x5X=eKg*Yh_~NP_ z8f+Un>&~RBLNjk1vXJVioXKAr*#CNWXThF%&o~Pj^*7867HAFLP#1XN-&_9H;DD+) zgBF7qOb!K2JR2C6GdBL%+_9{4f1`)u+Uvhwg>kY!SlH^ZaLbvqY8x2DOeUl(d(Fqj z!N$eMp|iqaK~pmipMru%fa1UAj`CMG9x6X^ZeSB-Vl>)vJV2PKQf_63@{5TIYNB42 z%moV>8(4(o6g(Op9Gn~!_{rhdlQk2$9B);rXjCz|P2k{{JF_Hnv(rT$jTJ{UB{ipS z^*SSC8RfFPe~zU~*_()s&d0j97$t1T#n>F0_3 zboe-*gnnED$H_)!cOG$_l$sv_O|G3Bf?ggS3J;AlE(l8UBp6KQRLEWK(iydPmg(z& z4V4eMoY?~AsVNwonZfwTsg+y8AcopA*2T5~?cTu4(I_$QgFk z!8qB()bG$#r9|F~Y=X(XqRdhHSDuRTSg%+xJ+6G>)$8%~o<=j=YknlQ%XSqqvIz9M zK5OF9+3=w;ZDyt#Bg=x?A2%}uJrks|_RaYemA!7`w`ZEmW*qWW+WDraL&B`%;|Yxk zPZ}NBTEhY|*jnNyII`tsKX9Aenm`yG)%$(nAev2hj z>A-lSQCm1@nb}3*1#NO$qYUoU-AGkLKlw$7eWQZ6`2P zKUHmLQn;j${lMg=pR-8&1--9pf>jkRG%K95VD?mAx`N?&=(8QK-}QXFRPk7;>x*H# zfb*9Z$^AAHp62trz7r^jOA&oJeR2bDnrC8-OxRL^2)i)m39(VnRA&ekh%vIH&eD>a z#Wg43u*j|hm0dZ@j(n3kb^F1j(GB9;2c=0S}T-eE1o^f9CjDm-zu0;YP zimcU{Zc`X`u()nHz?`yuxd6lEn~t5vQ*4UZ z8JTx{I3cI7K;Xzu<0B1Dyy|lf9A@R%R3#joY0PkddHTOFgTo6o91g?;zAi)9*gCY<#}O_?OLvGr?6a-=4|fGcgS-QdUmbX!u(o=gom*0?uC^xbg4h z|5MHHt@ejwik#%P4M`499x(HnUOA9DT_3j=zUnprozB%nBmC6vth!fgIQq$l?V8j zJY<*q@PO}Ud5 zjx%f{n)vG%ToRwOe9yD14s4S?UYfpdiRAf~=gA8rPceBJG;8!c@?G)ah{Ci07T$^l z9HAD@zh1TQ#!g_WYbZD*;^S!N*qbP_)WB7RZQ`2D8YM0^9ag1B4IP<0Ow23_2Ur6h zw7WYbuo;|i;O#J5A?>nIBwoRR$;wQaQ(*xk4~GNOf(1Q^PaksjYq)B@`@kTs)sVxS zc|PNYL#KpA2mAF0iYzP_IpkOv_|ENUUdI%0cC%_>>)T4D)faBgV@cti`){iD3ybT_ zjaqgG9Mud0k5>tBX3lYz`#hoD^`+o3{Yid3X(6$%7ftN(DmfZg{(@QN#)r0>tPguD zOjvw4i*0OgEaWQ{IFz)><-`dWe&fs0Dd|f(_hzp1WJ<|)6uD&1^+~spOG&~}(C7i9 z))K|qWjBuMsU&)-1#A+TAmQ>iE_1nU&7-LkPaM`<#(2(Vg9e|-20k_xLoU$_L1vAF zCL7}dW)BA^)?HGISh|vYbDlX$SvIh!Mm03a_&nz4FkrBf-_&r*^>w5}&1sK-h6a&2 z3Vb#V3^5iWH|yjQ7*n+0GNnA-ddlfz$ke~rg2a9)vT3N!64-Gt^hV>wyQiAv_WfJx zo^Nc$-#n?BQl4<1-?8IS{b6+25PDv7ADmW*4{{hc7F($6U3Z{!81{~=>7CQ4bIS4Y(-pX*K zQR-k83!9JtYxW0+R!+;?Os6DQuW@8xli0(+9~98o$9j-oGvgfhybE&8d@YQY`##>t zdn0hF@AHCnQAHxFPXuskEMQT6ArYez$W`*^@b-he@A6+Sd}g}2az~<$@}3y?ORiU5 z^Vb^ub7@L^#cu!RU}DIkNl7gQs&lRQt2(y$upF~iJ74j}Sn3bY&4>>y4s)7nLL3qb z?F5*9_buRhdHJZB%Ej(jg+<&!0!Lc7Y+BuBFbRHiX7OG&k$wJ*B$2`e8#KQuu&Nm> zu~s`$?^%!cELd8w?k8@N*pySkbsfBgKKCdIsAKm4wIY>MpjgB2O74 z228o<+bz`V!N@gF;4sIXCmo^la&EGG%2W3*J9TlAOzPS>0UK&x7kU3QVliOojCt{G z`0 z>dK4$*alA1-z@(WRyfEks7qiAIgzhfv9Z_XMUr@H0TXYTQ|m9@y@CxBoc=|vW!&>< zP2CBtqZ%xM94Yq`xC#?Ocozw%ak)mto z8RE=Nu__rptN-%t!KW4Nwhrr`Rrl=s!V|wa|L&QE4XNjx`h%!s<|;N4T%DrR{s zY=h((j|SGci76{>rrGvhRp4JWhly>Du+zu72J zj+%cC@J29m&5&g*+n)%)-@V|P%dyViKe zGLl;l9}NwkBGcxZDlU@1InAQ&^AUy#2|lR>4ezHOV9)3gV~Ves!8X&Sf$Pkz=1O_4 zISgWH$5>Z1h%T6V&XY+mpjok_k+p&0d;&xLjQa&D@{3+1KHhUs{SCj$pGFZACgBff z7RqoHq+D1L)8J#`z`}AMqs7CGrQsjLABGh+jBF+ccsLrESq{uvaGXU#box^XN0kGz z7S60(Oqu~s>!t+opJ`Zkq($VRbf)XQj2RDBO-Y*iHP|VY_o0nLCquIH&)((?IqN^p zGp9x#ni$7-&zb*Gvz^UjV>ZQCGjF;cog&Y0sI>Aef!$8IH7^-htJ|4K5Pb# z$|g)5yiFoEqF5I=EWRPSctHbmjKitS3D-@cx&Jtb2u!$kkWuDJy zW=}Sq-E)jhWa!UJ*&+?pW>0|=49>X!Y4#w}P47@*%eQR`B zq{HPT))gCjvElE*rj-ryE{$9{%Tg8{V7K5?&N{|Cq06+_i}`?q@)IVDm1&wD&0C)| znPi+cxxu)S!NF;Y`{b*_DlCp{p$DW*+(rJl$n0@^V0*BAMZgorDgL`oJ)Y`5)ilnj z*86FV_wkVDL3T~X5zNn39~&_(G=8TL7Nu(c;&4UZ^U!};s&*F20@s|CDpVD|Og#K& zV%bvF&kc)tPB?JOL@5A&qP~4uv_#*`9=F&v4S8 zaY*t5qxg$PjuZz6F|PV24-3v5sP*OTxOYIfAwu~Gqwtvnzf|K{?>NX6xF~Qm$vZH1 zq`0t{G+0e?WB$OP;&MoQhSYl3gVHXJvL=qgR~)yBFnwEa{E;RLlSz~D0SBHtj&c#q zI!2DaKSs&SahAE@vRZ)U?%icYo4j0}Eb3Tue7nGuVmHCgi07OZ9=jqATc2Qle(^!b zvju!fsg<9Y=YLySnWS3vs99yp^YAF~PbG`_uRM?1c7XfE^NNWv{}#D6)a%U(nsX<; zhC$fjzWM?tsfboZo2shD7e3`Lg7YSgBJ3=N^TSDc!~kn61|^&%JcmevSIPXD>tFWmZmFK7Uer zh1+93Egz)_r_#5QQ_nbldcg2m;9yjjaz&-5b=UMIb)Knb-kdr2R4I`mYt6s&%j%re zcn1L!-3i1tg?Zl;seLyu8hJD9F;_vG$%L-D_mn&QCNK^ zZL;D4?jMdDejJoGIm9_3fa!#*tSL9s8qdkHM|QZy>8(38o$nl1?7N4q>OtF9^6$O> zz;aDqV#v&g56WL<&d*vif1Td^vNfShZ7LE@C&b$1{|HJv;1=Lv)&VTESZ&RDk z_HRZ!n+9XNWuwrGhF@)tiV96hU5xS_jeJkq^Ix!Yv^b=0VfYt!fc=U%2ScO8l>@F$ z4zdz=O`b3*7@T6SIB2rxu=SF|;w}OeFDCDpq`gT>{y1df&?K=VMz-g@3a67?&q1CA#}wW$Du^^`ntae`WK!U0QtW9^*mKx^kF$kC z6JJdy>jsB^+%XMH>KwRh9C%+i&PzCKzJl4Tg@NIVyTTPl(}ZRP0S9IQ2R;uc^EHR< z-#97+9Aa$XD9Uu{q>$O3X4PtEcZEZ|9~{kB9Cq_Q#2(?WWLkr=djoTUA$!X~m-kF70-O1I zxH>9>H6@(b1RNM1{A*CmVKz;1R{Y_pvc`fbO#oFR3xMqLnrRo0-0`HmaH*7Wg z>>MB9q#DwsV$r0k<0Sv*pv;X%fenuRbB;!?ILiHHisI1&$rg&@DrFKaj1mow5*3aT zDrMp-WkM>Bw%xDJ)is^E(fQ@y=g?lEyKyIeY`U)QlCA7>P{YJY&WBM!!KwN!li7z( z37)23PKPvXm^6PJWM|1pJ7K_m#X;ePrykF$-wh0m4h(y)Iex#^SlrOS-NGpUfKgsS zQ*}qf&DMf|8@&cVWZXnpOXWOU`jQb1p7ku$njfuw}|& z%N|qH%PVc4ESQ&89KHRk-JkFEpI1h6SLp5Vi?u(b6w$Qz&S{x9C-{0=dEOjIV1931 zpUjuwnPhDyaidY9=V@DaqlCq`|Z1Zo7BD(2a(MC6tE*fn(C?|#R{weB&)7cesr}I}<1r{Z!+L;GrDm&X!&gfkZ@$+b z4u1=)oB!_Ux_3*yZ;zf|6CLl|>vFj0=2oRA|4ztkI4HA)vHuBc)RxVO@fNnljp8{! zRXV;@+oCe=s3t5(|%U-_t~4fgzhx(2G}wtGO(r`U`t^Lo%2HBN24dl zq41mi!YK`97Y^`RI4YPpE)Q^E4mjW%#lReKpvJ>JeS-sshr@qS2SyGD#t4Q!s}2SM z2ga6$5{q-QVwSMHIi-_s{)oZh=ERlvf8|bo`&xgxrtchsDub2-*LFT!z>@mw|HjQvzwD|C%`OQ2|YixV=@OM9t;@AGcFJ^~$4Wpcg`DErMxvlD7w?AJHW`9YzlH=bQ2fi9dIfH}z1y?(b9p~S0 zJTCfZYqz|x52J8^>wl9&3M?o71vs?zF$mvqa7u5GaA2BWd(hpT-9e-v;h>`%xRHsC zJx6LqXM$SO47+?SivtgypPj+pCD$4m^rVrUN6w=4*8@hD)v5|HJ3cU9+LCi+P3Gk? zzuB8hp0Al}-QG8MPyLrScXzk&L^V+o$W_udQ3{+C5EYrt;w~k&6qBm-|X9 z<(9eJQ%rW}k+CRZc=(8wN6v0OFRPb{)kGD~9S#qJY`LbZ1T6UH;mve~%Hgkz+%{d>?sK}+DCOPT#j>KgDdD5*{Rx*pPJL$;9 zChHNv|ERe|e$uj+x32L9w6X9m3b<_F7uCn1?zf_`m5t3oQ;mTkA)$dqTHt}oRHvX$ z^+hd{R;tbH=;~Ur&_dKq6J-jAyp?JM z4sq)r+R)5z&tQ0D!VSORvuC63^aPvU*vG)wBDLs;^91`xCzvM%*6m){{!io21SU(K zDb67~Sqw6^F!3z8;mjpnw84>GZ3cr1KM&W-WwT4}Jv_|Bc|p4)MBI+y;Dp?5Csn7% z9#J^Z$PzW-$<^EqjE@vJf*KlDO^}hbuB?w^&rZoFN`ta#J-WuI%HE zt+p-)64)MoIB*n5ndo z<==GqbIL11*xwkq*ax{q2VY?m;bNX(@Z$n=3x|{gW0QnIK?1{MDTVMRjU^M)q%w;p z9-EozRjG7nN`_NI6wg9at`&Ugw=NXSp1}0VIe+G&RYydGJs7k(81_84Te19N+pOxt zr^@4M>v=ZC@941!O4#$|(`tjmV%v;Pw{0)!;V?U*6U=X?bFkuU?_HJ3>GxOORQ++k zQ(sQpp@gBa&Aje)UcZ~dM`erqEH53?d30{feC{IpRBwmIq7x6>q&^8OX%%ZYz{q3& zK_fIPsIY=XHggXH6Nl`M3kw_E|DJF)^*pqIS*qayli&$QKBEf_JX;dj7&#nR?K z+As;VE>QTVvPQ9kOXWbL4x4u$omdx@UcP)Vp`VVm+aS%z_sjxGw!} z7MWqtYT=>C7t+D3A~lg+?N3g%_=Cve83Fq=c-lDw5BMrOOlahHVPMfKXmng6#u(_P zz*aY*MO-F;At-VI!^|~~vv?9YojV-)8$U2}898)%ylLcYn$gDgHn=_LNF$rl4N11T zfG+n13XDt{0Sy1x9CVi5U@0_9Y+!T~nX=qQL7gGs08@;G^C}ZPMl~T%OMZi$!U22^ z8V%y64HhnyX^W-5*d&=>yTq%yNLltxK>PlGs^XU)DtrE2AS5Pwd4{gpo9^NdO|s7l ztnaM#OfF1PHrvps9hPabSo&?NbhTlZU zqR+e_C1wc*2IoRW3-Ny!rX1@%9i#o}YIj9wqe#dDrimJ?rfnK7)k+^_eyvd#vh6-n zyUEq#ONIB#1s>u}af01ZF&fkMWw3Y5;w;~EC1Q5L$Bp+78ZMT<`ogV|J0|DBB5tvU zm#ka1+q$SGq+2^X`sQ-oeSP-<=ZvI7{HuO9OQkGet6Oj+-{oinn@PeHwx14;P6AA_ zC5-$X1}kC>!-6xd(ta+K#DkQUgoKw#nt2L`ci0$cYQ^kv2OOh59^<8c=Fk-ZpX6Tg5+BWYl!*SbKoha2; z#akA~a%cW7y>;m<>q4H>kEU*D_@{a+s_DcM&EDBg-*#L4GTU+IM%(1MGBGhPlX#RC zXr|8Y4%W2~va~K%NzFZFJHu%EPKg~1i)KyeF4edwrJxYXqOd@2lcz!`=zPn9kE?Qg z9&?K;9B{Jgw3iZDz$hOuQ^_@fS;}S+yJ!YOxSSzxcxkefS%r({424$r3}=B-g`)y8 z8x9E_b+P4OXz=V)jboUy*_FMa!Jp@LqTs~UDuz0OJ69BmGB9c|3sktM?0NJ^y!mkO z0|hgyzTBL;Ixd-)3qIOiUo@?3!Zg`06WWCZSGneHPvmo))|{go80~#WZ2QsfWcS9? zz4D>~}92?f8{&Uwn6?kf;W>Muc54T@DOU*(#gqJY#Z3uYG zEI#4D6Qu=mU&I=?C*5GvQ)=uEy3xqMa^gVx5rH<_Ck+hp83$RV4zWmGN#vg}gR!Xj zLx)wt!lyPNPRebTT-kTdN-kaD$fP7NnLUJ?QG9|U?-uSBi8BY-jUO=bBwS!#d3eDb zQ^x0hHY`f~3yWhy3k9wH0~j3{ZU;X!DBC>sZcdu=?6IE-i^_8H*bY`_&wn1O!^{tTNob>!_~y!lylJ zb}r!#6meyiV_jzBZ<&_B=Fq_TX93#@ZYCwYM^}_tY8-@iCGt;FVBfKTJBxwE>K&5< z1J^DGo?Q)WF$eheC@@Xgz%~1nV$xf{`R>4^MGKgWI9Uu1^93}p zF)6Y;FbM5wu+>|@7~>$o#K;oV$;76>!s00YZNc*#6=?-Vwo9ut=dd#VlWG)tcac$T zJ)?{Qv(p2%V+m|#2d2;HT;GzUW|5+q#Bu%DQWmc@;Xw+GObfLyIm(7jQoZje6Jexg zIPw1lzRo#s-?A0NU0B4h_Mz0i_fk^K-ckj8Zw?5ZSt9DqygecCMOD=m=r{cviQ_Jb*pSxoq+TlljsN)Wlx#p%+(xo<(x zrv$cLh1@f}_PofM%yo!~eMYGB<{z+$4z z$)v&WwNPY>A|Km9Nj+UYzucHLsto@WgrXMk?N;ELu|P2-$A(3b|3?mk$D==d3jY}Q zE!D_4VRoH=Q=AI(p+0^&Me#$5HV#WI?l&H|r>GVwB>eN`+rEyD3rXswPt$2OUoXonFKH9veRS83f9u zR?lN*U9><%A#sAs0ZxX6d>SmUd~0cGVWA|3~Z-kMVuW=6}<`l(9g7?JA=L12fY@!H_9Z+Z<&X5_#r4RYcHyo5?ddS4$z@WE4Kp|0PUn0+(1gTdJVZ6`PoSq0ARBa1- z@_LPvo6>?T4@TKZ!Ls)lW%h`KT-mDnw^PVcPoqujRuKAgB+TO==bHnPYztjyX!9v6Fm7^vA(Y_mxln|KkyXZp;S+;ENuyMkviKW? zqFE-cm;5!CiNqAV-MC><_@Q-tTnj}GCCcvmlE;~&dOgwd>DB*VG)pcuiBw+_`Cb~o z`eDK9g-Kbj-%pv;yYI8#;!XZni?xm>%1D&S96Km0w(Y|TM;@U?0s3b5`!?=8wC?n- zH!78GON|&(LLIp568TLW{&7vx=byBiDMvxz(*p-i4%VR5{Z0-nQxy0;5=FcirKk9F zeMk^l)+jRT0n?-d0)G-%3JwUwooCe0=JHS!$mpE*W`WzEgytxgRwrX;2?fSi4MM9{ zuhS9T8KuCP*T6YNfosk|{(BdcJDKKgc_*-Y-RpHtMH60g>+#0!y^{BFo5jqI{A`_f z6-B;3w@NK=l;RVYa!B^uxw)e5p>^Zygww_{W*eF>UY5PoDC1C(crsDu++-P+ZBj4UoMU%dh^av9%@w0k&3;|ML;;pW5uR4ADhs|Zy3A<~{__~PvKrW)YqNh) z5Kv+in6p4&+If*Ji6V;5ueF0%Zxx}bT}VwGfue5TZmy+%J0O*b@K?kF-WpL#V>CPAo} z@oJIzcJT;{YOjDcV>4|Hex|O8pN;%23piQx9`Gw16#3`CY@}c?m7h7lflF;=+Lyqz z9SZCj4D3e~gf=nCC^2%`ISS+{@J-q7UdqTX_0G}pfTGg@rkI9+x&s#j7zEr7`t3Mn zcBhR|rGe3@_=?PPl{X3kalrx%5BZlYuldI$Wg(@=5qGv_%}WiVCx6^>>ZKpN_G8f2 zW8kX#A#>}i#appSU$H zDKJApNa~?!HS_aF7nvLqShW1?1r~BPbXhK0pzZaLFM@&D&Ox?VAv5SH_m_#BX$wB> znw~A=5O(63{V~qg1x$=84;UhxR2L}-+*>d=Q^+naIWJGY?#Av@{o-&o<$oNbKy%)|O47Rog1r#cyPYzc|rr zmUXx2qgb(FqIg(hV&wAFol`w`v|KE{TadVf%R@)M)NX!tviABCOaJr&HO89aJ<%p zwhQvNUc8*>YGFI^&hqkECr=*SF;!+-X`%R(2h-M0KeH;PEpx{7i<>@4E&UuYeOIX( z^P>eKQi&pu6!_jn2yiu4wW#rbQ0O^wfd9{fBq_(${^yF$u<&O+U~XvO|2L_DE#rXE zQ$zM63Y<|2F0&Qb-)&qkI45US(wupVR8$(8&Hf5*V!Sa=fpL#_o#AFdh6OHMr#I~S z`bT6^7VAU4s)mfKf9=(ov~MhBDq5hZYABl&yY*onL;js5a~DcYQ(PKcb-nuT9fOL+ zK9ct)+`Zv>L-tBy;>@`x7s?gSmTR5+_iouj?w+XAN2azN=372VYWb|i>`~#YD;#*G zLwD`9ohhw&m%Y)Low15zCEq`VwA9mluN?S4P2jIu%s1~eQ$vHul17nF?BdHDg+3e) z`=P)b5+Zbjftg29NZ_D-z=1Don(9{`+Q2gViu(EKER8}r4h(-5i1E zrhgH(N22FkiBvkyFZD*yC}FkXXC|S9xq0$T2@FC88EWY^&!4xdf8P3hZsPUc`fW2C zCF_>IXgtkq{b76m+!o{VlZ)s5nz?tX;dYrhwKAs~L%%fW#HKW7SbRTxU!rxU@%IG+ zj}Gv2J(R4ydHVC`*()13A2sY^h~nyyV0K|(6`IEy^?;e-5G%KsZomvT_PT(BjN`1{DK_&8khDYyt)r6Bx9bR&ok$DG1Z@V-=Nn<>MTMG0CyVpo<*<175shh1=a&zw9ZIX6AdLOqR>5*2KGw+j0 zIq_b?eIHNcrza;~so8(axgpTXE$S8Skzi!{iYqyK`n3g0=T`(xc*J9LRoS_fXP5qq za}0_HuU*nGDmdDZxIK?Y(j(sE#Q{fF?u)W6J03W+>t!uDQm|01Q-o8Yjb|dKVw;qn zvjWemIE$wqDvGw7z!O89p`jE`$dF_T_J&i znL%MeLokQRhD2sY8>6$lOdJgdO_~;n-C^Z7X*i_3bk0=X=w z$mZXRaFeqRmwhIl75P^dVR?JP<|j!{EN;$7e-vW2H052!w#aEwS<@oY{p&XSDlHQ% zh?=RkP;E};veXF;T!kh{r$n?o6i@Jqya;2=%vLLBEIO6A!IAAuYQUZ1bD`l5EMhvp z4zlwxOyoA*`C=in-jWB0TKV2F3OVszd6DG7QRDLKD7(6efFtX*$nZv%b)UA&Gjpk2 zaA0C&=#b)3m)N>6P+YFz3L}q&>gx@MygV+qNmxAYk>7POERxT6u2j->SOrD##EiyH-Of28>i`~v%6~zDt zhJ&I44h_s4<_!ghMNJZ<*rfh>7M1L*v3-LvM(9WjNeq(+F6&fhXj|L+Ob&iXOb>0hc=zmOCcgj}YB4 zqj!64I`@z0XBR@R_dTC=JNZ%P`s^hW+cbAZO#T%0OfuzF%um0kQIB^2OnWsUm~Y08 z2b~72LMCep4zY-A*f1+^)}<3iMT|FQyj6NGX>@W=YmQNo#EL(2n0Xo)I4<6ijCJ6c z=Cj3}qwR?jC#Q*oQm2YSfx)S&7p|+^8UG1cI5f*$l|FQwLu}8hRZL;WS2-|j>eGD~ z^69vJVz6jfM`CLR|7iziog)_*jhdAk{BAJW~ziX6||XdG8E7EEGTHX%pK3S ze85>b+2^ksiWbN73$g3CefJN27GqYkdBV)+F1m>lkVWFkTp*C;RRv+-5 zDKt@e#fD#fv&9)0IW`#EI$xPwBf#m%=y9Ok;6(#dqCm5d+XcpsIm;RDDKfG;ZD`e# zNa|DKa8`0-^#2}lsCr#j15ZlBg{0IinxBg`g5)B%ZA@KJ7aaK{SgwKL0$;bPfB^%` zB%wx@xC8uC6T-YoOQxq<9Gt zXS~39m0W}J92S?-U6(~27F=sNlau##rTp!?@3cPbe*PnGP4d%atPUSqKV2~vY|!9$ zIU?Y1q>;!|rQm%2NCDf*ns^7Pn!>zdYK_cL8@b24XJq|Gs!x-C{U zKYtB})d|LxcW=0vH)-E_xUArW`k6`l-c8llR=jDvKwGic*}3sjKq+&`y$1F*F;7=` zmCoSZaUh~9V^*Ho^YSwv&xX&BjN?&YVSfJiknHs+f6pVvf|D3n=kP0bI-l7&FMYvg z6$Zu*=Y}MaDuL#OKNZ3dzTvdif1sJt$3I$U#@)O-@J>d zdY&0ELer8@&eqhLe6WOrF`$9z+T86nL*t9jKD&M;pV$ir z*@f5=B@ZaDv8!JDv&`wHds@fDjwKV^%}o}rN)Za*wn}tuLJ-sRH5yEn3tCL3IC7@0 zu{dVC>g$@+nC$4yLECq~%k8VSZ8u|(zHv~_Y@_9J@5gG^FOQTK`CF^cT>L2MmdFPN zp)(3yzBx_8D=z#}Gnv@wBhz>)T=#ph_oP+Jtpq!oPB4qkP+;ZZ5o8XUaEK==K*8oo z1DnOa4bDsSgjQBRb`oN~Z?4D{CujOc`oZg?tomh*y*fe6(hmdNExtG~w>L9zm}WTg z)&w+juQ{UGeB{C{CI$uugD*~O91gc+D$Q5~7#Nr*crqDnXjBSF&Rb$8kjVa>ZJTM6Y+qhL--)^2+a{jB z|K)^7o6LM>VSTF!)y5S~!ZFWX)V9?9_bE|&6Z(N+<_D3~r4yN!PAq$+rV(_3WB!G* zrVpHz1{_Tr*n0!mO$At+KCoIbFqtqg=_Rn&A7D>U;JoV0yE`GhM)K>ELH6>S^2 zZ*SniZJKI01o8^%nlEj>=W3_Hn6t^a8y2E;!RNeF+G! zwJ@PRaXaUoHm-#YTnWo_LjzbsTn+voSIc_jyQh&!>VulNFndz~*U|)sZG!AR0vxR$ zSd|l+l^vKp8CYy5vv>xCPix@b!N9d6fV-%TJ2Itb_XX}_9lSR#>-JT2BYaaYlTH?Uzv_ME)Nb%cckyXb7*10QszX(`)+?~OJ(Q5)*Y5;5a4wle_ zR__3|*a|jpfzTfxGb#&$&Mj!m(vZlG;A}GFykyR~b8GSS<;led*lulS7Tn2{m(X+f zdu`jrWNXzF|L_hO&pN#$F_Q)L?`+`y9?|)0dhQ$ru6+k4?K{9dk)v!@0^3{-1~G+( z?aR&Yi5N;GNCz1hRt9j*zrfMqz*@P1eZm3RhJf&-fbcp7|B^J0m;;QS4X%+#xECMb z_DGW0dw_Ru1MjT|P00(GO$?Z>XoT;+!CK17W;KDuqJh2c0eeFN`&0$qG6kDiOBCm& znMqx%5}%Zq{yD(Au}bNm0PFh**60A%J?5<53H_a&tzkR*`2&~=655tdXgf2ZjeX_J zvlAvve!wy3CCB{8TJvYcyV=-_F0kcXV5^+bo+mJIr+{XZNy&cJn1&nP&#q6@o08MN zP;%jA?yLozU#92IR^U4PfbTe`)13vp`zDkowzIf8FtRGh3K%qGE(*Q1gvp12qf&sK zFM})Jh$BdVwZVbwt^w~Z11|RVbgR#fR}>PgHgJ0sPhD_;dso8zuRFQ>!{;wBGLZei zx?6y6w*gnD0Y_&8N6iD4BmoX14vx(`*q1c0%RREI?=1A`Fk@d(wJu$e|7pN>BL>M0 zEU`0Yco?uIKWM0C;OMsO51+yLFVlc2)OhCRK(5&*c_vSo&{wc{-c6o`FF7opF1DDO zT&uud;K086B%9z)X6*?}v^Ovh%`z%?p`_q$k*m1&)vr_-5fs^`r3o+|L&Il#N% zK~Ag82+w3lvsP?OHjlilf?qiSGkr=LGJl;(Q*h zyq61BbS~h&`+@In0>|bVEENXKR$COOr$vgdVhCks@TyqN|J05#fWb3ihDQKvdV)i5 z!kS(N&Itw_J(B%11K1=FFbI5LS(d=HIN{&oIXf3ypIUtT6o>7t#ST(SiY~CPzRBh? zp}lGcQ?9{8FDcIHpOyrkYM02Ac*vl7vURq?%-Mn0m%dZrdv}2MjAiFhPoCp1c^?Vr zvz_KW@_~EN1SF7mN0O(H*idmT3I!X!?2sf^mn>^0K;2F&coj{zf~5zoyvVbf&V=NSJFxri4)Q* zrYF{ORyS`ktIJ&I(a6B4kQ=43CX<1q^#VutgPCSloV^BI6B1g#yx{b{z`*-~d0E4} zxZ7Lr{91eW*Ww<5bxv7J*6ytJ?Aq=VWoz|$-QD9WLjK)Kw)@tuY!dQ)(Nd+I`tLsQ zzWcC@XLspg2cC}t{7(y1<5unX^NaiNfelISp{YSq0uAbi7KT`3e~=A2{{9IVMVJT}cD%qfp>Aj{3%=< z9UE4BRN$MGz;#+;{=@_9CI#Bhe^g~AGwisvHa~mD2TNV*BctDHl^*N;~3zaYr5J-2Q-X(rf-5ELKkrtpE2=fsZ@8K9r%R zYOrhi9#`RdoXxRcdXEPW zlPL?^|CT+w3Z}m7c4ZB*Lfcaz=MH<9!n#_9 z0|gUixl2vFy?lw!sU@CS9Mc4vs(*L$+caJLu>I4m@P0xzOekq zb+wE8suo*{1Jl+O%qMSc@|(kvZou)vo1^0arztn*ClOQ21oqYi+g8y%SaJF2#4F$gm-_&7##OsJaPYQMdR`Mt)PeINLyt(;lS!}YXZ8|%rb{~4I!1uV})SV3* zW)^UU2yA3)R^t3(aAL~j(>1=k3fXT?;FFp=Z`lFvWd$5=4otR}jr|%pCcoMg@pIR% z1TMcGjs^c1IQY>2p`hfAhBWM2a#mcsqCf#9I z*IifCFiZO{Th0Y$>x7f#vro?2!&LL&`k#%~v>o`@+%P z59Ib7;QpV+;pM=#kn$rhrxL?VJauk~% zQIUDy>oY(4u&4k_`ojBHvbg%%r{0{*{jKx5ZuZKK1g_l%dvmOKcf8(IeRor)E!Q6J zwF@R(*SF@{d*SF_2ad&lO&_PQh<%WkXj0tKW!5ROmAycliQ#dy!Q<_7xT|U&vszv> z&E=XY@OYlUlXe5%i3LxJWkiN*{1xN!0Pk1GW$-%zUGc&Pv`NRir1_B zUdG!Kwe(^7GpDHsBWHQPe8BhM0pG0x?pX?4+Y7i4CU9>n;6Bpu{>M8$=GpIG_Oh9H zGB7)+WqdN=VZMKHsoH|oAH)S%mQLWi(Qv(w{ngd37i+UWZuee!ELEQfLr9Ibt8ZNXkO z;pvIDCp$l!yvQJA&3iE9R{Pxv6CWO*eX>=xxV|Ij9ryQt?sL9e=Q^msbEJUh_yfM< z0{jjBJO?ivay3w5yl=n#x#{x65A6OMIa8SV`guk8UmdW>c)9ZD)UMq#tfVXUUNiW; z$;5iI;q8|y|93BQc=dh*Z`%fmI~#bb!r0^AiX15Fo3;1RgX-0OhBe9`j%6AgTBN{r zdk27yC;qSj?P z^A0v1G+U^-InkMkiCMyC#RLV%qimv5CISr`A06jXl(r}k)RZ-gUF`7iAJb885sl!* zDSf|KUJ0LdHp=fa5-k>4-q& zH??WYi+-gOwIW|%Ixp{+IWzO&mYZje8A~hWI=#QK@zB|>OUuI7m_0n&Eo;3#@%F>y zliPAVcP@T-NHlkPe%9@#;KSQ;qqpbXKBTL@z2;8wi}NRa`~Cc4cN9KaH_vFYX7F;~ zISH)VY66J|Kg}^(x^Rmq{_@LNYN`e&7zv0>&5}}G z{?C8HFP2GxY?dj97VtT;2zqF^v`ld3(9BxN)+)7hdf`1q_UZMH49^|1)sawW+;Gh4 z^_s2cyts8YFeD@}Fg=-cnkUeOGhZU9OR;B0#e_TB61o#^Bu(Hk5m++CO>E+uQztbh zJ9}F^$ynTLaqGjHUd6JEW)qW(F}Y_=_X;$1TOOLg%KUla&H0nH^1yTfmN3--H4|1L54oGk6Bg>Yw5;UO zeDdKSoA#W7e^WA#{ZyY5v+5C>v?t%g!%b`g0SE4EWZ*ckX446!*J~IA7#J8j9z`W_ zs#sjyE}Gc!^1DRR1?Cx(b^1J#yLgQlmQ3;bx8qad=`B~*7@Hkgv*V2MWuM-&qFxM4 z9M|J!Gn?7J325oRmG?N<+<{fL#NwgQ-(|CBipToRbvxNuB2%RDw@o_sl)<9izE-WO zq7hx1{t}5z55HUyyrZnp@I$WRz@onrJNg@0woJRaf1Pm00sgq)M<*R-=0tJboFkC= zWMYaUBkLrA*+&glXEbgK(BPiWb=)O3XbIc&IXovDu5RFq@V+ON9dbZN_BpFVBZG#4 z(>mo-CtRi^v}pQ=G(FP%mnN)hAyaZ+zf!21f(ug8;@?ObjtS8(3>H7Y5IHE$(#{;ms?KdWq<{UT6VyzN$-X-~6^ zYiY98jf3aD7hT)@hT$!b@`nb&Qxn+U=OhWwbYQL6Q@l#aqqk0YhC>a<4O1qzhKBfu z^2{t93~U|={5M}5Gf86Na_MoLTIs;ztMswcBSAo@GND=ehQdr&lSP6}0xX+Nc5F}M zJv5!SYMS1jkNxEjn51_zi0Bsua!3gi)Hq0+Fy&FWMlAM=C;o`8pO?hZ|2 zdk%93-f$Ax{-8y2!h#vx ztJ#A_rIdAC_ZF^PxoQIoe?S8hON1!1mL-GNrc-rm7TxM=+ojCL!LX8lf`jb2K(Dz1 zPMpdMnuIFcjy+i8{Nj;x;KA0f z+a~tLdNi>yFSr<0Rv{+!?t)I*&4vD_oA$HBG;k(sFz-9@(@)o=bB_0oLt=dsw0?LT zm^Zb6Cq>P8ro|Dy?zw`Or?)9cT~rY<&0|o0Z^d@8qp4@nQD^1UfMv^VP2bPEIN8nP zZHBwc#4DCHIey=(-1>48m<42CFtFNeXq+MSkmIqmWt^8*K1@;D|-&->6~{)e$SNuwor&B}T9hp>d${n0ZJl<^*Y9vIcTW7umnV5d61k!^n1wb4_?`IG z(RoAEAzbAEyO+yE!S0SThj+0A=$EE(EcSfic=bt?*-wUT`!@$m97U>cFV-Al9OM zdgzD7^K+O51Z_ThoOeoX5MMEoFYCt>0Z)Tgl_$SDd6#{4s$a2G`roXBJRSzl(=PFbTE>PAgRteE7VC=MQXjm_OZeiaArHDg$6&p-#7_M?WU{uv< z+&FdVnwV`4th^g$FtR6P$rml`@iIvkJRQcO{6q7WVPl)m_T*eA@8aC+vlg&e9BAZE z(zek*a&l481LoX}%TK00RTJMg(QRQ}_4UUb;nim@8WnB7*!?7_S!!>DnrYd`{Ma{6 z;*9f;9az}T?Wtg6-Z{ZVQDI_-lf)r`#v9H@zaLtShcv-&|1~w|U;z$9x8iSy~Tbm@E9*R;*&#u(^~?Qh|Z-)t?4t83)F)32i1* z7WR1h91=WT_RVg$$lEJvJmqVyN@V@fJ0@BX{Xu^A3h^f`t*VF9Qi@jy=S(s0?Ax~H zZrdj1sUK(9&K8|66RmgLY=ZW_nYE9lA4=?f;T~oDq~+chZgN1VwI zdH3&3CnhtrEUi~yderamQ@Mcgv%3S6P{OKTYmZ%FJ!1b)c;l>_&XXE>T8=PEbxt>1 z8Tqe?Rin|qz&a(FEiI#M$82k<$vyR_7Ij++bKGg%`+0Zv;~7F6HcAi76cm~|wDwHg zIJsamvyR7NJB^u>53cq4F!^qRxn}kD>xXT0lBejLmYkxr_pEYz(T~M*PVR9%IP>*` z#qvDuB^>RqKO8JbXfI4?f2OcACVjV*2oEE}xsTm@PfZ!5Tb zU|FWYvXFr#)`3MqfN{Z%1#h~$3Od+=K1}wQaX3Y@#eGAIdjyNK0E_nrX8S+R_CJ{0 zGi6WhV78pmth0;RtfAR8h1s=+*`k5jRfNf`f;saxqq%~xcSl=L0bAOKBVH4>crLKk z+QnlwqxtebC8j$Y&F&_yHQ41KE?~oV=Ril%5tBq|B z4lLGA?kg#HFzr!s^Is-yvmeau)-8?-ExfBtoC3I zY_ zGp<#+imwPfc(^h0kA#i{(*uzvi;J$^PN!@=9W9vn%2zM8UNrLs&!N;M=VV1r+aKba zbJ6C#qMqB%V>;F=Ri*^pdCZ!gxKnR|{*nz7TyD&^pRvGxMT=WN@ZTqvyAr24{@A2a zz$B}{AQ;fZYt$OY(O&<9y|Ti2YKxp##hEr8)}W4-AcpWXj@#?=l+@rF>z7rvo) z45L(-6d5EZD9SC`vcme}WOZ+g)*d$Jg>2akr%pe1{N!^iHRjmEO9x*&Xs&;woL0-1 zwe?uOi2w5!Y>m>au@~4berC(R(aNOBCGcyD+kr!jy9GEi!{^=>;8kide8D8Upvgv~ zW#7U!Aw`d}gn-NiEa?kaGBvI^_O>`LZSm+Z?C3e+HRD8E?v2waY^e!s0ZT(BfAR6& z8{&5~f|oT^IQ8uHkc+M}3|$Lmh)q3v!QJlMTL=4dFtj?tqf!fJvZMnqe{gAEDS3BSatDUP_Sx4Kp?lZ@iw8Tuf;Jh}vWp04i3YMNJ zC%B42Qa#$bOnrUdh8&7*JNM^AA8Y9Jl9M|o@7|f*B4zC&{&&*Uqq1SC!V#s6k~2c1 zt~&6|Ioeq@S>AP1%$60!!IDp6yzcK{b1HPa#?4-`V^fJjdr8Kn>?vU@Z1wKQ>ZO%N z$)-%%6vAH0z&_;x=gbYq7d)6SCu707rPDGNPAFk%+{!=})Pqm2?XLWNE^LNk_e~tujg!Uc}e{_^`amuZ_2~ep0G)mSz|0LS$1~H)-9O*BXRBm z&RrX)aX#G1-Z`OakN%cN_w3#V7e{Q5nGn6R^|q_;eb?0Zwx_rE)!yiw8qvA-#;>JU zy55Eu#Koj>9CZ^&23QXtLCUpFYiQCaoWyGg?TFut=VO=)%QaEhW5IpE!h)pH|pL$5preM(&(>CefILErDWgm zoN=QqwJjwg#$aEBU+CE*x`BbYzG($4$4glzebqTNqj~b&W_`h4R<^|r3=E1tSvZUs z7#MUI7#J8N7(*BtIQ}s_=cqAmUvSWzLs%>3gh1jnbAEZJ774@dsXAi1`*vn5U3|1p z!8uIQbJLQk6N6VR2{~Dm>ODnw`H4A}sh++wy)uGkda8C#nQ2z5Hgl583x7?!r5drP zLRMZ}?mJh>(`f4}5!H~*T7f}n*?#lgmbCaTdg2qg-eZgThqiNxYKL4~!|5|aUqH)2NSp6a#c@7yJsz$(G0Xi9o}Mnsb;hXO_}Do?^|o!B z0+o;V$orQcQJVDRq}KZ6@SR4*+k9>ZuUWDrH+t#4nU>k_&UnV?i)z+fKen}U>Wixj z%O-kx&Nf*U^y>1`<-W6JJ16aWaPnW;|9NHEpI>=T*9d%H`Zj3s)7M%H*X>zWwaDvz za_IZ%r-W{_aH>g&=6XzQYi!eGYvFga*mjs#&&SMNhAGnQrX#E8vdWU;3Ud{{N^CH^I;t|Uo-y%;(ao+>7rk4Y*{tUS%a%Zpl z-SP*i>aignIqItAZb3({L_`FJPOb7jA^J4v^V6eur@M2SPK`P8cgp0Lb5EaKpB>5* zaig$9maRp?OXtRodbF$B_Yc4X8r!F477yBW$e47Sw577QOaQ?9Qz6yCzds zW-C^%Dh)arB@%q0>gwE}<7P`c*9N7ftIkYHJh^o0wM?UDAt{=>*PJlzbl2)%8D<^S zP!qT?H@ZV{nj3$&>51puV%HKU$Vl$1p5Qbu@?zv(KX+a!_inu^sT}pnl$ZNeZdPO- zuuQU=e1l{C7ir&6hjQVdrJHVr9G5WjRB*_b&ZvWNG6{C8^oBdMaO#DiP!g~i|6 z(8}xY@o}S?Tg@Jk(gaOQ(ZHjQ-QvY&Z@iEQzUmkX{-;^BJBwCk19 zl#)xlmER;gKAw^`?cDMF_loP&%4%mXov}(~n-AyfrP87t>i$Ch&rg2Y@HxxvsoYUn z_eyP{+_#qGD3WRO+)!+W!?Pv&qOU_6x-b1@bGP_#^FZN0=8{Q| zb`&e@j!ER+zpK0AUAgEEokaD*$jSdp+$1hYvKcP-J~L0DMUTs~BX)Y?^w&w@>nm!P zWQxs{y6W>}{lptDT=#I#TNa^YV^zmhX>BR7qT~4SwuM=`cLJ{l&q-I+m1L~;^mKZ# z>;m)C4-4wzxtNNo99Vt7JV-u%>dE72i_H6`rrX&%H1SUPv|3+NamwS@j`K7)Tjsxc zC{S&1NVF$l%5#pT7THOCg>IKDl7TYC=nZ^pyjK&u$&$;wkTrx#Ps0yG!CU z+j9FyYmTSQdF(A+qPB}c=mY!Xpu4ZGOkHPaRiOBB({)MBjAN!vPh2B+JGmDbuk_uk z;;wNsS;*>Ix`!2?9KFyizx{$H3CvO#9N0`(F!Ck|Fe~4f%2EE8LGa5jM&1Mm7Nr>tEIbJe0y0VB zyG|Suo3KD`lgcIcqlviu_9wF?{s*iG`14U-^|S)56j&&0ly##V+?kbNjwh zR;_@88&nk-3&k549Qdm)h&wbe-8jI1+2Xk6zKgwia}G=9JYbk9y`f=M_}yFg;~H3k z1X45KZ$B+8Q9MtZp-H#KQDn{w7Ktls^ZW{ugy$|`-F@%_i@=xZOyBHU^p0$7*EKm9 z`R?GQs(0TFR5RAqU;Fh`)HM45yXpjmxG#^Ggb!u%2QD#@Zag8nX+x#J+!I_@s)o`< zle5CvOIK{LdledKapYm#B%i=T>AV#e!lZX3u*yDYU_X_e%FHmAiG{)LVyj<`9(yg5 z$OEJ4vJH&d7KdN@_W!e7;>a%Q!Eh&q&1#4EzX>(}mN~FQ-uqN{oQILa;sAqqg_C&N z#&28em3U@9U{c@lq0_Hok?7lbuHsK(8LEYy#CKmPulne+SjnQ#RfbW$pfNGBb)rOR zkh+h8o|7_*^@rFQftk8rgZNDZWh4X_l(Wu0&X#|GZO(F53C1P`4c4#&Z04d>x@FQc z7m45B$aZ3*xWhHCNp6XGs?4{RG6f%Fw7FiKdMveaTKcuJ7S|iOEDj}iuIDDQmsBpR ziFGSEAzsF-z-Y?A`HZ3V;&tAe4|vZ_VAp=YkSJCb=23cIEJEW!={2@Io(l|o4tWU& zI1@IMWtnpa9AJ(4z*;qF1*}mGE_2wM_%ABy{1D%CLwx0R znPWy_TedgfVvRd&7WX(cHA<~D;X6YhM=P%agR4bp#PpWX={4curHR`aTr_%}-Fx{a zFy$-odR=ezb?1G4f%lvPhk*k_ibq;L6y+Bor6?hg#&3C!8k%St%X3)DDDzOlwX zD4QQp|HGWycSE{D0DF6d{TmTipGAf#i4|2B<-5{@N|{U?6&jy@lF<6VByl20V7er~ zis0l%K^2asr5Td33s{v`b`=YBTP7C?F7wY64Yw8u&lUa0oG;2O+fZ$Fz5D6a;#WU% zoz!~{ySIdFZ^_V@;(DSb%FAW zdw*NsDe*Fq4-5()Sm!<{n|*+D^9JrW3Vgc@I2hUDYi5)se&C#Ez}>{al#<@@NPvs! zo0yeJ#r=oEzAUnx*X0X0>E9At?VyjjuroJli5an=dR*&j_#Z%O=M#6GQ1W9miL zYKB>{7g&G%U=3Wr8d|{eV+p6@0#3675@rjUO%j+59cJ4|FyC4lE59JtbY}J8Z?hxB zT8{iEeJDQdu2^gE^x7o#DK5+Bh&1HhFsq5p?29Su3x3X^b%FijzYRGTxAWeK;Jy2U zga1Kp;&Fz{mW%=nd2z>6TfVY1Ey&BXp*u^O4V-<(zU{O^a``##~@c z5MWE#&>nvwj_+%V*@Z>AuflG*hVSs4T{3m{c5&wI(`TD&GCF85DlO=X*_P_nQFH2N z%gt{+XHL#JpHUi~zQo<5_2kJKp8~F%7fP;P;CpbB_jUvCoeL$m19+c1@Mb??NK)^; zoYwYW+PsIy=B2(&i`l@rXao0`Ox`F3G4~0gZVoK53LKj^^v{2gr?7!BilO1(&IPPi zPeNV?ibfg9mtGXxE#lC;aYo6aNP&})Gc_u8o=Uz=67bus$nbY+b9sO;+LB4YPNwjFp*C{P1Y3trg3I(gLj2)43xS1a_@5yfM-6P@+s@p@e0# zf~|1kqfH8YNr}AWk}{u6<_H#YFgniP$lfO$pM9Nc>jmz;3_MEFoAx?zmxPzgIn3Uc z{*PJsE7Q(QX8i}7^k&c2{*ZF->+JnMS2u;`X7B1b*;yNNa*6M(nj393-drWmcTBnF zna1~l!S({r^8miP4qI;@;0t-Z^yaUnr?xSj)GP~5Zw;%cWes2qoKQA@L)ktC{x=)g z%@SC5UNa0$V2OUfx@E%pLUuOk53CFpyi8^y*A7QS8p-xpSw>GZHrvSNWgwAhz}lQx z(W9=%#jd#eFw>NoDwLQ9YG&)TX=_9T*77MZ zW7*HA;GP$TxkPL)$!inqBp8EjH|&83=iNa-|J;(<5?gWU$JIrSM1n?=WN zojqH|VfT(*v3s;qo2I5VZB6CfkotJ`92cEl5r^E1JL4~S_WF3YI^Epkp4s+Ljd{Wc zp3qZk?|$G7y~TUAVe8Fk-t!OG#1GWyFNjZ>#$c{b`+7(ZBWvknN z-oO>Eck~LAX2C+)%9-AeGi9n+=Q=K0l)Y=u1Fm%wPHE_HYuIpWzu9E0&7ysvKu_oJ ztV6xg1%g?Z+Vp-~avwgx37>?gz5WK)3@L^3FSMNp6-s{`u`0SYC_;ZcN z?Aj+Umsa*yM#%$=_6OK(3gQYn7&SjIRiv%aRA3bUz*uwP_*ZxC z-3$vBi&%tyl0H)`WkDl<#n_(kU-L$yJoTYmGOHLW15c*27nq#v0uY z(>@R;Ghwk_z-;RU)rJR{|9xSQ5jb;Q{79JCkt04wqIXWs^_X&U#hkN0w|bspxZ&0l z_Kbn`1A}Y;YjXnkl?A+^yN|+kWXMt%;h|r!0B;-+2jL*b^&|C1-78?dt#0= zx+!qX5ny{|Rv!34+oeIIoyB7D6c*nDtos;F7ye=KHQ;7iBh=vS-4reIw@v%m#7UOU zCk3(;n3uQ;<>+x9v05-yPk!z}u7_-b4}D4;7Hx4jr0T=na3gJ%R{i3DLwi4Puf1@{ zGIq0J^ybY!W6T~f?%2*`dZ5Yu0As#3=d=%OLjM}h#H`J|u6e}I`^Y`FDfilP&qz$U zx@L~!$#{nsSNR`Y6>nfPb6}sAaP&sOwNu|XWFK5}XW+atf%i%P*TeuWtpMibA9$uK z?0d2!-Yvbo$(K7zoV8t7EU3Y*Y^lf=7nXzv+-nr-+4r(;Uci0XL8P&prRAD)>Bgcp zo78GH?wAmBaqVe(&voZG^0Xh(iSRx(wP2a26ZEAQAFq&Kt`=`~wC=tNG`hY>a;fm}B zw$M4ew-)eTOyDwXU=$BvZZ_aO?Z$gGVXg55#)$&FXCG{RP}ZXGpzhT(uB`2>?_GBE zoU*8MXPNWi#=EKuE&^P;9F`w+^$xih(Y`8q$)^(&Y957L5G^eVo3ZAkz#%4`2PY*? zDXyKH*=}^HNq}o}?xy_*`1TnbZ(q=*wu0671M8N6T}sy63JQ}Cy`7ckS}g3!T6BTu zTm!oS1G8ZObFBmK`3vmw4(#UxuJup2clm9qFYgg2ub#)(`d}j6t@6rJ;cHQUP~J1EcH(#`k{zwmvQ3dtk|1 zJAu*Lf%j|xGgAOVbR>hE!=eiAjsyW#{yLA(i7a8MEWRIDHhDLfnQr-r@rn z0yl{s6n&KQSkz|o4ogETuI$dJBNBEC3Y<1ve4nG3aAc=g_D1eTR;Ajvr3aR8b=aix zoBJID|2qY~T>@;cJ~ukQV^!?g6>wmc|FKnuJgb5Om~8`C8y@s74PchN@UFIj_XY!} zd;@FpJD$=22Air>eghUqo4CCac@L*C89k8TSiH;?523<+j0lqihmE@ zb3R~T+OWO&8t422Tx%3K;um=MJH*^G4A}dLb@A-=ACJdX9WQ^K;t_n1<2K^V!?D8m!2u^> zgV`3f|B_t2r%czF)#2gTw1iz$({!6e=b@9~>;30i)ZS!xG+|wS@dTN`xmH^zsT^9W z;i#nADfBFONEint9cJjMOIjKl-R34icq*9p7pDs|$ z*3F+bBauIA)yAV8l9Lu3=3uv4k;rMWs-U^g@t1~yfJa_qN*5DXMMJaHFO8>?vzIMA zHjyWx;FwJMv>yxj(^o8HW8Qi$tAUAaMb3g|(@9IbH|FiLc{4MJZDw+p#2+JuW|31C zsYiPueCjaOsq4jp=;Y8BE1JJry=%I$@S03s#X?rG z3$q{de!s*lW@mAyqWOmDlMkgzT`UV-7sgd29$y$cZ->)lomB^zxV1H2ExOr~(fFuY z!fjI{o5Zw&G^uE&$VYKILgJzG_Pm~{ z!}dUd%`#x+G4G=}oe!C!`MgyHLS>5Cn%P&yENIkr&U?TprEv0@!O>GcjUMT5Z|qUG zTJpo8O;w@u*@mKodu-2+ule^dZC3h%z$1)IG7$`ntRgoQxD0kET$voj^HhmTxo5(m zwvI^^kGKQVUaRov?O;f3;O9BmI@{Ln?-q8=DhDS{O^>P4mH|^|C|Rjqt2k%%dfjVA z4*8}R?{-Va6g2L!eB|(cZ`|4H#n-=2Xzh<^e0N&O!60p>uNYe` zHy!zk9yE#VQDEKEBvUgbVrP=+N84`;J!E$kOkmx(NG$x$QJod43%WQmx;P45b?qKb zEV}8y*7xeL@w~=XRh#Czs~Sh;|83xST>4nDXvRV16&HJhN)k9%Se)^m^-(XxLrF^c z)FIXfAC_c(Impqlfz!0YfPsghNvi*$1UpN@w9N($Ow$V+SSl1cf0aI#)>|BJE>e85 zXvSfgtUvPS_iZt@ygEs0#S2GHp$*f%1TJ9Y)M#Lw^N)eeuVtZG*d12)9UodXTpn`A zOyK3z4K)=AS->ED!&z^gBS+xQv$IW>B;E;Fsueypm~$i7oCj3_t7T>U!gVT~WzRV8 zJaO>2?R!Cvnri_%zb>RJ#m-pNRP~T6Or%6?>jfq8)f-sbn5J|w9~I$`5i;8@h``Z+Pw5$?XIvm;f(=*!sObcNSIrCIdYr~RBeFhE6 z7Kc@hD;n6W7)})JVBzO0I2AReN^JWc$0he)xG^lY+8=VQRd4#ANtU}N^5%TWk$(E) zSor=|Gty)pGIBLET)bk=81I(ApzyCzDpVoVdX6q`S;gI3LTKi*?R)JJ+kwX%L*bGLMU5>qrwWI_tb375`QfTA{>rP%ksJz zwGx{3Y~E%T-*w3~+t}&Ydaxz*_W?Gp4eAo7BcJ!QJ3_2V?B4?{xpNZMR;Gwlb?EWWh`%W%1olw2z zfp%`_znvv-Z!$k`kd6tv@m0M2Qj^%ahF0C0g*Vw*m9^(Nuo$0G(9r8({7}o!EP17Y zuiB?oFu|Z%`N%`xB{LYsj)nKC)Ep2L)NY;iYs2w*$qanYBrHEpb)0T|U;*cX1>W*) zcI}xrmR!9l(Za6h+`!48P#fZ9&GSlu)BgYa)a1Ku7K*z!9cC*D=s9*_>pW#0&I&I6 zhURpG_Kc2}H7h3VT)cbT!`*uyF>&46&Bw&Zd_z#@V}mwFYo-QEp>vapgUzFara;TS z;^3+K9vZ%?=#yP#b5K%7RzgY1Xi?RtJvEojFRnI*hY3+DZt6g+A2)W(@X4UCKeJ!~Hu%yzW?FYNSU zU}E~QgpGlL@3nzaMU%z_CW8%o^%IuqZeZklF!6IHgICJ}?TVIki*uno#fJqI`P-4$WK=57SCT)#Bldt9=Tn7?|(2x zWvU;M?3XujR~0&R@pO}^#+gfownp7eF##M8f;bWv9M|F6*!j5i{;z3wSI=ztb4av+ z@mx!z=mbVdkEY-kZ0Qr2MHTG+X$c5LC#(ya(QV(sxcm^e+yRDZHtlH>8rcpotSo7h zD`?>OF|VVNXV-=n?hg%VJ38NWo=i7r6~8^1v!GG7qLHJh;nfT(iOf#B1541W;Y=5v}qg+Ch7!TjW6|<$2`d+O#Sp3TE;K|bpLP|nl~mg1u$AHVB+Fd<%wwaDqyjEz{q!D^5T`}wi(P~+R>o=p*ea6 zTdc&wScjz{yY#X;nz=mAo1A2HRcJp_F*8bu-F3xz0fnu$k_}uEOk5lco90YnYhd7z zn91hkt!Z$<^97@F0Mmm7I!`8eKThD|Z0+02$i%(Z@%@9-(hRyPzxP}&GOAH>`laEN zd$37ML$>Wwi~0kW)(NcXKU*{RI;B@m3#?$0W@r@O(8xD|aj}5MLX8s&UzxKsf);cb zJ00j&Il<6t!fvypLF7eq=8K)|2N+Z*w74fUZvHo+!F&Z1TS?b{B|eo04SW+Cw*8o~ zMf>!o9B$46#&S#c^b-;)9gKW8CcpAo#c3cl`6g3>09(a}R<_xb^%u4=M6vKMU`XU( zOEhRu-od=;&eoq3Cl_0^vI{iEE@I%bX5q_L;7w)-PGwlRc)zA0gXWIb^c}4tEmzrP zMcI2#Un!WK-LhxpUOp`W=4UHx#-Y5{(MF7Xe6^0ADf6^x<`zWfeM!5M9q0gO@_H>4_B99J;z zKhWgpz{Gigf!m@FBkOK_=LvA0@oJ`#7MnpP@cggUC?L|e1|Wfk;kJ^qJx1agX`4QhV={l_$wNci@Vtk z7?v+^VVt|4eV4)8sk^>JGBD0y;BjEOG)1^nY3p#No`^tsi=Cy`*v>N<4Aho0U`xaL34o2n`=jNC;sPACS{b#W>E2BMA z;K0Kztl|s}bC%vqXlPCRamB(dgDpY3_1NRh9@;7jE&0jJ3O5*41)gZWV3gi)>}%)+ z$*8F(n?+;iPCa;&@o=HKN-}doVE@#-{%;<~OWvppZ*4L?(B!wS$&}+EJ7=f*gQi=L zTG{?(MrE*SoCvG`a#8$+&*Bfa<^MW8_%CJQ=v>Hr! zZu_u-XItZGiwJ&$23-r5T#ufBl=jRUt;hGZ#k^?bOlVN#2xabQ4WGcux47Z$+{gd? zzrKkMdCkVb$ZF8!{G&bgL=(%uQ@TGI_GC`E9}%y~z;wH959ig;r2?6UW=@smU{ZW> zTHIifVe!ioi7!tZPS7$AWHvlw%CKXm+r`<-nxYiN1sEE2A7nayWbunoQz}TE*pOB9 zGf;kM>aufB&$4*%RWN?jyCL$Tfydz7{v8cm6B-nEyic9Su)LuwGwfI(Yr&QS$=70= zf-bE3Z@eJC!C=)Vw&@*g(G@LL983ZktPvb-VGIY;^SXa_H!!VeP~u>Y5@27~H6z<& znYNRLhQjGz!P~ehSkJvP;99_N`kZU*mcn1v3LG2^0u0Sz7nI)CH7T6GCN)9Yx8wAV zId0D){_&Pg7R@`tcTiELY**ul6H{Xa&rI9a^!S^)nSp?&u|V$2Czmp(GOx>9v|>dU z7w0+^CS`{Fb!#46e;BrqgHu3)YoSDE0ju2u;VwRbE)fq#(H9Ky*BJOOF#M8hQhdO` z`67msqvwFfv6Md&UMAUFjtPZqPBXu;cvpbxF@>Wm7Rj%D&6dA|JAFcnfka!xkJnrY zt{hr3*?w5_GcZmSZHqn8mT&MeuAx2Q#McCdwy+AOz_kohYYVx>+rmDmrdBtF1w@*& zsLE^6O;10a zIQi+IjeDcRMZbT2?+o}=b{0x*kx?`dN*OAIsU%okg z@?uQTyTSK?VgHJG+!kLHJsO2H7&#euil!xPUh>Q{T%JLl;qRj$-}&<`Bo$UHTBZN7 zEoaBNDHqvdC$uJXw1zP-G9{F>{9U5{kHH|~TkM9mtQ)O{`^sjwvqo|-$FFKgsQ9?t zKRtF$(*9*t5go0GKT2EQ89ZriUD@6m{^9k#ySBa$8kf&Kc6(p0Rzz=c(RS&Y?d6j{ zeGuS(v@GR+sM4XDcZc45ni+O&AN$j+z*piIT0|Az^)6=Y<34*~52NIc2j@f*uRB)s z9^Mgo?+nksK+zAW3EvvdUeEf+w}T54{|etxpd^rz*DkG4ApLWH z=&KivCJ%J-biTh^A^Nb4QHFs-TtDti<>9nG(U#&HDV2{uES%aFxh7#_QlalO)9f`l zYmAO^2}zq|+^E=isI6C2ChUmE&ZO3E3H{g;3pOUDoSJB!d#gwB@p7M;+94~xJVa`CHttcnOjWFAYw&8f=4r_u!O*Ze+G>X%o`Fncjazapu@<-#3r&rkTZO1(bZk0 zlW*rr$L^}wS^79E_x85f15K>b%d}^yI@#KYR3tbs&5qvPasb#XHZ;7Q6?(9N zG2zey+l2`U9gX4ZQ)-tvJoH=>zA`&e_9Y?32G%tlcEkfS|uR$!DTKZmlKm(oY!7(WOr|5U=nok_{but zP{Yu~F1um@6PtuhK`5he%F?NfEIu0!^zh!;(a`%Z`6-831RsZj1EbcLMsCxa5;L#I zR;Y4HB;4h2yJ2vB&f;FP^FBt$*$qPqJIyDDhIN}h3J^POu(^RjDr3b?Ge%a?BWyj4 z>m(iSWcf=xn87I85*TBU(Zs-`Qz~}xp>Se`8aIcR!FA?h2ZQBDibFIO&Mmv0``o$Q zu|xeH-_xBJ4mogZiUd0GMSgUv%};2|auqpS9_X5#@bH+V_lt%Fyt=#*3SOGBKVGx% zTc#r9Z+udvIY6{(gHw=;h#;#-LJK3?)_9k|mJa!X4+jM9@_lSzF$+>q?oX_J`Y7D2 zt2MdL?3V@ud$!FZ9@ASTlh4LfK0Kl*Yg5QD*~7@^Z%dSKnNC-6a%b@;2FQWaaPHladvf zz;uzhY--r6x%_qvf+fq%PdqqOsk6Y!T`Xt8t1l~8SZ!IgMkF+GzH460+O3j#Gd8*i z7)>#G!xhPw5fs2aZ^<`L{U=`(c=YjK5`S7&8-tf+K&W$0HxpA8M7?<9&V^MHuXT^C{f?M$X~0 zX7!cHz0)^dahk5b@2?n7u0r!R(PR0?JP!&SJd$=mSXwKdv2f#-q<^zcZEV@tEaH{H zbV#`2(5Dt&cY|G<7oUH)hNWa`)G66t4+^zfxGi{=Y;ojR`TgEXv5k#KUA}~dv`YUB zZFCh~w@lDXBaz)*x$`!g3$yr#Cywl=rnWFlIgsS4;ds1>k!Oagl6p;}aNXlp?TCZi zaUB6I^By>Tnm36_>wqI$`hjLng$WHZbGvn{FHMZxvS{jw#B~fZ7EE#~A9=E_uA(~rpX+lvuHu5T!xddONG-(zp!RkLodq<6Zw{$SU%U}Pfo@Ru3O7&R0YJA{Cl9l zxpIE>4F=E9$R$cqo29O8Dckz`ivow4vU`rJh{u*DvjvBECl(x&6bfLHUg+(1<_wof{uKtcj8sOG zLl+-&JZNANS-|Y|qgi)ZqsvZ%lkq-xT;C*>_g)HRmfj*Ath;3*Z)^!8%i0O7hr1R_ zy<}Bj;acHf5Z1ix(Mq@8Y?};&I})}AZybb8cEo+t3TAQDbhkWrG4kA|#TR?yx~ykh z3R!-vc%JwYhYs9jYqvYxnHO;OL%l>m?=1kST&v&+H^=b?5V1LrXjaR#|hqZ2Fr|<1Q}T!7R2f; zN{q}pawa=USSLO60P_~lhGPW_*ouDKiC-Wgab;y+NloM>GlR*q6W%QHn(?4Pl2@QD zu|g+cQ^#Zuqi=lO!uN$;#dmQ8ny5QoOb10Z` z=UP>Er__M@q!7gm4mr-c0h`3c*YLX>Y*ye|7`k>Br~jJrmOXD=L^>|8q@Q@majL_G zQ{(|d+PiY5f77g2?>ba8ZQH!3+9pMU&%V88VA}AMNz$Q7F~G2QzfPgV3m+uek+!&Yh;7ybzs#-QS>!^dSc<=O zbX&hT&?$;5!^!#yU)Q&ljV6~A`b;>yLuUzwzg>Uo|L#vd_Cc*ok6Lo}Sq1oXbeYUr zqZ@AGDX@uaNfXa12R3;PCW#Ee9Ri!0#MTJ3YU~SOQFlocUaXL)8*#ByNrX|f`v!xG zQX(Vwl&g%^cLI%XPMx-8&S~w*%NSy$Hze%P+rc1iaDc(cfIU=W;(GBu*-0;wdZT(W z#ak+x3^&6MDXc6NB1{}<6CYk+Wi@!4Rdqr=Gut{VWUAv-q zk@ATf^WLAhE7&jKqBTKm2}51Lv?W&}x>oh3IMy@Tm=`fu+|k~Bkk{)$V~KEwkiZ)g zb#qC9O+~8P=era*X*W3XBnYh0_b{EV91^RyD}h|SxSt#IGN?1VI* z%{!P_{azpK)V!9Q(sh|h<#djTt)jVe{h{=Ck&+gI(dK3eOu|bV_MSh+#B%hNS6IXIL*|7)51@#n0SJ_g75r4moD@t8(13HK#5 zcNqQVxc7Np@e#h1mpScRnHKkR7=0(DMptg&pS;n*(SyZ#!Ku{@7hFTmxxpww0 z&4WS#j{FN;HZ(AJ#hh>B;l88e;XNhj&OeSuUIp#|oe8ZPM`zy+;<$Q9zlY7HNM7KOg2*8S7bZ;yC#@+=S~r?>Ee;!795#7zD4`%;yl~>lfcW!{@vIRIxf1;1 zrGZXIIcGCOOTAr~{h-Zmso27D zWjK0A#^Vk{Ym~A1CGP?5{{AvjXF&XHXX-=zc6G!lvfQ)QcX(wbwiACS3~6j_wANP zm}Z^Y;d;bvgCmPvxbMQ~d%BLII*z`+jQm?z1aln48xATqG#>hTYOm*siwhoX_>eSf z?-B2mjv$wV8V&R2>})nVbXZZNUC_gkSE5nPgHh1ppw6FvO>+yKME*1yEpd{Z*JS*F zNzaB^&!Rcek2y*0nN84P0p5xB`;@h&G^NaQ(zIbxv~ZI9!6>)kU|OG&u*N|Vg@f%0 z=heSBY3DF}YdM(vV2)`N{6{yOWNK;XSjgsScCK@-+bNz#$-N%( zeD1nWngkC>XR7uW++YyeaBVZIkgD79|BQ~Md}qrW+$T+J&-sL zhxA+yirFX&-eAzH;t=dOq(4DGLWEI%fs@p^O*ED#>oY0vluJ=+%e}$i}%fVL@3|0 z00y%_2iY0TCN&MLYvwHDJ;;5BL0}ET?1)*}F=uA!Ffc1Lgq5%|dN45c95`ywa5Tep z>MMcmEi$Yfb9!_dvNM{L?;JAu;=n5+AdtZ{>(+uIhxBYlp>s3Nmt~yVnyd7CM}Ik2 z_}>JV1!29FK?{P6RO@{XitJe`vVl?bNP}R8V+|9ZN7gj<8JC*hwD6p1xKT67V@uAN zunq&Od$Z3r>0gbUdy%Qbqfzk8{b(sh!HmY1sAP4O^9&IUF&{l5P1yR6OnrK1b%aT9 z!^78`s#v!~=}vpZ!R)}m(9vjVa+pWNfhECVN65LGRjwQlw71l~X6a#IYB_Lp&c9>K z4h&2R49p80cE_FJXldXuSpB$!d-fTQj{yxL0*7>Mm`hk4{jOe|%f={lM{D*kiT|PO ze%D^|Wh__XJTlMgK#j#&ufR7o3aPt9wRRb%R;^lI?8(T#=D_M_O(%u=x+@*_8BXB2 za$xNz2i_Y8_}=7wkac`AZ-H+~15Zi^yMUv5g!0_CLAnbB4ofl$9Z@x@I~XgK`zfNK z>F#R2-1m)Jb$58ZjO@A76L4!sk|yJh)uxe`S#IdEym7E}Vdh^TvqIsn5t^{IQEmgHz#fJVJb3~>hjbq}8Q*eP zmcr%z@0-JHJLx%X23r@NcW)HoopRivKw3lUh|87sHFx?|E>3ad(GpbJPqI)N*5u!&!r;yFC|*Zg3P;I4E@Fpj-gsvTsL%8W_|Y?tQu@ zDN(}lai@=mi{qTXO&62AS*N&0-aXM1GugzkPkJ#c@6z=$*E#wwa4?xLusmU~bU7@j z(ZF`6QQ5;m%KQLhK<1rE4EJ7L5MXhXUJxQ@vq||uqohW|9kF+>I}fUQI2(H$Huhjv z?{_wN;dp&hHiwA=uMN}wbAfDZjlBz51pYLzSNHetoF>E|^nWSi+&Hd^JDDCc-MKd~ za4%@!n4#po^WZ<8>&I0CAMBD1=iSXHI)|}Ji}8Np8sleNf+>zQ$ChijGzOhH>%nqR z@<5~2B}VBBjhX>!vUjE$&p9N$LdW~YS@sggb6o-28xH9dFbYaI@~SkR&S{j`9K})F zw0^1#%ap5+cYHM7*}ZJ)ohHfE%OtaZtqPS6&zyRz!^rTNPy_?N3bW~kgFLPdj@wt? z$zx?X(;;MX*yd4_iG{O?hqH;xVa+=Y96Rn!w`<_|a?rwrSh8=g=y#pKF&J+}_ zb>y90%J9)$G~giXyI9F7{}@f3!@YAF6`mZ@k!jK>aTH?V-v9QZltq&fLz;@fA!8F~ z%?pn53l8a;G;4lwm}Ps9P2}MG#L&5{h7v2XgiGd0oOjZXc+8pWvDxZ?Lbg<K=jf@2j)~D~iVqr< z7@SmWoQ?i8@Vsc?`_u65UxSjzVWk@`r>YJJJZV&0++gq6z+}P@^!c^uoh*S9N+p)d zOY9f+U(^)(6Zl^=vMej9at6Brck$%bJhwZt?mVxI=D#bfD96`B z`d6IhxiZeLbX32>WDucja5Sx_qfF|3cwfPth$-Ap7%chLnmyHN;4x)ja&U-}aM;jv zPB!9zyo;sc~RSY2ZpZAY$Td^u_4^vx-BcuT)IqobAsAuC4pnF!%j~eWANIa8E36?bl#7{ldr-!elyw$#jb2 zw%;e7KB+X3NJ|vzoRNHR&51^(f68(!Oe!0!+l;>%Gc;*FXkxW@mJ2zUwB4cZYbE0m z2E`eNOd<}-y;UD4B4Hrizl29?RK2V$*Z5gh+9Izpx@zt-PeGBFI{=EUS8X9 z*4a3C(^l4NMxs5}m^ym}&or`5W|Dn#fc?r5|1SrmG?-Nk9M`Egnp-)UvM}j6I4M{# z%bdyJJ#v8Oj6+9*gWv_HO`DxKG^=@jIq5T;Tlwvu6URaE&NU}aI!49JJ-t&&qF10n z=JD!#%k`(Z3G)dW@e4A{pUz~ebBM>o$$l2Iipv3pg5As~4#;XW8QnM}x1!0|<8Y$* zGn-$Hybqj=Lk{xrFll)>%U@~KU9d_10h9TO$(DN#3e9OUInZG5>JZW3pq5Z0cBO&6 zJcw7P;N81_*B|!WF#UCs`$5^Gh9%95I9l}FCw#3{xgOwZ+JEz{-EmvCf?C{T^o8OuAClsb{ z-m6`~pO6=AA5j;&V&W;rxco%Wf~4S4QXkvgmg6 zjnoWj&J&#E(mQ)+W`3VX)iWo?77kvH0)>=Qx28(DWqsMOQ1e8;oNdLOr-e`SgcWQX z7$!2gcqSZnZ1};@Wa05x)~W2yj*p92xkZ#zb|gMd>z#To+ai2n&{N~gi`mmU;x^6d znxyGHMI}(ll}mJ|NzRT18)v)m+~1L0vOqDJjo)&GmuuITp0(3@thW**rxix!HG?xV#Pu>bB~wF zoE|}$NmBzeB+u{pdM0}6)G)J3rWS6wiUy{MG7_Hle-*4&+y58ZvOTkT)L zf+g%GPTL$=85S-R(V4UGFr)k|$0Lm06TWu02?14WKKpH98j+c$w>(W5T* z$3e;R7SAf$WmJk68i~1iIJTH>NJ#EIe=p{+iB^o9uBq0v8BNcu5|f=cRVS?Q?UJ3b zM3F0uM3%ee?Oju^5h(3s3s}Y>`<1dhHaZ4F|cLm3R*y zG4R>g*Ex-+UHp_!A`7QY_I~5@D?apWxwtCgQQEBBw%o3{-{Tkcs7x?uoqExY%d3D{ z=8OQlxy}dP$cko}2Mp|%GKp+Q5B7<0JnXSO;-J?h?Z>0Fqs7DF5l?qOn;_3ZE_IPc z?!^+V>KTsRfh9@81SmCJvE+e_>1#N(>e=7K_MkXkeCEz;Nm2b32X>2fiu6oFN*ECBqt9_H4V_eA9A~ zy8H+&|foOK$>`Pm;2pGor5vv1PvuOD#Z+~CM0wBkdX-JeAM zMvfy&VvL*tE{jBt`mkzE=J>2ZcqFiD?`U{5qk)l8gImclM$C<+Md6>sMti3- z&PFQ?In?f46j?sO;olRN>kN!*9?GmRaAaf=VC0y<$Hbm+fJNwMbB4uok-Q(wT9X2~ z9?Gs@tDJE_VOC;i@DwMvT!tf>S`+6eFJWMoSivY0!oZQaCb9BYX9LSK1I`$i^^+

DCBr5jV)=;4>xsy@#gv&HL!48JHybVo%V%XGwU-eHjyM6y(nOJx< zTY$#414m=$8uF%xIEz=keQh$&us1U#S!#vFVaZ7at$J@1#h2`GQp_vtwmNdaf1jn3 zLV#e0oyKC$bcdr#HJa_AM;3B4Mz{)V1u$~gY-AJ)XcT2rWL1%PBvPw!a2wx}HEc2~ z!d!S{9NDEVu<2}MG>~vH`S)Z&OBRC?$Hfj>-Bj~VcERb&x|tKPnky_pft(lbAtou9za zEU}}Jf5L@B?0y9;!gC&M@HufHHBhmgTcKN#RpN+=QesEuk%j!~1rBRD3|Kl3{A{l~ zefy;UKHcsWG7R5m`tv9~?3uCqAj_44BW^R@|1pRLESB`~II->>a|5e@_ad$crsIzp z^gKM2_?LGayS#W|lZ2%byIMozabpG!4~s)W1v^?)!*V&L9~_h{mssScUXUF-;}Dzg z>J$MVb0waTk1M9^CHhz(SkbjHnr$h)AG)TLJ^g2gB5?sE6 zSy9eW=+0`^6YGR(*!gE1V2L^)sF&Ds&4DlKh@gxk+b;*HHLM3*9<0dY6`!+^?Z+Xu zGKN(y8&>MAT-ETPddHI+63*|>EH^r%!s^vf=XUt;vWBT?%eX}JKYn{YP3HZyDhAGJ z3Y=9AW)Bt!U0cAmt%svy0p~Uyt~mx=+Y(Mcc_1`tA^((<0^bt&e}>8qYh zo58@Tkk5HUjc>sO?p+K#2OI<}7S#4lLm8V-|gOfMd%7&P5O8zdLZRImz|O zfh){`HN}A~%7HgD;k(ZR7Kx%@s{_303VbUL3RpQPX(()$QDFYJKx7dUH%DV^+kr3z zCnh&0hB^*Mu@)w|MM_$Wo@hN{PJY~a2dN#e*<0rFKXb9Y7~o5Y|CmCt6C^lu~5wIAZO@7$r*<-Z}4(|Yp}le zfcM4%z9xllCjwY2-m^t5Fskb3$!1`i~v;$X56O)(6Pf{Q*01>_2JtdyLjDCC!*F5q!$o4^p$;g0t<^iFQ z2fS*DLQ53vE-5`J<8}!7q+p?N&g~GR-=;#IMO)hz*tM}UmvAt19bgP|IJY2}Rb>H# zO#{2qzrf#44cBCxSZ1i_A7KzL_h)_gKY=;upS(2dRcuBnr>kBCz71$h`!vI}5lTEhs;b z#46{`8gszkAb~Zlfm!0J&$E@x76-WU6i)>(iny%j4_nP5b->4`p)lpZj~OW(HVt`~ zn({0f@?I=skZNbJTEK3Tz+utAWYxgCqJjB_f^fzG&{?4^%E`uz{5cCu%QV>79QBqc z2y-~HzG4vlH?L9Z7emh}Cl-fQiZ)a86&RQ$lo-FA6g+fE%&}4~?!nav28A}}4tP~oo8W?O2BuH*$?l~aFbx?qdkvn9e6w4N$ z)G#K42TV1Ond}a*wJl&?(ZFldzU zbZ?>LmhaX1i9%ry{)t^+7{wSEKO7ES z$;^MJA$`UI_A?839vqO6NpyHvqF!`B@SFmxj{<8#1GCNn799pQrVyvM>)H7hp6{9glm;T?bPcj ztlJoL4hTM3z-;$`!z1xeQvrX90$4j68L{5aOFLed3C@=@1fwWgCa(b z(msmZB`Oo|FXYQ}l-{6pQtsbF*(2Abi+E4wJdiBP6*`kBWF>Zeo}zfpL&*sXxV$W- z*be?{YjHf05kBD?gBSyo*aHV0MWL|MOcJ5<3>L5|ISNKJgnOnpop0du;T3q*zrm83B6e~deME3LDBOK)`T z1or+0_MQU=dK@@6G#G7X;7VI4mUU2eUWJ&;K~9TCu{1`mMGAa(-t$g)z`N=ITUG+A z)dAL|X@?6I@;!6dl%ya$1Bs9)CA^4Ah zzs^Chp;G;Z5!0Thj++=yUSgEJwNSETfsmHtnKO$Rms$4N7U(H+&KB-DXmHY??!e?N zU4~QMuJ$;|8lS*&z|cE;O4=hWvtvyn+=Ny&FIcjsB^RNbHnP03H4_KAHel@zr9LJCroZv0{ zfa6!gW1fXVa~MTqqQ&nmls%#-GtW`j=789ohyVQMFXWR`)YjU?XY^1&&QZ7^k#A2q z-+>=oO$WG^E$~0_+n?(wTh{fRiDj&8e-=J(5cswz1U`LTqX* zN(yWojUsuBdRD9%RSE1;yPc~QHhsLNf8@bC0pow}vs%C`w266!^v=a;O1J$J+DEyD$(}NAK{3FVs?|o7C6XzGm3v>V6kfOPMvn( z^~;5?SH{a*u}!dkGBNrbR|4mghPxW!qCXl|zAq8_wR@h!0p`4Y-~J`Y9w@)ZJmX1K zE(6O9m20N6Hks`5{Kvwnar$!XfqmI~7qXYJPMdpJ?IG{C1tyA&Y#!+z_ly}eWR|EL zU{?tcJkzkwvS#^ON0}oBrPeU;YdNx8?GP!*dMU>!UZN;-#8G_BL)n5xHVJ{_$Ja2w zGZCMWq9?gauU)iS>9zq^@<;PyCyy>jULeDs*U+ZB_*AaRIxdZvfoc<4BQHik8Ks=SoEHRKDaDC$x}+|g7v+G)G(gn zM@)M%m?t%eC@kb!@_@_aA=f&E^&H^nc)+~v0Ee9e_cn&#y$6^lIdIq5{X6fd zz+Di>mE`auc>$A6{`2hfY<3A8Jq-O{ZJ1f66?z?zT6sY5o5P$rjei)F7+F|&B^)vs z9x6C-sLEJ2FIf|`)P70H%TuKqA$l>Pk3wcCKJYO1EdM3p_4UlPqcJz`Sw3#mh-{fD zpti=sMDzL-DVwY-H!dc#^UFC^elfVke3DgD99i2H=Wq(_2vBflSDyG!rt@QC zGw(lX?WBs1jqEIfx-JeGnV%MRPvdplmZPwVm8FCKN^XM$)1kKZH&^ZdZ2PF#Hs!5H zPKm=MMrJl{)rb=Xf-UD3+pe6m^iko51qaxKh5R}g8aew+m_1ZaIc(yPZXx{ul zsa1O7iv&iwvrB|dD%C&O)N6YAMG~+1^N1j4xsx3X3M_mTA5Q4*Jo7Nv-}=UZ%K@fx zU!D0K3=W2b1RAm~39<88I+Z`>Zo`vsoed34(OFwd!>4%MNWC6ca8~uYVA00|M*`eb zzPiaTclhKeGS?!^J8A0|CvV-w1)GjYw%sY};5FQGK$6$(YC^L}$N@uE-rYMlF!5?H z_;yG@#v*{>A0xj=LL-}-g%Jn8<{E<*MmZkC7IwLqg{&ODE*}{M10^oD2rgX4)g-t? zsL-`6JYadNpjpbsqwFOaA8*RI#cXUB41UviRAhq2#6}j?DG5#7`Zp4es?@G{)T8U8 zF(JT_)6l??nd59ryYV@VU@w{4l8L8vUap$mE$F4QsF|s-N19V*N66tm)9X2xd9BVr zSa?YG>=K28ENs;dCvj!Hxdn$@FU6b8NR+9+ zzRb1j@yRzwu0u9DFRaPy$ zZC6&!5v(rx*eKR?JDKg1$sCLI-RJ)_8~SX?uKb$w)t;9=3Q8wN{x zv=S5=IaKc`H1l{(ahh+uBc@w`nMv_LgX2B7fQ36QY9t3~?2cK&y<0v+kW-jh=b_Js z#s!!B%y-oM;IUZop`lUs>;?u$79kS@7DdaqiKhPck+xF;AHCl3I9S^F>c$B!S*0qI z^mICzxC++Zxz<;pS{lx8rtK;&6!mK9X5}jhY&NS83cXU`OqV&x5zlc{%jF`Ak^p0< zuZ5h-xee@AnMtfOA1d%>HT>Hu7tqLNsG-Pi^$4<+d4c>N04DSD$F&U6-)>Zbdrly^m8wS0*3zooiq# z)6mGHBEZO_^O9B1gGuXG19$oYCr-|Y4ksyx2`w5ASh@?2Ns2Xgg-mD?*}b^cwQM3c z|C)nBJ{t}QDqL*mIl?IZ;scx6j*lKCQ<&JK44jRl0)z#14lqj|IVxAPsaeVZ#KF6Xx}37kAL-NFg>}+l%2q$ z#dAo~Y=*Z&0b`f)0R}c>g#?8u8$13Pe^C^jJcEfRh@n~ZhYOdDX8(c|gRK%J39NDz zj{NNgj1^X|7)5{Z%CZ%<@Rc~Q@=jo2kGs&w@8QUG#jwe$HrSQF#9?9Bl?Hab39kNy z49vDQ3~T|Ho%t#rEL!2%%T@M)fwSWatNw(8QU!*(S9Ki~coaBf)DJM^J327xbhv1y z9bie-U=~@jfB`FF;QUR ztzeLiabf@PQ-#BobH1j3q+Qr5(OQRDT@o86i7GyE;P70P`X{Vx|DJ2C>C04}O)C*Q zBh{MERl37fD50RmZowhRhz$pMewZ|C{S!GP*>-|SGiW1=#1RKJg(XbVH!d*GzmTwH znWnC_<%A|#y91`I6$hAiMYOy7ENsz^c$>v{RI>KfHfEU<4(^qwZ+Cy%w)mgU4QEw` zUA=8z7V%j;WDaST6t_rVQu1x&E(%~yU$RazUBH7WMLd|r$3cNhCGv7=>%;yVGU2O9hl zTbPSES_JJDoaf&2ZN5xIqn%~PtuxLK8=8ipK>IaBI&h&f&0n< z9;1R5vqK;6|F2T!H!x_FoHVg{X2V2=QUzB%j)!{#{-tQxNyzuj*%W4&aV9zc?~Je) zUf0(*{9xI|;?$|zmK5K2UPJfZ%($#2%{l3_**$g~QvBAyVzDEUvv^Y<@3w$;y_|#m zE^RJaYzvurrnt2PJ@DnNayC78gJJIPkh20R1}xWGzWm!NJK?axJ|1RKivx^Zmon0W znkDtRd`)woX}Dj#6L`T_WT%cp9@mx6cQ*e0eP91ZBTq#_mxGH_Ou>UJpZgm3S``?W zv_3Ejv+Q**i*UK@WO_(YB#_Nb(}D3%BC~POTHeBpWTEVUgZv4-jFI9C_$Kc-s+=U^ zwQ1Lnte8VHeohHs(2yC#gBe=H9RcI*D;0ppOVH;?rQ;D_7Y0H zwFM16$z1~Zk}8T?L0gzi1ULgalFRM4w&`s!XttlD$i!KoTYu>TPso3BnO>WA>7ofe zMj8$*0uvY*pA~dOi#&QFc&{|_c=*2M8;UevN(<1NY_xjtPM*hXP$D2(U+dU|-?Dk&w(B!eD7D zkUBNoR7;FmcGjPVqf<_62zt+qlFqS`n4X)!R%Jdl-t5WuY(sD8y+Xscgz ziaXu$l^4Jj^?_YuBK!4VVcr4;qpJ+}pUVGZV8|3)z}T3; z@#X-pLJ{vS1KuBB*mDIKkA({hFW_|l&ZD=8H~1@GtpfAQ<26D73?It5CmJ*gig_tM zPO@;a^V%R{Z;;~{lvD7O*)lw}Eqx0@vmPto{lz#TQu;JvgijoPrpl%pORtlVJX(#;Rw*Y&^qkMPVwd z0;6;*qv;2Z{sgJ`E!D-BrL_c@XQg&0q%j*@U{n>6O9*p%nIX4yq5u08eh(f>@0i%f zwKZ~OVC;2O=8yuehyt4(AK0T<8Ba{FVGR`6cv<(VvO>)Q?i~VLuY|bnPy5GGae*x` zT=>o>rkMqt|0i(%FW^4k!1uC&-Lil)Re?#P$Tm!%e)0u_uj(S@=AFMLdj&NbN|~u_ zQ1ND#=u!?#yujvTxmk6Uh2cHnMpFfi0JcaQ0reFDzP=w==Uh%lV5e zwFTTQ8yuIKD(`;Sw0B{^?nag#_m&k8qa+?kDYY{42QW)s=`p%c`0t9QTLz0U1GAez zK*WLRZV4;~4NO#Y2a&Ez;v>J{i-sP@CSzQ4;(u`aP+Gx+bfEsx^~X!=yXV!>lh@uE<|PSjn2eI zX4Vr-HKIOTCz7JgMRtjKS$%MnR}q<`+IZE(!Q(-zZvadD2F_g{xOY8}sB__-9l+vJ zz>+tCt6qUwKS5)UBJ=AjGxj7(nlLR?Um_(Q%D|Kmn6;S6%z>k|VVd-GsgMxn;07%> z2NrLE8Cx%zpT5Xa{eaQzfUUd%XXppUY6HtU19syDtaTq4O&eI32D1J-Qo*UtoO(?% zXmPsD1CBfe_N5Pa`WhHrrVI33v$`ELVZt(of)6H>7-knUaIKucYTUqHv7tRil*z1+ z^`FUwsWT^VpI^ZH@&SiI0%z`qSqWDdLJT;!IdJvQD0Nbd5dF+-!=!TEQ#7Pe^#60_ zB2h8(0;aN7rb8|B!h>^Po)orTp#JV8^M{whGj~awHn4;SaQQM!UHpK1-vhpVp!4Xs zwtSFqHDI<8yFEQY~If2XP0()%&qe-BxHUqn@0qRQ`XSwTr*z@+bNVke9WAFRK(jb$EInr++xv;HB7N` z4KaJX7{o(cWIwRYh?ep=$-Jz6t-%LIV+UDDhP0CIwUP~MZ#7EGDfkK0Oq&(REVDpd z{X3id7PkBbMzO`9Wu7ZtnX3+nRDIy6WYEkm;F|oTCghc_@$$Bx4|TIQa8x{C%NJnJ zPSC5Jz}~ljL){>F$!USg1sn|x>@5#CDn79PkK!!kV$_mkbWGSi^OxhzW5QLJvc(tA z%`{!%a41pPdsAXy!@os8R`Ac*c>VXLca~9t$L6iKh%(xpZ{)Djb^}X61WSSg_ZkE4 zH40pDA6OrOj)`TNWx#4Sm6hj~^v>|KRJLuw#e4s>rLS~k7It9ZW2j!Wk-d(ABdmeF zvVnc3KvZ?Nw7~;m$qxtbo?`rB$*LW|?y-R>@-Sni1G~%twn_n(`~@7E2bdcSr1}I{ zj2PG}4=|dwGkUc#Uis?EJAvukF%7K%&a({6H=1;p{w!h&oWQYx;jbcN>IaTK29Dfh zy#|LPeF^MB2Lx0WFeoZ8Szcf(f57&B4@=$#HlGb_X#q@EKOYecU=D5Pb~k7YQ#O)Z zz0s0sh0P+vLuPwql5_68{3m*1^&YcFn@oigZ30&A^j>Lby;q*)c&ie#Q-IZm6U?%O z9OY|P%|F1Jcz|`!56;;E+{>F-yB{pQaZ0kriP^tGV}|1?n+Ys72iC5b$W*X^Y3dre z8fo^%0J$Io_R0iS>jz983z)x7soCAzr4y>i8*n}T2 zc?B@-Hy75sz{LB2NkV~PMe0KC>kN6evHT6Gea{$}CNO9nSYi}l%V|)2lgYjRwtVU> z1||VUO9p031!mm`mbwd=)ebPJF(@p~zUjY)CG7#5&jLoS1I!f<&Pp7pGZtjjzMynI zfpt+Rv&8G9y=fu`PmA~%h}s|W%Kg`BclTvtoZ&H@fRz_ktG|&R&2PGD^fVDVSr+IoO%e!|^(4Qg9Hutp28w=LkRHgGjukkk^CI^$%dPeFbt zo0WD$s@PM1g8=qv1uU!&7@8(&yxrm~CJ?1|fsrYpmzROzuF-WF1_l`eMv)0;Ssv(S zaxkRLV0gyEz|fS+o>0SdA(U?dgOvg!uOCC-wg=qPLoPWeK06bn-!9N-yVUq0gX#jt zAEJy4(@y$xDHtEGd%c{YVFJ(830#E?%#8}Xf{NL1)ERXHO3y2>&tYw}7HG7&kZu2g zIY#LA^TlBtx}v|+CnwBdPPoXlwS>vz=DdG%M7`c>maCQTU0<;>E^hBJU5$6YFD^Z3 zS;WAl;BaZr0q!*qxYjsu?|s1C|B$U=0@twhd9v4vec+8Tkbm<3gCl0!~J3d%&AebI*x!${hwdyH}ir45o!`E8j8j z&THdcz@YhnBS(Q@Y2O6)1=qD62z)%oB;+T+wo(4f_6bYYF<3p|5DN4VQ&8xe!2IJ( z%_>gDt_3{l0bEN9SW1_0_$}n4*ST5*tr`K?;d$8E@V^+Y#&&4}kAXq(rM$;o0=()1?56}ce581-1<+N$z=fy?SpV z-!rM$XGW_prmuUJJCFGyhg_KwYp4N7-A>k61+Egm=V32d{r+8Gx$%$Zjtc89l|F_4 z{ufjiUT`hkx;%x?fl)#s&G-S^iU6(+7o%1`)CdaTyt|HLV!=$N2X#yHYWOxpMO=Hx zz2Waly)TQ`z1n<_fp;QL>{DG~f#M~4j5+(+j=#}O-@$OgT;ONoBjcki=Mvc51$YZ4 zFm^fco@5ZP60zuDaCAJ_)IDWx$6_77Ij%)ZA}0wrHg#~Q@%cKXE^X_RGAVoUAn;Oq zpPYRa1LG%Wp3_DSN#~|Sx^>Q&Z65pT!OX+YIb|GEFI-Su{339%XZNKEfzHQ*m-#RC zT01e>jZebRE@!IIQ9(JKT8EWFs;h$44FWnOGMU^b8ig!nxy9`*)%+0aAFZ;7j;c8I;J>TSZa!@wCq8 zWZHzpHi;aDgKW}EBwQvc|2wdBg0ovCQ;T}e4~K5!UmYx>F$xEj*!V+sWN+a6@Q*8J zgXp3+-mH>Ln;Y5r5*9q_HN7|EpkavB4u_`0GpZe$tmR%VR&w7HUBqj;QEDk)hSu;Gjl2#YRxIoaR(i?YX;E;H zwe841zQYbo>~ax@h2)GH68qD`3K|%0p0f7!e-?4?ve~rAd1uY;zsqrG%LrmPkoHun z;s7J}gN|o>>g~uieEwFyS=gX-Mx0zn& zEFSW>Dg`cSZRVQ9!0hg`acRF~-_wUGlm1Cxa9|dXnXrsqoW;q>P|m03qs#%`!d1ue zCBn5NQdaCKIvv}vuvKi{FNeEQD|RRgne!MP?J^HyJj%8&XT_swJ{dET1ws@IkM?Wl zL_BP4=HHjlDmCH3fo8c?4a|L3GXfq7nm;R06SA^9a7Zk@qQJq3b5p$nv#Ltr;SRpO zj%DmzFP3a>{Ks)6MA!6E16!5pg2J?^CgQUMl`?O8NuOYp%vs3H%xJ>E$a~^S!7}Bo ztT|qc0ve90I*FYbFAgxNWjL#zNm%(Mca;LC24kyV0TXM)0#^Ag{BCt$8yxR2iI)V3 z@%}3oO8u$8A)0maA(Qz+j+;w(crPVN)|_ybXWPh=rjR5#@6(RuI}WfK1tbZnA3q|o zBZ1W@;vm-&3s=c43yS0o#6=$obgHsE5-aX#)^aFp5<9Yh*>lDbg$x5$E1N@tcXu3B zxN(7bPsIa9i3iGTNeZmm7aEuiE0X0D99y{$IIwzdVATs~WL2I38r@@HlWAb&_z=kI zzQIL7=3~2q$>B-=7!=O3tO@p-CfH@UYR}E0yvGd;n;QK@W;iV7P-tKhVPH03P{>Ny zEXC%+%fy^=i$y?T4o66UV5-6i)#e7q`yB^@ggF>l)E;QFDljlz)nU9^^Hkeg=#kjJ z*^D0FmNCn7yvVl?aN1n6+QPpf7vuf!&sx&v%yPqc_|1a{d9IB|!sXynyuXf#c1U>4PA z;OYrr(kG9*Ui7U}lurz+g3Z zd7$y?ry;q)SKXfKGp_m9lzJiIuu+iEaVFLc4y;-W8hK+B%#D;6FtYEMlbm%~fpbCj z>ZsHeVS&0V^A_D&7O}xWFr%1->qDrcivoi~k1?O`&4cy7wwcSBcyBUFO!w0WVN*Mk z;^cHhUFJ6R;?h`P@U_7*91D1bsQ&62yj&} z3TRZUIIz$xqFLmVLs!TIC6Qu-rdW%DMm~juOUgW%#CBY2=Fd1J)Z%eSBx^#8n#93a z--071{~WFf@mytc_U?&tJhm^$DuHQnWB$K4*|Wba_`@&9?JFqCtcyazR)0Y$MSLZm! z-+rRSCBnwl%b1vuaNAMgA+x83i(be@R;308mLIDeafxTh@XAMKExY??5a!y$( zJlnATSeWnOf3L!M7c_G{7jQ0q)xbIN>A`=3T1QwiWM6Eui)rHLcW0F^Gdt0tut`ba zBD<@Cs*#zO9>Gl#fc85o6b7;pxOBpr;pacFj~0(0-OMFP|BH)|Fw z?Dd?JELXguXmj=g7UhJ6yCMvZm;`M6G&MS9U!@nX_RPl&oHK;>ZFzH7T!GW7oeZoR77w{!m0Ml@{gzd?f#I6%_A3Po7W?jux_o#C+s7MC zZ?gp-SY_>6*3{7KxuMlvKw#2^Mv32yY&B*q2Rk`$>L{+VRyeS&+NwGC^X>zSTf9CT zP{~#*KRrpub5HSMq4L>lk0sAI&dnpv!z1UxWO;*GAcILyfHm&Hzt(t;y%#<=NzGvi zHfVKz(5!cZS@#D+Y>+nphi2OqO*%6i3#*#lcd%-$Xfh9Ilw@cL(r9%!!1QEOYo+BM4Hj`Ln#;`UoyXC;|170gwl6SDEW^XFgwmBJnP^82rK4$xY3yUhen=BU` z)OcYN?{O^UO-tYfRy~C#+aJt93ayHFnr#z=m=bvS4VZK{Gpd|mQklT`@L$mmyJ!~C zh~}V*Hg^jq%LbO16>QNBjZ-Hx7}YSm+^j#_@UZC~{kkP4b60P;$|1rf(ZHp^6nLX$ ziTD1m*`^W}jcOhnd>fkdA2732Z(aJ*g!PrkvWqA9l85^O7c-h6?ZMSKEP~zn#Joti=427Bx-X^ z$GiY%o)^0sxPLU5&uH2m(WrHRSv8=^tD$jm5u3-22L24DkQr>LGa9{r&3I$j|Ia*S z_3BCbYpS+PHSGN)(Gk8=|KSSf*Q*b+N$k>lz@i^;B8H)r{{jQAMI*1nYPAncfghUc z9y1HvXkcXEdp={U>z55@-q>d;Z4?k_6zDjT&vh=_s^4^mQ_zD>vFHQSN_LlZdCJW8 zH2h)sGRIYEF-uwoTc!kikwSZ+M0>`Bwkt)8RFt+KS-hyEdrw(#;|cB_k&08xpUgSX zz^HtpNzb53?}tr1gLnOnR*ixt{|l^s513OY>~%6=akgl&+A&>m1p{9P&x=P5QXPyt z-!#bmXqMZ+z;}Rw*Pp-i@1_s_8OS(Gt&6(Zn)$H?Yk^Z!W`p%IH z!Z-f&S>W05Dx$&4reTxOi6z1Iu?g)C8ybW<_=OzK@=BNnMp!LNUKy1o)x2SUZ_1J0 zC%b%CF3UX?l=#}$n`gIX1&gEf1lAM>NoCjBC)UjRp*ZV|nbM^L)+d;K64;75*sE7u zu3iyb?a^KotYr5#SasH-B|2Uula-dOS+ugm|Hl_^xf4yc9Zh;OSkHfMjb&)o>tOow zyET-9B_x0~=s=V0fhL^~&E_kVBwtLb=w_75m?+W7Xx`9b8o;=}<*Km<>-3U1_m{Kx_Nb{^9Eh$^ zoRr~^sy%_JQGjFO$(b=nt2Z^DZwk)zV6XbXUi*T*t|FrDMSI;1M)3tVj@`Vh{5V)m zYekcgqp(RxYgJ=eGUL~fMyU&@G$dG~EV@54GyA=0@x9O*dV?kAgOS}2X6FUWRx_H- zS1{XjFxxaR+q__Q_`tkam(_DaOIpEIy93Sk2be7%IH@tPrZco=y=dMXwbS#W%jE{U zHFI)pSzt=sdl|aO4QbgGSE+6;%g6$=Bh5HQP8&u3@uu z4PA56?8ink2Ns_WHmkGj?X@>*AGBY)+rChMNpeE0;sQqT4YBrT?-swlyo@KZHEa9X z#>V%zn3Xk{{1&i={a|_emL;qsB(Q@yfPpongC$dj#i^pj)8kf5H+?)4t3At0bCy?G^rDK@jG6{c3NLNK6I#PQu!LPW z82h2gwu7a-gzbq`%iOI^S`o~;I;`#u&6*ZWniHCJCa|O%uzClyr5#|Fb!h#{+hTDe zBz*>($AyN?Qis0==`VB&TzIH~r=x+#qmkQ#k$*x%kT0X;fd*BtM%foK@&S#kM|6xU znvD{eRW~#nXEaH6gnjwUTHnC#xI#tvhs0DbMiC80o)rzDFIxOxZ0DH^X=*9K=vpPFBtB0OZH&!r83t+Flz^)RO zRGyc#Y{lIhy<9h^a@|z*zq?gQ{KehHIp&fb2W?L@)jVc8fBK;DHODB4Znq+~^Z?e- z8O(}58~#NwyKi``+90TWLBr|*Yx)cptqIKL2O3mYw4}diw&!pTxxp4{(6lEpGj%1S zRB)4N0F!w`^DMq7haJt1Kbq|;T09C^JXU0QG_(XOywWvENfmgK7LnzcmeDq??Y&Q{ znuUj60aIB4d+dZ}-V+UWhZ=-_yb>~KIGtch-@xjzqLJqWgXoC{yNnmE4fnHlTs?ln>E*T-T^rWr zZ2XxOZRsc8WxRNov4HKvfxP8quTmmjd1$mQ@MCi=Xz@6~?0%xz`bD!bM~hNH!k5df zfdP-XJbHK-UJG3a6WhU{zkpTafw}UA<1AeaLMzn#Z>~OeS0w)MC7ssS6PARFy*n!u z(Wq(Cn*T0)-iw03JvxPAH!ixqDc;s>W6)M{fW5u-Uyk^FMzv`AYhYIkFzav|Feg;r0C7Kv}n4i7XvI#>*6 zvBh3!i@v}V6L4I41*60ZhW%GUf?lls{`P{eE^AOhvB!!Q-I5lsADORM%_lEoHlNTu z(~T*$vGKaMqi_Ht4+jrtK%-Cx!w$nnp@IgUh{+rrjXXCP*!xZdMg%!#7&zSzbGy!v zu#drU0ju7R7Ci@_1|yibp_0GwU4Ep=yS8oeV*zvU4_W_H>iF}P0O?n2}cE7URw#N&cU=8%&%4=~r zH|Uw)#1F^s`fAs{=4}6PujW_wre`7^ik=TRxfO$X3J;v)jcC-pA-td2CxX4k;nDHD zvS}PlTlkwM2ymV(OHyC<=H$HCihb$5bGgKKM5t9TTNyA(&uI2M(fUtq29r)klUhTE zYD9}&!TskyjqNwI^|fW_aAa92v^>;fak|0eG=t^J?3UmUjUEwE9y1#499UC3TBcrZ z_%_M=H}A}UZ*TqCJ9)ZgrMI!_)?Cr20+oyw{VW^{ZKg!)Nw8`>Xo(bH5x9}Ro-g}~ z4pY2^l~-xdI_1WByD#l1`|xspU3O5+{hnuc=lrYa4d|m>j$>kWGql< zVG@c|Jd@gBX`CjgXj6ahkL3}T3(9`;`3j?2SlpNS?Yy|~aXX)y)sZ{vbeHi=zMFPr z2B+MeKFLSrZ%mTh*30j&JvZOE{#={yY_seY0h~fd+yoS@_g(&>SK$7miIr1IM#H)N zu#DcF_Y9xfE==Kzcy?pbW3@?IK`W;OGM`jxTk?oUZ-U1nCU!Z218)2tHlI#N%Z40a z{3ldb^WeCOsm;C4}F?K{TQu^6E7fftIJCEYqhR zJ-}jbbfAevc*z4N7WXZQZJfo90S6eQYYuj51UMLWtMB^rs9Ph!#?eL4wMOxj-r|CX zuHuFv4h`Xvtqlia@()R>Po6(D?0ULVhXK>Ps!x`OSX$;3os`=4^1xyKEf2%mB=%0d zm?3$HH$0}W?9RaxMqdrY=Po<f{x#c;Luy_{HFaLQzSABfFu^ zBS*P@kB9#l*a}t_xM`MTTxblGO*r7#EfJB>$Sh{_fq^ZgQ#6dl{M~MS7W;QQl0zeW zA1E^mdN^ou7g#7YE1&du+%eB0ETB=arr;Cn)Oi&_-TLzHgSd6Kq}@2&#&}&}#nn`n4RXjbZb+}xQ60a;il$m*zy3hm7p93W?3U%|1@-JW?47hm~bQ3^`?FEDjuG)4FiiolUaFA&K$L zKI!$WkCkj5vm`YB+xlyN?9bCvPn*|GJ+7*H=V1?<&5nyas+t|5j*M-t2iwJC93CD~ zUtIB~+xi{H;dA|$V25=dhNEN9T9$n>$1-=~q!^aBH%+yq9Rj0G&qjbry7_|%~IvRn7}uZ!WEE^WG= zc$9DY1xA?_iA_u|9*9(4@M8WamoP(5g1>V14;OVCL!+pOm%;%Ln!fgJU}TMG)rv`B z;7kzP_o0%_xKz_O%Wc7|c?;8+DmSzki#X1{C8KDbdZBl*f~e->Esp#_zvb;`DIZ`= zThtJ_`f$mn#j;W@XB#)AJkU8FEU@Wm0=r|xVflL#7>_h`a?g3dBC&{}Nr{C?^jtuj z@v6YCAf8A3f>KA6Upa8hjQK3sE^$P3B0UOB6%$-c^bWTC_^S}OduROOy2mH~ zO0o+cRamO&zmwl7g=xh~l?BT$O^}s-!w}}!pvoK>dOBofuEK+d2c^C}V7p+su&P`p zQhlEYtEEm-+a~^PD z)Y;IuC*pmI`eUb=W*a%}Y8EbH-f*Dr#V38X0+a3SE-IN-t7ln1%=BNb*vQeM#F?d% zZ04bTOHF*~idjxw7L%HqWxp)AT>4<~6uZ(s|$sX*OMc%MZ#5Z&VsH?lg*ObBv{0LIkYpMKQ1D4>X6(EFLq0w zMf`3SM@-T#vO3BriC;U=s(bHaAFJ&HCPR&*M&1wGHcWHkEI8;;aZHu*##GnE_EFv} zLJ6V_2X?Sb)etlF3SiE%aALasTRiMps)X)=YgKVSn#6mKw>!8nvPsWa%XFyec3)H4 z_JUKN&hB{QVY^U-Ro&H>|i{wK(ZjJ6z;(eTZ*d+afJQ5uVeZsYkI2At_3hsFw_Q`=W zU!sL~5d&N7k0#j&>u*avMC1^<#XXA_Jbo~cqax;>6brzr6bbZl9wkeIg zNs->06c;dnc64!dI2up6z`)3}@^<9K0}M=tyrF9rai%l4DugMv^X445%hez_LE->| zdsY95-Df|=6p1ab=l*VzC}$z9b&<#QgxlHO3+4Y>Y08+q@DKXy&3A0;?z`R$^+Gdd zu$(ZS*y$h1$RXsj>eTsUxvwVs6*hh9E6+RH<}C4$bI}D>{X2~OS!4pYXY4uJ~_fm|L-ltjA&)=cqtIA<#p$m(%gz~cbxfezMxGK^Xd zO>74ah(t80tmt4q;h=iuU_``%yHi`4Z5&v(G%ELS$6S!@nR1Px!+X}vxe+O@EDV!a z=Qzq7X*AY3BpxuuXoVB2#sP)~)+mvl{s|6Q9sJ*KoXN4VD^8g#o;y+whhDFY z%)2n}Fza!NCvtwW$7K}`9ZkJ%mvLvA!CloEPRC`NKFY?Q)V+GrQvUSPvy!viT{s#A zb};aNXb|N&B<8}Xo+E15$S7WONTWAg@GsK09#6f%m+td4Od2ki#w*=m9%h{F<2mP(kxxjz$(HZ6u~5SfFq`dHN5BO zKc)jmB`usdr?B2%dz6{w0CPwrmxYtG2?HC4L(iJd`%@bx&0$bsI3y{<$fI#N=iueO zBQu^GGH@+uNZrF(xYtvFYavI*W982C1*~@jqmntd928QM`FrD-&tC`0EsuSl-a8U_ zN>=5#)eGlkKjvLEV7`3!kZQr44~(I5KJ2U3$b6A(;o)fE-f+iZ&O^s3;rts8DtH`R ze6dNrq)D#T@%W4DIwxjrvN`-)(n-a^N$83LZ_NRoH4eNt4hX$*R8V0QVqsFzaNLvP zz@BnIA?YA%4FkJC1Ir#)mMINvM;eV}8reb`wQQVtX7q7AIk-(zI`YF@76FDB14p@p zJ~p-654ZkXv}x->z7Me}77Y_z8#pu=m}WQzaP;nMa5B|6%wuryz6O(k0|WDu&gWM> z&t19wJ!9rM3!#FQazcyT_}v%;9Gr}%I4wRBSA3W4-+`G+B-oF33W=Gy+ePtSew%F1 zH$R}3OO!>**=ULLGIuAbCyp8_(zhJ=&m5PkaS-WY(yln9en3t8gOh%Sr}zXX(-3E_ zzm8f9WPg8c6y4(}>T^(zrz!raQ?n=&_sj;?84heJ2Nh!&g&sPJdN?rpFn|W}{xE9& z2@(=$k~zcBd7zs|fiXtqK=8J{Sr!cp2@PC5+>AO5>?RD1F2@X}xw7nW;M&02!x6^3 zq#2F{2TFCRvmzdAE_g&Y4Q z2Cfxtucj{Mw_(y+5_#!Po5Y!re|P3B379iSvRyRk(EmF(HSZoWE_tY1A`_rE;nUP` z={B{DT@M{pmN=WVNKb0#IdNQG;}Dw&qxc0T?F~-a9-8_;nDjdi$$eq`73!?AB>0~# zlimy`SqTN6D~@_UrkcKS6q8|6@p0A+Imjb%Q0zdIP@%Kc9>!;TCUbshkg;%JUDI*< zC`0Fg2CkTfrY+uF2Cdu)2e=g)tBn}>*Bp?#aDeMYqsSfxnGKA*8ywX<9Hcdz_(~M` z0~#0HaNrMY)U`Oo?9u$s@=qgE%oV{o2b4m<=k978mOZdaSy1xei)nK-gwA~tYb(-7 zXk+kVP2-&6AmGwuq%vn#?xlr$LVP_HZO+{+>=9zfeYnizu$>w6<-(W~lJe3q2elW* zRMn(aalCY@P;>TRIsC_w&BU?xuDJdVCjAeGG(I#c?r_wyaW-8dueIcmipL>imu8tA zcUc`cX5DiVPdTV^rcq@}vvEU{B8QX4i$kUpnDq7>RuOSv%u$F;@#A56&VJ^gvB05| zQKyYpo#LwP8}6<80z_(DH_pTt>6ykp{Cl&ayVnR#zI0_cV*dI9b)iSX&%Y zzH!)Cfzj&A1xuS|mN#pZJN|Vu&0)wryUN(bStX-^$>9Kpk6T8sh`=oW*gvN?@IB0U z;c2>Tp6R3moF^Pq7BnfRToRV!Ex6-<#Pj-*pSemeu2|@_i{*)5u6?bWaL7L8*~iAN ze!e4)7V~#G%@^39{ZwwrnkxyYnR1q6-a2F zXjFdSB)>zJz2<=5okNOe9K}?Y3obciwxUUaqe=Wplir=u~6I<~23b@Jm~iOAao+AH=1U&}4Ff*)C^#@ts2t zm*qd|Qq-QPSm7f7q-&Y;DXvVu%rz$%_%}3W>oCgtIBPZNX>=S?D`1irIVk6GLOz6< zYvrNeQ=8;N8XK>+2wq_jz0w#hcu3C2N$fy_ybrVH0V(+wC%rp|#8M8*=p60Z+QH?) zz%Fpe_(CJE$zho}ue=*h3#2rO?l~Zn(Zv0NQIv;KI>U+U%K=fF2DL8-xaKTV@n&S- zaX>_(MP$tZt}Bg8(%jj5P6@C$mMv-E|KY%ykp7QzhJ%btlk$#6&K6frjZ0Hw-Og$V z&AHL}GBq;((Mjc6$%J_aI71F)A8{*wnwvl4;3bn~HuDS)VARoW;M8$6p2KAPwUNt-fhp(TvD}x!u{Vt7-TB1Q3K{{sc~3@KN1H= z{h&L>&dpX=obB#>E#|rZq4jlnqgR>2Eyr~-{N2W_#w-G-z4&)DDytk6%y3lrQrsr` zM)|=(&5E+7XK9KH<~F!C$zM1qe)*uxjz;dZgT3;H6mK|eX~^O6ag?udICQ9+c}fGz ziw5qP^t~P4+RtWnsg$SWsFt06F50@3O>KLKM{AAOf%<#gF+WrSwsA!Gm9xBYkThww zN^p>z)1>->fk)%8W`aZ5i{SNbVX+PwKRgUFx8xT5EjkdH!MVYq)5}>V=aPht;#<}q z|6a%)6=hd4In;Q)+5XAlGTTG;RpGqUNUbD1p!4-dMfsHtn;kWle3oT#T4s}} z^kl{92Hpdg`u`v7v~<=~ag=%BWO~C%>&gzTIR}MUobv2~)22Jg%OnZiXqcmRP)_2X zlcK~Su?q)PLLAraoEvbCWx=jkwg}eE-WOlSTnu*Ey*HrpnTwlnWaU<2hi$^Qr%tv$ zR8pPUqnbVADBHB%!377nwsPZw$lc9Hv>0Ai;(Bb6#^Lr9~4+xg+-JmELd=6rBIR5K?aw%Ewj_g z6*L3{nVdM-6kU9NczjySA*AHe(-7d~(jutrG{b`N#3WrC_aiSZC@$;iGfJ8<<7eWs z9?7NN)9-Cees)gQ>%x_ryPt!`ETeapopsR&zTs1P;?+(jl@ol5dKniIUMH=M-kG0z zpj6wLBW|0~k6kMqTX`3<+dN_Ey1b-wvXZNri)Kp3#LZdfyd<^0c3e?Tzx=D_gTaww z3!^XRy}f9ZaehJZ(#0k>FQ;~kIBnk~azNmrS=${|gC!3RJajC0c(~JEBcNsnGv~qS z-V-Jq+S$T+Zn^z|f3k{A9M5H{0(Kr{c3@`YkTytIVBmP1BW=n-i{MrLMp@~<0(J{K zE_9lx;OwIy$>cC4LTh0Q2ZzkV7BLqF7ooW=RRQhtbqSAFaLo*Q)w6to(P2Jojfrlu zRc}tJ%dg#KW!P_XLTfeO=1Uh`bhkfBI<33o-lV|wyRxP{$G+Hs(n|C_*B0jpINDtyYz4zh|kx^TLR z2W@+CNX$u3;{nqht|JZG?*$!r(DZ<*Heu%jDTi}Ao(R-FFy}PL;A4KqVfw*Jw2r%p zsri)gc^eKZQAI}985#O~ssCoZSS+Ph;=v}sEpp=g<$nqqng;~j1RgFG@-tY-Dx9)o zVXM5)3+G8`zdo!~U(lk}%(>`P#WyFOsa0Hp?khH>yVUyFOk8dCEFzHKY+Z#Rhg8dq z(=p;#_6FYF`YI}MZRzWxg?jr^ZZvXP_4?*S1@c^vLn&72)kX78M~u5BXHjH9pGH z%iB52@<;G?j;7C%mYf<_`Aida4t46wlsW&;pgqwO3jPq!yk0Bsa~^q+N?B3pj9lc z=3%RNtW}^;3-7FkBQrxx`j{t$y!g?~XLYaQaIg8dj>G--{}h(;OME+^Iz^8~(8Wa8 zZbwoZlf@1POCFsAXPMj+`$Bfi7LWb%%*pV^%=6RNa_>)D=z2hbg+G7QYA1`l~wS{&kBCCDz^aF&UyvtiAT$MTaL5?F3%IvX!q*b`#mEU8e! zW~p$o-E+wy{u7s?@7D#iiEZ1U;VCJz)lNjI`ri*{`N+&WVK)+3wHO|@I|?X0k73YP z5-N->e;M<_&0SyeO9JQ83?{){23@f`4vT+iXxHCyfYZ;Sk;AjY#WWz0D`tw5=(Y|P z6}g8T-VTq1>N{K%EN<1yaO8F~U=o_Jp+&Rg0ptDZkROYjHJmiv zE%|3OsK^>L&i=_W`P#)L=UH4e?#@W4)%g&&WX@v-o{j@dw+hdthaK3$Y{J0wl2d_& zX@(rDgoe;kjmJDD3wX>U90dDz@Rm6-F}9uIn#rKW=wfrCMcJm9Ej6OWWR9aw%%6km zT0Ik5G@dvuw()NBzBnOd#izd2Pn_GWQ=Iut6kUAh{L7A=(R`XKZi3~Rlf|7smxK$Z zZfLsr)`7EO0<+*A2k!Wki}LSgxC$OH;L?{!O)umjPs{=(p`sPdGSe8^9W9bL z-8&8n+8H$SmpouHopDGojUmudf|1+w#{t0?3z#Jn4oo?;v$uGIvdL2)_7#^NX8J2D zm@G~zvTWXU;G9K5lSsoP2ZPxmKHprJrSBXP6tVDU5?H{n;wd|$l7$14%mZdiy@QNg z3@sY932Yn>(wlxA>}Q!EIeA8pmXPF)0}N&h7`f*xlsPq_S#!=u4!IQL*>|Q~2rNku zIqr6vFNkqs$DhV?B}Ls_wgOGOw;uURCA?#G`tUsZ1Lyj)qJsZAT|;s+fB$TfVqoN} z$#8PaTzE99M2RP$;n1X60v#ux=!q^5II3`=9kgFv&zAvT>2jZ8TYn2v=q%AUHw zC@XW2H}8kDkkJMfcZ)@A7PSmNiymvd5!t%pa%bO#4Vy2&cUACzYx_R;ST>XBpOY(9 zH{^5aU6Aqm$z#%Da9ou^U~7OT!@(_Q_#>qq7}yV7Vie3^C=*dIiP*~(E?K9&ZraM` zpeagxD;!#lRw=TxStau?t6)>Ue-R>Z3LuHcKzeO5O!YZfK z>@5z9O^9HUU8B&deI}7V$b(6NV`7Vg?o_st7mn;Y3mAD*9`LV-aOU0gn_VK}0aMUg zm8#0DD__@{PqK^VtEo+Ey!|88{MS}}jk&9s#8xN-?d)`5x&FODG36Ai-33OOTM2C| z4+>4&5OiX-QUSv*H3#dPC;g<~JrJ7aaa@&6DAY)7V@FxYVyU+rM+BD%_S)8L5V4xk z=wBqDD9YyQl$5b>+f<=*C+ad}KkiWM{54Uu?n}kou+;7;y*VityROauYO-n4>qvj` zJq-6h?knH-(z97EL{Uh=K}dVsnbfcki^R4HY!-Zyz#Vghk?o!=i^9CdZcm>kk?sE! zjw*T-_GKzGddL(oDok6@AZa7acH(U_m(hhrJDrEjd=}@Jr5?xxTK@`MuA{1&e9ZHX z$AL?`Pd0G2^~H;BS-?6?>tOXpXARLc4|ppJ4)EU!>-2iU$jmE|;FO@79pKo&s+!Tr z;x(a3Xu|cZvO%8e4fzEd1`i;-I`(VcTr01daoOw+dz{+uh|5ozyDCS7Y4E=#zbbWr5GZ z3ziF*L=QZ8p)Y*PK;D7b!sD=*oCAZ50;8M*ql|)$P!prngL(ypYiXQsVj5WE9I=%{O9DqxCbv^tY!AozyG3locO4zEOJy9SO)En*P|Mfui=Wi$%&DRQell*~yK$ZFt}dB`(ELDY*;$m5`ZN27?^ zLT)ui(Fq64`u_Q`rYSHdC~)#93g{`YB{2NE#=!ref&EVVGZlpZqfX1BbCw)Sr*Nq< zT~dEuk?dc^Jmp<>`WyY1?^ZjU)3(rYWeZYZP)T4|s>ZlO@0v`&!Fye+8xFAF^LrEb z;EmaYxA*v1^A_+bGz$4JF!?a#aTs#0dBESJz+CW9l1Wi0E+Djqi9Kb5I#)m2(FW$83Wq{2DS=?-47htv)Gv19R4=sU9!3uSTspZ{cXC?$v=1e znY|eP88I-wTYdS>`G5aX{@0|{mL>3gdcY``VS#3{T9 zYGA#yj>WA(T=f9!tp`GSjSe~vc}54;>nz|(d%%>FD8&*fX@aS1E8mYY_OBz%_w^dxOI6 z6%Tms91sg=zLO$A6q39IhrLtdwhDsQMH*!$>{;S>O?M_0K7L>hvx* z1(|o{?VrV^8UNr<0W(umGLwM=*Q|$P77rBUco>8l*4ea*iBHH($edP_@Yd@AtBM0l z(1AB<6U2`iU2R(A;G{6ah(S+RVVTK;=3fdzQx=Luy8MDMTxRS$-l>`775n-gwu*HT{^SgzsZl)N`d*=d2@j; zf7BG-nF;<;bBI0mSmEDf#b4_73B8QU3+lH8vBW)i8};DcIlVV|4GtV3OsWbj8y546 zDeCDaa865L_EMBmW)xLuFvqQh|U&H{e94k@LBVcw3RMGJ1N+o;d! z-#TM~hE%q}&0`-0ie2+P3TKEKtz0nQCD46-e!53-)W^4P*L;0@t<1eq#G|~8?bHKa zt%J-H76>d$lv=_e>ftD=kjQ^-E2o2pq}IZt(-L^E9k8=hU|Z85uu8$_Ln&Lx0yegT z93G7c?rb7k9z3a1U`;u|F1IwrFyYyc5)K0~W}5?nR-P6@zUFb)|5!9=#D6!pawz+4 zqE&w?P&vWsT07UQgEFrUXr*%oXY$%6eOqgGCgkj;w-E{dm@8FZp3`H!cHouI0*7A? zf-@HI>oAJWS}0<4K;W0&^eKs)DGIX}g|V^wJ0-1A{})uiwK?LbVPs@UqdvTSm5 zwqGH2j!|}AqsSizu5SmVjx?rx`6l_Rf%A_7Th9W{9tEy2;|Vqkn6nbt6Be+Z@G_S= zz^q`Vu3u=aa)8<6WvQWHvg)_ej^I{F!KkGRqB6Q1(Wm^ty*qC5j@86u8nBPAhGkc92nY8N-ih z4_X%h>&2qRxqLP_!U$wi+6Z4NO0x#B-}@%|#cfN75dW6V1{AGc?wSyg52 zaVs{kP+-gyWMVnMa?62lLo>s{9}>xdObrZ<7c0^u4^%BR=Qy>1SK|$jmLlKN1)+Bn z*n$+L91aRjStz;cWB97I&Zz+(eyBxwhiD2k<^_pFhEJJz`lHr6Mz>y_#(%m`b2M*G zx;?GI@#_-S3KPezlk1&Xa~7~MI0~jSaDO|%v1zD9zB+N%;VW_q zvxU!=#vUvCf8;rv>GPuRlh!O=!LgJvtATfxB0rM@V?rQ@T4iQ|A}Bn;ii4lTs|#e@6vF0kDBbgYZB*=`pVgp1xrlYZ?iQu@Z}shZ^rPC zrKg1>At8SDfh7qJeL;*~<&NSdjtOauyVuWN(HSMMt3e;x{6B7rmp z<^%?QwHD4K2IhGZv*tc7Ics%gn#D?vr^b^cnA60UE4?YLb^c$JHhI>$z_11EHyYTa z4lpKZGO-+Bn%B+8wh(kuw21=)--FOg%1lBB-kdroJ2{W}@VUUUgmjSuOg<0hi!byk zetCF`qa+jKin#}QO(xu8I`5n*aBR|2jWB7!NsLn06b)6^FD`#G@vkHQiiUYSJ0{I< zW)4U&SsHVyUV16xjqR<`ldUcP#5f2)W8nPN!@+Ti+a*!d!O_rWagioh@y$f>49DHf z$5!V1N}boP zkDR>Ey{6)@xn800ORIFfiR>{7pco5nXtQOWFiC-P6@!}NBF24wOt%;u?cZLWkjI?< zci$xjJ{?DPp9NVaS2k-qPIt+X>hhIbq{z3#m)&jmmeL0c1+%4ZZeJ8zU3aq3@SvIe zD+P`slO68G=>ZC-+8s`D{meczjlwR9XBIsWNa#x5q9DN3$fq>PC;I>s+eZ$*vn*Pflhinc<=5_0K6+7A zaP7rev-i4Z{JVGBYR=&|`;0Fu=Uj2i-IJ8SH7il*(oIGm2F_Or{LdK7`qvAra^Nd^ zus&IHTFe3=uUnVjtY$s;&SByK!6l7cJ>U3JVkF&qj)ZNH@?u=E`XSGXw>MVr3lDw4 z)?BLbM&X2DaouS_`7$4Q4;T3_4#HQ&9A|Q|205^_&ECHG&Z*z#J8n-qU9Zg6aD>a| z0B6Q)@hJ9lwvK1C9x$)q7hz%)adH#*vEYNb<3~w(p15FvSqi+&3%Q*dMK}_l{$UUj zkz+Hf<_vnk5wxKC0=IDYt9Jd3`(;+{-(%d#%(`md^T)Y-Rw`>#%s=k&DBBamz#qbJ zh1G!}?LiCQLsOnIsaFU7@!vQw&5DUp&4Fhg7xRakRo7>Pq?@p|IB>pLz#7x~%j^rs ztb{5334*H>ML$JI=55%r@Y=#^rA3=PLr*No|F>-8nF9v>rLO&RJ{7IF^X0)U?VGpn z&U<^4PoZsr#H2+DT#anPYELFKJrrn{QA&zPZ0?d52()_n=_#L_P1$^_(xRY;hi7^% zU@Bx{^%9WRU+|;hlalZhjlfuzNY5mV2@7I&dPsIEx=)hkHF)u5XM$sA7l(w3O@+YL zRV=FmWj*#bC~93_9cviFpuOy4lKlQ$P;@&kA?q`v@la<$bQL(`3E!$T9%7X@q2Rj5qSr%?;=a>?>-^S|xSC+2j{?Z37uz59} zIMVK2qq4)Zsi~syvUs#Yp^HLwMq!6SHBYFkO6?9o56u-kO2?%QR0O8S1bc>tg@kXt zF_lFw;XosM$ki7~-9?rQMA_I{MXyKa9~E_EW?IA0qI!140Y)C9GY;&E)gcn8TGDop z8W?&U4GuCYZ7^`+R0|1n;!^Fo;51|E&JF*jF{!#7a53sWBhZn3O6H)8>53m;Z*gup zt2R4#?@hH8e7mP6${1c+F}csS$I1L|$=?r${q6p|X}-(-LN;X{2MceYoFiN9@(F2d zhACOj0tc5L;&Vv(bVPj1pGVv_?|PVys{boxYGO-u5%O#@RFn{IZn~k;B_>)C(5&2O zvhb)_=#>eMLQ8+F;!&9}#mGa_FrnZPbI``nsXa0$I2N-!b$i;#k{tWtKt%4<6{5TJ zKb@Gy?6gRtk;UkZ0f(ZHfC2-XyUN5CLAQ*Bu0m!t8`${ODykir1QZMoa%lEA%-o(l zDbtBNdqt}wvtWwALC!^n3s0Fm-%@-;@SKKlht|0}{|>tFZ9lv0^Zfmt8)u()=uTW= zcv0v2ip3M$v{o*ezR%#oWto!aGg$6dexKp6Kx|5;k8U>C|2RS$LofC3p6+f}TL*6(iz>!fpqkyG7?~mXKp}LU9 zMppH6Ta&s|Q|CSE4Vn8|l(A3DRmeMeb(^y6Lgygx9;9g^6nZ|U5l@^OV zBvW~&GZ$L?S9@{3W!lH)3tbY zd(SkjXLn$5RrYoixKY3*C2)YLL8F=L$%RHHiGzjW5r;JY1$0?!C}{C^D4$e%z`&|; zfKji&FSv6(>v*&_lX`clQ5>p;ko#L2pG^2nibwQHg(gHWla|P_icMkF$(>zyT zw)5nTo`>nFJD63MIQCVXIV^X!Ql#|Lb`#qG#$f#ex60*|OhsILgCC##C&jdPn!^tl zmRO++jzWJfun5#7isdICyG*rLmFRFCE<7uW)wu zN!)OhmATm1sOiMYH(?&L$%l5Y5`RfUZWl%+fd)1g*UA%{8Q54pu-mLz$YwC(AkV4` z?T!l;NxE-fIK`B>T=|5ekh?*X@f^dm85+$JQx~wP$}Hq?5fK$E_sCK;?DTvr!7ezp z;n<%?2Uq-ZbVY!cJ=%Jy-i|8l8GOi~Jtd{H+X{t51I z;EvkRB)Tqvl|A}JqpZ(nfmnkHoL`Tz=iGAIH{oRv9$QopWH8 z3dm%Wd9b#LtFvRF_w*1y^QlZ8OLf2Bd7jDm>cBc529JV^;t^#I&59fhY<&+7GRPz} zTCZv-FIwRyBlhrw>jwu$)&K|ov;_>jIuH5WGul*jHnvDfEM#lmaa1?P*jeJk2Nna5 zJ4!PO*g_sW;$N|#&1lj@o}`FHLdOi;RG&q2CUJzW=t^*@6LsWFQgL4O%pgi@@;0xq zXN_D=FYYJ*VV8^cdRx!)bBfcg%kzQ^moz$OB=GcIXt&5=U@1;z)wMauy(M=MN5f@C zxqlH#?XD@oYOF?z8flF;oNp}PlDNe@WkW)Xmt2|mf4y+RcTA1E3bO;$ZGIz~bVwBck8=|5tw5Ro; z=C6cCDV;|=QVx?;o?O@~C6Od_PQt}_Mgk+RLz19-LnE8fhj!sP2TbcdSS^$2i&Wk18?W>mB zo{h`sNxl+nx;1q9Vg5DG`UR?9Gz#qU(puM=-jMWkicb1p39e>crEMt-BHAx7&i>}e z_U2ftUf9D{{WVUaD*}!R$z1H&)2zg&!+2Ebn_BYJFS_d7#z*8PEojy8VdM_ua8dbp zL8^nFiCJxdOjhWZe=cWhSI_%g(*5c{snW^?UGA%eYo|R{YUEtw%D}79z`&BgDEB0h z_ra+~r6&s*xMdOrS4S*iVl!Zv;mRx`X~Cr7wXsEYha;cHTt=Y>4(xWv6+Wv4xTwx3 zXyN(sho6~4zRF#;w@_s7i)Z%sCfD-6e4Sk{^LR#<^(w1`(4bSV`Ytq!lwG*$b1{nL z1yk6=hq4@uTG^>`5@LcZN5ZrMnDZ{MKNsdazkqAi1ff*{T;=6#OACsZP7u;li)dMD zzTg0-_7&E419mqC&ejba%?uok1sszCblnmd-7feQJrEOIP`T!~*qgqdb_?)MNML$?%K2)G ze00xxwLqgMC^UF`)|5{je;#mDUSOLgz+!TNkxR^!YX+0Bfa#n|vg^#to*vB3xhluM zF^vBpb6x@aW=8gN4dt4QTI0{IttO>K*DF&GM&7*9dw-GgcQLD2M!iQX^otCQ z90J(SGO#l(R%ZO*|0Bqqc>*KXgnx{S0^`zdu&-FabMslN6bJJPMP?rX&cXwcvI`hy zhEKSC-ML=DSWLjPEr2WFczaC-pO|F5lqa8bmQKQM}JU^HxCJuAVvih;wMf!Tcnv%8vA z^2FX-GkS9p#WafgTy`jjJ6e5L>tl5<&{AMDIlw5AAkK6kuuIYAc(Qw03ET1oJW~sp z_Lj5BgwOov;i2|G`(y&AM4y&uK+Us-VlqPf?#&SX~I#=R8qTb?ke zFJLZzz_VgPkf~EMT4`9{tNKf9~g1b{wwX#TW}Ut4dgWsx@7$j0%1} zVi35%V8p<(eF2C3E>+G4%nSxLN1GXVCopgbFdli#z;Ge@Y^%}L0=B*8fh!m}E-`Sn zUh{Kr;9B~jT2@19=>i^SDb6AXu5$%^FApr~tl;~w>L1@zg{8r(G{RQRj^>gLZa4XJ zfoItVj)>!IVGgWQJ(xMVH{5EJ3tQZ9>St#{x)!60)p8A2F>U7Un^~j<6bvq~Bu26n znKD1Ty3wJXtMGzOy1`1zt1SA(EK_E&mN>9CPsj^%;OKJaFbl|VQ(*7gkgj!sQ8&S8 z#pUTN+btI==L=LVd{klS`MNiMqZt2(D9c5u9HDxQ$_&B^Y*h(tl?m+oS1~9^F>)-B z(9Ckzn$>K`#rY(FYi~Q_iNy@v3cObx($gn!9ofpUl#w~TfKC1OcBuyEQw#XsF3@<_ zwfOat~DhdFH0EVKS?VAj=SF%ihtUm(Wj zxGODuIoA(wG3~j+4g$Itm@5?6B{wk22e7y{}kN!8o=QuFiq&w_QRqa=T0b>&z%0~hwkSkTjae85)&8BOkSvOwE9Po z&C$yY{1ce81K2A!FgZ?O?|Z;8V*!VegJIXfH435b8owm>UY~Jw0rUUc(fhU+s=Pfbg%q&xCotaPSQxlk`HR}|c}utOF4(e2d29H@7S#jIr9JzO3X5tTU^|<@tvz84 z2LtHPO%dUfyaC%vx6DvvV32r_!BxO=k%5DyhRN`%#|nnF6#=}L6lRJgFj;bO#YJ#k zEnqTc;5>U^*4u=Gk~IgVZPvf%nxyb%N8jqD6Qs}Vp2>gy0?+gblS(I?(V4Sz)@qgz zgTrZ2ou+%3tr>PL)MnPc5c+qynDra)r?4D!5wfh10t_9o~i+4+3U@>O? z$7vjJ)Zjv~+JWV9Z;sBG$|zpIXsp2AEWp_+pc*`ZRd!pRrvY>G2P3Zqy%rH$)|l^e zxv}Mb^p;vly&xC8|5wDGOEWSBFy(Dz@=6jv$91yZ(@0H;iE{yy)dD7?1&l|7x1|ek z`dnatTEM1yf!*pE@l#%!!Oz49q?X>}M~r%yHm;b3t&+CC-!ISfw3U9xUeZzEI2f$*AjDZTGRGk{1|b z5?M+&u-E+)h?HkwwoGSae8grE&g|YG9^7peG+XJzqn72)1=VjDgw7sU4baQ}s4Om( zFOeWFtjPNQ_X*B`6utn4AF~;l64)jNa5ik<=(@n9!L`;rfK%crhtUG&lML+Y0i27> z*6sbSR&gy_eF2BPSIjAg^zO?%R|@!^t>t^CwLWAn-_sB4cJv(lEHEkj)#>ht`=O?j zlq1h3K$jNMa0}I0i1~CPO{k1dp zaWSe2@b+C`(tW_Tyn#Kn+FiH6Sgm1^&IP8KYnV3}9_VvpQV(Y8OJkHguzg7(Q^SO~ z^_6^UG}o{3JpI=8#TTm`5<2(g|L}dzz5n$u&l!PbF)I(18n9Odu;&O|W2t6k&b!9q z$I7velhg0;r9Th2?arDeFt5mC7MJ6Qf55W%0JrQz?z>w!=YHT^{D8$JVEZ`%jv@w5 zk&g1_kPM}%=cDg18hl`DU{I~Tpv(8Tr@kS0&RJ$DfoP>ydlxQO^7tpVHum_|y<4g~ z8JUm2-6(3nVlYE5fT7Hgfl=Wf!>e`5ToV{#*Ku4<=A6KAQix$&xd`KdSQ|B?lRYQZ zqC7m_&tsfZ%j8na#v-44Wy6Q+9qmsY_?`-U6!c#&B>&=t@A|Gw_qwI;%irNsj6D6# zpngj9&e*s2r=2@A!*7{x&$W!)Yqkt5IlT`!_M`OFh{*u;i4$ABZsz%k=6_o4#r zEeE(upRyQiU|A(_PhMa}w*%LT2W*z%=k@s>Kb^$rWxz4%z*WNwj7AT70(`h?53r~$ zV9LC=*V(UNi>|mh!;?olj=Rl!;^3rT=m4X5AA7?EMKyzy7iOgu zI22t;wGjwdbl=;sgP)O8f$7-iONs$ZivJpHR`tkSUQqaS0#C>t_N5d|#fK!xqkPIOo92 zIeM%H^$$+ke>UQOl_SgI5YU+|P#GA&a`GAH;t!mPA78oT-|*P~TI~WOOTM~k1EZ9I z<@5S~5)&9-_cP{bvX>gLp4|QK-t^*bwF!Gnx0arN>w3F)-Q2ef9~U$-H}diLDRcxh zvG7T#2$=j(Sa`6VpJPj<2(KTH`!v6)n^HL48d(HIG%^ws8`^j{Wn4T0gq)fhc@=yb z7!(ira3~5*F_1i%!pUkdV-BZC`^nBlwO+2(*UtQ#X`Fd(QQ*|D&|9JFE^f-}6)TG| zDSCWrp6d3#IhNaA-C0$9?aeh6_ZFk4E1S3X&9f}N@n`F&>AX|59ahMg-D&QcayY2L zV%ep~h7(RYTw21Zw2W8UA*#zl(Dn4?4F5G1f2Bf~cuaOZrx0l&_?yLhnygOBi3CPx zwrRVi&%L;?u&swj!sI|laMs27_IW{0CK3k|j=a=g9X~(df@3qg(0bz|298W@?JV3C z%PneIS(#WW{%E#xi7pBD@{4LvIMBqvnyx21QFD2t$cB?@Y(-82j;wA1T`b&A84I@- zpIWIVz&*jbRVqDaBA00Tl$lyb_*Y5^x}+`%eC8q=_{VEj;lCIe^<$gQR6d)w^VO*% z{d-?sMGrw}-8w;eLx* zySVHoI8RMiuyky{VPwJBCS38j*X*&y0o`V9n;i_zOa=uFOC@<4Qzm-cnV`tVxulw5 zVdt{oFIH|BSNkn;k%Pk)$yB$;Z(a{0c?{jDs~@OVeup)iB0ZK!()zKW>3|r9{*k__emD)D_QhTp!dKf zck{V@8J_%I8JCu7PEAk{&xlZFUao3mPpV4oNQUXi-)Q>=bocB&5Hlh1)Zf)BC`M+^Gg` znpzWkWQCgBPkvZvsF%p2Bz95a#UYp2Yg0MpULMu2Q=B9!Fj?iw+H)S^TdpTnDYUA1 zByw-oJ|cKdfHU1ESv1|{s8*0*heN{(smljg#U3r_(oaa__l`Is6vD`r?8?L~`NM%d z;laPQAO|I(#snwD*^^~{{&D21h-l_f@G$Y)dfxg)@qs0~4$Cg(JR8XI(QmPU5o3^) zOQujjBTLK#28l0<0)8`3DJ3Ncu&z;*bicqLJZ%Aw{2wmI6m0<(iGYLL?^HAdE*ucN zA<m1!67{&0n2x3xji#^Dj+vZ# z!DE@hDB^PB01sP5TZoR5z>WYm&7OqD$0rsFHAJ{*-D&6wx}wL~sBvT^TcDlll;u_( zUGvZCSjzre=y_y0)5R-)jVd>@yErIvM|yClEMSn`q3hac(Q14r;gt0hCpPbfsU{B= zG;nk{G>NF(X4zlmxNhDOmZL$fMIjqIlp`GYmUgkoSsb^txgsi15#S=fW@q@mccp?; zeq<+TI&vmW*(@=2!%^K!r8`wWW#|X6$Vy)_V69?w6n5u2G`VWR&C8Z-y1lM^G<%?^6X3`xWWc=L zu#wa1N3-#*1ED;U%5pmq=Qe_A8rEW&! zErxttJB(@*F4#@CX;>0&cF@m6;=wE)l>_^}sIs^)Oy2s}Q8H5RxYCr39XtY;B>R3i zt~jm0rZy#!GgYHmGURTXnajV5L!xzGWi@&dUG6XncDy*e)>S6S>xM?`3=0=Ej)&_m zOyn~(o#rFZw4qtyM#D|(4TZg?6C)L#?6|djbB=(?MrO$a32a3cn{qOxIRX|q@dcPY z7BpjM-mb+g63LdRa!#QAgKDFyq`@J@-vXR#w9*BOeU1v56-cvASZ;l_w1HJ7fWg6A zg^5?7E8y7dxjuX;OwUC7S@M+PmPEGX{8Ub5;ae)q^I{33?gvLUk-{#Kxd{xpI;~1R z1@gR094zt!90N@?bms18VfbU|8BmuxX-*4+qpAjT&gq3bT&#=ir)IF}MHMv5>`-72 zKi|l|Medb+kJQ!p@U4<^IqaK)Z~Wugv~26z{Mu~kjhE-zJo4V7wD?V?_H8zk1J(C3 z3%>8YynyX=+O}TPjgh(9(Kl^gd}E(+p}ie&7(u413h28RY7O#NQlX;hexISJBXDe`a>x1@A!3l7z;|4;IS4$T)5}MWBJxg@IAo z!jaR!fthp70yeD^41ppB#glccSf1~%H=4M#QSHewrt}FdK?fOR0vKg77{v?@h-EOY zNp_IG&?tYwNaWjRcLth*xA;kQQp8s z@&~hx1=BPy)?7!ym8W`D3fO`duw^+gU9)c3wvsEoqbq$ySB3?1sAVTpz|=V(Rta!) zDO)rxJ|UsDp+U8QB{+kPGoewaf>A7iQCQ$u(!Z2O=8yv`cQ>XUv08hX(Xio|rsLKP zn;m$1y7)uHcU*4Pw%|#<%_thf$X9Vtv7%8?f{D-IxX6QsQXfYC3P!$$gOV|g3OSBa z5scCTOp*dl5(bP_-t$5pNLW@FNgS}5a&kp!b0aTszM?GXmm2@Q`gGuljGv}kAvVQ78O#o`*l;_+b6(Hb~75DV9bzd&M;v%Tp=iZkmXp?A}a~mYNb62 zC5#d>O%J7+L?q5C3N$JT@L!f;;j zRO2MkF!`m|iI$LM&MO%CFEH@mXpp+m{8VTOAIFl%9w&G&G^jMRcs4ZJ>}+7tVBnq6 zC}F|m(81#IgSpvY=ZpoM&UgBpFCMmCa@c}JP3;7;D94c}+4jr|4DAvP9S<6~B^bDO z^jbKys%~iDl4xMsA>dKL;&7|UR-ncCz*>nFjmid%GlKtda%(g&ykK}-#G72$<`KZ= zu3$A+Ka(K3fo&48(CiH%akWt|1gnll? zSTLD2T(;=F{H%&4XhkDWK?C1}CY=*Z_CJ(WBNn=AFa{fOP8VR9zJSx#hQapAVHO95 zpFE-B8O?ug9%jg3(AvSGxuP+80TX*hzvzKx^9DxI0%j8(R?C0A!4^*$TsvBHI+*Q0 zFzG&Ec7DL@B*Efvp}~uz$4hD5$~6DX~v-ZD67cqnHO{@|%k(U#_PdX%v>YqpG zipBC?w8b1~i$1_)^MlzUpy`E2!%UBpGd|e<+%nbj%+x4O7mfoBvvv0Uo1M}eEwQa6 zfthJRgXhv_^A${16I!GanmjJ3+8tnxInl)bf!X~6<8;v$J%(oI3rseFjJz7H;j7rP z9NJ3+77Hr{glMqqyzu31Fqyl7L5+jOxM8~k!+F;TF;w+9^ za<0b?>6~HlacESHkf~@(KhRnlwMO6wch#bed!iewM9=m}FLimj_gU_}?&ck>1&k^m znv>_wwS2*(d!kA21(VZ*X4@Og_8S__KQ!APXxgpQa{FYHlLBjyM{B-CJO6K((}#0>8yKZ$G)hG@r6jGkJ-}Lcp-oSt!A7H1SfhbU zfyJOBKv~1sA-cuRqS=2*TNy`7Y}9IF1*WbV>ClIk3msxv1uh6@GztHh@1Ve%wShrw zhwt7s;gYOAm7;D3t^_8FM2HDQu$mk&$!j!z7R&$VKuO>Ys~OCii#E!vaWs;(6I>9{ zee_1R_Y_x^22PGAYywYM8J2x2W)OeTXf>flhodECMqBoT6whsK*$>*@zh;#F!0eFF zWLd%DY``R45fH4<>Un@Q`bUE*2aAJ5OLPSb|B41~52jhAY(WAN_EHQTdzf@@H0l1E z5PbJQ!_{uS1jX6h^A6wJeR9Tv2gznnEK3@gD;k8SF-d=D__o5!*|f=a1{3#;M!Org z_n&RmmT0ox!Ln=xgX4!LXAV{?4F={L!YgDJhg7g-Wi%=;VDoEEVwKo2nZtm^kn?o+ z>?{V>8yqb`mEm81dmmsqBP)8$P|Qu#qVb@Jo6Luc4X%$BcrV>sd!}V8_Z|k>!wWWw z{PEH9Wvnvw{pc~}jY5j!yav{cmuvwp&T=VsG8{^C?&~VB#BN~AzR@~w8(ZuR)@*_H zvJVX^4b1ir7*!bVtNdV4p3v;6!SZ2hGw+NhhXu_lFPfVKFYtFXd%R%Qkx&WfVCH?n zP;j~RUkpR5?u!N(A-A6c> zL{(mj>?peIq7@|6;oNrb-r~%67x?)YRyf~ic6P`wOK6YPD4D;Ht>8eb#DgB?9SuA$ z8lL{SEVaR}wTDr1LbGE+pCd<${fjRTelgo6^xcZwt~#MPPK-s@fl1DwSV359GTWZ2&nfprXL8I~l7v6vi+!0JnCpKNt zZjkC|6t!TIyRc*Oj%KeJjf@J3OdA@+JOnK)y6$Q*t2H$6JmTaCVD#Fn_iKRu_V1XbL$D)_RMuaAEQV@Jyg z5B1j{-ZV&ZoS*!#LE=Sd#~T~&8x5*InBMJe;!kkhzO7?!NA?CLwfRd*cO5I8ZK7}W zgh862k$*uWd%%&HZH>#~F8_N}z%2J5hSipdJ)p_^0VC@VzfT1noJx(Wm=t&)oReah zXe}{wazmqQ!ReCooa_ghgLg2?&S{VNVcCM{|F#Sg%a|9uw7%KEmtY^G%21=+q?YljErOXlwT!2LIciFaYJvs( z1cd}94h{j2IV(%jF6w%3e{?`Y)i?6rO4sxr1Iwk%O-wxkQW^=9-kg~!**N9sNr&Yj zChldImIx}buy@VSiCD0}^0S(suzBoSa#BeYskH zm3#2@a0t5DXJ{K7isj%FF}ZQ$f%%8~0zxJZ3Ue4g-siLU!TIEPzW^sMOG!r36RT+w z5~p?qUV46B!NWZ(qUf@iJ@$c=qs~TVb`OFXbHq|h2g5lp-i7$bi zE{$^v4{_Fd2>BWx3vl98>U`1YCY7|O2vl_V2a|~W7^1E^mkiWY#D2ZtHR|cn@(vh_K;kv zvDn4ZOYeF_@d=UOl!-@KB^=advKeGJJWHRGxIvO_acaUr)}<#`Hn4F?IJ%3+zS($W zrl5_2BV(@Xg4=HU7>_qBJhZf3u84<4-GN!#!renPXUPQyMzbG`9g58^hYmBRA1FLh zKJT1?vuec>^$B^BiVGMx(`u%48Bel#%)`^Y;8GJS^PGPhTvR8#m=MUqDe`hYC&#=E z2RSDkE#x%4;IWx!>zRmdPvuUPLzg7<9u^!9(A#1l!uPb#1zCQ`u(DoSB+-FikAwKxAgpG_hxM)TV?vuo~WIIKs13)8RHVpN+xn96pul zFBpYn3K;SZ|4ox&>0>i@VYgnfp^1gXqT!%O&K}2Bd4q%p$CR{W6xkIDj|8}>cKx1ud)^wI4Cz<8{T#Wf1_XJz>ZM~Xw z$YYzxwkOR3GkzB^KcBn3;P8$I&nCRv;c+*KE7(`|(4xJiFA`gOOuiK;AJA@@+QhQs zea8kCyUnHtncdYoGns{6Txd70iC@M%^sP!Tl(nnGcYhV zsV{kC+K?|Xp;4MaaH6#iBY#&%i_x5g9RHUbXOwt3+g(YhbC)BN&^eD|8~!9W_Hd{u zcbww5Go^5Wu#S?X+K$7TOA`A^KLp6x#kAX0?C2?Vc^5UKL!kAD0h{5EhJT%wCmQ(| ztZ0^*^19Q1&p}~{=&<www=mSp*6i?kOy1 z^vk}%9CM-BiXo9*K)~^%2qTC4A4Z|23Com47IL_U91_gUP-J47F376#fF*sxGn3gn zIb07cWLY2Nz!7o6DQxd6wsMVDBdv)&t|}`;E(EX()UG&R`)@*ri_W7bn@kr@?O3?r zpIycy$D;K7CT^HcDJ?dFCKj=7BD?stwK3tY^v+PRg>YJXkD0Gu7i@G`Cl` zii(7eqG&@*tA@itcDs&8OIPi1(B(?ZXp%sO?EwYO!U!h8v<)qGM;>w* zTO5*mRG{U&U=hEzvn1QKfKGdhL%dQBPAZSeTHJpe6f!!{z!dXxxx3{7CS483DuWA6 zUsV#Ob4Vl<$UJL~4sn{KkNue!mMS{73MXPj}Mgxn^HG2kwkgB*=K}L&)e5MT! z3{tPn_wUte;7FLt^|zSCRpl|C`i^GRSrhvzPBd@*^JIB;y+*&HQ=jDV11zdijUCo6 z4)W#x^EfQUr4?hi;)USq?hFaz#`O}hj2w}iP5T*^9lPDxs-tA<9o1GMzLjH1I-doj zu){*0XSt7fd~Y0J@3_Dc;BZp^asaF5yhJW56-Ca81}+Lbg6z5tPWIh5l=Xr(cBpoU zi*{!;t^4O7X|c3{uesx(;1q+Ic2knL3U5qXa60MQ}GI19buwv+n~qWGQn-bh6iOS9|I##EtgOTFAhv6~x%(u)s-jm4mCo zqX(~ex9~7m6dd7UQ<%l4!zdKnd{EwxvH5gZn8%JlO_mP{oE8p?gscph6&3`xg{nSS zx8L)CzBmJef(bJpUq&NKy}|@D1|?>`ACA&`rB7mumfU{nC{o2zF|U_Fz@fS7>H;I7 z1EqP!%Gxq>q;($l7*F1LhOfNBR!Q%p;rq$$J?)Rs-hRh4z>(|x z+v|KO0vp6vJYbc$F!vuvo51H8>N>kMcy*@v7PL4xaEiopoa8jQ(8zy)fz9{>)09<# zy-qgD;?)LCwg06%E#4Fgwgwy(Vq)wHT9Kr3?#3>e9)~WAj0H?{UN*?+Y-}_BlgQ?I z;)znz7q`h8Cag~-}qxAM}Y)O-?0W}1uq4L8>$T~MJBy9(;g@tn4c50xS`2HS@xSi z@rwk1&cZ@F+d`ub{nASmYs#?}sm1YXUoBY!(aTWjHDDt@*mfSxF#v zd8ro13@r_3l^wT)CNwFkIP%Q#=QUtdS>U9(powpa6N8l~M>jM97j-b@iJah)!(ry+a=19yQ#ga-rri8~%kkIrW} zKOtmhdEx<{Hy79B9$?FG;GDw1Hltz28Hay@ALi(l99DdA$V8+`OyZ#Ao5nkT!-a3S zEA;rXtvn=X;HYB3eC_ST&LxM0beJ?{4hrfVYREhw_{UM<1H(#}L-H~Qx%N(C51hu# z(ZF=1(P#%_y+{LR#39)QqPO2fzbchxS>j+4(UkhtQF2NHlZUf>iHph{Csh{*Gs|0+ zi7nlN?N-l%3cnoV{uLDa^OBnZ@1tC?)gfVT&c>;%xF&Y1+4f4a{k>)fE)_eGC601W z9NQA#+s0)7ym0(!q}m=unPVKH8V7s++zj&w58raz_6j8^~ageM{~wsR{7D-rw?nzcJMA+yym!>p(3}X)P)HR zqE1f6D-PQfJQQnM=p>}#6sMNPr6Oe&V{=1NQDyq;4UUgPRowDgy;oe6`nvGlpN{?o zM+H=R&YuY6(24rE<$#b#f`>^G%N&MNHV!Ja2PD@tX>B;HAQ7b4;=)$qz-Ho|mcziK z<9#8rK_&NCs?RJLoux7cjtV-BGacGyM6{)!@Qk*c&2Gz(`j6KkI`S$H!)(qSjGhE{9aVH1bSo)<1NnZ({9?gEli!YqU&6i{R`){ z*IspP%aE`;q#`4(c;%q)S(*))0?4@k1+I&o5}@N}2o9l2KXD zJ1xe6HOE1<>Xzw>CdE64l>3fe&C zZ_MYvG24H5bN*Y-`Dd?Ezh$kji;?qTmH8qmnh~<`USglZ>V>_mkt{Dg_CAZea8ZE8 z|9nRSS4R5MrBdq+7qRBN=BYU-|B^{Bg;6o#pwJnHlQSIBa@f*R*ir-BR9r9d_(<)M zYc!wRSd?^N%dSS59dAX>9AFk{m=NPBRJQww>^WB^CpP0Y6<$vMQUAIi5`%!b(i42#$9IkARhIcy_ z$Yyi>^U&o=kpK2kaYE93`D1e;Sr)NuIB@RkL?6q>06vB)SH|kUtwui0f@g1P8ji*mP7_xs3)M+8ST?JQ@M&2z8%=3onB!nHrAcXulaYyY z+_z&aHPiGB8JKk%gr=`)p10KM+aznXjFVm(8LVP#Ee&TP4)CZ*o6cyqT5&k5GQ(k- zM%S_x7rfR>t^RmnQK9_|r(lbwo-`&!72P?u@lrFEEvR(uTcZE#-nsBK4t+eI!apdo z|5?c162kdH=HGco&*dK-6+Se|d~p1??YfrFVV)ZeyeAxaS2VD7oD5ud=2XoA#z#yn z^NQB6GTn1J_B69??zIE+KQ-{2VPId7&KMw;+4G!v%K`ZZj#@(RM7}WaUucw*?(icGZ+Uo5e{5K{S7geS5Z}EamI4O_ zZyb=8Y2=*a|Gi;3`-16gNB$+LNH7MTO3`(5IyYs*tThc>EL#JXH*gg=1WF%ZU^&1b z;lLNVIBr{5n*=*9|tBD2krxm(pwzP zf8pgee7MMfVR=_x@72=S_o=b+_f9aK5EMIg|HUF>18(e&Zd=Rv zw`>!+;ZU-&j92BL+~138Yn<}_Up)HfqM1aK^oIs%8OD^72El|zrfn&;rU!)>m^7C- z`H49xq&WWj=%BE|IXswIQRM5}84TtBiVrX+IPhL*R6O#+q~Ng1g5_LW4l4XPXzar* zyTPB?r_s2AIsAOX@+YcG9=toObv*yb(e12$=_mE9Z|OafwOIaaD^K&Pw~r>d1xQ+H zI&dy=8*7F|+J(wp1~b{9b%MYFmo`x0KZ%J9Sz$6PVNrwx!=@bd_Mp`jYX5< z8Qhv=b~OI{>UeD_lUW0^c>=SUNRzxkli(jlt1Zk{OPDp+ILydk^Xcm3Inux$a?rTu zr*R9jnZZ|PA4Y{ej+ze+$r+R~dK^@FaL96x^FL9BL#i#MyK%wQYuLefAxYPF zyJ%LEUB;=j%U0UH%dT&I%apdz;H>rUJQjy{3CTN59vHWuKK!xnN5?yz3v4E-*Q+|c z%huX68|$zJ7u=cslF@sKhPOwez`hS#B?9IXI3rPwNO`0zb8Alv8@hMk0!ytB} z$?QgxsZA4mi#eNz!-3gHnM)WLD;nf8n$0y>^=z2gDjXQUG{~zQl<{ejH`&U#=792p zCe47uf;>%1D;Uff+jTEWGl+6Ba4;-kKJFG%W%l<$&9i@Br$@r8qOTWWi6b&GBId*1bvQpqi)HBNWu-~YMRNz0*0 zIz+&^=f0dPlV(o3!jt>E|24{M993aBRLIOEbmV}{iGw0OuIvT(4|qFNtY?r3Xg1?v zk`1v|Jj2NP=cJ%TlYGKKnFK~5AIHaEe}q^tsl+t#FWIbL8&|KraEE9>Z>q4>>vhe$ z{xro~eXL&>=;GTkzVUj5Me`<_u{KI7lGS*s*D9OhVF zZb_J|>N(9|V&StB?7VL0rgVvPa86Y9eWJr9lHos1GkAi;%1h5Z*d&a?WELKJc(9pU z(7C9=DPb@7SpEC>q-p*;!NSYi-fO~T5s2j=crdcv891gKqdX+ zD$mtox>A;@Pfkn?UY>W)ruy5vyNBET=kKeV|NEn`Zk2rI<)z;pgU`MZGIZyk&gQcg%(`1Ho_Mm{w0CmY!4gFk=3hqNpUzbvN|=aY+?G)&?1#J$1`W+QH5>^qtZ(o&v+YHzxFv}bXLXJH_brv znW^PfjY=M)H%EdyriL$D$ga6!Lldvl_Y+TczEGR{RBUeXOUp|!Wqeg)?)*2CcMG56 zXxZ({%$WLbam@6HL%b3?4*Ja80e2ZfgME{f_>@g%IBUJ|~#vU1IK#`8a;UVD`M_WHP9kV)Ey(NZCE*=)_#l@*Cc7HO&{6>BVeZgl3) z#UA5c$Hx&{OmDle&2*Wdd9!j#;IX&orZvA%d^Xc~r|eaYEs#k{t6hB7C=-hxJcnS&g86AtsRJ?t%Bz${y5 z5w`J38zZLxLxlJahJUgf4)&(|JeDl32w?3g=+1n=B>Uh;!$Xw`O`Jk{U%M_d2@qb_CzhYcJp3s{sc z9ALipp(8CoxuBy`!g3X(WR`=P)H4HRt4o6Y<$vDE{kzd)*S3A*#W&0;>l-J3_F~(6 zn}gAO2d9&iiJ*qro<_@al`c!`#P-d5$t2!$<*2dT%(ac!RwkZ<4Du(OF8)(G>cg3^fLTVP;Srm_6gGhd2JHwwg?AU&tqv&i=N@Qa zU^!sHu*HKRL-Km)jjvPJ-r&&r@+yY6R%OZdIqUR&RlMVwlA8n~7O;pFG%M#B?!Fb6 zypTur_#wGp0a-HwrJpZ|u&Qg9d{~%PG4Cgj{fv*xo|*FZr>yY((lVjVo{5noC&5{4 z+T_k>AzC7ngHrUQZh9PyOqS~3(xz8&>Y#_$Ar4s!Mq#nQ)2?$81t%VGR_NQ%;-T}1 zL-qoLR-ok!-c(a&2@Pr44GFB;GK}_aGZ}>*DQGzPFgo%#bh4;@2-AvrBy(kfv)H-~ zt(G1KnFLyo*3hvgp?%nM0kSS@C7Fzm5A z`>&4;oO7+3m401d5-t&Cb6jz7Ba;Xt`-vrtTtzL$&m0<ZPV}QhiS=+?ygvE*;5^3IeTLH zGmphb_iVXX^CG$OO0C@7OABJnmOSLjjPR39k&2t$$3N+E!22pi*-Q`r$6}EiuCn_v zG|x{tz{27-l`D&ZSuP^sz!J5Ls~42lO6V{OH2iR0&A70`RU?VbA#*jmiog{11Eoww z%-6V54)_Q~U&v>gG@p0b2R0R!#-&;h7**mzn;k8f`Nazw7#=WmT39e~%X56?nI*_6 zaiHO{+KdkyyI7{J@u}RBxzcM|xyr5`Pf}0U6|T);7H0e>B(i0}&aJHl&8vHJQWi@y zzjr&hIphj z+#boy(${vdnQd$AoxWLSud+$h4Wj_YFCk~6Sf)*5lG(6u^}Jgwp}XJywNkKGm1#Wa z|Ow^uuHgd1ngXtnEstfkd9+sMG;(<_U=mv1A*&L=xncbh>$}OCLIM{>cWpWGRNr%x z_bVo6Cf*xm&fXF%T5A^a=J3qRbNsv7-BzvjxXi%yuej?u*kuoqCS>kp2dUT@gzP?Es>(y_En%S3{)#B}PC zZBrWqb4|a*Bru1ruB+0Uz>>azS^CzFqg;Db&DlAMXK8+LU|;9Ju_yGgq*;NOzs&)* ztOd+A430((%vKJ3%NT6$BnYfH#yHJE>C{8Xc?)HE9x?|k6nfIYsiY`%$3bw8BU?k$ z$7@Xt7n))osdlBTE;;w~?kcCP7Z!b*utZOUfl0}Mk*ka4(3%BP8U%bArTETCbL4UC zJ}AwX_i%#1*RcMtarx!P5@oMBetV(#{e;6~AFbU?yiXkTgx%H!33TpRrYJ7=kN46M z#h)|Y^Xzn#nZhrmRSQ(54ltLToo;jBt(qdwVRrFV3z(Er*p8gz_{6};!4PT0p!gw~Yg2+iMuS?~ z0+GBmB3_J}bR3jsCJgOSOBfos8BCI`l>jD$Q5k4K%l8_uw5 z>gz8$!N$DU)2xN_-vW_&2hW~!Y+UnEW`p3@Errr6H-0_SxI<-`p;WWfu|}C=2Yc2S ze7Er9zvRHZv!HsH!Bw4|B-@p>qn%5=VH`7BB}nN-{BOh%b~`bWl)d0mB2?=T5jAa*?su%=R7D{dS zD7{Hgdi_SJr$rApOxz)LuBRir=g10~lZsNu8V!#z?#@g8zQy2)(L;H@U@HO6j3epI zZ#M2d5g>hP!q0t;()$!;eod4w(!lf1pXHcY!}`!EqSo+#R9&h z18g4>c-O3A)=S{}#lUFUa3DlM=+^_rU&m${DF}Hba$ZPgV|g@dk`7~5fk)JX!(ZNs z=s61MJQVrjCmHZ?(-K9Yhz9L04!v!Y?j4#`q;Z}p<1~v$!vc{8b}nVP^{y<*|sEbv6YcUjIeZpq+DM+7DvGx(zN`RA1NM*AO1 zTN%sf2^-r6^V)wF>p9S5@?_PU34C8S{8mbEZkuwYYaRdEe#OKCyb(vISTI;xFEEo$ zP&W06P-$SBrOc+&;jELvmi6z!+Z=`{s|U<3y=u7!c(o3&aWt-9muH}qD6s0OMsBKL zfT2LxMMv&2zq%j(D5CU6UW zmj9O+B(^qPy;jkGV-mbc!OM)#8;fKBNjn~nn8s|I!h1#9C4{Qnm4o2`(3^N{%$ zgXp6N9BFTPq8KvTB| zNmmDB!x8;Yi`ZDzSd2_rd=6dBd02J&Ver+)^vlm=8oud%yd<;2vHaI3{aDkmP-Wkc z$C>LDgm)=i+;buDguCm}g1tu^>n?t%JGw2$aHrev?JfVVXa6mf`o_?3@0-H^W(UvQ z#;u{I`Yxur3x4^fvdw7VOnAp!i~1cR?GEKB(1 z65qd)sCE|Zzna#+S@4|7vxNzwVce0XCQ+8N9?NqmhL@L4y!CqTiGaGoEAqC*#*<5p ze{Ns;Q(Sh+RpaH20(J{lZ|hS0zj%s#gW~$B&7Nmf zlgN3iGxl00ED8w=j_5E|NeDgrQT2XD`g;rM?-JiauKruZXq>sQ zaf0cuWS&+58O9}t+xKXEFucwkanwiZV;%k2$(_Zqm$t_v*K z$#2KNt@ql>P=Rd~1EsUpv1r^;-HuJA!xNI^M?n5 zQy8m%=Iy+-urF9zTH8cMUrNfLYTI;1scDUpbC{R-KAhN_FU}VjvGC9K!YMnFXDyBW zCzX5g+EQ7l$_IwGRtnoy z9!_OrRlgK|YMsB-5f?|$*PY%pbW#rFMU=CwoTazHA*{$lPq-#m#v<%_$**e zS-_&?9&_S`mV!)-kb}0gf<|6ItRlnIPY?J58jru7k$!cdOvAK?=N2wfbgsFaBC|kI z@|dFJqPr)qrbsHW@_1T(xe{k|GmiKAG~eN#>s5qXae3GznKXPCfRWU2~2yEV$wcbYjiivEz(&t>FLbNUj1ugaZTKL($ zPu|MdAm%}U^JzJ&&-o(db8*t9PQIJ29b`QXSSeMTJu_k4gQ#W6$TGF$ng_zvc6JcEK^p?J~u7{!+3_XK?ZQSzlhbd2Y#> ztuF;LTOgu&st&Hp;8JBxhBhP$1W4yh_fL(9rnZ(1oJ8lHI@=U(->4cE@ ziU$l47DfUG!#U=#GWLd;J#b)|dSGLlQ5Pdq2cLoKVity~jk`BY;W4pn-wU z;*pO@YD57W|HMCy*HQ$gFdUR>Y4GWtb8WIQ-?xh^_G~)+PiM1?@tI90!_F9;Pze6K z@bFum2@ZDAT>mfJifq>`@zS=PVA@_`5O+1KTT1X`%3DRThJ&|57&+Bgnnc`p^s^dF zn9wojEK|wp2G-_J0vu5VDzb$KnDss|RNFc*Nq960ow&#oobr%s`;Mb3Z5vuWWt_Ma z6prL=`q1e9ph?8^MFW?dK_hF$IXB*po|g>HVy*`m8Kz}4Go886E~&Cu<_vRifv%n8ZajcQ*KbJ9O^xW8haIH9Gb*=b>blS0N6_NoufatU35Qv`Tp?j-Te zGH~T~6liw-@sLX{mU|^T~(7?tJINNlV1JjWrm%SzqOq>i2oD~-q z328X6<#seMFkN716ZP?`W=uY~VhRIOj3n3(7I+?5tpunk(%CUsHa z_&1qVMpNG!`qwbspZD*_@g#-AHP4;dl-myTx^X^{epC_v=Sbp~DR%9X?j~lI>O{)s zMJS6KMQ~iZ5j<^;>k@`LIu2`@Hg-7(Jme@WaMo7T5bDTyz@l)GfxS1$k<03zx5kbV z2S$qoW)+1-Hn|B0m`xV$NWAr+NPo(qryLI(wJp~1$=zsvGNFMn&@O>h=7j@$+Xn_d zfdx9Rl}`6bY+k~4LcxeJz=46K;Yh_dRYsl(ml);R&TBI&2u2MT;_X6l{*SHF|r&;W1q#q6=V94qi60gz#$&?fcfWL=Z+|kcJ4C|#m>%X;;=SowR-aK zAD4%O6XUAg;nI!8I`y3mZ!aj>98-Cq!|;QfN$A53JuZjC9aDwWMHv`e?>u0s*Km~a zI=F_BLqSQg!-2DcA*$#`=5fvFn`hF(E?!OAvel(+t-G*dbaukxz()$21-e)HN*0vt z=$})hvfk(EtAI^)_jVjJUKY^E6mmc@(z~BYYXQTTo|lX)I~>?&TwqtrDQ9!%8>2WONWad#6-oAsy_;o_p-(FMk$sZh^6g+I_N_i-G zae))#IxAKun`EK>87(3*K8#9IX>K3Ac=n#sTq@Ws!I1pZVCpl?rA#NH7^?#rrbJB; z6V_P3@=teyBQKN2%#&{%WV+~~3}(o&OHffS2xZgSz!*7;caO}$R9({x z6L?lEm?q}K_#nxkBj!rEP|J;`J!=g*eGH6++h;T{^}Nf<-;f}rwWdL|EV-3QqG9i( z3-@(3a+#R}8W`RR9Ta$MxtPI$fkA*{OU9?aaz zG2sA{)rljjOD6K9XUj^Rv~aur{{&B#{9=g|`}tcFSF@$$MDuo59Lry~F(t1;Ie4bi z%bi>2yweF@ey8$Zc}m%#jnOY7lBJ3n8W}gtG!1OE)QwbTVEpVW*t$a9?&Hf70e^hF zZJBRr$T+O$eNoVHL4FZ;^bRLho&}w$w-nY!&S={6dV>@Hng!h99$R=fP3Y8IafdBB zW1$TH#~$r~hO^2w2f1A?G&HO7FmBU+9=~bF2Idzp4$IfgXcKj-<@h1BA=l@Z`AUaqcrp4qXlb!ukX#>x;d6NtD-Hxc8^xAH^{lN7E8M>WLaK@V#i5IT4%tq(48YTfW3MH>%UA_j_?c2atk=!5|~99m}Ui3Tdm+UV_;%rU@|LU4L4w6 z{=gW$fj#y?Q*{DYu|cu30i*r~Cd~;<+yef=LXo>3C&oN+bxN-dZYt>(ZS!RJYhPHa zw1BDEfb-afx|&a=^;_!lGI-vmx2KesxlJppwy4jVz?S#C?4DZrH|B~Dm&a#eKiy^xWK5G7%s!$ zBxS_fU%;_JqiS=7rH(|E(H8cfr5x20Sal+*sui506(9K~7$^2|)-FE9rO zFgreAZ~GQE`2)XENq1`j$7BJn%@S2j3RTq~_}`ayTkd4{p251ag8TRiPUi;t=K?;Eh+ZP)W$-CuPjN*ZVZn#7z9^lGb;Fnw^-d`XVfg_HyH0do~juhn9bmac}F}=}}&7y&M$AkX!3wVWo$uh3wb=X|~W7{8NY+v0g~FC*eQdwnm7XK>DUNnr6^z*@bEHTD9F z?+3;iKQh|{8O0qqmIO>clEA~dYejd|g8r-pAEtBsSg|03VMe9_o6~~@e7|_&qB6`E zr~D9~aHcS?w{ya)ZRPHtXDqQ==+D3u^I)cC)kLwIGiM**oU>u#5>E#kC#|=Ri~U%c zeICgB7O*v5kni5W5%Iv(?L*VYr;PF&6zwIHY%JJ|O#X2$c`$j20!NDhw|@Cj*PCmX zOyF)cvNUq3a$Uh*oWN=uF!yRFOMwG>e*x!=%}jg_*?TVcayPif&Fp=-a$faE@%JvB zt{;~BEnxK%V2!`P8n3_}|AE!lfrT%EAvZ*_cqPw2&Xq?TI9YdXWb@)=Usaa9ygws> zJ-K0J@`VK{8&-Y|&1mJE&=D!Kq-(;mR}1@Z%04*2`#y|4{sO0Z*2KjdxK1tLzG}eb z+C6c*)-2hTX<@93jilraF0gbTU4QDDKXC6d;F`UGYxV<{T~bru8m4u+3rYnrsu!>{DzIk?b8K%~<6gkh zmcV^~fkliJXO0Eu4hNZAA1rz{OZQFY&i=imZQ9b7r21AR#S^%<8E{w|vR8g! z3m0GsN??t2;AnAR_XuE~bb-gvz>l$@H%ij~vE;Not>wOxTKR9V)O=w5H+A>K>8B6P z^qyEFz$$OR;H|`S1i}V=T*D5wqEV%kDMj{ttW3aqqod!28BwuUO|^=>svm z333si1^x*OWp3sOt7h*`VB1;55<8h|ZzK0A2d-wD6W_k?yRnIN)dH@C4qOWyPBhQq zZl1%n(0h!ft)wdw+rV$_D8vY8=xibH8ZdE>U1La$v7yU>0&<+*)Rkz``I> zz|dPgZ`)*7UWe@dr}KO#9Mbez(^#<2)o6`tz|Ob}tV=po`y9Ad1aMlN+0}D_qqpH7 zNACoVD{FEcWsZC?Ka$qKrlHNoH|vO34|9J5XG~PQ?v+ig(~J*DZ}v~?C^OH~SQ`D| zFmJchUX4@SS1<6Tp5gm_hVSZwJ&AibV-8F_=$y9L$wB_&@m%NG>ITd{4(1*LoL4{a zaD;G($8uzIFFdt{-E0EObuKPN-AhWb9E!0V69PC^8ZgU!U@$V^*gk>x?gO5NH7A-J zxGrDd`}e`beFCH51}2dRhR7C%5-&yu1qR{{*fxf6iPu!_iT2_Dav$%v|=egq5y2M?89VTQo57buqO(;Fuh6 z?sHVUz`s{3FAE%aeaBdWE0sAT=FPEa)}J!pV)o9l#r<>ooDW{1sgCaFo-@lnaWUt*-xVDV-^0JR5t{yj7wRpHrZQwZKuxm*)7i-s=lM`a!zhyO& z;aK^Ar)vSnrI>3s&fL$gees|7L59Kuw?Ah+=d3L3WztxXFw=o!X2JDmHRldSY6w~J zGH~u`J|PoyY~cszhi$d8|M%S3|KY|z2L5*!&c7+(oAP&0#-3w~EIYYAFtAP)d!bk< zSs|~S$y9Ifbi2bzt9|=VPhl}tSmws?k5lFB?HUD+QB7bsR?w zo-@>6O%K>PCxL}|1N*83?&fv(+vjm)e7LWo{c)PsgERv+djmGB3k%E}b{E^ev{=Ai z8opI>uT7T|-W25=LxM2c|{kf04Su)4=TzK_?f&b#`J;wier~c*L7qB-V zfMu43CCO{`@HWz@a`<&m?*<_`=3-%x@Ads^|Q#7(+_eDdJ-A; z#GLt%z%L&Cyu|N$_}Yy(LwD9ZU=cmQzCD4vuYsep;OB4MBhGuSsjOy8`|vX*;bT(5 zk>mm<-dW6(^sY}n$ELWRooimAavyjP;P>~Hw| z;Q{yc4cw~d`0jt;ePQt0;|c&PNPtVK{$ z(Niu}j{q@cHWm|)28Dx-EDSmwHXXmZbFL1HjMIG7X^q!f^Qx6ovR@;31_4M>6y{Kumj%n(BvF0cCm@SntNLa+o#3iGlvETqxHv9V2=nVqW&8$3p z21hE}-ph`!qA zwWrvbWo`3r)fdy&Me=>WpS=Xa-*(B=@EiFNluAswxOXn1E@vc}V<=kVq=~kEax&uxN zdrfY5EatPVaCxn>O|4N|Z)evQZ|S|?KAo06aDnr=!4V#3pABUkzn{&@E)hIBIk4hk zt5{gZ%ACz-Es~#44>Y^Eq~nrM#(_pgu8ai@S6`GYV6f8FSg`HJj~WJZtvQy@C4)l@ zOQuF1QD(Jp_T)Tj`P{`)MI^LjrJH!@JnfX=w+?|Jf&L3`n@rEWm8m}6R_svQ6kEQH z3z?WC92E95e7>D=;E&CsP-RB03?)_8pN-E$f)pB0I;H3zDKJqt%W)Eo*1DK{WbqWI z!hf1MiF3So^3u;F9+S#-NnEq>l0x?ytGXjBlee|)St=RRRg*TW@?+4XlLhBEdki#} zi@fnHn0KdbuFlmjE#CeS3)dJW-pQ1ZE3f-`bh7V)8w(m(rDkkyV6iB3U|tXIc$&-@=$KZND9lWLib0xbwQEF7$DHX=3fCe)CHD3%;9r;;(P)d@8W# zz{#f*oO>K{9saEno1o0-e(DDUqn3r{(!d9e?$cuoSeJ?<>-e-uxQeYjCY#`RGodpi zV4=#9tj}(OM@>TRd~;o$)KloT{)lWMZ?Ulbc6Zwgzt|@qH!Qq9eKv0`8eCR=&J}M`xAYJIOk|e21ZY zmL`Y7g2vmA`7r+x4vd+K>{O_=!yc?;?AWysh;9dQi3Wcc|GIYHcfwK z!an2dU+$8Y%rjF<9=2YdtCzU<#Zmn*_WrAD^OELIh-GC;ux8wF%vVmi;l#6&t9G3n6n03Rf1&p#Q zoSNJmxilvyP2sg$+N{6p)PsTpOot@-I2t&k3-^s-S_F*knNyEU^;1JirvNTv$+hQ7W1 zT?bYRzjxZkKZINH6dw_vW!>MV3L6@J(8}YpxdJJkF8chGy zBy648#N?-N%5{#Arr4d?tVa(l3O(h)6Md1Z_sG?$3K>egUQ3&T0xMcK8uU$1PZZxh z4Ld74lNM__v9dj|<2vEV z_{=0-^rx13shNVQ9CIXdNCRhVZj@)q~f)YaB zi&8Tf*Jz|TFv@qhXo?hiy@>MW@oNnHS9(+-M4+vpMMJ_-VA8BBLboRAXsMhIxz4mx zf+=|QXDzi!Uk>r+>2St3{Jb8)%+{A5kt|hxTU11zGq${Z&n1e z^sNqk^BG6ZZ0It*a6jTq?4Avcyk{2F`}}cv^Y7U%PJN4o9MUTs81Jw%cR=> z-MDLeNuuHUtP8hQ{Mpv+f6)5o z%CB5r69oo2hV#nzVnl3JjyCBOow}b;(rox(_KF8ub2xTpKa*_`=kO}cdQ_Gb}0jwg8Cm1 zrdRCFKMppv-fy_KWTSbZ0{_hgY7d)uXbx>D4ET%*Y~9Q zf(@Pi2B#;_zY@$KRlpG(!6fKB^P1+yt0D*PTHlXIwAsHkTiN)w!hKd7>8>|Ba2VE6`x;0lk%5^ z0{?Yb9CxO7*q(U+>Q9L&9h422GOd5dhGXHn>D`Vi8W6%9-qoUUv*)V;}sS%Ja(^PvaK&W1G%JTscpPp~~N zp5lC*|mm>X?B zjh7|uVzqkF@ZfWQ-v+7Gjtrn>FANO~9lIHSG)!K+o6W{jc>}}c868Xv44(fCTB8rN z%*tTl^=Ubj%w@7d?;z)9!|J8NEjCgUY)*dG^UcuG3Yc@od7Uz+PERMW5z}B&=)$>EMRYJ4oi$=Z*21Wq}-#7M5Z|rYmon(w* z@U~%KopFHiLIbZwQ~Xae&)?gW7tGZ-#nHvl;O)_tcAzC`MJG?qQELw_;~B0Jl8*mm zb~L{E)Ft(CyU3D8QJx(QpY?p^^I8g>5^)RWA`c{AxWiWr(W{r-q_r^+9FNp zkm4WiAFGaK-(br-aeB|4w!9e@_Mf}AA8E^8;dbSIR!r<-GFr|c-`$PlJjRxTr z=Q!`I{@uX9v7v)m<5Y@4TapHwk3w769f$0f3i2n+g;>PJUYHAQ5EL(HJaM}*?#odV zZg_~gSO}$ zEx{Z%o*P&^HLiO(gnMsj@v7)7tqfm!#=#-CrDE;%s-@vop)4*9Z7CLQb*61z68=k5 zSa@Z)mPDw^|Cq*aq*c${En2}S6>ueXi~pY|L0TTGmPKiaNO{X@u3xFtQtqptH&wjq zU|aCzwwwzaSiZL9WnB2l)y`3><`^A1BlQB0s>#6vH{}lL3;g9){~1{@gYC%;W?h9S zT>~cfMO)t%m|n1H{P#Bany^5#-Gnu78p1vsx9FYVTm8Ag_JwJ{4%SbV?Ii*1nHg=V zds>4%qJt;2Y@gBM&9SF+FU!ukw|APhdA|tuc@doK;FX-vw$HRJxq7icnm?G^uJo%kPD$rR%lCq?;} zs@n>DPD;9CE0_47i8#tMw7sZ7QWEv_fle10rv$I)WX!Q!-|N%w((;{#@o34IqAwU>3Y zt=`p=R?*Gfa=X;m)!U#oXv4pjpar+RR@~mr8u3x)d{RPNQUqI?MO&&s%wgYzL$NV+ zt!(wB59)Q>ws=Z^PGNCPXmK%!u->32A=vb7i&x=Hf5B^BC#{X$a~*vbEcNtV6}@YZ zEwrF-;!Qv6YA|b31@Wb>=c9XV66#owk-4lGR zUhU-->{S}KQ+;l8?_u@c!RqrP%PP6k4zy03%C_@DTZYBWn``e1JZ<|u%iAeC=|O3dx?oRT z$W<`J@LMAfA?0FS8elC zV{==;;>MAhHF3LRua}UHOqkfipD~VWS$3RX`|`cl!w*@Bo9;Z7@7x?cBQZ@S%k4_r zl5cHy-d(3%nu-FWZc;}DY+VcoOXZmU zA;?Mf;l7nk0(*Bzo+R@GjjfHY?YC!>U`?9Fvq@RsWN`C>Xm; zp!H_J_K*8Hl4jU`O?!58PUzQ=?}_Wmgz{(bMO*Z>c1XT{Y4AdDsZXZJoR=c6)n-LB z%QYCa=*C5*c;{CwNX{*|Z`hx2BbHILpPP(uP1eMHmJ6BsudL4 zw;kIyA*mrIWdRHO|0g_R6Bk=dTpl857~#)-{glh~nCX6BM4uczKle)Hqf1M)#FKRN zC6=vQay_x&udQP% znZaIK!Cvjre#?wC&zJ3c-M2#7>)ZD_6#7Lwdwl<*@%!2{=dZoet-0@-g5oUuPy%P zQx9HPt7<0wuflf5!Ii7De0FHH)VJ+Zf0d!YE^+^%V!B#`1^YQ^t9<5zjoj?-q%8^` z${D512vBfp5#d#H`EDR%p2vTGEA!1fm6MZHZ}z&JK&)=yGn)gvht>W znTczkoCs9)%DXG1+~$*aXV=kJw~l)ET#R!&A!O*K60u>fOP0{hjm^i-Nth%ZbCBDd z%-(xSJM6-aHI=OsMD@dVoOpAnb%KJK+HHZP!>$u#9pji69%?%!s+GBJNn&xd|FU~C z4SWw>`EqS-`k5xL>SenBW^H?RcCTZ*+`AZy=4)%)yLT8QIs@W{DD&UDk2 z%Fnla3bpTF{?pdC%)3A(ag+0Pz5jK8MZX?b=QBTl$Eibkd(UMa6ED@tKEa`$rXpd^ zcb%qcMpdeEMo9Z9d8+nL*(w&*7vPi>-4$@CL+QkZ8;iQD9v)Rvu2XFC>22Umi%814 zaZp(*+2Gp@i7BbmBGYHrev_Q!x-2X*buOpl;p`2WliG53B+i^Ix96*vjM{&L%To%^ zyEv=OJd>I1H|t_9%U#P0xyvuO6;*wn>&O-8a?w#IGi09moeE(;z3p!vvddQs6nQB2 zHyCkh_D@>GrMZ2@r0%8D?G|(EU5og#<+RDIoZ`tAHwreNHM@K#rPp*vN)e~Y*^bUB zL5WUZr-UqPnRzuNIJV-Us&rV;NtM*Ori+v(PVwOE%F}e>Q9AcebZz(LQx@U9X8)!p z_t`3KyOGrEC?>$5_>+ahh=GAYhk=2CL4q-av7L{HN0^sal9x+Fh*w&KS6YIDk57n? zPfCbiNRm&OUr>Z!NK!;tL|RltMp9H%L{eHzN>)lrOhSxLkXKcTQ%OuzSxQt{T3Sn1 zMom)OR90G7oI_4cKwXvHOpQZDQC3@7Mngr+NJYj>Q_9FhP*z$+M$|xAMoCLnSxrX6 zL{3RgQAb5tSzA?EM@v;zSzAk0TUT3KO;gQCRoPfa#Yk7%N>|%bQ`Oi|*V)XT}t z)zQY=#oEu)*4NWC$lEs5-7?VAFwH}wJVG-))UqPVEX&WJGQ}u2o5#Y^P0!3%S>D~% z+S%RK)z8-5+BwwRDZtn>-N(hr$IIE%)g#2$Dcs9FEWjlu*e4>`!`(AHIxHl}HzYN} zDnqDyQJJwI%=>zN+Oa6T?D--wYb7@Im@Sy4aAu9gu^IXc9?O0GA@Spf z{?#jLGur%Dw0mrtGg1JaxF} z?L2a2@0rJE&tATGfqKJ=KVVB4knb1C1ci)zt*VJnTko;Ea$ZtFbx^z@Z^ z+Y0ZMe411)em}puD)5zY=K1Z;tHtJ@x{~9+E@E==)T*yLOV;`}7TldT_05&-Za4q2 zxpEh4{Lt8Tv6xft4)>|urSA@`pM1T0f>r%rCdo(59uE(*D5hO#;FNWdbeUkeWrD-h zSe3w3UNa6hH|frA6WvsDugq+d*mfli zPwT~lRFNW)OX(pcEUgykVIHSHzG-XXv zq1(1Ka_K200$b1PNj!8Ah*O#5DL?be^Q^_ZRT|l=TN&9F>6R`O>YR9MS>BRevZ|WZ z`mRUw3j@l+L&_sJUe#ZF+c7o&;Ig9FY1i3`Pd=XV?C92chhHc5-o7cm-8TOAw~3j{ z1pfRq(BWZVi`eupSz@lt%%%%$?*vkW#rh^b6z&SNkmzm>T=d8!d&Q-UzWP8*uY@xq zk2;M@3mc(6;BwQoCJAasuBz+x$QAz<@<7AZd;I@`Hs(TJLv1 z{3Y>y2j|Z>h2|>Z5^2uPzwVxI|2Mr`{#i@KWPyqA)K+Q7<`pqdsgQhAdPDG1_%>ly zW~<(uUtL~8lNhF|o;_#HCDZ-Z<><63M;~1e%l=z&->r7lWcqxO44yg1z2)B$PPWU* z3%|~3o!G7LqSdfPF(c>3;R#dp>!!YD`duTj|3=5@iEpE)JoovsZk|P##nNCeVeKDF zYFfQQ&fN40*yrT6kmpQ-`Kr4jA6dn2*Io>HdhulLECHF#OP*TzU=V8WUbZF3y$+9MxOaKV~0Tli*u63Mb9whb*Cq0yF9q#>3hJ_ z?~kORhL)GQU(Tm#?UtDpzFLnbY}&C~&C;MJ{-5`xvY?}^Z>Ml+3muvJ_M4^ujVCHz zp?}W47s_(Gyz}Dp34`FqjT;$>k6)`S=~FtAST|8+7XsB~?J zk5q)X^V6fcT8=s_XHRz@abVjdyQax|q4@M4QxDGnda3#Q%Jau&xo`h>hj-?2tqVr> zQob{6h5c)fXvb|#J#1_1r+Q0=GvTDDu;RQGbcQ&e3`&ur?OJRYSx1H zJ<5(}_dcI4zb9gSjp>^*>AqMy^{P&{ioX=6%}=X@xJeEd&X@IWS#b7!`ecQuWm)VT!A}o` zPEKIalW=5Tc1x+&y@OfSB7$e}i=%pBsUr6bUA@;us|Vywb&Q`UYJcP}x10U3X5mc= z>-riaQ=FunPd!ld7eCpP7Pn@H4WpK+6esh7Go{J=GgH(Wx^842=#02gbCA=m&2v)u z=NAV94@)Y}Jh+D076px4VjY?@n;$SMb~w!C ze!#%pa)4Rof}_|rhj&HNjV!VY7+80m&1Dx+VCj5u)cVT@R=sKI+F4g*13RC~>9;0^ z3$H!D-s+gVoWx|-Ga(PRs4?AjP|GW<+TfgKv1vkYN_gSC&oL6Om^2jH^-i*xuDJA1 z<*HrSdzU9(aowd$HchFTTGW;JPk%;woV4>3CGlMsTs6NLnW(G}4If}elH(Yeh1i3b?KFAYo;KXhq zz{Fj!fI+6=yV*U*jwpFa@4Yi~ZpFDyj{f>-dZlBWf#xiEC7WC9$B#Z<@^}jSu>&X9 z@s>Z7m)9#a$hkF3|7_9DLk?{2*IJ&aByIZ5v&%%iIapor_Wh8>lW$#fWmcGm2ZdP)zJ=^^5%kr=W#&>TR*zQ;|39Z=0{6?mh)uo;x zX!!xQo_`xyohqE(%)Zg08zsmUeIw~j)(s~?r+@|~p9gn4UbI@X6$(dP`V<=Zi%GY1 z$~>NLzYiZc!kWEQ-pPSQwIzn*xKr0bfx0EO_k|^e%LT732~s*(nexCmDS%xl-O_rC z^TvzG1_vXyOjP>5OzfSpU*5!!_`;A)6hEn%iT(O z9^{EkU`=1ZnI=*9a5>vtfwHIJ+*<{>(ifB!h?lo1)VF;|@i5HTvPCW~Ad>&HaNZL$ z1(!fkQ=d{7oqq-kVpuL#s2*-8xm0lPv#r=9!IXl=Yg-!29yES>T=_KA;6Jm9-$&7a z2f}WLSpz9ddQfvQyp%_PQOBTn{f?gP5v8IIjIs$ViQ737x3gt!V4Ka*H){gt ztOs1r%DERl;B<3f%s z=R54tbHam}DIxvB4~FaCYXli;{ib(^v@_mb&i2rv@8NRJ^zyQ_1hx!~y6q3R(;hHZ zxiL0e;NH)`< zfrTA_{&sghY8Gjnaw-ZC;OR~R=6>2i|F2ZV{%w} z>7Ena6>2T}GD^>K^zvNj+4!R5+Vh%o>MimO?1wyh4n_9(L~`gpVBlq_y>O%FhB{;D z%Q-u5&SD8*beq8T@H*$C2<|=qKD5PHPD@+BHgf~p+zp&_66zusA}}$Z=5-KQ$_1K$yTm}TA>9D>n*3~STUZR zvB=13v2oSx3mxf#ADEY1;C(rP_s9#rTMT^nCbW3Y;B8G{W)ff!{J^k%=c3T|niJ1^ zm>n3T0@z}9vfX^i{qzCf{R7-t1#Hm)Z1-2TJ=)2+Ux3T~0i#F(+nxZvJqKonD9Q2{ zYi1hDNQOuSD8$H%sMRU@Y-D9AO>ycx;r_39lFrTMOrvCW)`vQqLj`VJUXaePa;gMd zpg|X>Tl{&pS*8KZCIQXL4$O)P&58xhMh(^O-x+Uxo3lZqbeBYnc~|%L%x)foDZ(2V zwq&j5-mqxft3`%UrF(z%oX%W)!GgKof%otazFQlnIC=Hha&a0fFkINdu)A_``8S3j z$=N~pV1DUjHKPeJCpT?4w@IgNvfxc+V!so_p^?)hj0n=GAW{V9g=R`GM7AskrRkE}F-zvS?t72x%_B%d%Tjp#{uEhqk zYMd=ww`ehLi<)g_#jw?K%avW5MHyK3x%b=?Sn^I`$vuN5hiCFOU0~q-z@R79eO8n4 zI@jE!4z_sZrO7ChuVr_(asJm3dKG;3=`8om|y=qfv?e)Icr7I0_HDEm8Py>YuUh+7s35} zJLjwkY>@>C3nX)m9GLV9V*V{;HvF*CqG6N50Vc=qjB9Uf^8a3Q?q=&n_BDGrmtHhy zG&{XWW7pfI5g`%W(kdd1ZGVDV`!z6T3<{jHY-{PXU)Ie}Zlfq{1c zgU*6Q;Svm?uQ#95Y+*HEx_e^k_6L0b7VtG+U`g8GT9VC@$iNagf$iyZ&c~VDsSg-a z4Awunz#1*!y)#9>C`x8eh*TbPMavIuWj6`K2^-X}1!^4AIa{=Vy~$|XQfBKCwyXf| zbp_mq6nNG<98%`tS-XKV%7Mvi0`nh7W(x@-(x%9`tA=?dE?=5xQ?)CDk7q@LRA$BEj_!wQH6=z9q<7w8=9s#J zRW646fWe{l6HcwlIdnkbgm(e683T)<1Jn61iRIb5%xrQU)JryXmqbc1iZ945TQ(=) z)NHM&J(sx_pM9}A*yiXSkJ;Nejz(Ocby?=@jvcM>32W{$Y;|76>v-yD;vC+~7g!h_ z7tb16_^8XC) zlYn(eORYbLccH|qk|{s(*ss}las33a%@>b8LMNCNjw%e<`$`E#cW?=>^EN}BHd(duiX zQ2vA)lWiva`k3Jo!0b}s!xe4Y!8lQ?EmN*nP}4atD}h^i&ZU(++*)%_sjuO57sxy7 zwo4_Td6{>v$=OZ1*(Fl5s>KC1AFZBx?#8U^&yNJ~^d@k$ZL^%>{de+y-lGoEt>H76 zD<*JX7N|MIc`W8%&)(Jvj4U4*0~6Ry8RjwvFzOmGOBSSa2QbMeFp4-Zbw22Qn9jZb z0r%r=t`P=eaRJs#e0bJ2%w1@Z*ZPL95SiJ7;T4J-NpIWV_`P6RE{6r>FEjT(a?3V z?I4r7fO_viA1$%YCXtnLJdf1waIHPSx#+^P`!Bh~*PYtF;gErD?t0%`&A7u0BCB_* zGaG&oF&1DlNnmeH;4mq;eMNl{b3k{*>(;RFJ?6JoJ9j@Z(psY_b@ighJ@pF=q6yb7 z$1$31U|;IMdsAS^(Hp!=7BJtv#xfy+_r(Ohi61zY1TgzdczgK))4BB82tTHlfU;Lv z-1la%cF6dz`XcQaz;?WV`-w+hL){5xn;Sb^MQ=EIckK1m+OSUt?dl|c(xyyo9LDIDE-j-0`5owCS!}s7ujx`u4Q%* zV5_*mV)Ad7!vpqK1-ALK*e_4uUF^Vo!~EJ#>DrLrt-EK}oNcc?dwo{=x~+z@YW%sj zFc_?MuAD9ZfmytnS)qZ6&)_Ym!dn9d_6Y`GZZ`1Ubl|dgU{Fk8Zgb#uy2X2Q0`HXz ztR@N^x6U!0vzWrPfia6=+iTwE(FLI?pG3SSuy_Y#ul!p2uHb&Q#I6Mj^)(4vfnl=S z)G~quZTK#m1v;E;eId@~@k^rDeM+3saV5ceW)Hn1>@l8~)O~pNJN(}MVQI1e+tY2F z%jcZZdCRDLA?ND1Tj#%Rl)KH+6u`Ck0*hh+qu##{%&iM}mm07c3vk?O;JvbeT}*)C zuneQU=hj1djPe(jSl(vXKDWAJ+h)}dOYCf0w`MY&p8j?3>l%v>97`YYOcda#3}7~H zU=$5tWDs0VE8*N5#mmIQ#-pLIfLXzJwpDIc zs<*G0a?FB?kB^pgPg0FZQHy-^GGg`QSv?XzuVl+C(DTeOD=gaLEvy$f%`8%CX{N}G z_&qKwAFF1|DjMuOBPbN6GeI%6_CtrD^D%C*nlF25yW9Ea?WlNhtFk-%%srdx^BtCx zPfRm7)Y2)~?fyo5y>ZHk9>vY7r)T;e;o5#m$muwTxJki-e+*5|?BcRZcS~GK+sN7> zWiQuwnT?}aF6Z8c&dvu7P0bwKMb;}mJUq|VpK^4ZwYzhk+n6_zT_WI^+Or5N|?ZQEB#RVNooQrn^G<9?57cQS_Y7%&ugWu-n z(m5_&QKveWEXZ1POe1T~M$VNxelTgT-Ezo_W1UHhr;cKS#Uvg1j*dijiHHP7J=+(X zxb=1|yW-rl{o5rYe!Fjv9`iUgX!dMKe&qOhy34i-pB{rl9cOrqS{eU6n`UZrp^-)W zO9^9-?HdMV0h>uDlBeVx={&;5ld*wuQqCSpmS$;|i>yLYIt7f)0!bej*)>of>?R0|SGtZ(97Kc35PsgH5wBggJ$$2kbculvRSiszP(olu7Wv)XPw^Vk?J~yfSH;ayU-fvjRr_uIi)$ffK-x4|0 z5-UEjY`@c$ouak!%Hh7rA8)+&HMLQEBbpe;sU}zcWqoV6!I2+LXCJ7rar}K$cVL67 z*uO7JnirNX6Sy*=P4C-;b|;0^B0d6)yfYX&+#L_G*3VGjh6BoShYc4na;|7x zoZvi>A=I$_N(YO=ECnXP0zGL4t^gOO=e^|?c~UnNS{;sDW?(bP;Lb8=ma;n7X?4)s zXooSEKt;G@Qb`i;Op9hA4FM*nEgb@8f%gt|DcQc1b=3-7;B`c)ZR*?D9*+b`?(#ED z)3$D4t$nF{^Vb#?wjXbg@>L#SmiTvl)31X{62~=;8J=S7(mC?JzOS=KDlc(n!j4BA zj@^yiY5x+K)t@YqUXsux=c72WJVTk^E1^k<#nCGG$pf~&LvFGL59Tu*3~+e$FwDwr z;!gD*CWp%l*czUxHE{7fDqh^cYs!__c#Si!K-eIZ*!fphxDBRrW3%p67u zjm94yNq6p;E;Oe=`I&5M<(X!=FLIKGU8(1j?jDYaIAO%1AaIb~x`Xj&L;;iS9Dlu_ zOF<{MJh9t-LA5UZ!V%qf-@C$Cm?aMUU^Te%VHx8jM!yt+W~r<~od?^BF z_L?#ZWvGN$Nq7C@aM;5oC^(Z%xXOX^tHUDJz!_~8B`cg#Ha>TFz@fwTPCzi%!HFyA zN*f1@L9<8AyZMc*2N)(TmU(KRz`$Fvs2(qejzIeDyjf=Lklg^k`cen8e9bv4Oenu1lNoibR9&%}3qtmv$TP zIqAz*=2M^jaZ~hxtHuY{H1aND;1qRpmG^yvJx zW+mHs1FBS*cIJl#N?*Rg%qr#3z@U&`DkpeS+B|WRZ$Z*@jyo(}PL8{y3zCE{w`#LX zU0`5O*e=eZ-6g!QL0M0wapKoW^FsDqnDS=sbnTeH^%J|iLgare@HCikkkjkKN$ph5 zDN96{wRTR4*HfJ#e7u4+Jo+J5_>Dt-DGbfJg`OR;_tqaQEOq2rr2NpsfvMGO#vz3X zQ+ZD{aO4&T3$K*m#Gn)IT<2b}SzHQZz<}Qu~@(m@9OO{u;D3FNp(|Iw$BjE%#c$%F6VjYkT+hBTed`jr;v=** z*+xRX=DCp9t_DT#52u6T-!QfP-t~Ufl$G-T?!4)Wx{}0gc6g>l(AjwZAMfgB96ZF4 zqF^F*L4m7(gSU#?1tzr_i#G0Yn`WG@z`-!}5Sxz%YYvwqqkzMq=thMG4wi*CoL2nJ ze3QbYx$j4-tbf+FTYe5qEFT&Usk$>R+Uk&ADH~9*WA@!-mbp&(zjfQ9R9ILV_AqmN zU=(jSpqwqzz}nMr(Pu^?FSGgK8Ls(4x@-O&22moE|9cKD`5 z&NWs;QQ0d84)Cn84*n_Q!pMG;sgX@lP%!<;!_cOPWyYZ!+Lzx=U^~s$!t7Acq0iOy zrl~>EgQ3x;LMDAh1Mi8(4bJU0D;jtzn0Nl3lf}I~IIw{=pfCO-lgEs<-LsXdF0++K z>sB^z(o$v>Sg@J#Lm&SQW>13_nHej(7BvXoU^2eYa_;vYAA`Lwx#UbOnoR$E zgcClnWT#BleX*WX(Ieu6q_PI@&P&2l84FHjTS;@Y**Aq0v)O)+2yXlVOeKiiYVni#S#^*nT(>bl|i`!^xNv zjEo6Pr?gmOGv*{NW>70&6WHAmc%ZGMp@qppmtjH!yGG+jj&&?68rTFFcvdh+ukP)< z;i_(J)4(+$h=GM+#Wc1AK8+V^N(5x@b~V+0=7`j3;8S2!ejuw;VIjo9$X=+#5#T>T zV68y|ON>OTow0=41i6E60^Tp;a(pfLcB6z6W7C2v_e~mFN)q$8{%BbCYsreJeIcu~ zX78T)jJf&j6Xu_qJij?jjV*a90-H@g=-=nu8ePy@pU5P^v8<;sJKCzzLOvT{G@kMUs7R1jqhI9LD3bcPP= zOP$s|f{aWYHaRzruo+BcPheDRnDj=X!LnE>`{W*Jh87Qt#j$VNa(}eNt~qbuB*1ur zfz5;A*8xVa6J~`!#7a)IN`2^i%rR#|!Gw^t3|t(Cm;(4XH5$LJnf6$s!JvXA<^Zee z2Lb-2f^ka0PSnqBfZmmn032&Cod#Siv6q z??f|C08^Gld!|Dp_licFv#d-JnhZ-=WI9f9akK=TV9T3xwNttU8Z06a=NTt# zkz27KKf_$3o28zalRcm>$T##ohYXi$1G@(UhXO-=vyH}tme>g_cMO}F8kwF&9@402 zy0v?{rbD+&;0fT6PNcxTj zUtX4fnH?=bi45`;E%qG^%l0;e&QwsyU~1%O;7(w4_&fE^iCFm^+9e5!js;DU9E`;u zIcjD_v1u~MUSRfIaVhVBYvGE>yah_$ol$QlFh~frh&Qdhy{PSB?ycNEalx965jqWT zG8i}v?yt;kP+QRwdxN#|WYgjXCXEjhsvl0U-#u~l0z|lxqA)pNajVhGi*uOS!UGgat{z?(xr=QQ*uzM?WXC{-XLZi-w+j)P_gfD2#<7GQ^ zwMkH;Noqyok_-dwrY=31|c$mmo-k<8bOX*L%OvwwCxUa|bmghmYm*%F8J6NPLtat*S( z8F+p$%LMHyzui`%(JK7!!Fii!=UEL!gI)QUJmxgyDYGqPko9PZ`N8IJ;p%58pQ0o2 zPQn+zR6Xf2+}W2nX-O61HV?ZW4>M-WwhUHUDkHjUm#^t8>99qTPkmop%ri}m2+-RR z#Pp)!DVtY!&z7@ni)_Rmq}ru0@+|0=U&jBWqv7G)qfaXu9`Bgu9J}bi%Cw77wg(NT z9#q`^B%$HLsRluY1Gg7-I2X#~o?vzS!6ar8`N?29;|x!!6U<%?>FFKJ>3KIa3ff9C zmR%5RSN|+#n%5-bC!>DLh&`J_0q6vwAuyy$=? z-;umq%l`Ip%g9(ydv!oE%kW1(Lt%S^)?x-h_L2;Kn+hiJ9Sv_jbcB>nV5ne_Yhavb z+h=yFLAt@^QX%`xsrS^q+t2mdOnsZUN2h^bgW+nT?30WJ_67zH`>f(lN6!y_E*?#v zxo+ROY;j`S#MKhlG)=yn zDdh`Hz8V(M4_Xw=nWtHVt7hzyl$iDC*gub+k|2SA7_o{V@fVo_2O8w3F&x>)(@Gc1$SoYSBHl0QGaS+^jP53u6y@1XLFQ$^eZT7cl#+7EX^nI??f)@xduv8Vc+6EY>HoYN-gW^6-oP?!~>ds&V_#6raVbSmWLr z$-HFeV^-MF)V1b>tUc2`OD1KD>w-HPWdg$OI=3#&-1YJB%buLZg+CLu(mC1^7@s9H z@N1-fed=#smTW!k*-{tL!-{i0WSqL_RQ%BC^09x3y4NRle3Cnu=g^+Ffld5h6FVzg zvPD~{2E)gWsKhN|QUWb;KUymc+GajvyYR4WXC=$e8|Odi{@^N@z!%W`>27jew41qJ zqfkQwH$$U~14|5trv2=Ag%`#L=j0UUG&yny@>X`v{%>pcpwG>kyYk2j(d}9_SM;wg zyBPQXtIX1!`W{t!0<$g^FfOdDk~%Tjrc+zG;pwu+JnX%UGB>I+<)xN7=V?y=#BXqu z+k&xeLH`#0XWR#h7ORQ96=}_0t=K zJQ@TIT3jBq8U!%OUJz7VF;T|gjLZZ%wGS8Eqf!Lgncg=tMTQ58#tJ=5z9imZQqA)# z@RUB=ztY*o^YtU%E~yumo$%{?#jl6n+-!^Lq*gGlSl%e`VDj4cbwU%W)cAkfq%b}c z>*Mf{-6qa=VY^u4gN94y=Q%2VFMYyr$npG6elfL0)xnG$HdnSZo@P5Y*CPMk0mGOb z6BKvk397ce{?;wNDROg6=>r1?21X`khYy9*@@}tBKGw(h_?W@$Z81JtAsPw~oSE79 zBV05NJe;8uB`al7U~nM0k;#HRWwdMQ<9%O+ zTDv3loffQURch&;u%ob(fzjv)dz_qM(t(D9O|5da6)cY>b=dkPY?Jt!4<@I76gQl> z|G~^ZtY^gatqKa%4&7|!7Bb4%ao}MS8?UUP!)1ZkSsp!78y#n}y=7|271yfs=Weae~1Dt8Z^^t#(gmIB;;f`;7yc#y2L|xa8jw z+4Oyy??bb^<0edxmD$`oxL6DxoMbhOwtV_0=$zThjLGLr0#g(iSXl!U95@9++#T3u zC5*cRI2{xmS%hK)9Jx&cA2_fI&S79=Q@GB`w3q?i_q zdQ3RLe92S%ZYh7rcLt_uQPc01-tfE-x}9+SJ6dNLi7|o-Xrex4Iq5)O)c~{kYzn)L948-?1*AC%gY9 z6Nhd+1JhzTk*GGYv>AbJVlS%q6c6ZYL^&Cz@ZLV&rSKGVAkc{R3O}o;ABQXJg|=xs2Ni%`74T ziR_zMLKilRH7Fe7aDTIB^ET<235;xw6Piv-g+Hlr5t|ylj#<+0i-f+o{W4b87rC;Q zzb1JgGLnNMd$@0H@aTE>5Y(qU#nMU^AQ0=;+cUu4U;a7W0uSIfg@0%%n-^p6N6V zlY=@LC%I-`5pvmiWof(vg9LxjadA!2HwJ!3CLIzxy!3d(N~1{|4~y$Cvl%^gU{^W7 z$l250eC1$UMfxh?j~T*yE?;S5Q)ZaQ?RLOOu;T!;yGOH{MPhdX%VN=59=2~ef?Y;0 z9HPY!IPkUvFe_C2Q)CgIz`)e=je*smVbX`m41XjHG#ACXTX6-v_l(sbDCZ4thE_`ek8s2Fv$hc9d;%9Cu zJJDsqWZ@r6F4-js3-&Pn6WSHgul4iz*0?jrC64@P)8;%hA+O}I^!*JSyuFM(0RaiT zQWITuVjgxyTv*h6!XS)8Z^{|IyhY3^H`Le{t`uCD^MGgZ3Ra_?oM*P~a^keS6BF6u zqOW*2+heiyB*FKS#FY*tbh2ps91{5^p>E{n6FjFUg~`iM@4VPze&r97WPD)0?P70x#^Pxo z4{GtoIx=znDH1z#r-A)NSl%}VaV|xt_g%7NIr8tcaZXp`lxH`OoV0T@{9dzBHf#FL(-$Oo zL@%FlbPl$e=zGOPM094BDVv+rolLe(mAafx3QHve5;ktefrtyV74)m<@nRUfmOtCIK=2mU1o z+FSyZv>CoF%6^%7-n`C{Tgt2I;z^#@f--j%`U9k#OEx#KXDmEpDq+lA=Cjl7+J=_h zaT&^@eNDo1R!&TodB_!XVNOKF4@{V+ zo74jgEQ-&M-8xXMk!j@i*0gX9n_J&O9tYtji=7YHyl?ztk@x00k@#nkSmp^P@tzGW zIysNF?R)AX*45Bqld(W#zJnuEmz$@T*xHX%Yxa~dDO{cz$DT25`yOVc2_MeXKVUk= z*~lTK*O__bdHs!mstFk8EKy=N}lR};Ji@RBAW>>8Kh4ae7V}8-BWPA z_xz$F&k85;qJox9Qi~4-6>N=orK-Yzn00PZ;_}$O6%q1B8aOonc^_nBZWFt2Tz05W zHtk>J9|7pJc#Op38wlD9q-el|LcyO#hqfzpXn9hfb21lB$ zcQ7kXXyl7=6>6EF#?gGA^}3KioN|p5SBk@uj(!%EIckOuhiVf_|NZIN64JmN)4&yW zfIZ;=`vwM%4Gg?z4zR!A!Xkd%rm#~a_JfedV4&PJUKjf)geyyCPd^`LZ&gm6mZDbskd4<{maNC_OB72V+% z^MzaHR@CG<5jh(Vu-nX)yU{4u6F=R-=Wu}F(S+EIm5ve%SOsLRaO~*h>tRq?&stKe;U8c&5kn%vHr`vDQX$gXk=gDS{6sh%fE@FDdfxF#mTE@=|eIVg0)L4@Uygn^@&E2Er?qg;rp z!W%~g4kty2WW@kwl^!O|2~K)FPfez%nx-(D_Jo_4Jf4@@Z2s=1$s4DbAZPs@0<|jt z4pnHnIIFmr^*FIFNM^5?&XCj4Tyj8b%j3-nCxn6$11cPBlN1e84@%t#WPc!DC)q97 z&?NpZ`B}mYp$Q9(Liq$Hw6?20nII9?F>waJ%G^twJO?*8aQ=BHm!1Fp{_HbP@^&<^PdO+j`P4wBS>xDY$q9}A zuNcH%G_-m;O5Hi6z0FBYf>FW2DZGzK<%yH#j3&J~y?QU0OmmdyZ+sfH>}k!pbUmMD zy`HBLZcGhTsfsyI755|ZDinTIKVN*(V#)Z;7*gZOtZCsv)GR%{*w;wvS<{^an$Hh zpZ@e_O%$_P!DF)^XU!>1dOe30IXS8DXehlo81}1C@k3(*gQ|Frthh*1(!5F6D z$Y*en=f(k!lm?C~%4{JHwqg$Df(=$drx-50QS)qIa^adK+PgI1(j}1t+CO@=-}t6x zbsJA@6l6Fcv?#gxNRwMjsA1{UQ)inD?wsw&3ov}^Bc_tZ8k%ceCZJovu=yI()>|DS zA2cN|T@YnaN%;E0^5{P)@ePhyTr1Z;Y;}3Zz;oh2_PvLfbD!8IioJWG<`L$TmNHTN zMWVRsLCGsj1`N)+8BKx)9s7UreSE@@!J)&~@-p+z{aY{DPWK!T>qyW4Iq`{{Bj3(h z{+dsOKz`^g1Xq<8*HrpU9UJ z&s3JIZj4y7GimWyYE}x=5?ZPY!laM zB_20@Xw7t)(^iU;!}-U(CgYFs-m}QKHnb_jjaZzumpH z-{XnI$&>eQ9lZSTKOcJ(#<%;D$0;G^;lX@ zns|zV$1?}5TaF?Z95`h7Ki_d<6KV84rm22_NqvgQcaO&27cbMr78rcFxM@GgH(a>a3a+5N5mCaK~XQrRGw- zheCWev@aZDk!?2E(IlC1kYmFN_7ulWa?C~_oMN|~txKJ1;B)!hf(dEw!fhCqY{{A- zc%pb^td5jXqf^s4tIl)#6gcnA;Sua0oOq|4x}9kP~f#~O|!FLc*9p37+B~@ZwUeQ6lW<;hjdpkdi@Obm)fcwq@cc} zk=-Y8=FIbt40$gvxT*e8r##^JthM|lrIX68gp`S#4J={M|8q#Jq(ShD)7_iAiE6Ir zxFk4BSX9an>V0w2%}7)F;iU0{Y5r43e=)|L0ewP z2NWML=`Ud1X`^6q^~5U9uXgV8?X7$h4}83MmGywY%N?s`xX#OR)XJa|-I~ z9`RL*i|mw__e`JIAT?#ZlmL@Lfn(O01il*wgnu+H7IV@xVNywH))YD9ZQZP3;jDMY z!S^yl-j)XTj-olc4rw-ryt&pSnbSBYpV6I_+3?FnwVXzFkpmUyKN)fJ9(<^CF!54J ziTlBs{j+uDN_FG*S9YTeM0IiN77$xMJr=|z*~hbGN4yDC09?ObzOJfKk|!&$%QAX`QQ*NOxG*m@k4dYtrm znsTBg_8cke|FR?^L8pIOT6%Taq#H-Kv-N!6>u~vC8s{HFx$0`!KZQH4Gwu{$m*-cK zR?^77@TmN6hX0m~3Jpve0*4gmGzv{%Zv6n=Dm)m}l*n zIJ4h@-Q&CNj7{owNpJF+q^2}VeTg;XVD^~F$ZLH3g5UAV@>2&ztx6PqFTD^clZh$) zvgyM=gNqVJ*_#|x8V)Pu9MIsgG2>|}zj||-ID`0uLwawROiw($aGgnV!WxMsjXLv~ z%qtFQ`ZViTH0#f4(hu3IDZr#J)3lMFN%T#V*^)-Kmf9Qn3`#vsdIE<;R2+Z3`nO=y zYXytO$!X;h(SM95iQE;AiQjry_|UD5veO)P)LYB-SnuSo+u8iY<*->!`KvuWDSU6_ zMSBh^9@(YIaERCB-`pn-Q@>XW861>&!e}z@>_P5B$^p&!$0s_r7W>)nnI@|cvM8O^R)o>Cf?fBf9P4=1DLf*qW!9GnDaG?~j>)#zwo zy|9Lr$ARO8gVF>GQwJu!FHB9xt=GIiq`t&apXabbNTcSGCbKz*)aNj*PTyt!6 zaW#L_jsI3`6-*j$4k>6j-tu03QjO8Uqflunr?8smER`G~Rxc67xCI=Y79TqW6rJ_B z7>)iNVdw8Rbl8`$aghs)P3fB(j*-uo%&>}2*!SntC2pS?vHM&eerokTuV*{^MZrSn z;~cVy43f7Fx4H$(=oLKJpxhfW&#~!~$f?UGW-6rjviLF^F}wH5xMcQx2smEUa~#ja~ey#2N8xWKm99YS9p9XmUBz2@>Q znV(wj+|_)t%)cnU`EMx+ZD*UC?%X-_j}E!ah@_#Yp~< zbLV&6uq_rWC;suv%T{(Uwu!q~Eo>1A+N9jC|0!T)-*kmJC)H+@{aU1^x2NM#w_e?= zPVOyjSzT?LQ#u-1WJ)wvx2Y?<c5#_tuve$kgrH@bs_cTr9)~zpXGI+9FmmEl zo3VXD4ePY<7Q;Ss>e7M}|QugrZ_L!=ly%XZN0 zI@R6x;}P+fRWDh0zTNcuiM4jfhdzOTGRqXs?_pm9IU6r@^D8QdWO;DtJ=^`B_m2+O zqeb#o*$=kMHr>WOTUa^BG3Af(0qso@SA$>@o~0(!mS|k4@pt zyPzs0rL$oHgMF@nilEUs0e-=l3eLRX@;1(V3I9_o9(8EhI4boC&0#0CHT#} z%p+X9dzjCQ#M!-kWFuc+lD|ML?y8xchFH@gJN<@^j&Q}AqtX$H$u}I?#QYK-9!L!g zGIL5#UA2}LmwXkhj z)&|A?K!%Brggw|ks*3pK6|NR-6|6JzQZ_lEz|6E}&JQ15{^TA*{i*2>9vfdi!}II( zp+66p#WQv!q>H;5R5I}KbRS@}+a#IFrym1_mNjfs6F6e~apL(J389H8DW6~z zR#?bw_rifEc10tf%Yx=*0S|c>|2QI&X43XS=ODAY!(la(g`EKzO1vH)9N4)GX0wGn zU{>7Vz`^pR@kBvpWv|b@nQD?oUXwh1Q;H5KuejLjWUx@=$^teOBZ0=rrU#fx53rfK zIWTT*OJLZjwO3>bh=d(m&PE#gx@N031Eh~g`j@V@LO**^@NUK5yv3a+r~|0`M89qZ#Per3k7$juM$xT`RF z_z5^i2uy5oQ8^@FWueKNm%uz>T2jeUiNmiyF>q)tc_^H2z@*`@P;Bd^HU>ckN1<;H z%xXU#3i@aGFy7gs##r35s_)5}PYQ?C#1l&SLnirvsf1WeYCFggR5@VMpk78Va`Pc&FUS7Jb?$Ar4Mh|sLx`}b6klrE9mzTW3R%q z*;Nl_F1T=1{fw}m>TeH)j}x2vXFXt1OIpntx1)dR?V0BPcp8;mW<2MOPguarXy~++ zSHr>h39Is|%1+5gPQv>ZuzLn5OW!=uX11a*-S&B=x$HO&2{bz#Q7>Vf z$fj|>N?wgQJ*X@&INIEEH_MdVcTt$WAJiX8v(m3B08D@r&9 zyBuiozmOpyr#rWoMUY*~Mp4LADk~}ETcbycve1MXZ3+y9SIpLC8g|_LrF8emozLvfePtbX;iAI}nvPO;myNe5up9KSvlNPJLQ?)$XvCLm;>58b{uy^{fUB ziQ;}ONfO&vI9=9jWKr33``AnkHXDQWk5AYg<_cZ$UBu@hcddTqNwpOIDdLA*nx1hz zbm~yd`pw+(pJl~X7OM+0lwG3lo_BH1TX#*M@axAmofgNs1xKEh9AHs>aeytoWsA_1 zgl4rLitgH9AMN^U@uaX-j$GU#ZY%oYu!JWbscSibqmED9*SRaNbGwk(bd?W<0I1K zC}PFPsi!!zO;Ob1;ipH8a%~TV0vx61C1zRmbE#>FEPE(&uA#}}d`p6ASeEKskJqg! z9P=_(u-hfEyC|LEd*WEb;Z)@R_=?WpQpy&<-l2UlHg&Z)l!EU(;9@9DK2qy6hE?nQO|)%<^zMZTe84nZ@K0-QircNDDclx;6JuPWQ(J~ z13ng;3(VXH*qjP@{;VgT~KR>t!9^i!4jz`ImcgMvj8tdCp^;bNTebh0cW=Fhw*h zeRv>%r?7uM&!i59td5e^+n+5y^d3H)A;oF^94Z%g1l<-mPv0nZr)-ZKw) zwmEPfTOgv6$n%9k;9LUFxd%K#i~>Ix1h^ChV;)@0dLZQLz!H@pl+M5w(7+nSz+%F{ z8t{PapH!k0gCmnt10#cviIM}8LZb**qxi7~5xIv%k0?klOMI)ef$2#eiwC33j|Vqp9NxbRO1D~Yi)Z1HhJ|8vic;AJMPnQ# z`Wz)J8ihL+N}M^sGwJ20GlddeilR-4M)d^}^Acs3yK!2-7oIm^<~Q93=bUXB^iJJU z<_gOH8ucQt=4kl0V{HX5Sc4LHY+dPPyCb8d>tXB~v! zar8$i_TPFWZgY_1oPxNQBY%zpXBtEAt^^*7CF){YYz_~i(->Iv7?@kq*(x5e1hBKX zG;qyG5GYIhC!(}~vFvQhjU|kK6vP=6g}yM9DLh~iSj@O9XyP$Fk>e~nw~sLT@ah~d zV%Xy#q|(TrlFYB-D3GxrC8CW%r-4mrq4W<0=7?ie0R}8i4a^n`m=84^Vfm~tcwV%@ zQEys{UWel+mG#0+5801ilsPXb*QID2{7~rD#n1N&do)him~?a1x6C=0`Bmmn>jDK1 zEv9)*3=vHV5fusSY%VH!=Xq3Jyk9FgzhdC3+T6<1w?%4F*dhgJ_<5 zlnDR7{$XLa$QRh z72ubMdnlUG==mXuy)xjF-a=suMNzfjm|#WGc?V1KSSNn&qB_OhCgcRECmYH znFmre4^%}Rs5N3>o}=LYCtqOM!GF@97$$U?vN$yO-gzF;%ATsPpm;!xUxz_!k{J6A zaiKp4Sa-MzEmP#-NtD)LV?lU(ligFZ}b4)gvxkPx!RCeowZlR2#MHcObjH2@%O6ol< zx?cLCBvIf;@BItRiZ;_tj%V7sdB-a+Qc_~#sAA<9WX;aesWcNFCw9h6PjAo}VGht6p!pF{zfEBrEA{8^V+V;DHT zBt(8Yz*pqJui(i4=~|Um14B-N$R`J`TmP7Y-5psbJy1WfK%K#p{eS~|oddswg@N@I>=X`AinG&^Ob9?d6TYxIKcTYFM(yw0+A9$aRyQG0>+u;2QThW z5HC9@RK_TMfI<8oqeX$F$iGGzhD4c3zXWzR$m~mO-hWWY!%}cVg1C?)zsFCm6%W@W z8*%D1@_%7q$dyV)9h zL)3~f`sB}OLq=h>&z}tPWewNQo&Rud>kTfuMsdGHfmIBhzt^_>IxBaP*>s_W{jZta z%?Zj(o4h)Ye(OmsyBeUFTRyiU+%ChFAUr8rD8uWcDDzb0$=}WIw-Kq%?#2kk1jUR~!_$a7$o=gU}I&07ga54Gr97M-FB- zXxuX^_<8M+{9E=P2Lu?t@-QsS__kcUAaUxSBaBu(%yv00u@5#_KG?9np*1#9bizWJ zI|pSu6y+uy6pc$1OK5D``{OW4_y|H{*h|43PBbXhZhUd#IJPn{Jrp0an`R@lL?v(SI;7mHU$7um;odcIxD7%A0;M(V!J)(iB{}cjGf2%5b#_pvj zUb0ZPm~p-8Lg_vH;%pOGxJ-F;7(sXc9ZKk$)S$7hfxAbkAo`C`v`y>7a2Hnx7kOQl zyx4qqhCun;KN%0$JDQdhFp6+Air>4O&-hZ0vB$-;VT0$FqqC>BMlLKUtYJ$p@JwrL zI~Xcj7vSD6bMmOHT+NQn%@XsT-V}Ktdg|@R`)r54%(2M(656H8wC(G%Hl`&JDaP$V z3TM;4n=2mTy(!Ssc)Psy_XFNTXLl;_EGp)_bU>_u(aY)}j}>F$8yoH^3<7`rdBkGb z@`PBF{zmT$UU&Al(84BGvA@hr3KJ$hQWa1X5>Q+u)F@J-C>YZ9$6-PKzn@|(E+)@w z^;psrSh5~m*Eg(oe^BLcpl8xP&Ql&sOB(o3dGL2N<+&-a$!%QW=D_oyTcAL3`kP5i zN(nB^vt2g+JSuK2cFs<00;6OfqvVE*5($lB6?58J7fL*l`26mk{IS37?SJlA>)-W^ z=9-(}!S2*@ZgFSeXU-!P@k?)nYbkAGuHx9Ufz9EOS82!D8$NjkOHM^}w6e`B&rD4+ zf6kt<)O|+-&kqIhWepxZEP64G!cmO+IS)B{8u&dP{$u&cZ1rHb;Mb}%)@&_IS}6x2 zQXK*d9hjILL~|YLo~)VjHR0gjge9js*cuWJg=Vo#I-n6&%W_D8-Kjw(Sc|8p=^y_r zCPfzo4Fk!g84ACzXRriuaa;*z7x~I0V*1jPOT_6RPf4cur7k9~4|k1w? zB!#YN<&Hh-l*S`@B>7mz#YwKKh1Vn>?>gH2{Cxbr35uT&dVhT*xyoBdKh&}G)s+RR z!CBX)7^$8VQ4gA<^6;ff*JL%nc`l7jUseccc-M1qe)@8Jg1pVDcO8t6k9Lbh?{xV1 zNR{ixdfyh4Or|F~Q#Sctb4nCq;Sg5zsk@RoX;PVJ1gpxSwv-bmrjuQILoPn zN6e`}-MM4BRNU^Wi3<`AHde4RaVkA2NEJ0|<54eKk|F3OB^i9iP@(k-zy04Y!a_Eh zmK~hlp%IN;n#&b}j*6H(+!EX#{N;&Dhl{MMQjoub>JedYR>q_Jjx2^pLPC!U9Og6H zbHIsHzrt~vxP!w%Rsox$L@^NuovUHOjvJ01i7sON=-ln}PO-zdIbh;3-3=^iJ_!do zioJE+B^W)D>mEE{V%M9oT$0b?$%9t8xSVOse4Ez(WaQ3b5@=vvzVsT4QfJ45$PA-~ z4~!fpJqzdj6ZWpy&?4lfv9XoitzhGvCCWZhhnDQN+!!r!s@IV%f9b1&HolaOi7pZi z9fD4kiY*S#>?tZjT#7q>TZxm&$3~9Oy+J<%MEN5!mxgkA z)V791#!X2Lo9O%J!x8>mQ^}|NMSl~n^OwHb+HTSsu+c?h`<+9a>f3HyD%9B7XUM8@ znW1O}x5sJ!H0t6)lw@WRJ>iH?e=p6Rvlf?RoHbz&k z^}B2J)O6=6{x>+n#mwlYu(YR$`HE8YBv(mynT;$`4Ihj=L?%p8;@~yvdEk_*Q=Ql@ zTY6{CzSLH!S4YnaMNDYq>+le2;+`;7jIr^80#CceVF|7stuAvOhH;y?tYSFG9@5#! zk}S|H7;;orPh+{6k^*CS$%KE+f>)MVzx>U>=CD9Rs^0;IKxK zA+uUXqQIUDzJl|@S_~DE#H&0GDR?zsym~R`%Me8f(j!?*(#s27tHPXJQUc4 zB^=H_aAtN^aB5wk*}&}7e8}L1BYTVjGv|~5t@!=Gqervp z(}E4G5*&`)F&`SGel)O(PhjNEX=wf@IN_l9f(k`R4)La?vQL<0El!AXeG&-UkfgZT z;gQ`2RZB%D@2Jfh0z$tV*XH~@?09~M`-^!eCI!D?DwQ%4_dmDPRWEO1w_E2SX_;R~ zm#=u?8uXHpKPy1_{?Y_iCy99iw_YrMba0}cQ^$>iQ=aeH<)Fi^!|Ei#vyfZL;34;< z1#U_iADFGrK44RmW>sKIN))QJG*aa)`}RNua-ChFVi(tOCoz%x7*eowJ3RYY^6!(QmK(-;rg1^YQIF6C~ z#G+Q~k0%w9&oD)8kvJo`_N!lQnV#wbVc8vvD!770{%zU%@=&Kpf_b#wwTGOEjfb4` zS1%H=RV@!a_=2x$`BY&M9$mL0^yUHasEt$sd_Dng2@R?!jl%X2d)uf zcdj@jK8u0PQ$>k?nFgzz+Dj(IT1K9LNxq%g0nE!TF|fy!%(ZY z;ecUg0JCh0qTr4dtzE*5ktePRw6h*OFM1{OX2c!g`HBf4fLOPwU1 z`gg^OY0YX68HH*JqUAmra0TC4z-l7ktn_UGyJNSb(6S2YRIYQZ*LxOBeo)za%9%mR z+dF}Qr@($Du;sDb07Q7~|Tb)@Qc9 zYBWrEIQ3-$&jJM(rHYT-8K(^z_%xQ+uBmO!PL959ml7ag&7BvzL`n2))Pyuims{*B zSr&wDm@sLj2CI6#NB5?wo1A5%@|RqB=kojA+n%xysrMz#`|P`do-Pat(vZB9s8wrm zWafnC4*P|CogC4MO<7z{A`3Z`6&%Zy9l7j%_J}k{IOgnWxZ-!r-|U~9PT|>IQV%XL zG&~b5aA4CjXmjc~Si?BYVdd(b#uscpGt07>ZLq$>Bw}UDygzKpA|r>8ZBC-h=MFMx zqzlYSEBeE}BI{dK)x-`ag}ye%o{f%|`dTGde^xA2_Y}}Jmw2RB&OXV-(?m$^BhPYG z#ZUp^*euUXz1G9cW6Z3Ns_RW=< zjE1wG+vsPA^F6n5OBGL2;AjwFN;cq%ZD)%rV7qnLwjqGsK7swQF^Bnvq8;M#Wru?N zTA5=eu&kS?!T5|p!JwE&fKjx7QSt%H1O<-Gip;`piJO-hNfsnLE0o>(CQ&C9%(Vn}e zf#?qO8j+1B%kz9@zwJDiHe2T&2>?pB|#~H8!4sW6}dgPByA6GWrCr-N+{&BV( zW;980woplqm{5O1O;Bc`&anVN28r^A)3}Z@@N2G|^#6f^hM*Pqi3USQ*L7DF6`EXI z#aZJ&1cYB}j4}({wOId`Qlt$Oc^CcygIpWf}99CS$Y1A)6=JSvW9d2r(H>FlT&bEX2UbSI}o(z%j8w zt>7f1dAiTZg)FB6Sxy;hp1Q~?T_U}0V|!m04KIKX0-!0ai&T3Nv1nZObizDK2YT&FsW}lap{L`5qOI8U@W;UO|?EGNL>q-vqf~f}-=AU-r*?)m^ zZ2@yJG$qVN?;{*LacZ8c5FdGLj8y}eIw%Dkv$V#{a!Axg_HY|6Zf{t*)uH4pT3%O zzCmc^0iiGhMg4`mMhmT$YBatQN-elBXR9#l&ZAw?Y|2JK%9*dMJ#Nmuy)($+0rL}0 zLqE&q?V*bctC)QR{;?)Ca71rlJ@*L{}Y+=P@F6J|fl+Wx|7X-E|7{0S+|3>Bg~mdamR8faFvrC4Om50)GSmcYco zu#46P4pTHAH0@a-IZM&dZh<8GY-aY~YDK@8+f-iyh$-?+9_ zau!E#FYD%7*}yf!VdyA`%H8`-ISh>{Ja7SRmdh5*8xFz|&XIWo+nQw4_S>K^a z`va4{hx)nV4YM}xoL#-azF>o2X2|1C<{K)~Zmsiz7OY9%z&h)I;p_>VX%pBo64=rb zxc3#X1Q$58FJRG7VP5x8s)8vrb}93}Z5+N<&CF#F7=M{5FEBY>U^053 z%&fp-_kdyQMiYeu#`$v?IieVUDA&s=a5BEx-}>1rJ>72tX8Yv_+at9a#X2U(t?J@C+Lgq`n)88W=j9#xKHX6lm+d9Mkr&MQP0aI8Z!k-UFikk(wv_o*WwYA?wgU;=voCOFU*KG%z%}~<*S-m?g~nWq z1=!MdN;(v5p8wE(>O@bSR7u7(Mz*M8zA{E7fqxtm7qEym%#5ADtdn4z!ofEE33HVw z^WrdO!3zvd8!j3sF!OwnWOPUrI>4YKz@XWCN#pEFApr*73k=K(msu_>U@VgoZCJn? zVDX8a>-L88Ii&)!1_Jg~Yu9vMiJZWiz2HjJ1lFhvoKX!d9v3)03=Rr9ZTFpZaK!`5 z;8O>ytGRAG*e>%ZCDNo(fzbBKl53ZYj;9zsWoH}b(#Wiyp#aS8j z9>kO!ir-a{w5y^^`|MAnroz~!>D&)X4(+VjyEE_51{;B@$2Ad4y_s8WnJgc$;*%O{;f0squATX*H8<&3m%~ox zD>6GP=U%tYd3ZPHVZPHj9`AFT`{pTaG4h?T=hR2>?H^R*m3R&aa77+qO-kUpY+!4v zw_x2w(LH-- zt`vhtu2;-G<6=uj)`Ivq{}@8@F7q*PUNT@VtL)<|VEpLF$jZQRD}jUcK!SJizgbc% zKXezfR^8)@oG>+V1LqM2{+R(U!U|pl&*l8^;k4GltIH+`{kXd>j3H#%gYExx8yWR> zYznO?Jm9+O2MhZ;rKDL(OZ|5JyY^tqkF)uUc2>PrvwwfK-Su44-<^+q&eN~l&xT9y?TUpPGKB-)a|FCUYFo@;@XW-V zF)Z$m*kM~PTEPAr2WMLCC zJBxvlh)Tyv4Ue6lYB&rU4l+y;3a|>u6mo207l^Z4@U!8dFPo5rO+Z1h12a2w!IM*g z3mcl*#C3I+9!Q_9(j%)=z!9i$gq<(avgE?4#v|d%dvjzzemySqdneJQyr_}{sJgT-O3T~qgmeVSrrnsF*oD@a5?%trN;h-TpumrmiQo>R zmNOn`PDwwtA}F1$QOhWaWvYDn9|x-z?(jt_dIEhD7@1t0#N;D*QWTXQ_B0vSmE8RN z*iFtuHus9AYgbzf+YZUiHtzb#{nPp2;>%|*y^?9-FsqVqGQC=H;cWZ$lEVkv?5j^d+Tfjsmkt|DpqoT)P&qi3?jVXkcDl^hAqg-KIz8Ec{G%M?K12ESWg{w|qR(rL?VN zQaO}#U# z6O_-}eCPQRa6`zp#NvU7Z-~YHv zfy)IP8XR6Qb4F-HGw~#?SlrpTDC1C{J4@%GHrpjPl4k`e2sU;Gc@;XjGEUD`?DBXg z@Z@Y}TZX|Q4)%bP>g)$o9h~P#b-ZMpBGa$QWMANLN{K_OV?k57vp_PBm@$WY)cD9?P*99~D1*iluU+@#6SkWAmqP z&OFsfsB_{l+t&W`w|U+>#S8~Q(Oc&KmR!`|wf%91#KQ$rGd3LVcYLIgDrI<%p*2e& zES*UxosqGj`Ua~K!$DRV4+SPch1;xB1&v}49&+n#aB#{I;P7x!Vb+?^q_9AkL+egH zkJLd9w`*TnJgzVbEnRR#IYNLTH&NR`aF^$Hwera)Lj9txJsp_23>cU^9AbnfFgBc- zIz=ex0JF5jLAGfp^to0n4;A5PWVZXTZsi0A7Fmf#cDDdFwU_HSK#LB37SRSbLTD7e8?3P)TqVxWXXtue9S37qel9W5L1TSDg-=EKKVKOcrv= z7c^8l9bgZtNn|rS(7>_e!~Vbv3_@2! z$iBIR(Kd&HbwlPlUfTyOf(i|+Tnk+IH4?c~J(wk{o;S)G8n_Ea#D*CKoEGRfVCBto zL*nEPM(qj*=CB(LY;Fmy%~=Z)nPpdS^*q1OV{5{{Ov5M1uFU;t$}`@Ho-@BL3Sd`^ zIbT&!AS~li#Xk8MBVTQYv;4X5m!x(mEHhl-$de_|Ecu|M_EBNu@{(IEwj77ptPH5tt$6)RyGF7~j<1mL_K_kZ%1%ax68_sd;QeacBQ50-@;3TA^$f+*D zB-GGwM9xi-Lra87x_Jjj(1})EMwX6f9*zT#e45U3ZG7UWepAuURCjJt3Bb>`WV9sJ?jv1Jn{WmcwJHGg_^h1K$@W~Ex@W%&n=D?}GCDcCr67)c!D zkXz8ewkUwhfkjDt;t5tg4#y5-6-6$;7mU0N8@Zk5JmhJA@vl{HMPqOMn0vx8<2RKqsxM*Yra{DiFTrlC_;YeY{ z#PXnnoSBynh?g`baK2}fesH7J_TEKSZU#{S^9dXZ-xjb)b1+Fy*rEQ%WEH*x1IEz&7+|9%*v6?DuKmVKuI{@8q<{13{TZ(I|`?1G_x;| zVB&6ZD3$!sVCR0pTW~`e^U;e9Gi9Tq&BPs9%_}yGPDpV3$FwTThaR`_vC&ZURiaF$~N;9!C_mDI~58PM&dv zp_#8}LZj3ZMS;o+Cpngn9YF!if)YE@8#Gdca=i7_A9Xpjmt^?n-c)Jhw{YP7ca*o0 zCFQ{;kFrf-6E3_n+A#TU+KHAuYyDhTzOZIl|4Q|M_5W43u3f8O)wggF;;=y5rR+j(Uzx;5wbn-@FBGsX2|J)9DsWKZ&<0k$36A{n z3@m(K7O)$bXtOJRxN5@nu_f>VYx;$@K!HZK3mlTW&GuimK7CR#@AO8thK0N*ch41= zKKVx{mxht5gOTou79$6yvLvCuQR=5&I~Y4OUwovmb!t2B0S5jLn|KZ|2pnMGUEnCi zV|$J3pRK3_W4l&gd(vXFC(DfnUJYjd3M1VE zuKos1uQ(d@lp1vdntdZSRo-smU9d>+vdyk&*|n=3rPmy6IP7>^$+kUd^A%^?YucIRlP6X?*-z55(6Dq~?4(`O;<f2!CyTV*%p;9e#Bku*H z+Q&BMRy*F@<(Oq++;-aS%H@V!6V`m6b;=^{Hq}kHv^9%sn5P)6EQxWqPuN+$hQ%v@ z&8vegbpl&v#moCPj0%5tz6MlhKKShe-AM+Yz~ zUtls_!RlnuvTq`b;e%%556!j=Okxf$VhN4h9vmMtriv|K+_AEe$AK|oCZk6|o6UkI z+Y4*~Q<&HV=B5`U%lVDgVJSGmAaUeT`lLjEQv zqritn$}3t+SMO2gkhuC%RpO4#F74Cdf{YhtpA0Y5;K}gr(As;++1TSkYmkKG&Dnd; zPO$CWblU!r_oPRsC!JnrAHm|jq9tPmTlR?y**5|*SFq{5IW+r}=KEPH$M3k;w(y($ zIUX_djEM@P!VPAJ7fr@HTElMqYYB^JHsa9Lz0neXq2=0Z149EATLUv21}6C*jW-W8 zswt={oDj7>(d@3!YSYl9U%?Sz*lP1)LFNx3!+?z`GZ&uN*f~!{C*n>g+e7UzmOTnq zr?^EJ-hI(A$uKYfz+NuE`1wb})s4Y|5sOTAwECTB@sZFiyJ)U^dwcC_{|n8cybR}> zZu>WL2Qc{VUEACee518Hnn&e~vhP-J%g$zX8%=go%h#`%?JHU`Hn5dNuvaf=5B=L- z8o^%D(93UX_u-VLWQ31rvh#bc6O95L5<7f6iW*fOG}~x28SP+=zR}|QqQ&$Ei(kOC zgdOYmDYiOjusAZX7&5Rpd$2HcwRrqfU=5OBEl_36n$ecIppCzn&GE(Thg+ruUTD)l z$=t1Qly#H!>5UF21s92aHj~LZ!1l+w+SH}!rwaQG2IU6_v^ZGnKeWd!X#DzOe&e17 zNr^_oPpzq;%_b7c+%pbxcO2B)ttD}#foFyzPX~vH0ONxv{-O?zuCX`&xin7gjhw!e z$@xNS=7~#6Dl=J@+G;*fwlBI+!gI^cV2kUFmMjPMsvYe$5x43xZq;09uQ|Y|)UosZ zq@A;O-FkT2y{?7->EV?p|G3M3V4S*#QTanln8j_s7c71pZ3!>-`6#qFO0e#!WC>>A zba7zW+Qc{E1iynui&IC-??SfXy^)S5*m8E<{HLiAo^_!uX;TM>hgj;#)tR>ziZWjo zV(JR2xpT2HhG&Ns&yNOS1Ad+c#@UY=rB^UY3oxn%7#UkIC4FeGkZ7GG%ki0^apn|V zeh${?6D^g-O#&CJCdi)VzOY?y@8^~=jxx+>jePDK_iXA-o`fbRi`Gno z3s0O5smQpq)iO_gB+uvPg+H#QX(I$c|5iDo^{_%r2qkgg-vh& zs=l+8v^cT&Svq-MHzjv8IvKPEHSpMn@u)qB z^DSNH)W$ruDB%vkS4r00&>9wDq#4z$CzM>*P_NfPZ0^ zUEEw*7{wy=#Wghe6@098=0;m3 zV~c@BOU#06wkKM{UQ2moFvT)tGGBag{|LiZXTc+%x9$1Wx!?k4&ckU7I@Yi>FgW*= z9{IhIrK2I@xLS@g7Xt&A>aC`;QQH$PF!^${8r^7E>eMJw!6>n#u^}+wo@=uj*9)#9 zhHL+hF(~VAKesk_a_v;L{7=@$lJB-f3Uo+KkYm}_XB+a+;G~I5SsJ=~<+R$jT!9&x7>FgUO^%+gc zVLWPjEFK@29Tv1~6k?VYVE)6`tjODvE86VB!Q#LW?#RHnPm$HO!|{(_6T{C2wvGnL zLzSmba&jEm&Dg<^*SwH>W#{Y$PRSS6q(o{VQ_@Ip@Cat`pZ3?oSAhYPkzo?(0O8#4od^WDXYc^dq(yN`}Q@j z$Vk({x({$BG53wLb9}6p2`Tdh)d_KmXqL z`l1GpMkZDs9<8)Dl23KcPf-t==g{&a;FNS%{=FjyIntHqSl2%I(s}v$`9&F@etmqr zJzw?oG@ZR1f=nzO{4!?$vVSQwEnaqZo<-6ZmafA`{bw5{38+Rfvrn*YHAv`C=9o0a zVOde{sj1%6R|YLz=35-L>H-hDg5HCLf$WzSTPZ39oM6=GTviqCQ z@M@*pSmfrbraLodE0bD}#D?gFb-$Iqt)4v7C$ley!*Pal3xAjK9E-+BOl+?L)-P_b z`f#X3MBSi9UZc*2OH}J$N$=s7lg{me^Y=8?fAkgUD|~n4CC4UzdHcRaR~PqtY4g4i z@-DrR*e-5xk>!Y=<&TER{&t+9A^%z(WLrh}EEyWvRal=2aHx1CJUCKS@nGTt#R*fK zd^IPUJaT4}Z#dw}Yri3sosF+XU=c^7%csMvyboFfG)sNT8(Afve05||Hxav)DBv)` zAw|&O!hzI@mCq!Zd2A-kWJ)Y^OI2R#BEV|q^X4Om>17vzMizkss!mcFGZwN6&COXf z+u*p&<8y`|MV!y5MJ zA$YT$iRj5#lafFVN$H9WjBLt39v+un+`H(XuV#*^BQuAB$wy{^)-7KTbC|p^@Zghs z^zhImg^UMBG%8a94)a(=BphTfc4AOvX_uXLlbJK)ro)U0rPCtQ61>zR(*;!o8swKO zIKW~R#G$~X+3NBkg3~!~!4b(VnHN8)&AwB_ZFq#G__W&02R&|*t~mye7yG$YF_m0$ z<(gVLt7!U{@=J|Y6S$ZTHXK@Re#G#UNr%V5Bh}IID%o{EYgHzB)K8egWAbAO%bLde z4 zUT~sauHntaR)vWyo7m+WJ|uETCOHUo$k((O9O6jf`NwpEBUWM(xBMB??+%=yaRQ7i zQE4-m9*%MnD4Z^o5Ggq$G3c5DQ@uxXBkz?b8xM+HxiQg^S>VpSh*OyvK{0A`GXn3N zo?8p&r~Q+AV9;8b#xSk&0K=gv z3~Uw;IIqlbWZN>aor~v?q`5-!H-!fbTqz4U%8i;O#WMH#7)+XIog9)NBd}b^;go?> zHy_smXW6AU&l^m>6u*K^Y1N9e41BspZPgDhGp{>)-0qiPw1?ZFj*JL{T*;J9Rh@_J z_CGEa6>~T@tGOJS{pM|pmF2?w`zCbg2OQw=zTl#rQ`nt#Er45&;fQ+F6X$b`N^=?v zTI2*2JDoZl=QJoVOY$+c8MyF?w9H7?-?ouE>&IegJ&s0ZJ%JWm4kv-i1g2Hr9FDPB zEMU^$IKbR?MY4V3N+aGG%0~rmH3jj6NC!`7iZF0c(Xw4IpI5+vE$^Q}tMP-*i4#ni zr3%bAzT`4Y{PQGm&O7myhj?_E@2%v!^hsQHQRVXbpk_y|j#K+3k0=Ty2A;3JccJ;` z8Fl6S2OJ&_Lhbf3rA<5bBs7-Y^o}n(EmY$%!KdTFg3XT>L|S=F<9xkrO|scT?$!&= z98(h5vhOiTm~LLC+ZVv%lA*levcge?FA8nOOD2~rX=suAa-iL@hf$z$0*kyuLq~2y zl0dowi+{)!wY-mNQGg;q?E%K8!@ zKGA|j<&0(5`iiC6iw#=!+ypu88yt8R&v=yPSm>%J(ZnrN;J_T=u*7@@lTf9`Q3Zv@ zUa^n`j1C&D0#X5+batxwq_{Njq!=tWNIk~Ll;I%4!pOpCa8R;vg+p?gxTKMoF5it0 zN7Yvav^nxDI;VVLn)*)f%N$b<)J|NrQloVm(~`(8R{Ou1)k~FEeLApxum6e`W4DD{ z|E-lZShe}q3HPkdtM(d?>=}>;c0W|K+@I2GA=i+&9m6C_H@nGXyq@=azA>x z8yt>sYzyd46gYfXF05J1EV0{o&Owoz2in5-DDlR69M;|?(5>ec_%>*QlW7wJBk!>V zEUG$}MH-k4;^#F+oXORKbXE$;alqK|E)KHV19@x-e76_JKE_ z1cRjTg=Vikii`nW2N-l_s+!GD;9B~F#ZN8^947*ZI=Ne{18f+y|ku99ixT&T>q8TUQ3o&0<#Xm~kd2 z+wW7@ zGt-5;_BKv_lV8dnR=wZkey7nZrKe$PXU%Rdn?yFVA1#{899ns64oQZbV3^FNzQLj} zfmN`Bkvm80*aBYNr#{{X7g<&^O+Re@^Pl%YVX22a=`pvSz3)6MEaIUZb*)H6$o=Sfv8Mr9zJH>8yrja*dLh0pB0Y#+)_ghZa)N|%eV%BeI6fx0g-WYwc%Xh<- z(+7IgdK~X2HkX>IWX~xIJ|Q?z;hv1sNq0- z06Uk7_{SHFss)T}28~WP4orw);0tMz<%wZ1kzr1AuxBsZ#88Hpu96}*9N1V6N%%BsK4~=eY3A)<5RKvEO*z0&GjYxlhADUa zSsCIh4SUN)?)wYgul789S-|DfLI?g09NLNj{6`w4Z5X8=%+ZOQ%A9&%{tvq?k8EQ~ zfSyb9kEKyRBb}NG9RKVLRq^XE+Ic^^kWt)&i9@1Mz2MO49XxT2&FU)BW(kK)5~Njh z68LHwg-smAO&o=m1poUX=_IajNaIe!;+6)MX9pE67+GH&;Al9&U=rih*q|8DB<|*@ z$#ICcWDY~f0l7N|6DPzm2uMn?Ixw#Yl<-;Wk z@-;{A$XrVl*&*Z-a9{X9qpOj?+7pZlT}M26TGqKZH@v*`eP>J911F`n`9B*cG`l>y zbTX>tXN%ts2j7B5S&t^+9tW`&C+!_h>LpC-H<}DM4(qRQl~@j>uy zDg$={14l>$gN6Iytqf9(Y>Yp=InFTf3OIxmMf24h;3#N(rE*aJ(?R{VgNaN{5$5rMI(-Y`u6pn&t$aD@RtBqA+x?5ZQ!gEix_f5UCIQ9( zpLAY^R&I+9@ehqE4;)XeU_A3<{(sI?-4%{0FPSv^TsJ#5Wz{&c1sve0VYGkeq&J6g z70Y#Ni9-?!(uy;fG#U<>O<-0KX<+#m5p?i`1FL`o$At!t00#*MC-D|Wr8P>L9~{L! z4#{;m>Pa{WC9q2NFzGck>g6=^Jz+BYz^K>L+}d|oQR0xbK$FTIX8j2Vt-mlS)HLgz zd10M%Q14H(W(PygjCQUk2i1L;Eqs{e1tYT)g62$En7HHoBFA>Ae2@@!&+%WXG}A1hS@Xy|O)Ew@7bm%f z1Cu%qSWJw6?%*Voqh;2>w5gIw|H@^_2TTej4o)xs@l7#Nn=4&NI0>7I4CLNAYQ`AS;8pk@Ic~417Am@utuX?$w3a5Cb^yl zxt>OjDU*dRERxV*6h6Wz^J+?`??GOb#JD4!dz!)%uVg3w3GrL%u>W5O>x_m!I#HYq zIS1A;)Xa6@WpUCAI4ry8()4|f0ud=+C$7|K_5D`+L`lWj`iobi&l@k7!+I`juKZm#9uf_s5tVfG|DGyu zzB>(k0gSwN4m906z*Et{(&fO`!&q&R%z8ojSCF$pibHn6f_E}!9E$P|@|=CH_3`7ruO>t#<}<2a5@((!aEjbYPZ*Ps1R-NiAAh!NpO;{@+Uuv}|3>*v8u9kn*-ksE0wp z;SkS-g=&kR&XznjRjonNtC-Efp-f}SRJV;uF~tWxKGxWOkFnfj)Lui0Sro^ z$2spXa5W^pyWlxH^+4tI2B)bz**rFz9{yhac(plu6st$!M_bOi+u!Pf56c~S7rWW9 zhNsG{NNVi^cimV~R@S+;Pnw&>x2byU_}LrUWcyrJ*+l+%;r$sCc)qX*Y`AcHs*_m{ z)4zE$4{2&V7U5}H+RK>WdsxH7X|;B;0!x#`k}}~JH|(U0lw6#4wkWW6FzCK=V6Qn~ z!8PyrU870Yr!ezKHtKyZny$7WkW($uf9q>MhkHisH@-~xx@DTOVM0PfT~f202m>pN z!&DUq!KBVPHk>S99F$i$$k;fsc{pmWU|0FWxKyJ}a#o(jeZMGmw*{TX)3!0J`eOd} z-qr?o$C@LTB$|B}9dA_nb5LbQv(1yucGGVyF;&JeJ*+O{Bqze8Y2&0fg-Nc5q0F37FTmOSg_GO@ zE`=YBDjokCxi_Cg0~%&5%Z?>iJaniVaST`KRm{C)6lNwv{82RWZ+t_){0oY->}s^6%zViH^!uHqs9= zU-#F{-)EOHTi|5YVp*FehL#3)8AhRoxneTy>?MsZvzyE!nvE@rHitf)FK*3o#X-r# zNvzjV&V@<7#nJo?lfs+liabsd9*5RiF)-E~Sit+0^#B8RLGsJDTXcoD7uxUs#lGbD zM*kn$otawI4s{O9|56-;yc*dIRxrdQr!lxNu?eZTC{#Q;#453D#*u|$>gP-xgXTES zny8T2$tC4IrQw3$p-v%nKNSzhNhxi@>VXLzOqsV9%lTLBd3mty>|}GtdtcsgvS(bd zD|q*%^YiomMV`-Y+}u5Vy}W(h?-gNNvpBWG>>}RY_{hY<&L?S+^gu!AvChN|87Eii zb{}Sq+?gNyW5s6nNvzzi(^4LloSwEeDyy$&LC~q0+cK-CI9v+V5>Uykc4%Q>JQT3V zrES-f&ME91dO15d8jZH*ic9J>)Syt_@TT(`HT-`@65U}3{+Zb>bdM59xjjNHts3Of`U8yz`$GX-@ zhM0teFmg`n;uifjpMz6TrQhw>iYDQv6U-~^r6M;7dVD&NJAy-Xt<@n*ytKW4JXpNi@(@E)$Bq+SjK_DL(|Wx|`Y6}wV_Q-R z7Ix}?Qcz_P+IGTaO7tud)eQmDZaB2s%qe^(EWP4{^K|!p3@-g@84`}|YF{oePq1Zc zbP*7AJIE&D#HQFH9(UwIW4FMO>^`e|5j)myd!TY-EhBqFwnCHBOyz_NPVA}Y8oBDU zC$zR`tv$E%X47O5RvUrJbP*;kfdgiZ|2UWem>y{bES&OV%S0vSU?0bY69a<|JZNG1 ztNhZwU+BSmv-D*f)i!Exy{I-jh^2AKBDonWE-cH)esogZ%;L#~6?Xq3vSh11HLjX3 z8Y+^kt&^}txo7gD%M|BaXDxra>N~r}H;Gmjoi$IW@;1K~wrj-@_wmM=F35%L^x4&@W zG`h5)sav|FgQ;IFN@L4~MT&p-{l2TvYxU~=^vNY=Vh%eRTD^o1P1)}uTy%g9=dubP#zT|vZcZiMu;6dlE-_DP3IP#19Yt1eSVRR1PBfzG|kjRnv zGN%8Qs=o5E1tmrbOq{Pm^(#ByvBw&)NcJq?j2BRri~QoMwrN6V)DlPb%EN8OB_Ab% zL>BpYUT_rWYUGgJv4&MVfRW3np;4wsky9?#Xwue-1KmXkBFz*|WB`(WzsBUV^(zjkCB?M-)R|*j9EMhHQQh2e!0DO)M%7 ztm}LlxfCK8MW(o@s=Pem;Uh4KW!fQ;${$Azd=)vgEgp+Z=s04sq_BDR41t2C)tzk| z={vITHWv0Z%Y>+yTk3N#vO6$zM>~m&U)*@Q&C+3gA%`5oMs33c4BQwnI5dXc~r zyMTf1Q3SKx7KK0t76C36jRT>f3XHlB`ME9~lb?6SAy?DGk#oy0ui%~}fm;Rb)(--^ zq~GufEuL`9#K(ix{DR7?#%gCjuHy@jXg0NdxshgHU}3r8g@TSlm-ma52A9K9+cvb? zDhZ`NICpT}WmdfvjW$OaIFx5BOql(mMbhXZSM(oOVd(%S&gB!?9UPkeiA>O7Q56#G z_^sO1zxV{3(vD(B=^qCdi8DAV9uMGfcvHsKnBc_P;}&y!i%8IZPmV(0Qyx#;E-{%c zeaRXX5G=NX!D?y~Lt(#IBTr8zi<1X{tgCa2?j<^n*}SiDpT2vKCI?m z#IRq&!I5{y0cO!F41A6+7)xEIel(FquwhbX+CI$T8>AB}Sjy>vJ-l)cz=x`X?L`UZC(wO=Tgc&kV*8 z(}pI0hoqJxKV0?Q7Ba1W=q`1$;hAxz?%Irv8`c#h9DbCsgKv3F_j=8!i2@AV;yaE= zbb7}aCau&>R@s%{VOq9m+v88na_tW|X0R4%H{IciT%O43=fE=ga{;sF1SPJr6Rg?p zik-TTl>~P$XybEoxG#2MGi&9BAh~>4W3qxdb*l^D1qfvsiFd+U81&b6iImK1%8` z$^VjeV@Z`qdV3u{xG5;(QS(FzzunJsg zV07`F%HsG>Rf{>ImSfLBe(?@QE*;HJVgZ|s6c;o~%t`E(3J7)dpOO6e(*j1vgl6BR zC){}Tczo9$|H>P8Js`HTwzsS#`^bUT^~;>Jel&8kXm1MeVSMfP{)ylFES+~U6E=KU z$N6c2ugL{$u?pA`7A8tlIh>``T=i zQ-Ra}XI=lr5``8ll)b?CRzY3(-_>>U3m!%9QD8rn`EJwO$3cZhLpG}UZe)Di$Lg{% zZovUQIYqu13~FVK;tq-24vpe*xh-moMAkUwYcL9ia7!jm;Ac~m=)S=JPl4agOTcZR zc!i=M$3fvLNAXVzoL?TW6)g~ma$tD2{Naq%W_5?n&ILZs=>PFm&*>>rhH{`P=FtGg#P`uo> z?_Pq^3y%F~+LRO*3TiM)S24V9OSt}Sy;BBLR9;iTnx*`I76>gmD0S>8V=M;(L8(F^0Q@`!b_V1Hr@`HcuuM{eC4rH%T6lZ$KQ_%Q|L6L9H!Wrw1 zh%g*vb6dbYX+5*c0_GJCy)F*SX%5VCxooo-d`cAk&M}BoF^YV9z&0u2>>CCC6B`4v zoSBvxJpN(y*yx>A8o!0q6&d4m%a&yCO5kG7;relFxzpbEj2-RvS_(XT52cwJ8U8J2 zoTdMZWkujOL5F|r$`ub-bU0a48d%c~DEK?Dwk7aOCBD>Jz^A~-w?1N#C6flmx9^BN+*unBl5Hv9a0DD2|F#lt0ZWWlB)#%&n~m`fA{ zsvL!OCGh-E5D;6($#jsDp;3V0prkvaz`h2yfL=DI1E!1AEV+`neqIWgH(^=BW0}(% zxlex7OFt|#HQOx1YyZ9F)eK84tF)M=IPhHXU|8hFy{7f8j8|7T8uLm2(s`=Dl#f7D+#eIEblA0UUtBAtDfNhWy+x^c*8OZ zQ@kwo9P7z6%9qd;A>79c@eC&eSvLe~r?aM0o zd@~&QZP*MI99cXw@^TJ*EOHboc_?f3P4)z%G@l|TL!$J*6ANXICCXTZ3FRpW$apPu zY2Z5M!0E7nsZEQW<7HU7KFciT#|_QjUKoAvSTFN_(zh>*mVN8_w&KH%dxg$|uEA5* zGjJ_n6k=c$co6>nBl|svh}G}IgC6j0ThF@fd}Opj6z5ykhzG0{2l(Y0*@YgQR8Ztw zrof*P==7~YN^P0uX9dMd2hYj|jyLBcFPTPNyQ+Rqflut9;HoV`Ur%Xn`V#$OYTW0C za_>yp(jHje6)6@ED`poDlR&r$LV$AO<5Rf{uwZws8ikpza!+-2$W#2d!EqV6I zsw`j6R4**IE2FbtFSBczQ9^K@Xt0rPpuz&nGt(J9JV`csASb`zC({BZIYl9Z1Li8K z*(SY=GqiF771$CO+~Xe2r~k z_6KKyj0UDO2BCci_!c~1T2Ri=q`d3J8=H!4jwcO+dd!%AOfuW1thZWOQR|;cz=`Ed zTun7qr_8-@p7|mnnUh%AcRB-mRjM(==1$72hbMJ#Mcg+h<%+&JP zd)+ei$MojNw!zoE327T#>-_ zk7;6-er}fX0=9$)ECC4(C%;9Se`aFY!T#<5e^rBZNP>{v!W^j-fA56?8*WN^Eo3V@ zFvUMox%w<`69eBpEsm_>LXRgh4@#%0UY}*A*gI?4 zqbvMb2bkO*aBMjsX8AC|h>=snfn|n+K+01AyGH3L5h8Pbh|RX>$~`E2A(H*a1A#{l ztkV`$@%e4dd9WzwLF%obpt8rZt!Y+OTzMIfGvZ%mELtsNSY7w)M!<&~mOGw@WE^91 zQs7*q!_3wYcu0ZsTmsLr2Rv<>Qi_V(_1<2JI>7(I)LiLi&it$Oi43fD4+J+o!T=Qb5Pi%@u1@bfvk&Sjt_Z+9NG6Nh<9F;Sm+~=*TAHsz{TWPJ;C52&(a#HD`rnV z)v{Rit~OA-WUtDr?pP_hyCMh@*+m$^>*i2p<_``6F~`^< z8g^vtG27N{R&dqwQIGw;i3a`KIJ=LT-%C4nx4ouAIdqA6&@Wx)Y5eR(3tn7u5dT-@ z=W|fDs8GPbp}s+MseZ!w#Wkl258vMT-Y7L8Lo0!O&O*T@3XCtD8B7{Dmn~F!;-Bv? z*{r?8^Kxjx^Q`D4ihG{knf5Z~?VCS))q4}a+Zb@}tL2eP2w!lX`LgfJXdin%jt>qz zJr3M1i4uK^-G2^>pE@A+Z@Ef}!b9O>31aDimyPF&tW$o-o^xQH?t^2&clxqxcdl~o6ECT57t7K)8&vRk+PB61_csP`Nd+n^vNter zd(w#+X0l_2C0(Q>-l`>Uh;uZ17J9z6Ha=G6B&4R1MjBMCe`47_!W-6_XqITk;>@tgwhn3jEGlvu#R=fLFRbo;S<_%%Uhg9Gdzb~gAf zVDVuRl1tBt#cciUwsN0dIBsO0EtfZML*?b>8r1>`6Gdh^3p~G`ns?=47?A^Gs5Iq$oNcV_)plUl+0K%Q9}& z>TNoX#o9A=S8P2u$8fRhlx0dvA8M>jI4&;V+-$Sb*V`j^MU)$tg@=;H`Cb7Pr;Y;? z*e}UBESBS}+Opi8o12%(BICh=CT4bV9hV&i{|+8}!ojB#&}S3*nVoySu617kqf_%q z@rW3U!p4%8%e)F69RiMrlDQSM0yY>Z9B5UGpAoUbKrKK{P;?)Mkcf(7Cp*7@O9cZX zQ`(tr^4=>RKKE=EQeQRQ;bVq?WZd3UOE*^sT+m2gl5|C4Q?gROtl-P=l%S7`T)XqX z7z$axzLC7Z>*t;)LiT@lsJQUxoDoUnw0=89m|HOB$K*EWpDmYt&2>c&1{wXCz^>pu zLy=2;ZO*1PsdYClozk7%V9Khpn8%1CYQ>VN)6{b-g-&Tkiv%?A9qV;yoT!-cfJLS9 zi|Ps9dV`A&tk(_GBvU7BT^5<$dr?g?ZBp;do9Xfv4gVO^c;~oDX6r1t5H-8oQ$ae7 zFCc-LxB5(g14Ai?Qj4rTheNZ%qyz_MmAa5e>@4C69~$QJ?_qOf5(t?fmBM!>ft^qE zh`@1S%{dbq71}xua_jF1*v!tc%%|Byh5f9+MQ4`)lOrz9UoK>?|6iJN(XBopsn3&r z-4~(eGoKE-@fgo_2xifa$OvxJE|{PjAj@Vd5@gYOdJ2!(n~m;*-Ythts=Y3|ll}7+EELnq&vk5MP-bD`;1d&YXk=6HXiQdYauGT)B}&NQd2+-w zO|^Obr8zTG=4YPC%wD!|n`Cazw4C(Z{=aG*rm;L18sy7u{#{_==1URa&@|2{2+-_J zQ8>V?uF!CRl~bjlkzuCY#^d}t2^Ru{^PdEEnVM(piV?1vbI@I{;st}JX48~MXG}l` z%--7V_2t6)y}y>13FPp6Jf?JM&f#xX5%_1mx%_WhuJ$iSrzisLX-43^l zDtW;&7g8R{n~O|5*0r|e9H-9F8%f95BzziA?6t1@lsv&J^)XxzM+UJawOF4>iY0 zObcBi|8mG|W-}LFAj9XZ+HqiO!~$JDn>^=9-7y^wSElgaP~fU#aN_8hz!|vZQSpWh zR|AVe7XFk5&Ucb7GUqm}*AqGvpY_p2*QSuo<%I%&(~ZMN7!=vm4H%P*7dWs*9bow< zxWJIFf+1OtS@KFvgORAk*`#&fC2l3WoY1Dyn=|9Pn!~o0CR;ntCQChNQe#=f$t=O3 zA~mr=$|aFaV}XN6$wQXlBZ*w+3feR^0;NQMJmi;bXo$b}fg>}(Nx<^OKSx;x#m*R& zB-8UYvy}!`TfA1l$0?!0_ zg+SiDJ3qA6M^E7SuE-+Kaa3LMQld@vBHiPR6P@oa>g1nkw21%30?xV*2ie^=uo|~K z5}V_3v}=_^D_6z=&Sr_Dipv_N87xW^n8_iel_l6^?6HXR=s{*V9y2zr3kw96o?sR> z3Sd;b!N`0*dvdmhu_Wgfw$$s@y{SBvm!Ixr^S-nD<;jfVy{}HR8H%28R9vCh!7Xrr zFGZP0MB(AQAcKW<^Di7zUlP!zuA;!lebIq+g;uMDOOsIVfkx@K23>()j5tLY9RJC6 ztYi{SdB8uvMw0EkBd3?fVxb1!oCNj2PEV6Xf>)0`Q8`sX4I2>uW zQEaP#lE=auE*d8oI1M`%3RD(2$sN1E`j9cHZt01m$`?31*l&a^TYGTQ7Y}2P>pt(T zF29b*-K2e)BO$@mq}Ml7CPDUQ!+A!93`SlphXzKD2mBK%T6m={hzA)sG6Yrhn=PBr zz>{!*ZNiK;&5(e`<{yombt}C2iw>~ddb^N2S%C4=vtR653l=c7Rlw~^@IO;Yu^B!a1%K33fBwd0<->0!R@Xf>YWezN^<)@?X2`TCB z5p~twHi5&&g;6L+=CH^K1GeukwsVT9NKZR-q=6%aD`H6f%oIBNxk|_;r#tIEbIVLnPurx3-cQ7zXJYev&Jt$ci5zltOiZ{ahpk(h1 zCz%iG4aZzKnMFMqHUv&+vI$uv7$VwYyvdN+!D6w%gaS9Kb&k22G0aD=o+#@I?n%n5 zus(X_T%q-ie~SGlRvnS&VlvwNPJu;6h3M57=uq zG_x&9;EL~2I@{vVa@Fo5`|q9V?khjDh}$m=H2l@X)4|{(q`=7K;qpj0_p60c9>W{4 zCkxn)W(zOltz~rCzUS$LEq?y{tAkHBtgmilkq-KnJ1^3i*+`u&@%0#29FsyJZhBqbH1@?sr_3V z@S*8kV}O&wI|WwN7Y>|e8yJOKj<;B>xN^XlgYoOC%SG3wEO>e*-Q?7vkk2m_Lzy;s zTAjUVa^mQPG_w=QjC`*iFtK0R{?a$!;j(H*k)nnQaRbY`DVqeI6#ldGf6Jyf_j+G+pS`3`|6BzOv>U#}1Dh)V#CzPGL(2#zZ&#H}2 z`Uc;U9gQy%_?|E565i3c!b9QH2mYGL{ND^J?7k(pJ!h{`V9}09aY?LfORk*CUTHH? z_-Kc)$RX#;O~OJ8n%zG(H#smX9AKTe!0X5t4h}}npbzZx3j`J`a3mLSv>b^3y-3-r zi7l#-Ew+H|m>@@h0PCa$+_x1tlO`}$3jE`=5UOexV66yX787>VNS1ADQ(ZDKc6y`a zC1cMwQ-hnYFqoAJPgy8+{^x!$ zPB_l!{gBD7fxXp$DNmBM@B>?j0h5;jo1X(~$O0yYbhG&etd$0=ZJw;&52klTvV4=P7A*EG&X@LZt2Tfk<8e< zOTooX9#;(~@;Wf+EMV@M?x|zIZ0aDWw6g96J3}j5hjORp%t;K(Ch!`}EYYd#yeUvt zn83CD8IRTnmei^2g%iqVYOwknnANdQePfXKVCEFJpHuufr@r36x6xwi)`+RC=1ul$ zoYN*W#db2k3lsUk7BzXg;)Lt-XPZ`PU+7V8QnXN z7Dm?xj8X+I$;>9rA7-{zdif>H2sU6#G++y4;+QL8)1JW5uz+1*0;7TfvyTF6lK{KV z0u6x{6W+*K8jq!#rUd6Ma-Uc>>%OS}<5pw)sP+fS!T%~aSOgoGs{=SEG=$kSOsacP z!h4{OQGt=;fsF1*QnLtihf|=t?FFCQL&nNJ`p1}7e zVfllZ%UKpJf1x=w_$B|O*1Ryz`9A|V`x7`BpR>ktvV4%-s;HCh(PAikG0c-PuW&E?*pGMLP{|vZ~Ge)_%RYc6p=vOg70?hLRGCGWjB9HzZ0b zEMP7zC}El)@sUgRjr#g`>I}M>8JrCag%>z9940SqVA3^UE>GZX_}9SnEuGoglK0vI zCXEKh*TU@Q82FkTSkHZ6oBV?3`UbvN4hki^HdfEtC^>uH`w#rDB_kVV@o&>;I#ivv z(}AzGk*?VCnmn_v59m{H7j`Q`63IR2nT{eqg~Avw(G>0nd>D zo+Ac43l+FB0$h!x>LnT&-*qucO<;^q(h_&rI^|<#WdMitu0^pQShZT2#I+V5`pzQZ zFzbSIdrzRG?&QTwSFYvH*?xA0l(NultE+KVrVNS=?U|1leAOA{|1hu%$e!Q6X08T@ zY6F8n0|W1ao!`=V>n|{0Iv{Y)f%n>h5RC^*&n56`3gl=7EOVLR6*iTze?!hc(ddoU zw>G}C+|3fSQB0eE_wUAuPj_#v$P1gpe^h&ZjX*&a1Dn?FO->i4c605GdzokWfa(1+ zrmMFryb?v0ePuK}z*1Gf=C*-#_6P1G0{m5P_#zwF76!1%J=k!rfXgI{*;JvXA%SsL z4)0Nd68o+=sscMmut7qHHnz*!f$>@ouflZdL|RAx)%viNYDsBT8f z1?+!TFH%lmoSn&RykX5RaaMl;?~+SuVxPCk9hNAYoq2oL+KFYhRa+K|%$arLd+`l- z2B8Ver3cubKA*^vkj2-j$x*||LCby{)`~nQ?H!fY;Hfeg&9QFq?0uE)K zlbP=Lu=Y-1kAKd-)Pc8dI$zl=zRl6g-#hFUmp-@i_3}47n}YY;_?V;cU4eIr0_U$i zM;XNnPQ5ex&YFwkVY*Ky?|L&xSrDYZa|?*^tm@W-SXB)-(>RdOxW~mO?S!Wq`YGpuv(ZtBi@eEQuc$2j(~1-;%ET;?=9nY)2AsAXSB zLi9!>jzh7#lmb{37Mz&2MpLhL|1_R|OlBWggD$XGGVDsc#~i3o+JEgWUeNJhlOh2Xo>W85o{3Gf#|u;IFRf z<|;cwPp&JP$&;Cp>)o=Y1{_)w7^){E+)X%qZVGSB^jGU5KAZV>t*x6!-8`cCb5MMkgmv?;5ugMpoa@%%&v_5voQH9IaaGUmrJRySlf8<;s&hEM!(_VR__d-Hi0N&l%a z=e?D{do=%#@%%rg>$_I$m~x+U-s=Ug+od>M)_0d3IA3MJUd8a(i~mA|0sE*f4>?Rm8*1bE;5@;y=e`79j{jmR&1vWE z>^%DR+*ar5-BXshwCO}XdVJhJ&ARSQ$32~1>c zVdc@~T692RX^5uBk`2u(E;y)qOp{_*1spTQZP$0mAy#cNIVS}H!9S>$(L-ga%*gcSmUOC%mo_P6<{G9}=` zjieS$E=^bIEU%eK{a?K_StbOz8M+F1+_;_G%;hknQPY!QV`Ha-(k8{eK(Cc1{f?g& zENuMoB>6<>DzjS}jT|qGKd7)uS2QrPTO=_s%~08~Vqa{yk-))OVL=mG`DF?m7!30y z7}o_dBxq&2K++QRZPS!<4j!gt2?3LS#0g#u)_wR^Q@FkKC@>r`HAq0ref>)$sG z&DQ&0Th~J~#5%%q*2@K9k&axLG!zwNMWBu|8vRmhYOM z@FG+5hc=aF*4}YvyKR=AI4qL0<>FD1_^OAj(;eL|9+S+iI=5KBV@X3~SE!=n6X9>i zPCQ_G|4G4Bz|p7hkch{hj%1BsFH4pQHm?>mDrTe?m9F~XAT-U9QSuMJYO%P^O69&U z%%1jhR1S2p$_EHEv<7z?am}_6y6~abqw2(hDQ-C`cNl~vH5xyNUX}IMIT5TP!>T#u zv=zUoQO9DTpk;!)ggg`s_fGOqTF4^g6_Nh8)GKXy?YFw@`L>BI(*x6Hb||ll5c1Y} zE%fiq*<*ixBpV#yINUcaxq5AiPF(QW?pN1SjL%*El96SbGv}l0><`s1TPx!Zr%_?|7uV zfrC?b(Yulr!LhHj4Bd>TZ4^4W;>aO^Y202OPC{)5B0ie6*-YrwF^f^~%}z;{R`{^7 z|GKfHz#E2toLPU)NZUKBDDltq2=-Cr>+ndBonbJ=(IYXdRD)GcYQcUR0nWy->K27O zhgP=>`D`L58vOJgHE&y6M(62DT|nasg(zX=Y3!kJ@Sy3)1l#U!j(lVG6!6BjFQ6ij{8apZbFAgpWIaR zT^uQVIxTp%{h>_LdLEU$HP2bCSr!S*_qcz?ud&y~H+hMzHLL0J2K{eW5=(TaOquX4 zppj>R0^jGR7KLRG=KXVXaAIvrn8Fdjd61*y?NsrLCz$P&9GMm#ao}>g;KiiUC=_v2 zg=dGTd)ZN;m3y8rZJ+djHSI%-yMQW}K)^wHzokrvGBubP1Q>*aK1^Jt)DRQh*bwsO z)jHnJ&y2B+3^F||eeBY--H=RRW zVRKxRoqnIM^K0tOt4Tg~AZ5>!xt*r@S-iO#tBmY_1)6T3n;~E9&@64>768YBJ>fdRhDz=n?u-3B9J%Hl z&eQa8MCPIDMMBR^Qsc#*&GrMohv^v@bEY`NeHya zW)$plf3j5MT1S!^r#f3sLbJ?&iRPe5Ozd7B%|d00T(O@XEb8mI;iBVofb+_trt>Qk znAHOux%5nXt*$h%+q^iqyWOBg{(?1!={-qi zV*y6CZ$9s&4OIChJUlp}4zQ$KSn21Nv6+4Q^f{A-iCKcPlX=mBm$o*EUHKkfObiNN z84MP@Vq|b&j#FS^SQEu4Z^p2Oc|tbhm4tsB8^TQ2Ng6b4eJsVOv!j`rsesYOhRJG$ zP>@LBLhjfpiyD;~T$F?wyEPpa95LwVHc5%$I_j@1J>kFsw@=?4_J|rXs{Lp&y%a8@ z_GN+XQ4DbQU0xq-vx!@)i60xjwr9v<8F-dy?}L%ZFbh5WT0 z%pXJr+g&vxjV7<`nZ7}WZR$^l$A$`wf-wrrk}mUqT|3^$Q_#@b9en4}-RV2{Rn{>J zv^22g>A9*rQs}VdND?qRz);~(@O(!10%xrijKM61P0cL~ES4HeJ0hmqDjcwRF~_FE zz|ljWDPt+u3`Xk=$%#ys%>HFb!g6Nb3C8xPjZ6A%KA)VfSio21S7ujspj_ zBr`A-Fl>`%nEs%VO@e_*g5l3h#syg|(i<41Ez~q6SkpWB1s^awDX=&QG)sCkh^=UJ zS73epiLdcM^H+&hw+GXMJ~i)YYG7_?*x=D1GJzxgM~mMnW{HlLpcict2O2Ia?-f(n zD_-C%QPC)Cuvg+n|(*}^rIzMq=jXZh=7AyR-<_OiYc=Cuo=o9I@hA;}#*!z{363%!mFEru5wekL|)2rxS+ zG#fLtI{s+7JG*h-F2;Sc*@PXor*iU>%1K#=^It;z@Yie-h*={37{e>%W9!RKVjoUNOE#27Z7ma!I}z+%wJ;yj^&VS%H~ zj$XY7%(4p_ojRCRFR>MBwD3xB$%!x;1~5A@v>IsW&zdBq<igTP`IC8SVvq(Ncxiw}(GrtAnCGAG>8CN_-{XgsYOEft9J`J%9@&D4o_$}qU z?2I<;nLWR@v^*A+ymep)+tPjWbQ(Dl7zHmaV2xzp+jTW2l!2G))Vz{L`5#Oc|D>25 zELvO?oRV!@oE_G>WHc?f**x`9>%zb$!-kf?2`wUb+oUoAJv*;8|7g%V(VC{*YSX~Y zx{5*SLaPodi!(=mlZAB7E778j+RO$Fp~(!#4~DZIxL&n)b+zkm$r1*=3ymrlnpHFy z*cBLpNQFf?eDR%qa{V0H*#j$vToUeLvMgL%IR*Ky$% zDGnh`4{p&3O&TW{7zNx*E*@I)VH(Q?29FGuZ9**`7h3f>{KXbr@rZ5gn{#D?%-$uX zhh-}c`6Ov7Ih6Kb&nS31FX&+ zEiM}x3^iEnCWv@2G$vnZbUx7j4xz-C)#4{!IJl&ZTDYRv3s}k1K1T# zFxuT<76@Q+;9ybR!Klu$$3=iyFM~>|pNbE7qu zq1DiXNxh&cc@IOmCxg7!^q|+GEFTtL6McOBtQs@NK_Ny41_nmE1{UWR$6YdfttT)! z2(Z`+Fgq)>=m{`8Ry6gBGH@j{y;p5jSTRBW0ke@rtVqZzdkxlYmpT~=8l)OpauQ4w zXR;MIu&De|%Wq*wv+sz&mejs@nC1+&QP|9hS;N>v7(9Fk{LKN8r2>&J1Q`{ z?1(P^!J3!%F!x7mZa};71139zmdHC^f;$?OBD@7Q1aSSZj}c)J=WWsCVsSKJad2q4 z{4$uggUNxR#rVYy#|@9&e>Ae3;5alhEl}E>=grj9N5c94Z4i=7<~d`v<`iS&)y76y zM)eJi@)=C30-3S_Og5EkSQHrARxt9MX!PKi&bFe_K!MTb0CQ2`#M!3;#ilSYdNeBh zZ26Zc?`W}ls=>r{6-(7m><;Q^%9mt~)nMsc67natQAEIhf(@fc1>>4;@l0m6E4QY8 z31MWO=FIA6@o1yxzvnNNU0H6=`*L3B0fU%fH0#5JqKAt7YSS4#qPMa#3ahq=l(iSV zNE3h7maEVnbD&A%0kaebW8(=1p06)HX>;>zV6;AvXg!UC$wh%h>AFP*W1B#ujX;Kj z1T%lf{`j3;s~2=-{^=6g5-u8YH#)fC#Y(E61U2AN6%Orb%p=d4_ z;|hj9KN-C~G_TBN6j;$HA;EOw-~nrmR)qwPtpyHV7X-Hn#q)G9^DuOZO>AH>U@$Y_ zVpdt=wt~UFfhD$s<+m5(d!F;5YW`tn{uWD8qs$nY&%GCU5%Se!FZbi8zf7`?BZUcRMLaV_84p2jkY2U_VORE`JaC%dco=}(P${a z^7;^ih)2i04Gi1{96SYu?<*E;S|B>>RKw&Ag_;)4CNClcJzmC~;yQFt|LWB6<2qT# zzdRDU$#pz&_ml&S{}(g1@G^u)Mobd3WO}iY>xTei2h)#kDWTaubyvGs7HkWYX|`B! zS~`L+J%ig8%iIfaC%XRl)RO^R{Q)p{&slxZ>bpaqM1T6$$?!1}AZJ)=3kKVUqN%~`={ ze(d(_ozd|hTB9=_=BcseX|(4aV3qy%fGI$cgC~J;PI8XGgpbQ+ehg`1;H_A&xJn_T z^Cs7VrmvCmsuwt?zgWPx?_Qnir`@N%$q6OP2YuS@R;us8q{qQ59>A^tH+Pa>li-R5 zYk?Ln4)0|hO=2A_ZUPOfgpbEP>=J8i3i{EQS;1aV!JeDYv~0$?uM=iv?r2`GCC)L! z>Pq08dArWpop1G6a9DiD{ok_wljgh<7r12};{30#WZkl;Rlfc!Sqs**E)LmiWHvFe z=l1y}U)FPUTwT@Fxm!{^c)vE2#6%RbxFvEZjOLg(6L;6J@llZ?a+_qm^ zzph&Kx*MN_oVNyxbwMLThJv18^BtcimW(xGGoEisViGwZ&KQsp=sfX*8-rXxRyqUw z$@6TQ4J_FUns_{x2kmOJirD(|633s~jW!84)3-2to?y-VVQZGbtmJTBwBhfKC96Bo;7Bn^U@F^&G1StM%?kInCLH#=SA(O7XrQ&MyKR?}UH0^ptC!FNmwukePlu25N$AHlaGY#pcIOe-NvZh}(B#_5A?W4Nq43Z+B znO$g+b9tQO5gsnFZPpSW4m74;@aEOi$*^1G#Olp&<6zI=(8Bod;^O#<_em_g0?Ilm zKbTjZ6ApT_<+%JgA7%$%PL~3QKmnBt4!jdNgxD1dpABa1-4>$4^voeduv(`II>F|sVE{c$rx&@(|QYu}ttQQ7M@etV|5 zY{ns9rJZkzIwZ_GKAzB+@TAd^tu-tlgRLcQf+Oo;)nBd(`#ss`G3%TuU}O}1!tTV% zawdS8joIV^!_4`;=C@cfl@1(O_0;l9LlwKg5fNs-E3XQX(v%=i z%_$q6_-I~^czlM_)pi0y^;6Y`CWT81*$+%^`ZpOvhxD?Ts(Bwh_DM| zo)8=LOm&7(ffyr8>MSj(SzL1h4vXwMP}!BU?8rB%Q@0;XI&ISaz|mE1DFaidf)~$n z#)X}HRKc)GS2 zGdR3Z!{IZhB4y=- zjfTGka^4&`CgA+#fgAr`{y)|H-fDk1rpQTt+mPh&6HVi({*yTMW!#%RApRg zJTvXqcGDimRcKAlv^;CXaXy<}Dqm+6iNO=mTBS~8v#spOf&ezuiE=_qgWgEuW4 z6-FIu28=>svmM#`CO6t{Ileke< zypm-8g5}n7UmDma2Hcn|cBSXJeCyRx&NkWzp%K@+^A)Dz){U0@OYH~XXYGdxz7{YU0(_w)1TzmlNJ*DdeOul zuacv25Mrh;>#_aE?V6Jz2ktYEqr zV!)C9W1%x|lY=1h?5zw(8l?_qv9JjVux5X7XyvrL&2&m~^%_S8HiH3Z;xMPFCd473&`yBqci#f8mzR&4sa))iRanFwBygmK%cj+B29w}NXBO{e z6WQm_ND?VrutD>i0;`(ALcWj&mrZVrNz*tMv7gjBD%`}Zw83ye2S3*VffbEwG*TQG zs%Nm>P)T^KuI^&{D)N*;V!)JpzTHB-9*kV`1P*iDdD0O&FXtx9r#yB4vQrl)$)v8G z6R@H7b&>Z^BNhXO&X^b9wtorueDLO)53R~`b|lE$-R-oh$J1w#fB~?x{k8R*I{mt@EVTFUtg1Q8@kQ4cu6&rh9UL=XP7BKOa zIko=c-7DBI!RcSrTE;z(*3_NQI;z1E$dPhCfvYefgm;kut4P8E_8<-??wn5xPPM+6 zb^VQFviHRfo)rm)JF0co?6~vxSD(f8hvJM^S`T zk;HT3#*C=T4&FVLtzwqP!Zt{r@n~S3o0ziFW}0p9RR#W4bC}rX2s?eOYcLO!IHFl{ zk=ye_l2qG+CRUdQM!^JkX2}nB3k?=9iWqn?i+-qM`xi8mU2}>Oe^W<`h{6IU_rL`f z2Uz?A)10(Q_!?$iPjT$N;;8xO0B-~{*9=((g99uY4R@YK^ld%v^}~_P!O4sxF3_XV zG@#jJ!7SrF3@kgMr~1ykt~1Zk#Koa^n!RAi(}UBF-MGdXav*ep^Yfo!FBY{@X^rlDKc%osp29DoYO4YJ|AJ2kl>SA(C~ih0rrd@ zF{b#68Ei9M8o18fYOa*$n!_NLc8qmJgXn^(=RBG80-6;&8d)0{&L=R`&$wTpBERTG z;^RFB)!*=|{Am<1VG{muW}ysMLCS>{F%3Q@4lFDOGFm*`SQ`E@{9#yO!^mcGfQO@j zndQK&1;<$=M5jNMa8x-UYvIhw#iSYFv~EfO|Cxq$M_NQ4N@u#>%b4+C)s&>EUxS@e zc^}$1bTTA6|LkqfkhA{dJacN~p^0&P_ni43HQU)dHfB?NHS?zH(JAr_he|8&lzE;> zc3_)u;J}H*hf0M9lnNDt9UPQv-B?Z>Xqb6OLF6EtK>PI6V|h07=N{nKd())&f{`_6 z*0--cixV2Se)uf@;lpO&sBFU2!P_KqBZ_r_!{QsFix)I7$2gqIoN(PFn){D~h`@wv z2N`9qG+wK1^sPL|zK7kZWcFm!**#Yoxn6L!Ej%Enz^&4X&CoFXY>tOtz%E0^M*tbT9MLJwgVqLMZ7aRT_Y+BhM@6yPXvn*xN0d@;M<*Z}O6S_=$ zy_gR;C_iDcSed5j(Y*CZlS#&DlN*dH862FZxKF++tis~R7J5M1#9ic%i_9L!2et>> zR|Gs^oZ`Rh)Z?k{Q%&QXYQ3MK9Krlt^|2AtLgRM|VNt5~FAi7qJrDht zrD|uPEO5M9A}veE~^Dt?%rKiw8_im$)b)m$F~biDRvX= zjCjsz;jt^?u=NS%=NBJ@JX^q*lv??TdH%PRl}V~qkD67sJP(f&|5UP=|H|{IZ3nnt zJg=A-^KX%BL%rUtpgDKaYZ!zb?yE0glFGQ$l;DgojG9B<-n10fOSiQutbx}2T@fACi#w2EB?%@v2b9H zaZu)HVhuT9@0G%OgF!qcgN=bvmE#brjRTY3J!cIEmLH8KcMdB1G@C4O=4Cm+dc;xL zrfH+eA#RzAjt98bJWz8qIU=}`%lVDAZ%E|snJ-HYiIqAvaPCnEnbK{%fZ2Kt^V~~^ z?boQ!d-gK)U1sH^dcq)o$3ZS@wZkVjxgA0(GE7P; zOz|_D6dDdh2Wm5>9AHmjsDE*Q;|jx#V+`Cp4J!hYm#?%9?UBt?`^Jdt`@OoZh-q z)A`PE#lCy!svfj$CI8<04=mT@C5Fs=_@Mk%=KQQR^VjLkFIyAJ)TScgbV966{*R!< z18xBxMm_(H`!Oj-xr zzI;3+q2kE*;QY7F_%^lqZ2xA&vuQBKTQ&;4X!zCUsHo7C)Ws;@(a86tJ^uweM~g%1 z7KVRu2iUKOb1*bYTsh$CKbCwNVwSC^_nLX#D=8r=r4owm}Vq|;Xt8hBW^&I3`a7^J1qk>42rpX75 zMkWQ0CdHlxg*}Ju_c&WPH1XARvTktr#~stKq|Sl6#)0>RdUoGvloXX3=YaYX}mD!miE#9kD^SMP70arX;!Uvc2_vW`@zwC#bGz^L+lX_OQtm_ zyEiZw7_zq}It-YPd}4G6 z3+O)k@B)iId+7nG&HHC5JovYX^PTq0RRsr}0uL^FxjDzz*;It(N{?}dvzbM+rH8Tg zi);3GUYh>jAn=~qe#2I?&(84yPO2eIDi%$uI!^L`4$9nU6xiU{Kj&!Vilf|LrYIgg zkZhqSu2Lq^!YI+;C{f`kp;9KUQYNJ0XxshjTwT+t8=YVNeGcsvx*K=m$ENG*F4@XH z2Q^Hb1yHty%S7 zxa1s{Gw0&s1*>_p4_l@jw(K!Ay}Z))$%1)l#nIcp+Wq-n|9NFJcZJ>#zgYW2N)b(a z@0^x-bAqp@mFLZY1m^eF^~rn*o=Mhb5;qzpdY-m*H%eH1OD=YlC^;zc#8HajkotoD zwA-A&1P-aoG-VM==1#Q&$nh2g*& zi32P@a?5s~a#~BIY;q`hR|L>%q%e z>c`fZM*CYe>`pTYXp(x^v3Evs^MBLGJI^D1*u<|hoJz9jY;Tl((Rg^z^J&|69k%F^ zJ7U!_nNjr4fwR7h>z-Gwb3Z8Yq(S%zgRn)T$eUH&QEL?f8o4|et2qzJaU2qHVf>}l z$lKGXVB*MTp~mroL7=8V+3mQJWfBX+LFFHe>^uzYA`I*$4GcUEOmiIgYf^+697Q-7 zMO+v~ZZL}IB+72M{V;?tP~Mb5WdWnZB8L}0y}HVU0{xy!ZS|j5{*3Ljp4uP%G9H7n zGpq*~Rcf}HHhi_z@#cH|;qbSxy7}*pu6wuS`}XMhHPP|Ty)K7~Zf;e2^6!MqhJ!L& z82g{FMs3-g7;j-)+$f&&V>Nq|Vo6K*ArVCZEuT^rg^q)~G3{qHf1ka%OXyAmZ-6ah zA_Hs60k#x|&^a#@el&V=916eLFPzd)cHsc8g`FM9 zcsTqQbztOhV2ohsv+7_FaA0g{D6u#SH zs4{3daBb&v<||X5-p*+7HL0EXGW~#X${w?fX444gMT-wFn%~Uhw8pk)4}bUbDDEBa z-aDJ_@LP7grmv+nV#`53f%?cftP?yG`C@j6*D%U?m``SIlH02Ob^G%LVfL4VD>?q1 zap0?QlruQUUvRb4*m3?1$K#@pwsy-4`!EUzxc)aeq`-3GUw}hfAA|4>2dDG~2?wV6 zwFlkZ*&Rd*5)L{#vJ2WpY!Fa7E^b)8%R+!5v2Cti6;Gq$jR!1TlDtd?0+K;bni&)% zSr`l&8W}kG1q=cNf*YCG*mI;-bS9`Z&9KYYvN-VI`PmulU2?6FK~EaldE_iwe?4Gi zS*@xNv*QEvr7byE)?{8T^P9c7dF1To^RjxGSWQ&%+~M#r$d+rm zO2C4D9^On>SdR2am9luAJebn2;L*flSSh6HFCrkSk#b@IV>6eC)|~SJjfz|fYLb&) z??_DcpC{dVWhJxtx|5DvY_c8!{EwPjkAdFvW)KpPA1qJYcxeNlZJ>V7L4TiMtg zG}Raw5)v9%qy-+ROmzzCRA1CGX{FlCj;^j13oTS$E#sf~NnqtdfetOL<#R4wII>)5 z16Pxfmc>IRX@x?!*&CK^RMROqWTh^>^;%N;-1;|~MeBCG+qGK1?%%D~>uXu0*GRfc z1mlf@uo3lq=A_njVw_Uo?Ok{!1zdkBdDQa)daZ-3=EtC5(-U|IZ6~7 znnSz}I5WHSc&cacIvj{xUh}B)@ccffh9k2Res!*5pVO-4xtem*lO!?Ac5`SrxS~}2PrkC^h+-}ahPA!=)eM|r$y%?1LyB# zev;(yA>mP*?Y9i3iJ3|ZS^iCzKc~DRg#C?yi+zw=bnq265iaHl20tz^w{S=~Fg8gT z6eKWAmQo0B(pWMvO)9f!;<1^TUX@CRreruZMDZ*%fvgJI8uyA{hXw#}+Oe5yRIww`BG{Ei-*poBeNKCL!5EVj+)bldil9uBi3I>G#Q zItMGx_TE*goPK}hP1PUwJN4zn9ZDD)+sx};=k>cOd{nl$&+^hSok!=!%;zqmPxW?a zEIRS9P3n`tl2);X1B^WOA2dRjH&;Dr*!wxKs`_>M%Mm%qwbO3G`;Xz%|iBBSn$f zhBeA`Wk7q-5~bx9*)h6{WY%j;XB2T!5}JLQ&FoXC{`tQh9WPF>S=qGmm8@{~_-_WqkGnu zN4Ff3+QrRpuotK5y0?|%|U0`4VFT)#0Ew;ktxe<6x1044lu=7 zIIl9%V^kCJwB$F~DICD(pwS?1+F;>QnYLK^i%pXGwM)FJi2STN>>|p38pZxXiVT+q}9OW zvhctX^Bs-nv^FrVO*t^pT;L(z6erjn6{9h2Uj}=}EY9*xS0ZK?eB5~d zpy6Wqt1sLdxnpu3EaDbxc*(kDyRD0ALb|oHqi-(P-Pd;?aL!0N#J}oyvsB6gwz>sJ z@?DNLu$d%GVf*Re=p?`-Tf)fSVc??1`iM!YM}hsNE=PIp0cn9P3j`*faA43oVB@hz zS@!Y*HkGV^#qZ0GN=}^6CZN=CXgY&Jft(TN_C-O9irAjcHdVaXwIpd#QQze@10zOj zX@*X@tZnn|G90&!)rnGlRlH?!EO+Mb(p#6#vM%H~{b=fjhJUKJqMA-D(d?b=^li7r zFS8wYZnRCFD-#p*GKoiNfoAIL?qFT}AWQ3FmDJo*wlj>j@08fVuxQqV?oy45QVI&8 zED8(cHhC(9g3h-r__!*^=P|dq!T~3%PJ1bl1&s0mGnHHun5AqMv5RIfgv%N7hLdVcktK*V+x!|MS^+nUlCQOt4GND~qaFuKB_C!9% zY0WvhfzjS~#I_&pPIhlR-J75DSo+?NW4rIK&C4rbmfkJl%&}oj>OXhQQ-P;eY8F*K z^Kkpcv(zkJW?6l|=ptGZ>4SKXh0XEPQGc;-uVW$(4QQtmM)aj!a4dli5SK8O0|!@^0a7 zkvMaJ-S`0`Pr?P}m4_G1F=c%2XTzezzpywav{2C6KY-DZ;dbytgR;$2@8+Z_&wiUb z-*8JK%aY8`-Ev<%Ui5FRRkf;7yW_HKMdqnk8w<0a(y!XSKNO6w$YDrg5&KcZWWM4l z+XEBkgtmA4w@)`d6<${@y1{{uN6^;Z;n5z!a=~C#1}42HlH$7;9}%~>AMg3^$3X$# z1&(qH7Hs0Vvp~^ug(K^Uux76hii*rH7#LXtyKIgeFLU*2)--1>(-A`GtA+N8O5Bn3?>thHq|n3cm+W87O}A}#xHM!nUUy8124AOCy! zVEL_dop)E1bk`h|Id_oZrqjYtd3-MvPPFx$IJS0?PFF=mw)700dqVF&9C4Jn*DDjj zD&TNnl^XxfElj69E_~XvX6F*_KoM7FIo4%1{+4M8Yz_^ae-^Nv;AT?NdvryarN%*M zS0eu;1@;{axU(2otllv>FmUa1;Mvu{7IT1aj{?(_4NS8f1l%5+k$S)sbwEH)k!AU* zRb2lLN-l9=p0Pks;321x1A`0$gOozF5CempqtvGbd|ik6J}~fqd%(L(nZMwiRMrDc zk%rj23x%aj4=)ZOpGi+olI;BEG&-V-xfU2QIS?) zWV^IVa}F!xKdDBccNZDe)-%c|Fgrb9JC?v^c3}F9&h;%xY8EM)NgUUYEoJdq6CR|{ z$h1)VlA~@8Km_vV1mnI)p$ z%-a(JPu}V{8NqtW=!5i;Mwxw$S}TKO4mpZ%PZTLR@X;^ew9*1;-w&r|X+N0qn8k#D zx`jXMq6Co}U7Rirock68eM(^4RmeTlYtM_E$y|q+I3C!{V2G>m;J49ZIT0xErXg5~ zk>A0Azwp6IsRq_P1uQ1YoJ<=0UJFIGDDtr#l+@GZ^UIA{qss74K`3eg-);r284DCc za%@->`G4dvcs%;Ur|^$)-%^c?6K2=>H^r$iAL`?mQxrd>XydTd;(p_Sdx~n2Lc%{^ zzU}MixR9h?`b52UQrRwE?;m-KR4(}5IMJu`L32LO$DBZ|rA1nY8)Z%;YF!o<*Y{)B zaM1BI(CIb2@3G;NpFyBpYV|y3)Qv=*2bG0GfzY|;Pkvc=(Mi_}T~Z}H8qYhfr=@jZJ`{iX;<3&SEi=Q}6P z`d;%mIp@>K8@$X9d&PfV(YpNEf9<8-A_v8NMRrexA51XZIYoQvE}i9nScL2tg;pqJ zR53HFoxiA+z?Qn+XV`-O`Xjg4<`hUbe@fhE?Z2mV5Oc@IV*sd~4FfcPc z6bzXnwarnMA(3a!L;j!#ypIl81~lw(>d=sR!Kbm1eZxUXriV-{4h(t=1QZfw_9gPn zNsxNw5XSpl&FP82LDjaPC$HByxhXBk@?eym6fAp>QD%=w$d#?Ce>;UN1!WiI>Xx`z z-P|Iz&M|&YYQQF@9$*M|g=WsM@U9xzQhAn+%FrQm=-+<8U~Z7vT*fsD>+Zx*=y zNobB@X>~GomQY}P)gZKL^*SBVoly#$c@3OX6u9ObM=3sWDTidgotrD_9$GiPPB?8WW459B;$_)O zjWP}ui6;|f&P|q)`6j*Lpa73C(A6ya&*s7YUCtYvyW=d=_1Q7s|%VpEmlcp$Y)C3*lYA7 z(R4$z<&Gl5@~Kx7WfFvn8Lt+ZZx@fSsP+nIGd9!K;AiTZ_}R$cvVfB{?*YHUL6LtB z%ti_ZQ~8+#9Jth0rhN%a+o8ao!N7h*L1+`Bj1nW4oufdW0^gMF?xl?UQtuoc4=6ev zV2Wu7s5@{mfI-0Rpx=&DW_Q{cRT>zbim%8#S9zl#5Em@K@Q{DW@|u54QWjE*9C2q` z*1Xg(dh*9Dr(XKOYd;2UJqE6-A2PSTTD%pT^mUu1d0=OQiZn~P@AWGMvK9t15>g8k z6Q)n9SkiMhLF(QLA@TJ>w&#nr!X=k`OH50>S)^NjS7qVHElE>)3X==7K4t7+s;ErP zwP1`mwX%Mh|;bdkv+fkn&DUSJ_-Lzm@}1=?N@`63vY z?Hpu_6*7aKa(|h~nYQ55uIbq_4q+#r*&pL7KLkAJ@ZhMmWf=fKJ&n}MoV+S`D_#ZiNwyH zY;D;TTD7f`UHnFd_=^+0W?6TOK8h6^CW?n8CPpq#-8t2BN6W?Hy9J3$xIA?9OYP=Y zCu^@SvGh+bP-CnqP7Wi!p1$D|Q}5aTQ1YNemWI3z(uF zu)jON=hw%YqQDl!%Aa|FZ3(*om*PYN26L67J=+49Tvjq}6KtK6$|#q>{Huon%6NZsb~2w(7N`L$CGjK@26XSr>dU(vY7S4 zR_}{T{H3IdYPN+QF7I+bbsPF!1S4#g1hX}vq|Mq}=%7)b<+d1VQ3ci!t>2%=yx7AE` z3M>u_I2#gJ+z#+PI>0t%0cX*9rntO4N4%$aHE6{x%00!pA$2l?PB-V4Z+vwp`PVqI zYAJFtEjT=J(fU=aF$}x4y7(A*_OMCi&b8aERo8H6v*V=Snp=Z+&p#ouWTCO|XVJpa znr)@AOBW|fRvg^csb0~rXS*PO>&457t`@cv?<_B$b@JrF9aCkdl@^Lmc`$A5^fRks z+A?QMzqsj>)Y8uZ(|47sF+W-$B9$oeNP+KNgaB7#Rf`(`2Zf#^2l)RyNRo0~?SHQ5 z3=4nO1LlSX{(qAi*fI_nJvC%MqQDuY;4)i*{oTg(f^%|KCC!<)NJXWg+3c_2CdM1{ z6d3n-*BNdWWLV(Bb$Y|DuYW`)WwAcwt7^!&`qy5aN&Ci9rlJLks)n*jv0ESJG34J_ zGIycWG{vRCRoAQU-Z7|H>?3(^!rdF5H)O9QCeECDa-m%DY`NCCfA5wp3`ubF$N8g7@FQ!8_-G4xA= zPHakZhQ;^8_a$0q8h>9P@aO{BeC{#Seu6Lb0JaHwE#bZTbj zm$0$8@Tswpg-3`ZW5V%BR*!t9Co88g1TD_2XHYR<*sR*b$tGZ6F@Zs=X(gx7mVz)X zKUPtRS3b^xTUA=t_AB{1x0~hOD$cq2FUmOm+^wxUudcFOy?c$+mc2E1Y+IS{%Dz-`KIVSptCOkm(IWI5m>C%Qy| zgQ3vj(Q!`qvtLA**cB2Om>ConGz4>~Y)E8gv@tr%%f!)e(4=XB*d11WlZHdeOXp1G zjn3Yhdd{-&WN3KbIfu<>jBNhB2sb(FaM@?#S&@Hb5tg?nY<`mT#Ny_R^hY6POHS*0l{GCQ-M?%xtxC#-dY+ z8ywlrqz2q6J{KD9z#^ve>mWNH!$fY=oi7$L>n(Y3sFm*>qmUEdl^01K95pV#jkUZqjW#^yvDmaIc5-s;g*P{*_djEEOWEZtHuH&Dw{BQpa$sKC&E&a> z+agmV%f#|6zu4{URZ$FZU^pl$;LyO#Vct-1SkxpzicRXDhoOgNrH5myLg0rq#(7H~ zp1hqWKIedRv6}hyJH<;ia~$~+8YVV#v(`Ld(wx+_uvum2ghkDKoew^7aOypn#4Z2g z;UaE%L5m9w%;Gy7nz?=AI5aNYHS~svSBAq)HjRLV8+bxaJd~bj zqv*j~9&ousVYy?%{Rq(=GkUkzrgQ&@es&@Bdf)R|x04@ruFqaFu}yPl#Nb` z#r*Vp8ue)R&$L$)g863bc+hFUDrB;z;1G+*h7GgwW?edQRK$2=##^Q5l13-@wB{HU zNv!xYhnc5=f#c#G$yf)DX+B%rIoh5madMhSD0QkR6d0VUdf~dto$;TLg+sI4Rp~>= zImGt7TE!H0e3b*kras+=A)k)hCkBg#btJZC@Sk>I);V&4K`z08wO*h_A&H?eYNkqP zQ$d^QCPVRj&w_%M%iQsN%Xb`Ixw_CbsU>mW$8{%bmlZ$$aLM9}sbuZT)3WntpV|9o zsio@MCN`0dzEvSdO_&UP+6Ag_%nJF?(jEAa(_x7tn^l?h{RKJYyR>I6s0)`7NoZhY z|9wENXk&M*$1B0a2@LE<1mtX668I;2u<#Xa<8BOmpwzG7AYOK{*Oo(}^2tIs6_tr5 zG9n2K%mq{BRTngHYBeP zxm{rFn6sSWo+2Zw(}q?(iKIRy4re7dM*r^-hpN|gHSnY~Tu4gYqWQU4BSiVZpVQGdX!*SIXbM`%de_?&m-9 z)+9e&#_I5)_0tt&!3GU}mm>oHMjDAcRSM4Mj})-&YN*K$#kZRt*?*z$t*VUIDNm2xx2|a& zc0begHYanoMcN#rsoP>z^YhnmSe;;8dH05!d6V{?hsz31sGpg%@7+{=ZN;0$3$zuB zot+ym1(Y&}+-qQ86Z3S1SLqDi9S0(+GG^tOJug4=@of0~$T%Ja7Ut)F56NDS^7lMq zEI5gQbq>E`r}LSe^U@b=R$*Z5aBfHvsS;>j_>;lkkfDj+N`ry<&F3bG6Ai4o3)gH^ z@lxb9zEUSO_d=_x)k5|H4XuL{{6Xs^^&zBQ!1f0?x7@8RoxB{e>vinYH+*aMdf$BS@PXOJM3KGb1)J%LgWcV0%J&xCD_7$9`2Opv zujy?JKQzAh?6d1v@`=50kX?u^QSyKS8@uYYKg*nMx~Fwa>{v3v-P~m1suZE{ZL37r zCIm4}U!%cPxuC^liX&(08jEALtG=#DjmeJQ9JGD+yWGBN+jcVs=^F>-%r;ss_kOHq z{qjg@k-xS2%*BtAZi#$g5IUpK<(tzayyC(yHIs>*J~EA`!gap~drw-m+)A*c=>)Ur z3FrqYZ>fPsN|f+v&F zhDN1;M9$zBwSp!J4LlqQ45|&DUNQ>OYmUS`GdgZ*95$yo)T@4K&w;PK<-8?k0*UP3 z*|wQB$@b+1^qrXNy=~(8`(I9Yw8_k87S^|#P;Fe%BpmbHMQuynf1eViH=!RGW_}P! zT{@9z>BO>EY8pWoIObm{Yx=-hX~5C6fxS0?-Bf_J=>w}31Ct2@lU@RQ{Q>s$1kS6@ zyt@<9TMkq-2&DhJR?)VR`}PLzT>}39t_X8Z4B${+!0hmV$v%O-Yy*2+07vBmCf)?a zACnc+gc*3I+ONCr;juAMSb^bxB7>*_bBFXnE@=y6Fm7Y3GoTA zFKZAz8NhjR0#DOIo@|NQZLQJ0m%WN#cv&9HcDT-FJ)!-sdx@)f%$AUpBU?*eig$RL z@x5D6_j-BVodeu%1uUroTniKG6Ss5TY2#Yhz?HB(H#C4H#MR*cakZ>RzIz&(q&}#L z3$r%`a4k)6*e1yCBf!!6fmJ!7S=oWvlYzx{GK*(G__PM@9SmGM0=SFXxFb_~c3z@LjJRZtSSdsQ!cQ29biqL((So`z2?C`r6mrWP78#@g%rPC7Fl&XV4b_7_ltm) z$K4qm7`-O2r3SEu?_dc{X!Q+Yu76|?EF{82|=-h&~EDee52+k%$&P(Q;JGT~J zU!GiifbG_HX2G3Ic?mstzt^^1Otw}{@el8i@vPH35;Iv)|IP;P?-8B9rsvL4;M#X! z(!K-S6FJIeC9uuaU=UMi*uLESo`|7Df^?99VPyc<{0kf%4y=_M*e4v2Z3qZY3J9-b z@GnW@h&jOM+29&^gnRJ;ZjU61y$5*rHt^nh(3HG@*~Eb9ibnYE8?2?QY*rIkEE?GB z97N6<8>^K539!D8V2uu7-DA$`ozUOe*&4Q^ zpFe=9AfavPgtjvi+SpglJUd~+rwPq+H)cts(>Y6G`N@zeze zxOXMY|GJaAKYacIBLmqFth)vHb{lYY8gO(raMV0tNfO{N;^5f4gMCQ@yWAtY`p!b1 z4m0)zRqN6f`JV=CH)4?7z!E!ShKB)b@`HwI299pa{_q)`|1u4jLXBr`4&<7Bl4tUS z34H~N=iTI4_>#lo>0*ni$+ZgX1rF@1PqGQ_WY(UrM0*3%k_Oj{3tXd8c)yF~See$z zc{-h$rh3kd@2LXcoddiJ4wTNmz&Tfe>)-+26PkSY9`HT=zx|0~kCLW_Sd!rYAV`CamdY;GAH<(IeSEGk{I<0E55>mSqWCixd7Wp0jhY^{K_T zPjT4ZTI?XTr04?s>YHpX6WXhGFy$Id^pfJ7{%J|zsdkA>iH8iTCtGJL%$yx~ed#*| zzIO+B&scUI_2fDJlJ}8-KHF*DBOkaIUEo}iz$WLwsFAe1?3A>R0K3Fx=8y$US`Mrd z2F#WU95)?!UuCW=ZR4KkUeRVSZ3zQsdjrQ5sg+gJI1IZvOn;}#2Qa)<(e$|fP-7cEuVssHW+@4FAncy^Z_cHsFa!2h&BHEz|8KfkySAJ~xO9-0~?CD5RL zXklnY6O;c_>pd5lngsZ68gR|h%D6e%wRt*+!UV>bo-5~n;A|=2n6Hr0@_|#on`5Gs z*2P4QsSdn%8+aEd%%8%=(XnC0M+Lq~30$Wo=1)ApZc?EA{6|%0GQ*BrdsCTpLY=p! zF!Xx_ur(`8F*~(2-kS4N0B3;`$Hy6L^8`|@7VMe$U}m25_6Mstilmn~bq8l8swS)X9hEZH>AF8vMzn!R;{p521-v^83TC{j(D$fFN(h^1 zz}3OPxh{Lb!Py-9rT2L7FqyKj{cqW`t6=KOPLAdS+&4e)y)T%0a>l$jrv5v3nccPa z==-we=jDL4;tY}>`n(sg&U?VU_yR}&1CFT%oc#fu)qnP#V&LqIWKGp(aC6|=#lSgV za#n%#_O1gbdhW1yDXgnyI8ZQQmb=u%+sl{uoLb_U#W78wsrq*}zfIG{58FT8N`4q{ z@JrB+C?Wk>g{ti{mYvzjxm952#|Qi$FD!d{;nX7szLyjDju`OVY-0-(NIi6M=fQ_N zvp{Axyg51^aGG*+eiAXY zOki(az`eI1|H`l3oe5lbE*#xez|p3)xzT~yzoTk<7K1PYgO6h*$Aqfst@hiCnBQxx z+4q5O+RB;LJe==eZ=EWzKm0kTv;otu0IpLPIILHl=-I&WZ#7514!g21`y?Iq5{Hwe ztJ_^3%wm7THtWsFwhtVfKAe*Tcy|c!hFUgl=FGAEmMpb%qEoAr1drr90lt5~xF=~G z+NQH}YWJbX3Ve?nPTkqCVP*kmh`>g+W+l!q1}COWK3(IxtC0QX1U{*`^OhapURJ>2 z=D=io+1RgvWAdv_5kGhBO5pPA;aKpGfzyxSy5|Ju*av&|1+3lC&9P?zZ|oWF_FS$7 z1r-e!GPVm{_%Gw(F=MrBhX})kz04OFLJT-UC$MHb;QM!hw?AQVQ{aSZn~Ti}tsV+o zt`8XBJ96g#UaV|;Y0@2bb=`GE4YRcWvgKT0woW)%KKtaXJxnzZPM+MmUBS`hqQR+6 zCs#P==4>{JX+NE_Cv(U2*WBASa2^ienIz56wfc1UCW#e$b{tXQN_1!CQw~*QG)$j; z=G2-p^#Uew0rndWd{-{qSR1`|0RsoC)XGT#9BZ{RcCY51xSji!0^i;byZ3Y-;a!!H zWYF8Z;9?~&*L;QZw=W#s{XlNd0q*~499|9#j0Oz{y%3ptEjet?U)NZ{IS zus6qwcgO2Z)ps{#+H&pjUb|qzb$x5Dy%&z|b>LX+*Yt4;i`WNwi6+GzU1pskTiFY= znHU~N8$8}Vhr6ogF{|Z8(_F5Z0*~hjJZU%JomlXsX!aAQS+{-q{>|EZmu}&2g_H-W4sd&B0?`6C_QA;1DKXaOTFmjgn%LjZ99`M~N;GU(xwY`A*U;_8H z0`4OX?|;1GW1ju~WiOkFCj+yCTE-^>9_ITOm#QsT{XtxSW$6UI8x7a{*k4`kda*Y9 z<96?rSAI3!2;dTbcZ4zTJEGJ(yzfn|Qe>7E_e1bdI0MRE2UJlGeot6`oW&1AK1G;u=oAhzw7MnJ+-qg zy?bJ9z;Y<}z|q>b))wql6P})Ud$RMx$%_m^*1QKpZnfW?F!AB>*(Y07i|adb-f@5b z=RW7lb*_U7JVy$6jz8c#F2LX5&vWp?Ay)$>#{2f$pPMdE{J`$Nku!yvub)?h|J4DD zjF&5aPVL$~!%Dhh?=^$pn@p@X8{U4Y@_+X-hga`6@V0G`xU+$`DvUk;t;m6*zFB)8 zJ*ZyoXIP{B;aH}@p+yQ@M;>r5pRlI-H%I#c?)?w=7B;YWKi~{rz}S&6Gf@BQq?+4j z>b`2pzMXo8IXLd^>#DcK3G9gu|8DwyI~8lYP^WU!%k81R+E?zeJbYDEwWZ@R17H3s z&RGnH<}h%tXAn5lCu&`$Gw)#IL9>OLn-iUxn3yGOR!mTEJjy02Wg^h9@zHTEMQMuy zK}}h+*u@SH|1llq7SRY^oYMD;<(2SRXQTW+Bhg}!Mb7gZn0&vie4!Z7oKYv_m2qWV z`TK32UElrgY;&IQA%x>Y10xd)yMRPO0z*UN#suC4vC;|+47}c_WL7>pJ$HrA_O6}3 zk9sd%61@M8pz^fNck0@bc2k?Syy#atQ7iKGrStNBnKLsVZn=5pn6b1{uG9M)8xNiB zy0k2OjoHJa-Llr}6K_9EKDjN|bLZlRheUIy=V#q+3O>9oH+p;C?L)fi+iUI=zc_!= zx8Kh%c1Pi(b@PlSYX&d(os+<-ttOCo@Y5Wlr3*JXHZXJwE2_+xzo7BxZ6D$Yz;xXaS!ii=c;wOUndj4$Z8UY^_pDrx)H+ zWS?ID$ne}DTOA37#tp}uUa#4D&Wl@j14BXr1Jjd9r+EThIP)cvx)gh6R7|*|EulN% zM$!Zx6M-dD+{7lnIdxKFva`3vlZ?f^7PmgE=~XPtXf`pq7?XR}bgw{Dx8rP9X~hCXRp28TW2Nqh1=Jlw=45OCnmMh1=p zYc`!wdcB50fPsOb<55%+r;5eJ?V^btFTYD9U0|LuS*On*-Up5Ah0 zjj`F0H9O82U-s!eE9%9-#Bn`tHnW-in}C-7TX~Oz%^g@}ODrA={arSDrg*I1T(^^r zB{D@Sf7_&EPZ=!Q?Q7MlDjLzH=`WGk^zh3S!8^(d4L{^64lMdBv7^6{Wy`dy`_~C~ z9N>=&est1dW=<5>%{c;@PbQ`)GO|t*n0?e>bw=Z+01fW>T*qBvgO;#OpTl#q;pzsy z2=9AR*&zpXWS_G-G%{!yIIUAYb;4yzLW`z%}JTZd>EY9k(yXmz>LdeKB?kl3@{Ott{l*6#XK*xdJ_ulZ#l#TPvw^iHb7AnD*W!+M0(n}e z@NuSQ^fMT3VE3{)R64nF?vY;?*epZd7$RWp(dH=u9Oea#9XWPx_;qSUI z`LoK#*)IZ>#oKPRnf5fxxRxec-8gved(pMcZy4V4D1T@WJT-yseNK|_Ob6DAJ;ke( zJbLSdXE@Yw+%RQgYiNjnD9_B&!NBH`z<=|_F_R=FE|(t1sg({azDgfEJrV?jDifNe zZz#-kHCZIsB*3!SWXJY2-b2%QtETDg`Pg6nfJu5cgNS}nAcvFy!xXjvnLRgH87d7M z+^kL|@-a_n;0bsb=kCxXw&yTs;0-5{?GIWcCp=i9D{;`4X+wjYh9akHM5ob|jXY_O z6xweV9!s*;5Ei_X5O8PdGLz+rXPIi+oBJ6ZISw)3%js!nwO4s>*7xGy91JV@CpgHS3-p>R;KZrCph@TxgCdVDhfq_6GuNFDq2>n;PP0AHtl(6kxG=?l z-%F!O}Wffvl?=I-1-CXE@x@kX4Oao`K2J^lX zKmBw~I_G%bI3(6LLFz*rkd3u|I)I}8$(>w;{_f~8dJDPeH z9d%Yt4Oq6^*7W_ni<8|v-e$PFOuS-QljHZj%B?RqfmuNI1p}+ihQ=9E4>=y&9Qv1P z?$G!>L_ulIECbB#|pxgIQ=(fZvH<9i2Bs9l})(uzR^o6zuLeb9fg^fPQHj$70VHj#r;Vnf+wg zwtsW5#G!=tM`t-tZaDb%)2#2uEW}P#-Q2ObCHI)j8VwDiR0>)0;hKBXh=N9%| zP>MK|SFyp=hT$s514dPy#*I^#u8G;^z{nx*zesF{|1%#VHJB+fYh*nx%Z+@1PXE8zzwk31=0)hn;+}8Pm5OS)`}@BQvcwNx;eTwZ@zV zMfE!hP7AKNg{=6e$f?&5!>+*5$hzX%_01KQz(d0F_|9x|k4o8`50CaH9cWYh@!^`YNb2*vY?u1|Txhwf5K*)+6@+ zgg4H*={%{Cr{xHvROfWFm689NST!2$3#?O;+0ruFcFeYxn%q-=YEiePFvp$7y`Ohy zKb|4PVWafGOhKWkLu=2(jgt#DGwXOPw$qq7`QTcw50mdEm}^#Vzkb+8CwYp_X~`){ zd(SGj7yVc~=j0yOgEL=0SS-)eUc%A-`oqD3g!aOO_Gbz^V*)sx7+8W0SPWkK z)7avcz_P*0&Q+j=@wS4?2bN_TEDIS}VjWl%1Q-|GSn#I1tDu7|=)+`>8HZCeTiiFa zxJR%!3$S>9V7C9`Z2yC~JyZ774ra?4%{sf7%^I3rQ;4Tgd6a!17YMS^7rvf{JO54$~eLH~(eQHv7TcZr$Rj(89aQ#3_Kw`2~}J zM2o*SOYi}R;C9?_Qi!-d(!VMfS-nRA*gKlEHQ5WezbV|Yws#2wGbC9a*n%nGdcn9S;( zcv1M7=fUTZ+b1k)%NF3XHgc2LGE*UkZ>m=Nt1~m#Ea|gb)R$?Z_xRDw_lY8NcFrt# z&=%VvV5HD&w7HQ-pwU=^YuCXAZZB5-;*ql2<7`V?s)py}KPN7>uoX+3YmK$?=tS9&yIqF_IG2>d5tN4n*gNGX<|48UaFg*}yvbgBl?R3i4)6s&7uYC1l z>qRqP@El5Aa!yv{wEZEzITvl-E9$xJJf>s4Qe{feoyV-{i97Wc=r7qY!R5wm`xy)D zSG2eV1pj?vxhrv+Rh((lVGZhN36f~# zm>Y0<&zbGsY`rmUeKBo(tQWI5E>19E7l>smUT|eni0{=DUuK_eEdS>0mYVDv;<8q5 z_u2igXI$-I6mJNXeBm2<$1qBTNs&Qvf}-4_Ei0@qPFDA}XzgKhUdWc+aO(79$4@@T zQe%!iymaujgXa1-%4xNHSzC|gi}*i(!PY3v8he55;%Bz}8?8*5TmrwgxE(mexLbfT zGkorC0bZpR!xv1l3z}>+TJ|k$6H@djO9;qZz>>azB~#;yV{eP|(iV>n!;YR4UNcU# z<=!}*!j_uA7O*sA@)sZPy&--_BY0Uug;UR754q?%!_c*0hS=1z7d&oW*x{?teKq3E z$%EZ1WR8aN>li&+BCK{;*E3_DD33@|?!jB#8?>|e($BihTWG$zWXh7Ym+T%X|4Vy% zDofRAppXQ zNlVOx3(jk!TjmCMtzhYya)PTUB-Nv>%hcERZOEb6wsU_@^s$CcFFCnm^6s6Z$|ZoTHspljU7E#cWwo94z@H#_RqLHm5?zYuxN5J2sUl zw3lRD%AOLo!dCB&tX^7alx)hBO(E>14D3@LaL(Lte8GbWb21jJTRJUc;e-+vXBUIk z%onW70+!veH#o1}-s5WKHQ`2QZ%Dw~ zh+>a_zK2)?C;Obe661R~_D-5dpy*sfsne6}970e16*3MKHf#!wINB&NBUI>wL*D5F z(SNs1*r@U7Ad{tmob?8lC3hmHbs8m`@}(TLDT#1O|9Z~$mY39@Q!fgl@}@jY?FpN7 znKj13l4WP7Y~6y{KN9CI;M}!w8t22E?41*;_ULbUbkFW>aB;-;mNqdZ$lqj5x|Dslaq@ zZ9>Pdm?)muhm1{@Gc@A1uCTjsGof|fRn;laYR;vk9y^nrSf#D^Mu2_M+nJvGV(<4} zljwV>xF;(1f^9(JJI;9>oIz?&6~a1M6{qd2RG7Eo(3)K)TYWFoZ)mSu+LArtcBAh7 z6Cqc2Esg%V)MqbmT1xf}&lxw`Qrl7@Vhr|0_=TQ5q8k{P>zh`)l*Oj3+rh(}F=Q%O`*SxQt{Oj=o5T1!So zOH#~4O4>kzOGQyuTUl01MchCpr+S1g-O4ravPEp*`Owz|t)J;}3Ojfl(R?XEy+sV|} z#n#lz*3`wu+RN73#mp+$+BV$E+)rOKPfssOQ##a5u)s$m!cEi7!PLvi%+=Ax$JyG? z(>lo2Hpt5^+}%9L%P7M`Bi+-yEW*4nNUtizxHN~`+|pgo%u7kp*W1R~-P$d{#;w@a z$=W&8%F)NzBh}l*$;;g>#K$Sj(<3_ACnDIx!y_!zDr93+|wLB}Wx;P^vtGv3bv^cLcE+O3B z!KzooaE+FEpR8tEzF23Jet)jfvI?WFM9F2AE|<(b`$7U+vZ5yx`Y$Q*ZOKV*EzfAF zDrv7QpHfpktvGXCQQ?ZPnDhCiuX8G{1tuKVwO-$Cwq&;E(WRD$=a~NdE%xJw-qkDG z8~Xjan=7ZbS5E0HT0N;~^|aU{v%F8O3^=>K;M#(Mw+|z;N;=CbI?8G%XB4!@WHooy zHLt5{`c%<&vAVgsd1_U2Z(Z+#?$(y+oo%zHbhLF%pE+&DqzTiO&gfh|YwG5?(^mI( ztX|Z!X36x;i>Gc`J!Q+1SzFi6-nC}tu1#~cZ=AVpMfao`OI9vjx^?xEotu_z-@0SX zx-D}Sues3E`=Ni*sZG7dcC7lfcHNgnv(6pe{QBaeO}mfHU3X~P(aU?!J-T-7*2z=n zF5h~7>&b`b&t5%y^X>JQKVN_U`}^aQtKV$Z;e%UEyGJ zsgRJ*i46-6TO}xY&GFdyNL)Wbuj=BCi{0-1v37MlH-ndXVl=BzyGd49fKbC#r4h(_QdyZ*d~MqfPr)dLpWwOWO~x;W2&j>E}aU(>=? z#4Oy<=rm31lCIy}teZ|H+*9=O=g#F?wqfE(r+MEl1&Znoz*A+?#CLKL5yu0DxjX<9h-77-Ner}qTa!PO6(Jd)2H*GsP zZCiZV#Mj$YuiVN=x2v?Aqboiqs_lNvyqM?b)%>@s=vGbj7GCDR)q!7Y%kK-S;fw#- zDO&$IS+?fM?q9ZA-zy4r_0pYY8vlRmxhLkug`2bMy=C-rS=1yF?L;IF^K}_so55Vs za4|ui&#j|fmd`HbVcP7*hmskKOC=_{Cv+UV=)R_t@lv+N#3Q#nRcCHv_10NiAzh&Q zaa(lmQ_pbeeDkB~ccT0bhRFEYIewQZ@wF(w8s###J*6mMndnphH>MM>PW79yJ*Di7 z=26#aR_|Em#}rOAaThOrXzE_)e`X?^c%A2^@Pt-LF=jr)6Cav5v}}$k$O)L;SfC&n zl%XLp!*vUc8Hi)hGUz$o)L; zyHoB6|2uxLB_t@`GBsu5EnCaG>tZ|>wW>Huu9S)1;&uH>;Db{4slsk~mQ&U{U!3wv zI5xmrCFaJh_Ghti7YeV(6>mAoD(=L>aU-Gi;Q?lTqc;bdn)*baELb4aB$(myMQ_T4 zwm(L;-K3- zrEDkP(rjTr6>r(Hvudfb!Phbwi??W)M=l9e^WVO7s{a;7?Fl#f%ColJxR-ui=J&;m z%LIx!1dj_(`cjj(%UWi~430iafkQ1!H*cG^h+%fBpO4J3Wg{?1cEmRE)UmoQBHaaEowb)Fxl{4iVe(&W}ztSk!(ac||x6CP7 z(V*k~ho5dQ5+pJfuPtkzP^Gfrp!U>~4_*5E*d}=u>^9lBTy2%g^`0WjLYabt7q28g z%y5)kySiWhX65OfPE+>7t19C*mq9u`D7(6LvLHlbCW%` z*=v2wum6>O=l_Aj#v5Mnq{dw2iU?3!a`(#de1|Jyb!pSv@(*e?GbstY7y8V^S8;){ zp6ek;d%BZ|A4^bQ)Vz*4&nGq2W+n3e4)IJ`a>9RS26z7HhxfCVEPlju-2Lf6u7DPm z{RxKNCEg2_rmxsMH&*wkjaB8Oe_I&lEnc(9<9oMiNz_fRZ5$`{zh;YVoVSU^->cmx zrL)_wt8H0sZMS{iuDkL7j9RR2dit01+-v@@^|-LpiT=#b0iTt=wu&#);OUxjHrqi| z$Yauz2__38?glZfDtq;!ZI9)hhm(SJW-$nOp7~dFP3VjH+*yqm=X*-Y23DQ)3X2o8 zSu6A6|D=`!Yec@-I8-pbjFB`}&tcI!sItg!U&^w@PoMkb%82jW!Qg&S&S9!`9TDM|`Pf$l6}wl|&}OWdX?@2&7tsPGf7x!J3W-fd+kpKds% zt@zqgxZ%M0@0=#n?tPRA*s)=D&7n^kHx5c}o|UR1t1(5R;M64NUK72DTt^|_2`mCp zk{4AP69wvCc#1D6P<|@#*{9+rfBiJE1^X{(#&$SPX!^i$DAUi_$zU#ruQaO%e`Q9( zY^5bRkCe}O9%`Fopg3P3^Rl1Gr;MmiJbnH~D=nsM^k03h>z``ryPdbbg`DTUkl6KX zP4m^?o~~LC3|GHsaIkuGs%XCVjMa{ltwUvAFtArNt>ZnD#Swi+iD$p2cSid%H+CR+1%n#-qfXBeOH*)w2)f&e8~4vQvX6oXPy|kL^kCS~|P6 z%r;&s(%I4$ocpUTNoW13oHa{CRyg!p?^>BA=qY^HN%q}=ht@Y3?cxfU3Olt~bU7M# z*h?4|7EfT7`J=#Y)v>YoqX@J3g$KvX7dZBmCa~zHByu{KHSyVfVB{}oV0P6wl&p85 z(J5rHRH6WL$h}QsMuk^>9^B%;^x#M;Rrzo)Ab4WFm&oTPfPP5Ys0=eh91tWI6; zpR+ocIEgHla8>?wq4hPx!lQN#dVCoVn#8s&U|wqWbz;9xv(lC*R?j>K))d=D&Kw37 z1p_CsV-r}7dlaMwOikfQq_qT`L=HUB# zH>ONm(WJU!!J7x0d@2?dUtrmMOoGE&Ew6FjYG!$x>&Hx9II!pKX>|i#anDhy{O^?m zGhd7XlVAn|i_`^%Df|piHPS85r1KTC>ohR3XdGa%`Oug!{~^zejHBvNf?vGn9OOCt zt$|%DfstG1HvhSA%@WfTWHk!jxNT&9Gi!oooPPanzmgOlmL_(m>Y1@ELXGct-Mp*M zTgk{jwf48~``>~drJG&nv#S66#q;69DVD;2lc!E-x-9ccHELSO)(x$~=L;l)W7y5z zwjQ$$U^%63c~VfK;aRL7!*urr93~r_bq@*lT=*KcSkLYM8O-C z^>dyIH2h@BIoH6ZAAu0KgbXD&y{4A~P6tdFim$+3oBo@O`gZOm0zzoK0bq*r`NT+@ujo}rAB$);k7yUAAfcVcGJn-r~D**}HXD=kcq3aG#MNXT-c zzYC*o#?*#?-;-mszcJe@BpbSA@!eorU6yj;I0N$r2IU!@Gc{Uf?qCqwz#zSXf$Kx# zq8FVTmosR5Z^d0F&pk zI=%E99tTGGf|j-h&S?#784a9n%Q%oFJ&+PI=w7# zS$d9Rs8vJ2dC@HMXAOE67!4IFrIr`UC=^?|Gt7!$m?h3&-qAU$y;DW9+FFBAqjS=- zjz(#Yrg=3g0;Bf>_W1>DZs9e45p4U~SknqPk58}J@_=38 z0i&b=W6}ifs|g(0A6s(@oWB;TRV_@NbipKV6KjTqd+rWB!wLO54ooS>g8D*jz6Lp| zKaosRE-z5$Omdj{O`R=8fbHP1Z0`cr3GAI-%S&V?E$onjFTN}iK6*sE14JExzRG=0b9EC@j{NSBb}3tIJ>65=#rM0tYkUq(915C0}K)$*v~XfxtTEK zwgK-p3*M#dd}lteOE}DXd@_RHU>WZR2Gi#ZC*2sOC$P3HnD@Gp`_2R29S$5bBkL0l z=1*V1={tjai$RXG0^9Zny#FtF{`-86qSV{l#3fQ98YU*$U0`mR*{|=w zl$<^_cC))s5PPD7V!x)~#7OqI1MF-&lf4CU(i+&(FL15%;$G)f;=#_uT2N`|KFj)7 zk)=oVos+ZGPIMWX7jHP;y`j9*zG}_-@10hbOFXT*Od}_2TP`;J*}Yr5nyG-%$ARnO z%%!&rc&}*A)wke1W5BMrYYzVg2EC{>9*1QQC2ja8FbHp8@sjAB-@yC!0N=I??B-io z5*^q)GJ0oB;M`fjCeOeW*}#2IfopGyR?0%w{KC|THlJEemcZh+#>BYhP6@tQGhSbx zp%M}-_9}0J=cUEwQ5-(&)AxaA%jn&CRy6w*>B-bTw=8soUKiySf(td%@r;HTmGmCDs*P`z+?2EMwmK zq3L?T(!0N!^jYLRJrzp4~JtP^2isw-fts$$z&u)fzpq%KhR zyE5CZZ=72iI5QGBcQ&wQcX3~A;NGhu8{M#Cb+S;5sblRlmc~#~kw|4K|{+nxu_3rf_HZgP89DBw%JA8}g z^4angdmOo|`7SW?%{lu`S{on6w|Oaw_mhv^dsR&MUK54d*S;o4=edE4(3ZVr{q2|2+wi)~hIiQwE4 zF>{^Kb%qmf7W=+FdV_niS^A_KvyU#5s7~I)V9UAJ`sAbt%e`zDiew$Q^eVR9`_=SN zpy-wY?>#Brn+G@yFH}n~>_7HO{YLjPeFny;&K-YluX@qIm$E^1NphGx7gx`l6Ke}t z*rgBJ3W>&S5WU)@+hL_w{6PMp(Z+>3r;0)jbvMq4I^4g;Ci3WJ{S~M6CoBCEtWai4 z-Y{?L@6&51a2om6aQxY1^JY_e0?SdhJ$o`5=ZP2ZJHGhjH;Fruj5-sT^%{!ZPc8P3 zIotDe^ZBeLD=dy)d(-(z`k0s1v3(j{F}+ z`!_J3x>hazL2r>XlYRgr%Lc}cs}FR)=G~bP#$_7VP&@C%1kQ^U-0lUelXJL(p1GED zvI?HmU>`CdOnmD<$OcBbv6sM&1oXq0Hy3MdOt@9}tqh0{B?t&b<1oqku*Q|DIvGcyVKx1Lu6z^`ylB+^1)-=TNW(eQo!Qguwm*S={Uxl z;orrCRrF%S*l$m>ZZVwEKCxfrLG!y)3JbI}KDe5FO}bRNLDE=9Fy#Wf{qao(ySXjq zaPMs3elMwXux*Fg+TB|iY7TlAAC$hrQotxPfw|-Y%N9=t#s(> zJb~H$_GYi$TV|WD(Q}_<)VbHTbMN8INmqEf^duQ2Ij8J3pUo1$Ty}utG(*wT+-uhq zn#CtDS}H76*v|2M;c5Q|4pV{sS_O=92`pz{^e}mIhX*`hnGxWiz~&&t;JfJ?vk z@SX{;Uo>#H38c>dx^Tt>?p3_4=?*y$e->R(-y;*i$XLLrcYw9mfXn{F5}pH0We!|t z64+%En9n9$YdyedGi&y3$?DT9XYZX^6#i!O{2!YIChXOc-0HrnQg#A!K@+p6!hNml z)iMWIn-B0@G~m^f;9DBNE^>guRDt94f~5}~c+VW*@kwCrSKz($f%)maB8{$=84EZ! zRB^jqNalF@(Ub8;cmeB_x__KKbu8N(Uhcob)|6428zl63W9c8mpjp{^I}bl>?L4$d zTY$Ck&YN#D5}8+tG(FOq_ef}A>$Y{fcTafzwt;WY2YyaXX*DIb@B^ESy}8#lWCuhZ z|NNF&;sTRd1EXjHbNKP_7Vmb{vFjj-5F#zu#{flICFvL zYy$gv@v3bLmR=6{#huSCpTK_p1CNga!)-}MVF$LeD>&C4I5AI?&Bf6wY`T$iB1`%K z?zI8)=O?hb2Ry#2z?N$ychmHuzW;)g|NQkHrAIg_WZG}gGc9;E>#t+lu^9`r%ofgi zv_$G(;k8|X8M}8L;C?&d-@O7h_RinaBiSt8Y+75u`ZIIOv0V0?2h8OT>=GYXE@bjD z1Zp!ga!I@~neFzt| zq6HT^850jM$;Y(l6l`!<&N7!zM&W{I!m+t&mzG3!f9nvOt7xbcrf@)kxsBg4$KZfN zGw;fR%X`Y^s;!+OVt!@G&4mXRG@f8eJ~wA$i)Q$0?W8R_7bksJ5z+A3^Wf^Gu5fLi z8Ah^}N~@&w^ke63Xk5hFHCa7+p2N(?Dd&XsVxFD(CwOV*Y2iyw^Ef({p12Fh+f{sM zV0_HlEug5-@*$F)f5zRt<=+{T)p;ZgN;__I#{Kc0`}A~o_sin8uywKPQ}92 zShlx9aI>rTcf&0eZ?jxq^K>w=O6gS8KNi%qUMhF@N03&GMVjojxE0I`qpTh*;CyDZ zU_rCgj18wcb!RP5Y*%}>;V_?+#0mulHmeN_`5fF5RVKJGoLtbTC%4j-oiSrY@|4tP zGcuQR@B}zC&N0Yonv%0=X4>(bH51cB9CmG3*vg}2VWg{K`Jzc=ebtw&<8rxzDgv1v z3mX`i#3~dTIgjjM5%$%&`E;7$x)aC{>GpjABFTYX47?R%E*wA@I;_zRZhWbAQ{nXLmg5P`xne zQj=J?{hr)Xp7!HuZxY%>co!ut@MfH5@R-NfWMQhf&!!WHIQCq~S>&%)@b4JQ+}H~m zIX%1%0<6=s4*mGdss2kuq-zED+c(CFX0LmU-RDa%aVWNZU~IphQjusTSa3VUJiy*H z&wN|IcBo}wr0rqRS8;`|8!UEPF1C*Ozh$*`T=_#+@hAhqZ5&K&D-JYqFHm5W30je` zXkx~;pMr}dVtyQ&mb2obBJ-4)2bGznf=@CAs(zqxzh{yoQDrOSg$?vnRaP zZE3Qc#d&;Iz=s8F9u6swJek;43>28yCC*$EF!Iy<(E)Y1O#ca?Hf6aEgF}s^;}K3xYNWHgcIfo$U8jjVJGQ$ZbClt(~>) zt#>UCua3Le753y%-tHCL>$ZGg)#Tu7dp2{SVv8svlgMf&i3BI5C5i%SE{Cyk3fmQaaj#|}L@vX(E-LbT!@ zlXm!prnYrESf!3I%+sw=voVQJBPV{ ziNaBhEX8B`B?<9|HZ$@*yE--1z{x!^vTaY$@eb}E4I&O)M|pD=%+Hyyh>1xcBVvzI zuVab=gVjS7ALmPrdMj2}b#8H!Ee|<9>FL- zp;;|vW%uuijOoEMngll$v;;kB5}$G4km{*|1Drk$0^3&{oA=|(mdOlUGiE3*%dk-H zberqeEX{IApkTwlX4w-AY&rpsxo$hmMC6w99ou;LPS55W`E{RLD=e5xFHDvTzjBsm z$FFnZ%Qm#?l(6&6zrd;=w@}XTK{3P8BnIw94qdJ`iv)!)9F&(U<6Yd)Tx#3pCd)I) zI_ORUThEGQ<5vdFPEY)#5?35>&Mj<~H*hYk)##YI&XJ4f<>^lAO|E=fM1tp@N%1y` zIHJfr?ZgAwOYT9Z*j|4N*8e(5$E(Yxn^j=K6;_Qi>Obc+XmGFK^qDb9tj&N~@>oLK zjH^20GZj)Lk1;fDnkDL=D9|r_t4h)_^O)gLleBAA$x`d@nsOu@ZsfN+!F21&F-C!c z1E!sqH6Ijgo~<8t)qJl(`#*_h$9X$9oa5nt&gJvvEqCmLh2onwuo#};-?Hz5YwFeq zjY?09nf}NeQm#vA4m3z$vQ%h2Vvs4)?6;U-BE!je(?K5fl7!yN3deZWHXO|~ab^~I z!L)ZqV6!xXzErG$1KT@;rCc!$Ox_wt1DqWh7+DnDEFY$XWGHxAed3td;iK~P-0wKe ziR;~8%!spEVaUOyA$pX@VI%XL>mmFpj`I?9d2~bWc5VL>esyYZcayyVBUeNNQ|peS zx(6nRvwb@1AAERLhMRKdhMTlmd9zGl>#Ge0Y=>p`ZO+ZD zZseQTdezcC(9g){;c*}BBC!(*9sd80tQ4Ipn`Ae27eil60^_Bu25ydvRt_&7@;7jN zF=d&^sP^QXnBM^gg;%a!oG)fdJsHksaU{{!6!1`n}gS>$A z+_W~f$Rww$tBzgcG* zoO5=bZ)ah$w5a&CT)nW+p75uvlCNcXYF9|ENw~;0Ywo4|_iuF8+g0qE*op0zfLm+81MX{++Ss%nu%u-; zOZhBl=v#8Jou^}6smX!{UWULua-PfrA{?APybPBoGBk;tx9IwMEP8K;qhZozgBMFK zYKO>mCb0$H6k>_^z+$an$|Uf>lSS=Aqt1+lJZT-PbXMBcd!Ok)kZ90!E&DG67mw2Q zxEz}2JRQkRR1{!Do3~F zfd=&n#?mwPrX|RVCp1Vu=*pbY%yOdP^5izR2$sMPT^T9NZH z*Ak|Nwb~NRZYBFyt+rZi+!U$ZFfF0MXG??q2~!&Z=Flps0F?uJ4NQ6kJr^Z-F9jS( z@pj$D%p~|(VSiGixPZ=`hK8*Ny&grZdHTR%T}ofx@223FeFiW1LYrBWOx$8K_E=ah zonhJR&#~uVG3$xX4NNzbbG~%`N@_?_WZ?VI^eC~7YsPk~1#AT$+@u*KGJiC4{a{?2 z(5APb!IgQ^>H`gK8|2J6JZ&txHk^=ENNnPcXt9!NzI|Yjy)g4z z^~vcX&6#r;Rvb9Ve&8I}f##?STbDO7`R6E{)(|=C;BbD2RLBzp_m{K6a=7F=)HZ7x zvU4=D@7U(oDE;KZsl&gTEN`6Ge{o9k^LhOnOa~{<7cw~(-oU!`OY7$i3lA^s`@BH$ z$WI3T73(i=atl1osje-Mwt_8f0wc!+4t<3csVh2cSIl@EZIYSMW~tC{if6HcLIYcc zhSi1!mK6<74R(e>7u7c~EUMsO6ljo%WV4E3;FRd(Uco4r!FXJdv$Nr>SS^#ohJUST z7E8`v**9_Gmb4wrY#SO_H!$Qo`skf%EBV27{|Li1ABHol8<UGkbh776<%*EW#Tuy(5P+E9KM1t;>5=A$bjFKQ(rAu7O3ZIN26726D8gMsVvHbXiAmPl#lb&fuI<*(N2?z}C_Dvx8}=5`*{w zX7QD+P7*ElFB(`^Fxt*&o_tAN;QGkOu=t1cOjTgA~KeubWKL4xFis?w;tO_4l$>=hI7E1~;B; zVc^hcurp|h=3sW1$i#Mu=@f(7rpG;>Vni-`xbk*Qkjv<#xpVwcJz>8O& zJ=neBOJu<(|J-PGy$ehN42(Vj-odT`uN$VyREK)bxo~~Wo_RN>@T^_G?~V<>s40g+ zgH=SEn}ZAAkLIWq%xnw|G6^lN2O1a!7}|F+b6gN$t5D-`Xt+G7fhmJgd;ue$21Ao1 zXDt^4;|vBSgDsb(IK>VyUyl@b|JAtnYUlR`{_mL#1#g(#3g#|+v~S+7whV@rwk-_o z6%0Zhj1C&-{=J+iVf41k?SfV3i%x-r2Bs70(;i;h!m!o(j(0#Z1J{A(7-r`8b9mUe zLYU9Y+7ac&8?}2+Q{?w2jQgt?g)C;@Ug0|Ds@K|_zV$hMArH@6yQj)Yjic4O<;_)t)r7Rme?jSavySj--Qg(@YM=j;;wF zTECQH=G<7!EY~nQr+=md!_0^|wpvQ39yh2bwEVrnuduS!bw#uMk83BMPP%qv!om%6 zCVUjs*wp5hz{%#oaJ{urD@j)VrIE(LNji!QKXc?ydCw?6cz0reRwu(&?imaQD_CO` znBUjTirvH$U~}r?jV2R^e_qpK53KvhxV`%5Ox`uC_C)$8uF1C6`D-&(p8LXv93F#b zP5FyR9Xn6d^mifb12L1&NSuVagGP`7A4!>)9pqtp>yD~t8 zIfZk9jYHNE-F;T{6pXr%{KWq(uG-x{9 zTlZZuC2;rYcU}@LjR{M=8cgPzfA7^~JD2fS*GihP!TK$O!3L$=13PmL$jA$}zuDL< zb$~;p+UQxZmgI%-Oph)Fk8>ufjFks1?@nPW`Ov1G)uyeXJx!#ci|_6)t)|_wuaqY2 zSh1ktwm2%6@! zL^QX>oY;Eo+8rTxzZoof{2grxyO-Q*eY5WTwHN=G!x}CJHO!M}VAAx+eX(-#w0U2v z*vjWJ%lznY?uqze5Y3j*l(dTRbnvQ@BeI1rCaya$N8OPU3dXe*( z3Rs`~YyYM(ZEHlZK*r<939WCZwMG`SCY)s2n%u1ZA#2H&v}`#ux0ePfuN$v*FlHp( zySd=#ZH?!}%d$)2ME{pOe|uJEK`sx+>210mkJBDCHhgN}JdnOP)nM^d8;%Wvtuv#V z|1OD|`l03CV`e=Dmv2wMMPziYz3%7rLZJQ$qeO0JpP;noS%wXbAG9PrFK^29X!qt) zF8j%>QNYO38L_qE)h)3!t{n{;5jVWFw3?aPjT`bJJ6IzN{_vlh9{}PuziXbk$Fzn^(^j^~<+5FSArQ~NE6C)g{V>-+qjM&k_anQH zU2Q=LE&3hdYz}_+jtJ9Mf&qwwL3mBZ)kAI>DQdj>XX6G z=g}Z^BK_^RPjB7w-rai_$uVtm0<+nHG^eW@lB%anuRS%R)nR1~qu@UQ#UeI7>-06R zo}S`+@}7Zr*%_{7A(5iGE1r9OY?RGl6nnrRI>APxy=YPO7mxZ(=V>|59bbHV+qtN@ zq-CS``>2M#>ysv~$oTI1#HW3wJ=2qm&(^i1J(;#SVUpR)y1P#%{uK1MosuTN>d??6 zHX(d!4MTHlgUMOOOM7+aI>)Z}c$)RSNWZ+5b3#K`8`ttG*7ymnTJsyG&CH6AsCw_l z9A3ccz2Tcm{WtB5CXE@*W)k`FUN@KMY?H4LF>hD1C^x&MVJN1=Ja=x~U$zT-pZ(VR z(U-vW6%e`f~JaD<+Rg=N@h0TiRzNfVp6QH_Dq(Nm(y5+4~k6hs^xm~C1Y zG%#};95~=2JSd>UeQErq$y^kPqv zJD>TM1a{+p4^y14n)v%pK4)^qLh%3#j|fAP_QoSk$6`0_2s*$b)bc?f)z4{VM8f)p z1vA*KOfGTAC3z$r5#9Xc=_%1D2gakqGjcCVisyO=v1U4Pd^{{r!E~{i-)0ZnY#xIX z4o(~@D;OL(DjF^x&)dbQE+cWe(NU&oidFlZ;+U9UX+@=19y0S;9Vxg_e!ujZF2zu~EnOv$BsbF9db?Yf)Q(KVmiBm1Tr|`%$0bPYgW|bDz z+<^0X7lK>m%AOo(Q3@@Dd{ zRVr8=(|KCy$wUQKwSpF%ghkD354hYlnAqvl5zsRI#38mvhZxjUn7HaaTKF53rp)`n z;(UFV*mjK)`F@@T29|^atVJ8x-3*ka^lrsGpHQJB=j$x}AR*Rln<8(C%~Q?u4Rcxj z3YgeT9(e4l?wGmei-5{gR$&E>dD?1b-AoH!F-bEpD&`q-sk=1t253yyscCG}_F&}l zyU--I<3o#@jUw-&#+d?57xpS@{JWI4uF{b^BUoBYpv%0kC`5ME=Q)cNc{5{wFkM~} zY#UIO$nAT;v5f5pqj-aZBFB?i)n%y@l}tV~3QSnQ@;q7Sx5oko-Wdnj*A%er-pM39 z?L`8+yTu{CG)5Mt69;6bDRL;?TFAsH;lQlYu!GOFqs?Sx2lIlZXD10HFxty7Fl$|4 zak#=%d{sj>%rB%@bYZHy*UO3h>xB3+8I;O4RXZ@rIV`;`x`3l?=iElF2`WbBAW!;^BdT5=PEiR-JYA7Y#te;%>+}>D>`zV&PzsvF zZF(ZgQnRzBibI&|!UDd98Ew|K=2qobKWnfp0C2JZma4rdD zmRS-Wvq_6d^v40gQxmqQPWm7`{e!dEri3toK2hO+)jt^wnJ%&gsVJ3|g~r5feJadc z^I2W8=t8r^181g+1kMHlSHYjAy!ku&O1w87GrhxfbF+uCfa(HXtVP7~&f@6}$jG9^bUMe+89mOG!y zIQ%6JeUGa$zWL$}s}jfi^EOX4=d1PB*&mhHbmo+u7}UO1?Q)a2S}QaEkp&7?0URIp zFiZ0+nBE=p#rV3P>S~U~pW6kP7O_7m+a7kINo+93Pb~A$XgSwsBf+$C*-1UsBJZfehETl^pLvWPusBUwRI2F59kyiS z`DHCTN-tL&;T3Xd+_0mH$xyQ2>2p#i@0G5jn=afH-)_>_$?0Y&wc(!w8&6<&bb}*z z1mDe$AsK#y>pwR$Mz(AbjThS~UhHzD-v} zcRpaXzV)!zNGFj~Ex$>4hJZrU9bVx&i$fwa5*qDU7O_S@H)ie%Xyk}l5Vq23YTcr) zMzJjnicj+oUgZg5mS4igVYh-YZ*hT))eT0j_zlepE{?sKXFh9rZtRMRIchfNoXG?x zVU7c=Ev!{G9?T0^x@FI>1Tb(a+>Vvu=~*euVR7cBuLDiAFx2mhuw}&L#{jd*YZ4Sh(kOxjpf6sc2!5Jjp(T@$+3@ zy)DerpPKkm7`M453L6|0l3_f)^-!c_qu`ALydI1S1_yZ!9Lt_K@IE=f|Kh+&14nUz zNiXiki{D}V``1zS2BX5Ag9;2x3I3BqD%H)CtTjnuQW_7|U38@T4{vap|USWtSk zw&Mtc$N}~_j(STJ^*1D1|6%4}iTr4LP{^QBqM^aH?tT>~GYpd!tFqi&?Sd zkWotOE`fx817~p#2Id3?W)+9kHVzy%4L3>zB^?!5=QvdU?K-H@IXR~7W9oy%9f#On z%yX^#*OU^{IXQ;WA?sk;7DuzD=(#450w#_EPdf9zcFw=?fcp+xo`Zvx+r5I2u1gNn zOU^2n1i0J&6D;*OF067~OeQ8}&O!SNO^QW8oSLfe>4jmKK-ro2gQl zob)#+TJ;>{sCpvA;3%9BE%xM)_7Tu4vLINmpcX^8R*E{)VGGN0a|q#*L5vY$}b3`h_1 zh&8^#$SUB#q{mRr65`v^z#MZ%e^Fyn%7HgJ4vaSa(TfhU=`afYP?pm?sk4JgYC)rR zNk-g*quc&@Iw_>K-Jd9uclG4lJDv|OCh#0^=ze#`vPg1~14Dtr@q#_dMGZk}w(TVi zLHlieg{%(Vo7^<1YK_JT4>ko` zMv)wT>5eu{&qMz*Zk*&lu)uuNF}^DY_zcvzuT1GFnpyN?K@o>&zf*f^+oTEaV#QS! zig!$zP{b%((3lbR#8v80oQ50slzHDA9atwEU@hQf-_XFF5b8Fgw>G53^+uZZ0VlN= zPO~pDaePUem38n%W~1bjLwY-!)IRvK@%XVPG;k*zIC?L8ZO8$poVg4>N1A&Typtwr}>2vs|o|-4F}BzXYrT=tO*XQ7R)esC&`qj z7&(c-L05rwjl1LCCjB2x+ANPbJ6c%}G$cKks6Qi+Gl8L=!@+3RA)OyisVOhybQ^_r z9JgN;oL$m(YKcgui{s?VWS+YIq)!dJdU5AhI9Ms2njv{yM&!ecQ1`oP8)o%tU&)#0 zVcU38;lA(%M%M7eLu`3^a}Ih4abEu?a%bu;2N9kLWd#TMO&EC#7J^BRsGS@q-Ejrbe})|3XeHSBkC4@k#4 zHX1bY-f*b7=iq%MqH&JnQqML)mYet1bt?3;r6nB8*uk^;Si(G?n@P76xDPZetT?+> z&YjU)PD9loO=+QBU$(fz2YJQ#9c(i49!*XS2ka9zC`vF+DPzBG;=peJx@=p4;km$F zZ`nTwV-B+SEZ`H7e{uAKUduYQ1p$_tGq_)~ ztXmN!d_SPsprcKpP{@7bO3MRB8w)unFj@R*JZf`LJ}-7)mFSae+YTo@H}4jWx*l5OEIE^w7ih%m7^bS+lKOqGG7VS;vq)4^-L z+IM0_!X8WOIZ9T@=tLZnlweFhd*r<9iCE1WQTzXWJ?iViWYM8#=h3XQKyTXtWsk#h z3nUbBT$$>ghU_+Ews`+$ii1K=lBU4nz((PC1O)E!U+J;vGf@>h)l)Q+IiP5XK*42? z>4Ap6LQA<%C5h zSD2J*8WnV$)!YvAZ17_E5nzyXK%!u)+MQnQk~@-j=88!;*7@Et7N0D+pj7{)slfiy z_7edsPYSg7d^X`f5ToCqcW$die=(*;wLHRB5T#`43lzmZWU z$<1TywF&<&p6+N<r>0`O zaD}6aOB1(BsOYzYx*p|<4;XnJ4ouzTz;~mu^1g$xz@)R+oHTfxH19OZ+As@y+|ZRh z!8u2aBXg_T1z!I6C6agUN+dWg`p6V5TkPb?5qth?OucGz{VvOyI*dImj4{FdYqy_# z$P60;NU3Y!?=n2 zkYkPtG{hu zd-A%TcD`wLj&YIln6*l-RLT z!=y=ei^D#>maIo7l%5g4Y7^Cn*2XUrJq=_QLo+R0OOl=tSU}s zS9U3+H2X6#vCVlE;?AgCu&t==RcW20hzp}=M77WyC&Q9Mnm*2kGn@_A{M*|V+^itr zq~H^*aHgT_cVlXHgI7kw1mk9f1;6hFmF(IaHs^-e+zD(va?5^b%sY3bdg}30%u7m2 zHf&K_A9}b&R-wgNhYp^zjH@e~dHHDf1wHrO zIWIpw(U`t8dO^;^O{|ZF)%~|5TxD`OEuU52iiZWQ@HW7A8IMQS<3DP0f0GN_1`5dbMqb_G2Tp;Srn$4W zWCj=nBpf)<#L5-OV9JxyE@>RpFvGVy?c5#d;Aa7>0oGJ4_W4ggG&Hta@G@w+No92?9=ja)E8&=Q{sqJ3lfODSE|vJ!xA2*SBZttD z;H*QD8X{Rf|1L6g{ySmtRLp+b0wylaISGe4`)~a4aFdkgQR>oY;z;UHsA=d_6E{&w zJS-sh;N)`sCZ((uOJ==l>SkW}>QR)q;i^}wRA*%*X?yNC<)|Z7vOtlIKjX!L*0$)5 zgRK$;3l=_=Es|tvuIpM zmI_*SVwni1#3iiv~ zCK6n5P+z22EUGM$_us7AZTIz8>=08~W?|pCgl)cubB9LMuSAX+dM%Bu;t>LkEoJ3J z4;Te|oldP>`Q~m?xB8Svf3);c_IUk}Zp=tpr9Le}iCwqkM$>AwHCtafNbF8{#Kfg{ zU_ld4$hxQfTy_r}&T%Vf%x~58bab5Xgk@)IkMfxXk##=jE*$C6SvTXrb4DqJg&7#>lb2k2mWBGoQf%7W)T`Y)lTDt~e+JU)Eq1Vn}4);OodG z<>A1jl_1Nd(8%Q>;G%G6gI&@DM*g}NF6w@UoFM^^MM5hMOP7@P`0aTp@_XYUkrfIY zjw*@*6DH5uq*2a&+P9HY`r$tgk6jnqUd~XWaA_@$`7pgFLP=S1%SXP37l-A43v}A?BngzRx`HJ_{BK%$cap|9~mrmdBK3 z8m?MFfr9VTO4d%cSjeev8PMFqz@nIw$k!~;s+$wYt$FW|*p37@lNk?JW_UbaerbcN z;-uxwhAoO=EfI(0r5fA$1QeLck`C~0dfcWt){MO*%L=GFOVf`*1nK<*?Y<4XgpVj=a$&%f&2p^G&1}*__Wz7igQXNHMFCH|N1Z zjz8TD%q$ODf?AjamP|OrBca$~DZ(VM@Wo;FUkTh!J`0#wcf4leo6ul3fk`MJf<@Eq zBI^QP2gYL|zRXGs4(@xm)lWOkfm=+GSO3fbsh%0my4xHWR=t|Rl*jRkS7Y^|ml#ZlmeCzH^m52-342Sv84P7q$Q zK+vTAh^xQHg)(`oT?!5!F(Ov$UR@Gx65haFwq>@s__BXSQ}0gwq8SmXDbBR)?wflW z38z0F5_dVc)V__8C((i{WZwb7o*N4V&j_??6eM>0cRUi_{=v;OFR-_yVj+W2M=+z> zghu<8B$4JBj;z}*w21vl6fEyxVpBTP!Fj^due{1ZWY$N`pq3`VGLCa|R2E#fS~%nA z)o;P3?1{Z@28=QdI}RHCTX2JorJczyq0OY~!&%`Ot2SNnOk`sUtqG0ywg`E%Bl6p(pchBNtqoaarcJVWZP9+ zqe-gkporB%39Ex5ehVeeJ^Zxnjf7d^gDDQ2%XCD08t-3ol)Z9L#^$|D&_b>$3SZYO z5Rpn(UZTLYr(mH(SF}-INz5X4zx3}B4d2b4F#0j{`Z4gdH9hG|iW6hGu%d}u@8u-1 zvrjjz<}X!9nD&7C14I6_100(gxIR4R+MvLF>;R9#MxGT9*jNpW=5n4q|~s23Dp-sauC-p7pZlzpz~*@cF`Gj-@%mb1q1nNfZrxD03z8OUVhY%>vwO z6*xCJ*gGknUD6P>=Ym7RlSh6|9Jd%^>K5!cp{iEVpqBH9)z5*&=o4d|K1+jATt~j< zvg`}99%u=@FJC;y_~)CzI4s^FLhPRgW2k6xbvZC8s5d7%*@xXq4Aj2vfhTk(5O6E8)@Tr8cmC3&JAAAm^4bPW<22gcYt#jr}Y8_W-$e}wU6065~Y?Ln7!%~ z)24|`UI&=G9A>jLN^mTaC@J75d}tfQSor3pOiv=`)CA++hZ126jn6ZhSYMXuImi`i z%)PeZOwK`J113dw6|S^k$9+q8PZR#G_UW7vi?XYO^4$r__Fft@nk*XAFHA7L7-Y12 zhQhOX4XOuRty0f%?mMuo=K+^NqIk(d|8gM_heY-B4>;;3ZT_dgQPjZssLT7s1NIIU zHa!Ivy#tzx3pkimS^oTc6{PooT`o~js8QsSx0DNqMO)hqi3Yw!2Ux!~2rl6~`eXU5 zH~z1DLSIQeU|N#*N~(cbDuE%!A%&xvQHw!Oe*v@A0wya3=BSOVMhy&V3z(A>sfyxsBh`7 z4L|RRR4qu~A*W(h|A6yZ1CNWMzkQ>4QKR^UM)8V9v2zPVqZozz7;Lz^1>YqIiZKdE zJQV!r&}8+1DP{qi#R1l+1=IeuD6k##4bTXgGwq<5$3iitMiI9UEndqQxehRkyk_!v za5Tdqt6U{Kwt-5l{ zm>U)(@+@ZDmLRx~N#Mf*PN_t}k_CJ(9N70T@HZUbo|P!DB|+dqxby)>9ydoB4$W0O z8w7VOkeqdpo2grKyTWp*g<@M8WnVquZ}XDk*#5tv^v;osOxJQh1~hz}^3=`O@qwS? zgRX^g>W#5Cuf{$t6}4y-o*)qynJC;>QWAQZtHq%7uj#iEZMTfe3lFezaD^>W5-Qb8 zVUJN*?5ChCZRRugP}~BAq{}6a0bNU1J>|b|;@7>Mlj)+t5@C)x2lzA&aby&!UtB1E zfl=I!u`Wtcd_x1*J_QlJ3kk0t)OTLhu{xl`_HUu&kpzLfgWS^=h|N0aaYBhr&w)WA zfp5_g=9~orXIArnND!1-IQ74xP{sk~Zwtf@Z5LBo$hYR8)T;%OYZ&?0G|I9q6kO9d zVT0nT8;LV2ck&%+l>P8P@DiiQ;e}!^9AsuO@=r2rQNK|y%_yF?P%LYq)HDX3YuZvw z3quT^?$=;QUG0e7H8Na z_8A8%VuP$^CUDM4IJk(h-k?!TXQB1^2Rxe;(s>lcj~(Fn$HMqtB2oLp1O5*U+$9TG ztr9%y!q^KAu$Hv2#T-aHafr3Z;kD3KwwPT4LL0;kX3Fkrl%95zJ0?Nk$U-qGM(J6O z>^l}pDIFC1#x1>Sq2ND8nW&p<_A0DXW;EEMuxe6~z#avm4+=sh3j|*|Xl_5i`hkJH z;Q(vH0)Z?>gQ!Rmmc~N6RHg&VLNY33gs)rem>T_fr|=yG&N~VMR*hS%7YZ3o7Wxud z{4`Xw%2D)MqHNVenW{wLC#Aw26$L&AMVuDKbEZ5>2#(%#w0PU<=%6U2YKGE;1oo`! z=7r%bP7b@?yr{S(Q87z@_o4?+A8bhuUsO@Vz){DrG;kMZmX_3^1OGVEc8TgJrk{DR zY_qxenFk!p8U$x86f9Z5B%zkLU|0BLt5uaLt4=;jUo7Pj639M-^RLhcF_%WKU5x8i zFO)7xVEV8?kg0P5+imV64E%c<*xuaU`N4r*<)OBXgHXu<<|9fvXAUuUDIATgIeM|o zy-ti>PJ;h{1NScl5u1bCRwN2+TenGL0b|w-mVFmp0~c7ynF|UnlsJ?qb!p+r?}DOM z!NuB+#Scq8t}m3ovQT!3!0DeiB}}FZd5DYt*czYMUm6)6y@jJ>537@c5yyoM>}3v# z!4A$ARg24>t6ceVZf5_*gzdW{m7XQ(Ej{?VA~4E|@fi1qN40tfjiSPYO%j)t{uAQf zqAxI`QBa62H@ZQmV{7>0UHkvfj$n;Bz^uTa^?)yB+T~TB`TwmDT;jo_lISk*jM;{P zrDOp^%@OyQJse#OEFp7_o=rM>nrYQprD^96>3IENE=bT>v@1!XjqQknh|5CZyhbUG zg;Cwgb8j!RGI5yw;Ai3StrCA6WiL6({$f1M7-ei4-f8_^V%wZMTVrG&l?sRK7GgUt zd~D7c{!eE%tGYQnSKNMj3HP=gNlEMvWZ0g~VexT@Ett+C=29NObw2u?<1B^U|0h^l zi!D89y>#}2-AASS1?NtBkggsn<{Q;07MCcV!N}d`z|G=#Xxaf*JB3w>Y^x?(u_ZAi z&X79rZ?e?N;suOM4GdZe$s7rs9F8J9iJHv6<*kyBrp`?akxBIVb3;E?$3o$zznt(*u{4LvT@K5zrj4sfv02v^Q;3&Wz*O!65Q<)CKkuslUNv{vw-dB#UJ9cVs$yd;^eS6{=jxWriyJ#mu`CP`0UN2Zu6zVR(lRxvj~)AyHhdIek-ub(2)C`CPWF1uGXuv40WZ*s(8B>X`1NU-haE z$JrhjmmJtQ+u`4HuNd~O2mALwVlGMG^r_*#63>(IfZ6O1^9ujJvmfYeQK)%z|46Fs zG;4=zjE|bxW#!dgTzL5KFfWg+Rm_P4jO@JX3vF1nTzy4Yxx}oBp4>2OW)qQe@-+@! z6Qb_Fe8o)T@>SQZN>_(TM!RNCwJJSUw(2#zNrrvt%AUE0Jy!-T)mq?JIg3ej)s&UN z3q2Z}rk)kn2%NX&O?Fslb#qae7n^M4 z2Ej(fNQDKgM@2rnX&jxQ)SaIWXSsM-Aaf-vGmpN^$|g?z6JHO7mESyiJ;8@3 zY(|0)!^9bh_8BvqlY2^cy6`0#9Xu-Ef2C-yOIlevKRZ^sst~sX;<|({bz{+6~ zvHY&k{HSo*(#RRR?y@I!1jfq0{kpN0ul~wMNBL46MmPD8D~uc>5?dM^IdyIX9Ijhv z;lXPx5isFEqk`fJ$E|9D!Dlq*axih}Eq^e{%g23Xk+8Pa&&T`$im#NF?EkJQK4WRJ zG~#5J?ku6HAt4(JMMd%+?6?|k)pa#lz+n+9Gr#7B)TS7v3rn3Q`+qumeQINDPwKHw zQ#HwCx$>FLGpwFmI3l;ip!o=Ua7|K|W|$1)XTit`id`B$XFhgl#BO=en600=}vO9}5WGr{!+2Fv)sN>+k-fY73uXU-eh@c~jtI2`J)rl`0j>#)DJUlLx z^kv~T#r0A`ZDMT=3lGbtzEEJ3Tla0DtiZxd567li#zAKcZk=jx8XGLIE~ z*IcmX)LK5r>6nP&E9>O_W?4%YhckCyJ$x|pZGcMbrO66^h5dNko+kN`pk&?=11zcDqw^szMWxG=A;HJCihJvfcq6ZHTE|pbTB`y0SfqUo6nb~$1 zn)r7tVD`~wVy`if;S^cUTra^QVP()P71F>U@SuU!;=;cc`vynu+7E}=4>*_`u5h^S2m7k33t#V}`s2aS>jPl8lWEa2_<+8}l(fz5=$Mbqh`P*%!H*97w{JKqq~_2pS;D{^kilXqv9ZPR$|C+#KWB0m7-)C~q?z|yddyEc;J}x&ii4ry z7DLs`Mtcz^e%%H~ZiNF391aHxpU!09wPk4H3`tsFb6mH2TXT z!=A)-bJsY>Zjn}g`=_w)&e|qn%~>w*pJ;Nd2sy|bQ0N?LlV!8Ug^}x&c7({R=QeT& z7^?I=7?1coxTMB$P~RlOtz-$)x1Ic)?~)r2ZxUt}KXAY-|Ayc8e@R=}x@4#E zmKd-`dvwf`yTM^+FYOv6lFT(XLO{!43*StRDW$PnJa?sB4=DTlMCMb}Q7PS9oG~+) z+-^J* zUdEA;6*_xi^QtD{xEOcqTNnFpzFl1QPV1sykYk`(+!qnaw=dk5JBY6Sa+)`A!$pt9 zA4D4!#dU37I4aCty`**RmR(o(t`yktdSh?yncWJT8IxPQLKa?Q;40H_QJnLE-63GB z*u9H~6eJWm93~v%tx9lHekagu8~BXf>cr~?{ux*A+FUiCp~Y0FvY~-Np@A_WfkjR% zw2@I`GON(H25!9%#vB_S%rHu0w2nM*+!^+T zUC~Dj|GDjREuOWuXt9}gj60ikrhM>%m{!Z{OI>dr6mDRUIBM|le0^Ncao)|b6I9t! z+WE2=vK8*h89d}}@bHt*O5<|YImn@K;M`K_Kn_iYCdr-+%`0|3V2#{XU|#d;K)CZ; z2R?@djEoZwToIbkz?IP`Q2(Q0o|M9MuG0>)bt4M7C){O7))u~1d{xcbT3{u&OV;c@ zS#{>UH+?gWQ`oLOW8i$&#Vmc`vdBCGR^vq*Su*DC?)2m7e5TRBGBHDebLZWpJ?vJ2 z$v!hrnOeF}OTWHuq0AJGf5+C^{8gB0UQ)uff9*us{^cLcj?7GOSY>>K`Q|01r3_Ww zoIC<84Ig_F%MJhCk-h(;=S#n(t>Y zv;`P2aUEXADE?}KbZ5<0e+P{#4YL<}i}9pqaeF)y;xg=DtZ!hBEO+4e_DUeTfAcNg z$OX5hnHm^pgfqOb&okArS>k-%E%|Oqgkb}-xdO*uL;rvYj2Dd*TH5nI2J3n_M6GDx z`&h=eYBQgCV1C>Zu2l>8-Wn-wzbw4%D3@3$>vRXp1=IEOA4CTq)HiX8S-xC3H^6{B zIbDm@Bge5tyR>lH7xDJS!iH_Jwp&@`AFJ9ukg#Qlvr-62-&mxtF5qUsmgU5~g@J2J z0cUtXv19>rtpisJ1Is^$&G9=K!`2;?VrFD{<;ZNbAd&Y0Q-E+br$M&W1&%3?jF|-( zwFH>F+rvMnmx?+tKltMNUszMJ&6!uxOTwk|e=)P=5*e<{3K&g4Fq;T8 z{t4u|q`)fr!-YMlf-^w3YdPbRRIcTdtrS*pt#wY_xr1xk^i(HC5w#)_(NO)#&S_Gj zqB2v{f&^H@AEuq!QN_dFV*8;gSF!qnadp?j!stWQa+@Voo=XI2sM#fCIF+_qHt6hH zSpJBEwL*c*#GN&jforZ{+k6M^`30=f0`hqRTrCrr{g@rfT|;~ht22HH+4;mF_%H*v zKsI*+bMFC;gb5mY0*o>T{xRqrU=Tf^{^X-9b0PEEkBpWMvJW3JwoPXAOOW;6z$$yv zOMMHA`3Ke=k60Q%F#0fXuo!Zz4rDf&Fxj+#S??srpO5lo0Sc^>{KXoWBqMY^1vus< z@Xh_exncv?O6R6k2e@{5;17&Q{udmK2T1e|v>hE*?8XX4bdd=Tot)K}&L69-%Nz9&q4FIi;vay2CgSYf!j}T zg|6&gHG%8-0e0Bv&b#vqM);^0&|s+T%A*b90Q}p0amsPC0voF6Ad`lu4InbsvcwBzIi7r=R$KU z2i7$cJ1dVGOC*%8eaI-u&~U!8!@@=OU!7_Fx`ho3Q<&`(7MVR@X;9z@a^1K$0g^P;yXs$0zelG(IdqjL3PK4p*m)drk7A*_s21(K-?Qa!5_Qr&01 zN;`6On)=g%jEPKzk{+AQ3$>?CmzY&u>zN_VWmGwxx$^qLD)&C~121h8JZIzJ$1_tpDHaxaynz^hJ{mC-{{lu$OIM z?-byu7qDF7z}sjrMQ-8pR(B!gMwZ(_TnwTAtiE%msjm?EJhNL}qjGgYlj}l7i-jy= zOIeaSxlJ-wvPUTgeb8UFeL(?Rdj3s!PUUL%Lt@&li~1RjBz`el6fAsrL-omv=^MM4 zU#2kIOz<;l&~f5siCwUI&nrpoD(*IeH4zuMb`-1$GSDn_VBWvbf#Hr9*bW=9=P0lQFmUugU=jJi*!Y2`=Yyc{5trujDN-Fw8W%Vl z19mNG<_eg}Wwww@(}`2eJAZpc)Ak*^{+{Gy;93y+&%knr0*7?!0-4g48EK+shnfQf z(rpWZaxb`ZDX+@@$aH?In0D&s8y0)-|5&wIQSCr_pXCI4A|roGJ6@=_6DwVy}q@2tAi6Wi`@e+*8@xz0qkrE3|u=I z7zBLk71*K~*eV1#D;(IpFB|(FV6?ixQsb1RkfK?BfYsE2z4!uC)dqIGgdMgFEDaaf zOgFGESjJxRfT{Wbhkyfn`2%J<2XcVC=*f3}X_ z1@>A7rZj<>(;C=5baSp-z31nPbMx;Cb0<L7rcFz zH@Sc_c@Jmu8WFF9N5f8XYHZ?6is4NC!{#-CB|m|SwYo?8Y4rZof9_oFF$HV(s83}} z7hKh_B}k8T?`C!uh0Dil|ESBpJuZ89*?O&z9dGt-oaNWQc=AgVX5R&@2RNDb%;KDR zfy;O+UAL^rT6 zUcf9Az;O0qR>EZlo`!ad07gLvhC>G#m?umSGDzf_If3&71Lw_a8_lwL1Q_@?FqF(? zcsqT<+vyAf0xmWMj64bvIk}7+8w5oju$^(>k!@s}JA3)Fu!?s*EE*G7Ci`AU{j0!PCcb(yDSkT+rEFmD#{@n!E6#cI<_J!77%& zkH<4f@9w?*L}T>{qkXQkCz~xep%u%#;|)uN1IGmuwgivM8yN2HQdsQ%fwen<+x{(U znW@007G{=ji(}Gf9Y~zDRFc{5f;V@;bC1RR^08#udG_~*+NKW1_rNZk?E{#*KX5dEU{1A_`L&l}rRK|jvM+z-UgI=)@OJrx9NP!{ z8;S!LyyUxgjfX*3On~)@1CK}pQ}5YxZk2iKJ6JpfxSl-_S{ZO`g+Smm-UVMDy*@U9 zYf8cVDGt$sTRA-)E=-PPT^7LAcp)(OAzS*i<})vj3Gk-JUt~VzeNoG_`dUV>!GkC3 zvo1Z}cl$}l3HQ~LcR4;?H*vE0hIdYvSt>4Ycwewvv_Wmb1kM=^oDmMSkr!AS6S$We za5Z08vu2qux5~fgK9`x6uJkRbV>GilRq5Q7g+m0usbNo zaM+k`Fnh_c?|#7B`!CjBf4GO?!5juA0oEf1>>?K)oOr@e=f`M%falT#28{^}>+13r z?tb`Q=IGo6zWfHR;|9l`ZFtRi_O+KG=W6S(&mG>ZYOr3>@FvgTW(q^fIles_r`eXY zH?N6RUVHY|s^xo5X+Jr+)Sy9RZ|Tn4kL_R> z7MsAatblud0*AXpn_(`Cw*Xsh0N2tB96J_W4b-~#NH?*nDfHU558V7WrW-L^rZJWr zC}wnE;CaBX`aJ`;0ZY#Yj=+Bf@jVZi84Vc9*G}MDz_9lIefF}1eJoc)n(t-DK6tTL zkkf#xe8EfJe++yKH$*ltcrfr@X?VDP-Hm=*At{H~&jKzyo3P6-pKofzn`ySUX7pMI z*Yiwucr?3!C(8b&fgkIl0Ji8RnCJA8#lV~N-Jb`{IQsH%u`$~`+|q3Gq;3W>6{LssjNLK zRHbbyE-X~_QFL1BWqPaV6Eph~PfHQUTZ^18@yX7)cH&~HU#nQD(BI0U!+geWJYqei z3I`dPxT}S3Zb*JExKe^qj7K8+`n!l-|08_&ey@AzZ zN3)<>)Cvdww7P_bb-M~;6qwkQR&<|GWsH8H&%WrN$Qi~HLJ2(+T{K>FH?qrQG%#{& zI5f`43^<|CeAdAEA&aRlhc2^!l6sv3yYb#G?r}k<9d7Rg5K^o2fPnok>fJTxY})UC`0wf zJ?WJy(rHKg_x+2o>THw9o6)E$ZpUz_+x1_D(iFEj6BbQzFj~RTEICi%X&+yX#FB|_ zOEOdV^)fV+m{caJEMVklsAyv2E2>!3ZOis^akCmjWYRIWS0^6yr+FkguG3&|ZfLcZ z+L+8&G4rFN9@9LT*ZLdhA?17BFnm`zPJd)5znLc+AJdI9bP- z*Q)rkujRxQhh#$fET8eJ-qV;8XkWZdc&gvM7n{50HCKmNJjzjI*{Nbw%4cP^q`9eg z`OO1=zQ~{dOI%sh_XulHLc+{_HXt8oP6Ty;-cdMB2wlQ&l|cMt<84>x7k^ z`Vx#RUG6g`F5(Ylx#`f%qgRkP(ftJn(*(D7D_SRbZ2R!AiGQL@!X5cnCmI|1^bRES zC8;iMY|_6c<=U<~Lot<2T~vUFiThQ93s;ar=97usECrYJ4!!D{vHqY?!GnfQb-f-& zuBsgmH?Sxa@Gw2$zkGoACI8_bUo+#~IVKk;6fB;RB=+@luepM0eEPpH@}f}|4_6qZ z25C*3ob=CocJi&uFW0S>di{6fv0qt%h3a-+KXz0<**t0Tkpn7RC5_xk6NLURNnzW+ zDB8=4fyG*cNx+wM#(cr z1O%4q`)PXBEesM@7n9J-^-LDoaNwkdLIY?rX5qyHZ1*bE&TY_SZa;Z|)2+G5Vxxbp zbFlpHbLDmmExl?rJ)HPklvx7?vt-`PW^x9LYsfvrTTx19vyT~ zN;oxP&Lq1+=}soe6At1R);96BDBR7sz$CN9fjyexh(h0qj*J_R#3E-LU@@D}$X~<2 z<{&N?56KVdZEV2fbSQDvWxY{D~+TP)C*kM23;FL#u`EGZuA^1J0TP?)F&^`gM&{Jbkr1+^3pwW=lx@o)Iv|E?{9@-~=X-XV>Q} z&&`^AO>(Jp&8uKr7E^x5tIl$F60X?DY?xyDppohC=PPVZSMB$Bon)3?u{z?|BzxU| z84LItF0eb)BrscXG%FrUVA&#<$gIH8to|>7J*Z=m$UK7<#d8-Xt6h0eyj8$e;#UHP z(+wr5tsd>BA09M0+AI=W+~A}f65gfyqCw=$i>7yC53KfFXielWXs{P8o4)zaXZ^*p z$@{y+nb=zxdNX4VaIhx$Xw5jtpK9Qodp3aWmfu5;md8D;Zz`5e=nC;`U|HhXT^%%4 zZc?qLgr7#vLw0Y|_PPrjGu6tuyjK1!S>b5gBDbrW!$HPr&!eyFO zkO0@df33PZ3c2%6B#W?qIH16Ak(K|>13uwf2YA(@+Zk+fy7`?OA_Nq~&)<7Gzhz=X zVup@btJg_G%Q}VxDT4+!kpw399d9L$B}6MHA3Tw|K|_ee!Q$o4mBtG>Rk%8&Pdrq* zzQoppsd&1!jL4oV7AF=qNoSnWoD;S@cs9#Z|3;}0hZl;14IVCY-_|^LIr3miVfXC% zn8;8E|EbfO=jrQ48#FubR?yN+WRhc==%YO2;QpOI-q-6JCTKJ89jvoB+|+%O@!a1= zfz*J4*u@t+qDvMF$}MPQ=bF&0FEV3h;|pfv8Hwz7S>2e*c0OlfaB$w(Q`YVGDd%v; z9~Bd?mEHZ74gWrfoobLzPvCSnV;1XcVAYOz$Q2f_Ni?fFT013ySvz`ifvd85$#KpJ zk#_5*?f;WEvGNdSaE8?0<F3_@ ztHAlV(M7&;F5X!a4yx)Lp%TaV+zT*kUy1|`EadaaJ%-mTzU z+U%yu$bREj)={CEsu#>ohFLg?9!qF7Oj*d~b<9ap&#_H~htKzRvGDf^38_}AqIZ-( zuP)rHGiU$i^*a*3wdnriO4OVcG~IIUt0{i7#k|bR|V!xF9tg$V1P4HDZT8|9E zQRAha&-9P!_3e)@1)VfJx9tYo3Wq>r4vDwBT$x0B8d#JSnhg&ysqStuTCjL$vgA8K z-aWUiOLlM4c5XV5toMvbg=XpSe?M1+zg%1Eau1+Xu{I8<-VeGDasias6O$n821M z(bBS0hc$qKQ-Xojz`j9>`-?R9&J24=VY4;SW@*71+Lg+@63q4nMq56!7%gDpJ-{HM z!N|L#QDufMlS#Apil$^K-nTz_iWe^~7G7t+$tXxgPh^3E!E0;xB0aYaB0nqno;66# z-_++XG}Zj`y872PA}Xv23T<&LZSh}R6AYHxHaAM?*#6w*c+J~3Eypd*h09f1%uRqr z`9Mqm<5s0V>;fHIj2T*uG+3e)TAluVSUvkUv+Rn6EENndIvMy9+?QlDa7#2f&S1WA zkm3B#=2Q+=sU3_7Ox#Qhx;rN}a7<{h>|tOoXt4fr_^UMcvgBQxUu^99*v-PhsGZnz zXafVA#3t_*x?2vj#(WT6nZYPLLqTLlllKc2rX{-WF2WNRS)Uhh4M;gqYJ4i?VRDwnVf4fmUOUJqu4PushIf zdxKf#1EYKgqhv+DOahbq3r6_^eAW#t4i!xQq%#<$cQo=8ER%^~icVS39&!8!}*MnvP>Yu7vr@nqb1gU?*6%XlE`HS?HeB1yIV^(wDKKj5Yu3kn9(>< zhf(E6gYk|Q?<+^9=S-fyho?x|b%xB*pd7#1TMigJ_6#iOi`=r(SJBISL35PNs>Pqy z*%q**Gq9ISv{!2QmrM8;F|_CZXj@*=_L<9e-4y+}Obdw}juI0XB{LZRT=8G`h(TIK zY2^ohz6*@%3@kPVO@<3teKxT89%!0yxW)SfOPm3-^8(K3AIv;kEo~%PTr*l+C0N`N zT5eZ%pNwKl%V10T!7M1t5|l80al#@E=Co? z=RYhy_d!|Hd*+mbyPxF*n;$(Xx4}pyf+bp`RX>4`?}+@%8^Hlt%x`V9tWGqatY*nj zXs?{WUcI5c`T=|04)*F5>{S!mD;>@kbuJO)OP2mAVf<6lxIyOh z6>V?66^!aXn(b~JGtOxB=?L{^V2xeC8n>Wz+w0~7LalZS4!KFRI6b)FHp40BrK)05 zOKJvN^lH`(1x;xI*W41CTs5pR1zP!jxHB07>CPJlkrS6vm|A@@b{lo5JfmkYE$BHKxu(zFRx9ArIhzN~whV3g1?*KD*lSL-*Ia0ye3gCb z)%MyO>@^$OJwjLY-|$*lwV>I0hhul6%#=nptx%;?{_8eLE9_{t`@tY^fXSyJbm>yo z*axlt7R|d>u|{uT<&HE^=vZi#!Q%G9QtAhT><%QRBy}@LV&{mMbB)g+|`tRoS1*T~Uw-dBjGzyy3A577E(WJYf*=hr` z!-i&ih8FH`27J0Lt^yYWD_F!8S%p+tlQdeRBG^Q>nMPN$iG+q_X0T;d{CgPmqWPwf zL4HPi^p6If3%W-Qiv(b~I*<=Wicy{}x%EF-i(E8j8=zEvBceK}U7 ztG`@PkPb8lcPU(sG&aqmTMoawdp{v91lo%DleP5rXyX7DMm zs8t95ESc?p_8>#y7Wp5H8VoW%AM^u$tT&pVeQc#q($cL?3b*V9cDX6ESg#1k_j<$?nO#vBgK?zM77BOEgH3ucIq(!heHY5gd1je(m z@UXR5&A5}ijoEBNv&IT$jSMEU1RphtGtvcTq*qMa=4p~~F-W{(nwZ06+fO`YsmfXc zEq_)w#T1-2BRf>^*qgG*}%n*lZ6paP45|xpGLiGrH)e&_}ua_Lto4 ziq~}ydOQidWFXjJmK1KZKv!o6OF)2;`w@oAwv6{$8pSR!2>fV?oAKUzL6gO+wXY+S zOiY6{>mHd-O|qA|dr>s!fEkBlL|fP#SH6%O`>X40I6{_)v=yyrKjn7w^uOobAH4TJ zX%zl{y8Fk!9O)a3U!-zFn+k1b`=?yFDfM8L)DK3hh!)KS4Q3abQ-3r|?qF2@!JxIF zS@j2lS^|@KK(y#vX1yO*?N3b5yAY)(F(Lb<_OX}Efh$;p9&jAbYLPy`uppYHr-euN z(_u}|Z3ism^nx@RW(hUSvgdAKIF_(jrL}>d!`de$&F4>x@(G4Za~XB7UgEW2id&J- z+%viNl$_~7xuU+LlDKelHEQj9WUbK}a3OdxkLu#2oVSmdT|C&zXS@;8|F|#b zqlh}w{+y5J#WI#Tt-r7?F7#2z99>%}2By+g%$Aj(93z^xF|^ng=r}brX$Y_+Eoc+i z%gmFo&+5iffvZht{@qb5e#0!6*(jgU$Z&u`zJrPFLLPTU<*G&HSvVq-;1# z==>X{7nX9xOB+N^_oOXuveRhx+rbp@;rb@I4ZeFA7uz=ZJG8UuG8;KGO*$#Z;$KkY z_MFwek)!^D@A267-pjTf+KZRDMu)r!-2U7-fF-q|eNySY(Ehlt7pEiKKSrE?6Rh=O z-$lU+o&15Zw5W+^1*4`yi&H_9V+320#jksMi8r@xbz@)=TotW40M$F=s`V2sI*UD#pLFbtrOIcDo*z2A>pTgcWwY+lr^wKVYjGO6NKLlP_ z*eP7d*80DKi}`DVHP z_JpkE++pw2NVmc|S6fOoFI>gB>rZ#WI3_(`cJ`J;ZKQfe5 zJSWKqE^+8Q#L^|JS+(GXu-6o6&D?)CHtf84?5v-8-j^LGKL?*zbbtEd;pOEenwOXP zsWLJ?O4ho%%AzLb=)vSwVKR}GS1w3xV~gBmlhjh`D0X79V}(V?f{Vu|2=dDr#QX^4 z5D=cfWL9g|5s&s*@u-p~5g{k|68TNoF3$MealVzClO^SX;H&<2PFb6n2!&gWnU@wR z^F&Oja%{QG<>6kiVS&lpTT8`b8jf@{%}Y0}y3u&-#3E*H9*u?>L2W!@+7WsegHCYI z)i2po)p#iAX^^D4ka5tGhKY;ZJ}CH3vyn7P^_r#@vB^ELT|{$s&Ye5YPX@0Dm|ykj zL*WVi_g5zK2eVmCT@V!GP#PM_pZ0WWi?El(;eRcOM;|n|N>2NFSh(ShATygujX|4W zNKfGrK?Ms(4$b}vPCmwGCLHPzFMUvPT;5k?(+T;6JB{peEfouoDm33Yc$8f>XG4p8 ztIkJ9*}Ou9SyLAoZeU<$-f~SUJN{VOY-XOGmj{@)*(NY5ERIMxZeq)E;FwXB#CHb{ z{s4yqg26KqkEx!wW#`gu-ZrUIjW^bbP1GZ0!bRr3s40P!a;;YaJ$2@q3wfz6c_6o3 zlhK{axa>(w{L0)f;voV5Ws5wFSROS$u;aSq#V#A*X&SmNh_SIbBr?%;;ReyH6^eW* z4?i51naFU-iCtb|qO0Wmnx>QL5i*HKWP414yi86{xb*LYe1ptR4rO11Lnq`X3OJom zXcg&vqI~vGBfDhEkB6Shtv!eOOl1uYIOIg7?NnuE(P=O^6uFJz%BItIrWzh)7l{x!#F-f;qUkDDpt5qQ zN@mp|59z(v?|HpnImL?#RrECMrTHpniX!r*EaXyC}DqI#olIfqijghoO48I7*4b~bb2jbYcj?naCO@TLcdxUp3jY6O^3^JhNj=t4?1-w4XjaL7Ky)z zV7EFIU>-H)AxDS+tMQDBJOze}1eyhsm9q@HQm;6P+!11}&0>_)clpdy`=MFGMv&J? zV>9b~X$J-ih7Pwmg@WEwjtPfdXL3_vVrEp1$Q9afbmG&zq)nU=1xqfn-8WR`;@mKI zZh<%xPk;k+i@yBvhnkj^AlAAr7l_hOJ$On z;*e?Ma8SlWk=NnJkvbd4scMaj*cT`~68PK9&fq$8fyabN{F55ETB^35;$S-MD99DY zV%*X!6Vu9FE^t(AT0&dQ8z$MeFWjw<6|foybo10kv~f&#JR5Z-N%De=c9iGBu;>uu z9^(^?64N%Y7~bJt-#4?FFW@jE6T^e#r3p+T?-sD9*)VpA8HjT;iEYrWh@Mho8!n6pWCW=!hs{uWwB&q#-VmMLoPR-NBrFu zmnJez)MK8UB&d79fn6=2kzZ#azx)dyR+|Y+uV=1g?h(jko6vAhnxlbH^ais^)PV+; z7*Df_SA#eu9NSI(bT;MN6lMwwu)2L|*iu^6kSsi5;(U%KsV0SHg=GOxRhXk+wP0`r-Ech+k z)rM5-=%C}(qT4Q3IyoIUtbAYr^M<5FqVo+7OLEn+w%q9!JTu{#^1Fog`)?cB>HUI1_)=>lLx2JU&x`}&9D;3}91R_=5e{$I3K*p)FbRmfV374PoFfp< zAfW1Yplg51QWl2>mhJ@RH*yztu}UbEi7jB*oPVH0BVfT6qrMQ1nmcVmE{!5h0!J2g z1akFFNl>+_yRz8t!b;{D$J6(TdMe+ZyH9uamnBhU&dT!?Ic7eJp1fL$tKmvcta);| zW_N^(%8vkcHq{B|Tn-#OB<;75w=(r%T;lB8-y9Arr!Wdj*)xlsyU^9|p3c*zz-pT2 z()FLKjWha0qu87U&8is-`CBqv$)@x!c%Qd5c z#bH9y!lfNdqF)&N)u$XdDb=`X$*H+{TC4Lj!tNDayw-ln|11Mn&w^urz67WJd!o<3 z#)+l$zih#B6}=U#BJU52%Q1K6uXtE7&*EjF0JH2LhgWAfSUCC$*a9+`glBcw>XvQf zj_O&+zfB-f_uqjpfgP^}n=_dB>=v*ZTv;S}bqA|NO@F)PmIa3-GbYN~J!mcY>A<+B z*n#ENh8}5w2W;gHPV5E&jEo@-%nlq9g6t3ewOD?$n3vjM<9wY#N$xn4gg^sZo&zI~ zLP%=Iao!XMHiLt#|3VsgEf^UWF`N`+HWhF-G+;LMU^cwrm}%>EcFuvhQ)d5|a^QY% z|AUqN1%h!8OHcLM9{%%%<)6d6Cz(NkD$P%0<$`%60v5>E=i#272`0QvCRYxL2{15LG)Vk$7Qfac@qkgez)>;fu-TIf z>J?0SDd%*TH0ug5w5c*MO=y^Q#zFSN1tFgW?_8bbW1OvSIGSWMFIws>Z{Tb-!AWC* zispu96CXz_4`#&_XA28Qt0m5|F3lEq7)>`g%Xc){ABnXRX;Rq0Y#MOTWJ$9^MuThO z0nP~wLK;!ows%?Y^yuV?oU3@?I_DpQTh7xdEBW;gMNgl5rFZT`)rbA7ZcKTSdr48J z>Awia0oFSQZa&cTbeiirr$wN#HRth*nR2&JFqm)hZR<%kzHvzGM&pwQht#(yYw+9> z{P9Ho&cQ{7&Kf;TF``bw66a(}99VrAc<(qWcQh%Q#4DRH@?LQ=nQ=&R!oe*DOB8P$ z(x_ly?3l&8;Kq><2jv?n!X3`0UmQ(*4hub5BHZI-R-@+2=VT$!BD@b48j{4 znd%yu{v8xIaGVp=D7#>}$PPwf0WlE;5!Mt2E)FJT7AD~WmYpXbomet$*VA;pK7P($ zNd<*mJbT=w44MkgIQl<1*cf)!H?nv2lxCZl=|LZy|HT}Ve8O`z)+i?7+0(-hQ(Mbk z9C@n6w}V08gQK#Ev#>|H0*jO8l7|X9j1n!5Eq5CgrZfrt6<5{>-y+7ey@yGq#mVc^ z@spK}>^Y5(IvjVT9Tc@_WPfphUBr<^q+zOo19ylUy91+mNduq2G0Q3kKAnRd7Z_M< zMA=IicrP%p?Kr@b(7?Op!2hZDS*OS|=t(e4y2fe{eZ=iejamp($^j7hc*Qr=`@v4Bcs{~jLra0T∾JH zv6}1Z`V?jJR|)ZH%C~ng%)59{At8y)qfxensl=>Fv%^V3=O3Q}N0XH=v$4ix-4>0g z-b2b)ni3LLfAo2-apjO~g@a%T+wHDK;RUl8L>yQYcx2lASQQdjTm0Ar8k)c6vI;n` znjBy_;QHc7;sJ+-dwdKm2@cFBCf|%?$TDmwTo9HTaey_#h3$bWE5qIYs#92RG>9-b zm7F`+Ca7ho)4ce|Bb`ao9962E9S!-sy<8l;oHIlUXF3SfXc=DN_dVEn_Lk0S5m(Ct z%s~!|gJe#|99y*^LF0&_Oxrt#+b;R-LF>)KR;;&kl6}%3!opUf+1sD3NhcEXPr{rcv0XQP{&#cmCLDOja$7s9WpXG3Yfb@c zOdfa6H4%m;6CY--1F;%Y8l)@MbN2L{cy;ZB-OMSTU1tMWI2Wur^==WDh#XG|7f*?l zvCiR@E@$7n#jTWMGN@^`KH+>`PX7YeRe=YoW>xdPM(M{_Z4BSh@RG4nT*pbahC_1- z(}bXA^&QL3Om&j~d zC-NK;wrMh%!Zf|@y>7)Jku6hXZFEgrR_w}Q{9t)d?nE=w6H!Nq^nhS2d^?R~x?&Q5m z8ypX4#p^NMXAt_5v1RGSO_4_yFOc44%iyiTz_q|(hpXe~Eze!whzZ>0F<$=0X#SP( zcmEnL95}#vKu5XZwCVL6wU5W^1z1(4G+Uo>w*7F}oP7>2--Q6n3oQvN`0_ZuvMm#T ztR;}+cfOQU13Bw}R1nRR$*ZGurVCoC*qWTwb!)-(_v*dlYoQ^R@01zaH)7cb}@Q zo4jz^Tn7h^H+>9A2joSZtpX0F&6W5rAQBMWz`27_d4Yox3lpmWlc5Xai5CauSI)k# zTkLirPp@c|^3(>-KL?8Y51H;h`2L>%swuC`{69ui8=JgqoN%QvN;ucryTiJCW6b8y zt&hsr^%Y9~%jafHVBk2zd)K7BE2BZ3#qFf4^P1MvCk>s*jz#enJ>--m1>F5I*A2Q7u(jQuQ z8B1%s>#=QVa9v=aeLp?Vxq++W25UltG>eml0%PV?iNgCI^t%PQSeP_-95ms$`DkZj zP``mdcETH@Vtp=O&L?;E4>j=KU{FXgJ!^KoqPW!0b7Mq!;p%e z?sHZ0+DGQ=mDYwc9QMlzQ6Gp=PC7=CQ**bp68M^bQmwYU6wO(*1d9&S)7rN z=a9XClS&7pn_c5S;fTY!Ezh1lVA6ZhAYpSzY|nkB2?tzunB3dO!0n>%h<8q#IG(7$@X=b4`HEA>}38 zZ7McL{tk)a7oE3tt93ZXg7^&a)7w9vYG~bYK;pluz=eZtl?U$3XuMm)q?^#>&~Qki zp@FTWQR9cPa7n{L>w`OPF!Jtj;Jn}<%h5UE&%uW#jT#}%8gm+-mOPYv(I8gQz+&;i zVP)VGvjc38TRjh3cpaDD!2a*|y7q5b_Pbf{{L0wOz;<9q&SZwoe-9?Ge@|XJQR=S2 zu22@mTziQjBpEbc9hr;cw)OFWa?c?uRtm|-EP@%<1 z<%ncVleJvqXNIPpzB~!N!$y|3miW|Yw#3vX-T3 zX0tdiBZxLAHA48w@mj=V!fYl9YdOq9CX5qKl8 zhk^0e=jVP)Y)!fK%@~1kC9$X#=%9#E!@0PB58MSFtMtbPfPK5@W3(WCO@Bz#YBaJ%xVjr z1e2FAvKpEn6rQ>vpvmQvfSBJ6yMuF%aPo*OF;M=ec7nIew#ms>S*+vACXJk120ayZ zi;mi~o9t?Se(~PE32HC9S%i$21x&a9DZEm1viQ5kA|i@8N!O&nDev`@_I;2RB9Ar+R4Z&KeM`lg?*m2$S!B^ z6AOBioE#IGMFJT7`ZGNkmxyIa7`@-)cekN=k+Ym;yYi2JPgbwDn-Otnp=v1imCla2 zMZfqpW`DcAQguPWhlAV>hb$KH1Y|sH6^}Y+@t8{|&+>4G!J>uBBwMXgQaUBqoT9-&U8pCfsvUn>ZO`P76D_v)Oy7giquga|m=0!_qugkDa zIHi~_SM_G&bGf>JrTxohMNLg~n4Y(3>YM3VwVGkGnQBh9>oL4seCVIQ!A$91<0oc| zr~3C z=5zRxp={2cHTi54i%vwJ=i@%@hdYJz3=a00rEU0kL0Chhk;!ZZ=M#~VGoI?Q>PtG7 ziMe%5JjSbX$3VYp+3kv||6ko?pWBA*Zm;p%YC3U)R=5?z?3)ct+nD~UgxPNty>YC} zY@_FfFuMnbWPdR*tvKT=thnT;URfje-o^ubM-tqW0vcMZ4GwW{>j=|zDeO-BcV&^_ zb^&+gE{9oMe-Z=?8xHU&cr!N_X2fkSP=%lKbb5pOzD8ZZ1gZLqpIcBk`ye#_2&TR!VQOcCVl5fQgPznUU)!GcEJo5lXlBT z*1Vki!kHwdJWy4e;m99wf!W=NomEVrC7;Exw`7L0j7LF}iN{5bMOzhl-IF~8WG-oT zN2mx&DYDxkr&tkE=87%BfmmBylJmfUrz#Ng~&}{qX zp98yP0+TXVB0HbVOQY0B;`{Z|)J{!s^X9Xdk>bTXOCaT>vFryPc7<)$;%5@d=Sm83 zu8}yN^Tm-tv_m*=(e~r)6LidbDlaGfdg7tP_K@AS!-;>!hD%eGbh&Ic%#(H!~CA+EkCjd>#QWFM9o)V7egP_Vf=n%R3hZ!_#zCK0Ahbmpo^Z)DXxCyQ->p z@|pwJwNtC(r#h~SYGT%_SSZLkgCWRnB4=#MBGFw7j;h-Ta@m_a(&|ZI5*AZrmHN{t z(v!iWD5c01$dkn1e}kEw<3I!Vkp---hZ=+>7+INAdYEKa9DY>rfbp2$MZ;Rn;8z^X z0jCe}{WI9O+96#}K$6Gc4U=rf4axHyk#7T+88K~W3Hq)I%PohOL3 zXiWH|_?%I+dRmm$j|5&7mMiXd96S2|gq5xlx$-K!Sn}G2TPX*Uwyt`&Nvm0GhUi+k z9}FBT3Ocu#BG}eECqfn(tMJ4-(Rg{Gk?*@=`*3p z#fDkHjJs7qqB7R$c7x<;j}yjs8n(!p+){K@c>1sJmi;%I=8!d0+OOI*ZT(kgmHj=U zput@?jFtVym87o@&8%}m7Zxz`Z~f4~ci;iLf58u-`3th;IRd#HYZOHyUN%p-d4bjX z3L{UP$2|ED3~l;14)T?Q&fEOZo(7uJP-s@<5oD8PND_=*a9D2H>tySvJiH~XP)lR@e0ty@YX?L7k|xy~>MT;e#A=x|Jh`v9BS=E})Qi_#g*Tog~6 z2dMDu6K9q<@nx6Uzm7wnB$}DrHf&&-D8nbrF=;~DhQ^M*KBmhmcPA`6D=u_yMpW?A zjhW#|yxHG({5+ewA#20WJ89gDvM8ZWlZh+s44ruzE?W5PQ zP@$hSa)Gyt0Gp(m(GOR%q-D%=90V91GQ}xBlv8+kz}=Flsn+hwuPU6|0%wHzL#NV`QQI5SaU>QAhqF7NWxW_W9-ish`oBicC1Hg zMIzVAMgfto;NxIw36Al6lG>DWN~`H=#W^3Z2k^M{-DE5Op1a(v_#w*rI-@gydLiHXq5WGz%x-{ip*KX8w)PQyc3X7l(<$P zb0^?e%*m`5iptq}0*r=NSqgvK2?>O+<%n7Goadszzn$wjtQgoE5(M8daGJSonBmAj z?Vy+oW8IdM%m*6Uq!LBe9F)4@F1^T6WRC-Po}z5QLFsjhGDj9lK1<+Vq`+zafTQOC z=b;2nn*~f33)r$4n5DWI*%}y_78_2vI6r8E@d;0Fku;Mtlje9d`)IIvDlKMuvv#YS zfuY3(#$^+k@)$U+9xw_#Fil$gRp!qs$7n zUuJLjy;}I|#RR_Wbqg;IMQnU+72pCTk)(5Sh#kyj{DaN23k6$(5| zj3U_ziBb)G5)b*L6c}t4lv?PUc&#>hz{?s~;4|Tg#|akJnTCl$uZ<#Bv#sn15K~}L zNMo~V;L>@(;KcCNNsX&(@z<2a_vIFM=p`6sFJRT->wVK>)ck<&#sd?HgRfsX2yh&H zz3hRBbEC)-M#W%-pAtnYl$dy@W<4vrEO*a9d96R6`C8=&hH{^v-w_UMAr3F5T(;~r zesStNvscjXV^aho9egqtaK1?p)L>vPdB8n~q1fsH+b#tjp@V!k9$ZxTw@^glpg`V3 zu_wnOdRlKs9TYG)C~$6p&2N&-@eZx$pl>M?YDYv3+P(3#xAq_80P{idx9 zZi#(Q?1kI5hD~5~4Ol4Ez$CW*Zq@~pB`NxU`8~#1@;th{rh$* zSMDI6mIK2x1_6l&oI4oGzoedw%J$$mZ{W5yL^F*wJX=zDnMZ(t@8Up~#H&(@4lGG+ zOmhzKJyQt(?8?xV&gFEF*DQg_q`_0hp-O16v4Mlp)DIJMgxL;+oi1Z%>pRROq`1MO z(OhJa$cl$SUJoV34o(y@;yGH%bNs5(4Q8onUgpm?#IDU(Zl287Y$o;GG()q)`=AN` z*7p|O58_@8Ua^P#^O~71Of#1fb;(H-VQ3VY#VB;FU}@6jJn^EK&kDTfr>-p9 z{A@``6SKHQbJ)u%dQy=K;v}!f9oLUPex7wlNb}>be5N8{u;gILs87mn8_gGzy$j2vFR>({zJh?;yX;cOC;p@g2N8bqfTz9&&$J!2jU^ zf6)PMxrKbc7O~vXwv0<)lv9pROW(TYMAwNF$&)`kG#n)7hbIa+&(ApSbBAZU(*bFH z1z(p&$((-;Oe_tIIScq4bOin^5cY8t`rE*1wcv9Avw*}w*1oS_4_#xuC%7w$f$c?t zq{qVvIt$EM7DmPqsvR^)hAwzTiqMHSsCw;K3l4#u2xjGg;R>FvT_W#@hw z&0yuZKF#okh4qc*eqmL^A5CvXeY+n_Q|yl~Z73>Ppk>b}@>ZWNOOZwIH@!!pPg3c%ZD6t4Nb1pc*_vlHc{Q=G={j+~R5L>}0 z{$>Gxo`Sx3GfTw+-hXlarw#IrA{U%HV8-!Hf#0CPu!`AaF$33)1H5Z~MzvqzJ0`F^ zcCupX>yJ+GHOFM0Q@?sJyW4cA2znL`+Sr%J7~m9X4tQn*=UC9imEBmR`)j`P- zqPvWOn3*r_Vy$>^W6dTJpM(4?i99w7MV2k(=W1m8V!`O7V7|yXNsr<44<0uAW>y~s zx5@_N31Yc#6`0c)`VAkj1sVSn*(9pCc!6oC*K$?fn9C0QGK^9O9AlL&R{T`FQt<8> zSNKbw?<))?_G}liTpApAJi4f}az^tu4!QLl84B$G5;#{J5GYVwqs$^JBF62I$Yb(= z`I_eK9c%wR+Qr}Xu|EpjTNW^1$jpCGSu7mBYdz5a2`xh22;XkT$qx>E8^PUYY9Rg-U>n`yQ3=oIhCQUW_e{w$MfJ}JUv z@`LHwRMz?iZVpABo(KCX7s}sK6hEfG-N(Rfbdbkk0rMM&4Gnt)v}XM~_29tMs*e1H zJPrv=777Bt4sezva85hGRxFXT?;Dw2Ct4!T3rz8aP=M|s5{~^oi?PZRP zhELtBP6njFS-eaD${H^)| z2BQT`MhW~9iGnN=0YS>Vs*}hD`BBo+hlXIw~Yf8*nt>$JPu~^HF)PoB>1UI>!D#{3W;v^^& z`%8qWkVRP6duCMTDVEjY>kJi4aw0`e_0K=l@Mp@)s|)e#v=9$JZzkb1`&IuoLD(FB!_Rfuz<1g>efQ@ioLrJvffx9x_#a$Cg#ic zS81-PV@O>1O1jMLYNA3y|6CRxg9wj=_9?PlOS}xFOAjA9$o^ux`i2P`*cVv(=GB~7 zc=+@Jw@dG4To>-@ouVJ0!gPYS-r({HM$ujuNiY)eN6`EFXXQA{_Ah zL-7GPF@HN*V>UG+7s7l8Wxo_aYY!bu1=%7g0xKH5!D4L zlft!kT_`#!C|$wes<4Pbi9@lD=MlSPcWUMh$@x!$j>(&3FnY)*e!0jl8T-QEAG>m~ ziUT97%>f4{CdH5qx0u#4K9^!XUKrrWENF1yK-TuFX)hKW_;|CSK(xStiCseH=&iYD zEg2eFdCnv-i%4)J9F%KFU_Z>v`qGh$Ye|n1hvo!>q;}T)SqEG=MZP>>lWmxA$wSfm z%PLmIttLrl6>i;psKut6!q6&P#$Y6*d`Y0VUGB;=M|bHPX}8&A+Z~wB`n~({I5^;& z$k&HKNb>!dr3SAb%;tr-Ll= z5{;e{-O`FqN=WB?I4IoM62u|d!t#hcx$Eo02>J9MA5ZGGbu4O@sJrv=qC&03N5_)# z1^*Nrm{b{ytLutBtQOC0x&Xagdo+eoM`#TiASh(v&lf(q;BYL#&U*{D32GF^^L-dq|68Thd&U`$%mUORSvsl5=e=*ijL+RBcvCB*OukU?VYRT0op(xO_d5J5loW4!=LyAzHmyBT&_3Meppdql7^1vGL$==3k^x-~EBq?GN?^Yw=|Mum&BN>40e zVA>PI7d7`Y1CK+aV8agHe~xkj4PQUxC`bt$;FS{W;7MWN)L6o1C}G%o@SJX*n1I7M zhDJuVhy|>Dx{cBbj^<&4$};8!%q9|yoLLRbQVUqu8ZR)6x*)ev##)ix+T=?ArBuUi zPS;rR{fC++wiWi|WzLoO{nWZ=-^aLpn^y#DMFlo`hAc9xI3&O21Eab@ljIqVqpO%6 zwoT`HBp47eRsB~2hqJ-6-M25#J6WK3#qmXgKx=}S+>E8|mS+xfO8szPw+m?G3~^wW zVMx#qk8F&*6%@G9bLmBwt~rYx-)3w{;bN8vn7u-GxgSpkqe4hfBg2VO+l*%~mhsx*YU7)f(E?qC#XeDQ|ojf1~(o!J(~ zRyT*h-IKNY`bwl9vF9XfE43Gi^rXzuI8b(7e|11*uTE9++if|4?n_;J)_y)HHtT1j zYRa43#)KA4BeDG#44On*41&wIyQjaK_fTm2hh{}7Lt~AA1xyk@7#Q!RFk9bo6j+?# zB5(1bS#JkJCWns*v&IB_$E|6B%}UwL`m)zpf3Pv_pXt>w{c{7m*>NV>8IJ4<1+!TN zQV#PSI2U`}lP7IMqP4Ap+gjGulQei(1SC}0q8E2-M=Q&ta zW-#()F4!h>N#c|7sm3Ut-ovtNwi-MA{*~|In|b?=QNzLTC5xJFx<0*|zvR=qbBFY` z|9>l0X^7?cfHPU1tnSz1Ti{;%vCl%)_vO!FnOPZMo<{Cy5CS`P&ZK6m>VrF&*T|+@Z)NV&Nfj zXF<04lN^4D7mUUMLR~IPBzt5djaz4H9@@KN@7YDE7IvyVTdiM5b}Rg|;rVl5a*rz`58vFG`~i(Ti4jh$ zc^gWDCMXGbD@^`$$bnVer;#hTfJwCHz;E3tpZUu&nly42^5*s=G0FaLR`y^FH~Xr< zq55Ixndy@qt|}~_*73@6-oNxrM@6kG3+z@e{q&{enME&OZ$wLIY;VI{UX~`lj{O(s zcP6f}Hx_g$5oXYBXv#2bWXjvXVLpK)Zo2lu4?*juH#AJ(xU#IA*_ChMm&^2I|FJMs+P?0uZyrme)uFjQa#QHDs0h`}~>b;lDeFR!d8`#Sa zIQe{FZ!_SiXJ8WzV37U5T2jnz8^EOZfLWp-bVW)@%_kE#=djf(GMiPrFNsU9nrOd4 zv1DDc3HOH*H};PDOH76VEI${Q<}z?pePH5iVBi;Es+Z_sKWMC;5H2!-S$%op)KC7>qm}&RCjoWPEv+~+N<)Doly&GS|SDlcuNN_g{ zVBRr_qk99p^Meky26ag@0rdig=LZEWgjtVl;9zW!5(!|M7m>YrlF%gvPRp50Js)^t zk2{`uz-GIF<1_=`Lj%5z9f}f{ri7YIec90cX+ifoSG~xW-Le+!F$Y*39D0sd^ymdm z)BPaAw>9;1kyz-)ru3~MABvipl->#9GZ19E(`e7`SRUxn=~gg>7JLd>FIM!8e~t zd$Jk>j{`%+q8Z!@42%K{Ob(1@3>;evxVHyzL|tI1FyQ#|P^i^x*ya{{<^v3Z0gTlLcrGxoFbgoSF)%O)WOH1Qu`3r4 z3FtJMnJBhkvdDsL$phY|AN*p(S)~Lx&Oa#K62zpsfU#^tS)K!LI|Fm~1)dEWe4#FU zFBzu3vYfhjbClF0;kBXj9VRwLzlnSb$<^GO^lhQ zZtz~ofc1iyV}o{xfJ*gDZi!RV=WkFEd(6_}z*TX9=~kkExPz>SK?b)2gQ!F1DZ#|j z4cfH^9L*0{%RjI;JYX?5U^l+tV<^Bn!L)YG!M3VT_FDrp>vznmOe*=;$dpxEA+s&A z?q{+FcL0mr1qQ|j2GIk?{0$6l?F`cn+x-g`M$FH~Rk4F$-44C~(f>T3~0ez~TYZlBflzZZPp}?cuwcnqp!QBDGM!fTfs$ z!+XK>1rxYmT;O~AfH!$Vq0B3mRsr5-2Ub;uWx55-W&w;W3Jm>~lf$>JnPSPF#$^1Z zk@2(^gK$EDxdF4kKy7cHsOSm(A~W1TaTxB~MAMQ{IG-a8uqF*`J{ z8%~&AYAV2!z;I|HgZ~DOB@;L#SeR8A7^1V7q_^-Kxxi-Qz|+aV{KK91!UFcokxUX> z1XT|BED4Zm66LZjVDjSRdv$Zl!wd7?T2AebYOFQqf5fy>k9*}`2Y!dbsoy@7*95Sa zOjv38dusg4mH#EDIXqx?l3ul_s>j4(fnLIbuIEp$V*M2MVPV(zXR~H6P%p5@7Yaz}^tR?9r^w(vYB+z+T@FH_ubnY6_!|0Edm#)9T;9h0oZsmRE;H(WFHFk-bzWCTK(4a0Zz{sq?QoVu4-=XyN zzea}1FN$9r)UqvL&T-)C-XOc-6N5p6u3CevE(5bb0F&(o_Jrk=#SSp5PY_^dWY%zC zloDWVDrmC|=Dsvx#j)-c$9~TfpTYOZlK+hYfB&yd6TJ8*X7N`PY%Xb7X?=S0++XwU zxwkCv+F~KF>QQx%o&dAw>@Dgim}Czy%094iVPL5iU@L9pez9PyV=7xj1M7kWHiZK$ z-2$FS)?;$YIk)2|ktC zdqOH3SBFkLcY0IU%}tkfvtRFKzqxz<%v~#OUT>bywXcF_-=pY#77omY4$OKCCkzFc zoe!{1bYSCS=30AzdtCxsx&ycCBJO{$3j~%+ww4tvVPM(n$gp-B!WB}@g?_P6}MFvyS-}rd3o0JUx&mUnBH+oM=~GQmYFDEu&>(TU;u;P z6(N^39~tgGWD_Z1Ub2CmnXz=KSUu+l2etyH=qv$AF(w`XCf0)jc>;1v9e6G~@Ypsm z&g`DYta@xiMtRboe^a_<@o7Xi2EXL{_J)6lh1Cz9y`LJ`|Hb&-n7w(H7H5e9XN&=R z*_*vpGFvKn7C4?hZnA*+xCDER0Y^f@>KPL_XEm@^FXY~}f%~5|Yl8#N(hV%z85zI* zV381DP_<&%{;*h`cT#Bqd&vhDQ--YkB*DGEEkcD?Ft1HVS&iP@VTf8hI8)BWIDdET4z=G~hutdFzC zUSQMZbo#LJ{_P7>&R($laN+U4)2!7OIO;wy#z`AW&E=F|%iU;cu9bS}?>0G+2kKp( znfiMfcze`k9he(*m`wv1j~rnLVBk=nIl^+|P#J?R+!|U`rukjitURL0p zotjhbP=59U&sX02KV$=L&&(~0<-6UP`=B6~(T{JXYWF*b#;<$W^Ukp6$FSEtV6|Ai zQs~^?32!$$EVv+1_fXvKCXej2FRO0$Z{U#Mz!=uRIXyvT0|U2QEvEuQM5jWzUjT#u zWUt9=4i|PE=$kov&YVT&oQ&q3jExiO{;f+o?0i`2?%Pw#XJ%^H+-aBYbkE)KueE*S zL}~3eCo{Hc@EpjfW?(U6*szr?c8LJN#{;Td-!1HS^( zi^j7r8n2c9dsL>t8YsYV>4Is!1J~69JZBkV+v;9ic*?Vnleb$g%Bjrd{s*4>A9$aB z;AON6SZLhHzV1Oq!3~~w4+Q6}jB_|Ha*u^`?!$YtA6DpWY1?CNtS}?QLB%ad<*9^3 zki)q%0&J228N~_A#s^r+FR)srFqa-+5ic;-owdmN97CzAto5vt-hxuQd!Z|zOi=Gh zxFp^=?b?&Fh(qta)ZpTKPXj4f{gyAK0{ zVdk~uxeWRL7}!5dDvqlQ$zjy5W0q=QGMUMtn!vvG1JBkA{sQtJ|4i{0x$l3@fk&J_ z>c|IPnJ>IY4PGjo&wcuVea^d=*QJlo^tvFjj@4-atHXm&Q3{-f`yVd=;*>>fOb_y&J zXIbh(nS2^pWG>vgxsW}`f$?e!)B7XOK3@Nk$M;;RfT5&MZt}nL0_qo-A2AkQOe(v3 zk!S0Js`>nuC#ShA_U8$3;qw3D;??rYYf1Jxf1XndvSt4BUNCrBExt|)&D+%0-9&qj`;NEtD{i-YPl>_dr z7ryi;u+|o^R|{~I1#sydGc_}qxgwE$!Ue7J56ml-r2aG(G%B#_sk8L9s;{5;hBtEu z!^MV!2iUl!EG$keP-tQ0=h160_;Bz*8?Ta9jDf(RW;T8$sXYf4xpFg@3vq68UM0EN z&0Iod!Gko(DLjEWI-83hG&ZyI^R*NRTse4tfo*M>7gWz=J833|fUst{(PS=Gj|+3e#3fT?BxHAeX}UPyxZp>OLURY7 zq}rJm0Sg@3ktESrAadm! zgNwyOG0zmito2*3MR|x7wg|c_{QMSOpeWcDlh_g{xYqvL4d#C9j3dnJ_wHff*`Ofm z?IZM0v8%=BWSwHPkHN8DQ(cS{3MR1hJ9?~aote34W|5TM(m9i6Wk?h*@^NEgOyRe7 zifrptO{sp*X>RDTQY2_mL+}I_rj3o9K}<6n85zYm76h~|YVKs3U)=p-(aRmp9@c#t zy$P2%Ha*U}JXdT-LPN7z(E&>yw^`2HtZJ4iPH|7$pwK8(ayfxb#OlF9$GCb?fg9{j z777|Wnw>s`vsTYCJ(TI8$STDBt~zak?Ux(P{AK+2l_wtI_iQOVF1*rA&|PKv9l_K4 zZNJ}12x3W_V6Aj;aqR9VU!8=i-X8rPUqAK8_OG>nbtlC)7cw+)%g*|7_+gG!vYS&h z1Lx6cQ=e%l{$rMwSb3SxYQ865m+Q2|C2jK-Cb$UnO?)9+p(&www3CH}gQ1aSfmlZa zyN<(rCKv4)E0*`I(*AX+Q>=htQJd0)EiYKV<*!iW;EHe%Y`CPr&2fayOrx>I#h8(i zp(&(6Lxk1Hp;6{S0%zF_MwV$tYglxSHnR&{U=}E7+TD4eh11HBe}Q3dj)1dQ=LrUB zsf#_X7Zmv`H!kG;x`5s8O+i72^$|Hf!z0IR`HH4&o?{?0ai{n}?YOFsPBJXhcGXGh z7_ADH-Tz~$mAuHC1c#1Bz8C{$S(gsUbRL(M6(3u94HS9IXEdDGX=oA>InN|=!C^&2 zYa^SAg1LCY;VdPQf2_71&Y|TA$1>dt8(9<@7}+=&xbKLz$bC@Y3Y^i%z-FNNPdB0F zOEw3O#ZCs6l1-BZJ~;Z@XzpWoC}QB2Xke6cXpwVJV6fld%u~hCabpUTKy8AX=8TCQ zl5=w;Fs5BvfNo0XSj`*sh z8g&!eG?qN%%&ka|yQOem`k5lX^`3P8Y73XTZ^nXM4R@YCa$3Cm4u@yU&PDt+4a{w3 zQ_3tR^oVZ!p|4*RxbW{HM&5Y>9YGasB4s-oMc58@NT)O~tkpUg^e%#lhl7FHIN^}E zl|uv98-8Y=h9iX#4 z>Z&kb1}#^PhM=&k4Dqb1ncEf|T%~t_p;zVrg9-yf(5D7=feZ)XcOtDpSDHi)>9p#W zdCjs6Sio2!Zo~WwK5JZ4=lOw=~ZE?Bg8ws?xuD*TZ?%Z`AEY!>3zaIjH~V5~Fjo zbhE^R1v_|j^NJ*QU3lsw!ok(6Q6jD-*_O10*{tA!r2VTF%QK21Rwo?T7Tjd|63M{K zxq-pNR-q%X#Ig4LN(O$9k32#DY?ZY86IkjL1-bSKJ{D9|P-S9bXyDx8&Hvi#h^*BK z28M#Is~8L#!W9iz*cBKKar{^n&io;iDL^B+W#T}T~N*Wl| zeH++zlru9P31pB=aOyQ)z^KM^h`&YQhOiUIl8LSx8r?MWJaDv&z>-=}K zH!~V|kKbvQo^g;((u`Taz~Ugc$+;%+6AWzn4U9a;_ApOceDjR01k>!x8$Ow>SjekC z-FR;9?serK9o;8L>hZ>0c%f9V;JGV9)rHUt#e6^Zyy&p|vLMzW=IJR0_6cqh@>bgh=vig^RenW!*D z0mTE4nh(gaJ#5r9Im9Z!*}%^*k(u9MA*b;VXAQd#jnV-JIprOg>gHZxxB1e@W6b8V zW)2sp!ySj$2`3y`Ux;w2d(4pxXmC;RXxO(TBH`MR1#W9P5{2D7k_E*%b{;d=tDDel zbH=*j;EB_3!nUsq{5$qt(F}g40+UPJ;!L?Qk|wuUbu^k5GVl7_(4dsjpw!W@ zfZJ;EZ_5Rf8TltPa`YVV@3gkt(5$|L*y=bDS1}3|7_dxWV2WsoVrX^~VCR!)xKq%y z?Q)ZS2FsMh7XBB_4+5w1bF?@*H1REHa#Ub;HfYg(pd;MS7;u5vVFFt=M|(~}lhuMY zCjpkA9Y(est*$$k32-sK$zbje3=WJ=h9ZaMnGVk9XgHSP zbATb_K$Z^E^2be`oO^Vp((1P?Q8yvX&!vm2~3U+^SC@2gg6A)1sGXYD03R^)9`2rIMOA2 z!s3(k{?I==WI1+D_O94X$N=l=E!hzJ5 z296Adj63sPKYC}BFie_qER$syi%6$@MUy{wv)zf+TAge$6=$+K+Hx;k>Ld6T^?qe zh$fzlCWjLYK^t1n{%GRqm|C{F!O?=*`2&lg27~#*gF7l%(i7Xn8ko{JSS(&SNbhKL zdbxl_ppjvw9ruq${s|Ti7LAWrvN}{WNc}j(Ve0xR`{+8K{a>2@xo(zn%`u@t=IF)rl!ndBhg21q zqYtzys$NoD+m`FWuD*fMDuTsqN2Aq`r7{ssx<8l$HCj9*SQJh$Xl!LJWog+Y%pkRc zIZ%Pc=0`*5Z$_qw1~m!Rm<+Z+j%LXdy==^8zb5X{F=)DG4kPa4EtG}y0Tb;@XQV_@cQXtF=ipc=uX`J!3>0E6^m2F42w_Ai>h zeQ9D6U|{o5aG1f!f1rWSz})&%OO zT@4SXFtFaZSbBGU;|m6Jr54o}%rPI>Vji&8=mzC`v|DRTcIjwhx7e+^V6Wd=ap>nd^$mjqd_Wx$>2njjz+WJ+-9?Y#?sJ+g$@nO z0Szx%wO`6cc22oB*_B~(teL%QLzoeRP)8%*2L}ENj8YdE`F6~bF*wP8f=R!E$%>(c zk0X@v1B29z2HOf2E{_JDgm~EtO(Cxoge+QIXE2^965g82mQ>K}trab|qmkvo)t#LG z7`az4x@E9=s<1geV3gKyPFr!B~aiF^c`O5KDWQ?q=UBeLCZ5Kvd09 zsj}05x=**RYCM|CIKTQ1#|iJfPfi~Y>0;dwIPvNHGjC5bu4s_gXlXst8ol9>;@NQR zbC0riuul2YY;~c-DKOaO;rt~x=Q9X2uxLE)Ji*(Vr^X=RH-jO_(BmHe4$0?dFS~7v zb58T?fA)CNQ#Hrf$jM(l*cR;ibd}Nn(_+a8rf;>GzWbUbUo>znXfR&OV5`6;_F(?> zjE)>D2B90wZZn#l5?TT)TJ#c{jI}JxW;AkeHt}&aGR*9-jbJgih;~SLAW^U&wSl28 z_?dV{W5}OVpEd{l%ib3~(RhD`=*}FzJDywZ?OyBdyKzm(R8KsEF=fdyCX@8NJr^^0 ze9Cy=?2|e7t5WsB5_yRg%s~#zqEE0YoozjS?|Pm?dtL{Nutko~~o6p?FC(dSrqZ~d&e>30uJ-1^qZEM>as&90s}7dHk4fw|TK zOZj#+$`>@1|7jAs@kH`Rll2Z}mEy)ELDif?YV$f~f3`4k5MVE4@JdQxQo4Q9g@ZLn zku9@|b76tNnHe*$g}zhBVG~r?&lwsvA^KUyJw{On`^;}%JiXT<*Bm|Y^^_Fb-0dtp zN3JIRTYUQX*3(K1k51j4|I6^7s&wyjqdHZ|1>EBQ<+;?| z`OcN$k{+DVFFyEv_`v(V#ec=usQx79e;l4tim5k3JJ>EP`K!rMd~J`^^ulv-&TR$f zG}s#$Uof0_z$Bj0V46~-SbtAZ{HC*7%2d&FFYdl|c&g@X_9^*Dqx_AX+#H7D8O+un zB&>Hd#4cji;%JuGW9!0|YO6k>t)js;qf>uFv%SVX4F%S;%1)L9hGiyP3k0}?4H*9& zWf0oHY&)aTDWWYkfW@Vw*`mQ->A?@#hArX@nt3S`_s*K;qN9e+HGKsHee07Q4nIl8uPp$#+)@Kqs8M#yX%f- znH`K8Gnh3_Fft~56!X_&$oS})_pMQZVHJ17zwk$Io^QCzq0x|bUxh)UdK;7B#MbJ< z@W}o!h8~%a&9^wun5LB9n;-J`k(7DG8hyrThGq^4%~lF6GAgW@98I6)xE4ytbL?Oa zifD0LA+}zUHKUUP$6!pw1ws6Y$``!hcQ7Y$5?62?rWj+BtZ13?3|4 z+{`S%Z`1SOKm#iqca}?LaI*^&8)ubH(h&}!je*@#t*b1%M3zYL2v}%%U1Ge*VvxtQ zDY7_F$y!r5p!UN8jj%O}0WTE4^+;^Am6W(z_3VxGt&~#}REw9q5LjeW_-IL>am$HV zIlCI3j|ZDM_~ooC4k+A7cITI9PUML)$+$4rBjLk^k4El70tyxdM-Du6V&fK*>1t_M zz{Hd_Suo7GK%jBqN!Io8yTc9?Ffxk;NX@kR%6V1Yf36klt0ylnFZY}8bGOUvMATaA z#3d}7m{|oDNSo&voOEz-7FILoFi3cGgq2acU*fNTfPy0f_rEf^t0_VWe41`%3ml?c z*03@$hNaAAlAim-AydZj1B0{NT$hW^f>tg<%>O2C@~C&T_~78nXfwefTr9va29+#l=GkOKgJAS>M;V zoML&(WoKA)*Pd1nnUV`rj|s%^eN>He{-U79Z#(0rb3!YRkY`H8j+d;mF*b}x`0aDV zoO$$GwxtUSOki_lS@BoncFyXPY|<|4SGG8c^vw8_QIzlV@t~l4fTD-`vWi17rfn|C z-txgQ>KzIc>**;nt* z2N&rxUj#0@D8?RnFRSJR|4C6gWPgTXE_E6Z@jp z1BWtenN~?H;ND_(d*LS6C$lb!8w7f2E^D~NDH)>B*upEHQQ#s`ClTId-fEJ3S|Q#; zkjrpMixP)O&x9Z^&Dt$l-Wm-&NvGHL7##Fa_TgYSuufN}h@Dg9M1+&lm-CoWP?PUCa+8`|aqg);byUf+c%<2|rJ}`?j1c}+a4~gG<^yYr$rAE@~ z%01EY?PuJlMJ_8e`jyG^>9}n9j>yQ=qDM8$xvSotbmDORGgFF%cgM3^SrgbCp4Pgk z3jAnPSoDn3$fuDrWCkN!T0;xJLL%4fO|GU=A2pp#CI~h%xM(bR&=SycD0d+jtER`r zc2^H&kyXDA$a)kqGX@+eknm`!%QIl-*|LD8o`Hq!!{TE7 z>fylI;n?2UvrcKUbrJ^;10(a6L_vcM4$?0^FbUc+_=C%*2VNP|4G-lDxzw}({O-SNMp{b z%m+;S_JA(sWYn7QFz~j)5#3oAI}A2x2?TJkC@)ap;JDk$u54#y|yE5x(gh!ZSO1#Z+$yiZ@(jPF=cK?kJaZ^^@bgkq5LMI$dD; z7r^bTw`r+fYKqOe?+$_V$ zr*j*6CH}cGIMqN#;liII`xlDxHpm;EWb}E& zFx5tZt@OYudAkjb+#wAt;xFRaHFmW4hCUWsw1HLe$c09ZxCCq60}Z@64UJqW4(xsf zhboK`lKZ(jrroIV$)W79HRbviQihWCL@Jj6lGp zpcK~?tqcs)3fmd}GzfM#u(2@-ux368le)yvF1KQn+r5wPly#K2y8Wux_e=Fuo-}z-R9*XL#oLtT0-=tV)cDjfEO-r-X zi!&=^Ud)^yRym7f&8MmBO0*b47Yj8WWZ+sT;M!z-fsv!&0h7#+SL|()9KkM+#d-tO zBr+y23Z*Y-6H#zr<+zZ|_GcF3x}^`xt&X#ZFFwH6x#NH?;|Dn=odgC(1qTK;hC3WB ze3}Q=UaZz*=;XGU!fC+Z!p+1G!{Cuna@XpS^EQiy$g2+pFC9>4ODJN`n83uPn!sFA znXZ3&-c1dwq=}R7{+Rfv)nwo z_dDiqzJ1B5`Ok${PnqZJQQ(|>`a1j6xSvb)d*he<^4vB5+mYkE^0L8rXC)xS}nVi?yz3=Kg2Ia7I!f_7(4sEH)9VS8J_%1Jt$b_<(y)Wys6OR-Snwvpr)zPngHyGhOzIljV_n^Y1#p zeb_9(rzrR+ZvNYw?`p+u&iMS{kry}-t~oO(qo9H3i_iHL3_+$17ruD@Tfp{(msf|C zt9t{3FpJZb5++fGShj=%>lkH4e>AGkY0`~hmJ~Upez8G+ts6L8UfxqW$s-rQ zEjwdQ+7q7B3onVRVtBBz;a|eSlBfMaS0C50axs`TcgMfS!)85VAS2S@%dvTR=t^y)Bh zzF}1HNMNx$B+PSwAtbu_z?0OC3>ilqxFZ_4H$?E<`Paao;J|*xL7IWJFQ;XD1`Cr4 zlkt`VyfzC=e6DdCuqtajmHEO~AnMvBb0=`Y&A<~!4xCVkh->8$bIsJ@cw=(@RqFxH zDaUG_NY4K%DUhUWelFGgp4{=bk+VbGWC|Kp-msnuy|r|sgXlee?hXd-2@Zk`zBaG; zwKoQ3orw8jdga2+28A0l`sfgaMZYY@Wh4&uaxL*f(#5U3=BLC z+&q4~cO3Z7H1h6X;61}A_rihCf|2)yi=fOzcMX2e8*B_W7)}2DVcGBKETVFN=|F?r z8^(;Sj6McOx?~&*6I-7sm4}4o`3Spj|xW_p85Q57}DSHh(;-9-0|;ha^rsE(=~5WMuMW4SYz@w zp{xi7Hl0QVk6z6`ZUq|cMK3fmW-(crT;RWRP}pVeuZ_>-^X}heJf!jGpwOHdyjL83 zZ5aCuSr}ay40c9w?{Mg1d&as$)}c~?fk%Pm4WrYIBQh>Yycv$l9?ljS&g>>k$|o8E zJC1a%Tg%rRczpypaF(I&QxJqK2=%SgX)AnnSk z^a?jE-$cP4x6h6a93>G>4V^H~9H(v^!FfuYmj|& zaINWKSpf%yEJou3X6v42b{;0>BMxpIE$$IV+y#7mYB+i}ESFXd3`?0}Aa{maF5F#l zacGAF)10M6GeeH7owwV8fpZFjyv7TQGtA547RPWcY;nt2{;GL-UDgUYCl!}}PA4)> z3V1LHHoV}QacLFTvV?66YlK4A{&GlTa1=fuF7V>OYbCxvlPMGS{ z)TFuK&_+4!8&zw~1Dxd<8f8luyU#W#ED2C@aJ(JUA-yHnc*$Y25N7i`P2wUth*|UIeDv_F6y3M(+NoX6 zWhdvI&Qo4~P3P>ToU@mnDm}SyTN-38|W35;Qqj=40u>!|E zpO%XK)6uqPTt#h=iaoXdO9-58Zeo(Fk8lCosN6F?3#{Py5UDHClv*!&*xYL|GYVW>;T_} z7oGDir7eg_D`*toQYc&DDBHlOv2~+(Mx*$F*Wdc|=U(yTd%_@Ea&}ooQ^AuJni}qk z`s_bFnI&E@{FCF^z-DmJB+bvwahC(ElZ~=C ze0VN6MqY8!)j4Ee)5jvgD{io%Y+Y2>HixTuj>0wv6>OY%Qy!(pUh6z@*a^B8cD!0_*(}@KacAl2eaWZajw%E~Z!LA?U^kUhmH!X7c zb}Nd-ykAWH&?s|;^-J4FzAMFFPaU{e)^>@B@y8}Z*)u-sCwtU)o=j7#fHqo}}Sg&&RzKN=M#Fs=;PqVT2hztLOSl7l=eX31*EDepMQYcfIM&wF-THo1m_ zLOKU!8ys6+Hn3l4nEmA+Co{_FB5nQ$bu`u3~1X)nY+ z9?HGGc zlv$I;{)B<|LZjRj23Z4V;|-r=H#q!X^4oWHOM;X+Z^c3Rieh7j?fb7C)G%?f<2}Ss zz>syM!SDXDNs1F0c~}c{*UYs#vOtN|SwO3%ee1!4(tCbBdKSz2R%zoqjm3-QRzg+KT z89A(fd+Y8=Q8ph2#)JmG3k|XxrpiuWez3{e_{VSEyCp@=49r^&tmAWf9M)_a;Vj7G zz$kD)-o%+@_ur}Cc6*)iaMyU_854HP(B+oq<_DsGI+sK)OuOq^b;ck`R4ynW|q zdi^x79Rp`U4QGr4=M4sBkxP3UYfi}ju<+P5FR;qe_=l+X3%e!8EvFrq*H^9Ru98qU z6X1zY-tC#@{VjRNF2yTN8h4mJoA3W@Z-3#pjrQSR3Jky5Z#Ybv&Q<#F0+;d?MqY(R z*)Ioqb84?llxV&hd*$j}UWV=BHx7zdFp4~B5Wmy7+0NNmhq>g(fuh|zy$Tvsjxabp z$h@}Z+Wx?P3m%Dp1I-KF+7_8=2`pq}5fc|PNN7lCWM*gM*&raaod*O;BFBM)iv$OXonPy!%*u2bJ-oQez$>pk$c+`uA zkW;fvvd^z+{H?_sV^Q?@*v{GJ{&CS&UtZmvtSufNTl4eXv&*Z)Bv$(S(}MUl9TSdh+iDFH81tro|(nZ&(7f1J!x>5O~ks4iId5tOHf(U^4}|i zO_LQ5w{nSS-PqUgkg4_F?(`cMlb83;)%UhCx%oMnZ?SC!8!r>nLZzc@qKXbTF7w3b zp18pNa6-U>hH1tLN4{JPYB|ixCQuY0;E=%3Je%{IP4uyx$p3%Od;zTp6w2UaWX_wV zxvY;bp+J3pMN+_WHrr1gD_L9)2pnd2iFmbAK*!=?qkyi$LKmK@ge78`dw#vr-grup zTc+q-P_}O6i-Q~I$-bR=dR^^@o9X&{8JK-~4_r}D@;oZ?c@dw@yM{9p9Hmx%l0Rc{ zc&TJe!OCT)_4nQ>Qi#dfm}(rj<--GJH5N{m2@cOD9P)Ks@*~OAVa|&OP24&g&Ti*2 zT5y2rAD4#2+dlg_2}RiG%KpSB~R&$`M(fq~(l2J3mfT^o}7TxLYJ9AFUpv9)oU z)g({H`Atn54lOKO6%f6AUX&K&3OSBM7j8L=g<8eB8jQ>`5{gHUitX+NatbMs<6YAW#?lP z>vtR}6Xi{^6#g|aT0A(g>$OQ!0}ETkheO@1$0j>8Eob-e5t99I;IOAnUfCi+H-^YX zf@yDF_%_(eZCuc=BW+Ni`C4% zjtD!gIi9F_`G$!$mQwlNtt%SX zjV?6a`p{->bK7@*UGVH%Z+dE4q^>N;Xl{D-a}A5m2|i|v10Crm&652q4l{6EX!!2% zfKecVfo0J#M%j#o{4Fb7O_UtvRtO3-=qa3M(Ynyc_n?tect)dwSRhk;J4YuqdyX$l+%4m|Z86llx!%2?i!d1x8CwFOHNKLcJ!c%8C{KI}V8YNq1$Z zJHC_<7bU)=9h4z{;huIULedOCGO z3b$_Q%H-l6WwQ;8?wyA=O7VxrDZi^|p~}p}uv1RqFvFRtOlK#l zS-f@D+pEV~^sq&{!=s^bRY*Qtw}h+7p?`%(j_8Ta@Z@nfd!or(Ww&^q@ghN+Whbm^ zuj&~T3aoHY6rOoM@R?Eel^Mq-u?JpVH*M?M))nSvYFzv#rv1`a7w4JHn*A>GO8VA? z4)V3L8`(NSW*J@KU}k&358=sZ={YYu@$oO zmn7^lIngLvbb-;DVG*C|2?pjFA3D@KHu1e&%%ItI(>!QHhlrSlo`2D&28BC}0^Ju5 zD}5{Ia9Qx4uR7s~Y)UXwL7~t(yEPjE72N$e-e_)6+2O!x<)C`BNm;rl_D#}hC96$W zf;XL?aLha*hEMW#vCNEC_UI=J?3NYZHZSZw;=D%XA6M5*&lM8JaV`_@+{#UGmR|kV zEqieww`|ISm7*tZ&Rdf+`*zQl2~AICdi)G&ydBWMD9W&8hNMG-Pf;R&)B~pqeSyX| zDelX;GaLjP47$=ynD&H9xv(-AG>F+`@jqO9P-t4&aXudgo-%=unbrZ!L1z{+%{;P% z)kBa=ZGi)WWI?mEkpe^Jp$Uxa2@ISnoV;l^%@Ri*@SFTI_RN^j#Cbodfz{%rvqIP9 zq8izW?bde~g%}nI9=PIWF=6hTf(B=4%{a4{uUy4Wd3}?y{o{G>3fSLO zoN2${wakc3Q#UQga6;CNLkmSiKbbv{eXp9oSlYX8Q`YR|&Ht7qHJl3jw%tv!LE88- zTl%iQz4IY1uRa*g#}tm+!oZ!url$L zDC`$|lE7`!v6N5hLW|=GqqX8M7&QM)IB9)PP2|XiW@#tKUY}scS=uUGq2Cxd)i`p~ z4R$c_D#bGM8zc&z?l`p8)JSjffvGWSOcU3+ZrN-X`;UPIrnFN>-bKs2Kk+W@t=R*M_f`IK z_wH`^GkN}=ce)Pw_jm5xH2;M7Ge^ej=?<^YR*SxycRy&aU_(OF*_jTFUjKL*nf$vO zn9b%*RJy>xF2~>?xMKALbsi>xYJrw{Q3+2at}NuV>Nv+<@}XUA#{)Cn8IE@~t~PLO zIl$DT;d-8VWBi|e*IVs2HW=j_F!Nk_z^1$6ApabLj=&xN_?C1WT4&DbvSZQ#jX&z) z97Viyj+{H}wP2!wf?!wf+bemDQ+1Cn;FVdj_T8pjruA#Jx3lFvFv*%=rd7@6 zdPazksp$jLf`r^R37k<3Tnr1@k`C}SG%#nK4^~o87JI-Hv4D|Hf$2pvwR-=g9 zzlDN13d|V|OjZv#84}qZtrwiNfZ>!Hd(?q#PK*L6>r5pUG7B|IOj{`OPC;beLB1IW z*_N!-nK2Z}B_S zl)zuJfXkjUYRv{l9uM{c2KG7!x!DaOGa4li2?*R+Ao%P6UsOL^#sO}rgUnqCE3F*1 zr!5fVbCvqHC{gCpLZ(Ch0$&(}JPz_rc)<6kTO%%m(Jh5vj8Wj-L_x6yOe_xU?+%C^ zNtBy&fO}dauR-8{tIn8?2^@!A)p93k9b4*R*TdqqfK_IVWPqUPO}=+d>P+ez-mg0= zW?Z0ITqvM)*h?qbI7d;+ue+klee0@+_d>MUCU4Z%@4vUJQTo8APYK*p)foC$7)pI_ z@h?dZFnM*lV!{#;zXAz8^BD{HdpZSYJYajnaFBtKjfsJY?ZCO32b?Vr*ykhek~BoVZ3RTVERmf zF^{3Tz+X9Lg{Ibne|&aEJd%laEDWpM9HrJJa_(x7e3r2O$NI@V!VH!xS&Dl9uUho_ zm|iRsH}kgzf(=F*H}!7Lc`KUS!n9deeC|W*gTgio9VPb(vuGxZwE29LdA0DtiIWKh zpC(+?-pMGtX@`i|iI5LeCa(zFP3l>!8?*#u*>FMSdlV)HSeuSRlBkf%(G% zF}a1DD<%qZ9pwB_B=9T2w6l?ysey406XOQYHj_`STbe~?Oc4CX!2FDvdCLQSE=Q4l z4V*I?dB1qI{W@>aX{^5V(RIIn4gOv(#TEyYXD7-WIq1_YBy&XZXv&kLcTC>@I%hpM zQS#7d>&XWtCn-uQq=~4x8P96G>+?aYwNU%MvEJ5 zw;{;!Rlmoo33bY6S^|D|1f0%M2wadLc*lV`FTwQ70w%VEbrSCa(-Ih25+0s9^K9NK z&b)+XrbNk{MnSs=Y)lX3q`xq|N)Y_g%f_~lPefxHSE7i-LH?32{2#Iw z7w|pVDEhj!Bx;S!Cf)awglt}l_<#PA@NY7s#H2)#bu1q*oz-I9y47HPfaugf-ET>{ z>+N<6En1bRAid!8x-&uL+-E#i1#Aj@`0CVlg9JXYg@PW60y7k7U-MydSyu4nMDd-k;x)Vje;Eu z1!EL6HnkRR=q%Jo;EQ={q~9SZw=mXCQAB65WX-!(Rwq5I4Xs(&uiq;DfA1@k$AUL{ z2h3(Z^a(D!IZgN0tChFX4E>&p*i1YqIe{^;kx@csp?^D9<((HN?o3YF{VC~aQL&_H z@?j&%=^Le2J(SO2n6^T2JBNummu3LR(o%n|Y7xDE)u(b6r!jE-bKp9j^L;{fZk19D%T1?+qpV;s2I8h>RR;N<9G z&{*(@;UKr#frokwjEj_5WZE@~4$Vq=!Ism&Iqv{lRsvhqg46{MIH#1eRU|k~U&Q3G zAb5-4`n+C_-zf}@YhBzf9_GuDypkt#CaK`})p>V9Zm)VP_IgU8xVe9`qC~?(@w$V8 zEen(C0{c|91vr~aZhaUKKBHK-Oj^h^P?uj;Prq-;n$O>IrPwq5kFj$^fA<%9XLh9L z%o!i$nzji#CwzOBRWLr}zrr3R$;g$}%v`iU;M5YquwucYe-8O?8HC&tW%L!MSS{dt zcWh7B`L8n`D2PAgd$T6eph@e8zcM>Zo78btp#-*Ds)Fwpu>3IMYI-mu$ARI*nfAWd zlkY9gD#%UOJJ4`C^L52B77>dLKOW?MyXwPQ-gI!`Erx0D_bE2-pKLQxaf-CLq{9z! z5l6vgmrisox7{kl7AqO_baLR)hXJxzlhQ4dQzc4Txj1~bo7rXQR&RLd#$|pieS1^I zX|Z?P|E%_}JIyPx(APsWw@{Dk(sSiZGxH1vo>foyG7c!ed@wnJful;{XUqcD3DcRK za$;r{@Nu1~ux#M2QLtXOLDM2rnR{lR?V=rC4iPLDc|SP|c9%=0IWYVaYGB&e$}&ON zqw&iA4d)u}Dl_g=kImz){(Be$V?vBzt*l91Y!XYu6WLxZ&@;C= zX{XXK`H1VnUhX!NOIK1d*dmw(<~4HOXpj(6Y^idPN;|M%`XM`^hRGsP%3reHR6Srx zd%*71!0yL9>H9VF{ z^(pDbGD*?4od*LiH*OQZs=aohX$P2{vKO5hQ!&GgYiYoXO=pxL@PIj4urPXB_0^__Z!UL#vNl%R-aU zw4(L0V9{=QVeC2zveL?oGjsFwM2aS7SC#lfas-y2P>j} zZ{-awPQSS7P%q5X&T7$^7>b15`wfup#=1tLMO%C_I3tmVUVMt&TViZ`Bz|X`elb3L%fLANb zYaw%hi^c=PPDcTq2bbMe#P}SE@o3;%#izrt?Hz;IF@qbD0X7m-8YL2BC7T+LPGsEL z{_tAtpDBV1w<@0c6tv@5>hGldJ=5A_k0 zM1DDNxiAV8wRMD_dYEJyo8#~|X93HMgw(tR3!2%OSe6IR*nMWsGb@V*wyZ{lJ9DgN zH*(t?FtRG<{P)gAs%rsf!R9%}evEnNCo4Re$>px>-Xiki08>ZeCI4n#xr0~Nd0lDP zGLQG>!Rbvg9IDKB!gd5Mm{InO#o)NhGX|k)2jA`Ze9JFJvZ+xbWnZ&(#532Q;(4}P zWA7XlbQBa|6pTA9lKCiTW@sQ*5o5?q_tB&Ve9q*p|Gc{tH=vL`12c_7GUUB9vaq~LCn7ZTV+wDJ$ zM3@|za}xHdg`8npcShpfrQOdjaGegpa zwQf-kCcCblY3_5|CUsGRX`y4Yk}H?U%7B!EO&okO86^`A9AsqU5Yq7wP;d}tV~}9+ zXi#8a?lrNtO8B6#+>b*{C_vzVhKRDGVwaiLE`xF`A#JhVN%l7K==q*?PjDOW{ybbdhY>jM&CP z;Tt`5M>`fLw3a4cPult-fYDzpCgC6_ThhM?Q-Y;)6mGJLE39EF6ny}muJPx z5#+}?C84o}XG;K+uzf&7BZEg$L8Cx-Na|FhlNOVGHXc;CEZU(o$)YDheVXcVgRG5P z#bgTp9!*Z{xEL|L_wKQb#oH~PUG+8HYVuHIqKZHz)1=wC6V^?Z_z=~^8(`qbDlEHY zf-^UdN$5e=b(W^V2Y3X|MCWm@>AdX2H{&J)d-*@T>t~GqR|HN}={FJb6pS))3Seyc zx4890K~sQ>@~uCQ9T`~|8r_YzOB9{fEq76H`bL=qQWmhf|WlEIF7Y_J{ zux6x`%Ols*RxvRXBqnw&JY3TJWg#<1!XCTE;x?X3Ma;2(0*%l9 zm#})p%nLg9G<(*qi5HyuyBe66ExI+&&C>9~MP~Ml`x80-`R-P4;!y12-`Xg)=J#Qx z(>lRxHlCD`R=SyD&%HJ(d+kv%nL>e$PTL;evRy1GxJ<8hr8Fb&WQoHcJPdDaXqA}b z)W{j3#w;7qm0~ejk6Ym-lS;(dv_H!vmou!e_hnX52xF`N-U{HI&Vk7d1Kb@gjG3OxT zDn|!4n+c8_V#nq&>p1j$m0T=o%{s}%YE9_sS2KM}%mjrnZ4G zATEt*-kF}sZ7GcN9FDaJ-MN&k@j&FXV;jS?le2<7J}hPBNoW;}XmGIzXm{dq3Od~5 z!mIkVKe1~ef33k0*&_jLsa*%VYA+mOpO(pSYu@-u~@=6Na&kNf{H(P=Qk%K_irz9f_WOwGwOV35T4_p{@i&fuf)b4cZTGJJhBs= z-LL34I(bL^NcL6|-`UiY^jP=$q9uuYRLu)MKCLft?71hl<6_pAg-ms`88oIZX;hA2 zWN~M3)R;a+QiU<6X?Ek%v;a-Zn9OCy@t!(K$+W{`>Oa4r-WBKU z)7&^_nKlW;^iEOqGB-}X)+3N`pjqJnLtElEMt)y`hM5r#Osq#6)Le=ktml58XW5j} z$3J8FH#5~qjJzx6XMF1P*AS5Zab$V3xWFY|1xKDFhQ|vhPmPaI^4gzq*FU!~V2P6P zs=)k$Dee!W6t;f|Q}(>{wtiAhE;H{##%X#lIYJE%t6CLy-kG(KE1aRx_u)~=kg{im z6PguLjzK)#k>Dv5CsdSOY#_9QVd12?IjPK<+k}0&>wjF1PdDVc)f2W@N_sB$&LgJg zyZGGPLHcEof2YE%j*0_F|jGS1k+DmxgJm9%+I$Aw8WrA%CFuFd{eQ|PE>7-M&&P1IhG zNyq*%OfmFKE=iV4Tr6e1Y^9_fW0%j9vRhY<<%&+?VOsh?E6nHP%Yw;MrbypV6lu88 ztiW)v!_Ma-J-s{ZHH&L~b+) zE;C>YeE&@-bcc&%o*$c*j|0Pnsq45E0=Z6FIZ3#v?@~Ush(q-5E?1SX=5^nX?_bdz z#;VnEY<=U~M|m5Zm#Bz_GoE~EmUEC*e7ixDs9qyyNP^P#SvQWTF8a&q^Puc)+J-|d zQ;W6+-(pL!|7W_8mt|syWWYh*Mg>-7CPohR8wa^TXOOZjWR>-hRf%?F(J-hmTb35o*~JmaLw;i)QUaAMHgDS-aq8)mN;r8=Gef>;J~Ed-oRpK z`tqLjt_*euhH4%KHy^`>rwR+4R+L+}X}SCt+HJ}#dFnyCPRk{4kr|H49gbZKyZ^s% z4?ohy;pwq#wJ+}*cf3YI21?HiRNS^v@(~+{c>_~`0*BK@jYULDQ487ch0LU@F{{ z)SpqWze)Ljxj1`^a{PAXBz5t71>7?lIA>4b%x&k~*}(mxfcHfM-@OFBdkx%fWznet zjR%g3OxlH7gQoV_uDy;OmtJ)OO?fL-stqHtm8VcU2a?LB?zPKT9djR*u z1p+ZoIj?4LT>VkN?D&u4U1<8^0``(17Q@MWj|{4|2NW!Pz}~FDwKahEq5+4m0efo! zyQl*LbASm~K+R!uw`1iDN7QW-P24VRW)OM67_5*L+Q6FJP}_K5Lbm}&odQR2K=e*i zHu(Ss(F=^d4qS@`I%IE7^e(qAU|`Q&z^1a(!&;!!=t6dngt~4d?|FvKNi&rHPUpKR zg&@4tk2tAcn&0|rI~#uMV12@&D_l5MZH6o)0Sr5Ugp9-q*2 zfMdb~jtLB$oeZ3f3KQ+Gai$nB`hVis%D~h9VWKwEq|1}oRWduWHZX6EoUOfqSyGkx zqkCzHy0PxdjtL#5vPYu~-Eu5HDZRMBdv5}_+cM6X3|xB~x^6$m1AB2_uR%Q za@rHt1(8eRJTU@RBl+PZ+(D7o*&(!6aGc;6`SEoxw|Z3w$IC4}ulhRiR` zzZ$kD(rYHHWUy0YkS>^cF`TVYfa~7@22MBgl0yufvotxp8fxb{u&RCGIBmeyUck}u zVX?wavC0OHf(vX#2D7`iuxV7z*1DO!ms6IbQNSpe_p5kk;_;~CW(_t;b6q;QXKpB) zxqy4c4ZbTg=UlznwX>Dgf_2^o7bhP@ORt5JCsLif9|p55U|y2Id-g*??}y5FQ`64} za2KCs%u|YcnxaBZLPkEhURsh;K>qm@hVcGVY7?Al?Fa^WW5 z+Y9_}7R*`6A%1H@*VUQa+3l>J%OvNEq@FNUn{=3=?ywij1j!9Y7+3@t^geJrJg~fO zR}U+@Y>NPA{53|887*IChSok{@2l#~i{w~%fg|b>WAp{~TUTS33aow-%5~|$N{bz# zi&llm-0GjKCdA>8`S(V@ld_xPMn;2n)`$z7R~)z&X!7(u;OKs^x&6WB&I25+3bory zSwtMTwtnE~IKbZFu;f_Qnxcd?r*^GT-I=ZabB*=~ChY~2LXLN69Z?$9XPuawDK+<5)VdWCWsf%SUYWUU`UIBJe^!oeMrwD;*6%pPz{?Q( z?xmFE0%?;E9M=?9)Mxc=y}<7MfbokPv(*IVo(U_yN=~m2*jRU9LgR*BzXE2<3C!Lf zSnC~DZe!RqZ$isijTz5gRK5u1h;v}*j?_9Ho}dzLqTnuE?#jHib;5!N{AVA`JTig9 z{`clCg)OJKIZ`&X%}rnt7T`MgYfE7Dnjr73Il c6zu3%sxMB_Rf_-15 z@Vq&lX0({c@dxjO9XWb#dQn}Sor!Z#&z#G>g8S(${xlbZRp(|;2X`r79GI;FP?!ddTL8v zCO6;po^1-6^Bi_9VPG}ez_RK$qm=*~-}7l3s<*v7z&Fo4Y~E-YvU| z;XB8kw=6Qj1)S#He18pi4>YVcy0xb+qpn?mW1hpz83t_EPV*dDz#g!BZ}4pP9AWm> zAICBU_NjNy-g$Ft%!)vV zXx}(zjYP_&$aQZIaPJA2%fci6Vgc`+16^AtM3<#AswPN2OJ0BCXzeA{*a;IYH<&Q6 z6fkxt@LrqH!~Q((_Kx1yD|+l>=BK}IzS_WdB_N$CoMVGDM{B^A&CDE(&v(vC;JU8x z@5qh>j+F4YiU*8ds0rxBe&X5I+(+W?qeM-5T zwmybQa`nSg|93ICI-I_g(0A$w=P8XVdwt?=WN_*G@XFg9UhU&I_iFVbj>8i;stdSxG_dZ; zViQi_Sn9y_>))C4nHSFg`?J@1L$*fbaqA5oS_wD0X6)Xz@is5=w0rv|BZjPIr z$r~7J4Y>L@oG+NLMtSF&Ru2{i*?k&6nV#L=%DVT4mI9N{iyQsCC*E-H-~V&c*X`SG zB=Fu^!26B+Rvz=ZIIUAN9UlE#+rYVY0r&0$-1{2%bmT6n-Q~_UXY)*x65QHSgwrJAr$9!IWhJ96P#JHwkbQ zuDcYrfOD<^tF%DqzrE)(6P767Jbyp>0lVD=?te_m3z$?4n0V#vN*dT}6lD7U@^kT? zsAyXo8dYzQ$t!j5;rzF2i>$YApAenh&N)Ex$?iwtD}PfomubJXRU|KZ>AqiatuD(LNP*!botXVS?#eOVg?ZXdex zl~E99n4NYVK&?B4o#Tdnloa~fQW z1qB1x&oAJ8c7x+g-G+GrSC0DLjW)g7(7=9jPMoL4uAApVc`|oju01nFi|@^Wq~?db zF<0633f^VgEi&}Jr?_hq=LAN11Gd~BWy>$T(mKbj)pzpgzgOI)=MOACmR!wXRB(X( zx6JH~mJe9$Zaj)U-v93f&%Wa^0_-&bAE(PbtUGfdXWHGMR~-`jw$0V;?3p8xxZUAJ z18;R2M|K70f*Y?i-*Io`yQHv|`_i=o?gw65oG;onQF`q}=`AAC8v_{}-(58^WIwxM z`QANW=09KlMRVGf*ewkU&dfQz^K0jd3vYKaRGo?3+Iw+9>5T=vZ#Lv87<0%k@Ya*Q zCvSD{72iGfhJXAHj3E_ei1k(%uQ29G$Eo6SDfYbJQ?AIL0MV_~3wN z@P-9aN*wKP0yZCh*e5Cwx+h?vQ%jeil1s`JkK^6qcJe=adP0nXl->VN`ulHnsrSvb zDQ8p8tqD6D<(qS3OX=g2lcUY^99!8~Uu{nJ&%eqOxv}Wv%cVXu6JFlhx;k7xcCB9K zu8^lESeSSuR1!8Q=rnHTIn-sFZT6;tk^9N9*aJ^znnrJTsTES~nh@_)&nj%>(jmCO zzDh_p>B@{iMYl$-N-mKOqlZpyT!LIKCxjpFKf7h)=4HCE#?kfc7bOG|8XK86@akAJ zD3s;i-j=9tQShL#fj{I}&&ndsDSEN%*Zg|*v|GH`Cv?LM#m}J)bKQGi95^Ywyn3EV zE{BxQCC87tl9(9}7@c0r^i1ot&aNku z0@rOy>S5qWJCNu$#m&X=$#Ekk!z0Jfb1AbJ*@IaHFLie%ge>5oLn{@l`7k^kySieYu(a`G8_yJOboY5iv3th zKHqpefnT7ZfoH;uLp(2e=C-WnH?5rMZ+t9*Nrz2fL9>Y4ryqU`%uZ-?1b>#gRl+Vh z!R?mPvZuj{Ex+CfbE^Ca`>H=_N=)d>X-w=s8^U6wudJLsx8qmmig?vGD`!QO++bi) zF2uZ1)H<;3(ig|AD_Gs0GNcNee+$_Q4$fjGt zX0my=v$V@}cFP@({ACxI6;`oy$S+)H(3)|CeOdwwe?Vg*-(feaBZ9oy9nR91Rpui0m;`e;+%RCxv}jS5VN})rxiDeo$sTns z!7kMsi~`;#Vx~$JcB^$Q7F(QgMB!gUyG%&~Lj=n_rW2_R3?2-OTo;l)yjs#=mT6e2 zd5J-E&IDGM-bS881!hsZ4UL()3`~3-{~DNH7<9RRSlIW#@WfK1rxQM|O$>hPdB564 zpwX#|fg^A-%O9WA%YWXe3x!3ysEB;*chAw4WL?C0qbStDw`;LFlT3oS9p|Zxj+K%- zW@H`xRK&eH$~0{84E33A2N$~uESmRK=ZM0a%xm^<6xVV%N66lJ)#|9@EPZiDtNN=h zc4w1A0?`W02@+-U?EanD`wo zF!EXjG;&4o7jio^h&6^^(7(3QJb@+IiJgIgfhU1U^1=ajJA-Ea1O;Zv1q}>x9~gMn zg$bQ&IHJhc*fCK`MR7)!hx0LZL3^j^N_wJK|G738T>f?8VTkG4m$jdQO~Y=!7u=|I zVU1Sg>e(H>>U>6C!5bfj{BjSO z#CLsQmiT>u-T4C}f5Cxfi8~f-vx-h|$1w78XS3jK0Hy6 zPhkyibJmqv*zx!oqabsmvkaFJ%j8cIN^fW8c_&s0&R{yoZel7r`~LU46F-DxJNI7o z$tz7~3uTtM#?Y=d+sYyTht>u0Pg83@R>w0hfjY zanhg1!5nvR0rSnX?>07ixX8I|ZxdYdaDz-(Y`RMDjHb>-e2G5}$oG9Ym$jr>_Q8sy zGMkN$yPXjf>Tht_*!z~l!y-xW-TWD#zfw_=To+I=G(JsiIRdH7A-R01aecg znNHU@Lc-b}}*m%4sD*k!VGdu`UQyCIRf*}Vatm6@U1UyL-=|GmCC zfA>S)5{a0H9cdQ+CY~=nvkzao8?Y)!bD#F6*jrMw-#N>lXka&*aEagjl}{Q|_p9LL z3wWy%&L~959&ylFBj~rFRDN4ZcUA|p+@lJ&^^RU$mQ7c#JAW@VlKaQlvzXcWrMJJ^ z<%x|v=^BrvgUXMY?+YwVPqBS{q@l`Sk|57g@!dBc*}YTTrM~i@(i59mQRk;z7H{g- zSTNyJ+-HFe(fL32>ht|n^z^KI)nWN-g=c)#=f;PnDH_VZPA#arvfr_1b@pA!gE#67 z_N@OB&+u40`TDnQwQ4Ir#dOtNT-veQ`9ogs3I_J9eeX)!1KxkD+`cU2^=|+FjQ=VQ zY71msn7`qnaSj8!Yxa8k3J?Se%ol=(NDeQ;~l(uKyve;usp2v=3%||?Qv=#}p zFPO1~JLLpNjndv7tr;)cax3)jeB=@4;W;;B(%ByqIWPBIb(GD%#j`(PO~8xp00oxL zBRUlq*d8kH3*=y}n&r9ZvuT3Qz62gOwGgWfj}N_7W?Q+~?#=E)D<0crlsF}Ncp3DV zY01IYQq_0n z$GmK_^WMW%vwoSD$GjVDt;QCbOw&4d@9~0-VF9Kh4xf~ebZ&m#>ud%sBBZw**2TUEvdyV zdC5M_m}3bioOGWYQVa1)+;QyFwJ!vE-`nl z@_V19^6O{+0qIq#3seqoUgG4l$9;`Myx@t&8@zX$%-*tc%c;{AowH56UbpdV+3}^v zAx%m6-tN3vmUUO6||RBuua${ z5ya8zF2MSDbpW^Qu0-Zzb6Qvu6V5JecGEREmc+xB!f-KxV}JYvH}@6Cw3T<~YMU1` z^Lue_))YSXrLa+=fN|{^ueDPQEpyIUY&hJ{&^dRJ{F;MO@!sBt4j*T|X`K5(-qV_k z%Vfu9MlLQ!u8%QCnr#d^SKGKnt zA*D%s`QLcB8MH86WII2Ho!Mxg(wYly0{f;P3Am!O&7HyKz-lx1ABUEh1PYiQOaHS^ z?}%@rL|azLsZZ4xC6-R{a_sOfv{o$Um-%<};^QZQYxk@(t#m0xer<*BFm)D$)xBJUAAbXuj%! zPTs8{9&?Yy&p5VZ&9TIQmWHb#pR2vzOkkFYVe+f|=-bCGV@a*1J)$&953bocIPX^GT->QKlr zIb%iJzg2A+2U@cZv_=a=Eu5?6D|{<+?X@gXR^Q~Pf6K0N|NPQjaF?f3h5ddqdu2x3 zgUhZLpB!}(HA|2-+qg75MLU{rYsj51`{Em}Y~32-6MHec=3?$-=3JBQjX%!j*{swo z=IHE=2wob|?RvH6YDCrSSr!3P{~4dQ+;lanqO*FB@$3kJTnC5v+6nfD+42I8bVQ%n z9C&`y%{vPMPPEQ(>3Mu^)`!z29_<%G?=DD)opzUP_ZPOCp%bz^y5`=!x!~)~j2p45 z7D>b;uxU(ctNzm_Vr;kgvnBW2t#Kw-lAJlylCNYJx761Y%UMt z{O)ejJ*@a~!;~$>dW#x6g8oK#9&ME2y<OHl%j4J1?RS^?F`N3X zgu)vSuH1c)_v3-kw+F)O1o8?5?nSn^SG2j^h?-v=urBmoMQ+QpyYU_wQ(t^lOY~@w zKXxTUn(fuvxTPvdKD-f%YB%0r-DItF*#E2BmjhE87#I|PvTztNFfiyaFfcGkForO; z^Kx?w^YTdYa)}7>NelBziF5Gr3G(wx3h@a`@(c3|iiik{NQ;WfNQ#PyNJ@)J$x2C! zNs93c@u*2~Dv63JONlCrNh?cBYstuHNs5_BNgGITsVK^7E6ZxBh?}ZQ8LM;4$|%W7 zt4T{)DN8G9ODk*3YO6}Bn#rhY%Nxll>d32^s3uOu+YMX1Q8yo6c8|zw`sB3DN80#2Y=$M!q8(Etg+gO>}m{{9cTbi0!=^EO| zDT;fVN%|Oyy2+}B$*LB}s<~QdJDD20*qVCTn!4Cnd)ZpMm{|o|+lE`2`{`@u>FGsj zN{6}$7Wha+xM{jMn0h&xxjNeTI9vO9S_iq>26@?qyPF4j8D)5Aq#H2)~ zlxL@=mS?3^7iVN-l~it0xt$o)&v#miMWZ0cY12 zTw74^_F-gJNoQF_M_KLUjDq%ke%@dU@};N7t_1I(h2cq1s{|#lIUXAyiR&lmRbAY1vD>{r z)~=4{=F%l6C&YOz`?6`N_i0`I(FnoRvpC&(F7O&XTkW(Fj~**Pr*$ z=!>Vndcb14R;$og7w7rUaX7i_Yg*Wfn1ve}ou+AB()F90bnn{&a>`SdjTxx+Uf1rfnyuZHq6P_KP%>k2sl-J0gpPw3 z-Pd$7Udq;(c;uF+>dbAd-a2b5qzhC(Zi~)+>KQJbZ+=w$PL$ul5E(x^$L}&Fz82+I zqg>{;rxXP&6MgFc#&qJK)7cn8K+h?&75nP2KDK&rD{7U;zJXMmd!B*IRUd93ls!{GBhM+*e-e4CR1lIOJZ^27gsl>l`jOdWzUvQ z%-T5hi&CD>q3>bdi+8cG`o!N7xu55Kcgh{%f5#8DgapM~rlw51WovnNU5v+~RuxCd zl`_#=yslpfd{F8>RoE@ha>{z=i&I_+#|C(-#N4>m{wy}`LgDqe;w>jx#hq9_Q&X!Mn^)6;w16Jl0bt_&5hrbJoS!N%f=QdDgZY_tLM+{JwZ`nLsgz;BnzeUuyDpS?aaHxgp=55m! z5ibe0y)zwC$`*^%vb1H$EX{bdTswEmQSsvaI~Ix0J?3uyuY2d+j@#aQc-Jb|I9Dy5 zyvED*=-hy_K{r*Vy5=#;Zcz6Bw|cW5<44t#J7(QciMqhFu=T~Qg{ooU%Y(e%MyDjc z7MrQIa;AL4@4cMrR~iL7n)xgBmN_LW8g#t>@YC%@f<(sRwPo!Ss#G={)Sg=Mp-X=s z+a#}o-6k8CtF2PG-cw{*C{u9o;+5ow8IF=`SNH4RtUSHbY3jaUPt$uX>lWSj-!$?2 z=C5}{Z#$lzdPm*l(-#)qsGD518$|d0Uh-0}{vzD0@nfh{MN=!&xHEo7n>RHCm zlL9`zNVVxS(WsOUlFrm9T{LmVG3A2Wx3khdN%@>@aoh58cCUt*-IfifbpKhuaw_(F zy8r7*pQ7g%`_9QdpRA;1=xu9xZnDQVd##W8^}n+3{6BEmc*6^x)R>E05dmsT?p`^b z?{Gz|E^T^S{z0u~CMALQLZ6xVDlRbAb3No}Pj?dWV+rbun%6Ps`J|@WtVG`5A)YBq zPWbQ4;Lbn&@P5{k#gBN7yFWe170{xxKf%zu#CxIA^c9=u#_Armv8tT(Zwte`#cMWs zeD78*iMr{vjpL;L*KDzk^ER>gd$s$dbawl7wJpo7?Y7U`bvOQ>QH#}0PyceBd(9uV z9v5~x(VzJ_;Iq=#R`F#TJY7@HW;=)qc}#jT!DL~?-5{n_Wv^be?XkS`a8j_&ECvD3 zGyjUN34Jl2JFD^Hd`~Iaz^ap8VR3>sYh_;ipVV?-jmQ@phYF^bF_OmWIV^eyRTlZ} zOIfz~>2sf48S$Mv7~BtvoL~>VEWGE#4E-mrPi7srUii_^!-+33MM=RZ(48gB_QsQA ziQ5$Ay%l~66@KD1H+yx_yRGcx(+#Jz6<>P_Hyk+sozrC6y^k^hJ2uR&IrK^6#zD!= zvr<)LHKu43oSNj^YoZsC>nP+qfkhxn@}f#(qCnjXPw^!M%1;G8`&8WIub(EiVE+Zp z*bc`DO&>T8W%?OA8O-JIm1gzeugpl8t+XWPk@7juLv3>m6z2 z2KI`kb-ZV?IHK<;@$A?1&S+oeX7X`4Z+6vWffKIF8wAMvu26N3Wr|nT`SWB zJ%#T&$-X=A(E0|WU0eZEVW&2WE=S`IdkMqB;t9+$e-zlQIyM%66k!&>@ZgyF0>_@x z1Qy+tL{0~@CO$jR$W{ZhtHz;Zy$g*_A&aFF1(-wbZ4xsoyz2Ad7XPIOM^YItu5_^Q zc$p-{%s6e|l(_GQnEy#WyX`)^YKf!yQT=;tZ09-}gAeSQc(y)2VzSNWv-ux43i|jx zU6p9~JZU#g2)xpF`WU+*+@~;c6uNf8|wQJDh%XrWvwq*hH zQnRlU`*oU?wnVXd<~gvY*fw(JFt8{XIEfvbz-rvX$Q^m2NzlQdk;&zOp+JHIE0+OB zOu=FPy*UfQeM;x+JyCDUUBG;K#L^GHpb}~|q%JT_;b(ZNk#2b=ov)Z(r-6}0 z;{c1zhsK2Y4|!%}9954J{Ng?5AkX1%4eVM8jNCf6`Okf8mYAj>t5NXAZ6ouWSratl z^y_c?m89^nG_gBX&x~ymYJ9)z=3RZ>N=E*vwZDDe{}%Kp-RwG_RsH8Lo(~sJu@wHB zJat0TWtm^9QPV=UZfF%gUmy`2!*1@j^_Xn{%PDoslY$Zr&tm-;rn@iTFxlX&dq}V^ zXNsLnZ-Eng#JvVqmjhq9J8D@Z3f`!!pYv3p;U`nhxdt{3X%<`i!|Dc&oezcWUu^NP zQ#)qxhRfgq6YmU%&AT1jbndjQ576kUee;ib-Mi*;{Ye@+b2&<8$ew6meRM5Kjy3f1 zG`neUdk#t8kydH@<;-=jSWfH6oCW36Tvm5fe5<$FsIt~1toylm$kHUuWl0yFGU**) zat~wCz0NTAd(kX)MrH*@t_78nKZ=9|7$rIyWilFNW>hNNV34`MvN(ZvX*%!C4^1~W z@c1<_iWL-dJ20%9Ub*pm(fdP8+TkocQ!B+h7)2H^dIzu_VJ~rTubI}s>2RHE%LMKP z4XoY{IpPJ3q6KW*FYu;6(7t3Swvbi4K~?Yg743o`z2Zyanr1Ba3}u{5HWf?UO}4tf z6ElR)K>fu>LY5Q#T^MyUrZ)Who*bk7joDry+0ZSE?*`NAvXl$Q z8JIUPD9`AesnIfX2ZPWC2I&80oJI55f=w6ryFPHSMxXy9~P&N-`qGoyg> zFnjBk51f%37)2WxT^rbUG;r4z)FwR?s!GjGdlV}yqW5@_c(D+x(+6h910AmqF&jLv zejJweCS1R|L_;XCU36i4DSP?X>1Bb-(sLX`tr`N(i)NWWYtXyEXsA#rwY*S9q1e)$ zVO9jgEO7?&j?P)_ohp*m)*6f&os*VzG)i+c&0E2+@%z6{m7R^u43p>2Xq>?@*`TwN zO`w|b0)w~#`X@IBnXQ&zw{<3!P>=T&?KIo@Gu!IF&Oo(#eh7`-2` z&o5we3$O8uVB6QmnpVJhe0t562kZ(D7$prDlO}LqP2kA>*qT$|{IyW6YGLZ63nqD+ zSTiKtb9d+&PUzQhU`ja_)E8>=HONW*iDa5`d4W1-lEchz>TD?jYzL2Jdl#@yVDI!= zULrFgM=&90W_k{DK%-hkv&Qj>`r(Z%4b9947^dy$6pLU`@|>i>UaeBuIsL?>={qJb z&*(CfkN$%*c5Vaz)5m$<$Hl zsL3RUxY|a&wm@d~x)joY%#DSU} zVT@D48CVqtQvvQwgwdjl4YA+_v(U`RUdzEq4B)eCO7jkqR>6~oD*){z| zm$bxWCCf>NUUsn@V37F0ex_l{&4ekp4S2U%@GfQNJM)2E!eQRylM(y|%XmL9m_BDX z>BcBMfwgVHyw{c7cOLNWaNw93S)XVyfBFJW-x=In405Cu*tS35{eQvp-{*@iu?N^P zjN~&GrQY5qE|C(^FfqyQ0&~mEetid~@AR! z*1(p2foq)?_d2f<4|XQjf=Wa8S=PUbEIq34oSdz8qRY^{c*F7T4dtEoRcqFN@3gX9 z;%U`o8aY|pap?%m?mOa+WS4qO*!F1=mAdqs1uz6I|Y19rV#bNDwf=tZURI4pZ8 zX~RE(L3jg;mqhRU2Hv*^__keOH{ZgN=)mTY(K}-T=gtB)c?PD)2JU+bTzgZrQWmo2 z7p6wE`P6E%1Qxe7CdM^)O7P8^@%s7|Uck1_fqVCcf81L> za2a@W@0w7PK7n;dTP5p(iMrQUTT4}&ecv41>Arhq^KQxNwZ|Lnveq1bRW*M_@?z>%q+lg>{s`dUkr=}%nc5_Hw$<#JIuY8)qQ;duX|=y)GZE!4Ghn# z)+KUnS!d3W{GGwog3YgjE#d>`j)L{A4_J$@X8L?!b(qF_yq$H%gqp+CSR1RjL&}BM ze73EW)MMdNuk+j($Iilfh&eQQ#{T4~^+rxVm4b7Q>;HYkD)6x^se#>AjV-N#bKL^& z-3*%zUUO~x&86$jomRl=70x2)z_50D_2H;J5t{RYJG%^SZnmAhC2;4Yt67sz-R}0- z)wTHF3kFxI$p=?1v99RaXEE<&8S~Z;P1g&S-u>02_oB(Ya_Qv{Ja!JV3x01_i%iq| zRi*f0od^R{T>)EF72D2&^}P-vb%DCymDzTE0vHz0KR8Vg!Ogg(nk6nma#~`iuBLg3UV7zjCu|#mL`l?mkg5R!ZubaTV zwR`vW-&{Macd!4jiJ80R*fYl2;afD9&z7&)TO`Bof7!dYa&xbJz_sfR z*Diz2+kT&LbEsra$O*PtY_n=h1m~8Bnd^+MGn{y{*!T6(8{Cu4(kI=ReRP>bb@Cnt zTh6`KCnrT%?q$1BB>Ft_% zogr!i$JP&pmoj-TT{xC=o5MbUQCxu8zk&JGwQBJXdW)o)^aB`KHZX2neW3d_@6LoU zF4Mq<+IcS~a9*t7b}wL^oWmXT%(a}8Rq$-zi7R1ZPf8cn#L1myicsFAI3=y;WZrL~ zTXPF9B?PcVe%Q2iHMhM^$?Y4CBo%huR8k3_}FS-6NXVUx|XHUI4t6^DuE{4J4I)g6*NB;#L zh407m&TPA+z-4xULE-^(^8t>Y1kjaA!VEL8;7Wak?Q~yZEG2RUSE+(v^7bC`gdzy8N;f(f){VETd-crq|HFa~a5FT21bae$@g0MFtH% zIs2lA$(uVo-~r2w00#v&_l5@-cRb+j_W@l^%3TqC)grWf`77o7a(ulFcD z!cig9euJKA!K+z+9n+4@SfFLLaMq(GQvV9C?F!7;z4HL~+X?^f6|k{){+=GmX7Ogz z+5*;}nOlzKvgbTtE_Yy;_`q@@lb0b-o0*YI;*H5{w^z<=j%Q|E-FaaFV>`Eu#g>MG zhfduRrWOVg43ij{`7CDH{A5&dXJ-;ExX{U%cz{VhrbVYAQ-EhR>b{S1)yiYx~SFlC@M?C8ehyJ8whdBG#_S>e2HYWZmD>i<@%bZgNap2r=tF`pr-Xw zxw}7tv|=pMWUs}oU|twy^DZCJ?X z;FhQ|!HwbMf<`^Lm9FfJ87q>fq&}OGxtxP1z@c%DK~B?@oJ}*+j_0hIm?q+|Ys11; z9xV$aT@}k0O(N^7zGNMj%N0}+$n;p)z`!I{q0q>AWCx3|ug=Y<(+t<0;NGxXV|Grb z&bkM!;^8Jg|E1UpvBL`E5d|bC8 z+sz1phi3C-Ui6y#+rB-!<4K3|)t&kWEJS(q4+uP-7Q1S*o^||%pXxUH+cMVd{&>XV z@t)WW$HN@_QXLG=OkNC2W~O;kc+ES@$TVc1lakJT}WwOX2_nr%H!HlT^M6 zt$5`gZUeL(t;dKyTot|~*$7fFUUm_x1E4bgjF;+Bt-DB)N zUxJB4vF!t6`}LHHL^HvH+acxw_O5y6+xoRbEdwKM4~xEvD|FppvDj0BZgllFIgIw|LwxJ*Q7Tg?GRHX#PZZsQFd+0$e4j#YC0 zogL9sKj-)fS8JojHt#u#+|oB1lKx42-m*M4&8X=WlQd7O#>JLnCT4|G1Qb*?ufJIk zv^lVm%jD@~zprXMd9Oom`*~>XtZi?-Yk7Ee+{Lc2Cx`NOui##{nF|$L zL>ZYxRx?Q?I4Lbr6i{+h0!@xoh!P zbsS(2yWYTUkSMrr^Fdy>gFM+e%>7Fgj%s8n9@8&Lh(ENMk@wlvsi_7|?un6YdxDO4 zaQ|o!ao{@2o3mhk&V)rwOad7Zdz5+|Qxq7i9;)~_Uux7_vAU{ri<@kD$ni;kSM;1N zND8k|vDKK#BVeC!fN9dMl^h}pb?FYxYB4Lje@|pg51!E^xT&Be=uwmSj01;MPZb>C z^l1>-zT()tA78diX5gAJLvdM#g>t9cT(@RvmO}ys8~!!Ro?u|p32@AH+hHamx18_T z#>01dHs8pv``lV#!CZP_vRwF;vphR~ofBWSp;f1ZooD_9R{gkza)u9z8IC3~a4&M` zaTs$vNcUo_9<=Y|>JoikBw@JhiMdoQI9>`vD4?4y6`dhI6*GW2FT{hjU0u!#V zYMfF3Ij2E`dj+S@j7ef`2F#Mj653{5)e)bmkSckMp=r}BQU64Ne&JhHl8%|j43CuMm`UZ`)C)5ok-~L|951i=v3Jx zyQ#Yv`eG6oFJ(1wb6m7?c=3?Gf#ZuQ%S1-CC+Ecc4lpRZa_!=LF;nVcMjM0H0Tyi| zCz~1M1)S%mwXsDeIbB_StVQ)gL#y!&#dK)_2G%Wn9Wk$s z4*3`Ji0JKUH4ET1IjF`f;5YfrI@92sv-5mA3zMZq#joY+g@yKnKV_ACEz488LUK*Q zMXp(MFXg{~qqE-bVpr^srnU$MN5QTQ$6GX~?o-ZeU}qET%{FmbDZ0Q>vFb<%o5jCG z$@#@jY`+BDS_2+%Uz^m%ruBd&EyG#LXF)^Xl8fy;9qUR>7BuiO1n!aZWEK$N;Pl~T zxIB@eN#wjm*VkjwdpjHrlP(*)SaMN2M6NT5E%2rgOT-5jYXws#fd`%}Y9AVPW-R1M z>sY0;(yre7O#gvIgQjcQe;K%Vl&;6^=(Z13)f2EzP*OXWAToQyYQEo%R~cq|O=R40 zc*^aDZCTM9{WbfJYVqy+&G&f0#+=6oa+R44Iuu_^*vOt{jj?Dp>^zhJI9IBDVwh6xkd0$(&pZ(uDvq06&^Sud-Ne?`M}=1JTN z3_8yLY#CQDyJ)ehFJN-C=*rm8$X&rS`_x>+Y9_V{%{dGzOPLsEGBmjch=1;q$egh= z=Yr9fM}i9<*;%WwN}mv#7|>|xARqLi>3ahMM*;(rMnjPFhE6rQbbW^KB z2MY&d^k@4Rj%L+_1-cVv#S2Yg_L*`{!7QnoX?>s?@2|$Ru1490#>`c`w-5C0Ql6Tb zyfFLmZvGgZf?o&ZnVF3`)Mre8tMa}^e` zK47pZSQPbPC)W!G?Tu|(yBkwEx-Abhs82AKp0PJAK~_AWLHa>g=8R^R6AhOqx4A{I z1b*ns_|Yiz?+0`Flr{%})(4d>QZv?@x@=+H(ZI;Tz_ftTT7WgJqpkg*%*@q?WjCr! zoH#Ehf=y#V!({Jnrb-qstyW8qRsO%0Fg2{zmS}b>*}rPF)oSCWNbQDc2@O758thM) z+6XX*R!Ie@9MEfE(ktk>D8YLv;6RGE>o#U4!Osf&lN!YZbnY}XY(41pC}Pdi2M+5} z`tp7^1;6Ywc)=Ii%$j847Mroh!gA>h%VvL$JqL?fPke4*x}lu&rSn%(Ly{usLiR_A zZCo?9TP>uG3UXW;yi!BCsUsrG}}ZA0se<}C#$_DvL=`LWU@Z38pYiH6CC z1wNdV^{VRNmf+-2nDOlcgY1Qw->Oee7irF%!?5DON%jNhxDGT&UD&$3k;y+t;k1Ux zSqF#nJETIM7`VTj6_&##*P*sq(~zB`k$uNDzeedN7fv1i)ns|&y#9+*lAq7(-(Wg8 zalVkrvG4}gtzTL{Z&-MEVc+KkibsAj@UK{Zd6Qe3>NqJjTMgLDLw`Co>chxa_mX}DU`!1bazrr98V@`iaUc{WNPJ!`>x!Gd@8 z3xlg2Q?50fN}J3mrqCGaVra9$|F??h;~$344ut%B;;-;}TPYL2;e|$Ri{|hZd=V!$ zhDQecuAKU6!NRZ;OacIx5-_#zkR@!xIQQ@6LVrIF9**X0)B^YK#%(2x{I`z0gJ)!0A z4St1{t*$GY<$qi|@pRI)BNG;Gm^0y{pvI;)w**c$2Zrmdjao^v@-K}v4o=chWcZmQ zf69AC`N6vr1GG9BwsOy4Fj&DFqrm*WW>)Mbo&cLu7jHC~IQ;XP7JFdbN5<{dM`!Y` zS+yt9KXFaAwa#Cgsq)+xHstUaJZs8-#H7!_BvZh+crR19M0~hGd_=?JFSg7dzB2GH zXvlK$&5_w96La`o(*xba2H%wdBFrhA3v3*+X0rT9_PZ4CH_6<+Z=%G>4UY3!$wZE-pC?Ob#1!_q3*EuqgjvQvYyd&jX&1H<+9eXN}*Zh00Cfm7;zq(e^ zj1AUr84NZkH-(W~l=lBGpFEg0&ZMw&Z;6OW3{SR_mK}=dZo^ z#~jvhIjCWtL<5tiNA8Q2lc&x5TE$jAms#dVhjUNF4})m7gr=lbjHiQFl^l^Rd@*s| zfjR1q?9nTv<3F@UCY*HkY0!(D$5g=j z^$%G~wxngtnYq0*NO|3Ot%ETm>E6u+M{jF9FJ6{i5-0k<3MlmrboLsmvY%pW{mBRGs~-^-AQN*>`Vv`LYJAf0M89*dg>~lR-y%zV~A(k4t?Wjbbkv ztn>I!Hpd?Go55i*L*`7xb5E`2t?|#g1&`lb+&puS%ocBky7^u6%sr+xT$;ABMJ|`^ z+6#er23|oXH|>YH1{$3+*}Nawee7xrN@&sV2xoKfyC?TRPnSblf<-TA<-$pFk6$-1 z-6+zZ->Tj5iFreVOHRM$bXK1Xem;)|p%dwEzkPb^miO-7yGV{{lM|TD4x~9<-H=p0 zWqR$Y8LbX0YZwLp2`Cn^@mZ&@dG+)Z-;?(YyvxpTEenYh)m`!2>tmyA2BX*m2GI#N z8tp}ks=s*DXF5;IdG7e)+uP1X%_S`xz28SQ^j)7caYe>=*C#&hEA5${Tzs~!CGE+y z%?XptUe?`xGV!ON$L*9f0ak~GCb0?OQ)?KSTN_NyGG5xNJJ&gOy~oq6??w9Mt(+4Y zy4tvwSFy%VXw{nEFl}a5d_>iIH|FpHR__hpRO-KJXEbTdXf~6`kN3K{L}#0Pg@}2( znnk(UEe%64CFZ$vhFb9#cYEC= z#c3z?#O(8Yv)?Q}cgZD5u!VC0E#mKV4CSgHSc<@A~>jSLPSG~<7*VsG5);4*E?H`V=~ zTr(K5!#{0$w91O@ep+~u$;PxCJFiO?AzO>5T+MJ$$jHCGVoC&0x}b)kkVd-1=GXb3 z&hNQwmizhl{>&b$gy#1g{JIPs{8kwi1&xPU+4$ zqfof$2nVwp-v^J6j;uUV4jDHdEM#nKXH~OWkr|wFoRLdO>xIR}6wmoi%px0&y`r`) z@m!v}ds^--)8&E7{kEFr&N9usrDkI+5XOE-Ne-GQ>t^$o+E-4xhG^Q`kf&^UM35KJDW0Na~!LHKBoXpGkX6TphsreX`-Y2eK+@=2h z(bDeX7w5K@tURo0$tz`%bfMtkA?JzjUAZE*oOr;<%+9alkTHMbUUxP@!*k-9^-r4E z#EohUoQ|FnO_un`$e|$O;J|Ftx}brX+u*sw4@m&Cr?j_MmaDZ6`qlMQBpkD zLx?rgiR0s8feNOJ&HOfd*koN#dBP+7s?$WhU7@p#@YMs*p9(~XWYMN_QW=M=}p z{7Ne-z4DNm&+16Qh4TBQ2M)4|h8$pE;lB}*!SObe#gUoUAVJWBFUsUntw;p}lc-xy zA)DHQgioAm={uX_>ed+j z>r(-vc!WXY^#6YrH;Y^NOuMcVp2El~9cyv%h**@3qNj9b&o>u=6}FAf`PUqMkiI}? z&jBWGnH$q?=XEi$-&wMYb9qeRX+d?_lB1;?S;YgYz1RiJjs!55-xEE;cwKzUuLZ05 zx0J7_mTGmGUn7*ra*;vQC*YuDdI)d^IIzb)knS~!cw6CX8t7^qYWmKPHPq}@%H&EjD=Svf;BZ#W4pz>H zgvN=i&rKtyZkDh(P!YQ85hJfcOyEuDl}SDx%FP~1oZ>M&^8dQ^!zB*33KyF!WS!%? zMNvwil&Mf^-s+#7%{+!@wr%EUb2`wpsQ2G8>B7UVzg!Eyx^FkR9PRUpRlK<5BQt+i zK=T`hd_~)|%k5K#@0-f2~r%>X^>cN>3&#u&Ncb=p-y^UVFghuEE4kpN@c* z=_d}cJvzjorozNk@6p2FpfqLP4;JU^yTrC@l*sqQ}(TX7a#eUvDEFfP6-EQm4+RB zt{rVAD?6AMEIm6(Ac4_dhJjh@0*k{HrsAs_vSEH9y`l?K-MwB;>|ZCum&u@1wyD~I zNzP&EZP5iBZ9C^Sa!pV3qo~o{#Z*pkkJ2xkv1qXT3cN|gm@^Feu zNaE>=WHM%GTwJH8z+iuJYJgJEByQ6aQI?vWHB}tKTo)GbEzD@Mwl%jZzxr9zJ?5^- zxlR5KEIQ2pw!vWK0lNrzFP2MNa(vWe;a+cpz>vpS#Np)Yk^)!|y zhb*2Z#M6AhIcVo;#)!CvVB*Jm>f;zubLx9gDdH$2_jLhE8*$+5y zL@#J9)%v%|tKUIpYIR(zpz(qOtSrwCx4R_hvB>uK3cI--iFtW!(~V1Sj3zX?-BFT? zi*TASUwp6jnw2RzQYnhJFSOkGRL0>iap-$omGR9NZ&;N$-k-O5sySb+x6b~kyrwg! z?8Kn#MN4v`Hw76unOS#u!mWiXTkLDm@mfH{Zvsl%q79t$z$Naz<6Srhepe}J{t+9mCH`*sTO%h z6*h$GefZ2{^nk@_%A!(5Kkl$48_zFm*-?7A;s~#hL*s@WRZNDG{Z5~gI(e^j9o=-{ zuK0G7#!gN*L#Yk_9N2gQyQ3Q%xg&o>$hB@~J#*&agvFjK=ay72m)F>oW~bo96T{p* z_iEc|*9V{SdxQ%-|6W;swUO`NlHVpTa~kXYFE>eSV`y1YcAQz>pjMmvQPYirg!5}2 zB{BZtxwqolsRH3M=hvz+F!61=D!TIltM#piy+%5ToND<^!ZQREqVDht*I67AnUTZ|G zR&a6b%{=p2%X4E_RLoJcIp<6!FbQ)UU~OTovhiSEz|t*yh9!W3Q{i^33{THWSq_Uc zH+>zrQXAMM4(VQLl5k<;_;=ujD8tNIDaX=zf;^Wt&Q%B!xNt@wAd=zWfk$t9rWHm` z_q-?LUP8v9h3ru=(H(vY>G zq9I1)K<~}gG)c!JGE*iOEZ8!)@t>=wIOJ@X_w+$Pn!eJ|n9XLRH-3li87 z^5|d#_n9`XmIDi4JSdbozT`_tk%9ZH)F8pKMlp$p&q^Jy*zl&aNbG2x)( z9LA5lN3y^8tlGdeUzdgbgu{Z;tF;|R7(@=R&vDdSqNu+i(fSWF2TSBf+k-*|jS>wF zu5}kAXE1WKG%DP1RA6XQU^%43a%k_F6b&0DjV>pRNlv;qoOBJE{pT&xOgg8tt2Jze zlIfdx6A4vQhh~4HX5AZ2T3*bGEr*O!T6YN~^cy&fb1*O`Ffgk)thRCBuxYqaA}Hyo zz&gjF@^9Bcjn2t2Z68w~B$8lkm<6<%~DRU0mUuaUCa`0g3 z0fw(U>=_H!SO|pJsQP_Y{A%@x1qLKZCx@JInkVmZX6-HJ82PQp+YL*b+mImgSGy01flTr@6(Q#n3 z>5pD?kWGhC=!deL=1H9$Oi~LPwM#PM9vt2F&(ldEwe9{yk-V!X=ic#rcrk(JfJ67Y zGnPe?iyRmV6pk0{Q7&o-QnPI@X$abH>nmh+@ZQ{>^d8AMFB=3q4%$CCq`sp`;t%Tw z85W5)*XkID&_Z9q50M@{iIo~h*j1#CrykP#z@-1=&^y<}kFE<%(-?K%FsWrQi8VAR zG$d&hDJknPeGOd{^Yx(YndnlM=VDJ7gs*f6UvOZ1;lQ?~fv@BMPYFY7CZM=j4g;JS-l=!*+#1gn>!ZM%_5%^0ml@0}N9dtr{AOZZdmiu+3rMjBwx`8I z8h0U;b&Er+-HMqzQd4U*PI#~>*fNUb@Jn~JX?h;|mvQ4H|A7VOn~w2aIlyP2#(iZ< zPtnYx9}9{&MEjlEQ`;s@co!?KvQWHZ%7h|D*@DK5s3)#chvGEcxTnnf=IFpW;Q(s^ zFZ+fD?u1ac8NIb3Ev`4xv=2C`y>Oa+iHYM&+N`XDFESe?pB&QL(WLglmyO4dJ)wa+ z;lR;**=s`%Fy+i;@Hx`#!@~OI0>d2{#yc|WKTL?5=#sGXdNW7Qfq(-%SDH*p8Y9+t zd0lA`J~4Hzhs0_I5f+crl@p#a|8bqFa%{Dc21`N%`<{pvJq^}`sKzB1*Wb#Ld)t(9 z;GtN_Y|p(tVgf;)Z?BoLHS)J4Gi=SUG*k0c?CROqf9Zhotk{3c{>v=%kQ7Z1;Lnx} z{akVI;KV1oCQ;gZUbC#Yw^HOFtA*Nii5Bh+4$KaB=Go3*ov8BprIXr{hYzwGMcfoc znHawC_C#<1!*2W`w%mRogJDSNlrNJP+H(lM45RFEFx(Cmv$U)0=b9Lx}VGKao3AcR7gg zOeiZj$Zx{PTfo3uz{alNQ8m?NqwfKUC5dVRI>$Zz^YijV+YU;;a5|#J&Ee2t*~;z7 z)6l}RF`$6^Lhk8pCLG%fx?OIt_&S^rn{_#&hsolGBU=Ci+l>R))_PgFN~|`}kDS+V z^vJ3o*KWk;FtDaHu&rUglY2lq*0Ir`k@to}%{>S2D-n%z9G7~w39{V0x2{v6pDiun zSjG;X&Bqev`P@vprNDilVPVDDt#a;+-f|kM25CwQ?fSCC9X`k_#_wR0k@skFYB*q@ zut8CRaY`BcbrT1E14k)_LkbMf1@3yw{y89T!=!wLfx+hptAz%m%q*Vs8&r23`X?E4 zkiBOCpNRa6qaXBI)~PKBu+*Hv{hDRniYVdx0nG*-Z3=}#?i*KH9yr=q$T@+@;!opI zn}hOsu`8QYSy)0M&h%Y7>mcjODB|HLQPJqd)Uzef;fL33HjXCs7AJ)#DwpwPBXF^MA2nIef*i|1*ibqB5HJLa|K!-H*d1>#rU zdS!olR_G?=#GXEfN1;*y;v?H7jUh~zy6Dtz-SX$3fvO-2D;*g{SWBS=6=Uq?4YTk(2 z|L^NjUl%5e4m~@MW}O9k+YTsu9F|)kp^)RsRQEJww;{8|`!`b@6nc_01r7&3I`l8F zStO%@Z9^ev1_PJD!N65Yh62sOPO7Fm7*nS^IR9?UYGvHEQEcNc!7Tolr*BM~t0E*I zaF72=k42w}s^F=fqM6JAMN0$^_7sHPLgQNoz1zXkb^lF#f zk-RfkOv16w_m;8vWXT1k`X@~V_LsJw2v~VipvC923IBl@{RX{rTP^yFISri5RGQfC zFc`Kl8~-aP?Bdj5iEvmmEsQOJL3qj`<0(y@vn`B84*%QBD5Sv1wWUHtrBPZYRbfY? z0tb_1OQWoabEmHvSI9S(ACCQvj50}X9$T+X_;>MiN1GyNV`7zZQkTirV;YNAXtAVZ z7yalj+EUVAWOVLv%i8;zCs z9fSoYoxSFy!Q-TPr%~31S=i%-uIvfUIbs}{Th%V`^2aZcymMC~!EwCr^&p z^Iv1?Rh#Q~S|tSy3FcqB{p2He{#q%^mD8IGPP|KEP;OyXPGJy=VKv?0_hR*aWcEIOChD%pNWZW&Z`i2M&*KSMQyK2>l{T~7)2wh zh2}UJmK@UbaW*Qc|*Ei`x3g>r9$gQnt8B7b(8CJj~R{&NC+?tx=+; zkvrzV+|*3&GY--M%$hBZvUeI)tq#iGaFRE0QhdN6n{b)w$IYYD9M~HUEXmv&cK@ug zeX-sPCjRhSk_iVzD;gMn2-N*x*-|mNA^iX9@^ow?wCMc2`9PQc7Z|5CU_`sEjHNxb}w^Q5|e0}JCJmyT2NwiO>%2r{$r z24}e; zPoHUO*3(m>Ys1#7r8_md@pLeWim^y+aA;({WGlWTPNC7{^(Ahf{cXJmOx|4KlX&oM zu@by_YvN}9)3e+^sowX%1z#t&uz=0-Ku0RG;o|JY;bF~1c_J7ZukaKk1UT}( z6*jvs6w%->Bs3=X^TJn;qQnhXy;`L@ zD#H<-3g5sMH z*yejUcW6ZYO5~WK*V5Q39wE@!QdVB{fKjm5>D0=VZ|){_t513KM@uhdkJk_B#*Cy@ z>eC{W*mX;8G_6)!v-OpO#O{PgOk8>g7BumMtb5weW%t109JhkT{8n90N5=_ISa!Dd zD4$snS?6=^!jT@Gbu$h;XOvP{n86|8u!(I$TULlP10(kg2j*pLj2sL6c(Xn*^BF8) zv46nG#^kW+ii1M%Werv#hD7!azK&c{9u7=e39?)Yja(iAE(&)x*deB^6*aajJh zK&Kr~l0fN7ZWW1(93Cx6f)ytu4jX-Fbh?owAbY_{*vi33z_NiwgTX~}Mgn_MO(NGq zfg@~v25k0o4)RRbI4seZ&@%7m8ph~kMk#%RO+4$=Ob-U~&Xv&6NaXt*uvn6JzRb}k zTbm@2289DMVaHk-`q~)sH#aX&b71@*;lLmw&~STMa-%ehlEBI&H^UCWuAm(UIizkl z2vyzaOx}_{@AwbtD!++cs!@!bVG&8{6Ixjsxuz7I`_$~VD#G<_W_GvQ!=!&Amjqf1 zc@H+3K49GCvtXgXoQdlE510aOc}!WR;i?rBDEL0DWbI^&g`Db^0nIH8EQ&dae9Z!_ zx;cT|n)eQg?MQGlnelLChR5UOmo~U6PFl`v*rFKL5^+dgsYGh$8DNZ z4)OVLxGEf)VB_@TAZOhN7l|+I%MV@=V#|Hu$h|0m-6}*OSl@}x;J$E6 zK1;FJbxVWDy+jt?M^_rMgbwgts$dh+C~yjjJkG>$L_{RURUlyMA?tt!5BtD{hxdyv zp0{Fx`lB;}ty&*0bIsl0Bh&kJ2d6a!YNvjH|KtsC$K@?q=QoY6tfz6a~>?@_|wh6%<`ZmsD(*j$%I2Z5{ey`B1{4cUmRxt zmB8)fvw(?p$7?3O2@Pfwn1li%STx-(vM%6tU_2J$%dE8E;J$ZT{j}2@xWyEC_0JrT z>Y3rJyUl@N)vFmyc^t2JHC7*bnX%UDl;pKvU4dLdH4oX|uW`zFTFDb~VARH z)wldq90gu@G6_xkkg5`LP-MI61mPtM1WoFXxcYlsD3iC^rQqNZBVx7g)g{p;;SJnn zTV{)kFZ*XS_3qR!nh}wj;!Ml#zPYE7aQgEhahHQj?b{f65-qqw_8k!Hxv^02j6kbK zL1MRm$0OnGAKXmy0((m;7BUER1T(5lXtZxh5^0{{$hz%9i`bt;!SW6!Hl;HioF_c} z%BvhiW_{EQYH1QI<2W}*Wx;i;g)@#`{T6J>p4jVVz$nwO#?%GDRCa1kEP4iGuFhYYTwGuET8>> z`KjtI?oxqP*`5#o8u?`$pWFUFz+1B5v$TSu;Imv8N;1w$dCKCq+ z$rlX=wPF~V64tXy>~LV4bcJJA*w&1d2OSdsUYP4@+Z-2HO^mXE# zydzZW*W9k+3D0G`UbGyNGKfjXd5nCnZC?#va zKIy#T62FIc-c5VN;<9!n6n69~knd9pKp1!1dud*9Had zV+VK?Hu9`^z{aAee`LX>O%6PJ7VxnA(-bjhsdb(kq*pRwc_AGq5rxO5Hjv^Q@OW|Ap-mfzKBf zb1cmfo^wIsOrmJmLzydyUrJ7JZ5H5OtH8O*!QM&n?2?A4Jr^7jo;>n%;<&{SQ@3Ep z301X<2DO|=tbPtGMxPk#^jR8|;yUs*mt|j=^*~GLEx+*E-E|6&7pSwd9OKJs6mUr3 zF?c9!bkHc{p;~65dS)WeiUw<`#>%QZ&TR}l|F$`B?{VP%z`$>jz-YEoy(ut*t$|Te zVYzj>B798g~t7kqCJYDVUAxy97WxVWrG?&Z!$O&vA%Gd16RrFc#%&I+c?fOCG42d zu*mOt%mnXPvrpga^7(onmAMEleWS5^#(Mtw57?QO?QVLYICC=R8de^cMgf6|VnvGL z6-Cz34|&!xaPDXp*>iw7Z2@zh0{cG(fqxF`UNrE1Tkwyy=NMx{pxxXcQ!NGdT?g1^ zF#MjeP%_7Xflt+R*(NqQMgb`wPrZg!4V{dCvofR@7?@V1)M#2Pb8hf@#-veVHRA!_ zzXP1RIIR~bFpDX$t$obqktns~!0c6@m^Mvh@;boeeqn;~#UP{IGZdc9YfwGlYL$A9bKikwJrB4H62(gv`j-od zI3%i{f51^UY4bk?j-m$6M_t}89~e{MLX9Gq zyro7p^L2wD@(I3laz43qL6Z%T(0n?JaS5ghkQV9$(4k;YXj9LtO z`U{w?7BE>UFh^}#3ApdTL((p5=^4U(fmrL48YqZTNXlq-sI>4mlO8`UjlP8hBh3{p}mYiyFl* zG>TU=ik({^8pSBw$6&+VE%+`$P>fMP;-TO_hbF5BOfd`CEDo?nEtvMNMS<;@Z-7R~ zoM{KeJQj*EHHx@>Xz^Oc$aR2O-C>u?-CC`k0;6{Ol6KGY@dzImB$y z!1iwepGCujx&zDw4Gc*R*E1TNGeOtdN0ct$U*XR!#OdkLD7DN{VAa;JiC38n9=PUS zus2~~mANi+v)SfPo3XL5=#5a$8&ice7(Kl|dpr*nJ(nn|*7$jq0_O=HaT~`+SKcVB z|FV=VjH76BOk0Ce_9NB`f90bl$`3jE&V^{L;8vY2oP58e;{F%*1_ky>>6~kh@_l$9 z*7tzR!BM=7QTEots=|j{=METcdGGz;H8)qI;D-l%?-IC;GP6n<_-CjK?RoHzc?AnU z*8?W02k+x@QX^70`|>0+5|VY;5*PF_H!MixSv{DZjLe>nyYv=2<})QIqM)dQ@7@Jh2>HU#kMrczIwpl<|W0k z{eMI0og){SuH}9VX!tngshh9k13$+HT?^&Z8)I)?jeS}wYSAb>K_V_PQMj+9B=j;@ zi$UpM({Cl(ZW)&s9$@9*3R|QkRH~Q49;2|>PeEDQ%xCVQxCIJHmrEQ2x|Xhb%75R) zuX{Tu(?x?N!W?rB@M#?4$S6|3xKREAqqrSoU6i8uh6b*E3L<VHL{j04Qy7Kk0% zE~d1QZ_PoeR|_Q9F!HZylx15exTbN!2E|o35@%HIM_ecL+PEOtP>tYZ_WC;VZ&#R zcQS1j#hQ)6Po@e#5ETt$j5BW(wJH{NGe0fS-ZgPWeEfMY^Y&=JCeCL%94b-Y`(Crg z9bk!JFE_rF*w|jKDyHhXYq!TN&ag-9GY(Y523gHa;GC0ia1mp@L8F+?LhJJncs41d z^C*fRJHYXeh4H;aqV|Ue{2v;)OBS$NC3w_@u@@X*EootkIgohb5NnacYoV=dF}nnW zHi#L_l-<=RJ?$oUOoG6XgQWK_rqU$@#Z zHTv;R;X4YPcN7Gy8n;+46f&AD^d++RX{czGqv*9n*{X*!Rf)n+N`*Tr3VaTVI4z9l zOnH{-{%3&UBQ9Cp2VQE^M6VwV2yMGu}n*peK+sG^90 zqmE%|;4aQAEvZ8X{&A%364g;mKl5POW^?f~4>*=J2+mq4Sh9dgLM?H@uJFlLt144g zoqUwOSjr{C|#1k^kIP@Q|AP>+uTPO`1drhy}7;fg9E$D zLv0%ep^^j4N0fBV9AfTLI2u`V^kSKNofx~E1pfgC?q3QbHV3z@NEFz%Zj;6W#;h4E z`!2c$F0hg_7Zh43aVSyh(!!J91x2lbi?tn#AC`JtUnqZNq3jZY(?4%Ym`oS)5EuQi zH9oPwG%`GT3rEQwRwo4`jtd*u%N!De9h@zy7MDF&x$@=Q%>IiB+jmDQJxkJCdhmBe zV3ZZ(G42nKYV{5pMTH5QBrYrcC&ax)UtmU~pb%Sbbc0UE*6_u<_Wz$9!5VXbS%E?8 z0bk0r%d0-~|63ur#DhmA(Ouvfvke1F$pVI&BknPKIJy{ELgpMjn{@Ov)2g#d)6O5# z@%qDDkf5_@SCT{<+Ytp3mxaQ4jZz#7qq>#n-d<*9;xPNc&%)zdCH^?dUUHQE#dw-A z%Gfl#)B3x_wmEmU#>hS@6%N@g#CBZx*qk%`pU!Mnb#r*Gxc&4J?rl4glGq=}usxf@ z;^PooFr7urr96P^eDpiVSqi)VPq4HWTYAuX>Ffu)k4pCo&YkoiT|H9FH>y!AE>S## zk-N`^R!y{GOJYczA$8#2WT};*@z>xrTc(}gt-Zu=T2e{-+= zpM`h+D&Fb-BEutpru4Unjid61zh@GS9c`St4f+x$N~>|z?KJ+y>T-a^$zgH)f$e@w z72B3B-Spb=*_%h*=1YUE_8him53mZHyqqsBQf;$BKxCMB6r)%|qPSXf`lbf!CaGNW zxolYrRxXTU|02M#V_%}wG2KbO>Qx<%vpp~_Ik0iI!@uWVG3;Fr_V0hhT#~@)Q^S2F zo+sl0v)Lc!75;x`KhW8tQ1j^ikyP7h)(+PgA2qYf%B#J&@bKYbULIMim=gyW*?H9$ z+OTT5`iih}iCGmrxnbDMCL-nJYaF^JMBRV+ikZgctFBv>t`3uocFmk>ReG#!)oXT> z4ExfRJ#!Cxt_)hLwZN}(7L(|zDJz2)dNejoJu9pcIB(0#Qz~8)Zyg1l=yidd+ zsKY?9iRF&C+PoXJm!9TLR`YK$Vf^&u_~iY8vpD`8e8N6KzN7w0#2FR$6j_H?e-uTz zj!g95nXxs?)m!|<`Q#pxjhoj5PUe%(*m3ORjAQ)r4j-;8W^Q&8}6_rZpT zhdFm>xOBL&%gA*p=sZ$fIE`O^S`EWPm*YaOIR)!J_&F;aY+>3TeEn@=XCIdv$GwD4 ziN~h%G3mr?$WLaQ?6A=4=Atk!HrdDxf{lui3JX||ihOp{I66bALvwiz6PL!aC7b+w zj};s`WoY;Ih<}F?)6#Q>wz*E8L6buox%3uJ2y_hz2~}m4j!6CLA`)9F+R|swa`CW0 z=1Nv(9(|dWO`Q5Cz8(rIzj^d}f)7vFj07Ksi8B)IGiEj?_mu8*;Y%_)cvQasO3_@& z`D;2o)u->NY*VcJ^RPv3Ys)5AcD)@BX7O1td|WOTcSX@N&+dmOa{=$)FqwZ0yaEA^ z?5!q?Sb23z0$dnfb50%1Q+Tm}mBS=r`CXy;QQ@+sku!GPWl!n|jFo@;bz>`E{gsc7 z@})YAZt@{l7&$~Fwlp|$>f8u8T({7|gV$IhV8Ve$1;rJPTh#=E&uGr&VB*qS{$P@q zkNe6ZVQs6QkNE=>Unwiu|6Nmj#?oYI#K|t*Swd4oLN*qPisU`maW&kk>uR!q!y;B@ ze$5T3O)*LrmO4%L|8(^F)W+DJ)MK5dYLdxvymcNT5PSnj~H!GV!c$H9TU*@Wp| z>r!12K}Qx>lLL*b6JIzSlUHbXcw8vy%ffAn>!pO+#M&Ab9+pjgp};1$?%P6HiQBb| z@*;kB99V@-3?8=dn%@X#;xxU|(4sJR#wU4Wse&brY)o7pj!m8)g6Cj9xM8;xnRwywS11#F%iR8*2(+LvX(9mXYRgw_+aMS z0F~HFlNJ67`|-FvP4X2)q=7-; zK?AGBg?}yf4UXKk9}ck}a4CAgT+*0V~gXJMf|6J z&g3pI(C`dMGw-+bn4fgOfiGtj2SdXxhN_p1_99IDx($xp3I`ZC91avdoyov!%h1Fb zlE8H3i-YOnV#a@b1q!nz5;)i;9tkXqeb9DSMs;IHD6{mG2TbA`2OsAsbOZUVKbDMv|Z^p{13J&Eh)u5pgtBCY)PPhsDkwN1jBvs~Uk(d1YW za*#Km&^gp5%Vv!WBiAYI2$5OOZR8FxROx##9`Si_NsZ&6zDekr_=bYZx|Kf5Cw>e{ zIr4>B`pE-@=f*zb9*wJC?Km%R;h`|EjRLFN6eXTjGA&|J51JMhGV*V`$P}3C8Pe%; zyfKEkfkoV44%dT)0;Mk;xV;z}IUF7^=@dAY*j#AbeaxVZZ%sqKO#mZX$r7e-JNY@^ zB{v@4B+M*+;DA~F4ZrRGlD4vS$xh=fF<_1M=$I#WgTu~V+BHZdnQLx@fR@1)zL^|T zN@KTp?n=2HQ1!PoAE5kf%&+s@fSA1A~Kn z3BQ`AUSl=5vNTUg^svNPKn$x0gu^)f4pfhQx`JV`KJ-Rj3Xl}boRpLRZYTiG49s4F81GiySVI~))O0$CdlwHWNGNhROgO|_mEflQPN3N~@EN<+iPsJM zGp^pXxoSQ`i>Xj$Lj!|C17ku0i=0?!BcsM-R-tbV+E~)3Y48(+rx05*9G&RLG=hH%-h4V2oPuwu5a>gsvlpl0bfivwFa7 z?yGXG!m(c#`HETXzVT_Xi_qPSxXX~NEqtr^s+zU6z)EhHtl53C>dbp@`equZuw8q`!1=6;S^B_bk$DEJ#)~$x zWX#>&>BrOgOrwEiVuk|e&bvu_*sTJSeP*6AwRE4Betq3SnJF6oj;*!%t1#8Pq=ak# z+KICL%RiVMnVI0Q%J>NL%}Yv48LGNDc?4P-KK3M*8~(c^d;dqvD`vG9b61r1y%+y= zp@a7cgTTBG&639g+6;Rhawu>#-_KxZ3ou~fI=qfi{M7{M&YG?M4jNY)W-s;@<4Mor z_IM`5W!S-3-@qJM?!fWwl|Xj?=3Bgx3vNp@H89QyXLwreZ{ifQe7SONfB}1Qx)!TPj$?~Fq(c~HW6t26UcQ*fmQT}3wuxnXMk?ka>ga8T+1h0DXidH z>zulC2iLUesZNX{YDFTVq56}Z)1*X2Wu~SD39yDgOgpusiif?$_Cr;!V)X^%>aK@{ z(TA$#HcO~Hmk82OvrEWuDs8oF(Al-H{1FFhg#wp}J8LQf*IdE2`3~Ik3s|KERraKp`W6=R53D;Lu{3^Q^kLv&G2~bs$ZRrUvS|Uc z-bs!>ALYve6j&$ui#0GwM(BD9aLh~KoBM%t#RjgG&P}ThaP9KQ|8$9u=_;RHA&*%i zkLXsV=^rY8I#s2f4E)nHtxScL>9eT)4I7Tj%x6Li^^S^dnQnMIPYc*t6rqe#HnTZ zAk=@UugnD|4z}!lPnh^#vdT`FtTN44$ALM@KvpA!(dq&F)j$q!2j-T43=``wu&ayA z@eY{mbAkQlK{N9WEItN|w&L+%34jh6fc&;dL%u7{JC}i?e zQJ5)gAELl9-+_182hPb1Tu+Pwx1Zn&UD>^A0@w2eTw6?qUpeTRE2d3ez;cA6%20`Q zQTVi@pY`*mRoR|!&k$@W7pvCZ%6#)^?A;dvI6r{eHz#ZNIIz~~tfhDiNG>SoHkz2?`L1$S7<|-k%I;R9V21bhmtZWxb zxFSs_8gQ&#$sDm&J;uC!^G;UIh2~ZctZOEARvtB$NGM(VkWrGM;e2I>g^TRJI@9`f z3mX=uFxx3CGJC+%puiF2%3I}-$1+Kw#azf^<&?Pwy!$`rMQ>45x0wGWvuU?Rw0?uYq$gV&H@$>2G(W+E{m%0Y9*5; zmP?eC*LE(d7Z6}(*ucPNAX8z%QD4BG(H?FQ5Y8hayIM?6)PUL6*!iNN7sHFP2VX*5 zRM!7*c3kz%e)^)xh7)3RJ!P(ZINsvH*7Bq4Y?=Keeun5S zU@j4tl)1ocdV$H_fZg~5v$ZjEY=cg50lUNj*5Ux>hQM&g3v7pt*mD$E0vI^@AFzmg zU~K%r)AK>l_lQe#`4p)RCXEZ6jRCusG;;;a;%Yt|~wIh_%^)O~Z?OS^-;H1KaCv)`$kq9R_Uj37NeNY!l zYS)x+ILQ2LhF|6Py}K_?zCT;X?*e-*15=v7%xMj5AG$f$E#N$w#rh(YV{rrf-o*AP zftqDUO`^Zs2kns9DZ;FBjQz$ghE#6`@q$c_#@gry%oYvIlP)mvE?_R%AX9w6!SVu= zpu!nJ2lkQ=4kZkXb_VSH4ovJ=Q zOb^(uIB;=1vh@9P2C>)_F_lbjlxIFn*HQ~$7eO<>7S z;9{-rk$xJzKlPtGmwQaXnmy`M+0q49HEapeW8J%%okijD@!CJ?vTu*e-d(m{D`dx; zy&Grw^)H_M(uCP}0qX%yraiMbXI|hkp3513f_rBIOJV?fZvc0%0_Vk#UQrXIcN{U< zztEdY#l*hY$IxI~l?4OG2L{m%?28vL3k5KoeVCPSnSrOF-6DWd(1GF5K?ddt6NC&B zxn@q_{J_9@^V&wUY#spy{tXNza~a-FpYV1%gMfgGO#vg1LPSn3BgY0okq2yN9C&0K zndZ)3{w%EGT@Q=K1eVFZ7gGOh^b)x6bIsA1JM-6ibN;LpPFcee8o(M{l6LU)v7EH3 zoDUatc2{LKu$<;Dyr>;};a{+d+H#93r=XoGVgf9QsKaH z!GtZr?=E0&ILp9) z;hMk(K_&&(GXY$FQ<$2|Q@CEca9i*C8^W{F`hxsyt`z~tX66RYt`+|Ia>b9CTntth zJOfxg4Oo{pa5o+Z%vWHQeae>Jx#`f>mFwQF6xO|Mb0EgmqS|1=DvJV*9d9r0;FW&V zaavYtOTY8e1A8aiuAJ=ml%+I)U%A&f4IaE*J|V~U0sn^Lzy&Y)?p@-3>G2ntPkCR|GOfOr(QEMF$@;8IkN4ev(s9Cl_2gZSPuESHY`)>0(`A;53mo1T z>=tcMTQGrhMgwPrLv7>**2ViqxyoI|TzLz;VH-Rs|f$O-zv1c1zGoF3zWyrbO`s;It zH>(<~S2VoIGq{<;kaCW1kH%@XCGE{?VwKmPy|rrjo>SUSPA)ZQ5ZPP0^Y$aV?>l_n z9jv{hd3Lhf>dB6e-`P|$dlg$`i?YQga4aj}o}a+s?$BnK%i=A-RvW;z^a96@MOOp0 z?mf~?tZE9qw(SEq|BdNJ%$8}4B?pQb9T<2XFsy#hz-_?Nvwv*EMXtZ)sW_U*|85^>=ooR;3{A6lJ_42AHxli4GbO(yjL0?u3vYf-&RP< z;q|kC3(qF(^2_I&+VEzY?X4NT*1`2WQym`7F5ro>ziHsdx~PDwQGiw2RW#V(o95}X zHFqz}kxI|z7x1{G5O{epd%v7P#gZG0%xumQ7DYcMLSIfTO;pxmR;Z{1QLue{%&k9v(n~DnyRecnlmU@}qD*D9CzQof~#PQZ5=SzID zbFQ7ZnCjOmRx0$jvgk0Mu^W$APpQH|MkelRp_?0$p9`*(U=-t#h`#&&B;}f&+P?&ZoF{n&KWy6O9ADP)iCH^F+@wT#_nQ0nrvGCxbRNjSw881@aomDA5 z8Z^^-`MSwMH#SZUnOXAc#w<+IcuepyOx3<1VjeufyYO%f*X-wcH(U~f z_g2l;xUq40=H=uBsb=e{j`SM^$NOZ&FE8K!Vk2AMcX{u1EI&W8)lN}PKQbfma4Vnq ztQE-uj{7uPChGp$u#}IU_1MuYs1~)tfj_M-p<&&wf*1uRHl-EaCsY}uALz3$ z`X_RR@q|!9&qNoE7u}8QG8qkw+!_vzGcp5CC^VlnaDK>Qs>`9vtYYGzG^Kcw2CryH zm*Mg4S`FTd{BNvsI65b}Yo<$Q5|``KUX?{2Or0tn$cfQ+V3Pmm{%cqT7iIT|XO*!YSn7IoXQ{aoCv z#t@lw%coG3K=@zU*r`am68-kUq<2ysGy!rUcp-Zxf#CckjjK zu6fPXAr_Bv6j^qv7?tu_nJsB<>Ro>GfZwY(TORv`1bV++$7|~6Yr3TOL9;)*Ooqnp zoL_GiD=v}tG162O&$^M{w!=DMWv9LbBTJY2jERf*16gi5H1p^cBu;dH!ND}a?cIvj z2_D-%JZ$2hD3fqU{?&=bMn1g*34KYbiyNEt?@76~tIkkNWm6Xw;9=r^)!@Pvq>%Y! zA~#FHCA~whx@N3DC{*yEp;KM2hmosl$HNUQ3I#k&Pxvn%;C;z|xX0Jbcz2G;#R&zA zXC#Sz{oHG=U>cwP?~A->l*Pjp2B|?>(+;KWtEFE5-FWO*R$!sJ-PeyD z)lW80ntbGd3Rg)ZchUr*|4UNX_AiR|vSMJd7GV-F=U|Co3FJ1aSimImqJh0AfZfW5 zQD6y&i{h^htgBrTTuFUUdtD#MQ+l^m09uMK&BbsiDvSnv7X^ z@c`Ss3bk_^G@08^9^iCqZnD_uU+Ww!|NC6I-9k&RT1^ip{uX7{zyM`|Gb^wBS#V5q zS)}{q>zq^n-?`A{-*&0~-=aqc-IEedO_(#uu28y@N%Dk)_=UAiye$fMGcGX6Y;jrhvPB)`Nas;}lO{Ef4pp zCY;$4Qom;e%&`ktSQj{fN#xn}Im>giCSQ|WDqZs`*p|hV-|?!m+?|9gHZmKgm_BG^ z`uq6`o6}YMJzgi7rB|$uI5x>%_g}^WzJ?3z4mAnPRvgWW#}Zh!$R#o>a5Ss`OJEP` zSR^vfphfZAg~@7H9u#jCaFzI#z~OX5NouP{yXl7qO^!B;1Q$0rDTjo2slI3sIrF0F zo!A4bJr`ONISd-?Ma!md{_|OXv261GE^#LI7KYx;m;)TF2|ijg4)UiOIOm=XV7uk_ zkfY^s59^zXWfQtW{2Ex6cy?C@O_iHet1029k@Jw<+qAvz!p2OsaxSlxKTB3P+P28; zs^)NzaoY3f>pD4~$mZZ>)5F)s=rb#GoL+6Up>fk0E!~wx?HklSb0ljVR?qm@Y1H$O z<8FzQxRWB+?k>lJwjU44STr;{T{$GcweMf6?v6t4yc5YHtRD_2FkEEizw>}k_|^el zwdi&Ro1AWb=Y|LY1@ZIup3ZNX7?GHvBi8D5($KPwAwkNZflVZV$$iILiDL=T%E<>$ zq;Ajv z1#A+{>W72hv3q^e0Tb}cl;`FK5lf8ubhi_)`WwqItRJtpKv{%>DX792hwcSn5_d@>mHl%N=N7*RNoF9wTu?tjvPzs^!9KOFP(%-zhXO zYV6kI_%N5Tu9rc{aE)H8f+h$#a-Nw z_8Ay7`7PK|rg5mupjrC_Q@{@vV*!>xj#eju)~FS&ZaO|)5B zu!eS}GOq-)y@An|&n!j@n0OB`h-fhK?r2n*q03~_?7gBXS&H}VPoCn%i;IQV*>5rm zQqdDx;9&6D+Pz56ZG*_qO1@_eQu8s97|jL7uN)XrMAtD zQaZLjcR613woS`%OLO6Jl@@aoU{OBM(*L+s=?}X=#};FTRwE6TXoXg%e;-!Q{>?1A zVj)Wf!;4M^z6AFr84cVLO^!2|ZyaPe|Fb!jgH>t=V*(R5(}M2Ki47bR8Z3Jlm2o_8i*4z$UTDdxh?n!>lnML|0}o3eQjwnbG9^f`w^` zuDgry#6{NU1zZDC4wM=n4ccS-)nf;~GuvN%m@+@hI6zCw(!0g= zg==~P+wvB+^a8e|30|vI+EN3SHgWIW@vvXY##Xk(P3*_1tJ#cVE2gPSv>0$$N9||{ zJJ6ym+^j9p?0=xuSYyw^6ASDPG~3=_mifRa-@zzZ(Jzz0B>#d@{s5nK1B*ii(?979 zM(G`mJO#^SBAB8R+A;r|;n@l6IXTb2KQ&Z}yf0 z29G@h3;H6rtn^j%GGEXfWwUDWr**akEa?pFbpnQN;=YWe*h4A?gi_d*f*7Tk^<>2mTIl<;fPs(jDQi))R)@apF z;Nv?Y|MEs~Ko;{`8!f97%_pl_G8Eb?C$Lv?2Ubln2dIfvcg!W2@^Flm1oc3ryjhC$@S zr4*)CpN!o`9lOu}=rNVNQDlAj0*9t~Y=osRV^B$i+5^w%8B7a$jz+HOv$xd>xOC3u zL9;DGTYdq1)du#O6YVt@+9zLSpL(^u_6B>+hIWt8RsA=-R#q)&w%*~`-6%7qkxeU9 z>6HJvP0|WGn(clt2pnMYX$W1qlr{E2tG`9_u2rnj8(6s`4HP;ST4k`fy|9$}!65sC zK`KBXNP>0k(K9Mc-TDfxI~K7;Cp2*?FmPow6s~Mwo6y6Rz{s_ufiI&$tClNmiE7-+ zM&(b9(jOS5UoieVl+vX3qftF!nPo+jrNj*D4b0{{n9U=Yj0~E!ESO4Ou$Qe+o2bGl z?9j+*)OeA#(TE|Kc@7hA!eS#)&E}^)rZevb*=&DS62Vn+!|d+KQ(Ksy6>MIpbE?Nx z%k}|t;*Yk<1?|-b+NZ|eoa}q^*;V$g2(3`B*|ydG&Djh5S+D-T5*eixDwVOt-XLo0 znU=5xtPwwU{I6PeDu*q)LdM`i<2qMa-5p|19L%~eICXC@86>n7q%g_uXrBJNIemd? zTEguFEf$S}CiMqX^jLMu{1VwO+LL?qRt$ zcX#hA7ciF`*veP5S6AG7 z(Hm!at-XIo$5JQ#;8|0@EV>zd$}4Ks!9Po8`=33?P`E|@2crgqjL!%CfFJ9PCTJg9 z>65f{tCPYldx2eU3N6+v0y4Q+ZC0Gg(3RoQXq0bgNqwQPa0c5pQLe4Fsar!IrggNX zukA^%eVA5oI6Z+aEtV~4L+Vyp)*uenO|dQ7L0(p~H@g%tI|#5?3$UntX#R7!DTaa7 z*nmkugYn**M4lHiQXIV64_I~ntzc5&<301C=k$Sjrc+}z^jtOUp4X<{2;P(Az~O$1 z^`7n5m1YL3>}9naS=ovtZcb{w_v~)FY1qw|UmL$aU~KFYwVcL(Ve(_^b&bE?c=@~T zxE?I1wxQAGLp|Sj$t=M~P2!An!5Qfl z)3$kTeCpo}kGZmU;0O8scm<6u$S!Dy?%mS*5?WWds8 z#2T2Ha|igC0f%~utirapfT6}#RGc}9ybkEhYU8` z0}Wg|7<#T8((Q~cx+(NgF2DUHce~Is@oMes$RrcfV9mNmrc;ybrS4u7%{gGk;TX{tHpi7OB**^h zIvb9VB_eG_E80)F-8}v8dG`nJ{ZAT&|DW#u@h?aE2ICj0+|Z^%+u8mpS8hr@SS9s? z(JG=vb3udIh33>B&5}D9m47g3ZD>~g!JwAFWF8PL`j%Pm$5s0i6Z9@b=}An;eyM%z zWpm&P)}RL*$Fo|b4=^l)Etuj~b6G}5b>nSN4&JsHRM(Ks6T=CKdk<&eCi<|5;n*DY#1$?-^Np6Gh z9>&GCP5ut;Y`V-w4o#Cz%CYzt6uCWTwQuC8|KNK(w!QbVZHM;aWv=idZtz1X*=asQgH&C~krv~wqaU@R?a;#t9{snFt7(Bv4w zmSpkko?hb3ZCl+KSOix^>tA5hX9(2)(6DPMqnp7?_YNkG6${t(Gg&7vF=;Tc8GPS$ z;82}s`B(39$?t}Loxh5|9%YTHNK>|8 zIx_vVt9XI?nR!AddiI=KY|`?G<-RZ*Uxsbqt99!}+!HnP1R(X2(*N)DHH# zXV0gwH%%?CoIbs@OCaNBy4DYY7Z!F37qYefui#?-+8}v?(aC@%ESpJzqliP2b6Sq%IcO?(MWYu7e&KiI$@!NR|y*}bBv-;;qSp_x1L-#!io9h1cs>$|twPty44 z)1Oc~zK9WaX6$Qrp-fH`ye$ zlsbx?nCw_#5whUo@d<+bat1L!0yzYP=P#MnnsvmZJytxbq8bxghweznxRoCMH7R7Gvh6Man!86RI3rE^~Rf7i?Hy^7htJ@tB4q9ZmDnO{;D+ z9y_s!nVUzWVMb6JkC=9Z-o>C3+;jCy_Ea?<3VIqOsV-z3w4`C;BDW6;zSC?ZjZ(d) zsYPsZPiz;_oSk#$&hwMOD+1RC&CK%FFctlXa!jVI>e}a>b@tFySI>bvKR2-N071?w` zKH*LyyIf1f!lMe!cMcw9m(AJGBHya>(NQ+9P+`{8MTQ#~n3=a+lgf@imNuK2r|0DX z=54kKj0%e*5{{eLavV5jR3-7Qv*6bz&3sNSSbv zxi4x;V5MB^l|WCOx#mJ%YD*r-E!Sjp=Q1vP(h|Qi_ltN)z<=2y4 znOSri3=TzZW4oP`I?YXuIepT%159EnPF!q0HpR}I4Fbi7nfqsD3bM+FBpxdZHMz3s z^qr}ON7+Rp1P*a#hKXppiWR7=T&j{;b;v_{@AZ3L?^jN7;`$sY%;CTEmG~7uC&sDA z6+8AsJqY5s^eRNQ#6aR|bC{*Xnhk0G9-(bwRbQFKt9gnJv%AMMa_TNfQ4&x~U!ieA zW9b=<8**0n3^|nD%#1iBid~I7_Ec09dYf#YlGvfRy&||*_tOSt7DhpX3k(9qHigV= zrWS_m%uFW#96m7cI4DHDSt@Jr=>)qENSngtp-a;d1^s9VmV6fvPu z(0xXu>wKO6Kj+L(4N=_8CSv1oXffvuhj~jL*Sq|yR9HIUV)dL&`}bY?G?B?q>DS%s zY^#F*Kbd^BN@G%wbyWG8)vCA0piB4aD3yEys^V}K^Z3p;}NYiMmZg3fu0F% zY7NWkBa2T0bFczvDc5nWDTzF(5$8l!Okn&p;egMX)gI@MnD6_- z^LD~Z@6HR?+P5rPm|}OiK+5mU_Q~^J=xbDNi28p|aP`tNS`EIPEOI{{u*SUfYE11; zU{bu|uu8{(JwN-1P)`GE)R#r#FCy5jP6e1pO?k)>BEV`q<04Oi;Ua-%fn?<@!>-gT zP9k@NSZlKwCG}lC^VEK5*02%eHPYD3I$zp>!GfW~ZBC(}_mpG8Vb_`5RG63!l>=POo z%eDG{@T_e7JmvgE)j+9BR{v6&B&IlI+Bh7P@lfP-_;IAp#&N1z<0AG23XcT-HnTIh z&RpOzVG{qO2CkN>t*1DcPCE*6g|Qg7G|R-aa+eDn6`PjO7W0Nl_U#LI>thA1#sS?t zwGnL`(;d%7T}hI>;G!MnxiBm`#JI=!1f#^X4J?LtxYzg1Y~~9%%*e#>AbDv5lgPUT z>}fWPU1A2}+)QE{wDlO~^C&dRRS7T|_2w{gE?{8xGH5=#tx?2$g9DSERRG5dL7UD6 zY^Gj@#{()Bi8MJhy4qjbE5KspYkj#xk$CX$Ryab|i|NxzMbdzDjo|JK#S3XkDl|@G6a^WBs53N{Ab3YC^}8#kyr5TZLD?%N!;=l&UzUOL*wQ+^2F!1 zYMyZ52y|I2*_d&t-OZ58jpq@6x5cH2OcV8(CnpK&9&lh+3uxrmS;#N{!iUvn!qV%R zE17!)a@i&{oRj8gU=+Qb0rwG2N)#t zTb%6w`$IT(wc;VctAA58c0LP!%XYOP)jB%pc(v%Zi8z)CY zhiinx8@2*Q=?P2%A}<(Zy$t6Fgfj@Jx*h1+pR$z2p@F44f%%QxgdB*YdeWISqx99HDo&9A= zRGG8#JVlO~kD@29mf~u-k`rs5T&~$2;iB>*fSpZs!a0`%2MW!*1%A>kdJMc7 z4zgSyTX~M8G72f2y`cD^CG|!llX(VgZd!6`Zl2ca{EV=B zMHjENU-Cc8z}2(h*q<-KY5$(+^RIDYDg7^7uv|rN1*^#W!{Tzxo%t&sR?M?_St!6P z`^Vwc84ebXz5=#@3?|`O9k#k<8@Z!;7V>WsNYwpz;7ee~Yr*CWCO*3btOi#WNnYK- zDpAwlZn#CTNCUG2hlC*egMTfS z-z?^(HrO~{XHb$m&Lknwz?SF0$fFRF+HssW#evP>AnU)723`wB#zhP#1({6+oDB__ zO+AlRsI9VSzii9#LX}6? z)S1z{_qr?3y|R490UjGh6@$a_F_W%b5P5R(Mf1JgyHAy^RR4#B-OgU`!-6hSs0t{`c3``Rmrk!z+y>LOuXTdvH zXZaXss~e6c8O@89I?Ee4TTO7%SfHZ0q1nX8(aM8aF~!-!!qIApv#d+A#T`b|4bJi% zP4-7(twfp>HZYq895h+dtdP;*ns|V70)vo7RJQG1);m2qc_Qa39=Oi=$KaOpbjnJ8 z{X@~y=U(ZZJ5lvv|Ee2Pp5$Iq)M@%J!f}B0&Vic`G(DZ>y3T14Xl%`S{9>lu?Gp^< zn|#}Pl8tX161&m(;WP ziYD>OCXBpSoJ?jMlALgGi@_4b8;3M17#KTdF)z4rB*a1aMv8EUv*{N{6Q9FEPnHPx zIGNR``SLkg2sFuhG%K_?UaoP_WI4p!!ldlOpxn~L_JvVXg7Hq~0ilFPLJWt5?l>si zXk_DPk`6e?dxJrE3Ip4gbWRa}kral_S9x}}9>}Xq4evXz>(y1TH~PWM1cOPbO8;g& zy|O`JzhLX?173Cqmip@+GMP5Xbk(H|KNJo>JP_}7!t7B)+cAe$FDDg)X5ExUx)n}} z51b?`KquF!TQG`GcrIJvZL(*X#*c8>hDOOwM4)WY!6u#r|>IsAJhDN5kMy7uU#SI+i1U1SoST3@IQCL7sL_vf#g@KEM zNtuO7xPWEn$ww!aOxyJ|U9XRy^H)+qAs5dccPWFWf-{c(PYyPQo%N0ET|K4QCT4oj z2j_n=ha{iyT#Yq~NqF}3@Wa&BvKL35YVqx05cuGzY~n2Jk*>hvq`Bmwf)1lZi(|{( zMujO&LVv}THNv-uF>UW*QfYDWx^(f z;>Pa4C|=UQCveQN%7IVkV8;ap78_Ca5(eH23~W0N@FXV4KJ@(g+s43n<0 z8blv)dsCwp!jy7A#D$SjO+>-qFtZu6Lc#)_duL8e$~hPE>PL@*+mal8tDtG6r+B_J zEb4VCmR-Cm;Or^^R)Z9#lw3V)JN}ZDA@g zYtrm+lF<3br@+x<<;!fWFwu8R|4IKJrLsk~Bw^DrZMSK5s7<2QTLg zk;0h{0ySEOSNMGoHlDquvs%Q}@&I#?!{Q*BlQG9uZAj2KVkp!0j^VaTetXb*^RN}` z?VMzvG>EV;8Tvd=>1xttY0|vGbmuD5mZ;Y^9yLm292CoOl&xtLc4-v$a1^=0*s%4$ zokeVHFFIITBv}3_G#vfFx1OozUKks5jKh?oJeH6H3<3vOb@Bw?%*jwOU@>8M+}3t8 z)Ip}~{JRMU-m%=)j%t}4%D|daz#5asopViup~=LDnd?BT#*_x>iuIg5JttmWJ7G6- zif7l^02a;#Yfime#3drfQ^LhlB4w;|c%{qP_ik}3<(Ld=nypVbpO@3Wz;#vNL8@8R zysuIE@l_kccQm|YY!ugVlC9y;oWe9As9Al-vNKbiiv0k5$F!&4OwFq!-F&wHk1^>pgSJd(w7*?SDMm3HGf8TFj_Z=5@u zk|6bzhjq>Y`9v0h+3ZI19p2PEHQF!D$)LRa-vK2ZXU!)r`?fw1j_wT%*l62wiRVz{ z!OTX+g=hJj=9#~HVE#|RD*8k`+ro)FhlFjKOr|hRZ+ov>aY$s#6j>Wx)0P#xau`2Y z9+bP%C}!d)oN;hMhZA3EBI?m3M!2bDIgj4yXov~ZOF?ZB;Iv|g2giT#XrJOihK!W);D zto3(U+xZ>^9q@du`^2wDd->g`YU?I1oHp0Nf#XddL(&0x5ofD_gK2Xmz6*#1L^p8m zU{qeMI( z5!J>f?;0muX^axiwf63?F5eil`E%=|@^yWMlK=9#850;d&hXwfY46HtP-k&F>FT_u z_4G+YC$lR|5*&Km3XU7z| zH!!fr9Pm2-g;hHGfX0VRbB6SX7GB2E+U|O6TN+#!7--*5&vS0z>bSw0&>+p?q@lo= zc~zqD{s;YTK`s_1%^e3#IBq`L*%;JsAdsE##;91I%a`-XUHwB1yf+vWQcTa9U9TuE z_4C{q5nj0ZTtdsdNf+E?ZtKKEPmlXtmAv+m`Ff?b;S2}=njK=RV2JN;JjQuSJ*G*N zW3uPDWDOn0%WjwDOq_ME9Ap+}~2qgWP@Bxx{^pUn@*g_4oAZqjB97V;fvoMT(y&PLM8742mU(; zJnuU2GdSr+6gS2R`QBU;Aah80$#$EH&5^%DqWDGUZQW`e&aogqL;Upi&!-w%cN~!T zuPSiiU|Z#ZJ2M*Z)-dTNG&wXJl4xjPD{0jDVJuwIu+aM8jvI`;I~+JKILLBzPWW^1 zp-H1gNVCSA#-}9@C0{g%RWz_zd~jG9_{8i0+v8Tx!xmo0k9nXWBa30ul@8m{-x6Dst_sN~u0AsiYNusfeCWaQ55F&!DRH`_NnJl-lVk|q#o#KL2w7n5;tk#P$*uarpIof}N7 zD(2HtJRUr7%(=l%Wf7S<7ENU?SBfd)SPTT zzwA(_JyUuVrk4)r;xM-Rn9OP9)!8;PXxhuJDf)E{O`Y<;5}x!+ zTNNy1=CjaP)WG&G=~1)lk`2k6))^X!e71`;7#mOPG%R3Z(^AkiI4rjy;iRhcYMu@I zmwd@?WRuf)>%rkAyq1;Eqkz#w+;Yc)6NmIHdYdf&EjZS+NPl99pOv%6+A?V{XweevR4RZm(2bQ1Ia(x5FWeMLYo+4_n2f&RIO> z(#f+t++nb2;WEiqtCW;Z$u+r&d~UB47Pj&!X(%vph&^y%w@FKQ&?Y~lV3DAP!HVRG z4owLPjBFkaYu4BBzxFmdvSrU1FSb=8KHj>_r?)d5(n?@t<_kN*&@8s=?HVR-j^+n( zT(3A%rL0*hjLj`yFZsMh*vH|aFHg^5g=Qgx4nC%1^NRffAA2P<%?P+>a8x8_g<}(^ zx8zFKS)XpbSjek#D}s5^(%I`WY!gl?rpr~m+4x+pE?{Z@vRP466CI}KZJPRKdRDDw z*lebnlkIv8FBc#B=Wj4my4U!L+2X1GeHKjuX)8Eg`Q6-Bvhr{GCcwh)@aBQir1dJ9 z2@Q-|2Ojbtda#F~*+r^w5wrA+AB$V65;vdOa7@m*_e7JY@S6v0VghfB8Vt2{lK8v| z5`DyDW>`EtK{ZxgpH%!6DgS3`{G|_zEj7IjUFI$i27m0N;@W zH>H4v7Hfk;+}k?BbX^L&)BasqB)DC`UAfC)7T2Ey0mFs^{EH0Qxi2&bmSr%q+HK%a zoA5IJmsP}@j+Dj=e@+{$ZjPPmb(x_%^o+sQ!w&3G4LiklFt7(TMPB!04*{7=n%xm9f>Mg?HqQ#T_KRF-_?5uvt&w52(f5KHcg%tt zd7DnNO?x}tqT-MY*G5Kx2o2$rK5DvE7rTv88o8Ph+zcfq>WS1g2yV-8;o~gqj{37$ z>~01NJJaO`z6%dI%{MSdWH~h3{`u#?ZkfQO%$3N_C-c%M^^y2~y)?B`6WqM{EM}y5 zG0zf6IcY5WfrnjTo3;3vgz~wPLY!+Pj^})FWDxBT&RewoIQs-0^PbAfNxzA~v>Ycpiz;*4^>iDUS>!O;N^(qz$vd&-#vYW^mo3coB z*Mg(!HiBICCXcjw5}1U=6j`PIG>Y_Ouqa9?as~1v@%P_gX6HE2z6A1^I(bel}zSx zrbh*}97hyb0@-y77IMlYFtA50V0SQh$Yv_U^4w}cv)Y9R?4>iD#WMsNFTN=@n(ldb zrpM~QONuiM;*>6>X2i5~7_MP8ZaC=deW1y7)-k4kW(7_P`y3b^c=4O7H}K9)S}bwf zW&ulqAB!}f$_$D&89hI&6M`5c1>IV)mdeKk0@wx*9~K3zi}n$t3xyEoX~{@jQm?aH1HjG!0uo0 zLumejY0k%*Vg6K-B$wZ6j06X!8c{sTjs{*8lt<)HI6KeVTT<}?(V6?p{N zWEqkKqZb^OTlTtHSy;`mSyOhR)8QwITgC3KGUd&F=-Xsa`g-e@(nx#H07garf$gE@N>`em2Rf1gL%KqXSZD~ zlhDYw!=NLL!I8T)p-tE4A*WHrBe6vsN98;OJ65?I7UODUiC_!-wuyO>Ky!eh0^3LS z&DS4_T~%n2w{zr(xuJBklz~b1%>;&?qX`}i!9^24X-ZveXuFrL)N@qjmZVUDgUQ^) zMyJd>OZlcOU{%>MW75qVGouM_!Kv)*_v z^CL?zM7LyP*8dlGmMeU>UBGl_rm4{R69tQIh9Env1uxc1M1Tk2rTxrW;X55=}Ia5OAntxMp)rND6Ggpt%? z54{E!g9TfY(pW-|vq&efL}&ZxH7r!ceU`*R>pKmcRSLXQ zma%)R+&gW(;4_%MG|;{SN; ziN}N^5|JO~NhNHJ5U4ozWUF2SYl3!~UbE+ojsUG>HZDb(SqoX59xys3^z%7RWE3T?70BEP_!V<9>xH6ncAfyE;Z>Hx-*!R*;cGc!mOSUV zDDZFRdJZcF_J#z(cMP0nZX0Ge@=rS`=E7LFSkbQae;B!M5a6jPOAru0uM}+7Jn5w!1Clte_wap zjKfUt6a=Fluq7q1y$bx6)$qtCVb_-jl5z)G)@TbdIdZWaG@R)ukhehOT)W5&Mga*W zUYVB}mlSx_ax>3;5Kv&0nAa$?g6)^t+kLMV{(3QiFMHjyPZIE5g| zbFx6;bH1JL|G9Dm`jj+q=H#&5I4ZUvQKV{tn)yV@V~^NgIf!JO5V+JRy=S4+9!I9- zkL0Hai5E0#u5IKMN)()SnsbE$4-=zEwnCy*1E0i0J}CtTn+2s7`X*khO&;*F1{U~C zc;az_g>|N3V$f@&h}CQ>I|9TMSQOIOtQxp<9xymDe05Ue>RS9YW$}Hv#T|MHM%fEk zb@+PU^cXci;Jfj_MB?D7FSmCEc(F!Fd-l@Gf-aZ z&u6|?If9|wC+K&C16zp0iz$~adyQY5I?wDC^!wNpfk+3Rj0K!;5(G6Em`fgT&tWLG zdcd|zfk)^d-;D>LLRN|Wj z35S@w=gLz2!6k5D}!5N-xGV`wyj|kSX~1aN;NQvt-qUf!DLB_{@*^P zHxAla3RO+raV!UTy$3!m@EaqFh(c)tOYs?akkOvkU zMFc)y@b(U}S{dwfWC@4O1E$TY5$X!e8a-S+3Hf#jTsaJ^ix_M?4hp*{+7&+(D`HI6 z7fUub6F=b~vSk6c)Ixzh1zi8WoywIv$fxDN@Qgt~;sNIlhVn0|C!?}GIL;fmZ4J>( zV-3%i6kg^LAmF<=kR|b|l%fMmQXA8p1ANaE!aut*w54-79pp7jU@~d&lyRsMT5N3K zU^Ml^1RY_v17WAj*xC9HGYKhf@Mts_StPRJVUX8DNwI?yg^YNPmhv3Gs&s={YMPh% z^9`|U^Oc(?^EI1EJvYtJ?C?Hl!oT&sMfZcams8^|esPsCFE3cY$+nQmhJmH;0M{u6 zExm<2s}68DBrs1?5b$zLvQ@M*WMr~`7{Ggw|JgI)jE7=p97GN{aPw&&4`4_#SitS} zkgw|r)1m`O9I7YT&VP7ObjnAxSY^t;bnmGlkA1gzEMQ(FsDG2g$SHwQj5*dMfy+Qa z*0{m1_dqpsBG)-{foTUUWDZCRD|5^{kQ%4&mM)T$y`ER>A^*H)rVG=|r9@qF5=9sq zMP@My9V=LxbU9DFDCV;Q@A;`K%Qin-64JyhZqXd}a*Cc*v6~Rk3FmN7oIP9P(Z1P6<+yf3X1?FW5f-a2$rxXGdH}Evw;MY6I zZ}XkUKv8@LFHhY90j`JK9~SU`c)(wDfLm@M->*e1ceE|z5*X!_(@iqSnmn$ieg}Uks#^uaDvVPbCyLCAstfl8l5E)J%klGo|P@_J9be; zcgn2>KAD3tCmmzwzEXO-@K@QnUq&-nd9F`0ykTK|qq$#L)$m8tTT$Qc2h$Y$<4YTg zN)~9@Gm1RXI&9#0_)G$`NdwoL1tL6&w_o^+{cV{2_W@7WkBA~hrq>SaQi%eG7R=^2 zDDWs#0d{f{z zXfUi|Hd)NTb>jfrrO!*=J*)eiWpu-J(M0AD2H&Yt zD{Un#cbXJ#7Fo$F-Wtikq~PByyIuA2caCEU;#G{YvzTO$F>o6^6k6lJe8p3QCy_^Q z$)epeXPuKQ7cm<(tbf+Pq@loJ;1K`g=m!IZ(yVJI zXNXrk=rxG=<+I>KQk+|oid%J1a)jtEqabGHOS@Pr9^6>7NyO(MKT9Hy%|el73;DSk z*}hmXIw_bha!%4?`22&1&Ayq{N5QSK!FYmL?pp=sG=_e|2W&yc|3o&4DlT4N8tS!N zl{eWs4O*6|WS$d&U+1lIQyhLy0}xMJ$&F#~qI@>a3j6yp2O{Jx7KD z`@aOv6$b@>0{4~$%vbXCxE}H! zQD84o(0O)%&tsv8%vZj;h7Us3AJ3?mNLVMX$w==O?2Vo3vE#ad%z73JhwoYs^#8c- za5@mYutD&UBmV?x<{1Y#elg7c-M|xfTZE-iWI+Sdmyi8_4&5{0w|v#ZGUvjQ!~^Ue z3M?K6EQB1hG+$57ZxEc}=+SwFZPV$AQJbec)K=0vJ9Tc!uRq#X9F(q zV%#cIcgraW!Tfo}XYYT=a(a82BctI{H>*>E&%N2Ka_?95OI_$LKL1?wPuI_QQ;El& zJ30R8EqH9LaDt05?_7h}zeMqvgFF`;@=O}VP9-oi?Gb#VC1ArS@QVKm?;d`Ig2N6D z3|0iwKg!c= z;u|&?@i;RZIWYfAVl-j9weONB^FhuP$$}ruOZ*I z%Gp*coqq@%RQvMLp!w!(*6xH0>!VnMzOvaZZ{GAE;@LAElZB#1j^ZT?uh=_qaV%WM zl=v>kuETlmzuEeV0hRZdCoO-gzJS4K0h3VzzeJ)Si-f?d2mBfd3<~~fu~(CV*gYeT zOFZ1#T};HR~ZGi1}vFj_(*UGd%uZo z(jgyVRo*i;-E$h1&Hd-uR6G28-}y_*^~lxVgu6R?P>Sk>elD(RXMb5^Ul znMW+vvLp52LJz@BuBVDJ0-iVt%EbN>VJc)1*7cqlm3fL~b@)0%1(TdekyHKi4>kOm z@-lhHnMcNLcC|*I{vA9lZXa(~!*J1&o!7dT!L_l7{Q?ggr=mf`UMVM5jt$A-8!jwh zY`nU)(7a;r?t`p1)`xDNcZ!Mm^8HnsYw8#h7rv4%bGw?TkkCJug~uSmBcXkYEY}h* zL+R4PhYqs8*si`|!UpyQmcDs4Cl($)y}<3#yBXJoyLzYS2dFTe;H@{fd_wt*O;E5v zrh!K=zk#EKt8_KPXCBMPU%m(j{Qgip#ZE!3RU|kpc0p68Mp)?8;G~lmLc_IoWQg{} zb()4nB~4rr#;U8+=&m3wlXygRLCU0X?Ohj&P6|p_Ft{o#Vo>5xtmApaF4>)$c|&sk zlb~brCK-$#@`+zAvP;IkF!;x=T&&{2$ZB)Ifr&{mWWz0{wT#cDn2#3*I5G>I8WPwKGqb*Q)T>83EW%9b%02`OI^C~lX# z^32g)`bOGqHraLurn7$Uemo8i_$Ko8q1`Wusp|rNo^WuAPInER7VNV!W2M#zZ*=`7Q61jB8p+~`|L!8UlR;7VS zp=S zX5t_yr`)sSBeR07MF2A=x5L@z7xsV5{E~I*-^;U$n!ZUdIQcB7EUVXLS&+#RO9dA0 zjhC-|n9#_ufI(=DlC(+(BX3ebgWy>Ke&q=Z*d0|AxXviJRz65)H+#}0=oir>JS(6h zx@VDC`2hyD8xK}RhAv<)(`Zy!;Mf&l^H{Q9fz6;{;Sp7)$AbQU?rc9`*vHe`?flK- z)UI=_>uPma6MPyEbas7MXY@=@``4MZ2het^}K<>FkCwnGAm8ZGkw4(vTtT?zzSx8ypNCca~w1ekUEs`bOYuGGSaP(h{HPldg^+@dUQvU0E z-<4W&HA*N7G;Lnu$|`5Ec<&k~vHH6iafy2lutqd6>oF+VRV1|ON-1_ZwKNGf8XQ$( zFyyi>xGXGV%yZ(@1&L!*4hruyU{U<{fpPkw62b0-Bg$@uU6uk0%-$Xm>`VcT+z&ea z%erpO%Q`7#`}2JLp^Z`D;;hmWix`;p#PCJU{mj7Q&?wljgZH1KoIu0Z4><}_0ta}d z1Uq%MD1S4lw1GCfu z*0sh945Kc{ZIrQAWVbfC(tjz{u$$90R($`VripEZJ$adPC4N7(?%DS-Zr|n=!CFy) zjh-Qk%qkAaZ~4HeZqOuoM&sxzriX3QxgH4yL`+rxmB8U_@ND<(%kxeaC|+@Vks#2T zASO3sX}jf_gPc-79N6sw8aYE8*ku?J^ur??BX0!-ZuDGw(WPt7V#l``TT-}~Wddfe z&|U7wlfkGE64c0WqEvawtrM~G6B=XI3JYm?uqdxM$l!1sY$x;d$fWuUu!gg|XGmA#nF(t-ihz=|}82$=XWoMIt>Zb2JW=UDsb7 zP}!?fmHc*FPN4fz*PgYX4~otD*{GWGCbu!6Mbk)Z{{@34krspC^6l>F@8&%e+Ww(g zQOeL*BVYlO#196>dnwG;cN_&4C%DL4d}!9&!H~(}Bf_jP!QOFeT41wMcC)_hHP#<& zO#5egHBA59z;1S&Np^-KyF$TiR)LhmdN>v(SIX?L^X|7MzvRJ{a{I7vg&0vwpLqQk$ zLx)Y(o~#mCx*<;D(Vo2y6*>Z~A0i~}CbE7pTgcyZ!cm^Xu*=>ek;yXQh(g>f1M4%6 z{CXVO^RixsF1{&q+1xye^+QW0TTEzNa%9&T!%{D{51%+2ZZz{SY+$fn$ZlIMdeBK? zLPP$xgEmFoO>#^Jc`|n>a*0@Yh}>C_ZT=*OU*ZL$aez>l%M!`D9))9`Zi>AHJXdbH z$gQ}ym-k&k{)!T>d+oM1Q(XBa@7*yinD{y;{v9GKkW%E-evcP4*8BTr(46Kmdv(x3@S0^SOfKOJ&lRrhJ+3NBz0?K$vUcgkn} zvWzB;oQ1r(JxNTmKb)047{krJDsZTN*m-99WQVH?%cpg`vYhuXJ=0N9>&gPV)k{Bp zDS2km%hwyx5*pjvFqfC5iLYb-#rd6yYwV2$T}p%*bQ_v73>%s9HgK3v;E0>9z3@ZO zy6FuK6F9CcD`$4)TR1uDi38ue34Ee2%GW-PYErN8T9)$Npkn<|ev1ImaehV#P0`zw z@)Qk174y>!^Z$Jcm>n)A^i<^kClSE{M&k!;jT1OD7&3GXnB@yt6a-YH4H$1JMzX7O zr5UmQOMJlQx1f6OC37Ev*3t&{@&isjAK2RrIO-YLL<1OPKd_b*v)cwR={;bUC_zEJ8VTB}A{oW06ON*Frn%7H<3x|23wN@_fHj@ zA}$)s<~BP#hUs#p&jlTc3yhKv808IEQwumF99q7GGHEcdE;Qh3e!$YCz#4SH+|q&B zw1D|*#lL>j1ZJ}j%;py)8=N`H7ck2?Fk4PwluZb3eo$m|A!72Owy@9Yt0rdJJ#XVS zoA|7}Hc&Ze<3{hs7x7gmq%0EL4Fi~WOycO?!0!B@gRMbb(o8_TfZ_Q;0SjT)BO5pv z8>B=6nC3-fZ=NJ{iGkB{CR5J`p4j7#XCAQGZs0i0!1vIAZ)1m|#HA^rCR1NFbbngV zz0OrH@@2QI1$)c^RtJZk;}t!6LDO_Ui12Mq{ahp#y0IyJtH_6F#7#knPY;*9r_<>pHLR9gJ1w- z^#Ps>3@pq73~USx3XZG}z|8s`vnrEH{xvdX)mF%Ci>&*Ztic_?B6op-v4KJKfH8jqgIhbpv`Gvc1&sGz z&ehA5%RRu6bAj!W1DDtVHpd0*mo{*ic=Bi{L|zGFVoPLH|G>Uffw%Dh%aH_jje>bT zntU5yOnIy@^>IRDkJS9nFNGg}D*yg6YHtRAX#jhi1AEMZl`#sO^SBn+87#1Pz_cW4 z!KoWed|P|?uBN7#7=%bI6fj^ZX5jE%Fnz%U?iUyM-ag<>-cTs>iltS6x7mSJRbiQK z0kc^EBZ~q@QHej5D*#XyWRh49!JLDmB)* zZP!@m%p|VByg||1|CaZT#(&HX4eW*!W|x`@@FXxCn#kb4fn&)84ha@!RR)IWEGFqK zJV!3D**NfYGBE#e=e@9i{cXQ`PYHpp>XQA59Kuh>?IRcTK=9I|8nJj$!QJ`n4P3oEvo74dax0bI=oII09#{VuRK1TcFv ztFtsD=q0e%H^j~Jl(m||=p(?PWyTzEfho~7bNR~&ciW}wZ!ox5nYdedpE)>d!$*x> zqNXoCwhJ_VK>^Xaj`Uxi41B|i{tXvpa zss-3e8@XRB*y@&0`|rOY~}$>_5q7>vuAl@v!MgC9>WPk0cPg|tP>sB_?WrY9^hV=z?Sa7?YfBj-|GT_<&v#s z1xpxMwmLGb-Nvvca|e$D^Qq-U0R@al4zn0PNLr-0%vF5J{a3|p)x~bF+J0W1_59Z% zaR;V%T+)%uhqYxU3K;CGb~qTo;CDsHWz9#1yARn!3YeE{U}t75T`E@3`N4s$fGIjl zKvIl}M}Ud-pg^90+)@Xg%MLuY4U99p=P|1u+mKP7^ylA{u33B<(T%|``M$m3-(g|( zgJ^@w0{O>et^#zW)4~%ir zhEj7m<=1jITAFL6Ui!ODPUL}lmuIH_UIyMCby)}I1|4S80LCLn7y=kL)MrjGx1Go- za>&tZNl$fXrt6i>A*UCp1?rtqpBgrM^+`s(fN1Uy%smENN?Ukx96~t-7&^bIX*^>q zZ9RLBr#|s-$J6jS{myH=hKZLIcxR{PlslB4{lN2;_x=yrfZH>3%VPO%cji7Q$Yu27 zTdCUp&Y|(^9`?L5?D;Y5H4j)VR<9H~w|BzZ%?=ALNYp(Px4X$BJMGJ=oBbO&Q(P}#u1EmzB_z!1@?Q0^DN;6K@GGMmGNT?hJR&Ym-8kvS)$c_(Azgt~w0k`6l` zmb&}))bg2`8a8*@r90hocl>K@-#AfP`_0LWtr|QBGO8I^%osLoWs6-Rz&zQLG3P?* z6@`E^0ZdB{8M6!2D_(fUUC6+%!1SW=?2E>0rT-q4DX<0#a9p}zTJOMh^#IRVhS;{c z7Z;xL?BnF^mWy&KbGiS4=l%!YryqD3?E)4WH?psLP*HG$=iLLrc`M@_j*Hx5;hg*M z-t31JI$PTIm>Vn12yswx3sQM1Ara(o?u-DNWI#r70<-Y}mhua%Rw>M-2Ux@ljCE%% zvOdR9>MCnJtE9J})b3vB$|n=ldlD{*cTT(Zq%7jld#`m7b(wOVha5g|JiNdmJAorG z;2+bSFC3y9FYzZZn?GaATfpwaz+jkpEqN|O{yzrx50i@H>OyiD_3N0W8kkIGa;PS- zZ~eft^@6{E{Kr32{6+5jpL5_5=Z`w_fmh}W?@@!73g>g5eqf*T?&Wpq<1@W3h^%9E zTEOb?;8TL-Sh&ZLzxCdGA|tn|-!jIisBdOT<~0x=<#c1{RqMw{9+E4{~6<+QRhy$g_{vf8_B! zS1MpA>64rM@4SHe1?ESLg%^{`?q1~C`k-n)zvankE{pwn0$jNKzqoj{{PJ3oz0RNK z)Pii8|GXCrURDcm&gr{Rz;pc1&dt#WSPj>+CfrzoPV2;k_;s=e*?EHK!1p-$No?l>F z+hveta*$K4f$1fui0eP*R;$H1y;)y8UWcuV@ywNF&CxdSIB9daYh?u5rTV$I2L@h*#OF}Q9foa8+xdDrq6FCkrG|L6mGucj>$sr)DS#C6$i`C=8 z95Hdp6d4KGU0<3m&NnXj5u?!D!6&J9=0(5)hc3dCAR8*-4`;OoB8>K zO^mHqj54dH8F45!&TSETwVc`I0;l5=Ig5vk!d@GiCq9|feTd20Es=AxtHjL$XAg-+ z6&FEwl|YwGXPDI1ZkeL!Hq-W>(!$q#b{{ry_wU(pp?m#4CgDY$M`lfE;XceUxy9o+ zQ?$;;BPAX^eU39Gx=Cd%=~0`NxnkzCnc7SDe3EpVA#ko$DsP9PszV*a5_7klhz2IM zHyPa`4sR0LX1Kmtz`*G+gR`|$+ABlPic4%m!U7hK4S5SV7DRus?30o@!N?{s>566B zy4U9v-f6Y2un|u=(7+(=rs&FLJ;9?z(wb$#fhOs9EG-jVIx3r37=;;5sV{JnFj&Ft zcR_RIJfRKAl8zlS+h!z5G!}?l`NrU4@leb&MKEjq)@xB7VudY&?g~G@MHeUvw#6j2 z1PZRT|8|49-#X(6^ZLDe7q*)mfg^PULm>5&|t(_v5qH3YA<=U=y)=u+TBCUR2-)yOV{2#*SvE58zsJ{4J+l34 z?O)wV@y&${P294xejI+7W0maY6wSbSblTKs8jAmzr6pEg=Chjb$=BsNEpbWPyoCua z0(}!-$W~}dC?4%(Vc}qCWLY5A(ZH_bFrUdqd&Y|8eXF#8UFsAoU|7_qG-1mN)^GVM z6gjve90VIKDR6TfVKdWcY;iGWWMpUxY0wa1HF9W_xsbqFHiMC6n$a2-oukd{0vDJC z3YvCz9%$jTa^zoN*qbBZ?A3XKL0al!kLv|R{>qIDdA}}Tw|i4i&|!TZ6kk%d}l}k~&7Kf@SyrSZXCN@+QHdqmeJhfLYe1Lo%JmrDes( zR$c=|9`hLu=XDyIghb9WiCl145z*SnrlMdjo^UuzN#q}^t%q}HdBU+wx57pig$718 z4hHT!qAhYC6u1IsG%~OmX#UepsQHr3!DF$Lfu&^AWPuNk{x+KX*d2-(xFs4GZnHDgf@*O4>@xy(&KI^oR@y4$Zx$Toxj?`rS6-t zU{}MPr;nT#@4my~*|KvHe@z2(o7t2yiwQlV8-M8QR|PKoyNHo@o?%V%>uTn<1qWB@9bo8{Il!R8 zz!3DQfn6ZOLHM0WYtWS@kwZGIx@BIoECUuWmI%3;ta4;#;&{MlK0`5J>B1Z79?iU* z2~RkW99yusVgrl$hE9Sk*qKMTA2et(_nZ86aFmrBTFtJtW2rO}|J-?EH-{T`s&_7!x?fwLoIz>UQ zeS(h#6%|yOm>3#3cX;!^_BtYKb%KGRVCyOdgNATL0~U4#hC>`bR)sTv2<3RPkd2SQ zf&KKU1}2dN1_6O=fr4WVR&O4%OKTm~U=!>LU}>se)%95Dp8-nFvqG#@`{3+^bCW;4hIgf7&bWalqo!u{m{UXr`nVm zI{TIhiz4s3)?(>T3hC9ig84fxFx#_#Y?r)vaC=iDPf@HvhUAV0=C)~!>_WXS_(ZJO zj1=;mXIjU$hgo^pXE3sv95Z*w;SgwTNY0ix(C8S*FpI@NllRqy4r`Hr3e3C;4$M^t z7Rq~EFnzP~0E@K%gN6e?6PrS6r|JbJLscf1o&<+Hz7GjJ(GrKb6Ap$p>N2jcn#x+e zfSGev?KQRv2PV-B2V-#uCbp6WMs?o?wjJfnj7I_)Bomx^jTbPg@f_lBQ8*%==Wt7r z`4JP7#YzS%2KA||hbEk0cJVs@-R#Yb2HxX$nx$tPWRo;w7BH|l$Zc}2N&Ex@n|=c$ z&#^tslNR4RV=KWl`|^fQW-Au*>Q6VGo4b2m`A0|h36greF&ADa6)bq}%20J7^g=P; zk3BCs?7l3Bb%=R-ih+HCn}mGP1tuwzpWLT6HcG!SXqQh}z!bmUfj!27nO~scADi!s z2EM!n&GrqAY_bOq3hMrm2vt)1*E^ug;dXm6qvZJ~#l7W%MVF9zr1ty6(0Zj#U z9jxiT4lT+&4^P@he41k3z;Y%kOi@7bz@z2^a%>M9bxjVj3UD^?GfZUWH(1DN{KHwp z?n9$=z(G!V2d28Y7uapSH1Zg;xvZJP#p!Uz;dR0ZN7ff2T%w?50}lU=eOEMt->Jal61O;0 zZj7YKEmj?kriIMAJ~uQdWi%*tG%Vn@TKwB`!DL4M35^^*2mCv&?KU*4?_hRV(R}mt zzXq#K4Lk=LI9D`q3OLH%Xm;ArtoEWgFrh_1g2|Af*=|R(y+ms|Lz5&!i`x(8Gail# zCm5|ISj7G`t4XxDpJ3efh&elh?O|e@+5%>e4a{~5t5bKcO8cx8)U_dqY1?`w)y-9m zLInma6Bw8xTA~=5-2~YABpU7%G;O=wWS_w@C9#G7Me~Eesr(!*jt)(H3z{4in4Jw; zbRXylH#7!ZV0M_mmd(+glh9Cj%5N|jBhfS`vZBC7+8c8nA{k2 zg$x*lemIF4F!pKNPjI&XBD^zPWoN{lMv(`NA|8!$9*m|Qjb;(eKfIfjv!^#55^r17 z=>G39gBXJYqm!Y?VR@#5^En!hW%wLm2sx0Y!?gTyljVyh-k(hB8=4IYn)C{q3VE6h zZZsXa%&7lhg87~<`v@j=1!kuMT+s>aZVwtb7+cyTTevS#Ng!1Q=Q;r4ucFzFk#L7Dt?_N+N>9x5{erb z85kHfm~;i2RX;HC8Ej-Y!FZR4*(RciC!@*X1Vhk<*0VpFcsiz*t!{9%V0Qk%VyMAj ze(>Oq3YPT5Hn9e#G!7PvR}RuU8l7G)U=e6!m}$rTqmh4tg@Z-oe ze#$<&&S(FZ=6|l6rW_UNI2x_8Q+c<&^ulJPpNvns8pS3siUl+>@i1nbIej38VZVv$ zKgouNZ%%VeXplL2F+HVWGxH%;1?K1jt%|Cb6xX)pda$c+V6=*0G278-^<$|_M3e3h z=0J@W4+$296AT(#nM+w(HVHFG?O+a6V6pkp5c->uDWXA5f;A?CEs&#G@YD_ET}THF|z`5T(- zPc*1TFloMM)<3`?y_kXVLWBK_=5Jq`m;@NuJQN&eF!CR0;4?5c|6nKbVWwn7zq|v( zG7|xIg~pJRYy}hAQvM|}OK<3LvtZP(W(-x?-+$RFZR-V|ucsz0cIDHx8G<2FUuxs3$eAa;V!(}e(o2(I$dP|&`IT#fWFkcXC_L$J3yMWpH29v%3 z^D7o+M-Dq{j;WjlTT@RRbbiQS@PWx-2eZCH!x|+{2%Kg?$;z7XP-lO497$qJs zu1{h4UFFR&(~c;?wI<3mDBjSj-*d8^HaBETFM8v|*t`19L#bOIGcdvXPxr?oD=Om>g?n@7fS%#30ns$oGMP z{{o}b1xCIdvt$fT@}FSRuVAubXyM}sW&FS(HKW0{f`!YYfhQqe_Ciz0D+M8o7S|b! zCyIo(=CUOfG<$1B%k5}nd2n?n=RZd76^w2fY@RA?jt>~6HJsB{oaVS{A>?pv7pvv2 z)N5a|U4=PLuNT?5Ma44ePNN$`;DwW6f4yU5K3o%FJ+N@|={+|6r`FD25HUsZ*qzg{ z4W(E2{Jpv4aI{b0kCx~ItR)}XE^SS^^tCOoqiyr<=Hr`YKA*v`O1fcUjcW4_M(Gue zCS{Bg0+XeFY!qXdvO&^-=|q!(2D3d!Gf#pxb448IiVd%2{qi0?e);wC%cXG%-O>rS z<9N8c)-yG1F^(7KU}%eA_;R(;JC2EK1*7$g1|E%#JUIRktR1XV{xn-%=x_=Qc6m5|$<6r;0u3x0k2_EB zw&tlZNchcQ2r~4z$G=1J`Ps{E+v1$l{Q93gp7d1BF*b7YR}Z!YyFOiI^#8P2GJ@$_ zZKm(OX2}-~TnieE*D}~Du!%jGKRu%($BIGd2D96YW~YRfz={^VgeGGx3$q!G9Gp#j zT#XDfJ8UCZ%q^lF5*|ntEJ$r&=nH-(p3xZc=hUao0spf1MNc%|pCP(4hwqN(R(reG zy8CWi6Ef8k&tObha*W9&eQ(dj3?82{-Z%SX&i$%XeXvAcVg+-M!?NfTtV(BFkKenV z=g^+l!6Gb@;e2rlL&H)fwWY6j%zXYqW&Sti$-K826c}VbusB8J8dUT-*4-wa7I`4}3i(#Wr_43(t|OiT@U#KECy|62qfYcjx~y{HH42``oBbRdPYI z+m5zqhF1R{ExgYQ`0f`({bFhoD99J zE%q8MEya!YD;PB#HU&uRJ-V*3eC_-loYDO|m^D(`TIaNxzMReDFkk9Iv-^%68%i1C zI0e@1VcfE2@kEA})QmRK8;znDPVM5|Db~Q~#_qi#=Ha$^MYn=pTw@+H5Z#7co+MIPfB?%b$7mVWw@jVXY`8?ejh&YzHjkg@inSH$@w3Lr<7vq zjnEFZ3rqfLaui?NBQ?G7T%2=T!8r}~2F4c*Cmt|~XEd0m6e-r1eiSuvdEUL$+ayIK$=% z-HcoGUhn;Cv~Aka+_ckn9WD26asRcJXp6Fv)3Zv=2~2Cb81(*hvt9VR35<3dSfdSC z18x+A7_i1XFNiT`jmc>7_|fjVqgiGLqs9zojT4NF2_MD$wHPu!dggs=RA5-e-S98` z(VOQR?s8}}q}^9xkf`3qWH_<4x-dMlKa8PACS>z1&NHSd<@e@?{Cy;4Ua>}>ahjo- zLqfBaLW_(FYbHn2XF0Bg67n27n1do(+*XLKmt@UoXxV5v|NK0Wg${=71Q)04pg$%6b`8Uus|bhjbgwH#cw?l8*L>eu2wyJ zBYi97)CASyB`*XP*%Urn5@_6VB390>hUeqKW)6Ni>xu&kcaq)tC7Kg?qD(R_%=Jk4 zaN(noyO4l_MZu8+51rV!#bml#8Wu1yWla_gb1o2QTzHaoef;jQ0|kuCVgXV!t-f+z zRrjB3#ro>W%gf9C=KI|3GCL8q);e(s%O+-4fd$g$IR+;k9Gr#K%sC7a9vxw2l*?OpIYEvzeslK5@vDas0sGEH~HXqO+itixBg_ ziJLs?9W6dM_%hl|a0nL*a1~|Xv}tf)a+lrE5U8aZpcKl)VUWbcVRPU>8{d){4jkc@ z7owQd1HL*d9eHu_kirt1;B(gZH7=)Eo^sh47TvX{)kCJ_!qj5|F?=6YgAtOR~3oaEy9~!VTZV;~Gm19F`i5X$plMnfM* z+mfm7wn0Z7I8ZQVY1ZnB88u$@R&si{b`>9-7M0|^`P)A@2=uEc9b#5vcyx@};7#DW4*953hhDdrvqL)>|DQGp3;!sW$H6Xh^)a)$ z#hDMx;tWAzHt$2?_a42uUwNsKw7POnw0!#+_i2&K3XOhc@_afjTfQSQGPUSY&2sLl zcPE`VT>s3JV&UEK>{iwUHixIRE~)}Q8Wk2j<23SVeJ^$i=GZak1UiLs?|iuLH6kh0Kfr2MQ!STI%u)*m<@rV5w(d zVf(PST`2d6GWP_JhJY>XK`aOV<*Ir(aCSJhclNAPnrxlK!Nb7Fyd_c4V1tA7iw{hK zwhX=^9t@9tQW}}x@M%gPy6)cBG*xONhr5;1$-p;Pl`UTtI-4g+a7q+3I_faXe6V<; z_;{g*tJ0;ao2MnB<}Bpt3+2>PJl%Za#R5;OE$3@1RyfOMm2Li6sK#b=V`fIpvL+4= z-hg#6z07~oHLi-Nn(;Io;1$xCvnule)4n~ROF0>}COizhZE!?)*2NBk4O#*L94yKU z6gc>=o#j&r300VpDC`!X*cPc`@ z;g_bfPt5IYW?rhRd39OKSwO?#?+bUeiBGdWM2Pe{Enq7XXw~F#WaUjbz$E%Xl`X^J zph3kVk=q$9y0a9ytZy(_n{8lLX1T~J()NhkZw0IAw2STAt~NukeTaEqOD!UAtgPLmV2HpD%>0ws^2*Dgl$x06pJ{bv@Jq}H6&0)vbKYf;X_B- zjRpqJ6$j{Jcy^5)Exw_T#TIQ~ zRXlQ`ks~g_TK7N$Z%#uaSBe9>U%{aYqXapg1O|bYqYLG>R5pJ%XqnF~a3D8i(}c9h z6lIPIeuhN{c!VrIvMt%bTq7e8a49Iobww)!!?eP7hCdC0-3@GPOaiQ#55lA_F|^CA z*yMKa<2z*?C9ZD2>h=9nJ(VX-U(S}luy~S^J_}F4zf8+#j{UaR7oV^DS3Li5VE@}H zak+=0`YI<^^Y}L@)|s8|;eXT8EcN2d3Yix(=Z96!;#l)(>bepwhS0@AjRzUH77DmF z8DC)JD0sjm^WznJn35-JN3)(~!99TInWV8L5#kg+i!*Z+REaHm~ z@OADupv(9{j!7qhfldgQ## zq9O9?L%~Z2)Y%e>*fSrCA7dSMq`5Bmt$;A7qNyzXgxLeM?q9xdG z@lv;d$)?8`+w(b3n`I~QFa%%o_;nh>-IeQd1C!fB~J~i&=QvKfe zCBHm(&HuLLjPu$R*QO|N1$;gtV&cHaWANju=#2(8oeZvMi{)aiE1J3g88MuZ6o`Gr z`y-1@#Ol>r>t4Y!h9k-v7gk67&<&=%DVbQgatm}-cuR!GP0Ft zUdU|En9mdDarjJ^z2anfT{ZOW0)mH z4yj*k&|j;_X2Nhcl!5I|5Hm;50i6TYZ@k#=FlO9jWc|aSaHV0_-AUY69C&Xu{Nvwp z;PKvAMzhnoKO9)Txhpxa@?AaRYvN?B+T<4LGTHSA zN&~CQL1PuBt_3rtCBqL|L>^i?^S}?m&j$VSCnJrz4sh=9{`}1lv2z zX7`u{^B$XBi<^J(-m#bWluq)<1#rvGn3MK|=k&r$BC8l4Y;5?Ku(0H5f6&#(b*x+r zrVXAt9GnJ>iZ)&9QyhhI7%Y4bu`7a!l z$xxBKaZrYZo8Q5a|BEc!mjk^z44iKmRXh?{><$U@9AF5EZa(lNbt6N@Q3vjb2JQ_J zJa_&z@FzI1UvZFTVC~Ck*`C3|q{3vpRnkSO;ze)-uDVv{5HNPi!{B7jy5I32EMwK_L zr$TQn-RK~CkDt4PfqR03AcL>XD}L>bL0KnazL;LQaI-<-M$?s&L#iQ+qA~{!CLSzY zaA@(ILnd=>2w!OA4LNYRcgEeT6RsE@;MZ}|)L}Nf!lbF=%%$tZTQTjTjRV7qBTOX> z0v8-LZXP_bp}{L9dYd2vg9`%#PXjlPAMYIp{xgldI~aJ+Fv`7f;Im-lec>V~Gtpgx z-}43=!wp80e}7o^J35P~9AG-oAoqqbV=JSN!I3T*hr-0x=Z>eJN%Az!@aQ|CaIiz< zr7eT;y$7#Yg=B4*OkPONKf2T`OWE#8bIUX5mP@L!RgdSNJoBT%QHEze{~Lz%H$0+I ziWztOJDc3N-}rRRo0O5DC??jJyiF)8f`Lt^QNg2E^N(ABMtjiE}qq2vyMTRrG36t`PhQN*^T`kLRxi(7cIEhSY*l>fx`OITa2Jt|VR|mH! zgo(-Ubfrd!u|_EI|9f?J!R{7^y}HfT1}o;LJ{Aai(NdPRJZwelt$Vg>IOPM5#T|2u z7dWW7R!g*rEo0At)$20SFC0j_vMRm8P0Ke?u*dDQqXS1t1d}b71_Qf>%JB_;z8mdM znXohozi5#C4*xA;a;Lsytf@ z8|NDyft*IM97c&NhjcxheBv5p-yB?PdRSJ#K_QFLxPaNZrTRRhCPrWnYb;g$<`S6m$0;lMO!Y0=D(BWvgFc3|L~!XU5l!r~0`vbe=D zTnk&=GM2w;US5~ALe5FW<)71ujFSQ$Oo9zB_-0&M#kDM98^ao*(6zrD(ij|t4~PrA zIB@w<1K$w_-hxc-JN%p_4ZIVkIyE(EE;zJNPWwjHTJr#Bd4@*W62|Vc4GK#FlpGvy z$8<<<$u(Yb*erzE{7#d&2-6vK)bHk{V{*?YMEmb;|-)vB2kra?fk;EtGQQ zm~kS)C@W&tyg45|JS#=_t-E$=mvh<4d8hM~mtWI4dnxDarKd^{a!#CSRXM^a_}~E7 zidU=n-mX&0<uUu;2^I6Ls@OH~92Z=MYmNh)v@Jgeo zMNQAdx$IG+Y>5Ng7Ew(WCk2i}CR|M-J+T^BoJ?d4%tV?bcn*oZXkfeQWE|qG!sEdF z#a}^6fTci_kEh8jqS?swuttRo^M`{9FB-%p4#|l$cCkW>_9=_F^M>?Vt*m_-cMg?K>A#cr_T9NRZK41S&L2C#x8X(Syh~{dV$up4#kUm7RyfKwFlubwD4x+Me&F@D zKK;2@Jo%n5h?bmPR?$@OWQC@NyP`h(Pfuov7YzU8cs8&Z95hLEFJ005H;`R!!BpKV zOnd${itRWcYvOcHtV!JE0PAF#+1K>3NbrgqY$#h7)wRvxYM!I8 z%|QhlC*G7tsj=5OPh56aNf0Z&k{X-pE|C?Mr0S@7r%vw7@w%M1^Onjj_4ssR!G)ct zWptd38=Nh6G+VIi$2h%McIr)wT)y3kVlnR*Q$IAyoMHXa_L1*O@z+xaE|#@jVq*NU z$x!x;kNU|T^_?eE8X6@U9HsLczi;{|-ohvH8gj}z4)U5zQ26tn-Ih(R;h>PtLD>e!mX{6e7aC@N`Nzr3@`%;>08;^jY=g6I zMH}CP2L1yXK93H_&Uh(5?RHhi`e|`3$gETkgaH75pgK-ZYbD2_vE(#?kfyD6B^QwFmV6d;=s4%z_c`l z!*<-EFpO(u>9I}91v{v3$D%eci>R#W(Zrg}N6wt0`6Ly53?;rh0%U)7kW z91xCiV7YV2ueia-EpL0l5uOKSs3a{5u)v zU~I=I(tdcatcB$b)6-##KRh!%v&dM?-J-?cLg_@~^3#lM=@k;E8&)Z;O?ti~efP;( z3w9`)9J23lQr=Ou=6~~Q;ij(t%&EdoDf~^dzA#GEII!Je5WaD+@3w>R6KzME*YZ0U zH-DG12tB}agMoKJLzw}ih)d&!*9T?Rq_IC?;Jwf&cZEULz}a}iC)o`S|CjvsUEPu( zWzJi1P`;wr*kSwrYX>znFj&ws7iY_#{@q4SqlYg|~py1z-|&tc^YR%^wb)+nGds39mhk-Gnf$u_t?1rhb6PO=tayI_) zTla2BkuwAHmILefoF0cYn?^Vb@;ERG9FRA0X4(CB>bKoqXFS|B-gw4@-7<8!rMdZm z=%3CdkqgtVyvVvzv-{or<|%LA`I%ln&1=WNSy00nixoQiE+zm$K~}^E4r&B)XfBV;*)oKrg?u$-my#ZN|VMNrqAa4Kik`1_-&(o z_?H61Z}uAwlcsZ({=2}Xe1(x$p;7kBLEfC&D-$J}uf|@vI+vGWyZDWR;uVY{Pa4GU zG;X$YHr8P-`Ej6V_fD^Z29+ZW4i7S~t+}>8u-}45BH%#tLbtX>rdk3E8Ck@{#S9V} z5*nG=*?2Yx2rW4+BVU&DV&%t#gH0?lvIQ{>4asL`a`Rcdc%h))G{auF;>b&d7tQSK zJxZooR}MBW^OiTT5NvX}DkL8Dq9NqeER*c>YZ`xR@y1vbJwCQ`wz+>?bk&zvcPDF$ z$H&(EeE00~>hO3w%lfA)1Z`$`Br4f8cuYF7luy=XqLSpKJ1^oFM{UhZy|!m&@$<7Y zxOGn&9A*=-E@R?ka_JIO*0lWh%3#xE#lx*!B3d{0H9TZ$y|+94#>M32{d4ubtxRry zPUc%|TfxT5#I#W9D4VFF!;Q;4F}f!%us@s-u%Ka@al(-=7lT?3v$6>k1qe7KFf`BR z{ALqS9>P!Ou2f0f!9R!8MZ32N(`da-cdKd4snALB;ZdtOfF>`V=UtYx_KC>A!64}(E_B~`XXW96; z^Pa(pvrOCu2NKeGShgzcaAn!~*u?rBN6JKblPraQO^g-~4(xhu($v7h7V+Uwck8jq z4o%D1J$!^@KO8vhX_HsBNYITTa*<%#n-{(fwsIR6_sOKaNNAIu!MT)$w_@D^C5AyF7pdzimo)d&#Syw zm3@5YyN%rQ_xG*&d%~jNyAofUeD}G9TB^$;&h(h;)jihhHLrW}c=68E37kd`)-o?J z3@~pN(mAo9bzbdahU$6M{~|B&TuOb!XUnkhasTJ<8y;s#HZU}?d70mxa{0U1goTWu z+z%Eoh?h(dXf0r1mp#z<=f>eiE}Mg#r5Y}53mkf^PPj9aDY*VqK6H@B&7_IbLxC;Z zX<~<~PLp8w0|sxqB#Y@ER5W^v8JJ5ZG5j_-r|h?JVX)r?kwqmdqLsC-Gy5#rS+aG3 z;NlI(1P@;l-6&&N;=4t>_tGQ2iX+XsYi|eIR;f&Hx~OMR)N4XV(0u6c!=UKEa zH1a)Y7l?5?kPikz7k`3*$x(sP zlGBSL<%LkMiK?<p2xJTJ+1EYKAp^Z}fp>fLZDjIo08kn1| ztZ-v|D*cI7tJ>~yup;kHV>XUe!E;xgXDax>QRv|;{ql!%wZemjbVqHbz-$Kbtiayr z3?C6D34g&D15Mor2bpDZ9ohF?Xyw=8$~4~K!1UrnhpkMC$YlvvHoFcMa{*_mz#YuS z1_p<-cweY8GcoLxQ#j0UW-8O!Nop2vo%QzWu@*gS(eChQXj~PN&(+|8b7@>ba`o@cyB&}P{QtJQ#g~LkU3OZaCyyvSU|AY`U>0CUipg-kP#EMfH!q-N(a=w34`kn~<}a4^ zuG^F~dwKJ}Wl0UE!oF>HQ*4knzRZ@s>u+zqPQo)Tg9YX(X%6SB+!}Za8W?05m}I*g z=FWNcC2|3aQ*mK|))Kb`^)jqXd?gC|#hxT^n{+JYle*C2c*1C{_zMQje-lnx-%}Gg zvY}bp$+6cb*m0J&3Rmbi22M4O+;oE-47^IQ%=`w4f~PwUtu-~$TYO+@j2hF#b*@`B z+r>U*Q8;!$VVX8yVN3Gc6L#M=9=`bIh?}&`=S|f;`xfTz&6bwBaAJ4Q%L|V${7V*i zDz+)@)RA}5GVf2kOM7efz~X(CzudjM8~#k5zvrE$BCO z@8;bP+AG+Q&~$dDL!;L}UPdPW?gnPFc@vc`FtE!pI0&v-Jwcs^NuXMwWnNUmQ;91J zIjuU*v6p;kSKINxOm~LkU5%>^Tw4w>^=P=BXWkh9XW#W!yNwM-`3B59S01qGt~kg) z$Dku{$3MO$9f#JLbGq!9bU@>edN@ZB@0=s&4tp(_XrLh2mHYNe9^+KqqYHRtmaKiZ zDVJ&eTgf>K-|JnF+_&)k5e}nMCls}slkybrPGH{}bb9L}UTLj-)|YEpU-C-p==aTH zuT;$v$1z_cJC_e}z46ayE-Lbjv>d<_lES?7b5 z6qLmtFhwk2WK&>z(adehAduB4;`VQ$V2%QFMgx=815SoSwnys)XDwhjrN$n0V4D-8 zK*~B(iG|EUjS|xqio8=0nRk$H#zD3vD>eBJTv0f{C$Z3yYYn%=2~LSbks}M`_c#dL zZeS`o+rB34^`_+OyZrt)W%zqFFnAs~I4_Y=GT`kL-kS#!r4%;2Taz32DUaD)fcft` zriBlgIA4`JO)zdt_YOM!aSqSDkO`k+CVbLcFTLmCryNH91kQddtis zeTjdb@`E!y(uNLPM!EuT67m@w`4=Vd*DT<&=Zspjfsx09y?}we&OvT=gUF0V$wLAH zHx>v!JHQv!&z5n3Tk0TlSHemwhwW(#1o>R0{w+$BxwMe!kiWne1|g4wd=nn<{pr?- z%V2a%;TK~RcsEf{YylIC1N*xJVn-6?<{aRj*2rrR_}{8CregxfVOO=>Nm|F2y4dxw zcr9R+StA)BD0-9cos&9~`iA%G&WafqXciX=C>{3FNjA<=l=ADY=yKn>>fyZ*ZMMl9 zwe|b&?P`=h@aay*{QF%f!w6ZGm8ek;YBEn{(cZCbuwc))k-o(E6aT z%|b`XeZnl7$s%n&A7x%Gd~o7qLcymA7qxdX%I-L*t*`Sc>YMTpR|1)Q%I{PQ{} zwxV&yhi;Ky$s%V`kp+fS=1zWM2d4j7HusUTwe5TXY($FMV{~?_YzzmrJq50p;0= zGDi;jGz-ZbQ9PRRpm?avgJ8+z4N~*kx8-dJa(vbAv1&q{@|l)^-yH#`a})v>BnaMdV9rZ0 z{jz|GEn%I+yTG&rMwWz!r_MZ^w~8|_p_wUBGN)0{?g1OqLpkX$Os^6Izx1-PE#wo? zn8uYTB5{zvb3za;#d%qTG_ zQDhy<$4h6mShsF9SRWudHBk3klJ0uD-9n32B`QcS_`L2+P&xM*k5vJi0w2CQwcQ|r zPi&!}hoZm?g({s4zDqp}ObrY=3;4b*sGY~ak)y!g(7^rfL2lMV=8j(9jz%eohv73B z1(pT+roCPn(spK%f>&g+2vehA$3npv1&vLug&R5xH4^w@-Wut52+A#tbyF14nJiiJ zZk5$Z4{Jkf7WV76O8?*c%H*-&jotyXnGbz}3vW)-z4dD4tu#Zwry@2J4@yp8Ol)M7 zkXh*8&Q*Em#fdwUlXibfI$BgLX_|c4NOJl{=~WNqGZ?0=5ZumTV$P)*z_GN{U#nU~ z?_c$)oW*GjT>l*SllqzR81~1kXTA2CudacYEm2U8QDEKy)|dviw)0|J7D_s;5zIKi z`s{$*S7!k`pT-ymZnnl>83#BydKfeoJYqP=t#;s{9s}bdB^H@>jiN)dQeLp-G;q#4 zz?PN3R<$5?!2`}IfLn@KC(&pkT|wq`JU9m2Cme=8{_<287Qj)-973G7Z$_m(|nn zTe9Z!w_GXqO#fr-9MRwXh2EJR={a-8N4chLg3bxwo@Et`5BaaKM@cerWi>MwEf6@h zL@=yau;`yd{#yniw?r9zg(+4G_}(4c({=vqj0Xzh5Bc7#i8N@^`r)t4&eA4zTvaH6 z?Ut(Gy9F#ijJTQ}%*b(IIB}-E@Ac$+i?a%H)AbHCoX&h*ag0U8V#ALIx!AD-S$7L{p(KiN-XsC5X~*rM1J-xXiy^g!1bwA8v1n3Z(WdDo4_c; z$(+K#JWnfYQWu-V((pvKR}1vaZBE*$G)z9?y0DkK&E(RRlnk~AW`TK)oHrUIgcMt< z9Hi0?ESP@CPN-qBNR;xItT$B;SkfM_do{58F;DuwP0Q&4(~ktftc4|R?C+V({ERK8 zc;8y${8Ri>iNvHt3BSm`gjxRr(tmwQy0J`Bv~B0Xz{`!>#II_vT_|~A!IN(-{Y}~d zW~b~$XU0^_FymSp@M6;$rgIN#)VQXZ)h=-`V`P8hHEsq#8o`RqUqtKlNo6fo(dU3$W<+v0>+2pTz z%m*h+I9V+bpT5PjS|T8Nso=qisNY+8LyOaIF3aCjoV~j^b4T0kh5Sc)%vm(qZN2RM zuetn~!TIPqpB)3&iU#v7{b$;`8Si!1%=x!KX6A)81~r-QzOy$xU>9KKXnDXXFqvB= zF-&6}V*!J0cA`MZ0>KqU-1qK?Sp{?FEEID|WSZ6>a;9r zAa=~)hGc+^#FR#f1X;iy?JnYQw)bH^PR9AfeU7oJ!3IA?(&R5XxhPd zJ3im?i;-+uNNEl z3)mzuYb7wRuv<~dHbHEr)MEF=7i#&P8b#(ba56RuXeR8?N;t)N)Sss+;7KzhOM>0I zovNEo-qlo^^3I1%u2I1%cyqR+kOPC!5eN1X2HVw%OeG7%W-xX~Dm-4{eQw@T#vQKe zR*M`so-hg=V3AN@n8&~_c7T;hlB?~q-5chp5+$t-S1$ER{bOCyC}pBh#jsE$AmPga z5vCUnf>Cu>cN^Pe+AYc6_dPdGvgq%%(!I|sZC^a(`21$_jqIx9`Bulf=l)EM*e1GF zddop6wxU;@c}v{9PB5nK`1yAG4{^NSoUXP*7q@g)Ar zyN)0wh95Bw=ed;FloD6$Il#=2bYZPql!M8xt7n?~+_p(w)L>fZ*sSErC9*Og?rwcLQ%3J zGaHwfmPPJij$4~oo?l?O`xVd3YY8#CtKR;4ecd4C@Tpl-7klt2Sr+a3m-w2^d|gal z$?Hp^>C^Le?Wy=$^m^O$*%k*XUzRziZ=by_YmuGB+5;laY)dC=rremY@X!+;akXTJ zr1p>nR>sd5eqC%}Vp+aYGN8csfNARu9=$zfCl$<@IV>v8I~g~g5ut=&?(h(`L4wFz8Y^loVYg8#W`hu~7I%PuJn#lDLSojNiwKgSFYwjfUcg40|Q^2!KDtxwhG6?3<4S#1X=iR z9EtPfP&jaDId{d9iHuA7Ud82E@p1(DaZX8SY~k4wz$9!R(9p=>kyOwq&>fOG)##+f zWS@-(6)uZ*C{42HiBO-WdfXsu<5n@5g1<+T6FV+OOz*vWEMxI@%V$@8O}Cmn6q%?Z zP{}lDcJ73AlO;YxHSq=*II;@MZkgcB&0`XJkaeAE3TmCI>JyFmU;G%r%&tpeM7KTQ5CkF?&NJx?=3pYBKqN!WUx!nhm`a^Ne)b$ z?&o&jOkqqH(@qh+c1)FRYvhFkJ|b*8ZZoCX@iekGJKQ>9-|h0q^|Vz?%mj&v9SaYa zG=Evh%#pCiuCcg{=TZ@K?4Lm6v;QTmUNQ56jy=tub!*}Ur~a-6=4Fd+&2zIfyl|13 zJ>&jFj(@(p)tfjJd-%6Dimmy5Sn0G*@S2S$Wu%pErr2|@jmlnoR7|E&V58Hv$G2=3 zOA0R2t6eG0$U9l$@COgW8yi|B<~TKSMyN5%26UxZOxEL8c*&#^aW?JGGRfr(Ycyw^ zJ}|>7c{Z0xg9m%EFpHP~L$8BRQd@`1k~ncr0g(f~NfC$mjs{F%%5db_pzR>G<-uX5 z1_$nmKN=X+9Q9qKsmBe>8^&~yky}oEk;vQA=f{#z@ zOB{RdiS4+U^<^Pb-E0Pp=}Q`wBN$oS85}jHPmxq%%xRk4xHK(5(=sM=nQ{E5mpMk^ zjf@&one_?|USl%t@R<6~FQ|9LIr}s>j#;Kn0x`W)6ur!ildts%Bphf~IKa@B_>Ga@ zSD;~LgaZ@n(FQe_Vh8KF-{)C2rS$R7SpLmSbrK`*iuoCzI{h^S9 zgTty;g`Ib1E#wMkX!Lz}R5GOOS>c3cg_L70F+5uum}Dcoia7-iv4t#Pi&?-d5VL^Q zB6M-uwjD?99n#LM^*SXGDZ9RsGqA(Sqf0>vvhsLA@-!pO;QC4Y4BWqs&+Rhh~^m7SgpjPp8X@@6D>3dIQ(WfvO=?O<3qX>Lv`bLKW- zU+(%Jm*dk7xo-7@EtZm=%f0i6srl}|OWh`~>CQBKIW0rr5wo5xlVlz@S3rW(iMb4| zd20(=RC!B1TC%;^W-)n3%qlo#&)}l)?f{F1$H6$i4UV!U1&w?;2l--ltmCd}V4q<7 zNd7{nu%6P1adTL&)<&RFO9}J?a9N3Iz9OO?~!6Z6iL!)|F8>7k&Mr9>!p3`w* z5=|)+*P3gy|J4*asu{-E9cdG_*JIMLe+*L$J(Ek4*^Q7$7m1DW0 zlX#exKF|vD`S`M6@{}pkHxxx0ZZs<}9PF_3Imy>J!REvWPFHfIFTPl(aU$y4!#@YPEg25&nQzc;ZIG0l%iz!; zCXnD>uXR8wGu^uVm}dVIHwKX#4T8%I*aF{w6AInoBAMsMrsd5TjbsSsY`1VoW2InOzqT!4upPJ)o|ZZHY#WnjC(yD`9YcdI&AA9qZM^Oq&tt&(R*vMF5i`xLcek8sh2 zmag{?`MM>J8i_eJurfF>DY!SV*qOe(XT2+f-GQN+N5Rd(q>=BBx!#ZMvB?Q+Zfd#H8@Lus;NB}xwUC`_enIq=bXLJO?TLZXGXoX3 zZIpb(#$n#T6rjN2bdjU1fVnt;-ERZWH3P2Z5A5|9eEuB{DF|cF5MIDKH{5tW=zOJu z9SPhEr>E~|;B8&NEN8%I6~Gbsl(nLPJ+*;7;Q@=^1(t*YcD)y@^#^#GCrBHn6fvn~ zvZ~oC3AQCx)F>usF)m=dZq8=X&Uz`mHk2VNIDl=DTdkD|BX0xaVh65=&pEm-v}+~R zHJ&KhF^#SDMqSYKB<%%ET`QOhHzoCF)a!3j{$DQ6-l818T{%fz{9Xa~j0Vox6F76* zId?X2zbN2+(ZF{vf$v@ew_90sYCz+GqaqVGtF|O}-!*4%Okgin;Al^0?<`=KJ-}La zfn{p~@0o^zcdA_T3b-$B2;3gPJ#m3R%u~**85~!C6fisf<9HXEzPNzBB#6auGT$SE zs_g*<3m>pID{yTM;Js+T;cLL&TEH&qz`z_}!WB?+*xc<{Il~ck+e8z$OPd))9xw(g zWQ8`cCO6bJ9+=Q=z)`2b5gZV`)09mxs<<-gPUZVEX(SQJ&77*ueuYr8;hjyw0#j;?zSQ`!%5FKmxC zNe@;?k!0Vjx=~15>Rj}Hs5P$Um&)EyH3j@-(1@OK(z}NdPA>OJWp3#7TQGxM< zcxFOGc)w)Zt1ZQ032bQwY=*}t^c>)r@PK0i17{}#XQRSI`)iyj28{lnIJPqIw11eW z%{1xqBzBd|j;syLTO(&{Z(x>GW&Y@18lrBj`?6y~N2%=5C_}d#%TG!#F7V!)!0on- zb0!1V-iEH*4|q>iN?h&a+cAOrN;#(|o0pENWx!4^H5PA)0%o5AX4wmw5)DkU4}vWM zxF22+h;ikZ7r=e^GS9^i6?>XnzPz0FgmppW(l}2IuKEXzycZbD1-P~@;5AC_d$BZc z+XUV>3Ve$i*lQcYu1yJHyO1ICOY^UW?TPf72`d@w6d9xoW?l?uYZT!6cYuM@&Aj9g z1LrJF4zGsVxelyqUpP)1aJ3h3bbMH>uv4tEfurC8Tam%+t}SdDm9w>OX7A;cbO~hP10PKPVSi-%4RO$UU7r(%FH=eZ+7i$Wwl_Px5359N72%2q2!5F zC+~;BEDM;IB=Da7P|*9K^4-+*^8wt&CmHjU;$FGVFICfQ72s$)u&C96`9dlCA_3lO z3EZ~?3(m64cr$^|T7uV3gRA)g>$e*jOq!WWQVA&;ZrmSSrgTn7d%(Gnf%AYtUEK?g zPKH?v1G3UCaHdYkT5^D=Uto!j(VEH$>{%1pl`JQzoUHRXWxuU+_Fm1kS_WL(C;a0n zG+L^sImc+_(z{*tg%i7W7^Gad$@lgG|CR&s}OP1a81J?uHB@?!nntmIiICTB{`eHOKDg+$q-4ZK%o zE}K4qrSzYbqnnZ1owD^i4l(dD#J+neCAmP_K(>fw^bG zim#H>>jO5{9hlI#q1Uf~*>VE2_XpN`hn3qHHqD#Ra#mx;vlo>wLOJ3b7`h|1j)y0x zgqtY13zxexZ*85h-~s>H2Q!aM;IRL_xl3WoX>N{`4Q+E1ScC<*4*uE_SiL65duvW` z$EKYgE&;R8&zimSWJiw%QwY1!{|P*APNx|y=5hSNJ7GtTo||4&S7&G9+|x7XaXr^V2hsGl9$QNcfDtug62GjT}v2P%{H*C`psx1z{dA{+J@?F zFAwm|GY{Jx%>MPnLcR~0PK$TTu44GkvF9y|OmG3Gc{kr*1KtA-tBr2$smrKq7vPxZ zFmr|h+qKg?M;5RL?A{wZn>|OEz4gbjOo4ssowIk|+!}Lx_Dk5QrglFyRYpEz23Ni}xDM9U2(3@inV-3h$cCiJjBkGs92_w|Y%`M~sE5m=ifFCWJ5xWV$@weNg=xw*$k|n$6P_ z)_pp_5Jz5qaEtLx)zv zjjkE{x^5lbFrZK2KKEz1@qXiIQ^}>8ZrNv?#{cZ zTrC$?)J=$caWwS9)D?LzcP*(_ten###fe{tf2~Cah83xu(^Fg+X?o#!seax3{wH zy`iPR}b)J%s*|JF8eu3f;r`vCX8 z20k6ROKNwyv(4E&(Psz2~di;{Jey|Qpa&y9{d zS6@7FmgU;;`U%sl7XH&5SKjVyz0h;#0N;Es@p~T%Zp@h>Ww`mM`n?nGnO-q9PLmlF z1ZvH@x!+FU-d-?enE=O*uGLKf9EIyHg)QKmYrrZk5c+TL`OJhR$~Vv7kAA>zcY*sK zlkx&46$2(-IlGbu_8JA5{=fWOyeBH!)`mva8)WiI-FrCy?b;&i?b|0rXSZ|Cbl}$9 z$E_%P>8-#eE}K&d28>%w)Pz*jl8n@B-bw9zs48`bLFytyS-_JMFX9|^x#oZPxBTeZ z(~AmvdmA>sdCHk|@=jmYMuFRhuKYMW<Z`rhj`?ELqs}Ec%R+iI8(P_p1_r(zIUTduQoKWpPUot zsj=(kxlo?W-Ir_6Owr;Nzz3(aR+Qd15QQm+p_ea_C3$L`! zaclLReERPdcj@^9i;pE&GZ+;dVE-*Md!yw87P}jdqL26gd%?5sc#HshO~A+Lau4gy zT*#SrH|SM|#J+8Fbvt|JNF;7|c+tRHoyL(}!MWhZE6sP@8~H9NtmVFR?ST7%*B0lC zc1@ICJ5hRzi1fxl2FG_-O$^!3Zdkr|&zJenmw(Zmb|rR8!-6w&PVfBMx#GgxoeWiH zBDeNlTu^#r0q>g)ISR%c@(aB6r0>aF-FwA%kG?}3=fWR2^2m!;2FGOfs_(Q`e6# zx451B&z_zTqabDX|C9dyTV3jXb8X7mlyhss&PMs>+}Kk3_~hhh^E}5^Hr7|0)BW?W z@(oMQDBT&(;k*ktRq{HZ;QyZ5cm&*y^hx^ZN z*|>R`Zme;1J^MuofrQ3J<_)|$77Yqzxwp3^s#_F1Xl&pQIo7kXh;xcw?D{poUOnv= zFZK!DFhlWkXv19h-WLZ>3NNpoXOhby{>cADrMJFshL@uPD=gDF1e&8Gr!I%OJiY|mDb972a7{oI&&_mtXh%4 zIB~V-I*TM`#sfyD*D^iRI<2$o$)vz_o057MIMNOzx=nF&F?@2|NXhWX@$+2DEJk)$ zJP!A|$auC09%Jgv+3d9HO^@**JL^v#{)-N_$>z-JTqYKHBE^NB>CvpWrp&io4m6lQ zZYegqQDfcE#J@~|uiJCMDVCjaYi_Ad_TDffuS-+SqoK)4F7k-=yGH^UdrYRdba42{ zS7|7T-l<-{sM|@@QK)~<1QnGD-yWY|;u;i{xil&%Hf>qjs>)?*Y|?i;XV2~1RG`7# zy~JI9|MO*gB-PNsk)pF&hbb*afnf*Rgahmu zhccf{bNj)R%qcrfpxNL+pG-=>nOlVO$q%;*4)a>82+#0xR(Z|bZ^iO*@hK;ljYp-* zwrpe-kJehZbfOFgg98)8t&(CtmXgmm9#7yGXlUS>Fyj!-vYA}+8n{3rEZn5i%xL6rL^p6uwu)vH^Q7Mf5N`%Pnr@F`f?f*yU&KO z80jl3XV2~U)wv>G_07szQ6)DRSX9i`9O_xVL7|XGyGzu8flDspKqKpsS*!14bLyKj zTz|x6zK)|{#nD-5zdpH$7?xaMYEd;hZM50#TZD3}snh>|HNA2b96KgBa9w-jadz4c zMi%}nTc6O{kdzK-Hr?1C0cH8X$oo8~*T)Zq2 z3mO@oNw5mdS2pcXQ0-J>+S}d{k^j=)ac=&VUoKWxyMKRpy?%*n6z_(m(=)b)9%S== zYd@Hn#RRyGM(LWha-R41!jd+EFJO-*BP{C9ATf9 zz``HUn89PuY74PH9UxvMc;>*l5(i$P=@G zNwI-}MIwQb%|~InpoRl8R{%qt*@5X9FB2kaUY-_ulpyjT;r0_3L6ty}B<}hTfe9wT z91b@OSTikJlw}xIwSO*5n0c~ColCGw^#-GW_lcOPQia`WU5mvQXB<)Z*U&Cg(!dbG zGLPv*Y6F7@10&akqz|u_G?--?R%%{i5S=rD)up$QCsBb})NVs#rY-{$U&p@&rWXcX z?jIKRJup16)adDik82Zy-+JD!b`fZF>SEvsoXqmaC-w55H|j!R(Jm??AN$>NbR}6A zao#8jb@1(4tj;8pU~b2GDx+hiNrbZ+|jE3s*By(QjN8YRmXX#HHBsPCLDA1+b zBCq1C>U5@CU|n*Aa*jG**%c;!hYO6nRsoG%5&VVR4h>?B;TQC;tu#+yNp@mqU|`@$ zV3NFWfZfiZnLj~+S#m)GgWLxOo^@eD=NgVE@-=o$lu}Wgk>%lhj9t*)X}Xf0=+%F& z4F;EgU3eH`y7p!5r(n~to9_iTYF${P6}ftLhp#%Hkyr4>have-T=spk_41qZD&$vP z(Eg8C-PDX<+JAku%7i&hz+#6Y-?q1@8@ptW2pxE!d?oaxK(j{+uls`aW^Wh;nkN`o zhMMnoymCmuy+%h-ZdzZV2{XUkLniTEADAV6A7FR>z{p>4pjqOM1>3Bm6WnnOg5N7v zDH(nE$er+@N$So47Mlh~UY`$7)ZBD%VKnRlG@=W#H{9bCYC^X$8gjUFy?F5BA#mpt4c6Be7U z557kher4=3z&gg};gCOV8}Xm+l6v3ew!Cy(#vV z)a-Z8@+TVDjV4^;cYo!R#?<{Pc=-a}s)RENQL;xIbk+#^Ehv@WmeQTo!7TTv!fm~y zSC?hemFv#mOO53IG4?EGc7EyY?{;}&BTu@(bsDVN2Yx-}L|_!Re9U_*5NkG=YQKNUSa>t1zOe%mc(xUtyy zLyfR+UeE<@`H~GHb=$c9t_l}@-g#7WU52;wIro$sO%g{QunPD%a4XmQOto3qF7iUV zWI^(!aOf}>{*?CSMuPEI)gpyzr-^<7EiwZZCkC{%1<#}H5Zq5>~{W;*Sms& zJ!{{)()NJ&-zv8+3wgcU|3Bluii6q$85ibncxar%!0wv8-adI1XXcKjzc>p!4#dqg z+4Pe0^kxf=!X3<;O>!f|onA1PU1C&WV!WBia_WMRV@5Nx2cw!N%SjIHw!*DiADJCE zn*EBIv}Q2OI=z}%u)R`(y-P`&SY^{yA2?N)NPz5dOn z^~5ffgKgbsyA8tj2?A`NjP*W0HYyR8|IBDK?U%@mphk`lA~F*;{+w*+Il)|Mg}JNM z)ZZ3UEt02BKEe0Ni*1I+p7b5vU#uLCMmx0LW@9tq()_f!Y4ZUNOX;On8x|#)Xm+uk zG-Q>ZxVKG^)s2Hy_)3q%3q=PHmYhchod1Mp9H`K7=4jSUSk5oNpnSmANkbr3n!QLu zf06aR1=3b3ENmVeyY)-#^v*cF+U%6F#xC{7zPG__Ne`ShaN8&TXifZ}R(RM**LhuI zNAtm`b(4+c%XfEtudx4lnz6FmS#t4y)5V&uOr5S>#`8AKkDoA=NpVm53Q?z&WqAra z;sT}ZnYS$V*pcV4<5=?%&m65q0__WCY~fBh!BL~McSmc+i?-Yf{W~9dgn4+*&6srd z$3)J{Jy#uNvv2Y24_FiMqB}r=rSphR#Ray9%KHL2SgU4vF8XYm;Il7*$4xE7YQy70 zZSD|X)g@0pd9V)e@M$@VfQyyR!tuVZ17 znc-}GXa5gD&eb3L=TBO4@U>L+o%t~@+w8peaMi3|rsXm3Mq8_~g(lOq&fR;wChy?j zY0ucuy07EJ$(7oNKePn6obk62I?Kp=>`u>>l{`~L+Otov#dWp?T+tSTzoRCKn@=5b4EaZ6sZPc!CN!U-qcCx_HRyb^aD`*eBJ#7~==XLx-s z-0d&uF1mtI>V%hUgY!C{~r>2XL?5 zhv{#(8%OJuBmVvY!REf6|E^3ra6w3E(q8^I9&QFLOc&YC&tYda+NZSUf}6m;=|=*t z=xlRmusN{W%>BoqB_@FarpMC%?9)5qn<&wiRdVW6^+k!LQ@k8IybG-ri}_{#9liMY zN#NQ&>rAVi|8z^)-LU@oaHYqO{Rz@q4k2xCWR9?E8n9nf@r*w2DziY#R@ybjI%69x;ux2%zGRZsd%|dMYt)`=MssD(3k2kevik2h|C^_0!zGWv6P9HS z?UggyE3NDVz$KCY@@LmZ>p1bYlz3(WAQVNEm?CcF`%X4YRKnmuXhueKl}`7 z>)=p`W|W*^Bw4`Nd3K|WbEw>1FV7#Ro*l7%-*o(60E>#K7IR?RU&SMAjO}*Sr`=qn zTee(cSY$F|=>grnJ6c*I^`ANva!k%x(e`guTgHLbtOKpl0#OU+YWWJ^%3OObOO(|& zIqKiCtK2`obQj#^=~Q9ApUhsF(e~i7>%}KWokYzNq|G)i4NuXI=Gz)_=gYqMhAUgQ zhWNx@%&xhZJDEAxWP9U}vw1cvHH$eqdn1CEMs&Mg?YSCJHG7stz|?=nr!6;KjjHIZ z-eWvFLLk?{A-;Bk{b9DefFm8zCpHJ3-*ofNf`AjPb6k2JpPTjJbcsj%h0wbT5@M&_ zW!wFQ?PlnNERU|acW*BEdNboj?5af)F$ruMliI5Pw22toE&goD{dQ}d$(1B$&a~t! z*~KmOdqcLYy_ZmMd%J9#vjm&VgE+stn{*E=e%vr+OR?Uf#*U!B(Va&dWq9wHPcf97 z;jHjh&tM0?)q>vU@VEBv9A=C^%}=Ph*k10`=-uI7qksNw%z_zPJ}z;Xu~b0s>7CQX zCWRm7bA<$y1jO?AwR8L3Wq!=2zAK^d#)B(&ALRXbAoT5l@H&CK0)cyxE$$Uh|Tp zlm-R{#h)x3MhpxLIt&a93=)hXjP1PK+`_y(lDu3ZLVVJ~yi(#Ee0+lZ{E|X^!jk;L z{DLAP!Xnb5qB4@AVj_~#qEfO_(qfWg{6aix5}Zn+qRLXD%3{*W($ZQoGFp;iCQ{M{ z5?m^Zvf9eBS}NkE>Qct)+_ExCveIhOQdY{+O4`!O+OpcJlB#Ahs@n2Ka*8_gY9=bm z%GxT*I-07g%G%m0I=b50>KYnGs>;UND#p6nR=V2e8tTS|y4J?JRwnA28Yad%#uhpz zrp89rrp7i_rZy(lcGi}rCRVzJHgbyMo@SChhN5n=s$sIK1+r?c7TQjx#xAy|Ubdz# zHr8IY)-GmN!Pd6nR_1>Cnt6J9QJT`BZh{3q5)p2iZVskiPG+u-Ha^bQexBAruC_s5 zcH!>kL0(1~9vbPM=4BD)g+Y2%DaNHa+~$_2$VZXrHSVV)k*!9Eee9v&WHpE51M z$&uCB(Pe27<(UED5otbQ*%>h@ktyZbsj1~zY1PFU8Cm7kWu?V=rEv-2_6}CP8is4M z%==_D+w#RaqxAc8jh0mybtOtJvvj#+?%5X-*pd}Jq0oOxiEm3zdTV(`OI1mGW%-nv z@@d7H>xv3jgvFfCFMXX;aV;?6xUTj3ZnGt`HIFW}JUqwr=WnqeKlH9%(caMS*WFw> zwY_pmXVL0OMXRU99+~BRYGuIL^##`!6uf;HnN`wRR?$&bJ2|7EJtnKUtFC!nUDKzE zwu{xx)y-3@ntSVd7j(C_Oz&))J*A_qYx>M-GbT-#wsc14@>x?i&z-iqw`299o;6FR zZ(clg%jzjxmdx6^cJ{6{Gk0y8vwh>tZ7aGb%~-N>>C&yMm+aiMZ2Q(7Yu0UoZ0@>4+m2q|d+yP-Yqw6GI(PZj^IK0o zJb(7;*_&^#zx?_7``_Px|Gs_w@$K{5Z+9*}*n6<9DTL!6!$*!I!RZPIn@fd+d`@gw zc-SgI$!m_s#z*4%33^o*cUs8~EJ*sj$o^wq_A{&O5o?)sV*wjyTXhDN7p zT9@N+&rO}?&BLNMv*dEwm+2X6%WoakN=TK03(tdvuF%Z_eIdAVuZ$!Xi-%O<|w zrh4U8M!H?4&Z)%sphsH>OmG}HM1ThBc)FD~4iUGFWUm&>9ik!U9(ahR{m@Y)RKiiV2`@_cR` z?XrA!DG$?TH$IfiSX?SG(LJH#;6?W}os5^VH6|Xp<*7Pz8>_d@+6w6c)sNevbDw&K zOXr&(RlgJEcQ8c8&(86?Oo^{W`PC?wx$P-M0n0?6`oA%qcy+4ZjO{69XEcwxPP2N) zGC!tps)@UJ=|fZZI{z~h*~IHSFNG(xN{TV_8J_si#Gz$#OhHb-?8X8G!JrHci5a#_ z9=6HUSHziNKqZPBe4IZVA-#mYr2gl?}d@$ymHa!#r|Hpql^orBnU4FltY@(N~_e?Z&oUJDUR)+n%prJO zc+!`eyj|8ZJ7#e7SqdC#VY+$Sv_-^Af^F|i$CR?gBDE}S88S;V9xd0--EvgCc>j(? z;&YF=oB!+HdAH-X_a5H0$~DeaODC`Kay>dX;B3%Mm8q_IjItY){r|1r?8o>~_2iCO zcT}P-FfD9-aciM!SorcF@3+w@iLb?Gs;!(U-|%}cr}~vf!H#DBO1))H$%+OY??3!> zdyycKv3PA+`-Cc$4F|QSmVD^a-^Vt|t6;au#^q|ORIc|FSr*C^9K3iX`C*2mEXgH~I91MK|guSM3JTeZQBy)GN7h zXx6M{CPk*c9GMam5?4)|VV8QA@$;mBk1tYfI!!bx<%6U%HA)vvoN-LK;P&mTv`^uJt95&wYf+scRB3DF!+LF6hj^{gE5vxm^-j;t*tC>kj;JwgiCccUbjP+a(Ioi{m zMEqES`l9A_%y~YksWvN-_jibA%90cQJ2SZRPd~h$wPf)lp5yLM4{`;xsO(QL^e*vU zs5E`W=DD%DM{TSsC;i*PFmLgiO&;I7RZF67dTrx4ssA-wY~#F5EdE~YJ}I5ueqC+L za%;Qo^LE{h|7X-w9otTT&2!1K(%qH985%;(N(yg1)eN;a_Sq*qv+pv_vD7yl=< z99Sdr#m1q6>1B+hv3d@R-a(Z`e*03EEq?mkCs#&%=MDz)90SGq0-2ZnOg?2qed6i!H(F^iWuyPvDOKHuj33m$H&9!80Bu<{X)wDX*S=D0h}d(2|`RJmE~{cYkb8de_p~t!1|HQjyM< zw&2`fbxAtwPvxvxBC^7v*Lv5=G(k_{yH2w24m`BJ!Dts(z*N|&&7#ZExWit;u&{Un zv&6&r!;{@HzkqN!K{hT4m7gW!0f7VC|U19qf^LYsYC(h zkb9fNj0&&%Jh;Vw>A{gy#)~T*EIeK&Nij1{+czcd`yu9klFx3t&#qeHXns`x9vj=a zj>g~vyC$Bk&ySdF^Z9K4$BlwMeot2=8a_|kIZ5?ooAyK1&U4|1S)IDxKWB9?aS~Z9 z;i~-WLhEaWg-7ig^!PF!G>L6lz`WG#>%@MYW~D7rte$xetSPpQoH-0E3IUJaA42d)9U8pEOm{eQu*I22WGw)1t!4^ z1{SFc3{&_So@%69o=N8`X4h$8WYIXlV)LOfVg5s&85u{_qXfTr&pF6*_*(7<8OP!Hffst!LrR0wyApu5-jz*b` zMwuCv3O5*JF0d?4;9Z)|d-Frn%?&($4UA$1#oP`I>!w$3{9g3_5R-N|OV89wF%L$O z1&rPSY)9Bj9NcTBHE=py=h`xXdqD%Mw?mG20i$RE+x82*=?}Co8Hz1r6>m`0JAOsG zAV{zHlDMWBi#3Xn$k2S4cK=%i_Diw7M+i!f^)X4GhXNI%jIM%-q2sw1Gi-1q0WI#zikWH!f$; z_}(c0qe$UL)kbzk!wno~9jXj=@EYvkDSptrCA~>-0Rvw{mp}lE)&VBZWp#S#IXn)G z@&zqz4V=>&*fJV8-IjCCD&Wj0;5^LUy5$3BqE>253C=DrM(H)uP)ILN^BQh*j~zB{&jj;;Ii}_$55+= zfb*hR=Fb}RE-)G@R7x!`lu;|_(DX1u^4Zoqyfp-Mla`=$Z!%?W%DH}I4dFwZzq zwCQ;jUqOy{SX!_oW4I@S@B~Kh2ki3;*xbTv{36))wXvoZa2}stv*iK1!UIN01IDBY z+*cDgvOl)w6gYn^RI6H;I_ZK*-X_)z3HRI`dWIAFbsU&djs^9F+I$UiQhy?urd(d2 z&Y9#e^P4(biU8ZeW7*yXtP|Kfy_T2AOvn*T$eEd*!yM44R?)0+e4>7MBTGXw^8tow zJ37T87?eCGX|PwTRCZ23F=_ga$;&gk%q-{Ze>r=f1Ov|o2F3tJy8|4T7@7?#c=aXb zK0LtFaDmC}-wuYg>Ql|!r|PyBRg@I-7%*BTu*Lr@P8Vq1Ucf!yVOp{QTPnk}Sq?RU z8Egt0a-uqTOFT1j9)w&GvQ{#6R61%h$sw+`QLinK**amy3-hwBm;FLfPOsD06B%Yo zIkDFS)LWa0gm1`>Y+$udpB8bTW=9y~lyC-C1x6MI#ydBQmhG(ECs{4}Vz%0giE}h2 zt^ZzSoHfbr)#8O5T}L`68*z3`f6*l^F5;ITQKi+CHI{NygM8?W=7U08qA-*fYWyd z_ZEX3X$7|J4|xAy@cj4rqD$-nwhSZrj76!pw~0%nL^MoHvb(_CGP7UbfhjqCYV2lr zp&<4|2gQC(!HJRVaR=Dhb|!lZN_WA ztDWdFG%wz8yn91=r+w9$_1`?g9!lEqPhb$&7Bf_vu3=$K0_rWSnO5a z1kY6k%bCMJw1*e4t#jbsz2P7CmJeJ8-rTz;)TB>f-O*ObdSIgN_0`r=)n?x}2Y0&f zUfH}`vU=_DM!T#v$6r;=U(tB#)uaU(#lF^EM>;!=I43g;FdX~UedQMeqXBb+1MkfO z-pdYi?`3sgU%>00Srv7Q!(ao$^Qv`;TwB(eGbDd!aJ69bt6+=xz`3Jfed`0(;;WfH zA6Olxu^w+{oiU;2@HE!OD(;YS;WeLaD<$<;1^U z#~>K59A7LEoU6WSRkz@`>)GojaBuD2z5O@W4(r|PKWt*=t~vIMad!9?&E>P@EA}{Y zSMyz9;7i!P$$X0%XH&`R?sYFF8U5^BAKn$RW|EfX9N+A%Yz>q4XB?gRomnA#TWa<; zx0$?pD|kILdCv;4@H|-bqH7(8z_Ok0`vn{rg$-E$9bwZ+aOqWSyZ5WSp7 zyf+VU7+$EBVAy}`mHLhDW%>+^QJp*f++OvffiGo)>XPIzc`mM=Hz(E>u&_%Xv=tJK z*&uqgNw>pFulRxdL!*rgbxsw99O`bI5p}qKjZNgy&H5`&>rYntCs?7(mb_u!*59Ys zPT(~1t>O5y$>zE`oU zOIBDMz4oT_lk_n!t7H2#x?*~}W?pBA+Q6~(L*b=N-b)vb<=p154`37*VD@idK6R~H z{Da;iX(s&uMwShX8&@Cbe$BfxA&kp3u%UL|iwT?;E4bYYSSROj2R(Bw=VTQ;+jrth znAnrjMKy79XPF|DcPUOu>p7YCTj^)}ZC%Z6uTygS#b%Dm@=+#TE|I3*) z|Hj!%7N3h@u(;0P%fQipfk)x{vAi?eE-7%CU0{%Sz}$R*qbC7$Wg_ndrpAYh zDoU=)F)-z7^)k)ouDy`Vx|Fp+_d(Ku={dl&cmlKg?af}h zx6C$Qqvt-!sB^Dv=ibAaldkY|=}9t5a!%Q6KARcDQv{YEC zu$|*y83Tuj!qff_9Hs*MwF(&J5?Ic@=wb5a4i9+1G9$o2fz7?)!Nna9c>B1wE6C<5 zxe5C|%y4R}y?Ij1y6&0lA%lb&vwh2yb8Q-#;upQjlbUo%pOvlc0hfO7;XM;xzi8lY z6G)x^b>WN&+^cw7(;adi{w%tnzDFj2k+FbL?*MDB0hj%UB|Hb1${e`PB(TdSFrQ7h z)_Q=^X4dT6lGUeI&fYt-DE!Ul`9C%bOxUX>xz&ADrR)Uef+l8Bh5K69t7Q(bHXq=* zXuzu{!M8MkUE~0RsRGC81xp_~@SZuqR7fnyxf0;b{>A%+IeV^wg7A6oj2cR zBr>lOX?mnJ?~%~L)@|!{@1F4bZ3Ew)5B!{((rQX<;RiMudvmXA$PS1+{`oDl#04g^ z21d~Y=JEqP4-Rm+crZ#Nu%Bh%ThhQNae$}j4u`~pYTfJAAG~Lua9eEtqe#@DQ}X0Q z!2(7Hk4df%nuA_1{;4zh`YD#?0Pfxo>?I5={5z_5x--abU@5)8apnTg*#!3U;#J!g zEWI4?i#wlPK7sxG2Ob{i!VYX_S8%R9aAKY&n~S4W*mNW3M3(de+-n2o&re`= z4|sf4fi2fa?xyKQeg6d~|M}}ZN{?_<$h6;}XIk)T)?df8V>1?LnJt|4Xo=Ln!fU$% zGj{Jh!2Nc@zk3C2?47@-N3vPG*|fHR^=IamW4Y`(517jx*d;!&T*%~Q2-Id~9j==$kX5N(rm-m#- zRa-kn#Qe&Vn+p#tXgtA`d~VLh7R~V0+DTh8NcBBJ55=fTxWUE$h3GmK;{l~zgV z>Br97(71@TYqEOuJcpT&Q_czN#XLLnPw>*r)54dW=5cf?J#iP1x2yQj!1$Q8TR>5v zr!1>H*!GdO~85>S@ z>dsoA*sk_$!(l!ti4_VAY*rf<@;SIAs!VWWIJuxvPj00vJ7dO*Hk$K`SbRRl6U7B(<2iB%{x zavs^iBJ8Vk^XW9hbtkwt?ADl_)2XxWL92MU$)FB_&t;lvWLg1m6^v3R^EHpB5S2ftJYgENyC!;+b4UK{Ca(Z|j1X!nM z9s2Q^Q~j5SNY@JPw{MIU&0hByyU&+k;!te+z}S90r6SQxu;6xxd4RoZp82+Z?NH0W zNZZ4rui^?_H(2bpTx=clf6Hp?xblat;!y^I+c=omRvc*JUZB7#6SN{>(Zq~xKLr;_ z#QZokEoa3=Mdm3p4=OWD%S~jHX*uw*OLFlcM)f&?c?}IMmTnW(W>0vn+tOq?i}U!b zfDa4UJRDLUc`~u97$`8YOPs%WJX7i3#aUH}@zW>TKKqc^#y2BDW!9wqU7t?Mc_uCs z5!zOBfRRmzL9yF-Lr3=Xn7m_^oPTFWG}X^Je!|t-sIkp^jv}}8jfSLu5}&s$k4-ab zdc`Eo)2eZ?<(P?C;S>P{Rn6;f76fe$Y~(U|I@#~58c*KqklTJ9T03joTkl#PULAL_ zE9}Xkyxl9f*KPU0s>#9E_H5=t#THRUCXv-l5(!R9OB4mvTn=krpK(ly?<%WE7$Xz$RwMTaa?N_;Q3BADe-2 zt7e0`RiXp;zX`iuhMt$Y_Cv6Lk;t(ilSh1I8Z5>M;rf~jn^Sg8W;1-Vp~3do6GrYE z4T4F_I*qRwh_jtxVxHf&StsG3&6W!+Qs)BL1Z2(%U)|6u@!^q)!&%3yzL!o0tN(p# zu`F4}8<`yOIT zuu2_an5SE#$dh;@RC38~b9k7{snOa2q5FuG@T&*X zOA_J_ZD!fGWcTOM+J(%%(5rwfw8D^zSXX7UKwCmdj!v}+}Yh(cYuL$g}U%I@D2 z8PkJjGzo4hXbF1MBtGN7A=Og_2RMBi1h%g@Ht)xmEt46zX3S7rmSLgX={DD`S(@dL zK*5H8&9WyL*mMFMbKQ2BiO4PIJGSxgou17%^6NggR#-5XUYINwe&sCBj$h}*mu+a( zDPiZCe}Pp$ZlRpvgJOoGNetYJ9J*X>76}SpI4Cby#=E$oxzx7HO_pbpb7tI;uaog){|%hR3Kn_T&}hy>3)lj3a>aYT`M+KC6U zm)wI+vAzBltp9bAj#rmWH><#eE36u4)PK%t(BNLd=`&-JSepT}5qwM23^`rh`1{B?-Nk6^`+$Z8(}~;>;}af@$xJz-DO% zeW_Rh2ex+xOSxhin7lQP1~@x1FtRAPSw2h&$x!gL`ouA@!$;-ox!-Y`6W6=Hm=R~S z!jOYYL-Z(*!$#&g*F*SI9OotI^5}-#?b`k&{OZ)+?k0N!My`kkrq&%tbq`DsXZv*2 zKlt#h3^(P@%QvnDPGn*dnDg?W#FSn$4L51C@@AR9)>j)0*bd9=+nk$Q-N-kw^{S

VuXA(5^b_-rT;dK%P*LB;z^*XCfc42{26+MJxoK@|kx5Qh zS08Ipz0lBVJVP;ET7ZFd3tvafYokN{#XKTgGTC11<()UJ?RlW>u1*4#_^@89UGx4YOC`=hBXg27R+Ys2vt&8hp8 zGaK011beehoK}i1a8#^1(!pl&FHv%Su@l=b0k_tG2i(^twXtbEU`fkxmhxH9(6{7b zJ5R^DQj-M@ybOVR8!M?_de5qAkm=dTJ~QCE*_=paXY&016B0| ztP_;f&LxP<-mseQcjHxt*e z4bmG}3s30stYFs5YU5wgaGiM)w*rHX^FLe070fPLtm+Gx94)#sHZ*cqFwH(S*RYz2 ztwM7SgUV7ShM5dat^wkoyCgDa?991f^yQJ@!bf)2Dy-5cgeC?wS~|!Fy=eO0z`&8f zz@*U-B)wtP;$7Yo?0q_Bem~vRs?fp0!5ID7K8B-NHDQ77gjw-IQ<#0GoKrAMs%Bar zsK)!NF|DgnwxKa|74Pi>y}OjBW+pGpe!QDMMyKG{0eNO-qmDI^296O8t>F(_e{Zxh zQfQVi>5r7K@iu5WQM~C_kn>!HMXV1PYzh`deb~wMf4GmildOeC*^Ynqkx|F`W-%Y_U`wU+2g*LM$nYhJf?6I(1I>WNrpJUI# zV%8I%8<=h==X~k>mDG@=2)dB{QDPg{jO|tn*a|+lNi#@f{%GdaSiEAyn) z2O8Wq$eD9^+E{dLI3cT$*u)*tVkOnecVLsjBc@+b4gY!?8rT^)e`GM!W^tOdU0Wb+1zXw#Mve&_`U)#jS9I8}nDIE;Br~JUQla4#&te6I2DS$1ZW);D}DbdNjf>ADm@wgyoXTw>sS|*1L|60>5mYlt^Z{ox) zX*-zNHZ-tqV90m$(L2>v@`LOC5r%6%3};q1Fr8@N|Ir{F!DRlI;pX8zPjVWr)--Uv zXpU(%h@ZS+-b$X0(nrr)@LsUso&Cb#YR8mo4X4s3Gm0rRM!Fc`iH+fr0lzD!zFM#_>;#j5#I&Q0nmo4qxk9<61X{Bz z*d!ltvMp!~U!LIU!<>Zf6(U0mjOOmpcDF6!e}n zDR;%rGzJzS4+f412BC@uDTbL}H<_dzI8zzjJ<&t!?`5mbrLoSm(ufHOJ7q2{fuzSOo$bwJ) zxzXx+7nlSX7<~e~gIxn&H%yhO4)vUK;rg0A^KMMxS-XDU9UFd8Qx1g&tB5u?2N%8{ z%~30u*%%sR5?WjjG%yM0!BU!h9*hQS}q3084OGY zTP{m+iXC9S9x3krt8wqu&hHKU-!mBs-Y~fp%w70s-@ILI84N9LTNv0Y7=$_)9W>7U zdpS|U=xvwV1*^^%odO9BOefT*J-oDqVXN~U?|@_mt^>_6%*^lS@UU@(FrS&VBg%_6 zYWJR|$nQ@W_g66rSvQ@-9-g;)!6dm`l&h;zn}PA=1=d)H&}5r{ z*C%ElpB%dHtm7}Kt-p&JwmRslJ!xjEkek@B>~hi^NeAbqnH-EAT@yUCeksMwxv`j8 zu3>gg|4a#nnGthrwUkahZctBX`Fn$3VP&i9ie~vA*G@d0bnVE5g&XEf_$a8csm(2c zlg)wQdTXOrlC1noBaMTTbQBqW=E$G&o>6}A?!*ACPKK@AGZ+k3u*N7bzpt4UyNM^j z=G4U-O(qWiyr#t-Soe`}d-c(oylYnNiS$oglWnc@*Ji3b_k|5PJO0`;ch~)8rY&K#`QvItA~qA4g-_J#@s!vX&EfaKbX`% z9NF`L=i?0~r_5&cjOHx0-sD}{p;JU=@XlWGN84km!)=DCdhZ@RwOC_$AtZz|;MobC z`Mll+7RfFL8D$xm*#6CKFsr*-n`Wvr!;N2KqKwZz(=!Ku*c$$5&~&)B?z?14;O^7! zyd+v06P9{4n9Mc*-mA%WF5|DRl{906^;-sm4NAEOcIF(Akr!-#v$0v~0EbAm(X(JJ z$qV6`9$g9^=S)-?D-T-Uox)b~p-nxjO&4@aLXWxQw+G|g#=Xl{!+vGv%s zJ3{V$Gg$KYJK7RYxXFAY*&H(u*t%t*R-bHUNu8qbTD zWtYT>{x5m{_N>l=Tpo_o+jKo1r#)(H_|(98AboMF!Q!bl92*2%XGS&uT@p3*L(9F# z%z6wi-=2Pp$mm>q-OuZVK>ZU&iQLXUL21vk3>z9hXi0ir-jwOl?#-oK_LEtofRUv$ zVr#{#TViQkI~p`1Zg^>FH8ZsvH{?ZjutpaAYh}{day6wvNuxojp!KaA^R%8ou8e-| z3RV^OPZ1uURHnCvTQK=gkvga0wL|IXk{prsnM|5x`7>2#KWn`bIcN6W8(zMw0qfu7 zD?D}xec5Etk)H4USjyv4Uq_?Ziw5gF{*%qI$NXk+Sj><)6Y<r(BP8OuQ{F7Cxf5Qqe19I`rB`x z-n!+zyZ0`VW7^~dX0ronPFFW1RZp2-dum3j!^#>)!G8jZMQnW5>1$p+J;nFrJp=Eu zGhE9;B1LsqJooz8D4W43_JBcjf{jLd(W2@v9`%{d({i3WzWDaGb5V0i%SP|_Q4M|9 zCrwpq*FM;dF`wL7` z7oueg82Mi?Fm2>;bDyDB{Kef~H%W2YNj)+9Jm2g$i%&Vba5+r)=Xh>8b6$qiw&=MA zUoW+NnOl$&7QU150K*+imD%PmCm!7LO8cSuB#F!otZo~ar2?+wx6yzbDrWhV1Z9n;xyQV!NLfUSzT{EyvF5l10eY;we`% z927G0udkRA!ILhiVJM`LF0uJ_{-^VME}P|k{=Gl5$10)uJqN!oLkGWAMnys6VODm2 zISY}@hy@24IJm_uIua&2w6IB<`^6{}E;_=&?8f)O1u`$JSz7w;^Mq{t2ZA(0t=kA`Cd&_ir;BvpMX1TLWGcS2CzWNumn3b7Z zBE#(w8(aHSs zzNb$+Jm%@u*+)+bn-$N?dHAQLdxDw&JdQ*o7mf*6?81KE5L9mCklb&azkjr}yZFVq?IkM@t6K6(StMO3czDQp;(J%Fh%F}` zFfz0A>o{c0-?-PEP0;Y1cxL^RCN^=S8Uv@Jr$mz_J~DDBh&VVf+q5odVCFVBaKJ^# zMa7Xt0&d#%^tSdJO(EmoH$fgFgS8l zG+aEMw~J9-M&fj%qfF5htM)m?F)_c=ib}6MWahIvQgEUCe(8aOtfCFu)wjkjXr&@YX;gM+qx(bcVDlMwH0q6BD1h>kS zJvq?E^+lzzT~I)wfra-%!EvKY5uuMlG`Cq#H3{B2BgEv4!J!jI4{WAR>+)uO*~Z2c zpm6;ki|4VXwA5x-2F5AtAGEqPM*sR$z$hMJ&^Z17pT*7M7CzIi>x8E;vP#EVTs$Hc zWuxdRo!RrvMPP+(<8%HsM<1jw(Ajfr-|}n0YW^+dE2^bhUFO#aC9+&(5cLT-DCj0I+f%z_dy%=u z+M1+eg7t|kj69}O3Y>XAWpcmUdbLWUal7<2k%t}voB}oq0hOn$!U`PowAIYInHIcal4f93%roRtcWLAe(3q@K)7Yl%!N}!zp-F7VhZZ#( zMcze?GX{ZnGcPVXMr6YGnu(X&!mw8`Ni0rD*a~3P|X2$+ty1XRVHlQkz+xLKD z8QTv=@dgJ)jwiFK%TgyQnS5vzn6QB5d9u)Nj|B|8GY+t?DPY^ZlSz2miv)Iei$i>A zj4Vti4#-SXW70yULm*ohHIRuO>(n5!i>~8icD@-GYKq8SU%%!CdcZ7jOSQfXA3-FGk-5VtDquE z)=Xl##mty<>WhD$)V<)pSV{Dp?4ApcyByrdWY%eW)Ecn)ddV)fpg6C=Vh*9fGFOMM`GIqn#HCaVC%Cx!{NI@SvrS%mKjUp1cw`I1r9uKmiutP^x0&_^Ldl^ z3A8k19I~9{H`Th`s$o*ymu@|crO6?SrwQ>iA8-!Zd75!;!i8R-m+SzjkG1?e=E$!T@As0fL0 zoV=jU@6-_BGfAHRWIH3X_jC3G4jj=7T1&P5ZSv}OkeON?*D7ed-~cPjv%~E!33@ED z{k_6&ZbxEX9@})|(i@`*jc#|8q~ao+Cd?P#tG#ArN{&>D;_VA9cRrPI_)8r69#>_2 z^TiuhC64#!ZJuh*SL>~_KPs>3%qcrDsC}#2WVt-P$J?ui0*d7OVtGvV()f7i|hlr*P7JN=V6B3!rHZXHZFn017 zI504tSmvS8a<0!tf@$TllX|K}-cf}Op?V)a^B6r~ahkHIRMC$+Y{|y+%UX7nUamO8 zE9B6)VMi5{p=7_)=cG>FD_uu7UAQa0-K4RT)6Gz7!#@W$p1|(t21oA59}#k`+gZQ_{{mWY79(#o34uPe86gb>tU~v zP9mpTev|MF0fneLyux)BheT#1G}^N)VvT-o%-j{w$Pu$3Y^BrGx8=4hd9D6g*eAe>Z*cBCX)NIZ=B72d>lxc8No}SDGYT*f{X%BF9pEdK$K1X(u61MaJcJT~G zM}ZE$498g#jeI;UR#etIsj^YB7Ufhirzr*MMg|TBHkAgZuCvW40XMeZWr=8r5joI%vo%f9@rcZn$ps6x%x(PV>gl*7kS(s` zplHuLi4M0(wQk=_`12VZdCYjN~J}Wgyu&hx` z;^DJW$166x=`51GFBmvpG)PQ1C^?7mBkz&yFFva_aLw0cVL#!pp!8~O#}NjR1MG7g z^_D2=Z%DNM!_2`F`O)^EkU^tFLxXGG1<4tV94(CsHyjlhniNddnQH0hDoE# zNn?_e?hPkhgJ%DEi!_tY>FjC^TcKq7Cf-Cs)zqQc->6ylMw6Bovtr93qm4zazM=UVr#DJ7(H zatxzG*1@zbj%H2Kb4?-zOdJKCbmo8UoPXm1_Z_x82L~&+dj%m~mmH>-oK-FfaJT&@ zSn6?HSmn5wOiaq0gZ3Ai6sH_KSbBirD-V0d!Zj8GA+{89B>2FZ9>N&_! z^+brlQ8*!5?8zbREl!cLjq(PEHG-T4=O!6@Fef>tYZ*Cf8!?;SX!5s9)%=p8QK6(< z(WFt5uE^r#{q3Oq4M%y7CjYaHC$BWh-Cz_pU{r`dCoxJRe?6;5p#X{qBrqk>nx=h607-1$&f>8iLep+e;dP_S^ajSslDL zwG6X=&)+_-ADCxOZho@VmuFv$MCRSVGvEnNM+sP5No$$=8n|V8jTYkYznrFB02oh9c`MP zhyG>UILUutf%&Fmd{++e8K`kznbK17h*7qn zF(c}UtJI-54L9y7^S(JcuueF@TENS`p@BOg)NMv@ZAgpjjWq28PHHclW?y3B_>wj& z>)?ybM#(3K^ma6Y!VZKT1H z(7?VYqD4=GH6f~T$;I`zvgF=2r5t!DRx;ajZ;zNjkmuWLW^9f8Ey)a9Gc3*2d=d-622@N5hy;J%Q1dYcKy_JVGg8!WyKC&Xr5j_6^s zxZ%haz`%Cnz_qnrR<07O4fG@DH5@&%>c_Pk@i`2vDGh9E*ze>XkdAe1G-%|#;ZSqW z!TU-?;~dAOo^66GH}9?MROn|*OE{LXgJ<)xgn2$UlWr++A81%uadxYmJEOOphN?lD z(n7nwY;lJV@`~|0*kt5Anw%OA*e7gIlwh1v#(v$zf#1MUis6s~!*hYV-m-rV$lEX} zUtwVIIl^k8!6-9}=lllM9f$r&#vEktS->YE|KjKey_R)q3j!=PXK=q}S+^oe_gd|vFzCRG-ekccyV*Umb~x-yD*I7(DBdNK8E z33T}3HJgp2Nxj8M;fado6edjvCN72&wkuENuP|wLF#fh;6ftO2_#G&;ty4^*$aAJh zV%6e#8fo1@tND(3t@-d^n_PkTmA78mpPm)E2|2N+&*4$%6cNATs8VpSTxCt>y+i5? z(&qBUu;{q(xiBi|95%YrB-_GaT;M935Mg3-=vu6dnJNQE!vyUJr-RpgweQ4=ggutl zbCj%*(TO-DDZ!Y2_Q-kH6S0~%qW1s$deqm2$)ZEg&ZAjpf!?+Q${vU17Dy=MxH8o} z4cTqTZ1Mih6bFT#Bu#wWpFTXm6D-AbFh=D=?=!!=?>1n z8?#y&w`~;L_)9R0|K;f$)8?uO2?*TdztUsTXQC>2s;6isb3oA&fr85((*q5Cg``AF zD~{b;c;@6AS$Q5Om5_()zZ{g5XexQ^C|tqrT;ag|;ed?6VdX!L$_a~1t}rRrG%Dyg ztGONK+2FD z`E0^}AV$AI@7z|4{$fr8Co`2MwmS@lEzHLM3JSY8HCQ4X)=UdyOJES5a>#f}Q|D|8 zW0AxE_A&}7Fmi3F5K(EAmPu9E(Wt<|B-zp^E8^VgYsMAwjpc`9emv>0pT+TPw#B}*K560!NF0)hjA14A;{v{_bUr~bs=$YLxwBFQ1rD6wOwhDnp`7KeR$ zEm@CFC_OnKbmM@~okj(bro9HviZzZVXBv-|3bIBxaONCfzta06is^W7=FCl@>{~Mn zr-?@S`<*bdV6t#vdhzGn_dVx~8)6%OHTi#zN|S#GO&OU|Uh!tI|405f?_$h-#rZPKG6iG<}>6XE+&r;Y>r< z@5a>Z2Cs~U3C7I|3x3}VD%rI;Y|ag_xf9rUeE_fk`>Q+4zI4@(d@<8BOXwjhZu@ zHTF2Fw=~@9?%%+|c*v#Wl)P=lhZTa%Y&=3bPESe%4jnva8CO>}^YYQ|3wrLmb6$RWqA`7G z^n#p+n^+$UtNU+BxXR>mT1Ypl>&1nKO>5Zs<*gD56c=-_^T|e?Vc4Q0u-K`7hN+fG z7U#Uc#aX7B2M*-+$ryV%EKGXhqvq3Rnws_Wl<3;9^=j!(&2Bs$Orl~e5*r*EnJ?Lj zFNsrVG1U?*WrHSNJ3zyjwZ@VDlYLi35Cn){U$V51axuO><{$$qXcawNsW%Ew1vN4xT4n~m zTCsr1fUQ!=f~O6GRfzH&1TWFXCxkXF>y&KG+h+yIKap$ z)4|Zd9dA2U{fy7A$-y zS$BY;nXhKU<31U#2Me0m#1w8aa_DSm6m&BYSTebXnv;SFhk)S zqg!`E<3*8%vlrZ?eSKeC5n7D(h;V@!I7Nv=SJayqh0yUe2O%Az3PD*@QNZx<5YPa3jU$H|>WtoM2=MuL09?l&aQNI#7X6Usvwu(mxG`5tL7d>DU>~%V| za^;)5N!{vG9{thMOWEV~L%J~|X_fl42qkvik{eB{)z)l%Ko%zCvbrpiG+|Cob_D(XI zy27QkIOfCjo(Lsn#VsHC8eSZh|1HpI$CD&bx{_N(;v$DfOOjy435ml-9~zx*Bnik~ za1yq1FcPqAV9{W3(VUUMo>Y^_wNT&)Tb}`&{hWh5(=`rD^d+>+`?-cOI+;;Q-(VBZ zIyKXSfxL4iG&BcNu)vHfK1r2R))SdhWyRV%hMbf|3^45hzK;? zUY6V_&7vf*GRe)bL$E7o$3YIM8xBHMH#(EIq|ZD4L%PauVwY+ZBWGAdlKO;JmPW2A zMdv;>yRC|FJ)4=`t@beKpU5SF)C@^QDI{yQvfLk6@mT9Q{P9k5kK&x&}Ah+hdLt;A;+)QRXT$$nVc=@Fbu8NbEGaI%j zhP6Z-l9y_1=Mzw1DoZ-RyXkS8=9EKxJ{+zJhbGuK{W!>3_rXQt3;XheSA^JdUpR6v zN?^ANQOG!;a^OPSHWSu&qBFQJ+>*~y>~-DJAaXB}h4;~whAg22yq7B2gft4Af+CMI zF&q&QiE$MOn0m-MpuxjFaN*(oqKoIPn4tdXOkk_lhs#`ZH#iCV8wR8*D)P+JXq9JU zVbv1zi1vOc=GkJfOT9&#L%)-=?VrgJroLr?ad8@4@A?i+xwCMJR^-jOALa>cP&et| z)W}>Z{_ex&441=VXE(40`XjFX z9v8~wt#&Cmc*KZUt$TGzv`KgaciEQN;^NEx8BM)A^^0aiq^3C2vb%5YX(XKfd`R5o z;8OcGMxI0qu8@5P1bc2Q6g(r)s!@>G?cec8c>4!8)4agml8S{4LLI@3Y7-jmTarYY zXE?HMyU-%`CsDAxgNaS)Ob6!)Prvdi2a#DHHG^831j{(i%~4r!-D=^CqgTHLo3bbN zx*0IaH0(HN^l!lpHkNiKzl1iEst;#{XRO+E#WRtODUd5`L;8V*3df|EIj|WmP!ip~ z>G|Zhom`Fnhvsds;(x@E-WS{wc}5{)nkJK>LYA*gw05Y#oc*flUHgv2tW<7vyWM)M z>~Bh(N9SYda`lWgF{;|Pax=?ke_(#9x{JG1pjEc#!@ov;8OP_g{}1q%Ech&~peXn( z*Tp#RW9QFVzph-|;VO8=ih;?*fkE;`!$GYWMy7=ItP(pM*d|@!*cG-lW931I#J?Bj zy4pB06s^jdyGpivsc!BGmKTi5O9IUu3>qg%o{(C`BGB5Vu+U1fqT%+cX1CWV4$MI> zj)YqqvdT=GWb@jh{lMFwZ3lgwI4AE2)%rEJt9Zh58Lt;Dha~x)srWP&?&xh2VcU|{ zzhcY6DwgX7zUj}eqDtzFpqkLk;H2Ehx3EIKju zf1_tGS)F5$o04;*Y1xryOZa9q3d~5{>(R(~$B$77G~sfqC^_H<%S^Kqfi)-11RgLd zJYd?il5dG3*OEq&MGHiP7#kQ~FbOqqb|^~EP}sDJIY+L6iLal@NP&5hf(S>GRM$Zf ztA!F)2SxlAN}PN6Y1tbIv&08e95|Qhi1swzzvd`=<)DnsdzqkxTvHUju2~==m9D%* zfoo5}LW!}IlW9ao`;Av}m(w7t`#&Tgr6Sv;WNn&T8Zd%P> zs*o`40rv-n{AmX`HZ^d4c+Ry!f&1729)*oOD;}`1DC!?saA}hR&z=Q5EdMlx4lEF4 zTgb7K(2_Ahaakj$k%IKfW~Noi zGR6$7Oo>vr4$D02WzT@3IlvKj>(5_k+A3L70X%6O=jnW&zb$g`rsTB@*llstDa{KQV#6RBffF{eT!)#uRGD{p< z|LL93Y53^$wDm$^c*TphKhI?R5=HwCN=&*iuVu$w_@3##?PA! z&P1#)+~&YlvN~SmlfyQSb4>|5W;87FJ03H^JJ#&e_qu$(-bZCFLQCIh?4GfnfBpk@ zre(XE9w^S7%(;e@$E8s~V4_%&qIgA-b@W4?bqt(4nnm^;U`|`WoTtG4k3ry{!@3s@ zeBT!QW9>P{*brzpH^@{=fqmBjwiyh+XDpP=abVz6HC?ueO^#7O%Ewc$VO2vXlvvGp!1wO}=Ppj`1q#e!3T$g1vw0*+EjciI)hDJ+6PdgY zFnKx5W@(h*SR_$Wz)|?nHi)tC%}be{M9!%R#=Q?E!WJ5zXEd?CEYow4E7X{KZNr(I zgTe+(itH*}X~B;BmhPS={9WzSIU^QjR|n<06O`?}G-fneG^SseV0gOMD)J@v_Pl2PTf%8$9_lpPY9V~2m3M_gD zG!++cFsZWq`S&VF?*Y49qM%Tt$R%$n7Y>WIwi^-+e2WgSerph1!g=(^@>y^EU-^W- zl6t_jB=40}1G7{DLySWTM>C@qgP#5ZW~&8ERtn5f8(EDS7}ORpCnYrH9pLr5!t2$_ zq}9N@j@`}Mfq7G4t&^h2Jq6Bt57_%1A3ErZuHfOEzClPUQOGP&xa*39S8-hb7URG~ zPR1fG2SqQ=mKKw@Uypq_yCY-Aq6F38rpJ0JrI((X*S}{uBhuHi{z6dS(q9{X-V>=> zkiJ7s#j5@R=d%VL7e#;jM)9IX@e7UO6^&x&7KlbM3imPCaCZy7OAr)e6p(l*_|Kuq z>H$;C0yc{StWgW5{cBNRJLVgp5i)1mK{1boVoZ%9ZXa5_mN9Z2U>14J~LdNfKca}-#$HEiNlCW8mAxfkqB7+7Vl%iL_X`O{`>EG&8>l=H?^Aq_@P@6R63 zLq*ReimEk!UZuc!f=ArO@zIqx3hTctWeejdnjF*Cpp^ZHwZdQdsEP7Jj=pmtS}V9! zXA39aFR8fyg}p(6eNsB-nxlLl9*Ffl;Bs&jFJqLwwXmx2A=kMBMqA!{KX}c})hPJk z0pGg>ZllbsQU?AR>Oy-S{9|6h!q4@9N$SD-xSZ686wbap$&7?#9k#>;easCD5_uLg zZc7l{$0YD!0jE@=V95f$7Y^)u82B3waL-B<*peV{Azb=^BafS-42R|_o(+OK7D&!I z$j#KPxm{to)IzZ>jk2#E@V9wMacuwJPC)>-fOW@j=%@IrYZa zn^$9>mWo<53Qv%Ti%b;mD=7)R%++F0`q%VZiMCtD<%I`WIk>_WDG8P8rLe~+EcR1S zmNxU5dnj&!Lek|D$AGS-tDf@TH}UJ<&dGGqV2LotoCACshd45d)GscSzrZMN$5R27nVf(jG@<@U}-a+nZ3&ds}^f;l!rsu#Qk-)cT33JW@ zfitW5KO_iBEu8vaQ7Gd8^S1?Jhqj9;E#zBsQ0mnJ$u*4pYZ_(Q77DIuoUlQ0)s4g% zl{@*4G|GN>Ab5#UU@rzq=$2hm%zzHZp?nd6;In?(h26|gOSE@Q zToE6C-pjl_+OLW8nGT0a)c3yEtZ@feV%W=#FC{j%m#d1Yy6)QTF^eQkf{9|E!FOjJI;Q{}L2JVsttX2sgbz$rU z2Uts5*kTSOo;bu>uQCKypNMMhG&<6#fk_CdV95lBdVEw?r-f)1mVSzvv zqd`=p2uow3T`JRoWg!_AGQ!ubc1(?ayi@p&0_PnC0jtI>)(eGY zZ(dZ~lBk%azkAVxrw_IyhcBuqV&JG_SQ@yCGfPYA(1Cv(X}d&q6w}Wj z9@iJjUs))-MBwz#n-V6|g*?PXe{79U>@STBkKV#jvWL}4!HDC+2KF+C#9#+!i>k$C z&sDB`IXAQaV#4;_kxI{!^p+m{T@e^%#dwVS!=qZggGNzd!X}B!O8*IQZ_yW+(I_ay zmK)um)3G&t@vi;3v5pyXI;5r}u&T*E)?*9`kt;Loev|c*< z!S18d{ep8RJxEuN6!VR06pKp~&tT;4bKquiJT&b9tDV9sMYdHFt=N(n5@$#q_%~T< zW$^+=rUnKrg=CHdP7X&AoTy;B*f3dn8U~zI-9DiWDA5+D)rAs%xc6|2cQMdWhV5>cct=R*t0w*u$ON&(7 ztPl_xCLYBomXIi})||en!MaH**L*Hp)`FD_qu9R)aO~KZD0NJC(yw||hvRGyj7tt| zobB-MxmOH(*Mt50A2F9CaQf76Uy0|*c))D-hk1qn-`NjzwkXs*x_>0qcAB-rHO5EH z?6UG|FD^WMc$k+*)+*-20Y-LS^@TR9TCTn#tXyJNMNe)RHnWLHIr$ogt_e~1U%q0d zarvt2R;8=MB%@t3r&^UBD_ixN-6X@lbY;)n!=5XHmTE2VtDMCox@yYG;DsKIO;gVb zYXr{Q^7535*97^b`Y$gIHXrX3F$n4~P;6qkBd#{@M(w4id6U)rTTB=~Jvlykf8Z>R ze+QqiPmu4Ze-d#<#XUvV;ng2SQLZBs{dZ<;&2se?e{nv!$7JK?HGz})TeB6Do;o)J<9U3kjZtOC0T?#sn6c3mE&F&pxe*(N(Iw7R({%!^Gna)V%_Vx+;61!k=xp`$(3Di$AejX77QPk zi^W}0^vtvS;mKUU`!`JH9|NyIfFpaW$s$%>-I4$oM%SEE2lEtOEMVm@iCBJDXns_< zY-!|-U3b}&Is#+m-+tZL%2$8oqoaJO4x^iV$Q4Eo5s57gj+{C-0uI+LwD90HmI#<| zpix0_h2vH=!QeBRb2*r}^p-!EgQwr0L53zO7?%(6rZs)SsHP&OLvyg z)R2&kg`y&P4|ZG)x9Yl@Ea0$+m6>02Luylu(uJi?ll?y(y*{-uwkP#ir>UA`vRwI0 z=NVQ{E*z0tV$ghqJ-8;ROEXM{@v~s$1jQ~5pEDmjG-9_rXw25z;=s&j^=AXCTnxj+ z71^Cd8#0zV@N95kWYlqRU~e{I`q#QtS47Z}#nt3M!YsUzx{>zH2U6b80Q0 z<8(~K@RfD)ezUBli^G|_uO2>_`8Gf$_R?g9zrubzZcmeZMbaXKBaN0Vofg-1bK?x< z?np`HvjQ%xlG`hS*0Noxb#T*NbVI>aW6^_$2bapKtdf@fk-)w4<;-lm3r+kx7BKs0 zGqKkg$Z(1*XReoEk+3pomI`TL5O~nQYH{ITi+zJ5ckPEm><1jo4Oh5Zvt0D8cy^re zlZ(3or(&41!GlIggC{|%Cl>H_d~FcBlfY)e;G*etQ79{AK{3as15a%h?VPwpML^@C zSexH^yUkPw=I({LdThyMqEnW-aIt(tb5fbCrONTP@Y*soafT!?UHRf*y11C}A76pOY>5O8c8Ny<%VHn2-IY<@ z*b&MsJ>>zDxW>W9ISL)Y0ew;zFElWil`GWrr8s(2Fq`^aTvWwN;^Vi)^i)V0}NIA9*jqP z9$Zr6IH+$DdM3W1;IeL|&+>^MgHn!sVU~XKK;gNukGMzU>Q_6?3tV_8%xj~->NZ7* zXO&EgSk!~2g@ug#+b%K%=6Z&7x*TteVQyd%H<-iqV4*Dx|z&UeX;hc^i`iyt^(mVd)<`@f{EY+bU`cuNdeqdhw2 z$=%?vvzK-a5=rKo8zG=&u!V0X$CT38EuOnlt_PI;eIoNI>Zp|NEzX!3Om06PX?U#v zw0=#`6yB4kCm!S}6Pv1bhQ+|(AYa0-rm5Fh4X!NBQxZKaaUvme`J^|Oj_XxVkon?Z z5T9^Fe%b_C**Ooz_Ebfblu$?xZ$G5;t!&YisHI9 zFB}!-u3pl*cFV4-dshlsY>xG2u~!0r&RRqWoyLkbd# z91ar>@m3|cDZdkFwhes7Zgt{y1OJSxcWthk&(LBjRN2tLpwPgWkia4*7TUVU*W7CaGU$8T&r;Gmqor}R=aO}TI?cpckT@P!mj8ehX365 zxfah_TeR3rJI0;OI#WJ)K}@UV^`)-24hlCgNE|hIc)mWa=Q!`?*a@m^DeZh&4A~0z zm1}zIB;&MbRdT&Lz86BhUOJJAFxJlD=@Elbs*gNtplIK0!GFO z2d)TBXyD3d6sZ5vFi%S1I@f83*}4&h+!O9HBx?)bD!!^_Z7r~p+a+supR79b-kZLe z#wl#qo-uGf>tdEZa9L!Y0ju$%jVu{+cX#^nbUxE)V40Yqz`65o(jIoJz+|79r%Wx~ zr=?$Cw@_w^#=m20ZT>1uH7_aQ+P`+9Z2$5PW=CcwIIJ>0!hG|R(o%-1ZcZM7mWGc# ziRFg>?#SN%(ejE}?Zw;`rG4+ke_iO{eZn9x??bcXv4A$io`)O?9L@JL7}^31n79tF zV-$ZiLAtYMtG|QBm4?}iy~TLav$#E;32_;AFxEFPN0vKqe0wF3-M{%3Z{&j8(o7AE zGr}2O*yow**er3r?v{MFB*L(P*<69+uc3dy1jdU-3N7t=fTeX?b zJTO0Q3D>Fxd~c1EwqF+Bc9csjly$m;<$~$@`46Il59*sZ#VlX0oEu=ko}8}5>XGBv zqFq`z?TdJOV`0O#Slg{E@{d*R9!S_S#91kXq;D+JR~K+IV9Rpi-on7OrGPU$pjfhi zxz>TJg@NUt!{+#%jA82zN-;CCymDkVT9C;5fGI#Yo6{iM>H^1+!c{V%L3+2+iv=q2IO`M;Ria)}IAVyDAo`NbuS1_g|!ADB%98vg`xT~c5b z{o%qMRKXdb+qIl=Nh;U!$yN$0xYjzS?%csOZF;H`qlj9Oh-j$(Wal&~QBj$xX+Z+4 z;SbYJ?Wp2mZ?XMQm8)2N!MM8XVPW*4YPro4D$gZ?G}P=8GMq|VEgN)pEi8Y;!CIlf zW#Z17%D^>Oux-8r_xu7@X#x2>0j`z_%zn%c<*p$7~Af#ry;7 zjz=tw9~gZYI9LohRtGYhOqgt1z^r$Y{kc__RQ_FiEvfNB(acf}xD3QMxbY?B*+6GSd z35*&E>^%+~Q3B4p8N;d{>kXhsw za#7G(R)M)nNUqK)L5_jZ;s7h#g%Ykv(}@NgYgaNyY*mjjZ{NI=m2;uFl>_UViJg^4 zjU^IF*FI#FWN0{F*)Qle%-=`g(=K-3X9Ahurw%e1iA87IpncSQfM(3@>n@# zt^x1<&w0^X6xA)}f5~jxtx>spF`u$W{%QlxoDfzR;LP?L!=7ri*r%TMLuJz22<}#|B&RltYVU>HI`2yybCCnWkSz86z+zv=;f$zHB zUcegez`e78#e;#h*?`NUD!f|BWQpYxW#zSD)O@EOQdSa8%AuxGS~TLgsj zh{&!MlM^*ywl#LXXz0c8qU^z!5Eqs8|C=3GeY2mwXtLo1zmf#@vJLE=0vz=MmP;IX z8x5w&EnME}E~MPZayy8NA@rZscg{5R6(XN!cB^Ytt}bYDU8rcWkVR}MOHwDdNybX{ zDCM9J`pdR2C}2y^zv<4YTHV;A$w6lR+VekKh% zPTVZ93s&!WC8=G--Da>R;sV!>f;B+~nxziR`xlzLI%Tp`#F=Gf;<88~#3f}eFq>XrvNvEi{=jT)%pBXGQ(V9LZ@}Uh!mOIWDDZ)~tbs{bfuYQSccFmp-0H^G^eIm|nRvZ< z{svmDtmewNGIi<%k@eA2O$~Q%iWWBA$oWrUHH)IQVxt0UI@_KT6`PJP-Zbab0*8on zgM{?6Ut>11+xC1^Z+XVNe$}eloW9yMyzd?>;gOA+-X3hnN z4?Sb%;$-IZWa9E<5}PoAQ!?VBQUucjwkr-?9FHu0{~WpMB(yks!P{qflM6VL_i!e! z5%D^BH0&g&#wO0B7|zr`Y+e&s@)NjNt9zuMM(`t=9_K@n-MFS$_SCC%-gd_FcewfRky@EY6u1xQyp=#-HHc znZS}5z}_3c-K)TP@uOGN1nC_|O!hDI=29`SFZMAs*j8o1!0~}WbOZb11G6F5IGaNfMO(JY%sfPsGlL&;o*x6>!Qoz5U2 z;9^t2$fFRElgr4lK~UrY+ZhKQ*+!$t9aMLqA`JGvhRh|KO4OSF8o|`H0I9y zwceaRD}__mu!IJ%2A8BAJbf%Dtt#ik1)behnGGzbxeG69$6ojstYZ25cs!Hz?%vx^ zG*+K5+UGiZve|+YTCvPK-mp|Sa9l89OYpe7f#L2hg~je4Si2Lr?ccJNnF?%bVP^TZ zI3|78fy7x$C7JCmcykwA=RCkzAHeO$z`*LjASlAf?T{pscG^VYdPeJk-)krAyKbSA@L#wlutAVXf%QxPm){ho=JFJ-*Dl=FyZ(mothBx$ zKbvbsz_FRRfwOCcf4*GtV)8XLl>x_A2n0^!UGVkM>thqRrWDMd;t(yk zmDAJV!sJ-iWdU4`7XpJHvZYUJKJ((30B?HyMdnl97qv{QuVwTaJb1D`>(b+Wx1V&J za9=%nm*dlQ6DON*c;|GPrQ!mI_XWE}8`Ks|;GEIG8R1YHd4aVtfqSU|SM!B6YnJ(P ztNeTJbD3%BO5c(?M#BpX+yWm1(i!VF@H8lNFur3j`NL6tfwlhwyMuxZhmGk5vzH9} z?gzZR|6=X+hkF>U_D~ME^^_)i6;zoevIY^crHy~(3rrmt}buk?uYMXj?PWs z%WvR1ZgA|`hS!W|UwavHuD1UA+~Lit2J00KZ}JRorZA+Oe$N-}8W((SV_R?F7CB3~TS-XD>_G$8t5K`CfMHgBN=RISshV7rf;C z$H2#MLu3Pk2LtbwhKK9d-RQR!l5%+cEa1Yk3A_CA`KC6!nPz)yMz3{nJ)P_^~c3;A#|Lm39>kHu$D_I&IC}3v;B>v-t%)E-3_FUd-MvXHc=^1|u_@vxG&_ zj|mZi$B#-`}+rnibdF|#l6v=ni?waEDrpX{7#CoZP?wThJr{jDrI%xCPzBi2)@aFCISyISbx zhUDjhDDLMq4aAcqo;3VPM9Kly_%UijM}(v|he$ve1o8I7vqb_|EQUH@e$O>vtuVbK%^qZJIzlJgXv_VMLNESc!GBr}CyFGE9# zNoAtS0!EI8iY7L`qKZY`wroEaH>)v3CLMEob>cyPnn$AJIt}*bhE{8-jmdl!Ge0`& zG0l^Ct-odltN zej$P0Z`bjf`uUnJ>3z`b&n}apu{-D2o5hMtqld+qIH7Dwhs@R_$SIF+>w8EqOp-r??6IdlIr5dCjEO- zuI;Kb6jRyMMFn`6xL-B6a0Mx3KAFhPQgBJ{(5tQ)>kkSQJZR`t*Xv>As@m~z1B*fd z57QI=%LjO0@*nQ;H8b9wV{&mq!QvT7VqZV^nk$&br~msRFB)a>aD_o?kk+)xN&mcO zC*QjKa@}gF*MB!2`;`?~sBZW5V@LIq&66e{IiSK-(#V}OLFoUI6t?|~qP?sbSgb{u z1k5>DB3J^sjVcx}iM(iFFA89{vSAch!r`L$YXht7iv=9q4Gf%71&v%j3z)1{9Ol{3 zbXdwDNl;he5o=ySvsKR{KFbU@-MRzTs*VR(%Nq{zi!pLYyvfsPlst1pKwzo9pQcyc z!XR;VF$uj~&t#De2Tp1zG=L^!7G6BScCSM1+y+hN_LB!V-I|*$Hu~2(2h0CHS8lh^ z(yLa}!->B|nKdv#S>VjdD}NRo(_9wmKKVN5)c(Lwj5gi{mdOtLGK z?qrfY;UIotZ4+;c!rhDuOfp*>*rOSaDD<7^$hh%HEON#H7PASB{51?*J`IPITnf80 z?mQAp_~GzesDR1Jz=>Oqfl+kMgBI12M1hhHM)_%m?K$Tf9V!%D#q$pIq}n*@r3s#4 zU^}o>UyXs`tn-2R6Gk%*wg~ndRrdMFCOqS~#R7f#=$>O<3btAujXWtIn&nqHv|8vs zV^Q}w;H)X&ZlCp_U)MOr(^t#GeX0p(wuIF083A+b0v6T>MVCB;fjsShAE~G8kzonzQX2o)qaoHNoMI4t0Rt0ve*5Wv4F4P0=q*^ z0<#rIv*NJ?mMwCL%nBUM>i-hhgE|(8%rj_FJa=KT+LZ^zTLoMtekE`?-B6O+>d|ic z;X#w5%_70Y4Nl4-;a#dP8br>#XnH61z-rHh)6`z2)?X}}yuVAFiM@rP zH#6n{2Wx_l){KMvsRquuX9L)7`90)ldECSLrefKIt`NTlmL;Cu)j?C`Ce><6_-W)k zWcN00ue-1@Q>~oKYvs?96^^zoa=WTI9AupKJo>s$&L^@tc-i#uwK4k4${eRxTWx6E zbVf^eWl{SEwa*;M8i&<0K6V=QJmk1r;w0{*$hEu6@u2O;LoyZ(%}!Si32^QE*Q&dt zkUQ^0vIy&k0}2cmS^4ig;1j-efLATLoxvujo8P%1LO?P2D#c&;4x_NDU~6U3{@4 zx@57S+=51St_jWhA~R+-zF;<$wQ8Dpa+1+2+ z@b81zsRsG<1WtD|X0g5oR_%y~Twwv5M6Nk-{>f(8aL}EL&)gxnbT8kXe(oK=3Y?D{UF0k0 z;+-|&psLP6uK6cik7qh|8z(S{OrOW3=;CoD-8!mjsupu6^=Ys0?cv;8rbzK z7@x;T91$zC;JRwL@Y>Q2Hsf~+4U8JQ^*BDvWvuIEP%>Pj*XsD~-3q>?&2E~E>^FX8 z9Tl3Xdco{un1z$*v4mE`l!aVg$DAbf9NScQ_cYJ`bM|ju zza#Nmi|#+JM9o=2(=F$|n&LNG%*&j=l>foj&B{WIobC=z60_P_p2<#(vweNDk-sK^ zg;Tar$!SgpuT9CTOicy162(h9U%2XDILICM=P}p(9j%gm0&UhUhn|$kwn*4CEa8e! zh_L?IAh9j7L1{+=f9Z|G%Fhb)ubzF#7SC|RWELX_ql&^yFaKXmDhUl6HD2oZO#hf( z-~RYg&`HB{+itL}a0oQ!ka)Yxl}WUxfkj!N+3*09>h2b!1&enkOTH82-E-TzWcMa* z=cWV6de4|txRx~WePYq>H%GzescM%&?afHd_;ImKLm`U8&3~!EA3}wB<94(E=vk0}LV>jJ!J4$0ytZ~P(sSD&^0SifS%cL4O@01CQ_VlG ztAA}HqQaV>&=$wi7XQUH!Ch+yq#Z547|@ZdLlj zF3_>Xn4#53gC$y_)#=}d)w6#y%dS|+Qo-<|lYuY6eMv?Gw?vcU4CWgL8P5M~PUT>g z+QFE>#LcvzyK`a#$AkvU9tP%u2J0_}ze;m2OWw8l#m26W-7Fl8+KD}fHZZVBZ1P^A zyX7!z%m>kx8H~a+6hvk;dB0#`TB7UjB0O=C^?3o;fRqEJ#z%wpSU+0j5aiM%$H5ZK z!CLj$f$z-rS0AR#&oU0s(z5h!aed*M-oUoJg)O~+Eop++DwVd>fTc~`dv`qSm$I>y zEpZe3aq4O|qu7dR>Jlvm9M(}gTEY&rC<`}hOEmi*Xf@W@v+%?My93R(H<)EUFv@o@ zN>=pCBrwUpV3a?=XWhW!P{H(1I)hPqMedxrBcaLwo*@w&f*lpSf(;P0^3bw2;{0C^3OiGK2BY75{aQ7^GE{R(|m3yTGW< zz+z+2WVnFUX9J7xfu;$ETf9%O#2GLx~f5gd@k{MJUx>O}L&YL*Oz_R0zD)f?KY zAF$W$V6R@mUNxb;(&2njM`O?{QCsKr-zMo_E^hp_Wa*z0#y=&E8)Qyj(e~zB!KnVD z+3v31r)M~fjkeft{(}N3cGn{f>swyV6q-L;1uV&p) z(3BQ%%`KtHRl_P%pq1~3I}^k7JqH5SUT#(T_jgyKaS->48C{w^&oUPqywqS^z`!$u zL50IWXGc@?i}pGOW~E)1?z~|TIdLh4snsWAw^7IL^FMk_C2tg2U%tShX&xJ4>B|^Y z5~23MGkONof}W$1Yx?YMwE`}kvw6^L%g~lzz+Sb1z2-!F&4u>KSJ|gtZLht-UbCUy zBXm{&4X>3|3!1HWICeM6Olf4(3RODgziyMX!j5LU9}EHqn0y*Smo8rG7BT{$P*_5D1cBU3>J53RAbfLhFu2Y|#l#oC*wF84ZOi z8`viFa3wHu?P%c3Xwa(VN?W2Dx3W?BQ={|;M(G!f{|==zsr_hFPgrJI(PSwx!+Ha= z`3`3D2qq(gW-SY*k{9e{E7T^cFbX>~avC*WWNkEJ2xgwc#GA0#NK~`=X^-j5yFoVF zpOr*#mE179d-Bv4=4S<)7wVkqan-VYz?}G_t#Uzo^?~-Ou{S6C-h6hIy(>a1)N8hF zwSRN=0)N)4|F1+wX@yE{C{xX`%HRaSS0m=g!H z?h8)c8%zcXZ3QVzvOAim|87oSV49Y2J3)&@qo7Is!4$n0O}ZPJtu`<_Y-qM;XyN{5 zz^B{dDsVBdf<;`BRY;XJNuxC?f=y(bX>>K4NN8AQ23uCezlT9Dnr{jja@-5TgTeT6|mt#dbE{kknoTtOu zmD|@{yQlWC-&>tH_Xf7|746j(_g?hInOA;Pk^jM{!64)FK|kQfdZP*2$5#3zE#2y*aLZm`mzzS1^@@N@E>@crXEJnUcr+U2 z8(LCdC@h@8woR04t8MDm(1&RqZRu-!(rX{46&y}aU`vZ-3)+ynRhBh~gLPAEi*}Hg z)$GkK13&g^Uyjul560oqsEsRQPz$eCRoS zV4msJSPeZ_4ZG*HsW*c6Bsp-npJKgd`*o$6!76)MEk{)OIl1DzK#)xEmR;bQ!S*X0+T=Yz<=Q4lGz!a6wFW0i(^2P)CW@v=wa8 z6$`o6vN~wYwSV!z-h;k)=8SlLpG+De_`#Lho#57p5?vd%# zBzvj57e#Xpm~l8pw1v%at4ThWU4Q*Jj;|9js3!F&IcM&bXbyMO%4 zk-owBMJhM6snB+|f6A4cQV&*1{b00;Xwh8IV0NK7^+&Vh4o2l43|bqSRevz3B`}!> zM2o&<*86eQ{=@{m3sHI!6S7}wAA8vxxPmq40mt#I7U=^F3!+(iT6lCn9oF>RcECbT zFG!3ejO^-XD8qt?Dh)*7t=7lIe_s4h;*dHaaj#e=PU#v391kNa{yil{T~&-r*> zEMu9|`U~sgLLY_9(Y2LgU@Be3Y+3oqF`{W3LyK*Jj#EREh5$>_f;NG@%sdJEtZpn7 zxY~5)-yOx`H_URGjq(|d3c|y?B{xbjX{)?a!S9SW+9>Czaj{?T_nvaXP~NW5oG4!CEi&t!do9=4=Fls8aI2ANGMzAGW{JN)?cyrrUHwG5LRnhtv81)$f^*=Q1TFU5V z@Y215iDSjWb^T1%2~12H3~UD9cO5uX=UM*MyWBXbTvM=H?zLgXdvP`m#&>0B#4KK^ z&tTMUYKkX`B;C^PF(21Ts=N6lu%-&b`$a{Om-O?Kz z$73A>qW^Gyt$gMebe`F$TkM{ma-AS1rjxQt;bOp{L!8`VYV%gi5M*`j(=eO&BST5WbCP`E5{J%1 zEM2miRSRwidrgtn%>8#`!_J$>&ia|>ec5sHbMSdZ_opu&US3|Jd3l+iDkI~gWUZ^K zENWto9!y>pCKFkC<$}~Uw#ZF3NiC(0Vkag$R#=2AxOjYmAitbJ%#T100pa;eX0>J= z@o0|~k1BZ*5pserk>7;v;*8H7=UcfsSyCU?yR%_~%x zHFc5U1_oy4E!U*7Y@*#=G%0f-9Y&v~ss^L*~kqCi9oS9)Fnyz96Dl3<&WL6#WkluU! zp4a=8Q=GUy2MTleFMTC`#m|Xx>T$)6Jy8#WI4->kku5QhxY`_MDY0fln!iVAn^@IX zX7OsCqQmU&F^!zM3sRH>)Y4aIoX}W$M&pK@)jdNFWj8Y;4vAt{Bab~56@}g=+ovRU zC~mI^?$!OYL79b7(BJ}tK(S3BGn=V}Av-gZ$v=k=3_K1BQE!&Y8hkpzu9Bd}A~0iS zgY4od46bH@298`RsyFJEb0|ejXcTmx(dar~=l{<+^HW0BcIm2PzlE?Kf z|0)%hPPkY-XVdBYC_d46E;Qvo1U#-%Z)MFi0zBaq&z~Qwj51V5mTeStf zX182;CYqdhG{SBVME<<3uH47(ybb^a+2g?$> zFO2*N87y)o1{^VW4hf`M99GYH$Q!t%SvF3Aftjg*(ZR-9>P&#R@w|&1TpSGgWj`F> zxFm1vuw78b$-#I;D~(Z3hgqO!LL1wi13V59N0|6M4t)A2!_dg$lfWFT09wj*Tx&`q zPin+DkrfjdKTSB`b7r;2`6K50zVN)A@Y1{U!nO7-ix#HXT`rLFd$WD=ychZ!l^de| z-xFND^o&-6Zzqe~j|Z$VFTEO5yAzlcuQ;sIF<{TnJ|fi9z#8>sk@$-UcB@kX=224~ za)b!58qc`MQ((AApjjYUIm@ss^@@|o9U<1*EJjIvm(M)4ADT661bK}#HnYx`c3`kz z=y01;DCj-qm~hy2CN~u(W=7?RT%ip|CqB(f+Qb=Au;e1!eM4m~&JAXOyJR3?cj4w*I%2W31I zc^!Tnsk3pMs@AxOeSyLwfxpe{46ZX5cubhYKdFJMrE2Rb4yMzNf?Q!N#x2bM2j z;T`VveKVW+0uD1WF+50Kn!qITZUK9m4P%#>fjBpl*amGqhWR`SjdE22Oh&yqjGPM? zn7s^|&u(iJG2h_8q-Pbtu|m+Ma{-&Fm*Me%ibWz#4vnt%m-Y&<82MUX?oi~PsriLd zx3*X1Wp&yCIf)&KB4;i%tER7#U*#CiUlY(`^x&iCe6xviQf95@197E3l}9BOwn7YEo!cSQgNgwBd-7o!!AB9u5=PIi@e( zr|aqAHtEE~60OsHCGlJd2JHa`$@~^4`~UtBj$N&INbu_46pfwFg5R=TZAi6_4mw^f zy6s}6lhc92$_ExOZ%A4sI^W>1Bv&nK%bjk)GZT&}ze{Mp|F)6MzJQU}$|3Wg)RP08 z-Y*!0FSRx@1Sl}@%s3#I>nhxzi@((kRj-aAZ+OAXnd%1XZiL zD~tUutYn^XJbj<2r}FK&`*df2SrS#|tUOPVW9Fmi$*ZNf8m{ETnkScQc1O6V{0LxY zQ=M?m<-oy1(tZnhD^nlFCC`*T+_#BdLr+3TH1UerQR((a2<;!J=8Cz@YKZ zsDZB`p>e9zv!zWF6ZrMFHB5_m>voaHgz|W8uYwef(&oXfJEI9V(OK{r1C;I$roLEZ#%N8tG z(ObbP^8T>69CK&>iiZ{REM680Fw6dNcy)$@g`=;4Eg*wQcvgq4ZrMiesGf!V+XNDI z{~h=e*zsDhIfIGMZUL*ol|_9awH{=#dt9 zz*gSi#BLD4$QaVV?7$%*$o}A8i{&?qd8rLH&es`~2*2sE(eIWY1lgrs&H=S^{7 zGdRflFQkFjf{}3%!%0DAQvqi~17=eXX2TnfnYLbM=Ny0L`anh@CTJ%)h^bPBWzv0F`O~L{W%nb|@cbJ~tX_ENTxXYv27h7&>+4W|pEgew}vY7WYMb7c29D6ex+M1)CL zhf%qOQCPr9{)vNP$D!M;Ork7Kx)zNJH=N|JI2zw*RPbreFm$#!A;r2yR#=D0RAr*j z%SPA8hVM2F|2TShD&AZ>SEEqn(KU5u^zOax%5$$QpK*Z4hEc`fuzbvapyqnM}d1OO>B)! zY9&qTUmPcR9`5OxV8YvEa^;Yi00U!1gTxE zdzbZ2k4~P*xrzs_bN(^7gZU=kww`3;8;8VhG(LH7NPUa42G1?QA5Y}(99(4R ztkJ_1BkCk9aZaYhfz^kB_l~1-N0XvSys`-+?-eJL8HXe%9Nc2CMDfNUjS2?Fj#$qLZPb?O$3;uD_BR(PB2S*GzLT(+T6wxW^G z;GnR03HyDNQIJ|noAiSZGsjiXf-$8K$ z$2mcbvI~}r>|hiY5ED@lVNGG+;$Tu{VG=H2*?IEOi6zr^Jx$l^Jw5y|wYBWUk*8XGI~W8$ zI4YYs3wxw1usCThd8nYnDAD5Ba<@@oN|VrEab=D0En-aDdze&OoV+d_KUvwxp40fK z!*NI2K~alF_7?}(MI2c~8m1aJaEG|DJ1~luH1G)=v#fI9(>d62fq}(Fl)Z$3_W}dk zjsrXi4ZK?p{GWQCb&5QLo&>|BYpe#*N8H}jsD&`491w9~WK z6O(ezg}nOFUNDP6#DPVDN2bk>RUv`3#g9#(q4{ertAGQm$pMA~t}l)x z9&l*5$H%~u;J|!h^36zwEW?Jv1!1WX2UsIq*dDmDGTi;II)(K{g9w9D$+?4Vf?9?; z&5Msb(wQX9QKib+(U8yE%f-RVIYXpyrh`C@mf;nC--C^3Z|STSakV_a9OSS#NakeB zu~i!qG>#a`w7p}v?ULUfwB9^y#dbV!j#vJ1?r6`Xj z57#_E^-3)b*DLeme!hv@zx3!~MCWkVx<`l5Tl|L`a`wGj z+)6nngPLaR6VB)5^e=E-6?l+pRyFTylzx2G#_$~tFBu!fb(~~tI5ej)O$cgM-?8k> zR44gAj*14(COx|9eNKuYx6(@-*i#zVWflq7H1I9)=H3v+pyR;(;-Hcb3zJU|qe26- zhXc=z2JVgtnnYt9rR;WF?zFL+CS-qvRSPL9PWD0-145Z z9bo$(&o+ZWG$;E1!*)hJC;l7f&ZZ_AYb3i_kMPN3&(R_zDbx)1sA~R*)m1eM%T1u#jYI250(ezt~837I0|PRoY3LKSK9dgDgzU{18azbtPYc~ z4-d0ML)S^ad+*M1MsN80_RcxW0~-CCZq77$!*YhLx!~4YzbDJ)PTrfe!SR4rydKki z2B9w*Tb5qj6nSLv0_k104BjdXTnij_xH^8`^4#@}n80lw(O3*_o>>t$qT2=b#UN# z)5nl>KwiYzD&Sz+T#4@jA_372oI4nm7dR-fFtHji8M-i@cyUmEq#cmh!^omv~ zPi^4*bD+5Ykm>G&@9+7qn)1rb|6@e8vB|r}30E4UgmbOEJFLq$#%%uF`lx(eU!ml` zd~U`B297hlcTL*6G8)ub+)lbWuW3Df($LB53X=qf9yh~5r|@P)4i)7oc~)1P@f$t&VON*jy|CAA=8{8{h@`Iv9z|k9@~}% z*98XJ_tW#78@M`duqHG}vp8ueFlJtrD7^nczgv)tg-LVAK@*Ofk9IZ&^&1FeC%iE# z*5~r&d~#R+Py_D`289&Uvu4*Tic9@GH%5dPu0EI0GH=oaH<{ZyG11fGK365LePq5~ zX>B;e!M|pQ*eV#}`x}pOo>Gr#66Kiec`jK)hw-x8WjPaP-75!~#Tof{4%rJhsdO;9 z*){$XjySB_^6cpYCcPI85;lj#_S|QhaKL4U$-Qk1+%9^L7Hzr7$OY)aP+)&1X| z_~m7XJ8=H7;EF!LwtGunbit!E2X=*nJPHTXmvhdGsTRB2(;wL&7n80e(#)n4C$huQ z@CM`B*>Cvbw+C14N64*U#Gx)H^VaYDW~*96EMQeLv%rebsC?~o{d z(Ro|9T8DEih|dr|z5Vm4hSnViB>t-kTsYWPdEm~B#=A94x(Q7V4TmHe8rVu2HGUWi zmozN2KDgrsBkv9e&I=B*9Gw&X9DHcfs1eevF{klq$wSE(4Pq4yEEXReRt7#XJHYn1 z)$_20*Kzp`?EikRYyXyIznk^WuZ+zMYzKDaOlH{p_h171_vEz`rS2N+3T5F9W>g4h zW({aG+2JT6(5x`wE_+qw{oDn?X)J6r8WdET6d0HkRGgHSa4YX%yz_Bg*zBc`k6wwA zI%;vC+4ja^TZhL7b@i?OtG506SrgoGDBR9UU3Y!eKK`D?x(=rW6V0WKz5$An>BW%RTJLq=pCUw%m&| z@p`^%y|ev#_m!+JnU4lv=iBFAYOH6Iz4dA($y?e>Ji<~zcHNQA0 z7&NneVKBYH$l?2)z3Ke9e|MgjqRrspJn8N-+(1(M9}K9Yv<$7Wb=Ry#9WyYkf$ zMd#(*GLwFo7$q(7(Tmx0%XIG~*30UKZZlXG9qXC0HfV9jM5(JCfj0ts7#M$je(tx# z)}*UK@`jU180p1i99(4F!p$osl6L0?6RV2(v=omA4;*uD^7GkPOjJ0?thUfeFnI|htD*Tp z;i($}np{2!i22>HJ2>YECy&Sy1Lc2eCwR+jo1AQw#X7ER(#W}G&{I*j=%`J*$*$(- z7w_$xp!TwxMaXzrz;yec!Yef=+s`jM)M?L@9>uAq$HjS=%eW|U)%?1GorgG#?LH=R z8hLfL%?z6MvTKTdT|-l+{I7&3{nAzi3z_*WG!`|my-Rx3th!`FGN*NhMk1f>A`QmI z(>e_cnAo%wbPW#6El4=2D!rO#!~P{-vK!gtG~Rk}cnPm%~i)#v7k4}$uW^x zB!I!MKhuM8iCC6|(fd7qcN>})Im>CbEC2ZSWc7Nx84-sTs)ll3>Fk(W^ow6(_P5(B zRTmU|ILPgA$YK#sK*qyX@u+hakGXX6EDv`WELylsvehajrBiZEZX%!CD}{xvd`cP$ zOdMhl9N2Br5+1b4&nQ?Vs9~@od7?v8f&wF(N5h)+HTDattzre>{2~9Hs?im~viCN*;#OW=$(skCSTQ3&!s@#fTUbJ-fx(wTdQ;O+wRc|&v zm#Yg{+P`d8)YL?W>3N%`zL}m?s~I+%spe$69>dGUhyM8+%#`jmeqy$Gs(+tFlR(-E zPFH?6x0S5?o4yIK@H@PDpfqW{N@hX>qt=0k{D&UwVQ6-dDqO@YJ>$pXma4?fXEq#@ zbM8IS8>0q8ZJi`OuYyD$@t7GF4-a{~Si`8IaUy|=tT5+KJL?gxKl{a;9#Fw+J=7@gf%o8napNzJ`pK73ae-3ai+;Es@(szy|6(|1fg$Lwh7tCNWX}5f2&C9tjoJnHJ z168#dj{E@^nB9%oS;Yie@>vXfOJ*p`coa05cwFRIv{jMUJ=sG*=8|T2go>b)BD>AA z!ma%x7aD#gaC&QGm~HgEpvE1u;6~o2(`?h;PPeEyB*V3lQ6NG?_@s}TZq>zZqm)Lj zrUW-biHUk5wGD#XGF8j0gMUmO`k zJB0HVZ9mRFLC3tO@^aFzCmu>{57})yocL#KxHMHsm&<0uJZUFE4o8t@sml}CL}wVR zP4zg;=MnJoqSwy}rVG+-Pyb-EymL`7JWW^Svty`t$#W)24S}4ntEze@uQ_mCJGDA~ zs^hw-CT6{gg@UXz7=r93a>k}C65X}nsJe|Hm%Yg&t)2uXVKGHksXvV(JsB*DQi@!G zJW2fhH<;Nu4m5BdS-|Rgs6kkQk(EiMhe>wD;YS4z7?1f~G_2JOe#OBYaQXn>KZA{{ z9n$p#BzX+pFv(`zkUY;3`8IHw5z~f7-U$yHdk=A%Z+w`Xlb1Z3`Oim2i4D#oc?QgK zcNo|Vb};fM1vClG>0!6^Xesa$lX1A=FVNJ$B2;&QCrQOcs>FlYd4gDr#)MCb&lyFl zr$uS~NZ?grx#Diev7`S_Sm_#(E3d+fC9iF`m2x0y>#BE~w3@|ch_03U!N9Sipwm0S zNqhmrQT0;~IHMVq1kOBIB7G&3xt!@yK`qA-1(rZ|-GYUjG6@XqQ481|3?8zX3b8!5 zTF|U^;Q@Q;3}^8SfyRq(ijAgw-ks^OI`ER>OoKS3OQ{(#EggnySdAMFI(r{zGM#md z>7QAFlfpg+h6i5!=IRZ+bCVWJ+_qW3QsBoT%_n*H{nbh~^B0YrJ`2B*Ewfhvk;NZdMjnGi=tBo#=Gx}`MI-ZMaw>kNayC5|JB4#!lu53reSuAH2-D4o&FMe($GfC|q(ab}4V zUv`=O>p1jDqM6BU!v>a#GJL`ulP08XXzb|gW4f$zcfzu>;zH+ULHhLoBe&q z&$Fo;vNrtO^L(Y7>FQwKFZ0=LSIZ5oyUL?>QV5q?Mk$v;^hhkS1TIB5fGx@{?)8Ep3D5m5)9ET*_ieJ#hv8}-)$E#-I-}BbpFIaA+C!& zYkw&6EE8~*{BxoGz_o*7!SYAM>JG3;Z`sV8v4Cw>!Xc{#%vuYWZ4$2ibKsUb*mADn zcELljZ44X@3s~zC_-`pN+&Ez*b=X6%fyH3K)}%C+(Bmx92`tgsK6(uc75Z5t7kIk} zut};J{ctr)TE;xbL4e^QQ=IZcIfaKe&iG|K_GRew3{hiSpnZOu&#r`q^V14adzuAV z4zd_AFb5>?ze=AavC#TX180>2@04Zi9xL}wTd#QSuwchS=~D+~GtMcqbo^Sg{@Dvh zxdZ-eTlo3+pW^H1dl`0?|NXy$HAnOXQXBq+BwUp_#{O)9*sBL@$9kkzByz266mU5x z%+@Hvk;s{KP^{nt_lyPM3q&9(&?3;fO@!hj~&7TO$N2jy>6`*T9;fou=39d7~ph zE18W;QD)Xc7N-Y{P6_>d4qrQ7vxK~7*|RZD=K=dIhg6Z1ELjO`I()m54zMiB=I?Oi z4?4`mq$v19OT?{FiYbxJ>){@cMyW3hJQEeB$edNYvEWk7I{_I*iE9NicLILJoXmQm zsGOZAz-V}trSP|%kU;oaj+iCSc`ge4+qs^@ih;c$LGT>|r6n9!lV}S-@nm zfGvxGS*n|nt$~4QvEh`9^Mf`RpYZe+Ni#V!X^uy;j|Q8k(qg7JYqz=?7+PFlTsDy@ zkAc(b0i(bJ)1<{;g$}ShdD7q49XI1J(>n#hs0VCG32d(dzhyN%@=4hB<$5f0#+Tq-Zwo)%@6o)JTQ?s`1+ND0LQ`C%O03GH;OD_ zR18-5DN(dSiHUb=*0Zw9a`z0B*ZT9BuT_p>k9!9|6C3q>Rj3gkT$dvYwIr}cK! zK>>q<0_PUkTo(~?YUDnwBp{XeWme}{iUbt;* z*aTMBfQ3>GOk(TrW?e8@lA`~&kLitrc9ud_Q+FK80bZ|zyhaMlk5mlbqkP-F?CK#Vc#tp(8s3-(3{PYK|ZnDbEjz(H9H zrC)lVcq@zf7H_op7{D5{gd^mE#YPc<&lkMCgRE8t`y5%qVe^1#vucF80<%UBS5HE| zT>@7Q1M4CN8;^s+E{b-=55c?1afE)HZ#yeg&Wz>?I)H0J={GllTa zt_*GITuujh%@UYQ8a!njs)QCB8#owE{V+jCnC(E==`wbU{hGc-HA51R0AeQ(kIAnxVV zxQkz0Wz5S97I3mHWU^sk={vx6N2>>md39^`-a zOgQ7A*ck_r0}kAL+Q$PJk_;AbyFKLVdcw5mKoW=QNw)JJUKE}35iM4k@-N+cYRF^X zEglP)7YXX$?ll@asKL&78<}&Rk&H0SlP}(!$Cd^A4oO>AR(i zG;=9Ymz+cqhDMQDj6%l>mL^@!6EBMStiXGI>dLas&z6KVF^gL? zhrOJlCl$FMPV#!(asBw?=UI1zG(Y~zXL=*f`%;|rgPxdA%r*?1#~cnjC<>dr5kB{T z!%TsBS%RQTqrfSJ0L2YFO*i=U4)WW4=P^(e-@(gMw?KgFA@_#`{2w0h7aic1Tgdlo z5z8HI%eVwaIpz4Y^sQ@7be%|%Jo&>z!$ER>c%p#w{EXv1cX+lt9gx;n@O5dF%=y>A z#L~c+vw+V*N8rx_VIN1KzYUyL3qA)h3rHMf?fd%m&^6Y3g1e#^*j^+^dOVzOZ$#pRMDMstAS7EV9ZI!*txHi-Y)!AcJ7za3|5}&(+qD| zSl?*w7gja=(ezf-xBI~~#s2uxhN6-MTK0@0PqYpjI37Nez--dM^=5$xPvY$t{$hU{ zX8(P_)Ab{wh>_{F1G`kBz@Y`RISvXu$`pCi&}Xwg zvs5hL{TJtd+92O3a>2O+W*pxX_zfBitC&p|GjQEFz`N#WRQnaaV*<-#Co85-Zd|y~ z^Lpv?l6TMQK4%%-a9uQ!IfTJ?s?6%Tq1B7XTSIFS_RmZaiV9h4j)y2~htnfcN# z)`|x=)@%~-ImpkF$YZlmWZ6P~u12;m7K}~`=8K$@^cX(>;9;|GX7y2Ut86fyAeQ@9 zfjNz#-|zujknul}O`?j67np{6Em!4@x$M9%!zgvYF;>}P#ZSd61@E45g}>zazQRyq z&vp^ZrNME>ql-E#XEbl)kXz4@p}_txfpf(Hfda)f$}F-XV%!diJSGpAuW8=ivG(7i zUHn}?4k%6#`=h|UWdZY*JUy<5{6`enOB8gT9pLj=C?fNfudd;PQ1!<%Dkc)viEA>_ zy9Ik=r+VzTZXmOs#lqpc)&u=Nt~;C#1TSn5JmknfL7I8S0ghh`vwt`6#N8HQX%tz| z!1U!~|DQwm4EQZy^{~vjuq5#SyN3dc#{ml=$1Kg)lk*z{XE=IvUSZpGdScY(DG#-k z^v+J5Tk`9V_7w-ED<$vbRQ^q0HTl-LnN};0PVt^BC9pH(&oZgzlOjweKbW3PWvy@E z=1}D6d9bf?q5LgH@nZ_yeGJ@22YDP8Fu!rw(6C29Yu3M04-P!7>d0TnNTQFj~N5l)x{M zD99oq@ah4-MgoI^e_HI-q#$ zn}O?21A`8OfKwuy5krtTqe7mF#Eb_mt?Viu78M1LSonEVbz%|@vmNHOD$lanpr8>r z*Ku)&Dfd-Ifvo{cW*9yaT*BUOVw-fxM_85jj7|5PMrCvVc{bG!Kkwc)FPVRFu}ANw zh7Bc}?aKr#Vk%ZOIfqKRro^1pYHsEci?!@XJ-Ey7oH+vlBPV!nKTmFAi{hQx)h zq|4l{CMqQK&t>5;i10{gpCZe(#LG~+^zfmB>@T*fZXhwDf$5_Oec8j4KANhK4TLUERbp75zKGkDB&ty&G4DW^6{51!U4ZO6i=~JP-_(l z4vSsT)Tt2`dNnxdNL75NXsN1QC*NSDO`Kkg`$&! z(iIG@3X2$&I27x699B^P_ zQViK}i)k(6b1CNIg#nJtf(923WNp8i_F}<-k2f0%L<=04*d=t1-kN*XlA)26=S%{# zhy+K%LAizm_QTAqFCDqKmh>obXihLlYG=)#b-;yF-@6}=g9E;ae0^y5OJeG} zz@H}^oTAfRL#GA%tjxH&SO1#CwBDq(5iT65jsY$qycHJ?@|QAvI>;g~(daqREv@LJ zgmlh_gTjq1K^&4TERWceyS_e*kWc^d@uY5B$D(G5x;q~)D%4thbSx=f@K3>kNtJ=g zndyx0uUV|?V-p&)ot+I{ELrnb?M|MnvqNLSs+C~}7jN*D`s&25!yv2K5KwSX{>Xs? z&Pui$0-Z!I-Ertq@aYieGPYG|U{dIr!`6{eJ>e6Fvd@=YEs6(k8FjEr?s%BRX`FDO zP1fwsMIOT?Hq5*#eL9akl%}f}bv&ElJc-BTX@Z5nzeC}Rg}YufNj!R`vf+ReN5H{z zB5!ma%y_@|+p&lvU1pCo#QIpw4>Dw0W7w|H(;~zn*(#BA zg?r=WYab>wGAv*aTB9Vb(!t1^6wn}eR)Al5!UA?j z6$P#{3a*t8(%H?PbP4)JGzrfN=!ouFBvyWaf$heFRgs|!*vm8;6&5&l1=u{6>{nnj zXjphemFcmd|DQYC4;c3G^maRc^EkEZTf+v0O;bfz zF3?RDxe>W+60?;~6Q|XIX5B@Oyn#9kdA1*5HD$RtSETKbK%z#A{J#TxkC~_$?No49 z=2_V6#PEna_{AZiT?XvdHjntLG+K2-F16mBz>vw~a4tJ+^QmiR()>NL4=@)zp0&y_ z!OXw%-13n_-uwfC96(M+7@lKqL2qPXDs5Tl2C`O4$0RJb^+^u!_traduyQFA{t z@HjLIHtgX2=O`!8@byEEf|S4kUMay2o)iX7jU{Y`5{9h@&*|og2{@c%Xk=uISist+ z+bFHzXdWi0EMs24Y$CzPnbp87wSaZ4@dCrB3vwG}tQFa_;c}1{RRA8fL$Re|fL-JcbFsd6gNuJR-x{B#x+jOo+f&mdz z)qf>$I2%0Mef#pflLd-b9A6{|v?hqj%~;xQdFCLe)DH)CyMRW{5C?V{h6Mfa$i~Q9 zL4g}RmtJ(~nzPvPZN`=qE@qj4*(-FH`|)HjDue_zGMp$?UUKV1to($=n6<(}8Xhdl zD-LovuF>UMc%ixSpI-oztpO9ON<*lNku;a%4n~2-7jJmpIQT2qnQdWgb#n;ZJz1-- zuSEJ0drq>pQhSj|Ps$vP17+9sR|iz~>Qp7a-If#RzSOm6?dOAHvwk+Jro72*OlZ+G z65D^lph={~Ah>+Hd-}V14~4dWXjYUmG}Z`Mz$Ed5f$?4nv-KTEfyD_f@)jSO^>#31 za`=caYfP|r+?p2Htd!lXFMEyk2OHD=nO+UkKR2+O9cPlA;mEE~Fq>5%S+nYZs4H5?3IvZ(2%>(jgWOFq3jcSv9R|F=?=hFFeI zzD%0yQ?)EsFf0FSU{o_$B=S(uMgGuXQ?)0nM3!!dlX$deuS127KNDw@75NOgN$tH_O2Kj3d7uNA|p|m!XSq%3L-#k7E7MlF1em8kZc|b;hvN zi|xZF&W0P!JPaEctQWG|mWv*Al9EoH z!Dt*H)a9~7vaUzrn5Ua!ZvoGhTP|`d?(OA$SCGG=#Oq$Wt<4lye#v`xObaHy&Iviz zxOKMXp}i~io?Vn`VW--&)%ta0x57Ugo<9dB_qa0h@XejcAJE8?7~#a4x1ls>f|7u@ z!sJhf99Y$T8o7cCm_&OH{MMcFnZGQfNh4<=Z*ET#lk5*?We>)1v#$yqsvmZqnLgR! zs>1SV9j`3s{Y%euRMfh%z;5-@PhU!&S@iPtMzn;+_BPDrWohE;*ne?;XW|-rV?mb^ zVFulXrVPVIro0Ut<`X#LrfVRf3?tp5@pu=y>h-h0X1N1(N|fxY~Ilg|hCHUo}& z1~$5OV%ZuaDOOq zWACWH#AF!2@^f)%E(1r^2PVD-27UpidWjD9gU0F!;UW__t{mX;NRaEB%3yY(R9}pB zX##suI+I3%wyFq6(}iZA3(SS$d`}p_32xxid^?di_mglDfgx;tp;1<3|yj`9V}at_Rv6BuO^ zf}0-{8C{5&e5ft#v-+xunRd_HxXmU$E3XYy4%)cUyYWSQ)d?w!1b4##<{gtbx;L;p zKj>gCiDC;T zi!8{NJm78m!7oOfRZ4*4{DaahK}@O(7|S-4_J)Oj)%RGTS2SekN;h2e8Op zU|?)u5Itbb-@xG3&M<8f14jYly_a+KGUaj)aO7NIyX3$nc7V-s0sEy5944MT8VZqD z!kE|+8Pz|qFIC`eJiu}!fnB3uo{uKq#url_D@=Wy(AXn2|MN@X$Dhi-zl_?O!CxA{ z9_PRwvtVV60_Qxg1$G7tEFLf|iCS>#1{2@b9=@xoDJBLXQVRtPSc(}qycbMgFoFBU z1-`crc#}62%DiG}72s`lU{zIErdz;l7Qo1&z|dbgIehDyDVFSMOvYat8Bc352qzSn z8!-C|)Gp)9>@u3T`X@tk(UMAyb#B`=);TkYD==?R^!C5yy`%9TvqJ;B;e^?xrUE<( z42LE%_;27?GJ!*ag;|w>Av%jmdJE5y3v4zHJe>^8KiqjQEMUJJ$t1BwQ00Knk^reD zQ7+p8CNECDS2w3TyfE*r<<#z|##(d!M@%dAxL5vl;CCpT`t3t`O#pkzgq4=Rr^dfr z`CoFH!vkg~=~auWdQ2P^=p`)ZdTy1zQ6%)2o3#Q<{RCF`0JdERxZhsjf7`$vn!uWN zpinv?ZCe0W^8t=30am{Y>Ctq>G~TC?o}r4R^De0&f4%%W0$Dui;wLB4eH_ojLZrw)f;&H9ZFySYh;-GqWHx@ zE!zU-90#uM4YC_PF&H%Hsx`>!GB67SFxhTkPgp)#>;SX+1OawNW(@~MDFN1|f;P)w z?n@I^9P3_j?DstJ8GN5C`QIq;_y5{7!Ha)l7Jo&-=8}e$)~7em{Wag7d&>f^EfxZ+ z9#!|~2{3!k-lBejN%jDv>;o$o29|08w$euK7YnvJrm{seur5eoQ#ioVEfAY1z_B>M z#pf$?_sj!bJEeA>WY95SZ++0({DDQhfzjW9-MoOk@c^570F!;dqTK9R-j|DoXSV0O z$mBNDhGud1~B+t5pr4ck>Tz` zHjx76B^%h88B3Rn)pLGuU@KsX&JvImW8x8DVm&C3Cm^@ff#e2D zlqdc9H>GP9pGI_J@JqgLZ}@juSpDGH`>BEbUyR?4*_&r+ah51>#u%`dz1dqOv!#+} zf#d1pCJUI4OR(1%a3mzGo-u)QRs(DGLhfA~xc^zRHaPGs-N3S)k@4FP76}0cRV#+= z4~x}#CzTekmwaF`Wys1;5)2JJsB^)K_hy@r+O_GB zeZjnv0A_~S4HE*GSqd3B4R$e2VC>i`yVUI{zexSkuxn|)M>AxZLhdlp6PngVZ`F7MT8*Q?$(@M|=ln7!%o2flAL-4Cvn=e;>^-o4qv`Z#Os1vXtyrw=Rd z-@Y*A>;=0I7aspR&02kdqwWJ^oV20TTu%A5+>MszTB(=*Zj%#vpx)(~slS(jw?|#p zfw@74*))Lh$PtDB1`hR^6U=QVa*7;s^jgwW9h&KSWpl{s1!{qMXVj;L&0c+yQ7<5x z`vY^20hiJio*aizP639_uWB04*h*W^-s7oH{M+#~yiULK8n0pEWd+{ZsX65i{%-phAzT2I-4+?S_{rFa@cE59I{JMud?+kl>413K3R*Tguh0g7r@OHDq zf(sIL55?_n^2kp6vg&641`hcRjA0F&(-TxSFmTJ&aw;%HbSjkl1u*ze_L|J*aADVh zzL~S<%vog4$!Olm*f^o?-@2s3&WEM$zCE>kW~PSCop$L?_uL)-TH7~Hl-7Q8GGnU- z&w-3;1{O1h4O`h_mk2OV_GHYtP zM`a4EfdU+tE|}Ika9usXbCw~tt?tEzr#$;OdAsGJoXTA8f8e?Qf%oYLUPilsg~pBS z>mF1T+~9flKycp5IEUjR_gFaRKD;;kVTI0?wms&?3Nu0+RNR78o=QjrIh;Eqz$O`x zQJlbRe1N6=0;^RDbLjyV@d9JrS&OXCF_gN>TF)x!Ehx3S7rOGv1ofVTOX8i=u01J> zIP~6YT|`}`T<0N&4;&9KaL7*J2n_hgbmt3)=*CO@3C!lt*zy*z`!FyVW?oC4%aH$% zf&Igz;<&ny97g>*W~l}ylbIZ<3G7=x@NB){FChQ%&lG==`~K$~c*Oamj(p&i`NDhD z;HAR(+@~Md=e&D)UHbS;uL~mUSe+KIIz0FkrNC*p|Kaj~o6B`LofkZ;xWLhT;ke%i z)}{ySa}^|06IdLTSQ`SkW@>Qw9Vp%Az*YKz{SV7mVKXlO1jhc84D-X;eLtl8FJL#l zz$m$VKI^{HX$4MJ+9<#cM9bw-4mfU8aZCB1{r@#_%mZdI~$)|xu z=EAL;3)zDl7_YW4y+88o0-SUDZWQnw|Fd&*^Z{1G^{fdu zS042LY$AV?bL~TC1r~?fGkYJfepcqalEA&_0q2ea?rj&?ue$PHIpE%U;Y*JKYi$90 zwE#z10GHk|Q!|5^D-zi!T+k~2z`RmP>Q7@qqXMg*I!kY>`ud4)cr$k}Tx>XafQ?(q z!s5gNg%(zR9=!&G4+jsl@hWM>7zi9{X5&|q+H+u$D>s9=5a%Z6Rg#3HHUrT|&m4oLO*w%I#WSJb~RBK>*$tmLckGa)qaZYd67mwFr>tZ}} zWm$8y4LlBEOb83Evr}iw`k%LRRaNCCzfnRCC`nI z4tL8kH@;!+;4^5>GE55bnx&g{#bWxbV?83KY1lLHSs%b_Xij8wygkCLYcDcamxJ1t4A)~O@ zhUSS+CUqZTvUW@4-0UiGv%uLyqEW>~&|M|aWz!iZwY6KOD7wwG{in3>HJ{yw4cz^E zc3kLQzmG|HQRk6a6I!?rb4+gWIL;KUv++oYM^B&QjEQbiSxb7ICXpyvLS#Y39 z`W;KlM3;`rCKg6vhEwVboFoiZF#BE5Tsco@L$ah}$IP}FNfM0(B3HgKxL7PJ1W#~b+Ste$#5A*!kx`6eK|t%G=1!*h#oaF!z1-34Vcnp0A3a?zf# zVtL;x?O&HV#R?b}wJA;5@`CkS{t877t_TOghD!?E97ov9G#Xo6j2RgjnnD^hL|Ba+ z8f7jdaF)$rWSM5PhDGOSGrPbAW`TmH-JJ(oIISG{7Z~>D2snFno?wucy4d4-L6N_5 z<3irA3)t=66clt=ACcoTJaWvIuV~8VIR-KlcZwg>j;s3UB*QXoSDmDe(W+qC{Xdpk z$&0*6aOh~{i!or9b?J~y=W%IS@v)WHK#|9MM#FiXh9)7A^GqTa99BfMHnOQGn2RSI z&QcQj$7<{099o`mEYq#9kwu|_k&T0a`;KUf+y@1&z!{AUYzCVDbQ5a6WOMLX>||gm z*)&<;gQLHV=00|ZA_i`W21YrD7C8q62Kx=pJXH)GH>NNN)F!xT&Y0LCIfqB!rNc6d z83J<|R2W#r7aZV~xx&oda~5=N0HaO;L*6!CN1j)XN)rN@L>4IIh_5=TQ8%GYW649# z+=}$LTMFl;pDFTN?@8ycws5KYW-Qp%aOdeGr^UPPaCo-tT*P0~z}#jwrOaYNkLbo9 z`ubIY3;!-+umTCIaY?;@CZI2f3X6Ap=6IW%y+;b-<~ zI3hXop!~m!Hq8=eBRGv`ZS1jTQD9(CXt?rX1H0Lah3sVvM`Snz>{vpst_t&I&~oKy z2nxH(5YM`rxoyG0ReA>)dSwnUs4y@DeQIDA$Z!yTC(;^prAg$FPOEO2*DT9`1&k#^ zt|qG-nVC2qFq+R$3|PAGM!H8cFK5CN&LhVbEUwtVV!okMjRg*Er!!o5wN5t6 z{YcJ)CF#M%`XCe7e<@gZh6iF*-L(H%mNNu!Bc8 zuSjy&g{Mv;99+E`CE{9=ZAn|0%?ch!+P`YCJfkRLb;5yd!A+(wkqpe78yHM%6*>Y- z9Ba?7WZ?Js$P@I>R!O@*fu&ASkZYgdV?jj)RVF5e2F@Mc{I9)^$XcCXU?|wSiou{E zT+x7qU4h{c$B$Lv%pXEIo-AbJV{l+UeX4;;B!NLdAX}i|ScBD@hwRc?M>W_4y8>96 zs#kSA7W!wv;V+WJCQ-q_TIIlFS(qc(JfYWwVPlJ=OaYsX$I6tu&lb$F>XE#n;3hr8 z;IPAi11yFOjyz=w&tyL|aO9~rWrohaWx}G!yRNlZ`jbL>^{rt3jtk88>>t}DFCN_9 z)W}m5Ymgziqk*|?8Y8<fsAJm;C#vF%}29`+fGY$nId9dbAXS{stHB@Q$? z1~SZIG0^0Fb)myr7v0#CHWVeW*3p^dtX>#L@+Rxe=YoK<^`t-^sxG{eDI+<}R$q=8Z0w}EX(IWyyt zKnBSKr(WX)jA}fG_*)c?i03)nQe=L_#ALCO!HPkBD(j&MCzxHl&VM(1Goyj`_?>3y z83)-U&6ouYEDmy;oNE$4!N8{9z{qoK5A&qOH_zBgFwMTa;gi{lg}nOHjpyd>URVCn z(S3rX9&gNr7fJ;Sp1U$sT?oBU%=cr@iw?Ul3t}B&o}OZ0pWr4TUvz;<%H${a>5Yxj zZw%VyQx-7AuXkXNabV^bX!ytG`=Ws_Z$YztLnE8)frG+&0nHJP2iYtcoP-M;nx&p3 zu$k?N+Q{I88Ztewkn=g$##%wNY=5TR3+;MoF zaKe%Gg$S3r#~jIk1{Vd7hJ8yS60RLt;I^hCQP|BRSx}r~=P`4=x(UrTXRIp@o;dv` zZ2P(}-pqi*zhmDO&ER(`FuBAn&XgM?X>yBIN26&W^RCYg4N4geN*xUgxUCldwp=io zk$*xXN6!KOPHVdj&FVXt9ac2oJpHf1YEuKxfd@Ic&R~0(*rv9C*<%B< zoxHT&0T-AZCa`65wC5x=SuJRD5?~41VPwnE z>bhf@02kw%4Ceko-XsPV;RGf(23;WoMxh^0Vg`(T+V&Hi?Y{``3|HA1ai>w_L8FLA zqnrn$sYjz(MDq{tX65YZO^3wW7B#y6d(0rl;K1l)C~{by>EL{hhGQ8%2N*&QWa%(1 zf81pGqKWq>llq2cgMuc#f~G>AW`i3|M=mq!KbT;?r^`NqNnL^2=>S)B0=wIT1`fuS zHpv#Q4-9-CnxZ<|&L%KDUeLH?5p&^&R<{+b(FaU~EciEnQrXP3nq#uZ1|x0G6MHgG z?_sz-pXaco#FeIcod%8xQnEi7WGmdicTM1Nn3sE^P4fhU^lXM%MlD7RM^qCO7-uj( z5NOFrXe;n&xBAhP=D}y3z~tC4kIR!mh(mx~fRSZ|GN;i#4UdL^BVEELEIvu^5B;-4 zmSg9H&!^U@SWeX59Bp!1?nR^Qi$>*cma>h4|MUf#{wkZKq%;aB97t_x;K*RexHI4N zqjyFL!=x$4GFf)9h;+(VH2HHk+nrdg)yWo9aVD#yE%ySOzQhT~1}61{7PlA7$^}gN z1}#wm9+@|qxmPqh{a|)l(dwwtz_y~nQlQnCt5wyYPv(XSUk8hU0JGtTrq8$JIXjmA z3S{Uwd zw4wFvk0zdusb#Ah94(lgKd=~TFqj`axTAt4J+V!!fhmoH#p0EN^o~ZSmkU?~8X0EV zasO!KpJ3r&(fD{Jt3ySD)Q>|Prmml|kFN9C|E2k#>!vA3MLLc~tL#+XZ7;pBS?MR^ zldeXw35;R^jZ8d@8D~x(h+)`oqWVv=;o+Op91|L3j$TYpY1qtsNL7J3`ar9q>Ltas zZMh!o>KhoXB3R6JG+O;wDihJ9`-3@9qs2plMd1X4##ZK1mX=Mz3{pFo0~J_oel&#s zW@L(JP?KPd$zTiQXqG(D%f@W>YvLXqlZMpO20Xu4F~11jY{js7)qIY}oO~A=9e1=w z?O^@!q(SUOgZ&Ctr;HXi24?<-Ci@c&su4_@FPilaFi0U~NsC?ibS?P~UOIG;Td$(2`D3Hh50R8FJ4J2#(`zoe&b;W?)$ni%1M7{8 zrFZ8yzF;s{YEgZ`9P@!K<^gMsZcx5QyS2t-myRZOi`}XV_Nu;EEAxPXU!ub*qD8TU zNj9TVZNiOCS$Btu7WIS%^@b3q0)NSr7LNyv>;Vm(CJgKvHz%JpVEu5J%lam3M5Nvl zCuR;t#RJS21e-l3wCFBiw!XonFTnhYh1rqA&YEK?XTjFgQwN@@U{mh?l+46!JqlCa@sUI7~7^ZBHG+;W>WT3%p&(X}2pv_zn z$GKv|Ygxa%M~`2Az5H@%Ttc^W!tFR7?ymJr4O@)k#W@(-A{f40ZS;;~;#$FI{i1r!$@gMAaOXDm(qB`*iE7 z#-pi>^Q-T0obcZJy$su zRu?*)0)t&1&R=qKK7&95i^k*56TGc?Y77#7GZ=ylJ?`=EkbHjjvfH*e=QO|mXOAa6 zRdbAuocz^;ZNaWjR~h|3EtZU6`c|9iyRTXDMFZD@2II91whC-w59UwL=*Y2R5W2za zHlx`op(U`QMK7VrSj)m}Mk5Dj6CYP2!^{rb2o`gTXorLc5(NuV8yNb6pNVHQhWt78 zX>-88?0wM_jrV7W?#$u4m4(ne^8nKO?fi! zZ3YDf*$*sE5xE8xeU5ea*3XKYe)rJCt#6&oQl^{U?CObgabr*rm}@Ptly6s~d_hzB zpC+LjPb7adS?^$0DQ-*>RLwc0Hm_s$XA2_-0ro-$ucQPfrQ0`MI9P)e*)pp*7ZwPd znKAQP=sSfRHbI5`oS|V8qMv2lV-$6;&-~`a(|av)&Cvs2Pf4-O-Oj>u=+xc$zYPDWO7}iDs#BF*(CoIOEt;X#|3?e&^8&v61yMg*wDQ{QJZx+}G;jrU zgs3(!Dx7SJNQ$^6&ydg%wCKYJM%6iFKdgvMoUX^qx}j-4Tntu5_^xXYb;+oe+OrD{|;u2 zl(yD6ZKf}0^Ek|xy3p*tW5A@NO z;)CCZ54`VN{8xO9>Q8e1$KffZn0h0$gYCkSznUDy*Y-$FFFY6L+*WW-gS~%HP<@&0#2>!EF6O!g@zT>>_3@ zj%JBHwk}+$w(1kwDjIAvI`ua++iUF8P+(1~>|{w`SZ2btK!8iwfbri^2B8hiwlf-? zBHB^|SX?@qEgI~V9{iAP*dorbc|tei7QNSdzZz|ub~HEbv|UHby<6OWttHx`tmO2p zQgZ^+S}q2?KizB>{%!)J-3HcZ1J-~W1tA8kG0zKP%vobHT0DNVyY6U~*}m|U=?b=V1Mb9t7Uu`eHYa|VJos^&{losL zNB@00b)eRD;_9OlvRRx@lw8o%sB-L?p}y`v^YvXK3M?#1#bpJOrs1WnAK3F5tqfX& zHhlGWXyrftHTpkGu)rm^6>V+;ENTr*L5i(`4NS8>Nj5ZZU2=-!$~>)IznjH4-ac1g zwCec9z|g@}&Tw<*g(Ut3>I`B!0S^u={MW?HCK3>maG-&uor6cm;K72$&CCM)Ha!mx zG_bO9XSrksH@h&gaaQRh9pMn#7}zbNr|gf&)!JiN;x$_wRp)3fkifjkCp@)x15NTv#a6xc(9p+ zU(UMXfWn<*cYcZHM4l*JM2IKBePh5)J&_doLANT=UTD8dh+t}a=-aLce~6^M6I<>T*9)6nN?tc zw0VxfNe2gKVKs9OgM>#%SQ(}JCH@KsC^#~3|0|Qbnj)0Ir|D+4z#+A6oFGG!b;FgVN2b-Cy)Xyqcr{BPnWk9tRo4-USJHWM7e#R6PK88~ek9GKi?H#7ul zsRk&8GI1CrF>%-&IMBwoWQGGrxaEZ?CiQ@?&PqpKTs)+(#3uNh^?i-YDVC>Pc7{cF z?P>LpDY-E9m_Q8QN7XpzFA8e>wli)zC$#bid8SnCc*!aoW5ala-#$monMbc>Te_gY z1U5&O6@N8u=d3=-Chf9*Ws9Ro&x}tQMfpA-4+^>mD0-+bt2h*6+UAn%Egu}C-l1^A zH}SZ}QiDe<3TLKV>JSdFaq?2WHu2CISz`_bVeU?be?f<2d<+blef92qaFIUqMc}fF zV(gJe?m8C^vdh{`Fld*nVqrYsEWr08fW!45gRivtyay-bl+>KK_{Ab7oKOx8OjYjZ zpWqVWp>yu^0cO=xTMmTsJebkY$I-TAs=IB_Q3nnc8wbY72NOQ?9Tg2elVTZYHPa`$ zPo{`NzUIloR2io&6Watb3nW>l$MJoPke|52GjdK$f#dVJ6{j9Bu`g;pa455uX_eFh z?k#4w7jANWGV7waL7<1`vW82Xk|7F>Exhs>1uhbG65(CuttQE*72-VvxeS-IC~=7N zObGJQtlg63tvVOB*f~W`L^#PV`S-%Hk+nhRuyZF{hlrv( z<0Z#4N{bynHOqGEY-V?3a|sA!nLOV`hWmb_^v}fs??#^ z?d9yyPR9SI4Z^}d%H?se%UpfTtZs4U1G6|okeJQ;kodhvZ|+xKY9y_$+!HO|e#U)T zT1vdQ`KVyXxIZCl1#?Go@H~cRah5HG$3HX|0Q@z>h|SMb9{m zd>T1JW-zj)HMH<6By!E(y##2CvosFFfwmR6g1f2ApPP4lb|hwuZRc3W1o~p<~Mwr(uc0Q_ccwG+Q{Kq~0o=}do0jUO zrr50eenL|-d6GFg78dz+b;>S|tH)^ZlmaQOSeU2WpitPc?) zy-o|*$^=?9c^p}J6Amzmeo$r0a5!jCu}I{0MvLw&MK0?b4Ay2Fn3Y*BvWm1l;`Up? zYC7#=`?jl1lFd6D6{ZDp1*ND9R$FkWcsyk0dXs2tS8+&5Po(9ZXNw9q$A#)Q4n1KT z6&b}Mjwo%55Md1oRFSOhU}X5vk#?hjfpf({IlT`Jwg!ipbQzAw%`%L9I=7)$;-4#n zb4|HO|AX#?qBEx$t3=jK$a}${r13J;Nq~vlpiJb!40oFsA1}St6OuEuU1F`%W~g3! zSaQmdl}g_pTt3ARD&%zH4F9(x^^i|J%vw7fF8n#Nf1xOEgS_EMMxRFvQ*9L3N)N1( zx7*Oj9n!!e{vw`TV@Hc`=wq=(8(0;OTxjHoOR&~G(7>D1(8!hI!0uOYsKO{gjwgXZ zpylX7xh<8=-wj&ka|;~E4cRmyEiy%!qk^Ad(E%PIi;rweHZa%72n1XTN^xD$%D^zK zu$|#ggJ5?98yk}VYvzM6sY?v)aw|5u-TU}XSx1Sh+pl_kzf@1Tn z6Yww7@|k15?e)dy>;4tbe;nBVwn|*?p{Tyf$<;jmO^S79r+fI{v@}b-II}|L#mxC( zm9se3e44thM2jJGu~6ed2Cjtyu1&@l7&!_aFvNtz|;sbo0I}YeFevo6*Nnl`9aA06#xWm!Hr+HxQ#cDl< zPHvkioCXXo+)NBH3?2z3cdZ^dZ?kBKy!ue^(gAh0gd+Bg2~1q73Ctyx>H4SV-PEv3 znmGCHkBN_3O?IB@HExm%xFoF5QCS_avW%VgMQ?G? zihb~5{q{Hb-+ZQ{v%>%RPy5GPz#{R5;jYCgezzM3*6A`ZvV|XD_E^xQm~ofKDP;qn zY(Xij)`EbQ@e0h$0UE3wvCeEKIG77sn`EU7n}42RV6b0T#=wy9n6V&*f#JakmrF7Y ztN{<0L5Ge7$H6u{G@tRpLlk%G5$~H_Y5s?B6;)hDRnHd;Z&d5GK$-&?t9ooZWyu^3a z6b6P4kD0QrJ`rJo54iVKhP;ey<(U^U+cW0#gn1l3(`BzXSsuAJ|E}}fhs^?fih_^g z=D)r9u2$UUjL#n)d4Ut*nlpnk3L1F6_?%zC5MFO zJ;J!+u8aw@^p(53Zw~TA9AI^5mY&kU>T=Lng{f=7OlisRgBFp8md-rzL-4afzx>Ha zqpky-JG?)CIl*brAgj@7^CQ9b4zt-kX2HD2X4m58U%YqhfaM z@RGH057lUborw#|F0i&W#m--Y(p&SMa-vcZ>!J;-@no|y$ zW;n`PEaAH%+ic3f>T>r=Ns=hI`LLayJ+LUu;K_)34_1|M~#~YPi$!LN{QYk$iU#jz`)bM&Ev;=$ASM$ zBkv9d-ZPAHFC6$R7;-A}R-%4m8NUVa(Xd=wooC zOU9uvvGuv*>1UEWO*1_DPADAg5P50KV0`bvD^?*{8zz$%lJk!)HOo@Ad(zzU%(>-~ zYHZcx`6ti(sBo0wna}@*A^i=HXp~~c9skZIH|{q+UGpYoBq)lBH70Kp%8FoM(`i)j z=+*q=R-n;d^g<(J7L%391^zn+gwbXRJG99V!(VcobOPFgo2hBIA<8o8hSJ;cSuN%x=P@e4-(+<49M_@>{Nr z(mGBeQyMnh;BY?k*pop#P~_FYEec^`GCW#gP-q4yHh4CO~Nl4WdAsDE>Yuk2;eOgWi>b`Z@{F$a!BFJ zL9vR1;s?5Tw`dxh98$<|{G=++*22d5hDRW$Q7ng1;>sai4=10v2H7_U*P0%d6>w0< zVl*ybw(eH!@MkRaSYeO7PpM$ubP+FWv!5NQgQj`bRy%VfCrOc!wbF{ zmsW8tOW4M+MksXcFNZV+N8tnF0xu3+e$>Esgn_ppllu-oXGsI^gsDzVO_~c1ZIsi# zQMJ}Qz*(N5QMQD!`)q^4k^m(K$J;R-(pz$kmmD?=VK%?hBrd|V$L{cfWz7~mPOG=x zVm+`}zxCMFM+c2FoXr!Mt$PkJ{Na%Ob8yeE(0nrozB^GCU!3{=98^AX;DIbh;F_d{ z8FxEwoN(Q;fK@E;d7<3%+ItJ7TsdZ(h%m~Em^E+CM-R_R(S7T#o!aGGc5>e7Jmuxr zbk1JNIeY1;(u156XIfQ`FbY06z_sGlD!#X?lyZ4A){12?ir2grD{$QNX{p#h9o;LJ z8u)zHatFNK^2$Nt%&cV%&o;c$C~8sDGjT3^)F@lxz_vwHlf_Aa+I0WR(_EYg`*wo5uR9Eg3&BJR8)y;du0pT?a-rBnLv-y>A1(suIY%S8-CPsQc-aFe2!J{&ztkd4)ASw(K+u@+Jcz0 zf=2Nzg|ZcnvJH$HTQ`blG>RX1{jE=b?iEkICk&z`XO~qp6+Bs?so}1u&;HYsS>gr5 zKRKQaYz7BS65UHz^!^QGms>DZ_X^XVe~n^04#=7~ofB&kcR9d1*(i&{hv$N0#Q(vPXUA$&`jhi3UgM{KoH_K8m+6iV93t z_~EGVqfuc3_n?9QK!(qw1F|z-%FlQy`+`ya%0Yz#>H6;( z=gw<+5Ic=e`KXJ|8rPQPE-|G>&fN>6bPjPEhTXXMCXeG_Lu%A%g;oRplamchk3uZ!0n#7sQWL7jv zwS46A$iFbnaNZkMfdYM*^FHUo%u>D0Qq2!_2Ag#!m#O?Y82tZWT7DyY#sPMNgJKsB z3jc6;YUn8Zq@m(WqwI!*ycaYq&NhfJ9Flc#l)p3o7DJPXy&Z=$R#yJ?pydZwz$oI@m4HG9j-a`xp3|U7S{O%u{ zq&Sh0hqXX=&0MP^3zS%$1+;3~w;ntwz31nnXR)krl{UW9SiD$n1?vO`-CqumA`crk z{F$z}!{RXW=|@|aneABq%`|>@@$&Ltt?mDQefg)u*Pzj%|E=S(W9Rc&`GVD2aVPl-1~v}sf@pT-TP~$%97K3_OJAAr%k^%Sk;D48x9*-4 zW%FTROlaV{&>*{Es_X>j2b-LYfBe?HTTF>n^t zaK<=r-e6D`xwN;j=7jtY3y)p%0;?>Ie~5a&uv=o>a@uiuebtKYDhYKn0iO8e-JWUQ z-;#IiQoPcnafj)%`To!L_7{HJXdnKi!0?;>hQp-kT&4dma4BD5t{c?~ur}oN3 ziRP=ZSFXfDB8W#tDr&U2!q3e%xi0|?GNm? z;E@P8(7e#CZIP*#z(PhAF>x`2gocDhW_C874FW<-PRq!b<-Az=G2vhni;QeROhZHR z*_qsY7B5~Xs5i~97p^$+QsG52JA03kY1Wm4&C9&y4J-tkT&@a7Y zjeQLdnOg7dPQP(6d3pa_eQzt1o1c^U7TZ>^@iH+jR65Egs_1azGEa=|i3{uxCj=~L zm}Z=C~=1r1qYb^acM}r z?X#bgpwzCKv4OGQHY1~fncsqesb4~3K}X|jg$?{n@^Xyola#s`7IN^jO<2Go!*+0u zr1JrWL);2l1&RXtX*&}7tgAc}7#RL(u%6f3wIR9BWkzJn0S2)jTN|fYP4aY{-_*3> z(897+0ny9nMQJgvkmE>n;g+*ls8y`1!N@Elp?LJD*zT_Hk{b`K-0miQ`;g$e>PNG< zPbhr+^m@JN6NL%`QIUyX)hAmq-St0W#O$@cR-0@QD}f8mv+mAsQ#u)+cXPjp4#BNSk2t)h_KU|Q(Am^ z8+hWxUap;fZ-dMHLYbl~P44q5?^R_V-}!DM_x$~RYyO_FDEO|#*CyY6ZlRXyvWPQ1 z=6ZFH^?J?go;+T>Gj#%|(SxAD~cc)zbE;eByV<`871q|XP69ifd7}#YGH2%49xRJ}|AZMwD3)=#R z9;*}X3}p(g|CA3MdJzQ=B8Ia*RbfE;A6Hp(2;J^EZM)}FayVhhVKp!7zH92SQZ^) zl+9Sk-?GBhM9EQZg`hx#p2B$+tqYBO4;ndzXEZ8^1v14)P7r+5(Pnxmu!DO_lSl&t zi}H$z9Bw9$*>y5Gx&OtVU|@1oV6^1);z)TR)N7)utXSc{qbas-O#am~+y?U%g z4_mZ5JQ^BTh2*n!OSqaG`d4`5h@RLCPab!(Cz`xfc8lj3FA}s_cEYOms-8iizzPRN z;hFaXpBZIenQ?3qd*Ib|)3&Z{U14sf#>H=9+An=|ah}<%+3zy1q;Fm5AYVJXk*y7V)W`U|^o{p+mi66W`0l44Pdx&4V^{h=^(E`4@d^P`JY=(0$>s(zk*Rmj&d_$IKIA z_#|%^%gktHkAA|yZdvhd^TOUE&TCZuadpk~Tp?i`=Q8omt=t4>>D6!DvKI$(%cd+? zDSG1OyfrzqZ})te(DY=c$Ip<)+W`%Xq6|xBNIEq56eaRUJ#ebf7ifHw;=Y_a!$GjY zpex;kX-}w>3oC;`gP2Vg|HHKhg{GAq=ksCUDH8~pX&t~EbY>yb%p*%!Jp{Sb7C10S z7Bov6DKKOnn!w1Oz`&`($(v@=EOF!kzsWyi&x{F8ocEI&SS?;UD|B5hs*#=8ZhePQ zh+&c7fh%qn6Xw1tXmFO+j5B-r%2n)?*Ei`)6>KK+KJrM^Kc4rlfc;&?nf41_%Z%7G zb<=VTCuH3?v`{qkli35=_p14erM>GmWzAmR{BK!O!>O=u+ual!q>V4LrSJONo3E4b zjLTqwc}kkY`6{;to`MDjSq3KAE{C~uo_&d2z~WR~SfI7UZ9%;ZD-&Od!hW$Q3EU7|mUeRN^$B*IrLDph`i+58jUzYRUQ$TG;y8lmd$ptPgxX>9Z;C2%~#lx{Pu+1w~dD{zB%G1E%SL( zb^HdJ0=~__@f@qQN%mv$hpH_ z3nm&U2zKSZy^_Z`RrlxuUYR9p-)+ieTK`sZ&cgS47bN#Be1C+)=+p^Ct>&aW#k&*O zw+5Zw`iNIrE1&h{TGp4m(mMKmvl#h*EjMF0YW72SJ6qlZldK75TGec>XN35enm#Zs zNXUJYz!}BB#jubq=>T6t19R5-U?l}*u?I{M3mDlHm|iqLz?{** zWc7fPA(8FTdcj!>7*46NM;+Ma#3+!m&QxL{vrwbNw1pz?6h!77gNqfC1`T8!u|4kYGUJVSM2M*3lWRwhe zJB9b=fkY{V4e!?E#(m0THWy(2`;KYhLnh8wKj z)@)$p@nA1tV6StKo82HXqfzpZfWVCfg3k`{MfI~~9N?BZ$lR5%(#m0b+5$m7SE+xC z5@jweWIE(8@P$Fh;~?LJ2Yi3JHR3WD-BS3)7zN%<6ck&)#NxpI?ts{lM7cQ!xTiJp z8U+5g>Wt}_z;W1BEq9XEv866{JuF@eSY_5o1_+AY4 zy>ya|a}=fgx+}Wex2}44FGQPd@8JY4xFobz}fPEecl4zMJ#)`*0e}Gl$&`_ zY{o(nuS7|Yg+`JO*^L%(N;$HhN)~vrfLThRii=U;*8;H|#+z0Nrq2`@^BAfN{FP%? zXlgz9$7g56BbjK&!m!HCQEFWx=dK3HX9??nte@N?%wV~arKtD+sztAl>BTZ}Gk;qk z*kGh_Q}5=Sx1z}{Oq+GZ=RUMPC~ULPQF5O!i)OM&o6kp?R|_AUIGIrJX~ISAos62VWI~#eK8W`s=F>df|Gx^lIrCDUg1i^m{%+Hvaw>;qIaunIuz&WFl_lsBC zuk#k2#_CHSUHALf;P2&9Y;iz&cB0IYgFekdGDj4TraU=%$K?I5bJlYcB@cbJo_tVp zlA@$Snuwa4@vO$XJ|DDN3$^bXOYU4~$8V~`Um7SLD)S&%GI@j4y!LH*8-g5P^?R(E zP^WyRCE#~Q!08-?zy%3{cO01W5=_4=U}8&HC-E*YErF3G;o+$>&*rV-%u8ryN|elL z6tsK5#`I85`U}&m1i>%8Y-|hpL^P&xC5lKK{e zSiqN)@HZoYt>gh)$pOxs1c3+7qIDQt(u&R%1-vH4 zus1Ytzk86I^^m!v*SDilO5$PoOh$oafxc<4SBA8mS)||X0`W=FD3uE0BMRX=h*1TI~b<)Gy(3*w)`mNIc_r5ZDEO?`Lz-;D2pWwoq z({yjWT6rtY(C?{;&BTL}6BrX486{*E`nPjc-g$B2&g7)spOTIi6-$~XA2yPlzEOJB zL-`DbX)6S`bC{TOX$EjCE%n!`7Sa1xeJW>h8Uxoq2mYjfraXrIG3!~cz2>WH;AKk` zlw%Z_cYrmffvxSl*p`KoPHO}+4zNBuAotZ-z|N;J#(|ry@mIzHPL3W1jRlVw4sxp< zc&Nv~xJZdbrd^}x(5#dfY&i{_^A50OC9qX3NL}!Nb4oc|MS|1xMNA$Gg17js&+Fy* zox;$#*2V4OVZI#6D|s?!k_vubop&eX_NupHucs7>oBKB_N;EtauRAE%vM{MGuuo-M zfU~*e)`tP%Gm3S~q=ifab@^rW^!t{q`TQ+ciapc+7&}MwcYmRGW=DF?obgevX`7&P z!nbEx1>-~hE9_B{j9gjG%tZ?XPAw4(D;6yJ=aB!FLC7spMqgoy)dIeE$M$rc|2pG= zg7`zeH)|pdnzVlSE3>n-NgY=eN?^OCD)?>z%MT;2rUx@}92icVY43YI`QGBJg4}ey z0}ZD$UsoJs5wY0t<3aAXt3IsdO$Qg=Vwm=RpJMa=$u<)er%0PiI{Xk9aTHv3=|tCZ z+pR)uv64YgCkGyV7$AE!Dcv$TRidPoi^FHTnO&A{^@fLTT;|8pw>MRs7JIk-&uag= z)4UQ3eLX~T3-!1zJy*^&GtXe)S@nc3cCuU{=AJ>@* z%LeWm1?zPiG%Yffxo7s-F52Pc5W#Yh_mi_=ce!Mm1H(U|2Bv+jEE9x18n5i%aIWF5 zGUGn=*gW3qZ`=Gu3MFrxi*sq;q$c2JZCN;1QNlz*a*E@Wi3_L5KAa+&xG-#|)SiX5 z%CGLdF)o%4EzXooF1D21_)zRp%i@2!RT+obYF|%pu1K%r?r5D6aN*2~&Fox$S0X+b z)LwdS;xzetfdkJ0h1FXZq}C=d%5XBLFfh;4%9_;0Cb2X;k?qw2J#(9rb}9{%kGL-E z%bR{K&ErMBKUL)s?1_>d>mMRCSv;zyKAF>l_m@E>d{3Yv6)dQBa2kc%A?0(FX zzHifVdcgD}K`?7!i5vTSCNn={iz(i>mN@?u|5PF|DN({NvM*uQzku{#pOS7YlN4>+ zc`)#D<2LcD+G`g|9$4_?TT6eFc7WL_d(oLO6*J7ZmIl1obcX5N!x}ZNX=b%c9L!i6 zbNgN@cju>f>su{)yu0CnRm6iCYzMim6xi7k`M>mXb3NoMdBA*P1@}7+{#Om01`j5e zI5Em7uw2=$Vb!#*(}dCMNbc-~`!x>C_`V{53j}knIgDw ztKz9oK|7A6{!Yr@Gp#-Lc;as908SP&hN>p36~0;%*XmTAdBYUbz1nNX=WYpqY({WqI(7-Dl=Jv$AMl%W71(GskLnBe%@~BdcQ0 zfA4Ihx)yL2Y@TE6$C!71vci*@T<*&5Eg~-tFm)te@^9vqJ9u@S*Oi7X^LTF_oZb|} zp~`$GY)9aN8D-B{434`zV-T8l@ZFBjxBOxxn;Ioj_BC5aJahdio@cu?_Rdj3M?nEb z!MM|Q)pOc%f8Ln5Px_MLaqH{F#{B{|3Cvmv%q#3xRI*JFn<=%}ees1_ey2u}ISrhQ zjRKkpJG2r`aUS*OsS0?~%*c{p_im@^rjvIym8QJ&VUufAunOLs?I`5HV06TRy@bJb zbs|&A0*tGd-92aYF<0tZ+m6d2|)u!|jFWs>A-yKMJ{IjTfS zYr~aGy;A>Jmo!S5C{!^l6bVT9azKRXMT1~e-PPU3Hko!yviE(@jgu_;d#!Zu^Ge$n z4>>-+nS3L=>Uh4@@$R`lQzN#CZk670P>QYS6=&WOH?I?nsXKnY-TuQ!gvpUPCtlWo;vg_)Z z=03M=QWrIt7CJU7xpIlD3`jZH#K9+%Q8MAcK}I$XAsr6^1qWd^1_>6A1_cJ@UK4Ap zgbxbK{W!#g0t60dh$uTsKATXKtjNs9C8lMOdzjeMvK2+`%2ld#eeSm?;D8@Dav zqk{rd3$t8qlh1?$ObuMz0wD$h50*8~RJSjCvZEo7|G|{WtZYFWI(SW4jF=Tpcd#+A zs>TF322V0^nra-D*xF+pqxpGi_at7s-arh;1wszR^>6v}18X zYiaWJq^&Ok82!a!5)N{*CHHP1+$!m?tG$7YSFAb1#XFX zB6ry2Sp-xZ7R=;lxzWJaeC5)aqV5*~E-V!e-}`yuOpLn3SzU@wYh02H>eiL3IS`<$ z=iGI%5vGzxTwq)s(DX))Pn<3WYXq8&<;EP5i;r>Pz{$lAD7Os3%P(d5LA zixJa%?;guoyxsEIRbSJsCJ#j>st8mvO`4rMVclej4^d6L0S1n&!m?W?ICJxugdSvF zXK5OIfJfj=bRPGb&dWZ0Gj1}lm;ckde#Yp3Mc_o0eiI>2!6*}_0LGSoi(5|=GzGXQ z-}>{|k&%U=(cO5vMA2#8au)?h4l~;Y#qP@vEfHi|JS&S~mC%H$65rLI?gVmh+FY1C zQ$|s39DDkyr5%G zvuE9!c)_W^tATmhqFeLaEDbMQWMb7C3UI{#_d|C*aB z(cy7A>c<8%Q-Q|%{D~L0{Sk8bFB#xg)0x8bzsX_8KNA6dg*`rOY8w~>;?kJro#~m} zmclsC;aH2%olD6Y4@6EowlPdQIV;%X!%|kBgjT_b1{aHfb|)UEpu;^bysBUO6T24j z*BTs=Jrcl{+I6t2_QE0dX_*{1o-W{yf8i`~D#6vQ!ogf1qmd_K!-hq!{ux{vizS?c zgubaHsQ7bteseN%|Mns$n5W@9qt1s0;W-ZK&z+a@N^IJ=~#Co(r&867EdhYjmmQ5*r{43tZw=aO6p1c)W1()c6P`ul*T!{c{TgmM9so3d}E<;{Gs7 zVf%+LWzS1*>nG*pGV?xUoTm4ZBh=uqs#Rg2i})fdZRe0wasdRX^qquLImO?pawMx?WYL@=s-_Cj;ZWj+wj} z37$f6LPgod20}X+7EYR*lggaAP1u*a{>SC`bVIINJz$!oeZ z4PQ>n5O~C_XUim+$ITUx;B;axLu=mJf)-WYQjeBwFSc1s-Vw73PT4cKD7-tsqTz8c z&ToUGY)L^QU(P|km>ui5YZ}-m*glfK&?&5^boA=_b=UW_1?Y!nU+c-LI2_a3HQ6L< zVXMpo26l%9%);|NuqY%v*v}Tg!1OLEVWFt+fn`z4ZN8qG*J}BrQ|bqU=qd*`qZtSJ zQ&upEPT0_>9@fUFvV&1sNt@?%T$n^t%EYzi+U$Qdg^p^5F?L7VMD6vMbnG9)6hqJC zl4QBW#ZuPGR!Z72cKJLhyLIJQuIMBlrlk+G!hAlyESNlHiu4Udk%k-13JeE3?0io0 zHO_Em&naNHdC9O;WM)@huG zy7utTL2gTiLwn{Mv|Ae_CFe3YG>8c#xYuhPkjhNAZa=2k|HO?!NRqPBq(j4b>oQYqQ9Iz56a%AZ8+32wPx1FYj6J%3yb3 zsOC{{^D%6As<6OmMY(mGmdk&k-KNZvryjKHv|REQnc=A1;n=mX`~M5~@FQIuo*v6q z`|`eVH~jCQ&Ae*A!uiU$!tVGd_7h%l=o@P#Bphkv`(v*6V|#3J0-Kv!?(_z(1rxaU z3REp*=bB#-eI=b$uuXenp!Cc@#cdlUAF*+mH!uY#a5!D$C@Wwt4q*4&z;n%jtN8)~^bt_$s2Np+1UN_I?RYrRnyG(Aas0aMosrov50{TcQ8o0R{T zi?g>V$8T3oQWw8hz&)dZbM^$z+;+~L4cspZcwaQ|-AmxR*TC&o7M&WXTLXA68gTd;u(uYli#jkc2bgdL z)EqW++G(iS_y(zJmi+>E`9fweq=UE>3@ z{{-g67Yd?Xd->I?&VHz}7{WJRzQ$5E0%l+4gEnaaaOdngN^P@d-T#I3_&cn83i< z$-vpDFwy=RXNm!%|0j;E3_R^0CTcTHx;%+pC9@-I1M}9%+1eYJB~_U}x|fEi8|%L8 zn9xxwdo;?>Eywbc(u)hc_a<<=E#sWYz_qub>-GcQQB;7$qiPwj z(@TxTTcUv3r+``ZLZ(Cmlk9_FivaG27X)HlIpzg$AHK|U@k7O)=9Vunr#)d^5VLf9^3$o$g$t6_U0y=KBn z20KLt>4KRT!`T`Ixc(hr;B+%DImEy@OOwN^p?0nVtJ)Wi(*|7a1sok87Ax!&t8CyX zxWHCqFuQ9Dn?~hqt()0~L^jau+BGt+JVKB=A<|PTdXFnA5eyDsmHT`@5ckxNaJf*l- zuJcRPG+PBY+72vgbzr_w%DzZ|_gVt?Ey04bEHmCr;Io$CwbS5ge!%+eMh25+rjk@b zN`@Qv2bU?G6Ve`VE@a?5U{F{0f}@jR*1~|Svzut;bFG#E*Y*kjcnXb{>S@j~TDkOYSAF5ct{ny`7jE*sy}IqY|NrxHg4tudoklb*Dfkl8p?*qrf1Iz1n^{}$bwg_;> zUt{!`(ehJt^Ubs zLL3g6e{b|VDZ3eNWHe}Jjkv&h#er*qCQsi3j_wDW+aGN1JiyVaP`ka9MZ|$?>j#dG z1MD3ROO9o&DN0y#YS$Xoo!RO?*JyuW(q1qrbjNOpW?6)``iPQgfe0ty>{c_Gkm|m6^+?PhctiXXWT-q;{ul{f{`I0uIANUh`H2`b?x3hu(?uFP9oCoFitfA+!5BNI66e{b$m*m9bi zBV|L|+yoY30j`6;wggtM3G&{W6WpyPjqrnuyZuEZw&zsX}MvHkI zfACJ&k)!9P7uD6-nK<|K%(>hvxS!tQe|v!MUcua(n|M1O^6qiqp4Grw_B};9!H(T% zN8F(uehV8fq&m$E?$^D*aZP}Cy93A8hTiP~zR?V9(E;rL;u+YYr?%u}a`Rp9*`}a5 z&tcaR23E5TEUSJqS_!c6J)gFrdfUqbeDlo1b_cV6J+YAQgQnBs-Lk6~zH{t(%OVq8 zz-ivi_t${;K*MUITYKs<>e>Z3<~huqVZe6nG|!O*>;b#?2G3^C5oT}waV%3{pL*x) zoj13}+@Af?yHrUawC@9d_B5WY6L?I6c~1r`)i&FoWgTUy8C3DGe)<-N_KkDaNTghf zT=(_>_nv^cEIi^b7VzFV(6wbkbXhv1YJ%jmj;Omb#ef`q5 zs8c5tmQCS3f-U ze;0$R!|6*2eW!kKp3=Co*C*~q2A94Mue{CSC9gT>M2GIo;aDWVbtNMGti};z;VR}t zj*1B(%mSG%k9QwbzsBvr@U&+0^n`Vv4)8d1uU0SOI6Q%)x`2B}1M8kFHsJ)0r4C%b z{+&6WdExxOKYN`wWNSnox8Bg9m2jhL#=dS{CXU_<-5WT#&+faxp`KpN{j^}4cG$(M z7kIBVEG_8HnJXhc#U{nklY9AG?wM6$ZYo#Hg%x!Z;$9pL{V;V!-pifK1bD+^xo665lu=-e z%hfvie8%611ntwCI0CL+3NPM!fd2(UZ_9=~ofDL*7_K)JaKCWi=D5k3yn(USfUAGQ z`GN^+ly|Oa^DleAtb1>0DKPoGxY5sh;tluy{XZvt-M;Nc0`IK_yx+KQ zgWN;nBaf4V-HiaPL0Ay{~~!NA8l^UG8jiHqSIkPKDbm|43P$t8MpY+1BHoShqG^>UGG{=>8slK@BIx=UdTIOiI$ zN(+Sk+j~AUVTtn1^Y^14u-jeW{>P-efJw!GiC509q=CIgL8kvNKNs(bing_(QS}Cy zyi)fb&VReM$a?$s3DMc@oHHG`HTQ8V%3gXaaEZ(2l!5`{785lg6}2QIHJf)*J0Ge_ z-C>Zr$WRvWPW-Pe1Jb`POK-G*nCl^K(TRpFs;K0t{ z_kro(UKL9gHoXf^X9sexJO1$T0|9^bxCg8?2iOJmZVK0}737^0_N(K;2QgW>?JA|Z zzjyC{6vZ}OpiApM*X{+6Hu=4Ndx3kmLT=iG6rJzF?yZlv)k^O@r@^&YP%wb~`~u!* zH#pAJZI~x;<*4u7Xw$0=4eTf9#CdA$x_K^?Cv*4Z+A~wM_}&~yYJSKYbCq4M;9a)e zB17+cin}&(PGFQbV9Whcw*0~?t#jO3eJ7v(d&OOP{=nj6$<+)-1qaxF%go+r`GCdl z#-r%t{r_I@>^mMKz+MyZak|{Yx-%DYrriyC)giHO+g#nwo;ebU+Z|ps@K&dBWLI!5 zxbaH!9rs4QOA2ebFI_v}e&Dsm`J!DDrPofB-XbEsF_6LW-BlAq_Olz7@7?od{`2Ku zG^bsO-O{k&%$(CZzjm&;@OCFd)tShxy%!gh-dMocg=@!ex@ z_{ZZ0v5|QLuZ~57LRs$ZZHej@ z1rHh<_(P8MtSsW3q8Gb<&97HayTyxrLO0A%{2baa*S+_}fs?|^tLK^Ia!C11p74-Q zDO+h>{l-O$xP+vhW=~Pk#ZsCEtdlg=8s#7&2H3KH#G4tQ{e0NTyTnIXWW`ws*}Ap%*gA~ zRP$(PvXYBDV*T!sK*k=EDJ~rxKJrx>N}_kF*DvaJ5_J^n-!nl)Wx}_|=a;w!MP)9H zN{UTembR*LnHrn)9naZw`!*G5aCfir6cI49Q0jiP?UswMZ$#oXXKlYZ4h`bs4ch;F z*&azXG;pNotkz*li&0?M!8YLld&Z&6XVct%FeP)!P7`Q0IM64P(r@M#;e7JLt%Ae6 z)+)j?yqr~DGxuAuyj*;^Nq(7_yrmo zcqYs^#PgD8Zp&(Z)5@9t#>XOf=>`r)_0?1VN)@Mo!8CG4UT+-@l?dm60R z^6QN-r^=tOulkdw#Du<_#>DQkAuLAv%F5YuJAQSph*y2Ha#mEy4F(n!vo(i$mTyof zJK3E2<_y;#ahb2-gumywu6y{|Ims_69RI63Q0%LQ>;GYHtRwKLw_>!OuwBS z>(8EL2;%AMv5nn!J3!}|Tr(Fhi^PIPhG!D2Li3ePI}}tq)tL6ScSPjB^mm+_f902p z)z$9bA6~Ct;u^)fVd?aYt)U0mydRq$TD0P9xXr;f(_3y0oH-A2-7T5~7fo%=kX;zT zwSdO%@RD0Y`PU}CYyIVOS?>G zx7^{#Uv`06VHHb<{K9nxtr9ByG6feLQ3wd z8)CN07+52;D~11EI3W1th|!C}6i;BV$NiqQwkWiB&WBg^`#(*+wIPr<`T=8{ zmBLf@o`O@_5{~Q&KO8n1H8AqTEMQV>U|^9*U}W=Am@cT{z|0lE5NCE^ddACyh?`~_u>{7kKDByh} zW~x+Sw_4X?vBeoj6#g}|%ak-QM6k?bI+5DI;K9Jibs_1)t0fI)nTC~`ml#CnOkj2C zZRAN*U>3F8(3q*qz{J<_uYu`>L6`f7g?$eUPb@WhI^pBm#NfA{_p4n58lAcrI07fL z{P9V>{O66jP*}8!ipa-)_Z(eG)P@m~`aIu@fqIq9+jwrmzyk`GKaV>{)gzTMHt&Td*(ieBMs=w-DcQ!dB5Us$R zAYs9)wLIIB9H=@wX*9HE?}&R2GYiQnM@Bd=9JBUc1} zA-6+=SY!AF{c9`D6Iha+*cliYcoLW-FC1XEGic^dP+*o^(7+(~fq`dTn9#X~BZ_>D z9TTNg6lY|4I3Hsdw0D}Wq$hgypKF7`S^Fv2H0P7|=$p~$!GZR*A@*&{*+ z9w=W4Jt@%a(ZcJ#V7=KJMuFxD29}}byB)6_5^%54QIwn3S7^e_FZYm1eAfqNiQfm< zoj)-07aVAoxMRUKtLOxG9E0HZid9NRA3ky?JZO@-bAZLB!I9VJ!xQ!R6xQH2XI+_v z9gm+e3Nkl3%WxU7O#URH^mb;RcVd;`45owZCZ?jZ?|;8L@k2<}7{ZMyH6bE??#yC;fRG%y9=7 zFyB1;Zeyc|i=4~$Ho+wiH^_v=rmFB@@ERqCIeoNcDeh-IhOysS5K4n{HzCEj!C@I)s(IWFrAU9Q;>2!@lp4_gO zl|HuOyBeqD&2)Tvsq4psT_#Jn*Jk~?8xpCT-5cOpnHj46#YjW_-|MUMcR%DUk%)QN zk!Im<;`!1u`|zc^0jq*E_i1m6y(KmKowNLj26m$fm-yXZ`J^#*zY1QyfVV2)j6#&` z5eJ<$f_@81<+r7DXLT^kJ*se9@95QK*>vT)^Y>CCxqpm3iMIW_J+YY;b$-fa@uqH#1rt8SeHPdd zo&RI6KHpD8PtUqn9hTpAiy3Y#HvUi}?3)*Kfm^<0gGk*ruD`3oMW1&b)m)e1?R?HX z&-yR%43EW= zuYcQCtG4n}Ojpgtr5(GSKjihUU|`SM_pY=(;QhDC?aM-5@Am)C_^;xiwm`;(`5PV@ z=P!j1!RGfg(V^ZZEo6pfWuOHsnv!>2_~9dY$pv_EfEl?n@+2gh#x5<9&!POmmQrL3__y|M3Y zFk8|Crw!cpi9cEsKd2QRHqv!o*Vxf~FlyaoBl+^(9p5YLf1YNn>~@x1yx(-OrYlpY zYnSo7P4nX?Ol4BslfFXKDP>un!j8B=X?x}^i#>MadF(jWe8e+HYmq?vf*D)5Q%-Qy zDDB7wo^vxMo&7P9^K#EsN7?LKJo^LI1ia`DP+;jiqEm5!?V<9% zKn~WbS)PkNn$^K z>z8SH%)8OnYHXp&G_7;@9b7dvZ zRFU@V6KrvvZ2=d0qBpSR&0sH6XiqiRH(lmzoD8dq$~G09ZL@jYl3Lu7m+aGwIhJt3 zN%zSiwGgkw9mhUh-Zb&krsf%5p9^>U3%ZN0V3a!HCEMV<&gXdP5_8uozxP=xzkc=~ zkY1I#K;`h}B~Ctj+}Ak73!YfK!F#vK>@6#|oH}jMIorhRbsNu?9bbAJ(v*Zxemo(+ zaIV`2mOmyeN3Q(yIQ-yz-w~n9yZ!fPw740xuD`T5>j7I?L3>FB+k{;bK^(2_0<4c$ z2XM>oN@PAZr-dak;q208H(irsNjz*R3>OnP_Qy|fb6;^xTX}b`ws|o#zZd6bP2qE2 z3L7N~7}uWhT06ziGUuGdhQs{~opTq-uQ@0c@9ll)@Nw3g#+wI2DI^~GJe?YLgujjujlMY-EQkt}v|BZ*6 zK?~DGw)1n?nT_@-t-0VPuy6X2fGaxN+!<^RtTuE1acGH2pn&PI^gsLbj`$`@v}Ki? z`c!>UV(Anw#}4m8YsF%InSVzwK7JCocF#J~YUe-QQg%12e?DC4@ne62w3b6i+Z&l9 zteOVw7gao?&%4Sj(6W_wjq&)VBE2xcgJYqI=Bv)wMvFMcrK>NQWc8jf+S?kn=bF)6 znezexd7`ZTJI?>+>Dh3}Bk+V}Swnl}jP^<^yGtgAW~rDhF*e(1Y{r}FKh*zYH^S7x+5xa@lI$x$a! zvjl0gjZ4E*w4?d9hTQqGFTUZ*)~z8vu@|#zF6K^V&NbQI_~UGz%}UK;j?UhQ;H44W zu2*}mMpVt7Wf3s-pYdtSO;@8TI;-~>&yEntb#REUonU{MEid3mNA!u!f#)~fyt5$S zMC%-vp2z2AeK=j>(S9NH?t+BaX?NLne_^{BIw8xWYwq2f3%=gWxDmT*kwi=ao5rNJ z>OXBF#&(N8TXMhM8fS7P$(b`P`AT+iOa0!EEo<*36x`k}+vY66=JFuU@9rku!-^j_ zOxaSbx2Ulr=x=oA(MB2GJLXdiC1*G*ywx+kW+y*q~)<4^MwsxG#dJ2iTD zxYy{Pe;c!4#+Hvu9A+#P(0h94bg@a{hxuF~0VM&kJbvxmes`H4v#IY&D7^9D%H0Qf zKOP8udmy|{Ag@5+USx}VMVs4=sQJ|a>q75U{lB_>IWVPxfx#L8l&k?n From adfcc7218eca0d6b3a69dda8a1cfc676a2e93906 Mon Sep 17 00:00:00 2001 From: "902449@58880@bigcat_chen@ASIC" Date: Thu, 16 Jul 2020 09:46:51 +0800 Subject: [PATCH 0075/1017] TFLM:update Himax WE1 EVB example micro speech animation gif link --- tensorflow/lite/micro/examples/micro_speech/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/lite/micro/examples/micro_speech/README.md b/tensorflow/lite/micro/examples/micro_speech/README.md index 5b291a4d6cf..5b25bc99da0 100644 --- a/tensorflow/lite/micro/examples/micro_speech/README.md +++ b/tensorflow/lite/micro/examples/micro_speech/README.md @@ -660,7 +660,7 @@ Following the Steps to run micro speech example at HIMAX WE1 EVB platform. After these steps, press reset button on the HIMAX WE1 EVB, you will see application output in the serial terminal and lighting LED. -![Animation on Himax WE1 EVB](https://github.com/HimaxWiseEyePlus/bsp_tflu/tree/master/HIMAX_WE1_EVB_user_guide/images/tflm_example_micro_speech_int8_led.gif) +![Animation on Himax WE1 EVB](https://raw.githubusercontent.com/HimaxWiseEyePlus/bsp_tflu/master/HIMAX_WE1_EVB_user_guide/images/tflm_example_micro_speech_int8_led.gif) ## Run on macOS From 02ca05721ecaac4b2f0ed4f29cc4b690523795e0 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Thu, 16 Jul 2020 18:50:22 +0000 Subject: [PATCH 0076/1017] removed unused imports --- .../python/kernel_tests/map_ops_test.py | 23 ++++++------------- tensorflow/python/ops/map_ops.py | 3 --- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index e9355af27ba..b71e8ca8ebe 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -17,27 +17,18 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.platform import test from absl.testing import parameterized -from tensorflow.python.framework import test_util - -from tensorflow.python.client import session from tensorflow.python.eager import backprop -from tensorflow.python.eager import context -from tensorflow.python.eager import def_function -from tensorflow.python.eager import function from tensorflow.python.framework import constant_op -from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import test_util from tensorflow.python.ops import map_ops +from tensorflow.python.platform import test @test_util.run_all_in_graph_and_eager_modes class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): - - def testEmptyTensorMap(self): - m = map_ops.empty_tensor_map() - - def testTensorMapSize(self): + + def testEmptyTensorMapSize(self): m = map_ops.empty_tensor_map() s = map_ops.tensor_map_size(m) self.assertAllEqual(s, 0) @@ -62,12 +53,12 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) v = constant_op.constant(2.0) - + with self.assertRaisesRegex(errors.InvalidArgumentError, "Trying to lookup non-existent key."): l = map_ops.tensor_map_lookup(m, k) self.evaluate(l) - + def testTensorMapReplace(self): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) @@ -115,7 +106,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): "Trying to erase non-existent item."): m, e = map_ops.tensor_map_erase(m, k) self.evaluate(e) - + def testTensorMapEraseMissingKeyFails(self): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 20806e6fd30..7813247c8e2 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -18,14 +18,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.framework import load_library -from tensorflow.python.platform import resource_loader # go/tf-wildcard-import # pylint: disable=wildcard-import from tensorflow.python.framework import ops from tensorflow.python.ops import gen_map_ops from tensorflow.python.ops.gen_map_ops import * -from tensorflow.python.framework import constant_op ops.NotDifferentiable("EmptyTensorMap") From e3d5bcc115dc449290336718eacae0f2fa03c4ff Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Thu, 16 Jul 2020 18:50:47 +0000 Subject: [PATCH 0077/1017] add api def files --- .../core/api_def/base_api/api_def_EmptyTensorMap.pbtxt | 7 +++++++ .../core/api_def/base_api/api_def_TensorMapErase.pbtxt | 10 ++++++++++ .../api_def/base_api/api_def_TensorMapInsert.pbtxt | 10 ++++++++++ .../api_def/base_api/api_def_TensorMapLookup.pbtxt | 9 +++++++++ .../api_def/base_api/api_def_TensorMapReplace.pbtxt | 10 ++++++++++ .../core/api_def/base_api/api_def_TensorMapSize.pbtxt | 8 ++++++++ tensorflow/core/ops/map_ops.cc | 2 +- 7 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_EmptyTensorMap.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorMapErase.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorMapInsert.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorMapLookup.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorMapReplace.pbtxt create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorMapSize.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_EmptyTensorMap.pbtxt b/tensorflow/core/api_def/base_api/api_def_EmptyTensorMap.pbtxt new file mode 100644 index 00000000000..fb5ce3d5413 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_EmptyTensorMap.pbtxt @@ -0,0 +1,7 @@ +op { + graph_op_name: "EmptyTensorMap" + summary: "Creates and returns an empty tensor map." + description: < Date: Thu, 16 Jul 2020 19:43:31 +0000 Subject: [PATCH 0078/1017] buildifier fix --- tensorflow/core/kernels/BUILD | 5 +++-- tensorflow/python/BUILD | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 85c2b9d175b..28f651fb33c 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2937,6 +2937,7 @@ tf_kernel_library( "//third_party/eigen3", ], ) + cc_library( name = "tensor_map", srcs = ["tensor_map.cc"], @@ -2968,13 +2969,13 @@ tf_cc_tests( name = "tensor_map_test", size = "small", srcs = [ - "tensor_map_test.cc" + "tensor_map_test.cc", ], deps = [ ":tensor_map", - "//tensorflow/core/framework:tensor_testutil", "//tensorflow/core:test", "//tensorflow/core:test_main", + "//tensorflow/core/framework:tensor_testutil", "@com_google_absl//absl/strings", ], ) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 3c465252007..87ad52c3bf8 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -168,9 +168,9 @@ py_library( ":kernels", ":lib", ":list_ops", - ":map_ops", ":manip_ops", ":map_fn", + ":map_ops", ":math_ops", ":metrics", ":nccl_ops", From 6b936b21d4f319c46951fc61f3dfae56212e8af2 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 16 Jul 2020 21:16:21 +0000 Subject: [PATCH 0079/1017] added summary_op dependency to C API build and Python BUILD --- tensorflow/core/BUILD | 2 ++ tensorflow/python/BUILD | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index d1909ea1bac..6da0208c653 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -892,6 +892,7 @@ cc_library( ":user_ops_op_lib", ":word2vec_ops", "//tensorflow/c/kernels:bitcast_op_lib", + "//tensorflow/c/kernels:summary_op_lib", "//tensorflow/compiler/mlir/tensorflow:mlir_passthrough_op", ] + if_chromiumos( [], @@ -998,6 +999,7 @@ cc_library( name = "all_kernels_impl", visibility = [":__subpackages__"], deps = [ + "//tensorflow/c/kernels:summary_op", "//tensorflow/c/kernels:bitcast_op", "//tensorflow/core/kernels:array", "//tensorflow/core/kernels:audio", diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 3901e6a1408..e76a0c60d7a 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2908,6 +2908,10 @@ tf_gen_op_wrapper_private_py( "//learning/brain/python/ops:__pkg__", "//tensorflow/python/kernel_tests:__pkg__", ], + deps = [ + "//tensorflow/c/kernels:summary_op_lib", + "//tensorflow/core:logging_ops_op_lib", + ], ) tf_gen_op_wrapper_private_py( From 4819021890ba58f17e7d56cc3208930942078705 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 16 Jul 2020 22:56:19 +0000 Subject: [PATCH 0080/1017] wrapped summary_op.cc in anonymous namespace and fixed naming in summary_op_test --- tensorflow/c/kernels/BUILD | 7 ++-- tensorflow/c/kernels/summary_op.cc | 56 +++++++++++++------------ tensorflow/c/kernels/summary_op_test.cc | 11 +++-- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index c5c652ab5d7..87ae0339c6f 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -84,7 +84,7 @@ cc_library( srcs = ["tensor_shape_utils.cc",], hdrs = ["tensor_shape_utils.h",], deps = [ "//tensorflow/c:tf_tensor",], - visibility = ["//visibility:public"], + visibility = ["//visibility:private"], ) tf_cc_test( @@ -116,7 +116,8 @@ filegroup( filegroup( name = "android_all_ops", - srcs = ["ops/bitcast.cc", - "ops/summary.cc" + srcs = [ + "ops/bitcast.cc", + "ops/summary.cc" ], ) diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 1bd14eaf9c9..6b611be7e4f 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -23,36 +23,37 @@ limitations under the License. #include "tensorflow/core/framework/summary.pb.h" #include "tensorflow/core/framework/types.h" +namespace { + // Struct that stores the status and TF_Tensor inputs to the opkernel. // Used to delete tensor and status in its destructor upon kernel return. -namespace { - struct Params { - TF_Tensor* tags; - TF_Tensor* values; - TF_Status* status; - Params(TF_OpKernelContext* ctx) : tags(nullptr), - values(nullptr), - status(nullptr) { - status = TF_NewStatus(); - TF_GetInput(ctx, 0, &tags, status); - if (TF_GetCode(status) == TF_OK) { - TF_GetInput(ctx, 1, &values, status); - } - }; - ~Params() { - TF_DeleteStatus(status); - TF_DeleteTensor(tags); - TF_DeleteTensor(values); +struct Params { + TF_Tensor* tags; + TF_Tensor* values; + TF_Status* status; + Params(TF_OpKernelContext* ctx) : tags(nullptr), + values(nullptr), + status(nullptr) { + status = TF_NewStatus(); + TF_GetInput(ctx, 0, &tags, status); + if (TF_GetCode(status) == TF_OK) { + TF_GetInput(ctx, 1, &values, status); } }; -} + ~Params() { + TF_DeleteStatus(status); + TF_DeleteTensor(tags); + TF_DeleteTensor(values); + } +}; + // dummy functions used for kernel registration -static void* ScalarSummaryOp_Create(TF_OpKernelConstruction* ctx) { +void* ScalarSummaryOp_Create(TF_OpKernelConstruction* ctx) { return nullptr; } -static void ScalarSummaryOp_Delete(void* kernel) { +void ScalarSummaryOp_Delete(void* kernel) { return; } @@ -60,10 +61,10 @@ static void ScalarSummaryOp_Delete(void* kernel) { bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2); // Returns a string representation of a single tag or empty string if there // are multiple tags -static tensorflow::string SingleTag(TF_Tensor* tags); +tensorflow::string SingleTag(TF_Tensor* tags); template -static void ScalarSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { +void ScalarSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { Params params(ctx); if (TF_GetCode(params.status) != TF_OK){ TF_OpKernelContext_Failure(ctx, params.status); @@ -105,7 +106,7 @@ static void ScalarSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { TF_DeleteTensor(summary_tensor); } -bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ +bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2) { if (TF_NumDims(tensor1) != TF_NumDims(tensor2)) { return false; } @@ -117,7 +118,7 @@ bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ return true; } -static tensorflow::string SingleTag(TF_Tensor* tags){ +tensorflow::string SingleTag(TF_Tensor* tags) { if (TF_TensorElementCount(tags) == 1) { const char* single_tag = static_cast( TF_TensorData(tags))->c_str(); @@ -150,7 +151,7 @@ void RegisterScalarSummaryOpKernel() { // A dummy static variable initialized by a lambda whose side-effect is to // register the ScalarSummary kernel. -TF_ATTRIBUTE_UNUSED static bool IsScalarSummaryOpKernelRegistered = []() { +TF_ATTRIBUTE_UNUSED bool IsScalarSummaryOpKernelRegistered = []() { if (SHOULD_REGISTER_OP_KERNEL("ScalarSummary")) { RegisterScalarSummaryOpKernel(); RegisterScalarSummaryOpKernel(); @@ -166,5 +167,6 @@ TF_ATTRIBUTE_UNUSED static bool IsScalarSummaryOpKernelRegistered = []() { RegisterScalarSummaryOpKernel(); } return true; -}(); +}(); +} // namespace diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc index ad5fafe5530..4c691379bed 100644 --- a/tensorflow/c/kernels/summary_op_test.cc +++ b/tensorflow/c/kernels/summary_op_test.cc @@ -32,10 +32,9 @@ class DummyDevice : public DeviceBase { }; // Helper for comparing ouput and expected output -static void EXPECT_SummaryMatches(const Summary& actual, - const string& expected_str) { +void ExpectSummaryMatches(const Summary& actual, const string& expected_str) { Summary expected; - (protobuf::TextFormat::ParseFromString(expected_str, &expected)); + ASSERT_TRUE(protobuf::TextFormat::ParseFromString(expected_str, &expected)); EXPECT_EQ(expected.DebugString(), actual.DebugString()); } @@ -77,8 +76,9 @@ void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, ASSERT_EQ(expected_code, ctx.status().code()); if (expected_code == error::OK){ Summary summary; - ParseProtoUnlimited(&summary, ctx.mutable_output(0)->scalar()()); - EXPECT_SummaryMatches(summary, expected_summary); + ASSERT_TRUE(ParseProtoUnlimited(&summary, ctx.mutable_output(0)-> + scalar()())); + ExpectSummaryMatches(summary, expected_summary); } } @@ -133,7 +133,6 @@ TEST(ScalarSummaryOpTest, SimpleHalf) { } TEST(ScalarSummaryOpTest, Error_WrongDimsTags) { - int vectorSize = 3; Tensor tags(DT_STRING, {2, 1}); Tensor values(DT_FLOAT, {2}); tags.matrix()(0, 0) = "tag1"; From 75b8d57f5b4eb0b5eca0623119783454ac853bce Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 16 Jul 2020 23:17:33 +0000 Subject: [PATCH 0081/1017] added wrapper for TF_Tensor in tensor_shape_utils --- .../c/kernels/tensor_shape_utils_test.cc | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tensorflow/c/kernels/tensor_shape_utils_test.cc b/tensorflow/c/kernels/tensor_shape_utils_test.cc index 25620838437..9c7f45c1256 100644 --- a/tensorflow/c/kernels/tensor_shape_utils_test.cc +++ b/tensorflow/c/kernels/tensor_shape_utils_test.cc @@ -23,11 +23,25 @@ limitations under the License. namespace tensorflow { -template -void TestShapeMatch(T shape) { +namespace { + +// A wrapper that will automatically delete the allocated TF_Tensor +// once out of scope. +struct TF_TensorWrapper { + TF_Tensor* tf_tensor; + TF_TensorWrapper(TF_Tensor* tensor){ + tf_tensor = tensor; + } + ~TF_TensorWrapper() { + TF_DeleteTensor(tf_tensor); + } +}; + +void TestShapeMatch(TensorShape shape) { Tensor tensor(DT_FLOAT, shape); Status status; TF_Tensor* tf_tensor = TF_TensorFromTensor(tensor, &status); + TF_TensorWrapper tensor_wrapper = TF_TensorWrapper(tf_tensor); ASSERT_TRUE(status.ok()) << status.ToString(); ASSERT_EQ(tensor.shape().DebugString(), ShapeDebugString(tf_tensor)); } @@ -40,4 +54,5 @@ TEST(ShapeDebugString, ScalarShape) { TestShapeMatch(TensorShape({})); } +} // namespace } // namespace tensorflow From 7a45b98508e4374a04c8ad2db1876b2e53f2e398 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 16 Jul 2020 23:22:31 +0000 Subject: [PATCH 0082/1017] spacing for tensor_shape_utils --- tensorflow/c/kernels/tensor_shape_utils.cc | 26 +++++++-------- .../c/kernels/tensor_shape_utils_test.cc | 32 +++++++++---------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tensorflow/c/kernels/tensor_shape_utils.cc b/tensorflow/c/kernels/tensor_shape_utils.cc index 6ca138584b7..0720414dea6 100644 --- a/tensorflow/c/kernels/tensor_shape_utils.cc +++ b/tensorflow/c/kernels/tensor_shape_utils.cc @@ -22,19 +22,19 @@ limitations under the License. #include "tensorflow/core/platform/logging.h" namespace tensorflow { - + std::string ShapeDebugString(TF_Tensor* tensor) { - // A TF_Tensor cannot have an unknown rank. - CHECK_GE(TF_NumDims(tensor), 0); - tensorflow::string s = "["; - for (int i = 0; i < TF_NumDims(tensor); ++i) { - if (i > 0) tensorflow::strings::StrAppend(&s, ","); - int64_t dim = TF_Dim(tensor, i); - // A TF_Tensor cannot have an unknown dimension. - CHECK_GE(dim, 0); - tensorflow::strings::StrAppend(&s, dim); - } - tensorflow::strings::StrAppend(&s, "]"); - return s; + // A TF_Tensor cannot have an unknown rank. + CHECK_GE(TF_NumDims(tensor), 0); + tensorflow::string s = "["; + for (int i = 0; i < TF_NumDims(tensor); ++i) { + if (i > 0) tensorflow::strings::StrAppend(&s, ","); + int64_t dim = TF_Dim(tensor, i); + // A TF_Tensor cannot have an unknown dimension. + CHECK_GE(dim, 0); + tensorflow::strings::StrAppend(&s, dim); + } + tensorflow::strings::StrAppend(&s, "]"); + return s; } } // namespace tensorflow \ No newline at end of file diff --git a/tensorflow/c/kernels/tensor_shape_utils_test.cc b/tensorflow/c/kernels/tensor_shape_utils_test.cc index 9c7f45c1256..a08e4a67e3e 100644 --- a/tensorflow/c/kernels/tensor_shape_utils_test.cc +++ b/tensorflow/c/kernels/tensor_shape_utils_test.cc @@ -28,31 +28,31 @@ namespace { // A wrapper that will automatically delete the allocated TF_Tensor // once out of scope. struct TF_TensorWrapper { - TF_Tensor* tf_tensor; - TF_TensorWrapper(TF_Tensor* tensor){ - tf_tensor = tensor; - } - ~TF_TensorWrapper() { - TF_DeleteTensor(tf_tensor); - } + TF_Tensor* tf_tensor; + TF_TensorWrapper(TF_Tensor* tensor){ + tf_tensor = tensor; + } + ~TF_TensorWrapper() { + TF_DeleteTensor(tf_tensor); + } }; void TestShapeMatch(TensorShape shape) { - Tensor tensor(DT_FLOAT, shape); - Status status; - TF_Tensor* tf_tensor = TF_TensorFromTensor(tensor, &status); - TF_TensorWrapper tensor_wrapper = TF_TensorWrapper(tf_tensor); - ASSERT_TRUE(status.ok()) << status.ToString(); - ASSERT_EQ(tensor.shape().DebugString(), ShapeDebugString(tf_tensor)); + Tensor tensor(DT_FLOAT, shape); + Status status; + TF_Tensor* tf_tensor = TF_TensorFromTensor(tensor, &status); + TF_TensorWrapper tensor_wrapper = TF_TensorWrapper(tf_tensor); + ASSERT_TRUE(status.ok()) << status.ToString(); + ASSERT_EQ(tensor.shape().DebugString(), ShapeDebugString(tf_tensor)); } TEST(ShapeDebugString, RegularShape) { - TestShapeMatch(TensorShape({5, 4, 7})); + TestShapeMatch(TensorShape({5, 4, 7})); } TEST(ShapeDebugString, ScalarShape) { - TestShapeMatch(TensorShape({})); + TestShapeMatch(TensorShape({})); } - + } // namespace } // namespace tensorflow From 2a3f88d4c8a1595936ff5d90f783fa043b88fa09 Mon Sep 17 00:00:00 2001 From: Yimei Sun Date: Fri, 10 Jul 2020 20:37:17 -0700 Subject: [PATCH 0083/1017] Add auto_mixed_precision_mkl to run-once optimizer list --- tensorflow/core/grappler/optimizers/meta_optimizer.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc index 2f1c869965d..0df32168787 100644 --- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc @@ -88,7 +88,8 @@ int NumIterations(const RewriterConfig& cfg) { // Check if optimizer is allowed to run only once. bool IsRunOnceOptimizer(const string& name) { return name == "layout" || name == "memory_optimizer" || - name == "loop_optimizer" || name == "auto_mixed_precision"; + name == "loop_optimizer" || name == "auto_mixed_precision" || + name == "auto_mixed_precision_mkl"; } bool IsTFDataFunction(const FunctionDef& func) { From 25197272cf1163cf6aa0fa6c20bea4b0369cbe0e Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 17 Jul 2020 20:33:53 +0000 Subject: [PATCH 0084/1017] TensorKey check shape and dtype --- tensorflow/core/kernels/tensor_map_test.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/kernels/tensor_map_test.cc b/tensorflow/core/kernels/tensor_map_test.cc index 1ee175be34d..8ea4efd75fa 100644 --- a/tensorflow/core/kernels/tensor_map_test.cc +++ b/tensorflow/core/kernels/tensor_map_test.cc @@ -36,10 +36,12 @@ TEST(TensorKeyTest, Equal) { TensorKey k1 = Tensor(15); TensorKey k2 = Tensor(15); EXPECT_EQ(k1, k2); + EXPECT_EQ(k1.shape(), k2.shape()); + EXPECT_EQ(k1.dtype(), k2.dtype()); - TensorKey k3 = Tensor(15); - TensorKey k4 = Tensor(37); - EXPECT_NE(k3, k4); + TensorKey k3 = Tensor(37.0); + EXPECT_NE(k1, k3); + EXPECT_NE(k1.dtype(), k3.dtype()); } TEST(TensorMapTest, Insert) { From 0ae7b5327815455487471cce1fb206b8ef036b24 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 17 Jul 2020 20:35:35 +0000 Subject: [PATCH 0085/1017] separate key and value dtypes --- tensorflow/core/kernels/map_kernels.cc | 2 - tensorflow/core/kernels/map_kernels.h | 45 +++++-------------- tensorflow/core/ops/map_ops.cc | 39 +++++++--------- .../python/kernel_tests/map_ops_test.py | 40 ++++------------- tensorflow/python/ops/map_ops.py | 11 ++--- 5 files changed, 37 insertions(+), 100 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index 45fa86c2bf6..c890ba77f54 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -35,6 +35,4 @@ REGISTER_KERNEL_BUILDER(Name("TensorMapInsert").Device(DEVICE_CPU), REGISTER_KERNEL_BUILDER(Name("TensorMapErase").Device(DEVICE_CPU), TensorMapErase); -REGISTER_KERNEL_BUILDER(Name("TensorMapReplace").Device(DEVICE_CPU), - TensorMapReplace); } diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 1ab6fbd2323..5fb856bf00f 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -112,9 +112,7 @@ class TensorMapSize : public OpKernel { class TensorMapInsert : public OpKernel { public: - explicit TensorMapInsert(OpKernelConstruction* c) : OpKernel(c) { - OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); - } + explicit TensorMapInsert(OpKernelConstruction* c) : OpKernel(c) {} ~TensorMapInsert() override {} void Compute(OpKernelContext* c) override { @@ -125,19 +123,14 @@ class TensorMapInsert : public OpKernel { TensorMap* output_map = nullptr; OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); - output_map->insert(key, value); + output_map->replace(key, value); } - - private: - DataType element_dtype_; }; class TensorMapLookup : public OpKernel { public: - explicit TensorMapLookup(OpKernelConstruction* c) : OpKernel(c) { - OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); - } + explicit TensorMapLookup(OpKernelConstruction* c) : OpKernel(c) {} ~TensorMapLookup() override {} void Compute(OpKernelContext* c) override { @@ -150,17 +143,12 @@ class TensorMapLookup : public OpKernel { c->set_output(0, m->tensors().find(key)->second); } - - private: - DataType element_dtype_; }; class TensorMapErase : public OpKernel { public: - explicit TensorMapErase(OpKernelConstruction* c) : OpKernel(c) { - OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); - } + explicit TensorMapErase(OpKernelConstruction* c) : OpKernel(c) {} void Compute(OpKernelContext* c) override { const TensorMap* m = nullptr; @@ -177,35 +165,22 @@ class TensorMapErase : public OpKernel { OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); output_map->tensors().erase(key); } - - private: - DataType element_dtype_; }; -class TensorMapReplace : public OpKernel { +class TensorMapHasKey : public OpKernel { public: - explicit TensorMapReplace(OpKernelConstruction* c) : OpKernel(c) { - OP_REQUIRES_OK(c, c->GetAttr("element_dtype", &element_dtype_)); - } - ~TensorMapReplace() override {} + explicit TensorMapHasKey(OpKernelConstruction* c) : OpKernel(c) {} + ~TensorMapHasKey() override {} void Compute(OpKernelContext* c) override { const TensorKey& key = c->input(1); - const Tensor& value = c->input(2); const TensorMap* m = nullptr; OP_REQUIRES_OK(c, GetInputMap(c, 0, &m)); - - OP_REQUIRES(c, m->tensors().find(key) != m->tensors().end(), - errors::InvalidArgument("Trying to replace non-existent key.")); - - TensorMap* output_map = nullptr; - OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); - output_map->replace(key, value); + Tensor* result; + OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape{}, &result)); + result->scalar()() = m->tensors().find(key) != m->tensors().end(); } - - private: - DataType element_dtype_; }; diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index f289a13c188..2e4284aa4a2 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -35,10 +35,11 @@ REGISTER_OP("TensorMapSize") REGISTER_OP("TensorMapInsert") .Input("input_handle: variant") - .Input("key: element_dtype") - .Input("value: element_dtype") + .Input("key: key_dtype") + .Input("value: value_dtype") .Output("output_handle: variant") - .Attr("element_dtype: type") + .Attr("key_dtype: type") + .Attr("value_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { c->set_output(0, c->Scalar()); return Status::OK(); @@ -46,38 +47,28 @@ REGISTER_OP("TensorMapInsert") REGISTER_OP("TensorMapLookup") .Input("input_handle: variant") - .Input("key: element_dtype") - .Output("value: element_dtype") - .Attr("element_dtype: type") + .Input("key: key_dtype") + .Output("value: value_dtype") + .Attr("key_dtype: type") + .Attr("value_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { - c->set_output(0, c->Scalar()); + c->set_output(0, c->UnknownShape()); return Status::OK(); }); REGISTER_OP("TensorMapErase") .Input("input_handle: variant") - .Input("key: element_dtype") + .Input("key: key_dtype") .Output("output_handle: variant") - .Output("value: element_dtype") - .Attr("element_dtype: type") + .Output("value: value_dtype") + .Attr("key_dtype: type") + .Attr("value_dtype: type") .SetShapeFn([](shape_inference::InferenceContext* c) { - DataType element_dtype; - TF_RETURN_IF_ERROR(c->GetAttr("element_dtype", &element_dtype)); - c->set_output(1, c->Scalar()); // removed element - c->set_output(0, c->Scalar()); // map + c->set_output(0, c->Scalar()); // output map + c->set_output(1, c->UnknownShape()); // removed element return Status::OK(); }); -REGISTER_OP("TensorMapReplace") - .Input("input_handle: variant") - .Input("key: element_dtype") - .Input("value: element_dtype") - .Output("output_handle: variant") - .Attr("element_dtype: type") - .SetShapeFn([](shape_inference::InferenceContext* c) { - c->set_output(0, c->Scalar()); - return Status::OK(); - }); } // namespace } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index b71e8ca8ebe..443b8fd34fc 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -20,6 +20,7 @@ from __future__ import print_function from absl.testing import parameterized from tensorflow.python.eager import backprop from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import test_util from tensorflow.python.ops import map_ops @@ -46,7 +47,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): k = constant_op.constant(1.0) v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k, v) - l = map_ops.tensor_map_lookup(m, k) + l = map_ops.tensor_map_lookup(m, k, dtypes.float32) self.assertAllClose(l, v) def testTensorMapLookupMissingKeyFails(self): @@ -56,34 +57,9 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.assertRaisesRegex(errors.InvalidArgumentError, "Trying to lookup non-existent key."): - l = map_ops.tensor_map_lookup(m, k) + l = map_ops.tensor_map_lookup(m, k, dtypes.float32) self.evaluate(l) - def testTensorMapReplace(self): - m = map_ops.empty_tensor_map() - k = constant_op.constant(1.0) - v = constant_op.constant(2.0) - m = map_ops.tensor_map_insert(m, k, v) - s = map_ops.tensor_map_size(m) - self.assertAllClose(s, 1) - - v2 = constant_op.constant(3.0) - m = map_ops.tensor_map_replace(m, k, v2) - l = map_ops.tensor_map_lookup(m, k) - self.assertAllClose(l, v2) - - def testTensorMapReplaceMissingKeyFails(self): - m = map_ops.empty_tensor_map() - k = constant_op.constant(1.0) - k2 = constant_op.constant(2.0) - v = constant_op.constant(2.0) - m = map_ops.tensor_map_insert(m, k2, v) - - with self.assertRaisesRegex(errors.InvalidArgumentError, - "Trying to replace non-existent key."): - m = map_ops.tensor_map_replace(m, k, v) - self.evaluate(m) - def testTensorMapErase(self): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) @@ -92,7 +68,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): s = map_ops.tensor_map_size(m) self.assertAllEqual(s, 1) - m, e = map_ops.tensor_map_erase(m, k) + m, e = map_ops.tensor_map_erase(m, k, dtypes.float32) s = map_ops.tensor_map_size(m) self.assertAllEqual(s, 0) self.assertAllClose(e, v) @@ -104,7 +80,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.assertRaisesRegex(errors.InvalidArgumentError, "Trying to erase non-existent item."): - m, e = map_ops.tensor_map_erase(m, k) + m, e = map_ops.tensor_map_erase(m, k, dtypes.float32) self.evaluate(e) def testTensorMapEraseMissingKeyFails(self): @@ -116,9 +92,9 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.assertRaisesRegex(errors.InvalidArgumentError, "Trying to erase non-existent item."): - m, e = map_ops.tensor_map_erase(m, k) + m, e = map_ops.tensor_map_erase(m, k, dtypes.float32) self.evaluate(e) - + ''' def testInsertLookupGrad(self): with backprop.GradientTape() as tape: m = map_ops.empty_tensor_map() @@ -129,7 +105,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = map_ops.tensor_map_lookup(m, k) l *= 5 g = tape.gradient(l, v) - self.assertAllClose(g, 5.0) + self.assertAllClose(g, 5.0)''' if __name__ == '__main__': diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 7813247c8e2..f14c1314d71 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -35,14 +35,11 @@ def tensor_map_size(input_handle): def tensor_map_insert(input_handle, key, value): return gen_map_ops.tensor_map_insert(input_handle, key, value) -def tensor_map_lookup(input_handle, key): - return gen_map_ops.tensor_map_lookup(input_handle, key) +def tensor_map_lookup(input_handle, key, value_dtype): + return gen_map_ops.tensor_map_lookup(input_handle, key, value_dtype) -def tensor_map_erase(input_handle, key): - return gen_map_ops.tensor_map_erase(input_handle, key) - -def tensor_map_replace(input_handle, key, value): - return gen_map_ops.tensor_map_replace(input_handle, key, value) +def tensor_map_erase(input_handle, key, value_dtype): + return gen_map_ops.tensor_map_erase(input_handle, key, value_dtype) @ops.RegisterGradient("TensorMapLookup") def LookupGrad(op, dval): From d910087ab6fc62b08f9a010d9f1f4c7e05bd02b0 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 17 Jul 2020 20:49:50 +0000 Subject: [PATCH 0086/1017] add TensorMapHasKey op --- tensorflow/core/kernels/map_kernels.cc | 3 +++ tensorflow/core/ops/map_ops.cc | 6 ++++++ .../python/kernel_tests/map_ops_test.py | 19 +++++++++++++++---- tensorflow/python/ops/map_ops.py | 5 ++++- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index c890ba77f54..db91a660809 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -35,4 +35,7 @@ REGISTER_KERNEL_BUILDER(Name("TensorMapInsert").Device(DEVICE_CPU), REGISTER_KERNEL_BUILDER(Name("TensorMapErase").Device(DEVICE_CPU), TensorMapErase); +REGISTER_KERNEL_BUILDER(Name("TensorMapHasKey").Device(DEVICE_CPU), + TensorMapHasKey); + } diff --git a/tensorflow/core/ops/map_ops.cc b/tensorflow/core/ops/map_ops.cc index 2e4284aa4a2..072c116fc29 100644 --- a/tensorflow/core/ops/map_ops.cc +++ b/tensorflow/core/ops/map_ops.cc @@ -69,6 +69,12 @@ REGISTER_OP("TensorMapErase") return Status::OK(); }); +REGISTER_OP("TensorMapHasKey") + .Input("input_handle: variant") + .Input("key: element_dtype") + .Output("has_key: bool") + .Attr("element_dtype: type") + .SetShapeFn(shape_inference::ScalarShape); } // namespace } // namespace tensorflow diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 443b8fd34fc..26f1ea93e1a 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -94,7 +94,19 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): "Trying to erase non-existent item."): m, e = map_ops.tensor_map_erase(m, k, dtypes.float32) self.evaluate(e) - ''' + + def testTensorMapHasKey(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + k2 = constant_op.constant(2.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + + b = map_ops.tensor_map_has_key(m, k) + b2 = map_ops.tensor_map_has_key(m, k2) + self.assertAllEqual(b, True) + self.assertAllEqual(b2, False) + def testInsertLookupGrad(self): with backprop.GradientTape() as tape: m = map_ops.empty_tensor_map() @@ -102,11 +114,10 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): v = constant_op.constant(2.0) tape.watch(v) m = map_ops.tensor_map_insert(m, k, v) - l = map_ops.tensor_map_lookup(m, k) + l = map_ops.tensor_map_lookup(m, k, dtypes.float32) l *= 5 g = tape.gradient(l, v) - self.assertAllClose(g, 5.0)''' - + self.assertAllClose(g, 5.0) if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index f14c1314d71..144346976d1 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -41,6 +41,9 @@ def tensor_map_lookup(input_handle, key, value_dtype): def tensor_map_erase(input_handle, key, value_dtype): return gen_map_ops.tensor_map_erase(input_handle, key, value_dtype) +def tensor_map_has_key(input_handle, key): + return gen_map_ops.tensor_map_has_key(input_handle, key) + @ops.RegisterGradient("TensorMapLookup") def LookupGrad(op, dval): m, k = op.inputs @@ -54,5 +57,5 @@ def InsertGrad(op, dmap): _, key, val = op.inputs map_grad = None key_grad = None - value_grad = tensor_map_lookup(dmap, key) + value_grad = tensor_map_lookup(dmap, key, val.dtype) return map_grad, key_grad, value_grad From a0a897784a2762b8b452fbf3d388fa50f8fa4203 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 20 Jul 2020 05:17:17 +0000 Subject: [PATCH 0087/1017] testHasKeyLookup --- .../python/kernel_tests/map_ops_test.py | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 26f1ea93e1a..b214909dc54 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -26,6 +26,10 @@ from tensorflow.python.framework import test_util from tensorflow.python.ops import map_ops from tensorflow.python.platform import test +from tensorflow.python.util.lazy_loader import LazyLoader +control_flow_ops = LazyLoader("control_flow_ops", globals(), + "tensorflow.python.ops.control_flow_ops") + @test_util.run_all_in_graph_and_eager_modes class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): @@ -96,16 +100,35 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.evaluate(e) def testTensorMapHasKey(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + k2 = constant_op.constant(2.0) + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + + b = map_ops.tensor_map_has_key(m, k) + b2 = map_ops.tensor_map_has_key(m, k2) + self.assertAllEqual(b, True) + self.assertAllEqual(b2, False) + + def testHasKeyLookup(self): + with self.test_session(): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) k2 = constant_op.constant(2.0) v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k, v) - b = map_ops.tensor_map_has_key(m, k) - b2 = map_ops.tensor_map_has_key(m, k2) - self.assertAllEqual(b, True) - self.assertAllEqual(b2, False) + default_value = constant_op.constant(0.0) + l = control_flow_ops.cond(map_ops.tensor_map_has_key(m, k), + lambda: map_ops.tensor_map_lookup(m, k, dtypes.float32), + lambda: default_value) + l2 = control_flow_ops.cond(map_ops.tensor_map_has_key(m, k2), + lambda: map_ops.tensor_map_lookup(m, k, dtypes.float32), + lambda: default_value) + + self.assertAllClose(l, v) + self.assertAllClose(l2, default_value) def testInsertLookupGrad(self): with backprop.GradientTape() as tape: From 6fe5847c19b7d4fdf777f14b7c8edb1b2870d397 Mon Sep 17 00:00:00 2001 From: Krzysztof Laskowski Date: Mon, 20 Jul 2020 14:19:13 +0200 Subject: [PATCH 0088/1017] Fix setting out_mtypes in MemoryTypesForNode Clear hostmem_attr vector before populating it with indices from "_output_hostmem" attribute so that it doesn't contain indices from "_input_hostmem" attribute. --- tensorflow/core/framework/memory_types.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/framework/memory_types.cc b/tensorflow/core/framework/memory_types.cc index d27ef1da61d..208ad20c21b 100644 --- a/tensorflow/core/framework/memory_types.cc +++ b/tensorflow/core/framework/memory_types.cc @@ -161,6 +161,7 @@ Status MemoryTypesForNode(const OpRegistryInterface* op_registry, } } } + hostmem_attr.clear(); if (TryGetNodeAttr(ndef, "_output_hostmem", &hostmem_attr)) { for (int32 i : hostmem_attr) { if (0 <= i && i < out_mtypes->size()) { From c99cade2628071213656fec80b7ab5f04b1fc04e Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 20 Jul 2020 18:11:16 +0000 Subject: [PATCH 0089/1017] modify gradient --- .../python/kernel_tests/map_ops_test.py | 43 +++++++++++++++---- tensorflow/python/ops/map_ops.py | 18 ++++++-- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index b214909dc54..c6f37251f21 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -26,7 +26,7 @@ from tensorflow.python.framework import test_util from tensorflow.python.ops import map_ops from tensorflow.python.platform import test -from tensorflow.python.util.lazy_loader import LazyLoader +from tensorflow.python.util.lazy_loader import LazyLoader control_flow_ops = LazyLoader("control_flow_ops", globals(), "tensorflow.python.ops.control_flow_ops") @@ -110,7 +110,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): b2 = map_ops.tensor_map_has_key(m, k2) self.assertAllEqual(b, True) self.assertAllEqual(b2, False) - + def testHasKeyLookup(self): with self.test_session(): m = map_ops.empty_tensor_map() @@ -119,17 +119,15 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k, v) - default_value = constant_op.constant(0.0) l = control_flow_ops.cond(map_ops.tensor_map_has_key(m, k), - lambda: map_ops.tensor_map_lookup(m, k, dtypes.float32), - lambda: default_value) + lambda: map_ops.tensor_map_lookup(m, k, dtypes.float32), + lambda: array_ops.zeros_like(v)) l2 = control_flow_ops.cond(map_ops.tensor_map_has_key(m, k2), - lambda: map_ops.tensor_map_lookup(m, k, dtypes.float32), - lambda: default_value) - + lambda: map_ops.tensor_map_lookup(m, k, dtypes.float32), + lambda: default_value) self.assertAllClose(l, v) self.assertAllClose(l2, default_value) - + def testInsertLookupGrad(self): with backprop.GradientTape() as tape: m = map_ops.empty_tensor_map() @@ -142,5 +140,32 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): g = tape.gradient(l, v) self.assertAllClose(g, 5.0) + #TODO(kattian): Test alternating inserts and lookups + def testMultipleInsertLookupGrad(self): + with backprop.GradientTape(persistent=True) as tape: + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + k2 = constant_op.constant(12.0) + v2 = constant_op.constant(22.0) + k3 = constant_op.constant(13.0) + v3 = constant_op.constant(23.0) + tape.watch(v) + tape.watch(v2) + tape.watch(v3) + m = map_ops.tensor_map_insert(m, k, v) + m = map_ops.tensor_map_insert(m, k2, v2) + m = map_ops.tensor_map_insert(m, k3, v3) + + l = map_ops.tensor_map_lookup(m, k, v.dtype) + l2 = map_ops.tensor_map_lookup(m, k2, v2.dtype) + l3 = map_ops.tensor_map_lookup(m, k3, v3.dtype) + g = tape.gradient(l * 5, v) + self.assertAllClose(g, 5) + g2 = tape.gradient(l2 * 5, v2) + self.assertAllClose(g2, 5) + g3 = tape.gradient(l3 * 5, v3) + self.assertAllClose(g3, 5) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index 144346976d1..f18623aa41a 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -22,7 +22,13 @@ from __future__ import print_function # pylint: disable=wildcard-import from tensorflow.python.framework import ops from tensorflow.python.ops import gen_map_ops +from tensorflow.python.ops import array_ops from tensorflow.python.ops.gen_map_ops import * +from tensorflow.python.framework import constant_op + +from tensorflow.python.util.lazy_loader import LazyLoader +control_flow_ops = LazyLoader("control_flow_ops", globals(), + "tensorflow.python.ops.control_flow_ops") ops.NotDifferentiable("EmptyTensorMap") @@ -46,7 +52,7 @@ def tensor_map_has_key(input_handle, key): @ops.RegisterGradient("TensorMapLookup") def LookupGrad(op, dval): - m, k = op.inputs + _, k = op.inputs map_grad = empty_tensor_map() map_grad = tensor_map_insert(map_grad, k, dval) key_grad = None @@ -54,8 +60,12 @@ def LookupGrad(op, dval): @ops.RegisterGradient("TensorMapInsert") def InsertGrad(op, dmap): - _, key, val = op.inputs - map_grad = None + _, k, v = op.inputs key_grad = None - value_grad = tensor_map_lookup(dmap, key, val.dtype) + value_grad = control_flow_ops.cond(tensor_map_has_key(dmap, k), + lambda: tensor_map_lookup(dmap, k, v.dtype), + lambda: array_ops.zeros_like(v)) + map_grad = control_flow_ops.cond(tensor_map_has_key(dmap, k), + lambda: tensor_map_erase(dmap, k, v.dtype)[0], + lambda: dmap) return map_grad, key_grad, value_grad From 0599a371346ca56ef28280670d9bd951e6794ce3 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 20 Jul 2020 18:38:43 +0000 Subject: [PATCH 0090/1017] same key tests --- .../python/kernel_tests/map_ops_test.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index c6f37251f21..c66b9bbad92 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -23,6 +23,7 @@ from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops from tensorflow.python.ops import map_ops from tensorflow.python.platform import test @@ -57,8 +58,6 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testTensorMapLookupMissingKeyFails(self): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) - v = constant_op.constant(2.0) - with self.assertRaisesRegex(errors.InvalidArgumentError, "Trying to lookup non-existent key."): l = map_ops.tensor_map_lookup(m, k, dtypes.float32) @@ -80,8 +79,6 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): def testTensorMapEraseFromEmptyMapFails(self): m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) - v = constant_op.constant(2.0) - with self.assertRaisesRegex(errors.InvalidArgumentError, "Trying to erase non-existent item."): m, e = map_ops.tensor_map_erase(m, k, dtypes.float32) @@ -93,7 +90,6 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): k2 = constant_op.constant(2.0) v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k2, v) - with self.assertRaisesRegex(errors.InvalidArgumentError, "Trying to erase non-existent item."): m, e = map_ops.tensor_map_erase(m, k, dtypes.float32) @@ -119,9 +115,10 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): v = constant_op.constant(2.0) m = map_ops.tensor_map_insert(m, k, v) + default_value = array_ops.zeros_like(v) l = control_flow_ops.cond(map_ops.tensor_map_has_key(m, k), lambda: map_ops.tensor_map_lookup(m, k, dtypes.float32), - lambda: array_ops.zeros_like(v)) + lambda: default_value) l2 = control_flow_ops.cond(map_ops.tensor_map_has_key(m, k2), lambda: map_ops.tensor_map_lookup(m, k, dtypes.float32), lambda: default_value) @@ -161,11 +158,27 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l2 = map_ops.tensor_map_lookup(m, k2, v2.dtype) l3 = map_ops.tensor_map_lookup(m, k3, v3.dtype) g = tape.gradient(l * 5, v) - self.assertAllClose(g, 5) g2 = tape.gradient(l2 * 5, v2) - self.assertAllClose(g2, 5) g3 = tape.gradient(l3 * 5, v3) + self.assertAllClose(g, 5) + self.assertAllClose(g2, 5) self.assertAllClose(g3, 5) + def testSameKeyInsertLookupGrad(self): + with backprop.GradientTape(persistent=True) as tape: + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + v2 = constant_op.constant(22.0) + tape.watch(v) + tape.watch(v2) + m = map_ops.tensor_map_insert(m, k, v) + m = map_ops.tensor_map_insert(m, k, v2) + l = map_ops.tensor_map_lookup(m, k, v.dtype) + g = tape.gradient(l * 5, v) + g2 = tape.gradient(l * 5, v2) + self.assertAllClose(g, array_ops.zeros_like(v)) + self.assertAllClose(g2, 5) + if __name__ == '__main__': test.main() From f2a506484fd391dd4ff099968b960bcdd9f7ed44 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 20 Jul 2020 19:54:22 +0000 Subject: [PATCH 0091/1017] add another test --- .../python/kernel_tests/map_ops_test.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index c66b9bbad92..688f4907457 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -180,5 +180,25 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllClose(g, array_ops.zeros_like(v)) self.assertAllClose(g2, 5) + def testSameKeyInsertLookupGrad2(self): + with backprop.GradientTape(persistent=True) as tape: + m = map_ops.empty_tensor_map() + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + v2 = constant_op.constant(22.0) + tape.watch(v) + tape.watch(v2) + m = map_ops.tensor_map_insert(m, k, v) + l = map_ops.tensor_map_lookup(m, k, v.dtype) + g = tape.gradient(l * 5, v) + self.assertAllClose(g, 5) + + m = map_ops.tensor_map_insert(m, k, v2) + l2 = map_ops.tensor_map_lookup(m, k, v2.dtype) + g2 = tape.gradient(l2 * 5, v2) + g3 = tape.gradient(l2 * 5, v) + self.assertAllClose(g2, 5) + self.assertAllClose(g3, array_ops.zeros_like(v)) + if __name__ == '__main__': test.main() From 47c1aeb1aac754a0a6c9797cfc1248c3bb302421 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 20 Jul 2020 20:25:24 +0000 Subject: [PATCH 0092/1017] update tests --- .../python/kernel_tests/map_ops_test.py | 22 +++++++++---------- tensorflow/python/ops/map_ops.py | 1 - 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 688f4907457..7fda6fb3efd 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -135,9 +135,8 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = map_ops.tensor_map_lookup(m, k, dtypes.float32) l *= 5 g = tape.gradient(l, v) - self.assertAllClose(g, 5.0) + self.assertAllClose(g, 5) - #TODO(kattian): Test alternating inserts and lookups def testMultipleInsertLookupGrad(self): with backprop.GradientTape(persistent=True) as tape: m = map_ops.empty_tensor_map() @@ -158,11 +157,11 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l2 = map_ops.tensor_map_lookup(m, k2, v2.dtype) l3 = map_ops.tensor_map_lookup(m, k3, v3.dtype) g = tape.gradient(l * 5, v) - g2 = tape.gradient(l2 * 5, v2) - g3 = tape.gradient(l3 * 5, v3) + g2 = tape.gradient(l2 * 6, v2) + g3 = tape.gradient(l3 * 7, v3) self.assertAllClose(g, 5) - self.assertAllClose(g2, 5) - self.assertAllClose(g3, 5) + self.assertAllClose(g2, 6) + self.assertAllClose(g3, 7) def testSameKeyInsertLookupGrad(self): with backprop.GradientTape(persistent=True) as tape: @@ -180,7 +179,7 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllClose(g, array_ops.zeros_like(v)) self.assertAllClose(g2, 5) - def testSameKeyInsertLookupGrad2(self): + def testSameKeyAlternatingInsertLookupGrad(self): with backprop.GradientTape(persistent=True) as tape: m = map_ops.empty_tensor_map() k = constant_op.constant(1.0) @@ -192,13 +191,12 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): l = map_ops.tensor_map_lookup(m, k, v.dtype) g = tape.gradient(l * 5, v) self.assertAllClose(g, 5) - m = map_ops.tensor_map_insert(m, k, v2) l2 = map_ops.tensor_map_lookup(m, k, v2.dtype) - g2 = tape.gradient(l2 * 5, v2) - g3 = tape.gradient(l2 * 5, v) - self.assertAllClose(g2, 5) - self.assertAllClose(g3, array_ops.zeros_like(v)) + g2 = tape.gradient(l2 * 6, v) + g3 = tape.gradient(l2 * 7, v2) + self.assertAllClose(g2, array_ops.zeros_like(v)) + self.assertAllClose(g3, 7) if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index f18623aa41a..443bb0b1934 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -24,7 +24,6 @@ from tensorflow.python.framework import ops from tensorflow.python.ops import gen_map_ops from tensorflow.python.ops import array_ops from tensorflow.python.ops.gen_map_ops import * -from tensorflow.python.framework import constant_op from tensorflow.python.util.lazy_loader import LazyLoader control_flow_ops = LazyLoader("control_flow_ops", globals(), From 1392800b57e7797c18113e7ed9ac5baa55a7eee2 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 20 Jul 2020 22:09:45 +0000 Subject: [PATCH 0093/1017] api def TensorMapHasKey --- .../core/api_def/base_api/api_def_TensorMapHasKey.pbtxt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tensorflow/core/api_def/base_api/api_def_TensorMapHasKey.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_TensorMapHasKey.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorMapHasKey.pbtxt new file mode 100644 index 00000000000..fc46a3abfd9 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_TensorMapHasKey.pbtxt @@ -0,0 +1,9 @@ +op { + graph_op_name: "TensorMapHasKey" + summary: "Returns whether the given key exists in the map." + description: < Date: Mon, 20 Jul 2020 21:34:56 -0700 Subject: [PATCH 0094/1017] Enable eager test for GradientDescentOptimizerTest --- .../optimizer_v2/gradient_descent_test.py | 159 +++++++++--------- 1 file changed, 79 insertions(+), 80 deletions(-) diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py index 0084f04bdd9..1f5a1004b48 100644 --- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py +++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py @@ -149,29 +149,28 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.evaluate(var0)) self.assertAllCloseAccordingToType([3.0 - 1.0], self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testMinimizeSparseResourceVariable(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. - with ops.Graph().as_default(): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) - var1 = variables.Variable([3.0], dtype=dtype) - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) + var1 = variables.Variable([3.0], dtype=dtype) + x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - def loss(): - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) # pylint: disable=cell-var-from-loop - pred += var1 # pylint: disable=cell-var-from-loop - return pred * pred + def loss(): + pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) # pylint: disable=cell-var-from-loop + pred += var1 # pylint: disable=cell-var-from-loop + return pred * pred - sgd_op = gradient_descent.SGD(1.0).minimize(loss, [var0, var1]) - self.evaluate(variables.global_variables_initializer()) - # Run 1 step of sgd - self.evaluate(sgd_op) - # Validate updated params - np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 - np_grad = 2 * np_pred - self.assertAllCloseAccordingToType( - [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0)) - self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1)) + sgd_op = gradient_descent.SGD(1.0).minimize(loss, [var0, var1]) + self.evaluate(variables.global_variables_initializer()) + # Run 1 step of sgd + self.evaluate(sgd_op) + # Validate updated params + np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 + np_grad = 2 * np_pred + self.assertAllCloseAccordingToType( + [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1)) def testTensorLearningRate(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: @@ -191,72 +190,72 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testGradWrtRef(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. - with ops.Graph().as_default(): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - opt = gradient_descent.SGD(3.0) - values = [1.0, 3.0] - vars_ = [variables.Variable([v], dtype=dtype) for v in values] - loss = lambda: vars_[0] + vars_[1] # pylint: disable=cell-var-from-loop - grads_and_vars = opt._compute_gradients(loss, vars_) - self.evaluate(variables.global_variables_initializer()) - for grad, _ in grads_and_vars: - self.assertAllCloseAccordingToType([1.0], self.evaluate(grad)) + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + opt = gradient_descent.SGD(3.0) + values = [1.0, 3.0] + vars_ = [variables.Variable([v], dtype=dtype) for v in values] + loss = lambda: vars_[0] + vars_[1] # pylint: disable=cell-var-from-loop + grads_and_vars = opt._compute_gradients(loss, vars_) + self.evaluate(variables.global_variables_initializer()) + for grad, _ in grads_and_vars: + self.assertAllCloseAccordingToType([1.0], self.evaluate(grad)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSparseBasic(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. - with ops.Graph().as_default(): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) - var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant([0.1], shape=[1, 1], dtype=dtype), - constant_op.constant([0]), constant_op.constant([2, 1])) - grads1 = ops.IndexedSlices( - constant_op.constant([0.01], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), constant_op.constant([2, 1])) - sgd_op = gradient_descent.SGD(3.0).apply_gradients( - zip([grads0, grads1], [var0, var1])) - self.evaluate(variables.global_variables_initializer()) - # Run 1 step of sgd - self.evaluate(sgd_op) - # Validate updated params - self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], - self.evaluate(var0)) - self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], - self.evaluate(var1)) + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) + var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) + grads0 = ops.IndexedSlices( + constant_op.constant([0.1], shape=[1, 1], dtype=dtype), + constant_op.constant([0]), constant_op.constant([2, 1])) + grads1 = ops.IndexedSlices( + constant_op.constant([0.01], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), constant_op.constant([2, 1])) + sgd_op = gradient_descent.SGD(3.0).apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Run 1 step of sgd + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], + self.evaluate(var0)) + self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], + self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSparseBasicWithLearningRateDecay(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. - with ops.Graph().as_default(): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) - var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant([0.1], shape=[1, 1], dtype=dtype), - constant_op.constant([0]), constant_op.constant([2, 1])) - grads1 = ops.IndexedSlices( - constant_op.constant([0.01], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), constant_op.constant([2, 1])) - sgd_op = gradient_descent.SGD( - 3.0, decay=0.5).apply_gradients( - zip([grads0, grads1], [var0, var1])) - self.evaluate(variables.global_variables_initializer()) - # Run 2 steps of sgd - self.evaluate(sgd_op) - # Validate updated params - self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], - self.evaluate(var0)) - self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], - self.evaluate(var1)) + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) + var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) + grads0 = ops.IndexedSlices( + constant_op.constant([0.1], shape=[1, 1], dtype=dtype), + constant_op.constant([0]), constant_op.constant([2, 1])) + grads1 = ops.IndexedSlices( + constant_op.constant([0.01], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), constant_op.constant([2, 1])) - self.evaluate(sgd_op) - # Validate updated params - self.assertAllCloseAccordingToType( - [[1.0 - 3.0 * 0.1 - 2.0 * 0.1], [2.0]], self.evaluate(var0)) - self.assertAllCloseAccordingToType( - [[3.0], [4.0 - 3.0 * 0.01 - 2.0 * 0.01]], self.evaluate(var1)) + opt = gradient_descent.SGD(3.0, decay=0.5) + update_op = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Run 2 steps of sgd + self.evaluate(update_op) + # Validate updated params + self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], + self.evaluate(var0)) + self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], + self.evaluate(var1)) + + if context.executing_eagerly(): + opt.apply_gradients(zip([grads0, grads1], [var0, var1])) + else: + self.evaluate(update_op) + # Validate updated params + self.assertAllCloseAccordingToType( + [[1.0 - 3.0 * 0.1 - 2.0 * 0.1], [2.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType( + [[3.0], [4.0 - 3.0 * 0.01 - 2.0 * 0.01]], self.evaluate(var1)) def testCapturingInDefunWhileExecutingEagerly(self): with context.eager_mode(): From a3a2eaf9a01cb5ced0a6322d561c05708e052e4f Mon Sep 17 00:00:00 2001 From: WindQAQ Date: Mon, 20 Jul 2020 22:20:36 -0700 Subject: [PATCH 0095/1017] Enable eager test for MomentumOptimizerTest --- .../optimizer_v2/gradient_descent_test.py | 507 +++++++++--------- 1 file changed, 262 insertions(+), 245 deletions(-) diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py index 1f5a1004b48..0872d6d3a29 100644 --- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py +++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py @@ -358,90 +358,100 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) ]), self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testNesterovMomentum(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. - with ops.Graph().as_default(): - for dtype in [dtypes.float32, dtypes.float64]: - var0 = variables.Variable([1.0, 2.0], dtype=dtype, name="var0") - var1 = variables.Variable([3.0, 4.0], dtype=dtype, name="var1") - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - loss = lambda: 5 * var0 * var0 + 3 * var1 # pylint: disable=cell-var-from-loop - mom_op = gradient_descent.SGD( - learning_rate=2.0, momentum=0.9, nesterov=True) - opt_op = mom_op.minimize(loss, [var0, var1]) - self.evaluate(variables.global_variables_initializer()) - for _ in range(1, 5): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([1.0, 2.0], dtype=dtype, name="var0") + var1 = variables.Variable([3.0, 4.0], dtype=dtype, name="var1") + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + loss = lambda: 5 * var0 * var0 + 3 * var1 # pylint: disable=cell-var-from-loop + mom_op = gradient_descent.SGD( + learning_rate=2.0, momentum=0.9, nesterov=True) + opt_op = mom_op.minimize(loss, [var0, var1]) + self.evaluate(variables.global_variables_initializer()) + for i in range(1, 5): + # already updated once in eager mode + if i != 1 and context.executing_eagerly(): + mom_op.minimize(loss, [var0, var1]) + else: self.evaluate(opt_op) - var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy( - var1_np, accum1_np, 3, 2.0, 0.9) - self.assertAllClose(var0_np, self.evaluate(var0)) - self.assertAllClose(var1_np, self.evaluate(var1)) + var0_np, accum0_np = self._update_nesterov_momentum_numpy( + var0_np, accum0_np, var0_np * 10, 2.0, 0.9) + var1_np, accum1_np = self._update_nesterov_momentum_numpy( + var1_np, accum1_np, 3, 2.0, 0.9) + self.assertAllClose(var0_np, self.evaluate(var0)) + self.assertAllClose(var1_np, self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSparseNesterovMomentum(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. for dtype in [dtypes.float32, dtypes.float64]: - with ops.Graph().as_default(), self.cached_session() as sess: - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - grads = [] + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + grads = [] + for t in range(1, 5): + grads.append(var0_np * 10) + var0_np, accum0_np = self._update_nesterov_momentum_numpy( + var0_np, accum0_np, var0_np * 10, 2.0, 0.9) + var1_np, accum1_np = self._update_nesterov_momentum_numpy( + var1_np, accum1_np, 3, 2.0, 0.9) + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + var0 = variables.Variable(var0_np, dtype=dtype, name="var0") + var1 = variables.Variable(var1_np, dtype=dtype, name="var1") + mom_op = gradient_descent.SGD( + learning_rate=2.0, momentum=0.9, nesterov=True) + grads_and_vars = [] + for t in range(1, 5): + y = ops.IndexedSlices(constant_op.constant(grads[t - 1], dtype=dtype), + constant_op.constant([0, 1]), + constant_op.constant([2])) + grads_and_vars.append([ + (y, var0), + (constant_op.constant([3.0, 3.0], dtype=dtype), var1)]) + if not context.executing_eagerly(): + opt_update = [] for t in range(1, 5): - grads.append(var0_np * 10) - var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy( - var1_np, accum1_np, 3, 2.0, 0.9) - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - var0 = variables.Variable(var0_np, dtype=dtype, name="var0") - var1 = variables.Variable(var1_np, dtype=dtype, name="var1") - mom_op = gradient_descent.SGD( - learning_rate=2.0, momentum=0.9, nesterov=True) - x_feed = array_ops.placeholder(dtype) - y_feed = ops.IndexedSlices(x_feed, constant_op.constant([0, 1]), - constant_op.constant([2])) - grads_and_vars = [(y_feed, var0), - (constant_op.constant([3.0, 3.0], dtype=dtype), var1)] - opt_update = mom_op.apply_gradients(grads_and_vars) - self.evaluate(variables.global_variables_initializer()) - for t in range(1, 5): - sess.run(opt_update, feed_dict={x_feed: grads[t - 1]}) - var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy( - var1_np, accum1_np, 3, 2.0, 0.9) - self.assertAllClose(var0_np, self.evaluate(var0)) - self.assertAllClose(var1_np, self.evaluate(var1)) + opt_update.append(mom_op.apply_gradients(grads_and_vars[t - 1])) + self.evaluate(variables.global_variables_initializer()) + for t in range(1, 5): + if context.executing_eagerly(): + mom_op.apply_gradients(grads_and_vars[t - 1]) + else: + self.evaluate(opt_update[t - 1]) + var0_np, accum0_np = self._update_nesterov_momentum_numpy( + var0_np, accum0_np, var0_np * 10, 2.0, 0.9) + var1_np, accum1_np = self._update_nesterov_momentum_numpy( + var1_np, accum1_np, 3, 2.0, 0.9) + self.assertAllClose(var0_np, self.evaluate(var0)) + self.assertAllClose(var1_np, self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testMinimizeSparseResourceVariable(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. - with ops.Graph().as_default(): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) - # pylint: disable=cell-var-from-loop - def loss(): - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - return pred * pred + # pylint: disable=cell-var-from-loop + def loss(): + x = constant_op.constant([[4.0], [5.0]], dtype=dtype) + pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) + return pred * pred - # pylint: enable=cell-var-from-loop + # pylint: enable=cell-var-from-loop - opt = gradient_descent.SGD(learning_rate=1.0, momentum=0.9) - sgd_op = opt.minimize(loss, [var0]) - self.evaluate(variables.global_variables_initializer()) - # Run 1 step of sgd - self.evaluate(sgd_op) - # Validate updated params - self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0)) + opt = gradient_descent.SGD(learning_rate=1.0, momentum=0.9) + sgd_op = opt.minimize(loss, [var0]) + self.evaluate(variables.global_variables_initializer()) + # Run 1 step of sgd + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0)) @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testMinimizeWith2DIndicesForEmbeddingLookup(self): @@ -456,151 +466,149 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): self.evaluate(sgd_op) self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testTensorLearningRateAndMomentum(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. - with ops.Graph().as_default(): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - mom_opt = gradient_descent.SGD( - learning_rate=constant_op.constant(2.0), - momentum=constant_op.constant(0.9)) - mom_update = mom_opt.apply_gradients( + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([1.0, 2.0], dtype=dtype) + var1 = variables.Variable([3.0, 4.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) + mom_opt = gradient_descent.SGD( + learning_rate=constant_op.constant(2.0), + momentum=constant_op.constant(0.9)) + mom_update = mom_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Check we have slots + slot0 = mom_opt.get_slot(var0, "momentum") + self.assertEqual(slot0.shape, var0.shape) + slot1 = mom_opt.get_slot(var1, "momentum") + self.assertEqual(slot1.shape, var1.shape) + + # Step 1: the momentum accumulators where 0. So we should see a normal + # update: v -= grad * learning_rate + self.evaluate(mom_update) + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([-0.2, -0.2]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([-0.02, -0.02]), self.evaluate(slot1)) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) + # Step 2: the momentum accumulators contain the previous update. + if context.executing_eagerly(): + mom_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) - self.evaluate(variables.global_variables_initializer()) - # Check we have slots - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEqual(slot0.shape, var0.shape) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEqual(slot1.shape, var1.shape) - - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - # Step 1: the momentum accumulators where 0. So we should see a normal - # update: v -= grad * learning_rate + else: self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([-0.2, -0.2]), self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([-0.02, -0.02]), self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), - self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), - self.evaluate(var1)) - # Step 2: the momentum accumulators contain the previous update. - self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), - self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.02) - 2.0 * 0.01), - (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), - 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), - 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) - ]), self.evaluate(var1)) + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), + self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([ + 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), + 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) + ]), self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([ + 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSparse(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. - with ops.Graph().as_default(): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable(array_ops.zeros([4, 2], dtype=dtype)) - var1 = variables.Variable(constant_op.constant(1.0, dtype, [4, 2])) - grads0 = ops.IndexedSlices( - constant_op.constant([[.1, .1]], dtype=dtype), - constant_op.constant([1]), constant_op.constant([4, 2])) - grads1 = ops.IndexedSlices( - constant_op.constant([[.01, .01], [.01, .01]], dtype=dtype), - constant_op.constant([2, 3]), constant_op.constant([4, 2])) - mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) - mom_update = mom_opt.apply_gradients( + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable(array_ops.zeros([4, 2], dtype=dtype)) + var1 = variables.Variable(constant_op.constant(1.0, dtype, [4, 2])) + grads0 = ops.IndexedSlices( + constant_op.constant([[.1, .1]], dtype=dtype), + constant_op.constant([1]), constant_op.constant([4, 2])) + grads1 = ops.IndexedSlices( + constant_op.constant([[.01, .01], [.01, .01]], dtype=dtype), + constant_op.constant([2, 3]), constant_op.constant([4, 2])) + mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) + mom_update = mom_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + + # Check we have slots + slot0 = mom_opt.get_slot(var0, "momentum") + self.assertEqual(slot0.shape, var0.shape) + slot1 = mom_opt.get_slot(var1, "momentum") + self.assertEqual(slot1.shape, var1.shape) + + # Step 1: the momentum accumulators are 0. So we should see a normal + # update: v -= grad * learning_rate + self.evaluate(mom_update) + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([0, 0]), + self.evaluate(slot0)[0]) + self.assertAllCloseAccordingToType( + np.array([-2.0 * .1, -2.0 * .1]), + self.evaluate(slot0)[1]) + self.assertAllCloseAccordingToType( + np.array([-2.0 * .01, -2.0 * .01]), + self.evaluate(slot1)[2]) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([0, 0]), + self.evaluate(var0)[0]) + self.assertAllCloseAccordingToType( + np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), + self.evaluate(var0)[1]) + self.assertAllCloseAccordingToType( + np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), + self.evaluate(var1)[2]) + # Step 2: the momentum accumulators contain the previous update. + if context.executing_eagerly(): + mom_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) - self.evaluate(variables.global_variables_initializer()) - - # Check we have slots - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEqual(slot0.shape, var0.shape) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEqual(slot1.shape, var1.shape) - - # Fetch params to validate initial values - self.assertAllClose([0, 0], self.evaluate(var0)[0]) - self.assertAllClose([0, 0], self.evaluate(var0)[1]) - self.assertAllClose([1, 1], self.evaluate(var1)[2]) - - # Step 1: the momentum accumulators are 0. So we should see a normal - # update: v -= grad * learning_rate + else: self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([0, 0]), - self.evaluate(slot0)[0]) - self.assertAllCloseAccordingToType( - np.array([-2.0 * .1, -2.0 * .1]), - self.evaluate(slot0)[1]) - self.assertAllCloseAccordingToType( - np.array([-2.0 * .01, -2.0 * .01]), - self.evaluate(slot1)[2]) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([0, 0]), - self.evaluate(var0)[0]) - self.assertAllCloseAccordingToType( - np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), - self.evaluate(var0)[1]) - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), - self.evaluate(var1)[2]) - # Step 2: the momentum accumulators contain the previous update. - self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0]) - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), - self.evaluate(slot0)[1]) - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.02) - 2.0 * 0.01), - (0.9 * (-0.02) - 2.0 * 0.01)]), - self.evaluate(slot1)[2]) - # Check that the parameters have been updated. - self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0]) - self.assertAllCloseAccordingToType( - np.array([ - -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), - -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), - self.evaluate(var0)[1]) - self.assertAllCloseAccordingToType( - np.array([ - 0.98 - ((0.9 * 0.01 + 0.01) * 2.0), - 0.98 - ((0.9 * 0.01 + 0.01) * 2.0) - ]), - self.evaluate(var1)[2]) + # Check that the momentum accumulators have been updated. + self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0]) + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), + self.evaluate(slot0)[1]) + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), + self.evaluate(slot1)[2]) + # Check that the parameters have been updated. + self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0]) + self.assertAllCloseAccordingToType( + np.array([ + -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), + -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) + ]), + self.evaluate(var0)[1]) + self.assertAllCloseAccordingToType( + np.array([ + 0.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 0.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), + self.evaluate(var1)[2]) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSharing(self): - # TODO(tanzheny, omalleyt): Fix test in eager mode. - with ops.Graph().as_default(): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([1.0, 2.0], dtype=dtype) + var1 = variables.Variable([3.0, 4.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) + mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) + if not context.executing_eagerly(): mom_update1 = mom_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) mom_update2 = mom_opt.apply_gradients( @@ -612,44 +620,53 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): slot1 = mom_opt.get_slot(var1, "momentum") self.assertEqual(slot1.shape, var1.shape) - # Fetch params to validate initial values - self.assertAllClose([1.0, 2.0], self.evaluate(var0)) - self.assertAllClose([3.0, 4.0], self.evaluate(var1)) - # Step 1: the momentum accumulators where 0. So we should see a normal - # update: v -= grad * learning_rate + # Step 1: the momentum accumulators where 0. So we should see a normal + # update: v -= grad * learning_rate + if context.executing_eagerly(): + mom_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + slot0 = mom_opt.get_slot(var0, "momentum") + self.assertEqual(slot0.shape, var0.shape) + slot1 = mom_opt.get_slot(var1, "momentum") + self.assertEqual(slot1.shape, var1.shape) + else: self.evaluate(mom_update1) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([-0.2, -0.2]), self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([-0.02, -0.02]), self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), - self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), - self.evaluate(var1)) - # Step 2: the second momentum accumulators contain the previous update. + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([-0.2, -0.2]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([-0.02, -0.02]), self.evaluate(slot1)) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) + # Step 2: the second momentum accumulators contain the previous update. + if context.executing_eagerly(): + mom_update2 = mom_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + else: self.evaluate(mom_update2) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), - self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.02) - 2.0 * 0.01), - (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), - 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), - 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) - ]), self.evaluate(var1)) + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), + self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([ + 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), + 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) + ]), self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([ + 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), self.evaluate(var1)) @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testConfig(self): From c06c26f032d746b190adf3250d9c7985d1e161be Mon Sep 17 00:00:00 2001 From: WindQAQ Date: Mon, 20 Jul 2020 22:39:01 -0700 Subject: [PATCH 0096/1017] Move decorator to class level --- .../optimizer_v2/gradient_descent_test.py | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py index 0872d6d3a29..4b05b3d2908 100644 --- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py +++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py @@ -37,9 +37,9 @@ from tensorflow.python.ops import variables from tensorflow.python.platform import test +@combinations.generate(combinations.combine(mode=["graph", "eager"])) class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasic(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([1.0, 2.0], dtype=dtype) @@ -88,7 +88,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): [3.0 - 3.0 * 0.01 - 2.0 * 0.01, 4.0 - 3.0 * 0.01 - 2.0 * 0.01], self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasicWithLearningRateDecay(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: learning_rate = 3.0 @@ -96,7 +95,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): sgd = gradient_descent.SGD(learning_rate=learning_rate, decay=decay) self._test_basic_sgd_with_learning_rate_decay(sgd, dtype) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasicWithLearningRateInverseTimeDecay(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: learning_rate = learning_rate_schedule.InverseTimeDecay( @@ -104,7 +102,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): sgd = gradient_descent.SGD(learning_rate=learning_rate) self._test_basic_sgd_with_learning_rate_decay(sgd, dtype) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasicWithLearningRateInverseTimeDecaySerializeAndDeserialize(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: learning_rate = learning_rate_schedule.InverseTimeDecay( @@ -113,7 +110,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): sgd = gradient_descent.SGD.from_config(sgd.get_config()) self._test_basic_sgd_with_learning_rate_decay(sgd, dtype) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasicCallableParams(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([1.0, 2.0], dtype=dtype) @@ -132,7 +128,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testMinimizeResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) @@ -149,7 +144,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.evaluate(var0)) self.assertAllCloseAccordingToType([3.0 - 1.0], self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) @@ -190,7 +184,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testGradWrtRef(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: opt = gradient_descent.SGD(3.0) @@ -202,7 +195,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): for grad, _ in grads_and_vars: self.assertAllCloseAccordingToType([1.0], self.evaluate(grad)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSparseBasic(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) @@ -224,7 +216,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSparseBasicWithLearningRateDecay(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) @@ -291,6 +282,7 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllClose(self.evaluate(opt_3.lr), (0.1)) +@combinations.generate(combinations.combine(mode=["graph", "eager"])) class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): def _update_nesterov_momentum_numpy(self, var, accum, g, lr, momentum): @@ -298,7 +290,6 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): var += (accum * momentum - g * lr) return var, accum - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasic(self): for _, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): var0 = variables.Variable([1.0, 2.0], dtype=dtype, name="var0") @@ -358,7 +349,6 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) ]), self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testNesterovMomentum(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([1.0, 2.0], dtype=dtype, name="var0") @@ -385,7 +375,6 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllClose(var0_np, self.evaluate(var0)) self.assertAllClose(var1_np, self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSparseNesterovMomentum(self): for dtype in [dtypes.float32, dtypes.float64]: var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) @@ -432,7 +421,6 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllClose(var0_np, self.evaluate(var0)) self.assertAllClose(var1_np, self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testMinimizeSparseResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) @@ -453,7 +441,6 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): # Validate updated params self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testMinimizeWith2DIndicesForEmbeddingLookup(self): var0 = variables.Variable(array_ops.ones([2, 2])) @@ -466,7 +453,6 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): self.evaluate(sgd_op) self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testTensorLearningRateAndMomentum(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([1.0, 2.0], dtype=dtype) @@ -525,7 +511,6 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) ]), self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSparse(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable(array_ops.zeros([4, 2], dtype=dtype)) @@ -600,7 +585,6 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): ]), self.evaluate(var1)[2]) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testSharing(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([1.0, 2.0], dtype=dtype) @@ -668,7 +652,6 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) ]), self.evaluate(var1)) - @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testConfig(self): opt = gradient_descent.SGD(learning_rate=1.0, momentum=0.9, nesterov=True) config = opt.get_config() From 836b99a27b930de801ecb17abf42e8aba3cac0b8 Mon Sep 17 00:00:00 2001 From: WindQAQ Date: Mon, 20 Jul 2020 22:42:27 -0700 Subject: [PATCH 0097/1017] Fix lint --- .../keras/optimizer_v2/gradient_descent_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py index 4b05b3d2908..bad83fd06cf 100644 --- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py +++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py @@ -212,9 +212,9 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.evaluate(sgd_op) # Validate updated params self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], - self.evaluate(var0)) + self.evaluate(var0)) self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], - self.evaluate(var1)) + self.evaluate(var1)) def testSparseBasicWithLearningRateDecay(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: @@ -234,9 +234,9 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.evaluate(update_op) # Validate updated params self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], - self.evaluate(var0)) + self.evaluate(var0)) self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], - self.evaluate(var1)) + self.evaluate(var1)) if context.executing_eagerly(): opt.apply_gradients(zip([grads0, grads1], [var0, var1])) @@ -402,8 +402,8 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): constant_op.constant([0, 1]), constant_op.constant([2])) grads_and_vars.append([ - (y, var0), - (constant_op.constant([3.0, 3.0], dtype=dtype), var1)]) + (y, var0), + (constant_op.constant([3.0, 3.0], dtype=dtype), var1)]) if not context.executing_eagerly(): opt_update = [] for t in range(1, 5): From 28ca3bf61ba54ef46c1906ac97d671af236c62a3 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 21 Jul 2020 15:33:08 +0000 Subject: [PATCH 0098/1017] erase and minor --- .../base_api/api_def_TensorMapReplace.pbtxt | 10 -------- tensorflow/core/ops/map_ops.cc | 2 +- .../python/kernel_tests/map_ops_test.py | 25 ++++++++++++++++--- tensorflow/python/ops/map_ops.py | 14 +++++++---- 4 files changed, 31 insertions(+), 20 deletions(-) delete mode 100644 tensorflow/core/api_def/base_api/api_def_TensorMapReplace.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_TensorMapReplace.pbtxt b/tensorflow/core/api_def/base_api/api_def_TensorMapReplace.pbtxt deleted file mode 100644 index 80c49cbbc25..00000000000 --- a/tensorflow/core/api_def/base_api/api_def_TensorMapReplace.pbtxt +++ /dev/null @@ -1,10 +0,0 @@ -op { - graph_op_name: "TensorMapReplace" - summary: "Returns a map that is the 'input_handle' after replacing the existing key value with the given value." - description: < Date: Tue, 21 Jul 2020 20:52:42 +0000 Subject: [PATCH 0099/1017] erase_grad and zerolike --- tensorflow/core/kernels/map_kernels.cc | 10 ++++++++++ tensorflow/core/kernels/map_kernels.h | 13 +++++++++++++ tensorflow/python/kernel_tests/map_ops_test.py | 3 +++ tensorflow/python/ops/map_ops.py | 2 +- 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index db91a660809..9caf8924e5d 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -12,6 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ + +#define EIGEN_USE_THREADS + #include "tensorflow/core/kernels/map_kernels.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/types.h" @@ -38,4 +41,11 @@ REGISTER_KERNEL_BUILDER(Name("TensorMapErase").Device(DEVICE_CPU), REGISTER_KERNEL_BUILDER(Name("TensorMapHasKey").Device(DEVICE_CPU), TensorMapHasKey); +#undef REGISTER_TENSOR_LIST_OPS_CPU + +#define REGISTER_TENSOR_LIST_OPS_CPU(T) + +REGISTER_UNARY_VARIANT_UNARY_OP_FUNCTION(ZEROS_LIKE_VARIANT_UNARY_OP, + DEVICE_CPU, TensorMap, + TensorMapZerosLike); } diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 5fb856bf00f..a098e428803 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/kernels/tensor_map.h" #include "tensorflow/core/framework/variant_encode_decode.h" +#include "tensorflow/core/util/tensor_ops_util.h" #include @@ -184,6 +185,18 @@ class TensorMapHasKey : public OpKernel { }; +template +Status TensorMapZerosLike(OpKernelContext* c, const TensorMap& x, TensorMap* y) { + y->element_dtype = x.element_dtype; + y->element_shape = x.element_shape; + for (const std::pair& p : x.tensors()) { + Tensor val; + TF_RETURN_IF_ERROR(ZerosLikeTensor(c, p.second, &val)); + y->tensors().emplace(p.first, val); + } + return Status::OK(); +} + } // namespace tensorflow #endif // TENSORFLOW_CORE_KERNELS_MAP_KERNELS_H_ diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index bdf06fc01d4..5d8b57e9955 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -215,5 +215,8 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): g = tape.gradient(l * 5, v) self.assertAllClose(g, 5) + g2 = tape.gradient(e * 6, v2) + self.assertAllClose(g2, 6) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index c28bc5754df..d57aae0672f 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -70,5 +70,5 @@ def InsertGrad(op, dmap): def EraseGrad(op, dmap, dval): _, k = op.inputs key_grad = None - map_grad = dmap + map_grad = tensor_map_insert(dmap, k, dval) #dmap return map_grad, key_grad From e286f3d5c5cf7b3466b4b1053f711d4ae754f7eb Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 22 Jul 2020 00:17:20 +0000 Subject: [PATCH 0100/1017] simplify --- tensorflow/python/kernel_tests/map_ops_test.py | 18 ------------------ tensorflow/python/ops/map_ops.py | 7 ------- 2 files changed, 25 deletions(-) diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index bdf06fc01d4..a80bab228d6 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -197,23 +197,5 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllClose(g2, array_ops.zeros_like(v)) self.assertAllClose(g3, 7) - def testEraseGrad(self): - with backprop.GradientTape(persistent=True) as tape: - m = map_ops.empty_tensor_map() - k = constant_op.constant(1.0) - v = constant_op.constant(2.0) - tape.watch(v) - k2 = constant_op.constant(12.0) - v2 = constant_op.constant(22.0) - tape.watch(v2) - m = map_ops.tensor_map_insert(m, k, v) - m = map_ops.tensor_map_insert(m, k2, v2) - m, e = map_ops.tensor_map_erase(m, k2, v2.dtype) - l = map_ops.tensor_map_lookup(m, k, v.dtype) - self.assertAllClose(l, v) - self.assertAllClose(e, v2) - g = tape.gradient(l * 5, v) - self.assertAllClose(g, 5) - if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index c28bc5754df..3d37247988c 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -65,10 +65,3 @@ def InsertGrad(op, dmap): lambda: tensor_map_erase(dmap, k, v.dtype)[0], lambda: dmap) return map_grad, key_grad, value_grad - -@ops.RegisterGradient("TensorMapErase") -def EraseGrad(op, dmap, dval): - _, k = op.inputs - key_grad = None - map_grad = dmap - return map_grad, key_grad From 1d8bfdbe149be2492c7a85d90f7311052bb1f9cf Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Wed, 22 Jul 2020 18:22:58 +0000 Subject: [PATCH 0101/1017] finished implementation of histogram_summary --- tensorflow/c/kernels/BUILD | 11 -- tensorflow/c/kernels/histogram_summary_op.cc | 154 +++++++++++++++++++ tensorflow/c/kernels/ops/histogram.cc | 55 +++++++ 3 files changed, 209 insertions(+), 11 deletions(-) create mode 100644 tensorflow/c/kernels/histogram_summary_op.cc create mode 100644 tensorflow/c/kernels/ops/histogram.cc diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index ee283d06169..9199559e89e 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -67,17 +67,6 @@ tf_gen_op_libs( ], ) -tf_cc_test( - name = "histogram_summary_op_test", - srcs = ["histogram_summary_op_test.cc"], - deps = [ - ":histogram_op_lib", - ":histogram_summary_op", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib" - ], -) - # Changes to the Android srcs here should be replicated in # tensorflow/contrib/makefile/tf_op_files.txt. # diff --git a/tensorflow/c/kernels/histogram_summary_op.cc b/tensorflow/c/kernels/histogram_summary_op.cc new file mode 100644 index 00000000000..3cfdf738c48 --- /dev/null +++ b/tensorflow/c/kernels/histogram_summary_op.cc @@ -0,0 +1,154 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/c/kernels.h" +#include "tensorflow/c/ops.h" +#include "tensorflow/c/tf_tensor.h" +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/selective_registration.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/lib/histogram/histogram.h" +#include "tensorflow/core/framework/summary.pb.h" +#include "tensorflow/core/platform/protobuf.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/types.h" + +// Struct that stores the status and TF_Tensor inputs to the opkernel. +// Used to delete tensor and status in its destructor upon kernel return. +typedef struct Params{ + TF_Tensor* tags; + TF_Tensor* values; + TF_Status* status; + Params(TF_OpKernelContext* ctx) { + status = TF_NewStatus(); + TF_GetInput(ctx, 0, &tags, status); + if (TF_GetCode(status) == TF_OK){ + TF_GetInput(ctx, 1, &values, status); + } + }; + ~Params(){ + TF_DeleteStatus(status); + TF_DeleteTensor(tags); + TF_DeleteTensor(values); + } +}; + +// dummy functions used for kernel registration +static void* HistogramSummaryOp_Create(TF_OpKernelConstruction* ctx) { + return nullptr; +} + +static void HistogramSummaryOp_Delete(void* kernel) { + return; +} + +template +static void HistogramSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { + Params params(ctx); + if (TF_GetCode(params.status) != TF_OK){ + TF_OpKernelContext_Failure(ctx, params.status); + return; + } + if (TF_NumDims(params.tags) != 0) { + std::ostringstream err; + err << "tags must be scalar"; + TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); + TF_OpKernelContext_Failure(ctx, params.status); + return; + } + // Cast values to array to access elements by index + auto values_array = static_cast(TF_TensorData(params.values)); + tensorflow::histogram::Histogram histo; + for (int i = 0; i < TF_TensorElementCount(params.values); ++i) { + const double double_val = static_cast(values_array[i]); + if (Eigen::numext::isnan(double_val)) { + std::ostringstream err; + err << "Nan in summary histogram for: "; + TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); + break; + } + else if (Eigen::numext::isinf(double_val)) { + std::ostringstream err; + err << "Infinity in Histogram for: "; + TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); + break; + } + histo.Add(double_val); + } + tensorflow::Summary s; + tensorflow::Summary::Value* v = s.add_value(); + const tensorflow::tstring& tag = *(static_cast( + TF_TensorData(params.tags))); + v->set_tag(tag.data(), tag.size()); + histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */); + + // Use a new status for AllocateOutput if params.status set to + // TF_INVALID_ARGUMENT + TF_Status* allocation_status = TF_NewStatus(); + TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, + TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, + sizeof(tensorflow::tstring), allocation_status); + if (TF_GetCode(allocation_status) != TF_OK){ + TF_DeleteTensor(summary_tensor); + TF_OpKernelContext_Failure(ctx, allocation_status); + TF_DeleteStatus(allocation_status); + return; + } + tensorflow::tstring* output_tstring = reinterpret_cast( + TF_TensorData(summary_tensor)); + SerializeToTString(s, output_tstring); + TF_DeleteTensor(summary_tensor); + TF_DeleteStatus(allocation_status); +} + +template +void RegisterHistogramSummaryOpKernel() { + TF_Status* status = TF_NewStatus(); + { + auto* builder = TF_NewKernelBuilder("HistogramSummary", + tensorflow::DEVICE_CPU, + &HistogramSummaryOp_Create, + &HistogramSummaryOp_Compute, + &HistogramSummaryOp_Delete); + TF_KernelBuilder_TypeConstraint(builder, "T", + static_cast(tensorflow::DataTypeToEnum::v()), status); + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while adding type constraint"; + TF_RegisterKernelBuilder("HistogramSummary", builder, status); + CHECK_EQ(TF_OK, TF_GetCode(status)) + << "Error while registering Histogram Summmary kernel"; + } + TF_DeleteStatus(status); +} + +// A dummy static variable initialized by a lambda whose side-effect is to +// register the Histogram Summary kernel. +TF_ATTRIBUTE_UNUSED static bool IsHistogramSummaryOpKernelRegistered = []() { + if (SHOULD_REGISTER_OP_KERNEL("HistogramSummary")) { + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + } + return true; +}(); \ No newline at end of file diff --git a/tensorflow/c/kernels/ops/histogram.cc b/tensorflow/c/kernels/ops/histogram.cc new file mode 100644 index 00000000000..da60c30b583 --- /dev/null +++ b/tensorflow/c/kernels/ops/histogram.cc @@ -0,0 +1,55 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/ops.h" +#include "tensorflow/core/framework/selective_registration.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/macros.h" + +static void histogram_summary_shape_inference_fn(TF_ShapeInferenceContext* ctx, + TF_Status* status) { + TF_SetStatus(status, TF_OK, ""); + TF_ShapeHandle* result = TF_NewShapeHandle(); + // Make shape handle a scalar value (empty shape) + TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); + if (TF_GetCode(status) != TF_OK) { + std::ostringstream err; + err << "Error in setting output shape inference"; + TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); + } + TF_DeleteShapeHandle(result); +} + +void Register_HistogramSummaryOp() { + TF_Status* status = TF_NewStatus(); + + TF_OpDefinitionBuilder* op_builder = + TF_NewOpDefinitionBuilder("HistogramSummary"); + TF_OpDefinitionBuilderAddInput(op_builder, "tags: string"); + TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); + TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); + TF_OpDefinitionBuilderAddAttr(op_builder, "T: realnumbertype"); + TF_OpDefinitionBuilderSetShapeInferenceFunction(op_builder, + &histogram_summary_shape_inference_fn); + + TF_RegisterOpDefinition(op_builder, status); + CHECK_EQ(TF_GetCode(status), TF_OK) + << "HistogramSummary op registration failed: " << TF_Message(status); + TF_DeleteStatus(status); +} + +TF_ATTRIBUTE_UNUSED static bool HistogramSummaryOpRegistered = []() { + if (SHOULD_REGISTER_OP("HistogramSummary")) { + Register_HistogramSummaryOp(); + } + return true; +}(); \ No newline at end of file From abba54ad2fb4d7c2c08069eacad505f38fab2637 Mon Sep 17 00:00:00 2001 From: "ag.ramesh" Date: Wed, 22 Jul 2020 18:10:21 -0700 Subject: [PATCH 0102/1017] Modified oneDNN build so as to not include the MKL blob when building with Eigen threadpool. --- .bazelrc | 10 +++++ .../core/kernels/mkl_batch_matmul_op.cc | 40 +++++++++++-------- .../core/kernels/mkl_matmul_ops_common.h | 7 +++- tensorflow/tensorflow.bzl | 10 ++--- third_party/mkl/build_defs.bzl | 22 ++-------- third_party/mkl_dnn/BUILD | 12 ++++++ third_party/mkl_dnn/build_defs.bzl | 4 +- third_party/mkl_dnn/mkldnn.BUILD | 6 +-- third_party/mkl_dnn/mkldnn_v1.BUILD | 23 +++-------- 9 files changed, 69 insertions(+), 65 deletions(-) diff --git a/.bazelrc b/.bazelrc index 82bb0605b08..a331ae12804 100644 --- a/.bazelrc +++ b/.bazelrc @@ -164,8 +164,18 @@ build:mkl -c opt # config to build OneDNN backend with a user specified threadpool. build:mkl_threadpool --define=build_with_mkl=true --define=enable_mkl=true build:mkl_threadpool --define=tensorflow_mkldnn_contraction_kernel=0 +build:mkl_threadpool --define=build_with_mkl_dnn_v1_only=true +build:mkl_threadpool --define=build_with_mkl_opensource=true build:mkl_threadpool --define=build_with_mkldnn_threadpool=true build:mkl_threadpool -c opt + +# Config setting to build with oneDNN and without the binary blob +build:mkl_opensource_only --define=build_with_mkl=true --define=enable_mkl=true +build:mkl_opensource_only --define=tensorflow_mkldnn_contraction_kernel=0 +build:mkl_opensource_only --define=build_with_mkl_dnn_v1_only=true +build:mkl_opensource_only --define=build_with_mkl_opensource=true +build:mkl_opensource_only -c opt + # This config refers to building with CUDA available. It does not necessarily # mean that we build CUDA op kernels. build:using_cuda --define=using_cuda=true diff --git a/tensorflow/core/kernels/mkl_batch_matmul_op.cc b/tensorflow/core/kernels/mkl_batch_matmul_op.cc index 87e6002d9cb..1a5821bc5af 100644 --- a/tensorflow/core/kernels/mkl_batch_matmul_op.cc +++ b/tensorflow/core/kernels/mkl_batch_matmul_op.cc @@ -15,21 +15,26 @@ limitations under the License. // See docs in ../ops/math_ops.cc. -// This file uses MKL CBLAS batched xGEMM for acceleration of TF Batch -// Matrix-Matrix Multiplication (MatMul) operations. -// We currently register this kernel only for MKL supported data -// types (float, double, complex64, complex128). The macro INTEL_MKL is defined -// by the build system only when MKL is chosen as an option at configure stage -// and when it is undefined at build time, this file becomes an empty -// compilation unit +// This file uses both oneDNN and MKL CBLAS batched xGEMM for acceleration of +// Batch Matrix-Matrix Multiplication (MatMul) operations. +// We currently register this kernel only for oneDNN supported data +// types (float, bfloat16). This file can be built with and without the use of +// the binary MKL CBLAS calls, controlled by the macro INTEL_MKL_DNN_ONLY. +// If INTEL_MKL_DNN_ONLY is defined, only oneDNN is used. For cases not +// supported by oneDNN (ex. Batchmatmul with broadcasting) we fall back to the +// default CPU implementation. +// if INTEL_MKL_DNN_ONLY is not defined, both oneDNN and MKL CBLAS +// implementations are used. This is only temporary, once we are able handle all +// cases with oneDNN, CBLAS calls will be removed. #define EIGEN_USE_THREADS #if defined(INTEL_MKL) #include +#if !defined(INTEL_MKL_DNN_ONLY) #include "mkl_cblas.h" -#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" +#endif // INTEL_MKL_DNN_ONLY #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" @@ -44,6 +49,7 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/matmul_bcast.h" #include "tensorflow/core/util/mkl_util.h" +#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" namespace tensorflow { @@ -105,14 +111,14 @@ class BatchMatMulMkl : public OpKernel { "In[0] and In[1] must have compatible batch dimensions: ", lhs.shape().DebugString(), " vs. ", rhs.shape().DebugString())); -#ifdef ENABLE_MKLDNN_THREADPOOL +#if defined(INTEL_MKL_DNN_ONLY) if (bcast.IsBroadcastingRequired()) { // Calling Eigen Kernel for broadcasting case and return. Eigen does // not have BF16 support, so we have to fail graciously in that case. eigen_batch_mm_v2_.Compute(ctx); return; } -#endif // ENABLE_MKLDNN_THREADPOOL +#endif // INTEL_MKL_DNN_ONLY TensorShape out_shape = bcast.output_batch_shape(); auto batch_size = bcast.output_batch_size(); @@ -158,11 +164,11 @@ class BatchMatMulMkl : public OpKernel { std::vector ldc_array(batch_size, N); std::vector group_size(1, batch_size); - bool threadpool_enabled = false; -#ifdef ENABLE_MKLDNN_THREADPOOL - threadpool_enabled = true; -#endif // ENABLE_MKLDNN_THREADPOOL - if (std::is_same::value || threadpool_enabled) { + bool bcast_not_supported = false; +#if defined(INTEL_MKL_DNN_ONLY) + bcast_not_supported = true; +#endif // INTEL_MKL_DNN_ONLY + if (std::is_same::value || bcast_not_supported) { // DNNL bfloat16 API requires a, b, and c as pointers to tensors // represented as flat-byte array. const Scalar* a = nullptr; @@ -227,7 +233,7 @@ class BatchMatMulMkl : public OpKernel { const std::vector& ldb_Array, float** C_Array, const std::vector& ldc_Array, const MKL_INT group_count, const std::vector& group_size, OpKernelContext* ctx) { -#ifndef ENABLE_MKLDNN_THREADPOOL +#if !defined(INTEL_MKL_DNN_ONLY) std::vector TransA_Array( group_size[0], TransA ? CblasTrans : CblasNoTrans); std::vector TransB_Array( @@ -249,7 +255,7 @@ class BatchMatMulMkl : public OpKernel { dnnl_gemm_batch(TransA_Array, TransB_Array, M_Array, N_Array, K_Array, alpha_Array, *A_Array, *B_Array, beta_Array, *C_Array, group_count, group_size, ctx); -#endif // !ENABLE_MKLDNN_THREADPOOL +#endif // INTEL_MKL_DNN_ONLY } // BatchMatMul BFloat16 support only exists in DNNL 1.2 onwards. #if defined(ENABLE_MKLDNN_V1) && defined(ENABLE_INTEL_MKL_BFLOAT16) diff --git a/tensorflow/core/kernels/mkl_matmul_ops_common.h b/tensorflow/core/kernels/mkl_matmul_ops_common.h index d7af614ad04..f8242d06fa6 100644 --- a/tensorflow/core/kernels/mkl_matmul_ops_common.h +++ b/tensorflow/core/kernels/mkl_matmul_ops_common.h @@ -35,7 +35,12 @@ using mkldnn::stream; namespace tensorflow { typedef Eigen::ThreadPoolDevice CPUDevice; - +#ifdef INTEL_MKL_DNN_ONLY +// Temporarily copying some definitions from mkl_cblas.h so the same code can +// be used when calling oneDNN or CBLAS batchmatmul in mkl_batch_matmul_op.cc. +typedef enum { CblasRowMajor, CblasColumnMajor } CBLAS_LAYOUT; +#define MKL_INT int +#endif // This structure aggregates multiple inputs to MklDnnMatMul* methods. struct MklDnnMatMulFwdParams { memory::dims src_dims; diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 1bf4b24559d..349c1e1532b 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -47,7 +47,7 @@ load( load( "//third_party/mkl_dnn:build_defs.bzl", "if_mkl_open_source_only", - "if_mkl_v1_open_source_only", + "if_mkl_v1", "if_mkldnn_threadpool", ) load( @@ -327,12 +327,8 @@ def tf_copts( if_tensorrt(["-DGOOGLE_TENSORRT=1"]) + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML"]) + if_mkl_open_source_only(["-DINTEL_MKL_DNN_ONLY"]) + - if_mkl_v1_open_source_only(["-DENABLE_MKLDNN_V1", "-DENABLE_INTEL_MKL_BFLOAT16"]) + - if_mkldnn_threadpool([ - "-DENABLE_MKLDNN_THREADPOOL", - "-DENABLE_MKLDNN_V1", - "-DINTEL_MKL_DNN_ONLY", - ]) + + if_mkl_v1(["-DENABLE_MKLDNN_V1", "-DENABLE_INTEL_MKL_BFLOAT16"]) + + if_mkldnn_threadpool(["-DENABLE_MKLDNN_THREADPOOL", "-DINTEL_MKL_DNN_ONLY"]) + if_enable_mkl(["-DENABLE_MKL"]) + if_ngraph(["-DINTEL_NGRAPH=1"]) + if_android_arm(["-mfpu=neon"]) + diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl index bd0686523bc..c1ab9f29686 100644 --- a/third_party/mkl/build_defs.bzl +++ b/third_party/mkl/build_defs.bzl @@ -41,24 +41,9 @@ def if_mkl_ml(if_true, if_false = []): a select evaluating to either if_true or if_false as appropriate. """ return select({ - "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_dnn_only": if_false, - "@org_tensorflow//third_party/mkl:build_with_mkl": if_true, - "//conditions:default": if_false, - }) - -def if_mkl_ml_only(if_true, if_false = []): - """Shorthand for select()'ing on whether we're building with MKL-ML only. - - Args: - if_true: expression to evaluate if building with MKL-ML only. - if_false: expression to evaluate if building without MKL, or with MKL-DNN. - - Returns: - a select evaluating to either if_true or if_false as appropriate. - """ - return select({ - "@org_tensorflow//third_party/mkl:build_with_mkl_ml_only": if_true, - "//conditions:default": if_false, + "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_opensource": if_false, + "@org_tensorflow//third_party/mkl_dnn:build_with_mkldnn_threadpool": if_false, + "//conditions:default": if_true, }) def if_mkl_lnx_x64(if_true, if_false = []): @@ -108,6 +93,7 @@ def mkl_deps(): "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_dnn_only": ["@mkl_dnn"], "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_dnn_v1_only": ["@mkl_dnn_v1//:mkl_dnn"], "@org_tensorflow//third_party/mkl_dnn:build_with_mkldnn_threadpool": ["@mkl_dnn_v1//:mkl_dnn"], + "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_opensource": ["@mkl_dnn_v1//:mkl_dnn"], "@org_tensorflow//third_party/mkl:build_with_mkl_ml_only": ["@org_tensorflow//third_party/mkl:intel_binary_blob"], "@org_tensorflow//third_party/mkl:build_with_mkl": [ "@org_tensorflow//third_party/mkl:intel_binary_blob", diff --git a/third_party/mkl_dnn/BUILD b/third_party/mkl_dnn/BUILD index fe558322916..c3059a3dc5c 100644 --- a/third_party/mkl_dnn/BUILD +++ b/third_party/mkl_dnn/BUILD @@ -18,6 +18,16 @@ config_setting( visibility = ["//visibility:public"], ) +config_setting( + name = "build_with_mkl_opensource", + define_values = { + "build_with_mkl": "true", + "build_with_mkl_dnn_v1_only": "true", + "build_with_mkl_opensource": "true", + }, + visibility = ["//visibility:public"], +) + config_setting( name = "build_with_mkl_dnn_v1_only", define_values = { @@ -31,6 +41,8 @@ config_setting( name = "build_with_mkldnn_threadpool", define_values = { "build_with_mkl": "true", + "build_with_mkl_dnn_v1_only": "true", + "build_with_mkl_opensource": "true", "build_with_mkldnn_threadpool": "true", }, visibility = ["//visibility:public"], diff --git a/third_party/mkl_dnn/build_defs.bzl b/third_party/mkl_dnn/build_defs.bzl index bd3b4b94f29..6a3e4f827ce 100644 --- a/third_party/mkl_dnn/build_defs.bzl +++ b/third_party/mkl_dnn/build_defs.bzl @@ -10,11 +10,11 @@ def if_mkl_open_source_only(if_true, if_false = []): """ return select({ - "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_dnn_only": if_true, + "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_opensource": if_true, "//conditions:default": if_false, }) -def if_mkl_v1_open_source_only(if_true, if_false = []): +def if_mkl_v1(if_true, if_false = []): """Returns `if_true` if MKL-DNN v1.x is used. Shorthand for select()'ing on whether we're building with diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD index 71dde75e2e0..5279043ad29 100644 --- a/third_party/mkl_dnn/mkldnn.BUILD +++ b/third_party/mkl_dnn/mkldnn.BUILD @@ -3,7 +3,7 @@ exports_files(["LICENSE"]) load( "@org_tensorflow//third_party/mkl_dnn:build_defs.bzl", "if_mkl_open_source_only", - "if_mkl_v1_open_source_only", + "if_mkl_v1", ) load( "@org_tensorflow//third_party:common.bzl", @@ -60,7 +60,7 @@ cc_library( "src/cpu/**/*.cpp", "src/cpu/**/*.hpp", "src/cpu/xbyak/*.h", - ]) + if_mkl_v1_open_source_only([ + ]) + if_mkl_v1([ ":mkldnn_config_h", ]) + [":mkldnn_version_h"], hdrs = glob(["include/*"]), @@ -71,7 +71,7 @@ cc_library( ] + if_mkl_open_source_only([ "-UUSE_MKL", "-UUSE_CBLAS", - ]) + if_mkl_v1_open_source_only([ + ]) + if_mkl_v1([ "-UUSE_MKL", "-UUSE_CBLAS", ]) + select({ diff --git a/third_party/mkl_dnn/mkldnn_v1.BUILD b/third_party/mkl_dnn/mkldnn_v1.BUILD index 7bdec138b99..438aa8dc03d 100644 --- a/third_party/mkl_dnn/mkldnn_v1.BUILD +++ b/third_party/mkl_dnn/mkldnn_v1.BUILD @@ -3,7 +3,7 @@ exports_files(["LICENSE"]) load( "@org_tensorflow//third_party/mkl_dnn:build_defs.bzl", "if_mkl_open_source_only", - "if_mkl_v1_open_source_only", + "if_mkl_v1", "if_mkldnn_threadpool", ) load( @@ -85,7 +85,7 @@ cc_library( ] + if_mkl_open_source_only([ "-UUSE_MKL", "-UUSE_CBLAS", - ]) + if_mkl_v1_open_source_only([ + ]) + if_mkl_v1([ "-UUSE_MKL", "-UUSE_CBLAS", ]) + if_mkldnn_threadpool([ @@ -109,21 +109,10 @@ cc_library( "src/cpu/xbyak", ], visibility = ["//visibility:public"], - deps = select({ - "@org_tensorflow//tensorflow:linux_x86_64": [ - "@mkl_linux//:mkl_headers", - "@mkl_linux//:mkl_libs_linux", - ], - "@org_tensorflow//tensorflow:macos": [ - "@mkl_darwin//:mkl_headers", - "@mkl_darwin//:mkl_libs_darwin", - ], - "@org_tensorflow//tensorflow:windows": [ - "@mkl_windows//:mkl_headers", - "@mkl_windows//:mkl_libs_windows", - ], - "//conditions:default": [], - }), + deps = if_mkl_open_source_only( + [], + ["@org_tensorflow//third_party/mkl:intel_binary_blob"], + ), ) cc_library( From 6d9afb6bf1710fea0122036fa7fd9704adadac03 Mon Sep 17 00:00:00 2001 From: "ag.ramesh" Date: Wed, 22 Jul 2020 23:05:10 -0700 Subject: [PATCH 0103/1017] Cleanup of oneDNN build files and removing obsolete code. --- tensorflow/tensorflow.bzl | 2 +- third_party/mkl/BUILD | 9 --------- third_party/mkl/build_defs.bzl | 4 ---- third_party/mkl_dnn/mkldnn_v1.BUILD | 10 ++++++++-- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 349c1e1532b..278bf1abfef 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -328,7 +328,7 @@ def tf_copts( if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML"]) + if_mkl_open_source_only(["-DINTEL_MKL_DNN_ONLY"]) + if_mkl_v1(["-DENABLE_MKLDNN_V1", "-DENABLE_INTEL_MKL_BFLOAT16"]) + - if_mkldnn_threadpool(["-DENABLE_MKLDNN_THREADPOOL", "-DINTEL_MKL_DNN_ONLY"]) + + if_mkldnn_threadpool(["-DENABLE_MKLDNN_THREADPOOL"]) + if_enable_mkl(["-DENABLE_MKL"]) + if_ngraph(["-DINTEL_NGRAPH=1"]) + if_android_arm(["-mfpu=neon"]) + diff --git a/third_party/mkl/BUILD b/third_party/mkl/BUILD index bbbec855ab7..470b3d50ea5 100644 --- a/third_party/mkl/BUILD +++ b/third_party/mkl/BUILD @@ -10,15 +10,6 @@ config_setting( visibility = ["//visibility:public"], ) -config_setting( - name = "build_with_mkl_ml_only", - define_values = { - "build_with_mkl": "true", - "build_with_mkl_ml_only": "true", - }, - visibility = ["//visibility:public"], -) - config_setting( name = "build_with_mkl_lnx_x64", define_values = { diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl index c1ab9f29686..7708aa387d9 100644 --- a/third_party/mkl/build_defs.bzl +++ b/third_party/mkl/build_defs.bzl @@ -42,7 +42,6 @@ def if_mkl_ml(if_true, if_false = []): """ return select({ "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_opensource": if_false, - "@org_tensorflow//third_party/mkl_dnn:build_with_mkldnn_threadpool": if_false, "//conditions:default": if_true, }) @@ -92,9 +91,6 @@ def mkl_deps(): return select({ "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_dnn_only": ["@mkl_dnn"], "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_dnn_v1_only": ["@mkl_dnn_v1//:mkl_dnn"], - "@org_tensorflow//third_party/mkl_dnn:build_with_mkldnn_threadpool": ["@mkl_dnn_v1//:mkl_dnn"], - "@org_tensorflow//third_party/mkl_dnn:build_with_mkl_opensource": ["@mkl_dnn_v1//:mkl_dnn"], - "@org_tensorflow//third_party/mkl:build_with_mkl_ml_only": ["@org_tensorflow//third_party/mkl:intel_binary_blob"], "@org_tensorflow//third_party/mkl:build_with_mkl": [ "@org_tensorflow//third_party/mkl:intel_binary_blob", "@mkl_dnn", diff --git a/third_party/mkl_dnn/mkldnn_v1.BUILD b/third_party/mkl_dnn/mkldnn_v1.BUILD index 438aa8dc03d..adbf1161781 100644 --- a/third_party/mkl_dnn/mkldnn_v1.BUILD +++ b/third_party/mkl_dnn/mkldnn_v1.BUILD @@ -6,6 +6,12 @@ load( "if_mkl_v1", "if_mkldnn_threadpool", ) + +load( + "@org_tensorflow//third_party/mkl:build_defs.bzl", + "if_mkl_ml", +) + load( "@org_tensorflow//third_party:common.bzl", "template_rule", @@ -109,9 +115,9 @@ cc_library( "src/cpu/xbyak", ], visibility = ["//visibility:public"], - deps = if_mkl_open_source_only( - [], + deps = if_mkl_ml( ["@org_tensorflow//third_party/mkl:intel_binary_blob"], + [], ), ) From 4fcab24e7ea71de3da16ec919dafb719e1764d7c Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 23 Jul 2020 18:59:05 +0000 Subject: [PATCH 0104/1017] removed unnecessary includes --- tensorflow/c/kernels/BUILD | 1 - tensorflow/c/kernels/histogram_summary_op.cc | 10 ++-------- tensorflow/c/kernels/ops/histogram.cc | 3 +-- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 9199559e89e..813e788fd45 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -53,7 +53,6 @@ tf_kernel_library( prefix = "histogram_summary_op", deps = [ "//tensorflow/c:kernels", - "//tensorflow/c:ops", "//tensorflow/c:tf_tensor", "//tensorflow/core:framework", ], diff --git a/tensorflow/c/kernels/histogram_summary_op.cc b/tensorflow/c/kernels/histogram_summary_op.cc index 3cfdf738c48..0ffa70a0920 100644 --- a/tensorflow/c/kernels/histogram_summary_op.cc +++ b/tensorflow/c/kernels/histogram_summary_op.cc @@ -1,4 +1,4 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -13,21 +13,15 @@ limitations under the License. #include #include "tensorflow/c/kernels.h" -#include "tensorflow/c/ops.h" #include "tensorflow/c/tf_tensor.h" -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/selective_registration.h" -#include "tensorflow/core/platform/macros.h" #include "tensorflow/core/lib/histogram/histogram.h" #include "tensorflow/core/framework/summary.pb.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/types.h" // Struct that stores the status and TF_Tensor inputs to the opkernel. // Used to delete tensor and status in its destructor upon kernel return. -typedef struct Params{ +typedef struct Params { TF_Tensor* tags; TF_Tensor* values; TF_Status* status; diff --git a/tensorflow/c/kernels/ops/histogram.cc b/tensorflow/c/kernels/ops/histogram.cc index da60c30b583..aa7502f1e37 100644 --- a/tensorflow/c/kernels/ops/histogram.cc +++ b/tensorflow/c/kernels/ops/histogram.cc @@ -1,4 +1,4 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -13,7 +13,6 @@ limitations under the License. #include "tensorflow/c/ops.h" #include "tensorflow/core/framework/selective_registration.h" #include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/macros.h" static void histogram_summary_shape_inference_fn(TF_ShapeInferenceContext* ctx, TF_Status* status) { From da1917bcf42261ddfac43f33b46b1b8b98f5e38a Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Thu, 23 Jul 2020 19:44:59 +0000 Subject: [PATCH 0105/1017] TensorMapBinaryAdd and testLookupAddGrad --- tensorflow/core/kernels/map_kernels.cc | 4 ++ tensorflow/core/kernels/map_kernels.h | 35 ++++++++++++++++++ .../python/kernel_tests/map_ops_test.py | 37 +++++++++++++++++++ tensorflow/python/ops/map_ops.py | 2 +- 4 files changed, 77 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index 9caf8924e5d..26f67c576e5 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -45,6 +45,10 @@ REGISTER_KERNEL_BUILDER(Name("TensorMapHasKey").Device(DEVICE_CPU), #define REGISTER_TENSOR_LIST_OPS_CPU(T) +REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_CPU, + TensorMap, + TensorMapBinaryAdd); + REGISTER_UNARY_VARIANT_UNARY_OP_FUNCTION(ZEROS_LIKE_VARIANT_UNARY_OP, DEVICE_CPU, TensorMap, TensorMapZerosLike); diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index a098e428803..ca616e1b50c 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -185,6 +185,41 @@ class TensorMapHasKey : public OpKernel { }; +template +Status TensorMapBinaryAdd(OpKernelContext* c, const TensorMap& a, + const TensorMap& b, TensorMap* out) { + /*if (a.element_dtype != b.element_dtype) { + return errors::InvalidArgument( + "Trying to add two maps of tensors of different dtypes. One is ", + DataTypeString(a.element_dtype), " and the other is ", + DataTypeString(b.element_dtype)); + }*/ + if (!a.element_shape.IsCompatibleWith(b.element_shape)) { + return errors::InvalidArgument( + "Trying to add two maps of tensors with incompatible element shapes. " + "One is ", + a.element_shape.DebugString(), " and the other is ", + b.element_shape.DebugString()); + } + + out->element_dtype = a.element_dtype; + TF_RETURN_IF_ERROR(a.element_shape.MergeWith(b.element_shape, &out->element_shape)); + out->tensors() = a.tensors(); + for (const std::pair& p : b.tensors()) { + absl::flat_hash_map::iterator it = out->tensors().find(p.first); + if (it != out->tensors().end()) { + Tensor out_tensor; + TF_RETURN_IF_ERROR(BinaryAddTensors(c, p.second, it->second, &out_tensor)); + it->second = out_tensor; + } + else { + out->tensors().emplace(p.first, p.second); + } + } + return Status::OK(); +} + + template Status TensorMapZerosLike(OpKernelContext* c, const TensorMap& x, TensorMap* y) { y->element_dtype = x.element_dtype; diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 5d8b57e9955..9395eb416ec 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -197,6 +197,22 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllClose(g2, array_ops.zeros_like(v)) self.assertAllClose(g3, 7) + def testLookupAddGrad(self): + with backprop.GradientTape(persistent=True) as tape: + k = constant_op.constant(1.0) + v = constant_op.constant(2.0) + k2 = constant_op.constant(12.0) + v2 = constant_op.constant(22.0) + tape.watch(v) + tape.watch(v2) + m = map_ops.empty_tensor_map() + m = map_ops.tensor_map_insert(m, k, v) + m = map_ops.tensor_map_insert(m, k2, v2) + l1 = map_ops.tensor_map_lookup(m, k, v.dtype) + l2 = map_ops.tensor_map_lookup(m, k2, v2.dtype) + g = tape.gradient(l1 + l2, [l1, l2]) + self.assertAllClose(g, [1, 1]) + def testEraseGrad(self): with backprop.GradientTape(persistent=True) as tape: m = map_ops.empty_tensor_map() @@ -218,5 +234,26 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): g2 = tape.gradient(e * 6, v2) self.assertAllClose(g2, 6) + def testStringKey(self): + with backprop.GradientTape(persistent=True) as tape: + m = map_ops.empty_tensor_map() + k = constant_op.constant("key") + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + s = map_ops.tensor_map_size(m) + self.assertAllEqual(s, 1) + l = map_ops.tensor_map_lookup(m, k, v.dtype) + self.assertAllEqual(l, v) + + m, e = map_ops.tensor_map_erase(m, k, v.dtype) + s = map_ops.tensor_map_size(m) + self.assertAllEqual(s, 0) + self.assertAllClose(e, v) + + def testVectorValue(self): + with backprop.GradientTape(persistent=True) as tape: + + + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/ops/map_ops.py b/tensorflow/python/ops/map_ops.py index d57aae0672f..05408886360 100644 --- a/tensorflow/python/ops/map_ops.py +++ b/tensorflow/python/ops/map_ops.py @@ -70,5 +70,5 @@ def InsertGrad(op, dmap): def EraseGrad(op, dmap, dval): _, k = op.inputs key_grad = None - map_grad = tensor_map_insert(dmap, k, dval) #dmap + map_grad = tensor_map_insert(dmap, k, dval) return map_grad, key_grad From f6eb8538d0addb8873f6319d5a95f70d27a028fe Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 23 Jul 2020 20:51:41 +0000 Subject: [PATCH 0106/1017] changed python wrapper to use new histogram summary --- tensorflow/c/kernels/BUILD | 6 +++- tensorflow/c/kernels/ops/histogram.cc | 2 +- tensorflow/core/BUILD | 2 ++ tensorflow/core/kernels/summary_op.cc | 44 --------------------------- tensorflow/core/ops/logging_ops.cc | 7 ----- tensorflow/python/BUILD | 4 +++ 6 files changed, 12 insertions(+), 53 deletions(-) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 813e788fd45..6f6b1f155bb 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -77,11 +77,15 @@ filegroup( name = "android_all_op_kernels", srcs = [ "bitcast_op.cc", + "histogram_summary_op.cc", ], ) # LINT.ThenChange(//tensorflow/contrib/makefile/tf_op_files.txt) filegroup( name = "android_all_ops", - srcs = ["ops/bitcast.cc"], + srcs = [ + "ops/bitcast.cc", + "ops/histogram.cc", + ], ) diff --git a/tensorflow/c/kernels/ops/histogram.cc b/tensorflow/c/kernels/ops/histogram.cc index aa7502f1e37..f2d60b75f34 100644 --- a/tensorflow/c/kernels/ops/histogram.cc +++ b/tensorflow/c/kernels/ops/histogram.cc @@ -33,7 +33,7 @@ void Register_HistogramSummaryOp() { TF_OpDefinitionBuilder* op_builder = TF_NewOpDefinitionBuilder("HistogramSummary"); - TF_OpDefinitionBuilderAddInput(op_builder, "tags: string"); + TF_OpDefinitionBuilderAddInput(op_builder, "tag: string"); TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); TF_OpDefinitionBuilderAddAttr(op_builder, "T: realnumbertype"); diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index d1909ea1bac..d11aaef33bf 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -892,6 +892,7 @@ cc_library( ":user_ops_op_lib", ":word2vec_ops", "//tensorflow/c/kernels:bitcast_op_lib", + "//tensorflow/c/kernels:histogram_op_lib", "//tensorflow/compiler/mlir/tensorflow:mlir_passthrough_op", ] + if_chromiumos( [], @@ -998,6 +999,7 @@ cc_library( name = "all_kernels_impl", visibility = [":__subpackages__"], deps = [ + "//tensorflow/c/kernels:histogram_summary_op", "//tensorflow/c/kernels:bitcast_op", "//tensorflow/core/kernels:array", "//tensorflow/core/kernels:audio", diff --git a/tensorflow/core/kernels/summary_op.cc b/tensorflow/core/kernels/summary_op.cc index b35759cb7bc..96edec89452 100644 --- a/tensorflow/core/kernels/summary_op.cc +++ b/tensorflow/core/kernels/summary_op.cc @@ -72,54 +72,10 @@ class SummaryScalarOp : public OpKernel { } }; -// template -// class SummaryHistoOp : public OpKernel { -// public: -// // SummaryHistoOp could be extended to take a list of custom bucket -// // boundaries as an option. -// explicit SummaryHistoOp(OpKernelConstruction* context) : OpKernel(context) {} - -// void Compute(OpKernelContext* c) override { -// const Tensor& tags = c->input(0); -// const Tensor& values = c->input(1); -// const auto flat = values.flat(); -// OP_REQUIRES(c, TensorShapeUtils::IsScalar(tags.shape()), -// errors::InvalidArgument("tags must be scalar")); -// // Build histogram of values in "values" tensor -// histogram::Histogram histo; -// for (int64 i = 0; i < flat.size(); i++) { -// const double double_val = static_cast(flat(i)); -// if (Eigen::numext::isnan(double_val)) { -// c->SetStatus( -// errors::InvalidArgument("Nan in summary histogram for: ", name())); -// break; -// } else if (Eigen::numext::isinf(double_val)) { -// c->SetStatus(errors::InvalidArgument( -// "Infinity in summary histogram for: ", name())); -// break; -// } -// histo.Add(double_val); -// } - -// Summary s; -// Summary::Value* v = s.add_value(); -// const tstring& tags0 = tags.scalar()(); -// v->set_tag(tags0.data(), tags0.size()); -// histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */); - -// Tensor* summary_tensor = nullptr; -// OP_REQUIRES_OK(c, c->allocate_output(0, TensorShape({}), &summary_tensor)); -// CHECK(SerializeToTString(s, &summary_tensor->scalar()())); -// } -// }; - #define REGISTER(T) \ REGISTER_KERNEL_BUILDER( \ Name("ScalarSummary").Device(DEVICE_CPU).TypeConstraint("T"), \ SummaryScalarOp); - // REGISTER_KERNEL_BUILDER( \ - // Name("HistogramSummary").Device(DEVICE_CPU).TypeConstraint("T"), \ - // SummaryHistoOp); TF_CALL_REAL_NUMBER_TYPES(REGISTER) #undef REGISTER diff --git a/tensorflow/core/ops/logging_ops.cc b/tensorflow/core/ops/logging_ops.cc index e620da86c68..96ef71931dd 100644 --- a/tensorflow/core/ops/logging_ops.cc +++ b/tensorflow/core/ops/logging_ops.cc @@ -94,13 +94,6 @@ REGISTER_OP("ScalarSummary") .Attr("T: realnumbertype") .SetShapeFn(shape_inference::ScalarShape); -// REGISTER_OP("HistogramSummary") -// .Input("tag: string") -// .Input("values: T") -// .Output("summary: string") -// .Attr("T: realnumbertype = DT_FLOAT") -// .SetShapeFn(shape_inference::ScalarShape); - REGISTER_OP("ImageSummary") .Input("tag: string") .Input("tensor: T") diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index c0b982be3c9..3f269922ec1 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -2907,6 +2907,10 @@ tf_gen_op_wrapper_private_py( "//learning/brain/python/ops:__pkg__", "//tensorflow/python/kernel_tests:__pkg__", ], + deps = [ + "//tensorflow/c/kernels:histogram_op_lib", + "//tensorflow/core:logging_ops_op_lib", + ], ) tf_gen_op_wrapper_private_py( From 7dbe3fb4ca410264a6db6c0a0345859fd7a2eb11 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 23 Jul 2020 21:05:41 +0000 Subject: [PATCH 0107/1017] switched to use scalar shape function --- tensorflow/c/kernels/ops/summary.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc index 98b8b743fa1..20a935aeb0a 100644 --- a/tensorflow/c/kernels/ops/summary.cc +++ b/tensorflow/c/kernels/ops/summary.cc @@ -20,8 +20,7 @@ limitations under the License. static void scalar_summary_shape_inference_fn(TF_ShapeInferenceContext* ctx, TF_Status* status) { TF_SetStatus(status, TF_OK, ""); - TF_ShapeHandle* result = TF_NewShapeHandle(); - // Make shape handle a scalar value (empty shape) + TF_ShapeHandle* result = TF_ShapeInferenceContextScalar(ctx); TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); if (TF_GetCode(status) != TF_OK) { TF_SetStatus(status, TF_INVALID_ARGUMENT, From ba6d661ba81671aba5c7ce30e000b40d71f02b58 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 23 Jul 2020 22:08:33 +0000 Subject: [PATCH 0108/1017] clean up --- tensorflow/c/kernels/ops/summary.cc | 4 ---- tensorflow/c/kernels/summary_op.cc | 8 +++----- tensorflow/c/kernels/tensor_shape_utils_test.cc | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc index 20a935aeb0a..a9c3b697f14 100644 --- a/tensorflow/c/kernels/ops/summary.cc +++ b/tensorflow/c/kernels/ops/summary.cc @@ -22,10 +22,6 @@ static void scalar_summary_shape_inference_fn(TF_ShapeInferenceContext* ctx, TF_SetStatus(status, TF_OK, ""); TF_ShapeHandle* result = TF_ShapeInferenceContextScalar(ctx); TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); - if (TF_GetCode(status) != TF_OK) { - TF_SetStatus(status, TF_INVALID_ARGUMENT, - "Error in setting output shape inference"); - } TF_DeleteShapeHandle(result); } diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc index 6b611be7e4f..5d98d0c0477 100644 --- a/tensorflow/c/kernels/summary_op.cc +++ b/tensorflow/c/kernels/summary_op.cc @@ -47,7 +47,6 @@ struct Params { } }; - // dummy functions used for kernel registration void* ScalarSummaryOp_Create(TF_OpKernelConstruction* ctx) { return nullptr; @@ -66,7 +65,7 @@ tensorflow::string SingleTag(TF_Tensor* tags); template void ScalarSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { Params params(ctx); - if (TF_GetCode(params.status) != TF_OK){ + if (TF_GetCode(params.status) != TF_OK) { TF_OpKernelContext_Failure(ctx, params.status); return; } @@ -167,6 +166,5 @@ TF_ATTRIBUTE_UNUSED bool IsScalarSummaryOpKernelRegistered = []() { RegisterScalarSummaryOpKernel(); } return true; -}(); - -} // namespace +}(); +} // namespace diff --git a/tensorflow/c/kernels/tensor_shape_utils_test.cc b/tensorflow/c/kernels/tensor_shape_utils_test.cc index a08e4a67e3e..23e5940dc7b 100644 --- a/tensorflow/c/kernels/tensor_shape_utils_test.cc +++ b/tensorflow/c/kernels/tensor_shape_utils_test.cc @@ -29,7 +29,7 @@ namespace { // once out of scope. struct TF_TensorWrapper { TF_Tensor* tf_tensor; - TF_TensorWrapper(TF_Tensor* tensor){ + TF_TensorWrapper(TF_Tensor* tensor) { tf_tensor = tensor; } ~TF_TensorWrapper() { From 284f16a59bf03dac279967a1cd2c0d2b89edc9a0 Mon Sep 17 00:00:00 2001 From: Vignesh Kothapalli Date: Fri, 24 Jul 2020 04:02:34 +0530 Subject: [PATCH 0109/1017] added missing docstrings in dataset_utils module --- .../keras/preprocessing/dataset_utils.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/keras/preprocessing/dataset_utils.py b/tensorflow/python/keras/preprocessing/dataset_utils.py index 1c9d283c2f1..055f37e5ca2 100644 --- a/tensorflow/python/keras/preprocessing/dataset_utils.py +++ b/tensorflow/python/keras/preprocessing/dataset_utils.py @@ -189,6 +189,19 @@ def get_training_or_validation_split(samples, labels, validation_split, subset): def labels_to_dataset(labels, label_mode, num_classes): + """Create a tf.data.Dataset from the list/tuple of labels. + + Args: + labels: list/tuple of labels to be converted into a tf.data.Dataset. + label_mode: + - 'binary' indicates that the labels (there can be only 2) + are encoded as `float32` scalars with values 0 or 1 + (e.g. for `binary_crossentropy`). + - 'categorical' means that the labels are + mapped into a categorical vector. + (e.g. for `categorical_crossentropy` loss). + num_classes: number of classes of labels. + """ label_ds = dataset_ops.Dataset.from_tensor_slices(labels) if label_mode == 'binary': label_ds = label_ds.map( @@ -199,7 +212,17 @@ def labels_to_dataset(labels, label_mode, num_classes): def check_validation_split_arg(validation_split, subset, shuffle, seed): - """Raise errors in case of invalid argument values.""" + """Raise errors in case of invalid argument values. + + Args: + shuffle: Whether to shuffle the data. Default: True. + If set to False, sorts the data in alphanumeric order. + seed: Optional random seed for shuffling and transformations. + validation_split: Optional float between 0 and 1, + fraction of data to reserve for validation. + subset: One of "training" or "validation". + Only used if `validation_split` is set. + """ if validation_split and not 0 < validation_split < 1: raise ValueError( '`validation_split` must be between 0 and 1, received: %s' % From edb88b4be341f74920a8438e2055c29877141463 Mon Sep 17 00:00:00 2001 From: Vignesh Kothapalli Date: Fri, 24 Jul 2020 04:07:19 +0530 Subject: [PATCH 0110/1017] fixed docstring for check_validation_split_arg --- tensorflow/python/keras/preprocessing/dataset_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/python/keras/preprocessing/dataset_utils.py b/tensorflow/python/keras/preprocessing/dataset_utils.py index 055f37e5ca2..09e6485a492 100644 --- a/tensorflow/python/keras/preprocessing/dataset_utils.py +++ b/tensorflow/python/keras/preprocessing/dataset_utils.py @@ -215,8 +215,7 @@ def check_validation_split_arg(validation_split, subset, shuffle, seed): """Raise errors in case of invalid argument values. Args: - shuffle: Whether to shuffle the data. Default: True. - If set to False, sorts the data in alphanumeric order. + shuffle: Whether to shuffle the data. Either True or False. seed: Optional random seed for shuffling and transformations. validation_split: Optional float between 0 and 1, fraction of data to reserve for validation. From 1a3a8deba522de7afa4767edb464efc08b4dcc8a Mon Sep 17 00:00:00 2001 From: Vignesh Kothapalli Date: Fri, 24 Jul 2020 04:10:08 +0530 Subject: [PATCH 0111/1017] removed 'optional' string from docstrings --- tensorflow/python/keras/preprocessing/dataset_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/preprocessing/dataset_utils.py b/tensorflow/python/keras/preprocessing/dataset_utils.py index 09e6485a492..688115862c1 100644 --- a/tensorflow/python/keras/preprocessing/dataset_utils.py +++ b/tensorflow/python/keras/preprocessing/dataset_utils.py @@ -216,9 +216,9 @@ def check_validation_split_arg(validation_split, subset, shuffle, seed): Args: shuffle: Whether to shuffle the data. Either True or False. - seed: Optional random seed for shuffling and transformations. - validation_split: Optional float between 0 and 1, - fraction of data to reserve for validation. + seed: random seed for shuffling and transformations. + validation_split: float between 0 and 1, + fraction of data to reserve for validation. subset: One of "training" or "validation". Only used if `validation_split` is set. """ From d9cdd0336fec8e8ac040cdd3eb6aac92d08d7509 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 24 Jul 2020 13:38:03 +0000 Subject: [PATCH 0112/1017] more tests --- .../python/kernel_tests/map_ops_test.py | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 9395eb416ec..5c8729475c1 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -235,6 +235,38 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllClose(g2, 6) def testStringKey(self): + + m = map_ops.empty_tensor_map() + k = constant_op.constant("key") + v = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + s = map_ops.tensor_map_size(m) + self.assertAllEqual(s, 1) + l = map_ops.tensor_map_lookup(m, k, v.dtype) + self.assertAllEqual(l, v) + + m, e = map_ops.tensor_map_erase(m, k, v.dtype) + s = map_ops.tensor_map_size(m) + self.assertAllEqual(s, 0) + self.assertAllClose(e, v) + + def testKeyTypes(self): + m = map_ops.empty_tensor_map() + k = constant_op.constant("key") + v = constant_op.constant("value") + k2 = constant_op.constant(1.0) + v2 = constant_op.constant(2.0) + m = map_ops.tensor_map_insert(m, k, v) + m = map_ops.tensor_map_insert(m, k2, v2) + l = map_ops.tensor_map_lookup(m, k, v.dtype) + #self.assertAllClose(l, v) + l2 = map_ops.tensor_map_lookup(m, k2, v2.dtype) + #self.assertAllClose(l2, v2) + m, e = map_ops.tensor_map_erase(m, k, v.dtype) + #self.assertAllClose(e, v) + + + '''def testVectorValue(self): with backprop.GradientTape(persistent=True) as tape: m = map_ops.empty_tensor_map() k = constant_op.constant("key") @@ -248,11 +280,8 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): m, e = map_ops.tensor_map_erase(m, k, v.dtype) s = map_ops.tensor_map_size(m) self.assertAllEqual(s, 0) - self.assertAllClose(e, v) + self.assertAllClose(e, v)''' - def testVectorValue(self): - with backprop.GradientTape(persistent=True) as tape: - if __name__ == '__main__': From f3375f0e267cb24146df9a59e320cc527f3ab285 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 24 Jul 2020 13:58:05 +0000 Subject: [PATCH 0113/1017] change to CPU-only test for now --- tensorflow/core/kernels/map_kernels.cc | 1 - tensorflow/python/kernel_tests/BUILD | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index db91a660809..6a691538956 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -19,7 +19,6 @@ limitations under the License. namespace tensorflow { - REGISTER_KERNEL_BUILDER(Name("EmptyTensorMap").Device(DEVICE_CPU), EmptyTensorMap); diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 55a8feeb053..7c3a6a43995 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -142,7 +142,8 @@ cuda_py_test( ], ) -cuda_py_test( +# TODO(kattian): add GPU capability and change to cuda_py_test +tf_py_test( name = "map_ops_test", size = "small", srcs = ["map_ops_test.py"], From 1007559c220863de8c0051154229a1f556852813 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 24 Jul 2020 15:50:08 +0000 Subject: [PATCH 0114/1017] added allocator attributes to allocate_temp --- tensorflow/c/kernels.cc | 10 ++++++++-- tensorflow/c/kernels.h | 7 +++---- tensorflow/c/kernels_test.cc | 15 ++++++++++++--- tensorflow/c/tf_tensor.h | 2 +- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 3ac0af13b2d..4d9b35861f9 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -277,18 +277,24 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, } TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, - int64_t* dims, int num_dims, TF_Status* status){ + int64_t* dims, int num_dims, + TF_AllocatorAttributes attributes, + TF_Status* status) { auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); TF_SetStatus(status, TF_OK, ""); tensorflow::TensorShape shape; for(int i = 0; i < num_dims; ++i){ shape.AddDim(dims[i]); } + tensorflow::AllocatorAttributes allocator_attr; + if (attributes.on_host) { + allocator_attr.set_on_host(true); + } tensorflow::Status s; tensorflow::Tensor tensor_temp; TF_Tensor* tf_tensor_temp; s = cc_ctx->allocate_temp(static_cast(dtype), shape, - &tensor_temp); + &tensor_temp, allocator_attr); if (s.ok()){ tf_tensor_temp = TF_TensorFromTensor(tensor_temp, &s); } diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index 5c8f36cde9c..658ad3ac404 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/c/tf_datatype.h" #include "tensorflow/c/tf_status.h" +#include "tensorflow/c/tf_tensor.h" // Macro to control visibility of exported symbols in the shared library (.so, // .dylib, .dll). @@ -200,10 +201,8 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, // num_dims must equal the size of array dims TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, - TF_DataType dtype, - int64_t* dims, - int num_dims, - TF_Status* status); + TF_DataType dtype, int64_t* dims, int num_dims, TF_AllocatorAttributes + alloc_attrs, TF_Status* status); #ifdef __cplusplus diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index 738c1e12c80..780cd8958d4 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -459,9 +459,12 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSizeOne) { // Allocate output TF_Status* s = TF_NewStatus(); int64_t dim = 1; + TF_AllocatorAttributes alloc_attrs; + alloc_attrs.struct_size = TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE; + alloc_attrs.on_host = 1; TF_Tensor* output = TF_AllocateTemp( /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, s); + /*num_dims=*/1, /*allocator_attributes*/ alloc_attrs, s); size_t tensor_size_bytes = TF_DataTypeSize(TF_FLOAT); EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, &dim, 1, TF_FLOAT); @@ -489,9 +492,12 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempEmpty) { TF_Status* s = TF_NewStatus(); // Allocate empty output int64_t dim = 0; + TF_AllocatorAttributes alloc_attrs; + alloc_attrs.struct_size = TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE; + alloc_attrs.on_host = 1; TF_Tensor* output = TF_AllocateTemp( /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, s); + /*num_dims=*/1, /*allocator_attributes*/ alloc_attrs, s); EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, &dim, 1, TF_FLOAT); TF_SetOutput(ctx, 0, output, s); @@ -515,9 +521,12 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSize2x3) { size_t tensor_size_bytes = 6 * TF_DataTypeSize(TF_FLOAT); // Allocate 2x3 output int64_t dim[2] = {2, 3}; + TF_AllocatorAttributes alloc_attrs; + alloc_attrs.struct_size = TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE; + alloc_attrs.on_host = 1; TF_Tensor* output = TF_AllocateTemp( /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/dim, - /*num_dims=*/2, s); + /*num_dims=*/2, /*allocator_attributes*/ alloc_attrs, s); EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, dim, 2, TF_FLOAT); diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index d7ea6c0a8ce..d8d8155e8cd 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -58,7 +58,7 @@ extern "C" { // Allocator Attributes used for tensor allocation. typedef struct TF_AllocatorAttributes { size_t struct_size; - // Set boolean to 0 for CPU allocation, else 1. + // Set boolean to 1 for CPU allocation, else 0. unsigned char on_host; } TF_AllocatorAttributes; From 461ca1e9e12272c186783cac1621aa91501d80fe Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 24 Jul 2020 16:59:13 +0000 Subject: [PATCH 0115/1017] used simple tensor and status wrapper instead of params --- tensorflow/c/kernels/histogram_summary_op.cc | 86 ++++++++++---------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/tensorflow/c/kernels/histogram_summary_op.cc b/tensorflow/c/kernels/histogram_summary_op.cc index 0ffa70a0920..5c0f7c0ee24 100644 --- a/tensorflow/c/kernels/histogram_summary_op.cc +++ b/tensorflow/c/kernels/histogram_summary_op.cc @@ -19,25 +19,24 @@ limitations under the License. #include "tensorflow/core/framework/summary.pb.h" #include "tensorflow/core/framework/types.h" -// Struct that stores the status and TF_Tensor inputs to the opkernel. -// Used to delete tensor and status in its destructor upon kernel return. -typedef struct Params { - TF_Tensor* tags; - TF_Tensor* values; - TF_Status* status; - Params(TF_OpKernelContext* ctx) { - status = TF_NewStatus(); - TF_GetInput(ctx, 0, &tags, status); - if (TF_GetCode(status) == TF_OK){ - TF_GetInput(ctx, 1, &values, status); - } - }; - ~Params(){ - TF_DeleteStatus(status); - TF_DeleteTensor(tags); - TF_DeleteTensor(values); +// Wrappers to clean up resources once the resource is out of scope. +struct Tensor_Wrapper { + TF_Tensor* t; + Tensor_Wrapper() : t(nullptr) {} + ~Tensor_Wrapper() { + TF_DeleteTensor(t); } -}; +}; + +struct Status_Wrapper { + TF_Status* s; + Status_Wrapper() { + s = TF_NewStatus(); + } + ~Status_Wrapper() { + TF_DeleteStatus(s); + } +}; // dummy functions used for kernel registration static void* HistogramSummaryOp_Create(TF_OpKernelConstruction* ctx) { @@ -50,33 +49,39 @@ static void HistogramSummaryOp_Delete(void* kernel) { template static void HistogramSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { - Params params(ctx); - if (TF_GetCode(params.status) != TF_OK){ - TF_OpKernelContext_Failure(ctx, params.status); + Tensor_Wrapper tags_wrapper; + Tensor_Wrapper values_wrapper; + Status_Wrapper status_wrapper; + TF_GetInput(ctx, 0, &tags_wrapper.t, status_wrapper.s); + if (TF_GetCode(status_wrapper.s) != TF_OK) { + TF_OpKernelContext_Failure(ctx, status_wrapper.s); return; } - if (TF_NumDims(params.tags) != 0) { - std::ostringstream err; - err << "tags must be scalar"; - TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); - TF_OpKernelContext_Failure(ctx, params.status); + TF_GetInput(ctx, 1, &values_wrapper.t, status_wrapper.s); + if (TF_GetCode(status_wrapper.s) != TF_OK) { + TF_OpKernelContext_Failure(ctx, status_wrapper.s); + return; + } + if (TF_NumDims(tags_wrapper.t) != 0) { + TF_SetStatus(status_wrapper.s, TF_INVALID_ARGUMENT, "tags must be scalar"); + TF_OpKernelContext_Failure(ctx, status_wrapper.s); return; } // Cast values to array to access elements by index - auto values_array = static_cast(TF_TensorData(params.values)); + auto values_array = static_cast(TF_TensorData(values_wrapper.t)); tensorflow::histogram::Histogram histo; - for (int i = 0; i < TF_TensorElementCount(params.values); ++i) { + for (int i = 0; i < TF_TensorElementCount(values_wrapper.t); ++i) { const double double_val = static_cast(values_array[i]); if (Eigen::numext::isnan(double_val)) { std::ostringstream err; err << "Nan in summary histogram for: "; - TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); + TF_SetStatus(status_wrapper.s, TF_INVALID_ARGUMENT, err.str().c_str()); break; } else if (Eigen::numext::isinf(double_val)) { std::ostringstream err; err << "Infinity in Histogram for: "; - TF_SetStatus(params.status, TF_INVALID_ARGUMENT, err.str().c_str()); + TF_SetStatus(status_wrapper.s, TF_INVALID_ARGUMENT, err.str().c_str()); break; } histo.Add(double_val); @@ -84,27 +89,26 @@ static void HistogramSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { tensorflow::Summary s; tensorflow::Summary::Value* v = s.add_value(); const tensorflow::tstring& tag = *(static_cast( - TF_TensorData(params.tags))); + TF_TensorData(tags_wrapper.t))); v->set_tag(tag.data(), tag.size()); histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */); - // Use a new status for AllocateOutput if params.status set to + // Use a new status for AllocateOutput if status_wrapper.s set to // TF_INVALID_ARGUMENT - TF_Status* allocation_status = TF_NewStatus(); - TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, + Status_Wrapper allocation_status_wrapper; + Tensor_Wrapper summary_tensor_wrapper; + TF_Tensor* summary_tensor= TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, - sizeof(tensorflow::tstring), allocation_status); - if (TF_GetCode(allocation_status) != TF_OK){ - TF_DeleteTensor(summary_tensor); - TF_OpKernelContext_Failure(ctx, allocation_status); - TF_DeleteStatus(allocation_status); + sizeof(tensorflow::tstring), allocation_status_wrapper.s); + summary_tensor_wrapper.t = summary_tensor; + + if (TF_GetCode(allocation_status_wrapper.s) != TF_OK){ + TF_OpKernelContext_Failure(ctx, allocation_status_wrapper.s); return; } tensorflow::tstring* output_tstring = reinterpret_cast( TF_TensorData(summary_tensor)); SerializeToTString(s, output_tstring); - TF_DeleteTensor(summary_tensor); - TF_DeleteStatus(allocation_status); } template From 2de04ca046406bd208f4f04ceaf938b520e1281f Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 24 Jul 2020 20:48:09 +0000 Subject: [PATCH 0116/1017] clean up --- tensorflow/c/kernels.cc | 6 +++--- tensorflow/c/kernels_test.cc | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 4d9b35861f9..8c2c4e67b3a 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -283,7 +283,7 @@ TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); TF_SetStatus(status, TF_OK, ""); tensorflow::TensorShape shape; - for(int i = 0; i < num_dims; ++i){ + for (int i = 0; i < num_dims; ++i) { shape.AddDim(dims[i]); } tensorflow::AllocatorAttributes allocator_attr; @@ -295,10 +295,10 @@ TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, TF_Tensor* tf_tensor_temp; s = cc_ctx->allocate_temp(static_cast(dtype), shape, &tensor_temp, allocator_attr); - if (s.ok()){ + if (s.ok()) { tf_tensor_temp = TF_TensorFromTensor(tensor_temp, &s); } - if (s.ok()){ + if (s.ok()) { ::tensorflow::Set_TF_Status_from_Status(status, s); return tf_tensor_temp; } diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index 780cd8958d4..4c925402cb7 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -456,7 +456,7 @@ REGISTER_OP("AllocateTempOp1").Output("output1: float"); TEST_F(DeviceKernelOpTest, TestAllocateTempSizeOne) { auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { - // Allocate output + // Allocate scalar TF_Tensor TF_Status* s = TF_NewStatus(); int64_t dim = 1; TF_AllocatorAttributes alloc_attrs; @@ -469,7 +469,7 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSizeOne) { EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, &dim, 1, TF_FLOAT); - // Set output to 3 + // Set TF_Tensor value to 3 float values[1] = {3.0f}; set_tensor_data(output, values, tensor_size_bytes, ctx); TF_SetOutput(ctx, 0, output, s); @@ -490,7 +490,7 @@ REGISTER_OP("AllocateTempOp0").Output("output1: float"); TEST_F(DeviceKernelOpTest, TestAllocateTempEmpty) { auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { TF_Status* s = TF_NewStatus(); - // Allocate empty output + // Allocate empty TF_Tensor int64_t dim = 0; TF_AllocatorAttributes alloc_attrs; alloc_attrs.struct_size = TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE; @@ -519,7 +519,7 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSize2x3) { auto my_compute_func = [](void* kernel, TF_OpKernelContext* ctx) { TF_Status* s = TF_NewStatus(); size_t tensor_size_bytes = 6 * TF_DataTypeSize(TF_FLOAT); - // Allocate 2x3 output + // Allocate 2x3 TF_Tensor int64_t dim[2] = {2, 3}; TF_AllocatorAttributes alloc_attrs; alloc_attrs.struct_size = TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE; @@ -530,7 +530,7 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSize2x3) { EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, dim, 2, TF_FLOAT); - // Set output to [1 2 3 4 5 6] + // Set TF_Tensor values to [1 2 3 4 5 6] void* data = TF_TensorData(output); float values[6] = {1, 2, 3, 4, 5, 6}; set_tensor_data(output, values, tensor_size_bytes, ctx); @@ -558,7 +558,7 @@ void validate_tensor(TF_Tensor* tensor, int64_t* dims, int64_t num_dims, template void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, - TF_OpKernelContext* ctx){ + TF_OpKernelContext* ctx) { T* data = reinterpret_cast(TF_TensorData(tensor)); #if GOOGLE_CUDA OpKernelContext* cc_ctx = reinterpret_cast(ctx); From aa3291d4402ce91dfd40ed32801fd70b1b6d5447 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 24 Jul 2020 20:56:17 +0000 Subject: [PATCH 0117/1017] clean up --- tensorflow/c/kernels.cc | 11 +- tensorflow/c/kernels.h | 2 +- ...-summary_op-needs-tstring-C-API-sync.patch | 377 ------------------ tensorflow/c/kernels/BUILD | 39 -- tensorflow/c/kernels/diff.patch | 0 tensorflow/c/kernels/ops/summary.cc | 70 ---- 6 files changed, 5 insertions(+), 494 deletions(-) delete mode 100644 tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch delete mode 100644 tensorflow/c/kernels/diff.patch delete mode 100644 tensorflow/c/kernels/ops/summary.cc diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 8c2c4e67b3a..5bd3989e66c 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -26,9 +26,6 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/platform/types.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/lib/gtl/array_slice.h" - // This file forms the basis of a stable ABI for third-party kernel // implementations. It is crucial that changes to this file are made cautiously // and with a focus on maintaining both source and binary compatibility. @@ -100,9 +97,9 @@ void TF_KernelBuilder_HostMemory(TF_KernelBuilder* kernel_builder, kernel_builder->cc_builder->HostMemory(arg_name); } -void TF_KernelBuilder_Priority(TF_KernelBuilder* kernel_builder, - int32_t priority_number){ - kernel_builder->cc_builder->Priority(priority_number); +void TF_KernelBuilder_Priority(TF_KernelBuilder* kernel_builder, + int32_t priority_number) { + kernel_builder->cc_builder->Priority(priority_number); } namespace tensorflow { @@ -277,7 +274,7 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, } TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, - int64_t* dims, int num_dims, + int64_t* dims, int num_dims, TF_AllocatorAttributes attributes, TF_Status* status) { auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index 658ad3ac404..a564bd42797 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -110,7 +110,7 @@ TF_CAPI_EXPORT extern void TF_KernelBuilder_HostMemory( // Specify a priority number for this kernel. TF_CAPI_EXPORT extern void TF_KernelBuilder_Priority( - TF_KernelBuilder* kernel_builder, int32_t priority_number); + TF_KernelBuilder* kernel_builder, int32_t priority_number); // Register the given kernel builder with the TensorFlow runtime. If // registration fails, the given status will be populated. diff --git a/tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch b/tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch deleted file mode 100644 index 856f4a554c3..00000000000 --- a/tensorflow/c/kernels/0001-summary_op-needs-tstring-C-API-sync.patch +++ /dev/null @@ -1,377 +0,0 @@ -From 9134fbb13794865a45288d2e722ad47c362e0ae4 Mon Sep 17 00:00:00 2001 -From: Daniel Nguyen -Date: Thu, 18 Jun 2020 23:13:11 +0000 -Subject: [PATCH] summary_op needs tstring C API sync - ---- - tensorflow/c/kernels/diff.patch | 0 - tensorflow/c/kernels/ops/summary.cc | 70 ++++++++++ - tensorflow/c/kernels/summary_op.cc | 171 ++++++++++++++++++++++++ - tensorflow/c/kernels/summary_op_test.cc | 96 +++++++++++++ - 4 files changed, 337 insertions(+) - create mode 100644 tensorflow/c/kernels/diff.patch - create mode 100644 tensorflow/c/kernels/ops/summary.cc - create mode 100644 tensorflow/c/kernels/summary_op.cc - create mode 100644 tensorflow/c/kernels/summary_op_test.cc - -diff --git a/tensorflow/c/kernels/diff.patch b/tensorflow/c/kernels/diff.patch -new file mode 100644 -index 0000000000..e69de29bb2 -diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc -new file mode 100644 -index 0000000000..550a663d00 ---- /dev/null -+++ b/tensorflow/c/kernels/ops/summary.cc -@@ -0,0 +1,70 @@ -+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. -+ -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ -+ http://www.apache.org/licenses/LICENSE-2.0 -+ -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+#include -+#include -+ -+#include "tensorflow/c/ops.h" -+#include "tensorflow/core/framework/selective_registration.h" -+#include "tensorflow/core/platform/logging.h" -+#include "tensorflow/core/platform/macros.h" -+ -+ -+static void TF_ScalarSummary_shape_inference_fn(TF_ShapeInferenceContext* ctx, -+ TF_Status* status) { -+ TF_ShapeHandle* result = TF_NewShapeHandle(); -+ // TODO: what to do in the case of unknown input shape? -+ if (TF_GetCode(status) == TF_OK && -+ !TF_ShapeInferenceContextRankKnown(ctx, result)) { -+ TF_ShapeInferenceContextSetUnknownShape(ctx, status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while setting unknown shape function"; -+ TF_DeleteShapeHandle(result); -+ return; -+ } -+ // make shape handle a scalar value (empty shape) -+ if (TF_GetCode(status) == TF_OK) { -+ TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while setting shape function"; -+ } -+ TF_DeleteShapeHandle(result); -+} -+ -+void Register_ScalarSummaryOp() { -+ TF_Status* status = TF_NewStatus(); -+ -+ TF_OpDefinitionBuilder* op_builder = TF_NewOpDefinitionBuilder("SummaryScalar"); -+ TF_OpDefinitionBuilderAddInput(op_builder, "tags: string"); -+ TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); -+ TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); -+ TF_OpDefinitionBuilderAddAttr( -+ op_builder, -+ "T: realnumbertype"); -+ TF_OpDefinitionBuilderSetShapeInferenceFunction(op_builder, -+ &TF_ScalarSummary_shape_inference_fn); -+ -+ TF_RegisterOpDefinition(op_builder, status); -+ CHECK_EQ(TF_GetCode(status), TF_OK) -+ << "TF_ScalarSummary op registration failed: " << TF_Message(status); -+ TF_DeleteStatus(status); -+} -+ -+TF_ATTRIBUTE_UNUSED static bool SummaryScalarOpRegistered = []() { -+ if (SHOULD_REGISTER_OP("SummaryScalar")) { -+ Register_ScalarSummaryOp(); -+ } -+ return true; -+}(); -diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc -new file mode 100644 -index 0000000000..3a78d321d7 ---- /dev/null -+++ b/tensorflow/c/kernels/summary_op.cc -@@ -0,0 +1,171 @@ -+ -+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. -+ -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ -+ http://www.apache.org/licenses/LICENSE-2.0 -+ -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+#include -+ -+#include "tensorflow/c/kernels.h" -+#include "tensorflow/c/ops.h" -+#include "tensorflow/c/tf_tensor.h" -+#include "tensorflow/core/framework/common_shape_fns.h" -+#include "tensorflow/core/framework/op.h" -+#include "tensorflow/core/framework/selective_registration.h" -+#include "tensorflow/core/framework/shape_inference.h" -+#include "tensorflow/core/platform/macros.h" -+#include "tensorflow/core/framework/summary.pb.h" -+#include "tensorflow/core/platform/protobuf.h" -+#include "tensorflow/core/framework/register_types.h" -+ -+#include "tensorflow/core/framework/types.h" -+ -+// BitcastOp implements a bitcast kernel, creating an output tensor that shares -+// the same data buffer as the input but with a different shape and/or data -+// type. Its inputs are: -+// -+// * the input tensor -+// * an attribute named "T" containing the TF_DataType of the input tensor -+// * an attribute named "type" containing the TF_DataType of the output tensor -+// -+// Given an input tensor of shape [...], if the input DataType "T" is larger -+// than the output DataType "type", then the shape changes from [...] -+// to [..., sizeof(T)/sizeof(type)]. -+// -+// If "T" is smaller than "type", the operator requires that the rightmost -+// dimension be equal to sizeof(type)/sizeof(T). The shape then goes from -+// [..., sizeof(type)/sizeof(T)] to [...]. -+// -+// Bitcast is implemented as a low-level cast, so machines with different endian -+// orderings will give different results. -+ -+static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { -+ // TODO: replace with a void* pointer type later -+ int a = 4; -+ return static_cast(&a); -+} -+ -+static void SummaryScalarOp_Delete(void* kernel) { -+ return; -+} -+ -+bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ -+ if (TF_NumDims(tensor1) != TF_NumDims(tensor2)){ -+ return false; -+ } -+ for(int d = 0; d < TF_NumDims(tensor1); d++){ -+ if (TF_Dim(tensor1, d) != TF_Dim(tensor2, d)){ -+ return false; -+ } -+ } -+ return true; -+} -+ -+template -+static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { -+ TF_Tensor* tags; -+ TF_Tensor* values; -+ TF_Status* status = TF_NewStatus(); -+ TF_GetInput(ctx, 0, &tags, status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while getting input"; -+ if (TF_GetCode(status) == TF_OK){ -+ TF_GetInput(ctx, 1, &values, status); -+ } -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while getting input"; -+ if (TF_GetCode(status) == TF_OK) { -+ if (!IsSameSize(tags, values)) { -+ std::ostringstream err; -+ err << "tags and values not the same shape: "; -+ TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); -+ } -+ } -+ -+ tensorflow::Summary s; -+ if (TF_GetCode(status) == TF_OK) { -+ auto Ttags_array = static_cast(TF_TensorData(tags)); -+ auto values_array = static_cast(TF_TensorData(values)); -+ for (int i = 0; i < TF_TensorElementCount(tags); ++i){ -+ tensorflow::Summary::Value* v = s.add_value(); -+ TF_TString_Init(Ttags_array[i]); -+ v->set_tag(TF_TString_GetDataPointer(Ttags_array[i]), TF_TString_GetSize(Ttags_array[i])); -+ v->set_simple_value(float(values_array[i])); -+ } -+ -+ -+ // TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), 0, 0) -+ -+ // TF_Tensor* output = TF_AllocateTensor(k->output_data_type, dims, 0, -+ // TF_DataTypeSize(k->output_data_type)); -+ // if (TF_GetCode(status) == TF_OK) { -+ // TF_SetOutput(ctx, 0, output, status); -+ // } -+ // TF_DeleteTensor(output); -+ } -+ -+ // if (TF_GetCode(status) != TF_OK) { -+ // TF_OpKernelContext_Failure(ctx, status); -+ // } -+ // TF_DeleteStatus(status); -+ // TF_DeleteTensor(tags); -+} -+ -+template -+void RegisterSummaryScalarOpKernel() { -+ TF_Status* status = TF_NewStatus(); -+ { -+ auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_CPU, -+ &SummaryScalarOp_Create, &SummaryScalarOp_Compute, -+ &SummaryScalarOp_Delete); -+ TF_KernelBuilder_TypeConstraint(builder, "T", static_cast(tensorflow::DataTypeToEnum::v()), status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while adding type constraint"; -+ TF_RegisterKernelBuilder("SummaryScalar", builder, status); -+ CHECK_EQ(TF_OK, TF_GetCode(status)) -+ << "Error while registering Summary Scalar kernel"; -+ } -+// template -+// #if GOOGLE_CUDA -+// { -+// auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_GPU, -+// &SummaryScalarOp_Create, &SummaryScalarOp_Compute, -+// &SummaryScalarOp_Delete); -+// TF_RegisterKernelBuilder("SummaryScalar", builder, status); -+// CHECK_EQ(TF_OK, TF_GetCode(status)) -+// << "Error while registering CUDA SummaryScalar kernel"; -+// } -+// #endif -+ -+ TF_DeleteStatus(status); -+} -+ -+// A dummy static variable initialized by a lambda whose side-effect is to -+// register the bitcast kernel. -+ -+ -+TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { -+ if (SHOULD_REGISTER_OP_KERNEL("SummaryScalar")) { -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ RegisterSummaryScalarOpKernel(); -+ } -+ return true; -+}(); -+ -diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc -new file mode 100644 -index 0000000000..fd6199abd6 ---- /dev/null -+++ b/tensorflow/c/kernels/summary_op_test.cc -@@ -0,0 +1,96 @@ -+/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. -+ -+Licensed under the Apache License, Version 2.0 (the "License"); -+you may not use this file except in compliance with the License. -+You may obtain a copy of the License at -+ -+ http://www.apache.org/licenses/LICENSE-2.0 -+ -+Unless required by applicable law or agreed to in writing, software -+distributed under the License is distributed on an "AS IS" BASIS, -+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+See the License for the specific language governing permissions and -+limitations under the License. -+==============================================================================*/ -+ -+#include "tensorflow/core/framework/attr_value.pb.h" -+#include "tensorflow/core/framework/attr_value_util.h" -+#include "tensorflow/core/framework/fake_input.h" -+#include "tensorflow/core/framework/node_def.pb.h" -+#include "tensorflow/core/framework/node_def_builder.h" -+#include "tensorflow/core/framework/op_kernel.h" -+#include "tensorflow/core/framework/shape_inference.h" -+#include "tensorflow/core/platform/test.h" -+ -+#include -+#include -+#include -+namespace tensorflow { -+namespace { -+ -+class DummyDevice : public DeviceBase { -+ public: -+ explicit DummyDevice(Env* env) : DeviceBase(env) {} -+ Allocator* GetAllocator(AllocatorAttributes /*attr*/) override { -+ return cpu_allocator(); -+ } -+}; -+ -+void TestScalarSummaryOp(Tensor* tags, Tensor* values, error::Code expected_code) { -+ Status status; -+ NodeDef def; -+ def.set_op("SummaryScalar"); -+ -+ def.set_device(DEVICE_CPU); -+ -+ AttrValue valuesTypeAttr; -+ SetAttrValue(values->dtype(), &valuesTypeAttr); -+ (*def.mutable_attr())["T"] = valuesTypeAttr; -+ -+ def.add_input( -+ strings::StrCat("input1: ", DataTypeString(tags->dtype()))); -+ def.add_input( -+ strings::StrCat("input2: ", DataTypeString(values->dtype()))); -+ -+ std::unique_ptr kernel = -+ CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status); -+ ASSERT_TRUE(status.ok()) << status.ToString(); -+ OpKernelContext::Params params; -+ DummyDevice dummy_device(nullptr); -+ params.device = &dummy_device; -+ params.op_kernel = kernel.get(); -+ gtl::InlinedVector inputs; -+ inputs.emplace_back(tags); -+ inputs.emplace_back(values); -+ params.inputs = &inputs; -+ OpKernelContext ctx(¶ms, 1); -+ kernel->Compute(&ctx); -+ -+ ASSERT_EQ(expected_code, ctx.status().code()); -+ if (expected_code == error::OK) { -+ ASSERT_EQ(true, false) -+ << ctx.mutable_output(0)->shape().DebugString(); -+ } -+} -+ -+TEST(ScalarSummaryOpTest, Test) { -+ int vectorSize = 2; -+ Tensor tags(DT_STRING, {vectorSize}); -+ Tensor values(DT_FLOAT, {vectorSize}); -+ for (int i = 0; i < vectorSize; ++i){ -+ values.vec()(i) = static_cast(i); -+ } -+ tags.vec()(0) = "tag 1"; -+ tags.vec()(1) = "tag 2"; -+ TestScalarSummaryOp(&tags, &values, error::INVALID_ARGUMENT); -+} -+ -+ -+PartialTensorShape S(std::initializer_list dims) { -+ return PartialTensorShape(dims); -+} -+ -+ -+ -+} // namespace -+} // namespace tensorflow --- -2.27.0.111.gc72c7da667-goog - diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 3ce53309841..770352c62c1 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -24,21 +24,6 @@ tf_kernel_library( ], ) -tf_kernel_library( - name = "summary_op", - prefix = "summary_op", - deps = [ - "//tensorflow/c:kernels", - "//tensorflow/c:ops", - "//tensorflow/c:tf_datatype", - "//tensorflow/c:tf_status", - "//tensorflow/c:tf_tensor", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ], -) - - tf_gen_op_libs( op_lib_names = ["bitcast"], deps = [ @@ -50,17 +35,6 @@ tf_gen_op_libs( ], ) -tf_gen_op_libs( - op_lib_names = ["summary"], - deps = [ - "//tensorflow/c:ops", - "//tensorflow/c:tf_datatype", - "//tensorflow/c:tf_status", - "//tensorflow/c:tf_tensor", - "//tensorflow/core:lib", - ], -) - tf_cc_test( name = "bitcast_op_test", srcs = ["bitcast_op_test.cc"], @@ -74,19 +48,6 @@ tf_cc_test( ], ) -tf_cc_test( - name = "summary_op_test", - srcs = ["summary_op_test.cc"], - deps = [ - ":summary_op", - ":summary_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) # Changes to the Android srcs here should be replicated in # tensorflow/contrib/makefile/tf_op_files.txt. # diff --git a/tensorflow/c/kernels/diff.patch b/tensorflow/c/kernels/diff.patch deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tensorflow/c/kernels/ops/summary.cc b/tensorflow/c/kernels/ops/summary.cc deleted file mode 100644 index 550a663d006..00000000000 --- a/tensorflow/c/kernels/ops/summary.cc +++ /dev/null @@ -1,70 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include -#include - -#include "tensorflow/c/ops.h" -#include "tensorflow/core/framework/selective_registration.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/macros.h" - - -static void TF_ScalarSummary_shape_inference_fn(TF_ShapeInferenceContext* ctx, - TF_Status* status) { - TF_ShapeHandle* result = TF_NewShapeHandle(); - // TODO: what to do in the case of unknown input shape? - if (TF_GetCode(status) == TF_OK && - !TF_ShapeInferenceContextRankKnown(ctx, result)) { - TF_ShapeInferenceContextSetUnknownShape(ctx, status); - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while setting unknown shape function"; - TF_DeleteShapeHandle(result); - return; - } - // make shape handle a scalar value (empty shape) - if (TF_GetCode(status) == TF_OK) { - TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while setting shape function"; - } - TF_DeleteShapeHandle(result); -} - -void Register_ScalarSummaryOp() { - TF_Status* status = TF_NewStatus(); - - TF_OpDefinitionBuilder* op_builder = TF_NewOpDefinitionBuilder("SummaryScalar"); - TF_OpDefinitionBuilderAddInput(op_builder, "tags: string"); - TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); - TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); - TF_OpDefinitionBuilderAddAttr( - op_builder, - "T: realnumbertype"); - TF_OpDefinitionBuilderSetShapeInferenceFunction(op_builder, - &TF_ScalarSummary_shape_inference_fn); - - TF_RegisterOpDefinition(op_builder, status); - CHECK_EQ(TF_GetCode(status), TF_OK) - << "TF_ScalarSummary op registration failed: " << TF_Message(status); - TF_DeleteStatus(status); -} - -TF_ATTRIBUTE_UNUSED static bool SummaryScalarOpRegistered = []() { - if (SHOULD_REGISTER_OP("SummaryScalar")) { - Register_ScalarSummaryOp(); - } - return true; -}(); From 953a8d99a4bae4fd74b32aa71ae3bca3b18dae84 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 24 Jul 2020 21:09:37 +0000 Subject: [PATCH 0118/1017] clean up --- tensorflow/c/kernels/summary_op.cc | 152 -------------------- tensorflow/c/kernels/summary_op_test.cc | 177 ------------------------ tensorflow/c/kernels_test.cc | 1 - tensorflow/c/tf_tensor.h | 5 - tensorflow/c/tf_tensor_internal.h | 3 - 5 files changed, 338 deletions(-) delete mode 100644 tensorflow/c/kernels/summary_op.cc delete mode 100644 tensorflow/c/kernels/summary_op_test.cc diff --git a/tensorflow/c/kernels/summary_op.cc b/tensorflow/c/kernels/summary_op.cc deleted file mode 100644 index 002de6fb6e8..00000000000 --- a/tensorflow/c/kernels/summary_op.cc +++ /dev/null @@ -1,152 +0,0 @@ - -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include - -#include "tensorflow/c/kernels.h" -#include "tensorflow/c/ops.h" -#include "tensorflow/c/tf_tensor.h" -#include "tensorflow/core/framework/common_shape_fns.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/selective_registration.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/framework/summary.pb.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/core/framework/register_types.h" - -#include "tensorflow/core/framework/types.h" -#include - -// TODO: Copy over Summary Scalar Op Doc - -static void* SummaryScalarOp_Create(TF_OpKernelConstruction* ctx) { - // TODO: replace with a void* pointer type later - void* ptr; - return ptr; -} - -static void SummaryScalarOp_Delete(void* kernel) { - return; -} - -bool IsSameSize(TF_Tensor* tensor1, TF_Tensor* tensor2){ - if (TF_NumDims(tensor1) != TF_NumDims(tensor2)){ - return false; - } - for(int d = 0; d < TF_NumDims(tensor1); d++){ - if (TF_Dim(tensor1, d) != TF_Dim(tensor2, d)){ - return false; - } - } - return true; -} - -template -static void SummaryScalarOp_Compute(void* kernel, TF_OpKernelContext* ctx) { - TF_Tensor* tags; - TF_Tensor* values; - TF_Status* status = TF_NewStatus(); - TF_GetInput(ctx, 0, &tags, status); - if (TF_GetCode(status) == TF_OK){ - TF_GetInput(ctx, 1, &values, status); - } - - if (TF_GetCode(status) == TF_OK) { - if (!IsSameSize(tags, values)) { - std::ostringstream err; - err << "tags and values not the same shape: " << TF_ShapeDebugString(tags) - << " != " << TF_ShapeDebugString(values); - TF_SetStatus(status, TF_INVALID_ARGUMENT, err.str().c_str()); - } - } - - // Copy tag and string data into summary protobuf - tensorflow::Summary s; - if (TF_GetCode(status) == TF_OK) { - // Convert tags and values tensor to array to access elements by index - auto tags_array = static_cast(TF_TensorData(tags)); - auto values_array = static_cast(TF_TensorData(values)); - for (int i = 0; i < TF_TensorElementCount(tags); ++i){ - tensorflow::Summary::Value* v = s.add_value(); - v->set_tag(TF_TString_GetDataPointer(&tags_array[i]), - TF_TString_GetSize(&tags_array[i])); - v->set_simple_value(float(values_array[i])); - } - TF_Tensor* summary_tensor = TF_AllocateOutput(ctx, 0, - TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, - sizeof(TF_TString), status); - if (TF_GetCode(status) == TF_OK){ - SerializeToTString(s, static_cast - (TF_TensorData(summary_tensor))); - } - TF_DeleteTensor(summary_tensor); - } - - if (TF_GetCode(status) != TF_OK) { - TF_OpKernelContext_Failure(ctx, status); - } - TF_DeleteStatus(status); - TF_DeleteTensor(tags); -} - -template -void RegisterSummaryScalarOpKernel() { - TF_Status* status = TF_NewStatus(); - { - auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_CPU, - &SummaryScalarOp_Create, &SummaryScalarOp_Compute, - &SummaryScalarOp_Delete); - TF_KernelBuilder_TypeConstraint(builder, "T", static_cast(tensorflow::DataTypeToEnum::v()), status); - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while adding type constraint"; - TF_RegisterKernelBuilder("SummaryScalarOp", builder, status); - CHECK_EQ(TF_OK, TF_GetCode(status)) - << "Error while registering Summary Scalar kernel"; - } -// #if GOOGLE_CUDA -// { -// auto* builder = TF_NewKernelBuilder("SummaryScalar", tensorflow::DEVICE_GPU, -// &SummaryScalarOp_Create, &SummaryScalarOp_Compute, -// &SummaryScalarOp_Delete); -// TF_RegisterKernelBuilder("SummaryScalar", builder, status); -// CHECK_EQ(TF_OK, TF_GetCode(status)) -// << "Error while registering CUDA SummaryScalar kernel"; -// } -// #endif - - TF_DeleteStatus(status); -} - -// A dummy static variable initialized by a lambda whose side-effect is to -// register the bitcast kernel. - - -TF_ATTRIBUTE_UNUSED static bool IsSummaryScalarOpKernelRegistered = []() { - if (SHOULD_REGISTER_OP_KERNEL("SummaryScalarOp")) { - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - RegisterSummaryScalarOpKernel(); - } - return true; -}(); - diff --git a/tensorflow/c/kernels/summary_op_test.cc b/tensorflow/c/kernels/summary_op_test.cc deleted file mode 100644 index afc818fb7b5..00000000000 --- a/tensorflow/c/kernels/summary_op_test.cc +++ /dev/null @@ -1,177 +0,0 @@ -/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/framework/attr_value.pb.h" -#include "tensorflow/core/framework/attr_value_util.h" -#include "tensorflow/core/framework/fake_input.h" -#include "tensorflow/core/framework/node_def.pb.h" -#include "tensorflow/core/framework/node_def_builder.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/platform/test.h" - -#include "tensorflow/core/framework/summary.pb.h" -#include "tensorflow/core/platform/protobuf.h" -#include "tensorflow/c/tf_tensor.h" -#include "tensorflow/c/tf_tensor_internal.h" - -#include -#include -#include -namespace tensorflow { -namespace { - -class DummyDevice : public DeviceBase { - public: - explicit DummyDevice(Env* env) : DeviceBase(env) {} - Allocator* GetAllocator(AllocatorAttributes /*attr*/) override { - return cpu_allocator(); - } -}; - -// Helper for comparing ouput and expected output -static void EXPECT_SummaryMatches(const Summary& actual, - const string& expected_str) { - Summary expected; - (protobuf::TextFormat::ParseFromString(expected_str, &expected)); - EXPECT_EQ(expected.DebugString(), actual.DebugString()); -} - - -void TestScalarSummaryOp(Tensor* tags, Tensor* values, string expected_summary, - error::Code expected_code) { - // initialize node used to fetch OpKernel - Status status; - NodeDef def; - def.set_op("SummaryScalar"); - def.set_device(DEVICE_CPU); - AttrValue valuesTypeAttr; - SetAttrValue(values->dtype(), &valuesTypeAttr); - (*def.mutable_attr())["T"] = valuesTypeAttr; - def.add_input( - strings::StrCat("input1: ", DataTypeString(tags->dtype()))); - def.add_input( - strings::StrCat("input2: ", DataTypeString(values->dtype()))); - - std::unique_ptr kernel = - CreateOpKernel(DeviceType(DEVICE_CPU), nullptr, nullptr, def, 1, &status); - ASSERT_TRUE(status.ok()) << status.ToString(); - - // initialize OpKernel parameters - OpKernelContext::Params params; - DummyDevice dummy_device(nullptr); - params.device = &dummy_device; - params.op_kernel = kernel.get(); - gtl::InlinedVector inputs; - inputs.emplace_back(tags); - inputs.emplace_back(values); - params.inputs = &inputs; - OpKernelContext ctx(¶ms, 1); - AllocatorAttributes alloc_attrs; - std::vector output_alloc_attrs({alloc_attrs}); - params.output_attr_array = output_alloc_attrs.data(); - kernel->Compute(&ctx); - ASSERT_EQ(expected_code, ctx.status().code()); - if (expected_code == error::OK){ - Summary summary; - ParseProtoUnlimited(&summary, ctx.mutable_output(0)->scalar()()); - EXPECT_SummaryMatches(summary, expected_summary); - } -} - -TEST(ScalarSummaryOpTest, SimpleFloat) { - int vectorSize = 3; - Tensor tags(DT_STRING, {vectorSize}); - Tensor values(DT_FLOAT, {vectorSize}); - tags.vec()(0) = "tag1"; - tags.vec()(1) = "tag2"; - tags.vec()(2) = "tag3"; - values.vec()(0) = 1.0f; - values.vec()(1) = -0.73f; - values.vec()(2) = 10000.0f; - TestScalarSummaryOp(&tags, &values, R"( - value { tag: 'tag1' simple_value: 1.0 } - value { tag: 'tag2' simple_value: -0.73} - value { tag: 'tag3' simple_value: 10000.0})", error::OK); -} - -TEST(ScalarSummaryOpTest, SimpleDouble) { - int vectorSize = 3; - Tensor tags(DT_STRING, {vectorSize}); - Tensor values(DT_DOUBLE, {vectorSize}); - tags.vec()(0) = "tag1"; - tags.vec()(1) = "tag2"; - tags.vec()(2) = "tag3"; - values.vec()(0) = 1.0; - values.vec()(1) = -0.73; - values.vec()(2) = 10000.0; - TestScalarSummaryOp(&tags, &values, R"( - value { tag: 'tag1' simple_value: 1.0 } - value { tag: 'tag2' simple_value: -0.73} - value { tag: 'tag3' simple_value: 10000.0})", error::OK); -} - -TEST(ScalarSummaryOpTest, SimpleHalf) { - int vectorSize = 3; - Tensor tags(DT_STRING, {vectorSize}); - Tensor values(DT_HALF, {vectorSize}); - tags.vec()(0) = "tag1"; - tags.vec()(1) = "tag2"; - tags.vec()(2) = "tag3"; - values.vec()(0) = static_cast(1.0); - values.vec()(1) = static_cast(-2.0); - values.vec()(2) = static_cast(10000.0); - TestScalarSummaryOp(&tags, &values, R"( - value { tag: 'tag1' simple_value: 1.0 } - value { tag: 'tag2' simple_value: -2.0} - value { tag: 'tag3' simple_value: 10000.0})", error::OK); -} - -TEST(ScalarSummaryOpTest, Error_WrongDimsTags) { - int vectorSize = 3; - Tensor tags(DT_STRING, {2, 1}); - Tensor values(DT_FLOAT, {2}); - tags.matrix()(0, 0) = "tag1"; - tags.matrix()(1, 0) = "tag2"; - values.vec()(0) = 1.0f; - values.vec()(1) = -2.0f; - TestScalarSummaryOp(&tags, &values, R"()", error::INVALID_ARGUMENT); -} - -TEST(ScalarSummaryOpTest, Error_WrongValuesTags) { - Tensor tags(DT_STRING, {2}); - Tensor values(DT_FLOAT, {2, 1}); - tags.vec()(0) = "tag1"; - tags.vec()(1) = "tag2"; - values.matrix()(0, 0) = 1.0f; - values.matrix()(1, 0) = -2.0f; - TestScalarSummaryOp(&tags, &values, R"()", error::INVALID_ARGUMENT); -} - -TEST(ScalarSummaryOpTest, IsRegistered){ - const OpRegistrationData* reg; - TF_CHECK_OK(OpRegistry::Global()->LookUp("SummaryScalar", ®)); -} - - - -PartialTensorShape S(std::initializer_list dims) { - return PartialTensorShape(dims); -} - - - -} // namespace -} // namespace tensorflow diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index 4c925402cb7..e2eef11db0e 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -568,5 +568,4 @@ void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, memcpy(data, values, tensor_size_bytes); #endif } - } // namespace tensorflow \ No newline at end of file diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index d8d8155e8cd..91263fb3b7f 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -22,9 +22,6 @@ limitations under the License. #include "tensorflow/c/tf_datatype.h" #include "tensorflow/c/tf_status.h" -#include -#include - // Macro to control visibility of exported symbols in the shared library (.so, // .dylib, .dll). // This duplicates the TF_EXPORT macro definition in @@ -172,8 +169,6 @@ TF_CAPI_EXPORT extern void TF_TensorBitcastFrom(const TF_Tensor* from, // Returns bool iff this tensor is aligned. TF_CAPI_EXPORT extern bool TF_TensorIsAligned(const TF_Tensor*); -TF_CAPI_EXPORT extern std::string TF_ShapeDebugString(const TF_Tensor*); - #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/tf_tensor_internal.h b/tensorflow/c/tf_tensor_internal.h index b3f44c71245..7a896dc5d11 100644 --- a/tensorflow/c/tf_tensor_internal.h +++ b/tensorflow/c/tf_tensor_internal.h @@ -24,8 +24,6 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/platform/casts.h" -#include -#include // Internal structures used by the C API. These are likely to change and should // not be depended on. @@ -106,7 +104,6 @@ class TensorInterface : public AbstractTensorInterface { void* Data() const override; bool IsAligned() const override; bool CanMove() const override; - std::string ShapeDebugString() const; Status ToTensor(tensorflow::Tensor* dst) const; Status BitcastFrom(const TensorInterface& from, DataType type, From be5f945a62daa9d90df2a7098cd4b156a001e4c4 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 24 Jul 2020 21:23:41 +0000 Subject: [PATCH 0119/1017] clean up for cleaner merge --- tensorflow/c/tf_tensor.cc | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc index 39f0176c0bf..0feb986ce44 100644 --- a/tensorflow/c/tf_tensor.cc +++ b/tensorflow/c/tf_tensor.cc @@ -28,8 +28,6 @@ limitations under the License. #include "tensorflow/core/framework/types.pb.h" #include "tensorflow/core/lib/core/coding.h" #include "tensorflow/core/platform/casts.h" -#include -#include using tensorflow::Status; using tensorflow::Tensor; @@ -182,11 +180,6 @@ void TF_TensorBitcastFrom(const TF_Tensor* from, TF_DataType type, Set_TF_Status_from_Status(status, cc_status); } -std::string TF_ShapeDebugString(const TF_Tensor* t){ - return tensorflow::down_cast(t->tensor) - ->ShapeDebugString(); -} - namespace tensorflow { void TensorInterface::Release() { delete this; } @@ -232,10 +225,6 @@ Status TensorInterface::BitcastFrom(const TensorInterface& from, DataType type, return tensor_.BitcastFrom(from.tensor_, type, s); } -std::string TensorInterface::ShapeDebugString() const { - return tensor_.shape().DebugString(); -} - } // namespace tensorflow // -------------------------------------------------------------------------- @@ -267,7 +256,6 @@ static TF_Tensor* EmptyTensor(TF_DataType dtype, namespace tensorflow { // Non-static for testing. - TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, Status* status) { *status = tensorflow::Status::OK(); if (!src.IsInitialized()) { @@ -328,10 +316,8 @@ Status TensorInterface::ToTensor(tensorflow::Tensor* dst) const { return Status::OK(); } - bool TensorInterface::IsAligned() const { return tensor_.IsAligned(); } } // namespace tensorflow bool TF_TensorIsAligned(const TF_Tensor* t) { return t->tensor->IsAligned(); } - From e31572f0adbd24a061d8dc6475bca93367cc9637 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Fri, 24 Jul 2020 22:11:06 +0000 Subject: [PATCH 0120/1017] incorporate previous changes to allocate_temp --- tensorflow/c/kernels.cc | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 5bd3989e66c..d89c222568f 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -281,22 +281,25 @@ TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, TF_SetStatus(status, TF_OK, ""); tensorflow::TensorShape shape; for (int i = 0; i < num_dims; ++i) { - shape.AddDim(dims[i]); + shape.AddDim(dims[i]); } tensorflow::AllocatorAttributes allocator_attr; if (attributes.on_host) { allocator_attr.set_on_host(true); } tensorflow::Status s; - tensorflow::Tensor tensor_temp; - TF_Tensor* tf_tensor_temp; - s = cc_ctx->allocate_temp(static_cast(dtype), shape, + tensorflow::Tensor tensor; + TF_Tensor* tf_tensor; + s = cc_ctx->allocate_temp(static_cast(dtype), shape, &tensor_temp, allocator_attr); - if (s.ok()) { - tf_tensor_temp = TF_TensorFromTensor(tensor_temp, &s); + if (!s.ok()) { + ::tensorflow::Set_TF_Status_fromStatus(status, s); + return nullptr; } - if (s.ok()) { - ::tensorflow::Set_TF_Status_from_Status(status, s); - return tf_tensor_temp; - } + tf_tensor = TF_TensorFromTensor(tensor, &s); + if (!s.ok()) { + ::tensorflow::Set_TF_Status_from_Status(status, s); + return nullptr; + } + return tf_tensor; } From 4732d2aebfd22b8dde7ce3ee9abe6efed7b0c485 Mon Sep 17 00:00:00 2001 From: Steenu Johnson Date: Sat, 25 Jul 2020 11:02:37 +0530 Subject: [PATCH 0121/1017] Adding dependency tensorflow/core/platform:logging to ignore_errors_dataset_op. Signed-off-by: Steenu Johnson --- tensorflow/core/kernels/data/experimental/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD index 8457cfa6145..49cf7a284c6 100644 --- a/tensorflow/core/kernels/data/experimental/BUILD +++ b/tensorflow/core/kernels/data/experimental/BUILD @@ -248,6 +248,7 @@ tf_kernel_library( deps = [ "//tensorflow/core:experimental_dataset_ops_op_lib", "//tensorflow/core:framework", + "//tensorflow/core/platform:logging", "//third_party/eigen3", ], ) From 2bf057f5a33a6889e323d6144d622f5f448ee5c0 Mon Sep 17 00:00:00 2001 From: Steenu Johnson Date: Sat, 25 Jul 2020 11:04:41 +0530 Subject: [PATCH 0122/1017] Updating docstring with "Args:" section to describe log_warning argument. Signed-off-by: Steenu Johnson --- tensorflow/python/data/experimental/ops/error_ops.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/python/data/experimental/ops/error_ops.py b/tensorflow/python/data/experimental/ops/error_ops.py index 14b8802d920..12ad4b323f1 100644 --- a/tensorflow/python/data/experimental/ops/error_ops.py +++ b/tensorflow/python/data/experimental/ops/error_ops.py @@ -41,6 +41,9 @@ def ignore_errors(log_warning=False): dataset = dataset.apply(tf.data.experimental.ignore_errors()) # ==> {1., 0.5, 0.2} ``` + Args: + log_warning: (Optional.) A 'tf.bool' scalar indicating whether ignored + errors should be logged to stderr. Defaults to 'False'. Returns: A `Dataset` transformation function, which can be passed to From c39b8215d9715e136f6bc297e53b574772f8ab65 Mon Sep 17 00:00:00 2001 From: Steenu Johnson Date: Sat, 25 Jul 2020 11:05:37 +0530 Subject: [PATCH 0123/1017] Adding api_def_IgnoreErrorsDatasetV2.pbtxt file. Signed-off-by: Steenu Johnson --- .../api_def/base_api/api_def_IgnoreErrorsDatasetV2.pbtxt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 tensorflow/core/api_def/base_api/api_def_IgnoreErrorsDatasetV2.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_IgnoreErrorsDatasetV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_IgnoreErrorsDatasetV2.pbtxt new file mode 100644 index 00000000000..2d8f8470869 --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_IgnoreErrorsDatasetV2.pbtxt @@ -0,0 +1,7 @@ +op { + graph_op_name: "IgnoreErrorsDatasetV2" + summary: <S`#eZv?ggtU0+XdKc1-+WXtIcr^MUNSiJHHTwVXR7E8aZKRa5g*@`R9| z-S?elPI@rOs-1iCqA8Cjan0Re&Tv3yR>6!_9|Y!Fwry0j?l>sBLSaUz=G5aRg65Wj zx|by`|ET3>32b|=sbOa4<2ZfFPE#XsgW1O__AX1$n9k`Wq&I~@$gq;pQGikWAV-TO z*Fu5GVHYIiChEF5%r>t~o~OwnC&(~)gVmgsT#g2;tR@1hG_5zkFr1i_;4-;bFVjAI zg0A3#SyK)$DJZgKADH~sgHs}bwc`@!+zIZim#xG<3ur&oSi;GmnZSCVQ+&~KuGce+ zmUo&pSr&fe_^;jnq`FK)O~gtdvd;G8%($#8DP<%+ zSx}2v@OERH?GvTihw~aB##fcRQej4mCTCE=G|n4cm6{=EJcHz# z#hS91TD(>$U1aUB0`0R8y09XsVTIwLX=yBVdpEXD-^uxK((?N69FsN*X%y=^2(XzI zE$EoYV0}yDu!Detp-|TW+29L;!KXCXJ{ol_WMB|r{I__DA*aUX3c;KU;&O%aS8TBA z6JX0epd@!ddC~!;Ne%xQCmINS)-(^*a@f?#_4uKc=ZC)5iHwhe7V=cJe3?0OzXs<8 zuC|z<$cZ9V3MSd+)5X@$;NnVPHJ`BfT%zX61uQb{zK3syJX7SfYnow@T=2$|)5w5R zM}e7JbLqofiUo~bE0yQo&6@J!;xaSA`HmZ$TW4~HB?wf`a#q_Y!O5^-@k`xR1`DJd zrp$aWW5UHL9fi6Nzp4N2YP%=2j-^rJX;$$j^RSz0m695Da>=VFEnvt=n8jc%@aM+9e}}!Y!*dFtT1K6Wf|`K48l^#pN24`nIgxIfr5Ocj1jw8dmJ%6n9nFJVV;3 z+>3KtRhs#N$wArc-nl3rG8Wh=X~SGNVP2O|-^rD0H?tZ3%oUxrP59!*FFX4fcC*dW z+U+WyHm!Sle!BIH*Xy!(3c3H@(NnD{nVq5LCUqlV`EHI4`xofG_Sll0u)0**h&J)wB-RtPs_r-HM4yD&*k7PHd$edN5XoEF7pYi_iF2K z>TmFTZz{T^Vr_``o+$?6^(R*`$V^x+xqDqh`^*Qc-&Qc+Yu)-v+n)RO8m16AaYJug znZ?gjHW>1==H z$QjY8v+9;r;*Is|Ui3})vuN$2i4Q~CUNE@a{-%+BbM3WLLfPfbeRIk-P43R@?$fN& za1NU7`mMw(qp5V;=Cg|z+3T#ZQdyzwU_W`np>I)VZp~g?V6iBn^#4v)wq7Ik#*b6m z9-E|cGSpaBi<0wZMVG^hc`HwwnxB38 zIIU-UpC2mVgnlddkG=TNpsa?u{PWLg2_iCT$xqay5o5CqibAF1YY`N%_DpC7x zYsnqc!idY0v)L~mVTpX9krV6N#wuFtvo&|^_4KzFt4n$UQetb&=B94DbJ^fdXu9kF ztPY7Qf8WSBL|-`UbD^eIlgW48?$Rud7F!PAC|i#NxoGeIZ5_A5dCk=Q1dn8|>A2Xo zMs;sT2>-H}eTSl^e^JQU<{Mouy;no^BzpnJjeH8B9B4)n#*;i9@BsD>3=xvzYm<&YnxRd}cX&=!C~% zb<^(ljQHdK8;vfqzSBB%&XDQOtLyH6uV1ykT%$Tk-F;KRtRuFo1cS5InaSlm z?QdliKC)gd?_9t2u&oK_dMVE0mbS@3nsN+6+q0)Au0CiReP`B=0J9l)ZmUEGr9Et5 znbxJg^8FO8liRD_Ty3B4S(iIw&Rw&X&bDZekU-6w?wy80DtGR!JiuljaN(HD4TiHX z8ErW)Y`k1ydie3}$>)n!T1f3}5)ixUh>KDYiKVY~VT~)-N z-c@qewpPvS>zlOjWe&!*|1*rTa#h(|q!JzlEqGe8n8jm*d0OGyiZ`e2b*2S%Wc++4 zcS!c}biI>P4lp=6tlzLp_ae)!eQG*~${HL_biPpvVwhC4Z(`2s?K@{ChrYN|H^DcT zPfhf}sseTM9PS(M!s1-mrgA8MU{x-B`)`eyBt!c#K~v9knInoafyV+i*1hj~F!K=a zhWlBE-8DNq`YK{Sg}$(Ao%Uk6{)sxXqYqoQItDy{TOIdWku&oEXRDrnz|l#C+ASPv z^B7}4FuAQajWHJYGAPo0^5XiHqT2S`!8V&xK5tof?pW2Tj9=zK=l&Mkct832{nMu} z8Rq@BT()X2Qctb)oMXrSxtQ;D#WWY6{r|J2pL6RwyzN}b(s7D`$G|LhO@n81t*_rx z&K2+0omsg*c=>8>Bh~zy4*k(w z@{{+fylM1nov>cz`6c_<1uVq}*H)cR-<+0RTpGJgMyAK3g?-k&kX5f9a(8|(N%qTg zI-cv+zco?mtWVj>Ek{byUq3BTnfWb(J)(MglG>}ix#fQ}c2?AiI;ZRmQFq%Zz2^3& zwDw9tsYyJuCrvmgxA?>dyR3cc9#>s{$)EXje)W@&HyrZJLyA~Aq?)9Xnp>NaI$PSi zI@($~+9yozo6^(T+uc98cgFNNQzy-x);wj_%plZ|~f=d;5pQ8+Na_x$ND-Erq+M`7hW}@~h%*)QNWw9O~!RIrg;Y&vwmc zm|!4%Ebyj*<5h<~j>z04Y>7D*ize?lIYF=MwNvM;pXV2zj=J~AmuLIrxqD|%Q`+|X z%cn`YZFh|J>ab6ldSuSm1UcWR0^tMg|GxR&J-$_K$6EK*TyxkrNiIsxSj#KiETkLV zcieB5ZDha9p1k!RidwS-?jM?K(tT?4sq%IWQ-Qv9QNw*>a8d2PW?Jj$i=Yj#49u2wc*p_3rtHtr|!5^ zcrC86xhT$D)Hr|Rwua`ubCa!}uI*L~m)TLNskwA&V{^Wz>HBwkoH&I-`8a23-JF!s zo?RZ~IV1U-#yr10A^*$&{`)H`>+{6LY=PC{CGHWf9Huuyw7H_T#bte}n0BFXRis92 zC5P_!%Z+CYU9TnXpF7FVc0;4^b^q;iTW^`K?|l;GEGkvK1Rc@@%C+viR!7gUX%Tk4v*xI*t~=e;@mRZAw%SYEqTy*8B9?NN=V z=+60B6E~@(<)3>S@v^Ub({YiUz9p5*7r3UjUR*GJ+VtcW<5tx<^9%2#F7j7kYtQO) z`?q*$QnHZOwFL!1QHwg3{kwIuY}2G|Pb_vku#zfsSCe|swSPyYf6dOl7PGwW&(WBf zWZtz@bLmf}-~(29t@s| zxjR-Z`+j-z-dWe~L{zPOHTD1Wo2F?i=BaOV(e+5kWzw2;bm!+QrzsKm z>|5E36?WxwUNrw%`X+PP+*>=hp0@aIP&roAmzI-ANtwIdl4E%T`O2q}1s>g)6#?BBiLPUXeVvRAt&?=SCrlM=0b zb^WvNkJim~-Fz=g?sH@G^{QP*g7Y<>`(G9g*>rsU+03*FM(1{&Dw`N?w?6Q#^po8S z&0l+7d&2GYVoqKBuDx?j?!RC0%Sd?AU!8Vc8`1yk{C*nu_6sbLd!x%2qqK>00$=D} zub1zNlvXD!ENQdZ_oqKkd6L4iM+8Efsb##uu1e!OcxvTE+@HsLD|ZB{Lll{)*@$bOL=k1WTs z1H30IZO+8T@0zhjM@}dClhO&HQ1^9`c3D=R7EfZ}Tz>fF%Y3JFmuqhrTK5|%Wu*M* z>Pi0f&D5@|FTdr>InnqTw##?EEy*b6<=+3?@ZOJiO)orWJl~OSwC0Ov(3j>IHD2aZ zetc)*LzYNipT=&_+3V@O$pTzIyy_~RMl z>Wfhq&x9%Hzv^m#dg98vAII0&@O`p*6m;e4p~l2{#Wb>~VhH!s^Q6R)26?sn;CkLJa$xs7=iyY`%Ym;bWo)?}8IHawkt z3)bYUc=bVIMYtUQ1m`dOI*Fp242^Opw>90i_^1}dAGW33_K1Wif8-|bu4Udno7MHB z&0YGVd=i7AR~XO!F4Fh&WT@4wM@%ahd7D1UcGle^=2xkuc~DbtIo~g3G5)HMS#M1m zLgro(Z`3&db>^9g`jaFrwsJ{mxD`xgH+^(j#O(Bq#82}z&VY-5fii(*fO%Qn}i zF8-0Z$s6>nBTwX&iEnZgonW;(l4C8;#*>>b+={cdXNwAK6u2^1DYxQM$CGu-f;WYG zFPyGr_4Z?K>f;5mUsvqw`gbiMF!uUBj&GM1E}7Q{-Pn$e5`SsKsZT1x_S&Qd&+)Zb@zB6uv>y6nLd(@re-KIN4 zSsc0k(5=;s%t%cGV7*WRPVi6wtDB)8Qh%9&Ar}yd%M%VWXs`>C%2}Zc;;)-DB&b` zcUHxnWl4uZZ}eQ6();>eL{FniV$-*0cP!R@@C+#0w!G->%bG=(B43MS<$Mh9Ojx3F z^`HV%^fBId@za0j)g+Y_YF_!j$iRB$f(0)&Z7dAd6??z@?&mz-n(Mu(i+;{-S=Q{D zReq{FSARfgtT-MJCzq@J4+!uGAra2f$EGX)ZgGNf0gGyn zKp1Cj&e3gil$w7nZkuBvHEY?Kr&^z0?RqQrdGcDB)Sr&a1Xj+y7#IBX(~&&EJr88F z>v>P#uw~xfa$uEy@$4nyJ?_qK3^xw2wQMY1w|v*@gAPsyTp#OMU0R&Xr21>ygjEx! zJ!ieq!(nyqYwvvHn{!X;Zn{?TaM8oRVVu2Hw(Ei>Ta-)+ny|VhdKvFjljHv`2)o~A z*v@powniZe>YQ4W)++DlaHB3ePBgzO3}1122OEZ|w5M z1@p$!;`ipO39K>^ zyb>o$XP7eyT=O(d4`480$}V7L2w<&hmrhgUNmOetVk}XcQD1bhsYH;+?nmt>ah|dr zVuB0m7!I)I2xy%S;5~LgFh_tf?0Mtq4`K%|u-O5fqb$uMX zehrM1IC$TM*L`|kchR9EwNdDd1g}Ja(4HGSrX0LeZpg_6R^>MKJZcvb5azvjqP=io zh-Y%EcWU}ACl~WFyWp!Kr(Hd{B(mR3b-P<`(L13{U`7^~KyOX~<0g*(ys|SI#db8x z?O?nhZk%mg@c#l6pCqpod*_7$`9lu%^&Wk#9K6zoOmYjlAE;HU1>}kSYziw7Qm(9C z+ur_2qWb4_v9lk<%ql9jFKPJ48WNN~x#DTu2IbhzEHxsb{=b?XFER(rNGKJusLWZw zz!gw=QJ{@wW@*CpQpQU2o7eJJh~{@{@FbPz3GHlcJ;7G{qFrHOW6+1LF!QOU;bP}j zOnSVd<7xw=i=r9+5+poGk+r4W`jf5so6xHB~ zxZdx1vL{w@!j%WD{+>n5={_B51*J`ei8CiA6-+vxz#;76*!hg+#*XYQ;=GBHi|_tuwdO+Tl(t&u-RC4X-qBsEqx&ko{z7?12V*u}l+1p3gUH9(NW>C9pasOl#XAxJ{7v zT^g(OL4z=MA$88#HR-&rp374cmn%o+6>#d@IBa%)p}Dwl#=nvVBg@ILn;ZI_v$k!C zI4G#t9MBtOCB!@{Yl23%&`akNt+p~7*}6SCo-bSZG;Nj53P#QYJf|dhbtglj{5`dBqle}-7mLuR#& z?Z-QM>?ZV1n(@E>Va4*ToF!KRde=^D%VgsHcvDbhXWt>0Ty4dQouPAn9m(f>E_(K; z@41z7UEg)Cl`mkp%IdYCmEpl!`-n>E0_L03Yo#M*J~+0iK|`m^GG*%lDPu38E$s^# zqh{|}IsaDsYCTEm#*XrL2WzxGO^QkDeJ|MV@_?nZy}@Bx4$t*0q3|>#@9wS4OMR=N z7I8!zV2>zOVRMZDxY`6rtOzUX|Qaybk=&;eCHVoYy8J2TXX!9<&gFhw}?|pEfn8SCb zh}+p$eTQ9k$Xu=UewAd_>}{du5cNCaKY@5x(QTQZjS zeK;if)6M&IbMm%hN&B2ql%6>qG3md3?SDm%nt52vy46Rm>RSs6zU+AX#PM9#f?sKE z^C!nHXE}Q7@cx+^p$8qOI(^K`6%}ll9q6}8&ESl17_W|<+eL4!;JLc>Y*lB8 z*na2lyTfOA`$V{=FIG$uiMNQ{6;zu3^Xtw4E#I(`nO~wePx7&dUFP*gV&B1Qb5c_) z?l`Z_SsoFfdX&9p*Y9W5Gt9d3R=e18Ixv>b5X{M4CN{-Uu)s~++1+T$UB>9^At`;6 z_1?Crc0M>QQj+rg%aRAyQO#Eimil;~h%V^P>c1GWls{{Ct&Z2N5>++Tv)$L#Pp7PS z_u*vq&amy?2me?*om%D@xiMv8ltWq;7jv4s*B|?W(*IVGp)QQZTURYzB6-dI`l_;} zTTlLVo^#4M?(>TCbKgYYXzAJ*y-w(rUy$Q@KJn6n(*&G@n^F^tj~>r_n7Vg&`MLLp zxA8rTZF`w@@#m?F-`qEpm1P>&NHXt#uHh7TdYj;>%IBtr*Z0&rOgz7C`b?JR+5O*~ z$}7rw!t{5j@4LNw4#VX=v%+>Th%F7&vJhSydvvwO&ERFV(*goZ5}UW2$lB(cfAsib zhbxoQPOPauGW|=Q>9yiAc2m|+l_$|&ZNmOCVx}EgM_-2?)jxBQYr40suTuA$YkWU8 zZd#hyaCVK`tg{>ahUAro3 zu2sa|npxAXe*AhPdt0>R>!TM-H!?R}(%ySrl+Ew|Qg4aGc?pkarP# zT=f1wx%DZKb94IN?qdOY`}3YmzI-iOMAtyPS7m?AT+YfPT=;>=YCq>`#;^8(W5qV+1>};PZn3~ufAgzGWYg2=hFB!fib<8LVNeN z$i3M6bZ&{wnWsK?R`8|?aL;MDo2Gr{*5QdyvNqc;dgaL)cI?`bjNQE{|4pw>sLBZc z^GvWz*5O#~;<+A|mmis^_xga(>q+nC2KxB8=_$^tJ!Em^W<{L0=(0DL%lZOlB>!PK z?|kjVoVc_>lUpiJ# zo?m-+^;x%uYaj1thh5ZKHTUV7+wPZQ`p<3B3!5?P&fIsmxP4z5wk*n)sP1^z%ytTMK`_`(n2Ykb?9k>72y7}W0w_7<+zF&=7c4OAf zzJ=wPVm}U?mw45)B8rV8Nvc^Ysja!am7%4zr>Uu;Z zUz}tz@xENt=4}(_AC3KM;xw!Ct*TG3X}s;8%5%>=^KLUe)0Qx~ezIeWTU8FnTJb%3 z%+r&ObgjF(Z;jzi$>^;eQoe!7>T0guQ|Fy5zilG4ch)35@AaRG7M_YeCwF)1JEy|A zQ@*@kSHe*F@b_|_b^6;|+uxPuMC5xc4p!fEwf$PO>jFdjF10m9SN5LxIj>9bx}bRW zl*0=wpH<{s+j}M?c5d5^{AK%BI@x}I`i$f8g(B6HhWmQ1D1Df++1h>zFP~)W>J5{H z*Z+Up8+-4{mTvxc5rJh7CZ4beon0GYQ`XWp^?0epG}V$d+uhx@cF)qdvpVTz$W)In zwNHie87`I0HTrUfJ>Kv3TV;8Jo=k=VtvN9o_ab((USF8O^Y7|ZUF%MbgaYfSOL^_? z&j}4}ZBjk4u$BFq>!Nt4rKz(#Tjw5)3pZxFxxj(*))Rs7i>hMs-DY)a5oIc3pH-B_ z!!lF7c?vz|yRg@mC`AY;hF?ieNj15iUbpa?tY@;+mU)FiwN8_r9$(Cm(a<_KQ^MV_ z?4;y``f1Cb=DX}`x}40|c5+HYoYnD2+ps9hrl~FhiTvK>wngE}X|2yz%#6}=7gm?b z+a#*9+p}`{|Eg1W(iW`D&^i*IS$3(zM}1%IkyY-yjh;SPB+VomzIfgj$z}8XBCjoV z-&e)Cux_L1(zL?qZdq%Y7)lyjwua?JJ1o&~xy34|7iFrmHL%b)ecc?Z%dexfzosv= zxW017#QwWqx)v2(j?j3&|MuMSmHXLut_t71Gu1-sL zFy-N!M!|{CRA%<_`n{cg)@#|S_Wj2WS*#B|&&{wcpLs#c z3GM53s&|i{db{*Q@axl>yoqnvB+|b{N*?=EA6)rxTS(H%pl7CUvl2fRRO{Sb;~PEu zfXI@lnG9#c_!1IDuicA%y80hy)$0j&EU%}PiOFxB>2NsIC&%$u%b{&2wnRstEMjlq zeX%Zj+#M8YlD>TZ#8pw^yBHZF5{!5*S=q&C-aENkV-P-F-^y}9Gr zuJ()RvQw=Rmbgu5+@5=&A%C{W92K=?Q#U9wO;{bf)~MX^(^Jh{hK@&GYo;omi0osG zV!6q2?F8Ey!vlV7TbdrFT-R1_*;tg#p)6}yqH0*sSADH0S)#$fQQOcV#XRD$BBKC{ ziQ75b4yUB3D@}_GOO3*sqk}5v!!0^Ab9N@T)6{DQ`tle47suUL^EAq5i@$HvrO7-_ zmjksvtMB7kW%h0N)vXaH*LwCIV9|<}JeAeqI@PtSVY95tDc?#ms5-$=XyF*U4&hcCMFcgkwBY0U|ZhHYG*tUv*i`y*8z#^FtCxm`0gZ zyyT~PHEqeaYL!Pl)=u5F_3nWsKOXHq^+Ic6_6#OBMg``jVc(pd$}Vd?*A>IQ$uQ`X z$xJ~vo=tb(l+H8Jj$^dxx))@bxI=8x!+1gV9WnDFn#8pnL|w%mIi9fC>JxNfafgQe zD$$MB&YSuE+I?AgMfbQ*lZ2yq%<64|J`x+Yx@xCwKNKqYm-VpOy4a6DJnnvqTG`%v z=KtF>RVzcpEySnEh$VALcb;a+xy*5IN<#Vm>Xj4vPBrx}m6YFJx$xuC-gb#o&#x%4 zEN-xr*vhfD<3)(+Mkhme``>FPhB>Wf)b!VC$XQ&pP0uQkCAow}#BhqgQ;J8nXyVdy zTLZWl3}U<|g%sy`zbu;W=CheWVUPRPOWOt3n7Pe5GFv8Xg>q$NvfZxYJ;&#;vDqXX zn0jPdw28)h@o8;yrDe0CoUcXI_v+j)NxY=#^`!NmMEb3=6z&7NoSWEKvsIZVeb5X@ znvuDy;L4-QD_jPqUs}5g@~gC?4)n#r0 z-77XbTlv{XbMAB1os#n>Z1%T4tN*`i)wSUFC*ygqo#JQv)4c6@z`U;bYGJQkQ@g*Z zZ~OeSRZU-W$!_8LtD0fl6Yu}$*%*%d6+-N%CJzE#xletsEJ?UYiRq+DW^88%zj|rmmt5b!0~&M>K`@zW6X8FiuPQ>LL9>8`xX{8G#V>M zaNT^wd~!2yBXiTPMKRZu#bc+$Cj?h;m6$%hmS?zKA>LUrb+eN155w)wWqe1?#0wZD z9m;kTrg(mNH3MhJf!)tNi}VT{I_rmEk)(!hBZIMHU5QGJliI@ z>Zo+gHM{qzEr}n6FFo|+W=Z`O)-XZ9b@B=pwFa}C4eGKhOnH~Z2b*@RD|7jMU1>v7*9?YkbBpfv4f>}S$k{1%Pjc?M zAucidIp4R9?em={dI-z$xN%hl$Mxa=|L=<@_x=!NdpaTK zP$%zZHD{5`SM6FB&V6;|Tq+t|!X?IAg1T#ho0WGet7T|8dZ_z7mPy^>ezYU+^z+u~ z$2v{6%Y=%Wr$$b=&nV+`)0%%}S8?#tnqm?IepOttDF9vzaM*wVVALb|mGOtv{G2-=F6B z^4LtRPMz73@uupAb6-pq+&MWWc#gv)gm9Xn>N_-{KUaPDJ;p4clccV|c) zm$aIs|GEKP=`%pL99ZG5_!hyB}9)Z*{B- zNL8qr@_3oS zbkltqy%RW0R-aV8yCTz7dB*<6;%`qIBaSvq+NM#kYR-X`OCN`&u&$cZq!r+Gy{LD3 z=Q6Hov4@0T|CHHf5xUC5?0C@p9VcdG@AT&to;gpv;qS`1$30vee?`WrF0u7o^xauK z@X-SC;!YT{Gvc&}zxvRNQ}Z+OrR9w$57g?(q^|6IJV3Gq&G~Yur?`a@PFj z@XQC1%hqM3+d{_D$nr%R#dR<-yat1#<~ z>d9FDEN#W-UmIM!tS0?haKLAD7 zJLZx(R{y=`do9egT+*ba=rLo$`pw-D%*OwxFXGcEzG7$=<29krZ1NoOwI4RlGn~Ej z-uBjg%<&x7DHF`*uKAqt>qh0*S?lbbwtRn@@{3cq<5%PI&;Gd=o6c~qd$)KizjvOj z(tO3Bo@DP{H)i`u&*n|HlA1Vca@Fn`S2&k2rf&C~u|P=5V8P|>KWDB;;M^jAYi8$; z&E?V?`>t&-muhS2w%VLE-}}qlz|~9V&R&^*dl}D`?kLucugoSebZslYS>xide1gRi zR;8WyB(2^p-JrJHVeRV$C)Fo7RJjOvZ#k^FUOIc}Zp+>+**c1|Mbo&1TQavgX>aT4 zGG(115ahkmt7*5N$-JGf|L@^mDI&Ss^~bXLH-6XZ&f2RMJ)y6?_tNWDc8(2-w^Q$l zg|)5TbC7$zRMsjF?99W=Mnxjv0y=Z2ouI%jun zUH;YoUv~GM<1?5}t=;-t^NaPo+f@g%E7vL5>^~n$JTR` z)Y2KAQPbm=ZtcyO{+)ZJx=)R=cGvPRTOYsbFt9xEeA}A69%~$0`s{Y&uc(=S zP8~XPE91QPzV1`qr+4nVy|cJymdxYX+v}DeYLVL8mcB0{$FwnOmF?*XUtiDgwO)1f z=6;7O{dZ2v1}KIq81C%qKCt9U-e}>oYhX=PAdmRZ#nsa020pULfSf3u!o-^S}mdl?>`xLY$99q5m?~LwGzmENX z(G>rB*O~9fPtQ8G;Z3BmwQA4uS&q|c8MV)_XVe!zXkFkb_- zEI<4xY{O^cJpy}|Ja|3lZ_*?o>s1S8ZuDQZ+dpYVfiz|m3z+(X3ajg`qqk%C(mF0WpriFRo}T9D-QVbWtEKamG9bo-MTwCbC098i>B}K z=W~`eWKMoGeVuXUUYqJ0|9tkWX1gQNJ#BiF!uJzSYYeA_oH=>s_JOHh7h-v5OA6oJ z-WL7$5 zy!+l>W2=VS=K{HVuG~F1TU+VMgavGu^EMo`(kgi%Xm#qsrK3wu?Mc%*`z4*f{fm8R z%hU~H`nzQyN7aokd`&>5b@^!QS%5@&|PVDOrO@Gt5O7_mq3zy1nt-5l1`RABr zmeISTPqutrdj4?D)33rekM*wH_HcI0-|0@1Za!i=)}b!E%;%x1u#tUu+5Jh$rE#jP z4i1&`k|iEZ?>ux>WA1YGwf|47*ST@IOJAvJdrjohM|vaf&RNzP`y^aaBX|(cHqK2M@xN<{pVmJ}R~($TeeSDDR8=(^XC4 zdmW#u(F#7J8P7|0HV1CM8s>QC`Y}c$l`E`o&b^a)IdP3FPwK=F(;u?y*mYI~ScWyQ zyin$SVZkv=eTHIS#<{vFwSmS>ZFQAJQ&q#tL_}=XoDf_Sq{`=C{&`Yx%<)KxNoJdZ z9e+D+pZ0fk+fD_Mo@1T19REKTU;SPj5%T|?X-UZj7YD}kFQ4B&B*w&&wyl)qwopA= zM(N^u#lAXLjc8*#-p}EawiJXVf4P{N8{rcy{HJZ|wN1HdH;rGkBpc6g{%PW0dCdJp znl*#N*YzG(Y;6{&y}2p-=7eg`C(Zk2qWQYzc_rJXrWVzgurlm1xZBAq|1NP`%wv&^ zsBrhRkPs0T^UdMOE1xreQ}CNEBD&CASd}I1wSZ=0nw#zZ4{h-t-wp(C{aWb%IYRG< z+tmaX#|_CP`9Ggcb+&z$(^%ZQF!$v8KN|zy98j~XT&G$o>hUFgnss*Q_dxz@)n}|X z88bNwL|j*6%Q;XMn;z1VqA2+w*vi6DQs}S7x0sjA|G%ZD`)y}qP;q7HQT!=+=;Il_ zZ91INE9t=`# z3`tVmT|K>By;2h=_fP7co;0;@M$fc4vuDnoKXKZUc?`-;t5&aRQeL?_Y?U;_CWbBB zw}frlvD&L`Qvc*ROAalX#=zy+)ZHYwii^Q3ySXEARce#bm8ST$_s_4rle0SU4ujkC zj;7exjy!8-pIsxj`qlka*Ha%d=rr9)T=y{exKgWD1?QYOyIc4eEQ^||?g}4q>sjUe zSwJ>wYvB>2b_N3utrEs3u8v$ga#<7vl-&{}+F567dZlRuFU)v0<-dAy$hrB}&5l{$ zxa9YRD|>z0bTvEla`fs4u~ySEGd3=qyJ6ju6{o}+SQwV9NOE4sb0o}-`PRy=m=s1kUu;<%|Xt5WEk@YBXWCO&3P<5Nm*u|KZ)-1C2c z)VXP)hMO9Ly8MG3ldgs;o+{=@dbhFTeE!xAheGuHI!%>ARj+MX=yRS?pwXe3Bki=; z^_}hEc5BucD!RLyse3j(4LsW8x`BbAKkv_i@4mI_6U&!4TtBf?H9Erd%Z%vUPR^d1 z0}W-@6`%ZQRdlaedBrEH_rQ|pxtmvRe>h(ut<)!H+sh2$`KF4k{oSh+rZU>?sMs3P z<8#fVr)|~W&hW_%cX#p$C@`*+aPRiJ!c(dKbZ4M|@FZg%uWYT)zT)4nv_FhYDz#kJ ze1y|1URz*ZlOqfN!^^DF%rm~kxaG<I((+4Ax4TZV7kR7Hcjaz%({j0K zS>Ug6YstypmZ()q*;XstU#f3;`b%$(vW-oNwm)O8lHbgGOBQK8%zdg?$izKK=irYC z%~{tTXWY%J&R5zrTPx|{=J{7-r1!WnaY?M@VtcY7;B|OZoU)tQsY$aK?s=QBM_GN_ zP~&=4!6TEJiLE-TqU2mp1WCC#6bK5`EO5SI z&|fLg7Fna|d86RUija_&CcTX(*dMA$eVOq=$KB~6o9>B&QwflrMfIpvaV#*`T&}fIdit)4q^Z(hUlunCT@ijA-l>23u5@Y<$x4xP+-&s6L|M4`$>fYn%TKsW@ahjdw0Pc^8y+r?)S_Q9M!hlH zeoE_+hUJ!pk}tfLpVB=S#O(D${&mN-K0%2!rPG~y&nDg6`M0-eilx?*6Mh%hh&AO-Mxv3B!OFAhLMgK%n>Ykttvu@F@^OdeQ-iKkEz4I) zq`M|Bc^0?&PNYt&Bd6B2p!i=;VpROTcgL+$5?!y6VWvOTC$p|woxyc_mRQ6&#e1o< zuGa8Ak|^5mDbv5$%leVpEtjCZXIkcn*&g*$OP2e8`3g^Zs?o~lAzkN0Tc2B;%~`Hi z>~Z|&)XR#SIHoV?$z1W2A*8s~C~Kiri;V2mHj$>EtU*EFSN#8~v!J?DnwbkU z=up+ljDP9h(RE_W5?eu=zb7x8`F4K8y4JNhAHQ!-xF)pj&!4p&`=VTD&e1bj>D-kb zdNO>$5&u3-r#F8hW?o%uHeuGnsmpQ=kB8mT;$6o5_nN@rtw*)o{%6mBVHzHD-7E5j z1?R5sR;x2UYx-%_`E@VMeI0T5cc@NX_=<;f^md(?ZU6S!`E`G|UKk{X#|y1B-w-UL zlk@lVg+E-P={2glCi*`^<{t~&V6{2l&H2oJZLt~CYIdc!2%6sBac{f-oY1vQj^6eM zkzE8W>++3F?e8Sj5%xtFIq*O&wi{#2@7ALkJU6i}>bd;2E#m1vL zdG*QFS6>EQT(@{?-qg31SK1#5ue@%mH8o!*roHROh96%F_j%1p`*EsmoyN8K6?;w3 z-1yM5X`75Bv#Qo3QRnHqt+`pfo<=;p8utD0A)g2JZ>Pt-+Vttvw7nlDOtgvN{O%v~ zzh~VVyHxXQX7hiUgkmrLzl5w>-^)U4w-*I8Y>H?`o; zq^uX$s%{=T(5qBE^VrQ*FJ{bd7mZzBJm=c`u;`ULdp~8*=Za8$U3x%p-R8*nF3upM z9c$w5zTI|mS8}*i*~O~6pSDNtiDpUPzFOj%&JOdx&o+OP4CGWR*?#)WmdfJN{rkTa zM?OFEs`FRqUE`^RD-(Ic>XX0qnoiT-#((sN@_+IDFOQWKaqz@t=a=1-nErGsd-ds5 z=^Zol`AgPajVZ`WpMG?%npt~NjDOPZFpUFloBGSwhFV;jF2t*!pP4NCWhVQst?Q4y zTF0CHsxSWAiC?+@Q_r``U2@)j$zLh|%fi_?r?n^TI_PNDJYwwlpo^jpkpW!0sJzs?`?_L;t&ob(2 zXq684mG`s5c9b^eeP5X#)wTTmhY4?Q#@c#c{3O0?*^ByjKl@JoI$pd?cdn?8y+XG9 zq3G>qb3UK``qu5w=^J-zH--N%{CaA7y;A(g*uN?966>qahnt(uf5~6+wK?+r>aT9o zL}R_(s};l>o_(paTb^?Jc=X-!1ZIu@4GXTv{4}X@*YVh%rBkE4mgl&c zr?DSTR&2Uio!%Xw&m@Keha> z>s9Nn*GVrg;@@5nUf8T)(fmTKAb45Gt7{JL-NSEAZ;4K83ach@tNA$L7UE&fYmEoPQws(YRq|BWBztO&%Ra!tr{IY28PDbWJtONq3JcD=V42J@@L>-Y4qaWjlHd#H+TacKS|h zoAxZ>Tt(E(>8=dy9foPO{m;sFO>f#)p2a5Cv*cOxf%29&;vGqC1#{Xeu1>Fc>d`Vu zy;1Ia%VG0+k&e<;Y!mxdwDYQ!-}v7AdvWULif&1Xg6!>`Ev%_mUu5j&D9=A$%U@dc zdc`CbkBNrvHEz!m&y-DYiD=MUmO4SBI&xe0dG|^4-D{qkCoeo)ckfzR?DYJa>$(4K z#P&bydD5lP_lYrD>E+Z0mgFn)Y;2vsrn2RB3+x%=)645wU798l02#IP2za?ecuy zvg$K@4XsPDhn$Bdg^1jm(75g|Ql>TV_ zdOXLKt;tLxw^^c8=y{0o@_FGWbFC%QB!0{YOq<(s!&rGmXJ=>YkD34D=2&*kemVck z_THTDndTn@Z(Qj+#n$-ndR21y^ed7*2GetX{FwRX==6Kfd-qkepJUIkRG${lIr+-Y zigT$Nwvw^VKeG>gsca5k7<@f^dVBxcZSA2qD>q-ykj^ZMy1DSH*`%2>%GZR()TWmv zb%YpuP7_^Gem-sTVULDmFDFV|pAl0s<&Nqy3&|xsGg4o^$Yb*C{P#25@6cS`Z%IqV zmxRqsKa-l&|04G2@gAO*oJNhQOq?xZCubLFOf+$ud~@0InHdu@X0*w?T)`>1{0_&G zkK1RJ&6-okwczyh-fx^sMJ0PLv#;2{qIbGg=4FY+=IYb(mv^0!=yNMqTAkImBVu`B z=KqBmzq%Jotze7j+`Mc09;v0vR!#o!qNpgm-_)aAfUEmZTi=;klRn<;UcNun8~LonoJLS#tVkul3w* zdFLWKx-#1SoNVD!pZl_NO`qk02(K9iPZxCkSUD?e-IJXhA2)HiIIzrE?p$A1a^gm- z!cr;AtsAAL>)Bq^@iW>avD8OdI{(2FpL0`Ow3Y@bzFwB|wCTRo#9vXX9%#1j*6fO^ zDCL*PoB4dh;qv+WPa6Gj*z#XUIM^ue*~*QN#Qolf=2;&}QrG7HaC4Jih*GuF)^&mj z>$W7iPYDwJDpc9@U$3?$?)Zrcj;L_0WvdOk(yml(^Ol;kf+JpyYvIb7O6ko?)GM%%9XsCaZ7;1&IP-# zFf+tXn87XGY!DT!k}|dW^x1%1`EVv)221O_H6p z{CURaqqioV)vTFPUd_2<`5viFD+D$Q8=Fi%$h9wlucOP=i0 zoFmO{YPRXYp+kx?KNhgqWv!j#;J2#5?q2X5mn}Z}XXKTi$z>ZTh`x5;AMCySV6Q!;dfED}n78T4;rcJ0@16e#iEmYW@<8>)y$si$=giZ(L{iZC1A3Orp} zGPh;g?64D=Q+}LbS>EPcH80sYKG=-HgMf5SY?_uT% z%cpHr-*fJJla2Rm^X%WxKDJrl2XFA*1tUaLdei7f> z4Q%Pgd#^ORUogJF`qqVBkVtS^n z>YKC2Y4=+7T@#WFHonWgw(#S2p(_{fC&;AyO0nq{YdXN>#^8MF5lecY{3XWo3_e<| ziX3_egft7p^qr4z)e_tFdxN*u|t^^bA6*(f4lk|O8^dH|PT9n8ttY|o4;WgH`0%8Z1UT(g} zUT9;X%aQni#qYtnxhxEClaC}P-F_0p`G*0q}R{SS}J3;a%*ex))f&?+5h*> ziOo1WXSMW;j71Fr?++VH~vs&q=7#G1uC8%^^JB?TVJ|B}8?#_OeX`LMx5 zmiN32vmE(U54uR^xm(=Pcs5z$N1^+Sg)RTLPi48)E@_itU@<+Qx8xVcb{6g__0ul| zSk`&o;?Q4U<@Qj?gYnXgzpGYOT&z%hl<{d_45Me~<$4y|YmYvvC%xe{p8h|8Jkzk+Vpec8L932q;ABVY4p)OWWXvI8gw`LAe&b~@cW>1XE(QP zxU*_a!$W-~jtiSrWhTE&?hD?N!%=N>RU_x%BadW<-O>7C3s`pZt>a8T@O7`z?nC0eTMoQs zG3uHvnD29YtBnzZ14}^SUF|K02DO%tsg!L%qMMxq9BH=Gwv_*z&Ke%vzEwiyy;5h7E#iK1v~$Iw@1JJ8y}NAt-<^wCbKEb8 zYe%nRJ7=|oUwUo(=XaKyzvn0(44<`ub8)EWG^@mC&vcvF)+8!2s4kosY{rqgGhOiR z&+IEbDN1Fc(U**$PD<1dU{qYjHB0ctnYIsYr9MAC#_B8(2!CnB6`FNCF1$Z?&GV9H zHY?dw`C?ZU-nhPAg58bRYTHY%jjn4iMXwf~W4_Ad{jr*=sT%)dGK(3@Pe%j=du~gA zd1}tGkd?NT^;;LMjk>YKyjw^;K5Y7pg5WjtYMWHIUA%a6a*U;tD(~qn>*j60)iuwh zPyYB^kIrR$y!@LF99X)%syFi2eY;xml`H?wyit~StUBA!ukO!vmpj+bmU)>9ZaRAL z<1(cj-L5A;UeDbjxvlSvzFyCqH=CGr_CEc7gM0ETv)UQ87k_ia8% z9%TG46!&rYdG`3!)aJcT@d+0#HLex3oBoJw+^##Vul@4g=X{AqitRUiMXo-&ZuE~s z?Ml<*rKvroTv`#zl~I*P`J->ooi?|h{b-t4Z_&gst|SSk87gJ#UVc@maLi&4``>t4 z^!t^GcVbVZ_C}hmypSE{bRw>)>EYq%Nmf>pQ;XE4RPKev9SfZ>xAdKucy@+btJ3t$ zb;qvf2!7fgH*vGhk z%bkDyT&cf1a%uMZ0yCza`Zf2mrsae?H=TYzPvqF@HCq=rzAEVVFUwpW$2#}-`+mNQ z-hx}pb@$x-KQ}bjbIFN{dp|z*%uKyKW1D8U^qPW&=T2{!eb?!Z#Bu5MD>k2fxQVl* z!~JH}n{9{QsobnAu)O_c!;`*j!z;l;Z1QU_P3<-~`-PKh)g{S!E7n|#l>Re$Vn>kH z#>-60)@EIJD;1l&%saTtbWQwZuk!Te>nu}G|2y~m#pQxWGpFrPejO!SA}v>8(f{*7 zso5hJ_ByWBiz-!CF01|iQ|rkT=6iE5+*->0@``)wOO=UvdgAJJJL*<0U#xm*#jdv( zPQILTKl#-T>lyFu=6rtA({{Y-sxkXX#aH{chHh9ob+5MBKHb&bW>s?Y*BX6IVCE2r zSJ}NM&^P6%oLzqJ^L^%fs=8h*`rme$@2c(Un|AkgzKJ*M>fSoJIok66#}k*{C5hT! zlJp9X|IB8wPE<81^kg+RBSX)TYl`n;^K|tV&cA+W(yBKhFKXr92*2I%eAl&$=Tjsm z3m(l{s>1d#x#gOXsIv~U($xj0{cR3kza+!H*8ieFP1mn`zF|$$R|C__Vm5j{44C4q zcum^z0|!Oc!-UW){?K z>i%Q!}?80uj10TMdKi``(q4LV&o+-

xoC#4Wb2WThQLR1#zB*(%{VT>%`@S@*P_BT(`)*TQq}~mcgi@(dC|wk zu=)N}qn+LpN;B;%m&7vtXFtFpsHAdh@;x2XXS#iYXIhuWxcUb;>2Mx;B6T82)G8yz zH}1=Yel=m)?>)lvbm}f796BN*TB~m7`>$}W@oU8|7b;vGulck*lX|xHR@?DQf~&j3 zZ$EQ2xHP$T!g0r~iDx%w6#M31bLpG3!k6XXrc-l!`&uqpUoS7)Jb#a2z{EL`H-8sQ zU3~D1v1_38;*Aa^TWt?I$xJ*Go)u=MV|wtI-PAd9-hDSO&_j{Ip zzsSE?-*r0YnMZD0qG>E5V_S70UE(T7rs|s3KM}{-J_atF@mp-ITawpw6_e&uFPa2* zDafyWcWA4W0TVC70tTf|r+Nai-JLeFZQ9?M9UP{do9bG6V^!(uT*fC)#O6!O26CL) zbc0p(#)VU{g;&BB-~Vz%Gw{TcZyJlI^m{y2-1oajS|mi6A;G!4e%9pMn^HRV=w_H# z9x`U#n;K<)HY4SITF{w&oA%$Snw9hBxR>}%Lzh&YS-1Axl-6*p=x}4mEkB=q;s4*h zMYHG0UNn1oI(F8NFUy}*yl~d$e3ZYzai7+-^`d7K*(9pXw%nbf(DUMD>f*mCw9o zb7HZFf=lC661qMs~UfS@z6S{d{+=(7m2(3H6CLPA+JyX8iER zEL-BjR6Y}Hy`9IB7H(d)Gr6c~ySral5pU-^n}Vi)yyxEDElLPtTx0a)bw+LQtRoBz zhmOjh&3V@qeQuwbF>iv+WAVw-H_vN)**ZD#?S#csw`sXsF1cv+{^uiWp#nAs#z_t= zTeiup-IbPa>3CvA%HN`E?OF+cQaVnYnzO!mudBxo%)M;jaJoYPfC-?p7Z#K z-R3}H1y;=$q5HfRFy`vdYx#N2;)-#Y+tI+d+$l>e&q`naRr66UKJT~HIrEK70=w2d za*sc>LT$o>8H}~}obFdF|Ia4Z7yWov#ERnVDItdy!v7%47GPU-o5@ynkld;;-l4@gAS?S*Ogie9wD(wLdc@ z1K+N1U;MSfcjMCEeT>JduYGiTy8f9c_g%H=C+59+^~o|?H1RQ4{%Yx^7iQkls#_W# z?vxT{`0=;h7aYAItv>Ele#_T9I#fJtym1P7VXh6|tytw;Yid z)lOyUr@}M*GjdOrGu~8pRFsH1)fRv1Lvdbv@$u#53<@oW)Ph9HR9~rBDTO*NKJH<9 zLMJlV_T1G#%N2g&rsl;N<_E6`v377|E=$y$o}5&kyfHjyt$BXsG={`)Nem9T`R;8R z-x@MRi~HMaa=#_NGiCMc2unN8w%{rAuGB*9bn&ZZMJLiao-Hk3Egs*@78T!~#OZAN z$0TIcG5tx~Q;c2&Ie+)6HEEu8IBQ#L!b!)Td?B~&M;z*DHCm1p=OX zH^(4t-cdGIhw6*rnL!^|kGCYwG_k5+bU+%fHcZMk^-iv{JaA6gvK+wO^GEu6}^ zuPj;mLz?=;-tz@{8p+-Z;SBp>M3B`?NcwBqq^4{+oK% zKU2pk%R+wsn7AyYf3^qTwTM2c6ID*nA{8Y2;=fgCa%L_Q2~?fYp%KX~*HLd^nAxz^ zzbiP=yx7I&hDd;lxlghm`!N&Nm7f1Egg%)bbLDG-|HrtUo4sBgwBt>UzxE+s{b`t| zvR7My*VO|GepeIx7YPXlMYkIyyB+Yi{uyMjP_^56iq~RurV_tYW4Fac$~%JV_?|>O zS!5RxnehLRxrK;-kqY0`5AiRr8WeU$=SLdd65!dFD*b8V|CCEd615$Zt{2WcQ?CDA zblTq!0jrjadp@1XzS;7~Rmazlr&VxxMjxyEb69Odr;3kZYU^i*?JFXVy_m3tkw-Wr zz~ySdS<^XFHt@=;O$^w{c{wEelYsBnqXBQ(dp=eOt#F(u6*1W*H1I{6?;`h@lS^io zS=3bowqGs|@W+ZdtxYjTrZXozpMBXd zZoY|Y;?1UC6&0sK;$Euiyt){(ILP3Zv%}l(K9Nu6ZU5=$7hYPGT(Z(_!jaE)6&{J_ z9Fvc4ou6~LfTwAWZ)mAbOYxbHu5z4{ZftRtJ~HPz=i1`h_iOId$}OaxGGfqS(c0N;`S#?UVL$0cG#e(YS~uiRNl*3{L8x^u@u}_ zpBC_|$twrvxJfpjeKffzWx0yiqM5(UoZT(kq?+58Cag&FHtA?CES@r7J>jy@ z)S1&Ox!KHpCA}YiTFR)p)Pu|6^wD7do3r$`ER@$;;L&bTW~Z2GA2 z*Ydw6^Ag`}P2mb&(|oMVGJ}1dcFkn^-afI}eD+iCiL9&BTs;mHN6uTG`cG`_<)71M zO|*N!?qmA7NjqZV8%h7Elh#OWuCzN`yQjn?;#&NfEh`tRItv8Pe##Y4Gc)zbjJ$@5b2 zZ`bD7i&OSHZ#h`tdUTib=}_mxNt1#_tUhoxAN^b_J;j2hJfPuaf%5KkFC*odkGOa| z@||*JE4$0WAknI0PaVa#7OXCFwhHkHncnQMbJm8i>E+$qr%hkqu*D_r^O|+cTaW#) zUioUxx=ZVxzgqP;US)9EM3r?*Z?yN0A692A7G~|*wKKx*r|~YICzVMhyG~n}#BWVF z7aZ4ETEN@B-fGJ3wNsY2PO))~wy*!a`^77_2c{NRZaWyxPVJWpKC~-BGJ9d8$2_01 zlFOG%{I)1}zut6_Y31J4Q46!0d>^fu*kpIOD=5f1G2)6x*CM5D+M8F1h5k32GWoOF z&dv=z-a8I(?S9}`7jiVB@4zh9Y29<%!)LKd-Q2Aac)f{#iPHfO^C{``^QCuQD|K9Y zEAsyAMT~ze{Id@yWv`z2dw<@M-4TZWH?OKZpnqnY;}UDFYDtYb3*Dqwm`Ll$zBY`| zHnI7=W6di+rE2|<-#f)$)+U7>`84U!ya@);gQKB1seIh5Z9(n0J_ftx?_w3dCq8G9}UuPJ+C)-==oaT=A zkp&Gt6Yh5(u6=P(VUNqwBLDv;t898EZh1ACf0e0h&knhqt=o>J#&Rc4)7EmnW9R0x z%`s+q^$Xki#_HNaYk8~pU%ItD^!E~>IYH_w1(Sa-3bH)KXCA(I&B`UgOHM_(hw=Ce zecBbsV}D9#Wm$&_m+GOGm-pc5A{dotccmF?%esW?a{|R`o62H1qJ!edikD7FaE~o^Vc| zG1%ouipMv@xGKdAo6w&*yO<+a*lu3%nz`!E>k`-Q(hoTo)|!NASM0VAo;P92k_pSF zWf;aKd#z*7TClU`RHw`7O*#QJ6~14d9<_LVW~=m|wr$@~D%B)(M^7v6+sqS( zR<_?*Wz53tF2`wj|LZ=Fx3f3AZL8V6L9=W5sb9yx8}Epc)%}%xtMaBxh;QnHoguy4 zb8O%IzhPiEUpr-qQQ)o+wU}LrOUs!8ztqg1W4=4cCQCbX-tesf>EFE`{}s(c(< zl58J`$2*O@uHxc-bLXv_n>6bJW0_ClvqqQp1q{`~FK^s2v|6NfWmVvv%a_hPx}UrD z=Bq8?&mX>SkC?6N%DTtrT4BH6hh(mXj3aK*T4iyG52AN{NO~ABEjrR^(aYe~>HXH< zcybo7xY*XtPfa4+A-!i=PhxUC zR=Kn9kGm8&qi4x>r*{krxwG^fH5D8^4GK-&3lkekp2)FXzW2N`@p^cru!KO)!vMBM zwoj+^%1W%QJCYmsS}%5P5=sc6$vZ#r^vD(r@9M8~%5E63zMKTEpev#kGI3+ky`{@##@FCYP-%pVrZq zs}|2SDFcj zH-?SUb1p?pSjMw zWpjI4I9sOg>sT`3MAL-rr%rD_aqi-Ui5JhDOgeq@|HNx|E}y!8>&o@}PfnhHaOKh2 z*T-MJJM;GAwVUTYym)@&-KE=K-rxFpZ`VoY1rGvO>9si|1|87|Ug>$xo`hGHn+Be!{wU*VMZ<^^f_=YY24|Yw>wcfTb z{p%jHY~6QK3uiZ%X?5`^7$0`)ta9>c-JHLin_A%YrVjq82CM&~&teikk|srVy0@UP8}1@;n?Qs)Hk zUC9<*EHrQGEbEp`70z<;t4oy{PR()i5YVhBny-E!n8RzFoWI>GRAZ;4}4 zedeaiHCck5fh=7oTu(@eo%NVFMN_j)yvWmX>Fvnqw4=R+TU}M9o4+>*PbhF+SSD>3 zvT^6E&~VeQs@ciC9HII4e=kcgmGr~q&R09Nv+^IYGEC*RP3@}bDsa1{ zduwKxl%iy9w7kE}Mfbz8cHbo4C4YI?(qFdb*YxQ-`xZW$uy^^~qU^bua>COhQzTcU zM(kmn8e_UfbWP&n9ZOecXC*&6W-xDhVa39ONBOj4S~!vqALBJWH8VPjp++#?OO{*a zbL#S|!UfBEFO(K%b4)J!te|<`hl6dw1|I(dPub?YHCq~3k$(RAJgymfQ=c!cB z)XhzlJ2B|A`$IIyWK=V?NFDHq=fEdR$UFXL@1Tx+~8! zWVMry7MewTvu4_R^V_CeJD=it>te6!&Iz7gy{JLXJ#OBdE3@Lx-Zv@F`8+4;rR(V%vU6zG!@PI<(MXx!Ld1b(;5;rH|}5 zCcP%`O_luXN!MbgJFINw{vXA&XU3Zop>}UsU+5kADKRa;qc=}5r=fFE|JE~13U5~O zd)`orm?k8$X48)iEz&wO{_k_z#M;>7bn|k=;VWTkNk`8m-&p)GZ;q?W=D+S~E+2c> z>pf9wa$MuO!a7-EDu=*JHv28?)fB6liSC{QoMunI%}THUf_Rahf_w0<_wP~qKw%mHYIhP zD+&1`xwFWEcM|V~=5>>&1zsp!{cz2jnd$#02j5xp-IAa6Md#fe60CwAhm+Q=%Zlj~ zXh~>4QP8HFyKl-8o5Y(gzRx~~AKIiIV)1lg_${`GCDO|!UM`nys;G%eJcA-j!cg zU4OS~=Sn@(sZ}9MLi#wJHhYyiC%LY6xE$els7`m4h|BH&?4n!FWLaM8^_wzFtM}IR z-b~|;d6)08&t<(jbM9I*WfO(;(A$pnen!dh*^4*~Umv+-RH)TyyYldg9|x94O`K$8 zqi*$ZS48)&X&pbZUqxrWI=*Amxy8auUh0a6dEN;8vh*dl=J89Ou^ZR+I^HZ@-Znwo z{gK=2uGHSt(vwna4#{OquHaJn=j3_G)UJ2uf0KOX)i0c?x-!?VWnP{BQhJwZX;H`4 z-^D#AOmJgO)yE?6QYSzyUohL4RE#7m(CR<-|kF#KW*z)abpBOH; z4u0q>airWgjdAME)gsnA4q5)o>j<A?ZO0>{gugWTKr!v>Yk~9 zr>&w&Vd?LrBh!p`EPrc}0{eaNfDc^SnmHk<3oXURIGNjFPYvR7FdBxtLC7aAF3fGq2 zdUX2yN5RrHVP)B~VtF42AG=j?U24-hNn2}2|NNV~D^Jc9KmBImGJ~~e{;bfrZM|t~ znDuqnuiKKQ<~~(A_i0+pu^U0X+~;lOM3-Lwb~W+n-mG78+poHu%Mr5GO>^Lv@!e^s zeRNGn?Ef`oH|D9!X$C(3 z+4SrCucs=;Ka6?b`%Tbid+?h}d~>qg?54P=xZFQipXqQ}%VH1PvQ4vs_r6L0xbes# zTeA-h+Be&)XV#kdnK1LuiTHnx^MtpIe!)Xc>83}eD-ODTXgVn79`LL*=l$-@%Od-h zoxEgpvwq#q6Pr#x+1*}J+xeuryv_Oh+V(6_f9_`c?5TKX>eZ*a{@zWQH+e_I zBe_4_qIywx9?qF%Y-ZXo?Q_+U?ate!Vq2aY@|?I(9Q#V@`HLCS>s=b|y?bTE^5XCu zP5<2MIu0BDt=q7ebJ5QK|0}+D>hADfe(mGh>3f!MQqff}m%l5z*k-HY{w>#ZWB)w6 zY_|999*eAROY=4#`*Y=LzPp(9tvkQ3Z3;2h zbrCD-URLv?t-j4PEKtPy>(qwj&XtXAb6^bs@k}?2YPEUn!uO;l z-Tkqf7f1PO!Pd1ml9!+8IK$!~;o=@G*lMZazthmq{Ic`iFL@42tXI4!&S+`fr(~<= zR2O|X+&VQ%>vCno_jsA$_HWNirI&Q4iTRs7Gk5!J%<$0q$uyfq-s%R;(0ebI*Zu(?zl|sYHas5@AH~g*`8Wc{=J3KG?eQwf6fI- zD@&Wg7cxgB^dl_fcLa%BP3bFtB<{tm@wG@UAd-9X44z3J)jtRIn@<&g=n!y}L&I~k zeQkTq3Ss@q6DBtkC#AONmH+G(+d2K@W~IHXML}v|8_i1>A4@wIR><4#blOF5au7%J zBuQ>HL!PdgiuxzcHoQoZWR~``hlqkrcz@^T_(6K>lUqD$6lfK%=$%Ptz zZ5LR3HYv>$m@o2?wfTh6-;2uqGh1#Xa;uh1{OmY&+adl5FJ^9D$|?8Jc=b*(1_8F! z2`mYg91AvZB?d@nM(Vdb5L&%6`0FJBm*=*}#4X!P@;_hBlyz^?Sd}@;L73A(`30kj z@Ij&fmYm8L)Zbc)G88a7I*3nsBzNP1e8*>vHUs763GxOPxt+Zxo?oDM{wC*61=hKd z@+&o4cw6T?XK5MLk;L-$;EJ|{dF3}b=WgT@ z4s@S+(dc=SVA3tloQ;}mE-aqBSt%!R;k&{Wu2!pTS87as#o5Q8=`6K+-{e&@7R-O} zQcGqP=i7%XLQdADMZ{>|s;fCB6T{rRNz~~y$E<>6t(9C0;;jOk56oL1CELotB65(a zKa#EI($Xon#Dp7_HVP=m{aSt_kwemeYtDlef(ja&9b_3EsI+|0i&?4LX2{fPXyJEK zzV@Q{Wf#HPLis)csmYGhH63NY2(tBTV&igD-Ee_(Qxb>uC9z5EB@rdXWy{hVQltM~ z|L^DcGWgXd;TX$>{Wk=juVUOSCI6*k9RtInHiz}oqBvXx7+=1cowZTvp_Gv1MY$P9 z9G)ANia%JWY`uPV0>{(~+?zHrN`DmWeV`&xxVlY%?GDohPpRcw9OO3}Ft}b4-}r!o zFPq!ph@eU|$9t!BtsmC9u2#NS72NnNsc}iF+q3dxTv@lin&YLVKUrDmUn6%;*4mX57;+lq-X2sJFOblAut|5e;G|OvryLYp z8^EQ!fz5O^ulYn}p+H#&rHvu4B>y(bMoDoAzv3|eD6S(ge{BNmN(H5=hjN}DHm`pz zo@21X#AuUR^#8(rJCgTLO;p}qek;^>&GU+&mI>9`OWCCNvpE9iiEQ_L4V=t)NcyZ+@&hvMg8k;2^_ImsJ*@814 ztQLwApYm(N2B}3a8aM2l$uQMhV#9{jtq)cl5LnltF?&|pnpab2@V5DjdDkRQ>0EqS zqWR&%<_+?yr`Nx%SW@S>mVseCgY!zAJ4zuhC09+){uUEQ_2Rp(;u z8IF6KtZVnEyZ_qv{$iyII0asAfxdjYU1U)Uk zr}*S(PHtz#!%a2|So>-= zH>h9fcynQ@?27=2J3N{{ea=~@L`-hHYPOwaZg9cQMBD9CJQn?L&EV|Z^2ajoL|Nv} zei6y^M78=`qK9mc?>!XT|0}gwX!^}*K^#<}#4JK-@)D|Gc%HeRx-tBdN- zO^;aq)#66wjfgGXViB2LD?4;o^*8LERwB{5u|qrHRsRl8nX2Tu@gem>%V%bU=Fc@P zH?AmP^T;={X!=(ARFiAEOP?XDUMRnw)yHBhhdR4mC)=GGyA?|9maFD2eEPrHDP+PO zU(bFIUzJ447~Uv?`_+Kuyv=UzsF_l2|ZZ$Vy#%__lSGnm~VZFytg3O zM&yKR>gT$ny`g6`$^zY=Wqu3&)z)GtccE!(tEpO~r+T{6vAoS}K3#cvhLXP=ZNqwO^JMGGB;qobwJCY>lRb9`GJNH_x z>fZl%;XB?4Zn=_wJDH)Z=aNaJcVdrLMLSDdat7ap$mLB!{CB6hFPG@9a*n*x_%x*G zY3!2|JqM0evfTV^{{Gp=1GnOqKVQ@S`dWv#|3B^3;gx$AKAWRw>UE=T8++56vt7O6 z;Vyf>2DFvepUvXkHrqU}bwyv=1@q|p%j^YKwp(9$X!5=CP*kG&@_U`rCY#A6_f>Sr z{jM+WmwCqL;aUIjLUg&#)dr9JW394Rbi^iGTXcuknwrH=&$v9%QTDwvZx`Fq$kaI% zQKbbJ&pmeEKH>l5gVW`^UWOl^(O>q1EhF-4WKL^A$op`$d1_z(hD?$7EnEI0)>G7m z)z0{MlNINyaHapJTh4tl`SJWv{)Jg~U-qtBbj|nW3ctSnOO_i}OtF#PRpg`hN?{{^aB}EHnRmeQ&*L#|+N9 zv47Y>N7^0&>oxP`4oIHH-_?{)ZuAbhyc-hvQ zb5Gycd~MT>+4r9X&8_ph@aMMfut>6xy%Wq=6 zj`Jkw6izHSxVh}TjCsV=6Re+~=FC|Z`Q-nmo!!TKo*4H|cd82B+4sP@R*_k#`?_AF zK7;Y!leab2EOhT--r8Dyt$6nSZ1IPzx?Nu~w0gWOWnWtuEm>UBAaD{K8K=I6PlEt4}>Mf^8o(y9FZc-wvl-dQ!#p>gh!zs~6D@NCje zPVZj%(X{H^vw$@@Kiz763+e9aS5AD!b9&=)zrq6w{XJ$Aw_e$#!N@#yj(Ps$N!;2|w z<=o%VYIH}sd~fcByzl$^@0;#TTDROdXiaUsk#$SU*6_RkO(K-Huu@i`yGn|s|!!2FU;P*rs7iPqO(T? zIt;{?%VeslibX_MTs`tIpN~^?a_X%;5?8y0w>_CM*<+i@65rClC*qpsMzT97S}dF% z-_NqkymyL=?DwRp3Q?>Iaj8-gVUx7ZO?@2vM}$+fTS`&=X*lDwZ3*21?}DyR`z`xJ zcwt7C_~XWwYZg3Zk%{J>(^s^7lg<4ld3~<+BiS&|*rbN4$GN8-gs)VZ^6uqJbrH^r zndzxYDT~r3buEeSP1zX|V96x7Y+CdDx9aiBK7?52bSa&Z(nzRW#nJOW`DE_%?8*DN z#3H=7j(&e-%D}^wm9fk;dP=V8S`h&+L%q<2F4j_<3lh%%4UbQYUaK{=e@>2Odhl#M zrx!)voKo&NW;kgpc<;G)^4319Z9mp%ScJ}86~)W`A}#w~&+DfdUuKxzIM8}f>wUA% zx|8!uedJ80u^n6F5%0a|6t}peQ1{Lcg>U0p-_AQCw=FA;bG@0wo_T+zOOEf_c{5~g zVHEq?lHGefm(0+f_w?n_Q|opr)>sSa22Q`3Cw6!L{Y|_5Q{8Syy`OEjIqUPq?yT@@ zIk)b$Nf<2<`_pxCk5^fF>Yf+93MbPxv#;DXS+Y_y?^>5mT=ON{HyhloWQ{T=`7y5I z{D1$IU#yPeF{XQS*K`_bmDCz9*2&)gZsni&Gk(prs}5dY*Q=eCBh}S+;8fYH1+Eg& zeVM3? zd3%Pz=a}a6k4ujmUsF3hrR4V89n|TZptvOMU)PKkCP8NR zWAc)IHqZF`;!v04#vH+mkMFcOsc-IRbt-jzWgoSU zDM_1Q_|EgSQ;)Rpzmkjt&Knjo2=^#VPdUEn_r>)cC7el~0w%E{!|Wp13U8;%9o&#)>C!Va!tX2R9CP|6jd`>34-aGn2z4*F6VV|7En8 zN;o!7vrO9VlxnY47bqWp=g5MN2@M{b7B=auR*H3d;Ak8aIBDvVc!rx62W_UQb|`y1 z`2EV<#p<15 zjpFpgq+8B0=~FpO(^Jh=yd4*{nV2haO=6h9D6l%>T%M{zqfSTi_Co<}MmN?^ttvh$ z6)D!%x{I~GFW`Z?;z=Xz2gUyvCH`~58E2_p$Y*{$(jTdgMGVc7)`F=i5#G0Tm1;^< zzG*WH`2XG?w#kRnZi5EHub1b&r8Y*({Wn$3H%ti7f48l}Z^F~BPO3pMekLn9Li1D} zM%|jyb*Y=ngF$zQ%iTOVtZJ&JZU0uxBoqDe`0;mZqqp42jTEekXxzu` zCY$wSaYoaLU9uHby@|r+HQRUQ8Gi2-d=nP1zx%#U)#W;q1&)nBPe;$+)Wasm%gp7Q zWv}_-%1&|CxA$~rg>-gKNi|B?DDHYDDeBhw#d4x6RXcwlZ;Mi7{QfnmRI$=va@X_& zR>#(>L>e>%=7sG4?`>!|^VBtsz#XC=FH4=j7pb*Eb)Cc7MOpl3IT*5keeVjLlOY;> z;I@C3c-6Pl4rPiblYeVITxYT`g=NZv;~$NFZi$Ux{x7hQ`>tur!VR8wIXbuh98rC| zoqL*@W@P;v*T}Bojq`jR#kX-b_}3bDR3Xjr@8b-=`_ zteh*h?l-#`KW*lMD_mQ)_3)p4p(S=;^FOYf_MLySC5)q12!8k?;J-KE;-s>RBE8S* z-^L#CoqTD_GnWj3L%+NVxgu_7w}d1eGcPo7?DR|7cgf(WknG+&7vt^TU2vvn&HMlE%m0;MxU)ZO z)KlEacHHUHv^~7(LXomx()pye*u51yKlN1F41KY%%|T~sV$ymH-2E&v+`d&;mClo@ z{;+h%ocEUA<3dTjm6boOcItHLMsZHrZuV{ff(p0FeB z&X>cQ{#T=Wwtgf%oHh_QDw)VDjRVS`D+!hLtJSO_YM&&qv-D43;hw&31L)FIL2KmDNok`OwU0uUdEI4Orj5G(ssn&*TQ@!tQs?2%ndMG@ZIlc16gC?EJ@mxZNg2w*MmpJng)$Y)^VA*Ui$%Aw25Y7A z9ZPb$TJYaC$|5R_IVj{}>V_u;Q(Ii;TXd~Z5{wOSjQJAKyF7?3CFDi1@~Rf@ZOdHO zgm(Wbjwx$%Fk+Uy@2o9eWHU1(ZCO$^=QJ6;$C`yHHB90bOd=j#-#PA!^c^l!nbOpi zx+AHw$gNu0aS@C0sx-&`V#laWecwYw<(BnLY;|K!R||R`t{mjOYm5DDje^R{F07FW z+rHX;P7;g^Dl95A5NGdH-E6}pDZM9A{AjcB+KWb?jg?qilwUj%{n_lgZ-?zOVarWl zZM+Ye{1kG0n(WN;Sw|_;_}6#!NfSbouPMr}RFR*QbHCWCQb}1@%xhXlOB!2Q`J;*F z7MgkAoD%)n>7V)kwpC8nS39QoT{1X5S%fn*It$9>`#42AeRJ0urJQLNoz=~bsW+vHS62IGPWLagtj)Aj*gXAEv&FkFJu4W(H#?il zDf_otCe}D5=31%;N;W(b(*1hCY<0)X0O#p#m!x?EjhG}FdS~h!VyQmI9IC)z7OUxta``mRxWyhNg7p%0BIXrD^ zg6qXk9U+m{S!KOP8RuBCh%div*xPCCA>286r&!+MgmXJ)*w55-(o_+As@~x2c1_7# z#Y%aLhdx7w#g?rae#$vZ4$UpJoWduySWro2|Kf@JIYJLJr)^$pp|&|8_>jym&c!Ly z^-c(-@xEMGv2tOE)tm#H^`Ga7;ChW54Gnq8~_x5?7 z9t$0AiORne{?o2_#c{#sNroT|83_y{mssCQnS;q=`GNlzoW{y&T55blE;XqMj#{@(V&s;`64C01{;$sy7Mb`{(l&Bqxr_h5 zmvcFaRU^6AEiv?$(5jBzEX#ds;!`D=$EV5;1}{|N-1a1E$vQ6o#U3kMEms6aP0#o} zp}ECsGRKnCXVWLI+BQ#lZGxh`;In%9MaJJxSqto19I@F@X;w@9@9jL={tpy4zY6w9 zT5X+?K7ISkIa@Dj`G4H%kd%}1+hDo&RvnWOP()`Aynr2a^n z7VDH$x13-)kjmaOL$deZ#_7S;TeCEGHCpXib<)(2b@TSy+jw8@xgcP*^P(L~_KM(3 zTYOIl_#Ekf9XP>k_3{VND;XuLpRe9?Xw%}YpN}4P)OWOMeG)W>FWT8QdF~I(oX62? zU#&Lze_UDeven$z9n3pi^9}dCK0HtL&!L+$X9{&Em%`>{+lz}=$?HxPs8TO(my-nCiB1ldH7IZ(Q7NRv)-}V+W_s znPWaXYi{psU45#5;;BB)v*wvgKK%~%pM5gE$Vemm?D;)A{zj+$JSi`KsP+8rzMazM zt-B|;?6G?L;AGXWHJ@`NWxkw0a^-~P;_j%f!cgO!zs0+mW-fiYDC+PY+n-$aGk;$= zHv53Edf91oa$>W zw(XU%^A)*}x}-nkuzgLiNbQRME$Lai+|DlEJ^RWk1-l@VS?9~TJWWn8bxJ-zv%dTB z(UddWVona$#v2DjZkM0SwHz(QNoDzF;TJ6mlb8pVtdvng+oAdtORG&S| zZ{=Y{=|wClYkW3cm=Mk3a^cpxzqdB{x^9rYonvr&TkY)~b8l~1dwZSk?e(?ZTmRlZ zsC#FN?VYu?-g{#2oSb`mpYI))3wJj2-aWnd_T|{S*L3e5&%JYg?cLkCw=b+c{`KX_ z0$U586*iW=!CzKLaHr^;xO?l&l{*L4-nnh-d*;i%1GWF}AMCyVYwi6rA$LB$z5Dwt z+h5%~jOXrNjD2u<%H8j8?=$7y|NZu!@`c;mpFH5&_kgAE!QsF6f7L!*Z~Nfx--p6- z4`uA`-|2lQt@rQ%?-cRvCYB~=rp%bI`-f?cLfnf2w`B?0>+Z7h!w7wc1N_N3Uy8nAu}5BulDqIC6^ zbOTHIyvJ5@&(7XuEx-3>a@?z~dk=o-vO2DNvoY^&hu+(%Pv4w9$L7WGjPc)_GgBhJ zy?x`7_in|xS1t!$Y_NO#-S^#%d2cJ^-d^OpbLQUL4fk%nnfLh4zIU+~-rcEt^Jd-K zwY&^L4$s!ueYhES|MR<-OV_<^yr;ZHdv~t)NjoLO6YgO;#(@#f%5tYMw%ud>rT57} z@3nZqE75s(LRenLuDdVZaC==}BG@7Mzj z|KlPh)-jmo|9|0?!1OKm_A9>!8uvcs6ueoxmW6AKMAzIzq$6q zzjyc7ul2k+?`7q`4>=PSZL?M}?L4r4ajWKY_rwFmyIQmT%_BpWd|CVUyTrZh18Y57 z&a+K+eQPx@^8dLX0d`Ci);*r}2k8 z|5MM6#p}T>Fa2M0?LVwN$JQSIRXO0-!LvWs<}&U2|4MQF&sc*y?w!(&p9F z&c&eL-NMk-(m7*hY;04LxVZBCPA>7;9Xa|dm(0vr)H!?QBo~+2n}a$x_s*WHrnYj; z8n4Yex5Un9Z*D!=IlpiA{GGkc2i6}td^m`UYwPSCo5i`VADz2-M$__6FRxn#tCYr%liY)h3V~f-0aG|>dedozjQ@8W=b9DIc*8QVHW3H-4~{F54Qh=20qd{O$$CEZ)L5PHnuIw|=Tx^365JK9zUeSQ8QS zS#lCe zrXGs3a;RG5>o8-&r@P!+d#>L1zs})&LhYxFnQ&j}S0?|WS0_`a#+*L*J#^;cm|*UZ z!x_S-3l98nZ<(}hPk8K9j(3O8hB7Wxp4U2A(Ce(JOxKACnF}*pZg%u0MNDj4v2{m) zgu=#)Ngg~539f9`p`KiM6CP%ysl9)}a>x6*OOvdiL!oO(R5g>;!XTeRtuyT1H$L*x zIxx}mM#{WBT-m8=C&VN4vNookNphKE^q{o3Y?I&S;*%M3*UdZ<&N=Vmu9sp){8_iI z&-9cFy&TgPC9%-FGeJaZv1FIU%8X^XVN&kObyu9Ybr)@GbXU*ynpAN#$7tREwR>Jo zJ7X3;lWVH>`z=;eqoU=CuKLZ_I5RKutZm!5tpfIkx{Ry>j|(3XoxNvX(JGIx{kz2X z%d9n2-OXB9&7i^KU07JUIeB{xH+N4h z{L8;j`dRe*#b28&wzW%6y>T&H^zH@MWo-4U7pEkudY-*-ymt{paFzO@?@{+J-EHkl zk1`F9{5(@x@}8o7*pB;8jy#!jmoYRpvVWUn-Sm5NYh1;i==z>}xAKS{Z&hSymg)+vDtuf2@a8-F z{ad2{{y%@YIytp+uJ?|$&-Qs~+}nMBd-hWGl#Me_mcQi++t>R3ck$Ukht&N2sk^^h zuiU%u{d|EBU0TtdM-H?}OE_|}OrGMVvSjfU*B8Y*yA!_6x@=W`cU{SaW}gVT3dfH} z`9oWNyR-cdTozJux~zWpeNJ7E-d8u@>A&7}?C*i}#a{#u#w9MEzQn0y{r}V@CZB$J zIQs-mYE}_Y)Mq)y{{HyZ{f?P!0aF&d4Z3)Q%k*iph|esZ#pc3%=L))GFE6q<)N#nj z>to5P*K%+3p1TUJGHjOZy25X5vhCi>fVjWP_uPLSo#n;2q&fGAlE_}M6ZWe<_M7Nj zk>vH9DaaKxu~Q|5eS6z~j|-AqRee4wBC1cEPLyn#^fu^0>W`)NGI4J4OCIvLcC}9w z;r&qmOUU`Hk)~o)+@-F`cb-VGb2`T5ot-wzC6Tq1^AzX4cbyaVBpGy8D%!ldI8)}O z{LKz?soo#`uGKxN+S?-q(wru>US?KPO`PGKKWlUBI_GDW*CYHAR|yN>y(1-ⅇ~V z-sa51J2-TfOS*)7>hE*xdDb51xiUv;iO3WRQZ3Wp&`#_(VurBp|NS37&r}9*)N&EpknbwG=4|? zdLzeXO`R*NcUX&r?p?;B9+4=ab1_NjNCBg^!2``#2@P?Q3+CxZEac|C>L8TC5E}pJ z^U@Vpp7R&pI)C@>mSulpLQG8){HJ*>h7LxK>;uoE z7&SXg860_LE?#J48RdG-^Gk_O?Q(@91-?f#B^i=gXEb- z49k)ZJC#{8s2>Psz8=x>?ubJRN5yn0)vE@T=^H0<NDaT1*uv4{z>}JAP&Y%M<$l%!Ho*y5BL9C* z*z{4qJ^iyh|5ndaN0OdQZ0#{B-hE+$o`~W(PL9J)`yFIed=9b~zb#Oo(9n{YlE7u& zFo$hQ!z)3Hhe!0YKXK#*G&G&5;^I-zmtND*_$F-;!;1fX>5P0AvkVOmu&;d98E$UC zx*+QSS8#!m+Jd*Qx2_Pm!Kx9=XfUDC+?n;pRlg(R%Mw^sSrVCdr5zB@o6E>_FM&%k zfpznJtL~!LN4O?&XsgZWy%u(-!O5M~NirjV?Lk)~tL954p`6Wcv%DRbUYJ%^JjWr* zOoNeG+2N4Xm73>9OB|*p85}B^+B3o6$jnn8eOrVd*>HDxOLkeAeOcrf?9T9db9twh z>RHw~Nsittcv3IC*I5>D$EW=o^UqXf-fIGh>IM!xvez~Uw!{c1uW@AaOE}2NwW;y* zCWg(0=Z^3;1+ek&nYPz&#(R;rz_znSwtAd9s;(8UWl(Q%WYbaj{a++Ppv^QxcBP{S zW3z2S%ZE(|n3*$PUA;b`$@+c*FK5ERhwKb@R9G6{*Pj2%J!!)ak%U7Wau0U%tdJ`> zwpoEGd&Z&kxeplS1p>Iz%!;`*<~A+3Z^^mw-~lOy+pRa)89AipIWg>8@Y=;Cfh+z+ zaYEz-7DJH(%o)AAXP$mgal%{fW7lN|Ml-%7rG_J1cjOz{B0fa1+U!0*;r2$6J2EV6 zZ_HTb_wVKDl0WI}$V4yQbFfke1#SUA7$Y=vpl|7RB(rQ)XMe{xc~Lw#2QP<;J*H8Dh761j%z}ocx*2g^4&m7X>3dmgk{&Q+E3h&=V3spT z(cHl1n$hYZk?vs-#cs%O!!<6zyp??-6XS$S+g(HByTTWCzig;JaPWg z#4v?6U$JQGZ_(Q7-jCfQ&a*eV&1khf(X!%c%xO_ZK9#x$#|taI6-sSm%v_djA;9RJ;yCeegz0jxzn3!pU61Hsi|lps3Z3ex`#ox#bK=H}th^e`AD1&q zDzq>kU`j4$=Kk8D(-@NdAzjWvc70k*KYPR;Q5GjdZovo4QXAM5UZlA#U{qs^PTKIl zb*mWT64kmV$AuUK*z$H%m))on{K$GCy|=I|-9$h-`MFTuw!&E)MUy<_1UK|0UFchG zTC_narFCn+!u8H#Q`T$)(dS}4b{ncyr^P=gZ`qYzm{gHI^@hMM=6Y_yQmL|re7BnK z97Vq@${Lo|-Vra5JH~bUc}K88>y8=aFE(UP)~GYKD3L2@y1Lv*`ze3druq!$1iK^F zi!6M*zfQI_cV_3XOKPZ=TEO7=pz63pYVdPL)tMa2k0&HH#3?IGm{HI>Q=zl&SWJi{ z+jKJ~#|b&brHsj(p|KmLyy6Ib^1S^KM@97aoIA@Iqa#9|yH2fBXZsq__x2gf`ll6+ z4jknt|Chgg8aaDMmRv)gie>m*j{aZ)mfg=7FDe(x+2b8H zZgG!T>414pS29MNRF1L?Pgdtq+|DLl!2I!NcxGklqxKg6qr&MoCk92`1yWw_zuzqX9!$0drUatH2Bf$LSn1INBTnCY^rDvfFin-puUs zZ4>jv7=sQlUD)wIX64tkCu;Mh8D?)h5%cb7mD7**^_dHvW-b)?(Z+kK_UD&{sY3l* zIypTiWVs2j&74pZwji6UV4+k){EZ(Kx+g>CX9Ss>dYQBaA38kQnAPFu;bQ%AZ)L5S zEH2^l8r@P3-9JOCRn*vau3%(#XpGJXlVVsdo!BmSA&>Ro{NG|TcwaH34}FBv8@jkVE<9UqcuItZMtIv3;(YuX+gnQbspUZ3*4r!aNJOOcjxl= zmGiA5Vi?`ig4h?aWO8_O7KOEQCQitV{lKJmiz|JnU|q%B&Izky7lg+Kq)%xpcyoNE z;DeP)+ji*tzPjnWi0;}W%ruZA2!aJhpS}fWAGT#i|%}r*f9|>dktm88jcnzLgX#Uy*iwLv49!NWL5E z3Z`7a05<=?)n^YfNINjEUB341jyUE8|3hy!CtTDt>wOl~Q{>*I>T9W~bmj`H=Yu%S z2eUi_3b;P>a6e=gEU4lvNZ;MTIa969{6SozLygn{hR~BMbT;%wY-7|~C9Jt&=besJ z*=uo~%M&FY#CbMw>Y1;(r#i9aYenHtX2A*BB|F(NrtW?_m1+OeUDKl&1ShOysZ5iX znttjv%jDY&W?fyA$J#c(d$*3kqOKVnCJMW^&*sqh-u>{E*wgAN6N5xe&&A7+uD!ls z;eXDK?1WuP+xNKNDBV6`k^8CLp620U0eiSr=iC)d-Mz!r{bb3*vK0~m0znl^WK=wF zeO{U7ta)apd)IFNzP944F2#Ze)lRs0Cn>PbVRM?J{J&0l!lB6r^-dXZS1nDPIf2o= z!~A%Fdb~m6tbmaJD;VM@BvvpVc3jMPy+LHt7gqNfhc~uq@m*UMu`Sd4&XOr}aVt|_aLA6sSxcrp@HNy_vbn>x# znekiQtLIuo^&RKxH;QiCJPw(J>P%aDX|YR!i-5D`jxXhY$tBvBZ6_*}*2K3a-CZts zG;*EUVmV#*rtMCl@h03l7kZ1128uN$n7T?${<^a1j&sYjdZ$SD{+g4asSzhjV|Y#d zV&1qt;mY*e66>=xYLSn|=A%AGXEORJp*-8u6+HS~dx&(_e$e-nDM%#InCxVj|7bEVn)7zVSl&fVW! zAOC78+d(Cvg=VKLPkd=Pb$8CW!*|YEwq5wT-E+3h+}$Z>xO&fvzAb+6<-)@~Q4(k4 zS@*6xU|cLyWA<~gdzP@2`d{bgdoJmmb@?iC<{ppTp`LSmp$8>Goxj+eV<T>e0^QR&cY&jF!!xB8oq};DezF>1^*Bierr9Ow3_(g7s-EZUbq~xOB-Ag-MF9#pJ z`aH(k_1D49wBoehh(B-qC5kS&-j$RqoSd-NPjuq039_g}HC zu8qFG=Vo*6iS^DGW4`D$9ufJkbGFAjxKH$wg73B0uQZrKmkFP}DLB{md(Oq(wrBS4 zy|_5nCotEeu=bA4+v48c8GD4Myu0Nux#sNEBiAP6Uee_aoXp!*;_JU)@97h|CwKPV zdT4Tay3UQqwKs!XyZ(QzFJ@VPF7pc0w#n;{>`=dW!XW+J&9j$Q>AD@2xi@#N)0QVU zzkNAvvGzjWlAHJ6Ubufn#_H!BhKw#dPp^|ZPyKRok#f3lBjx_R%l99@y*tU)*4p=s zjNdKETEB_kZu7AIzb?J?0+(+C`@BH2Uos-3Dc6-h7S%C{zPgHVFv0)C45D5}y zdExi?&s~mtCzbOwtd=StT>C)5|p1aOd%q9q!&zi(g(9EbXaW%sRp;=4}sT;(X z6q%bH(A#Qv(($8RMf*A^I_XX1M=z<2&JSC})G-o*b}V&$@>2f0Eo1s5-5*=P_pp;_6k z`Dpk@ezS|u?p@&FYm(J>{OP1G-ePx(^WWY18Mk}AA1;?zc0=dH6*&i;My2$;XX`}X ztrHaT5@1v)Wd3zbTX75b<07dDL+#i-hstw4FR2sFW8{)QFLHLpF{_VEvWdc8$^5$e ze>TU6QaK4ltL%lW|4He5FiSOH&}QMtX=;*Wl?n^Wk4ANf9co>wG z!?SoiPoy}K1?)>`c!}lLK zzkdJzwfE0OX^8?kQ^%QHzYe?W2CEq+$+%7V5peiOr>L%5i^oQ#WBo#oQA7XC9y6oNux= zu=LrPDYB=Q$wVnXKcN`^ye&ug-Rb*@26sZaw?DhGLi>K-o44zpDbLZD6sdhV=cSkG zQR~wy^JY%|yR+zQ2ya>BrGhD8X;nL$b=~Hq zEuL8-<*DAtax0njP+GJ*Uy<5u&&3PHX6Nxs%}~siJj?blYu?Ul58H+J9h1sfb8`9I z>Ze;ZSiSpxJmqt}E4y9E`{%^=MqxLV15UNqD#eponl>D9^0lAjE@pN)@o`Mg;h(O; zkqj*lJKFYb6cJ%~q#B-DrO6xSn$J*k_2`ssogq@4($B(frmmPNv9MF&l;*?PIY}J1 z|IcQ9v`jjeW!trfoq~_p0$YWaeY-JV=p?H$!&^p$JH07EmP;owpEH@^<>l~mcJa~7 zMN33OW3^IMQ(Aw9m!CdrdWmgD)YPRb^+H+0B{-AXmaY8aTPPzf%vg}NxbW-Jn{&A9 z$|hwLD(zC6yC}6HC$(~s#oAfRe5bwMSnXty;h~_Hp+0-^#IoDYXAVw%oFD1skvh#a z%KY)3|B)@38|s!$pSSnJrmXjqw631cTlz%tTmHc{+h*vuTW_^qJS8jkiTtjjn_qKh zsb)z(&$(e%oqkNQs%ClC>8?3B9e0f~6c=u6-JR2!S$0i*j;8#pnHdas5?X{_F0zi! zm$7=1aU}g9Px1fr>W1HoiYt3F=6Tw3zvYwG6j|`);^!uv=n2`UOtL#|tN(79?{t3q z?DiU~vv2aQ%njA^96;!GjkU{Q1klK(Xfa&{bb(zv){J_ZI}Ac^VBX^sewJe zx7VLvIqzP>OtY>EBV`M=EorlV|5>m&$J6Kip15+o`9d~@-T6l*SMPgqY>)2F#O4aE z>dmv7mhIixlvBOh{A_OD?26o?yxrfni%IR+H2d*6xfdU!?AKWwD}7bAbzxQhwVhk0 zY_zLdbkn#`@OAC?l`Gtn+_TrE%6sc?YiiiezvqMbZj`!F zu)tSk;#r*=YxaLN=vDb6|7OBf8{V|Q)E&yvJ9~cYnjhQa_)YuH-7DLUG6iKvO88vo zvY%qL;BJ4*pQ5Pdybg1RQoVN9GxM1^XV~uM%&+5nC9$1N;#jEI#2%4L!A(&&bw6M2 z?^$KDXm;CnZ;rwVlQ!&0*kF*d*}jgymd$(a-+x>Sf zO!xPWzGJVt_|V5ny}st+c#-p09eWCU(uz%P?F_qijAPTJN{yPOf3&I;+qou2cK%)= zoml3#cpkSH=|MqtU;!%C2zS>TC3HO)}w>^qM=3f7QE?Opcv1Pj#Dy&NKS@ z^!T%IU*)bhUD<0-d^4IH%)DsJnwp;1n`_kUnC&i2tFT?&Hp%_}ii2$*>XuDOcdp%` zR;Xk!+iE778>=hmEe6a=fkf<4@{?3KMPKk<*u}5vHG^T{_mUEV7HsDti~0V zTb^dlw(`E(GdZrpo4>j*ZsZ&SWmuTegID{%Y9O>GvUGOLo1%y5 zXQ!oI-Vy7n7%?^aD)Xy}b+6VudS@YZbwOPD>J6&zL&{$VB%j(4U~}b`r=XN=cB<&6 z>YPXcW-IqwAI$UZTUG{@TODR)x?%Lc_i3$lRb0%w>q}TflNI>q%xb!Hd65q9tY&4c zizWA_&;I#$wtanv{O%L8s{-~b&ns%(qv;y>&LXN{$;-yD zY)6gVAxB$|7;U*GeoXTCG-uv*ZmPLx9Ka(#A5Pdd2jstq<60Aib00`te1LCI%fsHWiOBJalXIvYU|e5#T(D>xa=rr zaDvIDq_3Uh$m@C2PL;&HpYZqhYn$B$Qx#j^eO^@Mu^}${lzEAk?t8D(-rK@zb-BxP zAI9D)ud|ff{#W>U&%?9Nk4`f>HG6N$+X=6qG0v_$bYb`Bb=pE442nNlSZx>>7<3pI z7#TJ&vNN|aFfg#Nu>79K(9+WK>q^U!BS*f!JK*Ev^Yw7ZqGf9$Qp%n_dvWE;l`rcQ zr%ag=6BG0M`{U1#mu}gzMMXvB{|bhS7ta4Yo!vWSp}A{xc4^CovoeTPg;O#b}+ zA|fL4>+PxJ+}iJ3b#1*8Yieo&Vsig&5MR7_@%JSxKd*KFJQnkPqsrvz^EYkVd+O|^ zuNP~rJ>s4`xc&RttrkUwQw*lita5j-5KcdhMpS?>_#1yU*F# zxum9(mzTG@yZglPdo2N+0S3Rx^d%ra&q#YKd*m1S)-$)^Jf+J z_j6?vr!TN{i+z0m)}PN;etiG_<<8U}$CAG7wDyh4{k>YyB_OS_bMnvUyY?MAVQp>w z=l_2h8JXXo&J-6H2getFTgLh8X3wupDzj(L-nDDj*$Y=^FI@fj@#E8{Pp{jw?fr+3 zl}&vN3=Hi58yK3JTUy)NJ370%dwTo&Crq3)dCJsj(`U?_)x$J<;@o))7cE}0blLJ1 zD_5;vvv%G34I4LY-m-Pu_N^j2cJJ9cdDp&!hh{F=c;Mji!&^?CI(zPX=h1_gE?k{{ zrsu|#xwo#}yZ_+fqsLF4K6~DI_x6i7Z{NND@bS~P6qK4pIk4_Gy)6@4_ z9FW;k^h|lWS$JpTNguCs^Y3+pBnc_XFsk*1M6Af{QT225Z{+Lj>S)jvSbmM+?y0GQ zl8#YgUjsvyz1}xDjrC;Bv5<)!8(RL?$o;C)WN+A(sT_DVnlvV4lPV}rDJ(wfLXo>ZAv6;2AC*$egBSZ2IDb89jmgH@JAv%)Nq zpc8A^1r`En*b7H^;|`y$SB<_P;E}trO=)hA*TUm1chWk6w|au zq;Y`)zt^P(6<33dhpqZ=!{pjz%3eHdS11x%C)NOD~{hSbKK;-KWcE2kCLCu9?eZ z@MdK}&@Anh%gj!#+0^hbOJnJbsqM4Y9`AYmX2%`3XtizTIoi5wSGcLap4DZdrN8!g z*r_+O*E~|qUVUNH&-Yt(0|gAGx4!D~Tr}~?te)jv>#Xl6Uwjz)?Ek?DX;PYNTQAHk zocA)~*M&sE^kDtaNBy&z7&ohxYO8 z^_b}vpXGU}$so7ma;J*z(kuNQZ#b^ZOUONY{&gAeNx{V}3q2)LCjb4an%1nmO*x_a zQkiPi0$-1tlcxB7SI%cH*skorpxAML5<{G~e2X}r!^31nryUC2>Tw+pJFM#}o=jG@ z>v+<%WI@HFS=y6+2sJOcQ2BHkGs7>T1@iwa9#6Ek`}Jzoc0J2i)BVrwc-wlx&+2L0 zmUF)zP5*q)LUEI`->=8XiG91@EmE$pesNZtzxM4x`}&TDd)41rJeyVT=ka+}`n{e1 z3%a}S)xK<<5nuW3=yv`;Z{MlM|M|Gz{~ROxKIQ`Zf78})wg3N}d;Y&~>3i-v<`w?F zeV5_A`_8v#`y2LfCot50xBtiMubb^Tk5Tu#-P^B?jl4OrtcBv10%qr$<)?6d;JdNk zaYjLffX_m%ujU6hu1zT5EV;vd+^EekC!kqk&cZ)I9XmKL6|`?uPU6bFzE6Q`W5@r) zcc0YEaByO|(60Byfq$(=hi+1VK*Sd%vBeojC9Mn_xYsNwxZ-h4Bd)L~Z%vZSMT-L_ zmo9XcPT>^2x#Jj%*M^=P9;NDa8XSr@1p8`x9?9L_?BKw(sjF`-2_ zL{{DNgiY0?|C5WpD624YHh3-C*fs0SB9+T8?7f7XI5za8DkgJ!>oYk_+dgHfZlX=kDol>c58{(pw~64!R!+!eNYMQ^F5k8Ad|GeX}s zZe(4Y@%-kMbH9Q%-aexm&ip#`A=kGpcSBw!%#XTxILLUFbnL5y?BCJnnM~#<_O8wp z%!=9hYumDeqOU^*wIgSL3Y&dI=54Ng^r=J3wk-c;@;0UYW^AR{lwB{6W(PBC#eWns z-P#(eSGQL3{#L&4+Z$8Y1#(t}>_4@A2VdBl|8=il-J5PTb=y@}%~bWMYl{wjo>Rg5 z(s!fvr8iE~=d8S%(d2(BO5pLfUGl7JUHEyfCTNMRl2uz*5_&3<`~Hn%w%@e-S#RI} z&h~YorkbXo{^qEtylZoQ?paq>nG+%wEk2)V?>aMXp2!>Lwybmg_dZgX$M1ic=^AS_ zV~35u5`?V9*WFyZ-1Yfw|7UtTj|QH7nW}i|#!J2#3j_ZZl}`;mNb@QQffRckbVrGJ&HJZ=`db=q00toKz{PH*yEKlAPXS*cSGg)P(dT^&&S zBR2K<)kQl*KFrtlO5n4v=AueUs5y7YZg z`JRjW=l#5TVb11b-)CLPv-)+kPxpPKZ(ZiQUzLx3sjafLJyW1O|MHEGw&_zkbMJI) zzdc_x>_bE9+2?!gL{8fIm@L1al@eNBaU%a}!PbSDabL7PzdYiqweEKAC5hvgAD!K6 zd1!ifOlbU`E1!IeH@@yl5itL{a9i(Bzw)y_cEP{zebP;hd1{vz=P!OGdfmG~`KU`S!=gFhJ>OrM@ zZ(OaCy!=gedBeq(e`dMu`NGw|@ow(9;Iqpu7fzgBeIr%sjoWI=*GtxIxS#&|Rnz9% zZ^W*B>bjm;-rg_ux86N}V($A(Z|weV{gx6h{5@~qH?O^^wPxaR-@|+li{?z5mXX_D z#rxc6-PXjpqA?}r3Db`!e=JQkY_Go;UcN&#reCbc)UAR|!(a4zGJ8i=%k?M@v(y{u zX&@$~WNPWIA`sug!#lEql- zE3Y;jcTIVwUaEDhl*k#D&^%i=SiWlR)nsCAFBNDKYZUivt#ihHZCgGSMXZ?Rj% z%6~1%2$+!~c_XIQtZqqKsZMz5#qTZN;X$0!Do=eY44GDtDUn@R9`S2x)tslLZ`vwd zmQ_`$`9~d1vwYedT~_t+Uu$oy!p+rqLvLEUlv=uIHO~QvjopW5FGd$#ctVnNRdrEVFv`-FM~H zn2I0$-5R~OLw$G@7+-9OvfJ4w9bWQmM*7U_O$pC>J5K~h-^?^SS@-(M)Fm@&j5wQF zrWP+^ui-mdJE|7>+YH`R6Y(|s}|6n+2&%JtIWm_@S*pLTFI2BlSQBUY!gYh**ruHwYc5B>HYYm>zz{fW&?&w_Wy3P)fZp-n(9|NuUWwS(-DuGksKPU zIL@fZu(>b(xJ-=g)`X4?t~s5HG|MNyQ?DuAIr))Cod31ZY|BcO&M>zV>9b$9Y<<$X z{cCjJ^A4X>ABptx#?U2tAsmGl%y^?OZmqKh_3(DDyLPeusls=dXN6hMJ*Rr zH6^U7N|?4((%h_og=fR^jtg>S4r{xkIQlKks|zd*tvC`d{9oHVVSW|!x-BOek{^1D zPGD`}V%?g#iXniZkb(2e70YHVjw*w-Ppam(U0Ai%QcP$9Ygqw%(}uMhqvrp}TDQ28 zqk5KkYXC#o2Cp^)*&AH7O24w@rBZI+XitmqGTGVvEF+V5XT$I3sdkz*wPrDYU78Eo z3q^iRZ%;OhR9MHrU>1KHQSx%Iqc?e$d0ddSUdR@$D&`8Z_VU5 zWXPPBIql#QM$_o6C!^-CT*_XUum1ZeFqE=d0bI5F1-fy*~H)=`6t~Gt9{&SQota|6Q z#BR#We(fFSTHMN3W%Ncd7;o@OzPMKO_u^>oDwo|m85)@0|JoVOy*fnOEN%6C2Wv64 zrCXQ2WOIML_RYr)#upqvdd)7=V*ELi=>^;RvTJRzW{KISdt@wIxBu7{aAMZ-@YY*9 zilcu{tg@clr`Eo_bB-r#rIq`Ssy<+El52z$>KnlRII^^y;#ecGZqytDVe%Bj(n@w2PlCY}z_(pNA?5kfUvF(gm;kv1Uqhjx_VquT8zgwzPQYUk{gyE{?z77Kh z4;{a%W$eZa4qVY<5%!sFvD?+I#^OrWj=s%{R&5U#JlT1~y*}R3J3XhSdTH+Z9mf`D z6g}tc_~z2dDpr;Mt55$;x3%^_C3Z0(%OY<1wuE%sYB95m7-foYNF}`^q#b=N+$#UeRnct#yOs z^&_{oJx`ln|EKZ(4?nYI``ZNC{X+krIPq$8?3AWH-a5a?#Tu+P|C=o8dNoBqI_-S* zP4AfcmJOvoWjA}a&9$yQe&_4_tgq8wm0z5ymfm`7QNpsGAIGPt&e^MUB5sXF2K)2{ zKGO@Ie%^j|?fV8D@8HcW)ICl2x z676Z(H5$i-+HdgGomud;sd`1ec~;-?_WL@Sy;s(*cxbWatlE*FWp~|Y25G&U%_W*Q zae3xJovzl3+XjC3LnRAh*FCZQRgt=^v5j>GYihmvjGN~=a)dkEiZlG}pJl4$9ABQf z=l9u(8ntb2=El9eIg5{N;kmMujDMxKy^ovk4AXtre%-82`g^$6?gj>l+?u0L#j6)y zzFVHgcD7;Ngx`B_$1c4vf0I7ZhW!u8h85secc7JeMvJ{zg*t0z4rf;b76Z~ z$}Z(gJk*)7@sHSxgkL$`TOSu~O}}$vf%2V4(iS;t6-f$l3#~f}Or{0Jt*h9WR+exy zF6zfN&3DCC#~!}SeYd?bKyP)|@`w)~&n?{>R>pC@cw^)O#`iwi;zgO2xi2EBH^r5m z?1}on?nOe@2laLJQ@^)fkNCiGZhEli*<-dPe`my3gnf+72;Dwoqjq`!o>?2;?yVN& zd@op+v~QL7hq|bld=EMAM>6^6>t;?(ld4;vdL}XiinoTnv)Akn z@vplcK1b~S<7267%CApfYLTY%YU-bNvu=01m(8g?CHe8{w#SU-uallnKj%09?(xYy zyFNde`@g;@yzZ6U^cOQWDuuo}IyFpv-OXC0?OM=vS$e`AsIz$2+UEV)I0*_`!1*RKC9$(JK1QSA7= zM(5n5k~*KG&z@%Nc@zArsXo7XQQaT!<9~$DEn_P0i9Pq{wE3LYkYXA3>EpJo0~cd36c9_a5$P4f^(P&YTB%`kOS~^v>tw$jftTX=Pw&Yw2+6 zZffsIPVQ=IZA)&R+S4?nyQOL3q^WazrY@MtFr}xvy?@ob88f?t?N-yQbsl#+A$eFUULEzF}AEkv(mzx1T#c?P}+yqqDmv&hG8od*be)JuBv~ zzO&@b)%gqGoqe|C>XvVNckP>cs_p&+y`7w;+;>-d{9RD`d8_BceK8krJq-@|Fk@5s zOS#m^I}DZd>ILT?6~5tl$3B8#hGvdN;cSh%i&tJ|SL)9^)U?WB71uJq1p8F zofF!c-L{DOP5koi-y*lODshw7+f;4w?q6iHnmLv2S()zg>z-YIk7l2lac$Khe)U2n zlk$xZX66|0XE8sfd1w0NH?iKf-9{Tazh?OT`On#X!);xR=As~(9{r@PPtRDr(u{bl zbHIAF*8Sar3h!@5^|mkjaY}K?`L&JPl-b1$0}Dm7W!qlVsC=DqT2JdKi^$^L>zMty zL}p~@zS+yhZXRrDxARD;k~AM)QBR53WXL{0lYLsUxV_&-wL2N6dM^Z`H^z26b@W-q7U{I;OZ)sP^?w#U zO%D^F&&YpyAuP6m=Ov4H-g(Iy>7o$~k<_k_%Yi70Fbid+( z>e3lnwO<$;*6&>wGND^=XKP5Or{>J0MpiG*slEp;E}N7l@b;CgyXvjvm01fXm7Ua_ z@4~%!+R~pa=c*#vD&NJw33%0Q_)B6V@_J3#wmzGfUCz*)g^=eYajJt)KSUsCA;TgHrmr34-=&3)k8>RcP)rcP*D+ zwR>;)d=Ku`zX~STgiRMRf1DJNw@81gi2S+fX}`I3+01V&SpWIWvXC<@mzpF0hfMF2 z44dWHKC^E@+o5kgrb=%$i?c7@C{wy|NH^!^;kt*<;wKs;1wT)Cp~~@YTI;4C{Jqn+ z`h@n2UFr%vt6*sQ?W*A6!zb7GXy^KHxif4yF2@?e=kSy@rtjyoIm`mbRzHt9tzOF% z-Tm@n*(b9Atp+i>sWWPWkHj8(>uMp<_~G4^+t(K#^fvNm?0z>p*L+@^>{*j8iCwN9 zA6a*)GOc|2?Sn)6iFJ8vtfyx_?$e&j#8BSC-@x)@v(P`;2aD@Hejan-TOMb6aj(Q? z*@tH9Ox@jO_t-6U`&AzB$aCK02_J1=XRuV{T$-!Kkg#F;MNjb~LXqw%S2YU0>iTYH z=+EhK|3B^Qj^j#&pL*U}g{T=Ds%J9%kkf8jx`oMOLA61M)K5G0{Vj!FrCBG#w%-$; z$G3kzUWsBM2i!JhX zniGv=61#b`JUXI;ZZ&s2<(hZEvwUCILc=Q2HgT!uTHc`hiWkz|KbRKily7AH7m_wh zF?Ny3Lxy?p5?1VHzLKAN$4O*fL}7zY#zrk0$H+{+h3smdKE$>+9AY#(c|;;8FjL22 z>7`i>E3VD#&~saK^>u*bvNbPUY=eK@xZk3-RN-Tg@a!v5?DL(MS~GIDX@xvg44Sa> zU1xr6Uv(Ozgh+qx|CKuFg-H(-XU&|$eCnb^<;tTf``Na=d#N{7!{UgvL?!?6mwk*3 z4a>iXo$TV=z{I8FvGmfW1&#SyZ>QY8vTUNri#^;~XT7gInjy;jNoT^}G-LZlxvH3B z%U^mVw@f(E-Mwr184Ig(i@k#7<=*}{WT!XT!65jBXVFpbyt94U&eyA|$WcF}sT$K1QXNmjc6CENN#R>aPadSrsvWHE%BoVIrO0lDg zqv2O^XtLn4sU{N@w%eF!Gh1EhQ@eI-_guBSt6Edv_-y08Uio54yltlZgP%TOUjmm@ z^St@N)_rO2UXK}0SrcqEZ!WNvQTRVK@`RSQQn=JQtzPb*CT@1d)g0P34+OgZre!dG zxWT6CJAYwLfb^@(p0q_5R>f>+yd*X|RIP5EqtBEBR+bLY3_J@QS1w^cvcpwr7h6YD zj?4tM?-TDty;`d5zAKHFeWs}NrwR4tCtu4XuWNv}Oant?CDS+1E{pJZ+rA z+x~I0z%mUvnabGOuq{%y&fmqqufEn*;yq{{5;9(bvoL?lN-dpWZFCYfJ0aNIi4cz)TON@P+KQ7NNWEKT)puXz^@v zS#osm&(sT-eg+4gwmcg!!-#vzi6ilkg)3w-4vS3r*6`0Puza>k)_<`e$@iN4bM7)Y z>`3F^*lvDf)|LgVFMr%)2(DU^u;q30A=z&V0SlNKrdnIRTIv1Pt%x(gf{#%o#loU* z(_NLQ6ElLYUyv0~j52AKK3r+sz!20~zMfyNBu0L1PLQ>jg7FdNfWm|mOjVO^&i}G+ zYy08dSL3ud9iHFzP{Kei=j77wtN066T6~S@z3?h+#r-=cuUY;n;FCJqeWkE>!Q3L7 zUFO;J+N{1OVB4Aq-5TTem#j<&?w)l}ynm_tP|SYjr&eFidh)O;`5(n$fq|=c#kv zQ|5?AVR^nO(&f#Lceityi8^n8%Et4+&WkyyDIwcW-CZ=n!1E|aVnO8^155kwPPS9y z3`N|RKGc?ZUisX_FQzeb4MP)qbLcxY^A`rzB1`2{Tm<$qW~5v!51nk!yTfq8ch_Ai zu^khNTZI0bbqTP|S8feh%3i3#D0AI%PI3E;!we=OR_7+0$7~W?zohv57^=d+^p!$xISlgv~q zPC)nRvl;{SNUHrEPtIH8D@Dt+(eBp0 z?`}w}b&~>fC#QSK|GusWzm7-JyAM@QI>KdEkRpAxQ>4OXaYV7Gk;cu3$-1$XrB2NAN@tyInu4!$Z%$ebnhoU ze^1qmhmtK6T`Uwmc3x@Rbhu+{z$A_0aPh@zUki-+O-=Kjn;vwW=51C}(H^)}M6o9* z!%5k7S@5KW?P}X9Qa`e^q(taft4;lH+$i}|x+}QN>az9!!-`2KdoDChx$(mByl|h* zBWuUW5^+i{RxXkj;<2-YWDEmsen4-6;7&V@+{gq?Xkhsy9VO=;r(ozx-GY8MkVU>Jn&>VA|0s_Tdb=8xIl_e zv(Ze@tgLAI@o#FMAM;tesvr3#bicxk=dvC-w-E7XoO!w}_Q_RcxzDrb1qxnn z@@x{R{dLg%?6xKLi)OOBSouFxHlH3l`-8&wlL42%%@oLTTj3;DdU&qzq6I4#_UZ4M zX?V$NQ&>+M$H|Pt9qIw zZ)h!CwrZk|@g%)Q8LgW$kBHB>a!i4HaZ{e_d}FN@K|&r9CQ~k4kjr{m%^VUSI(5}O zg9RFsX7m?%{CPa<`z4{!lX7X3CL9l6b1bt)u50nJ9siSBGL!RD=Bm2qOf0gRaiweK zN4<5QCg(pfxKZeGl%;!NQ(pP2Y0n~cI8Suu&Yb^y@tg_eiw%#-F3ec7_m^rf*T!h) zV%29eCjVR-c*)?Ml;_2-tJ`-5X?$EXZ)QM{)N)zll^vbFm08m_OqMvgb)tQ@;)$86 zW>!u+vSwLFZwTq0aC7nMfSGe^!ndj}o_J*Q?2?0>Ctgiy_-wz@c%HrS@+qrqj!n)? z^;qbrQr_q-UH^Nf(212tL>hS_CTa?+o#k4zP|0jAhrwr&u=2%=t<=01N%i++nRO~{ z=d4cJJ0WNDubTJ_iN}ufULKpDTeV#EvXSg6n@t>Er#G!qs9GesbfVSm|2y|piT0e@ zdPveo;HAOYjTT!9wa?93w5vtKbn=od(UO^`W~APnRQD@pOHl5LZ&M8q?&7|>`d)Q$ z^|NWQLb5$w8yZwHe73Fj3Q~M{bJn6N{RXqC9&Y9HR_r~K(6vBojh&q?a>+!qadq&H{ zd|v;z&Jg0_J+a#Jj_>N?qt~7&-IuD+vi6BkS)8=RdZxl+-8n&5F0YfZTDywtu(Zv3 z?`~ZNMT^U!E)x?su^&}kT(RfW?+q^>%{;VN`OfqsD=QbL<;dCWUeqG8(0k8eYn~db z%K^ooX0+2&i?i*#hlb8q*l z@99f>J>it(Ces=t-8=s`uZXhsWOEVnJ)N{r`&sJb{-lk@nR|8|Nb=bVw~2d~d7<#TFnfyWouDbdCdHnYzB0|nPfxwu%W;A6 zP8AkgZa&{NYA+9*__*I!`Lg?72aA|%x00PWswG!zNUSV8z<4%A@aV-;k6U$_jsCN3 zKluFU+fOt8WrTCvPFuRV&)A8F+h+bG-*pvT5#o0z-1&Xte2wVih-F`IZE3097*@0F z#Tu(iv#z^tR-M2VD8wSF&~i1_dt36C|0b8UcDT>-ef3qeoBQmw-ek*izSwts+}->#{XgU+$%K;@2|S!#Xj-T zoLes>bEoy(o@sN)E?XmG(L80YbCwrR%--8LkK;)ETa9lz+x{}!CrX^U->vv6#^qn* z)-Odp3$w%0PhLK=bMEV&>yJ+#YpI@}nB9|E;c~?Ndg@AN>lf9YJoM%~i|T1c#k+>#%N6G}oLrhzcK4BUTyJ3_4 zX^)aHUwh*G9>d+HXK%W!75bjFu-|LW1*_Hl(c-^kPH$)aFE8Tjp;YIEyS72g2>(R<$DR-n1X-R0!eXB&6ih$&pR&ieEZ%e#vdHG+N5ebv0H7<+Qnt4&>J zcNJ~D^w+6PD(CeD#w))zok-lfSl>)TI`?v&`2Da`FPa-}-3Y#;Vl3KWyXDK@c^jQ3 z$eWm#U+c1U-1D*NReDtK_H~VKW`~{bnI!t+j**^E5Z@-RT~ETYl{fh6)y@2W=gYjX zIGuS}lZux;I&;VC{=UpRa=RVR6;9hQW9Ev}+jYKwoHB9miPfw6S3apIzSa87E9Y#D zq4&btPmb()hS&FbM;|usEb^XEne&M6a%rFOj0G!W8Bb@Ny1MSE@O5n(hiSYd+4tEp_9H(60+j)hZ9UKHSl(Ie+h3yWf;@v3pKVedV$h zIqT2wnpC{=QM$oyHJ7u`^kOfq&^>8VthAseJaf+MqYGb$KKjOWcfu^5si!Mn>Hl0O zGJo6ZrmcJ0_f4pORGI&9+B=h5liwTU2_BlOXmYQ|Z>iG)zq##9-*HA-Cy-u&+fot>Fzt_hkpE<(D`!up4U}pEmyen_FPjq%+3F( z&bxk=Pf>+Wu-r0d&gmj7iy~7l#B>MldCU{x@iA2UV5H=&fYr-nAKPF3u=DrD z=~)ur+L_;3EIG64!Co1D4yUFjhUV6`_LgR+_Po6AzE+0Gle?$%OmFM!>+0<8nbDWL zc-n$xGv-a2F~6m@cFKm@swwN&uh_6+`B%eP!`F6|o_T#fI?|!}V!mF!$JE!iMw&UpOFGo(_nSE^0 zM4>OVBddYXJPr{Z-!=dw_ZOMU4vNk9A zr|)Q7aL@P7v`)!QHKKx+YqlFN(=V}W)w%s$NM_BHcTRH`yA+BW*Y_ODxvj#fyf5YR zso2GqC!-ycEpM}Koqm3me{sspjeEkkYAh+vYCI_7l`9p|S<0BfG9fAGet+%Z1|H9f zNiBzkPMLjfeY<4m6qnebsZmpJoH`y8ysCM2Y2mj1>2G*VtTQ%U+4JLys?ajy;62CB zc060O{8z@7*g4v5t95!}&Yj!tWXAiYtkKjXQ62vuD7$-2VJNT>O3kP(X`dJP@L+RO z?#{cH+S#~zndKet2QF%N+v{^;iuAjw(`Wg1x#-IFG|>`0=tyv>rjBOdY#-JZ&*u`lD6TX$4((W03v zYzhu6J7YJ4KXh`yuBX#FoD3OPZ{b+h$R_i{+OvmY&&r2dOFuYiS2FQRtX7Rr`YxU! zS9(Wqid?YLK^?=^_N(4vT{0)lJ9+sGiyslu3(J1%tAzzGGFYdcAo8jqPmwvi)+qDaD*d3P8w~&Nxx;%b zdp(1WrX>T@x)h7_pjn3x-LINx#kehbLgc$sRlC@%bb=2UIli7EQ_e8GK5@6{)+Jqk z16M3xwHap4tO*w)-u}_8mOEIzg8fe1lf5gt zXFq2AE_Z?5H%4y5b3dIMww=s6Pi8QkUfy+oMc9T0{r(9DQq3m%`E3+cm2-DE9_p9k zw@9W?BxBN-jAurP3tK*PI4`yjT$EmQV&d<gy zc21dg?(M=esvo(lM~SqT&zse_L`BNkXc(&1O$aQkkl!A2|huY9_{ zRx2L-zsil1-NzyGyK#$b#*U>=RQ-iInAlJEJ=TAErdc}iB@07oa>wz5rwjXEUzzO0 zwDeTqI%}<2r%mrIo1H7s%y?{;gP2yuwtkaBsomXkF7$m?3o$%Yc|NN9SChx8xX|=; zJ(a79^S8@JeChReo$ataK#={>jl|mU$qZZ*Sh}A~4V23L5LbBTyX%gLs{%hbh4uw; zbWaP-YnoHzxxeA6-R@Sdsn0$I7u0;@E3Y{!_cS5-xBsKEB5jc9;_tBiJORCSRDn(B1NaV4-?Z)}nl(Bci$&v+{4IgA0 z1Xurk+aa-Yu6*Lg|DIlb*{SaW-E<1CwpD*LW?hlQFg=YS;7!h_rcXyate+>Jo~!Um z)S}1p{)X>@?rp}R>y%H4DgL~JF-Ai}n$c4M!bzjrrEtBTE;N8aP`(>xDtZ8(U|2KQuN{1C(9>;nU zZbWi9%&b@+siNk-kn82VJylBkYme|f%{g7V=|jl%gX&f83(w}aef6F9#$T?M!Rgq) zM{a$h&suJ`xUyQ5Og$Fjt$FG|X5j=A-+6(H81n93vN#gH%+;dAYO=!>P0IrlU#;jB zdMqHJy5Yhp^&~IXdxzFOD{DHe^Z!ME{>H0Xk(!F9Q^cO!?_~i_WKZq zPZ#<{J~f?fJ^J=n5yPpU+N(0F`VTQ3jrS_tz##c~0Sn)SZMqXB7(NsqQ_)(pP;q){ zAXhJ&kFR3Ntq3+<=}rC;x3r@!cD1AkPJU*Z($0^jD#UXL2S5VI$F@{YkE-Mrw zBTcx)g@rqcU!@s3npJ)ht=epy5wohXx{&k#LL28~|agdQsgID=5U+l(Z zm)UzVpWOWudU9r4&7D~qk2gQ?+#0sTCB#ie@y7P1w;n!+wAf}WVY;^M+bSKqROjnj z8gnPGOg+&oJblTE^@eWBE(%w-uH3@mI-SvhyP!0u`M0LmrT?!l?pS&JgU81zn?GLP z5|1_=>yP0)wcy}$ac4)Vw24oG#04MBJYd)@%>VdqxPHjmtW)-@PH0?vv(CtGiSpTu zPsvx$7054p7Svn5fuqMo+@~cft#jvR@z9BR zCO>1j(WZ@gf7OM*KjkkjcbmIRPF-1Mb-Txp$vIgnhQ1f8!WQMdex19#qyRiRGh5yvBRf290VkYjye`r>Mn%l3%R4t0v*3WCD?n!>5I%WIyiw}>QG zes^j8;oi)YQTFS4W%M(Vg^b!h$I3n`2G5%+zQxIQ?L-BQ6B3FMQoEju`b8LJnQ*YV zs8%Vy%~-UcXvhbMAuHQb$Rj70o^z30lU@_QxYbBI0%`YBzOo3*1cf=dlyH)ZVw%LS>uMrcOb&<1)-B8o0bzZr;W>Lc-CdGx*^R{dhnxdh9LqKiP zk5=C6*^)c>c3RXlKkwKnR_k-H-u6VwqUUYxGvYRHYTf^%`65SAk7)3%0Hyj0(QB6k zo=7xm9JZWjsMB(}?8&!|xD|cpPIT{$XyOUypK_x=?MKCe6{Q!$yAw{7K1^urmgr07 zNd14KFE_pM-0?2fhZ7Se>Y~B~CPqx;u5^1?Q84S^B;N40vn6#Op34@A%S!ANJHpQA z|Ey5$iwMKS|3+6eIA$G>6c(;~aUy$HcyFJhu-{G7lIcCpN9}te1O<2Ydn}YPljL`= zX!ZWVbIja#zF?Q$iHW(_oBe-Ix}6@s>PGPghNcr7eOq?qMR~@#?9}ThnUIp9>lab- zZbkj+@TLzO6SUlWwkA#Qyqq#+d(XBL{qrh1|FVl7R+6$!$`E!e%AcsFJ-wE1MqlPk zjpC0}zh8)Gn^DcBKDG6`veOE8%fOas&2p!yqQ5_jJz}ks`l;LXqDnc*zx-!;#+Rwx zt+Tdes2?~n>x6)B(~XH64t2VR&ndq-?M^{2`}P@Y*?Ctow!iw`D)L!Y!D7O*R`KTY z`2QEBlV;4j(lGOkMEA^@|2s`urLEd!3`G4{Ib;$z=g$|clI0NnA=+-&E>rSR{@PFB zo1ATSnSwWdcC8HynI*i?_~nFSnbYQT_-Q4!E{QaHBGB-2=8XFhJyX=H^uHFeeXnZM z&~X)?*d5uecVpJcmz}vgyYt=U6td=}Rm>GnUi4#gX;`ILqi403TKl69y)mBCa}`z1 zBF%$uNDE0U{&{KkUiYa_(q~6nF0|Vz%C@ua*7qgz8w-qfraZQgQNC7_wIaeKQq?+h z659*AYZa}VcZ#f)Xlu%t<60`b@zTN>fm3f}lJrOHkgTwrhu^`pfzI z8vFMu&RYF4i&JR%SxLvOfgSF0ZaZN;Sd zCv_rMiS2wj^}_dwi!=G>{FwB^eaiXFp6y~L8X|8+a+V*|t`u!OV9b0;~Rjz5^ z5dOAe^6rKyb)6GhJxUlQ1%DURmfh^Gv#QzlYTknON-gvCuh^#qWG>R^t}Z`O@+WDE zxKsHX&UI(qSGOtgZO>RUdFFyuKZ2aH)<2k4>T0@V@yY*Hj;EJSzO|yiQ2eyiVkhg# zMds7jTW$S*R6?k8iRDh=y5qefMZvVDQ<6TWDC)b6as=b*|%l2{&gZA_pTzv}N;&sp09jYo-c3ZB$dRoZrrTs_k z-EQ>it=x3}_`2rR)y@*tyR{~7>sspdal-Vj?e}+WQ{CApsnPv_eXZ@z?x0H*M{jji zq!$EL?{rzQdfl`ICQs-7{JMOt^pv8G2~1uq-bplU>+YFwV*z7@z$b~xyKXJYbbOF;VUU^Ww8ktRxEmy&3Acb&GmMrZjas3f&X__MDBgM zYi4HDGLMXDKHbZITXgN7*7vY_6Em0OAEzxlI@j$#G41k~sotwP=STEBQ9I!MVp`YN z&22Mx|G1HLEMm3d>$SQYMMZjM^WB=N_;T^v*GsCob{WoGJU4hpM$`;7&W>`c$yrtk z@>w&AIhTp8p7_gakJ@Tc^X|5LOHx};t=**=clKBDrx#*BXK_!cXq#|KMmwX@A!&>G zs;SOf_nex!<{;C|?hHxO?uGNxj%?bof6|S{{kL|r9F(^1+OXo(<~u9Icx$%r+Gz35 zJ?~j{*@f9nA*=Ssc`yE{x;NVKaD7L-&Z`B%(PyP-ubf+TUM2?jp=gw(Ngzhd!WUVPT|r8 z8scJ;ckP>4IS@Wa3j%@lBsnsFgIHTr$Rl%Fv3m#RRd%0@6X^oh)^ctIZ1W}zN#Ctpb*6Laj?{KOi3^EL0vj@qZVX(tO-D0XvM7BE-w_@~gZ4|2+?|&y zSH;~v^Rs+I^P4H3c#7HI{^u;SIKT4F{QVQ>-~Ou6S*i5-(ySzlJ)3(3&a|9#n8{b3 z-Y%u(x29}fR##KKWL~D1{Ktc8bp<$d+FTmF z;L^*YxEK3Etq=7r-JV~0f^+)JtI=mm%u9n8{ogY0uDX@?#itQygpvq{IolAqjLbi>e(J;t}EOOW)BzwCOls8N!8F% z=(wV2*7PMGZkC+8rp=#nJa^&a$p#FrC9c1M_N-X&c!iUtfw&Y~ppkolb)cb=|4lLe z2}vJL*~qj!VCUHGYIRoA_gpaVr9!XUjodflyB4)%Wr%>Um*FUV#j!noRK}hn@wwx)LwjN~2QB{+pXt6@U4n!Ivj9;d#tK zLuOAA>xYY+E(J?|{}cb|#oS7pb2mMkk`7*Ye06U!x1-RS4WYX^Za(i#`utwkP-!h&{h)OV8eSNCF@U7A!)l!R6vCS7=Ol1pl zFO$1`Y2QW8WfIw^T8p_Z2;RtEefgAUwAk`>@^@|&#l199w%}g!c3X=1Ta#CB-Qsr| z`As%pmYl>Y)1~Vj%5Y3ITY{0b`rwDh{}_Z4OwHAK7bxp}kyym_SRt)hmQDYb6SEXU zzgXyVjXkY`EUC=yE~zUP1mAPeeBk#gnN{kns~VT^dd0aH4$XZw_w$`6IsaE#d-17G zU}q>`%bdW*uz*3WA$!(qAI_$;lP?5v9Sgs=n6b7`mU9`qPJ#Z4Mc$8up4>lInHQR{ zVv(x%)^PR({tN+L3m!B53yokg{jB_uDe&nXt!ulYO`FoKJtHN957s{r5*JC#x*`Ix4U0qRoq(bNv6+Wzk@cMb0Du3V1X>hyN)V=J7NguczXAiyA#nE!twM%2ITL{SI5$b}T>B6tDg;`)7(ur{ZxHkE`dN2x)o5wcHWn zs@D2?`C6-744Z+fQkH*~;qIo>+SGP8rk7U10l@r%3PRNj`+QjL(t@Oz) z_lYmQ3FhjgHVdg3Ow(ZbFUG)`Lk@!TY(%RzONde>$c=f&uVJD;q}@<;?|?(>d#bvonzcF zkV^6K0Gz*d!{NhU9iaXyzMrPV<|2= z7cTiLu967W(B%}%@XNTN;pM@t`=Ii{|4BQ4m}= Mf3rc7CqL;)5DRT%ju#RZU`7 zZ(cL?PNKou9UDCALm$LWyZXm#T}O%RYHn-R1CldO&b={VRqW#llU>EO2`te!YpfDF zzQ1M1ELX;Wo}M+ccch(CQr=oS>5P5LnGanFqK?!1&$RBC#cklePf5)-z+%?f^q)mV zIy1}o6im`vme};g^Azv7G(lVIrJ&}tUCSPwp2;Iss5oCv=%XO-agN(dg5-^-&g@t@ z<-w!Ahp)B;Gx#1bneD3FE1DQ;`t(lvtb1LL@@^>ENmx(osIu5FW4`OB2dk|0gq}@} zpI+Imrns%}!GGVS8&<7hDmb#d`ZI6MhtlmLqTRY(Gd}#E>0P?((yE1%5AiMYep&tY zvn!KBn#9WPcYhyjoy~bkPsVXx<^&Eq^>0>@rROb{w9S{Zew;e*+t%oL%P9K@`rY_8vj%`|~mby>C~Y|E=DCTJ-K0W251p4Wm3cSJb{|Y>-{!2qsvgnN<(g9-`sGNb)^y!3wQH51R0qD( zapz!s_N+N?L;TC0?efZc?Y--F%c(ak;Wo8;Cxz)Y&nok}R#{!k zPB>xzzi^3k^OoRW5{xH5859&xyd`_c$6}E}=qc5iZxjtDS|0W-YnfhZJnI?L%MCL! z_@=qL%rmhxlfG9fFtI6l*)ao4pFO7Un@nt;`dwjQ)35tzKmE^B-nl32E4lDyD ztl>xVJ!VFUQ(rjsX3T!Y@kGwTfK`uw{*;U!E6zlgQmajtv%5L1`wj&EtL!V7oG``q zzx6Du1}nFGTf>zWJXJ?;F5xQL!L!u+G`o+TUOndtsG;5r(VxNMRfyKiKmmj}6WO7O4__+tsIWgW9 zK2~#&%DgmQKcm5i@xavhQq@2~PtB-bT^G2UzLsew&1VnHx|Y4GyydZOQg%*7^ox>D zg8Eg_ev>o)$J;5LuVi60s1kLXeL3aKX2yL9xnCyqJhJ!^ymSqxZ_VS`7M6@W;>A4Y z=3Ljz2eC1&4q-MHFV0zTEhrKsC*IH%yO}9y7RXg=%VZhUpC4AAdT`br# zkD2USFRdk2;i(l+bHack~>R>h@IIlNp!qzkHGp4+o@g-DLSoiNH-}`JqIkj?n zf@`e9w+YO*TY9nZ>>p0m|1vK&n=gA(T@*U8ywoY2VO7jItA+D5G|jwa1jKl_w|5v8 zi3I(W%~o;feH|cP*`SouHzE1)ens{T#jD>ftL~O`x8$0!?Zy=SgPRt)?MgE(kUV^Q z%NuiF&#U5%OO;zXs#~VWtz=~2;b;ysx7gq>qoBho)hp28sQfX;qs*-}d|l5&ql-FA zHg;I^Z-|Hud-g73t4#Zk#=S2czeVocvUiJ;XzJ>Oud4YIcLbVvt6M$2HIL&oyIkz) zRu8Y>OWx$xY-q`|^$f|ADHv z!iyQ$JWSR}NbUME*(Fbv&GfhORUZ+XH6y{*2lav%n3JeaLjTJ*30sb=yjAV-L`PP9Up$YvwU-2Gir+(Uo`{Y+Pd7|9@U@a7AuRd&eNH=yL|ty`Oe$R zk84VvEi5;P4~t`2U-x;2 zDec&sxm-5B+yB8t?n(bdc?~}2$G!Vuf4}HuSdD{FUiX*0vkN`WyT>ISiR0Z`{X)&u zG>wNVFx|vlx*@#aSD5UDr%YxG*fkEQ>}@KnE3*3C{NI8nfHyy&Q04$@{BzFx%S#s* zS1@+eJ8rLF7B9(}7InkTbAeg$i`AvVba&nV#cAW#II|R!T7j}Dp0$?W%~21!^dh*o23+N z_00>duncK2iYPzSl9Io?VPZ>d!PELJ+xd(u+;~MYJf}tPW&K~XdTXbVQ%zNd!>#X5 zzTb+Nr}<{7MocT}RmrgY_rxNv&F<;~f$quDXPp%fEUI&@P-}6Eea7mu*sYOE!&Wkx z^W5az%VF(dUqYfRTqi9H*zPR;iN$3Xg(lcgJGuq-?cX+ctcQHQnwdBki zsg_9}S@lbomF;D%Z88fuQ)=K85kGHo#l90W=7nbUevg!!I`N5GX8pqOjmHDbmf2hm zt4_GW8zeE)LZZUlbE??OX&ax17#5YOhGuM04p&h3x#Pkq5*hkiG~6M=yD+7`)3ZqT zdC^S{dpQ>?jw|l1KP^oy>}EWgv2Lrw0giaRnJ(+DSc^(nTx`?jzS6SAGv$uwhhx*Yl+6DsypPW&Qrn1(x3>uZcO!O`oJ%l=nBa_0zLz7PHwu#6zF5)C=93`(E^a z^M!?RsUaS1-)1e^=C#0bj;5+@tz- zhqKb1+bdzdXq2>MxsGPw>6NMHEhL=Xr$(*}t4dEc_!LzTF+t$v?2q5Qgtgl1E2q9z z%d5EMrK=ixjxqCa$>gU?UDk+&XKd{LC{|tl)T3ph^Q0?zd)+O#+){O4)jXe>b$LhB zR^h42r)HKc&r10<^`uF^t4rqMux`H2rCYsXmsnNKVXD|sW|9>(Tg$HWU6K|4$quVPyMW!4PNT`_7^HtrhBv##<^UmCx7XT|-Q zQ+FMlsWhu)@wa*Dvu59L+weAY=_Agm(lb-TrY&)6o3(6J_Jpt+#*o|t&sX1aS*2n% zdH0i`D{V^LtAq}x7k}Hj(X!l+{Y&tLiVdr-MqT#^W%=RrL84YJZNrokQGdfT)^@FN z_bOn!y~u0nEV0>>GfpK=XX~;5?S6bxd2na+_Dz}5)pHttEcfV$o=`dUsb_uQi#7Ld zWnGY5vg*0-j*P{t+xurkP01Dg?{xHMr`W8r>d-k&Vhbfg#D9L>_K|(k)34qRW;33h zl3s9YbKUIbu%{C>ICu6$_+}u8I(vCWnU-GB9x7@EbU!%No^TBzazpbB| zQvNr?gROH-s8qDn?}aTdL#DXxYuN4S#4T#^ebM&rIrTSs&V(nt`Mg+P`~br&lY=jJ zGD-=r%U;5NCr8z#i1&r>myAs_e$L*r**R$D{_L&)7w!ALdu#VL!z_3@^*eF&U%3)`u)4frb>6GwNq4wgi<{T| zwA%K4`;Pl8+jgJaSyOuWP)Anv$`$92O+6D;uD9s$tHaB^yQizOFV$&D?%!0OAtSAG zJ!HzE4H6wI`I)!Xha>(gHP@|pVD2`aMD+s*)1%kNtp+Wt@a-MyuGJ#xz^OV)77j$woWWOzP4xe*}J<|hg@|wdDH$# zI?SPL+b-*^C$779c(1Gd5w$>Mr}B~Y9>(?4uH*z|*b_&YO+(jMLp(wuZ-?`hI17oXol(VHTn*_jkJZ3chiBx4XslO5QZ<(_tkpeP^m> z)EwKp^HlX8-|Nl+Ss@d9Z=PIkJ-2_+u7gYB1NYQ!z8>G?eg5`Nca7`4ZnAzw8V>Wf zd$5}AKL29-tw~nGXP4@Kb@%cv^4}SDzq9w{s&${XERBt9{jg-kMd5(&8WFFy??^40 zpt$wFl~$+gtkl$U1vMkdz8`D2V{c!tJ8OCI@MPs(d7Gx&|4U#mYv|hh(o1{!+=3U% zH&1de`=G#D(!rp8IgIm!VaTl2*{(dlOYGX44Xe*aWrnXe>egGH%^<+8lWg0-8>H!( z{QV09zkzYtSD86o&ub{w`y>Qvx- zec{h6t6oOd)bRSxM;uN!rk2X2hIjd0mN^viQ)%C>!0TN%8akq67v&rm$PH$UJr>5H zCs6g_vfD8k)dEJ=#fsjs7jK7n7#%l?O?7IL{K&G}aFetacSNy7Ckq4f0gVuD>qp7H zPsMGIvS~N${crCz)CaBPI=7!P)TeX=N zJ$T+OnESW+;hO~Ru+8i-28;`z7tH?{wfAq(>Ibo#Y^{3)l)M`sY&yc-=)`@m@yzKL zPj+5t3b@;4_vw1|RXSCE*%ONzb7cglOWOk zQRmPAZ+TefP~2l0+8jvlg@| zO}yXR`fT;*K|lOBQ!UfG2fB5tme=Ppu+BE*5@t}3*%UMH16x?L zx@ZEkKz&__>Y15)uf5rR|LzLu3w?@2ZnSn-d!}8jmvME2!NG5M?d;Kj&b<$DIjWJLN8vYxGY3EAr~4*%#aLJ-SjC z{-xO-4epr#qeGsp$3Crhzq`kl{n~f)S{AZzp2uD6$ekr->#Wb1x?cQhf^LfddyWJz z-}Kk3E|g^l^RhbXr9G6(4|vHY#Q#4*YSo0C$VYts|C0kYs4Y(9Hdf%Ctq_^xVK(u= zm#U8}>yMnqjthUMJ>k?dVD_^U4_CbFGvnsX zo+-A+SG`=b&Cc57o;jE9vc&3ldvCp1FiXN<+2?Gv`<%WE9KHc%3_1+$0qtGg40+v7 z6P$E9dxRLOy7}9MYukk)dl=k%s`waW7@QblI{z<-T)&__FL@E$_lb}k*RmIJ_SL+s&klcobFR@@X2Q{+A1lO< zXY3OxOkn7}rdY3J$TYzqQLvkFw$Y>%kz+^a1$Rg6nj@xl>EtAaG#42SZiOXgPX1f( zJUem9)$PWjk2xK$Cu~dVVG~PeDZO|pBYNZ3NR!Pn=bRT;+2xp@pJ!i@SG$Vq$GN%A zWuJWNOm3W*023_sPw1pZ0!`edwY9= zR;=67u`s^vpQ!o;&vMR>a$a_hmxD5Nd={OE=)S`=yFBD#h>vvc6Q%<}Cv=XshEI1q zbV&M)?k6t|j@%$_W{GnN9mzcbOs+ zIR$a*rQUS2PrbaLI3U;gp<-BzQb@RPN}*tH;I^&FG37g_-ZS{N(Kx|AXKrTb|E}&8 z%i`z!X8d+t_TYguPUrg6Vu{)UUlWCbx^IgYkJ}Ylhypy`GF!zDzz^d)!l0|q!j9fIF2pXU~$TR(EDNT z>={wvd6S=IDfU|NSgvB3ebqS6Y4K&HZ;zIr(2@BlA!Grv3!3Q_t0m1)-Fo&%oh;-e%W@*oU+2BZx-z=Tq&Yzw&_COi_)CS zJwCl^H|!o?_BiXSou#^mced{pn>b+`!`e@h&xQN*uJ(Mk*>@m(-Ttl&Zk~B%UAuzn zn~(ZTI#9!{^U0w~`G(=8k7l33wp`gIT5>~!+g2dD>*Q5S-BjK)*-18^wcqLl{JwPU z>2a^yJCjzY{@HqK@grUHGW$Lqc)IzVtfXqmh2x**T7K}e?h(Hy|9DUT=eJLJ zGGcCPDXDkW3hsXXuywzW*wNnyjQ<`?sk{5<*(biM|7Nl?^NOF{5_A51>eJb)<&w{{ z%-;Wj-AjJvxogwraQ^>buBJWR;lVeZIjmmE)tj$Z*O)szS=wdyy~+Er&nBH8iF_&9 zmtyYD`o3n3JNHL+qDjxN0-{)>yAZF{jZuH{J0 zl~uR-T088`72V66j?7_tdgDk9$2(;XNq*%YBF;q-PIVhDKRRy}zUnulo=C)pM`Fq8 zwsYSyS1sKAAZY^g_P83KiI*?Wx%^_j1c9qXCor*Td8QcTAqSV_#`UG{^fYRg+j7y>C3pXZzyhZ{E77 zv+S6f-lU|T2@gc`JFNVQJ~oBrDDX3iI;76?p8PB(Y}We_u7tz({M-qQsd7n%MnKk9Snc@69A#s8Q*FG(%hSU0IgL$5MIkAKmd zMw9G}#IT<`JeDZ%+_YXIU(D3N<#49rw1}Mj_sX5c6|7!Uxml-tiO|vMO`ci%xwED! z%OU99m#UDd%Pmw_`uWZJIIa7Q;;Ty)L4D^$yC(VkpY7XSW!8Blc(IA>@W`9yV;_NTw2*V~L&cHq|r;nmBQev4I)^Dtj zb8KQ1+Bh$3>$3l!HtpQ7s6;nb_y+UhcE_(%d)BWxa-HG6qzS&GRW3)})V!i{HPm;!)_k|+ z#+4J3L{Bu=R6o_ce9h#!Q^?wPQ(is3(CVfobQUyEJ_qJsCV) z%3=;p-2Cpv*6T)j*RM&$ABg*4t8-TM?zH6xnE!<~SkBeoEGKSs_=@O}hd!B;0-Nr5 z_MM$(RWeO_CC_Ok=bH(lOB8E^EV|yNDf4Vs;N*MyH6`nn@%rYgQPojL?V644RjSmbTZr%Bu4#L`Y_lu&vo~kWdl`LQuJNnvDdr9H*2aV}+5R%9-M!3|TllyIW^!( z$j)o&al!oRK9}Un^V}m&Sx?(Pb-VMyPcwBr-WO<3p1YB+OknT1>@T4kjxNf}-fCv+ z{&Ru$*~_m#`LC78Z$07^Wf{jB9;vX|xo5eH@b~DF#_hXU*}+D z{R_<2zsNTIpYQjCb)M$_rqm+;KAAg9j_)}3RM-FVWy5FDlQ;QgXX*)?O{q%1Z+nVs z!NLgDR?USPUlh|A)I(Ul+s)py$9=cP-z)Du&K`XC;-pi#%l7csy@_+LWxcdmpK|H$ zEqk}m+1CB9Zf~CdqV4JnugcXC+mBCu^Y!Yj=ZU*4_g;K+$gVKCeTjJBMYA`P^qRJt zHZFI5Emo{FrT*6PT;=f4)1qnrw-r^UH9kmjVrnn$+M4d~lG{6_$mFT4+cjsM>oE=G zrK_8bs={ODD~D9F<(0aYd=4(VcDUZ}sVmd5hM=jT=fs+oD`cXUSAS8>(QGxC|J?kZ ziHikmshox1#CDSnkIg2F{pXzerKTa(QZ*!ziN)+~W963Q+JuhC|BLw>OO39vnO?jW zGheJUu)^fy#HRjhQ7)S!Bd6HqmU3(rG%vo9@+RHo@zki9%cCZzxp{LWd=4v~QEEKz zdwla(ljKXatHausEzxQTkFQ7#GC$$!xYR8*yzax&I^`ctJ51Y{H#fA3ZUGfjnq zrcL*9{p!pgmUcp=de@g2`4x`N?%pf5$8Wyi*c;ORE-iC+s#D12j0w|P_#;Z2OH2ef z0@;`4zF=Ox-MO>2r0D?jgET&UqM8d>)v!Z!R_yt_)XqDjx49TU$y z?G~TjFC>;6vm>+BqV$7Bdw_c5-5rxnUv}uGi~ERm9OKYksNDQ2%{aML&GU(iad>aa zWot>vjk1})l>!g9Pe`en-Wh^9Q%n!XB~#~H0&VXimT9h|QlU%NE* ztZLdm2mhx(JdO$1gl?It5X_%(xuN-p*xVO7pTaT~JuEfFOB5dlZFrixs@;F}_TJ|$ z{hT}c%Tn8(icFs}-7=WdDatLnu%r7*M<&zc|Fvsbf;?B&-cB!^_k7~lsa%sZ{nkwF zsNzW4%Q`XPXXVH3{!Y*8mqkvQ7?L$dy#MnxvCd7 zX}S2nZ%G>JLFUirt`77(A34QfN3_WHIWH~xg1$_+a5;)$N#|a+DYw~UH|z{A_>q4% zeM;q08}(MXA75-%xtfKC`>#1U=|r=^!*JVQN5ac)MhU59W~eWEU!@enN4h^sQ|={V7N zTRgWQa?+Iasoz*Lb0n+o&WPd6@IIi~6n$#pj`m3$o!v!V)3>EBWBp$DW&5I#WnC3q zvxGCuwtSx0G($W@r8(0m*IcsbbF%KN&n;!allFa^J)N!f!^-#-65d=U+8L9C@S^7G8&C)aE{)f(lNA30-YikQbms|W+vwwbD~dNaH1 zyDH^7mhFt1@OSH)yD7E*o@J{tuVZtWsvqVnpJ_Aka!cILxq*^X9?tZv^ZcLr==#jp zC9TIpnliOQpR8y67sxfi+5t6IFn znnkbu&-A%j_32dJqBO6G$J_os3NBwA^EqPWj2TlsIf}nuowP))Z2GCCu}>yn&e&Dx zlHj%5t!1Un*BzT?xh;R@vfX#uoPfx=iL@nS1F&cb9Mbn>pN{z6RoHIz3Q-Kcdrw-e_`f4)16aJWv}pgA;>t}b>_}~_t|}R zyB2=n&bN<>dmOda>11bU=^EvgYxb(H?K-_?cSVeBduxJMijK!TX?3-?tEQQX&fdV@ zur)a7!t%DauMQ~uDto&7f0_H8t-q9)I-H5LZ;4nd7SZkDn-;l2MSP*~$9fmhEezl0 zS+sX??J9G~p8R9?#?{}dq|G+1vRvA5BdPo0LaW_j57HM1diEaVT$S`SP)Vcdftfo; zvVRNLtmf6z?KLZ%vo@W-eg1Xjf}1ZE%9VHL&M1AL({n4O%yISATdSvTl030u0+a0m zmiP~>ZX1|X8@5gQohfPUr#K}gU*^CDwaF~fD_$gDl$Dw7P+KCVdv>XH(8(VYy8qbh zVxJbfJy$`)_VtqqzCIC4+5CP6ZJCx+T(y=i;mR2{w~kCYo7MVzznH7OQvnl~0<)0=lgb3PXTJ9m9GJu>Fzr}1JwYu$(Pod- zQookDTc+qFZkT)b*zM)pXI-1J`b^5ol>VNuV?TOy{@h%}bZ38fn=;pNl{H7t?Ji&9 z86%^<>q7x+cmR{szK5y>EOPrE%JV&vTgNCBz`D%$p2oaKFXl3aZwT8Qarj$@FWa|C z4}46QtEDXUEKN*qOk13^#67j3dEPr_({D$X$!)AFnSRbY=a6#g2C2*=*|q-H_>-9l-$zjlcT9qb@z^o=iZpo zOD6(ftvGjU)|&R}CWGg`%C>WvnL>B()bYN+yy6~n#fDdF=Dk|?@73zKXW9zvg>kQ! z<+ALmn$PxE_{dNF)KeS37P(EDacN;eme@4SZsdaQtl4$$VZMjg zHJaaey!}r6&y{QTa8~eH*X4!I?~F+eJhHTS_Thcz*KT}z_3q!h5BHe;CfwU5_q?cq zb-~`7H|PD=aeBSwlJ?}@xWb4}3@rJ9_d^sdF4z9h0ue1<$!T+3r-K(e)>xPnSyZ9_jyagY|W-dkoV@xzF$SF@4Q@ozU?5 zuk?ql4V-RDF~z}#ftS-KI|l?TvO6m%azwGym#JCCtd{C@EXjFdU zt?rEtbE3WlEuXqIW4mv~|Jgfq63%#c>*@JDli0Fs`HJ1Q_`@H3%e8;zC-A*!{i7f2 zeigk^V_m4dvRGZ99gy?@6^)SGW!;vi(FvFAd`3QJVWpthT{unwKUC} z*UWILL*^tBPD0(_=2>I$bUB{=k<1Jbgm{dHR0eF68^4w>(a^cz5cZ2U4H* z&D&Fa=-txIwLbQqLh6%0e`k2Wx9gAJQU>$Oq1zfhwdt?AbIDC-=^eHM(gzC;&bGeD z>i)o3rr>}ALtHJZSizY!n_@I=Hx_tlIyQc~6{xfDsaVOK&clLucbg+QMEPLy7|AHMiLKHOS z_DaO0ujgU)Dhw?Y5b@c^@SR~s-kE0#KKDL|D|IYPF?MX9=cYep;+;2rUDU_(u<`g5rMVLh{FjV~Vqz1roVd%s zep=VmUhR#)jP9&mI>*DS&3I5Dn^^$2ng~hfT)os-+neE4txlPxqYC?Tx=Ca%fg@sy|xPEMn zI>}ipqZK%%VBf{dAqi~LkFuO%4CoQycW#Z|sMA!G=vMMh%YV)Sn*(YE%xBzYS@Lf% zU_7kNcl67}~ZDO`AnUmU{ z{#H>8uXxa?6sDgv|7uM9UFPfBPY!0S&YZH>S>mn0u9v%RG#VXXINS87K_ijp)PesY z>(*49?woJ=`Ep)@-x9-njZKrj87n-S_E9sSX_Ii(>XKt0^>egpWBX10r^guUbI&~rX^!@v6;54 z>G#L$KG!*|S>JNOT0C-}@#6cT*H63YImzx}6SVd0UF^+#XUduiv#k|do5S>X9TJsD zXc2hwoZbG!0nQ8G>}EdHFN^(k?6&{S{Vuy~# zc77Dw=+G$h|H@8v*`iMG2sib4rvp{&xc~6y)J&To*~*@1QslizWU6y@Z(f$O6F*b_ z@xGs}Pt#vSJ^XxX;z9!h2TraDjXYWl-6UHacv%lPaqV&6zqoZmNyL+j&(yxF9%MOb z`}xL^$|a$D#XfDY-{bKhVM~RR>c{sn;VtJm9xggM)#be25{rJ>yN{N9oLW%F8zcYw z-J@M)s+zphg54<1-4~$ zEzz0!lHu|7OPcSRXN0@(SnNIGms`ZQ^4gbkBEpr^(xgm1r)It0dnbHLp5c>(DJGMa z?cq9l@WIBgpKqR3ZF;O2(4-xAi}7*yv;z@37S}e`e((y54$`mlnYLl3w$Qspn}QnL zHZK31qMpFB-9-6f|K|F+FGF2dM@p_!+46tJmiQ?%SZ(qaf%Xhv)<=Tw9=j)|y zAML$S9O2O^zIl1btp;|TkT)hX)-0ZNy)e2mddj>l>%Od-zl=}Zr|j?2?ySR$az%IU z`n7A4r@c_J=i^0(MY#Da&gIOJx0LjgKK=E;{ev;JoOdoiP|DJG^;`AoLF>-BYQf8c z9?NYPiP2uV>(i;;!>eQ;GAnEi_*{02Q`b}dmfepH<$7%U{$~qsjMiV9W@vY0kB`~Y zr=h1i{6chVgI+B9BgQ7b;k5QT<+nU<4!`Hho4oa`q~gavN^gzI*SKh>7UXkly(-eN zsmT3QJe4cJN?PFDZC*pOj}{Ln2z~6FnW|D$>4eP>Xh9A4Sa@b=8b1xY*sSe zzD)Y7*yfqH%h`4|hPE2)wMff`H}=*8Y2@pfVJ$=^3F?b{HxyZ2pwl0-@HQ{%YR zoO+Rl7F(C-mRY8|-<@SUNw@Z7X+eI}3R|{M4^R0nHQ9Js<@2`O8|=*H+5fx0z4WQP zvs>`0x={OSXY>Cz?&Y z5laS_lxGr_b`syOt!MtGc5dHOzt@j;2TZv3+$z;NS}l4;zuURu-$^AZc`tR=EcpHH zy6Ds*M(5bZ{=QB9xjOHi4huJ>ybV+N?if-0J9VwSsOrY`cV4zu7tSx8_i0B!exHy8Xk0xeSl89nD-Ug{<{iQbd_tS{WU!O-nU4OG{y~pAt(@pUUzm+9#Pks8WSnpX*(6Z#PmU>OLn4ThErfrf5M;i-{He^jp zue)x`=u)kD!@?mstmLBOH}>2&!Nq|`0uNr1_MF!I=$e_wGntP^f?h83npBpd8sRQ* zTzCEFhG$IKku&mrH;0;kNo%?;bF!pz$MXD5(_?Z3eXXvPzD;zw$`Ze*H8Phi@Wl4W zd&{lex}x( z)`)#m#o3?Rl}zuHd*1x5)wfo(c2!B@(k~8s%yW2t$Tv<6dYGAp zuL{pino&IelY4@A0sA%=Be9U%9fGFc>!!J-)}%)pEr~MtQS7K%!1%58UrM!7MaA9n zYTawSm)PV!vRSrFYn*UBAx6zVY)iULg;nYC;xo_FxmIQyAI^O+y>ypF%W{oe8}Ya+ zj@_p$yqMCG)7Xm&%X));{LgRyn!0?l$hO?U7W>bKc_VX_oN@M%=_B) zIic`ZdqkO9&uxobrsZ=NmM60b&U_=#TVWo&sKe9y#?*wbMXEncjz)BRxTrgSbMvFd zx`?uLw~WvZv-~2>sNFL|PO?ot^Kk0ww)Xm`DV9fiTDLc^QP1Yh@+i3)#BZLldTF`E zak=SFY<?ME|(p_Gvv6KP{MYEI952-`5>&TGtl_%(9g5$kr`iaH-X`%`(=^ z+|R00V$({M6>cdWojuwuxi`PGl~p!6%+Nh~y}n-S|5697^c~y%78rXyWs2+llsmm* zLHfkLj_b+4&6{J{;?-ZyH@ZIUYq~?U$;9d(k)9EyRWCXm%I3@{Y&FlAx#5L-sQP62 zr!)LsHNKTter20V*Y=7R3+FA~vdmz`%CtqL#Ib%sLnYy zt?>z$Z{+hVuP;d#W^QQyQS_l)(vW3}Ip@MXS{%QlHnclN-N>{!e`1UEqm?rrX{f3$ z+wrsfkcg_?c8y6=6PLJ&y{>5ayd-XwaM_X_NuQ*;HZ7gI9UVg@;d1jTLw8gq(D?6{u zYKcBR-FaD4n6=BdsE#SWTIE>sHz%*E(r%c4vR`}3?y}HDY_I&Iq^H`foLe`$IK%CK zpVE)WM-N07F=$jVXl(MfzN4!0|7gnGqVn`rg=RvtS1L~un!3k!_xALw8JoZ5t8ec< zoi6ddX?pw9Gw=UJtvGbx{`QqFxi!@V`tNVQ0YW0G~!%Fh#a9h?~dWYWPWLd8cG z`Cn+u{ko(7*X6Li4{a)5$4Gi*`j>R2yY2Wsqw}QHu9WC1QK7w;L|o*>*Ok1UbGCZN z;)p$!8p;d3y8mp5UT|XTBCFPQpGEthPiraXZZP7qEMzecWC)8n8eVhMY~s;K9tQUY z4{@vT-rHgCfAbh0UBhstVt%IQ($-}8js-1G^!A1>DiA7O`NH>n<<9v+U1AaqS}&sN zHkSFHE?=Vc-zRa|b_4Gc?lRe|L$Ef3>Of)g{3rBCE}lf*hT`FzKe!o>j$ zM>i_Szvb3k08fX zp4lf1YmV5b$v)bmB2?az|D)|})pnUj+;JN(BuvsNZ@7Ari6LSy_u>s#XDIOLp4nPG zBWP)X@ODPwEf%w6ImFIhvRwRuWkCU#&I5(3m(&-f+lHhHmYof|JjG**sT}Vok-G<& ziejY~2QZecy}Z;xB+KZ^)Zg>BN3FeSb8^=9RUeWkMRLwfUcG9=x8A2EXV-t=Ir&IC z`s~%ny;s#AX}5X%+!c*Z-e8ozNNEv+jLZV|f+XFKcht{5=FWH|dQ{=;=N`$&+IE+Z zsGD75D_E?w$<|i(k*IjnZH5OCPnPQ}o*>H3@ZY@3_|h&TwI!_QnQNykj_}m|U3E8H zti0XoypiUKTSe=SZsv;#JC&omG3n}5o>NO0E}He8{jx*(^>f9l$85V2Ss6Ac-e6|1 zz340@aG6J$aplG9>ozI{6fm4N6xO*QnP?zg9dNzWm+SDu2P-GC-t^+&5sF<HlyzEMb#}0}do+yas7esIUVB}o<@61Sn$DW0WhG|}kDhLD>%X13ig{04zT4)TA$NFqk4E)9J#pdc zinCW4jTRjhDXFyK+StIWX~cCRkU_XWRHuRWXrTF38{vQncITRGIg7Xj-rWxQXr5*y zJ^eD<8bi@{2c&sFpSwEuZHpam5g$W4FW3D%o)kXk?S;pGUz)vX*0JdFw9ns9E1NF( zbaE2Al}Ba!iAP)lwGL+&`QBR1_j*CWtzIA_4BZp4_eUpnB! zx#tI&8ti0poJ30Vcy8obFgq%^eb-?P;HhGGySsp0I-W~!f}wcE>b@y64flLbUw%f8 zbB^oV6LBkIX4KB$-c>qVfbGy-gIb4x-3%9(#(v$vcab$=`$snaI>Ton2N?{{UOp7} z?3%AyufOm*RxZQ8Y(9ljS^M4|a`=2Dm_s~X`qBp}euEdx2EQlIS(T#{(gNZ|rp}AS6IiRJpjlq{es#CLtC%>avubjcjsk3u-i%tt`ey8vp z290IStSejc@@6?LU}zKStXk8Yo7=o;<-E?k<4dOI~_`XvtixSTeA+mx_{~^a|ieI ztxX)9tt+4B$sW#ovg_<`(+tJwj*B^b6BxT(#TX8=SxSGocttntLJaRJ(@7T=xv&@Z zetI!UcnT-GMP||e(vQ*&%$DqtGg~J~DLL=1nRUqhyr07WL2|x;ce56Kjq?@zpsmrSblU{-B*+MZ=yHf+~#ZYj8{LpqBcrN`$oD!LsW}_ ztX)rqVRG=ZGi5@Lv{deG&0Wu%l$Nb|g6pV8(T&Y6K`Johu<3K{cO=k|j zdfL2qh8CMb{Tp4Kd8LkpH3}T%>|b`sFT^H6v8AK==PGx<$V z>{hBNb}lWlIsN=W{Xzr2-cX5oGl)&qjV8fmAB_H|tM3P0B2e8P5Cu4zlu65>}!v-d8u%sKRkcs_v>oOM~;U#PFWcuSw6}4WJ2xsmXFi@ zb&g#Psq+3ebwMkeo_km@=dp;mFsZ5T6Dt@_K2DwTu;z!#EWcjmn_b6Fu25{>&k}lV z!N#30VuGbk9AdQ9d-gP~s`M!H6Z@AMl1rmpq&Obh+D;NpS)5nSbBlXlR5Gr^%A8qJ`p* z!fJG1aMtp?b-mmC!eYmy_DKib@7mnEr~3T=#FWVL?CLGAB;xGJe4>&z>}2wJnL^8xH(U&vZkFr{#6^vx-(vo)8WERZ@iZ_dm*tI)ZV&b@xK{7pbt?2ghTXU%1NbDcAy z&N3aznak4k*r)g`@1&-!>20TtPi?Y`DqUkJ?=w9%$V%hGIpxFCH{X1idMj+k(Wu(z z>k87NrkA94-P+RHaESf&`6pZ~7jL-VnzPJe;@g|0mu6lqJ>QtIHa(^4_b<_%&R^D6 zxg5$m?PVToKl}Uk`wQZhy)Eb6xIMQ#pz6D2-v%$OpPQGopS$+|>cT_3AFXd~+g=gh zAE#?Lar9(6i&&6`V`z681z2Q%HjYc$=8uaCQA%jzix9c>5QG~4Cv7KwlUptfq! z%HGt%$&)v2GF$$7fp@K&l+?p}M;&h5)pv`oc0VX}ZIjTlw>6WZFD;psav(SJZOlSO z{$>Bn?Re{s*6qKcaF(lm$^5k*;cvJio_zV{sODwZWm^>gVO7Z9$glh@MS+iZw*9{@ zkbbnlpYNf_^L@fStU-p(`D?Cx-1f}r&)#;A;@Nj*-1=sxp{Cz-X!DiFCgtyCmfCK3IT@-|D#RzP`Y!Q!)2meBZe+QN%Oy za-{V3?;HLHNr}vqzk5$McloX~+mOXQn=hSl@H#zZTCm4^$yoDujv?y~zAo83ng7Tx z@zc8kw-o5vUSaT8^LxY?SaNQay7W%b&>sglA|!>xmUJcVpDp?_?2%EFe?Ld)wT=2y zK2~Pwb1q-9Iw{}#_oE4?_n4crdue$(2X!7hv}?D;TAxAfKJr|W9OPt3f>KY!tBzf(LKg5qiZ`OB6hyQFTp`Po3Ydg~_%pVwDcs*4%5 znye&ViC zP>;%;Ao;7M*97NnSy{9>*_YQnblo#%>qWkAo=-E^S|G-~vY_a?@cL@6tuHQy?^yEA zMKn5kPLjyXHxrZ1m+rn>aq3G1!ykRI%UMa4$9VNJ*1Xc5c>L(!l_xyZ`{G2TXZZ3l zcdefOcYSry4J$tRBE2^*{_hv8N%*o%DA#nK`~{^crxUll(|JB;e)8()N16A3TCFLn z!Mk$hy{3)Rm&KnxCU9MK9s8lPp4KseF;;EQ+qbMqF8#M~nq$u1{)PT^>b4&Z4P#EX zI!&#Abf7P6rKi=p9Omu`8<(CB__;j!)i>?`VL^|U&b08kAhagcy!2Cu;;~y?lLMFC zJA8Gzt@;xs<=mdDCbA+cGu9_~+?=6%Pe^!k)`8>4zIER}nO*l;RnkZMx{bwEoi*8~ z4+`k+SbSs^U-^&kysthoUr|^R6n)ce_pVJ5^?O;~clK6aV-}pb_Db~zS@~nR68o+c z^Vcfw5M+C?P^|diy&Gml?78{b)dIyEOAXf@pZi?*bh5;`+(lE3t#b;rc1G+!Hg}pW2Mr0e}4zDvwv7_(UtNlN@~ki&bzPnEdKvVYWCMek>s5@nQJ%n z{n8R_=#PHNth~|OgGVWn%$_Ld)*>UwMGv-9shzg3Hkm3eb(?aQMTrc$ThuelKZ zI5ck8-qjw8+h2!Q6|emly!Oe)lb;xrO}1WXzUZJ|lW5(XQs345v*7wd?-#ih$*pte zT+QVx*%O;0;^=`C^o{@Za z^6ZFywR<(L7H{rN+qBSJ?E4=s%cC{%YfgmxzPr<@D!Nbj$35#ecbNCy_{uST|CaV$ zW?_5pUrRQW{^@13HAD9952mgyC#Aj_t}cuDKgIsdv#i8TiIbP4K1g-Bb2x;h@qg~q z5_Q2NwUf7d+_>tuJj}3{rF!<28b#4Ef0YQ=kZQKW)rXGzXDin)RW4rS60*!p_s)j6 z7Wd|oWyMWb%zaIgMMX2WsnzXnDV~=ad|5m*&t1+!Nq6UFuk+6JDVQt&= zMdf{)-Qys!-vLz}VrDbLn{I{JO?aB?E)+d6-1|g$%H&D8?c({jRn(45tGL@Q=EEBDF#bYo$CQL%$g@y;|X`2BUN-i$)8u#&!`lVZ1*EK$kw?o69_rHuKp$~=*_ z(4a{(Qp?g#cFpmaBvVm%x~b%@q2&Lz!pkBql1pnYP1oHb-fpqjrg>$UiDn$flbTzX z+K;t1J~T`!_pG}0++ah9TxNRA&!jwou<7hy`g?;tz8tC!e%LCrGRvNmUtmYhiYwDI z97Fm&ql?rNYK!ASmsjj^ue_QOVtcabEN9e=F9kw+Nb4(T`keb40CawzwgpKQ%0d6&Av&W zm1ncr6g6sJ4L3e6CLg)<|K#_cvT+)9EZM0a$70uQu_?|wENrq zGN#797L55bz3b}MPU}n`$4d*@N~;cRmHiqhaiT$fTC3%{(rGn~-8qJFapDUnmPS_{ z>3=P>*g4W`!qn8MB`#4We%a*{ud>zd)od<4Syi;7;Gv`_6ki)=H%mDFS!TkD;GQ{}+0vz@X)6mhADQQ2(pvJvI`3h2 z;q|7S)2IB{Z0FP(b?9nahDD-wS@|>f=G`R@S*9yIci3x4mEGfrXmd*m3X|V4VR7vN z#%=|<FEEctaQ8S>E?eqf<4?9a#T&+{nQ}(W}4T@`PbBJ zRd+?&bajPap0a7_6m5ybua{N4wmX?$l{&nNzx#Ob%1i9(2Nl$c*wu_Uo^vrgou%cp zdBGNqd2Jjzx{D@kIqqs&I(uByOg7Ux!eSGiW^Vd*s*T9HmPQ`kgIv86xw-EF4`20#nyK1wPCC6W-S&7SJ zP6ZmuP2$>a;G26&VcvqpvWM(c!)>-7S?w}8e8LOG z3pTYBuoQ6Zy1O#kZ-byLBNzJtDLLmjBW4D}#PF)uY>OANJMZp2zF0K0P%*Y~%S<-= z!wsy;#)eE*d#6vz+^~zQfSW;vk?X9X(}syN1dF$<5!eyEQSdSY!^91Oo46UyEzsyQ zU<~}c-8Q7jH9Crwv!7GKTeRA{eEY7d1?<93iYu&kcP4TP1^nk;AI)ufbGhAbKHUpU zeG3?+K5UUVv*UWVKy3g+o1k5Ilm3c_oYyvRMqb#r(^HoH&cS614k@2eRp#0AsDVQ{ zdZ+B1#LfVr2e$>=5)ZFgu;=^+*{}tNx*J4=z1KeU2-=doG2>(0_U5%8FT2!fr(fdU zG{-A#=H&e&5Dd`4oZD%Wb(MpKIfv^TtlT5 zjvP%)43dQsmk!BQG4RUgNElue(sbO@a^P6sZFaK<`{o83y(;C}zJWuyK|+9$_55j0 zbDjM%OozfIa)lmbn)>17`~s0rr=<=rV7%I}X3oo$4` z$ncHCvsDLHUO2G6d&0ISj^Qz!3ktZU3Is14V5+{O7Ji0{b;8N>4+Pf~GF&bYUUNa8 zLBMpjjQC?K&Lta88}RV5PY^M2;@uG-6M2BiU=ELt0Z-cmnKGH(53&;$BJ{HckYgn-y?X8muL5vJqK=iUyx%-6n5gS zfme;a5)e*y4dYa3|M&@fUkFe5t_uJ=m_8t~qa8@(n zT-OEevW9&cx@R9Vurm0HbOh|-e|uj1A>WtNm%L=p%{g#*xq8jEl_8-&Y>r=5+aO1ZP zx4z-2TmkkuIlLPjueMweR%GO5O*kUIh~2%GW8z1ML=lMLB z&K1k$8Ph zF0e5t+*`Hbte&<|Z|&ulhD&m1?`~Pe)#M=J%e04W&Q8~lkJQd_ZrO0Rt>M<^Gn{{O z@2>nI<6_4>L+1S7Ib05Ym#!r7sxqGU&f|_*aQA|C;q6nF&v(YoQm>D*x^PXU*G`a) z|J}jN+J{_k4zLC?wx}!=T@oRE_=aJOXiy@DW1_VG?3-G4LWeJ26#l@yx8T0-MuEn& zdruY|FMYf3vH{}^Z=thy^z(Tb4jw!n{)gf3S+2#U9PWP^at)MYHgNp)y|Sj@{z2)h z20Ry>rTb+org^BWe%!7Ywj}+xcQ~@JRjKW2;TC3iGG+Juvv&j;63$(;;V=#my)1D1%)B=Oc2|Alo@u}G<=0J%J3->8peD|hHX;OlW=%!^Y2fim9 zVQka>&eQOE)*jj3bDVYY9BmDIiv553?7lf^!o!t~i*KxE*WJjm-9Sj=(BasJKW`>T z>|gMmPxa@=+-Sqg!B*4D%BPg(Sp=_qvENC6?OegPH(e4u8%`dM|I=Ks|NHHFO_wgc zS23~cw{KsnrnM-W+&=4_V_QdewLI{Z7LY%WJcV8-b< zE0K)qt-E?>HEwy=tYPcmKp)dmnnSa4 zQ}Ya`S+l0kW=LKzd(n)!PI-BA81j;r=XK6mw`6&8=YqM@CT^S8wQ<$j#Vb44u9&@f zd-vM)(@!2)KK;v!)tw{KmS@Z*D%mn>eo{oLM}J8r*S z{&xGEnKuvIJ@W0(=3^h4L;GX2qn2_BJ-*`3+@RbpqF5xN#xAy`1S*(OYY=ZY^l?9jb%UOo4E?C&RHTz9OU80z; z??(ad{|s!wR}V$4*pzYY;JOo>VY4d~{35!%`Olw$L@@zrqs-2hb#VoliB&5FP%(WDpc$UO_wVLe zGoJ2^;GRBj=k|mw4YL=q>Sr7#M^=89-Fmb9*wv6LbKjrbEFa{!v-44=S)AsQ{QW20 zl=?Mv-7Z!7{cO9`tmPXT6aDyEFr_h zs55q_`MMde*0*GRO|6nRnjtfPuE_GfRcHLRyIC z^l3Y;OfQhUa;fw9-|#-Yg{dZUVrQ-@ne>0=f~2jl_s+R>dhX_xio0fav97+oWcR~g zhn5D#J(_lAB8QS?RQRWF)+yzivv$8()%(vgZTp?C%4g5nJaq2A_*M0lY{92A8}%fC z*`Z(7y*p7p!EgQt^9|XBT4~1fJ}B+Iam}r`TCpbQ`Yr8){_CpaS8_d%Oj)(v?T+d( z)1PxT_wH2V?aTUZ_gLr7SFx1n854uA70#|&uaU7T;kNd|uCH;|)*fGdWO1f3x30$h z>z8K8_^M5i?^tmAPSwBk(@Mp@UdP;oVjnj>X4*Q{=l>gygrvRqPi#t?=zQ3y_Uppl zyQfaHFSq`<@K@)=o>HNg?<_jEPn~;ZV(hu?+wXl^DY{harws4^w%W<+E8i?H{!u#H z?%MvqOV7%5*R*ZE;BjWwCoR#rZt)6%d|?$AH920k*JKFJ>1Sh6WO>z`p)YmmLBUn6 z&k~-^cYnm(TeVGnrmR=Po|EU>I7|;DywUU(F6n8zuy~iuL~k+ERr9N~j$7Y-kzan{ z^Yk5Bj}M-bJ*GQ6)*UB?Xg11bo#bPS{M$dffTY!R7FL`AzHrVahk76ke>5krFI*R9?ieGEHG}!@E*x zD-qf0BC~kd{QGD8?48hIKI5KekLk`C!D5CX;x&ohj<+i>M}~ZzAFUK$y<;c1#M4PAv-M`;O|cGLEk&)8$;N!M8-55!eO&Fb z@$u4wKg7i@PW3L*Ty#_SrHNT{u3O~Q6oI!rp;Fqu7Yh>@6Wfk>ZYp}PB-m+6@QE8I zT!jjcyM-mm=^1`K_?6LEwae(tfvU_CYc*aixcx|U=7AH74JTdlWjqt?baacA#$Nxr zgE3K>p{AR)P9>EYC_ZAby+5OeS{gs+V_U8o?-aMW(r>d>_chAjf0Pl~(TEk``-=8j7fUtKPHbmBqMU&~z$pWJ)Cy%Aa! zQxauq_o6^YCZ+P7rq4_k)5H@s$5%}Fxl`Ozt=nK$-s`NV#_oS7%sgv7`$SKPQ~9db zlO(#t^XI-0n>lZV=Or1X%WjH}g*S{Wc5$t8Q7zHti8?#kd@q0O_2U5nva2$$9(i;| zM{-V(1#j4$CTYT+l%F_J>T1ypO&-kC& zS#D{0I{Mq9>I>Colj|+)49XH!LM|T>HRsh{*ZO_hl6`WI!@S#)n$~A3wPY0UdU|$l zj&sG48BS$ucdD**o;%Ce;o9rN9Ij{G40avbSizgJuQ>WSn@8w`&L*!iukgz*+;UlW zBf|L3EpEQO%&<8rcLzh@Z9CzUNq?vQNa=Xi$vRUKeTG`OM_sORIwZ3uj4mExk}zwy-bj{ggY=I&-G|QLEnhC{4p9 z=fwokwdv;rZe++_FRyx)o4cn{VsVs8*UnXk)WbrSq=m-6;bF@t40^XmJM^_wdFGXZ zsV4a;stY6~VJ z^yoFiHmPq*UY-!W;`#ZW=Gj%*dJnhE-4OLR=h+tb{uBRC-H}Rqdu1OFTUYh|w%m=` zxAdwMY!7G`r~UU@B6FM}zvKJk#bteWT_!0%-!_$VrsjE#C)@Uim+e(Z=QV9TQaLlA z{B~V!__eU@cOD7`IK0iecGkUS#=%1IgtOB+w_Z8@vu{~C_ot+)Ln=M~Rlzn!I#;Cg zPbTx%t-8qGeD8EYwOVzv+t;*Q!=u~Yls+`A$h-eR%FOZHN_!)*iseUgpPE~#PY*nK zxmsYGZJuGRhkAN>u+p__emjfoMb#5apLiP`bxS#Ba68@k)YJgYCjk{HQ7(rXPp~Cj zW-p#)UOc}%I;*^9>SWE8?zR6OReMcId2-xX_Da2rO3hif3KsXYNR$8eZ$F9@rpFgC zmufAIwm6|bdAf<-!rVJoBTGwVO1C&kxdjT8c`TmjLq z63_U#xFPdJaBotR51V2nn;Bnn)!x9WSJ!Kf3T0_OcVm1MU9hG2M_@)72VW^i0Bd5v zPPWwT;S%*za$SmApB@ zGiJ+2$4st$i4wf-zWYb`7Wa@VC+x+Zo3tMnjuT4Wx+rzGaMR)R>Y~pgQw@}+iVBA& z^|^X9?so1}Gz*%&s3v*2lg#p@cNP=eF6P78E=*Q@7cp^e@WhGb&8hCG(~O;8 z?MR$r8g0KL<;xsOq2?moVbVN>UDQWlm_&8~X_9bxrKY z_MYh_Ndhze`^|CZbezfIx{55cMHr)KEPCC z#V0b0i^qV`*^s$Ri}Pd_??NrkG6g~51Rhl)=Cx?;gf%)J82Y6Q$^%yay~w68drjnoWegkE z7z(cGy1>S4uw22L^XI8;RS&i%oZi+sL7Z)Nm~`7j-laLKgDRv|CTx8ko;juAb9vLT z=UKn1DwCRP+pfufa*J3gpmers$)pMYncNi^y`|Q#lA7b=HDkhomRB?C+oy7bIjmw} zV9hlUtuoyDZx_dv1-nwMIiE@jqoHnvwtr+Hj~5Sw11a*?pE$KT@M%*YR-|FurP*U!;Km1CAqg;a+}M@yn&%%xr~8G zSr%Vi;JnENyYqMN+PrI*PQr5T+a+gqL@={9Jbn?i%VYiBhNSmtt~a7)8H+bd_ZU{U z>HVlwn7L!+mjs?w-uvF(V&`F4KSf}{+iu=~R?X>Y|3#K}FWM!&zUCLh)eoY=2?t6} zZ*#h^W^v%$Z`!+c3YKYf2{>=uGU)(=@SR=3HV3;K7U_Ht%@a6!=7#c}Yi$cQoA!z> zc2w4zxU}%!^Dfq}hcBu5oU$-YVN2cD)Z;L1%2m&uwHIa_-nr9Ndxo3#8r@*GCbd2iD?IXDuY-M>>Yn1o;t`hRT>pL^zbd)lOwI|v zuEVX5n+SqFWre_e%cf$+n(z^CaWU zIY&30j-GIF!4}WxSJNaHdt1DUjQr=)k$!O2gr|2P z{*mOYp0H+P`_%q!>!@bgP2v$x!q)x{XqYGF{9YyOtGhkV*^u{2wJy(dnYKwfC>)4; zID6@zbzmjS2GcJ~=62*A1CjWxqBojZtL3a3JsZn)2B@!m5;`wOI1Gmu*?V zE^EuV?6wty!FB%5V%t^44~(X&D=JSB4fHd~Gv(~R|FuJ|u~@`x>VJuVhBBjQNsZkP z{vV&ra3c8cx^zIIWq6*%s?yE#FJ+QZAxEUTDN)Q*?H^}%@5CA^5EQ| zmC4s9^pk$DJVmygC27tImU~%tC$4|ZtT?#5al`Zat6B-^Lh>^&-7L_1 zW*&B#X`}Jxn9Y5LA|C%ZuF6i||2RBnn^@Q4HrA6dDsze=TgyU1{I{|O{5aGiey_-( zC!i-IQmyCU<*V_Vzt^XDwCU~f>XgxaXqB_r&DN?^>$X^(_u}(ckE}kXes5NK(@GU*WFv8UYBWQ67)6fhP_MS`J~p@0$mo7k6EWw zZ*D2bj!1reD}_brany6uC3%l7n#pWiVX^#h&$kYPAJg16e_2}}8t2=-sa3Y`Wmra1 z$;Z5{D=yv~i+W?sjRf9n|4Dl{sj6p(>hqT08mFIDlsRz+ft`&tpXUyXJYS&uy#bws+FUogWD|%h`E`E|8E~(gCR+HYJ zvinIk+fGBBI_D_P&3}1g9%SF{+$a4iU0o~ezj%+?yC#=6D=fsfB^39qy}wVjnyKx> z$4dSZIlZ&@lG+USzQ1y;ytMn@+1sinPLez25)&szF4sN7+gP&GzxHL3eV3ge%VgDW zy)}z_9Qz7(B6ztENf3JAv>yu#t7Ph{9c6PBVLi^owXWF~OJ-QNVyp!$gy?2i) z(w^_-iRR5u=hB$QtM#&~{QU978b7v1ajmO#eEV!?Z;Y|UlV|qAU*1f+$j9N-#E_TQ z)ZWtC+}57l<zX?k zES)!X!?N8w`!~&9mA7N-&h`UGPfVJydh^cp-93v>t?67hKl%U0T=x~)kBKQI zJ$)^)Fnik$*&{pFEnbl?vA8p5*{jGcC0qQGZim0VyQ^iw2iYYSxm&^1Mao#Nqx=gyO44(MRvN1L4R{wCx!||NRp*t+b%AWIuOFg;7LLLV$y&B;7 z#zTM2>74A{Cwp&nzR}$B*h_7WtLX~QHT(9p24$U^>zx<-G)Ol3TXx8HhtkS9;g?nK z&owR$yIP(nov^94R%)wM=rsFz(XR81=T>~Vv!?g?zEfX(=PdTPwt(?ll=Yj(%gX-y z@7lLTxbSIcb#m(5wbD8{z3&=Un&fo+4~}t3G!HFQ6EwbQ_bEa} zI_;kCmh-`4T_!xnOP7koyT29Sn2=D9Fl6X=+<_Qo7b^Q$KomfmCF zm}?>`=a!V7dR9!l{Nz?)zfh-l*OLn$+8?l^^|JOE!2(Mi7#A*cO?#qF|BvOWjjvCOw}Jpg1+e z{BVoYy}A`jQ8!Dzooo;GOm=%RE9>-*P@TlS!t3WdS41tIro7u_+M1QS5}ACas9IfK z5>#;UXtd?&#?WZzM_y72twP7FrYLRJD_&N&KT`Sb+EbxHbJ|quUM?x*?LEC9pk2)C zQpD}Jld8FEg{)U5|Jq}+I*m#3MCQKtF3b5cE zSS#?iVEa@1r!y;jlG1f6HMX>TI}~yDAL9$3n7@y9g?nhmo%UF=K5vsw@|x1DlHE4t zj~A+3WbzQWGUeKi^($vCHJAE-_Sdy*k(j%7apqpbbrOpz}XKym5XLS zo*%O-<6G3Bn<-_>9=_=I3(q@lkv_*vpQCKU?FY4j&dopF6z-GD#%FzQeZe=}I{Zz0*H@@iV zxt1pm-KERrQkgk++CAfKs$EgBQkwVnwA6#myxIRt9bU=BhHT~7nscsVC1*l`RL7Dd z4ZPudq&(L3sV{oh>8F&O?0Sfeb^ES67kn2cDSpt|WKpAg%F%S9)Mqo-r9FYU?pLi7 zs{aTr`?%w@tyZNddoi1iuL(=~Gmq0(mptDz^I>Oloa41}2bRgzADY+w1zMXQ=Wo^WVpnX%}K>ZIEpX>lz#_)0UFg#siy(%NT+Ka~rezU(7=opy4o zoWubQTd#tqtlg7ua!ok?M)hA>=3$9Bb$hWw<(Bmmb5-&puLN9bm|-Vg zxT*R2q9DUwSMkM10)zu*JXss?QDLIva;K`EE3F)ChLbbYruqMm>isPx&XwT!kmLS_ zIPPesR!Ig2{!ST*n_`AD7{dG}9#|8jle|qfr|)Bx{^qCWPwhOwaZ0&^@ygK~Tb@sA z-gU{1QI>zh43D0T2PRvI>&`6k*t%NtS-$Jf%lFHyEUogZnCpZJ)C*S}W@HFCAj|jQ z!s9eAk@6T3-3%7Q)GhVnPsUyK~UUsvZ(XYf;Gb$%(nyt3B6lga- z{cT6$ytT7;W?dJ%a@9cf!e;ApIWrhPlnea2dL%Qq>zYvPqtl|#J~Ui?b-67pP4&X# z1%APq)|nGFIARu6tm>X~zH{MIHjSfG z-l?uXH+`n7u<_Na4h?H7Cza1giFaSE-qw8EAV+PpH{%4RKTa`6mz>?TR+{g*p|JCf zwueP`?w`3j?U3<}$vVoB3m-h1R&@CW=dNug(mvOoO8FNA`*&L3@;RTK z`ahC?&C!pWz8+l4^?H$8dGNYDbItqie7mvarOkW&J(oW6$QXzT{Qh=&`i4p!Re|;y zMn865IaoeRA%^K1Q|$!SvaLA`8!{FK6cwe%TuQ0uSsrRq7Q^Hgn{myrGCa}psqgWv z(c72m@)M!a_(H~# z6?O$7p=`|+YtqD4UgdFhshk;Fx_7bF*)8tfPBwMMS^poXcO6v`6%}J;ac?~sx=h9Q zlAuZX*K&y~_G{TR<)+$9P6@LulCC%JTCu{}W|_P0lIF%#?fma8Cz`9-($(vm#H>^D zS1t+p?Hn55YPdjDz(?KGj#ZlTs@0;jIL#1&b;kVGSGDE7O0zESnIRaI{8?vDhC2_t zu1|9JhwV}UA>AsbRky?3<{y;o-PUyBh^Ez7;q8vik2Z)1Yz~Z=)LU`Hd1iZwvsk;y z6yr@z`Hd6#R<+qZ`4$^o6f$4DYu%Ur+1Hg=U=CV%9Y`>k`f> z+ry_n&1g>67y6R5y~OTciol$uQu75g^glICD9H*u(mN}xvMZyu#LE+ zgi2#RMyqU=fXfqA`kPYLpPY4hiz-(#@4IPE%sVIC5YYbkER0vOQ{m&>V?sQ(CsjNi zYO_qqm$>8>s%hr(V+!YFKi_ttyDh%X&UukXRPR2mEc@vgw=|zSS#zP7(r?aLlfEdQ ze5^4;TyUkyg8m%~{HF4JSX}v$$$@pIlAHSsPqk@#nX`6CdikGR;AW~S$u#HVHl8O> zYYjDJPZx?txEOdJc3xrJGq2NH{-Ld9W6!a)fPVFq$|FrvmV~@jojIYg&rC}AMx?we z%lw~ik{6hCtCh@yw#q+@&_4BnGo6$F(ZkG)6usG@k!&;N>LMq4Zk|0`LoZS}i$&}I zEI&2nlPj0`EH*FuslsGva`mLn2Xz;Bra1*W7rpTGc41q7Up1)iqSnF^b7LoawJoOJ zBGYbiEHf+O_L-`x;5c1H)u7mOfveUM7mgX9zS^;bs`y`)e0VAUc(YQsVcnbQGv;Wy zO`QLj(fY0uF=$A*KFr@xjdL+S$fNz>(K0zi}a@K+MsCYQ!VM#r)8AH z+POv(XsI^}umt3~$iqxaxt*5+17bNGYqF)+{()7apI!h_vBr#J*O)R88bI7?U-7!;1h?6=7i9q^=DJ;!ls(PKj~q) zWr=}T@5h&-zE2hez7))#`G4uLg|c5#Ym1&(oK?5FmSNlGDH(LhMT>L(O)Z7xDGC9n zx8HDg*vT~ebcx*4E869k6*FA4`+pitn6Zs>*FwHXyGgA1Hq4$pODj)bUdJJ7_T!=A znZ(L>n?(0otl`m|Uf5-H%zH&k>lT$oYtGFG`1s4ih247I;k{1T+us*jXnL>HE)KO; zUGmIhAzO{t(o2=cvqPLBwXeTk`1$C8=#~{;*^Z{FKF@;rRn}*t zVsnXyrtqx8Emn(Cc5B;wab9FuXu95Djw92Kt4HP@^;{j6zGnOB6`vRlpBz?t*1B?e zQ2vfW{;Arg)r)=DYLuOtBwre>@%ud`_~QRfcB;;j$?LY=j_j4(v+mas&rm!0ua*y7 z4&F@hM$-_fU) zq}tb^`;TLFLgRtMGTXXp;*S0jTXfoE;g8j?uFdS#S)NoalC!w!*vWbPGxuz}c;L(@ zuf!uN7i3m_&hF!rv9;0O;B0L3P3`zI#&i20)&AL4u~gcGUuN%#7sV%c7+(Cdcv8ZN z&BogsUrH98iId@Sa=Lx)YuAx=UHa#pgb#45sDH5*^_`x(=>Jh2E3IkY17H2ntd(9B z>pZ{cr^AgC=W;S_HFnLH|H*&2)8%#ON%_eK7f-Xhm~-sGL9N<17V}HDlm?j`m?#!q zIAgPnjMe9DXQM7Vh?_H+C@y_@DCLm~Q_cdumBQot+Owz4*tt1s z(d);}Jv)SyA}=dgA2xIhY-H^Ef5Ll;bE;(Y>3GehcODt{DV#1Vp0()N{)NYFzZ^Et zl62CyJaF>O_3lF#E3!hui)X)eTr)*t#=J?}K5&HmzPaedw1dT4_AU6i=1$hluRLidV=PF5+u{J+ue zP^G}8H4!l)QJMZ>Ur*+qKAyGbwpb&--DwL0nFEtDRxCf9nQ+)J>-GW*$IEkSch_y{ zb+A3Rx-za|x+9CEd!o=fzuTDxTJE==92MC)Z_|c@8%{1wx)kgdGmlSDqPC{ugOMM* z$y}?&3u?Yho8>!0dUdUxgo^l{AoedNd0X5GdZQ?Gb{3>9_^31wtYeIjEK!2 zmiss?-=lpt0H1^XlATMd#TH+e_Z15axuxUg>+SUNoM!C>hI{${JEupnsLTDm zTjYD`RmuPS_pt}RWY4ipnpvE^Mxx}@IiBmAoFmTsUfX@^!OOB6EH1V;dT-BUTl%r} zpuv@zmA~_T%-G`f=w9>HIA_=1jCb3lJh%M1Cf^-%#LDP_DC^oSA}g;yF{}5nI(x=D z=iL>v%AHI>&7s*7cfM$9uyQ$4QrxsuibeL?i4v}Oy}k0adGkzeSglxfI^u5Xs&(W!?UB1rPbd^i7vyPR zTEntrnkyps}73MRh{ zwULVQdv$mbdrUxhtASVOVTPTi-HR76N(FER7%)b&n$|XONR|htDKqY7m52^2i@wHQ z^?;34f${1a)s+U!LJ9i1tsFcPnCdUH^EA|yb=Tw>Fqj?K_g8{FCV{=lF)m+uM%4uI z#rIx28GPH$YFny&KknxZp6u<9ch2CCS*DJM!KD%eHduVoR){E;YeiN@G z=}yi~aC=w6z#hOj=>P-!0#2syi5$=WzjYE|_cdVUH~7vlftC5fcD@O$A;t{B57Fay)maf~dv}5fR-a`KlKD-W~=@7&nGFRIw^3g*Xu7kG%p-G)ndJn z?@YmALA5SUo;!E0FdT53trK&|r6f7|lt<8dvCz#%%9oe2>*ZK1&ERy97ppRAeHk1i zw%A*hQK3bwC*o4!#&fzx+GlTU%1Ga@b>^5^;L`kz!mmxoIP@Z0<6rM%VmteK<)*B& zh0R;vNGx)`d28L#CGAI5Io~p5%s9~4EZbH1<7HyPMfn4oe0*;+4}WdkU$B&m*PzOH zvz&E|B~L{H^TSm>tvsP>wH&9IIV00jdMyqJUJqF^JQpX%2Juqh^{9eRFjM} zjvZBu>^W!=(Z2n{CcZ-FhvBCcs}jOP?ftek9#Z8=P@SgbarBv6F;Brk4u|Rj$CJ$L z9t=mNS5A!ZP&|C_`2lHTj|)ebXZlP$DjVGLC?((vY$CJUWA6jhP#(aKNKu zBHt#IEV9yh62dFOvvZ@6Xe82fdE*tnaGd>&{eCn$dE z^KsoGr%`z3Nt>+iiBAmvca|RL6OfHn;#@iH?_t*FWZT874V8aQ4J z?VO=;GDZ50(@Y;lJ{!fUfmh!gWD5LTaKU50*TgAhB{}O%4o>;I^QT)`vXjju#V2cv z*#CyGkmn+rU$ zjb=ry>3L?lKCnt8@ z(PsZ`b>x^q(pi@azaMSg^JA5(jA66R48}cSrp;zQnpvx7NGWjdl$9w`Y)vwJ;9+v} zP~`{R_{gA>Jg*X4&2J?CXEWyHo2hnzui;Fa;Djj)>^B><{@!|_%p;NU=^|s+j#^cQ zlOOwJ#WqUn+{u{wG&|NtXyrQd32p6sH`o<#8MJa0EC`+?ae!wBgD_`=r|iUoo+v=cxH;$J1VFx3Rt=47BT6f4bCOBnL`LU^q z(>+eA9%5*fJdh$@e&M-bN0{Jmo&<)OnJtXoPr7XHG&-BDaF7o;AUgNQR)z%^9xI6& zMhb45$ z5`~o9cB?ptB(U)~9sDOz5TWxXlxf!g4UJy+m}Oq595YP2DAmidSUK&FUVV)mDw7`;ra@yIh;vCVRhPukoQ3tB)^QWTiE8CeP3E${QJ1!z~Kbs=quc zJN5@-TZza<|vBy+T+?rJqUiE6q`_i*!3p3^x<|SUPs-)d$zm6w__EWhe^_)=cXqBq=;&4LH!nO)42 z>`IEh{aN|x?5Vx7pY0zW`sH##tJJVSM|aP$Dxa4*_oA0>N$B;{mE_L#PJT9PwQ~5b zi~7`Rn-x7KVwtSnN|5akIn9CT9*E+ zf^iL(e`#cJTv2yX%@*JOK-Z9c>w34wn*I4zeaAd)$D!$Ap_>;9-WIbj>b0>e+7;$| zJh$ZP1hXfvHg~NJ6;^w`=FMx9+Sx}QnzNOtF>L;<+2yg|oA0OU^Z&!cs~+9nEpdHE z+o$Z#+o|v8?vpc?=)K329C`TC^0b`IN2QNTbZ*bc-fQ-a&*Q${G#Nwdc{{I9RejGT z|I#_C{O8S9*AD>_ckYL;pDO!t^;K2&U*|LC^L$rpIq>J-oNVLz^{3Zre72Cgll72Q zy5Ll-;NA^^GXCbrFI>3$J^xwFrF*RF%Q@@Z*BkO&dX}Nxp`^pZ70>)SqQvCL-SSUY zK5JwY?R=HCeWl+S~a^pPr8E7_C!(MLldzV`RU7=b%a{EuD2X8@;j5>ep$Id?Yo`0dMo!0 zQxT_pr|o_%SKTV7=REzAQtsY3V|x0F%Z(YTX;!vH(u<=d8ckv**Sf!&w@I`o1&5e^-=1?rtYrR)nr!jD*Q`oQmuvqs zDqJk4{Bm;dt-~#mrPb>04K*C)QI|u{-01cBW^z8cB4tNslt)`~nQp9yZ9Zd_Q8}+; zM*f!Rjk$)I-t28_B~mq4wB0uCVshtrzr0uITl)0veHX=BBd6zjt?1^T*hiT`mg1>&Q_m5-H&|F`JdcA~Phy(qh4%6auv`RDyBPgGwHYB11fk7*Ns zBvDc%-mq0Ye8scA-5GgrUbL&J*C#Ah3UZ$Kba|P|w(jtoO(j3eLp|EIJ~D~>*3)`? zhGh8kJ1KD+H@0kV>J9r^oHo_M`C9p1xB9=^>K1%!{d1$^l3M?l?bDa-u>Gm7)|S!# zM5J=&&xVrc4Nu*AwrjM`SkcsH-c)kE@3LFtoSBnfZSUQgS*X|~aejyRyQ$ehKa!(g z7A_Oht({RS`m;JsbG9hQq>L#AoXbR38&gwWn9lkEkoI{6D)qW73CC-`*dl49gUQBT92L+8WyW{bx+S=3J5I*0+9H zjq~%CH20RRoQYRE_%@b@8AVQzSXo!s9;-Ed24mDBJMp$nX%jLE=b0OsIGoHqvn+Sr ziD^XF0ahcQ1{3jaSZkJaqTM>Uu;{Sv{oD|l#^}_Xcj3z?zZBiIy#MLU;y3EEO}{VMTvaFVW0Kk~ z+t1tWHhbhB5nC*^EVYGW)m68ZycMy0x7K-cE-hv(@|a#I&9=1Ns@KY*=BSl#;myTa z5vxkJ_k?Sj*)XH%7MZ7y#|KX^){&Rk3 z<#N^xzslQv34tkd%{IlXo|E}~{oBPGcwUx2NpG34t!j09vt;+IW7FGKuh=-By}#qo z>a_ML0W)nmq~_)PUUh|Y!ZfjMQ`2^Qw9z1=G_V$g%W{p<_ zR)%I(U)UkBEzK?F^okYfv*TYbmAJItg*$HR=^1&`SNCq;IeBKk%l4f~*8jC{-ONkT z+9^`C*YU@ydFkP)*H>x3So`_stY^!6^CM?|5I*qLEvrppedgEdV=s1H+O_mz_^R9) z{i_+uX0|VnHCY^bd)vD$dz37;A3w=^&9ukmXW#$R%P-HE-5(yWtKBQGsG{`yN`u|2 z>R%bBb?!`^RkCdN%D>s+hoe)pZx`6IE!_1;@AdDA?7K4Zq)Ik*)cNi)(T-W4$a3&@ zr`wyU)3!=ZlGm!wZ1)cPuETzOt%!Py*$i!^Jxx;O%Pv32PDz|R@#Rz=(Yk2&73CHE zKehMVuj;Xyw$kR2o8Yz$v0QEsw$*Ooa!=g6W|!5(MAo3%=b;=Sp@ z%_R&s|97xu?)QwbyO%w`eFgW1iW&}X@seM8N7%BSaGbatzUlt&{tcb^5;sn;NtK=w zUmdyi#7_5ax0$m~R!xiexr1Y7dV`Odct=fZOl3y*f(wspwZdb6uxqompVenuF1=$K zM@UgkMniIGcf{1(_hKf$Qf=+U3z{bCl!zbobmN^Ov8wm>j1GylO+A8pUN-!<+Wddc zd6Qk@I?s};mAe?f?^*2KI-hgeqZ27_EVCC^E>&e|UDv(#l~>vtwf$>O);avJ|FcSQ zzu+aSY1yx>C%)Hmd#YvsIn~!$1>5A`EMH{m>EP9 zm{}KWYhmKtkd~K!{r~1)$Jg3wtWXgwxc0K~!n4-kx0C))3OmlQ=V*n{nwBMb!4VUd z2d1lWaml(YePz$nz_cJhY3>_|vOE25HJp=G8GnB+`lXQ?`$F9O_o_*^+Z$x({PZqq z{>9a>iCty^du@qa%*PGg1#(M@7*?{j%$&o-!yw=nS-W9@Q2E!c-{Lxv=GTAC&YruG zgMra#@r1PdR%~Vl8oH6Z&ku@+UE6-^!!0qkoxW*S+e%e^PBKhsxR!re;)oH0y4dtT zf$H0rGHM1~ycWDuhEsM&uW6M4yPe3zD>iF`d@^LD8r;0MJ67#<$=G4NXz7j}ISq@P zd+W0J+tRV% zQR#J6)t+YcK*>_&p9%^4E`sf+BQ2MqHjXfjOZS21Mve_*xJ zweM@DAo~PS(GP5ki|i8@dYuyS)-;IFG_#v~i#^wXk$#B&V7 z6GY_;W!Z}KruJQHEnsIXU|4s6A%B9tR@wgxldc)uoVb^1Cr8y*DZNAm1`&3bvmtE3 z#>N+n+9t5hf1qm{XpqR;qWV~K$wtFvjC)@F5_PzlG_Or|b(+)7Mw645PBOl1t@fDi z$MtrekNauO6ZV^roSdMWXT}g_AfTS+@h8DT$bhw{h$D+><>~;{rI&?$85t5Alm<>GdSunuzokv;#i~*X7FIwM~1QnrqwM;WfQpWZxs5uL1?j%Jkt{f7NG#q2Mi() z84je1>M(FUjSI0oc&|EvS<@hpsYtIZL55deM#$kNt33nz1(rxffwl&wmr2Y*4p9bj zQW_0!z9z9oCPbGrq!}~hUP|Oh{2=J|h)sI2{=J9)`7S@y2-xBh7s$~R$5^1vEnX<9 z$s%C+-(z86l3^YD!h@Wj`h-5MGyE~}bNK?sU+a9DCs-cOQ9iP%`q!yM$Fw7UyXtJZ z8WsuUKHHJq-*JQ6SR#1+I@hFVwJUs_`iv`|W@{X95nRTX`yl+Ei-?W@JI@C8ZS!O; z{j`=fX|)|zjeEp&X$v>wlrI*^7Lf-e#SDcb3mD27SSPMy&o|(hpl8t2Af@$$VM%Bb z<3$!31g41l>o+V~v25-3)ywmCt=h4F>rSVmTUV_(zHalbl^a)|Uzf|XZXI9n8HVa$;X;Pw zJmavu!orp15AwEb=hIoqQ(ivnK0_XlPBH`g%e*rF6*)eI#p?5=`Gidhyc%B}TpV@v z7sHIC z9UXHGzLvy=2cJzbi1|{p>G`tdvnQ{3vnBDAlarQ9UXDxwv+C_#1+r^jasR)rlCr~$ zPeGQUBE9L7b^e;DnGYVEV=R(kII?k3Uvu03Wos7b7&tC6t!7_vgZpt9|8#?u5e}YL zg&IVQzTdllhR{$?hVyAIpUAK&?8_9lpXqSOb+*=l*qwp4ydI5m4wZdQ@( zi~QGf)gFHFIiU2Ap~22q{$;UgLf1SI;|=MpvaClE5A#%+96ab)n3BjUF1pY_RPl90 zusrjJ)bc+>t4PDHR$!7di%A*(uV;#=fRC-lEC#`83-U zw?!wL7A!fylwusp;*k1iVtSnX;~LSqEy7b+oCGETVORa<6qQRHsMgitZ#8w-vz^F=70Vl-}<$keLYaLGmD z#EZfP%WnqOt#(msw_H|Ni7Qj`cpo$4yqrPZp^4JdZA}?UYaWJ+G3Z=jJs>;JK{0sa z(wUP~BC6h`9Q~j1P^7?9x@v|=q^y5`%0R@vBdkGK$=sr@4OCDW(f; zvkQ)CHL}QG-Id(WLR z2~(wsOH_h#&M90j{ygEiXV+<&%^q5d3MX<^hb_D0X3xYvrT6T^kTVT(`60%aQhw=p z8rB8hi9S@GI3*%fDR1E>Bdw+ns;4IS`#LOQ7Pd*6@=&I?kfD=r*0Jmr%|~aaRG9d& ziHOhPt2>~)QseeEeG%T|^ZOn?iJcInG}VKTL4MJuEUtBHBzjCYMwjJR_?d0JDzeb? zxGPtcz*H|`9d^kDMyD1fZ2P|xj5C^(A_5xECA=xyqcmUdL*QbIFQ<7mGY*MtD6|fW zbY+_OnLS-4VS>mGg|taCPp7yw+T4i9cZ}WC&Kz?ZmxkU;|CYUm<3Z93lbR$D-K(Ft?1WdJjoiilHvP;3oToG z@(UlXI2V5HVC(zNMfTajlh`(0XkpjsV6L3WBqAr#H%H^R$amo$+rD7sAV)U&uo=oq z788xc6dt+V^tiyXyueyJjj?-I@j<^Y6aO!H(y_$Y-&-Vm&a++frmhcI!7*RA?4!ny zxonety+yMQnQL5Ue4Ba9v9H8p(JAXQ3|wA2;}3H!*sXR$+4S|RX4$+*hgy!!k)oFl zGaN}`*63+u46|h7N_wsn;*r2st#C~v;K16rF9+FpZXR~Ov!FeoCuxT0gitM^Dc%J= zPRvP#uBPUKLPdJQ8WR*f7!Pce_&P;cv}!|RVt@fNr$vtfYY8j6j6pNk9mUkD1}5b< z4+DJK7Bk&2Y-LecEF-eOK}nb6r69+boDb4ORJJ4kKSlzKD@=5`=sW@TBk>g zS2{GRZ(F!G`BsBSvcq#{##=16dXMyY2haaC#Z6|(e@$L9ho?7k54r94)ml8I*Xr#5 zS+`8CuW_+!-^ga5m8#h!b3oyi;^ZjpB{E;%w&$eG*3v%ljN!zT1g5FMet)0+63*6m zZ17N`;nA(<=Yn5cU0=hiZ6BTJdGuJD=ARIGgMU?v?jL;^=lpx_?=0c#zbAHz1m8GW zaA+bchbZ@kF4nl1VDD?UJ_@@Zo5`s1zQZJ7UGwIdS5p3nWb9k9_I>*PKq-USUUoN1 z#2EBS*IxN#)-%4j%fylR<-_6X)JfEKBRp-l{nN(3Zo3MZZt3`Q@`~RmJOj*Cm7CJcJJT~Qs z?m3O?<_VAHyx%Ku?Csxz-603=P1`$X6@&8(>!hnoBFt|Ja89O(PQS){-`MtN4H=3^UG7JCs%(~Y0Xn#zZ*xE=NvWqwEDn#wXW4B`&5=T zWMxk4)89Wu&m_5Bc#HMq_cbdyrSetJZjnAMx=Q8VmnBX=jdO4OxG;-Pdy$uIa^UZu z>dXFnnw;OXMk;7k+umrtZ}t6Kf&xA@O;wqBAqB@2|?1)(m^jf8=HK+m!#gX>kn}_cm#si?Fo*oP1nr@}$2z)T;Hk z<=o%a?q%J&ZTd6grRIB1JY4%}M{10n^zJ*8HK!ZT(XV{+U{d9)W~OcFwcT%p&86!@ z`@h_NcrIC{{O^sq$2R?MOil{c@{9zDLc=5V2Xe(`tPwdIJcc0b2ea2HTy<}&4iRm@2nXa|0?Iod4r7gm}jxG&3mYVjw ztol`ZZcl2=WVft`%ZpAv&tacY=A*1pt@giUp}MBQvQ(L48FJq=8k5Sir$xRuPus91 zYs<1cmFXH)$D~x$D?LA#RJKDpNIZDFGhn@M28pVv<36rANg|}SY z9)0L*o$ItBn~J7=3SReFo9&F-Qoh;m`5yhY***GbZP4`=t{+lOVolT1>)$EIE>*9( zxU6kO7CFBva|r3Jt+iJ+m?@?te{6{#G_+tJG2T{1t&MuHQ?Shj+Jx zXD{1soEy=(#;hgpTV;Sn>GTx49T|n|F4wiE6>NL{zq6$&`q8(#<;$uAUQ`K$XU025 zzfnUA$z z{!xGMc(2aPO2%#dQ9BxD9nbn1p11jVblagspU|RJU*ZKh>tC@&Bz*L~P;U48qSwc3 zjoWY7aoHLGMeC-h~c_`LMeX)k1! zES{h?%~~V=V0jr=#k7eQoetM}6qmIZek;CK)>M14^)P$M-|heV_qk3~IFY+-Mp$rU zPFH61iDfyP)N7I}D!%O~{T!CL?}gXPhzWgR1uZit+3uK8_q}HNvgy;OP3Pjw>A04# zL1n6Y(6oyYIgg7|CYa7%P%c&dv|{77##=i&KDAEbd!aGevyRQ8@5%Rqyz&$xy{>BFQ_ft!C5+eWyjaUY5zH6oQj)2Ol|1lOiq0s5;Xn)q>pO3 z-0oF;Vl!VauUIi9Ido^c+KrBA)wn>`_M1~nB)uwsSh%&XoGbCOx<-BRh9AirJ16Q^ zPMT9WLA+zC?<@7}@QPP2=3U;}xaH@P$86a@%f0-I>vx7W$!;&V+nnL~Irmj$n7#9o z0v~5 zh-;O|jhUxUOuqYUWwED>ocUa@j@-1Ya*>FYwJ&Pgi>6$4O*!IKx9xiB;p2-cPE^Tq zWVO!tzi49m{B@m0-rFZn{4wXy)=8oqGs3oaxdzYKR+^gNRb6(XOp0TMyj7>SYvRcj zT_2m5-I_76JZj<6knWOO>%z(wXO`wqR?YQ}sGjs}YUc4-Uykwhu39!@X0wCy{0jGI z+4AOFzp8#|t~)K&ENWg6>Q?(}agAD3kz_}zXk_W~w28(q%8g$}MYOILn3-`YeL}3p zYV(_uo^m!^d|5p+WwILA7MGWs@@6hmJiYna^#)be^&*uk4~kZNDV?p$wc6FIoaK3= z>5BA#&bY>oHTuh!YFt|p5$VNbm3eCW%ropMi>=zH?btNs$J~m{We-*@juKt=%A#ap zc}3Lie6#9Vt&iHiwEtgJ$FbP1bN%A-MF&Hx+N8VUJ7+|vrThymI2KWEsV<`(HMjBg z#tmD#*SgHF)9QFLeZotr6=~)(EGpKC{g(a_KAr#Nmiq2yW2?EFC7WkYTYbQEnda^G zrt4dLG;_jU=KriH$(X%+Zs+vg*Skbl?7ID;bn2}d)oWSn7OsfAxlng!O~muE?OblL zQ@5YbUcD}2Wxhw)`)BK7JX@?X7tK7Ge>}58M7kmBc&S`<-=B{89J`Yiyv|#kHfc`Q zo`+iVpUm{3kuvH}(ge>U^+te0IrMsVa9SC$^a8j_uu&e#^ zR{5)CdwpCr^H0Rg{9gQ~YIX3{eWrg3_gHl*?M_`-U0d>_tjQv6)y)1~;$EqtOG~p3 zZshFPTDkpo`09HeZ6fCTBy!q|Gm#wc0KkwoI5A7iFGo^|-Wd zchm0c5tXMK?Xr%=zKE(aUCcWz<#*x=r8_3;oR58Tt>@2lh?sru-}5s+f6mKHJO4gA z`tgd+3l+1trbbEMjQa33;lTl!hpUV?dR(lpo}r<+pHb$3btebY$l-wCQX4Lb8fr0JeQY<&n-5)cx`J~nN*$R^_*BYwMXR*60(cDf*7n=H>#Xk z_3iYOligKGzgNbbS)iC9vvR6Jgv@m^toJo6yO5H~-g143ae=Q|z2k}x0*2YsxPEM8fAZl1qlQ}8hQt|a z(>FL8eww`AemWD+28C}q3F{he9eQy6OtXB|1nJtn<_)|;e?_gm9&rgWGiVeTe%i28 zJCR-I14HzghAH1qMV&eQSi8JzXUlBMCCj|H{v=BWPvD!nmp@Wqr%<8(wFLQ^yVh(v zT&x$^*dHjrwq>nNFgbX5a@JKT(SVTM2fLY8u+Lh+^+9(6^8qII3v93MvKd|F>=I9n zk>+#1z%tK(t(-yhngO%WlY0!UI=%rmbCeveGz#tV;<`|%F)@Mp2X79~0>)nfvavP) zXYb`mTq(ews~~rhaq9{P#bpMp3~~<`92NI(W`A{-i9yd%SWxQ2U#^JgyVs{vnH?#7f9%fgxMQ1Gah)jG56lO2{Hb{ zY(fW_sy;lE<46#G7QvqQDD-S2_o2rp8%sXArmgsp`a6|XMu1JGfJLW(^})CBJXSTm zP&r=#749VFyf`Jjk0M5eGF^rC7E^3wF0hGC$m^XT{^SwUO@nnyR4w!6!fkA(d|Ua_ z^{bDt!lQL-!y0m>U6R;a>sX}2UVA0^Tx(*=oA=7}QFv}ESBspSDx;o3to11dX9e3S zxe`U;^ZsYd-Z4@2#GBtUs}gk@6AI(cDqk0S#O1nCUT>>Hd4tWhg%REgaZ_gA%PnOw zp1>M$kfAk!Rj`=dz{JiK^IQ^p_Z<)-&Sl%5$il$9mKMn{ieaL2@qNBj5{d|K$Qb@$QTL!rs z6dpPVvL^(uOkkGQmwFW|q@VoeF$1fKzXDUR;BG^%*bv3DDg5OHk{SwJW(9@^13c#n zel2|SPWzyaS%C7hJZ2u&;6`V~XA`(S=)G5o=g?C$c>Qb6p7O1|ceXWoC-r{YbAfx+ zBKO1-g%Wza(@#G9|FUUo)a(gS8W~)@1@^}ea`ZU}=rQODC2I2>VC_8U_}oEat`Xxx zQ--+-HsuVgON7~s4=_D25#ldk*IOsTZ@|bpRa$Pdg1Z5SM#Imm4Rwe;?>lZ(?HYg=3@?Blv`-VrAjI0Y7EXxBJvM2H_W@6|%z+sbQ zq5b&t(-kTcoRzc{8DkRIuT2)qXe?P^AnpF{_R(+w6My}f2Mh=PaRfYI3S;n#+29|> ztatrBM?V8cxgI|sLjXU6Q&)FqXSY+|gf50D6Z`XKFwE$8n%UbwWx<45dEGOX&RsEW z&ZIf>yJjz9sAAxmy1jRMa`J!3cx)bn?^GV0sR0Z!3s*ap>&VO~+@E(m@4{h*T%lQ! z!Lx*X^F(uHs$#a?Hp>&5kjGw@r=B~@?CXqRV+MZX<>w;*cydqn3zja6(c;;>DS!HG z=Q(T&lh~G?HL%V%*&%s(8Siv zgP&NMPF>mS9rw4*Qi45|YnfecS<+IrHMa~IZW_pbV&T4ed23JHTq&{ctD47K=9N|sq!^* z-nb=s#5LO&GfY?I=8n1$zN~cduj1DecnSo?RE%W`4#?`?Wnv&!X`F$jq^S{v)4dIjXXB>EPX`-=+LL-Z{?M^Xs z@77=+-{%_^1QsitTj0p$aqHL_A3ff02N=TZ%sBNQ@TzF4u`g^sv{1;0OWtGNR`%@N z@{9$uq%{i`HmYevn6PqxeR$HRL9^hIfWE(1@o^S5J&sF-wFR69+MQ%I`%UJnUG0?) z%ei{PxcO5v2e-B7h~WI(ca3(!YQ<_a%c1M5a&s;;i34n_-mHg!06 zeKKl}J)H4HMJ2iAgl?fmPq1Inq^wIVyfS+hvafn$b5i@i_>wJxQ;eLmo=s>yy`#|O z{A-4SD+bRuq%&LW2?!C<*U9S0TgSol-~i*bH-+ve_zf1UaqTvJG-1KzUz<;RI>+a3 z^S{k&9MpLDiy>o^S$TC-Me+uP^G+Q;g^AWarAKcV^cWeRsnaw#thCAY-xuXrh0+<* zw;tg5{7F-)v-qjvRE;wmQhFL@Mx5H{FRA@Tu-WY7lc^W$B4<9VJhWW7CeLJ#$Cf#V zuFRV{gN$nfX@Q!7sccNqgiAj5T9E4GO@ucsYtwg$HJGQb2?e31x;LM+Q`|rPdVc4HxAAym42Tml9P@F z8JSy!CNW&N9C0x1;*s4J32bi)oFxJ_aEMG`mo5=XH2>1jq*y8>tdZcr{MMkEOXPsZ zjTKxPikZi@CNfKEJBO(iUh;Hp2@>XUc+$mV+{)p*i0REESJpQQUU3177}!z{B>uN6 zK2@g3nzO5!#pj4K=Xr(&OdgB}6j<&26622gjCSs{-+_y-G?}%%zg4o_EGX(=o zRWJMIXKiTk>=8*__0o5%;-m#8hkGt;k&rFDwOc_hVcAuaG}RMd`c~SVX=d2KsI+J% zcg~(fHj7ou{8K;kcREfjGrfCQB%pZrwNg!SyQ^p3Y_Vj`*tSK}Vd{ElZY#V1?UZA#Y%hV0WMB1Ls++gd( z&sWf39&a65(d)GJeC+A8;}e=@|9QEzzG`l{m|DvdjjPK-MMbL@6j^UjTqgPcri91~ z7sEMq+EuBn_hO6<-4CZ-PrBGMBg8i0O z8$F*bp5j|Glk+m0Vbzfrx?GEI=Nb~+H!Z%O$}Rmk{l?@Z*2`=giq~qi zEbR$?FA(PYAoO#hv9XrvE~T)DY18iPIQ2v?ecI{Bu%fpz3*GIjcP`PoRy^&Y$7c2S zA9qbQ{SlJ2e1m%W^?iGJC7 zY0k%rr#f2;kF06?@MhhwBp>%k?_D3Z%-_32b;7gjmv-+i?Vfn%&3)IIPG5VKH*oGf zw))Y-fFj z!m*}nSHAuG(&E^x;}v0MQG2(%t($Fi`;eUR)VM!On+3luN<5cQf3^2*UB2}Nmbx!V zjJk;$tCeQmDSe%@VNa^I{-%qvdtcY5=e@NJ`?J%F^=(3-`D?xJ66NQ`eja_RQ4%Wd zw!-JXUee;~xNP;ki>{YxA9{M4GfVo<-oR~@t5oaM%ubs=-~TlKb;I|YOMgc`75H~; zy1!IpvAp$?x~E%Kb-(+jHo5ZZWxo<_JMMyQ>u+8vQoh)-_Rr^O_by&f(t2ICm+$$% zTNjsZs`%igKW+EM`Cpo)&wpFKa&Ag9pVhm{|8rlQk1GiHTzRAV|NGZZE@pk1D0TNy z|I{40(84r3`9JsAtlz)$ue}sJXZzh?`QN)Q=7p`iXLq>If6eoohw>M@r(HU7{O``- z`fm@a78#yq*ZdxRugL$!-JYW3MZ0g-)$ZW4sghJRYb=%v^%U2+lN7%#@~i)kPX99x z>eUNoPBeO`ZaBx?Ak!>Db-8Nzb!*<{DyEK{40g$_?m?R7NejN_Upf>!=eU?qMy^JA zU0`X=tMI~@@8!2r>tBk+er8LbvM}N5G%l9qiH+aOyu>r+A1l{0Z?yc{G+90RX<2pj zv>JWZ*plVpE>Fv2(?co$*J4^=O#v^ zPP&@ZyS?fIYb6s$qgh7lt?6z0$CX^zvbe$%o3^*7=;VY#iWVBW&O&Z|WW z928k3ayrzkZ-!>{mR4@NoSW%h?sYv){zhY+m?O7(?Sb~pooN*tTXL(|YxhQ^eNby$ z;UUfNy>(T4v3f_HjB4{m@q*(Xg|iv!d?bWUx1~gN#5<|AOO$sl2yZiWZ~m}7^J!Ym zqDz&>EOIN`O2gHgW~rs^QY&qX$o$#bqW@g=SV+RlXN3l3-3{s8e&+RC!kTw%&zvgw zzeVna!O;~B9F@g(<-K7_v6kQ37Hs#}{xyC3^4i-GMbg(IzC>96sHpJEC|rA@W(7x2 zS7>eWa%-`reP_+GzFjj=<>)$l%>AOoM6Herxt&$}kJp@G?=Us1UCmawWqIly_U=6= z#6*tvtZ7S|sxje1dYS0az=fYuTr2y!)VPm@c5$^A3zg?EJezXKwbR=oXUX?69rnnS z(CY0Oz3+can5~w3!L364dr!l&(onbh=JcjXp;fayYVOX+h_P&RoYouuqWY;v%98d< zxr`LOu&Rq5IV&ejv<*#j5KWsT-pb8BdCm4F*`4VJuT}7QWXo$bZEBC5>(OfawL5uH zcfD!Xj;a6iFMITc3U&HgW@gRk7V;>0ajj?T$_cAJOt~~;cGxD5tP>S4H}(04t1TCw zHCLiy(#;-i*Sa+?CtY|py|bgRLOm|;yNE=4L&0|In09OLh_02>(q36)aQM^?Q(nxN{M{%2*Gj>e*!%UU^SRR-LcUmnr(x3YEf zjXtIHsTYrz)mEfljZ9j9Z9+oCoC`CjuaRijdLnntj>Wxg1@2}GL(3QLagX=inWuIl zKRT)Fm{QIE$mA4B??Tc4&7NFAD^3=-x;5XOJ~?z#pU*E2X~{1A$nNeRv%^+R%lu?; zZfAppXkOs+5^Iahozpr_AJ44roZ9uRH)d&6gV_XCuZ4%71qYU;wI)uV!)~4ZYtEJx z^A4AvhdW4LP?8NqK~IdS+V+2RD#**qURn4Z_C`f!>2V& z??2bDP9$kTy~P?f^#!3FXVm@31qH0mZZXT6c$+8n*FX;(w7^g6ziQ*NA|8eP7M9W@Cd_J|Y85s)`UJ_M_&z{YUp1g;{@$9PjhZiH7c{zd@1&a)>x^f!4Ws68)o-!w81yc(Yiy9;bAf&4U6w-%K9_18-B=&+IvKW*5&{QGr|R0gRDee7+B^pBK3AYjAhr6}C^sisG@2 z*`6(xx01TT8(V&>S6yIJc2lz#t^>HrarI$=<76K~N*$vY3%ZB7-dRhAY!O`GgN#3tJ!{^MHYIp`iPM zH+N%X=Rftk@klr@_e|8y`Q>viO-kJQ-(mKY3u-$a*l3liDF0(9T6Fs6B3YaMROSUw zofM?3C$j$Gdy(*w%{uYPqAh|q|1!8K$p$z$h;QO${-82#9|wD*=_w{>p#@Gi3^A7bL$K^S1DU?rD?f-CLlrtM|FyXYQhkcQP0Lv#ns!+_A>D?+zK%G{?9?cB*~d&7e~s+bH;TFl1qGJ z7Q8t6F|t}gA*g0{3;*Xhcu&8^B) zjaihJYu1w7mz!Id*i}6t&n$OEaHlU{my~c7kFaLfp_cR2!8*b7yV%RmmgN;rDr;_? z-jzJvS0hhmsm@jzVJV)N73`b0c6ZKj;tzP<9oZE&HIHRs!G~jW{x38ue|J+cG^2-& zm*K)yp(QUI7e!6^vZP|_Dk*lc>ZH)qMkjh%5)!#IRvlQ>KfAh%A?0Dw0l&4jF&AWa z9bVwum9psFp2`miUW;w*?nbDTtk-N<7NBq2`fJN8@0AQv_m~YrJCuAF0!l@ntF4K! z)sZMTdTQ(1=(#adj|CoC=){`t_10C?=s|R3OJ2~K6G^S7b)+XO6Ok_0^j2E>i-~FW zrM4GSI1}G(-8lKtLT_^oxg#@;Je-7`6EE$$c;HdTqO)593MYJM@IGOk8*?o4iO?x# zbA|7w6BQaotd5_5u`|&i;n3rvo7+l%U;Ci7GxEi|Iea#ajV+uen^Fqps=M~R`uTre zYi98SM)i;t3k+&>AIfnhbp1-nGj? zBauyEeNHo29k_B=h>C>^?Fe;BX+zd13y3Aqmo-}oy z>U14f|NrtrUlQHzWnLU~EX%!Pe8;vU@_N8r#=px#lwbUL#1_@abf8I=A)`ROx!a&I#j=TC+xc#qwJoCc0X})0*VDK4@HOvC}J_)pX2Uz93H5_yN1ua^D2z zRc3`}T(jhh4m~->$ab`ck70d=j*9XAyvrwgKD_Ig<~1+OxH&$#qNqPVV4=~(O)qUv zx0Ji(3@~-tcBAGtQmlB>c?RaQcba23bdWl54_E#Si@P^}qAi zmyetK=!=vt(fykP4i+iAo^+J4X~v;x5e@clD|=X(I}S^K30lp}lfcHYrbFE(w1=ga zOG0CTUk=Y^e$Exj26_P}8QRR7)mTmmXjnJ|&q?Hv4)BoPx`C0&ewk3!h6X=nZMQud zOj>#dT1~4n-4lKkTZkt#bN^R<$)4f&k>OTAIjqM>m54d##E z%r_lxcqb=X(7>FtVgHjGiEN@84OhNgU`bO^7cIMRO!nB(hQ}|n*kpINvxs#gniU5D(d z7`&xW5vdhrY5XYj!OR7Q|1Zei5NpdaIL^qhfsuU$%Y042^-^13ES9~n zkX>Yf^6XW2?wrxt#Zl|BG;UeY;umu^xdlrqEm^!%aPh5(%UB~f+i{OxW3YzID))@otnl(JS0r`L#s}(V?L9Fmq$x(F zByCHe)J>tO2fbLAEM3$U@vc;#;l)M9tdAG>q?|goDevkgvxYa_eW^FENis1^;Y=@G z!4+{Xu5lxp6%0Zrxnirb$f1;-e#VN*HT9pSsgp%*PA`F ztIGZM4T-aXZW%u7_ny(Xz0@gu&-WY5?+(^6_Oo$ z<4~zS|1#}9Hk-Dntm#{=RbLxR7#!VO@Ky4M>4u8euOF;*W8IqiW_`=|Rr)J?Li$by zzE2E&er%?dg}ceVfU4y5sr6U&cHVXU82oaTDf{*rCkp?}DHYmtjpcCZQY~K1rkQ(^ z7~{9O2X5OmO?}T|HF?P;mwH##Z#7EME1!N~pYO)06S*JG)015Cly^piP|W{ibEZzM z!aVMm3(mMPs$V-D#aA+k-BwxCKk}@+Y4L)UIvG=Uc^vt-YHhE?-4|ZmvrbJ}Rx&+% zb&_*8b4=tp!OHhT51_dfK_tq+Hn~*Z+}= zr`s9z6!vC6e)em_p1hJUCY{j4?kjr|GFL>ei4rwo@^Nd;+q!>Q(ldAQ@7L~^C2yK} zDkr`F=e`Ge<$EsV1|1MHi|bG|=*nSRY=hHkWe&i`MVJH2I`>kKUNgmgZh}&Izx#XYu8>`vh4yM~YYj59te5+rv z?Cuw(%k_R7uH2$O_cwRx;dNY(o8{hn%>Mr2N9@-Lzt=uC-M{nb%r@II%WZzU_5Yu~`BwIGqx^%Hv**2fulI8AWBEUw%jdjEivRy-uie+v z{ZVazZITo?6-FlCCX`r}R^!6H;>G=}d z8Y{P3xnC>&=9H=OqISphEai-fn(bNK%Nv!B)>wp#ZEUZOJeYQYts&)m{xY+OOWV>P zAFGUdR-B#QWSQZ1x-9j6T2W70)?W39smJSFmZt<)6eKm~sYGPw9xGU6mbhwrjg4sp z8++p#^GL0V%Fr3r&KV7v64}M=Etcvvu5M-QGcqoeT76J0?KswWzO4C>nsVmzjO2|i zi_D6;%ag^=Vr>PO@f4d$h6cDBk-cFY-oqfxEJ! zc}b9Cm(%eMJM*4LTeEh%=lLz{I_l<9M&`2Q;J9Z!IXkQ-PjBFp=v9`;C{*m8 zeN^IjMp>3d@78C%Hzk7e)RSM8T3L(6m%7LO3`;-#y~gE7U64hK+KeWph^l#R9R^Lk zT$wE!kEN_uO-wV1Q!kHj??_3R(NdMJzwvU}J@xwEZGBOb3;1?6?7UI+<;Mis$il*s zo`oyQW7EryxA)ChroZ}Gj^MWTHSGP%ih3vB_@BLLdar}zlxc2NTc2lS&B)uit*=zQ z;HbIS|Bn6{;mHqfWE~Bcm21~?&77#tHYxjQ^H-;|<`-@97CBdUWS+~Y*SVIoJ3TAr zS%>P(DMvh7Qz|CMh~+O1o$^nj-C;)yuSU&7wvwZ54X)wc+m`G59?v$NG1YG6j5{x? zVpdj9Th^1jEzes$)Jd|GYmzOR57P|$3(M}MH3=B zD$El#f8?7-c5T`@`@H0YNVQN&w>pc=_V$xAxI5O{R{x5oBUid4$;pp*##na;s zyeyv1KFz6UPFmPxw&4B+VU@cir-wXOdKunpe0=_9@s=%`nXYNG^vmb{y0*Y$`+`=@ zgxQwuXHPEpb*#^-yuvc9f0bFQuSWAdk9qfQPKa-ts@gSs^32})?OpdHryrYM5@WgO zX$ zMmLle?^IhnO)Sy4a;oaCc_EQac3xROCncxOsQQv*HTUOyv+pS!FGKw`rd;)`p0Ygi zMOodhXVrJVRdM~8T*SFtmc1rTJD!s^1!HHx2??DGfP^kQEO(q zyHfMgYfFSRSG@>ds?v|SA?mF`s*9zC4EgQ>A%Zf!WS1$IE>eKXE z`9VEnL&TDm$!o7wRWd~NZ#bNMJw2C6dQRE%xie1HS5>ThT3J)`blQ`hOE+6C&aYav zd}+hCX)z~0Eecs)^8Q!tF84l>^gQ9!GnjVw=x=ksCe@KJqvBXce~Nlr)wbk$KN26b zC+_$7pZQ{1ns`}S-p(+g>NSzy;$NTI=rpy%R4dhUX{V;!(tW?upPlNqTD53$TC{5Q zl+6}R(>l_-QkR&vWrwcpJ8@Es)p%C=c7rbu+Aq6uluz(il8#sqr5<-x+2rZ6XFH3Y zYON0!++uY)Bjs4Q!nUN13rpsSZg+a1_VmjHPqCFBF0$u8a@~1z#>`iR5)$Q=r&s^u zoS3F%_50QO85-@@YF4IG`)w8#xF534a}@u2f?-v)ip&9qjO^_)46MQha_-ZZcCFrh zUUch}NiwUQv}Y_}G*d7VJZjdsY`0(|muLdtRPSkN(^xkY@Exk=P(8oXFoA z(l4|=F~M`ihGRCZGR->SZ3_SGWIhP^Uf`Ea3XPxYQ!-J{rIfv6^6{7g6zYX{Mgg}`O_A`38Y9~m3FR*ydtrj8dC!Dfs_U!09wRusickT#} zZ9AH}xH^}?GBSx{*Io09Pcx1sY&!5&kwr6LPmCaY|=$rob2(z!25K)RW+8W}x@Zh9NtMyEed3PeiiHK;q~InX&@*umskJrHb1g zv9U7nn>ARxj4{$H^j{MGMZfwdCv1+-qwJB@1Bis!&Zv@>YhBicHf^_N|~*;&sGF? zh9~H_FZ$M+7ARZvMy!x|{}u<&?aCade=~5T^5_IGJ+2XaHvMwVAtvJj_TUc;(+@DE z-Hkb|%(z6+z}L zQ`V7Xvuio=`~pjufZ~z_cE80OqL%}-Y#jG{uHT}*@}lLHP@nQ;FN&m2O`31q(bqLG zRzkTpNu~ZH=L`Yf{0kb9oAz{V+_Of|kKw}cw3Pq5rv*BCJz?`;VaP48iF6WIW768} zz%%nPn<#_p*$Eu1ww%ioxLzAD>ZfwJIBJ?1*eo{UkUC(db5VQMLBkh!Jwg{>_`xXF zP{hBo&{%4*kkkRzUl-W<4VVjJ71QyWHmglz5T4ukiAkMJWd#8cx zCDC~g+STJ2d@tO*9m{>UmrvA>;~_6=D!XXF1r~3n0Mm(FUk@lSe9(Atf#ueLOY`3v zd+*`*c9fjbhJBx^Xl2%s~+&bZ5 zT*%8uG2Dlg<#Gisd^mXXK%vd|k37#7dW+PUEoYKH<0xlrVE5&MV)6u5(K$@%ewqwd zFML@T?Q2>XvZpG#GPyWw2A9^I#?sZNeY@vfuR1el|H7Y(54EojGQ1lpDAo|jvF3m% z2P*^j!Mj(QUaGrX{dYo?e?!*7L&rM}9P{no75eF#D(dDmOLyTb*>pS=pAB z8zw(^czGG_UV5(EC^l=+y*nYt*9&QMDLrGDAf;NMsCwYVR%74b2Hv}`g+fI9RgrpmXo+&w>PP=8l5}P) z`n@zLGHbqS!})z~pKqiE`vuL}u}P0-0T);3eD1r^jbamBkrFqUFMvVflM{$8ZHK+G~)J_yUYUtnG@U_)%-JP$eBYy0@a#F%7(jxfpuQRs0 zp6GM*tZdup&$;A6PkNbKTmmv$HS?insFd>!E(<~ zA&CP?1_$_DoP z7dBBLyONzVHRj#NsKaH`D`KlT7`Q&>{)l?E%=!zL@qpXx;87|6;9uYE2F{RNXss*)4g_g6+O(cT2C$)!>U=e4IzmMQOgf zkm!-~ZC^{aD*7zDJFVYk3G4Hy|Apdh9!go-N$PQJI&IS*Rc={YAs}LX^!bCHdz@1Q zYIP4jlPO$t^yoSB>jecWb$>d`W3yK)?wmidi*e@s^59d?WAY+=3R%hoTz|w?atE1r ziK_o}pUo2BAr>5dq0l^eYVwkVc3Fw3FCX{s`kEQnIrl`^1D{o*?R^~vtWx(DsjL!? zDZlQzH)`c(~9p3FDEYK{dr?S$=5wMRxI3V zddq+2ZpGaK3){Oq&N-jBcB9{N);ckPwnx{*7XM$|T$Y-Yes`0`<+XR+Zb~~F?fl_a zqsk?&zwAleCapsD$;|m>D;7>)dywtmlRGsp&PHv_`R!10Yo5fvw)YEGnucuLcW2(` z+qP58tkxep_lo^;rv zcCFd^>#mPsyUzWX-2KeQwJv8jo9Vjj-sdWvw^K!pSMi{<@mj@peZ`V(+qF*}^Sk1< z`_ApP2hC(OuW(I`{eFG-x$0Z_0tg8gj~u?U%n^f4@gMR`>Q}rspa>)9>HC{c!uoAFuu&zFVeKIkjCV z_s4s-z~8fODVBfJIG1Ohu63p<>8$a4!L8A^zOw$mZSmcxZpFvK>*Z(udhhs&iFT zE6e@VS$ee|eXx5} z6%va&SleynMXsH$D7Sgvsay2Olk@JRj$PX=we!9(++|;EYFF9VTfxRH`>bop!Mi#C zdjl(C&Yzffg|GPwZ+6fV{=Ww_oU%*#6K!H9zKvP7lqWKKBFmcOTE7g7MVzO4k586f z$Yg#Dz%~6SgSf}MwKFhmqld?XMIUPMBs^!o8WoGpi&3hszBHU)6_={_<#Zmb?e-d{t zXDdxqE?R6hnaj@loM-%n>9>DbVm)A)Fnr|!``-Rrn3pC4T6H@$4hle8rZKN zNjn?&Bq}{Z^8385+;!D&zdUZ1V7JVhdA|N7)2s=8r^7n#2Bq;Et0s89^2%S?Yr|h0 zuJ%5j-^={G%vtIK)1^126dbJcy0+H5RP zl2-4Ct}-ibD_{S_C1zF7j_vBN#E!VX)~htKJoR$=Tg$7QEujuKz0UGX+}^Not6F7L z*0Osh->0g&2=O#N_p@3Q(zdBKxH_8c;te(?2F0H&tTqe`3_1)93=A6>IhflS7#LVt zS%1%CXlZHrdb9b+kt5&V9r&`_#mC2I(Xus9pS}39PBA+>`^uFoQ>IM$b}BU_B;?Px zN8j%){J(->%a$!VIy!gm+_`w+{GZQPzHKv_IDNsdTa$ks2{w0)e*5-qO-;?ubH$!v zSwFsi|Grw}?)^t1A|kb|laHUeux8Dg@2|IQ+kI&I+~rPwDc_f{96oyL*V|Lyx9VD3 zTmSxf<=al{x9>iFJ74+d@0YLZWPV=j{=VO9!LkkCH>xD()?T`LbMo~0F)=YepYPhV zZSRu@w@;nD^lgX9&xdn<9*eQ{N-U}A42a3S|KN#?j7;z3Ig1xB{E2)}KGGe?3|AXBD@JiOJ*pw|;-U^K-k=q*+UT-JNae z7W?aB$@d>Wey5Q|p^Y0&be_pQr^Z)

_WauCxpejBUz=2B&z_x}oV;t-uCo`e{yrI3+0+*tU-@~+@;I9M8?md?(81zq)gB^YZia9h$jhy+j<&E%KPG6?);aF?uip!Utp8JAbZp5B%ly*cmkskz?U*WKOi)qktS zJ3!xWPXjZzoX?I84-Y4+e{iiS`slb{Qqg#x&(2Lw$EO)*=XKm%_T=mW(=tBaU0=?g z&k3Bpj3eXg5wUfRtN7lid^;$%A@{Z0%%hJ)I919mY>WfAU6>qvnixMEdeC$CLak7P z)Mu06G+^UQ2p{@_yrCoOUF4KOlQR>&(Kh7VRSAMTBYgkTQ!l% z(N%||tw%_(QE(=AuKR?k?1Gj7GU*fkTM0~<-Iww#D|GJm3mhR@E2lJ?im^mT9*}r^ZPPVl?PHU!PpFD5 zJhtO*&&ET?B5M}hh-%-nNvt5)H{hmh{aT4TC9AJHU8`>Xe|D>b)U!Y4>FMj+Z{O_O zxTW9@TW9UJE2-Ukx7~}9=L^48?7yz;dL8pR6SsoLbEPjXYVTY4Qbc^qhUs0(HJe^K zN8kH(rZ~E9!u{I#W!s(Y4)+MWHuMj%aO5zK-TZ#0HH&zRx%ih$d;S*vseD|duTp#Q z_<8A~5A&@11s;dKx7+&NI==RgR^^<9o+&2;rTmVxF+?c3#H~0iduT!{i)W(C|EL>>N5YFOT(yLjbTc(7iEmwT z%yQ|+&a9S2QU`aq8W{z4rJY&i@WJD##W_xanlnjqyCaS_I!x?JhZQjRN68a@;1p1EhK+Tk0s^gd0T(XH}WC-~+GKeNpf>t&u9`&;^E*ln7T zws>J}KswOU`CCO`iSG+i1?PxGitQx}!+T&xN56h9SMxNb_9@zodJ{9PBPwOv`3C!*<^ zUUjK^Tl&%$TQWQ{UmctF?aJc%m{<0ZURPF}3t8q7xzFEEx_18x3w>I;UZ0ggCgf`vAM3UiCQmaG zPA?9h@ow{MaowC0ZtLJfL7^;@V|82~9tb^oXv;=dUCq?-o_+qTB!Y>wyOEP)e! z3uoVEeCs4F9Wmqoxo=yQ64tuMdj+nWWwyTKt4_TCsqh<9&u!Ng$`BAH#AZ(rR+3Lo= zQ%mz25>GwoJ}18Yhpj=b;+{nI*%DitOV>6@>qLL;o3a0+>bj2UvmbFZZ`-KLx{|}D zB%=G;jpLtFH4}2X6BOfrY_k=6lQ22zK`qyfBS%V$0-w%~mhk?$cSr2Io`u$rrHw^* z#$4T)kneL(zMt`Qh*@6I*`9~fuif~6?C;jR$XXm7u)c@GkTWdVsdv%h6@wPP%+rH$ky!H4@QSrH_m75n`l}WuU zCADZazt{Y9KCAVj?X-@0iePdtS>|>Yp zK04g#ept+W=U(l;&j+k?^L794IyG_M=bgv@Jb2nP=hl(`+jg8dYMb}qTlMbGt+{93 zze-v9t@LT1-j_p%dlQdd+WqKVY;Kpe_T%@TZXd~4+diN3+u;XEyJh;GH@o@17arVm{ehIt+hcOy z%P(txbFcpLUYgxH>%Ug9Y`69CUwOrix6hruvFP&37p=xU6JuXxo}PYBb)Wvm;Ov*~ z+kagse7-3-IH#iZbl7VXKBIQWdzG&Cvo3PAzuoZoUlQMOv(06Cn>MSS{~pV}^Je0` zt@HeQpL?Z8zWSHC#^K1F+>`U8_Z|t|TQv1e>RS1^_l_*jeb(>u{b_gQ=C@R`4nt%$Sth&iD~+)@QMe= zV|RyTR~=8&S5GJkui7#_H_0u0k$T)mwd9X#<&0+KLE9rwO)vlPt?{mS&dzH!+uKXC zzZW>C#j9RxVm?;%+B`<)Yu(ZE%;V4NFPZ0m`&!d$UjOD>&5!Lxldo3%T-vzrd#xQu z(e&^Xq2sB~mu5?9WL`F}`#7y}wtMT@>lOQ^H!e=CYiDhd`&KPABks0(;s@8@7S{U7 z>h-77;t!gY?3!MazoKrMTH}K7@=0n*)zh*)%<5;0#hZ9Uo-oVHj3~}Duh;a*sOTs@ zVxCj*|5>o?x0VB^rnT{U*Zyy*51*EQ`j*G=ti(yYeEsnfHT619xAJpt`ET8dGT2iN zJ&!b(=xTWpm%257%dyg!=N;ZZD)!DuxO*df!;1vzh_=VuB9xxhxuo}W9&6=vtD3bv zTsu8(wP}lNdyPALn?P91m#uZeWyQyy*6v^0)yA5r=-#flJ>S)%YEyarH?b`5W7$Fx zxfj0oq(w~J=32knC3z9vX6FlReFN^;ZQ6A-1x@krW^Rr&bqp_3P z`@}pmTr+D=wRe0|=A{Y|rJYyHHd(~!mrq~s z)|7KKXtMc~=dNWJQfI51l@~;mKZ&S5%Gt#e-WQ=Bq^>dbgL&6tw>YsIxgy(YYL?Zp zsJE4;Pj%i=t(-bTW>d{$#N%CGFB|) zHcRC^F=6qEEXh-W>>d+OoeZ))kt zTV6$L)U^h+A5$lKE@FLNCUGiH?%D#GXEBWxJ!fVv+TvDF=+^OUXZ^y=>IED*3}%Zj zKZ{`bxOAar^ZV)jQzI&hUS%;I?@GyNj{bkMEwF6Gdo7u{I5^UcwW0mBTxBy$)l}v-8&~Yu3GhU+bkWo(gQmxXIn0lXsuMSn$SLT z`s(x*&EJ+^UEU=5DtcpBrMzYOZjFX>9!oTC)IVPt>DiW%v9n+9MCFWQ6K_1PKDu@3 zgqbssWz@($tKWMwis>T5TEV*9tfJiKtM%B|8lvt-&XHZd)ZC`V|z$ zz!35v{zq2%)8l#duVzYTmR(rY@b_!>*Vbhj$FpP2TPIs3{=8nE^R0bPS>zY1nyt$> zI$dA?rlVlN%_{$_rC-Cc)V=(eA2RqlFuqJ=52EfUX{htCS>P@R9|M*5DK zDMn?{d(6tWAI*y6Y%iprZ(h6F}49tPVp zhnW;v&2Jt+6+I^IaKx3H z!K`J+f<1fsjvSh^XLFCvwD+r4syXCtutjGWG!)69-XW^@;v zKepIp#tgfa>m~o+*t(Z1Y09-O*YNFYwyy}wTxszuN5p-*enijno6Xazn!DPUGI+4w zl0G85_X3k6qt@OdC7Uj~-4S3lI1xDE&<36(ak=afHrvvs9GWM7C_8|4(iN#?(kHjC z*|T@GK%fD`OTj~853Vs!JhVwi;MQ$s7DI-(zkAmvMl=3iuk?U1bOW2<-MuUmuZ4VB zzjon`*xvIjjXO@wp0o1h8l_otgnU}lvPxes&-Uh8cSWoG;nAR?TYHnA@A(bS?f0WuE5utBj4Nw0vmQrQM!?I>G1vPiHmQuXr7rj;V#QO z9Rce-o5f<6aRr=zD7|;>#6x8Yr)&QIR!QnzsO8JV>gcqs^^X0Wz5KeDs^?ymX=2Jt zVZA)}cGBE+EQL2s&6|zc8)i!Ef1$I+J7dnno%g%9u4BzieRgu9N$lkJKUObEEB?PM ze)E=^gSG{WpgvbX_FmvEY1RXGgVlPsGH%~dY;<<5BdenCmV-77A$v~$ z`gw{|@Q@a7C|l!Qox59<`>yEJ9qp9$<>BQpTqF<_d)56ai;m5Kyc-+1DtD>9%-mMq zef7mE-Z}BN%x2z{om6c#^9*}w>-D|1FIQ}xwyUCY*OKdt#XYuI*A(584!OMc+MOwf zbm9cU{|IQ>T#RhFes%8cGiPrs{iTv&zRPOMJ%e3)l>cA7BNlsJvi7-SjX>g>=d6Wy zmDcVxxO!q@-KjR4!XVw3l5)P9FK@Y8L}qjs{a?K)SEjl(GqucPx^3lBAKBv^+L!e; zwl-a#67D(wSapQXzY{BU&nxoWzh(XK)0yjf)+d=9oeX*oKRSJZN$}n-TlE`n_g0)e z6d7}3H5bE&xvoZMo~!lVJ#{_aP36>|CybZl1eD`g7t9v$lDlKi_w;J?-I}=qDml?S z6OJks9MQS+*nR2r3+>yTy31?B_pxu!S?_&7B&%8MS@WM=R}TK1Yb-u}Piy**$f&$4 zI~VMJH#wWNA%Nj!;qxb7POLSYYpty&vf*t@@1xXNd%jwqZscaTJaeNu00opOH6^q#ZUG`FfNpYPy| z+Rg_c~%}iemQ#AO)idwUB72}wXgc&J=bRC{C~G|mY&>O z{(qXJ{)(hGEl<66FmnW}l)V2?yy&SZ}XizQ*QF+>Y4kai#BU zKF&DOE$;RI%+(jcbBf=ta;?0v^4bc#T z#JMpa9ov;kG?%@6@+EWa{wY$Mu3qiB{n~-~v(*-#^-7mlne%Nudhx96**SW*1!vlZ z9Y5u9`1}Uve&4sQ_q2G;9PVH16+e_dS+wl*{*vs=krsPvKQ4Npan>_jOFu?`clgOD zZk=npuIQU6q@VHd_5Rzn{r>G=QQ>>fYDNFh-|}Eh+1=lBc7>XKE0Yy+_A<7czM5?> z%el*k1KyN4$;bA32Pt|#Er{s-ek9-cmszsM-UU~y1up$R9p1?209{k^*hx*r96BR^?UQmmcE>?heeMT zpE$TH`iqB^@E#dWvCfdMmK{sd)_pyrw0dUMokC=L?yd(GRSp!lMKA#b?SX}X> z@c8tXtGE|0e=4+NRf9T9m%+RQ=1qtET-W(#1?7sB2b`EY-Cxz2|83mub=`mT%C>NnNuLK|)`!2OHH!c8&v+9oVbS{9bJj+wW)EihX_BwA!*@M&`J%q|XbG=RgEsf2 zu1jm*zHJHKC>gi1%v|9|>GC2b=7|#~6)Ulc@+x~8PdpqTyg(;P$HP)+{@yt|^w!Eg z4Xax;aa)AxzHnJz4?E|@4VzBL-+tKF;F>&3d~xw{b2IL&oW;|GKA$jRN!{2he9K$7 zp4r8~&aiG`(`4n_9jazk6VtCRp7rep(?VU_<&zSAFA|89I2Cl%AV|yR%G7Th!s2zu z3tlGAIkbjt`!~s;`naw~*XFd(UO3lqn#i1}o@0k+vghUaO{!Ln{SYP_@l&X5!Xz2F z>i%tI3wF(P}>wGqt@&?bWi^FPOX^&O94^Y|`Z#151%(B1O~Pn1!|_eRVv@ z!D%1wID7fGj8!WeR&>Sx6uWin+4O{4Tsh3At{z>Nd^Wf@$3uWqt}jZ&yL6FjrTb|< z$)6VIJ3Zg!9s4M=`be5250j4dfjj-D!)~naUmHBz%5$!vdX!yF>WA_hcV;IYa$gds z=(8~T(#jc&L>C;{qqE=#3-7BLt0bcT3fIP}2(|{iw0`c}W|{A-?7R9#u({~g!fuy9 zY3Ud86;@koryt3z;t@OIajeVBrh3xNYgy++eun5?S|y_V`?Jo)HOGw3cMB9Qt31DB zm7z_o=_2lnhmQZx)q7=``sUinAG2Hz#BH&u`695>#B!a~)-C0eokcDjD3|`58sH!Q zM?QSsSLHm*v>IlSCn{GteIDd&UU1luPk(;Iiqcud3)e?ak;yzBQl1yuva#r4k<*h! zSFfI2waxp*9g{SZosL|tGpDHO$genE;~BiYqj^Dv@9`D+7sJeRm4&@tsBFBgYk&gC-p{@vSjJ$(B`uF&+TPf{|XWueKTd{h4`Kbn*NCgc8!6UQeVzw%Z0{cCaA zlWYtP0St?{Wpfz{9x5sy{BexYKuEJ>&z!!#YpRt-MpC7T1bgiggLAjzvqi1TXO~{_G|wzQ@^<;uODC&@w07-^@Yt-< zzdikQcG)IZyN}!q4NOV$)(jsMCT{AMeS7+yv$_3Tt)mZ)c#D=@Ps~d#2wm-kEnXBba{8KtTu6Q~8?1(S?C3renxj3kC*85df zxqGgeeOP{?xn%LYO^RoyaVWGX2s$6W_ef$n!}09>gmrkNbv zV%-qOqEpg%`eM%4tu@JY?_T`nI^)UQz^u0JjjGbE>kJE;&KkzGo(`Ehf2HZv;4%)s z{d2P(-s0X|!Me|7{;t+17Oz&9R@$Hc)$`}qS+y;jey`8D7Hxgn!+NXq?mNo6Z!K;( z@p{sKp{;_yr=>c}zjK`T{Pw=$?iI(p?RIx=+t96Z*3fLhguTZD<~uPqXl&r=e!OG- z<{q^O$77HC5@yeH?%Epg)1awXTXKm+z0d}sU5mSm9M~V5?X8xRuUS>bHS3AP7KS$# zPMdf9@7}g~a)-vB9hWnR~idnN18fH-2NmywE#3|W+&!124LCkqzVY6} zma)4%qrr5E2Ezs>ZHG?vlEb1dCYF1ot!x~-Hy~PIU+KnicmsH8+1UNL}o9dWRdshr?VIg4Hz~92YEHs@*yjPE@lr zSXXqYHZ=UNWMSxFVQ8%BXx_r3b>c+(4(HB{=1i6bevcE~!DbFOnruS0ah-7P-hC|1 z#c}g$XNLuDF2X1Kyr=SZ?pv~3X*Q!tg_4Z#tiA_U+M#FGG4D2Waaz48YH50>`>$?~ z&JPSi6P(s8u|0i&d7TVrwU7Sni<7yPJ4_iGB6AMDwl>k+VzDK}yJwGGV~NMC7$(IX zCp$OqNZYc5f5FsO zb62TOW0IR^rQ>9|e8z6m4NG?VC_0~!Syp{{RT`6Q>%?{GMoShba;;Bxy0Mw3H`(5xe+Tim4)*+r%CI|a2d*;S8s|I+S-r&~A>7kKvx^InZXMm&2>BD=s z^hgBwB*Zw*4QS}g>6jScXdPqLJNrc6;gb($*e&*1V%NR%;==uX7xk~?OnY-nX}*v9 zkD%_g7OP?eSDe}?fA`edz7GscUp>T^G@9xh4Gghn%RML7%|DaD@1?WJ7LoH@8|tr1MYwy|D_RDv_(f%JH+ax*|-zcAZzqv=_#O0o9 zj@Q{90=41}8O$sZW)t?feL3l|lgHCG)sgYYj)^NSHq7x9`OthsnM-TOMaBm^{$;ya znR3qEVqqWPXvAgv!*P1SqX~BoI-Ys$dGB{;&diN%LZ@b&nk4+1D>O>KrJA>Nx4_X^ z)_NIk{wc>&xesjG?YlYH?CuffzV1_>9XBjunl2c-&1lW#_1ztBxAaKg^?#}2F>Qz2 zr%LZmk1JVcuf6_!iof)V?Ni=H7T#wfmXkM}p4)KceUbajOud;44T^%-yPuj-vv{X= z@R3W=OXIHUSEcqhtlV^VVPpJX=l+^Qs%L!Hx>&RJ?)e{`EVuu$&r9iZ#kOZ2TZbR5 z;jjzwoA~ycNa*PaTbqOe%;K3_aze}|wH!XW**xd%#F#xC)n_i=oZ{K;VDeGtA{VR-}VkOTx&zNCEY!xsk8gP_DpHU`OIBcqkKI6bF(yTljhLo zxR;uJY-+=mwuV^om}d1Hj>HyjJq>S$330RT1a~IH_Df&m`y0XSVx|&!XUdkw_0gVF zw=UQaG)>MiK1%pPawg}7%so{~5$fF9-}kN+sEqpSboYUl^uJp^3k&5pi@2Uv2wh795Xos6<2yIPnBLhOM11?v49o7xo+KA>9~m5@LTR;ffDm)2V+|{ zTTPnG@s!o!-)s98$xXas`|kec{Woiy%MpY6Nq0(b&x>U>G5O-UFnYVjU9JV1_wV;6 z{cjT3lx1T1!t}6FjHH2^M%Wx3*Db46UMDJL9xf4{BJs~n;@M+Y=Ogy-F8vWbyRE@` zdF3ggLz|UCml-DPW$HEKudwC*F>&rijhI8Lyo+a4Zk$m6NpE-afn1)2@^2O{n7nnH zhPH(B>f0WMPY?6dPwAMzyjgdd^882EHaxpIt(BHp-@R(IIV4G7fzd)yp@4(Zb~1so zg;G+T#k(4<^cOJ1I=6MpU*5sZa5SKXL+Aw~!-~_#3kCk&jNMvC=gFKlPvVwsU{svEV8w<8yNvm}?#LdJ zbj&$^JyO}>;N<@uHx=GVHz@{~z5L4HFgZhwNwDON-jYp0-&4=twiFh<%#pTB@sY<0 zUZeQ5ON-tW3QYUvdPnkb#H?$fcIw}%%=wJc+ZLxkIk;V8mh#?*M*l-*_A$+8RGgjh z$ab62ltpq2R2tVMRvxML<7e8&ur|)Q+uI=fB%jN(`16PDRf5j{&6)o6;6biU(=OY( zv|Q47Vrcg>c*Q2kgN9$`Ep@#0X-4MK=ALtRCoJ6;&sP){&h6_j<{aUi@vVDzQjpb* zjoXAi@BLo8p^)e9GUek^Yr?gsYMk>~{d|If;~o_S#zp&#qm!9-=|0~4$yIXM619DA z#HA1N&nS#nUVI>j@%1d@=?4to{+D_j9yrZZuGv^?+x(7Qp$;3b%kA1GTRlZsYWAex zrk6XlYCH2bdrS8_T#m!a!z zRkwil;+fG|5#5({n8c?yIV2XJd2}(~D9SYP@hmUS+|P@4e+Yc1Dmy4Ouw=5SMa`6b;i7tMLCc0Jpgs z;Y+9f+s(MUF!0xc_D9RByp`8K+VIasbu;=xN@cc_glG2-#qVeIQ8FJV(@D+$JzPw{>)oYrNYVYH`O@m-NCD0d~#>czBFg^ zq%ygeOMOZf*Sz1xvD!6z!;YIlFZm)?e%$tFignn!4coKAvXdsR-)8by(DWxRub&gugco;cI?r+5CAmYeRyD)roR^pSuqhN3Eanf8(@}II*zD@ji?8 zPbk|{;hm+ob=Ezh$<225w+oKn^P4oQ>=aLvd&T}uZjbvv*RBe!S@ZwYvBKh8+gINe zj_%y*kR}x3I$xQ2zfW|KQHY5i@6*8$D0tWq7?{=RT`>tLGJ1t1;%6rq*dL-mZJd%B5cL z<$`6Ct|Y9f_cGZTcI)pRrI?A6o+m!Z+lm5xnzE7LaXbWaZ3Y3q zgHF@#&HJPfmh-)%BJ<|E81txm)%H)bq?nt7k513r=4pHMg68WLvVYDR7ntYvx*N}4 zrDqtvepdAtxh0%y5^vcUttgoEr>g2b1t9Yp&U+rIk zO*6lq=~S|xzxk${owtjrrmg&?nGaoi6eAza_>@uGT)6(}&+v7QmO+y=!iDH4InhiCHT^6_YHRz2II&V6f&<+7!Rx|K@w&d%7PJzZIXI;-`b3;s%KY5uU=PM@l7$HvxwF1&B<%8Bbp+)H|^aO zy2NXBY@=F<_y0>8kB+GBT=XUChtkb;O{vp-dUg~Y+2N{t`s?c{8%}q!oXu0yp0z1< zX8%IIFJ~;bJUz7H=&mgl<{PsY&**)`!t--ez=}n>zpm^xFbdjf;F-0B=aYr^q1oA1 z=WW02x_5r3`>bbYH!k*{bF{Wvk4wbt`3jY2;r9_y)#aZ|y}#|tDg9HqJ=InI_>Yg3 z8C9OCGsDhJWSt%Ge(e#_%yUr>soh;E;ovZ=A; zVp9Cfr&k&+9(|3UsBLR@Bg8J~|CVVrK56Ys-NLHUp5+GpJ2EY*~dNm3b`(yepqBJ*^?<6E;f;K+Rx;b zU*2V8)g1B)osrhEQ0#gB>cCgg7y6&NN9M?HON(C_E_n9Zh0uvsQHK=V*gx-N-z}Av zYvw#Na&B-}<;~J7y_NT`<_Nufx8>HZ^hIaRxlX-O`RdWu~Cot$&@(*nef->A2kwW5b%(EqS*o zZ?@`}ja=`)tPyVN)0$)6^9V92n zXJByZw8WzqL89uC+Q~A>TzdpMR*v4r0bUWxG_A=HGEMTJJS!-6f%!Ha}aOL+SJiLaPC^=)p~YW|~EhxZx3%d?H>obje)g?oeIE1lHC0v9;c41=1U_UPPv zD*b=D&H7p)hq&ER+}@RwlDsDCJ-!nD?2qx5y;oglyxem4XHv=LDVjyPF_#lGBX?}@ z*|)h*%yZJ;o^MNz2>MM*kDNTK;;CBw)9$JwZ_(;GmV23B<#wp6%q@R#X|V)bQB13j z%+-I}-2Ml;X7ln!h`fBJbAdrSoO_LkL;4EgnQx!ZllG3BH|z7X4Bsi~xgsxE4j%W- zbL_6FnWVnJVu`nISYhv>q6w4ls2udS%`6l;6;T_oL?>TOKqtBAylacswDuYN9&vkT zGCzO*|LUSMm#Xcw#bp{dA74?m*0*ob>{A+T+ZLa?GUKhMoZndUHCV6`d@|g z2~Sr{h}>M5YP`2dk#}98liRAXR+i&m6|WiUQB@naTUQH9CdP8@3*!@t zxOUd>?XA>3c!CYIl|YFT9vNZ}KGhv)+%6-7<5FzUwt*%d%UG-DXVBc%<4KBynh) zT*s_+C%cZT?3^K%Ua_pg>SG*V^UQx=S8P6JDDIoP*jjMzw8b()oxC{(i!PXzl)jC= z*2>ipz*cPG#dBw&gZJwYW^ciw#7Ux+dEd^SP&=?Zd6BvLyV}~;DY}w-&Q3045tzr;u-x|S?Z|6JyBDlX zdUT{CkS$*Ks`S!ic~P@1_pfYh);Srt>%!wly;|an6=YPZB35{xQ@GWaU?lGoLFaQ5|^-hV=q^sbXh^VLd+qn)V`zE$3E?-u4 zBjei58L#X@t3K{tuT~apZah0aBDH4I9v0VxdCQ#K1M=P#^FO)sNbI{q0$-L1pLu~? zzTt#Uwu~(SA)9ZSJlK5A_L+&+Zuexdz86`%Q>_yxM66G_;+OSnbqkkHso!_mZT@9R zXJdC3y4iPk~#rF&wnsy{V)qMZx^!|4{FD7_euJrgmeM9_zRiSpd`1gILd#$aPtlGzw zx$p1Uje#b~zxVFu44=ff=njm%;v@=hzuOPxohhsci4WyLfHi&42!Ba=%yp5c#riSJ$SXua>S`-Wx_d zZhIrXjbp~kRhJJ$d_LhY>Eo%o+v(Py&1bXZnHazH(*Nn`wX`7oRmz=sMR_K*1=T0! zc&CKsvPQhVATF)Ru4c=k5YTvL;heimWf&3`h$uYxB6{G9qdbS-{V%Z%`jtNKO*k%n z*gf`{Xyi8sps%op3n$>I|8HIkcQU&1`z zBU*jZ?z}C`B@ga~oG7{~q*14IbLXP!AB-KYiY^zcU}Kta$(KSJ8OCV zw@BUNl~9sD-I?$?Fv;B^N%h`Y4mtk!p}jWy65S0PH0q+<7bsYN;xiFC`tW6})~|!M zMoBVychCAL%PTE@7Mh}W&)2vuQEyU&`j7R7%%a&DcgH^a1cr>G@jhh$XT z9yuf=$t=2~W)$;(&ef*US@I@~?oEn9Wh{|F5AU=-VF*b5s;jVKAw%T9%dAgiqnA8h zyzq{M#Jo5iC4M26_(w~2%uUvMq$J1aAbN*ayyS?%s)%E$Yzz(#qGy!OOk`mSaqv=m z5OYrE%-#QsMIVHwG)c*vnOmKcw!HR9e2{k<({YPVYU6S;XIjlz|7YXG`W!cgyCUpWO9T`iC`Bz&{q?x;+hXCmXFRK@J?2jr2ebO}7dNBW#QjXj6uOTbU`<8F6TIE^h zVzn;T!?o>0#f%#Tt(y)lUG0^)^w|H^h5KG~^R;|h8XNwrXIX$8r%^_|pN55&eEcRi zR)fskSF$V@UfD-#Tdzw>?F+TO`^3>LQ)ZJg!x~3U28XtP!Sml`O;k$gH%su`we(vd zEm|B+UvQwCGontC^vEqxm0eye9F-BnP$pT^Zw zt#h#YR2t)%mv==Ng3>yCnI5;*DTy{{GQ4=>(C|Ogi!W}4)r-41%gk=A*?y~^Pf31r z<^3H$y03S(E-!41UzW7{Tc9~2wCvtND=!tTvLozqIdhfb zSxp{?Ov-Yes?2og&Yho)%0U{LVVO+x9^Y4bV3CxpnIs<0z_fZ7J9E$Tf-712Obths z=1*^0_cSt{No|?8hy$mE9#fb~Hjmb)z01v*bfp)iFr8CcR@?h;>0JdY$ybi7Z;m}= zy`=QXQ>*aU!!L|X&nLDeofSEf5RkoP%@(Eu)3_QNryclm(C7cZjAQfO$mi+zG`WZ7 zG@d=BCaUnHyei5oDZiEJ(6uraktIqjTXIv>;-;&;FXQsb?f5S?YtyqbwKPVVmDNAq zY4KeW-hI5qOidxCfqOpRnTbnH|2-_4=NIwt&EizOXJsy&CQI&T&C~go@$5rr)YdqW z!r-j|jRHsV7VpxYC|Af>kow)$&C@4yH%lNj<0^wdHbw#0u68nGb#`F*m7rMsY9n(mu2H z(T21~RdY5yydmpCSNs>P|SJ)mZmc36se&_coLsGf%69g&T3S@Yc&&Dl}Q zXC7qW(I92~iv9G#;2FDgBX(Z%>Vf)k+GsF4B6~>6EZ-?Zqv>!e_6$>JhhPS#9)N%Zsrm68VJk z8utF5_j1cQuXT}{qIrC8d7f)7bPN3^5yFrt#m~(6=>5u&J*rWjo6_aYeO3krpFQh# zT31Olpz-KGsV9Fs`Z~=vxXeE3aJxrI_!VT~bwPRLES##s+n_m1|GbvK&^tni}D)y7l&pf<71B zg$t|vBcgdH9q4U%d0wHA@y)bo6VvW%ITy0kr}H0tJg;zuQK;t%4?#C6o|3D=4IjK6 z-s%N(mcRaRescf!8P#&KvPnsksFPONx;>_zA z?Vc|KGxvr}t&u$_8}|EaAam^%jqhq2aYsA*kZUaPXvRO{fxd-5O3e%?9~`60IV|K0n`Z~grA_XKktPuUr^*dNhr zJbM#`A3h14cD?-y&-AqQ@vpphuCI*!W8gV`NBrIk9icO_tr*2MUta2zm8#<1FzJsS z%T~*nl>1Bh>)p4?ZQdKQFST}`|Fu0f9NU`b-aZrGu_7dd*)-{W+k5%woZRm>b*|LP z+wnw(eUI>8m?660hgtp4na!QDH$HJ%F4Vqn;a9JD`O4oLaXiJ1j4U!1$sA%T8cr-S zf(#4-ITmaOoZ9$!`vhlfNOEixmA7lrxR~hF$zYadVj-C1Je8SWDJ61}^1(@R%A0(4 z3Osab=aN+0k|~toI7?n|)tm_9=iU}P{jW?qHY`0fNiQ;JuSU`TRsJiSTy5S;i6~v1 zXQ}+`h-T2@LtEmntLeV}^fq#<(eo;vliMDIEpR=|<@$4@j^`f7uTrXB7oUW#wqLJj zo4rFP;$)@#;uNcrH!KkB-Qvnt$>B+ zyLRMnm+{&9)N}XN&Q~hF({m1OPl#Myb|%d5@bl*UKAC;i=3ifJ6Rh`}GyBu^wbOIW zmfgPgHDkY~VjSzMdCAA0%1@mnYEf@;f58X$?^AdscD5!+aT`q$`F(E9)M-{LH{8%_ zI=$}oH>1}kEXA8wYR@oRY+&8zC*8evJBLn}`Ijw0TGou4t-VZmy?38pvc#(8;*$S2 zPHcLu#5-HNYuTolBYYmh|C282T`LjMJkrj`!+&I{&u>n%b`_gT3gVnR{)Y2|jL#`c z7JqtmWXBApiR!a11+QFTqQ1KD>~g&;0UPC)`g9qw#lFhhthIS^mh0s`O9al(d@^~1 zY3G?$!7H{-i3#x1;AZ`-$gwwC*u4M2oz08cw|cKWwpK&rjOKHuzbidg8hQEqAGsjn ze=OwpE}zx<&7XU(xY<83ar@1q`tfMwK^yI`9y1voKX>P@E~`~6Q}}l4$L-{W{IlrE$y=-6b`;)t6WWtPRT>7&*qL6n_!DZ*{FPB$3?VSDj=I&Q(-`SiL zswp*esO=W>`5!;^tJju4F2$VshHqVk5}z!2%74-O{g<;>Z~FNPg+BRrC@lDtS>5y{ za}V4;V;ujuIX9e9^^oq-Ra0i~xUha&j&X36U|Ck$8?QsUjt510rrTB@WD*RS8T@AD z9wF<)p1T(2TCH=LW_*vwOTfA7!~Wkvr#{-*-;T+D(OY~iIATL!(%J`S-aUP_TG4mf zy4TIlb1q*JEq>OYu|GulN@T;*C$}RD9((UdcxHAtV&B(UM!S;tTF$w(TI&7esy&CS zXI)JBS0!c`w0y$8n)k;}rcKSxe}Bk2K0JHTrbjlyQ_W+J7p}YTq`uJTPh{3a)84$Y zzrB<7rmZ&3d-K0dI5*(*1JP-(|4;i<^84`)iLkD-hWoFDoLg{{<&AD~@e$LrozCxc zj;>E`?kwUB-}U2|IHzp+$B(D_D-AQ3KJQ9j5b7$Ob#94BUa;ZI-W|t^_i~zYr&^zq z?>9-F^7!xjWby6?*;!&X$*pVMO+Ly8dZkR=xl-3^ZnW>3H|4T7E4rg^HJx|8A-3U_ zsgBlaqsd)WS7#n7>~~aflDHQkv4{OzdiL`}ah!!)4!(*xk(|2dd`8ih8E+wp3%=0I{FijVH)Y;th#QXo3IP*28|9q^Oxn>{zA~Lc2WoF86 z>-eq-PnNh!O>(b&cu`JKd}+?aoX8J5uZewa+w&+u+fm$#Yj4jYud61n{5jTKQRh6q zZQa@4sXL^?+!iIPtnArT z=IqRuCA@OF$O5j~!07Ef6Xnowf_oDy9AE*zxv~RoS{>9 zvUHPB%Nyro|M#A1ntJ*5B9&(<)4MJen@{u?u3sEp|5nh~C z^i1zT?E3Zgxk4ndr`*zfK;qcs6eHVeP3s;pT-Mqly;qvHBU;Va2hm)mv zJvnwwG_MwlED z-j_3FX3)IS58URNt@(@RtEw&Tn6oEg;)ji)Zf9$5pGf5JDlT*lTI-Z)vtyn^_mWRu z6N;AvU;8GIUAJITR(;y1XIX1s6YA;le6Wl z@=UETzI!JpXHMT$e#&!R@u$N3$iwmVaTvsILn3n0qoWR>Q&2L+N z*{k0R6u1H<98NGMrO5 zxZSe*h==bd30IELAE%XSuU+D1c);hio-?ex^JDuFhbxsViC5S0gmE%iTPh#aoN{>i zKC>k5HwSuVc0W13Zl&ETv;VF4DyQlyl+LpkbK=?9z&tgXRYQa!te5Xkdcf%v28Ra5 zkQJL;W)-{$_pN!XYX1F#e9nu#;zzmeM5;2ZVPIUQ^YFvG||naU)-Fp-f5V)i-$SteeDVBy7TYe|FuhJxc**6Wc^+`98P-W9ymU~uHnd%&cV@aun)nt_zc{$tDn(-wSWVEoH>p8uEsX%T(~ zi325Da=xxQ^F(36cRgmNGrdeU>$!3+*njK!#wGCAMxQHb!X>A-&w7}cY6AXAFudNf zrteP^8>0im)-@dUJtyW&P+T_QtH1(Iu|piU85q9xop__bGgIm7wpL9BMm`2ccHs${ zbr-%eFz|Q2;qK&Nw_o-)i+ksvv-vHD_p!aZc0l9a1{Q`(3=ESV7@b<6dFaK{U)n3$ z-n?AJ^JUeUcTx+45A!nRX6j0_EbRPw(6FM9v?h=AmC11{BbcdjKo zuhM_)B$A}xD*6_7JPGgz*V_HZt2<286PA&*D#&&Wr)t_e$%IEr2losqQ5o=ey=cm ztuVo3i_TYueoe1_e*MMYb2@lz&p#;(U~4Sk)n^vS`1qISyp-i9qjUWygbKtv*L>f? zWb|ukZ^e>yljFfvD~sE@)xJ+)PZnU4RN~px&)1+j&mwF7w6zS33E%b3HM(5lG@W$f z-GQ%K3Y@`$|NnGw3SZzCInP-rbo<2lJ6sD`6h66?34K^INpahOXD=?S|8LXJn5C~N z`RHHF0sa<+&oT}w@rL^R{OmJ7u~lv`oNUO+&(9Fxpz?8|X21i!Bk$F!4UFU`X=*vV z{CClywqMh0!R1@WihrE0D?4`I@1=i7cHp5l-p(A3B?%l7Ny3Z|Sjr|V<_UlJ@j&Z* z&n>sy%YGUbojLFMCcCzLG;->*WdCIJ;ZygQ5Bmc$)a&y!f30US5fUid$e-kFnS9}y z!~)@&pE>lME&sPYIkR@TORj;w^G6@y|2aWM$&dL$gcvl0_NVc4Yz{iD;J|32XOtME zeRC4`#<@SL3S7Um5zR9i!Ay$MVZaR zu$yIyx{8y0XOEb^g2>6`64_VoWK8%!O~AUCcECbP?Gar!g{e+eo)BPH_!KZJm2s| z{J)Xf#|z%2A%2S+#Lm8D-7a$bO&<4Fu7{J7-1hr~AM_H4&iOSZ!P#(&-%KZ7E0f<7 zLVxa1f9Um?{r|@T&TR$ivTGRJ1(}YV*FL`Kk4=A2zKhP`0F^{Rra!N|Z}b0U+Q@C6 z@Z#2aezpaiw+?)s!pXaHg6&4-rG1-O4t#Q874bfK<^Kc*20kI9v%ON&bw$!Y=t?@N ze4PC4frHQLGyE=>OjZi7*YMc<Y{dsc8RGqunC@KQjZY38AT!?`cCCx@9g^gL8{+dMC@P1`JD4&VRjB8k`0WJ z>whyan{qD`6!cZQ(SED0#qa+)5r!oSDUyd|PcLA8V#3+tFhgvWnaVk3_e1O0-iS+I zzw+mAohaOF{+hCEk+`h!zxjDGLdb-6XPuGfL7TFzFb}V65=WTzncv^thvmj%3*$mAo z#cg*|_jNq0nrP6*a{N-s*WNELEtqs; zVZSm>zT;>=Vr-^wSZr6<)_m~~aIH9tSaEz)M& zmMhOyrgX=h^Zfm5TRVFI@1X?#6%6_xp53=-;__hdyUcHRWrl*rq~%k>o@N+5jSA=b z@y*EKS4G~=Dc^5yG_?T^AI@w)vopy!SIang`I_~%AI{C|>DpQ( za45m}(1kP2WoNEyd^vFHm~s`vjMr26IF>T!HyP(`a25Stq+CGKq8FXxe+0kqts3<=1a1gl&^lv_wt%UdQ%eo%7s(%KW|ulM)s_N zh3dtxLE3B2XYkCQ{&|HlfAOSmTn-=IPCwz>{C~#+?p(ettKTSj2djKcuuksLUiIp0 z$L|F}WmW}imYa4p_XV9gB{{GEY-+^qNtrrbLN+!<9S^Ui`z~|qS>d#V^>Ek;weTfN z_iOYO{+B$&>^4WyL0aB4h>>ZE%KA?$G94^7_^-IQJT1~NXlH3=hxp^yH7zfDW^SThnnx+ za}!^k=ydpXMUH!^tIL5TmFpMIMD=8xP~kbe%KQq$K_-)^M=utzb4)+Q?2(&uK4)QY z)9cBK3mo`czex1=8T*E0G8ev>CwcqQj(L{_HTlAKPFUOBeQf8k^frNs{g)O`vFmO8 z@-uebQ=cuT4qMzUc-Ax5wKku3L9zMIe-x_51Xg{QD)?qmoq9YfgR79wDt;CB zTu(wtlJGb+-Tg zJ}f;xTd^+h1=oXP?)^2zA9r^ja@W89`BBrExYGQ*v^9| zOV7H#<@nvO#d-Eaot8&?pPyp8qv;hco_+U&$88b!YeI|f%;#BgSZnT?J3cl|+#SD| z41O)jm(8@LohyK==M8CxdlwR)W;5m4>t4fJGS)twFQ=w ze6o9aW$N{_zV=I3wk5az;XTA|U#GiD_lU(ChsV1=aBlbMo+frSzfbwz8jkviY~Q5U z(>i}XU9v?p_)P7*j+V{0g0xM;E>`bxj(L=Mh)JT3Z~Fs(+kQFDkbf7p2lD7u&pNRq zQnH@!_3=6O@kh>VUUBNztfV{3w01s>TWHl+b;j1^wv7y<6KK%%SciZ|!5IfZ1|hMU ziU|*$8IlguU@7OmY|G_n!46!Z7Lo$>su&Xta^wEj`q8I&9tTW>)2EtewG%G1H{dZ@E~0S9RW3$a%SScEx&! zMup4gO8*#bc(jX6^4VGrtEVZu7Y66aR7@`R+|I|p+2usV`-cZ+gtM&xns=lAXFMe*9OMfjlKO#4Ju9D>Q<<(c5e!QJ}A*$=4d3^PUng6#-WyzoYKE1&v z(R^Y^Ngs1BvCCX^!_i`su#DT@i#S!M*EdzRi}%mEs}i5wVtDMad*0JX(V}zTZ?-{nlUoI^9>%>BxePf2BvFtJ#>^(r0+oMa*;O zb(;1t&FSZn8PoZ1ZN2Vdd-scI-YUjtY5`7HE5*WBrb;fKT(R&*WRmKoZ2=xD-li;E zRg<;t#k|IS?vp29*%~IFzhmch;hNtIJ)iqkeezUlOflk|l6&x0l~CsVl}{(HnS1d6 z*QXtN7r&{e*>GvyTH)|6ZRNs3Q>%-MHG4B3x)rfjgfjYtMrY03z9?Hte9^BCv9vY0 zw*}_^`1R}FoOOXK*X}*5cU!&2i1F8w850uA*1Cy1RVA)@Gcohsx;LT~FKaqyW-VP- zp1LO}dzt1-&A2a7)en7C4(ZH0wd|vb_GSZP^BZ}`<$mj(SlR2odW}xxDZLE$GPXCn z51qTYAU~6JP04J#sMq(Wy*+lSbgsfn)40`6eIXALQ~jr@H5CfwKEL>cGxXc(rkCFn z5+=OM`gpx;?#|B}RP1ACT`QI?+qg*a@7mnM0=J)CTVT6vmC96g?lTIFJbreox18I@ zwtN-StnV5t4KM$X+;Zi1+KsI6>HEHA<(GS_Nv2M`)>4@0)1OxmTnS7~|96pUeEY_X8n>`Ggu9PKT_W}^FYRbm434bJ8%P8L)9 z%(r%5diKKHX_jbCEB8sCwTr$;n{Q>S+Ue*~+VOdL+RvrwJdXVjGMKiP1-|uq9QmMe zS&9+o+Sj>_uab7o36;3p!Jora^*@?x?T+~x^PZQfe~I7ut7M1Fyw|~;j>^|Le#kEU zsgi%7>Gm1FNm^^ep$-q^6vuczc|j= zCMG&RX|1c-(O=z*F0^_r`g_d#>0-W>9?zDzN%DI?y(#nS)8kVXlXA@}Ckp3ZV_vxW zgx?g!-P{|0Tv^a8Xb^Nkb1vuh?fbTx7Ye`vCx+( z=6;gdNtab0a>{I;XWo-I>6mwMuFmNtJinit6u$~iHobh}e}>x8D^pk1f3vyTd50Iv|40F-H}bDs;3gR{Sw;uUru$N$j$e`lV2|Mu+NyK@bqov zmeq+L|2$gPqI9{p>WN~`l;is+O%hppz361FMf;YsYhU>FYd+%K{PozE6?yf=SqWDT zsj5!b&1}2!P=4u|#gaD*vW=c(`wMudD5FBLj zmpc;qsAhPk!&sL`xpx^{Nq{w>p7#WT;`dNN^}`Woe()4rN@ zggw>Waw%0t`b4}lqp5+&;X75U!%uvd+<0Kk?R%ytnG;SNJsBJtVo)jh=HWcvH>Ru3 zUdZ^n+2OaZU(w`6OD-;R*Z*54G5@J^%9#n@{crbe*}pb+{+b87!sQS18n7SZKcpZ$ z_irJ0=^3L!lld=Q{#SYL-Rg2`u*<>wVl=+rx{pvFt z_vkWRi_zZW)yrw#t6-S+L=u9|1!>O8%Nk|8TC zPA#(kyIO7cG~@S2g5~~-e(28SR^Hd;oAlat&Gq(lWdQ!T9@K}b&r64@7%w$(nX>#n!4m_C$pJJU=9u)idNI~^&pKsOTGj;qNmoGYV{#cdbrubK1I#pK{uIZk) zSFcIusO5(Q;nEfIw#jz)FMSG+EnKtGn8WGNFRjzzO53OXnw>0ISZI6W*VHxKJ12ip z*t50g-s|)C{1!398QojE_0rpr*9pafT=p00jF$+UyRdS7dgTV;mChd34=e8L`Um}) znH()=Y5FlV_1DtX^QyNT=lnW-eWlCK4!xD1BY$tvvyzhW@C`ROcGoQJko|)@=J(!k za>h)W+;so&7T!(e)#^VKzNhSWZ~xx1J7>>s`F%&-)}P}OoSR;_Y*%92(PCT2<%LF@ zQf`|EyKHL@Nip>F-QSe0bluI;`00PWkL?O`RXit)6iFSDdt7G!TirpiIBe%v-7s;5 zwuMzw+C6>Ul`k$-`?*ayi&=Ei=Eezbm9MU;u?C3AnpLS!N}01sUD?Cj_j>7#!x0io z-Je|*4Y|PLJ3+bcNNUDr?|Dm_XFLw~ofx6|sK#5wfjcNgu_=!0b4^@Q?FQx=84@Od|#~eu}C=3K;NiCyL7Wy%tYmYz_zk)nUC2S><`EWFa~)9V`L_-AKb09fFIE<7?1>QXwN4UbED%mKEC{I(sfl29 z+bCls(ebuX_e5#?$pm)-YPV#9mbR1JDhe(u(0;;6g8B6d*pwMA2^hDxf1 zOx0$KODno_e@v9M&~Iu|4ma#QWg)21DQY;OMWRJp^rwi?%f20tD*s)V`F5i3!o==> z+t~sF#4J5^?K$0;eop)>G-X*$(Qwes zw%1UVZ;8irRT;5|j2A6s7K9nyaEah$OPOD0K2xwGw_=LF;SAA+$@!K|Qh^g$H}q-T z>=X}}bL_Bt?TzU%g|mVu&Q`XVax}7SpGF762DTp+GVM22Bi#P0Zu>cT?hE#}C#T04 zvhAo8c9&#jJs{xyVMf47Z=P48h7Y=9zV$IQu)pB!$v-~D{NpU1U2GZ*)6-s*Xtl>~ zaZ3Gf8oz9zmy*ETiy^ce(?wzo%|yC1ZyBome8K4Uaqdrv zPDQCsl_G)MgoQgfIm{kRbTe8iQz=|`pxw7&_Uj)#6L-vv5uT!)p%U^-#IRwqGnXi@ zKxe4I6x)w;OE^Wv8^r21=oeHhlGbYGh_asPq9Ok&6=eQwsh?{w zzvjde9)syQUhU?yrYJB@a;p+9^O~=u)f;HA@bsk_3>#QN4QAAOFfP8irrb#FXj-m} znZ>)5Rbq)M(`L<9`z7M|L4-Ad@!|G{chCI%x)!@Tu9nK0Syv@&SkSZn?wEe# z)eNH-A}2H^v3oC9*4W%Rb?q*WjkPb|)5A!!Lqhr7w(=h)@;%*)8y+kR=VlO@u(jgBF5TCg{$;f_Yo&ia!RnJG znB3S?{##__tI1JODnGi`EfQiC=jPz}u!@mMbjNCWLG9*%1>N^=vN>nY4mq%N`R=uz z583syMN7)#K5#NfbBmhoUbs)YIsc?`_RggzW^^l7$;_45>7KaqvUWym?Gq52 z)?AduyE$;q*8gcAB)0~3uMfGfZcX-9_A{buR#L}9lHK+N_>W}yA zKgPxIxpKdQ_qMEwGin1Em+fSIJ!78hN731x{Zc!2EXx)V4HU~=ux6&zA+?*ElcHxt zR+W0Nbc#C6Y|Iv{KFM%BdWrj;1^x#YG;$c!eGq+pLg461cR$5>u8mXjPH%Pz?6*I# z$J}wp&da*4+v75W)^GS>bB>Y2EbB1ypY_g%b_xDD#&>5OU(bdQhfFsz>(!)*`yCY7 zV$~GpFmLy*wxcWzWm5aoZ?U9*oF$XUI`_sj4Toh$FWAEvHu4d$HxjoG>)wp5tE`U~RtigJcfAVB4Z7OYvw26< zq{fh$4H0K1alR;;_<}t>tIy)p8jDpfoHI;5s@kMCcV3yi_uZYw499wv*Rm}$&)-_Q z;feNwK+f*dhm8|mE!CDr_8S{0uk2RuRyMiS9nyV(J?5fW(?$P4F^@~JalhSWi7Y#- za_K|jipe)zy2N`$6Pe8zS~vde-}y%5PZqQNkEWBX^B1n#^E9(@UUp%LilIcBaO#hF zug_>s+U~3K%pml|p%}L-j~*V%7i-a(S<}4qzwNxV`ZL?zI$8tfACY0vY3{Mo=a)IV zVfB3WITu5C(=(02ZZYj@``Z1LUDKQ;?v0{~?$ipu$(>SxV)hx5rdLW2Z0X&7#V$U% zCM0FBXlxE_Nj%+j@X|(tvAEg-;C~^C)^+#9&P*d#)oTaXCGeW zuU)<2_PTv1_U7jZrs;}JS`>eD@qVGo#@?CECvWamO4Il?yM2FigBM$XSV!C}_R>>n zXIj4Oyq{gnvslj~R$A$qLdTA>05MCRvu65-ZEu`S4iC*R+o7e^IwL6TFA58K_o;J(~spTswcQ}$ZxvWEW2c4^>sw$yO?vR#Gyufx$W{TFH~iY(4L zQ3X=p^)Iq!&uTi8ac1-Os$1N1wu;<6uqOBH*@^y}*Jhj-zVYzSA^${1HLYmlpuU%v zS}PAZzMB}{ccl1s`c=af`(=unZ(mw0(flyXJ!?zzmZVGaf15K`Jr_Ltx7Lxl!p+S# zQrW{dI_qtTW%1YOc^fZ?QjO|1-bbF;rx}VRF#3IzNzYQ5HEXMQt#H22*1A6j98`gF{Y?Xvn$F@}ZbBBWf_3x!0?oa-{<=!C^_ zC8<~I@{T+GIb612hVDV52`ba0yEog*{e8B0qg0H$Q_KO@)DJ6J zHvHE!QcPq2&Mo?ISb$|$UL^k<X*uM)0qGNkG^kR42EtG90ZjfFi2#$U^!GcL*StUgQ%pF&kuv7V}`=k zW-%3pOE|h!tlL_Ym6Syp#9J5^I5MaCNQrr^db8p~id#REWLita1m*MnD!h6w5gU_F zEfHm8$l_vf@?hZ6Dq>_fy6SRQ-Hxz1ze01)H-&hsiFgSt_n+f+_L+=S_`8@+*L;J3n&z`HC*({@JtoEz4^nAbYWVe*{f}2y$HE9~L=H2`e zbi7z|a*Ku$fYJ4fa|o$>th^T8xC=VtXcSn<+Z=! zv(+67--G=k~Goo;yref1xukvIRNU{= zn-}e4&Nu&fY&EyXGKPo3&Z^3jgpdB*#v@>|)28dq{^m(h`D<2|6lXQ)mWX7pelh1z zkJqmgD!IFPx`jVq_;Kt=$>tq8GWxsyQd4}D|1+xp5?RsvMTGP0+%q!yOB`Nq+PvXZ zolNN3-RrtHpLM=0#ThW6O83f!$!#e)c5a1x*B=lLRK9F;RwqXE){2Cla~cd>5)4h! zvkyOB&C3|Ec<07>zNfA;>wnf?boxYeuJT@^RXQe@eN0{$T0FjcI#SDY>WYPChnuH` z<%`;RY;RB%`sBy@P-|5{b7yB5sdrLacJ<_DLSWcpdt$Yri2Tu;BxvdKEB^fM_cdC3Rm z8y-jZJl6JSO1$cy_JQy2%4d6R_+ zzNuDd-`lWD*Ov>%iuruEa#GlNafgu9@fC}?wF~3t?)iCEJ(_#-RfEj8yEMBF-#$~4 z#28akvT;qyjI{^T{Wfhi-FPNHGq^b4J1qCTOW3LZKG~-aZ4b0I(`{Y&gkv{*hqS4Pp1|4nY@}jPi$Stk2}fIAz2$;LyG&i@$UGiGEry! z0souN@89#0zwWo~l#q7sk3PGV+7B9EEK|8^>R<3xy6aN-G%GygZ-pIS@ZyAP=wda>uqLN+C3$sP+5&Gjnna2Gs-5r#x6L*XU z$P{=)&H7P+{awpb`kBXf2g_}&@ZmYn?O(jj>6)Hnn$%R2=sP^AeN_nu3|{^Els9>C`G*y7SL0>8ToqXEqo_zPo&`-;C{O zzu}pL6)28f(y&8Hn?qED=KtTfL-EP zQ(*3t)h%a4w2$xJz4Cdj(y14<8>Z_{o%<%QbM1AT!WBvluU^_53rStZaAfkyNxVH> zOiKb+E_Yq{cf&-+ARWK(|2y_i{>HXyN#I?3&D~QN^W)>x+ zOgpZ!ztBDMs86uX*YdfS+)OvH%ZN%J+!bgr8PqK>7L!>&}7O)1XG>W$W1&S)y!vuU#aq~)*Tl6k`y3#uj@v?6Cye8`C zAuGPAsY_o4F?p(*uh-wPjYnmci+jw%*3h=hT z1XgXD<1l4Of=-uW=xff^Sy>_LxK#IA<*J`zIVUnb%gV;)F3X9qEzguz-@K>Je62TP zm-wM0yt_)QHs^d;t2BM(q_YQ)?MUwV-S^B+JG;y9MOgIZvP0jly?wy@#c|`Tgn2$I zk7-+WbcFZ5oBCSo1*h3Wi$^+Vtkuu0QWid^!LN6Am7UMQ6YKs@cImbhG8Zp>`e!Ep z`r;zq9NW%i@u%1Le!ji=H`Df)A$pq})^1$*VA}K@am_L{YH9Zy^IpA)T9m#Z{@>5j zTc;WeeOp_qT`O08Lh`m8^UGV?TO^skOMN}5R(Q|y_nQ3r6WRWr0naX9zIyr4*^Iw` zuZD(By7a1T?JNi7_cc4?Chakp_U_ZE>0bYizDWCgPw9Z#_Xc66_qYGwvkp5{{AlaH zsPMy?Zbq|rIfX5GdG6dVkDgh8lmk}JwPC%gQ2y?*=u z6<*mr%k~{itM)l`*zJe-#obSDaGPHfYdhn*Rj;h?8R@vX( z!l#pFs@&{VSh*vQQUAmNzU9)(E(>fd>)LX|X~({gs#XUj_dVt__^7n|@uJH+Eg4f} z*B>^oELxDVdYditj%C4WD{oI>tXw0w+4iExwgsy<8#m7HsM7Edo*}zyVq*+PmCSm&~4Y#e32I$<}P$OZhgdL{cqG36#5=@4!0iH9nzdYvcFI`**l=Y_*Pfrh&enVWU* zexD>NnSIEi_^{IdnDsFiHyCH>AGo}A(E(=xrG>`AvooDGTkKr7-f+zg&aHB*cWjaF zJyGPs@o{xclR?1h=@SC?9$e-4_=N5(!4(l#)F)n8S6H;9x_ENA)V!ys7UVkd<|?_y zNloT^qSknNlEdZ>!NPf-TUI9-T7Ca*?>tFs`c22i-#ylW6YWLziL>mE)YL3FC6v^x z+5Bp0QI1up?p0pp@Ld98JE!lNxm0S6uwUe!?N+ObnjG3f zgg@_IUev7byzyY@8vW$gtLJda|5tLl_fcy{hg!MxWVXv&N*-@Gz^&dWG&AUxn3|`{ zhfT+Wi%#X*xNMBlx8LJZP<5ia`}Ah1mC|3#V=t~QmbRU%;=M@5UZDB7r;^piH~N!) zo$wVl-#vT%GpU)(zxX~~Jbd3+^T_YRA*W~itU8$AVtpV+qj8d{`ld6PF~_qUJtqa8 zILWPa?be1XI`-G5Xg>OQX5&t&2xafOptJUw@);-HzBg~4{@L;L>D?zcXf3F+*vh>8 z(=FFC-mA|a+8mlBJE!EprITm0UKuXcTwJ+&`!iu_wf~7;7nodHmTW!{yz5P*?!n0p zMN&wqs%LnFUYhT=>59-gFmILdvY@~Pvja^pp59hzDwn&7f!kJlw?XGm+x=E*Yilg} zm=6AawRUaOg=akiGZ!@L|L6*w<@Y4Tq3fb*+T>jZM?A0FoEJTKZmEF&^3&@NB`dj_ zNZ3i6KPueyB*#hgsoLT!wRb6-<=)QN?`7T2a^;P3_fMOk-G^4a|7CpRzme&`Co3lz zp89t9@W#bT+$qamIqc=Jp0;79);3PRH?z-$8?M!Oc_d_`*|8pe?<$wPC&Bt3Eu{83 z1k72obmbQL#}|KgnKe7fr{tUncp`Q!Vb-PWZM_$R?o4q#v~X*s%93v#I}Yy>@p`5B zo`FG*p}~NGK`fy$>+j+)Z4s}hUUtQnT$5)XnCbK6&U%JY8Nbp^pPAH|M19i|sXc$({9WO;5nwQ+v0}?Q!Z^5p(|O;a#T}h%8rVY%h%v zdcf7u+xzL|!UKfEO4TTIk@FVwzZ;z#zUp#YG_F|N zv0~{h;SRo4tPJ8CBG$cV;I^EuQjGh3Hzf0o6gqIx>#sK1o-T$L!PwXd&w zGhZzzyy>%L+MK=G`wY%~sB%yAG+$rkYIB3$JwsT7C(v8m*E^T6HkqDTmw&J$zL#`JLi`I&65!F2I@NnA?LD~0Kw|U-3`R`h7 zv}VP^-q0TsfyK;w*t*wGsIjsqtb%g%g%ykPpr>9K~l8kfpH3pBf&8EyMH_@HQt$rN3gGY>W>TeOt8 zB?or+#yvde%Xg+Xarp*@=!3TV)^zy16XfUUW@=!twwP|l(4gDU=eqJq*M=tf0rJ}C|~>3Y)t z&)FjT&NOae&qoGPvsvy{SRc!}b>m8}PrR*ST8KiX@hL;8%Y74W*!)Pn&G9VpUzhNX zaE)v!kByAd3=H;oJR7gWgqIry624nxt;rB)_#k(rOuSokp?coLY`N}v)A&v^ zHh8aFy!_nbr=lX;JHr24-f8R=`|@zbzCz3NKM@~RPm*7K{fdU^0^jxa!RC68L{u3V z66Lsi_ua~P`ugtbv%Wex%NgFfJd>1h?3>RxozdZq;e}^~H`6x$Z@52sK||M#ChvRi zUOwwO&L;li!nsoy&z%wL);K4~aDae_sb6M) zwM-|@G?!-UpKrDF$eV+Pt3!>ZE}lFyYBg)Tc;3S&1&n*2z2;W1zk27P9b3cmzN9rj zx)y#*J2S6K*CNYaBI`xRBz}4B%C!7T3@n*tSrYb-magkqtMN$f!n5aPTu0hs_e(3W zSF3Fn6V*!H^*rW<^T)m4@7SuPpP6##cHc~yZCcu|n#|ve@b&odJ?4A1LbthV!{d)f zds*M9sx^JAj^q2ju0vkq(cH8?AHNj4!u-5@6HdLnYu)i#!JPYJVG<9cxb5;{>Hil6 z^X|QqYdrV%@3cdS2i{H5XWMFK!6g^qGVPtH{UYZ%eJl%}eq6L!TDt71M3>&)pUjF6 zcNFYf)+}JrB$4-ZVyn-sl^nw5?+&mrL@iJ1W-apfo27B?Nr^VY=VNy_eqcV6_wu@8 zr_#GOpC0zLa+C$=mHgUv*lhKJikp4wtgr6t{aRkSH~Xentd0|(Z_!B?hbt$qJ}HWI zPW5C+U!=}8_rvVhkN$CeT>AZtZDYy97fc-IcP?J|S*%;>UtafLM*ACGS#9suH!ul* z=PGWz7BzjdH)n-lLGkvxcld9->)g?Nu`WrrGfBJP^Mu+xYyP~C`n)GOCpCsK&v%iF z+?-7bJB^I~x5#bc6`Ov^%!769>;?N>pTw-&QLyrAMuOQp7KXbYuQCWVbTMs_KjzDS zt|-hZoln~#VRiyvx4!&}6*m;zb}l&=pe@0Wv+rYWY1jK>lREvUe>;$q=EPXm7OP?x z*%STl^1_WacV=p3-~ZjQEyBBIZ7w9C5Tq<@Elke)!T6@1H`16gn|Iyt~Exei}SL$uPVE<%me#FJi zmHQ)%{lX`oc%vC`U7hjcx}&NA^M60u_1V3mS>xH_D@=D4_AW}eSN;9-;e~f!FfZq5 z{`|DdsQn+)1_r4c+pJP%m;85*YOW5u!(Q!iGC1v$ajcJ#?_$GbR_U}RzJ)*McWH)k zu9A86?rPLLgPubFo$))>R1~j03lF{((ZFDkwtz#BnZJ)QK!D+J6W>Fbl#UGwPED*2 zI~p=JK6LJ6whil<@!_#+kGOG`jI-eqXF(xWE|r%;40EP%Wa};AbXPnxOR6?WgsFj# zzs*f@Q^Jb2g{?CPc43WI{oX$jg^zb;UbWluMne6}ax+P$|D}I68DzAm=t=NyG(Q=zxWOPz_RV&)j~zAk zk9_oIS@_Hn^G!SQ%XEiF-!aDhIqL$Kf7u#2_uW;?Z`ntuiKXx3J$Wtl#L3e4TK2ZW zYs1^q7M|Px=gP{;qgNDHn}%lJ>ur%J(bzlnp5w%w>M~Jp7`IK1KixQ8xM8Vms+-h9 z(+P2&+Oet<89jn(iFN&@YC(mnQ5-k24%`f%Rk&v7br}zXK(|HJ2O>-6W=8NX7m7Ri z_KJLYh2@dJNsfGGGpo+3o|Fi%u`|8fVteyQ)CBoO+k$$#cYa+Q?02X%rn>i{TY`_Z z+YjNX=Q=%-LSM*kJr)>JQIa~7t?p|;_3c>ms7do;|GQ}9sBPSGVO{~(a)H^|lG7JW zu={9|yy)W6ud@CJm#z%;T)5O_B)@zmKFQOju-3YChmR**V`E%lrg(;bG=GU9I%)53W@rtJQlck>) zmOSw0?ko#B6m8hEYOiY6l<`qMaKihQ6;$`}t*F;gWsYTXNo~EtBe6EX%3k>o))Y^~aOF{jR>K zoy{rzra|vo*^7(O#q86Xr!O^`p5-OCN+VGww)1hN_~9JAiUOfI>#DS$U3qZ9bycq3 zOu=OqD^@T1cw4FahpSG~Nn^&$YxSvYXC)?g#ixm^^Y89jY2 z=e35-Z_`!x>6$y2m&eC`c$_0#dvDjKx4YJa?Mm{|ShX_q>~7D_oy$x!16J`x?WsAG zdNc6EG@qIl)v2~6Qzk#Jvp9Y)>4)(>&g5%(NBGrkp6_%z0y=)Sb58Az*SO{VN$u(^&Z!p*g+KF` zt6WieWXUy=YtkvNKU>%H3mRIUnk=<@!cj@jnS9;LehYc>c)I>|oz!8br6L`#ylHvi zhl0&cm!{g3?^x2fB5}p9+xOK()O77O^@t|P>w7;taC@Uck&sZ>i5rI`HCeSi*M-E1 zwol*Tu4EWEQ~AHluLsi&uAVsO+*`t#9d~Rs&#(TT5);Xnm$vI%sGXR2WX=N~_6pz9 zsL6pAUmldzxLWSYbZr)XJ$u(3iy10T$9?a#@Y((>pWkTvZvOAzYS-Q5Wa47Kw|>9(Ln; zy1wyA;-r~AC#FwxE1YbZvOp&|>*T%~wF%Fx6ZmaS?Pjfoa7JxBd%HEn%=i3ziA2b zkG@vpR}=NFsUS|jYr*Ghj~C}nJZ%sm7@Cl?@_)4G`L74{xHqQnW-$;f3WziLw(Q+C zF8PaJ16Q|t$0;^*Z?U}jF^Tb8f0Vzejms(afHU430~cNinBBF`X`$76UE38l4DZsj zi$2ee{V?NXS00yT3g@|>X~~PqZ)rswip|xSzeC$0DpOm3mVC>e9rwF}&iKY^a%V4f ziw!K*oqDZ!O5fV@6#msGwma#>>6|QCx3Z?7aOuI^D4WQ;yBRIE?0fw}BAHk1CPPKK z?ArP1k3R>r%`H0+`R{eD)|R!g&eON5w5jpy$8O2EC%WlPZ|LtiS+{LJmNl*xdD+9{ z^{(Ll_F4Bc7gzZ|(tg?`c0*~F^n_iJuQv8orc|u)ap9dS_36Ld`AK*Abo!Rf$Shy) zo^!G?yTc^rY6F**pOI{mgy*irr5}HtnA>9LvSY@L{B>U@`I^k`(=7`=$!O4WQpLb; z>96cC^FIf!e||XAx6Sp`7Om`GcOQKIb*0GPSoopM*V&;xJ5GPCT%F~3+(+bl=r#$i z?Aq;46XqvMFTQeO+j;4l!a%{|C#(KR$8OP`U@Cs%%fh}nr5}ZPbJC{g{qHL)L&n$~b+@x*|};PV|@dF#JSyKERJxpVtjzGf|5n_WGbS1%O!{{Nb{ z=1j*ryJKE;a}2Y~xi^)6Q(4liRgz{Jvs`vX;(7NYSIeB=yj^W$`QhH{8D@nNmu9mq z>9egnxzJhl_ZdmCxS%Wb=8tD<-ffRw{AIn{ji+(a_l1u*=(fGs%fF{g=U>{j`_nCM z3cM`wFMSa#94Vcu8f3V3>ij!V>^pD#u|4ZiDZm=Uy8GujH&@YzColg!!^5O>jeF&( zSGv;9U*#5W@kqTAY4NnRn73E=XoQ34T-LKUgn0dxw#Lm{8ljsw|Mv-zi$||t6%jkG zI+givAg5Aj#oV~wm6I+Ggj%KS@U{~{QA zYG$0qTb@!MStnhYWjl{qbWX2L?2ekqbL)+0n}bx!9R>dogSCNtx+6EE;E}*Xt(au?Ds7c=WF1WbZ`IpQe&wt=D85lTz0{V3U$N z^;2qj3%7cx=gE-!R}!Yboe`j5dwKg6iS-ZTXR6A5lns6uIrr_7(BC&BTMtTEZ{-m3 zowI#z*8vsZ^&N9n+K$zC_CC3IIa;>IUiLy@#&wpxf!)35JXt5q|9G{^V1d%L%O`YP z9!#A3d;-^}x9vTm{_2*q{;xA#V)Hg$_*yr!Rp1v+kAm1Mb4x`EbiHrd9+$j&u6NB8 zjcF>1g}05T19ly>Y<_rBf1L8)C0yN(-g7*|gpHZg{R` z;i8GX{QQEcazT+xH_Qy4l_k=<`6?Fn zILwuslym%$rt)i7x!#LW%0I8ZS$TYm@6EGImQL55$~`G|UgO;ZiT4+t5UY6Op+DEl z>D&2;8ysaWaz4CM*0l<<^(kp;UA{4K*7~n=x7G5cc5z83g}q;R|Cqr7riHv0U2g8$ z8)Y3e+t2n&uciFkmbsp(=L2HrX>l$IQ;GktleqBJ6Y*nb_;b^gyMA6PzId+xN<_&< z**}^~v^0;4HLARSp#1TxRLI=3QyKilriuQkoXk;o<)(w=$u}PBE>4_ybCyb1Fw-rL zxUX@GGJKp*25vYroo$}?-%$P?fe~CU1CPADA)3lN;mqRDlS}X4m{-$y{%nodskIlh zF5arvYdbLKYWmEO^4w+F_s&QerUy=%t59_}^Z>WaSHAxlcRVvA9XcO=6b;vT_0W=) zLv+&=>6w0=Nn&~DI;Vb6>lX7o8!}0tOz{`ToCe+NFQ-QPZar7k%g=K$tSdomHe>1p z$G%w~=Z3MShO>Dscsf)27LCqgIy|}p^0kO+>*-tir-E)=aUpX=rvnF z@b1ZH6NQfDv%WfiWs!G913Lr5ycTxnidDQV**m;WgdB5sKboo8co!=aHzH?=K(<;KiaB7i`xR3Wd-^c0uc$<&O z-`|ny@%Q=6r!TghdAe%CtNyhU3{|hjMP%-eI(Z^;jq=j4+^1d=leAjTEG}ugY_*mn z?^R?7@1#>XlP7nISWOby0#CZ`AI%s9U-BzQ1(knWntS)oardmF^xP-)ritb?>EFoZbE`<#AQI zZDV*vKx5?8$fLXwid$bsC_bDUDj4Q)w$J5Vb}0YjuTvXeJjve~9bR@s^$4>9>pzSU}1T9(DLG z>+Jjhrh!X5(fc@trhq$~Z{=K>x!>Q}we}+u1Iy*N zr<%m3En4E!o4&(|;oz*^|6FbDd{IG9r+IRk9OMXg6luto3SGBTNO!kHK^)KJeQUjQ zroQ&+F=V)LQvSZ4bB)7}Uyb=c7~gM8)88fTxGQD$370cRCcO*1+3%*$E41WEYSHW> zk!wpfFft!{Tem5YLH`bu{`;_vcbp2u9&a!Ty_0%~RUyRT=PdmlZAQ^8MRNDo1@N5N z^5CSxZ`~P;CF^|%sG}T4M{RI^v2PKLRib=G}^Bf2a zX$Z_=eBYLR>QAA_7Di?VMrWNy(FHmNv-7Rqx5}US5LNKys}!r~2@{{$Z-mZlIKaTb za^h&*kpu40B|nW0NYrO38Mnw!>GWZCu$tT{b2HU{A;&mQ^g8OmTfr=Bz*IR2uVSi+qlubB6)L zlzhY0`phg1Qcq2kChPN+cgQywCoksVe(u8fLhtjguihTThRvS?jPolrzKXScxtXNL zHRFxXkAquw9Pza-v_5+)L+e!jul}%FUNGUckL;zT3*z6-Ipeml zV`)-+Pj~c^+?`F@2JvO0m)Cw+q$6ig!uX)^&ZVuf+~Tn^*H=&bynNN0z?zZ}=`z+i zxxpXw6qXc=oG3ZT^fhosnJ`aT#e|Z8_eZ0Qk6N{7H*R5MGBIP}C>EA3Fu80fKeeP{ zfdl7-BF+zaijzON{!X@^$tm9|8*nh{e5&w0(}xdC9;xQGS~s6bo7(!oMXj8t(QJ!5f(mmwpgq@C*-q^)u*N-S@yhA1jy}m>CZ6^O!PkVPsgb2;e~i}QS^JrqpIeWbsvYwIrlgn!dE2(_@9rI_{|ba?(Q zDgIld|HtU_rk?61V&R7KYThiGsB&TDsle}B1AlBk`AKi;_aE}i4(9nE9GC0-F?31s z=U%yei-d5Qh>N4}ifuv7f4$3lMrBdw8>uP!KJEu6eK(YUIf=i>?Q^SC z@2=a)l1hxhzE@v%uz4zcx81yN>qGZ~gGJeSj@@gt+&{e2`=Ghq;{7$B`y2GC-Mp4d zZxAi`vfS8Z!eVo0jfUO#^$h3be`Rv0karbw__@4{!}&&M;^bdz_>PFxn9Pm%@;`9P zN1q8hS9R>_VLVX5-NC57HE_d+pylQ9n`128u9zF1D&W2!Ea-W{c*>~|g{q}WRi~bR z`>9rWO!Cmi=wh>~o%d>XuKVOl>xDI3IAKxc* zuMUiG>p2y9I^IX9a96#F(3-*GmK4?=|E%oDG|gqLz5?BU6xu)OvTy!5&FKBC z>L>Yy7Xr_Bs9m@E-B^FRgwf&aG5HOQ{6#6Neg5)FbJ!9Yg9eAr|9}N54?Ejfy(CvuBnPeIc0JJOWm@#4E5Xn{Y(}uI=WH2Sb=~Yh z^^+4kEc4i8XDPj0Qxkr?O*1h2Ok?dIg=g8nm#1&Gd3o-e)(7pB8%&{_dTb(d3a`f8 zoixp6X5QtOtL8?P{EJ-k?RE03V0Qb`%;M8i<Us-uNFrHjjBeak1~(4b76^O`H~964q2)vo648riGLE6+TZ3+-B+Ssr@f zu$1qbFS~*y|6DoBv{!f2x?+A4(|xQPTQ2GTuVUqnO1*39f7DUBMEBa4bs?dfzg@`< zi+#08IotN`x3`DUwN~+YuW23vtGZwx@+T#q+rjG^|N>c96p@7tuHmF>(zDJ-=gb7 zo#n-fT;pH5T?kp|CQ@+V%(RezkkgM=b)|_0KHKS`%=x|A_EcJ&MDmp@+jgaS@6R*7 zdtM{XU1-t|pCw>|r~{gg)X`5kfFLnb_5XrUUlWj1$j zU1{m*KbLjB$C;Jfntw9-OWF3mH^JXe&boiiYg@#gywy4DMK>jGJ7O1~`~Q2}wA)!( zE5#*@MXx{S)1DR2cHp|MPVST0et+VEZ6>=eZMe94+Tm?i=We;UrhbjLzx+n zd3sJ(m}}LlaMIB@cWO>c}Ao?CN1B>A?~EuSA|eQ(!<=iI)xJ3x8E_6^(~&d!q# z?nu&G5ZRkD=k&B>n-JNH7j+wD9{L~W`EfA9_f(|+te`V8GZy~I_0W8j6?uKvu_^C6 zc&jZYZX8*Bk(uLJGR-apDs^!@?;ehj^r8~rIeH4#QUNCvadXY-W ztCA-Ig8$#T^!DGC?NuMOlGB-&ie!3j(BCDLF7(G&vDkgVf{hpZBhMr~<%_g#Pn{S& zZFV>dBfIOVi1k$$7DceewmQ95{S=p9)%$79imB%`Ud(J*v+TKu*n;Dk>x<8yy0%U2 zgoad8Uus^<47uri-}_SpT}1)(X^FUP#J z@^asH=X65clygRvX6L6EzK$zXdTP^na*9Y+i0;STN+177E?S!B(|(FeOYHs6gVkcD z(=H~-yDXe~Q1je~*k?BIo+mA|Ev8W`O*sddb=I^6-Td6LzWwVN z(TrDLFC1&SlH2-e=^wS1r;b#nZ{2j{TH6vMAGfB@s~63knwQz)_(OA5u~vx4?p;<> zRZfMNtQNiXrs=n3`MrrY*WPkH*0^pFdd?#+AX`f3E0=wiTxRwA%uGreggz%c)x{+sC>2BLS5a#>bD6Le){cKlV<2WsdWAmhMB8Qbd}{cL~|C0rh4taSa>0B?nJHW z$B$bTdQUjLI%eL~lw+FZrxvWpox|_1dPAm+-($;`PY)WRul+wBdVZ=&)!SrcvyIEX zo!s_$>b!MZ!(LtcDYT+W@8Od*PqcNe-df%G>;CTZ*OY2E^UU0Sdg<4^udXI+o1?zo zEOOXAH#G0^sXn2km;xjB*()W5j?QoNW-P6`*pl~0lV2@;_IFKNN5%d!rfTKgz4IV@Lxxtx#Vq$3hg+(jO<}$_^~%}AE?>So!H53$ z?0KhB5beA2YWKg&oR`0TDb7DSE06bXlg^x{EMM0d{fM#ZHfSG;o zy9>#a12jMLq?kSPh^u@0e+ld3n0E(OWX(0mZ|Kr*S6SM7&F6Dn{?0kiO1pZueoHTr z=-Ihq-JM%G*Xr^Yb~OL+@V!*=rpHZhu7}tEnCC@iD>HOn{yO5*S;0Q7Xr*ac`WK;& zRT|xQih8T>{XcukiGS^m<4ZPq&uU{ou|sdR!TDKx^!Dvo)zRl}Y4pN;){58<%M;`{ z&rkV~+s3()X?@Iv4W~bx4lGG4U-9fg7yrtvlSjEyR<2qZW8uwXsk7)+|AuM#T#U=y zHn??gyxK41^75Vj)HdA-pLnjlaKE-IiS;(igAK}kSM(1s8D{S~nfXg&hS5rwS1B=h z{EMzF3*O-7EwyH9i}%VC9jBN&1U4E>auY1&)0yhEy4>I-H1E`&bkx?%s91V zR?5qsf9KcN_5It&Ha+U%85^GKZW@`hx(o7B|F45QgAUPb+5|DgHb`B1o&^K94S1u!LE+$;iKcK zj9ed5UdZ?dOgo+bBsBY5&SEF#_0Rm9>eN21IrnYdij8f7b6AdieUMu5?tIc4 zP44%ZssAqBujuDcEnNN1``U*S7hE=4_8c$HQIGfe-+Lp(AoEA>w>fIdzNlT?@uW9>&Fkf6FGb|JmRDpSX7PS6^>$9crllOm zO+i zt+_VKlU4iO5uu4|$~?t$58F5!-oLbx;q~$J7oK!pNp+4cT(YF2_0Ed0YnO`kCWk6% z_=UNI>rGna(Z+q%#g6ysb(zUIe+>8O>A5$oUvU?mU<<@5dIm%NaXfg}n4CJd>9#viseimX-BU zNB{iGbLf44`NH!jx6&`fxZd7 BRg;R_Fg`(NIt_3qM}d|hhU>;^Z3M)sM9mIbz- zIwJJSJePe_4XbagyUg5=w@ThD-?Ujm@=QX@N0-17ojPsayQe=nZ51x|l)KDSR@7~M zPj7S8@nr#%wfAS|+2n<~XEFW1Vj-?pwk7q-K8@esyfWg7ijQp#ea7L-By4T&DboA@ zd8t^+?KyouQN5*`4D`HQYm**-K5|sRf88fBzi8Fg_dEPbIyP;4_nJLfyY122(BB2D z;yJgc6f0ijFV^ZhyTyC!uM{Q6rH9uXd6lB){w`yw--@3)A0C&O&OgQe;L8;2>HHQG zPkZls;+@8uxG&`Go?{b_WnN(`R*dBDxDb@H&Ns5N@6&na#(S%`0lXf$OGI*OZNwy4?8UaoVGU&G`mIrf9Wl|qs|)oVWr)2PEDgl z+wwEI+6=@0pJUtp%s1VoG>Q986>z zd#0$-;{7O0^^5D5vg?jZO7-~-jxOqsKjrgU@3gDWg!HNWe=FF{a~E?QF3WrOx;`c_ zVkevS*Fe6dt1RyZ?shu*ZjHIaWnX9cw|{P|+M{P~btUL;M@eycYE) zO)5JY4A`6vzHB%6v)w>rLB0M1lO6rfj7se8s|%bq4DJ6Ocy(LSCU%#+T&++&FLuKc zo7BLSdLEWMOn*QBGimc(v_(O`^mv+y%Z(D#2w#nUlFdXhJxHDl`fDCtu+-A`WI zY?nT{JWFE>b3Rjh|6%0jr@|zOnrnW! zR!6z~4s(B1`Qr1VQrJX|!I5i;L%310RnCH`f=wNpdbuPRA`O`JTwIQbJ&#OemSS&W z$yls+mhY>JztW8gg|Zzpypx|YESON69HJh&UB~dX53k6_X;nVD_RF7G^YSrdY>Vic z+53E2ghb>-Y0HU{l7`>P)oOUQiu|8CBO;06PXc$1vf8vQ^$wC|KU8-vU>4wN;QF$) zE@Vw0Ta$sxOuZWtF0U3V_S~>nbF)w5sr)`gRgNiLg^B6ff$E9?+ens~+AQi;k5u(i z6ci5dYaN)Nn$*tjr0R5wp?_HqgXT{W9`)0{?5>q-k8oUmci4_4a{124iRz0c*6vd2 z*)g&AhoN&<_zbKxv(-lldP>hG`D=2WF|)B$;`Y zwbm__I(Vu+Bmedt)3of!^nGV;DTph7I$W3gU%Y*P<;<5>9g;ie^hZqOUeLkmJooUb zmlNL3E?^V*)4->;Nb!iC$_EFQ6|1!F-b$F! z>Dp0oh*^(idok0})(7jI7jx?EVhCBp&+t&iXtoK%!im>^)rD4>FgmPHOkUSJ({zU9 znx9U4Vw02}S*h4aI@J7jee?OphwPRoD_nSAKhyFQmgU?uG5j-huMD~c2@HLqIK%( zZ;KTRPO3IB3NSb%$xkqZN~Ng_o9wFsB7AL>9fahR9P$W#VCp0oYT3Z-M)H5 z!De>Bl0*D^)!#X6W@Naw_|`4WlxvI+7HK~zt4&%io5bm-q<-tb`OLYu7X7~yA2m~L zP0YH^n}=>zY$$Z%tU7c)^pJYh?XtqAd;9NhFK=@FvP+vGk^7eQ`ocTsPgZLyJYX+c zbb0^pOU$*GG-dZ5t?ukTciyOH%br3V55KrYX|sMlS@$KQegaE*snzmgCV>;bZyf(Q zr}f2uOhg#g#4qCQs^;wasXc4XB<@PyfFW|Iu_D9W+{H^mlb7_` zsjvELud~lh<(Yk5fl9rGn|hYhy$h=RQzzU>y(Z$o=uoS4AX0Tk?E%}}?aGI0?ill) z7t>hyzvh7VqSu9gnBV=nzj~jtLc+uNJ$D}OQ=akfg4!dMoa)vOF^myP=O5PY-7kBk zkV$c$9K)8l&8dGcG4D|PWBbm0PDA9Py<6{^wCp#WAoudV)wP-P=CeCp6@PT}W{pzF zSJqbc#M_eDb;mBhIAf^ZUG;fgl1cPa_B+fU7+5(Lx8Iey*=jNIW(}{>XDxf-I-Y5@ zeKkrG>?b~CxLsNEgOw>VOJ0G&fgyqQ04oCngQCj|_M=Yr$zn_^lnQ-ZI>l77SQMD} z4rvJO%&GZtL7733TQcX3vcN))7G_Z|9mmuq#~K;l-00bqy4bTfTF>c#<)jyG-Qu}k zUz#r@IW5x@7ZP>+FUlv=9TK~54OhEX(DKODLA56XS%Me@_xEkgy7tm(efI8cy%C{r zU3ZsVytQVfh@!$osgFHN92gxQwl;}ZJ?OZ;_Uf^W^KqrO1sU%5%87aFED&cn(aQYm zQOJrSeYB!h!<#RJOxy1_w3e55CHg&rmQ(2f2 z6jfOJ^sbZVSCgP1p~fWwfgctBw=WFYqN~zc;OlwiUXX<5)njR*XI2De*Gs;g2(=dc`FIdx9CRjN{F)k%}&G=?i0De+tnr^~xnyjs4XPM`J1)KIe& zt_~k%F_pjwPR*2y%f3vMdbO-=)~pGl-MLY5Asm%a*CHYfJ2G19O4Tyl-np`Lh?<@( z5!~>3;f{;TJSN_nxN74vu8!a{QH;}FXBu*AI_(jQp1j73b;bnOYO&W#SE+S%KVI6< zq%9u)^xDkFOXTLMX6z|DDEVOC+puqn^SWIBcTRHYR!wttPI&o>o(m{aB$}5 zx-Gj(FFKlNEPZvpJL^f{kuOR|%l3#Z|D(0{&pYmECmsCRO3z<^TJyQ1+{r64rEZ#c z@l_#1fhASt>vm6=)5Kl(Ju-4H%!&t{!hYIvnQkLTbt&PT7!7KmA^ zPT#fV)6S~v>yMkRd^4r%-i`$tyLyCjyHndPvDUWu8eR%i{_nrCqsYrvy(9N_xnX#8 zbLp1r@xIx2PtFnkThme9=6&-{b5Z2#`Llnjh(D24PksL5-`wqb?{iImEKAjhdA85Z z)N;Lb_Y}wL{tSY1K0RR9`&T_<x~ZOq^K{2)u4u zbi3;#Q@h^p6{}nL7W;ph@O<83*AKoiNvD~jHFlMx_6C?Zd(C-uc-__53;rpL4<(LV zvX-^ds{650@ov@%bMbSm4zh~#Xa6Y^-#GVpM_|nKV;vl;RVEquC#|0TYQ=K}G;t~jr=Ibm!2L|Og!4YwVQ96_94Cj4C!F>hb) z($Xc}^WMpc>|i#!)DSpDncvv+*y0OI?au7xzgjW<@aqldrgT1PGFqvib?;KnUsW8IR=>bry@O>!Td%T_6txFc}n*qVbv+7`u$ktvzh-CDTs0huQM>|_~&6>-@z0)_cOsz{zWSOtzQo~Bd3!x=|;;dI{63>OhK-N{OG<;B9eHqM&HSJlAd{j)_Z$WG|f@&jki zFWDk~Y=_{N%BhE*OD<+Vqp*vm$yYVN^5S)cg;AH*ewnlCgi55(=cQ-b5?U9&y_9-> zx0PW{=F}xlt|t0NZ~66B&kfhwD!E>I&FesS3GcEb!3A8stHX1@op_~npn;pB^xD|BW>#F_0r zmT6};D{g6q<5R8+CF_dj-!ER%q@S7iwdrE&bLWTk9&c`4Xw`2KpS()rs`lX*(=I!= zeqFm~LEAm^s%u%bS>4kA&6oH8D_&%3-|xZJ(3R_?A&N{OB#CXY&$L=2<-D zc%td%v*1qj`F=jsDsjs_Ih$2I>*J@Vw=%;| zY~y)#_tEt|0d=2ieC%>6KEA#x-TrZ3zOm8%x+>rIcgs({o@|h$Eqv5Z-G66v>Xx$a zZkEp5eXm5MMCYtuwSH~Z^@6fI)sEQd|6f%!RprmD6_IjW{@~NfjmN^`v^{?xx%O;p z?)1D(*^__nyeKT+ekK5moGd~-^d{$FLXtZ~y-)wTDUZqB{`TybiA!?U#q7RQ`8!7F*P=G>N<@vD5@ zxpkM-xK`-SoU?2DC-yT_?>(NX+Gc+9-p@8W=@-5HQ?gDry?QKj|C3bmo9EG8e;VH1 zEb+0mD`cAexrno6=?Rs*q$&ARcm8bo5XGsU%*eO*?!w&fTVmpZ^z}S+=EdIqb?c=| zWc`Vi=9}}BgjePqw-QR#nm4an{_UoBZnu3W9=YuPw5!nKe}CRf=DO@TZa;D?`=3v7 zso$wPvFEmHbd?{NoA8S#9auv z`0K%9&$Kto-d+9ccI)YGr|+S~WmbjE6)!*A#O*(sYkYnar~irExl^>*Vx-M2+Aaq* ztA%xmE0r`BOqv{Aaj4{7a8}3mjN7Sk$)=G3=Hj>90-L4=P5;`+$Dz=-CHen{L&*_W z3&U01gwxWCt`^Kzi`cBuuQL`7TUh&olSQ7jyMjJlB(Wn*M&VyzuqXsF1MGDW%sHYtqb$JC|^C@TkyK4B~ zk3Fs-C}Q`N_IsO>CtgbMQuC^O?)>L^ChH^9l@+}&Kb0!DrbSM5UH>TVW1GbcR{tPp z>(9&G*0N?S`l=zhBk17tkQLt>4}Xnbd?dB#nMVyr`hv;&F%gD0EfN=BDJ@km4BH;J z#-+S?n}?rxTd8v~r)Bo@)aHOxFCnpn1JC>fzN_EAlHZk*yqTr!mP&2h4lAqY1^=!T z>ZnZi@Gw6*y`Vp$VBz+*?~@(FM2mRU&9{l zxAl7&6&h*J{w&QnBVk{tW<|5L(08{6mV!TmPK9AL@7>(i9B-SpX;SLf`dy~Qwx))S zq6J=N75`bcH{CO~y7p>8|8sD0kvHfV^ldoB~+o!!q zo>3h>adK+R;$wkDR|A)R^$uAPu~)1)=}Y4tH;-3GE2e}6=84wlerphHnKY?!y0t|0 z@*TDNT8#wSW-qx}lIvM^t990;^30XRx#pqTq9O6U7PIDlb-yreW{=sd=jv4xj`o^l z1kQ1(4_3{Uaq~VQIcJ~b)JczR9z&@~O&i)P0;#^NA$}G=Q`P?(%V8Q9-3ZK)v!bMZAaZKXc zF2?T`_2bE;`b#d8Q}zEJ4x7hmnQt#y^?Zqo+s%opscsxWb+aOhrOOiQrkL2T=-P5Q zkkiVWYg%hH(_EIQ-c*ZRhiwU84_2yqiLJcREvq&2$dO7Tj^d}{4chKX$rVvW>>c-( zINW@abLVQOpGHU9&H0Mo7U+doy9Uob{$g(5^Z7fS+oy@mH9FSt_R69yOH)cuq#S8q zR=p##&Na(_diwG!e$na19$^zsA1zj4ceI#3bBba~%`dOZt@U17Nf(yX?{sP1&lxwn zbGGHR$s0=)CS3823ad4#aNPDiVCz+r4Lj{xu2#*lw9a~-_g%@?F3aM&^Xewk#{XfB zUdQ|nO^LXDJk|YIU7eTt(qFyXe@y*vWwF|5i|r59gh!VhY;Wc0CKAsKUbgp4z8rDaeYv=2e2IC$bJ5D2@|fn8 zdY`!Z*;T3nCF1ISHKlXb9+=_JcsjNB#e$bziFHynFH5`TH?J==?$deUF0I|s;x<=H zRZuK!^8J*!G_M67mH{DY_RU<@&)hW6ceOJ{cGeh z7}aR{Q_SV@f;CfJgVY^*W>(Ddm^o7|#^b1As94O`iuwuL=iln8e2`MVYx&G`-zP9s zub5n379;AV!?7;*yPK1C_A=K{o$jhAcFC&6Y5(mvnte`5Dpv3LkiMZpxTEZpYP*U)h~*J0Hoaaw;T_uy-|gG%ZW=|cYPc*m$%iZDRC(6V z;|>UYWQ8SAE3 zm8=lS*}xfO{iEfOnN=(M>`&WM&v5I{RoTltBjB~`&V0_6V;P%_vvqW5&Td*!C!ig4 zufiux#N6lQF@vq0(LNKd38|_qHp%~8W^KOiPKEBFki$Wo_P3+=S)@!2*tz3BmrdSd zwd~?!o3wT|s5`4SnD2e4F-JLNd*kN$8*C18YxLXnEaxstO_dBk*Xnk-`f!xb!B8&4 zz_K}4%=4LE>efHbRNJKKzgZxpL1V=tsTD<9|9!On`cw(N>Sbx`wRjbyvv|pV&h7b( zb`7q+an(jI+>BnBm46Ye=X3~KSAsa+nX6A9PYVeiXv?4t$ z<>w{)E(@*A^?Nxd>F`YL`>bidvHI*6&A+YFAEXusRR=$iYFqPdQ_G&5e9is#)3$V+ zD*iv$FxYeTy6UW*e_cyTEGxdN9BWwR^}j~aROrg1AcmGeiIzZyj=4A5FY0cLK6&tJ zsFH=1(rM$iEvJ8Pu6NyX-Rq#`Qq`=d6VHEDRXKTGeBH}M<))$6r=IZ7U3&fVF4Nm7 zB_}WWuvqQfwq*OuYm2s2a#_deEV<^hb++C_$DEkhu8UgFHVK~DA}7^!qj&F(^R*JK ziV{*m5_1^)u1bpDd_HGK&pA`YeNP^SF&xld&6>^q@y3@#5wX*O$`3WQ4{K})Vwk^C zDaUHcG2hr<;(JU*MXQ$8*e)$C`|EWx{qX!{vuz$Ad>Hd!2W0{6F~chU6y2h-jaCHu`3aN8PMr%^bC?1f_X`BBTYGjr;_ol=MD2 zcXKaP$-Sb=-NBq`uX$UQR$gxnet?5SF!Dqd^3_ZO9*WTXA{imb-s!+Zv z^=HRTw_bq+BcY0(zyNUtu7i^7ftvFc8tGm(3=3eK7sP59^ms>+FgK%`TmYj;DcX4nZlShwMg;Qf3=jBnxGH0~kX}Y#R??cE%{H;3(nBsb1@XMJX8yMRZL z?rlZY8GH=kObiPJmE#|Yf1G%jBaq>4q7;JwbKL=Lo(Jq+i{#V-?{77f=i6}S>_UML zff8Ct8bXJ3i)W_H7q3>+o@LH#y;8R0U)1?$Q+I4JW^rGqm*trE_O3;S z(#(Y#8FfN8c;D#0zactb;`K%WcE`Cc(Wd`*vTTuCFUWU6iSKyy{gm`X zwEjza7f4z>l;3#J_zB;07Du7ZJOh3Hw?d8Y0@*Z9Z9UMZoO@tRYf$LrbGI+l7hhV# zmVD*R#_)Y@8_RdkIDBkbxrH3hYq?~j(B4%gMhng?7D%qwwzK~!S$KEf{fDCUZ;qe) zwl{d*vQv92lCSDE3D_=t*|v#wP)#2u)wuNgi*%DV!|m0erCo1#w8B_pLkA`Tf?&?pn-{Dx=!*Y zmWJmlt<#lM#pZZiTyS(gA7hD5MxY7{Lm-=2#HWu-nspf_Ffp(k@M~Qa#GbZv*Thqk zRtmCn$#Aeg^<0|%cY%n}lr<+h(_Nz`Xm5ACyGAK#iEj7B4EGM_yKy-ZUliS%(_Bk< zZe7oJ?q!kh6SbM44D|!+DkYp3m=}JAe!WoTe6&k zL9S$R2*Zh&p357S9GKF1Ira23hN7-5oS#!e&$R!|dZQuvEpTGC-#XDqhbPn4)NS%R z%IomQf&Gqa+>)N%mQR-ytxkFBbW|~K*S-6Ee*&)mIkh%|NniENDQiyi{}&XZPHgI} z(#kOCwCgr7=HRa4-FmE}>%!KJMvpul7YKP9m>j8kbyBHUMEjKU9kG&)+g_PSA6_nO z%fq$#f?QIr(WZ(+yaIY&pYoUN3OL}obK0K6FHbP-+5A$~=AN*RVxLgA?usL7oAoxV zlwerUr0DwS)tXhaUYu%DWmw^~j`eVfFvDa8CYQah1U~RHEfKk;wQ4iNopmO%vL*p* zSYxYmk2!nuT-0d_ncU}fK;Ul2=5uF$ac?`5G(~5T^zj!Z+XOj_yG=!+Kd-)c!^!sQ zWe54Uno8WJT%1eq2h6(4WIX#!rl)#jl%Lllsi}Uu1h?-AT(#lVB8IMnpDvu|*D+nm z^85crEBb=PRkB9u&=LI}q(f%#2TF!cJlS_53$=M_Ps%vK$xIE_CbZwW-i{@2N z@0Cp6c3LKB%jLN|)dEYK>I7as-nDF1*KRE~3(dQY0uNT*-^n5^!oVgYal&8UTINLG zrV6VKYcl;8Gu<-a-7JktseHhXRo+%GG9G*Sxh7_)LFe~gVz+nEf-I{>bO~|Jrv-&ZfYq+ZUX7hlSpF;{U(;?T$9zFB^Q{8(({~TiI*EqW&|s-O2p7HaT&8 zI&J>HhT&ZP>cg_3JUvzyj5c4$VP<|Fy@ffl<@;K*)lr`3cYCH7OzJ$H$yPRL&57L1 zq=iC@Ke$RwS##3RrfYiOr>hD+s##u=N*=4^n6;1Dq&clTbo)bxs>Gw0OB~Iue16M) zIS>))RcV-Yb)ig|mWWt(v*{v1h8+(M8_qq-TrzX&iC3SjE4RKi%5#kPD6-5+!0)-E z;Hp~;vEln4?mXL-Bs^svU&xAzjcNat%r3?Bv#~br`SGGjdymq)oadUWPWnfz-Z-h` z!D9jb-j1x#i+sDIT=ZA*8ibf|-C;l2;i&4hKICa(1gj|Pj2oTDl$<1*SUs2+q<*w} z9XU0RLvrQz;I$b!l_`S%G=h)u?NVzsXzFFQmQXw@a-r!)b1b`i_@Og{|D!5Vt4mtgBbN+O$k}dl(GozTMQ)NKax2lDb`P)6Xf1WX0 zZzL;z=I6w1Uldgz(EeM?uG`>d!ZS2I|wbIm4BwfmnKH@k&BfcLIjl@ZSzm95+NpPIlE zR`@gOSj365$M(E1WbGD_;f$K?`*2GTXR!3NkSjj(P1ny`Xf!M0gPYSVDW6n1nY5r* zCG&`VT{=(h^yhY@Zr^^vz{M>%!{)A=P0Po1Qv;4Jy~HE2UNrKvRj}!7mERW5nzueU zeUB2Y&WdtR(OciYq)1aDxpaTctK!z4okINK$Gr{}>ALB8mG3O70pu%zPd3+-Mdk(B==LX)_=bKm&d+u^{nfcNw5fh zrr_=*+}HR~vq*mB(p6j6PmMmL>Z?8VmgiL7d0d|}T?1d9%=^Tj6!BLf$D?z*=F#=8 z8LA7@T~#l9l;WCH`ZrDU@XW)H+}J0Ei0CCfEDPeQ{A{Q7SUN}EX|W!g>7?YV+QJ)e zdE~{dnRVpwjiQ}ev42+a9-GAUQX}_P3`f`0qk5-;@)LUuZv8Us%PP*yX|{^ocur!D z&ab2{_3Z62|F-O!of*Q}TxEacl=14CJy%v7(4KPP(B+=JtZ@m7kq=&P+q!LnDYtHC zwX)%-v$?+?^~@=J5pFe4U|0Q1p1#Xk4^F(fXgFo+#~IGvQ|8PwYY>mi?6~tkLsvAm zBf!=-sZ&Svu|4~yGt54ZS4#Z7`eR)qBc1_+scrrY*f$M|_^{RMlD8v@J@U`&@ecT)l^<+)@*3KRqvZ zQ!`~-cJF{-?5TF~o!3tv)t?`~eed44n{(&huBe&5Bk|1KT_(Aq8&0uXrOB%AI9P67 zsl+;C#sBGUQp*n?*4}eE>S@zE@dd|I3>Xbn6!#0woUrVk0ax!cCGYU4V)ruLqfS+a z-uv~e3KK6l_4wG=9Zx?L*~LxSTz%c~a%_Oc@-Iv7{E7Xx;Z(paBc>-^GM#a0Zp#yR zlrqk@|2wxx*q~KxiOu@k^MXP+B_gsCSUDn%`ZZ`Y3ygSk< z&8D_4TmI4%H`|bt?D-;d>#OTCXB?gKDR;hS+A$u(%0)h5NxQSFthR7{$YHfvad@NL z!#za?rSEW&XKJ;o@%Fr=Rv;R5-vQz2>{)?339R_M3S1n78+S*ziPf-+_baW zy?m0^x~IDZ<{TE$S}a$beW329;QHNL3qH+V`)P@oivH0T2i5oN{P^1R)9O{XAFf?B zbElNx@}$Ot{8SlO|LIDBxls#iZuZ}gwin)P>U3H0@~SOgvlY#E_aE(A zXx(CU|B;&QB&}~+X1T^rhqw++DPFfO(9}uUSjc4mx67LjY3unXYgBGF+4)$XP5JQ6 z*=sH_9d!3ODD=zu^69-bi%jou&Azv5&GS?Lx7s$l2>h9&ad>@aHcz*b)=y^rcZavP zeO{!NvMDKR@+(Q1h%VRq-_uV&(ybOW^An!3z*~C9W{I`Jhq`WwoIbfZIYxw!$6TOh zr>n?H*N2PhG^f^bZhA3O;Hl)cbH!SLlN}71=JGvMDbrpn+NA29Y}v!PFHl;=?Tf~Q z!;2SLZ-29JLecD=8C?!7n(ow+=DxMSd*_BNQd+kiSIghtdi=8b+K(&rdF*pf zYajcdt|en4>$7xSlcm?{*#a_4qfW~@C~KYDuyMj`^|M`5qn+1OnoO{Gy}Vk}L;1|6 z^_v&<8!cS3`=nUnlv}5F-Z{Cp_qO_Svsk=nigaQA<$y_uR{e)r5fV!l*) zEhp!K%-PHSRoM&g){3ywn3^oPt-9Sp8XPcq$MoHW<1%o z*ji-O%p<`UXYDCoHvP}xD;Y~y6e+AbV|sU$CX0ymqM&15pFMJX1mZ;OJr}y1*3WsDn`SAUymf-vM&jxvuZ_i`mR9ca&bzN`NI1!C=B!!zD0rXfmF;>$BF!dx z0za*tIgeG8nAA@?!Jp;3q*~lkY1bSk(@WmClgvvpDXJxk8r{msU788J%}H zefD(Vve&-Hc#M5&W-PmP=-io&D}{p?F1a@qgUa2$w#Y%@9NA~Q1@S76zZqcso}pHZx0aXuMi7Ust-9i*f5_E6$3f1GgUw z?A{{BGI{NDpS?YMbf*6DZkAbR`t`7Dt6@ot8+WkYl_ydqe=K$^nG@(^`D5eC`btfm zRhK^nizi&&ZQdHJ)*`zjWSI*0Wz(*?1&f+?TkQV%bLP3jfiBIROdOqTylWX*W@s%s zc1TKf{uWD#LecjAu|&y7u@jv)vyz{Sp#wzj^Y-&jZ_xSNvanYwwv!@wpaRRTE-1M{w|N z@O$Hy#kH%g_=tF~?Ec2%uNjYN+?aNYRhD=5&8%IKZJx@TKgeC4$o+3(_r=P&>wV`I z6&~06EXlBgq5ap)=tHqmp_6ZT-`um<;&q_+xw})|=%~4hx`ewdTB>QkerHJd5#0n9 znd>ZTPcB)QC=`4>XP)NS<>fi1Du>o_*7(atEqE93f6iuy#zmr>#uFE;`q99c(X?vE zQjH(-Q@--gU(3zHH(BGy!i2!dcUE-&6X;?5(U8m7z^@>F`)mRu8^iylQ%pG}CqBHn z%|M*}LSM0EHlbu9knyI<4Wfo1b4#$;;;!7wAGZPU+M`hT@jHo15`==?Mu z*=N6xFL}A_y_KApurS9uZgvgX)eb-h|;@2=mcs+R2kIz{3M>*89^y&Ib4?(0ll(DdKpPWAOU(zE}a zUNLcTkHREwb^)oM5_1>*UASt(H?G9tV$GZ$vyI2oPE#1T_eL;p>wTN%AKG)Io zPcAcxuW;_!oOk!i<)W~DFVPt~J9V?!l>9K3t?K&OPCkxXdJPrU?_SamKZE>$f%A)(Km1aHhC*I!x={HciY> zGy8{e=KqT9JtCZXACfnDbg!0c2wHsc9rK!PM{HW(ycCZ-%@BKfk<8M0a>FRtk;pb;e4MiF^@M_qtvdZ$8vR>VaK*SL26Qv+nfLgKZo|J7 z`&Q2konxK#ezk4U-#s2e2X8)6FWz_PMA)MpGb64@*=7;-bVec0iS!G^cG~YIY`}^F;#8DUD>tC`d*(JKit>&aQ`6h zlL-L~Yz1+}nzMSo@iBZ$)7+RnsqAsl%YHR`PUFG`4G9LFbxkkq?ier2o|_=T#~)<> ztoy%F@x%g#|E(E9?;e<*Ja4RW=!wp@x-x_2;8R^;Bb zh^X;m-Cu`-UVTil;$&{uU-#pGhoj`qJ8qrne8SVtz1WeO5fOjyQ$d&fL9u4BCIi2l z{RYxnVz2L)71(s7hRiC>k}Z8ynzi~x+7At-;Ps!UtPHSEwOnHM^c1Uhg>p`X?3|N) z>(9o7`{#PA{f+kTKkaYt@gQf#dS|24OLr?@$}O2|DX0^X;CsI=QZjY>ckc8VbDrLv z95g-lX{`F5r3PQqe7%&~9?n#p`Qp&NKs{ma(7lQ;Rh`!Skx~r3Q>~;Q&^%AC$8GuH z!-3kX__D8_T(co@uHXe<$%#H{rv)v-om?wU$5o!M$j>uwnwl|jvsFUOOmDr}rm4b` z!s2ggw)}8Xd35%$*}`Pi*X61*yB|I}b@}N3?I&DbEmA9=z9-_xg8db1!ld{!mOYx{ zdNQ(NS-iun1A>bJGn41b?9o1NQyO_W{D+6{-IMYxzP7hMSuHqadv2GEd&wcrr`5H_ zucdsiKQ-h%*8Skvr|0Ycm47$hBrWe&?Co-0c5BS1bLUUqTihC@+F}}6v?Q|R%84IK zyblLRo{nf!-M_Q*SZ(YrY5vg7@|NCV_YcgvBfrih;_ap_s-8v0XI>cXuY0yygnvun zx}ZAg-ghapR(jQD99sKrweKY<-Zzu=os{ejY8`3&ZuDd21m;jJf!F6_bJaJxZrE9Q zT!PU%#aWjjwMy{MAq8DtzMaZ(&Z28>n;#3VdpC8T@oT+loBumnU)B5Z__-uA2Lr<~ z0U6^01xE&@2Rb$yPAv`{Y}{Hs91RbhEeg1bWF{9bc5D^mw9AnwTIA5KDw%agW6=Yz z39_19I+~l3ecM<~)kJQ5T6myEK>kpShVjBPGwd6=Om7;i9-8NpUUcQh}A;-LKR@e_ zzmMMM^#7ITC$#(B42SCZ;xoTlRvtLq>uj@U!phqp19i=wz4=sAy6f)M+12J#ug$FR zWlU8_X6b6XoBDg=)*A`QZ`sYg<6Y_ zr6**lzF0Qb&TOlCYWdFgklGHl>lzWa=Y_8=ESZ)OoSCevn(lj{?bPBqHSdlGO?+&3 z@>0>ulcEb+-zq(iuMJqla%;19=FIzxj|YB@E)ZLHRBns@tWud(GW%9)ERG5EUcX)C z;MdC=-d)>zvn=_T}-E_QjD(m1Sphl{jDT_`dSbvQFDu zt7bU&-jdmLS|;ghq_5w^@84HwKF^N5Q!cemW4iO|aN`5I+ADJ6o3nPlKVR4NRPotb z`(H~BMu}OeA7Z_CCwqIj_S~W~Zw{G8-I%fYdgbNvQ#V3RXttU?epdAF`RqMjD?4Y_ z8T_id_hc65nTos7KUJgOX&&E}@k7v`Rn=v0@O^<5#qXuc$~~kzZ4ytgul1~S%RRbT zlA(5+D$A}OwObcO7z@9O-JS5vc~xWd_Jkvf4m11nYMN$TzVlT~U)ERS`>Af0&r7$I zF8$!UMtl0P9`>CVC!R8B))M(CuuXQwlmAaVG#B$9PD&QLVUyaS^tm@`lbP_`ttZ+$ zf+9Mu1zU+6OsmT-ymeZ_L;KZUePfkLCwE!72)~eARU9$Yi2I3o zZHZQNb}9~`u|n3Cq}Df<|dt0a}}ETv!6)r zmH(!jl)xzByDTrTpaMPrBb{HE#K} zFot#QO5M*_1*Xk2>FVon_Bhv5 zVj>nXByW5EHf?ofO2%ZXb^{aE6-kpz6sCOhTKH*Ur|HHb-WaJT+ZM1*nkchmiQVm! zvafkBEjTD8WPQ=)SaWPf-<+zL`&N;&T#B~X#G3SY<%kP=>SYE<75z_Mbj9TK{iWTx zH4j5lC%c~DHR5k6s8YQkBQbTo%p2`nQtuA`6H}XO^=_W>o;A6BN+!u+$Ksa=MdvMi zyexrNqj2u6)P&|$5q??6S5M!&W{;TTq{tq2%MLL7$ zBv!`uw28*Ht)1M|!*ugP#g{2ID|2d#nEhW&V6!>3@cEP=FWn@$?UMKN^wuq`)e4&J z9<=3Nakhul)6(Uon!SC`y0RpbVm4WsI}5FvTCi8qNIm70@Unoc(#}=4%I*YSuPSl( zG2K`$q`Y53YVF?Lg$idCmd~$#`eDhHNpZ0|xC0fwpITyI9PW3t`_U8K9g9_t%$oK; zGwGmlCYyy;=M_WUv=ci^vlia%HhQ(E6!a9E#9KF zd2P3Mp7|VZw{rHpJMP(;P7|li-)tDP_NUyla{;s2XT6=edgnQw%^6MQj%71WH;Wvz zoNTgPZF1*IsVg`C1Q}=9nN7`|o5m(rT)g{TWZy!r8|iD02kFe7>0-?8soEczsk86= z6%lFwyreT3`(h@C7|ngP>}meZsMjICk7cHGrind0&*2)*nX*nzKv+7Bz_bblid-7#J#qMUsEshE|etC>#)9Sq& zOIEr6)?wH*HHIyScgE7S>lX&E&G2wvR&`T##os5B=IOWUPW;2Vsr}l=k`tcKTb8_j z=yv$xU5mRvldqc?^S=`9;>-QHZO#^p=Pju^MY3TUUjE&&w08BkkTC6q*TYusstw&;8(ze9t=ngP$=8wz z-M6ML?pl3$ZTR&U+TRK;?o6?LoV7}NruOk-oy2A3e(&ENRZG7;E77h_`|6fcUfH=4 z*2l{Gjw$o3V?OrbdZLp_i8sG-?EjyauCDlUYU-am?>0POjnA$8J?X)l@Q~y zr)ak+ZF%Q;rswg(stY~2|L)#@Q)u_-gD%Hwg>x?==2#tVWwe{KTrg{v z%YF7R(H9Tz&R30^6&b<&v7LdB|AlJ+!@N-ThddXTxFu;yT$ci?^Ku#g+)RT~~<|R1GZOBmeNFl=)0?x0S-O6T@v- z{l)60_x%-J_v57P#B&E#ukI_ov;3&yjyZuhQiN6iUtRb+L1u#_N9vj5r6Q;I$6R-d zWZ!Wo@x%P6(X?s*h5zSiT5O)F49Pym^4*s z-JT>4B{2nyb_Op2Lp~3$FUi(k$xacB3wa`P?oRpYGcRRNQt8)O?`Fsup%2fBLhrwx_gDYvTh4g(Qbn9hlbBap&zq|cQ&reS85mgpJFq&<>k!~z3};aJ z!s#ON!%<|*u{wbi|1Zi4Umn}o1qCuW1Zp)f9Z1fc+3gU*$S;}dab*b`W0PpW(vL?L z?>;Nc%-|r*BmY)5)#uk@AI+x>KT@2e{8yai=3K`9bz-l&(TVeg57S&jzu9uV;g~S{ zOw!stD#v*ynzc=}j^(??d*<_!`Lzv;Y=b(OavZ%pp89;ckixgbr!6)7Y)T~4Q)R{j z;meMs9BW_?NOL-t!sC-F5~5};HBtK(Ba;Y2<~rq&2nGg)1|g0kY41c`4Gu6kxC$R} zuY@&6{C`ulcO{ihbWQ!mn9IZ@?&D!|=}6kNr~aFs zXE8B|Gc<^Bw8#fAgm^4I8+K^>`I5pb-1-7j^`o1>M>gYVTi;?>{ z&as{Ae9b#Wxp4Mj#yee6+}m4yD_rh>n>%^agMQ|NE`GdS2^*Koi>RhmJqw?v%KRtA ze$_KEfs}@@xeh+dS{5l4I;D6WSrV$I$uK3={?oI`Ob#L%>5Yq$B8^t$sHx^|TkiAo zZ2GsyQVmP9`ksaNIr6Kd&oXnIzG#uCNn^8u=Pa>W))JbQ8NM(n|I$kg9g;J_G?=CR_L-!6CIkY&|*88anWm}?kTH)YP+l;x$s;Fq;hpg}7{ z<3+(DCC#prI}|0~EnJwSBc!-8@U#r?;t8kuBm450?jAlVE1!Au1M7tY1(UvThVr*b zUzA+5XX46`JVzg)=fP~U(r44+m06l=nHmbYGz|FCrJpGKx2?)) zW65=U;<_L+gzHGYlk4=pWe#f^t<6@XUQ$nqkjjYEcF>#E`r~?8%i+OuZz@m&+uGnI@TV|XG|85ExIhp>b_ia&Hov!>|js&B8Y znzEF4Doy>QeOc*Ex2|^1Dm4X`Cyp1?I6cl>P|7aqTQym4wZj+2keo)*h=UW=Tv(r^ z`P^F>lyrVQ@2k%T68k<~;hlM*>)V=z{~pagadc;^fBIj}te?GyKD4!J3VPm>eeC4- z@Yn;}e-SxrxxMB*s#a1F6-cp9%G-M`<=C!8M{j9={-S+DYt7L!%lGPKdRb(+s4n#1 zpuYE$fZwfi>FaLT{?y7)Vqo!DyMc@4*0Lk43U9Cg6n;2v5K9s7edBR%s+iFg z(MMCS8OFw6irW0+^18_UwErfzjh%Ru51qWdG^}jfWOu(w{#zu{bi0^%bV5GpF@zkH z_2{Wt^yr}6^GklJ;tm;mCuJr?s!a`J(28<6&bsE}E+xk8%C%L3AtKprQ95mp99=Co zOj|TLpf^v7>+bC*i{H-_m3f$YYU?H8u6d@FhnC zD|sz!Px=S9)K_dTc%_e|Onb-j{kher?7U9`&3yrr!!o?4Fcju#vivZpXVPIf@HX5l zHKL8BZkN0C9nJF<%N!IO|8lW7ap^MYy!H`NXJy!MY>~s1X$;5vW~&4(ShirU;oQ4+ zydg>_J1nmX9N!z6KhwWg=|Oeka#`VMrT;TKPW?;RFrh-b47*XzTx#FA(7tm+{m40oym?T;;SY%t2(rL4>o~~EoKcDKdDz3oW7yuu6I6Je`yPQ#0x_&9-CLno24GM zT=8#HYsvZ+yn5f}b% zc~O<&_bpz{=ex5>;ONJRDpo-*_HK+9ik9tN$2=^C3*K69e6aSt~BhwbJHY_t?(JEdFPTdsgC?TTW?NMKf87NiY+$t zI`WP!T4bd1VV+*k%oO)|9c!*g>^gZR)Kuc?TB*wtR}v*u7U@ii35;54`k*tlq;uVw zV@JcLn1t=Pb!LfU+f*Inm8rogOF|;QD=rV2KRq}4Yo%#)`~BB;rnf{wbFwde@ku`P zp-qOf?ckaxcdQcQ9&o20oZvERxw&bw_EWZL74iF21PvHp3GTYjbGbM6fA!~&bGH5e z<1_Jx>4LxfrYpZ5iIiENu~u02@wYkeIj5SR{He(r8YG_8VLH9z4dX;-?(3SdAqx_t zZUle(Qt ze-|#ExU-<5eqkQly@w~}b*RmiITpBh?@f0}GwXKw$B#M!W3}&1S=g=pBR`aP^BTX$ zB0nE(lw_TKAtq|-4ap_?r&^QWh88fhMZWb^Q&r@Cc>BMHM3%ue#!UkA+C^j3BBp#` zm-+wn&+XA$z<<#b?8N91*)^CR@fmt#^ z*LHsX*|$^lt>BKi-x&RE;UkQYhS)-cIOKRz3peF z&G1O1;yATf>(B0-r>k!#ACneJEl!W#TB@+T^qFfQxBnAHuBhwYGZc;ql&ocWHH5$h{!`f6bQC59cgge^T0TvjcmeZ9qFeK7Kajs^eJHMRd==ExmQw)5^u_cgKP)*_}aA)2!vk#YZ!m1U0)XB#Ratn#`6cw?=Z}1Lq#; zq(-O@dgui`s%RR8~8aywk<(YpHjM|*cwyh@y|lv>TU zX4dkD9bWD6b~mJqUSHvuIA1oV$nf0)p_#>Tucv6eUFW8$$2xOzMd}i9*Wb2!+w+6> z+-jd5vuoG$>#sU@hPBtnsrfz7xy>){Z?gN@zT3X*CORl<8Q*Vn^51$ssO#L;Cwj^k z^w)j)b=Gppb5p~>k}+d7^+;@i4P?^o}Uebm&S7ttFCCu*w%$8c zWU%n%fvOKSU#e72nC`o=Yr?i)W=)z$PIveOI-Y0Ry~cSD#}=avj}(kg8f-pw_~XJS z87y9=YuIEyt=W5`G8?TPhFP6*;>bSoam|E1r;EJTUh)wL{a43y z>MEnB>KSF<=tm`&_gi!qZ#Ao5?d2(Entj^oYn2z5#t)y*MK%{4Qde#Eh(EK=IHL7a z=Pq9U*)P?!l|R4UwQNJP*3Lv@<@qEFgNX}@gDmz6$Zp&8Q{}F?skh@*gUg3} z_H`aN3v|m*i2VNQ%84{}?#XAnb<6rwkGsUaS#s?~ZHeQLIaQJ;KA+kWw(PD*aEZnL zZ!e^T90JoH3A*KOeIjYJeoyz!@E+%+HJ6uOx;-&CK6*8Oi2C7s{zn%45IuXdvs&z& zwYTnJu~onBP5E$vNj+xTk|h;oyAQp{Nxkx9x!RT6*IEC^Oxw7BBG(Q5go&!Y{)i%7e3v(e|1h&IFS~m#P57qf%yRS4^p_!L@9x;L z=Td)&-r?<`*Ps8Vs!!qZHE!kb_8Xq&0E7k6+T1zTd z)E33Kmqj6>{Qooi17vS`eUTA4zu%H7VuPV}W0p z`B_UYow%v>U59gKX6wq}Ndl!$l?pz-?5_WOig~fgg0+F2iEpFg55=V0FWs=*e{<3m zF3%9n)iYi>b)Vv1y6V`nhkFGbE00`T+%$blvSX%-&aTAQT4^(XZ}I$|JhYJemO!CmJwYBhI^{~xUy%?J{G&l!envOt7EQC zYdG~(%ctDszW#I7Pez@&Z>P+ae<|r$+0+x9ar2&4<<}{X8YVa?{_$I?7AAE)p)g0D z>ylHus&xLO)Q-QCp73=i?-r9SoLrL=vFu9I>f}aU_spi}TSayp75KPjn$7Cdhnpe| zczKPy*Pprmg71}WPS?`NIF|=&Vt%P=t;}xO6g8o0&vUzswAXVdec|_eeC7PqMO|HM zyF(eApY3q`%56Q9^M>ByC+;r~8ZNiFdsA@tjy+5>-*}x^xUh7KmDuvNp^N%1b?$gI z=|GSfzw6zTt6C2Up9$yF^Y~tUsY^l4{Nq_wz^ZELS~aJ*SsDF;Z*g$xV-4OKXI=x!AtnbW4eI-}&m&O8fL~ugyO` z&%atZImkw3;swp?ZBLVyhu)q!{g%@88784CrgV7+Ug!AWF5dYlZlBu1KQbArx46~Q zo;BT8_v>r4IomtyS=H7!$?Xe2rpO=ja1J@WWs37I>4lVZ0k5ztu zOJx+8Eo^c=a$eSWYw+dRnnfj__`NSpx^b(dT~T=Yt{%z8O~>Ry^0Zo zkl=Q$Apd#iZoj&f-4t9=tIlK$~BGE^L3=DP4>qK?44hi7)* zoMhN3I)zu|@-*qk6)HO00(tKgv+8Y;XJSz>F#4Rz{OJD2u+MR~Vkb@dJ}oe-DfmcJ zOsnkn+nQk!+t{bXoo;LR(|7wytgw&K`?oBnJ6}#Jy>*HAut%8g&OJ|b3>~-3_bfeL zw(W|<-0)pz^_wL0V{U9bs=LmbV@u-MH*5|fmugS8Z`u9M>tTWOHHTLD=s8aItXn)^ z)u|+|-4*sRt>K;B0&z_Sg%AIyZM)C#fITC&^NbCXSftv*o4x#;8`MK$)m&}3lM zW;(#7`oU&;QtYdr+_iU}y!&LO|7XcshBBYuF-e~o4*U+vKlSVUy&rr}G?sqJ=>2np z|I|92S#MtLt4cak^k&Ycjuab#w-rx}ue6@9XiuBcp(K*PJuB~@M8c;jy!;9WUYNaS za$V0Q7VtUf{8yhvoO}WddJldxG`w1}mQzfC!}~nnzx9d<`3%|qUurIJ`Z6)LEKp1c z_`~49S?bQ&T5x>P0`BrPk4@hHXM4MrVcUaeBI^A+HgJ5+yE!G`*OU+JeH#?ZCiwqi z;7?wlSa;!`)Qf;QeT5aL_CCt4yrK~mwMLii#LXS6EaSQqm%M*p#=|uC!oyV?_@$fp z4F#X9JO6c?%dLCs75^-_+tbJR!+`5s{*#c6iZTKBT>3?QAF)3_`&;S}pW8;R4-4*c zZBRTAAhz_ZXsJ8*(ht0;7au*-{h+WxNHXAqkJCS?MW3wBy>kov;-L4prI2%r!D)R*mZ;7gH&<5NDcy)F3vZ;^t;YJn$z9^9|F_?e60{eum! zrue?h$~*26_?qDWqtqd$Z~naX7Zs+TeRM2Am|ZvV8PA+ek4^Q~TJK=#Ilx!trIPG( ztWfRjl>@$}D-={$7fM(b{a$uNQ=v#II`EcC(AQaKng2LQon0?l9dK&`1NT&30j>3) zpRH3doTwRoN#$Q)`JaGW0t!6!4R^H++3%ib>|OVc?*ijS1KsY8I@yKyVv>~SU%bck zUc#|o)i+7GTyL|7y1MjoEfu~xj#ZU;E*ItWx>foN1yfJixIFP%t|+3wwxvM$=GhOw zCcG<2=RWu0!5qPVUoI-F6k_<`ck|UoQQdd)PuFpNW%@c}6aTin{|0kEh|RsAC8rZsw6j>+h?%Kfk14;gt2gG zJO65ZiQN~Jv?pAPm?*mWGH<2+*MEi|E(mE(f6IQdNkaUwed%UJ-$Pbc`8PLQ1jd~8Ty7#BseffXo z^2Zt60y zz%X^~uL6NR8EX|cOknaoByPI?UWxnXgb7@GCfOBE@-Nc!nQOq#el?V}@Eg|=y_*}v zTfZdj5b;@=AhxsUNy=7<;t7)5Mf$?dy*J1S?%;2lw>-MA*=|Rn_KD^0msXvTVq*|F z@cGT;BrU;61_qw?lipiD|68M}(rR$+(=(>d&%v)`=Y-ZRahzda)|7huIr?^yR&|$qq|AzG(_YVLdA~BOD?x_2t7M&z z_nT9P1R@Pjz4BWBe~!}r3|=O?;Gkz|-xsz1KantYcmbTKg`^RM$ubVUP z9OFr4=GfARzZ-shJM>lWgV)BA7aD?{JjLeS2Ik7@$BQ<(7qSF}zBux3y_@gihbn!| zdGEtExbNrqR{h6$(Iel|JL$Fp-$Hk-oV11Mi%I|Kf~WVU993Mh;O-0-=l(ZA$vGQm zEcM)?TNt1G_Dktslh%h^&hs0(|J+Dwusfe`p?l7t_l$s;Lvq;dO__-qM^7_(Yff!G zQr5i2z@Qd;81y)KpIy>(sO{LKhH1RVl~(&{ z`sMg0To6CAX8GwD7We&yngQK;zr81lEV-nxDo@CLrxs_`@8--IzUil5_R5?o^v}we ztsTby;_@n<3R}PBAH24o>9%GRNMYkH?0(`tmnW8ejr)DElNU1EpRLIi&^emqd`8Qn zk*Rf|+>cc|%w|4HE4l0%{k112>|9`wcY|f;!)<(#zg7r{zRI&qvEtjT!*h13&an$d zI|9{1@2qmqSo*lH;Pkg2kCrHGW#R7nf8g5U#YVk5)i}F&Iac4v>)`2*2{`rSj>Loy zo*I_?EB&)ZR^|**- zoJ8fFsHUKn+#_t}>1Wn-T0LvNobH#Q+P__*_9go-j*j*tXQjW+XiYnM7XGHI?Ue1~%ofY&%C%9#2WAovB#&s^&Hn%=h zcWc>c@qXsUMQmAS%c~bZyE^T3uadcHSyD>!bIU3H|0iG5y|1P7q(u3n-dk7YBX3(e zXPB~kJz2HnUUEt`cbSZ5=h5Z7e^>vAP}ltNyyxqVHVgC4BJl<9X1HyCT{uf?_X7(X z{=SV}-W3Ms{jHiy4>Fe@=1a|NW|1|ub+zbyne2DuYy7phQ$Jk06w*_9@uwS~sD1O| zQ!+1lT4vn1bYww5&f@oLI=TEF+`BO~IhEzG`v#VrNIn%VkbHiGVMf2W2bWpQ!@oL_E z*}x_$$8*Pw3dyCDe)^SdiR)Q*dWXcCou_PLwoJa!xu+txAu%KM%#8oNw*r;d9DV%e z(G*FRwmGwwe_fnebk?wW_teAP(@whup1;waueDrQvpeWXQ7vcU3(L+j*785hn;9~q zk{&;`$}D}oHc{@>^7dr2mvV zt-d+E;tTuk(${lttZ1LzJmsFY=eLU8?pf!zweFAn-LXsU<2%r-F!#K3I7eJ-Ul77_^Ww+AM@6V$Z56j%d~fIJHh5t z6XLr?DeIit!pi7PX1jJ7u3Fq<^~sNCORUH7HR0=yyqd_=*Bm3UL-%TT$99>wCy&jJ z{C|96*A7`N!N-!zPlmQnklyp?d)aQua!1|1RjPth&y;OfJ=XbS+FJF_{F&<*C04xI zlQ%!}@V@T6<6`HlPHkA4l6UI%%b?Vq)xW$|ZeN^QZgV?p{jL)`EvHT1V>Lb9d*ih^ z>mp`M`m=P2WOv1z^yO=7sx7S#8;W1)OWTlr=dg9h77v@LJhSSyRQX&puDnv1w&KaO ztg}13S2sAdX0PztdEO)Wc=+O3?9uJg>y)&!TDFCAb!5u4G!_23wk6!ou9?noz@^)Gf>z}2;?uWkqs$FtoXLk3J(DE6jv$uVSU3JTIj)HLN z{K%ZSHVIL0=IMq17ciBbz2iqo(3--Bcl%0qOC~B`yZ=?yVeN(8H8-T+zH(!|D8o8$ z@740sOKSB&D;j21ns~W&wJwd#StDc_k~}BprPtJpoU`t~P2e!p{bjR{D=R4L+H+x_ z^6<6|xecSi!5Z>=@@jC17wc8$(%wJX+ z8NWZ0Qs3J;^TNLqMSIUhZ0~;Jvd=*}q0q!g*{%4h_5M`x8AYou-qu)k=Vv_QQQ1g? zoqVq8+itA6z~jipzU6B4%)`GznA_S&sA8 zrL4I5T`f1J#&*Wj>pmVG($kJ^N`dRcQL9wBoK}2qjMnR90g|K1TpB0463kMww^?9y#5l$*#xy@`K{hZV_`O zlNkZ4BbuzVCzW^!WnJoz(e#u3ZM4d7MPm7?|8H(=Sr@UpX>-?!6_-*D9%=u2$v30uEMLAnbA2i9lTt3HRp5=~&w52mule~g;)fd$4 zRSlWk(y?>ty!if|rXjK2vfHJiq~^KDw_bgDOvc-5+7=nlwx|D(N~FEK_+z2>A;%@N zT|Z8oF}X47qgYU>P-0|6$35T5c>ziPRy+#zP}(-5($FnqQi;UVEmP`N-)+B{++QfN zIQT-Uns8Wt+!HCs_Gi-sX4%<3jhq+qwlbuT>72*Q-lZ4WXXNzGs}$|*4ttqiclxZ@ zlTI~W)_eV%`N~$8oaa@|=#0O2G$80^>h)-!)=#Fh;wAYk6ql|ybzfR2WA@@w*2~hF z@-e%zJZH5kE)=q)Y%K|kc)9NFN|vNcja;>XUB3#;H8SF7&Jel2&d+@Io+Z8Y9pLkBU9~B&{e-!0ARmw~}&@8xBX@Rk3?X-Y%FS$QN zahIh_oY@ex?0$(%x6I||L1MCr&+knueb`>NYR$auYJ4jqic`)1m|t{bi+nft?lZoG zOODx{S?`tZ^)6W4Z~E-*<%icQpQSBP-E-+q!~0)b7N4=5RpGJZ?bp8zACGm^es@pu z{WWDT+y66vjyzxcZcoShs?tlj=l`F$);{a`&22v;x=inD-}%5XZA0kY_OPkOvaMa; zZ8l$N^;KxLx5ozfwsJ;-~`jh%bK!miZSmlvq! z{pj|~u?shO^Oq-idZXVHhH{x=#(<<^!@ifNj5p5Rsqb*dYHC^1Wvd68dX)3R- z(JAF7wq%*fhVI>8lR8uUb}VAs+nvf}bfJ3A^u0gVthAmid2zAUqYXP%zd*x0MtpMSu|3*(dgaag&hP>qH&AWMyiFt>DV(6r14x>bE zbtx|vCI`mo6LL&k3nndaTENTWkj-PXOO3;1i(+f)tXbDCOptI9V=6tWvd*ORxJ1X) z#eq?iBYazhQ|A?Nr+;;J@jvFih~dEUWM3V3(bLBpKL~BK$bE9mZ{=ktx2eioQx|#v@C;lfd?Gs$yeRUfpRylEe%p`lyEx3W&$?kt0O z^5K`<%hEKQ{#9}^%x7&7VK~sRsBF^W%PmdL)}1%beVVk`DI-5SnwvxK!sfZW4^s{`SrIQ2a zEtqxB#WHlk#N@LMIUB+iy;Zn)Yc*J^q*Joud{^)aI*Cd=Xz&YqxFhVhI>Iv zHZoWgxUw5~nh6JV%TM6oyq5A@F0!#(C2gNz0amEWP_ce&0q0LHXY8`UJ{NJH;xpYb{kA8^f!(u;)fPid4E7clp};zi$_qA7SEMaL8Qz_MK#1w^{BNY4`iLFWT-oJ`~6|F5Y%oz`MHg zOGfTnu?&a$d2O}+?8S#$RlYy1UdtMxe5vH)6;IpcSy2aTR!poZnO<{kS;XICu2BaW zYPV&)3TvEfQvaEagY`gQ@%N|#wZhKQ7`0-lU)Q62)f@WR7&IL6ABD&qZ}cjdB%N^B z=<+3buE|P5|38WU7M8nw#BA#$p_;7~iZ?vVo+V~aN)uwt5Bluea8zK`H8#O(5$D+$ zCLHyC>sA%j79sgH(Ckv9tRhQNMTXCkjGwIarQcf0)Iu2^wEACc;|&Z75b@Df_10xxO^4HKI9$&x2R zbY0a%T`R&p!{lX!QZ<+Rddx@&eGu> z6#<@QAqEx==Pep0{C7{km|E~ixhU<6d%lUs+l`)CpN-Fih?yjN&e>$qF)?X}x?jNo z@2YK0(a#z`eX;O&c4%No=VNC-Rnaj&wUmV;BWhd2vmI;`j)u;?>U~i$ws1#9g;?PC zu$ToYCI8x-q}d_@6~fpg^Itkm{PZPWs-694Sa{{7;Xe1w zpA<6|QyVjrt`;LXza#$lpG$ic`P!A(nKh(_H1tF-@|$|JNGQC2euuwOskxhY&01Fv zbIp*K<*G?#v1MZIFE<6dyW}?pmj;`CELJkl}=DHm_1*HC!YPvNpv$nV#U8 zR(G@Gj*H%UW+j8oRx>XfKKuVPunF0%<{DsVVzWG)JPCp=~bNz;g z(LtfcNT+p`?#ElYMOKK;SIs#T($#o!x;J~GhlCYdsaF^SqhY1inT=_0TxT8An8SPA z=KmqVW1TX+nc;^%d+Mu(y$TKgrxY?ZRpI-zl4~l~{!8cfJK1e>?9UYTQfDukV663> z#m_KAH*ZPh0$23~rPIAQEN&zj{Z5py-&F3AGUMLN`OhrGHfvZiB>Bf>Mn{=AcDcmx z3j5y+H2D8xexXIt9aHzH&Zx=EYJ3r?2OMRNDY*uglrPyd*Qd#Pi=|A@VLb(QsRyEV zv%l1uZR(u9sqP_r_MIonJ(I>AHQ(#dlfM3qaT7A;?CE^j6I-$^L1NHz4L6W2a22f$ZYWlUur8HoNPmD2gqy3iGn+5h-Fkyfl}|cJU(T6AU=g&P^q~ zg01siKg~@jRCY6;TiLoQm}Tm`#S41499>q;a8)zgyE*9KF8L!-j!lPk_Y{SfsLF^i zEAG7%w#~>(=Zl}?3UlEkhu@c!c@8bLa9o{{RTuaFV4*^rx@zI_i&qxiRyOYmn!37b znf58YcR{AhE2qtMUYvGI^yXp9vS!of%WD=rQEIy^ zzsqo=dbD5CEaRsw%GHA6t2VDZvCwYP534mMLG#&+iHg zjqIA7X?OYROzqDeT*AJEOU%BWa&AinCg&%diJ$@Ds)D&A!1pscF5jXQMbZ;SK4 zoVQI;S$BASuhHh*K$Fmx4VJfUB)?dBn0Y-<+M%&nz3_?Jte2f%gYCCHQFJ(^^X*El zv5Nex73*iMRxSw(Bl|5PK1&5A3V)>=oo#zb{-tg(5wTKx1E$*^N}cc$9&KQ^}y6_H4m3NnjR z-Iaawk!0W3S*0#U!b|)6itS&2a=Nk5!mwq7x22<5*5P|j8$y)L+P)lWm>g>Ve^0!F z&JkyyBd$D0W+Wc*Tyw;G&JmrXNBsXBW@S7Y>~l2o>5&i_MIW7`!EcU4#~gK^6B=i8 z#B`O@#aV|78twfrY6=x)cL_;PsoGU(x>3=zMd`!Q@}A?BbB>reFju`fUe9x)QRhUn z&52nz4=Gr$Feq7cJ=HJMam%ymilVQD(-xX}xSg2Zb6mdli0Ij~%(-WO*qo6&dsO7J^y}#w zTdj>=H@a0NukZ7TlQLTW$dJwD%=w=@m&9x@OYA+*aQ5_{H5cpeTz=Sd*=X(MmL93M zn-3|^@p;VYy(n2$OVTdukI}rC%in!2$jDv^mpx}=dnKaxWPYJVqO-x@&B~uoZwO`d z;bdJ|v#HB4>-a~Tt5^P9j*-2P|Ms%Y+_O=8uNIv>{pr*m{=1f1dnX&Gq*!IlcgeC< zm>3%N_xv`U3y;>^sO-63y!QIu|324EY;Sh+9=Z4DXj`?irT5}Lfm(vImoC|?pJ6kT zt@7r|wP(X?FR$`F(cgR1%O-Z?rtOZ4biHmYHVTnHpXGZ0;^B3F&lldkc)<70O5WS` zHuE)@wEtahepe`wb=J;$b=k9`8%P$_@;k;ydmAGl}DRafZJ2!IfO@DLO$@aw0xu@5@y%tb&KlF{${)N|e$olez zxTe0=diQgK@Yx%;bZmK&z zKJi?4ymZaVmS4N)u$Tot_O4sIZ1Q56&pD4>@7y(9_k>~HW&OI-m3JRSG<+jQca8Eon@l8jTlcoxP>t`=t2n)3kL@66%gW3qIt&X;UYY-{FbIZc)9O zcQP*5JiI;Uc?#dlnt3;F{XJQI?&Ox9>veiBLt+-skhJo1+w5()^O@x`u^U~>0t5H+ zJgixAI*65nDey?hgI6JUf~UV@tXXqkzW24;pJe+zubuC(*FHGj^Y8GHbH{6Kj?8E= zbn#&|WIcK$?(~;CZ|3;DlFwU@WR!<JAW|mxf{>cyVl}M9WBEz7b+cg{ihmc{3C;BLa5q#*1K-3@>he^{|5>CA7Kq( zjQ+rEAHd{v|6SbzhJp{NhPE%|4A?X_u!wyKthp0x?tA(tAA6v9sodO`o9{5l`kl_m zXFR^{W2EfOlK9v5Jl`4LpB4{bIv@7B-2Ua=Fh=%M zUb`<->VJ64fAsIM6xr1HV#?}$Q=2)ToatUzAtV^@VHt1qK9uno>$d+vDhl5nd{{*p z7!RL}1)UO(N`DM; zQef%*_}rgZVEIh>L4Y$q(bQ2G=2uZZ8n{o)Q;GR2r7NC zc&6axpp??I2@_dHhbsZ zUF=-fay#sO+;YF~+gLR(uXH~YvMqv9lSRdOLNB9BBSQn5xR%>_sVxZ)b-Xv3&C=Pq zLGS=SOM%RfhmR5+gx4|)C^T>|u)dh>F(c=z04pPlwP2`OV*|qvehV=z=0`!S=VjTn zK2AuxaDZ9dL0n=>V}lcisFs-Bn>5}P{!`R^<8BK+e6W?dS$tK%ic8P@n%i^hL&n)Kavj;fxS^3@)J{o>JFxlV+E!SJ|Pu8vU$hZ4t{DKA1!GIys8M6uhErGKWfK;(JYZYF#Py9U zVv!d62G@q`W`8y~uy7}J9PBFyY3B|GW@vP<whCk^5e{Mc&LpRs71_o!2y52|Ziv&9sbqTR8ILMUs@CO4I1DnKyaCYw%Ogn_G zMLcfS6(~5sWU=jU!rin~?uHO+jx7v}c4$tzyh4sAh)Gxa?U|QKg8$BJKHpul%6q-t zU#Z8sN1htR6m%EeX}WmKEo;r?E3c06e70K5wDDN;89UyK67L$6Z*tenI@=UxoAHBb zj&Ot%M|TkWk|wQxa>{Ivw%y{2WoME4+C2H#2g!#2Q{t=6@LRBbINUW^Lxb^@YH`zo zrfGE7w85J?9aJxrDQ1mcGKu?qDWX38PC#3p#?XJdkOg za!T^3V#ZCLr9DBym!6za2$MLRxx{#v&(?$HwG)^&<%ruZ{m<}-J^FxiH z&Q27{t8+LgvEwnj^&1wZiWf`P8YO!_x$?m(WI>zxU!5_*_OEAQa})QwA0<1M-+21pzxUwXhkw`@ zj%vKp?2dAl?iJGXu05x~w@^tT&C`FD#d*FDn;7L47hHIEA;C~J;?Rk-Pt5jP7V;We zWHWw9FyhPGY9SZp;j+M8fAcCK21!L#-wF-6i4B~590nP_NiS`DuU?um^Wf4Hkptd> zIgC;1S8o{ja&jpCXsnLz^%Rg4VzyUc*d%bm*OhmYpl58UPA$iwGdzzxG`~0|az=9M zKiqK5ZpSgrzL;jGZ3Wjh-)o#Vz3}OgZ5x~f?<8_^IAljLOSQHLm>B7`s+KbZL@4S! zEZ5BDbhS-rW(sle=Gifoqa|Qzf5#Ptj#oF*^(L49X zx0PEgd}prtXR5dFZleT)Rfg_;oLnI)?K3@hZnySdnif*Cf#tD_N-*1735JsCnj8_Y zeN`lwSr;_0volp+NB`QxdO&$WnoG zhlB+g#8jhsua%fD)#q|qvdMQwM|Snh^P;{>*ts^ZkiT@{_zVqUhQqv4;(tnqs%p7K3kaQo`o%^GnFnibo4zm<`vlFG0DS!vBrXVC7WNYNY9!y=b39j z0VmJ@g$&=PA9%sKzlzIewSUu!la_4Px>xz6J>t1!QOvd~aIFqApOO8uSAs1(4TtTe z8&U}SQ6cI=Zw#mTvHAB0ru7$TUBDo?5vFJu?}5xa@yh+o@{RV%M8YudLU zM#i2;q2Ytj`K?PQ9NFr%p4T%mF7S;wGxwH!MWuuXS=<39&u=xKA|T1kwcvqK^E!?%?3-bok}l!hL)38Ue2lyY03pHYvqkWtHgM%pD=y zCVwLS{qKgg8axR*VrE`rj}dNQTw(Fix53G_IiTr6g~JiO-3=Rs%kDSuGo%i$?yQW~SiZ7} zzoYTo?mcImdA1ZXR29xm>74#QJ#qi0>Xpq7+y_@H*<4(2_-WC6C6PZ`8?>Y5#91rb z&yYFwbL+ln(;b$kTO*oVelg0=T&oqaf}exGMML9Sg(Ta8rf-Q2+&h-7{j{Ump>gde zb$Q1{T00oBwHjKuCiwki;J6{wCDE{!i=W}dislm^h7fy--oZnY0XPUVuVP#X36XW6l;r*F{ zh8r6^|1^532sr30@t(+=&BS8;gVAgyPm9$djRFR=Wz`Y`&dN8(vwOJeSx!t;;BKCx)ta(Z#^~ zb0WI?6pnT+I9{bE~q8FO#LeZ2fa-WrQ$i+7{2P5y$>AubjC+zxPhZhTHbbdANU7c{wcT z{Q2tC@++q{J~_2s=J;wA?zKGLTW54`aXD>Sb$acc*;`{w|4V95GIZeHy2QKE#EUU| z&cT2=;mkd*j8nEu-hJ%k?gf9i_p+ScTXN=j%b61le!iiQ94}NmHS2MNF}$^;D+^yu6-PyuX((+8=ThboZl9}%@WHkweZZRBR*WI{%m_MtW&wb^E6}na$4z@@kI0L64PBCyXRl>(aK=pKCyP~Z7YWthq`&Zxo@P7P z-?jvO`r`ZF)7M|CG~`~$wRV+@48aHIZ8~;L(ICA0*uewcOE^w#ukN;qytb(&q;e1Q zjT4@3aEMsgR8>lK zTZuL@D!8^xoqT86l?T&d5(To&&k*`N9L}qHSN5aamcV?=9vl= ziwPZNnw}dMD$Tfg=)Lg_!yOG~5ln##X9;CA$4{IQ!^-z0!qgR#J`xlcPAXKqlif5F&PdPBqa)@Gk;$87Jh zl|8uY5-j%h^!Cj+e6MQivId-*dr#-lrT>+CG(XNgyZKL4wo;_3MLd`DbhQNyqIy9V zi=^hKUJzjJ{T|!w6~f;fpn7d;uPQ@}21mo%MMCd|1aCG>pHTC_{cY5e-g}&>cOI8s z$$YwK^X0IGJ?9TU;eNWsRHrPe^XmC+9uJoD++4cgrm$Lv(1Ga-47#@ZCf`(b6&G3& zxKiO!ER&W-gW3f#)y@T-pQFn|ZalMnbau*==e!T?^=_%|J0lizZn4S+zqwnbuiVg` z>n|e{cx3G&gJn)!)(ClGg zd2cK`Bog%emF<<)o;(#zJNoxgZP^)P-(WveU#)uyHjImPzuj3V(sQlHclsoehK(zh zK4>z1(b4f{x}{?Gser3(Cv??Zm(5wwz_mc_K5-f%BDcOk$w>#U7i^3*FX zsc}q4vv++TQ1YogA8l$h`xFCdg+`aCIRb~ZGExlCC{^%>}PozweRiM`FvTEx70M^ z*q+?FW9PEZJi7Aa(o&y`vu5pX^UGmk);iqgxBSEFn=u7{&7QRIK6!g4|6SjQeaqG# znp?1~bE=&^RA<_=G|tX5$@Of(*1Qj|d-64MKPKJ1b;8u^5a*Ox z|L$G8wmwt0@X8+l!*U;U-=5k0=Xpx(NAc0tg8AuE~#@Hk5oa>tB9><7}j;1D7Bu*vv3CAse)^(?Mmw8;=NiOZ7s`Y zb15$@dV6r%Zb_RBu8j=g<=z6e6R4-Lo)jg@~$)KPGsHPF8*rDGR}_m zv%B>p-q$SNe1mK81KC}T6_f4z-Zgw*u})=5kY@JrzzL5kraP5ax_=6AkF);yL1){o zCN-IZaV4qww@i31pU>rqE9Q}&GG}RdkD224WNrUtr+l5* zXQp-uZ_)TU&44c`>!(*&MiKu$7u{;6o$CLWMVYb&@K_{qaUb|*Tz+2Zd+ySh{Ht@4 zI(v^@HU)IA4(Gne&cf%T$29l z6I^q?>KaBuQvrl}nj`;rQrqL4i{!WCe#z#zO!3a%`MY3=A2M9r|Q+FK~GNU%bGvOM0q?;|W1UCst`^ zw=Wu-A0J~2F{yfT=%}-^gB5R4j>M;A7d^dnCErygnwOXP&3Brc8KBZ@7EOi&$ z8YhS~B=s`vU^p@38ms5bkU*g&OedbWDN4&1y%DxrlHtJBm&d2c!tine+u^>K!IPAy z_p;gA?D5_B<1q6n-YJj&thj&h`u6)GfBr80xwPuTl-Qik{|4fT=`%$m zqVr{EBt&J2o)B%U(2ZCS6H~c#dMLk=#55)b1~J8`3`Qo2q+Z4k&LVxSu1rzoYgU9X zvOWlR@DYzuV3LcOan*rQM)U#$gCkRuyL?(<<8cYGKRW^$EyWs`Sc4ZaILKENxHP88 ztlh}wdgy0H7-L0@0;?Qb;!Z(1*1(3QDZU~KPLqsT9hbXvJ!ov|=j}K!aYoI%jHv>d zOioYId0#vUWVDe=ztE#pqA5I?+4KT~g7}4s*&LI_qGkuB6}AWmEl^T@wW9Xbb&-{F zwl6NOT61p8jL4|7x5Q+MAI^FmlNH-49url#V&S?yc^uN=8HK6W+Cp*?lj4<+{Wn+Y z>T|rn5~a7!Y*EB9J+Wy?=b0KBn6I>LWsPEPVU6WU+?trnsyvJPK*EK;0%A-IYZw{i zSQ#7+c6dMG7Gsewk)9@-__3RvAtOV@g>}-=*X*pxY@Uy{CED_YrM&O!&{Sy7JT0*+ znniFLx2S{orWJXN0?tzvtlHY;H8*n{IJl@vTYLA@HrAZjxTjiSF}vMnj zyfM^do#*RW3oi%S`+k+kl8x1F;^wm`jf{J26&Akv*wQtPT(7=PZIqu?vF4QGeHPjM zn)wrRBeeC(R^P7JeKum3@-!ZXd;NtmYwzERDv(HG2r*o{FL#csarCWawH9|Kw!T^K zX1?-2Pwc+T{EO^S2l{?*d#T0z`}IrX>^?D3Uk$Zw_m4*LMJH6*pS{*%yIpk4y(3X; z&Dx}1{}YRq_FGqSF(BUJ!G)d2mriZjt9)0iJNT8hv1LsW($+QS>yG{oP3Yc# zwqaYH$1>fuGP~q|c7*2qE)wzPh}*aA)pnoH51v@9ORJhWtvO^{%j^yBOO_kQ+!9$X z_ABw6mVJg#vO=*Hk1cOh*U$NTz0x;-Ze3F}h57d9RbpF~xbl~m9F%IlaQ{c)8Ep{_pEuk()7d;!y_=iTyM3U8IcyPsD6;&5cdy(c)Gp*}>GImf_c# zA3bqGM%y-7tqUbNIx~KMd zal|95ze?ju^m#QAQ;yUir^$!su6@U}W=ZHJeVO2yU$1buv3}~+k@8MHF{$FT+M;(; zLu6!C^TWOG-Tc)3PCAgWw9-m&R^bfA&&fZgsGV^znpkxG$(=^I5|34my%YVErPm!e z?({)(y_4adsEY*=en|`0wqJd7^;(zfxy_tsb8c;ZrulN`o>he#yUTl+5(Hiz+OpLs z#m9KAt)Q0JrqIuh>(@MfvF_yB=6C-uY46UF6TW-RNx;u&mQJ=*lnz51SAc7>bQv}U)=Qnz0AbFN&u zdd@OS&$nc4Y$^|pdT=D+0q*HupoPW&}<|j#$ez@q=%mYzkU= z|JEYmiN3x8H^Z`%7dd@fXmQNQbMvcdzqMYpPwrMtt5@Co%Way|SD*g|$!FVw?=37p zGt2dJg{rp>({s!0U}u9q*`60{ku#@E__}a0cSOjgKGPkJX&06V%AVp_9eMa1bK%B} z-Yu1jgf?j{i&~zm{@_Cw@7`FoThlM!O1}EmyqVqZs;ib6Uke)FJBcS+t>eSeqZ`j#hSR< zTYOLMzl%DW9CKfObHFzH>MuVV%FjoM-@F#J$H>~!%CuZ*Pne{X(sV<;=i%lvrati9 zI^{pFZ$UxRrVSF`*00^8Y3eaeJlo~WsxarZpDg%qtkto8biHm_-kX~{55x&tt!8H_ zD{LzbyB@lvqVagx)O_nBMg7~OZ{K`zoz3)YU|H$)*E$6=cgt-0Vmj+uy!D5N`$TsA z6RX}ce_LeU*OI%w(**AM`=y<~7IPt)z3}ej*tjc9r^BoMXw52}`DWJs%}z$`vo=<5 z^IIXmJz65gY{%Y{mkYZz_CB3E!QsH(p)x z(?0JoXMDxM_uiYHJ*@rzXJ>a@v%{Tc)9F`IZ!_+iwd`i>)w@%dPmOr@_QHI{nCgSl z8&^u%y`6a@GU(|+{{ylLVQ*!!bQLsDUY%(wxjObD`#sUX%qyQa%D&|YW1DoHXH`Pv zg4+#>b5#$<@9yllzE$R(<-Pc)vwo*u`0+O3uxEUR#3T*2`Ep0YpK!(~{!@&9EViyTRy0N z8GCQfGJTtjQP3ZaClZ*Nq TFYG=z&Gc5Nl2@n$1A{dHAANZQ literal 0 HcmV?d00001 diff --git a/tensorflow/lite/g3doc/models/recommendation/overview.md b/tensorflow/lite/g3doc/models/recommendation/overview.md new file mode 100644 index 00000000000..582c5cd405e --- /dev/null +++ b/tensorflow/lite/g3doc/models/recommendation/overview.md @@ -0,0 +1,122 @@ +# Recommendation + +Personalized recommendations are widely used for a variety of use cases on +mobile devices, such as media content retrieval, shopping product suggestion, +and next app recommendation. If you are interested in providing personalized +recommendations in your application while respecting user privacy, we recommend +exploring the following example and toolkit. + +## Get started + + + +We provide a TensorFlow Lite sample application that demonstrates how to +recommend relevant items to users on Android. + +Android +example + +If you are using a platform other than Android, or you are already familiar with +the TensorFlow Lite APIs, you can download our starter recommendation model. + +Download +starter model + +We also provide training script in Github to train your own model. + +Training +code + +## Understand the model architecture + +We leverage a dual-encoder model architecture, with context-encoder to encode +sequential user history and label-encoder to encode predicted recommendation +candidate. Similarity between context and label encodings is used to represent +the likelihood that the predicted candidate meets the user's needs. + +Three different sequential user history encoding techniques are provided with +this code base: + +* Bag-of-words encoder (BOW): averaging user activities' embeddings without + considering context order. +* Convolutional neural network encoder (CNN): applying multiple layers of + convolutional neural networks to generate context encoding. +* Recurrent neural network encoder (RNN): applying recurrent neural network to + encode context sequence. + +*Note: The model is trained based on +[MovieLens](https://grouplens.org/datasets/movielens/1m/) dataset for research +purpose. + +## Examples + +Input IDs: + +* Matrix (ID: 260) +* Saving Private Ryan (ID: 2028) +* (and more) + +Output IDs: + +* Star Wars: Episode VI - Return of the Jedi (ID: 1210) +* (and more) + +## Performance benchmarks + +Performance benchmark numbers are generated with the tool +[described here](https://www.tensorflow.org/lite/performance/benchmarks). + + + + + + + + + + + + + + + + + + + + +
Model NameModel Size Device CPU
+ recommendation + + 0.52 Mb + Pixel 30.09ms*
Pixel 4 0.05ms*
+ +\* 4 threads used. + +## Use your training data + +In addition to the trained model, we provide an open-sourced +[toolkit in GitHub](https://github.com/tensorflow/examples/tree/master/lite/examples/recommendation/ml) +to train models with your own data. You can follow this tutorial to learn how to +use the toolkit and deploy trained models in your own mobile applications. + +Please follow this +[tutorial](https://github.com/tensorflow/examples/tree/master/lite/examples/recommendation/ml/ondevice_recommendation.ipynb) +to apply the same technique used here to train a recommendation model using your +own datasets. + +## Tips for model customization with your data + +The pretrained model integrated in this demo application is trained with +[MovieLens](https://grouplens.org/datasets/movielens/1m/) dataset, you may want +to modify model configuration based on your own data, such as vocab size, +embedding dims and input context length. Here are a few tips: + +* Input context length: The best input context length varies with datasets. We + suggest selecting input context length based on how much label events are + correlated with long-term interests vs short-term context. + +* Encoder type selection: we suggest selecting encoder type based on input + context length. Bag-of-words encoder works well for short input context + length (e.g. <10), CNN and RNN encoders bring in more summarization ability + for long input context length. From 37a0da627ce2b8075c77d6ef576315af7ed8f179 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Sun, 2 Aug 2020 21:43:37 -0700 Subject: [PATCH 0238/1017] Fix GuaranteeAllFuncsOneUse to only clone a function after checking if the limit for cloning is reached. This fixes a bug determined via ASAN. PiperOrigin-RevId: 324540320 Change-Id: I607082ed26198f70c33b86267105f5271fcdd270 --- tensorflow/compiler/mlir/tensorflow/tests/BUILD | 1 - .../mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/BUILD b/tensorflow/compiler/mlir/tensorflow/tests/BUILD index 1fc35f37058..daa583bed0e 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/BUILD +++ b/tensorflow/compiler/mlir/tensorflow/tests/BUILD @@ -5,7 +5,6 @@ package(licenses = ["notice"]) glob_lit_tests( data = [":test_utilities"], driver = "@llvm-project//mlir:run_lit.sh", - exclude = ["guarantee-all-funcs-one-use.mlir"], # TODO(b/162700124): Re-enable. tags_override = { "optimize.mlir": ["no_rocm"], "tf_optimize.mlir": ["no_rocm"], diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc b/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc index a1aed65bd36..6112ff500c5 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc @@ -84,13 +84,13 @@ class GuaranteeAllFuncsOneUse // At this point, we know we are going to change the module. made_changes = true; for (const SymbolTable::SymbolUse &use : llvm::drop_begin(uses, 1)) { - auto new_func = func.clone(); if (num_clones++ > k_max_clones) { return func.emitError() << "reached cloning limit (likely recursive call graph or " "repeated diamond-like call structure " "or just very large program)"; } + auto new_func = func.clone(); symbol_table.insert(new_func); new_func.setVisibility(SymbolTable::Visibility::Private); if (failed(symbol_table.replaceAllSymbolUses(func, new_func.getName(), From 151bd5901aad789d309697ea4ea634f430145896 Mon Sep 17 00:00:00 2001 From: Yuefeng Zhou Date: Sun, 2 Aug 2020 22:11:18 -0700 Subject: [PATCH 0239/1017] Cancel in-flight closures when there is an error. PiperOrigin-RevId: 324542620 Change-Id: I1d6cddf8130df74f00ce7b0a3b6b84f553990e78 --- tensorflow/python/distribute/client/BUILD | 1 + tensorflow/python/distribute/client/client.py | 175 ++++++------ .../python/distribute/client/client_test.py | 253 ++++++++---------- .../client/parameter_server_client_test.py | 61 ++++- 4 files changed, 252 insertions(+), 238 deletions(-) diff --git a/tensorflow/python/distribute/client/BUILD b/tensorflow/python/distribute/client/BUILD index 0f7b7df145f..35d8de95276 100644 --- a/tensorflow/python/distribute/client/BUILD +++ b/tensorflow/python/distribute/client/BUILD @@ -32,6 +32,7 @@ py_library( "//tensorflow/python/distribute:input_lib", "//tensorflow/python/distribute:parameter_server_strategy_v2", "//tensorflow/python/distribute:values", + "//tensorflow/python/eager:cancellation", "//tensorflow/python/eager:context", "//tensorflow/python/eager:def_function", "//tensorflow/python/eager:executor", diff --git a/tensorflow/python/distribute/client/client.py b/tensorflow/python/distribute/client/client.py index 533d5f19042..7bef5e2385c 100644 --- a/tensorflow/python/distribute/client/client.py +++ b/tensorflow/python/distribute/client/client.py @@ -31,15 +31,19 @@ import threading import weakref from absl import logging from six.moves import queue + from tensorflow.python.distribute import distribute_lib from tensorflow.python.distribute import input_lib from tensorflow.python.distribute import parameter_server_strategy_v2 from tensorflow.python.distribute.client import metric_utils +from tensorflow.python.eager import cancellation from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.eager import executor from tensorflow.python.eager import function as tf_function from tensorflow.python.eager import remote +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import func_graph from tensorflow.python.framework import ops @@ -247,20 +251,28 @@ class PerWorkerValues(object): self._values = tuple(values) +def _select_worker_slice(worker_id, structured): + """Selects the worker slice of each of the items in `structured`.""" + + def _get(x): + return x._values[worker_id] if isinstance(x, PerWorkerValues) else x # pylint: disable=protected-access + + return nest.map_structure(_get, structured) + + class Closure(object): """Hold a function to be scheduled and its arguments.""" - def __init__(self, function, args=None, kwargs=None): + def __init__(self, function, cancellation_mgr, args=None, kwargs=None): if not callable(function): raise ValueError("Function passed to `Client.schedule` must be a " "callable object.") self._args = args or () self._kwargs = kwargs or {} - self._function = function if isinstance(function, def_function.Function): - replica_args = self._select_worker_slice(0, self._args) - replica_kwargs = self._select_worker_slice(0, self._kwargs) + replica_args = _select_worker_slice(0, self._args) + replica_kwargs = _select_worker_slice(0, self._kwargs) # Note: no need to handle function registration failure since this kind of # failure will not raise exceptions as designed in the runtime. The client @@ -276,25 +288,22 @@ class Closure(object): concrete_function = function.get_concrete_function( *nest.map_structure(_maybe_as_type_spec, replica_args), **nest.map_structure(_maybe_as_type_spec, replica_kwargs)) + self._function = cancellation_mgr.get_cancelable_function( + concrete_function) self._output_remote_values = nest.map_structure( lambda x: RemoteValue(self, x), concrete_function.structured_outputs) elif isinstance(function, tf_function.ConcreteFunction): + self._function = cancellation_mgr.get_cancelable_function( + concrete_function) self._output_remote_values = nest.map_structure( lambda x: RemoteValue(self, x), function.structured_outputs) else: # Regular python functions. + self._function = function # TODO(yuefengz): maybe we should trace python functions if their inputs # are Python primitives, tensors and composite tensors. self._output_remote_values = RemoteValue(self, None) - def _select_worker_slice(self, worker_id, structured): - """Selects the worker slice of each of the items in `structured`.""" - - def _get(x): - return x._values[worker_id] if isinstance(x, PerWorkerValues) else x # pylint: disable=protected-access - - return nest.map_structure(_get, structured) - def _fetch_output_remote_values(self): """Temporary method used to sync the scheduler.""" # It will do nothing if there is no return value. @@ -319,9 +328,8 @@ class Closure(object): Args: worker: a `Worker` object. """ - replica_args = self._select_worker_slice(worker.worker_index, self._args) - replica_kwargs = self._select_worker_slice(worker.worker_index, - self._kwargs) + replica_args = _select_worker_slice(worker.worker_index, self._args) + replica_kwargs = _select_worker_slice(worker.worker_index, self._kwargs) e = ( _maybe_get_error_and_rebuild_remote_values(worker, replica_args) or @@ -350,8 +358,7 @@ class _CoordinatedClosureQueue(object): This class is thread-safe. """ - def __init__(self): - + def __init__(self, cancellation_mgr): # `self._inflight_closure_count` only tracks the number of inflight closures # that are "in generation". Once an error occurs, error generation is # incremented and all subsequent arriving closures (from inflight) are @@ -359,17 +366,26 @@ class _CoordinatedClosureQueue(object): self._inflight_closure_count = 0 self._queue_lock = threading.Lock() + # Condition indicating that all pending closures (either queued or inflight) # have been processed, failed, or cancelled. self._stop_waiting_condition = threading.Condition(self._queue_lock) + # Condition indicating that an item becomes available in queue (not empty). self._closures_queued_condition = threading.Condition(self._queue_lock) + # Condition indicating that a queue slot becomes available (not full). # Note that even with "infinite" queue size, there is still a "practical" # size limit for the queue depending on host memory capacity, and thus the # queue will eventually become full with a lot of enqueued closures. self._queue_free_slot_condition = threading.Condition(self._queue_lock) + # Condition indicating there is no inflight closures. + self._no_inflight_closure_condition = threading.Condition(self._queue_lock) + + # Use to cancel in-flight closures. + self._cancellation_mgr = cancellation_mgr + if _CLOSURE_QUEUE_MAX_SIZE <= 0: logging.warning( "In ParameterServerClient, creating an infinite closure queue can " @@ -377,31 +393,6 @@ class _CoordinatedClosureQueue(object): self._queue = queue.Queue(maxsize=_CLOSURE_QUEUE_MAX_SIZE) self._error = None - # Error generation is a counter that helps us track whether a closure - # should be cancelled when it is being put back to `self._queue`. It works - # in the following way: - # 1) Error generation starts off at 0. - # 2) When a worker thread calls `get()`, the closure's error generation - # is copied from this queue's error generation. - # 3) If any worker thread experiences an error that's categorized as a - # non-retryable error, the queue's error will be set, error generation - # increments by 1, and the queue is cleared (with the closures marked - # with cancelled error), so other worker threads stop getting closures - # from the queue. Worker preemption is categorized as a retryable error. - # 4) At this point, if `put()` or `wait()` is called (usually by the main - # thread via `schedule` and `join`), the error is raised through that - # call. - # 5) The closures that are inflight, i.e. that are being executed remotely, - # will not be aware of such error event. If the worker that's executing - # the closure happens to be interrupted, the closure should not be put - # back to the queue, and be cancelled with error instead. Checking the - # generation id of the closure and queue is how the worker thread tells - # whether the closure should be put back. Likewise for `mark_finished` - # and `mark_failed`: if the arriving closure is considered out of - # generation in those two methods, it is simply discarded (the inflight - # closure count still decrements). - self._error_generation = 0 - # The following is a lock to make sure when `wait` is called and before it # returns no `put` can be executed during this period. It is because `wait` # won't know what to do with newly put closures. This lock adds an cutoff @@ -415,11 +406,14 @@ class _CoordinatedClosureQueue(object): # of the code. self._put_wait_lock = threading.Lock() - def _cancel_closures_in_queue(self): + def _cancel_all_closures(self): """Clears the queue and sets remaining closures cancelled error. This method expects self._queue_lock to be held prior to entry. """ + self._cancellation_mgr.start_cancel() + while self._inflight_closure_count > 0: + self._no_inflight_closure_condition.wait() while True: try: closure = self._queue.get(block=False) @@ -437,8 +431,8 @@ class _CoordinatedClosureQueue(object): This method expects self._queue_lock to be held prior to entry. """ if self._error: + self._cancel_all_closures() try: - self._cancel_closures_in_queue() raise self._error # pylint: disable=raising-bad-type finally: self._error = None @@ -466,16 +460,17 @@ class _CoordinatedClosureQueue(object): return None closure = self._queue.get(block=False) self._queue_free_slot_condition.notify() - closure._error_generation = self._error_generation # pylint: disable=protected-access self._inflight_closure_count += 1 return closure - def mark_finished(self, closure): + def mark_finished(self): """Let the queue know that a closure has been successfully executed.""" with self._queue_lock: if self._inflight_closure_count < 1: raise AssertionError("There is no inflight closures to mark_finished.") self._inflight_closure_count -= 1 + if self._inflight_closure_count == 0: + self._no_inflight_closure_condition.notifyAll() if self._queue.empty() and self._inflight_closure_count == 0: self._stop_waiting_condition.notifyAll() @@ -484,17 +479,15 @@ class _CoordinatedClosureQueue(object): with self._queue_lock: if self._inflight_closure_count < 1: raise AssertionError("There is no inflight closures to put_back.") - self._inflight_closure_count -= 1 - if closure._error_generation < self._error_generation: # pylint: disable=protected-access - # If the closure to put back is out of generation, cancel the closure - # and ignore it. - logging.info("Function %r should no longer be dispatched; marking " - "as cancelled.") + if self._error: closure._set_output_remote_values_cancelled() # pylint: disable=protected-access - return - self._queue_free_slot_condition.wait_for(lambda: not self._queue.full()) - self._queue.put(closure, block=False) - self._closures_queued_condition.notify() + else: + self._queue_free_slot_condition.wait_for(lambda: not self._queue.full()) + self._queue.put(closure, block=False) + self._closures_queued_condition.notify() + self._inflight_closure_count -= 1 + if self._inflight_closure_count == 0: + self._no_inflight_closure_condition.notifyAll() def wait(self, timeout=None): """Wait for all closures to be finished before returning. @@ -516,22 +509,18 @@ class _CoordinatedClosureQueue(object): self._raise_if_error() return True - def mark_failed(self, e, closure): + def mark_failed(self, e): """Sets error and unblocks any wait() call.""" with self._queue_lock: # TODO(yuefengz): maybe record all failure and give users more # information? if self._inflight_closure_count < 1: raise AssertionError("There is no inflight closures to mark_failed.") + if self._error is None: + self._error = e self._inflight_closure_count -= 1 - if closure._error_generation < self._error_generation: # pylint: disable=protected-access - # If the closure to mark fail is out of generation, simply ignore it - # (with the actual error associated with the closure preserved). - return - assert self._error is None - self._error = e - self._error_generation += 1 - self._cancel_closures_in_queue() + if self._inflight_closure_count == 0: + self._no_inflight_closure_condition.notifyAll() self._stop_waiting_condition.notifyAll() def done(self): @@ -678,7 +667,7 @@ class Worker(object): # TODO(yuefengz): we don't have to materialize results every step. with metric_utils.monitored_timer("remote_value_fetch"): closure._fetch_output_remote_values() # pylint: disable=protected-access - self._cluster._closure_queue.mark_finished(closure) # pylint: disable=protected-access + self._cluster._closure_queue.mark_finished() # pylint: disable=protected-access except Exception as e: # pylint: disable=broad-except logging.error( "/job:worker/task:%d encountered the following error when processing " @@ -686,7 +675,7 @@ class Worker(object): nest.map_structure( lambda x: x._set_error(e), # pylint: disable=protected-access closure._output_remote_values) # pylint: disable=protected-access - self._cluster._closure_queue.mark_failed(e, closure) # pylint: disable=protected-access + self._cluster._closure_queue.mark_failed(e) # pylint: disable=protected-access def _process_queue(self): while True: @@ -710,7 +699,8 @@ class Worker(object): # the same worker such as creating resources, setting resources' aborted # status, and executing closures happen on the same thread. This allows us # to have simpler logic of concurrency. - closure = Closure(function=function, args=args, kwargs=kwargs) + closure = Closure( + function, self._cluster._cancellation_mgr, args=args, kwargs=kwargs) # pylint: disable=protected-access resource_remote_value = closure._output_remote_values # pylint: disable=protected-access self._register_resource(resource_remote_value) @@ -775,7 +765,8 @@ class Cluster(object): protocol=cluster_resolver.rpc_layer, cluster_device_filters=device_filters) - self._closure_queue = _CoordinatedClosureQueue() + self._cancellation_mgr = cancellation.CancellationManager() + self._closure_queue = _CoordinatedClosureQueue(self._cancellation_mgr) self.failure_handler = WorkerPreemptionHandler(context.get_server_def()) worker_device_strings = [ "/job:worker/replica:0/task:%d" % i for i in range(self._num_workers) @@ -796,7 +787,8 @@ class Cluster(object): Returns: A structure of `RemoteValue` object. """ - closure = Closure(function=function, args=args, kwargs=kwargs) + closure = Closure( + function, self._cancellation_mgr, args=args, kwargs=kwargs) self._closure_queue.put(closure) return closure._output_remote_values # pylint: disable=protected-access @@ -893,8 +885,8 @@ class Client(object): function execution to finish and retrieve its output from the remote worker. `schedule` guarantees that `fn` will be executed on a worker at least once; - it could be more than once if a worker fails and restarts in the middle of - function scheduling. Note that since worker can fail at any point when + it could be more than once if its corresponding worker fails in the middle + of its execution. Note that since worker can fail at any point when executing the function, it is possible that the function is partially executed, but `Client` guarantees that in those events, the function will eventually be fully executed, possibly on a different worker that is @@ -904,14 +896,12 @@ class Client(object): by raising any one of those errors, and clear the errors collected so far. There are two implications when this happens: 1) user should call `schedule` with `fn` again to re-schedule, and 2) some of the previously scheduled - functions may no longer execute. User can call `fetch` on the returned + functions may have not been executed. User can call `fetch` on the returned `RemoteValue` to inspect if they have executed, failed, or cancelled, and reschedule the corresponding function if needed. - When `schedule` raises, it is possible that there are still functions being - executed on workers, at the time `schedule` raises. When this happens, users - can call `join` again to wait for all pending async function execution to - finish, and bring the cluster into a consistent state. + When `schedule` raises, it guarantees that there is no function that is + still being executed. At this time, there is no support of worker assignment for function execution, or priority of the workers. @@ -940,7 +930,8 @@ class Client(object): # TODO(b/160702436): Invoke `strategy.run` for user's function so it enters # a `ReplicaContext` in a logically correct way. with distribute_lib.ReplicaContext( - self._strategy, replica_id_in_sync_group=0): + self._strategy, + replica_id_in_sync_group=constant_op.constant(0, dtypes.int32)): with self._translate_parameter_server_failure(): return self.cluster.schedule(fn, args=args, kwargs=kwargs) @@ -949,17 +940,14 @@ class Client(object): If any previously scheduled function raises an error, `join` will fail by raising any one of those errors, and clear the errors collected so far. If - this happens, some of the previously scheduled functions may no longer - execute. Users can call `fetch` on the returned `RemoteValue` to inspect if + this happens, some of the previously scheduled functions may have not been + executed. Users can call `fetch` on the returned `RemoteValue` to inspect if they have executed, failed, or cancelled. If some that have been cancelled need to be rescheduled, users should call `schedule` with the function again. - Note: `join` raises an exception as soon as the client detects one, and this - means it is possible that there are still functions being executed on - workers, at the time `join` raises. When this happens, users can call `join` - again to wait for all pending async function execution to finish, and bring - the cluster into a consistent state. + When `join` returns or raises, it guarantees that there is no function that + is still being executed. Raises: Exception: one of the exceptions caught by the client by any previously @@ -976,6 +964,9 @@ class Client(object): If any previously scheduled function raises an error, `done` will fail by raising any one of those errors. + + When `done` returns True or raises, it guarantees that there is no function + that is still being executed. """ return self.cluster.done() @@ -1091,7 +1082,7 @@ class Client(object): raise -class _PerWorkerDistributedDataset(object): # pylint: disable=protected-access +class _PerWorkerDistributedDataset(object): """Represents worker-distributed datasets created from dataset function.""" def __init__(self, dataset_fn, input_workers, client): @@ -1107,13 +1098,13 @@ class _PerWorkerDistributedDataset(object): # pylint: disable=protected-access if isinstance(dataset_fn, def_function.Function): with variable_scope.variable_creator_scope(disallow_variable_creation): - self._dataset_fn = dataset_fn.get_concrete_function() - elif isinstance(dataset_fn, tf_function.ConcreteFunction): - self._dataset_fn = dataset_fn - else: + dataset_fn = dataset_fn.get_concrete_function() + elif not isinstance(dataset_fn, tf_function.ConcreteFunction): with variable_scope.variable_creator_scope(disallow_variable_creation): - self._dataset_fn = def_function.function( - dataset_fn).get_concrete_function() + dataset_fn = def_function.function(dataset_fn).get_concrete_function() + self._dataset_fn = ( + client.cluster._cancellation_mgr.get_cancelable_function( # pylint: disable=protected-access + dataset_fn)) self._input_workers = input_workers self._client = client self._element_spec = None diff --git a/tensorflow/python/distribute/client/client_test.py b/tensorflow/python/distribute/client/client_test.py index 12152407c5d..459633aca2b 100644 --- a/tensorflow/python/distribute/client/client_test.py +++ b/tensorflow/python/distribute/client/client_test.py @@ -30,22 +30,34 @@ from tensorflow.python.training import coordinator from tensorflow.python.util import nest +class MockCancellationManager(object): + + def __init__(self): + self.cancelled = False + + def start_cancel(self): + self.cancelled = True + + def get_cancelable_function(self, func): + return func + + class CoordinatedClosureQueueTest(test.TestCase): def testBasic(self): - queue = client._CoordinatedClosureQueue() + queue = client._CoordinatedClosureQueue(MockCancellationManager()) closure1 = self._create_closure() queue.put(closure1) self.assertIs(closure1, queue.get()) self.assertFalse(queue.done()) queue.put_back(closure1) self.assertEqual(closure1, queue.get()) - queue.mark_finished(closure1) + queue.mark_finished() self.assertTrue(queue.done()) queue.wait() def testProcessAtLeaseOnce(self): - closure_queue = client._CoordinatedClosureQueue() + closure_queue = client._CoordinatedClosureQueue(MockCancellationManager()) labels = ['A', 'B', 'C', 'D', 'E'] processed_count = collections.defaultdict(int) @@ -63,7 +75,7 @@ class CoordinatedClosureQueueTest(test.TestCase): closure_queue.put_back(closure) continue closure._function() - closure_queue.mark_finished(closure) + closure_queue.mark_finished() def get_func(label): @@ -76,7 +88,8 @@ class CoordinatedClosureQueueTest(test.TestCase): return func for label in labels: - closure_queue.put(client.Closure(get_func(label))) + closure_queue.put( + client.Closure(get_func(label), MockCancellationManager())) t1 = threading.Thread(target=process_queue, daemon=True) t1.start() t2 = threading.Thread(target=process_queue, daemon=True) @@ -93,7 +106,7 @@ class CoordinatedClosureQueueTest(test.TestCase): coord.join([t1, t2]) def testNotifyBeforeWait(self): - closure_queue = client._CoordinatedClosureQueue() + closure_queue = client._CoordinatedClosureQueue(MockCancellationManager()) def func(): logging.info('func running') @@ -102,10 +115,10 @@ class CoordinatedClosureQueueTest(test.TestCase): def process_queue(): with coord.stop_on_exception(): - closure = closure_queue.get() - closure_queue.mark_finished(closure) + closure_queue.get() + closure_queue.mark_finished() - closure_queue.put(client.Closure(func)) + closure_queue.put(client.Closure(func, MockCancellationManager())) t = threading.Thread(target=process_queue) t.start() coord.join([t]) @@ -114,8 +127,30 @@ class CoordinatedClosureQueueTest(test.TestCase): # doesn't time out. closure_queue.wait() + def _assert_one_unblock_the_other(self, first_fn, second_fn): + """Asserts `second_fn` wouldn't return before `first_fn` is finished.""" + first_fn_done = threading.Event() + second_fn_done = threading.Event() + coord = coordinator.Coordinator(clean_stop_exception_types=[]) + + def wrapped_first_fn(): + with coord.stop_on_exception(): + self.assertFalse(second_fn_done.is_set()) + first_fn() + first_fn_done.set() + + self.assertFalse(first_fn_done.is_set()) + t = threading.Thread(target=wrapped_first_fn) + t.start() + + second_fn() + self.assertTrue(first_fn_done.is_set()) + second_fn_done.set() + + coord.join([t]) + def testWaitRaiseErrorAfterMarkFailure(self): - closure_queue = client._CoordinatedClosureQueue() + closure_queue = client._CoordinatedClosureQueue(MockCancellationManager()) closure_queue.put(self._create_closure()) closure = closure_queue.get() @@ -126,22 +161,17 @@ class CoordinatedClosureQueueTest(test.TestCase): # all inflight closures are finished. def mark_finished_fn(): - with coord.stop_on_exception(): - self.assertFalse(wait_finish_event.is_set()) - try: - raise ValueError('Some error.') - except ValueError as e: - closure_queue.mark_failed(e, closure) - wait_finish_event.wait() + try: + raise ValueError('Some error.') + except ValueError as e: + closure_queue.mark_failed(e) - t = threading.Thread(target=mark_finished_fn) - t.start() + def wait_fn(): + with self.assertRaises(ValueError): + closure_queue.wait() - with self.assertRaises(ValueError): - closure_queue.wait() - wait_finish_event.set() + self._assert_one_unblock_the_other(mark_finished_fn, wait_fn) - coord.join([t]) self.assertTrue(closure_queue.done()) def _create_closure(self): @@ -150,10 +180,10 @@ class CoordinatedClosureQueueTest(test.TestCase): def some_function(): return 1.0 - return client.Closure(some_function) + return client.Closure(some_function, MockCancellationManager()) def _put_two_closures_and_get_one(self): - closure_queue = client._CoordinatedClosureQueue() + closure_queue = client._CoordinatedClosureQueue(MockCancellationManager()) closure1 = self._create_closure() closure_queue.put(closure1) @@ -166,9 +196,9 @@ class CoordinatedClosureQueueTest(test.TestCase): return closure_queue, closure1, closure2 def testPutRaiseError(self): - closure_queue, closure1, closure2 = self._put_two_closures_and_get_one() + closure_queue, _, closure2 = self._put_two_closures_and_get_one() - closure_queue.mark_failed(ValueError(), closure1) + closure_queue.mark_failed(ValueError()) with self.assertRaises(ValueError): closure_queue.put(self._create_closure()) @@ -185,9 +215,9 @@ class CoordinatedClosureQueueTest(test.TestCase): closure_queue.put(self._create_closure()) def testWaitRaiseError(self): - closure_queue, closure1, closure2 = self._put_two_closures_and_get_one() + closure_queue, _, closure2 = self._put_two_closures_and_get_one() - closure_queue.mark_failed(ValueError(), closure1) + closure_queue.mark_failed(ValueError()) with self.assertRaises(ValueError): closure_queue.wait() @@ -203,15 +233,22 @@ class CoordinatedClosureQueueTest(test.TestCase): closure_queue.wait() def testDoneRaiseError(self): - closure_queue, closure1, _ = self._put_two_closures_and_get_one() - closure_queue.get() + closure_queue, _, _ = self._put_two_closures_and_get_one() self.assertFalse(closure_queue.done()) - closure_queue.mark_failed(ValueError(), closure1) + closure_queue.mark_failed(ValueError()) with self.assertRaises(ValueError): closure_queue.done() - def _test_error_reporting_and_cancel_flow(self, call_wait): + def _set_error(self, closure_queue, closure, error): + try: + raise error + except Exception as e: # pylint: disable=broad-except + nest.map_structure(lambda x: x._set_error(e), + closure._output_remote_values) + closure_queue.mark_failed(e) + + def _test_cancel_closure_when_error(self, call_wait): closure_queue, closure1, closure2 = self._put_two_closures_and_get_one() closure_queue.put(self._create_closure()) closure_queue.get() @@ -219,34 +256,37 @@ class CoordinatedClosureQueueTest(test.TestCase): self.assertEqual(closure_queue._inflight_closure_count, 2) # Simulating closure1 fails. - try: - raise ValueError('Some error.') - except ValueError as e: - nest.map_structure(lambda x: x._set_error(e), - closure1._output_remote_values) - self.assertEqual(closure_queue._error_generation, 0) # pylint: disable=g-assert-in-except - closure_queue.mark_failed(e, closure1) - self.assertEqual(closure_queue._error_generation, 1) - # At this moment, there are one inflight, nothing - # in queue (because the ones in queue should have been removed and - # cancelled). - self.assertTrue(closure_queue._queue.empty()) - # Doesn't include out of generation closures. + self._set_error(closure_queue, closure1, ValueError('Some error.')) + + # At this moment, there are one inflight, one in queue. + self.assertEqual(closure_queue._queue.qsize(), 1) self.assertEqual(closure_queue._inflight_closure_count, 1) - coord = coordinator.Coordinator(clean_stop_exception_types=[]) closure3 = self._create_closure() - with self.assertRaises(ValueError): - # Verifying `wait()` or `put()` raises even if one closure is in - # flight. - if call_wait: - closure_queue.wait() - else: - closure_queue.put(closure3) - # At this moment, there is one inflight, nothing in queue. + def fake_cancellation(): + self._set_error(closure_queue, closure2, + ValueError('Fake cancellation error.')) + + def report_error(): + # It should not report the fake cancellation error. + with self.assertRaisesRegex(ValueError, 'Some error.'): + # Verifying `wait()` or `put()` raises even if one closure is in + # flight. + if call_wait: + closure_queue.wait() + else: + closure_queue.put(closure3) + + self._assert_one_unblock_the_other(fake_cancellation, report_error) + + # Cancellation manager has been called. + self.assertTrue(closure_queue._cancellation_mgr.cancelled) + + # At this moment, there is zero inflight, nothing in queue. self.assertTrue(closure_queue._queue.empty()) - self.assertEqual(closure_queue._inflight_closure_count, 1) + self.assertEqual(closure_queue._inflight_closure_count, 0) + self.assertIsNone(closure_queue._error) # This asserts that closure1 has errored. with self.assertRaisesRegex(ValueError, 'Some error.'): @@ -260,107 +300,36 @@ class CoordinatedClosureQueueTest(test.TestCase): 'function.'): closure3._fetch_output_remote_values() - # Closure2 is inflight, so it shouldn't be ready. + # Closure2 was an inflight closure when it got cancelled. self.assertEqual(closure2._output_remote_values._status, - client._RemoteValueStatus.NOT_READY) - - # And `wait` should block because closure2 is not back yet. - self.assertFalse(closure_queue.wait(timeout=20)) - - # Now let's assume that closure2 isn't successful due to worker preemption, - # and now it's attempted to be put back, but ends up getting cancelled. - self.assertEqual(closure2._error_generation, 0) - self.assertEqual(closure_queue._error_generation, 1) - closure_queue.put_back(closure2) - - with self.assertRaisesRegex( - client.FunctionRetryableError, - 'The corresponding function is cancelled. Please reschedule the ' - 'function.'): + client._RemoteValueStatus.READY) + with self.assertRaisesRegex(ValueError, 'Fake cancellation error.'): closure2._fetch_output_remote_values() - # At this moment, there is nothing inflight, and the queue is also empty - # (because closure2 should not be added back to the queue). - self.assertTrue(closure_queue._queue.empty()) - self.assertEqual(closure_queue._inflight_closure_count, 0) + # This asserts that the queue has a clear state. + self.testBasic() - closure4 = self._create_closure() + def testWaitRaiseErrorAfterCancelClosure(self): + self._test_cancel_closure_when_error(call_wait=True) - e = threading.Event() - - def get_fn(): - with coord.stop_on_exception(): - # This should end up getting closure4, not closure2, because closure2 - # has been cancelled and should not be got. - closure_got = closure_queue.get() - e.set() - self.assertEqual(closure_got._error_generation, 1) - self.assertEqual(closure_queue._error_generation, 1) - self.assertIs(closure4, closure_got) - self.assertIsNot(closure2, closure_got) - - t = threading.Thread(target=get_fn) - t.start() - - time.sleep(10) - - # Make sure `closure_got = closure_queue.get()` is unblocked as a result of - # `closure_queue.put(closure4)`. - self.assertFalse(e.is_set()) - closure_queue.put(closure4) - self.assertTrue(e.wait()) - coord.join([t]) - - self.assertEqual(closure_queue._inflight_closure_count, 1) - closure_queue.mark_finished(closure4) - # The queue is now cleared and nothing inflight. - self.assertEqual(closure_queue._inflight_closure_count, 0) - closure_queue.wait() - - def testWaitRaiseErrorAfterAnErrorIsReported(self): - self._test_error_reporting_and_cancel_flow(call_wait=True) - - def testPutRaiseErrorAfterAnErrorIsReported(self): - self._test_error_reporting_and_cancel_flow(call_wait=False) + def testPutRaiseErrorAfterCancelClosure(self): + self._test_cancel_closure_when_error(call_wait=False) def testStateIsRestoredAfterJoinIsCalled(self): - closure_queue, closure1, closure2 = self._put_two_closures_and_get_one() - closure_queue.get() - self.assertEqual(closure_queue._inflight_closure_count, 2) - closure_queue.mark_failed(ValueError('test error'), closure1) + closure_queue, _, _ = self._put_two_closures_and_get_one() + self.assertEqual(closure_queue._inflight_closure_count, 1) + closure_queue.mark_failed(ValueError('test error')) with self.assertRaises(ValueError): closure_queue.put(self._create_closure()) - closure_queue.mark_failed(ValueError('test error'), closure2) - # closure2's error is previous generation so should not raise at this - # following put, and _error should have been cleared. + # Its error should have been cleared. self.assertIsNone(closure_queue._error) closure_queue.put(self._create_closure()) self.assertIsNone(closure_queue._error) - def testStateIsRestoredAfterJoinIsCalled_WaitShouldReturn(self): - closure_queue, closure1, closure2 = self._put_two_closures_and_get_one() - closure_queue.put(self._create_closure()) - closure_queue.get() # got closure2 - self.assertFalse(closure_queue._queue.empty()) # still has closure3 - self.assertEqual(closure_queue._inflight_closure_count, 2) # closure1,2 - closure_queue.mark_failed(ValueError('test error'), closure1) - self.assertTrue(closure_queue._queue.empty()) # closure3 cancelled - self.assertEqual(closure_queue._inflight_closure_count, 1) - with self.assertRaises(ValueError): - closure_queue.wait() # reports error from closure1 - - # `wait` should block because closure2 is not back yet, even if closure2 - # was sent inflight before the error. - self.assertFalse(closure_queue.wait(timeout=20)) - self.assertEqual(closure_queue._inflight_closure_count, 1) - closure_queue.mark_finished(closure2) - closure_queue.wait() # wait should pass immediately - self.assertEqual(closure_queue._inflight_closure_count, 0) - def testThreadSafey(self): thread_count = 10 - queue = client._CoordinatedClosureQueue() + queue = client._CoordinatedClosureQueue(MockCancellationManager()) # Each thread performs 20 queue actions: 10 are `put_back` and 10 are # `mark_finished`. @@ -372,7 +341,7 @@ class CoordinatedClosureQueueTest(test.TestCase): if i % 2 == 0: queue.put_back(closure) else: - queue.mark_finished(closure) + queue.mark_finished() threads = [threading.Thread(target=func) for i in range(thread_count)] for t in threads: diff --git a/tensorflow/python/distribute/client/parameter_server_client_test.py b/tensorflow/python/distribute/client/parameter_server_client_test.py index db22a476b4a..32c7ff9c7e9 100644 --- a/tensorflow/python/distribute/client/parameter_server_client_test.py +++ b/tensorflow/python/distribute/client/parameter_server_client_test.py @@ -19,7 +19,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools +import threading from absl import logging + from tensorflow.python.data.ops import dataset_ops from tensorflow.python.distribute import multi_worker_test_base from tensorflow.python.distribute import sharded_variable @@ -40,6 +43,48 @@ from tensorflow.python.ops import variables from tensorflow.python.training.server_lib import ClusterSpec +class ErrorReportingThread(threading.Thread): + + error = None + + def __init__(self, *args, **kwargs): + assert "target" in kwargs + target = kwargs["target"] + + @functools.wraps(target) + def wrapped_target(*args, **kwargs): + try: + return target(*args, **kwargs) + except Exception as e: # pylint: disable=broad-except + ErrorReportingThread.error = e + + kwargs["target"] = wrapped_target + super(ErrorReportingThread, self).__init__(*args, **kwargs) + + +class TestCaseWithErrorReportingThread(test.TestCase): + + @classmethod + def setUpClass(cls): + cls._threading_thread = threading.Thread + threading.Thread = ErrorReportingThread + super(TestCaseWithErrorReportingThread, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + super(TestCaseWithErrorReportingThread, cls).tearDownClass() + threading.Thread = cls._threading_thread + + def setUp(self): + ErrorReportingThread.error = None + super(TestCaseWithErrorReportingThread, self).setUp() + + def tearDown(self): + super(TestCaseWithErrorReportingThread, self).tearDown() + if ErrorReportingThread.error: + raise ErrorReportingThread.error # pylint: disable=raising-bad-type + + def make_client(num_workers, num_ps): # TODO(rchao): Test the internal rpc_layer version. cluster_def = multi_worker_test_base.create_in_process_cluster( @@ -52,7 +97,7 @@ def make_client(num_workers, num_ps): return parameter_server_client.ParameterServerClient(cluster_resolver) -class ParameterServerClientTest(test.TestCase): +class ParameterServerClientTest(TestCaseWithErrorReportingThread): @classmethod def setUpClass(cls): @@ -304,7 +349,7 @@ class VariablePartitioningScopeTest(test.TestCase): self.assertEqual(var_sum, 10.0) -class ErrorReportingTest(test.TestCase): +class ErrorReportingTest(TestCaseWithErrorReportingThread): @classmethod def setUpClass(cls): @@ -344,8 +389,16 @@ class ErrorReportingTest(test.TestCase): while True: self.client.schedule(self._normal_function) + def testScheduleRaiseErrorWithMultipleFailure(self): + for _ in range(3): + self.client.schedule(self._normal_function) + self.client.schedule(self._error_function) + with self.assertRaises(errors.InvalidArgumentError): + while True: + self.client.schedule(self._error_function) + self.client.join() + def testErrorWillbeCleared(self): - self.skipTest("b/157597579") self.client.schedule(self._error_function) with self.assertRaises(errors.InvalidArgumentError): self.client.join() @@ -356,7 +409,7 @@ class ErrorReportingTest(test.TestCase): with self.assertRaises(errors.InvalidArgumentError): self.client.join() - def testFutureReturnError(self): + def testRemoteValueReturnError(self): result = self.client.schedule(self._error_function) with self.assertRaises(errors.InvalidArgumentError): From 1cb7ce30b747bc0afb301a4ee3af3dbd0e23be85 Mon Sep 17 00:00:00 2001 From: Yuanzhong Xu Date: Sun, 2 Aug 2020 23:17:55 -0700 Subject: [PATCH 0240/1017] Explicitly disable SPMD in TPU strategy. Mirrored variables are not yet supported for SPMD. This is in preparation of turning SPMD by default. PiperOrigin-RevId: 324548129 Change-Id: Ie7adf563402bd5ef31b7759232b1cd8f441586c7 --- tensorflow/python/distribute/tpu_strategy.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/distribute/tpu_strategy.py b/tensorflow/python/distribute/tpu_strategy.py index 3446f78288d..22aeb37ff7c 100644 --- a/tensorflow/python/distribute/tpu_strategy.py +++ b/tensorflow/python/distribute/tpu_strategy.py @@ -690,7 +690,10 @@ class TPUExtended(distribute_lib.StrategyExtendedV1): select_replica, per_replica_inputs),)) replicate_outputs = tpu.replicate( - run_fn, replicate_inputs, device_assignment=self._device_assignment) + run_fn, + replicate_inputs, + device_assignment=self._device_assignment, + xla_options=tpu.XLAOptions(use_spmd_for_xla_partitioning=False)) # If run_fn has tensor outputs, tpu.replicate returns a list of list. We # will flatten it in this case. If run_fn has no tensor outputs, @@ -1166,7 +1169,8 @@ class TPUExtended(distribute_lib.StrategyExtendedV1): replicate_inputs, device_assignment=self._device_assignment, maximum_shapes=maximum_shapes, - padding_spec=padding_spec) + padding_spec=padding_spec, + xla_options=tpu.XLAOptions(use_spmd_for_xla_partitioning=False)) # Remove all no ops that may have been added during 'tpu.replicate()' if isinstance(result[0], list): From d201be6284693c9ac5b93bdccfeeac2524d05239 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 00:54:27 -0700 Subject: [PATCH 0241/1017] Minor improvement, saying why the object is invalid. PiperOrigin-RevId: 324556545 Change-Id: Ieb0728e400ff2c220d07a29fecb40c37070d2f08 --- tensorflow/compiler/xla/pjrt/pjrt_client.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/xla/pjrt/pjrt_client.cc b/tensorflow/compiler/xla/pjrt/pjrt_client.cc index 126b74b9b98..c5dce4a37f7 100644 --- a/tensorflow/compiler/xla/pjrt/pjrt_client.cc +++ b/tensorflow/compiler/xla/pjrt/pjrt_client.cc @@ -1004,7 +1004,7 @@ PjRtBuffer::GetBufferForHoldLocked(ScopedHold::Type type) { // acquiring any other kind of hold. WaitForOutstandingDonationHold(); if (device_buffer_ == nullptr) { - return InvalidArgument("Hold requested on invalid buffer"); + return InvalidArgument("Hold requested on deleted or donated buffer"); } else { ++holds_[type]; } @@ -1084,7 +1084,8 @@ PjRtBuffer::CopyToHostAsyncInternal(bool discard_cached_copy, // We can't perform any other action while a donation hold is in progress. WaitForOutstandingDonationHold(); if (device_buffer_ == nullptr) { - return InvalidArgument("CopyToHostAsync() called on invalid buffer."); + return InvalidArgument( + "CopyToHostAsync() called on deleted or donated buffer"); } if (discard_cached_copy) { auto it = host_values_.find(host_layout); @@ -1154,7 +1155,7 @@ StatusOr> PjRtBuffer::ToLiteral( TF_ASSIGN_OR_RETURN(std::shared_ptr host_value, CopyToHostAsyncInternal(discard_cached_copy, layout)); if (host_value == nullptr) { - return InvalidArgument("ToLiteral called on invalid buffer"); + return InvalidArgument("ToLiteral called on deleted or donated buffer"); } host_value->ready.WaitForNotification(); TF_RETURN_IF_ERROR(host_value->status); @@ -1272,7 +1273,8 @@ StatusOr> PjRtBuffer::CopyToDevice( // We can't perform any other action while a donation hold is in progress. WaitForOutstandingDonationHold(); if (device_buffer_ == nullptr) { - return InvalidArgument("CopyToDevice called on invalid buffer"); + return InvalidArgument( + "CopyToDevice called on deleted or donated buffer"); } AcquireHoldLocked(&src_device_buffer); } @@ -1313,7 +1315,8 @@ Status PjRtBuffer::BlockHostUntilReady() { { absl::MutexLock lock(&mu_); if (device_buffer_ == nullptr) { - return InvalidArgument("BlockHostUntilReady() called on invalid buffer."); + return InvalidArgument( + "BlockHostUntilReady() called on deleted or donated buffer"); } device_buffer = device_buffer_; } From 82126e56ddb0c76330290e09815ec4240bef8bd3 Mon Sep 17 00:00:00 2001 From: Alexander Belyaev Date: Mon, 3 Aug 2020 01:22:43 -0700 Subject: [PATCH 0242/1017] [MLIR][KERNEL_GEN] Legalize TF Framework dialect to LLVM. PiperOrigin-RevId: 324559430 Change-Id: I1685be7f2aace9cf9658fe05574cf957ee67bd37 --- .../compiler/mlir/tools/kernel_gen/BUILD | 7 + .../compiler/mlir/tools/kernel_gen/ir/BUILD | 8 +- .../tests/tf_framework_legalize_to_llvm.mlir | 75 +++++++ .../mlir/tools/kernel_gen/transforms/BUILD | 48 +++++ .../mlir/tools/kernel_gen/transforms/passes.h | 38 ++++ .../tools/kernel_gen/transforms/passes.td | 27 +++ .../kernel_gen/transforms/register_passes.cc | 30 +++ .../tools/kernel_gen/transforms/rewriters.h | 36 ++++ .../tf_framework_legalize_to_llvm.cc | 187 ++++++++++++++++++ .../tf_framework_legalize_to_llvm_pass.cc | 71 +++++++ 10 files changed, 520 insertions(+), 7 deletions(-) create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tests/tf_framework_legalize_to_llvm.mlir create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm.cc create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/BUILD index de5926301dd..32fae8a8305 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/BUILD @@ -3,6 +3,12 @@ load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") licenses(["notice"]) +package_group( + name = "friends", + includes = ["//third_party/mlir:subpackages"], + packages = ["//tensorflow/compiler/mlir/..."], +) + cc_library( name = "cubin_creator", srcs = ["cubin_creator.cc"], @@ -57,6 +63,7 @@ tf_cc_binary( deps = [ "//tensorflow/compiler/mlir/tensorflow:tensorflow_dialect_registration", "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_dialect_registration", + "//tensorflow/compiler/mlir/tools/kernel_gen/transforms:passes", "@llvm-project//mlir:AllPassesAndDialects", "@llvm-project//mlir:MlirOptLib", "@llvm-project//mlir:MlirOptMain", diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/ir/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/ir/BUILD index 0c3db5fa4ab..3a28d4815d2 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/ir/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/ir/BUILD @@ -1,16 +1,10 @@ load("//third_party/mlir:tblgen.bzl", "gentbl") package( - default_visibility = [":friends"], + default_visibility = ["//tensorflow/compiler/mlir/tools/kernel_gen:friends"], licenses = ["notice"], # Apache 2.0 ) -package_group( - name = "friends", - includes = ["//third_party/mlir:subpackages"], - packages = ["//tensorflow/compiler/mlir/..."], -) - gentbl( name = "tf_framework_ops_inc_gen", tbl_outs = [ diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tests/tf_framework_legalize_to_llvm.mlir b/tensorflow/compiler/mlir/tools/kernel_gen/tests/tf_framework_legalize_to_llvm.mlir new file mode 100644 index 00000000000..77328aa7738 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/tests/tf_framework_legalize_to_llvm.mlir @@ -0,0 +1,75 @@ +// RUN: kernel-gen-opt %s -test-tf-framework-legalize-to-llvm -split-input-file | FileCheck %s + +// CHECK: llvm.func @_mlir_ciface_tf_alloc_raw +// CHECK-SAME: (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*"> + +// CHECK-LABEL: llvm.func @alloc_raw( +// CHECK-SAME: [[TF_CTX:%.*]]: !llvm<"i8*">, +// CHECK-SAME: [[SIZE_0:%.*]]: !llvm.i64, +// CHECK-SAME: [[SIZE_2:%.*]]: !llvm.i64) -> [[DESC_TY:!.*]] { +func @alloc_raw(%ctx: !tf_framework.op_kernel_context, + %size_0 : index , %size_2 : index) -> memref { + %buf = tf_framework.alloc_raw(%ctx, %size_0, %size_2) : memref + std.return %buf : memref +} +// Compute number of elements. +// CHECK: [[SIZE_1:%.*]] = llvm.mlir.constant(10 : index) : !llvm.i64 +// CHECK: [[NUM_ELEM_0:%.*]] = llvm.mul [[SIZE_0]], [[SIZE_1]] : !llvm.i64 +// CHECK: [[NUM_ELEM_1:%.*]] = llvm.mul [[NUM_ELEM_0]], [[SIZE_2]] : !llvm.i64 + +// Compute the size of an individual element. +// CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm<"float*"> +// CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : index) : !llvm.i64 +// CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]]{{\[}}[[C1]]] +// CHECK-SAME: (!llvm<"float*">, !llvm.i64) -> !llvm<"float*"> +// CHECK: [[SIZE_OF_FLOAT:%.*]] = llvm.ptrtoint [[GEP]] +// CHECK-SAME: !llvm<"float*"> to !llvm.i64 + +// Allocate memory. +// CHECK: [[NUM_BYTES:%.*]] = llvm.mul [[NUM_ELEM_1]], [[SIZE_OF_FLOAT]] +// CHECK: [[BYTES_PTR:%.*]] = llvm.call @{{.*}}([[TF_CTX]], [[NUM_BYTES]]) +// CHECK-SAME: (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*"> + +// Build memref descriptor. +// CHECK: [[DESC_0:%.*]] = llvm.mlir.undef : [[DESC_TY]] + +// Set pointers and offset. +// CHECK: [[FLOAT_PTR:%.*]] = llvm.bitcast [[BYTES_PTR]] +// CHECK-SAME: !llvm<"i8*"> to !llvm<"float*"> +// CHECK: [[DESC_1:%.*]] = llvm.insertvalue [[FLOAT_PTR]], [[DESC_0]][0] +// CHECK: [[DESC_2:%.*]] = llvm.insertvalue [[FLOAT_PTR]], [[DESC_1]][1] +// CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : index) : !llvm.i64 +// CHECK: [[DESC_3:%.*]] = llvm.insertvalue [[C0]], [[DESC_2]][2] : [[DESC_TY]] + +// Set sizes and strides. +// CHECK: [[STRIDE_2:%.*]] = llvm.mlir.constant(1 : index) : !llvm.i64 +// CHECK: [[DESC_4:%.*]] = llvm.insertvalue [[SIZE_2]], [[DESC_3]][3, 2] +// CHECK: [[DESC_5:%.*]] = llvm.insertvalue [[STRIDE_2]], [[DESC_4]][4, 2] +// CHECK: [[STRIDE_1:%.*]] = llvm.mul [[STRIDE_2]], [[SIZE_2]] : !llvm.i64 +// CHECK: [[DESC_6:%.*]] = llvm.insertvalue [[SIZE_1]], [[DESC_5]][3, 1] +// CHECK: [[DESC_7:%.*]] = llvm.insertvalue [[STRIDE_1]], [[DESC_6]][4, 1] +// CHECK: [[STRIDE_0:%.*]] = llvm.mul [[STRIDE_1]], [[SIZE_1]] : !llvm.i64 +// CHECK: [[DESC_8:%.*]] = llvm.insertvalue [[SIZE_0]], [[DESC_7]][3, 0] +// CHECK: [[DESC_9:%.*]] = llvm.insertvalue [[STRIDE_0]], [[DESC_8]][4, 0] +// CHECK: llvm.return [[DESC_9]] : [[DESC_TY]] + +// ----- + +// CHECK: llvm.func @_mlir_ciface_tf_dealloc_raw(!llvm<"i8*">) + +// CHECK-LABEL: llvm.func @dealloc_raw( +// CHECK-SAME: [[TF_CTX:%.*]]: !llvm<"i8*">, +func @dealloc_raw(%ctx: !tf_framework.op_kernel_context, + %memref : memref) { + tf_framework.dealloc_raw(%ctx, %memref) : memref + return +} +// Extract allocated ptr from the memref descriptor. +// CHECK: %{{.*}} = llvm.mlir.undef : [[DESC_TY:!.*]] +// CHECK: [[FLOAT_PTR:%.*]] = llvm.extractvalue %{{.*}}[0] : [[DESC_TY]] +// CHECK-NEXT: [[VOID_PTR:%.*]] = llvm.bitcast [[FLOAT_PTR]] +// CHECK-SAME: !llvm<"float*"> to !llvm<"i8*"> + +// Deallocate. +// CHECK: llvm.call @_mlir_ciface_tf_dealloc_raw( +// CHECK-SAME: [[TF_CTX]], [[VOID_PTR]]) : (!llvm<"i8*">, !llvm<"i8*">) -> () diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD new file mode 100644 index 00000000000..15c0d571e61 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD @@ -0,0 +1,48 @@ +load("//third_party/mlir:tblgen.bzl", "gentbl") + +package( + default_visibility = ["//tensorflow/compiler/mlir/tools/kernel_gen:friends"], + licenses = ["notice"], # Apache 2.0 +) + +cc_library( + name = "tf_framework_legalize_to_llvm", + srcs = ["tf_framework_legalize_to_llvm.cc"], + hdrs = ["rewriters.h"], + deps = [ + "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_ops", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:LLVMTransforms", + "@llvm-project//mlir:StandardOps", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + +gentbl( + name = "tf_framework_passes_inc_gen", + tbl_outs = [("-gen-pass-decls", "tf_framework_passes.h.inc")], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "passes.td", + td_srcs = ["@llvm-project//mlir:PassBaseTdFiles"], +) + +cc_library( + name = "passes", + srcs = [ + "register_passes.cc", + "tf_framework_legalize_to_llvm_pass.cc", + ], + hdrs = ["passes.h"], + deps = [ + ":tf_framework_legalize_to_llvm", + ":tf_framework_passes_inc_gen", + "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_ops", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:LLVMTransforms", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:StandardOps", + ], + alwayslink = 1, +) diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h new file mode 100644 index 00000000000..89871ba3faf --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h @@ -0,0 +1,38 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TRANSFORMS_PASSES_H_ +#define TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TRANSFORMS_PASSES_H_ + +#include + +namespace mlir { + +class ModuleOp; +template +class OperationPass; + +namespace kernel_gen { +namespace tf_framework { + +// Test pass for applying TF Framework -> LLVM patterns. +std::unique_ptr > +createTestTFFrameworkLegalizeToLLVMPass(); + +} // namespace tf_framework +} // namespace kernel_gen +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TRANSFORMS_PASSES_H_ diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td new file mode 100644 index 00000000000..71e50379ce7 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td @@ -0,0 +1,27 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TF_FRAMEWORK_PASSES +#define TF_FRAMEWORK_PASSES + +include "mlir/Pass/PassBase.td" + +def TestTFFrameworkLegalizeToLLVMPass + : Pass<"test-tf-framework-legalize-to-llvm", "ModuleOp"> { + let summary = "Test pass for applying TF Framework -> LLVM patterns."; + let constructor = "createTestTFFrameworkLegalizeToLLVMPass()"; +} + +#endif // TF_FRAMEWORK_PASSES diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc new file mode 100644 index 00000000000..b9bad8e18d2 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc @@ -0,0 +1,30 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" + +namespace mlir { +namespace kernel_gen { +namespace tf_framework { + +bool register_all_passes = ([] { +#define GEN_PASS_REGISTRATION +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_passes.h.inc" +}(), true); + +} // namespace tf_framework +} // namespace kernel_gen +} // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h new file mode 100644 index 00000000000..28dba379738 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h @@ -0,0 +1,36 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TRANSFORMS_REWRITERS_H_ +#define TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TRANSFORMS_REWRITERS_H_ + +namespace mlir { + +class LLVMTypeConverter; +class LowerToLLVMOptions; +class OwningRewritePatternList; + +namespace kernel_gen { +namespace tf_framework { + +/// Collect a set of patterns to convert from the TF Framework dialect to LLVM. +void PopulateTFFrameworkToLLVMConversionPatterns( + LLVMTypeConverter *converter, OwningRewritePatternList *patterns); + +} // namespace tf_framework +} // namespace kernel_gen +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TRANSFORMS_REWRITERS_H_ diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm.cc new file mode 100644 index 00000000000..2edcaabd7b4 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm.cc @@ -0,0 +1,187 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h" // from @llvm-project +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" // from @llvm-project +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/Module.h" // from @llvm-project +#include "mlir/IR/StandardTypes.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h" + +namespace mlir { +namespace kernel_gen { +namespace tf_framework { +namespace { + +using LLVM::LLVMFuncOp; +using LLVM::LLVMType; + +static constexpr StringRef kCInterfaceAlloc = "_mlir_ciface_tf_alloc_raw"; +static constexpr StringRef kCInterfaceDealloc = "_mlir_ciface_tf_dealloc_raw"; + +/// Base class for patterns converting TF Framework ops to function calls. +template +class ConvertToLLVMCallOpPattern : public ConvertOpToLLVMPattern { + public: + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + // Attempts to find function symbol in the module, adds it if not found. + FlatSymbolRefAttr getOrInsertTFFunction(PatternRewriter &rewriter, + Operation *op) const { + ModuleOp module = op->getParentOfType(); + StringRef tf_func_name = GetFuncName(); + auto tf_func = module.lookupSymbol(tf_func_name); + if (!tf_func) { + OpBuilder::InsertionGuard guard(rewriter); + rewriter.setInsertionPointToStart(module.getBody()); + auto func_type = GetFuncType(); + tf_func = rewriter.create(rewriter.getUnknownLoc(), + tf_func_name, func_type); + } + return SymbolRefAttr::get(tf_func_name, rewriter.getContext()); + } + + protected: + virtual StringRef GetFuncName() const = 0; + virtual LLVMType GetFuncType() const = 0; +}; + +class AllocRawOpConverter : public ConvertToLLVMCallOpPattern { + public: + using ConvertToLLVMCallOpPattern::ConvertToLLVMCallOpPattern; + + LogicalResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + Location loc = op->getLoc(); + AllocRawOp alloc_raw_op = cast(op); + AllocRawOp::Adaptor transformed(operands); + + MemRefType memref_type = alloc_raw_op.getType(); + + // Get memref descriptor sizes. + SmallVector sizes; + getMemRefDescriptorSizes(loc, memref_type, + llvm::to_vector<4>(transformed.dyn_sizes()), + rewriter, sizes); + // Get memory block size in bytes. + Value num_bytes = getCumulativeSizeInBytes( + loc, memref_type.getElementType(), sizes, rewriter); + + // Insert function call. + FlatSymbolRefAttr tf_func_ref = getOrInsertTFFunction(rewriter, op); + Value allocated_byte_ptr = + rewriter + .create( + loc, getVoidPtrType(), tf_func_ref, + llvm::makeArrayRef({transformed.ctx(), num_bytes})) + .getResult(0); + + MemRefDescriptor memRefDescriptor = CreateMemRefDescriptor( + loc, rewriter, memref_type, allocated_byte_ptr, sizes); + + // Return the final value of the descriptor. + rewriter.replaceOp(op, {memRefDescriptor}); + return success(); + } + + protected: + StringRef GetFuncName() const override { return kCInterfaceAlloc; } + LLVMType GetFuncType() const override { + LLVMType llvm_void_ptr_type = getVoidPtrType(); + return LLVM::LLVMType::getFunctionTy( + llvm_void_ptr_type, + llvm::makeArrayRef({llvm_void_ptr_type, getIndexType()}), + /*isVarArg=*/false); + } + + private: + MemRefDescriptor CreateMemRefDescriptor(Location loc, + ConversionPatternRewriter &rewriter, + MemRefType memref_type, + Value allocated_byte_ptr, + ArrayRef sizes) const { + auto memref_desc = MemRefDescriptor::undef( + rewriter, loc, typeConverter.convertType(memref_type)); + + // TF AllocateRaw returns aligned pointer => AllocatedPtr == AlignedPtr. + Value allocated_type_ptr = rewriter.create( + loc, getElementPtrType(memref_type), allocated_byte_ptr); + memref_desc.setAllocatedPtr(rewriter, loc, allocated_type_ptr); + memref_desc.setAlignedPtr(rewriter, loc, allocated_type_ptr); + memref_desc.setConstantOffset(rewriter, loc, 0); + + if (memref_type.getRank() == 0) { + return memref_desc; + } + + // Compute strides and populate descriptor `size` and `stride` fields. + Value stride_carried = createIndexConstant(rewriter, loc, 1); + for (int pos = sizes.size() - 1; pos >= 0; --pos) { + Value size = sizes[pos]; + memref_desc.setSize(rewriter, loc, pos, size); + memref_desc.setStride(rewriter, loc, pos, stride_carried); + // Update stride + if (pos > 0) { + stride_carried = + rewriter.create(loc, stride_carried, size); + } + } + return memref_desc; + } +}; + +class DeallocRawOpConverter : public ConvertToLLVMCallOpPattern { + public: + using ConvertToLLVMCallOpPattern::ConvertToLLVMCallOpPattern; + + LogicalResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + DeallocRawOp::Adaptor transformed(operands); + MemRefDescriptor memref(transformed.memref()); + + Value allocated_bytes_ptr = rewriter.create( + op->getLoc(), getVoidPtrType(), + memref.allocatedPtr(rewriter, op->getLoc())); + + // Insert function call. + FlatSymbolRefAttr tf_func_ref = getOrInsertTFFunction(rewriter, op); + rewriter.replaceOpWithNewOp( + op, llvm::None, tf_func_ref, + llvm::makeArrayRef({transformed.ctx(), allocated_bytes_ptr})); + return success(); + } + + protected: + StringRef GetFuncName() const override { return kCInterfaceDealloc; } + LLVMType GetFuncType() const override { + return LLVM::LLVMType::getFunctionTy(getVoidType(), getVoidPtrType(), + /*isVarArg=*/false); + } +}; + +} // namespace + +void PopulateTFFrameworkToLLVMConversionPatterns( + LLVMTypeConverter *converter, OwningRewritePatternList *patterns) { + patterns->insert(*converter); +} + +} // namespace tf_framework +} // namespace kernel_gen +} // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc new file mode 100644 index 00000000000..8439e1617e0 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc @@ -0,0 +1,71 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h" // from @llvm-project +#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h" // from @llvm-project +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" // from @llvm-project +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h" + +namespace mlir { +namespace kernel_gen { +namespace tf_framework { +namespace { + +#define GEN_PASS_CLASSES +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_passes.h.inc" + +class TestTFFrameworkToLLVMPass + : public TestTFFrameworkLegalizeToLLVMPassBase { + public: + void runOnOperation() override { + ModuleOp m = getOperation(); + + // Populate type conversions. + LLVMTypeConverter type_converter(m.getContext()); + type_converter.addConversion([&](tf_framework::OpKernelContextType type) { + return LLVM::LLVMType::getInt8PtrTy(type_converter.getDialect()); + }); + + // Populate patterns. + OwningRewritePatternList patterns; + populateStdToLLVMConversionPatterns(type_converter, patterns); + PopulateTFFrameworkToLLVMConversionPatterns(&type_converter, &patterns); + + // Set target. + ConversionTarget target(getContext()); + target.addLegalDialect(); + target.addIllegalDialect(); + target.addLegalOp(); + + if (failed(applyFullConversion(m, target, patterns))) { + signalPassFailure(); + } + } +}; + +} // namespace + +std::unique_ptr > +createTestTFFrameworkLegalizeToLLVMPass() { + return std::make_unique(); +} + +} // namespace tf_framework +} // namespace kernel_gen +} // namespace mlir From c730b889e20569833c50c5aa224a943fc92027b0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 02:01:42 -0700 Subject: [PATCH 0243/1017] Update GraphDef version to 482. PiperOrigin-RevId: 324562958 Change-Id: Ie80c4ff6b968b2281711d8437fcdde7205c22518 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 3784f2b212d..dae48097aa8 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 481 // Updated: 2020/8/2 +#define TF_GRAPH_DEF_VERSION 482 // Updated: 2020/8/3 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From 71329959e563a6278a31d45029b8b4a86642aad2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 02:01:42 -0700 Subject: [PATCH 0244/1017] compat: Update forward compatibility horizon to 2020-08-03 PiperOrigin-RevId: 324562959 Change-Id: I688798867a67903d517e9402be2441a1657c24da --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index e36055c6a93..e0f751a4376 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 2) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 3) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From 4305e83f8cc65d9bf941b7136859fc9397368da6 Mon Sep 17 00:00:00 2001 From: Alexander Belyaev Date: Mon, 3 Aug 2020 02:41:09 -0700 Subject: [PATCH 0245/1017] [MLIR][KERNEL_GEN] Use TF_FRAMEWORK_TYPE instead of PRIVATE_EXPERIMENTAL_0. This is just a clean up. PiperOrigin-RevId: 324567279 Change-Id: I1a14ff07b60ed24baf94ef8c544f175f88e1dc89 --- .../compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h index ae621cb386a..8d6e433d9b9 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h +++ b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h @@ -32,10 +32,7 @@ namespace tf_framework { namespace TFFrameworkTypes { enum Kind { - // TODO(pifon): Replace enum value with - // OpKernelContextType = Type::FIRST_TF_FRAMEWORK_TYPE, - // after DialectSymbolRegistry.def is updated. - OpKernelContextType = Type::FIRST_PRIVATE_EXPERIMENTAL_0_TYPE, + OpKernelContextType = Type::FIRST_TF_FRAMEWORK_TYPE, }; } // namespace TFFrameworkTypes From 123af9ba8605aedeb11c9f1eca0a3e7c64f324c2 Mon Sep 17 00:00:00 2001 From: Stephan Herhut Date: Mon, 3 Aug 2020 04:00:34 -0700 Subject: [PATCH 0246/1017] Also allow older compute capabilities 32 and 30 when generating ptx. PiperOrigin-RevId: 324575272 Change-Id: I1a9a49edb55b36cb971ad713e070ce1ee58b34fb --- .../xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc index a93810b53f7..1228a1b4823 100644 --- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc +++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/gpu_backend_lib.cc @@ -83,10 +83,10 @@ const int kDefaultInlineThreshold = 1100; static string GetSmName(std::pair compute_capability) { int compute_capability_version = compute_capability.first * 10 + compute_capability.second; - int sm_version = 35; + int sm_version = 30; // If the current compute capability isn't known, fallback to the // most recent version before it. - for (int v : {75, 72, 70, 62, 61, 60, 53, 52, 50, 37, 35}) { + for (int v : {75, 72, 70, 62, 61, 60, 53, 52, 50, 37, 35, 32, 30}) { if (v <= compute_capability_version) { sm_version = v; break; From 8e9f3196fd8841de83bd6a622df696ea191d1d78 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 08:22:33 -0700 Subject: [PATCH 0247/1017] Added a bunch of unary ops to the estimator. PiperOrigin-RevId: 324607213 Change-Id: I24369f36cc29f68caac412a5d3076f5ef43859fe --- .../grappler/costs/op_level_cost_estimator.cc | 8 ++++ .../costs/op_level_cost_estimator_test.cc | 39 +++++++++++-------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc index d2e56cd2f1c..62e6e361ef8 100644 --- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc +++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc @@ -522,6 +522,8 @@ OpLevelCostEstimator::OpLevelCostEstimator() { // Unary ops alphabetically sorted elementwise_ops_.emplace("Acos", EIGEN_COST(scalar_acos_op)); + elementwise_ops_.emplace("All", EIGEN_COST(scalar_boolean_and_op)); + elementwise_ops_.emplace("ArgMax", EIGEN_COST(scalar_max_op)); elementwise_ops_.emplace("Asin", EIGEN_COST(scalar_asin_op)); elementwise_ops_.emplace("Atan", EIGEN_COST(scalar_atan_op)); elementwise_ops_.emplace("Atan2", EIGEN_COST(scalar_quotient_op) + @@ -546,7 +548,10 @@ OpLevelCostEstimator::OpLevelCostEstimator() { elementwise_ops_.emplace("Lgamma", 1); elementwise_ops_.emplace("Log", EIGEN_COST(scalar_log_op)); elementwise_ops_.emplace("Log1p", EIGEN_COST(scalar_log1p_op)); + elementwise_ops_.emplace("Max", EIGEN_COST(scalar_max_op)); + elementwise_ops_.emplace("Min", EIGEN_COST(scalar_min_op)); elementwise_ops_.emplace("Neg", EIGEN_COST(scalar_opposite_op)); + elementwise_ops_.emplace("Prod", EIGEN_COST(scalar_product_op)); elementwise_ops_.emplace("QuantizeAndDequantizeV2", quantize_and_dequantize_v2_cost); elementwise_ops_.emplace("QuantizedSigmoid", @@ -554,6 +559,7 @@ OpLevelCostEstimator::OpLevelCostEstimator() { elementwise_ops_.emplace("QuantizeV2", quantize_v2_cost); elementwise_ops_.emplace("Reciprocal", EIGEN_COST(scalar_inverse_op)); elementwise_ops_.emplace("Relu", EIGEN_COST(scalar_max_op)); + elementwise_ops_.emplace("Relu6", EIGEN_COST(scalar_max_op)); elementwise_ops_.emplace("Rint", 1); elementwise_ops_.emplace("Round", EIGEN_COST(scalar_round_op)); elementwise_ops_.emplace("Rsqrt", EIGEN_COST(scalar_rsqrt_op)); @@ -562,8 +568,10 @@ OpLevelCostEstimator::OpLevelCostEstimator() { elementwise_ops_.emplace("Sin", EIGEN_COST(scalar_sin_op)); elementwise_ops_.emplace("Sqrt", EIGEN_COST(scalar_sqrt_op)); elementwise_ops_.emplace("Square", EIGEN_COST(scalar_square_op)); + elementwise_ops_.emplace("Sum", EIGEN_COST(scalar_sum_op)); elementwise_ops_.emplace("Tan", EIGEN_COST(scalar_tan_op)); elementwise_ops_.emplace("Tanh", EIGEN_COST(scalar_tanh_op)); + elementwise_ops_.emplace("TopKV2", EIGEN_COST(scalar_max_op)); // Binary ops alphabetically sorted elementwise_ops_.emplace("Add", EIGEN_COST(scalar_sum_op)); elementwise_ops_.emplace("AddV2", EIGEN_COST(scalar_sum_op)); diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc index d24533cf532..0f19b54feec 100644 --- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc +++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc @@ -939,24 +939,29 @@ TEST_F(OpLevelCostEstimatorTest, SquaredDifferenceExecutionTime) { EXPECT_EQ(cost.num_ops_with_unknown_shapes, 0); } -TEST_F(OpLevelCostEstimatorTest, ReluExecutionTime) { - auto cost = PredictCosts(DescribeUnaryOp("Relu", 1000)); - EXPECT_EQ(Costs::Duration(800), cost.memory_time); - EXPECT_EQ(Costs::Duration(100), cost.compute_time); - EXPECT_EQ(Costs::Duration(900), cost.execution_time); - EXPECT_EQ(1, cost.num_ops_total); - EXPECT_FALSE(cost.inaccurate); - EXPECT_EQ(0, cost.num_ops_with_unknown_shapes); -} +TEST_F(OpLevelCostEstimatorTest, UnaryOpExecutionTime) { + std::vector> unary_ops = { + {"All", 1}, {"ArgMax", 1}, {"Cast", 1}, {"Max", 1}, {"Min", 1}, + {"Prod", 1}, {"Relu", 1}, {"Relu6", 1}, {"Sum", 1}, {"TopKV2", 1}}; -TEST_F(OpLevelCostEstimatorTest, CastExecutionTime) { - auto cost = PredictCosts(DescribeUnaryOp("Cast", 1000)); - EXPECT_EQ(Costs::Duration(800), cost.memory_time); - EXPECT_EQ(Costs::Duration(100), cost.compute_time); - EXPECT_EQ(Costs::Duration(900), cost.execution_time); - EXPECT_EQ(1, cost.num_ops_total); - EXPECT_FALSE(cost.inaccurate); - EXPECT_EQ(0, cost.num_ops_with_unknown_shapes); + const int kTensorSize = 1000; + for (auto unary_op : unary_ops) { + OpContext op_context = DescribeUnaryOp(unary_op.first, kTensorSize); + + const int kExpectedMemoryTime = 800; + int expected_compute_time = std::ceil( + unary_op.second * kTensorSize / + estimator_.GetDeviceInfo(op_context.op_info.device()).gigaops); + + auto cost = PredictCosts(op_context); + EXPECT_EQ(cost.memory_time, Costs::Duration(kExpectedMemoryTime)); + EXPECT_EQ(cost.compute_time, Costs::Duration(expected_compute_time)); + EXPECT_EQ(cost.execution_time, + Costs::Duration(expected_compute_time + kExpectedMemoryTime)); + EXPECT_EQ(cost.num_ops_total, 1); + EXPECT_EQ(cost.num_ops_with_unknown_shapes, 0); + EXPECT_FALSE(cost.inaccurate); + } } TEST_F(OpLevelCostEstimatorTest, BroadcastAddExecutionTime) { From fbb9c59ab276664ba1a3c09adbe5f2d397c71ea4 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Mon, 3 Aug 2020 16:01:21 +0000 Subject: [PATCH 0248/1017] Update tensorflow/python/ops/custom_gradient.py --- tensorflow/python/ops/custom_gradient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py index e67f19099fc..6715d3bf0a7 100644 --- a/tensorflow/python/ops/custom_gradient.py +++ b/tensorflow/python/ops/custom_gradient.py @@ -139,7 +139,6 @@ def custom_gradient(f=None): the same number of variables. We take the function `z = x * y` as an example. ```python - >>> import tensorflow as tf >>> @tf.custom_gradient def bar(x, y): def grad(upstream): From 86927dd8ddc90ffc5a9bdfb4f8b48ecf76e8aa44 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Mon, 3 Aug 2020 16:02:28 +0000 Subject: [PATCH 0249/1017] Update tensorflow/python/ops/custom_gradient.py --- tensorflow/python/ops/custom_gradient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py index 6715d3bf0a7..c0ee970af3e 100644 --- a/tensorflow/python/ops/custom_gradient.py +++ b/tensorflow/python/ops/custom_gradient.py @@ -138,7 +138,6 @@ def custom_gradient(f=None): In case the function takes multiple variables as input, the `grad` function must also return the same number of variables. We take the function `z = x * y` as an example. - ```python >>> @tf.custom_gradient def bar(x, y): def grad(upstream): From 84ad9375524def803dc0b1e6470688d893f8cf1b Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Mon, 3 Aug 2020 16:02:35 +0000 Subject: [PATCH 0250/1017] Update tensorflow/python/ops/custom_gradient.py --- tensorflow/python/ops/custom_gradient.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py index c0ee970af3e..f2675b422ac 100644 --- a/tensorflow/python/ops/custom_gradient.py +++ b/tensorflow/python/ops/custom_gradient.py @@ -165,7 +165,6 @@ def custom_gradient(f=None): 2 >>> tape.gradient(x, y) None - ``` Nesting custom gradients can lead to unintuitive results. The default behavior does not correspond to n-th order derivatives. For example From cad412bfcbaea7952e7758620e3928f64d83be32 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Mon, 3 Aug 2020 16:12:00 +0000 Subject: [PATCH 0251/1017] Update tensorflow/python/ops/math_ops.py --- tensorflow/python/ops/math_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 2eae0ade4dd..1d43802ab41 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -3850,8 +3850,8 @@ def conj(x, name=None): Given a tensor `x` of complex numbers, this operation returns a tensor of complex numbers that are the complex conjugate of each element in `x`. The - complex numbers in `x` must be of the form \\(a + bj\\), where *a* is the - real part and *b* is the imaginary part. + complex numbers in `x` must be of the form \\(a + bj\\), where `a` is the + real part and `b` is the imaginary part. The complex conjugate returned by this operation is of the form \\(a - bj\\). From 3ccad31d3963077b9879c9e18921e32213b9c32f Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Mon, 3 Aug 2020 09:48:09 -0700 Subject: [PATCH 0252/1017] Update copyright in GuaranteeAllFuncsOneUse to be of TensorFlow and update function names to match Google C++ Style Guide (NFC). PiperOrigin-RevId: 324622347 Change-Id: Ib1d19560afa26e4fef197b1694328e26f53ade8e --- .../transforms/guarantee_all_funcs_one_use.cc | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc b/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc index 6112ff500c5..776afd72ad5 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/guarantee_all_funcs_one_use.cc @@ -1,16 +1,17 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ #include "llvm/ADT/STLExtras.h" #include "mlir/IR/SymbolTable.h" // from @llvm-project @@ -51,12 +52,12 @@ class GuaranteeAllFuncsOneUse : public PassWrapper> { public: void runOnOperation() override { - if (failed(run())) { + if (failed(Run())) { signalPassFailure(); } } - LogicalResult run() { + LogicalResult Run() { auto module = getOperation(); // Overall strategy: From 856dc4f7b6b1c4b35142961e432a5fa66c6d1259 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 10:01:28 -0700 Subject: [PATCH 0253/1017] Add MLIR definition for StatelessTruncatedNormalOp. PiperOrigin-RevId: 324625125 Change-Id: Ia17f1179c18c509b60427765134c377be3aef403 --- .../mlir/tensorflow/ir/tf_generated_ops.td | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index e00ea4c342a..63138489ef7 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -9596,6 +9596,33 @@ The outputs are a deterministic function of `shape` and `seed`. TF_DerivedResultTypeAttr dtype = TF_DerivedResultTypeAttr<0>; } +def TF_StatelessTruncatedNormalOp : TF_Op<"StatelessTruncatedNormal", [NoSideEffect]> { + let summary = [{ +Outputs deterministic pseudorandom values from a truncated normal distribution. + }]; + + let description = [{ +The generated values follow a normal distribution with mean 0 and standard +deviation 1, except that values whose magnitude is more than 2 standard +deviations from the mean are dropped and re-picked. + +The outputs are a deterministic function of `shape` and `seed`. + }]; + + let arguments = (ins + TF_I32OrI64Tensor:$shape, + TF_I32OrI64Tensor:$seed + ); + + let results = (outs + TF_FpTensor:$output + ); + + TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<0>; + TF_DerivedOperandTypeAttr Tseed = TF_DerivedOperandTypeAttr<1>; + TF_DerivedResultTypeAttr dtype = TF_DerivedResultTypeAttr<0>; +} + def TF_StopGradientOp : TF_Op<"StopGradient", [NoSideEffect, TF_AllTypesMatch<["input", "output"]>]> { let summary = "Stops gradient computation."; From 59dc165d26caaed925bcfe7b40752b8429d922ea Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Mon, 3 Aug 2020 10:05:11 -0700 Subject: [PATCH 0254/1017] Enable input spec checking for Functional models. PiperOrigin-RevId: 324625967 Change-Id: Ide0a8cb4d6d7614f86f22088a5ef95d72636c54e --- RELEASE.md | 9 +- .../distribute/distribute_strategy_test.py | 2 +- tensorflow/python/keras/engine/base_layer.py | 3 +- tensorflow/python/keras/engine/functional.py | 36 +++++ .../python/keras/engine/functional_test.py | 59 +++++++- tensorflow/python/keras/engine/input_spec.py | 140 ++++++++++++------ tensorflow/python/keras/engine/sequential.py | 6 + .../keras/layers/tensorflow_op_layer_test.py | 14 +- .../keras/legacy_tf_layers/base_test.py | 25 +--- .../tensorflow.keras.layers.-input-spec.pbtxt | 2 +- .../v1/tensorflow.layers.-input-spec.pbtxt | 2 +- .../tensorflow.keras.layers.-input-spec.pbtxt | 2 +- 12 files changed, 214 insertions(+), 86 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 13369fd92f7..0c6b0f556e8 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -25,7 +25,14 @@ * Code that requires very tricky shape manipulation via converted op layers in order to work, where the Keras symbolic shape inference proves insufficient. * Code that tries manually walking a `tf.keras.Model` layer by layer and assumes layers only ever have one positional argument. This assumption doesn't hold true before TF 2.4 either, but is more likely to cause issues know. * Code that manually enters `keras.backend.get_graph()` before building a functional model. This is no longer needed. - +* Start enforcing input shape assumptions when calling Functional API Keras + models. This may potentially break some users, in case there is a mismatch + between the shape used when creating `Input` objects in a Functional model, + and the shape of the data passed to that model. You can fix this mismatch by + either calling the model with correctly-shaped data, or by relaxing `Input` + shape assumptions (note that you can pass shapes with `None` entries for axes + that are meant to be dynamic). You can also disable the input checking + entirely by setting `model.input_spec = None`. ## Known Caveats diff --git a/tensorflow/python/keras/distribute/distribute_strategy_test.py b/tensorflow/python/keras/distribute/distribute_strategy_test.py index df8f4e29764..4b6d3a80730 100644 --- a/tensorflow/python/keras/distribute/distribute_strategy_test.py +++ b/tensorflow/python/keras/distribute/distribute_strategy_test.py @@ -1239,7 +1239,7 @@ class TestDistributionStrategyWithDatasets(test.TestCase, dataset = dataset.repeat(100) dataset = dataset.batch(10) - with self.assertRaisesRegex(ValueError, 'incompatible with the layer'): + with self.assertRaisesRegex(ValueError, 'is incompatible with'): model.fit(dataset, epochs=1, steps_per_epoch=2, verbose=0) @combinations.generate( diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index a63d499400a..d7cc3fd38a8 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -970,12 +970,11 @@ class Layer(module.Module, version_utils.LayerVersionSelector): if self._autocast: inputs = self._maybe_cast_inputs(inputs, input_list) + input_spec.assert_input_compatibility(self.input_spec, inputs, self.name) if eager: call_fn = self.call name_scope = self._name else: - input_spec.assert_input_compatibility(self.input_spec, inputs, - self.name) name_scope = self._name_scope() # Avoid autoincrementing. call_fn = self._autographed_call() diff --git a/tensorflow/python/keras/engine/functional.py b/tensorflow/python/keras/engine/functional.py index 707dedac028..d2592ac1c42 100644 --- a/tensorflow/python/keras/engine/functional.py +++ b/tensorflow/python/keras/engine/functional.py @@ -33,6 +33,7 @@ from tensorflow.python.keras import backend from tensorflow.python.keras.engine import base_layer from tensorflow.python.keras.engine import base_layer_utils from tensorflow.python.keras.engine import input_layer as input_layer_module +from tensorflow.python.keras.engine import input_spec from tensorflow.python.keras.engine import keras_tensor from tensorflow.python.keras.engine import node as node_module from tensorflow.python.keras.engine import training as training_lib @@ -248,6 +249,32 @@ class Functional(training_lib.Model): """ return nest.map_structure(backend.int_shape, self.input) + @property + def input_spec(self): + if hasattr(self, '_manual_input_spec'): + return self._manual_input_spec + if (isinstance(self._nested_inputs, (dict, list, tuple)) and + len(self._nested_inputs) != len(self.inputs)): + # Case where we have a nested structure. + # In such a case we can't safely run any checks. + return None + if isinstance(self._nested_inputs, dict): + # Case where `_nested_inputs` is a plain dict of Inputs. + names = sorted(self._nested_inputs.keys()) + return [input_spec.InputSpec( + shape=shape_with_no_batch_size(self._nested_inputs[name]), + allow_last_axis_squeeze=True, name=name) for name in names] + else: + # Single input, or list / tuple of inputs. + # The data may be passed as a dict keyed by input name. + return [input_spec.InputSpec( + shape=shape_with_no_batch_size(x), allow_last_axis_squeeze=True, + name=x._keras_history.layer.name) for x in self.inputs] + + @input_spec.setter + def input_spec(self, value): + self._manual_input_spec = value + @property def output(self): """Retrieves the output tensor(s) of a layer. @@ -1312,3 +1339,12 @@ def get_network_config(network, serialize_layer_fn=None): model_outputs = tf_utils.convert_inner_node_data(model_outputs) config['output_layers'] = model_outputs return config + + +def shape_with_no_batch_size(x): + if x.shape.rank is None: + return None + shape = x.shape.as_list() + if shape: + shape[0] = None + return shape diff --git a/tensorflow/python/keras/engine/functional_test.py b/tensorflow/python/keras/engine/functional_test.py index b104668c9e1..1b6d15863e6 100644 --- a/tensorflow/python/keras/engine/functional_test.py +++ b/tensorflow/python/keras/engine/functional_test.py @@ -1059,7 +1059,7 @@ class NetworkConstructionTest(keras_parameterized.TestCase): self.assertEqual(history.history['loss'][0], 0.0) # Check the output dtype - self.assertEqual(model(array_ops.ones(3, 3)).dtype, dtypes.float16) + self.assertEqual(model(array_ops.ones((3, 10))).dtype, dtypes.float16) model = training_lib.Model.from_config( model.get_config(), custom_objects={'Double': Double}) @@ -1075,7 +1075,7 @@ class NetworkConstructionTest(keras_parameterized.TestCase): self.assertEqual(history.history['loss'][0], 0.0) # Check the output dtype - self.assertEqual(model(array_ops.ones(3, 3)).dtype, dtypes.float16) + self.assertEqual(model(array_ops.ones((3, 10))).dtype, dtypes.float16) @combinations.generate(combinations.keras_mode_combinations()) def test_call_kwarg_nonserializable(self): @@ -1793,8 +1793,8 @@ class NestedNetworkTest(keras_parameterized.TestCase): network = functional.Functional.from_config(network.get_config()) result_tensor = network({ - 'x': array_ops.ones((1, 1), 'float32'), - 'y': array_ops.ones((1, 1), 'float32') + 'x1': array_ops.ones((1, 1), 'float32'), + 'x2': array_ops.ones((1, 1), 'float32') }) result = self.evaluate(result_tensor) self.assertAllEqual(result, [[2.]]) @@ -2340,6 +2340,57 @@ class InputsOutputsErrorTest(keras_parameterized.TestCase): TypeError, "('Keyword argument not understood:', 'output')"): models.Model(inputs=inputs, output=outputs) + def test_input_spec(self): + if not context.executing_eagerly(): + return + inputs = input_layer_lib.Input((10,)) + outputs = layers.Dense(10)(inputs) + model = models.Model(inputs, outputs) + with self.assertRaisesRegex( + ValueError, r'.*expected shape=.*'): + model(np.zeros((3, 11))) + + def test_input_spec_list_of_inputs(self): + if not context.executing_eagerly(): + return + input_1 = input_layer_lib.Input((10,), name='1') + input_2 = input_layer_lib.Input((5,), name='2') + x = layers.Concatenate()([input_1, input_2]) + outputs = layers.Dense(10)(x) + model = models.Model([input_1, input_2], outputs) + with self.assertRaisesRegex( + ValueError, r'.*expects 2 input.*'): + model(np.zeros((3, 10))) + with self.assertRaisesRegex( + ValueError, r'.*expects 2 input.*'): + model([np.zeros((3, 10)), np.zeros((3, 5)), np.zeros((3, 10))]) + with self.assertRaisesRegex( + ValueError, r'.*expected shape=.*'): + model([np.zeros((3, 10)), np.zeros((3, 6))]) + + # Test passing data via dict keyed by input name + with self.assertRaisesRegex( + ValueError, r'Missing data for input.*'): + model({'1': np.zeros((3, 10))}) + with self.assertRaisesRegex( + ValueError, r'.*expected shape=.*'): + model({'1': np.zeros((3, 10)), '2': np.zeros((3, 6))}) + + def test_input_spec_dict(self): + if not context.executing_eagerly(): + return + input_1 = input_layer_lib.Input((10,)) + input_2 = input_layer_lib.Input((5,)) + x = layers.Concatenate()([input_1, input_2]) + outputs = layers.Dense(10)(x) + model = models.Model({'1': input_1, '2': input_2}, outputs) + with self.assertRaisesRegex( + ValueError, r'Missing data for input.*'): + model({'1': np.zeros((3, 10))}) + with self.assertRaisesRegex( + ValueError, r'.*expected shape=.*'): + model({'1': np.zeros((3, 10)), '2': np.zeros((3, 6))}) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/keras/engine/input_spec.py b/tensorflow/python/keras/engine/input_spec.py index b57b2974aae..52a2829ffdb 100644 --- a/tensorflow/python/keras/engine/input_spec.py +++ b/tensorflow/python/keras/engine/input_spec.py @@ -44,14 +44,32 @@ class InputSpec(object): a None shape is compatible with any shape. Arguments: - dtype: Expected DataType of the input. - shape: Shape tuple, expected shape of the input - (may include None for unchecked axes). - ndim: Integer, expected rank of the input. - max_ndim: Integer, maximum rank of the input. - min_ndim: Integer, minimum rank of the input. - axes: Dictionary mapping integer axes to - a specific dimension value. + dtype: Expected DataType of the input. + shape: Shape tuple, expected shape of the input + (may include None for unchecked axes). Includes the batch size. + ndim: Integer, expected rank of the input. + max_ndim: Integer, maximum rank of the input. + min_ndim: Integer, minimum rank of the input. + axes: Dictionary mapping integer axes to + a specific dimension value. + allow_last_axis_squeeze: If True, then allow inputs of rank N+1 as long + as the last axis of the input is 1, as well as inputs of rank N-1 + as long as the last axis of the spec is 1. + name: Expected key corresponding to this input when passing data as + a dictionary. + + Example: + + ```python + class MyLayer(Layer): + def __init__(self): + super(MyLayer, self).__init__() + # The layer will accept inputs with shape (?, 28, 28) & (?, 28, 28, 1) + # and raise an appropriate error message otherwise. + self.input_spec = InputSpec( + shape=(None, 28, 28, 1), + allow_last_axis_squeeze=True) + ``` """ def __init__(self, @@ -60,8 +78,15 @@ class InputSpec(object): ndim=None, max_ndim=None, min_ndim=None, - axes=None): + axes=None, + allow_last_axis_squeeze=False, + name=None): self.dtype = dtypes.as_dtype(dtype).name if dtype is not None else None + shape = tensor_shape.TensorShape(shape) + if shape.rank is None: + shape = None + else: + shape = tuple(shape.as_list()) if shape is not None: self.ndim = len(shape) self.shape = shape @@ -70,6 +95,8 @@ class InputSpec(object): self.shape = None self.max_ndim = max_ndim self.min_ndim = min_ndim + self.name = name + self.allow_last_axis_squeeze = allow_last_axis_squeeze try: axes = axes or {} self.axes = {int(k): axes[k] for k in axes} @@ -149,6 +176,21 @@ def assert_input_compatibility(input_spec, inputs, layer_name): if not input_spec: return + input_spec = nest.flatten(input_spec) + if isinstance(inputs, dict): + # Flatten `inputs` by reference order if input spec names are provided + names = [spec.name for spec in input_spec] + if all(names): + list_inputs = [] + for name in names: + if name not in inputs: + raise ValueError('Missing data for input "%s". ' + 'You passed a data dictionary with keys %s. ' + 'Expected the following keys: %s' % + (name, list(inputs.keys()), names)) + list_inputs.append(inputs[name]) + inputs = list_inputs + inputs = nest.flatten(inputs) for x in inputs: # Having a shape/dtype is the only commonality of the various tensor-like @@ -157,81 +199,83 @@ def assert_input_compatibility(input_spec, inputs, layer_name): # have a `shape` attribute. if not hasattr(x, 'shape'): raise TypeError('Inputs to a layer should be tensors. Got: %s' % (x,)) - input_spec = nest.flatten(input_spec) + if len(inputs) != len(input_spec): raise ValueError('Layer ' + layer_name + ' expects ' + - str(len(input_spec)) + ' inputs, ' + str(len(input_spec)) + ' input(s), ' 'but it received ' + str(len(inputs)) + ' input tensors. Inputs received: ' + str(inputs)) for input_index, (x, spec) in enumerate(zip(inputs, input_spec)): if spec is None: continue - if (spec.ndim is not None or - spec.min_ndim is not None or - spec.max_ndim is not None): - if x.shape.ndims is None: - raise ValueError('Input ' + str(input_index) + ' of layer ' + - layer_name + ' is incompatible with the layer: ' - 'its rank is undefined, but the layer requires a ' - 'defined rank.') - + shape = tensor_shape.TensorShape(x.shape) + if shape.rank is None: + return # Check ndim. - if spec.ndim is not None: - ndim = x.shape.ndims + if spec.ndim is not None and not spec.allow_last_axis_squeeze: + ndim = shape.rank if ndim != spec.ndim: raise ValueError('Input ' + str(input_index) + ' of layer ' + layer_name + ' is incompatible with the layer: ' 'expected ndim=' + str(spec.ndim) + ', found ndim=' + str(ndim) + '. Full shape received: ' + - str(x.shape.as_list())) + str(tuple(shape))) if spec.max_ndim is not None: - ndim = x.shape.ndims + ndim = x.shape.rank if ndim is not None and ndim > spec.max_ndim: raise ValueError('Input ' + str(input_index) + ' of layer ' + layer_name + ' is incompatible with the layer: ' 'expected max_ndim=' + str(spec.max_ndim) + ', found ndim=' + str(ndim)) if spec.min_ndim is not None: - ndim = x.shape.ndims + ndim = x.shape.rank if ndim is not None and ndim < spec.min_ndim: raise ValueError('Input ' + str(input_index) + ' of layer ' + layer_name + ' is incompatible with the layer: ' ': expected min_ndim=' + str(spec.min_ndim) + ', found ndim=' + str(ndim) + '. Full shape received: ' + - str(x.shape.as_list())) + str(tuple(shape))) # Check dtype. if spec.dtype is not None: - if x.dtype != spec.dtype: + if x.dtype.name != spec.dtype: raise ValueError('Input ' + str(input_index) + ' of layer ' + layer_name + ' is incompatible with the layer: ' 'expected dtype=' + str(spec.dtype) + ', found dtype=' + str(x.dtype)) + # Check specific shape axes. + shape_as_list = shape.as_list() if spec.axes: - shape = x.shape.as_list() - if shape is not None: - for axis, value in spec.axes.items(): - if hasattr(value, 'value'): - value = value.value - if value is not None and shape[int(axis)] not in {value, None}: - raise ValueError( - 'Input ' + str(input_index) + ' of layer ' + layer_name + ' is' - ' incompatible with the layer: expected axis ' + str(axis) + - ' of input shape to have value ' + str(value) + - ' but received input with shape ' + str(shape)) + for axis, value in spec.axes.items(): + if hasattr(value, 'value'): + value = value.value + if value is not None and shape_as_list[int(axis)] not in {value, None}: + raise ValueError( + 'Input ' + str(input_index) + ' of layer ' + layer_name + ' is' + ' incompatible with the layer: expected axis ' + str(axis) + + ' of input shape to have value ' + str(value) + + ' but received input with shape ' + display_shape(x.shape)) # Check shape. - if spec.shape is not None: - shape = x.shape.as_list() - if shape is not None: - for spec_dim, dim in zip(spec.shape, shape): - if spec_dim is not None and dim is not None: - if spec_dim != dim: - raise ValueError('Input ' + str(input_index) + - ' is incompatible with layer ' + layer_name + - ': expected shape=' + str(spec.shape) + - ', found shape=' + str(shape)) + if spec.shape is not None and shape.rank is not None: + spec_shape = spec.shape + if spec.allow_last_axis_squeeze: + if shape_as_list and shape_as_list[-1] == 1: + shape_as_list = shape_as_list[:-1] + if spec_shape and spec_shape[-1] == 1: + spec_shape = spec_shape[:-1] + for spec_dim, dim in zip(spec_shape, shape_as_list): + if spec_dim is not None and dim is not None: + if spec_dim != dim: + raise ValueError('Input ' + str(input_index) + + ' is incompatible with layer ' + layer_name + + ': expected shape=' + str(spec.shape) + + ', found shape=' + display_shape(x.shape)) + + +def display_shape(shape): + return str(tuple(shape.as_list())) def to_tensor_spec(input_spec, default_dtype=None): diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py index 595757672ce..e22c4921102 100644 --- a/tensorflow/python/keras/engine/sequential.py +++ b/tensorflow/python/keras/engine/sequential.py @@ -495,10 +495,16 @@ class Sequential(functional.Functional): @property def input_spec(self): + if hasattr(self, '_manual_input_spec'): + return self._manual_input_spec if self.layers and hasattr(self.layers[0], 'input_spec'): return self.layers[0].input_spec return None + @input_spec.setter + def input_spec(self, value): + self._manual_input_spec = value + @property def _trackable_saved_model_saver(self): return model_serialization.SequentialSavedModelSaver(self) diff --git a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py index 18eb82624c1..e128323a1a6 100644 --- a/tensorflow/python/keras/layers/tensorflow_op_layer_test.py +++ b/tensorflow/python/keras/layers/tensorflow_op_layer_test.py @@ -324,9 +324,9 @@ class AutoLambdaTest(keras_parameterized.TestCase): run_eagerly=testing_utils.should_run_eagerly()) np_inputs = nest.map_structure( - lambda x: np.ones((10,) + tuple(x.shape[1:]), 'float32'), model.inputs) + lambda x: np.ones((2,) + tuple(x.shape[1:]), 'float32'), model.inputs) np_outputs = nest.map_structure( - lambda x: np.ones((10,) + tuple(x.shape[1:]), 'float32'), model.outputs) + lambda x: np.ones((2,) + tuple(x.shape[1:]), 'float32'), model.outputs) model.fit(np_inputs, np_outputs, batch_size=2) model(np_inputs) # Test calling the model directly on inputs. @@ -402,7 +402,7 @@ class AutoLambdaTest(keras_parameterized.TestCase): def test_getitem_slice_with_step_only(self): if not context.executing_eagerly(): self.skipTest('Complex slicing like this fails in v1') - inp = keras.Input(shape=(4, 3, 8)) + inp = keras.Input(shape=(8,)) slice_step = keras.Input(shape=(), dtype='int32') out = inp[..., ::slice_step[0]] @@ -508,7 +508,7 @@ class AutoLambdaTest(keras_parameterized.TestCase): def test_getitem_slice_with_stop_only(self): if not context.executing_eagerly(): self.skipTest('Complex slicing like this fails in v1') - inp = keras.Input(shape=(4, 3, 8)) + inp = keras.Input(shape=(8,)) slice_stop = keras.Input(shape=(), dtype='int32') out = inp[:slice_stop[0]] @@ -544,7 +544,7 @@ class AutoLambdaTest(keras_parameterized.TestCase): def test_getitem_slice_with_stop_and_ellipsis_only(self): if not context.executing_eagerly(): self.skipTest('Complex slicing like this fails in v1') - inp = keras.Input(shape=(4, 3, 8)) + inp = keras.Input(shape=(8,)) slice_stop = keras.Input(shape=(), dtype='int32') out = inp[..., :slice_stop[0]] @@ -646,14 +646,14 @@ class AutoLambdaTest(keras_parameterized.TestCase): def test_numerical_correctness_with_attrs(self): x = ops.convert_to_tensor_v2([[1.5, 1.5], [2.5, 3.5]]) - inputs = keras.Input(shape=(10,)) + inputs = keras.Input(shape=(2,)) outputs = math_ops.reduce_mean(inputs, axis=1) model = keras.Model(inputs, outputs) y = self.evaluate(model(x)) self.assertAllClose(y, [1.5, 3.]) def test_numerical_correctness_serialization(self): - x = ops.convert_to_tensor_v2([-1., 0., -2., 1.]) + x = ops.convert_to_tensor_v2([[-1., 0., -2., 1.]]) inputs = keras.Input(shape=(4,)) outputs = gen_nn_ops.relu(inputs) model1 = keras.Model(inputs, outputs) diff --git a/tensorflow/python/keras/legacy_tf_layers/base_test.py b/tensorflow/python/keras/legacy_tf_layers/base_test.py index 36be60f7657..2c9810c4109 100644 --- a/tensorflow/python/keras/legacy_tf_layers/base_test.py +++ b/tensorflow/python/keras/legacy_tf_layers/base_test.py @@ -277,11 +277,6 @@ class BaseLayerTest(test.TestCase, parameterized.TestCase): def call(self, inputs): return inputs - if not context.executing_eagerly(): - layer = CustomerLayer() - with self.assertRaisesRegex(ValueError, r'requires a defined rank'): - layer.apply(array_ops.placeholder('int32')) - layer = CustomerLayer() with self.assertRaisesRegex(ValueError, r'expected ndim=2'): layer.apply(constant_op.constant([1])) @@ -295,29 +290,24 @@ class BaseLayerTest(test.TestCase, parameterized.TestCase): @combinations.generate(combinations.combine(mode=['graph', 'eager'])) def testInputSpecMinNdimCheck(self): - class CustomerLayer(base_layers.Layer): + class CustomLayer(base_layers.Layer): def __init__(self): - super(CustomerLayer, self).__init__() + super(CustomLayer, self).__init__() self.input_spec = input_spec.InputSpec(min_ndim=2) def call(self, inputs): return inputs - if not context.executing_eagerly(): - layer = CustomerLayer() - with self.assertRaisesRegex(ValueError, r'requires a defined rank'): - layer.apply(array_ops.placeholder('int32')) - - layer = CustomerLayer() + layer = CustomLayer() with self.assertRaisesRegex(ValueError, r'expected min_ndim=2'): layer.apply(constant_op.constant([1])) # Works - layer = CustomerLayer() + layer = CustomLayer() layer.apply(constant_op.constant([[1], [2]])) - layer = CustomerLayer() + layer = CustomLayer() layer.apply(constant_op.constant([[[1], [2]]])) @combinations.generate(combinations.combine(mode=['graph', 'eager'])) @@ -332,11 +322,6 @@ class BaseLayerTest(test.TestCase, parameterized.TestCase): def call(self, inputs): return inputs - if not context.executing_eagerly(): - layer = CustomerLayer() - with self.assertRaisesRegex(ValueError, r'requires a defined rank'): - layer.apply(array_ops.placeholder('int32')) - layer = CustomerLayer() with self.assertRaisesRegex(ValueError, r'expected max_ndim=2'): layer.apply(constant_op.constant([[[1], [2]]])) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt index fce381e8a0a..c95e3135df8 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-input-spec.pbtxt @@ -4,7 +4,7 @@ tf_class { is_instance: "" member_method { name: "__init__" - argspec: "args=[\'self\', \'dtype\', \'shape\', \'ndim\', \'max_ndim\', \'min_ndim\', \'axes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'dtype\', \'shape\', \'ndim\', \'max_ndim\', \'min_ndim\', \'axes\', \'allow_last_axis_squeeze\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\'], " } member_method { name: "from_config" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt index 17b89c29fb2..75dbd5e386a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.layers.-input-spec.pbtxt @@ -4,7 +4,7 @@ tf_class { is_instance: "" member_method { name: "__init__" - argspec: "args=[\'self\', \'dtype\', \'shape\', \'ndim\', \'max_ndim\', \'min_ndim\', \'axes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'dtype\', \'shape\', \'ndim\', \'max_ndim\', \'min_ndim\', \'axes\', \'allow_last_axis_squeeze\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\'], " } member_method { name: "from_config" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt index fce381e8a0a..c95e3135df8 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-input-spec.pbtxt @@ -4,7 +4,7 @@ tf_class { is_instance: "" member_method { name: "__init__" - argspec: "args=[\'self\', \'dtype\', \'shape\', \'ndim\', \'max_ndim\', \'min_ndim\', \'axes\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'dtype\', \'shape\', \'ndim\', \'max_ndim\', \'min_ndim\', \'axes\', \'allow_last_axis_squeeze\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'False\', \'None\'], " } member_method { name: "from_config" From 60c8033ebc325550cae779c393f9e4c73108a75e Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Mon, 3 Aug 2020 10:17:46 -0700 Subject: [PATCH 0255/1017] Add support for token operands to mhlo.tuple. mhlo.get_tuple_element supports extracting a mhlo.token type from a tuple. This updates the creation of tuples to allow for mhlo.token typed operands. PiperOrigin-RevId: 324628663 Change-Id: I18c77aabdfcb2d84ae70d49e85a52d751bc962c2 --- .../mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td | 2 +- tensorflow/compiler/mlir/hlo/tests/ops.mlir | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td index db98bd16f76..e83bf874c62 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td @@ -664,7 +664,7 @@ def HLO_GetTupleElementOp: HLO_Op<"get_tuple_element", [NoSideEffect]>, BASE_HLO } def HLO_TupleOp : HLO_Op<"tuple", [NoSideEffect]>, BASE_HLO_TupleOp { - let arguments = (ins Variadic:$val); + let arguments = (ins Variadic:$val); let results = (outs HLO_Tuple); let builders = [OpBuilder< diff --git a/tensorflow/compiler/mlir/hlo/tests/ops.mlir b/tensorflow/compiler/mlir/hlo/tests/ops.mlir index 920e62e57b4..212e79432b1 100644 --- a/tensorflow/compiler/mlir/hlo/tests/ops.mlir +++ b/tensorflow/compiler/mlir/hlo/tests/ops.mlir @@ -847,6 +847,13 @@ func @tuple(%arg0: tensor<1xi32>, %arg1: tensor<1x2xf32>) -> tuple // ----- +func @tuple_token(%arg0: tensor, %arg1: !mhlo.token) -> tuple, !mhlo.token> { + %0 = "mhlo.tuple"(%arg0, %arg1) : (tensor, !mhlo.token) -> tuple, !mhlo.token> + return %0 : tuple, !mhlo.token> +} + +// ----- + func @tuple_arg_size_mismatch(%arg0: tensor, %arg1: tensor) -> tuple, tensor, tensor> { // expected-error@+1 {{has return type tuple, tensor, tensor>, but expected tuple, tensor>}} %0 = "mhlo.tuple"(%arg0, %arg1) : (tensor, tensor) -> tuple, tensor, tensor> From 74d526257013ee74c34e6e48cd52e650b4bde6ec Mon Sep 17 00:00:00 2001 From: Robert David Date: Mon, 3 Aug 2020 10:24:31 -0700 Subject: [PATCH 0256/1017] Use workgroup-local reductions for MeanStdDevNormalization. PiperOrigin-RevId: 324630237 Change-Id: Ie0fe32a072039809b7b1b51bbeda8665e7f1a5ce --- .../cl/kernels/mean_stddev_normalization.cc | 76 ++++++++++++++++--- .../cl/kernels/mean_stddev_normalization.h | 3 + 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc index fb206cc0692..a6ce7e55253 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc @@ -24,9 +24,55 @@ limitations under the License. namespace tflite { namespace gpu { namespace cl { +namespace { + +std::string GetVectorReduceCode() { + return R"(static inline float reduce_vector(float4 v) { + return dot(v, (float4)(1.0f)); +})"; +} + +std::string GetReduceCode(size_t work_group_size_x, size_t work_group_size_y) { + // If it is supported, use the built-in work_group_reduce_add function. + // Otherwise, implement a reduction using __local memory. Note this only works + // with power-of-two work group sizes. + return R"( +static inline float local_reduce(float input) { +#if (__OPENCL_C_VERSION__ >= 300 && __opencl_c_work_group_collective_functions) || \ + (__OPENCL_C_VERSION__ >= 200) + return work_group_reduce_add(input); +#else + __local float data[)" + + std::to_string(work_group_size_y) + "][" + + std::to_string(work_group_size_x) + R"(]; + const size_t local_id_x = get_local_id(0); + const size_t local_id_y = get_local_id(1); + data[local_id_y][local_id_x] = input; + mem_fence(CLK_LOCAL_MEM_FENCE); + size_t reduction_size = get_local_size(0) / 2; + while (reduction_size > 0) { + if (local_id_x < reduction_size) { + data[local_id_y][local_id_x] += data[local_id_y][local_id_x + reduction_size]; + } + mem_fence(CLK_LOCAL_MEM_FENCE); + reduction_size /= 2; + } + return data[local_id_y][0]; +} +#endif +)"; +} +} // namespace MeanStdDevNormalization::MeanStdDevNormalization(const OperationDef& definition) : GPUOperation(definition) { + // The kernel code does not inherently need a fixed size, but in order to not + // hardcode the __local array's size for the reductions, we would need to pass + // that size to the kernel at runtime, and that is currently not supported. + // For now, fix workgroup size to 128 threads. + work_group_size_.x = 128; + work_group_size_.y = 1; + work_group_size_.z = 1; code_ = GetNormalizationCode(); } @@ -35,35 +81,43 @@ std::string MeanStdDevNormalization::GetNormalizationCode() { AddDstTensor("dst_tensor", definition_.dst_tensors[0]); std::string c = GetCommonDefines(definition_.precision); - c += R"(__kernel void main_function( + c += GetVectorReduceCode(); + c += GetReduceCode(work_group_size_.x, work_group_size_.y); + c += R"(__attribute__((reqd_work_group_size(128, 1, 1))) +__kernel void main_function( $0) { - if (get_global_id(0) > 0) { return; } size_t B = get_global_id(1); if (get_global_id(2) > 0) { return; } if (B >= args.src_tensor.Batch()) { return; } // Calculate the total sum of the input tensor. // First, get a local sum of input[local_id_x + N*local_size_x] for all N. - float sum = 0.0f; - for (int S = 0; S < args.src_tensor.Slices(); ++S) { + float4 private_sum4 = (float4)(0.0f); + for (int S = get_local_id(0); S < args.src_tensor.Slices(); S += get_local_size(0)) { const float4 t = args.src_tensor.Read(0, 0, S, B); // Filter out reads beyond the end of the tensor. const int4 is_after_end_of_tensor = (int4)(0, 1, 2, 3) >= (args.src_tensor.Channels() - S * 4); const float4 filtered_t = select(t, (float4)(0.0f), is_after_end_of_tensor); - sum += filtered_t.x + filtered_t.y + filtered_t.z + filtered_t.w; + private_sum4 += filtered_t; } + // Reduce the vector to a single float and do a workgroup reduce. + const float private_sum = reduce_vector(private_sum4); + const float sum = local_reduce(private_sum); // Calculate the mean const float mean = sum / args.src_tensor.Channels(); // Calculate the squared sum of the difference from the mean. - float sum_diff_sq = 0.0f; - for (int S = 0; S < args.src_tensor.Slices(); ++S) { + float4 private_sum_diff_sq4 = (float4)(0.0f); + for (int S = get_local_id(0); S < args.src_tensor.Slices(); S += get_local_size(0)) { const float4 t = args.src_tensor.Read(0, 0, S, B); const float4 diff = t - mean; // Filter out reads beyond the end of the tensor. const int4 is_after_end_of_tensor = (int4)(0, 1, 2, 3) >= (args.src_tensor.Channels() - S * 4); const float4 filtered_diff = select(diff, (float4)(0.0f), is_after_end_of_tensor); - float dotprod = dot(filtered_diff, filtered_diff); - sum_diff_sq += dotprod; + // sum_diff_sq += diff² + private_sum_diff_sq4 = mad(filtered_diff, filtered_diff, private_sum_diff_sq4); } + // Reduce + const float private_sum_diff_sq = reduce_vector(private_sum_diff_sq4); + const float sum_diff_sq = local_reduce(private_sum_diff_sq); // Calculate 1/stddev (with the 'regulazing constant' as in tensor_utils.cc) const float variance = sum_diff_sq / args.src_tensor.Channels(); const float stddev_inv = rsqrt(variance + 1.0e-8f); @@ -78,7 +132,9 @@ $0) { } int3 MeanStdDevNormalization::GetGridSize() const { - const int grid_x = 1; + // To avoid dealing with global reductions, we restrict the grid size to the + // work group size in the first dimension. + const int grid_x = work_group_size_.x; const int grid_y = src_[0]->Batch(); const int grid_z = 1; return int3(grid_x, grid_y, grid_z); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.h b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.h index 5724d72bcd1..7dd45fcb86a 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.h @@ -30,6 +30,9 @@ class MeanStdDevNormalization : public GPUOperation { public: explicit MeanStdDevNormalization(const OperationDef& definition); + absl::Status Tune(const TuningParameters& params) override { + return absl::OkStatus(); + } int3 GetGridSize() const override; // Move only From b25ed5a8d47d96860e828139b90aec55a085c137 Mon Sep 17 00:00:00 2001 From: Tomer Kaftan Date: Mon, 3 Aug 2020 10:31:42 -0700 Subject: [PATCH 0257/1017] Fixes error when reconstructing functional models from config if layer outputs accessed in call **kwargs come from previously-called layers. Specifically, this may happen when cloning functional models w/ weight sharing in place. PiperOrigin-RevId: 324631738 Change-Id: I8a75b053e16116a55e2a99a8ee011f5ba52c171c --- tensorflow/python/keras/engine/functional.py | 39 ++++++++++---------- tensorflow/python/keras/models_test.py | 33 +++++++++++++---- 2 files changed, 45 insertions(+), 27 deletions(-) diff --git a/tensorflow/python/keras/engine/functional.py b/tensorflow/python/keras/engine/functional.py index d2592ac1c42..71d6faa71b6 100644 --- a/tensorflow/python/keras/engine/functional.py +++ b/tensorflow/python/keras/engine/functional.py @@ -1056,26 +1056,6 @@ def _should_skip_first_node(layer): isinstance(layer._layers[0], input_layer_module.InputLayer)) -def _deserialize_keras_tensors(kwargs, layer_map): - """Deserializes Keras Tensors passed to `call`..""" - - def _deserialize_keras_tensor(t): - """Deserializes a single Keras Tensor passed to `call`.""" - if isinstance(t, tf_utils.ListWrapper): - t = t.as_list() - layer_name = t[0] - node_index = t[1] - tensor_index = t[2] - - layer = layer_map[layer_name] - node = layer._inbound_nodes[node_index] - return nest.flatten(node.outputs)[tensor_index] - return t - - kwargs = tf_utils.convert_inner_node_data(kwargs, wrap=True) - return nest.map_structure(_deserialize_keras_tensor, kwargs) - - def connect_ancillary_layers(model, created_layers): """Adds layers that are not connected to the outputs to the model.""" # Layers not connected to outputs, such as those added in `add_loss`. @@ -1135,6 +1115,25 @@ def reconstruct_from_config(config, custom_objects=None, created_layers=None): return 0 return node_index_map.get((layer.name, config_node_index), None) + def _deserialize_keras_tensors(kwargs, layer_map): + """Deserializes Keras Tensors passed to `call`..""" + + def _deserialize_keras_tensor(t): + """Deserializes a single Keras Tensor passed to `call`.""" + if isinstance(t, tf_utils.ListWrapper): + t = t.as_list() + layer_name = t[0] + node_index = t[1] + tensor_index = t[2] + + layer = layer_map[layer_name] + node = layer._inbound_nodes[get_node_index(layer, node_index)] + return nest.flatten(node.outputs)[tensor_index] + return t + + kwargs = tf_utils.convert_inner_node_data(kwargs, wrap=True) + return nest.map_structure(_deserialize_keras_tensor, kwargs) + def process_node(layer, node_data): """Deserialize a node. diff --git a/tensorflow/python/keras/models_test.py b/tensorflow/python/keras/models_test.py index ea0dc148326..8411ed0d3ea 100644 --- a/tensorflow/python/keras/models_test.py +++ b/tensorflow/python/keras/models_test.py @@ -278,9 +278,20 @@ class TestModelCloning(keras_parameterized.TestCase): has_placeholder = _has_placeholder(graph) self.assertFalse(has_placeholder) - def test_functional_cloning_with_tensor_kwarg(self): + @keras_parameterized.run_all_keras_modes + @parameterized.named_parameters([ + {'testcase_name': 'clone_weights', 'share_weights': False}, + {'testcase_name': 'share_weights', 'share_weights': True}, + ]) + def test_functional_cloning_with_tensor_kwarg(self, share_weights): """Test that cloning works with models that use Tensor kwargs.""" + if share_weights: + clone_fn = functools.partial( + keras.models.clone_model, clone_function=models.share_weights) + else: + clone_fn = keras.models.clone_model + class LayerWithTensorKwarg(keras.layers.Layer): def call(self, inputs, tensor=None): @@ -295,13 +306,21 @@ class TestModelCloning(keras_parameterized.TestCase): model.add_loss(math_ops.reduce_sum(model.outputs)) input_arr = np.random.random((1, 3)).astype(np.float32) - with ops.Graph().as_default(): - with self.session() as sess: - clone = keras.models.clone_model(model) - self.assertLen(clone.losses, 1) + clone = clone_fn(model) - loss = sess.run(clone.losses[0], feed_dict={clone.input: input_arr}) - self.assertAllClose(np.sum(input_arr), loss) + if context.executing_eagerly(): + clone(input_arr) + loss = clone.losses[0] + else: + with self.session() as sess: + clone(input_arr) + if share_weights: + self.skipTest('Weight sharing with inputs in call **kwargs does ' + 'not work correctly in v1') + else: + feed_dict = {clone.input: input_arr} + loss = sess.run(clone.losses[0], feed_dict=feed_dict) + self.assertAllClose(np.sum(input_arr), loss) def _has_placeholder(graph): From 88ee42a7e7a8ac0dba3968fdb48c3c79324611e5 Mon Sep 17 00:00:00 2001 From: Kaixi Hou Date: Fri, 26 Jun 2020 15:13:28 -0700 Subject: [PATCH 0258/1017] Layout grappler for NDHWC Conv3D --- tensorflow/core/grappler/op_types.cc | 8 ++ tensorflow/core/grappler/op_types.h | 2 + .../generic_layout_optimizer_transposer.cc | 86 ++++++++++++- .../generic_layout_optimizer_transposer.h | 24 ++++ ...ric_layout_optimizer_transposer_factory.cc | 11 ++ tensorflow/core/kernels/data_format_ops.cc | 5 +- .../python/grappler/layout_optimizer_test.py | 121 ++++++++++++++++++ 7 files changed, 253 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc index 9d30f24e047..6b961c1e18f 100644 --- a/tensorflow/core/grappler/op_types.cc +++ b/tensorflow/core/grappler/op_types.cc @@ -186,6 +186,14 @@ bool IsConv2DBackpropInput(const NodeDef& node) { bool IsConv3D(const NodeDef& node) { return node.op() == "Conv3D"; } +bool IsConv3DBackpropFilterV2(const NodeDef& node) { + return node.op() == "Conv3DBackpropFilterV2"; +} + +bool IsConv3DBackpropInputV2(const NodeDef& node) { + return node.op() == "Conv3DBackpropInputV2"; +} + bool IsDepthwiseConv2dNative(const NodeDef& node) { return node.op() == "DepthwiseConv2dNative"; } diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h index 141eda7415a..1bf26721847 100644 --- a/tensorflow/core/grappler/op_types.h +++ b/tensorflow/core/grappler/op_types.h @@ -63,6 +63,8 @@ bool IsConv2D(const NodeDef& node); bool IsConv2DBackpropFilter(const NodeDef& node); bool IsConv2DBackpropInput(const NodeDef& node); bool IsConv3D(const NodeDef& node); +bool IsConv3DBackpropFilterV2(const NodeDef& node); +bool IsConv3DBackpropInputV2(const NodeDef& node); bool IsDepthwiseConv2dNative(const NodeDef& node); bool IsDepthwiseConv2dNativeBackpropFilter(const NodeDef& node); bool IsDepthwiseConv2dNativeBackpropInput(const NodeDef& node); diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc index 9d702971dd7..2ac47ec36a4 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.cc @@ -241,7 +241,7 @@ Status Transposer::CreateConstPermNode(TransposeContext* context, node.mutable_attr()->insert({"dtype", attr_data_type}); AttrValue attr_tensor; - Tensor tensor(DT_INT32, TensorShape({4})); + Tensor tensor(DT_INT32, TensorShape({permutation.size()})); for (int i = 0, end = permutation.size(); i < end; i++) { tensor.flat()(i) = permutation[i]; } @@ -752,6 +752,86 @@ Status Conv2DBackpropInputTransposer::TransposeNode( return context->graph_view->GetMutationBuilder()->Apply(); } +Status Conv3DTransposer::TransposeNode( + TransposeContext* context, utils::MutableNodeView* node) { + DCHECK(IsConv3D(*node->node())); + // Update the format from 4D to 5D layout. + std::string src_format = context->src_format; + std::string dst_format = context->dst_format; + std::string src_format_3d = src_format == "NHWC" ? "NDHWC": "NCDHW"; + std::string dst_format_3d = dst_format == "NHWC" ? "NDHWC": "NCDHW"; + context->AssignDeviceAndDataFormats(context->target_device, src_format_3d, + dst_format_3d); + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 5)) { + return Status::OK(); + } + VLOG(3) << "GenericLayoutOptimizer: transforming node '" << node->GetName() + << "' with op '" << node->GetOp() << "' from data format '" + << context->src_format << "' to '" << context->dst_format << "'"; + TF_RETURN_IF_ERROR(UpdateNode(context, node)); + TF_RETURN_IF_ERROR(UpdateFaninEdgesWithOp(context, {0}, node, kOpTranspose)); + TF_RETURN_IF_ERROR(UpdateFanoutEdgesWithOp(context, {0}, node, kOpTranspose)); + // Change back the format from 5D to 4D layout. + context->AssignDeviceAndDataFormats(context->target_device, src_format, + dst_format); + return context->graph_view->GetMutationBuilder()->Apply(); +} + +Status Conv3DBackpropFilterTransposer::TransposeNode( + TransposeContext* context, utils::MutableNodeView* node) { + DCHECK(IsConv3DBackpropFilterV2(*node->node())); + // Update the format from 4D to 5D layout. + std::string src_format = context->src_format; + std::string dst_format = context->dst_format; + std::string src_format_3d = src_format == "NHWC" ? "NDHWC": "NCDHW"; + std::string dst_format_3d = dst_format == "NHWC" ? "NDHWC": "NCDHW"; + context->AssignDeviceAndDataFormats(context->target_device, src_format_3d, + dst_format_3d); + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 5)) { + return Status::OK(); + } + VLOG(3) << "GenericLayoutOptimizer: transforming node '" << node->GetName() + << "' with op '" << node->GetOp() << "' from data format '" + << context->src_format << "' to '" << context->dst_format << "'"; + TF_RETURN_IF_ERROR(UpdateNode(context, node)); + TF_RETURN_IF_ERROR( + UpdateFaninEdgesWithOp(context, {0, 2}, node, kOpTranspose)); + // No need to update output shape, as it is always of shape + // [filter_height, filter_width, in_channels, out_channels], regardless of + // whether NCHW or NHWC is used. + // Change back the format from 5D to 4D layout. + context->AssignDeviceAndDataFormats(context->target_device, src_format, + dst_format); + return context->graph_view->GetMutationBuilder()->Apply(); +} + +Status Conv3DBackpropInputTransposer::TransposeNode( + TransposeContext* context, utils::MutableNodeView* node) { + DCHECK(IsConv3DBackpropInputV2(*node->node())); + // Update the format from 4D to 5D layout. + std::string src_format = context->src_format; + std::string dst_format = context->dst_format; + std::string src_format_3d = src_format == "NHWC" ? "NDHWC": "NCDHW"; + std::string dst_format_3d = dst_format == "NHWC" ? "NDHWC": "NCDHW"; + context->AssignDeviceAndDataFormats(context->target_device, src_format_3d, + dst_format_3d); + if (!ShouldProcess(*context, *node) || !IsFanoutPortRankN(*node, 0, 5)) { + return Status::OK(); + } + VLOG(3) << "GenericLayoutOptimizer: transforming node '" << node->GetName() + << "' with op '" << node->GetOp() << "' from data format '" + << context->src_format << "' to '" << context->dst_format << "'"; + TF_RETURN_IF_ERROR(UpdateNode(context, node)); + TF_RETURN_IF_ERROR( + UpdateFaninEdgesWithOp(context, {0}, node, kOpDataFormatVecPermute)); + TF_RETURN_IF_ERROR(UpdateFaninEdgesWithOp(context, {2}, node, kOpTranspose)); + TF_RETURN_IF_ERROR(UpdateFanoutEdgesWithOp(context, {0}, node, kOpTranspose)); + // Change back the format from 5D to 4D layout. + context->AssignDeviceAndDataFormats(context->target_device, src_format, + dst_format); + return context->graph_view->GetMutationBuilder()->Apply(); +} + Status FusedBatchNormExTransposer::TransposeNode(TransposeContext* context, utils::MutableNodeView* node) { DCHECK(IsFusedBatchNormEx(*node->node())); @@ -1684,7 +1764,9 @@ bool IsLayoutSensitiveOp(const NodeDef& node) { IsDepthwiseConv2dNativeBackpropInput(node) || IsFusedBatchNormEx(node) || IsFusedBatchNormGrad(node) || IsMaxPoolV2(node) || IsMaxPoolGrad(node) || IsMaxPoolGradV2(node) || - IsMaxPoolGradGradV1(node) || IsMaxPoolGradGradV2(node); + IsMaxPoolGradGradV1(node) || IsMaxPoolGradGradV2(node) || + IsConv3D(node) || IsConv3DBackpropInputV2(node) || + IsConv3DBackpropFilterV2(node); } bool IsDefaultLayoutAgnosticOp(const NodeDef& node) { diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.h b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.h index 95af7933d10..b1de9561bfc 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.h +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer.h @@ -239,6 +239,30 @@ class Conv2DBackpropInputTransposer : public LayoutSensitiveOpTransposer { utils::MutableNodeView* node) override; }; +class Conv3DTransposer : public LayoutSensitiveOpTransposer { + public: + explicit Conv3DTransposer() : LayoutSensitiveOpTransposer() {} + + Status TransposeNode(TransposeContext* context, + utils::MutableNodeView* node) override; +}; + +class Conv3DBackpropFilterTransposer: public LayoutSensitiveOpTransposer { + public: + explicit Conv3DBackpropFilterTransposer() : LayoutSensitiveOpTransposer() {} + + Status TransposeNode(TransposeContext* context, + utils::MutableNodeView* node) override; +}; + +class Conv3DBackpropInputTransposer: public LayoutSensitiveOpTransposer { + public: + explicit Conv3DBackpropInputTransposer() : LayoutSensitiveOpTransposer() {} + + Status TransposeNode(TransposeContext* context, + utils::MutableNodeView* node) override; +}; + class FusedBatchNormExTransposer : public LayoutSensitiveOpTransposer { public: explicit FusedBatchNormExTransposer() : LayoutSensitiveOpTransposer() {} diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer_factory.cc b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer_factory.cc index 59c06d42441..15bbc08079c 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer_factory.cc +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer_transposer_factory.cc @@ -43,6 +43,17 @@ std::shared_ptr TransposerFactory::GetTransposer( return GetOrCreateIfNotFound( "Conv2DBackpropInput"); } + if (IsConv3D(node)) { + return GetOrCreateIfNotFound("Conv3D"); + } + if (IsConv3DBackpropInputV2(node)) { + return GetOrCreateIfNotFound( + "Conv3DBackpropInput"); + } + if (IsConv3DBackpropFilterV2(node)) { + return GetOrCreateIfNotFound( + "Conv3DBackpropFilter"); + } if (IsFusedBatchNormEx(node)) { return GetOrCreateIfNotFound( "FusedBatchNormEx"); diff --git a/tensorflow/core/kernels/data_format_ops.cc b/tensorflow/core/kernels/data_format_ops.cc index 181aa1b8a2c..14f3ea472c3 100644 --- a/tensorflow/core/kernels/data_format_ops.cc +++ b/tensorflow/core/kernels/data_format_ops.cc @@ -90,9 +90,10 @@ class DataFormatVecPermuteOp : public OpKernel { "input must be a vector or 2D tensor, but got shape ", input.shape().DebugString())); if (input.dims() == 1) { - OP_REQUIRES(context, input.NumElements() == 2 || input.NumElements() == 4, + OP_REQUIRES(context, input.NumElements() == 2 || + input.NumElements() == 4 || input.NumElements() == 5, errors::InvalidArgument( - "1D input must be of size 2 or 4, but got shape ", + "1D input must be of size 2, 4 or 5, but got shape ", input.shape().DebugString())); } else if (input.dims() == 2) { OP_REQUIRES(context, input.dim_size(0) == 2 || input.dim_size(0) == 4, diff --git a/tensorflow/python/grappler/layout_optimizer_test.py b/tensorflow/python/grappler/layout_optimizer_test.py index 10f869805d8..9b37bc9fc1b 100644 --- a/tensorflow/python/grappler/layout_optimizer_test.py +++ b/tensorflow/python/grappler/layout_optimizer_test.py @@ -212,6 +212,12 @@ class LayoutOptimizerTest(test.TestCase): def _assert_trans_nhwc_to_nchw(self, name, nodes): self.assertIn(name + '-TransposeNHWCToNCHW-LayoutOptimizer', nodes) + def _assert_trans_ncdhw_to_ndhwc(self, name, nodes): + self.assertIn(name + '-TransposeNCDHWToNDHWC-LayoutOptimizer', nodes) + + def _assert_trans_ndhwc_to_ncdhw(self, name, nodes): + self.assertIn(name + '-TransposeNDHWCToNCDHW-LayoutOptimizer', nodes) + def _assert_map_nhwc_to_nchw(self, name, nodes): self.assertIn(name + '-DimMapNHWCToNCHW-LayoutOptimizer', nodes) @@ -221,6 +227,14 @@ class LayoutOptimizerTest(test.TestCase): def _assert_vec_nhwc_to_nchw(self, name, nodes): self.assertIn(name + '-VecPermuteNHWCToNCHW-LayoutOptimizer', nodes) + def _assert_vec_ncdhw_to_ndhwc(self, name, nodes): + self.assertIn(name + '-DataFormatVecPermuteNCDHWToNDHWC-LayoutOptimizer', + nodes) + + def _assert_vec_ndhwc_to_ncdhw(self, name, nodes): + self.assertIn(name + '-DataFormatVecPermuteNDHWCToNCDHW-LayoutOptimizer', + nodes) + def _train(self, checkpoint_path, layout_optimizer=False, restore=False): ops.reset_default_graph() graph = ops.get_default_graph() @@ -1121,6 +1135,113 @@ class LayoutOptimizerTest(test.TestCase): self.assertIn('MaxPoolGradV2-3-LayoutOptimizer', nodes) self.assertAllClose(output_val_ref, output_val, atol=1e-3) + @test_util.deprecated_graph_mode_only + def testConv3D(self): + if test.is_gpu_available(cuda_only=True): + random_seed.set_random_seed(0) + x = random_ops.truncated_normal([1, 784], seed=0) + conv = _two_layer_model(x) + filters = random_ops.truncated_normal([2, 2, 2, 1, 2], seed=0) + strides_val = [1, 1, 1, 1, 1] + x_3d = array_ops.reshape(conv, [-1, 4, 14, 14, 1]) + conv3d = gen_nn_ops.conv3d(x_3d, filters, strides_val, 'VALID') + output = array_ops.identity(conv3d) + + with session.Session(config=_get_config(False)) as sess: + output_val_ref = sess.run(output) + + with session.Session(config=_get_config()) as sess: + metadata = config_pb2.RunMetadata() + output_val = sess.run( + output, run_metadata=metadata) + + nodes = [] + num_transposes = 0 + for node in metadata.cost_graph.node: + if _is_transpose(node.name): + num_transposes += 1 + nodes.append(node.name) + + expected_num_transposes = 2 + self.assertEqual(expected_num_transposes, num_transposes) + self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) + self._assert_trans_ndhwc_to_ncdhw('Conv3D-0', nodes) + self._assert_trans_ncdhw_to_ndhwc('Conv3D-0-0', nodes) + self.assertAllClose(output_val_ref, output_val, atol=1e-3) + + @test_util.deprecated_graph_mode_only + def testConv3DBackpropInput(self): + if test.is_gpu_available(cuda_only=True): + random_seed.set_random_seed(0) + x = random_ops.truncated_normal([1, 784], seed=0) + conv = _two_layer_model(x) + x_3d = array_ops.reshape(conv, [-1, 4, 14, 14, 1]) + filters = random_ops.truncated_normal([2, 2, 2, 1, 1], seed=0) + strides_val = [1, 1, 1, 1, 1] + shape = array_ops.shape(x_3d) + conv3d_grad = gen_nn_ops.conv3d_backprop_input_v2( + shape, filters, x_3d, strides_val, 'SAME') + output = array_ops.identity(conv3d_grad) + + with session.Session(config=_get_config(False)) as sess: + output_val_ref = sess.run(output) + + with session.Session(config=_get_config()) as sess: + metadata = config_pb2.RunMetadata() + output_val = sess.run( + output, run_metadata=metadata) + + nodes = [] + num_transposes = 0 + for node in metadata.cost_graph.node: + if _is_transpose(node.name): + num_transposes += 1 + nodes.append(node.name) + + expected_num_transposes = 2 + self.assertEqual(expected_num_transposes, num_transposes) + self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) + self._assert_vec_ndhwc_to_ncdhw('Conv3DBackpropInputV2-0', nodes) + self._assert_trans_ndhwc_to_ncdhw('Conv3DBackpropInputV2-2', nodes) + self._assert_trans_ncdhw_to_ndhwc('Conv3DBackpropInputV2-0-0', nodes) + self.assertAllClose(output_val_ref, output_val, atol=1e-3) + + @test_util.deprecated_graph_mode_only + def testConv3DBackpropFilter(self): + if test.is_gpu_available(cuda_only=True): + random_seed.set_random_seed(0) + x = random_ops.truncated_normal([1, 784], seed=0) + conv = _two_layer_model(x) + x_3d = array_ops.reshape(conv, [-1, 4, 14, 14, 1]) + filters = random_ops.truncated_normal([2, 2, 2, 1, 1], seed=0) + strides_val = [1, 1, 1, 1, 1] + shape = constant_op.constant([2, 2, 2, 1, 1], shape=[5]) + conv3d_grad = gen_nn_ops.conv3d_backprop_filter_v2( + x_3d, shape, x_3d, strides_val, 'SAME') + output = array_ops.identity(conv3d_grad) + + with session.Session(config=_get_config(False)) as sess: + output_val_ref = sess.run(output) + + with session.Session(config=_get_config()) as sess: + metadata = config_pb2.RunMetadata() + output_val = sess.run( + output, run_metadata=metadata) + + nodes = [] + num_transposes = 0 + for node in metadata.cost_graph.node: + if _is_transpose(node.name): + num_transposes += 1 + nodes.append(node.name) + + expected_num_transposes = 2 + self.assertEqual(expected_num_transposes, num_transposes) + self._assert_trans_nhwc_to_nchw('Conv2D-0', nodes) + self._assert_trans_ndhwc_to_ncdhw('Conv3DBackpropFilterV2-0', nodes) + self._assert_trans_ndhwc_to_ncdhw('Conv3DBackpropFilterV2-2', nodes) + self.assertAllClose(output_val_ref, output_val, atol=1e-3) + @test_util.deprecated_graph_mode_only def testSliceWithNonConstAxis(self): if test.is_gpu_available(cuda_only=True): From d623697b83541087f759f5f5533fde2c4d86059c Mon Sep 17 00:00:00 2001 From: Kaixi Hou Date: Wed, 22 Jul 2020 15:46:32 -0700 Subject: [PATCH 0259/1017] Add Conv3D in layout opt decision --- .../optimizers/generic_layout_optimizer.cc | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer.cc b/tensorflow/core/grappler/optimizers/generic_layout_optimizer.cc index 969857879af..6053f96ae08 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer.cc @@ -38,7 +38,7 @@ namespace { constexpr char kNHWC[] = "NHWC"; constexpr char kNCHW[] = "NCHW"; constexpr float kVoltaGPURatioThreshold = 0.5; -constexpr float kConv2DGPUFP16Threshold = 0.5; +constexpr float kConvGPUFP16Threshold = 0.5; struct MutableNodeViewFormatter { void operator()(std::string* out, utils::MutableNodeView* node_view) const { @@ -69,15 +69,15 @@ inline std::pair GetNumGPUs(const Cluster& cluster) { return {num_gpus, num_volta}; } -inline bool NumConv2DOnDeviceWithDataTypeOverThreshold( +inline bool NumConvOnDeviceWithDataTypeOverThreshold( const TransposeContext& context, absl::string_view device, const DataType& data_type) { - int num_conv2d_gpu = 0; - int num_conv2d_gpu_fp16 = 0; + int num_conv_gpu = 0; + int num_conv_gpu_fp16 = 0; for (const auto& node : context.graph_view->GetNodes()) { const auto* node_def = node.node(); - if (!IsConv2D(*node_def)) { + if (!IsConv2D(*node_def) or !IsConv3D(*node_def)) { continue; } const string& device_name = @@ -89,20 +89,20 @@ inline bool NumConv2DOnDeviceWithDataTypeOverThreshold( absl::AsciiStrToLower(device))) { continue; } - num_conv2d_gpu++; + num_conv_gpu++; const auto* t_attr = node.GetAttr("T"); if (t_attr == nullptr) { continue; } if (t_attr->type() == data_type) { - num_conv2d_gpu_fp16++; + num_conv_gpu_fp16++; } } - if (num_conv2d_gpu == 0) return false; + if (num_conv_gpu == 0) return false; - return (static_cast(num_conv2d_gpu_fp16) / - static_cast(num_conv2d_gpu)) >= kConv2DGPUFP16Threshold; + return (static_cast(num_conv_gpu_fp16) / + static_cast(num_conv_gpu)) >= kConvGPUFP16Threshold; } inline std::pair GetSrcAndDstDataFormats( @@ -111,7 +111,7 @@ inline std::pair GetSrcAndDstDataFormats( string dst_format = kNCHW; if (((static_cast(num_voltas) / static_cast(num_gpus)) >= kVoltaGPURatioThreshold) && - NumConv2DOnDeviceWithDataTypeOverThreshold(context, kGPU, DT_HALF)) { + NumConvOnDeviceWithDataTypeOverThreshold(context, kGPU, DT_HALF)) { std::swap(src_format, dst_format); } return {src_format, dst_format}; From 0b68f7509ff304bf28289b7da100f4853c45dbfb Mon Sep 17 00:00:00 2001 From: Alexander Belyaev Date: Mon, 3 Aug 2020 10:55:29 -0700 Subject: [PATCH 0260/1017] [MLIR][KERNEL_GEN] Add embed-tf-framework pass. The pass rewrites the function marked with `tf_entry` attribute. * adds tf_framework::OpKernelContextType argument to the function * std.alloc becomes tf_framework.alloc_raw * std.dealloc becomes tf_framework.dealloc_raw PiperOrigin-RevId: 324636895 Change-Id: I5d733ec05c69438f4a7677573cf69155f785105e --- .../kernel_gen/tests/embed_tf_framework.mlir | 37 +++++ .../mlir/tools/kernel_gen/transforms/BUILD | 18 +++ .../transforms/embed_tf_framework.cc | 127 ++++++++++++++++++ .../transforms/embed_tf_framework_pass.cc | 77 +++++++++++ .../mlir/tools/kernel_gen/transforms/passes.h | 6 + .../tools/kernel_gen/transforms/passes.td | 5 + .../tools/kernel_gen/transforms/rewriters.h | 11 +- 7 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tests/embed_tf_framework.mlir create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework.cc create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework_pass.cc diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tests/embed_tf_framework.mlir b/tensorflow/compiler/mlir/tools/kernel_gen/tests/embed_tf_framework.mlir new file mode 100644 index 00000000000..bb0f1926cda --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/tests/embed_tf_framework.mlir @@ -0,0 +1,37 @@ +// RUN: kernel-gen-opt %s -embed-tf-framework -split-input-file | FileCheck %s + +// CHECK-LABEL: func @tf_entry( +// CHECK-SAME: [[CTX:%.*]]: !tf_framework.op_kernel_context, +// CHECK-SAME: [[SIZE_0:%.*]]: index, +// CHECK-SAME: [[SIZE_2:%.*]]: index) -> index attributes {tf_entry} { +func @tf_entry(%size_0 : index , %size_2 : index) -> index + attributes {tf_entry} { + %buf = alloc(%size_0, %size_2)[] : memref + dealloc %buf : memref + std.return %size_0 : index +} +// CHECK-NEXT: [[VAL_3:%.*]] = tf_framework.alloc_raw +// CHECK-SAME: ([[CTX]], [[SIZE_0]], [[SIZE_2]]) : memref +// CHECK-NEXT: tf_framework.dealloc_raw([[CTX]], [[VAL_3]]) : memref +// CHECK-NEXT: return [[SIZE_0]] : index + +// ----- + +// CHECK-LABEL: func @non_tf_entry( +// CHECK-SAME: [[SIZE_0:%.*]]: index, [[SIZE_2:%.*]]: index) -> index +func @non_tf_entry(%size_0 : index , %size_2 : index) -> index { + std.return %size_0 : index +} + +// ----- + +// CHECK-LABEL: func @tf_entry( +func @tf_entry(%size : index) attributes {tf_entry} { + %buf = alloc()[%size] : memref<64xf32, affine_map<(d0)[s0] -> (d0 + s0)>> + dealloc %buf : memref<64xf32, affine_map<(d0)[s0] -> (d0 + s0)>> + std.return +} +// CHECK_NOT: alloc_raw +// CHECK: alloc() +// CHECK_NOT: dealloc_raw +// CHECK: dealloc % diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD index 15c0d571e61..0119b2e46ea 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD @@ -20,6 +20,21 @@ cc_library( ], ) +cc_library( + name = "embed_tf_framework", + srcs = ["embed_tf_framework.cc"], + hdrs = ["rewriters.h"], + deps = [ + "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_ops", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:LLVMTransforms", + "@llvm-project//mlir:StandardOps", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + gentbl( name = "tf_framework_passes_inc_gen", tbl_outs = [("-gen-pass-decls", "tf_framework_passes.h.inc")], @@ -31,11 +46,13 @@ gentbl( cc_library( name = "passes", srcs = [ + "embed_tf_framework_pass.cc", "register_passes.cc", "tf_framework_legalize_to_llvm_pass.cc", ], hdrs = ["passes.h"], deps = [ + ":embed_tf_framework", ":tf_framework_legalize_to_llvm", ":tf_framework_passes_inc_gen", "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_ops", @@ -43,6 +60,7 @@ cc_library( "@llvm-project//mlir:LLVMTransforms", "@llvm-project//mlir:Pass", "@llvm-project//mlir:StandardOps", + "@llvm-project//mlir:Transforms", ], alwayslink = 1, ) diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework.cc new file mode 100644 index 00000000000..aa02aefa9d2 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework.cc @@ -0,0 +1,127 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/Function.h" // from @llvm-project +#include "mlir/IR/Module.h" // from @llvm-project +#include "mlir/IR/StandardTypes.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h" + +namespace mlir { +namespace kernel_gen { +namespace tf_framework { +namespace { + +// Prepends argument type list of the function with an OpKernelContextType arg. +class FuncOpConverter : public OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite( + FuncOp func, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + // Convert function arguments using the provided TypeConverter. + auto func_type = func.getType(); + TypeConverter::SignatureConversion conversion(func_type.getNumInputs()); + + conversion.addInputs(OpKernelContextType::get(rewriter.getContext())); + for (auto arg_type : llvm::enumerate(func_type.getInputs())) { + conversion.addInputs(arg_type.index(), arg_type.value()); + } + + TypeConverter type_converter; + if (failed(rewriter.convertRegionTypes(&func.getBody(), type_converter, + &conversion))) { + return failure(); + } + + // Update the signature of the function. + rewriter.updateRootInPlace(func, [&] { + func.setType(rewriter.getFunctionType(conversion.getConvertedTypes(), + func_type.getResults())); + }); + return success(); + } +}; + +// Converts std.alloc to tf_framework.alloc_raw using OpKernelContextType arg of +// the parent function. +class AllocOpConverter : public OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite( + AllocOp alloc, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + auto func = alloc.getParentOfType(); + if (func.getNumArguments() == 0) { + return failure(); + } + Value ctx = func.getArgument(0); + if (!ctx.getType().isa()) { + return failure(); + } + // Symbolic operands that bind to the symbols of the memref's layout map are + // not supported by AllocRawOp. + if (alloc.getNumSymbolicOperands() != 0) { + return failure(); + } + rewriter.replaceOpWithNewOp(alloc, alloc.getType(), ctx, + operands); + return success(); + } +}; + +// Converts std.dealloc to tf_framework.dealloc_raw using OpKernelContextType +// arg of the parent function. +class DeallocOpConverter : public OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite( + DeallocOp dealloc, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + FuncOp func = dealloc.getParentOfType(); + if (func.getNumArguments() == 0) { + return failure(); + } + Value ctx = func.getArgument(0); + if (!ctx.getType().isa()) { + return failure(); + } + // Operand with no layout is expected. + auto operand_memref_type = dealloc.memref().getType().cast(); + if (!operand_memref_type.getAffineMaps().empty()) { + return failure(); + } + DeallocOp::Adaptor transformed(operands); + rewriter.replaceOpWithNewOp(dealloc, ctx, + transformed.memref()); + return success(); + } +}; + +} // namespace + +void PopulateEmbedTFFrameworkConversionPatterns( + MLIRContext *context, OwningRewritePatternList *patterns) { + patterns->insert( + context); +} + +} // namespace tf_framework +} // namespace kernel_gen +} // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework_pass.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework_pass.cc new file mode 100644 index 00000000000..615c596e353 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework_pass.cc @@ -0,0 +1,77 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h" + +namespace mlir { +namespace kernel_gen { +namespace tf_framework { +namespace { + +#define GEN_PASS_CLASSES +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_passes.h.inc" + +static constexpr StringRef kTFEntry = "tf_entry"; + +// The pass rewrites the function marked with `tf_entry` attribute. +// * adds tf_framework::OpKernelContextType argument to the function, +// * std.alloc becomes tf_framework.alloc_raw, +// * std.dealloc becomes tf_framework.dealloc_raw. +class EmbedTFFrameworkPass + : public EmbedTFFrameworkPassBase { + public: + void runOnOperation() override { + ModuleOp m = getOperation(); + + // Populate patterns. + OwningRewritePatternList patterns; + PopulateEmbedTFFrameworkConversionPatterns(m.getContext(), &patterns); + + // Set target. + ConversionTarget target(getContext()); + target.addLegalDialect(); + + target.addDynamicallyLegalOp([&](FuncOp op) { + if (!op.getAttrOfType(kTFEntry)) { + return true; + } + FunctionType func_type = op.getType(); + return func_type.getNumInputs() > 0 && + func_type.getInput(0).isa(); + }); + target.addDynamicallyLegalOp([](Operation* op) { + return !op->getParentOfType().getAttrOfType(kTFEntry); + }); + + if (failed(applyPartialConversion(m, target, patterns))) { + signalPassFailure(); + } + } +}; + +} // namespace + +std::unique_ptr > createEmbedTFFrameworkPass() { + return std::make_unique(); +} + +} // namespace tf_framework +} // namespace kernel_gen +} // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h index 89871ba3faf..c6aaeb92c56 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h @@ -31,6 +31,12 @@ namespace tf_framework { std::unique_ptr > createTestTFFrameworkLegalizeToLLVMPass(); +// Pass to replace some of the Standard ops with TF Framework ops. +// * adds tf_framework::OpKernelContextType argument to the function +// * std.alloc becomes tf_framework.alloc_raw +// * std.dealloc becomes tf_framework.dealloc_raw +std::unique_ptr > createEmbedTFFrameworkPass(); + } // namespace tf_framework } // namespace kernel_gen } // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td index 71e50379ce7..8c4d5801f51 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td @@ -24,4 +24,9 @@ def TestTFFrameworkLegalizeToLLVMPass let constructor = "createTestTFFrameworkLegalizeToLLVMPass()"; } +def EmbedTFFrameworkPass : Pass<"embed-tf-framework", "ModuleOp"> { + let summary = "Pass to embed TF Framework for allocation and error reporting"; + let constructor = "createEmbedTFFrameworkPass()"; +} + #endif // TF_FRAMEWORK_PASSES diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h index 28dba379738..257e84b4a21 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h @@ -16,19 +16,26 @@ limitations under the License. #ifndef TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TRANSFORMS_REWRITERS_H_ #define TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TRANSFORMS_REWRITERS_H_ +#include "mlir/IR/MLIRContext.h" // from @llvm-project + namespace mlir { class LLVMTypeConverter; -class LowerToLLVMOptions; +class MLIRContext; class OwningRewritePatternList; +class TypeConverter; namespace kernel_gen { namespace tf_framework { -/// Collect a set of patterns to convert from the TF Framework dialect to LLVM. +/// Collects a set of patterns to convert from the TF Framework dialect to LLVM. void PopulateTFFrameworkToLLVMConversionPatterns( LLVMTypeConverter *converter, OwningRewritePatternList *patterns); +/// Collects a set of patterns to embed TF Framework. +void PopulateEmbedTFFrameworkConversionPatterns( + MLIRContext *context, OwningRewritePatternList *patterns); + } // namespace tf_framework } // namespace kernel_gen } // namespace mlir From 49bb08c055889f90d380d5c28ca7506e02c7e44a Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Sat, 1 Aug 2020 16:04:53 +0700 Subject: [PATCH 0261/1017] Add c interface for `core/platform:logging` --- tensorflow/c/BUILD | 11 ++++++++ tensorflow/c/logging.cc | 59 +++++++++++++++++++++++++++++++++++++++++ tensorflow/c/logging.h | 42 +++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+) create mode 100644 tensorflow/c/logging.cc create mode 100644 tensorflow/c/logging.h diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index 410fc22069f..5f64c43dfd3 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -213,6 +213,17 @@ tf_cuda_library( alwayslink = 1, ) +cc_library( + name = "logging", + srcs = ["logging.cc"], + hdrs = ["logging.h"], + deps = [ + ":c_api", + "//tensorflow/core/platform:logging", + "//tensorflow/core/platform:stringprintf", + ], +) + tf_cuda_library( name = "tf_status_internal", hdrs = [ diff --git a/tensorflow/c/logging.cc b/tensorflow/c/logging.cc new file mode 100644 index 00000000000..bf6bf069fff --- /dev/null +++ b/tensorflow/c/logging.cc @@ -0,0 +1,59 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/c/logging.h" + +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/stringprintf.h" + +static ::tensorflow::string BuildMessage(const char* fmt, va_list args) { + ::tensorflow::string message; + ::tensorflow::strings::Appendv(&message, fmt, args); + return message; +} + +void TF_Log(TF_LogLevel level, const char* fmt, ...) { + if (level < TF_INFO || level > TF_FATAL) return; + va_list args; + va_start(args, fmt); + auto message = BuildMessage(fmt, args); + switch (level) { + case TF_INFO: + LOG(INFO) << message; + break; + case TF_WARNING: + LOG(WARNING) << message; + break; + case TF_ERROR: + LOG(ERROR) << message; + break; + case TF_FATAL: + LOG(FATAL) << message; + break; + } +} + +void TF_VLog(int level, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + auto message = BuildMessage(fmt, args); + VLOG(level) << message; +} + +void TF_DVLog(int level, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + auto message = BuildMessage(fmt, args); + DVLOG(level) << message; +} diff --git a/tensorflow/c/logging.h b/tensorflow/c/logging.h new file mode 100644 index 00000000000..ad97cbf8c8a --- /dev/null +++ b/tensorflow/c/logging.h @@ -0,0 +1,42 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_C_LOGGING_H_ +#define TENSORFLOW_C_LOGGING_H_ + +#include "tensorflow/c/c_api.h" + +// -------------------------------------------------------------------------- +// C API for tensorflow::Logging. + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum TF_LogLevel { + TF_INFO = 0, + TF_WARNING = 1, + TF_ERROR = 2, + TF_FATAL = 3, +} TF_LogLevel; + +TF_CAPI_EXPORT extern void TF_Log(TF_LogLevel level, const char* fmt, ...); +TF_CAPI_EXPORT extern void TF_VLog(int level, const char* fmt, ...); +TF_CAPI_EXPORT extern void TF_DVLog(int level, const char* fmt, ...); + +#ifdef __cplusplus +} +#endif + +#endif // TENSORFLOW_C_LOGGING_H_ From a1b5894af841bc9c267f81bde384427facad074b Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Mon, 3 Aug 2020 11:00:05 -0700 Subject: [PATCH 0262/1017] Update LegalizeTFCommunication to deference optionals instead of calling getValue() explicitly (NFC). PiperOrigin-RevId: 324638102 Change-Id: I56de41274619bb31a4a2212762d560710886022c --- .../transforms/legalize_tf_communication.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc index 42c719da266..588e31ab669 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc @@ -96,7 +96,7 @@ llvm::SmallDenseMap GetFunctionsToRewrite( for (FuncOp& func : funcs_to_visit) { auto uses = func.getSymbolUses(module); if (!uses) continue; - for (auto& use : uses.getValue()) { + for (auto& use : *uses) { // Only `mlir::CallOp` is supported as this requires knowing how to // rewrite arguments and results to a function. if (!isa(use.getUser())) continue; @@ -189,13 +189,13 @@ Value CreateSendOp(OpBuilder& builder, int64_t& channel_id, Location loc, /*is_host_transfer=*/builder.getBoolAttr(true)); if (index) { - SetFrontendAttributes(send, index.getValue(), key, operand.getType(), + SetFrontendAttributes(send, *index, key, operand.getType(), /*device_to_host=*/true); } else { SetFrontendAttributes(send, key, operand.getType()); } - if (tpu_core) SetOpSharding(send, tpu_core.getValue()); + if (tpu_core) SetOpSharding(send, *tpu_core); return send.getResult(); } @@ -217,22 +217,22 @@ Value CreateRecvOp(OpBuilder& builder, int64_t& channel_id, Location loc, builder.create(loc, recv_result_type, token, channel_handle, /*is_host_transfer=*/builder.getBoolAttr(true)); if (index) { - SetFrontendAttributes(recv, index.getValue(), key, result_type, + SetFrontendAttributes(recv, *index, key, result_type, /*device_to_host=*/false); } else { SetFrontendAttributes(recv, key, result.getType()); } - if (tpu_core) SetOpSharding(recv, tpu_core.getValue()); + if (tpu_core) SetOpSharding(recv, *tpu_core); auto get_tuple_element = builder.create(loc, recv.getResult(), /*index=*/0); - if (tpu_core) SetOpSharding(get_tuple_element, tpu_core.getValue()); + if (tpu_core) SetOpSharding(get_tuple_element, *tpu_core); result.replaceAllUsesWith(get_tuple_element); auto new_token = builder.create(loc, recv.getResult(), /*index=*/1); - if (tpu_core) SetOpSharding(new_token, tpu_core.getValue()); + if (tpu_core) SetOpSharding(new_token, *tpu_core); return new_token.getResult(); } @@ -320,8 +320,8 @@ Value RewriteCallOp(OpBuilder& builder, CallOp call, auto new_result_types = llvm::to_vector<4>(call.getResultTypes()); new_result_types.push_back(token.getType()); auto new_call = builder.create( - call.getLoc(), new_result_types, - new_symbol ? new_symbol.getValue() : call.callee(), new_operands); + call.getLoc(), new_result_types, new_symbol ? *new_symbol : call.callee(), + new_operands); for (auto results : llvm::zip(call.getResults(), new_call.getResults())) std::get<0>(results).replaceAllUsesWith(std::get<1>(results)); From 7fafd216eb116a092a85d35d1e56d486c36d8727 Mon Sep 17 00:00:00 2001 From: amturati Date: Mon, 3 Aug 2020 18:19:43 +0000 Subject: [PATCH 0263/1017] changed OnesLike to ZerosLike --- tensorflow/c/eager/gradients.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/c/eager/gradients.cc b/tensorflow/c/eager/gradients.cc index cf62dcea926..406da1291ae 100644 --- a/tensorflow/c/eager/gradients.cc +++ b/tensorflow/c/eager/gradients.cc @@ -101,7 +101,7 @@ AbstractTensorHandle* TapeTensor::ZerosLike() const { } if (isa(op.get())) { s = dyn_cast(op.get())->SetOpName( - absl::StrCat("OnesLike", ToId(handle_)).c_str()); + absl::StrCat("ZerosLike", ToId(handle_)).c_str()); if (!s.ok()) { return nullptr; } From 9bcf2657a228115381f85275711e90501480690d Mon Sep 17 00:00:00 2001 From: Jose Baiocchi Date: Mon, 3 Aug 2020 11:07:06 -0700 Subject: [PATCH 0264/1017] Fix TraceMe in external/compat.h PiperOrigin-RevId: 324640019 Change-Id: I8248b642248ae8b562d85f05a98bd1d2ce3dbab5 --- .../xla/python/tpu_driver/platform/external/compat.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/python/tpu_driver/platform/external/compat.h b/tensorflow/compiler/xla/python/tpu_driver/platform/external/compat.h index 285d59e2304..0c7cc370e2a 100644 --- a/tensorflow/compiler/xla/python/tpu_driver/platform/external/compat.h +++ b/tensorflow/compiler/xla/python/tpu_driver/platform/external/compat.h @@ -35,7 +35,13 @@ class Thread { class TraceMe { public: - explicit TraceMe(absl::string_view tag, int level = 1) {} + explicit TraceMe(absl::string_view name, int level = 1) {} + explicit TraceMe(std::string&& name, int level = 1) = delete; + explicit TraceMe(const std::string& name, int level = 1) = delete; + explicit TraceMe(const char* raw, int level = 1) + : TraceMe(absl::string_view(raw), level) {} + template + explicit TraceMe(NameGeneratorT name_generator, int level = 1) {} ~TraceMe() {} }; From 043c4d515f352eff052b42dfc5a4bf5fe0dc00f6 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 01:26:23 +0700 Subject: [PATCH 0265/1017] Use c_api_macros instead of c_api --- tensorflow/c/BUILD | 2 +- tensorflow/c/logging.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index 5f64c43dfd3..e5efe323922 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -218,7 +218,7 @@ cc_library( srcs = ["logging.cc"], hdrs = ["logging.h"], deps = [ - ":c_api", + ":c_api_macros", "//tensorflow/core/platform:logging", "//tensorflow/core/platform:stringprintf", ], diff --git a/tensorflow/c/logging.h b/tensorflow/c/logging.h index ad97cbf8c8a..9583777b661 100644 --- a/tensorflow/c/logging.h +++ b/tensorflow/c/logging.h @@ -15,7 +15,7 @@ limitations under the License. #ifndef TENSORFLOW_C_LOGGING_H_ #define TENSORFLOW_C_LOGGING_H_ -#include "tensorflow/c/c_api.h" +#include "tensorflow/c/c_api_macros.h" // -------------------------------------------------------------------------- // C API for tensorflow::Logging. From cf43fd2af62a1d2188462176d6ffd36d9765231a Mon Sep 17 00:00:00 2001 From: Zhenyu Tan Date: Mon, 3 Aug 2020 11:20:03 -0700 Subject: [PATCH 0266/1017] Let linear model support dict inputs. PiperOrigin-RevId: 324642874 Change-Id: I09dd6555671258a15870471649616865972f2be3 --- tensorflow/python/keras/premade/linear.py | 73 ++++++++++++++----- .../python/keras/premade/linear_test.py | 22 +++++- 2 files changed, 76 insertions(+), 19 deletions(-) diff --git a/tensorflow/python/keras/premade/linear.py b/tensorflow/python/keras/premade/linear.py index edb4dc4b442..20f2ce560e2 100644 --- a/tensorflow/python/keras/premade/linear.py +++ b/tensorflow/python/keras/premade/linear.py @@ -18,10 +18,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import tensor_shape from tensorflow.python.keras import activations from tensorflow.python.keras import initializers from tensorflow.python.keras import regularizers from tensorflow.python.keras.engine import base_layer +from tensorflow.python.keras.engine import input_spec from tensorflow.python.keras.engine import training from tensorflow.python.keras.layers import core from tensorflow.python.ops import nn @@ -96,24 +98,42 @@ class LinearModel(training.Model): base_layer._keras_model_gauge.get_cell('Linear').set(True) # pylint: disable=protected-access def build(self, input_shape): - self.dense_layers = [] - if isinstance(input_shape, (tuple, list)): - for shape in input_shape: + if isinstance(input_shape, dict): + names = sorted(list(input_shape.keys())) + self.input_specs = [] + self.dense_layers = [] + for name in names: + shape = input_shape[name] layer = core.Dense( units=self.units, use_bias=False, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer, - input_shape=shape) + name=name) + layer.build(shape) + self.input_specs.append( + input_spec.InputSpec(shape=shape, name=name)) + self.dense_layers.append(layer) + elif isinstance(input_shape, (tuple, list)) and all( + isinstance(shape, tensor_shape.TensorShape) for shape in input_shape): + self.dense_layers = [] + for shape in input_shape: + layer = core.Dense( + units=self.units, + use_bias=False, + kernel_initializer=self.kernel_initializer, + kernel_regularizer=self.kernel_regularizer) + layer.build(shape) self.dense_layers.append(layer) else: + # input_shape can be a single TensorShape or a tuple of ints. layer = core.Dense( units=self.units, use_bias=False, kernel_initializer=self.kernel_initializer, - kernel_regularizer=self.kernel_regularizer, - input_shape=input_shape) - self.dense_layers.append(layer) + kernel_regularizer=self.kernel_regularizer) + layer.build(input_shape) + self.dense_layers = [layer] if self.use_bias: self.bias = self.add_weight( @@ -125,20 +145,37 @@ class LinearModel(training.Model): trainable=True) else: self.bias = None + self.built = True def call(self, inputs): - if not isinstance(inputs, (tuple, list)): - inputs = [inputs] - if len(inputs) != len(self.dense_layers): - raise ValueError('Expected {} inputs, but got {} inputs'.format( - len(self.dense_layers), len(inputs))) result = None - for inp, layer in zip(inputs, self.dense_layers): - output = layer(inp) - if result is None: - result = output - else: - result += output + if isinstance(inputs, dict): + names = [layer.name for layer in self.dense_layers] + different_keys = set(names) - set(inputs.keys()) + if different_keys: + raise ValueError( + 'The input dictionary does not match ' + 'the structure expected by the model.' + '\n\tExpected keys: {}' + '\n\tReceived keys: {}' + '\n\tMissing keys: {}'.format(set(names), set(inputs.keys()), + different_keys)) + inputs = [inputs[name] for name in names] + for inp, layer in zip(inputs, self.dense_layers): + output = layer(inp) + if result is None: + result = output + else: + result += output + elif isinstance(inputs, (tuple, list)): + for inp, layer in zip(inputs, self.dense_layers): + output = layer(inp) + if result is None: + result = output + else: + result += output + else: + result = self.dense_layers[0](inputs) if self.use_bias: result = nn.bias_add(result, self.bias) diff --git a/tensorflow/python/keras/premade/linear_test.py b/tensorflow/python/keras/premade/linear_test.py index 676f29bb840..ad57baa7813 100644 --- a/tensorflow/python/keras/premade/linear_test.py +++ b/tensorflow/python/keras/premade/linear_test.py @@ -26,6 +26,7 @@ from tensorflow.python.feature_column import feature_column_v2 as fc from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import sparse_tensor +from tensorflow.python.framework import tensor_shape from tensorflow.python.keras import backend from tensorflow.python.keras import keras_parameterized from tensorflow.python.keras import losses @@ -51,7 +52,7 @@ class LinearModelTest(keras_parameterized.TestCase): model.fit(inp, output, epochs=5) self.assertTrue(model.built) - def test_linear_model_with_multi_input(self): + def test_linear_model_with_list_input(self): model = linear.LinearModel() input_a = np.random.uniform(low=-5, high=5, size=(64, 1)) input_b = np.random.uniform(low=-5, high=5, size=(64, 1)) @@ -59,6 +60,25 @@ class LinearModelTest(keras_parameterized.TestCase): model.compile('sgd', 'mse', []) model.fit([input_a, input_b], output, epochs=5) + def test_linear_model_with_mismatched_dict_inputs(self): + model = linear.LinearModel() + input_a = np.random.uniform(low=-5, high=5, size=(64, 1)) + input_b = np.random.uniform(low=-5, high=5, size=(64, 1)) + output = .3 * input_a + .2 * input_b + model.compile('sgd', 'mse', []) + model.build({'a': tensor_shape.TensorShape([None, 1]), + 'b': tensor_shape.TensorShape([None, 1])}) + with self.assertRaisesRegex(ValueError, 'Missing keys'): + model.fit({'c': input_a, 'b': input_b}, output, epochs=5) + + def test_linear_model_with_dict_input(self): + model = linear.LinearModel() + input_a = np.random.uniform(low=-5, high=5, size=(64, 1)) + input_b = np.random.uniform(low=-5, high=5, size=(64, 1)) + output = .3 * input_a + .2 * input_b + model.compile('sgd', 'mse', []) + model.fit({'a': input_a, 'b': input_b}, output, epochs=5) + def test_linear_model_as_layer(self): input_a = input_layer.Input(shape=(1,), name='a') output_a = linear.LinearModel()(input_a) From dbc843d6ec806def963ed8016aca337595faa10d Mon Sep 17 00:00:00 2001 From: Haoyu Zhang Date: Mon, 3 Aug 2020 11:23:13 -0700 Subject: [PATCH 0267/1017] Garbage collect old WorkerSession when the restarted master task create new one. PiperOrigin-RevId: 324643608 Change-Id: I10165604d7ae03b25f15a31676d90f62aa6181be --- .../distributed_runtime/master_session.cc | 10 +++ .../core/distributed_runtime/session_mgr.cc | 39 +++++++++ .../core/distributed_runtime/session_mgr.h | 22 +++++ .../distributed_runtime/session_mgr_test.cc | 84 +++++++++++++++++++ tensorflow/core/distributed_runtime/worker.cc | 3 +- tensorflow/core/protobuf/worker.proto | 11 +++ tensorflow/python/training/server_lib_test.py | 58 +++++++++++++ 7 files changed, 226 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/distributed_runtime/master_session.cc b/tensorflow/core/distributed_runtime/master_session.cc index fb3a6659848..af98a0e4997 100644 --- a/tensorflow/core/distributed_runtime/master_session.cc +++ b/tensorflow/core/distributed_runtime/master_session.cc @@ -57,6 +57,7 @@ limitations under the License. #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/tracing.h" #include "tensorflow/core/public/session_options.h" +#include "tensorflow/core/util/device_name_utils.h" namespace tensorflow { @@ -1314,12 +1315,21 @@ Status MasterSession::CreateWorkerSessions( } }); + string task_name; + string local_device_name; + DeviceNameUtils::SplitDeviceName(devices_->client_device()->name(), + &task_name, &local_device_name); + const int64 client_device_incarnation = + devices_->client_device()->attributes().incarnation(); + Status status = Status::OK(); // Create all the workers & kick off the computations. for (size_t i = 0; i < worker_names.size(); ++i) { workers[i].name = &worker_names[i]; workers[i].worker = worker_cache->GetOrCreateWorker(worker_names[i]); workers[i].request.set_session_handle(handle_); + workers[i].request.set_master_task(task_name); + workers[i].request.set_master_incarnation(client_device_incarnation); if (session_opts_.config.share_cluster_devices_in_session() || session_opts_.config.experimental() .share_cluster_devices_in_session()) { diff --git a/tensorflow/core/distributed_runtime/session_mgr.cc b/tensorflow/core/distributed_runtime/session_mgr.cc index 37f47848f75..0dd657c7fd9 100644 --- a/tensorflow/core/distributed_runtime/session_mgr.cc +++ b/tensorflow/core/distributed_runtime/session_mgr.cc @@ -62,11 +62,46 @@ Status SessionMgr::CreateSession( const protobuf::RepeatedPtrField& cluster_device_attributes, bool isolate_session_state) { + return CreateSession(session, server_def, cluster_device_attributes, + isolate_session_state, /*master_task=*/"", + /*master_incarnation=*/0); +} + +Status SessionMgr::CreateSession( + const string& session, const ServerDef& server_def, + const protobuf::RepeatedPtrField& + cluster_device_attributes, + bool isolate_session_state, string master_task, int64 master_incarnation) { mutex_lock l(mu_); if (session.empty()) { return errors::InvalidArgument("Session must be non-empty."); } + // For given master task name, check if one or more `WorkerSession`s have been + // created previously on this worker, and if so garbage collect the expired + // `WorkerSession`s. This happens when the master fails before sending + // `DeleteSession` requests, which can cause `WorkerSession`s to be leaked. + if (!master_task.empty()) { + auto it_range = master_to_associated_sessions_.equal_range(master_task); + if (it_range.first != it_range.second && + it_range.first->second.master_incarnation != master_incarnation) { + LOG(INFO) << "When creating WorkerSession for master task " << master_task + << ", found old WorkerSessions created by the same master task " + << "with a different incarnation. These sessions will " + << "be garbage collected. Current WorkerSession count: " + << sessions_.size(); + + auto it = it_range.first; + while (it != it_range.second) { + auto session_it = sessions_.find(it->second.session_handle); + if (session_it != sessions_.end()) { + sessions_.erase(session_it); + } + it = master_to_associated_sessions_.erase(it); + } + } + } + WorkerCacheInterface* worker_cache = nullptr; string worker_name; if (server_def.cluster().job().empty()) { @@ -141,6 +176,10 @@ Status SessionMgr::CreateSession( } sessions_.insert(std::make_pair(session, std::move(worker_session))); + if (!master_task.empty()) { + MasterAssociatedSession s{master_incarnation, session}; + master_to_associated_sessions_.emplace(master_task, s); + } return Status::OK(); } diff --git a/tensorflow/core/distributed_runtime/session_mgr.h b/tensorflow/core/distributed_runtime/session_mgr.h index a9467708870..dfcc69463c4 100644 --- a/tensorflow/core/distributed_runtime/session_mgr.h +++ b/tensorflow/core/distributed_runtime/session_mgr.h @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/distributed_runtime/worker_session.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/thread_annotations.h" #include "tensorflow/core/protobuf/tensorflow_server.pb.h" #include "tensorflow/core/protobuf/worker.pb.h" @@ -53,6 +54,18 @@ class SessionMgr { const protobuf::RepeatedPtrField& device_attributes, bool isolate_session_state); + // Create WorkerSession from the master with the given `master_task` and + // `master_incarnation`. We first look for existing WorkerSessions associated + // with the specified master task. If there are sessions created by the same + // master but with a different incarnation, it indicates that the remote + // master has restarted before deleting the sessions on worker. When it + // happens, old sessions associated with the master will be automatically + // removed before the new session is created. + Status CreateSession( + const string& session, const ServerDef& server_def, + const protobuf::RepeatedPtrField& device_attributes, + bool isolate_session_state, string master_task, int64 master_incarnation); + void ResetDefaultWorkerCache(WorkerCacheInterface* worker_cache); // Updates state (worker cache, devices) of worker session identified by @@ -107,6 +120,15 @@ class SessionMgr { mutex mu_; // A map from session identifier to internal session structure. std::map> sessions_ TF_GUARDED_BY(mu_); + + // Incarnation and WorkerSession handle associated with a master task. + struct MasterAssociatedSession { + const int64 master_incarnation; + const string session_handle; + }; + // A map from master task name to its associated worker sessions. + std::unordered_multimap + master_to_associated_sessions_ TF_GUARDED_BY(mu_); }; } // namespace tensorflow diff --git a/tensorflow/core/distributed_runtime/session_mgr_test.cc b/tensorflow/core/distributed_runtime/session_mgr_test.cc index f6e0551ff56..1f5e26b7a0b 100644 --- a/tensorflow/core/distributed_runtime/session_mgr_test.cc +++ b/tensorflow/core/distributed_runtime/session_mgr_test.cc @@ -152,6 +152,90 @@ TEST_F(SessionMgrTest, CreateSessionIsolateSessionState) { EXPECT_NE(devices_3[0]->resource_manager(), devices_4[0]->resource_manager()); } +TEST_F(SessionMgrTest, CreateSessionWithMasterName) { + ServerDef server_def; + server_def.set_job_name("worker"); + server_def.set_task_index(3); + auto job = server_def.mutable_cluster()->add_job(); + job->set_name("worker"); + job->mutable_tasks()->insert({3, "localhost:3333"}); + + protobuf::RepeatedPtrField cluster_device_attributes; + + const string master_name = "/job:master/replica:0/task:1"; + const int64 old_incarnation = random::New64(); + const int64 new_incarnation = random::New64(); + + // Allow multiple worker sessions to be created by the same master + string sess_handle1 = "test_session_handle_1"; + TF_EXPECT_OK(mgr_.CreateSession(sess_handle1, server_def, + cluster_device_attributes, true, master_name, + old_incarnation)); + string sess_handle2 = "test_session_handle_2"; + TF_EXPECT_OK(mgr_.CreateSession(sess_handle2, server_def, + cluster_device_attributes, true, master_name, + old_incarnation)); + + std::shared_ptr session; + TF_EXPECT_OK(mgr_.WorkerSessionForSession(sess_handle1, &session)); + EXPECT_NE(nullptr, session) << "Session for " << sess_handle1 << "was null"; + + TF_EXPECT_OK(mgr_.WorkerSessionForSession(sess_handle2, &session)); + EXPECT_NE(nullptr, session) << "Session for " << sess_handle2 << "was null"; + + // When the master creates a WorkerSession with new incarnation, the old + // WorkerSessions should be garbage collected. + string sess_handle3 = "test_session_handle_3"; + TF_EXPECT_OK(mgr_.CreateSession(sess_handle3, server_def, + cluster_device_attributes, true, master_name, + new_incarnation)); + + EXPECT_NE(mgr_.WorkerSessionForSession(sess_handle1, &session), + tensorflow::Status::OK()) + << "Session for " << sess_handle1 + << " should have been garbage collected."; + + EXPECT_NE(mgr_.WorkerSessionForSession(sess_handle2, &session), + tensorflow::Status::OK()) + << "Session for " << sess_handle2 + << " should have been garbage collected."; + + TF_EXPECT_OK(mgr_.WorkerSessionForSession(sess_handle3, &session)); + EXPECT_NE(nullptr, session) << "Session for " << sess_handle3 << "was null"; + + TF_EXPECT_OK(mgr_.DeleteSession(sess_handle2)); + TF_EXPECT_OK(mgr_.DeleteSession(sess_handle3)); +} + +TEST_F(SessionMgrTest, CreateSessionWithoutMasterName) { + ServerDef server_def; + server_def.set_job_name("worker"); + server_def.set_task_index(3); + auto job = server_def.mutable_cluster()->add_job(); + job->set_name("worker"); + job->mutable_tasks()->insert({3, "localhost:3333"}); + + protobuf::RepeatedPtrField cluster_device_attributes; + + // WorkerSession will NOT be garbage collected for empty master names. + string sess_handle1 = "test_session_handle_no_master_1"; + TF_EXPECT_OK(mgr_.CreateSession(sess_handle1, server_def, + cluster_device_attributes, true, "", 0)); + string sess_handle2 = "test_session_handle_no_master_2"; + TF_EXPECT_OK(mgr_.CreateSession(sess_handle2, server_def, + cluster_device_attributes, true, "", 0)); + + std::shared_ptr session; + TF_EXPECT_OK(mgr_.WorkerSessionForSession(sess_handle1, &session)); + EXPECT_NE(nullptr, session) << "Session for " << sess_handle1 << "was null"; + + TF_EXPECT_OK(mgr_.WorkerSessionForSession(sess_handle2, &session)); + EXPECT_NE(nullptr, session) << "Session for " << sess_handle2 << "was null"; + + TF_EXPECT_OK(mgr_.DeleteSession(sess_handle1)); + TF_EXPECT_OK(mgr_.DeleteSession(sess_handle2)); +} + TEST_F(SessionMgrTest, LegacySession) { string session_handle = ""; std::shared_ptr session; diff --git a/tensorflow/core/distributed_runtime/worker.cc b/tensorflow/core/distributed_runtime/worker.cc index f857a63e64d..5212f51d491 100644 --- a/tensorflow/core/distributed_runtime/worker.cc +++ b/tensorflow/core/distributed_runtime/worker.cc @@ -53,7 +53,8 @@ void Worker::CreateWorkerSessionAsync(const CreateWorkerSessionRequest* request, StatusCallback done) { Status s = env_->session_mgr->CreateSession( request->session_handle(), request->server_def(), - request->cluster_device_attributes(), request->isolate_session_state()); + request->cluster_device_attributes(), request->isolate_session_state(), + request->master_task(), request->master_incarnation()); done(s); } diff --git a/tensorflow/core/protobuf/worker.proto b/tensorflow/core/protobuf/worker.proto index f10283531da..739ba8e03e6 100644 --- a/tensorflow/core/protobuf/worker.proto +++ b/tensorflow/core/protobuf/worker.proto @@ -70,6 +70,17 @@ message CreateWorkerSessionRequest { // The device attributes of all the devices in the cluster. repeated DeviceAttributes cluster_device_attributes = 4; + + // The master task name from which the request is sent. + string master_task = 5; + + // The incarnation ID of the master task local CPU device. + // If the target worker already has a WorkerSession created previously with + // the same master task name but a different incarnation, it usually indicates + // that the previous master failed before deleting the WorkerSession on the + // worker. To prevent memory leaks, the worker should garbage collect the old + // WorkerSessions. + int64 master_incarnation = 6; } message CreateWorkerSessionResponse {} diff --git a/tensorflow/python/training/server_lib_test.py b/tensorflow/python/training/server_lib_test.py index 54ede81c9ea..75008985aae 100644 --- a/tensorflow/python/training/server_lib_test.py +++ b/tensorflow/python/training/server_lib_test.py @@ -22,6 +22,7 @@ import time import numpy as np +from tensorflow.core.protobuf import cluster_pb2 from tensorflow.core.protobuf import config_pb2 from tensorflow.core.protobuf import tensorflow_server_pb2 from tensorflow.python.client import session @@ -202,6 +203,63 @@ class GrpcServerTest(test.TestCase): self.assertEqual(0.1, server.server_def.default_session_config.gpu_options. per_process_gpu_memory_fraction) + def testRestartedMaster(self): + master_old = server_lib.Server.create_local_server() + master_new = server_lib.Server.create_local_server() + worker = self._cached_server + + def get_cluster_def(master, worker): + cluster_def = cluster_pb2.ClusterDef() + job = cluster_def.job.add() + job.name = "master" + job.tasks[0] = master.target[len("grpc://"):] + job = cluster_def.job.add() + job.name = "worker" + job.tasks[0] = worker.target[len("grpc://"):] + return cluster_def + + def check_session_devices(sess): + # Make sure we have the correct set of cluster devices + devices = sess.list_devices() + device_names = set(d.name for d in devices) + self.assertIn("/job:master/replica:0/task:0/device:CPU:0", device_names) + self.assertIn("/job:worker/replica:0/task:0/device:CPU:0", device_names) + + with ops.Graph().as_default(): + # Construct a simple graph that runs ops on remote worker + with ops.device("/job:worker/replica:0/task:0/device:CPU:0"): + a = constant_op.constant([1.0]) + b = a + a + + config = config_pb2.ConfigProto( + cluster_def=get_cluster_def(master_old, worker)) + sess_old = session.Session(master_old.target, config=config) + check_session_devices(sess_old) + + # Create a session with the new master and the worker. + # The new master has the same task name ('/job:master/replica:0/task:0') + # as the old master, but is initiated from a different server thus has a + # different incarnation. This triggers the WorkerSession on worker with + # the old master incarnation to be garbage collected. + + config = config_pb2.ConfigProto( + cluster_def=get_cluster_def(master_new, worker)) + sess_new = session.Session(master_new.target, config=config) + check_session_devices(sess_new) + + # Running on worker with the new session should work as expected + v = sess_new.run(b) + self.assertAllEqual(v, [2.0]) + + # Running on worker with the old session should raise an exception since + # the WorkerSession of the old session has been garbage collected + with self.assertRaisesRegex(errors_impl.AbortedError, + "Session handle is not found"): + sess_old.run(b) + + sess_old.close() + sess_new.close() + def testInvalidHostname(self): with self.assertRaisesRegex(errors_impl.InvalidArgumentError, "port"): _ = server_lib.Server( From cddca76312f5ae4fb92a101e79eeff6d5ac16932 Mon Sep 17 00:00:00 2001 From: Robert David Date: Mon, 3 Aug 2020 11:35:42 -0700 Subject: [PATCH 0268/1017] Add check for reading input tensors at an index that is out of range. PiperOrigin-RevId: 324646398 Change-Id: I602b23b2f28504c20a6d099874cdba2ddbf5ca83 --- tensorflow/lite/delegates/gpu/common/object_reader.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tensorflow/lite/delegates/gpu/common/object_reader.h b/tensorflow/lite/delegates/gpu/common/object_reader.h index f360bcf9302..be9a89e1b4e 100644 --- a/tensorflow/lite/delegates/gpu/common/object_reader.h +++ b/tensorflow/lite/delegates/gpu/common/object_reader.h @@ -58,6 +58,11 @@ class ObjectReader { template absl::Status ReadTensor(uint32_t idx, TensorT* t) const { + if (idx < 0 || idx >= node_->inputs->size) { + // If larger, this can be an older model with fewer input tensors than the + // current implementation. + return absl::OutOfRangeError("Invalid data index found."); + } const int32_t tensor_idx = node_->inputs->data[idx]; if (tensor_idx < 0) { return absl::InvalidArgumentError( From 7a4b89dbaf78503882dfeefa634c5bbff303d563 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 3 Aug 2020 11:45:27 -0700 Subject: [PATCH 0269/1017] Adding `doc_controls.decorate_all_class_attributes` to tensorflow_docs. `_hide_layer_and_module` can be removed. PiperOrigin-RevId: 324648586 Change-Id: I30559a92f02d69772fc2719cb69d36470b1f9fd7 --- tensorflow/tools/docs/generate2.py | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/tensorflow/tools/docs/generate2.py b/tensorflow/tools/docs/generate2.py index d61c03548e3..44152ba30ef 100644 --- a/tensorflow/tools/docs/generate2.py +++ b/tensorflow/tools/docs/generate2.py @@ -151,28 +151,6 @@ class TfExportAwareVisitor(doc_generator_visitor.DocGeneratorVisitor): return (canonical_score,) + scores -def _hide_layer_and_module_methods(): - """Hide methods and properties defined in the base classes of keras layers.""" - # __dict__ only sees attributes defined in *this* class, not on parent classes - module_contents = list(tf.Module.__dict__.items()) - layer_contents = list(tf.keras.layers.Layer.__dict__.items()) - - for name, obj in module_contents + layer_contents: - if name == "__init__": - continue - - if isinstance(obj, property): - obj = obj.fget - - if isinstance(obj, (staticmethod, classmethod)): - obj = obj.__func__ - - try: - doc_controls.do_not_doc_in_subclasses(obj) - except AttributeError: - pass - - def build_docs(output_dir, code_url_prefix, search_hints=True): """Build api docs for tensorflow v2. @@ -189,7 +167,11 @@ def build_docs(output_dir, code_url_prefix, search_hints=True): if not name.startswith("_"): doc_controls.hide_from_search(obj) - _hide_layer_and_module_methods() + for cls in [tf.Module, tf.keras.layers.Layer]: + doc_controls.decorate_all_class_attributes( + decorator=doc_controls.do_not_doc_in_subclasses, + cls=cls, + skip=["__init__"]) try: doc_controls.do_not_generate_docs(tf.__operators__) From 3b83a25110d66c12654d74e64b1f7a77b14e00c6 Mon Sep 17 00:00:00 2001 From: Ran Chen Date: Mon, 3 Aug 2020 12:12:20 -0700 Subject: [PATCH 0270/1017] Test with MWMS in custom_training_loop_optimizer_test PiperOrigin-RevId: 324654618 Change-Id: I1a43c1b275bdb9b084432220519044fe4a69e1a8 --- tensorflow/python/keras/distribute/BUILD | 2 +- .../custom_training_loop_optimizer_test.py | 25 ++++++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/keras/distribute/BUILD b/tensorflow/python/keras/distribute/BUILD index 5a5cff01e33..2c8ba97dbfa 100644 --- a/tensorflow/python/keras/distribute/BUILD +++ b/tensorflow/python/keras/distribute/BUILD @@ -275,7 +275,7 @@ distribute_py_test( "//tensorflow/python:variables", "//tensorflow/python/distribute:combinations", "//tensorflow/python/distribute:strategy_combinations", - "//tensorflow/python/distribute:values", + "//tensorflow/python/distribute:test_util", "//tensorflow/python/eager:def_function", "//tensorflow/python/eager:test", "//tensorflow/python/keras/optimizer_v2", diff --git a/tensorflow/python/keras/distribute/custom_training_loop_optimizer_test.py b/tensorflow/python/keras/distribute/custom_training_loop_optimizer_test.py index b9eee26220a..0a12d85bebd 100644 --- a/tensorflow/python/keras/distribute/custom_training_loop_optimizer_test.py +++ b/tensorflow/python/keras/distribute/custom_training_loop_optimizer_test.py @@ -22,7 +22,7 @@ from absl.testing import parameterized from tensorflow.python.distribute import combinations from tensorflow.python.distribute import strategy_combinations -from tensorflow.python.distribute import values +from tensorflow.python.distribute import test_util from tensorflow.python.eager import def_function from tensorflow.python.eager import test from tensorflow.python.framework import ops @@ -35,7 +35,14 @@ class OptimizerTest(test.TestCase, parameterized.TestCase): @combinations.generate( combinations.times( combinations.combine( - distribution=strategy_combinations.multidevice_strategies, + distribution=[ + strategy_combinations.mirrored_strategy_with_gpu_and_cpu, + strategy_combinations.mirrored_strategy_with_two_gpus, + strategy_combinations.multi_worker_mirrored_2x1_cpu, + strategy_combinations.multi_worker_mirrored_2x1_gpu, + strategy_combinations.tpu_strategy, + strategy_combinations.tpu_strategy_one_step, + ], mode=["eager"], ), combinations.concat( @@ -55,10 +62,10 @@ class OptimizerTest(test.TestCase, parameterized.TestCase): @def_function.function def optimize(): - grads = values.PerReplica([ - ops.convert_to_tensor([1., 1.]), - ops.convert_to_tensor([2., 2.]), - ]) + grads = ops.convert_to_tensor([[1., 1.], + [2., 2.]]) + grads = distribution.experimental_distribute_values_from_function( + lambda ctx: grads[ctx.replica_id_in_sync_group]) def step_fn(grads): optimizer.apply_gradients( @@ -66,8 +73,8 @@ class OptimizerTest(test.TestCase, parameterized.TestCase): experimental_aggregate_gradients=experimental_aggregate_gradients) return v.read_value() - return distribution.experimental_local_results( - distribution.run(step_fn, args=(grads,))) + return test_util.gather(distribution, + distribution.run(step_fn, args=(grads,))) self.assertAllClose(optimize(), expected) @@ -118,4 +125,4 @@ class OptimizerTest(test.TestCase, parameterized.TestCase): if __name__ == "__main__": - test.main() + combinations.main() From 99fc31e82fc8d5a5506a8f2de4dd7eb5f7f160e2 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Mon, 3 Aug 2020 12:35:28 -0700 Subject: [PATCH 0271/1017] Add a module config option to enable hlo deduplication. PiperOrigin-RevId: 324660155 Change-Id: Ic7aac0daf851bb93b4f6c24e56b20234200efdbc --- .../compiler/xla/client/executable_build_options.cc | 6 ++++++ .../compiler/xla/client/executable_build_options.h | 4 ++++ .../compiler/xla/service/compile_only_service.cc | 1 + tensorflow/compiler/xla/service/compiler.h | 1 + tensorflow/compiler/xla/service/hlo_module.cc | 1 + tensorflow/compiler/xla/service/hlo_module_config.h | 11 +++++++++++ tensorflow/compiler/xla/service/local_service.cc | 1 + tensorflow/compiler/xla/service/service.cc | 1 + tensorflow/compiler/xla/xla.proto | 4 ++++ 9 files changed, 30 insertions(+) diff --git a/tensorflow/compiler/xla/client/executable_build_options.cc b/tensorflow/compiler/xla/client/executable_build_options.cc index 404f9eb7519..f39a3e79fe5 100644 --- a/tensorflow/compiler/xla/client/executable_build_options.cc +++ b/tensorflow/compiler/xla/client/executable_build_options.cc @@ -76,6 +76,12 @@ ExecutableBuildOptions& ExecutableBuildOptions::set_use_spmd_partitioning( return *this; } +ExecutableBuildOptions& ExecutableBuildOptions::set_deduplicate_hlo( + bool deduplicate_hlo) { + deduplicate_hlo_ = deduplicate_hlo; + return *this; +} + ExecutableBuildOptions& ExecutableBuildOptions::set_device_assignment( const DeviceAssignment& device_assignment) { device_assignment_ = device_assignment; diff --git a/tensorflow/compiler/xla/client/executable_build_options.h b/tensorflow/compiler/xla/client/executable_build_options.h index 9a7fdd974b1..d034eaa7fd6 100644 --- a/tensorflow/compiler/xla/client/executable_build_options.h +++ b/tensorflow/compiler/xla/client/executable_build_options.h @@ -82,6 +82,9 @@ class ExecutableBuildOptions { bool use_spmd_partitioning() const { return use_spmd_partitioning_; } ExecutableBuildOptions& set_use_spmd_partitioning(bool use_spmd_partitioning); + bool deduplicate_hlo() const { return deduplicate_hlo_; } + ExecutableBuildOptions& set_deduplicate_hlo(bool deduplicate_hlo); + // If set, this specifies a static device assignment for the computation. // Otherwise, the computation will be compiled generically and can be run with // any device assignment compatible with the computation's replica and @@ -110,6 +113,7 @@ class ExecutableBuildOptions { int num_replicas_ = 1; int num_partitions_ = 1; bool use_spmd_partitioning_ = false; + bool deduplicate_hlo_ = false; absl::optional device_assignment_; bool alias_passthrough_params_ = false; }; diff --git a/tensorflow/compiler/xla/service/compile_only_service.cc b/tensorflow/compiler/xla/service/compile_only_service.cc index ce9c8a4ea62..f8e4f591a5d 100644 --- a/tensorflow/compiler/xla/service/compile_only_service.cc +++ b/tensorflow/compiler/xla/service/compile_only_service.cc @@ -92,6 +92,7 @@ CompileOnlyService::CompileAheadOfTime( execution_options.mutable_device_assignment())); } execution_options.set_use_spmd_partitioning(options.use_spmd_partitioning()); + execution_options.set_deduplicate_hlo(options.deduplicate_hlo()); for (const AotXlaComputationInstance& instance : computations) { TF_RET_CHECK(instance.computation.has_host_program_shape()); *execution_options.mutable_shape_with_output_layout() = diff --git a/tensorflow/compiler/xla/service/compiler.h b/tensorflow/compiler/xla/service/compiler.h index 57b24e372e6..312a068ba65 100644 --- a/tensorflow/compiler/xla/service/compiler.h +++ b/tensorflow/compiler/xla/service/compiler.h @@ -77,6 +77,7 @@ class AotCompilationOptions { virtual int64 replica_count() const { return 0; } virtual int64 num_cores() const { return 0; } virtual bool use_spmd_partitioning() const { return false; } + virtual bool deduplicate_hlo() const { return false; } // Optional allocator that may be used for allocating temp space on the device // during compilation. diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc index 308b8e8f095..4a67c1d2146 100644 --- a/tensorflow/compiler/xla/service/hlo_module.cc +++ b/tensorflow/compiler/xla/service/hlo_module.cc @@ -443,6 +443,7 @@ StatusOr HloModule::CreateModuleConfigFromShape( } module_config.set_use_spmd_partitioning( execution_options->use_spmd_partitioning()); + module_config.set_deduplicate_hlo(execution_options->deduplicate_hlo()); if (execution_options->has_device_assignment()) { TF_ASSIGN_OR_RETURN(std::unique_ptr device_assignment, DeviceAssignment::Deserialize( diff --git a/tensorflow/compiler/xla/service/hlo_module_config.h b/tensorflow/compiler/xla/service/hlo_module_config.h index 7ab0f24d06e..ae0a8aae838 100644 --- a/tensorflow/compiler/xla/service/hlo_module_config.h +++ b/tensorflow/compiler/xla/service/hlo_module_config.h @@ -138,6 +138,13 @@ class HloModuleConfig { } bool use_spmd_partitioning() const { return use_spmd_partitioning_; } + // If enabled, deduplicate equivalent hlos into function calls to reduce code + // size. + void set_deduplicate_hlo(bool deduplicate_hlo) { + deduplicate_hlo_ = deduplicate_hlo; + } + bool deduplicate_hlo() const { return deduplicate_hlo_; } + // Return a string which unambiguously represents all the fields of this data // structure. Used for generating a cache key for storing the compiled // executable. @@ -246,6 +253,10 @@ class HloModuleConfig { // needs to partition the module. bool use_spmd_partitioning_ = false; + // If enabled, deduplicate equivalent hlos into function calls to reduce code + // size. + bool deduplicate_hlo_ = false; + // The target maximum parallelism at which to partition HLOs for parallel // execution on the CPU backend. int64 intra_op_parallelism_threads_ = -1; diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc index c80646e0c70..5def5bbe9db 100644 --- a/tensorflow/compiler/xla/service/local_service.cc +++ b/tensorflow/compiler/xla/service/local_service.cc @@ -114,6 +114,7 @@ ExecutionOptions CreateExecutionOptions( execution_options.set_num_partitions(build_options.num_partitions()); execution_options.set_use_spmd_partitioning( build_options.use_spmd_partitioning()); + execution_options.set_deduplicate_hlo(build_options.deduplicate_hlo()); if (build_options.has_device_assignment()) { TF_CHECK_OK(build_options.device_assignment().Serialize( execution_options.mutable_device_assignment())); diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc index 2ed5e709d81..4437ec3d452 100644 --- a/tensorflow/compiler/xla/service/service.cc +++ b/tensorflow/compiler/xla/service/service.cc @@ -315,6 +315,7 @@ StatusOr> Service::CreateModuleConfig( } config->set_use_spmd_partitioning( execution_options->use_spmd_partitioning()); + config->set_deduplicate_hlo(execution_options->deduplicate_hlo()); config->set_seed(execution_options->seed()); config->set_launch_id(execution_options->launch_id()); config->set_debug_options(execution_options->debug_options()); diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto index 6b9917eac53..1cf30b10373 100644 --- a/tensorflow/compiler/xla/xla.proto +++ b/tensorflow/compiler/xla/xla.proto @@ -349,6 +349,10 @@ message ExecutionOptions { // Indicates whether to use SPMD (true) or MPMD (false) partitioning when // num_partitions > 1 and XLA is requested to partition the input program. bool use_spmd_partitioning = 11; + + // If set, deduplicate hlo into function calls to reduce binary size. Only + // works on TPU. + bool deduplicate_hlo = 12; } message GetDeviceHandlesRequest { From ec87d847f222e6ad5d53126d37264e02c5772587 Mon Sep 17 00:00:00 2001 From: HanBin Yoon Date: Mon, 3 Aug 2020 13:20:16 -0700 Subject: [PATCH 0272/1017] Add compiler pass to remove duplicate 'tf_saved_model.bound_input' bindings. Consolidate identical bound inputs so that resource variables do not alias in modules with tf_saved_model semantics. PiperOrigin-RevId: 324669700 Change-Id: I546c212e50c889aa6526c0df88f7b9051ed28fa8 --- tensorflow/compiler/mlir/tensorflow/BUILD | 1 + .../mlir/tensorflow/ir/tf_saved_model.cc | 1 + .../tf_saved_model/hash_table_asset_v1.py | 17 +++-- ...odel_deduplicate_bound_input_bindings.mlir | 33 ++++++++++ .../tensorflow/tests/tf_saved_model_ops.mlir | 13 ++++ .../tests/tf_saved_model_ops_invalid.mlir | 14 ++++ .../deduplicate_bound_input_bindings.cc | 65 +++++++++++++++++++ .../transforms/tf_saved_model_passes.h | 3 + .../mlir/tensorflow/translate/import_model.cc | 4 +- 9 files changed, 145 insertions(+), 6 deletions(-) create mode 100644 tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_deduplicate_bound_input_bindings.mlir create mode 100644 tensorflow/compiler/mlir/tensorflow/transforms/deduplicate_bound_input_bindings.cc diff --git a/tensorflow/compiler/mlir/tensorflow/BUILD b/tensorflow/compiler/mlir/tensorflow/BUILD index 518992d03db..c6f0083fc92 100644 --- a/tensorflow/compiler/mlir/tensorflow/BUILD +++ b/tensorflow/compiler/mlir/tensorflow/BUILD @@ -676,6 +676,7 @@ cc_library( cc_library( name = "tf_saved_model_passes", srcs = [ + "transforms/deduplicate_bound_input_bindings.cc", "transforms/freeze_global_tensors.cc", "transforms/lift_variables_pass.cc", "transforms/optimize_global_tensors.cc", diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc index edfc7feefd5..94a792ec3db 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc @@ -337,6 +337,7 @@ LogicalResult VerifyExportedFunc(FuncOp func) { if (auto attr = func.getArgAttrOfType( i, "tf_saved_model.bound_input")) { if (!unique_bound_inputs.insert(attr.getValue()).second) { + if (module.getAttr("tf_saved_model.under_construction")) continue; return func.emitError() << "duplicate 'tf_saved_model.bound_input' binding"; } diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_asset_v1.py b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_asset_v1.py index 7e86953eb8f..4cb931253b3 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_asset_v1.py +++ b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model/hash_table_asset_v1.py @@ -27,13 +27,15 @@ import tensorflow.compat.v1 as tf from tensorflow.compiler.mlir.tensorflow.tests.tf_saved_model import common_v1 # CHECK: "tf_saved_model.session_initializer"() {initializer = [[init:@.*]]} : () -> () -# CHECK: "tf_saved_model.asset"() {filename = {{.*}}, sym_name = "[[asset:.*]]"} +# CHECK: "tf_saved_model.asset"() {filename = {{.*}}, sym_name = "[[asset1:__tf_saved_model_asset1_.*]]"} +# CHECK: "tf_saved_model.asset"() {filename = {{.*}}, sym_name = "[[asset0:__tf_saved_model_asset0_.*]]"} # CHECK: func [[init]] -# CHECK-SAME: [[ARG:%.*]]: tensor {tf_saved_model.bound_input = @[[asset]]} +# CHECK-SAME: [[ARG0:%.*]]: tensor {tf_saved_model.bound_input = @[[asset0]]} +# CHECK-SAME: [[ARG1:%.*]]: tensor {tf_saved_model.bound_input = @[[asset1]]} # CHECK-NEXT: [[R0:%.*]] = "tf.HashTableV2"() # CHECK-SAME: shared_name = "[[hash_table:.*]]" -# CHECK-NEXT: "tf.InitializeTableFromTextFileV2"([[R0]], [[ARG]]) +# CHECK-NEXT: "tf.InitializeTableFromTextFileV2"([[R0]], [[ARG0]]) def write_vocabulary_file(vocabulary): @@ -48,11 +50,16 @@ def write_vocabulary_file(vocabulary): def test(): + vocabulary_file = write_vocabulary_file(['cat', 'is', 'on', 'the', 'mat']) table_initializer = tf.lookup.TextFileInitializer( - write_vocabulary_file(['cat', 'is', 'on', 'the', 'mat']), tf.string, - tf.lookup.TextFileIndex.WHOLE_LINE, tf.int64, + vocabulary_file, tf.string, tf.lookup.TextFileIndex.WHOLE_LINE, tf.int64, tf.lookup.TextFileIndex.LINE_NUMBER) + # Incur another bound_input on the asset, but with a different sym_name, i.e., + # __tf_saved_model_asset1_tokens.txt vs. __tf_saved_model_asset0_tokens.txt. table = tf.lookup.StaticVocabularyTable(table_initializer, num_oov_buckets=10) + vocab_file_tensor = tf.convert_to_tensor(vocabulary_file, tf.string, + name='asset_filepath') + tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, vocab_file_tensor) x = tf.placeholder(tf.string, shape=(), name='input') r = table.lookup(x) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_deduplicate_bound_input_bindings.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_deduplicate_bound_input_bindings.mlir new file mode 100644 index 00000000000..22fd3d86068 --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_deduplicate_bound_input_bindings.mlir @@ -0,0 +1,33 @@ +// RUN: tf-opt %s -split-input-file -verify-diagnostics -tf-saved-model-dedup-bound-input-binding-pass | FileCheck %s + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + // Test case: Remove duplicate bound_input symbols. + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.0> : tensor } : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "w", type = tensor, value = dense<43.0> : tensor } : () -> () + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "x", type = tensor, value = dense<44.0> : tensor } : () -> () + // CHECK: func @f + // CHECK: %arg0: tensor>> {tf_saved_model.bound_input = @v} + // CHECK: %arg1: tensor>> {tf_saved_model.bound_input = @w} + // CHECK: %arg2: tensor>> {tf_saved_model.bound_input = @x} + // CHECK-NOT: %arg3 + // CHECK-NOT: %arg4 + func @f( + %arg0: tensor>> {tf_saved_model.bound_input = @v}, + %arg1: tensor>> {tf_saved_model.bound_input = @w}, + %arg2: tensor>> {tf_saved_model.bound_input = @v}, + %arg3: tensor>> {tf_saved_model.bound_input = @x}, + %arg4: tensor>> {tf_saved_model.bound_input = @v} + ) attributes {tf_saved_model.exported_names = ["f"]} { + // CHECK: "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg2) : (tensor>>) -> tensor + // CHECK: "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %val0 = "tf.ReadVariableOp"(%arg0) : (tensor>>) -> tensor + %val1 = "tf.ReadVariableOp"(%arg1) : (tensor>>) -> tensor + %val2 = "tf.ReadVariableOp"(%arg2) : (tensor>>) -> tensor + %val3 = "tf.ReadVariableOp"(%arg3) : (tensor>>) -> tensor + %val4 = "tf.ReadVariableOp"(%arg4) : (tensor>>) -> tensor + return + } +} diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops.mlir index 7156a1fab63..d2c5509b52d 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops.mlir @@ -76,3 +76,16 @@ module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} } } + +// ----- + +module attributes {tf_saved_model.semantics, tf_saved_model.under_construction} { + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.0> : tensor } : () -> () + // CHECK: func @f + func @f( + %arg0: tensor>> {tf_saved_model.bound_input = @v}, + %arg1: tensor>> {tf_saved_model.bound_input = @v} + ) attributes {tf_saved_model.exported_names = ["f"]} { + return + } +} diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops_invalid.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops_invalid.mlir index dcb889ff99e..714c8908825 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops_invalid.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/tf_saved_model_ops_invalid.mlir @@ -400,3 +400,17 @@ module attributes {tf_saved_model.semantics} { } } + +// ----- + +module attributes {tf_saved_model.semantics} { + + "tf_saved_model.global_tensor"() { is_mutable, sym_name = "v", type = tensor, value = dense<42.0> : tensor } : () -> () + // expected-error@+1 {{duplicate 'tf_saved_model.bound_input' binding}} + func @f( + %arg0: tensor>> {tf_saved_model.bound_input = @v}, + %arg1: tensor>> {tf_saved_model.bound_input = @v} + ) attributes {tf_saved_model.exported_names = ["f"]} { + return + } +} diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/deduplicate_bound_input_bindings.cc b/tensorflow/compiler/mlir/tensorflow/transforms/deduplicate_bound_input_bindings.cc new file mode 100644 index 00000000000..c1514dfa357 --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/transforms/deduplicate_bound_input_bindings.cc @@ -0,0 +1,65 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "llvm/ADT/DenseMap.h" +#include "mlir/IR/Function.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.h" + +namespace mlir { +namespace tf_saved_model { +namespace { + +class DedupBoundInputBindingPass + : public PassWrapper { + public: + void runOnFunction() override; +}; + +void DedupBoundInputBindingPass::runOnFunction() { + FuncOp func = getFunction(); + if (!mlir::tf_saved_model::IsExported(func)) return; + llvm::SmallDenseMap unique_bound_inputs; + llvm::SmallVector arg_indices_to_erase; + for (unsigned i = 0, e = func.getNumArguments(); i < e; i++) { + auto attr = func.getArgAttrOfType( + i, "tf_saved_model.bound_input"); + if (!attr) continue; + auto inserted = unique_bound_inputs.insert(std::make_pair(attr, i)); + if (inserted.second) continue; + auto duplicate_arg = func.getArgument(i); + auto original_arg = func.getArgument(unique_bound_inputs[attr]); + duplicate_arg.replaceAllUsesWith(original_arg); + arg_indices_to_erase.push_back(i); + } + func.eraseArguments(arg_indices_to_erase); +} + +} // namespace + +static PassRegistration pass( + "tf-saved-model-dedup-bound-input-binding-pass", + "Remove duplicate 'tf_saved_model.bound_input' bindings."); + +std::unique_ptr> CreateDedupBoundInputBindingPass() { + return std::make_unique(); +} + +} // namespace tf_saved_model +} // namespace mlir diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h b/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h index f7a73dc1561..d46b81156f9 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h +++ b/tensorflow/compiler/mlir/tensorflow/transforms/tf_saved_model_passes.h @@ -46,6 +46,9 @@ CreateRemoveVariablesInSessionInitializerPass(); std::unique_ptr> CreateLiftVariablesPass( ::tensorflow::Session* session); +// Creates a pass that removes duplicate 'tf_saved_model.bound_input' bindings. +std::unique_ptr> CreateDedupBoundInputBindingPass(); + } // namespace tf_saved_model } // namespace mlir diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc b/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc index 2c44aaa5c42..27385e81262 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc @@ -3368,12 +3368,13 @@ SavedModelSignatureDefImporter::ConvertAssets() { results.reserve(asset_file_defs.size()); mlir::OpBuilder builder(module_->getBodyRegion()); + unsigned i = 0; // Use to generate unique sym_name(s) for duplicate assets. for (const auto& asset : asset_file_defs) { auto asset_op = builder.create( module_->getLoc(), /*sym_name=*/ builder.getStringAttr( - absl::StrCat("__tf_saved_model_asset_", asset.filename())), + absl::StrCat("__tf_saved_model_asset", i++, "_", asset.filename())), /*filename=*/ builder.getStringAttr( io::JoinPath(kSavedModelAssetsDirectory, asset.filename()))); @@ -3569,6 +3570,7 @@ Status SavedModelSignatureDefImporter::LiftVariables() { pm.addPass(mlir::TF::CreatePromoteVarHandlesToArgsPass()); pm.addPass( mlir::tf_saved_model::CreateLiftVariablesPass(bundle_.GetSession())); + pm.addPass(mlir::tf_saved_model::CreateDedupBoundInputBindingPass()); if (mlir::failed(pm.run(*module_))) return diag_handler.Combine(errors::Internal("Failed to lift variables.")); From cec4c62fa2545135065003d3ca19f8df3e688efd Mon Sep 17 00:00:00 2001 From: Nat Jeffries Date: Mon, 3 Aug 2020 13:20:41 -0700 Subject: [PATCH 0273/1017] Remove all instances of initializer_list from TF Micro kernel tests and clean up tests. PiperOrigin-RevId: 324669800 Change-Id: Ib47abca412a47f30bfc804f21e3194ded3e72bc5 --- .../micro_speech/recognize_commands_test.cc | 45 +- .../lite/micro/kernels/concatenation_test.cc | 127 +- .../micro/kernels/maximum_minimum_test.cc | 205 ++- tensorflow/lite/micro/kernels/mul_test.cc | 467 ++---- tensorflow/lite/micro/kernels/neg_test.cc | 38 +- tensorflow/lite/micro/kernels/pack_test.cc | 316 ++-- tensorflow/lite/micro/kernels/pooling_test.cc | 1319 +++++++---------- tensorflow/lite/micro/kernels/prelu_test.cc | 240 +-- tensorflow/lite/micro/kernels/reshape_test.cc | 9 +- tensorflow/lite/micro/testing/test_utils.cc | 28 - tensorflow/lite/micro/testing/test_utils.h | 34 - 11 files changed, 1121 insertions(+), 1707 deletions(-) diff --git a/tensorflow/lite/micro/examples/micro_speech/recognize_commands_test.cc b/tensorflow/lite/micro/examples/micro_speech/recognize_commands_test.cc index eff7b4eb37b..089da9173c7 100644 --- a/tensorflow/lite/micro/examples/micro_speech/recognize_commands_test.cc +++ b/tensorflow/lite/micro/examples/micro_speech/recognize_commands_test.cc @@ -75,11 +75,11 @@ TF_LITE_MICRO_TEST(RecognizeCommandsTestBasic) { RecognizeCommands recognize_commands(µ_error_reporter); - std::initializer_list result_data = {127, -128, -128, -128}; - auto result_dims = {2, 1, 4}; + const int8_t result_data[] = {127, -128, -128, -128}; + const int result_dims[] = {2, 1, 4}; TfLiteTensor results = tflite::testing::CreateQuantizedTensor( - result_data, tflite::testing::IntArrayFromInitializer(result_dims), - -128.0f, 127.0f); + result_data, tflite::testing::IntArrayFromInts(result_dims), -128.0f, + 127.0f); const char* found_command; uint8_t score; @@ -94,11 +94,10 @@ TF_LITE_MICRO_TEST(RecognizeCommandsTestFindCommands) { RecognizeCommands recognize_commands(µ_error_reporter, 1000, 51); - std::initializer_list yes_data = {-128, -128, 127, -128}; - auto yes_dims = {2, 1, 4}; + const int8_t yes_data[] = {-128, -128, 127, -128}; + const int yes_dims[] = {2, 1, 4}; TfLiteTensor yes_results = tflite::testing::CreateQuantizedTensor( - yes_data, tflite::testing::IntArrayFromInitializer(yes_dims), -128.0f, - 127.0f); + yes_data, tflite::testing::IntArrayFromInts(yes_dims), -128.0f, 127.0f); bool has_found_new_command = false; const char* new_command; @@ -122,11 +121,10 @@ TF_LITE_MICRO_TEST(RecognizeCommandsTestFindCommands) { TF_LITE_MICRO_EXPECT_EQ(0, tflite::testing::TestStrcmp("yes", new_command)); } - std::initializer_list no_data = {-128, -128, -128, 127}; - auto no_dims = {2, 1, 4}; + const int8_t no_data[] = {-128, -128, -128, 127}; + const int no_dims[] = {2, 1, 4}; TfLiteTensor no_results = tflite::testing::CreateQuantizedTensor( - no_data, tflite::testing::IntArrayFromInitializer(no_dims), -128.0f, - 127.0f); + no_data, tflite::testing::IntArrayFromInts(no_dims), -128.0f, 127.0f); has_found_new_command = false; new_command = ""; uint8_t score; @@ -156,11 +154,10 @@ TF_LITE_MICRO_TEST(RecognizeCommandsTestBadInputLength) { RecognizeCommands recognize_commands(µ_error_reporter, 1000, 51); - std::initializer_list bad_data = {-128, -128, 127}; - auto bad_dims = {2, 1, 3}; + const int8_t bad_data[] = {-128, -128, 127}; + const int bad_dims[] = {2, 1, 3}; TfLiteTensor bad_results = tflite::testing::CreateQuantizedTensor( - bad_data, tflite::testing::IntArrayFromInitializer(bad_dims), -128.0f, - 127.0f); + bad_data, tflite::testing::IntArrayFromInts(bad_dims), -128.0f, 127.0f); const char* found_command; uint8_t score; @@ -175,11 +172,11 @@ TF_LITE_MICRO_TEST(RecognizeCommandsTestBadInputTimes) { RecognizeCommands recognize_commands(µ_error_reporter, 1000, 51); - std::initializer_list result_data = {-128, -128, 127, -128}; - auto result_dims = {2, 1, 4}; + const int8_t result_data[] = {-128, -128, 127, -128}; + const int result_dims[] = {2, 1, 4}; TfLiteTensor results = tflite::testing::CreateQuantizedTensor( - result_data, tflite::testing::IntArrayFromInitializer(result_dims), - -128.0f, 127.0f); + result_data, tflite::testing::IntArrayFromInts(result_dims), -128.0f, + 127.0f); const char* found_command; uint8_t score; @@ -197,11 +194,11 @@ TF_LITE_MICRO_TEST(RecognizeCommandsTestTooFewInputs) { RecognizeCommands recognize_commands(µ_error_reporter, 1000, 51); - std::initializer_list result_data = {-128, -128, 127, -128}; - auto result_dims = {2, 1, 4}; + const int8_t result_data[] = {-128, -128, 127, -128}; + const int result_dims[] = {2, 1, 4}; TfLiteTensor results = tflite::testing::CreateQuantizedTensor( - result_data, tflite::testing::IntArrayFromInitializer(result_dims), - -128.0f, 127.0f); + result_data, tflite::testing::IntArrayFromInts(result_dims), -128.0f, + 127.0f); const char* found_command; uint8_t score; diff --git a/tensorflow/lite/micro/kernels/concatenation_test.cc b/tensorflow/lite/micro/kernels/concatenation_test.cc index c3fa395600f..d82a804e659 100644 --- a/tensorflow/lite/micro/kernels/concatenation_test.cc +++ b/tensorflow/lite/micro/kernels/concatenation_test.cc @@ -24,17 +24,16 @@ namespace tflite { namespace testing { namespace { -void TestConcatenateTwoInputs(std::initializer_list input1_dims_data, - std::initializer_list input1_data, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, - int axis, - std::initializer_list output_dims_data, - std::initializer_list expected_output_data, +void TestConcatenateTwoInputs(const int* input1_dims_data, + const float* input1_data, + const int* input2_dims_data, + const float* input2_data, int axis, + const int* output_dims_data, + const float* expected_output_data, float* output_data) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); constexpr int input_size = 2; constexpr int output_size = 1; @@ -65,30 +64,31 @@ void TestConcatenateTwoInputs(std::initializer_list input1_dims_data, const int output_dims_count = ElementCount(*output_dims); for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - 1e-5f); + TF_LITE_MICRO_EXPECT_NEAR(expected_output_data[i], output_data[i], 1e-5f); } } void TestConcatenateQuantizedTwoInputs( - std::initializer_list input1_dims_data, - std::initializer_list input1_data, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, float input_min, - float input_max, int axis, std::initializer_list output_dims_data, - std::initializer_list expected_output_data, float output_min, - float output_max, uint8_t* output_data) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); + const int* input1_dims_data, const uint8_t* input1_data, + const int* input2_dims_data, const uint8_t* input2_data, + const float input_scale, const int input_zero_point, int axis, + const int* output_dims_data, const uint8_t* expected_output_data, + const float output_scale, const int output_zero_point, + uint8_t* output_data) { + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); constexpr int input_size = 2; constexpr int output_size = 1; constexpr int tensors_size = input_size + output_size; TfLiteTensor tensors[tensors_size] = { - CreateQuantizedTensor(input1_data, input1_dims, input_min, input_max), - CreateQuantizedTensor(input2_data, input2_dims, input_min, input_max), - CreateQuantizedTensor(output_data, output_dims, output_min, output_max)}; + CreateQuantizedTensor(input1_data, input1_dims, input_scale, + input_zero_point), + CreateQuantizedTensor(input2_data, input2_dims, input_scale, + input_zero_point), + CreateQuantizedTensor(output_data, output_dims, output_scale, + output_zero_point)}; int inputs_array_data[] = {2, 0, 1}; TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); @@ -111,7 +111,7 @@ void TestConcatenateQuantizedTwoInputs( const int output_dims_count = ElementCount(*output_dims); for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_EQ(expected_output_data.begin()[i], output_data[i]); + TF_LITE_MICRO_EXPECT_EQ(expected_output_data[i], output_data[i]); } } @@ -124,19 +124,19 @@ TF_LITE_MICRO_TESTS_BEGIN TF_LITE_MICRO_TEST(TwoInputsAllAxesCombinations) { // Concatenate the same two input tensors along all possible axes. - auto input_shape = {2, 2, 3}; - auto input1_value = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; - auto input2_value = {7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f}; + const int input_shape[] = {2, 2, 3}; + const float input1_value[] = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + const float input2_value[] = {7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f}; // expected output when concatenating on axis 0 - auto output_shape_axis0 = {2, 4, 3}; - auto output_value_axis0 = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, - 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f}; + const int output_shape_axis0[] = {2, 4, 3}; + const float output_value_axis0[] = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, + 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f}; // expected output when concatenating on axis 1 - auto output_shape_axis1 = {2, 2, 6}; - auto output_value_axis1 = {1.0f, 2.0f, 3.0f, 7.0f, 8.0f, 9.0f, - 4.0f, 5.0f, 6.0f, 10.0f, 11.0f, 12.0f}; + const int output_shape_axis1[] = {2, 2, 6}; + const float output_value_axis1[] = {1.0f, 2.0f, 3.0f, 7.0f, 8.0f, 9.0f, + 4.0f, 5.0f, 6.0f, 10.0f, 11.0f, 12.0f}; float output_data[12]; @@ -162,59 +162,48 @@ TF_LITE_MICRO_TEST(TwoInputsAllAxesCombinations) { } TF_LITE_MICRO_TEST(TwoInputsQuantizedUint8) { - using tflite::testing::F2Q; - const int axis = 2; - auto input_shape = {3, 2, 1, 2}; - auto output_shape = {3, 2, 1, 4}; + const int input_shape[] = {3, 2, 1, 2}; + const int output_shape[] = {3, 2, 1, 4}; - const float input_min = -12.7f; - const float input_max = 12.8f; - const float output_min = -12.7f; - const float output_max = 12.8f; + const float input_scale = 0.1f; + const int input_zero_point = 127; + const float output_scale = 0.1f; + const int output_zero_point = 127; - auto input1_value = { - F2Q(1.0, input_min, input_max), - F2Q(3.0, input_min, input_max), - F2Q(4.0, input_min, input_max), - F2Q(7.0, input_min, input_max), - }; + const uint8_t input1_values[] = {137, 157, 167, 197}; - auto input2_value = { - F2Q(1.1, input_min, input_max), - F2Q(3.1, input_min, input_max), - F2Q(4.1, input_min, input_max), - F2Q(7.1, input_min, input_max), - }; + const uint8_t input2_values[] = {138, 158, 168, 198}; - std::initializer_list output_value = { + const uint8_t output_value[] = { 137, 157, 138, 158, 167, 197, 168, 198, }; uint8_t output_data[8]; tflite::testing::TestConcatenateQuantizedTwoInputs( - input_shape, input1_value, input_shape, input2_value, input_min, - input_max, axis, output_shape, output_value, output_min, output_max, - output_data); + input_shape, input1_values, input_shape, input2_values, input_scale, + input_zero_point, axis, output_shape, output_value, output_scale, + output_zero_point, output_data); } TF_LITE_MICRO_TEST(ThreeDimensionalTwoInputsDifferentShapes) { const int axis = 1; - auto input1_shape = {3, 2, 1, 2}; - auto input2_shape = {3, 2, 3, 2}; - auto output_shape = {3, 2, 4, 2}; + const int input1_shape[] = {3, 2, 1, 2}; + const int input2_shape[] = {3, 2, 3, 2}; + const int output_shape[] = {3, 2, 4, 2}; - auto input1_value = {1.0f, 3.0f, 4.0f, 7.0f}; - auto input2_value = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, - 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f}; - auto output_value = {1.0f, 3.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, - 4.0f, 7.0f, 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f}; + const float input1_values[] = {1.0f, 3.0f, 4.0f, 7.0f}; + const float input2_values[] = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, + 7.0f, 8.0f, 9.0f, 10.0f, 11.0f, 12.0f}; + const float output_values[] = {1.0f, 3.0f, 1.0f, 2.0f, 3.0f, 4.0f, + 5.0f, 6.0f, 4.0f, 7.0f, 7.0f, 8.0f, + 9.0f, 10.0f, 11.0f, 12.0f}; float output_data[16]; tflite::testing::TestConcatenateTwoInputs( - input1_shape, input1_value, input2_shape, input2_value, axis, - output_shape, output_value, output_data); + input1_shape, input1_values, input2_shape, input2_values, axis, + output_shape, output_values, output_data); } TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/kernels/maximum_minimum_test.cc b/tensorflow/lite/micro/kernels/maximum_minimum_test.cc index ee84fcba497..7fab5407cdb 100644 --- a/tensorflow/lite/micro/kernels/maximum_minimum_test.cc +++ b/tensorflow/lite/micro/kernels/maximum_minimum_test.cc @@ -25,16 +25,13 @@ namespace testing { namespace { void TestMaxMinFloat(const TfLiteRegistration& registration, - std::initializer_list input1_dims_data, - std::initializer_list input1_data, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, - std::initializer_list expected_output_data, - std::initializer_list output_dims_data, - float* output_data) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); + const int* input1_dims_data, const float* input1_data, + const int* input2_dims_data, const float* input2_data, + const float* expected_output_data, + const int* output_dims_data, float* output_data) { + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 2; @@ -59,34 +56,35 @@ void TestMaxMinFloat(const TfLiteRegistration& registration, TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - 1e-5f); + TF_LITE_MICRO_EXPECT_NEAR(expected_output_data[i], output_data[i], 1e-5f); } } void TestMaxMinQuantized(const TfLiteRegistration& registration, - std::initializer_list input1_dims_data, - std::initializer_list input1_data, - float input1_min, float input1_max, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, - float input2_min, float input2_max, - std::initializer_list expected_output_data, - float output_min, float output_max, - std::initializer_list output_dims_data, - uint8_t* output_data) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); + const int* input1_dims_data, + const uint8_t* input1_data, float const input1_scale, + const int input1_zero_point, + const int* input2_dims_data, + const uint8_t* input2_data, const float input2_scale, + const int input2_zero_point, + const uint8_t* expected_output_data, + const float output_scale, const int output_zero_point, + const int* output_dims_data, uint8_t* output_data) { + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 2; constexpr int outputs_size = 1; constexpr int tensors_size = inputs_size + outputs_size; TfLiteTensor tensors[tensors_size] = { - CreateQuantizedTensor(input1_data, input1_dims, input1_min, input1_max), - CreateQuantizedTensor(input2_data, input2_dims, input2_min, input2_max), - CreateQuantizedTensor(output_data, output_dims, output_min, output_max), + CreateQuantizedTensor(input1_data, input1_dims, input1_scale, + input1_zero_point), + CreateQuantizedTensor(input2_data, input2_dims, input2_scale, + input2_zero_point), + CreateQuantizedTensor(output_data, output_dims, output_scale, + output_zero_point), }; int inputs_array_data[] = {2, 0, 1}; @@ -102,21 +100,21 @@ void TestMaxMinQuantized(const TfLiteRegistration& registration, TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_EQ(expected_output_data.begin()[i], output_data[i]); + TF_LITE_MICRO_EXPECT_EQ(expected_output_data[i], output_data[i]); } } -void TestMaxMinQuantizedInt32( - const TfLiteRegistration& registration, - std::initializer_list input1_dims_data, - std::initializer_list input1_data, float input1_scale, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, float input2_scale, - std::initializer_list expected_output_data, float output_scale, - std::initializer_list output_dims_data, int32_t* output_data) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); +void TestMaxMinQuantizedInt32(const TfLiteRegistration& registration, + const int* input1_dims_data, + const int32_t* input1_data, float input1_scale, + const int* input2_dims_data, + const int32_t* input2_data, float input2_scale, + const int32_t* expected_output_data, + float output_scale, const int* output_dims_data, + int32_t* output_data) { + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 2; @@ -141,7 +139,7 @@ void TestMaxMinQuantizedInt32( TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_EQ(expected_output_data.begin()[i], output_data[i]); + TF_LITE_MICRO_EXPECT_EQ(expected_output_data[i], output_data[i]); } } @@ -152,109 +150,86 @@ void TestMaxMinQuantizedInt32( TF_LITE_MICRO_TESTS_BEGIN TF_LITE_MICRO_TEST(FloatTest) { - std::initializer_list data1 = {1.0, 0.0, -1.0, 11.0, -2.0, -1.44}; - std::initializer_list data2 = {-1.0, 0.0, 1.0, 12.0, -3.0, -1.43}; + const int dims[] = {3, 3, 1, 2}; + const float data1[] = {1.0, 0.0, -1.0, 11.0, -2.0, -1.44}; + const float data2[] = {-1.0, 0.0, 1.0, 12.0, -3.0, -1.43}; + const float golden_max[] = {1.0, 0.0, 1.0, 12.0, -2.0, -1.43}; + const float golden_min[] = {-1.0, 0.0, -1.0, 11.0, -3.0, -1.44}; float output_data[6]; - tflite::testing::TestMaxMinFloat( - tflite::ops::micro::Register_MAXIMUM(), {3, 3, 1, 2}, - data1, // input1 shape and data - {3, 3, 1, 2}, data2, // input2 shape and data - {1.0, 0.0, 1.0, 12.0, -2.0, -1.43}, // expected output - {3, 3, 1, 2}, output_data); // output shape and data buffer + tflite::testing::TestMaxMinFloat(tflite::ops::micro::Register_MAXIMUM(), dims, + data1, dims, data2, golden_max, dims, + output_data); - tflite::testing::TestMaxMinFloat( - tflite::ops::micro::Register_MINIMUM(), {3, 3, 1, 2}, - data1, // input1 shape and data - {3, 3, 1, 2}, data2, // input2 shape and data - {-1.0, 0.0, -1.0, 11.0, -3.0, -1.44}, // expected output - {3, 3, 1, 2}, output_data); // output shape and data buffer + tflite::testing::TestMaxMinFloat(tflite::ops::micro::Register_MINIMUM(), dims, + data1, dims, data2, golden_min, dims, + output_data); } TF_LITE_MICRO_TEST(Uint8Test) { - std::initializer_list data1 = {1, 0, 2, 11, 2, 23}; - std::initializer_list data2 = {0, 0, 1, 12, 255, 1}; - const float input1_min = -63.5; - const float input1_max = 64; - const float input2_min = -63.5; - const float input2_max = 64; - const float output_min = -63.5; - const float output_max = 64; + const int dims[] = {3, 3, 1, 2}; + const uint8_t data1[] = {1, 0, 2, 11, 2, 23}; + const uint8_t data2[] = {0, 0, 1, 12, 255, 1}; + const uint8_t golden_max[] = {1, 0, 2, 12, 255, 23}; + const uint8_t golden_min[] = {0, 0, 1, 11, 2, 1}; + + const float input_scale = 1.0; + const int input_zero_point = 0; + const float output_scale = 1.0; + const int output_zero_point = 0; uint8_t output_data[6]; tflite::testing::TestMaxMinQuantized( - tflite::ops::micro::Register_MAXIMUM(), - // input1 shape, data and bounds - {3, 3, 1, 2}, data1, input1_min, input1_max, - // input2 shape, data and bounds - {3, 3, 1, 2}, data2, input2_min, input2_max, - // expected output - {1, 0, 2, 12, 255, 23}, - // output bounds, shape and data buffer - output_min, output_max, {3, 3, 1, 2}, output_data); + tflite::ops::micro::Register_MAXIMUM(), dims, data1, input_scale, + input_zero_point, dims, data2, input_scale, input_zero_point, golden_max, + output_scale, output_zero_point, dims, output_data); tflite::testing::TestMaxMinQuantized( - tflite::ops::micro::Register_MINIMUM(), - // input1 shape, data and bounds - {3, 3, 1, 2}, data1, input1_min, input1_max, - // input2 shape, data and bounds - {3, 3, 1, 2}, data2, input2_min, input2_max, - // expected output - {0, 0, 1, 11, 2, 1}, - // output bounds, shape and data buffer - output_min, output_max, {3, 3, 1, 2}, output_data); + tflite::ops::micro::Register_MINIMUM(), dims, data1, input_scale, + input_zero_point, dims, data2, input_scale, input_zero_point, golden_min, + output_scale, output_zero_point, dims, output_data); } TF_LITE_MICRO_TEST(FloatWithBroadcastTest) { - std::initializer_list data1 = {1.0, 0.0, -1.0, -2.0, -1.44, 11.0}; - std::initializer_list data2 = {0.5, 2.0}; + const int dims[] = {3, 3, 1, 2}; + const int dims_scalar[] = {1, 2}; + const float data1[] = {1.0, 0.0, -1.0, -2.0, -1.44, 11.0}; + const float data2[] = {0.5, 2.0}; + const float golden_max[] = {1.0, 2.0, 0.5, 2.0, 0.5, 11.0}; + const float golden_min[] = {0.5, 0.0, -1.0, -2.0, -1.44, 2.0}; float output_data[6]; - tflite::testing::TestMaxMinFloat( - tflite::ops::micro::Register_MAXIMUM(), {3, 3, 1, 2}, - data1, // input1 shape and data - {1, 2}, data2, // input2 shape and data - {1.0, 2.0, 0.5, 2.0, 0.5, 11.0}, // expected output - {3, 3, 1, 2}, output_data); // output shape and data buffer + tflite::testing::TestMaxMinFloat(tflite::ops::micro::Register_MAXIMUM(), dims, + data1, dims_scalar, data2, golden_max, dims, + output_data); - tflite::testing::TestMaxMinFloat( - tflite::ops::micro::Register_MINIMUM(), {3, 3, 1, 2}, - data1, // input1 shape and data - {1, 2}, data2, // input2 shape and data - {0.5, 0.0, -1.0, -2.0, -1.44, 2.0}, // expected output - {3, 3, 1, 2}, output_data); // output shape and data buffer + tflite::testing::TestMaxMinFloat(tflite::ops::micro::Register_MINIMUM(), dims, + data1, dims_scalar, data2, golden_min, dims, + output_data); } TF_LITE_MICRO_TEST(Int32WithBroadcastTest) { const float input1_scale = 0.5; const float input2_scale = 0.5; const float output_scale = 0.5; - std::initializer_list data1 = {1, 0, -1, -2, 3, 11}; - std::initializer_list data2 = {2}; + const int dims[] = {3, 3, 1, 2}; + const int dims_scalar[] = {1, 1}; + const int32_t data1[] = {1, 0, -1, -2, 3, 11}; + const int32_t data2[] = {2}; + const int32_t golden_max[] = {2, 2, 2, 2, 3, 11}; + const int32_t golden_min[] = {1, 0, -1, -2, 2, 2}; int32_t output_data[6]; tflite::testing::TestMaxMinQuantizedInt32( - tflite::ops::micro::Register_MAXIMUM(), - // input1 shape, data and scale - {3, 3, 1, 2}, data1, input1_scale, - // input2 shape, data and scale - {1, 1}, data2, input2_scale, - // expected output - {2, 2, 2, 2, 3, 11}, - // output scale, shape and data buffer - output_scale, {3, 3, 1, 2}, output_data); + tflite::ops::micro::Register_MAXIMUM(), dims, data1, input1_scale, + dims_scalar, data2, input2_scale, golden_max, output_scale, dims, + output_data); tflite::testing::TestMaxMinQuantizedInt32( - tflite::ops::micro::Register_MINIMUM(), - // input1 shape, data and scale - {3, 3, 1, 2}, data1, input1_scale, - // input2 shape, data and scale - {1, 1}, data2, input2_scale, - // expected output - {1, 0, -1, -2, 2, 2}, - // output scale, shape and data buffer - output_scale, {3, 3, 1, 2}, output_data); + tflite::ops::micro::Register_MINIMUM(), dims, data1, input1_scale, + dims_scalar, data2, input2_scale, golden_min, output_scale, dims, + output_data); } TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/kernels/mul_test.cc b/tensorflow/lite/micro/kernels/mul_test.cc index e39f6693fcd..86b4d8be57c 100644 --- a/tensorflow/lite/micro/kernels/mul_test.cc +++ b/tensorflow/lite/micro/kernels/mul_test.cc @@ -23,16 +23,57 @@ namespace tflite { namespace testing { namespace { -void TestMulFloat(std::initializer_list input1_dims_data, - std::initializer_list input1_data, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, - std::initializer_list output_dims_data, - std::initializer_list expected_output_data, +const int flat_size_simple = 4; +const float scale_simple = 0.01; +const int dims_simple[] = {4, 1, 2, 2, 1}; +const float input1_simple[] = {-0.8, 0.2, 0.9, 0.7}; +const float input2_simple[] = {0.6, 0.4, 0.9, 0.8}; +const float golden_simple[] = {-0.48, 0.08, 0.81, 0.56}; +const float golden_simple_relu[] = {0.0, 0.08, 0.81, 0.56}; + +const int flat_size_broadcast = 6; +const float input_scale_broadcast = 0.05f; +const float output_scale_broadcast = 0.01f; +const int dims_broadcast[] = {4, 1, 3, 1, 2}; +const int dims_scalar_broadcast[] = {1, 1}; +const float input1_broadcast[] = {-2.0, 0.2, 0.7, 0.8, 1.1, 2.0}; +const float input2_broadcast[] = {0.1}; +const float golden_broadcast[] = {-0.2, 0.02, 0.07, 0.08, 0.11, 0.2}; +const float golden_broadcast_relu[] = {0, 0.02, 0.07, 0.08, 0.11, 0.2}; + +template +void ValidateMulGoldens(TfLiteTensor* tensors, int tensors_size, + TfLiteFusedActivation activation, const T* golden, + int output_len, float tolerance, T* output) { + TfLiteMulParams builtin_data = { + .activation = activation, + }; + + int inputs_array_data[] = {2, 0, 1}; + TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); + int outputs_array_data[] = {1, 2}; + TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); + + const TfLiteRegistration registration = tflite::ops::micro::Register_MUL(); + micro::KernelRunner runner( + registration, tensors, tensors_size, inputs_array, outputs_array, + reinterpret_cast(&builtin_data), micro_test::reporter); + + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); + + for (int i = 0; i < output_len; i++) { + TF_LITE_MICRO_EXPECT_NEAR(golden[i], output[i], tolerance); + } +} + +void TestMulFloat(const int* input1_dims_data, const float* input1_data, + const int* input2_dims_data, const float* input2_data, + const int* output_dims_data, const float* golden, float* output_data, TfLiteFusedActivation activation) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 2; @@ -44,75 +85,40 @@ void TestMulFloat(std::initializer_list input1_dims_data, CreateFloatTensor(output_data, output_dims), }; - TfLiteMulParams builtin_data = { - .activation = activation, - }; - - int inputs_array_data[] = {2, 0, 1}; - TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); - int outputs_array_data[] = {1, 2}; - TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - - const TfLiteRegistration registration = tflite::ops::micro::Register_MUL(); - micro::KernelRunner runner( - registration, tensors, tensors_size, inputs_array, outputs_array, - reinterpret_cast(&builtin_data), micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; i++) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - 1e-5f); - } + ValidateMulGoldens(tensors, tensors_size, activation, golden, + output_dims_count, 1e-5, output_data); } template -void TestMulQuantized(std::initializer_list input1_dims_data, - std::initializer_list input1_data, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, - const float input_min, const float input_max, - std::initializer_list output_dims_data, - const float output_min, const float output_max, - std::initializer_list expected_output_data, - T* output_data, TfLiteFusedActivation activation, - int error_tolerance) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); +void TestMulQuantized(const int* input1_dims_data, const float* input1_data, + T* input1_quantized, const int* input2_dims_data, + const float* input2_data, T* input2_quantized, + const float input_scale, const int input_zero_point, + const int* output_dims_data, const float* golden, + T* golden_quantized, const float output_scale, + const int output_zero_point, T* output_data, + TfLiteFusedActivation activation) { + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 2; constexpr int outputs_size = 1; constexpr int tensors_size = inputs_size + outputs_size; TfLiteTensor tensors[tensors_size] = { - CreateQuantizedTensor(input1_data, input1_dims, input_min, input_max), - CreateQuantizedTensor(input2_data, input2_dims, input_min, input_max), - CreateQuantizedTensor(output_data, output_dims, output_min, output_max), - }; + CreateQuantizedTensor(input1_data, input1_quantized, input1_dims, + input_scale, input_zero_point), + CreateQuantizedTensor(input2_data, input2_quantized, input2_dims, + input_scale, input_zero_point), + CreateQuantizedTensor(output_data, output_dims, output_scale, + output_zero_point)}; - TfLiteMulParams builtin_data = { - .activation = activation, - }; + AsymmetricQuantize(golden, golden_quantized, output_dims_count, output_scale, + output_zero_point); - int inputs_array_data[] = {2, 0, 1}; - TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); - int outputs_array_data[] = {1, 2}; - TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - - const TfLiteRegistration registration = tflite::ops::micro::Register_MUL(); - micro::KernelRunner runner( - registration, tensors, tensors_size, inputs_array, outputs_array, - reinterpret_cast(&builtin_data), micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; i++) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - error_tolerance); - } + ValidateMulGoldens(tensors, tensors_size, activation, golden_quantized, + output_dims_count, 1.0f, output_data); } } // namespace @@ -122,250 +128,105 @@ void TestMulQuantized(std::initializer_list input1_dims_data, TF_LITE_MICRO_TESTS_BEGIN -TF_LITE_MICRO_TEST(Int8NoActivation) { - using tflite::testing::F2QS; - const float input_min = -1; - const float input_max = 1; - const float output_min = -1; - const float output_max = 1; +TF_LITE_MICRO_TEST(SimpleFloatNoAcativationShouldMatchGolden) { + float output_data[tflite::testing::flat_size_simple]; - int8_t output_data[4]; - tflite::testing::TestMulQuantized({4, 1, 2, 2, 1}, // input1 dims - { - F2QS(-0.8, input_min, input_max), - F2QS(0.2, input_min, input_max), - F2QS(0.9, input_min, input_max), - F2QS(0.7, input_min, input_max), - }, // input1 data - {4, 1, 2, 2, 1}, // input2 dims - { - F2QS(0.6, input_min, input_max), - F2QS(0.4, input_min, input_max), - F2QS(0.9, input_min, input_max), - F2QS(0.8, input_min, input_max), - }, // input2 data - input_min, input_max, - {4, 1, 2, 2, 1}, // output dims - output_min, output_max, - { - F2QS(-0.48, output_min, output_max), - F2QS(0.08, output_min, output_max), - F2QS(0.81, output_min, output_max), - F2QS(0.56, output_min, output_max), - }, // expected output data - output_data, kTfLiteActNone, 1); -} - -TF_LITE_MICRO_TEST(Int8NoActivationLargeMultiplier) { - using tflite::testing::F2QS; - const float input_min = -100; - const float input_max = 100; - const float output_min = -10; - const float output_max = 10; - - int8_t output_data[4]; - tflite::testing::TestMulQuantized( - {4, 1, 2, 2, 1}, - { - F2QS(-4, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(3, input_min, input_max), - F2QS(1, input_min, input_max), - }, - {4, 1, 2, 2, 1}, - { - /* F2QS(-1, input_min, input_max), F2QS(-3, input_min, input_max), */ - F2QS(-1, input_min, input_max), - F2QS(-3, input_min, input_max), - F2QS(4, input_min, input_max), - F2QS(2, input_min, input_max), - }, - input_min, input_max, {4, 1, 2, 2, 1}, output_min, output_max, - { - F2QS(4, output_min, output_max), - F2QS(-6, output_min, output_max), - F2QS(12, output_min, output_max), - F2QS(2, output_min, output_max), - }, - // In Tensorflow Lite, this test have a max allowed error of 1.4f. - // A difference of 1.4 in floating points corresponds to 18 quantized - // for the output min/max [-10, 10]. - output_data, kTfLiteActNone, 18); -} - -TF_LITE_MICRO_TEST(Int8NoActivationBroadcast) { - using tflite::testing::F2QS; - const float input_min = -3.0; - const float input_max = 3.0; - const float output_min = -3.0; - const float output_max = 3.0; - - int8_t output_data[6]; - tflite::testing::TestMulQuantized({4, 1, 3, 1, 2}, // input1 shape - { - F2QS(-2.0, input_min, input_max), - F2QS(0.2, input_min, input_max), - F2QS(0.7, input_min, input_max), - F2QS(0.8, input_min, input_max), - F2QS(1.1, input_min, input_max), - F2QS(2.0, input_min, input_max), - }, // input1 data - {1, 1}, // input2 shape - { - F2QS(0.1, input_min, input_max), - }, // input2 data - input_min, input_max, - {4, 1, 3, 1, 2}, // output shape - output_min, output_max, - { - F2QS(-0.2, output_min, output_max), - F2QS(0.02, output_min, output_max), - F2QS(0.07, output_min, output_max), - F2QS(0.08, output_min, output_max), - F2QS(0.11, output_min, output_max), - F2QS(0.2, output_min, output_max), - }, // expected output data - output_data, kTfLiteActNone, 1); -} - -TF_LITE_MICRO_TEST(UInt8NoActivation) { - using tflite::testing::F2Q; - const float input_min = -1; - const float input_max = 1; - const float output_min = -1; - const float output_max = 1; - - uint8_t output_data[4]; - tflite::testing::TestMulQuantized({4, 1, 2, 2, 1}, // input1 dims - { - F2Q(-0.8, input_min, input_max), - F2Q(0.2, input_min, input_max), - F2Q(0.9, input_min, input_max), - F2Q(0.7, input_min, input_max), - }, // input1 data - {4, 1, 2, 2, 1}, // input2 dims - { - F2Q(0.6, input_min, input_max), - F2Q(0.4, input_min, input_max), - F2Q(0.9, input_min, input_max), - F2Q(0.8, input_min, input_max), - }, // input2 data - input_min, input_max, - {4, 1, 2, 2, 1}, // output dims - output_min, output_max, - { - F2Q(-0.48, output_min, output_max), - F2Q(0.08, output_min, output_max), - F2Q(0.81, output_min, output_max), - F2Q(0.56, output_min, output_max), - }, // expected output data - output_data, kTfLiteActNone, 1); -} - -TF_LITE_MICRO_TEST(UInt8NoActivationLargeMultiplier) { - using tflite::testing::F2Q; - const float input_min = -100; - const float input_max = 100; - const float output_min = -10; - const float output_max = 10; - - uint8_t output_data[4]; - tflite::testing::TestMulQuantized( - {4, 1, 2, 2, 1}, - { - F2Q(-4, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(3, input_min, input_max), - F2Q(1, input_min, input_max), - }, - {4, 1, 2, 2, 1}, - { - F2Q(-1, input_min, input_max), - F2Q(-3, input_min, input_max), - F2Q(4, input_min, input_max), - F2Q(2, input_min, input_max), - }, - input_min, input_max, {4, 1, 2, 2, 1}, output_min, output_max, - { - F2Q(4, output_min, output_max), - F2Q(-6, output_min, output_max), - F2Q(12, output_min, output_max), - F2Q(2, output_min, output_max), - }, - // In Tensorflow Lite, this test have a max allowed error of 1.4f. - // A difference of 1.4 in floating points corresponds to 18 quantized - // for the output min/max [-10, 10]. - output_data, kTfLiteActNone, 18); -} - -TF_LITE_MICRO_TEST(UInt8NoActivationBroadcast) { - using tflite::testing::F2Q; - const float input_min = -3.0; - const float input_max = 3.0; - const float output_min = -3.0; - const float output_max = 3.0; - - uint8_t output_data[6]; - tflite::testing::TestMulQuantized({4, 1, 3, 1, 2}, // input1 shape - { - F2Q(-2.0, input_min, input_max), - F2Q(0.2, input_min, input_max), - F2Q(0.7, input_min, input_max), - F2Q(0.8, input_min, input_max), - F2Q(1.1, input_min, input_max), - F2Q(2.0, input_min, input_max), - }, // input1 data - {1, 1}, // input2 shape - { - F2Q(0.1, input_min, input_max), - }, // input2 data - input_min, input_max, - {4, 1, 3, 1, 2}, // output shape - output_min, output_max, - { - F2Q(-0.2, output_min, output_max), - F2Q(0.02, output_min, output_max), - F2Q(0.07, output_min, output_max), - F2Q(0.08, output_min, output_max), - F2Q(0.11, output_min, output_max), - F2Q(0.2, output_min, output_max), - }, // expected output data - output_data, kTfLiteActNone, 1); -} - -TF_LITE_MICRO_TEST(FloatNoActivation) { - float output_data[4]; tflite::testing::TestMulFloat( - {4, 1, 2, 2, 1}, // input1 shape - {-2.0, 0.2, 0.7, 0.8}, // input1 data - {4, 1, 2, 2, 1}, // input2 shape - {0.1, 0.2, 0.3, 0.5}, // input2 data - {4, 1, 2, 2, 1}, // output shape - {-0.2, 0.04, 0.21, 0.4}, // expected output data + tflite::testing::dims_simple, tflite::testing::input1_simple, + tflite::testing::dims_simple, tflite::testing::input2_simple, + tflite::testing::dims_simple, tflite::testing::golden_simple, output_data, + kTfLiteActNone); +} + +TF_LITE_MICRO_TEST(SimpleFloatReluShouldMatchGolden) { + float output_data[tflite::testing::flat_size_simple]; + + tflite::testing::TestMulFloat( + tflite::testing::dims_simple, tflite::testing::input1_simple, + tflite::testing::dims_simple, tflite::testing::input2_simple, + tflite::testing::dims_simple, tflite::testing::golden_simple_relu, + output_data, kTfLiteActRelu); +} + +TF_LITE_MICRO_TEST(SimpleInt8NoAcativationShouldMatchGolden) { + int8_t input1_quantized[tflite::testing::flat_size_simple]; + int8_t input2_quantized[tflite::testing::flat_size_simple]; + int8_t golden_quantized[tflite::testing::flat_size_simple]; + int8_t output_data[tflite::testing::flat_size_simple]; + + tflite::testing::TestMulQuantized( + tflite::testing::dims_simple, tflite::testing::input1_simple, + input1_quantized, tflite::testing::dims_simple, + tflite::testing::input2_simple, input2_quantized, + tflite::testing::scale_simple, 0, tflite::testing::dims_simple, + tflite::testing::golden_simple, golden_quantized, + tflite::testing::scale_simple, 0, output_data, kTfLiteActNone); +} + +TF_LITE_MICRO_TEST(SimpleUInt8NoAcativationShouldMatchGolden) { + uint8_t input1_quantized[tflite::testing::flat_size_simple]; + uint8_t input2_quantized[tflite::testing::flat_size_simple]; + uint8_t golden_quantized[tflite::testing::flat_size_simple]; + uint8_t output_data[tflite::testing::flat_size_simple]; + + tflite::testing::TestMulQuantized( + tflite::testing::dims_simple, tflite::testing::input1_simple, + input1_quantized, tflite::testing::dims_simple, + tflite::testing::input2_simple, input2_quantized, + tflite::testing::scale_simple, 128, tflite::testing::dims_simple, + tflite::testing::golden_simple, golden_quantized, + tflite::testing::scale_simple, 128, output_data, kTfLiteActNone); +} + +TF_LITE_MICRO_TEST(BroadcastFloatNoActivationShouldMatchGolden) { + float output_data[tflite::testing::flat_size_broadcast]; + + tflite::testing::TestMulFloat( + tflite::testing::dims_broadcast, tflite::testing::input1_broadcast, + tflite::testing::dims_scalar_broadcast, tflite::testing::input2_broadcast, + tflite::testing::dims_broadcast, tflite::testing::golden_broadcast, output_data, kTfLiteActNone); } -TF_LITE_MICRO_TEST(FloatRelu) { - float output_data[4]; +TF_LITE_MICRO_TEST(BroadcastFloatReluShouldMatchGolden) { + float output_data[tflite::testing::flat_size_broadcast]; + tflite::testing::TestMulFloat( - {4, 1, 2, 2, 1}, // input1 shape - {-2.0, 0.2, 0.7, 0.8}, // input1 data - {4, 1, 2, 2, 1}, // input2 shape - {0.1, 0.2, 0.3, 0.5}, // input2 data - {4, 1, 2, 2, 1}, // output shape - {-0.2, 0.04, 0.21, 0.4}, // expected output data - output_data, kTfLiteActReluN1To1); + tflite::testing::dims_broadcast, tflite::testing::input1_broadcast, + tflite::testing::dims_scalar_broadcast, tflite::testing::input2_broadcast, + tflite::testing::dims_broadcast, tflite::testing::golden_broadcast_relu, + output_data, kTfLiteActRelu); } -TF_LITE_MICRO_TEST(FloatBroadcast) { - float output_data[6]; - tflite::testing::TestMulFloat( - {4, 1, 3, 1, 2}, // input1 shape - {-2.0, 0.2, 0.7, 0.8, 1.1, 2.0}, // input1 data - {1, 1}, // input2 shape - {0.1}, // input2 data - {4, 1, 3, 1, 2}, // output shape - {-0.2, 0.02, 0.07, 0.08, 0.11, 0.2}, // expected output data +TF_LITE_MICRO_TEST(BroadcastInt8NoAcativationShouldMatchGolden) { + int8_t input1_quantized[tflite::testing::flat_size_broadcast]; + int8_t input2_quantized[tflite::testing::flat_size_broadcast]; + int8_t golden_quantized[tflite::testing::flat_size_broadcast]; + int8_t output_data[tflite::testing::flat_size_broadcast]; + + tflite::testing::TestMulQuantized( + tflite::testing::dims_broadcast, tflite::testing::input1_broadcast, + input1_quantized, tflite::testing::dims_scalar_broadcast, + tflite::testing::input2_broadcast, input2_quantized, + tflite::testing::input_scale_broadcast, 0, + tflite::testing::dims_broadcast, tflite::testing::golden_broadcast, + golden_quantized, tflite::testing::output_scale_broadcast, 0, output_data, + kTfLiteActNone); +} + +TF_LITE_MICRO_TEST(BroadcastUInt8NoAcativationShouldMatchGolden) { + uint8_t input1_quantized[tflite::testing::flat_size_broadcast]; + uint8_t input2_quantized[1]; + uint8_t golden_quantized[tflite::testing::flat_size_broadcast]; + uint8_t output_data[tflite::testing::flat_size_broadcast]; + + tflite::testing::TestMulQuantized( + tflite::testing::dims_broadcast, tflite::testing::input1_broadcast, + input1_quantized, tflite::testing::dims_scalar_broadcast, + tflite::testing::input2_broadcast, input2_quantized, + tflite::testing::input_scale_broadcast, 128, + tflite::testing::dims_broadcast, tflite::testing::golden_broadcast, + golden_quantized, tflite::testing::output_scale_broadcast, 128, output_data, kTfLiteActNone); } diff --git a/tensorflow/lite/micro/kernels/neg_test.cc b/tensorflow/lite/micro/kernels/neg_test.cc index 2d7c449fcef..544a3eddc1c 100644 --- a/tensorflow/lite/micro/kernels/neg_test.cc +++ b/tensorflow/lite/micro/kernels/neg_test.cc @@ -24,13 +24,11 @@ namespace tflite { namespace testing { namespace { -void TestNegFloat(std::initializer_list input_dims_data, - std::initializer_list input_data, - std::initializer_list expected_output_data, - std::initializer_list output_dims_data, - float* output_data) { - TfLiteIntArray* input_dims = IntArrayFromInitializer(input_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); +void TestNegFloat(const int* input_dims_data, const float* input_data, + const float* expected_output_data, + const int* output_dims_data, float* output_data) { + TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 1; constexpr int outputs_size = 1; @@ -53,9 +51,9 @@ void TestNegFloat(std::initializer_list input_dims_data, TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - TF_LITE_MICRO_EXPECT_EQ(expected_output_data.begin()[0], output_data[0]); + TF_LITE_MICRO_EXPECT_EQ(expected_output_data[0], output_data[0]); for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_EQ(expected_output_data.begin()[i], output_data[i]); + TF_LITE_MICRO_EXPECT_EQ(expected_output_data[i], output_data[i]); } } @@ -66,23 +64,21 @@ void TestNegFloat(std::initializer_list input_dims_data, TF_LITE_MICRO_TESTS_BEGIN TF_LITE_MICRO_TEST(NegOpSingleFloat) { + const int dims[] = {1, 2}; + const float input_data[] = {8.5, 0.0}; + const float golden[] = {-8.5, 0.0}; float output_data[2]; - tflite::testing::TestNegFloat(/*input_dims_data=*/{1, 2}, - /*input_data=*/{8.5f, 0.0f}, - /*expected_output_data=*/{-8.5f, 0.0f}, - /*output_dims_data*/ {1, 2}, - /*output_data=*/output_data); + + tflite::testing::TestNegFloat(dims, input_data, golden, dims, output_data); } TF_LITE_MICRO_TEST(NegOpFloat) { + const int dims[] = {2, 2, 3}; + const float input_data[] = {-2.0f, -1.0f, 0.f, 1.0f, 2.0f, 3.0f}; + const float golden[] = {2.0f, 1.0f, -0.f, -1.0f, -2.0f, -3.0f}; float output_data[6]; - tflite::testing::TestNegFloat(/*input_dims_data=*/{2, 2, 3}, - /*input_data=*/ - {-2.0f, -1.0f, 0.f, 1.0f, 2.0f, 3.0f}, - /*expected_output_data=*/ - {2.0f, 1.0f, -0.f, -1.0f, -2.0f, -3.0f}, - /*output_dims_data=*/{2, 2, 3}, - /*output_data=*/output_data); + + tflite::testing::TestNegFloat(dims, input_data, golden, dims, output_data); } TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/kernels/pack_test.cc b/tensorflow/lite/micro/kernels/pack_test.cc index 45d5e32ef48..c05595df146 100644 --- a/tensorflow/lite/micro/kernels/pack_test.cc +++ b/tensorflow/lite/micro/kernels/pack_test.cc @@ -23,16 +23,39 @@ limitations under the License. namespace tflite { namespace testing { -void TestPackTwoInputsFloat(std::initializer_list input1_dims_data, - std::initializer_list input1_data, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, int axis, - std::initializer_list output_dims_data, - std::initializer_list expected_output_data, +template +void ValidatePackGoldens(TfLiteTensor* tensors, int tensors_size, + TfLitePackParams params, TfLiteIntArray* inputs_array, + TfLiteIntArray* outputs_array, const T* golden, + int output_len, float tolerance, T* output) { + // Place a unique value in the uninitialized output buffer. + for (int i = 0; i < output_len; ++i) { + output[i] = 23; + } + + const TfLiteRegistration registration = tflite::ops::micro::Register_PACK(); + micro::KernelRunner runner(registration, tensors, tensors_size, inputs_array, + outputs_array, reinterpret_cast(¶ms), + micro_test::reporter); + + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); + + for (int i = 0; i < output_len; ++i) { + TF_LITE_MICRO_EXPECT_NEAR(golden[i], output[i], tolerance); + } +} + +void TestPackTwoInputsFloat(const int* input1_dims_data, + const float* input1_data, + const int* input2_dims_data, + const float* input2_data, int axis, + const int* output_dims_data, + const float* expected_output_data, float* output_data) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int input_size = 2; @@ -43,11 +66,6 @@ void TestPackTwoInputsFloat(std::initializer_list input1_dims_data, CreateFloatTensor(input2_data, input2_dims), CreateFloatTensor(output_data, output_dims)}; - // Place a unique value in the uninitialized output buffer. - for (int i = 0; i < output_dims_count; ++i) { - output_data[i] = 23; - } - TfLitePackParams builtin_data = { .values_count = 2, .axis = axis, @@ -57,34 +75,21 @@ void TestPackTwoInputsFloat(std::initializer_list input1_dims_data, int outputs_array_data[] = {1, 2}; TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - const TfLiteRegistration registration = tflite::ops::micro::Register_PACK(); - micro::KernelRunner runner( - registration, tensors, tensors_size, inputs_array, outputs_array, - reinterpret_cast(&builtin_data), micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - 1e-5f); - } + ValidatePackGoldens(tensors, tensors_size, builtin_data, inputs_array, + outputs_array, expected_output_data, output_dims_count, + 1e-5f, output_data); } -void TestPackThreeInputsFloat(std::initializer_list input1_dims_data, - std::initializer_list input1_data, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, - std::initializer_list input3_dims_data, - std::initializer_list input3_data, - int axis, - std::initializer_list output_dims_data, - std::initializer_list expected_output_data, - float* output_data) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* input3_dims = IntArrayFromInitializer(input3_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); +void TestPackThreeInputsFloat( + const int* input1_dims_data, const float* input1_data, + const int* input2_dims_data, const float* input2_data, + const int* input3_dims_data, const float* input3_data, int axis, + const int* output_dims_data, const float* expected_output_data, + float* output_data) { + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* input3_dims = IntArrayFromInts(input3_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int input_size = 3; @@ -96,95 +101,67 @@ void TestPackThreeInputsFloat(std::initializer_list input1_dims_data, CreateFloatTensor(input3_data, input3_dims), CreateFloatTensor(output_data, output_dims)}; - // Place a unique value in the uninitialized output buffer. - for (int i = 0; i < output_dims_count; ++i) { - output_data[i] = 23; - } - TfLitePackParams builtin_data = { .values_count = 3, .axis = axis, }; - int inputs_array_data[] = {3, 0, 1, 2}; TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); int outputs_array_data[] = {1, 3}; TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - const TfLiteRegistration registration = tflite::ops::micro::Register_PACK(); - micro::KernelRunner runner( - registration, tensors, tensors_size, inputs_array, outputs_array, - reinterpret_cast(&builtin_data), micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - 1e-5f); - } + ValidatePackGoldens(tensors, tensors_size, builtin_data, inputs_array, + outputs_array, expected_output_data, output_dims_count, + 1e-5f, output_data); } -void TestPackTwoInputsQuantized( - std::initializer_list input1_dims_data, - std::initializer_list input1_data, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, int axis, - std::initializer_list output_dims_data, - std::initializer_list expected_output_data, uint8_t* output_data) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); +void TestPackTwoInputsQuantized(const int* input1_dims_data, + const uint8_t* input1_data, + const int* input2_dims_data, + const uint8_t* input2_data, int axis, + const int* output_dims_data, + const uint8_t* expected_output_data, + uint8_t* output_data) { + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int input_size = 2; constexpr int output_size = 1; constexpr int tensors_size = input_size + output_size; TfLiteTensor tensors[tensors_size] = { - // CreateQuantizedTensor needs min/max values as input, but these values - // don't matter as to the functionality of PACK, so just set as 0 and 10. - CreateQuantizedTensor(input1_data, input1_dims, 0, 10), - CreateQuantizedTensor(input2_data, input2_dims, 0, 10), - CreateQuantizedTensor(output_data, output_dims, 0, 10)}; - - // Place a unique value in the uninitialized output buffer. - for (int i = 0; i < output_dims_count; ++i) { - output_data[i] = 23; - } + // CreateQuantizedTensor needs scale/zero_point values as input, but these + // values don't matter as to the functionality of PACK, so just set as 1.0 + // and 128. + CreateQuantizedTensor(input1_data, input1_dims, 1.0, 128), + CreateQuantizedTensor(input2_data, input2_dims, 1.0, 128), + CreateQuantizedTensor(output_data, output_dims, 1.0, 128)}; TfLitePackParams builtin_data = { .values_count = 2, .axis = axis, }; - int inputs_array_data[] = {2, 0, 1}; TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); int outputs_array_data[] = {1, 2}; TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - const TfLiteRegistration registration = tflite::ops::micro::Register_PACK(); - micro::KernelRunner runner( - registration, tensors, tensors_size, inputs_array, outputs_array, - reinterpret_cast(&builtin_data), micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_EQ(expected_output_data.begin()[i], output_data[i]); - } + ValidatePackGoldens(tensors, tensors_size, builtin_data, inputs_array, + outputs_array, expected_output_data, output_dims_count, + 1e-5f, output_data); } -void TestPackTwoInputsQuantized32( - std::initializer_list input1_dims_data, - std::initializer_list input1_data, - std::initializer_list input2_dims_data, - std::initializer_list input2_data, int axis, - std::initializer_list output_dims_data, - std::initializer_list expected_output_data, int32_t* output_data) { - TfLiteIntArray* input1_dims = IntArrayFromInitializer(input1_dims_data); - TfLiteIntArray* input2_dims = IntArrayFromInitializer(input2_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); +void TestPackTwoInputsQuantized32(const int* input1_dims_data, + const int32_t* input1_data, + const int* input2_dims_data, + const int32_t* input2_data, int axis, + const int* output_dims_data, + const int32_t* expected_output_data, + int32_t* output_data) { + TfLiteIntArray* input1_dims = IntArrayFromInts(input1_dims_data); + TfLiteIntArray* input2_dims = IntArrayFromInts(input2_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int input_size = 2; @@ -195,32 +172,18 @@ void TestPackTwoInputsQuantized32( CreateQuantized32Tensor(input2_data, input2_dims, 1.0), CreateQuantized32Tensor(output_data, output_dims, 1.0)}; - // Place a unique value in the uninitialized output buffer. - for (int i = 0; i < output_dims_count; ++i) { - output_data[i] = 23; - } - TfLitePackParams builtin_data = { .values_count = 2, .axis = axis, }; - int inputs_array_data[] = {2, 0, 1}; TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); int outputs_array_data[] = {1, 2}; TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - const TfLiteRegistration registration = tflite::ops::micro::Register_PACK(); - micro::KernelRunner runner( - registration, tensors, tensors_size, inputs_array, outputs_array, - reinterpret_cast(&builtin_data), micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_EQ(expected_output_data.begin()[i], output_data[i]); - } + ValidatePackGoldens(tensors, tensors_size, builtin_data, inputs_array, + outputs_array, expected_output_data, output_dims_count, + 1e-5f, output_data); } } // namespace testing @@ -229,99 +192,96 @@ void TestPackTwoInputsQuantized32( TF_LITE_MICRO_TESTS_BEGIN TF_LITE_MICRO_TEST(PackFloatThreeInputs) { + const int input_shape[] = {1, 2}; + const int output_shape[] = {2, 3, 2}; + const float input1_values[] = {1, 4}; + const float input2_values[] = {2, 5}; + const float input3_values[] = {3, 6}; + const float golden[] = {1, 4, 2, 5, 3, 6}; + const int axis = 0; constexpr int output_dims_count = 6; float output_data[output_dims_count]; + tflite::testing::TestPackThreeInputsFloat( - {1, 2}, // Input1 shape - {1, 4}, // Input1 values - {1, 2}, // Input2 shape - {2, 5}, // Input2 values - {1, 2}, // Input3 shape - {3, 6}, // Input3 values - 0, {2, 3, 2}, // Output shape - { - 1, 4, 2, 5, 3, 6 // Output values - }, - output_data); + input_shape, input1_values, input_shape, input2_values, input_shape, + input3_values, axis, output_shape, golden, output_data); } TF_LITE_MICRO_TEST(PackFloatThreeInputsDifferentAxis) { + const int input_shape[] = {1, 2}; + const int output_shape[] = {2, 2, 3}; + const float input1_values[] = {1, 4}; + const float input2_values[] = {2, 5}; + const float input3_values[] = {3, 6}; + const float golden[] = {1, 2, 3, 4, 5, 6}; + const int axis = 1; constexpr int output_dims_count = 6; float output_data[output_dims_count]; + tflite::testing::TestPackThreeInputsFloat( - {1, 2}, // Input1 shape - {1, 4}, // Input1 values - {1, 2}, // Input2 shape - {2, 5}, // Input2 values - {1, 2}, // Input3 shape - {3, 6}, // Input3 values - 1, {2, 2, 3}, // Output shape - { - 1, 2, 3, 4, 5, 6 // Output values - }, - output_data); + input_shape, input1_values, input_shape, input2_values, input_shape, + input3_values, axis, output_shape, golden, output_data); } TF_LITE_MICRO_TEST(PackFloatThreeInputsNegativeAxis) { + const int input_shape[] = {1, 2}; + const int output_shape[] = {2, 2, 3}; + const float input1_values[] = {1, 4}; + const float input2_values[] = {2, 5}; + const float input3_values[] = {3, 6}; + const float golden[] = {1, 2, 3, 4, 5, 6}; + const int axis = -1; constexpr int output_dims_count = 6; float output_data[output_dims_count]; + tflite::testing::TestPackThreeInputsFloat( - {1, 2}, // Input1 shape - {1, 4}, // Input1 values - {1, 2}, // Input2 shape - {2, 5}, // Input2 values - {1, 2}, // Input3 shape - {3, 6}, // Input3 values - -1, {2, 2, 3}, // Output shape - { - 1, 2, 3, 4, 5, 6 // Output values - }, - output_data); + input_shape, input1_values, input_shape, input2_values, input_shape, + input3_values, axis, output_shape, golden, output_data); } TF_LITE_MICRO_TEST(PackFloatMultilDimensions) { + const int input_shape[] = {2, 2, 3}; + const int output_shape[] = {3, 2, 2, 3}; + const float input1_values[] = {1, 2, 3, 4, 5, 6}; + const float input2_values[] = {7, 8, 9, 10, 11, 12}; + const float golden[] = {1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12}; + const int axis = 1; constexpr int output_dims_count = 12; float output_data[output_dims_count]; - tflite::testing::TestPackTwoInputsFloat( - {2, 2, 3}, // Input1 shape - {1, 2, 3, 4, 5, 6}, // Input1 values - {2, 2, 3}, // Input2 shape - {7, 8, 9, 10, 11, 12}, // Input2 values - 1, {3, 2, 2, 3}, // Output shape - { - 1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12 // Output values - }, - output_data); + + tflite::testing::TestPackTwoInputsFloat(input_shape, input1_values, + input_shape, input2_values, axis, + output_shape, golden, output_data); } TF_LITE_MICRO_TEST(PackQuantizedMultilDimensions) { + const int input_shape[] = {2, 2, 3}; + const int output_shape[] = {3, 2, 2, 3}; + const uint8_t input1_values[] = {1, 2, 3, 4, 5, 6}; + const uint8_t input2_values[] = {7, 8, 9, 10, 11, 12}; + const uint8_t golden[] = {1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12}; + const int axis = 1; constexpr int output_dims_count = 12; uint8_t output_data[output_dims_count]; + tflite::testing::TestPackTwoInputsQuantized( - {2, 2, 3}, // Input1 shape - {1, 2, 3, 4, 5, 6}, // Input1 values - {2, 2, 3}, // Input2 shape - {7, 8, 9, 10, 11, 12}, // Input2 values - 1, {3, 2, 2, 3}, // Output shape - { - 1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12 // Output values - }, - output_data); + input_shape, input1_values, input_shape, input2_values, axis, + output_shape, golden, output_data); } TF_LITE_MICRO_TEST(PackQuantized32MultilDimensions) { + const int input_shape[] = {2, 2, 3}; + const int output_shape[] = {3, 2, 2, 3}; + const int32_t input1_values[] = {1, 2, 3, 4, 5, 6}; + const int32_t input2_values[] = {7, 8, 9, 10, 11, 12}; + const int32_t golden[] = {1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12}; + const int axis = 1; constexpr int output_dims_count = 12; int32_t output_data[output_dims_count]; + tflite::testing::TestPackTwoInputsQuantized32( - {2, 2, 3}, // Input1 shape - {1, 2, 3, 4, 5, 6}, // Input1 values - {2, 2, 3}, // Input2 shape - {7, 8, 9, 10, 11, 12}, // Input2 values - 1, {3, 2, 2, 3}, // Output shape - { - 1, 2, 3, 7, 8, 9, 4, 5, 6, 10, 11, 12 // Output values - }, - output_data); + input_shape, input1_values, input_shape, input2_values, axis, + output_shape, golden, output_data); } TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/kernels/pooling_test.cc b/tensorflow/lite/micro/kernels/pooling_test.cc index ec5eb47d0da..a33f5df6fd4 100644 --- a/tensorflow/lite/micro/kernels/pooling_test.cc +++ b/tensorflow/lite/micro/kernels/pooling_test.cc @@ -25,27 +25,14 @@ namespace tflite { namespace testing { namespace { -void TestAveragePoolingFloat(std::initializer_list input_dims_data, - std::initializer_list input_data, - const int filter_height, const int filter_width, - const int stride_height, const int stride_width, - std::initializer_list expected_output_data, - std::initializer_list output_dims_data, - TfLitePadding padding, - TfLiteFusedActivation activation, - float* output_data) { - TfLiteIntArray* input_dims = IntArrayFromInitializer(input_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); - const int output_dims_count = ElementCount(*output_dims); - - constexpr int inputs_size = 1; - constexpr int outputs_size = 1; - constexpr int tensors_size = inputs_size + outputs_size; - TfLiteTensor tensors[tensors_size] = { - CreateFloatTensor(input_data, input_dims), - CreateFloatTensor(output_data, output_dims), - }; - +template +void ValidatePoolingGoldens(TfLiteTensor* tensors, int tensors_size, + const TfLiteRegistration registration, + const int filter_height, const int filter_width, + const int stride_height, const int stride_width, + const T* golden, const int output_length, + TfLitePadding padding, + TfLiteFusedActivation activation, T* output_data) { int inputs_array_data[] = {1, 0}; TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); int outputs_array_data[] = {1, 1}; @@ -59,8 +46,6 @@ void TestAveragePoolingFloat(std::initializer_list input_dims_data, activation, {}}; - const TfLiteRegistration registration = - tflite::ops::micro::Register_AVERAGE_POOL_2D(); micro::KernelRunner runner( registration, tensors, tensors_size, inputs_array, outputs_array, reinterpret_cast(&builtin_data), micro_test::reporter); @@ -68,73 +53,78 @@ void TestAveragePoolingFloat(std::initializer_list input_dims_data, TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - 1e-5f); + for (int i = 0; i < output_length; ++i) { + TF_LITE_MICRO_EXPECT_NEAR(golden[i], output_data[i], 1e-5f); } } +void TestAveragePoolFloat(const int* input_dims_data, const float* input_data, + const int filter_height, const int filter_width, + const int stride_height, const int stride_width, + const float* expected_output_data, + const int* output_dims_data, TfLitePadding padding, + TfLiteFusedActivation activation, + float* output_data) { + TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); + const int output_dims_count = ElementCount(*output_dims); + + constexpr int inputs_size = 1; + constexpr int outputs_size = 1; + constexpr int tensors_size = inputs_size + outputs_size; + TfLiteTensor tensors[tensors_size] = { + CreateFloatTensor(input_data, input_dims), + CreateFloatTensor(output_data, output_dims), + }; + + const TfLiteRegistration registration = + tflite::ops::micro::Register_AVERAGE_POOL_2D(); + + ValidatePoolingGoldens(tensors, tensors_size, registration, filter_height, + filter_width, stride_height, stride_width, + expected_output_data, output_dims_count, padding, + activation, output_data); +} + template -void TestAveragePoolingQuantized( - std::initializer_list input_dims_data, - std::initializer_list input_data, const float input_min, - const float input_max, const int filter_height, const int filter_width, +void TestAveragePoolQuantized( + const int* input_dims_data, const T* input_data, const float input_scale, + const int input_zero_point, const int filter_height, const int filter_width, const int stride_height, const int stride_width, - std::initializer_list expected_output_data, - std::initializer_list output_dims_data, float output_min, - float output_max, TfLitePadding padding, TfLiteFusedActivation activation, - T* output_data) { + const T* expected_output_data, const int* output_dims_data, + const float output_scale, const int output_zero_point, + TfLitePadding padding, TfLiteFusedActivation activation, T* output_data) { static_assert(sizeof(T) == 1, "Only int8_t/uint8_t data types allowed."); - TfLiteIntArray* input_dims = IntArrayFromInitializer(input_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); + TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 1; constexpr int outputs_size = 1; constexpr int tensors_size = inputs_size + outputs_size; TfLiteTensor tensors[tensors_size] = { - CreateQuantizedTensor(input_data, input_dims, input_min, input_max), - CreateQuantizedTensor(output_data, output_dims, output_min, output_max), + CreateQuantizedTensor(input_data, input_dims, input_scale, + input_zero_point), + CreateQuantizedTensor(output_data, output_dims, output_scale, + output_zero_point), }; - int inputs_array_data[] = {1, 0}; - TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); - int outputs_array_data[] = {1, 1}; - TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - - TfLitePoolParams builtin_data = {padding, - stride_width, - stride_height, - filter_width, - filter_height, - activation, - {}}; - const TfLiteRegistration registration = tflite::ops::micro::Register_AVERAGE_POOL_2D(); - micro::KernelRunner runner( - registration, tensors, tensors_size, inputs_array, outputs_array, - reinterpret_cast(&builtin_data), micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - 1e-5f); - } + ValidatePoolingGoldens(tensors, tensors_size, registration, filter_height, + filter_width, stride_height, stride_width, + expected_output_data, output_dims_count, padding, + activation, output_data); } -void TestMaxPoolFloat(std::initializer_list input_dims_data, - std::initializer_list input_data, int filter_width, - int filter_height, int stride_width, int stride_height, - std::initializer_list expected_output_data, - std::initializer_list output_dims_data, - TfLitePadding padding, TfLiteFusedActivation activation, - float* output_data) { - TfLiteIntArray* input_dims = IntArrayFromInitializer(input_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); +void TestMaxPoolFloat(const int* input_dims_data, const float* input_data, + int filter_width, int filter_height, int stride_width, + int stride_height, const float* expected_output_data, + const int* output_dims_data, TfLitePadding padding, + TfLiteFusedActivation activation, float* output_data) { + TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 1; @@ -145,83 +135,43 @@ void TestMaxPoolFloat(std::initializer_list input_dims_data, CreateFloatTensor(output_data, output_dims), }; - int inputs_array_data[] = {1, 0}; - TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); - int outputs_array_data[] = {1, 1}; - TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - - TfLitePoolParams builtin_data = {padding, - stride_width, - stride_height, - filter_width, - filter_height, - activation, - {}}; - const TfLiteRegistration registration = tflite::ops::micro::Register_MAX_POOL_2D(); - micro::KernelRunner runner( - registration, tensors, tensors_size, inputs_array, outputs_array, - reinterpret_cast(&builtin_data), micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - 1e-5f); - } + ValidatePoolingGoldens(tensors, tensors_size, registration, filter_height, + filter_width, stride_height, stride_width, + expected_output_data, output_dims_count, padding, + activation, output_data); } template -void TestMaxPoolQuantized(std::initializer_list input_dims_data, - std::initializer_list input_data, float input_min, - float input_max, int filter_width, int filter_height, - int stride_width, int stride_height, - std::initializer_list expected_output_data, - float output_min, float output_max, - std::initializer_list output_dims_data, - TfLitePadding padding, +void TestMaxPoolQuantized(const int* input_dims_data, const T* input_data, + const float input_scale, const int input_zero_point, + const int filter_height, const int filter_width, + const int stride_height, const int stride_width, + const T* expected_output_data, + const int* output_dims_data, const float output_scale, + const int output_zero_point, TfLitePadding padding, TfLiteFusedActivation activation, T* output_data) { - static_assert(sizeof(T) == 1, "Only int8_t/uint8_t data types allowed."); - - TfLiteIntArray* input_dims = IntArrayFromInitializer(input_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); + TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 1; constexpr int outputs_size = 1; constexpr int tensors_size = inputs_size + outputs_size; TfLiteTensor tensors[tensors_size] = { - CreateQuantizedTensor(input_data, input_dims, input_min, input_max), - CreateQuantizedTensor(output_data, output_dims, output_min, output_max), + CreateQuantizedTensor(input_data, input_dims, input_scale, + input_zero_point), + CreateQuantizedTensor(output_data, output_dims, output_scale, + output_zero_point), }; - int inputs_array_data[] = {1, 0}; - TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); - int outputs_array_data[] = {1, 1}; - TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - - TfLitePoolParams builtin_data = {padding, - stride_width, - stride_height, - filter_width, - filter_height, - activation, - {}}; - const TfLiteRegistration registration = tflite::ops::micro::Register_MAX_POOL_2D(); - micro::KernelRunner runner( - registration, tensors, tensors_size, inputs_array, outputs_array, - reinterpret_cast(&builtin_data), micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_EQ(expected_output_data.begin()[i], output_data[i]); - } + ValidatePoolingGoldens(tensors, tensors_size, registration, filter_height, + filter_width, stride_height, stride_width, + expected_output_data, output_dims_count, padding, + activation, output_data); } } // namespace @@ -232,798 +182,535 @@ void TestMaxPoolQuantized(std::initializer_list input_dims_data, TF_LITE_MICRO_TESTS_BEGIN TF_LITE_MICRO_TEST(SimpleAveragePoolTestFloat) { + const int input_shape[] = {4, 1, 2, 4, 1}; + const float input_values[] = {0, 6, 2, 4, 3, 2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const float golden[] = {2.75, 5.75}; + const int output_shape[] = {4, 1, 1, 2, 1}; float output_data[2]; - tflite::testing::TestAveragePoolingFloat({4, 1, 2, 4, 1}, // Input shape - { // Input values - 0., 6., 2., 4., 3., 2., 10., 7.}, - 2, 2, // filter width, filter height - 2, 2, // stride width, stride height - { - // Output values - 2.75, - 5.75, - }, - {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActNone, - output_data); + tflite::testing::TestAveragePoolFloat( + input_shape, input_values, filter_height, filter_width, stride_height, + stride_width, golden, output_shape, kTfLitePaddingValid, kTfLiteActNone, + output_data); } TF_LITE_MICRO_TEST(SimpleAveragePoolTestUint8) { - using tflite::testing::F2Q; - - const float input_min = -15.9375; - const float input_max = 15.9375; - const float output_min = -15.9375; - const float output_max = 15.9375; + const int input_shape[] = {4, 1, 2, 4, 1}; + const uint8_t input_values[] = {0, 24, 8, 16, 12, 8, 40, 28}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const uint8_t golden[] = {11, 23}; + const int output_shape[] = {4, 1, 1, 2, 1}; uint8_t output_data[2]; - tflite::testing::TestAveragePoolingQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2Q(0., input_min, input_max), - F2Q(-6., input_min, input_max), - F2Q(2., input_min, input_max), - F2Q(4., input_min, input_max), - F2Q(3., input_min, input_max), - F2Q(2., input_min, input_max), - F2Q(-10., input_min, input_max), - F2Q(7., input_min, input_max), - }, - input_min, input_max, // input quantization range - 2, 2, // filter width, filter height - 2, 2, // stride width, stride height - { - // Output values - F2Q(0., output_min, output_max), - F2Q(0.75, output_min, output_max), - }, - {4, 1, 1, 2, 1}, // Output shape - output_min, output_max, // output quantization range - kTfLitePaddingValid, kTfLiteActRelu, output_data); + + const float input_scale = 0.25; + const int input_zero_point = 0; + const float output_scale = .25; + const int output_zero_point = 0; + tflite::testing::TestAveragePoolQuantized( + input_shape, input_values, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActNone, + output_data); } TF_LITE_MICRO_TEST(SimpleAveragePoolTestInt8PaddingValidStride2ActNone) { - using tflite::testing::F2QS; - - const float input_min = -15.9375; - const float input_max = 15.8130; - const float output_min = -15.9375; - const float output_max = 15.8130; + const int input_shape[] = {4, 1, 2, 4, 1}; + const int8_t input_values[] = {0, -24, 8, 16, 12, 8, -40, 28}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const int8_t golden[] = {-1, 3}; + const int output_shape[] = {4, 1, 1, 2, 1}; int8_t output_data[2]; - tflite::testing::TestAveragePoolingQuantized( - {4, 1, 2, 4, 1}, // Input shape - { // Input values - F2QS(0., input_min, input_max), F2QS(-6., input_min, input_max), - F2QS(2., input_min, input_max), F2QS(4., input_min, input_max), - F2QS(3., input_min, input_max), F2QS(2., input_min, input_max), - F2QS(-10., input_min, input_max), F2QS(7., input_min, input_max)}, - input_min, input_max, // input quantization range - 2, 2, // filter height, filter width - 2, 2, // stride height, stride width - { // Output values - F2QS(-0.25, output_min, output_max), F2QS(0.75, output_min, output_max)}, - {4, 1, 1, 2, 1}, // Output shape - output_min, output_max, // output quantization range - kTfLitePaddingValid, kTfLiteActNone, output_data); + + const float input_scale = .25; + const int input_zero_point = 0; + const float output_scale = .25; + const int output_zero_point = 0; + tflite::testing::TestAveragePoolQuantized( + input_shape, input_values, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActNone, + output_data); } TF_LITE_MICRO_TEST(SimpleAveragePoolTestInt8PaddingValidStride1Stride2Relu) { - using tflite::testing::F2QS; - - const float input_min = -15.9375; - const float input_max = 15.8130; - const float output_min = -15.9375; - const float output_max = 15.8130; + const int input_shape[] = {4, 1, 2, 4, 1}; + const int8_t input_values[] = {0, -24, 8, 16, 12, 8, -40, 28}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 1; + const int stride_height = 2; + const int8_t golden[] = {0, 0, 3}; + const int output_shape[] = {4, 1, 1, 3, 1}; int8_t output_data[3]; - tflite::testing::TestAveragePoolingQuantized( - {4, 1, 2, 4, 1}, // Input shape - { // Input values - F2QS(0., input_min, input_max), F2QS(-6., input_min, input_max), - F2QS(2., input_min, input_max), F2QS(4., input_min, input_max), - F2QS(3., input_min, input_max), F2QS(2., input_min, input_max), - F2QS(-10., input_min, input_max), F2QS(7., input_min, input_max)}, - input_min, input_max, // input quantization range - 2, 2, // filter height, filter width - 2, 1, // stride height, stride width - { // Output values - F2QS(0., output_min, output_max), F2QS(0., output_min, output_max), - F2QS(0.75, output_min, output_max)}, - {4, 1, 1, 3, 1}, // Output shape - output_min, output_max, // output quantization range - kTfLitePaddingValid, kTfLiteActRelu, output_data); + + const float input_scale = .25; + const int input_zero_point = 0; + const float output_scale = .25; + const int output_zero_point = 0; + tflite::testing::TestAveragePoolQuantized( + input_shape, input_values, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActRelu, + output_data); } TF_LITE_MICRO_TEST( SimpleAveragePoolTestInt8PaddingValidStride2Stride1ReluN1To1) { - using tflite::testing::F2QS; - - const float input_min = -15.9375; - const float input_max = 15.8130; - const float output_min = -15.9375; - const float output_max = 15.8130; + const int input_shape[] = {4, 1, 2, 4, 1}; + const int8_t input_values[] = {0, -24, 8, 16, 12, 8, -40, 28}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 1; + const int8_t golden[] = {-1, 3}; + const int output_shape[] = {4, 1, 1, 2, 1}; int8_t output_data[2]; - tflite::testing::TestAveragePoolingQuantized( - {4, 1, 2, 4, 1}, // Input shape - { // Input values - F2QS(0., input_min, input_max), F2QS(-6., input_min, input_max), - F2QS(2., input_min, input_max), F2QS(4., input_min, input_max), - F2QS(3., input_min, input_max), F2QS(2., input_min, input_max), - F2QS(-10., input_min, input_max), F2QS(7., input_min, input_max)}, - input_min, input_max, // input quantization range - 2, 2, // filter height, filter width - 1, 2, // stride height, stride width - { // Output values - F2QS(-0.25, output_min, output_max), F2QS(0.75, output_min, output_max)}, - {4, 1, 1, 2, 1}, // Output shape - output_min, output_max, // output quantization range - kTfLitePaddingValid, kTfLiteActReluN1To1, output_data); + + const float input_scale = .25; + const int input_zero_point = 0; + const float output_scale = .25; + const int output_zero_point = 0; + tflite::testing::TestAveragePoolQuantized( + input_shape, input_values, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActReluN1To1, + output_data); } TF_LITE_MICRO_TEST(SimpleAveragePoolTestInt8PaddingValidStride2Relu6) { - using tflite::testing::F2QS; - - const float input_min = -15.9375; - const float input_max = 15.8130; - const float output_min = -15.9375; - const float output_max = 15.8130; + const int input_shape[] = {4, 1, 2, 4, 1}; + const int8_t input_values[] = {12, -24, 32, 16, 12, 8, 40, 28}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const int8_t golden[] = {2, 24}; + const int output_shape[] = {4, 1, 1, 2, 1}; int8_t output_data[2]; - tflite::testing::TestAveragePoolingQuantized( - {4, 1, 2, 4, 1}, // Input shape - { // Input values - F2QS(3., input_min, input_max), F2QS(-6., input_min, input_max), - F2QS(8., input_min, input_max), F2QS(4., input_min, input_max), - F2QS(3., input_min, input_max), F2QS(2., input_min, input_max), - F2QS(10., input_min, input_max), F2QS(7., input_min, input_max)}, - input_min, input_max, // input quantization range - 2, 2, // filter height, filter width - 2, 2, // stride height, stride width - { // Output values - F2QS(0.5, output_min, output_max), F2QS(6., output_min, output_max)}, - {4, 1, 1, 2, 1}, // Output shape - output_min, output_max, // output quantization range - kTfLitePaddingValid, kTfLiteActRelu6, output_data); + + const float input_scale = .25; + const int input_zero_point = 0; + const float output_scale = .25; + const int output_zero_point = 0; + tflite::testing::TestAveragePoolQuantized( + input_shape, input_values, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActRelu6, + output_data); } TF_LITE_MICRO_TEST(SimpleAveragePoolTestInt8PaddingSameStride1ActNone) { - using tflite::testing::F2QS; - - const float input_min = -15.9375; - const float input_max = 15.8130; - const float output_min = -15.9375; - const float output_max = 15.8130; + const int input_shape[] = {4, 1, 2, 4, 1}; + const int8_t input_values[] = {12, -24, 32, 16, 12, 8, 40, 28}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 1; + const int stride_height = 1; + const int8_t golden[] = {2, 14, 29, 22, 10, 24, 34, 28}; + const int output_shape[] = {4, 1, 2, 4, 1}; int8_t output_data[8]; - tflite::testing::TestAveragePoolingQuantized( - {4, 1, 2, 4, 1}, // Input shape - { // Input values - F2QS(3., input_min, input_max), F2QS(-6., input_min, input_max), - F2QS(8., input_min, input_max), F2QS(4., input_min, input_max), - F2QS(3., input_min, input_max), F2QS(2., input_min, input_max), - F2QS(10., input_min, input_max), F2QS(7., input_min, input_max)}, - input_min, input_max, // input quantization range - 2, 2, // filter height, filter width - 1, 1, // stride height, stride width - { // Output values - F2QS(0.5, output_min, output_max), F2QS(3.5, output_min, output_max), - F2QS(7.25, output_min, output_max), F2QS(5.5, output_min, output_max), - F2QS(2.5, output_min, output_max), F2QS(6., output_min, output_max), - F2QS(8.5, output_min, output_max), F2QS(7., output_min, output_max)}, - {4, 1, 2, 4, 1}, // Output shape - output_min, output_max, // output quantization range - kTfLitePaddingSame, kTfLiteActNone, output_data); + + const float input_scale = .25; + const int input_zero_point = 0; + const float output_scale = .25; + const int output_zero_point = 0; + tflite::testing::TestAveragePoolQuantized( + input_shape, input_values, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActNone, + output_data); } TF_LITE_MICRO_TEST(SimpleMaxPoolTestFloat) { + const int input_shape[] = {4, 1, 2, 4, 1}; + const float input_values[] = {0, 6, 2, 4, 3, 2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const float golden[] = {6, 10}; + const int output_shape[] = {4, 1, 1, 2, 1}; float output_data[2]; - tflite::testing::TestMaxPoolFloat({4, 1, 2, 4, 1}, // Input shape - { // Input values - 0, 6, 2, 4, 3, 2, 10, 7}, - 2, 2, // filter width, filter height - 2, 2, // stride width, stride height - { - // Output values - 6, - 10, - }, - {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActNone, - output_data); + tflite::testing::TestMaxPoolFloat(input_shape, input_values, filter_height, + filter_width, stride_height, stride_width, + golden, output_shape, kTfLitePaddingValid, + kTfLiteActNone, output_data); } TF_LITE_MICRO_TEST(SimpleMaxPoolTestFloatRelu) { + const int input_shape[] = {4, 1, 2, 4, 1}; + const float input_values[] = {-1, -6, 2, 4, -3, -2, 10.5, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const float golden[] = {0, 10.5}; + const int output_shape[] = {4, 1, 1, 2, 1}; float output_data[2]; - tflite::testing::TestMaxPoolFloat({4, 1, 2, 4, 1}, // Input shape - { - // Input values - -1, -6, 2, 4, // - -3, -2, 10.5, 7, // - }, - 2, 2, // filter width, filter height - 2, 2, // stride width, stride height - { - // Output values - 0.0, - 10.5, - }, - {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActRelu, - output_data); + tflite::testing::TestMaxPoolFloat(input_shape, input_values, filter_height, + filter_width, stride_height, stride_width, + golden, output_shape, kTfLitePaddingValid, + kTfLiteActRelu, output_data); } TF_LITE_MICRO_TEST(SimpleMaxPoolTestFloatReluN1To1) { + const int input_shape[] = {4, 1, 2, 4, 1}; + const float input_values1[] = {-2.75, -6, 0.2, 0.4, -3, -2, -0.3, 0.7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const float golden1[] = {-1.0, 0.7}; + const int output_shape[] = {4, 1, 1, 2, 1}; float output_data[2]; - tflite::testing::TestMaxPoolFloat({4, 1, 2, 4, 1}, // Input shape - { - // Input values - -2.75, -6, 0.2, 0.4, // - -3, -2, -0.3, 0.7, // - }, - 2, 2, // filter width, filter height - 2, 2, // stride width, stride height - { - // Output values - -1.0, - 0.7, - }, - {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActReluN1To1, - output_data); + tflite::testing::TestMaxPoolFloat(input_shape, input_values1, filter_height, + filter_width, stride_height, stride_width, + golden1, output_shape, kTfLitePaddingValid, + kTfLiteActReluN1To1, output_data); - tflite::testing::TestMaxPoolFloat({4, 1, 2, 4, 1}, // Input shape - { - // Input values - -2.75, -6, -2, -4, // - -3, -2, 10, -7, // - }, - 2, 2, // filter width, filter height - 2, 2, // stride width, stride height - { - // Output values - -1.0, - 1.0, - }, - {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActReluN1To1, - output_data); + const float input_values2[] = {-2.75, -6, -2, -4, -3, -2, 10, -7}; + const float golden2[] = {-1.0, 1.0}; + tflite::testing::TestMaxPoolFloat(input_shape, input_values2, filter_height, + filter_width, stride_height, stride_width, + golden2, output_shape, kTfLitePaddingValid, + kTfLiteActReluN1To1, output_data); } TF_LITE_MICRO_TEST(SimpleMaxPoolTestFloatRelu6) { + const int input_shape[] = {4, 1, 2, 4, 1}; + const float input_values1[] = {-1.5, -6, 12, 4, -3, -2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const float golden1[] = {0, 6}; + const int output_shape[] = {4, 1, 1, 2, 1}; float output_data[2]; - tflite::testing::TestMaxPoolFloat({4, 1, 2, 4, 1}, // Input shape - { - // Input values - -1.5, -6, 12, 4, // - -3, -2, 10, 7, // - }, - 2, 2, // filter width, filter height - 2, 2, // stride width, stride height - { - // Output values - 0.0, - 6.0, - }, - {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActRelu6, - output_data); + tflite::testing::TestMaxPoolFloat(input_shape, input_values1, filter_height, + filter_width, stride_height, stride_width, + golden1, output_shape, kTfLitePaddingValid, + kTfLiteActRelu6, output_data); - tflite::testing::TestMaxPoolFloat({4, 1, 2, 4, 1}, // Input shape - { - // Input values - 0, 4.5, 12, 4, // - 3, 2, 10, 7, // - }, - 2, 2, // filter width, filter height - 2, 2, // stride width, stride height - { - // Output values - 4.5, - 6.0, - }, - {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActRelu6, - output_data); + const float input_values2[] = {0, 4.5, 12, 4, 3, 2, 10, 7}; + const float golden2[] = {4.5, 6}; + tflite::testing::TestMaxPoolFloat(input_shape, input_values2, filter_height, + filter_width, stride_height, stride_width, + golden2, output_shape, kTfLitePaddingValid, + kTfLiteActRelu6, output_data); } TF_LITE_MICRO_TEST(SimpleMaxPoolTestPaddingSameStride1) { + const int input_shape[] = {4, 1, 2, 4, 1}; + const float input_values[] = {0, 6, 2, 4, 3, 2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 1; + const int stride_height = 1; + const float golden[] = {6, 10, 10, 7, 3, 10, 10, 7}; + const int output_shape[] = {4, 1, 2, 4, 1}; float output_data[8]; - tflite::testing::TestMaxPoolFloat({4, 1, 2, 4, 1}, // Input shape - { - // Input values - 0, 6, 2, 4, // - 3, 2, 10, 7, // - }, - 2, 2, // filter width, filter height - 1, 1, // stride width, stride height - { - // Output values - 6, 10, 10, 7, // - 3, 10, 10, 7, // - }, - {4, 1, 2, 4, 1}, // Output shape - kTfLitePaddingSame, kTfLiteActNone, - output_data); + tflite::testing::TestMaxPoolFloat(input_shape, input_values, filter_height, + filter_width, stride_height, stride_width, + golden, output_shape, kTfLitePaddingSame, + kTfLiteActNone, output_data); } TF_LITE_MICRO_TEST(SimpleMaxPoolTestPaddingValidStride1) { - float output_data[3]; - tflite::testing::TestMaxPoolFloat({4, 1, 2, 4, 1}, // Input shape - { - // Input values - 0, 6, 2, 4, // - 3, 2, 10, 7, // - }, - 2, 2, // filter width, filter height - 1, 1, // stride width, stride height - { - // Output values - 6, - 10, - 10, - }, - {4, 1, 1, 3, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActNone, - output_data); + const int input_shape[] = {4, 1, 2, 4, 1}; + const float input_values[] = {0, 6, 2, 4, 3, 2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 1; + const int stride_height = 1; + const float golden[] = {6, 10, 10}; + const int output_shape[] = {4, 1, 1, 3, 1}; + float output_data[8]; + tflite::testing::TestMaxPoolFloat(input_shape, input_values, filter_height, + filter_width, stride_height, stride_width, + golden, output_shape, kTfLitePaddingValid, + kTfLiteActNone, output_data); } TF_LITE_MICRO_TEST(SimpleMaxPoolTestUInt8ActNone) { - using tflite::testing::F2Q; - + const int input_shape[] = {4, 1, 2, 4, 1}; + const uint8_t input_values[] = {0, 12, 4, 8, 6, 4, 20, 14}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const uint8_t golden[] = {12, 20}; + const int output_shape[] = {4, 1, 1, 2, 1}; uint8_t output_data[2]; - float input_min = 0; - float input_max = 15.9375; - float output_min = 0; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 2; - int stride_height = 2; + + const float input_scale = 1.0; + const int input_zero_point = 0; + const float output_scale = 1.0; + const int output_zero_point = 0; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2Q(0, input_min, input_max), - F2Q(6, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(4, input_min, input_max), - F2Q(3, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(10, input_min, input_max), - F2Q(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2Q(6, output_min, output_max), F2Q(10, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActNone, output_data); + input_shape, input_values, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActNone, + output_data); } TF_LITE_MICRO_TEST(MaxPoolTestUInt8ActRelu) { - using tflite::testing::F2Q; - + const int input_shape[] = {4, 1, 2, 4, 1}; + const uint8_t input_values[] = {0, 4, 2, 4, 3, 2, 14, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const uint8_t golden[] = {4, 14}; + const int output_shape[] = {4, 1, 1, 2, 1}; uint8_t output_data[2]; - float input_min = -15.9375; - float input_max = 15.9375; - float output_min = -15.9375; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 2; - int stride_height = 2; + + const float input_scale = 1.0; + const int input_zero_point = 4; + const float output_scale = 1.0; + const int output_zero_point = 4; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2Q(-1.5, input_min, input_max), - F2Q(-6, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(4, input_min, input_max), - F2Q(-3, input_min, input_max), - F2Q(-2, input_min, input_max), - F2Q(10, input_min, input_max), - F2Q(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2Q(0, output_min, output_max), F2Q(10, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActRelu, output_data); + input_shape, input_values, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActRelu, + output_data); } TF_LITE_MICRO_TEST(MaxPoolTestUInt8ActReluN1To1) { - using tflite::testing::F2Q; - + const int input_shape[] = {4, 1, 2, 4, 1}; + const uint8_t input_values[] = {0, 4, 2, 4, 3, 2, 14, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const uint8_t golden[] = {3, 5}; + const int output_shape[] = {4, 1, 1, 2, 1}; uint8_t output_data[2]; - float input_min = -15.9375; - float input_max = 15.9375; - float output_min = -15.9375; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 2; - int stride_height = 2; - tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2Q(-1.7, input_min, input_max), - F2Q(-6, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(4, input_min, input_max), - F2Q(-3, input_min, input_max), - F2Q(-2, input_min, input_max), - F2Q(-10, input_min, input_max), - F2Q(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2Q(-1.0, output_min, output_max), F2Q(1.0, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActReluN1To1, output_data); + + const float input_scale = 1.0; + const int input_zero_point = 4; + const float output_scale = 1.0; + const int output_zero_point = 4; + tflite::testing::TestAveragePoolQuantized( + input_shape, input_values, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActReluN1To1, + output_data); } TF_LITE_MICRO_TEST(MaxPoolTestUInt8ActRelu6) { - using tflite::testing::F2Q; - + const int input_shape[] = {4, 1, 2, 4, 1}; + const uint8_t input_values1[] = {12, 0, 36, 20, 6, 8, 32, 26}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const uint8_t golden1[] = {12, 24}; + const int output_shape[] = {4, 1, 1, 2, 1}; uint8_t output_data[8]; - float input_min = -15.9375; - float input_max = 15.9375; - float output_min = -15.9375; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 2; - int stride_height = 2; - tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2Q(0, input_min, input_max), - F2Q(-6, input_min, input_max), - F2Q(12, input_min, input_max), - F2Q(4, input_min, input_max), - F2Q(-3, input_min, input_max), - F2Q(-2, input_min, input_max), - F2Q(10, input_min, input_max), - F2Q(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2Q(0.0, output_min, output_max), F2Q(6.0, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActRelu6, output_data); + const float input_scale = 0.5; + const int input_zero_point = 12; + const float output_scale = 0.5; + const int output_zero_point = 12; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2Q(0, input_min, input_max), - F2Q(4.5, input_min, input_max), - F2Q(12, input_min, input_max), - F2Q(4, input_min, input_max), - F2Q(3, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(10, input_min, input_max), - F2Q(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2Q(4.5, output_min, output_max), F2Q(6.0, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActRelu6, output_data); + input_shape, input_values1, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden1, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActRelu6, + output_data); + + const uint8_t input_values2[] = {12, 21, 36, 16, 18, 16, 32, 26}; + + const uint8_t golden2[] = {21, 24}; + tflite::testing::TestMaxPoolQuantized( + input_shape, input_values2, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden2, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActRelu6, + output_data); } TF_LITE_MICRO_TEST(MaxPoolTestUInt8PaddingSameStride1) { - using tflite::testing::F2Q; - + const int input_shape[] = {4, 1, 2, 4, 1}; + const uint8_t input_values1[] = {0, 6, 2, 4, 3, 2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 1; + const int stride_height = 1; + const uint8_t golden1[] = {6, 10, 10, 7, 3, 10, 10, 7}; + const int output_shape[] = {4, 1, 2, 4, 1}; uint8_t output_data[8]; - float input_min = 0; - float input_max = 15.9375; - float output_min = 0; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 1; - int stride_height = 1; + + const float input_scale = 1.0; + const int input_zero_point = 0; + const float output_scale = 1.0; + const int output_zero_point = 0; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2Q(0, input_min, input_max), - F2Q(6, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(4, input_min, input_max), - F2Q(3, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(10, input_min, input_max), - F2Q(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - { - // Output values - F2Q(6, output_min, output_max), - F2Q(10, output_min, output_max), - F2Q(10, output_min, output_max), - F2Q(7, output_min, output_max), - F2Q(3, output_min, output_max), - F2Q(10, output_min, output_max), - F2Q(10, output_min, output_max), - F2Q(7, output_min, output_max), - }, - output_min, output_max, {4, 1, 2, 4, 1}, // Output shape - kTfLitePaddingSame, kTfLiteActNone, output_data); + input_shape, input_values1, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden1, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActNone, + output_data); } TF_LITE_MICRO_TEST(MaxPoolTestUInt8PaddingValidStride1) { - using tflite::testing::F2Q; - + const int input_shape[] = {4, 1, 2, 4, 1}; + const uint8_t input_values1[] = {0, 6, 2, 4, 3, 2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 1; + const int stride_height = 1; + const uint8_t golden1[] = {6, 10, 10}; + const int output_shape[] = {4, 1, 1, 3, 1}; uint8_t output_data[3]; - float input_min = 0; - float input_max = 15.9375; - float output_min = 0; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 1; - int stride_height = 1; + + const float input_scale = 1.0; + const int input_zero_point = 0; + const float output_scale = 1.0; + const int output_zero_point = 0; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2Q(0, input_min, input_max), - F2Q(6, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(4, input_min, input_max), - F2Q(3, input_min, input_max), - F2Q(2, input_min, input_max), - F2Q(10, input_min, input_max), - F2Q(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - { - // Output values - F2Q(6, output_min, output_max), - F2Q(10, output_min, output_max), - F2Q(10, output_min, output_max), - }, - output_min, output_max, {4, 1, 1, 3, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActNone, output_data); + input_shape, input_values1, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden1, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActNone, + output_data); } TF_LITE_MICRO_TEST(SimpleMaxPoolTestInt8ActNone) { - using tflite::testing::F2QS; - + const int input_shape[] = {4, 1, 2, 4, 1}; + const int8_t input_values1[] = {0, 6, 2, 4, 3, 2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const int8_t golden1[] = {6, 10}; + const int output_shape[] = {4, 1, 1, 2, 1}; int8_t output_data[2]; - float input_min = 0; - float input_max = 15.9375; - float output_min = 0; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 2; - int stride_height = 2; + + const float input_scale = 1.0; + const int input_zero_point = 0; + const float output_scale = 1.0; + const int output_zero_point = 0; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2QS(0, input_min, input_max), - F2QS(6, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(4, input_min, input_max), - F2QS(3, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(10, input_min, input_max), - F2QS(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2QS(6, output_min, output_max), F2QS(10, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActNone, output_data); + input_shape, input_values1, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden1, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActNone, + output_data); } -TF_LITE_MICRO_TEST(MaxPoolTestUInt8ActRelu) { - using tflite::testing::F2QS; - +TF_LITE_MICRO_TEST(MaxPoolTestInt8ActRelu) { + const int input_shape[] = {4, 1, 2, 4, 1}; + const int8_t input_values1[] = {-3, -12, 4, 8, -6, -4, 20, 14}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const int8_t golden1[] = {0, 20}; + const int output_shape[] = {4, 1, 1, 2, 1}; int8_t output_data[2]; - float input_min = -15.9375; - float input_max = 15.9375; - float output_min = -15.9375; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 2; - int stride_height = 2; + + const float input_scale = 0.5; + const int input_zero_point = 0; + const float output_scale = 0.5; + const int output_zero_point = 0; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2QS(-1.5, input_min, input_max), - F2QS(-6, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(4, input_min, input_max), - F2QS(-3, input_min, input_max), - F2QS(-2, input_min, input_max), - F2QS(10, input_min, input_max), - F2QS(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2QS(0, output_min, output_max), F2QS(10, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActRelu, output_data); + input_shape, input_values1, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden1, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActRelu, + output_data); } -TF_LITE_MICRO_TEST(MaxPoolTestUInt8ActReluN1To1) { - using tflite::testing::F2QS; - +TF_LITE_MICRO_TEST(MaxPoolTestInt8ActReluN1To1) { + const int input_shape[] = {4, 1, 2, 4, 1}; + const int8_t input_values1[] = {-2, -6, -2, -4, -3, -2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const int8_t golden1[] = {-1, 1}; + const int output_shape[] = {4, 1, 1, 2, 1}; int8_t output_data[2]; - float input_min = -15.9375; - float input_max = 15.9375; - float output_min = -15.9375; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 2; - int stride_height = 2; + + const float input_scale = 1.0; + const int input_zero_point = 0; + const float output_scale = 1.0; + const int output_zero_point = 0; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2QS(-1.7, input_min, input_max), - F2QS(-6, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(4, input_min, input_max), - F2QS(-3, input_min, input_max), - F2QS(-2, input_min, input_max), - F2QS(-10, input_min, input_max), - F2QS(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2QS(-1.0, output_min, output_max), F2QS(1.0, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActReluN1To1, output_data); + input_shape, input_values1, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden1, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActReluN1To1, + output_data); } -TF_LITE_MICRO_TEST(MaxPoolTestUInt8ActRelu6) { - using tflite::testing::F2QS; +TF_LITE_MICRO_TEST(MaxPoolTestInt8ActRelu6) { + const int input_shape[] = {4, 1, 2, 4, 1}; + const int8_t input_values1[] = {0, -6, 12, 4, -3, -2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 2; + const int stride_height = 2; + const int8_t golden1[] = {0, 6}; + const int output_shape[] = {4, 1, 1, 2, 1}; + int8_t output_data[2]; - int8_t output_data[8]; - float input_min = -15.9375; - float input_max = 15.9375; - float output_min = -15.9375; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 2; - int stride_height = 2; + const float input_scale = 1.0; + const int input_zero_point = 0; + const float output_scale = 1.0; + const int output_zero_point = 0; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2QS(0, input_min, input_max), - F2QS(-6, input_min, input_max), - F2QS(12, input_min, input_max), - F2QS(4, input_min, input_max), - F2QS(-3, input_min, input_max), - F2QS(-2, input_min, input_max), - F2QS(10, input_min, input_max), - F2QS(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2QS(0.0, output_min, output_max), F2QS(6.0, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActRelu6, output_data); - - tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2QS(0, input_min, input_max), - F2QS(4.5, input_min, input_max), - F2QS(12, input_min, input_max), - F2QS(4, input_min, input_max), - F2QS(3, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(10, input_min, input_max), - F2QS(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - {// Output values - F2QS(4.5, output_min, output_max), F2QS(6.0, output_min, output_max)}, - output_min, output_max, {4, 1, 1, 2, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActRelu6, output_data); + input_shape, input_values1, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden1, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActRelu6, + output_data); } TF_LITE_MICRO_TEST(MaxPoolTestUInt8PaddingSameStride1) { - using tflite::testing::F2QS; + const int input_shape[] = {4, 1, 2, 4, 1}; + const uint8_t input_values1[] = {0, 6, 2, 4, 3, 2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 1; + const int stride_height = 1; + const uint8_t golden1[] = {6, 10, 10, 7, 3, 10, 10, 7}; + const int output_shape[] = {4, 1, 2, 4, 1}; + uint8_t output_data[8]; - int8_t output_data[8]; - float input_min = 0; - float input_max = 15.9375; - float output_min = 0; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 1; - int stride_height = 1; + const float input_scale = 1.0; + const int input_zero_point = 0; + const float output_scale = 1.0; + const int output_zero_point = 0; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2QS(0, input_min, input_max), - F2QS(6, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(4, input_min, input_max), - F2QS(3, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(10, input_min, input_max), - F2QS(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - { - // Output values - F2QS(6, output_min, output_max), - F2QS(10, output_min, output_max), - F2QS(10, output_min, output_max), - F2QS(7, output_min, output_max), - F2QS(3, output_min, output_max), - F2QS(10, output_min, output_max), - F2QS(10, output_min, output_max), - F2QS(7, output_min, output_max), - }, - output_min, output_max, {4, 1, 2, 4, 1}, // Output shape - kTfLitePaddingSame, kTfLiteActNone, output_data); + input_shape, input_values1, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden1, output_shape, + output_scale, output_zero_point, kTfLitePaddingSame, kTfLiteActNone, + output_data); } TF_LITE_MICRO_TEST(MaxPoolTestUInt8PaddingValidStride1) { - using tflite::testing::F2QS; + const int input_shape[] = {4, 1, 2, 4, 1}; + const uint8_t input_values1[] = {0, 6, 2, 4, 3, 2, 10, 7}; + const int filter_width = 2; + const int filter_height = 2; + const int stride_width = 1; + const int stride_height = 1; + const uint8_t golden1[] = {6, 10, 10}; + const int output_shape[] = {4, 1, 1, 3, 1}; + uint8_t output_data[3]; - int8_t output_data[3]; - float input_min = 0; - float input_max = 15.9375; - float output_min = 0; - float output_max = 15.9375; - int filter_width = 2; - int filter_height = 2; - int stride_width = 1; - int stride_height = 1; + const float input_scale = 1.0; + const int input_zero_point = 0; + const float output_scale = 1.0; + const int output_zero_point = 0; tflite::testing::TestMaxPoolQuantized( - {4, 1, 2, 4, 1}, // Input shape - { - // Input values - F2QS(0, input_min, input_max), - F2QS(6, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(4, input_min, input_max), - F2QS(3, input_min, input_max), - F2QS(2, input_min, input_max), - F2QS(10, input_min, input_max), - F2QS(7, input_min, input_max), - }, - input_min, input_max, filter_width, filter_height, stride_width, - stride_height, - { - // Output values - F2QS(6, output_min, output_max), - F2QS(10, output_min, output_max), - F2QS(10, output_min, output_max), - }, - output_min, output_max, {4, 1, 1, 3, 1}, // Output shape - kTfLitePaddingValid, kTfLiteActNone, output_data); + input_shape, input_values1, input_scale, input_zero_point, filter_height, + filter_width, stride_height, stride_width, golden1, output_shape, + output_scale, output_zero_point, kTfLitePaddingValid, kTfLiteActNone, + output_data); } TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/kernels/prelu_test.cc b/tensorflow/lite/micro/kernels/prelu_test.cc index 4e352763cec..f559ddff993 100644 --- a/tensorflow/lite/micro/kernels/prelu_test.cc +++ b/tensorflow/lite/micro/kernels/prelu_test.cc @@ -23,16 +23,35 @@ namespace tflite { namespace testing { namespace { -void TestPreluFloat(std::initializer_list input_dims_data, - std::initializer_list input_data, - std::initializer_list alpha_dims_data, - std::initializer_list alpha_data, - std::initializer_list expected_output_data, - std::initializer_list output_dims_data, - float* output_data) { - TfLiteIntArray* input_dims = IntArrayFromInitializer(input_dims_data); - TfLiteIntArray* alpha_dims = IntArrayFromInitializer(alpha_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); +template +void ValidatePreluGoldens(TfLiteTensor* tensors, int tensors_size, + const T* golden, const int output_length, + T* output_data) { + int inputs_array_data[] = {2, 0, 1}; + TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); + int outputs_array_data[] = {1, 2}; + TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); + + const TfLiteRegistration registration = tflite::ops::micro::Register_PRELU(); + micro::KernelRunner runner(registration, tensors, tensors_size, inputs_array, + outputs_array, + /*builtin_data=*/nullptr, micro_test::reporter); + + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); + TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); + + for (int i = 0; i < output_length; ++i) { + TF_LITE_MICRO_EXPECT_NEAR(golden[i], output_data[i], 1e-5f); + } +} + +void TestPreluFloat(const int* input_dims_data, const float* input_data, + const int* alpha_dims_data, const float* alpha_data, + const float* expected_output_data, + const int* output_dims_data, float* output_data) { + TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data); + TfLiteIntArray* alpha_dims = IntArrayFromInts(alpha_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 2; constexpr int outputs_size = 1; @@ -43,66 +62,42 @@ void TestPreluFloat(std::initializer_list input_dims_data, CreateFloatTensor(output_data, output_dims), }; - int inputs_array_data[] = {2, 0, 1}; - TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); - int outputs_array_data[] = {1, 2}; - TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); - - const TfLiteRegistration registration = tflite::ops::micro::Register_PRELU(); - micro::KernelRunner runner(registration, tensors, tensors_size, inputs_array, - outputs_array, - /*builtin_data=*/nullptr, micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_NEAR(expected_output_data.begin()[i], output_data[i], - 1e-5f); - } + ValidatePreluGoldens(tensors, tensors_size, expected_output_data, + output_dims_count, output_data); } // Template argument T can be either uint8_t or int8_t depending on which type // of quantization required to be tested. template -void TestPreluQuantized(std::initializer_list input_dims_data, - std::initializer_list input_data, float input_min, - float input_max, - std::initializer_list alpha_dims_data, - std::initializer_list alpha_data, float alpha_min, - float alpha_max, - std::initializer_list expected_output_data, - std::initializer_list output_dims_data, - float output_min, float output_max, T* output_data) { - TfLiteIntArray* input_dims = IntArrayFromInitializer(input_dims_data); - TfLiteIntArray* alpha_dims = IntArrayFromInitializer(alpha_dims_data); - TfLiteIntArray* output_dims = IntArrayFromInitializer(output_dims_data); +void TestPreluQuantized(const int* input_dims_data, const float* input_data, + T* input_quantized, const float input_scale, + const int input_zero_point, const int* alpha_dims_data, + const float* alpha_data, T* alpha_quantized, + const float alpha_scale, const int alpha_zero_point, + const float* golden, T* golden_quantized, + const float output_scale, const int output_zero_point, + const int* output_dims_data, T* output_data) { + TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data); + TfLiteIntArray* alpha_dims = IntArrayFromInts(alpha_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); const int output_dims_count = ElementCount(*output_dims); constexpr int inputs_size = 2; constexpr int outputs_size = 1; constexpr int tensors_size = inputs_size + outputs_size; TfLiteTensor tensors[tensors_size] = { - CreateQuantizedTensor(input_data, input_dims, input_min, input_max), - CreateQuantizedTensor(alpha_data, alpha_dims, alpha_min, alpha_max), - CreateQuantizedTensor(output_data, output_dims, output_min, output_max), + CreateQuantizedTensor(input_data, input_quantized, input_dims, + input_scale, input_zero_point), + CreateQuantizedTensor(alpha_data, alpha_quantized, alpha_dims, + alpha_scale, alpha_zero_point), + CreateQuantizedTensor(output_data, output_dims, output_scale, + output_zero_point), }; - int inputs_array_data[] = {2, 0, 1}; - TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); - int outputs_array_data[] = {1, 2}; - TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); + AsymmetricQuantize(golden, golden_quantized, output_dims_count, output_scale, + output_zero_point); - const TfLiteRegistration registration = tflite::ops::micro::Register_PRELU(); - micro::KernelRunner runner(registration, tensors, tensors_size, inputs_array, - outputs_array, - /*builtin_data=*/nullptr, micro_test::reporter); - - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.InitAndPrepare()); - TF_LITE_MICRO_EXPECT_EQ(kTfLiteOk, runner.Invoke()); - - for (int i = 0; i < output_dims_count; ++i) { - TF_LITE_MICRO_EXPECT_EQ(expected_output_data.begin()[i], output_data[i]); - } + ValidatePreluGoldens(tensors, tensors_size, golden_quantized, + output_dims_count, output_data); } } // namespace } // namespace testing @@ -111,74 +106,89 @@ void TestPreluQuantized(std::initializer_list input_dims_data, TF_LITE_MICRO_TESTS_BEGIN TF_LITE_MICRO_TEST(FloatPreluActivationsOpTest) { + const int input_shape[] = {3, 2, 2, 3}; + const float input_values[] = { + 0.0f, 0.0f, 0.0f, // Row 1, Column 1 + 1.0f, 1.0f, 1.0f, // Row 1, Column 2 + -1.0f, -1.0f, -1.0f, // Row 2, Column 1 + -2.0f, -2.0f, -2.0f, // Row 1, Column 2 + }; + const int alpha_shape[] = {3, 1, 1, 3}; + const float alpha_values[] = {0.0f, 1.0f, 2.0f}; + const int output_shape[] = {3, 2, 2, 3}; + const float golden[] = { + 0.0f, 0.0f, 0.0f, // Row 1, Column 1 + 1.0f, 1.0f, 1.0f, // Row 1, Column 2 + 0.0f, -1.0f, -2.0f, // Row 2, Column 1 + 0.0f, -2.0f, -4.0f, // Row 1, Column 2 + }; const int output_dims_count = 12; float output_data[output_dims_count]; - tflite::testing::TestPreluFloat({3, 2, 2, 3}, // input shape - { - 0.0f, 0.0f, 0.0f, // Row 1, Column 1 - 1.0f, 1.0f, 1.0f, // Row 1, Column 2 - -1.0f, -1.0f, -1.0f, // Row 2, Column 1 - -2.0f, -2.0f, -2.0f, // Row 1, Column 2 - }, - {3, 1, 1, 3}, // alpha shape - {0.0f, 1.0f, 2.0f}, // alpha values - { - 0.0f, 0.0f, 0.0f, // Row 1, Column 1 - 1.0f, 1.0f, 1.0f, // Row 1, Column 2 - 0.0f, -1.0f, -2.0f, // Row 2, Column 1 - 0.0f, -2.0f, -4.0f, // Row 1, Column 2 - }, - {3, 2, 2, 3}, // output shape + tflite::testing::TestPreluFloat(input_shape, input_values, alpha_shape, + alpha_values, golden, output_shape, output_data); } TF_LITE_MICRO_TEST(QuantizedUint8PreluActivationsOpTest) { - using tflite::testing::F2Q; - const float kMin = -4; - const float kMax = 127.f / 32.f; - const int output_dims_count = 12; - uint8_t output_data[output_dims_count]; + const int input_shape[] = {3, 2, 2, 3}; + const float input_values[] = { + 0.0f, 0.0f, 0.0f, // Row 1, Column 1 + 0.5f, 0.5f, 0.5f, // Row 1, Column 2 + -1.0f, -1.0f, -1.0f, // Row 2, Column 1 + -0.25f, -0.25f, -0.25f, // Row 1, Column 2 + }; + const int alpha_shape[] = {3, 1, 1, 3}; + const float alpha_values[] = {0.0f, 0.5f, -0.5f}; + const int output_shape[] = {3, 2, 2, 3}; + const float golden[] = { + 0.0f, 0.0f, 0.0f, // Row 1, Column 1 + 0.5f, 0.5f, 0.5f, // Row 1, Column 2 + 0.0f, -0.5f, 0.5f, // Row 2, Column 1 + 0.0f, -0.125f, 0.125f, // Row 1, Column 2 + }; + + const int dims_count = 12; + + uint8_t input_quantized[dims_count]; + uint8_t alpha_quantized[3]; + uint8_t golden_quantized[dims_count]; + float scale = 0.125; + int zero_point = 127; + uint8_t output_data[dims_count]; + tflite::testing::TestPreluQuantized( - {3, 2, 2, 3}, // input shape - {F2Q(0.0f, kMin, kMax), F2Q(0.0f, kMin, kMax), F2Q(0.0f, kMin, kMax), - F2Q(0.5f, kMin, kMax), F2Q(0.5f, kMin, kMax), F2Q(0.5f, kMin, kMax), - F2Q(-1.0f, kMin, kMax), F2Q(-1.0f, kMin, kMax), F2Q(-1.0f, kMin, kMax), - F2Q(-0.25f, kMin, kMax), F2Q(-0.25f, kMin, kMax), - F2Q(-0.25f, kMin, kMax)}, - kMin, kMax, {3, 1, 1, 3}, // alpha shape - {F2Q(0.0f, kMin, kMax), F2Q(0.5f, kMin, kMax), F2Q(-0.5f, kMin, kMax)}, - kMin, kMax, - {F2Q(0.0f, kMin, kMax), F2Q(0.0f, kMin, kMax), F2Q(0.0f, kMin, kMax), - F2Q(0.5f, kMin, kMax), F2Q(0.5f, kMin, kMax), F2Q(0.5f, kMin, kMax), - F2Q(0.0f, kMin, kMax), F2Q(-0.5f, kMin, kMax), F2Q(0.5f, kMin, kMax), - F2Q(0.0f, kMin, kMax), F2Q(-0.125f, kMin, kMax), - F2Q(0.125f, kMin, kMax)}, - {3, 2, 2, 3}, // output shape - kMin, kMax, output_data); + input_shape, input_values, input_quantized, scale, zero_point, + alpha_shape, alpha_values, alpha_quantized, scale, zero_point, golden, + golden_quantized, scale, zero_point, output_shape, output_data); } TF_LITE_MICRO_TEST(QuantizedInt8PreluActivationsOpTest) { - using tflite::testing::F2QS; - const float kMin = -1; - const float kMax = 127.f / 128.f; - const int output_dims_count = 12; - int8_t output_data[output_dims_count]; + const int input_shape[] = {3, 2, 2, 3}; + const float input_values[] = { + 0.0f, 0.0f, 0.0f, // Row 1, Column 1 + 0.5f, 0.5f, 0.5f, // Row 1, Column 2 + -1.0f, -1.0f, -1.0f, // Row 2, Column 1 + -0.25f, -0.25f, -0.25f, // Row 1, Column 2 + }; + const int alpha_shape[] = {3, 1, 1, 3}; + const float alpha_values[] = {0.0f, 0.5f, -0.5f}; + const int output_shape[] = {3, 2, 2, 3}; + const float golden[] = { + 0.0f, 0.0f, 0.0f, // Row 1, Column 1 + 0.5f, 0.5f, 0.5f, // Row 1, Column 2 + 0.0f, -0.5f, 0.5f, // Row 2, Column 1 + 0.0f, -0.125f, 0.125f, // Row 1, Column 2 + }; + const int dims_count = 12; + int8_t input_quantized[dims_count]; + int8_t alpha_quantized[3]; + int8_t golden_quantized[dims_count]; + float scale = 2.0 / 255.0; + int zero_point = 0; + int8_t output_data[dims_count]; tflite::testing::TestPreluQuantized( - {3, 2, 2, 3}, // input shape - {F2QS(0.0f, kMin, kMax), F2QS(0.0f, kMin, kMax), F2QS(0.0f, kMin, kMax), - F2QS(0.5f, kMin, kMax), F2QS(0.5f, kMin, kMax), F2QS(0.5f, kMin, kMax), - F2QS(-1.0f, kMin, kMax), F2QS(-1.0f, kMin, kMax), - F2QS(-1.0f, kMin, kMax), F2QS(-0.25f, kMin, kMax), - F2QS(-0.25f, kMin, kMax), F2QS(-0.25f, kMin, kMax)}, - kMin, kMax, {3, 1, 1, 3}, // alpha shape - {F2QS(0.0f, kMin, kMax), F2QS(0.5f, kMin, kMax), F2QS(-0.5f, kMin, kMax)}, - kMin, kMax, - {F2QS(0.0f, kMin, kMax), F2QS(0.0f, kMin, kMax), F2QS(0.0f, kMin, kMax), - F2QS(0.5f, kMin, kMax), F2QS(0.5f, kMin, kMax), F2QS(0.5f, kMin, kMax), - F2QS(0.0f, kMin, kMax), F2QS(-0.5f, kMin, kMax), F2QS(0.5f, kMin, kMax), - F2QS(0.0f, kMin, kMax), F2QS(-0.125f, kMin, kMax), - F2QS(0.125f, kMin, kMax)}, - {3, 2, 2, 3}, // output shape - kMin, kMax, output_data); + input_shape, input_values, input_quantized, scale, zero_point, + alpha_shape, alpha_values, alpha_quantized, scale, zero_point, golden, + golden_quantized, scale, zero_point, output_shape, output_data); } TF_LITE_MICRO_TESTS_END diff --git a/tensorflow/lite/micro/kernels/reshape_test.cc b/tensorflow/lite/micro/kernels/reshape_test.cc index 59d5577774f..91ecbdc7a49 100644 --- a/tensorflow/lite/micro/kernels/reshape_test.cc +++ b/tensorflow/lite/micro/kernels/reshape_test.cc @@ -194,7 +194,7 @@ TF_LITE_MICRO_TEST(ReshapeWithInvalidShapeShouldFail) { int input_dims_data[] = {3, 1, 2, 2}; TfLiteIntArray* input_dims = tflite::testing::IntArrayFromInts(input_dims_data); - auto input_data = {3.0f}; + const float input_data[] = {3.0f}; auto input_tensor = tflite::testing::CreateFloatTensor(input_data, input_dims); float output_data[4]; @@ -315,7 +315,7 @@ TF_LITE_MICRO_TEST(ReshapeWithLegacyScalarOutputShouldSucceed) { int input_dims_data[] = {1, 1}; TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data); - auto input_data = {3.0f}; + const float input_data[] = {3.0f}; auto input_tensor = CreateFloatTensor(input_data, input_dims); float output_data[1]; @@ -326,8 +326,9 @@ TF_LITE_MICRO_TEST(ReshapeWithLegacyScalarOutputShouldSucceed) { int shape_dims_data[] = {1, 0}; TfLiteIntArray* shape_dims = IntArrayFromInts(shape_dims_data); - auto shape_tensor = - tflite::testing::CreateTensor({0}, shape_dims); + const int32_t shape_data[] = {0}; + auto shape_tensor = tflite::testing::CreateTensor( + shape_data, shape_dims); const float expected_output_with_shape[] = {}; const int expected_output_with_shape_len = 0; const float expected_output_no_shape[] = {3}; diff --git a/tensorflow/lite/micro/testing/test_utils.cc b/tensorflow/lite/micro/testing/test_utils.cc index 0bb97854a41..4d931bdd33b 100644 --- a/tensorflow/lite/micro/testing/test_utils.cc +++ b/tensorflow/lite/micro/testing/test_utils.cc @@ -150,16 +150,6 @@ void PopulateContext(TfLiteTensor* tensors, int tensors_size, } } -TfLiteTensor CreateFloatTensor(std::initializer_list data, - TfLiteIntArray* dims, bool is_variable) { - return CreateFloatTensor(data.begin(), dims, is_variable); -} - -TfLiteTensor CreateBoolTensor(std::initializer_list data, - TfLiteIntArray* dims, bool is_variable) { - return CreateBoolTensor(data.begin(), dims, is_variable); -} - TfLiteTensor CreateQuantizedTensor(const uint8_t* data, TfLiteIntArray* dims, float min, float max, bool is_variable) { TfLiteTensor result; @@ -174,12 +164,6 @@ TfLiteTensor CreateQuantizedTensor(const uint8_t* data, TfLiteIntArray* dims, return result; } -TfLiteTensor CreateQuantizedTensor(std::initializer_list data, - TfLiteIntArray* dims, float min, float max, - bool is_variable) { - return CreateQuantizedTensor(data.begin(), dims, min, max, is_variable); -} - TfLiteTensor CreateQuantizedTensor(const int8_t* data, TfLiteIntArray* dims, float min, float max, bool is_variable) { TfLiteTensor result; @@ -194,12 +178,6 @@ TfLiteTensor CreateQuantizedTensor(const int8_t* data, TfLiteIntArray* dims, return result; } -TfLiteTensor CreateQuantizedTensor(std::initializer_list data, - TfLiteIntArray* dims, float min, float max, - bool is_variable) { - return CreateQuantizedTensor(data.begin(), dims, min, max, is_variable); -} - TfLiteTensor CreateQuantizedTensor(float* data, uint8_t* quantized_data, TfLiteIntArray* dims, bool is_variable) { TfLiteTensor result; @@ -258,11 +236,5 @@ TfLiteTensor CreateQuantized32Tensor(const int32_t* data, TfLiteIntArray* dims, return result; } -TfLiteTensor CreateQuantized32Tensor(std::initializer_list data, - TfLiteIntArray* dims, float scale, - bool is_variable) { - return CreateQuantized32Tensor(data.begin(), dims, scale, is_variable); -} - } // namespace testing } // namespace tflite diff --git a/tensorflow/lite/micro/testing/test_utils.h b/tensorflow/lite/micro/testing/test_utils.h index 053c4417f52..e83ac806d8a 100644 --- a/tensorflow/lite/micro/testing/test_utils.h +++ b/tensorflow/lite/micro/testing/test_utils.h @@ -17,7 +17,6 @@ limitations under the License. #include #include -#include #include #include "tensorflow/lite/c/common.h" @@ -31,12 +30,6 @@ namespace testing { // Note: These methods are deprecated, do not use. See b/141332970. -// TODO(kreeger): Don't use this anymore in our tests. Optimized compiler -// settings can play with pointer placement on the stack (b/140130236). -inline TfLiteIntArray* IntArrayFromInitializer( - std::initializer_list int_initializer) { - return IntArrayFromInts(int_initializer.begin()); -} // Derives the quantization range max from scaling factor and zero point. template @@ -80,28 +73,14 @@ int32_t F2Q32(const float value, const float scale); void PopulateContext(TfLiteTensor* tensors, int tensors_size, ErrorReporter* error_reporter, TfLiteContext* context); -TfLiteTensor CreateFloatTensor(std::initializer_list data, - TfLiteIntArray* dims, bool is_variable = false); - -TfLiteTensor CreateBoolTensor(std::initializer_list data, - TfLiteIntArray* dims, bool is_variable = false); - TfLiteTensor CreateQuantizedTensor(const uint8_t* data, TfLiteIntArray* dims, float min, float max, bool is_variable = false); -TfLiteTensor CreateQuantizedTensor(std::initializer_list data, - TfLiteIntArray* dims, float min, float max, - bool is_variable = false); - TfLiteTensor CreateQuantizedTensor(const int8_t* data, TfLiteIntArray* dims, float min, float max, bool is_variable = false); -TfLiteTensor CreateQuantizedTensor(std::initializer_list data, - TfLiteIntArray* dims, float min, float max, - bool is_variable = false); - TfLiteTensor CreateQuantizedTensor(float* data, uint8_t* quantized_data, TfLiteIntArray* dims, bool is_variable = false); @@ -117,10 +96,6 @@ TfLiteTensor CreateQuantizedTensor(float* data, int16_t* quantized_data, TfLiteTensor CreateQuantized32Tensor(const int32_t* data, TfLiteIntArray* dims, float scale, bool is_variable = false); -TfLiteTensor CreateQuantized32Tensor(std::initializer_list data, - TfLiteIntArray* dims, float scale, - bool is_variable = false); - template inline TfLiteTensor CreateTensor(const input_type* data, TfLiteIntArray* dims, @@ -135,15 +110,6 @@ inline TfLiteTensor CreateTensor(const input_type* data, TfLiteIntArray* dims, return result; } -template -inline TfLiteTensor CreateTensor(std::initializer_list data, - TfLiteIntArray* dims, - bool is_variable = false) { - return CreateTensor(data.begin(), dims, - is_variable); -} - } // namespace testing } // namespace tflite From d9586bfdbec7138be08176c639ec8d5bd36a41a6 Mon Sep 17 00:00:00 2001 From: Meghna Natraj Date: Mon, 3 Aug 2020 13:22:18 -0700 Subject: [PATCH 0274/1017] Fix error message for missing `quantized_input_stats` flag PiperOrigin-RevId: 324670162 Change-Id: I0521c697183b246bad1525b0fe2d313d2d7017bc --- tensorflow/lite/python/lite.py | 7 ++++--- tensorflow/lite/python/lite_test.py | 12 ++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tensorflow/lite/python/lite.py b/tensorflow/lite/python/lite.py index c0a8c33331b..a853cc953fd 100644 --- a/tensorflow/lite/python/lite.py +++ b/tensorflow/lite/python/lite.py @@ -1245,7 +1245,7 @@ class TFLiteConverterBaseV1(TFLiteConverterBase): return object.__getattribute__(self, name) def _validate_quantized_input_stats(self, converter_kwargs, calibrate): - """Ensure quantized_input_stats provided if required.""" + """Ensure the `quantized_input_stats` flag is provided if required.""" quantized_types = frozenset({constants.INT8, constants.QUANTIZED_UINT8}) @@ -1256,8 +1256,9 @@ class TFLiteConverterBaseV1(TFLiteConverterBase): if (requires_quantized_input_stats and not converter_kwargs["quantized_input_stats"]): - raise ValueError("std_dev and mean must be defined when inference_type " - "or inference_input_type is QUANTIZED_UINT8 or INT8.") + raise ValueError("The `quantized_input_stats` flag must be defined when " + "either `inference_type` flag or `inference_input_type` " + "flag is set to tf.uint8 or tf.int8.") def convert(self): """Converts a TensorFlow GraphDef based on instance variables. diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py index ede24b2ede5..e9853c7f17c 100644 --- a/tensorflow/lite/python/lite_test.py +++ b/tensorflow/lite/python/lite_test.py @@ -1126,18 +1126,18 @@ class FromSessionTest(TestModels, parameterized.TestCase): quantized_converter.inference_type = quantized_type quantized_converter.convert() self.assertEqual( - 'std_dev and mean must be defined when inference_type or ' - 'inference_input_type is QUANTIZED_UINT8 or INT8.', - str(error.exception)) + 'The `quantized_input_stats` flag must be defined when ' + 'either `inference_type` flag or `inference_input_type` ' + 'flag is set to tf.uint8 or tf.int8.', str(error.exception)) with self.assertRaises(ValueError) as error: quantized_converter.inference_type = lite_constants.FLOAT quantized_converter.inference_input_type = quantized_type quantized_converter.convert() self.assertEqual( - 'std_dev and mean must be defined when inference_type or ' - 'inference_input_type is QUANTIZED_UINT8 or INT8.', - str(error.exception)) + 'The `quantized_input_stats` flag must be defined when ' + 'either `inference_type` flag or `inference_input_type` ' + 'flag is set to tf.uint8 or tf.int8.', str(error.exception)) quantized_converter.inference_type = quantized_type quantized_converter.inference_input_type = quantized_type From fcec95e19189cadc00a33bf65d0b894ed6722a2b Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Mon, 3 Aug 2020 13:28:30 -0700 Subject: [PATCH 0275/1017] In tf.nondifferentiable_batch_function (https://www.tensorflow.org/api_docs/python/tf/nondifferentiable_batch_function), flip `enable_large_batch_splitting` to True. PiperOrigin-RevId: 324671442 Change-Id: I890d191a778016db9b58ca7638463aba7997e3ad --- tensorflow/python/ops/batch_ops.py | 13 ++++++++++++- tensorflow/tools/api/golden/v1/tensorflow.pbtxt | 2 +- tensorflow/tools/api/golden/v2/tensorflow.pbtxt | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/ops/batch_ops.py b/tensorflow/python/ops/batch_ops.py index 4c470270975..8f163d078ff 100644 --- a/tensorflow/python/ops/batch_ops.py +++ b/tensorflow/python/ops/batch_ops.py @@ -34,7 +34,8 @@ def batch_function(num_batch_threads, batch_timeout_micros, allowed_batch_sizes=None, max_enqueued_batches=10, - autograph=True): + autograph=True, + enable_large_batch_splitting=True): """Batches the computation done by the decorated function. So, for example, in the following code @@ -71,6 +72,15 @@ def batch_function(num_batch_threads, max_enqueued_batches: The maximum depth of the batch queue. Defaults to 10. autograph: Whether to use autograph to compile python and eager style code for efficient graph-mode execution. + enable_large_batch_splitting: The value of this option doesn't affect + processing output given the same input; it affects implementation details + as stated below: 1. Improve batching efficiency by eliminating unnecessary + adding. 2.`max_batch_size` specifies the limit of input and + `allowed_batch_sizes` specifies the limit of a task to be processed. API + user can give an input of size 128 when 'max_execution_batch_size' + is 32 -> implementation can split input of 128 into 4 x 32, schedule + concurrent processing, and then return concatenated results corresponding + to 128. Returns: The decorated function will return the unbatched computation output Tensors. @@ -101,6 +111,7 @@ def batch_function(num_batch_threads, allowed_batch_sizes=allowed_batch_sizes, max_enqueued_batches=max_enqueued_batches, shared_name=name, + enable_large_batch_splitting=enable_large_batch_splitting, f=computation, in_tensors=list(args), captured_tensors=computation.captured_inputs, diff --git a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt index 6adfb231c38..ba64d009908 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.pbtxt @@ -1734,7 +1734,7 @@ tf_module { } member_method { name: "nondifferentiable_batch_function" - argspec: "args=[\'num_batch_threads\', \'max_batch_size\', \'batch_timeout_micros\', \'allowed_batch_sizes\', \'max_enqueued_batches\', \'autograph\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\'], " + argspec: "args=[\'num_batch_threads\', \'max_batch_size\', \'batch_timeout_micros\', \'allowed_batch_sizes\', \'max_enqueued_batches\', \'autograph\', \'enable_large_batch_splitting\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'True\'], " } member_method { name: "norm" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt index 7cf617ddf8b..83baba1b1ce 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.pbtxt @@ -834,7 +834,7 @@ tf_module { } member_method { name: "nondifferentiable_batch_function" - argspec: "args=[\'num_batch_threads\', \'max_batch_size\', \'batch_timeout_micros\', \'allowed_batch_sizes\', \'max_enqueued_batches\', \'autograph\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\'], " + argspec: "args=[\'num_batch_threads\', \'max_batch_size\', \'batch_timeout_micros\', \'allowed_batch_sizes\', \'max_enqueued_batches\', \'autograph\', \'enable_large_batch_splitting\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'True\', \'True\'], " } member_method { name: "norm" From af7bad693cdc07e59e866c0f1a914d6d20a06d2a Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Mon, 3 Aug 2020 13:51:06 -0700 Subject: [PATCH 0276/1017] Expose device memory allocator mem usage via context PiperOrigin-RevId: 324676035 Change-Id: Ib6ad38ccb2f0f6c399c53f650d077b8db4e2c8a8 --- tensorflow/python/eager/context.py | 6 ++ tensorflow/python/eager/context_test.py | 23 +++++++ tensorflow/python/lib/core/pybind11_lib.h | 5 ++ tensorflow/python/tfe_wrapper.cc | 78 +++++++++++++++++++++++ 4 files changed, 112 insertions(+) diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index fbd63f764cf..765c77af7cd 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -1413,6 +1413,12 @@ class Context(object): self._visible_device_list = visible_device_list + def get_total_memory_usage(self, dev): + """Returns total memory usage in bytes for the current device.""" + self._initialize_physical_devices() + self.ensure_initialized() + return pywrap_tfe.TFE_GetTotalMemoryUsage(self._context_handle, dev) + def get_memory_growth(self, dev): """Get if memory growth is enabled for a PhysicalDevice.""" self._initialize_physical_devices() diff --git a/tensorflow/python/eager/context_test.py b/tensorflow/python/eager/context_test.py index fd815fe7433..086f943b3b0 100644 --- a/tensorflow/python/eager/context_test.py +++ b/tensorflow/python/eager/context_test.py @@ -26,6 +26,8 @@ from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import test_util +from tensorflow.python.ops import array_ops from tensorflow.python.platform import test @@ -108,6 +110,27 @@ class ContextTest(test.TestCase): with self.assertRaises(errors.NotFoundError): _ = context.get_function_def('this_should_not_be_found') + @test_util.run_gpu_only + def testGetMemoryUsage(self): + array_ops.zeros([10]) # Allocate some memory on the GPU. + self.assertGreater( + context.context().get_total_memory_usage('GPU:0'), 0) + + def testGetMemoryUsageCPU(self): + with self.assertRaisesRegex(ValueError, 'CPU does not support'): + context.context().get_total_memory_usage('CPU:0') + + def testGetMemoryUsageUnknownDevice(self): + with self.assertRaisesRegex(ValueError, 'Failed parsing device name'): + context.context().get_total_memory_usage('unknown_device') + + @test_util.run_gpu_only + def testGetMemoryUsageAmbiguousDevice(self): + if len(context.context().list_physical_devices('GPU')) < 2: + self.skipTest('Need at least 2 GPUs') + with self.assertRaisesRegex(ValueError, 'Multiple devices'): + context.context().get_total_memory_usage('GPU') + if __name__ == '__main__': ops.enable_eager_execution() diff --git a/tensorflow/python/lib/core/pybind11_lib.h b/tensorflow/python/lib/core/pybind11_lib.h index 6a0471cb4da..a0fb45a5152 100644 --- a/tensorflow/python/lib/core/pybind11_lib.h +++ b/tensorflow/python/lib/core/pybind11_lib.h @@ -60,6 +60,11 @@ void ThrowTypeError(const char* error_message) { throw pybind11::error_already_set(); } +void ThrowValueError(const char* error_message) { + PyErr_SetString(PyExc_ValueError, error_message); + throw pybind11::error_already_set(); +} + } // namespace tensorflow #endif // TENSORFLOW_PYTHON_LIB_CORE_PYBIND11_LIB_H_ diff --git a/tensorflow/python/tfe_wrapper.cc b/tensorflow/python/tfe_wrapper.cc index 9234ecd7102..ec0a1ac1c23 100644 --- a/tensorflow/python/tfe_wrapper.cc +++ b/tensorflow/python/tfe_wrapper.cc @@ -16,6 +16,7 @@ limitations under the License. #include #include "Python.h" +#include "absl/strings/str_format.h" #include "pybind11/chrono.h" #include "pybind11/complex.h" #include "pybind11/functional.h" @@ -351,6 +352,83 @@ PYBIND11_MODULE(_pywrap_tfe, m) { TFE_Py_RegisterFallbackExceptionClass(e.ptr())); }); + m.def( + "TFE_GetTotalMemoryUsage", [](py::handle& ctx, const char* device_name) { + tensorflow::EagerContext* context = tensorflow::ContextFromInterface( + reinterpret_cast( + tensorflow::InputTFE_Context(ctx))); + + tensorflow::DeviceNameUtils::ParsedName input_device_name; + if (!tensorflow::DeviceNameUtils::ParseFullName(device_name, + &input_device_name) && + !tensorflow::DeviceNameUtils::ParseLocalName(device_name, + &input_device_name)) { + tensorflow::ThrowValueError( + absl::StrFormat("Failed parsing device name: '%s'", device_name) + .c_str()); + } + + std::vector devices = + context->local_device_mgr()->ListDevices(); + + tensorflow::Device* matched_device = nullptr; + for (int device_idx = 0; device_idx < devices.size(); device_idx++) { + tensorflow::Device* device = devices[device_idx]; + + if (absl::StrContains(device->name(), "XLA") && + !absl::StrContains(device_name, "XLA")) { + continue; + } + + if (tensorflow::DeviceNameUtils::AreCompatibleDevNames( + input_device_name, device->parsed_name())) { + if (device->device_type() == tensorflow::DEVICE_CPU) { + tensorflow::ThrowValueError( + "CPU does not support getting allocator information"); + } + + if (absl::StrContains(device->device_type(), "XLA") && + !absl::StrContains(device_name, "XLA")) { + // TODO(b/140134773): Remove this workaround. + // Do not accidentally match XLA devices. + continue; + } + + if (matched_device != nullptr) { + tensorflow::ThrowValueError( + absl::StrFormat( + "Multiple devices matching the provided string " + "'%s': '%s' and " + "'%s' ", + device_name, matched_device->name(), device->name()) + .c_str()); + } + matched_device = device; + } + } + + if (matched_device == nullptr) { + tensorflow::ThrowValueError( + absl::StrFormat("No matching devices found for '%s'", device_name) + .c_str()); + } + CHECK(matched_device); + + tensorflow::AllocatorAttributes attrs; + tensorflow::Allocator* allocator = matched_device->GetAllocator(attrs); + + if (absl::optional stats = + allocator->GetStats()) { + return stats->bytes_in_use; + } + + tensorflow::ThrowTypeError( + absl::StrFormat("Allocator stats not available for device '%s'", + matched_device->name()) + .c_str()); + LOG(FATAL) << "Unreachable"; + }); + // XLA Eager Logic m.def("TF_SetXlaEnableLazyCompilation", &TF_SetXlaEnableLazyCompilation); m.def("TF_SetTfXlaCpuGlobalJit", &TF_SetTfXlaCpuGlobalJit); From 2ffd0a6fb6a10c93d5cfe9ad621e84efa9346c76 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 13:54:39 -0700 Subject: [PATCH 0277/1017] tracking device memory transfer in steps_db. PiperOrigin-RevId: 324676713 Change-Id: I7d42f3746f35458ecf2d68a00c65f900ff673449 --- .../convert/step_events_to_steps_db.cc | 6 +++ .../profiler/convert/xplane_to_step_events.cc | 49 ++++++++++++++++--- .../core/profiler/protobuf/steps_db.proto | 15 +++++- tensorflow/core/profiler/utils/event_span.cc | 47 ++++++++++++++++++ tensorflow/core/profiler/utils/event_span.h | 20 ++++++++ 5 files changed, 129 insertions(+), 8 deletions(-) diff --git a/tensorflow/core/profiler/convert/step_events_to_steps_db.cc b/tensorflow/core/profiler/convert/step_events_to_steps_db.cc index 6841929dea7..f37cd6ed103 100644 --- a/tensorflow/core/profiler/convert/step_events_to_steps_db.cc +++ b/tensorflow/core/profiler/convert/step_events_to_steps_db.cc @@ -142,6 +142,12 @@ StepDatabaseResult ConvertStepEventsToStepDb( for (const auto& it : step_details->Collectives()) { collectives[it.first] = it.second; } + // Populates the device transfer stats for this step. + auto& device_memory_transfers = + *per_core_step_info.mutable_device_memory_transfers(); + for (const auto& dma : step_details->DeviceMemoryTransfers()) { + *device_memory_transfers.Add() = dma; + } // The remaining fields in PerCoreStepInfo are not filled. *step_db.add_step_sequence() = per_core_step_info; } diff --git a/tensorflow/core/profiler/convert/xplane_to_step_events.cc b/tensorflow/core/profiler/convert/xplane_to_step_events.cc index 1d80d308193..0af9ecaf4d3 100644 --- a/tensorflow/core/profiler/convert/xplane_to_step_events.cc +++ b/tensorflow/core/profiler/convert/xplane_to_step_events.cc @@ -17,6 +17,7 @@ limitations under the License. #include "absl/container/flat_hash_map.h" #include "absl/strings/match.h" +#include "absl/strings/str_split.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "tensorflow/core/platform/types.h" @@ -51,6 +52,20 @@ inline bool IsRealCpuCompute(absl::string_view event_name) { return !not_real; } +uint64 ParseNumBytesFromMemcpyDetail(absl::string_view memcpy_detail) { + const std::vector params = + absl::StrSplit(memcpy_detail, absl::ByAnyChar(":\n")); + + // Processes value pairs. + for (uint32 ii = 0; ii < params.size(); ii += 2) { + if (params[ii] != "num_bytes") continue; + uint64 value = 0; + if (absl::SimpleAtoi(params[ii + 1], &value)) return value; + break; + } + return 0ULL; +} + } // namespace StepEvents ConvertHostThreadsXLineToStepEvents( @@ -134,6 +149,7 @@ StepEvents ConvertDeviceTraceXLineToStepEvents(const uint64 device_id, int64 correlation_id = -1; int64 group_id = -1; absl::string_view tensor_shapes; + absl::string_view memcpy_details; event.ForEachStat([&](const XStatVisitor& stat) { if (!stat.Type().has_value()) return; switch (stat.Type().value()) { @@ -146,6 +162,9 @@ StepEvents ConvertDeviceTraceXLineToStepEvents(const uint64 device_id, case StatType::kTensorShapes: tensor_shapes = stat.StrOrRefValue(); break; + case StatType::kMemcpyDetails: + memcpy_details = stat.StrOrRefValue(); + break; } }); @@ -153,13 +172,29 @@ StepEvents ConvertDeviceTraceXLineToStepEvents(const uint64 device_id, EventType event_type = ClassifyGpuEvent(event.Name(), tensor_shapes); EventTypeSpan event_type_span(event_type, event.GetTimespan()); result[group_id].AddEvent(event_type_span); - if (event_type == DEVICE_COLLECTIVES) { - AllReduceInfo collective_ops; - collective_ops.set_name(string(event.Name())); - collective_ops.set_start_time_ps(event.TimestampPs()); - collective_ops.set_end_time_ps(event.EndOffsetPs()); - // TODO(jiesun): figure out how to get size info etc. - result[group_id].AddCollectiveOpEvent(device_id, collective_ops); + switch (event_type) { + case DEVICE_COLLECTIVES: { + AllReduceInfo collective_ops; + collective_ops.set_name(string(event.Name())); + collective_ops.set_start_time_ps(event.TimestampPs()); + collective_ops.set_end_time_ps(event.EndOffsetPs()); + // TODO(jiesun): figure out how to get size info etc. + result[group_id].AddCollectiveOpEvent(device_id, collective_ops); + break; + } + case HOST_TO_DEVICE: + case DEVICE_TO_DEVICE: + case DEVICE_TO_HOST: { + // TODO(jiesun): not all memcpy events are grouped, figure out a + // better way to attribute them to steps. + uint64 bytes_transferred = + ParseNumBytesFromMemcpyDetail(memcpy_details); + result[group_id].AddDeviceMemoryTransferEvent( + event_type, event.GetTimespan(), bytes_transferred); + break; + } + default: + return; } } }); diff --git a/tensorflow/core/profiler/protobuf/steps_db.proto b/tensorflow/core/profiler/protobuf/steps_db.proto index 7d5e87fad5a..cf44b817ac8 100644 --- a/tensorflow/core/profiler/protobuf/steps_db.proto +++ b/tensorflow/core/profiler/protobuf/steps_db.proto @@ -15,6 +15,13 @@ message GenericStepBreakdown { map type_ps = 1; } +// Information about memory transfer to/from device memory. +message DeviceMemoryTransfer { + uint64 occurrence = 1; + double time_us = 2; + uint64 bytes_transferred = 3; +} + // Next ID: 5 // Result proto for StepInfo. message StepInfoResult { @@ -90,8 +97,14 @@ message PerCoreStepInfo { // A map from core ID to program replica id. Replica id map could change // during a profile session, but should stay stable within a step. map core_id_to_replica_id_map = 5; - // The result for all-reduce ops.hlo_metrics_db + // A map from core_id to all-reduce ops. map all_reduce_db_per_core = 6; + // Information about deivce memory transfers, categoried by source and + // destination. Ordered by following categories: + // 1. HostToDevice + // 2. DeviceToHost + // 3. DeviceToDevice + repeated DeviceMemoryTransfer device_memory_transfers = 7; } // Result proto for a StepDatabase. diff --git a/tensorflow/core/profiler/utils/event_span.cc b/tensorflow/core/profiler/utils/event_span.cc index acb037420e0..137a798c7f8 100644 --- a/tensorflow/core/profiler/utils/event_span.cc +++ b/tensorflow/core/profiler/utils/event_span.cc @@ -141,6 +141,7 @@ void CombineStepDetails(const StepDetails& src, StepDetails* dst) { dst->AppendMarkers(src.Markers()); dst->AppendEvents(src.Events()); dst->AppendCollectives(src.Collectives()); + dst->AggregateDeviceMemoryTransfers(src.DeviceMemoryTransfers()); } EventType ClassifyDeviceCompute(absl::string_view event_name, @@ -288,6 +289,8 @@ StepEvents ToNonOverlappedStepEvents(const StepEvents& overlapped_step_events) { ToNonOverlappedEvents(step_details.Events()); *non_overlapped_step_events[step_id].MutableCollectives() = step_details.Collectives(); + *non_overlapped_step_events[step_id].MutableDeviceMemoryTransfers() = + step_details.DeviceMemoryTransfers(); } return non_overlapped_step_events; } @@ -311,10 +314,54 @@ void StepDetails::AppendCollectives( } } +void StepDetails::AggregateDeviceMemoryTransfers( + const std::vector device_memory_transfers) { + if (device_memory_transfers.size() != device_memory_transfers_.size()) { + return; // Sanity check. + } + for (size_t i = 0; i < device_memory_transfers.size(); ++i) { + device_memory_transfers_[i].set_occurrence( + device_memory_transfers_[i].occurrence() + + device_memory_transfers[i].occurrence()); + device_memory_transfers_[i].set_bytes_transferred( + device_memory_transfers_[i].bytes_transferred() + + device_memory_transfers[i].bytes_transferred()); + device_memory_transfers_[i].set_time_us( + device_memory_transfers_[i].time_us() + + device_memory_transfers[i].time_us()); + } +} + void StepDetails::AddCollectiveOpEvent(uint64 core_id, const AllReduceInfo& e) { *collectives_[core_id].add_all_reduce_info() = e; } +void StepDetails::AddDeviceMemoryTransferEvent(EventType event_type, + const Timespan& time_span, + uint64 bytes) { + int index = 0; + switch (event_type) { + case HOST_TO_DEVICE: + index = 0; + break; + case DEVICE_TO_HOST: + index = 1; + break; + case DEVICE_TO_DEVICE: + index = 2; + break; + default: + return; + } + device_memory_transfers_[index].set_occurrence( + device_memory_transfers_[index].occurrence() + 1); + device_memory_transfers_[index].set_time_us( + device_memory_transfers_[index].time_us() + + time_span.duration_ps() / 1000000.0); + device_memory_transfers_[index].set_bytes_transferred( + device_memory_transfers_[index].bytes_transferred() + bytes); +} + Timespan StepDetails::StepTime() const { Timespan max_host_step_time; Timespan max_device_step_time; diff --git a/tensorflow/core/profiler/utils/event_span.h b/tensorflow/core/profiler/utils/event_span.h index b1f325b08e2..6ffbd228d5e 100644 --- a/tensorflow/core/profiler/utils/event_span.h +++ b/tensorflow/core/profiler/utils/event_span.h @@ -112,11 +112,16 @@ struct StepMarker { // StepDetails of the same step executed on different cores. class StepDetails { public: + StepDetails() : device_memory_transfers_(3) {} + const std::vector& Markers() const { return markers_; } const std::vector& Events() const { return events_; } const absl::flat_hash_map& Collectives() const { return collectives_; } + const std::vector& DeviceMemoryTransfers() const { + return device_memory_transfers_; + } // Returns the step time. Timespan StepTime() const; std::vector* MutableMarkers() { return &markers_; } @@ -124,12 +129,20 @@ class StepDetails { absl::flat_hash_map* MutableCollectives() { return &collectives_; } + std::vector* MutableDeviceMemoryTransfers() { + return &device_memory_transfers_; + } // Adds a step-marker to this step. void AddMarker(const StepMarker& m); // Adds an EventTypeSpan to this step. void AddEvent(const EventTypeSpan& e); // Adds a collective op to this step. void AddCollectiveOpEvent(uint64 core_id, const AllReduceInfo& e); + // Appends device memory transfer events to this step. + // Only event type of HOST_TO_DEVICE/DEVICE_TO_DEVICE/DEVICE_TO_HOST are + // allowed. + void AddDeviceMemoryTransferEvent(EventType event_type, + const Timespan& time_span, uint64 bytes); // Appends the step-markers from another step to this step. void AppendMarkers(const std::vector& other_markers); // Appends the events from another step to this step. @@ -137,6 +150,9 @@ class StepDetails { // Appends the collectives from another step to this step. void AppendCollectives( const absl::flat_hash_map& collectives); + // Accumulates the device memory transfers from another step to this step. + void AggregateDeviceMemoryTransfers( + const std::vector device_memory_transfers); // Equality test. bool operator==(const StepDetails& other) const; // Inequality test. @@ -155,6 +171,10 @@ class StepDetails { std::vector events_; // Collective operation related events such as all-reduce etc. absl::flat_hash_map collectives_; + // Device memory transfers (including time and bytes involved). + // TODO(jiesun): Consider to use IntervalSet instead of just sum up the event + // durations. + std::vector device_memory_transfers_; }; // Map from step_id to the events happened in that step. From a3df6cff1bd7e43329e4dc8600d1e4defed72b90 Mon Sep 17 00:00:00 2001 From: Robert David Date: Mon, 3 Aug 2020 14:05:37 -0700 Subject: [PATCH 0278/1017] Add MeanStddevNormalization to the list of operations and the OpenCL operation selector. PiperOrigin-RevId: 324678979 Change-Id: Ie4ba8df7d57f184f2bdcd790d4ec761ba1681dee --- .../delegates/gpu/cl/selectors/operation_selector.cc | 7 +++++++ tensorflow/lite/delegates/gpu/common/operations.cc | 9 ++++++--- tensorflow/lite/delegates/gpu/common/operations.h | 1 + tensorflow/lite/delegates/gpu/gl/kernels/registry.cc | 1 + tensorflow/lite/delegates/gpu/metal/api.cc | 2 ++ 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc index ffe9acb8299..088677ba7e2 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc @@ -19,6 +19,7 @@ limitations under the License. #include "absl/types/any.h" #include "tensorflow/lite/delegates/gpu/cl/cl_device.h" #include "tensorflow/lite/delegates/gpu/cl/kernels/elementwise.h" +#include "tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.h" #include "tensorflow/lite/delegates/gpu/cl/selectors/convolution_selector.h" #include "tensorflow/lite/delegates/gpu/cl/selectors/convolution_transposed_selector.h" #include "tensorflow/lite/delegates/gpu/cl/selectors/default_selector.h" @@ -286,6 +287,12 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, return SelectMean(attr, op_def, creation_context.device->GetInfo(), gpu_op); } + case OperationType::MEAN_STDDEV_NORMALIZATION: { + MeanStdDevNormalization operation = CreateMeanStdDevNormalization(op_def); + *gpu_op = + absl::make_unique(std::move(operation)); + return absl::OkStatus(); + } case OperationType::MUL: { if (inputs.size() == 2) { ElementwiseTwoInput operation = diff --git a/tensorflow/lite/delegates/gpu/common/operations.cc b/tensorflow/lite/delegates/gpu/common/operations.cc index dd0a91b2705..245a5a80639 100644 --- a/tensorflow/lite/delegates/gpu/common/operations.cc +++ b/tensorflow/lite/delegates/gpu/common/operations.cc @@ -110,6 +110,8 @@ std::string ToString(enum OperationType op) { return "max_unpooling"; case OperationType::MEAN: return "mean"; + case OperationType::MEAN_STDDEV_NORMALIZATION: + return "mean_stddev_normalization"; case OperationType::MINIMUM: return "minimum"; case OperationType::MUL: @@ -156,10 +158,9 @@ std::string ToString(enum OperationType op) { return "tanh"; case OperationType::TRANSPOSE: return "transpose"; - default: - break; + case OperationType::UNKNOWN: + return "unknown_operation"; } - return "unknown_operation"; } OperationType OperationTypeFromString(const std::string& name) { @@ -185,6 +186,8 @@ OperationType OperationTypeFromString(const std::string& name) { {"maximum", OperationType::MAXIMUM}, {"max_unpooling", OperationType::MAX_UNPOOLING_2D}, {"mean", OperationType::MEAN}, + {"mean_stddev_normalization", + OperationType::MEAN_STDDEV_NORMALIZATION}, {"minimum", OperationType::MINIMUM}, {"mul", OperationType::MUL}, {"pad", OperationType::PAD}, diff --git a/tensorflow/lite/delegates/gpu/common/operations.h b/tensorflow/lite/delegates/gpu/common/operations.h index fcce6532c1d..225165589ae 100644 --- a/tensorflow/lite/delegates/gpu/common/operations.h +++ b/tensorflow/lite/delegates/gpu/common/operations.h @@ -53,6 +53,7 @@ enum class OperationType { MAXIMUM, MAX_UNPOOLING_2D, MEAN, + MEAN_STDDEV_NORMALIZATION, MINIMUM, MUL, PAD, diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc b/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc index b4bfbcd8f56..0d2438aacc6 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc @@ -82,6 +82,7 @@ class Registry : public NodeShader { insert_op(Type::FULLY_CONNECTED, NewFullyConnectedNodeShader); insert_op(Type::LSTM, NewLstmNodeShader); insert_op(Type::MEAN, NewMeanNodeShader); + // TODO(b/162763635): implement MeanStddevNormalization for OpenGL. insert_op(Type::MUL, NewMultiplyNodeShader); insert_op(Type::PAD, NewPadNodeShader); insert_op(Type::POOLING_2D, NewPoolingNodeShader); diff --git a/tensorflow/lite/delegates/gpu/metal/api.cc b/tensorflow/lite/delegates/gpu/metal/api.cc index 7b086a6d130..fcab962ee61 100644 --- a/tensorflow/lite/delegates/gpu/metal/api.cc +++ b/tensorflow/lite/delegates/gpu/metal/api.cc @@ -406,6 +406,8 @@ absl::Status RegisterPrimaryOps(const GraphFloat32& graph, const Node* node, case OperationType::BATCH_TO_SPACE: case OperationType::CONST: case OperationType::LSTM: + // TODO(b/162763635): implement MeanStddevNormalization for Metal. + case OperationType::MEAN_STDDEV_NORMALIZATION: case OperationType::SPACE_TO_BATCH: case OperationType::TRANSPOSE: case OperationType::UNKNOWN: From 729b23995f7a655194748afe24a8e7a065f4309b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 14:06:04 -0700 Subject: [PATCH 0279/1017] Test with MWMS in custom_training_loop_optimizer_test PiperOrigin-RevId: 324679085 Change-Id: I3a550fde03380d0327906dcc9d449837a5f6cc8b --- tensorflow/python/keras/distribute/BUILD | 2 +- .../custom_training_loop_optimizer_test.py | 25 +++++++------------ 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/tensorflow/python/keras/distribute/BUILD b/tensorflow/python/keras/distribute/BUILD index 2c8ba97dbfa..5a5cff01e33 100644 --- a/tensorflow/python/keras/distribute/BUILD +++ b/tensorflow/python/keras/distribute/BUILD @@ -275,7 +275,7 @@ distribute_py_test( "//tensorflow/python:variables", "//tensorflow/python/distribute:combinations", "//tensorflow/python/distribute:strategy_combinations", - "//tensorflow/python/distribute:test_util", + "//tensorflow/python/distribute:values", "//tensorflow/python/eager:def_function", "//tensorflow/python/eager:test", "//tensorflow/python/keras/optimizer_v2", diff --git a/tensorflow/python/keras/distribute/custom_training_loop_optimizer_test.py b/tensorflow/python/keras/distribute/custom_training_loop_optimizer_test.py index 0a12d85bebd..b9eee26220a 100644 --- a/tensorflow/python/keras/distribute/custom_training_loop_optimizer_test.py +++ b/tensorflow/python/keras/distribute/custom_training_loop_optimizer_test.py @@ -22,7 +22,7 @@ from absl.testing import parameterized from tensorflow.python.distribute import combinations from tensorflow.python.distribute import strategy_combinations -from tensorflow.python.distribute import test_util +from tensorflow.python.distribute import values from tensorflow.python.eager import def_function from tensorflow.python.eager import test from tensorflow.python.framework import ops @@ -35,14 +35,7 @@ class OptimizerTest(test.TestCase, parameterized.TestCase): @combinations.generate( combinations.times( combinations.combine( - distribution=[ - strategy_combinations.mirrored_strategy_with_gpu_and_cpu, - strategy_combinations.mirrored_strategy_with_two_gpus, - strategy_combinations.multi_worker_mirrored_2x1_cpu, - strategy_combinations.multi_worker_mirrored_2x1_gpu, - strategy_combinations.tpu_strategy, - strategy_combinations.tpu_strategy_one_step, - ], + distribution=strategy_combinations.multidevice_strategies, mode=["eager"], ), combinations.concat( @@ -62,10 +55,10 @@ class OptimizerTest(test.TestCase, parameterized.TestCase): @def_function.function def optimize(): - grads = ops.convert_to_tensor([[1., 1.], - [2., 2.]]) - grads = distribution.experimental_distribute_values_from_function( - lambda ctx: grads[ctx.replica_id_in_sync_group]) + grads = values.PerReplica([ + ops.convert_to_tensor([1., 1.]), + ops.convert_to_tensor([2., 2.]), + ]) def step_fn(grads): optimizer.apply_gradients( @@ -73,8 +66,8 @@ class OptimizerTest(test.TestCase, parameterized.TestCase): experimental_aggregate_gradients=experimental_aggregate_gradients) return v.read_value() - return test_util.gather(distribution, - distribution.run(step_fn, args=(grads,))) + return distribution.experimental_local_results( + distribution.run(step_fn, args=(grads,))) self.assertAllClose(optimize(), expected) @@ -125,4 +118,4 @@ class OptimizerTest(test.TestCase, parameterized.TestCase): if __name__ == "__main__": - combinations.main() + test.main() From 6b3990c84e1740d3a543cb934762a6bd4815c241 Mon Sep 17 00:00:00 2001 From: Andy Lou Date: Mon, 3 Aug 2020 14:10:33 -0700 Subject: [PATCH 0280/1017] Fix asan error, when there are no matching files in matching_files_op. PiperOrigin-RevId: 324680130 Change-Id: Idec9a7826bf85780eb5adcebe6a168f1858144e7 --- tensorflow/core/kernels/matching_files_op.cc | 12 +++++++----- .../python/data/kernel_tests/list_files_test.py | 5 ++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tensorflow/core/kernels/matching_files_op.cc b/tensorflow/core/kernels/matching_files_op.cc index 0ba718c88ec..515e58d518a 100644 --- a/tensorflow/core/kernels/matching_files_op.cc +++ b/tensorflow/core/kernels/matching_files_op.cc @@ -54,13 +54,15 @@ class MatchingFilesOp : public OpKernel { context, context->allocate_output("filenames", TensorShape({num_files}), &output_t)); auto output = output_t->vec(); - int index = 0; - for (int i = 0; i < num_patterns; ++i) { - for (int j = 0; j < all_fnames[i].size(); j++) { - output(index++) = all_fnames[i][j]; + if (output.size() > 0) { + int index = 0; + for (int i = 0; i < num_patterns; ++i) { + for (int j = 0; j < all_fnames[i].size(); j++) { + output(index++) = all_fnames[i][j]; + } } + std::sort(&output(0), &output(0) + num_files); } - std::sort(&output(0), &output(0) + num_files); } }; diff --git a/tensorflow/python/data/kernel_tests/list_files_test.py b/tensorflow/python/data/kernel_tests/list_files_test.py index 40b4b77116c..b57bfb7293b 100644 --- a/tensorflow/python/data/kernel_tests/list_files_test.py +++ b/tensorflow/python/data/kernel_tests/list_files_test.py @@ -113,7 +113,7 @@ class ListFilesTest(test_base.DatasetTestBase, parameterized.TestCase): # Each run should produce the same set of filenames, which may be # different from the order of `expected_filenames`. - self.assertItemsEqual(expected_filenames, all_actual_filenames[0]) + self.assertCountEqual(expected_filenames, all_actual_filenames[0]) # However, the different runs should produce filenames in the same order # as each other. self.assertEqual(all_actual_filenames[0], all_actual_filenames[1]) @@ -199,7 +199,7 @@ class ListFilesTest(test_base.DatasetTestBase, parameterized.TestCase): actual_filenames.append(compat.as_bytes(self.evaluate(next_element()))) with self.assertRaises(errors.OutOfRangeError): self.evaluate(next_element()) - self.assertItemsEqual(expected_filenames, actual_filenames) + self.assertCountEqual(expected_filenames, actual_filenames) self.assertEqual(actual_filenames[:len(filenames)], actual_filenames[len(filenames):]) @@ -234,6 +234,5 @@ class ListFilesTest(test_base.DatasetTestBase, parameterized.TestCase): assert_items_equal=True) - if __name__ == '__main__': test.main() From b58a8717b1aad125f893c520e897e6f56c6345b4 Mon Sep 17 00:00:00 2001 From: Ran Chen Date: Mon, 3 Aug 2020 14:22:52 -0700 Subject: [PATCH 0281/1017] Add auto_restart to multi_process_runner This helps creating fault tolerance test cases. MWMS currently requires an external system which brings back tasks that are down, otherwise the remaining workers may hang forever. Ideally the remaining workers should error, which is what I'm working on. But it's beneficial to have test cases reflecting the current behavior since in many deployment, we do have a cluster management system that does the restart (e.g. k8s). This also changes the behavior of dependence_on_chief. We used to terminate the cluster if chief exits when join() is called. Now with a watchdog thread, that happens immediately after the chief's terminate. PiperOrigin-RevId: 324682642 Change-Id: I56ce27658298916d1ddd4507b90b79db0a2d4673 --- .../python/distribute/multi_process_runner.py | 287 +++++++++++++----- .../distribute/multi_process_runner_test.py | 98 +++++- 2 files changed, 298 insertions(+), 87 deletions(-) diff --git a/tensorflow/python/distribute/multi_process_runner.py b/tensorflow/python/distribute/multi_process_runner.py index e5be4fa4a14..4ded663e588 100644 --- a/tensorflow/python/distribute/multi_process_runner.py +++ b/tensorflow/python/distribute/multi_process_runner.py @@ -67,7 +67,8 @@ except ImportError: # exception stack trace info is stored in exc_info to pass on to parent process # to be re-raised. _ProcessStatusInfo = collections.namedtuple( - '_ProcessStatusInfo', ['is_successful', 'exc_info', 'return_value']) + '_ProcessStatusInfo', + ['task_type', 'task_id', 'is_successful', 'exc_info', 'return_value']) # Information returned from a successful MultiProcessRunner run. MultiProcessRunnerResult = collections.namedtuple('MultiProcessRunnerResult', @@ -97,6 +98,11 @@ Resources = collections.namedtuple('Resources', [ # "medium" timeout of the test runs. _DEFAULT_TIMEOUT_SEC = 200 +# The timeout in seconds to wait to force kill a child process. When a child +# process times out we first try to SIGTERM it so that it has a chance to dump +# stacktraces. However dumping stacktrace can take a long time. +_FORCE_KILL_WAIT_SEC = 30 + class MultiProcessRunner(object): """A utility class to start multiple processes to simulate a cluster. @@ -124,6 +130,8 @@ class MultiProcessRunner(object): list_stdout=False, use_dill_for_args=True, daemon=False, + dependence_on_chief=True, + auto_restart=False, args=None, kwargs=None): """Creates a multi-process runner. @@ -161,6 +169,11 @@ class MultiProcessRunner(object): can pickle more objects, but doesn't work with types in `multiprocessing` library like `Mutex`. daemon: Whether to start processes as daemons. + dependence_on_chief: Whether to terminates the cluster if the chief exits. + If auto_restart is True, it only terminates the cluster if the chief + exits with a zero exit code. + auto_restart: Whether to automatically restart processes that exit with + non-zero exit code. args: Positional arguments to be sent to functions run on processes. kwargs: Keyword arguments to be sent to functions run on processes. @@ -190,9 +203,10 @@ class MultiProcessRunner(object): self._stream_stdout = stream_stdout # TODO(rchao): Revisit list_stdout argument to consider other solution. self._list_stdout = list_stdout - self._dependence_on_chief = True + self._dependence_on_chief = dependence_on_chief self._use_dill_for_args = use_dill_for_args self._daemon = daemon + self._auto_restart = auto_restart self._args = args or () self._kwargs = kwargs or {} @@ -201,8 +215,15 @@ class MultiProcessRunner(object): self._executing_eagerly = context.executing_eagerly() self._joined = False + self._process_lock = threading.Lock() + # Guarded by self._process_lock. self._processes = {} - self._outstanding_subprocess_count = 0 + # Record which processes are terminated. Due to a bug in Python<3.7, + # terminated processes return 255 exit code, which should cause an exception + # in join(). + # https://bugs.python.org/issue30589 + # Guarded by self._process_lock. + self._terminated = set() self._reading_threads = [] self._manager = manager() @@ -215,8 +236,7 @@ class MultiProcessRunner(object): # safe. self._streaming_queue = self._manager.Queue() - # This flag will be set to True once terminate_all() is called. - self._all_forced_terminated = False + self._watchdog_thread = None def set_args(self, args=None, kwargs=None): self._args = args or self._args @@ -281,7 +301,7 @@ class MultiProcessRunner(object): daemon=self._daemon) p.start() self._processes[(task_type, task_id)] = p - self._outstanding_subprocess_count += 1 + self._terminated.discard((task_type, task_id)) # For each subprocess, we dedicate a thread continuously reading lines # from them. @@ -291,17 +311,26 @@ class MultiProcessRunner(object): thread.start() self._reading_threads.append(thread) + if self._watchdog_thread is None or not self._watchdog_thread.is_alive(): + self._watchdog_thread = threading.Thread(target=self._process_watchdog) + self._watchdog_thread.start() + def start(self): """Starts processes, one for each task in `cluster_spec`. Note that this is best effort by the applicable multiprocessing library, and it may take up to seconds for a subprocess to be successfully started. """ - if self._processes: - raise ValueError('MultiProcessRunner already started.') - for task_type, addresses in self._cluster_spec.items(): - for task_id, _ in enumerate(addresses): - self._start_subprocess_and_reading_thread(task_type, task_id) + with self._process_lock: + if self._processes: + raise ValueError('MultiProcessRunner already started.') + if self._joined: + raise ValueError('cannot start new processes after' + 'MultiProcessRunner.join() is called') + + for task_type, addresses in self._cluster_spec.items(): + for task_id, _ in enumerate(addresses): + self._start_subprocess_and_reading_thread(task_type, task_id) # TODO(rchao): Remove the need of using SIGALRM if possible. At this time, # without this the tests become very flaky. @@ -353,10 +382,14 @@ class MultiProcessRunner(object): """ if self._processes: raise ValueError('MultiProcessRunner already started.') - for task_type, addresses in self._cluster_spec.items(): - for task_id, _ in enumerate(addresses): - if not (task_type == as_task_type and task_id == as_task_id): - self._start_subprocess_and_reading_thread(task_type, task_id) + with self._process_lock: + if self._joined: + raise ValueError('cannot start new processes after' + 'MultiProcessRunner.join() is called') + for task_type, addresses in self._cluster_spec.items(): + for task_id, _ in enumerate(addresses): + if not (task_type == as_task_type and task_id == as_task_id): + self._start_subprocess_and_reading_thread(task_type, task_id) _set_tf_config(as_task_type, as_task_id, self._cluster_spec, self._rpc_layer) @@ -392,13 +425,17 @@ class MultiProcessRunner(object): args: Optional positional arguments to be supplied in `proc_func`. kwargs: Optional keyword arguments to be supplied in `proc_func`. """ - self._start_subprocess_and_reading_thread( - task_type, - task_id, - cluster_spec=cluster_spec, - proc_func=proc_func, - args=args or (), - kwargs=kwargs or {}) + with self._process_lock: + if self._joined: + raise ValueError('cannot start new processes after' + 'MultiProcessRunner.join() is called') + self._start_subprocess_and_reading_thread( + task_type, + task_id, + cluster_spec=cluster_spec, + proc_func=proc_func, + args=args or (), + kwargs=kwargs or {}) def _queue_to_list(self, queue_to_convert): """Convert `queue.Queue` to `list`.""" @@ -411,9 +448,17 @@ class MultiProcessRunner(object): break return list_to_return + def _get_process_statuses(self): + # One worker may have multiple statuses. We only keep the last one. + statuses = {} + for status in self._queue_to_list(self._process_status_queue): + statuses[(status.task_type, status.task_id)] = status + return statuses + def get_process_id(self, task_type, task_id): """Returns the subprocess id given the task type and task id.""" - p = self._processes.get((task_type, task_id), None) + with self._process_lock: + p = self._processes.get((task_type, task_id), None) return p.pid if p else None def get_process_exit_code(self, task_type, task_id): @@ -430,22 +475,54 @@ class MultiProcessRunner(object): KeyError: If the corresponding subprocess is not found with `task_type` and `task_id`. """ - p = self._processes[(task_type, task_id)] + with self._process_lock: + p = self._processes[(task_type, task_id)] return p.exitcode if p else None - def _join_or_terminate(self, task_type, task_id, process, timeout): - """Joins a process. If it times out, terminate all procsses.""" - logging.info('joining %s-%d', task_type, task_id) - process.join(timeout) - # If exitcode is None, the process aren't terminated and this is a - # timeout. - if process.exitcode is None: - # Force termination to dump worker processes stack trace. - self.terminate_all(sig=signal.SIGTERM) - process_statuses = self._queue_to_list(self._process_status_queue) - raise SubprocessTimeoutError( - '%s-%d and possibly more subprocesses timed out.' % - (task_type, task_id), self._get_mpr_result(process_statuses)) + def _process_watchdog(self): + """Simulates a cluster management system. + + - If auto_restart is True, it restarts processes that exit with a non-zero + exit code. Note that when join() times out it overrides auto_restart to + False. + - If dependence_on_chief is True, it terminates all processes once the chief + exits. If auto_restart is also True, it only terminates all processes if + the chief exit with a zero exit code, otherwise it restarts the chief. + + This runs in self._watchdog_thread. + """ + while True: + time.sleep(1) + with self._process_lock: + chief = self._processes.get(('chief', 0), None) + # Terminate the cluster when _dependence_on_chief is True if either: + # - chief has exited with zero exit code. + # - chief has exited with non-zero exit code and self._auto_restart is + # False. + if chief and self._dependence_on_chief and chief.exitcode is not None: + if chief.exitcode == 0 or (not self._auto_restart): + for p in self._processes.values(): + # Give other processes a chance to exit on their own. + p.join(timeout=3) + self._terminate_all() + for p in self._processes.values(): + p.join() + return + + # Auto restart failed processes if self._auto_restart is True. + if self._auto_restart: + has_failure = False + for (task_type, task_id), p in self._processes.items(): + if p.exitcode is not None and p.exitcode != 0: + has_failure = True + logging.info('Restarting failed %s-%d', task_type, task_id) + self._start_subprocess_and_reading_thread(task_type, task_id) + if has_failure: + continue + + # Exit the thread if all processes have exited at this point. + if all(p.exitcode is not None for p in self._processes.values()): + return def join(self, timeout=_DEFAULT_TIMEOUT_SEC): """Joins all the processes with timeout. @@ -489,41 +566,48 @@ class MultiProcessRunner(object): cases. Exception: if there is an Exception propagated from any subprocess. """ - if self._joined: - raise ValueError("MultiProcessRunner can't be joined twice.") - self._joined = True + with self._process_lock: + if self._joined: + raise ValueError("MultiProcessRunner can't be joined twice.") + self._joined = True - chief = self._processes.get(('chief', 0), None) - if self._dependence_on_chief and chief: - self._join_or_terminate('chief', 0, chief, timeout) - # Give other processes a chance to exit on their own. - for p in self._processes.values(): - p.join(timeout=3) - self.terminate_all() - else: - for (task_type, task_id), p in self._processes.items(): - self._join_or_terminate(task_type, task_id, p, timeout) + self._watchdog_thread.join(timeout) + if self._watchdog_thread.is_alive(): + # Timeout. Force termination to dump worker processes stack trace. + with self._process_lock: + self._auto_restart = False + logging.error('Timeout when joining for child processes. Terminating...') + self.terminate_all(sig=signal.SIGTERM) + # Wait for the processes to terminate by themselves first, so they have a + # chance to dump stacktraces. After _FORCE_KILL_WAIT_SEC, we SIGKILL them. + self._watchdog_thread.join(_FORCE_KILL_WAIT_SEC) + if self._watchdog_thread.is_alive(): + logging.error('Timeout when waiting for child processes to ' + 'print stacktrace. Sending SIGKILL...') + self.terminate_all() + self._watchdog_thread.join() + process_statuses = self._get_process_statuses() + raise SubprocessTimeoutError('one or more subprocesses timed out.', + self._get_mpr_result(process_statuses)) for (task_type, task_id), p in self._processes.items(): logging.info('%s-%d exit code: %s', task_type, task_id, p.exitcode) - process_statuses = self._queue_to_list(self._process_status_queue) - for process_status in process_statuses: + process_statuses = self._get_process_statuses() + for process_status in process_statuses.values(): assert isinstance(process_status, _ProcessStatusInfo) if not process_status.is_successful: six.reraise(*process_status.exc_info) # Checking all the processes that are expected to exit properly. for (task_type, task_id), p in self._processes.items(): - if self._dependence_on_chief and chief and task_type != 'chief': - # If _dependence_on_chief, other processes may have been - # forced-terminated, which is expected. - continue - # Successfully exiting process has exit code 0. - if p.exitcode is None or p.exitcode > 0: + # Successfully exiting process has exit code 0. We ignore processes that + # are terminated. + assert p.exitcode is not None + if (p.exitcode > 0 and (task_type, task_id) not in self._terminated): raise UnexpectedSubprocessExitError( - 'Subprocess %s-%d exited with exit code %d. See logs for details.' % - (task_type, task_id, p.exitcode), + 'Subprocess %s-%d exited with exit code %s. See logs for details.' + % (task_type, task_id, p.exitcode), self._get_mpr_result(process_statuses)) logging.info('Joining log reading threads.') @@ -539,34 +623,60 @@ class MultiProcessRunner(object): def _get_mpr_result(self, process_statuses): stdout = self._queue_to_list(self._streaming_queue) return_values = [] - for process_status in process_statuses: + for process_status in process_statuses.values(): if process_status.return_value is not None: return_values.append(process_status.return_value) return MultiProcessRunnerResult(stdout=stdout, return_value=return_values) def terminate(self, task_type, task_id): - """Terminates the process with `task_type` and `task_id`.""" - p = self._processes.get((task_type, task_id), None) - if p is None: - raise ValueError('{}-{} does not exist'.format(task_type, task_id)) - # TODO(crccw): change to use Process.terminate() as well. - self._parent_to_sub_queue.put('terminate {} {}'.format(task_type, task_id)) - p.join() + """Terminates the process with `task_type` and `task_id`. + + If auto_retart=True, the terminated task will be restarted unless the chief + has already exited with zero exit code. + + Args: + task_type: the task type. + task_id: the task id. + + """ + with self._process_lock: + p = self._processes.get((task_type, task_id), None) + if p is None: + raise ValueError('{}-{} does not exist'.format(task_type, task_id)) + self._terminated.add((task_type, task_id)) + # TODO(crccw): change to use Process.terminate() as well. + self._parent_to_sub_queue.put('terminate {} {}'.format( + task_type, task_id)) + p.join() + + def _terminate_all(self, sig=None): + """Terminates all subprocesses. + + The caller is required to hold self._process_lock. + + Args: + sig: the signal used to terminate the process. The default is SIGKILL. + """ - def terminate_all(self, sig=None): - """Terminates all subprocesses.""" # Use SIGKILL as default. In systems where that's unavailable such as # windows, use SIGTERM. sig = sig or getattr(signal, 'SIGKILL', signal.SIGTERM) for (task_type, task_id), p in self._processes.items(): + if p.exitcode is not None: + continue try: os.kill(p.pid, sig) + self._terminated.add((task_type, task_id)) logging.info('%s-%d terminated with signal %r.', task_type, task_id, sig) except ProcessLookupError: logging.info('Attempting to kill %s-%d but it does not exist.', task_type, task_id) - self._all_forced_terminated = True + + def terminate_all(self, sig=None): + """Terminates all subprocesses.""" + with self._process_lock: + self._terminate_all(sig) class _Process(multi_process_lib.Process): @@ -625,11 +735,13 @@ class _ProcFunc(object): time.sleep(0.1) self._resources.process_status_queue.put( _ProcessStatusInfo( + task_type=task_type, + task_id=task_id, is_successful=True, exc_info=None, return_value=None)) - # `os._exit(0)` is used to more reliably terminate a subprocess. - os._exit(0) # pylint: disable=protected-access + # `os._exit(1)` is used to more reliably terminate a subprocess. + os._exit(1) # pylint: disable=protected-access def _close_streaming(self): """Close stdout, stderr and streaming pipe. @@ -685,7 +797,8 @@ class _ProcFunc(object): v2_compat.enable_v2_behavior() with self._runtime_mode(test_env.executing_eagerly): - info = _run_contained(proc_func, args, kwargs) + info = _run_contained(test_env.task_type, test_env.task_id, proc_func, + args, kwargs) self._resources.process_status_queue.put(info) # Re-raise the exception in addition to reporting it to the parent @@ -774,7 +887,7 @@ class MultiProcessPoolRunner(object): task_type, task_id, proc_func=_pool_runner_worker, - args=(initializer, conn2)) + args=(task_type, task_id, initializer, conn2)) def run(self, proc_func, args=None, kwargs=None): """Runs `proc_func` with `args` and `kwargs` on all jobs. @@ -819,7 +932,7 @@ class MultiProcessPoolRunner(object): return return_values -def _pool_runner_worker(initializer, conn): +def _pool_runner_worker(task_type, task_id, initializer, conn): """Function that runs on the workers in a pool. It listens for callables to run and returns the result until `conn` is closed. @@ -827,8 +940,10 @@ def _pool_runner_worker(initializer, conn): `conn`. Args: - initializer: A callable to execute during startup. - conn: A multiprocessing.Connection object to listen for tasks and send + task_type: the task type. + task_id: the task index. + initializer: a callable to execute during startup. + conn: a multiprocessing.Connection object to listen for tasks and send results. """ if initializer: @@ -840,22 +955,24 @@ def _pool_runner_worker(initializer, conn): except EOFError: break proc_func = dill.loads(proc_func) - info = _run_contained(proc_func, args, kwargs) + info = _run_contained(task_type, task_id, proc_func, args, kwargs) sys.stdout.flush() sys.stderr.flush() conn.send(info) -def _run_contained(proc_func, args, kwargs): +def _run_contained(task_type, task_id, proc_func, args, kwargs): """Runs `proc_func` with `args` and `kwargs`. The function returns _ProcessStatusInfo which captures the return value and the exception. Args: - proc_func: The function to be run. - args: Optional positional arguments to be supplied in `proc_func`. - kwargs: Optional keyword arguments to be supplied in `proc_func`. + task_type: the task type. + task_id: the task index. + proc_func: the function to be run. + args: optional positional arguments to be supplied in `proc_func`. + kwargs: optional keyword arguments to be supplied in `proc_func`. Returns: a _ProcessStatusInfo. @@ -868,6 +985,8 @@ def _run_contained(proc_func, args, kwargs): return_value = proc_func(*args, **kwargs) is_successful = True return _ProcessStatusInfo( + task_type=task_type, + task_id=task_id, is_successful=is_successful, exc_info=exc_info, return_value=return_value) @@ -877,6 +996,8 @@ def _run_contained(proc_func, args, kwargs): except Exception: # pylint: disable=broad-except exc_info = sys.exc_info() return _ProcessStatusInfo( + task_type=task_type, + task_id=task_id, is_successful=is_successful, exc_info=exc_info, return_value=return_value) diff --git a/tensorflow/python/distribute/multi_process_runner_test.py b/tensorflow/python/distribute/multi_process_runner_test.py index c6266a5be26..0aa214d3ca4 100644 --- a/tensorflow/python/distribute/multi_process_runner_test.py +++ b/tensorflow/python/distribute/multi_process_runner_test.py @@ -156,11 +156,8 @@ class MultiProcessRunnerTest(test.TestCase): mpr.start() time.sleep(5) mpr.terminate('worker', 0) - with self.assertRaises( - multi_process_runner.UnexpectedSubprocessExitError) as cm: - mpr.join() - std_stream_results = cm.exception.mpr_result.stdout + std_stream_results = mpr.join().stdout # Worker 0 is terminated in the middle, so it should not have iteration 9 # printed. @@ -388,6 +385,99 @@ class MultiProcessRunnerTest(test.TestCase): 'Subprocess worker-0 exited with exit code 10'): mpr.join() + def test_auto_restart(self): + + def proc_func(counter): + counter.value += 1 + if counter.value == 1: + raise ValueError + + manager = multi_process_runner.manager() + counter = manager.Value(int, 0) + mpr = multi_process_runner.MultiProcessRunner( + proc_func, + multi_worker_test_base.create_cluster_spec(num_workers=1), + args=(counter,), + auto_restart=True) + mpr.start() + mpr.join() + self.assertEqual(counter.value, 2) + + def test_auto_restart_and_timeout(self): + + def proc_func(): + time.sleep(1) + raise ValueError + + mpr = multi_process_runner.MultiProcessRunner( + proc_func, + multi_worker_test_base.create_cluster_spec(num_workers=1), + auto_restart=True) + mpr.start() + with self.assertRaises(multi_process_runner.SubprocessTimeoutError): + mpr.join(timeout=10) + + def test_auto_restart_and_chief(self): + # If the chief has exited with zero exit code, auto restart should stop + # restarting other tasks even if they fail. + + def proc_func(): + time.sleep(1) + if multi_worker_test_base.get_task_type() != 'chief': + raise ValueError + + manager = multi_process_runner.manager() + mpr = multi_process_runner.MultiProcessRunner( + proc_func, + multi_worker_test_base.create_cluster_spec( + has_chief=True, num_workers=1), + auto_restart=True) + mpr.start() + with self.assertRaises(ValueError): + mpr.join(timeout=10) + + def test_auto_restart_failure_immediate_after_restart(self): + # Test the case when worker-0 fails immediately after worker-1 restarts. + + def proc_func(): + time.sleep(5) + + mpr = multi_process_runner.MultiProcessRunner( + proc_func, + multi_worker_test_base.create_cluster_spec( + has_chief=False, num_workers=2), + auto_restart=True) + mpr.start() + pid = mpr.get_process_id('worker', 1) + mpr.terminate('worker', 1) + while mpr.get_process_id('worker', 1) == pid: + time.sleep(0.1) + mpr.terminate('worker', 0) + mpr.join(timeout=20) + + def test_auto_restart_terminate(self): + # Tasks terminated by the user should also be restarted. + + def proc_func(counter): + counter.value += 1 + if counter.value == 1: + time.sleep(100) + + manager = multi_process_runner.manager() + counter = manager.Value(int, 0) + + mpr = multi_process_runner.MultiProcessRunner( + proc_func, + multi_worker_test_base.create_cluster_spec( + has_chief=False, num_workers=1), + args=(counter,), + auto_restart=True) + mpr.start() + time.sleep(3) + mpr.terminate('worker', 0) + mpr.join(timeout=20) + self.assertEqual(counter.value, 2) + class MultiProcessPoolRunnerTest(test.TestCase): From 6dbc50195e516d6d1144c6eb29184d75e2a1da1e Mon Sep 17 00:00:00 2001 From: Robert David Date: Mon, 3 Aug 2020 14:22:57 -0700 Subject: [PATCH 0282/1017] Automatic readability finding fixes. PiperOrigin-RevId: 324682656 Change-Id: I009a949fbf08eeb5e45d19d8a1918a988960311b --- .../lite/delegates/gpu/gl/object_manager.cc | 11 +++++------ .../delegates/gpu/metal/kernels/elementwise.cc | 2 +- .../delegates/gpu/metal/kernels/max_unpooling.cc | 16 +++++++--------- .../lite/delegates/gpu/metal/kernels/mean.cc | 3 +-- .../lite/delegates/gpu/metal/kernels/winograd.cc | 3 +-- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/gl/object_manager.cc b/tensorflow/lite/delegates/gpu/gl/object_manager.cc index c37be507b2b..ba48b7323a9 100644 --- a/tensorflow/lite/delegates/gpu/gl/object_manager.cc +++ b/tensorflow/lite/delegates/gpu/gl/object_manager.cc @@ -40,12 +40,11 @@ absl::Status CreatePHWC4BufferFromTensorRef(const TensorRef& tensor_ref, absl::Status CopyFromPHWC4Buffer(const GlBuffer& buffer, TensorFloat32* tensor) { - return buffer.MappedRead( - [tensor, &buffer](absl::Span data) { - tensor->data.resize(tensor->shape.DimensionsProduct()); - return ConvertFromPHWC4(absl::MakeConstSpan(data), tensor->shape, - absl::MakeSpan(tensor->data)); - }); + return buffer.MappedRead([tensor](absl::Span data) { + tensor->data.resize(tensor->shape.DimensionsProduct()); + return ConvertFromPHWC4(absl::MakeConstSpan(data), tensor->shape, + absl::MakeSpan(tensor->data)); + }); } absl::Status ObjectManager::RegisterBuffer(uint32_t id, GlBuffer buffer) { diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/elementwise.cc b/tensorflow/lite/delegates/gpu/metal/kernels/elementwise.cc index 7bac1402fd2..53c1c5b38dd 100644 --- a/tensorflow/lite/delegates/gpu/metal/kernels/elementwise.cc +++ b/tensorflow/lite/delegates/gpu/metal/kernels/elementwise.cc @@ -115,7 +115,7 @@ std::vector ElementwiseWithTwoInputs( desc->uniform_buffers = { {"constant int2&", - [input_ids, output_id](const std::map& buffers) { + [input_ids](const std::map& buffers) { const auto& input_dim_1 = buffers.find(input_ids[1])->second; std::vector uniform_params{ input_dim_1.w, diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/max_unpooling.cc b/tensorflow/lite/delegates/gpu/metal/kernels/max_unpooling.cc index d0e326baf2c..39b4c8fde0e 100644 --- a/tensorflow/lite/delegates/gpu/metal/kernels/max_unpooling.cc +++ b/tensorflow/lite/delegates/gpu/metal/kernels/max_unpooling.cc @@ -99,17 +99,15 @@ std::vector MaxUnpooling( {input_indices_id, "device FLT4* const src_indices_buffer"}, }; - desc->output_buffer = {output_id, "device FLT4* output_buffer", - [input_id, input_indices_id, - params](const std::map& buffers) { - return CalculateOutputShape( - buffers.find(input_id)->second, params); - }}; + desc->output_buffer = { + output_id, "device FLT4* output_buffer", + [input_id, params](const std::map& buffers) { + return CalculateOutputShape(buffers.find(input_id)->second, params); + }}; desc->uniform_buffers = { {"constant uniforms& params", - [input_id, input_indices_id, output_id, - params](const std::map& buffers) { + [input_id, output_id, params](const std::map& buffers) { const auto& dimension = buffers.find(input_id)->second; const auto& output_dimension = buffers.find(output_id)->second; std::vector uniform_params{ @@ -126,7 +124,7 @@ std::vector MaxUnpooling( }}, }; - desc->resize_function = [input_id, input_indices_id, + desc->resize_function = [input_id, params](const std::map& buffers) { const auto& src_shape = buffers.find(input_id)->second; BHWC dst_shape = CalculateOutputShape(src_shape, params); diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/mean.cc b/tensorflow/lite/delegates/gpu/metal/kernels/mean.cc index 431b1e5d6db..d67c9e7f275 100644 --- a/tensorflow/lite/delegates/gpu/metal/kernels/mean.cc +++ b/tensorflow/lite/delegates/gpu/metal/kernels/mean.cc @@ -130,8 +130,7 @@ std::vector Mean(int id, ValueId input_id, }}; desc->uniform_buffers = { {"constant uniforms& params", - [input_id, output_id, - work_group_size](const std::map& buffers) { + [input_id, work_group_size](const std::map& buffers) { const auto& src_shape = buffers.find(input_id)->second; const int src_slices = DivideRoundUp(src_shape.c, 4); struct uniforms { diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/winograd.cc b/tensorflow/lite/delegates/gpu/metal/kernels/winograd.cc index 2098155888d..d62c6a7fcbe 100644 --- a/tensorflow/lite/delegates/gpu/metal/kernels/winograd.cc +++ b/tensorflow/lite/delegates/gpu/metal/kernels/winograd.cc @@ -613,8 +613,7 @@ std::vector Winograd4x4To36TileX6( }}, }; - desc->resize_function = [output_id, - attr](const std::map& buffers) { + desc->resize_function = [output_id](const std::map& buffers) { const uint3 groups_size{4, 6, 1}; const auto& dst_shape = buffers.find(output_id)->second; int grid_x = dst_shape.w; From a4ff9ed01fa5299f17c560fbe05891e31f5e71e5 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Mon, 3 Aug 2020 14:34:06 -0700 Subject: [PATCH 0283/1017] Add CI job name to sizetracker table schema PiperOrigin-RevId: 324684951 Change-Id: Iab9f8fdf6c54f9001beeeaeaee87156abcab93f5 --- tensorflow/tools/ci_build/sizetrack_helper.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tensorflow/tools/ci_build/sizetrack_helper.py b/tensorflow/tools/ci_build/sizetrack_helper.py index 675a16d9a97..032dbdf7490 100755 --- a/tensorflow/tools/ci_build/sizetrack_helper.py +++ b/tensorflow/tools/ci_build/sizetrack_helper.py @@ -52,6 +52,7 @@ from __future__ import print_function import argparse import csv import datetime +import os import os.path import platform import subprocess @@ -92,6 +93,10 @@ parser.add_argument( "--dry_run", action="store_true", help="Dry run: do not load to BigQuery or upload to GCS.") +parser.add_argument( + "--job", + type=str, + help="Name of job calling this script. Default: $KOKORO_JOB_NAME.") parser.add_argument( "--print_schema", action="store_true", @@ -140,6 +145,7 @@ SCHEMA = ",".join([ "team:string", "logged_date:timestamp", "uploaded_to:string", + "job:string", ]) # Select the earliest recorded commit in the same table for the same artifact # and team. Used to determine the full range of tested commits for each @@ -313,6 +319,7 @@ def build_row(): FLAGS.team, current_time, get_upload_path(), + FLAGS.job, ] @@ -330,6 +337,9 @@ def main(): "\nPass -h or --help for usage.") exit(1) + if not FLAGS.job: + FLAGS.job = os.environ.get("KOKORO_JOB_NAME", "NO_JOB") + # Generate data about this artifact into a Tab Separated Value file next_tsv_row = build_row() From d84acd6e45d5c33743d032885e4f5ee727f57db8 Mon Sep 17 00:00:00 2001 From: Anjali Sridhar Date: Mon, 3 Aug 2020 14:43:36 -0700 Subject: [PATCH 0284/1017] Remove unused symbols in vars_test. PiperOrigin-RevId: 324686959 Change-Id: If5a8d2ccf6d4baa4e1f19d83a2d54d359c6e6514 --- tensorflow/python/distribute/vars_test.py | 28 ----------------------- 1 file changed, 28 deletions(-) diff --git a/tensorflow/python/distribute/vars_test.py b/tensorflow/python/distribute/vars_test.py index 98d0c1bb2d2..efbb6c23aaa 100644 --- a/tensorflow/python/distribute/vars_test.py +++ b/tensorflow/python/distribute/vars_test.py @@ -26,7 +26,6 @@ from tensorflow.python.distribute import combinations from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import strategy_combinations from tensorflow.python.distribute import tpu_strategy -from tensorflow.python.distribute import tpu_values from tensorflow.python.distribute import values from tensorflow.python.distribute.cluster_resolver import tpu_cluster_resolver from tensorflow.python.eager import context @@ -664,26 +663,6 @@ class OnWriteVariableSyncScatterTests(test.TestCase, parameterized.TestCase): self.assertAllEqual([1, 1, 1], self.evaluate(v2.read_value())) -def _make_replica_local(method, strategy=None): - if strategy is None: - devices = ("/device:GPU:0", "/device:CPU:0") - else: - devices = strategy.extended.worker_devices - - v = [] - for d, n, init in zip(devices, ["v", "v/replica"], [1., 2.]): - with ops.device(d): - v.append(variable_scope.get_variable( - name=n, initializer=init, use_resource=True)) - - if (strategy is not None) and isinstance(strategy, _TPU_STRATEGIES): - var_cls = tpu_values.TPUSyncOnReadVariable - else: - var_cls = values.SyncOnReadVariable - replica_local = var_cls(strategy, v, method) - return v, replica_local - - class OnReadVariableSyncTest(test.TestCase, parameterized.TestCase): @combinations.generate(strategy_and_run_tf_function_combinations()) @@ -1258,12 +1237,5 @@ class SyncOnReadScatterReplicaTest(test.TestCase, parameterized.TestCase): self.evaluate(distribution.run(v.scatter_min, args=(delta,))) -def _make_index_slices(vals, indices, dense_shape=None): - if dense_shape: - dense_shape = array_ops.identity(dense_shape) - return indexed_slices.IndexedSlices( - array_ops.identity(vals), array_ops.identity(indices), dense_shape) - - if __name__ == "__main__": test.main() From 64c753b9ee94b11493abfceefd234fa6b8497f71 Mon Sep 17 00:00:00 2001 From: Tomer Kaftan Date: Mon, 3 Aug 2020 14:49:34 -0700 Subject: [PATCH 0285/1017] When a non-KerasTensor Tensor is passed to the `tensor` argument of keras.Input or InputLayer, make a KerasTensor directly from that tensor rather than erroring out. PiperOrigin-RevId: 324688220 Change-Id: I2b06682f8ea706be4e36e0b8807c0f07bec55a4e --- tensorflow/python/keras/engine/BUILD | 17 ++ tensorflow/python/keras/engine/functional.py | 5 +- tensorflow/python/keras/engine/input_layer.py | 24 ++- .../python/keras/engine/input_layer_test.py | 148 ++++++++++++++++++ 4 files changed, 179 insertions(+), 15 deletions(-) create mode 100644 tensorflow/python/keras/engine/input_layer_test.py diff --git a/tensorflow/python/keras/engine/BUILD b/tensorflow/python/keras/engine/BUILD index c71069b3657..0d2ddb46049 100644 --- a/tensorflow/python/keras/engine/BUILD +++ b/tensorflow/python/keras/engine/BUILD @@ -545,6 +545,23 @@ tf_py_test( ], ) +tf_py_test( + name = "input_layer_test", + size = "medium", + srcs = ["input_layer_test.py"], + python_version = "PY3", + shard_count = 3, + tags = [ + "nomac", # TODO(mihaimaruseac): b/127695564 + ], + deps = [ + ":base_layer", + ":engine", + "//tensorflow/python/keras:testing_utils", + "//tensorflow/python/keras/utils:layer_utils", + ], +) + tf_py_test( name = "functional_test", size = "medium", diff --git a/tensorflow/python/keras/engine/functional.py b/tensorflow/python/keras/engine/functional.py index 71d6faa71b6..8422bf923d8 100644 --- a/tensorflow/python/keras/engine/functional.py +++ b/tensorflow/python/keras/engine/functional.py @@ -135,8 +135,9 @@ class Functional(training_lib.Model): (isinstance(self._nested_inputs, (list, tuple, dict)) and not any(nest.is_nested(t) for t in self._nested_inputs))) - if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs): - base_layer_utils.create_keras_history(self._nested_outputs) + if not keras_tensor.keras_tensors_enabled(): + if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs): + base_layer_utils.create_keras_history(self._nested_outputs) self._validate_graph_inputs_and_outputs() diff --git a/tensorflow/python/keras/engine/input_layer.py b/tensorflow/python/keras/engine/input_layer.py index 4818c5c59a7..33f9320e516 100644 --- a/tensorflow/python/keras/engine/input_layer.py +++ b/tensorflow/python/keras/engine/input_layer.py @@ -76,8 +76,9 @@ class InputLayer(base_layer.Layer): batch_size: Optional input batch size (integer or None). dtype: Optional datatype of the input. When not provided, the Keras default float type will be used. - input_tensor: Optional tensor to use as layer input - instead of creating a placeholder. + input_tensor: Optional tensor to use as layer input. If set, the layer + will use the `tf.TypeSpec` of this tensor rather + than creating a new placeholder tensor. sparse: Boolean, whether the placeholder created is meant to be sparse. Default to False. ragged: Boolean, whether the placeholder created is meant to be ragged. @@ -162,19 +163,15 @@ class InputLayer(base_layer.Layer): self.is_placeholder = True self._batch_input_shape = batch_input_shape else: - raise_eager_tensor_error = False if keras_tensor.keras_tensors_enabled(): - if (not isinstance(input_tensor, keras_tensor.KerasTensor) and - not tf_utils.is_symbolic_tensor(input_tensor)): - raise_eager_tensor_error = True + if not isinstance(input_tensor, keras_tensor.KerasTensor): + input_tensor = keras_tensor.keras_tensor_from_tensor(input_tensor) else: if not tf_utils.is_symbolic_tensor(input_tensor): - raise_eager_tensor_error = True - if raise_eager_tensor_error: - raise ValueError('You should not pass an EagerTensor to `Input`. ' - 'For example, instead of creating an ' - 'InputLayer, you should instantiate your model and ' - 'directly call it on your input.') + raise ValueError('You should not pass an EagerTensor to `Input`. ' + 'For example, instead of creating an ' + 'InputLayer, you should instantiate your model and ' + 'directly call it on your input.') self.is_placeholder = False try: self._batch_input_shape = tuple(input_tensor.shape.as_list()) @@ -245,7 +242,8 @@ def Input( # pylint: disable=invalid-name if `sparse` is False, sparse tensors can still be passed into the input - they will be densified with a default value of 0. tensor: Optional existing tensor to wrap into the `Input` layer. - If set, the layer will not create a placeholder tensor. + If set, the layer will use the `tf.TypeSpec` of this tensor rather + than creating a new placeholder tensor. ragged: A boolean specifying whether the placeholder to be created is ragged. Only one of 'ragged' and 'sparse' can be True. In this case, values of 'None' in the 'shape' argument represent ragged dimensions. diff --git a/tensorflow/python/keras/engine/input_layer_test.py b/tensorflow/python/keras/engine/input_layer_test.py new file mode 100644 index 00000000000..1b15f34458c --- /dev/null +++ b/tensorflow/python/keras/engine/input_layer_test.py @@ -0,0 +1,148 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#,============================================================================ +"""Tests for InputLayer construction.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from tensorflow.python.eager import def_function +from tensorflow.python.keras import combinations +from tensorflow.python.keras import keras_parameterized +from tensorflow.python.keras import testing_utils +from tensorflow.python.keras.engine import functional +from tensorflow.python.keras.engine import input_layer as input_layer_lib +from tensorflow.python.ops import array_ops +from tensorflow.python.ops.ragged import ragged_tensor +from tensorflow.python.platform import test + + +class InputLayerTest(keras_parameterized.TestCase): + + @combinations.generate(combinations.combine(mode=['graph', 'eager'])) + def testBasicOutputShapeNoBatchSize(self): + # Create a Keras Input + x = input_layer_lib.Input(shape=(32,), name='input_a') + self.assertAllEqual(x.shape.as_list(), [None, 32]) + + # Verify you can construct and use a model w/ this input + model = functional.Functional(x, x * 2.0) + self.assertAllEqual(model(array_ops.ones((3, 32))), + array_ops.ones((3, 32)) * 2.0) + + @combinations.generate(combinations.combine(mode=['graph', 'eager'])) + def testBasicOutputShapeWithBatchSize(self): + # Create a Keras Input + x = input_layer_lib.Input(batch_size=6, shape=(32,), name='input_b') + self.assertAllEqual(x.shape.as_list(), [6, 32]) + + # Verify you can construct and use a model w/ this input + model = functional.Functional(x, x * 2.0) + self.assertAllEqual(model(array_ops.ones(x.shape)), + array_ops.ones(x.shape) * 2.0) + + @combinations.generate(combinations.combine(mode=['eager'])) + def testBasicOutputShapeNoBatchSizeInTFFunction(self): + model = None + @def_function.function + def run_model(inp): + nonlocal model + if not model: + # Create a Keras Input + x = input_layer_lib.Input(shape=(8,), name='input_a') + self.assertAllEqual(x.shape.as_list(), [None, 8]) + + # Verify you can construct and use a model w/ this input + model = functional.Functional(x, x * 2.0) + return model(inp) + + self.assertAllEqual(run_model(array_ops.ones((10, 8))), + array_ops.ones((10, 8)) * 2.0) + + @combinations.generate(combinations.combine(mode=['graph', 'eager'])) + def testInputTensorArg(self): + with testing_utils.use_keras_tensors_scope(True): + # Create a Keras Input + x = input_layer_lib.Input(tensor=array_ops.zeros((7, 32))) + self.assertAllEqual(x.shape.as_list(), [7, 32]) + + # Verify you can construct and use a model w/ this input + model = functional.Functional(x, x * 2.0) + self.assertAllEqual(model(array_ops.ones(x.shape)), + array_ops.ones(x.shape) * 2.0) + + @combinations.generate(combinations.combine(mode=['eager'])) + def testInputTensorArgInTFFunction(self): + with testing_utils.use_keras_tensors_scope(True): + # We use a mutable model container instead of a model python variable, + # because python 2.7 does not have `nonlocal` + model_container = {} + + @def_function.function + def run_model(inp): + if not model_container: + # Create a Keras Input + x = input_layer_lib.Input(tensor=array_ops.zeros((10, 16))) + self.assertAllEqual(x.shape.as_list(), [10, 16]) + + # Verify you can construct and use a model w/ this input + model_container['model'] = functional.Functional(x, x * 3.0) + return model_container['model'](inp) + + self.assertAllEqual(run_model(array_ops.ones((10, 16))), + array_ops.ones((10, 16)) * 3.0) + + @combinations.generate(combinations.combine(mode=['eager'])) + def testCompositeInputTensorArg(self): + with testing_utils.use_keras_tensors_scope(True): + # Create a Keras Input + rt = ragged_tensor.RaggedTensor.from_row_splits( + values=[3, 1, 4, 1, 5, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8]) + x = input_layer_lib.Input(tensor=rt) + + # Verify you can construct and use a model w/ this input + model = functional.Functional(x, x * 2) + + # And that the model works + rt = ragged_tensor.RaggedTensor.from_row_splits( + values=[3, 21, 4, 1, 53, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8]) + self.assertAllEqual(model(rt), rt * 2) + + @combinations.generate(combinations.combine(mode=['eager'])) + def testCompositeInputTensorArgInTFFunction(self): + with testing_utils.use_keras_tensors_scope(True): + # We use a mutable model container instead of a model python variable, + # because python 2.7 does not have `nonlocal` + model_container = {} + + @def_function.function + def run_model(inp): + if not model_container: + # Create a Keras Input + rt = ragged_tensor.RaggedTensor.from_row_splits( + values=[3, 1, 4, 1, 5, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8]) + x = input_layer_lib.Input(tensor=rt) + + # Verify you can construct and use a model w/ this input + model_container['model'] = functional.Functional(x, x * 3) + return model_container['model'](inp) + + # And verify the model works + rt = ragged_tensor.RaggedTensor.from_row_splits( + values=[3, 21, 4, 1, 53, 9, 2, 6], row_splits=[0, 4, 4, 7, 8, 8]) + self.assertAllEqual(run_model(rt), rt * 3) + +if __name__ == '__main__': + test.main() From 3d273680d5428cc7426b78f129f4d69ba2dd0bb2 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Mon, 3 Aug 2020 14:38:10 -0700 Subject: [PATCH 0286/1017] Adding a macro for exposing base class methods to FileSystem derived classes to workaround name-hiding issue. --- tensorflow/core/platform/file_system.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h index b2086b5968e..77ba34bcdc6 100644 --- a/tensorflow/core/platform/file_system.h +++ b/tensorflow/core/platform/file_system.h @@ -691,6 +691,32 @@ class WrappedFileSystem : public FileSystem { TransactionToken* token_; }; +/// This macro adds forwarding methods from FileSystem class to +/// used class since name hiding will prevent these to be accessed from +/// derived classes and would require all use locations to migrate to +/// Transactional API. This is an interim solution until ModularFileSystem class +/// becomes a singleton. +// TODO(sami): Remove this macro when filesystem plugins migration is complete. + +#define TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT \ + using FileSystem::NewRandomAccessFile; \ + using FileSystem::NewWritableFile; \ + using FileSystem::NewAppendableFile; \ + using FileSystem::NewReadOnlyMemoryRegionFromFile; \ + using FileSystem::FileExists; \ + using FileSystem::GetChildren; \ + using FileSystem::GetMatchingPaths; \ + using FileSystem::Stat; \ + using FileSystem::DeleteFile; \ + using FileSystem::RecursivelyCreateDir; \ + using FileSystem::DeleteDir; \ + using FileSystem::DeleteRecursively; \ + using FileSystem::GetFileSize; \ + using FileSystem::RenameFile; \ + using FileSystem::CopyFile; \ + using FileSystem::IsDirectory; \ + using FileSystem::FlushCaches + /// A file abstraction for randomly reading the contents of a file. class RandomAccessFile { public: From 2e680a5dde23f242ee1cb45a627898e3746b877d Mon Sep 17 00:00:00 2001 From: Jose Baiocchi Date: Mon, 3 Aug 2020 14:52:03 -0700 Subject: [PATCH 0287/1017] Remove ProfilerSession::Create with no arguments PiperOrigin-RevId: 324688779 Change-Id: Ia285b30de66e04ad3e6286c315a4563420287cf8 --- tensorflow/core/distributed_runtime/worker.cc | 4 +++- tensorflow/core/profiler/lib/BUILD | 1 - .../core/profiler/lib/profiler_session.cc | 22 +++++-------------- .../core/profiler/lib/profiler_session.h | 3 +-- 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/tensorflow/core/distributed_runtime/worker.cc b/tensorflow/core/distributed_runtime/worker.cc index 5212f51d491..c4dc51ce47d 100644 --- a/tensorflow/core/distributed_runtime/worker.cc +++ b/tensorflow/core/distributed_runtime/worker.cc @@ -198,7 +198,9 @@ void Worker::DoRunGraph(CallOptions* opts, RunGraphRequestWrapper* request, ProfilerSession* profiler_session = nullptr; if (collector && request->exec_opts().record_timeline()) { // If timeline was requested, assume we want hardware level tracing. - profiler_session = ProfilerSession::Create().release(); + ProfileOptions options = ProfilerSession::DefaultOptions(); + options.set_host_tracer_level(0); + profiler_session = ProfilerSession::Create(options).release(); } CancellationManager* cm = new CancellationManager; opts->SetCancelCallback([this, cm, step_id]() { diff --git a/tensorflow/core/profiler/lib/BUILD b/tensorflow/core/profiler/lib/BUILD index 0f92ffd5a70..57a3fa8a586 100644 --- a/tensorflow/core/profiler/lib/BUILD +++ b/tensorflow/core/profiler/lib/BUILD @@ -46,7 +46,6 @@ cc_library( ], deps = [ "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", "//tensorflow/core/platform", "//tensorflow/core/profiler/internal:profiler_interface", "//tensorflow/core/profiler/protobuf:xplane_proto_cc", diff --git a/tensorflow/core/profiler/lib/profiler_session.cc b/tensorflow/core/profiler/lib/profiler_session.cc index 90857ea8b51..ee6eb55300e 100644 --- a/tensorflow/core/profiler/lib/profiler_session.cc +++ b/tensorflow/core/profiler/lib/profiler_session.cc @@ -29,7 +29,6 @@ limitations under the License. #include "tensorflow/core/profiler/protobuf/xplane.pb.h" #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/protobuf/error_codes.pb.h" -#include "tensorflow/core/util/env_var.h" #if !defined(IS_MOBILE_PLATFORM) #include "tensorflow/core/profiler/internal/profiler_factory.h" @@ -41,31 +40,20 @@ limitations under the License. #endif namespace tensorflow { - namespace { + ProfileOptions GetOptions(const ProfileOptions& opts) { if (opts.version()) return opts; ProfileOptions options = ProfilerSession::DefaultOptions(); options.set_include_dataset_ops(opts.include_dataset_ops()); return options; } + }; // namespace /*static*/ std::unique_ptr ProfilerSession::Create( const ProfileOptions& options) { - return absl::WrapUnique(new ProfilerSession(options)); -} - -/*static*/ std::unique_ptr ProfilerSession::Create() { - int64 host_tracer_level = 2; - tensorflow::Status s = ReadInt64FromEnvVar("TF_PROFILER_HOST_TRACER_LEVEL", 2, - &host_tracer_level); - if (!s.ok()) { - LOG(WARNING) << "ProfilerSession: " << s.error_message(); - } - ProfileOptions options = DefaultOptions(); - options.set_host_tracer_level(host_tracer_level); - return Create(options); + return absl::WrapUnique(new ProfilerSession(GetOptions(options))); } tensorflow::Status ProfilerSession::Status() { @@ -141,14 +129,14 @@ Status ProfilerSession::CollectData(RunMetadata* run_metadata) { return Status::OK(); } -ProfilerSession::ProfilerSession(const ProfileOptions& options) +ProfilerSession::ProfilerSession(ProfileOptions options) #if !defined(IS_MOBILE_PLATFORM) : active_(profiler::AcquireProfilerLock()), #else : active_(false), #endif start_time_ns_(EnvTime::NowNanos()), - options_(GetOptions(options)) { + options_(std::move(options)) { if (!active_) { #if !defined(IS_MOBILE_PLATFORM) status_ = tensorflow::Status(error::UNAVAILABLE, diff --git a/tensorflow/core/profiler/lib/profiler_session.h b/tensorflow/core/profiler/lib/profiler_session.h index 6f92b047eb7..93541f501ce 100644 --- a/tensorflow/core/profiler/lib/profiler_session.h +++ b/tensorflow/core/profiler/lib/profiler_session.h @@ -40,7 +40,6 @@ class ProfilerSession { public: // Creates and ProfilerSession and starts profiling. static std::unique_ptr Create(const ProfileOptions& options); - static std::unique_ptr Create(); static ProfileOptions DefaultOptions() { ProfileOptions options; @@ -67,7 +66,7 @@ class ProfilerSession { private: // Constructs an instance of the class and starts profiling - explicit ProfilerSession(const ProfileOptions& options); + explicit ProfilerSession(ProfileOptions options); // ProfilerSession is neither copyable or movable. ProfilerSession(const ProfilerSession&) = delete; From fcb71ce45b544ff8702b8faca84a763b3c2ebb57 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 15:04:48 -0700 Subject: [PATCH 0288/1017] Added beta parameter (as in https://research.google.com/pubs/archive/41159.pdf) to FTRL implementation for TPU embeddings. PiperOrigin-RevId: 324691573 Change-Id: I6d41c7d631e034ad95e12c6b1a3c24d4482e9171 --- RELEASE.md | 3 +++ .../core/protobuf/tpu/optimization_parameters.proto | 12 ++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 0c6b0f556e8..b0c785c7d68 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -111,6 +111,9 @@ * Math and Linear Algebra: * * TPU Enhancements: + * Added support for the `beta` parameter of the FTRL optimizer for TPU + embeddings. Users of other TensorFlow platforms can implement equivalent + behavior by adjusting the `l2` parameter. * * XLA Support: * diff --git a/tensorflow/core/protobuf/tpu/optimization_parameters.proto b/tensorflow/core/protobuf/tpu/optimization_parameters.proto index f29beb3bc48..53905a33a3b 100644 --- a/tensorflow/core/protobuf/tpu/optimization_parameters.proto +++ b/tensorflow/core/protobuf/tpu/optimization_parameters.proto @@ -85,10 +85,13 @@ message StochasticGradientDescentParameters {} // https://github.com/tensorflow/tensorflow/blob/6b6471f3ffb7f1fefe42d814aa5fb9ab7a535b58/tensorflow/core/kernels/training_ops.cc#L2646 // // The hyperparameters for FTRL are the same as for the Keras implementation, -// with some additions. When the multiply_linear_by_lr field is set to true, a -// modified formula is used for FTRL that treats the "linear" accumulator as -// being pre-multiplied by the learning rate (i.e., the accumulator named -// "linear" actually stores "linear * learning_rate"). Other than checkpoint +// with some additions. The "beta" parameter matches the behavior described in +// the second link above; "beta" / (2 * learning rate) should be added to "l2" +// to get equivalent behavior in the other TensorFlow implementations of this +// optimizer. When the multiply_linear_by_lr field is set to true, a modified +// formula is used for FTRL that treats the "linear" accumulator as being +// pre-multiplied by the learning rate (i.e., the accumulator named "linear" +// actually stores "linear * learning_rate"). Other than checkpoint // compatibility, this is mathematically equivalent for a static learning rate; // for a dynamic learning rate, it is nearly the same as long as the learning // rate does not change quickly. The benefit of setting multiply_linear_by_lr to @@ -98,6 +101,7 @@ message FtrlParameters { float l1 = 1; float l2 = 2; float lr_power = 3; + float beta = 7; bool multiply_linear_by_lr = 6; // Old initial accumulator parameters. From f292f31b57480d0b33f5c0feb5fb128e43c865dc Mon Sep 17 00:00:00 2001 From: Chuanhao Zhuge Date: Mon, 3 Aug 2020 15:22:17 -0700 Subject: [PATCH 0289/1017] Disabling benchmarkScanDefun for TFRT due to lack of MLIR lowering support. PiperOrigin-RevId: 324694962 Change-Id: I2398161dff9403ac115a031c5942f753daff7871 --- tensorflow/python/eager/benchmarks_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py index d8d331a8fc7..93766d809f2 100644 --- a/tensorflow/python/eager/benchmarks_test.py +++ b/tensorflow/python/eager/benchmarks_test.py @@ -1260,6 +1260,8 @@ class MicroBenchmarks(benchmarks_test_base.MicroBenchmarksBase): self._run(scan, 100) + @test_util.disable_tfrt( + "tf.While not supported in TF to CoreRT lowing. b/162685874") def benchmarkScanDefun(self): elems = math_ops.range(1600) From f18d09553b2f26a07b0b5cd2ee96f68834fd3c10 Mon Sep 17 00:00:00 2001 From: Jiho Choi Date: Mon, 3 Aug 2020 15:31:34 -0700 Subject: [PATCH 0290/1017] Add element tracing for tf.data.experimental.parallel_interleave. PiperOrigin-RevId: 324696858 Change-Id: I099b9b8935a38e263bd24f008e123c0623432e40 --- .../core/kernels/data/experimental/BUILD | 2 ++ .../parallel_interleave_dataset_op.cc | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/kernels/data/experimental/BUILD b/tensorflow/core/kernels/data/experimental/BUILD index d2142ee69d1..bf28d175e6d 100644 --- a/tensorflow/core/kernels/data/experimental/BUILD +++ b/tensorflow/core/kernels/data/experimental/BUILD @@ -394,6 +394,8 @@ tf_kernel_library( "//tensorflow/core/kernels/data:captured_function", "//tensorflow/core/kernels/data:dataset_utils", "//tensorflow/core/kernels/data:name_utils", + "//tensorflow/core/profiler/lib:traceme", + "//tensorflow/core/profiler/lib:traceme_encode", ], ) diff --git a/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc index 2167c5d9b98..9c344e01c6a 100644 --- a/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/parallel_interleave_dataset_op.cc @@ -31,6 +31,8 @@ limitations under the License. #include "tensorflow/core/lib/random/random.h" #include "tensorflow/core/platform/blocking_counter.h" #include "tensorflow/core/platform/stringprintf.h" +#include "tensorflow/core/profiler/lib/traceme.h" +#include "tensorflow/core/profiler/lib/traceme_encode.h" namespace tensorflow { namespace data { @@ -323,6 +325,11 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { } *end_of_sequence = false; Status s = current_worker->outputs.front().status; + profiler::TraceMe traceme([&] { + return profiler::TraceMeEncode( + "ParallelInterleaveConsume", + {{"element_id", current_worker->outputs.front().id}}); + }); current_worker->outputs.front().output.swap(*out_tensors); current_worker->outputs.pop_front(); current_worker->cond_var.notify_one(); @@ -564,8 +571,10 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { Status status; // The buffered data element. std::vector output; + int64 id = -1; explicit OutputElem(const Status& s) : status(s) {} + OutputElem(const Status& s, int64 id) : status(s), id(id) {} }; // Worker threads operate on their relevant WorkerState structs. @@ -813,6 +822,14 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { worker_thread_states_[thread_index] .output_elem.output.empty() && !worker_thread_states_[thread_index].end_of_sequence) { + int64& id = worker_thread_states_[thread_index].output_elem.id; + profiler::TraceMe traceme( + [&] { + id = profiler::TraceMe::NewActivityId(); + return profiler::TraceMeEncode( + "ParallelInterleaveProduce", {{"element_id", id}}); + }, + profiler::kInfo); worker_thread_states_[thread_index].output_elem.status = worker_thread_states_[thread_index].iterator->GetNext( ctx.get(), @@ -856,7 +873,8 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { worker_thread_states_[thread_index].end_of_sequence = false; } else { workers_[thread_index].outputs.emplace_back( - worker_thread_states_[thread_index].output_elem.status); + worker_thread_states_[thread_index].output_elem.status, + worker_thread_states_[thread_index].output_elem.id); workers_[thread_index].outputs.back().output.swap( worker_thread_states_[thread_index].output_elem.output); } From a0f7e214ae4f79f959a96c106b2379c17ea3b60f Mon Sep 17 00:00:00 2001 From: Zhenyu Tan Date: Mon, 3 Aug 2020 15:34:22 -0700 Subject: [PATCH 0291/1017] Fixit for sequence feature column test. PiperOrigin-RevId: 324697411 Change-Id: Idd0568a80b4b3f82c5c676920a592154efdcc604 --- .../sequence_feature_column_test.py | 114 +++++++++--------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/tensorflow/python/feature_column/sequence_feature_column_test.py b/tensorflow/python/feature_column/sequence_feature_column_test.py index da6d1dee4ba..e98a202bc5a 100644 --- a/tensorflow/python/feature_column/sequence_feature_column_test.py +++ b/tensorflow/python/feature_column/sequence_feature_column_test.py @@ -516,7 +516,6 @@ class SequenceEmbeddingColumnTest( class SequenceSharedEmbeddingColumnTest(test.TestCase): - @test_util.run_deprecated_v1 def test_get_sequence_dense_tensor(self): vocabulary_size = 3 embedding_dimension = 2 @@ -532,67 +531,68 @@ class SequenceSharedEmbeddingColumnTest(test.TestCase): self.assertIsNone(partition_info) return embedding_values - sparse_input_a = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - indices=((0, 0), (1, 0), (1, 1), (3, 0)), - values=(2, 0, 1, 1), - dense_shape=(4, 2)) - sparse_input_b = sparse_tensor.SparseTensorValue( - # example 0, ids [1] - # example 1, ids [0, 2] - # example 2, ids [0] - # example 3, ids [] - indices=((0, 0), (1, 0), (1, 1), (2, 0)), - values=(1, 0, 2, 0), - dense_shape=(4, 2)) + with ops.Graph().as_default(): + sparse_input_a = sparse_tensor.SparseTensorValue( + # example 0, ids [2] + # example 1, ids [0, 1] + # example 2, ids [] + # example 3, ids [1] + indices=((0, 0), (1, 0), (1, 1), (3, 0)), + values=(2, 0, 1, 1), + dense_shape=(4, 2)) + sparse_input_b = sparse_tensor.SparseTensorValue( + # example 0, ids [1] + # example 1, ids [0, 2] + # example 2, ids [0] + # example 3, ids [] + indices=((0, 0), (1, 0), (1, 1), (2, 0)), + values=(1, 0, 2, 0), + dense_shape=(4, 2)) - expected_lookups_a = [ - # example 0, ids [2] - [[7., 11.], [0., 0.]], - # example 1, ids [0, 1] - [[1., 2.], [3., 5.]], - # example 2, ids [] - [[0., 0.], [0., 0.]], - # example 3, ids [1] - [[3., 5.], [0., 0.]], - ] + expected_lookups_a = [ + # example 0, ids [2] + [[7., 11.], [0., 0.]], + # example 1, ids [0, 1] + [[1., 2.], [3., 5.]], + # example 2, ids [] + [[0., 0.], [0., 0.]], + # example 3, ids [1] + [[3., 5.], [0., 0.]], + ] - expected_lookups_b = [ - # example 0, ids [1] - [[3., 5.], [0., 0.]], - # example 1, ids [0, 2] - [[1., 2.], [7., 11.]], - # example 2, ids [0] - [[1., 2.], [0., 0.]], - # example 3, ids [] - [[0., 0.], [0., 0.]], - ] + expected_lookups_b = [ + # example 0, ids [1] + [[3., 5.], [0., 0.]], + # example 1, ids [0, 2] + [[1., 2.], [7., 11.]], + # example 2, ids [0] + [[1., 2.], [0., 0.]], + # example 3, ids [] + [[0., 0.], [0., 0.]], + ] - categorical_column_a = sfc.sequence_categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - categorical_column_b = sfc.sequence_categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - shared_embedding_columns = fc.shared_embedding_columns_v2( - [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, - initializer=_initializer) + categorical_column_a = sfc.sequence_categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + categorical_column_b = sfc.sequence_categorical_column_with_identity( + key='bbb', num_buckets=vocabulary_size) + shared_embedding_columns = fc.shared_embedding_columns_v2( + [categorical_column_a, categorical_column_b], + dimension=embedding_dimension, + initializer=_initializer) - embedding_lookup_a = _get_sequence_dense_tensor( - shared_embedding_columns[0], {'aaa': sparse_input_a})[0] - embedding_lookup_b = _get_sequence_dense_tensor( - shared_embedding_columns[1], {'bbb': sparse_input_b})[0] + embedding_lookup_a = _get_sequence_dense_tensor( + shared_embedding_columns[0], {'aaa': sparse_input_a})[0] + embedding_lookup_b = _get_sequence_dense_tensor( + shared_embedding_columns[1], {'bbb': sparse_input_b})[0] - self.evaluate(variables_lib.global_variables_initializer()) - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertItemsEqual(('aaa_bbb_shared_embedding:0',), - tuple([v.name for v in global_vars])) - self.assertAllEqual(embedding_values, self.evaluate(global_vars[0])) - self.assertAllEqual( - expected_lookups_a, self.evaluate(embedding_lookup_a)) - self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b)) + self.evaluate(variables_lib.global_variables_initializer()) + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + self.assertItemsEqual(('aaa_bbb_shared_embedding:0',), + tuple([v.name for v in global_vars])) + self.assertAllEqual(embedding_values, self.evaluate(global_vars[0])) + self.assertAllEqual( + expected_lookups_a, self.evaluate(embedding_lookup_a)) + self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b)) def test_sequence_length(self): with ops.Graph().as_default(): From fc9c057b1ac7883551e72c833a5429f4bb6dc47a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 15:36:31 -0700 Subject: [PATCH 0292/1017] Adds ParseShardingFromEdgeSource(). Makes DistributedTPURewritePass::AssignArgsAndRetvalsToCores() support TUPLE sharding for return values. PiperOrigin-RevId: 324697874 Change-Id: I3039da1731c9622ebeb0bf9c3b45185e220267af --- tensorflow/compiler/tf2xla/sharding_util.cc | 24 +++++++++++++++++++ tensorflow/compiler/tf2xla/sharding_util.h | 3 +++ .../distributed_tpu_rewrite_pass.cc | 20 ++++++++++------ .../distributed_tpu_rewrite_pass.h | 4 +++- tensorflow/core/tpu/kernels/BUILD | 1 - .../core/tpu/kernels/tpu_compile_op_common.cc | 2 +- 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/tensorflow/compiler/tf2xla/sharding_util.cc b/tensorflow/compiler/tf2xla/sharding_util.cc index 366e8d49228..90585c9d98a 100644 --- a/tensorflow/compiler/tf2xla/sharding_util.cc +++ b/tensorflow/compiler/tf2xla/sharding_util.cc @@ -80,6 +80,30 @@ xla::StatusOr> ParseShardingFromDevice( return ParseShardingFromDevice(device_name, num_cores_per_replica, sharding); } +xla::StatusOr> ParseShardingFromEdgeSource( + const Edge& edge, int num_cores_per_replica) { + if (edge.src() == nullptr) { + return tensorflow::errors::InvalidArgument( + "Null src for ParseShardingFromEdgeSource edge=", edge.DebugString()); + } + TF_ASSIGN_OR_RETURN( + absl::optional sharding, + ParseShardingFromDevice(*edge.src(), num_cores_per_replica)); + if (sharding.has_value() && + sharding.value().type() == xla::OpSharding::TUPLE) { + if (edge.src_output() < 0 || + edge.src_output() >= sharding.value().tuple_shardings_size()) { + return tensorflow::errors::InvalidArgument( + "Tuple index out of bound: edge=", edge.DebugString(), + " sharding=", sharding->DebugString()); + } + absl::optional subsharding = + sharding.value().tuple_shardings(edge.src_output()); + return subsharding; + } + return sharding; +} + void SetShardingDeviceAssignmentFromNode(const Node& src, Node* dst) { string device_name = src.assigned_device_name(); if (device_name.empty()) { diff --git a/tensorflow/compiler/tf2xla/sharding_util.h b/tensorflow/compiler/tf2xla/sharding_util.h index 196434826f9..07657c656d3 100644 --- a/tensorflow/compiler/tf2xla/sharding_util.h +++ b/tensorflow/compiler/tf2xla/sharding_util.h @@ -43,6 +43,9 @@ xla::StatusOr> ParseShardingFromDevice( xla::StatusOr> ParseShardingFromDevice( const NodeDef& node_def, int num_cores_per_replica); +xla::StatusOr> ParseShardingFromEdgeSource( + const Edge& edge, int num_cores_per_replica); + void SetShardingDeviceAssignmentFromNode(const Node& src, Node* dst); // Get sharding inforamtion from node. diff --git a/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.cc b/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.cc index 075a1ec9069..5fdc74b79fc 100644 --- a/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.cc +++ b/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.cc @@ -1813,7 +1813,8 @@ Status DistributedTPURewritePass::AssignArgsAndRetvalsToCores( } else if (sharding->type() != xla::OpSharding::REPLICATED && sharding->type() != xla::OpSharding::OTHER) { return tensorflow::errors::InvalidArgument( - "Unsupported argument sharding: ", sharding->DebugString()); + "Unsupported argument sharding (for arg ", n->DebugString(), + "): ", sharding->DebugString()); } if (assigned_core.has_value()) { args_device_selector.ReportDeviceAssigned(*assigned_core, i); @@ -1855,7 +1856,7 @@ Status DistributedTPURewritePass::AssignArgsAndRetvalsToCores( TF_ASSIGN_OR_RETURN( absl::optional sharding, - ParseShardingFromDevice(*edge->src(), num_cores_per_replica)); + ParseShardingFromEdgeSource(*edge, num_cores_per_replica)); if (partitioned_output_nodes.contains(i)) { Node* output_node = partitioned_output_nodes[i]; @@ -1883,7 +1884,9 @@ Status DistributedTPURewritePass::AssignArgsAndRetvalsToCores( } else if (sharding.value().type() != xla::OpSharding::REPLICATED && sharding.value().type() != xla::OpSharding::OTHER) { return tensorflow::errors::InvalidArgument( - "Unsupported argument sharding: ", sharding->DebugString()); + "Unsupported argument sharding for retval ", + retvals[i]->DebugString(), " edge=", edge->DebugString(), ": ", + sharding->DebugString()); } } else { if (use_spmd) { @@ -2472,7 +2475,8 @@ xla::StatusOr CreateOrGetPerHostVariableCopy( Status DistributedTPURewritePass::BuildExecuteNodes( const ParameterInfo& params_info, int num_tasks, int num_cores_per_replica, - const Node& replicate_node, const DataTypeVector& arg_types, + const Node& replicate_node, const std::vector& arg_names, + const DataTypeVector& arg_types, const std::vector& arg_shapes, const DataTypeVector& retval_types, const std::vector& arg_shardings, @@ -2595,7 +2599,9 @@ Status DistributedTPURewritePass::BuildExecuteNodes( } } else { return tensorflow::errors::InvalidArgument( - "Unsupported argument sharding: ", sharding.DebugString()); + "Unsupported argument sharding for arg=", arg_names[i], + " shape=", arg_shapes[i].shape.DebugString(), ": ", + sharding.DebugString()); } } std::vector> core_retval_nums(num_cores_per_replica); @@ -3922,8 +3928,8 @@ Status DistributedTPURewritePass::FingerprintFunctionLibrary( std::vector variable_writes; TF_RETURN_IF_ERROR(BuildExecuteNodes( - params_info, num_tasks, num_cores_per_replica, *replicate_node, arg_types, - arg_shapes, retval_types, arg_sharding, retval_sharding, + params_info, num_tasks, num_cores_per_replica, *replicate_node, arg_names, + arg_types, arg_shapes, retval_types, arg_sharding, retval_sharding, tf_device_assignment, compile_node, variable_reads, control_after_compilation, control_after, &variable_writes, graph)); bool contains_resource_write_op = diff --git a/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.h b/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.h index 1931b4ac80f..a9692cc0edb 100644 --- a/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.h +++ b/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.h @@ -413,9 +413,10 @@ class DistributedTPURewritePass : public GraphOptimizationPass { // * `num_cores_per_replica` is the number of cores which are dedicated to // each replica. // * `replicate_node` is the original TPUReplicate node. - // * `arg_types` are the types of the arguments to the computation function + // * `arg_names` are the names of the arguments to the computation function // passed as argument to TPUReplicate, including per-replica, // broadcast, and variable arguments. + // * `arg_types` are the corresponding types of the arguments. // * `arg_shapes` are the corresponding shapes (and handle types/shapes, if // applicable). // * `arg_shardings` and `retval_shardings` are mappings from @@ -431,6 +432,7 @@ class DistributedTPURewritePass : public GraphOptimizationPass { static Status BuildExecuteNodes( const ParameterInfo& params_info, int num_tasks, int num_cores_per_replica, const Node& replicate_node, + const std::vector& arg_names, const DataTypeVector& arg_types, const std::vector& arg_shapes, const DataTypeVector& retval_types, diff --git a/tensorflow/core/tpu/kernels/BUILD b/tensorflow/core/tpu/kernels/BUILD index 6f74123131f..1336f52ed34 100644 --- a/tensorflow/core/tpu/kernels/BUILD +++ b/tensorflow/core/tpu/kernels/BUILD @@ -71,7 +71,6 @@ cc_library( "//tensorflow/core/tpu:tpu_api", "//tensorflow/core/tpu:tpu_configuration", "//tensorflow/core/tpu:tpu_defs", - "//tensorflow/stream_executor/tpu:status_helper", "//tensorflow/stream_executor/tpu:tpu_platform_interface", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:span", diff --git a/tensorflow/core/tpu/kernels/tpu_compile_op_common.cc b/tensorflow/core/tpu/kernels/tpu_compile_op_common.cc index 8bd45db2206..ce18e844e66 100644 --- a/tensorflow/core/tpu/kernels/tpu_compile_op_common.cc +++ b/tensorflow/core/tpu/kernels/tpu_compile_op_common.cc @@ -117,7 +117,7 @@ Status SetPerCoreArgShapes( } else { TF_RET_CHECK(proto_arg.sharding().type() == xla::OpSharding::REPLICATED) << "Unsupported argument sharding: " - << proto_arg.sharding().DebugString(); + << " proto_arg=" << proto_arg.DebugString(); for (int core = 0; core < per_core_arg_shapes->size(); ++core) { (*arg_core_mapping)[arg_index].indices.push_back( (*per_core_arg_shapes)[core].size()); From 0e5562be23660c29f5625e750ea5afcba981dc9f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 15:45:48 -0700 Subject: [PATCH 0293/1017] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 324699433 Change-Id: I980612c967b3d5948fe2cbf614060425a261091c --- tensorflow/go/op/wrappers.go | 1716 +++++++++++++++++----------------- 1 file changed, 858 insertions(+), 858 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 1bff193830a..34ff57636ca 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -27463,524 +27463,6 @@ func DecodePaddedRaw(scope *Scope, input_bytes tf.Output, fixed_length tf.Output return op.Output(0) } -// QuantizeV2Attr is an optional argument to QuantizeV2. -type QuantizeV2Attr func(optionalAttr) - -// QuantizeV2Mode sets the optional mode attribute to value. -// If not specified, defaults to "MIN_COMBINED" -func QuantizeV2Mode(value string) QuantizeV2Attr { - return func(m optionalAttr) { - m["mode"] = value - } -} - -// QuantizeV2RoundMode sets the optional round_mode attribute to value. -// If not specified, defaults to "HALF_AWAY_FROM_ZERO" -func QuantizeV2RoundMode(value string) QuantizeV2Attr { - return func(m optionalAttr) { - m["round_mode"] = value - } -} - -// QuantizeV2NarrowRange sets the optional narrow_range attribute to value. -// If not specified, defaults to false -func QuantizeV2NarrowRange(value bool) QuantizeV2Attr { - return func(m optionalAttr) { - m["narrow_range"] = value - } -} - -// QuantizeV2Axis sets the optional axis attribute to value. -// If not specified, defaults to -1 -func QuantizeV2Axis(value int64) QuantizeV2Attr { - return func(m optionalAttr) { - m["axis"] = value - } -} - -// QuantizeV2EnsureMinimumRange sets the optional ensure_minimum_range attribute to value. -// If not specified, defaults to 0.01 -func QuantizeV2EnsureMinimumRange(value float32) QuantizeV2Attr { - return func(m optionalAttr) { - m["ensure_minimum_range"] = value - } -} - -// Quantize the 'input' tensor of type float to 'output' tensor of type 'T'. -// -// [min_range, max_range] are scalar floats that specify the range for -// the 'input' data. The 'mode' attribute controls exactly which calculations are -// used to convert the float values to their quantized equivalents. The -// 'round_mode' attribute controls which rounding tie-breaking algorithm is used -// when rounding float values to their quantized equivalents. -// -// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: -// -// ``` -// out[i] = (in[i] - min_range) * range(T) / (max_range - min_range) -// if T == qint8: out[i] -= (range(T) + 1) / 2.0 -// ``` -// -// here `range(T) = numeric_limits::max() - numeric_limits::min()` -// -// *MIN_COMBINED Mode Example* -// -// Assume the input is type float and has a possible range of [0.0, 6.0] and the -// output type is quint8 ([0, 255]). The min_range and max_range values should be -// specified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each -// value of the input by 255/6 and cast to quint8. -// -// If the output type was qint8 ([-128, 127]), the operation will additionally -// subtract each value by 128 prior to casting, so that the range of values aligns -// with the range of qint8. -// -// If the mode is 'MIN_FIRST', then this approach is used: -// -// ``` -// num_discrete_values = 1 << (# of bits in T) -// range_adjust = num_discrete_values / (num_discrete_values - 1) -// range = (range_max - range_min) * range_adjust -// range_scale = num_discrete_values / range -// quantized = round(input * range_scale) - round(range_min * range_scale) + -// numeric_limits::min() -// quantized = max(quantized, numeric_limits::min()) -// quantized = min(quantized, numeric_limits::max()) -// ``` -// -// The biggest difference between this and MIN_COMBINED is that the minimum range -// is rounded first, before it's subtracted from the rounded value. With -// MIN_COMBINED, a small bias is introduced where repeated iterations of quantizing -// and dequantizing will introduce a larger and larger error. -// -// *SCALED mode Example* -// -// `SCALED` mode matches the quantization approach used in -// `QuantizeAndDequantize{V2|V3}`. -// -// If the mode is `SCALED`, the quantization is performed by multiplying each -// input value by a scaling_factor. -// The scaling_factor is determined from `min_range` and `max_range` to be as large -// as possible such that the range from `min_range` to `max_range` is representable -// within values of type T. -// -// ```c++ -// -// const int min_T = std::numeric_limits::min(); -// const int max_T = std::numeric_limits::max(); -// const float max_float = std::numeric_limits::max(); -// -// const float scale_factor_from_min_side = -// (min_T * min_range > 0) ? min_T / min_range : max_float; -// const float scale_factor_from_max_side = -// (max_T * max_range > 0) ? max_T / max_range : max_float; -// -// const float scale_factor = std::min(scale_factor_from_min_side, -// scale_factor_from_max_side); -// ``` -// -// We next use the scale_factor to adjust min_range and max_range as follows: -// -// ```c++ -// min_range = min_T / scale_factor; -// max_range = max_T / scale_factor; -// ``` -// -// -// e.g. if T = qint8, and initially min_range = -10, and max_range = 9, we would -// compare -128/-10.0 = 12.8 to 127/9.0 = 14.11, and set scaling_factor = 12.8 -// In this case, min_range would remain -10, but max_range would be adjusted to -// 127 / 12.8 = 9.921875 -// -// So we will quantize input values in the range (-10, 9.921875) to (-128, 127). -// -// The input tensor can now be quantized by clipping values to the range -// `min_range` to `max_range`, then multiplying by scale_factor as follows: -// -// ```c++ -// result = round(min(max_range, max(min_range, input)) * scale_factor) -// ``` -// -// The adjusted `min_range` and `max_range` are returned as outputs 2 and 3 of -// this operation. These outputs should be used as the range for any further -// calculations. -// -// -// *narrow_range (bool) attribute* -// -// If true, we do not use the minimum quantized value. -// i.e. for int8 the quantized output, it would be restricted to the range -// -127..127 instead of the full -128..127 range. -// This is provided for compatibility with certain inference backends. -// (Only applies to SCALED mode) -// -// -// *axis (int) attribute* -// -// An optional `axis` attribute can specify a dimension index of the input tensor, -// such that quantization ranges will be calculated and applied separately for each -// slice of the tensor along that dimension. This is useful for per-channel -// quantization. -// -// If axis is specified, min_range and max_range -// -// if `axis`=None, per-tensor quantization is performed as normal. -// -// -// *ensure_minimum_range (float) attribute* -// -// Ensures the minimum quantization range is at least this value. -// The legacy default value for this is 0.01, but it is strongly suggested to -// set it to 0 for new uses. -// -// -// Arguments: -// -// min_range: The minimum value of the quantization range. This value may be adjusted by the -// op depending on other parameters. The adjusted value is written to `output_min`. -// If the `axis` attribute is specified, this must be a 1-D tensor whose size -// matches the `axis` dimension of the input and output tensors. -// max_range: The maximum value of the quantization range. This value may be adjusted by the -// op depending on other parameters. The adjusted value is written to `output_max`. -// If the `axis` attribute is specified, this must be a 1-D tensor whose size -// matches the `axis` dimension of the input and output tensors. -// -// -// Returns: -// output: The quantized data produced from the float input. -// output_min: The final quantization range minimum, used to clip input values before scaling -// and rounding them to quantized values. -// If the `axis` attribute is specified, this will be a 1-D tensor whose size -// matches the `axis` dimension of the input and output tensors. -// output_max: The final quantization range maximum, used to clip input values before scaling -// and rounding them to quantized values. -// If the `axis` attribute is specified, this will be a 1-D tensor whose size -// matches the `axis` dimension of the input and output tensors. -func QuantizeV2(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, T tf.DataType, optional ...QuantizeV2Attr) (output tf.Output, output_min tf.Output, output_max tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"T": T} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "QuantizeV2", - Input: []tf.Input{ - input, min_range, max_range, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) -} - -// Returns the truth value of (x >= y) element-wise. -// -// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting -// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) -// -// Example: -// -// ```python -// x = tf.constant([5, 4, 6, 7]) -// y = tf.constant([5, 2, 5, 10]) -// tf.math.greater_equal(x, y) ==> [True, True, True, False] -// -// x = tf.constant([5, 4, 6, 7]) -// y = tf.constant([5]) -// tf.math.greater_equal(x, y) ==> [True, False, True, True] -// ``` -func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "GreaterEqual", - Input: []tf.Input{ - x, y, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// BatchAttr is an optional argument to Batch. -type BatchAttr func(optionalAttr) - -// BatchMaxEnqueuedBatches sets the optional max_enqueued_batches attribute to value. -// If not specified, defaults to 10 -func BatchMaxEnqueuedBatches(value int64) BatchAttr { - return func(m optionalAttr) { - m["max_enqueued_batches"] = value - } -} - -// BatchAllowedBatchSizes sets the optional allowed_batch_sizes attribute to value. -// If not specified, defaults to <> -func BatchAllowedBatchSizes(value []int64) BatchAttr { - return func(m optionalAttr) { - m["allowed_batch_sizes"] = value - } -} - -// BatchContainer sets the optional container attribute to value. -// If not specified, defaults to "" -func BatchContainer(value string) BatchAttr { - return func(m optionalAttr) { - m["container"] = value - } -} - -// BatchSharedName sets the optional shared_name attribute to value. -// If not specified, defaults to "" -func BatchSharedName(value string) BatchAttr { - return func(m optionalAttr) { - m["shared_name"] = value - } -} - -// BatchBatchingQueue sets the optional batching_queue attribute to value. -// If not specified, defaults to "" -func BatchBatchingQueue(value string) BatchAttr { - return func(m optionalAttr) { - m["batching_queue"] = value - } -} - -// Batches all input tensors nondeterministically. -// -// When many instances of this Op are being run concurrently with the same -// container/shared_name in the same device, some will output zero-shaped Tensors -// and others will output Tensors of size up to max_batch_size. -// -// All Tensors in in_tensors are batched together (so, for example, labels and -// features should be batched with a single instance of this operation. -// -// Each invocation of batch emits an `id` scalar which will be used to identify -// this particular invocation when doing unbatch or its gradient. -// -// Each op which emits a non-empty batch will also emit a non-empty batch_index -// Tensor, which, is a [K, 3] matrix where each row contains the invocation's id, -// start, and length of elements of each set of Tensors present in batched_tensors. -// -// Batched tensors are concatenated along the first dimension, and all tensors in -// in_tensors must have the first dimension of the same size. -// -// in_tensors: The tensors to be batched. -// num_batch_threads: Number of scheduling threads for processing batches of work. -// Determines the number of batches processed in parallel. -// max_batch_size: Batch sizes will never be bigger than this. -// batch_timeout_micros: Maximum number of microseconds to wait before outputting -// an incomplete batch. -// allowed_batch_sizes: Optional list of allowed batch sizes. If left empty, does -// nothing. Otherwise, supplies a list of batch sizes, causing the op to pad -// batches up to one of those sizes. The entries must increase monotonically, and -// the final entry must equal max_batch_size. -// grad_timeout_micros: The timeout to use for the gradient. See Unbatch. -// batched_tensors: Either empty tensors or a batch of concatenated Tensors. -// batch_index: If out_tensors is non-empty, has information to invert it. -// container: Controls the scope of sharing of this batch. -// id: always contains a scalar with a unique ID for this invocation of Batch. -// shared_name: Concurrently running instances of batch in the same device with the -// same container and shared_name will batch their elements together. If left -// empty, the op name will be used as the shared name. -// T: the types of tensors to be batched. -func Batch(scope *Scope, in_tensors []tf.Output, num_batch_threads int64, max_batch_size int64, batch_timeout_micros int64, grad_timeout_micros int64, optional ...BatchAttr) (batched_tensors []tf.Output, batch_index tf.Output, id tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"num_batch_threads": num_batch_threads, "max_batch_size": max_batch_size, "batch_timeout_micros": batch_timeout_micros, "grad_timeout_micros": grad_timeout_micros} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "Batch", - Input: []tf.Input{ - tf.OutputList(in_tensors), - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - if scope.Err() != nil { - return - } - var idx int - var err error - if batched_tensors, idx, err = makeOutputList(op, idx, "batched_tensors"); err != nil { - scope.UpdateErr("Batch", err) - return - } - batch_index = op.Output(idx) - id = op.Output(idx) - return batched_tensors, batch_index, id -} - -// UnicodeDecodeAttr is an optional argument to UnicodeDecode. -type UnicodeDecodeAttr func(optionalAttr) - -// UnicodeDecodeErrors sets the optional errors attribute to value. -// -// value: Error handling policy when there is invalid formatting found in the input. -// The value of 'strict' will cause the operation to produce a InvalidArgument -// error on any invalid input formatting. A value of 'replace' (the default) will -// cause the operation to replace any invalid formatting in the input with the -// `replacement_char` codepoint. A value of 'ignore' will cause the operation to -// skip any invalid formatting in the input and produce no corresponding output -// character. -// If not specified, defaults to "replace" -func UnicodeDecodeErrors(value string) UnicodeDecodeAttr { - return func(m optionalAttr) { - m["errors"] = value - } -} - -// UnicodeDecodeReplacementChar sets the optional replacement_char attribute to value. -// -// value: The replacement character codepoint to be used in place of any invalid -// formatting in the input when `errors='replace'`. Any valid unicode codepoint may -// be used. The default value is the default unicode replacement character is -// 0xFFFD or U+65533.) -// If not specified, defaults to 65533 -func UnicodeDecodeReplacementChar(value int64) UnicodeDecodeAttr { - return func(m optionalAttr) { - m["replacement_char"] = value - } -} - -// UnicodeDecodeReplaceControlCharacters sets the optional replace_control_characters attribute to value. -// -// value: Whether to replace the C0 control characters (00-1F) with the -// `replacement_char`. Default is false. -// If not specified, defaults to false -func UnicodeDecodeReplaceControlCharacters(value bool) UnicodeDecodeAttr { - return func(m optionalAttr) { - m["replace_control_characters"] = value - } -} - -// UnicodeDecodeTsplits sets the optional Tsplits attribute to value. -// If not specified, defaults to DT_INT64 -func UnicodeDecodeTsplits(value tf.DataType) UnicodeDecodeAttr { - return func(m optionalAttr) { - m["Tsplits"] = value - } -} - -// Decodes each string in `input` into a sequence of Unicode code points. -// -// The character codepoints for all strings are returned using a single vector -// `char_values`, with strings expanded to characters in row-major order. -// -// The `row_splits` tensor indicates where the codepoints for -// each input string begin and end within the `char_values` tensor. -// In particular, the values for the `i`th -// string (in row-major order) are stored in the slice -// `[row_splits[i]:row_splits[i+1]]`. Thus: -// -// * `char_values[row_splits[i]+j]` is the Unicode codepoint for the `j`th -// character in the `i`th string (in row-major order). -// * `row_splits[i+1] - row_splits[i]` is the number of characters in the `i`th -// string (in row-major order). -// -// Arguments: -// input: The text to be decoded. Can have any shape. Note that the output is flattened -// to a vector of char values. -// input_encoding: Text encoding of the input strings. This is any of the encodings supported -// by ICU ucnv algorithmic converters. Examples: `"UTF-16", "US ASCII", "UTF-8"`. -// -// Returns: -// row_splits: A 1D int32 tensor containing the row splits. -// char_values: A 1D int32 Tensor containing the decoded codepoints. -func UnicodeDecode(scope *Scope, input tf.Output, input_encoding string, optional ...UnicodeDecodeAttr) (row_splits tf.Output, char_values tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"input_encoding": input_encoding} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "UnicodeDecode", - Input: []tf.Input{ - input, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) -} - -// Create a dense tensor from a ragged tensor, possibly altering its shape. -// -// The `ragged_to_dense` op creates a dense tensor from a list of row partition -// tensors, a value vector, and default values. If the shape is unspecified, the -// minimal shape required to contain all the elements in the ragged tensor (the -// natural shape) will be used. If some dimensions are left unspecified, then the -// size of the natural shape is used in that dimension. -// -// The default_value will be broadcast to the output shape. After that, the values -// from the ragged tensor overwrite the default values. Note that the default_value -// must have less dimensions than the value. -// -// The row partition tensors are in the order of the dimensions. -// At present, the types can be: -// * "ROW_SPLITS": the row_splits tensor from the ragged tensor. -// * "VALUE_ROWIDS": the value_rowids tensor from the ragged tensor. -// * "FIRST_DIM_SIZE": if value_rowids is used for the first dimension, then it -// is preceded by "FIRST_DIM_SIZE". -// -// Arguments: -// shape: The desired shape of the the output tensor. If left unspecified (empty), -// the minimal shape required to contain all the elements in the ragged tensor -// (the natural shape) will be used. If some dimensions are left unspecified, then -// the size of the natural shape is used in that dimension. -// -// Note that dense dimensions cannot be modified by the shape argument. Trying to -// change the size of a dense dimension will cause the op to fail. -// Examples: -// natural shape: [4, 5, 6] -// shape: -1 -// output shape: [4, 5, 6] -// -// natural shape: [4, 5, 6] -// shape: [3, -1, 2] -// output shape: [3, 5, 2] -// -// natural shape: [4, 5, 6] -// shape: [3, 7, 2] -// output shape: [3, 7, 2] -// -// values: A 1D tensor representing the values of the ragged tensor. -// default_value: The default_value when the shape is larger than the ragged tensor. The -// default_value is broadcast until it is the shape of the output tensor, and -// then overwritten by values in the ragged tensor. The default value must be -// compatible with this broadcast operation, and must have fewer dimensions than -// the value tensor. -// -// row_partition_types: The types of the row partition tensors. At present, these can be: -// * "ROW_SPLITS": the row_splits tensor from the ragged tensor. -// * "VALUE_ROWIDS": the value_rowids tensor from the ragged tensor. -// * "FIRST_DIM_SIZE": if value_rowids is used for the first dimension, then it -// is preceeded by "FIRST_DIM_SIZE". -// The tensors are in the order of the dimensions. -// -// Returns The resulting dense tensor. -func RaggedTensorToTensor(scope *Scope, shape tf.Output, values tf.Output, default_value tf.Output, row_partition_tensors []tf.Output, row_partition_types []string) (result tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"row_partition_types": row_partition_types} - opspec := tf.OpSpec{ - Type: "RaggedTensorToTensor", - Input: []tf.Input{ - shape, values, default_value, tf.OutputList(row_partition_tensors), - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // BatchMatMulAttr is an optional argument to BatchMatMul. type BatchMatMulAttr func(optionalAttr) @@ -28653,45 +28135,6 @@ func BlockLSTMV2(scope *Scope, seq_len_max tf.Output, x tf.Output, cs_prev tf.Ou return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4), op.Output(5), op.Output(6) } -// Return a tensor with the same shape and contents as the input tensor or value. -func Identity(scope *Scope, input tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "Identity", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Outputs a `Summary` protocol buffer with scalar values. -// -// The input `tags` and `values` must have the same shape. The generated summary -// has a summary value for each tag-value pair in `tags` and `values`. -// -// Arguments: -// tags: Tags for the summary. -// values: Same shape as `tags. Values for the summary. -// -// Returns Scalar. Serialized `Summary` protocol buffer. -func ScalarSummary(scope *Scope, tags tf.Output, values tf.Output) (summary tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "ScalarSummary", - Input: []tf.Input{ - tags, values, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // ResourceSparseApplyProximalAdagradAttr is an optional argument to ResourceSparseApplyProximalAdagrad. type ResourceSparseApplyProximalAdagradAttr func(optionalAttr) @@ -28759,6 +28202,45 @@ func Neg(scope *Scope, x tf.Output) (y tf.Output) { return op.Output(0) } +// Return a tensor with the same shape and contents as the input tensor or value. +func Identity(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "Identity", + Input: []tf.Input{ + input, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Outputs a `Summary` protocol buffer with scalar values. +// +// The input `tags` and `values` must have the same shape. The generated summary +// has a summary value for each tag-value pair in `tags` and `values`. +// +// Arguments: +// tags: Tags for the summary. +// values: Same shape as `tags. Values for the summary. +// +// Returns Scalar. Serialized `Summary` protocol buffer. +func ScalarSummary(scope *Scope, tags tf.Output, values tf.Output) (summary tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "ScalarSummary", + Input: []tf.Input{ + tags, values, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Concatenates tensors along one dimension. // // Arguments: @@ -34257,307 +33739,6 @@ func SparseCrossV2(scope *Scope, indices []tf.Output, values []tf.Output, shapes return op.Output(0), op.Output(1), op.Output(2) } -// Pads a tensor with mirrored values. -// -// This operation pads a `input` with mirrored values according to the `paddings` -// you specify. `paddings` is an integer tensor with shape `[n, 2]`, where n is -// the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates -// how many values to add before the contents of `input` in that dimension, and -// `paddings[D, 1]` indicates how many values to add after the contents of `input` -// in that dimension. Both `paddings[D, 0]` and `paddings[D, 1]` must be no greater -// than `input.dim_size(D)` (or `input.dim_size(D) - 1`) if `copy_border` is true -// (if false, respectively). -// -// The padded size of each dimension D of the output is: -// -// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)` -// -// For example: -// -// ``` -// # 't' is [[1, 2, 3], [4, 5, 6]]. -// # 'paddings' is [[1, 1]], [2, 2]]. -// # 'mode' is SYMMETRIC. -// # rank of 't' is 2. -// pad(t, paddings) ==> [[2, 1, 1, 2, 3, 3, 2] -// [2, 1, 1, 2, 3, 3, 2] -// [5, 4, 4, 5, 6, 6, 5] -// [5, 4, 4, 5, 6, 6, 5]] -// ``` -// -// Arguments: -// input: The input tensor to be padded. -// paddings: A two-column matrix specifying the padding sizes. The number of -// rows must be the same as the rank of `input`. -// mode: Either `REFLECT` or `SYMMETRIC`. In reflect mode the padded regions -// do not include the borders, while in symmetric mode the padded regions -// do include the borders. For example, if `input` is `[1, 2, 3]` and `paddings` -// is `[0, 2]`, then the output is `[1, 2, 3, 2, 1]` in reflect mode, and -// it is `[1, 2, 3, 3, 2]` in symmetric mode. -// -// Returns The padded tensor. -func MirrorPad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"mode": mode} - opspec := tf.OpSpec{ - Type: "MirrorPad", - Input: []tf.Input{ - input, paddings, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// TensorArrayV3Attr is an optional argument to TensorArrayV3. -type TensorArrayV3Attr func(optionalAttr) - -// TensorArrayV3ElementShape sets the optional element_shape attribute to value. -// -// value: The expected shape of an element, if known. Used to -// validate the shapes of TensorArray elements. If this shape is not -// fully specified, gathering zero-size TensorArrays is an error. -// If not specified, defaults to -func TensorArrayV3ElementShape(value tf.Shape) TensorArrayV3Attr { - return func(m optionalAttr) { - m["element_shape"] = value - } -} - -// TensorArrayV3DynamicSize sets the optional dynamic_size attribute to value. -// -// value: A boolean that determines whether writes to the TensorArray -// are allowed to grow the size. By default, this is not allowed. -// If not specified, defaults to false -func TensorArrayV3DynamicSize(value bool) TensorArrayV3Attr { - return func(m optionalAttr) { - m["dynamic_size"] = value - } -} - -// TensorArrayV3ClearAfterRead sets the optional clear_after_read attribute to value. -// -// value: If true (default), Tensors in the TensorArray are cleared -// after being read. This disables multiple read semantics but allows early -// release of memory. -// If not specified, defaults to true -func TensorArrayV3ClearAfterRead(value bool) TensorArrayV3Attr { - return func(m optionalAttr) { - m["clear_after_read"] = value - } -} - -// TensorArrayV3IdenticalElementShapes sets the optional identical_element_shapes attribute to value. -// -// value: If true (default is false), then all -// elements in the TensorArray will be expected to have have identical shapes. -// This allows certain behaviors, like dynamically checking for -// consistent shapes on write, and being able to fill in properly -// shaped zero tensors on stack -- even if the element_shape attribute -// is not fully defined. -// If not specified, defaults to false -func TensorArrayV3IdenticalElementShapes(value bool) TensorArrayV3Attr { - return func(m optionalAttr) { - m["identical_element_shapes"] = value - } -} - -// TensorArrayV3TensorArrayName sets the optional tensor_array_name attribute to value. -// -// value: Overrides the name used for the temporary tensor_array -// resource. Default value is the name of the 'TensorArray' op (which -// is guaranteed unique). -// If not specified, defaults to "" -func TensorArrayV3TensorArrayName(value string) TensorArrayV3Attr { - return func(m optionalAttr) { - m["tensor_array_name"] = value - } -} - -// An array of Tensors of given size. -// -// Write data via Write and read via Read or Pack. -// -// Arguments: -// size: The size of the array. -// dtype: The type of the elements on the tensor_array. -// -// Returns: -// handle: The handle to the TensorArray. -// flow: A scalar used to control gradient flow. -func TensorArrayV3(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV3Attr) (handle tf.Output, flow tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"dtype": dtype} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "TensorArrayV3", - Input: []tf.Input{ - size, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1) -} - -// MatrixSolveLsAttr is an optional argument to MatrixSolveLs. -type MatrixSolveLsAttr func(optionalAttr) - -// MatrixSolveLsFast sets the optional fast attribute to value. -// If not specified, defaults to true -func MatrixSolveLsFast(value bool) MatrixSolveLsAttr { - return func(m optionalAttr) { - m["fast"] = value - } -} - -// Solves one or more linear least-squares problems. -// -// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions -// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same -// type as `matrix` and shape `[..., M, K]`. -// The output is a tensor shape `[..., N, K]` where each output matrix solves -// each of the equations -// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]` -// in the least squares sense. -// -// We use the following notation for (complex) matrix and right-hand sides -// in the batch: -// -// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\), -// `rhs`=\\(B \in \mathbb{C}^{m \times k}\\), -// `output`=\\(X \in \mathbb{C}^{n \times k}\\), -// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\). -// -// If `fast` is `True`, then the solution is computed by solving the normal -// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then -// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares -// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 + \lambda ||Z||_F^2\\). -// If \\(m \lt n\\) then `output` is computed as -// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the -// minimum-norm solution to the under-determined linear system, i.e. -// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\), -// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable -// when \\(A\\) is numerically full rank and has a condition number -// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or \\(\lambda\\) is -// sufficiently large. -// -// If `fast` is `False` an algorithm based on the numerically robust complete -// orthogonal decomposition is used. This computes the minimum-norm -// least-squares solution, even when \\(A\\) is rank deficient. This path is -// typically 6-7 times slower than the fast path. If `fast` is `False` then -// `l2_regularizer` is ignored. -// -// Arguments: -// matrix: Shape is `[..., M, N]`. -// rhs: Shape is `[..., M, K]`. -// l2_regularizer: Scalar tensor. -// -// @compatibility(numpy) -// Equivalent to np.linalg.lstsq -// @end_compatibility -// -// Returns Shape is `[..., N, K]`. -func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{} - for _, a := range optional { - a(attrs) - } - opspec := tf.OpSpec{ - Type: "MatrixSolveLs", - Input: []tf.Input{ - matrix, rhs, l2_regularizer, - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - -// Generates sparse cross from a list of sparse and dense tensors. -// -// The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each -// representing features of one feature column. It outputs a 2D `SparseTensor` with -// the batchwise crosses of these features. -// -// For example, if the inputs are -// -// inputs[0]: SparseTensor with shape = [2, 2] -// [0, 0]: "a" -// [1, 0]: "b" -// [1, 1]: "c" -// -// inputs[1]: SparseTensor with shape = [2, 1] -// [0, 0]: "d" -// [1, 0]: "e" -// -// inputs[2]: Tensor [["f"], ["g"]] -// -// then the output will be -// -// shape = [2, 2] -// [0, 0]: "a_X_d_X_f" -// [1, 0]: "b_X_e_X_g" -// [1, 1]: "c_X_e_X_g" -// -// if hashed_output=true then the output will be -// -// shape = [2, 2] -// [0, 0]: FingerprintCat64( -// Fingerprint64("f"), FingerprintCat64( -// Fingerprint64("d"), Fingerprint64("a"))) -// [1, 0]: FingerprintCat64( -// Fingerprint64("g"), FingerprintCat64( -// Fingerprint64("e"), Fingerprint64("b"))) -// [1, 1]: FingerprintCat64( -// Fingerprint64("g"), FingerprintCat64( -// Fingerprint64("e"), Fingerprint64("c"))) -// -// Arguments: -// indices: 2-D. Indices of each input `SparseTensor`. -// values: 1-D. values of each `SparseTensor`. -// shapes: 1-D. Shapes of each `SparseTensor`. -// dense_inputs: 2-D. Columns represented by dense `Tensor`. -// hashed_output: If true, returns the hash of the cross instead of the string. -// This will allow us avoiding string manipulations. -// num_buckets: It is used if hashed_output is true. -// output = hashed_value%num_buckets if num_buckets > 0 else hashed_value. -// hash_key: Specify the hash_key that will be used by the `FingerprintCat64` -// function to combine the crosses fingerprints. -// -// -// -// Returns: -// output_indices: 2-D. Indices of the concatenated `SparseTensor`. -// output_values: 1-D. Non-empty values of the concatenated or hashed -// `SparseTensor`. -// output_shape: 1-D. Shape of the concatenated `SparseTensor`. -func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, dense_inputs []tf.Output, hashed_output bool, num_buckets int64, hash_key int64, out_type tf.DataType, internal_type tf.DataType) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) { - if scope.Err() != nil { - return - } - attrs := map[string]interface{}{"hashed_output": hashed_output, "num_buckets": num_buckets, "hash_key": hash_key, "out_type": out_type, "internal_type": internal_type} - opspec := tf.OpSpec{ - Type: "SparseCross", - Input: []tf.Input{ - tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), tf.OutputList(dense_inputs), - }, - Attrs: attrs, - } - op := scope.AddOperation(opspec) - return op.Output(0), op.Output(1), op.Output(2) -} - // Generate a glob pattern matching all sharded file names. func ShardedFilespec(scope *Scope, basename tf.Output, num_shards tf.Output) (filename tf.Output) { if scope.Err() != nil { @@ -37294,6 +36475,77 @@ func InfeedEnqueuePrelinearizedBuffer(scope *Scope, input tf.Output, optional .. return scope.AddOperation(opspec) } +// Create a dense tensor from a ragged tensor, possibly altering its shape. +// +// The `ragged_to_dense` op creates a dense tensor from a list of row partition +// tensors, a value vector, and default values. If the shape is unspecified, the +// minimal shape required to contain all the elements in the ragged tensor (the +// natural shape) will be used. If some dimensions are left unspecified, then the +// size of the natural shape is used in that dimension. +// +// The default_value will be broadcast to the output shape. After that, the values +// from the ragged tensor overwrite the default values. Note that the default_value +// must have less dimensions than the value. +// +// The row partition tensors are in the order of the dimensions. +// At present, the types can be: +// * "ROW_SPLITS": the row_splits tensor from the ragged tensor. +// * "VALUE_ROWIDS": the value_rowids tensor from the ragged tensor. +// * "FIRST_DIM_SIZE": if value_rowids is used for the first dimension, then it +// is preceded by "FIRST_DIM_SIZE". +// +// Arguments: +// shape: The desired shape of the the output tensor. If left unspecified (empty), +// the minimal shape required to contain all the elements in the ragged tensor +// (the natural shape) will be used. If some dimensions are left unspecified, then +// the size of the natural shape is used in that dimension. +// +// Note that dense dimensions cannot be modified by the shape argument. Trying to +// change the size of a dense dimension will cause the op to fail. +// Examples: +// natural shape: [4, 5, 6] +// shape: -1 +// output shape: [4, 5, 6] +// +// natural shape: [4, 5, 6] +// shape: [3, -1, 2] +// output shape: [3, 5, 2] +// +// natural shape: [4, 5, 6] +// shape: [3, 7, 2] +// output shape: [3, 7, 2] +// +// values: A 1D tensor representing the values of the ragged tensor. +// default_value: The default_value when the shape is larger than the ragged tensor. The +// default_value is broadcast until it is the shape of the output tensor, and +// then overwritten by values in the ragged tensor. The default value must be +// compatible with this broadcast operation, and must have fewer dimensions than +// the value tensor. +// +// row_partition_types: The types of the row partition tensors. At present, these can be: +// * "ROW_SPLITS": the row_splits tensor from the ragged tensor. +// * "VALUE_ROWIDS": the value_rowids tensor from the ragged tensor. +// * "FIRST_DIM_SIZE": if value_rowids is used for the first dimension, then it +// is preceeded by "FIRST_DIM_SIZE". +// The tensors are in the order of the dimensions. +// +// Returns The resulting dense tensor. +func RaggedTensorToTensor(scope *Scope, shape tf.Output, values tf.Output, default_value tf.Output, row_partition_tensors []tf.Output, row_partition_types []string) (result tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"row_partition_types": row_partition_types} + opspec := tf.OpSpec{ + Type: "RaggedTensorToTensor", + Input: []tf.Input{ + shape, values, default_value, tf.OutputList(row_partition_tensors), + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Computes the derivative of a Gamma random sample w.r.t. `alpha`. func RandomGammaGrad(scope *Scope, alpha tf.Output, sample tf.Output) (output tf.Output) { if scope.Err() != nil { @@ -41826,6 +41078,453 @@ func SparseSparseMaximum(scope *Scope, a_indices tf.Output, a_values tf.Output, return op.Output(0), op.Output(1) } +// QuantizeV2Attr is an optional argument to QuantizeV2. +type QuantizeV2Attr func(optionalAttr) + +// QuantizeV2Mode sets the optional mode attribute to value. +// If not specified, defaults to "MIN_COMBINED" +func QuantizeV2Mode(value string) QuantizeV2Attr { + return func(m optionalAttr) { + m["mode"] = value + } +} + +// QuantizeV2RoundMode sets the optional round_mode attribute to value. +// If not specified, defaults to "HALF_AWAY_FROM_ZERO" +func QuantizeV2RoundMode(value string) QuantizeV2Attr { + return func(m optionalAttr) { + m["round_mode"] = value + } +} + +// QuantizeV2NarrowRange sets the optional narrow_range attribute to value. +// If not specified, defaults to false +func QuantizeV2NarrowRange(value bool) QuantizeV2Attr { + return func(m optionalAttr) { + m["narrow_range"] = value + } +} + +// QuantizeV2Axis sets the optional axis attribute to value. +// If not specified, defaults to -1 +func QuantizeV2Axis(value int64) QuantizeV2Attr { + return func(m optionalAttr) { + m["axis"] = value + } +} + +// QuantizeV2EnsureMinimumRange sets the optional ensure_minimum_range attribute to value. +// If not specified, defaults to 0.01 +func QuantizeV2EnsureMinimumRange(value float32) QuantizeV2Attr { + return func(m optionalAttr) { + m["ensure_minimum_range"] = value + } +} + +// Quantize the 'input' tensor of type float to 'output' tensor of type 'T'. +// +// [min_range, max_range] are scalar floats that specify the range for +// the 'input' data. The 'mode' attribute controls exactly which calculations are +// used to convert the float values to their quantized equivalents. The +// 'round_mode' attribute controls which rounding tie-breaking algorithm is used +// when rounding float values to their quantized equivalents. +// +// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following: +// +// ``` +// out[i] = (in[i] - min_range) * range(T) / (max_range - min_range) +// if T == qint8: out[i] -= (range(T) + 1) / 2.0 +// ``` +// +// here `range(T) = numeric_limits::max() - numeric_limits::min()` +// +// *MIN_COMBINED Mode Example* +// +// Assume the input is type float and has a possible range of [0.0, 6.0] and the +// output type is quint8 ([0, 255]). The min_range and max_range values should be +// specified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each +// value of the input by 255/6 and cast to quint8. +// +// If the output type was qint8 ([-128, 127]), the operation will additionally +// subtract each value by 128 prior to casting, so that the range of values aligns +// with the range of qint8. +// +// If the mode is 'MIN_FIRST', then this approach is used: +// +// ``` +// num_discrete_values = 1 << (# of bits in T) +// range_adjust = num_discrete_values / (num_discrete_values - 1) +// range = (range_max - range_min) * range_adjust +// range_scale = num_discrete_values / range +// quantized = round(input * range_scale) - round(range_min * range_scale) + +// numeric_limits::min() +// quantized = max(quantized, numeric_limits::min()) +// quantized = min(quantized, numeric_limits::max()) +// ``` +// +// The biggest difference between this and MIN_COMBINED is that the minimum range +// is rounded first, before it's subtracted from the rounded value. With +// MIN_COMBINED, a small bias is introduced where repeated iterations of quantizing +// and dequantizing will introduce a larger and larger error. +// +// *SCALED mode Example* +// +// `SCALED` mode matches the quantization approach used in +// `QuantizeAndDequantize{V2|V3}`. +// +// If the mode is `SCALED`, the quantization is performed by multiplying each +// input value by a scaling_factor. +// The scaling_factor is determined from `min_range` and `max_range` to be as large +// as possible such that the range from `min_range` to `max_range` is representable +// within values of type T. +// +// ```c++ +// +// const int min_T = std::numeric_limits::min(); +// const int max_T = std::numeric_limits::max(); +// const float max_float = std::numeric_limits::max(); +// +// const float scale_factor_from_min_side = +// (min_T * min_range > 0) ? min_T / min_range : max_float; +// const float scale_factor_from_max_side = +// (max_T * max_range > 0) ? max_T / max_range : max_float; +// +// const float scale_factor = std::min(scale_factor_from_min_side, +// scale_factor_from_max_side); +// ``` +// +// We next use the scale_factor to adjust min_range and max_range as follows: +// +// ```c++ +// min_range = min_T / scale_factor; +// max_range = max_T / scale_factor; +// ``` +// +// +// e.g. if T = qint8, and initially min_range = -10, and max_range = 9, we would +// compare -128/-10.0 = 12.8 to 127/9.0 = 14.11, and set scaling_factor = 12.8 +// In this case, min_range would remain -10, but max_range would be adjusted to +// 127 / 12.8 = 9.921875 +// +// So we will quantize input values in the range (-10, 9.921875) to (-128, 127). +// +// The input tensor can now be quantized by clipping values to the range +// `min_range` to `max_range`, then multiplying by scale_factor as follows: +// +// ```c++ +// result = round(min(max_range, max(min_range, input)) * scale_factor) +// ``` +// +// The adjusted `min_range` and `max_range` are returned as outputs 2 and 3 of +// this operation. These outputs should be used as the range for any further +// calculations. +// +// +// *narrow_range (bool) attribute* +// +// If true, we do not use the minimum quantized value. +// i.e. for int8 the quantized output, it would be restricted to the range +// -127..127 instead of the full -128..127 range. +// This is provided for compatibility with certain inference backends. +// (Only applies to SCALED mode) +// +// +// *axis (int) attribute* +// +// An optional `axis` attribute can specify a dimension index of the input tensor, +// such that quantization ranges will be calculated and applied separately for each +// slice of the tensor along that dimension. This is useful for per-channel +// quantization. +// +// If axis is specified, min_range and max_range +// +// if `axis`=None, per-tensor quantization is performed as normal. +// +// +// *ensure_minimum_range (float) attribute* +// +// Ensures the minimum quantization range is at least this value. +// The legacy default value for this is 0.01, but it is strongly suggested to +// set it to 0 for new uses. +// +// +// Arguments: +// +// min_range: The minimum value of the quantization range. This value may be adjusted by the +// op depending on other parameters. The adjusted value is written to `output_min`. +// If the `axis` attribute is specified, this must be a 1-D tensor whose size +// matches the `axis` dimension of the input and output tensors. +// max_range: The maximum value of the quantization range. This value may be adjusted by the +// op depending on other parameters. The adjusted value is written to `output_max`. +// If the `axis` attribute is specified, this must be a 1-D tensor whose size +// matches the `axis` dimension of the input and output tensors. +// +// +// Returns: +// output: The quantized data produced from the float input. +// output_min: The final quantization range minimum, used to clip input values before scaling +// and rounding them to quantized values. +// If the `axis` attribute is specified, this will be a 1-D tensor whose size +// matches the `axis` dimension of the input and output tensors. +// output_max: The final quantization range maximum, used to clip input values before scaling +// and rounding them to quantized values. +// If the `axis` attribute is specified, this will be a 1-D tensor whose size +// matches the `axis` dimension of the input and output tensors. +func QuantizeV2(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, T tf.DataType, optional ...QuantizeV2Attr) (output tf.Output, output_min tf.Output, output_max tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"T": T} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "QuantizeV2", + Input: []tf.Input{ + input, min_range, max_range, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1), op.Output(2) +} + +// Returns the truth value of (x >= y) element-wise. +// +// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting +// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) +// +// Example: +// +// ```python +// x = tf.constant([5, 4, 6, 7]) +// y = tf.constant([5, 2, 5, 10]) +// tf.math.greater_equal(x, y) ==> [True, True, True, False] +// +// x = tf.constant([5, 4, 6, 7]) +// y = tf.constant([5]) +// tf.math.greater_equal(x, y) ==> [True, False, True, True] +// ``` +func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "GreaterEqual", + Input: []tf.Input{ + x, y, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// BatchAttr is an optional argument to Batch. +type BatchAttr func(optionalAttr) + +// BatchMaxEnqueuedBatches sets the optional max_enqueued_batches attribute to value. +// If not specified, defaults to 10 +func BatchMaxEnqueuedBatches(value int64) BatchAttr { + return func(m optionalAttr) { + m["max_enqueued_batches"] = value + } +} + +// BatchAllowedBatchSizes sets the optional allowed_batch_sizes attribute to value. +// If not specified, defaults to <> +func BatchAllowedBatchSizes(value []int64) BatchAttr { + return func(m optionalAttr) { + m["allowed_batch_sizes"] = value + } +} + +// BatchContainer sets the optional container attribute to value. +// If not specified, defaults to "" +func BatchContainer(value string) BatchAttr { + return func(m optionalAttr) { + m["container"] = value + } +} + +// BatchSharedName sets the optional shared_name attribute to value. +// If not specified, defaults to "" +func BatchSharedName(value string) BatchAttr { + return func(m optionalAttr) { + m["shared_name"] = value + } +} + +// BatchBatchingQueue sets the optional batching_queue attribute to value. +// If not specified, defaults to "" +func BatchBatchingQueue(value string) BatchAttr { + return func(m optionalAttr) { + m["batching_queue"] = value + } +} + +// Batches all input tensors nondeterministically. +// +// When many instances of this Op are being run concurrently with the same +// container/shared_name in the same device, some will output zero-shaped Tensors +// and others will output Tensors of size up to max_batch_size. +// +// All Tensors in in_tensors are batched together (so, for example, labels and +// features should be batched with a single instance of this operation. +// +// Each invocation of batch emits an `id` scalar which will be used to identify +// this particular invocation when doing unbatch or its gradient. +// +// Each op which emits a non-empty batch will also emit a non-empty batch_index +// Tensor, which, is a [K, 3] matrix where each row contains the invocation's id, +// start, and length of elements of each set of Tensors present in batched_tensors. +// +// Batched tensors are concatenated along the first dimension, and all tensors in +// in_tensors must have the first dimension of the same size. +// +// in_tensors: The tensors to be batched. +// num_batch_threads: Number of scheduling threads for processing batches of work. +// Determines the number of batches processed in parallel. +// max_batch_size: Batch sizes will never be bigger than this. +// batch_timeout_micros: Maximum number of microseconds to wait before outputting +// an incomplete batch. +// allowed_batch_sizes: Optional list of allowed batch sizes. If left empty, does +// nothing. Otherwise, supplies a list of batch sizes, causing the op to pad +// batches up to one of those sizes. The entries must increase monotonically, and +// the final entry must equal max_batch_size. +// grad_timeout_micros: The timeout to use for the gradient. See Unbatch. +// batched_tensors: Either empty tensors or a batch of concatenated Tensors. +// batch_index: If out_tensors is non-empty, has information to invert it. +// container: Controls the scope of sharing of this batch. +// id: always contains a scalar with a unique ID for this invocation of Batch. +// shared_name: Concurrently running instances of batch in the same device with the +// same container and shared_name will batch their elements together. If left +// empty, the op name will be used as the shared name. +// T: the types of tensors to be batched. +func Batch(scope *Scope, in_tensors []tf.Output, num_batch_threads int64, max_batch_size int64, batch_timeout_micros int64, grad_timeout_micros int64, optional ...BatchAttr) (batched_tensors []tf.Output, batch_index tf.Output, id tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"num_batch_threads": num_batch_threads, "max_batch_size": max_batch_size, "batch_timeout_micros": batch_timeout_micros, "grad_timeout_micros": grad_timeout_micros} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "Batch", + Input: []tf.Input{ + tf.OutputList(in_tensors), + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + if scope.Err() != nil { + return + } + var idx int + var err error + if batched_tensors, idx, err = makeOutputList(op, idx, "batched_tensors"); err != nil { + scope.UpdateErr("Batch", err) + return + } + batch_index = op.Output(idx) + id = op.Output(idx) + return batched_tensors, batch_index, id +} + +// UnicodeDecodeAttr is an optional argument to UnicodeDecode. +type UnicodeDecodeAttr func(optionalAttr) + +// UnicodeDecodeErrors sets the optional errors attribute to value. +// +// value: Error handling policy when there is invalid formatting found in the input. +// The value of 'strict' will cause the operation to produce a InvalidArgument +// error on any invalid input formatting. A value of 'replace' (the default) will +// cause the operation to replace any invalid formatting in the input with the +// `replacement_char` codepoint. A value of 'ignore' will cause the operation to +// skip any invalid formatting in the input and produce no corresponding output +// character. +// If not specified, defaults to "replace" +func UnicodeDecodeErrors(value string) UnicodeDecodeAttr { + return func(m optionalAttr) { + m["errors"] = value + } +} + +// UnicodeDecodeReplacementChar sets the optional replacement_char attribute to value. +// +// value: The replacement character codepoint to be used in place of any invalid +// formatting in the input when `errors='replace'`. Any valid unicode codepoint may +// be used. The default value is the default unicode replacement character is +// 0xFFFD or U+65533.) +// If not specified, defaults to 65533 +func UnicodeDecodeReplacementChar(value int64) UnicodeDecodeAttr { + return func(m optionalAttr) { + m["replacement_char"] = value + } +} + +// UnicodeDecodeReplaceControlCharacters sets the optional replace_control_characters attribute to value. +// +// value: Whether to replace the C0 control characters (00-1F) with the +// `replacement_char`. Default is false. +// If not specified, defaults to false +func UnicodeDecodeReplaceControlCharacters(value bool) UnicodeDecodeAttr { + return func(m optionalAttr) { + m["replace_control_characters"] = value + } +} + +// UnicodeDecodeTsplits sets the optional Tsplits attribute to value. +// If not specified, defaults to DT_INT64 +func UnicodeDecodeTsplits(value tf.DataType) UnicodeDecodeAttr { + return func(m optionalAttr) { + m["Tsplits"] = value + } +} + +// Decodes each string in `input` into a sequence of Unicode code points. +// +// The character codepoints for all strings are returned using a single vector +// `char_values`, with strings expanded to characters in row-major order. +// +// The `row_splits` tensor indicates where the codepoints for +// each input string begin and end within the `char_values` tensor. +// In particular, the values for the `i`th +// string (in row-major order) are stored in the slice +// `[row_splits[i]:row_splits[i+1]]`. Thus: +// +// * `char_values[row_splits[i]+j]` is the Unicode codepoint for the `j`th +// character in the `i`th string (in row-major order). +// * `row_splits[i+1] - row_splits[i]` is the number of characters in the `i`th +// string (in row-major order). +// +// Arguments: +// input: The text to be decoded. Can have any shape. Note that the output is flattened +// to a vector of char values. +// input_encoding: Text encoding of the input strings. This is any of the encodings supported +// by ICU ucnv algorithmic converters. Examples: `"UTF-16", "US ASCII", "UTF-8"`. +// +// Returns: +// row_splits: A 1D int32 tensor containing the row splits. +// char_values: A 1D int32 Tensor containing the decoded codepoints. +func UnicodeDecode(scope *Scope, input tf.Output, input_encoding string, optional ...UnicodeDecodeAttr) (row_splits tf.Output, char_values tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"input_encoding": input_encoding} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "UnicodeDecode", + Input: []tf.Input{ + input, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + // LSTMBlockCellAttr is an optional argument to LSTMBlockCell. type LSTMBlockCellAttr func(optionalAttr) @@ -43215,6 +42914,307 @@ func RetrieveTPUEmbeddingAdadeltaParametersGradAccumDebug(scope *Scope, num_shar return op.Output(0), op.Output(1), op.Output(2), op.Output(3) } +// Pads a tensor with mirrored values. +// +// This operation pads a `input` with mirrored values according to the `paddings` +// you specify. `paddings` is an integer tensor with shape `[n, 2]`, where n is +// the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates +// how many values to add before the contents of `input` in that dimension, and +// `paddings[D, 1]` indicates how many values to add after the contents of `input` +// in that dimension. Both `paddings[D, 0]` and `paddings[D, 1]` must be no greater +// than `input.dim_size(D)` (or `input.dim_size(D) - 1`) if `copy_border` is true +// (if false, respectively). +// +// The padded size of each dimension D of the output is: +// +// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)` +// +// For example: +// +// ``` +// # 't' is [[1, 2, 3], [4, 5, 6]]. +// # 'paddings' is [[1, 1]], [2, 2]]. +// # 'mode' is SYMMETRIC. +// # rank of 't' is 2. +// pad(t, paddings) ==> [[2, 1, 1, 2, 3, 3, 2] +// [2, 1, 1, 2, 3, 3, 2] +// [5, 4, 4, 5, 6, 6, 5] +// [5, 4, 4, 5, 6, 6, 5]] +// ``` +// +// Arguments: +// input: The input tensor to be padded. +// paddings: A two-column matrix specifying the padding sizes. The number of +// rows must be the same as the rank of `input`. +// mode: Either `REFLECT` or `SYMMETRIC`. In reflect mode the padded regions +// do not include the borders, while in symmetric mode the padded regions +// do include the borders. For example, if `input` is `[1, 2, 3]` and `paddings` +// is `[0, 2]`, then the output is `[1, 2, 3, 2, 1]` in reflect mode, and +// it is `[1, 2, 3, 3, 2]` in symmetric mode. +// +// Returns The padded tensor. +func MirrorPad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"mode": mode} + opspec := tf.OpSpec{ + Type: "MirrorPad", + Input: []tf.Input{ + input, paddings, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// TensorArrayV3Attr is an optional argument to TensorArrayV3. +type TensorArrayV3Attr func(optionalAttr) + +// TensorArrayV3ElementShape sets the optional element_shape attribute to value. +// +// value: The expected shape of an element, if known. Used to +// validate the shapes of TensorArray elements. If this shape is not +// fully specified, gathering zero-size TensorArrays is an error. +// If not specified, defaults to +func TensorArrayV3ElementShape(value tf.Shape) TensorArrayV3Attr { + return func(m optionalAttr) { + m["element_shape"] = value + } +} + +// TensorArrayV3DynamicSize sets the optional dynamic_size attribute to value. +// +// value: A boolean that determines whether writes to the TensorArray +// are allowed to grow the size. By default, this is not allowed. +// If not specified, defaults to false +func TensorArrayV3DynamicSize(value bool) TensorArrayV3Attr { + return func(m optionalAttr) { + m["dynamic_size"] = value + } +} + +// TensorArrayV3ClearAfterRead sets the optional clear_after_read attribute to value. +// +// value: If true (default), Tensors in the TensorArray are cleared +// after being read. This disables multiple read semantics but allows early +// release of memory. +// If not specified, defaults to true +func TensorArrayV3ClearAfterRead(value bool) TensorArrayV3Attr { + return func(m optionalAttr) { + m["clear_after_read"] = value + } +} + +// TensorArrayV3IdenticalElementShapes sets the optional identical_element_shapes attribute to value. +// +// value: If true (default is false), then all +// elements in the TensorArray will be expected to have have identical shapes. +// This allows certain behaviors, like dynamically checking for +// consistent shapes on write, and being able to fill in properly +// shaped zero tensors on stack -- even if the element_shape attribute +// is not fully defined. +// If not specified, defaults to false +func TensorArrayV3IdenticalElementShapes(value bool) TensorArrayV3Attr { + return func(m optionalAttr) { + m["identical_element_shapes"] = value + } +} + +// TensorArrayV3TensorArrayName sets the optional tensor_array_name attribute to value. +// +// value: Overrides the name used for the temporary tensor_array +// resource. Default value is the name of the 'TensorArray' op (which +// is guaranteed unique). +// If not specified, defaults to "" +func TensorArrayV3TensorArrayName(value string) TensorArrayV3Attr { + return func(m optionalAttr) { + m["tensor_array_name"] = value + } +} + +// An array of Tensors of given size. +// +// Write data via Write and read via Read or Pack. +// +// Arguments: +// size: The size of the array. +// dtype: The type of the elements on the tensor_array. +// +// Returns: +// handle: The handle to the TensorArray. +// flow: A scalar used to control gradient flow. +func TensorArrayV3(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV3Attr) (handle tf.Output, flow tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"dtype": dtype} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "TensorArrayV3", + Input: []tf.Input{ + size, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + +// MatrixSolveLsAttr is an optional argument to MatrixSolveLs. +type MatrixSolveLsAttr func(optionalAttr) + +// MatrixSolveLsFast sets the optional fast attribute to value. +// If not specified, defaults to true +func MatrixSolveLsFast(value bool) MatrixSolveLsAttr { + return func(m optionalAttr) { + m["fast"] = value + } +} + +// Solves one or more linear least-squares problems. +// +// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions +// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same +// type as `matrix` and shape `[..., M, K]`. +// The output is a tensor shape `[..., N, K]` where each output matrix solves +// each of the equations +// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]` +// in the least squares sense. +// +// We use the following notation for (complex) matrix and right-hand sides +// in the batch: +// +// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\), +// `rhs`=\\(B \in \mathbb{C}^{m \times k}\\), +// `output`=\\(X \in \mathbb{C}^{n \times k}\\), +// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\). +// +// If `fast` is `True`, then the solution is computed by solving the normal +// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then +// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares +// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 + \lambda ||Z||_F^2\\). +// If \\(m \lt n\\) then `output` is computed as +// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the +// minimum-norm solution to the under-determined linear system, i.e. +// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\), +// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable +// when \\(A\\) is numerically full rank and has a condition number +// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or \\(\lambda\\) is +// sufficiently large. +// +// If `fast` is `False` an algorithm based on the numerically robust complete +// orthogonal decomposition is used. This computes the minimum-norm +// least-squares solution, even when \\(A\\) is rank deficient. This path is +// typically 6-7 times slower than the fast path. If `fast` is `False` then +// `l2_regularizer` is ignored. +// +// Arguments: +// matrix: Shape is `[..., M, N]`. +// rhs: Shape is `[..., M, K]`. +// l2_regularizer: Scalar tensor. +// +// @compatibility(numpy) +// Equivalent to np.linalg.lstsq +// @end_compatibility +// +// Returns Shape is `[..., N, K]`. +func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "MatrixSolveLs", + Input: []tf.Input{ + matrix, rhs, l2_regularizer, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Generates sparse cross from a list of sparse and dense tensors. +// +// The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each +// representing features of one feature column. It outputs a 2D `SparseTensor` with +// the batchwise crosses of these features. +// +// For example, if the inputs are +// +// inputs[0]: SparseTensor with shape = [2, 2] +// [0, 0]: "a" +// [1, 0]: "b" +// [1, 1]: "c" +// +// inputs[1]: SparseTensor with shape = [2, 1] +// [0, 0]: "d" +// [1, 0]: "e" +// +// inputs[2]: Tensor [["f"], ["g"]] +// +// then the output will be +// +// shape = [2, 2] +// [0, 0]: "a_X_d_X_f" +// [1, 0]: "b_X_e_X_g" +// [1, 1]: "c_X_e_X_g" +// +// if hashed_output=true then the output will be +// +// shape = [2, 2] +// [0, 0]: FingerprintCat64( +// Fingerprint64("f"), FingerprintCat64( +// Fingerprint64("d"), Fingerprint64("a"))) +// [1, 0]: FingerprintCat64( +// Fingerprint64("g"), FingerprintCat64( +// Fingerprint64("e"), Fingerprint64("b"))) +// [1, 1]: FingerprintCat64( +// Fingerprint64("g"), FingerprintCat64( +// Fingerprint64("e"), Fingerprint64("c"))) +// +// Arguments: +// indices: 2-D. Indices of each input `SparseTensor`. +// values: 1-D. values of each `SparseTensor`. +// shapes: 1-D. Shapes of each `SparseTensor`. +// dense_inputs: 2-D. Columns represented by dense `Tensor`. +// hashed_output: If true, returns the hash of the cross instead of the string. +// This will allow us avoiding string manipulations. +// num_buckets: It is used if hashed_output is true. +// output = hashed_value%num_buckets if num_buckets > 0 else hashed_value. +// hash_key: Specify the hash_key that will be used by the `FingerprintCat64` +// function to combine the crosses fingerprints. +// +// +// +// Returns: +// output_indices: 2-D. Indices of the concatenated `SparseTensor`. +// output_values: 1-D. Non-empty values of the concatenated or hashed +// `SparseTensor`. +// output_shape: 1-D. Shape of the concatenated `SparseTensor`. +func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, dense_inputs []tf.Output, hashed_output bool, num_buckets int64, hash_key int64, out_type tf.DataType, internal_type tf.DataType) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"hashed_output": hashed_output, "num_buckets": num_buckets, "hash_key": hash_key, "out_type": out_type, "internal_type": internal_type} + opspec := tf.OpSpec{ + Type: "SparseCross", + Input: []tf.Input{ + tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), tf.OutputList(dense_inputs), + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1), op.Output(2) +} + // Reverses specific dimensions of a tensor. // // Given a `tensor`, and a `bool` tensor `dims` representing the dimensions From 52821ea9359775a16cfd31cc649d4b08d93bb53d Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 3 Aug 2020 15:50:36 -0700 Subject: [PATCH 0294/1017] Add error if calling `jacobian` or `batch_jacobian` on an exhausted tape. Currently, this is failing silently: returns None. PiperOrigin-RevId: 324700276 Change-Id: If5b4fc76bc3bfd2280ca67395015aca5bcf62f91 --- tensorflow/python/eager/backprop.py | 28 +++++++++++++++++++----- tensorflow/python/eager/backprop_test.py | 26 +++++++++++++++++++++- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 3c6ffc99fa4..7cb3abf4e07 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -997,6 +997,9 @@ class GradientTape(object): unconnected_gradients=UnconnectedGradients.NONE): """Computes the gradient using operations recorded in context of this tape. + Note: Unless you set `persistent=True` a GradientTape can only be used to + compute one set of gradients (or jacobians). + Args: target: a list or nested structure of Tensors or Variables to be differentiated. @@ -1015,14 +1018,14 @@ class GradientTape(object): the structure of `sources`. Raises: - RuntimeError: if called inside the context of the tape, or if called more - than once on a non-persistent tape. - ValueError: if the target is a variable or if unconnected gradients is + RuntimeError: If called on a used, non-persistent tape. + RuntimeError: If called inside the context of the tape. + ValueError: If the target is a variable or if unconnected gradients is called with an unknown value. """ if self._tape is None: - raise RuntimeError("GradientTape.gradient can only be called once on " - "non-persistent tapes.") + raise RuntimeError("A non-persistent GradientTape can only be used to" + "compute one set of gradients (or jacobians)") if self._recording: if not self._persistent: self._pop_tape() @@ -1101,6 +1104,9 @@ class GradientTape(object): experimental_use_pfor=True): """Computes the jacobian using operations recorded in context of this tape. + Note: Unless you set `persistent=True` a GradientTape can only be used to + compute one set of gradients (or jacobians). + See[wikipedia article](http://en.wikipedia.org/wiki/jacobian_matrix_and_determinant) for the definition of a Jacobian. @@ -1139,10 +1145,15 @@ class GradientTape(object): Raises: + RuntimeError: If called on a used, non-persistent tape. RuntimeError: If called on a non-persistent tape with eager execution enabled and without enabling experimental_use_pfor. ValueError: If vectorization of jacobian computation fails. """ + if self._tape is None: + raise RuntimeError("A non-persistent GradientTape can only be used to" + "compute one set of gradients (or jacobians)") + flat_sources = nest.flatten(sources) rewrap_as_ndarray = False if isinstance(target, np_arrays.ndarray): @@ -1225,6 +1236,9 @@ class GradientTape(object): are lower dimensional and avoid a bunch of redundant zeros which would result in the jacobian computation given the independence assumption. + Note: Unless you set `persistent=True` a GradientTape can only be used to + compute one set of gradients (or jacobians). + Example usage: ```python @@ -1255,11 +1269,15 @@ class GradientTape(object): per-example jacobians. Raises: + RuntimeError: If called on a used, non-persistent tape. RuntimeError: If called on a non-persistent tape with eager execution enabled and without enabling experimental_use_pfor. ValueError: If vectorization of jacobian computation fails or if first dimension of `target` and `source` do not match. """ + if self._tape is None: + raise RuntimeError("A non-persistent GradientTape can only be used to" + "compute one set of gradients (or jacobians)") rewrap_as_ndarray = False if isinstance(target, np_arrays.ndarray): target = target.data diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py index 6ae2a4c9a5e..0adb4698529 100644 --- a/tensorflow/python/eager/backprop_test.py +++ b/tensorflow/python/eager/backprop_test.py @@ -837,9 +837,33 @@ class BackpropTest(test.TestCase, parameterized.TestCase): z = y * y g.gradient(z, [x]) with self.assertRaisesRegex( - RuntimeError, 'GradientTape.gradient can only be called once'): + RuntimeError, 'A non-persistent GradientTape can only'): g.gradient(y, [x]) + @test_util.assert_no_new_tensors + def testGradientTapeJacobianCalledMultipleTimes(self): + with backprop.GradientTape() as g: + x = constant_op.constant(3.0) + g.watch(x) + y = x * x + z = y * y + g.jacobian(z, [x]) + with self.assertRaisesRegex( + RuntimeError, 'A non-persistent GradientTape can only'): + g.jacobian(y, [x]) + + @test_util.assert_no_new_tensors + def testGradientTapeBatchJacobianCalledMultipleTimes(self): + with backprop.GradientTape() as g: + x = constant_op.constant([[3.0]]) + g.watch(x) + y = x * x + z = y * y + g.batch_jacobian(z, x) + with self.assertRaisesRegex( + RuntimeError, 'A non-persistent GradientTape can only'): + g.batch_jacobian(y, [x]) + @test_util.assert_no_new_tensors @test_util.run_in_graph_and_eager_modes @test_util.run_v1_only('b/120545219') From 9474df4a1273601606018ae788a165e703253ed3 Mon Sep 17 00:00:00 2001 From: Rahul Joshi Date: Mon, 3 Aug 2020 16:04:39 -0700 Subject: [PATCH 0295/1017] [MLIR][NFC] Adopt SymbolTable::UseRange::empty() PiperOrigin-RevId: 324703092 Change-Id: Ieb4303522df0215cc2df2461e56dcfe25b3d834a --- tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc index 7a791afb24d..abff4c21cf1 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc @@ -113,7 +113,7 @@ bool HasSingleUse(FuncOp func) { // If no uses in this scope, continue looking in parent module SymbolTable::UseRange func_uses = func_uses_optional.getValue(); - if (llvm::empty(func_uses)) continue; + if (func_uses.empty()) continue; // Check if multiple uses at this scope or another use already seen. if (!llvm::hasSingleElement(func_uses) || use_seen) return false; From 3cf7683cfea4277223b940bfb5563efd541badd4 Mon Sep 17 00:00:00 2001 From: Russell Power Date: Mon, 3 Aug 2020 16:16:10 -0700 Subject: [PATCH 0296/1017] Refactor `TpuCompilationCacheEntry` interface to return `TpuProgramGroupInterface` and `core_index` and makes CacheEntry less transparent and move application specific logics outside of cache. PiperOrigin-RevId: 324705343 Change-Id: I9dc421df069dbe7dc9bb57695f06e8b636fbc945 --- tensorflow/core/tpu/kernels/BUILD | 28 ++- .../kernels/tpu_compilation_cache_entry.cc | 54 +++++ .../tpu/kernels/tpu_compilation_cache_entry.h | 26 ++- .../tpu_compilation_cache_entry_impl.h | 94 +++++++++ .../kernels/tpu_compilation_cache_external.cc | 53 ++--- .../kernels/tpu_compilation_cache_external.h | 12 ++ .../tpu_compilation_cache_interface.cc | 144 ++----------- .../kernels/tpu_compilation_cache_interface.h | 111 ++++++---- .../tpu_compilation_cache_local_lookup.cc | 43 +++- .../tpu_compilation_cache_local_lookup.h | 13 +- .../kernels/tpu_compilation_cache_lookup.h | 18 +- .../core/tpu/kernels/tpu_compile_op_common.cc | 40 ++++ .../core/tpu/kernels/tpu_compile_op_common.h | 9 + .../tpu/kernels/tpu_compile_op_support.cc | 38 ---- .../core/tpu/kernels/tpu_compile_op_support.h | 8 - .../core/tpu/kernels/tpu_configuration_ops.cc | 13 -- tensorflow/core/tpu/kernels/tpu_execute_op.cc | 58 +++--- .../core/tpu/kernels/tpu_program_c_api.h | 14 -- .../core/tpu/kernels/tpu_program_group.cc | 189 ++++++------------ .../core/tpu/kernels/tpu_program_group.h | 58 ++++-- .../tpu/kernels/tpu_program_group_interface.h | 7 +- tensorflow/core/tpu/tpu_library_init_fns.inc | 2 - 22 files changed, 523 insertions(+), 509 deletions(-) create mode 100644 tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.cc create mode 100644 tensorflow/core/tpu/kernels/tpu_compilation_cache_entry_impl.h diff --git a/tensorflow/core/tpu/kernels/BUILD b/tensorflow/core/tpu/kernels/BUILD index 1336f52ed34..3b7d0e09c08 100644 --- a/tensorflow/core/tpu/kernels/BUILD +++ b/tensorflow/core/tpu/kernels/BUILD @@ -92,8 +92,6 @@ tf_kernel_library( deps = [ ":tpu_compilation_cache_factory", ":tpu_compilation_cache_interface", - ":tpu_compilation_cache_local_lookup", - ":tpu_compilation_cache_lookup", ":tpu_mesh_state_interface", ":tpu_op_consts", "//tensorflow/c:tf_status", @@ -210,14 +208,30 @@ cc_library( cc_library( name = "tpu_compilation_cache_entry", + srcs = ["tpu_compilation_cache_entry.cc"], hdrs = [ "tpu_compilation_cache_entry.h", ], deps = [ + ":compiled_subgraph", + ":tpu_compilation_cache_proto_cc", ":tpu_executable_info_proto_cc", - ":tpu_program_group_interface", + ":tpu_program_group", "//tensorflow/compiler/xla/service:hlo_proto_cc", + "//tensorflow/core:framework", "//tensorflow/core/lib/core:refcount", + "//tensorflow/core/platform:casts", + ], +) + +cc_library( + name = "tpu_compilation_cache_entry_impl", + srcs = [], + hdrs = ["tpu_compilation_cache_entry_impl.h"], + deps = [ + ":compiled_subgraph", + ":tpu_compilation_cache_interface", + ":tpu_executable_info_proto_cc", ], ) @@ -288,8 +302,6 @@ cc_library( "//tensorflow/compiler/tf2xla:host_compute_metadata_proto_cc", "//tensorflow/compiler/xla/service:hlo_proto_cc", "//tensorflow/core/lib/core:status", - "@com_google_absl//absl/time", - "@com_google_absl//absl/types:span", ], ) @@ -329,7 +341,6 @@ cc_library( hdrs = ["tpu_compilation_cache_interface.h"], deps = [ ":compiled_subgraph", - ":tpu_compilation_cache_entry", ":tpu_compilation_cache_key", ":tpu_compilation_cache_proto_cc", ":tpu_compilation_metrics_hdrs", @@ -361,6 +372,7 @@ cc_library( deps = [ ":compiled_subgraph", ":tpu_compilation_cache_entry", + ":tpu_compilation_cache_entry_impl", ":tpu_compilation_cache_interface", ":tpu_compilation_cache_key", ":tpu_compilation_cache_proto_cc", @@ -370,7 +382,6 @@ cc_library( ":tpu_compile_op_support", ":tpu_mesh_state_interface", ":tpu_op_consts", - ":tpu_program_c_api_hdrs", ":tpu_program_group", ":tpu_util", ":trace_util_hdrs", @@ -380,10 +391,10 @@ cc_library( "//tensorflow/core:framework_internal", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", "//tensorflow/core/profiler/lib:traceme", "//tensorflow/core/protobuf/tpu:compile_metadata_proto_cc", "@com_google_absl//absl/container:node_hash_map", - "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@com_google_absl//absl/synchronization", "@com_google_absl//absl/types:span", @@ -604,7 +615,6 @@ cc_library( deps = [ ":tpu_compilation_cache_entry", ":tpu_compilation_cache_external", - ":tpu_compilation_cache_interface", ":tpu_compilation_cache_local_lookup", ":tpu_compilation_cache_lookup", ":tpu_executable_info_proto_cc", diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.cc b/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.cc new file mode 100644 index 00000000000..73f55853306 --- /dev/null +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.cc @@ -0,0 +1,54 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.h" + +#include "tensorflow/core/platform/casts.h" + +namespace tensorflow { +namespace tpu { + +TpuCompilationCacheEntry::TpuCompilationCacheEntry( + const TpuProgramGroupInterface* tpu_program_group, int core_index) + : tpu_program_group_( + tensorflow::down_cast(tpu_program_group)), + core_index_(core_index) {} + +// Constructor for an empty entry. +TpuCompilationCacheEntry::TpuCompilationCacheEntry() + : tpu_program_group_(nullptr) {} + +const TPUExecutableInfoProto* TpuCompilationCacheEntry::get_executable_info() + const { + return &(tpu_program_group_->executable_info()); +} + +const TPUHostTransferInfoProto* +TpuCompilationCacheEntry::get_host_transfer_info() const { + return &(tpu_program_group_->host_transfer_info()); +} + +const xla::HloProto* TpuCompilationCacheEntry::get_hlo_metadata() const { + return tpu_program_group_->hlo_metadatas()[core_index_]; +} + +// TODO(henrytan,jiawenhao): When should we expect more than one +// XLA_TpuProgram* per TpuProgram? Remove the program_count CHECK below then. +const XLA_TpuProgram* TpuCompilationCacheEntry::get_tpu_program() const { + CHECK_EQ(tpu_program_group_->program_count(), 1); + return tpu_program_group_->tpu_programs()[core_index_]; +} + +} // namespace tpu +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.h b/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.h index 832d76bfceb..b3766b8b4dd 100644 --- a/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.h +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.h @@ -18,32 +18,30 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/core/lib/core/refcount.h" #include "tensorflow/core/tpu/kernels/tpu_executable_info.pb.h" -#include "tensorflow/core/tpu/kernels/tpu_program_group_interface.h" +#include "tensorflow/core/tpu/kernels/tpu_program_group.h" namespace tensorflow { namespace tpu { -// Cache entry to hold a `TpuProgramGroupInterface` object that can be used to -// fetch a TPU program for a given TPU core index. +// A version of `CompilationCacheEntry` to access Tpu binary program +// `XLA_TpuProgram`. class TpuCompilationCacheEntry { public: explicit TpuCompilationCacheEntry( - const TpuProgramGroupInterface* tpu_program_group, int core_index) - : tpu_program_group_(tpu_program_group), core_index_(core_index) {} - + const TpuProgramGroupInterface* tpu_program_group, int core_index); // Constructor for an empty entry. - TpuCompilationCacheEntry() : tpu_program_group_(nullptr), core_index_(-1) {} - - const TpuProgramGroupInterface* tpu_program_group() const { - return tpu_program_group_; - } - - int core_index() const { return core_index_; } + TpuCompilationCacheEntry(); + const TPUExecutableInfoProto* get_executable_info() const; + const TPUHostTransferInfoProto* get_host_transfer_info() const; + const xla::HloProto* get_hlo_metadata() const; + // TODO(henrytan): maybe nicer to return C++ wrapper of `XLA_TpuProgram` + const XLA_TpuProgram* get_tpu_program() const; private: - const TpuProgramGroupInterface* tpu_program_group_; + const TpuProgramGroup* tpu_program_group_; int core_index_; }; + } // namespace tpu } // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry_impl.h b/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry_impl.h new file mode 100644 index 00000000000..0632d9a163f --- /dev/null +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_entry_impl.h @@ -0,0 +1,94 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_TPU_KERNELS_TPU_COMPILATION_CACHE_ENTRY_IMPL_H_ +#define TENSORFLOW_CORE_TPU_KERNELS_TPU_COMPILATION_CACHE_ENTRY_IMPL_H_ +#include "tensorflow/core/tpu/kernels/compiled_subgraph.h" +#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h" +#include "tensorflow/core/tpu/kernels/tpu_executable_info.pb.h" +namespace tensorflow { +namespace tpu { +// Wrapper for a cache entry that holds a reference to the entry until the +// wrapper is deleted. This wrapper is the concrete type of +// CompilationCacheEntryRef returned by Lookup. +template +class CompilationCacheEntryRefImpl + : public CompilationCacheEntryRef { + public: + CompilationCacheEntryRefImpl(TpuCompilationCacheInterface* parent, + CompiledSubgraph* entry, int index); + ~CompilationCacheEntryRefImpl() override; + Status ToSubEntryRef(CompilationCacheFetchTarget fetch_target) override; + + protected: + TpuCompilationCacheInterface* parent_; // Not owned. + // A reference to entry_ is acquired in the constructor and released via + // parent->DiscardEntryRefs in the destructor. + CompiledSubgraph* entry_; + // The index of the program in entry_ that is returned by the get method. + int index_; +}; +template +CompilationCacheEntryRefImpl::CompilationCacheEntryRefImpl( + TpuCompilationCacheInterface* parent, CompiledSubgraph* entry, int index) + : parent_(parent), entry_(entry), index_(index) { + if (entry_ == nullptr) { + return; + } + if (entry_->main_entry == nullptr) { + entry_->Ref(); + } else { + // This is a sharding/unsharding entry nested in a main entry. Only + // refcount the main entry. + entry_->main_entry->Ref(); + } +} +template +CompilationCacheEntryRefImpl::~CompilationCacheEntryRefImpl() { + if (entry_ == nullptr) { + return; + } + if (entry_->main_entry == nullptr) { + parent_->DiscardEntryRefs({entry_}); + } else { + parent_->DiscardEntryRefs({entry_->main_entry}); + } +} +template +Status CompilationCacheEntryRefImpl::ToSubEntryRef( + CompilationCacheFetchTarget fetch_target) { + CompiledSubgraph* target = nullptr; + switch (fetch_target) { + case CompilationCacheFetchTarget::MAIN: + target = entry_; + break; + case CompilationCacheFetchTarget::SHARDING: + target = entry_->sharding_entry.get(); + break; + case CompilationCacheFetchTarget::UNSHARDING: + target = entry_->unsharding_entry.get(); + break; + default: + return xla::InvalidArgument("Invalid fetch target: %d", fetch_target); + } + if (target == nullptr) { + // Cache entry does not have an unsharding subentry. Unref and replace + // with nullptr. + parent_->DiscardEntryRefs({entry_}); + } + // Otherwise, since the refcount is always on the main entry, we don't + // need ref/unref. + entry_ = target; + return Status::OK(); +} +} // namespace tpu +} // namespace tensorflow +#endif // TENSORFLOW_CORE_TPU_KERNELS_TPU_COMPILATION_CACHE_ENTRY_IMPL_H_ diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_external.cc b/tensorflow/core/tpu/kernels/tpu_compilation_cache_external.cc index 80010d70cd4..b4b18d1743b 100644 --- a/tensorflow/core/tpu/kernels/tpu_compilation_cache_external.cc +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_external.cc @@ -16,18 +16,15 @@ limitations under the License. #include -#include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/platform/random.h" #include "tensorflow/core/profiler/lib/traceme.h" -#include "tensorflow/core/tpu/kernels/compiled_subgraph.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_metrics.h" #include "tensorflow/core/tpu/kernels/tpu_compile_c_api.h" #include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" -#include "tensorflow/core/tpu/kernels/tpu_program_c_api.h" #include "tensorflow/core/tpu/kernels/tpu_util.h" #include "tensorflow/core/tpu/kernels/trace_util.h" @@ -51,22 +48,23 @@ void PopulateEntry(const std::string& key, CompiledSubgraph* entry, entry->tpu_program_group = absl::make_unique(std::move(tpu_program_group)); entry->initialized = true; - - if (entry->initialization_status.ok()) { - // Compute the entries total size once all members are initialized. - entry->total_size = entry->ComputeTotalSize(); - } -} - -std::unique_ptr CreateAndInitializeCompiledSubgraph( - CompiledSubgraph* main_entry) { - auto entry = absl::make_unique(); - entry->main_entry = main_entry; - entry->tpu_program_group = absl::make_unique(); - return entry; } } // namespace +TpuCompilationCacheExternal::EntryRefImpl::EntryRefImpl( + TpuCompilationCacheInterface* parent, CompiledSubgraph* entry, int index) + : CompilationCacheEntryRefImpl(parent, entry, + index) {} + +TpuCompilationCacheEntry TpuCompilationCacheExternal::EntryRefImpl::get() { + if (entry_ == nullptr) { + // Create an empty entry if the entry is nullptr. This corresponds to + // non-existing sharding/unsharding entries. + return TpuCompilationCacheEntry(); + } + return TpuCompilationCacheEntry(entry_->tpu_program_group.get(), index_); +} + CompiledSubgraph* TpuCompilationCacheExternal::InitializeEntry( const string& key, const std::function& initialize_program, @@ -75,6 +73,7 @@ CompiledSubgraph* TpuCompilationCacheExternal::InitializeEntry( main_entry->parent = this; main_entry->subgraph_key = key; main_entry->uid = get_uid(); + // TODO(henrytan): implement TpuCompilationCacheKey.debug_string. main_entry->cache_entry_debug_string = subgraph_key.prefix; VLOG(1) << "Cache Initializing Entry Session Debug " << main_entry->cache_entry_debug_string; @@ -113,29 +112,17 @@ CompiledSubgraph* TpuCompilationCacheExternal::InitializeEntry( std::pair(main_entry->uid, main_entry)); CHECK(uid_inserted.second); - if (tpu_program_group.has_sharding_program()) { - main_entry->sharding_entry = - CreateAndInitializeCompiledSubgraph(main_entry); - TpuProgramGroup sharding_programs; - sharding_programs.Initialize( - tpu_program_group.tpu_programs(TpuProgramShardingType::kSharding)); - PopulateEntry(key, main_entry->sharding_entry.get(), - std::move(sharding_programs)); - - main_entry->unsharding_entry = - CreateAndInitializeCompiledSubgraph(main_entry); - TpuProgramGroup unsharding_programs; - unsharding_programs.Initialize( - tpu_program_group.tpu_programs(TpuProgramShardingType::kUnsharding)); - PopulateEntry(key, main_entry->unsharding_entry.get(), - std::move(unsharding_programs)); + if (initialization_status.ok()) { + // Compute the entries total size once all members are initialized. + main_entry->total_size = tpu_program_group.program_size(); } + // TODO(henrytan): handle sharding/unsharding. PopulateEntry(key, main_entry, std::move(tpu_program_group)); for (int64 i = 0; i < main_entry->proto_key.size(); ++i) { auto entry_inserted = entries_by_proto_key_.insert( - std::pair>( + std::pair>( main_entry->proto_key[i], std::make_pair(main_entry, i))); CHECK(entry_inserted.second); } diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_external.h b/tensorflow/core/tpu/kernels/tpu_compilation_cache_external.h index 51b5ffbed0d..86615b15d4c 100644 --- a/tensorflow/core/tpu/kernels/tpu_compilation_cache_external.h +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_external.h @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/core/tpu/kernels/compiled_subgraph.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache.pb.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.h" +#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_entry_impl.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_key.h" #include "tensorflow/core/tpu/kernels/tpu_compile_c_api.h" @@ -45,6 +46,17 @@ namespace tpu { class TpuCompilationCacheExternal : public TpuCompilationCacheInterface { public: + using Status = ::stream_executor::port::Status; + + class EntryRefImpl + : public CompilationCacheEntryRefImpl { + public: + EntryRefImpl(TpuCompilationCacheInterface* parent, CompiledSubgraph* entry, + int index); + + TpuCompilationCacheEntry get() override; + }; + explicit TpuCompilationCacheExternal(int64 max_cache_size) : TpuCompilationCacheInterface(max_cache_size) {} diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.cc b/tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.cc index 4cd2b864203..9e1aedf92ce 100644 --- a/tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.cc +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.cc @@ -38,77 +38,10 @@ void TpuCompilationCacheInterface::RefHolder::AddRef(CompiledSubgraph* entry) { entries_.push_back(entry); } -std::string TpuCompilationCacheInterface::RefHolder::DebugString() const { +string TpuCompilationCacheInterface::RefHolder::DebugString() const { return "TpuCompilationCacheRefHolder"; } -CompilationCacheEntryRef::CompilationCacheEntryRef() - : parent_(nullptr), entry_(nullptr), index_(0) {} - -CompilationCacheEntryRef::CompilationCacheEntryRef( - TpuCompilationCacheInterface* parent, CompiledSubgraph* entry, int index) - : parent_(parent), entry_(entry), index_(index) { - if (entry_ == nullptr) { - return; - } - if (entry_->main_entry == nullptr) { - entry_->Ref(); - } else { - // This is a sharding/unsharding entry nested in a main entry. Only - // refcount the main entry. - entry_->main_entry->Ref(); - } -} - -CompilationCacheEntryRef::~CompilationCacheEntryRef() { - if (entry_ == nullptr) { - return; - } - if (entry_->main_entry == nullptr) { - parent_->DiscardEntryRefs({entry_}); - } else { - parent_->DiscardEntryRefs({entry_->main_entry}); - } -} - -TpuCompilationCacheEntry CompilationCacheEntryRef::get() { - if (entry_ == nullptr) { - // Create an empty entry if the entry is nullptr. This corresponds to - // non-existing sharding/unsharding entries. - return TpuCompilationCacheEntry(); - } - - return TpuCompilationCacheEntry(entry_->tpu_program_group.get(), index_); -} - -Status CompilationCacheEntryRef::ToSubEntryRef( - CompilationCacheFetchTarget fetch_target) { - CompiledSubgraph* target = nullptr; - switch (fetch_target) { - case CompilationCacheFetchTarget::MAIN: - target = entry_; - break; - case CompilationCacheFetchTarget::SHARDING: - target = entry_->sharding_entry.get(); - break; - case CompilationCacheFetchTarget::UNSHARDING: - target = entry_->unsharding_entry.get(); - break; - default: - return xla::InvalidArgument("Invalid fetch target: %d", fetch_target); - } - - if (target == nullptr) { - // Cache entry does not have an unsharding subentry. Unref and replace - // with nullptr. - parent_->DiscardEntryRefs({entry_}); - } - // Otherwise, since the refcount is always on the main entry, we don't - // need ref/unref. - entry_ = target; - return Status::OK(); -} - TpuCompilationCacheInterface::TpuCompilationCacheInterface(int64 max_cache_size) : max_cache_size_(max_cache_size) { CHECK_GE(max_cache_size_, 0); @@ -223,7 +156,7 @@ void TpuCompilationCacheInterface::UnloadAndDestroy(CompiledSubgraph* entry) { entry->Unref(); } -size_t TpuCompilationCacheInterface::RemoveEntry(const std::string& key) { +size_t TpuCompilationCacheInterface::RemoveEntry(const string& key) { auto erased = cache_.erase(key); TpuCompilationMetrics::SetCacheEntryCount(cache_.size()); @@ -263,7 +196,7 @@ CompiledSubgraph* TpuCompilationCacheInterface::DiscardEntryRef( } erased = entries_by_uid_.erase(entry->uid); CHECK_EQ(erased, 1); - for (const std::string& key : entry->proto_key) { + for (const string& key : entry->proto_key) { erased = entries_by_proto_key_.erase(key); CHECK_EQ(erased, 1); } @@ -336,10 +269,10 @@ void TpuCompilationCacheInterface::LookupEntryMarkedForEviction( } } -void TpuCompilationCacheInterface::InsertEntry(const std::string& key, +void TpuCompilationCacheInterface::InsertEntry(const string& key, CompiledSubgraph* entry) { auto cache_inserted = - cache_.insert(std::pair(key, entry)); + cache_.insert(std::pair(key, entry)); CHECK(cache_inserted.second); TpuCompilationMetrics::SetCacheEntryCount(cache_.size()); @@ -362,8 +295,7 @@ Status TpuCompilationCacheInterface::CompileIfKeyAbsent( const TpuCompilationCacheKey& subgraph_key, const SessionMetadata* session_metadata, CompilationRefHolder* per_step_ref_holder, int64* uid, - std::vector* proto_key, - std::vector* may_modify_variables, + std::vector* proto_key, std::vector* may_modify_variables, absl::Span* hlo_metadatas, const std::function& compile_function) { std::vector removed_entries; @@ -376,7 +308,7 @@ Status TpuCompilationCacheInterface::CompileIfKeyAbsent( return status; } -std::string TpuCompilationCacheInterface::FindCacheKey( +string TpuCompilationCacheInterface::FindCacheKey( const TpuCompilationCacheKey& subgraph_key) { if (!subgraph_key.has_guaranteed_const) { return subgraph_key.prefix; @@ -399,8 +331,7 @@ Status TpuCompilationCacheInterface::CompileIfKeyAbsentHelper( const TpuCompilationCacheKey& subgraph_key, const SessionMetadata* session_metadata, CompilationRefHolder* per_step_ref_holder, int64* uid, - std::vector* proto_key, - std::vector* may_modify_variables, + std::vector* proto_key, std::vector* may_modify_variables, std::vector* removed_entries, absl::Span* hlo_metadatas, const std::function& compile_function) { @@ -414,18 +345,17 @@ Status TpuCompilationCacheInterface::CompileIfKeyAbsentHelper( // for the lifetime of the object, see InitializeEntry() call below. absl::MutexLock lock(&mu_); - std::string cache_key = FindCacheKey(subgraph_key); + string cache_key = FindCacheKey(subgraph_key); auto iter = cache_.find(cache_key); bool is_new_key = iter == cache_.end(); - const std::string session_name = - tpu::SessionNameFromMetadata(session_metadata); + const string session_name = tpu::SessionNameFromMetadata(session_metadata); if (is_new_key) { cache_key = subgraph_key.ToString(); TpuCompilationMetrics::IncrementCacheLookupCount( /*is_cache_hit=*/false, session_name); - const std::string msg = + const string msg = strings::StrCat("TPU host compilation cache miss: cache_key(", cache_key, "), session_name(", session_name, ")"); TRACESTRING(msg); @@ -434,7 +364,7 @@ Status TpuCompilationCacheInterface::CompileIfKeyAbsentHelper( // Check if caller has disabled compilation. Set using // internal::ScopedTpuCompileDisabler. if (!UtilApiFn()->TpuCompile_IsTpuCompilationEnabledFn()) { - const std::string error_msg = strings::StrCat( + const string error_msg = strings::StrCat( "[TpuCompilationDisabled]: Compilation cache miss, but compilation " "disabled, session_name(", session_name, ") Debug String: ", subgraph_key.debug_string); @@ -473,7 +403,7 @@ Status TpuCompilationCacheInterface::CompileIfKeyAbsentHelper( } else { TpuCompilationMetrics::IncrementCacheLookupCount( /*is_cache_hit=*/true, session_name); - const std::string msg = + const string msg = strings::StrCat("TPU host compilation cache hit: cache_key(", cache_key, "), session_name(", session_name, ")"); TRACESTRING(msg); @@ -536,8 +466,8 @@ Status TpuCompilationCacheInterface::CompileIfKeyAbsentHelper( return entry->initialization_status; } -Status TpuCompilationCacheInterface::GetKeysFromUid( - int64 uid, std::vector* keys) { +Status TpuCompilationCacheInterface::GetKeysFromUid(int64 uid, + std::vector* keys) { keys->clear(); absl::MutexLock lock(&mu_); @@ -549,49 +479,5 @@ Status TpuCompilationCacheInterface::GetKeysFromUid( return Status::OK(); } -Status TpuCompilationCacheInterface::Lookup( - int64 uid, int proto_index, - std::unique_ptr* entry) { - entry->reset(); - - profiler::TraceMe proto_lookup_traceme( - "TPU compilation cache proto lookup by uid", - /*level=*/2); - - absl::MutexLock lock(&mu_); - const auto iter = entries_by_uid_.find(uid); - if (iter == entries_by_uid_.end()) { - return errors::NotFound("No subgraph found for uid ", uid); - } - CompiledSubgraph* cache_entry = iter->second; - if (proto_index < 0 || - proto_index >= cache_entry->tpu_program_group->program_count()) { - return errors::NotFound("No proto found for core index ", proto_index, - " in subgraph with uid ", uid); - } - *entry = absl::make_unique(this, cache_entry, - proto_index); - return Status::OK(); -} - -Status TpuCompilationCacheInterface::Lookup( - const std::string& proto_key, - std::unique_ptr* entry) { - entry->reset(); - - profiler::TraceMe proto_lookup_traceme("TPU compilation cache proto lookup", - /*level=*/2); - - absl::MutexLock lock(&mu_); - const auto iter = entries_by_proto_key_.find(proto_key); - if (iter == entries_by_proto_key_.end()) { - return errors::NotFound("No proto found for key ", proto_key); - } - CompiledSubgraph* cache_entry = iter->second.first; - int proto_index = iter->second.second; - *entry = absl::make_unique(this, cache_entry, - proto_index); - return Status::OK(); -} } // namespace tpu } // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h b/tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h index 7b206fb1cf4..cde6467b7af 100644 --- a/tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h @@ -32,7 +32,6 @@ limitations under the License. #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/tpu/kernels/compiled_subgraph.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache.pb.h" -#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_key.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_metrics.h" #include "tensorflow/core/tpu/kernels/trace_util.h" @@ -49,20 +48,18 @@ class CompilationRefHolder : public ResourceBase { ~CompilationRefHolder() override = default; }; -// Wrapper for a cache entry returned by all the TpuCompilationCacheInterface -// `Lookup` methods, and ensures the underlying proto is not garbage-collected -// until the client discards the ptr. +// Base class for a reference to a cached tpu program. A unique_ptr to a +// CompilationCacheEntryRef is returned by all the cache Lookup methods below, +// and ensures the underlying proto is not garbage-collected until the client +// discards the ptr. +template class CompilationCacheEntryRef { public: - CompilationCacheEntryRef(); - CompilationCacheEntryRef(TpuCompilationCacheInterface* parent, - CompiledSubgraph* entry, int index); + virtual ~CompilationCacheEntryRef() = default; - virtual ~CompilationCacheEntryRef(); - - // Returns a TpuCompilationCacheEntry that should not be used beyond the - // lifetime of the CompilationCacheEntryRef. - virtual TpuCompilationCacheEntry get(); + // Returns a CompilationCacheEntry that should not be used beyond the lifetime + // of the tpu::CompilationCacheEntryRef. + virtual CacheEntryType get() = 0; // Mutates this ref to point to the entry's subentry (for // sharding/unsharding) or main entry (unchanged) as specified by @@ -72,15 +69,7 @@ class CompilationCacheEntryRef { // // If the requested subentry does not exist, the ref will point to a nullptr // entry, and the original entry will be unref'ed. - virtual Status ToSubEntryRef(CompilationCacheFetchTarget fetch_target); - - protected: - TpuCompilationCacheInterface* parent_; // Not owned. - // A reference to entry_ is acquired in the constructor and released via - // parent->DiscardEntryRefs in the destructor. - CompiledSubgraph* entry_; - // The index of the program in entry_ that is returned by the get method. - int index_; + virtual Status ToSubEntryRef(CompilationCacheFetchTarget fetch_target) = 0; }; class TpuCompilationCacheInterface : public ResourceBase { @@ -108,8 +97,7 @@ class TpuCompilationCacheInterface : public ResourceBase { const TpuCompilationCacheKey& subgraph_key, const SessionMetadata* session_metadata, CompilationRefHolder* per_step_ref_holder, int64* uid, - std::vector* proto_key, - std::vector* may_modify_variables, + std::vector* proto_key, std::vector* may_modify_variables, absl::Span* hlo_metadatas, const std::function& compile_function); @@ -136,18 +124,19 @@ class TpuCompilationCacheInterface : public ResourceBase { // Looks up an executable corresponding to the model-parallel core index of // the subgraph represented by key. On success a pointer to an EntryRef // holding the program is returned in entry. - Status Lookup(const std::string& proto_key, - std::unique_ptr* entry); + template + Status Lookup(const string& proto_key, std::unique_ptr* entry); // Looks up an executable corresponding to the model-parallel core index of // the subgraph represented by uid. On success a pointer to an EntryRef // holding the program is returned in entry. + template Status Lookup(int64 uid, int proto_index, - std::unique_ptr* entry); + std::unique_ptr* entry); // Looks up the subgraph represented by uid, and returns the vector of keys, // one per core, corresponding to that subgraph. - Status GetKeysFromUid(int64 uid, std::vector* keys); + Status GetKeysFromUid(int64 uid, std::vector* keys); // Makes a reference holder for this cache, that can be stored in the per-step // resource manager and will ensure that compiled entries persist until the @@ -181,7 +170,7 @@ class TpuCompilationCacheInterface : public ResourceBase { // parent_->DiscardEntryRefs. void AddRef(CompiledSubgraph* entry); - std::string DebugString() const override; + string DebugString() const override; private: TpuCompilationCacheInterface* parent_; // Not owned. @@ -196,8 +185,7 @@ class TpuCompilationCacheInterface : public ResourceBase { const TpuCompilationCacheKey& subgraph_key, const SessionMetadata* session_metadata, CompilationRefHolder* per_step_ref_holder, int64* uid, - std::vector* proto_key, - std::vector* may_modify_variables, + std::vector* proto_key, std::vector* may_modify_variables, std::vector* removed_entries, absl::Span* hlo_metadatas, const std::function& compile_function); @@ -242,14 +230,14 @@ class TpuCompilationCacheInterface : public ResourceBase { ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Removes the entry with given key from cache. - size_t RemoveEntry(const std::string& key) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + size_t RemoveEntry(const string& key) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Inserts the given key and entry to cache. - void InsertEntry(const std::string& key, CompiledSubgraph* entry) + void InsertEntry(const string& key, CompiledSubgraph* entry) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Returns the cache key matching given subgraph_key. - std::string FindCacheKey(const TpuCompilationCacheKey& subgraph_key) + string FindCacheKey(const TpuCompilationCacheKey& subgraph_key) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Creates a new entry by running initialize_programs and places it in the @@ -259,7 +247,7 @@ class TpuCompilationCacheInterface : public ResourceBase { // // **InitializeEntry releases mu_ during the call to initialize_programs.** virtual CompiledSubgraph* InitializeEntry( - const std::string& key, + const string& key, const std::function& initialize_programs, const TpuCompilationCacheKey& subgraph_key) @@ -288,16 +276,13 @@ class TpuCompilationCacheInterface : public ResourceBase { // cache_ key matching a given subgraph key. When doing a lookup, check // session_key_map_ first to avoid unnecessay fingerprint computation. // Map from key prefix + session_handle to a cache_ key. - absl::node_hash_map session_key_map_ - ABSL_GUARDED_BY(mu_); + absl::node_hash_map session_key_map_ ABSL_GUARDED_BY(mu_); // Map from key prefix + fingerprint to a cache_ key. - absl::node_hash_map fingerprint_key_map_ - ABSL_GUARDED_BY(mu_); + absl::node_hash_map fingerprint_key_map_ ABSL_GUARDED_BY(mu_); // All the subgraph entries that can be looked up in the cache. An entry is // marked for eviction iff it is present in cache_ and not in // entries_by_last_use_. - std::unordered_map cache_ - ABSL_GUARDED_BY(mu_); + std::unordered_map cache_ ABSL_GUARDED_BY(mu_); // All the subgraph entries that can be looked up in the cache, indexed by // uid. absl::node_hash_map entries_by_uid_ @@ -305,7 +290,7 @@ class TpuCompilationCacheInterface : public ResourceBase { // All the protos that can be looked up in the cache, indexed by proto // key. The value of the map is a subgraph and the index of the proto compiled // for that subgraph. - std::unordered_map> + std::unordered_map> entries_by_proto_key_ ABSL_GUARDED_BY(mu_); // Map from last_use to entry, used to mark entries for eviction in LRU // order. If an entry's last_use counter is not present as a key in @@ -319,6 +304,50 @@ class TpuCompilationCacheInterface : public ResourceBase { TpuCompilationCacheInterface& operator=(const TpuCompilationCacheInterface&) = delete; }; + +template +Status TpuCompilationCacheInterface::Lookup( + int64 uid, int proto_index, std::unique_ptr* entry) { + entry->reset(); + + profiler::TraceMe proto_lookup_traceme( + "TPU compilation cache proto lookup by uid", + /*level=*/2); + + absl::MutexLock lock(&mu_); + const auto iter = entries_by_uid_.find(uid); + if (iter == entries_by_uid_.end()) { + return errors::NotFound("No subgraph found for uid ", uid); + } + CompiledSubgraph* cache_entry = iter->second; + if (proto_index < 0 || + proto_index >= cache_entry->tpu_program_group->program_count()) { + return errors::NotFound("No proto found for core index ", proto_index, + " in subgraph with uid ", uid); + } + *entry = absl::make_unique(this, cache_entry, proto_index); + return Status::OK(); +} + +template +Status TpuCompilationCacheInterface::Lookup( + const string& proto_key, std::unique_ptr* entry) { + entry->reset(); + + profiler::TraceMe proto_lookup_traceme("TPU compilation cache proto lookup", + /*level=*/2); + + absl::MutexLock lock(&mu_); + const auto iter = entries_by_proto_key_.find(proto_key); + if (iter == entries_by_proto_key_.end()) { + return errors::NotFound("No proto found for key ", proto_key); + } + CompiledSubgraph* cache_entry = iter->second.first; + int proto_index = iter->second.second; + *entry = absl::make_unique(this, cache_entry, proto_index); + return Status::OK(); +} + } // namespace tpu } // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.cc b/tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.cc index 29864a310d1..f30a503d2d2 100644 --- a/tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.cc +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.cc @@ -16,50 +16,70 @@ limitations under the License. namespace tensorflow { namespace tpu { +namespace { +class CompilationCacheFetchTargetUtility { + public: + CompilationCacheFetchTargetUtility() + : names_({"Invalid", "Main", "Sharding", "Unsharding"}) {} + + std::string name(CompilationCacheFetchTarget target) const { + return names_[static_cast(target)]; + } + + private: + const std::vector names_; +}; + +std::string GetName(CompilationCacheFetchTarget target) { + static const auto* util = new CompilationCacheFetchTargetUtility(); + return util->name(target); +} + +} // namespace TpuCompilationCacheLocalLookup::TpuCompilationCacheLocalLookup( TpuCompilationCacheInterface* cache) - : cache_(cache) { - cache_->Ref(); -} + : cache_(cache) {} TpuCompilationCacheLocalLookup::~TpuCompilationCacheLocalLookup() { cache_->Unref(); } Status TpuCompilationCacheLocalLookup::Lookup( - const string& proto_key, std::unique_ptr* entry, + const string& proto_key, + std::unique_ptr* entry, CompilationCacheFetchTarget fetch_target) { profiler::TraceMe proto_lookup_traceme("Local TPU proto cache lookup", /*level=*/2); - Status s = cache_->Lookup(proto_key, entry); + Status s = cache_->Lookup( + proto_key, entry); VLOG(1) << "Looked up key " << proto_key << " in local subgraph cache status " << s; if (!s.ok()) { return s; } s = (*entry)->ToSubEntryRef(fetch_target); - VLOG(1) << "Fetched subentry: " - << CompilationCacheFetchTarget_Name(fetch_target) << " with status " + + VLOG(1) << "Fetched subentry: " << GetName(fetch_target) << " with status " << s; return s; } Status TpuCompilationCacheLocalLookup::Lookup( int64 uid, int proto_index, - std::unique_ptr* entry, + std::unique_ptr* entry, CompilationCacheFetchTarget fetch_target) { profiler::TraceMe proto_lookup_traceme("Local TPU proto cache lookup by uid", /*level=*/2); - Status s = cache_->Lookup(uid, proto_index, entry); + Status s = cache_->Lookup( + uid, proto_index, entry); VLOG(1) << "Looked up uid " << uid << ", index " << proto_index << " in local subgraph cache status " << s; if (!s.ok()) { return s; } s = (*entry)->ToSubEntryRef(fetch_target); - VLOG(1) << "Fetched subentry: " - << CompilationCacheFetchTarget_Name(fetch_target) << " with status " + VLOG(1) << "Fetched subentry: " << GetName(fetch_target) << " with status " << s; return s; } @@ -67,5 +87,6 @@ Status TpuCompilationCacheLocalLookup::Lookup( string TpuCompilationCacheLocalLookup::DebugString() const { return "TpuCompilationCacheLocalLookup"; } + } // namespace tpu } // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.h b/tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.h index 8db4c11ebea..eb5aadcd3e2 100644 --- a/tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.h +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.h @@ -28,17 +28,24 @@ namespace tpu { // Class for looking up TPU programs when the execute and compile Op are in the // same address space. The proto is simply looked up in the compilation cache, // without any serialization taking place. -class TpuCompilationCacheLocalLookup : public TpuCompilationCacheLookup { +class TpuCompilationCacheLocalLookup + : public TpuCompilationCacheLookup< + CompilationCacheEntryRef> { public: + using TpuCompilationCacheEntryRef = + ::tensorflow::tpu::CompilationCacheEntryRef; + using EntryRefImpl = + ::tensorflow::tpu::TpuCompilationCacheExternal::EntryRefImpl; + explicit TpuCompilationCacheLocalLookup(TpuCompilationCacheInterface* cache); ~TpuCompilationCacheLocalLookup() override; Status Lookup(const string& proto_key, - std::unique_ptr* entry, + std::unique_ptr* entry, CompilationCacheFetchTarget fetch_target) override; Status Lookup(int64 uid, int proto_index, - std::unique_ptr* entry, + std::unique_ptr* entry, CompilationCacheFetchTarget fetch_target) override; string DebugString() const override; diff --git a/tensorflow/core/tpu/kernels/tpu_compilation_cache_lookup.h b/tensorflow/core/tpu/kernels/tpu_compilation_cache_lookup.h index ab476322a8a..0d1a53d31d2 100644 --- a/tensorflow/core/tpu/kernels/tpu_compilation_cache_lookup.h +++ b/tensorflow/core/tpu/kernels/tpu_compilation_cache_lookup.h @@ -23,11 +23,10 @@ limitations under the License. namespace tensorflow { namespace tpu { -// TODO(b/162241759): consider merging TpuCompilationCacheLookup and -// TpuCompilationCacheInterface. // Base class allowing Execute Ops to look up TPU programs. Different subclasses // are used when the execute Op is in the same address space as the compile Op, // and when they need to communicate over RPC. +template class TpuCompilationCacheLookup : public ResourceBase { public: ~TpuCompilationCacheLookup() override = default; @@ -44,11 +43,12 @@ class TpuCompilationCacheLookup : public ResourceBase { // fetch_target requests one of them, then after this call // (*entry)->get().get_executable() will return nullptr. virtual Status Lookup(const string& proto_key, - std::unique_ptr* entry, + std::unique_ptr* entry, CompilationCacheFetchTarget fetch_target) = 0; - virtual Status Lookup(const string& proto_key, - std::unique_ptr* entry) { + virtual Status Lookup( + const string& proto_key, + std::unique_ptr* entry) { return Lookup(proto_key, std::move(entry), CompilationCacheFetchTarget::MAIN); } @@ -58,15 +58,17 @@ class TpuCompilationCacheLookup : public ResourceBase { // returned in program. The wrapper is guaranteed to be valid only during the // execution of the Op requesting the proto. virtual Status Lookup(int64 uid, int proto_index, - std::unique_ptr* entry, + std::unique_ptr* entry, CompilationCacheFetchTarget fetch_target) = 0; - virtual Status Lookup(int64 uid, int proto_index, - std::unique_ptr* entry) { + virtual Status Lookup( + int64 uid, int proto_index, + std::unique_ptr* entry) { return Lookup(uid, proto_index, std::move(entry), CompilationCacheFetchTarget::MAIN); } }; + } // namespace tpu } // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_compile_op_common.cc b/tensorflow/core/tpu/kernels/tpu_compile_op_common.cc index ce18e844e66..4ed646af302 100644 --- a/tensorflow/core/tpu/kernels/tpu_compile_op_common.cc +++ b/tensorflow/core/tpu/kernels/tpu_compile_op_common.cc @@ -413,6 +413,46 @@ Status TpuCompileOpKernelCommon::CompileTFFunctionToHlo( return Status::OK(); } +/* static */ +Status TpuCompileOpKernelCommon::ComputeArgumentShapes( + const tpu::TPUCompileMetadataProto& metadata, + const std::vector& dynamic_shapes, + std::vector* arg_shapes) { + arg_shapes->resize(metadata.args_size()); + int dynamic_shape_pos = 0; + for (int i = 0; i < metadata.args_size(); ++i) { + const tpu::TPUCompileMetadataProto::Arg& arg = metadata.args(i); + // The XLA compiler determines the shape of each constant by inspecting the + // value of its corresponding host-memory tensor. As a result, we don't need + // to give the compiler graph-inferred shapes for constant arguments. + if (arg.kind() == tpu::TPUCompileMetadataProto::Arg::GUARANTEED_CONSTANT) { + continue; + } + TF_RETURN_IF_ERROR(PartialTensorShape::IsValidShape(arg.shape())); + PartialTensorShape static_shape(arg.shape()); + + TensorShape& shape = (*arg_shapes)[i]; + if (static_shape.IsFullyDefined()) { + TF_RET_CHECK(static_shape.AsTensorShape(&shape)); + } else { + TF_RET_CHECK(dynamic_shape_pos < dynamic_shapes.size()) + << "Too few dynamic shapes"; + shape = dynamic_shapes[dynamic_shape_pos++]; + if (!static_shape.IsCompatibleWith(shape)) { + return errors::InvalidArgument( + "Mismatch between static and dynamic shape for argument. Static " + "shape: ", + static_shape.DebugString(), + "; dynamic shape: ", shape.DebugString()); + } + } + } + // Checks we consumed all of the dynamic shapes. + TF_RET_CHECK(dynamic_shape_pos == dynamic_shapes.size()) + << "Too many dynamic shapes"; + return Status::OK(); +} + // Function arguments and return values lose their device assignments, so we // must recreate them. /* static */ Status TpuCompileOpKernelCommon::AssignDevicesToArgsAndRetvals( diff --git a/tensorflow/core/tpu/kernels/tpu_compile_op_common.h b/tensorflow/core/tpu/kernels/tpu_compile_op_common.h index 327aa460ddd..3d3f0afcdb7 100644 --- a/tensorflow/core/tpu/kernels/tpu_compile_op_common.h +++ b/tensorflow/core/tpu/kernels/tpu_compile_op_common.h @@ -99,6 +99,15 @@ class TpuCompileOpKernelCommon { const std::vector& arg_shapes, TpuProgramGroupInterface* tpu_program_group) = 0; + // Computes shapes for each argument. Uses both the static shape from the + // metadata, and the dynamic shapes where the static shape is not + // defined. There must be one dynamic_shape for each argument with a + // partially defined shape, in index order. + static Status ComputeArgumentShapes( + const tpu::TPUCompileMetadataProto& metadata, + const std::vector& dynamic_shapes, + std::vector* arg_shapes); + // Performs shape inference on `computation`, filling shape_info with operator // shapes. The shapes of the _Arg nodes are taken from `arg_shapes`. static Status RunShapeInferenceOnComputation( diff --git a/tensorflow/core/tpu/kernels/tpu_compile_op_support.cc b/tensorflow/core/tpu/kernels/tpu_compile_op_support.cc index 3440b6d265a..5cc35a07e66 100644 --- a/tensorflow/core/tpu/kernels/tpu_compile_op_support.cc +++ b/tensorflow/core/tpu/kernels/tpu_compile_op_support.cc @@ -540,43 +540,5 @@ Status CompileOpMetadataFromContext(OpKernelConstruction* ctx, } return Status::OK(); } - -Status ComputeArgumentShapes(const tpu::TPUCompileMetadataProto& metadata, - const std::vector& dynamic_shapes, - std::vector* arg_shapes) { - arg_shapes->resize(metadata.args_size()); - int dynamic_shape_pos = 0; - for (int i = 0; i < metadata.args_size(); ++i) { - const tpu::TPUCompileMetadataProto::Arg& arg = metadata.args(i); - // The XLA compiler determines the shape of each constant by inspecting the - // value of its corresponding host-memory tensor. As a result, we don't need - // to give the compiler graph-inferred shapes for constant arguments. - if (arg.kind() == tpu::TPUCompileMetadataProto::Arg::GUARANTEED_CONSTANT) { - continue; - } - TF_RETURN_IF_ERROR(PartialTensorShape::IsValidShape(arg.shape())); - PartialTensorShape static_shape(arg.shape()); - - TensorShape& shape = (*arg_shapes)[i]; - if (static_shape.IsFullyDefined()) { - TF_RET_CHECK(static_shape.AsTensorShape(&shape)); - } else { - TF_RET_CHECK(dynamic_shape_pos < dynamic_shapes.size()) - << "Too few dynamic shapes"; - shape = dynamic_shapes[dynamic_shape_pos++]; - if (!static_shape.IsCompatibleWith(shape)) { - return errors::InvalidArgument( - "Mismatch between static and dynamic shape for argument. Static " - "shape: ", - static_shape.DebugString(), - "; dynamic shape: ", shape.DebugString()); - } - } - } - // Checks we consumed all of the dynamic shapes. - TF_RET_CHECK(dynamic_shape_pos == dynamic_shapes.size()) - << "Too many dynamic shapes"; - return Status::OK(); -} } // namespace tpu } // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_compile_op_support.h b/tensorflow/core/tpu/kernels/tpu_compile_op_support.h index ea13d33b521..bc60f64286a 100644 --- a/tensorflow/core/tpu/kernels/tpu_compile_op_support.h +++ b/tensorflow/core/tpu/kernels/tpu_compile_op_support.h @@ -159,14 +159,6 @@ se::port::Status CompileOpMetadataFromContext(OpKernelConstruction* ctx, TPUCompileMetadataProto* metadata, NameAttrList* function_name, std::string* mlir_module); - -// Computes shapes for each argument. Uses both the static shape from the -// metadata, and the dynamic shapes where the static shape is not -// defined. There must be one dynamic_shape for each argument with a -// partially defined shape, in index order. -Status ComputeArgumentShapes(const TPUCompileMetadataProto& metadata, - const std::vector& dynamic_shapes, - std::vector* arg_shapes); } // namespace tpu } // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc b/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc index 5a8c283c7c2..e098dbd682c 100644 --- a/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc +++ b/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc @@ -25,8 +25,6 @@ limitations under the License. #include "tensorflow/core/platform/refcount.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_factory.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h" -#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.h" -#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_lookup.h" #include "tensorflow/core/tpu/kernels/tpu_mesh_state_interface.h" #include "tensorflow/core/tpu/kernels/tpu_op_consts.h" #include "tensorflow/core/tpu/tpu_api.h" @@ -255,10 +253,6 @@ void InitializeHostForDistributedTpuOp::Compute(OpKernelContext* ctx) { mesh_state_interface)); } - VLOG(1) << "Removing existing proto compilation cache lookup if it exists"; - OP_REQUIRES_OK(ctx, DeleteIfExists( - rmgr, tpu::kCompiledProtoCacheResourceName)); - if (enable_whole_mesh_compilations_) { // If this is a whole mesh compilation mode, create the compilation cache, // if missing. @@ -282,13 +276,6 @@ void InitializeHostForDistributedTpuOp::Compute(OpKernelContext* ctx) { if (local_compilation_cache != nullptr) { local_compilation_cache->Unref(); - - tpu::TpuCompilationCacheLookup* proto_lookup; - proto_lookup = - new tpu::TpuCompilationCacheLocalLookup(local_compilation_cache); - OP_REQUIRES_OK( - ctx, rmgr->Create(rmgr->default_container(), - tpu::kCompiledProtoCacheResourceName, proto_lookup)); } Tensor* ctx_output; diff --git a/tensorflow/core/tpu/kernels/tpu_execute_op.cc b/tensorflow/core/tpu/kernels/tpu_execute_op.cc index 3522ace379a..51c9dd481a3 100644 --- a/tensorflow/core/tpu/kernels/tpu_execute_op.cc +++ b/tensorflow/core/tpu/kernels/tpu_execute_op.cc @@ -40,12 +40,10 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/platform/casts.h" #include "tensorflow/core/platform/tracing.h" #include "tensorflow/core/profiler/lib/traceme.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_entry.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_external.h" -#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_lookup.h" #include "tensorflow/core/tpu/kernels/tpu_executable_info.pb.h" @@ -58,10 +56,14 @@ limitations under the License. #include "tensorflow/stream_executor/tpu/tpu_node_context.h" namespace tensorflow { + namespace { -using ::tensorflow::tpu::CompilationCacheEntryRef; -using ::tensorflow::tpu::TpuCompilationCacheLookup; + using ::tensorflow::tpu::TpuNodeContext; +using CompilationCacheEntryRef = ::tensorflow::tpu::CompilationCacheEntryRef< + ::tensorflow::tpu::TpuCompilationCacheEntry>; +using TpuCompilationCacheLookup = + ::tensorflow::tpu::TpuCompilationCacheLookup; // Looks up the input `key` in the compilation cache, populating // `*rendezvous_key_base` and `*entry`. @@ -639,35 +641,28 @@ Status TPUExecuteOp::DoWork(OpKernelContext* context) { profiler::TraceMe trace_me_init("TPUExecuteOp::Init", /*level=*/2); string rendezvous_key_base; - std::unique_ptr entry_ref; + std::unique_ptr entry; TF_RETURN_IF_ERROR( - GetComputationCacheEntry(context, &rendezvous_key_base, &entry_ref)); + GetComputationCacheEntry(context, &rendezvous_key_base, &entry)); // Shapes of the inputs and outputs, in xla::Shape form. - tpu::TpuCompilationCacheEntry entry = entry_ref->get(); - const tpu::TpuProgramGroup* tpu_program_group = - tensorflow::down_cast( - entry.tpu_program_group()); - CHECK_NE(tpu_program_group, nullptr); - const int core_index = entry.core_index(); - const TPUExecutableInfoProto& executable = - tpu_program_group->executable_info(core_index); + const TPUExecutableInfoProto* proto = entry->get().get_executable_info(); xla::Backend* const backend = node_context->backend(); xla::TransferManager* const transfer_manager = backend->transfer_manager(); TF_RET_CHECK(context->op_device_context()); se::Stream* stream = context->op_device_context()->stream(); - TF_RET_CHECK(executable.input_shapes_size() == 1); + TF_RET_CHECK(proto->input_shapes_size() == 1); - xla::Shape host_shape(executable.input_shapes(0)); + xla::Shape host_shape(proto->input_shapes(0)); TF_ASSIGN_OR_RETURN( auto variable_update_map, - BuildVariableUpdateMap(executable.variable_indices(), + BuildVariableUpdateMap(proto->variable_indices(), fused_device_var_reads_in_computation_inputs_, fused_device_var_updates_in_computation_outputs_, - executable.output_tensor_shapes().size())); + proto->output_tensor_shapes().size())); TF_ASSIGN_OR_RETURN( std::unique_ptr input_buffers, BuildComputationInputs(context, host_shape, variable_update_map, backend, @@ -702,9 +697,8 @@ Status TPUExecuteOp::DoWork(OpKernelContext* context) { // Snapshot the inputs, if a snapshot was requested. std::shared_ptr hlo_snapshot; - if (executable.has_session_module()) { - hlo_snapshot = - std::make_shared(executable.session_module()); + if (proto->has_session_module()) { + hlo_snapshot = std::make_shared(proto->session_module()); auto literal = std::make_shared(shaped_buffer.on_host_shape()); transfer_manager->TransferLiteralFromDevice( @@ -729,9 +723,9 @@ Status TPUExecuteOp::DoWork(OpKernelContext* context) { const uint32 rng_seed = GetXLARandomSeed(); std::unique_ptr device_assignment; - if (executable.has_device_assignment()) { + if (proto->has_device_assignment()) { TF_ASSIGN_OR_RETURN(device_assignment, xla::DeviceAssignment::Deserialize( - executable.device_assignment())); + proto->device_assignment())); } VLOG(4) << "Input buffers after alias resolution: " @@ -749,24 +743,24 @@ Status TPUExecuteOp::DoWork(OpKernelContext* context) { // we free a memory and reassign it to other users while a program is running, // all subsequent writes to the program that could possibly clobber the memory // will depend on the program to finish. - const TPUHostTransferInfoProto& host_transfer_info = - tpu_program_group->host_transfer_info(core_index); + const TPUHostTransferInfoProto* host_transfer_info = + entry->get().get_host_transfer_info(); + const xla::HloProto* hlo_metadata = entry->get().get_hlo_metadata(); TF_ASSIGN_OR_RETURN( xla::ExecutionOutput output, - TPUExecute(executable, host_transfer_info, - *tpu_program_group->hlo_metadata(core_index), std::move(input), + TPUExecute(*proto, *host_transfer_info, *hlo_metadata, std::move(input), rendezvous_key_base, rng_seed, node_context.get(), device_assignment.get(), context->cancellation_manager(), context, stream, transfer_stream_ptr.get(), - tpu_program_group->tpu_program(core_index))); + entry->get().get_tpu_program())); stream->ThenRecordEvent(definition_event.get()); TF_ASSIGN_OR_RETURN( std::unique_ptr output_buffers, - AllocateOutputTensors( - context, output.ConsumeResult(), executable.output_tensor_shapes(), - variable_update_map, node_context.get(), stream, device_ordinal, - input_buffers.get(), definition_event)); + AllocateOutputTensors(context, output.ConsumeResult(), + proto->output_tensor_shapes(), variable_update_map, + node_context.get(), stream, device_ordinal, + input_buffers.get(), definition_event)); // Transfer the outputs and save the snapshot to disk. if (hlo_snapshot) { diff --git a/tensorflow/core/tpu/kernels/tpu_program_c_api.h b/tensorflow/core/tpu/kernels/tpu_program_c_api.h index 41c7d47cf97..c9951e4d5ce 100644 --- a/tensorflow/core/tpu/kernels/tpu_program_c_api.h +++ b/tensorflow/core/tpu/kernels/tpu_program_c_api.h @@ -21,9 +21,6 @@ limitations under the License. typedef struct XLA_TpuProgram XLA_TpuProgram; -// Enum for choosing sharding/unsharding program from a `XLA_TpuProgram` obj. -enum TpuProgramShardingType { kInvalid = 0, kMain, kSharding, kUnsharding }; - extern "C" { // Creates a new TPU program. @@ -67,15 +64,6 @@ TFTPU_CAPI_EXPORT void TpuProgram_GetHloMetadata( TFTPU_CAPI_EXPORT void TpuProgram_GetMayModifyVariables( const XLA_TpuProgram* tpu_program, bool* may_modify_variables); -// Check if TPU program has sharding. -TFTPU_CAPI_EXPORT bool TpuProgram_HasSharding( - const XLA_TpuProgram* tpu_program); - -// Gets TPU program by sharding type. Return value is valid only when the -// `status.status()` returns `OK`. -TFTPU_CAPI_EXPORT XLA_TpuProgram* TpuProgram_GetTpuProgram( - XLA_TpuProgram* tpu_program, TpuProgramShardingType type); - struct TfTpu_TpuProgramApiFn { TFTPU_ADD_FN_IN_STRUCT(TpuProgram_New); TFTPU_ADD_FN_IN_STRUCT(TpuProgram_Free); @@ -88,8 +76,6 @@ struct TfTpu_TpuProgramApiFn { TFTPU_ADD_FN_IN_STRUCT(TpuProgram_GetHostTransferInfo); TFTPU_ADD_FN_IN_STRUCT(TpuProgram_GetHloMetadata); TFTPU_ADD_FN_IN_STRUCT(TpuProgram_GetMayModifyVariables); - TFTPU_ADD_FN_IN_STRUCT(TpuProgram_HasSharding); - TFTPU_ADD_FN_IN_STRUCT(TpuProgram_GetTpuProgram); }; } // extern "C" diff --git a/tensorflow/core/tpu/kernels/tpu_program_group.cc b/tensorflow/core/tpu/kernels/tpu_program_group.cc index 39d1f38b104..e22175af270 100644 --- a/tensorflow/core/tpu/kernels/tpu_program_group.cc +++ b/tensorflow/core/tpu/kernels/tpu_program_group.cc @@ -22,7 +22,6 @@ limitations under the License. #include "tensorflow/core/tpu/kernels/tpu_compile.pb.h" #include "tensorflow/core/tpu/kernels/tpu_compile_c_api.h" #include "tensorflow/core/tpu/kernels/tpu_compile_op_support.h" -#include "tensorflow/core/tpu/kernels/tpu_program_c_api.h" #include "tensorflow/core/tpu/tpu_api.h" #include "tensorflow/stream_executor/tpu/proto_helper.h" #include "tensorflow/stream_executor/tpu/status_helper.h" @@ -99,71 +98,55 @@ StatusOr> CompileAheadOfTime( compilation_result, metadata, per_core_arg_shapes, per_core_output_shapes, per_core_variable_indices, device_assignment); } -} // namespace -void TpuProgramGroup::Initialize( - absl::Span xla_tpu_programs) { +Status CreateTpuProgramGroup( + absl::Span xla_tpu_programs, + TpuProgramGroupInterface* tpu_program_group_interface) { CHECK_GT(xla_tpu_programs.size(), 0); - set_tpu_programs(xla_tpu_programs); + TpuProgramGroup* tpu_program_group = + tensorflow::down_cast(tpu_program_group_interface); + CHECK_NE(tpu_program_group, nullptr); + tpu_program_group->set_tpu_programs(xla_tpu_programs); - std::vector may_modify_variables_array(xla_tpu_programs.size(), false); - std::vector executable_infos(xla_tpu_programs.size()); - std::vector host_transfer_infos( - xla_tpu_programs.size()); - std::vector hlo_metadatas(xla_tpu_programs.size()); - for (size_t i = 0; i < xla_tpu_programs.size(); ++i) { - const XLA_TpuProgram* xla_tpu_program = xla_tpu_programs[i]; - bool may_modify_variables; - TpuProgramApiFn()->TpuProgram_GetMayModifyVariablesFn( - xla_tpu_program, &may_modify_variables); - may_modify_variables_array[i] = may_modify_variables; + // TODO(jiawenhao): Handle the case of xla_tpu_programs.size() > 1. + bool may_modify_variables; + TpuProgramApiFn()->TpuProgram_GetMayModifyVariablesFn(xla_tpu_programs[0], + &may_modify_variables); + tpu_program_group->set_may_modify_variables( + std::vector(1, may_modify_variables)); - TpuSerializedProto serialized_executable_info; - TpuProgramApiFn()->TpuProgram_GetExecutableInfoFn( - xla_tpu_program, &serialized_executable_info); - TPUExecutableInfoProto executable_info = - se_tpu::DeserializeProto( - serialized_executable_info); - executable_infos[i] = executable_info; - StreamExecutor_Tpu_FreeSerializedProto(&serialized_executable_info); + TpuSerializedProto serialized_executable_info; + TpuProgramApiFn()->TpuProgram_GetExecutableInfoFn( + xla_tpu_programs[0], &serialized_executable_info); + TPUExecutableInfoProto executable_info = + se_tpu::DeserializeProto( + serialized_executable_info); + tpu_program_group->set_executable_info(executable_info); + StreamExecutor_Tpu_FreeSerializedProto(&serialized_executable_info); - TPUHostTransferInfoProto host_transfer_info; - TpuSerializedProto serialized_host_transfer_info; - TpuProgramApiFn()->TpuProgram_GetHostTransferInfoFn( - xla_tpu_program, &serialized_host_transfer_info); - if (serialized_host_transfer_info.size > 0) { - host_transfer_info = se_tpu::DeserializeProto( - serialized_host_transfer_info); - StreamExecutor_Tpu_FreeSerializedProto(&serialized_host_transfer_info); - } - host_transfer_infos[i] = host_transfer_info; - - TpuSerializedProto serialized_hlo_metadata; - TpuProgramApiFn()->TpuProgram_GetHloMetadataFn(xla_tpu_program, - &serialized_hlo_metadata); - xla::HloProto hlo_metadata = - se_tpu::DeserializeProto(serialized_hlo_metadata); - hlo_metadatas[i] = hlo_metadata; - StreamExecutor_Tpu_FreeSerializedProto(&serialized_hlo_metadata); + TPUHostTransferInfoProto host_transfer_info; + TpuSerializedProto serialized_host_transfer_info; + TpuProgramApiFn()->TpuProgram_GetHostTransferInfoFn( + xla_tpu_programs[0], &serialized_host_transfer_info); + if (serialized_host_transfer_info.size > 0) { + host_transfer_info = se_tpu::DeserializeProto( + serialized_host_transfer_info); + StreamExecutor_Tpu_FreeSerializedProto(&serialized_host_transfer_info); } + tpu_program_group->set_host_transfer_info(host_transfer_info); - may_modify_variables_ = may_modify_variables_array; - executable_infos_ = executable_infos; - host_transfer_infos_ = host_transfer_infos; - hlo_metadatas_ = hlo_metadatas; - RefreshHloMetadatasPtrs(); + TpuSerializedProto serialized_hlo_metadata; + TpuProgramApiFn()->TpuProgram_GetHloMetadataFn(xla_tpu_programs[0], + &serialized_hlo_metadata); + xla::HloProto hlo_metadata = + se_tpu::DeserializeProto(serialized_hlo_metadata); + tpu_program_group->set_hlo_metadata(hlo_metadata); + StreamExecutor_Tpu_FreeSerializedProto(&serialized_hlo_metadata); + + return Status::OK(); } -bool TpuProgramGroup::has_sharding_program() const { - for (const XLA_TpuProgram* tpu_program : tpu_programs_) { - if (!TpuProgramApiFn()->TpuProgram_HasShardingFn(tpu_program)) { - return false; - } - } - return true; -} - -size_t TpuProgramGroup::program_count() const { return tpu_programs_.size(); } +} // namespace int64_t TpuProgramGroup::program_size() const { int64_t total_size = 0; @@ -218,6 +201,12 @@ void TpuProgramGroup::UnloadAndDestroyPrograms() { TF_RET_CHECK(per_core_output_shapes.size() == per_core_variable_indices.size()); + // TODO(henrytan): add an interface to TpuProgramGroupInterface to set + // may_modify_variables. + TpuProgramGroup* tpu_program_group = + tensorflow::down_cast(tpu_program_group_interface); + tpu_program_group->may_modify_variables_ = may_modify_variables; + // With shardable input/output pairs, XLA could generate separate // sharding/unsharding programs along with the main program. The // sharding/unsharding programs will be in nested entries of the AOT @@ -232,20 +221,17 @@ void TpuProgramGroup::UnloadAndDestroyPrograms() { TF_RET_CHECK(xla_tpu_programs.size() == 1 || xla_tpu_programs.size() == metadata.num_cores_per_replica()); - // TODO(henrytan): add an interface to TpuProgramGroupInterface to set - // may_modify_variables. - TpuProgramGroup* tpu_program_group = - tensorflow::down_cast(tpu_program_group_interface); - tpu_program_group->Initialize(xla_tpu_programs); - tpu_program_group->may_modify_variables_ = may_modify_variables; + TF_RETURN_IF_ERROR( + CreateTpuProgramGroup(xla_tpu_programs, tpu_program_group)); return Status::OK(); } TpuProgramGroup::TpuProgramGroup(TpuProgramGroup&& other) : may_modify_variables_(std::move(other.may_modify_variables_)), + host_compute_metadata_(std::move(other.host_compute_metadata_)), tpu_programs_(std::move(other.tpu_programs_)), - executable_infos_(std::move(other.executable_infos_)), - host_transfer_infos_(std::move(other.host_transfer_infos_)), + executable_info_(std::move(other.executable_info_)), + host_transfer_info_(std::move(other.host_transfer_info_)), hlo_metadatas_(std::move(other.hlo_metadatas_)) { RefreshHloMetadatasPtrs(); } @@ -262,12 +248,6 @@ absl::Span TpuProgramGroup::hlo_metadatas() const { return hlo_metadatas_ptrs_; } -const xla::HloProto* TpuProgramGroup::hlo_metadata(int index) const { - CHECK_GE(index, 0); - CHECK_LT(index, hlo_metadatas_ptrs_.size()); - return hlo_metadatas_ptrs_[index]; -} - void TpuProgramGroup::RefreshHloMetadatasPtrs() { hlo_metadatas_ptrs_.reserve(hlo_metadatas_.size()); for (const auto& hlo_metadata_internal_ : hlo_metadatas_) { @@ -282,47 +262,6 @@ Status TpuProgramGroup::LogCompilationStats(const TpuCompilationCacheKey& key, return Status::OK(); } -const std::vector& TpuProgramGroup::may_modify_variables() const { - return may_modify_variables_; -} - -void TpuProgramGroup::set_may_modify_variables( - const std::vector& may_modify_variables) { - may_modify_variables_ = may_modify_variables; -} - -const std::vector& TpuProgramGroup::tpu_programs() const { - return tpu_programs_; -} - -const XLA_TpuProgram* TpuProgramGroup::tpu_program(int index) const { - CHECK_GE(index, 0); - CHECK_LT(index, tpu_programs_.size()); - return tpu_programs_[index]; -} - -void TpuProgramGroup::set_tpu_programs( - absl::Span tpu_programs) { - tpu_programs_.resize(tpu_programs.size()); - for (size_t i = 0; i < tpu_programs.size(); ++i) { - tpu_programs_[i] = tpu_programs[i]; - } -} - -const TPUExecutableInfoProto& TpuProgramGroup::executable_info( - int index) const { - CHECK_GE(index, 0); - CHECK_LT(index, executable_infos_.size()); - return executable_infos_[index]; -} - -const TPUHostTransferInfoProto& TpuProgramGroup::host_transfer_info( - int index) const { - CHECK_GE(index, 0); - CHECK_LT(index, host_transfer_infos_.size()); - return host_transfer_infos_[index]; -} - /*static*/ Status TpuProgramGroup::CompileAndBuild( const TpuCompilationRequestProto& compilation_request, @@ -348,27 +287,15 @@ Status TpuProgramGroup::CompileAndBuild( TF_RET_CHECK(count == 1 || count == compilation_request.metadata().num_cores_per_replica()); - VLOG(1) << "Initialize TpuProgramGroup."; - TpuProgramGroup* tpu_program_group = - tensorflow::down_cast(tpu_program_group_interface); - tpu_program_group->Initialize( - absl::MakeConstSpan(&xla_tpu_programs[0], count)); + VLOG(1) << "CreateTpuProgramGroup"; + Status serialize_status = + CreateTpuProgramGroup(absl::MakeConstSpan(&xla_tpu_programs[0], count), + tpu_program_group_interface); + VLOG(1) << absl::StrCat("Run CreateTpuProgramGroup completed. StatusCode: ", + serialize_status.code()); TpuProgramApiFn()->TpuProgram_FreeArrayFn(xla_tpu_programs); - return status.status(); + return serialize_status; } -std::vector TpuProgramGroup::tpu_programs( - TpuProgramShardingType sharding_type) const { - std::vector tpu_programs; - tpu_programs.reserve(tpu_programs_.size()); - for (size_t i = 0; i < tpu_programs_.size(); ++i) { - if (TpuProgramApiFn()->TpuProgram_HasShardingFn(tpu_programs_[i])) { - tpu_programs.push_back(TpuProgramApiFn()->TpuProgram_GetTpuProgramFn( - tpu_programs_[i], sharding_type)); - CHECK_NE(tpu_programs[i], nullptr); - } - } - return tpu_programs; -} } // namespace tpu } // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_program_group.h b/tensorflow/core/tpu/kernels/tpu_program_group.h index b76ef3d507a..4bc8cdd003a 100644 --- a/tensorflow/core/tpu/kernels/tpu_program_group.h +++ b/tensorflow/core/tpu/kernels/tpu_program_group.h @@ -102,16 +102,11 @@ class TpuProgramGroup : public TpuProgramGroupInterface { const absl::optional& xla_device_assignment, TpuProgramGroupInterface* tpu_program_group_interface); - // Initializes `TpuProgramGroup` object with `xla_tpu_programs`. - void Initialize(absl::Span xla_tpu_programs); - TpuProgramGroup() = default; TpuProgramGroup(TpuProgramGroup&& other); TpuProgramGroup& operator=(TpuProgramGroup&&) = delete; - bool has_sharding_program() const override; - - size_t program_count() const override; + size_t program_count() const override { return tpu_programs_.size(); } int64_t program_size() const override; @@ -122,29 +117,58 @@ class TpuProgramGroup : public TpuProgramGroupInterface { Status LogCompilationStats(const TpuCompilationCacheKey& key, absl::Duration duration) override; - const std::vector& may_modify_variables() const override; - void set_may_modify_variables(const std::vector& may_modify_variables); + const std::vector& may_modify_variables() const override { + return may_modify_variables_; + } + void set_may_modify_variables(const std::vector& may_modify_variables) { + may_modify_variables_ = may_modify_variables; + } - const std::vector& tpu_programs() const; - std::vector tpu_programs(TpuProgramShardingType type) const; - const XLA_TpuProgram* tpu_program(int index) const; - void set_tpu_programs(absl::Span tpu_programs); + const tf2xla::HostComputeMetadata& host_compute_metadata() const { + return host_compute_metadata_; + } + void set_host_compute_metadata( + const tf2xla::HostComputeMetadata& host_compute_metadata) { + host_compute_metadata_ = host_compute_metadata; + } - const TPUExecutableInfoProto& executable_info(int index) const; + const std::vector& tpu_programs() const { + return tpu_programs_; + } + void set_tpu_programs(absl::Span tpu_programs) { + tpu_programs_.resize(tpu_programs.size()); + for (size_t i = 0; i < tpu_programs.size(); ++i) { + tpu_programs_[i] = tpu_programs[i]; + } + } + + const TPUExecutableInfoProto& executable_info() const { + return executable_info_; + } + void set_executable_info(const TPUExecutableInfoProto& executable_info) { + executable_info_ = executable_info; + } + + const TPUHostTransferInfoProto& host_transfer_info() const { + return host_transfer_info_; + } + void set_host_transfer_info( + const TPUHostTransferInfoProto& host_transfer_info) { + host_transfer_info_ = host_transfer_info; + } - const TPUHostTransferInfoProto& host_transfer_info(int index) const; void set_hlo_metadata(const xla::HloProto& hlo_metadata); - const xla::HloProto* hlo_metadata(int index) const; absl::Span hlo_metadatas() const override; private: void RefreshHloMetadatasPtrs(); std::vector may_modify_variables_; + tf2xla::HostComputeMetadata host_compute_metadata_; std::vector tpu_programs_; // Not owned. - std::vector executable_infos_; - std::vector host_transfer_infos_; + TPUExecutableInfoProto executable_info_; + TPUHostTransferInfoProto host_transfer_info_; // To be consistent with the TpuProgramGroupInterface::hlo_metadatas() // signature, we store HloProto values in hlo_metadatas_ when diff --git a/tensorflow/core/tpu/kernels/tpu_program_group_interface.h b/tensorflow/core/tpu/kernels/tpu_program_group_interface.h index 4af94f8e1ad..cb7347783b1 100644 --- a/tensorflow/core/tpu/kernels/tpu_program_group_interface.h +++ b/tensorflow/core/tpu/kernels/tpu_program_group_interface.h @@ -20,8 +20,6 @@ limitations under the License. #include #include -#include "absl/time/time.h" -#include "absl/types/span.h" #include "tensorflow/compiler/tf2xla/host_compute_metadata.pb.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/core/lib/core/status.h" @@ -36,16 +34,13 @@ class TpuProgramGroupInterface { public: virtual ~TpuProgramGroupInterface() = default; - // Check if whether sharding/unsharding program exists. - virtual bool has_sharding_program() const = 0; - // Computes program count. virtual size_t program_count() const = 0; // Computes total program size. virtual int64_t program_size() const = 0; - // Unloads and destroys safely TPU programs. + // Unloads and destroys safely Tpu programs. virtual void UnloadAndDestroyPrograms() = 0; // Logs program memory summary. diff --git a/tensorflow/core/tpu/tpu_library_init_fns.inc b/tensorflow/core/tpu/tpu_library_init_fns.inc index 6914a8cd102..682cc8b1c13 100644 --- a/tensorflow/core/tpu/tpu_library_init_fns.inc +++ b/tensorflow/core/tpu/tpu_library_init_fns.inc @@ -64,8 +64,6 @@ tensorflow::Status SetTpuProgramStructFn(void* library_handle) { TFTPU_SET_FN(tpu_program_fn, TpuProgram_GetHostTransferInfo); TFTPU_SET_FN(tpu_program_fn, TpuProgram_GetHloMetadata); TFTPU_SET_FN(tpu_program_fn, TpuProgram_GetMayModifyVariables); - TFTPU_SET_FN(tpu_program_fn, TpuProgram_HasSharding); - TFTPU_SET_FN(tpu_program_fn, TpuProgram_GetTpuProgram); return tensorflow::Status::OK(); } From 474d3df724c402048ae2f8773bfc49f1587229ff Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Mon, 3 Aug 2020 16:19:08 -0700 Subject: [PATCH 0297/1017] [TF2XLA] [NFC] Test that aliased updates do not actually increase memory usage PiperOrigin-RevId: 324705992 Change-Id: I1c7c19867c3b7086dab61427da7e7b78547e4c59 --- .../python/eager/def_function_xla_jit_test.py | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/tensorflow/python/eager/def_function_xla_jit_test.py b/tensorflow/python/eager/def_function_xla_jit_test.py index 10982070c00..44a4c99f5d6 100644 --- a/tensorflow/python/eager/def_function_xla_jit_test.py +++ b/tensorflow/python/eager/def_function_xla_jit_test.py @@ -461,26 +461,23 @@ class DefFunctionTest(xla_test.XLATestCase): def testUpdateVariable(self): with ops.device('device:{}:0'.format(self.device)): - v = variables.Variable(3.1) + + on_gpu = 'gpu' in self.device.lower() + v = variables.Variable([3.1, 3.2]) @def_function.function(experimental_compile=True) def update_var(a, b): v.assign_add(a * b) - update_var(constant_op.constant(0.7), constant_op.constant(0.6)) - self.assertAllClose(v, 3.52) + arg1 = random_ops.random_normal([2]) + arg2 = random_ops.random_normal([2]) - def testUpdateVariableVector(self): - with ops.device('device:{}:0'.format(self.device)): - v = variables.Variable([3.1, 3.1]) - - @def_function.function(experimental_compile=True) - def update_var(a, b): - v.assign_add(a * b) - - update_var( - constant_op.constant([0.7, 0.7]), constant_op.constant([0.6, 0.6])) - self.assertAllClose(v, [3.52, 3.52]) + initial_usage = context.context().get_total_memory_usage( + v.device) if on_gpu else 0 + update_var(arg1, arg2) + final_usage = context.context().get_total_memory_usage( + v.device) if on_gpu else 0 + self.assertEqual(initial_usage, final_usage) @test_util.disable_mlir_bridge('TODO(b/162381930): MLIR bridge renames ' ' functions') @@ -524,11 +521,19 @@ class DefFunctionTest(xla_test.XLATestCase): def f(a, b): return (a, b) - a = constant_op.constant([0.7]) - b = constant_op.constant([0.6]) + a = random_ops.random_normal([10, 10]) + b = random_ops.random_normal([10, 10]) + + on_gpu = 'gpu' in self.device.lower() + initial_usage = context.context().get_total_memory_usage( + b.backing_device) if on_gpu else 0 f(a, b) + final_usage = context.context().get_total_memory_usage( + b.backing_device) if on_gpu else 0 + self.assertEqual(initial_usage, final_usage) + if __name__ == '__main__': ops.enable_eager_execution() From 7e10ef560d3bb0cbe837bfd2b17c0e999f84d9da Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Mon, 3 Aug 2020 16:20:05 -0700 Subject: [PATCH 0298/1017] [tf2xla] Add support for PopulationCount PiperOrigin-RevId: 324706175 Change-Id: I567c1fec90022573acf7a6dbe18a06191a088c90 --- .../compiler/jit/mark_for_compilation_pass.cc | 1 + tensorflow/compiler/tests/unary_ops_test.py | 37 +++++++++++++++++++ .../compiler/tf2xla/kernels/unary_ops.cc | 3 ++ 3 files changed, 41 insertions(+) diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc index d1ec66b1559..19eb61b6f72 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc @@ -1952,6 +1952,7 @@ absl::flat_hash_set GetKnownXLAAllowlistOp() { "ParallelDynamicStitch", "ParameterizedTruncatedNormal", "PartitionedCall", + "PopulationCount", "Qr", "QuantizeAndDequantizeV2", "QuantizeAndDequantizeV3", diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index f5fe6986a98..e3a82610027 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import unittest import numpy as np +import six from six.moves import xrange # pylint: disable=redefined-builtin from tensorflow.compiler.tests import xla_test @@ -90,6 +91,10 @@ class UnaryOpsTest(xla_test.XLATestCase): self.assertAllClose(result, expected, rtol, atol) self.assertAllEqual(np.sort(result), result) + def AssertAllEqual(self, result, expected, rtol, atol): + """Tests that result and expeted are exactly equal.""" + self.assertAllEqual(result, expected) + @test_util.disable_mlir_bridge( "MlirHloBuilder::Iota missing required for xla::Diag") def testAllTypeOps(self): @@ -779,6 +784,10 @@ class UnaryOpsTest(xla_test.XLATestCase): np.array([1 + 3j, -4 + 7j, 2.7, -3j], dtype=dtype), expected=np.array([1, -4, 2.7, 0], dtype=ctypes[dtype])) + @test_util.disable_mlir_bridge( + "TF_PopulationCount is missing and is required to translate to " + "xla::PopulationCount." + ) def testIntOps(self): for dtype in self.int_types: self._assertOpOutputMatchesExpected( @@ -786,6 +795,34 @@ class UnaryOpsTest(xla_test.XLATestCase): np.array([0, -1, 1, 16, 42], dtype=dtype), expected=np.array([-1, 0, -2, -17, -43], dtype=dtype)) + # Test population_count for array inputs. + raw_inputs = [ + 0, 1, -1, 3, -3, 5, -5, 14, -14, 127, 128, 255, 256, 65535, 65536, + 2**31 - 1, 2**31, 2**32 - 1, 2**32, -2**32 + 1, -2**32, -2**63 + 1, + 2**63 - 1 + ] + inputs = np.array(raw_inputs, dtype=dtype) + + def count_bits(x): + return sum(bin(z).count("1") for z in six.iterbytes(x.tobytes())) + + truth = [count_bits(x) for x in inputs] + self._assertOpOutputMatchesExpected( + bitwise_ops.population_count, + inputs, + expected=np.array(truth, dtype=np.uint8), + equality_test=self.AssertAllEqual) + + # Test population_count for scalar inputs. + for raw_inp in raw_inputs: + inp = dtype(raw_inp) + truth = count_bits(inp) + self._assertOpOutputMatchesExpected( + bitwise_ops.population_count, + inp, + expected=np.uint8(truth), + equality_test=self.AssertAllEqual) + def testNumericOps(self): for dtype in self.numeric_types - {np.int8, np.uint8}: self._assertOpOutputMatchesExpected( diff --git a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc index 6d4393ee006..6fe6b164951 100644 --- a/tensorflow/compiler/tf2xla/kernels/unary_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/unary_ops.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/lib/math.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/primitive_util.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" #include "tensorflow/core/framework/kernel_def_builder.h" namespace tensorflow { @@ -76,6 +77,8 @@ XLAJIT_MAKE_UNARY(Log1p, xla::Log1p(x)); XLAJIT_MAKE_UNARY(Invert, xla::Not(x)); XLAJIT_MAKE_UNARY(LogicalNot, xla::Not(x)); +XLAJIT_MAKE_UNARY(PopulationCount, + xla::ConvertElementType(xla::PopulationCount(x), xla::U8)); XLAJIT_MAKE_UNARY(Neg, -x); XLAJIT_MAKE_UNARY(Rint, xla::RoundToEven(x)); From eed154ca65961d14b99f207ea57f8dd421bb9df3 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 3 Aug 2020 23:42:27 +0000 Subject: [PATCH 0299/1017] changed TF_AllocatorAttributes usage to pass by pointer --- tensorflow/c/kernels.cc | 14 ++++++-------- tensorflow/c/kernels.h | 2 +- tensorflow/c/kernels_test.cc | 6 +++--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index b2200f7c9af..6d9d7d57517 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -283,23 +283,21 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, int64_t* dims, int num_dims, - TF_AllocatorAttributes attributes, + TF_AllocatorAttributes* attributes, TF_Status* status) { auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); TF_SetStatus(status, TF_OK, ""); - tensorflow::TensorShape shape; - for (int i = 0; i < num_dims; ++i) { - shape.AddDim(dims[i]); - } + tensorflow::gtl::ArraySlice dimarray( + reinterpret_cast(dims), num_dims); tensorflow::AllocatorAttributes allocator_attr; - if (attributes.on_host) { + if (attributes->on_host) { allocator_attr.set_on_host(true); } tensorflow::Status s; tensorflow::Tensor tensor; TF_Tensor* tf_tensor; - s = cc_ctx->allocate_temp(static_cast(dtype), shape, - &tensor, allocator_attr); + s = cc_ctx->allocate_temp(static_cast(dtype), + tensorflow::TensorShape(dimarray), &tensor, allocator_attr); if (!s.ok()) { ::tensorflow::Set_TF_Status_from_Status(status, s); return nullptr; diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index 178e1a29267..d6a7070de06 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -206,7 +206,7 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, // num_dims must equal the size of array dims TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, - TF_DataType dtype, int64_t* dims, int num_dims, TF_AllocatorAttributes + TF_DataType dtype, int64_t* dims, int num_dims, TF_AllocatorAttributes* alloc_attrs, TF_Status* status); diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index 82027ffd1e0..708c39690a8 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -476,7 +476,7 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSizeOne) { #endif TF_Tensor* output = TF_AllocateTemp( /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, /*allocator_attributes*/ alloc_attrs, s); + /*num_dims=*/1, /*allocator_attributes*/ &alloc_attrs, s); size_t tensor_size_bytes = TF_DataTypeSize(TF_FLOAT); EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, &dim, 1, TF_FLOAT); @@ -513,7 +513,7 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempEmpty) { #endif TF_Tensor* output = TF_AllocateTemp( /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, /*allocator_attributes*/ alloc_attrs, s); + /*num_dims=*/1, /*allocator_attributes*/ &alloc_attrs, s); EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, &dim, 1, TF_FLOAT); TF_SetOutput(ctx, 0, output, s); @@ -546,7 +546,7 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSize2x3) { #endif TF_Tensor* output = TF_AllocateTemp( /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/dim, - /*num_dims=*/2, /*allocator_attributes*/ alloc_attrs, s); + /*num_dims=*/2, /*allocator_attributes*/ &alloc_attrs, s); EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, dim, 2, TF_FLOAT); From 68468d791fe616a89501edd369267814be17a254 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 3 Aug 2020 23:44:00 +0000 Subject: [PATCH 0300/1017] added histogram_summary op registration --- tensorflow/c/kernels/ops/histogram_summary.cc | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tensorflow/c/kernels/ops/histogram_summary.cc diff --git a/tensorflow/c/kernels/ops/histogram_summary.cc b/tensorflow/c/kernels/ops/histogram_summary.cc new file mode 100644 index 00000000000..b4889efefc9 --- /dev/null +++ b/tensorflow/c/kernels/ops/histogram_summary.cc @@ -0,0 +1,50 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/ops.h" +#include "tensorflow/c/tf_status.h" +#include "tensorflow/core/framework/selective_registration.h" +#include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/platform/macros.h" + +static void histogram_summary_shape_inference_fn(TF_ShapeInferenceContext* ctx, + TF_Status* status) { + TF_SetStatus(status, TF_OK, ""); + TF_ShapeHandle* result = TF_ShapeInferenceContextScalar(ctx); + TF_ShapeInferenceContextSetOutput(ctx, 0, result, status); + TF_DeleteShapeHandle(result); +} + +void Register_HistogramSummaryOp() { + TF_Status* status = TF_NewStatus(); + + TF_OpDefinitionBuilder* op_builder = + TF_NewOpDefinitionBuilder("HistogramSummary"); + TF_OpDefinitionBuilderAddInput(op_builder, "tag: string"); + TF_OpDefinitionBuilderAddInput(op_builder, "values: T"); + TF_OpDefinitionBuilderAddOutput(op_builder, "summary: string"); + TF_OpDefinitionBuilderAddAttr(op_builder, "T: realnumbertype"); + TF_OpDefinitionBuilderSetShapeInferenceFunction(op_builder, + &histogram_summary_shape_inference_fn); + + TF_RegisterOpDefinition(op_builder, status); + CHECK_EQ(TF_GetCode(status), TF_OK) + << "HistogramSummary op registration failed: " << TF_Message(status); + TF_DeleteStatus(status); +} + +TF_ATTRIBUTE_UNUSED static bool HistogramSummaryOpRegistered = []() { + if (SHOULD_REGISTER_OP("HistogramSummary")) { + Register_HistogramSummaryOp(); + } + return true; +}(); From 9fefb3b50ac69a074faf5829e9c7b4666fe906d0 Mon Sep 17 00:00:00 2001 From: Robert David Date: Mon, 3 Aug 2020 16:22:21 -0700 Subject: [PATCH 0301/1017] Tool generated fixes for unused dependencies. PiperOrigin-RevId: 324706583 Change-Id: Ia8512b20d27791d935d207fd50180593ca5e2ccc --- tensorflow/lite/delegates/gpu/cl/BUILD | 1 - .../lite/delegates/gpu/cl/kernels/BUILD | 29 ------------------- tensorflow/lite/delegates/gpu/common/BUILD | 1 - .../testing/feature_parity/generators/BUILD | 1 - .../gpu/common/transformations/BUILD | 1 - .../lite/delegates/gpu/gl/kernels/BUILD | 5 ---- .../lite/delegates/gpu/gl/runtime/BUILD | 1 - tensorflow/lite/delegates/gpu/metal/BUILD | 1 - .../lite/delegates/gpu/metal/kernels/BUILD | 26 ----------------- 9 files changed, 66 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/BUILD b/tensorflow/lite/delegates/gpu/cl/BUILD index 36cafdb4d3b..2344a7c6c40 100644 --- a/tensorflow/lite/delegates/gpu/cl/BUILD +++ b/tensorflow/lite/delegates/gpu/cl/BUILD @@ -259,7 +259,6 @@ cc_library( "EGL_EGLEXT_PROTOTYPES", ], deps = [ - ":cl_device", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/gl:gl_call", ], diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/BUILD b/tensorflow/lite/delegates/gpu/cl/kernels/BUILD index 35ed09633a0..b89e7d7252a 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/cl/kernels/BUILD @@ -172,7 +172,6 @@ cc_test( deps = [ ":cl_test", ":conv_buffer_1x1", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -212,7 +211,6 @@ cc_test( deps = [ ":cl_test", ":conv_constants", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -302,7 +300,6 @@ cc_test( deps = [ ":cl_test", ":conv_texture", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -380,7 +377,6 @@ cc_test( deps = [ ":cl_test", ":convolution_transposed", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -445,7 +441,6 @@ cc_test( deps = [ ":cl_test", ":convolution_transposed_3x3", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -484,7 +479,6 @@ cc_test( deps = [ ":cl_test", ":convolution_transposed_3x3_thin", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -525,7 +519,6 @@ cc_test( deps = [ ":cl_test", ":convolution_transposed_4x4", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -565,7 +558,6 @@ cc_test( deps = [ ":cl_test", ":convolution_transposed_thin", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -606,7 +598,6 @@ cc_test( deps = [ ":cl_test", ":depthwise_conv", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -646,7 +637,6 @@ cc_test( deps = [ ":cl_test", ":depthwise_conv_3x3", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -678,7 +668,6 @@ cc_test( deps = [ ":cl_test", ":elementwise", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -716,7 +705,6 @@ cc_test( deps = [ ":cl_test", ":fully_connected", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -772,7 +760,6 @@ cc_test( deps = [ ":cl_test", ":lstm", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -837,7 +824,6 @@ cc_test( deps = [ ":cl_test", ":max_unpooling", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -872,7 +858,6 @@ cc_test( deps = [ ":cl_test", ":mean", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -904,7 +889,6 @@ cc_test( deps = [ ":cl_test", ":padding", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -938,7 +922,6 @@ cc_test( deps = [ ":cl_test", ":pooling", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -975,7 +958,6 @@ cc_test( deps = [ ":cl_test", ":prelu", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1012,7 +994,6 @@ cc_test( deps = [ ":cl_test", ":quantize_and_dequantize", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/kernels/internal:quantization_util", @@ -1043,7 +1024,6 @@ cc_test( deps = [ ":cl_test", ":relu", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1075,7 +1055,6 @@ cc_test( deps = [ ":cl_test", ":reshape", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1108,7 +1087,6 @@ cc_test( deps = [ ":cl_test", ":reshapex4", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1142,7 +1120,6 @@ cc_test( deps = [ ":cl_test", ":softmax", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1174,7 +1151,6 @@ cc_test( deps = [ ":cl_test", ":softmax1x1", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1207,7 +1183,6 @@ cc_test( deps = [ ":cl_test", ":space_to_depth", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1238,7 +1213,6 @@ cc_test( deps = [ ":cl_test", ":strided_slice", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1270,7 +1244,6 @@ cc_test( deps = [ ":cl_test", ":transpose", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1312,7 +1285,6 @@ cc_test( deps = [ ":cl_test", ":resize", - "//tensorflow/lite/delegates/gpu/cl:tensor", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "@com_google_googletest//:gtest_main", @@ -1357,7 +1329,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:shape", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:winograd_util", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ], ) diff --git a/tensorflow/lite/delegates/gpu/common/BUILD b/tensorflow/lite/delegates/gpu/common/BUILD index e9877b63fb3..ab2d5d033f7 100644 --- a/tensorflow/lite/delegates/gpu/common/BUILD +++ b/tensorflow/lite/delegates/gpu/common/BUILD @@ -97,7 +97,6 @@ cc_test( srcs = ["model_test.cc"], deps = [ ":model", - ":status", "@com_google_googletest//:gtest_main", ], ) diff --git a/tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators/BUILD b/tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators/BUILD index ae746cdb08d..4fef0a28525 100644 --- a/tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators/BUILD +++ b/tensorflow/lite/delegates/gpu/common/testing/feature_parity/generators/BUILD @@ -24,7 +24,6 @@ cc_library( "//tensorflow/lite:schema_fbs_version", "//tensorflow/lite/delegates/gpu/common:shape", "//tensorflow/lite/delegates/gpu/common/testing/feature_parity:utils", - "//tensorflow/lite/kernels:builtin_ops", "@flatbuffers", ], ) diff --git a/tensorflow/lite/delegates/gpu/common/transformations/BUILD b/tensorflow/lite/delegates/gpu/common/transformations/BUILD index 4c76e4a81d3..bf26b03f534 100644 --- a/tensorflow/lite/delegates/gpu/common/transformations/BUILD +++ b/tensorflow/lite/delegates/gpu/common/transformations/BUILD @@ -59,7 +59,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:model_transformer", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", - "@com_google_absl//absl/strings", ], ) diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/BUILD b/tensorflow/lite/delegates/gpu/gl/kernels/BUILD index 700a553a125..a367a60ba41 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/kernels/BUILD @@ -317,13 +317,11 @@ cc_library( srcs = ["mean.cc"], hdrs = ["mean.h"], deps = [ - "//tensorflow/lite/delegates/gpu/common:data_type", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", ], ) @@ -470,7 +468,6 @@ cc_library( srcs = ["quantize_and_dequantize.cc"], hdrs = ["quantize_and_dequantize.h"], deps = [ - "//tensorflow/lite/delegates/gpu/common:convert", "//tensorflow/lite/delegates/gpu/common:data_type", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:shape", @@ -645,7 +642,6 @@ cc_test( ":space_to_depth", ":test_util", "//tensorflow/lite/delegates/gpu/common:operations", - "//tensorflow/lite/delegates/gpu/common:shape", "@com_google_googletest//:gtest", ], ) @@ -720,7 +716,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:node_shader", - "//tensorflow/lite/delegates/gpu/gl:variable", "@com_google_absl//absl/memory", ], ) diff --git a/tensorflow/lite/delegates/gpu/gl/runtime/BUILD b/tensorflow/lite/delegates/gpu/gl/runtime/BUILD index 20b307359db..c7418810f2d 100644 --- a/tensorflow/lite/delegates/gpu/gl/runtime/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/runtime/BUILD @@ -10,7 +10,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/gl:gl_buffer", - "//tensorflow/lite/delegates/gpu/gl:gl_call", "//tensorflow/lite/delegates/gpu/gl:object", "//tensorflow/lite/delegates/gpu/gl:portable", ], diff --git a/tensorflow/lite/delegates/gpu/metal/BUILD b/tensorflow/lite/delegates/gpu/metal/BUILD index 4db8f3d071d..c4e7ca7c10d 100644 --- a/tensorflow/lite/delegates/gpu/metal/BUILD +++ b/tensorflow/lite/delegates/gpu/metal/BUILD @@ -100,7 +100,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", ], ) diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/BUILD b/tensorflow/lite/delegates/gpu/metal/kernels/BUILD index 6385b87c403..f4f4c180976 100644 --- a/tensorflow/lite/delegates/gpu/metal/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/metal/kernels/BUILD @@ -91,7 +91,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", ], ) @@ -176,7 +175,6 @@ cc_library( deps = [ "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:status", - "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", "//tensorflow/lite/delegates/gpu/metal:runtime_options", ], @@ -230,12 +228,9 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:convert", "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:operations", - "//tensorflow/lite/delegates/gpu/common:shape", - "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", "//tensorflow/lite/delegates/gpu/metal:environment", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", "@com_google_absl//absl/strings", ], ) @@ -270,7 +265,6 @@ cc_library( deps = [ "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:operations", - "//tensorflow/lite/delegates/gpu/common:shape", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", @@ -314,7 +308,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", "@com_google_absl//absl/strings", ], ) @@ -388,11 +381,9 @@ cc_library( deps = [ "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:operations", - "//tensorflow/lite/delegates/gpu/common:shape", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", "@com_google_absl//absl/strings", ], ) @@ -429,10 +420,8 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:shape", - "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", "@com_google_absl//absl/strings", ], ) @@ -469,8 +458,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:shape", - "//tensorflow/lite/delegates/gpu/common:types", - "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", "//tensorflow/lite/delegates/gpu/metal:runtime_options", "@com_google_absl//absl/strings", @@ -509,10 +496,8 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:shape", "//tensorflow/lite/delegates/gpu/common:types", - "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", "//tensorflow/lite/delegates/gpu/metal:runtime_options", - "@com_google_absl//absl/strings", ], ) @@ -556,9 +541,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:shape", "//tensorflow/lite/delegates/gpu/common:types", - "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", "@com_google_absl//absl/strings", ], ) @@ -591,13 +574,10 @@ cc_library( srcs = ["resize.cc"], hdrs = ["resize.h"], deps = [ - ":util", "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:operations", - "//tensorflow/lite/delegates/gpu/common:tensor", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", "@com_google_absl//absl/types:variant", ], ) @@ -637,7 +617,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", "@com_google_absl//absl/strings", ], ) @@ -676,7 +655,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", "@com_google_absl//absl/strings", ], ) @@ -717,7 +695,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", "//tensorflow/lite/delegates/gpu/metal:environment", "//tensorflow/lite/delegates/gpu/metal:runtime_options", - "@com_google_absl//absl/strings", ], ) @@ -753,7 +730,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", - "//tensorflow/lite/delegates/gpu/metal:runtime_options", "//tensorflow/lite/delegates/gpu/metal/kernels:util", ], ) @@ -789,7 +765,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:model", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:shape", - "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", "//tensorflow/lite/delegates/gpu/metal:environment", @@ -873,7 +848,6 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:winograd_util", "//tensorflow/lite/delegates/gpu/metal:compute_task_descriptor", "//tensorflow/lite/delegates/gpu/metal:runtime_options", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ], ) From a73b5ce940b3eeb5e322463e42c862e46c49a58e Mon Sep 17 00:00:00 2001 From: Zhenyu Tan Date: Mon, 3 Aug 2020 16:58:43 -0700 Subject: [PATCH 0302/1017] partial fixit for feature_columns_test PiperOrigin-RevId: 324713509 Change-Id: Ie1b69ed70ac787d8782f48fa2f9831c9bd622a17 --- .../feature_column/feature_column_test.py | 51 ++++++------------- 1 file changed, 15 insertions(+), 36 deletions(-) diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index 2ea7face467..d6d4d2eb1a1 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -171,7 +171,6 @@ class LazyColumnTest(test.TestCase): TypeError, '"key" must be either a "str" or "_FeatureColumn".'): builder.get(NotAFeatureColumn()) - @test_util.run_deprecated_v1 def test_expand_dim_rank_1_sparse_tensor_empty_batch(self): # empty 1-D sparse tensor: builder = _LazyBuilder(features={'a': sparse_tensor.SparseTensor( @@ -179,7 +178,7 @@ class LazyColumnTest(test.TestCase): dense_shape=[0], values=np.array([]))}) with self.cached_session(): - spv = builder.get('a').eval() + spv = builder.get('a') self.assertAllEqual(np.array([0, 1], dtype=np.int64), spv.dense_shape) self.assertAllEqual( np.reshape(np.array([], dtype=np.int64), (0, 2)), spv.indices) @@ -187,7 +186,6 @@ class LazyColumnTest(test.TestCase): class NumericColumnTest(test.TestCase): - @test_util.run_deprecated_v1 def test_defaults(self): a = fc._numeric_column('aaa') self.assertEqual('aaa', a.key) @@ -266,7 +264,6 @@ class NumericColumnTest(test.TestCase): 'aaa': parsing_ops.FixedLenFeature((2, 3), dtype=dtypes.int32) }, a._parse_example_spec) - @test_util.run_deprecated_v1 def test_parse_example_no_default_value(self): price = fc._numeric_column('price', shape=[2]) data = example_pb2.Example(features=feature_pb2.Features( @@ -309,7 +306,6 @@ class NumericColumnTest(test.TestCase): with self.assertRaisesRegex(TypeError, 'must be a callable'): fc._numeric_column('price', normalizer_fn='NotACallable') - @test_util.run_deprecated_v1 def test_normalizer_fn_transform_feature(self): def _increment_two(input_tensor): @@ -328,7 +324,7 @@ class NumericColumnTest(test.TestCase): price = fc._numeric_column('price', shape=[2], normalizer_fn=_increment_two) builder = _LazyBuilder({'price': [[1., 2.], [5., 6.]]}) - self.assertEqual(builder.get(price), price._get_dense_tensor(builder)) + self.assertAllClose(builder.get(price), price._get_dense_tensor(builder)) def test_sparse_tensor_not_supported(self): price = fc._numeric_column('price') @@ -340,7 +336,6 @@ class NumericColumnTest(test.TestCase): with self.assertRaisesRegex(ValueError, 'must be a Tensor'): price._transform_feature(builder) - @test_util.run_deprecated_v1 def test_deep_copy(self): a = fc._numeric_column('aaa', shape=[1, 2], default_value=[[3., 2.]]) a_copy = copy.deepcopy(a) @@ -353,7 +348,6 @@ class NumericColumnTest(test.TestCase): 'aaa', shape=[1, 2], default_value=np.array([[3., 2.]])) self.assertEqual(a.default_value, ((3., 2.),)) - @test_util.run_deprecated_v1 def test_linear_model(self): price = fc._numeric_column('price') with ops.Graph().as_default(): @@ -368,7 +362,6 @@ class NumericColumnTest(test.TestCase): sess.run(price_var.assign([[10.]])) self.assertAllClose([[10.], [50.]], self.evaluate(predictions)) - @test_util.run_deprecated_v1 def test_keras_linear_model(self): price = fc._numeric_column('price') with ops.Graph().as_default(): @@ -465,8 +458,8 @@ class BucketizedColumnTest(test.TestCase): 'price': [[-1., 1.], [5., 6.]] }, [bucketized_price]) with _initialized_session(): - self.assertAllEqual([[0, 1], [3, 4]], - transformed_tensor[bucketized_price].eval()) + self.assertAllClose([[0, 1], [3, 4]], + transformed_tensor[bucketized_price]) def test_get_dense_tensor_one_input_value(self): """Tests _get_dense_tensor() for input with shape=[1].""" @@ -539,7 +532,6 @@ class BucketizedColumnTest(test.TestCase): with self.assertRaisesRegex(ValueError, 'must be a Tensor'): bucketized_price._transform_feature(builder) - @test_util.run_deprecated_v1 def test_deep_copy(self): a = fc._numeric_column('aaa', shape=[2]) a_bucketized = fc._bucketized_column(a, boundaries=[0, 1]) @@ -667,7 +659,6 @@ class BucketizedColumnTest(test.TestCase): class HashedCategoricalColumnTest(test.TestCase): - @test_util.run_deprecated_v1 def test_defaults(self): a = fc._categorical_column_with_hash_bucket('aaa', 10) self.assertEqual('aaa', a.name) @@ -695,7 +686,6 @@ class HashedCategoricalColumnTest(test.TestCase): with self.assertRaisesRegex(ValueError, 'dtype must be string or integer'): fc._categorical_column_with_hash_bucket('aaa', 10, dtype=dtypes.float32) - @test_util.run_deprecated_v1 def test_deep_copy(self): original = fc._categorical_column_with_hash_bucket('aaa', 10) for column in (original, copy.deepcopy(original)): @@ -735,10 +725,8 @@ class HashedCategoricalColumnTest(test.TestCase): sparse_tensor.SparseTensorValue( indices=[[0, 0], [0, 1]], values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) + dense_shape=[1, 2]), features['aaa'].eval()) - @test_util.run_deprecated_v1 def test_strings_should_be_hashed(self): hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10) wire_tensor = sparse_tensor.SparseTensor( @@ -753,7 +741,7 @@ class HashedCategoricalColumnTest(test.TestCase): self.assertEqual(dtypes.int64, output.values.dtype) self.assertAllEqual(expected_values, output.values) self.assertAllEqual(wire_tensor.indices, output.indices) - self.assertAllEqual(wire_tensor.dense_shape, output.dense_shape.eval()) + self.assertAllEqual(wire_tensor.dense_shape, output.dense_shape) def test_tensor_dtype_should_be_string_or_integer(self): string_fc = fc._categorical_column_with_hash_bucket( @@ -793,7 +781,6 @@ class HashedCategoricalColumnTest(test.TestCase): with self.assertRaisesRegex(ValueError, 'dtype must be compatible'): builder.get(hashed_sparse) - @test_util.run_deprecated_v1 def test_ints_should_be_hashed(self): hashed_sparse = fc._categorical_column_with_hash_bucket( 'wire', 10, dtype=dtypes.int64) @@ -852,7 +839,6 @@ class HashedCategoricalColumnTest(test.TestCase): ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)) self.assertCountEqual([], ops.get_collection('my_weights')) - @test_util.run_deprecated_v1 def test_get_sparse_tensors_dense_input(self): hashed_sparse = fc._categorical_column_with_hash_bucket('wire', 10) builder = _LazyBuilder({'wire': (('omar', ''), ('stringer', 'marlo'))}) @@ -860,7 +846,6 @@ class HashedCategoricalColumnTest(test.TestCase): self.assertIsNone(id_weight_pair.weight_tensor) self.assertEqual(builder.get(hashed_sparse), id_weight_pair.id_tensor) - @test_util.run_deprecated_v1 def test_linear_model(self): wire_column = fc._categorical_column_with_hash_bucket('wire', 4) self.assertEqual(4, wire_column._num_buckets) @@ -878,12 +863,11 @@ class HashedCategoricalColumnTest(test.TestCase): self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var)) self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) - wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() + self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,)))) # 'marlo' -> 3: wire_var[3] = 4 # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6 self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions)) - @test_util.run_deprecated_v1 def test_keras_linear_model(self): wire_column = fc._categorical_column_with_hash_bucket('wire', 4) self.assertEqual(4, wire_column._num_buckets) @@ -902,7 +886,7 @@ class HashedCategoricalColumnTest(test.TestCase): self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), self.evaluate(wire_var)) self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) - wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval() + self.evaluate(wire_var.assign(((1.,), (2.,), (3.,), (4.,)))) # 'marlo' -> 3: wire_var[3] = 4 # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6 self.assertAllClose(((4.,), (6.,)), self.evaluate(predictions)) @@ -990,7 +974,6 @@ class CrossedColumnTest(test.TestCase): crossed = fc._crossed_column([b, 'c'], 15) self.assertEqual(15, crossed._num_buckets) - @test_util.run_deprecated_v1 def test_deep_copy(self): a = fc._numeric_column('a', dtype=dtypes.int32) b = fc._bucketized_column(a, boundaries=[0, 1]) @@ -1001,7 +984,6 @@ class CrossedColumnTest(test.TestCase): self.assertEqual(15, crossed2_copy.hash_bucket_size) self.assertEqual(5, crossed2_copy.hash_key) - @test_util.run_deprecated_v1 def test_parse_example(self): price = fc._numeric_column('price', shape=[2]) bucketized_price = fc._bucketized_column(price, boundaries=[0, 50]) @@ -1044,7 +1026,7 @@ class CrossedColumnTest(test.TestCase): } outputs = _transform_features(features, [price_cross_wire]) output = outputs[price_cross_wire] - with self.cached_session() as sess: + with self.cached_session(): output_val = self.evaluate(output) self.assertAllEqual( [[0, 0], [0, 1], [1, 0], [1, 1], [1, 2], [1, 3]], output_val.indices) @@ -1052,7 +1034,6 @@ class CrossedColumnTest(test.TestCase): self.assertIn(val, list(range(hash_bucket_size))) self.assertAllEqual([2, 4], output_val.dense_shape) - @test_util.run_deprecated_v1 def test_get_sparse_tensors(self): a = fc._numeric_column('a', dtype=dtypes.int32, shape=(2,)) b = fc._bucketized_column(a, boundaries=(0, 1)) @@ -1120,7 +1101,6 @@ class CrossedColumnTest(test.TestCase): self.assertAllEqual(expected_values, id_tensor_eval.values) self.assertAllEqual((2, 4), id_tensor_eval.dense_shape) - @test_util.run_deprecated_v1 def test_linear_model(self): """Tests linear_model. @@ -1139,15 +1119,15 @@ class CrossedColumnTest(test.TestCase): }, (crossed,)) bias = get_linear_model_bias() crossed_var = get_linear_model_column_var(crossed) - with _initialized_session() as sess: + with _initialized_session(): self.assertAllClose((0.,), self.evaluate(bias)) self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)), self.evaluate(crossed_var)) self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) - sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,)))) + self.evaluate(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,)))) # Expected ids after cross = (1, 0, 1, 3, 4, 2) self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions)) - sess.run(bias.assign((.1,))) + self.evaluate(bias.assign((.1,))) self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions)) def test_linear_model_with_weights(self): @@ -1202,7 +1182,6 @@ class CrossedColumnTest(test.TestCase): dense_shape=(2, 2)), }, (crossed,)) - @test_util.run_deprecated_v1 def test_keras_linear_model(self): """Tests _LinearModel. @@ -1223,15 +1202,15 @@ class CrossedColumnTest(test.TestCase): }, (crossed,)) bias = get_linear_model_bias() crossed_var = get_linear_model_column_var(crossed) - with _initialized_session() as sess: + with _initialized_session(): self.assertAllClose((0.,), self.evaluate(bias)) self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)), self.evaluate(crossed_var)) self.assertAllClose(((0.,), (0.,)), self.evaluate(predictions)) - sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,)))) + self.evaluate(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,)))) # Expected ids after cross = (1, 0, 1, 3, 4, 2) self.assertAllClose(((3.,), (14.,)), self.evaluate(predictions)) - sess.run(bias.assign((.1,))) + self.evaluate(bias.assign((.1,))) self.assertAllClose(((3.1,), (14.1,)), self.evaluate(predictions)) def test_keras_linear_model_with_weights(self): From bb55b48537eb8373941cc812fa140e14599c3d17 Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Tue, 4 Aug 2020 00:04:45 +0000 Subject: [PATCH 0303/1017] Passing function_test --- tensorflow/python/eager/def_function.py | 11 ++++++----- tensorflow/python/eager/function.py | 6 ++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py index a3bc0516403..03bc48050e0 100644 --- a/tensorflow/python/eager/def_function.py +++ b/tensorflow/python/eager/def_function.py @@ -861,7 +861,7 @@ class Function(object): # If we did not create any variables the trace we have is good enough. return self._concrete_stateful_fn._filtered_call(flat_args, flat_kwds) # pylint: disable=protected-access - def fn_with_cond(*inner_args, **inner_kwds): + def fn_with_cond(inner_args, inner_kwds, inner_flat_args, inner_flat_kwds): """Conditionally runs initialization if it's needed.""" condition = True for wr in self._created_variables: @@ -910,15 +910,16 @@ class Function(object): condition, lambda: self._stateless_fn(*inner_args, **inner_kwds), functools.partial(self._concrete_stateful_fn._filtered_call, # pylint: disable=protected-access - inner_args, inner_kwds)) + inner_flat_args, inner_flat_kwds)) # We've created variables and are unable to lift the initialization graphs, # so we fall back to initializing with conds while running the function. - canon_args, canon_kwds, _, _ = \ + canon_args, canon_kwds, flat_args, flat_kwds = \ self._stateful_fn._function_spec.canonicalize_function_inputs( # pylint: disable=protected-access *args, **kwds) - # TODO(jlchu): fix arguments for this, two cases for fn_with_cond - return function_lib.defun(fn_with_cond)(*canon_args, **canon_kwds) + # TODO(jlchu): verify that mdofication to fn_with_cond works + return function_lib.defun(fn_with_cond)(canon_args, canon_kwds, + flat_args, flat_kwds) @property def python_function(self): diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 8c5815e21b3..b0a2b996ab8 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -3285,6 +3285,8 @@ class Function(object): if self.input_signature is None or args is not None or kwargs is not None: args, kwargs, flat_args, flat_kwargs = \ self._function_spec.canonicalize_function_inputs(*args, **kwargs) + else: + flat_args, flat_kwargs = [], [] cache_key = self._cache_key(args, kwargs) @@ -3324,7 +3326,7 @@ class Function(object): and self.input_signature is None and call_context_key in self._function_cache.missed): return_function, _, _ = \ - self.define_function_with_shape_relaxation(args, kwargs) + self._define_function_with_shape_relaxation(args, kwargs) #TODO(jlchu): Investigate modifying above function sig directly return return_function, flat_args, flat_kwargs @@ -3334,7 +3336,7 @@ class Function(object): if ops.get_default_graph()._distribution_strategy_stack: self._traced_with_distribution_strategy = True - + return graph_function, flat_args, flat_kwargs From 2e89544a158acf0e0cd9fd4e34d7bb869dea5256 Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Tue, 4 Aug 2020 00:13:49 +0000 Subject: [PATCH 0304/1017] Pylint polishing --- tensorflow/python/eager/def_function.py | 2 +- tensorflow/python/eager/function.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py index 03bc48050e0..a1ffd2f9efc 100644 --- a/tensorflow/python/eager/def_function.py +++ b/tensorflow/python/eager/def_function.py @@ -918,7 +918,7 @@ class Function(object): self._stateful_fn._function_spec.canonicalize_function_inputs( # pylint: disable=protected-access *args, **kwds) # TODO(jlchu): verify that mdofication to fn_with_cond works - return function_lib.defun(fn_with_cond)(canon_args, canon_kwds, + return function_lib.defun(fn_with_cond)(canon_args, canon_kwds, flat_args, flat_kwds) @property diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index b0a2b996ab8..d9b141e049a 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -3336,7 +3336,7 @@ class Function(object): if ops.get_default_graph()._distribution_strategy_stack: self._traced_with_distribution_strategy = True - + return graph_function, flat_args, flat_kwargs From 611067afc6efb94ce3b9917928c20a3454d83962 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 4 Aug 2020 00:12:27 +0000 Subject: [PATCH 0305/1017] clean up --- tensorflow/c/kernels/histogram_summary_op.cc | 21 ++++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tensorflow/c/kernels/histogram_summary_op.cc b/tensorflow/c/kernels/histogram_summary_op.cc index 7219f4d92d0..a3dfc102472 100644 --- a/tensorflow/c/kernels/histogram_summary_op.cc +++ b/tensorflow/c/kernels/histogram_summary_op.cc @@ -113,17 +113,16 @@ static void HistogramSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { // because of an invalid values argument. StatusWrapper allocation_status_wrapper; TensorWrapper summary_tensor_wrapper; - TF_Tensor* summary_tensor= TF_AllocateOutput(ctx, 0, + summary_tensor_wrapper.t = TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, sizeof(tensorflow::tstring), allocation_status_wrapper.s); - summary_tensor_wrapper.t = summary_tensor; if (TF_GetCode(allocation_status_wrapper.s) != TF_OK){ TF_OpKernelContext_Failure(ctx, allocation_status_wrapper.s); return; } tensorflow::tstring* output_tstring = reinterpret_cast( - TF_TensorData(summary_tensor)); + TF_TensorData(summary_tensor_wrapper.t)); CHECK(SerializeToTString(s, output_tstring)); } @@ -151,14 +150,14 @@ void RegisterHistogramSummaryOpKernel() { // register the Histogram Summary kernel. TF_ATTRIBUTE_UNUSED static bool IsHistogramSummaryOpKernelRegistered = []() { if (SHOULD_REGISTER_OP_KERNEL("HistogramSummary")) { - // RegisterHistogramSummaryOpKernel(); - // RegisterHistogramSummaryOpKernel(); - // RegisterHistogramSummaryOpKernel(); - // RegisterHistogramSummaryOpKernel(); - // RegisterHistogramSummaryOpKernel(); - // RegisterHistogramSummaryOpKernel(); - // RegisterHistogramSummaryOpKernel(); - // RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); + RegisterHistogramSummaryOpKernel(); RegisterHistogramSummaryOpKernel(); RegisterHistogramSummaryOpKernel(); RegisterHistogramSummaryOpKernel(); From dab856a93fdecb88880e08bc94928f3e0f141cf9 Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Mon, 3 Aug 2020 17:10:40 -0700 Subject: [PATCH 0306/1017] Removed ElementwiseOperation. Simplified ex-ElementwiseOperations, ReLU, PReLU, etc. PiperOrigin-RevId: 324715510 Change-Id: I3d98cdbcc8075bb91f20e065b0aca2ab16a4e8e5 --- .../delegates/gpu/cl/inference_context.cc | 8 +- .../lite/delegates/gpu/cl/kernels/add.cc | 43 ++-- .../lite/delegates/gpu/cl/kernels/add.h | 22 +- .../lite/delegates/gpu/cl/kernels/add_test.cc | 6 +- .../delegates/gpu/cl/kernels/elementwise.cc | 215 ++++++------------ .../delegates/gpu/cl/kernels/elementwise.h | 91 ++------ .../gpu/cl/kernels/elementwise_test.cc | 62 +++-- .../delegates/gpu/cl/kernels/gpu_operation.cc | 158 +++++++------ .../delegates/gpu/cl/kernels/gpu_operation.h | 76 ++----- .../lite/delegates/gpu/cl/kernels/prelu.cc | 46 ++-- .../lite/delegates/gpu/cl/kernels/prelu.h | 41 +--- .../delegates/gpu/cl/kernels/prelu_test.cc | 4 +- .../gpu/cl/kernels/quantize_and_dequantize.cc | 66 ++---- .../gpu/cl/kernels/quantize_and_dequantize.h | 38 +--- .../kernels/quantize_and_dequantize_test.cc | 20 +- .../lite/delegates/gpu/cl/kernels/relu.cc | 42 ++-- .../lite/delegates/gpu/cl/kernels/relu.h | 22 +- .../delegates/gpu/cl/kernels/relu_test.cc | 8 +- .../gpu/cl/selectors/operation_selector.cc | 75 +++--- .../gpu/cl/selectors/simple_selectors.cc | 28 ++- .../gpu/cl/selectors/simple_selectors.h | 8 +- 21 files changed, 368 insertions(+), 711 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/inference_context.cc b/tensorflow/lite/delegates/gpu/cl/inference_context.cc index 3067c81ec94..8e23eb1bcee 100644 --- a/tensorflow/lite/delegates/gpu/cl/inference_context.cc +++ b/tensorflow/lite/delegates/gpu/cl/inference_context.cc @@ -390,9 +390,7 @@ void InferenceContext::Merge() { continue; } auto& linkable_node = nodes_[next_nodes[0]]; - auto* elementwise = - dynamic_cast(linkable_node.operations[0].get()); - if (!elementwise || !elementwise->IsLinkable() || + if (!linkable_node.operations[0]->IsLinkable() || linkable_node.outputs.size() != 1 || !IsReady(ready_tensors, linkable_node)) { continue; @@ -410,9 +408,7 @@ void InferenceContext::Merge() { } for (auto& node : nodes_) { for (int j = 1; j < node.operations.size(); ++j) { - auto* elementwise = - dynamic_cast(node.operations[j].get()); - node.operations[0]->AddOperation(elementwise); + node.operations[0]->AddOperation(node.operations[j].get()); } } } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/add.cc b/tensorflow/lite/delegates/gpu/cl/kernels/add.cc index 1d09e39b83b..1cb41e79d88 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/add.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/add.cc @@ -25,42 +25,29 @@ namespace tflite { namespace gpu { namespace cl { -Add::Add(const OperationDef& definition, const std::vector& channels, - int dst_channels) - : ElementwiseOperation(definition) { +GPUOperation CreateAdd(const OperationDef& definition, + const std::vector& channels, int dst_channels) { + GPUOperation add(definition); int dst_depth = DivideRoundUp(dst_channels, 4); int src0_depth = DivideRoundUp(channels[0], 4); - linkable_ = dst_depth == src0_depth; + add.elementwise_ = true; + add.linkable_ = dst_depth == src0_depth; if (src0_depth < dst_depth) { - check_src_channels_size_ = true; + add.check_src_channels_size_ = true; } - for (int i = 1; i < definition_.src_tensors.size(); ++i) { + for (int i = 1; i < definition.src_tensors.size(); ++i) { const std::string tensor_name = absl::StrCat("src_data_", i); - auto src_desc = definition_.src_tensors[i]; - if (definition_.IsBatchSupported()) { + auto src_desc = definition.src_tensors[i]; + if (definition.IsBatchSupported()) { src_desc.SetStateVar("BatchedWidth", "true"); } - AddSrcTensor(tensor_name, src_desc); - code_ += "if (S_COORD < args." + tensor_name + ".Slices()) {\n"; - code_ += " in_out_value += args." + tensor_name + - ".Read(X_COORD, Y_COORD, S_COORD);\n"; - code_ += "}\n"; + add.AddSrcTensor(tensor_name, src_desc); + add.code_ += "if (S_COORD < args." + tensor_name + ".Slices()) {\n"; + add.code_ += " in_out_value += args." + tensor_name + + ".Read(X_COORD, Y_COORD, S_COORD);\n"; + add.code_ += "}\n"; } -} - -Add::Add(Add&& operation) : ElementwiseOperation(std::move(operation)) {} - -Add& Add::operator=(Add&& operation) { - if (this != &operation) { - ElementwiseOperation::operator=(std::move(operation)); - } - return *this; -} - -Add CreateAdd(const OperationDef& definition, const std::vector& channels, - int dst_channels) { - Add operation(definition, channels, dst_channels); - return operation; + return add; } } // namespace cl diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/add.h b/tensorflow/lite/delegates/gpu/cl/kernels/add.h index 81b2fed116f..0e9d7e0d333 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/add.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/add.h @@ -27,24 +27,10 @@ namespace tflite { namespace gpu { namespace cl { -// Add operation inherited from ElementwiseOperation, but it is more -// complicated than usual elementwise, that is why it has own versions for -// Compile. Add operation support not equal tensors on input (for possibility to -// remove Padding operation with zeroes in Z dimension) -class Add : public ElementwiseOperation { - public: - Add(const OperationDef& definition, const std::vector& channels, - int dst_channels); - - // Move only - Add(Add&& operation); - Add& operator=(Add&& operation); - Add(const Add&) = delete; - Add& operator=(const Add&) = delete; -}; - -Add CreateAdd(const OperationDef& definition, const std::vector& channels, - int dst_channels); +// Add operation supports not equal tensors on input (for possibility to +// remove Padding operation with zeroes in channels dimension) +GPUOperation CreateAdd(const OperationDef& definition, + const std::vector& channels, int dst_channels); } // namespace cl } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/add_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/add_test.cc index 1eccab87646..2856b37a497 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/add_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/add_test.cc @@ -49,7 +49,7 @@ TEST_F(OpenCLOperationTest, AddTwoEqualTensors) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - Add operation = CreateAdd(op_def, channels, channels[0]); + GPUOperation operation = CreateAdd(op_def, channels, channels[0]); ASSERT_OK(ExecuteGPUOperation({src0, src1}, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -77,7 +77,7 @@ TEST_F(OpenCLOperationTest, AddFirstTensorHasMoreChannelsThanSecond) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - Add operation = CreateAdd(op_def, channels, channels[0]); + GPUOperation operation = CreateAdd(op_def, channels, channels[0]); ASSERT_OK(ExecuteGPUOperation({src0, src1}, creation_context_, &operation, BHWC(1, 2, 1, 6), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -107,7 +107,7 @@ TEST_F(OpenCLOperationTest, AddFirstTensorHasLessChannelsThanSecond) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - Add operation = CreateAdd(op_def, channels, 6); + GPUOperation operation = CreateAdd(op_def, channels, 6); ASSERT_OK(ExecuteGPUOperation({src0, src1}, creation_context_, &operation, BHWC(1, 2, 1, 6), &dst_tensor)); EXPECT_THAT(dst_tensor.data, diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc index 21866021e91..063b15c1b69 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc @@ -134,128 +134,33 @@ std::string GetTwoInputCode(const OperationType& op_type, } } // namespace -ElementwiseOneInput::ElementwiseOneInput(const OperationDef& definition, - const OperationType& op_type) - : ElementwiseOperation(definition) { - code_ = GetOneInputCode(op_type, definition.precision, "in_out_value"); +GPUOperation CreateElementwiseOneInput(const OperationDef& definition, + const OperationType& op_type) { + GPUOperation op(definition); + op.elementwise_ = true; + op.code_ = GetOneInputCode(op_type, definition.precision, "in_out_value"); + return op; } -ElementwiseOneInput::ElementwiseOneInput(ElementwiseOneInput&& operation) - : ElementwiseOperation(std::move(operation)) {} - -ElementwiseOneInput& ElementwiseOneInput::operator=( - ElementwiseOneInput&& operation) { - if (this != &operation) { - ElementwiseOperation::operator=(std::move(operation)); - } - return *this; -} - -ElementwiseOneInput CreateElementwiseOneInput(const OperationDef& definition, - const OperationType& op_type) { - ElementwiseOneInput operation(definition, op_type); - return operation; -} - -ElementwiseOneRuntimeOneScalar::ElementwiseOneRuntimeOneScalar( - const OperationDef& definition, const OperationType& op_type, - float scalar_parameter, CalculationsPrecision scalar_precision) - : ElementwiseOperation(definition) { - if (definition.precision == CalculationsPrecision::F32) { - args_.AddFloat("scalar", scalar_parameter); - } else { - args_.AddHalf("scalar", half(scalar_parameter)); - } - code_ = GetTwoInputCode(op_type, "in_out_value", "args.scalar"); -} - -ElementwiseOneRuntimeOneScalar::ElementwiseOneRuntimeOneScalar( - ElementwiseOneRuntimeOneScalar&& operation) - : ElementwiseOperation(std::move(operation)) {} - -ElementwiseOneRuntimeOneScalar& ElementwiseOneRuntimeOneScalar::operator=( - ElementwiseOneRuntimeOneScalar&& operation) { - if (this != &operation) { - ElementwiseOperation::operator=(std::move(operation)); - } - return *this; -} - -ElementwiseOneRuntimeOneScalar CreateElementwiseOneRuntimeOneScalar( +GPUOperation CreateElementwiseOneRuntimeOneScalar( const CreationContext& creation_context, const OperationDef& definition, const OperationType& op_type, float scalar_parameter) { - const auto scalar_precision = creation_context.device->IsPowerVR() - ? CalculationsPrecision::F32 - : definition.precision; - ElementwiseOneRuntimeOneScalar operation(definition, op_type, - scalar_parameter, scalar_precision); - return operation; -} - -ElementwiseTwoInput::ElementwiseTwoInput(const OperationDef& definition, - const OperationType& op_type, - const BroadcastSettings& broadcast) - : ElementwiseOperation(definition), - broadcast_(broadcast) { - auto src_desc = definition.src_tensors[1]; - if (definition.IsBatchSupported()) { - src_desc.SetStateVar("BatchedWidth", "true"); + GPUOperation op(definition); + op.elementwise_ = true; + if (definition.precision == CalculationsPrecision::F32) { + op.args_.AddFloat("scalar", scalar_parameter); + } else { + op.args_.AddHalf("scalar", half(scalar_parameter)); } - AddSrcTensor("second_tensor", src_desc); - const std::string x_coord = broadcast.width ? "0" : "X_COORD"; - const std::string y_coord = broadcast.height ? "0" : "Y_COORD"; - const std::string s_coord = broadcast.channels ? "0" : "S_COORD"; - code_ = absl::StrCat("FLT4 second_val = args.second_tensor.Read(", x_coord, - ", ", y_coord, ", ", s_coord, ");\n"); - if (broadcast.channels) { - code_ += " second_val.y = second_val.x;\n"; - code_ += " second_val.z = second_val.x;\n"; - code_ += " second_val.w = second_val.x;\n"; - } - code_ += GetTwoInputCode(op_type, "in_out_value", "second_val"); -} - -ElementwiseTwoInput::ElementwiseTwoInput(const OperationDef& definition, - const OperationType& op_type, - const BroadcastSettings& broadcast, - Tensor&& constant_tensor) - : ElementwiseOperation(definition), - broadcast_(broadcast) { - auto descriptor = constant_tensor.GetDescriptor(); - args_.AddObject("second_tensor", AccessType::READ, - absl::make_unique(std::move(constant_tensor)), - absl::make_unique(descriptor)); - const std::string x_coord = broadcast.width ? "0" : "X_COORD"; - const std::string y_coord = broadcast.height ? "0" : "Y_COORD"; - const std::string s_coord = broadcast.channels ? "0" : "S_COORD"; - code_ = absl::StrCat("FLT4 second_val = args.second_tensor.Read(", x_coord, - ", ", y_coord, ", ", s_coord, ");\n"); - if (broadcast.channels) { - code_ += " second_val.y = second_val.x;\n"; - code_ += " second_val.z = second_val.x;\n"; - code_ += " second_val.w = second_val.x;\n"; - } - code_ += GetTwoInputCode(op_type, "in_out_value", "second_val"); -} - -ElementwiseTwoInput::ElementwiseTwoInput(ElementwiseTwoInput&& operation) - : ElementwiseOperation(std::move(operation)), - broadcast_(operation.broadcast_) {} - -ElementwiseTwoInput& ElementwiseTwoInput::operator=( - ElementwiseTwoInput&& operation) { - if (this != &operation) { - broadcast_ = operation.broadcast_; - ElementwiseOperation::operator=(std::move(operation)); - } - return *this; + op.code_ = GetTwoInputCode(op_type, "in_out_value", "args.scalar"); + return op; } absl::Status CreateElementwiseTwoInput( const CreationContext& creation_context, const OperationDef& definition, const OperationType& op_type, const tflite::gpu::Tensor& constant_tensor, - ElementwiseTwoInput* result) { + GPUOperation* result) { const BHWC shape = BHWC(1, 1, 1, constant_tensor.shape.v); TensorStorageType storage_type = SelectBestStorageType(*creation_context.context, *creation_context.device, @@ -268,12 +173,21 @@ absl::Status CreateElementwiseTwoInput( &gpu_tensor)); RETURN_IF_ERROR( gpu_tensor.WriteData(creation_context.queue, constant_tensor)); - BroadcastSettings broadcast; - broadcast.width = true; - broadcast.height = true; - broadcast.channels = shape.c == 1; - *result = ElementwiseTwoInput(definition, op_type, broadcast, - std::move(gpu_tensor)); + + *result = GPUOperation(definition); + result->elementwise_ = true; + result->args_.AddObject("second_tensor", AccessType::READ, + absl::make_unique(std::move(gpu_tensor)), + absl::make_unique(desc)); + const std::string s_coord = shape.c == 1 ? "0" : "S_COORD"; + result->code_ = absl::StrCat( + "FLT4 second_val = args.second_tensor.Read(0, 0, ", s_coord, ");\n"); + if (shape.c == 1) { + result->code_ += " second_val.y = second_val.x;\n"; + result->code_ += " second_val.z = second_val.x;\n"; + result->code_ += " second_val.w = second_val.x;\n"; + } + result->code_ += GetTwoInputCode(op_type, "in_out_value", "second_val"); return absl::OkStatus(); } @@ -281,7 +195,7 @@ absl::Status CreateElementwiseTwoInput( const CreationContext& creation_context, const OperationDef& definition, const OperationType& op_type, const tflite::gpu::Tensor& constant_tensor, - ElementwiseTwoInput* result) { + GPUOperation* result) { const BHWC shape = BHWC(1, constant_tensor.shape.h, constant_tensor.shape.w, constant_tensor.shape.c); TensorStorageType storage_type = @@ -295,34 +209,49 @@ absl::Status CreateElementwiseTwoInput( &gpu_tensor)); RETURN_IF_ERROR( gpu_tensor.WriteData(creation_context.queue, constant_tensor)); - BroadcastSettings broadcast; - broadcast.width = shape.w == 1; - broadcast.height = shape.h == 1; - broadcast.channels = shape.c == 1; - *result = ElementwiseTwoInput(definition, op_type, broadcast, - std::move(gpu_tensor)); + + *result = GPUOperation(definition); + result->elementwise_ = true; + result->args_.AddObject("second_tensor", AccessType::READ, + absl::make_unique(std::move(gpu_tensor)), + absl::make_unique(desc)); + const std::string x_coord = shape.w == 1 ? "0" : "X_COORD"; + const std::string y_coord = shape.h == 1 ? "0" : "Y_COORD"; + const std::string s_coord = shape.c == 1 ? "0" : "S_COORD"; + result->code_ = absl::StrCat("FLT4 second_val = args.second_tensor.Read(", + x_coord, ", ", y_coord, ", ", s_coord, ");\n"); + if (shape.c == 1) { + result->code_ += " second_val.y = second_val.x;\n"; + result->code_ += " second_val.z = second_val.x;\n"; + result->code_ += " second_val.w = second_val.x;\n"; + } + result->code_ += GetTwoInputCode(op_type, "in_out_value", "second_val"); + return absl::OkStatus(); } -ElementwiseTwoInput CreateElementwiseTwoInput(const OperationDef& definition, - const OperationType& op_type, - const BHWC& shape) { - BroadcastSettings broadcast; - broadcast.width = shape.w == 1; - broadcast.height = shape.h == 1; - broadcast.channels = shape.c == 1; - ElementwiseTwoInput operation(definition, op_type, broadcast); - return operation; -} - -ElementwiseTwoInput CreateElementwiseTwoInput(const OperationDef& definition, - const OperationType& op_type) { - BroadcastSettings broadcast; - broadcast.width = false; - broadcast.height = false; - broadcast.channels = false; - ElementwiseTwoInput operation(definition, op_type, broadcast); - return operation; +GPUOperation CreateElementwiseTwoInput(const OperationDef& definition, + const OperationType& op_type, + const BHWC& shape) { + GPUOperation op(definition); + op.elementwise_ = true; + auto src_desc = definition.src_tensors[1]; + if (definition.IsBatchSupported()) { + src_desc.SetStateVar("BatchedWidth", "true"); + } + op.AddSrcTensor("second_tensor", src_desc); + const std::string x_coord = shape.w == 1 ? "0" : "X_COORD"; + const std::string y_coord = shape.h == 1 ? "0" : "Y_COORD"; + const std::string s_coord = shape.c == 1 ? "0" : "S_COORD"; + op.code_ = absl::StrCat("FLT4 second_val = args.second_tensor.Read(", x_coord, + ", ", y_coord, ", ", s_coord, ");\n"); + if (shape.c == 1) { + op.code_ += " second_val.y = second_val.x;\n"; + op.code_ += " second_val.z = second_val.x;\n"; + op.code_ += " second_val.w = second_val.x;\n"; + } + op.code_ += GetTwoInputCode(op_type, "in_out_value", "second_val"); + return op; } } // namespace cl diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.h b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.h index 9712ee96b90..d03d535b39a 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.h @@ -26,93 +26,38 @@ namespace tflite { namespace gpu { namespace cl { -// Class for simple one input operations without any parameters, for example -// log, sin, cos and etc. -class ElementwiseOneInput : public ElementwiseOperation { - public: - ElementwiseOneInput(const OperationDef& definition, - const OperationType& op_type); +// Creates simple one input operation without any parameters, for example +// log, sin, cos, etc. +GPUOperation CreateElementwiseOneInput(const OperationDef& definition, + const OperationType& op_type); - // Move only - ElementwiseOneInput(ElementwiseOneInput&& operation); - ElementwiseOneInput& operator=(ElementwiseOneInput&& operation); - ElementwiseOneInput(const ElementwiseOneInput&) = delete; - ElementwiseOneInput& operator=(const ElementwiseOneInput&) = delete; -}; - -ElementwiseOneInput CreateElementwiseOneInput(const OperationDef& definition, - const OperationType& op_type); - -// Class for simple two input (first input is runtime tensor and second input is -// scalar argument) operations without any parameters, for example sub, div and -// etc. -class ElementwiseOneRuntimeOneScalar : public ElementwiseOperation { - public: - ElementwiseOneRuntimeOneScalar(const OperationDef& definition, - const OperationType& op_type, - float scalar_parameter, - CalculationsPrecision scalar_precision); - - // Move only - ElementwiseOneRuntimeOneScalar(ElementwiseOneRuntimeOneScalar&& operation); - ElementwiseOneRuntimeOneScalar& operator=( - ElementwiseOneRuntimeOneScalar&& operation); - ElementwiseOneRuntimeOneScalar(const ElementwiseOneRuntimeOneScalar&) = - delete; - ElementwiseOneRuntimeOneScalar& operator=( - const ElementwiseOneRuntimeOneScalar&) = delete; -}; - -ElementwiseOneRuntimeOneScalar CreateElementwiseOneRuntimeOneScalar( +// Creates simple two input (first input is runtime tensor and second input is +// scalar argument) operation, for example sub, div, pow, etc. +GPUOperation CreateElementwiseOneRuntimeOneScalar( const CreationContext& creation_context, const OperationDef& definition, const OperationType& op_type, float scalar_parameter); -struct BroadcastSettings { - bool width; - bool height; - bool channels; -}; - -// Class for simple two input(first input is runtime tensor and second input is -// runtime or constant tensor) operations without any parameters, for example -// sub, div and etc. -class ElementwiseTwoInput : public ElementwiseOperation { - public: - ElementwiseTwoInput() = default; - ElementwiseTwoInput(const OperationDef& definition, - const OperationType& op_type, - const BroadcastSettings& broadcast); - - ElementwiseTwoInput(const OperationDef& definition, - const OperationType& op_type, - const BroadcastSettings& broadcast, - Tensor&& constant_tensor); - - // Move only - ElementwiseTwoInput(ElementwiseTwoInput&& operation); - ElementwiseTwoInput& operator=(ElementwiseTwoInput&& operation); - ElementwiseTwoInput(const ElementwiseTwoInput&) = delete; - ElementwiseTwoInput& operator=(const ElementwiseTwoInput&) = delete; - - private: - BroadcastSettings broadcast_; -}; - +// Creates simple two input(first input is runtime tensor and second input is +// constant linear tensor) operation, for example sub, div and etc. absl::Status CreateElementwiseTwoInput( const CreationContext& creation_context, const OperationDef& definition, const OperationType& op_type, const tflite::gpu::Tensor& constant_tensor, - ElementwiseTwoInput* result); + GPUOperation* result); +// Creates simple two input(first input is runtime tensor and second input is +// constant HWC tensor) operation, for example sub, div and etc. absl::Status CreateElementwiseTwoInput( const CreationContext& creation_context, const OperationDef& definition, const OperationType& op_type, const tflite::gpu::Tensor& constant_tensor, - ElementwiseTwoInput* result); + GPUOperation* result); -ElementwiseTwoInput CreateElementwiseTwoInput(const OperationDef& definition, - const OperationType& op_type, - const BHWC& shape); +// Creates simple two input(2 runtime tensors) operation, for example +// sub, div and etc. +GPUOperation CreateElementwiseTwoInput(const OperationDef& definition, + const OperationType& op_type, + const BHWC& shape); } // namespace cl } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise_test.cc index ac825c0cdfc..11a651df901 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise_test.cc @@ -45,7 +45,7 @@ TEST_F(OpenCLOperationTest, Abs) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::ABS); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -70,7 +70,7 @@ TEST_F(OpenCLOperationTest, Cos) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::COS); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -95,7 +95,7 @@ TEST_F(OpenCLOperationTest, Copy) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::COPY); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -118,7 +118,7 @@ TEST_F(OpenCLOperationTest, Elu) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::ELU); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 1, 1, 7), &dst_tensor)); @@ -144,7 +144,7 @@ TEST_F(OpenCLOperationTest, Exp) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::EXP); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 1, 1, 7), &dst_tensor)); @@ -171,7 +171,7 @@ TEST_F(OpenCLOperationTest, HardSwish) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::HARD_SWISH); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, src_tensor.shape, &dst_tensor)); @@ -197,7 +197,7 @@ TEST_F(OpenCLOperationTest, Log) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::LOG); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -222,7 +222,7 @@ TEST_F(OpenCLOperationTest, Rsqrt) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::RSQRT); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -249,7 +249,7 @@ TEST_F(OpenCLOperationTest, Sigmoid) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::SIGMOID); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -273,7 +273,7 @@ TEST_F(OpenCLOperationTest, Sin) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::SIN); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -299,7 +299,7 @@ TEST_F(OpenCLOperationTest, Sqrt) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::SQRT); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -325,7 +325,7 @@ TEST_F(OpenCLOperationTest, Square) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::SQUARE); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -349,7 +349,7 @@ TEST_F(OpenCLOperationTest, Tanh) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseOneInput operation = + GPUOperation operation = CreateElementwiseOneInput(op_def, OperationType::TANH); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -378,7 +378,7 @@ TEST_F(OpenCLOperationTest, Sub) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::SUB, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, @@ -406,7 +406,7 @@ TEST_F(OpenCLOperationTest, SquaredDiff) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::SQUARED_DIFF, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, @@ -434,7 +434,7 @@ TEST_F(OpenCLOperationTest, Div) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::DIV, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, @@ -462,7 +462,7 @@ TEST_F(OpenCLOperationTest, Pow) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::POW, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, @@ -490,7 +490,7 @@ TEST_F(OpenCLOperationTest, Add) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::ADD, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, @@ -518,7 +518,7 @@ TEST_F(OpenCLOperationTest, Maximum) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::MAXIMUM, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, @@ -547,9 +547,8 @@ TEST_F(OpenCLOperationTest, MaximumWithScalar) { op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; const float* scalar = absl::get_if(&attr.param); - ElementwiseOneRuntimeOneScalar operation = - CreateElementwiseOneRuntimeOneScalar(creation_context_, op_def, - OperationType::MAXIMUM, *scalar); + GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( + creation_context_, op_def, OperationType::MAXIMUM, *scalar); ASSERT_OK(ExecuteGPUOperation(src_tensor_0, creation_context_, &operation, BHWC(1, 4, 1, 1), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -578,7 +577,7 @@ TEST_F(OpenCLOperationTest, MaximumWithConstantLinearTensor) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation; + GPUOperation operation; ASSERT_OK(CreateElementwiseTwoInput(creation_context_, op_def, OperationType::MAXIMUM, linear_tensor, &operation)); @@ -608,7 +607,7 @@ TEST_F(OpenCLOperationTest, MaximumWithConstantHWCTensor) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation; + GPUOperation operation; ASSERT_OK(CreateElementwiseTwoInput(creation_context_, op_def, OperationType::MAXIMUM, hwc_tensor, &operation)); @@ -637,7 +636,7 @@ TEST_F(OpenCLOperationTest, MaximumWithConstantHWCTensorBroadcastChannels) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation; + GPUOperation operation; ASSERT_OK(CreateElementwiseTwoInput(creation_context_, op_def, OperationType::MAXIMUM, hwc_tensor, &operation)); @@ -666,7 +665,7 @@ TEST_F(OpenCLOperationTest, Minimum) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::MINIMUM, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, @@ -695,9 +694,8 @@ TEST_F(OpenCLOperationTest, MinimumWithScalar) { op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; const float* scalar = absl::get_if(&attr.param); - ElementwiseOneRuntimeOneScalar operation = - CreateElementwiseOneRuntimeOneScalar(creation_context_, op_def, - OperationType::MINIMUM, *scalar); + GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( + creation_context_, op_def, OperationType::MINIMUM, *scalar); ASSERT_OK(ExecuteGPUOperation(src_tensor_0, creation_context_, &operation, BHWC(1, 4, 1, 1), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -723,7 +721,7 @@ TEST_F(OpenCLOperationTest, Mul) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::MUL, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, @@ -751,7 +749,7 @@ TEST_F(OpenCLOperationTest, MulBroadcastHW) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::MUL, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, @@ -779,7 +777,7 @@ TEST_F(OpenCLOperationTest, MulBroadcastChannels) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ElementwiseTwoInput operation = CreateElementwiseTwoInput( + GPUOperation operation = CreateElementwiseTwoInput( op_def, OperationType::MUL, src_tensor_1.shape); ASSERT_OK(ExecuteGPUOperation({src_tensor_0, src_tensor_1}, creation_context_, &operation, diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc index beb62632099..7260048c6d3 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc @@ -49,6 +49,20 @@ std::string GetElementWiseCode(const OperationDef& op_def, return c; } +absl::Status MergeOperations(const std::vector& linked_ops, + Arguments* merged_args, std::string* merged_code) { + for (int i = 0; i < linked_ops.size(); ++i) { + std::string code = linked_ops[i]->code_; + std::string unique_postfix = absl::StrCat("_link", i + 1); + linked_ops[i]->args_.RenameArgs(unique_postfix, &code); + *merged_code += "{\n" + code + "\n}\n"; + RETURN_IF_ERROR( + merged_args->Merge(std::move(linked_ops[i]->args_), unique_postfix)); + linked_ops[i]->AddUniquePostfix(unique_postfix); + } + return absl::OkStatus(); +} + } // namespace DataType OperationDef::GetDataType() const { @@ -108,14 +122,17 @@ void GPUOperation::SetDst(Tensor* ptr, int index) { } GPUOperation::GPUOperation(GPUOperation&& operation) - : definition_(std::move(operation.definition_)), + : args_(std::move(operation.args_)), + code_(std::move(operation.code_)), + elementwise_(operation.elementwise_), + linkable_(operation.linkable_), + check_src_channels_size_(operation.check_src_channels_size_), + definition_(std::move(operation.definition_)), src_(std::move(operation.src_)), dst_(std::move(operation.dst_)), - args_(std::move(operation.args_)), kernel_(std::move(operation.kernel_)), work_group_size_(operation.work_group_size_), grid_size_(operation.grid_size_), - code_(std::move(operation.code_)), src_tensors_names_(std::move(operation.src_tensors_names_)), dst_tensors_names_(std::move(operation.dst_tensors_names_)), compiler_options_(std::move(operation.compiler_options_)), @@ -123,14 +140,17 @@ GPUOperation::GPUOperation(GPUOperation&& operation) GPUOperation& GPUOperation::operator=(GPUOperation&& operation) { if (this != &operation) { + args_ = std::move(operation.args_); + code_ = std::move(operation.code_); + elementwise_ = operation.elementwise_; + linkable_ = operation.linkable_; + check_src_channels_size_ = operation.check_src_channels_size_; definition_ = std::move(operation.definition_); src_ = std::move(operation.src_); dst_ = std::move(operation.dst_); - args_ = std::move(operation.args_); kernel_ = std::move(operation.kernel_); std::swap(work_group_size_, operation.work_group_size_); std::swap(grid_size_, operation.grid_size_); - code_ = std::move(operation.code_); src_tensors_names_ = std::move(operation.src_tensors_names_); dst_tensors_names_ = std::move(operation.dst_tensors_names_); compiler_options_ = std::move(operation.compiler_options_); @@ -139,7 +159,7 @@ GPUOperation& GPUOperation::operator=(GPUOperation&& operation) { return *this; } -void GPUOperation::AddOperation(ElementwiseOperation* operation) { +void GPUOperation::AddOperation(GPUOperation* operation) { linked_operations_.push_back(operation); } @@ -183,73 +203,62 @@ absl::Status GPUOperation::UpdateParams() { } absl::Status GPUOperation::Compile(const CreationContext& creation_context) { - std::string element_wise_code; - RETURN_IF_ERROR( - MergeOperations(linked_operations_, &args_, &element_wise_code)); - RETURN_IF_ERROR(args_.TransformToCLCode( - creation_context.device->GetInfo(), - {{dst_tensors_names_[0], element_wise_code}}, &code_)); - RETURN_IF_ERROR(creation_context.cache->GetOrCreateCLKernel( - code_, "main_function", compiler_options_, *creation_context.context, - *creation_context.device, &kernel_)); + if (elementwise_) { + auto src_desc = + absl::make_unique(definition_.src_tensors[0]); + if (definition_.IsBatchSupported()) { + src_desc->SetStateVar("BatchedWidth", "true"); + } + src_tensors_names_.insert(src_tensors_names_.begin(), "src_tensor"); + args_.AddObjectRef("src_tensor", AccessType::READ, std::move(src_desc)); + + auto dst_desc = + absl::make_unique(definition_.dst_tensors[0]); + if (definition_.IsBatchSupported()) { + dst_desc->SetStateVar("BatchedWidth", "true"); + } + dst_tensors_names_.insert(dst_tensors_names_.begin(), "dst_tensor"); + args_.AddObjectRef("dst_tensor", AccessType::WRITE, std::move(dst_desc)); + + std::string code = + GetElementWiseCode(definition_, check_src_channels_size_); + std::string element_wise_code; + element_wise_code += "{\n" + code_ + "\n}\n"; + RETURN_IF_ERROR( + MergeOperations(linked_operations_, &args_, &element_wise_code)); + RETURN_IF_ERROR(args_.TransformToCLCode( + creation_context.device->GetInfo(), + {{dst_tensors_names_[0], element_wise_code}}, &code)); + code = absl::Substitute(code, args_.GetListOfArgs()); + RETURN_IF_ERROR(creation_context.cache->GetOrCreateCLKernel( + code, "main_function", *creation_context.context, + *creation_context.device, &kernel_)); + } else { + std::string element_wise_code; + RETURN_IF_ERROR( + MergeOperations(linked_operations_, &args_, &element_wise_code)); + RETURN_IF_ERROR(args_.TransformToCLCode( + creation_context.device->GetInfo(), + {{dst_tensors_names_[0], element_wise_code}}, &code_)); + RETURN_IF_ERROR(creation_context.cache->GetOrCreateCLKernel( + code_, "main_function", compiler_options_, *creation_context.context, + *creation_context.device, &kernel_)); + } return PostCompileCheck(creation_context.device->GetInfo()); } -ElementwiseOperation::ElementwiseOperation(ElementwiseOperation&& operation) - : GPUOperation(std::move(operation)), - check_src_channels_size_(operation.check_src_channels_size_), - linkable_(operation.linkable_) {} - -ElementwiseOperation& ElementwiseOperation::operator=( - ElementwiseOperation&& operation) { - if (this != &operation) { - check_src_channels_size_ = operation.check_src_channels_size_; - linkable_ = operation.linkable_; - GPUOperation::operator=(std::move(operation)); +int3 GPUOperation::GetGridSize() const { + if (elementwise_) { + const int grid_x = dst_[0]->Width() * dst_[0]->Batch(); + const int grid_y = dst_[0]->Height(); + const int grid_z = dst_[0]->Slices(); + return int3(grid_x, grid_y, grid_z); + } else { + return int3(0, 0, 0); } - return *this; } -int3 ElementwiseOperation::GetGridSize() const { - const int grid_x = dst_[0]->Width() * dst_[0]->Batch(); - const int grid_y = dst_[0]->Height(); - const int grid_z = dst_[0]->Slices(); - return int3(grid_x, grid_y, grid_z); -} - -absl::Status ElementwiseOperation::Compile( - const CreationContext& creation_context) { - auto src_desc = - absl::make_unique(definition_.src_tensors[0]); - if (definition_.IsBatchSupported()) { - src_desc->SetStateVar("BatchedWidth", "true"); - } - src_tensors_names_.insert(src_tensors_names_.begin(), "src_tensor"); - args_.AddObjectRef("src_tensor", AccessType::READ, std::move(src_desc)); - - auto dst_desc = - absl::make_unique(definition_.dst_tensors[0]); - if (definition_.IsBatchSupported()) { - dst_desc->SetStateVar("BatchedWidth", "true"); - } - dst_tensors_names_.insert(dst_tensors_names_.begin(), "dst_tensor"); - args_.AddObjectRef("dst_tensor", AccessType::WRITE, std::move(dst_desc)); - - std::string code = GetElementWiseCode(definition_, check_src_channels_size_); - std::string element_wise_code; - element_wise_code += "{\n" + code_ + "\n}\n"; - RETURN_IF_ERROR( - MergeOperations(linked_operations_, &args_, &element_wise_code)); - RETURN_IF_ERROR(args_.TransformToCLCode( - creation_context.device->GetInfo(), - {{dst_tensors_names_[0], element_wise_code}}, &code)); - code = absl::Substitute(code, args_.GetListOfArgs()); - return creation_context.cache->GetOrCreateCLKernel( - code, "main_function", *creation_context.context, - *creation_context.device, &kernel_); -} - -void ElementwiseOperation::AddUniquePostfix(const std::string& unique_postfix) { +void GPUOperation::AddUniquePostfix(const std::string& unique_postfix) { for (int i = 0; i < src_tensors_names_.size(); ++i) { src_tensors_names_[i] += unique_postfix; } @@ -258,21 +267,6 @@ void ElementwiseOperation::AddUniquePostfix(const std::string& unique_postfix) { } } -absl::Status MergeOperations( - const std::vector& linked_ops, - Arguments* merged_args, std::string* merged_code) { - for (int i = 0; i < linked_ops.size(); ++i) { - std::string code = linked_ops[i]->GetCode(); - std::string unique_postfix = absl::StrCat("_link", i + 1); - auto&& link_args = linked_ops[i]->MoveArgs(); - link_args.RenameArgs(unique_postfix, &code); - *merged_code += "{\n" + code + "\n}\n"; - RETURN_IF_ERROR(merged_args->Merge(std::move(link_args), unique_postfix)); - linked_ops[i]->AddUniquePostfix(unique_postfix); - } - return absl::OkStatus(); -} - } // namespace cl } // namespace gpu } // namespace tflite diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h index 01e11f3ea64..620883f26f4 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h @@ -59,18 +59,15 @@ struct OperationDef { bool IsBatchSupported() const; }; -class ElementwiseOperation; - // GPUOperation represents some implementation of neural network operation on -// GPU. GPUOperation can contain ElementwiseOperation operations, in this case, -// ElementwiseOperation still hold necessary data and should be alive. -// When GPUOperation contains ElementwiseOperations, this GPUoperation replaces -// some sequence of operations Op + el_op0 + el_op1 + ... +// GPU. GPUOperation can contain another GPU operations with flag elementwise_. +// When GPUOperation contains another GPU ops, this GPUoperation replaces +// some sequence of operations Op + op0 + op1 + ... // Because of this abilities of GPUOperation, usage scenario is next: // Create instance of GPUOperation. -// Create all instances of ElementwiseOperations that we will(probably) attach -// to GPUOperation. Attach all ElementwiseOperations to GPUOperation. Call -// GPUOperation.Compile(). Don't call ElementwiseOperation.Compile() if it +// Create all instances of GPUOperations that we will(probably) attach +// to GPUOperation. Attach all GPUOperations to GPUOperation. Call +// GPUOperation.Compile(). Don't call GPUOperations.Compile() if it // attached, it useless(and may be error) class GPUOperation { public: @@ -83,7 +80,7 @@ class GPUOperation { GPUOperation(const GPUOperation&) = delete; GPUOperation& operator=(const GPUOperation&) = delete; - void AddOperation(ElementwiseOperation* operation); + void AddOperation(GPUOperation* operation); void SetSrc(Tensor* ptr, int index = 0); void SetDst(Tensor* ptr, int index = 0); @@ -116,64 +113,37 @@ class GPUOperation { void AddDstTensor(const std::string& tensor_name, const TensorDescriptor& desc); + bool IsLinkable() const { return elementwise_ && linkable_; } + + // for linking + void AddUniquePostfix(const std::string& unique_postfix); + + Arguments args_; + std::string code_; + + bool elementwise_ = false; + // applicable only with elementwise_ = true; + bool linkable_ = true; // by default every elementwise is linkable + // applicable only with elementwise_ = true; + bool check_src_channels_size_ = false; + protected: virtual absl::Status BindArguments() { return absl::OkStatus(); } - virtual int3 GetGridSize() const = 0; + virtual int3 GetGridSize() const; // Defines operation calculation precision and format of src/dst tensors. OperationDef definition_; std::vector src_; std::vector dst_; - Arguments args_; CLKernel kernel_; int3 work_group_size_ = int3(8, 4, 1); int3 grid_size_ = int3(0, 0, 0); - std::string code_; std::vector src_tensors_names_; std::vector dst_tensors_names_; std::vector compiler_options_; - std::vector linked_operations_; + std::vector linked_operations_; }; -// ElementwiseOperation can be fused(linked) to another operation. -// field linked_ indicate about this -// link_index_ used mostly for generating of correct names for -// linked code variables -// link_index_ is number of operation in sequence of linked operations -// and should be unique in this sequence -// link_index_ = 0 is equivalent that operation not linked. -class ElementwiseOperation : public GPUOperation { - public: - ElementwiseOperation() {} - explicit ElementwiseOperation(const OperationDef& definition) - : GPUOperation(definition) {} - - virtual ~ElementwiseOperation() {} - - absl::Status Compile(const CreationContext& creation_context) override; - int3 GetGridSize() const override; - - // Move only - ElementwiseOperation(ElementwiseOperation&& operation); - ElementwiseOperation& operator=(ElementwiseOperation&& operation); - ElementwiseOperation(const ElementwiseOperation&) = delete; - ElementwiseOperation& operator=(const ElementwiseOperation&) = delete; - - Arguments&& MoveArgs() { return std::move(args_); } - std::string GetCode() const { return code_; } - void AddUniquePostfix(const std::string& unique_postfix); - - bool IsLinkable() const { return linkable_; } - - protected: - bool check_src_channels_size_ = false; - bool linkable_ = true; -}; - -absl::Status MergeOperations( - const std::vector& linked_ops, - Arguments* merged_args, std::string* merged_code); - } // namespace cl } // namespace gpu } // namespace tflite diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/prelu.cc b/tensorflow/lite/delegates/gpu/cl/kernels/prelu.cc index 85c88f3b51b..1ca2e096a0e 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/prelu.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/prelu.cc @@ -24,47 +24,43 @@ namespace tflite { namespace gpu { namespace cl { -PReLU::PReLU(const OperationDef& definition, const PReLUAttributes& attr, - CalculationsPrecision scalar_precision) - : ElementwiseOperation(definition) { +absl::Status CreatePReLU(const CreationContext& creation_context, + const OperationDef& definition, + const PReLUAttributes& attr, GPUOperation* result) { + *result = GPUOperation(definition); + result->elementwise_ = true; if (attr.clip != 0) { if (definition.precision == CalculationsPrecision::F32) { - args_.AddFloat("clip", attr.clip); + result->args_.AddFloat("clip", attr.clip); } else { - args_.AddHalf("clip", half(attr.clip)); + result->args_.AddHalf("clip", half(attr.clip)); } - code_ = + result->code_ = "in_out_value = clamp(in_out_value, (FLT4)(0.0f), (FLT4)(args.clip)) + " "min((FLT4)(0.0f), in_out_value) * args.alpha.Read(S_COORD);"; } else { - code_ = + result->code_ = "in_out_value = max((FLT4)(0.0f), in_out_value) + min((FLT4)(0.0f), " "in_out_value) * args.alpha.Read(S_COORD);"; } -} -PReLU::PReLU(PReLU&& operation) : ElementwiseOperation(std::move(operation)) {} - -PReLU& PReLU::operator=(PReLU&& operation) { - if (this != &operation) { - ElementwiseOperation::operator=(std::move(operation)); - } - return *this; -} - -absl::Status CreatePReLU(const CreationContext& creation_context, - const OperationDef& definition, - const PReLUAttributes& attr, PReLU* result) { auto alpha = absl::get_if>(&attr.alpha); if (!alpha) { return absl::InvalidArgumentError("Alpha is missing"); } - const auto scalar_precision = creation_context.device->IsPowerVR() - ? CalculationsPrecision::F32 - : definition.precision; - *result = PReLU(definition, attr, scalar_precision); - RETURN_IF_ERROR(result->UploadParameters(*alpha, creation_context.context)); + TensorLinearDescriptor desc; + desc.storage_type = + DeduceLinearStorageType(definition.GetPrimaryStorageType()); + desc.element_type = definition.GetPrimaryDataType(); + + LinearStorage lt; + RETURN_IF_ERROR( + CreateLinearStorage(desc, *alpha, creation_context.context, <)); + result->args_.AddObject("alpha", AccessType::READ, + absl::make_unique(std::move(lt)), + absl::make_unique(desc)); + return absl::OkStatus(); } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/prelu.h b/tensorflow/lite/delegates/gpu/cl/kernels/prelu.h index e65559cf7c7..b673217c799 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/prelu.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/prelu.h @@ -31,48 +31,9 @@ namespace tflite { namespace gpu { namespace cl { -class PReLU : public ElementwiseOperation { - public: - PReLU() = default; - // Move only - PReLU(PReLU&& operation); - PReLU& operator=(PReLU&& operation); - PReLU(const PReLU&) = delete; - PReLU& operator=(const PReLU&) = delete; - - friend absl::Status CreatePReLU(const CreationContext& creation_context, - const OperationDef& definition, - const PReLUAttributes& attr, PReLU* result); - - private: - PReLU(const OperationDef& definition, const PReLUAttributes& attr, - CalculationsPrecision scalar_precision); - - template - absl::Status UploadParameters( - const tflite::gpu::Tensor& parameters, CLContext* context); -}; - absl::Status CreatePReLU(const CreationContext& creation_context, const OperationDef& definition, - const PReLUAttributes& attr, PReLU* result); - -template -absl::Status PReLU::UploadParameters( - const tflite::gpu::Tensor& parameters, CLContext* context) { - TensorLinearDescriptor desc; - desc.storage_type = - DeduceLinearStorageType(definition_.GetPrimaryStorageType()); - desc.element_type = definition_.GetPrimaryDataType(); - - LinearStorage lt; - RETURN_IF_ERROR(CreateLinearStorage(desc, parameters, context, <)); - args_.AddObject("alpha", AccessType::READ, - absl::make_unique(std::move(lt)), - absl::make_unique(desc)); - - return absl::OkStatus(); -} + const PReLUAttributes& attr, GPUOperation* result); } // namespace cl } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/prelu_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/prelu_test.cc index 4b0006c7f32..06ff09ccca7 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/prelu_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/prelu_test.cc @@ -52,7 +52,7 @@ TEST_F(OpenCLOperationTest, PReLUAlpha) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - PReLU operation; + GPUOperation operation; ASSERT_OK(CreatePReLU(creation_context_, op_def, attr, &operation)); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); @@ -83,7 +83,7 @@ TEST_F(OpenCLOperationTest, PReLUAlphaClip) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - PReLU operation; + GPUOperation operation; ASSERT_OK(CreatePReLU(creation_context_, op_def, attr, &operation)); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize.cc b/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize.cc index 957fc9bbb98..e0c44e1cda7 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize.cc @@ -25,59 +25,37 @@ limitations under the License. namespace tflite { namespace gpu { namespace cl { - -QuantizeAndDequantize::QuantizeAndDequantize( - const OperationDef& definition, const QuantizeAndDequantizeAttributes& attr, - CalculationsPrecision scalar_precision) - : ElementwiseOperation(definition) { - if (definition.precision == CalculationsPrecision::F32) { - args_.AddFloat("min", attr.min); - args_.AddFloat("max", attr.max); - args_.AddFloat("scale", attr.scale); - } else { - args_.AddHalf("min", half(attr.min)); - args_.AddHalf("max", half(attr.max)); - args_.AddHalf("scale", half(attr.scale)); - } - code_ = R"( -FLT4 clamped_value = min((FLT4)(args.max), max((FLT4)(args.min), in_out_value)); -FLT4 quantized_value = round((clamped_value - (FLT4)(args.min)) / (FLT4)(args.scale)); -FLT4 dequantized_value = quantized_value * (FLT4)(args.scale) + (FLT4)(args.min); -in_out_value = dequantized_value;)"; -} - -QuantizeAndDequantize::QuantizeAndDequantize(QuantizeAndDequantize&& operation) - : ElementwiseOperation(std::move(operation)) {} - -QuantizeAndDequantize& QuantizeAndDequantize::operator=( - QuantizeAndDequantize&& operation) { - if (this != &operation) { - ElementwiseOperation::operator=(std::move(operation)); - } - return *this; -} - -absl::Status CreateQuantizeAndDequantize( +GPUOperation CreateQuantizeAndDequantize( const CreationContext& creation_context, const OperationDef& definition, - const QuantizeAndDequantizeAttributes& attr, - QuantizeAndDequantize* result) { - const auto scalar_precision = creation_context.device->IsPowerVR() - ? CalculationsPrecision::F32 - : definition.precision; + const QuantizeAndDequantizeAttributes& attr) { + QuantizeAndDequantizeAttributes adjusted_attr = attr; const bool is_fp16 = definition.precision == CalculationsPrecision::F16 || definition.precision == CalculationsPrecision::F32_F16; if (is_fp16 && attr.scale < 0.000062f) { // The smallest positive normal number for Half-precision floating-point // format is 2^-14 ~ 0.000062f. Therefore, if the scale is lesser than this // number, we just reset it accordingly. - QuantizeAndDequantizeAttributes adjusted_attr = attr; adjusted_attr.scale = 0.000062f; - *result = - QuantizeAndDequantize(definition, adjusted_attr, scalar_precision); - } else { - *result = QuantizeAndDequantize(definition, attr, scalar_precision); } - return absl::OkStatus(); + + GPUOperation op(definition); + op.elementwise_ = true; + if (definition.precision == CalculationsPrecision::F32) { + op.args_.AddFloat("min", adjusted_attr.min); + op.args_.AddFloat("max", adjusted_attr.max); + op.args_.AddFloat("scale", adjusted_attr.scale); + } else { + op.args_.AddHalf("min", half(adjusted_attr.min)); + op.args_.AddHalf("max", half(adjusted_attr.max)); + op.args_.AddHalf("scale", half(adjusted_attr.scale)); + } + op.code_ = R"( +FLT4 clamped_value = min((FLT4)(args.max), max((FLT4)(args.min), in_out_value)); +FLT4 quantized_value = round((clamped_value - (FLT4)(args.min)) / (FLT4)(args.scale)); +FLT4 dequantized_value = quantized_value * (FLT4)(args.scale) + (FLT4)(args.min); +in_out_value = dequantized_value;)"; + + return op; } } // namespace cl diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize.h b/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize.h index a40aa21d23c..6e028625852 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize.h @@ -43,43 +43,9 @@ namespace cl { // // NOTE: We do not need to nudge min/max values in this op, since they would // already be adjusted while generating the quantized model. -class QuantizeAndDequantize : public ElementwiseOperation { - public: - QuantizeAndDequantize() = default; - // Move only - QuantizeAndDequantize(QuantizeAndDequantize&& operation); - QuantizeAndDequantize& operator=(QuantizeAndDequantize&& operation); - QuantizeAndDequantize(const QuantizeAndDequantize&) = delete; - QuantizeAndDequantize& operator=(const QuantizeAndDequantize&) = delete; - - friend absl::Status CreateQuantizeAndDequantize( - const CreationContext& creation_context, const OperationDef& definition, - const QuantizeAndDequantizeAttributes& attr, - QuantizeAndDequantize* result); - - private: - QuantizeAndDequantize(const OperationDef& definition, - const QuantizeAndDequantizeAttributes& attr, - CalculationsPrecision scalar_precision); - - template - absl::Status UploadParameters( - const tflite::gpu::Tensor& parameters, CLContext* context); -}; - -absl::Status CreateQuantizeAndDequantize( +GPUOperation CreateQuantizeAndDequantize( const CreationContext& creation_context, const OperationDef& definition, - const QuantizeAndDequantizeAttributes& attr, QuantizeAndDequantize* result); - -template -absl::Status QuantizeAndDequantize::UploadParameters( - const tflite::gpu::Tensor& parameters, CLContext* context) { - LinearStorageCreateInfo create_info; - create_info.storage_type = - DeduceLinearStorageType(definition_.GetPrimaryStorageType()); - create_info.data_type = definition_.GetPrimaryDataType(); - return absl::OkStatus(); -} + const QuantizeAndDequantizeAttributes& attr); } // namespace cl } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize_test.cc index 71d6d066b9b..43b5d69323d 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/quantize_and_dequantize_test.cc @@ -56,9 +56,8 @@ TEST_F(OpenCLOperationTest, QuantAndDequant_Dim2Bits8) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - QuantizeAndDequantize operation; - ASSERT_OK(CreateQuantizeAndDequantize(creation_context_, op_def, attr, - &operation)); + GPUOperation operation = + CreateQuantizeAndDequantize(creation_context_, op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 3, 2, 1), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -92,9 +91,8 @@ TEST_F(OpenCLOperationTest, QuantAndDequant_Dim3Bits8_NegativeRange) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - QuantizeAndDequantize operation; - ASSERT_OK(CreateQuantizeAndDequantize(creation_context_, op_def, attr, - &operation)); + GPUOperation operation = + CreateQuantizeAndDequantize(creation_context_, op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 3, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -128,9 +126,8 @@ TEST_F(OpenCLOperationTest, QuantAndDequant_Dim3Bits16) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - QuantizeAndDequantize operation; - ASSERT_OK(CreateQuantizeAndDequantize(creation_context_, op_def, attr, - &operation)); + GPUOperation operation = + CreateQuantizeAndDequantize(creation_context_, op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 3, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -164,9 +161,8 @@ TEST_F(OpenCLOperationTest, QuantAndDequant_Dim2Bits16_NegativeRange) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - QuantizeAndDequantize operation; - ASSERT_OK(CreateQuantizeAndDequantize(creation_context_, op_def, attr, - &operation)); + GPUOperation operation = + CreateQuantizeAndDequantize(creation_context_, op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 3, 2, 1), &dst_tensor)); EXPECT_THAT(dst_tensor.data, diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/relu.cc b/tensorflow/lite/delegates/gpu/cl/kernels/relu.cc index 774c030545a..a80dccd6259 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/relu.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/relu.cc @@ -21,50 +21,36 @@ limitations under the License. namespace tflite { namespace gpu { namespace cl { +GPUOperation CreateReLU(const CreationContext& creation_context, + const OperationDef& definition, + const ReLUAttributes& attr) { + GPUOperation op(definition); + op.elementwise_ = true; -ReLU::ReLU(const OperationDef& definition, const ReLUAttributes& attr, - CalculationsPrecision scalar_precision) - : ElementwiseOperation(definition) { std::string min_func; if (attr.alpha != 0.0f) { min_func = "min(in_out_value * args.alpha, (FLT)(0.0f))"; if (definition.precision == CalculationsPrecision::F32) { - args_.AddFloat("alpha", attr.alpha); + op.args_.AddFloat("alpha", attr.alpha); } else { - args_.AddHalf("alpha", half(attr.alpha)); + op.args_.AddHalf("alpha", half(attr.alpha)); } } else { min_func = "(FLT)(0.0f)"; } if (attr.clip != 0.0f) { if (definition.precision == CalculationsPrecision::F32) { - args_.AddFloat("clip", attr.clip); + op.args_.AddFloat("clip", attr.clip); } else { - args_.AddHalf("clip", half(attr.clip)); + op.args_.AddHalf("clip", half(attr.clip)); } - code_ = absl::StrCat("in_out_value = clamp(in_out_value, " + min_func + - ", args.clip);"); + op.code_ = absl::StrCat("in_out_value = clamp(in_out_value, " + min_func + + ", args.clip);"); } else { - code_ = absl::StrCat("in_out_value = max(in_out_value, ", min_func, ");"); + op.code_ = + absl::StrCat("in_out_value = max(in_out_value, ", min_func, ");"); } -} - -ReLU::ReLU(ReLU&& operation) : ElementwiseOperation(std::move(operation)) {} - -ReLU& ReLU::operator=(ReLU&& operation) { - if (this != &operation) { - ElementwiseOperation::operator=(std::move(operation)); - } - return *this; -} - -ReLU CreateReLU(const CreationContext& creation_context, - const OperationDef& definition, const ReLUAttributes& attr) { - const auto scalar_precision = creation_context.device->IsPowerVR() - ? CalculationsPrecision::F32 - : definition.precision; - ReLU operation(definition, attr, scalar_precision); - return operation; + return op; } } // namespace cl diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/relu.h b/tensorflow/lite/delegates/gpu/cl/kernels/relu.h index ccb6f6ca37f..001e23da41c 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/relu.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/relu.h @@ -25,25 +25,9 @@ namespace tflite { namespace gpu { namespace cl { -class ReLU : public ElementwiseOperation { - public: - // Move only - ReLU(ReLU&& operation); - ReLU& operator=(ReLU&& operation); - ReLU(const ReLU&) = delete; - ReLU& operator=(const ReLU&) = delete; - - friend ReLU CreateReLU(const CreationContext& creation_context, - const OperationDef& definition, - const ReLUAttributes& attr); - - private: - ReLU(const OperationDef& definition, const ReLUAttributes& attr, - CalculationsPrecision scalar_precision); -}; - -ReLU CreateReLU(const CreationContext& creation_context, - const OperationDef& definition, const ReLUAttributes& attr); +GPUOperation CreateReLU(const CreationContext& creation_context, + const OperationDef& definition, + const ReLUAttributes& attr); } // namespace cl } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/relu_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/relu_test.cc index cebc9886ba5..f741a408661 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/relu_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/relu_test.cc @@ -49,7 +49,7 @@ TEST_F(OpenCLOperationTest, ReLUNoClipNoAlpha) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ReLU operation = CreateReLU(creation_context_, op_def, attr); + GPUOperation operation = CreateReLU(creation_context_, op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -76,7 +76,7 @@ TEST_F(OpenCLOperationTest, ReLUClip) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ReLU operation = CreateReLU(creation_context_, op_def, attr); + GPUOperation operation = CreateReLU(creation_context_, op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -103,7 +103,7 @@ TEST_F(OpenCLOperationTest, ReLUAlpha) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ReLU operation = CreateReLU(creation_context_, op_def, attr); + GPUOperation operation = CreateReLU(creation_context_, op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -130,7 +130,7 @@ TEST_F(OpenCLOperationTest, ReLUAlphaClip) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - ReLU operation = CreateReLU(creation_context_, op_def, attr); + GPUOperation operation = CreateReLU(creation_context_, op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc index 088677ba7e2..f60af5f730d 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc @@ -144,9 +144,9 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, if (inputs.size() == 2 && (inputs[0]->tensor.shape.c == inputs[1]->tensor.shape.c || inputs[1]->tensor.shape.c == 1)) { - ElementwiseTwoInput operation = + GPUOperation operation = CreateElementwiseTwoInput(op_def, op_type, inputs[1]->tensor.shape); - *gpu_op = absl::make_unique(std::move(operation)); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } else if (inputs.size() >= 2) { auto output = outputs[0]; @@ -167,25 +167,21 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, absl::get_if>( &attr.param); if (scalar) { - ElementwiseOneRuntimeOneScalar operation = - CreateElementwiseOneRuntimeOneScalar(creation_context, op_def, - op_type, *scalar); - *gpu_op = absl::make_unique( - std::move(operation)); + GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( + creation_context, op_def, op_type, *scalar); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } else if (linear_tensor) { - ElementwiseTwoInput operation; + GPUOperation operation; RETURN_IF_ERROR(CreateElementwiseTwoInput( creation_context, op_def, op_type, *linear_tensor, &operation)); - *gpu_op = - absl::make_unique(std::move(operation)); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } else if (hwc_tensor) { - ElementwiseTwoInput operation; + GPUOperation operation; RETURN_IF_ERROR(CreateElementwiseTwoInput( creation_context, op_def, op_type, *hwc_tensor, &operation)); - *gpu_op = - absl::make_unique(std::move(operation)); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } } @@ -295,9 +291,9 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, } case OperationType::MUL: { if (inputs.size() == 2) { - ElementwiseTwoInput operation = + GPUOperation operation = CreateElementwiseTwoInput(op_def, op_type, inputs[1]->tensor.shape); - *gpu_op = absl::make_unique(std::move(operation)); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } else if (inputs.size() == 1 && node.operation.attributes.has_value()) { auto attr = @@ -310,25 +306,21 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, absl::get_if>( &attr.param); if (scalar) { - ElementwiseOneRuntimeOneScalar operation = - CreateElementwiseOneRuntimeOneScalar(creation_context, op_def, - op_type, *scalar); - *gpu_op = absl::make_unique( - std::move(operation)); + GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( + creation_context, op_def, op_type, *scalar); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } else if (linear_tensor) { - ElementwiseTwoInput operation; + GPUOperation operation; RETURN_IF_ERROR(CreateElementwiseTwoInput( creation_context, op_def, op_type, *linear_tensor, &operation)); - *gpu_op = - absl::make_unique(std::move(operation)); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } else if (hwc_tensor) { - ElementwiseTwoInput operation; + GPUOperation operation; RETURN_IF_ERROR(CreateElementwiseTwoInput( creation_context, op_def, op_type, *hwc_tensor, &operation)); - *gpu_op = - absl::make_unique(std::move(operation)); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } } @@ -353,8 +345,8 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, case OperationType::QUANTIZE_AND_DEQUANTIZE: { auto attr = absl::any_cast( node.operation.attributes); - return SelectQuantizeAndDequantize(attr, creation_context, op_def, - gpu_op); + SelectQuantizeAndDequantize(attr, creation_context, op_def, gpu_op); + return absl::OkStatus(); } case OperationType::RELU: { auto attr = absl::any_cast(node.operation.attributes); @@ -405,9 +397,8 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, case OperationType::SQRT: case OperationType::SQUARE: case OperationType::TANH: { - ElementwiseOneInput operation = - CreateElementwiseOneInput(op_def, op_type); - *gpu_op = absl::make_unique(std::move(operation)); + GPUOperation operation = CreateElementwiseOneInput(op_def, op_type); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } case OperationType::DIV: @@ -417,9 +408,9 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, case OperationType::SQUARED_DIFF: case OperationType::SUB: { if (inputs.size() == 2) { - ElementwiseTwoInput operation = + GPUOperation operation = CreateElementwiseTwoInput(op_def, op_type, inputs[1]->tensor.shape); - *gpu_op = absl::make_unique(std::move(operation)); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } else if (inputs.size() == 1 && node.operation.attributes.has_value()) { auto attr = @@ -432,25 +423,21 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, absl::get_if>( &attr.param); if (scalar) { - ElementwiseOneRuntimeOneScalar operation = - CreateElementwiseOneRuntimeOneScalar(creation_context, op_def, - op_type, *scalar); - *gpu_op = absl::make_unique( - std::move(operation)); + GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( + creation_context, op_def, op_type, *scalar); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } else if (linear_tensor) { - ElementwiseTwoInput operation; + GPUOperation operation; RETURN_IF_ERROR(CreateElementwiseTwoInput( creation_context, op_def, op_type, *linear_tensor, &operation)); - *gpu_op = - absl::make_unique(std::move(operation)); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } else if (hwc_tensor) { - ElementwiseTwoInput operation; + GPUOperation operation; RETURN_IF_ERROR(CreateElementwiseTwoInput( creation_context, op_def, op_type, *hwc_tensor, &operation)); - *gpu_op = - absl::make_unique(std::move(operation)); + *gpu_op = absl::make_unique(std::move(operation)); return absl::OkStatus(); } } diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.cc b/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.cc index a32efd5dd2c..1c0bed74422 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.cc @@ -54,17 +54,17 @@ void SelectLSTM(const OperationDef& op_def, const DeviceInfo& device_info, void SelectReLU(const CreationContext& creation_context, const ReLUAttributes& attr, const OperationDef& op_def, std::unique_ptr* ptr) { - ReLU relu = CreateReLU(creation_context, op_def, attr); - *ptr = absl::make_unique(std::move(relu)); + GPUOperation relu = CreateReLU(creation_context, op_def, attr); + *ptr = absl::make_unique(std::move(relu)); } absl::Status SelectPReLU(const PReLUAttributes& attr, const CreationContext& creation_context, const OperationDef& op_def, std::unique_ptr* ptr) { - PReLU operation; + GPUOperation operation; RETURN_IF_ERROR(CreatePReLU(creation_context, op_def, attr, &operation)); - *ptr = absl::make_unique(std::move(operation)); + *ptr = absl::make_unique(std::move(operation)); return absl::OkStatus(); } @@ -85,8 +85,8 @@ void SelectMaxUnpooling(const MaxUnpooling2DAttributes& attr, void SelectAdd(const OperationDef& op_def, const std::vector& channels, int dst_channels, std::unique_ptr* ptr) { - Add operation = CreateAdd(op_def, channels, dst_channels); - *ptr = absl::make_unique(std::move(operation)); + GPUOperation operation = CreateAdd(op_def, channels, dst_channels); + *ptr = absl::make_unique(std::move(operation)); } absl::Status SelectResize(const Resize2DAttributes& attr, @@ -203,15 +203,13 @@ absl::Status SelectWinograd36To4x4( return absl::OkStatus(); } -absl::Status SelectQuantizeAndDequantize( - const QuantizeAndDequantizeAttributes& attr, - const CreationContext& creation_context, const OperationDef& op_def, - std::unique_ptr* ptr) { - QuantizeAndDequantize operation; - RETURN_IF_ERROR( - CreateQuantizeAndDequantize(creation_context, op_def, attr, &operation)); - *ptr = absl::make_unique(std::move(operation)); - return absl::OkStatus(); +void SelectQuantizeAndDequantize(const QuantizeAndDequantizeAttributes& attr, + const CreationContext& creation_context, + const OperationDef& op_def, + std::unique_ptr* ptr) { + GPUOperation operation = + CreateQuantizeAndDequantize(creation_context, op_def, attr); + *ptr = absl::make_unique(std::move(operation)); } } // namespace cl diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.h b/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.h index f266882a458..7133aa94502 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.h +++ b/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.h @@ -97,10 +97,10 @@ absl::Status SelectWinograd36To4x4( const tflite::gpu::Tensor& biases, std::unique_ptr* ptr); -absl::Status SelectQuantizeAndDequantize( - const QuantizeAndDequantizeAttributes& attr, - const CreationContext& creation_context, const OperationDef& op_def, - std::unique_ptr* ptr); +void SelectQuantizeAndDequantize(const QuantizeAndDequantizeAttributes& attr, + const CreationContext& creation_context, + const OperationDef& op_def, + std::unique_ptr* ptr); } // namespace cl } // namespace gpu From 1a8f8965b242e186d1082dca088433823d784a60 Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Mon, 3 Aug 2020 17:38:51 -0700 Subject: [PATCH 0307/1017] Device and DeviceInfo separated. DeviceInfo doesn't has OpenCL API calls/elements. PiperOrigin-RevId: 324719724 Change-Id: I0ffb6eaf6cd7c1edc77e14b28528710aced34519 --- tensorflow/lite/delegates/gpu/cl/BUILD | 10 + tensorflow/lite/delegates/gpu/cl/arguments.cc | 2 +- .../lite/delegates/gpu/cl/cl_command_queue.cc | 8 +- tensorflow/lite/delegates/gpu/cl/cl_device.cc | 337 ++++-------------- tensorflow/lite/delegates/gpu/cl/cl_device.h | 134 +------ .../lite/delegates/gpu/cl/device_info.cc | 268 ++++++++++++++ .../lite/delegates/gpu/cl/device_info.h | 168 +++++++++ .../gpu/cl/kernels/conv_constants.cc | 2 +- .../gpu/cl/kernels/work_group_picking.cc | 11 +- .../gpu/cl/selectors/convolution_selector.cc | 77 ++-- .../convolution_transposed_selector.cc | 27 +- .../cl/selectors/dw_convolution_selector.cc | 18 +- .../cl/selectors/fully_connected_selector.cc | 28 +- 13 files changed, 589 insertions(+), 501 deletions(-) create mode 100644 tensorflow/lite/delegates/gpu/cl/device_info.cc create mode 100644 tensorflow/lite/delegates/gpu/cl/device_info.h diff --git a/tensorflow/lite/delegates/gpu/cl/BUILD b/tensorflow/lite/delegates/gpu/cl/BUILD index 2344a7c6c40..ebfb2cff41b 100644 --- a/tensorflow/lite/delegates/gpu/cl/BUILD +++ b/tensorflow/lite/delegates/gpu/cl/BUILD @@ -166,6 +166,7 @@ cc_library( srcs = ["cl_device.cc"], hdrs = ["cl_device.h"], deps = [ + ":device_info", ":opencl_wrapper", ":util", "//tensorflow/lite/delegates/gpu/common:status", @@ -251,6 +252,15 @@ flatbuffer_cc_library( ], ) +cc_library( + name = "device_info", + srcs = ["device_info.cc"], + hdrs = ["device_info.h"], + deps = [ + "@com_google_absl//absl/strings", + ], +) + cc_library( name = "egl_sync", srcs = ["egl_sync.cc"], diff --git a/tensorflow/lite/delegates/gpu/cl/arguments.cc b/tensorflow/lite/delegates/gpu/cl/arguments.cc index 79241091b14..ed72bcc7c97 100644 --- a/tensorflow/lite/delegates/gpu/cl/arguments.cc +++ b/tensorflow/lite/delegates/gpu/cl/arguments.cc @@ -690,7 +690,7 @@ std::string Arguments::AddActiveArgument(const std::string& arg_name, void Arguments::ResolveArgsPass(const DeviceInfo& device_info, std::string* code) { - bool use_f32_for_half_arguments = device_info.vendor == Vendor::POWERVR; + bool use_f32_for_half_arguments = device_info.IsPowerVR(); size_t position = 0; size_t next_position = code->find(kArgsPrefix); while (next_position != std::string::npos) { diff --git a/tensorflow/lite/delegates/gpu/cl/cl_command_queue.cc b/tensorflow/lite/delegates/gpu/cl/cl_command_queue.cc index f7501dab5af..a1795b18b27 100644 --- a/tensorflow/lite/delegates/gpu/cl/cl_command_queue.cc +++ b/tensorflow/lite/delegates/gpu/cl/cl_command_queue.cc @@ -216,16 +216,14 @@ absl::Status ProfilingCommandQueue::GetBestWorkGroupIndex( const CLKernel& kernel, const DeviceInfo& device_info, const int3& grid, const std::vector& work_group_sizes, int* index) { // Some Adreno 3xx can have wrong numbers for some events - const bool possible_bug_with_events = - device_info.vendor == Vendor::QUALCOMM && - device_info.adreno_info.gpu_version < 400; + const bool possible_bug_with_events = device_info.IsAdreno3xx(); events_.resize(work_group_sizes.size()); for (int i = 0; i < work_group_sizes.size(); ++i) { RETURN_IF_ERROR(CLCommandQueue::DispatchImplicit( kernel, grid, work_group_sizes[i], &events_[i])); // reducing the speed of memory leak on Mali for some kernels - if (device_info.vendor == Vendor::MALI && i % 8 == 7) { + if (device_info.IsMali() && i % 8 == 7) { events_[i - 7].Wait(); } if (possible_bug_with_events) { @@ -237,7 +235,7 @@ absl::Status ProfilingCommandQueue::GetBestWorkGroupIndex( RETURN_IF_ERROR(WaitForCompletion()); // To release memory of some kernel pool on Mali. - if (device_info.vendor == Vendor::MALI) { + if (device_info.IsMali()) { RETURN_IF_ERROR(kernel.ReInit()); } diff --git a/tensorflow/lite/delegates/gpu/cl/cl_device.cc b/tensorflow/lite/delegates/gpu/cl/cl_device.cc index f4f1f1c923f..b93bfb25ad1 100644 --- a/tensorflow/lite/delegates/gpu/cl/cl_device.cc +++ b/tensorflow/lite/delegates/gpu/cl/cl_device.cc @@ -128,24 +128,24 @@ Vendor ParseVendor(const std::string& device_name, std::transform(v_name.begin(), v_name.end(), v_name.begin(), ::tolower); if (d_name.find("qualcomm") != std::string::npos || v_name.find("qualcomm") != std::string::npos) { - return Vendor::QUALCOMM; + return Vendor::kQualcomm; } else if (d_name.find("mali") != std::string::npos || v_name.find("mali") != std::string::npos) { - return Vendor::MALI; + return Vendor::kMali; } else if (d_name.find("power") != std::string::npos || v_name.find("power") != std::string::npos) { - return Vendor::POWERVR; + return Vendor::kPowerVR; } else if (d_name.find("nvidia") != std::string::npos || v_name.find("nvidia") != std::string::npos) { - return Vendor::NVIDIA; + return Vendor::kNvidia; } else if (d_name.find("advanced micro devices") != std::string::npos || v_name.find("advanced micro devices") != std::string::npos) { - return Vendor::AMD; + return Vendor::kAMD; } else if (d_name.find("intel") != std::string::npos || v_name.find("intel") != std::string::npos) { - return Vendor::INTEL; + return Vendor::kIntel; } else { - return Vendor::UNKNOWN; + return Vendor::kUnknown; } } @@ -156,316 +156,99 @@ bool IsGPUVersionInRange(int gpu_version, int min_version, int max_version) { } } // namespace -// There is no rule for gpu version encoding, but we found these samples: -// Version: OpenCL C 2.0 Adreno(TM) 540 // Pixel 2 -// Version: OpenCL C 2.0 Adreno(TM) 630 // Sony Compact XZ2 -// Version: OpenCL C 2.0 Adreno(TM) 630 // Pixel 3 -// Version: OpenCL C 2.0 Adreno(TM) 540 // Samsung S8 -// Version: OpenCL C 1.2 Adreno(TM) 430 // HTC One M9 -// Version: OpenCL C 2.0 Adreno(TM) 530 // Samsung S7 Edge -// Version: OpenCL C 1.2 Adreno(TM) 405 // Motorola Moto G(4) -// After the number string ends. -// It is assumed that the for Adreno GPUs has -// the following format: -// Adreno(TM) -// Returns -1 if vendor-specific information cannot be parsed -int GetAdrenoGPUVersion(const std::string& gpu_version) { - const std::string gpu = absl::AsciiStrToLower(gpu_version); - const std::vector words = absl::StrSplit(gpu, ' '); - int i = 0; - for (; i < words.size(); ++i) { - if (words[i].find("adreno") != words[i].npos) { - break; - } - } - i += 1; - for (; i < words.size(); ++i) { - int number; - bool is_number = absl::SimpleAtoi(words[i], &number); - // Adreno GPUs starts from 2xx, but opencl support should be only from 3xx - if (is_number && number >= 300) { - return number; - } - } - return -1; -} - -MaliGPU GetMaliGPUVersion(const std::string& device_name) { - const std::map kMapping = { - {"T604", MaliGPU::T604}, {"T622", MaliGPU::T622}, {"T624", MaliGPU::T624}, - {"T628", MaliGPU::T628}, {"T658", MaliGPU::T658}, {"T678", MaliGPU::T678}, - {"T720", MaliGPU::T720}, {"T760", MaliGPU::T760}, {"T820", MaliGPU::T820}, - {"T830", MaliGPU::T830}, {"T860", MaliGPU::T860}, {"T880", MaliGPU::T880}, - {"G31", MaliGPU::G31}, {"G51", MaliGPU::G51}, {"G71", MaliGPU::G71}, - {"G52", MaliGPU::G52}, {"G72", MaliGPU::G72}, {"G76", MaliGPU::G76}, - {"G57", MaliGPU::G57}, {"G77", MaliGPU::G77}, - }; - for (const auto& v : kMapping) { - if (device_name.find(v.first) != std::string::npos) { - return v.second; - } - } - return MaliGPU::UNKNOWN; -} - -std::string VendorToString(Vendor v) { - switch (v) { - case Vendor::QUALCOMM: - return "Qualcomm"; - case Vendor::MALI: - return "Mali"; - case Vendor::POWERVR: - return "PowerVR"; - case Vendor::NVIDIA: - return "NVIDIA"; - case Vendor::AMD: - return "AMD"; - case Vendor::INTEL: - return "Intel"; - case Vendor::UNKNOWN: - return "unknown vendor"; - } -} - -std::string OpenCLVersionToString(OpenCLVersion version) { - switch (version) { - case OpenCLVersion::CL_1_0: - return "1.0"; - case OpenCLVersion::CL_1_1: - return "1.1"; - case OpenCLVersion::CL_1_2: - return "1.2"; - case OpenCLVersion::CL_2_0: - return "2.0"; - case OpenCLVersion::CL_2_1: - return "2.1"; - case OpenCLVersion::CL_2_2: - return "2.2"; - case OpenCLVersion::CL_3_0: - return "3.0"; - } -} - -AdrenoInfo::AdrenoInfo(const std::string& device_version) - : gpu_version(GetAdrenoGPUVersion(device_version)) {} - -int AdrenoInfo::GetMaximumWavesCount() const { - if (gpu_version < 400) { - return -1; // Adreno 3xx does not support it currently - } else if (gpu_version >= 400 && gpu_version < 500) { - return -1; // Adreno 4xx does not support it currently - } else if (gpu_version >= 500 && gpu_version < 600) { - return -1; // Adreno 5xx does not support it currently - } else if (gpu_version >= 600 && gpu_version < 700) { - return gpu_version == 640 ? 30 : 16; - } else { - return -1; // Adreno 7xx and higher does not exist yet - } -} - -int AdrenoInfo::GetRegisterMemorySizePerComputeUnit() const { - if (gpu_version < 400) { - return -1; // Adreno 3xx does not support it currently - } else if (gpu_version >= 400 && gpu_version < 500) { - return -1; // Adreno 4xx does not support it currently - } else if (gpu_version >= 500 && gpu_version < 600) { - return -1; // Adreno 5xx does not support it currently - } else if (gpu_version >= 600 && gpu_version < 700) { - return gpu_version == 640 ? 128 * 144 * 16 : 128 * 96 * 16; - } else { - return -1; // Adreno 7xx and higher does not exist yet - } -} - -int AdrenoInfo::GetMaximumWavesCount(int register_footprint_per_tread, - bool full_wave) const { - const int register_usage_per_wave = - GetWaveSize(full_wave) * register_footprint_per_tread; - const int possible_waves_count = - GetRegisterMemorySizePerComputeUnit() / register_usage_per_wave; - return std::min(possible_waves_count, GetMaximumWavesCount()); -} - -int AdrenoInfo::GetWaveSize(bool full_wave) const { - if (gpu_version < 400) { - return -1; // Adreno 3xx does not support it currently - } else if (gpu_version < 600) { - return full_wave ? 64 : 32; - } else { - return full_wave ? 128 : 64; - } -} - -MaliInfo::MaliInfo(const std::string& device_name) - : gpu_version(GetMaliGPUVersion(device_name)) {} - -bool MaliInfo::IsMaliT6xx() const { - return gpu_version == MaliGPU::T604 || gpu_version == MaliGPU::T622 || - gpu_version == MaliGPU::T624 || gpu_version == MaliGPU::T628 || - gpu_version == MaliGPU::T658 || gpu_version == MaliGPU::T678; -} - -bool MaliInfo::IsMaliT7xx() const { - return gpu_version == MaliGPU::T720 || gpu_version == MaliGPU::T760; -} - -bool MaliInfo::IsMaliT8xx() const { - return gpu_version == MaliGPU::T820 || gpu_version == MaliGPU::T830 || - gpu_version == MaliGPU::T860 || gpu_version == MaliGPU::T880; -} - -bool MaliInfo::IsMidgard() const { - return IsMaliT6xx() || IsMaliT7xx() || IsMaliT8xx(); -} - -bool MaliInfo::IsBifrostGen1() const { - return gpu_version == MaliGPU::G31 || gpu_version == MaliGPU::G51 || - gpu_version == MaliGPU::G71; -} - -bool MaliInfo::IsBifrostGen2() const { - return gpu_version == MaliGPU::G52 || gpu_version == MaliGPU::G72; -} - -bool MaliInfo::IsBifrostGen3() const { return gpu_version == MaliGPU::G76; } - -bool MaliInfo::IsBifrost() const { - return IsBifrostGen1() || IsBifrostGen2() || IsBifrostGen3(); -} - -bool MaliInfo::IsValhall() const { - return gpu_version == MaliGPU::G57 || gpu_version == MaliGPU::G77; -} - -DeviceInfo::DeviceInfo(cl_device_id id) { +DeviceInfo DeviceInfoFromDeviceID(cl_device_id id) { + DeviceInfo info; const auto device_name = GetDeviceInfo(id, CL_DEVICE_NAME); const auto vendor_name = GetDeviceInfo(id, CL_DEVICE_VENDOR); const auto opencl_c_version = GetDeviceInfo(id, CL_DEVICE_OPENCL_C_VERSION); - vendor = ParseVendor(device_name, vendor_name); - if (vendor == Vendor::QUALCOMM) { - adreno_info = AdrenoInfo(opencl_c_version); - } else if (vendor == Vendor::MALI) { - mali_info = MaliInfo(device_name); + info.vendor = ParseVendor(device_name, vendor_name); + if (info.vendor == Vendor::kQualcomm) { + info.adreno_info = AdrenoInfo(opencl_c_version); + } else if (info.vendor == Vendor::kMali) { + info.mali_info = MaliInfo(device_name); } - cl_version = ParseCLVersion(opencl_c_version); - extensions = + info.cl_version = ParseCLVersion(opencl_c_version); + info.extensions = absl::StrSplit(GetDeviceInfo(id, CL_DEVICE_EXTENSIONS), ' '); - supports_fp16 = false; - supports_image3d_writes = false; - for (const auto& ext : extensions) { + info.supports_fp16 = false; + info.supports_image3d_writes = false; + for (const auto& ext : info.extensions) { if (ext == "cl_khr_fp16") { - supports_fp16 = true; + info.supports_fp16 = true; } if (ext == "cl_khr_3d_image_writes") { - supports_image3d_writes = true; + info.supports_image3d_writes = true; } } - f32_config = + cl_device_fp_config f32_config = GetDeviceInfo(id, CL_DEVICE_SINGLE_FP_CONFIG); - supports_fp32_rtn = f32_config & CL_FP_ROUND_TO_NEAREST; + info.supports_fp32_rtn = f32_config & CL_FP_ROUND_TO_NEAREST; - if (supports_fp16) { + if (info.supports_fp16) { + cl_device_fp_config f16_config; auto status = GetDeviceInfo( id, CL_DEVICE_HALF_FP_CONFIG, &f16_config); // AMD supports cl_khr_fp16 but CL_DEVICE_HALF_FP_CONFIG is empty. - if (status.ok() && vendor != Vendor::AMD) { - supports_fp16_rtn = f16_config & CL_FP_ROUND_TO_NEAREST; + if (status.ok() && info.vendor != Vendor::kAMD) { + info.supports_fp16_rtn = f16_config & CL_FP_ROUND_TO_NEAREST; } else { // happens on PowerVR f16_config = f32_config; - supports_fp16_rtn = supports_fp32_rtn; + info.supports_fp16_rtn = info.supports_fp32_rtn; } } else { - f16_config = 0; - supports_fp16_rtn = false; + info.supports_fp16_rtn = false; } - if (vendor == Vendor::POWERVR && !supports_fp16) { + if (info.vendor == Vendor::kPowerVR && !info.supports_fp16) { // PowerVR doesn't have full support of fp16 and so doesn't list this // extension. But it can support fp16 in MADs and as buffers/textures types, // so we will use it. - supports_fp16 = true; - f16_config = f32_config; - supports_fp16_rtn = supports_fp32_rtn; + info.supports_fp16 = true; + info.supports_fp16_rtn = info.supports_fp32_rtn; } - if (!supports_image3d_writes && - ((vendor == Vendor::QUALCOMM && - IsGPUVersionInRange(adreno_info.gpu_version, 400, 500)) || - vendor == Vendor::NVIDIA)) { + if (!info.supports_image3d_writes && + ((info.vendor == Vendor::kQualcomm && + IsGPUVersionInRange(info.adreno_info.gpu_version, 400, 500)) || + info.vendor == Vendor::kNvidia)) { // in local tests Adreno 430 can write in image 3d, at least on small sizes, // but it doesn't have cl_khr_3d_image_writes in list of available // extensions // The same for NVidia - supports_image3d_writes = true; + info.supports_image3d_writes = true; } - compute_units_count = GetDeviceInfo(id, CL_DEVICE_MAX_COMPUTE_UNITS); - image2d_max_width = GetDeviceInfo(id, CL_DEVICE_IMAGE2D_MAX_WIDTH); - image2d_max_height = GetDeviceInfo(id, CL_DEVICE_IMAGE2D_MAX_HEIGHT); - buffer_max_size = GetDeviceInfo(id, CL_DEVICE_MAX_MEM_ALLOC_SIZE); - if (cl_version >= OpenCLVersion::CL_1_2) { - image_buffer_max_size = + info.compute_units_count = + GetDeviceInfo(id, CL_DEVICE_MAX_COMPUTE_UNITS); + info.image2d_max_width = + GetDeviceInfo(id, CL_DEVICE_IMAGE2D_MAX_WIDTH); + info.image2d_max_height = + GetDeviceInfo(id, CL_DEVICE_IMAGE2D_MAX_HEIGHT); + info.buffer_max_size = + GetDeviceInfo(id, CL_DEVICE_MAX_MEM_ALLOC_SIZE); + if (info.cl_version >= OpenCLVersion::CL_1_2) { + info.image_buffer_max_size = GetDeviceInfo(id, CL_DEVICE_IMAGE_MAX_BUFFER_SIZE); - image_array_max_layers = + info.image_array_max_layers = GetDeviceInfo(id, CL_DEVICE_IMAGE_MAX_ARRAY_SIZE); } - image3d_max_width = GetDeviceInfo(id, CL_DEVICE_IMAGE3D_MAX_WIDTH); - image3d_max_height = GetDeviceInfo(id, CL_DEVICE_IMAGE2D_MAX_HEIGHT); - image3d_max_depth = GetDeviceInfo(id, CL_DEVICE_IMAGE3D_MAX_DEPTH); + info.image3d_max_width = + GetDeviceInfo(id, CL_DEVICE_IMAGE3D_MAX_WIDTH); + info.image3d_max_height = + GetDeviceInfo(id, CL_DEVICE_IMAGE2D_MAX_HEIGHT); + info.image3d_max_depth = + GetDeviceInfo(id, CL_DEVICE_IMAGE3D_MAX_DEPTH); + int3 max_work_group_sizes; GetDeviceWorkDimsSizes(id, &max_work_group_sizes); + info.max_work_group_size_x = max_work_group_sizes.x; + info.max_work_group_size_y = max_work_group_sizes.y; + info.max_work_group_size_z = max_work_group_sizes.z; + return info; } -bool DeviceInfo::SupportsTextureArray() const { - return cl_version >= OpenCLVersion::CL_1_2; -} - -bool DeviceInfo::SupportsImageBuffer() const { - return cl_version >= OpenCLVersion::CL_1_2; -} - -bool DeviceInfo::SupportsImage3D() const { - if (vendor == Vendor::MALI) { - // On Mali T880 read_imageh doesn't compile with image3d_t - return false; - } - return supports_image3d_writes; -} - -bool DeviceInfo::IsAdreno() const { return vendor == Vendor::QUALCOMM; } - -bool DeviceInfo::IsAdreno3xx() const { - return IsAdreno() && IsGPUVersionInRange(adreno_info.gpu_version, 300, 400); -} - -bool DeviceInfo::IsAdreno4xx() const { - return IsAdreno() && IsGPUVersionInRange(adreno_info.gpu_version, 400, 500); -} - -bool DeviceInfo::IsAdreno5xx() const { - return IsAdreno() && IsGPUVersionInRange(adreno_info.gpu_version, 500, 600); -} - -bool DeviceInfo::IsAdreno6xx() const { - return IsAdreno() && IsGPUVersionInRange(adreno_info.gpu_version, 600, 700); -} - -bool DeviceInfo::IsAdreno6xxOrHigher() const { - return IsAdreno() && adreno_info.gpu_version >= 600; -} - -bool DeviceInfo::IsPowerVR() const { return vendor == Vendor::POWERVR; } - -bool DeviceInfo::IsNvidia() const { return vendor == Vendor::NVIDIA; } - -bool DeviceInfo::IsMali() const { return vendor == Vendor::MALI; } - -bool DeviceInfo::IsAMD() const { return vendor == Vendor::AMD; } - -bool DeviceInfo::IsIntel() const { return vendor == Vendor::INTEL; } - CLDevice::CLDevice(cl_device_id id, cl_platform_id platform_id) - : id_(id), platform_id_(platform_id), info_(id) {} + : id_(id), platform_id_(platform_id), info_(DeviceInfoFromDeviceID(id)) {} CLDevice::CLDevice(const CLDevice& device) : id_(device.id_), platform_id_(device.platform_id_), info_(device.info_) {} diff --git a/tensorflow/lite/delegates/gpu/cl/cl_device.h b/tensorflow/lite/delegates/gpu/cl/cl_device.h index 217111c475d..7e4792b0a53 100644 --- a/tensorflow/lite/delegates/gpu/cl/cl_device.h +++ b/tensorflow/lite/delegates/gpu/cl/cl_device.h @@ -19,6 +19,7 @@ limitations under the License. #include #include +#include "tensorflow/lite/delegates/gpu/cl/device_info.h" #include "tensorflow/lite/delegates/gpu/cl/opencl_wrapper.h" #include "tensorflow/lite/delegates/gpu/cl/util.h" #include "tensorflow/lite/delegates/gpu/common/status.h" @@ -28,139 +29,6 @@ namespace tflite { namespace gpu { namespace cl { -enum class Vendor { QUALCOMM, MALI, POWERVR, NVIDIA, AMD, INTEL, UNKNOWN }; -std::string VendorToString(Vendor v); - -enum class OpenCLVersion { - CL_1_0, - CL_1_1, - CL_1_2, - CL_2_0, - CL_2_1, - CL_2_2, - CL_3_0 -}; -std::string OpenCLVersionToString(OpenCLVersion version); - -// for use only in cl_device.cc, but putted here to make tests -int GetAdrenoGPUVersion(const std::string& gpu_version); - -struct AdrenoInfo { - AdrenoInfo() = default; - explicit AdrenoInfo(const std::string& device_version); - int gpu_version = -1; // can be, for example, 405/430/540/530/630 etc. - - // This function returns some not very documented physical parameter of - // Adreno6xx GPU. - // We obtained it using Snapdragon Profiler. - int GetMaximumWavesCount() const; - - // returns amount of register memory per CU(Compute Unit) in bytes. - int GetRegisterMemorySizePerComputeUnit() const; - - // returns maximum possible amount of waves based on register usage. - int GetMaximumWavesCount(int register_footprint_per_tread, - bool full_wave = true) const; - - int GetWaveSize(bool full_wave) const; - - // Not supported on some Adreno devices with specific driver version. - // b/131099086 - bool support_one_layer_texture_array = true; -}; - -enum class MaliGPU { - T604, - T622, - T624, - T628, - T658, - T678, - T720, - T760, - T820, - T830, - T860, - T880, - G31, - G51, - G71, - G52, - G72, - G76, - G57, - G77, - UNKNOWN -}; - -struct MaliInfo { - MaliInfo() = default; - explicit MaliInfo(const std::string& device_name); - MaliGPU gpu_version; - - bool IsMaliT6xx() const; - bool IsMaliT7xx() const; - bool IsMaliT8xx() const; - bool IsMidgard() const; - bool IsBifrostGen1() const; - bool IsBifrostGen2() const; - bool IsBifrostGen3() const; - bool IsBifrost() const; - bool IsValhall() const; -}; - -struct DeviceInfo { - DeviceInfo() = default; - explicit DeviceInfo(cl_device_id id); - - bool IsAdreno() const; - bool IsAdreno3xx() const; - bool IsAdreno4xx() const; - bool IsAdreno5xx() const; - bool IsAdreno6xx() const; - bool IsAdreno6xxOrHigher() const; - bool IsPowerVR() const; - bool IsNvidia() const; - bool IsMali() const; - bool IsAMD() const; - bool IsIntel() const; - - bool SupportsTextureArray() const; - bool SupportsImageBuffer() const; - bool SupportsImage3D() const; - - std::vector extensions; - bool supports_fp16; - bool supports_image3d_writes; - Vendor vendor; - OpenCLVersion cl_version; - int compute_units_count; - uint64_t buffer_max_size; - uint64_t image2d_max_width; - uint64_t image2d_max_height; - uint64_t image_buffer_max_size; - uint64_t image_array_max_layers; - uint64_t image3d_max_width; - uint64_t image3d_max_height; - uint64_t image3d_max_depth; - int3 max_work_group_sizes; - - cl_device_fp_config f32_config; - // valid only with cl_khr_fp16 - cl_device_fp_config f16_config; - - // rtn is ROUND_TO_NEAREST - // with rtn precision is much better then with rtz (ROUND_TO_ZERO) - // Adreno 3xx supports only rtz, Adreno 4xx and more support rtn - // Mali from T6xx supports rtn - // PowerVR supports only rtz - bool supports_fp32_rtn; - bool supports_fp16_rtn; - - AdrenoInfo adreno_info; - MaliInfo mali_info; -}; - // A wrapper around opencl device id class CLDevice { public: diff --git a/tensorflow/lite/delegates/gpu/cl/device_info.cc b/tensorflow/lite/delegates/gpu/cl/device_info.cc new file mode 100644 index 00000000000..7e0acb87ab7 --- /dev/null +++ b/tensorflow/lite/delegates/gpu/cl/device_info.cc @@ -0,0 +1,268 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/lite/delegates/gpu/cl/device_info.h" + +#include +#include +#include + +#include "absl/strings/numbers.h" +#include "absl/strings/str_split.h" + +namespace tflite { +namespace gpu { +namespace cl { +namespace { +// check that gpu_version belong to range min_version-max_version +// min_version is included and max_version is excluded. +bool IsGPUVersionInRange(int gpu_version, int min_version, int max_version) { + return gpu_version >= min_version && gpu_version < max_version; +} + +MaliGPU GetMaliGPUVersion(const std::string& device_name) { + const std::map kMapping = { + {"T604", MaliGPU::T604}, {"T622", MaliGPU::T622}, {"T624", MaliGPU::T624}, + {"T628", MaliGPU::T628}, {"T658", MaliGPU::T658}, {"T678", MaliGPU::T678}, + {"T720", MaliGPU::T720}, {"T760", MaliGPU::T760}, {"T820", MaliGPU::T820}, + {"T830", MaliGPU::T830}, {"T860", MaliGPU::T860}, {"T880", MaliGPU::T880}, + {"G31", MaliGPU::G31}, {"G51", MaliGPU::G51}, {"G71", MaliGPU::G71}, + {"G52", MaliGPU::G52}, {"G72", MaliGPU::G72}, {"G76", MaliGPU::G76}, + {"G57", MaliGPU::G57}, {"G77", MaliGPU::G77}, + }; + for (const auto& v : kMapping) { + if (device_name.find(v.first) != std::string::npos) { + return v.second; + } + } + return MaliGPU::UNKNOWN; +} + +} // namespace + +// There is no rule for gpu version encoding, but we found these samples: +// Version: OpenCL C 2.0 Adreno(TM) 540 // Pixel 2 +// Version: OpenCL C 2.0 Adreno(TM) 630 // Sony Compact XZ2 +// Version: OpenCL C 2.0 Adreno(TM) 630 // Pixel 3 +// Version: OpenCL C 2.0 Adreno(TM) 540 // Samsung S8 +// Version: OpenCL C 1.2 Adreno(TM) 430 // HTC One M9 +// Version: OpenCL C 2.0 Adreno(TM) 530 // Samsung S7 Edge +// Version: OpenCL C 1.2 Adreno(TM) 405 // Motorola Moto G(4) +// After the number string ends. +// It is assumed that the for Adreno GPUs has +// the following format: +// Adreno(TM) +// Returns -1 if vendor-specific information cannot be parsed +int GetAdrenoGPUVersion(const std::string& gpu_version) { + const std::string gpu = absl::AsciiStrToLower(gpu_version); + const std::vector words = absl::StrSplit(gpu, ' '); + int i = 0; + for (; i < words.size(); ++i) { + if (words[i].find("adreno") != words[i].npos) { + break; + } + } + i += 1; + for (; i < words.size(); ++i) { + int number; + bool is_number = absl::SimpleAtoi(words[i], &number); + // Adreno GPUs starts from 2xx, but opencl support should be only from 3xx + if (is_number && number >= 300) { + return number; + } + } + return -1; +} + +std::string VendorToString(Vendor v) { + switch (v) { + case Vendor::kQualcomm: + return "Qualcomm"; + case Vendor::kMali: + return "Mali"; + case Vendor::kPowerVR: + return "PowerVR"; + case Vendor::kNvidia: + return "NVIDIA"; + case Vendor::kAMD: + return "AMD"; + case Vendor::kIntel: + return "Intel"; + case Vendor::kUnknown: + return "unknown vendor"; + } +} + +std::string OpenCLVersionToString(OpenCLVersion version) { + switch (version) { + case OpenCLVersion::CL_1_0: + return "1.0"; + case OpenCLVersion::CL_1_1: + return "1.1"; + case OpenCLVersion::CL_1_2: + return "1.2"; + case OpenCLVersion::CL_2_0: + return "2.0"; + case OpenCLVersion::CL_2_1: + return "2.1"; + case OpenCLVersion::CL_2_2: + return "2.2"; + case OpenCLVersion::CL_3_0: + return "3.0"; + } +} + +AdrenoInfo::AdrenoInfo(const std::string& device_version) + : gpu_version(GetAdrenoGPUVersion(device_version)) {} + +int AdrenoInfo::GetMaximumWavesCount() const { + if (gpu_version < 400) { + return -1; // Adreno 3xx does not support it currently + } else if (gpu_version >= 400 && gpu_version < 500) { + return -1; // Adreno 4xx does not support it currently + } else if (gpu_version >= 500 && gpu_version < 600) { + return -1; // Adreno 5xx does not support it currently + } else if (gpu_version >= 600 && gpu_version < 700) { + return gpu_version == 640 ? 30 : 16; + } else { + return -1; // Adreno 7xx and higher does not exist yet + } +} + +int AdrenoInfo::GetRegisterMemorySizePerComputeUnit() const { + if (gpu_version < 400) { + return -1; // Adreno 3xx does not support it currently + } else if (gpu_version >= 400 && gpu_version < 500) { + return -1; // Adreno 4xx does not support it currently + } else if (gpu_version >= 500 && gpu_version < 600) { + return -1; // Adreno 5xx does not support it currently + } else if (gpu_version >= 600 && gpu_version < 700) { + return gpu_version == 640 ? 128 * 144 * 16 : 128 * 96 * 16; + } else { + return -1; // Adreno 7xx and higher does not exist yet + } +} + +int AdrenoInfo::GetMaximumWavesCount(int register_footprint_per_tread, + bool full_wave) const { + const int register_usage_per_wave = + GetWaveSize(full_wave) * register_footprint_per_tread; + const int possible_waves_count = + GetRegisterMemorySizePerComputeUnit() / register_usage_per_wave; + return std::min(possible_waves_count, GetMaximumWavesCount()); +} + +int AdrenoInfo::GetWaveSize(bool full_wave) const { + if (gpu_version < 400) { + return -1; // Adreno 3xx does not support it currently + } else if (gpu_version < 600) { + return full_wave ? 64 : 32; + } else { + return full_wave ? 128 : 64; + } +} + +MaliInfo::MaliInfo(const std::string& device_name) + : gpu_version(GetMaliGPUVersion(device_name)) {} + +bool MaliInfo::IsMaliT6xx() const { + return gpu_version == MaliGPU::T604 || gpu_version == MaliGPU::T622 || + gpu_version == MaliGPU::T624 || gpu_version == MaliGPU::T628 || + gpu_version == MaliGPU::T658 || gpu_version == MaliGPU::T678; +} + +bool MaliInfo::IsMaliT7xx() const { + return gpu_version == MaliGPU::T720 || gpu_version == MaliGPU::T760; +} + +bool MaliInfo::IsMaliT8xx() const { + return gpu_version == MaliGPU::T820 || gpu_version == MaliGPU::T830 || + gpu_version == MaliGPU::T860 || gpu_version == MaliGPU::T880; +} + +bool MaliInfo::IsMidgard() const { + return IsMaliT6xx() || IsMaliT7xx() || IsMaliT8xx(); +} + +bool MaliInfo::IsBifrostGen1() const { + return gpu_version == MaliGPU::G31 || gpu_version == MaliGPU::G51 || + gpu_version == MaliGPU::G71; +} + +bool MaliInfo::IsBifrostGen2() const { + return gpu_version == MaliGPU::G52 || gpu_version == MaliGPU::G72; +} + +bool MaliInfo::IsBifrostGen3() const { return gpu_version == MaliGPU::G76; } + +bool MaliInfo::IsBifrost() const { + return IsBifrostGen1() || IsBifrostGen2() || IsBifrostGen3(); +} + +bool MaliInfo::IsValhall() const { + return gpu_version == MaliGPU::G57 || gpu_version == MaliGPU::G77; +} + +bool DeviceInfo::SupportsTextureArray() const { + return cl_version >= OpenCLVersion::CL_1_2; +} + +bool DeviceInfo::SupportsImageBuffer() const { + return cl_version >= OpenCLVersion::CL_1_2; +} + +bool DeviceInfo::SupportsImage3D() const { + if (vendor == Vendor::kMali) { + // On Mali T880 read_imageh doesn't compile with image3d_t + return false; + } + return supports_image3d_writes; +} + +bool DeviceInfo::IsAdreno() const { return vendor == Vendor::kQualcomm; } + +bool DeviceInfo::IsAdreno3xx() const { + return IsAdreno() && IsGPUVersionInRange(adreno_info.gpu_version, 300, 400); +} + +bool DeviceInfo::IsAdreno4xx() const { + return IsAdreno() && IsGPUVersionInRange(adreno_info.gpu_version, 400, 500); +} + +bool DeviceInfo::IsAdreno5xx() const { + return IsAdreno() && IsGPUVersionInRange(adreno_info.gpu_version, 500, 600); +} + +bool DeviceInfo::IsAdreno6xx() const { + return IsAdreno() && IsGPUVersionInRange(adreno_info.gpu_version, 600, 700); +} + +bool DeviceInfo::IsAdreno6xxOrHigher() const { + return IsAdreno() && adreno_info.gpu_version >= 600; +} + +bool DeviceInfo::IsPowerVR() const { return vendor == Vendor::kPowerVR; } + +bool DeviceInfo::IsNvidia() const { return vendor == Vendor::kNvidia; } + +bool DeviceInfo::IsMali() const { return vendor == Vendor::kMali; } + +bool DeviceInfo::IsAMD() const { return vendor == Vendor::kAMD; } + +bool DeviceInfo::IsIntel() const { return vendor == Vendor::kIntel; } + +} // namespace cl +} // namespace gpu +} // namespace tflite diff --git a/tensorflow/lite/delegates/gpu/cl/device_info.h b/tensorflow/lite/delegates/gpu/cl/device_info.h new file mode 100644 index 00000000000..b13fe3df846 --- /dev/null +++ b/tensorflow/lite/delegates/gpu/cl/device_info.h @@ -0,0 +1,168 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_DELEGATES_GPU_CL_DEVICE_INFO_H_ +#define TENSORFLOW_LITE_DELEGATES_GPU_CL_DEVICE_INFO_H_ + +#include +#include + +// for use only in device_info.cc, but keep here to make tests +int GetAdrenoGPUVersion(const std::string& gpu_version); + +namespace tflite { +namespace gpu { +namespace cl { + +enum class Vendor { + kQualcomm, + kMali, + kPowerVR, + kNvidia, + kAMD, + kIntel, + kUnknown +}; +std::string VendorToString(Vendor v); + +enum class OpenCLVersion { + CL_1_0, + CL_1_1, + CL_1_2, + CL_2_0, + CL_2_1, + CL_2_2, + CL_3_0 +}; +std::string OpenCLVersionToString(OpenCLVersion version); + +struct AdrenoInfo { + AdrenoInfo() = default; + explicit AdrenoInfo(const std::string& device_version); + int gpu_version = -1; // can be, for example, 405/430/540/530/630 etc. + + // This function returns some not very documented physical parameter of + // Adreno6xx GPU. + // We obtained it using Snapdragon Profiler. + int GetMaximumWavesCount() const; + + // returns amount of register memory per CU(Compute Unit) in bytes. + int GetRegisterMemorySizePerComputeUnit() const; + + // returns maximum possible amount of waves based on register usage. + int GetMaximumWavesCount(int register_footprint_per_tread, + bool full_wave = true) const; + + int GetWaveSize(bool full_wave) const; + + // Not supported on some Adreno devices with specific driver version. + // b/131099086 + bool support_one_layer_texture_array = true; +}; + +enum class MaliGPU { + T604, + T622, + T624, + T628, + T658, + T678, + T720, + T760, + T820, + T830, + T860, + T880, + G31, + G51, + G71, + G52, + G72, + G76, + G57, + G77, + UNKNOWN +}; + +struct MaliInfo { + MaliInfo() = default; + explicit MaliInfo(const std::string& device_name); + MaliGPU gpu_version; + + bool IsMaliT6xx() const; + bool IsMaliT7xx() const; + bool IsMaliT8xx() const; + bool IsMidgard() const; + bool IsBifrostGen1() const; + bool IsBifrostGen2() const; + bool IsBifrostGen3() const; + bool IsBifrost() const; + bool IsValhall() const; +}; + +struct DeviceInfo { + DeviceInfo() = default; + + bool IsAdreno() const; + bool IsAdreno3xx() const; + bool IsAdreno4xx() const; + bool IsAdreno5xx() const; + bool IsAdreno6xx() const; + bool IsAdreno6xxOrHigher() const; + bool IsPowerVR() const; + bool IsNvidia() const; + bool IsMali() const; + bool IsAMD() const; + bool IsIntel() const; + + bool SupportsTextureArray() const; + bool SupportsImageBuffer() const; + bool SupportsImage3D() const; + + std::vector extensions; + bool supports_fp16; + bool supports_image3d_writes; + Vendor vendor; + OpenCLVersion cl_version; + int compute_units_count; + uint64_t buffer_max_size; + uint64_t image2d_max_width; + uint64_t image2d_max_height; + uint64_t image_buffer_max_size; + uint64_t image_array_max_layers; + uint64_t image3d_max_width; + uint64_t image3d_max_height; + uint64_t image3d_max_depth; + int max_work_group_size_x; + int max_work_group_size_y; + int max_work_group_size_z; + + // rtn is ROUND_TO_NEAREST + // with rtn precision is much better then with rtz (ROUND_TO_ZERO) + // Adreno 3xx supports only rtz, Adreno 4xx and more support rtn + // Mali from T6xx supports rtn + // PowerVR supports only rtz + bool supports_fp32_rtn; + bool supports_fp16_rtn; + + AdrenoInfo adreno_info; + MaliInfo mali_info; +}; + +} // namespace cl +} // namespace gpu +} // namespace tflite + +#endif // TENSORFLOW_LITE_DELEGATES_GPU_CL_DEVICE_INFO_H_ diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc index b93fe113d89..ed1ec8be7b1 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc @@ -37,7 +37,7 @@ int GetAdrenoOptimalMaxConstantSize(int gpu_version) { } int GetOptimalMaxConstantSize(const DeviceInfo& info) { - if (info.vendor != Vendor::QUALCOMM) { + if (!info.IsAdreno()) { // In general we do not expect that this kernel will be used with non Adreno // so as it tuned for __constant memory that have big profit on Adreno return 1024; // 1KB diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc index 5e280d5f98b..3771a5b033a 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc @@ -80,9 +80,12 @@ absl::Status GetBestWorkGroupAlignedToGrid(const TuningParameters& params, const int3& grid, int3* best_work_group) { std::vector work_groups; + int3 max_wg_size; + max_wg_size.x = params.info->max_work_group_size_x; + max_wg_size.y = params.info->max_work_group_size_y; + max_wg_size.z = params.info->max_work_group_size_z; RETURN_IF_ERROR(GenerateWorkGroupSizesAlignedToGrid( - grid, params.info->max_work_group_sizes, kernel.GetMaxWorkGroupSize(), - &work_groups)); + grid, max_wg_size, kernel.GetMaxWorkGroupSize(), &work_groups)); int best_work_group_index; RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( kernel, *params.info, grid, work_groups, &best_work_group_index)); @@ -268,10 +271,10 @@ absl::Status GetBestWorkGroupConv(const TuningParameters& params, switch (params.tuning_type) { case TuningType::FAST: { int max_z_size = 16; - if (params.info->vendor == Vendor::QUALCOMM) { + if (params.info->IsAdreno()) { max_z_size = params.info->adreno_info.gpu_version < 400 ? 16 : 64; } - max_z_size = std::min(max_z_size, params.info->max_work_group_sizes.z); + max_z_size = std::min(max_z_size, params.info->max_work_group_size_z); *best_work_group = GetWorkGroupConv(grid, kernel.GetMaxWorkGroupSize(), max_z_size); return absl::OkStatus(); diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/convolution_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/convolution_selector.cc index 3e2531c02b3..b577757057e 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/convolution_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/convolution_selector.cc @@ -167,22 +167,21 @@ absl::Status SelectConvolution(const Convolution2DAttributes& attr, const CreationContext& creation_context, const OperationDef& op_def, ModelHints hints, std::unique_ptr* ptr) { - switch (creation_context.device->vendor()) { - case Vendor::QUALCOMM: - return SelectConvolutionAdreno(attr, dst_shape, creation_context, op_def, + const auto& device_info = creation_context.device->GetInfo(); + if (device_info.IsAdreno()) { + return SelectConvolutionAdreno(attr, dst_shape, creation_context, op_def, hints, ptr); - case Vendor::POWERVR: - case Vendor::INTEL: - case Vendor::AMD: - return SelectConvolutionPowerVR(attr, creation_context, op_def, ptr); - case Vendor::NVIDIA: - return SelectConvolutionNVidia(attr, dst_shape, creation_context, op_def, + } else if (device_info.IsPowerVR() || device_info.IsAMD() || + device_info.IsIntel()) { + return SelectConvolutionPowerVR(attr, creation_context, op_def, ptr); + } else if (device_info.IsNvidia()) { + return SelectConvolutionNVidia(attr, dst_shape, creation_context, op_def, ptr); - case Vendor::MALI: - return SelectConvolutionMali(attr, dst_shape, creation_context, op_def, + } else if (device_info.IsMali()) { + return SelectConvolutionMali(attr, dst_shape, creation_context, op_def, ptr); - default: - return SelectConvolutionAdreno(attr, dst_shape, creation_context, op_def, + } else { + return SelectConvolutionAdreno(attr, dst_shape, creation_context, op_def, hints, ptr); } } @@ -191,25 +190,22 @@ absl::Status SelectConvolutionForWinograd( const Convolution2DAttributes& attr, const BHWC& dst_shape, const CreationContext& creation_context, const OperationDef& op_def, ModelHints hints, std::unique_ptr* ptr) { - switch (creation_context.device->vendor()) { - case Vendor::QUALCOMM: - return SelectConvolutionWinogradAdreno(attr, dst_shape, creation_context, + const auto& device_info = creation_context.device->GetInfo(); + if (device_info.IsAdreno()) { + return SelectConvolutionWinogradAdreno(attr, dst_shape, creation_context, op_def, hints, ptr); - case Vendor::POWERVR: - case Vendor::AMD: - case Vendor::INTEL: - case Vendor::NVIDIA: { - ConvPowerVR conv; + } else if (device_info.IsPowerVR() || device_info.IsAMD() || + device_info.IsNvidia() || device_info.IsIntel()) { + ConvPowerVR conv; RETURN_IF_ERROR(CreateConvPowerVRWino4x4To6x6(creation_context, op_def, attr, &conv, &dst_shape)); *ptr = absl::make_unique(std::move(conv)); return absl::OkStatus(); - } - case Vendor::MALI: - return SelectConvolutionWinogradMali(attr, dst_shape, creation_context, + } else if (device_info.IsMali()) { + return SelectConvolutionWinogradMali(attr, dst_shape, creation_context, op_def, ptr); - default: - return SelectConvolutionWinogradAdreno(attr, dst_shape, creation_context, + } else { + return SelectConvolutionWinogradAdreno(attr, dst_shape, creation_context, op_def, hints, ptr); } } @@ -219,23 +215,22 @@ absl::Status SelectConvolutionWithDynamicWeights( const BHWC& dst_shape, const CreationContext& creation_context, const OperationDef& op_def, ModelHints hints, std::unique_ptr* ptr, ConvWeightsDescription* weights_desc) { - switch (creation_context.device->vendor()) { - case Vendor::QUALCOMM: - return SelectConvolutionDynamicWeightsAdreno( - attr, weights_shape, dst_shape, creation_context, op_def, hints, ptr, - weights_desc); - case Vendor::MALI: - return SelectConvolutionDynamicWeightsMali(attr, weights_shape, dst_shape, + const auto& device_info = creation_context.device->GetInfo(); + if (device_info.IsAdreno()) { + return SelectConvolutionDynamicWeightsAdreno(attr, weights_shape, dst_shape, creation_context, op_def, hints, ptr, weights_desc); - default: { - ConvPowerVR conv; - RETURN_IF_ERROR(CreateConvPowerVRDynamicWeights( - creation_context, op_def, attr, weights_shape, &conv, &dst_shape)); - *weights_desc = conv.GetConvWeightsDescription(); - *ptr = absl::make_unique(std::move(conv)); - return absl::OkStatus(); - } + } else if (device_info.IsMali()) { + return SelectConvolutionDynamicWeightsMali(attr, weights_shape, dst_shape, + creation_context, op_def, hints, + ptr, weights_desc); + } else { + ConvPowerVR conv; + RETURN_IF_ERROR(CreateConvPowerVRDynamicWeights( + creation_context, op_def, attr, weights_shape, &conv, &dst_shape)); + *weights_desc = conv.GetConvWeightsDescription(); + *ptr = absl::make_unique(std::move(conv)); + return absl::OkStatus(); } } diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/convolution_transposed_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/convolution_transposed_selector.cc index 5fdfdca073e..56864f2c575 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/convolution_transposed_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/convolution_transposed_selector.cc @@ -105,22 +105,19 @@ absl::Status SelectConvolutionTransposed( const ConvolutionTransposedAttributes& attr, const CreationContext& creation_context, const OperationDef& op_def, std::unique_ptr* ptr) { - switch (creation_context.device->vendor()) { - case Vendor::QUALCOMM: - return SelectConvolutionTransposedAdreno(attr, creation_context, op_def, - ptr); - case Vendor::POWERVR: - case Vendor::NVIDIA: - case Vendor::AMD: - case Vendor::INTEL: - return SelectConvolutionTransposedPowerVR(attr, creation_context, op_def, - ptr); - case Vendor::MALI: - return SelectConvolutionTransposedMali(attr, creation_context, op_def, + const auto& device_info = creation_context.device->GetInfo(); + if (device_info.IsAdreno()) { + return SelectConvolutionTransposedAdreno(attr, creation_context, op_def, + ptr); + } else if (device_info.IsPowerVR() || device_info.IsAMD() || + device_info.IsNvidia() || device_info.IsIntel()) { + return SelectConvolutionTransposedPowerVR(attr, creation_context, op_def, + ptr); + } else if (device_info.IsMali()) { + return SelectConvolutionTransposedMali(attr, creation_context, op_def, ptr); + } else { + return SelectConvolutionTransposedAdreno(attr, creation_context, op_def, ptr); - default: - return SelectConvolutionTransposedAdreno(attr, creation_context, op_def, - ptr); } } diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/dw_convolution_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/dw_convolution_selector.cc index 54ff45d182a..fafd9078f6f 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/dw_convolution_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/dw_convolution_selector.cc @@ -90,15 +90,15 @@ absl::Status SelectDWConvolution(const DepthwiseConvolution2DAttributes& attr, const CreationContext& creation_context, const OperationDef& op_def, std::unique_ptr* ptr) { - switch (creation_context.device->vendor()) { - case Vendor::QUALCOMM: - return SelectDWConvolutionAdreno(attr, creation_context, op_def, ptr); - case Vendor::POWERVR: - return SelectDWConvolutionPowerVR(attr, creation_context, op_def, ptr); - case Vendor::MALI: - return SelectDWConvolutionMali(attr, creation_context, op_def, ptr); - default: - return SelectDWConvolutionAdreno(attr, creation_context, op_def, ptr); + const auto& device_info = creation_context.device->GetInfo(); + if (device_info.IsAdreno()) { + return SelectDWConvolutionAdreno(attr, creation_context, op_def, ptr); + } else if (device_info.IsPowerVR()) { + return SelectDWConvolutionPowerVR(attr, creation_context, op_def, ptr); + } else if (device_info.IsMali()) { + return SelectDWConvolutionMali(attr, creation_context, op_def, ptr); + } else { + return SelectDWConvolutionAdreno(attr, creation_context, op_def, ptr); } } diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/fully_connected_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/fully_connected_selector.cc index eacbea8b586..cb967e45b52 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/fully_connected_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/fully_connected_selector.cc @@ -104,22 +104,20 @@ absl::Status SelectFullyConnected(const FullyConnectedAttributes& attr, const CreationContext& creation_context, const OperationDef& op_def, int batch_size, std::unique_ptr* ptr) { - switch (creation_context.device->vendor()) { - case Vendor::QUALCOMM: - return SelectFullyConnectedAdreno(attr, creation_context, op_def, - batch_size, ptr); - case Vendor::POWERVR: - case Vendor::AMD: - case Vendor::NVIDIA: - case Vendor::INTEL: - return SelectFullyConnectedPowerVR(attr, creation_context, op_def, - batch_size, ptr); - case Vendor::MALI: - return SelectFullyConnectedMali(attr, creation_context, op_def, + const auto& device_info = creation_context.device->GetInfo(); + if (device_info.IsAdreno()) { + return SelectFullyConnectedAdreno(attr, creation_context, op_def, batch_size, ptr); - default: - return SelectFullyConnectedGeneric(attr, creation_context, op_def, - batch_size, ptr); + } else if (device_info.IsPowerVR() || device_info.IsAMD() || + device_info.IsNvidia() || device_info.IsIntel()) { + return SelectFullyConnectedPowerVR(attr, creation_context, op_def, + batch_size, ptr); + } else if (device_info.IsMali()) { + return SelectFullyConnectedMali(attr, creation_context, op_def, batch_size, + ptr); + } else { + return SelectFullyConnectedGeneric(attr, creation_context, op_def, + batch_size, ptr); } } From a04564d58816e10bcedd1b9911605fb1d878781d Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Tue, 4 Aug 2020 00:58:27 +0000 Subject: [PATCH 0308/1017] Modify _define_function_with_shape_relaxation, remove some calls to nest.flatten --- tensorflow/python/eager/function.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index d9b141e049a..6febd72b54f 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -3198,10 +3198,16 @@ class Function(object): shared_func_graph=False) return graph_function - def _define_function_with_shape_relaxation(self, args, kwargs): + def _define_function_with_shape_relaxation(self, + args, + kwargs, + flat_args, + flat_kwargs): """Define a function, relaxing arg shapes to avoid unnecessary retracing.""" + flat_args_all = nest.flatten((args, kwargs), expand_composites=False) + any_composite_args = any(isinstance(x, composite_tensor.CompositeTensor) - for x in nest.flatten((args, kwargs))) + for x in flat_args_all) # Build a cache key where TensorShapes include only rank information (and # not information about the size of each dimension). @@ -3216,7 +3222,7 @@ class Function(object): rank_only_cache_key = self._cache_key( cache_key_args, cache_key_kwargs, include_tensor_ranks_only=True) - arg_specs = [_type_spec_for(x) for x in nest.flatten((args, kwargs))] + arg_specs = [_type_spec_for(x) for x in flat_args_all] relaxed_arg_specs = self._function_cache.arg_relaxed_specs.get( rank_only_cache_key, None) relaxed_arg_function = self._function_cache.arg_relaxed.get( @@ -3225,7 +3231,7 @@ class Function(object): if (relaxed_arg_function is not None and all(_is_type_subset(x, y) for (x, y) in zip(relaxed_arg_specs, arg_specs))): - return relaxed_arg_function, args, kwargs + return relaxed_arg_function, flat_args, flat_kwargs if relaxed_arg_specs is None: relaxed_arg_specs = arg_specs @@ -3251,14 +3257,16 @@ class Function(object): (args, kwargs), relaxed_arg_specs, expand_composites=False) (args, kwargs) = nest.pack_sequence_as( (relaxed_arg_specs, relaxed_kwarg_specs), - nest.flatten((args, kwargs), expand_composites=True), + flat_args + flat_kwargs, expand_composites=True) graph_function = self._create_graph_function( args, kwargs, override_flat_arg_shapes=relaxed_arg_shapes) self._function_cache.arg_relaxed[rank_only_cache_key] = graph_function - return graph_function, args, kwargs + return (graph_function, + nest.flatten(args, expand_composites=True), + nest.flatten(kwargs, expand_composites=True)) def _maybe_define_function(self, args, kwargs): """Gets a function for these inputs, defining it if necessary. @@ -3286,6 +3294,7 @@ class Function(object): args, kwargs, flat_args, flat_kwargs = \ self._function_spec.canonicalize_function_inputs(*args, **kwargs) else: + # TODO(jlchu): Check - empty lists or Nones? flat_args, flat_kwargs = [], [] cache_key = self._cache_key(args, kwargs) @@ -3325,10 +3334,8 @@ class Function(object): if (self._experimental_relax_shapes and self.input_signature is None and call_context_key in self._function_cache.missed): - return_function, _, _ = \ - self._define_function_with_shape_relaxation(args, kwargs) - #TODO(jlchu): Investigate modifying above function sig directly - return return_function, flat_args, flat_kwargs + return self._define_function_with_shape_relaxation( + args, kwargs, flat_args, flat_kwargs) self._function_cache.missed.add(call_context_key) graph_function = self._create_graph_function(args, kwargs) From c94f5813e65b4a48901053ef86d48f13b2571b24 Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Mon, 3 Aug 2020 17:59:24 -0700 Subject: [PATCH 0309/1017] Address comments --- .../compiler/mlir/tensorflow/ir/tf_ops_n_z.cc | 43 +++++++++---------- .../mlir/tensorflow/tests/tf-ops.mlir | 2 +- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc index 0b9b757da55..0941345a76c 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc @@ -1760,6 +1760,9 @@ void ToBoolOp::getCanonicalizationPatterns(OwningRewritePatternList &results, static LogicalResult Verify(TransposeOp op) { auto perm_type = op.perm().getType().dyn_cast(); + auto x_type = op.x().getType().dyn_cast(); + auto y_type = op.y().getType().dyn_cast(); + if (!perm_type) { return success(); } @@ -1770,33 +1773,23 @@ static LogicalResult Verify(TransposeOp op) { << perm_type.getRank(); } - if (!perm_type.hasStaticShape()) { + if (x_type && y_type && x_type.getRank() != y_type.getRank()) { + return op.emitOpError() + << "x should be of the same rank with y, got " + << "x of rank " << x_type.getRank() << ", and y of rank " + << y_type.getRank(); + } + + if (!x_type || !y_type || !perm_type.hasStaticShape()) { return success(); } - auto x_type = op.x().getType().dyn_cast(); - if (!x_type) { - return success(); - } - - const int64_t x_rank = x_type.getRank(); - if (x_rank != perm_type.getNumElements()) { + if (x_type.getRank() != perm_type.getNumElements()) { return op.emitOpError() << "expected perm to be a 1-D Tensor of size " << "equal to the rank of x, got perm of size " - << perm_type.getNumElements() << ", and x of rank " << x_rank; - } - - auto y_type = op.y().getType().dyn_cast(); - if (!y_type) { - return success(); - } - - const int64_t y_rank = y_type.getRank(); - if (x_rank != y_rank) { - return op.emitOpError() - << "x should be of the same rank with y, got " - << "x of rank " << x_rank << ", and y of rank " << y_rank; + << perm_type.getNumElements() << ", and x of rank " + << x_type.getRank(); } DenseIntElementsAttr attr_perm; @@ -1808,10 +1801,14 @@ static LogicalResult Verify(TransposeOp op) { const int64_t y_dim = y_type.getDimSize(y_idx); const int64_t x_idx = e.value().getSExtValue(); const int64_t x_dim = x_type.getDimSize(x_idx); + if (y_dim == ShapedType::kDynamicSize || x_dim == ShapedType::kDynamicSize) { + continue; + } if (y_dim != x_dim) { return op.emitOpError() - << "y.shape[" << y_idx << "] = " << y_dim - << " != x.shape[perm[" << x_idx << "]] = " << x_dim; + << "requires y.shape[" << y_idx << "] (" << y_dim << ") " + << "to be equal to x.shape[perm[" << x_idx << "]] " + << "(" << x_dim << ")"; } } } diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir index 4fd691cc104..04469e69684 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir @@ -2091,7 +2091,7 @@ func @testTranspose(tensor<2x3xf32>) -> tensor<3x2x1xf32> { func @testTranspose(tensor<2x3x4xf32>) -> tensor<3x2x4xf32> { ^bb0(%arg0: tensor<2x3x4xf32>): %cst = constant dense<[2, 0, 1]> : tensor<3xi32> - // expected-error @+1 {{y.shape[0] = 3 != x.shape[perm[2]] = 4}} + // expected-error @+1 {{requires y.shape[0] (3) to be equal to x.shape[perm[2]] (4)}} %0 = "tf.Transpose"(%arg0, %cst) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x3x4xf32>, tensor<3xi32>) -> tensor<3x2x4xf32> return %0 : tensor<3x2x4xf32> } From 3410880bd3447efad4a273b9e65948d2a26d03ae Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Tue, 4 Aug 2020 01:07:52 +0000 Subject: [PATCH 0310/1017] Remove / modify temp comments --- tensorflow/python/eager/function.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 6febd72b54f..e5dc33b513a 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -1853,8 +1853,7 @@ class ConcreteFunction(object): `flat_args` and `flat_kwargs`. """ return self._call_flat( - [t for t in flat_args + flat_kwargs \ - # TODO(jlchu): delete when final [t for t in nest.flatten((args, kwargs), expand_composites=True) + [t for t in flat_args + flat_kwargs if isinstance(t, (ops.Tensor, resource_variable_ops.BaseResourceVariable))], captured_inputs=self.captured_inputs, @@ -2727,9 +2726,11 @@ def _is_ndarray(value): def _convert_numpy_inputs(inputs): """Convert numpy array inputs to tensors.""" - ## TODO(jlchu): Modify/delete comment when change is final!!! # We assume that any CompositeTensors have already converted their components - # from numpy arrays to Tensors, so we don't need to expand composites here. + # from numpy arrays to Tensors, so we don't need to expand composites here for + # the numpy array conversion. Instead, we do so because the flattened inputs + # are eventually passed to ConcreteFunction()._filtered_call, which requires + # expanded composites. flat_inputs = nest.flatten(inputs, expand_composites=True) # Check for NumPy arrays in arguments and convert them to Tensors. From 69e1ca4cdea60cbff7881f83c21b5a3578615bfc Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 3 Aug 2020 18:03:08 -0700 Subject: [PATCH 0311/1017] Add Compilation Cache local lookup feature to TF-TPU support PiperOrigin-RevId: 324722948 Change-Id: Iab0cadf375fb2a23e2aa6d3c5bcb7aa08328a9c3 --- tensorflow/core/tpu/kernels/BUILD | 6 ++++++ .../core/tpu/kernels/tpu_configuration_ops.cc | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tensorflow/core/tpu/kernels/BUILD b/tensorflow/core/tpu/kernels/BUILD index 3b7d0e09c08..75d12f89426 100644 --- a/tensorflow/core/tpu/kernels/BUILD +++ b/tensorflow/core/tpu/kernels/BUILD @@ -89,9 +89,15 @@ tf_kernel_library( name = "tpu_configuration_ops", srcs = ["tpu_configuration_ops.cc"], hdrs = ["tpu_configuration_ops.h"], + copts = select({ + WITH_TPU_SUPPORT: ["-DLIBTFTPU"], + DEFAULT: [], + }), deps = [ ":tpu_compilation_cache_factory", ":tpu_compilation_cache_interface", + ":tpu_compilation_cache_local_lookup", + ":tpu_compilation_cache_lookup", ":tpu_mesh_state_interface", ":tpu_op_consts", "//tensorflow/c:tf_status", diff --git a/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc b/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc index e098dbd682c..4030cf86910 100644 --- a/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc +++ b/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc @@ -25,6 +25,8 @@ limitations under the License. #include "tensorflow/core/platform/refcount.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_factory.h" #include "tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h" +#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_local_lookup.h" +#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_lookup.h" #include "tensorflow/core/tpu/kernels/tpu_mesh_state_interface.h" #include "tensorflow/core/tpu/kernels/tpu_op_consts.h" #include "tensorflow/core/tpu/tpu_api.h" @@ -253,6 +255,12 @@ void InitializeHostForDistributedTpuOp::Compute(OpKernelContext* ctx) { mesh_state_interface)); } +#if defined(LIBTFTPU) + VLOG(1) << "Removing existing proto compilation cache lookup if it exists"; + OP_REQUIRES_OK(ctx, DeleteIfExists( + rmgr, tpu::kCompiledProtoCacheResourceName)); +#endif + if (enable_whole_mesh_compilations_) { // If this is a whole mesh compilation mode, create the compilation cache, // if missing. @@ -276,6 +284,15 @@ void InitializeHostForDistributedTpuOp::Compute(OpKernelContext* ctx) { if (local_compilation_cache != nullptr) { local_compilation_cache->Unref(); + +#if defined(LIBTFTPU) + tpu::TpuCompilationCacheLookup* proto_lookup; + proto_lookup = + new tpu::TpuCompilationCacheLocalLookup(local_compilation_cache); + OP_REQUIRES_OK( + ctx, rmgr->Create(rmgr->default_container(), + tpu::kCompiledProtoCacheResourceName, proto_lookup)); +#endif } Tensor* ctx_output; From 4d765c332e6c6046e7ab71f7aea9d5c0bd890913 Mon Sep 17 00:00:00 2001 From: bbbboom Date: Tue, 4 Aug 2020 09:10:04 +0800 Subject: [PATCH 0312/1017] Update generate.sh Add corresponding annotation. --- tensorflow/go/genop/generate.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/go/genop/generate.sh b/tensorflow/go/genop/generate.sh index 54541106f13..547dd790e05 100644 --- a/tensorflow/go/genop/generate.sh +++ b/tensorflow/go/genop/generate.sh @@ -24,10 +24,14 @@ then GOPATH=$(go env GOPATH) fi -# change GOPATH style +# convert GOPATH's Windows style to UNIX style if [ $1 == "win" ]; then + # eg: convert "D:\go-14;D:\go-13" to "D\go-14;D\go-13" + GOPATH=${GOPATH//:\\/\\} + # eg: convert "D\go-14;D\go-13" to "\D\go-14:\D\go-13" + GOPATH=\\${GOPATH//;/:\\} + # eg: convert "\D\go-14:\D\go-13" to "/D/go-14:/D/go-13" GOPATH=${GOPATH//\\/\/} - GOPATH=/${GOPATH//:/} fi cd $(dirname $0) From 2d968de1433b9afc9752aa29954b013cea7d110b Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Tue, 4 Aug 2020 01:23:13 +0000 Subject: [PATCH 0313/1017] Resolve minor TODO in function.py --- tensorflow/python/eager/def_function.py | 2 +- tensorflow/python/eager/function.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py index a1ffd2f9efc..8447245b524 100644 --- a/tensorflow/python/eager/def_function.py +++ b/tensorflow/python/eager/def_function.py @@ -917,7 +917,7 @@ class Function(object): canon_args, canon_kwds, flat_args, flat_kwds = \ self._stateful_fn._function_spec.canonicalize_function_inputs( # pylint: disable=protected-access *args, **kwds) - # TODO(jlchu): verify that mdofication to fn_with_cond works + # TODO(jlchu): verify that modification to fn_with_cond works return function_lib.defun(fn_with_cond)(canon_args, canon_kwds, flat_args, flat_kwds) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index e5dc33b513a..a833d351c84 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -3295,8 +3295,7 @@ class Function(object): args, kwargs, flat_args, flat_kwargs = \ self._function_spec.canonicalize_function_inputs(*args, **kwargs) else: - # TODO(jlchu): Check - empty lists or Nones? - flat_args, flat_kwargs = [], [] + flat_args, flat_kwargs = [None], [None] cache_key = self._cache_key(args, kwargs) From 9e6c87150b098953584acca770a397065ea548e3 Mon Sep 17 00:00:00 2001 From: Feng Liu Date: Mon, 3 Aug 2020 18:21:39 -0700 Subject: [PATCH 0314/1017] run shape inference after the constants are frozen PiperOrigin-RevId: 324725487 Change-Id: I7fa6dd741d502c83817cbd0009b03b07ac761170 --- tensorflow/compiler/mlir/lite/tf_tfl_passes.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow/compiler/mlir/lite/tf_tfl_passes.cc b/tensorflow/compiler/mlir/lite/tf_tfl_passes.cc index c49d9a10716..d63eb481376 100644 --- a/tensorflow/compiler/mlir/lite/tf_tfl_passes.cc +++ b/tensorflow/compiler/mlir/lite/tf_tfl_passes.cc @@ -166,6 +166,10 @@ void AddTFToTFLConversionPasses(const mlir::TFL::PassConfig& pass_config, // The below passes only make sense if Builtin TFLite ops are enabled // for emission. if (pass_config.emit_builtin_tflite_ops) { + // Run shape inference after variables are converted to constants. + if (pass_config.shape_inference) { + pass_manager->addPass(mlir::TF::CreateTFShapeInferencePass()); + } // Prepare for TFLite dialect, rerun canonicalization, and then legalize to // the TFLite dialect. pass_manager->addPass( @@ -173,6 +177,9 @@ void AddTFToTFLConversionPasses(const mlir::TFL::PassConfig& pass_config, pass_manager->addNestedPass(mlir::createCanonicalizerPass()); if (pass_config.shape_inference) { // Add a shape inference pass to optimize away the unnecessary casts. + // This also fixes the unranked shapes due to TF ops constant folding. + // TODO(fengliuai): remove this pass if TableGen patterns have a better + // to control the shapes for the intermediate results. pass_manager->addPass(mlir::TF::CreateTFShapeInferencePass()); } From 23d2e73f82418f395cab1b4eec9c6f9bde63615b Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Tue, 4 Aug 2020 01:24:52 +0000 Subject: [PATCH 0315/1017] Update tensorflow/core/platform/file_system.h --- tensorflow/core/platform/file_system.h | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h index 77ba34bcdc6..4a8d9e63023 100644 --- a/tensorflow/core/platform/file_system.h +++ b/tensorflow/core/platform/file_system.h @@ -697,7 +697,6 @@ class WrappedFileSystem : public FileSystem { /// Transactional API. This is an interim solution until ModularFileSystem class /// becomes a singleton. // TODO(sami): Remove this macro when filesystem plugins migration is complete. - #define TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT \ using FileSystem::NewRandomAccessFile; \ using FileSystem::NewWritableFile; \ From e2865bb150fb9371f96dc4f33fc6cea26b1c143b Mon Sep 17 00:00:00 2001 From: Anjali Sridhar Date: Mon, 3 Aug 2020 18:26:43 -0700 Subject: [PATCH 0316/1017] Add MWMS combinations to custom_training_loop_metrics_test. PiperOrigin-RevId: 324726066 Change-Id: I05fcf563c34c216d9a59a1656e3ce4d409e2a8f3 --- .../keras/distribute/custom_training_loop_metrics_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/distribute/custom_training_loop_metrics_test.py b/tensorflow/python/keras/distribute/custom_training_loop_metrics_test.py index 8704b8378bf..a41d1f369a4 100644 --- a/tensorflow/python/keras/distribute/custom_training_loop_metrics_test.py +++ b/tensorflow/python/keras/distribute/custom_training_loop_metrics_test.py @@ -34,7 +34,8 @@ class KerasMetricsTest(test.TestCase, parameterized.TestCase): @combinations.generate( combinations.combine( - distribution=strategy_combinations.all_strategies, + distribution=strategy_combinations.all_strategies + + strategy_combinations.multiworker_strategies, mode=["eager"] )) def test_multiple_keras_metrics_experimental_run(self, distribution): @@ -58,7 +59,8 @@ class KerasMetricsTest(test.TestCase, parameterized.TestCase): @combinations.generate( combinations.combine( - distribution=strategy_combinations.all_strategies, + distribution=strategy_combinations.all_strategies+ + strategy_combinations.multiworker_strategies, mode=["eager"] )) def test_update_keras_metric_declared_in_strategy_scope(self, distribution): @@ -98,4 +100,4 @@ class KerasMetricsTest(test.TestCase, parameterized.TestCase): if __name__ == "__main__": - test.main() + combinations.main() From d353f49989edd1e9e8cf41466fb2d46c226eea5d Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Mon, 3 Aug 2020 18:28:29 -0700 Subject: [PATCH 0317/1017] Extended support of SUB (and other elementwise ops). OpenCL delegate supports SUB with runtime tensor as second argument. PiperOrigin-RevId: 324726289 Change-Id: If26a72a5214bffc7b664f1902344ab04038ed3f5 --- .../delegates/gpu/cl/kernels/elementwise.cc | 103 +++++++++++++----- .../delegates/gpu/cl/kernels/elementwise.h | 26 +---- .../gpu/cl/kernels/elementwise_test.cc | 60 +++++++--- .../gpu/cl/selectors/operation_selector.cc | 99 ++--------------- .../delegates/gpu/common/model_builder.cc | 2 + .../lite/delegates/gpu/common/operations.h | 4 + 6 files changed, 143 insertions(+), 151 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc index 063b15c1b69..f735f1aa047 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc @@ -98,53 +98,51 @@ std::string GetOneInputCode(const OperationType& op_type, } std::string GetTwoInputCode(const OperationType& op_type, + const std::string& result_var, const std::string& input0, - const std::string& input1) { + const std::string& input1, + bool swap_inputs = false) { std::string result; switch (op_type) { case OperationType::ADD: - result += "$0 += $1;\n"; + result += "$0 = $1 + $2;\n"; break; case OperationType::DIV: - result += "$0 /= $1;\n"; + result += "$0 = $1 / $2;\n"; break; case OperationType::MAXIMUM: - result += "$0 = max($0, $1);\n"; + result += "$0 = max($1, $2);\n"; break; case OperationType::MINIMUM: - result += "$0 = min($0, $1);\n"; + result += "$0 = min($1, $2);\n"; break; case OperationType::MUL: - result += "$0 *= $1;\n"; + result += "$0 = $1 * $2;\n"; break; case OperationType::POW: - result += "$0 = pow($0, $1);\n"; + result += "$0 = pow($1, $2);\n"; break; case OperationType::SQUARED_DIFF: - result += "$0 -= $1;\n"; - result += "$0 *= $0;\n"; + result += "$0 = ($1 - $2) * ($1 - $2);\n"; break; case OperationType::SUB: - result += "$0 -= $1;\n"; + result += "$0 = $1 - $2;\n"; break; default: return "Unknown operation type;\n"; } - return absl::Substitute(result, input0, input1); -} -} // namespace - -GPUOperation CreateElementwiseOneInput(const OperationDef& definition, - const OperationType& op_type) { - GPUOperation op(definition); - op.elementwise_ = true; - op.code_ = GetOneInputCode(op_type, definition.precision, "in_out_value"); - return op; + if (swap_inputs) { + return absl::Substitute(result, result_var, input1, input0); + } else { + return absl::Substitute(result, result_var, input0, input1); + } } +// Creates simple two input (first input is runtime tensor and second input is +// scalar argument) operation, for example sub, div, pow, etc. GPUOperation CreateElementwiseOneRuntimeOneScalar( - const CreationContext& creation_context, const OperationDef& definition, - const OperationType& op_type, float scalar_parameter) { + const OperationDef& definition, const OperationType& op_type, + float scalar_parameter, bool swap_inputs) { GPUOperation op(definition); op.elementwise_ = true; if (definition.precision == CalculationsPrecision::F32) { @@ -152,15 +150,21 @@ GPUOperation CreateElementwiseOneRuntimeOneScalar( } else { op.args_.AddHalf("scalar", half(scalar_parameter)); } - op.code_ = GetTwoInputCode(op_type, "in_out_value", "args.scalar"); + op.code_ = + "FLT4 second_val = (FLT4)(args.scalar, args.scalar, args.scalar, " + "args.scalar);\n"; + op.code_ += GetTwoInputCode(op_type, "in_out_value", "in_out_value", + "second_val", swap_inputs); return op; } +// Creates simple two input(first input is runtime tensor and second input is +// constant linear tensor) operation, for example sub, div and etc. absl::Status CreateElementwiseTwoInput( const CreationContext& creation_context, const OperationDef& definition, const OperationType& op_type, const tflite::gpu::Tensor& constant_tensor, - GPUOperation* result) { + bool swap_inputs, GPUOperation* result) { const BHWC shape = BHWC(1, 1, 1, constant_tensor.shape.v); TensorStorageType storage_type = SelectBestStorageType(*creation_context.context, *creation_context.device, @@ -187,15 +191,18 @@ absl::Status CreateElementwiseTwoInput( result->code_ += " second_val.z = second_val.x;\n"; result->code_ += " second_val.w = second_val.x;\n"; } - result->code_ += GetTwoInputCode(op_type, "in_out_value", "second_val"); + result->code_ += GetTwoInputCode(op_type, "in_out_value", "in_out_value", + "second_val", swap_inputs); return absl::OkStatus(); } +// Creates simple two input(first input is runtime tensor and second input is +// constant HWC tensor) operation, for example sub, div and etc. absl::Status CreateElementwiseTwoInput( const CreationContext& creation_context, const OperationDef& definition, const OperationType& op_type, const tflite::gpu::Tensor& constant_tensor, - GPUOperation* result) { + bool swap_inputs, GPUOperation* result) { const BHWC shape = BHWC(1, constant_tensor.shape.h, constant_tensor.shape.w, constant_tensor.shape.c); TensorStorageType storage_type = @@ -225,11 +232,50 @@ absl::Status CreateElementwiseTwoInput( result->code_ += " second_val.z = second_val.x;\n"; result->code_ += " second_val.w = second_val.x;\n"; } - result->code_ += GetTwoInputCode(op_type, "in_out_value", "second_val"); + result->code_ += GetTwoInputCode(op_type, "in_out_value", "in_out_value", + "second_val", swap_inputs); return absl::OkStatus(); } +} // namespace + +GPUOperation CreateElementwiseOneInput(const OperationDef& definition, + const OperationType& op_type) { + GPUOperation op(definition); + op.elementwise_ = true; + op.code_ = GetOneInputCode(op_type, definition.precision, "in_out_value"); + return op; +} + +absl::Status CreateElementwise(const CreationContext& creation_context, + const OperationDef& definition, + const OperationType& op_type, + const ElementwiseAttributes& attr, + GPUOperation* result) { + const float* scalar = absl::get_if(&attr.param); + const auto* linear_tensor = + absl::get_if>(&attr.param); + const auto* hwc_tensor = + absl::get_if>(&attr.param); + + if (scalar) { + *result = CreateElementwiseOneRuntimeOneScalar( + definition, op_type, *scalar, attr.runtime_tensor_is_second); + return absl::OkStatus(); + } else if (linear_tensor) { + return CreateElementwiseTwoInput(creation_context, definition, op_type, + *linear_tensor, + attr.runtime_tensor_is_second, result); + } else if (hwc_tensor) { + return CreateElementwiseTwoInput(creation_context, definition, op_type, + *hwc_tensor, attr.runtime_tensor_is_second, + result); + } + return absl::UnimplementedError( + "No elementwise implementation for this case"); +} + GPUOperation CreateElementwiseTwoInput(const OperationDef& definition, const OperationType& op_type, const BHWC& shape) { @@ -250,7 +296,8 @@ GPUOperation CreateElementwiseTwoInput(const OperationDef& definition, op.code_ += " second_val.z = second_val.x;\n"; op.code_ += " second_val.w = second_val.x;\n"; } - op.code_ += GetTwoInputCode(op_type, "in_out_value", "second_val"); + op.code_ += GetTwoInputCode(op_type, "in_out_value", "in_out_value", + "second_val", false); return op; } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.h b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.h index d03d535b39a..f841cdba9fb 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.h @@ -31,27 +31,13 @@ namespace cl { GPUOperation CreateElementwiseOneInput(const OperationDef& definition, const OperationType& op_type); -// Creates simple two input (first input is runtime tensor and second input is -// scalar argument) operation, for example sub, div, pow, etc. -GPUOperation CreateElementwiseOneRuntimeOneScalar( - const CreationContext& creation_context, const OperationDef& definition, - const OperationType& op_type, float scalar_parameter); - // Creates simple two input(first input is runtime tensor and second input is -// constant linear tensor) operation, for example sub, div and etc. -absl::Status CreateElementwiseTwoInput( - const CreationContext& creation_context, const OperationDef& definition, - const OperationType& op_type, - const tflite::gpu::Tensor& constant_tensor, - GPUOperation* result); - -// Creates simple two input(first input is runtime tensor and second input is -// constant HWC tensor) operation, for example sub, div and etc. -absl::Status CreateElementwiseTwoInput( - const CreationContext& creation_context, const OperationDef& definition, - const OperationType& op_type, - const tflite::gpu::Tensor& constant_tensor, - GPUOperation* result); +// constant or linear/hwc tensor) operation, for example sub, div and etc. +absl::Status CreateElementwise(const CreationContext& creation_context, + const OperationDef& definition, + const OperationType& op_type, + const ElementwiseAttributes& attr, + GPUOperation* result); // Creates simple two input(2 runtime tensors) operation, for example // sub, div and etc. diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise_test.cc index 11a651df901..23ee6622e8c 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise_test.cc @@ -546,9 +546,9 @@ TEST_F(OpenCLOperationTest, MaximumWithScalar) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - const float* scalar = absl::get_if(&attr.param); - GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( - creation_context_, op_def, OperationType::MAXIMUM, *scalar); + GPUOperation operation; + ASSERT_OK(CreateElementwise(creation_context_, op_def, + OperationType::MAXIMUM, attr, &operation)); ASSERT_OK(ExecuteGPUOperation(src_tensor_0, creation_context_, &operation, BHWC(1, 4, 1, 1), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -578,9 +578,8 @@ TEST_F(OpenCLOperationTest, MaximumWithConstantLinearTensor) { op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; GPUOperation operation; - ASSERT_OK(CreateElementwiseTwoInput(creation_context_, op_def, - OperationType::MAXIMUM, linear_tensor, - &operation)); + ASSERT_OK(CreateElementwise(creation_context_, op_def, + OperationType::MAXIMUM, attr, &operation)); ASSERT_OK(ExecuteGPUOperation(src_tensor_0, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -597,6 +596,8 @@ TEST_F(OpenCLOperationTest, MaximumWithConstantHWCTensor) { ::tflite::gpu::Tensor hwc_tensor; hwc_tensor.shape = HWC(2, 1, 2); hwc_tensor.data = {0.5f, 2.0f, 0.7f, 4.7f}; + ElementwiseAttributes attr; + attr.param = hwc_tensor; for (auto storage : env_.GetSupportedStorages()) { for (auto precision : env_.GetSupportedPrecisions()) { @@ -608,9 +609,8 @@ TEST_F(OpenCLOperationTest, MaximumWithConstantHWCTensor) { op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; GPUOperation operation; - ASSERT_OK(CreateElementwiseTwoInput(creation_context_, op_def, - OperationType::MAXIMUM, hwc_tensor, - &operation)); + ASSERT_OK(CreateElementwise(creation_context_, op_def, + OperationType::MAXIMUM, attr, &operation)); ASSERT_OK(ExecuteGPUOperation(src_tensor_0, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -626,6 +626,8 @@ TEST_F(OpenCLOperationTest, MaximumWithConstantHWCTensorBroadcastChannels) { ::tflite::gpu::Tensor hwc_tensor; hwc_tensor.shape = HWC(2, 1, 1); hwc_tensor.data = {0.5f, 2.0f}; + ElementwiseAttributes attr; + attr.param = hwc_tensor; for (auto storage : env_.GetSupportedStorages()) { for (auto precision : env_.GetSupportedPrecisions()) { @@ -637,9 +639,8 @@ TEST_F(OpenCLOperationTest, MaximumWithConstantHWCTensorBroadcastChannels) { op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; GPUOperation operation; - ASSERT_OK(CreateElementwiseTwoInput(creation_context_, op_def, - OperationType::MAXIMUM, hwc_tensor, - &operation)); + ASSERT_OK(CreateElementwise(creation_context_, op_def, + OperationType::MAXIMUM, attr, &operation)); ASSERT_OK(ExecuteGPUOperation(src_tensor_0, creation_context_, &operation, BHWC(1, 2, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -693,9 +694,9 @@ TEST_F(OpenCLOperationTest, MinimumWithScalar) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - const float* scalar = absl::get_if(&attr.param); - GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( - creation_context_, op_def, OperationType::MINIMUM, *scalar); + GPUOperation operation; + ASSERT_OK(CreateElementwise(creation_context_, op_def, + OperationType::MINIMUM, attr, &operation)); ASSERT_OK(ExecuteGPUOperation(src_tensor_0, creation_context_, &operation, BHWC(1, 4, 1, 1), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -788,6 +789,35 @@ TEST_F(OpenCLOperationTest, MulBroadcastChannels) { } } +TEST_F(OpenCLOperationTest, SubWithScalarAtFirstPosition) { + TensorFloat32 src_tensor_0; + src_tensor_0.shape = BHWC(1, 4, 1, 1); + src_tensor_0.data = {0.0f, -6.2f, 2.0f, -3.0f}; + + ElementwiseAttributes attr; + attr.param = 4.0f; + attr.runtime_tensor_is_second = true; + + for (auto storage : env_.GetSupportedStorages()) { + for (auto precision : env_.GetSupportedPrecisions()) { + const float eps = precision == CalculationsPrecision::F32 ? 1e-6f : 1e-2f; + OperationDef op_def; + op_def.precision = precision; + auto data_type = DeduceDataTypeFromPrecision(precision); + op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); + op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); + TensorFloat32 dst_tensor; + GPUOperation operation; + ASSERT_OK(CreateElementwise(creation_context_, op_def, OperationType::SUB, + attr, &operation)); + ASSERT_OK(ExecuteGPUOperation(src_tensor_0, creation_context_, &operation, + BHWC(1, 4, 1, 1), &dst_tensor)); + EXPECT_THAT(dst_tensor.data, + Pointwise(FloatNear(eps), {4.0f, 10.2f, 2.0f, 7.0f})); + } + } +} + } // namespace } // namespace cl } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc index f60af5f730d..e1225e83e95 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc @@ -159,31 +159,11 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, } else if (inputs.size() == 1 && node.operation.attributes.has_value()) { auto attr = absl::any_cast(node.operation.attributes); - const float* scalar = absl::get_if(&attr.param); - const auto* linear_tensor = - absl::get_if>( - &attr.param); - const auto* hwc_tensor = - absl::get_if>( - &attr.param); - if (scalar) { - GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( - creation_context, op_def, op_type, *scalar); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } else if (linear_tensor) { - GPUOperation operation; - RETURN_IF_ERROR(CreateElementwiseTwoInput( - creation_context, op_def, op_type, *linear_tensor, &operation)); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } else if (hwc_tensor) { - GPUOperation operation; - RETURN_IF_ERROR(CreateElementwiseTwoInput( - creation_context, op_def, op_type, *hwc_tensor, &operation)); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } + GPUOperation operation; + RETURN_IF_ERROR(CreateElementwise(creation_context, op_def, op_type, + attr, &operation)); + *gpu_op = absl::make_unique(std::move(operation)); + return absl::OkStatus(); } return absl::UnimplementedError(absl::StrCat( "No support of ", node.operation.type, " with this parameters")); @@ -289,44 +269,6 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, absl::make_unique(std::move(operation)); return absl::OkStatus(); } - case OperationType::MUL: { - if (inputs.size() == 2) { - GPUOperation operation = - CreateElementwiseTwoInput(op_def, op_type, inputs[1]->tensor.shape); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } else if (inputs.size() == 1 && node.operation.attributes.has_value()) { - auto attr = - absl::any_cast(node.operation.attributes); - const float* scalar = absl::get_if(&attr.param); - const auto* linear_tensor = - absl::get_if>( - &attr.param); - const auto* hwc_tensor = - absl::get_if>( - &attr.param); - if (scalar) { - GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( - creation_context, op_def, op_type, *scalar); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } else if (linear_tensor) { - GPUOperation operation; - RETURN_IF_ERROR(CreateElementwiseTwoInput( - creation_context, op_def, op_type, *linear_tensor, &operation)); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } else if (hwc_tensor) { - GPUOperation operation; - RETURN_IF_ERROR(CreateElementwiseTwoInput( - creation_context, op_def, op_type, *hwc_tensor, &operation)); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } - } - return absl::UnimplementedError(absl::StrCat( - "No support of ", node.operation.type, " with this parameters")); - } case OperationType::PAD: { auto attr = absl::any_cast(node.operation.attributes); SelectPadding(attr, op_def, gpu_op); @@ -404,6 +346,7 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, case OperationType::DIV: case OperationType::MAXIMUM: case OperationType::MINIMUM: + case OperationType::MUL: case OperationType::POW: case OperationType::SQUARED_DIFF: case OperationType::SUB: { @@ -415,31 +358,11 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, } else if (inputs.size() == 1 && node.operation.attributes.has_value()) { auto attr = absl::any_cast(node.operation.attributes); - const float* scalar = absl::get_if(&attr.param); - const auto* linear_tensor = - absl::get_if>( - &attr.param); - const auto* hwc_tensor = - absl::get_if>( - &attr.param); - if (scalar) { - GPUOperation operation = CreateElementwiseOneRuntimeOneScalar( - creation_context, op_def, op_type, *scalar); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } else if (linear_tensor) { - GPUOperation operation; - RETURN_IF_ERROR(CreateElementwiseTwoInput( - creation_context, op_def, op_type, *linear_tensor, &operation)); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } else if (hwc_tensor) { - GPUOperation operation; - RETURN_IF_ERROR(CreateElementwiseTwoInput( - creation_context, op_def, op_type, *hwc_tensor, &operation)); - *gpu_op = absl::make_unique(std::move(operation)); - return absl::OkStatus(); - } + GPUOperation operation; + RETURN_IF_ERROR(CreateElementwise(creation_context, op_def, op_type, + attr, &operation)); + *gpu_op = absl::make_unique(std::move(operation)); + return absl::OkStatus(); } return absl::UnimplementedError(absl::StrCat( "No support of ", node.operation.type, " with this parameters")); diff --git a/tensorflow/lite/delegates/gpu/common/model_builder.cc b/tensorflow/lite/delegates/gpu/common/model_builder.cc index bf24e0d9eff..4c0fd827834 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder.cc +++ b/tensorflow/lite/delegates/gpu/common/model_builder.cc @@ -847,6 +847,8 @@ class ElementwiseOperationParser : public TFLiteOperationParser { /*outputs=*/1)); ElementwiseAttributes attr; RETURN_IF_ERROR(ParseInputsWithConstTensor(node, reader, &attr.param)); + attr.runtime_tensor_is_second = + IsConstantTensor(reader->GetInputTensor(0)); node->operation.attributes = std::move(attr); } else { return absl::InvalidArgumentError("Incorrect operation type passed"); diff --git a/tensorflow/lite/delegates/gpu/common/operations.h b/tensorflow/lite/delegates/gpu/common/operations.h index 225165589ae..563dbdec96e 100644 --- a/tensorflow/lite/delegates/gpu/common/operations.h +++ b/tensorflow/lite/delegates/gpu/common/operations.h @@ -490,6 +490,10 @@ BHWC CalculateOutputShape(const BHWC& input, const MeanAttributes& attr); struct ElementwiseAttributes { TensorOrScalar param; + // For elementwise operation with 2 inputs op(A, B), runtime_tensor_is_second + // true when runtime tensor is B(on second position). this is important for + // ops that non commutative, for example substract. + bool runtime_tensor_is_second = false; }; struct ReshapeAttributes { From e4592dad255abb37e6ad675d4caa50161aba7e82 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 18:29:31 -0700 Subject: [PATCH 0318/1017] Internal change PiperOrigin-RevId: 324726407 Change-Id: Ie720508eeb1375dfd82d7ac5ef208c5f2e6edb45 --- tensorflow/core/kernels/cwise_op_exp.cc | 4 ++-- tensorflow/python/kernel_tests/cwise_ops_unary_test.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/kernels/cwise_op_exp.cc b/tensorflow/core/kernels/cwise_op_exp.cc index 48b6823cbdc..2b157f0e7a9 100644 --- a/tensorflow/core/kernels/cwise_op_exp.cc +++ b/tensorflow/core/kernels/cwise_op_exp.cc @@ -16,8 +16,8 @@ limitations under the License. #include "tensorflow/core/kernels/cwise_ops_common.h" namespace tensorflow { -REGISTER6(UnaryOp, CPU, "Exp", functor::exp, float, Eigen::half, double, - bfloat16, complex64, complex128); +REGISTER5(UnaryOp, CPU, "Exp", functor::exp, float, Eigen::half, double, + complex64, complex128); #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER5(UnaryOp, GPU, "Exp", functor::exp, float, Eigen::half, double, diff --git a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py index 368f3509dc6..df848a653d4 100644 --- a/tensorflow/python/kernel_tests/cwise_ops_unary_test.py +++ b/tensorflow/python/kernel_tests/cwise_ops_unary_test.py @@ -389,7 +389,6 @@ class UnaryOpTest(test.TestCase): 2).reshape(1, 3, 2).astype(dtypes_lib.bfloat16.as_numpy_dtype) self._compareCpu(x, np.abs, math_ops.abs) self._compareCpu(x, np.abs, _ABS) - self._compareCpu(x, np.exp, math_ops.exp) self._compareBoth(x, np.negative, math_ops.negative) self._compareBoth(x, np.negative, _NEG) From 1277f67514466d3d571071d9c6b6f0119677ccc2 Mon Sep 17 00:00:00 2001 From: Meghna Natraj Date: Mon, 3 Aug 2020 18:37:33 -0700 Subject: [PATCH 0319/1017] Refactor and Fix lint errors in util.py and lite*.py files PiperOrigin-RevId: 324727472 Change-Id: I3766b0724564f91216bffcc8b55f70744fd94334 --- tensorflow/lite/python/lite.py | 11 +- tensorflow/lite/python/lite_test.py | 1031 +++++++++---------- tensorflow/lite/python/lite_v2_test.py | 148 +-- tensorflow/lite/python/lite_v2_test_util.py | 1 + tensorflow/lite/python/util.py | 11 +- tensorflow/lite/python/util_test.py | 70 +- 6 files changed, 607 insertions(+), 665 deletions(-) diff --git a/tensorflow/lite/python/lite.py b/tensorflow/lite/python/lite.py index a853cc953fd..56397110e5b 100644 --- a/tensorflow/lite/python/lite.py +++ b/tensorflow/lite/python/lite.py @@ -125,7 +125,7 @@ class Optimize(enum.Enum): OPTIMIZE_FOR_LATENCY = "OPTIMIZE_FOR_LATENCY" def __str__(self): - return self.value + return str(self.value) @_tf_export("lite.RepresentativeDataset") @@ -230,7 +230,7 @@ class QuantizationMode(object): def post_training_int16x8_allow_float(self): """Post training int16x8 quantize, allow float fallback.""" - return (self._is_int16x8_target_required() and self._is_allow_float()) + return self._is_int16x8_target_required() and self._is_allow_float() def post_training_dynamic_range_int8(self): """Post training int8 const, on-the-fly int8 quantize of dynamic tensors.""" @@ -907,7 +907,7 @@ class TFLiteFrozenGraphConverterV2(TFLiteConverterBaseV2): """ # TODO(b/130297984): Add support for converting multiple function. - if len(self._funcs) == 0: + if len(self._funcs) == 0: # pylint: disable=g-explicit-length-test raise ValueError("No ConcreteFunction is specified.") if len(self._funcs) > 1: @@ -1127,7 +1127,7 @@ class TFLiteConverterBaseV1(TFLiteConverterBase): parameter is ignored. (default tf.float32) inference_input_type: Target data type of real-number input arrays. Allows for a different type for input arrays. If an integer type is provided and - `optimizations` are not used, `quantized_inputs_stats` must be provided. + `optimizations` are not used, `quantized_input_stats` must be provided. If `inference_type` is tf.uint8, signaling conversion to a fully quantized model from a quantization-aware trained input model, then `inference_input_type` defaults to tf.uint8. In all other cases, @@ -1681,7 +1681,7 @@ class TFLiteConverter(TFLiteFrozenGraphConverter): inference_input_type: Target data type of real-number input arrays. Allows for a different type for input arrays. If an integer type is provided and `optimizations` are not used, - `quantized_inputs_stats` must be provided. + `quantized_input_stats` must be provided. If `inference_type` is tf.uint8, signaling conversion to a fully quantized model from a quantization-aware trained input model, then `inference_input_type` defaults to tf.uint8. @@ -2012,6 +2012,7 @@ class TFLiteConverter(TFLiteFrozenGraphConverter): """ return super(TFLiteConverter, self).convert() + @_tf_export(v1=["lite.TocoConverter"]) class TocoConverter(object): """Convert a TensorFlow model into `output_format` using TOCO. diff --git a/tensorflow/lite/python/lite_test.py b/tensorflow/lite/python/lite_test.py index e9853c7f17c..d17fc94cd20 100644 --- a/tensorflow/lite/python/lite_test.py +++ b/tensorflow/lite/python/lite_test.py @@ -114,7 +114,7 @@ class FromConstructor(TestModels): class FromSessionTest(TestModels, parameterized.TestCase): - def testFloat(self): + def testFloatModel(self): with ops.Graph().as_default(): in_tensor = array_ops.placeholder( shape=[1, 16, 16, 3], dtype=dtypes.float32) @@ -125,130 +125,27 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('add', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) - def testForgottenCallToAllocateTensors(self): - with ops.Graph().as_default(): - in_tensor = array_ops.placeholder( - shape=[1, 16, 16, 3], dtype=dtypes.float32) - out_tensor = in_tensor + in_tensor - sess = session.Session() - # Convert model and ensure model is not None. - converter = lite.TFLiteConverter.from_session(sess, [in_tensor], - [out_tensor]) - tflite_model = converter.convert() - self.assertTrue(tflite_model) - - # Check values from converted model. - interpreter = Interpreter(model_content=tflite_model) - input_index = interpreter.get_input_details()[0]['index'] - dummy_tensor = np.ones(shape=[1, 16, 16, 3], dtype=np.float32) - with self.assertRaises(ValueError): - interpreter.set_tensor(input_index, dummy_tensor) - - @parameterized.named_parameters( - ('EnableMlirConverter', True), # enable mlir - ('DisableMlirConverter', False)) # disable mlir - def testString(self, enable_mlir): - with ops.Graph().as_default(): - in_tensor = array_ops.placeholder(shape=[4], dtype=dtypes.string) - out_tensor = array_ops.reshape(in_tensor, shape=[2, 2]) - sess = session.Session() - - # Convert model and ensure model is not None. - converter = lite.TFLiteConverter.from_session(sess, [in_tensor], - [out_tensor]) - converter.experimental_new_converter = enable_mlir - tflite_model = converter.convert() - self.assertTrue(tflite_model) - - # Check values from converted model. - interpreter = Interpreter(model_content=tflite_model) - interpreter.allocate_tensors() - - input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) - self.assertEqual('Placeholder', input_details[0]['name']) - self.assertEqual(np.string_, input_details[0]['dtype']) - self.assertTrue(([4] == input_details[0]['shape']).all()) - - output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) - self.assertEqual('Reshape', output_details[0]['name']) - self.assertEqual(np.string_, output_details[0]['dtype']) - self.assertTrue(([2, 2] == output_details[0]['shape']).all()) - # TODO(b/122659643): Test setting/getting string data via the python - # interpreter API after support has been added. - - @parameterized.named_parameters( - ('EnableMlirConverter', True), # enable mlir - ('DisableMlirConverter', False)) # disable mlir - def testQuantization(self, enable_mlir): - with ops.Graph().as_default(): - in_tensor_1 = array_ops.placeholder( - shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputA') - in_tensor_2 = array_ops.placeholder( - shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputB') - out_tensor = array_ops.fake_quant_with_min_max_args( - in_tensor_1 + in_tensor_2, min=0., max=1., name='output') - sess = session.Session() - - # Convert model and ensure model is not None. - converter = lite.TFLiteConverter.from_session(sess, - [in_tensor_1, in_tensor_2], - [out_tensor]) - converter.inference_type = lite_constants.QUANTIZED_UINT8 - converter.quantized_input_stats = { - 'inputA': (0., 1.), - 'inputB': (0., 1.) - } # mean, std_dev - converter.experimental_new_converter = enable_mlir - tflite_model = converter.convert() - self.assertTrue(tflite_model) - - # Check values from converted model. - interpreter = Interpreter(model_content=tflite_model) - interpreter.allocate_tensors() - - input_details = interpreter.get_input_details() - self.assertEqual(2, len(input_details)) - self.assertEqual('inputA', input_details[0]['name']) - self.assertEqual(np.uint8, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) - self.assertEqual((1., 0.), - input_details[0]['quantization']) # scale, zero_point - - self.assertEqual('inputB', input_details[1]['name']) - self.assertEqual(np.uint8, input_details[1]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[1]['shape']).all()) - self.assertEqual((1., 0.), - input_details[1]['quantization']) # scale, zero_point - - output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) - self.assertEqual(np.uint8, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) - self.assertTrue(output_details[0]['quantization'][0] > 0) # scale - - def testQuantizedInput(self): + def testFloatModelQuantizedInput(self): with ops.Graph().as_default(): in_tensor = array_ops.placeholder( shape=[1, 16, 16, 3], dtype=dtypes.float32) @@ -262,7 +159,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter.inference_type = lite_constants.FLOAT converter.quantized_input_stats = {'Placeholder': (0., 1.)} # mean, std_dev tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) @@ -272,38 +169,68 @@ class FromSessionTest(TestModels, parameterized.TestCase): self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.uint8, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) - self.assertEqual((1., 0.), - input_details[0]['quantization']) # scale, zero_point + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) + self.assertEqual((1., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() self.assertLen(output_details, 1) self.assertEqual('add', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) # float - def testQuantizationInvalid(self): + def testForgottenCallToAllocateTensors(self): with ops.Graph().as_default(): - in_tensor_1 = array_ops.placeholder( - shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputA') - in_tensor_2 = array_ops.placeholder( - shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputB') - out_tensor = array_ops.fake_quant_with_min_max_args( - in_tensor_1 + in_tensor_2, min=0., max=1., name='output') + in_tensor = array_ops.placeholder( + shape=[1, 16, 16, 3], dtype=dtypes.float32) + out_tensor = in_tensor + in_tensor + sess = session.Session() + # Convert model and ensure model is not None. + converter = lite.TFLiteConverter.from_session(sess, [in_tensor], + [out_tensor]) + tflite_model = converter.convert() + self.assertIsNotNone(tflite_model) + + # Check values from converted model. + interpreter = Interpreter(model_content=tflite_model) + input_index = interpreter.get_input_details()[0]['index'] + dummy_tensor = np.ones(shape=[1, 16, 16, 3], dtype=np.float32) + with self.assertRaises(ValueError): + interpreter.set_tensor(input_index, dummy_tensor) + + @parameterized.named_parameters( + ('EnableMlirConverter', True), # enable mlir + ('DisableMlirConverter', False)) # disable mlir + def testString(self, enable_mlir_converter): + with ops.Graph().as_default(): + in_tensor = array_ops.placeholder(shape=[4], dtype=dtypes.string) + out_tensor = array_ops.reshape(in_tensor, shape=[2, 2]) sess = session.Session() # Convert model and ensure model is not None. - converter = lite.TFLiteConverter.from_session(sess, - [in_tensor_1, in_tensor_2], + converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) - converter.inference_type = lite_constants.QUANTIZED_UINT8 - converter.quantized_input_stats = {'inputA': (0., 1.)} # mean, std_dev - with self.assertRaises(ValueError) as error: - converter.convert() - self.assertEqual( - 'Quantization input stats are not available for input tensors ' - '\'inputB\'.', str(error.exception)) + converter.experimental_new_converter = enable_mlir_converter + tflite_model = converter.convert() + self.assertIsNotNone(tflite_model) + + # Check values from converted model. + interpreter = Interpreter(model_content=tflite_model) + interpreter.allocate_tensors() + + input_details = interpreter.get_input_details() + self.assertLen(input_details, 1) + self.assertEqual('Placeholder', input_details[0]['name']) + self.assertEqual(np.string_, input_details[0]['dtype']) + self.assertAllEqual([4], input_details[0]['shape']) + + output_details = interpreter.get_output_details() + self.assertLen(output_details, 1) + self.assertEqual('Reshape', output_details[0]['name']) + self.assertEqual(np.string_, output_details[0]['dtype']) + self.assertAllEqual([2, 2], output_details[0]['shape']) + # TODO(b/122659643): Test setting/getting string data via the python + # interpreter API after support has been added. def testIntermediateInputArray(self): """Convert a model from an intermediate input array.""" @@ -318,24 +245,24 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_session(sess, [in_tensor_final], [out_tensor]) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('add', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('add_1', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) def testSizeNoneInvalid(self): @@ -357,7 +284,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir - def testScalarValid(self, enable_mlir): + def testScalarValid(self, enable_mlir_converter): # Construct a graph using a scalar (empty shape) input. with ops.Graph().as_default(): in_tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[]) @@ -367,25 +294,25 @@ class FromSessionTest(TestModels, parameterized.TestCase): # Test conversion with the scalar input shape. converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) - converter.experimental_new_converter = enable_mlir + converter.experimental_new_converter = enable_mlir_converter tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([] == input_details[0]['shape']).all()) + self.assertEmpty(input_details[0]['shape']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('add', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([] == input_details[0]['shape']).all()) + self.assertEmpty(input_details[0]['shape']) # Validate inference using the scalar inputs/outputs. test_input = np.array(4.0, dtype=np.float32) @@ -394,7 +321,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): interpreter.invoke() output_data = interpreter.get_tensor(output_details[0]['index']) - self.assertTrue((expected_output == output_data).all()) + self.assertEqual(expected_output, output_data) def testSizeInvalid(self): with ops.Graph().as_default(): @@ -433,9 +360,8 @@ class FromSessionTest(TestModels, parameterized.TestCase): self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 1, 16, 3] == input_details[0]['shape']).all()) - self.assertTrue(([1, -1, 16, - 3] == input_details[0]['shape_signature']).all()) + self.assertAllEqual([1, 1, 16, 3], input_details[0]['shape']) + self.assertAllEqual([1, -1, 16, 3], input_details[0]['shape_signature']) self.assertEqual((0., 0.), input_details[0]['quantization']) # Resize tensor with strict checking. @@ -452,13 +378,11 @@ class FromSessionTest(TestModels, parameterized.TestCase): input_details = interpreter.get_input_details() self.assertLen(input_details, 1) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) - self.assertTrue(([1, -1, 16, - 3] == input_details[0]['shape_signature']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) + self.assertAllEqual([1, -1, 16, 3], input_details[0]['shape_signature']) output_details = interpreter.get_output_details() - self.assertTrue(([1, -1, 16, - 3] == output_details[0]['shape_signature']).all()) + self.assertAllEqual([1, -1, 16, 3], output_details[0]['shape_signature']) def testResizeTensorInputStrict(self): # Ensures that resize_tensor_input(strict=True) works as expected. @@ -472,7 +396,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) @@ -499,24 +423,24 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('add', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) def testBatchSizeNonZero(self): @@ -533,7 +457,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): [in_tensor_1, in_tensor_2], [out_tensor]) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) @@ -542,9 +466,9 @@ class FromSessionTest(TestModels, parameterized.TestCase): input_details = interpreter.get_input_details() self.assertLen(input_details, 2) self.assertEqual('input1', input_details[0]['name']) - self.assertTrue(([1, 4] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 4], input_details[0]['shape']) self.assertEqual('input2', input_details[1]['name']) - self.assertTrue(([4, 10] == input_details[1]['shape']).all()) + self.assertAllEqual([4, 10], input_details[1]['shape']) def testFreezeGraph(self): with ops.Graph().as_default(): @@ -562,24 +486,24 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('top_k:1', output_details[0]['name']) self.assertEqual(np.int32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 1] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 1], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) def testGraphviz(self): @@ -594,12 +518,12 @@ class FromSessionTest(TestModels, parameterized.TestCase): [out_tensor]) converter.output_format = lite_constants.GRAPHVIZ_DOT graphviz_output = converter.convert() - self.assertTrue(graphviz_output) + self.assertIsNotNone(graphviz_output) @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir - def testDumpGraphviz(self, enable_mlir): + def testDumpGraphviz(self, enable_mlir_converter): with ops.Graph().as_default(): in_tensor = array_ops.placeholder( shape=[1, 16, 16, 3], dtype=dtypes.float32) @@ -609,35 +533,35 @@ class FromSessionTest(TestModels, parameterized.TestCase): # Convert model and ensure model is not None. converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) - converter.experimental_new_converter = enable_mlir + converter.experimental_new_converter = enable_mlir_converter graphviz_dir = self.get_temp_dir() converter.dump_graphviz_dir = graphviz_dir tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Ensure interpreter is able to allocate and check graphviz data. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() num_items_graphviz = len(os.listdir(graphviz_dir)) - self.assertTrue(num_items_graphviz) - self.assertTrue( + self.assertIsNotNone(num_items_graphviz) + self.assertIsNotNone( os.path.exists(os.path.join(graphviz_dir, 'toco_AT_IMPORT.dot'))) - self.assertTrue( + self.assertIsNotNone( os.path.exists( os.path.join(graphviz_dir, 'toco_AFTER_TRANSFORMATIONS.dot'))) # new converter doesn't support `dump_graphviz_video` flag - if not enable_mlir: + if not enable_mlir_converter: # Convert model and ensure model is not None. converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) - converter.experimental_new_converter = enable_mlir + converter.experimental_new_converter = enable_mlir_converter graphviz_dir = self.get_temp_dir() converter.dump_graphviz_dir = graphviz_dir converter.dump_graphviz_video = True tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Ensure graphviz folder has more data after using video flag. num_items_graphviz_video = len(os.listdir(graphviz_dir)) @@ -656,10 +580,9 @@ class FromSessionTest(TestModels, parameterized.TestCase): log_dir = self.get_temp_dir() converter.conversion_summary_dir = log_dir tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) - num_items_conversion_summary = len(os.listdir(log_dir)) - self.assertTrue(num_items_conversion_summary) + self.assertNotEmpty(os.listdir(log_dir)) def testDumpConversionSummaryWithOldConverter(self): with ops.Graph().as_default(): @@ -675,7 +598,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): log_dir = self.get_temp_dir() converter.conversion_summary_dir = log_dir tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check nothing is generated under the conversion summary path. num_items_conversion_summary = len(os.listdir(log_dir)) self.assertEqual(num_items_conversion_summary, 0) @@ -683,104 +606,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir - def testInferenceInputType(self, enable_mlir): - with ops.Graph().as_default(): - in_tensor = array_ops.placeholder( - shape=[1, 16, 16, 3], dtype=dtypes.float32) - out_tensor = in_tensor + in_tensor - sess = session.Session() - - # Convert model and ensure model is not None. - converter = lite.TFLiteConverter.from_session(sess, [in_tensor], - [out_tensor]) - converter.experimental_new_converter = enable_mlir - converter.inference_input_type = lite_constants.QUANTIZED_UINT8 - converter.quantized_input_stats = {'Placeholder': (0., 1.)} # mean, std_dev - tflite_model = converter.convert() - self.assertTrue(tflite_model) - - # Check values from converted model. - interpreter = Interpreter(model_content=tflite_model) - interpreter.allocate_tensors() - - input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) - self.assertEqual('Placeholder', input_details[0]['name']) - self.assertEqual(np.uint8, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) - self.assertEqual((1., 0.), input_details[0]['quantization']) - - output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) - self.assertEqual('add', output_details[0]['name']) - self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) - - def testDefaultRangesStats(self): - with ops.Graph().as_default(): - in_tensor = array_ops.placeholder( - shape=[1, 16, 16, 3], dtype=dtypes.float32) - out_tensor = in_tensor + in_tensor - sess = session.Session() - - # Convert model and ensure model is not None. - converter = lite.TFLiteConverter.from_session(sess, [in_tensor], - [out_tensor]) - converter.inference_type = lite_constants.QUANTIZED_UINT8 - converter.quantized_input_stats = {'Placeholder': (0., 1.)} # mean, std_dev - converter.default_ranges_stats = (0, 6) # min, max - tflite_model = converter.convert() - self.assertTrue(tflite_model) - - # Check values from converted model. - interpreter = Interpreter(model_content=tflite_model) - interpreter.allocate_tensors() - - input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) - self.assertEqual('Placeholder', input_details[0]['name']) - self.assertEqual(np.uint8, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) - self.assertEqual((1., 0.), input_details[0]['quantization']) - - output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) - self.assertEqual('add', output_details[0]['name']) - self.assertEqual(np.uint8, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) - self.assertTrue(output_details[0]['quantization'][0] > 0) # scale - - @parameterized.named_parameters( - ('EnableMlirConverter', True), # enable mlir - ('DisableMlirConverter', False)) # disable mlir - def testPostTrainingQuantizeDeprecatedAttribute(self, enable_mlir): - with ops.Graph().as_default(): - in_tensor_1 = array_ops.placeholder( - shape=[33, 33], dtype=dtypes.float32, name='inputA') - in_tensor_2 = constant_op.constant( - np.random.uniform(low=-10., high=10., size=(33, 33)), - shape=[33, 33], - dtype=dtypes.float32, - name='inputB') - out_tensor = math_ops.matmul(in_tensor_1, in_tensor_2, name='output') - sess = session.Session() - - quantized_converter = lite.TFLiteConverter.from_session( - sess, [in_tensor_1], [out_tensor]) - self.assertFalse(quantized_converter.post_training_quantize) - quantized_converter.experimental_new_converter = enable_mlir - - quantized_converter.post_training_quantize = True - self.assertTrue(quantized_converter.post_training_quantize) - self.assertEqual(quantized_converter.optimizations, [lite.Optimize.DEFAULT]) - - quantized_tflite = quantized_converter.convert() - self.assertTrue(quantized_tflite) - - @parameterized.named_parameters( - ('EnableMlirConverter', True), # enable mlir - ('DisableMlirConverter', False)) # disable mlir - def testPostTrainingQuantize(self, enable_mlir): + def testQuantizeDynamicRange(self, enable_mlir_converter): np.random.seed(0) with ops.Graph().as_default(): # We need the tensor to have more than 1024 elements for quantize_weights @@ -796,26 +622,53 @@ class FromSessionTest(TestModels, parameterized.TestCase): sess = session.Session() # Convert float model. - float_converter = lite.TFLiteConverter.from_session(sess, [in_tensor_1], - [out_tensor]) - float_converter.experimental_new_converter = enable_mlir - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) + float_converter = lite.TFLiteConverter.from_session( + sess, [in_tensor_1], [out_tensor]) + float_converter.experimental_new_converter = enable_mlir_converter + float_tflite_model = float_converter.convert() + self.assertIsNotNone(float_tflite_model) # Convert quantized weights model. quantized_converter = lite.TFLiteConverter.from_session( sess, [in_tensor_1], [out_tensor]) - quantized_converter.experimental_new_converter = enable_mlir quantized_converter.optimizations = [lite.Optimize.DEFAULT] - quantized_converter.experimental_new_converter = enable_mlir - quantized_tflite = quantized_converter.convert() - self.assertTrue(quantized_tflite) + quantized_converter.experimental_new_converter = enable_mlir_converter + quantized_tflite_model = quantized_converter.convert() + self.assertIsNotNone(quantized_tflite_model) # Ensure that the quantized weights tflite model is smaller. - self.assertTrue(len(quantized_tflite) < len(float_tflite)) + self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) - def _getCalibrationQuantizeModel(self): + @parameterized.named_parameters( + ('EnableMlirConverter', True), # enable mlir + ('DisableMlirConverter', False)) # disable mlir + def testQuantizeDynamicRangeDeprecatedPostTrainingQuantizeAttribute( + self, enable_mlir_converter): + with ops.Graph().as_default(): + in_tensor_1 = array_ops.placeholder( + shape=[33, 33], dtype=dtypes.float32, name='inputA') + in_tensor_2 = constant_op.constant( + np.random.uniform(low=-10., high=10., size=(33, 33)), + shape=[33, 33], + dtype=dtypes.float32, + name='inputB') + out_tensor = math_ops.matmul(in_tensor_1, in_tensor_2, name='output') + sess = session.Session() + + quantized_converter = lite.TFLiteConverter.from_session( + sess, [in_tensor_1], [out_tensor]) + self.assertFalse(quantized_converter.post_training_quantize) + quantized_converter.experimental_new_converter = enable_mlir_converter + + quantized_converter.post_training_quantize = True + self.assertTrue(quantized_converter.post_training_quantize) + self.assertEqual(quantized_converter.optimizations, [lite.Optimize.DEFAULT]) + + quantized_tflite_model = quantized_converter.convert() + self.assertIsNotNone(quantized_tflite_model) + + def _getIntegerQuantizeModel(self): np.random.seed(0) inp = array_ops.placeholder( dtype=dtypes.float32, shape=(1, 5, 5, 3), name='input') @@ -835,37 +688,37 @@ class FromSessionTest(TestModels, parameterized.TestCase): @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir - def testPostTrainingCalibrateAndQuantize(self, enable_mlir): + def testQuantizeInt8AllowFloat(self, enable_mlir_converter): with ops.Graph().as_default(): - inp, output, calibration_gen = self._getCalibrationQuantizeModel() + inp, output, calibration_gen = self._getIntegerQuantizeModel() sess = session.Session() # Convert float model. float_converter = lite.TFLiteConverter.from_session(sess, [inp], [output]) - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) + float_tflite_model = float_converter.convert() + self.assertIsNotNone(float_tflite_model) # Convert quantized model. quantized_converter = lite.TFLiteConverter.from_session( sess, [inp], [output]) - quantized_converter.experimental_new_converter = enable_mlir + quantized_converter.experimental_new_converter = enable_mlir_converter quantized_converter.optimizations = [lite.Optimize.DEFAULT] quantized_converter.representative_dataset = calibration_gen - quantized_tflite = quantized_converter.convert() - self.assertTrue(quantized_tflite) + quantized_tflite_model = quantized_converter.convert() + self.assertIsNotNone(quantized_tflite_model) # The default input and output types should be float. - interpreter = Interpreter(model_content=quantized_tflite) + interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual(np.float32, input_details[0]['dtype']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual(np.float32, output_details[0]['dtype']) # Ensure that the quantized weights tflite model is smaller. - self.assertLess(len(quantized_tflite), len(float_tflite)) + self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) @parameterized.named_parameters( # Quantize model to Int8: with enable mlir @@ -883,138 +736,82 @@ class FromSessionTest(TestModels, parameterized.TestCase): [lite.OpsSet.\ EXPERIMENTAL_TFLITE_BUILTINS_ACTIVATIONS_INT16_WEIGHTS_INT8], True)) - def testCalibrateAndQuantizeBuiltinInt(self, supported_ops, enable_mlir): + def testQuantizeInt8And16x8(self, supported_ops, enable_mlir_converter): with ops.Graph().as_default(): - inp, output, calibration_gen = self._getCalibrationQuantizeModel() + inp, output, calibration_gen = self._getIntegerQuantizeModel() sess = session.Session() # Convert float model. float_converter = lite.TFLiteConverter.from_session(sess, [inp], [output]) - float_converter.experimental_new_converter = enable_mlir - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) + float_converter.experimental_new_converter = enable_mlir_converter + float_tflite_model = float_converter.convert() + self.assertIsNotNone(float_tflite_model) # Convert model by specifying target spec (instead of optimizations), since # when targeting an integer only backend, quantization is mandatory. quantized_converter = lite.TFLiteConverter.from_session( sess, [inp], [output]) - quantized_converter.experimental_new_converter = enable_mlir + quantized_converter.experimental_new_converter = enable_mlir_converter quantized_converter.target_spec.supported_ops = supported_ops quantized_converter.representative_dataset = calibration_gen - quantized_tflite = quantized_converter.convert() - self.assertTrue(quantized_tflite) + quantized_tflite_model = quantized_converter.convert() + self.assertIsNotNone(quantized_tflite_model) # The default input and output types should be float. - interpreter = Interpreter(model_content=quantized_tflite) + interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual(np.float32, input_details[0]['dtype']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual(np.float32, output_details[0]['dtype']) # Ensure that the quantized weights tflite model is smaller. - self.assertLess(len(quantized_tflite), len(float_tflite)) + self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) @parameterized.named_parameters( - # Quantize to Float16 even if rep data provided. - ('UseRepresentativeData', True, False, True, False, False, False), - # Quantize to Float16 if no rep data provided. - ('NoRepresentativeData', False, False, True, False, False, False), - # Post training quantization if both rep data and int8 included. - ('UseSampleDataIncludeInt8', True, True, False, False, True, False), - - # Quantize to Float16 even if rep data provided with mlir. - ('UseRepresentativeDataMlir', True, False, True, False, False, True), - # Quantize to Float16 if no rep data provided with mlir. - ('NoRepresentativeDataMlir', False, False, True, False, False, True), - # Post training quantization if both rep data and int8 included with mlir. - ('SampleDataIncludeInt8Mlir', True, True, False, False, True, True)) - def testQuantizeFloat16(self, use_rep_data, include_int8, - is_float16_quantized, is_error, - is_post_training_quantized, enable_mlir): + ('EnableMlirConverter', True), # enable mlir + ('DisableMlirConverter', False)) # disable mlir + def testQuantizeInt8InputOutput(self, enable_mlir_converter): with ops.Graph().as_default(): - inp, output, calibration_gen = self._getCalibrationQuantizeModel() + inp, output, calibration_gen = self._getIntegerQuantizeModel() sess = session.Session() - idx = 1 if enable_mlir else 0 - node_name = 'Conv2D' if enable_mlir else 'Conv2D_bias' # Convert float model. float_converter = lite.TFLiteConverter.from_session(sess, [inp], [output]) - float_converter.experimental_new_converter = enable_mlir - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) - interpreter = Interpreter(model_content=float_tflite) + float_converter.experimental_new_converter = enable_mlir_converter + float_tflite_model = float_converter.convert() + self.assertIsNotNone(float_tflite_model) + + # Convert quantized weights model. + quantized_converter = lite.TFLiteConverter.from_session( + sess, [inp], [output]) + quantized_converter.experimental_new_converter = enable_mlir_converter + quantized_converter.inference_input_type = lite_constants.INT8 + quantized_converter.inference_output_type = lite_constants.INT8 + quantized_converter.optimizations = [lite.Optimize.DEFAULT] + quantized_converter.representative_dataset = calibration_gen + quantized_tflite_model = quantized_converter.convert() + self.assertIsNotNone(quantized_tflite_model) + + # The input and output types should be int8. + interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() - self.assertEqual(interpreter.get_tensor_details()[idx]['name'], node_name) - self.assertEqual(interpreter.get_tensor_details()[idx]['dtype'], - lite.constants.FLOAT) - # Convert model to quantized version - quantized_converter = lite.TFLiteConverter.from_session( - sess, [inp], [output]) - quantized_converter.experimental_new_converter = enable_mlir - quantized_converter.optimizations = [lite.Optimize.DEFAULT] - quantized_converter.target_spec.supported_types = [lite.constants.FLOAT16] - if include_int8: - quantized_converter.target_spec.supported_types.append( - lite.constants.INT8) - if use_rep_data: - quantized_converter.representative_dataset = calibration_gen + input_details = interpreter.get_input_details() + self.assertLen(input_details, 1) + self.assertEqual(np.int8, input_details[0]['dtype']) + output_details = interpreter.get_output_details() + self.assertLen(output_details, 1) + self.assertEqual(np.int8, output_details[0]['dtype']) - if is_error: - with self.assertRaises(ValueError) as error: - quantized_converter.convert() - self.assertEqual( - 'representative_dataset is required when specifying ' - 'TFLITE_BUILTINS_INT8 or INT8 supported types.', str(error.exception)) - - else: - quantized_tflite = quantized_converter.convert() - self.assertTrue(quantized_tflite) - interpreter = Interpreter(model_content=quantized_tflite) - interpreter.allocate_tensors() - self.assertEqual(interpreter.get_tensor_details()[idx]['name'], node_name) - - if is_float16_quantized: - # Verify that bias constant is float16 type. - self.assertEqual(interpreter.get_tensor_details()[idx]['dtype'], - lite.constants.FLOAT16) - elif is_post_training_quantized: - # Verify that bias constants is int32 type. - self.assertEqual(interpreter.get_tensor_details()[idx]['dtype'], - lite.constants.INT32) - else: - raise ValueError('Invalid test options.') + # Ensure that the quantized weights tflite model is smaller. + self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir - def testInvalidQuantizeFloat16(self, enable_mlir): - with ops.Graph().as_default(): - inp, output, _ = self._getCalibrationQuantizeModel() - sess = session.Session() - - # Specify float16 quantization - quantized_converter = lite.TFLiteConverter.from_session( - sess, [inp], [output]) - quantized_converter.experimental_new_converter = enable_mlir - quantized_converter.optimizations = [lite.Optimize.DEFAULT] - quantized_converter.target_spec.supported_types = [lite.constants.FLOAT16] - # Specify only int8 builtin ops - quantized_converter.target_spec.supported_ops = [ - lite.OpsSet.TFLITE_BUILTINS_INT8 - ] - with self.assertRaises(ValueError) as error: - quantized_converter.convert() - self.assertEqual( - 'TFLITE_BUILTINS_INT8 requires smallest supported type to be INT8.', - str(error.exception)) - - @parameterized.named_parameters( - ('EnableMlirConverter', True), # enable mlir - ('DisableMlirConverter', False)) # disable mlir - def testInvalidPostTrainingQuantize(self, enable_mlir): + def testInvalidQuantizeInt8(self, enable_mlir_converter): np.random.seed(0) with ops.Graph().as_default(): # We need the tensor to have more than 1024 elements for quantize_weights @@ -1032,7 +829,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): # Attempt to convert to quantized weights model. quantized_converter = lite.TFLiteConverter.from_session( sess, [in_tensor_1], [out_tensor]) - quantized_converter.experimental_new_converter = enable_mlir + quantized_converter.experimental_new_converter = enable_mlir_converter quantized_converter.optimizations = [lite.Optimize.DEFAULT] # Restricting to int8 type only quantized_converter.target_spec.supported_types = [lite.constants.INT8] @@ -1046,72 +843,183 @@ class FromSessionTest(TestModels, parameterized.TestCase): @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir - def testPostTrainingCalibrateAndQuantizeFloatNotAllowed(self, enable_mlir): + def testQuantizeUInt8(self, enable_mlir_converter): with ops.Graph().as_default(): - inp, output, calibration_gen = self._getCalibrationQuantizeModel() + in_tensor_1 = array_ops.placeholder( + shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputA') + in_tensor_2 = array_ops.placeholder( + shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputB') + out_tensor = array_ops.fake_quant_with_min_max_args( + in_tensor_1 + in_tensor_2, min=0., max=1., name='output') sess = session.Session() + # Convert model and ensure model is not None. + converter = lite.TFLiteConverter.from_session(sess, + [in_tensor_1, in_tensor_2], + [out_tensor]) + converter.inference_type = lite_constants.QUANTIZED_UINT8 + converter.quantized_input_stats = { + 'inputA': (0., 1.), + 'inputB': (0., 1.) + } # mean, std_dev + converter.experimental_new_converter = enable_mlir_converter + tflite_model = converter.convert() + self.assertIsNotNone(tflite_model) + + # Check values from converted model. + interpreter = Interpreter(model_content=tflite_model) + interpreter.allocate_tensors() + + input_details = interpreter.get_input_details() + self.assertLen(input_details, 2) + self.assertEqual('inputA', input_details[0]['name']) + self.assertEqual(np.uint8, input_details[0]['dtype']) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) + self.assertEqual((1., 0.), input_details[0]['quantization']) + + self.assertEqual('inputB', input_details[1]['name']) + self.assertEqual(np.uint8, input_details[1]['dtype']) + self.assertAllEqual([1, 16, 16, 3], input_details[1]['shape']) + self.assertEqual((1., 0.), input_details[1]['quantization']) + + output_details = interpreter.get_output_details() + self.assertLen(output_details, 1) + self.assertEqual(np.uint8, output_details[0]['dtype']) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) + self.assertGreater(output_details[0]['quantization'][0], 0) # scale + + def testQuantizeUInt8UsingDefaultRangeStats(self): + with ops.Graph().as_default(): + in_tensor = array_ops.placeholder( + shape=[1, 16, 16, 3], dtype=dtypes.float32) + out_tensor = in_tensor + in_tensor + sess = session.Session() + + # Convert model and ensure model is not None. + converter = lite.TFLiteConverter.from_session(sess, [in_tensor], + [out_tensor]) + converter.inference_type = lite_constants.QUANTIZED_UINT8 + converter.quantized_input_stats = {'Placeholder': (0., 1.)} # mean, std_dev + converter.default_ranges_stats = (0, 6) # min, max + tflite_model = converter.convert() + self.assertIsNotNone(tflite_model) + + # Check values from converted model. + interpreter = Interpreter(model_content=tflite_model) + interpreter.allocate_tensors() + + input_details = interpreter.get_input_details() + self.assertLen(input_details, 1) + self.assertEqual('Placeholder', input_details[0]['name']) + self.assertEqual(np.uint8, input_details[0]['dtype']) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) + self.assertEqual((1., 0.), input_details[0]['quantization']) + + output_details = interpreter.get_output_details() + self.assertLen(output_details, 1) + self.assertEqual('add', output_details[0]['name']) + self.assertEqual(np.uint8, output_details[0]['dtype']) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) + self.assertGreater(output_details[0]['quantization'][0], 0) # scale + + @parameterized.named_parameters( + # Quantize to Float16 even if rep data provided. + ('UseRepresentativeData', True, False, True, False, False, False), + # Quantize to Float16 if no rep data provided. + ('NoRepresentativeData', False, False, True, False, False, False), + # Post training quantization if both rep data and int8 included. + ('UseSampleDataIncludeInt8', True, True, False, False, True, False), + + # Quantize to Float16 even if rep data provided with mlir. + ('UseRepresentativeDataMlir', True, False, True, False, False, True), + # Quantize to Float16 if no rep data provided with mlir. + ('NoRepresentativeDataMlir', False, False, True, False, False, True), + # Post training quantization if both rep data and int8 included with mlir. + ('SampleDataIncludeInt8Mlir', True, True, False, False, True, True)) + def testQuantizeFloat16(self, use_rep_data, include_int8, + is_float16_quantized, is_error, + is_post_training_quantized, enable_mlir_converter): + with ops.Graph().as_default(): + inp, output, calibration_gen = self._getIntegerQuantizeModel() + sess = session.Session() + + idx = 1 if enable_mlir_converter else 0 + node_name = 'Conv2D' if enable_mlir_converter else 'Conv2D_bias' # Convert float model. float_converter = lite.TFLiteConverter.from_session(sess, [inp], [output]) - float_converter.experimental_new_converter = enable_mlir - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) - - # Convert quantized model. + float_converter.experimental_new_converter = enable_mlir_converter + float_tflite_model = float_converter.convert() + self.assertIsNotNone(float_tflite_model) + interpreter = Interpreter(model_content=float_tflite_model) + interpreter.allocate_tensors() + self.assertEqual(interpreter.get_tensor_details()[idx]['name'], node_name) + self.assertEqual(interpreter.get_tensor_details()[idx]['dtype'], + lite.constants.FLOAT) + # Convert model to quantized version quantized_converter = lite.TFLiteConverter.from_session( sess, [inp], [output]) - quantized_converter.experimental_new_converter = enable_mlir + quantized_converter.experimental_new_converter = enable_mlir_converter quantized_converter.optimizations = [lite.Optimize.DEFAULT] - quantized_converter.representative_dataset = calibration_gen - quantized_converter.target_spec.supported_types = [lite.constants.INT8] - quantized_tflite = quantized_converter.convert() - self.assertTrue(quantized_tflite) + quantized_converter.target_spec.supported_types = [lite.constants.FLOAT16] + if include_int8: + quantized_converter.target_spec.supported_types.append( + lite.constants.INT8) + if use_rep_data: + quantized_converter.representative_dataset = calibration_gen - # Ensure that the quantized weights tflite model is smaller. - self.assertLess(len(quantized_tflite), len(float_tflite)) + if is_error: + with self.assertRaises(ValueError) as error: + quantized_converter.convert() + self.assertEqual( + 'representative_dataset is required when specifying ' + 'TFLITE_BUILTINS_INT8 or INT8 supported types.', str(error.exception)) + + else: + quantized_tflite_model = quantized_converter.convert() + self.assertIsNotNone(quantized_tflite_model) + interpreter = Interpreter(model_content=quantized_tflite_model) + interpreter.allocate_tensors() + self.assertEqual(interpreter.get_tensor_details()[idx]['name'], node_name) + + if is_float16_quantized: + # Verify that bias constant is float16 type. + self.assertEqual(interpreter.get_tensor_details()[idx]['dtype'], + lite.constants.FLOAT16) + elif is_post_training_quantized: + # Verify that bias constants is int32 type. + self.assertEqual(interpreter.get_tensor_details()[idx]['dtype'], + lite.constants.INT32) + else: + raise ValueError('Invalid test options.') @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir - def testPostTrainingCalibrateAndQuantizeInt8Inputs(self, enable_mlir): + def testInvalidQuantizeFloat16(self, enable_mlir_converter): with ops.Graph().as_default(): - inp, output, calibration_gen = self._getCalibrationQuantizeModel() + inp, output, _ = self._getIntegerQuantizeModel() sess = session.Session() - # Convert float model. - float_converter = lite.TFLiteConverter.from_session(sess, [inp], [output]) - float_converter.experimental_new_converter = enable_mlir - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) - - # Convert quantized weights model. + # Specify float16 quantization quantized_converter = lite.TFLiteConverter.from_session( sess, [inp], [output]) - quantized_converter.experimental_new_converter = enable_mlir - quantized_converter.inference_input_type = lite_constants.INT8 - quantized_converter.inference_output_type = lite_constants.INT8 + quantized_converter.experimental_new_converter = enable_mlir_converter quantized_converter.optimizations = [lite.Optimize.DEFAULT] - quantized_converter.representative_dataset = calibration_gen - quantized_tflite = quantized_converter.convert() - self.assertTrue(quantized_tflite) - - # The input and output types should be int8. - interpreter = Interpreter(model_content=quantized_tflite) - interpreter.allocate_tensors() - input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) - self.assertEqual(np.int8, input_details[0]['dtype']) - output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) - self.assertEqual(np.int8, output_details[0]['dtype']) - - # Ensure that the quantized weights tflite model is smaller. - self.assertLess(len(quantized_tflite), len(float_tflite)) + quantized_converter.target_spec.supported_types = [lite.constants.FLOAT16] + # Specify only int8 builtin ops + quantized_converter.target_spec.supported_ops = [ + lite.OpsSet.TFLITE_BUILTINS_INT8 + ] + with self.assertRaises(ValueError) as error: + quantized_converter.convert() + self.assertEqual( + 'TFLITE_BUILTINS_INT8 requires smallest supported type to be INT8.', + str(error.exception)) @parameterized.named_parameters( ('InferenceType_INT8', lite_constants.INT8), - ('InferenceType_QUANTIZED_INT8', lite_constants.QUANTIZED_UINT8)) - def testRequiresInputStatsForTrainingTimeQuantization(self, quantized_type): + ('InferenceType_UINT8', lite_constants.QUANTIZED_UINT8)) + def testInvalidQuantizeQATModelRequiresInputStats(self, quantized_type): with ops.Graph().as_default(): in_tensor = array_ops.placeholder( shape=[1, 16, 16, 3], dtype=dtypes.float32) @@ -1148,15 +1056,37 @@ class FromSessionTest(TestModels, parameterized.TestCase): } quantized_converter.convert() + def testInvalidQuantizeQATModelMissingInputStats(self): + with ops.Graph().as_default(): + in_tensor_1 = array_ops.placeholder( + shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputA') + in_tensor_2 = array_ops.placeholder( + shape=[1, 16, 16, 3], dtype=dtypes.float32, name='inputB') + out_tensor = array_ops.fake_quant_with_min_max_args( + in_tensor_1 + in_tensor_2, min=0., max=1., name='output') + sess = session.Session() + + # Convert model and ensure model is not None. + converter = lite.TFLiteConverter.from_session(sess, + [in_tensor_1, in_tensor_2], + [out_tensor]) + converter.inference_type = lite_constants.QUANTIZED_UINT8 + converter.quantized_input_stats = {'inputA': (0., 1.)} # mean, std_dev + with self.assertRaises(ValueError) as error: + converter.convert() + self.assertEqual( + 'Quantization input stats are not available for input tensors ' + '\'inputB\'.', str(error.exception)) + def testTrainingTimeAndPostTrainingCalibrateAndQuantize(self): with ops.Graph().as_default(): - inp, output, calibration_gen = self._getCalibrationQuantizeModel() + inp, output, calibration_gen = self._getIntegerQuantizeModel() sess = session.Session() # Convert float model. float_converter = lite.TFLiteConverter.from_session(sess, [inp], [output]) - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) + float_tflite_model = float_converter.convert() + self.assertIsNotNone(float_tflite_model) converter = lite.TFLiteConverter.from_session(sess, [inp], [output]) @@ -1172,15 +1102,16 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter.optimizations = [lite.Optimize.DEFAULT] converter.representative_dataset = calibration_gen converter._experimental_new_quantizer = True - quantized_tflite = converter.convert() - self.assertTrue(quantized_tflite) - self.assertLess(len(quantized_tflite), len(float_tflite)) + quantized_tflite_model = converter.convert() + self.assertIsNotNone(quantized_tflite_model) + self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) # calibration only api converter._experimental_calibrate_only = True calibrated_tflite = converter.convert() - quantized_tflite = mlir_quantize(calibrated_tflite, fully_quantize=True) - interpreter = Interpreter(model_content=quantized_tflite) + quantized_tflite_model = mlir_quantize( + calibrated_tflite, fully_quantize=True) + interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() self.assertEqual(np.int8, input_details[0]['dtype']) @@ -1200,7 +1131,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): # Convert model and ensure model is not None. converter = lite.TocoConverter.from_session(sess, [in_tensor], [out_tensor]) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Ensure the interpreter is able to load. interpreter = Interpreter(model_content=tflite_model) @@ -1218,20 +1149,20 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_session(sess, [input_tensor], [out0, out1, out2, out3]) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) interpreter.set_tensor(input_details[0]['index'], np.asarray([1.0, 2.0, 3.0, 4.0], dtype=np.float32)) interpreter.invoke() output_details = interpreter.get_output_details() - self.assertEqual(4, len(output_details)) + self.assertLen(output_details, 4) self.assertEqual(1.0, interpreter.get_tensor(output_details[0]['index'])) self.assertEqual(2.0, interpreter.get_tensor(output_details[1]['index'])) self.assertEqual(3.0, interpreter.get_tensor(output_details[2]['index'])) @@ -1241,7 +1172,7 @@ class FromSessionTest(TestModels, parameterized.TestCase): ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir @test_util.run_in_graph_and_eager_modes - def testFunctions(self, enable_mlir): + def testFunctions(self, enable_mlir_converter): """Tests tf.function in 1.X.""" @def_function.function @@ -1262,26 +1193,26 @@ class FromSessionTest(TestModels, parameterized.TestCase): # Convert model and ensure model is not None. converter = lite.TFLiteConverter.from_session(sess, [placeholder], [output_node]) - converter.experimental_new_converter = enable_mlir + converter.experimental_new_converter = enable_mlir_converter tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('input', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1] == input_details[0]['shape']).all()) + self.assertAllEqual([1], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('output_node', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1] == output_details[0]['shape']).all()) + self.assertAllEqual([1], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) def testInferenceInputOutputTypeFloatDefault(self): @@ -1295,23 +1226,23 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_session(sess, [in_tensor], [out_tensor]) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('add', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) def testInferenceInputOutputTypeQuantizedUint8Default(self): with ops.Graph().as_default(): @@ -1327,23 +1258,23 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter.inference_type = lite_constants.QUANTIZED_UINT8 converter.quantized_input_stats = {'Placeholder': (0., 1.)} # mean, std_dev tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.uint8, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('output', output_details[0]['name']) self.assertEqual(np.uint8, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) def testReusingConverterWithDifferentPostTrainingQuantization(self): with ops.Graph().as_default(): @@ -1359,11 +1290,11 @@ class FromSessionTest(TestModels, parameterized.TestCase): converter.post_training_quantize = True tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) converter.post_training_quantize = False tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) def testResizeWithShape(self): with ops.Graph().as_default(): @@ -1383,8 +1314,8 @@ class FromSessionTest(TestModels, parameterized.TestCase): interpreter = Interpreter(model_content=tflite_model) input_details = interpreter.get_input_details() self.assertLen(input_details, 1) - self.assertTrue(([1, 1] == input_details[0]['shape']).all()) - self.assertTrue(([-1, -1] == input_details[0]['shape_signature']).all()) + self.assertAllEqual([1, 1], input_details[0]['shape']) + self.assertAllEqual([-1, -1], input_details[0]['shape_signature']) # Resize tensor and invoke. interpreter.resize_tensor_input(0, [4]) @@ -1395,9 +1326,9 @@ class FromSessionTest(TestModels, parameterized.TestCase): output_details = interpreter.get_output_details() self.assertLen(output_details, 1) self.assertEqual(np.int32, output_details[0]['dtype']) - self.assertTrue(([4] == output_details[0]['shape']).all()) + self.assertAllEqual([4], output_details[0]['shape']) output_data = interpreter.get_tensor(output_details[0]['index']) - self.assertTrue(([1, 2, 3, 4] == output_data).all()) + self.assertAllEqual([1, 2, 3, 4], output_data) def testResizingIntermediateDynamicTensor(self): # This is a regression test for the case where shape of dynamic output @@ -1479,24 +1410,24 @@ class FromFrozenGraphFile(LiteTest): converter = lite.TFLiteConverter.from_frozen_graph(graph_def_file, ['Placeholder'], ['add']) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('add', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) def testFloatWithShapesArray(self): @@ -1516,15 +1447,15 @@ class FromFrozenGraphFile(LiteTest): graph_def_file, ['Placeholder'], ['add'], input_shapes={'Placeholder': [1, 16, 16, 3]}) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertLen(input_details, 1) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) def testFreezeGraph(self): with ops.Graph().as_default(): @@ -1563,24 +1494,24 @@ class FromFrozenGraphFile(LiteTest): converter = lite.TFLiteConverter.from_frozen_graph(graph_def_file, ['Placeholder'], ['add']) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('add', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) def testInvalidFileNotFound(self): @@ -1620,7 +1551,7 @@ class FromFrozenGraphFile(LiteTest): converter = lite.TocoConverter.from_frozen_graph(graph_def_file, ['Placeholder'], ['add']) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Ensure the model is able to load. interpreter = Interpreter(model_content=tflite_model) @@ -1644,7 +1575,7 @@ class FromFrozenGraphFile(LiteTest): ['Placeholder'], ['add']) converter.convert() # GraphDebugInfo should be none for frozen graph. - self.assertTrue(not converter._debug_info) + self.assertFalse(converter._debug_info) class FromFrozenGraphObjectDetection(LiteTest): @@ -1679,35 +1610,35 @@ class FromFrozenGraphObjectDetection(LiteTest): self._input_shapes) converter.allow_custom_ops = True tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('normalized_input_image_tensor', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 300, 300, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 300, 300, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(4, len(output_details)) + self.assertLen(output_details, 4) self.assertEqual('TFLite_Detection_PostProcess', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 10, 4] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 10, 4], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) self.assertEqual('TFLite_Detection_PostProcess:1', output_details[1]['name']) - self.assertTrue(([1, 10] == output_details[1]['shape']).all()) + self.assertAllEqual([1, 10], output_details[1]['shape']) self.assertEqual('TFLite_Detection_PostProcess:2', output_details[2]['name']) - self.assertTrue(([1, 10] == output_details[2]['shape']).all()) + self.assertAllEqual([1, 10], output_details[2]['shape']) self.assertEqual('TFLite_Detection_PostProcess:3', output_details[3]['name']) - self.assertTrue(([1] == output_details[3]['shape']).all()) + self.assertAllEqual([1], output_details[3]['shape']) class FromSavedModelTest(TestModels): @@ -1734,28 +1665,28 @@ class FromSavedModelTest(TestModels): # Convert model and ensure model is not None. converter = lite.TFLiteConverter.from_saved_model(saved_model_dir) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(2, len(input_details)) + self.assertLen(input_details, 2) self.assertStartsWith(input_details[0]['name'], 'inputA') self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) self.assertStartsWith(input_details[1]['name'], 'inputB') self.assertEqual(np.float32, input_details[1]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[1]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[1]['shape']) self.assertEqual((0., 0.), input_details[1]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertStartsWith(output_details[0]['name'], 'add') self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) def testOldConverterWarning(self): @@ -1769,7 +1700,7 @@ class FromSavedModelTest(TestModels): converter = lite.TFLiteConverter.from_saved_model(saved_model_dir) converter.experimental_new_converter = False tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) self.assertIn(warning_message, log.getvalue()) logging.root.removeHandler(handler) @@ -1784,7 +1715,7 @@ class FromSavedModelTest(TestModels): # Convert model and ensure model is not None. converter = lite.TFLiteConverter.from_saved_model(saved_model_dir) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) self.assertIn(optout_message, log.getvalue()) logging.root.removeHandler(handler) @@ -1794,29 +1725,29 @@ class FromSavedModelTest(TestModels): converter = lite.TFLiteConverter.from_saved_model(saved_model_dir) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(2, len(input_details)) + self.assertLen(input_details, 2) self.assertStartsWith(input_details[0]['name'], 'inputA') self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) self.assertStartsWith(input_details[1]['name'], 'inputB') self.assertEqual(np.float32, input_details[1]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[1]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[1]['shape']) self.assertEqual((0., 0.), input_details[1]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertStartsWith(output_details[0]['name'], 'add') self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) def testOrderInputArrays(self): @@ -1826,29 +1757,29 @@ class FromSavedModelTest(TestModels): converter = lite.TFLiteConverter.from_saved_model( saved_model_dir, input_arrays=['inputB', 'inputA']) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(2, len(input_details)) + self.assertLen(input_details, 2) self.assertStartsWith(input_details[0]['name'], 'inputA') self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) self.assertStartsWith(input_details[1]['name'], 'inputB') self.assertEqual(np.float32, input_details[1]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[1]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[1]['shape']) self.assertEqual((0., 0.), input_details[1]['quantization']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertStartsWith(output_details[0]['name'], 'add') self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) def testSubsetInputArrays(self): @@ -1880,7 +1811,7 @@ class FromSavedModelTest(TestModels): # Convert model and ensure model is not None. converter = lite.TocoConverter.from_saved_model(saved_model_dir) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Ensure the model is able to load. interpreter = Interpreter(model_content=tflite_model) @@ -1958,7 +1889,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_keras_model_file(self._keras_file) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check tensor details of converted model. interpreter = Interpreter(model_content=tflite_model) @@ -1968,13 +1899,13 @@ class FromKerasFile(TestModels, parameterized.TestCase): self.assertLen(input_details, 1) self.assertEndsWith(input_details[0]['name'], 'dense_input') self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() self.assertLen(output_details, 1) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 3, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 3, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) # Check inference of converted model. @@ -1998,7 +1929,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_keras_model_file( self._keras_file, custom_objects=self._custom_objects) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check tensor details of converted model. interpreter = Interpreter(model_content=tflite_model) @@ -2035,7 +1966,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_keras_model_file( self._keras_file, input_arrays=['dense_input']) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) def testSequentialModelInputShape(self): """Test a Sequential tf.keras model testing input shapes argument.""" @@ -2053,7 +1984,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_keras_model_file( self._keras_file, input_shapes={'dense_input': [2, 3]}) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check input shape from converted model. interpreter = Interpreter(model_content=tflite_model) @@ -2062,7 +1993,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): input_details = interpreter.get_input_details() self.assertLen(input_details, 1) self.assertEndsWith(input_details[0]['name'], 'dense_input') - self.assertTrue(([2, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([2, 3], input_details[0]['shape']) def testSequentialModelOutputArray(self): """Test a Sequential tf.keras model testing output arrays argument.""" @@ -2080,7 +2011,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): converter = lite.TFLiteConverter.from_keras_model_file( self._keras_file, output_arrays=['time_distributed/Reshape_1']) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) @parameterized.named_parameters(('_graph', context.graph_mode), ('_eager', context.eager_mode)) @@ -2110,7 +2041,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): # Convert to TFLite model. converter = lite.TFLiteConverter.from_keras_model_file(self._keras_file) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check tensor details of converted model. interpreter = Interpreter(model_content=tflite_model) @@ -2120,13 +2051,13 @@ class FromKerasFile(TestModels, parameterized.TestCase): self.assertLen(input_details, 1) self.assertEqual('input', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() self.assertLen(output_details, 1) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) # Check inference of converted model. @@ -2172,7 +2103,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): # Convert to TFLite model. converter = lite.TFLiteConverter.from_keras_model_file(self._keras_file) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check values from converted model. interpreter = Interpreter(model_content=tflite_model) @@ -2182,22 +2113,22 @@ class FromKerasFile(TestModels, parameterized.TestCase): self.assertLen(input_details, 2) self.assertEndsWith(input_details[0]['name'], 'input_a') self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) self.assertEndsWith(input_details[1]['name'], 'input_b') self.assertEqual(np.float32, input_details[1]['dtype']) - self.assertTrue(([1, 3] == input_details[1]['shape']).all()) + self.assertAllEqual([1, 3], input_details[1]['shape']) self.assertEqual((0., 0.), input_details[1]['quantization']) output_details = interpreter.get_output_details() self.assertLen(output_details, 2) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 4] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 4], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) self.assertEqual(np.float32, output_details[1]['dtype']) - self.assertTrue(([1, 4] == output_details[1]['shape']).all()) + self.assertAllEqual([1, 4], output_details[1]['shape']) self.assertEqual((0., 0.), output_details[1]['quantization']) def testFunctionalSequentialModel(self): @@ -2228,7 +2159,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): # Convert to TFLite model. converter = lite.TFLiteConverter.from_keras_model_file(self._keras_file) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Check tensor details of converted model. interpreter = Interpreter(model_content=tflite_model) @@ -2238,13 +2169,13 @@ class FromKerasFile(TestModels, parameterized.TestCase): self.assertLen(input_details, 1) self.assertEndsWith(input_details[0]['name'], 'dense_input') self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) output_details = interpreter.get_output_details() self.assertLen(output_details, 1) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 3, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([1, 3, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) # Check inference of converted model. @@ -2264,7 +2195,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): converter = lite.TocoConverter.from_keras_model_file(self._keras_file) tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) # Ensure the model is able to load. interpreter = Interpreter(model_content=tflite_model) @@ -2286,7 +2217,7 @@ class FromKerasFile(TestModels, parameterized.TestCase): converter = lite.TocoConverter.from_keras_model_file(self._keras_file) converter._experimental_sparsify_model = True tflite_model = converter.convert() - self.assertTrue(tflite_model) + self.assertIsNotNone(tflite_model) class GrapplerTest(TestModels, parameterized.TestCase): @@ -2312,21 +2243,21 @@ class GrapplerTest(TestModels, parameterized.TestCase): interpreter.allocate_tensors() input_details = interpreter.get_input_details() - self.assertEqual(1, len(input_details)) + self.assertLen(input_details, 1) self.assertEqual('Placeholder', input_details[0]['name']) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([3, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([3, 3], input_details[0]['shape']) output_details = interpreter.get_output_details() - self.assertEqual(1, len(output_details)) + self.assertLen(output_details, 1) self.assertEqual('output', output_details[0]['name']) self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([3, 3] == output_details[0]['shape']).all()) + self.assertAllEqual([3, 3], output_details[0]['shape']) @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir - def testInputNodeIsNotFolded(self, enable_mlir): + def testInputNodeIsNotFolded(self, enable_mlir_converter): ops.disable_eager_execution() # Constant folding handles the tf.broadcast_to operation which was not # supported by the TFLite at the time this test was added. @@ -2340,7 +2271,7 @@ class GrapplerTest(TestModels, parameterized.TestCase): # Convert model. converter = lite.TFLiteConverter.from_session(sess, [in_tensor, y_const], [out_tensor]) - converter.experimental_new_converter = enable_mlir + converter.experimental_new_converter = enable_mlir_converter tflite_model = converter.convert() # Check values from converted model. @@ -2416,7 +2347,7 @@ class DefaultConverterAttrsTest(LiteTest): self.assertFalse(converter.change_concat_input_ranges) # Assert dropping control dependency is enabled by default. - self.assertTrue(converter.drop_control_dependency) + self.assertIsNotNone(converter.drop_control_dependency) # Assert dumping extra information is disabled by default. self.assertIsNone(converter.dump_graphviz_dir) diff --git a/tensorflow/lite/python/lite_v2_test.py b/tensorflow/lite/python/lite_v2_test.py index c1b566ff8ad..714eb249ec9 100644 --- a/tensorflow/lite/python/lite_v2_test.py +++ b/tensorflow/lite/python/lite_v2_test.py @@ -58,14 +58,14 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir @test_util.run_v2_only - def testFloat(self, enable_mlir): + def testFloat(self, enable_mlir_converter): root = self._getSimpleVariableModel() input_data = tf.constant(1., shape=[1]) concrete_func = root.f.get_concrete_function(input_data) # Convert model. converter = lite.TFLiteConverterV2.from_concrete_functions([concrete_func]) - converter.experimental_new_converter = enable_mlir + converter.experimental_new_converter = enable_mlir_converter tflite_model = converter.convert() # Check values from converted model. @@ -142,7 +142,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): self.assertIn('can only convert a single ConcreteFunction', str(error.exception)) - def _getCalibrationQuantizeModel(self): + def _getIntegerQuantizeModel(self): np.random.seed(0) root = tracking.AutoTrackable() @@ -167,23 +167,23 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): ('EnableMlirQuantizer', True), # enable mlir quantizer ('DisableMlirQuantizer', False)) # disable mlir quantizer def testPostTrainingCalibrateAndQuantize(self, mlir_quantizer): - func, calibration_gen = self._getCalibrationQuantizeModel() + func, calibration_gen = self._getIntegerQuantizeModel() # Convert float model. float_converter = lite.TFLiteConverterV2.from_concrete_functions([func]) - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) + float_tflite_model = float_converter.convert() + self.assertIsNotNone(float_tflite_model) # Convert quantized model. quantized_converter = lite.TFLiteConverterV2.from_concrete_functions([func]) quantized_converter.optimizations = [lite.Optimize.DEFAULT] quantized_converter.representative_dataset = calibration_gen quantized_converter._experimental_new_quantizer = mlir_quantizer - quantized_tflite = quantized_converter.convert() - self.assertTrue(quantized_tflite) + quantized_tflite_model = quantized_converter.convert() + self.assertIsNotNone(quantized_tflite_model) # The default input and output types should be float. - interpreter = Interpreter(model_content=quantized_tflite) + interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() self.assertLen(input_details, 1) @@ -193,7 +193,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): self.assertEqual(np.float32, output_details[0]['dtype']) # Ensure that the quantized weights tflite model is smaller. - self.assertLess(len(quantized_tflite), len(float_tflite)) + self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) @parameterized.named_parameters( ('_INT8InputOutput', lite.constants.INT8), @@ -202,7 +202,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): @test_util.run_v2_only def testInvalidPostTrainingDynamicRangeQuantization( self, inference_input_output_type): - func, _ = self._getCalibrationQuantizeModel() + func, _ = self._getIntegerQuantizeModel() # Convert float model. converter = lite.TFLiteConverterV2.from_concrete_functions([func]) @@ -228,7 +228,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): ('_UINT8InputOutput', lite.constants.QUANTIZED_UINT8)) def testPostTrainingIntegerAllowFloatQuantization( self, inference_input_output_type): - func, calibration_gen = self._getCalibrationQuantizeModel() + func, calibration_gen = self._getIntegerQuantizeModel() # Convert float model. converter = lite.TFLiteConverterV2.from_concrete_functions([func]) @@ -242,7 +242,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): quantized_converter.inference_input_type = inference_input_output_type quantized_converter.inference_output_type = inference_input_output_type quantized_tflite_model = quantized_converter.convert() - self.assertTrue(quantized_tflite_model) + self.assertIsNotNone(quantized_tflite_model) interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() @@ -259,7 +259,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): self.assertLess(len(quantized_tflite_model), len(tflite_model)) def testPostTrainingIntegerAllowFloatQuantizationINT16InputOutput(self): - func, calibration_gen = self._getCalibrationQuantizeModel() + func, calibration_gen = self._getIntegerQuantizeModel() # Convert float model. converter = lite.TFLiteConverterV2.from_concrete_functions([func]) @@ -279,7 +279,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): quantized_converter.inference_input_type = inference_input_output_type quantized_converter.inference_output_type = inference_input_output_type quantized_tflite_model = quantized_converter.convert() - self.assertTrue(quantized_tflite_model) + self.assertIsNotNone(quantized_tflite_model) interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() @@ -299,7 +299,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): # In this test we check that when we do 16x8 post-training # quantization and set inference_input(output)_type to # constants.INT8, we have an error. - func, calibration_gen = self._getCalibrationQuantizeModel() + func, calibration_gen = self._getIntegerQuantizeModel() # Convert quantized model. quantized_converter = lite.TFLiteConverterV2.from_concrete_functions([func]) @@ -330,7 +330,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): inference_input_output_type, use_target_ops_flag, quantization_16x8): - func, calibration_gen = self._getCalibrationQuantizeModel() + func, calibration_gen = self._getIntegerQuantizeModel() # Convert float model. converter = lite.TFLiteConverterV2.from_concrete_functions([func]) @@ -357,7 +357,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): quantized_converter.inference_input_type = inference_input_output_type quantized_converter.inference_output_type = inference_input_output_type quantized_tflite_model = quantized_converter.convert() - self.assertTrue(quantized_tflite_model) + self.assertIsNotNone(quantized_tflite_model) interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() @@ -374,12 +374,12 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): self.assertLess(len(quantized_tflite_model), len(tflite_model)) def testCalibrateAndQuantizeBuiltinInt16(self): - func, calibration_gen = self._getCalibrationQuantizeModel() + func, calibration_gen = self._getIntegerQuantizeModel() # Convert float model. float_converter = lite.TFLiteConverterV2.from_concrete_functions([func]) - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) + float_tflite_model = float_converter.convert() + self.assertIsNotNone(float_tflite_model) converter = lite.TFLiteConverterV2.from_concrete_functions([func]) # TODO(b/156309549): We should add INT16 to the builtin types. @@ -389,13 +389,13 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): converter.representative_dataset = calibration_gen converter._experimental_calibrate_only = True calibrated_tflite = converter.convert() - quantized_tflite = mlir_quantize(calibrated_tflite, - inference_type=_types_pb2.QUANTIZED_INT16) + quantized_tflite_model = mlir_quantize( + calibrated_tflite, inference_type=_types_pb2.QUANTIZED_INT16) - self.assertTrue(quantized_tflite) + self.assertIsNotNone(quantized_tflite_model) # The default input and output types should be float. - interpreter = Interpreter(model_content=quantized_tflite) + interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() self.assertLen(input_details, 1) @@ -405,7 +405,7 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): self.assertEqual(np.float32, output_details[0]['dtype']) # Ensure that the quantized weights tflite model is smaller. - self.assertLess(len(quantized_tflite), len(float_tflite)) + self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) def _getTrainingTimeQuantizedModel(self): @@ -454,17 +454,17 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): model = self._getTrainingTimeQuantizedModel() float_converter = lite.TFLiteConverterV2.from_keras_model(model) - float_tflite = float_converter.convert() - self.assertTrue(float_tflite) + float_tflite_model = float_converter.convert() + self.assertIsNotNone(float_tflite_model) quantized_converter = lite.TFLiteConverterV2.from_keras_model(model) quantized_converter.optimizations = [lite.Optimize.DEFAULT] quantized_converter.inference_input_type = inference_input_output_type quantized_converter.inference_output_type = inference_input_output_type - quantized_tflite = quantized_converter.convert() - self.assertTrue(quantized_tflite) + quantized_tflite_model = quantized_converter.convert() + self.assertIsNotNone(quantized_tflite_model) - interpreter = Interpreter(model_content=quantized_tflite) + interpreter = Interpreter(model_content=quantized_tflite_model) interpreter.allocate_tensors() input_details = interpreter.get_input_details() self.assertLen(input_details, 1) @@ -476,12 +476,12 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): output_details[0]['dtype']) # Ensure that the quantized tflite model is smaller. - self.assertLess(len(quantized_tflite), len(float_tflite)) + self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) @test_util.run_v2_only def testNewQuantizer(self): """Test the model quantized by the new converter.""" - func, calibration_gen = self._getCalibrationQuantizeModel() + func, calibration_gen = self._getIntegerQuantizeModel() quantized_converter = lite.TFLiteConverterV2.from_concrete_functions([func]) quantized_converter.target_spec.supported_ops = [ @@ -502,13 +502,13 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): np.random.uniform(-1, 1, size=(1, 5, 5, 3)).astype(np.float32)) old_value = self._evaluateTFLiteModel(old_tflite, [input_data]) new_value = self._evaluateTFLiteModel(new_tflite, [input_data]) - np.testing.assert_almost_equal(old_value, new_value, 1) + self.assertAllClose(old_value, new_value, atol=1e-01) @parameterized.named_parameters( ('EnableMlirConverter', True), # enable mlir ('DisableMlirConverter', False)) # disable mlir @test_util.run_v2_only - def testEmbeddings(self, enable_mlir): + def testEmbeddings(self, enable_mlir_converter): """Test model with embeddings.""" input_data = tf.constant( np.array(np.random.random_sample((20)), dtype=np.int32)) @@ -534,13 +534,13 @@ class FromConcreteFunctionTest(lite_v2_test_util.ModelTest): # Convert model. converter = lite.TFLiteConverterV2.from_concrete_functions([concrete_func]) - converter.experimental_new_converter = enable_mlir + converter.experimental_new_converter = enable_mlir_converter tflite_model = converter.convert() # Check values from converted model. expected_value = root.func(input_data) actual_value = self._evaluateTFLiteModel(tflite_model, [input_data]) - np.testing.assert_almost_equal(expected_value.numpy(), actual_value[0], 5) + self.assertAllClose(expected_value.numpy(), actual_value[0], atol=1e-05) @test_util.run_v2_only def testGraphDebugInfo(self): @@ -594,7 +594,7 @@ class FromSavedModelTest(lite_v2_test_util.ModelTest): self.assertLen(input_details, 2) self.assertStartsWith(input_details[0]['name'], 'inputA') self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[0]['shape']).all()) + self.assertAllEqual([1, 16, 16, 3], input_details[0]['shape']) self.assertEqual((0., 0.), input_details[0]['quantization']) self.assertStartsWith( @@ -602,14 +602,14 @@ class FromSavedModelTest(lite_v2_test_util.ModelTest): 'inputB', ) self.assertEqual(np.float32, input_details[1]['dtype']) - self.assertTrue(([1, 16, 16, 3] == input_details[1]['shape']).all()) + self.assertTrue([1, 16, 16, 3], input_details[1]['shape']) self.assertEqual((0., 0.), input_details[1]['quantization']) output_details = interpreter.get_output_details() self.assertLen(output_details, 1) self.assertStartsWith(output_details[0]['name'], 'add') self.assertEqual(np.float32, output_details[0]['dtype']) - self.assertTrue(([1, 16, 16, 3] == output_details[0]['shape']).all()) + self.assertTrue([1, 16, 16, 3], output_details[0]['shape']) self.assertEqual((0., 0.), output_details[0]['quantization']) @test_util.run_v2_only @@ -715,7 +715,6 @@ class FromSavedModelTest(lite_v2_test_util.ModelTest): @test_util.run_v2_only def testNoConcreteFunctionModel(self): root = self._getMultiFunctionModel() - input_data = tf.constant(1., shape=[1]) save_dir = os.path.join(self.get_temp_dir(), 'saved_model') save(root, save_dir) @@ -836,7 +835,7 @@ class FromKerasModelTest(lite_v2_test_util.ModelTest): expected_value = model.predict(input_data) actual_value = self._evaluateTFLiteModel(tflite_model, input_data) for tf_result, tflite_result in zip(expected_value, actual_value): - np.testing.assert_almost_equal(tf_result, tflite_result, 5) + self.assertAllClose(tf_result, tflite_result, atol=1e-05) @test_util.run_v2_only def testGraphDebugInfo(self): @@ -919,7 +918,7 @@ class ControlFlowTest(lite_v2_test_util.ModelTest): expected_value = concrete_func(**input_data) actual_value = self._evaluateTFLiteModel( tflite_model, [input_data['x'], input_data['b']])[0] - np.testing.assert_almost_equal(expected_value.numpy(), actual_value) + self.assertAllClose(expected_value, actual_value) @test_util.run_v2_only def testStaticRnn(self): @@ -945,7 +944,7 @@ class ControlFlowTest(lite_v2_test_util.ModelTest): expected_value = concrete_func(input_data)[0] actual_value = self._evaluateTFLiteModel(tflite_model, [input_data]) for expected, actual in zip(expected_value, actual_value): - np.testing.assert_almost_equal(expected.numpy(), actual) + self.assertAllClose(expected, actual) @test_util.run_v2_only def testWhileLoop(self): @@ -973,7 +972,7 @@ class ControlFlowTest(lite_v2_test_util.ModelTest): # Check values from converted model. expected_value = concrete_func(input_data)[0] actual_value = self._evaluateTFLiteModel(tflite_model, [input_data])[0] - np.testing.assert_almost_equal(expected_value.numpy(), actual_value) + self.assertAllClose(expected_value, actual_value) @test_util.run_v2_only def testDynamicRnn(self): @@ -997,11 +996,9 @@ class ControlFlowTest(lite_v2_test_util.ModelTest): expected_value = concrete_func(input_data) actual_value = self._evaluateTFLiteModel(tflite_model, [input_data]) for expected, actual in zip(expected_value, actual_value): - if isinstance(expected, ops.EagerTensor): - expected = expected.numpy() - else: - expected = expected.c.numpy() - np.testing.assert_almost_equal(expected, actual) + if not isinstance(expected, ops.EagerTensor): + expected = expected.c + self.assertAllClose(expected, actual) @parameterized.named_parameters(('LSTM', recurrent_v2.LSTM), ('SimpleRNN', recurrent.SimpleRNN), @@ -1025,7 +1022,7 @@ class ControlFlowTest(lite_v2_test_util.ModelTest): # Check values from converted model. expected_value = model.predict(input_data) - np.testing.assert_almost_equal(expected_value, actual_value, decimal=5) + self.assertAllClose(expected_value, actual_value, atol=1e-05) @parameterized.named_parameters(('LSTM', recurrent_v2.LSTM), ('SimpleRNN', recurrent.SimpleRNN), @@ -1046,7 +1043,7 @@ class ControlFlowTest(lite_v2_test_util.ModelTest): # Check values from converted model. expected_value = model.predict(input_data) - np.testing.assert_almost_equal(expected_value, actual_value, decimal=5) + self.assertAllClose(expected_value, actual_value, atol=1e-05) @test_util.run_v2_only def testKerasBidirectionalRNN(self): @@ -1069,7 +1066,7 @@ class ControlFlowTest(lite_v2_test_util.ModelTest): # Check values from converted model. expected_value = model.predict(input_data) - np.testing.assert_almost_equal(expected_value, actual_value, decimal=5) + self.assertAllClose(expected_value, actual_value, atol=1e-05) class GrapplerTest(lite_v2_test_util.ModelTest): @@ -1096,14 +1093,14 @@ class GrapplerTest(lite_v2_test_util.ModelTest): # Check values from converted model. expected_value = root.f(input_data) - actual_value = self._evaluateTFLiteModel(tflite_model, [input_data]) - np.testing.assert_almost_equal(expected_value.numpy(), actual_value[0]) + actual_value = self._evaluateTFLiteModel(tflite_model, [input_data])[0] + self.assertAllClose(expected_value, actual_value) # Enable hybrid quantization, same result converter.optimizations = [lite.Optimize.DEFAULT] - hybrid_tflite_model = converter.convert() - actual_value = self._evaluateTFLiteModel(hybrid_tflite_model, [input_data]) - np.testing.assert_almost_equal(expected_value.numpy(), actual_value[0]) + tflite_model = converter.convert() + actual_value = self._evaluateTFLiteModel(tflite_model, [input_data])[0] + self.assertAllClose(expected_value, actual_value) class UnknownShapes(lite_v2_test_util.ModelTest): @@ -1128,15 +1125,16 @@ class UnknownShapes(lite_v2_test_util.ModelTest): # Check values from converted model. expected_value = concrete_func(input_data) actual_value = self._evaluateTFLiteModel( - tflite_model, [input_data], input_shapes=[([-1, 4], [10, 4])]) - np.testing.assert_almost_equal( - expected_value.numpy(), actual_value[0], decimal=6) + tflite_model, [input_data], input_shapes=[([-1, 4], [10, 4])])[0] + self.assertAllClose(expected_value, actual_value, atol=1e-06) + + def _getIntegerQuantizeModelWithUnknownShapes(self): + np.random.seed(0) - def _getQuantizedModel(self): - # Returns a model with tf.MatMul and unknown dimensions. @tf.function( input_signature=[tf.TensorSpec(shape=[None, 33], dtype=tf.float32)]) - def model(in_tensor): + def model(input_tensor): + """Define a model with tf.MatMul and unknown shapes.""" # We need the tensor to have more than 1024 elements for quantize_weights # to kick in. Thus, the [33, 33] shape. const_tensor = tf.constant( @@ -1145,12 +1143,14 @@ class UnknownShapes(lite_v2_test_util.ModelTest): dtype=tf.float32, name='inputB') - shape = tf.shape(in_tensor) + shape = tf.shape(input_tensor) fill = tf.transpose(tf.fill(shape, 1.)) - mult = tf.matmul(fill, in_tensor) + mult = tf.matmul(fill, input_tensor) return tf.matmul(mult, const_tensor) - concrete_func = model.get_concrete_function() + root = tracking.AutoTrackable() + root.f = model + concrete_func = root.f.get_concrete_function() def calibration_gen(): for batch in range(5, 20, 5): @@ -1161,7 +1161,7 @@ class UnknownShapes(lite_v2_test_util.ModelTest): @test_util.run_v2_only def testMatMulQuantize(self): - concrete_func, _ = self._getQuantizedModel() + concrete_func, _ = self._getIntegerQuantizeModelWithUnknownShapes() float_converter = lite.TFLiteConverterV2.from_concrete_functions( [concrete_func]) float_tflite_model = float_converter.convert() @@ -1177,14 +1177,15 @@ class UnknownShapes(lite_v2_test_util.ModelTest): input_details = quantized_interpreter.get_input_details() self.assertLen(input_details, 1) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue((input_details[0]['shape_signature'] == [-1, 33]).all()) + self.assertAllEqual([-1, 33], input_details[0]['shape_signature']) # Ensure that the quantized weights tflite model is smaller. self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) @test_util.run_v2_only def testMatMulCalibrateAndQuantize(self): - concrete_func, calibration_gen = self._getQuantizedModel() + concrete_func, calibration_gen = \ + self._getIntegerQuantizeModelWithUnknownShapes() float_converter = lite.TFLiteConverterV2.from_concrete_functions( [concrete_func]) float_tflite_model = float_converter.convert() @@ -1201,7 +1202,7 @@ class UnknownShapes(lite_v2_test_util.ModelTest): input_details = quantized_interpreter.get_input_details() self.assertLen(input_details, 1) self.assertEqual(np.float32, input_details[0]['dtype']) - self.assertTrue((input_details[0]['shape_signature'] == [-1, 33]).all()) + self.assertAllEqual([-1, 33], input_details[0]['shape_signature']) # Ensure that the quantized weights tflite model is smaller. self.assertLess(len(quantized_tflite_model), len(float_tflite_model)) @@ -1228,9 +1229,8 @@ class UnknownShapes(lite_v2_test_util.ModelTest): expected_value = concrete_func(input_data_1, input_data_2) actual_value = self._evaluateTFLiteModel( tflite_model, [input_data_1, input_data_2], - input_shapes=[([-1, 256, 256], [1, 256, 256])]) - np.testing.assert_almost_equal( - expected_value.numpy(), actual_value[0], decimal=4) + input_shapes=[([-1, 256, 256], [1, 256, 256])])[0] + self.assertAllClose(expected_value, actual_value, atol=4) def testSizeInvalid(self): diff --git a/tensorflow/lite/python/lite_v2_test_util.py b/tensorflow/lite/python/lite_v2_test_util.py index d8f764711cd..1493b240913 100644 --- a/tensorflow/lite/python/lite_v2_test_util.py +++ b/tensorflow/lite/python/lite_v2_test_util.py @@ -77,6 +77,7 @@ class ModelTest(test_util.TensorFlowTestCase, parameterized.TestCase): def _getMultiFunctionModel(self): class BasicModel(tracking.AutoTrackable): + """Basic model with multiple functions.""" def __init__(self): self.y = None diff --git a/tensorflow/lite/python/util.py b/tensorflow/lite/python/util.py index 720e53de509..79d2775d1dc 100644 --- a/tensorflow/lite/python/util.py +++ b/tensorflow/lite/python/util.py @@ -48,16 +48,16 @@ from tensorflow.python.training.saver import export_meta_graph as _export_meta_g _MAP_TF_TO_TFLITE_TYPES = { dtypes.float32: _types_pb2.FLOAT, dtypes.float16: _types_pb2.FLOAT16, - dtypes.float64: _types_pb2.FLOAT64, dtypes.int32: _types_pb2.INT32, + dtypes.uint8: _types_pb2.QUANTIZED_UINT8, dtypes.int64: _types_pb2.INT64, dtypes.string: _types_pb2.STRING, - dtypes.uint8: _types_pb2.QUANTIZED_UINT8, - dtypes.int8: _types_pb2.INT8, + dtypes.bool: _types_pb2.BOOL, dtypes.int16: _types_pb2.QUANTIZED_INT16, dtypes.complex64: _types_pb2.COMPLEX64, + dtypes.int8: _types_pb2.INT8, + dtypes.float64: _types_pb2.FLOAT64, dtypes.complex128: _types_pb2.COMPLEX128, - dtypes.bool: _types_pb2.BOOL, } _MAP_TFLITE_ENUM_TO_TF_TYPES = { @@ -72,6 +72,7 @@ _MAP_TFLITE_ENUM_TO_TF_TYPES = { 8: dtypes.complex64, 9: dtypes.int8, 10: dtypes.float64, + 11: dtypes.complex128, } _TFLITE_FILE_IDENTIFIER = b"TFL3" @@ -113,7 +114,7 @@ def _convert_tflite_enum_type_to_tf_type(tflite_enum_type): tf_type = _MAP_TFLITE_ENUM_TO_TF_TYPES.get(tflite_enum_type) if tf_type is None: raise ValueError( - "Unsupported enum {}. The valid map of enum to tf.dtypes is : {}" + "Unsupported enum {}. The valid map of enum to tf types is : {}" .format(tflite_enum_type, _MAP_TFLITE_ENUM_TO_TF_TYPES)) return tf_type diff --git a/tensorflow/lite/python/util_test.py b/tensorflow/lite/python/util_test.py index 0e9cbc1e58a..820cda4c7d6 100644 --- a/tensorflow/lite/python/util_test.py +++ b/tensorflow/lite/python/util_test.py @@ -42,27 +42,34 @@ from tensorflow.python.platform import test class UtilTest(test_util.TensorFlowTestCase): def testConvertDtype(self): - self.assertEqual( - util.convert_dtype_to_tflite_type(lite_constants.FLOAT), - _types_pb2.FLOAT) self.assertEqual( util.convert_dtype_to_tflite_type(dtypes.float32), _types_pb2.FLOAT) + self.assertEqual( + util.convert_dtype_to_tflite_type(dtypes.float16), _types_pb2.FLOAT16) self.assertEqual( util.convert_dtype_to_tflite_type(dtypes.int32), _types_pb2.INT32) + self.assertEqual( + util.convert_dtype_to_tflite_type(dtypes.uint8), + _types_pb2.QUANTIZED_UINT8) self.assertEqual( util.convert_dtype_to_tflite_type(dtypes.int64), _types_pb2.INT64) self.assertEqual( util.convert_dtype_to_tflite_type(dtypes.string), _types_pb2.STRING) self.assertEqual( - util.convert_dtype_to_tflite_type(dtypes.uint8), - _types_pb2.QUANTIZED_UINT8) + util.convert_dtype_to_tflite_type(dtypes.bool), _types_pb2.BOOL) + self.assertEqual( + util.convert_dtype_to_tflite_type(dtypes.int16), + _types_pb2.QUANTIZED_INT16) self.assertEqual( util.convert_dtype_to_tflite_type(dtypes.complex64), _types_pb2.COMPLEX64) self.assertEqual( - util.convert_dtype_to_tflite_type(dtypes.half), _types_pb2.FLOAT16) + util.convert_dtype_to_tflite_type(dtypes.int8), _types_pb2.INT8) self.assertEqual( - util.convert_dtype_to_tflite_type(dtypes.bool), _types_pb2.BOOL) + util.convert_dtype_to_tflite_type(dtypes.float64), _types_pb2.FLOAT64) + self.assertEqual( + util.convert_dtype_to_tflite_type(dtypes.complex128), + _types_pb2.COMPLEX128) def testConvertEnumToDtype(self): self.assertEqual( @@ -81,17 +88,19 @@ class UtilTest(test_util.TensorFlowTestCase): self.assertEqual(util._convert_tflite_enum_type_to_tf_type(9), dtypes.int8) self.assertEqual( util._convert_tflite_enum_type_to_tf_type(10), dtypes.float64) - with self.assertRaises(ValueError) as error: - util._convert_tflite_enum_type_to_tf_type(11) self.assertEqual( - "Unsupported enum 11. The valid map of enum to tf.dtypes is : " + util._convert_tflite_enum_type_to_tf_type(11), dtypes.complex128) + with self.assertRaises(ValueError) as error: + util._convert_tflite_enum_type_to_tf_type(20) + self.assertEqual( + "Unsupported enum 20. The valid map of enum to tf types is : " "{0: tf.float32, 1: tf.float16, 2: tf.int32, 3: tf.uint8, 4: tf.int64, " "5: tf.string, 6: tf.bool, 7: tf.int16, 8: tf.complex64, 9: tf.int8, " - "10: tf.float64}", str(error.exception)) + "10: tf.float64, 11: tf.complex128}", str(error.exception)) def testTensorName(self): with ops.Graph().as_default(): - in_tensor = array_ops.placeholder(shape=[4], dtype=dtypes.float32) + in_tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[4]) out_tensors = array_ops.split( value=in_tensor, num_or_size_splits=[1, 1, 1, 1], axis=0) @@ -103,7 +112,7 @@ class UtilTest(test_util.TensorFlowTestCase): @test_util.enable_control_flow_v2 def testRemoveLowerUsingSwitchMerge(self): with ops.Graph().as_default(): - i = array_ops.placeholder(shape=(), dtype=dtypes.int32) + i = array_ops.placeholder(dtype=dtypes.int32, shape=()) c = lambda i: math_ops.less(i, 10) b = lambda i: math_ops.add(i, 1) control_flow_ops.while_loop(c, b, [i]) @@ -116,7 +125,7 @@ class UtilTest(test_util.TensorFlowTestCase): if node.op == "While" or node.op == "StatelessWhile": if not node.attr["_lower_using_switch_merge"].b: lower_using_switch_merge_is_removed = True - self.assertEqual(lower_using_switch_merge_is_removed, True) + self.assertTrue(lower_using_switch_merge_is_removed) def testConvertBytes(self): source, header = util.convert_bytes_to_c_source( @@ -154,7 +163,7 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase): def testGetTensorsValid(self): with ops.Graph().as_default(): in_tensor = array_ops.placeholder( - shape=[1, 16, 16, 3], dtype=dtypes.float32) + dtype=dtypes.float32, shape=[1, 16, 16, 3]) _ = in_tensor + in_tensor sess = session.Session() @@ -164,7 +173,7 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase): def testGetTensorsInvalid(self): with ops.Graph().as_default(): in_tensor = array_ops.placeholder( - shape=[1, 16, 16, 3], dtype=dtypes.float32) + dtype=dtypes.float32, shape=[1, 16, 16, 3]) _ = in_tensor + in_tensor sess = session.Session() @@ -175,52 +184,51 @@ class TensorFunctionsTest(test_util.TensorFlowTestCase): def testSetTensorShapeValid(self): with ops.Graph().as_default(): - tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32) - self.assertEqual([None, 3, 5], tensor.shape.as_list()) + tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[None, 3, 5]) + self.assertAllEqual([None, 3, 5], tensor.shape) util.set_tensor_shapes([tensor], {"Placeholder": [5, 3, 5]}) - self.assertEqual([5, 3, 5], tensor.shape.as_list()) + self.assertAllEqual([5, 3, 5], tensor.shape) def testSetTensorShapeNoneValid(self): with ops.Graph().as_default(): tensor = array_ops.placeholder(dtype=dtypes.float32) - self.assertEqual(None, tensor.shape) util.set_tensor_shapes([tensor], {"Placeholder": [1, 3, 5]}) - self.assertEqual([1, 3, 5], tensor.shape.as_list()) + self.assertAllEqual([1, 3, 5], tensor.shape) def testSetTensorShapeArrayInvalid(self): # Tests set_tensor_shape where the tensor name passed in doesn't exist. with ops.Graph().as_default(): - tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32) - self.assertEqual([None, 3, 5], tensor.shape.as_list()) + tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[None, 3, 5]) + self.assertAllEqual([None, 3, 5], tensor.shape) with self.assertRaises(ValueError) as error: util.set_tensor_shapes([tensor], {"invalid-input": [5, 3, 5]}) self.assertEqual( "Invalid tensor 'invalid-input' found in tensor shapes map.", str(error.exception)) - self.assertEqual([None, 3, 5], tensor.shape.as_list()) + self.assertAllEqual([None, 3, 5], tensor.shape) def testSetTensorShapeDimensionInvalid(self): # Tests set_tensor_shape where the shape passed in is incompatible. with ops.Graph().as_default(): - tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32) - self.assertEqual([None, 3, 5], tensor.shape.as_list()) + tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[None, 3, 5]) + self.assertAllEqual([None, 3, 5], tensor.shape) with self.assertRaises(ValueError) as error: util.set_tensor_shapes([tensor], {"Placeholder": [1, 5, 5]}) self.assertIn("The shape of tensor 'Placeholder' cannot be changed", str(error.exception)) - self.assertEqual([None, 3, 5], tensor.shape.as_list()) + self.assertAllEqual([None, 3, 5], tensor.shape) def testSetTensorShapeEmpty(self): with ops.Graph().as_default(): - tensor = array_ops.placeholder(shape=[None, 3, 5], dtype=dtypes.float32) - self.assertEqual([None, 3, 5], tensor.shape.as_list()) + tensor = array_ops.placeholder(dtype=dtypes.float32, shape=[None, 3, 5]) + self.assertAllEqual([None, 3, 5], tensor.shape) util.set_tensor_shapes([tensor], {}) - self.assertEqual([None, 3, 5], tensor.shape.as_list()) + self.assertAllEqual([None, 3, 5], tensor.shape) def _generate_integer_tflite_model(): @@ -355,7 +363,7 @@ class UtilModifyIntegerQuantizedModelIOTypeTest( output_io_data = _run_tflite_inference(model_io, in_tftype, out_tftype) # Validate that both the outputs are the same - self.assertTrue(np.allclose(output_data, output_io_data, atol=1.0)) + self.assertAllClose(output_data, output_io_data, atol=1.0) if __name__ == "__main__": From fc424cb9c535c0bca6f69437f7a9ec6f0c084b0f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 19:01:27 -0700 Subject: [PATCH 0320/1017] Internal change PiperOrigin-RevId: 324730398 Change-Id: I9c45c92bac9702ee8ef18599e2f85402bd2f0cac --- .../optimizer_v2/gradient_descent_test.py | 658 +++++++++--------- 1 file changed, 333 insertions(+), 325 deletions(-) diff --git a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py index 56f0b217578..0f25beacc9a 100644 --- a/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py +++ b/tensorflow/python/keras/optimizer_v2/gradient_descent_test.py @@ -37,9 +37,9 @@ from tensorflow.python.ops import variables from tensorflow.python.platform import test -@combinations.generate(combinations.combine(mode=["graph", "eager"])) class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasic(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([1.0, 2.0], dtype=dtype) @@ -88,6 +88,7 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): [3.0 - 3.0 * 0.01 - 2.0 * 0.01, 4.0 - 3.0 * 0.01 - 2.0 * 0.01], self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasicWithLearningRateDecay(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: learning_rate = 3.0 @@ -95,6 +96,7 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): sgd = gradient_descent.SGD(learning_rate=learning_rate, decay=decay) self._test_basic_sgd_with_learning_rate_decay(sgd, dtype) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasicWithLearningRateInverseTimeDecay(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: learning_rate = learning_rate_schedule.InverseTimeDecay( @@ -102,6 +104,7 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): sgd = gradient_descent.SGD(learning_rate=learning_rate) self._test_basic_sgd_with_learning_rate_decay(sgd, dtype) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasicWithLearningRateInverseTimeDecaySerializeAndDeserialize(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: learning_rate = learning_rate_schedule.InverseTimeDecay( @@ -110,6 +113,7 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): sgd = gradient_descent.SGD.from_config(sgd.get_config()) self._test_basic_sgd_with_learning_rate_decay(sgd, dtype) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasicCallableParams(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([1.0, 2.0], dtype=dtype) @@ -128,6 +132,7 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01], self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testMinimizeResourceVariable(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) @@ -145,26 +150,28 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllCloseAccordingToType([3.0 - 1.0], self.evaluate(var1)) def testMinimizeSparseResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) - var1 = variables.Variable([3.0], dtype=dtype) - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) + # TODO(tanzheny, omalleyt): Fix test in eager mode. + with ops.Graph().as_default(): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) + var1 = variables.Variable([3.0], dtype=dtype) + x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - def loss(): - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) # pylint: disable=cell-var-from-loop - pred += var1 # pylint: disable=cell-var-from-loop - return pred * pred + def loss(): + pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) # pylint: disable=cell-var-from-loop + pred += var1 # pylint: disable=cell-var-from-loop + return pred * pred - sgd_op = gradient_descent.SGD(1.0).minimize(loss, [var0, var1]) - self.evaluate(variables.global_variables_initializer()) - # Run 1 step of sgd - self.evaluate(sgd_op) - # Validate updated params - np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 - np_grad = 2 * np_pred - self.assertAllCloseAccordingToType( - [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0)) - self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1)) + sgd_op = gradient_descent.SGD(1.0).minimize(loss, [var0, var1]) + self.evaluate(variables.global_variables_initializer()) + # Run 1 step of sgd + self.evaluate(sgd_op) + # Validate updated params + np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0 + np_grad = 2 * np_pred + self.assertAllCloseAccordingToType( + [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType([3.0 - np_grad], self.evaluate(var1)) def testTensorLearningRate(self): for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: @@ -185,68 +192,71 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.evaluate(var1)) def testGradWrtRef(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - opt = gradient_descent.SGD(3.0) - values = [1.0, 3.0] - vars_ = [variables.Variable([v], dtype=dtype) for v in values] - loss = lambda: vars_[0] + vars_[1] # pylint: disable=cell-var-from-loop - grads_and_vars = opt._compute_gradients(loss, vars_) - self.evaluate(variables.global_variables_initializer()) - for grad, _ in grads_and_vars: - self.assertAllCloseAccordingToType([1.0], self.evaluate(grad)) + # TODO(tanzheny, omalleyt): Fix test in eager mode. + with ops.Graph().as_default(): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + opt = gradient_descent.SGD(3.0) + values = [1.0, 3.0] + vars_ = [variables.Variable([v], dtype=dtype) for v in values] + loss = lambda: vars_[0] + vars_[1] # pylint: disable=cell-var-from-loop + grads_and_vars = opt._compute_gradients(loss, vars_) + self.evaluate(variables.global_variables_initializer()) + for grad, _ in grads_and_vars: + self.assertAllCloseAccordingToType([1.0], self.evaluate(grad)) def testSparseBasic(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) - var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant([0.1], shape=[1, 1], dtype=dtype), - constant_op.constant([0]), constant_op.constant([2, 1])) - grads1 = ops.IndexedSlices( - constant_op.constant([0.01], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), constant_op.constant([2, 1])) - sgd_op = gradient_descent.SGD(3.0).apply_gradients( - zip([grads0, grads1], [var0, var1])) - self.evaluate(variables.global_variables_initializer()) - # Run 1 step of sgd - self.evaluate(sgd_op) - # Validate updated params - self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], - self.evaluate(var0)) - self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], - self.evaluate(var1)) + # TODO(tanzheny, omalleyt): Fix test in eager mode. + with ops.Graph().as_default(): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) + var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) + grads0 = ops.IndexedSlices( + constant_op.constant([0.1], shape=[1, 1], dtype=dtype), + constant_op.constant([0]), constant_op.constant([2, 1])) + grads1 = ops.IndexedSlices( + constant_op.constant([0.01], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), constant_op.constant([2, 1])) + sgd_op = gradient_descent.SGD(3.0).apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Run 1 step of sgd + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], + self.evaluate(var0)) + self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], + self.evaluate(var1)) def testSparseBasicWithLearningRateDecay(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) - var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) - grads0 = ops.IndexedSlices( - constant_op.constant([0.1], shape=[1, 1], dtype=dtype), - constant_op.constant([0]), constant_op.constant([2, 1])) - grads1 = ops.IndexedSlices( - constant_op.constant([0.01], shape=[1, 1], dtype=dtype), - constant_op.constant([1]), constant_op.constant([2, 1])) + # TODO(tanzheny, omalleyt): Fix test in eager mode. + with ops.Graph().as_default(): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([[1.0], [2.0]], dtype=dtype) + var1 = variables.Variable([[3.0], [4.0]], dtype=dtype) + grads0 = ops.IndexedSlices( + constant_op.constant([0.1], shape=[1, 1], dtype=dtype), + constant_op.constant([0]), constant_op.constant([2, 1])) + grads1 = ops.IndexedSlices( + constant_op.constant([0.01], shape=[1, 1], dtype=dtype), + constant_op.constant([1]), constant_op.constant([2, 1])) + sgd_op = gradient_descent.SGD( + 3.0, decay=0.5).apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Run 2 steps of sgd + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], + self.evaluate(var0)) + self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], + self.evaluate(var1)) - opt = gradient_descent.SGD(3.0, decay=0.5) - update_op = opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - self.evaluate(variables.global_variables_initializer()) - # Run 2 steps of sgd - self.evaluate(update_op) - # Validate updated params - self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]], - self.evaluate(var0)) - self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]], - self.evaluate(var1)) - - if context.executing_eagerly(): - opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - else: - self.evaluate(update_op) - # Validate updated params - self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1 - 2.0 * 0.1], [2.0]], - self.evaluate(var0)) - self.assertAllCloseAccordingToType( - [[3.0], [4.0 - 3.0 * 0.01 - 2.0 * 0.01]], self.evaluate(var1)) + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType( + [[1.0 - 3.0 * 0.1 - 2.0 * 0.1], [2.0]], self.evaluate(var0)) + self.assertAllCloseAccordingToType( + [[3.0], [4.0 - 3.0 * 0.01 - 2.0 * 0.01]], self.evaluate(var1)) def testCapturingInDefunWhileExecutingEagerly(self): with context.eager_mode(): @@ -282,7 +292,6 @@ class GradientDescentOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllClose(self.evaluate(opt_3.lr), (0.1)) -@combinations.generate(combinations.combine(mode=["graph", "eager"])) class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): def _update_nesterov_momentum_numpy(self, var, accum, g, lr, momentum): @@ -290,6 +299,7 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): var += (accum * momentum - g * lr) return var, accum + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testBasic(self): for _, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]): var0 = variables.Variable([1.0, 2.0], dtype=dtype, name="var0") @@ -350,97 +360,91 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): ]), self.evaluate(var1)) def testNesterovMomentum(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([1.0, 2.0], dtype=dtype, name="var0") - var1 = variables.Variable([3.0, 4.0], dtype=dtype, name="var1") - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - loss = lambda: 5 * var0 * var0 + 3 * var1 # pylint: disable=cell-var-from-loop - mom_op = gradient_descent.SGD( - learning_rate=2.0, momentum=0.9, nesterov=True) - opt_op = mom_op.minimize(loss, [var0, var1]) - self.evaluate(variables.global_variables_initializer()) - for i in range(1, 5): - # already updated once in eager mode - if i != 1 and context.executing_eagerly(): - mom_op.minimize(loss, [var0, var1]) - else: + # TODO(tanzheny, omalleyt): Fix test in eager mode. + with ops.Graph().as_default(): + for dtype in [dtypes.float32, dtypes.float64]: + var0 = variables.Variable([1.0, 2.0], dtype=dtype, name="var0") + var1 = variables.Variable([3.0, 4.0], dtype=dtype, name="var1") + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + loss = lambda: 5 * var0 * var0 + 3 * var1 # pylint: disable=cell-var-from-loop + mom_op = gradient_descent.SGD( + learning_rate=2.0, momentum=0.9, nesterov=True) + opt_op = mom_op.minimize(loss, [var0, var1]) + self.evaluate(variables.global_variables_initializer()) + for _ in range(1, 5): self.evaluate(opt_op) - var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy( - var1_np, accum1_np, 3, 2.0, 0.9) - self.assertAllClose(var0_np, self.evaluate(var0)) - self.assertAllClose(var1_np, self.evaluate(var1)) + var0_np, accum0_np = self._update_nesterov_momentum_numpy( + var0_np, accum0_np, var0_np * 10, 2.0, 0.9) + var1_np, accum1_np = self._update_nesterov_momentum_numpy( + var1_np, accum1_np, 3, 2.0, 0.9) + self.assertAllClose(var0_np, self.evaluate(var0)) + self.assertAllClose(var1_np, self.evaluate(var1)) def testSparseNesterovMomentum(self): + # TODO(tanzheny, omalleyt): Fix test in eager mode. for dtype in [dtypes.float32, dtypes.float64]: - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - grads = [] - for t in range(1, 5): - grads.append(var0_np * 10) - var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy( - var1_np, accum1_np, 3, 2.0, 0.9) - var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) - var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) - accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) - var0 = variables.Variable(var0_np, dtype=dtype, name="var0") - var1 = variables.Variable(var1_np, dtype=dtype, name="var1") - mom_op = gradient_descent.SGD( - learning_rate=2.0, momentum=0.9, nesterov=True) - grads_and_vars = [] - for t in range(1, 5): - y = ops.IndexedSlices( - constant_op.constant(grads[t - 1], dtype=dtype), - constant_op.constant([0, 1]), constant_op.constant([2])) - grads_and_vars.append([(y, var0), - (constant_op.constant([3.0, 3.0], - dtype=dtype), var1)]) - if not context.executing_eagerly(): - opt_update = [] + with ops.Graph().as_default(), self.cached_session() as sess: + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + grads = [] for t in range(1, 5): - opt_update.append(mom_op.apply_gradients(grads_and_vars[t - 1])) - self.evaluate(variables.global_variables_initializer()) - for t in range(1, 5): - if context.executing_eagerly(): - mom_op.apply_gradients(grads_and_vars[t - 1]) - else: - self.evaluate(opt_update[t - 1]) - var0_np, accum0_np = self._update_nesterov_momentum_numpy( - var0_np, accum0_np, var0_np * 10, 2.0, 0.9) - var1_np, accum1_np = self._update_nesterov_momentum_numpy( - var1_np, accum1_np, 3, 2.0, 0.9) - self.assertAllClose(var0_np, self.evaluate(var0)) - self.assertAllClose(var1_np, self.evaluate(var1)) + grads.append(var0_np * 10) + var0_np, accum0_np = self._update_nesterov_momentum_numpy( + var0_np, accum0_np, var0_np * 10, 2.0, 0.9) + var1_np, accum1_np = self._update_nesterov_momentum_numpy( + var1_np, accum1_np, 3, 2.0, 0.9) + var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype) + var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype) + accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype) + var0 = variables.Variable(var0_np, dtype=dtype, name="var0") + var1 = variables.Variable(var1_np, dtype=dtype, name="var1") + mom_op = gradient_descent.SGD( + learning_rate=2.0, momentum=0.9, nesterov=True) + x_feed = array_ops.placeholder(dtype) + y_feed = ops.IndexedSlices(x_feed, constant_op.constant([0, 1]), + constant_op.constant([2])) + grads_and_vars = [(y_feed, var0), + (constant_op.constant([3.0, 3.0], dtype=dtype), var1)] + opt_update = mom_op.apply_gradients(grads_and_vars) + self.evaluate(variables.global_variables_initializer()) + for t in range(1, 5): + sess.run(opt_update, feed_dict={x_feed: grads[t - 1]}) + var0_np, accum0_np = self._update_nesterov_momentum_numpy( + var0_np, accum0_np, var0_np * 10, 2.0, 0.9) + var1_np, accum1_np = self._update_nesterov_momentum_numpy( + var1_np, accum1_np, 3, 2.0, 0.9) + self.assertAllClose(var0_np, self.evaluate(var0)) + self.assertAllClose(var1_np, self.evaluate(var1)) def testMinimizeSparseResourceVariable(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) + # TODO(tanzheny, omalleyt): Fix test in eager mode. + with ops.Graph().as_default(): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([[1.0, 2.0]], dtype=dtype) - # pylint: disable=cell-var-from-loop - def loss(): - x = constant_op.constant([[4.0], [5.0]], dtype=dtype) - pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) - return pred * pred + # pylint: disable=cell-var-from-loop + def loss(): + x = constant_op.constant([[4.0], [5.0]], dtype=dtype) + pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x) + return pred * pred - # pylint: enable=cell-var-from-loop + # pylint: enable=cell-var-from-loop - opt = gradient_descent.SGD(learning_rate=1.0, momentum=0.9) - sgd_op = opt.minimize(loss, [var0]) - self.evaluate(variables.global_variables_initializer()) - # Run 1 step of sgd - self.evaluate(sgd_op) - # Validate updated params - self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0)) + opt = gradient_descent.SGD(learning_rate=1.0, momentum=0.9) + sgd_op = opt.minimize(loss, [var0]) + self.evaluate(variables.global_variables_initializer()) + # Run 1 step of sgd + self.evaluate(sgd_op) + # Validate updated params + self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testMinimizeWith2DIndicesForEmbeddingLookup(self): var0 = variables.Variable(array_ops.ones([2, 2])) @@ -454,140 +458,150 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0)) def testTensorLearningRateAndMomentum(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - mom_opt = gradient_descent.SGD( - learning_rate=constant_op.constant(2.0), - momentum=constant_op.constant(0.9)) - mom_update = mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - self.evaluate(variables.global_variables_initializer()) - # Check we have slots - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEqual(slot0.shape, var0.shape) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEqual(slot1.shape, var1.shape) + # TODO(tanzheny, omalleyt): Fix test in eager mode. + with ops.Graph().as_default(): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([1.0, 2.0], dtype=dtype) + var1 = variables.Variable([3.0, 4.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) + mom_opt = gradient_descent.SGD( + learning_rate=constant_op.constant(2.0), + momentum=constant_op.constant(0.9)) + mom_update = mom_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) + # Check we have slots + slot0 = mom_opt.get_slot(var0, "momentum") + self.assertEqual(slot0.shape, var0.shape) + slot1 = mom_opt.get_slot(var1, "momentum") + self.assertEqual(slot1.shape, var1.shape) - # Step 1: the momentum accumulators where 0. So we should see a normal - # update: v -= grad * learning_rate - self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([-0.2, -0.2]), self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([-0.02, -0.02]), self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), - self.evaluate(var1)) - # Step 2: the momentum accumulators contain the previous update. - if context.executing_eagerly(): - mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - else: + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + # Step 1: the momentum accumulators where 0. So we should see a normal + # update: v -= grad * learning_rate self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), - self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.02) - 2.0 * 0.01), - (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), - 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), - 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) - ]), self.evaluate(var1)) + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([-0.2, -0.2]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([-0.02, -0.02]), self.evaluate(slot1)) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) + # Step 2: the momentum accumulators contain the previous update. + self.evaluate(mom_update) + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), + self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([ + 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), + 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) + ]), self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([ + 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), self.evaluate(var1)) def testSparse(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable(array_ops.zeros([4, 2], dtype=dtype)) - var1 = variables.Variable(constant_op.constant(1.0, dtype, [4, 2])) - grads0 = ops.IndexedSlices( - constant_op.constant([[.1, .1]], dtype=dtype), - constant_op.constant([1]), constant_op.constant([4, 2])) - grads1 = ops.IndexedSlices( - constant_op.constant([[.01, .01], [.01, .01]], dtype=dtype), - constant_op.constant([2, 3]), constant_op.constant([4, 2])) - mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) - mom_update = mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - self.evaluate(variables.global_variables_initializer()) + # TODO(tanzheny, omalleyt): Fix test in eager mode. + with ops.Graph().as_default(): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable(array_ops.zeros([4, 2], dtype=dtype)) + var1 = variables.Variable(constant_op.constant(1.0, dtype, [4, 2])) + grads0 = ops.IndexedSlices( + constant_op.constant([[.1, .1]], dtype=dtype), + constant_op.constant([1]), constant_op.constant([4, 2])) + grads1 = ops.IndexedSlices( + constant_op.constant([[.01, .01], [.01, .01]], dtype=dtype), + constant_op.constant([2, 3]), constant_op.constant([4, 2])) + mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) + mom_update = mom_opt.apply_gradients( + zip([grads0, grads1], [var0, var1])) + self.evaluate(variables.global_variables_initializer()) - # Check we have slots - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEqual(slot0.shape, var0.shape) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEqual(slot1.shape, var1.shape) + # Check we have slots + slot0 = mom_opt.get_slot(var0, "momentum") + self.assertEqual(slot0.shape, var0.shape) + slot1 = mom_opt.get_slot(var1, "momentum") + self.assertEqual(slot1.shape, var1.shape) - # Step 1: the momentum accumulators are 0. So we should see a normal - # update: v -= grad * learning_rate - self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([0, 0]), - self.evaluate(slot0)[0]) - self.assertAllCloseAccordingToType( - np.array([-2.0 * .1, -2.0 * .1]), - self.evaluate(slot0)[1]) - self.assertAllCloseAccordingToType( - np.array([-2.0 * .01, -2.0 * .01]), - self.evaluate(slot1)[2]) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([0, 0]), - self.evaluate(var0)[0]) - self.assertAllCloseAccordingToType( - np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), - self.evaluate(var0)[1]) - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), - self.evaluate(var1)[2]) - # Step 2: the momentum accumulators contain the previous update. - if context.executing_eagerly(): - mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - else: + # Fetch params to validate initial values + self.assertAllClose([0, 0], self.evaluate(var0)[0]) + self.assertAllClose([0, 0], self.evaluate(var0)[1]) + self.assertAllClose([1, 1], self.evaluate(var1)[2]) + + # Step 1: the momentum accumulators are 0. So we should see a normal + # update: v -= grad * learning_rate self.evaluate(mom_update) - # Check that the momentum accumulators have been updated. - self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0]) - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), - self.evaluate(slot0)[1]) - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.02) - 2.0 * 0.01), - (0.9 * (-0.02) - 2.0 * 0.01)]), - self.evaluate(slot1)[2]) - # Check that the parameters have been updated. - self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0]) - self.assertAllCloseAccordingToType( - np.array([ - -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), - -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), - self.evaluate(var0)[1]) - self.assertAllCloseAccordingToType( - np.array([ - 0.98 - ((0.9 * 0.01 + 0.01) * 2.0), - 0.98 - ((0.9 * 0.01 + 0.01) * 2.0) - ]), - self.evaluate(var1)[2]) + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([0, 0]), + self.evaluate(slot0)[0]) + self.assertAllCloseAccordingToType( + np.array([-2.0 * .1, -2.0 * .1]), + self.evaluate(slot0)[1]) + self.assertAllCloseAccordingToType( + np.array([-2.0 * .01, -2.0 * .01]), + self.evaluate(slot1)[2]) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([0, 0]), + self.evaluate(var0)[0]) + self.assertAllCloseAccordingToType( + np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), + self.evaluate(var0)[1]) + self.assertAllCloseAccordingToType( + np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), + self.evaluate(var1)[2]) + # Step 2: the momentum accumulators contain the previous update. + self.evaluate(mom_update) + # Check that the momentum accumulators have been updated. + self.assertAllClose(np.array([0, 0]), self.evaluate(slot0)[0]) + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), + self.evaluate(slot0)[1]) + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), + self.evaluate(slot1)[2]) + # Check that the parameters have been updated. + self.assertAllClose(np.array([0, 0]), self.evaluate(var0)[0]) + self.assertAllCloseAccordingToType( + np.array([ + -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), + -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) + ]), + self.evaluate(var0)[1]) + self.assertAllCloseAccordingToType( + np.array([ + 0.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 0.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), + self.evaluate(var1)[2]) def testSharing(self): - for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: - var0 = variables.Variable([1.0, 2.0], dtype=dtype) - var1 = variables.Variable([3.0, 4.0], dtype=dtype) - grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) - grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) - mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) - if not context.executing_eagerly(): + # TODO(tanzheny, omalleyt): Fix test in eager mode. + with ops.Graph().as_default(): + for dtype in [dtypes.half, dtypes.float32, dtypes.float64]: + var0 = variables.Variable([1.0, 2.0], dtype=dtype) + var1 = variables.Variable([3.0, 4.0], dtype=dtype) + grads0 = constant_op.constant([0.1, 0.1], dtype=dtype) + grads1 = constant_op.constant([0.01, 0.01], dtype=dtype) + mom_opt = gradient_descent.SGD(learning_rate=2.0, momentum=0.9) mom_update1 = mom_opt.apply_gradients( zip([grads0, grads1], [var0, var1])) mom_update2 = mom_opt.apply_gradients( @@ -599,52 +613,46 @@ class MomentumOptimizerTest(test.TestCase, parameterized.TestCase): slot1 = mom_opt.get_slot(var1, "momentum") self.assertEqual(slot1.shape, var1.shape) - # Step 1: the momentum accumulators where 0. So we should see a normal - # update: v -= grad * learning_rate - if context.executing_eagerly(): - mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1])) - slot0 = mom_opt.get_slot(var0, "momentum") - self.assertEqual(slot0.shape, var0.shape) - slot1 = mom_opt.get_slot(var1, "momentum") - self.assertEqual(slot1.shape, var1.shape) - else: + # Fetch params to validate initial values + self.assertAllClose([1.0, 2.0], self.evaluate(var0)) + self.assertAllClose([3.0, 4.0], self.evaluate(var1)) + # Step 1: the momentum accumulators where 0. So we should see a normal + # update: v -= grad * learning_rate self.evaluate(mom_update1) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([-0.2, -0.2]), self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([-0.02, -0.02]), self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), - self.evaluate(var1)) - # Step 2: the second momentum accumulators contain the previous update. - if context.executing_eagerly(): - mom_update2 = mom_opt.apply_gradients( - zip([grads0, grads1], [var0, var1])) - else: + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([-0.2, -0.2]), self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([-0.02, -0.02]), self.evaluate(slot1)) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), + self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), + self.evaluate(var1)) + # Step 2: the second momentum accumulators contain the previous update. self.evaluate(mom_update2) - # Check that the momentum accumulators have been updated. - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), - self.evaluate(slot0)) - self.assertAllCloseAccordingToType( - np.array([(0.9 * (-0.02) - 2.0 * 0.01), - (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) - # Check that the parameters have been updated. - self.assertAllCloseAccordingToType( - np.array([ - 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), - 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) - ]), self.evaluate(var0)) - self.assertAllCloseAccordingToType( - np.array([ - 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), - 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) - ]), self.evaluate(var1)) + # Check that the momentum accumulators have been updated. + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.2) - 2.0 * 0.1), (0.9 * (-0.2) - 2.0 * 0.1)]), + self.evaluate(slot0)) + self.assertAllCloseAccordingToType( + np.array([(0.9 * (-0.02) - 2.0 * 0.01), + (0.9 * (-0.02) - 2.0 * 0.01)]), self.evaluate(slot1)) + # Check that the parameters have been updated. + self.assertAllCloseAccordingToType( + np.array([ + 1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), + 2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0) + ]), self.evaluate(var0)) + self.assertAllCloseAccordingToType( + np.array([ + 2.98 - ((0.9 * 0.01 + 0.01) * 2.0), + 3.98 - ((0.9 * 0.01 + 0.01) * 2.0) + ]), self.evaluate(var1)) + @combinations.generate(combinations.combine(mode=["graph", "eager"])) def testConfig(self): opt = gradient_descent.SGD(learning_rate=1.0, momentum=0.9, nesterov=True) config = opt.get_config() From f6741ff2261e8d8112736d2443f1c580762ceecc Mon Sep 17 00:00:00 2001 From: bigcat-himax Date: Tue, 4 Aug 2020 10:06:05 +0800 Subject: [PATCH 0321/1017] TFLM:update HIMAX_WE1_SDK_URL --- .../tools/make/third_party_downloads.inc | 59 ++++++++++--------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/tensorflow/lite/micro/tools/make/third_party_downloads.inc b/tensorflow/lite/micro/tools/make/third_party_downloads.inc index d4d5c1c73be..e2777d9fbb5 100644 --- a/tensorflow/lite/micro/tools/make/third_party_downloads.inc +++ b/tensorflow/lite/micro/tools/make/third_party_downloads.inc @@ -1,65 +1,70 @@ # Add URLs and MD5 checksums for third-party libraries here. +# We use mirror.tensorflow.org to cache copies of third-party files, +# but this is just an optimization applied manually by TensorFlow +# engineers, so add non-mirrored URLs if you need to update this +# in a pull request and we'll periodically copy them and update +# the URL. GEMMLOWP_URL := "https://github.com/google/gemmlowp/archive/719139ce755a0f31cbf1c37f7f98adcc7fc9f425.zip" GEMMLOWP_MD5 := "7e8191b24853d75de2af87622ad293ba" ifeq ($(HOST_OS),windows) - FLATBUFFERS_URL := "https://github.com/google/flatbuffers/archive/v1.12.0.zip" + FLATBUFFERS_URL := "http://mirror.tensorflow.org/github.com/google/flatbuffers/archive/v1.12.0.zip" FLATBUFFERS_MD5 := "a1afdbf114dec01a861c1b8c917d0fc7" else - FLATBUFFERS_URL := "https://github.com/google/flatbuffers/archive/v1.12.0.tar.gz" + FLATBUFFERS_URL := "http://mirror.tensorflow.org/github.com/google/flatbuffers/archive/v1.12.0.tar.gz" FLATBUFFERS_MD5 := "c62ffefb3d4548b127cca14ce047f16c" endif ifeq ($(HOST_OS),osx) - GCC_EMBEDDED_URL := "https://developer.arm.com/-/media/Files/downloads/gnu-rm/7-2018q2/gcc-arm-none-eabi-7-2018-q2-update-mac.tar.bz2" + GCC_EMBEDDED_URL := "http://mirror.tensorflow.org/developer.arm.com/-/media/Files/downloads/gnu-rm/7-2018q2/gcc-arm-none-eabi-7-2018-q2-update-mac.tar.bz2" GCC_EMBEDDED_MD5 := "a66be9828cf3c57d7d21178e07cd8904" else ifeq ($(HOST_OS),windows) - GCC_EMBEDDED_URL := "https://developer.arm.com/-/media/Files/downloads/gnu-rm/7-2018q2/gcc-arm-none-eabi-7-2018-q2-update-win32.zip" + GCC_EMBEDDED_URL := "http://mirror.tensorflow.org/developer.arm.com/-/media/Files/downloads/gnu-rm/7-2018q2/gcc-arm-none-eabi-7-2018-q2-update-win32.zip" GCC_EMBEDDED_MD5 := "bc8ae26d7c429f30d583a605a4bcf9bc" else - GCC_EMBEDDED_URL := "https://developer.arm.com/-/media/Files/downloads/gnu-rm/7-2018q2/gcc-arm-none-eabi-7-2018-q2-update-linux.tar.bz2" + GCC_EMBEDDED_URL := "http://mirror.tensorflow.org/developer.arm.com/-/media/Files/downloads/gnu-rm/7-2018q2/gcc-arm-none-eabi-7-2018-q2-update-linux.tar.bz2" GCC_EMBEDDED_MD5 := "299ebd3f1c2c90930d28ab82e5d8d6c0" endif -LEON_BCC2_URL := "https://www.gaisler.com/anonftp/bcc2/bin/bcc-2.0.7-gcc-linux64.tar.xz" +LEON_BCC2_URL := "http://mirror.tensorflow.org/www.gaisler.com/anonftp/bcc2/bin/bcc-2.0.7-gcc-linux64.tar.xz" LEON_BCC2_MD5 := "cdf78082be4882da2a92c9baa82fe765" -TSIM_URL := "https://www.gaisler.com/anonftp/tsim/tsim-eval-2.0.63.tar.gz" +TSIM_URL := "http://mirror.tensorflow.org/www.gaisler.com/anonftp/tsim/tsim-eval-2.0.63.tar.gz" TSIM_MD5 := "afa0095d3ed989a949e1467f94e41d2f" -CMSIS_URL := "https://github.com/ARM-software/CMSIS_5/archive/9daaa7a34a5627a24009462b8fa8413a00c4fdb1.zip" +CMSIS_URL := "http://mirror.tensorflow.org/github.com/ARM-software/CMSIS_5/archive/9daaa7a34a5627a24009462b8fa8413a00c4fdb1.zip" CMSIS_MD5 := "b988dacff8925ffffcb7e5079cc713b7" -AM_SDK_URL := "http://s3.asia.ambiqmicro.com/downloads/AmbiqSuite-Rel2.2.0.zip" +AM_SDK_URL := "http://mirror.tensorflow.org/s3.asia.ambiqmicro.com/downloads/AmbiqSuite-Rel2.2.0.zip" AM_SDK_MD5 := "7605fa2d4d97e6bb7a1190c92b66b597" AM_SDK_DEST := AmbiqSuite-Rel2.2.0 -SF_BSPS_URL := "https://github.com/sparkfun/SparkFun_Apollo3_AmbiqSuite_BSPs/archive/v0.0.7.zip" +SF_BSPS_URL := "http://mirror.tensorflow.org/github.com/sparkfun/SparkFun_Apollo3_AmbiqSuite_BSPs/archive/v0.0.7.zip" SF_BSPS_MD5 := "34199f7e754735661d1c8a70a40ca7a3" SF_BSPS_DEST := boards_sfe -STM32_BARE_LIB_URL := "https://github.com/google/stm32_bare_lib/archive/c07d611fb0af58450c5a3e0ab4d52b47f99bc82d.zip" +STM32_BARE_LIB_URL := "http://mirror.tensorflow.org/github.com/google/stm32_bare_lib/archive/c07d611fb0af58450c5a3e0ab4d52b47f99bc82d.zip" STM32_BARE_LIB_MD5 := "282bff40d4d0b92278fd123a3b6e3123" ifeq ($(HOST_OS),osx) - RISCV_TOOLCHAIN_URL := "https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.1.0-2019.01.0-x86_64-apple-darwin.tar.gz" + RISCV_TOOLCHAIN_URL := "http://mirror.tensorflow.org/static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-8.1.0-2019.01.0-x86_64-apple-darwin.tar.gz" RISCV_TOOLCHAIN_MD5 := "2ac2fa00618b9ab7fa0c7d0ec173de94" else - RISCV_TOOLCHAIN_URL := "https://static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-20181030-x86_64-linux-ubuntu14.tar.gz" + RISCV_TOOLCHAIN_URL := "http://mirror.tensorflow.org/static.dev.sifive.com/dev-tools/riscv64-unknown-elf-gcc-20181030-x86_64-linux-ubuntu14.tar.gz" RISCV_TOOLCHAIN_MD5="2366b7afe36a54dc94fb0ff8a0830934" endif -SIFIVE_FE310_LIB_URL := "https://github.com/sifive/freedom-e-sdk/archive/baeeb8fd497a99b3c141d7494309ec2e64f19bdf.zip" +SIFIVE_FE310_LIB_URL := "http://mirror.tensorflow.org/github.com/sifive/freedom-e-sdk/archive/baeeb8fd497a99b3c141d7494309ec2e64f19bdf.zip" SIFIVE_FE310_LIB_MD5 := "06ee24c4956f8e21670ab3395861fe64" -KISSFFT_URL="https://github.com/mborgerding/kissfft/archive/v130.zip" +KISSFFT_URL="http://mirror.tensorflow.org/github.com/mborgerding/kissfft/archive/v130.zip" KISSFFT_MD5="438ba1fef5783cc5f5f201395cc477ca" -RUY_URL="https://github.com/google/ruy/archive/34ea9f4993955fa1ff4eb58e504421806b7f2e8f.zip" -RUY_MD5="18613212e9c01aba85c7d19010b194a9" +RUY_URL="https://github.com/google/ruy/archive/5bb02fbf90824c2eb6cd7418f766c593106a332b.zip" +RUY_MD5="c720b1743360259ac45809a321f8f26c" -CIFAR10_DATASET_URL="https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz" +CIFAR10_DATASET_URL="http://mirror.tensorflow.org/www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz" CIFAR10_DATASET_MD5="c32a1d4ab5d03f1284b67883e8d87530" IMAGE_RECOGNITION_MODEL_URL := "https://storage.googleapis.com/download.tensorflow.org/models/tflite/cifar_image_recognition_model_2020_05_27.zip" @@ -68,25 +73,23 @@ IMAGE_RECOGNITION_MODEL_MD5 := "1f4607b05ac45b8a6146fb883dbc2d7b" PERSON_MODEL_URL := "https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_grayscale_2020_05_27.zip" PERSON_MODEL_MD5 := "55b85f76e2995153e660391d4a209ef1" -PERSON_MODEL_INT8_URL := "https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_int8_grayscale_2020_05_27.zip" -PERSON_MODEL_INT8_MD5 := "a0ede2d058aa2a1d413893455dd55352" +PERSON_MODEL_INT8_URL := "https://storage.googleapis.com/download.tensorflow.org/data/tf_lite_micro_person_data_int8_grayscale_2020_06_23.zip" +PERSON_MODEL_INT8_MD5 := "9b5b6d4677dd0a91b1bb992d1c4c0417" -EMBARC_MLI_URL := "https://github.com/foss-for-synopsys-dwc-arc-processors/embarc_mli/archive/58284867ca52d1f43b25045e8601999d7359d986.zip" +EMBARC_MLI_URL := "http://mirror.tensorflow.org/github.com/foss-for-synopsys-dwc-arc-processors/embarc_mli/archive/58284867ca52d1f43b25045e8601999d7359d986.zip" EMBARC_MLI_MD5 := "2bf4982a327fdaa9d475803ce014d1ef" -EMBARC_MLI_PRE_COMPILED_URL := "https://github.com/foss-for-synopsys-dwc-arc-processors/embarc_mli/releases/download/Release_1.1_RC2/embARC_MLI_package.zip" +EMBARC_MLI_PRE_COMPILED_URL := "http://mirror.tensorflow.org/github.com/foss-for-synopsys-dwc-arc-processors/embarc_mli/releases/download/Release_1.1_RC2/embARC_MLI_package.zip" EMBARC_MLI_PRE_COMPILED_MD5 := "a95ff9e0370434484f14e7e4114327f6" -ZEPHYR_URL := "https://github.com/antmicro/zephyr/archive/55e36b9.zip" +ZEPHYR_URL := "http://mirror.tensorflow.org/github.com/antmicro/zephyr/archive/55e36b9.zip" ZEPHYR_MD5 := "755622eb4812fde918a6382b65d50c3b" -XTENSA_HIFI4_URL :="https://github.com/foss-xtensa/nnlib-hifi4/raw/master/archive/xa_nnlib_04_07.zip" -XTENSA_HIFI4_MD5 :="f234764928f9a42901df33a27e118c8b" +XTENSA_HIFI4_URL :="http://mirror.tensorflow.org/github.com/foss-xtensa/nnlib-hifi4/raw/master/archive/xa_nnlib_06_27.zip" +XTENSA_HIFI4_MD5 :="45fdc1209a8da62ab568aa6040f7eabf" -ETHOSU_URL := "https://git.mlplatform.org/ml/ethos-u/ethos-u-core-driver.git/snapshot/ethos-u-core-driver-bcb5aaa99756f1b5c1295b079ebdd60996bc75a5.tar.gz" +ETHOSU_URL := "http://mirror.tensorflow.org/git.mlplatform.org/ml/ethos-u/ethos-u-core-driver.git/snapshot/ethos-u-core-driver-bcb5aaa99756f1b5c1295b079ebdd60996bc75a5.tar.gz" ETHOSU_MD5 := "d2073c8d88fc167fd5c46b5dcda58ea1" HIMAX_WE1_SDK_URL ="https://www.himax.com.tw/we-i/himax_we1_sdk_v03.zip" HIMAX_WE1_SDK_MD5 ="1cd9b17f3fdb3e9a1dfd1cc356694325" - - From ebbe7dd6c4f6385633dedae18301cf4ad95b93da Mon Sep 17 00:00:00 2001 From: Souradeep Nanda Date: Tue, 4 Aug 2020 07:47:34 +0530 Subject: [PATCH 0322/1017] Fixed indentation --- tensorflow/python/ops/custom_gradient.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py index f2675b422ac..ebbb0e4aede 100644 --- a/tensorflow/python/ops/custom_gradient.py +++ b/tensorflow/python/ops/custom_gradient.py @@ -139,23 +139,23 @@ def custom_gradient(f=None): the same number of variables. We take the function `z = x * y` as an example. >>> @tf.custom_gradient - def bar(x, y): - def grad(upstream): - dz_dx = y - dz_dy = x - return upstream * dz_dx, upstream * dz_dy - - z = x * y - - return z, grad + def bar(x, y): + def grad(upstream): + dz_dx = y + dz_dy = x + return upstream * dz_dx, upstream * dz_dy + + z = x * y + + return z, grad >>> x = tf.constant(2.0, dtype=tf.float32) >>> y = tf.constant(3.0, dtype=tf.float32) >>> with tf.GradientTape(persistent=True) as tape: - tape.watch(x) - tape.watch(y) - z = bar(x, y) + tape.watch(x) + tape.watch(y) + z = bar(x, y) >>> z 6 From 4457dcc8c57cf970ef9403bf367863e22eaa8d7d Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Mon, 3 Aug 2020 19:23:23 -0700 Subject: [PATCH 0323/1017] Introduce early documentation on mlir-hlo PiperOrigin-RevId: 324732753 Change-Id: I48c355931959b4aa05fc21cb643e48b58f11b92e --- tensorflow/compiler/mlir/hlo/README.md | 200 +++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 tensorflow/compiler/mlir/hlo/README.md diff --git a/tensorflow/compiler/mlir/hlo/README.md b/tensorflow/compiler/mlir/hlo/README.md new file mode 100644 index 00000000000..1be6fb29d13 --- /dev/null +++ b/tensorflow/compiler/mlir/hlo/README.md @@ -0,0 +1,200 @@ +# MLIR-HLO + +The code here exists in two places: + +* https://github.com/tensorflow/tensorflow/tree/master/tensorflow/compiler/mlir/hlo; + this is the canonical location and where contributions should be made using + GitHub pull-requests. +* https://github.com/tensorflow/mlir-hlo; this is a standalone repository with + a view to the same code to allow other projects to use this without + depending on the entire TF monorepo. + +This implements a self-contained compiler for a linear algebra set of operations +inspired by XLA +[HLO IR](https://www.tensorflow.org/xla/architecture#how_does_xla_work) using +MLIR components. It is designed to provide an end-to-end flow independent of +TensorFlow and XLA, but usable inside of these projects. + +Coding practice and conventions in this repository follow the +[MLIR Developer Guide](https://mlir.llvm.org/getting_started/DeveloperGuide/) in +this repo as part of the intent to act as an incubator for technology to +upstream. + +## QuickStart: building and testing + +TODO + +## Overview + +[XLA](https://www.tensorflow.org/xla/) (Accelerated Linear Algebra) is a +domain-specific compiler framework and execution environment for linear algebra, +which powers code-generation for ML frameworks like TensorFlow, JAX, and others. + +A cornerstone of XLA is the HLO (High Level Optimizer) IR, which offers a +carefully fixed selected list of operations, mostly orthogonal to each other. It +provides an efficient optimizer for computations expressed with this set of +operations and generate codes for hardware platforms like CPU, GPU, and TPUs. +Its goal is to provide a uniform interface to compile and execute these +optimized HLO programs independently of the targeted device. It is not a +front-end ML system like TensorFlow or JAX, rather it is a backend framework +that optimizes HLO and lowers to machine code. + +The HLO set of operations is closed and has well defined semantics. HLO +operations operate on immutable Tensors with static shapes (actually bounded +shapes to be exact) and explicit broadcasts. + +[MLIR](https://mlir.llvm.org/) is a compiler infrastructure which intends to +come with "battery included", as such it intends to provide all the blocks +required to assemble graph optimization and codegen pipelines. The longer term +roadmap for MLIR is to provide a +[Tensor Compute Primitive](https://llvm.discourse.group/c/mlir/MLIR-TCP-WG/36) +(TCP) dialect, which should hopefully be general enough to model what HLO +represents today (see +[slides](https://drive.google.com/open?id=1iljcpTQ5NPaMfGpoPDFml1XkYxjK_6A4) and +[recording](https://drive.google.com/open?id=1jSPa8TwPKUt0WuLquGc8OgSUVYJHMvWZ) +for a technical discussion on this topic). + +The work on MLIR-HLO can be seen as a stepping stone towards building TCP, while +integrating intermediate components into XLA itself by relying on the +well-proven HLO IR and introducing more pieces from upstream MLIR +([Linalg](https://mlir.llvm.org/docs/Dialects/Linalg/), +[Vector](https://mlir.llvm.org/docs/Dialects/Vector/), +[GPU](https://mlir.llvm.org/docs/Dialects/GPU/) dialect, ...). +[This document](https://www.tensorflow.org/mlir/xla_gpu_codegen) provides more +information on the current migration of the XLA GPU codegen. + +## MLIR Dialects for XLA-style compilation + +This repository defines three dialects to support a HLO-like compilation +pipeline using MLIR: + +* `chlo`: the "client" HLO dialect, intended to be closer to the frontend + (including implicit broadcast semantics). +* `mhlo`: "meta"-HLO dialect ; similar to `xla_hlo`, but with extensions for + dynamic shape support. +* `lmhlo`: "late"-"meta"-HLO, it is the IR after buffer allocation is + performed. In XLA the buffer allocation is a side-datastructure which keeps + track of these informations, while this separate dialect materializes it in + the IR. + +We describe these in more details below. + +### HLO Client Dialect: `chlo`. + +* It was originaly designed to map the + [XLA client APIs](https://www.tensorflow.org/xla/operation_semantics) (e.g., + ops supports implicit broadcast and roughly modeled on XlaBuilder API) + modulo support for dynamic shapes and additional ops required to support + dynamic client side HLOs. +* Ops can be from either the XlaBuilder or XLA helper functions can be + converted into ops (e.g., given ambiguity in what constitutes these ops, + there is some freedom to decide), the goal of this dialect is to correspond + close to client level and enable a thin layer between client use and op + construction (making it cheap to construct and optimizations on the dialect + close to optimizations on the client ops). + +Entry: + +* The vast majority of old "client" interactions are via the XlaBuilder APIs. + These APIs are used by TF2XLA kernels, JAX, PyTorch bridge and directly. The + legalization path (described below) can also reuse the XlaBuilder's APIs to + construct XLA Client HLO ops directly (this uses MlirXlaBuilder which is a + subclass of XlaBuilder). +* The other entry point is during legalization from TensorFlow ops in the TF + Graph Compiler and other tools (e.g., SavedModel lowering and TFCompile). + +Exit: + +* MHLO +* May be exported to xla::HloInstructionProto by invoking the XlaBuilder APIs + (with regular XlaBuilder) + +The `chlo` dialect started originally as mapping to the XLA client Builder APIs. +It enables it to both be constructed and converted back to existing XLA +interfaces using the XlaBuilder API. Due to the way that translation into and +out of the dialect works, there is no expectation that this dialect roundtrips +to XLA (e.g., it is only intended to be translated to MLIR and then legalized to +another dialect or translated to HloInstructionProto). + +The export approach of reusing the XlaBuilders enables reusing a lot of logic +that was already implemented in terms of computing shapes, inserting broadcasts +etc. + +An important topic here is that XLA Client HLO ops are not a well defined set. +And in particular what some would consider helper functions, others would +consider ops. It should be easy to move between these and so define a new op +along with the helper function or autogenerate the helper functions from the +descriptions of the ops. For the former, a simple approach would be to simply +consider the context in which the op is being constructed and if an MLIR one, +construct a op in the client dialect instead of further calls into XlaBuilder. +The latter could be implemented by adding the op and a legalization of the op to +other known ops, from which a helper function can get generated that could be +used as regular. + +Status: Exists but need to be cleaned up. + +### Meta HLO Dialect `mhlo` + +* Dialect is closer to current HLO server ops (e.g., no implicit broadcast) +* MHLO dialect where we can deviate from the requirements of the client or + server dialect, in particular: + * Control flow ops with implicit capture to enable simpler optimizations + (e.g., generic LICM, unroll & jam, etc.) + * Multiple results ops (e.g., no tuples) + * More ops (for example, unique op or assert op), and ops that don't need + to be added to either client or server dialect. + * Op set not constrained by implementation (e.g., hlo.add operating on say + i79 or !mydialect.weird_type is allowed even though no XLA backend + supports it). Verification on types happening at the boundaries. + * It does not need to preserve some deprecated XLA constructs (e.g. + stateful RNG HLO). + * More dynamic shape support ops without need for updating all + users/backends. +* This dialect enables evolving HLO independently from XLA in order to + experiment with features we'd like to upstream in MLIR TCP. In particular it + intends to be user-extensible through + [interfaces](https://mlir.llvm.org/docs/Interfaces/). +* It should have no TensorFlow, or proto, or other Google internal + dependencies. +* It need not be a complete superset of ops compared to XLA HLO dialect. + +Entry: + +* Legalization from `chlo` dialect or conversion from XLA HLO. +* Directly emitted from TF Graph Compiler; +* Builder call (e.g., EDSL); + +Exit: + +* LMHLO, Linalg IREE, directly used in codegen. +* XLA HLO. + +The MHLO dialect has no direct export format, it is only meant as an +intermediate optimization dialect/format. It is also where we can experiment +cheaply with new ops. This format will be where the representation would differ +from existing end points. + +Status: Exists but need to be cleaned up and evolved, in particular with respect +to supporting dynamic shapes. + +### LMHLO + +LMHLO corresponds to late `mhlo` and operates on buffer domain (e.g., memref) +with side-effecting operations. The lowering from `mhlo` dialect proceeds by way +of scheduling, memory and buffer allocation. The current mapping is directly on +XLA Client HLOs but without implicit broadcast and with operation on memrefs. +This dialect will instead be rebased on `mhlo` dialect but operating on buffers +still. + +Entry: + +* Post buffer assignment on `mhlo` dialect, or from XLA after buffer + assignment. + +Exit: + +* Codegen (LLVM IR in the common cases at the moment) + +## End-to-End pipeline + +TODO From 6a5edccf1d0a93a52f6e13a3ab0720f1e8562240 Mon Sep 17 00:00:00 2001 From: Souradeep Nanda Date: Tue, 4 Aug 2020 08:12:23 +0530 Subject: [PATCH 0324/1017] tidy up doctest --- tensorflow/python/ops/custom_gradient.py | 32 +++++++++++------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py index ebbb0e4aede..db5690f6e80 100644 --- a/tensorflow/python/ops/custom_gradient.py +++ b/tensorflow/python/ops/custom_gradient.py @@ -139,23 +139,21 @@ def custom_gradient(f=None): the same number of variables. We take the function `z = x * y` as an example. >>> @tf.custom_gradient - def bar(x, y): - def grad(upstream): - dz_dx = y - dz_dy = x - return upstream * dz_dx, upstream * dz_dy - - z = x * y - - return z, grad - - >>> x = tf.constant(2.0, dtype=tf.float32) - >>> y = tf.constant(3.0, dtype=tf.float32) - - >>> with tf.GradientTape(persistent=True) as tape: - tape.watch(x) - tape.watch(y) - z = bar(x, y) + ... def bar(x, y): + ... def grad(upstream): + ... dz_dx = y + ... dz_dy = x + ... return upstream * dz_dx, upstream * dz_dy + ... z = x * y + ... return z, grad + ... + ... x = tf.constant(2.0, dtype=tf.float32) + ... y = tf.constant(3.0, dtype=tf.float32) + ... + ... with tf.GradientTape(persistent=True) as tape: + ... tape.watch(x) + ... tape.watch(y) + ... z = bar(x, y) >>> z 6 From f4d60080b0b5f96907fe44af5e607b4cca9391f2 Mon Sep 17 00:00:00 2001 From: Souradeep Nanda Date: Tue, 4 Aug 2020 08:25:36 +0530 Subject: [PATCH 0325/1017] Fixed indentation errors --- tensorflow/python/ops/custom_gradient.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py index db5690f6e80..fc1f7f6fbc9 100644 --- a/tensorflow/python/ops/custom_gradient.py +++ b/tensorflow/python/ops/custom_gradient.py @@ -146,23 +146,18 @@ def custom_gradient(f=None): ... return upstream * dz_dx, upstream * dz_dy ... z = x * y ... return z, grad - ... - ... x = tf.constant(2.0, dtype=tf.float32) - ... y = tf.constant(3.0, dtype=tf.float32) - ... - ... with tf.GradientTape(persistent=True) as tape: + >>> x = tf.constant(2.0, dtype=tf.float32) + >>> y = tf.constant(3.0, dtype=tf.float32) + >>> with tf.GradientTape(persistent=True) as tape: ... tape.watch(x) ... tape.watch(y) ... z = bar(x, y) - >>> z - 6 + >>> tape.gradient(z, x) - 3 + >>> tape.gradient(z, y) - 2 - >>> tape.gradient(x, y) - None + Nesting custom gradients can lead to unintuitive results. The default behavior does not correspond to n-th order derivatives. For example From 90d864cd1d03d554014466495e4dec9164fb6c05 Mon Sep 17 00:00:00 2001 From: Blake Hechtman Date: Mon, 3 Aug 2020 19:50:45 -0700 Subject: [PATCH 0326/1017] [XLA] Allow kBitcast to also change types. PiperOrigin-RevId: 324735064 Change-Id: Iba6ca80b5511feeaa738334e6746a3dd7d53dea3 --- tensorflow/compiler/xla/service/BUILD | 1 + .../compiler/xla/service/hlo_creation_utils.cc | 6 ++++++ .../compiler/xla/service/hlo_instruction.cc | 2 +- .../compiler/xla/service/hlo_verifier.cc | 8 -------- .../compiler/xla/service/hlo_verifier_test.cc | 18 ------------------ 5 files changed, 8 insertions(+), 27 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 540cd7fecd2..4d15bc432a2 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -1705,6 +1705,7 @@ cc_library( ":hlo", ":hlo_module_config", ":shape_inference", + "//tensorflow/compiler/xla:comparison_util", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:statusor", diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc index 0f5267e9fbc..4ba67888409 100644 --- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc +++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/lib/comparators.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/client/xla_computation.h" +#include "tensorflow/compiler/xla/comparison_util.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/service/hlo_clone_context.h" @@ -258,6 +259,11 @@ HloInstruction* MakeBitcastConvertToHlo(HloInstruction* hlo, PrimitiveType type) { CHECK_NE(hlo->shape().element_type(), type); Shape shape = ShapeUtil::ChangeElementType(hlo->shape(), type); + // PRED are stored as one byte, PRED have a BitWidth of 1, avoid this problem + // by using a convert instead of bitcast convert. + if (type == PRED || hlo->shape().element_type() == PRED) { + return MakeConvertToHlo(hlo, type); + } hlo = hlo->parent()->AddInstruction( HloInstruction::CreateBitcastConvert(shape, hlo)); CHECK_EQ(hlo->shape().element_type(), type); diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 4335ed312c3..94d53ebe0b1 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -2207,7 +2207,7 @@ Status HloInstruction::ReplaceUsesWith(absl::Span users, Status HloInstruction::ReplaceAllUsesWithDifferentShape( absl::Span users, HloInstruction* new_producer) { for (HloInstruction* user : users) { - TF_RETURN_IF_ERROR(ReplaceUseWith(user, new_producer)); + TF_RETURN_IF_ERROR(ReplaceUseWithDifferentShape(user, new_producer)); } if (parent_ && parent_->root_instruction() == this) { diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 62b0d98418c..d395fddcc5d 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -670,14 +670,6 @@ Status ShapeVerifier::HandleReduce(HloInstruction* reduce) { } Status ShapeVerifier::HandleBitcast(HloInstruction* bitcast) { - // Bitcasts are not allowed to change the element type. - if (bitcast->operand(0)->shape().element_type() != - bitcast->shape().element_type()) { - return InternalError( - "Bitcast can not change the element type from %s to %s", - PrimitiveType_Name(bitcast->operand(0)->shape().element_type()), - PrimitiveType_Name(bitcast->shape().element_type())); - } if (layout_sensitive_ && shape_size_function_(bitcast->shape()) != shape_size_function_(bitcast->operand(0)->shape())) { diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc index d9709c50df9..1f71c9586d5 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc @@ -540,24 +540,6 @@ TEST_F(HloVerifierTestLayoutSensitive, ConcatWithLayoutChangeNotAllowed) { HasSubstr("Instruction shouldn't change layouts")); } -TEST_F(HloVerifierTest, BitcastCanNotChangeElementType) { - const char* const hlo_string = R"( - HloModule Module - - ENTRY BitcastCanNotChangeElementType { - constant.0 = f32[2] constant({0.0, 0.0}) - ROOT bitcast = s32[2] bitcast(constant.0) - } - )"; - TF_ASSERT_OK_AND_ASSIGN(auto module, - ParseAndReturnUnverifiedModule(hlo_string)); - - auto status = verifier().Run(module.get()).status(); - ASSERT_FALSE(status.ok()); - EXPECT_THAT(status.error_message(), - HasSubstr("Bitcast can not change the element type")); -} - TEST_F(HloVerifierTestLayoutSensitive, BitcastNeedsSameNumberOfElements) { const char* const hlo_string = R"( HloModule Module From 5297c4ada6c631125908cbbdaa389b85e9ca6f2f Mon Sep 17 00:00:00 2001 From: Souradeep Nanda Date: Tue, 4 Aug 2020 09:03:06 +0530 Subject: [PATCH 0327/1017] Fix pylint --- tensorflow/python/ops/custom_gradient.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py index fc1f7f6fbc9..f081f036b58 100644 --- a/tensorflow/python/ops/custom_gradient.py +++ b/tensorflow/python/ops/custom_gradient.py @@ -131,12 +131,14 @@ def custom_gradient(f=None): By chain rule we know that `dy/dx = dy/x_0 * dx_0/dx_1 * ... * dx_i/dx_i+1 * ... * dx_n/dx` - In this case the gradient of our current function defined as `dx_i/dx_i+1 = (1 - 1 / (1 + e))`. - The upstream gradient `dy` would be `dx_i+1/dx_i+2 * dx_i+2/dx_i+3 * ... * dx_n/dx`. - The upstream gradient multiplied by the current gradient is then passed downstream. + In this case the gradient of our current function defined as + `dx_i/dx_i+1 = (1 - 1 / (1 + e))`. The upstream gradient `dy` would be + `dx_i+1/dx_i+2 * dx_i+2/dx_i+3 * ... * dx_n/dx`. The upstream gradient + multiplied by the current gradient is then passed downstream. - In case the function takes multiple variables as input, the `grad` function must also return - the same number of variables. We take the function `z = x * y` as an example. + In case the function takes multiple variables as input, the `grad` + function must also return the same number of variables. + We take the function `z = x * y` as an example. >>> @tf.custom_gradient ... def bar(x, y): From 66dc5f61c9f19050133a1026780c9ae6d918af46 Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Mon, 3 Aug 2020 20:41:27 -0700 Subject: [PATCH 0328/1017] Fix the Windows build PiperOrigin-RevId: 324740036 Change-Id: I5bb28a33cf85393c41a6792e4238cc66b62944c3 --- tensorflow/compiler/tests/unary_ops_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py index e3a82610027..eb022da6895 100644 --- a/tensorflow/compiler/tests/unary_ops_test.py +++ b/tensorflow/compiler/tests/unary_ops_test.py @@ -801,6 +801,10 @@ class UnaryOpsTest(xla_test.XLATestCase): 2**31 - 1, 2**31, 2**32 - 1, 2**32, -2**32 + 1, -2**32, -2**63 + 1, 2**63 - 1 ] + # Only choose inputs which fit in the int dtype. + raw_inputs = list( + filter(lambda x: np.iinfo(dtype).min <= x <= np.iinfo(dtype).max, + raw_inputs)) inputs = np.array(raw_inputs, dtype=dtype) def count_bits(x): From d46875f207c1e45808b699bc3fb218392e4bf8a5 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Mon, 3 Aug 2020 20:46:54 -0700 Subject: [PATCH 0329/1017] Unify TF and XLA error messages for the SplitV op PiperOrigin-RevId: 324740588 Change-Id: I8bb30059cbf2c474087a040d786b632360f1ecb4 --- tensorflow/compiler/tf2xla/kernels/split_op.cc | 4 ++++ tensorflow/python/kernel_tests/split_op_test.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/tf2xla/kernels/split_op.cc b/tensorflow/compiler/tf2xla/kernels/split_op.cc index 7a0e240400b..dbaa84c223d 100644 --- a/tensorflow/compiler/tf2xla/kernels/split_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/split_op.cc @@ -105,6 +105,10 @@ class SplitVOp : public XlaOpKernel { const TensorShape input_shape = ctx->InputShape(0); const TensorShape index_shape = ctx->InputShape(2); + OP_REQUIRES(ctx, index_shape.num_elements() == 1, + errors::InvalidArgument( + "split_dim_tensor must have exactly one element.")); + int64 split_dim_orig; OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(2, &split_dim_orig)); int64 split_dim = split_dim_orig < 0 ? split_dim_orig + input_shape.dims() diff --git a/tensorflow/python/kernel_tests/split_op_test.py b/tensorflow/python/kernel_tests/split_op_test.py index ef66d8dda0b..16f92dbd875 100644 --- a/tensorflow/python/kernel_tests/split_op_test.py +++ b/tensorflow/python/kernel_tests/split_op_test.py @@ -373,7 +373,6 @@ class SplitOpTest(test.TestCase): assert s1.shape.as_list() == [1] @test_util.run_deprecated_v1 - @test_util.disable_xla("b/123337890") # Error messages differ def testNonexistentDimTensor(self): x = array_ops.placeholder(dtypes.int32) values = np.zeros([5, 30]) From 3a4385ca6943e626a82be44acccc37c7743e723a Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Mon, 3 Aug 2020 20:50:01 -0700 Subject: [PATCH 0330/1017] Address comment --- .../compiler/mlir/tensorflow/ir/tf_ops_n_z.cc | 13 +++---------- .../compiler/mlir/tensorflow/tests/tf-ops.mlir | 11 +++++++++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc index 0941345a76c..1275591e6ed 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc @@ -1763,11 +1763,7 @@ static LogicalResult Verify(TransposeOp op) { auto x_type = op.x().getType().dyn_cast(); auto y_type = op.y().getType().dyn_cast(); - if (!perm_type) { - return success(); - } - - if (perm_type.getRank() != 1) { + if (perm_type && perm_type.getRank() != 1) { return op.emitOpError() << "expected perm to be a 1-D Tensor, got perm of rank " << perm_type.getRank(); @@ -1780,7 +1776,7 @@ static LogicalResult Verify(TransposeOp op) { << y_type.getRank(); } - if (!x_type || !y_type || !perm_type.hasStaticShape()) { + if (!x_type || !y_type || !perm_type || !perm_type.hasStaticShape()) { return success(); } @@ -1801,10 +1797,7 @@ static LogicalResult Verify(TransposeOp op) { const int64_t y_dim = y_type.getDimSize(y_idx); const int64_t x_idx = e.value().getSExtValue(); const int64_t x_dim = x_type.getDimSize(x_idx); - if (y_dim == ShapedType::kDynamicSize || x_dim == ShapedType::kDynamicSize) { - continue; - } - if (y_dim != x_dim) { + if (y_dim != ShapedType::kDynamicSize && x_dim != ShapedType::kDynamicSize && y_dim != x_dim) { return op.emitOpError() << "requires y.shape[" << y_idx << "] (" << y_dim << ") " << "to be equal to x.shape[perm[" << x_idx << "]] " diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir index 04469e69684..20a0e22c48e 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/tf-ops.mlir @@ -2055,6 +2055,17 @@ func @testTranspose(tensor<2x?xf32>) -> tensor { // ----- +// Test tf.Transpose with different partial unknown shape +// CHECK-LABEL: testTranspose +func @testTranspose(tensor<2x?x?xf32>) -> tensor<3x?x2xf32> { +^bb0(%arg0: tensor<2x?x?xf32>): + %cst = constant dense<[2, 1, 0]> : tensor<3xi32> + %0 = "tf.Transpose"(%arg0, %cst) {T = "tfdtype$DT_FLOAT", Tperm = "tfdtype$DT_INT32"} : (tensor<2x?x?xf32>, tensor<3xi32>) -> tensor<3x?x2xf32> + return %0 : tensor<3x?x2xf32> +} + +// ----- + // Test tf.Transpose with invalid rank of perm func @testTranspose(tensor<2x3xf32>, tensor<1x2xi32>) -> tensor<3x2xf32> { ^bb0(%arg0: tensor<2x3xf32>, %arg1: tensor<1x2xi32>): From be269ac3d2ca0d283747ba0fef3b7f680b2a87c8 Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Mon, 3 Aug 2020 21:15:52 -0700 Subject: [PATCH 0331/1017] Break too long line --- tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc index 1275591e6ed..b917be76500 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc @@ -1797,7 +1797,8 @@ static LogicalResult Verify(TransposeOp op) { const int64_t y_dim = y_type.getDimSize(y_idx); const int64_t x_idx = e.value().getSExtValue(); const int64_t x_dim = x_type.getDimSize(x_idx); - if (y_dim != ShapedType::kDynamicSize && x_dim != ShapedType::kDynamicSize && y_dim != x_dim) { + if (y_dim != ShapedType::kDynamicSize && + x_dim != ShapedType::kDynamicSize && y_dim != x_dim) { return op.emitOpError() << "requires y.shape[" << y_idx << "] (" << y_dim << ") " << "to be equal to x.shape[perm[" << x_idx << "]] " From befe7e95c6e4fd9758d506f1b9656ab74cb3c270 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 21:27:05 -0700 Subject: [PATCH 0332/1017] [XLA] Allow kBitcast to also change types. PiperOrigin-RevId: 324744627 Change-Id: Ie672116cf39d4bdef3a5dea788793ee5674a2f45 --- tensorflow/compiler/xla/service/BUILD | 1 - .../compiler/xla/service/hlo_creation_utils.cc | 6 ------ .../compiler/xla/service/hlo_instruction.cc | 2 +- .../compiler/xla/service/hlo_verifier.cc | 8 ++++++++ .../compiler/xla/service/hlo_verifier_test.cc | 18 ++++++++++++++++++ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 4d15bc432a2..540cd7fecd2 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -1705,7 +1705,6 @@ cc_library( ":hlo", ":hlo_module_config", ":shape_inference", - "//tensorflow/compiler/xla:comparison_util", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:statusor", diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc index 4ba67888409..0f5267e9fbc 100644 --- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc +++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc @@ -21,7 +21,6 @@ limitations under the License. #include "tensorflow/compiler/xla/client/lib/comparators.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/client/xla_computation.h" -#include "tensorflow/compiler/xla/comparison_util.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/service/hlo_clone_context.h" @@ -259,11 +258,6 @@ HloInstruction* MakeBitcastConvertToHlo(HloInstruction* hlo, PrimitiveType type) { CHECK_NE(hlo->shape().element_type(), type); Shape shape = ShapeUtil::ChangeElementType(hlo->shape(), type); - // PRED are stored as one byte, PRED have a BitWidth of 1, avoid this problem - // by using a convert instead of bitcast convert. - if (type == PRED || hlo->shape().element_type() == PRED) { - return MakeConvertToHlo(hlo, type); - } hlo = hlo->parent()->AddInstruction( HloInstruction::CreateBitcastConvert(shape, hlo)); CHECK_EQ(hlo->shape().element_type(), type); diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 94d53ebe0b1..4335ed312c3 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -2207,7 +2207,7 @@ Status HloInstruction::ReplaceUsesWith(absl::Span users, Status HloInstruction::ReplaceAllUsesWithDifferentShape( absl::Span users, HloInstruction* new_producer) { for (HloInstruction* user : users) { - TF_RETURN_IF_ERROR(ReplaceUseWithDifferentShape(user, new_producer)); + TF_RETURN_IF_ERROR(ReplaceUseWith(user, new_producer)); } if (parent_ && parent_->root_instruction() == this) { diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index d395fddcc5d..62b0d98418c 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -670,6 +670,14 @@ Status ShapeVerifier::HandleReduce(HloInstruction* reduce) { } Status ShapeVerifier::HandleBitcast(HloInstruction* bitcast) { + // Bitcasts are not allowed to change the element type. + if (bitcast->operand(0)->shape().element_type() != + bitcast->shape().element_type()) { + return InternalError( + "Bitcast can not change the element type from %s to %s", + PrimitiveType_Name(bitcast->operand(0)->shape().element_type()), + PrimitiveType_Name(bitcast->shape().element_type())); + } if (layout_sensitive_ && shape_size_function_(bitcast->shape()) != shape_size_function_(bitcast->operand(0)->shape())) { diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc index 1f71c9586d5..d9709c50df9 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc @@ -540,6 +540,24 @@ TEST_F(HloVerifierTestLayoutSensitive, ConcatWithLayoutChangeNotAllowed) { HasSubstr("Instruction shouldn't change layouts")); } +TEST_F(HloVerifierTest, BitcastCanNotChangeElementType) { + const char* const hlo_string = R"( + HloModule Module + + ENTRY BitcastCanNotChangeElementType { + constant.0 = f32[2] constant({0.0, 0.0}) + ROOT bitcast = s32[2] bitcast(constant.0) + } + )"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnUnverifiedModule(hlo_string)); + + auto status = verifier().Run(module.get()).status(); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.error_message(), + HasSubstr("Bitcast can not change the element type")); +} + TEST_F(HloVerifierTestLayoutSensitive, BitcastNeedsSameNumberOfElements) { const char* const hlo_string = R"( HloModule Module From 486296671c8302af98edc17effa3edcb28931b58 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Mon, 3 Aug 2020 22:03:17 -0700 Subject: [PATCH 0333/1017] Fix tf2xla error message in the ConcatOffset and MatrixDiagPart ops PiperOrigin-RevId: 324748010 Change-Id: I22846e9fe5d30049f2272d7b8c9140c18ea00da8 --- tensorflow/compiler/tf2xla/kernels/concat_op.cc | 8 +++++--- tensorflow/compiler/tf2xla/kernels/matrix_diag_ops.cc | 5 +++-- tensorflow/python/kernel_tests/concat_op_test.py | 1 - tensorflow/python/kernel_tests/diag_op_test.py | 2 -- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/concat_op.cc b/tensorflow/compiler/tf2xla/kernels/concat_op.cc index 09c97de13eb..d0f24b5f561 100644 --- a/tensorflow/compiler/tf2xla/kernels/concat_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/concat_op.cc @@ -186,9 +186,11 @@ class ConcatOffsetOp : public XlaOpKernel { const int32 inp0_element = inp0_dims[j]; const int32 inp_element = inp_dims[j]; OP_REQUIRES(ctx, inp0_element == inp_element, - errors::InvalidArgument("input[", i, ",", j, - "] mismatch: ", inp0_element, - " vs. ", inp_element)); + errors::InvalidArgument( + "All dimensions except ", axis, " must match. Input ", + i, " has shape [", absl::StrJoin(inp_dims, " "), + "] and doesn't match input 0 with shape [", + absl::StrJoin(inp0_dims, " "), "].")); out_vec(j) = 0; } } diff --git a/tensorflow/compiler/tf2xla/kernels/matrix_diag_ops.cc b/tensorflow/compiler/tf2xla/kernels/matrix_diag_ops.cc index 57e961917cc..c8da75157fc 100644 --- a/tensorflow/compiler/tf2xla/kernels/matrix_diag_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/matrix_diag_ops.cc @@ -243,8 +243,9 @@ class MatrixDiagOp : public XlaOpKernel { errors::InvalidArgument("MatrixDiag op must have at least one input")); const TensorShape diag_shape = context->InputShape(0); OP_REQUIRES(context, TensorShapeUtils::IsVectorOrHigher(diag_shape), - errors::InvalidArgument("Expected >= 1 dims, got shape ", - diag_shape.DebugString())); + errors::InvalidArgument( + "diagonal must be at least 1-dim, received shape: ", + diag_shape.DebugString())); const DataType dtype = context->expected_output_dtype(0); const xla::XlaOp zero = XlaHelpers::Zero(context->builder(), dtype); diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py index 8d05b278aa6..334e25cfc4e 100644 --- a/tensorflow/python/kernel_tests/concat_op_test.py +++ b/tensorflow/python/kernel_tests/concat_op_test.py @@ -701,7 +701,6 @@ class ConcatOffsetTest(test.TestCase): self.evaluate(off) @test_util.run_deprecated_v1 - @test_util.disable_xla("b/123337890") # Error messages differ def testSizeMismatch(self): cdim = constant_op.constant(1, dtypes.int32) s0 = constant_op.constant([2, 3, 5], dtypes.int32) diff --git a/tensorflow/python/kernel_tests/diag_op_test.py b/tensorflow/python/kernel_tests/diag_op_test.py index 9c679ff34c9..8e8586b88d1 100644 --- a/tensorflow/python/kernel_tests/diag_op_test.py +++ b/tensorflow/python/kernel_tests/diag_op_test.py @@ -541,7 +541,6 @@ class MatrixDiagTest(test.TestCase): array_ops.matrix_diag(0) @test_util.run_deprecated_v1 - @test_util.disable_xla("b/123337890") # Error messages differ def testInvalidShapeAtEval(self): with self.session(use_gpu=True): v = array_ops.placeholder(dtype=dtypes_lib.float32) @@ -891,7 +890,6 @@ class MatrixDiagPartTest(test.TestCase): array_ops.matrix_diag_part(0) @test_util.run_deprecated_v1 - @test_util.disable_xla("b/123337890") # Error messages differ def testInvalidShapeAtEval(self): with self.session(use_gpu=True): v = array_ops.placeholder(dtype=dtypes_lib.float32) From 6032b6a8888f5a1a38a7a719b4086dd5a7041061 Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Mon, 3 Aug 2020 22:12:15 -0700 Subject: [PATCH 0334/1017] [tf.data service] Write dispatcher state changes to a journal. When a work directory is configured, the dispatcher will write journal entries for its state changes to a journal within the work directory. If no work directory is configured, the dispatcher uses a NoopJournalWriter, which writes nothing. We don't yet read from the journal on dispatcher start. This support will be added in the next CL. PiperOrigin-RevId: 324749061 Change-Id: I0baf10cef05a53e0fa94139d84d5cfd284550acf --- tensorflow/core/data/service/BUILD | 3 ++ .../core/data/service/dispatcher_impl.cc | 26 +++++++-- .../core/data/service/dispatcher_impl.h | 3 ++ .../core/data/service/dispatcher_state.cc | 4 +- .../core/data/service/dispatcher_state.h | 3 +- .../data/service/dispatcher_state_test.cc | 16 ++++-- tensorflow/core/data/service/journal.cc | 16 +++--- tensorflow/core/data/service/journal.h | 54 ++++++++++++++----- tensorflow/core/data/service/journal_test.cc | 12 ++--- .../data/experimental/service_config.proto | 3 ++ 10 files changed, 103 insertions(+), 37 deletions(-) diff --git a/tensorflow/core/data/service/BUILD b/tensorflow/core/data/service/BUILD index 55dccdf080b..19fe0263df2 100644 --- a/tensorflow/core/data/service/BUILD +++ b/tensorflow/core/data/service/BUILD @@ -61,6 +61,7 @@ cc_library( ":dispatcher_proto_cc", ":dispatcher_state", ":grpc_util", + ":journal", ":worker_cc_grpc_proto", ":worker_proto_cc", "//tensorflow/c:c_api_internal", @@ -87,6 +88,7 @@ cc_library( deps = [ ":common_proto_cc", ":data_service", + ":journal", ":journal_proto_cc", "//tensorflow/core:lib", "@com_google_absl//absl/container:flat_hash_map", @@ -100,6 +102,7 @@ tf_cc_test( deps = [ ":common_proto_cc", ":dispatcher_state", + ":journal", ":journal_proto_cc", "//tensorflow/core:lib", "//tensorflow/core:test", diff --git a/tensorflow/core/data/service/dispatcher_impl.cc b/tensorflow/core/data/service/dispatcher_impl.cc index 7a20f553b19..4bc4d409fd7 100644 --- a/tensorflow/core/data/service/dispatcher_impl.cc +++ b/tensorflow/core/data/service/dispatcher_impl.cc @@ -28,11 +28,13 @@ limitations under the License. #include "tensorflow/core/data/service/data_service.h" #include "tensorflow/core/data/service/dispatcher.pb.h" #include "tensorflow/core/data/service/grpc_util.h" +#include "tensorflow/core/data/service/journal.h" #include "tensorflow/core/data/service/worker.grpc.pb.h" #include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/kernels/data/dataset_utils.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/path.h" #include "tensorflow/core/protobuf/data/experimental/service_config.pb.h" #include "tensorflow/core/public/session_options.h" @@ -40,6 +42,9 @@ namespace tensorflow { namespace data { namespace { +// The name of the journal directory inside the dispatcher's working directory. +constexpr StringPiece kJournalDir = "journal"; + using Dataset = DispatcherState::Dataset; using NamedJobKey = DispatcherState::NamedJobKey; using Job = DispatcherState::Job; @@ -59,7 +64,15 @@ Status CreateWorkerStub(const std::string& address, const std::string& protocol, DataServiceDispatcherImpl::DataServiceDispatcherImpl( const experimental::DispatcherConfig& config) - : config_(config) {} + : config_(config) { + if (config_.work_dir().empty()) { + journal_writer_ = absl::make_unique(); + } else { + std::string journal_dir = io::JoinPath(config_.work_dir(), kJournalDir); + journal_writer_ = + absl::make_unique(Env::Default(), journal_dir); + } +} Status DataServiceDispatcherImpl::RegisterWorker( const RegisterWorkerRequest* request, RegisterWorkerResponse* response) { @@ -124,7 +137,7 @@ Status DataServiceDispatcherImpl::WorkerUpdate( Update update; FinishJobUpdate* finish_job = update.mutable_finish_job(); finish_job->set_job_id(task->job_id); - TF_RETURN_IF_ERROR(state_.Apply(update)); + TF_RETURN_IF_ERROR(Apply(update)); } VLOG(3) << "Task " << task_id << " from job " << task->job_id << " completed"; @@ -170,7 +183,7 @@ Status DataServiceDispatcherImpl::RegisterDataset(uint64 fingerprint, register_dataset->set_dataset_id(*dataset_id); register_dataset->set_fingerprint(fingerprint); *register_dataset->mutable_dataset_def() = dataset; - return state_.Apply(update); + return Apply(update); } Status DataServiceDispatcherImpl::CreateJob(const CreateJobRequest* request, @@ -278,7 +291,7 @@ Status DataServiceDispatcherImpl::CreateJob( key->set_name(named_job_key->name); key->set_index(named_job_key->index); } - TF_RETURN_IF_ERROR(state_.Apply(update)); + TF_RETURN_IF_ERROR(Apply(update)); TF_RETURN_IF_ERROR(state_.JobFromId(job_id, job)); return Status::OK(); } @@ -394,5 +407,10 @@ Status DataServiceDispatcherImpl::GetWorkers(const GetWorkersRequest* request, return Status::OK(); } +Status DataServiceDispatcherImpl::Apply(const Update& update) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return state_.Apply(update, journal_writer_.get()); +} + } // namespace data } // namespace tensorflow diff --git a/tensorflow/core/data/service/dispatcher_impl.h b/tensorflow/core/data/service/dispatcher_impl.h index f44f9e8d807..3e8b8dc6fbe 100644 --- a/tensorflow/core/data/service/dispatcher_impl.h +++ b/tensorflow/core/data/service/dispatcher_impl.h @@ -124,6 +124,8 @@ class DataServiceDispatcherImpl { Status ValidateMatchingJob(std::shared_ptr job, ProcessingMode processing_mode, int64 dataset_id) EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Applies a state update, updating both the journal and the in-memory state. + Status Apply(const Update& update) EXCLUSIVE_LOCKS_REQUIRED(mu_); const experimental::DispatcherConfig& config_; @@ -141,6 +143,7 @@ class DataServiceDispatcherImpl { absl::flat_hash_map>> tasks_by_job_ TF_GUARDED_BY(mu_); + std::unique_ptr journal_writer_ TF_GUARDED_BY(mu_); DispatcherState state_ TF_GUARDED_BY(mu_); TF_DISALLOW_COPY_AND_ASSIGN(DataServiceDispatcherImpl); diff --git a/tensorflow/core/data/service/dispatcher_state.cc b/tensorflow/core/data/service/dispatcher_state.cc index 2e6709b2287..f22672c4363 100644 --- a/tensorflow/core/data/service/dispatcher_state.cc +++ b/tensorflow/core/data/service/dispatcher_state.cc @@ -16,6 +16,7 @@ limitations under the License. #include +#include "tensorflow/core/data/service/journal.h" #include "tensorflow/core/data/service/journal.pb.h" #include "tensorflow/core/platform/errors.h" @@ -24,7 +25,8 @@ namespace data { DispatcherState::DispatcherState() {} -Status DispatcherState::Apply(Update update) { +Status DispatcherState::Apply(Update update, JournalWriter* journal_writer) { + TF_RETURN_IF_ERROR(journal_writer->Write(update)); switch (update.update_type_case()) { case Update::kRegisterDataset: RegisterDataset(update.register_dataset()); diff --git a/tensorflow/core/data/service/dispatcher_state.h b/tensorflow/core/data/service/dispatcher_state.h index 936558e55a2..1959afa61eb 100644 --- a/tensorflow/core/data/service/dispatcher_state.h +++ b/tensorflow/core/data/service/dispatcher_state.h @@ -18,6 +18,7 @@ limitations under the License. #include "absl/container/flat_hash_map.h" #include "tensorflow/core/data/service/common.pb.h" #include "tensorflow/core/data/service/data_service.h" +#include "tensorflow/core/data/service/journal.h" #include "tensorflow/core/data/service/journal.pb.h" #include "tensorflow/core/lib/core/status.h" @@ -60,7 +61,7 @@ class DispatcherState { DispatcherState& operator=(const DispatcherState&) = delete; // Applies the given update to the dispatcher's state. - Status Apply(Update update); + Status Apply(Update update, JournalWriter* journal_writer); // A dataset registered with the dispatcher. struct Dataset { diff --git a/tensorflow/core/data/service/dispatcher_state_test.cc b/tensorflow/core/data/service/dispatcher_state_test.cc index 0a943b2dd92..e1fd47805a7 100644 --- a/tensorflow/core/data/service/dispatcher_state_test.cc +++ b/tensorflow/core/data/service/dispatcher_state_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/core/data/service/dispatcher_state.h" #include "tensorflow/core/data/service/common.pb.h" +#include "tensorflow/core/data/service/journal.h" #include "tensorflow/core/data/service/journal.pb.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/errors.h" @@ -27,28 +28,31 @@ namespace data { namespace { Status RegisterDatasetWithIdAndFingerprint(int64 id, uint64 fingerprint, DispatcherState* state) { + NoopJournalWriter journal_writer; Update update; RegisterDatasetUpdate* register_dataset = update.mutable_register_dataset(); register_dataset->set_dataset_id(id); register_dataset->set_fingerprint(fingerprint); - TF_RETURN_IF_ERROR(state->Apply(update)); + TF_RETURN_IF_ERROR(state->Apply(update, &journal_writer)); return Status::OK(); } Status CreateAnonymousJob(int64 job_id, int64 dataset_id, DispatcherState* state) { + NoopJournalWriter journal_writer; Update update; CreateJobUpdate* create_job = update.mutable_create_job(); create_job->set_job_id(job_id); create_job->set_dataset_id(dataset_id); create_job->set_processing_mode(ProcessingModeDef::PARALLEL_EPOCHS); - TF_RETURN_IF_ERROR(state->Apply(update)); + TF_RETURN_IF_ERROR(state->Apply(update, &journal_writer)); return Status::OK(); } Status CreateNamedJob(int64 job_id, int64 dataset_id, DispatcherState::NamedJobKey named_job_key, DispatcherState* state) { + NoopJournalWriter journal_writer; Update update; CreateJobUpdate* create_job = update.mutable_create_job(); create_job->set_job_id(job_id); @@ -57,15 +61,16 @@ Status CreateNamedJob(int64 job_id, int64 dataset_id, NamedJobKeyDef* key = create_job->mutable_named_job_key(); key->set_name(named_job_key.name); key->set_index(named_job_key.index); - TF_RETURN_IF_ERROR(state->Apply(update)); + TF_RETURN_IF_ERROR(state->Apply(update, &journal_writer)); return Status::OK(); } Status FinishJob(int64 job_id, DispatcherState* state) { + NoopJournalWriter journal_writer; Update update; FinishJobUpdate* finish_job = update.mutable_finish_job(); finish_job->set_job_id(job_id); - TF_RETURN_IF_ERROR(state->Apply(update)); + TF_RETURN_IF_ERROR(state->Apply(update, &journal_writer)); return Status::OK(); } } // namespace @@ -112,9 +117,10 @@ TEST(DispatcherState, NextAvailableDatasetId) { } TEST(DispatcherState, UnknownUpdate) { + NoopJournalWriter journal_writer; DispatcherState state; Update update; - Status s = state.Apply(update); + Status s = state.Apply(update, &journal_writer); EXPECT_EQ(s.code(), error::INTERNAL); } diff --git a/tensorflow/core/data/service/journal.cc b/tensorflow/core/data/service/journal.cc index 9f2d4908f05..6856c69deb3 100644 --- a/tensorflow/core/data/service/journal.cc +++ b/tensorflow/core/data/service/journal.cc @@ -34,10 +34,10 @@ std::string DataServiceJournalFile(StringPiece journal_dir) { return io::JoinPath(journal_dir, kJournal); } -JournalWriter::JournalWriter(Env* env, StringPiece journal_dir) +FileJournalWriter::FileJournalWriter(Env* env, StringPiece journal_dir) : env_(env), journal_dir_(journal_dir) {} -Status JournalWriter::EnsureInitialized() { +Status FileJournalWriter::EnsureInitialized() { if (writer_) { return Status::OK(); } @@ -48,7 +48,7 @@ Status JournalWriter::EnsureInitialized() { return Status::OK(); } -Status JournalWriter::Write(Update update) { +Status FileJournalWriter::Write(Update update) { TF_RETURN_IF_ERROR(EnsureInitialized()); std::string s = update.SerializeAsString(); if (s.empty()) { @@ -61,10 +61,14 @@ Status JournalWriter::Write(Update update) { return Status::OK(); } -JournalReader::JournalReader(Env* env, StringPiece journal_dir) +NoopJournalWriter::NoopJournalWriter() {} + +Status NoopJournalWriter::Write(Update update) { return Status::OK(); } + +FileJournalReader::FileJournalReader(Env* env, StringPiece journal_dir) : env_(env), journal_dir_(journal_dir) {} -Status JournalReader::EnsureInitialized() { +Status FileJournalReader::EnsureInitialized() { if (reader_) { return Status::OK(); } @@ -74,7 +78,7 @@ Status JournalReader::EnsureInitialized() { return Status::OK(); } -Status JournalReader::Read(Update* update, bool* end_of_journal) { +Status FileJournalReader::Read(Update* update, bool* end_of_journal) { TF_RETURN_IF_ERROR(EnsureInitialized()); tstring record; Status s = reader_->ReadRecord(&offset_, &record); diff --git a/tensorflow/core/data/service/journal.h b/tensorflow/core/data/service/journal.h index b2d718ad652..f5b3e26ba18 100644 --- a/tensorflow/core/data/service/journal.h +++ b/tensorflow/core/data/service/journal.h @@ -27,19 +27,26 @@ namespace data { // Returns the location of the journal file within the journal directory. std::string DataServiceJournalFile(StringPiece journal_dir); -// JournalWriter is not thread-safe, requiring external synchronization when -// used by multiple threads. +// Interface for writing to a journal. class JournalWriter { + public: + virtual ~JournalWriter() = default; + // Writes and syncs an update to the journal. + virtual Status Write(Update update) = 0; +}; + +// FileJournalWriter is not thread-safe, requiring external synchronization when +// used by multiple threads. +class FileJournalWriter : public JournalWriter { public: // Creates a journal writer to write to the given journal directory. // If there is already journal data there, the journal writer will append to // the existing journal. - explicit JournalWriter(Env* env, StringPiece journal_dir); - JournalWriter(const JournalWriter&) = delete; - JournalWriter& operator=(const JournalWriter&) = delete; + explicit FileJournalWriter(Env* env, StringPiece journal_dir); + FileJournalWriter(const FileJournalWriter&) = delete; + FileJournalWriter& operator=(const FileJournalWriter&) = delete; - // Writes and syncs an update to the journal. - Status Write(Update update); + Status Write(Update update) override; private: // Initializes the writer if it is not yet initialized. @@ -51,17 +58,36 @@ class JournalWriter { std::unique_ptr writer_; }; -// JournalReader is not thread-safe, requiring external synchronization when -// used by multiple threads. +// NoopJournalWriter implements the JournalWriter interface, but doesn't +// actually write journal entries anywhere. +class NoopJournalWriter : public JournalWriter { + public: + // Creates a journal writer which does nothing. + explicit NoopJournalWriter(); + NoopJournalWriter(const NoopJournalWriter&) = delete; + NoopJournalWriter& operator=(const NoopJournalWriter&) = delete; + + Status Write(Update update) override; +}; + +// Interface for reading from a journal. class JournalReader { public: - explicit JournalReader(Env* env, StringPiece journal_dir); - JournalReader(const JournalReader&) = delete; - JournalReader& operator=(const JournalReader&) = delete; - + virtual ~JournalReader() = default; // Reads the next update from the journal. Sets `*end_of_journal=true` if // there are no more updates left in the journal. - Status Read(Update* update, bool* end_of_journal); + virtual Status Read(Update* update, bool* end_of_journal) = 0; +}; + +// JournalReader is not thread-safe, requiring external synchronization when +// used by multiple threads. +class FileJournalReader : public JournalReader { + public: + explicit FileJournalReader(Env* env, StringPiece journal_dir); + FileJournalReader(const FileJournalReader&) = delete; + FileJournalReader& operator=(const FileJournalReader&) = delete; + + Status Read(Update* update, bool* end_of_journal) override; private: // Initializes the reader if it is not yet initialized. diff --git a/tensorflow/core/data/service/journal_test.cc b/tensorflow/core/data/service/journal_test.cc index dc1006e280e..3c43cf763e9 100644 --- a/tensorflow/core/data/service/journal_test.cc +++ b/tensorflow/core/data/service/journal_test.cc @@ -63,7 +63,7 @@ Update MakeRegisterDatasetUpdate() { Status CheckJournalContent(StringPiece journal_dir, const std::vector& expected) { - JournalReader reader(Env::Default(), journal_dir); + FileJournalReader reader(Env::Default(), journal_dir); for (const auto& update : expected) { Update result; bool end_of_journal = true; @@ -87,7 +87,7 @@ TEST(Journal, RoundTripMultiple) { std::vector updates = {MakeCreateJobUpdate(), MakeRegisterDatasetUpdate(), MakeFinishJobUpdate()}; - JournalWriter writer(Env::Default(), journal_dir); + FileJournalWriter writer(Env::Default(), journal_dir); for (const auto& update : updates) { TF_EXPECT_OK(writer.Write(update)); } @@ -102,7 +102,7 @@ TEST(Journal, AppendExistingFile) { MakeRegisterDatasetUpdate(), MakeFinishJobUpdate()}; for (const auto& update : updates) { - JournalWriter writer(Env::Default(), journal_dir); + FileJournalWriter writer(Env::Default(), journal_dir); TF_EXPECT_OK(writer.Write(update)); } @@ -112,7 +112,7 @@ TEST(Journal, AppendExistingFile) { TEST(Journal, MissingFile) { std::string journal_dir; EXPECT_TRUE(NewJournalDir(&journal_dir)); - JournalReader reader(Env::Default(), journal_dir); + FileJournalReader reader(Env::Default(), journal_dir); Update result; bool end_of_journal = true; Status s = reader.Read(&result, &end_of_journal); @@ -131,7 +131,7 @@ TEST(Journal, NonRecordData) { TF_ASSERT_OK(file->Append("not record data")); } - JournalReader reader(Env::Default(), journal_dir); + FileJournalReader reader(Env::Default(), journal_dir); Update result; bool end_of_journal = true; Status s = reader.Read(&result, &end_of_journal); @@ -152,7 +152,7 @@ TEST(Journal, InvalidRecordData) { TF_ASSERT_OK(writer->WriteRecord("not serializd proto")); } - JournalReader reader(Env::Default(), journal_dir); + FileJournalReader reader(Env::Default(), journal_dir); Update result; bool end_of_journal = true; Status s = reader.Read(&result, &end_of_journal); diff --git a/tensorflow/core/protobuf/data/experimental/service_config.proto b/tensorflow/core/protobuf/data/experimental/service_config.proto index 8708b923720..872a47013eb 100644 --- a/tensorflow/core/protobuf/data/experimental/service_config.proto +++ b/tensorflow/core/protobuf/data/experimental/service_config.proto @@ -9,6 +9,9 @@ message DispatcherConfig { int64 port = 1; // The protocol for the dispatcher to use when connecting to workers. string protocol = 2; + // An optional work directory to use for storing dispatcher state, and for + // recovering during restarts. + string work_dir = 3; } // Configuration for a tf.data service WorkerServer. From f6c698c9eba6e32550da26f3606101ee348c9cea Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 3 Aug 2020 22:35:38 -0700 Subject: [PATCH 0335/1017] Fix compilation issue in TensorFlow TPU support PiperOrigin-RevId: 324751447 Change-Id: Icb96e301dc0bf080bc37ee31fbcb0c510a94f1c5 --- tensorflow/core/tpu/kernels/tpu_configuration_ops.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc b/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc index 4030cf86910..71735f0639f 100644 --- a/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc +++ b/tensorflow/core/tpu/kernels/tpu_configuration_ops.cc @@ -257,8 +257,10 @@ void InitializeHostForDistributedTpuOp::Compute(OpKernelContext* ctx) { #if defined(LIBTFTPU) VLOG(1) << "Removing existing proto compilation cache lookup if it exists"; - OP_REQUIRES_OK(ctx, DeleteIfExists( - rmgr, tpu::kCompiledProtoCacheResourceName)); + OP_REQUIRES_OK( + ctx, DeleteIfExists>>( + rmgr, tpu::kCompiledProtoCacheResourceName)); #endif if (enable_whole_mesh_compilations_) { @@ -286,7 +288,9 @@ void InitializeHostForDistributedTpuOp::Compute(OpKernelContext* ctx) { local_compilation_cache->Unref(); #if defined(LIBTFTPU) - tpu::TpuCompilationCacheLookup* proto_lookup; + tpu::TpuCompilationCacheLookup< + tpu::CompilationCacheEntryRef>* + proto_lookup; proto_lookup = new tpu::TpuCompilationCacheLocalLookup(local_compilation_cache); OP_REQUIRES_OK( From 68bfd34e2644178f8af446692e4993c93e1837e6 Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Mon, 3 Aug 2020 22:48:18 -0700 Subject: [PATCH 0336/1017] [TF2XLA] Enable more tests with MLIR bridge and on TPUs PiperOrigin-RevId: 324752612 Change-Id: Ia5dd37ac768461082be514618986a054f13c2aa0 --- tensorflow/python/eager/BUILD | 8 ++- .../python/eager/def_function_xla_jit_test.py | 51 +++++++++++-------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 68afa637daf..3c0c3894a64 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -853,12 +853,10 @@ tf_py_test( tf_xla_py_test( name = "def_function_xla_jit_test", srcs = ["def_function_xla_jit_test.py"], - enable_mlir_bridge = True, - enabled_backends = [ - # TODO(b/162438052): Enable the test on TPU. - "cpu", - "gpu", + disabled_backends = [ + "cpu_ondemand", ], + enable_mlir_bridge = True, python_version = "PY3", tags = [ "no_mac", diff --git a/tensorflow/python/eager/def_function_xla_jit_test.py b/tensorflow/python/eager/def_function_xla_jit_test.py index 44a4c99f5d6..b9457159217 100644 --- a/tensorflow/python/eager/def_function_xla_jit_test.py +++ b/tensorflow/python/eager/def_function_xla_jit_test.py @@ -39,9 +39,10 @@ from tensorflow.python.platform import test class DefFunctionTest(xla_test.XLATestCase): - @test_util.disable_mlir_bridge('TODO(b/162381930): MLIR bridge renames ' - ' functions') def testAutoclusteringWithTfFunction(self): + if 'tpu' in self.device.lower(): + self.skipTest('Autoclustering does not run on TPU') + with ops.device('device:{}:0'.format(self.device)): @def_function.function(experimental_compile=False) @@ -80,16 +81,16 @@ class DefFunctionTest(xla_test.XLATestCase): self.assertAllClose([2, 3, 3, 4, 4], xla_func(inputs, 1)) def testBasicInt32(self): + with ops.device('device:{}:0'.format(self.device)): - def fn(x, a): - return x + a + @def_function.function(experimental_compile=True) + def fn(x, a): + return x + a - xla_func = def_function.function(fn, experimental_compile=True) - - inputs = constant_op.constant([1, 2, 2, 3, 3], dtype=dtypes.int32) - if not test.is_built_with_rocm(): - # XLA support is not yet enabled for TF ROCm - self.assertAllClose([2, 3, 3, 4, 4], xla_func(inputs, 1)) + inputs = constant_op.constant([1, 2, 2, 3, 3], dtype=dtypes.int32) + if not test.is_built_with_rocm(): + # XLA support is not yet enabled for TF ROCm + self.assertAllClose([2, 3, 3, 4, 4], fn(inputs, 1)) def testDerivative(self): with ops.device('device:{}:0'.format(self.device)): @@ -119,25 +120,24 @@ class DefFunctionTest(xla_test.XLATestCase): # Calling function with experimental_compile=True from # experimental_compile=False should compile the inner func. - @test_util.disable_mlir_bridge('TODO(b/162381930): MLIR bridge renames ' - ' functions') def testNestedCall(self): + if 'tpu' in self.device.lower(): + self.skipTest('b/162800687: Inner function runs on host') + with ops.device('device:{}:0'.format(self.device)): + @def_function.function(experimental_compile=True) def fn(x, a): return x + a - xla_func = def_function.function(fn, experimental_compile=True) - + @def_function.function(experimental_compile=False) def fn2(x, a): - return xla_func(x, a) - - func = def_function.function(fn2, experimental_compile=False) + return fn(x, a) inputs = constant_op.constant([1, 2, 2, 3, 3]) if not test.is_built_with_rocm(): # XLA support is not yet enabled for TF ROCm - self.assertAllClose([2, 3, 3, 4, 4], func(inputs, 1)) + self.assertAllClose([2, 3, 3, 4, 4], fn2(inputs, 1)) @test_util.disable_mlir_bridge('TODO(b/162272821): MLIR bridge returns' ' wrong status type') @@ -268,9 +268,10 @@ class DefFunctionTest(xla_test.XLATestCase): 'not compilable'): c.f1(inputs) - @test_util.disable_mlir_bridge('TODO(b/162381930): MLIR bridge renames ' - ' functions') def testMustBeConstantPropagation(self): + if 'tpu' in self.device.lower(): + self.skipTest('b/162799319: Cannot resolve constant on TPU') + with ops.device('device:{}:0'.format(self.device)): if test.is_built_with_rocm(): return @@ -292,6 +293,9 @@ class DefFunctionTest(xla_test.XLATestCase): @test_util.disable_mlir_bridge('TODO(b/162271237): argmax gives different' ' results in MLIR-based bridge') def testArgMinMax(self): + if 'tpu' in self.device.lower(): + self.skipTest('b/162800904: Tie resolution is wrong on TPU for tf.func') + with ops.device('device:{}:0'.format(self.device)): @def_function.function(experimental_compile=True) @@ -429,6 +433,9 @@ class DefFunctionTest(xla_test.XLATestCase): self.assertAllClose([5.0, 5.0, 5.0], g()) def testCumsum(self): + if 'tpu' in self.device.lower(): + self.skipTest('b/162771302: 64bit rewrite of cumsum not supported') + with ops.device('device:{}:0'.format(self.device)): @def_function.function(experimental_compile=True) @@ -438,8 +445,6 @@ class DefFunctionTest(xla_test.XLATestCase): f64_input = constant_op.constant([1.1, 2.2, 3.3], dtype=dtypes.float64) self.assertAllClose([1.1, 3.3, 6.6], f(f64_input)) - @test_util.disable_mlir_bridge('TODO(b/162381930): MLIR bridge renames ' - ' functions') def testNoExcessiveRetracing(self): with ops.device('device:{}:0'.format(self.device)): inner_retracings = 0 @@ -501,6 +506,8 @@ class DefFunctionTest(xla_test.XLATestCase): outer() self.assertAllClose(c.v, 3.52) + @test_util.disable_mlir_bridge('TODO(b/162801728): MLIR bridge causes ' + ' invalid free on TPUs') def testUpdateVariableMultipleOutputs(self): with ops.device('device:{}:0'.format(self.device)): v = variables.Variable(3.1) From 19d6fdc42d51ddbc978f9e2bcbf921a1460acc86 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Mon, 3 Aug 2020 22:59:23 -0700 Subject: [PATCH 0337/1017] Internal change PiperOrigin-RevId: 324753700 Change-Id: I8305c64636ef358daa691059d719debb3d1228f9 --- .../Dialect/mhlo/transforms/sink_constants_to_control_flow.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/sink_constants_to_control_flow.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/sink_constants_to_control_flow.cc index 14d89a7e196..8d677f45c19 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/sink_constants_to_control_flow.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/sink_constants_to_control_flow.cc @@ -16,12 +16,12 @@ limitations under the License. #include "llvm/ADT/DenseMap.h" #include "llvm/Support/Casting.h" #include "mlir-hlo/Dialect/mhlo/IR/hlo_ops.h" +#include "mlir/Dialect/StandardOps/IR/Ops.h" #include "mlir/IR/Operation.h" #include "mlir/Pass/Pass.h" #include "mlir/Pass/PassManager.h" #include "mlir/Support/LLVM.h" #include "mlir/Transforms/RegionUtils.h" -#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project namespace mlir { namespace mhlo { From 88ab72153e13d925f951e9464a121cda47083e3b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 3 Aug 2020 23:04:41 -0700 Subject: [PATCH 0338/1017] Integrate LLVM at llvm/llvm-project@b5059b714023 Updates LLVM usage to match [b5059b714023](https://github.com/llvm/llvm-project/commit/b5059b714023) PiperOrigin-RevId: 324754243 Change-Id: I54dfd7b80179c7a25699bb41e91868037d0eaf89 --- tensorflow/compiler/mlir/hlo/BUILD | 4 +- .../Dialect/mhlo/transforms/register_passes.h | 8 +-- .../mlir/tools/kernel_gen/transforms/BUILD | 2 +- .../kernel_gen/transforms/register_passes.cc | 5 +- tensorflow/workspace.bzl | 4 +- third_party/mlir/BUILD | 72 ++++++++++++++----- third_party/mlir/test.BUILD | 12 ++++ 7 files changed, 78 insertions(+), 29 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/BUILD b/tensorflow/compiler/mlir/hlo/BUILD index e003e9ba279..9eee39894e4 100644 --- a/tensorflow/compiler/mlir/hlo/BUILD +++ b/tensorflow/compiler/mlir/hlo/BUILD @@ -60,7 +60,7 @@ gentbl( strip_include_prefix = "include/mlir-hlo/Dialect/mhlo/transforms/", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name MHLO", "include/mlir-hlo/Dialect/mhlo/transforms/mhlo_passes.h.inc", ), ], @@ -76,7 +76,7 @@ gentbl( strip_include_prefix = "include/mlir-hlo/Dialect/mhlo/transforms/", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name LMHLO", "include/mlir-hlo/Dialect/mhlo/transforms/lmhlo_passes.h.inc", ), ], diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/register_passes.h b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/register_passes.h index 5c862d83fee..8f70f64359b 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/register_passes.h +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/register_passes.h @@ -27,10 +27,10 @@ std::unique_ptr createTestInferShapedTypeMethodsPass(); std::unique_ptr createTestMaterializeBroadcastsPass(); std::unique_ptr createTestUnfuseBatchNormPass(); -inline void registerAllMhloPasses() { #define GEN_PASS_REGISTRATION #include "mlir-hlo/Dialect/mhlo/transforms/mhlo_passes.h.inc" -} + +inline void registerAllMhloPasses() { registerMHLOPasses(); } } // namespace mhlo @@ -38,10 +38,10 @@ namespace lmhlo { std::unique_ptr createTestLhloToLLVMPass(); -inline void registerAllLmhloPasses() { #define GEN_PASS_REGISTRATION #include "mlir-hlo/Dialect/mhlo/transforms/lmhlo_passes.h.inc" -} + +inline void registerAllLmhloPasses() { registerLMHLOPasses(); } } // namespace lmhlo } // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD index 0119b2e46ea..613422e6128 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD @@ -37,7 +37,7 @@ cc_library( gentbl( name = "tf_framework_passes_inc_gen", - tbl_outs = [("-gen-pass-decls", "tf_framework_passes.h.inc")], + tbl_outs = [("-gen-pass-decls -name TFFramework", "tf_framework_passes.h.inc")], tblgen = "@llvm-project//mlir:mlir-tblgen", td_file = "passes.td", td_srcs = ["@llvm-project//mlir:PassBaseTdFiles"], diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc index b9bad8e18d2..b9cdb2085a3 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc @@ -19,11 +19,10 @@ limitations under the License. namespace mlir { namespace kernel_gen { namespace tf_framework { - -bool register_all_passes = ([] { #define GEN_PASS_REGISTRATION #include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_passes.h.inc" -}(), true); + +bool register_all_passes = ([] { registerTFFrameworkPasses(); }(), true); } // namespace tf_framework } // namespace kernel_gen diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 47cc5951579..29cba080fa1 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "b7cfa6ca92830b3c331cb44706bb279996663439" - LLVM_SHA256 = "bad1849f86e5b83571d8a83c849e07dd66c5ddbc01a73432d4fef4da2db21543" + LLVM_COMMIT = "b5059b7140232559ed123cb94d4e8f75ca9a44dc" + LLVM_SHA256 = "3075583f88b572da4afb1340281b0e170d51ef03ba6eb2965e7dc8288cbff153" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), diff --git a/third_party/mlir/BUILD b/third_party/mlir/BUILD index 94e8f4520a6..04238bae943 100644 --- a/third_party/mlir/BUILD +++ b/third_party/mlir/BUILD @@ -99,7 +99,6 @@ cc_library( "-lpthread", ], deps = [ - ":Analysis", ":IR", ":Support", "@llvm-project//llvm:Support", @@ -344,11 +343,11 @@ gentbl( ) gentbl( - name = "LoopPassIncGen", + name = "SCFPassIncGen", strip_include_prefix = "include", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name SCF", "include/mlir/Dialect/SCF/Passes.h.inc", ), ], @@ -370,9 +369,9 @@ cc_library( deps = [ ":Affine", ":IR", - ":LoopPassIncGen", ":Pass", ":SCFDialect", + ":SCFPassIncGen", ":StandardOps", ":Transforms", "@llvm-project//llvm:Support", @@ -433,6 +432,7 @@ cc_library( ]), hdrs = glob([ "include/mlir/Dialect/*.h", + "include/mlir/Dialect/*.h", ]), includes = ["include"], deps = [ @@ -510,7 +510,7 @@ gentbl( strip_include_prefix = "include", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name Affine", "include/mlir/Dialect/Affine/Passes.h.inc", ), ], @@ -552,7 +552,7 @@ gentbl( strip_include_prefix = "include", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name Conversion", "include/mlir/Conversion/Passes.h.inc", ), ], @@ -563,6 +563,35 @@ gentbl( ], ) +cc_library( + name = "ConversionPasses", + hdrs = ["include/mlir/Conversion/Passes.h"], + includes = ["include"], + deps = [ + ":AVX512ToLLVM", + ":AffineToStandard", + ":ConversionPassIncGen", + ":GPUToGPURuntimeTransforms", + ":GPUToNVVMTransforms", + ":GPUToROCDLTransforms", + ":GPUToSPIRVTransforms", + ":GPUToVulkanTransforms", + ":LinalgToLLVM", + ":LinalgToSPIRV", + ":LinalgToStandard", + ":SCFToGPUPass", + ":SCFToStandard", + ":SPIRVToLLVM", + ":ShapeToSCF", + ":ShapeToStandard", + ":StandardToLLVM", + ":StandardToSPIRVTransforms", + ":VectorToLLVM", + ":VectorToROCDL", + ":VectorToSCF", + ], +) + cc_library( name = "AffineToStandard", srcs = glob([ @@ -628,7 +657,9 @@ cc_library( ":EDSC", ":IR", ":LoopLikeInterface", + ":Pass", ":SCFIncGen", + ":SCFPassIncGen", ":SideEffectInterfaces", ":StandardOps", ":Support", @@ -788,7 +819,7 @@ gentbl( name = "ShapeTransformsPassIncGen", strip_include_prefix = "include", tbl_outs = [( - "-gen-pass-decls", + "-gen-pass-decls -name Shape", "include/mlir/Dialect/Shape/Transforms/Passes.h.inc", )], tblgen = ":mlir-tblgen", @@ -847,7 +878,7 @@ gentbl( name = "StandardOpsTransformsPassIncGen", strip_include_prefix = "include", tbl_outs = [( - "-gen-pass-decls", + "-gen-pass-decls -name Standard", "include/mlir/Dialect/StandardOps/Transforms/Passes.h.inc", )], tblgen = ":mlir-tblgen", @@ -897,6 +928,7 @@ cc_library( ":DialectUtils", ":EDSC", ":IR", + ":SCFDialect", ":SideEffectInterfaces", ":StandardOps", ":Support", @@ -1087,7 +1119,7 @@ gentbl( strip_include_prefix = "include", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name LLVM", "include/mlir/Dialect/LLVMIR/Transforms/Passes.h.inc", ), ], @@ -1206,7 +1238,7 @@ gentbl( strip_include_prefix = "include", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name GPU", "include/mlir/Dialect/GPU/Passes.h.inc", ), ], @@ -1896,6 +1928,7 @@ cc_library( ":SPIRVCanonicalizationIncGen", ":SPIRVOpUtilsIncGen", ":SPIRVOpsIncGen", + ":SPIRVPassIncGen", ":SPIRVSerializationGen", ":SPIRVTargetAndABIStructGen", ":SideEffectInterfaces", @@ -1910,7 +1943,7 @@ gentbl( strip_include_prefix = "include", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name SPIRV", "include/mlir/Dialect/SPIRV/Passes.h.inc", ), ], @@ -2041,11 +2074,12 @@ cc_library( ":Analysis", ":ControlFlowInterfaces", ":IR", - ":LoopLikeInterface", + ":Pass", ":SCFDialect", ":SideEffectInterfaces", ":StandardOps", ":Support", + ":TransformsPassIncGen", "@llvm-project//llvm:Support", ], ) @@ -2172,7 +2206,7 @@ gentbl( strip_include_prefix = "include", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name Transforms", "include/mlir/Transforms/Passes.h.inc", ), ], @@ -2709,6 +2743,7 @@ cc_library( includes = ["include"], deps = [ ":Analysis", + ":ConversionPasses", ":GPUToGPURuntimeTransforms", ":GPUToNVVMTransforms", ":GPUToROCDLTransforms", @@ -2741,6 +2776,7 @@ cc_library( "@llvm-project//mlir/test:TestReducer", "@llvm-project//mlir/test:TestSPIRV", "@llvm-project//mlir/test:TestTransforms", + "@llvm-project//mlir/test:TestTypeDialect", ], ) @@ -2788,8 +2824,9 @@ cc_library( ":AVX512ToLLVM", ":Affine", ":AffinePassIncGen", + ":AffineToStandard", ":AffineTransforms", - ":ConversionPassIncGen", + ":ConversionPasses", ":GPUDialect", ":GPUPassIncGen", ":GPUToGPURuntimeTransforms", @@ -2809,13 +2846,13 @@ cc_library( ":LinalgToSPIRV", ":LinalgToStandard", ":LinalgTransforms", - ":LoopPassIncGen", ":NVVMDialect", ":OpenMPDialect", ":QuantOps", ":QuantPassIncGen", ":ROCDLDialect", ":SCFDialect", + ":SCFPassIncGen", ":SCFToGPUPass", ":SCFToStandard", ":SCFTransforms", @@ -2890,6 +2927,7 @@ cc_binary( "@llvm-project//mlir/test:TestReducer", "@llvm-project//mlir/test:TestSPIRV", "@llvm-project//mlir/test:TestTransforms", + "@llvm-project//mlir/test:TestTypeDialect", ], ) @@ -3211,7 +3249,7 @@ gentbl( strip_include_prefix = "include", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name Quant", "include/mlir/Dialect/Quant/Passes.h.inc", ), ], @@ -3506,7 +3544,7 @@ gentbl( strip_include_prefix = "include", tbl_outs = [ ( - "-gen-pass-decls", + "-gen-pass-decls -name Linalg", "include/mlir/Dialect/Linalg/Passes.h.inc", ), ], diff --git a/third_party/mlir/test.BUILD b/third_party/mlir/test.BUILD index a1dd9f0c168..f507842a639 100644 --- a/third_party/mlir/test.BUILD +++ b/third_party/mlir/test.BUILD @@ -229,3 +229,15 @@ cc_library( "@llvm-project//mlir:SPIRVLowering", ], ) + +cc_library( + name = "TestTypeDialect", + srcs = glob([ + "lib/Dialect/LLVMIR/*.cpp", + ]), + deps = [ + ":TestDialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + ], +) From 451ca2badeb4851e01ad0780dc42aa66eac30b41 Mon Sep 17 00:00:00 2001 From: Thai Nguyen Date: Tue, 4 Aug 2020 00:56:21 -0700 Subject: [PATCH 0339/1017] Support User's defined library in Flex delegate The user's libraries should be listed in the additional_deps as follows: tflite_flex_cc_library( name = "sample_delegate", models = ["model1.tflite", "model2.tflite"], additional_deps = ["your_custom_ops_lib"], ) Converter support will be done in a separate cl. PiperOrigin-RevId: 324764988 Change-Id: I3c284ee154c5cb17a98b148634c2bcccc4db530d --- tensorflow/lite/delegates/flex/BUILD | 12 +- tensorflow/lite/delegates/flex/build_def.bzl | 126 +++++++++---------- tensorflow/lite/tools/BUILD | 15 ++- 3 files changed, 84 insertions(+), 69 deletions(-) diff --git a/tensorflow/lite/delegates/flex/BUILD b/tensorflow/lite/delegates/flex/BUILD index a6d71881a3d..6210007361a 100644 --- a/tensorflow/lite/delegates/flex/BUILD +++ b/tensorflow/lite/delegates/flex/BUILD @@ -58,12 +58,14 @@ tf_cc_test( # Define the standard flex delegate library, that pulls in the standard set # of TensorFlow ops and kernels, using tflite_flex_cc_library with no -# portable_tensorflow_lib parameter. Custom flex delegate can be defined with -# tflite_flex_cc_library if the parameter portable_tensorflow_lib -# is provided. Ex: +# models parameter. Custom flex delegate can be defined with +# tflite_flex_cc_library if the parameter models is provided. Tensorflow +# user-provided ops could also be supported by passing to additional_deps. +# Ex: # tflite_flex_cc_library( -# name = "sample", -# portable_tensorflow_lib = custom_portable_tensorflow_lib, +# name = "sample_delegate", +# models = ["model1.tflite", "model2.tflite"], +# additional_deps = ["your_custom_ops_lib"], # ) tflite_flex_cc_library( name = "delegate", diff --git a/tensorflow/lite/delegates/flex/build_def.bzl b/tensorflow/lite/delegates/flex/build_def.bzl index 2ff762b658b..9b0771e79e6 100644 --- a/tensorflow/lite/delegates/flex/build_def.bzl +++ b/tensorflow/lite/delegates/flex/build_def.bzl @@ -5,6 +5,7 @@ load( "if_android", "if_ios", "if_mobile", + "tf_cc_binary", "tf_copts", "tf_defines_nortti_if_lite_protos", "tf_features_nomodules_if_mobile", @@ -21,12 +22,14 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") def generate_flex_kernel_header( name, - models): + models, + additional_deps = []): """A rule to generate a header file listing only used operators. Args: name: Name of the generated library. models: TFLite models to interpret. + additional_deps: Dependencies for additional TF ops. Returns: A struct with 'header' and 'include_path' fields that @@ -44,6 +47,14 @@ def generate_flex_kernel_header( ) list_ops_output = include_path + "/list_flex_ops" list_ops_tool = "//tensorflow/lite/tools:list_flex_ops_main" + if additional_deps: + tf_cc_binary( + name = "%s_list_flex_ops_main" % name, + deps = [ + "//tensorflow/lite/tools:list_flex_ops_main_lib", + ] + additional_deps, + ) + list_ops_tool = ":%s_list_flex_ops_main" % name native.genrule( name = "%s_list_flex_ops" % name, srcs = models, @@ -70,59 +81,18 @@ def generate_flex_kernel_header( return struct(include_path = include_path, header = header) def tflite_flex_cc_library( - name, - portable_tensorflow_lib = "//tensorflow/core:portable_tensorflow_lib", - visibility = ["//visibility:public"]): - """A rule to generate a flex delegate with custom portable tensorflow lib. - - This lib should be a custom version of portable_tensorflow_lib and contains ops - registrations and kernels. If not defined, the default libs will be used. - - Args: - name: Name of the generated rule. - portable_tensorflow_lib: the tensorflow_lib to be added in deps for android and ios, - can be a full or trimmed version. - visibility: visibility of the generated rule. - """ - native.cc_library( - name = name, - hdrs = [ - "//tensorflow/lite/delegates/flex:delegate.h", - ], - visibility = visibility, - deps = [ - "//tensorflow/lite/delegates/flex:delegate_data", - "//tensorflow/lite/delegates/flex:delegate_only_runtime", - "//tensorflow/lite/delegates/utils:simple_delegate", - ] + select({ - "//tensorflow:android": [ - portable_tensorflow_lib, - ], - "//tensorflow:ios": [ - portable_tensorflow_lib, - ], - "//conditions:default": [ - "//tensorflow/core:tensorflow", - "//tensorflow/lite/c:common", - ], - }), - alwayslink = 1, - ) - -def tflite_flex_jni_library( name, models = [], - visibility = ["//visibility:private"]): - """A rule to generate a jni library listing only used operators. - - The libtensorflowlite_flex_jni.so name is fixed due to a limitation in JNI - Java wrapper, so please make sure there is no naming conflicts. + additional_deps = [], + visibility = ["//visibility:public"]): + """A rule to generate a flex delegate with only ops to run listed models. Args: - name: Prefix of the generated libraries. + name: Name of the generated flex delegate. models: TFLite models to interpret. The library will only include ops and kernels to support these models. If empty, the library will include all Tensorflow ops and kernels. + additional_deps: Dependencies for additional TF ops. visibility: visibility of the generated rules. """ portable_tensorflow_lib = "//tensorflow/core:portable_tensorflow_lib" @@ -130,6 +100,7 @@ def tflite_flex_jni_library( CUSTOM_KERNEL_HEADER = generate_flex_kernel_header( name = "%s_tf_op_headers" % name, models = models, + additional_deps = additional_deps, ) # Define a custom tensorflow_lib with selective registration. @@ -172,52 +143,81 @@ def tflite_flex_jni_library( ) portable_tensorflow_lib = ":%s_tensorflow_lib" % name - # Define a custom init_tensorflow that depends on the above tensorflow_lib. - # This will avoid the symbols re-definition errors. + # Define a custom flex delegate with above tensorflow_lib. native.cc_library( - name = "%s_init_tensorflow" % name, - srcs = [ - "//tensorflow/lite/testing:init_tensorflow.cc", - ], + name = name, hdrs = [ - "//tensorflow/lite/testing:init_tensorflow.h", + "//tensorflow/lite/delegates/flex:delegate.h", ], visibility = visibility, - deps = select({ - "//conditions:default": [ - "//tensorflow/core:lib", - ], + deps = [ + "//tensorflow/lite/delegates/flex:delegate_data", + "//tensorflow/lite/delegates/flex:delegate_only_runtime", + "//tensorflow/lite/delegates/utils:simple_delegate", + ] + select({ "//tensorflow:android": [ portable_tensorflow_lib, ], "//tensorflow:ios": [ portable_tensorflow_lib, ], - }), + "//conditions:default": [ + "//tensorflow/core:tensorflow", + "//tensorflow/lite/c:common", + ], + }) + additional_deps, + alwayslink = 1, ) +def tflite_flex_jni_library( + name, + models = [], + additional_deps = [], + visibility = ["//visibility:private"]): + """A rule to generate a jni library listing only used operators. + + The libtensorflowlite_flex_jni.so name is fixed due to a limitation in JNI + Java wrapper, so please make sure there is no naming conflicts. + + Args: + name: Prefix of the generated libraries. + models: TFLite models to interpret. The library will only include ops and kernels + to support these models. If empty, the library will include all Tensorflow + ops and kernels. + additional_deps: Dependencies for additional TF ops. + visibility: visibility of the generated rules. + """ + # Define a custom flex_delegate that depends on above tensorflow_lib. # This will reduce the binary size comparing to the original flex delegate. tflite_flex_cc_library( name = "%s_flex_delegate" % name, - portable_tensorflow_lib = portable_tensorflow_lib, + models = models, + additional_deps = additional_deps, visibility = visibility, ) - # Define a custom flex_native that depends on above flex_delegate and init_tensorflow. + # Define a custom flex_native that depends on above flex_delegate. native.cc_library( name = "%s_flex_native" % name, srcs = [ + "//tensorflow/lite/testing:init_tensorflow.h", + "//tensorflow/lite/testing:init_tensorflow.cc", "//tensorflow/lite/delegates/flex/java/src/main/native:flex_delegate_jni.cc", ], copts = tflite_copts(), visibility = visibility, deps = [ ":%s_flex_delegate" % name, - ":%s_init_tensorflow" % name, "//tensorflow/lite/java/jni", "//tensorflow/lite/delegates/utils:simple_delegate", - ], + ] + select({ + "//tensorflow:android": [], + "//tensorflow:ios": [], + "//conditions:default": [ + "//tensorflow/core:lib", + ], + }), alwayslink = 1, ) diff --git a/tensorflow/lite/tools/BUILD b/tensorflow/lite/tools/BUILD index 89d3da1ec6a..1f57cad7f7a 100644 --- a/tensorflow/lite/tools/BUILD +++ b/tensorflow/lite/tools/BUILD @@ -9,7 +9,9 @@ package( licenses = ["notice"], # Apache 2.0 ) -exports_files(["logging.h"]) +exports_files([ + "logging.h", +]) common_copts = ["-Wall"] @@ -283,6 +285,17 @@ tf_cc_binary( ], ) +cc_library( + name = "list_flex_ops_main_lib", + srcs = ["list_flex_ops_main.cc"], + visibility = ["//visibility:public"], + deps = [ + ":list_flex_ops", + "//tensorflow/lite/tools:command_line_flags", + "@com_google_absl//absl/strings", + ], +) + tf_cc_test( name = "list_flex_ops_test", srcs = ["list_flex_ops_test.cc"], From 4187ccbe2536d43b2fb6e794c4ca72ad631da4c3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 02:02:26 -0700 Subject: [PATCH 0340/1017] compat: Update forward compatibility horizon to 2020-08-04 PiperOrigin-RevId: 324771488 Change-Id: I2c256c3a92b758fde313f9fa27f590e9899513aa --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index e0f751a4376..c40337f00be 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 3) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 4) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From c7fa71b32a3635eb25596ae80d007b41007769c4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 02:02:27 -0700 Subject: [PATCH 0341/1017] Update GraphDef version to 483. PiperOrigin-RevId: 324771489 Change-Id: Idc9972f0e6200d41978a5ec0f6b6f3e5b8edd320 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index dae48097aa8..7febc640348 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 482 // Updated: 2020/8/3 +#define TF_GRAPH_DEF_VERSION 483 // Updated: 2020/8/4 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From 0f592c3c557267a1055dee892eaa52e9c61b8888 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 16:28:17 +0700 Subject: [PATCH 0342/1017] Add NewReadOnlyMemoryRegionFromFile --- .../filesystem/plugins/hadoop/hadoop_filesystem.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index de0a36816e0..20c907affee 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -440,6 +440,20 @@ void NewWritableFile(const TF_Filesystem* filesystem, const char* path, TF_SetStatus(status, TF_OK, ""); } +void NewReadOnlyMemoryRegionFromFile(const TF_Filesystem* filesystem, + const char* path, + TF_ReadOnlyMemoryRegion* region, + TF_Status* status) { + // hadoopReadZero() technically supports this call with the following + // caveats: + // - It only works up to 2 GB. We'd have to Stat() the file to ensure that + // it fits. + // - If not on the local filesystem, the entire file will be read, making + // it inefficient for callers that assume typical mmap() behavior. + TF_SetStatus(status, TF_UNIMPLEMENTED, + "HDFS does not support ReadOnlyMemoryRegion"); +} + // TODO(vnvo2409): Implement later } // namespace tf_hadoop_filesystem From cb7fd5197e2e435b9d15c6b21c7659a493563d64 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 16:29:01 +0700 Subject: [PATCH 0343/1017] Add PathExists --- .../plugins/hadoop/hadoop_filesystem.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index 20c907affee..1bd72bdf059 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -454,6 +454,22 @@ void NewReadOnlyMemoryRegionFromFile(const TF_Filesystem* filesystem, "HDFS does not support ReadOnlyMemoryRegion"); } +void PathExists(const TF_Filesystem* filesystem, const char* path, + TF_Status* status) { + auto libhdfs = static_cast(filesystem->plugin_filesystem); + auto fs = Connect(libhdfs, path, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string scheme, namenode, hdfs_path; + ParseHadoopPath(path, &scheme, &namenode, &hdfs_path); + + if (libhdfs->hdfsExists(fs, hdfs_path.c_str()) == 0) + TF_SetStatus(status, TF_OK, ""); + else + TF_SetStatus(status, TF_NOT_FOUND, + (std::string(path) + " not found").c_str()); +} + // TODO(vnvo2409): Implement later } // namespace tf_hadoop_filesystem From b3c8cc141a5c6c4becc642ea09f34537eba3b1d2 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 16:29:25 +0700 Subject: [PATCH 0344/1017] Add Stat --- .../plugins/hadoop/hadoop_filesystem.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index 1bd72bdf059..b3ba79cbc77 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -470,6 +470,25 @@ void PathExists(const TF_Filesystem* filesystem, const char* path, (std::string(path) + " not found").c_str()); } +void Stat(const TF_Filesystem* filesystem, const char* path, + TF_FileStatistics* stats, TF_Status* status) { + auto libhdfs = static_cast(filesystem->plugin_filesystem); + auto fs = Connect(libhdfs, path, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string scheme, namenode, hdfs_path; + ParseHadoopPath(path, &scheme, &namenode, &hdfs_path); + + auto info = libhdfs->hdfsGetPathInfo(fs, hdfs_path.c_str()); + if (info == nullptr) return TF_SetStatusFromIOError(status, errno, path); + + stats->length = static_cast(info->mSize); + stats->mtime_nsec = static_cast(info->mLastMod) * 1e9; + stats->is_directory = info->mKind == kObjectKindDirectory; + libhdfs->hdfsFreeFileInfo(info, 1); + TF_SetStatus(status, TF_OK, ""); +} + // TODO(vnvo2409): Implement later } // namespace tf_hadoop_filesystem From 9fee1f54b2ce504cbdc31d5f2958a6277cadb3c0 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 16:29:39 +0700 Subject: [PATCH 0345/1017] Add GetFileSize --- .../plugins/hadoop/hadoop_filesystem.cc | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index b3ba79cbc77..dce6e27f9ee 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -489,6 +489,27 @@ void Stat(const TF_Filesystem* filesystem, const char* path, TF_SetStatus(status, TF_OK, ""); } +int64_t GetFileSize(const TF_Filesystem* filesystem, const char* path, + TF_Status* status) { + auto libhdfs = static_cast(filesystem->plugin_filesystem); + auto fs = Connect(libhdfs, path, status); + if (TF_GetCode(status) != TF_OK) return -1; + + std::string scheme, namenode, hdfs_path; + ParseHadoopPath(path, &scheme, &namenode, &hdfs_path); + + auto info = libhdfs->hdfsGetPathInfo(fs, hdfs_path.c_str()); + if (info == nullptr) { + TF_SetStatusFromIOError(status, errno, path); + return -1; + } + + TF_SetStatus(status, TF_OK, ""); + auto size = static_cast(info->mSize); + libhdfs->hdfsFreeFileInfo(info, 1); + return size; +} + // TODO(vnvo2409): Implement later } // namespace tf_hadoop_filesystem From 557663d0ac0c5e2f000cb8fde679b6ca3649a28d Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 16:33:24 +0700 Subject: [PATCH 0346/1017] Add DeleteFile --- .../plugins/hadoop/hadoop_filesystem.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index dce6e27f9ee..7b7a575873a 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -510,6 +510,21 @@ int64_t GetFileSize(const TF_Filesystem* filesystem, const char* path, return size; } +void DeleteFile(const TF_Filesystem* filesystem, const char* path, + TF_Status* status) { + auto libhdfs = static_cast(filesystem->plugin_filesystem); + auto fs = Connect(libhdfs, path, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string scheme, namenode, hdfs_path; + ParseHadoopPath(path, &scheme, &namenode, &hdfs_path); + + if (libhdfs->hdfsDelete(fs, hdfs_path.c_str(), /*recursive=*/0) != 0) + TF_SetStatusFromIOError(status, errno, path); + else + TF_SetStatus(status, TF_OK, ""); +} + // TODO(vnvo2409): Implement later } // namespace tf_hadoop_filesystem From 56a86ce36e09fdedeb84b5ebfa8f83f7778edf4a Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 16:38:02 +0700 Subject: [PATCH 0347/1017] Add CreateDir --- .../plugins/hadoop/hadoop_filesystem.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index 7b7a575873a..7e12fc9483f 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -525,6 +525,21 @@ void DeleteFile(const TF_Filesystem* filesystem, const char* path, TF_SetStatus(status, TF_OK, ""); } +void CreateDir(const TF_Filesystem* filesystem, const char* path, + TF_Status* status) { + auto libhdfs = static_cast(filesystem->plugin_filesystem); + auto fs = Connect(libhdfs, path, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string scheme, namenode, hdfs_path; + ParseHadoopPath(path, &scheme, &namenode, &hdfs_path); + + if (libhdfs->hdfsCreateDirectory(fs, hdfs_path.c_str()) != 0) + TF_SetStatusFromIOError(status, errno, path); + else + TF_SetStatus(status, TF_OK, ""); +} + // TODO(vnvo2409): Implement later } // namespace tf_hadoop_filesystem From 209ff596bd429599c8bc7039c15094b169ff605f Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 16:46:07 +0700 Subject: [PATCH 0348/1017] Add DeleteDir --- .../plugins/hadoop/hadoop_filesystem.cc | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index 7e12fc9483f..76d2538cd31 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -540,6 +540,42 @@ void CreateDir(const TF_Filesystem* filesystem, const char* path, TF_SetStatus(status, TF_OK, ""); } +void DeleteDir(const TF_Filesystem* filesystem, const char* path, + TF_Status* status) { + auto libhdfs = static_cast(filesystem->plugin_filesystem); + auto fs = Connect(libhdfs, path, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string scheme, namenode, hdfs_path; + ParseHadoopPath(path, &scheme, &namenode, &hdfs_path); + + // Count the number of entries in the directory, and only delete if it's + // non-empty. This is consistent with the interface, but note that there's + // a race condition where a file may be added after this check, in which + // case the directory will still be deleted. + int entries = 0; + auto info = libhdfs->hdfsListDirectory(fs, hdfs_path.c_str(), &entries); + if (info != nullptr) libhdfs->hdfsFreeFileInfo(info, entries); + + // Due to HDFS bug HDFS-8407, we can't distinguish between an error and empty + // folder, especially for Kerberos enable setup, EAGAIN is quite common when + // the call is actually successful. Check again by Stat. + if (info == nullptr && errno != 0) { + TF_FileStatistics stat; + Stat(filesystem, path, &stat, status); + if (TF_GetCode(status) != TF_OK) return; + } + + if (entries > 0) + return TF_SetStatus(status, TF_FAILED_PRECONDITION, + "Cannot delete a non-empty directory."); + + if (libhdfs->hdfsDelete(fs, hdfs_path.c_str(), /*recursive=*/1) != 0) + TF_SetStatusFromIOError(status, errno, path); + else + TF_SetStatus(status, TF_OK, ""); +} + // TODO(vnvo2409): Implement later } // namespace tf_hadoop_filesystem From 0579ea25ffc4230ab2d9c327ac79aeeeaddc56ff Mon Sep 17 00:00:00 2001 From: Stephan Herhut Date: Tue, 4 Aug 2020 02:51:28 -0700 Subject: [PATCH 0349/1017] Roll back "Enable mlir generated GPU kernels by default for cuda builds." It breaks some internal builds. PiperOrigin-RevId: 324776590 Change-Id: If5c7cebc54e450a91f13aec7969c86265253e90c --- .bazelrc | 5 +++++ tensorflow/core/kernels/mlir_generated/BUILD | 4 ++-- tensorflow/core/kernels/mlir_generated/build_defs.bzl | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.bazelrc b/.bazelrc index 6a448b267e0..da988e4c928 100644 --- a/.bazelrc +++ b/.bazelrc @@ -173,6 +173,11 @@ build:using_cuda --define=using_cuda=true build:using_cuda --action_env TF_NEED_CUDA=1 build:using_cuda --crosstool_top=@local_config_cuda//crosstool:toolchain +# Enable the mlir generated GPU kernels only for cuda builds. +build --define=tensorflow_enable_mlir_generated_gpu_kernels=0 +# This is a more specific option, so it takes precedence over the line above for cuda builds. +build:using_cuda --define=tensorflow_enable_mlir_generated_gpu_kernels=1 + # This config refers to building CUDA op kernels with nvcc. build:cuda --config=using_cuda build:cuda --define=using_cuda_nvcc=true diff --git a/tensorflow/core/kernels/mlir_generated/BUILD b/tensorflow/core/kernels/mlir_generated/BUILD index 79ccda50c87..9f3efe9d972 100644 --- a/tensorflow/core/kernels/mlir_generated/BUILD +++ b/tensorflow/core/kernels/mlir_generated/BUILD @@ -18,9 +18,9 @@ package( ) config_setting( - name = "mlir_generated_gpu_kernels_enabled", + name = "mlir_generated_gpu_kernels_disabled", define_values = { - "tensorflow_enable_mlir_generated_gpu_kernels": "1", + "tensorflow_enable_mlir_generated_gpu_kernels": "0", }, ) diff --git a/tensorflow/core/kernels/mlir_generated/build_defs.bzl b/tensorflow/core/kernels/mlir_generated/build_defs.bzl index 3426aba94a4..2bf6e8fa3bb 100644 --- a/tensorflow/core/kernels/mlir_generated/build_defs.bzl +++ b/tensorflow/core/kernels/mlir_generated/build_defs.bzl @@ -4,8 +4,8 @@ load("@local_config_cuda//cuda:build_defs.bzl", "cuda_gpu_architectures", "if_cu def if_mlir_generated_gpu_kernels_enabled(if_true, if_false = []): return select({ - "//tensorflow/core/kernels/mlir_generated:mlir_generated_gpu_kernels_enabled": if_true, - "//conditions:default": if_false, + "//tensorflow/core/kernels/mlir_generated:mlir_generated_gpu_kernels_disabled": if_false, + "//conditions:default": if_true, }) def _lookup_file(filegroup, path): From 79b1c712df008964833a2d920b5398d58c48e03b Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 17:00:08 +0700 Subject: [PATCH 0350/1017] Add RenameFile --- .../plugins/hadoop/hadoop_filesystem.cc | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index 76d2538cd31..37b0fdc55fb 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -576,6 +576,27 @@ void DeleteDir(const TF_Filesystem* filesystem, const char* path, TF_SetStatus(status, TF_OK, ""); } +void RenameFile(const TF_Filesystem* filesystem, const char* src, + const char* dst, TF_Status* status) { + auto libhdfs = static_cast(filesystem->plugin_filesystem); + auto fs = Connect(libhdfs, src, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string scheme, namenode, hdfs_path_src, hdfs_path_dst; + ParseHadoopPath(src, &scheme, &namenode, &hdfs_path_src); + ParseHadoopPath(dst, &scheme, &namenode, &hdfs_path_dst); + + if (libhdfs->hdfsExists(fs, hdfs_path_dst.c_str()) == 0 && + libhdfs->hdfsDelete(fs, hdfs_path_dst.c_str(), /*recursive=*/0) != 0) + return TF_SetStatusFromIOError(status, errno, dst); + + if (libhdfs->hdfsRename(fs, hdfs_path_src.c_str(), hdfs_path_dst.c_str()) != + 0) + TF_SetStatusFromIOError(status, errno, src); + else + TF_SetStatus(status, TF_OK, ""); +} + // TODO(vnvo2409): Implement later } // namespace tf_hadoop_filesystem From 2bdbc19821f6b299d5e8c97e4f2d5311c925a5d8 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 17:20:22 +0700 Subject: [PATCH 0351/1017] Add GetChildren --- .../plugins/hadoop/hadoop_filesystem.cc | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index 37b0fdc55fb..fd481d38075 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -597,6 +597,44 @@ void RenameFile(const TF_Filesystem* filesystem, const char* src, TF_SetStatus(status, TF_OK, ""); } +int GetChildren(const TF_Filesystem* filesystem, const char* path, + char*** entries, TF_Status* status) { + auto libhdfs = static_cast(filesystem->plugin_filesystem); + auto fs = Connect(libhdfs, path, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string scheme, namenode, hdfs_path; + ParseHadoopPath(path, &scheme, &namenode, &hdfs_path); + + // hdfsListDirectory returns nullptr if the directory is empty. Do a separate + // check to verify the directory exists first. + TF_FileStatistics stat; + Stat(filesystem, path, &stat, status); + if (TF_GetCode(status) != TF_OK) return; + + int num_entries = 0; + auto info = libhdfs->hdfsListDirectory(fs, hdfs_path.c_str(), &num_entries); + if (info == nullptr) { + if (stat.is_directory) { + // Assume it's an empty directory. + TF_SetStatus(status, TF_OK, ""); + return 0; + } + TF_SetStatusFromIOError(status, errno, path); + return -1; + } + *entries = static_cast( + plugin_memory_allocate(num_entries * sizeof((*entries)[0]))); + auto BaseName = [](const std::string& name) { + return name.substr(name.find_last_of('/') + 1); + }; + for (int i = 0; i < num_entries; i++) { + (*entries)[i] = strdup(BaseName(info[i].mName).c_str()); + } + libhdfs->hdfsFreeFileInfo(info, num_entries); + TF_SetStatus(status, TF_OK, ""); +} + // TODO(vnvo2409): Implement later } // namespace tf_hadoop_filesystem From 2c652e599091155a4bd26d5b5dbf4f0af08f65dd Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 17:31:46 +0700 Subject: [PATCH 0352/1017] Fix GetChildrens --- .../filesystem/plugins/hadoop/hadoop_filesystem.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc index fd481d38075..fe5e1992ff2 100644 --- a/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/hadoop/hadoop_filesystem.cc @@ -601,7 +601,7 @@ int GetChildren(const TF_Filesystem* filesystem, const char* path, char*** entries, TF_Status* status) { auto libhdfs = static_cast(filesystem->plugin_filesystem); auto fs = Connect(libhdfs, path, status); - if (TF_GetCode(status) != TF_OK) return; + if (TF_GetCode(status) != TF_OK) return -1; std::string scheme, namenode, hdfs_path; ParseHadoopPath(path, &scheme, &namenode, &hdfs_path); @@ -610,7 +610,7 @@ int GetChildren(const TF_Filesystem* filesystem, const char* path, // check to verify the directory exists first. TF_FileStatistics stat; Stat(filesystem, path, &stat, status); - if (TF_GetCode(status) != TF_OK) return; + if (TF_GetCode(status) != TF_OK) return -1; int num_entries = 0; auto info = libhdfs->hdfsListDirectory(fs, hdfs_path.c_str(), &num_entries); @@ -633,6 +633,7 @@ int GetChildren(const TF_Filesystem* filesystem, const char* path, } libhdfs->hdfsFreeFileInfo(info, num_entries); TF_SetStatus(status, TF_OK, ""); + return num_entries; } // TODO(vnvo2409): Implement later From ec4a78f4431203c2c352feaf07b9f8194dfa1052 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Tue, 4 Aug 2020 17:30:19 +0700 Subject: [PATCH 0353/1017] Bump google-cloud-cpp to 1.16.0 --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 76d23dd81ab..a74e287a30f 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -336,8 +336,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "com_github_googlecloudplatform_google_cloud_cpp", - sha256 = "839b2d4dcb36a671734dac6b30ea8c298bbeaafcf7a45ee4a7d7aa5986b16569", - strip_prefix = "google-cloud-cpp-1.14.0", + sha256 = "d9d1358f464328b8fd6d24a98d4c2876fde0d3fdb06c8b6bd617be7fb9b0fbac", + strip_prefix = "google-cloud-cpp-1.16.0", repo_mapping = { "@com_github_curl_curl": "@curl", }, @@ -346,8 +346,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): "//third_party/systemlibs:google_cloud_cpp.google.cloud.bigtable.BUILD": "google/cloud/bigtable/BUILD", }, urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/googleapis/google-cloud-cpp/archive/v1.14.0.tar.gz", - "https://github.com/googleapis/google-cloud-cpp/archive/v1.14.0.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/googleapis/google-cloud-cpp/archive/v1.16.0.tar.gz", + "https://github.com/googleapis/google-cloud-cpp/archive/v1.16.0.tar.gz", ], ) From 2f00b55557703302d0f4ff43212638574e63ab87 Mon Sep 17 00:00:00 2001 From: Ben Barsdell Date: Tue, 4 Aug 2020 20:55:49 +1000 Subject: [PATCH 0354/1017] Enable depthwise convs in auto_mixed_precision - These are well-supported as of CUDNN v8. - Also adds a Python test. --- .../optimizers/auto_mixed_precision_lists.h | 10 +++--- .../grappler/auto_mixed_precision_test.py | 35 +++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h b/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h index 6643149a6e5..ce0af4ac4b1 100644 --- a/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h +++ b/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h @@ -126,11 +126,6 @@ class AutoMixedPrecisionListsCuda : public AutoMixedPrecisionLists { "GRUBlockCellGrad", "LSTMBlockCell", "LSTMBlockCellGrad", - // TODO(benbarsdell): Enable these when fast and safe fp16 kernels are - // available for depthwise convolutions. - // "DepthwiseConv2dNative", - // "DepthwiseConv2dNativeBackpropFilter", - // "DepthwiseConv2dNativeBackpropInput", "MatMul", }; if (cuda_version_ >= 9010) { @@ -146,6 +141,11 @@ class AutoMixedPrecisionListsCuda : public AutoMixedPrecisionLists { list.insert("Conv3DBackpropInput"); list.insert("Conv3DBackpropInputV2"); } + if (cudnn_version_ >= 8000) { + list.insert("DepthwiseConv2dNative"); + list.insert("DepthwiseConv2dNativeBackpropFilter"); + list.insert("DepthwiseConv2dNativeBackpropInput"); + } UpdateList("ALLOWLIST", &list); // For backwards compatibility, keeping the original env variable here. // TODO(reedwm): This should be removed if we don't have active users. diff --git a/tensorflow/python/grappler/auto_mixed_precision_test.py b/tensorflow/python/grappler/auto_mixed_precision_test.py index 539c2bca9f3..f7f3777f7a9 100644 --- a/tensorflow/python/grappler/auto_mixed_precision_test.py +++ b/tensorflow/python/grappler/auto_mixed_precision_test.py @@ -138,6 +138,11 @@ def _conv_pool(x): return h_pool2 +def _depthwise_conv2d(x, w): + """Returns a 2d depthwise convolution layer with full stride.""" + return nn.depthwise_conv2d(x, w, strides=[1, 1, 1, 1], padding='SAME') + + def _simple_loop(x, functor): """Simple loop whose body is provided by the functor.""" init = (constant_op.constant(0), x) @@ -566,6 +571,36 @@ class AutoMixedPrecisionTest(test.TestCase, parameterized.TestCase): tol = 5e-3 if mode == 'mkl' else 1e-3 self.assertAllClose(output_val_ref, output_val, atol=tol, rtol=tol) + # TODO(benbarsdell): This test has not been tried with MKL. + @parameterized.parameters(['cuda']) + @test_util.run_deprecated_v1 + @test_util.disable_xla('This test does not pass with XLA') + def test_depthwise_conv2d(self, mode): + """Test grad ops with depthwise convolution2d graph.""" + self._maybe_skip(mode) + random_seed.set_random_seed(0) + x = _input([2, 8, 8, 1]) + f = _weight([3, 3, 1, 4]) + y = _depthwise_conv2d(x, f) + y = array_ops.identity(y) + optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.01) + g = optimizer.compute_gradients(y, [x, f]) + output = (y, g) + + output_val_ref, output_val, cost_graph = self._run(mode, output) + node_map = _build_node_map(cost_graph.node) + self._assert_output_f16(mode, node_map, 'depthwise') + self._assert_output_f16( + mode, node_map, + 'gradients/depthwise_grad/DepthwiseConv2dNativeBackpropInput') + self._assert_output_f16( + mode, node_map, + 'gradients/depthwise_grad/DepthwiseConv2dNativeBackpropFilter') + + output_val_ref, output_val, cost_graph = self._run(mode, output) + tol = 2e-3 + self.assertAllClose(output_val_ref, output_val, atol=tol, rtol=tol) + @parameterized.parameters(['cuda', 'mkl']) @test_util.run_v1_only('b/138749235') @test_util.disable_xla('This test does not pass with XLA') From 1b64996606a5a17335f1ef52ae902bae359dfea0 Mon Sep 17 00:00:00 2001 From: Thai Nguyen Date: Tue, 4 Aug 2020 05:56:30 -0700 Subject: [PATCH 0355/1017] Add support for additional deps in tflite_flex_android_library PiperOrigin-RevId: 324796318 Change-Id: I8590cf68360f486c6ef4c6014d91969a0fab3954 --- tensorflow/lite/delegates/flex/build_def.bzl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/lite/delegates/flex/build_def.bzl b/tensorflow/lite/delegates/flex/build_def.bzl index 9b0771e79e6..b4965d1076e 100644 --- a/tensorflow/lite/delegates/flex/build_def.bzl +++ b/tensorflow/lite/delegates/flex/build_def.bzl @@ -234,6 +234,7 @@ def tflite_flex_jni_library( def tflite_flex_android_library( name, models = [], + additional_deps = [], custom_package = "org.tensorflow.lite.flex", visibility = ["//visibility:private"]): """A rule to generate an android library based on the selective-built jni library. @@ -243,12 +244,14 @@ def tflite_flex_android_library( models: TFLite models used for selective build. The library will only include ops and kernels to support these models. If empty, the library will include all Tensorflow ops and kernels. + additional_deps: Dependencies for additional TF ops. custom_package: Java package for which java sources will be generated. visibility: visibility of the generated rules. """ tflite_flex_jni_library( name = name, models = models, + additional_deps = additional_deps, visibility = visibility, ) From ea6516dcadbaf0f6cc01d2c2168d3ba0b5dab895 Mon Sep 17 00:00:00 2001 From: Stephan Herhut Date: Tue, 4 Aug 2020 06:06:23 -0700 Subject: [PATCH 0356/1017] Add a shape_to_descriptors pass that combines the various shape lowering patterns needed for code generation with descriptors. PiperOrigin-RevId: 324797518 Change-Id: Icb0a836668c273773834249ce6ad7b06c6aa200a --- .../compiler/mlir/tools/kernel_gen/BUILD | 1 - .../mlir/tools/kernel_gen/tests/BUILD | 19 ----- .../kernel_gen/tests/embed_tf_framework.mlir | 37 --------- .../mlir/tools/kernel_gen/tests/invalid.mlir | 7 -- .../mlir/tools/kernel_gen/tests/ops.mlir | 19 ----- .../tests/tf_framework_legalize_to_llvm.mlir | 75 ------------------- .../mlir/tools/kernel_gen/transforms/BUILD | 22 ++++++ .../mlir/tools/kernel_gen/transforms/passes.h | 9 +++ .../tools/kernel_gen/transforms/passes.td | 9 ++- .../kernel_gen/transforms/register_passes.cc | 2 - .../transforms/shape_to_descriptors.cc | 72 ++++++++++++++++++ 11 files changed, 110 insertions(+), 162 deletions(-) delete mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tests/BUILD delete mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tests/embed_tf_framework.mlir delete mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tests/invalid.mlir delete mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tests/ops.mlir delete mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tests/tf_framework_legalize_to_llvm.mlir create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors.cc diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/BUILD index 32fae8a8305..b40d6cb3abf 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/BUILD @@ -61,7 +61,6 @@ tf_cc_binary( name = "kernel-gen-opt", visibility = ["//tensorflow/compiler/mlir/tools/kernel_gen/tests:__pkg__"], deps = [ - "//tensorflow/compiler/mlir/tensorflow:tensorflow_dialect_registration", "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_dialect_registration", "//tensorflow/compiler/mlir/tools/kernel_gen/transforms:passes", "@llvm-project//mlir:AllPassesAndDialects", diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tests/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/tests/BUILD deleted file mode 100644 index db878df991b..00000000000 --- a/tensorflow/compiler/mlir/tools/kernel_gen/tests/BUILD +++ /dev/null @@ -1,19 +0,0 @@ -load("//tensorflow/compiler/mlir:glob_lit_test.bzl", "glob_lit_tests") - -package(licenses = ["notice"]) - -glob_lit_tests( - data = [":test_utilities"], - driver = "@llvm-project//mlir:run_lit.sh", - test_file_exts = ["mlir"], -) - -# Bundle together all of the test utilities that are used by tests. -filegroup( - name = "test_utilities", - testonly = True, - data = [ - "//tensorflow/compiler/mlir/tools/kernel_gen:kernel-gen-opt", - "@llvm-project//llvm:FileCheck", - ], -) diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tests/embed_tf_framework.mlir b/tensorflow/compiler/mlir/tools/kernel_gen/tests/embed_tf_framework.mlir deleted file mode 100644 index bb0f1926cda..00000000000 --- a/tensorflow/compiler/mlir/tools/kernel_gen/tests/embed_tf_framework.mlir +++ /dev/null @@ -1,37 +0,0 @@ -// RUN: kernel-gen-opt %s -embed-tf-framework -split-input-file | FileCheck %s - -// CHECK-LABEL: func @tf_entry( -// CHECK-SAME: [[CTX:%.*]]: !tf_framework.op_kernel_context, -// CHECK-SAME: [[SIZE_0:%.*]]: index, -// CHECK-SAME: [[SIZE_2:%.*]]: index) -> index attributes {tf_entry} { -func @tf_entry(%size_0 : index , %size_2 : index) -> index - attributes {tf_entry} { - %buf = alloc(%size_0, %size_2)[] : memref - dealloc %buf : memref - std.return %size_0 : index -} -// CHECK-NEXT: [[VAL_3:%.*]] = tf_framework.alloc_raw -// CHECK-SAME: ([[CTX]], [[SIZE_0]], [[SIZE_2]]) : memref -// CHECK-NEXT: tf_framework.dealloc_raw([[CTX]], [[VAL_3]]) : memref -// CHECK-NEXT: return [[SIZE_0]] : index - -// ----- - -// CHECK-LABEL: func @non_tf_entry( -// CHECK-SAME: [[SIZE_0:%.*]]: index, [[SIZE_2:%.*]]: index) -> index -func @non_tf_entry(%size_0 : index , %size_2 : index) -> index { - std.return %size_0 : index -} - -// ----- - -// CHECK-LABEL: func @tf_entry( -func @tf_entry(%size : index) attributes {tf_entry} { - %buf = alloc()[%size] : memref<64xf32, affine_map<(d0)[s0] -> (d0 + s0)>> - dealloc %buf : memref<64xf32, affine_map<(d0)[s0] -> (d0 + s0)>> - std.return -} -// CHECK_NOT: alloc_raw -// CHECK: alloc() -// CHECK_NOT: dealloc_raw -// CHECK: dealloc % diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tests/invalid.mlir b/tensorflow/compiler/mlir/tools/kernel_gen/tests/invalid.mlir deleted file mode 100644 index 1d1b3319515..00000000000 --- a/tensorflow/compiler/mlir/tools/kernel_gen/tests/invalid.mlir +++ /dev/null @@ -1,7 +0,0 @@ -// RUN: kernel-gen-opt %s -split-input-file -verify-diagnostics - -func @alloc_raw(%ctx: !tf_framework.op_kernel_context, %size : index) { - // expected-error @+1 {{`dyn_sizes` count 1 does not match dynamic dimensions}} - %buf = tf_framework.alloc_raw(%ctx, %size) : memref - return -} diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tests/ops.mlir b/tensorflow/compiler/mlir/tools/kernel_gen/tests/ops.mlir deleted file mode 100644 index 19974ec9482..00000000000 --- a/tensorflow/compiler/mlir/tools/kernel_gen/tests/ops.mlir +++ /dev/null @@ -1,19 +0,0 @@ -// RUN: kernel-gen-opt %s | FileCheck %s -// Verify the printed output can be parsed. -// RUN: kernel-gen-opt %s | kernel-gen-opt | FileCheck %s -// Verify the generic form can be parsed. -// RUN: kernel-gen-opt -mlir-print-op-generic %s | kernel-gen-opt | FileCheck %s - -// CHECK-LABEL: func @alloc_raw -func @alloc_raw(%ctx: !tf_framework.op_kernel_context, - %size_0 : index , %size_2 : index) { - %buf_0 = tf_framework.alloc_raw(%ctx) : memref<10xi8> - %buf_1 = tf_framework.alloc_raw(%ctx, %size_0, %size_2) : memref - return -} - -// CHECK-LABEL: func @dealloc_raw -func @dealloc_raw(%ctx: !tf_framework.op_kernel_context, %memref : memref) { - tf_framework.dealloc_raw(%ctx, %memref) : memref - return -} diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tests/tf_framework_legalize_to_llvm.mlir b/tensorflow/compiler/mlir/tools/kernel_gen/tests/tf_framework_legalize_to_llvm.mlir deleted file mode 100644 index 77328aa7738..00000000000 --- a/tensorflow/compiler/mlir/tools/kernel_gen/tests/tf_framework_legalize_to_llvm.mlir +++ /dev/null @@ -1,75 +0,0 @@ -// RUN: kernel-gen-opt %s -test-tf-framework-legalize-to-llvm -split-input-file | FileCheck %s - -// CHECK: llvm.func @_mlir_ciface_tf_alloc_raw -// CHECK-SAME: (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*"> - -// CHECK-LABEL: llvm.func @alloc_raw( -// CHECK-SAME: [[TF_CTX:%.*]]: !llvm<"i8*">, -// CHECK-SAME: [[SIZE_0:%.*]]: !llvm.i64, -// CHECK-SAME: [[SIZE_2:%.*]]: !llvm.i64) -> [[DESC_TY:!.*]] { -func @alloc_raw(%ctx: !tf_framework.op_kernel_context, - %size_0 : index , %size_2 : index) -> memref { - %buf = tf_framework.alloc_raw(%ctx, %size_0, %size_2) : memref - std.return %buf : memref -} -// Compute number of elements. -// CHECK: [[SIZE_1:%.*]] = llvm.mlir.constant(10 : index) : !llvm.i64 -// CHECK: [[NUM_ELEM_0:%.*]] = llvm.mul [[SIZE_0]], [[SIZE_1]] : !llvm.i64 -// CHECK: [[NUM_ELEM_1:%.*]] = llvm.mul [[NUM_ELEM_0]], [[SIZE_2]] : !llvm.i64 - -// Compute the size of an individual element. -// CHECK: [[NULL:%.*]] = llvm.mlir.null : !llvm<"float*"> -// CHECK: [[C1:%.*]] = llvm.mlir.constant(1 : index) : !llvm.i64 -// CHECK: [[GEP:%.*]] = llvm.getelementptr [[NULL]]{{\[}}[[C1]]] -// CHECK-SAME: (!llvm<"float*">, !llvm.i64) -> !llvm<"float*"> -// CHECK: [[SIZE_OF_FLOAT:%.*]] = llvm.ptrtoint [[GEP]] -// CHECK-SAME: !llvm<"float*"> to !llvm.i64 - -// Allocate memory. -// CHECK: [[NUM_BYTES:%.*]] = llvm.mul [[NUM_ELEM_1]], [[SIZE_OF_FLOAT]] -// CHECK: [[BYTES_PTR:%.*]] = llvm.call @{{.*}}([[TF_CTX]], [[NUM_BYTES]]) -// CHECK-SAME: (!llvm<"i8*">, !llvm.i64) -> !llvm<"i8*"> - -// Build memref descriptor. -// CHECK: [[DESC_0:%.*]] = llvm.mlir.undef : [[DESC_TY]] - -// Set pointers and offset. -// CHECK: [[FLOAT_PTR:%.*]] = llvm.bitcast [[BYTES_PTR]] -// CHECK-SAME: !llvm<"i8*"> to !llvm<"float*"> -// CHECK: [[DESC_1:%.*]] = llvm.insertvalue [[FLOAT_PTR]], [[DESC_0]][0] -// CHECK: [[DESC_2:%.*]] = llvm.insertvalue [[FLOAT_PTR]], [[DESC_1]][1] -// CHECK: [[C0:%.*]] = llvm.mlir.constant(0 : index) : !llvm.i64 -// CHECK: [[DESC_3:%.*]] = llvm.insertvalue [[C0]], [[DESC_2]][2] : [[DESC_TY]] - -// Set sizes and strides. -// CHECK: [[STRIDE_2:%.*]] = llvm.mlir.constant(1 : index) : !llvm.i64 -// CHECK: [[DESC_4:%.*]] = llvm.insertvalue [[SIZE_2]], [[DESC_3]][3, 2] -// CHECK: [[DESC_5:%.*]] = llvm.insertvalue [[STRIDE_2]], [[DESC_4]][4, 2] -// CHECK: [[STRIDE_1:%.*]] = llvm.mul [[STRIDE_2]], [[SIZE_2]] : !llvm.i64 -// CHECK: [[DESC_6:%.*]] = llvm.insertvalue [[SIZE_1]], [[DESC_5]][3, 1] -// CHECK: [[DESC_7:%.*]] = llvm.insertvalue [[STRIDE_1]], [[DESC_6]][4, 1] -// CHECK: [[STRIDE_0:%.*]] = llvm.mul [[STRIDE_1]], [[SIZE_1]] : !llvm.i64 -// CHECK: [[DESC_8:%.*]] = llvm.insertvalue [[SIZE_0]], [[DESC_7]][3, 0] -// CHECK: [[DESC_9:%.*]] = llvm.insertvalue [[STRIDE_0]], [[DESC_8]][4, 0] -// CHECK: llvm.return [[DESC_9]] : [[DESC_TY]] - -// ----- - -// CHECK: llvm.func @_mlir_ciface_tf_dealloc_raw(!llvm<"i8*">) - -// CHECK-LABEL: llvm.func @dealloc_raw( -// CHECK-SAME: [[TF_CTX:%.*]]: !llvm<"i8*">, -func @dealloc_raw(%ctx: !tf_framework.op_kernel_context, - %memref : memref) { - tf_framework.dealloc_raw(%ctx, %memref) : memref - return -} -// Extract allocated ptr from the memref descriptor. -// CHECK: %{{.*}} = llvm.mlir.undef : [[DESC_TY:!.*]] -// CHECK: [[FLOAT_PTR:%.*]] = llvm.extractvalue %{{.*}}[0] : [[DESC_TY]] -// CHECK-NEXT: [[VOID_PTR:%.*]] = llvm.bitcast [[FLOAT_PTR]] -// CHECK-SAME: !llvm<"float*"> to !llvm<"i8*"> - -// Deallocate. -// CHECK: llvm.call @_mlir_ciface_tf_dealloc_raw( -// CHECK-SAME: [[TF_CTX]], [[VOID_PTR]]) : (!llvm<"i8*">, !llvm<"i8*">) -> () diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD index 613422e6128..c0808ae08c4 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD @@ -35,6 +35,27 @@ cc_library( ], ) +cc_library( + name = "shape_to_descriptors", + srcs = ["shape_to_descriptors.cc"], + hdrs = [ + "passes.h", + ], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:Shape", + "@llvm-project//mlir:ShapeToSCF", + "@llvm-project//mlir:ShapeToStandard", + "@llvm-project//mlir:ShapeTransforms", + "@llvm-project//mlir:StandardOps", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + gentbl( name = "tf_framework_passes_inc_gen", tbl_outs = [("-gen-pass-decls -name TFFramework", "tf_framework_passes.h.inc")], @@ -53,6 +74,7 @@ cc_library( hdrs = ["passes.h"], deps = [ ":embed_tf_framework", + ":shape_to_descriptors", ":tf_framework_legalize_to_llvm", ":tf_framework_passes_inc_gen", "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_ops", diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h index c6aaeb92c56..5e240b8d01c 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h @@ -23,6 +23,7 @@ namespace mlir { class ModuleOp; template class OperationPass; +class Pass; namespace kernel_gen { namespace tf_framework { @@ -38,6 +39,14 @@ createTestTFFrameworkLegalizeToLLVMPass(); std::unique_ptr > createEmbedTFFrameworkPass(); } // namespace tf_framework + +namespace transforms { + +// Pass to tranform shape computations in shape dialect to standard and scf +// using memref descriptors. +std::unique_ptr CreateShapeToDescriptorsPass(); + +} // namespace transforms } // namespace kernel_gen } // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td index 8c4d5801f51..61720674926 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td @@ -21,12 +21,17 @@ include "mlir/Pass/PassBase.td" def TestTFFrameworkLegalizeToLLVMPass : Pass<"test-tf-framework-legalize-to-llvm", "ModuleOp"> { let summary = "Test pass for applying TF Framework -> LLVM patterns."; - let constructor = "createTestTFFrameworkLegalizeToLLVMPass()"; + let constructor = "tf_framework::createTestTFFrameworkLegalizeToLLVMPass()"; } def EmbedTFFrameworkPass : Pass<"embed-tf-framework", "ModuleOp"> { let summary = "Pass to embed TF Framework for allocation and error reporting"; - let constructor = "createEmbedTFFrameworkPass()"; + let constructor = "tf_framework::createEmbedTFFrameworkPass()"; +} + +def ShapeToDescriptorsPass : Pass<"test-shape-to-descriptors", "ModuleOp"> { + let summary = "Pass to transform shape computations to descriptors"; + let constructor = "transforms::CreateShapeToDescriptorsPass()"; } #endif // TF_FRAMEWORK_PASSES diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc index b9cdb2085a3..3a42d03355c 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc @@ -18,12 +18,10 @@ limitations under the License. namespace mlir { namespace kernel_gen { -namespace tf_framework { #define GEN_PASS_REGISTRATION #include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_passes.h.inc" bool register_all_passes = ([] { registerTFFrameworkPasses(); }(), true); -} // namespace tf_framework } // namespace kernel_gen } // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors.cc new file mode 100644 index 00000000000..32c2f9641b5 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors.cc @@ -0,0 +1,72 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file combines patterns for lowering shape dialect to standard ops, +// structured control flow and descriptors. + +#include "mlir/Conversion/ShapeToSCF/ShapeToSCF.h" // from @llvm-project +#include "mlir/Conversion/ShapeToStandard/ShapeToStandard.h" // from @llvm-project +#include "mlir/Dialect/SCF/SCF.h" // from @llvm-project +#include "mlir/Dialect/Shape/IR/Shape.h" // from @llvm-project +#include "mlir/Dialect/Shape/Transforms/Passes.h" // from @llvm-project +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" + +namespace mlir { +namespace kernel_gen { +namespace transforms { + +namespace { + +struct ShapeToDescriptorsPass + : public PassWrapper> { + public: + ShapeToDescriptorsPass() = default; + + void runOnOperation() override { + MLIRContext &ctx = getContext(); + + // Setup target legality. + ConversionTarget target(ctx); + target.addIllegalDialect(); + target.addLegalDialect(); + target.addLegalDialect(); + + // Setup conversion patterns. + OwningRewritePatternList patterns; + populateShapeRewritePatterns(&ctx, patterns); + populateShapeToStandardConversionPatterns(patterns, &ctx); + populateShapeToSCFConversionPatterns(patterns, &ctx); + + // Apply conversion. + auto module = getOperation(); + if (failed(applyPartialConversion(module, target, patterns))) + signalPassFailure(); + } +}; + +} // namespace + +std::unique_ptr CreateShapeToDescriptorsPass() { + return std::make_unique(); +} + +} // namespace transforms +} // namespace kernel_gen +} // namespace mlir From 6effd1682821a662629ad9b0249083728cd9c351 Mon Sep 17 00:00:00 2001 From: Tamas Nyiri Date: Mon, 3 Aug 2020 19:41:13 +0100 Subject: [PATCH 0357/1017] Add clustering to TF Model Optimization overview --- .../g3doc/performance/model_optimization.md | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tensorflow/lite/g3doc/performance/model_optimization.md b/tensorflow/lite/g3doc/performance/model_optimization.md index 14ab9c6a0c4..5e2156fc357 100644 --- a/tensorflow/lite/g3doc/performance/model_optimization.md +++ b/tensorflow/lite/g3doc/performance/model_optimization.md @@ -33,8 +33,8 @@ models have the following benefits: translate to better performance and stability. Quantization can reduce the size of a model in all of these cases, potentially -at the expense of some accuracy. Pruning can reduce the size of a model for -download by making it more easily compressible. +at the expense of some accuracy. Pruning and clustering can reduce the size of +a model for download by making it more easily compressible. ### Latency reduction @@ -54,7 +54,7 @@ Some hardware accelerators, such as the with models that have been correctly optimized. Generally, these types of devices require models to be quantized in a specific -way. See each hardware accelerators documentation to learn more about their +way. See each hardware accelerator's documentation to learn more about their requirements. ## Trade-offs @@ -70,7 +70,7 @@ certain models may gain some accuracy as a result of the optimization process. ## Types of optimization -TensorFlow Lite currently supports optimization via quantization and pruning. +TensorFlow Lite currently supports optimization via quantization, pruning and clustering. These are part of the [TensorFlow Model Optimization Toolkit](https://www.tensorflow.org/model_optimization), @@ -134,6 +134,17 @@ technique for reducing model download size. In the future, TensorFlow Lite will provide latency reduction for pruned models. +### Clustering + +[Clustering](https://www.tensorflow.org/model_optimization/guide/clustering) +works by grouping the weights of each layer in a model into a predefined +number of clusters, then sharing the centroid values for the weights +belonging to each individual cluster. This reduces the number of unique weight +values in a model, thus reducing its complexity. + +As a result, clustered models can be compressed more effectively, providing +deployment benefits similar to pruning. + ## Development workflow As a starting point, check if the models in @@ -149,4 +160,4 @@ is the better option. See additional optimization techniques under the [TensorFlow Model Optimization Toolkit](https://www.tensorflow.org/model_optimization). If you want to further reduce your model size, you can try [pruning](#pruning) -prior to quantizing your models. +and/or [clustering](#clustering) prior to quantizing your models. \ No newline at end of file From 3b6172f744fca952af09113eee709d1bed0db4de Mon Sep 17 00:00:00 2001 From: Tamas Bela Feher Date: Tue, 2 Jun 2020 20:11:18 +0200 Subject: [PATCH 0358/1017] Add TF-TRT op converter tests for reduce ops --- .../tf2tensorrt/convert/convert_nodes.cc | 2 +- .../tf2tensorrt/convert/convert_nodes_test.cc | 142 ++++++++++++++++++ 2 files changed, 143 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc index 369b339d01a..eaff361f09d 100644 --- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc +++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc @@ -4479,7 +4479,7 @@ Status ConvertReduce(OpConverterParams* params) { int trt_axis; TF_RETURN_IF_ERROR( ConvertAxis(tf_axes_list[i], tensor->getDimensions().nbDims, - node_def.name(), /*use_implicit_batch=*/true, &trt_axis)); + node_def.name(), params->use_implicit_batch, &trt_axis)); axes |= (1 << trt_axis); } diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc index 52d05ff8225..0de2916857b 100644 --- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc +++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc @@ -5052,6 +5052,148 @@ TEST_P(OpConverterTest3, ConvertGather) { } } +template +NodeDef CreateReduceOp(DataType tf_type, bool keep_dims) { + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), tf_type); + auto axis = ops::Placeholder(s.WithOpName("axis"), DT_INT32); + typename OpType::Attrs op_attrs; + op_attrs.keep_dims_ = keep_dims; + auto op = OpType(s.WithOpName("my_reduce"), input, axis, op_attrs); + return op.operation.node()->def(); +} + +// Applies reduction op on sub-sequences of input +// output[i] = reduce(input[m * i : m * (i +1)]) +std::vector CalcReduce(string op_name, std::vector input, int m, + float (*op)(float, float), float init) { + std::vector output(input.size() / m); + for (int i = 0; i < output.size(); i++) { + auto begin = input.begin() + i * m; + auto end = input.begin() + (i + 1) * m; + output[i] = std::accumulate(begin, end, init, op); + if (op_name == "Mean") { + output[i] /= m; + } + } + return output; +} +TEST_P(OpConverterTest1, ConvertReduce) { + { + // Input is weights, should fail. + Reset(); + const NodeDef node_def = CreateReduceOp(tf_type, false); + AddTestWeights("input", {1, 2, 3}, {-3, -2, -1, 0, 1, 2}); + AddTestWeights("axis", {1}, {1}); + RunValidationAndConversion( + node_def, error::UNIMPLEMENTED, + "The input \"input\" for Sum must be a tensor, at my_reduce"); + } + { + // Axis is weights, should fail. + Reset(); + const NodeDef node_def = CreateReduceOp(tf_type, false); + AddTestTensor("input", {1, 2, 3}, {-3, -2, -1, 0, 1, 2}); + AddTestTensor("axis", {1}, DT_INT32, {1}); + RunValidationAndConversion( + node_def, error::UNIMPLEMENTED, + "The input \"axis\" for Sum must be a constant, at my_reduce"); + } + using OpFunc = std::function; + using ValFunc = float (*)(float, float); + struct ReduceTestDescriptor { + string name; + OpFunc get_node; + ValFunc val_func; + float init_val; + }; + std::vector op_test_info{ + {"Sum", CreateReduceOp, [](float x, float y) { return x + y; }, + 0}, + {"Prod", CreateReduceOp, + [](float x, float y) { return x * y; }, 1}, + {"Mean", CreateReduceOp, + [](float x, float y) { return x + y; }, 0}, + {"Min", CreateReduceOp, + [](float x, float y) { return y < x ? y : x; }, 1000}, + {"Max", CreateReduceOp, + [](float x, float y) { return x < y ? y : x; }, -1000}}; + + std::vector input_values{1, 2, 3, 4, 5, 6}; + struct TestParams { + std::vector input_dims; + std::vector input_values; + // Helper array contains the same elements as input but permuted in a way + // that the reduction can be calculated over contiguous elements using + // CalcReduce + std::vector helper_array; + std::vector axis; + int stride; // product of input_dims along axis + Status conversion_status; + }; + std::vector params{ + // Out of range tests + TestParams{{2, 3, 1}, input_values, input_values, {3}, 3}, + TestParams{{2, 3, 1}, input_values, input_values, {-4}, 3}, + // Ok tests + TestParams{{2, 3, 1}, input_values, {1, 4, 2, 5, 3, 6}, {0}, 2}, + TestParams{{2, 3, 1}, input_values, input_values, {1}, 3}, + TestParams{{2, 3, 1}, input_values, input_values, {2}, 1}, + TestParams{{2, 3, 1}, input_values, input_values, {0, 1}, 6}, + // Ok tests with negative axis values + TestParams{{2, 3, 1}, input_values, {1, 4, 2, 5, 3, 6}, {-3}, 2}, + TestParams{{2, 3, 1}, input_values, input_values, {-2}, 3}, + TestParams{{2, 3, 1}, input_values, input_values, {-1}, 1}, + TestParams{{2, 3, 1}, input_values, input_values, {-3, 1}, 6}, + }; + + for (bool keep_dims : {false, true}) { + for (auto& op : op_test_info) { + for (auto p : params) { + SCOPED_TRACE(StrCat(op.name, keep_dims ? "keep_dims" : "")); + Reset(); + NodeDef node_def = op.get_node(tf_type, keep_dims); + + AddTestTensor("input", p.input_dims, p.input_values); + AddTestWeights("axis", {static_cast(p.axis.size())}, + p.axis); + std::vector expected_output_dims(p.input_dims); + + // Set expected output dim and conversion error messages + for (int ax : p.axis) { + int rank = p.input_dims.size(); + if (ax >= rank || ax < -rank) { + p.conversion_status = + errors::InvalidArgument("Axis value of ", ax, + " is out of bounds, must be in " + "range [", + -rank, ", ", rank, "), at my_reduce"); + } else { + int ax_positive = ax >= 0 ? ax : ax + rank; + // Zero marks elements that we will remove later. + expected_output_dims[ax_positive] = keep_dims ? 1 : 0; + if (trt_mode == TrtTestMode::kImplicitBatch && + (ax == 0 || ax == -rank)) { + p.conversion_status = errors::Unimplemented( + "TensorRT does not allow manipulation of the batch " + "dimension, at my_reduce"); + } + } + } + expected_output_dims.erase(std::remove(expected_output_dims.begin(), + expected_output_dims.end(), 0), + expected_output_dims.end()); + VLOG(2) << "out dims " << expected_output_dims; + std::vector expected_values = CalcReduce( + op.name, p.helper_array, p.stride, op.val_func, op.init_val); + TestOpConverter("my_reduce", node_def, expected_output_dims, + p.conversion_status, Status::OK(), + ArrayFloatNear(expected_values)); + } + } + } +} + NodeDef CreateCastOp(DataType tf_type) { Scope s = Scope::NewRootScope(); auto input = ops::Placeholder(s.WithOpName("input"), DT_HALF); From 447537756f834098176fba4abf44b19cce59434e Mon Sep 17 00:00:00 2001 From: Zhenyu Tan Date: Tue, 4 Aug 2020 08:12:40 -0700 Subject: [PATCH 0359/1017] fix skip test feature column part 2: Embedding Column PiperOrigin-RevId: 324813679 Change-Id: I7849ac67794d759e8b3cbd5fb1279e5bbb775b28 --- .../feature_column/feature_column_test.py | 663 +++++++++--------- 1 file changed, 328 insertions(+), 335 deletions(-) diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index d6d4d2eb1a1..e351c5da572 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -4794,7 +4794,6 @@ class IndicatorColumnTest(test.TestCase): class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): - @test_util.run_deprecated_v1 def test_defaults(self): categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=3) @@ -4816,7 +4815,6 @@ class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column._parse_example_spec) - @test_util.run_deprecated_v1 def test_all_constructor_args(self): categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=3) @@ -4845,7 +4843,6 @@ class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column._parse_example_spec) - @test_util.run_deprecated_v1 def test_deep_copy(self): categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=3) @@ -4879,7 +4876,6 @@ class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column._parse_example_spec) - @test_util.run_deprecated_v1 def test_invalid_initializer(self): categorical_column = fc._categorical_column_with_identity( key='aaa', num_buckets=3) @@ -4908,25 +4904,24 @@ class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): sparse_tensor.SparseTensorValue( indices=[[0, 0], [0, 1]], values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) + dense_shape=[1, 2]), features['aaa'].eval()) - @test_util.run_deprecated_v1 def test_transform_feature(self): - a = fc._categorical_column_with_identity(key='aaa', num_buckets=3) - a_embedded = fc._embedding_column(a, dimension=2) - features = { - 'aaa': sparse_tensor.SparseTensor( - indices=((0, 0), (1, 0), (1, 1)), - values=(0, 1, 0), - dense_shape=(2, 2)) - } - outputs = _transform_features(features, [a, a_embedded]) - output_a = outputs[a] - output_embedded = outputs[a_embedded] - with _initialized_session(): - _assert_sparse_tensor_value(self, self.evaluate(output_a), - self.evaluate(output_embedded)) + with ops.Graph().as_default(): + a = fc._categorical_column_with_identity(key='aaa', num_buckets=3) + a_embedded = fc._embedding_column(a, dimension=2) + features = { + 'aaa': sparse_tensor.SparseTensor( + indices=((0, 0), (1, 0), (1, 1)), + values=(0, 1, 0), + dense_shape=(2, 2)) + } + outputs = _transform_features(features, [a, a_embedded]) + output_a = outputs[a] + output_embedded = outputs[a_embedded] + with _initialized_session(): + _assert_sparse_tensor_value(self, self.evaluate(output_a), + self.evaluate(output_embedded)) @parameterized.named_parameters( { @@ -4946,184 +4941,183 @@ class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): 'use_safe_embedding_lookup': False, 'partition_variables': True, }) - @test_util.run_deprecated_v1 + def test_get_dense_tensor(self, use_safe_embedding_lookup, partition_variables): - # Inputs. - vocabulary_size = 4 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - indices=((0, 0), (1, 0), (1, 4), (3, 0)), - values=(2, 0, 1, 1), - dense_shape=(4, 5)) + with ops.Graph().as_default(): + # Inputs. + vocabulary_size = 4 + sparse_input = sparse_tensor.SparseTensorValue( + # example 0, ids [2] + # example 1, ids [0, 1] + # example 2, ids [] + # example 3, ids [1] + indices=((0, 0), (1, 0), (1, 4), (3, 0)), + values=(2, 0, 1, 1), + dense_shape=(4, 5)) - # Embedding variable. - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.), # id 2 - (9., 13.) # id 3 - ) + # Embedding variable. + embedding_dimension = 2 + embedding_values = ( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.), # id 2 + (9., 13.) # id 3 + ) - def _initializer(shape, dtype, partition_info=None): + def _initializer(shape, dtype, partition_info=None): + if partition_variables: + self.assertEqual([vocabulary_size, embedding_dimension], + partition_info.full_shape) + self.assertAllEqual((2, embedding_dimension), shape) + else: + self.assertAllEqual((vocabulary_size, embedding_dimension), shape) + self.assertIsNone(partition_info) + + self.assertEqual(dtypes.float32, dtype) + return embedding_values + + # Expected lookup result, using combiner='mean'. + expected_lookups = ( + # example 0, ids [2], embedding = [7, 11] + (7., 11.), + # example 1, ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] + (2., 3.5), + # example 2, ids [], embedding = [0, 0] + (0., 0.), + # example 3, ids [1], embedding = [3, 5] + (3., 5.), + ) + + # Build columns. + categorical_column = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + partitioner = None if partition_variables: - self.assertEqual([vocabulary_size, embedding_dimension], - partition_info.full_shape) - self.assertAllEqual((2, embedding_dimension), shape) + partitioner = partitioned_variables.fixed_size_partitioner(2, axis=0) + with variable_scope.variable_scope('vars', partitioner=partitioner): + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, + initializer=_initializer, + use_safe_embedding_lookup=use_safe_embedding_lookup) + + # Provide sparse input and get dense result. + embedding_lookup = embedding_column._get_dense_tensor( + _LazyBuilder({'aaa': sparse_input})) + + # Assert expected embedding variable and lookups. + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + if partition_variables: + self.assertCountEqual(('vars/embedding_weights/part_0:0', + 'vars/embedding_weights/part_1:0'), + tuple([v.name for v in global_vars])) else: + self.assertCountEqual(('vars/embedding_weights:0',), + tuple([v.name for v in global_vars])) + for v in global_vars: + self.assertIsInstance(v, variables_lib.Variable) + with _initialized_session(): + self.assertAllEqual(embedding_values, global_vars[0]) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + + if use_safe_embedding_lookup: + self.assertIn( + 'SparseFillEmptyRows', + [x.type for x in ops.get_default_graph().get_operations()]) + else: + self.assertNotIn( + 'SparseFillEmptyRows', + [x.type for x in ops.get_default_graph().get_operations()]) + + def test_get_dense_tensor_3d(self): + with ops.Graph().as_default(): + # Inputs. + vocabulary_size = 4 + sparse_input = sparse_tensor.SparseTensorValue( + # example 0, ids [2] + # example 1, ids [0, 1] + # example 2, ids [] + # example 3, ids [1] + indices=((0, 0, 0), (1, 1, 0), (1, 1, 4), (3, 0, 0), (3, 1, 2)), + values=(2, 0, 1, 1, 2), + dense_shape=(4, 2, 5)) + + # Embedding variable. + embedding_dimension = 3 + embedding_values = ( + (1., 2., 4.), # id 0 + (3., 5., 1.), # id 1 + (7., 11., 2.), # id 2 + (2., 7., 12.) # id 3 + ) + + def _initializer(shape, dtype, partition_info): self.assertAllEqual((vocabulary_size, embedding_dimension), shape) + self.assertEqual(dtypes.float32, dtype) self.assertIsNone(partition_info) + return embedding_values - self.assertEqual(dtypes.float32, dtype) - return embedding_values + # Expected lookup result, using combiner='mean'. + expected_lookups = ( + # example 0, ids [[2], []], embedding = [[7, 11, 2], [0, 0, 0]] + ((7., 11., 2.), (0., 0., 0.)), + # example 1, ids [[], [0, 1]], embedding + # = mean([[], [1, 2, 4] + [3, 5, 1]]) = [[0, 0, 0], [2, 3.5, 2.5]] + ((0., 0., 0.), (2., 3.5, 2.5)), + # example 2, ids [[], []], embedding = [[0, 0, 0], [0, 0, 0]] + ((0., 0., 0.), (0., 0., 0.)), + # example 3, ids [[1], [2]], embedding = [[3, 5, 1], [7, 11, 2]] + ((3., 5., 1.), (7., 11., 2.)), + ) - # Expected lookup result, using combiner='mean'. - expected_lookups = ( - # example 0, ids [2], embedding = [7, 11] - (7., 11.), - # example 1, ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] - (2., 3.5), - # example 2, ids [], embedding = [0, 0] - (0., 0.), - # example 3, ids [1], embedding = [3, 5] - (3., 5.), - ) - - # Build columns. - categorical_column = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - partitioner = None - if partition_variables: - partitioner = partitioned_variables.fixed_size_partitioner(2, axis=0) - with variable_scope.variable_scope('vars', partitioner=partitioner): + # Build columns. + categorical_column = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) embedding_column = fc._embedding_column( categorical_column, dimension=embedding_dimension, - initializer=_initializer, - use_safe_embedding_lookup=use_safe_embedding_lookup) + initializer=_initializer) # Provide sparse input and get dense result. embedding_lookup = embedding_column._get_dense_tensor( _LazyBuilder({'aaa': sparse_input})) - # Assert expected embedding variable and lookups. - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - if partition_variables: - self.assertCountEqual(('vars/embedding_weights/part_0:0', - 'vars/embedding_weights/part_1:0'), + # Assert expected embedding variable and lookups. + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + self.assertCountEqual(('embedding_weights:0',), tuple([v.name for v in global_vars])) - else: - self.assertCountEqual(('vars/embedding_weights:0',), - tuple([v.name for v in global_vars])) - for v in global_vars: - self.assertIsInstance(v, variables_lib.Variable) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0]) - self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + with _initialized_session(): + self.assertAllEqual(embedding_values, global_vars[0]) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) - if use_safe_embedding_lookup: - self.assertIn('SparseFillEmptyRows', - [x.type for x in ops.get_default_graph().get_operations()]) - else: - self.assertNotIn( - 'SparseFillEmptyRows', - [x.type for x in ops.get_default_graph().get_operations()]) - - @test_util.run_deprecated_v1 - def test_get_dense_tensor_3d(self): - # Inputs. - vocabulary_size = 4 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - indices=((0, 0, 0), (1, 1, 0), (1, 1, 4), (3, 0, 0), (3, 1, 2)), - values=(2, 0, 1, 1, 2), - dense_shape=(4, 2, 5)) - - # Embedding variable. - embedding_dimension = 3 - embedding_values = ( - (1., 2., 4.), # id 0 - (3., 5., 1.), # id 1 - (7., 11., 2.), # id 2 - (2., 7., 12.) # id 3 - ) - def _initializer(shape, dtype, partition_info): - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return embedding_values - - # Expected lookup result, using combiner='mean'. - expected_lookups = ( - # example 0, ids [[2], []], embedding = [[7, 11, 2], [0, 0, 0]] - ((7., 11., 2.), (0., 0., 0.)), - # example 1, ids [[], [0, 1]], embedding - # = mean([[], [1, 2, 4] + [3, 5, 1]]) = [[0, 0, 0], [2, 3.5, 2.5]] - ((0., 0., 0.), (2., 3.5, 2.5)), - # example 2, ids [[], []], embedding = [[0, 0, 0], [0, 0, 0]] - ((0., 0., 0.), (0., 0., 0.)), - # example 3, ids [[1], [2]], embedding = [[3, 5, 1], [7, 11, 2]] - ((3., 5., 1.), (7., 11., 2.)), - ) - - # Build columns. - categorical_column = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column = fc._embedding_column( - categorical_column, - dimension=embedding_dimension, - initializer=_initializer) - - # Provide sparse input and get dense result. - embedding_lookup = embedding_column._get_dense_tensor( - _LazyBuilder({ - 'aaa': sparse_input - })) - - # Assert expected embedding variable and lookups. - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertCountEqual(('embedding_weights:0',), - tuple([v.name for v in global_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0]) - self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) - - @test_util.run_deprecated_v1 def test_get_dense_tensor_weight_collections(self): - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - indices=((0, 0), (1, 0), (1, 4), (3, 0)), - values=(2, 0, 1, 1), - dense_shape=(4, 5)) + with ops.Graph().as_default(): + sparse_input = sparse_tensor.SparseTensorValue( + # example 0, ids [2] + # example 1, ids [0, 1] + # example 2, ids [] + # example 3, ids [1] + indices=((0, 0), (1, 0), (1, 4), (3, 0)), + values=(2, 0, 1, 1), + dense_shape=(4, 5)) - # Build columns. - categorical_column = fc._categorical_column_with_identity( - key='aaa', num_buckets=3) - embedding_column = fc._embedding_column(categorical_column, dimension=2) + # Build columns. + categorical_column = fc._categorical_column_with_identity( + key='aaa', num_buckets=3) + embedding_column = fc._embedding_column(categorical_column, dimension=2) - # Provide sparse input and get dense result. - embedding_column._get_dense_tensor( - _LazyBuilder({ - 'aaa': sparse_input - }), weight_collections=('my_vars',)) + # Provide sparse input and get dense result. + embedding_column._get_dense_tensor( + _LazyBuilder({'aaa': sparse_input}), weight_collections=('my_vars',)) - # Assert expected embedding variable and lookups. - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertCountEqual(('embedding_weights:0',), - tuple([v.name for v in global_vars])) - my_vars = ops.get_collection('my_vars') - self.assertCountEqual(('embedding_weights:0',), - tuple([v.name for v in my_vars])) + # Assert expected embedding variable and lookups. + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + self.assertCountEqual(('embedding_weights:0',), + tuple([v.name for v in global_vars])) + my_vars = ops.get_collection('my_vars') + self.assertCountEqual(('embedding_weights:0',), + tuple([v.name for v in my_vars])) @test_util.run_deprecated_v1 def test_get_dense_tensor_placeholder_inputs(self): @@ -5197,66 +5191,63 @@ class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): input_shape: sparse_input.dense_shape, })) - @test_util.run_deprecated_v1 def test_get_dense_tensor_restore_from_ckpt(self): - # Inputs. - vocabulary_size = 3 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - indices=((0, 0), (1, 0), (1, 4), (3, 0)), - values=(2, 0, 1, 1), - dense_shape=(4, 5)) + with ops.Graph().as_default(): + # Inputs. + vocabulary_size = 3 + sparse_input = sparse_tensor.SparseTensorValue( + # example 0, ids [2] + # example 1, ids [0, 1] + # example 2, ids [] + # example 3, ids [1] + indices=((0, 0), (1, 0), (1, 4), (3, 0)), + values=(2, 0, 1, 1), + dense_shape=(4, 5)) - # Embedding variable. The checkpoint file contains _embedding_values. - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - ) - ckpt_path = test.test_src_dir_path( - 'python/feature_column/testdata/embedding.ckpt') - ckpt_tensor = 'my_embedding' + # Embedding variable. The checkpoint file contains _embedding_values. + embedding_dimension = 2 + embedding_values = ( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.) # id 2 + ) + ckpt_path = test.test_src_dir_path( + 'python/feature_column/testdata/embedding.ckpt') + ckpt_tensor = 'my_embedding' - # Expected lookup result, using combiner='mean'. - expected_lookups = ( - # example 0, ids [2], embedding = [7, 11] - (7., 11.), - # example 1, ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] - (2., 3.5), - # example 2, ids [], embedding = [0, 0] - (0., 0.), - # example 3, ids [1], embedding = [3, 5] - (3., 5.), - ) + # Expected lookup result, using combiner='mean'. + expected_lookups = ( + # example 0, ids [2], embedding = [7, 11] + (7., 11.), + # example 1, ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] + (2., 3.5), + # example 2, ids [], embedding = [0, 0] + (0., 0.), + # example 3, ids [1], embedding = [3, 5] + (3., 5.), + ) - # Build columns. - categorical_column = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column = fc._embedding_column( - categorical_column, - dimension=embedding_dimension, - ckpt_to_load_from=ckpt_path, - tensor_name_in_ckpt=ckpt_tensor) + # Build columns. + categorical_column = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, + ckpt_to_load_from=ckpt_path, + tensor_name_in_ckpt=ckpt_tensor) - # Provide sparse input and get dense result. - embedding_lookup = embedding_column._get_dense_tensor( - _LazyBuilder({ - 'aaa': sparse_input - })) + # Provide sparse input and get dense result. + embedding_lookup = embedding_column._get_dense_tensor( + _LazyBuilder({'aaa': sparse_input})) - # Assert expected embedding variable and lookups. - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertCountEqual(('embedding_weights:0',), - tuple([v.name for v in global_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0]) - self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) + # Assert expected embedding variable and lookups. + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + self.assertCountEqual(('embedding_weights:0',), + tuple([v.name for v in global_vars])) + with _initialized_session(): + self.assertAllEqual(embedding_values, global_vars[0]) + self.assertAllEqual(expected_lookups, self.evaluate(embedding_lookup)) - @test_util.run_deprecated_v1 def test_linear_model(self): # Inputs. batch_size = 4 @@ -5336,7 +5327,6 @@ class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), self.evaluate(predictions)) - @test_util.run_deprecated_v1 def test_keras_linear_model(self): # Inputs. batch_size = 4 @@ -5416,125 +5406,128 @@ class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), self.evaluate(predictions)) - @test_util.run_deprecated_v1 def test_input_layer(self): - # Inputs. - vocabulary_size = 3 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - indices=((0, 0), (1, 0), (1, 4), (3, 0)), - values=(2, 0, 1, 1), - dense_shape=(4, 5)) + with ops.Graph().as_default(): + # Inputs. + vocabulary_size = 3 + sparse_input = sparse_tensor.SparseTensorValue( + # example 0, ids [2] + # example 1, ids [0, 1] + # example 2, ids [] + # example 3, ids [1] + indices=((0, 0), (1, 0), (1, 4), (3, 0)), + values=(2, 0, 1, 1), + dense_shape=(4, 5)) - # Embedding variable. - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - ) - def _initializer(shape, dtype, partition_info): - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return embedding_values + # Embedding variable. + embedding_dimension = 2 + embedding_values = ( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.) # id 2 + ) - # Expected lookup result, using combiner='mean'. - expected_lookups = ( - # example 0, ids [2], embedding = [7, 11] - (7., 11.), - # example 1, ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] - (2., 3.5), - # example 2, ids [], embedding = [0, 0] - (0., 0.), - # example 3, ids [1], embedding = [3, 5] - (3., 5.), - ) + def _initializer(shape, dtype, partition_info): + self.assertAllEqual((vocabulary_size, embedding_dimension), shape) + self.assertEqual(dtypes.float32, dtype) + self.assertIsNone(partition_info) + return embedding_values - # Build columns. - categorical_column = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column = fc._embedding_column( - categorical_column, - dimension=embedding_dimension, - initializer=_initializer) + # Expected lookup result, using combiner='mean'. + expected_lookups = ( + # example 0, ids [2], embedding = [7, 11] + (7., 11.), + # example 1, ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] + (2., 3.5), + # example 2, ids [], embedding = [0, 0] + (0., 0.), + # example 3, ids [1], embedding = [3, 5] + (3., 5.), + ) - # Provide sparse input and get dense result. - input_layer = fc.input_layer({'aaa': sparse_input}, (embedding_column,)) + # Build columns. + categorical_column = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, + initializer=_initializer) - # Assert expected embedding variable and lookups. - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertCountEqual(('input_layer/aaa_embedding/embedding_weights:0',), - tuple([v.name for v in global_vars])) - trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) - self.assertCountEqual(('input_layer/aaa_embedding/embedding_weights:0',), - tuple([v.name for v in trainable_vars])) - with _initialized_session(): - self.assertAllEqual(embedding_values, trainable_vars[0]) - self.assertAllEqual(expected_lookups, self.evaluate(input_layer)) + # Provide sparse input and get dense result. + input_layer = fc.input_layer({'aaa': sparse_input}, (embedding_column,)) + + # Assert expected embedding variable and lookups. + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + self.assertCountEqual(('input_layer/aaa_embedding/embedding_weights:0',), + tuple([v.name for v in global_vars])) + trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) + self.assertCountEqual(('input_layer/aaa_embedding/embedding_weights:0',), + tuple([v.name for v in trainable_vars])) + with _initialized_session(): + self.assertAllEqual(embedding_values, trainable_vars[0]) + self.assertAllEqual(expected_lookups, self.evaluate(input_layer)) - @test_util.run_deprecated_v1 def test_input_layer_not_trainable(self): - # Inputs. - vocabulary_size = 3 - sparse_input = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - # example 2, ids [] - # example 3, ids [1] - indices=((0, 0), (1, 0), (1, 4), (3, 0)), - values=(2, 0, 1, 1), - dense_shape=(4, 5)) + with ops.Graph().as_default(): + # Inputs. + vocabulary_size = 3 + sparse_input = sparse_tensor.SparseTensorValue( + # example 0, ids [2] + # example 1, ids [0, 1] + # example 2, ids [] + # example 3, ids [1] + indices=((0, 0), (1, 0), (1, 4), (3, 0)), + values=(2, 0, 1, 1), + dense_shape=(4, 5)) - # Embedding variable. - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - ) - def _initializer(shape, dtype, partition_info): - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return embedding_values + # Embedding variable. + embedding_dimension = 2 + embedding_values = ( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.) # id 2 + ) - # Expected lookup result, using combiner='mean'. - expected_lookups = ( - # example 0, ids [2], embedding = [7, 11] - (7., 11.), - # example 1, ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] - (2., 3.5), - # example 2, ids [], embedding = [0, 0] - (0., 0.), - # example 3, ids [1], embedding = [3, 5] - (3., 5.), - ) + def _initializer(shape, dtype, partition_info): + self.assertAllEqual((vocabulary_size, embedding_dimension), shape) + self.assertEqual(dtypes.float32, dtype) + self.assertIsNone(partition_info) + return embedding_values - # Build columns. - categorical_column = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - embedding_column = fc._embedding_column( - categorical_column, - dimension=embedding_dimension, - initializer=_initializer, - trainable=False) + # Expected lookup result, using combiner='mean'. + expected_lookups = ( + # example 0, ids [2], embedding = [7, 11] + (7., 11.), + # example 1, ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] + (2., 3.5), + # example 2, ids [], embedding = [0, 0] + (0., 0.), + # example 3, ids [1], embedding = [3, 5] + (3., 5.), + ) - # Provide sparse input and get dense result. - input_layer = fc.input_layer({'aaa': sparse_input}, (embedding_column,)) + # Build columns. + categorical_column = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + embedding_column = fc._embedding_column( + categorical_column, + dimension=embedding_dimension, + initializer=_initializer, + trainable=False) - # Assert expected embedding variable and lookups. - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertCountEqual(('input_layer/aaa_embedding/embedding_weights:0',), - tuple([v.name for v in global_vars])) - self.assertCountEqual([], - ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)) - with _initialized_session(): - self.assertAllEqual(embedding_values, global_vars[0]) - self.assertAllEqual(expected_lookups, self.evaluate(input_layer)) + # Provide sparse input and get dense result. + input_layer = fc.input_layer({'aaa': sparse_input}, (embedding_column,)) + + # Assert expected embedding variable and lookups. + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + self.assertCountEqual(('input_layer/aaa_embedding/embedding_weights:0',), + tuple([v.name for v in global_vars])) + self.assertCountEqual([], + ops.get_collection( + ops.GraphKeys.TRAINABLE_VARIABLES)) + with _initialized_session(): + self.assertAllEqual(embedding_values, global_vars[0]) + self.assertAllEqual(expected_lookups, self.evaluate(input_layer)) class SharedEmbeddingColumnTest(test.TestCase, parameterized.TestCase): From cfaecb4257b7808cedac06de4ddb417796211c87 Mon Sep 17 00:00:00 2001 From: Yixing Fu Date: Tue, 4 Aug 2020 15:59:30 +0000 Subject: [PATCH 0360/1017] throw error when h5 save fail; create dir before saving --- tensorflow/python/keras/callbacks.py | 2 ++ tensorflow/python/keras/engine/training.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py index 5a191263241..0d795a99d06 100644 --- a/tensorflow/python/keras/callbacks.py +++ b/tensorflow/python/keras/callbacks.py @@ -1358,6 +1358,8 @@ class ModelCheckpoint(Callback): raise IOError('Please specify a non-directory filepath for ' 'ModelCheckpoint. Filepath used is an existing ' 'directory: {}'.format(filepath)) + # Re-throw the error for any other causes. + raise e def _get_file_path(self, epoch, logs): """Returns the file path for checkpoint.""" diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index 15f77ab8a96..c69c5bb2ebd 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -69,6 +69,7 @@ from tensorflow.python.ops import variables from tensorflow.python.ops.ragged import ragged_concat_ops from tensorflow.python.ops.ragged import ragged_tensor from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.platform import gfile from tensorflow.python.profiler import trace from tensorflow.python.training import checkpoint_management from tensorflow.python.training import py_checkpoint_reader @@ -2094,6 +2095,8 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): if not proceed: return if save_format == 'h5': + dirpath = os.path.dirname(filepath) + gfile.MakeDirs(dirpath) with h5py.File(filepath, 'w') as f: hdf5_format.save_weights_to_hdf5_group(f, self.layers) else: From 9ce602d8206b1f3a78613c69ae1e3fe3de477f26 Mon Sep 17 00:00:00 2001 From: Yixing Fu Date: Tue, 4 Aug 2020 16:09:55 +0000 Subject: [PATCH 0361/1017] change in saving instead of training --- tensorflow/python/keras/engine/training.py | 3 --- tensorflow/python/keras/saving/hdf5_format.py | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index c69c5bb2ebd..15f77ab8a96 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -69,7 +69,6 @@ from tensorflow.python.ops import variables from tensorflow.python.ops.ragged import ragged_concat_ops from tensorflow.python.ops.ragged import ragged_tensor from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.platform import gfile from tensorflow.python.profiler import trace from tensorflow.python.training import checkpoint_management from tensorflow.python.training import py_checkpoint_reader @@ -2095,8 +2094,6 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): if not proceed: return if save_format == 'h5': - dirpath = os.path.dirname(filepath) - gfile.MakeDirs(dirpath) with h5py.File(filepath, 'w') as f: hdf5_format.save_weights_to_hdf5_group(f, self.layers) else: diff --git a/tensorflow/python/keras/saving/hdf5_format.py b/tensorflow/python/keras/saving/hdf5_format.py index 31c9a6e14e0..4bc881ae5e2 100644 --- a/tensorflow/python/keras/saving/hdf5_format.py +++ b/tensorflow/python/keras/saving/hdf5_format.py @@ -35,6 +35,7 @@ from tensorflow.python.keras.utils.generic_utils import LazyLoader from tensorflow.python.keras.utils.io_utils import ask_to_proceed_with_overwrite from tensorflow.python.ops import variables as variables_module from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.platform import gfile # pylint: disable=g-import-not-at-top try: @@ -99,6 +100,10 @@ def save_model_to_hdf5(model, filepath, overwrite=True, include_optimizer=True): if not proceed: return + # Try creating dir if not exist + dirpath = os.path.dirname(filepath) + gfile.MakeDirs(dirpath) + f = h5py.File(filepath, mode='w') opened_new_file = True else: From 14a136df8632233c81d46e331be53e9d3b30f8cd Mon Sep 17 00:00:00 2001 From: Bixia Zheng Date: Tue, 4 Aug 2020 09:01:21 -0700 Subject: [PATCH 0362/1017] [TF:TRT] Disable some python tests for TensorRT 7.1.3. Add IsTensorRTVersionGreaterEqual to check the linked TensorRT version and use the routine to skip the tests that fail with TensorRT 7.1.3. Modify lru_cache_test to not use Conv2D. This is to workaround a bug in TensorRT 7.1.3. PiperOrigin-RevId: 324821921 Change-Id: Icf7f0b239917e0417f090f904192837181380eae --- .../python/compiler/tensorrt/test/base_test.py | 12 ++++++++++++ .../tensorrt/test/combined_nms_test.py | 3 +++ .../tensorrt/test/const_broadcast_test.py | 6 ++++++ .../compiler/tensorrt/test/conv2d_test.py | 18 ++++++++++++++++++ .../tensorrt/test/dynamic_input_shapes_test.py | 3 +++ .../compiler/tensorrt/test/lru_cache_test.py | 14 +++----------- .../tensorrt/test/memory_alignment_test.py | 6 ++++++ .../multi_connection_neighbor_engine_test.py | 6 ++++++ .../tensorrt/test/neighboring_engine_test.py | 6 ++++++ .../tensorrt/test/quantization_mnist_test.py | 8 ++++++-- .../test/tf_trt_integration_test_base.py | 7 +++++++ .../tensorrt/test/vgg_block_nchw_test.py | 6 ++++++ .../compiler/tensorrt/test/vgg_block_test.py | 6 ++++++ 13 files changed, 88 insertions(+), 13 deletions(-) diff --git a/tensorflow/python/compiler/tensorrt/test/base_test.py b/tensorflow/python/compiler/tensorrt/test/base_test.py index 9d2d3abd4fb..195382cd8ed 100644 --- a/tensorflow/python/compiler/tensorrt/test/base_test.py +++ b/tensorflow/python/compiler/tensorrt/test/base_test.py @@ -70,6 +70,12 @@ class SimpleSingleEngineTest(trt_test.TfTrtIntegrationTestBase): ] } + def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162448349") + return super().ShouldRunTest(run_params) + class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase): @@ -130,6 +136,12 @@ class SimpleMultiEnginesTest(trt_test.TfTrtIntegrationTestBase): return conversion_params._replace( rewriter_config_template=rewrite_config_with_trt) + def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162448349") + return super().ShouldRunTest(run_params) + class SimpleMultiEnginesTest2(trt_test.TfTrtIntegrationTestBase): diff --git a/tensorflow/python/compiler/tensorrt/test/combined_nms_test.py b/tensorflow/python/compiler/tensorrt/test/combined_nms_test.py index ffb1bf85e87..71f5139d049 100644 --- a/tensorflow/python/compiler/tensorrt/test/combined_nms_test.py +++ b/tensorflow/python/compiler/tensorrt/test/combined_nms_test.py @@ -90,6 +90,9 @@ class CombinedNmsTest(trt_test.TfTrtIntegrationTestBase): } def ShouldRunTest(self, run_params): + # TODO(b/162447069): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, 'Skip test due to b/162447069') # There is no CombinedNonMaxSuppression op for GPU at the moment, so # calibration will fail. # TODO(laigd): fix this. diff --git a/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py b/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py index ccbaf9e52fa..9e71b9e3f75 100644 --- a/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py +++ b/tensorflow/python/compiler/tensorrt/test/const_broadcast_test.py @@ -60,6 +60,12 @@ class ConstBroadcastTest(trt_test.TfTrtIntegrationTestBase): """The relative tolerance to compare floating point results.""" return 1.e-04 if run_params.precision_mode == 'FP32' else 1.e-02 + def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, 'Skip test due to b/162448349') + return super().ShouldRunTest(run_params) + if __name__ == '__main__': test.main() diff --git a/tensorflow/python/compiler/tensorrt/test/conv2d_test.py b/tensorflow/python/compiler/tensorrt/test/conv2d_test.py index df1adce2178..400c17b343e 100644 --- a/tensorflow/python/compiler/tensorrt/test/conv2d_test.py +++ b/tensorflow/python/compiler/tensorrt/test/conv2d_test.py @@ -114,6 +114,12 @@ class Conv2DNCHWTest(trt_test.TfTrtIntegrationTestBase): return 4e-02 return super(Conv2DNCHWTest, self).ExpectedRelativeTolerance(run_params) + def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162448349") + return super().ShouldRunTest(run_params) + class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase): """Testing conversion of Conv2D (data_format=NCHW) in TF-TRT conversion.""" @@ -137,6 +143,12 @@ class Conv2DNHWCTest(trt_test.TfTrtIntegrationTestBase): """Return the expected engines to build.""" return ["TRTEngineOp_0"] + def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162448349") + return super().ShouldRunTest(run_params) + class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase): """Testing conversion of strided Conv2D (data_format=NCHW).""" @@ -168,6 +180,12 @@ class Conv2DStridedNCHWTest(trt_test.TfTrtIntegrationTestBase): """Return the expected engines to build.""" return ["TRTEngineOp_0"] + def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162448349") + return super().ShouldRunTest(run_params) + class Conv2DTranposeTest(trt_test.TfTrtIntegrationTestBase): """Testing conversion of conv2d_transpose (AKA Conv2DBackpropInput)""" diff --git a/tensorflow/python/compiler/tensorrt/test/dynamic_input_shapes_test.py b/tensorflow/python/compiler/tensorrt/test/dynamic_input_shapes_test.py index 95dbe727ac3..f02ad08777e 100644 --- a/tensorflow/python/compiler/tensorrt/test/dynamic_input_shapes_test.py +++ b/tensorflow/python/compiler/tensorrt/test/dynamic_input_shapes_test.py @@ -98,6 +98,9 @@ class DynamicInputShapesTest(trt_test.TfTrtIntegrationTestBase): return ["TRTEngineOp_0"] def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162448349") return (run_params.dynamic_engine and not trt_test.IsQuantizationMode( run_params.precision_mode)), "test dynamic engine and non-INT8" diff --git a/tensorflow/python/compiler/tensorrt/test/lru_cache_test.py b/tensorflow/python/compiler/tensorrt/test/lru_cache_test.py index 9ebbfd51bc6..a2caa070011 100644 --- a/tensorflow/python/compiler/tensorrt/test/lru_cache_test.py +++ b/tensorflow/python/compiler/tensorrt/test/lru_cache_test.py @@ -33,14 +33,6 @@ from tensorflow.python.platform import test class LRUCacheTest(trt_test.TfTrtIntegrationTestBase): def GraphFn(self, x): - conv_filter = constant_op.constant( - np.random.randn(3, 3, 2, 1), dtype=dtypes.float32) - x = nn.conv2d( - input=x, - filter=conv_filter, - strides=[1, 1, 1, 1], - padding="SAME", - name="conv") bias = constant_op.constant( np.random.randn(1, 10, 10, 1), dtype=dtypes.float32) x = math_ops.add(x, bias) @@ -51,9 +43,9 @@ class LRUCacheTest(trt_test.TfTrtIntegrationTestBase): dtype = dtypes.float32 input_dims = [[[1, 10, 10, 2]], [[2, 10, 10, 2]], [[4, 10, 10, 2]], [[2, 10, 10, 2]]] - expected_output_dims = [[[1, 10, 10, 1]], [[2, 10, 10, 1]], [[4, 10, 10, - 1]], - [[2, 10, 10, 1]]] + expected_output_dims = [[[1, 10, 10, 2]], [[2, 10, 10, 2]], [[4, 10, 10, + 2]], + [[2, 10, 10, 2]]] return trt_test.TfTrtIntegrationTestParams( graph_fn=self.GraphFn, input_specs=[ diff --git a/tensorflow/python/compiler/tensorrt/test/memory_alignment_test.py b/tensorflow/python/compiler/tensorrt/test/memory_alignment_test.py index 056edc3e4d4..c1f0a007bf8 100644 --- a/tensorflow/python/compiler/tensorrt/test/memory_alignment_test.py +++ b/tensorflow/python/compiler/tensorrt/test/memory_alignment_test.py @@ -67,6 +67,12 @@ class MemoryAlignmentTest(trt_test.TfTrtIntegrationTestBase): """The relative tolerance to compare floating point results.""" return 0.1 + def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162448349") + return super().ShouldRunTest(run_params) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/compiler/tensorrt/test/multi_connection_neighbor_engine_test.py b/tensorflow/python/compiler/tensorrt/test/multi_connection_neighbor_engine_test.py index b57bee6c5d7..687a12486b7 100644 --- a/tensorflow/python/compiler/tensorrt/test/multi_connection_neighbor_engine_test.py +++ b/tensorflow/python/compiler/tensorrt/test/multi_connection_neighbor_engine_test.py @@ -72,6 +72,12 @@ class MultiConnectionNeighborEngineTest(trt_test.TfTrtIntegrationTestBase): """Return the expected engines to build.""" return ["TRTEngineOp_0", "TRTEngineOp_1"] + def ShouldRunTest(self, run_params): + # TODO(b/162447069): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162447069") + return super().ShouldRunTest(run_params) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/compiler/tensorrt/test/neighboring_engine_test.py b/tensorflow/python/compiler/tensorrt/test/neighboring_engine_test.py index f377fe8dceb..39fee5cba5d 100644 --- a/tensorflow/python/compiler/tensorrt/test/neighboring_engine_test.py +++ b/tensorflow/python/compiler/tensorrt/test/neighboring_engine_test.py @@ -61,6 +61,12 @@ class NeighboringEngineTest(trt_test.TfTrtIntegrationTestBase): "TRTEngineOp_1": ["weights", "conv"] } + def ShouldRunTest(self, run_params): + # TODO(b/162447069): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162447069") + return super().ShouldRunTest(run_params) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py index 2716a933336..6b2b4ba77c2 100644 --- a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py +++ b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py @@ -18,13 +18,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function - import tensorflow_datasets as tfds from tensorflow.compiler.tf2tensorrt._pywrap_py_utils import get_linked_tensorrt_version from tensorflow.compiler.tf2tensorrt._pywrap_py_utils import is_tensorrt_enabled from tensorflow.core.protobuf import config_pb2 from tensorflow.python.compiler.tensorrt import trt_convert +from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test from tensorflow.python.data.ops import dataset_ops from tensorflow.python.estimator.estimator import Estimator from tensorflow.python.estimator.model_fn import EstimatorSpec @@ -262,6 +262,11 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase): def testEval(self): if not is_tensorrt_enabled(): return + + # TODO(b/162447069): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return + model_dir = test.test_src_dir_path( 'python/compiler/tensorrt/test/testdata/mnist') @@ -286,6 +291,5 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase): logging.info('accuracy_tf_trt: %f', accuracy_tf_trt) self.assertAllClose(0.9675, accuracy_tf_trt, rtol=1e-3, atol=1e-3) - if __name__ == '__main__': test.main() diff --git a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py index 87fa55a32bd..27133a14203 100644 --- a/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py +++ b/tensorflow/python/compiler/tensorrt/test/tf_trt_integration_test_base.py @@ -31,6 +31,7 @@ import warnings import numpy as np import six +from tensorflow.compiler.tf2tensorrt._pywrap_py_utils import get_linked_tensorrt_version from tensorflow.compiler.tf2tensorrt._pywrap_py_utils import is_tensorrt_enabled from tensorflow.core.framework import graph_pb2 from tensorflow.core.protobuf import config_pb2 @@ -100,6 +101,12 @@ def IsQuantizationWithCalibration(params): return IsQuantizationMode(params.precision_mode) and params.use_calibration +def IsTensorRTVersionGreaterEqual(major, minor=0, patch=0): + ver = get_linked_tensorrt_version() + return ver[0] > major or (ver[0] == major and ver[1] > minor) or ( + ver[0] == major and ver[1] == minor and ver[2] >= patch) + + class GraphState(object): ORIGINAL = 0 CALIBRATE = 1 diff --git a/tensorflow/python/compiler/tensorrt/test/vgg_block_nchw_test.py b/tensorflow/python/compiler/tensorrt/test/vgg_block_nchw_test.py index 8fd9606812d..43034e8b31e 100644 --- a/tensorflow/python/compiler/tensorrt/test/vgg_block_nchw_test.py +++ b/tensorflow/python/compiler/tensorrt/test/vgg_block_nchw_test.py @@ -76,6 +76,12 @@ class VGGBlockNCHWTest(trt_test.TfTrtIntegrationTestBase): super(trt_test.TfTrtIntegrationTestBase, self).setUp() os.environ["TF_TRT_ALLOW_ENGINE_NATIVE_SEGMENT_EXECUTION"] = "True" + def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162448349") + return super().ShouldRunTest(run_params) + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/compiler/tensorrt/test/vgg_block_test.py b/tensorflow/python/compiler/tensorrt/test/vgg_block_test.py index 9d81cd6dcc3..7b1f7e062d7 100644 --- a/tensorflow/python/compiler/tensorrt/test/vgg_block_test.py +++ b/tensorflow/python/compiler/tensorrt/test/vgg_block_test.py @@ -67,6 +67,12 @@ class VGGBlockTest(trt_test.TfTrtIntegrationTestBase): super(trt_test.TfTrtIntegrationTestBase, self).setUp() os.environ["TF_TRT_ALLOW_ENGINE_NATIVE_SEGMENT_EXECUTION"] = "True" + def ShouldRunTest(self, run_params): + # TODO(b/162448349): Enable the test for TRT 7.1.3. + if trt_test.IsTensorRTVersionGreaterEqual(7, 1, 3): + return (False, "Skip test due to b/162448349") + return super().ShouldRunTest(run_params) + if __name__ == "__main__": test.main() From 465b9f4258dfe922c6d385551233b9ab7319af37 Mon Sep 17 00:00:00 2001 From: Bixia Zheng Date: Tue, 4 Aug 2020 09:27:04 -0700 Subject: [PATCH 0363/1017] [TF:TRT] Use IsTensorRTVersionGreaterEqual to check TensorRT version. Rewrite a few places that check TensorRT version to use IsTensorRTVersionGreaterEqual. PiperOrigin-RevId: 324826641 Change-Id: I667d317574f025422cedbed9b14cb8df647a5a70 --- .../python/compiler/tensorrt/test/combined_nms_test.py | 8 +++----- .../compiler/tensorrt/test/quantization_mnist_test.py | 4 +--- .../python/compiler/tensorrt/test/quantization_test.py | 5 ++--- tensorflow/python/compiler/tensorrt/test/trt_mode_test.py | 7 ++----- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/tensorflow/python/compiler/tensorrt/test/combined_nms_test.py b/tensorflow/python/compiler/tensorrt/test/combined_nms_test.py index 71f5139d049..26e911e3b0b 100644 --- a/tensorflow/python/compiler/tensorrt/test/combined_nms_test.py +++ b/tensorflow/python/compiler/tensorrt/test/combined_nms_test.py @@ -18,7 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.compiler.tf2tensorrt._pywrap_py_utils import get_linked_tensorrt_version from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -97,10 +96,9 @@ class CombinedNmsTest(trt_test.TfTrtIntegrationTestBase): # calibration will fail. # TODO(laigd): fix this. # Only run for TRT 5.1 and above. - ver = get_linked_tensorrt_version() - return (ver[0] > 5 or - (ver[0] == 5 and ver[1] >= 1)) and not trt_test.IsQuantizationMode( - run_params.precision_mode), 'test >=TRT5.1 and non-INT8' + return trt_test.IsTensorRTVersionGreaterEqual( + 5, 1) and not trt_test.IsQuantizationMode( + run_params.precision_mode), 'test >=TRT5.1 and non-INT8' if __name__ == '__main__': diff --git a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py index 6b2b4ba77c2..d859407f1f7 100644 --- a/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py +++ b/tensorflow/python/compiler/tensorrt/test/quantization_mnist_test.py @@ -19,8 +19,6 @@ from __future__ import division from __future__ import print_function import tensorflow_datasets as tfds - -from tensorflow.compiler.tf2tensorrt._pywrap_py_utils import get_linked_tensorrt_version from tensorflow.compiler.tf2tensorrt._pywrap_py_utils import is_tensorrt_enabled from tensorflow.core.protobuf import config_pb2 from tensorflow.python.compiler.tensorrt import trt_convert @@ -279,7 +277,7 @@ class QuantizationAwareTrainingMNISTTest(test_util.TensorFlowTestCase): logging.info('accuracy_tf_native: %f', accuracy_tf_native) self.assertAllClose(0.9662, accuracy_tf_native, rtol=3e-3, atol=3e-3) - if get_linked_tensorrt_version()[0] < 5: + if not trt_test.IsTensorRTVersionGreaterEqual(5): return accuracy_tf_trt = self._Run( diff --git a/tensorflow/python/compiler/tensorrt/test/quantization_test.py b/tensorflow/python/compiler/tensorrt/test/quantization_test.py index 7ed3414817c..c41afbb29c5 100644 --- a/tensorflow/python/compiler/tensorrt/test/quantization_test.py +++ b/tensorflow/python/compiler/tensorrt/test/quantization_test.py @@ -20,7 +20,6 @@ from __future__ import print_function import numpy as np -from tensorflow.compiler.tf2tensorrt._pywrap_py_utils import get_linked_tensorrt_version from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes @@ -65,7 +64,7 @@ class QuantizationMissingAllRangesTest(trt_test.TfTrtIntegrationTestBase): def ShouldRunTest(self, run_params): # Only test static engine mode, with or without calibration. - return (get_linked_tensorrt_version()[0] >= 5 and + return (trt_test.IsTensorRTVersionGreaterEqual(5) and trt_test.IsQuantizationMode(run_params.precision_mode) and not run_params.convert_online and not run_params.dynamic_engine ), "test static engine, offline conversion and INT8" @@ -90,7 +89,7 @@ class QuantizationWithRangesTest(trt_test.TfTrtIntegrationTestBase): def ShouldRunTest(self, run_params): # Test static/dynamic engine with/without calibration. - return (get_linked_tensorrt_version()[0] >= 5 and + return (trt_test.IsTensorRTVersionGreaterEqual(5) and trt_test.IsQuantizationMode(run_params.precision_mode) and not run_params.convert_online), "test offline conversion and INT8" diff --git a/tensorflow/python/compiler/tensorrt/test/trt_mode_test.py b/tensorflow/python/compiler/tensorrt/test/trt_mode_test.py index c67de7432cd..7d991678748 100644 --- a/tensorflow/python/compiler/tensorrt/test/trt_mode_test.py +++ b/tensorflow/python/compiler/tensorrt/test/trt_mode_test.py @@ -20,7 +20,6 @@ from __future__ import print_function from unittest import SkipTest # pylint: disable=g-importing-member -from tensorflow.compiler.tf2tensorrt._pywrap_py_utils import get_linked_tensorrt_version from tensorflow.python.compiler.tensorrt.test import tf_trt_integration_test_base as trt_test from tensorflow.python.framework import dtypes from tensorflow.python.ops import array_ops @@ -132,8 +131,7 @@ class ExplicitBatchTest(TrtModeTestBase): def ShouldRunTest(self, run_params): # Only run for TRT 6 and above. - ver = get_linked_tensorrt_version() - return run_params.is_v2 and ver[0] >= 6 and ( + return run_params.is_v2 and trt_test.IsTensorRTVersionGreaterEqual(6) and ( not run_params.use_calibration), "test v2, >=TRT6 and non-calibration" @@ -169,8 +167,7 @@ class DynamicShapesTest(TrtModeTestBase): def ShouldRunTest(self, run_params): # Only run for TRT 6 and above. - ver = get_linked_tensorrt_version() - return run_params.is_v2 and ver[0] >= 6 and ( + return run_params.is_v2 and trt_test.IsTensorRTVersionGreaterEqual(6) and ( not run_params.use_calibration), "test v2 >=TRT6 and non-calibration" From 935a1f07c60867aa627d4cb898faab8f83ce0004 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 09:28:45 -0700 Subject: [PATCH 0364/1017] Add COPY operation to node shader registry in OpenGL PiperOrigin-RevId: 324826897 Change-Id: Ieb7a43609551f277f9a7dc24eb8ab8a33406be33 --- tensorflow/lite/delegates/gpu/gl/kernels/registry.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc b/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc index 0d2438aacc6..da6aad720a2 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc @@ -96,6 +96,7 @@ class Registry : public NodeShader { insert_op(Type::SOFTMAX, NewSoftmaxNodeShader); insert_elementwise_op(Type::ABS); + insert_elementwise_op(Type::COPY); insert_elementwise_op(Type::COS); insert_elementwise_op(Type::DIV); insert_elementwise_op(Type::ELU); From 920c7f93633b87f7f703617a72aa504509d59939 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Tue, 4 Aug 2020 09:37:22 -0700 Subject: [PATCH 0365/1017] Delete duplicate glob expression PiperOrigin-RevId: 324828618 Change-Id: I6ebb0dcf912ff4813a47e344651d6092088bb108 --- third_party/mlir/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/third_party/mlir/BUILD b/third_party/mlir/BUILD index 04238bae943..4f2873af3dd 100644 --- a/third_party/mlir/BUILD +++ b/third_party/mlir/BUILD @@ -432,7 +432,6 @@ cc_library( ]), hdrs = glob([ "include/mlir/Dialect/*.h", - "include/mlir/Dialect/*.h", ]), includes = ["include"], deps = [ From 1cbe64ec056ae48d66df51cdcb8c0a8d61fe6cf4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 09:45:46 -0700 Subject: [PATCH 0366/1017] Move scalar multiply to the smaller side of convolution. PiperOrigin-RevId: 324830166 Change-Id: Ia71f882d838356158118c7f9d69d41d8a0ff6512 --- .../xla/service/algebraic_simplifier.cc | 202 ++++++++++++++++++ .../xla/service/algebraic_simplifier.h | 12 ++ .../xla/service/algebraic_simplifier_test.cc | 53 +++++ 3 files changed, 267 insertions(+) diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index 1f82c062df9..0b588048e4a 100755 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -428,6 +428,10 @@ class AlgebraicSimplifierVisitor : public DfsHloRewriteVisitor { shape, hlo, zero, dims, AddReduce_computation)); } + // Move scalar multiply to the smallest side of convolution to + // reduce multiply computations. + Status ScalarMultiplyReduction(HloInstruction* dot); + // Convenience method for replacing an instruction with a bitcast. If operand // is not null, then the bitcast will use the specified operand instead of the // operand of the instruction. @@ -563,6 +567,200 @@ bool AlgebraicSimplifierVisitor::SameShape(const HloInstruction* lhs, } } +namespace { + +float GetConstantValue(HloInstruction* inst) { + switch (inst->shape().element_type()) { + case BF16: + return static_cast(inst->literal().GetFirstElement()); + case F32: + return inst->literal().GetFirstElement(); + default: + LOG(FATAL) << "Unsupported data type: " << inst->shape().element_type(); + } +} + +bool IsOpCodeMultiplyCommutative(HloOpcode opcode) { + switch (opcode) { + case HloOpcode::kMultiply: + case HloOpcode::kTranspose: + case HloOpcode::kReshape: + case HloOpcode::kSelect: + return true; + default: + return false; + } +} + +std::unique_ptr MakeScalarInstruction(HloInstruction* target, + float multiplier) { + switch (target->shape().element_type()) { + case BF16: + return HloInstruction::CreateConstant(LiteralUtil::ConvertF32ToBF16( + LiteralUtil::CreateR0(multiplier))); + break; + case F32: + return HloInstruction::CreateConstant( + LiteralUtil::CreateR0(multiplier)); + break; + default: + LOG(FATAL) << "Unsupported data type: " << target->shape().element_type(); + } +} + +} // namespace + +Status AlgebraicSimplifierVisitor::ScalarMultiplyReduction( + HloInstruction* dot) { + // We only process bfloat16 and float32 for now. + if (dot->shape().element_type() != BF16 && + dot->shape().element_type() != F32) { + return Status::OK(); + } + + auto lhs = dot->mutable_operand(0); + auto rhs = dot->mutable_operand(1); + + const int64 dot_size = ShapeUtil::ElementsIn(dot->shape()); + const int64 lhs_size = ShapeUtil::ElementsIn(lhs->shape()); + const int64 rhs_size = ShapeUtil::ElementsIn(rhs->shape()); + + HloInstruction* target = nullptr; + // (current node, user, operand_index) + std::vector> operands; + std::vector users; + + // Find which side of dot has the smallest size: + // operand 0, operand 1, or output. + if (dot_size <= std::min(lhs_size, rhs_size)) { + target = dot; + if (dot_size < lhs_size) { + operands.emplace_back(lhs, dot, 0); + } + if (dot_size < rhs_size) { + operands.emplace_back(rhs, dot, 1); + } + } else if (lhs_size <= rhs_size) { + target = lhs; + if (lhs_size < rhs_size) { + operands.emplace_back(rhs, dot, 1); + } + if (lhs_size < dot_size && dot->user_count() == 1) { + users.push_back(dot->users().front()); + } + } else { + target = rhs; + if (rhs_size < lhs_size) { + operands.emplace_back(lhs, dot, 0); + } + if (rhs_size < dot_size && dot->user_count() == 1) { + users.push_back(dot->users().front()); + } + } + + std::vector values; + + // DFS to find scalar multiply ops from the operands. + while (!operands.empty()) { + HloInstruction* inst; + HloInstruction* user; + int64 index; + std::tie (inst, user, index) = operands.back(); + operands.pop_back(); + + // Skip the op types that are not commutative with multiply. + if (!IsOpCodeMultiplyCommutative(inst->opcode())) { + continue; + } + + HloInstruction* operand; + HloInstruction* multiplier; + // Pattern match a scalar multiply. + if (Match(inst, m::MultiplyAnyOrder( + m::Op(&operand), + m::Broadcast(m::ConstantScalar(&multiplier))))) { + CHECK_LT(index, user->operand_count()); + CHECK_EQ(inst, user->operands()[index]); + + // When found a scalar multiply, save its scalar value. + values.push_back(GetConstantValue(multiplier)); + // And remove the scalar multiply op. + TF_RETURN_IF_ERROR(user->ReplaceOperandWith(index, operand)); + inst = operand; + } + + // Push the operands of inst. + int64 i = 0; + for (auto* operand : inst->operands()) { + operands.emplace_back(operand, inst, i++); + } + } + + // DFS to find scalar multiply ops from the users. + while (!users.empty()) { + auto inst = users.back(); + users.pop_back(); + + if (!IsOpCodeMultiplyCommutative(inst->opcode())) { + continue; + } + + HloInstruction* operand; + HloInstruction* multiplier; + if (Match(inst, m::MultiplyAnyOrder( + m::Op(&operand), + m::Broadcast(m::ConstantScalar(&multiplier))))) { + values.push_back(GetConstantValue(multiplier)); + + TF_RETURN_IF_ERROR(inst->ReplaceAllUsesWith(operand)); + inst = operand; + } + + // Process the instructions with only one user. + // Otherwise moving scalar multiply to the operands changes the values of + // other users. + if (inst->user_count() == 1) { + users.push_back(inst->users().front()); + } + } + + if (values.empty()) { + return Status::OK(); + } + + changed_ = true; + + // Combine all constant multipliers. + float multiplier = 1.0; + for (const float v : values) { + multiplier *= v; + } + + // Create a new const scalar multiply instruction. + HloInstruction* new_const_inst; + new_const_inst = + computation_->AddInstruction(MakeScalarInstruction(target, multiplier)); + + // Broadcast the scalar multiplier. + HloInstruction* new_broadcast = computation_->AddInstruction( + HloInstruction::CreateBroadcast(target->shape(), new_const_inst, {})); + // Create a new scalar multiply instruction. + HloInstruction* new_multiply = + computation_->AddInstruction(HloInstruction::CreateBinary( + target->shape(), HloOpcode::kMultiply, target, new_broadcast)); + CHECK_EQ(new_multiply->shape(), target->shape()); + + // Update the dependency with the rest of the instructions. + if (target == lhs) { + return dot->ReplaceOperandWith(0, new_multiply); + } else if (target == rhs) { + return dot->ReplaceOperandWith(1, new_multiply); + } else { + CHECK_EQ(target, dot); + return dot->ReplaceAllUsesWith(new_multiply); + } +} + void AlgebraicSimplifierVisitor::ReplaceWithBitcast(HloInstruction* instruction, HloInstruction* operand) { CHECK_EQ(1, instruction->operand_count()); @@ -5042,6 +5240,10 @@ StatusOr AlgebraicSimplifierVisitor::SimplifyConvToDot( Status AlgebraicSimplifierVisitor::HandleConvolution( HloInstruction* convolution) { + if (options_.enable_scalar_multiply_reduction()) { + TF_RETURN_IF_ERROR(ScalarMultiplyReduction(convolution)); + } + // Zero-sized input or filter. if (ShapeUtil::IsZeroElementArray(convolution->operand(0)->shape()) || ShapeUtil::IsZeroElementArray(convolution->operand(1)->shape())) { diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h index 9f29df3c209..9f2a3404116 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.h +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h @@ -86,6 +86,17 @@ class AlgebraicSimplifierOptions { } bool enable_conv_operand_swap() const { return enable_conv_operand_swap_; } + // Move constant scalar multiply to one operand or output of convolutions with + // the smallest tensor size, to reduce the number of scalar multiply. + void set_enable_scalar_multiply_reduction( + bool enable_scalar_multiply_reduction) { + enable_scalar_multiply_reduction_ = enable_scalar_multiply_reduction; + } + + bool enable_scalar_multiply_reduction() const { + return enable_scalar_multiply_reduction_; + } + // If enable_window_reduce_replacement is true, the kReduceWindow instruction // can be optimized by replacement with simpler operations. void set_enable_window_reduce_to_reduce_replacement( @@ -146,6 +157,7 @@ class AlgebraicSimplifierOptions { bool enable_dot_to_multiply_rewrite_{true}; bool enable_conv_simplification_{true}; bool enable_conv_operand_swap_{true}; + bool enable_scalar_multiply_reduction_{false}; bool enable_window_reduce_to_reduce_replacement_{true}; bool enable_reduce_of_reshape_{true}; bool replace_transpose_with_bitcast_{true}; diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc index 034d8ec4361..90ca44714f7 100644 --- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc @@ -5343,6 +5343,59 @@ ENTRY AddBroadcastZeroWithDynamicSlice { EXPECT_THAT(root->operand(1)->opcode(), HloOpcode::kPad); } +TEST_F(AlgebraicSimplifierTest, ScalarMultiplyReduction) { + const char* hlo_string = R"( +HloModule ConstScalarMultiply +ENTRY ConstScalarMultiply { + param0 = f32[16,512,4096]{2,1,0} parameter(0) + constant.0 = f32[] constant(0.5) + broadcast.0 = f32[16,512,4096] broadcast(constant.0), dimensions={} + multiply.0 = f32[16,512,4096]{2,1,0} multiply(param0, broadcast.0) + param1 = f32[16,512,4096]{2,1,0} parameter(1) + multiply.1 = f32[16,512,4096]{2,1,0} multiply(multiply.0, param1) + param2 = f32[16,512,1024]{2,1,0} parameter(2) + constant.1 = f32[] constant(1.109) + broadcast.1 = f32[16,512,1024] broadcast(constant.1), dimensions={} + multiply.2 = f32[16,512,1024]{2,1,0} multiply(param2, broadcast.1) + ROOT convolution = f32[4096,1024,1]{1,0,2} convolution(multiply.1, multiply.2), window={size=16}, dim_labels=0fb_0io->bf0 +} +)"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + AlgebraicSimplifierOptions options; + options.set_enable_scalar_multiply_reduction(true); + AlgebraicSimplifier simplifier(options); + ASSERT_TRUE(simplifier.Run(module.get()).ValueOrDie()); + auto root = module->entry_computation()->root_instruction(); + EXPECT_EQ(root->opcode(), HloOpcode::kMultiply); + EXPECT_THAT(root, + GmockMatch(m::MultiplyAnyOrder( + m::Op(), m::Broadcast(m::ConstantScalar(0.5f * 1.109f))))); +} + +TEST_F(AlgebraicSimplifierTest, ScalarMultiplyReductionMultiUser) { + const char* hlo_string = R"( +HloModule ConstScalarMultiply +ENTRY ConstScalarMultiply { + param0 = f32[16,512,1024] parameter(0) + param1 = f32[4096,1024,1] parameter(1) + convolution = f32[16,512,4096] convolution(param0, param1), window={size=1}, dim_labels=0bf_oi0->0bf + constant.1 = f32[] constant(0.5) + broadcast.1 = f32[16,512,4096] broadcast(constant.1), dimensions={} + multiply.1 = f32[16,512,4096] multiply(convolution, broadcast.1) + param2 = f32[16,512,4096] parameter(2) + multiply.2 = f32[16,512,4096] multiply(convolution, param2) + ROOT add.1 = f32[16,512,4096] add(multiply.1, multiply.2) +} +)"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + AlgebraicSimplifierOptions options; + options.set_enable_scalar_multiply_reduction(true); + AlgebraicSimplifier simplifier(options); + ASSERT_FALSE(simplifier.Run(module.get()).ValueOrDie()); +} + INSTANTIATE_TEST_SUITE_P(DotOfConcatSimplificationTestInstantiation, DotOfConcatSimplificationTest, ::testing::ValuesIn(kDotOfConcatTestSpecs)); From 0224b563c6d614ef61fdcec1ba1953dfabb1a70b Mon Sep 17 00:00:00 2001 From: Zhenyu Tan Date: Tue, 4 Aug 2020 09:46:34 -0700 Subject: [PATCH 0367/1017] fix skip test feature column part 3: Shared Embedding Column PiperOrigin-RevId: 324830308 Change-Id: I5fe94e971a64fba18a750ce08b86f09d1a1b3cff --- .../feature_column/feature_column_test.py | 992 +++++++++--------- 1 file changed, 499 insertions(+), 493 deletions(-) diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index e351c5da572..c8e24e46c2f 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -5532,251 +5532,251 @@ class EmbeddingColumnTest(test.TestCase, parameterized.TestCase): class SharedEmbeddingColumnTest(test.TestCase, parameterized.TestCase): - @test_util.run_deprecated_v1 def test_defaults(self): - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=3) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=3) - embedding_dimension = 2 - embedding_column_b, embedding_column_a = fc_new.shared_embedding_columns( - [categorical_column_b, categorical_column_a], - dimension=embedding_dimension) - self.assertIs(categorical_column_a, embedding_column_a.categorical_column) - self.assertIs(categorical_column_b, embedding_column_b.categorical_column) - self.assertEqual(embedding_dimension, embedding_column_a.dimension) - self.assertEqual(embedding_dimension, embedding_column_b.dimension) - self.assertEqual('mean', embedding_column_a.combiner) - self.assertEqual('mean', embedding_column_b.combiner) - self.assertIsNone(embedding_column_a.ckpt_to_load_from) - self.assertIsNone(embedding_column_b.ckpt_to_load_from) - self.assertEqual('aaa_bbb_shared_embedding', - embedding_column_a.shared_embedding_collection_name) - self.assertEqual('aaa_bbb_shared_embedding', - embedding_column_b.shared_embedding_collection_name) - self.assertIsNone(embedding_column_a.tensor_name_in_ckpt) - self.assertIsNone(embedding_column_b.tensor_name_in_ckpt) - self.assertIsNone(embedding_column_a.max_norm) - self.assertIsNone(embedding_column_b.max_norm) - self.assertTrue(embedding_column_a.trainable) - self.assertTrue(embedding_column_b.trainable) - self.assertEqual('aaa_shared_embedding', embedding_column_a.name) - self.assertEqual('bbb_shared_embedding', embedding_column_b.name) - self.assertEqual( - 'aaa_bbb_shared_embedding', embedding_column_a._var_scope_name) - self.assertEqual( - 'aaa_bbb_shared_embedding', embedding_column_b._var_scope_name) - self.assertEqual( - (embedding_dimension,), embedding_column_a._variable_shape) - self.assertEqual( - (embedding_dimension,), embedding_column_b._variable_shape) - self.assertEqual({ - 'aaa': parsing_ops.VarLenFeature(dtypes.int64) - }, embedding_column_a._parse_example_spec) - self.assertEqual({ - 'bbb': parsing_ops.VarLenFeature(dtypes.int64) - }, embedding_column_b._parse_example_spec) - - @test_util.run_deprecated_v1 - def test_all_constructor_args(self): - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=3) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=3) - embedding_dimension = 2 - embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( - [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, - combiner='my_combiner', - initializer=lambda: 'my_initializer', - shared_embedding_collection_name='shared_embedding_collection_name', - ckpt_to_load_from='my_ckpt', - tensor_name_in_ckpt='my_ckpt_tensor', - max_norm=42., - trainable=False) - self.assertIs(categorical_column_a, embedding_column_a.categorical_column) - self.assertIs(categorical_column_b, embedding_column_b.categorical_column) - self.assertEqual(embedding_dimension, embedding_column_a.dimension) - self.assertEqual(embedding_dimension, embedding_column_b.dimension) - self.assertEqual('my_combiner', embedding_column_a.combiner) - self.assertEqual('my_combiner', embedding_column_b.combiner) - self.assertEqual('shared_embedding_collection_name', - embedding_column_a.shared_embedding_collection_name) - self.assertEqual('shared_embedding_collection_name', - embedding_column_b.shared_embedding_collection_name) - self.assertEqual('my_ckpt', embedding_column_a.ckpt_to_load_from) - self.assertEqual('my_ckpt', embedding_column_b.ckpt_to_load_from) - self.assertEqual('my_ckpt_tensor', embedding_column_a.tensor_name_in_ckpt) - self.assertEqual('my_ckpt_tensor', embedding_column_b.tensor_name_in_ckpt) - self.assertEqual(42., embedding_column_a.max_norm) - self.assertEqual(42., embedding_column_b.max_norm) - self.assertFalse(embedding_column_a.trainable) - self.assertFalse(embedding_column_b.trainable) - self.assertEqual('aaa_shared_embedding', embedding_column_a.name) - self.assertEqual('bbb_shared_embedding', embedding_column_b.name) - self.assertEqual( - 'shared_embedding_collection_name', embedding_column_a._var_scope_name) - self.assertEqual( - 'shared_embedding_collection_name', embedding_column_b._var_scope_name) - self.assertEqual( - (embedding_dimension,), embedding_column_a._variable_shape) - self.assertEqual( - (embedding_dimension,), embedding_column_b._variable_shape) - self.assertEqual({ - 'aaa': parsing_ops.VarLenFeature(dtypes.int64) - }, embedding_column_a._parse_example_spec) - self.assertEqual({ - 'bbb': parsing_ops.VarLenFeature(dtypes.int64) - }, embedding_column_b._parse_example_spec) - - @test_util.run_deprecated_v1 - def test_deep_copy(self): - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=3) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=3) - embedding_dimension = 2 - original_a, _ = fc_new.shared_embedding_columns( - [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, - combiner='my_combiner', - initializer=lambda: 'my_initializer', - shared_embedding_collection_name='shared_embedding_collection_name', - ckpt_to_load_from='my_ckpt', - tensor_name_in_ckpt='my_ckpt_tensor', - max_norm=42., - trainable=False) - for embedding_column_a in (original_a, copy.deepcopy(original_a)): - self.assertEqual('aaa', embedding_column_a.categorical_column.name) - self.assertEqual(3, embedding_column_a.categorical_column._num_buckets) - self.assertEqual({ - 'aaa': parsing_ops.VarLenFeature(dtypes.int64) - }, embedding_column_a.categorical_column._parse_example_spec) - + with ops.Graph().as_default(): + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=3) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=3) + embedding_dimension = 2 + embedding_column_b, embedding_column_a = fc_new.shared_embedding_columns( + [categorical_column_b, categorical_column_a], + dimension=embedding_dimension) + self.assertIs(categorical_column_a, embedding_column_a.categorical_column) + self.assertIs(categorical_column_b, embedding_column_b.categorical_column) self.assertEqual(embedding_dimension, embedding_column_a.dimension) + self.assertEqual(embedding_dimension, embedding_column_b.dimension) + self.assertEqual('mean', embedding_column_a.combiner) + self.assertEqual('mean', embedding_column_b.combiner) + self.assertIsNone(embedding_column_a.ckpt_to_load_from) + self.assertIsNone(embedding_column_b.ckpt_to_load_from) + self.assertEqual('aaa_bbb_shared_embedding', + embedding_column_a.shared_embedding_collection_name) + self.assertEqual('aaa_bbb_shared_embedding', + embedding_column_b.shared_embedding_collection_name) + self.assertIsNone(embedding_column_a.tensor_name_in_ckpt) + self.assertIsNone(embedding_column_b.tensor_name_in_ckpt) + self.assertIsNone(embedding_column_a.max_norm) + self.assertIsNone(embedding_column_b.max_norm) + self.assertTrue(embedding_column_a.trainable) + self.assertTrue(embedding_column_b.trainable) + self.assertEqual('aaa_shared_embedding', embedding_column_a.name) + self.assertEqual('bbb_shared_embedding', embedding_column_b.name) + self.assertEqual('aaa_bbb_shared_embedding', + embedding_column_a._var_scope_name) + self.assertEqual('aaa_bbb_shared_embedding', + embedding_column_b._var_scope_name) + self.assertEqual((embedding_dimension,), + embedding_column_a._variable_shape) + self.assertEqual((embedding_dimension,), + embedding_column_b._variable_shape) + self.assertEqual({'aaa': parsing_ops.VarLenFeature(dtypes.int64)}, + embedding_column_a._parse_example_spec) + self.assertEqual({'bbb': parsing_ops.VarLenFeature(dtypes.int64)}, + embedding_column_b._parse_example_spec) + + def test_all_constructor_args(self): + with ops.Graph().as_default(): + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=3) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=3) + embedding_dimension = 2 + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( + [categorical_column_a, categorical_column_b], + dimension=embedding_dimension, + combiner='my_combiner', + initializer=lambda: 'my_initializer', + shared_embedding_collection_name='shared_embedding_collection_name', + ckpt_to_load_from='my_ckpt', + tensor_name_in_ckpt='my_ckpt_tensor', + max_norm=42., + trainable=False) + self.assertIs(categorical_column_a, embedding_column_a.categorical_column) + self.assertIs(categorical_column_b, embedding_column_b.categorical_column) + self.assertEqual(embedding_dimension, embedding_column_a.dimension) + self.assertEqual(embedding_dimension, embedding_column_b.dimension) self.assertEqual('my_combiner', embedding_column_a.combiner) + self.assertEqual('my_combiner', embedding_column_b.combiner) self.assertEqual('shared_embedding_collection_name', embedding_column_a.shared_embedding_collection_name) + self.assertEqual('shared_embedding_collection_name', + embedding_column_b.shared_embedding_collection_name) self.assertEqual('my_ckpt', embedding_column_a.ckpt_to_load_from) + self.assertEqual('my_ckpt', embedding_column_b.ckpt_to_load_from) self.assertEqual('my_ckpt_tensor', embedding_column_a.tensor_name_in_ckpt) + self.assertEqual('my_ckpt_tensor', embedding_column_b.tensor_name_in_ckpt) self.assertEqual(42., embedding_column_a.max_norm) + self.assertEqual(42., embedding_column_b.max_norm) self.assertFalse(embedding_column_a.trainable) + self.assertFalse(embedding_column_b.trainable) self.assertEqual('aaa_shared_embedding', embedding_column_a.name) + self.assertEqual('bbb_shared_embedding', embedding_column_b.name) + self.assertEqual('shared_embedding_collection_name', + embedding_column_a._var_scope_name) + self.assertEqual('shared_embedding_collection_name', + embedding_column_b._var_scope_name) self.assertEqual( (embedding_dimension,), embedding_column_a._variable_shape) + self.assertEqual((embedding_dimension,), + embedding_column_b._variable_shape) self.assertEqual({ 'aaa': parsing_ops.VarLenFeature(dtypes.int64) }, embedding_column_a._parse_example_spec) + self.assertEqual({'bbb': parsing_ops.VarLenFeature(dtypes.int64)}, + embedding_column_b._parse_example_spec) - @test_util.run_deprecated_v1 - def test_invalid_initializer(self): - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=3) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=3) - with self.assertRaisesRegex(ValueError, 'initializer must be callable'): - fc_new.shared_embedding_columns( + def test_deep_copy(self): + with ops.Graph().as_default(): + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=3) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=3) + embedding_dimension = 2 + original_a, _ = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], - dimension=2, - initializer='not_fn') + dimension=embedding_dimension, + combiner='my_combiner', + initializer=lambda: 'my_initializer', + shared_embedding_collection_name='shared_embedding_collection_name', + ckpt_to_load_from='my_ckpt', + tensor_name_in_ckpt='my_ckpt_tensor', + max_norm=42., + trainable=False) + for embedding_column_a in (original_a, copy.deepcopy(original_a)): + self.assertEqual('aaa', embedding_column_a.categorical_column.name) + self.assertEqual(3, embedding_column_a.categorical_column._num_buckets) + self.assertEqual( + {'aaa': parsing_ops.VarLenFeature(dtypes.int64)}, + embedding_column_a.categorical_column._parse_example_spec) + + self.assertEqual(embedding_dimension, embedding_column_a.dimension) + self.assertEqual('my_combiner', embedding_column_a.combiner) + self.assertEqual('shared_embedding_collection_name', + embedding_column_a.shared_embedding_collection_name) + self.assertEqual('my_ckpt', embedding_column_a.ckpt_to_load_from) + self.assertEqual('my_ckpt_tensor', + embedding_column_a.tensor_name_in_ckpt) + self.assertEqual(42., embedding_column_a.max_norm) + self.assertFalse(embedding_column_a.trainable) + self.assertEqual('aaa_shared_embedding', embedding_column_a.name) + self.assertEqual((embedding_dimension,), + embedding_column_a._variable_shape) + self.assertEqual({'aaa': parsing_ops.VarLenFeature(dtypes.int64)}, + embedding_column_a._parse_example_spec) + + def test_invalid_initializer(self): + with ops.Graph().as_default(): + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=3) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=3) + with self.assertRaisesRegex(ValueError, 'initializer must be callable'): + fc_new.shared_embedding_columns( + [categorical_column_a, categorical_column_b], + dimension=2, + initializer='not_fn') - @test_util.run_deprecated_v1 def test_incompatible_column_type(self): - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=3) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=3) - categorical_column_c = fc._categorical_column_with_hash_bucket( - key='ccc', hash_bucket_size=3) - with self.assertRaisesRegex( - ValueError, 'all categorical_columns must have the same type.*' - '_IdentityCategoricalColumn.*_HashedCategoricalColumn'): + with ops.Graph().as_default(): + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=3) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=3) + categorical_column_c = fc._categorical_column_with_hash_bucket( + key='ccc', hash_bucket_size=3) + with self.assertRaisesRegex( + ValueError, 'all categorical_columns must have the same type.*' + '_IdentityCategoricalColumn.*_HashedCategoricalColumn'): + fc_new.shared_embedding_columns( + [categorical_column_a, categorical_column_b, categorical_column_c], + dimension=2) + + def test_weighted_categorical_column_ok(self): + with ops.Graph().as_default(): + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=3) + weighted_categorical_column_a = fc._weighted_categorical_column( + categorical_column_a, weight_feature_key='aaa_weights') + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=3) + weighted_categorical_column_b = fc._weighted_categorical_column( + categorical_column_b, weight_feature_key='bbb_weights') fc_new.shared_embedding_columns( - [categorical_column_a, categorical_column_b, categorical_column_c], + [weighted_categorical_column_a, categorical_column_b], dimension=2) + fc_new.shared_embedding_columns( + [categorical_column_a, weighted_categorical_column_b], dimension=2) + fc_new.shared_embedding_columns( + [weighted_categorical_column_a, weighted_categorical_column_b], dimension=2) - @test_util.run_deprecated_v1 - def test_weighted_categorical_column_ok(self): - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=3) - weighted_categorical_column_a = fc._weighted_categorical_column( - categorical_column_a, weight_feature_key='aaa_weights') - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=3) - weighted_categorical_column_b = fc._weighted_categorical_column( - categorical_column_b, weight_feature_key='bbb_weights') - fc_new.shared_embedding_columns( - [weighted_categorical_column_a, categorical_column_b], dimension=2) - fc_new.shared_embedding_columns( - [categorical_column_a, weighted_categorical_column_b], dimension=2) - fc_new.shared_embedding_columns( - [weighted_categorical_column_a, weighted_categorical_column_b], - dimension=2) - - @test_util.run_deprecated_v1 def test_parse_example(self): - a = fc._categorical_column_with_vocabulary_list( - key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) - b = fc._categorical_column_with_vocabulary_list( - key='bbb', vocabulary_list=('omar', 'stringer', 'marlo')) - a_embedded, b_embedded = fc_new.shared_embedding_columns([a, b], - dimension=2) - data = example_pb2.Example(features=feature_pb2.Features( - feature={ - 'aaa': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'omar', b'stringer'])), - 'bbb': - feature_pb2.Feature(bytes_list=feature_pb2.BytesList( - value=[b'stringer', b'marlo'])), - })) - features = parsing_ops.parse_example( - serialized=[data.SerializeToString()], - features=fc.make_parse_example_spec([a_embedded, b_embedded])) - self.assertIn('aaa', features) - self.assertIn('bbb', features) - with self.cached_session(): - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'omar', b'stringer'], dtype=np.object_), - dense_shape=[1, 2]), - features['aaa'].eval()) - _assert_sparse_tensor_value( - self, - sparse_tensor.SparseTensorValue( - indices=[[0, 0], [0, 1]], - values=np.array([b'stringer', b'marlo'], dtype=np.object_), - dense_shape=[1, 2]), - features['bbb'].eval()) + with ops.Graph().as_default(): + a = fc._categorical_column_with_vocabulary_list( + key='aaa', vocabulary_list=('omar', 'stringer', 'marlo')) + b = fc._categorical_column_with_vocabulary_list( + key='bbb', vocabulary_list=('omar', 'stringer', 'marlo')) + a_embedded, b_embedded = fc_new.shared_embedding_columns([a, b], + dimension=2) + data = example_pb2.Example( + features=feature_pb2.Features( + feature={ + 'aaa': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'omar', b'stringer'])), + 'bbb': + feature_pb2.Feature( + bytes_list=feature_pb2.BytesList( + value=[b'stringer', b'marlo'])), + })) + features = parsing_ops.parse_example( + serialized=[data.SerializeToString()], + features=fc.make_parse_example_spec([a_embedded, b_embedded])) + self.assertIn('aaa', features) + self.assertIn('bbb', features) + with self.cached_session(): + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'omar', b'stringer'], dtype=np.object_), + dense_shape=[1, 2]), features['aaa'].eval()) + _assert_sparse_tensor_value( + self, + sparse_tensor.SparseTensorValue( + indices=[[0, 0], [0, 1]], + values=np.array([b'stringer', b'marlo'], dtype=np.object_), + dense_shape=[1, 2]), features['bbb'].eval()) - @test_util.run_deprecated_v1 def test_transform_feature(self): - a = fc._categorical_column_with_identity(key='aaa', num_buckets=3) - b = fc._categorical_column_with_identity(key='bbb', num_buckets=3) - a_embedded, b_embedded = fc_new.shared_embedding_columns([a, b], - dimension=2) - features = { - 'aaa': sparse_tensor.SparseTensor( - indices=((0, 0), (1, 0), (1, 1)), - values=(0, 1, 0), - dense_shape=(2, 2)), - 'bbb': sparse_tensor.SparseTensor( - indices=((0, 0), (1, 0), (1, 1)), - values=(1, 2, 1), - dense_shape=(2, 2)), - } - outputs = _transform_features(features, [a, a_embedded, b, b_embedded]) - output_a = outputs[a] - output_a_embedded = outputs[a_embedded] - output_b = outputs[b] - output_b_embedded = outputs[b_embedded] - with _initialized_session(): - _assert_sparse_tensor_value(self, self.evaluate(output_a), - self.evaluate(output_a_embedded)) - _assert_sparse_tensor_value(self, self.evaluate(output_b), - self.evaluate(output_b_embedded)) + with ops.Graph().as_default(): + a = fc._categorical_column_with_identity(key='aaa', num_buckets=3) + b = fc._categorical_column_with_identity(key='bbb', num_buckets=3) + a_embedded, b_embedded = fc_new.shared_embedding_columns([a, b], + dimension=2) + features = { + 'aaa': + sparse_tensor.SparseTensor( + indices=((0, 0), (1, 0), (1, 1)), + values=(0, 1, 0), + dense_shape=(2, 2)), + 'bbb': + sparse_tensor.SparseTensor( + indices=((0, 0), (1, 0), (1, 1)), + values=(1, 2, 1), + dense_shape=(2, 2)), + } + outputs = _transform_features(features, [a, a_embedded, b, b_embedded]) + output_a = outputs[a] + output_a_embedded = outputs[a_embedded] + output_b = outputs[b] + output_b_embedded = outputs[b_embedded] + with _initialized_session(): + _assert_sparse_tensor_value(self, self.evaluate(output_a), + self.evaluate(output_a_embedded)) + _assert_sparse_tensor_value(self, self.evaluate(output_b), + self.evaluate(output_b_embedded)) @parameterized.named_parameters( { @@ -5796,162 +5796,164 @@ class SharedEmbeddingColumnTest(test.TestCase, parameterized.TestCase): 'use_safe_embedding_lookup': False, 'partition_variables': True, }) - @test_util.run_deprecated_v1 + def test_get_dense_tensor(self, use_safe_embedding_lookup, partition_variables): - # Inputs. - vocabulary_size = 4 - # -1 values are ignored. - input_a = np.array([ - [2, -1, -1], # example 0, ids [2] - [0, 1, -1] - ]) # example 1, ids [0, 1] - input_b = np.array([ - [0, -1, -1], # example 0, ids [0] - [-1, -1, -1] - ]) # example 1, ids [] - input_features = {'aaa': input_a, 'bbb': input_b} + with ops.Graph().as_default(): + # Inputs. + vocabulary_size = 4 + # -1 values are ignored. + input_a = np.array([ + [2, -1, -1], # example 0, ids [2] + [0, 1, -1] + ]) # example 1, ids [0, 1] + input_b = np.array([ + [0, -1, -1], # example 0, ids [0] + [-1, -1, -1] + ]) # example 1, ids [] + input_features = {'aaa': input_a, 'bbb': input_b} - # Embedding variable. - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.), # id 2 - (9., 13.) # id 3 - ) + # Embedding variable. + embedding_dimension = 2 + embedding_values = ( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.), # id 2 + (9., 13.) # id 3 + ) - def _initializer(shape, dtype, partition_info=None): - if partition_variables: - self.assertEqual([vocabulary_size, embedding_dimension], - partition_info.full_shape) - self.assertAllEqual((2, embedding_dimension), shape) - else: - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertIsNone(partition_info) + def _initializer(shape, dtype, partition_info=None): + if partition_variables: + self.assertEqual([vocabulary_size, embedding_dimension], + partition_info.full_shape) + self.assertAllEqual((2, embedding_dimension), shape) + else: + self.assertAllEqual((vocabulary_size, embedding_dimension), shape) + self.assertIsNone(partition_info) - self.assertEqual(dtypes.float32, dtype) - return embedding_values + self.assertEqual(dtypes.float32, dtype) + return embedding_values - # Expected lookup result, using combiner='mean'. - expected_lookups_a = ( - # example 0: - (7., 11.), # ids [2], embedding = [7, 11] - # example 1: - (2., 3.5), # ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] - ) - if use_safe_embedding_lookup: - expected_lookups_b = ( + # Expected lookup result, using combiner='mean'. + expected_lookups_a = ( # example 0: - (1., 2.), # ids [0], embedding = [1, 2] + (7., 11.), # ids [2], embedding = [7, 11] # example 1: - (0., 0.), # ids [], embedding = [0, 0] + (2., 3.5), # ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] ) - else: - expected_lookups_b = ( - # example 0: - (1., 2.), # ids [0], embedding = [1, 2] + if use_safe_embedding_lookup: + expected_lookups_b = ( + # example 0: + (1., 2.), # ids [0], embedding = [1, 2] + # example 1: + (0., 0.), # ids [], embedding = [0, 0] + ) + else: + expected_lookups_b = ( + # example 0: + (1., 2.), # ids [0], embedding = [1, 2] + ) + + # Build columns. + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=vocabulary_size) + + partitioner = None + if partition_variables: + partitioner = partitioned_variables.fixed_size_partitioner(2, axis=0) + + with variable_scope.variable_scope('vars', partitioner=partitioner): + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( + [categorical_column_a, categorical_column_b], + dimension=embedding_dimension, + initializer=_initializer, + use_safe_embedding_lookup=use_safe_embedding_lookup) + # Provide sparse input and get dense result. + embedding_lookup_a = embedding_column_a._get_dense_tensor( + _LazyBuilder(input_features)) + embedding_lookup_b = embedding_column_b._get_dense_tensor( + _LazyBuilder(input_features)) + # Assert expected embedding variable and lookups. + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + if partition_variables: + self.assertCountEqual(('vars/embedding_weights/part_0:0', + 'vars/embedding_weights/part_1:0'), + tuple([v.name for v in global_vars])) + else: + self.assertCountEqual(('vars/embedding_weights:0',), + tuple([v.name for v in global_vars])) + embedding_var = global_vars[0] + + self.evaluate(variables_lib.global_variables_initializer()) + self.evaluate(lookup_ops.tables_initializer()) + + self.assertAllEqual(embedding_values, self.evaluate(embedding_var)) + self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a)) + self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b)) + + if use_safe_embedding_lookup: + self.assertIn( + 'SparseFillEmptyRows', + [x.type for x in ops.get_default_graph().get_operations()]) + else: + self.assertNotIn( + 'SparseFillEmptyRows', + [x.type for x in ops.get_default_graph().get_operations()]) + + def test_get_dense_tensor_weight_collections(self): + with ops.Graph().as_default(): + # Inputs. + vocabulary_size = 3 + # -1 values are ignored. + input_a = np.array([ + [2, -1, -1], # example 0, ids [2] + [0, 1, -1] + ]) # example 1, ids [0, 1] + input_b = np.array([ + [0, -1, -1], # example 0, ids [0] + [-1, -1, -1] + ]) # example 1, ids [] + input_features = {'aaa': input_a, 'bbb': input_b} + + # Embedding variable. + embedding_dimension = 2 + embedding_values = ( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.) # id 2 ) - # Build columns. - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) + def _initializer(shape, dtype, partition_info): + self.assertAllEqual((vocabulary_size, embedding_dimension), shape) + self.assertEqual(dtypes.float32, dtype) + self.assertIsNone(partition_info) + return embedding_values - partitioner = None - if partition_variables: - partitioner = partitioned_variables.fixed_size_partitioner(2, axis=0) - - with variable_scope.variable_scope('vars', partitioner=partitioner): + # Build columns. + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=vocabulary_size) embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( [categorical_column_a, categorical_column_b], dimension=embedding_dimension, - initializer=_initializer, - use_safe_embedding_lookup=use_safe_embedding_lookup) - # Provide sparse input and get dense result. - embedding_lookup_a = embedding_column_a._get_dense_tensor( - _LazyBuilder(input_features)) - embedding_lookup_b = embedding_column_b._get_dense_tensor( - _LazyBuilder(input_features)) - # Assert expected embedding variable and lookups. - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - if partition_variables: - self.assertCountEqual(('vars/embedding_weights/part_0:0', - 'vars/embedding_weights/part_1:0'), - tuple([v.name for v in global_vars])) - else: - self.assertCountEqual(('vars/embedding_weights:0',), - tuple([v.name for v in global_vars])) - embedding_var = global_vars[0] + initializer=_initializer) - self.evaluate(variables_lib.global_variables_initializer()) - self.evaluate(lookup_ops.tables_initializer()) + fc.input_layer( + input_features, [embedding_column_a, embedding_column_b], + weight_collections=('my_vars',)) - self.assertAllEqual(embedding_values, self.evaluate(embedding_var)) - self.assertAllEqual(expected_lookups_a, self.evaluate(embedding_lookup_a)) - self.assertAllEqual(expected_lookups_b, self.evaluate(embedding_lookup_b)) - - if use_safe_embedding_lookup: - self.assertIn('SparseFillEmptyRows', - [x.type for x in ops.get_default_graph().get_operations()]) - else: - self.assertNotIn( - 'SparseFillEmptyRows', - [x.type for x in ops.get_default_graph().get_operations()]) - - @test_util.run_deprecated_v1 - def test_get_dense_tensor_weight_collections(self): - # Inputs. - vocabulary_size = 3 - # -1 values are ignored. - input_a = np.array([ - [2, -1, -1], # example 0, ids [2] - [0, 1, -1] - ]) # example 1, ids [0, 1] - input_b = np.array([ - [0, -1, -1], # example 0, ids [0] - [-1, -1, -1] - ]) # example 1, ids [] - input_features = {'aaa': input_a, 'bbb': input_b} - - # Embedding variable. - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - ) - - def _initializer(shape, dtype, partition_info): - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return embedding_values - - # Build columns. - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( - [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, - initializer=_initializer) - - fc.input_layer( - input_features, [embedding_column_a, embedding_column_b], - weight_collections=('my_vars',)) - - # Assert expected embedding variable and lookups. - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertCountEqual( - ('input_layer/aaa_bbb_shared_embedding/embedding_weights:0',), - tuple(v.name for v in global_vars)) - my_vars = ops.get_collection('my_vars') - self.assertCountEqual( - ('input_layer/aaa_bbb_shared_embedding/embedding_weights:0',), - tuple(v.name for v in my_vars)) + # Assert expected embedding variable and lookups. + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) + self.assertCountEqual( + ('input_layer/aaa_bbb_shared_embedding/embedding_weights:0',), + tuple(v.name for v in global_vars)) + my_vars = ops.get_collection('my_vars') + self.assertCountEqual( + ('input_layer/aaa_bbb_shared_embedding/embedding_weights:0',), + tuple(v.name for v in my_vars)) @test_util.run_deprecated_v1 def test_get_dense_tensor_placeholder_inputs(self): @@ -6010,40 +6012,42 @@ class SharedEmbeddingColumnTest(test.TestCase, parameterized.TestCase): with _initialized_session() as sess: sess.run([embedding_lookup_a, embedding_lookup_b], feed_dict=feed_dict) - @test_util.run_deprecated_v1 def test_linear_model(self): - # Inputs. - batch_size = 2 - vocabulary_size = 3 - # -1 values are ignored. - input_a = np.array( - [[2, -1, -1], # example 0, ids [2] - [0, 1, -1]]) # example 1, ids [0, 1] - input_b = np.array( - [[0, -1, -1], # example 0, ids [0] - [-1, -1, -1]]) # example 1, ids [] - - # Embedding variable. - embedding_dimension = 2 - embedding_shape = (vocabulary_size, embedding_dimension) - zeros_embedding_values = np.zeros(embedding_shape) - def _initializer(shape, dtype, partition_info): - self.assertAllEqual(embedding_shape, shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return zeros_embedding_values - - # Build columns. - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( - [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, - initializer=_initializer) - with ops.Graph().as_default(): + # Inputs. + batch_size = 2 + vocabulary_size = 3 + # -1 values are ignored. + input_a = np.array([ + [2, -1, -1], # example 0, ids [2] + [0, 1, -1] + ]) # example 1, ids [0, 1] + input_b = np.array([ + [0, -1, -1], # example 0, ids [0] + [-1, -1, -1] + ]) # example 1, ids [] + + # Embedding variable. + embedding_dimension = 2 + embedding_shape = (vocabulary_size, embedding_dimension) + zeros_embedding_values = np.zeros(embedding_shape) + + def _initializer(shape, dtype, partition_info): + self.assertAllEqual(embedding_shape, shape) + self.assertEqual(dtypes.float32, dtype) + self.assertIsNone(partition_info) + return zeros_embedding_values + + # Build columns. + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=vocabulary_size) + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( + [categorical_column_a, categorical_column_b], + dimension=embedding_dimension, + initializer=_initializer) + predictions = fc.linear_model({ categorical_column_a.name: input_a, categorical_column_b.name: input_b, @@ -6101,43 +6105,42 @@ class SharedEmbeddingColumnTest(test.TestCase, parameterized.TestCase): # = [3*1 + 5*2, 3*0 +5*0] = [13, 0] self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions)) - @test_util.run_deprecated_v1 def test_keras_linear_model(self): - # Inputs. - batch_size = 2 - vocabulary_size = 3 - # -1 values are ignored. - input_a = np.array([ - [2, -1, -1], # example 0, ids [2] - [0, 1, -1] - ]) # example 1, ids [0, 1] - input_b = np.array([ - [0, -1, -1], # example 0, ids [0] - [-1, -1, -1] - ]) # example 1, ids [] - - # Embedding variable. - embedding_dimension = 2 - embedding_shape = (vocabulary_size, embedding_dimension) - zeros_embedding_values = np.zeros(embedding_shape) - - def _initializer(shape, dtype, partition_info): - self.assertAllEqual(embedding_shape, shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return zeros_embedding_values - - # Build columns. - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( - [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, - initializer=_initializer) - with ops.Graph().as_default(): + # Inputs. + batch_size = 2 + vocabulary_size = 3 + # -1 values are ignored. + input_a = np.array([ + [2, -1, -1], # example 0, ids [2] + [0, 1, -1] + ]) # example 1, ids [0, 1] + input_b = np.array([ + [0, -1, -1], # example 0, ids [0] + [-1, -1, -1] + ]) # example 1, ids [] + + # Embedding variable. + embedding_dimension = 2 + embedding_shape = (vocabulary_size, embedding_dimension) + zeros_embedding_values = np.zeros(embedding_shape) + + def _initializer(shape, dtype, partition_info): + self.assertAllEqual(embedding_shape, shape) + self.assertEqual(dtypes.float32, dtype) + self.assertIsNone(partition_info) + return zeros_embedding_values + + # Build columns. + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=vocabulary_size) + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( + [categorical_column_a, categorical_column_b], + dimension=embedding_dimension, + initializer=_initializer) + predictions = get_keras_linear_model_predictions({ categorical_column_a.name: input_a, categorical_column_b.name: input_b, @@ -6196,84 +6199,87 @@ class SharedEmbeddingColumnTest(test.TestCase, parameterized.TestCase): self.assertAllClose([[94. + 13.], [29.]], self.evaluate(predictions)) def _test_input_layer(self, trainable=True): - # Inputs. - vocabulary_size = 3 - sparse_input_a = sparse_tensor.SparseTensorValue( - # example 0, ids [2] - # example 1, ids [0, 1] - indices=((0, 0), (1, 0), (1, 4)), - values=(2, 0, 1), - dense_shape=(2, 5)) - sparse_input_b = sparse_tensor.SparseTensorValue( - # example 0, ids [0] - # example 1, ids [] - indices=((0, 0),), - values=(0,), - dense_shape=(2, 5)) + with ops.Graph().as_default(): + # Inputs. + vocabulary_size = 3 + sparse_input_a = sparse_tensor.SparseTensorValue( + # example 0, ids [2] + # example 1, ids [0, 1] + indices=((0, 0), (1, 0), (1, 4)), + values=(2, 0, 1), + dense_shape=(2, 5)) + sparse_input_b = sparse_tensor.SparseTensorValue( + # example 0, ids [0] + # example 1, ids [] + indices=((0, 0),), + values=(0,), + dense_shape=(2, 5)) - # Embedding variable. - embedding_dimension = 2 - embedding_values = ( - (1., 2.), # id 0 - (3., 5.), # id 1 - (7., 11.) # id 2 - ) - def _initializer(shape, dtype, partition_info): - self.assertAllEqual((vocabulary_size, embedding_dimension), shape) - self.assertEqual(dtypes.float32, dtype) - self.assertIsNone(partition_info) - return embedding_values + # Embedding variable. + embedding_dimension = 2 + embedding_values = ( + (1., 2.), # id 0 + (3., 5.), # id 1 + (7., 11.) # id 2 + ) - # Expected lookup result, using combiner='mean'. - expected_lookups = ( - # example 0: - # A ids [2], embedding = [7, 11] - # B ids [0], embedding = [1, 2] - (7., 11., 1., 2.), - # example 1: - # A ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] - # B ids [], embedding = [0, 0] - (2., 3.5, 0., 0.), - ) + def _initializer(shape, dtype, partition_info): + self.assertAllEqual((vocabulary_size, embedding_dimension), shape) + self.assertEqual(dtypes.float32, dtype) + self.assertIsNone(partition_info) + return embedding_values - # Build columns. - categorical_column_a = fc._categorical_column_with_identity( - key='aaa', num_buckets=vocabulary_size) - categorical_column_b = fc._categorical_column_with_identity( - key='bbb', num_buckets=vocabulary_size) - embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( - [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, - initializer=_initializer, - trainable=trainable) + # Expected lookup result, using combiner='mean'. + expected_lookups = ( + # example 0: + # A ids [2], embedding = [7, 11] + # B ids [0], embedding = [1, 2] + (7., 11., 1., 2.), + # example 1: + # A ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5] + # B ids [], embedding = [0, 0] + (2., 3.5, 0., 0.), + ) - # Provide sparse input and get dense result. - input_layer = fc.input_layer( - features={'aaa': sparse_input_a, 'bbb': sparse_input_b}, - feature_columns=(embedding_column_b, embedding_column_a)) + # Build columns. + categorical_column_a = fc._categorical_column_with_identity( + key='aaa', num_buckets=vocabulary_size) + categorical_column_b = fc._categorical_column_with_identity( + key='bbb', num_buckets=vocabulary_size) + embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( + [categorical_column_a, categorical_column_b], + dimension=embedding_dimension, + initializer=_initializer, + trainable=trainable) - # Assert expected embedding variable and lookups. - global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) - self.assertCountEqual( - ['input_layer/aaa_bbb_shared_embedding/embedding_weights:0'], - tuple([v.name for v in global_vars])) - trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) - if trainable: + # Provide sparse input and get dense result. + input_layer = fc.input_layer( + features={ + 'aaa': sparse_input_a, + 'bbb': sparse_input_b + }, + feature_columns=(embedding_column_b, embedding_column_a)) + + # Assert expected embedding variable and lookups. + global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES) self.assertCountEqual( ['input_layer/aaa_bbb_shared_embedding/embedding_weights:0'], - tuple([v.name for v in trainable_vars])) - else: - self.assertCountEqual([], tuple([v.name for v in trainable_vars])) - shared_embedding_vars = global_vars - with _initialized_session(): - self.assertAllEqual(embedding_values, shared_embedding_vars[0]) - self.assertAllEqual(expected_lookups, self.evaluate(input_layer)) + tuple([v.name for v in global_vars])) + trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES) + if trainable: + self.assertCountEqual( + ['input_layer/aaa_bbb_shared_embedding/embedding_weights:0'], + tuple([v.name for v in trainable_vars])) + else: + self.assertCountEqual([], tuple([v.name for v in trainable_vars])) + shared_embedding_vars = global_vars + with _initialized_session(): + self.assertAllEqual(embedding_values, shared_embedding_vars[0]) + self.assertAllEqual(expected_lookups, self.evaluate(input_layer)) - @test_util.run_deprecated_v1 def test_input_layer(self): self._test_input_layer() - @test_util.run_deprecated_v1 def test_input_layer_no_trainable(self): self._test_input_layer(trainable=False) From 9ce5ad0f0a8ddc901b74b590473ff52480132df6 Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Tue, 4 Aug 2020 09:46:49 -0700 Subject: [PATCH 0368/1017] Sampler resolve moved to TransformToCLCode. Removed DeviceInfo from many generation functions. PiperOrigin-RevId: 324830359 Change-Id: Ia640efdb558bfaa0e1bb854f1a048cea385690ec --- tensorflow/lite/delegates/gpu/cl/arguments.cc | 28 +++++++++++++++++ .../gpu/cl/kernels/conv_constants.cc | 10 +++---- .../delegates/gpu/cl/kernels/conv_constants.h | 8 +++-- .../delegates/gpu/cl/kernels/conv_powervr.cc | 2 +- .../delegates/gpu/cl/kernels/conv_texture.cc | 11 ++++--- .../delegates/gpu/cl/kernels/conv_texture.h | 3 +- .../delegates/gpu/cl/kernels/converter.cc | 2 -- .../gpu/cl/kernels/convolution_transposed.cc | 24 +++++++-------- .../gpu/cl/kernels/convolution_transposed.h | 4 +-- .../cl/kernels/convolution_transposed_3d.cc | 18 ++++++----- .../cl/kernels/convolution_transposed_3d.h | 3 +- .../convolution_transposed_3x3_thin.cc | 13 ++++---- .../kernels/convolution_transposed_3x3_thin.h | 6 ++-- .../gpu/cl/kernels/depthwise_conv.cc | 26 +++++++--------- .../delegates/gpu/cl/kernels/depthwise_conv.h | 7 ++--- .../gpu/cl/kernels/depthwise_conv_3x3.cc | 10 +++---- .../gpu/cl/kernels/depthwise_conv_3x3.h | 1 - .../delegates/gpu/cl/kernels/max_unpooling.cc | 26 +++++++--------- .../delegates/gpu/cl/kernels/max_unpooling.h | 15 ++++------ .../gpu/cl/kernels/max_unpooling_test.cc | 3 +- .../lite/delegates/gpu/cl/kernels/pooling.cc | 30 ++++++++----------- .../lite/delegates/gpu/cl/kernels/pooling.h | 17 ++++------- .../delegates/gpu/cl/kernels/pooling_test.cc | 12 +++----- .../special/depthwise_conv_plus_1x1_conv.cc | 14 ++++----- .../special/depthwise_conv_plus_1x1_conv.h | 5 ++-- .../lite/delegates/gpu/cl/kernels/util.cc | 11 ------- .../lite/delegates/gpu/cl/kernels/util.h | 14 --------- .../gpu/cl/selectors/operation_selector.cc | 5 ++-- .../gpu/cl/selectors/simple_selectors.cc | 6 ++-- .../gpu/cl/selectors/simple_selectors.h | 2 -- 30 files changed, 147 insertions(+), 189 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/arguments.cc b/tensorflow/lite/delegates/gpu/cl/arguments.cc index ed72bcc7c97..8db58e5e81b 100644 --- a/tensorflow/lite/delegates/gpu/cl/arguments.cc +++ b/tensorflow/lite/delegates/gpu/cl/arguments.cc @@ -145,6 +145,33 @@ std::string GetImageModifier(AccessType access) { } } +std::string GetDefaultSamplers(const DeviceInfo& device_info) { + std::string result; + result += + "__constant sampler_t smp_none = CLK_NORMALIZED_COORDS_FALSE | " + "CLK_ADDRESS_NONE | CLK_FILTER_NEAREST;\n"; + if (device_info.IsAdreno3xx()) { + // Unfortunately, CLK_ADDRESS_CLAMP is very slow on Adreno3xx and + // we can observe huge register overhead when compared to other modes. + + // While using CLK_ADDRESS_NONE with out-of-range image coordinates is + // undefined in the OpenCL specification, we have observed that + // CLK_ADDRESS_NONE works like CLK_ADDRESS_CLAMP for out-of-range image + // coordinates for RGBA F16/F32 textures on Adreno3xx devices. Using + // CLK_ADDRESS_NONE is significantly faster than CLK_ADDRESS_CLAMP on Adreno + // 3xx. + result += + "__constant sampler_t smp_zero = CLK_NORMALIZED_COORDS_FALSE | " + "CLK_ADDRESS_NONE | CLK_FILTER_NEAREST;\n"; + } else { + result += + "__constant sampler_t smp_zero = CLK_NORMALIZED_COORDS_FALSE | " + "CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;\n"; + } + + return result; +} + } // namespace // Static @@ -483,6 +510,7 @@ absl::Status Arguments::TransformToCLCode( RETURN_IF_ERROR(ResolveSelectorsPass(linkables, code)); ResolveArgsPass(device_info, code); *code = absl::Substitute(*code, GetListOfArgs()); + *code = GetDefaultSamplers(device_info) + *code; return absl::OkStatus(); } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc index ed1ec8be7b1..d5a2a56c19c 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc @@ -59,9 +59,9 @@ ConvConstants::ConvConstants(const OperationDef& definition, dst_channels_(attr.weights.shape.o) { const bool stride_correction = definition_.IsBatchSupported() && stride_.x != 1; - code_ = GenerateConvolutionConstantCode(definition_, kernel_size_, - src_channels_, dst_channels_, - stride_correction, device_info); + code_ = + GenerateConvolutionConstantCode(definition_, kernel_size_, src_channels_, + dst_channels_, stride_correction); if (definition_.precision == CalculationsPrecision::F16 && device_info.IsAdreno3xx()) { compiler_options_.push_back(CompilerOptions::ADRENO_FULL_SIMD_LINE); @@ -97,9 +97,9 @@ ConvConstants& ConvConstants::operator=(ConvConstants&& kernel) { std::string ConvConstants::GenerateConvolutionConstantCode( const OperationDef& op_def, const int2& kernel_size, int src_channels, - int dst_channels, bool stride_correction, const DeviceInfo& device_info) { + int dst_channels, bool stride_correction) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); if (op_def.IsBatchSupported()) { src_desc.SetStateVar("BatchedWidth", "true"); } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.h b/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.h index 877f32bdf4c..6504b828158 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.h @@ -60,9 +60,11 @@ class ConvConstants : public GPUOperation { void RearrangeWeightsData(const tflite::gpu::Tensor& weights, absl::Span dst); - std::string GenerateConvolutionConstantCode( - const OperationDef& op_def, const int2& kernel_size, int src_channels, - int dst_channels, bool stride_correction, const DeviceInfo& device_info); + std::string GenerateConvolutionConstantCode(const OperationDef& op_def, + const int2& kernel_size, + int src_channels, + int dst_channels, + bool stride_correction); int2 kernel_size_; int2 stride_; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc index 40060007b4e..c4e26725f74 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc @@ -269,7 +269,7 @@ std::string ConvPowerVR::GenerateConv(const DeviceInfo& device_info, bool stride_correction, const ConvParams& conv_params) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); if (op_def.IsBatchSupported()) { src_desc.SetStateVar("BatchedWidth", "true"); } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc index 6f7ebf2b64b..88035556c86 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc @@ -95,10 +95,9 @@ std::string ConvTexture::GenerateConvCode(const OperationDef& op_def, const int3& block_size, bool is1x1, bool adreno4xx_optimization, bool stride_correction, - bool different_weights_for_height, - const DeviceInfo& device_info) { + bool different_weights_for_height) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); if (op_def.IsBatchSupported()) { src_desc.SetStateVar("BatchedWidth", "true"); } @@ -390,9 +389,9 @@ void ConvTexture::GenerateCode(const DeviceInfo& device_info) { definition_.precision == CalculationsPrecision::F16; const bool stride_correction = definition_.IsBatchSupported() && stride_.x != 1; - code_ = GenerateConvCode(definition_, block_size_, is1x1, - adreno4xx_optimization, stride_correction, - different_weights_for_height_, device_info); + code_ = + GenerateConvCode(definition_, block_size_, is1x1, adreno4xx_optimization, + stride_correction, different_weights_for_height_); if (UseFP16SIMD(device_info, definition_.precision, is1x1)) { compiler_options_.push_back(CompilerOptions::ADRENO_FULL_SIMD_LINE); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.h b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.h index 9d50f0291da..10efc23a044 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.h @@ -94,8 +94,7 @@ class ConvTexture : public GPUOperation { const int3& block_size, bool is1x1, bool adreno4xx_optimization, bool stride_correction, - bool different_weights_for_height, - const DeviceInfo& device_info); + bool different_weights_for_height); int2 kernel_size_; int2 stride_; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/converter.cc b/tensorflow/lite/delegates/gpu/cl/kernels/converter.cc index 69873aa9922..bd5aaed8bc3 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/converter.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/converter.cc @@ -136,8 +136,6 @@ class FromTensorConverter : public OpenClConverterImpl { R"( #pragma OPENCL EXTENSION cl_khr_fp16 : enable -const sampler_t smp_none = CLK_NORMALIZED_COORDS_FALSE | CLK_ADDRESS_NONE | CLK_FILTER_NEAREST; - __kernel void from_tensor()" + params_kernel.first + R"(, $0) { int linear_id = get_global_id(0); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc index ecd2fcbc6e1..a139b3affc9 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc @@ -31,17 +31,16 @@ namespace cl { ConvolutionTransposed::ConvolutionTransposed( const OperationDef& definition, const ConvolutionTransposedAttributes& attr, - const CLDevice& device) + const DeviceInfo& device_info) : GPUOperation(definition), - weights_are_buffer_(device.IsMali()), + weights_are_buffer_(device_info.IsMali()), kernel_size_(attr.weights.shape.w, attr.weights.shape.h), stride_(attr.stride.w, attr.stride.h), padding_(attr.padding.prepended.w, attr.padding.prepended.h), block_size_(2, 2, 2) { const bool is_f16 = definition.precision == CalculationsPrecision::F16; - if (device.IsMali()) { - MaliInfo mali_info = device.GetInfo().mali_info; - if (mali_info.IsMidgard()) { + if (device_info.IsMali()) { + if (device_info.mali_info.IsMidgard()) { block_size_ = is_f16 ? int3(2, 1, 2) : int3(2, 1, 1); } else { block_size_ = is_f16 ? int3(2, 2, 2) : int3(2, 2, 1); @@ -49,13 +48,13 @@ ConvolutionTransposed::ConvolutionTransposed( } const int dst_depth = DivideRoundUp(attr.weights.shape.o, 4); if (dst_depth == 1 || dst_depth == 3) { - if (!device.IsMali()) { + if (!device_info.IsMali()) { block_size_.y *= block_size_.z; } block_size_.z = 1; } - code_ = GenerateConvolutionTransposedCode(definition_, device, + code_ = GenerateConvolutionTransposedCode(definition_, device_info, weights_are_buffer_, block_size_); } @@ -81,10 +80,10 @@ ConvolutionTransposed& ConvolutionTransposed::operator=( } std::string ConvolutionTransposed::GenerateConvolutionTransposedCode( - const OperationDef& op_def, const CLDevice& device, bool weights_are_buffer, - const int3& block_size) { + const OperationDef& op_def, const DeviceInfo& device_info, + bool weights_are_buffer, const int3& block_size) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); AddSrcTensor("src_tensor", src_desc); AddDstTensor("dst_tensor", op_def.dst_tensors[0]); @@ -256,7 +255,7 @@ std::string ConvolutionTransposed::GenerateConvolutionTransposedCode( c += " int x_c = kernel_index * args.src_tensor.Slices();\n"; } c += " for (int s = 0; s < args.src_tensor.Slices(); ++s) {\n"; - const bool conditional_read = device.IsMali(); + const bool conditional_read = device_info.IsMali(); for (int y = 0; y < block_size.y; ++y) { const std::string yindex = std::to_string(y); for (int x = 0; x < block_size.x; ++x) { @@ -361,7 +360,8 @@ absl::Status CreateConvolutionTransposed( const CreationContext& creation_context, const OperationDef& definition, const ConvolutionTransposedAttributes& attr, ConvolutionTransposed* result) { - *result = ConvolutionTransposed(definition, attr, *creation_context.device); + *result = ConvolutionTransposed(definition, attr, + creation_context.device->GetInfo()); RETURN_IF_ERROR( result->UploadWeights(attr.weights, creation_context.context)); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.h b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.h index 2263e7d2e4f..44e1c942925 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.h @@ -55,7 +55,7 @@ class ConvolutionTransposed : public GPUOperation { ConvolutionTransposed* result); explicit ConvolutionTransposed(const OperationDef& definition, const ConvolutionTransposedAttributes& attr, - const CLDevice& device); + const DeviceInfo& device_info); template absl::Status UploadWeights(const tflite::gpu::Tensor& weights, CLContext* context); @@ -65,7 +65,7 @@ class ConvolutionTransposed : public GPUOperation { absl::Span dst); std::string GenerateConvolutionTransposedCode(const OperationDef& op_def, - const CLDevice& device, + const DeviceInfo& device_info, bool weights_are_buffer, const int3& block_size); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.cc b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.cc index 5385c09eb0f..eeb3ae15e51 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.cc @@ -30,18 +30,19 @@ namespace cl { ConvolutionTransposed3D::ConvolutionTransposed3D( const OperationDef& definition, - const ConvolutionTransposed3DAttributes& attr, const CLDevice& device) + const ConvolutionTransposed3DAttributes& attr, + const DeviceInfo& device_info) : GPUOperation(definition), - weights_are_buffer_(device.IsMali()), + weights_are_buffer_(device_info.IsMali()), kernel_size_(attr.weights.shape.w, attr.weights.shape.h, attr.weights.shape.d), stride_(attr.stride.w, attr.stride.h, attr.stride.d), padding_(attr.padding.prepended.w, attr.padding.prepended.h, attr.padding.prepended.d), block_size_(2, 2, 1, 2) { - code_ = GenerateConvolutionTransposed3DCode(definition_, device, - weights_are_buffer_, block_size_); - if (device.IsPowerVR() && block_size_.y != 1) { + code_ = GenerateConvolutionTransposed3DCode(definition_, weights_are_buffer_, + block_size_); + if (device_info.IsPowerVR() && block_size_.y != 1) { bool is_texture3d = definition_.src_tensors[0].storage_type == TensorStorageType::TEXTURE_3D; bool is_texture_array = definition_.src_tensors[0].storage_type == @@ -75,10 +76,10 @@ ConvolutionTransposed3D& ConvolutionTransposed3D::operator=( } std::string ConvolutionTransposed3D::GenerateConvolutionTransposed3DCode( - const OperationDef& op_def, const CLDevice& device, bool weights_are_buffer, + const OperationDef& op_def, bool weights_are_buffer, const int4& block_size) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); AddSrcTensor("src_tensor", src_desc); AddDstTensor("dst_tensor", op_def.dst_tensors[0]); @@ -402,7 +403,8 @@ absl::Status CreateConvolutionTransposed3D( const CreationContext& creation_context, const OperationDef& definition, const ConvolutionTransposed3DAttributes& attr, ConvolutionTransposed3D* result) { - *result = ConvolutionTransposed3D(definition, attr, *creation_context.device); + *result = ConvolutionTransposed3D(definition, attr, + creation_context.device->GetInfo()); RETURN_IF_ERROR( result->UploadWeights(attr.weights, creation_context.context)); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.h b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.h index b8b4aa75df2..0025d9da7b6 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.h @@ -55,7 +55,7 @@ class ConvolutionTransposed3D : public GPUOperation { ConvolutionTransposed3D* result); ConvolutionTransposed3D(const OperationDef& definition, const ConvolutionTransposed3DAttributes& attr, - const CLDevice& device); + const DeviceInfo& device_info); template absl::Status UploadWeights(const tflite::gpu::Tensor& weights, CLContext* context); @@ -65,7 +65,6 @@ class ConvolutionTransposed3D : public GPUOperation { absl::Span dst); std::string GenerateConvolutionTransposed3DCode(const OperationDef& op_def, - const CLDevice& device, bool weights_are_buffer, const int4& block_size); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3_thin.cc b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3_thin.cc index 8f8282781df..4fb93dd3263 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3_thin.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3_thin.cc @@ -27,12 +27,11 @@ namespace gpu { namespace cl { ConvolutionTransposed3x3Thin::ConvolutionTransposed3x3Thin( - const OperationDef& definition, const ConvolutionTransposedAttributes& attr, - const DeviceInfo& device_info) + const OperationDef& definition, const ConvolutionTransposedAttributes& attr) : GPUOperation(definition) { code_ = GenerateConvolutionTransposedCode( definition_, DivideRoundUp(attr.weights.shape.i, 4), - DivideRoundUp(attr.weights.shape.o, 4), device_info); + DivideRoundUp(attr.weights.shape.o, 4)); } ConvolutionTransposed3x3Thin::ConvolutionTransposed3x3Thin( @@ -48,10 +47,9 @@ ConvolutionTransposed3x3Thin& ConvolutionTransposed3x3Thin::operator=( } std::string ConvolutionTransposed3x3Thin::GenerateConvolutionTransposedCode( - const OperationDef& op_def, int src_depth, int dst_depth, - const DeviceInfo& device_info) { + const OperationDef& op_def, int src_depth, int dst_depth) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); AddSrcTensor("src_tensor", src_desc); AddDstTensor("dst_tensor", op_def.dst_tensors[0]); @@ -208,8 +206,7 @@ absl::Status CreateConvolutionTransposed3x3Thin( return absl::InvalidArgumentError( "ConvolutionTransposed3x3Thin doesn't support this attributes"); } - *result = ConvolutionTransposed3x3Thin(definition, attr, - creation_context.device->GetInfo()); + *result = ConvolutionTransposed3x3Thin(definition, attr); RETURN_IF_ERROR( result->UploadData(attr.weights, attr.bias, creation_context.context)); return absl::OkStatus(); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3_thin.h b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3_thin.h index 274d75cb167..5b4c4d05bac 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3_thin.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3_thin.h @@ -54,8 +54,7 @@ class ConvolutionTransposed3x3Thin : public GPUOperation { ConvolutionTransposed3x3Thin* result); explicit ConvolutionTransposed3x3Thin( const OperationDef& definition, - const ConvolutionTransposedAttributes& attr, - const DeviceInfo& device_info); + const ConvolutionTransposedAttributes& attr); template absl::Status UploadData(const tflite::gpu::Tensor& weights, const tflite::gpu::Tensor& biases, @@ -66,8 +65,7 @@ class ConvolutionTransposed3x3Thin : public GPUOperation { absl::Span dst); std::string GenerateConvolutionTransposedCode(const OperationDef& op_def, - int src_depth, int dst_depth, - const DeviceInfo& device_info); + int src_depth, int dst_depth); }; template diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv.cc b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv.cc index 3ab05134bd6..4b4416751fb 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv.cc @@ -70,8 +70,7 @@ std::string GetSrcValue(int channel_multiplier, const std::string coords) { DepthwiseConvolution::DepthwiseConvolution( const OperationDef& definition, - const DepthwiseConvolution2DAttributes& attr, bool weights_are_buffer, - const DeviceInfo& device_info) + const DepthwiseConvolution2DAttributes& attr, bool weights_are_buffer) : GPUOperation(definition), weights_are_buffer_(weights_are_buffer), kernel_size_(attr.weights.shape.w, attr.weights.shape.h, 0, 0), @@ -82,15 +81,13 @@ DepthwiseConvolution::DepthwiseConvolution( work_group_size_ = int3(8, 8, 1); const bool stride_correction = definition_.IsBatchSupported() && stride_.x != 1; - code_ = GenerateDepthwiseConvolutionCode(definition_, stride_correction, - channel_multiplier_, - weights_are_buffer_, device_info); + code_ = GenerateDepthwiseConvolutionCode( + definition_, stride_correction, channel_multiplier_, weights_are_buffer_); } DepthwiseConvolution::DepthwiseConvolution( const OperationDef& definition, - const DepthwiseConvolution3DAttributes& attr, bool weights_are_buffer, - const DeviceInfo& device_info) + const DepthwiseConvolution3DAttributes& attr, bool weights_are_buffer) : GPUOperation(definition), weights_are_buffer_(weights_are_buffer), kernel_size_(attr.weights.shape.w, attr.weights.shape.h, @@ -103,9 +100,8 @@ DepthwiseConvolution::DepthwiseConvolution( work_group_size_ = int3(8, 8, 1); const bool stride_correction = definition_.IsBatchSupported() && stride_.x != 1; - code_ = GenerateDepthwiseConvolutionCode(definition_, stride_correction, - channel_multiplier_, - weights_are_buffer_, device_info); + code_ = GenerateDepthwiseConvolutionCode( + definition_, stride_correction, channel_multiplier_, weights_are_buffer_); } DepthwiseConvolution::DepthwiseConvolution(DepthwiseConvolution&& operation) @@ -133,9 +129,9 @@ DepthwiseConvolution& DepthwiseConvolution::operator=( std::string DepthwiseConvolution::GenerateDepthwiseConvolutionCode( const OperationDef& op_def, bool stride_correction, int channel_multiplier, - bool weights_are_buffer, const DeviceInfo& device_info) { + bool weights_are_buffer) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); if (op_def.IsBatchSupported()) { src_desc.SetStateVar("BatchedWidth", "true"); } @@ -315,8 +311,7 @@ absl::Status CreateDepthwiseConvolution( const DepthwiseConvolution2DAttributes& attr, DepthwiseConvolution* result) { bool weights_are_buffer = creation_context.device->IsMali(); - *result = DepthwiseConvolution(definition, attr, weights_are_buffer, - creation_context.device->GetInfo()); + *result = DepthwiseConvolution(definition, attr, weights_are_buffer); RETURN_IF_ERROR( result->UploadWeights(attr.weights, creation_context.context)); @@ -339,8 +334,7 @@ absl::Status CreateDepthwiseConvolution( const DepthwiseConvolution3DAttributes& attr, DepthwiseConvolution* result) { bool weights_are_buffer = creation_context.device->IsMali(); - *result = DepthwiseConvolution(definition, attr, weights_are_buffer, - creation_context.device->GetInfo()); + *result = DepthwiseConvolution(definition, attr, weights_are_buffer); RETURN_IF_ERROR( result->UploadWeights(attr.weights, creation_context.context)); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv.h b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv.h index be87c182880..9a841db82ab 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv.h @@ -58,10 +58,10 @@ class DepthwiseConvolution : public GPUOperation { DepthwiseConvolution* result); DepthwiseConvolution(const OperationDef& definition, const DepthwiseConvolution2DAttributes& attr, - bool weights_are_buffer, const DeviceInfo& device_info); + bool weights_are_buffer); DepthwiseConvolution(const OperationDef& definition, const DepthwiseConvolution3DAttributes& attr, - bool weights_are_buffer, const DeviceInfo& device_info); + bool weights_are_buffer); template absl::Status UploadWeights(const tflite::gpu::Tensor& weights, @@ -82,8 +82,7 @@ class DepthwiseConvolution : public GPUOperation { std::string GenerateDepthwiseConvolutionCode(const OperationDef& op_def, bool stride_correction, int channel_multiplier, - bool weights_are_buffer, - const DeviceInfo& device_info); + bool weights_are_buffer); bool weights_are_buffer_; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc index 0bd84c3b116..e171231fc0a 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc @@ -35,8 +35,8 @@ DepthwiseConv3x3::DepthwiseConv3x3(const OperationDef& definition, weights_are_buffer_(weights_are_buffer), local_mem_uploads_(local_mem_uploads) { work_group_size_ = int3(8, 4, 1); - code_ = GenerateDepthwiseConvCode(definition_, device_info, - weights_are_buffer_, local_mem_uploads_); + code_ = GenerateDepthwiseConvCode(definition_, weights_are_buffer_, + local_mem_uploads_); if (definition_.precision == CalculationsPrecision::F16 && device_info.IsPowerVR()) { @@ -59,10 +59,10 @@ DepthwiseConv3x3& DepthwiseConv3x3::operator=(DepthwiseConv3x3&& operation) { } std::string DepthwiseConv3x3::GenerateDepthwiseConvCode( - const OperationDef& op_def, const DeviceInfo& device_info, - bool weights_are_buffer, bool local_mem_uploads) { + const OperationDef& op_def, bool weights_are_buffer, + bool local_mem_uploads) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); AddSrcTensor("src_tensor", src_desc); AddDstTensor("dst_tensor", op_def.dst_tensors[0]); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.h b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.h index d02d65b4e38..dedc9b530bb 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.h @@ -67,7 +67,6 @@ class DepthwiseConv3x3 : public GPUOperation { const tflite::gpu::Tensor& biases, absl::Span dst); std::string GenerateDepthwiseConvCode(const OperationDef& op_def, - const DeviceInfo& device_info, bool weights_are_buffer, bool local_mem_uploads); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling.cc b/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling.cc index 7be6cc0b9b4..97ee4878572 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling.cc @@ -25,24 +25,22 @@ namespace gpu { namespace cl { MaxUnpooling::MaxUnpooling(const OperationDef& definition, - const MaxUnpooling2DAttributes& attr, - const DeviceInfo& device_info) + const MaxUnpooling2DAttributes& attr) : GPUOperation(definition), stride_(attr.strides.w, attr.strides.h, 0, 0), padding_(attr.padding.appended.w, attr.padding.appended.h, 0, 0), kernel_size_(attr.kernel.w, attr.kernel.h, 0, 0) { - code_ = GetMaxUnpoolingKernelCode(definition_, device_info); + code_ = GetMaxUnpoolingKernelCode(definition_); } MaxUnpooling::MaxUnpooling(const OperationDef& definition, - const MaxUnpooling3DAttributes& attr, - const DeviceInfo& device_info) + const MaxUnpooling3DAttributes& attr) : GPUOperation(definition), stride_(attr.strides.w, attr.strides.h, attr.strides.d, 0), padding_(attr.padding.appended.w, attr.padding.appended.h, attr.padding.appended.d, 0), kernel_size_(attr.kernel.w, attr.kernel.h, attr.kernel.d, 0) { - code_ = GetMaxUnpoolingKernelCode(definition_, device_info); + code_ = GetMaxUnpoolingKernelCode(definition_); } MaxUnpooling::MaxUnpooling(MaxUnpooling&& kernel) @@ -62,15 +60,15 @@ MaxUnpooling& MaxUnpooling::operator=(MaxUnpooling&& kernel) { } std::string MaxUnpooling::GetMaxUnpoolingKernelCode( - const OperationDef& op_def, const DeviceInfo& device_info) { + const OperationDef& op_def) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); if (op_def.IsBatchSupported()) { src_desc.SetStateVar("BatchedWidth", "true"); } AddSrcTensor("src_tensor", src_desc); auto src_ind_desc = op_def.src_tensors[1]; - src_ind_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_ind_desc.SetTextureAddressMode(TextureAddressMode::ZERO); if (op_def.IsBatchSupported()) { src_ind_desc.SetStateVar("BatchedWidth", "true"); } @@ -202,15 +200,13 @@ int3 MaxUnpooling::GetGridSize() const { } MaxUnpooling CreateMaxUnpooling(const OperationDef& definition, - const MaxUnpooling2DAttributes& attr, - const DeviceInfo& device_info) { - return MaxUnpooling(definition, attr, device_info); + const MaxUnpooling2DAttributes& attr) { + return MaxUnpooling(definition, attr); } MaxUnpooling CreateMaxUnpooling(const OperationDef& definition, - const MaxUnpooling3DAttributes& attr, - const DeviceInfo& device_info) { - return MaxUnpooling(definition, attr, device_info); + const MaxUnpooling3DAttributes& attr) { + return MaxUnpooling(definition, attr); } } // namespace cl diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling.h b/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling.h index da4b0e28cec..0b1420a67c9 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling.h @@ -28,11 +28,9 @@ namespace cl { class MaxUnpooling : public GPUOperation { public: MaxUnpooling(const OperationDef& definition, - const MaxUnpooling2DAttributes& attr, - const DeviceInfo& device_info); + const MaxUnpooling2DAttributes& attr); MaxUnpooling(const OperationDef& definition, - const MaxUnpooling3DAttributes& attr, - const DeviceInfo& device_info); + const MaxUnpooling3DAttributes& attr); absl::Status BindArguments() override; int3 GetGridSize() const override; @@ -44,8 +42,7 @@ class MaxUnpooling : public GPUOperation { MaxUnpooling& operator=(const MaxUnpooling&) = delete; private: - std::string GetMaxUnpoolingKernelCode(const OperationDef& op_def, - const DeviceInfo& device_info); + std::string GetMaxUnpoolingKernelCode(const OperationDef& op_def); int4 stride_; int4 padding_; @@ -53,12 +50,10 @@ class MaxUnpooling : public GPUOperation { }; MaxUnpooling CreateMaxUnpooling(const OperationDef& definition, - const MaxUnpooling2DAttributes& attr, - const DeviceInfo& device_info); + const MaxUnpooling2DAttributes& attr); MaxUnpooling CreateMaxUnpooling(const OperationDef& definition, - const MaxUnpooling3DAttributes& attr, - const DeviceInfo& device_info); + const MaxUnpooling3DAttributes& attr); } // namespace cl } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling_test.cc index 77e92c8950b..c03cb4f89d7 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/max_unpooling_test.cc @@ -55,8 +55,7 @@ TEST_F(OpenCLOperationTest, MaxUnpooling) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - MaxUnpooling operation = - CreateMaxUnpooling(op_def, attr, env_.GetDevicePtr()->GetInfo()); + MaxUnpooling operation = CreateMaxUnpooling(op_def, attr); ASSERT_OK(ExecuteGPUOperation({src_tensor, src_ind_tensor}, creation_context_, &operation, BHWC(1, 4, 4, 1), &dst_tensor)); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/pooling.cc b/tensorflow/lite/delegates/gpu/cl/kernels/pooling.cc index 0c5a7a64d15..fb077fe4a1a 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/pooling.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/pooling.cc @@ -25,18 +25,18 @@ namespace gpu { namespace cl { Pooling::Pooling(const OperationDef& definition, - const Pooling2DAttributes& attr, const DeviceInfo& device_info) + const Pooling2DAttributes& attr) : GPUOperation(definition), stride_(attr.strides.w, attr.strides.h, 0, 0), padding_(-attr.padding.prepended.w, -attr.padding.prepended.h, 0, 0), kernel_size_(attr.kernel.w, attr.kernel.h, 0, 0), type_(attr.type), output_indices_(attr.output_indices) { - GenerateCode(device_info); + GenerateCode(); } Pooling::Pooling(const OperationDef& definition, - const Pooling3DAttributes& attr, const DeviceInfo& device_info) + const Pooling3DAttributes& attr) : GPUOperation(definition), stride_(attr.strides.w, attr.strides.h, attr.strides.d, 0), padding_(-attr.padding.prepended.w, -attr.padding.prepended.h, @@ -44,7 +44,7 @@ Pooling::Pooling(const OperationDef& definition, kernel_size_(attr.kernel.w, attr.kernel.h, attr.kernel.d, 0), type_(attr.type), output_indices_(attr.output_indices) { - GenerateCode(device_info); + GenerateCode(); } Pooling::Pooling(Pooling&& kernel) @@ -67,11 +67,10 @@ Pooling& Pooling::operator=(Pooling&& kernel) { return *this; } -std::string Pooling::GetAveragePoolingKernelCode( - const OperationDef& op_def, bool stride_correction, - const DeviceInfo& device_info) { +std::string Pooling::GetAveragePoolingKernelCode(const OperationDef& op_def, + bool stride_correction) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); if (op_def.IsBatchSupported()) { src_desc.SetStateVar("BatchedWidth", "true"); } @@ -348,12 +347,11 @@ std::string Pooling::GetMaxPoolingKernelCode(const OperationDef& op_def, return c; } -void Pooling::GenerateCode(const DeviceInfo& device_info) { +void Pooling::GenerateCode() { const bool stride_correction = definition_.IsBatchSupported() && stride_.x != 1; if (type_ == PoolingType::AVERAGE) { - code_ = GetAveragePoolingKernelCode(definition_, stride_correction, - device_info); + code_ = GetAveragePoolingKernelCode(definition_, stride_correction); } else if (type_ == PoolingType::MAX) { code_ = GetMaxPoolingKernelCode(definition_, stride_correction, output_indices_); @@ -387,15 +385,13 @@ int3 Pooling::GetGridSize() const { } Pooling CreatePooling(const OperationDef& definition, - const Pooling2DAttributes& attr, - const DeviceInfo& device_info) { - return Pooling(definition, attr, device_info); + const Pooling2DAttributes& attr) { + return Pooling(definition, attr); } Pooling CreatePooling(const OperationDef& definition, - const Pooling3DAttributes& attr, - const DeviceInfo& device_info) { - return Pooling(definition, attr, device_info); + const Pooling3DAttributes& attr) { + return Pooling(definition, attr); } } // namespace cl diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/pooling.h b/tensorflow/lite/delegates/gpu/cl/kernels/pooling.h index 07c3c6d85da..18bb426f259 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/pooling.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/pooling.h @@ -29,10 +29,8 @@ namespace cl { class Pooling : public GPUOperation { public: - Pooling(const OperationDef& definition, const Pooling2DAttributes& attr, - const DeviceInfo& device_info); - Pooling(const OperationDef& definition, const Pooling3DAttributes& attr, - const DeviceInfo& device_info); + Pooling(const OperationDef& definition, const Pooling2DAttributes& attr); + Pooling(const OperationDef& definition, const Pooling3DAttributes& attr); absl::Status BindArguments() override; int3 GetGridSize() const override; @@ -45,13 +43,12 @@ class Pooling : public GPUOperation { private: std::string GetAveragePoolingKernelCode(const OperationDef& op_def, - bool stride_correction, - const DeviceInfo& device_info); + bool stride_correction); std::string GetMaxPoolingKernelCode(const OperationDef& op_def, bool stride_correction, bool output_indices); - void GenerateCode(const DeviceInfo& device_info); + void GenerateCode(); int4 stride_; int4 padding_; @@ -62,12 +59,10 @@ class Pooling : public GPUOperation { }; Pooling CreatePooling(const OperationDef& definition, - const Pooling2DAttributes& attr, - const DeviceInfo& device_info); + const Pooling2DAttributes& attr); Pooling CreatePooling(const OperationDef& definition, - const Pooling3DAttributes& attr, - const DeviceInfo& device_info); + const Pooling3DAttributes& attr); } // namespace cl } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/pooling_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/pooling_test.cc index 7ebcc4871c5..12efd56f5d2 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/pooling_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/pooling_test.cc @@ -52,8 +52,7 @@ TEST_F(OpenCLOperationTest, AveragePooling) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - Pooling operation = - CreatePooling(op_def, attr, env_.GetDevicePtr()->GetInfo()); + Pooling operation = CreatePooling(op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 1, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, Pointwise(FloatNear(eps), {3.0f, 4.0f})); @@ -82,8 +81,7 @@ TEST_F(OpenCLOperationTest, AveragePoolingNonEmptyPadding) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - Pooling operation = - CreatePooling(op_def, attr, env_.GetDevicePtr()->GetInfo()); + Pooling operation = CreatePooling(op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 2, 2, 1), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -113,8 +111,7 @@ TEST_F(OpenCLOperationTest, MaxPooling) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - Pooling operation = - CreatePooling(op_def, attr, env_.GetDevicePtr()->GetInfo()); + Pooling operation = CreatePooling(op_def, attr); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 1, 1, 2), &dst_tensor)); EXPECT_THAT(dst_tensor.data, Pointwise(FloatNear(eps), {8.0f, 7.0f})); @@ -146,8 +143,7 @@ TEST_F(OpenCLOperationTest, MaxPoolingIndices) { op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; TensorFloat32 dst_tensor_ind; - Pooling operation = - CreatePooling(op_def, attr, env_.GetDevicePtr()->GetInfo()); + Pooling operation = CreatePooling(op_def, attr); ASSERT_OK(ExecuteGPUOperation({src_tensor}, creation_context_, &operation, {BHWC(1, 1, 1, 2), BHWC(1, 1, 1, 2)}, {&dst_tensor, &dst_tensor_ind})); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/special/depthwise_conv_plus_1x1_conv.cc b/tensorflow/lite/delegates/gpu/cl/kernels/special/depthwise_conv_plus_1x1_conv.cc index 88417ce6f1e..e95e758fc95 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/special/depthwise_conv_plus_1x1_conv.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/special/depthwise_conv_plus_1x1_conv.cc @@ -30,12 +30,11 @@ namespace cl { DepthwiseConvPlus1x1Conv::DepthwiseConvPlus1x1Conv( const OperationDef& definition, const DepthwiseConvolution2DAttributes& dw_attr, - const Convolution2DAttributes& conv_attr, const DeviceInfo& device_info) + const Convolution2DAttributes& conv_attr) : GPUOperation(definition), dw_attr_(dw_attr) { work_group_size_ = int3(8, 8, 1); - code_ = - GenerateCode(definition_, dw_attr_, - DivideRoundUp(conv_attr.weights.shape.o, 4), device_info); + code_ = GenerateCode(definition_, dw_attr_, + DivideRoundUp(conv_attr.weights.shape.o, 4)); } DepthwiseConvPlus1x1Conv::DepthwiseConvPlus1x1Conv( @@ -146,9 +145,9 @@ absl::Status DepthwiseConvPlus1x1Conv::UploadWeights( std::string DepthwiseConvPlus1x1Conv::GenerateCode( const OperationDef& op_def, const DepthwiseConvolution2DAttributes& dw_attr, - int result_depth, const DeviceInfo& device_info) { + int result_depth) { auto src_desc = op_def.src_tensors[0]; - src_desc.SetTextureAddressMode(GetFastestZeroMode(device_info)); + src_desc.SetTextureAddressMode(TextureAddressMode::ZERO); AddSrcTensor("src_tensor", src_desc); AddDstTensor("dst_tensor", op_def.dst_tensors[0]); @@ -273,8 +272,7 @@ absl::Status CreateDepthwiseConvPlus1x1Conv( const DepthwiseConvolution2DAttributes& dw_attr, const Convolution2DAttributes& conv_attr, DepthwiseConvPlus1x1Conv* result) { - *result = DepthwiseConvPlus1x1Conv(definition, dw_attr, conv_attr, - creation_context.device->GetInfo()); + *result = DepthwiseConvPlus1x1Conv(definition, dw_attr, conv_attr); RETURN_IF_ERROR( result->UploadWeights(dw_attr, conv_attr, creation_context.context)); return absl::OkStatus(); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/special/depthwise_conv_plus_1x1_conv.h b/tensorflow/lite/delegates/gpu/cl/kernels/special/depthwise_conv_plus_1x1_conv.h index d4037c83b30..b2d3b05d285 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/special/depthwise_conv_plus_1x1_conv.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/special/depthwise_conv_plus_1x1_conv.h @@ -52,8 +52,7 @@ class DepthwiseConvPlus1x1Conv : public GPUOperation { DepthwiseConvPlus1x1Conv* result); DepthwiseConvPlus1x1Conv(const OperationDef& definition, const DepthwiseConvolution2DAttributes& dw_attr, - const Convolution2DAttributes& conv_attr, - const DeviceInfo& device_info); + const Convolution2DAttributes& conv_attr); absl::Status UploadWeights(const DepthwiseConvolution2DAttributes& dw_attr, const Convolution2DAttributes& conv_attr, @@ -61,7 +60,7 @@ class DepthwiseConvPlus1x1Conv : public GPUOperation { std::string GenerateCode(const OperationDef& op_def, const DepthwiseConvolution2DAttributes& dw_attr, - int result_depth, const DeviceInfo& device_info); + int result_depth); DepthwiseConvolution2DAttributes dw_attr_; }; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/util.cc b/tensorflow/lite/delegates/gpu/cl/kernels/util.cc index e3599eb5044..3fe4ffb4acd 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/util.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/util.cc @@ -69,17 +69,6 @@ std::string GetCommonDefines(CalculationsPrecision precision) { result += "#define TO_ACCUM_FLT convert_float\n"; break; } - - result += - "__constant sampler_t smp_edge = CLK_NORMALIZED_COORDS_FALSE | " - "CLK_ADDRESS_CLAMP_TO_EDGE | CLK_FILTER_NEAREST;\n"; - result += - "__constant sampler_t smp_none = CLK_NORMALIZED_COORDS_FALSE | " - "CLK_ADDRESS_NONE | CLK_FILTER_NEAREST;\n"; - result += - "__constant sampler_t smp_zero = CLK_NORMALIZED_COORDS_FALSE | " - "CLK_ADDRESS_CLAMP | CLK_FILTER_NEAREST;\n"; - return result; } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/util.h b/tensorflow/lite/delegates/gpu/cl/kernels/util.h index b41d0efb91e..173a4d43072 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/util.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/util.h @@ -83,20 +83,6 @@ void RearrangeWeightsToOHWIOGroupI4O4( } } -// Returns fastest TextureAddressMode that return ZERO for out-of-range image -// coordinates. -// -// Unfortunately, CLK_ADDRESS_CLAMP is very slow on Adreno3xx and -// we can observe huge register overhead when compared to other modes. - -// While using CLK_ADDRESS_NONE with out-of-range image coordinates is undefined -// in the OpenCL specification, we have observed that CLK_ADDRESS_NONE works -// like CLK_ADDRESS_CLAMP for out-of-range image coordinates for RGBA F16/F32 -// textures on Adreno3xx devices. Using CLK_ADDRESS_NONE is significantly faster -// than CLK_ADDRESS_CLAMP on Adreno 3xx. -TextureAddressMode GetFastestZeroMode(const CLDevice& device); -TextureAddressMode GetFastestZeroMode(const DeviceInfo& device_info); - // Returns float4 mask for last plane(batch of 4 channels) // assumes that plane size is 4; // for example we have 7 channels, in our data structures we align it to 8 diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc index e1225e83e95..5661c3d0a37 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc @@ -254,8 +254,7 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, case OperationType::MAX_UNPOOLING_2D: { auto attr = absl::any_cast(node.operation.attributes); - SelectMaxUnpooling(attr, op_def, creation_context.device->GetInfo(), - gpu_op); + SelectMaxUnpooling(attr, op_def, gpu_op); return absl::OkStatus(); } case OperationType::MEAN: { @@ -277,7 +276,7 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, case OperationType::POOLING_2D: { auto attr = absl::any_cast(node.operation.attributes); - SelectPooling(attr, op_def, creation_context.device->GetInfo(), gpu_op); + SelectPooling(attr, op_def, gpu_op); return absl::OkStatus(); } case OperationType::PRELU: { diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.cc b/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.cc index 1c0bed74422..ca5ec9f4f23 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.cc @@ -69,17 +69,15 @@ absl::Status SelectPReLU(const PReLUAttributes& attr, } void SelectPooling(const Pooling2DAttributes& attr, const OperationDef& op_def, - const DeviceInfo& device_info, std::unique_ptr* ptr) { - Pooling pooling = CreatePooling(op_def, attr, device_info); + Pooling pooling = CreatePooling(op_def, attr); *ptr = absl::make_unique(std::move(pooling)); } void SelectMaxUnpooling(const MaxUnpooling2DAttributes& attr, const OperationDef& op_def, - const DeviceInfo& device_info, std::unique_ptr* ptr) { - MaxUnpooling operation = CreateMaxUnpooling(op_def, attr, device_info); + MaxUnpooling operation = CreateMaxUnpooling(op_def, attr); *ptr = absl::make_unique(std::move(operation)); } diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.h b/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.h index 7133aa94502..556698ef62f 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.h +++ b/tensorflow/lite/delegates/gpu/cl/selectors/simple_selectors.h @@ -41,12 +41,10 @@ absl::Status SelectPReLU(const PReLUAttributes& attr, std::unique_ptr* ptr); void SelectPooling(const Pooling2DAttributes& attr, const OperationDef& op_def, - const DeviceInfo& device_info, std::unique_ptr* ptr); void SelectMaxUnpooling(const MaxUnpooling2DAttributes& attr, const OperationDef& op_def, - const DeviceInfo& device_info, std::unique_ptr* ptr); void SelectAdd(const OperationDef& op_def, const std::vector& channels, From 6ccee7b92432a2a0c41dc3b8ae207f3b570d6737 Mon Sep 17 00:00:00 2001 From: Doe Hyun Yoon Date: Tue, 4 Aug 2020 10:02:42 -0700 Subject: [PATCH 0369/1017] Add ImmutableNodeMap for const GraphDef. NodeMap and ImmutableNodeMap are subclass of NodeMapInternal. PiperOrigin-RevId: 324833428 Change-Id: I6fbc95fa7ee8ff38e0409314589806aa217aa5b3 --- tensorflow/core/grappler/utils.cc | 31 ++++---- tensorflow/core/grappler/utils.h | 64 ++++++++++++--- tensorflow/core/grappler/utils_test.cc | 103 ++++++++++++++++++------- 3 files changed, 138 insertions(+), 60 deletions(-) diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc index 151bb9d5d86..e342f7dfdf0 100644 --- a/tensorflow/core/grappler/utils.cc +++ b/tensorflow/core/grappler/utils.cc @@ -73,26 +73,21 @@ bool IsShapeConsumer(const NodeDef& node) { } // namespace -NodeMap::NodeMap(GraphDef* graph) { - CHECK(graph != nullptr); - nodes_.reserve(graph->node_size()); - outputs_.reserve(graph->node_size()); - for (int i = 0; i < graph->node_size(); i++) { - NodeDef* node = graph->mutable_node(i); - const string& node_name = node->name(); - auto rslt = nodes_.emplace(node_name, node); - // Check that the graph doesn't contain multiple nodes with the same name. - if (!rslt.second) { - // The first node found with a given name becomes the canonical. - LOG(WARNING) << "Duplicated node in the graph: " << node_name; - } - NodeDef* canonical = rslt.second ? node : rslt.first->second; - for (const auto& input : node->input()) { - outputs_[NodeName(input)].insert(canonical); - } - } +namespace internal { +// Specialized template class method GetNodeDefFromGraph. +template <> +NodeDef* NodeMapInternal::GetNodeDefFromGraph( + GraphDef* graph, int64 i) const { + return graph->mutable_node(i); } +template <> +const NodeDef* +NodeMapInternal::GetNodeDefFromGraph( + const GraphDef* graph, int64 i) const { + return &graph->node(i); +} +} // namespace internal string TensorIdToString(const TensorId& tensor_id) { return tensor_id.index() == 0 ? string(tensor_id.node()) : tensor_id.ToString(); diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h index e529d5fb4ad..e9ab5b7da12 100644 --- a/tensorflow/core/grappler/utils.h +++ b/tensorflow/core/grappler/utils.h @@ -98,16 +98,39 @@ inline int NodePosition(const string& name) { return position; } -// A utility class to lookup a node and its outputs by node name. -class NodeMap { +namespace internal { +// Base template class for NodeMap and ImmutableNodeMap. +template +class NodeMapInternal { public: // Note: The NodeMap will store pointers to nodes in graph, which may become // invalid if graph is changed. - explicit NodeMap(GraphDef* graph); + explicit NodeMapInternal(GraphDefT* graph) { + if (graph == nullptr) { + LOG(WARNING) << "NodeMapInternal constructor is called with a nullptr!"; + return; + } + nodes_.reserve(graph->node_size()); + outputs_.reserve(graph->node_size()); + for (int i = 0; i < graph->node_size(); i++) { + NodeDefT* node = GetNodeDefFromGraph(graph, i); + const string& node_name = node->name(); + auto rslt = nodes_.emplace(node_name, node); + // Check that the graph doesn't contain multiple nodes with the same name. + if (!rslt.second) { + // The first node found with a given name becomes the canonical. + LOG(WARNING) << "Duplicated node in the graph: " << node_name; + } + NodeDefT* canonical = rslt.second ? node : rslt.first->second; + for (const auto& input : node->input()) { + outputs_[NodeName(input)].insert(canonical); + } + } + } // Get unordered list of fanouts from node. Notice, that the order is // non-deterministic. - const absl::flat_hash_set& GetOutputs( + const absl::flat_hash_set& GetOutputs( const string& node_name) const { auto it = outputs_.find(node_name); if (it == outputs_.end()) { @@ -117,12 +140,12 @@ class NodeMap { } // Get fanouts ordered by name. - std::vector GetOutputsOrderedByNodeName( + std::vector GetOutputsOrderedByNodeName( const string& node_name) const { - std::vector result; + std::vector result; auto it = outputs_.find(node_name); if (it != outputs_.end()) { - const absl::flat_hash_set& outputs = it->second; + const absl::flat_hash_set& outputs = it->second; result.reserve(outputs.size()); result.assign(outputs.begin(), outputs.end()); std::sort(result.begin(), result.end(), @@ -135,7 +158,7 @@ class NodeMap { // This method doesn't record the outputs of the added node; the outputs need // to be explicitly added by the AddOutput method. - void AddNode(const string& node_name, NodeDef* node) { + void AddNode(const string& node_name, NodeDefT* node) { DCHECK(node != nullptr); auto ret = nodes_.emplace(node_name, node); DCHECK(ret.second) @@ -148,7 +171,7 @@ class NodeMap { outputs_.erase(NodeName(name)); } - NodeDef* GetNode(const string& name) const { + NodeDefT* GetNode(const string& name) const { const string node_name = NodeName(name); auto it = nodes_.find(node_name); if (it == nodes_.end()) { @@ -197,9 +220,26 @@ class NodeMap { } private: - const absl::flat_hash_set empty_set_; - absl::node_hash_map nodes_; - absl::node_hash_map> outputs_; + // Helper method to get the NodeDef pointer of i-th node in a graph. + NodeDefT* GetNodeDefFromGraph(GraphDefT* graph, int64 i) const; + + const absl::flat_hash_set empty_set_; + absl::node_hash_map nodes_; + absl::node_hash_map> outputs_; +}; +} // namespace internal + +// A utility class to lookup a node and its outputs by node name. +class NodeMap : public internal::NodeMapInternal { + public: + explicit NodeMap(GraphDef* graph) : NodeMapInternal(graph) {} +}; + +// Same to NodeMap, but uses const GraphDef. +class ImmutableNodeMap + : public internal::NodeMapInternal { + public: + explicit ImmutableNodeMap(const GraphDef* graph) : NodeMapInternal(graph) {} }; // A vector with a set. The set stores the same elements as the vector, and diff --git a/tensorflow/core/grappler/utils_test.cc b/tensorflow/core/grappler/utils_test.cc index 6231fb7a780..e7e57e9b7d7 100644 --- a/tensorflow/core/grappler/utils_test.cc +++ b/tensorflow/core/grappler/utils_test.cc @@ -349,39 +349,69 @@ TEST_F(UtilsTest, NumNonControlOutputs) { GraphDef graph; TF_CHECK_OK(s.ToGraphDef(&graph)); - NodeMap node_map(&graph); - const NodeDef* add_node = node_map.GetNode("add"); - const NodeDef* mul_node = node_map.GetNode("mul"); - ASSERT_NE(add_node, nullptr); + { + NodeMap node_map(&graph); - // [a, b] are only non-control inputs - EXPECT_EQ(NumNonControlInputs(*add_node), 2); - EXPECT_EQ(NumControlInputs(*add_node), 1); - // [sqrt, shape] are non control outputs - EXPECT_EQ(NumNonControlOutputs(*add_node, node_map), 2); - // sqrt is the only data output - EXPECT_EQ(NumNonControlDataOutputs(*add_node, node_map), 1); - EXPECT_EQ(NumControlInputs(*mul_node), 0); + const NodeDef* add_node = node_map.GetNode("add"); + const NodeDef* mul_node = node_map.GetNode("mul"); + ASSERT_NE(add_node, nullptr); - EXPECT_TRUE(HasControlInputs(*add_node)); - EXPECT_TRUE(HasRegularInputs(*add_node)); - EXPECT_TRUE(HasControlOutputs(*add_node, node_map)); - EXPECT_TRUE(HasRegularOutputs(*add_node, node_map)); + // [a, b] are only non-control inputs + EXPECT_EQ(NumNonControlInputs(*add_node), 2); + EXPECT_EQ(NumControlInputs(*add_node), 1); + // [sqrt, shape] are non control outputs + EXPECT_EQ(NumNonControlOutputs(*add_node, node_map), 2); + // sqrt is the only data output + EXPECT_EQ(NumNonControlDataOutputs(*add_node, node_map), 1); + EXPECT_EQ(NumControlInputs(*mul_node), 0); - const NodeDef* x_node = node_map.GetNode("x"); - ASSERT_NE(x_node, nullptr); - EXPECT_FALSE(HasControlInputs(*x_node)); - EXPECT_FALSE(HasRegularInputs(*x_node)); - EXPECT_FALSE(HasControlOutputs(*x_node, node_map)); - EXPECT_TRUE(HasRegularOutputs(*x_node, node_map)); + EXPECT_TRUE(HasControlInputs(*add_node)); + EXPECT_TRUE(HasRegularInputs(*add_node)); + EXPECT_TRUE(HasControlOutputs(*add_node, node_map)); + EXPECT_TRUE(HasRegularOutputs(*add_node, node_map)); - const NodeDef* round_node = node_map.GetNode("round"); - ASSERT_NE(round_node, nullptr); - EXPECT_TRUE(HasControlInputs(*round_node)); - EXPECT_TRUE(HasRegularInputs(*round_node)); - EXPECT_FALSE(HasControlOutputs(*round_node, node_map)); - EXPECT_FALSE(HasRegularOutputs(*round_node, node_map)); + const NodeDef* x_node = node_map.GetNode("x"); + ASSERT_NE(x_node, nullptr); + EXPECT_FALSE(HasControlInputs(*x_node)); + EXPECT_FALSE(HasRegularInputs(*x_node)); + EXPECT_FALSE(HasControlOutputs(*x_node, node_map)); + EXPECT_TRUE(HasRegularOutputs(*x_node, node_map)); + + const NodeDef* round_node = node_map.GetNode("round"); + ASSERT_NE(round_node, nullptr); + EXPECT_TRUE(HasControlInputs(*round_node)); + EXPECT_TRUE(HasRegularInputs(*round_node)); + EXPECT_FALSE(HasControlOutputs(*round_node, node_map)); + EXPECT_FALSE(HasRegularOutputs(*round_node, node_map)); + } + + { + // Similar test for ImmutableNodeMap. + ImmutableNodeMap node_map(&graph); + + const NodeDef* add_node = node_map.GetNode("add"); + const NodeDef* mul_node = node_map.GetNode("mul"); + ASSERT_NE(add_node, nullptr); + + // [a, b] are only non-control inputs + EXPECT_EQ(NumNonControlInputs(*add_node), 2); + EXPECT_EQ(NumControlInputs(*add_node), 1); + EXPECT_EQ(NumControlInputs(*mul_node), 0); + + EXPECT_TRUE(HasControlInputs(*add_node)); + EXPECT_TRUE(HasRegularInputs(*add_node)); + + const NodeDef* x_node = node_map.GetNode("x"); + ASSERT_NE(x_node, nullptr); + EXPECT_FALSE(HasControlInputs(*x_node)); + EXPECT_FALSE(HasRegularInputs(*x_node)); + + const NodeDef* round_node = node_map.GetNode("round"); + ASSERT_NE(round_node, nullptr); + EXPECT_TRUE(HasControlInputs(*round_node)); + EXPECT_TRUE(HasRegularInputs(*round_node)); + } } TEST(CheckAttrExists, All) { @@ -653,17 +683,30 @@ TEST(SetTensorValueTest, Quantized) { /*error_msg=*/""); } -static void BM_NodeMapConstruct(int iters, int size) { +static void BM_NodeMapConstruct(benchmark::State& state) { + const int size = state.range(0); testing::StopTiming(); GraphDef graph = test::CreateRandomGraph(size); testing::StartTiming(); - for (int i = 0; i < iters; i++) { + for (auto s : state) { NodeMap node_map(&graph); } testing::StopTiming(); } BENCHMARK(BM_NodeMapConstruct)->Range(1, 1 << 20); +static void BM_ImmutableNodeMapConstruct(benchmark::State& state) { + const int size = state.range(0); + testing::StopTiming(); + GraphDef graph = test::CreateRandomGraph(size); + testing::StartTiming(); + for (auto s : state) { + ImmutableNodeMap node_map(&graph); + } + testing::StopTiming(); +} +BENCHMARK(BM_ImmutableNodeMapConstruct)->Range(1, 1 << 20); + } // namespace } // namespace grappler } // namespace tensorflow From c8ddf5a1dfae741079955bd11e09894ac1ee2f46 Mon Sep 17 00:00:00 2001 From: Blake Hechtman Date: Tue, 4 Aug 2020 10:05:23 -0700 Subject: [PATCH 0370/1017] [XLA] Allow kBitcast to also change types. PiperOrigin-RevId: 324834056 Change-Id: I3376ff3984564a043c5b1f7f077bc31b30adef2c --- tensorflow/compiler/xla/service/BUILD | 1 + .../compiler/xla/service/hlo_creation_utils.cc | 6 ++++++ .../compiler/xla/service/hlo_instruction.cc | 2 +- .../compiler/xla/service/hlo_verifier.cc | 8 -------- .../compiler/xla/service/hlo_verifier_test.cc | 18 ------------------ 5 files changed, 8 insertions(+), 27 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 540cd7fecd2..4d15bc432a2 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -1705,6 +1705,7 @@ cc_library( ":hlo", ":hlo_module_config", ":shape_inference", + "//tensorflow/compiler/xla:comparison_util", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:statusor", diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc index 0f5267e9fbc..4ba67888409 100644 --- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc +++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/lib/comparators.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/client/xla_computation.h" +#include "tensorflow/compiler/xla/comparison_util.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/service/hlo_clone_context.h" @@ -258,6 +259,11 @@ HloInstruction* MakeBitcastConvertToHlo(HloInstruction* hlo, PrimitiveType type) { CHECK_NE(hlo->shape().element_type(), type); Shape shape = ShapeUtil::ChangeElementType(hlo->shape(), type); + // PRED are stored as one byte, PRED have a BitWidth of 1, avoid this problem + // by using a convert instead of bitcast convert. + if (type == PRED || hlo->shape().element_type() == PRED) { + return MakeConvertToHlo(hlo, type); + } hlo = hlo->parent()->AddInstruction( HloInstruction::CreateBitcastConvert(shape, hlo)); CHECK_EQ(hlo->shape().element_type(), type); diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 4335ed312c3..94d53ebe0b1 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -2207,7 +2207,7 @@ Status HloInstruction::ReplaceUsesWith(absl::Span users, Status HloInstruction::ReplaceAllUsesWithDifferentShape( absl::Span users, HloInstruction* new_producer) { for (HloInstruction* user : users) { - TF_RETURN_IF_ERROR(ReplaceUseWith(user, new_producer)); + TF_RETURN_IF_ERROR(ReplaceUseWithDifferentShape(user, new_producer)); } if (parent_ && parent_->root_instruction() == this) { diff --git a/tensorflow/compiler/xla/service/hlo_verifier.cc b/tensorflow/compiler/xla/service/hlo_verifier.cc index 62b0d98418c..d395fddcc5d 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier.cc @@ -670,14 +670,6 @@ Status ShapeVerifier::HandleReduce(HloInstruction* reduce) { } Status ShapeVerifier::HandleBitcast(HloInstruction* bitcast) { - // Bitcasts are not allowed to change the element type. - if (bitcast->operand(0)->shape().element_type() != - bitcast->shape().element_type()) { - return InternalError( - "Bitcast can not change the element type from %s to %s", - PrimitiveType_Name(bitcast->operand(0)->shape().element_type()), - PrimitiveType_Name(bitcast->shape().element_type())); - } if (layout_sensitive_ && shape_size_function_(bitcast->shape()) != shape_size_function_(bitcast->operand(0)->shape())) { diff --git a/tensorflow/compiler/xla/service/hlo_verifier_test.cc b/tensorflow/compiler/xla/service/hlo_verifier_test.cc index d9709c50df9..1f71c9586d5 100644 --- a/tensorflow/compiler/xla/service/hlo_verifier_test.cc +++ b/tensorflow/compiler/xla/service/hlo_verifier_test.cc @@ -540,24 +540,6 @@ TEST_F(HloVerifierTestLayoutSensitive, ConcatWithLayoutChangeNotAllowed) { HasSubstr("Instruction shouldn't change layouts")); } -TEST_F(HloVerifierTest, BitcastCanNotChangeElementType) { - const char* const hlo_string = R"( - HloModule Module - - ENTRY BitcastCanNotChangeElementType { - constant.0 = f32[2] constant({0.0, 0.0}) - ROOT bitcast = s32[2] bitcast(constant.0) - } - )"; - TF_ASSERT_OK_AND_ASSIGN(auto module, - ParseAndReturnUnverifiedModule(hlo_string)); - - auto status = verifier().Run(module.get()).status(); - ASSERT_FALSE(status.ok()); - EXPECT_THAT(status.error_message(), - HasSubstr("Bitcast can not change the element type")); -} - TEST_F(HloVerifierTestLayoutSensitive, BitcastNeedsSameNumberOfElements) { const char* const hlo_string = R"( HloModule Module From 12b7b9e06d9d7b0f29ebe4b8a3645daa058c3bd1 Mon Sep 17 00:00:00 2001 From: Tomer Kaftan Date: Tue, 4 Aug 2020 10:08:00 -0700 Subject: [PATCH 0371/1017] The `linalg.LinearOperator*` Module APIs do not support top-level dispatching because they are classes w/ methods instead of top-level methods in TF's APIs. But, their class methods call out to APIs that do support dispatching. This CL updates the convert_to_tensor calls in the `linalg.LinearOperator*` APIs to use the publicly exposed, dispatching `convert_to_tensor_v2_with_dispatch`, which enables the Operators to effectively work with dispatching as the APIs they call out to support dispatching as well. PiperOrigin-RevId: 324834645 Change-Id: If2e9f17be101e74f8835497d8ca51a0174055053 --- .../python/ops/linalg/linear_operator.py | 16 +++---- .../ops/linalg/linear_operator_block_diag.py | 18 ++++---- .../linear_operator_block_lower_triangular.py | 18 ++++---- .../ops/linalg/linear_operator_circulant.py | 4 +- .../python/ops/linalg/linear_operator_diag.py | 2 +- .../ops/linalg/linear_operator_full_matrix.py | 2 +- .../ops/linalg/linear_operator_householder.py | 9 ++-- .../ops/linalg/linear_operator_identity.py | 4 +- .../ops/linalg/linear_operator_permutation.py | 6 +-- .../ops/linalg/linear_operator_toeplitz.py | 8 ++-- .../ops/linalg/linear_operator_tridiag.py | 6 ++- .../python/ops/linalg/linear_operator_util.py | 22 ++++----- tensorflow/python/util/dispatch_test.py | 45 +++++++++++++++++++ 13 files changed, 106 insertions(+), 54 deletions(-) diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py index 8e1967f63c1..cf14cdb6eae 100644 --- a/tensorflow/python/ops/linalg/linear_operator.py +++ b/tensorflow/python/ops/linalg/linear_operator.py @@ -385,7 +385,7 @@ class LinearOperator(module.Module): # `shape` may be passed in if this can be pre-computed in a # more efficient manner, e.g. without excessive Tensor conversions. if self.tensor_rank is not None: - return ops.convert_to_tensor(self.tensor_rank) + return ops.convert_to_tensor_v2_with_dispatch(self.tensor_rank) else: shape = self.shape_tensor() if shape is None else shape return array_ops.size(shape) @@ -429,7 +429,7 @@ class LinearOperator(module.Module): # more efficient manner, e.g. without excessive Tensor conversions. dim_value = tensor_shape.dimension_value(self.domain_dimension) if dim_value is not None: - return ops.convert_to_tensor(dim_value) + return ops.convert_to_tensor_v2_with_dispatch(dim_value) else: shape = self.shape_tensor() if shape is None else shape return shape[-1] @@ -473,7 +473,7 @@ class LinearOperator(module.Module): # more efficient manner, e.g. without excessive Tensor conversions. dim_value = tensor_shape.dimension_value(self.range_dimension) if dim_value is not None: - return ops.convert_to_tensor(dim_value) + return ops.convert_to_tensor_v2_with_dispatch(dim_value) else: shape = self.shape_tensor() if shape is None else shape return shape[-2] @@ -641,7 +641,7 @@ class LinearOperator(module.Module): return linear_operator_algebra.matmul(left_operator, right_operator) with self._name_scope(name): - x = ops.convert_to_tensor(x, name="x") + x = ops.convert_to_tensor_v2_with_dispatch(x, name="x") self._check_input_dtype(x) self_dim = -2 if adjoint else -1 @@ -688,7 +688,7 @@ class LinearOperator(module.Module): A `Tensor` with shape `[..., M]` and same `dtype` as `self`. """ with self._name_scope(name): - x = ops.convert_to_tensor(x, name="x") + x = ops.convert_to_tensor_v2_with_dispatch(x, name="x") self._check_input_dtype(x) self_dim = -2 if adjoint else -1 tensor_shape.dimension_at_index( @@ -834,7 +834,7 @@ class LinearOperator(module.Module): return linear_operator_algebra.solve(left_operator, right_operator) with self._name_scope(name): - rhs = ops.convert_to_tensor(rhs, name="rhs") + rhs = ops.convert_to_tensor_v2_with_dispatch(rhs, name="rhs") self._check_input_dtype(rhs) self_dim = -1 if adjoint else -2 @@ -891,7 +891,7 @@ class LinearOperator(module.Module): NotImplementedError: If `self.is_non_singular` or `is_square` is False. """ with self._name_scope(name): - rhs = ops.convert_to_tensor(rhs, name="rhs") + rhs = ops.convert_to_tensor_v2_with_dispatch(rhs, name="rhs") self._check_input_dtype(rhs) self_dim = -1 if adjoint else -2 tensor_shape.dimension_at_index( @@ -1054,7 +1054,7 @@ class LinearOperator(module.Module): A `Tensor` with broadcast shape and same `dtype` as `self`. """ with self._name_scope(name): - x = ops.convert_to_tensor(x, name="x") + x = ops.convert_to_tensor_v2_with_dispatch(x, name="x") self._check_input_dtype(x) return self._add_to_tensor(x) diff --git a/tensorflow/python/ops/linalg/linear_operator_block_diag.py b/tensorflow/python/ops/linalg/linear_operator_block_diag.py index 7c50d00a055..7afa15ae069 100644 --- a/tensorflow/python/ops/linalg/linear_operator_block_diag.py +++ b/tensorflow/python/ops/linalg/linear_operator_block_diag.py @@ -263,7 +263,7 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator): def _shape_tensor(self): # Avoid messy broadcasting if possible. if self.shape.is_fully_defined(): - return ops.convert_to_tensor( + return ops.convert_to_tensor_v2_with_dispatch( self.shape.as_list(), dtype=dtypes.int32, name="shape") domain_dimension = sum(self._block_domain_dimension_tensors()) @@ -330,12 +330,12 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator): if linear_operator_util.arg_is_blockwise(block_dimensions, x, arg_dim): for i, block in enumerate(x): if not isinstance(block, linear_operator.LinearOperator): - block = ops.convert_to_tensor(block) + block = ops.convert_to_tensor_v2_with_dispatch(block) self._check_input_dtype(block) block_dimensions[i].assert_is_compatible_with(block.shape[arg_dim]) x[i] = block else: - x = ops.convert_to_tensor(x, name="x") + x = ops.convert_to_tensor_v2_with_dispatch(x, name="x") self._check_input_dtype(x) op_dimension = (self.range_dimension if adjoint else self.domain_dimension) @@ -404,7 +404,7 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator): if linear_operator_util.arg_is_blockwise(block_dimensions, x, -1): for i, block in enumerate(x): if not isinstance(block, linear_operator.LinearOperator): - block = ops.convert_to_tensor(block) + block = ops.convert_to_tensor_v2_with_dispatch(block) self._check_input_dtype(block) block_dimensions[i].assert_is_compatible_with(block.shape[-1]) x[i] = block @@ -412,7 +412,7 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator): y_mat = self.matmul(x_mat, adjoint=adjoint) return [array_ops.squeeze(y, axis=-1) for y in y_mat] - x = ops.convert_to_tensor(x, name="x") + x = ops.convert_to_tensor_v2_with_dispatch(x, name="x") self._check_input_dtype(x) op_dimension = (self.range_dimension if adjoint else self.domain_dimension) @@ -508,12 +508,12 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator): split_rhs = rhs for i, block in enumerate(split_rhs): if not isinstance(block, linear_operator.LinearOperator): - block = ops.convert_to_tensor(block) + block = ops.convert_to_tensor_v2_with_dispatch(block) self._check_input_dtype(block) block_dimensions[i].assert_is_compatible_with(block.shape[arg_dim]) split_rhs[i] = block else: - rhs = ops.convert_to_tensor(rhs, name="rhs") + rhs = ops.convert_to_tensor_v2_with_dispatch(rhs, name="rhs") self._check_input_dtype(rhs) op_dimension = (self.domain_dimension if adjoint else self.range_dimension) @@ -583,7 +583,7 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator): if linear_operator_util.arg_is_blockwise(block_dimensions, rhs, -1): for i, block in enumerate(rhs): if not isinstance(block, linear_operator.LinearOperator): - block = ops.convert_to_tensor(block) + block = ops.convert_to_tensor_v2_with_dispatch(block) self._check_input_dtype(block) block_dimensions[i].assert_is_compatible_with(block.shape[-1]) rhs[i] = block @@ -591,7 +591,7 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator): solution_mat = self.solve(rhs_mat, adjoint=adjoint) return [array_ops.squeeze(x, axis=-1) for x in solution_mat] - rhs = ops.convert_to_tensor(rhs, name="rhs") + rhs = ops.convert_to_tensor_v2_with_dispatch(rhs, name="rhs") self._check_input_dtype(rhs) op_dimension = (self.domain_dimension if adjoint else self.range_dimension) diff --git a/tensorflow/python/ops/linalg/linear_operator_block_lower_triangular.py b/tensorflow/python/ops/linalg/linear_operator_block_lower_triangular.py index b4bf8bdb142..84f2ff15345 100644 --- a/tensorflow/python/ops/linalg/linear_operator_block_lower_triangular.py +++ b/tensorflow/python/ops/linalg/linear_operator_block_lower_triangular.py @@ -366,7 +366,7 @@ class LinearOperatorBlockLowerTriangular(linear_operator.LinearOperator): def _shape_tensor(self): # Avoid messy broadcasting if possible. if self.shape.is_fully_defined(): - return ops.convert_to_tensor( + return ops.convert_to_tensor_v2_with_dispatch( self.shape.as_list(), dtype=dtypes.int32, name="shape") domain_dimension = sum(self._block_domain_dimension_tensors()) @@ -433,12 +433,12 @@ class LinearOperatorBlockLowerTriangular(linear_operator.LinearOperator): if linear_operator_util.arg_is_blockwise(block_dimensions, x, arg_dim): for i, block in enumerate(x): if not isinstance(block, linear_operator.LinearOperator): - block = ops.convert_to_tensor(block) + block = ops.convert_to_tensor_v2_with_dispatch(block) self._check_input_dtype(block) block_dimensions[i].assert_is_compatible_with(block.shape[arg_dim]) x[i] = block else: - x = ops.convert_to_tensor(x, name="x") + x = ops.convert_to_tensor_v2_with_dispatch(x, name="x") self._check_input_dtype(x) op_dimension = (self.range_dimension if adjoint else self.domain_dimension) @@ -543,7 +543,7 @@ class LinearOperatorBlockLowerTriangular(linear_operator.LinearOperator): if linear_operator_util.arg_is_blockwise(block_dimensions, x, -1): for i, block in enumerate(x): if not isinstance(block, linear_operator.LinearOperator): - block = ops.convert_to_tensor(block) + block = ops.convert_to_tensor_v2_with_dispatch(block) self._check_input_dtype(block) block_dimensions[i].assert_is_compatible_with(block.shape[-1]) x[i] = block @@ -551,7 +551,7 @@ class LinearOperatorBlockLowerTriangular(linear_operator.LinearOperator): y_mat = self.matmul(x_mat, adjoint=adjoint) return [array_ops.squeeze(y, axis=-1) for y in y_mat] - x = ops.convert_to_tensor(x, name="x") + x = ops.convert_to_tensor_v2_with_dispatch(x, name="x") self._check_input_dtype(x) op_dimension = (self.range_dimension if adjoint else self.domain_dimension) @@ -674,7 +674,7 @@ class LinearOperatorBlockLowerTriangular(linear_operator.LinearOperator): if blockwise_arg: for i, block in enumerate(rhs): if not isinstance(block, linear_operator.LinearOperator): - block = ops.convert_to_tensor(block) + block = ops.convert_to_tensor_v2_with_dispatch(block) self._check_input_dtype(block) block_dimensions[i].assert_is_compatible_with(block.shape[arg_dim]) rhs[i] = block @@ -684,7 +684,7 @@ class LinearOperatorBlockLowerTriangular(linear_operator.LinearOperator): split_rhs = rhs else: - rhs = ops.convert_to_tensor(rhs, name="rhs") + rhs = ops.convert_to_tensor_v2_with_dispatch(rhs, name="rhs") self._check_input_dtype(rhs) op_dimension = (self.domain_dimension if adjoint else self.range_dimension) @@ -795,14 +795,14 @@ class LinearOperatorBlockLowerTriangular(linear_operator.LinearOperator): if linear_operator_util.arg_is_blockwise(block_dimensions, rhs, -1): for i, block in enumerate(rhs): if not isinstance(block, linear_operator.LinearOperator): - block = ops.convert_to_tensor(block) + block = ops.convert_to_tensor_v2_with_dispatch(block) self._check_input_dtype(block) block_dimensions[i].assert_is_compatible_with(block.shape[-1]) rhs[i] = block rhs_mat = [array_ops.expand_dims(block, axis=-1) for block in rhs] solution_mat = self.solve(rhs_mat, adjoint=adjoint) return [array_ops.squeeze(x, axis=-1) for x in solution_mat] - rhs = ops.convert_to_tensor(rhs, name="rhs") + rhs = ops.convert_to_tensor_v2_with_dispatch(rhs, name="rhs") self._check_input_dtype(rhs) op_dimension = (self.domain_dimension if adjoint else self.range_dimension) diff --git a/tensorflow/python/ops/linalg/linear_operator_circulant.py b/tensorflow/python/ops/linalg/linear_operator_circulant.py index ace276900fc..d4b671c53bd 100644 --- a/tensorflow/python/ops/linalg/linear_operator_circulant.py +++ b/tensorflow/python/ops/linalg/linear_operator_circulant.py @@ -378,7 +378,7 @@ class _BaseLinearOperatorCirculant(linear_operator.LinearOperator): def _broadcast_batch_dims(self, x, spectrum): """Broadcast batch dims of batch matrix `x` and spectrum.""" - spectrum = ops.convert_to_tensor(spectrum, name="spectrum") + spectrum = ops.convert_to_tensor_v2_with_dispatch(spectrum, name="spectrum") # spectrum.shape = batch_shape + block_shape # First make spectrum a batch matrix with # spectrum.shape = batch_shape + [prod(block_shape), 1] @@ -755,7 +755,7 @@ class LinearOperatorCirculant(_BaseLinearOperatorCirculant): name=name) def _eigvals(self): - return ops.convert_to_tensor(self.spectrum) + return ops.convert_to_tensor_v2_with_dispatch(self.spectrum) @tf_export("linalg.LinearOperatorCirculant2D") diff --git a/tensorflow/python/ops/linalg/linear_operator_diag.py b/tensorflow/python/ops/linalg/linear_operator_diag.py index d51d6b81c5d..b5e81b267ce 100644 --- a/tensorflow/python/ops/linalg/linear_operator_diag.py +++ b/tensorflow/python/ops/linalg/linear_operator_diag.py @@ -251,7 +251,7 @@ class LinearOperatorDiag(linear_operator.LinearOperator): return array_ops.matrix_set_diag(x, new_diag) def _eigvals(self): - return ops.convert_to_tensor(self.diag) + return ops.convert_to_tensor_v2_with_dispatch(self.diag) def _cond(self): abs_diag = math_ops.abs(self.diag) diff --git a/tensorflow/python/ops/linalg/linear_operator_full_matrix.py b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py index 8d92d1accaa..b10822589d5 100644 --- a/tensorflow/python/ops/linalg/linear_operator_full_matrix.py +++ b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py @@ -160,7 +160,7 @@ class LinearOperatorFullMatrix(linear_operator.LinearOperator): dtypes.complex128, ] - matrix = ops.convert_to_tensor(matrix, name="matrix") + matrix = ops.convert_to_tensor_v2_with_dispatch(matrix, name="matrix") dtype = matrix.dtype if dtype not in allowed_dtypes: diff --git a/tensorflow/python/ops/linalg/linear_operator_householder.py b/tensorflow/python/ops/linalg/linear_operator_householder.py index 142d48c5331..265c862ea03 100644 --- a/tensorflow/python/ops/linalg/linear_operator_householder.py +++ b/tensorflow/python/ops/linalg/linear_operator_householder.py @@ -198,7 +198,8 @@ class LinearOperatorHouseholder(linear_operator.LinearOperator): # Note that because this is a reflection, it lies in O(n) (for real vector # spaces) or U(n) (for complex vector spaces), and thus is its own adjoint. - reflection_axis = ops.convert_to_tensor(self.reflection_axis) + reflection_axis = ops.convert_to_tensor_v2_with_dispatch( + self.reflection_axis) x = linalg.adjoint(x) if adjoint_arg else x normalized_axis = reflection_axis / linalg.norm( reflection_axis, axis=-1, keepdims=True) @@ -229,7 +230,8 @@ class LinearOperatorHouseholder(linear_operator.LinearOperator): return self._matmul(rhs, adjoint, adjoint_arg) def _to_dense(self): - reflection_axis = ops.convert_to_tensor(self.reflection_axis) + reflection_axis = ops.convert_to_tensor_v2_with_dispatch( + self.reflection_axis) normalized_axis = reflection_axis / linalg.norm( reflection_axis, axis=-1, keepdims=True) mat = normalized_axis[..., array_ops.newaxis] @@ -238,7 +240,8 @@ class LinearOperatorHouseholder(linear_operator.LinearOperator): matrix, 1. + array_ops.matrix_diag_part(matrix)) def _diag_part(self): - reflection_axis = ops.convert_to_tensor(self.reflection_axis) + reflection_axis = ops.convert_to_tensor_v2_with_dispatch( + self.reflection_axis) normalized_axis = reflection_axis / linalg.norm( reflection_axis, axis=-1, keepdims=True) return 1. - 2 * normalized_axis * math_ops.conj(normalized_axis) diff --git a/tensorflow/python/ops/linalg/linear_operator_identity.py b/tensorflow/python/ops/linalg/linear_operator_identity.py index 8226e74bacd..a0f7ead42d6 100644 --- a/tensorflow/python/ops/linalg/linear_operator_identity.py +++ b/tensorflow/python/ops/linalg/linear_operator_identity.py @@ -394,7 +394,7 @@ class LinearOperatorIdentity(BaseLinearOperatorIdentity): A `Tensor` with broadcast shape and same `dtype` as `self`. """ with self._name_scope(name): - mat = ops.convert_to_tensor(mat, name="mat") + mat = ops.convert_to_tensor_v2_with_dispatch(mat, name="mat") mat_diag = array_ops.matrix_diag_part(mat) new_diag = 1 + mat_diag return array_ops.matrix_set_diag(mat, new_diag) @@ -720,7 +720,7 @@ class LinearOperatorScaledIdentity(BaseLinearOperatorIdentity): multiplier_vector = array_ops.expand_dims(self.multiplier, -1) # Shape [C1,...,Cc, M, M] - mat = ops.convert_to_tensor(mat, name="mat") + mat = ops.convert_to_tensor_v2_with_dispatch(mat, name="mat") # Shape [C1,...,Cc, M] mat_diag = array_ops.matrix_diag_part(mat) diff --git a/tensorflow/python/ops/linalg/linear_operator_permutation.py b/tensorflow/python/ops/linalg/linear_operator_permutation.py index 3a44cd5ef1b..9cc8e158a21 100644 --- a/tensorflow/python/ops/linalg/linear_operator_permutation.py +++ b/tensorflow/python/ops/linalg/linear_operator_permutation.py @@ -197,7 +197,7 @@ class LinearOperatorPermutation(linear_operator.LinearOperator): return array_ops.shape(perm)[-1] def _matmul(self, x, adjoint=False, adjoint_arg=False): - perm = ops.convert_to_tensor(self.perm) + perm = ops.convert_to_tensor_v2_with_dispatch(self.perm) if adjoint and not self.is_self_adjoint: # TODO(srvasude): invert_permutation doesn't work on batches so we use # argsort. @@ -232,13 +232,13 @@ class LinearOperatorPermutation(linear_operator.LinearOperator): return self._matmul(rhs, adjoint=(not adjoint), adjoint_arg=adjoint_arg) def _to_dense(self): - perm = ops.convert_to_tensor(self.perm) + perm = ops.convert_to_tensor_v2_with_dispatch(self.perm) return math_ops.cast(math_ops.equal( math_ops.range(0, self._domain_dimension_tensor(perm)), perm[..., array_ops.newaxis]), self.dtype) def _diag_part(self): - perm = ops.convert_to_tensor(self.perm) + perm = ops.convert_to_tensor_v2_with_dispatch(self.perm) return math_ops.cast(math_ops.equal( math_ops.range(0, self._domain_dimension_tensor(perm)), perm), self.dtype) diff --git a/tensorflow/python/ops/linalg/linear_operator_toeplitz.py b/tensorflow/python/ops/linalg/linear_operator_toeplitz.py index 71fff44da44..2d61a536e29 100644 --- a/tensorflow/python/ops/linalg/linear_operator_toeplitz.py +++ b/tensorflow/python/ops/linalg/linear_operator_toeplitz.py @@ -209,8 +209,8 @@ class LinearOperatorToeplitz(linear_operator.LinearOperator): # for more details. x = linalg.adjoint(x) if adjoint_arg else x expanded_x = array_ops.concat([x, array_ops.zeros_like(x)], axis=-2) - col = ops.convert_to_tensor(self.col) - row = ops.convert_to_tensor(self.row) + col = ops.convert_to_tensor_v2_with_dispatch(self.col) + row = ops.convert_to_tensor_v2_with_dispatch(self.row) circulant_col = array_ops.concat( [col, array_ops.zeros_like(col[..., 0:1]), @@ -236,8 +236,8 @@ class LinearOperatorToeplitz(linear_operator.LinearOperator): [self.domain_dimension_tensor()], self.dtype) def _to_dense(self): - row = ops.convert_to_tensor(self.row) - col = ops.convert_to_tensor(self.col) + row = ops.convert_to_tensor_v2_with_dispatch(self.row) + col = ops.convert_to_tensor_v2_with_dispatch(self.col) total_shape = array_ops.broadcast_dynamic_shape( array_ops.shape(row), array_ops.shape(col)) n = array_ops.shape(row)[-1] diff --git a/tensorflow/python/ops/linalg/linear_operator_tridiag.py b/tensorflow/python/ops/linalg/linear_operator_tridiag.py index 422747848c0..2ba310f75bf 100644 --- a/tensorflow/python/ops/linalg/linear_operator_tridiag.py +++ b/tensorflow/python/ops/linalg/linear_operator_tridiag.py @@ -246,7 +246,7 @@ class LinearOperatorTridiag(linear_operator.LinearOperator): self.diagonals, linalg.adjoint(self.diagonals), message='Matrix was not equal to its adjoint.')] elif self.diagonals_format == _COMPACT: - diagonals = ops.convert_to_tensor(self.diagonals) + diagonals = ops.convert_to_tensor_v2_with_dispatch(self.diagonals) asserts += [linear_operator_util.assert_zero_imag_part( diagonals[..., 1, :], message=diag_message)] # Roll the subdiagonal so the shifted argument is at the end. @@ -353,7 +353,9 @@ class LinearOperatorTridiag(linear_operator.LinearOperator): align='LEFT_RIGHT', padding_value=0.) - diagonals = [ops.convert_to_tensor(d) for d in self.diagonals] + diagonals = [ + ops.convert_to_tensor_v2_with_dispatch(d) for d in self.diagonals + ] diagonals = array_ops.stack(diagonals, axis=-2) return gen_array_ops.matrix_diag_v3( diff --git a/tensorflow/python/ops/linalg/linear_operator_util.py b/tensorflow/python/ops/linalg/linear_operator_util.py index 948f2f86a53..096ad3fb4bb 100644 --- a/tensorflow/python/ops/linalg/linear_operator_util.py +++ b/tensorflow/python/ops/linalg/linear_operator_util.py @@ -114,7 +114,7 @@ def convert_nonref_to_tensor(value, dtype=None, dtype_hint=None, name=None): raise TypeError('Mutable type must be of dtype "{}" but is "{}".'.format( dtype_name(dtype_base), dtype_name(value_dtype_base))) return value - return ops.convert_to_tensor( + return ops.convert_to_tensor_v2_with_dispatch( value, dtype=dtype, dtype_hint=dtype_hint, name=name) @@ -189,10 +189,10 @@ def assert_no_entries_with_modulus_zero( An `Op` that asserts `x` has no entries with modulus zero. """ with ops.name_scope(name, values=[x]): - x = ops.convert_to_tensor(x, name="x") + x = ops.convert_to_tensor_v2_with_dispatch(x, name="x") dtype = x.dtype.base_dtype should_be_nonzero = math_ops.abs(x) - zero = ops.convert_to_tensor(0, dtype=dtype.real_dtype) + zero = ops.convert_to_tensor_v2_with_dispatch(0, dtype=dtype.real_dtype) return check_ops.assert_less(zero, should_be_nonzero, message=message) @@ -208,13 +208,13 @@ def assert_zero_imag_part(x, message=None, name="assert_zero_imag_part"): An `Op` that asserts `x` has no entries with modulus zero. """ with ops.name_scope(name, values=[x]): - x = ops.convert_to_tensor(x, name="x") + x = ops.convert_to_tensor_v2_with_dispatch(x, name="x") dtype = x.dtype.base_dtype if dtype.is_floating: return control_flow_ops.no_op() - zero = ops.convert_to_tensor(0, dtype=dtype.real_dtype) + zero = ops.convert_to_tensor_v2_with_dispatch(0, dtype=dtype.real_dtype) return check_ops.assert_equal(zero, math_ops.imag(x), message=message) @@ -261,7 +261,7 @@ def shape_tensor(shape, name=None): dtype = dtypes.int32 else: dtype = None - return ops.convert_to_tensor(shape, dtype=dtype, name=name) + return ops.convert_to_tensor_v2_with_dispatch(shape, dtype=dtype, name=name) ################################################################################ @@ -323,7 +323,7 @@ def broadcast_matrix_batch_dims(batch_matrices, name=None): batch_matrices = list(batch_matrices) for i, mat in enumerate(batch_matrices): - batch_matrices[i] = ops.convert_to_tensor(mat) + batch_matrices[i] = ops.convert_to_tensor_v2_with_dispatch(mat) assert_is_batch_matrix(batch_matrices[i]) if len(batch_matrices) < 2: @@ -366,8 +366,9 @@ def broadcast_matrix_batch_dims(batch_matrices, name=None): def matrix_solve_with_broadcast(matrix, rhs, adjoint=False, name=None): """Solve systems of linear equations.""" with ops.name_scope(name, "MatrixSolveWithBroadcast", [matrix, rhs]): - matrix = ops.convert_to_tensor(matrix, name="matrix") - rhs = ops.convert_to_tensor(rhs, name="rhs", dtype=matrix.dtype) + matrix = ops.convert_to_tensor_v2_with_dispatch(matrix, name="matrix") + rhs = ops.convert_to_tensor_v2_with_dispatch( + rhs, name="rhs", dtype=matrix.dtype) # If either matrix/rhs has extra dims, we can reshape to get rid of them. matrix, rhs, reshape_inv, still_need_to_transpose = _reshape_for_efficiency( @@ -526,7 +527,8 @@ def arg_is_blockwise(block_dimensions, arg, arg_split_dim): if not any(nest.is_nested(x) for x in arg): return True else: - arg_dims = [ops.convert_to_tensor(x).shape[arg_split_dim] for x in arg] + arg_dims = [ops.convert_to_tensor_v2_with_dispatch( + x).shape[arg_split_dim] for x in arg] self_dims = [dim.value for dim in block_dimensions] # If none of the operator dimensions are known, interpret the input as diff --git a/tensorflow/python/util/dispatch_test.py b/tensorflow/python/util/dispatch_test.py index 2b3946ce9f7..f06f2fda7e3 100644 --- a/tensorflow/python/util/dispatch_test.py +++ b/tensorflow/python/util/dispatch_test.py @@ -18,11 +18,13 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_math_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops.linalg import linear_operator_diag from tensorflow.python.ops.proto_ops import decode_proto from tensorflow.python.platform import googletest from tensorflow.python.platform import test @@ -60,6 +62,8 @@ class TensorTracer(object): self.name = name self.args = args self.kwargs = kwargs + self.shape = array_ops.ones(shape=(4, 4)).shape + self.dtype = dtypes.float32 def __repr__(self): if self.args is None and self.kwargs is None: @@ -70,6 +74,10 @@ class TensorTracer(object): ["{}={}".format(name, x) for (name, x) in self.kwargs.items()]) return "{}({})".format(self.name, ", ".join(args)) + @property + def is_tensor_like(self): + return True + @classmethod def _overload_all_operators(cls): # pylint: disable=invalid-name """Register overloads for all operators.""" @@ -282,5 +290,42 @@ class DispatchTest(test_util.TensorFlowTestCase): # Clean up. dispatch._GLOBAL_DISPATCHERS = original_global_dispatchers + def testGlobalDispatcherLinearOperators(self): + original_global_dispatchers = dispatch._GLOBAL_DISPATCHERS + try: + TensorTracerOpDispatcher().register() + + x = TensorTracer("x") + + # To grab the eigenvalues the diag operator just calls convert_to_tensor + # (twice) in this case. + trace = linear_operator_diag.LinearOperatorDiag(x).eigvals() + self.assertEqual( + str(trace), + "convert_to_tensor(convert_to_tensor(x, dtype=None, dtype_hint=None, " + "name=diag))") + + # The diagonal tensor addition gets traced even though the linear_operator + # API only uses dispatchable ops instead of directly exposing dispatching. + trace = linear_operator_diag.LinearOperatorDiag(x).add_to_tensor(x) + self.assertIn( + "linalg.set_diag(convert_to_tensor(x, name=x), __operators__.add(" + "convert_to_tensor(x, dtype=None, dtype_hint=None, name=diag), " + "linalg.diag_part(convert_to_tensor(x, name=x)), " + "name=", + str(trace)) + + # The dispatch-supporting ops the non-singular check calls out to + # get traced. + trace = linear_operator_diag.LinearOperatorDiag(x).assert_non_singular() + self.assertIn("debugging.assert_less", str(trace)) + self.assertIn( + "message=Singular operator: Diagonal contained zero values.", + str(trace)) + + finally: + # Clean up. + dispatch._GLOBAL_DISPATCHERS = original_global_dispatchers + if __name__ == "__main__": googletest.main() From b4b0d9da40feb4c7b056ead6b3a69b280adc7943 Mon Sep 17 00:00:00 2001 From: Zhenyu Tan Date: Tue, 4 Aug 2020 10:22:57 -0700 Subject: [PATCH 0372/1017] fix for lint in feature column test. PiperOrigin-RevId: 324837891 Change-Id: Ia86ecb5dd962f7aa70f676644e9f6b9b24707e46 --- .../python/feature_column/feature_column_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py index c8e24e46c2f..e598848282f 100644 --- a/tensorflow/python/feature_column/feature_column_test.py +++ b/tensorflow/python/feature_column/feature_column_test.py @@ -5865,11 +5865,12 @@ class SharedEmbeddingColumnTest(test.TestCase, parameterized.TestCase): partitioner = partitioned_variables.fixed_size_partitioner(2, axis=0) with variable_scope.variable_scope('vars', partitioner=partitioner): - embedding_column_a, embedding_column_b = fc_new.shared_embedding_columns( - [categorical_column_a, categorical_column_b], - dimension=embedding_dimension, - initializer=_initializer, - use_safe_embedding_lookup=use_safe_embedding_lookup) + embedding_column_a, embedding_column_b = ( + fc_new.shared_embedding_columns( + [categorical_column_a, categorical_column_b], + dimension=embedding_dimension, + initializer=_initializer, + use_safe_embedding_lookup=use_safe_embedding_lookup)) # Provide sparse input and get dense result. embedding_lookup_a = embedding_column_a._get_dense_tensor( _LazyBuilder(input_features)) From be1cfeb96e23a8e24be6b141a15e39bf78116c10 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Tue, 4 Aug 2020 10:45:09 -0700 Subject: [PATCH 0373/1017] Add build helper to mlir-hlo PiperOrigin-RevId: 324843181 Change-Id: I3ab290395883173c590651fbf3d2bfb2ae7cb9fc --- .../mlir/hlo/build_tools/build_mlir.sh | 52 +++++++++++++++++++ .../mlir/hlo/build_tools/llvm_version.txt | 2 + 2 files changed, 54 insertions(+) create mode 100755 tensorflow/compiler/mlir/hlo/build_tools/build_mlir.sh create mode 100644 tensorflow/compiler/mlir/hlo/build_tools/llvm_version.txt diff --git a/tensorflow/compiler/mlir/hlo/build_tools/build_mlir.sh b/tensorflow/compiler/mlir/hlo/build_tools/build_mlir.sh new file mode 100755 index 00000000000..5ccefb9416f --- /dev/null +++ b/tensorflow/compiler/mlir/hlo/build_tools/build_mlir.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +if [[ $# -ne 2 ]] ; then + echo "Usage: $0 " + exit 1 +fi + +# LLVM source +LLVM_SRC_DIR="$1" +build_dir="$2" + +if ! [ -f "$LLVM_SRC_DIR/llvm/CMakeLists.txt" ]; then + echo "Expected the path to LLVM to be set correctly (got '$LLVM_SRC_DIR'): can't find CMakeLists.txt" + exit 1 +fi +echo "Using LLVM source dir: $LLVM_SRC_DIR" + +# Setup directories. +echo "Building MLIR in $build_dir" +mkdir -p "$build_dir" + +echo "Beginning build (commands will echo)" +set -x + +cmake -GNinja \ + "-H$LLVM_SRC_DIR/llvm" \ + "-B$build_dir" \ + -DLLVM_INSTALL_UTILS=ON \ + -DLLVM_ENABLE_LLD=ON \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_TARGETS_TO_BUILD="X86;NVPTX;AMDGPU" \ + -DLLVM_INCLUDE_TOOLS=ON \ + -DLLVM_BUILD_TOOLS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DLLVM_ENABLE_ASSERTIONS=On + +cmake --build "$build_dir" --target all --target mlir-cpu-runner diff --git a/tensorflow/compiler/mlir/hlo/build_tools/llvm_version.txt b/tensorflow/compiler/mlir/hlo/build_tools/llvm_version.txt new file mode 100644 index 00000000000..0d5446142ec --- /dev/null +++ b/tensorflow/compiler/mlir/hlo/build_tools/llvm_version.txt @@ -0,0 +1,2 @@ + + From e1da2cba6db064c83b827a775d92804e7783f99e Mon Sep 17 00:00:00 2001 From: Stephan Herhut Date: Tue, 4 Aug 2020 10:47:39 -0700 Subject: [PATCH 0374/1017] Make kernel-gen-opt its own tool with a cc file. PiperOrigin-RevId: 324843789 Change-Id: I2ce05b73ea3ef6272ab131f7ca11d9cd301f2d59 --- .../compiler/mlir/tools/kernel_gen/BUILD | 8 +- .../tools/kernel-gen-opt/kernel-gen-opt.cc | 122 ++++++++++++++++++ .../mlir/tools/kernel_gen/transforms/BUILD | 55 +++----- .../transforms/embed_tf_framework_pass.cc | 2 +- .../mlir/tools/kernel_gen/transforms/passes.h | 13 +- .../kernel_gen/transforms/register_passes.cc | 27 ---- ...iptors.cc => shape_to_descriptors_pass.cc} | 15 +-- .../tf_framework_legalize_to_llvm_pass.cc | 2 +- 8 files changed, 165 insertions(+), 79 deletions(-) create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tools/kernel-gen-opt/kernel-gen-opt.cc delete mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc rename tensorflow/compiler/mlir/tools/kernel_gen/transforms/{shape_to_descriptors.cc => shape_to_descriptors_pass.cc} (88%) diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/BUILD index b40d6cb3abf..066ca221d5d 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/BUILD @@ -59,12 +59,18 @@ tf_cc_binary( tf_cc_binary( name = "kernel-gen-opt", + srcs = ["tools/kernel-gen-opt/kernel-gen-opt.cc"], visibility = ["//tensorflow/compiler/mlir/tools/kernel_gen/tests:__pkg__"], deps = [ + "//tensorflow/compiler/mlir/hlo:hlo_dialect_registration", "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_dialect_registration", "//tensorflow/compiler/mlir/tools/kernel_gen/transforms:passes", - "@llvm-project//mlir:AllPassesAndDialects", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:AllPassesAndDialectsNoRegistration", + "@llvm-project//mlir:IR", "@llvm-project//mlir:MlirOptLib", "@llvm-project//mlir:MlirOptMain", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:Support", ], ) diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tools/kernel-gen-opt/kernel-gen-opt.cc b/tensorflow/compiler/mlir/tools/kernel_gen/tools/kernel-gen-opt/kernel-gen-opt.cc new file mode 100644 index 00000000000..c1af35617b1 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/tools/kernel-gen-opt/kernel-gen-opt.cc @@ -0,0 +1,122 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/InitLLVM.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/ToolOutputFile.h" +#include "mlir/IR/AsmState.h" // from @llvm-project +#include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/InitAllDialects.h" // from @llvm-project +#include "mlir/InitAllPasses.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/FileUtilities.h" // from @llvm-project +#include "mlir/Support/MlirOptMain.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/register.h" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" + +// NOLINTNEXTLINE +static llvm::cl::opt inputFilename(llvm::cl::Positional, + llvm::cl::desc(""), + llvm::cl::init("-")); + +// NOLINTNEXTLINE +static llvm::cl::opt outputFilename( + "o", llvm::cl::desc("Output filename"), llvm::cl::value_desc("filename"), + llvm::cl::init("-")); + +// NOLINTNEXTLINE +static llvm::cl::opt splitInputFile( + "split-input-file", + llvm::cl::desc("Split the input file into pieces and process each " + "chunk independently"), + llvm::cl::init(false)); + +// NOLINTNEXTLINE +static llvm::cl::opt verifyDiagnostics( + "verify-diagnostics", + llvm::cl::desc("Check that emitted diagnostics match " + "expected-* lines on the corresponding line"), + llvm::cl::init(false)); + +// NOLINTNEXTLINE +static llvm::cl::opt verifyPasses( + "verify-each", + llvm::cl::desc("Run the verifier after each transformation pass"), + llvm::cl::init(true)); + +// NOLINTNEXTLINE +static llvm::cl::opt allowUnregisteredDialects( + "allow-unregistered-dialect", + llvm::cl::desc("Allow operation with no registered dialects"), + llvm::cl::init(false)); + +// NOLINTNEXTLINE +static llvm::cl::opt showDialects( + "show-dialects", llvm::cl::desc("Print the list of registered dialects"), + llvm::cl::init(false)); + +int main(int argc, char **argv) { + mlir::registerAllDialects(); + mlir::registerAllPasses(); + + mlir::mhlo::registerAllDialects(); + mlir::kernel_gen::registerKernelGenPasses(); + + llvm::InitLLVM y(argc, argv); + + // Register any pass manager command line options. + mlir::registerAsmPrinterCLOptions(); + mlir::registerPassManagerCLOptions(); + mlir::PassPipelineCLParser passPipeline("", "Compiler passes to run"); + + // Parse pass names in main to ensure static initialization completed. + llvm::cl::ParseCommandLineOptions(argc, argv, + "MLIR modular optimizer driver\n"); + + if (showDialects) { + mlir::MLIRContext context; + llvm::outs() << "Registered Dialects:\n"; + for (mlir::Dialect *dialect : context.getRegisteredDialects()) { + llvm::outs() << dialect->getNamespace() << "\n"; + } + return 0; + } + + // Set up the input file. + std::string errorMessage; + auto file = mlir::openInputFile(inputFilename, &errorMessage); + if (!file) { + llvm::errs() << errorMessage << "\n"; + return 1; + } + + auto output = mlir::openOutputFile(outputFilename, &errorMessage); + if (!output) { + llvm::errs() << errorMessage << "\n"; + exit(1); + } + + if (failed(MlirOptMain(output->os(), std::move(file), passPipeline, + splitInputFile, verifyDiagnostics, verifyPasses, + allowUnregisteredDialects))) { + return 1; + } + // Keep the output file if the invocation of MlirOptMain was successful. + output->keep(); + return 0; +} diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD index c0808ae08c4..0d346da9956 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD @@ -35,15 +35,31 @@ cc_library( ], ) +gentbl( + name = "tf_framework_passes_inc_gen", + tbl_outs = [("-gen-pass-decls -name KernelGen", "kernel_gen_passes.h.inc")], + tblgen = "@llvm-project//mlir:mlir-tblgen", + td_file = "passes.td", + td_srcs = ["@llvm-project//mlir:PassBaseTdFiles"], +) + cc_library( - name = "shape_to_descriptors", - srcs = ["shape_to_descriptors.cc"], - hdrs = [ - "passes.h", + name = "passes", + srcs = [ + "embed_tf_framework_pass.cc", + "shape_to_descriptors_pass.cc", + "tf_framework_legalize_to_llvm_pass.cc", ], + hdrs = ["passes.h"], deps = [ + ":embed_tf_framework", + ":tf_framework_legalize_to_llvm", + ":tf_framework_passes_inc_gen", + "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_ops", "@llvm-project//llvm:Support", "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:LLVMTransforms", "@llvm-project//mlir:Pass", "@llvm-project//mlir:SCFDialect", "@llvm-project//mlir:Shape", @@ -55,34 +71,3 @@ cc_library( "@llvm-project//mlir:Transforms", ], ) - -gentbl( - name = "tf_framework_passes_inc_gen", - tbl_outs = [("-gen-pass-decls -name TFFramework", "tf_framework_passes.h.inc")], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "passes.td", - td_srcs = ["@llvm-project//mlir:PassBaseTdFiles"], -) - -cc_library( - name = "passes", - srcs = [ - "embed_tf_framework_pass.cc", - "register_passes.cc", - "tf_framework_legalize_to_llvm_pass.cc", - ], - hdrs = ["passes.h"], - deps = [ - ":embed_tf_framework", - ":shape_to_descriptors", - ":tf_framework_legalize_to_llvm", - ":tf_framework_passes_inc_gen", - "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_ops", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:LLVMTransforms", - "@llvm-project//mlir:Pass", - "@llvm-project//mlir:StandardOps", - "@llvm-project//mlir:Transforms", - ], - alwayslink = 1, -) diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework_pass.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework_pass.cc index 615c596e353..a0cfcae65d1 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework_pass.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/embed_tf_framework_pass.cc @@ -26,7 +26,7 @@ namespace tf_framework { namespace { #define GEN_PASS_CLASSES -#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_passes.h.inc" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/kernel_gen_passes.h.inc" static constexpr StringRef kTFEntry = "tf_entry"; diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h index 5e240b8d01c..13f367c9fe4 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h @@ -18,13 +18,10 @@ limitations under the License. #include +#include "mlir/IR/Module.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project + namespace mlir { - -class ModuleOp; -template -class OperationPass; -class Pass; - namespace kernel_gen { namespace tf_framework { @@ -47,6 +44,10 @@ namespace transforms { std::unique_ptr CreateShapeToDescriptorsPass(); } // namespace transforms + +#define GEN_PASS_REGISTRATION +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/kernel_gen_passes.h.inc" + } // namespace kernel_gen } // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc deleted file mode 100644 index 3a42d03355c..00000000000 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/register_passes.cc +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "mlir/Pass/Pass.h" // from @llvm-project -#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" - -namespace mlir { -namespace kernel_gen { -#define GEN_PASS_REGISTRATION -#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_passes.h.inc" - -bool register_all_passes = ([] { registerTFFrameworkPasses(); }(), true); - -} // namespace kernel_gen -} // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors_pass.cc similarity index 88% rename from tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors.cc rename to tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors_pass.cc index 32c2f9641b5..9c1b434b9b2 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors_pass.cc @@ -24,21 +24,20 @@ limitations under the License. #include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project #include "mlir/IR/MLIRContext.h" // from @llvm-project #include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Transforms/DialectConversion.h" // from @llvm-project -#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" namespace mlir { namespace kernel_gen { namespace transforms { - namespace { -struct ShapeToDescriptorsPass - : public PassWrapper> { - public: - ShapeToDescriptorsPass() = default; +#define GEN_PASS_CLASSES +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/kernel_gen_passes.h.inc" +struct ShapeToDescriptorsPass + : public ShapeToDescriptorsPassBase { + public: void runOnOperation() override { MLIRContext &ctx = getContext(); @@ -63,7 +62,7 @@ struct ShapeToDescriptorsPass } // namespace -std::unique_ptr CreateShapeToDescriptorsPass() { +std::unique_ptr > CreateShapeToDescriptorsPass() { return std::make_unique(); } diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc index 8439e1617e0..916eedb55de 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc @@ -28,7 +28,7 @@ namespace tf_framework { namespace { #define GEN_PASS_CLASSES -#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_passes.h.inc" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/kernel_gen_passes.h.inc" class TestTFFrameworkToLLVMPass : public TestTFFrameworkLegalizeToLLVMPassBase { From f1f65d45f70af76d62a019838a8d939969a74428 Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Tue, 4 Aug 2020 10:48:46 -0700 Subject: [PATCH 0375/1017] [tf.data service] Support restoring state from the journal on dispatcher startup. PiperOrigin-RevId: 324844108 Change-Id: I805c59ee6cf40a08fa5348c4232f92c0a6de8d3c --- .../core/data/service/dispatcher_impl.cc | 46 ++++++++++++++++--- .../core/data/service/dispatcher_impl.h | 11 ++++- .../core/data/service/dispatcher_state.cc | 3 +- .../core/data/service/dispatcher_state.h | 2 +- .../data/service/dispatcher_state_test.cc | 15 ++---- .../core/data/service/grpc_dispatcher_impl.cc | 13 +++--- .../core/data/service/grpc_dispatcher_impl.h | 2 + tensorflow/core/data/service/journal.cc | 4 -- tensorflow/core/data/service/journal.h | 12 ----- tensorflow/core/data/service/server_lib.cc | 4 ++ tensorflow/core/data/service/server_lib.h | 2 +- 11 files changed, 70 insertions(+), 44 deletions(-) diff --git a/tensorflow/core/data/service/dispatcher_impl.cc b/tensorflow/core/data/service/dispatcher_impl.cc index 4bc4d409fd7..4a3764ecea3 100644 --- a/tensorflow/core/data/service/dispatcher_impl.cc +++ b/tensorflow/core/data/service/dispatcher_impl.cc @@ -49,6 +49,10 @@ using Dataset = DispatcherState::Dataset; using NamedJobKey = DispatcherState::NamedJobKey; using Job = DispatcherState::Job; +std::string JournalDir(StringPiece work_dir) { + return io::JoinPath(work_dir, kJournalDir); +} + Status CreateWorkerStub(const std::string& address, const std::string& protocol, std::unique_ptr* stub) { ::grpc::ChannelArguments args; @@ -65,15 +69,35 @@ Status CreateWorkerStub(const std::string& address, const std::string& protocol, DataServiceDispatcherImpl::DataServiceDispatcherImpl( const experimental::DispatcherConfig& config) : config_(config) { - if (config_.work_dir().empty()) { - journal_writer_ = absl::make_unique(); - } else { - std::string journal_dir = io::JoinPath(config_.work_dir(), kJournalDir); - journal_writer_ = - absl::make_unique(Env::Default(), journal_dir); + if (!config_.work_dir().empty()) { + journal_writer_ = absl::make_unique( + Env::Default(), JournalDir(config_.work_dir())); } } +Status DataServiceDispatcherImpl::Start() { + if (config_.work_dir().empty()) { + return Status::OK(); + } + mutex_lock l(mu_); + Update update; + bool end_of_journal = false; + FileJournalReader reader(Env::Default(), JournalDir(config_.work_dir())); + Status s = reader.Read(&update, &end_of_journal); + if (errors::IsNotFound(s)) { + LOG(INFO) << "No journal found. Starting dispatcher from new state."; + return Status::OK(); + } + TF_RETURN_IF_ERROR(s); + LOG(INFO) << "Restoring dispatcher state from journal in " + << JournalDir(config_.work_dir()); + while (!end_of_journal) { + TF_RETURN_IF_ERROR(ApplyWithoutJournaling(update)); + TF_RETURN_IF_ERROR(reader.Read(&update, &end_of_journal)); + } + return Status::OK(); +} + Status DataServiceDispatcherImpl::RegisterWorker( const RegisterWorkerRequest* request, RegisterWorkerResponse* response) { VLOG(3) << "Received register worker request"; @@ -407,9 +431,17 @@ Status DataServiceDispatcherImpl::GetWorkers(const GetWorkersRequest* request, return Status::OK(); } +Status DataServiceDispatcherImpl::ApplyWithoutJournaling(const Update& update) + EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return state_.Apply(update); +} + Status DataServiceDispatcherImpl::Apply(const Update& update) EXCLUSIVE_LOCKS_REQUIRED(mu_) { - return state_.Apply(update, journal_writer_.get()); + if (journal_writer_.has_value()) { + TF_RETURN_IF_ERROR(journal_writer_.value()->Write(update)); + } + return state_.Apply(update); } } // namespace data diff --git a/tensorflow/core/data/service/dispatcher_impl.h b/tensorflow/core/data/service/dispatcher_impl.h index 3e8b8dc6fbe..e39f3269d02 100644 --- a/tensorflow/core/data/service/dispatcher_impl.h +++ b/tensorflow/core/data/service/dispatcher_impl.h @@ -47,6 +47,10 @@ class DataServiceDispatcherImpl { explicit DataServiceDispatcherImpl( const experimental::DispatcherConfig& config); + // Starts the dispatcher. If there is a journal, this will read from the + // journal to restore the dispatcher's state. + Status Start(); + // See dispatcher.proto for API documentation. /// Worker-facing API. @@ -126,6 +130,10 @@ class DataServiceDispatcherImpl { EXCLUSIVE_LOCKS_REQUIRED(mu_); // Applies a state update, updating both the journal and the in-memory state. Status Apply(const Update& update) EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Applies a state update, but doesn't update the journal. Only meant to be + // used when recovering state when the dispatcher starts. + Status ApplyWithoutJournaling(const Update& update) + EXCLUSIVE_LOCKS_REQUIRED(mu_); const experimental::DispatcherConfig& config_; @@ -143,7 +151,8 @@ class DataServiceDispatcherImpl { absl::flat_hash_map>> tasks_by_job_ TF_GUARDED_BY(mu_); - std::unique_ptr journal_writer_ TF_GUARDED_BY(mu_); + absl::optional> journal_writer_ + TF_GUARDED_BY(mu_); DispatcherState state_ TF_GUARDED_BY(mu_); TF_DISALLOW_COPY_AND_ASSIGN(DataServiceDispatcherImpl); diff --git a/tensorflow/core/data/service/dispatcher_state.cc b/tensorflow/core/data/service/dispatcher_state.cc index f22672c4363..64be7fbc54e 100644 --- a/tensorflow/core/data/service/dispatcher_state.cc +++ b/tensorflow/core/data/service/dispatcher_state.cc @@ -25,8 +25,7 @@ namespace data { DispatcherState::DispatcherState() {} -Status DispatcherState::Apply(Update update, JournalWriter* journal_writer) { - TF_RETURN_IF_ERROR(journal_writer->Write(update)); +Status DispatcherState::Apply(Update update) { switch (update.update_type_case()) { case Update::kRegisterDataset: RegisterDataset(update.register_dataset()); diff --git a/tensorflow/core/data/service/dispatcher_state.h b/tensorflow/core/data/service/dispatcher_state.h index 1959afa61eb..e54f51ba499 100644 --- a/tensorflow/core/data/service/dispatcher_state.h +++ b/tensorflow/core/data/service/dispatcher_state.h @@ -61,7 +61,7 @@ class DispatcherState { DispatcherState& operator=(const DispatcherState&) = delete; // Applies the given update to the dispatcher's state. - Status Apply(Update update, JournalWriter* journal_writer); + Status Apply(Update update); // A dataset registered with the dispatcher. struct Dataset { diff --git a/tensorflow/core/data/service/dispatcher_state_test.cc b/tensorflow/core/data/service/dispatcher_state_test.cc index e1fd47805a7..02961d5bd1d 100644 --- a/tensorflow/core/data/service/dispatcher_state_test.cc +++ b/tensorflow/core/data/service/dispatcher_state_test.cc @@ -28,31 +28,28 @@ namespace data { namespace { Status RegisterDatasetWithIdAndFingerprint(int64 id, uint64 fingerprint, DispatcherState* state) { - NoopJournalWriter journal_writer; Update update; RegisterDatasetUpdate* register_dataset = update.mutable_register_dataset(); register_dataset->set_dataset_id(id); register_dataset->set_fingerprint(fingerprint); - TF_RETURN_IF_ERROR(state->Apply(update, &journal_writer)); + TF_RETURN_IF_ERROR(state->Apply(update)); return Status::OK(); } Status CreateAnonymousJob(int64 job_id, int64 dataset_id, DispatcherState* state) { - NoopJournalWriter journal_writer; Update update; CreateJobUpdate* create_job = update.mutable_create_job(); create_job->set_job_id(job_id); create_job->set_dataset_id(dataset_id); create_job->set_processing_mode(ProcessingModeDef::PARALLEL_EPOCHS); - TF_RETURN_IF_ERROR(state->Apply(update, &journal_writer)); + TF_RETURN_IF_ERROR(state->Apply(update)); return Status::OK(); } Status CreateNamedJob(int64 job_id, int64 dataset_id, DispatcherState::NamedJobKey named_job_key, DispatcherState* state) { - NoopJournalWriter journal_writer; Update update; CreateJobUpdate* create_job = update.mutable_create_job(); create_job->set_job_id(job_id); @@ -61,16 +58,15 @@ Status CreateNamedJob(int64 job_id, int64 dataset_id, NamedJobKeyDef* key = create_job->mutable_named_job_key(); key->set_name(named_job_key.name); key->set_index(named_job_key.index); - TF_RETURN_IF_ERROR(state->Apply(update, &journal_writer)); + TF_RETURN_IF_ERROR(state->Apply(update)); return Status::OK(); } Status FinishJob(int64 job_id, DispatcherState* state) { - NoopJournalWriter journal_writer; Update update; FinishJobUpdate* finish_job = update.mutable_finish_job(); finish_job->set_job_id(job_id); - TF_RETURN_IF_ERROR(state->Apply(update, &journal_writer)); + TF_RETURN_IF_ERROR(state->Apply(update)); return Status::OK(); } } // namespace @@ -117,10 +113,9 @@ TEST(DispatcherState, NextAvailableDatasetId) { } TEST(DispatcherState, UnknownUpdate) { - NoopJournalWriter journal_writer; DispatcherState state; Update update; - Status s = state.Apply(update, &journal_writer); + Status s = state.Apply(update); EXPECT_EQ(s.code(), error::INTERNAL); } diff --git a/tensorflow/core/data/service/grpc_dispatcher_impl.cc b/tensorflow/core/data/service/grpc_dispatcher_impl.cc index a26164ed48f..f62b487fcdf 100644 --- a/tensorflow/core/data/service/grpc_dispatcher_impl.cc +++ b/tensorflow/core/data/service/grpc_dispatcher_impl.cc @@ -24,7 +24,6 @@ namespace data { using ::grpc::ServerBuilder; using ::grpc::ServerContext; -using ::grpc::Status; GrpcDispatcherImpl::GrpcDispatcherImpl( ServerBuilder* server_builder, const experimental::DispatcherConfig& config) @@ -33,11 +32,13 @@ GrpcDispatcherImpl::GrpcDispatcherImpl( VLOG(1) << "Registered data service dispatcher"; } -#define HANDLER(method) \ - Status GrpcDispatcherImpl::method(ServerContext* context, \ - const method##Request* request, \ - method##Response* response) { \ - return ToGrpcStatus(impl_.method(request, response)); \ +Status GrpcDispatcherImpl::Start() { return impl_.Start(); } + +#define HANDLER(method) \ + grpc::Status GrpcDispatcherImpl::method(ServerContext* context, \ + const method##Request* request, \ + method##Response* response) { \ + return ToGrpcStatus(impl_.method(request, response)); \ } HANDLER(RegisterWorker); HANDLER(WorkerUpdate); diff --git a/tensorflow/core/data/service/grpc_dispatcher_impl.h b/tensorflow/core/data/service/grpc_dispatcher_impl.h index 24bf2d79061..1810c3fb6ac 100644 --- a/tensorflow/core/data/service/grpc_dispatcher_impl.h +++ b/tensorflow/core/data/service/grpc_dispatcher_impl.h @@ -39,6 +39,8 @@ class GrpcDispatcherImpl : public DispatcherService::Service { const experimental::DispatcherConfig& config); ~GrpcDispatcherImpl() override {} + Status Start(); + #define HANDLER(method) \ grpc::Status method(grpc::ServerContext* context, \ const method##Request* request, \ diff --git a/tensorflow/core/data/service/journal.cc b/tensorflow/core/data/service/journal.cc index 6856c69deb3..a9aa43b9758 100644 --- a/tensorflow/core/data/service/journal.cc +++ b/tensorflow/core/data/service/journal.cc @@ -61,10 +61,6 @@ Status FileJournalWriter::Write(Update update) { return Status::OK(); } -NoopJournalWriter::NoopJournalWriter() {} - -Status NoopJournalWriter::Write(Update update) { return Status::OK(); } - FileJournalReader::FileJournalReader(Env* env, StringPiece journal_dir) : env_(env), journal_dir_(journal_dir) {} diff --git a/tensorflow/core/data/service/journal.h b/tensorflow/core/data/service/journal.h index f5b3e26ba18..112c3b614be 100644 --- a/tensorflow/core/data/service/journal.h +++ b/tensorflow/core/data/service/journal.h @@ -58,18 +58,6 @@ class FileJournalWriter : public JournalWriter { std::unique_ptr writer_; }; -// NoopJournalWriter implements the JournalWriter interface, but doesn't -// actually write journal entries anywhere. -class NoopJournalWriter : public JournalWriter { - public: - // Creates a journal writer which does nothing. - explicit NoopJournalWriter(); - NoopJournalWriter(const NoopJournalWriter&) = delete; - NoopJournalWriter& operator=(const NoopJournalWriter&) = delete; - - Status Write(Update update) override; -}; - // Interface for reading from a journal. class JournalReader { public: diff --git a/tensorflow/core/data/service/server_lib.cc b/tensorflow/core/data/service/server_lib.cc index 648a189717e..751fa6ca2a8 100644 --- a/tensorflow/core/data/service/server_lib.cc +++ b/tensorflow/core/data/service/server_lib.cc @@ -82,6 +82,10 @@ void DispatchGrpcDataServer::AddServiceToBuilder(grpc::ServerBuilder* builder) { service_ = absl::make_unique(builder, config_).release(); } +Status DispatchGrpcDataServer::StartServiceInternal() { + return service_->Start(); +} + Status DispatchGrpcDataServer::NumWorkers(int* num_workers) { GetWorkersRequest req; GetWorkersResponse resp; diff --git a/tensorflow/core/data/service/server_lib.h b/tensorflow/core/data/service/server_lib.h index 365241753fb..2c300947f63 100644 --- a/tensorflow/core/data/service/server_lib.h +++ b/tensorflow/core/data/service/server_lib.h @@ -81,7 +81,7 @@ class DispatchGrpcDataServer : public GrpcDataServerBase { protected: void AddServiceToBuilder(grpc::ServerBuilder* builder) override; - Status StartServiceInternal() override { return Status::OK(); } + Status StartServiceInternal() override; private: const experimental::DispatcherConfig config_; From 0907c6e0547b9bdd2bfde9acdaea33392c45617b Mon Sep 17 00:00:00 2001 From: Katherine Wu Date: Tue, 4 Aug 2020 10:54:17 -0700 Subject: [PATCH 0376/1017] Add explanation for v1-only tests (training_ops_tests). PiperOrigin-RevId: 324845593 Change-Id: I0c167e3a90514e83740e84f032917df62a1fddf6 --- .../python/training/training_ops_test.py | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tensorflow/python/training/training_ops_test.py b/tensorflow/python/training/training_ops_test.py index 118636c551e..3dd1283c924 100644 --- a/tensorflow/python/training/training_ops_test.py +++ b/tensorflow/python/training/training_ops_test.py @@ -60,7 +60,8 @@ class TrainingOpsTest(TensorFlowTestCase): self.assertShapeEqual(out, apply_sgd) self.assertAllCloseAccordingToType(x - alpha * delta, out) - @test_util.run_v1_only("b/120545219") + @test_util.run_v1_only("ApplyGradientDescent op returns a ref, so it is not " + "supported in eager mode.") def testApplyGradientDescent(self): for (dtype, use_gpu) in itertools.product( [np.float16, np.float32, np.float64], [False, True]): @@ -184,7 +185,8 @@ class TrainingOpsTest(TensorFlowTestCase): self.assertAllClose(linear_update, self.evaluate(linear)) self.assertAllClose(expected_out, out) - @test_util.run_v1_only("b/120545219") + @test_util.run_v1_only("ApplyAdagrad op returns a ref, so it is not " + "supported in eager mode.") def testApplyAdagrad(self): for (dtype, use_gpu) in itertools.product( [np.float16, np.float32, np.float64], [False, True]): @@ -194,7 +196,8 @@ class TrainingOpsTest(TensorFlowTestCase): grad = np.arange(100).astype(dtype) self._testTypesForAdagrad(x, y, lr, grad, use_gpu) - @test_util.run_v1_only("b/120545219") + @test_util.run_v1_only("ApplyFtrl op returns a ref, so it is not " + "supported in eager mode.") def testApplyFtrl(self): for dtype in [np.float16, np.float32, np.float64]: x = np.arange(100).astype(dtype) @@ -206,7 +209,8 @@ class TrainingOpsTest(TensorFlowTestCase): grad = np.arange(100).astype(dtype) self._testTypesForFtrl(x, y, z, lr, grad, use_gpu=False, l1=l1, l2=l2) - @test_util.run_v1_only("b/120545219") + @test_util.run_v1_only("ApplyFtrlMultiplyLinearByLr op returns a ref, so it " + "is not supported in eager mode.") def testApplyFtrlMultiplyLinearByLr(self): for dtype in [np.float16, np.float32, np.float64]: x = np.arange(100).astype(dtype) @@ -320,7 +324,8 @@ class TrainingOpsTest(TensorFlowTestCase): self.assertAllCloseAccordingToType(y[index] + grad[i] * grad[i], self.evaluate(accum)[index]) - @test_util.run_v1_only("b/120545219") + @test_util.run_v1_only("SparseApplyAdagrad op returns a ref, so it is not " + "supported in eager mode.") def testSparseApplyAdagrad(self): for (dtype, index_type) in itertools.product( [np.float16, np.float32, np.float64], [np.int32, np.int64]): @@ -334,7 +339,8 @@ class TrainingOpsTest(TensorFlowTestCase): indices = np.array([0, 2]).astype(index_type) self._testTypesForSparseAdagrad(x, y, lr, grad, indices) - @test_util.run_v1_only("b/120545219") + @test_util.run_v1_only("SparseApplyAdagrad op returns a ref, so it is not " + "supported in eager mode.") def testSparseApplyAdagradDim1(self): for (dtype, index_type) in itertools.product( [np.float16, np.float32, np.float64], [np.int32, np.int64]): @@ -348,7 +354,8 @@ class TrainingOpsTest(TensorFlowTestCase): indices = np.array([0, 2]).astype(index_type) self._testTypesForSparseAdagrad(x, y, lr, grad, indices) - @test_util.run_v1_only("b/120545219") + @test_util.run_v1_only("SparseApplyFtrl op returns a ref, so it is not " + "supported in eager mode.") def testSparseApplyFtrlDim1(self): for (dtype, index_type) in itertools.product( [np.float16, np.float32, np.float64], [np.int32, np.int64]): @@ -364,7 +371,8 @@ class TrainingOpsTest(TensorFlowTestCase): indices = np.array([0, 2]).astype(index_type) self._testTypesForSparseFtrl(x, y, z, lr, grad, indices) - @test_util.run_v1_only("b/120545219") + @test_util.run_v1_only("SparseApplyFtrlMultiplyLinearByLr op returns a ref, " + "so it is not supported in eager mode.") def testSparseApplyFtrlMultiplyLinearByLrDim1(self): for (dtype, index_type) in itertools.product([np.float16, np.float32, np.float64], @@ -381,7 +389,8 @@ class TrainingOpsTest(TensorFlowTestCase): indices = np.array([0, 2]).astype(index_type) self._testTypesForSparseFtrlMultiplyLinearByLr(x, y, z, lr, grad, indices) - @test_util.run_v1_only("b/120545219") + @test_util.run_v1_only("ApplyAdam op returns a ref, so it is not " + "supported in eager mode.") def testApplyAdam(self): for dtype, use_gpu in itertools.product( [np.float16, np.float32, np.float64], [False, True]): From bafcc6de33d1c5e976777e425b90a23b3e609d5d Mon Sep 17 00:00:00 2001 From: Haoyu Zhang Date: Tue, 4 Aug 2020 11:00:20 -0700 Subject: [PATCH 0377/1017] Disable client_test on windows. PiperOrigin-RevId: 324847100 Change-Id: I1181459063c65cbbecd7aa15ab9ede12cbb30d69 --- tensorflow/python/distribute/client/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/distribute/client/BUILD b/tensorflow/python/distribute/client/BUILD index 35d8de95276..d37d855a390 100644 --- a/tensorflow/python/distribute/client/BUILD +++ b/tensorflow/python/distribute/client/BUILD @@ -49,6 +49,7 @@ tf_py_test( srcs = ["client_test.py"], python_version = "PY3", shard_count = 12, + tags = ["no_windows"], # TODO(b/162751266) deps = [ ":client", "//tensorflow/python:client_testlib", From 7a2383c18f2f5be28d9c70e6362911d2c011f729 Mon Sep 17 00:00:00 2001 From: Pete Warden Date: Tue, 4 Aug 2020 11:16:42 -0700 Subject: [PATCH 0378/1017] Improve error catching for downloading PiperOrigin-RevId: 324850944 Change-Id: I1f00bf4ab6ea0e4f96e391b8129b8df4126755a2 --- tensorflow/lite/micro/tools/make/download_and_extract.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tensorflow/lite/micro/tools/make/download_and_extract.sh b/tensorflow/lite/micro/tools/make/download_and_extract.sh index fa5e57dd91a..e72fd7a0184 100755 --- a/tensorflow/lite/micro/tools/make/download_and_extract.sh +++ b/tensorflow/lite/micro/tools/make/download_and_extract.sh @@ -171,12 +171,20 @@ download_and_extract() { # loop to attempt to recover from them. for (( i=1; i<=$curl_retries; ++i )) do + # We have to use this approach because we normally halt the script when + # there's an error, and instead we want to catch errors so we can retry. + set +e curl -Ls --fail --retry 5 "${url}" > ${tempfile} CURL_RESULT=$? + set -e + + # Was the command successful? If so, continue. if [[ $CURL_RESULT -eq 0 ]] then break fi + + # Keep trying if we see the '56' error code. if [[ ( $CURL_RESULT -ne 56 ) || ( $i -eq $curl_retries ) ]] then echo "Error $CURL_RESULT downloading '${url}'" From 4e03f13e6a7c8fc760abb5badc32d7f6557f73c7 Mon Sep 17 00:00:00 2001 From: Yujing Zhang Date: Tue, 4 Aug 2020 11:17:45 -0700 Subject: [PATCH 0379/1017] Support packed TensorHandles on CustomDevices. PiperOrigin-RevId: 324851163 Change-Id: I017665b5e2bd90da37f4d45aa94e1278be7bd2f7 --- tensorflow/core/common_runtime/eager/tensor_handle.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc index 12bd70d705d..d7b2ef4be1e 100644 --- a/tensorflow/core/common_runtime/eager/tensor_handle.cc +++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc @@ -316,8 +316,7 @@ Status TensorHandle::CreatePackedHandle(std::vector&& handles, std::vector devices; for (auto* handle : handles) { if (VariantDeviceIsCustom(handle->device())) { - return errors::InvalidArgument( - "CustomDevice is not supported for packing."); + devices.push_back(absl::get(handle->device())->name()); } else { devices.push_back(handle->op_device() ? handle->op_device()->name() : ctx->HostCPU()->name()); From 6acd86d539464b611d37b8dc13251fafab25fb5c Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Tue, 4 Aug 2020 11:18:29 -0700 Subject: [PATCH 0380/1017] [TF2XLA] Make tf.argmin stable on XLA:TPU PiperOrigin-RevId: 324851314 Change-Id: Icdecbe87c545d4254bcdb508f76e31de30bc8f86 --- tensorflow/compiler/tf2xla/kernels/index_ops.cc | 2 +- tensorflow/python/eager/def_function_xla_jit_test.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/index_ops.cc b/tensorflow/compiler/tf2xla/kernels/index_ops.cc index 31637d9d8a0..df6d9b475dc 100644 --- a/tensorflow/compiler/tf2xla/kernels/index_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/index_ops.cc @@ -71,7 +71,7 @@ void XlaArgMinMaxOp::Compile(XlaOpKernelContext* ctx) { if (is_gpu_) { output = xla::ArgMinTwoPass(input, index_xla_type, axis); } else { - output = xla::ArgMin(input, index_xla_type, axis); + output = xla::ArgMin(input, index_xla_type, axis, /*stable=*/true); } } else { if (is_gpu_) { diff --git a/tensorflow/python/eager/def_function_xla_jit_test.py b/tensorflow/python/eager/def_function_xla_jit_test.py index b9457159217..813c1377cd9 100644 --- a/tensorflow/python/eager/def_function_xla_jit_test.py +++ b/tensorflow/python/eager/def_function_xla_jit_test.py @@ -293,9 +293,6 @@ class DefFunctionTest(xla_test.XLATestCase): @test_util.disable_mlir_bridge('TODO(b/162271237): argmax gives different' ' results in MLIR-based bridge') def testArgMinMax(self): - if 'tpu' in self.device.lower(): - self.skipTest('b/162800904: Tie resolution is wrong on TPU for tf.func') - with ops.device('device:{}:0'.format(self.device)): @def_function.function(experimental_compile=True) From 8c5f64c993b2a6a24335664db9891c8b7ecb2c73 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 4 Aug 2020 18:40:33 +0000 Subject: [PATCH 0381/1017] fix build --- tensorflow/core/kernels/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 589165fcd2f..47c7d41d0fe 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2960,6 +2960,7 @@ tf_kernel_library( ":fill_functor", ":tensor_map", "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", "//tensorflow/core:lib", "//third_party/eigen3", ], From adc09622af25ab8b48ef5f18ed25d18c50db8e13 Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Mon, 3 Aug 2020 10:33:01 -0700 Subject: [PATCH 0382/1017] Fix python math doc Run tests --- tensorflow/python/ops/math_ops.py | 46 ++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py index 0a16c18f7b2..d5c3650916b 100644 --- a/tensorflow/python/ops/math_ops.py +++ b/tensorflow/python/ops/math_ops.py @@ -367,8 +367,17 @@ def abs(x, name=None): # pylint: disable=redefined-builtin Given a tensor `x` of complex numbers, this operation returns a tensor of type `float32` or `float64` that is the absolute value of each element in `x`. For a complex number \\(a + bj\\), its absolute value is computed as - \\(\sqrt{a^2 + b^2}\\). For example: + \\(\sqrt{a^2 + b^2}\\). + For example: + + >>> # real number + >>> x = tf.constant([-2.25, 3.25]) + >>> tf.abs(x) + + + >>> # complex number >>> x = tf.constant([[-2.25 + 4.75j], [-3.25 + 5.75j]]) >>> tf.abs(x) 0. + `y = sign(x) = -1 if x < 0; 0 if x == 0; 1 if x > 0`. - For complex numbers, y = sign(x) = x / |x| if x != 0, otherwise y = 0. + For complex numbers, `y = sign(x) = x / |x| if x != 0, otherwise y = 0`. Example usage: + >>> # real number >>> tf.math.sign([0., 2., -3.]) - + + + >>> # complex number + >>> tf.math.sign([1 + 1j, 0 + 0j]) + Args: x: A Tensor. Must be one of the following types: bfloat16, half, float32, @@ -708,7 +724,7 @@ def sign(x, name=None): tf.math.sign(x.values, ...), x.dense_shape). """ x = ops.convert_to_tensor(x) - if x.dtype in (dtypes.complex64, dtypes.complex128): + if x.dtype.is_complex: return gen_math_ops.div_no_nan( x, cast( @@ -3615,9 +3631,9 @@ def _accumulate_n_grad(op, grad): def sigmoid(x, name=None): r"""Computes sigmoid of `x` element-wise. - Formula for calculating sigmoid(x): `y = 1 / (1 + exp(-x))`. + Formula for calculating $\mathrm{sigmoid}(x) = y = 1 / (1 + \exp(-x))$. - For x \in (-inf, inf) => sigmoid(x) \in (0, 1) + For $x \in (-\infty, \infty)$, $\mathrm{sigmoid}(x) \in (0, 1)$. Example Usage: @@ -4568,12 +4584,12 @@ def polyval(coeffs, x, name=None): If `x` is a tensor and `coeffs` is a list n + 1 tensors, this function returns the value of the n-th order polynomial - p(x) = coeffs[n-1] + coeffs[n-2] * x + ... + coeffs[0] * x**(n-1) + `p(x) = coeffs[n-1] + coeffs[n-2] * x + ... + coeffs[0] * x**(n-1)` evaluated using Horner's method, i.e. - p(x) = coeffs[n-1] + x * (coeffs[n-2] + ... + x * (coeffs[1] + - x * coeffs[0])) + `p(x) = coeffs[n-1] + x * (coeffs[n-2] + ... + x * (coeffs[1] + + x * coeffs[0]))` Usage Example: @@ -4820,10 +4836,14 @@ def exp(x, name=None): numpy=array([ 7.389056, 2980.958 ], dtype=float32)> For complex numbers, the exponential value is calculated as - \\(e^{x+iy}={e^x}{e^{iy}}={e^x}{\\cos(y)+i\\sin(y)}\\) + $$ + e^{x+iy} = {e^x} {e^{iy}} = {e^x} ({\cos (y) + i \sin (y)}) + $$ For `1+1j` the value would be computed as: - \\(e^1{\\cos(1)+i\\sin(1)} = 2.7182817 \\times (0.5403023+0.84147096j)\\) + $$ + e^1 (\cos (1) + i \sin (1)) = 2.7182817 \times (0.5403023+0.84147096j) + $$ >>> x = tf.constant(1 + 1j) >>> tf.math.exp(x) From e1e264594584d740ce5b077d92d58ca167318f22 Mon Sep 17 00:00:00 2001 From: Ran Chen Date: Tue, 4 Aug 2020 11:56:38 -0700 Subject: [PATCH 0383/1017] Put distribution and cluster parameter modifiers in their own combination It's easier to add more strategy/cluster modifiers. PiperOrigin-RevId: 324859132 Change-Id: Ided1a3d1106d71e86335ca3c59d7698550fa2155 --- .../integration_tests/saved_model_test.py | 4 +- tensorflow/python/distribute/combinations.py | 43 +++++++++---------- .../python/distribute/combinations_test.py | 14 +++--- tensorflow/python/distribute/vars_test.py | 31 ++++++------- 4 files changed, 42 insertions(+), 50 deletions(-) diff --git a/tensorflow/examples/saved_model/integration_tests/saved_model_test.py b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py index 6333e55999e..434d5ed4ad5 100644 --- a/tensorflow/examples/saved_model/integration_tests/saved_model_test.py +++ b/tensorflow/examples/saved_model/integration_tests/saved_model_test.py @@ -90,8 +90,8 @@ class SavedModelTest(scripts.TestCase, parameterized.TestCase): retrain_flag_value=["true", "false"], regularization_loss_multiplier=[None, 2], # Test for b/134528831. )), - test_combinations=(distribute_combinations.NamedGPUCombination(), - distribute_combinations.NamedTPUCombination())) + test_combinations=(distribute_combinations.GPUCombination(), + distribute_combinations.TPUCombination())) @combinations.generate(**TEST_MNIST_CNN_GENERATE_KWARGS) def test_mnist_cnn(self, use_keras_save_api, named_strategy, diff --git a/tensorflow/python/distribute/combinations.py b/tensorflow/python/distribute/combinations.py index 17bc285b222..3856b6fd132 100644 --- a/tensorflow/python/distribute/combinations.py +++ b/tensorflow/python/distribute/combinations.py @@ -99,7 +99,24 @@ class ClusterParameters(combinations_lib.ParameterModifier): return update -class NamedGPUCombination(combinations_lib.TestCombination): +class DistributionCombination(combinations_lib.TestCombination): + """Sets up distribution strategy for tests.""" + + def parameter_modifiers(self): + return [ + DistributionParameter(), + combinations_lib.OptionalParameter("use_var_policy"), + ] + + +class ClusterCombination(combinations_lib.TestCombination): + """Sets up multi worker tests.""" + + def parameter_modifiers(self): + return [ClusterParameters()] + + +class GPUCombination(combinations_lib.TestCombination): """Enable tests to request GPU hardware and skip non-GPU combinations. This class expects test_combinations to be generated with `NamedDistribution` @@ -141,17 +158,7 @@ class NamedGPUCombination(combinations_lib.TestCombination): return [combinations_lib.OptionalParameter("required_gpus")] -class GPUCombination(NamedGPUCombination): - """NamedGPUCombination that passes `tf.distribute.Strategy` to the tests.""" - - def parameter_modifiers(self): - return [ - ClusterParameters(), - DistributionParameter(), - ] + NamedGPUCombination.parameter_modifiers(self) - - -class NamedTPUCombination(combinations_lib.TestCombination): +class TPUCombination(combinations_lib.TestCombination): """Allow to request TPU hardware and skip non-TPU combinations. This class expects test_combinations to be generated with `NamedDistribution` @@ -213,16 +220,6 @@ class NamedTPUCombination(combinations_lib.TestCombination): ] -class TPUCombination(NamedTPUCombination): - """NamedTPUCombination that passes `tf.distribute.Strategy` to the tests.""" - - def parameter_modifiers(self): - return [ - ClusterParameters(), - DistributionParameter(), - ] + NamedTPUCombination.parameter_modifiers(self) - - class NamedDistribution(object): """Wraps a `tf.distribute.Strategy` and adds a name for test titles.""" @@ -304,6 +301,8 @@ def generate(combinations, test_combinations=()): default_combinations = ( framework_combinations.EagerGraphCombination(), framework_combinations.TFVersionCombination(), + ClusterCombination(), + DistributionCombination(), GPUCombination(), TPUCombination(), ) diff --git a/tensorflow/python/distribute/combinations_test.py b/tensorflow/python/distribute/combinations_test.py index 6d9d0b2570f..3fc3735d560 100644 --- a/tensorflow/python/distribute/combinations_test.py +++ b/tensorflow/python/distribute/combinations_test.py @@ -30,7 +30,7 @@ from tensorflow.python.framework import combinations as framework_combinations from tensorflow.python.platform import test -class ClusterParametersTest(test.TestCase, parameterized.TestCase): +class ClusterCombinationTest(test.TestCase, parameterized.TestCase): # For this test we need to use `framework.test_combinations` because our # `generate` eats the cluster parameters. # @@ -42,7 +42,7 @@ class ClusterParametersTest(test.TestCase, parameterized.TestCase): combinations.NamedDistribution( "HasClusterParams", lambda: None, has_chief=True, num_workers=2), ]), - test_combinations=(combinations.GPUCombination(),)) + test_combinations=(combinations.ClusterCombination(),)) def testClusterParams(self, distribution, has_chief, num_workers): self.assertTrue(has_chief) self.assertEqual(num_workers, 2) @@ -51,14 +51,14 @@ class ClusterParametersTest(test.TestCase, parameterized.TestCase): framework_combinations.combine(distribution=[ combinations.NamedDistribution("NoClusterParams", lambda: None), ]), - test_combinations=(combinations.GPUCombination(),)) + test_combinations=(combinations.ClusterCombination(),)) def testClusterParamsHasDefault(self, distribution, has_chief, num_workers): self.assertFalse(has_chief) self.assertEqual(num_workers, 1) @framework_combinations.generate( framework_combinations.combine(v=1), - test_combinations=(combinations.GPUCombination(),)) + test_combinations=(combinations.ClusterCombination(),)) def testClusterParamsNoStrategy(self, v, has_chief, num_workers): self.assertFalse(has_chief) self.assertEqual(num_workers, 1) @@ -69,7 +69,7 @@ class ClusterParametersTest(test.TestCase, parameterized.TestCase): "WithClusterParams", lambda: None, has_chief=True, num_workers=2), combinations.NamedDistribution("WithoutClusterParams", lambda: None), ]), - test_combinations=(combinations.GPUCombination(),)) + test_combinations=(combinations.ClusterCombination(),)) def testClusterParamsAreOptional(self, distribution): # If combinations library doesn't raise an exception, the test is passed. pass @@ -83,7 +83,7 @@ class ClusterParametersTest(test.TestCase, parameterized.TestCase): ds3=combinations.NamedDistribution( "Strategy3", lambda: None, has_chief=True, num_workers=0), ), - test_combinations=(combinations.GPUCombination(),)) + test_combinations=(combinations.ClusterCombination(),)) def testMultipleDistributionSingleWorker(self, ds1, ds2, ds3): # If combinations library doesn't raise an exception, the test is passed. pass @@ -101,7 +101,7 @@ class ClusterParametersShouldFailTest(test.TestCase, parameterized.TestCase): ds2=combinations.NamedDistribution( "Strategy2", lambda: None, has_chief=True, num_workers=2), ), - test_combinations=(combinations.GPUCombination(),)) + test_combinations=(combinations.ClusterCombination(),)) def testMultipleDistributionMultiWorker(self, ds1, ds2): # combinations library should raise an exception. pass diff --git a/tensorflow/python/distribute/vars_test.py b/tensorflow/python/distribute/vars_test.py index efbb6c23aaa..a8605a3f2da 100644 --- a/tensorflow/python/distribute/vars_test.py +++ b/tensorflow/python/distribute/vars_test.py @@ -95,8 +95,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): sess.run({"complicated": mirrored}) @combinations.generate(strategy_and_run_tf_function_combinations()) - def testAssign(self, distribution, experimental_run_tf_function, - use_var_policy): + def testAssign(self, distribution, experimental_run_tf_function): def assign(fn, v, update_value, cross_replica): update_fn = lambda: getattr(v, fn)(update_value) @@ -136,8 +135,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): self.evaluate(array_ops.ones_like(component))) @combinations.generate(strategy_and_run_tf_function_combinations()) - def testAssignOnWriteVar(self, distribution, experimental_run_tf_function, - use_var_policy): + def testAssignOnWriteVar(self, distribution, experimental_run_tf_function): with distribution.scope(): v_to_assign = variable_scope.variable( @@ -182,8 +180,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): self.assertAllEqual(2.0, self.evaluate(component.read_value())) @combinations.generate(strategy_and_run_tf_function_combinations()) - def testAssignPerReplicaVal(self, distribution, experimental_run_tf_function, - use_var_policy): + def testAssignPerReplicaVal(self, distribution, experimental_run_tf_function): if isinstance(distribution, _TPU_STRATEGIES): self.skipTest("Assigning PerReplica values is not supported. See" @@ -241,7 +238,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): self.assertAllEqual(expected, self.evaluate(component.read_value())) @combinations.generate(strategy_with_var_policy()) - def testValueInReplicaContext(self, distribution, use_var_policy): + def testValueInReplicaContext(self, distribution): with distribution.scope(): v = variables_lib.Variable( 1., aggregation=variables_lib.VariableAggregation.MEAN) @@ -260,8 +257,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): @combinations.generate(strategy_and_run_tf_function_combinations()) def testReadValueInReplicaContext(self, distribution, - experimental_run_tf_function, - use_var_policy): + experimental_run_tf_function): aggregations = [ variables_lib.VariableAggregation.NONE, variables_lib.VariableAggregation.SUM, @@ -286,8 +282,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): @combinations.generate(strategy_and_run_tf_function_combinations()) def testReadValueInCrossReplicaContext(self, distribution, - experimental_run_tf_function, - use_var_policy): + experimental_run_tf_function): aggregations = [ variables_lib.VariableAggregation.NONE, variables_lib.VariableAggregation.SUM, @@ -312,7 +307,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): self.evaluate(results)) @combinations.generate(strategy_with_var_policy()) - def testAssignOutOfScope(self, distribution, use_var_policy): + def testAssignOutOfScope(self, distribution): with distribution.scope(): mirrored = variables_lib.Variable(1.) self.evaluate(mirrored.assign(3.)) @@ -321,8 +316,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): self.assertEqual(self.evaluate(component.read_value()), 3.) @combinations.generate(strategy_with_var_policy()) - def testAssignAggregationMeanDTypeNonFloat(self, distribution, - use_var_policy): + def testAssignAggregationMeanDTypeNonFloat(self, distribution): if isinstance(distribution, _TPU_STRATEGIES): self.skipTest("Fix sponge/6e8ab540-4c0f-4da5-aedf-86505ff810c9 before " "reenabling test.") @@ -379,8 +373,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): self.assertEqual(self.evaluate(v.read_value()), 4) @combinations.generate(strategy_with_var_policy()) - def testInitializedToSameValueInsideEagerRun(self, distribution, - use_var_policy): + def testInitializedToSameValueInsideEagerRun(self, distribution): if not context.executing_eagerly(): self.skipTest("eager only test") v = [None] @@ -399,7 +392,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): self.assertAllEqual(vals[0], vals[1]) @combinations.generate(strategy_with_var_policy()) - def testAggregationOnlyFirstReplica(self, distribution, use_var_policy): + def testAggregationOnlyFirstReplica(self, distribution): with distribution.scope(): v = variable_scope.variable( 15., @@ -420,7 +413,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): per_replica_results) @combinations.generate(strategy_with_var_policy()) - def testInitScope(self, distribution, use_var_policy): + def testInitScope(self, distribution): if not context.executing_eagerly(): self.skipTest("eager only") class C(object): @@ -448,7 +441,7 @@ class OnWriteVariableSync(test.TestCase, parameterized.TestCase): self.assertAllEqual([2, 2], per_replica_results) @combinations.generate(strategy_with_var_policy()) - def testOperatorOverride(self, distribution, use_var_policy): + def testOperatorOverride(self, distribution): with distribution.scope(): v = variable_scope.variable( From 773541470562915763bd7fd61d76cda49df78aa2 Mon Sep 17 00:00:00 2001 From: Xingyu Long Date: Tue, 4 Aug 2020 15:08:19 -0400 Subject: [PATCH 0384/1017] Add README.md for keras_examples_benchmarks folder --- .../keras_examples_benchmarks/README.md | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 tensorflow/python/keras/benchmarks/keras_examples_benchmarks/README.md diff --git a/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/README.md b/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/README.md new file mode 100644 index 00000000000..595f94b7eda --- /dev/null +++ b/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/README.md @@ -0,0 +1,183 @@ +# Benchmarks for keras model exmaples + +- [Benchmarks for keras model exmaples](#benchmarks-for-keras-model-exmaples) + - [Keras Benchmarks](#keras-benchmarks) + - [Available models](#available-models) + - [Computer Vision examples](#computer-vision-examples) + - [Text & Sequence examples](#text--sequence-examples) + - [Other examples](#other-examples) + - [Available benchmark results](#available-benchmark-results) + - [Cifar10 CNN benchmark](#cifar10-cnn-benchmark) + - [MNIST Conv benchmark](#mnist-conv-benchmark) + - [MNIST Hierarchical RNN (HRNN) benchmark](#mnist-hierarchical-rnn-hrnn-benchmark) + - [Bidirectional LSTM benchmark](#bidirectional-lstm-benchmark) + - [Text classification with transformer benchmark](#text-classification-with-transformer-benchmark) + - [MLP benchmark](#mlp-benchmark) + - [Antirectifier benchmark](#antirectifier-benchmark) + - [IRNN benchmark](#irnn-benchmark) + - [Installing Bazel](#installing-bazel) + - [How to run benchmarks](#how-to-run-benchmarks) + - [How to add new benchmark tests that use `fit`](#how-to-add-new-benchmark-tests-that-use-fit) + - [Troubleshooting](#troubleshooting) + +## Keras Benchmarks + +These are benchmark tests running on keras models: models from [keras/examples](https://github.com/keras-team/keras/tree/master/examples). Benchmarks in the current folder (`tensorflow/python/keras/benchmarks/keras_examples_benchmarks`) use Keras [built-in dataset](https://keras.io/api/datasets/) or synthetic data. In addition, these benchmarks support different distribution strategies and measure the performance with distributed training. + +### Available models + +These examples are implemented by functional API and Sequential API. + +#### Computer Vision examples + +- [cifar10_cnn_benchmark_test.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/cifar10_cnn_benchmark_test.py): Simple CNN on CIFAR10 image dataset. +- [mnist_conv_benchmark_test.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/mnist_conv_benchmark_test.py): Simple Convnet that achieves ~99% test accuracy on MNIST. +- [mnist_hierarchical_rnn_benchmark_test.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/mnist_hierarchical_rnn_benchmark_test.py): Hierarchical RNN (HRNN) to classify MNIST digits. + +#### Text & Sequence examples + +[Bidirectional_lstm_benchmark_test.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/bidirectional_lstm_benchmark_test.py): 2-layer bidirectional LSTM on IMDB movie review dataset. +[text_classification_transformer_benchmark_test.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/text_classification_transformer_benchmark_test.py): Text classification with custom transformer block. +[reuters_mlp_benchmark_test.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/reuters_mlp_benchmark_test.py): Simple MLP on Reuters newswire topic classification dataset. + +#### Other examples + +[antirectifier_benchmark_test.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/antirectifier_benchmark_test.py): Simple custom layer example. +[mnist_irnn_benchmark_test.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/mnist_irnn_benchmark_test.py): Reproduction of the IRNN experiment with pixel-by-pixel sequential MNIST in ["A Simple Way to Initialize Recurrent Networks of Rectified Linear Units"](https://arxiv.org/abs/1504.00941) by Le et al. + +### Available benchmark results + +We run benchmarks on Google Cloud Platform (GCP) and here is current environment for running benchmarks tests:
+GPU: 2 x Tesla V100 (only for GPU test)
+OS: Ubuntu 18.04
+CPU: 8 x vCPUs, 30 GB memory
+CUDA: 10.1
+Bazel: 3.1.0
+ +If you want to run benchmark tests on GPU, please make sure you already installed CUDA and other dependencies and you can follow the instructions from the [official tutorial](https://www.tensorflow.org/install/gpu) for GPU support. + +#### Cifar10 CNN benchmark + +| | Batch_size | Wall_time | Avg_epoch_time | Exp_per_sec | Distribution_Strategy | +| :---: | :--------: | :-------: | :------------: | :---------: | :-------------------: | +| CPU | 256 | 1393.4896 | 3.21 | 15397.69 | `off` | +| GPU:2 | 256 | 76.49 | 2.59 | 18758.01 | `mirrored` | + +#### MNIST Conv benchmark + +| | Batch_size | Wall_time | Avg_epoch_time | Exp_per_sec | Distribution_Strategy | +| :---: | :--------: | :-------: | :------------: | :---------: | :-------------------: | +| CPU | 256 | 196.52 | 12.19 | 4915.26 | `off` | +| GPU:2 | 256 | 24.5794 | 1.21 | 47899.32 | `mirrored` | + +#### MNIST Hierarchical RNN (HRNN) benchmark + +| | Batch_size | Wall_time | Avg_epoch_time | Exp_per_sec | Distribution_Strategy | +| :---: | :--------: | :-------: | :------------: | :---------: | :-------------------: | +| CPU | 256 | 654.05 | 218.68 | 274.24 | `off` | +| GPU:2 | 256 | 20.77 | 3.73 | 15088.06 | `mirrored` | + +#### Bidirectional LSTM benchmark + +| | Batch_size | Wall_time | Avg_epoch_time | Exp_per_sec | Distribution_Strategy | +| :---: | :--------: | :-------: | :------------: | :---------: | :-------------------: | +| CPU | 512 | 225.57 | 72.55 | 344.70 | `off` | +| GPU:2 | 512 | 23.54 | 3.23 | 7532.53 | `mirrored` | + +#### Text classification with transformer benchmark + +| | Batch_size | Wall_time | Avg_epoch_time | Exp_per_sec | Distribution_Strategy | +| :---: | :--------: | :-------: | :------------: | :---------: | :-------------------: | +| CPU | 512 | 109.22 | 35.93 | 698.10 | `off` | +| GPU:2 | 512 | 9.28 | 0.83 | 26567.54 | `mirrored` | + +#### MLP benchmark + +| | Batch_size | Wall_time | Avg_epoch_time | Exp_per_sec | Distribution_Strategy | +| :---: | :--------: | :-------: | :------------: | :---------: | :-------------------: | +| CPU | 128 | 3.76 | 0.54 | 17678.54 | `off` | +| GPU:2 | 128 | 5.91 | 0.30 | 25435.14 | `mirrored` | + +#### Antirectifier benchmark + +| | Batch_size | Wall_time | Avg_epoch_time | Exp_per_sec | Distribution_Strategy | +| :---: | :--------: | :-------: | :------------: | :---------: | :-------------------: | +| CPU | 512 | 6.77 | 1.79 | 30916.39 | `off` | +| GPU:2 | 512 | 6.81 | 0.66 | 66563.17 | `mirrored` | + +#### IRNN benchmark + +| | Batch_size | Wall_time | Avg_epoch_time | Exp_per_sec | Distribution_Strategy | +| :---: | :--------: | :-------: | :------------: | :---------: | :-------------------: | +| CPU | 1024 | 213.00 | 69.01 | 868.08 | `off` | +| GPU:2 | 1024 | 92.71 | 29.12 | 2042.94 | `mirrored` | + +**Note**: For the smaller models, running models with GPU may be slower than running models with CPU as training small models is not computation dominant and there might be some overhead on model replication and data sharding with distributed training on GPUs. + +## Installing Bazel + +This step can be skipped if Bazel is already installed.
+ +We need to use [Bazel](https://bazel.build/) to build targets based on BUILD files. It will take a while for the first time because it will compile all dependencies from your BUILD file. For the next time, Bazel will use the cache and it’ll be much faster. Since we use Ubuntu OS, we can install bazel by using apt repository. + +1. Add bazel as package source + + ```shell + sudo apt install curl gnupg + ``` + + ```shell + curl https://bazel.build/bazel-release.pub.gpg | sudo apt-key add - + ``` + + ```shell + echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | sudo tee /etc/apt/sources.list.d/bazel.list + ``` + + Before we install the bazel, We should take a look for a bazel version that can build the specific tensorflow version, you can check it from [here](https://www.tensorflow.org/install/source#tested_build_configurations). In addition, you can follow the instructions from [Bazel website](https://docs.bazel.build/versions/3.4.0/install.html). + +2. Install Bazel + + ```shell + sudo apt update && sudo apt install bazel-`version` + ``` + +## How to run benchmarks + +To run benchmarks in [keras/benchmarks](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/keras/benchmarks), please take the following steps: + +1. Pull the latest tensorflow repo from github. +2. Install the Bazel tool which works with tensorflow, please take a look for the Tool installation section. +3. To run benchmarks with Bazel, use the `--benchmarks=.` flags to specify the benchmarks to run. + - To run all benchmarks on CPU + + ```shell + bazel run -c opt benchmark_test -- --benchmarks=. + ``` + + - To run all benchmarks on GPU + + ```shell + bazel run run --config=cuda -c opt --copt="-mavx" benchmarks_test -- \ --benchmarks=. + ``` + + - To run a subset of benchmarks using `--benchmarks` flag, `--benchmarks`: the list of benchmarks to run. The specified value is interpreted as a regular expression and any benchmarks whose name contains a partial match to the regular expression is executed. e.g. `--benchmarks=".*lstm*."`, will run all lstm layer related benchmarks. + +## How to add new benchmark tests that use `fit` + +To add a new benchmark, please follow the steps: + +1. Create your own benchmark test file, `xxxx_benchmark_test.py`. +2. Import `benchmark_util` to measure and track performance. +3. Create class which inherits from `tf.test.Benchmark` +4. Define and load dataset in `__init__` method. +5. Design and create a model in `_build_model` method. +6. Define the `benchmark_xxx` method and it will pass essential parameters, which includes `batch_size`, `run_iters`, `train_data` and etc. You can check examples from [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/python/keras/benchmarks/keras_examples_benchmarks). +7. In addition, you need to add a benchmark target in the [BUILD](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/keras/benchmarks/BUILD) file and write the target name and dependencies. You can take current BUILD as a reference. + +## Troubleshooting + +1. tensorflow.python.framework.errors_impl.InternalError: CUDA runtime implicit initialization on GPU:0 failed. Status: device kernel image is invalid + + - Make sure CUDA was installed on your machine. + - Pull the latest tensorflow repo and run the `./configure` in the root folder of tensorflow, it will help you to create the configuration file which shows your local environment. Please check [this post](https://www.tensorflow.org/install/source#configure_the_build) to know the details. \ No newline at end of file From beab9b83eaf0a5227cc8c314726bb00a32c866ed Mon Sep 17 00:00:00 2001 From: Robert David Date: Tue, 4 Aug 2020 12:05:27 -0700 Subject: [PATCH 0385/1017] Fix missing dependency for GPU delegate PiperOrigin-RevId: 324861084 Change-Id: I6a894d90b5844ea1f5a04526ec396bfa75ff9cb0 --- tensorflow/lite/delegates/gpu/cl/selectors/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/BUILD b/tensorflow/lite/delegates/gpu/cl/selectors/BUILD index ebee4b03b6e..7ea0ac35f89 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/BUILD +++ b/tensorflow/lite/delegates/gpu/cl/selectors/BUILD @@ -108,6 +108,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/cl/kernels:conv_common", "//tensorflow/lite/delegates/gpu/cl/kernels:elementwise", "//tensorflow/lite/delegates/gpu/cl/kernels:gpu_operation", + "//tensorflow/lite/delegates/gpu/cl/kernels:mean_stddev_normalization", "//tensorflow/lite/delegates/gpu/cl/selectors:default_selector", "//tensorflow/lite/delegates/gpu/common:data_type", "//tensorflow/lite/delegates/gpu/common:model", From 6be604aaacd9d270de01c37ec6e9a9a077397848 Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Tue, 4 Aug 2020 12:26:02 -0700 Subject: [PATCH 0386/1017] Reland (Attempt #3) PR #35985: [TFLite int16] 16-bit version of ADD/SUB reference kernel operators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Imported from GitHub PR https://github.com/tensorflow/tensorflow/pull/35985 This PR is one of steps to extend 8-bit quantization to support symmetric 16-bit activations. Each activation is of type int16 and symmetric around zero. The weight tensor precision remains at 8-bit signed values. The bias is set to int64 precision. In this PR we introduce implementation and tests for ADD/SUB kernel reference function. The specification of this operator: SUB   Input 0:     data_type  : int16     range      : [-32768, 32767]     granularity: per-tensor, zero_point=0   Input 1:     data_type  : int16     range      : [-32768, 32767]     granularity: per-tensor, zero_point=0   Output 0:     data_type  : int16     range      : [-32768, 32767]     granularity: per-tensor, zero_point=0 ADD   Input 0:     data_type  : int16     range      : [-32768, 32767]     granularity: per-tensor, zero_point=0   Input 1:     data_type  : int16     range      : [-32768, 32767]     granularity: per-tensor, zero_point=0   Output 0:     data_type  : int16     range      : [-32768, 32767]     granularity: per-tensor, zero_point=0 Copybara import of the project: -- b94cb4732ab536828e565fd1c7b557f124432e29 by Elena Zhelezina : Added 16-bit version of ADD/SUB operators. Broadcasting is included. -- 924d0b72c568f249f2fd224a942f8922524bfede by Elena Zhelezina : Addressed reviewer comments. -- dd0d9e8f03d1fb1b887609fffb8ea5a86638c63e by Elena Zhelezina : Added versioning to ADD/SUB + some rework of the existing code. -- abae3fd9a9b894c07d13c9ef416092c9004bc913 by Elena Zhelezina : Added versioning for ADD/SUB with new option in the schema.fbs schema_generated.h is edited manually. -- 24f3f5593a06d24fa1ca6be257f1265b5293d492 by Elena Zhelezina : Fix for broken build. -- d252fe175aef3a1a08c65155815efb706aa80afd by Elena Zhelezina : Fix for the failing internal test for NN delegates. -- 2223a5c380bb821eb05f8034703c687269353e32 by Elena Zhelezina : Fix for asan failures. Change-Id: I2cf421ddda7f9e802202239136ab062bcd63b4aa -- 3c219a46ce5888e8e402b64cc943ac6522156ef5 by Elena Zhelezina : Added broadcast params to addsub structure. Change-Id: I61d7d4a94087d052a782890799211031f6ed3015 -- 9131a38c776109cdbcfa60be602667ec7aafe00f by Elena Zhelezina : Corrected defaults. Change-Id: I9ea50c75014cc03ac91fdef0f5b4fe11395f7074 PiperOrigin-RevId: 324865496 --- tensorflow/lite/c/builtin_op_data.h | 4 + .../lite/core/api/flatbuffer_conversions.cc | 2 + .../experimental/writer/writer_lib_test.cc | 4 + tensorflow/lite/kernels/add.cc | 85 ++++++++++++---- tensorflow/lite/kernels/add_test.cc | 31 ++++-- .../lite/kernels/internal/reference/add.h | 66 ++++++++++--- tensorflow/lite/kernels/register.cc | 6 +- tensorflow/lite/kernels/sub.cc | 97 +++++++++++++++---- tensorflow/lite/kernels/sub_test.cc | 12 +++ tensorflow/lite/schema/schema.fbs | 4 + tensorflow/lite/schema/schema_generated.h | 22 ++++- tensorflow/lite/toco/tflite/op_version.cc | 3 + tensorflow/lite/toco/tflite/operator.cc | 4 +- .../lite/tools/versioning/op_version.cc | 46 ++++++++- tensorflow/lite/tools/versioning/op_version.h | 5 + .../lite/tools/versioning/runtime_version.cc | 3 + 16 files changed, 321 insertions(+), 73 deletions(-) diff --git a/tensorflow/lite/c/builtin_op_data.h b/tensorflow/lite/c/builtin_op_data.h index 232f5f95928..e205f075b43 100644 --- a/tensorflow/lite/c/builtin_op_data.h +++ b/tensorflow/lite/c/builtin_op_data.h @@ -199,6 +199,8 @@ typedef struct { typedef struct { TfLiteFusedActivation activation; + // Parameter added for the version 4. + bool pot_scale_int16; } TfLiteAddParams; typedef struct { @@ -220,6 +222,8 @@ typedef struct { typedef struct { TfLiteFusedActivation activation; + // Parameter added for the version 5. + bool pot_scale_int16; } TfLiteSubParams; typedef struct { diff --git a/tensorflow/lite/core/api/flatbuffer_conversions.cc b/tensorflow/lite/core/api/flatbuffer_conversions.cc index 0652c64f6c2..7fb04f5b89e 100644 --- a/tensorflow/lite/core/api/flatbuffer_conversions.cc +++ b/tensorflow/lite/core/api/flatbuffer_conversions.cc @@ -896,6 +896,7 @@ TfLiteStatus ParseAdd(const Operator* op, ErrorReporter* error_reporter, if (schema_params != nullptr) { params->activation = ConvertActivation(schema_params->fused_activation_function()); + params->pot_scale_int16 = schema_params->pot_scale_int16(); } else { // TODO(b/157480169): We should either return kTfLiteError or fill in some // reasonable defaults in the params struct. We are not doing so until we @@ -1631,6 +1632,7 @@ TfLiteStatus ParseSub(const Operator* op, ErrorReporter* error_reporter, if (schema_params != nullptr) { params->activation = ConvertActivation(schema_params->fused_activation_function()); + params->pot_scale_int16 = schema_params->pot_scale_int16(); } else { // TODO(b/157480169): We should either return kTfLiteError or fill in some // reasonable defaults in the params struct. We are not doing so until we diff --git a/tensorflow/lite/experimental/writer/writer_lib_test.cc b/tensorflow/lite/experimental/writer/writer_lib_test.cc index fb59482f705..bf50d4944f1 100644 --- a/tensorflow/lite/experimental/writer/writer_lib_test.cc +++ b/tensorflow/lite/experimental/writer/writer_lib_test.cc @@ -47,6 +47,7 @@ TEST(Writer, FloatModelTest) { TfLiteAddParams* builtin_data = reinterpret_cast(malloc(sizeof(TfLiteAddParams))); builtin_data->activation = kTfLiteActNone; + builtin_data->pot_scale_int16 = false; const TfLiteRegistration* reg = resolver.FindOp(BuiltinOperator_ADD, 1); interpreter.AddNodeWithParameters({0, 1}, {2}, initial_data, 0, reinterpret_cast(builtin_data), reg); @@ -84,6 +85,7 @@ TEST(Writer, CustomInputOutputTest) { TfLiteAddParams* builtin_data = reinterpret_cast(malloc(sizeof(TfLiteAddParams))); builtin_data->activation = kTfLiteActNone; + builtin_data->pot_scale_int16 = false; const TfLiteRegistration* reg = resolver.FindOp(BuiltinOperator_ADD, 1); interpreter.AddNodeWithParameters({0, 1}, {2}, initial_data, 0, reinterpret_cast(builtin_data), reg); @@ -131,6 +133,7 @@ TEST(Writer, CustomInputOutputErrorCasesTest) { TfLiteAddParams* builtin_data = reinterpret_cast(malloc(sizeof(TfLiteAddParams))); builtin_data->activation = kTfLiteActNone; + builtin_data->pot_scale_int16 = false; const TfLiteRegistration* reg = resolver.FindOp(BuiltinOperator_ADD, 1); interpreter.AddNodeWithParameters({0, 1}, {2}, initial_data, 0, reinterpret_cast(builtin_data), reg); @@ -173,6 +176,7 @@ TEST(Writer, PerTensorQuantizedModelTest) { TfLiteAddParams* builtin_data = reinterpret_cast(malloc(sizeof(TfLiteAddParams))); builtin_data->activation = kTfLiteActNone; + builtin_data->pot_scale_int16 = false; const TfLiteRegistration* reg = resolver.FindOp(BuiltinOperator_ADD, 1); interpreter.AddNodeWithParameters({0, 1}, {2}, initial_data, 0, reinterpret_cast(builtin_data), reg); diff --git a/tensorflow/lite/kernels/add.cc b/tensorflow/lite/kernels/add.cc index bda475bdc35..7692ae9e54b 100644 --- a/tensorflow/lite/kernels/add.cc +++ b/tensorflow/lite/kernels/add.cc @@ -68,6 +68,11 @@ struct OpData { int32 input1_offset; int32 input2_offset; int32 output_offset; + + // This parameter is used to indicate whether + // parameter scale is power of two. + // It is used in 16-bit -> 16-bit quantization. + bool pot_scale_int16; }; void* Init(TfLiteContext* context, const char* buffer, size_t length) { @@ -103,12 +108,55 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { output_size = TfLiteIntArrayCopy(input1->dims); } - if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8) { + // 8bit -> 8bit general quantized path, with general rescalings + // as well as, int16 -> int16 with general rescalings + bool pot_scale_int16 = true; + + bool input1_scale_is_pot = false; + bool input2_scale_is_pot = false; + bool output_scale_is_pot = false; + + int input1_scale_log2_rounded{0}; + int input2_scale_log2_rounded{0}; + int output_scale_log2_rounded{0}; + + if (input1->type == kTfLiteInt16 && input2->type == kTfLiteInt16 && + output->type == kTfLiteInt16) { + // In case of 16-bit, there are two implementation: + // the scale parameter is a general number + // the scale parameter is POT and + // zero_point is zero for inputs/output. + pot_scale_int16 = (input1->params.zero_point == 0) && + (input2->params.zero_point == 0) && + (output->params.zero_point == 0); + + input1_scale_is_pot = + CheckedLog2(input1->params.scale, &input1_scale_log2_rounded); + + input2_scale_is_pot = + CheckedLog2(input2->params.scale, &input2_scale_log2_rounded); + + output_scale_is_pot = + CheckedLog2(output->params.scale, &output_scale_log2_rounded); + + pot_scale_int16 &= + input1_scale_is_pot && input2_scale_is_pot && output_scale_is_pot; + } + + data->pot_scale_int16 = pot_scale_int16; + + if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8 || + !pot_scale_int16) { // 8bit -> 8bit general quantized path, with general rescalings + // as well as, 16bit -> 16bit with general rescalings data->input1_offset = -input1->params.zero_point; data->input2_offset = -input2->params.zero_point; data->output_offset = output->params.zero_point; - data->left_shift = 20; + + // The shift is set to 15 for 16-bit and 20 in case of 8-bit, accordingly. + // In case of 16-bit we have 65535 << 15 which is less than 1 << 31, + // therefore the addition will still fit in a 32 bit accumulator. + data->left_shift = !pot_scale_int16 ? 15 : 20; const double twice_max_input_scale = 2 * std::max(input1->params.scale, input2->params.scale); const double real_input1_multiplier = @@ -144,19 +192,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, input2->params.zero_point, 0); TF_LITE_ENSURE_EQ(context, output->params.zero_point, 0); - int input1_scale_log2_rounded; - bool input1_scale_is_pot = - CheckedLog2(input1->params.scale, &input1_scale_log2_rounded); TF_LITE_ENSURE(context, input1_scale_is_pot); - - int input2_scale_log2_rounded; - bool input2_scale_is_pot = - CheckedLog2(input2->params.scale, &input2_scale_log2_rounded); TF_LITE_ENSURE(context, input2_scale_is_pot); - - int output_scale_log2_rounded; - bool output_scale_is_pot = - CheckedLog2(output->params.scale, &output_scale_log2_rounded); TF_LITE_ENSURE(context, output_scale_is_pot); data->input1_shift = input1_scale_log2_rounded - output_scale_log2_rounded; @@ -231,7 +268,8 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, const TfLiteTensor* input1, const TfLiteTensor* input2, TfLiteTensor* output) { - if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8) { + if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8 || + !data->pot_scale_int16) { tflite::ArithmeticParams op_params; op_params.left_shift = data->left_shift; op_params.input1_offset = data->input1_offset; @@ -266,6 +304,15 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, TF_LITE_ADD(optimized_integer_ops, Add, int8_t); } } + } else if (output->type == kTfLiteInt16) { + if (need_broadcast) { + TF_LITE_ADD(reference_ops, BroadcastAdd4DSlow, int16_t); + } else { + reference_ops::Add( + op_params, GetTensorShape(input1), GetTensorData(input1), + GetTensorShape(input2), GetTensorData(input2), + GetTensorShape(output), GetTensorData(output), false); + } } else { if (kernel_type == kReference) { if (need_broadcast) { @@ -283,12 +330,12 @@ TfLiteStatus EvalAddQuantized(TfLiteContext* context, TfLiteNode* node, } #undef TF_LITE_ADD } else if (output->type == kTfLiteInt16) { + tflite::ArithmeticParams op_params; + op_params.input1_shift = data->input1_shift; + op_params.input2_shift = data->input2_shift; + SetActivationParams(data->output_activation_min, + data->output_activation_max, &op_params); #define TF_LITE_ADD(type, opname) \ - tflite::ArithmeticParams op_params; \ - op_params.input1_shift = data->input1_shift; \ - op_params.input2_shift = data->input2_shift; \ - SetActivationParams(data->output_activation_min, \ - data->output_activation_max, &op_params); \ type::opname(op_params, GetTensorShape(input1), \ GetTensorData(input1), GetTensorShape(input2), \ GetTensorData(input2), GetTensorShape(output), \ diff --git a/tensorflow/lite/kernels/add_test.cc b/tensorflow/lite/kernels/add_test.cc index bb883dd9b05..fc78f930897 100644 --- a/tensorflow/lite/kernels/add_test.cc +++ b/tensorflow/lite/kernels/add_test.cc @@ -310,15 +310,18 @@ TEST(QuantizedAddOpModel, QuantizedTestsNoActivationInt16) { const float kMin = -1.f; const float kMax = 32767.f / 32768.f; float kQuantizedTolerance = GetToleranceInt16(kMin, kMax); - std::vector> inputs1 = { - {0.1, 0.2, 0.3, 0.4}, {-0.8, 0.2, 0.4, 0.7}, {-0.8, 0.2, 0.7, 0.3}}; - std::vector> inputs2 = { - {0.6, 0.4, 0.3, 0.1}, {0.6, 0.4, 0.5, -0.8}, {0.6, 0.4, -0.8, 0.5}}; - std::vector> results = { - {0.7, 0.6, 0.6, 0.5}, {-0.2, 0.6, 0.9, -0.1}, {-0.2, 0.6, -0.1, 0.8}}; + std::vector> inputs1 = {{0.1, 0.2, 0.3, 0.4, 0.9, 0.7}, + {-0.8, 0.2, 0.4, 0.7, 0.1, 0.0}, + {-0.8, 0.2, 0.7, 0.3, 0.9, 0.1}}; + std::vector> inputs2 = {{0.6, 0.4, 0.3, 0.1, -0.1, 0.3}, + {0.6, 0.4, 0.5, -0.8, 0.0, -1.0}, + {0.6, 0.4, -0.8, 0.5, -0.9, 0.1}}; + std::vector> results = {{0.7, 0.6, 0.6, 0.5, 0.8, 1.0}, + {-0.2, 0.6, 0.9, -0.1, 0.1, -1.0}, + {-0.2, 0.6, -0.1, 0.8, 0.0, 0.2}}; for (size_t i = 0; i < inputs1.size(); ++i) { - QuantizedAddOpModel m({TensorType_INT16, {1, 2, 2, 1}, kMin, kMax}, - {TensorType_INT16, {1, 2, 2, 1}, kMin, kMax}, + QuantizedAddOpModel m({TensorType_INT16, {1, 2, 3, 1}, kMin, kMax}, + {TensorType_INT16, {1, 2, 3, 1}, kMin, kMax}, {TensorType_INT16, {}, kMin, kMax}, ActivationFunctionType_NONE); m.QuantizeAndPopulate(m.input1(), inputs1[i]); @@ -439,6 +442,10 @@ TEST(QuantizedAddOpModel, QuantizedWithScalarBroadcastInt8) { QuantizedWithScalarBroadcast(); } +TEST(QuantizedAddOpModel, QuantizedWithScalarBroadcastInt16) { + QuantizedWithScalarBroadcast(); +} + template void QuantizedWithMixedBroadcast() { float kQuantizedTolerance = GetTolerance(-3.f, 3.f); @@ -501,6 +508,10 @@ TEST(QuantizedAddOpModel, QuantizedWithMixedBroadcastInt8) { QuantizedWithMixedBroadcast(); } +TEST(QuantizedAddOpModel, QuantizedWithMixedBroadcastInt16) { + QuantizedWithMixedBroadcast(); +} + template void QuantizedWithGenericBroadcast() { float kQuantizedTolerance = GetTolerance(-1.0, 1.0); @@ -527,5 +538,9 @@ TEST(QuantizedAddOpModel, QuantizedWithGenericdBroadcastInt8) { QuantizedWithGenericBroadcast(); } +TEST(QuantizedAddOpModel, QuantizedWithGenericdBroadcastInt16) { + QuantizedWithGenericBroadcast(); +} + } // namespace } // namespace tflite diff --git a/tensorflow/lite/kernels/internal/reference/add.h b/tensorflow/lite/kernels/internal/reference/add.h index 94c58097154..5be7ab4dc0c 100644 --- a/tensorflow/lite/kernels/internal/reference/add.h +++ b/tensorflow/lite/kernels/internal/reference/add.h @@ -51,13 +51,18 @@ inline void Add(const ArithmeticParams& params, // Element-wise add that can often be used for inner loop of broadcast add as // well as the non-broadcast add. + +// This function is used for 8-bit as well as for 16-bit, but the accumulator +// is 32-bit for both cases. The overflow does not happen due to the +// choice of the shift (20 or 15, accordingly - see add.cc for more comments). +template inline void AddElementwise(int size, const ArithmeticParams& params, - const uint8_t* input1_data, - const uint8_t* input2_data, uint8_t* output_data) { - TFLITE_DCHECK_GT(params.input1_offset, -256); - TFLITE_DCHECK_GT(params.input2_offset, -256); - TFLITE_DCHECK_LT(params.input1_offset, 256); - TFLITE_DCHECK_LT(params.input2_offset, 256); + const T* input1_data, const T* input2_data, + T* output_data) { + TFLITE_DCHECK_GT(params.input1_offset, -std::numeric_limits::max()); + TFLITE_DCHECK_GT(params.input2_offset, -std::numeric_limits::max()); + TFLITE_DCHECK_LT(params.input1_offset, std::numeric_limits::max()); + TFLITE_DCHECK_LT(params.input2_offset, std::numeric_limits::max()); for (int i = 0; i < size; ++i) { const int32_t input1_val = params.input1_offset + input1_data[i]; @@ -78,7 +83,7 @@ inline void AddElementwise(int size, const ArithmeticParams& params, const int32_t clamped_output = std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output)); - output_data[i] = static_cast(clamped_output); + output_data[i] = static_cast(clamped_output); } } @@ -132,10 +137,38 @@ inline void Add(const ArithmeticParams& params, AddElementwise(flat_size, params, input1_data, input2_data, output_data); } +inline void AddGeneralParamScale(const ArithmeticParams& params, + const RuntimeShape& input1_shape, + const int16_t* input1_data, + const RuntimeShape& input2_shape, + const int16_t* input2_data, + const RuntimeShape& output_shape, + int16_t* output_data) { + TFLITE_DCHECK_LE(params.quantized_activation_min, + params.quantized_activation_max); + const int flat_size = + MatchingElementsSize(input1_shape, input2_shape, output_shape); + + int max_value = std::numeric_limits::max(); + + TFLITE_DCHECK_GT(params.input1_offset, -max_value); + TFLITE_DCHECK_GT(params.input2_offset, -max_value); + TFLITE_DCHECK_LT(params.input1_offset, max_value); + TFLITE_DCHECK_LT(params.input2_offset, max_value); + AddElementwise(flat_size, params, input1_data, input2_data, output_data); +} + inline void Add(const ArithmeticParams& params, const RuntimeShape& input1_shape, const int16_t* input1_data, const RuntimeShape& input2_shape, const int16_t* input2_data, - const RuntimeShape& output_shape, int16_t* output_data) { + const RuntimeShape& output_shape, int16_t* output_data, + bool pot_scale = true) { + if (!pot_scale) { + AddGeneralParamScale(params, input1_shape, input1_data, input2_shape, + input2_data, output_shape, output_data); + return; + } + TFLITE_DCHECK_LE(params.quantized_activation_min, params.quantized_activation_max); @@ -258,13 +291,14 @@ inline void BroadcastAdd4DSlow(const ArithmeticParams& params, } } -inline void BroadcastAdd4DSlow(const ArithmeticParams& params, - const RuntimeShape& input1_shape, - const uint8_t* input1_data, - const RuntimeShape& input2_shape, - const uint8_t* input2_data, - const RuntimeShape& output_shape, - uint8_t* output_data) { +// This function is used for 8-bit as well as for 16-bit, but the accumulator +// is 32-bit for both cases. The overflow does not happen due to the +// choice of the shift (20 or 15, accordingly - see add.cc for more comments). +template +inline void BroadcastAdd4DSlow( + const ArithmeticParams& params, const RuntimeShape& input1_shape, + const T* input1_data, const RuntimeShape& input2_shape, + const T* input2_data, const RuntimeShape& output_shape, T* output_data) { NdArrayDesc<4> desc1; NdArrayDesc<4> desc2; NdArrayDescsForElementwiseBroadcast(input1_shape, input2_shape, &desc1, @@ -314,7 +348,7 @@ inline void BroadcastAdd4DSlow(const ArithmeticParams& params, std::min(params.quantized_activation_max, std::max(params.quantized_activation_min, raw_output)); output_data[Offset(extended_output_shape, b, y, x, c)] = - static_cast(clamped_output); + static_cast(clamped_output); } } } diff --git a/tensorflow/lite/kernels/register.cc b/tensorflow/lite/kernels/register.cc index adffa19c4e1..1d1db9e0403 100644 --- a/tensorflow/lite/kernels/register.cc +++ b/tensorflow/lite/kernels/register.cc @@ -89,8 +89,8 @@ BuiltinOpResolver::BuiltinOpResolver() { /* min_version = */ 1, /* max_version = */ 3); AddBuiltin(BuiltinOperator_ADD, Register_ADD(), - /* min_version = */ 1, - /* max_version = */ 2); + /* min_version */ 1, + /* max_version */ 4); AddBuiltin(BuiltinOperator_SPACE_TO_BATCH_ND, Register_SPACE_TO_BATCH_ND(), /* min_version = */ 1, /* max_version = */ 3); @@ -143,7 +143,7 @@ BuiltinOpResolver::BuiltinOpResolver() { /* max_version */ 2); AddBuiltin(BuiltinOperator_SUB, Register_SUB(), /* min_version = */ 1, - /* max_version = */ 4); + /* max_version = */ 5); AddBuiltin(BuiltinOperator_SPLIT, Register_SPLIT(), /* min_version = */ 1, /* max_version = */ 4); diff --git a/tensorflow/lite/kernels/sub.cc b/tensorflow/lite/kernels/sub.cc index 4cd9dd7ff60..f93ebecd46d 100644 --- a/tensorflow/lite/kernels/sub.cc +++ b/tensorflow/lite/kernels/sub.cc @@ -71,6 +71,11 @@ struct OpData { int32 input1_offset; int32 input2_offset; int32 output_offset; + + // This parameter is used to indicate whether + // parameter scale is power of two. + // It is used in 16-bit -> 16-bit quantization. + bool pot_scale_int16; }; void* Init(TfLiteContext* context, const char* buffer, size_t length) { @@ -83,13 +88,14 @@ void Free(TfLiteContext* context, void* buffer) { delete reinterpret_cast(buffer); } -TfLiteStatus Prepare8BitSubOp(TfLiteContext* context, - const TfLiteTensor* input_1, - const TfLiteTensor* input_2, TfLiteTensor* output, - TfLiteSubParams* params, OpData* op_params, - int op_sign) { - TF_LITE_ENSURE(context, - output->type == kTfLiteUInt8 || output->type == kTfLiteInt8); +TfLiteStatus PrepareGeneralSubOp(TfLiteContext* context, + const TfLiteTensor* input_1, + const TfLiteTensor* input_2, + TfLiteTensor* output, TfLiteSubParams* params, + OpData* op_params, int op_sign) { + TF_LITE_ENSURE(context, output->type == kTfLiteUInt8 || + output->type == kTfLiteInt8 || + output->type == kTfLiteInt16); const auto& input1_quantization_params = input_1->params; const auto& input2_quantization_params = input_2->params; const auto& output_quantization_params = output->params; @@ -98,6 +104,9 @@ TfLiteStatus Prepare8BitSubOp(TfLiteContext* context, if (output->type == kTfLiteUInt8) { integer_type_min = std::numeric_limits::min(); integer_type_max = std::numeric_limits::max(); + } else if (output->type == kTfLiteInt16) { + integer_type_min = std::numeric_limits::min(); + integer_type_max = std::numeric_limits::max(); } else { // output->type == kTfLiteInt8 integer_type_min = std::numeric_limits::min(); @@ -120,7 +129,11 @@ TfLiteStatus Prepare8BitSubOp(TfLiteContext* context, op_params->input1_offset = -input1_quantization_params.zero_point; op_params->input2_offset = -input2_quantization_params.zero_point; op_params->output_offset = output_quantization_params.zero_point; - op_params->left_shift = 20; + + // The shift is set to 15 in case of 16-bit and 20 in case of 8-bit, + // accordingly. In case of 16-bit we have 65535 << 15 which is less than 1 << + // 31, therefore the addition will still fit in a 32 bit accumulator. + op_params->left_shift = output->type == kTfLiteInt16 ? 15 : 20; const double twice_max_input_scale = 2 * std::max(input1_quantization_params.scale, input2_quantization_params.scale); @@ -146,13 +159,15 @@ TfLiteStatus Prepare8BitSubOp(TfLiteContext* context, TF_LITE_ENSURE_STATUS(CalculateActivationRangeQuantized( context, params->activation, output, &op_params->output_activation_min, &op_params->output_activation_max)); + return kTfLiteOk; } -TfLiteStatus PrepareInt16SubOp(TfLiteContext* context, - const TfLiteTensor* input1, - const TfLiteTensor* input2, TfLiteTensor* output, - TfLiteSubParams* params, OpData* data) { +TfLiteStatus PrepareInt16SubOpPOT(TfLiteContext* context, + const TfLiteTensor* input1, + const TfLiteTensor* input2, + TfLiteTensor* output, TfLiteSubParams* params, + OpData* data) { // 16bit -> 16bit special quantized path, supporting only a rather // narrow case of quantization parameters: zero_points must all be 0 // ("symmetric quantization") and scales must be power-of-two (which @@ -219,12 +234,51 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { output_size = TfLiteIntArrayCopy(input1->dims); } - if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8) { - TF_LITE_ENSURE_OK(context, Prepare8BitSubOp(context, input1, input2, output, - params, data, -1)); + // 8bit -> 8bit general quantized path, with general rescalings + // as well as, 16bit -> 16bit with general rescalings + bool pot_scale_int16 = true; + + bool input1_scale_is_pot = false; + bool input2_scale_is_pot = false; + bool output_scale_is_pot = false; + + int input1_scale_log2_rounded{0}; + int input2_scale_log2_rounded{0}; + int output_scale_log2_rounded{0}; + + if (input1->type == kTfLiteInt16 && input2->type == kTfLiteInt16 && + output->type == kTfLiteInt16) { + // In case of 16-bit, there are two implementation: + // the scale parameter is a general number + // the scale parameter is POT and + // zero_point is zero for inputs/output. + pot_scale_int16 = (input1->params.zero_point == 0) && + (input2->params.zero_point == 0) && + (output->params.zero_point == 0); + + input1_scale_is_pot = + CheckedLog2(input1->params.scale, &input1_scale_log2_rounded); + + input2_scale_is_pot = + CheckedLog2(input2->params.scale, &input2_scale_log2_rounded); + + output_scale_is_pot = + CheckedLog2(output->params.scale, &output_scale_log2_rounded); + + pot_scale_int16 &= + input1_scale_is_pot && input2_scale_is_pot && output_scale_is_pot; + } + + data->pot_scale_int16 = pot_scale_int16; + + if (output->type == kTfLiteUInt8 || output->type == kTfLiteInt8 || + !pot_scale_int16) { + TF_LITE_ENSURE_OK(context, PrepareGeneralSubOp(context, input1, input2, + output, params, data, -1)); } else if (output->type == kTfLiteInt16) { - TF_LITE_ENSURE_OK(context, PrepareInt16SubOp(context, input1, input2, - output, params, data)); + // LSTM-special case with scale parameter of POT + TF_LITE_ENSURE_OK(context, PrepareInt16SubOpPOT(context, input1, input2, + output, params, data)); } return context->ResizeTensor(context, output, output_size); @@ -332,6 +386,15 @@ void EvalQuantized(TfLiteContext* context, TfLiteNode* node, } else { TF_LITE_SUB(reference_integer_ops, Add, int8_t); } + } else if (!data->pot_scale_int16) { + if (need_broadcast) { + TF_LITE_SUB(reference_ops, BroadcastAdd4DSlow, int16_t); + } else { + reference_ops::Add(op_params, GetTensorShape(input1), + GetTensorData(input1), GetTensorShape(input2), + GetTensorData(input2), GetTensorShape(output), + GetTensorData(output), false); + } } else if (output->type == kTfLiteUInt8) { if (kernel_type == kReference) { if (need_broadcast) { diff --git a/tensorflow/lite/kernels/sub_test.cc b/tensorflow/lite/kernels/sub_test.cc index 67054fe4903..24d9c251afb 100644 --- a/tensorflow/lite/kernels/sub_test.cc +++ b/tensorflow/lite/kernels/sub_test.cc @@ -304,6 +304,10 @@ TEST(QuantizedSubOpModel, QuantizedTestsNoActivationInt8) { QuantizedTestsNoActivation(); } +TEST(QuantizedSubOpModel, QuantizedTestsNoActivationGenericInt16) { + QuantizedTestsNoActivation(); +} + template void QuantizedTestsActivationRELU_N1_TO_1() { float kQuantizedTolerance = GetTolerance(-1.0, 1.0); @@ -365,6 +369,10 @@ TEST(QuantizedSubOpModel, QuantizedVariousInputShapesInt8) { QuantizedVariousInputShapes(); } +TEST(QuantizedSubOpModel, QuantizedVariousInputShapesInt16) { + QuantizedVariousInputShapes(); +} + template void QuantizedWithBroadcast() { float kQuantizedTolerance = GetTolerance(-3.0, 3.0); @@ -393,6 +401,10 @@ TEST(QuantizedSubOpModel, QuantizedWithBroadcastInt8) { QuantizedWithBroadcast(); } +TEST(QuantizedSubOpModel, QuantizedWithBroadcastInt16) { + QuantizedWithBroadcast(); +} + TEST(QuantizedSubOpModel, QuantizedTestsNoActivationInt16) { const float kMin = -1.f; const float kMax = diff --git a/tensorflow/lite/schema/schema.fbs b/tensorflow/lite/schema/schema.fbs index 878acde1e16..baeb49f7b7a 100644 --- a/tensorflow/lite/schema/schema.fbs +++ b/tensorflow/lite/schema/schema.fbs @@ -583,6 +583,8 @@ table ConcatenationOptions { table AddOptions { fused_activation_function:ActivationFunctionType; + // Parameters supported by version 4. + pot_scale_int16:bool = true; } table MulOptions { @@ -704,6 +706,8 @@ table DepthToSpaceOptions { table SubOptions { fused_activation_function:ActivationFunctionType; + // Parameters supported by version 5 + pot_scale_int16:bool = true; } table DivOptions { diff --git a/tensorflow/lite/schema/schema_generated.h b/tensorflow/lite/schema/schema_generated.h index a6117dc72ab..a4691b70e49 100755 --- a/tensorflow/lite/schema/schema_generated.h +++ b/tensorflow/lite/schema/schema_generated.h @@ -4742,22 +4742,29 @@ flatbuffers::Offset CreateConcatenationOptions(flatbuffers struct AddOptionsT : public flatbuffers::NativeTable { typedef AddOptions TableType; + bool pot_scale_int16; tflite::ActivationFunctionType fused_activation_function; AddOptionsT() - : fused_activation_function(tflite::ActivationFunctionType_NONE) { + : pot_scale_int16(true), + fused_activation_function(tflite::ActivationFunctionType_NONE) { } }; struct AddOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef AddOptionsT NativeTableType; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_FUSED_ACTIVATION_FUNCTION = 4 + VT_FUSED_ACTIVATION_FUNCTION = 4, + VT_POT_SCALE_INT16 = 6 }; + bool pot_scale_int16() const { + return GetField(VT_POT_SCALE_INT16, 0) != 0; + } tflite::ActivationFunctionType fused_activation_function() const { return static_cast(GetField(VT_FUSED_ACTIVATION_FUNCTION, 0)); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && + VerifyField(verifier, VT_POT_SCALE_INT16) && VerifyField(verifier, VT_FUSED_ACTIVATION_FUNCTION) && verifier.EndTable(); } @@ -5907,22 +5914,29 @@ flatbuffers::Offset CreateDepthToSpaceOptions(flatbuffers:: struct SubOptionsT : public flatbuffers::NativeTable { typedef SubOptions TableType; + bool pot_scale_int16; tflite::ActivationFunctionType fused_activation_function; SubOptionsT() - : fused_activation_function(tflite::ActivationFunctionType_NONE) { + : pot_scale_int16(true), + fused_activation_function(tflite::ActivationFunctionType_NONE) { } }; struct SubOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table { typedef SubOptionsT NativeTableType; enum FlatBuffersVTableOffset FLATBUFFERS_VTABLE_UNDERLYING_TYPE { - VT_FUSED_ACTIVATION_FUNCTION = 4 + VT_FUSED_ACTIVATION_FUNCTION = 4, + VT_POT_SCALE_INT16 = 6 }; + bool pot_scale_int16() const { + return GetField(VT_POT_SCALE_INT16, 0) != 0; + } tflite::ActivationFunctionType fused_activation_function() const { return static_cast(GetField(VT_FUSED_ACTIVATION_FUNCTION, 0)); } bool Verify(flatbuffers::Verifier &verifier) const { return VerifyTableStart(verifier) && + VerifyField(verifier, VT_POT_SCALE_INT16) && VerifyField(verifier, VT_FUSED_ACTIVATION_FUNCTION) && verifier.EndTable(); } diff --git a/tensorflow/lite/toco/tflite/op_version.cc b/tensorflow/lite/toco/tflite/op_version.cc index 567d000dab6..222be969560 100644 --- a/tensorflow/lite/toco/tflite/op_version.cc +++ b/tensorflow/lite/toco/tflite/op_version.cc @@ -53,12 +53,15 @@ std::string GetMinimumRuntimeVersionForModel(const Model& model) { {{OperatorType::kDepthwiseConv, 5}, kPendingReleaseOpVersion}, {{OperatorType::kAdd, 1}, "1.5.0"}, {{OperatorType::kAdd, 2}, "1.14.0"}, + {{OperatorType::kAdd, 3}, kPendingReleaseOpVersion}, {{OperatorType::kAddN, 1}, "1.14.0"}, {{OperatorType::kSpaceToBatchND, 1}, "1.6.0"}, {{OperatorType::kSpaceToBatchND, 2}, "1.14.0"}, {{OperatorType::kSub, 1}, "1.6.0"}, {{OperatorType::kSub, 2}, "1.14.0"}, + {{OperatorType::kSub, 3}, "1.15.0"}, {{OperatorType::kSub, 4}, kPendingReleaseOpVersion}, + {{OperatorType::kSub, 5}, kPendingReleaseOpVersion}, {{OperatorType::kDiv, 1}, "1.6.0"}, {{OperatorType::kBatchToSpaceND, 1}, "1.6.0"}, {{OperatorType::kBatchToSpaceND, 2}, "1.14.0"}, diff --git a/tensorflow/lite/toco/tflite/operator.cc b/tensorflow/lite/toco/tflite/operator.cc index 794691f5724..585b15bae2e 100644 --- a/tensorflow/lite/toco/tflite/operator.cc +++ b/tensorflow/lite/toco/tflite/operator.cc @@ -276,10 +276,10 @@ class Sub : public BuiltinOperator 4) { + if (op_sig.options.addsub.need_broadcast && + op_sig.options.addsub.num_dims > 4) { return 3; } if (op_sig.input_types.at(0) == TensorType_INT8) { @@ -542,7 +560,7 @@ int GetBuiltinOperatorVersion(const OpSignature& op_sig) { } } return 1; - case BuiltinOperator_ADD: + case BuiltinOperator_SPACE_TO_DEPTH: case BuiltinOperator_SPLIT_V: case BuiltinOperator_SUM: @@ -669,6 +687,26 @@ OpSignature GetOpSignature(const OperatorCode* op_code, const Operator* op, } } break; + case BuiltinOperator_ADD: { + auto add_option = op->builtin_options_as_AddOptions(); + op_sig.options.addsub.pot_scale_int16 = true; + if (add_option) { + op_sig.options.addsub.pot_scale_int16 = add_option->pot_scale_int16(); + } + } break; + + case BuiltinOperator_SUB: { + auto sub_option = op->builtin_options_as_SubOptions(); + op_sig.options.addsub.need_broadcast = + !HaveSameShapes(subgraph, op, 0, 1); + op_sig.options.addsub.num_dims = + std::max(GetNumDims(subgraph, op, 0), GetNumDims(subgraph, op, 1)); + op_sig.options.addsub.pot_scale_int16 = true; + if (sub_option) { + op_sig.options.addsub.pot_scale_int16 = sub_option->pot_scale_int16(); + } + } break; + case BuiltinOperator_LSTM: { auto lstm_option = op->builtin_options_as_LSTMOptions(); if (lstm_option) { @@ -714,7 +752,7 @@ OpSignature GetOpSignature(const OperatorCode* op_code, const Operator* op, case BuiltinOperator_TRANSPOSE: { op_sig.options.single_input_op.num_dims = GetNumDims(subgraph, op, 0); } break; - case BuiltinOperator_SUB: + case BuiltinOperator_DIV: case BuiltinOperator_MAXIMUM: case BuiltinOperator_MINIMUM: { diff --git a/tensorflow/lite/tools/versioning/op_version.h b/tensorflow/lite/tools/versioning/op_version.h index 71362001387..67a7b79fe38 100644 --- a/tensorflow/lite/tools/versioning/op_version.h +++ b/tensorflow/lite/tools/versioning/op_version.h @@ -63,6 +63,11 @@ typedef struct { int32_t num_dims; bool need_broadcast; } broadcast; + struct { + bool pot_scale_int16; + int32_t num_dims; + bool need_broadcast; + } addsub; struct { bool is_per_channel_quantized; } conv_2d; diff --git a/tensorflow/lite/tools/versioning/runtime_version.cc b/tensorflow/lite/tools/versioning/runtime_version.cc index ccbbaa27d68..5a454224b92 100644 --- a/tensorflow/lite/tools/versioning/runtime_version.cc +++ b/tensorflow/lite/tools/versioning/runtime_version.cc @@ -72,6 +72,8 @@ std::string FindMinimumRuntimeVersionForOp(tflite::BuiltinOperator op_code, {{BuiltinOperator_DEPTHWISE_CONV_2D, 6}, "2.3.0"}, {{BuiltinOperator_ADD, 1}, "1.5.0"}, {{BuiltinOperator_ADD, 2}, "1.14.0"}, + {{BuiltinOperator_ADD, 3}, kPendingReleaseVersion}, + {{BuiltinOperator_ADD, 4}, kPendingReleaseVersion}, {{BuiltinOperator_ADD_N, 1}, "1.14.0"}, {{BuiltinOperator_SPACE_TO_BATCH_ND, 1}, "1.6.0"}, {{BuiltinOperator_SPACE_TO_BATCH_ND, 2}, "1.14.0"}, @@ -80,6 +82,7 @@ std::string FindMinimumRuntimeVersionForOp(tflite::BuiltinOperator op_code, {{BuiltinOperator_SUB, 2}, "1.14.0"}, {{BuiltinOperator_SUB, 3}, "2.3.0"}, {{BuiltinOperator_SUB, 4}, kPendingReleaseVersion}, + {{BuiltinOperator_SUB, 5}, kPendingReleaseVersion}, {{BuiltinOperator_DENSIFY, 1}, "2.2.0"}, {{BuiltinOperator_DIV, 1}, "1.6.0"}, {{BuiltinOperator_DIV, 2}, "2.3.0"}, From 52c4411c26fd27377da12da6d1158e8b49b22034 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 12:35:41 -0700 Subject: [PATCH 0387/1017] Split GPU Compatibility Lib into two. PiperOrigin-RevId: 324867524 Change-Id: Ibf5403a725aee72ebeba4d6681660c289326eb69 --- .../delegates/gpu/java/src/main/native/BUILD | 2 +- .../java/src/main/native/gpu_delegate_jni.cc | 6 +- .../acceleration/compatibility/BUILD | 38 ++++++- .../compatibility/gpu_compatibility.cc | 9 +- .../compatibility/gpu_compatibility.h | 38 +++---- .../gpu_compatibility_recommender.cc | 30 ++++++ .../gpu_compatibility_recommender.h | 64 +++++++++++ .../gpu_compatibility_recommender_test.cc | 100 ++++++++++++++++++ .../compatibility/gpu_compatibility_test.cc | 61 +++++++++++ 9 files changed, 310 insertions(+), 38 deletions(-) create mode 100644 tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.cc create mode 100644 tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h create mode 100644 tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender_test.cc create mode 100644 tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_test.cc diff --git a/tensorflow/lite/delegates/gpu/java/src/main/native/BUILD b/tensorflow/lite/delegates/gpu/java/src/main/native/BUILD index 00b56bb0c06..7b340e20562 100644 --- a/tensorflow/lite/delegates/gpu/java/src/main/native/BUILD +++ b/tensorflow/lite/delegates/gpu/java/src/main/native/BUILD @@ -30,7 +30,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/gl:egl_environment", "//tensorflow/lite/delegates/gpu/gl:request_gpu_info", "//tensorflow/lite/experimental/acceleration/compatibility:android_info", - "//tensorflow/lite/experimental/acceleration/compatibility:gpu_compatibility", + "//tensorflow/lite/experimental/acceleration/compatibility:gpu_compatibility_recommender", "//tensorflow/lite/java/jni", "@com_google_absl//absl/status", ], diff --git a/tensorflow/lite/delegates/gpu/java/src/main/native/gpu_delegate_jni.cc b/tensorflow/lite/delegates/gpu/java/src/main/native/gpu_delegate_jni.cc index d31d058b796..c4571100818 100644 --- a/tensorflow/lite/delegates/gpu/java/src/main/native/gpu_delegate_jni.cc +++ b/tensorflow/lite/delegates/gpu/java/src/main/native/gpu_delegate_jni.cc @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/egl_environment.h" #include "tensorflow/lite/delegates/gpu/gl/request_gpu_info.h" #include "tensorflow/lite/experimental/acceleration/compatibility/android_info.h" -#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h" +#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h" #ifdef __cplusplus extern "C" { @@ -74,13 +74,13 @@ class CompatibilityListHelper { } bool IsDelegateSupportedOnThisDevice() { - return compatibility_list_.Includes(android_info_, gpu_info_); + return compatibility_recommender_.Includes(android_info_, gpu_info_); } private: tflite::acceleration::AndroidInfo android_info_; tflite::gpu::GpuInfo gpu_info_; - tflite::acceleration::GPUCompatibilityList compatibility_list_; + tflite::acceleration::GPUCompatibilityRecommender compatibility_recommender_; }; } // namespace diff --git a/tensorflow/lite/experimental/acceleration/compatibility/BUILD b/tensorflow/lite/experimental/acceleration/compatibility/BUILD index 78a9d2eb8d8..6adb6daaa6f 100644 --- a/tensorflow/lite/experimental/acceleration/compatibility/BUILD +++ b/tensorflow/lite/experimental/acceleration/compatibility/BUILD @@ -152,7 +152,6 @@ cc_library( ":android_info", ":database_fbs", ":devicedb", - "//tensorflow/lite/delegates/gpu:delegate", "//tensorflow/lite/delegates/gpu/common:gpu_info", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", @@ -160,4 +159,41 @@ cc_library( ], ) +cc_test( + name = "gpu_compatibility_test", + srcs = ["gpu_compatibility_test.cc"], + deps = [ + ":gpu_compatibility", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "gpu_compatibility_recommender", + srcs = [ + "gpu_compatibility_recommender.cc", + ], + hdrs = [ + "gpu_compatibility_recommender.h", + ], + deps = [ + ":android_info", + ":gpu_compatibility", + "//tensorflow/lite/delegates/gpu:delegate", + "//tensorflow/lite/delegates/gpu/common:gpu_info", + ], +) + +cc_test( + name = "gpu_compatibility_recommender_test", + srcs = ["gpu_compatibility_recommender_test.cc"], + tags = ["notap"], # Needs to be built with --copt=-DCL_DELEGATE_NO_GL + deps = [ + ":gpu_compatibility_recommender", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + tflite_portable_test_suite() diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.cc b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.cc index e04f5d18db4..1911d26b8df 100644 --- a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.cc +++ b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.cc @@ -89,13 +89,8 @@ bool GPUCompatibilityList::Includes( return variables[gpu::kStatus] == std::string(gpu::kStatusSupported); } -TfLiteGpuDelegateOptionsV2 GPUCompatibilityList::GetBestOptionsFor( - const AndroidInfo& /* android_info */, - const ::tflite::gpu::GpuInfo& /* gpu_info */) const { - // This method is for forwards-compatibility: the list may later include - // information about which backend to choose (OpenGL/OpenCL/Vulkan) or other - // options. - return TfLiteGpuDelegateOptionsV2Default(); +bool GPUCompatibilityList::IsDatabaseLoaded() const { + return database_ != nullptr; } } // namespace acceleration diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h index f975fe04f22..873151dca66 100644 --- a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h +++ b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h @@ -19,7 +19,6 @@ limitations under the License. #include #include "tensorflow/lite/delegates/gpu/common/gpu_info.h" -#include "tensorflow/lite/delegates/gpu/delegate.h" #include "tensorflow/lite/experimental/acceleration/compatibility/android_info.h" #include "tensorflow/lite/experimental/acceleration/compatibility/devicedb.h" @@ -32,54 +31,41 @@ namespace acceleration { // Android version, OpenGL ES version, GPU chipset etc. The support is based on // measure stability, correctness and peformance. For more detail see README.md. // +// Reads from the flatbuffer. // Example usage: -// tflite::Interpreter* interpreter = ... ; +// tflite::acceleration::GPUCompatibilityList list; // tflite::acceleration::AndroidInfo android_info; // tflite::gpu::GpuInfo gpu_info; -// EXPECT_OK(tflite::acceleration::RequestAndroidInfo(&android_info)); -// EXPECT_OK(tflite::gpu::gl::EglEnvironment::NewEglEnvironment(&env)); -// EXPECT_OK(tflite::gpu::gl::RequestGpuInfo(&tflite_gpu_info)); -// tflite::acceleration::GPUCompatibilityList list; -// TfLiteDelegate* gpu_delegate = nullptr; -// TfLiteGpuDelegateOptions gpu_options; -// if (list.Includes(android_info, gpu_info)) { -// gpu_options = list.BestOptionsFor(android_info, gpu_info); -// gpu_delegate = TfLiteGpuDelegateCreate(&gpu_options); -// EXPECT_EQ(interpreter->ModifyGraphWithDelegate(gpu_delegate), TfLiteOk); -// } else { -// // Fallback path. +// ... +// if(list.Includes(android_info, gpu_info)){ +// // SUPPORTED. +// } else{ +// // UNSUPPORTED. // } class GPUCompatibilityList { public: // Construct list from bundled data. GPUCompatibilityList(); + // Constructs list from the given flatbuffer. + explicit GPUCompatibilityList( + const unsigned char* compatibility_list_flatbuffer); // Returns true if the provided device specs are supported by the database. bool Includes(const AndroidInfo& android_info, const ::tflite::gpu::GpuInfo& gpu_info) const; - - // Returns the best TfLiteGpuDelegateOptionsV2 for the provided device specs - // based on the database. The output can be modified as desired before passing - // to delegate creation. - TfLiteGpuDelegateOptionsV2 GetBestOptionsFor( - const AndroidInfo& android_info, - const ::tflite::gpu::GpuInfo& gpu_info) const; - // Convert android_info and gpu_info into a set of variables used for querying // the list, and update variables from list data. See variables.h // and devicedb.h for more information. std::map CalculateVariables( const AndroidInfo& android_info, const ::tflite::gpu::GpuInfo& gpu_info) const; - GPUCompatibilityList(const GPUCompatibilityList&) = delete; GPUCompatibilityList& operator=(const GPUCompatibilityList&) = delete; + // Indicates if the database is loaded. + bool IsDatabaseLoaded() const; protected: - explicit GPUCompatibilityList( - const unsigned char* compatibility_list_flatbuffer); const DeviceDatabase* database_; }; - } // namespace acceleration } // namespace tflite diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.cc b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.cc new file mode 100644 index 00000000000..1b625913323 --- /dev/null +++ b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.cc @@ -0,0 +1,30 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h" + +namespace tflite { +namespace acceleration { + +TfLiteGpuDelegateOptionsV2 GPUCompatibilityRecommender::GetBestOptionsFor( + const AndroidInfo& /* android_info */, + const ::tflite::gpu::GpuInfo& /* gpu_info */) const { + // This method is for forwards-compatibility: the list may later include + // information about which backend to choose (OpenGL/OpenCL/Vulkan) or other + // options. + return TfLiteGpuDelegateOptionsV2Default(); +} + +} // namespace acceleration +} // namespace tflite diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h new file mode 100644 index 00000000000..4443cfdf70f --- /dev/null +++ b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h @@ -0,0 +1,64 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_COMPATIBILITY_GPU_COMPATIBILITY_RECOMMENDER_H_ +#define TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_COMPATIBILITY_GPU_COMPATIBILITY_RECOMMENDER_H_ + +#include "tensorflow/lite/delegates/gpu/common/gpu_info.h" +#include "tensorflow/lite/delegates/gpu/delegate.h" +#include "tensorflow/lite/experimental/acceleration/compatibility/android_info.h" +#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h" + +namespace tflite { +namespace acceleration { + +// This class recommends best TfLiteGPU delegate options for Android devices. +// +// Example usage: +// tflite::Interpreter* interpreter = ... ; +// tflite::acceleration::AndroidInfo android_info; +// tflite::gpu::GpuInfo gpu_info; +// CHECK(tflite::acceleration::RequestAndroidInfo(&android_info)); +// CHECK(tflite::gpu::gl::EglEnvironment::NewEglEnvironment(&env)); +// CHECK(tflite::gpu::gl::RequestGpuInfo(&tflite_gpu_info)); +// tflite::acceleration::GPUCompatibilityRecommender recommender; +// TfLiteDelegate* gpu_delegate = nullptr; +// TfLiteGpuDelegateOptions gpu_options; +// if (list.Includes(android_info, gpu_info)) { +// gpu_options = recommender.BestOptionsFor(android_info, gpu_info); +// gpu_delegate = TfLiteGpuDelegateCreate(&gpu_options); +// CHECK_EQ(interpreter->ModifyGraphWithDelegate(gpu_delegate), TfLiteOk); +// } else { +// // Fallback path. +// } + +class GPUCompatibilityRecommender : public GPUCompatibilityList { + public: + GPUCompatibilityRecommender() {} + GPUCompatibilityRecommender(const GPUCompatibilityRecommender&) = delete; + GPUCompatibilityRecommender& operator=(const GPUCompatibilityRecommender&) = + delete; + + // Returns the best TfLiteGpuDelegateOptionsV2 for the provided device specs + // based on the database. The output can be modified as desired before passing + // to delegate creation. + TfLiteGpuDelegateOptionsV2 GetBestOptionsFor( + const AndroidInfo& android_info, + const ::tflite::gpu::GpuInfo& gpu_info) const; +}; + +} // namespace acceleration +} // namespace tflite + +#endif // TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_COMPATIBILITY_GPU_COMPATIBILITY_RECOMMENDER_H_ diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender_test.cc b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender_test.cc new file mode 100644 index 00000000000..ebf793d5a94 --- /dev/null +++ b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender_test.cc @@ -0,0 +1,100 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h" + +#include +#include + +namespace { + +class GPUCompatibilityRecommenderTest : public ::testing::Test { + protected: + GPUCompatibilityRecommenderTest() { + recommender_ = + absl::make_unique(); + } + + std::unique_ptr + recommender_; +}; + +TEST_F(GPUCompatibilityRecommenderTest, Load) { + EXPECT_TRUE(recommender_->IsDatabaseLoaded()); +} + +TEST_F(GPUCompatibilityRecommenderTest, ReturnsSupportedForFullMatch) { + tflite::acceleration::AndroidInfo android_info = { + .android_sdk_version = "28", + .model = "redmi_note_7G960F", + .device = "lavender", + .manufacturer = "xiaomi"}; + tflite::gpu::GpuInfo tflite_gpu_info = { + .renderer_name = "adreno_(tm)_512", + .major_version = 3, + .minor_version = 2, + }; + EXPECT_TRUE(recommender_->Includes(android_info, tflite_gpu_info)); +} + +TEST_F(GPUCompatibilityRecommenderTest, ReturnsUnsupported) { + tflite::acceleration::AndroidInfo android_info = {.android_sdk_version = "28", + .model = "sm_g960f", + .device = "starlte", + .manufacturer = "samsung"}; + tflite::gpu::GpuInfo tflite_gpu_info = { + .renderer_name = "mali_g72", + .major_version = 3, + .minor_version = 2, + }; + + EXPECT_FALSE(recommender_->Includes(android_info, tflite_gpu_info)); +} + +TEST_F(GPUCompatibilityRecommenderTest, MissingInfoReturnsUnsupported) { + tflite::acceleration::AndroidInfo android_info = {.android_sdk_version = "23", + .model = "sm_g532f", + .device = "grandpplte", + .manufacturer = "samsung"}; + tflite::gpu::GpuInfo tflite_gpu_info = { + .renderer_name = "mali_t720", + .major_version = 3, + .minor_version = 1, + }; + EXPECT_FALSE(recommender_->Includes(android_info, tflite_gpu_info)); +} + +TEST_F(GPUCompatibilityRecommenderTest, ReturnsDefaultOptions) { + tflite::acceleration::AndroidInfo android_info; + tflite::gpu::GpuInfo tflite_gpu_info; + auto default_options = TfLiteGpuDelegateOptionsV2Default(); + auto best_options = + recommender_->GetBestOptionsFor(android_info, tflite_gpu_info); + EXPECT_EQ(best_options.is_precision_loss_allowed, + default_options.is_precision_loss_allowed); + EXPECT_EQ(best_options.inference_preference, + default_options.inference_preference); + EXPECT_EQ(best_options.inference_priority1, + default_options.inference_priority1); + EXPECT_EQ(best_options.inference_priority2, + default_options.inference_priority2); + EXPECT_EQ(best_options.inference_priority3, + default_options.inference_priority3); + EXPECT_EQ(best_options.experimental_flags, + default_options.experimental_flags); + EXPECT_EQ(best_options.max_delegated_partitions, + default_options.max_delegated_partitions); +} + +} // namespace diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_test.cc b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_test.cc new file mode 100644 index 00000000000..d300867a8b0 --- /dev/null +++ b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_test.cc @@ -0,0 +1,61 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h" + +#include + +#include +#include + +namespace { + +class GPUCompatibilityTest : public ::testing::Test { + protected: + GPUCompatibilityTest() { + list_ = absl::make_unique(); + } + + std::unique_ptr list_; +}; + +TEST_F(GPUCompatibilityTest, Load) { EXPECT_TRUE(list_->IsDatabaseLoaded()); } + +TEST_F(GPUCompatibilityTest, ReturnsSupportedForFullMatch) { + tflite::acceleration::AndroidInfo android_info = {.android_sdk_version = "27", + .model = "cph1803", + .device = "cph1803", + .manufacturer = "Oppo"}; + tflite::gpu::GpuInfo tflite_gpu_info = { + .renderer_name = "Adreno (TM) 506", + .major_version = 3, + .minor_version = 2, + }; + EXPECT_TRUE(list_->Includes(android_info, tflite_gpu_info)); +} + +TEST_F(GPUCompatibilityTest, ReturnsUnsupportedForFullMatch) { + tflite::acceleration::AndroidInfo android_info = {.android_sdk_version = "28", + .model = "SM-G960F", + .device = "starlte", + .manufacturer = "Samsung"}; + tflite::gpu::GpuInfo tflite_gpu_info = { + .renderer_name = "Mali-G72", + .major_version = 3, + .minor_version = 2, + }; + EXPECT_FALSE(list_->Includes(android_info, tflite_gpu_info)); +} + +} // namespace From 7972c284ac705b06066dffc85b7a36ef9662fb74 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 4 Aug 2020 19:56:52 +0000 Subject: [PATCH 0388/1017] fixed bug that doesn't return on TF_InvalidArugment --- tensorflow/c/kernels/histogram_summary_op.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tensorflow/c/kernels/histogram_summary_op.cc b/tensorflow/c/kernels/histogram_summary_op.cc index a3dfc102472..a816305d9e5 100644 --- a/tensorflow/c/kernels/histogram_summary_op.cc +++ b/tensorflow/c/kernels/histogram_summary_op.cc @@ -92,13 +92,13 @@ static void HistogramSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { std::ostringstream err; err << "Nan in summary histogram for: " << k->op_node_name; TF_SetStatus(status_wrapper.s, TF_INVALID_ARGUMENT, err.str().c_str()); - break; + return; } else if (Eigen::numext::isinf(double_val)) { std::ostringstream err; err << "Infinity in Histogram for: " << k->op_node_name; TF_SetStatus(status_wrapper.s, TF_INVALID_ARGUMENT, err.str().c_str()); - break; + return; } histo.Add(double_val); } @@ -109,16 +109,13 @@ static void HistogramSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { v->set_tag(tag.data(), tag.size()); histo.EncodeToProto(v->mutable_histo(), false /* Drop zero buckets */); - // Must use new status for TF_AllocateOutput if previous status is set - // because of an invalid values argument. - StatusWrapper allocation_status_wrapper; TensorWrapper summary_tensor_wrapper; summary_tensor_wrapper.t = TF_AllocateOutput(ctx, 0, TF_ExpectedOutputDataType(ctx, 0), nullptr, 0, - sizeof(tensorflow::tstring), allocation_status_wrapper.s); + sizeof(tensorflow::tstring), status_wrapper.s); - if (TF_GetCode(allocation_status_wrapper.s) != TF_OK){ - TF_OpKernelContext_Failure(ctx, allocation_status_wrapper.s); + if (TF_GetCode(status_wrapper.s) != TF_OK){ + TF_OpKernelContext_Failure(ctx, status_wrapper.s); return; } tensorflow::tstring* output_tstring = reinterpret_cast( From 53f0fa916fad58b5f914dc983f1068a005e98606 Mon Sep 17 00:00:00 2001 From: Karim Nosir Date: Tue, 4 Aug 2020 12:53:40 -0700 Subject: [PATCH 0389/1017] NFC: Remove unused dependencies PiperOrigin-RevId: 324871458 Change-Id: I9c11f4a67a27a38d4288a80df6bc4b4f9f096955 --- tensorflow/lite/tools/optimize/BUILD | 8 -------- tensorflow/lite/tools/optimize/calibration/BUILD | 4 ---- 2 files changed, 12 deletions(-) diff --git a/tensorflow/lite/tools/optimize/BUILD b/tensorflow/lite/tools/optimize/BUILD index 146f869a906..ab153afc2cf 100644 --- a/tensorflow/lite/tools/optimize/BUILD +++ b/tensorflow/lite/tools/optimize/BUILD @@ -49,7 +49,6 @@ cc_binary( srcs = ["modify_model_interface_main.cc"], deps = [ ":modify_model_interface", - ":quantize_model", ], ) @@ -90,8 +89,6 @@ cc_library( hdrs = ["quantization_wrapper.h"], deps = [ ":quantization_wrapper_utils", - "//tensorflow/lite:framework", - "//tensorflow/lite/core/api", "//tensorflow/lite/schema:schema_fbs", "//tensorflow/lite/tools/optimize:quantize_model", "@flatbuffers", @@ -115,7 +112,6 @@ cc_library( "//tensorflow/lite/schema:schema_fbs", "//third_party/eigen3", "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", ], ) @@ -130,7 +126,6 @@ cc_library( "//tensorflow/lite/kernels/internal:types", "//tensorflow/lite/schema:schema_fbs", "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", ], ) @@ -159,7 +154,6 @@ cc_library( hdrs = ["operator_property.h"], deps = [ "//tensorflow/lite:framework", - "//tensorflow/lite/kernels/internal:types", "//tensorflow/lite/schema:schema_fbs", ], ) @@ -200,7 +194,6 @@ cc_library( ":quantization_utils", ":model_utils", "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", "@com_google_absl//absl/container:flat_hash_map", "@flatbuffers", "//tensorflow/lite:framework", @@ -245,7 +238,6 @@ cc_library( srcs = ["test_util.cc"], hdrs = ["test_util.h"], deps = [ - "//tensorflow/lite:framework", "//tensorflow/lite/core/api", "@com_google_googletest//:gtest", "@flatbuffers", diff --git a/tensorflow/lite/tools/optimize/calibration/BUILD b/tensorflow/lite/tools/optimize/calibration/BUILD index 06183353e44..f641b151aa9 100644 --- a/tensorflow/lite/tools/optimize/calibration/BUILD +++ b/tensorflow/lite/tools/optimize/calibration/BUILD @@ -51,7 +51,6 @@ cc_library( "//tensorflow/lite/schema:schema_fbs", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", "@flatbuffers", ], ) @@ -105,7 +104,6 @@ cc_test( deps = [ ":logging_op_resolver", "//tensorflow/lite:framework", - "//tensorflow/lite/kernels:builtin_ops", "@com_google_googletest//:gtest", ], ) @@ -120,7 +118,6 @@ cc_library( "//tensorflow/lite:framework", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", ], ) @@ -130,7 +127,6 @@ cc_library( hdrs = ["calibration_logger.h"], copts = tflite_copts(), deps = [ - "//tensorflow/lite:framework", "//tensorflow/lite:minimal_logging", "//tensorflow/lite/c:common", "//tensorflow/lite/core/api", From 8ef70baf379615028d34395e8d44865e3df3828f Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 4 Aug 2020 20:25:26 +0000 Subject: [PATCH 0390/1017] moved TF_OFFSET_OF_END to c_api_macros --- tensorflow/c/BUILD | 1 + tensorflow/c/c_api_macros.h | 7 +++++++ tensorflow/c/tf_tensor.h | 8 +------- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index 410fc22069f..4a0ab232200 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -299,6 +299,7 @@ cc_library( hdrs = ["tf_tensor.h"], visibility = ["//visibility:public"], deps = [ + ":c_api_macros", ":tensor_interface", ":tf_datatype", ":tf_status", diff --git a/tensorflow/c/c_api_macros.h b/tensorflow/c/c_api_macros.h index 85c9507db87..ce24e4d8cbd 100644 --- a/tensorflow/c/c_api_macros.h +++ b/tensorflow/c/c_api_macros.h @@ -30,4 +30,11 @@ limitations under the License. #endif // _WIN32 #endif // SWIG +// Macro used to calculate struct size for maintaining ABI stability across +// different struct implementations. +#ifndef TF_OFFSET_OF_END +#define TF_OFFSET_OF_END(TYPE, MEMBER) (offsetof(TYPE, MEMBER) + \ + sizeof(((TYPE *)0)->MEMBER)) +#endif // TF_OFFSET_OF_END + #endif // TENSORFLOW_C_C_API_MACROS_H_ diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index 91263fb3b7f..ced57df77d4 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -19,6 +19,7 @@ limitations under the License. #include #include +#include "tensorflow/c/c_api_macros.h" #include "tensorflow/c/tf_datatype.h" #include "tensorflow/c/tf_status.h" @@ -45,13 +46,6 @@ limitations under the License. extern "C" { #endif -// Macro used to calculate struct size for maintaining ABI stability across -// different struct implementations. -#ifndef TF_OFFSET_OF_END -#define TF_OFFSET_OF_END(TYPE, MEMBER) (offsetof(TYPE, MEMBER) + \ - sizeof(((TYPE *)0)->MEMBER)) -#endif // TF_OFFSET_OF_END - // Allocator Attributes used for tensor allocation. typedef struct TF_AllocatorAttributes { size_t struct_size; From 9bf172a8d8b83c7c21fffc39943f9eb069d2ede2 Mon Sep 17 00:00:00 2001 From: Robert David Date: Tue, 4 Aug 2020 13:24:03 -0700 Subject: [PATCH 0391/1017] Fix MeanStddevNormalization on ARM Mali: The OpenCL standard requires __local variables to be declared at __kernel scope. PiperOrigin-RevId: 324877612 Change-Id: I4dd665948878f04a1155d1583942940cbac38390 --- .../cl/kernels/mean_stddev_normalization.cc | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc index a6ce7e55253..bf2ae33ec6d 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc @@ -37,29 +37,32 @@ std::string GetReduceCode(size_t work_group_size_x, size_t work_group_size_y) { // Otherwise, implement a reduction using __local memory. Note this only works // with power-of-two work group sizes. return R"( -static inline float local_reduce(float input) { -#if (__OPENCL_C_VERSION__ >= 300 && __opencl_c_work_group_collective_functions) || \ - (__OPENCL_C_VERSION__ >= 200) - return work_group_reduce_add(input); -#else - __local float data[)" + +#if (__OPENCL_C_VERSION__ >= 200) && (__OPENCL_C_VERSION__ < 300) && \ + !defined(__opencl_c_work_group_collective_functions) + #define __opencl_c_work_group_collective_functions 1 +#endif + +#ifdef __opencl_c_work_group_collective_functions +#define local_reduce(input, tmp) work_group_reduce_add(input) +#else // !defined(__opencl_c_work_group_collective_functions) +static inline float local_reduce(float input, __local float tmp[)" + std::to_string(work_group_size_y) + "][" + - std::to_string(work_group_size_x) + R"(]; + std::to_string(work_group_size_x) + R"(]) { const size_t local_id_x = get_local_id(0); const size_t local_id_y = get_local_id(1); - data[local_id_y][local_id_x] = input; + tmp[local_id_y][local_id_x] = input; mem_fence(CLK_LOCAL_MEM_FENCE); size_t reduction_size = get_local_size(0) / 2; while (reduction_size > 0) { if (local_id_x < reduction_size) { - data[local_id_y][local_id_x] += data[local_id_y][local_id_x + reduction_size]; + tmp[local_id_y][local_id_x] += tmp[local_id_y][local_id_x + reduction_size]; } mem_fence(CLK_LOCAL_MEM_FENCE); reduction_size /= 2; } - return data[local_id_y][0]; + return tmp[local_id_y][0]; } -#endif +#endif // defined(__opencl_c_work_group_collective_functions) )"; } } // namespace @@ -86,6 +89,11 @@ std::string MeanStdDevNormalization::GetNormalizationCode() { c += R"(__attribute__((reqd_work_group_size(128, 1, 1))) __kernel void main_function( $0) { +#ifndef __opencl_c_work_group_collective_functions + __local float tmp[)" + + std::to_string(work_group_size_.y) + "][" + + std::to_string(work_group_size_.x) + R"(]; +#endif size_t B = get_global_id(1); if (get_global_id(2) > 0) { return; } if (B >= args.src_tensor.Batch()) { return; } @@ -101,7 +109,7 @@ $0) { } // Reduce the vector to a single float and do a workgroup reduce. const float private_sum = reduce_vector(private_sum4); - const float sum = local_reduce(private_sum); + const float sum = local_reduce(private_sum, tmp); // Calculate the mean const float mean = sum / args.src_tensor.Channels(); // Calculate the squared sum of the difference from the mean. @@ -117,7 +125,7 @@ $0) { } // Reduce const float private_sum_diff_sq = reduce_vector(private_sum_diff_sq4); - const float sum_diff_sq = local_reduce(private_sum_diff_sq); + const float sum_diff_sq = local_reduce(private_sum_diff_sq, tmp); // Calculate 1/stddev (with the 'regulazing constant' as in tensor_utils.cc) const float variance = sum_diff_sq / args.src_tensor.Channels(); const float stddev_inv = rsqrt(variance + 1.0e-8f); From 6de5b8dea2e2c51ec94705d852ebd01201547dbd Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Tue, 4 Aug 2020 13:32:37 -0700 Subject: [PATCH 0392/1017] Add ImmutableNodeMap for const GraphDef. NodeMap and ImmutableNodeMap are subclass of NodeMapInternal. PiperOrigin-RevId: 324879172 Change-Id: If5c311e7a4992e327f4cbe6a04c727825d2920d2 --- tensorflow/core/grappler/utils.cc | 30 +++---- tensorflow/core/grappler/utils.h | 64 +++------------ tensorflow/core/grappler/utils_test.cc | 103 +++++++------------------ 3 files changed, 59 insertions(+), 138 deletions(-) diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc index e342f7dfdf0..7cf303654ed 100644 --- a/tensorflow/core/grappler/utils.cc +++ b/tensorflow/core/grappler/utils.cc @@ -73,21 +73,25 @@ bool IsShapeConsumer(const NodeDef& node) { } // namespace -namespace internal { -// Specialized template class method GetNodeDefFromGraph. -template <> -NodeDef* NodeMapInternal::GetNodeDefFromGraph( - GraphDef* graph, int64 i) const { - return graph->mutable_node(i); +NodeMap::NodeMap(GraphDef* graph) { + nodes_.reserve(graph->node_size()); + outputs_.reserve(graph->node_size()); + for (int i = 0; i < graph->node_size(); i++) { + NodeDef* node = graph->mutable_node(i); + const string& node_name = node->name(); + auto rslt = nodes_.emplace(node_name, node); + // Check that the graph doesn't contain multiple nodes with the same name. + if (!rslt.second) { + // The first node found with a given name becomes the canonical. + LOG(WARNING) << "Duplicated node in the graph: " << node_name; + } + NodeDef* canonical = rslt.second ? node : rslt.first->second; + for (const auto& input : node->input()) { + outputs_[NodeName(input)].insert(canonical); + } + } } -template <> -const NodeDef* -NodeMapInternal::GetNodeDefFromGraph( - const GraphDef* graph, int64 i) const { - return &graph->node(i); -} -} // namespace internal string TensorIdToString(const TensorId& tensor_id) { return tensor_id.index() == 0 ? string(tensor_id.node()) : tensor_id.ToString(); diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h index e9ab5b7da12..e529d5fb4ad 100644 --- a/tensorflow/core/grappler/utils.h +++ b/tensorflow/core/grappler/utils.h @@ -98,39 +98,16 @@ inline int NodePosition(const string& name) { return position; } -namespace internal { -// Base template class for NodeMap and ImmutableNodeMap. -template -class NodeMapInternal { +// A utility class to lookup a node and its outputs by node name. +class NodeMap { public: // Note: The NodeMap will store pointers to nodes in graph, which may become // invalid if graph is changed. - explicit NodeMapInternal(GraphDefT* graph) { - if (graph == nullptr) { - LOG(WARNING) << "NodeMapInternal constructor is called with a nullptr!"; - return; - } - nodes_.reserve(graph->node_size()); - outputs_.reserve(graph->node_size()); - for (int i = 0; i < graph->node_size(); i++) { - NodeDefT* node = GetNodeDefFromGraph(graph, i); - const string& node_name = node->name(); - auto rslt = nodes_.emplace(node_name, node); - // Check that the graph doesn't contain multiple nodes with the same name. - if (!rslt.second) { - // The first node found with a given name becomes the canonical. - LOG(WARNING) << "Duplicated node in the graph: " << node_name; - } - NodeDefT* canonical = rslt.second ? node : rslt.first->second; - for (const auto& input : node->input()) { - outputs_[NodeName(input)].insert(canonical); - } - } - } + explicit NodeMap(GraphDef* graph); // Get unordered list of fanouts from node. Notice, that the order is // non-deterministic. - const absl::flat_hash_set& GetOutputs( + const absl::flat_hash_set& GetOutputs( const string& node_name) const { auto it = outputs_.find(node_name); if (it == outputs_.end()) { @@ -140,12 +117,12 @@ class NodeMapInternal { } // Get fanouts ordered by name. - std::vector GetOutputsOrderedByNodeName( + std::vector GetOutputsOrderedByNodeName( const string& node_name) const { - std::vector result; + std::vector result; auto it = outputs_.find(node_name); if (it != outputs_.end()) { - const absl::flat_hash_set& outputs = it->second; + const absl::flat_hash_set& outputs = it->second; result.reserve(outputs.size()); result.assign(outputs.begin(), outputs.end()); std::sort(result.begin(), result.end(), @@ -158,7 +135,7 @@ class NodeMapInternal { // This method doesn't record the outputs of the added node; the outputs need // to be explicitly added by the AddOutput method. - void AddNode(const string& node_name, NodeDefT* node) { + void AddNode(const string& node_name, NodeDef* node) { DCHECK(node != nullptr); auto ret = nodes_.emplace(node_name, node); DCHECK(ret.second) @@ -171,7 +148,7 @@ class NodeMapInternal { outputs_.erase(NodeName(name)); } - NodeDefT* GetNode(const string& name) const { + NodeDef* GetNode(const string& name) const { const string node_name = NodeName(name); auto it = nodes_.find(node_name); if (it == nodes_.end()) { @@ -220,26 +197,9 @@ class NodeMapInternal { } private: - // Helper method to get the NodeDef pointer of i-th node in a graph. - NodeDefT* GetNodeDefFromGraph(GraphDefT* graph, int64 i) const; - - const absl::flat_hash_set empty_set_; - absl::node_hash_map nodes_; - absl::node_hash_map> outputs_; -}; -} // namespace internal - -// A utility class to lookup a node and its outputs by node name. -class NodeMap : public internal::NodeMapInternal { - public: - explicit NodeMap(GraphDef* graph) : NodeMapInternal(graph) {} -}; - -// Same to NodeMap, but uses const GraphDef. -class ImmutableNodeMap - : public internal::NodeMapInternal { - public: - explicit ImmutableNodeMap(const GraphDef* graph) : NodeMapInternal(graph) {} + const absl::flat_hash_set empty_set_; + absl::node_hash_map nodes_; + absl::node_hash_map> outputs_; }; // A vector with a set. The set stores the same elements as the vector, and diff --git a/tensorflow/core/grappler/utils_test.cc b/tensorflow/core/grappler/utils_test.cc index e7e57e9b7d7..6231fb7a780 100644 --- a/tensorflow/core/grappler/utils_test.cc +++ b/tensorflow/core/grappler/utils_test.cc @@ -349,69 +349,39 @@ TEST_F(UtilsTest, NumNonControlOutputs) { GraphDef graph; TF_CHECK_OK(s.ToGraphDef(&graph)); + NodeMap node_map(&graph); - { - NodeMap node_map(&graph); + const NodeDef* add_node = node_map.GetNode("add"); + const NodeDef* mul_node = node_map.GetNode("mul"); + ASSERT_NE(add_node, nullptr); - const NodeDef* add_node = node_map.GetNode("add"); - const NodeDef* mul_node = node_map.GetNode("mul"); - ASSERT_NE(add_node, nullptr); + // [a, b] are only non-control inputs + EXPECT_EQ(NumNonControlInputs(*add_node), 2); + EXPECT_EQ(NumControlInputs(*add_node), 1); + // [sqrt, shape] are non control outputs + EXPECT_EQ(NumNonControlOutputs(*add_node, node_map), 2); + // sqrt is the only data output + EXPECT_EQ(NumNonControlDataOutputs(*add_node, node_map), 1); + EXPECT_EQ(NumControlInputs(*mul_node), 0); - // [a, b] are only non-control inputs - EXPECT_EQ(NumNonControlInputs(*add_node), 2); - EXPECT_EQ(NumControlInputs(*add_node), 1); - // [sqrt, shape] are non control outputs - EXPECT_EQ(NumNonControlOutputs(*add_node, node_map), 2); - // sqrt is the only data output - EXPECT_EQ(NumNonControlDataOutputs(*add_node, node_map), 1); - EXPECT_EQ(NumControlInputs(*mul_node), 0); + EXPECT_TRUE(HasControlInputs(*add_node)); + EXPECT_TRUE(HasRegularInputs(*add_node)); + EXPECT_TRUE(HasControlOutputs(*add_node, node_map)); + EXPECT_TRUE(HasRegularOutputs(*add_node, node_map)); - EXPECT_TRUE(HasControlInputs(*add_node)); - EXPECT_TRUE(HasRegularInputs(*add_node)); - EXPECT_TRUE(HasControlOutputs(*add_node, node_map)); - EXPECT_TRUE(HasRegularOutputs(*add_node, node_map)); + const NodeDef* x_node = node_map.GetNode("x"); + ASSERT_NE(x_node, nullptr); + EXPECT_FALSE(HasControlInputs(*x_node)); + EXPECT_FALSE(HasRegularInputs(*x_node)); + EXPECT_FALSE(HasControlOutputs(*x_node, node_map)); + EXPECT_TRUE(HasRegularOutputs(*x_node, node_map)); - const NodeDef* x_node = node_map.GetNode("x"); - ASSERT_NE(x_node, nullptr); - EXPECT_FALSE(HasControlInputs(*x_node)); - EXPECT_FALSE(HasRegularInputs(*x_node)); - EXPECT_FALSE(HasControlOutputs(*x_node, node_map)); - EXPECT_TRUE(HasRegularOutputs(*x_node, node_map)); - - const NodeDef* round_node = node_map.GetNode("round"); - ASSERT_NE(round_node, nullptr); - EXPECT_TRUE(HasControlInputs(*round_node)); - EXPECT_TRUE(HasRegularInputs(*round_node)); - EXPECT_FALSE(HasControlOutputs(*round_node, node_map)); - EXPECT_FALSE(HasRegularOutputs(*round_node, node_map)); - } - - { - // Similar test for ImmutableNodeMap. - ImmutableNodeMap node_map(&graph); - - const NodeDef* add_node = node_map.GetNode("add"); - const NodeDef* mul_node = node_map.GetNode("mul"); - ASSERT_NE(add_node, nullptr); - - // [a, b] are only non-control inputs - EXPECT_EQ(NumNonControlInputs(*add_node), 2); - EXPECT_EQ(NumControlInputs(*add_node), 1); - EXPECT_EQ(NumControlInputs(*mul_node), 0); - - EXPECT_TRUE(HasControlInputs(*add_node)); - EXPECT_TRUE(HasRegularInputs(*add_node)); - - const NodeDef* x_node = node_map.GetNode("x"); - ASSERT_NE(x_node, nullptr); - EXPECT_FALSE(HasControlInputs(*x_node)); - EXPECT_FALSE(HasRegularInputs(*x_node)); - - const NodeDef* round_node = node_map.GetNode("round"); - ASSERT_NE(round_node, nullptr); - EXPECT_TRUE(HasControlInputs(*round_node)); - EXPECT_TRUE(HasRegularInputs(*round_node)); - } + const NodeDef* round_node = node_map.GetNode("round"); + ASSERT_NE(round_node, nullptr); + EXPECT_TRUE(HasControlInputs(*round_node)); + EXPECT_TRUE(HasRegularInputs(*round_node)); + EXPECT_FALSE(HasControlOutputs(*round_node, node_map)); + EXPECT_FALSE(HasRegularOutputs(*round_node, node_map)); } TEST(CheckAttrExists, All) { @@ -683,30 +653,17 @@ TEST(SetTensorValueTest, Quantized) { /*error_msg=*/""); } -static void BM_NodeMapConstruct(benchmark::State& state) { - const int size = state.range(0); +static void BM_NodeMapConstruct(int iters, int size) { testing::StopTiming(); GraphDef graph = test::CreateRandomGraph(size); testing::StartTiming(); - for (auto s : state) { + for (int i = 0; i < iters; i++) { NodeMap node_map(&graph); } testing::StopTiming(); } BENCHMARK(BM_NodeMapConstruct)->Range(1, 1 << 20); -static void BM_ImmutableNodeMapConstruct(benchmark::State& state) { - const int size = state.range(0); - testing::StopTiming(); - GraphDef graph = test::CreateRandomGraph(size); - testing::StartTiming(); - for (auto s : state) { - ImmutableNodeMap node_map(&graph); - } - testing::StopTiming(); -} -BENCHMARK(BM_ImmutableNodeMapConstruct)->Range(1, 1 << 20); - } // namespace } // namespace grappler } // namespace tensorflow From 3b30b1b495e018a23025f71877024324eaddb00f Mon Sep 17 00:00:00 2001 From: Stephan Herhut Date: Tue, 4 Aug 2020 13:33:31 -0700 Subject: [PATCH 0393/1017] Allow index typed memrefs in reshape_memref_cast. With the recent change to allow memref of index in MLIR core, we should also allow this in the mhlo dialect. PiperOrigin-RevId: 324879354 Change-Id: Id18d6a5951906d4f5b4438e93c49d3518cff5a3d --- .../mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.td | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.td b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.td index 87082219db7..3fa46584ca2 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.td +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.td @@ -66,6 +66,8 @@ def LHLO_PredOrIntBuffer : MemRefOf<[HLO_Int, HLO_Pred]>; def LHLO_Buffer : MemRefOf<[AnyFloat, AnySignlessInteger, AnyComplex]>; +def LHLO_ExtentBuffer : MemRefRankOf<[AnySignlessInteger, Index], [1]>; + //===----------------------------------------------------------------------===// // LMHLO nullary op definitions. //===----------------------------------------------------------------------===// @@ -467,7 +469,7 @@ def ReshapeMemRefCastOp: Op:$shape + LHLO_ExtentBuffer:$shape ); let results = (outs AnyRankedOrUnrankedMemRef:$result); From 442c7015fce8889fac11f415f20d43e0576520a3 Mon Sep 17 00:00:00 2001 From: Pavithra Vijay Date: Tue, 4 Aug 2020 13:44:26 -0700 Subject: [PATCH 0394/1017] Remove usages of `smart_cond` module from Keras. We have a version of smart_cond in keras/utils/tf_utils.py, removing that and adding smart_cond from smart_cond TF module to keras/utils/control_flow_util.py PiperOrigin-RevId: 324881600 Change-Id: I94a2e9666d877b49703d2aa9dd10c1e954e70fda --- .../python/keras/engine/data_adapter.py | 4 +- .../python/keras/engine/training_utils.py | 4 +- .../python/keras/layers/normalization.py | 4 +- tensorflow/python/keras/losses.py | 15 +++--- .../experimental/loss_scale_optimizer.py | 6 +-- .../python/keras/utils/control_flow_util.py | 46 ++++++++----------- 6 files changed, 34 insertions(+), 45 deletions(-) diff --git a/tensorflow/python/keras/engine/data_adapter.py b/tensorflow/python/keras/engine/data_adapter.py index 0e4886fc8cb..e9662da73e7 100644 --- a/tensorflow/python/keras/engine/data_adapter.py +++ b/tensorflow/python/keras/engine/data_adapter.py @@ -37,12 +37,12 @@ from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import smart_cond from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape from tensorflow.python.framework.ops import composite_tensor from tensorflow.python.keras import backend from tensorflow.python.keras.engine import training_utils -from tensorflow.python.keras.utils import control_flow_util from tensorflow.python.keras.utils import data_utils from tensorflow.python.ops import array_ops from tensorflow.python.ops import math_ops @@ -1296,7 +1296,7 @@ def _make_class_weight_map_fn(class_weight): raise ValueError("`class_weight` not supported for " "3+ dimensional targets.") - y_classes = control_flow_util.smart_cond( + y_classes = smart_cond.smart_cond( y.shape.rank == 2 and backend.shape(y)[1] > 1, lambda: backend.argmax(y, axis=1), lambda: math_ops.cast(backend.reshape(y, (-1,)), dtypes.int64)) diff --git a/tensorflow/python/keras/engine/training_utils.py b/tensorflow/python/keras/engine/training_utils.py index 157a0c77ebf..84bcd99922f 100644 --- a/tensorflow/python/keras/engine/training_utils.py +++ b/tensorflow/python/keras/engine/training_utils.py @@ -40,6 +40,7 @@ from tensorflow.python.framework import composite_tensor_utils from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors from tensorflow.python.framework import ops +from tensorflow.python.framework import smart_cond from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import tensor_util @@ -47,7 +48,6 @@ from tensorflow.python.keras import backend as K from tensorflow.python.keras import callbacks as cbks from tensorflow.python.keras import losses from tensorflow.python.keras import metrics as metrics_module -from tensorflow.python.keras.utils import control_flow_util from tensorflow.python.keras.utils import data_utils from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import losses_utils @@ -997,7 +997,7 @@ def standardize_weights(y, weight_vector[:] = np.nan weight_vector[keys] = values - y_classes = control_flow_util.smart_cond( + y_classes = smart_cond.smart_cond( len(y.shape.as_list()) == 2 and K.shape(y)[1] > 1, lambda: K.argmax(y, axis=1), lambda: math_ops.cast(K.reshape(y, (-1,)), dtypes.int64)) diff --git a/tensorflow/python/keras/layers/normalization.py b/tensorflow/python/keras/layers/normalization.py index fd77cddb08d..12013882ff5 100644 --- a/tensorflow/python/keras/layers/normalization.py +++ b/tensorflow/python/keras/layers/normalization.py @@ -577,7 +577,7 @@ class BatchNormalizationBase(Layer): training, train_op, _fused_batch_norm_inference) variance = _maybe_add_or_remove_bessels_correction(variance, remove=True) - training_value = control_flow_util.smart_constant_value(training) + training_value = control_flow_util.constant_value(training) if training_value or training_value is None: if not use_fused_avg_updates: if training_value is None: @@ -762,7 +762,7 @@ class BatchNormalizationBase(Layer): return (scale, offset) # Determine a boolean value for `training`: could be True, False, or None. - training_value = control_flow_util.smart_constant_value(training) + training_value = control_flow_util.constant_value(training) if training_value == False: # pylint: disable=singleton-comparison,g-explicit-bool-comparison mean, variance = self.moving_mean, self.moving_variance else: diff --git a/tensorflow/python/keras/losses.py b/tensorflow/python/keras/losses.py index a149418fdd8..f75e6af6e30 100644 --- a/tensorflow/python/keras/losses.py +++ b/tensorflow/python/keras/losses.py @@ -26,9 +26,9 @@ from tensorflow.python.autograph.core import ag_ctx from tensorflow.python.autograph.impl import api as autograph from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.framework import ops +from tensorflow.python.framework import smart_cond from tensorflow.python.framework import tensor_util from tensorflow.python.keras import backend as K -from tensorflow.python.keras.utils import control_flow_util from tensorflow.python.keras.utils import losses_utils from tensorflow.python.keras.utils import tf_utils from tensorflow.python.keras.utils.generic_utils import deserialize_keras_object @@ -1313,9 +1313,8 @@ def _maybe_convert_labels(y_true): # Convert the binary labels to -1 or 1. return 2. * y_true - 1. - updated_y_true = control_flow_util.smart_cond(is_binary, - _convert_binary_labels, - lambda: y_true) + updated_y_true = smart_cond.smart_cond(is_binary, _convert_binary_labels, + lambda: y_true) return updated_y_true @@ -1527,8 +1526,8 @@ def categorical_crossentropy(y_true, num_classes = math_ops.cast(array_ops.shape(y_true)[-1], y_pred.dtype) return y_true * (1.0 - label_smoothing) + (label_smoothing / num_classes) - y_true = control_flow_util.smart_cond(label_smoothing, _smooth_labels, - lambda: y_true) + y_true = smart_cond.smart_cond(label_smoothing, _smooth_labels, + lambda: y_true) return K.categorical_crossentropy(y_true, y_pred, from_logits=from_logits) @@ -1596,8 +1595,8 @@ def binary_crossentropy(y_true, y_pred, from_logits=False, label_smoothing=0): def _smooth_labels(): return y_true * (1.0 - label_smoothing) + 0.5 * label_smoothing - y_true = control_flow_util.smart_cond(label_smoothing, _smooth_labels, - lambda: y_true) + y_true = smart_cond.smart_cond(label_smoothing, _smooth_labels, + lambda: y_true) return K.mean( K.binary_crossentropy(y_true, y_pred, from_logits=from_logits), axis=-1) diff --git a/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py b/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py index f09c8c92e8c..4a3f459de80 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py +++ b/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py @@ -24,11 +24,11 @@ from tensorflow.python.distribute import one_device_strategy from tensorflow.python.distribute import tpu_strategy from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import smart_cond from tensorflow.python.keras import backend from tensorflow.python.keras import optimizers from tensorflow.python.keras.mixed_precision.experimental import loss_scale as keras_loss_scale_module from tensorflow.python.keras.optimizer_v2 import optimizer_v2 -from tensorflow.python.keras.utils import control_flow_util from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.training.experimental import mixed_precision @@ -406,8 +406,8 @@ class LossScaleOptimizer(_DelegatingTrackableMixin, optimizer_v2.OptimizerV2): # DistributionStrategy does not support having a cond in a replica context # with a branch that calls `merge_call`, and self._optimizer.apply_gradients # calls `merge_call`. - maybe_apply_op = control_flow_util.smart_cond(should_apply_grads, apply_fn, - do_not_apply_fn) + maybe_apply_op = smart_cond.smart_cond(should_apply_grads, apply_fn, + do_not_apply_fn) return control_flow_ops.group(maybe_apply_op, loss_scale_update_op) def _apply_gradients(self, grads, wrapped_vars, name, diff --git a/tensorflow/python/keras/utils/control_flow_util.py b/tensorflow/python/keras/utils/control_flow_util.py index 4aadf691d70..8d13c573149 100644 --- a/tensorflow/python/keras/utils/control_flow_util.py +++ b/tensorflow/python/keras/utils/control_flow_util.py @@ -22,8 +22,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.framework import ops -from tensorflow.python.framework import tensor_util +from tensorflow.python.framework import smart_cond as smart_module from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import variables @@ -107,43 +106,34 @@ def smart_cond(pred, true_fn=None, false_fn=None, name=None): # pylint: disable Raises: TypeError: If `true_fn` or `false_fn` is not callable. """ - if not callable(true_fn): - raise TypeError("`true_fn` must be callable.") - if not callable(false_fn): - raise TypeError("`false_fn` must be callable.") - pred_value = smart_constant_value(pred) - if pred_value is not None: - if pred_value: - return true_fn() - else: - return false_fn() - else: + if isinstance(pred, variables.Variable): return control_flow_ops.cond( pred, true_fn=true_fn, false_fn=false_fn, name=name) + return smart_module.smart_cond( + pred, true_fn=true_fn, false_fn=false_fn, name=name) -def smart_constant_value(pred): # pylint: disable=invalid-name +def constant_value(pred): # pylint: disable=invalid-name """Return the bool value for `pred`, or None if `pred` had a dynamic value. Arguments: - pred: A scalar, either a Python bool or tensor. + pred: A scalar, either a Python bool or a TensorFlow boolean variable + or tensor, or the Python integer 1 or 0. Returns: True or False if `pred` has a constant boolean value, None otherwise. Raises: - TypeError: If `pred` is not a Tensor or bool. + TypeError: If `pred` is not a Variable, Tensor or bool, or Python + integer 1 or 0. """ - if isinstance(pred, ops.Tensor): - pred_value = tensor_util.constant_value(pred) - elif isinstance(pred, variables.Variable): - pred_value = None - elif pred in {0, 1}: # Accept 1/0 as valid boolean values - pred_value = bool(pred) - elif isinstance(pred, bool): - pred_value = pred - else: - raise TypeError("`pred` must be a Tensor, or a Python bool, or 1 or 0. " - "Found instead: %s" % type(pred)) + # Allow integer booleans. + if isinstance(pred, int): + if pred == 1: + pred = True + elif pred == 0: + pred = False - return pred_value + if isinstance(pred, variables.Variable): + return None + return smart_module.smart_constant_value(pred) From 895ed30e83b57c8c7702839f3a4c2697a4b4e52a Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Tue, 4 Aug 2020 13:46:05 -0700 Subject: [PATCH 0395/1017] Update visibility of build target. PiperOrigin-RevId: 324881891 Change-Id: I2a71619158aebc895b8cd708e25c3d12bc3ad08d --- .../mlir/tensorflow/ir/tf_generated_ops.td | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index 63138489ef7..84f3fa9c463 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -7226,6 +7226,28 @@ tf.range(start, limit, delta) ==> [3, 6, 9, 12, 15] ]; } +def TF_RangeDatasetOp : TF_Op<"RangeDataset", []> { + let summary = [{ +Creates a dataset with a range of values. Corresponds to python's xrange. + }]; + + let description = [{ + }]; + + let arguments = (ins + I64Tensor:$start, + I64Tensor:$stop, + I64Tensor:$step, + + Confined]>:$output_types, + Confined]>:$output_shapes + ); + + let results = (outs + TF_VariantTensor:$handle + ); +} + def TF_RankOp : TF_Op<"Rank", [NoSideEffect]> { let summary = "Returns the rank of a tensor."; From 6dae832b2cebab6792abcfa55a39263f5f4f483a Mon Sep 17 00:00:00 2001 From: Scott Main Date: Tue, 4 Aug 2020 13:54:10 -0700 Subject: [PATCH 0396/1017] Update post-training quant tutorial to use int-only quant with TF2.3 PiperOrigin-RevId: 324883545 Change-Id: I425debca1604354d37939e1141ac7cf425422067 --- .../post_training_integer_quant.ipynb | 957 +++++++++++------- 1 file changed, 576 insertions(+), 381 deletions(-) diff --git a/tensorflow/lite/g3doc/performance/post_training_integer_quant.ipynb b/tensorflow/lite/g3doc/performance/post_training_integer_quant.ipynb index cff1e773938..a2835f53d82 100644 --- a/tensorflow/lite/g3doc/performance/post_training_integer_quant.ipynb +++ b/tensorflow/lite/g3doc/performance/post_training_integer_quant.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": { "cellView": "form", "colab": {}, @@ -76,22 +76,15 @@ "source": [ "## Overview\n", "\n", - "[TensorFlow Lite](https://www.tensorflow.org/lite/) now supports\n", - "converting all model values (weights and activations) to 8-bit integers when converting from TensorFlow to TensorFlow Lite's flat buffer format. This results in a 4x reduction in model size and a 3 to 4x performance improvement on CPU performance. In addition, this fully quantized model can be consumed by integer-only hardware accelerators.\n", + "Integer quantization is an optimization strategy that converts 32-bit floating-point numbers (such as weights and activation outputs) to the nearest 8-bit fixed-point numbers. This results in a smaller model and increased inferencing speed, which is valuable for low-power devices such as [microcontrollers](https://www.tensorflow.org/lite/microcontrollers). This data format is also required by integer-only accelerators such as the [Edge TPU](https://coral.ai/).\n", "\n", - "In contrast to [post-training \"on-the-fly\" quantization](https://colab.sandbox.google.com/github/tensorflow/tensorflow/blob/master/tensorflow/lite/tutorials/post_training_quant.ipynb)—which stores only the weights as 8-bit integers—this technique statically quantizes all weights *and* activations during model conversion.\n", + "In this tutorial, you'll train an MNIST model from scratch, convert it into a Tensorflow Lite file, and quantize it using [post-training quantization](https://www.tensorflow.org/lite/performance/post_training_quantization). Finally, you'll check the accuracy of the converted model and compare it to the original float model.\n", "\n", - "In this tutorial, you'll train an MNIST model from scratch, check its accuracy in TensorFlow, and then convert the model into a Tensorflow Lite flatbuffer with full quantization. Finally, you'll check the accuracy of the converted model and compare it to the original float model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "2XsEP17Zelz9" - }, - "source": [ - "## Build an MNIST model" + "You actually have several options as to how much you want to quantize a model. In this tutorial, you'll perform \"full integer quantization,\" which converts all weights and activation outputs into 8-bit integer data—whereas other strategies may leave some amount of data in floating-point.\n", + "\n", + "To learn more about the various quantization strategies, read about [TensorFlow Lite model optimization](https://www.tensorflow.org/lite/performance/model_optimization).\n", + "\n", + "\n" ] }, { @@ -101,12 +94,22 @@ "id": "dDqqUIZjZjac" }, "source": [ - "### Setup" + "## Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "I0nR5AMEWq0H" + }, + "source": [ + "In order to quantize both the input and output tensors, we need to use APIs added in TensorFlow r2.3:" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", @@ -118,82 +121,18 @@ "logging.getLogger(\"tensorflow\").setLevel(logging.DEBUG)\n", "\n", "import tensorflow as tf\n", - "from tensorflow import keras\n", "import numpy as np\n", - "import pathlib" + "assert float(tf.__version__[:3]) \u003e= 2.3" ] }, { "cell_type": "markdown", "metadata": { "colab_type": "text", - "id": "eQ6Q0qqKZogR" + "id": "2XsEP17Zelz9" }, "source": [ - "### Train and export the model" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": { - "colab": { - "height": 51 - }, - "colab_type": "code", - "id": "eMsw_6HujaqM", - "outputId": "5662a5f3-fc64-458f-958a-98f9c6348143" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2782 - accuracy: 0.9221 - val_loss: 0.1230 - val_accuracy: 0.9664\n" - ] - }, - { - "data": { - "text/plain": [ - "\u003ctensorflow.python.keras.callbacks.History at 0x7f33f1817588\u003e" - ] - }, - "execution_count": 19, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "# Load MNIST dataset\n", - "mnist = keras.datasets.mnist\n", - "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n", - "\n", - "# Normalize the input image so that each pixel value is between 0 to 1.\n", - "train_images = train_images / 255.0\n", - "test_images = test_images / 255.0\n", - "\n", - "# Define the model architecture\n", - "model = keras.Sequential([\n", - " keras.layers.InputLayer(input_shape=(28, 28)),\n", - " keras.layers.Reshape(target_shape=(28, 28, 1)),\n", - " keras.layers.Conv2D(filters=12, kernel_size=(3, 3), activation='relu'),\n", - " keras.layers.MaxPooling2D(pool_size=(2, 2)),\n", - " keras.layers.Flatten(),\n", - " keras.layers.Dense(10)\n", - "])\n", - "\n", - "# Train the digit classification model\n", - "model.compile(optimizer='adam',\n", - " loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n", - " metrics=['accuracy'])\n", - "model.fit(\n", - " train_images,\n", - " train_labels,\n", - " epochs=1,\n", - " validation_data=(test_images, test_labels)\n", - ")" + "## Generate a TensorFlow Model" ] }, { @@ -203,7 +142,94 @@ "id": "5NMaNZQCkW9X" }, "source": [ - "This training won't take long because you're training the model for just a single epoch, which trains to about 96% accuracy." + "We'll build a simple model to classify numbers from the [MNIST dataset](https://www.tensorflow.org/datasets/catalog/mnist).\n", + "\n", + "This training won't take long because you're training the model for just a 5 epochs, which trains to about ~98% accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "height": 51 + }, + "colab_type": "code", + "id": "eMsw_6HujaqM", + "outputId": "0f362bef-a5b8-46f2-c41c-cba008998b72" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n", + "11493376/11490434 [==============================] - 0s 0us/step\n", + "Epoch 1/5\n", + "1875/1875 [==============================] - 5s 2ms/step - loss: 0.2793 - accuracy: 0.9227 - val_loss: 0.1392 - val_accuracy: 0.9618\n", + "Epoch 2/5\n", + "1875/1875 [==============================] - 5s 2ms/step - loss: 0.1179 - accuracy: 0.9667 - val_loss: 0.0928 - val_accuracy: 0.9719\n", + "Epoch 3/5\n", + "1875/1875 [==============================] - 4s 2ms/step - loss: 0.0860 - accuracy: 0.9754 - val_loss: 0.0742 - val_accuracy: 0.9755\n", + "Epoch 4/5\n", + "1875/1875 [==============================] - 4s 2ms/step - loss: 0.0691 - accuracy: 0.9796 - val_loss: 0.0686 - val_accuracy: 0.9776\n", + "Epoch 5/5\n", + "1875/1875 [==============================] - 4s 2ms/step - loss: 0.0589 - accuracy: 0.9823 - val_loss: 0.0654 - val_accuracy: 0.9787\n" + ] + }, + { + "data": { + "text/plain": [ + "\u003ctensorflow.python.keras.callbacks.History at 0x7f69e0275a58\u003e" + ] + }, + "execution_count": null, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "# Load MNIST dataset\n", + "mnist = tf.keras.datasets.mnist\n", + "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n", + "\n", + "# Normalize the input image so that each pixel value is between 0 to 1.\n", + "train_images = train_images.astype(np.float32) / 255.0\n", + "test_images = test_images.astype(np.float32) / 255.0\n", + "\n", + "# Define the model architecture\n", + "model = tf.keras.Sequential([\n", + " tf.keras.layers.InputLayer(input_shape=(28, 28)),\n", + " tf.keras.layers.Reshape(target_shape=(28, 28, 1)),\n", + " tf.keras.layers.Conv2D(filters=12, kernel_size=(3, 3), activation='relu'),\n", + " tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),\n", + " tf.keras.layers.Flatten(),\n", + " tf.keras.layers.Dense(10)\n", + "])\n", + "\n", + "# Train the digit classification model\n", + "model.compile(optimizer='adam',\n", + " loss=tf.keras.losses.SparseCategoricalCrossentropy(\n", + " from_logits=True),\n", + " metrics=['accuracy'])\n", + "model.fit(\n", + " train_images,\n", + " train_labels,\n", + " epochs=5,\n", + " validation_data=(test_images, test_labels)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "KuTEoGFYd8aM" + }, + "source": [ + "## Convert to a TensorFlow Lite model" ] }, { @@ -213,16 +239,16 @@ "id": "xl8_fzVAZwOh" }, "source": [ - "### Convert to a TensorFlow Lite model\n", + "Now you can convert the trained model to TensorFlow Lite format using the [`TFLiteConverter`](https://www.tensorflow.org/lite/convert/python_api) API, and apply varying degrees of quantization.\n", "\n", - "Using the Python [TFLiteConverter](https://www.tensorflow.org/lite/convert/python_api), you can now convert the trained model into a TensorFlow Lite model.\n", + "Beware that some versions of quantization leave some of the data in float format. So the following sections show each option with increasing amounts of quantization, until we get a model that's entirely int8 or uint8 data. (Notice we duplicate some code in each section so you can see all the quantization steps for each option.)\n", "\n", - "Now load the model using the `TFLiteConverter`:" + "First, here's a converted model with no quantization:" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", @@ -231,63 +257,10 @@ "outputs": [], "source": [ "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n", + "\n", "tflite_model = converter.convert()" ] }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "F2o2ZfF0aiCx" - }, - "source": [ - "Write it out to a `.tflite` file:" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "vptWZq2xnclo" - }, - "outputs": [], - "source": [ - "tflite_models_dir = pathlib.Path(\"/tmp/mnist_tflite_models/\")\n", - "tflite_models_dir.mkdir(exist_ok=True, parents=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": { - "colab": { - "height": 34 - }, - "colab_type": "code", - "id": "Ie9pQaQrn5ue", - "outputId": "8580b835-61f0-42b3-a21e-b8d476042c11" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "84528" - ] - }, - "execution_count": 22, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "tflite_model_file = tflite_models_dir/\"mnist_model.tflite\"\n", - "tflite_model_file.write_bytes(tflite_model)" - ] - }, { "cell_type": "markdown", "metadata": { @@ -295,25 +268,81 @@ "id": "7BONhYtYocQY" }, "source": [ - "Now you have a trained MNIST model that's converted to a `.tflite` file, but it's still using 32-bit float values for all parameter data.\n", - "\n", - "So let's convert the model again, this time using quantization...\n", - "\n", - "#### Convert using quantization\n", - "First, first set the `optimizations` flag to optimize for size:" + "It's now a TensorFlow Lite model, but it's still using 32-bit float values for all parameter data." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "jPYZwgZTwJMT" + }, + "source": [ + "### Convert using dynamic range quantization\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "Hjvq1vpJd4U_" + }, + "source": [ + "Now let's enable the default `optimizations` flag to quantize all fixed parameters (such as weights):" ] }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": { - "colab": {}, + "colab": { + "height": 34 + }, "colab_type": "code", - "id": "HEZ6ET1AHAS3" + "id": "HEZ6ET1AHAS3", + "outputId": "82a75458-10d2-484a-8e09-a8af56212e10" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /tmp/tmpcojyiqri/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /tmp/tmpcojyiqri/assets\n" + ] + } + ], "source": [ - "converter.optimizations = [tf.lite.Optimize.DEFAULT]" + "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n", + "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n", + "\n", + "tflite_model_quant = converter.convert()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "o5wuE-RcdX_3" + }, + "source": [ + "The model is now a bit smaller with quantized weights, but other variable data is still in float format." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "UgKDdnHQEhpb" + }, + "source": [ + "### Convert using float fallback quantization" ] }, { @@ -323,107 +352,87 @@ "id": "rTe8avZJHMDO" }, "source": [ - "Now, in order to create quantized values with an accurate dynamic range of activations, you need to provide a representative dataset.\n", + "To quantize the variable data (such as model input/output and intermediates between layers), you need to provide a [`RepresentativeDataset`](https://www.tensorflow.org/api_docs/python/tf/lite/RepresentativeDataset). This is a generator function that provides a set of input data that's large enough to represent typical values. It allows the converter to estimate a dynamic range for all the variable data. (The dataset does not need to be unique compared to the training or evaluation dataset.)\n", "To support multiple inputs, each representative data point is a list and elements in the list are fed to the model according to their indices.\n" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", - "id": "FiwiWU3gHdkW" - }, - "outputs": [], - "source": [ - "mnist_train, _ = tf.keras.datasets.mnist.load_data()\n", - "images = tf.cast(mnist_train[0], tf.float32) / 255.0\n", - "mnist_ds = tf.data.Dataset.from_tensor_slices((images)).batch(1)\n", - "def representative_data_gen():\n", - " for input_value in mnist_ds.take(100):\n", - " # Model has only one input so each data point has one element.\n", - " yield [input_value]\n", - "\n", - "converter.representative_dataset = representative_data_gen" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "xW84iMYjHd9t" - }, - "source": [ - "Finally, convert the model to TensorFlow Lite format:" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": { - "colab": { - "height": 34 - }, - "colab_type": "code", - "id": "yuNfl3CoHNK3", - "outputId": "79a19679-87a2-4dc6-eee4-b33f3e5c1c5d" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "24720" - ] - }, - "execution_count": 25, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" - } - ], - "source": [ - "tflite_model_quant = converter.convert()\n", - "tflite_model_quant_file = tflite_models_dir/\"mnist_model_quant.tflite\"\n", - "tflite_model_quant_file.write_bytes(tflite_model_quant)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "PhMmUTl4sbkz" - }, - "source": [ - "Note how the resulting file is approximately `1/4` the size:" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": { - "colab": { - "height": 85 - }, - "colab_type": "code", - "id": "JExfcfLDscu4", - "outputId": "58238f92-01b0-4faa-e293-35451d08dd7c" + "id": "FiwiWU3gHdkW", + "outputId": "61093d59-5b47-4e59-a577-46f056281bab" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "total 140K\n", - "-rw-rw-r-- 1 yashkatariya 10086651 25K Jun 23 06:06 mnist_model_quant_io.tflite\n", - "-rw-rw-r-- 1 yashkatariya 10086651 25K Jun 23 06:07 mnist_model_quant.tflite\n", - "-rw-rw-r-- 1 yashkatariya 10086651 83K Jun 23 06:06 mnist_model.tflite\n" + "INFO:tensorflow:Assets written to: /tmp/tmp1bvfr71i/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /tmp/tmp1bvfr71i/assets\n" ] } ], "source": [ - "!ls -lh {tflite_models_dir}" + "def representative_data_gen():\n", + " for input_value in tf.data.Dataset.from_tensor_slices(train_images).batch(1).take(100):\n", + " # Model has only one input so each data point has one element.\n", + " yield [input_value]\n", + "\n", + "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n", + "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n", + "converter.representative_dataset = representative_data_gen\n", + "\n", + "tflite_model_quant = converter.convert()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "_GC3HFlptf7x" + }, + "source": [ + "Now all weights and variable data are quantized, and the model is significantly smaller compared to the original TensorFlow Lite model.\n", + "\n", + "However, to maintain compatibility with applications that traditionally use float model input and output tensors, the TensorFlow Lite Converter leaves the model input and output tensors in float:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "height": 51 + }, + "colab_type": "code", + "id": "id1OEKFELQwp", + "outputId": "024a710f-44cc-43d1-89a7-456a1727523c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input: \u003cclass 'numpy.float32'\u003e\n", + "output: \u003cclass 'numpy.float32'\u003e\n" + ] + } + ], + "source": [ + "interpreter = tf.lite.Interpreter(model_content=tflite_model_quant)\n", + "input_type = interpreter.get_input_details()[0]['dtype']\n", + "print('input: ', input_type)\n", + "output_type = interpreter.get_output_details()[0]['dtype']\n", + "print('output: ', output_type)" ] }, { @@ -433,44 +442,75 @@ "id": "RACBJuj2XO8x" }, "source": [ - "Your model should now be fully quantized. However, if you convert a model that includes any operations that TensorFlow Lite cannot quantize, those ops are left in floating point. This allows for conversion to complete so you have a smaller and more efficient model, but the model won't be compatible with some ML accelerators that require full integer quantization. Also, by default, the converted model still use float input and outputs, which also is not compatible with some accelerators.\n", + "That's usually good for compatibility, but it won't be compatible with devices that perform only integer-based operations, such as the Edge TPU.\n", "\n", - "So to ensure that the converted model is fully quantized (make the converter throw an error if it encounters an operation it cannot quantize), and to use integers for the model's input and output, you need to convert the model again using these additional configurations:" + "Additionally, the above process may leave an operation in float format if TensorFlow Lite doesn't include a quantized implementation for that operation. This strategy allows conversion to complete so you have a smaller and more efficient model, but again, it won't be compatible with integer-only hardware. (All ops in this MNIST model have a quantized implementation.)\n", + "\n", + "So to ensure an end-to-end integer-only model, you need a couple more parameters..." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "FQgTqbvPvxGJ" + }, + "source": [ + "### Convert using integer-only quantization" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "mwR9keYAwArA" + }, + "source": [ + "To quantize the input and output tensors, and make the converter throw an error if it encounters an operation it cannot quantize, convert the model again with some additional parameters:" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": { "colab": { - "height": 34 + "height": 51 }, "colab_type": "code", "id": "kzjEjcDs3BHa", - "outputId": "8d7370ec-3f3f-41a2-8afb-4ecdd40e9efc" + "outputId": "0462645b-f8e1-489a-f703-8093f83645d5" }, "outputs": [ { - "data": { - "text/plain": [ - "24784" - ] - }, - "execution_count": 27, - "metadata": { - "tags": [] - }, - "output_type": "execute_result" + "name": "stdout", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /tmp/tmpvnuxq9pa/assets\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:tensorflow:Assets written to: /tmp/tmpvnuxq9pa/assets\n" + ] } ], "source": [ + "def representative_data_gen():\n", + " for input_value in tf.data.Dataset.from_tensor_slices(train_images).batch(1).take(100):\n", + " yield [input_value]\n", + "\n", + "converter = tf.lite.TFLiteConverter.from_keras_model(model)\n", + "converter.optimizations = [tf.lite.Optimize.DEFAULT]\n", + "converter.representative_dataset = representative_data_gen\n", + "# Ensure that if any ops can't be quantized, the converter throws an error\n", "converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]\n", + "# Set the input and output tensors to uint8 (APIs added in r2.3)\n", "converter.inference_input_type = tf.uint8\n", "converter.inference_output_type = tf.uint8\n", "\n", - "tflite_model_quant = converter.convert()\n", - "tflite_model_quant_file = tflite_models_dir/\"mnist_model_quant_io.tflite\"\n", - "tflite_model_quant_file.write_bytes(tflite_model_quant)" + "tflite_model_quant = converter.convert()" ] }, { @@ -480,9 +520,115 @@ "id": "wYd6NxD03yjB" }, "source": [ - "In this example, the resulting model size remains the same because all operations successfully quantized to begin with. However, this new model now uses quantized input and output, making it compatible with more accelerators, such as the Coral Edge TPU.\n", + "The internal quantization remains the same as above, but you can see the input and output tensors are now integer format:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "height": 51 + }, + "colab_type": "code", + "id": "PaNkOS-twz4k", + "outputId": "b7b22b48-c305-4b4c-80c6-506d9f3c2013" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "input: \u003cclass 'numpy.uint8'\u003e\n", + "output: \u003cclass 'numpy.uint8'\u003e\n" + ] + } + ], + "source": [ + "interpreter = tf.lite.Interpreter(model_content=tflite_model_quant)\n", + "input_type = interpreter.get_input_details()[0]['dtype']\n", + "print('input: ', input_type)\n", + "output_type = interpreter.get_output_details()[0]['dtype']\n", + "print('output: ', output_type)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "TO17AP84wzBb" + }, + "source": [ + "Now you have an integer quantized model that uses integer data for the model's input and output tensors, so it's compatible with integer-only hardware such as the [Edge TPU](https://coral.ai)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "sse224YJ4KMm" + }, + "source": [ + "### Save the models as files" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "4_9nZ4nv4b9P" + }, + "source": [ + "You'll need a `.tflite` file to deploy your model on other devices. So let's save the converted models to files and then load them when we run inferences below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "height": 34 + }, + "colab_type": "code", + "id": "BEY59dC14uRv", + "outputId": "20a3397a-1466-48eb-f421-adc8ebf3f60f" + }, + "outputs": [ + { + "data": { + "text/plain": [ + "24720" + ] + }, + "execution_count": null, + "metadata": { + "tags": [] + }, + "output_type": "execute_result" + } + ], + "source": [ + "import pathlib\n", "\n", - "In the following sections, notice that we are now handling two TensorFlow Lite models: `tflite_model_file` is the converted model that still uses floating-point parameters, and `tflite_model_quant_file` is the same model converted with full integer quantization, including uint8 input and output." + "tflite_models_dir = pathlib.Path(\"/tmp/mnist_tflite_models/\")\n", + "tflite_models_dir.mkdir(exist_ok=True, parents=True)\n", + "\n", + "# Save the unquantized/float model:\n", + "tflite_model_file = tflite_models_dir/\"mnist_model.tflite\"\n", + "tflite_model_file.write_bytes(tflite_model)\n", + "# Save the quantized model:\n", + "tflite_model_quant_file = tflite_models_dir/\"mnist_model_quant.tflite\"\n", + "tflite_model_quant_file.write_bytes(tflite_model_quant)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "9t9yaTeF9fyM" + }, + "source": [ + "## Run the TensorFlow Lite models" ] }, { @@ -492,50 +638,56 @@ "id": "L8lQHMp_asCq" }, "source": [ - "## Run the TensorFlow Lite models\n", + "Now we'll run inferences using the TensorFlow Lite [`Interpreter`](https://www.tensorflow.org/api_docs/python/tf/lite/Interpreter) to compare the model accuracies.\n", "\n", - "Run the TensorFlow Lite model using the Python TensorFlow Lite\n", - "Interpreter. " - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "Ap_jE7QRvhPf" - }, - "source": [ - "### Load the model into the interpreters" + "First, we need a function that runs inference with a given model and images, and then returns the predictions:\n" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", - "id": "Jn16Rc23zTss" + "id": "X092SbeWfd1A" }, "outputs": [], "source": [ - "interpreter = tf.lite.Interpreter(model_path=str(tflite_model_file))\n", - "interpreter.allocate_tensors()" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "J8Pztk1mvNVL" - }, - "outputs": [], - "source": [ - "interpreter_quant = tf.lite.Interpreter(model_path=str(tflite_model_quant_file))\n", - "interpreter_quant.allocate_tensors()\n", - "input_index_quant = interpreter_quant.get_input_details()[0][\"index\"]\n", - "output_index_quant = interpreter_quant.get_output_details()[0][\"index\"]" + "# Helper function to run inference on a TFLite model\n", + "def run_tflite_model(tflite_file, test_image_indices):\n", + " global test_images\n", + "\n", + " # Initialize the interpreter\n", + " interpreter = tf.lite.Interpreter(model_path=str(tflite_file))\n", + " interpreter.allocate_tensors()\n", + "\n", + " input_details = interpreter.get_input_details()[0]\n", + " output_details = interpreter.get_output_details()[0]\n", + "\n", + " predictions = np.zeros((len(test_image_indices),), dtype=int)\n", + " for i, test_image_index in enumerate(test_image_indices):\n", + " test_image = test_images[test_image_index]\n", + " test_label = test_labels[test_image_index]\n", + "\n", + " # Check if the input type is quantized, then rescale input data to uint8\n", + " if input_details['dtype'] == np.uint8:\n", + " input_scale, input_zero_point = input_details[\"quantization\"]\n", + " test_image = test_image / input_scale + input_zero_point\n", + "\n", + " test_image = np.expand_dims(test_image, axis=0).astype(input_details[\"dtype\"])\n", + " interpreter.set_tensor(input_details[\"index\"], test_image)\n", + " interpreter.invoke()\n", + " output = interpreter.get_tensor(output_details[\"index\"])[0]\n", + "\n", + " # Check if the output type is quantized, then rescale output data to float\n", + " if output_details['dtype'] == np.uint8:\n", + " output_scale, output_zero_point = output_details[\"quantization\"]\n", + " test_image = test_image.astype(np.float32)\n", + " test_image = test_image / input_scale + input_zero_point\n", + "\n", + " predictions[i] = output.argmax()\n", + "\n", + " return predictions\n" ] }, { @@ -546,62 +698,88 @@ }, "source": [ "### Test the models on one image\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "QpPpFPaz7eEM" + }, + "source": [ + "Now we'll compare the performance of the float model and quantized model:\n", + "+ `tflite_model_file` is the original TensorFlow Lite model with floating-point data.\n", + "+ `tflite_model_quant_file` is the last model we converted using integer-only quantization (it uses uint8 data for input and output).\n", "\n", - "First test it on the float model:" + "Let's create another function to print our predictions:" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "metadata": { "colab": {}, "colab_type": "code", - "id": "AKslvo2kwWac" + "id": "zR2cHRUcUZ6e" }, "outputs": [], "source": [ - "test_image = np.expand_dims(test_images[0], axis=0).astype(np.float32)\n", + "import matplotlib.pylab as plt\n", "\n", - "input_index = interpreter.get_input_details()[0][\"index\"]\n", - "output_index = interpreter.get_output_details()[0][\"index\"]\n", - "interpreter.set_tensor(input_index, test_image)\n", - "interpreter.invoke()\n", - "predictions = interpreter.get_tensor(output_index)" + "# Change this to test a different image\n", + "test_image_index = 1\n", + "\n", + "## Helper function to test the models on one image\n", + "def test_model(tflite_file, test_image_index, model_type):\n", + " global test_labels\n", + "\n", + " predictions = run_tflite_model(tflite_file, [test_image_index])\n", + "\n", + " plt.imshow(test_images[test_image_index])\n", + " template = model_type + \" Model \\n True:{true}, Predicted:{predict}\"\n", + " _ = plt.title(template.format(true= str(test_labels[test_image_index]), predict=str(predictions[0])))\n", + " plt.grid(False)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "A5OTJ_6Vcslt" + }, + "source": [ + "Now test the float model:" ] }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": { "colab": { - "height": 281 + "height": 296 }, "colab_type": "code", - "id": "XZClM2vo3_bm", - "outputId": "3af2e31c-44c6-41f2-c51f-da9d7b71bdfb" + "id": "iTK0x980coto", + "outputId": "1881b045-e953-416f-a25f-6c083409c7be" }, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAEICAYAAACQ6CLfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAFxZJREFUeJzt3XtU1HXeB/D3cE0RVDSG4eKMPJBL\nIrI6ZqXhBTFrVwwpw5WEAGnLc9ZL2nbbI1arPPV4nix99jRR7aiFz7qmtIu6KhulVrJj4baYHiKI\nq6DCE4pyG7/PH51mI5nf4DAX9Pt+neM5zO/z/f2+H37ynt/M/GbmpxJCCBCRdDzc3QARuQfDTyQp\nhp9IUgw/kaQYfiJJMfxEkmL4yeF6enqgUqlQXV0NAMjOzsaGDRucPm9+fj5mzpzp9HluFgy/nYYN\nG2b55+HhgSFDhlhuv/vuu06fPzs7u1cPvr6+GDlypNPntUd+fj6effZZm+OmT5+OP/7xj07p4Ztv\nvum1v4YNGwaVSoXNmzc7Zb4bgZe7G7hRXbp0yfKzTqdDfn4+5syZY3V8T08PvLwct7vz8/ORn59v\nuZ2WloahQ4c6bPs/Zjab4enp6ZRtu0pERESv/7Ovv/4a48aNw8KFC93YlXvxyO8kzz//PB5++GEs\nXrwY/v7+2LFjB9LS0pCbm2sZc/jwYeh0Osvturo6JCcn49Zbb8XYsWOxdevWfs118eJF7NmzB+np\n6f0a/8O8L7zwAkaNGoWxY8di586dlnpaWhqWL1+OefPmwc/PD0eOHEFHRwdWr16N8PBwqNVqPPHE\nE+jo6LCsk5eXh+DgYISGhsJoNPaa76e/9/vvv4+4uDgEBAQgMjISBw8exG9/+1t8+umn+PWvf41h\nw4Zh5cqVAIBTp05hzpw5CAwMxM9+9jPs3r3bsp1z587hl7/8JQICAnDnnXeiqqqqX78/ABiNRsye\nPRvh4eH9XuemI2jAtFqtOHToUK9lzz33nPD29hYffPCBMJvN4vLly2LJkiVi3bp1ljGHDh0SWq1W\nCCFET0+PmDhxovj9738vOjs7RUVFhdBqteLw4cNCCCFKSkrEqFGj+pz/rbfeEpGRkf3u99ChQ8LT\n01OsWbNGdHR0iOLiYjFkyBBRUVEhhBBiyZIlYsSIEeKTTz4RZrNZdHR0iOXLl4sHHnhAtLS0iO++\n+07cd9994vnnnxdCCPGXv/xFBAcHi/LycnHp0iXx0EMPCQCiqqrKsr0ffu9jx46J4cOHi8OHDwuz\n2SxqamrE6dOnhRBCTJs2TbzzzjuWPtva2kRISIgwGo2iu7tbmEwmERgYaBmfkpIiUlNTRXt7uzh5\n8qQIDg4WM2bMsKw/b9488corr1zz+1+9elVotVqxffv2fu+zmxHD7wDWwj9r1qxey5TCf/ToUTF2\n7Nhe41944QWRnZ1tc/74+Hjx4osv9rvfQ4cOCW9vb9He3m5ZlpycLDZs2GDp89FHH7XUzGaz8PX1\nFdXV1ZZlH3/8seUO55FHHhHPPfecpVZeXm41/JmZmWLNmjV99vXT8O/YsUPMnDmz15jMzEzx0ksv\nia6uLuHp6Wm5wxJCiLVr1/YKvzV///vfhb+/f6/fX0Z8zu9E1/OQ8ttvv0VNTQ1GjBhhWWY2m22+\nel1VVYWjR49i27Zt19XbqFGjer1GoNVq0dDQYLn9497Pnj2Lzs5OTJw40bJM/OjzYA0NDZg2bVqv\nbVlTW1uLKVOm9KvHb7/9FseOHeu1T3p6epCRkYGmpiaYzeZefWq1WpSWltrcrtFoxEMPPeS010hu\nFAy/E6lUql63/fz8cPnyZcvts2fPWn4ODw9HVFQUvvrqq+uaY9u2bZgxY4Zi4Ppy4cIFXLlyBUOG\nDAEA1NTUQK/X99m7Wq2Gj48Pzpw5A7Vafc22NBoNamtrLbdramqszhseHo7Kyso+az/dX+Hh4UhI\nSMD+/fuvGdvd3Q0PDw/U1tYiMjLS5rw/aG9vx+7du1FUVGRz7M2OL/i5UFxcHIqKitDa2orGxka8\n9tprltpdd90FHx8fbNq0CR0dHTCbzfjyyy9x4sQJxW1u27YNGRkZ1yxPS0tDdna21fWuXr2K3Nxc\ndHV1oaSkBPv378eDDz7Y51hPT09kZ2dj5cqVOHfuHIQQqKurw8GDBwEAixYtwttvv43Tp0+jvb0d\n69evtzpvVlYW8vPz8eGHH+Lq1auoq6vDmTNnAHx/J/PNN99YxiYlJaG8vBzvvfceuru70d3djdLS\nUpw5cwbe3t544IEHsG7dOly5cgX/+te/sH37dsV9BQC7d+9GUFAQ7rnnHptjb3YMvwtlZGQgOjoa\nWq0W8+bNQ2pqqqXm5eWFffv2obS0FDqdDqNHj8Zjjz2GtrY2AEBJSUmvh78AcOTIETQ1NSElJeWa\nuWpra3s9FP+psLAw+Pn5QaPRID09Hfn5+YiKirI6ftOmTdBqtbjjjjswfPhwzJ07FxUVFQCA+fPn\nY/ny5ZgxYwZuu+02JCYmWt3O3XffjTfffBO/+c1vMHz4cMyaNcvyqGHlypUoKCjAiBEjsHr1agwf\nPhx/+9vfsGPHDmg0GgQHB+OZZ55BZ2cnAOAPf/gDWltboVarkZWVhUcffbTXXHPnzsXLL7/ca5nR\naMTSpUuveZQhI5UQ/DKPm01HRwd+/vOf48svv+zzvQWHDx9Gdna25R14JCc+578J3XLLLdf92gHJ\nhw/7iSTFh/1EkuKRn0hSLn3O76PyxS3wc+WURFLpQDu6RGe/xg4o/AcOHMCKFStgNpuRnZ2Np59+\nWnH8LfDDVFXCQKYkIgXHRXG/x9r9sN9sNmP58uXYv38/Tp06hYKCApw6dcrezRGRi9kd/tLSUkRG\nRiIiIgI+Pj5ITU1FYWGhI3sjIieyO/z19fW9PlQRFhaG+vr6a8YZDAbo9Xro9Xp0o3/PRYjI+ewO\nf19nCPt6y2ROTg5MJhNMJhO84WvvdETkYHaHPywsrNcnuerq6hASEuKQpojI+ewO/5QpU1BRUYGq\nqip0dXVh586dSEpKcmRvROREdp/q8/LywpYtW3DvvffCbDYjMzMT48ePd2RvRORELn17b4AqkOf5\niZzouChGm2jp11i+vZdIUgw/kaQYfiJJMfxEkmL4iSTF8BNJiuEnkhTDTyQphp9IUgw/kaQYfiJJ\nMfxEkmL4iSTF8BNJiuEnkhTDTyQphp9IUgw/kaQYfiJJMfxEkmL4iSTF8BNJiuEnkhTDTyQphp9I\nUgw/kaQYfiJJMfxEkmL4iSTF8BNJymsgK+t0Ovj7+8PT0xNeXl4wmUyO6ouInGxA4QeADz/8EKNH\nj3ZEL0TkQnzYTySpAYVfpVJh7ty5mDx5MgwGQ59jDAYD9Ho99Ho9utE5kOmIyIFUQghh78oNDQ0I\nCQlBc3MzEhMT8frrryM+Pt7q+ABVIKaqEuydjohsOC6K0SZa+jV2QEf+kJAQAEBQUBCSk5NRWlo6\nkM0RkQvZHf729nZcvHjR8vPBgwcRExPjsMaIyLnsfrW/qakJycnJAICenh786le/wrx58xzWGBE5\nl93hj4iIwMmTJx3ZCxG5EE/1EUmK4SeSFMNPJCmGn0hSDD+RpAb8wR5ZXFh2l9XamEe+Vlz3dLNa\nsd7V6a1YDy1Qrg+tu2S1drXslOK6JC8e+YkkxfATSYrhJ5IUw08kKYafSFIMP5GkGH4iSfE8fz89\ntfY9q7UUv1bllf9jgJPPVC5X91y2Wtt8btYAJ79xlTZrrdb8Ng1XXNer+ISj2xl0eOQnkhTDTyQp\nhp9IUgw/kaQYfiJJMfxEkmL4iSQ1oCv2XK8b+Yo97Q9OtVo7H6t8HzryK+Vd3BqtUqz7xP6fYv3l\nmPet1hKHXFFct+jyMMX6L4Za/66AgboiuhTrxzv9FOszb+m2e+7IoscU67fl/MPubbuTy67YQ0Q3\nLoafSFIMP5GkGH4iSTH8RJJi+IkkxfATSYqf5+8nvz8fV6gNbNsBA1sdrwfPtFp7aZpOee6PlK85\n8PLMSDs66h+vK1cV637/bFSsj/p4t2J9go/16x0MrVa+FoIMbB75MzMzERQUhJiYGMuylpYWJCYm\nIioqComJiWhttfFlFkQ06NgMf0ZGBg4cONBrWV5eHhISElBRUYGEhATk5eU5rUEicg6b4Y+Pj0dg\nYGCvZYWFhUhPTwcApKenY+/evc7pjoicxq7n/E1NTdBoNAAAjUaD5uZmq2MNBgMMBgMAoBud9kxH\nRE7g9Ff7c3JyYDKZYDKZ4A1fZ09HRP1kV/jVajUaG79/JbaxsRFBQUEObYqInM+u8CclJcFoNAIA\njEYjFixY4NCmiMj5bD7nX7x4MUpKSnD+/HmEhYVh/fr1ePrpp7Fo0SK89dZbGDNmDHbt2uWKXsmK\nnrNNVmt+u63XAMBsY9t+f75gR0eO0ZR9l2J9vI/yn+9/tYyzWtO9843iuj2K1ZuDzfAXFBT0uby4\nuNjhzRCR6/DtvUSSYviJJMXwE0mK4SeSFMNPJCl+pJfcxksbrljf8uwWxbq3ylOxvmvzHKu1UY2f\nKq4rAx75iSTF8BNJiuEnkhTDTyQphp9IUgw/kaQYfiJJ8Tw/uc3pVaGK9Sm+ypcuL+9Svvx44KnL\n192TTHjkJ5IUw08kKYafSFIMP5GkGH4iSTH8RJJi+IkkxfP85FSdv5hitfb5g/9tY23lKzw9vmKF\nYn3IJ6U2ti83HvmJJMXwE0mK4SeSFMNPJCmGn0hSDD+RpBh+IknxPD85Vc191o8vw1TK5/EXVyUq\n1oceOKlYF4pVsnnkz8zMRFBQEGJiYizLcnNzERoairi4OMTFxWHfvn1ObZKIHM9m+DMyMnDgwIFr\nlq9atQplZWUoKyvD/fff75TmiMh5bIY/Pj4egYGBruiFiFzI7hf8tmzZgtjYWGRmZqK1tdXqOIPB\nAL1eD71ej2502jsdETmYXeF//PHHUVlZibKyMmg0Gjz55JNWx+bk5MBkMsFkMsHbxgc1iMh17Aq/\nWq2Gp6cnPDw8sGzZMpSW8tNTRDcau8Lf2Nho+XnPnj29zgQQ0Y3B5nn+xYsXo6SkBOfPn0dYWBjW\nr1+PkpISlJWVQaVSQafT4Y033nBFrzQIefj7K9Yfueeo1Vrb1Q7FdZs3RCjWfTv/oVgnZTbDX1BQ\ncM2yrKwspzRDRK7Dt/cSSYrhJ5IUw08kKYafSFIMP5Gk+JFeGpCK3PGK9b+O/h+rtQUVKYrr+u7j\nqTxn4pGfSFIMP5GkGH4iSTH8RJJi+IkkxfATSYrhJ5IUz/OTou/S7lSs//Ph1xTrlT3dVmuX/jNM\ncV1fNCrWaWB45CeSFMNPJCmGn0hSDD+RpBh+Ikkx/ESSYviJJMXz/JLzCg1RrK/83f8q1n1Vyn9C\nqScfsVq7dT8/r+9OPPITSYrhJ5IUw08kKYafSFIMP5GkGH4iSTH8RJKyeZ6/trYWS5cuxdmzZ+Hh\n4YGcnBysWLECLS0tePjhh1FdXQ2dToc//elPGDlypCt6puug8lL+L5741zrF+kPDLijW370YpFhX\n/8768eWq4prkbDaP/F5eXti0aRO++uorfPbZZ9i6dStOnTqFvLw8JCQkoKKiAgkJCcjLy3NFv0Tk\nIDbDr9FoMGnSJACAv78/oqOjUV9fj8LCQqSnpwMA0tPTsXfvXud2SkQOdV3P+aurq/HFF19g6tSp\naGpqgkajAfD9HURzc7NTGiQi5+j3e/svXbqElJQUvPrqqwgICOj3BAaDAQaDAQDQjc7r75CInKJf\nR/7u7m6kpKRgyZIlWLhwIQBArVajsfH7L1hsbGxEUFDfL/zk5OTAZDLBZDLBG74OapuIBspm+IUQ\nyMrKQnR0NFavXm1ZnpSUBKPRCAAwGo1YsGCB87okIodTCSGE0oCjR4/innvuwYQJE+Dh8f19xYYN\nGzB16lQsWrQINTU1GDNmDHbt2oXAwEDFyQJUgZiqSnBc92STarLyJbSLPtg+oO3f/cxyxfqIbZ8O\naPt0fY6LYrSJln6Ntfmcf/r06bB2/1BcXHx9nRHRoMF3+BFJiuEnkhTDTyQphp9IUgw/kaQYfiJJ\n8au7bwKet99mtZazs3BA2779beXz+Lrtnw1o++Q+PPITSYrhJ5IUw08kKYafSFIMP5GkGH4iSTH8\nRJLief6bwOknrH9l+vyhbQPadlhJl/IA5a+DoEGMR34iSTH8RJJi+IkkxfATSYrhJ5IUw08kKYaf\nSFI8z38D6Jh/h2K9eP4mhepQxzZDNw0e+YkkxfATSYrhJ5IUw08kKYafSFIMP5GkGH4iSdk8z19b\nW4ulS5fi7Nmz8PDwQE5ODlasWIHc3Fy8+eabuPXWWwEAGzZswP333+/0hmXUMM1TsT7Gy/5z+e9e\nDFKse7cpf56fn+a/cdkMv5eXFzZt2oRJkybh4sWLmDx5MhITEwEAq1atwpo1a5zeJBE5ns3wazQa\naDQaAIC/vz+io6NRX1/v9MaIyLmu6zl/dXU1vvjiC0ydOhUAsGXLFsTGxiIzMxOtra19rmMwGKDX\n66HX69GNzoF3TEQO0e/wX7p0CSkpKXj11VcREBCAxx9/HJWVlSgrK4NGo8GTTz7Z53o5OTkwmUww\nmUzwhq/DGieigelX+Lu7u5GSkoIlS5Zg4cKFAAC1Wg1PT094eHhg2bJlKC0tdWqjRORYNsMvhEBW\nVhaio6OxevVqy/LGxkbLz3v27EFMTIxzOiQip7D5gt+xY8ewfft2TJgwAXFxcQC+P61XUFCAsrIy\nqFQq6HQ6vPHGG05vlq7fxgu3K9Y/vVenWBeNXzqwGxpMbIZ/+vTpEH18NzvP6RPd2PgOPyJJMfxE\nkmL4iSTF8BNJiuEnkhTDTyQplejrPJ6TBKgCMVWV4KrpiKRzXBSjTbT0ayyP/ESSYviJJMXwE0mK\n4SeSFMNPJCmGn0hSDD+RpFx6iW6fUR5o1VVZbp87d87y1d+DzWDtbbD2BbA3ezmyN5/q/h/PXfom\nn5/S6/UwmUzuml7RYO1tsPYFsDd7uas3PuwnkhTDTyQpz9zc3Fx3NjB58mR3Tq9osPY2WPsC2Ju9\n3NGbW5/zE5H78GE/kaQYfiJJuSX8Bw4cwLhx4xAZGYm8vDx3tGCVTqezXKNAr9e7tZfMzEwEBQX1\nuiBKS0sLEhMTERUVhcTERKvXSHRHb7m5uQgNDUVcXBzi4uKwb98+t/RWW1uLWbNmITo6GuPHj8fm\nzZsBuH/fWevLbftNuFhPT4+IiIgQlZWVorOzU8TGxory8nJXt2GVVqsV586dc3cbQgghPvroI3Hi\nxAkxfvx4y7K1a9eKjRs3CiGE2Lhxo3jqqacGTW/r1q0Tr7zyilv6+bGGhgZx4sQJIYQQbW1tIioq\nSpSXl7t931nry137zeVH/tLSUkRGRiIiIgI+Pj5ITU1FYWGhq9u4IcTHxyMwMLDXssLCQqSnpwMA\n0tPTsXfvXne01mdvg4VGo8GkSZMA9L6svLv3nbW+3MXl4a+vr0d4eLjldlhYmFt3wE+pVCrMnTsX\nkydPhsFgcHc712hqaoJGowHw/R9Tc3OzmzvqrT+XbXelH19WfjDtO3sud+9oLg+/6OPMokqlcnUb\nVh07dgyff/459u/fj61bt+Ljjz92d0s3jP5ett1VfnpZ+cHC3svdO5rLwx8WFoba2lrL7bq6OoSE\nhLi6Dat+6CUoKAjJycmD7tLjarXacoXkxsZGBAUFubmjfxtMl223dll5d++7wXS5e5eHf8qUKaio\nqEBVVRW6urqwc+dOJCUlubqNPrW3t+PixYuWnw8ePDjoLj2elJQEo9EIADAajViwYIGbO/q3wXLZ\ndmHlsvLu3nfW+nLbfnP5S4xCiKKiIhEVFSUiIiLESy+95I4W+lRZWSliY2NFbGysuP32293eW2pq\nqggODhZeXl4iNDRU5Ofni/Pnz4vZs2eLyMhIMXv2bHHhwoVB01taWpqIiYkREyZMEPPnzxcNDQ1u\n6e3IkSMCgJgwYYKYOHGimDhxoigqKnL7vrPWl7v2G9/eSyQpvsOPSFIMP5GkGH4iSTH8RJJi+Ikk\nxfATSYrhJ5LU/wOdAGX9nfSgHgAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEXCAYAAABrgzLrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVZUlEQVR4nO3de9RVdZ3H8fcH5aKICsIQIEFeWN5mxGK8pGM2aBplWtNYTBk2GjVljrOYlWatpEmdVpNZM5VGaqJ5ibximomUYxqhaCgqlTcU6EE0YEArLo/f+WPvpw6Pz9nn4dwffp/XWmdxzv7ty5cDn7Ovv70VEZjZ9q9fqwsws+Zw2M0S4bCbJcJhN0uEw26WCIfdLBEOex8habykkLRjq2uplqSrJF3Qy3GXSTq20TWlxGFvM/l/8j9KeqXkNbrOywhJ+xS0n5aPc0m34Sflw6+qZz3WHA57ezoxInYpef2uBTU8A5zSbUtiGvDbFtRideCw91GSRkuaK2mNpKclfayk7VBJCyStk9Qh6ZuSBuRt9+WjPZpvNXygzCJWAUuA4/PphgFvBeZ2q+M9kp7Il3WvpP1L2g6R9IikDZJ+AAzqNu27JS3Op/2FpL+p8WuxAg5733UDsAIYDbwfuEjS3+dtncC/AcOBI4DJwCcBIuLofJyD862GHxQs42rgI/n7DwK3ARu7GiVNAK4HzgZGAHcCt0sakP+43ApcAwwDfgj8Q8m0hwBXAh8H9gC+A8yVNHCbvwnrFYe9Pd2ar+3WSbq1e6OkscCRwDkR8aeIWAxcTh7MiHg4In4ZEVsiYhlZkN5WRR23AMdI2i2f99Xd2j8A3BER8yJiM/BVYCeyLYDDgf7A1yNic0TcCDxUMu104DsRsTAiOiNiNtkPyeFV1Gm94LC3p5MjYvf8dXIP7aOBNRGxoWTY88AYyNa4kn4kaZWk9cBFZGv5bRIRfwTuAD4P7BERD/RQx/Ml478GLM/rGA2sjK17Wj1f8n4cMKPkR20dMDafzhrAYe+bfgcMkzSkZNgbgZX5+0uBXwP7RsSuwHmAqlzW1cAM4Ptl6hjX9UGSyAK7EugAxuTDSmvsshy4sORHbfeI2Dkirq+yTqvAYe+DImI58AvgPyUNyg9snc5fAjkEWA+8Imk/4F+6zeJFYK9eLu5/geOA/+mhbQ7wLkmTJfUn+1HYmNe2ANgCnCWpv6T3AYeWTPtd4BOSDlNmsKR3dfsBszpy2PuuqcB4srXrLcD5EXFP3vbvwD8BG8hC1f0g3Exgdr75fErRQiIzPyLW9ND2G+DDZD8ELwMnkp023BQRm4D3AacBa8j2728umXYR8DHgm8Ba4Ol8XGsQ+eYVZmnwmt0sEQ67WSIcdrNEOOxmiXDYrSFKu6hKOk/S5U1Y5jGSVjR6OX2Vw14jSW/s1h01JL1a8vnvGrjsd0m6Pz+FtkrS5b09T13SP76rzmWSzm1EnRFxUUSc0Yuaet3ffVtJGijpCknP5x1zFkt6ZyOW1a4c9hpFxAul3VHzwQeXDPt517gNuPHEbsAFZJeY7k92mep/beM8ds/rngp8QdIJ3UfoyzfMKLEj2VV7byP73j4PzJE0voU1NZXD3kD5TSAekHSJpN8DMyXNlPT9knG2ugONpN3yNVCHpJWSLpC0Q0/zj4jrIuKuiPhDRKwlu4DmyGpqjYgFwBPAQV2bw5LOkbQK+J6kfpLOlfSMpN9LmpN3e+36e5yarzV/L+lz3b6H7n/no/IureskLc+/p+nAh4DP5Fsat+fjjpZ0k6SXJD0n6ayS+eyUbw2slfQk8LcFf79XI2JmRCyLiNci4kfAc8Bbqvm++iKHvfEOA54FRgIX9mL8q8guM90HOAR4B3AG/HmXYZ2kN5aZ9miywG6T/HLVI4EDgV/lg99A1jV1HFkPtU8DJ5OtGUeTXfX2rXz6A8iuxz81b9sD2LPMssYBPya76m4EMBFYHBGzgGuBr+RbRCdK6gfcDjxKttUyGThb0vH57M4H9s5fx5PdXKN0Wd+W9O0ydYwEJlDF99VnRYRfdXwBAeyTvz8NeKFb+0zg+yWfx+fT7Ej2g7AR2KmkfSrws14s9ziyAE7oZZ1dy12XT7cUOCtvOwbYBAwqGX8pMLnk8yhgc173F4AbStoG59Mf2/3vDHwWuKVMTVcBF5R8PqyH7++zwPfy988CJ5S0TQdW9OLv3h+4h6yLbcv/zzTrtT3si7W75dsw7jiy/4gdJZ3F+lWah6TDgeuA90fEtt42anhEbOlh+EsR8adutd0i6bWSYZ1kP1CjS2uMiFfz3ZaejCW75VVvjANGK+v+2mUHoOs4yFbLZesutD3KtxauIfsxOrOXdWwXHPbG69754FVg55LPbyh5v5xszV4ugK+j7I4vc4F/joj5tRTaTfe6l+fL6N6nHUkdZAcIuz7vTLYp35PlbN37rdIyn4uIfcuM30H249G1KV5u96arLgFXkP1ATYnshhvJ8D578y0Gjs73v3cj2ywFICI6gLuBiyXtmh8U21tSj3eZkXQQcBfw6Yi4vYf2mZLurVPdlwEX5vvcSBoh6aS87Ubg3fmBtwHAf1D+/9a1wLGSTpG0o6Q9JE3M27p3vX0Q2JAfKNxJ0g6SDpLUdSBuDvBZSUMl7Ul2XKHIpWQ/SidGdmOOpDjsTRYR88i6nD4GPAz8qNsoHwEGAE+S7UvfSLZ/XHpOv2sNNoPsINcVJefLSw84jQVetyau0jfItiDulrQB+CXZPjUR8QTwKbJdiY687h4vbomIF4Apee1ryH78Ds6brwAOyA9C3hoRncC7yQ7iPUfWjfZyslNnAF8k23R/juxH8prSZUm6TNJl+ftxZPe7mwisKvm+PlTLl9KXuIvrdkzSYrKDauX2ny0hDrtZIrwZb5YIh90sEQ67WSKaep59gAbGIAY3c5FmSfkTr7IpNvZ42/Cawp73kPoG2VVNl0fEl4vGH8RgDtPkWhZpZgUWFlxXVfVmfN4T61vAO4EDgKl5hwgza0O17LMfCjwdEc9Gdo/wG4CTKkxjZi1SS9jHsHUnhBX5sK1Imi5pkaRFm//yAFAza7KGH42PiFkRMSkiJvXHT+M1a5Vawr6S7NrrLnvylwcLmlmbqSXsDwH7SnpT3tPpg2QdJcysDVV96i0itkg6E/gJ2am3K/PeT2bWhmo6zx4RdwJ31qkWM2sgXy5rlgiH3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhMNulgiH3SwRDrtZIhx2s0Q47GaJ8COb+4BlFxxR2N45qPwjvEYc+FLhtAsOvqmqmrrs/dOPFrYPeXCnsm0j//sXNS3bto3X7GaJcNjNEuGwmyXCYTdLhMNulgiH3SwRDrtZInyevQ2svWPfwvbHJ36zYcveXP4Ufa/8+u2XF7ZfO2lU2bY5895WOG3n0qeqqsl65jW7WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TDbpYIn2dvgkrn0R+YeEPDln3Zur0K27+24LjC9vHjivvD333AzYXtHxrSUbbtwtOGF0671zk+z15PNYVd0jJgA9AJbImISfUoyszqrx5r9rdHxMt1mI+ZNZD32c0SUWvYA7hb0sOSpvc0gqTpkhZJWrSZjTUuzsyqVetm/FERsVLSXwHzJP06Iu4rHSEiZgGzAHbVsBq7XZhZtWpas0fEyvzP1cAtwKH1KMrM6q/qsEsaLGlI13vgHcDj9SrMzOqrls34kcAtkrrmc11E3FWXqvqYLZPfUtj+04O/VWEO/Qtbv752QmH7zz5QcMbzd6sLp52wdlFhe79BgwrbL1r414Xt5w1fUrZty9AthdNafVUd9oh4Fji4jrWYWQP51JtZIhx2s0Q47GaJcNjNEuGwmyXCXVzr4JUxAwrb+1X4Ta10au3e9xSf3up89jeF7bV4+ouHFLZfN+ziCnMYWLZlz7u8rmkmf9tmiXDYzRLhsJslwmE3S4TDbpYIh90sEQ67WSJ8nr0Odr96QWH7+xd9uLBda9cXtm/pWLaNFdXPGVPuKWzfpV/58+jWXrxmN0uEw26WCIfdLBEOu1kiHHazRDjsZolw2M0S4fPsTdD55G9bXUJZyy48orD99N2/WmEOxbeantFxeNm2IfcsLZy2s8KSbdt4zW6WCIfdLBEOu1kiHHazRDjsZolw2M0S4bCbJcLn2bdz604tPo/+wEeKz6Pv1q/4PPqCjTsUti++oPx953da/2DhtFZfFdfskq6UtFrS4yXDhkmaJ+mp/M+hjS3TzGrVm834q4ATug07F5gfEfsC8/PPZtbGKoY9Iu4D1nQbfBIwO38/Gzi5znWZWZ1Vu88+MiI68vergJHlRpQ0HZgOMIidq1ycmdWq5qPxERFAFLTPiohJETGpf8FD/syssaoN+4uSRgHkf66uX0lm1gjVhn0uMC1/Pw24rT7lmFmjVNxnl3Q9cAwwXNIK4Hzgy8AcSacDzwOnNLJIq97Lby67hwVUPo9eybR7zyhsn3Crz6W3i4phj4ipZZom17kWM2sgXy5rlgiH3SwRDrtZIhx2s0Q47GaJcBfX7cCmeePKti3Y7+IKUxefejt4wbTC9v1nPFPY7ttBtw+v2c0S4bCbJcJhN0uEw26WCIfdLBEOu1kiHHazRPg8ex+w417jC9u/tM8Py7YNrdCF9eGNxcse96XiM+Wda9cWz8DahtfsZolw2M0S4bCbJcJhN0uEw26WCIfdLBEOu1kifJ69D9h7zsrC9kMGVP+bPXX+JwrbJzz6UNXztvbiNbtZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhMNulgifZ28Da6cdUdj+xZGV7v0+sGzLtGXHFk65/2eeLmz3fd+3HxXX7JKulLRa0uMlw2ZKWilpcf6a0tgyzaxWvdmMvwo4oYfhl0TExPx1Z33LMrN6qxj2iLgPWNOEWsysgWo5QHempMfyzfyh5UaSNF3SIkmLNlPhhmdm1jDVhv1SYG9gItABlD2CFBGzImJSREzqX3Agycwaq6qwR8SLEdEZEa8B3wUOrW9ZZlZvVYVd0qiSj+8FHi83rpm1h4rn2SVdDxwDDJe0AjgfOEbSRCCAZcDHG1hjn7fjmNGF7X931sLC9l36Vb/7s+DJfQrbJ6x1f/VUVAx7REztYfAVDajFzBrIl8uaJcJhN0uEw26WCIfdLBEOu1ki3MW1CZaeN7aw/dY33F7T/N++5B/LtrkLq3Xxmt0sEQ67WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TPszfBw++5pMIYtd3BZ7dPvla2bcvatTXN27YfXrObJcJhN0uEw26WCIfdLBEOu1kiHHazRDjsZonwefbtwOaRu5Vt679pTBMreb3Ol14u2xYbix8HpoHF1x/sMGJ4VTUBdI7YvbD9qRkDqp53b0Snyrbt9+kK9yBYv76qZXrNbpYIh90sEQ67WSIcdrNEOOxmiXDYzRLhsJslojePbB4LXA2MJHtE86yI+IakYcAPgPFkj20+JSLceboF7rjxylaXUNZbf9XTQ4AzL7+4a+G0Q0dsKGxf+Jbrqqqp3R3w+TML2/f6zIKq5tubNfsWYEZEHAAcDnxK0gHAucD8iNgXmJ9/NrM2VTHsEdEREY/k7zcAS4ExwEnA7Hy02cDJjSrSzGq3TfvsksYDhwALgZER0ZE3rSLbzDezNtXrsEvaBbgJODsitro4NyKCbH++p+mmS1okadFmiq+FNrPG6VXYJfUnC/q1EXFzPvhFSaPy9lHA6p6mjYhZETEpIib1r/HGimZWvYphlyTgCmBpRHytpGkuMC1/Pw24rf7lmVm9KNsCLxhBOgr4ObAE6Lpn8Xlk++1zgDcCz5OdeltTNK9dNSwO0+Raa+5z/viTNxW2zz/oxiZVkpY/xKaybZuj/O23e2PKY6cVtv/f4uq73466f0th+8AfP1S2bWHMZ32s6bH/bMXz7BFxP1Cu8216yTXro3wFnVkiHHazRDjsZolw2M0S4bCbJcJhN0uEbyXdBDsd/1xh+4EXFXdpjAb+Kw3Zr/DSiIZ2Iz3w5x8tbI8XBtc0/71ufKV844NLapr3UJ6qqb0VvGY3S4TDbpYIh90sEQ67WSIcdrNEOOxmiXDYzRJRsT97PaXan92sWYr6s3vNbpYIh90sEQ67WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TDbpYIh90sEQ67WSIcdrNEOOxmiXDYzRLhsJslomLYJY2V9DNJT0p6QtK/5sNnSlopaXH+mtL4cs2sWr15/MAWYEZEPCJpCPCwpHl52yUR8dXGlWdm9VIx7BHRAXTk7zdIWgqMaXRhZlZf27TPLmk8cAiwMB90pqTHJF0paWiZaaZLWiRp0WY21lSsmVWv12GXtAtwE3B2RKwHLgX2BiaSrfkv7mm6iJgVEZMiYlJ/BtahZDOrRq/CLqk/WdCvjYibASLixYjojIjXgO8ChzauTDOrVW+Oxgu4AlgaEV8rGT6qZLT3Ao/Xvzwzq5feHI0/EjgVWCJpcT7sPGCqpIlAAMuAjzekQjOri94cjb8f6Ok+1HfWvxwzaxRfQWeWCIfdLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEw26WCIfdLBEOu1kiHHazRDjsZolw2M0SoYho3sKkl4DnSwYNB15uWgHbpl1ra9e6wLVVq561jYuIET01NDXsr1u4tCgiJrWsgALtWlu71gWurVrNqs2b8WaJcNjNEtHqsM9q8fKLtGtt7VoXuLZqNaW2lu6zm1nztHrNbmZN4rCbJaIlYZd0gqTfSHpa0rmtqKEcScskLckfQ72oxbVcKWm1pMdLhg2TNE/SU/mfPT5jr0W1tcVjvAseM97S767Vjz9v+j67pB2A3wLHASuAh4CpEfFkUwspQ9IyYFJEtPwCDElHA68AV0fEQfmwrwBrIuLL+Q/l0Ig4p01qmwm80urHeOdPKxpV+phx4GTgNFr43RXUdQpN+N5asWY/FHg6Ip6NiE3ADcBJLaij7UXEfcCaboNPAmbn72eT/WdpujK1tYWI6IiIR/L3G4Cux4y39LsrqKspWhH2McDyks8raK/nvQdwt6SHJU1vdTE9GBkRHfn7VcDIVhbTg4qP8W6mbo8Zb5vvrprHn9fKB+he76iIeDPwTuBT+eZqW4psH6ydzp326jHezdLDY8b/rJXfXbWPP69VK8K+Ehhb8nnPfFhbiIiV+Z+rgVtov0dRv9j1BN38z9UtrufP2ukx3j09Zpw2+O5a+fjzVoT9IWBfSW+SNAD4IDC3BXW8jqTB+YETJA0G3kH7PYp6LjAtfz8NuK2FtWylXR7jXe4x47T4u2v5488joukvYArZEflngM+1ooYyde0FPJq/nmh1bcD1ZJt1m8mObZwO7AHMB54C7gGGtVFt1wBLgMfIgjWqRbUdRbaJ/hiwOH9NafV3V1BXU743Xy5rlggfoDNLhMNulgiH3SwRDrtZIhx2s0Q47GaJcNjNEvH/9ALsS7Cy9ngAAAAASUVORK5CYII=\n", "text/plain": [ - "\u003cFigure size 600x400 with 1 Axes\u003e" + "\u003cFigure size 432x288 with 1 Axes\u003e" ] }, "metadata": { + "needs_background": "light", "tags": [] }, "output_type": "display_data" } ], "source": [ - "import matplotlib.pylab as plt\n", - "\n", - "plt.imshow(test_images[0])\n", - "template = \"True:{true}, predicted:{predict}\"\n", - "_ = plt.title(template.format(true= str(test_labels[0]),\n", - " predict=str(np.argmax(predictions[0]))))\n", - "plt.grid(False)" + "test_model(tflite_model_file, test_image_index, model_type=\"Float\")" ] }, { @@ -611,41 +789,37 @@ "id": "o3N6-UGl1dfE" }, "source": [ - "Now test the quantized model (using the uint8 data):" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": { - "colab": {}, - "colab_type": "code", - "id": "3gwhv4lKbYZ4" - }, - "outputs": [], - "source": [ - "input_index = interpreter_quant.get_input_details()[0][\"index\"]\n", - "output_index = interpreter_quant.get_output_details()[0][\"index\"]\n", - "interpreter_quant.set_tensor(input_index, test_image)\n", - "interpreter_quant.invoke()\n", - "predictions = interpreter_quant.get_tensor(output_index)" + "And test the quantized model:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "colab": {}, + "colab": { + "height": 296 + }, "colab_type": "code", - "id": "CIH7G_MwbY2x" + "id": "rc1i9umMcp0t", + "outputId": "480bc68f-812b-460e-82fe-d66f70b4345e" }, - "outputs": [], + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEXCAYAAABrgzLrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAWRklEQVR4nO3de9RVdZ3H8fcHRVRQBHEQ0SBvldoSi9FKKxu1lKm0Vjk5jWLlYGuyci3XlGlTNKPWNJrZTQcvqeUl0kwtM5VyeYkx0UhQKm94oUfRwEQtBPzOH/v32PHhnH0O5w6/z2utZ3HO/u3L9zk8n7Ovv70VEZjZhm9Yrwsws+5w2M0y4bCbZcJhN8uEw26WCYfdLBMOu1kmHPYMSHpO0o5tnufNko5p5zzbuUxJIWnnTte0PnHYO0DS0ZIWSHpB0hOSviNpdJeWvVYgImJURDzUjeWnGmamsH16yPBPp+Ezu1WL/Y3D3maSTgD+G/h3YDTwJmAycIOk4T0srdv+ABw1ZNj0NNx6wGFvI0lbAl8CPhkR10fEqohYDBwO7Aj8cxrvQkmnVEy3v6THK96fKOlBSSsk3SfpfRVtR0u6TdLpkpZLeljSIantVOCtwLfSpvu30vCQtLOk7dLwwZ8XJEXFvD8qaVGa788lTapoO0jS7yT9Oc1XdT6OO4HNJe2ept8d2DQNr/zM/lXSA5KWSbpG0naNLrOsXlubw95eb6H4g/5R5cCIeA64Dnhng/N5kCK0oym+PL4vaUJF+z7A74FxwFeB8yUpIk4GbgWOS5vuxw2p449p+KiIGAVcBVwOIOlQ4CTg/cA2aT6XpbZx6Xf6fFrmg8C+Dfwe3+Nva/fp6f3LJP0D8GWKL8MJwCMV9ZQus6xeq85hb69xwNMRsbpK2wDFH2VdEfHDFMyXIuIHwP3A3hWjPBIR50bEGuAiiqCMX5dCJX0WeC3w0TTo48CXI2JRqv80YEpaW04D7o2IKyJiFfB14IkGFvN94Ii0+/Kh9L7Sh4ELIuLuiFgJfA54s6TJDSyzrF6rwmFvr6eBcZI2rtI2IbXXJekoSfMlPSPpGWAPii+SQS//0UfEC+nlqEaLTJv9nwYOi4i/pMGTgLMqlrmMYrN5IrAd8FjFMqPyfS0R8SjwAEUQ74+IodNsR7E2Hxz/OeBPDS6zrF6rwmFvr7nASopNy5dJGgUcAtycBj0PbF4xyrYV404CzgWOA7aOiK2AhdTfRx5U2mdZ0msotgYOHxK+x4BjI2Krip/NIuJXFFslO1TMQ5Xv67gYOCH9O9QfKUI7ON+RwNbAkgaWWVavVeGwt1FE/JliH/ubkg6WNDxtks6mWKtfkkadD0yTNFbStsDxFbMZSRHYpwAkfYRizd6oJykOBq4lHUC8Gjg5Im4b0nwO8LmKA2qjJX0wtf0U2F3S+9NWy6eo+IKq4wcUxypmV2m7DPiIpCmSRlBsAdyRDmrWW2ZZvVaFw95mEfFVigNHpwMrgIcp1uIHRsTzabTvAb8FFgM3UARicPr7gDMothKeBF4P3L4OJZwFfCAdof7GkLY3AK8Bzqw8Kp+WexXFKcPLJT1LsTVxSGp7Gvgg8BWKzexdGq0pIv4SETdV7C5Utt0E/AdwJcWafCeKffu6yyyr16qT71TTWWnN/J/Avmkf1qwnHPYukHQksCoiLu91LZYvh90sE95nN8uEw24dIWmxpAPT65MkndeFZb7ismN7JYe9RZJeNeR685D0fMX7t3Zw2f+YrpN/RkXvuvMkbdHgtJNTrYN1LpZ0YifqjIjTIqJu11QN6TPQTpJGSDpf0iMq+hzMTxcXZcNhb1FEPDrkenOAPSuG3To4bo0r61oxGjiF4mqz11FcPfY/6ziPrVLdRwBfkHTw0BE6UHcvbExxIc7bKT63zwOz03UQWXDYO0hFD7XbJZ0p6U/ATBV9vb9fMc7gGnbj9H50WgMNSFoi6RRJG1Wbf0RcmnrXvRARyymuvGukg0q1ec0F7gX2GNwclvRZSU8A35U0TH/rjfcnSbMlja34PY5Ma80/STp5yOcw9HfeT9Kv0hbJY+lzmkFxrfxn0pbGtWnc7SRdKekpFT38PlUxn83S1sBySfcBf1/y+z0fETMjYnHqc/ATimsg3tjM57U+ctg7bx/gIYqOKqc2MP6FwGpgZ2AviqvPjoGXdxmekfSqGtO+jSKw60SFfYHdgd+kwdsCYykuZ50BfBI4jGLNuB2wHPh2mn434GzgyNS2NbB9jWVNAn4GfJOiY9AUYH5EzKK4wvCraYvoPZKGAddSXIA0ETgAOF7Su9LsvkhxIc5OwLsoetZVLus7kr5To47xwK408XmttyLCP238objUdef0+mjg0SHtM4HvV7yfnKbZmOILYSWwWUX7EcAvG1juQRQB3LXBOgeX+0yabhHwqdS2P/AisGnF+IuAAyreTwBWpbq/AFxe0TYyTX/g0N+ZomfbVTVquhA4peL9PlU+v88B302vHwIOrmibATzewO8+HLgJ+N9e/71082dD2Bfrd3V7h1WYRPGHOCC93O9lWL15SHoTcCnwgYhY1zvBjIvqXXKfioi/DqntKkkvVQxbQ/EFNbSH2vNpt6WaHSj6pjdiErCdil5tgzai6LvO0OVS0YOulrS18D2KL6Pj6oy+QXHYO2/oVUs1e7xR/OGupHYA1yJpL+Aa4KMRMaeVQocYWvdjaRlrXRMvaYDiAOHg+80pNuWreYxX9s2vt8yHI2KXGuMP9owb3BSvtXszWJeA8ym+oKZF0U8+G95n7775wNvS/vdois1SACJigKJjzBmStkwHxXaS9PZqM5K0B3A9xW2wrq3SPlPSzW2q+xzg1LTPjaRtVNwtBuAK4N3pwNsmFH0Bav1tXQIcKOlwSRtL2lrSlNQ2tMfer4EV6UDhZpI2krSHpMEDcbMper6NkbQ9xXGFMmdTfCm9J6p0zNnQOexdFhE3UvRyuwe4C/jJkFGOAjYB7qPYl76CYv+48pz+4BrsBIqDXOdXnC+vPOC0A+vWY67MWRRbEDdIWgH8H8U+NRFxL/AJil2JgVR31YtbougMNC3Vvoziy2/P1Hw+sFs6CPnjKO7E826Kg3gPU3QTPo/i1BkU3YkfSW03sPZtr86RdE56PQk4Ns3riYrP68OtfCjrE18bvwGTNJ/ioFqt/WfLiMNulglvxptlwmE3y4TDbpaJrp5n30QjYlNGdnORZln5K8/zYqyseifilsKeekidRXFV03kR8ZWy8TdlJPvogFYWaWYl7ii5rqrpzfjUE+vbFHf03I3iyR+7NTs/M+usVvbZ9wYeiIiHIuJFimd0HVpnGjPrkVbCPpFXdkJ4nCqP3pE0Q9I8SfNWsbKFxZlZKzp+ND4iZkXE1IiYOpwRnV6cmdXQStiX8Mpnb22fhplZH2ol7HcCu0h6derp9CGKjhJm1oeaPvUWEaslHQf8nOLU2wWp95OZ9aGWzrNHxHXAdW2qxcw6yJfLmmXCYTfLhMNulgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XCYTfLhMNulgmH3SwTDrtZJvzI5vXA4lPeXNq+ZtPaj/DaZvenSqedu+eVTdU0aKdffKS0fYtfb1azbfw3ftXSsm3deM1ulgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XC59n7wPKf7lLavnDKtzq27FW1T9E35HfvOK+0/ZKpE2q2zb7x7aXTrll0f1M1WXVes5tlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmfB59i6odx799imXd2zZ5zyzY2n71+YeVNo+eVJ5f/gbdvtRafuHtxio2Xbq0eNKp93xsz7P3k4thV3SYmAFsAZYHRFT21GUmbVfO9bs74iIp9swHzPrIO+zm2Wi1bAHcIOkuyTNqDaCpBmS5kmat4qVLS7OzJrV6mb8fhGxRNLfATdK+l1E3FI5QkTMAmYBbKmxLXa7MLNmtbRmj4gl6d+lwFXA3u0oyszar+mwSxopaYvB18A7gYXtKszM2quVzfjxwFWSBudzaURc35aq1jOrD3hjafsv9vx2nTkML239+vJdS9t/+U8lZzz/uLR02l2XzyttH7bppqXtp93x+tL2k8YtqNm2eszq0mmtvZoOe0Q8BOzZxlrMrIN86s0sEw67WSYcdrNMOOxmmXDYzTLhLq5t8NzETUrbh9X5Tq13au3m95af3lrz0O9L21vxwJf2Km2/dOwZdeYwombL9td7XdNN/rTNMuGwm2XCYTfLhMNulgmH3SwTDrtZJhx2s0z4PHsbbHXx3NL2D8z7l9J2LX+2tH31wOJ1rKh9jpl2U2n7qGG1z6Nbf/Ga3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XCYTfLhM+zd8Ga+/7Q6xJqWnzqm0vbP7bV6XXmUH6r6RMG3lSzbYubFpVOu6bOkm3deM1ulgmH3SwTDrtZJhx2s0w47GaZcNjNMuGwm2XC59k3cM8cWX4e/fajys+jjx5Wfh597sqNStvnn1L7vvObPfvr0mmtvequ2SVdIGmppIUVw8ZKulHS/enfMZ0t08xa1chm/IXAwUOGnQjMiYhdgDnpvZn1sbphj4hbgGVDBh8KXJReXwQc1ua6zKzNmt1nHx8RA+n1E8D4WiNKmgHMANiUzZtcnJm1quWj8RERQJS0z4qIqRExdXjJQ/7MrLOaDfuTkiYApH+Xtq8kM+uEZsN+DTA9vZ4OXN2ecsysU+rus0u6DNgfGCfpceCLwFeA2ZI+BjwCHN7JIq15T7+h5h4WUP88ej3Tbz6mtH3XH/tcer+oG/aIOKJG0wFtrsXMOsiXy5plwmE3y4TDbpYJh90sEw67WSbcxXUD8OKNk2q2zX3tGXWmLj/1tufc6aXtrzvhwdJ23w66f3jNbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwufZ1wMb7zi5tP2/dv5hzbYxdbqw3rWyfNmT/qv8TPma5cvLZ2B9w2t2s0w47GaZcNjNMuGwm2XCYTfLhMNulgmH3SwTPs++Hthp9pLS9r02af47+4g5Hy9t3/W3dzY9b+svXrObZcJhN8uEw26WCYfdLBMOu1kmHHazTDjsZpnwefY+sHz6m0vbvzS+3r3fR9Rsmb74wNIpX/eZB0rbfd/3DUfdNbukCyQtlbSwYthMSUskzU8/0zpbppm1qpHN+AuBg6sMPzMipqSf69pblpm1W92wR8QtwLIu1GJmHdTKAbrjJN2TNvPH1BpJ0gxJ8yTNW0WdG56ZWcc0G/azgZ2AKcAAUPMIUkTMioipETF1eMmBJDPrrKbCHhFPRsSaiHgJOBfYu71lmVm7NRV2SRMq3r4PWFhrXDPrD3XPs0u6DNgfGCfpceCLwP6SpgABLAaO7WCN672NJ25X2v7WT91R2j5qWPO7P3Pv27m0fdfl7q+ei7phj4gjqgw+vwO1mFkH+XJZs0w47GaZcNjNMuGwm2XCYTfLhLu4dsGik3Yobf/xtte2NP93LPhgzTZ3YbVBXrObZcJhN8uEw26WCYfdLBMOu1kmHHazTDjsZpnwefYuuOu9Z9YZo7U7+Iz+t5dqtq1evryleduGw2t2s0w47GaZcNjNMuGwm2XCYTfLhMNulgmH3SwTPs++AVg1fnTNtuEvTuxiJWtb89TTNdtiZfnjwDSi/PqDjbYZ11RNAGu22aq0/f4TNml63o2INarZ9tpP1rkHwbPPNrVMr9nNMuGwm2XCYTfLhMNulgmH3SwTDrtZJhx2s0w08sjmHYCLgfEUj2ieFRFnSRoL/ACYTPHY5sMjwp2ne+CnV1zQ6xJqestvqj0EuPD0k1uWTjtmmxWl7Xe88dKmaup3u33+uNL2HT8zt6n5NrJmXw2cEBG7AW8CPiFpN+BEYE5E7ALMSe/NrE/VDXtEDETE3en1CmARMBE4FLgojXYRcFinijSz1q3TPrukycBewB3A+IgYSE1PUGzmm1mfajjskkYBVwLHR8QrLs6NiKDYn6823QxJ8yTNW0X5tdBm1jkNhV3ScIqgXxIRP0qDn5Q0IbVPAJZWmzYiZkXE1IiYOrzFGyuaWfPqhl2SgPOBRRHxtYqma4Dp6fV04Or2l2dm7aJiC7xkBGk/4FZgATB4z+KTKPbbZwOvAh6hOPW2rGxeW2ps7KMDWq15vfOXn7+6tH3OHld0qZK8vBAv1mxbFbVvv92IafccXdr+5/nNd7+dcNvq0vYRP7uzZtsdMYdnY1nV/rN1z7NHxG1Arc63+SXXbD3lK+jMMuGwm2XCYTfLhMNulgmH3SwTDrtZJnwr6S7Y7F0Pl7bvflp5l8bo4P/SFq8tvTSio91Id7/1I6Xt8ejIlua/4xXP1W789YKW5j2G+1tq7wWv2c0y4bCbZcJhN8uEw26WCYfdLBMOu1kmHHazTNTtz95OufZnN+uWsv7sXrObZcJhN8uEw26WCYfdLBMOu1kmHHazTDjsZplw2M0y4bCbZcJhN8uEw26WCYfdLBMOu1kmHHazTDjsZpmoG3ZJO0j6paT7JN0r6dNp+ExJSyTNTz/TOl+umTWrkccPrAZOiIi7JW0B3CXpxtR2ZkSc3rnyzKxd6oY9IgaAgfR6haRFwMROF2Zm7bVO++ySJgN7AXekQcdJukfSBZLG1JhmhqR5kuatYmVLxZpZ8xoOu6RRwJXA8RHxLHA2sBMwhWLNf0a16SJiVkRMjYipwxnRhpLNrBkNhV3ScIqgXxIRPwKIiCcjYk1EvAScC+zduTLNrFWNHI0XcD6wKCK+VjF8QsVo7wMWtr88M2uXRo7G7wscCSyQND8NOwk4QtIUIIDFwLEdqdDM2qKRo/G3AdXuQ31d+8sxs07xFXRmmXDYzTLhsJtlwmE3y4TDbpYJh90sEw67WSYcdrNMOOxmmXDYzTLhsJtlwmE3y4TDbpYJh90sE4qI7i1Megp4pGLQOODprhWwbvq1tn6tC1xbs9pZ26SI2KZaQ1fDvtbCpXkRMbVnBZTo19r6tS5wbc3qVm3ejDfLhMNuloleh31Wj5dfpl9r69e6wLU1qyu19XSf3cy6p9drdjPrEofdLBM9CbukgyX9XtIDkk7sRQ21SFosaUF6DPW8HtdygaSlkhZWDBsr6UZJ96d/qz5jr0e19cVjvEseM97Tz67Xjz/v+j67pI2APwAHAY8DdwJHRMR9XS2kBkmLgakR0fMLMCS9DXgOuDgi9kjDvgosi4ivpC/KMRHx2T6pbSbwXK8f452eVjSh8jHjwGHA0fTwsyup63C68Ln1Ys2+N/BARDwUES8ClwOH9qCOvhcRtwDLhgw+FLgovb6I4o+l62rU1hciYiAi7k6vVwCDjxnv6WdXUldX9CLsE4HHKt4/Tn897z2AGyTdJWlGr4upYnxEDKTXTwDje1lMFXUf491NQx4z3jefXTOPP2+VD9Ctbb+IeANwCPCJtLnal6LYB+unc6cNPca7W6o8Zvxlvfzsmn38eat6EfYlwA4V77dPw/pCRCxJ/y4FrqL/HkX95OATdNO/S3tcz8v66THe1R4zTh98dr18/Hkvwn4nsIukV0vaBPgQcE0P6liLpJHpwAmSRgLvpP8eRX0NMD29ng5c3cNaXqFfHuNd6zHj9Piz6/njzyOi6z/ANIoj8g8CJ/eihhp17Qj8Nv3c2+vagMsoNutWURzb+BiwNTAHuB+4CRjbR7V9D1gA3EMRrAk9qm0/ik30e4D56Wdarz+7krq68rn5clmzTPgAnVkmHHazTDjsZplw2M0y4bCbZcJhN8uEw26Wif8HteKJB66NhMUAAAAASUVORK5CYII=\n", + "text/plain": [ + "\u003cFigure size 432x288 with 1 Axes\u003e" + ] + }, + "metadata": { + "needs_background": "light", + "tags": [] + }, + "output_type": "display_data" + } + ], "source": [ - "plt.imshow(test_images[0])\n", - "template = \"True:{true}, predicted:{predict}\"\n", - "_ = plt.title(template.format(true= str(test_labels[0]),\n", - " predict=str(np.argmax(predictions[0]))))\n", - "plt.grid(False)" + "test_model(tflite_model_quant_file, test_image_index, model_type=\"Quantized\")" ] }, { @@ -655,7 +829,17 @@ "id": "LwN7uIdCd8Gw" }, "source": [ - "### Evaluate the models" + "### Evaluate the models on all images" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "RFKOD4DG8XmU" + }, + "source": [ + "Now let's run both models using all the test images we loaded at the beginning of this tutorial:" ] }, { @@ -668,49 +852,52 @@ }, "outputs": [], "source": [ - "# A helper function to evaluate the TF Lite model using \"test\" dataset.\n", - "def evaluate_model(interpreter):\n", - " input_index = interpreter.get_input_details()[0][\"index\"]\n", - " output_index = interpreter.get_output_details()[0][\"index\"]\n", + "# Helper function to evaluate a TFLite model on all images\n", + "def evaluate_model(tflite_file, model_type):\n", + " global test_images\n", + " global test_labels\n", "\n", - " # Run predictions on every image in the \"test\" dataset.\n", - " prediction_digits = []\n", - " for test_image in test_images:\n", - " # Pre-processing: add batch dimension and convert to float32 to match with\n", - " # the model's input data format.\n", - " test_image = np.expand_dims(test_image, axis=0).astype(np.float32)\n", - " interpreter.set_tensor(input_index, test_image)\n", + " test_image_indices = range(test_images.shape[0])\n", + " predictions = run_tflite_model(tflite_file, test_image_indices)\n", "\n", - " # Run inference.\n", - " interpreter.invoke()\n", + " accuracy = (np.sum(test_labels== predictions) * 100) / len(test_images)\n", "\n", - " # Post-processing: remove batch dimension and find the digit with highest\n", - " # probability.\n", - " output = interpreter.tensor(output_index)\n", - " digit = np.argmax(output()[0])\n", - " prediction_digits.append(digit)\n", - "\n", - " # Compare prediction results with ground truth labels to calculate accuracy.\n", - " accurate_count = 0\n", - " for index in range(len(prediction_digits)):\n", - " if prediction_digits[index] == test_labels[index]:\n", - " accurate_count += 1\n", - " accuracy = accurate_count * 1.0 / len(prediction_digits)\n", - "\n", - " return accuracy" + " print('%s model accuracy is %.4f%% (Number of test samples=%d)' % (\n", + " model_type, accuracy, len(test_images)))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "colab_type": "text", + "id": "xnFilQpBuMh5" + }, + "source": [ + "Evaluate the float model:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "colab": {}, + "colab": { + "height": 34 + }, "colab_type": "code", - "id": "T5mWkSbMcU5z" + "id": "T5mWkSbMcU5z", + "outputId": "7e05d400-1455-4c1a-f3f0-b81422c3a0ba" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Float model accuracy is 97.8700% (Number of test samples=10000)\n" + ] + } + ], "source": [ - "print(evaluate_model(interpreter))" + "evaluate_model(tflite_model_file, model_type=\"Float\")" ] }, { @@ -720,25 +907,31 @@ "id": "Km3cY9ry8ZlG" }, "source": [ - "Repeat the evaluation on the fully quantized model using the uint8 data:" + "Evaluate the quantized model:" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "colab": {}, + "colab": { + "height": 34 + }, "colab_type": "code", - "id": "-9cnwiPp6EGm" + "id": "-9cnwiPp6EGm", + "outputId": "1e7409cf-748d-45c9-aa2f-36ccd9454f45" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Quantized model accuracy is 97.8100% (Number of test samples=10000)\n" + ] + } + ], "source": [ - "# NOTE: Colab runs on server CPUs, and TensorFlow Lite currently\n", - "# doesn't have super optimized server CPU kernels. So this part may be\n", - "# slower than the above float interpreter. But for mobile CPUs, considerable\n", - "# speedup can be observed.\n", - "\n", - "print(evaluate_model(interpreter_quant))" + "evaluate_model(tflite_model_quant_file, model_type=\"Quantized\")" ] }, { @@ -748,7 +941,9 @@ "id": "L7lfxkor8pgv" }, "source": [ - "In this example, you have fully quantized a model with almost no difference in the accuracy, compared to the above float model." + "So you now have an integer quantized a model with almost no difference in the accuracy, compared to the float model.\n", + "\n", + "To learn more about other quantization strategies, read about [TensorFlow Lite model optimization](https://www.tensorflow.org/lite/performance/model_optimization)." ] } ], From dcbf62e6b2588c83c3b0fdfcb1fe101eb2afae47 Mon Sep 17 00:00:00 2001 From: Terry Huang Date: Tue, 4 Aug 2020 14:14:34 -0700 Subject: [PATCH 0397/1017] reduce over-broad dependencies in regex_split library PiperOrigin-RevId: 324887712 Change-Id: I1368673dd1f4b04331de98f656f8d642a1d210c0 --- tensorflow/core/platform/BUILD | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow/core/platform/BUILD b/tensorflow/core/platform/BUILD index 4fe09498e93..a889666c608 100644 --- a/tensorflow/core/platform/BUILD +++ b/tensorflow/core/platform/BUILD @@ -780,7 +780,10 @@ cc_library( name = "types", hdrs = ["types.h"], # TODO(b/161569340): Short-term fix. Remove this visibility rule. - visibility = ["//tensorflow:__subpackages__"], + visibility = [ + "//tensorflow:__subpackages__", + "//tensorflow_text:__subpackages__", + ], deps = [ ":platform", ":tstring", From 5c6f844c0208a605a9909953cca00ece1373e990 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 14:41:18 -0700 Subject: [PATCH 0398/1017] In preparation for making the proto ParseFromString() method accept string_view, this change makes the conversion from py::bytes to std::string explicit, since this will no longer happen implicitly after the parameter type change. PiperOrigin-RevId: 324893229 Change-Id: I5ba90b32d5e68b45281c0000b747df3d9b6bf532 --- .../python/profiler/internal/profiler_wrapper.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/profiler/internal/profiler_wrapper.cc b/tensorflow/python/profiler/internal/profiler_wrapper.cc index 0f57204d1d0..0984a8b45c5 100644 --- a/tensorflow/python/profiler/internal/profiler_wrapper.cc +++ b/tensorflow/python/profiler/internal/profiler_wrapper.cc @@ -177,7 +177,7 @@ PYBIND11_MODULE(_pywrap_profiler, m) { m.def("xspace_to_trace_events", [](const py::bytes& serialized_xspace_proto) { tensorflow::string content; tensorflow::profiler::XSpace xspace; - xspace.ParseFromString(serialized_xspace_proto); + xspace.ParseFromString(std::string(serialized_xspace_proto)); tensorflow::profiler::ConvertXSpaceToTraceEventsString(xspace, &content); return py::bytes(content); }); @@ -185,7 +185,7 @@ PYBIND11_MODULE(_pywrap_profiler, m) { m.def("xspace_to_overview_page", [](const py::bytes& serialized_xspace_proto) { tensorflow::profiler::XSpace xspace; - xspace.ParseFromString(serialized_xspace_proto); + xspace.ParseFromString(std::string(serialized_xspace_proto)); tensorflow::profiler::OverviewPage overview_page = tensorflow::profiler::ConvertOpStatsToOverviewPage( ConvertXSpaceToOpStats( @@ -196,7 +196,7 @@ PYBIND11_MODULE(_pywrap_profiler, m) { m.def("xspace_to_input_pipeline", [](const py::bytes& serialized_xspace_proto) { tensorflow::profiler::XSpace xspace; - xspace.ParseFromString(serialized_xspace_proto); + xspace.ParseFromString(std::string(serialized_xspace_proto)); tensorflow::profiler::InputPipelineAnalysisResult input_pipeline = tensorflow::profiler::ConvertOpStatsToInputPipelineAnalysis( ConvertXSpaceToOpStats(xspace, {OP_METRICS_DB, STEP_DB})); @@ -205,7 +205,7 @@ PYBIND11_MODULE(_pywrap_profiler, m) { m.def("xspace_to_tf_stats", [](const py::bytes& serialized_xspace_proto) { tensorflow::profiler::XSpace xspace; - xspace.ParseFromString(serialized_xspace_proto); + xspace.ParseFromString(std::string(serialized_xspace_proto)); tensorflow::profiler::TfStatsDatabase tf_stats_db = tensorflow::profiler::ConvertOpStatsToTfStats( ConvertXSpaceToOpStats(xspace, {OP_METRICS_DB})); @@ -214,7 +214,7 @@ PYBIND11_MODULE(_pywrap_profiler, m) { m.def("xspace_to_kernel_stats", [](const py::bytes& serialized_xspace_proto) { tensorflow::profiler::XSpace xspace; - xspace.ParseFromString(serialized_xspace_proto); + xspace.ParseFromString(std::string(serialized_xspace_proto)); tensorflow::profiler::OpStats op_stats = ConvertXSpaceToOpStats(xspace, {KERNEL_STATS_DB}); return py::bytes(op_stats.kernel_stats_db().SerializeAsString()); @@ -223,7 +223,7 @@ PYBIND11_MODULE(_pywrap_profiler, m) { m.def("xspace_to_memory_profile", [](const py::bytes& serialized_xspace_proto) { tensorflow::profiler::XSpace xspace; - xspace.ParseFromString(serialized_xspace_proto); + xspace.ParseFromString(std::string(serialized_xspace_proto)); std::string json_output; tensorflow::profiler::ConvertXSpaceToMemoryProfileJson(xspace, &json_output); From 12208cd82db68e19b81680a2b98a6f593256bfff Mon Sep 17 00:00:00 2001 From: Brian Zhao Date: Tue, 4 Aug 2020 14:42:20 -0700 Subject: [PATCH 0399/1017] Cleaning up unused TF_TensorHandleList, now that captures are not exposed on the C API. PiperOrigin-RevId: 324893440 Change-Id: Ie89e64894e19af2e72a187bc07065595d0704925 --- .../c/experimental/saved_model/internal/BUILD | 35 --------------- .../saved_model/internal/concrete_function.cc | 1 - .../internal/saved_model_api_test.cc | 1 - .../saved_model/internal/tensorhandle_list.cc | 36 ---------------- .../internal/tensorhandle_list_type.h | 37 ---------------- .../c/experimental/saved_model/public/BUILD | 7 --- .../saved_model/public/c_saved_model_api.h | 1 - .../saved_model/public/concrete_function.h | 1 - .../saved_model/public/tensorhandle_list.h | 43 ------------------- 9 files changed, 162 deletions(-) delete mode 100644 tensorflow/c/experimental/saved_model/internal/tensorhandle_list.cc delete mode 100644 tensorflow/c/experimental/saved_model/internal/tensorhandle_list_type.h delete mode 100644 tensorflow/c/experimental/saved_model/public/tensorhandle_list.h diff --git a/tensorflow/c/experimental/saved_model/internal/BUILD b/tensorflow/c/experimental/saved_model/internal/BUILD index 60ca0134602..323298c5fc1 100644 --- a/tensorflow/c/experimental/saved_model/internal/BUILD +++ b/tensorflow/c/experimental/saved_model/internal/BUILD @@ -38,8 +38,6 @@ cc_library( ":concrete_function_type", ":function_metadata", ":function_metadata_type", - ":tensorhandle_list", - ":tensorhandle_list_type", "//tensorflow/c:c_api_macros", "//tensorflow/c:tf_status_internal", "//tensorflow/c/eager:abstract_tensor_handle", @@ -167,38 +165,6 @@ cc_library( ], ) -cc_library( - name = "tensorhandle_list", - srcs = [ - "tensorhandle_list.cc", - ], - hdrs = [ - "//tensorflow/c/experimental/saved_model/public:tensorhandle_list.h", - ], - copts = tf_copts(), - visibility = [ - "//tensorflow/c/experimental/saved_model/public:__pkg__", - ], - deps = [ - ":tensorhandle_list_type", - "//tensorflow/c:c_api_macros", - "//tensorflow/c/eager:c_api", - "//tensorflow/c/eager:immediate_execution_tensor_handle", - "//tensorflow/c/eager:tfe_tensorhandle_internal", - ], -) - -cc_library( - name = "tensorhandle_list_type", - hdrs = [ - "tensorhandle_list_type.h", - ], - deps = [ - "//tensorflow/c:conversion_macros", - "//tensorflow/c/eager:immediate_execution_tensor_handle", - ], -) - tf_cc_test( name = "saved_model_api_test", size = "small", @@ -216,7 +182,6 @@ tf_cc_test( "//tensorflow/c/eager:c_api_test_util", "//tensorflow/c/experimental/saved_model/public:concrete_function", "//tensorflow/c/experimental/saved_model/public:saved_model_api", - "//tensorflow/c/experimental/saved_model/public:tensorhandle_list", "//tensorflow/core:lib", "//tensorflow/core:test", "//tensorflow/core:test_main", diff --git a/tensorflow/c/experimental/saved_model/internal/concrete_function.cc b/tensorflow/c/experimental/saved_model/internal/concrete_function.cc index 9f421a7b9b7..65c6eca5623 100644 --- a/tensorflow/c/experimental/saved_model/internal/concrete_function.cc +++ b/tensorflow/c/experimental/saved_model/internal/concrete_function.cc @@ -24,7 +24,6 @@ limitations under the License. #include "tensorflow/c/experimental/saved_model/core/function_metadata.h" #include "tensorflow/c/experimental/saved_model/internal/concrete_function_type.h" #include "tensorflow/c/experimental/saved_model/internal/function_metadata_type.h" -#include "tensorflow/c/experimental/saved_model/internal/tensorhandle_list_type.h" #include "tensorflow/c/tf_status_internal.h" #include "tensorflow/core/platform/status.h" diff --git a/tensorflow/c/experimental/saved_model/internal/saved_model_api_test.cc b/tensorflow/c/experimental/saved_model/internal/saved_model_api_test.cc index 10b5677a48b..e58b232f9c9 100644 --- a/tensorflow/c/experimental/saved_model/internal/saved_model_api_test.cc +++ b/tensorflow/c/experimental/saved_model/internal/saved_model_api_test.cc @@ -22,7 +22,6 @@ limitations under the License. #include "tensorflow/c/eager/c_api_experimental.h" #include "tensorflow/c/eager/c_api_test_util.h" #include "tensorflow/c/experimental/saved_model/public/concrete_function.h" -#include "tensorflow/c/experimental/saved_model/public/tensorhandle_list.h" #include "tensorflow/c/tf_status.h" #include "tensorflow/c/tf_tensor.h" #include "tensorflow/core/lib/io/path.h" diff --git a/tensorflow/c/experimental/saved_model/internal/tensorhandle_list.cc b/tensorflow/c/experimental/saved_model/internal/tensorhandle_list.cc deleted file mode 100644 index c8f00c1f7c0..00000000000 --- a/tensorflow/c/experimental/saved_model/internal/tensorhandle_list.cc +++ /dev/null @@ -1,36 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/c/experimental/saved_model/public/tensorhandle_list.h" - -#include - -#include "tensorflow/c/eager/immediate_execution_tensor_handle.h" -#include "tensorflow/c/eager/tfe_tensorhandle_internal.h" -#include "tensorflow/c/experimental/saved_model/internal/tensorhandle_list_type.h" - -extern "C" { - -size_t TF_TensorHandleListSize(const TF_TensorHandleList* list) { - return tensorflow::unwrap(list)->size(); -} - -TFE_TensorHandle* TF_TensorHandleListGet(const TF_TensorHandleList* list, - int i) { - return tensorflow::wrap((*tensorflow::unwrap(list))[i]); -} - - -} // end extern "C" diff --git a/tensorflow/c/experimental/saved_model/internal/tensorhandle_list_type.h b/tensorflow/c/experimental/saved_model/internal/tensorhandle_list_type.h deleted file mode 100644 index 566417df025..00000000000 --- a/tensorflow/c/experimental/saved_model/internal/tensorhandle_list_type.h +++ /dev/null @@ -1,37 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_C_EXPERIMENTAL_SAVED_MODEL_INTERNAL_CONCRETE_FUNCTION_LIST_TYPE_H_ -#define TENSORFLOW_C_EXPERIMENTAL_SAVED_MODEL_INTERNAL_CONCRETE_FUNCTION_LIST_TYPE_H_ - -#include - -#include "tensorflow/c/conversion_macros.h" -#include "tensorflow/c/eager/immediate_execution_tensor_handle.h" - -// Internal structures used by the SavedModel C API. These are likely to -// change and should not be depended on. - -typedef struct TF_TensorHandleList TF_TensorHandleList; - -namespace tensorflow { - -DEFINE_CONVERSION_FUNCTIONS( - std::vector, - TF_TensorHandleList) - -} // namespace tensorflow - -#endif // TENSORFLOW_C_EXPERIMENTAL_SAVED_MODEL_INTERNAL_CONCRETE_FUNCTION_LIST_TYPE_H_ diff --git a/tensorflow/c/experimental/saved_model/public/BUILD b/tensorflow/c/experimental/saved_model/public/BUILD index 0cfa0a2c005..af65e05e7f6 100644 --- a/tensorflow/c/experimental/saved_model/public/BUILD +++ b/tensorflow/c/experimental/saved_model/public/BUILD @@ -24,7 +24,6 @@ exports_files( "concrete_function_list.h", "function_metadata.h", "saved_model_api.h", - "tensorhandle_list.h", ], visibility = ["//tensorflow/c/experimental/saved_model/internal:__pkg__"], ) @@ -40,7 +39,6 @@ cc_library( ":concrete_function_list", ":function_metadata", ":saved_model_api", - ":tensorhandle_list", ], ) @@ -63,8 +61,3 @@ alias( name = "saved_model_api", actual = "//tensorflow/c/experimental/saved_model/internal:saved_model_api", ) - -alias( - name = "tensorhandle_list", - actual = "//tensorflow/c/experimental/saved_model/internal:tensorhandle_list", -) diff --git a/tensorflow/c/experimental/saved_model/public/c_saved_model_api.h b/tensorflow/c/experimental/saved_model/public/c_saved_model_api.h index aae95a5477c..30f533f140a 100644 --- a/tensorflow/c/experimental/saved_model/public/c_saved_model_api.h +++ b/tensorflow/c/experimental/saved_model/public/c_saved_model_api.h @@ -21,7 +21,6 @@ limitations under the License. #include "tensorflow/c/experimental/saved_model/public/concrete_function_list.h" #include "tensorflow/c/experimental/saved_model/public/function_metadata.h" #include "tensorflow/c/experimental/saved_model/public/saved_model_api.h" -#include "tensorflow/c/experimental/saved_model/public/tensorhandle_list.h" // IWYU pragma: end_exports #endif // TENSORFLOW_C_EXPERIMENTAL_SAVED_MODEL_PUBLIC_C_SAVED_MODEL_API_H_ diff --git a/tensorflow/c/experimental/saved_model/public/concrete_function.h b/tensorflow/c/experimental/saved_model/public/concrete_function.h index 4cc2a4b4f05..ee5292294d6 100644 --- a/tensorflow/c/experimental/saved_model/public/concrete_function.h +++ b/tensorflow/c/experimental/saved_model/public/concrete_function.h @@ -19,7 +19,6 @@ limitations under the License. #include "tensorflow/c/c_api_macros.h" #include "tensorflow/c/eager/c_api.h" #include "tensorflow/c/experimental/saved_model/public/function_metadata.h" -#include "tensorflow/c/experimental/saved_model/public/tensorhandle_list.h" #ifdef __cplusplus extern "C" { diff --git a/tensorflow/c/experimental/saved_model/public/tensorhandle_list.h b/tensorflow/c/experimental/saved_model/public/tensorhandle_list.h deleted file mode 100644 index a1e88db3474..00000000000 --- a/tensorflow/c/experimental/saved_model/public/tensorhandle_list.h +++ /dev/null @@ -1,43 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_C_EXPERIMENTAL_SAVED_MODEL_PUBLIC_TENSORHANDLE_LIST_H_ -#define TENSORFLOW_C_EXPERIMENTAL_SAVED_MODEL_PUBLIC_TENSORHANDLE_LIST_H_ - -#include - -#include "tensorflow/c/c_api_macros.h" -#include "tensorflow/c/eager/c_api.h" - -#ifdef __cplusplus -extern "C" { -#endif // __cplusplus - -// An opaque type that is acts like a list of TF_ConcreteFunction pointers. -typedef struct TF_TensorHandleList TF_TensorHandleList; - -// Returns the size of `list`. -TF_CAPI_EXPORT extern size_t TF_TensorHandleListSize( - const TF_TensorHandleList* list); - -// Returns the `i`th TFE_TensorHandle in the list. -TF_CAPI_EXPORT extern TFE_TensorHandle* TF_TensorHandleListGet( - const TF_TensorHandleList* list, int i); - -#ifdef __cplusplus -} // end extern "C" -#endif // __cplusplus - -#endif // TENSORFLOW_C_EXPERIMENTAL_SAVED_MODEL_PUBLIC_TENSORHANDLE_LIST_H_ From 541832fca1728779929f3c1416e585ffe57c8373 Mon Sep 17 00:00:00 2001 From: Jiho Choi Date: Tue, 4 Aug 2020 14:51:22 -0700 Subject: [PATCH 0400/1017] Support multiple producers of the same (iterator_id, element_id) pair. PiperOrigin-RevId: 324895347 Change-Id: I4c560807486d43b977624dc47217738a12ecda02 --- .../core/profiler/utils/group_events.cc | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/tensorflow/core/profiler/utils/group_events.cc b/tensorflow/core/profiler/utils/group_events.cc index 2be2da3b445..86566822252 100644 --- a/tensorflow/core/profiler/utils/group_events.cc +++ b/tensorflow/core/profiler/utils/group_events.cc @@ -649,8 +649,9 @@ void EventForest::ProcessModelIds() { void EventForest::ProcessTfDataEvents() { absl::flat_hash_map, - EventNode*> - produce_iterators; + std::vector> + produce_iterator_map; + uint64 num_producers = 0; for (HostEventType event_type : {HostEventType::kPrefetchProduce, HostEventType::kParallelInterleaveProduce, @@ -670,14 +671,16 @@ void EventForest::ProcessTfDataEvents() { absl::optional iterator_id = produce_iterator->GetEventVisitor().GetStat(StatType::kParentId); if (!iterator_id.has_value()) break; - produce_iterators[{iterator_id->IntValue(), element_id->IntValue()}] = - produce_iterator; + produce_iterator_map[{iterator_id->IntValue(), + element_id->IntValue()}] + .push_back(produce_iterator); + ++num_producers; break; } } } } - VLOG(1) << produce_iterators.size() << " producer iterators found."; + VLOG(1) << num_producers << " producer iterators found."; uint64 num_matched = 0; for (HostEventType event_type : {HostEventType::kPrefetchConsume, @@ -701,11 +704,13 @@ void EventForest::ProcessTfDataEvents() { absl::optional iterator_id = consume_iterator->GetEventVisitor().GetStat(StatType::kStepId); if (!iterator_id.has_value()) continue; - if (auto produce_iterator = gtl::FindOrNull( - produce_iterators, std::make_pair(iterator_id->IntValue(), - element_id->IntValue()))) { - consume_iterator->AddChild(*produce_iterator); - ++num_matched; + if (auto produce_iterators = gtl::FindOrNull( + produce_iterator_map, std::make_pair(iterator_id->IntValue(), + element_id->IntValue()))) { + for (EventNode* produce_iterator : *produce_iterators) { + consume_iterator->AddChild(produce_iterator); + ++num_matched; + } } } } From 2b220dae89d64be05276018e171fcfefa95c1bc5 Mon Sep 17 00:00:00 2001 From: Jose Baiocchi Date: Tue, 4 Aug 2020 15:04:20 -0700 Subject: [PATCH 0401/1017] Export CombineOpMetrics via header PiperOrigin-RevId: 324898471 Change-Id: I77fb4c4fa86d695a701b51110485324dc76fd20a --- .../convert/op_metrics_db_combiner.cc | 43 +++++++++++++------ .../profiler/convert/op_metrics_db_combiner.h | 3 ++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/tensorflow/core/profiler/convert/op_metrics_db_combiner.cc b/tensorflow/core/profiler/convert/op_metrics_db_combiner.cc index e91869885c5..ad1d4bf380a 100644 --- a/tensorflow/core/profiler/convert/op_metrics_db_combiner.cc +++ b/tensorflow/core/profiler/convert/op_metrics_db_combiner.cc @@ -25,21 +25,43 @@ namespace { using OperationType = OpMetrics::MemoryAccessed::OperationType; -// Combines the src OpMetrics into the dst OpMetrics. -void CombineOpMetrics(const OpMetrics& src, OpMetrics* dst) { +// Copies OpMetrics symbol data from src to dst. +void CopyOpMetricsSymbolData(const OpMetrics& src, OpMetrics* dst) { DCHECK(dst != nullptr); DCHECK_EQ(src.hlo_module_id(), dst->hlo_module_id()); DCHECK_EQ(src.name(), dst->name()); - dst->set_category(src.category()); - dst->set_provenance(src.provenance()); - dst->set_is_eager(dst->is_eager() || src.is_eager()); - dst->set_deduplicated_name(src.deduplicated_name()); + if (dst->category().empty()) { + dst->set_category(src.category()); + } + if (dst->provenance().empty()) { + dst->set_provenance(src.provenance()); + } + if (dst->deduplicated_name().empty()) { + dst->set_deduplicated_name(src.deduplicated_name()); + } if (!dst->has_layout() && src.has_layout()) { *dst->mutable_layout() = src.layout(); } if (!dst->has_children() && src.has_children()) { *dst->mutable_children() = src.children(); } +} + +void CombinePrecisionStats(const PrecisionStats& src, PrecisionStats* dst) { + dst->set_compute_16bit_ps(src.compute_16bit_ps() + dst->compute_16bit_ps()); + dst->set_compute_32bit_ps(src.compute_32bit_ps() + dst->compute_32bit_ps()); +} + +} // namespace + +void CombineOpMetrics(const OpMetrics& src, OpMetrics* dst) { + DCHECK(dst != nullptr); + if (dst->occurrences() == 0) { + dst->set_min_time_ps(src.min_time_ps()); + } else { + dst->set_min_time_ps(std::min(src.min_time_ps(), dst->min_time_ps())); + } + dst->set_is_eager(dst->is_eager() || src.is_eager()); dst->set_occurrences(src.occurrences() + dst->occurrences()); dst->set_time_ps(src.time_ps() + dst->time_ps()); dst->set_self_time_ps(src.self_time_ps() + dst->self_time_ps()); @@ -50,16 +72,10 @@ void CombineOpMetrics(const OpMetrics& src, OpMetrics* dst) { dst->set_dma_stall_ps(src.dma_stall_ps() + dst->dma_stall_ps()); } -void CombinePrecisionStats(const PrecisionStats& src, PrecisionStats* dst) { - dst->set_compute_16bit_ps(src.compute_16bit_ps() + dst->compute_16bit_ps()); - dst->set_compute_32bit_ps(src.compute_32bit_ps() + dst->compute_32bit_ps()); -} - -} // namespace - void CombineMemoryAccessedBreakdown( const protobuf::RepeatedPtrField& src, protobuf::RepeatedPtrField* dst) { + if (src.empty()) return; absl::flat_hash_map, OpMetrics_MemoryAccessed*> dst_memory_accessed_map; @@ -99,6 +115,7 @@ void OpMetricsDbCombiner::Combine(const OpMetricsDb& src) { for (const auto& src_metrics : src.metrics_db()) { auto* dst_metrics = LookupOrInsertNewOpMetrics(src_metrics.hlo_module_id(), src_metrics.name()); + CopyOpMetricsSymbolData(src_metrics, dst_metrics); CombineOpMetrics(src_metrics, dst_metrics); } } diff --git a/tensorflow/core/profiler/convert/op_metrics_db_combiner.h b/tensorflow/core/profiler/convert/op_metrics_db_combiner.h index a0ca3387e7a..a87a2b53500 100644 --- a/tensorflow/core/profiler/convert/op_metrics_db_combiner.h +++ b/tensorflow/core/profiler/convert/op_metrics_db_combiner.h @@ -23,6 +23,9 @@ limitations under the License. namespace tensorflow { namespace profiler { +// Combines the src OpMetrics into the dst OpMetrics. +void CombineOpMetrics(const OpMetrics& src, OpMetrics* dst); + // Combines the memory access breakdown. void CombineMemoryAccessedBreakdown( const protobuf::RepeatedPtrField& src, From 54b76dc588d28643fbfd0a297870d79326b11abb Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 4 Aug 2020 22:08:35 +0000 Subject: [PATCH 0402/1017] switched from allocator attributes struct to opaque pointer --- tensorflow/c/kernels.cc | 6 +----- tensorflow/c/kernels.h | 1 - tensorflow/c/kernels_test.cc | 36 +++++++++++++------------------ tensorflow/c/tf_tensor.cc | 17 +++++++++++++++ tensorflow/c/tf_tensor.h | 22 +++++++++---------- tensorflow/c/tf_tensor_internal.h | 5 +++++ 6 files changed, 49 insertions(+), 38 deletions(-) diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 6d9d7d57517..096c8e41812 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -289,15 +289,11 @@ TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, TF_SetStatus(status, TF_OK, ""); tensorflow::gtl::ArraySlice dimarray( reinterpret_cast(dims), num_dims); - tensorflow::AllocatorAttributes allocator_attr; - if (attributes->on_host) { - allocator_attr.set_on_host(true); - } tensorflow::Status s; tensorflow::Tensor tensor; TF_Tensor* tf_tensor; s = cc_ctx->allocate_temp(static_cast(dtype), - tensorflow::TensorShape(dimarray), &tensor, allocator_attr); + tensorflow::TensorShape(dimarray), &tensor, attributes->alloc_attrs); if (!s.ok()) { ::tensorflow::Set_TF_Status_from_Status(status, s); return nullptr; diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index d6a7070de06..ee865613b6e 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -209,7 +209,6 @@ TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, int64_t* dims, int num_dims, TF_AllocatorAttributes* alloc_attrs, TF_Status* status); - #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index 708c39690a8..5239274a2ce 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -467,16 +467,13 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSizeOne) { // Allocate scalar TF_Tensor TF_Status* s = TF_NewStatus(); int64_t dim = 1; - TF_AllocatorAttributes alloc_attrs; - alloc_attrs.struct_size = TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE; -#if GOOGLE_CUDA - alloc_attrs.on_host = 0; -#else - alloc_attrs.on_host = 1; + TF_AllocatorAttributes* alloc_attrs = TF_NewAllocatorAttributes(); +#if !GOOGLE_CUDA + TF_AllocatorAttributesSetOnHost(alloc_attrs); #endif TF_Tensor* output = TF_AllocateTemp( /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, /*allocator_attributes*/ &alloc_attrs, s); + /*num_dims=*/1, /*allocator_attributes*/ alloc_attrs, s); size_t tensor_size_bytes = TF_DataTypeSize(TF_FLOAT); EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, &dim, 1, TF_FLOAT); @@ -487,6 +484,7 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSizeOne) { TF_SetOutput(ctx, 0, output, s); TF_DeleteStatus(s); TF_DeleteTensor(output); + TF_DeleteAllocatorAttributes(alloc_attrs); }; SetupOp("AllocateTempOp1", "AllocateTemp1", my_compute_func); @@ -504,21 +502,19 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempEmpty) { TF_Status* s = TF_NewStatus(); // Allocate empty TF_Tensor int64_t dim = 0; - TF_AllocatorAttributes alloc_attrs; - alloc_attrs.struct_size = TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE; -#if GOOGLE_CUDA - alloc_attrs.on_host = 0; -#else - alloc_attrs.on_host = 1; + TF_AllocatorAttributes* alloc_attrs = TF_NewAllocatorAttributes(); +#if !GOOGLE_CUDA + TF_AllocatorAttributesSetOnHost(alloc_attrs); #endif TF_Tensor* output = TF_AllocateTemp( /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/&dim, - /*num_dims=*/1, /*allocator_attributes*/ &alloc_attrs, s); + /*num_dims=*/1, /*allocator_attributes*/ alloc_attrs, s); EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, &dim, 1, TF_FLOAT); TF_SetOutput(ctx, 0, output, s); TF_DeleteStatus(s); TF_DeleteTensor(output); + TF_DeleteAllocatorAttributes(alloc_attrs); }; SetupOp("AllocateTempOp0", "AllocateTemp0", my_compute_func); @@ -537,16 +533,13 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSize2x3) { size_t tensor_size_bytes = 6 * TF_DataTypeSize(TF_FLOAT); // Allocate 2x3 TF_Tensor int64_t dim[2] = {2, 3}; - TF_AllocatorAttributes alloc_attrs; - alloc_attrs.struct_size = TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE; -#if GOOGLE_CUDA - alloc_attrs.on_host = 0; -#else - alloc_attrs.on_host = 1; + TF_AllocatorAttributes* alloc_attrs = TF_NewAllocatorAttributes(); +#if !GOOGLE_CUDA + TF_AllocatorAttributesSetOnHost(alloc_attrs); #endif TF_Tensor* output = TF_AllocateTemp( /*context=*/ctx, /*dtype=*/TF_FLOAT, /*dims=*/dim, - /*num_dims=*/2, /*allocator_attributes*/ &alloc_attrs, s); + /*num_dims=*/2, /*allocator_attributes*/ alloc_attrs, s); EXPECT_EQ(TF_OK, TF_GetCode(s)); validate_tensor(output, dim, 2, TF_FLOAT); @@ -556,6 +549,7 @@ TEST_F(DeviceKernelOpTest, TestAllocateTempSize2x3) { TF_SetOutput(ctx, 0, output, s); TF_DeleteStatus(s); TF_DeleteTensor(output); + TF_DeleteAllocatorAttributes(alloc_attrs); }; SetupOp("AllocateTempOp2x3", "AllocateTempOp2x3", my_compute_func); diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc index 0feb986ce44..12405939bfc 100644 --- a/tensorflow/c/tf_tensor.cc +++ b/tensorflow/c/tf_tensor.cc @@ -321,3 +321,20 @@ bool TensorInterface::IsAligned() const { return tensor_.IsAligned(); } } // namespace tensorflow bool TF_TensorIsAligned(const TF_Tensor* t) { return t->tensor->IsAligned(); } + +TF_AllocatorAttributes* TF_NewAllocatorAttributes() { + return new TF_AllocatorAttributes{tensorflow::AllocatorAttributes()}; +} + +void TF_AllocatorAttributesSetOnHost(TF_AllocatorAttributes* tf_alloc_attrs) { + tf_alloc_attrs->alloc_attrs.set_on_host(true); +} + +void TF_DeleteAllocatorAttributes(TF_AllocatorAttributes* tf_alloc_attrs) { + if (tf_alloc_attrs == nullptr) { + return; + } + else { + delete tf_alloc_attrs; + } +} diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index ced57df77d4..9a043ce0538 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -46,17 +46,6 @@ limitations under the License. extern "C" { #endif -// Allocator Attributes used for tensor allocation. -typedef struct TF_AllocatorAttributes { - size_t struct_size; - // Set boolean to 1 for CPU allocation, else 0. - unsigned char on_host; -} TF_AllocatorAttributes; - - -#define TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE \ - TF_OFFSET_OF_END(TF_AllocatorAttributes, on_host) - // -------------------------------------------------------------------------- // TF_Tensor holds a multi-dimensional array of elements of a single data type. // For all types other than TF_STRING, the data buffer stores elements @@ -163,6 +152,17 @@ TF_CAPI_EXPORT extern void TF_TensorBitcastFrom(const TF_Tensor* from, // Returns bool iff this tensor is aligned. TF_CAPI_EXPORT extern bool TF_TensorIsAligned(const TF_Tensor*); +// Allocator Attributes used for tensor allocation. +typedef struct TF_AllocatorAttributes TF_AllocatorAttributes; + +TF_CAPI_EXPORT extern TF_AllocatorAttributes* TF_NewAllocatorAttributes(); + +TF_CAPI_EXPORT extern void TF_AllocatorAttributesSetOnHost( + TF_AllocatorAttributes* tf_alloc_attrs); + +TF_CAPI_EXPORT extern void TF_DeleteAllocatorAttributes( + TF_AllocatorAttributes* tf_alloc_attrs); + #ifdef __cplusplus } /* end extern "C" */ #endif diff --git a/tensorflow/c/tf_tensor_internal.h b/tensorflow/c/tf_tensor_internal.h index 7a896dc5d11..71cac0afeb1 100644 --- a/tensorflow/c/tf_tensor_internal.h +++ b/tensorflow/c/tf_tensor_internal.h @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow/core/framework/allocation_description.pb.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/allocator.h" #include "tensorflow/core/platform/casts.h" // Internal structures used by the C API. These are likely to change and should @@ -124,4 +125,8 @@ Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst); TF_Tensor* TF_TensorFromTensor(const Tensor& src, Status* status); } // namespace tensorflow +typedef struct TF_AllocatorAttributes { + tensorflow::AllocatorAttributes alloc_attrs; +} TF_AllocatorAttributes; + #endif // TENSORFLOW_C_TF_TENSOR_INTERNAL_H_ From 087d17541a0e7ff0f8bdd36145dbcac93be91235 Mon Sep 17 00:00:00 2001 From: Pete Warden Date: Tue, 4 Aug 2020 15:23:21 -0700 Subject: [PATCH 0403/1017] Fix for hello world Colab dependency error PiperOrigin-RevId: 324902066 Change-Id: Ib111d520f4de962c4c506c5641da09d3683ad6be --- .../examples/hello_world/train/train_hello_world_model.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/micro/examples/hello_world/train/train_hello_world_model.ipynb b/tensorflow/lite/micro/examples/hello_world/train/train_hello_world_model.ipynb index d0fb0eaa1b5..aea609cbb39 100644 --- a/tensorflow/lite/micro/examples/hello_world/train/train_hello_world_model.ipynb +++ b/tensorflow/lite/micro/examples/hello_world/train/train_hello_world_model.ipynb @@ -103,7 +103,8 @@ } }, "source": [ - "! pip install -q tensorflow==2" + "! pip2 install gast==0.3.3\n", + "! pip install -q tensorflow==2\n" ], "execution_count": 2, "outputs": [ From 5ad23f0dd45a53fb2781f11f6f5ae8eb9924c6d5 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Tue, 4 Aug 2020 15:32:27 -0700 Subject: [PATCH 0404/1017] Internal change PiperOrigin-RevId: 324903902 Change-Id: I5eba083589d52f72cb26bcebabff1a4795ba586a --- tensorflow/python/keras/engine/base_layer.py | 11 ++++++---- .../python/keras/engine/base_layer_v1.py | 2 ++ tensorflow/python/keras/engine/functional.py | 1 + tensorflow/python/keras/engine/sequential.py | 1 + tensorflow/python/keras/engine/training.py | 20 +++++++------------ tensorflow/python/keras/engine/training_v1.py | 14 +++++-------- tensorflow/python/keras/premade/linear.py | 2 +- tensorflow/python/keras/premade/wide_deep.py | 2 +- 8 files changed, 25 insertions(+), 28 deletions(-) diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index d7cc3fd38a8..373e17a4004 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -94,9 +94,11 @@ _TF_OP_LAYER_NAME_PREFIX = 'tf_op_layer_' _AUTOCAST_TYPES = (ops.Tensor, sparse_tensor.SparseTensor, ragged_tensor.RaggedTensor) -_keras_layers_gauge = monitoring.BoolGauge('/tensorflow/api/keras/layers', - 'keras layers usage', 'method') -_keras_model_gauge = monitoring.BoolGauge( +keras_layers_gauge = monitoring.BoolGauge('/tensorflow/api/keras/layers', + 'keras layers usage', 'method') +keras_api_gauge = monitoring.BoolGauge('/tensorflow/api/keras', + 'keras api usage', 'method') +keras_model_gauge = monitoring.BoolGauge( '/tensorflow/api/keras/premade_models', 'premade keras model usage', 'type') @@ -301,6 +303,8 @@ class Layer(module.Module, version_utils.LayerVersionSelector): dtype=None, dynamic=False, **kwargs): + keras_api_gauge.get_cell('layer').set(True) + keras_layers_gauge.get_cell(self.__class__.__name__).set(True) # These properties should be set by the user via keyword arguments. # note that 'dtype', 'input_shape' and 'batch_input_shape' # are only applicable to input layers: do not pass these keywords @@ -3084,7 +3088,6 @@ class TensorFlowOpLayer(Layer): super(TensorFlowOpLayer, self).__init__( name=_TF_OP_LAYER_NAME_PREFIX + name, trainable=trainable, dtype=dtype, autocast=False) - _keras_layers_gauge.get_cell('TensorflowOpLayer').set(True) if isinstance(node_def, dict): self.node_def = json_format.ParseDict(node_def, node_def_pb2.NodeDef()) else: diff --git a/tensorflow/python/keras/engine/base_layer_v1.py b/tensorflow/python/keras/engine/base_layer_v1.py index 9822094df26..e9ebc170b96 100644 --- a/tensorflow/python/keras/engine/base_layer_v1.py +++ b/tensorflow/python/keras/engine/base_layer_v1.py @@ -153,6 +153,8 @@ class Layer(base_layer.Layer): @trackable.no_automatic_dependency_tracking def __init__(self, trainable=True, name=None, dtype=None, dynamic=False, **kwargs): + base_layer.keras_api_gauge.get_cell('layer v1').set(True) + base_layer.keras_layers_gauge.get_cell(self.__class__.__name__).set(True) # These properties should be set by the user via keyword arguments. # note that 'dtype', 'input_shape' and 'batch_input_shape' # are only applicable to input layers: do not pass these keywords diff --git a/tensorflow/python/keras/engine/functional.py b/tensorflow/python/keras/engine/functional.py index 8422bf923d8..7c1fd4d1c72 100644 --- a/tensorflow/python/keras/engine/functional.py +++ b/tensorflow/python/keras/engine/functional.py @@ -113,6 +113,7 @@ class Functional(training_lib.Model): @trackable.no_automatic_dependency_tracking def _init_graph_network(self, inputs, outputs): + base_layer.keras_api_gauge.get_cell('Functional').set(True) # This method is needed for Sequential to reinitialize graph network when # layer is added or removed. self._is_graph_network = True diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py index e22c4921102..3b50506370b 100644 --- a/tensorflow/python/keras/engine/sequential.py +++ b/tensorflow/python/keras/engine/sequential.py @@ -114,6 +114,7 @@ class Sequential(functional.Functional): # Skip the init in FunctionalModel since model doesn't have input/output yet super(functional.Functional, self).__init__( # pylint: disable=bad-super-call name=name, autocast=False) + base_layer.keras_api_gauge.get_cell('Sequential').set(True) self.supports_masking = True self._compute_output_and_mask_jointly = True self._auto_track_sub_layers = False diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index 15f77ab8a96..bf542129e5c 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -33,7 +33,6 @@ from tensorflow.python.distribute import values as ds_values from tensorflow.python.eager import backprop from tensorflow.python.eager import context from tensorflow.python.eager import def_function -from tensorflow.python.eager import monitoring from tensorflow.python.framework import errors from tensorflow.python.framework import errors_impl from tensorflow.python.framework import func_graph @@ -96,10 +95,6 @@ except ImportError: # pylint: enable=g-import-not-at-top -_keras_api_gauge = monitoring.BoolGauge('/tensorflow/api/keras', - 'keras api usage', 'method') - - def enable_multi_worker(method): """Decorator that handles running `method` with multi-worker strategy.""" @@ -245,6 +240,8 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): @trackable.no_automatic_dependency_tracking def __init__(self, *args, **kwargs): + base_layer.keras_api_gauge.get_cell('model').set(True) + # Special case for Subclassed Functional Model, which we couldn't detect # when __new__ is called. We only realize it is a functional model when it # calls super.__init__ with input and output tensor. @@ -255,6 +252,7 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): functional.Functional.__init__(self, *args, **kwargs) return + base_layer.keras_api_gauge.get_cell('Model subclass').set(True) # The following are implemented as property functions: # self.trainable_weights # self.non_trainable_weights @@ -309,7 +307,6 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): self._init_batch_counters() self._base_model_initialized = True - _keras_api_gauge.get_cell('model').set(True) @trackable.no_automatic_dependency_tracking def _init_batch_counters(self): @@ -538,7 +535,7 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): ValueError: In case of invalid arguments for `optimizer`, `loss` or `metrics`. """ - _keras_api_gauge.get_cell('compile').set(True) + base_layer.keras_api_gauge.get_cell('compile').set(True) with self.distribute_strategy.scope(): self._validate_compile(optimizer, metrics, **kwargs) self._run_eagerly = run_eagerly @@ -1031,7 +1028,7 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): ValueError: In case of mismatch between the provided input data and what the model expects or when the input data is empty. """ - _keras_api_gauge.get_cell('fit').set(True) + base_layer.keras_api_gauge.get_cell('fit').set(True) # Legacy graph support is contained in `training_v1.Model`. version_utils.disallow_legacy_graph('Model', 'fit') self._assert_compile_was_called() @@ -1340,7 +1337,7 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): RuntimeError: If `model.evaluate` is wrapped in `tf.function`. ValueError: in case of invalid arguments. """ - _keras_api_gauge.get_cell('evaluate').set(True) + base_layer.keras_api_gauge.get_cell('evaluate').set(True) version_utils.disallow_legacy_graph('Model', 'evaluate') self._assert_compile_was_called() self._check_call_args('evaluate') @@ -1568,7 +1565,7 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): or in case a stateful model receives a number of samples that is not a multiple of the batch size. """ - _keras_api_gauge.get_cell('predict').set(True) + base_layer.keras_api_gauge.get_cell('predict').set(True) version_utils.disallow_legacy_graph('Model', 'predict') self._check_call_args('predict') _disallow_inside_tf_function('predict') @@ -1824,7 +1821,6 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): `Model.fit` now supports generators, so there is no longer any need to use this endpoint. """ - _keras_api_gauge.get_cell('fit_generator').set(True) return self.fit( generator, steps_per_epoch=steps_per_epoch, @@ -1857,7 +1853,6 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): `Model.evaluate` now supports generators, so there is no longer any need to use this endpoint. """ - _keras_api_gauge.get_cell('evaluate_generator').set(True) self._check_call_args('evaluate_generator') return self.evaluate( @@ -1885,7 +1880,6 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): `Model.predict` now supports generators, so there is no longer any need to use this endpoint. """ - _keras_api_gauge.get_cell('predict_generator').set(True) return self.predict( generator, steps=steps, diff --git a/tensorflow/python/keras/engine/training_v1.py b/tensorflow/python/keras/engine/training_v1.py index 29591e8ffb7..2ac3337948a 100644 --- a/tensorflow/python/keras/engine/training_v1.py +++ b/tensorflow/python/keras/engine/training_v1.py @@ -28,7 +28,6 @@ from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import parameter_server_strategy from tensorflow.python.eager import context from tensorflow.python.eager import def_function -from tensorflow.python.eager import monitoring from tensorflow.python.framework import composite_tensor from tensorflow.python.framework import composite_tensor_utils from tensorflow.python.framework import constant_op @@ -72,9 +71,6 @@ try: except ImportError: issparse = None -_keras_api_gauge = monitoring.BoolGauge('/tensorflow/api/keras/model_v1', - 'keras model v1 usage', 'method') - class Model(training_lib.Model): """`Model` groups layers into an object with training and inference features. @@ -142,7 +138,7 @@ class Model(training_lib.Model): def __init__(self, *args, **kwargs): super(Model, self).__init__(*args, **kwargs) - _keras_api_gauge.get_cell('model_v1').set(True) + base_layer.keras_api_gauge.get_cell('model v1').set(True) # initializing _distribution_strategy here since it is possible to call # predict on a model without compiling it. self._distribution_strategy = None @@ -413,7 +409,7 @@ class Model(training_lib.Model): # time the model gets called on training data. return self._is_compiled = True - _keras_api_gauge.get_cell('compile_v1').set(True) + base_layer.keras_api_gauge.get_cell('compile_v1').set(True) # Prepare list of loss functions, same size of model outputs. self.loss_functions = training_utils.prepare_loss_functions( @@ -774,7 +770,7 @@ class Model(training_lib.Model): and what the model expects. """ self._assert_built_as_v1() - _keras_api_gauge.get_cell('fit_v1').set(True) + base_layer.keras_api_gauge.get_cell('fit_v1').set(True) # Legacy support if 'nb_epoch' in kwargs: logging.warning( @@ -895,7 +891,7 @@ class Model(training_lib.Model): ValueError: in case of invalid arguments. """ self._assert_built_as_v1() - _keras_api_gauge.get_cell('evaluate_v1').set(True) + base_layer.keras_api_gauge.get_cell('evaluate_v1').set(True) self._assert_compile_was_called() self._check_call_args('evaluate') @@ -975,7 +971,7 @@ class Model(training_lib.Model): that is not a multiple of the batch size. """ self._assert_built_as_v1() - _keras_api_gauge.get_cell('predict_v1').set(True) + base_layer.keras_api_gauge.get_cell('predict_v1').set(True) self._check_call_args('predict') func = self._select_training_loop(x) diff --git a/tensorflow/python/keras/premade/linear.py b/tensorflow/python/keras/premade/linear.py index 20f2ce560e2..438e3270021 100644 --- a/tensorflow/python/keras/premade/linear.py +++ b/tensorflow/python/keras/premade/linear.py @@ -95,7 +95,7 @@ class LinearModel(training.Model): self.kernel_regularizer = regularizers.get(kernel_regularizer) self.bias_regularizer = regularizers.get(bias_regularizer) super(LinearModel, self).__init__(**kwargs) - base_layer._keras_model_gauge.get_cell('Linear').set(True) # pylint: disable=protected-access + base_layer.keras_model_gauge.get_cell('Linear').set(True) def build(self, input_shape): if isinstance(input_shape, dict): diff --git a/tensorflow/python/keras/premade/wide_deep.py b/tensorflow/python/keras/premade/wide_deep.py index 8638d3afc71..edb0124276f 100644 --- a/tensorflow/python/keras/premade/wide_deep.py +++ b/tensorflow/python/keras/premade/wide_deep.py @@ -85,7 +85,7 @@ class WideDeepModel(keras_training.Model): Allowed keyword arguments include `name`. """ super(WideDeepModel, self).__init__(**kwargs) - base_layer._keras_model_gauge.get_cell('WideDeep').set(True) # pylint: disable=protected-access + base_layer.keras_model_gauge.get_cell('WideDeep').set(True) self.linear_model = linear_model self.dnn_model = dnn_model self.activation = activations.get(activation) From cf59ede2e4b98b7f4ec868fe9c6c8e6f8dbffee3 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Tue, 4 Aug 2020 15:40:08 -0700 Subject: [PATCH 0405/1017] If an input-output pair is configured to be must-alias(off by default), they must be aliased at runtime. PiperOrigin-RevId: 324905361 Change-Id: Id12e9583ec25d6464f29479c48ddef37027ef61a --- .../utils/compile_mlir_util_test.cc | 2 +- tensorflow/compiler/xla/client/xla_builder.cc | 2 +- tensorflow/compiler/xla/client/xla_builder.h | 17 ++++-- .../xla/service/cpu/cpu_executable.cc | 6 ++ .../xla/service/gpu/gpu_executable.cc | 6 ++ tensorflow/compiler/xla/service/hlo.proto | 14 ++++- .../service/hlo_input_output_alias_config.cc | 38 +++++++++--- .../service/hlo_input_output_alias_config.h | 32 +++++++--- tensorflow/compiler/xla/service/hlo_parser.cc | 59 +++++++++++-------- .../compiler/xla/service/hlo_parser_test.cc | 41 ++----------- .../xla/tests/buffer_donation_test.cc | 49 +++++++++++++-- .../tpu/tpu_executable_interface.cc | 18 ++++++ 12 files changed, 194 insertions(+), 90 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc index 6ebf6897bb1..8a07aab11e1 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc @@ -524,7 +524,7 @@ TEST(CompileGraphToXlaHlo, Resources) { ASSERT_TRUE(status_or_hlo_module.ok()); constexpr char expected_hlo_module_string[] = - R"(HloModule main.4, input_output_alias={ {0}: 1 } + R"(HloModule main.4, input_output_alias={ {0}: (1, {}, may_alias) } ENTRY %main.4 (Arg_0.1: f32[2], Arg_1.2: f32[2]) -> (f32[2]) { %Arg_1.2 = f32[2]{0} parameter(1) diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc index 52f61408cbb..484fb0aabe7 100644 --- a/tensorflow/compiler/xla/client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_builder.cc @@ -446,7 +446,7 @@ StatusOr XlaBuilder::Build(int64 root_id, alias.param_index.ToString().c_str()); } TF_RETURN_IF_ERROR(config.SetUpAlias(alias.output_index, alias.param_number, - alias.param_index)); + alias.param_index, alias.kind)); } *module->mutable_input_output_alias() = config.ToProto(); return Status::OK(); diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h index 1960d0c4632..aa5074d28d9 100644 --- a/tensorflow/compiler/xla/client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_builder.h @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" +#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -349,12 +350,16 @@ class XlaBuilder { // not available until the computation is built, and eventual error in the // arguments of this API will be detected only at computation Build() time. // - // Note: Aliasing API is 'may-alias' and only donated buffer at runtime will - // be aliased with output. If a buffer is not donated at runtime, a copy will - // be inserted by XLA to prevent buffer clobbering. + // Note: Except when 'must-alias' is true, alias is assumed to be 'may-alias' + // and only donated buffer at runtime will be aliased with output. If a buffer + // is not donated at runtime, a copy will be inserted by XLA to prevent buffer + // clobbering. void SetUpAlias(const ShapeIndex& output_index, int64 param_number, - const ShapeIndex& param_index) { - input_output_aliases_.push_back({output_index, param_number, param_index}); + const ShapeIndex& param_index, + HloInputOutputAliasConfig::AliasKind kind = + HloInputOutputAliasConfig::AliasKind::kMayAlias) { + input_output_aliases_.push_back( + {output_index, param_number, param_index, kind}); } // Describes an input/output alias as inserted by the SetUpAlias() API. @@ -365,6 +370,8 @@ class XlaBuilder { int64 param_number; // Specifies the index of the aliased buffer in the parameter ShapeIndex param_index; + // Specifies if the alias is a must alias or may alias. + HloInputOutputAliasConfig::AliasKind kind; }; // Looks up the HloInstruction and sets the frontend attribute "attribute" to diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc index 0abcc91a1d7..7431e829b8e 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc @@ -247,6 +247,12 @@ StatusOr CpuExecutable::CreateResultShapedBuffer( ExecutionInput& input = arguments[alias->parameter_number]; MaybeOwningDeviceMemory* maybe_owning_memory = input.MutableBuffer(alias->parameter_index); + if (alias->must_alias() && !maybe_owning_memory->HasOwnership()) { + return InvalidArgument( + "An input was configured to be must-alias at " + "compile time but not donated at runtime: %s", + alias->ToString()); + } if (absl::optional owning = maybe_owning_memory->Release()) { // If the caller passes the ownership of the device memory, reuse it diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc index 469f2919fba..726f1963545 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc @@ -480,6 +480,12 @@ StatusOr GpuExecutable::ExecuteAsyncOnStream( ExecutionInput& input = arguments[alias->parameter_number]; MaybeOwningDeviceMemory* maybe_owning_memory = input.MutableBuffer(alias->parameter_index); + if (alias->must_alias() && !maybe_owning_memory->HasOwnership()) { + return InvalidArgument( + "An input was configured to be must-alias at " + "compile time but not donated at runtime: %s", + alias->ToString()); + } if (absl::optional owning = maybe_owning_memory->Release()) { // If the caller passes the ownership of the device memory, reuse it diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto index 960f60fe882..e043216c17e 100644 --- a/tensorflow/compiler/xla/service/hlo.proto +++ b/tensorflow/compiler/xla/service/hlo.proto @@ -283,6 +283,16 @@ message HloScheduleProto { map sequences = 1; } +enum Kind { + // Define a UNDEFINED_ALIAS equal to zero to get around the default-0 proto3 + // behavior and missing has_*() APIs. + UNDEFINED_ALIAS = 0; + // The buffers may or may not alias at runtime. + MAY_ALIAS = 1; + // The buffers must alias at runtime. + MUST_ALIAS = 2; +} + message HloInputOutputAliasProto { // The following proto describes a pair of aliased an input // (described by parameter number and a ShapeIndex of the parameter) @@ -304,8 +314,8 @@ message HloInputOutputAliasProto { int64 parameter_number = 2; // ShapeIndex of the parameter instruction. repeated int64 parameter_shape_index = 3; - reserved 4; - reserved "kind"; + // The kind of alias to be setup. + Kind kind = 4; } repeated AliasEntryProto entries = 1; diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc index e123161720b..34bc30d641f 100644 --- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc +++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h" +#include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_module.h" namespace xla { @@ -24,9 +25,10 @@ bool HloInputOutputAliasConfig::OutputHasAlias( return alias_.element(output_index).has_value(); } -Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index, - int64 param_number, - const ShapeIndex& param_index) { +Status HloInputOutputAliasConfig::SetUpAlias( + const ShapeIndex& output_index, int64 param_number, + const ShapeIndex& param_index, + HloInputOutputAliasConfig::AliasKind must_alias) { TF_RET_CHECK(ShapeUtil::IndexIsValid(alias_.shape(), output_index)) << "Trying to set up alias at " << output_index.ToString() << " which is an invalid index for shape " @@ -41,7 +43,8 @@ Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index, param_number, param_index.ToString(), output_index.ToString(), alias_.element(output_index)->parameter_number, alias_.element(output_index)->parameter_index.ToString()); - (*alias_.mutable_element(output_index)) = Alias(param_number, param_index); + (*alias_.mutable_element(output_index)) = + Alias(param_number, param_index, must_alias); VLOG(4) << "Set up alias between output index " << output_index.ToString() << " and parameter " << param_index << " at index " << param_index.ToString(); @@ -61,6 +64,11 @@ HloInputOutputAliasProto HloInputOutputAliasConfig::ToProto() const { for (int64 i : data->parameter_index) { entry.add_parameter_shape_index(i); } + if (data->must_alias()) { + entry.set_kind(Kind::MUST_ALIAS); + } else { + entry.set_kind(Kind::MAY_ALIAS); + } result.add_entries()->Swap(&entry); } }); @@ -77,8 +85,9 @@ StatusOr HloInputOutputAliasConfig::CreateFromProto( int64 param_number = entry.parameter_number(); ShapeIndex param_index(entry.parameter_shape_index().begin(), entry.parameter_shape_index().end()); + AliasKind kind = entry.kind() == Kind::MAY_ALIAS ? kMayAlias : kMustAlias; TF_RETURN_IF_ERROR( - result.SetUpAlias(output_index, param_number, param_index)); + result.SetUpAlias(output_index, param_number, param_index, kind)); } return result; } @@ -93,9 +102,9 @@ string HloInputOutputAliasConfig::ToString() const { ForEachAlias([&](const ShapeIndex& output_index, const Alias& alias) { pieces.push_back(absl::StrFormat( - " OutputIndex %s is aliased with parameter %lld at %s:", - output_index.ToString(), alias.parameter_number, - alias.parameter_index.ToString())); + " OutputIndex %s is %saliased with parameter %lld at %s:", + output_index.ToString(), alias.kind == kMustAlias ? "must-" : "may-", + alias.parameter_number, alias.parameter_index.ToString())); }); return absl::StrJoin(pieces, "\n"); } @@ -112,6 +121,19 @@ string HloInputOutputAliasConfig::ToShortString() const { return absl::StrJoin(pieces, ", "); } +bool HloInputOutputAliasConfig::ParameterMustAlias( + int64 param_number, const ShapeIndex& param_index) const { + bool result = false; + alias_.ForEachElement( + [&](const xla::ShapeIndex&, absl::optional alias) { + if (alias && alias->parameter_number == param_number && + alias->parameter_index == param_index && alias->must_alias()) { + result = true; + } + }); + return result; +} + absl::optional HloInputOutputAliasConfig::GetAliasedOutput( int64 param_number, const ShapeIndex& param_index) const { absl::optional output; diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h index d5ca28e9387..6b84bdb6a68 100644 --- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h +++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h @@ -32,22 +32,32 @@ class HloModule; // parameter index in the entry computation. class HloInputOutputAliasConfig { public: + // The kind of aliases which can be set. A kMayAlias is one setup at + // compilation time by the user, and has to be respected. A kMustAlias one + // might be setup by the compiler, if it decides it is convenient to do so. + enum AliasKind { + kMayAlias, + kMustAlias, + }; // Defines the alias information for a given output buffer. A given output // buffer shape index can refer only to one parameter+index. struct Alias { - Alias(int64 parameter_number, ShapeIndex parameter_index) + Alias(int64 parameter_number, ShapeIndex parameter_index, + AliasKind kind = kMayAlias) : parameter_number(parameter_number), - parameter_index(std::move(parameter_index)) {} + parameter_index(std::move(parameter_index)), + kind(kind) {} int64 parameter_number; ShapeIndex parameter_index; + AliasKind kind; + + bool must_alias() const { return kind == kMustAlias; } std::string ToString() { - if (parameter_index.empty()) { - return absl::StrCat(parameter_number); - } - return absl::StrFormat("(%lld, %s)", parameter_number, - parameter_index.ToString()); + return absl::StrFormat("(%lld, %s, %s)", parameter_number, + parameter_index.ToString(), + kind == kMustAlias ? "must_alias" : "may_alias"); } }; @@ -61,7 +71,8 @@ class HloInputOutputAliasConfig { // Sets up alias config from `output_index` to `param_index` at // `param_number`. Status SetUpAlias(const ShapeIndex& output_index, int64 param_number, - const ShapeIndex& param_index); + const ShapeIndex& param_index, + AliasKind must_alias = kMayAlias); // Returns true if the given parameter is aliased with one of the output // buffers. @@ -92,6 +103,11 @@ class HloInputOutputAliasConfig { absl::optional GetAliasedParameter( const ShapeIndex& output_index) const; + // Returns if the parameter at the given parameter number and parameter + // index must-alias with an output. + bool ParameterMustAlias(int64 param_number, + const ShapeIndex& param_index) const; + using AliasFn = std::function; diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc index 0530062c43b..31afe2a3673 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.cc +++ b/tensorflow/compiler/xla/service/hlo_parser.cc @@ -552,33 +552,39 @@ bool HloParserImpl::ParseAliasing(AliasingData* data) { return false; } - if (lexer_.GetKind() != TokKind::kLparen) { - // Short form: "{0}: 0", output index "{}" is assumed. - int64 param_num; - ParseInt64(¶m_num); - data->emplace(std::piecewise_construct, std::forward_as_tuple(out), - std::forward_as_tuple(param_num, ShapeIndex{})); - } else { - // Long form: "{0}: (0, {0})", output index is explicitly specified. - if (!ParseToken(TokKind::kLparen, errmsg)) { - return false; - } - int64 param_num; - ParseInt64(¶m_num); - if (!ParseToken(TokKind::kComma, errmsg)) { - return false; - } - ShapeIndex param_idx; - if (!ParseShapeIndex(¶m_idx)) { - return false; - } - data->emplace(std::piecewise_construct, std::forward_as_tuple(out), - std::forward_as_tuple(param_num, param_idx)); - if (!ParseToken(TokKind::kRparen, errmsg)) { - return false; + if (!ParseToken(TokKind::kLparen, errmsg)) { + return false; + } + int64 param_num; + ParseInt64(¶m_num); + if (!ParseToken(TokKind::kComma, errmsg)) { + return false; + } + ShapeIndex param_idx; + if (!ParseShapeIndex(¶m_idx)) { + return false; + } + + HloInputOutputAliasConfig::AliasKind alias_kind = + HloInputOutputAliasConfig::kMayAlias; + if (EatIfPresent(TokKind::kComma)) { + std::string type; + ParseName(&type); + if (type == "must-alias") { + alias_kind = HloInputOutputAliasConfig::kMustAlias; + } else if (type == "may-alias") { + alias_kind = HloInputOutputAliasConfig::kMayAlias; + } else { + return TokenError("Unexpected aliasing kind; expected SYSTEM or USER"); } } + data->emplace(std::piecewise_construct, std::forward_as_tuple(out), + std::forward_as_tuple(param_num, param_idx, alias_kind)); + if (!ParseToken(TokKind::kRparen, errmsg)) { + return false; + } + if (!EatIfPresent(TokKind::kComma)) { break; } @@ -624,8 +630,9 @@ bool HloParserImpl::ParseHloModule(HloModule* module) { if (aliasing_data) { HloInputOutputAliasConfig alias_config(module->result_shape()); for (auto& p : *aliasing_data) { - Status st = alias_config.SetUpAlias(p.first, p.second.parameter_number, - p.second.parameter_index); + Status st = + alias_config.SetUpAlias(p.first, p.second.parameter_number, + p.second.parameter_index, p.second.kind); if (!st.ok()) { return TokenError(st.error_message()); } diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc index 484578e5e0e..86b6b1bedd9 100644 --- a/tensorflow/compiler/xla/service/hlo_parser_test.cc +++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc @@ -2399,7 +2399,7 @@ ENTRY c2 { TEST_F(HloParserTest, SimpleAliasing) { const string original = R"( -HloModule Module, input_output_alias={ {0}: (0, {0}), {1}: (0, {1}) } +HloModule Module, input_output_alias={ {0}: (0, {0}, must-alias), {1}: (0, {1}) } ENTRY entry { %p = (f32[], f32[]) parameter(0) @@ -2413,42 +2413,13 @@ ENTRY entry { std::unique_ptr parsed_module = module.ConsumeValueOrDie(); EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(0, {0}), ShapeIndex{0}); + + EXPECT_TRUE( + parsed_module->input_output_alias_config().ParameterMustAlias(0, {0})); EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(0, {1}), ShapeIndex{1}); -} - -TEST_F(HloParserTest, SimpleAliasingShortForm) { - const string original = R"( -HloModule Module, input_output_alias={ {0}: 0, {1}: 1 } - -ENTRY entry { - %p0 = f32[] parameter(0) - %p1 = f32[] parameter(1) - ROOT %out = (f32[], f32[]) tuple(%p0, %p1) -} - )"; - auto module = ParseAndReturnVerifiedModule(original); - TF_ASSERT_OK(module.status()); - std::unique_ptr parsed_module = module.ConsumeValueOrDie(); - EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(0, {}), - ShapeIndex{0}); - EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(1, {}), - ShapeIndex{1}); -} - -TEST_F(HloParserTest, SimpleAliasingShortFormError) { - const string original = R"( -HloModule Module, input_output_alias={ {0}: A, {1}: 1 } - -ENTRY entry { - %p0 = f32[] parameter(0) - %p1 = f32[] parameter(1) - ROOT %out = (f32[], f32[]) tuple(%p0, %p1) -} - )"; - ExpectHasSubstr( - ParseAndReturnUnverifiedModule(original).status().error_message(), - "expects integer"); + EXPECT_FALSE( + parsed_module->input_output_alias_config().ParameterMustAlias(0, {1})); } TEST_F(HloParserTest, NestedAliasing) { diff --git a/tensorflow/compiler/xla/tests/buffer_donation_test.cc b/tensorflow/compiler/xla/tests/buffer_donation_test.cc index 856ea7c9b44..f78083fe2af 100644 --- a/tensorflow/compiler/xla/tests/buffer_donation_test.cc +++ b/tensorflow/compiler/xla/tests/buffer_donation_test.cc @@ -61,7 +61,7 @@ class BufferDonationTest : public HloTestBase { absl::Span argument_literals, absl::Span donate_arguments, absl::Span expected_runtime_aliasing, - const Literal& expected) { + const Literal& expected, std::string expected_failure = "") { // Create a copy of the output shape because the HLO module is std::moved // into the compiler and may be deallocated. const Shape output_shape = hlo_module->result_shape(); @@ -123,10 +123,19 @@ class BufferDonationTest : public HloTestBase { ExecutionInput(std::move(owned_buffers), argument_literal.shape())); } - TF_ASSERT_OK_AND_ASSIGN( - ExecutionOutput output, + StatusOr output_status = executable->ExecuteAsyncOnStream(&service_run_options, std::move(args), - /*hlo_execution_profile=*/nullptr)); + /*hlo_execution_profile=*/nullptr); + if (!expected_failure.empty()) { + ASSERT_FALSE(output_status.ok()); + ASSERT_TRUE(absl::StrContains(output_status.status().error_message(), + expected_failure)) + << "got: \n" + << output_status.status().error_message() << " \nvs want\n" + << expected_failure; + return; + } + ExecutionOutput output = output_status.ConsumeValueOrDie(); se::DeviceMemoryBase result_root_buffer = output.Result().root_buffer(); LOG(INFO) << "result allocation = " << result_root_buffer.opaque() @@ -303,5 +312,37 @@ ENTRY entry { #endif } +TEST_F(BufferDonationTest, TestMustAliasNotDonated) { + HloModuleConfig config; + + StatusOr> module = + ParseAndReturnVerifiedModule(R"( +HloModule module + +ENTRY entry { + a = f32[] parameter(0) + b = f32[] parameter(1) + ROOT out = (f32[], f32[]) tuple(a, b) +} + )", + config); + + TF_ASSERT_OK(module->get()->input_output_alias_config().SetUpAlias( + {0}, 0, {}, HloInputOutputAliasConfig::kMustAlias)); + + std::vector args; + args.push_back(LiteralUtil::CreateR0(0.1)); + args.push_back(LiteralUtil::CreateR0(0.2)); + Literal expected = LiteralUtil::MakeTupleFromSlices( + {LiteralUtil::CreateR0(0.1), LiteralUtil::CreateR0(0.2)}); + +#ifndef XLA_TEST_BACKEND_INTERPRETER + RunAndCheck(std::move(*module), args, + /*donate_arguments=*/{false, false}, {true, false}, expected, + "An input was configured to be must-alias at " + "compile time but not donated at runtime:"); +#endif +} + } // namespace } // namespace xla diff --git a/tensorflow/stream_executor/tpu/tpu_executable_interface.cc b/tensorflow/stream_executor/tpu/tpu_executable_interface.cc index 13f9db98e5d..f260cc1631f 100644 --- a/tensorflow/stream_executor/tpu/tpu_executable_interface.cc +++ b/tensorflow/stream_executor/tpu/tpu_executable_interface.cc @@ -62,6 +62,24 @@ TpuExecutableInterface::AllocateOutputMemoryWithInputReuse( << " host_shape = " << ShapeUtil::HumanStringWithLayout(host_shape); Shape device_shape = HostShapeToDeviceShape(host_shape); + TF_RETURN_IF_ERROR(alias_config.ForEachAliasWithStatus( + [&](const ShapeIndex& output_index, + absl::optional alias) { + if (alias && alias->must_alias()) { + VLOG(1) << alias->ToString(); + const MaybeOwningDeviceMemory& original_input = + (*arguments)[alias->parameter_number].Buffers().element( + alias->parameter_index); + if (!original_input.HasOwnership()) { + return InvalidArgument( + "An input was configured to be must-alias at " + "compile time but not donated at runtime: %s", + alias->ToString()); + } + } + return Status::OK(); + })); + if (VLOG_IS_ON(3)) { VLOG(3) << "AllocateOutputMemoryWithInputReuse, device = " << device_ordinal << " host_shape = " << ShapeUtil::HumanStringWithLayout(host_shape); From c96f601e0caabe294c906aa0355b18e970713091 Mon Sep 17 00:00:00 2001 From: Yanhua Sun Date: Tue, 4 Aug 2020 15:47:24 -0700 Subject: [PATCH 0406/1017] remove obsolete comment, the bug is already fixed PiperOrigin-RevId: 324906751 Change-Id: I625e5f9509044c40130733168d7c6a92c25d57bd --- tensorflow/python/eager/function.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index 53d4b62b9b5..289b8a32cdb 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -2756,9 +2756,6 @@ def _convert_inputs_to_signature(inputs, input_signature, flat_input_signature): ",\n ".join(str(i) for i in input_signature) + ")") try: - # TODO(b/124370185): Use all elements as inputs to throw an error if there - # are ignored arguments. Calling with arguments that are not part of the - # signature should throw an error. flatten_inputs = nest.flatten_up_to( input_signature, inputs[:len(input_signature)], From 683b1bbc357da95a144ed7e7f7b368b9b7e468c3 Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Tue, 4 Aug 2020 16:02:27 -0700 Subject: [PATCH 0407/1017] [tf.data service] Track task creation in the dispatcher journal. This way, the dispatcher will remember which tasks exist on startup, so that when workers reconnect the dispatcher will understand their tasks and allow them to continue processing their existing tasks. Without this journaling, the dispatcher won't know what task ids to assign to new tasks, since not-yet-reconnected workers could already be using task ids. Now that task state is managed by dispatcher_state, we can replace FinishJobUpdate with FinishTaskUpdate, since dispatcher_state can identify a job as finished when its last task finishes. PiperOrigin-RevId: 324909615 Change-Id: Iee5b877aac79046662231e4b9d2a01a271a71d5d --- .../core/data/service/dispatcher_impl.cc | 95 +++++------ .../core/data/service/dispatcher_impl.h | 46 ++--- .../core/data/service/dispatcher_state.cc | 65 ++++++- .../core/data/service/dispatcher_state.h | 33 +++- .../data/service/dispatcher_state_test.cc | 160 ++++++++++++++++-- tensorflow/core/data/service/journal.proto | 14 +- tensorflow/core/data/service/journal_test.cc | 10 +- .../experimental/data_service_dataset_op.cc | 12 +- 8 files changed, 312 insertions(+), 123 deletions(-) diff --git a/tensorflow/core/data/service/dispatcher_impl.cc b/tensorflow/core/data/service/dispatcher_impl.cc index 4a3764ecea3..77477df71e4 100644 --- a/tensorflow/core/data/service/dispatcher_impl.cc +++ b/tensorflow/core/data/service/dispatcher_impl.cc @@ -43,13 +43,14 @@ namespace data { namespace { // The name of the journal directory inside the dispatcher's working directory. -constexpr StringPiece kJournalDir = "journal"; +constexpr char kJournalDir[] = "journal"; using Dataset = DispatcherState::Dataset; using NamedJobKey = DispatcherState::NamedJobKey; using Job = DispatcherState::Job; +using Task = DispatcherState::Task; -std::string JournalDir(StringPiece work_dir) { +std::string JournalDir(const std::string& work_dir) { return io::JoinPath(work_dir, kJournalDir); } @@ -115,8 +116,8 @@ Status DataServiceDispatcherImpl::RegisterWorker( if (job->finished) { continue; } - std::shared_ptr task = CreateTask(job, worker_address); - + std::shared_ptr task; + TF_RETURN_IF_ERROR(CreateTask(job, worker_address, &task)); TaskDef* task_def = response->add_tasks(); std::shared_ptr dataset; TF_RETURN_IF_ERROR(state_.DatasetFromId(job->dataset_id, &dataset)); @@ -134,35 +135,20 @@ Status DataServiceDispatcherImpl::RegisterWorker( Status DataServiceDispatcherImpl::WorkerUpdate( const WorkerUpdateRequest* request, WorkerUpdateResponse* response) { mutex_lock l(mu_); - int64 worker_id = request->worker_id(); for (auto& update : request->updates()) { int64 task_id = update.task_id(); - const auto it = tasks_.find(task_id); - if (it == tasks_.end()) { - return errors::NotFound("WorkerUpdate called for worker ", worker_id, - " with unknown task id ", task_id); - } - std::shared_ptr task = it->second; + std::shared_ptr task; + TF_RETURN_IF_ERROR(state_.TaskFromId(task_id, &task)); if (update.completed()) { if (task->finished) { VLOG(1) << "Received completion update for already-finished task " << task->task_id << " on worker " << task->worker_address; continue; } - task->finished = true; - bool finished = true; - for (const auto& job_task : tasks_by_job_[task->job_id]) { - if (!job_task->finished) { - finished = false; - break; - } - } - if (finished) { - Update update; - FinishJobUpdate* finish_job = update.mutable_finish_job(); - finish_job->set_job_id(task->job_id); - TF_RETURN_IF_ERROR(Apply(update)); - } + Update update; + FinishTaskUpdate* finish_task = update.mutable_finish_task(); + finish_task->set_task_id(task_id); + TF_RETURN_IF_ERROR(Apply(update)); VLOG(3) << "Task " << task_id << " from job " << task->job_id << " completed"; } @@ -221,7 +207,7 @@ Status DataServiceDispatcherImpl::CreateJob(const CreateJobRequest* request, mutex_lock l(mu_); TF_RETURN_IF_ERROR(CreateJob(request->dataset_id(), processing_mode, absl::optional(), &job)); - tasks = CreateTasksForJob(job); + TF_RETURN_IF_ERROR(CreateTasksForJob(job, &tasks)); } response->set_job_id(job->job_id); TF_RETURN_IF_ERROR(AssignTasks(tasks)); @@ -256,7 +242,7 @@ Status DataServiceDispatcherImpl::GetOrCreateJob( } TF_RETURN_IF_ERROR( CreateJob(request->dataset_id(), requested_processing_mode, key, &job)); - tasks = CreateTasksForJob(job); + TF_RETURN_IF_ERROR(CreateTasksForJob(job, &tasks)); } TF_RETURN_IF_ERROR(AssignTasks(tasks)); response->set_job_id(job->job_id); @@ -320,28 +306,35 @@ Status DataServiceDispatcherImpl::CreateJob( return Status::OK(); } -std::vector> -DataServiceDispatcherImpl::CreateTasksForJob(std::shared_ptr job) +Status DataServiceDispatcherImpl::CreateTasksForJob( + std::shared_ptr job, + std::vector>* tasks) EXCLUSIVE_LOCKS_REQUIRED(mu_) { - std::vector> tasks; - tasks.reserve(workers_.size()); + tasks->clear(); + tasks->reserve(workers_.size()); for (const auto& it : workers_) { std::shared_ptr worker = it.second; - tasks.push_back(CreateTask(job, worker->address)); + std::shared_ptr task; + TF_RETURN_IF_ERROR(CreateTask(job, worker->address, &task)); + tasks->push_back(task); } - return tasks; + return Status::OK(); } -std::shared_ptr -DataServiceDispatcherImpl::CreateTask(std::shared_ptr job, - const std::string& worker_address) +Status DataServiceDispatcherImpl::CreateTask(std::shared_ptr job, + const std::string& worker_address, + std::shared_ptr* task) EXCLUSIVE_LOCKS_REQUIRED(mu_) { - int64 task_id = next_task_id_++; - DCHECK(!tasks_.contains(task_id)); - tasks_[task_id] = std::make_shared(task_id, job->job_id, - job->dataset_id, worker_address); - tasks_by_job_[job->job_id].push_back(tasks_[task_id]); - return tasks_[task_id]; + int64 task_id = state_.NextAvailableTaskId(); + Update update; + CreateTaskUpdate* create_task = update.mutable_create_task(); + create_task->set_task_id(task_id); + create_task->set_job_id(job->job_id); + create_task->set_dataset_id(job->dataset_id); + create_task->set_worker_address(worker_address); + TF_RETURN_IF_ERROR(Apply(update)); + TF_RETURN_IF_ERROR(state_.TaskFromId(task_id, task)); + return Status::OK(); } Status DataServiceDispatcherImpl::AssignTasks( @@ -393,24 +386,16 @@ Status DataServiceDispatcherImpl::GetTasks(const GetTasksRequest* request, GetTasksResponse* response) { mutex_lock l(mu_); VLOG(3) << "Looking up tasks for job id " << request->job_id(); - auto it = tasks_by_job_.find(request->job_id()); - if (it == tasks_by_job_.end()) { - return errors::NotFound("GetTasks failed. Job id <", request->job_id(), - "> not found."); - } - std::vector>& tasks = it->second; - bool has_finished_tasks = false; + std::vector> tasks; + TF_RETURN_IF_ERROR(state_.TasksForJob(request->job_id(), &tasks)); for (const auto& task : tasks) { - if (task->finished) { - has_finished_tasks = true; - continue; - } TaskInfo* task_info = response->mutable_task_info()->Add(); task_info->set_worker_address(task->worker_address); task_info->set_id(task->task_id); } - response->set_job_finished(has_finished_tasks && - response->task_info_size() == 0); + std::shared_ptr job; + TF_RETURN_IF_ERROR(state_.JobFromId(request->job_id(), &job)); + response->set_job_finished(job->finished); VLOG(3) << "Found " << response->task_info_size() << " tasks for job id " << request->job_id(); return Status::OK(); diff --git a/tensorflow/core/data/service/dispatcher_impl.h b/tensorflow/core/data/service/dispatcher_impl.h index e39f3269d02..6fa1815e9eb 100644 --- a/tensorflow/core/data/service/dispatcher_impl.h +++ b/tensorflow/core/data/service/dispatcher_impl.h @@ -80,21 +80,6 @@ class DataServiceDispatcherImpl { std::unique_ptr stub; }; - struct Task { - Task(int64 task_id, int64 job_id, int64 dataset_id, - const std::string& worker_address) - : task_id(task_id), - job_id(job_id), - dataset_id(dataset_id), - worker_address(worker_address) {} - - const int64 task_id; - const int64 job_id; - const int64 dataset_id; - const std::string worker_address; - bool finished = false; - }; - // Registers a dataset with the given fingerprint, storing the new dataset's // id in `*dataset-id`. Status RegisterDataset(uint64 fingerprint, const DatasetDef& dataset, @@ -107,22 +92,26 @@ class DataServiceDispatcherImpl { absl::optional named_job_key, std::shared_ptr* job) EXCLUSIVE_LOCKS_REQUIRED(mu_); - // Creates one task for each worker, for the given job. This method only - // updates dispatcher metadata with the new tasks, but doesn't assign the - // tasks to the workers. - std::vector> CreateTasksForJob( - std::shared_ptr job) - EXCLUSIVE_LOCKS_REQUIRED(mu_); - // Creates a new task for a job, returning a pointer to the created task. - std::shared_ptr CreateTask( + // Creates one task for each worker, for the given job. The created tasks are + // stored in `*tasks`. This method only updates dispatcher metadata with the + // new tasks, but doesn't assign the tasks to the workers. + Status CreateTasksForJob( std::shared_ptr job, - const std::string& worker_address) EXCLUSIVE_LOCKS_REQUIRED(mu_); + std::vector>* tasks) + EXCLUSIVE_LOCKS_REQUIRED(mu_); + + // Creates a new task for a job, storing the created task in `*task`. + Status CreateTask(std::shared_ptr job, + const std::string& worker_address, + std::shared_ptr* task); // Assigns the list of tasks to the workers indicated by their // `worker_address` fields. - Status AssignTasks(std::vector> tasks) + Status AssignTasks( + std::vector> tasks) LOCKS_EXCLUDED(mu_); // Assigns a task to the worker indicated by its `worker_address` field. - Status AssignTask(std::shared_ptr task) LOCKS_EXCLUDED(mu_); + Status AssignTask(std::shared_ptr task) + LOCKS_EXCLUDED(mu_); // Validates that an existing job matches the given processing_mode and // dataset_id, returning an error status describing any difference. Status ValidateMatchingJob(std::shared_ptr job, @@ -145,11 +134,6 @@ class DataServiceDispatcherImpl { // Registered workers, keyed by their addresses. absl::flat_hash_map> workers_ TF_GUARDED_BY(mu_); - // Tasks, keyed by task ids. - absl::flat_hash_map> tasks_ TF_GUARDED_BY(mu_); - // Mapping from job id to the tasks for that job. - absl::flat_hash_map>> tasks_by_job_ - TF_GUARDED_BY(mu_); absl::optional> journal_writer_ TF_GUARDED_BY(mu_); diff --git a/tensorflow/core/data/service/dispatcher_state.cc b/tensorflow/core/data/service/dispatcher_state.cc index 64be7fbc54e..093457a55af 100644 --- a/tensorflow/core/data/service/dispatcher_state.cc +++ b/tensorflow/core/data/service/dispatcher_state.cc @@ -33,8 +33,11 @@ Status DispatcherState::Apply(Update update) { case Update::kCreateJob: CreateJob(update.create_job()); break; - case Update::kFinishJob: - FinishJob(update.finish_job()); + case Update::kCreateTask: + CreateTask(update.create_task()); + break; + case Update::kFinishTask: + FinishTask(update.finish_task()); break; case Update::UPDATE_TYPE_NOT_SET: return errors::Internal("Update type not set."); @@ -68,7 +71,6 @@ void DispatcherState::CreateJob(const CreateJobUpdate& create_job) { named_job_key); DCHECK(!jobs_.contains(job_id)); jobs_[job_id] = job; - LOG(INFO) << "Created a new job with id " << job_id; if (named_job_key.has_value()) { DCHECK(!named_jobs_.contains(named_job_key.value())); named_jobs_[named_job_key.value()] = job; @@ -76,10 +78,31 @@ void DispatcherState::CreateJob(const CreateJobUpdate& create_job) { next_available_job_id_ = std::max(next_available_job_id_, job_id + 1); } -void DispatcherState::FinishJob(const FinishJobUpdate& finish_job) { - int64 job_id = finish_job.job_id(); - DCHECK(jobs_.contains(job_id)); - jobs_[job_id]->finished = true; +void DispatcherState::CreateTask(const CreateTaskUpdate& create_task) { + int64 task_id = create_task.task_id(); + auto& task = tasks_[task_id]; + DCHECK_EQ(task, nullptr); + task = std::make_shared(task_id, create_task.job_id(), + create_task.dataset_id(), + create_task.worker_address()); + tasks_by_job_[create_task.job_id()].push_back(task); + next_available_task_id_ = std::max(next_available_task_id_, task_id + 1); +} + +void DispatcherState::FinishTask(const FinishTaskUpdate& finish_task) { + VLOG(2) << "Marking task " << finish_task.task_id() << " as finished"; + int64 task_id = finish_task.task_id(); + auto& task = tasks_[task_id]; + DCHECK(task != nullptr); + task->finished = true; + bool all_finished = true; + for (const auto& task_for_job : tasks_by_job_[task->job_id]) { + if (!task_for_job->finished) { + all_finished = false; + } + } + VLOG(3) << "Job " << task->job_id << " finished: " << all_finished; + jobs_[task->job_id]->finished = all_finished; } int64 DispatcherState::NextAvailableDatasetId() const { @@ -141,5 +164,33 @@ int64 DispatcherState::NextAvailableJobId() const { return next_available_job_id_; } +Status DispatcherState::TaskFromId(int64 id, + std::shared_ptr* task) const { + auto it = tasks_.find(id); + if (it == tasks_.end()) { + return errors::NotFound("Task ", id, " not found"); + } + *task = it->second; + return Status::OK(); +} + +Status DispatcherState::TasksForJob( + int64 job_id, std::vector>* tasks) const { + auto it = tasks_by_job_.find(job_id); + if (it == tasks_by_job_.end()) { + return errors::NotFound("Job ", job_id, " not found"); + } + tasks->clear(); + tasks->reserve(it->second.size()); + for (const auto& task : it->second) { + tasks->push_back(task); + } + return Status::OK(); +} + +int64 DispatcherState::NextAvailableTaskId() const { + return next_available_task_id_; +} + } // namespace data } // namespace tensorflow diff --git a/tensorflow/core/data/service/dispatcher_state.h b/tensorflow/core/data/service/dispatcher_state.h index e54f51ba499..7313274ae71 100644 --- a/tensorflow/core/data/service/dispatcher_state.h +++ b/tensorflow/core/data/service/dispatcher_state.h @@ -110,6 +110,21 @@ class DispatcherState { bool finished = false; }; + struct Task { + Task(int64 task_id, int64 job_id, int64 dataset_id, + const std::string& worker_address) + : task_id(task_id), + job_id(job_id), + dataset_id(dataset_id), + worker_address(worker_address) {} + + const int64 task_id; + const int64 job_id; + const int64 dataset_id; + const std::string worker_address; + bool finished = false; + }; + // Returns the next available dataset id. int64 NextAvailableDatasetId() const; // Gets a dataset by id. Returns NOT_FOUND if there is no such dataset. @@ -128,11 +143,21 @@ class DispatcherState { // Gets a named job by key. Returns NOT_FOUND if there is no such job. Status NamedJobByKey(NamedJobKey key, std::shared_ptr* job) const; + // Returns the next available task id. + int64 NextAvailableTaskId() const; + // Gets a task by id. Returns NOT_FOUND if there is no such task. + Status TaskFromId(int64 id, std::shared_ptr* task) const; + // Stores a list of all tasks for the given job to `*tasks`. Returns NOT_FOUND + // if there is no such job. + Status TasksForJob(int64 job_id, + std::vector>* tasks) const; + private: // Registers a dataset. The dataset must not already be registered. void RegisterDataset(const RegisterDatasetUpdate& register_dataset); void CreateJob(const CreateJobUpdate& create_job); - void FinishJob(const FinishJobUpdate& finish_job); + void CreateTask(const CreateTaskUpdate& create_task); + void FinishTask(const FinishTaskUpdate& finish_task); int64 next_available_dataset_id_ = 0; // Registered datasets, keyed by dataset ids. @@ -147,6 +172,12 @@ class DispatcherState { // Named jobs, keyed by their names and indices. Not all jobs have names, so // this is a subset of the jobs stored in `jobs_`. absl::flat_hash_map> named_jobs_; + + int64 next_available_task_id_ = 0; + // Tasks, keyed by task ids. + absl::flat_hash_map> tasks_; + // Tasks, keyed by job ids. + absl::flat_hash_map>> tasks_by_job_; }; } // namespace data diff --git a/tensorflow/core/data/service/dispatcher_state_test.cc b/tensorflow/core/data/service/dispatcher_state_test.cc index 02961d5bd1d..933d783d227 100644 --- a/tensorflow/core/data/service/dispatcher_state_test.cc +++ b/tensorflow/core/data/service/dispatcher_state_test.cc @@ -26,6 +26,12 @@ namespace tensorflow { namespace data { namespace { +using Dataset = DispatcherState::Dataset; +using NamedJobKey = DispatcherState::NamedJobKey; +using Job = DispatcherState::Job; +using Task = DispatcherState::Task; +using ::testing::SizeIs; + Status RegisterDatasetWithIdAndFingerprint(int64 id, uint64 fingerprint, DispatcherState* state) { Update update; @@ -47,8 +53,7 @@ Status CreateAnonymousJob(int64 job_id, int64 dataset_id, return Status::OK(); } -Status CreateNamedJob(int64 job_id, int64 dataset_id, - DispatcherState::NamedJobKey named_job_key, +Status CreateNamedJob(int64 job_id, int64 dataset_id, NamedJobKey named_job_key, DispatcherState* state) { Update update; CreateJobUpdate* create_job = update.mutable_create_job(); @@ -62,10 +67,22 @@ Status CreateNamedJob(int64 job_id, int64 dataset_id, return Status::OK(); } -Status FinishJob(int64 job_id, DispatcherState* state) { +Status CreateTask(int64 task_id, int64 job_id, int64 dataset_id, + StringPiece worker_address, DispatcherState* state) { Update update; - FinishJobUpdate* finish_job = update.mutable_finish_job(); - finish_job->set_job_id(job_id); + CreateTaskUpdate* create_task = update.mutable_create_task(); + create_task->set_task_id(task_id); + create_task->set_job_id(job_id); + create_task->set_dataset_id(dataset_id); + create_task->set_worker_address(worker_address); + TF_RETURN_IF_ERROR(state->Apply(update)); + return Status::OK(); +} + +Status FinishTask(int64 task_id, DispatcherState* state) { + Update update; + FinishTaskUpdate* finish_task = update.mutable_finish_task(); + finish_task->set_task_id(task_id); TF_RETURN_IF_ERROR(state->Apply(update)); return Status::OK(); } @@ -76,14 +93,15 @@ TEST(DispatcherState, RegisterDataset) { uint64 fingerprint = 20; DispatcherState state; TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(id, fingerprint, &state)); + EXPECT_EQ(state.NextAvailableDatasetId(), id + 1); { - std::shared_ptr dataset; + std::shared_ptr dataset; TF_EXPECT_OK(state.DatasetFromFingerprint(fingerprint, &dataset)); EXPECT_EQ(id, dataset->dataset_id); } { - std::shared_ptr dataset; + std::shared_ptr dataset; TF_EXPECT_OK(state.DatasetFromId(id, &dataset)); EXPECT_EQ(fingerprint, dataset->fingerprint); } @@ -91,14 +109,14 @@ TEST(DispatcherState, RegisterDataset) { TEST(DispatcherState, MissingDatasetId) { DispatcherState state; - std::shared_ptr dataset; + std::shared_ptr dataset; Status s = state.DatasetFromId(0, &dataset); EXPECT_EQ(s.code(), error::NOT_FOUND); } TEST(DispatcherState, MissingDatasetFingerprint) { DispatcherState state; - std::shared_ptr dataset; + std::shared_ptr dataset; Status s = state.DatasetFromFingerprint(0, &dataset); EXPECT_EQ(s.code(), error::NOT_FOUND); } @@ -123,11 +141,11 @@ TEST(DispatcherState, AnonymousJob) { int64 job_id = 3; int64 dataset_id = 10; DispatcherState state; - Update update; TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); - std::shared_ptr job; + std::shared_ptr job; TF_EXPECT_OK(state.JobFromId(job_id, &job)); + EXPECT_EQ(state.NextAvailableJobId(), job_id + 1); EXPECT_EQ(dataset_id, job->dataset_id); EXPECT_EQ(job_id, job->job_id); EXPECT_FALSE(job->finished); @@ -137,29 +155,135 @@ TEST(DispatcherState, NamedJob) { int64 job_id = 3; int64 dataset_id = 10; DispatcherState state; - Update update; TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); - DispatcherState::NamedJobKey named_job_key("test", 1); + NamedJobKey named_job_key("test", 1); TF_EXPECT_OK(CreateNamedJob(job_id, dataset_id, named_job_key, &state)); - std::shared_ptr job; + std::shared_ptr job; TF_EXPECT_OK(state.NamedJobByKey(named_job_key, &job)); + EXPECT_EQ(state.NextAvailableJobId(), job_id + 1); EXPECT_EQ(dataset_id, job->dataset_id); EXPECT_EQ(job_id, job->job_id); EXPECT_FALSE(job->finished); } -TEST(DispatcherState, FinishJob) { +TEST(DispatcherState, CreateTask) { int64 job_id = 3; int64 dataset_id = 10; + int64 task_id = 8; + std::string worker_address = "test_worker_address"; DispatcherState state; - Update update; TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); - TF_EXPECT_OK(FinishJob(job_id, &state)); - std::shared_ptr job; + TF_EXPECT_OK(CreateTask(task_id, job_id, dataset_id, worker_address, &state)); + EXPECT_EQ(state.NextAvailableTaskId(), task_id + 1); + { + std::shared_ptr task; + TF_EXPECT_OK(state.TaskFromId(task_id, &task)); + EXPECT_EQ(task_id, task->task_id); + EXPECT_EQ(job_id, task->job_id); + EXPECT_EQ(dataset_id, task->dataset_id); + EXPECT_EQ(worker_address, task->worker_address); + } + { + std::vector> tasks; + TF_EXPECT_OK(state.TasksForJob(job_id, &tasks)); + EXPECT_THAT(tasks, SizeIs(1)); + } +} + +TEST(DispatcherState, CreateTasksForSameJob) { + int64 job_id = 3; + int64 dataset_id = 10; + int64 task_id_1 = 8; + int64 task_id_2 = 9; + std::string worker_address = "test_worker_address"; + DispatcherState state; + TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); + TF_EXPECT_OK( + CreateTask(task_id_1, job_id, dataset_id, worker_address, &state)); + TF_EXPECT_OK( + CreateTask(task_id_2, job_id, dataset_id, worker_address, &state)); + { + std::vector> tasks; + TF_EXPECT_OK(state.TasksForJob(job_id, &tasks)); + EXPECT_EQ(2, tasks.size()); + } +} + +TEST(DispatcherState, CreateTasksForDifferentJobs) { + int64 job_id_1 = 3; + int64 job_id_2 = 4; + int64 dataset_id = 10; + int64 task_id_1 = 8; + int64 task_id_2 = 9; + std::string worker_address = "test_worker_address"; + DispatcherState state; + TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(CreateAnonymousJob(job_id_1, dataset_id, &state)); + TF_EXPECT_OK(CreateAnonymousJob(job_id_2, dataset_id, &state)); + TF_EXPECT_OK( + CreateTask(task_id_1, job_id_1, dataset_id, worker_address, &state)); + TF_EXPECT_OK( + CreateTask(task_id_2, job_id_2, dataset_id, worker_address, &state)); + { + std::vector> tasks; + TF_EXPECT_OK(state.TasksForJob(job_id_1, &tasks)); + EXPECT_EQ(1, tasks.size()); + } + { + std::vector> tasks; + TF_EXPECT_OK(state.TasksForJob(job_id_2, &tasks)); + EXPECT_EQ(1, tasks.size()); + } +} + +TEST(DispatcherState, FinishTask) { + int64 job_id = 3; + int64 dataset_id = 10; + int64 task_id = 4; + std::string worker_address = "test_worker_address"; + DispatcherState state; + TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); + TF_EXPECT_OK(CreateTask(task_id, job_id, dataset_id, worker_address, &state)); + TF_EXPECT_OK(FinishTask(task_id, &state)); + std::shared_ptr task; + TF_EXPECT_OK(state.TaskFromId(task_id, &task)); + EXPECT_TRUE(task->finished); + std::shared_ptr job; TF_EXPECT_OK(state.JobFromId(job_id, &job)); EXPECT_TRUE(job->finished); } +TEST(DispatcherState, FinishMultiTaskJob) { + int64 job_id = 3; + int64 dataset_id = 10; + int64 task_id_1 = 4; + int64 task_id_2 = 5; + std::string worker_address = "test_worker_address"; + DispatcherState state; + TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); + TF_EXPECT_OK( + CreateTask(task_id_1, job_id, dataset_id, worker_address, &state)); + TF_EXPECT_OK( + CreateTask(task_id_2, job_id, dataset_id, worker_address, &state)); + + TF_EXPECT_OK(FinishTask(task_id_1, &state)); + { + std::shared_ptr job; + TF_EXPECT_OK(state.JobFromId(job_id, &job)); + EXPECT_FALSE(job->finished); + } + + TF_EXPECT_OK(FinishTask(task_id_2, &state)); + { + std::shared_ptr job; + TF_EXPECT_OK(state.JobFromId(job_id, &job)); + EXPECT_TRUE(job->finished); + } +} + } // namespace data } // namespace tensorflow diff --git a/tensorflow/core/data/service/journal.proto b/tensorflow/core/data/service/journal.proto index 944b77b87f1..fd4c5863ca9 100644 --- a/tensorflow/core/data/service/journal.proto +++ b/tensorflow/core/data/service/journal.proto @@ -11,7 +11,8 @@ message Update { oneof update_type { RegisterDatasetUpdate register_dataset = 1; CreateJobUpdate create_job = 2; - FinishJobUpdate finish_job = 3; + CreateTaskUpdate create_task = 3; + FinishTaskUpdate finish_task = 4; } } @@ -34,6 +35,13 @@ message CreateJobUpdate { NamedJobKeyDef named_job_key = 4; } -message FinishJobUpdate { - int64 job_id = 1; +message CreateTaskUpdate { + int64 task_id = 1; + int64 job_id = 2; + int64 dataset_id = 3; + string worker_address = 4; +} + +message FinishTaskUpdate { + int64 task_id = 1; } diff --git a/tensorflow/core/data/service/journal_test.cc b/tensorflow/core/data/service/journal_test.cc index 3c43cf763e9..169e58ed048 100644 --- a/tensorflow/core/data/service/journal_test.cc +++ b/tensorflow/core/data/service/journal_test.cc @@ -46,10 +46,10 @@ Update MakeCreateJobUpdate() { return update; } -Update MakeFinishJobUpdate() { +Update MakeFinishTaskUpdate() { Update update; - FinishJobUpdate* finish_job = update.mutable_finish_job(); - finish_job->set_job_id(8); + FinishTaskUpdate* finish_task = update.mutable_finish_task(); + finish_task->set_task_id(8); return update; } @@ -86,7 +86,7 @@ TEST(Journal, RoundTripMultiple) { EXPECT_TRUE(NewJournalDir(&journal_dir)); std::vector updates = {MakeCreateJobUpdate(), MakeRegisterDatasetUpdate(), - MakeFinishJobUpdate()}; + MakeFinishTaskUpdate()}; FileJournalWriter writer(Env::Default(), journal_dir); for (const auto& update : updates) { TF_EXPECT_OK(writer.Write(update)); @@ -100,7 +100,7 @@ TEST(Journal, AppendExistingFile) { EXPECT_TRUE(NewJournalDir(&journal_dir)); std::vector updates = {MakeCreateJobUpdate(), MakeRegisterDatasetUpdate(), - MakeFinishJobUpdate()}; + MakeFinishTaskUpdate()}; for (const auto& update : updates) { FileJournalWriter writer(Env::Default(), journal_dir); TF_EXPECT_OK(writer.Write(update)); diff --git a/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc b/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc index 0b4e8cbbbae..8e1713e2d77 100644 --- a/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc @@ -195,6 +195,7 @@ class DataServiceDatasetOp::Dataset : public DatasetBase { void CancelThreads() TF_LOCKS_EXCLUDED(mu_) { mutex_lock l(mu_); + VLOG(1) << "Cancelling threads in DataServiceDataset::Iterator"; cancelled_ = true; worker_thread_cv_.notify_all(); manager_thread_cv_.notify_all(); @@ -295,7 +296,9 @@ class DataServiceDatasetOp::Dataset : public DatasetBase { // TODO(aaudibert): Instead of polling, have dispatcher send updates when // the list of tasks changes. void TaskThreadManager(std::unique_ptr ctx) { - VLOG(3) << "Starting task thread manager"; + auto cleanup = + gtl::MakeCleanup([] { VLOG(1) << "Task thread manager exiting"; }); + VLOG(1) << "Starting task thread manager"; DataServiceDispatcherClient dispatcher(dataset()->address_, dataset()->protocol_); uint64 next_check = Env::Default()->NowMicros(); @@ -396,8 +399,11 @@ class DataServiceDatasetOp::Dataset : public DatasetBase { } void RunWorkerThread(std::function done) { - auto cleanup = gtl::MakeCleanup([done = std::move(done)]() { done(); }); - VLOG(3) << "Starting worker thread"; + auto cleanup = gtl::MakeCleanup([done = std::move(done)]() { + done(); + VLOG(1) << "Worker thread exiting"; + }); + VLOG(1) << "Starting worker thread"; std::shared_ptr task_to_process; while (true) { { From 0f142c8ae150cef022f9fea0d3185c886db0ba02 Mon Sep 17 00:00:00 2001 From: Ruoxin Sang Date: Tue, 4 Aug 2020 16:09:51 -0700 Subject: [PATCH 0408/1017] Add a sync point before DistributedDataset `__iter__` method returns. This is to avoid cases if users are writing code as below: ``` iterator = iter(dist_dataset) outputs = multi_device_function(iterator) ``` In async eager, function and eager ops go into different execution queues. The iterator may not finish initialization when the multi device function is called. PiperOrigin-RevId: 324911136 Change-Id: I0833756c5e48775a7abee6968dce41549beb9669 --- tensorflow/python/distribute/input_lib.py | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tensorflow/python/distribute/input_lib.py b/tensorflow/python/distribute/input_lib.py index 6484cf14e42..b77739c1274 100644 --- a/tensorflow/python/distribute/input_lib.py +++ b/tensorflow/python/distribute/input_lib.py @@ -1032,6 +1032,13 @@ class DistributedDataset(_IterableInput): iterator = DistributedIterator(self._input_workers, worker_iterators, self._strategy) iterator._element_spec = self.element_spec # pylint: disable=protected-access + + # When async eager is enabled, sometimes the iterator may not finish + # initialization before passing to a multi device function, add a sync point + # here to make sure all underlying iterators are initialized. + if context.executing_eagerly(): + context.async_wait() + return iterator @property @@ -1106,6 +1113,13 @@ class DistributedDatasetV1(DistributedDataset): iterator = DistributedIteratorV1(self._input_workers, worker_iterators, self._strategy) iterator._element_spec = self.element_spec # pylint: disable=protected-access + + # When async eager is enabled, sometimes the iterator may not finish + # initialization before passing to a multi device function, add a sync point + # here to make sure all underlying iterators are initialized. + if context.executing_eagerly(): + context.async_wait() + return iterator def __iter__(self): @@ -1173,6 +1187,13 @@ class DistributedDatasetsFromFunction(_IterableInput): iterator = DistributedIterator(self._input_workers, iterators, self._strategy) iterator._element_spec = self._element_spec # pylint: disable=protected-access + + # When async eager is enabled, sometimes the iterator may not finish + # initialization before passing to a multi device function, add a sync + # point here to make sure all underlying iterators are initialized. + if context.executing_eagerly(): + context.async_wait() + return iterator raise RuntimeError("__iter__() is only supported inside of tf.function " @@ -1213,6 +1234,13 @@ class DistributedDatasetsFromFunctionV1(DistributedDatasetsFromFunction): iterator = DistributedIteratorV1(self._input_workers, iterators, self._strategy) iterator._element_spec = self._element_spec # pylint: disable=protected-access + + # When async eager is enabled, sometimes the iterator may not finish + # initialization before passing to a multi device function, add a sync point + # here to make sure all underlying iterators are initialized. + if context.executing_eagerly(): + context.async_wait() + return iterator def __iter__(self): From 460115529023a3b8a2b0ed743c152f0467f2daa1 Mon Sep 17 00:00:00 2001 From: Eugene Zhulenev Date: Tue, 4 Aug 2020 16:26:17 -0700 Subject: [PATCH 0409/1017] [MLIR:TF] Hoist cwise binary op out of concat PiperOrigin-RevId: 324914057 Change-Id: I4e007a3dfc64c6182920c1bea4dd6b217f4c1866 --- .../mlir/tensorflow/ir/tf_generated_ops.td | 10 +- .../compiler/mlir/tensorflow/ir/tf_op_base.td | 4 + .../compiler/mlir/tensorflow/ir/tf_ops_a_m.cc | 153 ++++++++++++++++++ .../compiler/mlir/tensorflow/ir/tf_traits.h | 5 + .../mlir/tensorflow/tests/canonicalize.mlir | 37 +++++ 5 files changed, 205 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index 84f3fa9c463..bba468acddb 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -136,7 +136,7 @@ Inputs must be of same size and shape. let hasFolder = 1; } -def TF_AddV2Op : TF_Op<"AddV2", [Commutative, NoSideEffect, ResultsBroadcastableShape, TF_LayoutAgnostic, TF_SameOperandsAndResultElementTypeResolveRef]>, +def TF_AddV2Op : TF_Op<"AddV2", [Commutative, NoSideEffect, ResultsBroadcastableShape, TF_LayoutAgnostic, TF_SameOperandsAndResultElementTypeResolveRef, TF_CwiseBinary]>, WithBroadcastableBinOpBuilder { let summary = "Returns x + y element-wise."; @@ -1711,6 +1711,8 @@ def TF_ConcatV2Op : TF_Op<"ConcatV2", [NoSideEffect]> { let verifier = [{ return Verify(*this); }]; + + let hasCanonicalizer = 1; } def TF_ConjOp : TF_Op<"Conj", [NoSideEffect, SameOperandsAndResultType]> { @@ -6070,7 +6072,7 @@ the result here is consistent with a truncating divide. E.g. TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<0>; } -def TF_MulOp : TF_Op<"Mul", [Commutative, NoSideEffect, ResultsBroadcastableShape, TF_SameOperandsAndResultElementTypeResolveRef]>, +def TF_MulOp : TF_Op<"Mul", [Commutative, NoSideEffect, ResultsBroadcastableShape, TF_SameOperandsAndResultElementTypeResolveRef, TF_CwiseBinary]>, WithBroadcastableBinOpBuilder { let summary = "Returns x * y element-wise."; @@ -7338,7 +7340,7 @@ tf.real(input) ==> [-2.25, 3.25] TF_DerivedResultTypeAttr Tout = TF_DerivedResultTypeAttr<0>; } -def TF_RealDivOp : TF_Op<"RealDiv", [NoSideEffect, ResultsBroadcastableShape]>, +def TF_RealDivOp : TF_Op<"RealDiv", [NoSideEffect, ResultsBroadcastableShape, TF_CwiseBinary]>, WithBroadcastableBinOpBuilder { let summary = "Returns x / y element-wise for real types."; @@ -9887,7 +9889,7 @@ Examples: TF_DerivedOperandSizeAttr N = TF_DerivedOperandSizeAttr<0>; } -def TF_SubOp : TF_Op<"Sub", [NoSideEffect, ResultsBroadcastableShape, TF_SameOperandsAndResultElementTypeResolveRef]>, +def TF_SubOp : TF_Op<"Sub", [NoSideEffect, ResultsBroadcastableShape, TF_SameOperandsAndResultElementTypeResolveRef, TF_CwiseBinary]>, WithBroadcastableBinOpBuilder { let summary = "Returns x - y element-wise."; diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td index 544cfb8af64..81a0e1bd1a5 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td @@ -73,6 +73,10 @@ def TF_LayoutAgnostic : NativeOpTrait<"TF::LayoutAgnostic">; // certain state around within their implementations. def TF_CannotDuplicate : NativeOpTrait<"TF::CannotDuplicate">; +// Coefficient wise binary operation with implicit broadcasting support, for +// example tf.Sub operation. +def TF_CwiseBinary : NativeOpTrait<"TF::CwiseBinary">; + // Variant of broadcastable trait that considers TF's subtype behavior. class TF_OpIsBroadcastableToRes : And<[ TCOpResIsShapedTypePred, diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc index 485e4fa5315..2ed44fd3fc7 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc @@ -512,6 +512,159 @@ void ConcatOp::getCanonicalizationPatterns(OwningRewritePatternList &results, results.insert(context); } +namespace { + +// Hoist coefficient-wise binary operation out of the Concat op: +// +// %0 = tf.Mul(%lhs_0, %rhs_0) +// %1 = tf.Mul(%lhs_1, %rhs_1) +// ... +// %n = tf.Mul(%lhs_n, %rhs_n) +// %m = tf.ConcatV2(%0, %1, ..., %n, %axis) +// +// Rewrite it to: +// +// %0 = tf.ConcatV2(%lhs0, %lhs1, ..., %lhs_n, %lhs_concat_axis) +// %1 = tf.ConcatV2(%rhs0, %rhs1, ..., %rhs_n, %rhs_concat_axis) +// %2 = tf.Mul(%0, %1) +// +// Because coefficient-wise binary operations support implicit broadcasting, we +// should be very careful with this optimization, and do not accidentally +// produce incorrect concat operations. +class HoistCwiseBinaryOutOfConcat : public OpRewritePattern { + public: + explicit HoistCwiseBinaryOutOfConcat(MLIRContext *context) + : OpRewritePattern(context) {} + LogicalResult matchAndRewrite(TF::ConcatV2Op op, + PatternRewriter &rewriter) const override; + + private: + struct HoistParams { + SmallVector lhs_args; + SmallVector rhs_args; + int64_t lhs_axis; + int64_t rhs_axis; + Type lhs_concat_type; + Type rhs_concat_type; + }; + + // Returns parameters of a binary op hoisting out of concatenation if all of + // the operands are in one of the compatible configurations. + Optional GetHoistParams(TF::ConcatV2Op op, int64_t axis) const; +}; + +LogicalResult HoistCwiseBinaryOutOfConcat::matchAndRewrite( + TF::ConcatV2Op op, PatternRewriter &rewriter) const { + auto loc = op.getLoc(); + + // Axis must be a constant scalar value. + DenseIntElementsAttr axis_attr; + if (!matchPattern(op.axis(), m_Constant(&axis_attr))) return failure(); + if (axis_attr.getNumElements() != 1) return failure(); + int64_t axis = + axis_attr.getSplatValue().getValue().getSExtValue(); + + // All concat operands must be defined by ops. + Operation *first_arg_op = op.values().front().getDefiningOp(); + if (first_arg_op == nullptr) return failure(); + + // All concat operands must be produced by the coeff-wise binary operation. + if (!first_arg_op->hasTrait()) return failure(); + + // All concat operands must be defined by the op of same kind. + bool args_same_op = llvm::all_of(op.values(), [&](Value arg) -> bool { + Operation *arg_op = arg.getDefiningOp(); + return arg_op && arg_op->getName() == first_arg_op->getName(); + }); + if (!args_same_op) return failure(); + + // Compute binary operands hoist parameters. + auto hoist_params = GetHoistParams(op, axis); + if (!hoist_params.hasValue()) return failure(); + + // New lhs and rhs concatenation axis. + auto axis_type = mlir::RankedTensorType::get({}, rewriter.getIntegerType(64)); + auto lhs_axis = rewriter.create( + loc, DenseIntElementsAttr::get(axis_type, hoist_params->lhs_axis)); + auto rhs_axis = rewriter.create( + loc, DenseIntElementsAttr::get(axis_type, hoist_params->rhs_axis)); + + // Concatenate binary ops operands on the new axis. + auto lhs_concat = rewriter.create( + loc, hoist_params->lhs_concat_type, hoist_params->lhs_args, lhs_axis); + auto rhs_concat = rewriter.create( + loc, hoist_params->rhs_concat_type, hoist_params->rhs_args, rhs_axis); + + // Replace original concat with a binary op. + OperationState new_binary_op_state( + loc, first_arg_op->getName().getStringRef(), + {lhs_concat.getResult(), rhs_concat.getResult()}, + op.getResult().getType(), ArrayRef()); + Operation *new_binary_op = rewriter.createOperation(new_binary_op_state); + + rewriter.replaceOp(op, new_binary_op->getResults()); + + return success(); +} + +Optional +HoistCwiseBinaryOutOfConcat::GetHoistParams(TF::ConcatV2Op op, + int64_t axis) const { + // Collects lhs or rhs arguments of concat op operands. + auto args = [&](int operand_idx) -> SmallVector { + auto range = llvm::map_range(op.values(), [&](Value arg) { + return arg.getDefiningOp()->getOperand(operand_idx); + }); + return {range.begin(), range.end()}; + }; + + // Returns true if all binary ops operands at `operand_idx` index are tensors + // of `axis + 1` rank and axis dim has size `1`. + auto is_all_tensors = [&](int operand_idx, int axis) -> bool { + return llvm::all_of(op.values(), [&](Value arg) -> bool { + auto lhs = arg.getDefiningOp()->getOperand(operand_idx); + auto ranked = lhs.getType().dyn_cast(); + return ranked && ranked.getRank() == (axis + 1) && + ranked.getShape()[axis] == 1; + }); + }; + + // Returns true if all binary ops operands at `operand_idx` index are scalars. + auto is_all_scalars = [&](int operand_idx) -> bool { + return llvm::all_of(op.values(), [&](Value arg) -> bool { + auto lhs = arg.getDefiningOp()->getOperand(operand_idx); + auto ranked = lhs.getType().dyn_cast(); + return ranked && ranked.hasRank() && ranked.getRank() == 0; + }); + }; + + auto ranked = op.getType().cast(); + if (!ranked) return None; + + // TODO(ezhulenev): Add support for more valid concat patterns. + + // Tensor + Scalar: [..., 1] + [] <- scalar + // ^ + // \- axis is the innermost dimension. + // + // Concatenate tensor arguments on the same axis as the original operation, + // and concatenate scalars into the vector. + if (is_all_tensors(0, axis) && is_all_scalars(1)) { + std::array rhs_dims{static_cast(op.values().size())}; + auto rhs_type = RankedTensorType::get(rhs_dims, ranked.getElementType()); + return HoistParams{args(0), args(1), axis, 0, op.getType(), rhs_type}; + } + + return None; +} + +} // namespace + +void ConcatV2Op::getCanonicalizationPatterns(OwningRewritePatternList &results, + MLIRContext *context) { + results.insert(context); +} + //===----------------------------------------------------------------------===// // ConcatOffsetOp //===----------------------------------------------------------------------===// diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h index b9a781b99e7..6cf2df60a3f 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h @@ -124,6 +124,11 @@ class CannotDuplicate : public TraitBase { } }; +// Coefficient wise binary operation with implicit broadcasting support, for +// example tf.Sub operation. +template +class CwiseBinary : public TraitBase {}; + } // namespace TF } // namespace OpTrait } // namespace mlir diff --git a/tensorflow/compiler/mlir/tensorflow/tests/canonicalize.mlir b/tensorflow/compiler/mlir/tensorflow/tests/canonicalize.mlir index 007c123a034..5808b03c909 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/canonicalize.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/canonicalize.mlir @@ -143,6 +143,43 @@ func @testConcatCanonicalization(%arg0: tensor<2x1xi32>, %arg1: tensor<2x1xi32>) return %1 : tensor<2x2xi32> } +// CHECK-LABEL: testConcatCwiseBinaryOnInnerDim +func @testConcatCwiseBinaryOnInnerDim(%arg0: tensor, + %arg1: tensor, %arg2: tensor, %arg3: tensor) -> tensor { + + // CHECK: %[[LHS_AXIS:.*]] = "tf.Const"() {value = dense<1> : tensor} + // CHECK: %[[RHS_AXIS:.*]] = "tf.Const"() {value = dense<0> : tensor} + + // CHECK: %[[LHS_CONCAT:.*]] = "tf.ConcatV2"(%arg0, %arg1, %[[LHS_AXIS]]) + // CHECK: %[[RHS_CONCAT:.*]] = "tf.ConcatV2"(%arg2, %arg3, %[[RHS_AXIS]]) + + // CHECK: %[[MUL:.*]] = "tf.Mul"(%[[LHS_CONCAT]], %[[RHS_CONCAT]]) + // CHECK-SAME: (tensor, tensor<2xf32>) -> tensor + // CHECK: return %[[MUL]] + + %0 = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %1 = "tf.Mul"(%arg0, %arg2) : (tensor, tensor) -> tensor + %2 = "tf.Mul"(%arg1, %arg3) : (tensor, tensor) -> tensor + %3 = "tf.ConcatV2"(%1, %2, %0) : (tensor, tensor, tensor) -> tensor + + return %3 : tensor +} + +// CHECK-LABEL: testConcatCwiseBinaryInvalidInnerDim +func @testConcatCwiseBinaryInvalidInnerDim(%arg0: tensor, + %arg1: tensor, %arg2: tensor, %arg3: tensor) -> tensor { + // Each individual binary operation has an implicit broadcast that will be + // lost if we would reorder them with the concat. + + // CHECK: "tf.ConcatV2"(%1, %2, %0) + %0 = "tf.Const"() { value = dense<1> : tensor } : () -> tensor + %1 = "tf.Mul"(%arg0, %arg2) : (tensor, tensor) -> tensor + %2 = "tf.Mul"(%arg1, %arg3) : (tensor, tensor) -> tensor + %3 = "tf.ConcatV2"(%1, %2, %0) : (tensor, tensor, tensor) -> tensor + + return %3 : tensor +} + // CHECK-LABEL: testLogOfSoftmax func @testLogOfSoftmax(%arg0: tensor<8x16xf32>) -> tensor<8x16xf32> { %0 = "tf.Softmax"(%arg0) : (tensor<8x16xf32>) -> tensor<8x16xf32> From 60040ffabfce4d7c89b7cb3b57528aa721e0ead2 Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Tue, 4 Aug 2020 16:30:17 -0700 Subject: [PATCH 0410/1017] NFC: Remove unused dependencies PiperOrigin-RevId: 324914796 Change-Id: I230ad19f82048474f77170a6244fcf4e7a835ba8 --- tensorflow/lite/tools/optimize/BUILD | 8 ++++++++ tensorflow/lite/tools/optimize/calibration/BUILD | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/tensorflow/lite/tools/optimize/BUILD b/tensorflow/lite/tools/optimize/BUILD index ab153afc2cf..146f869a906 100644 --- a/tensorflow/lite/tools/optimize/BUILD +++ b/tensorflow/lite/tools/optimize/BUILD @@ -49,6 +49,7 @@ cc_binary( srcs = ["modify_model_interface_main.cc"], deps = [ ":modify_model_interface", + ":quantize_model", ], ) @@ -89,6 +90,8 @@ cc_library( hdrs = ["quantization_wrapper.h"], deps = [ ":quantization_wrapper_utils", + "//tensorflow/lite:framework", + "//tensorflow/lite/core/api", "//tensorflow/lite/schema:schema_fbs", "//tensorflow/lite/tools/optimize:quantize_model", "@flatbuffers", @@ -112,6 +115,7 @@ cc_library( "//tensorflow/lite/schema:schema_fbs", "//third_party/eigen3", "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", ], ) @@ -126,6 +130,7 @@ cc_library( "//tensorflow/lite/kernels/internal:types", "//tensorflow/lite/schema:schema_fbs", "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", ], ) @@ -154,6 +159,7 @@ cc_library( hdrs = ["operator_property.h"], deps = [ "//tensorflow/lite:framework", + "//tensorflow/lite/kernels/internal:types", "//tensorflow/lite/schema:schema_fbs", ], ) @@ -194,6 +200,7 @@ cc_library( ":quantization_utils", ":model_utils", "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", "@com_google_absl//absl/container:flat_hash_map", "@flatbuffers", "//tensorflow/lite:framework", @@ -238,6 +245,7 @@ cc_library( srcs = ["test_util.cc"], hdrs = ["test_util.h"], deps = [ + "//tensorflow/lite:framework", "//tensorflow/lite/core/api", "@com_google_googletest//:gtest", "@flatbuffers", diff --git a/tensorflow/lite/tools/optimize/calibration/BUILD b/tensorflow/lite/tools/optimize/calibration/BUILD index f641b151aa9..06183353e44 100644 --- a/tensorflow/lite/tools/optimize/calibration/BUILD +++ b/tensorflow/lite/tools/optimize/calibration/BUILD @@ -51,6 +51,7 @@ cc_library( "//tensorflow/lite/schema:schema_fbs", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", "@flatbuffers", ], ) @@ -104,6 +105,7 @@ cc_test( deps = [ ":logging_op_resolver", "//tensorflow/lite:framework", + "//tensorflow/lite/kernels:builtin_ops", "@com_google_googletest//:gtest", ], ) @@ -118,6 +120,7 @@ cc_library( "//tensorflow/lite:framework", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", ], ) @@ -127,6 +130,7 @@ cc_library( hdrs = ["calibration_logger.h"], copts = tflite_copts(), deps = [ + "//tensorflow/lite:framework", "//tensorflow/lite:minimal_logging", "//tensorflow/lite/c:common", "//tensorflow/lite/core/api", From 5a0d95f7eb6476e015d9dff644567076325f242e Mon Sep 17 00:00:00 2001 From: Eugene Zhulenev Date: Tue, 4 Aug 2020 16:33:13 -0700 Subject: [PATCH 0411/1017] [MLIR:TF] Hoist cwise unary op out of concat PiperOrigin-RevId: 324915363 Change-Id: I3537157af5762054a068f4aadaa6dd9761b8204d --- .../mlir/tensorflow/ir/tf_generated_ops.td | 2 +- .../compiler/mlir/tensorflow/ir/tf_op_base.td | 3 + .../compiler/mlir/tensorflow/ir/tf_ops_a_m.cc | 63 ++++++++++++++++++- .../compiler/mlir/tensorflow/ir/tf_traits.h | 6 +- .../mlir/tensorflow/tests/canonicalize.mlir | 13 ++++ 5 files changed, 84 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index bba468acddb..081903d13cf 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -4799,7 +4799,7 @@ tf.math.log(x) ==> [-inf, -0.6931472, 0. , 1.609438] let hasCanonicalizer = 1; } -def TF_Log1pOp : TF_Op<"Log1p", [NoSideEffect, SameOperandsAndResultType]> { +def TF_Log1pOp : TF_Op<"Log1p", [NoSideEffect, SameOperandsAndResultType, TF_CwiseUnary]> { let summary = "Computes natural logarithm of (1 + x) element-wise."; let description = [{ diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td index 81a0e1bd1a5..1755c975c23 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_op_base.td @@ -77,6 +77,9 @@ def TF_CannotDuplicate : NativeOpTrait<"TF::CannotDuplicate">; // example tf.Sub operation. def TF_CwiseBinary : NativeOpTrait<"TF::CwiseBinary">; +// Coefficient wise unary operation, for example tf.Sqrt operation. +def TF_CwiseUnary : NativeOpTrait<"TF::CwiseUnary">; + // Variant of broadcastable trait that considers TF's subtype behavior. class TF_OpIsBroadcastableToRes : And<[ TCOpResIsShapedTypePred, diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc index 2ed44fd3fc7..791323ca992 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc @@ -514,6 +514,66 @@ void ConcatOp::getCanonicalizationPatterns(OwningRewritePatternList &results, namespace { +// Hoist coefficient-wise unary operation out of the Concat op: +// +// %0 = "tf.Log1p"(%arg_0) +// %1 = "tf.Log1p"(%arg_1) +// ... +// %n = "tf.Log1p"(%arg_n) +// %m = "tf.ConcatV2"(%0, %1, ..., %n, %axis) +// +// Rewrite it to: +// +// %0 = "tf.ConcatV2"(%arg_0, %arg_1, ..., %arg_n, %axis) +// %1 = "tf.Log1p"(%0) +class HoistCwiseUnaryOutOfConcat : public OpRewritePattern { + public: + explicit HoistCwiseUnaryOutOfConcat(MLIRContext *context) + : OpRewritePattern(context) {} + LogicalResult matchAndRewrite(TF::ConcatV2Op op, + PatternRewriter &rewriter) const override; +}; + +LogicalResult HoistCwiseUnaryOutOfConcat::matchAndRewrite( + TF::ConcatV2Op op, PatternRewriter &rewriter) const { + auto loc = op.getLoc(); + + // All concat operands must be defined by ops. + Operation *first_arg_op = op.values().front().getDefiningOp(); + if (first_arg_op == nullptr) return failure(); + + // All concat operands must be produced by the coeff-wise unary operation. + if (!first_arg_op->hasTrait()) return failure(); + + // All concat operands must be defined by the op of same kind. + bool args_same_op = llvm::all_of(op.values(), [&](Value arg) -> bool { + Operation *arg_op = arg.getDefiningOp(); + return arg_op && arg_op->getName() == first_arg_op->getName(); + }); + if (!args_same_op) return failure(); + + // Collect unary operations operands. + auto unary_operands = llvm::map_range(op.values(), [](Value arg) -> Value { + return arg.getDefiningOp()->getOperand(0); + }); + SmallVector unary_ops_args(unary_operands); + + // Concatenate unary ops operands. + auto concat_unary_operands = + rewriter.create(loc, op.getType(), unary_ops_args, op.axis()); + + // Replace original concat with an unary op. + OperationState new_unary_op_state(loc, first_arg_op->getName().getStringRef(), + concat_unary_operands.getResult(), + op.getResult().getType(), + ArrayRef()); + Operation *new_unary_op = rewriter.createOperation(new_unary_op_state); + + rewriter.replaceOp(op, new_unary_op->getResults()); + + return success(); +} + // Hoist coefficient-wise binary operation out of the Concat op: // // %0 = tf.Mul(%lhs_0, %rhs_0) @@ -662,7 +722,8 @@ HoistCwiseBinaryOutOfConcat::GetHoistParams(TF::ConcatV2Op op, void ConcatV2Op::getCanonicalizationPatterns(OwningRewritePatternList &results, MLIRContext *context) { - results.insert(context); + results.insert( + context); } //===----------------------------------------------------------------------===// diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h index 6cf2df60a3f..fc8e6f40f65 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_traits.h @@ -124,11 +124,15 @@ class CannotDuplicate : public TraitBase { } }; -// Coefficient wise binary operation with implicit broadcasting support, for +// Coefficient-wise binary operation with implicit broadcasting support, for // example tf.Sub operation. template class CwiseBinary : public TraitBase {}; +// Coefficient-wise unary operation, for example tf.Sqrt operation. +template +class CwiseUnary : public TraitBase {}; + } // namespace TF } // namespace OpTrait } // namespace mlir diff --git a/tensorflow/compiler/mlir/tensorflow/tests/canonicalize.mlir b/tensorflow/compiler/mlir/tensorflow/tests/canonicalize.mlir index 5808b03c909..595bdce5be4 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/canonicalize.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/canonicalize.mlir @@ -143,6 +143,19 @@ func @testConcatCanonicalization(%arg0: tensor<2x1xi32>, %arg1: tensor<2x1xi32>) return %1 : tensor<2x2xi32> } +// CHECK-LABEL: testConcatCwiseUnary +func @testConcatCwiseUnary(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + + // CHECK: %[[CONCAT:.*]] = "tf.ConcatV2"(%arg0, %arg1, %arg2) + // CHECK: %[[LOG1P:.*]] = "tf.Log1p"(%[[CONCAT]]) + // CHECK: return %[[LOG1P]] + %0 = "tf.Log1p"(%arg0) : (tensor) -> tensor + %1 = "tf.Log1p"(%arg1) : (tensor) -> tensor + %2 = "tf.ConcatV2"(%0, %1, %arg2) : (tensor, tensor, tensor) -> tensor + + return %2 : tensor +} + // CHECK-LABEL: testConcatCwiseBinaryOnInnerDim func @testConcatCwiseBinaryOnInnerDim(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor) -> tensor { From 53576063848800b6ee906c6e94d840fbdbbecd1b Mon Sep 17 00:00:00 2001 From: Thomas O'Malley Date: Tue, 4 Aug 2020 16:33:15 -0700 Subject: [PATCH 0412/1017] Enable clipnorm and clipvalue arguments in Optimizer with tf.distribute.Strategy. Apply gradient clipping after aggregation. CentralStorageStrategy is still not supported with these arguments. PiperOrigin-RevId: 324915370 Change-Id: Ib9b41511b5b9b77ec95ff9543b9aa68e4ed6b4d8 --- RELEASE.md | 6 + .../distribute/distribute_strategy_test.py | 31 +++++ tensorflow/python/keras/engine/training.py | 1 - .../python/keras/engine/training_eager.py | 1 - .../experimental/loss_scale_optimizer.py | 4 +- tensorflow/python/keras/optimizer_v2/BUILD | 1 + .../python/keras/optimizer_v2/optimizer_v2.py | 108 +++++++++--------- tensorflow/python/keras/optimizer_v2/utils.py | 38 ++++++ ...n.experimental.-loss-scale-optimizer.pbtxt | 8 ++ ...ensorflow.keras.optimizers.-adadelta.pbtxt | 8 ++ ...tensorflow.keras.optimizers.-adagrad.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-adam.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-adamax.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-ftrl.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-nadam.pbtxt | 8 ++ ...nsorflow.keras.optimizers.-optimizer.pbtxt | 8 ++ ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-s-g-d.pbtxt | 8 ++ ...n.experimental.-loss-scale-optimizer.pbtxt | 8 ++ ...ensorflow.keras.optimizers.-adadelta.pbtxt | 8 ++ ...tensorflow.keras.optimizers.-adagrad.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-adam.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-adamax.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-ftrl.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-nadam.pbtxt | 8 ++ ...nsorflow.keras.optimizers.-optimizer.pbtxt | 8 ++ ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt | 8 ++ .../tensorflow.keras.optimizers.-s-g-d.pbtxt | 8 ++ .../v2/tensorflow.optimizers.-adadelta.pbtxt | 8 ++ .../v2/tensorflow.optimizers.-adagrad.pbtxt | 8 ++ .../v2/tensorflow.optimizers.-adam.pbtxt | 8 ++ .../v2/tensorflow.optimizers.-adamax.pbtxt | 8 ++ .../v2/tensorflow.optimizers.-ftrl.pbtxt | 8 ++ .../v2/tensorflow.optimizers.-nadam.pbtxt | 8 ++ .../v2/tensorflow.optimizers.-optimizer.pbtxt | 8 ++ .../v2/tensorflow.optimizers.-r-m-sprop.pbtxt | 8 ++ .../v2/tensorflow.optimizers.-s-g-d.pbtxt | 8 ++ 37 files changed, 365 insertions(+), 57 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index b0c785c7d68..d7a345c7c76 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -33,6 +33,10 @@ shape assumptions (note that you can pass shapes with `None` entries for axes that are meant to be dynamic). You can also disable the input checking entirely by setting `model.input_spec = None`. +* `tf.keras.optimizers.Optimizer.get_gradients` no longer performs gradient + clipping. Instead, gradient clipping is performed in + `tf.keras.optimizers.Optimizer.apply_gradients`, after the gradients on each + device have been aggregated. ## Known Caveats @@ -95,6 +99,8 @@ * Error messages when Functional API construction goes wrong (and when ops cannot be converted to Keras layers automatically) should be clearer and easier to understand. * `Optimizer.minimize` can now accept a loss `Tensor` and a `GradientTape` as an alternative to accepting a `callable` loss. + * `Optimizer` arguments `clipnorm` and `clipvalue` are now supported with + `tf.distribute.Strategy` (`CentralStorageStrategy` is not yet supported). * `tf.function` / AutoGraph: * Added `experimental_follow_type_hints` argument for `tf.function`. When True, the function may use type annotations to optimize the tracing diff --git a/tensorflow/python/keras/distribute/distribute_strategy_test.py b/tensorflow/python/keras/distribute/distribute_strategy_test.py index 4b6d3a80730..abcb5d1c0e8 100644 --- a/tensorflow/python/keras/distribute/distribute_strategy_test.py +++ b/tensorflow/python/keras/distribute/distribute_strategy_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python import keras from tensorflow.python.data.experimental.ops import cardinality from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.distribute import central_storage_strategy from tensorflow.python.distribute import combinations from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import mirrored_strategy @@ -1863,6 +1864,36 @@ class TestDistributionStrategyWithKerasModels(test.TestCase, self.assertEqual(bc.predict_begin_batches, [0]) self.assertEqual(bc.predict_end_batches, [24]) + @combinations.generate( + combinations.combine(distribution=all_strategies, mode=['eager'])) + def test_gradient_clipping(self, distribution): + + class MyLayer(keras.layers.Layer): + + def build(self, _): + self.v1 = variables.Variable(1.) + self.v2 = variables.Variable(1.) + + def call(self, x): + return 3 * self.v1 - 3 * self.v2 + + x, y = np.ones((10, 1)), np.ones((10, 1)) + + with distribution.scope(): + layer = MyLayer() + model = keras.Sequential([layer]) + optimizer = gradient_descent_keras.SGD(1., clipnorm=2., clipvalue=2.) + model.compile(optimizer, 'mae') + + if isinstance(distribution, + central_storage_strategy.CentralStorageStrategy): + with self.assertRaisesRegex(ValueError, 'not supported'): + model.fit(x, y, batch_size=10, epochs=1) + else: + model.fit(x, y, batch_size=10, epochs=1) + self.assertAllClose(self.evaluate(layer.v1), 3.) + self.assertAllClose(self.evaluate(layer.v2), -1.) + @combinations.generate( combinations.times( all_strategy_combinations_minus_default())) diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index bf542129e5c..a1fb329feab 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -2744,7 +2744,6 @@ def _minimize(strategy, tape, optimizer, loss, trainable_variables): trainable_variables)) if isinstance(optimizer, lso.LossScaleOptimizer): gradients = optimizer.get_unscaled_gradients(gradients) - gradients = optimizer._clip_gradients(gradients) # pylint: disable=protected-access if trainable_variables: if aggregate_grads_outside_optimizer: optimizer.apply_gradients( diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py index 8064bf2a7ab..b3ce3d13ed7 100644 --- a/tensorflow/python/keras/engine/training_eager.py +++ b/tensorflow/python/keras/engine/training_eager.py @@ -273,7 +273,6 @@ def _process_single_batch(model, if isinstance(model.optimizer, loss_scale_optimizer.LossScaleOptimizer): grads = model.optimizer.get_unscaled_gradients(grads) - grads = model.optimizer._clip_gradients(grads) model.optimizer.apply_gradients(zip(grads, trainable_weights)) else: logging.warning('The list of trainable weights is empty. Make sure that' diff --git a/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py b/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py index 4a3f459de80..59a49b03ad5 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py +++ b/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py @@ -258,8 +258,8 @@ class LossScaleOptimizer(_DelegatingTrackableMixin, optimizer_v2.OptimizerV2): 'clipvalue %s' % (optimizer, optimizer.clipvalue)) self._raise_if_strategy_unsupported() - self.clipnorm = None - self.clipvalue = None + self._clipnorm = None + self._clipvalue = None self._optimizer = optimizer self._loss_scale = keras_loss_scale_module.get(loss_scale) diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD index b519ec7fb3d..9a317e5d114 100644 --- a/tensorflow/python/keras/optimizer_v2/BUILD +++ b/tensorflow/python/keras/optimizer_v2/BUILD @@ -40,6 +40,7 @@ py_library( "//tensorflow/python:state_ops", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/distribute:central_storage_strategy", "//tensorflow/python/distribute:distribute_lib", "//tensorflow/python/distribute:parameter_server_strategy", "//tensorflow/python/distribute:reduce_util", diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py index 18d94594542..0ecca63a64f 100644 --- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py +++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py @@ -41,7 +41,6 @@ from tensorflow.python.keras.optimizer_v2 import utils as optimizer_utils from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import tf_utils from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_resource_variable_ops from tensorflow.python.ops import gradients @@ -332,15 +331,6 @@ class OptimizerV2(trackable.Trackable): raise ValueError("decay cannot be less than 0: {}".format(decay)) self._initial_decay = decay - # Set the gradient clipping properties - self.clipnorm = kwargs.pop("clipnorm", None) - self.clipvalue = kwargs.pop("clipvalue", None) - if ((self.clipnorm is not None or self.clipvalue is not None) - and distribute_ctx.has_strategy()): - raise ValueError("Gradient clipping in the optimizer " - "(by setting clipnorm or clipvalue) is currently " - "unsupported when using a distribution strategy.") - self._hypers_created = False # Store the distribution strategy object if the optimizer is created inside @@ -350,6 +340,33 @@ class OptimizerV2(trackable.Trackable): else: self._distribution_strategy = None + # Set the gradient clipping properties + self._clipnorm = kwargs.pop("clipnorm", None) + self._clipvalue = kwargs.pop("clipvalue", None) + + # Configure gradient transforms. + self._transform_gradients_fns = [] + + if self._clipnorm is not None: + self._transform_gradients_fns.append( + optimizer_utils.make_gradient_clipnorm_fn(self._clipnorm)) + if self._clipvalue is not None: + self._transform_gradients_fns.append( + optimizer_utils.make_gradient_clipvalue_fn(self._clipvalue)) + + @property + def clipnorm(self): + """`float` or `None`. If set, clips gradients to this maximum norm.""" + return self._clipnorm + + @property + def clipvalue(self): + """`float` or `None`. + + If set, clips gradients to this maximum absolute value. + """ + return self._clipvalue + def minimize(self, loss, var_list, grad_loss=None, name=None, tape=None): """Minimize `loss` by updating `var_list`. @@ -385,26 +402,6 @@ class OptimizerV2(trackable.Trackable): loss, var_list=var_list, grad_loss=grad_loss, tape=tape) return self.apply_gradients(grads_and_vars, name=name) - def _clip_gradients(self, grads): - """Clip gradients according to the clipnorm and clipvalue attributes.""" - if self.clipnorm is not None: - if distribute_ctx.has_strategy(): - raise ValueError("Gradient clipping in the optimizer " - "(by setting clipnorm or clipvalue) is currently " - "unsupported when using a distribution strategy.") - grads = [None if g is None else clip_ops.clip_by_norm(g, self.clipnorm) - for g in grads] - if self.clipvalue is not None: - if distribute_ctx.has_strategy(): - raise ValueError("Gradient clipping in the optimizer " - "(by setting clipnorm or clipvalue) is currently " - "unsupported when using a distribution strategy.") - v = self.clipvalue - grads = [ - None if g is None else clip_ops.clip_by_value(g, -v, v) for g in grads - ] - return grads - def _compute_gradients(self, loss, var_list, grad_loss=None, tape=None): """Compute gradients of `loss` for the variables in `var_list`. @@ -454,8 +451,6 @@ class OptimizerV2(trackable.Trackable): var_list = nest.flatten(var_list) with ops.name_scope_v2(self._name + "/gradients"): grads = tape.gradient(loss, var_list, grad_loss) - # TODO(omalleyt): Move to post-aggregation. - grads = self._clip_gradients(grads) grads_and_vars = list(zip(grads, var_list)) self._assert_valid_dtypes([ @@ -465,6 +460,12 @@ class OptimizerV2(trackable.Trackable): return grads_and_vars + def _transform_gradients(self, grads_and_vars): + """Transformations to apply aggregated gradients.""" + for fn in self._transform_gradients_fns: + grads_and_vars = fn(grads_and_vars) + return grads_and_vars + def get_gradients(self, loss, params): """Returns gradients of `loss` with respect to `params`. @@ -483,14 +484,15 @@ class OptimizerV2(trackable.Trackable): with backend.get_graph().as_default(), backend.name_scope(self._name + "/gradients"): grads = gradients.gradients(loss, params) - for grad, param in zip(grads, params): + grads_and_vars = list(zip(grads, params)) + for grad, param in grads_and_vars: if grad is None: raise ValueError("Variable {} has `None` for gradient. " "Please make sure that all of your ops have a " "gradient defined (i.e. are differentiable). " "Common ops without gradient: " "K.argmax, K.round, K.eval.".format(param)) - grads = self._clip_gradients(grads) + grads = [g for g, _ in grads_and_vars] return grads def apply_gradients(self, @@ -534,10 +536,23 @@ class OptimizerV2(trackable.Trackable): ValueError: If none of the variables have gradients. RuntimeError: If called in a cross-replica context. """ - grads_and_vars = optimizer_utils.filter_empty_gradients(grads_and_vars) - var_list = [v for (_, v) in grads_and_vars] + if distribute_ctx.in_cross_replica_context(): + raise RuntimeError( + "`apply_gradients() cannot be called in cross-replica context. " + "Use `tf.distribute.Strategy.run` to enter replica " + "context.") - with backend.name_scope(self._name): + strategy = distribute_ctx.get_strategy() + if (not experimental_aggregate_gradients and strategy and + isinstance(strategy.extended, + parameter_server_strategy.ParameterServerStrategyExtended)): + raise NotImplementedError( + "`experimental_aggregate_gradients=False is not supported for " + "ParameterServerStrategy and CentralStorageStrategy") + + grads_and_vars = optimizer_utils.filter_empty_gradients(grads_and_vars) + var_list = [v for _, v in grads_and_vars] + with ops.name_scope_v2(self._name): # Create iteration if necessary. with ops.init_scope(): self._create_all_weights(var_list) @@ -547,25 +562,12 @@ class OptimizerV2(trackable.Trackable): # gradients return control_flow_ops.no_op() - if distribute_ctx.in_cross_replica_context(): - raise RuntimeError( - "`apply_gradients() cannot be called in cross-replica context. " - "Use `tf.distribute.Strategy.run` to enter replica " - "context.") - - strategy = distribute_ctx.get_strategy() - if (not experimental_aggregate_gradients and strategy and isinstance( - strategy.extended, - parameter_server_strategy.ParameterServerStrategyExtended)): - raise NotImplementedError( - "`experimental_aggregate_gradients=False is not supported for " - "ParameterServerStrategy and CentralStorageStrategy") - - apply_state = self._prepare(var_list) if experimental_aggregate_gradients: reduced_grads = self._aggregate_gradients(grads_and_vars) - var_list = [v for _, v in grads_and_vars] grads_and_vars = list(zip(reduced_grads, var_list)) + grads_and_vars = self._transform_gradients(grads_and_vars) + + apply_state = self._prepare(var_list) return distribute_ctx.get_replica_context().merge_call( functools.partial(self._distributed_apply, apply_state=apply_state), args=(grads_and_vars,), diff --git a/tensorflow/python/keras/optimizer_v2/utils.py b/tensorflow/python/keras/optimizer_v2/utils.py index 9f680e04dd6..f723c6d8b64 100644 --- a/tensorflow/python/keras/optimizer_v2/utils.py +++ b/tensorflow/python/keras/optimizer_v2/utils.py @@ -18,8 +18,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.distribute import central_storage_strategy from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx from tensorflow.python.distribute import reduce_util as ds_reduce_util +from tensorflow.python.ops import clip_ops from tensorflow.python.platform import tf_logging as logging @@ -57,6 +59,42 @@ def all_reduce_sum_gradients(grads_and_vars): return reduced_with_nones +def make_gradient_clipnorm_fn(clipnorm): + """Creates a gradient transformation function for clipping by norm.""" + + def gradient_clipnorm_fn(grads_and_vars): + + if isinstance(distribute_ctx.get_strategy(), + central_storage_strategy.CentralStorageStrategy): + raise ValueError( + "`clipnorm` is not supported with `CenteralStorageStrategy`") + + clipped_grads_and_vars = [ + (clip_ops.clip_by_norm(g, clipnorm), v) for g, v in grads_and_vars + ] + return clipped_grads_and_vars + + return gradient_clipnorm_fn + + +def make_gradient_clipvalue_fn(clipvalue): + """Creates a gradient transformation function for clipping by value.""" + + def gradient_clipvalue_fn(grads_and_vars): + + if isinstance(distribute_ctx.get_strategy(), + central_storage_strategy.CentralStorageStrategy): + raise ValueError( + "`clipvalue` is not supported with `CenteralStorageStrategy`") + + clipped_grads_and_vars = [(clip_ops.clip_by_value(g, -clipvalue, + clipvalue), v) + for g, v in grads_and_vars] + return clipped_grads_and_vars + + return gradient_clipvalue_fn + + def filter_empty_gradients(grads_and_vars): """Filter out `(grad, var)` pairs that have a gradient equal to `None`.""" grads_and_vars = tuple(grads_and_vars) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt index dbab3abae8e..58f8cf24495 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt @@ -5,6 +5,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt index af854e98013..fb341cb24dd 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt index e89cc5cef75..d8039ed21ef 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt index 15414d7234f..912f92f83a6 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt index 8b3c429e6b5..3abc6d39b3f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt index 51ab675db74..00880d3f73b 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt index 342c0951bbe..2ce311d3504 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt index f007b4b971a..2020de9fa5c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt @@ -3,6 +3,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt index d5bf6fa7f47..80a1449613c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt index df904f72511..8acfe214256 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt index dbab3abae8e..58f8cf24495 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt @@ -5,6 +5,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt index af854e98013..fb341cb24dd 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt index e89cc5cef75..d8039ed21ef 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt index 15414d7234f..912f92f83a6 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt index 8b3c429e6b5..3abc6d39b3f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt index 51ab675db74..00880d3f73b 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt index 342c0951bbe..2ce311d3504 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt index f007b4b971a..2020de9fa5c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt @@ -3,6 +3,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt index d5bf6fa7f47..80a1449613c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt index df904f72511..8acfe214256 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt index cb3d38246a7..06212bdc95d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt index c7b2bca4b6b..09fff0514d8 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt index 209c9fe6620..195ba9e4f56 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt index 12bbb14fb71..9859da430bd 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt index 1482ed54eb9..a4ed911e39d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt index 2a422fa2340..128f223fdc7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt index e7021e02772..5ea1ed521ef 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt @@ -3,6 +3,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt index 6543f4023a4..db89ecbabe7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt index 94ff8dfcdfc..0cb0205e65e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" From 2d68aaede743bac80efe8dc427f34838388e0a73 Mon Sep 17 00:00:00 2001 From: Doe Hyun Yoon Date: Tue, 4 Aug 2020 16:45:28 -0700 Subject: [PATCH 0413/1017] Add ImmutableNodeMap for const GraphDef. NodeMap and ImmutableNodeMap are subclass of NodeMapInternal. PiperOrigin-RevId: 324917672 Change-Id: I9be113ae4134934f6c8b402e5ceb0fd5e90b2e80 --- tensorflow/core/grappler/utils.cc | 30 ++++---- tensorflow/core/grappler/utils.h | 64 +++++++++++++---- tensorflow/core/grappler/utils_test.cc | 97 ++++++++++++++++++-------- 3 files changed, 134 insertions(+), 57 deletions(-) diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc index 7cf303654ed..e342f7dfdf0 100644 --- a/tensorflow/core/grappler/utils.cc +++ b/tensorflow/core/grappler/utils.cc @@ -73,25 +73,21 @@ bool IsShapeConsumer(const NodeDef& node) { } // namespace -NodeMap::NodeMap(GraphDef* graph) { - nodes_.reserve(graph->node_size()); - outputs_.reserve(graph->node_size()); - for (int i = 0; i < graph->node_size(); i++) { - NodeDef* node = graph->mutable_node(i); - const string& node_name = node->name(); - auto rslt = nodes_.emplace(node_name, node); - // Check that the graph doesn't contain multiple nodes with the same name. - if (!rslt.second) { - // The first node found with a given name becomes the canonical. - LOG(WARNING) << "Duplicated node in the graph: " << node_name; - } - NodeDef* canonical = rslt.second ? node : rslt.first->second; - for (const auto& input : node->input()) { - outputs_[NodeName(input)].insert(canonical); - } - } +namespace internal { +// Specialized template class method GetNodeDefFromGraph. +template <> +NodeDef* NodeMapInternal::GetNodeDefFromGraph( + GraphDef* graph, int64 i) const { + return graph->mutable_node(i); } +template <> +const NodeDef* +NodeMapInternal::GetNodeDefFromGraph( + const GraphDef* graph, int64 i) const { + return &graph->node(i); +} +} // namespace internal string TensorIdToString(const TensorId& tensor_id) { return tensor_id.index() == 0 ? string(tensor_id.node()) : tensor_id.ToString(); diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h index e529d5fb4ad..e9ab5b7da12 100644 --- a/tensorflow/core/grappler/utils.h +++ b/tensorflow/core/grappler/utils.h @@ -98,16 +98,39 @@ inline int NodePosition(const string& name) { return position; } -// A utility class to lookup a node and its outputs by node name. -class NodeMap { +namespace internal { +// Base template class for NodeMap and ImmutableNodeMap. +template +class NodeMapInternal { public: // Note: The NodeMap will store pointers to nodes in graph, which may become // invalid if graph is changed. - explicit NodeMap(GraphDef* graph); + explicit NodeMapInternal(GraphDefT* graph) { + if (graph == nullptr) { + LOG(WARNING) << "NodeMapInternal constructor is called with a nullptr!"; + return; + } + nodes_.reserve(graph->node_size()); + outputs_.reserve(graph->node_size()); + for (int i = 0; i < graph->node_size(); i++) { + NodeDefT* node = GetNodeDefFromGraph(graph, i); + const string& node_name = node->name(); + auto rslt = nodes_.emplace(node_name, node); + // Check that the graph doesn't contain multiple nodes with the same name. + if (!rslt.second) { + // The first node found with a given name becomes the canonical. + LOG(WARNING) << "Duplicated node in the graph: " << node_name; + } + NodeDefT* canonical = rslt.second ? node : rslt.first->second; + for (const auto& input : node->input()) { + outputs_[NodeName(input)].insert(canonical); + } + } + } // Get unordered list of fanouts from node. Notice, that the order is // non-deterministic. - const absl::flat_hash_set& GetOutputs( + const absl::flat_hash_set& GetOutputs( const string& node_name) const { auto it = outputs_.find(node_name); if (it == outputs_.end()) { @@ -117,12 +140,12 @@ class NodeMap { } // Get fanouts ordered by name. - std::vector GetOutputsOrderedByNodeName( + std::vector GetOutputsOrderedByNodeName( const string& node_name) const { - std::vector result; + std::vector result; auto it = outputs_.find(node_name); if (it != outputs_.end()) { - const absl::flat_hash_set& outputs = it->second; + const absl::flat_hash_set& outputs = it->second; result.reserve(outputs.size()); result.assign(outputs.begin(), outputs.end()); std::sort(result.begin(), result.end(), @@ -135,7 +158,7 @@ class NodeMap { // This method doesn't record the outputs of the added node; the outputs need // to be explicitly added by the AddOutput method. - void AddNode(const string& node_name, NodeDef* node) { + void AddNode(const string& node_name, NodeDefT* node) { DCHECK(node != nullptr); auto ret = nodes_.emplace(node_name, node); DCHECK(ret.second) @@ -148,7 +171,7 @@ class NodeMap { outputs_.erase(NodeName(name)); } - NodeDef* GetNode(const string& name) const { + NodeDefT* GetNode(const string& name) const { const string node_name = NodeName(name); auto it = nodes_.find(node_name); if (it == nodes_.end()) { @@ -197,9 +220,26 @@ class NodeMap { } private: - const absl::flat_hash_set empty_set_; - absl::node_hash_map nodes_; - absl::node_hash_map> outputs_; + // Helper method to get the NodeDef pointer of i-th node in a graph. + NodeDefT* GetNodeDefFromGraph(GraphDefT* graph, int64 i) const; + + const absl::flat_hash_set empty_set_; + absl::node_hash_map nodes_; + absl::node_hash_map> outputs_; +}; +} // namespace internal + +// A utility class to lookup a node and its outputs by node name. +class NodeMap : public internal::NodeMapInternal { + public: + explicit NodeMap(GraphDef* graph) : NodeMapInternal(graph) {} +}; + +// Same to NodeMap, but uses const GraphDef. +class ImmutableNodeMap + : public internal::NodeMapInternal { + public: + explicit ImmutableNodeMap(const GraphDef* graph) : NodeMapInternal(graph) {} }; // A vector with a set. The set stores the same elements as the vector, and diff --git a/tensorflow/core/grappler/utils_test.cc b/tensorflow/core/grappler/utils_test.cc index 6231fb7a780..31444735b20 100644 --- a/tensorflow/core/grappler/utils_test.cc +++ b/tensorflow/core/grappler/utils_test.cc @@ -349,39 +349,69 @@ TEST_F(UtilsTest, NumNonControlOutputs) { GraphDef graph; TF_CHECK_OK(s.ToGraphDef(&graph)); - NodeMap node_map(&graph); - const NodeDef* add_node = node_map.GetNode("add"); - const NodeDef* mul_node = node_map.GetNode("mul"); - ASSERT_NE(add_node, nullptr); + { + NodeMap node_map(&graph); - // [a, b] are only non-control inputs - EXPECT_EQ(NumNonControlInputs(*add_node), 2); - EXPECT_EQ(NumControlInputs(*add_node), 1); - // [sqrt, shape] are non control outputs - EXPECT_EQ(NumNonControlOutputs(*add_node, node_map), 2); - // sqrt is the only data output - EXPECT_EQ(NumNonControlDataOutputs(*add_node, node_map), 1); - EXPECT_EQ(NumControlInputs(*mul_node), 0); + const NodeDef* add_node = node_map.GetNode("add"); + const NodeDef* mul_node = node_map.GetNode("mul"); + ASSERT_NE(add_node, nullptr); - EXPECT_TRUE(HasControlInputs(*add_node)); - EXPECT_TRUE(HasRegularInputs(*add_node)); - EXPECT_TRUE(HasControlOutputs(*add_node, node_map)); - EXPECT_TRUE(HasRegularOutputs(*add_node, node_map)); + // [a, b] are only non-control inputs + EXPECT_EQ(NumNonControlInputs(*add_node), 2); + EXPECT_EQ(NumControlInputs(*add_node), 1); + // [sqrt, shape] are non control outputs + EXPECT_EQ(NumNonControlOutputs(*add_node, node_map), 2); + // sqrt is the only data output + EXPECT_EQ(NumNonControlDataOutputs(*add_node, node_map), 1); + EXPECT_EQ(NumControlInputs(*mul_node), 0); - const NodeDef* x_node = node_map.GetNode("x"); - ASSERT_NE(x_node, nullptr); - EXPECT_FALSE(HasControlInputs(*x_node)); - EXPECT_FALSE(HasRegularInputs(*x_node)); - EXPECT_FALSE(HasControlOutputs(*x_node, node_map)); - EXPECT_TRUE(HasRegularOutputs(*x_node, node_map)); + EXPECT_TRUE(HasControlInputs(*add_node)); + EXPECT_TRUE(HasRegularInputs(*add_node)); + EXPECT_TRUE(HasControlOutputs(*add_node, node_map)); + EXPECT_TRUE(HasRegularOutputs(*add_node, node_map)); - const NodeDef* round_node = node_map.GetNode("round"); - ASSERT_NE(round_node, nullptr); - EXPECT_TRUE(HasControlInputs(*round_node)); - EXPECT_TRUE(HasRegularInputs(*round_node)); - EXPECT_FALSE(HasControlOutputs(*round_node, node_map)); - EXPECT_FALSE(HasRegularOutputs(*round_node, node_map)); + const NodeDef* x_node = node_map.GetNode("x"); + ASSERT_NE(x_node, nullptr); + EXPECT_FALSE(HasControlInputs(*x_node)); + EXPECT_FALSE(HasRegularInputs(*x_node)); + EXPECT_FALSE(HasControlOutputs(*x_node, node_map)); + EXPECT_TRUE(HasRegularOutputs(*x_node, node_map)); + + const NodeDef* round_node = node_map.GetNode("round"); + ASSERT_NE(round_node, nullptr); + EXPECT_TRUE(HasControlInputs(*round_node)); + EXPECT_TRUE(HasRegularInputs(*round_node)); + EXPECT_FALSE(HasControlOutputs(*round_node, node_map)); + EXPECT_FALSE(HasRegularOutputs(*round_node, node_map)); + } + + { + // Similar test for ImmutableNodeMap. + ImmutableNodeMap node_map(&graph); + + const NodeDef* add_node = node_map.GetNode("add"); + const NodeDef* mul_node = node_map.GetNode("mul"); + ASSERT_NE(add_node, nullptr); + + // [a, b] are only non-control inputs + EXPECT_EQ(NumNonControlInputs(*add_node), 2); + EXPECT_EQ(NumControlInputs(*add_node), 1); + EXPECT_EQ(NumControlInputs(*mul_node), 0); + + EXPECT_TRUE(HasControlInputs(*add_node)); + EXPECT_TRUE(HasRegularInputs(*add_node)); + + const NodeDef* x_node = node_map.GetNode("x"); + ASSERT_NE(x_node, nullptr); + EXPECT_FALSE(HasControlInputs(*x_node)); + EXPECT_FALSE(HasRegularInputs(*x_node)); + + const NodeDef* round_node = node_map.GetNode("round"); + ASSERT_NE(round_node, nullptr); + EXPECT_TRUE(HasControlInputs(*round_node)); + EXPECT_TRUE(HasRegularInputs(*round_node)); + } } TEST(CheckAttrExists, All) { @@ -664,6 +694,17 @@ static void BM_NodeMapConstruct(int iters, int size) { } BENCHMARK(BM_NodeMapConstruct)->Range(1, 1 << 20); +static void BM_ImmutableNodeMapConstruct(int iters, int size) { + testing::StopTiming(); + GraphDef graph = test::CreateRandomGraph(size); + testing::StartTiming(); + for (int i = 0; i < iters; i++) { + ImmutableNodeMap node_map(&graph); + } + testing::StopTiming(); +} +BENCHMARK(BM_ImmutableNodeMapConstruct)->Range(1, 1 << 20); + } // namespace } // namespace grappler } // namespace tensorflow From 79594069bb6b5b0e43aeb7d6c6504f10ab8b8be8 Mon Sep 17 00:00:00 2001 From: Yujing Zhang Date: Tue, 4 Aug 2020 16:57:42 -0700 Subject: [PATCH 0414/1017] Preserve composite devices when cloning a ProcessFunctionLibraryRuntime. PiperOrigin-RevId: 324919856 Change-Id: Ibfe2df7e511593730ebe54e57204b136b0fff5a9 --- .../process_function_library_runtime.cc | 4 ++++ .../process_function_library_runtime.h | 4 ++++ .../process_function_library_runtime_test.cc | 22 +++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc index 515477cd16a..aee482d92da 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc @@ -1667,6 +1667,10 @@ Status ProcessFunctionLibraryRuntime::Clone( device_mgr_, env, config_ ? &(*config_) : nullptr, graph_def_version, out_lib_def->get(), optimizer_options, default_thread_pool_, parent_, custom_kernel_creator, session_metadata_, rendezvous_factory_); + { + tf_shared_lock l(mu_); + for (auto* d : composite_devices_) (*out_pflr)->AddCompositeDevice(d); + } return Status::OK(); } diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.h b/tensorflow/core/common_runtime/process_function_library_runtime.h index bc68c9c2807..0bd85c62df5 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.h +++ b/tensorflow/core/common_runtime/process_function_library_runtime.h @@ -221,6 +221,7 @@ class ProcessFunctionLibraryRuntime { void AddCompositeDevice(CompositeDevice* d) TF_LOCKS_EXCLUDED(mu_) { mutex_lock l(mu_); device_set_->AddDevice(d); + composite_devices_.push_back(d); } protected: @@ -452,6 +453,9 @@ class ProcessFunctionLibraryRuntime { // fail if it spans the changed remote devices. std::shared_ptr device_set_ TF_GUARDED_BY(mu_); + // Composite devices owned by a EagerContext. + std::vector composite_devices_ TF_GUARDED_BY(mu_); + // Holds all the function instantiations. Maps function_keys to handles. std::unordered_map table_ TF_GUARDED_BY(mu_); diff --git a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc index 19c33a53d20..be279c84d1a 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc @@ -1188,6 +1188,28 @@ TEST_F(ProcessFunctionLibraryRuntimeTest, SessionMetadataPresent) { EXPECT_EQ(session_metadata.version(), read_metadata.version()); } +TEST_F(ProcessFunctionLibraryRuntimeTest, CompositeDevicesAfterCloning) { + Init({AddVarAcrossDevices()}); + + Status s; + std::unique_ptr composite_device = + CompositeDevice::MakeDevice({device0_->name(), device1_->name()}, + /*unique_device_id=*/0, + device_mgr_->HostCPU()->parsed_name(), &s); + TF_ASSERT_OK(s); + AddCompositeDevice(composite_device.get()); + + auto* flr = proc_flr_->GetFLR("/job:a/replica:0/task:0/cpu:0"); + ASSERT_NE(nullptr, flr); + std::unique_ptr cloned_lib_def; + std::unique_ptr cloned_proc_flr; + FunctionLibraryRuntime* cloned_flr; + TF_ASSERT_OK(flr->Clone(&cloned_lib_def, &cloned_proc_flr, &cloned_flr)); + EXPECT_EQ( + cloned_proc_flr->device_set()->FindDeviceByName(composite_device->name()), + composite_device.get()); +} + TEST_F(ProcessFunctionLibraryRuntimeTest, SessionMetadataPresentAfterCloning) { const SessionMetadata session_metadata = GenerateSessionMetadata(); Init({SessionMetadataReaderOpFn()}, &session_metadata); From 84d053187cb80d975ef2b9684d4b61981bca0c41 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 17:19:50 -0700 Subject: [PATCH 0415/1017] Break up core/kernels/BUILD (part 1 of N): Move linear algebra kernels to subdirectory tensorflow/core/kernels/linalg with its own BUILD file. PiperOrigin-RevId: 324923762 Change-Id: Id17aac690729b62ae97525df5bb57d6a073d6b0c --- tensorflow/core/BUILD | 2 +- tensorflow/core/kernels/BUILD | 412 ++---------------- tensorflow/core/kernels/linalg/BUILD | 376 ++++++++++++++++ .../banded_triangular_solve_op.cc | 2 +- .../banded_triangular_solve_op_test.cc | 2 +- .../kernels/{ => linalg}/cholesky_grad.cc | 2 +- .../core/kernels/{ => linalg}/cholesky_op.cc | 6 +- .../kernels/{ => linalg}/determinant_op.cc | 6 +- .../kernels/{ => linalg}/determinant_op.h | 6 +- .../{ => linalg}/determinant_op_gpu.cu.cc | 4 +- .../kernels/{ => linalg}/eig_op_complex128.cc | 2 +- .../kernels/{ => linalg}/eig_op_complex64.cc | 2 +- .../kernels/{ => linalg}/eig_op_double.cc | 2 +- .../core/kernels/{ => linalg}/eig_op_float.cc | 2 +- .../core/kernels/{ => linalg}/eig_op_impl.h | 8 +- .../core/kernels/{ => linalg}/einsum_op.h | 4 +- .../kernels/{ => linalg}/einsum_op_gpu.cu.cc | 2 +- .../kernels/{ => linalg}/einsum_op_impl.h | 8 +- .../{ => linalg}/einsum_op_impl_bfloat16.cc | 2 +- .../{ => linalg}/einsum_op_impl_complex128.cc | 2 +- .../{ => linalg}/einsum_op_impl_complex64.cc | 2 +- .../{ => linalg}/einsum_op_impl_double.cc | 2 +- .../{ => linalg}/einsum_op_impl_float.cc | 2 +- .../{ => linalg}/einsum_op_impl_half.cc | 2 +- .../{ => linalg}/einsum_op_impl_int32.cc | 2 +- .../{ => linalg}/einsum_op_impl_int64.cc | 2 +- .../core/kernels/{ => linalg}/eye_functor.h | 4 +- .../{ => linalg}/eye_functor_gpu.cu.cc | 2 +- .../kernels/{ => linalg}/linalg_ops_common.cc | 2 +- .../core/kernels/linalg/linalg_ops_common.h | 221 ++++++++++ tensorflow/core/kernels/{ => linalg}/lu_op.cc | 0 .../core/kernels/{ => linalg}/lu_op_gpu.cu.cc | 2 +- .../{ => linalg}/matrix_band_part_op.cc | 3 +- .../{ => linalg}/matrix_band_part_op.h | 6 +- .../matrix_band_part_op_gpu.cu.cc | 2 +- .../kernels/{ => linalg}/matrix_diag_op.cc | 2 +- .../kernels/{ => linalg}/matrix_diag_op.h | 6 +- .../{ => linalg}/matrix_diag_op_gpu.cu.cc | 2 +- .../{ => linalg}/matrix_exponential_op.cc | 2 +- .../kernels/{ => linalg}/matrix_inverse_op.cc | 6 +- .../{ => linalg}/matrix_logarithm_op.cc | 2 +- .../{ => linalg}/matrix_set_diag_op.cc | 4 +- .../kernels/{ => linalg}/matrix_set_diag_op.h | 6 +- .../{ => linalg}/matrix_set_diag_op_gpu.cu.cc | 2 +- .../matrix_solve_ls_op_complex128.cc | 2 +- .../matrix_solve_ls_op_complex64.cc | 2 +- .../{ => linalg}/matrix_solve_ls_op_double.cc | 2 +- .../{ => linalg}/matrix_solve_ls_op_float.cc | 2 +- .../{ => linalg}/matrix_solve_ls_op_impl.h | 8 +- .../kernels/{ => linalg}/matrix_solve_op.cc | 4 +- .../{ => linalg}/matrix_square_root_op.cc | 2 +- .../matrix_triangular_solve_op_complex.cc | 2 +- .../matrix_triangular_solve_op_impl.h | 12 +- .../matrix_triangular_solve_op_real.cc | 2 +- .../matrix_triangular_solve_op_test.cc | 0 .../kernels/{ => linalg}/qr_op_complex128.cc | 2 +- .../kernels/{ => linalg}/qr_op_complex64.cc | 2 +- .../core/kernels/{ => linalg}/qr_op_double.cc | 2 +- .../core/kernels/{ => linalg}/qr_op_float.cc | 2 +- .../core/kernels/{ => linalg}/qr_op_impl.h | 14 +- .../{ => linalg}/self_adjoint_eig_op.cc | 2 +- .../self_adjoint_eig_v2_op_complex128.cc | 2 +- .../self_adjoint_eig_v2_op_complex64.cc | 2 +- .../self_adjoint_eig_v2_op_double.cc | 2 +- .../self_adjoint_eig_v2_op_float.cc | 2 +- .../self_adjoint_eig_v2_op_gpu.cc | 2 +- .../self_adjoint_eig_v2_op_impl.h | 8 +- .../kernels/{ => linalg}/svd_op_complex128.cc | 2 +- .../kernels/{ => linalg}/svd_op_complex64.cc | 2 +- .../kernels/{ => linalg}/svd_op_double.cc | 2 +- .../core/kernels/{ => linalg}/svd_op_float.cc | 2 +- .../kernels/{ => linalg}/svd_op_gpu.cu.cc | 6 +- .../core/kernels/{ => linalg}/svd_op_impl.h | 8 +- .../{ => linalg}/tridiagonal_matmul_op.cc | 2 +- .../tridiagonal_matmul_op_gpu.cu.cc | 6 +- .../{ => linalg}/tridiagonal_solve_op.cc | 2 +- .../tridiagonal_solve_op_gpu.cu.cc | 6 +- tensorflow/core/kernels/linalg_ops_common.h | 205 +-------- .../core/kernels/segment_reduction_ops_impl.h | 4 +- tensorflow/core/kernels/sparse/BUILD | 4 +- tensorflow/core/kernels/sparse/add_op.cc | 4 +- tensorflow/core/kernels/sparse/conj_op.cc | 4 +- .../sparse/csr_sparse_matrix_to_dense_op.cc | 4 +- .../csr_sparse_matrix_to_sparse_tensor_op.cc | 4 +- .../sparse/dense_to_csr_sparse_matrix_op.cc | 4 +- .../core/kernels/sparse/kernels_gpu.cu.cc | 2 +- tensorflow/core/kernels/sparse/mat_mul_op.cc | 4 +- tensorflow/core/kernels/sparse/mul_op.cc | 2 +- tensorflow/core/kernels/sparse/nnz_op.cc | 4 +- tensorflow/core/kernels/sparse/softmax_op.cc | 2 +- .../core/kernels/sparse/sparse_mat_mul_op.cc | 4 +- .../sparse/sparse_matrix_components_op.cc | 4 +- .../sparse_tensor_to_csr_sparse_matrix_op.cc | 4 +- .../core/kernels/sparse/transpose_op.cc | 2 +- tensorflow/core/kernels/where_op.cc | 2 +- tensorflow/core/util/BUILD | 63 +++ .../core/{kernels => util}/cuda_solvers.cc | 2 +- .../core/{kernels => util}/cuda_solvers.h | 8 +- .../core/{kernels => util}/cuda_sparse.cc | 4 +- .../core/{kernels => util}/cuda_sparse.h | 49 +-- .../core/{kernels => util}/rocm_solvers.cc | 2 +- .../core/{kernels => util}/rocm_solvers.h | 6 +- .../core/{kernels => util}/rocm_sparse.cc | 4 +- 103 files changed, 885 insertions(+), 772 deletions(-) create mode 100644 tensorflow/core/kernels/linalg/BUILD rename tensorflow/core/kernels/{ => linalg}/banded_triangular_solve_op.cc (99%) rename tensorflow/core/kernels/{ => linalg}/banded_triangular_solve_op_test.cc (99%) rename tensorflow/core/kernels/{ => linalg}/cholesky_grad.cc (99%) rename tensorflow/core/kernels/{ => linalg}/cholesky_op.cc (98%) rename tensorflow/core/kernels/{ => linalg}/determinant_op.cc (99%) rename tensorflow/core/kernels/{ => linalg}/determinant_op.h (90%) rename tensorflow/core/kernels/{ => linalg}/determinant_op_gpu.cu.cc (98%) rename tensorflow/core/kernels/{ => linalg}/eig_op_complex128.cc (93%) rename tensorflow/core/kernels/{ => linalg}/eig_op_complex64.cc (93%) rename tensorflow/core/kernels/{ => linalg}/eig_op_double.cc (93%) rename tensorflow/core/kernels/{ => linalg}/eig_op_float.cc (93%) rename tensorflow/core/kernels/{ => linalg}/eig_op_impl.h (93%) rename tensorflow/core/kernels/{ => linalg}/einsum_op.h (94%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_gpu.cu.cc (96%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_impl.h (99%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_impl_bfloat16.cc (94%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_impl_complex128.cc (95%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_impl_complex64.cc (95%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_impl_double.cc (95%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_impl_float.cc (95%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_impl_half.cc (95%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_impl_int32.cc (94%) rename tensorflow/core/kernels/{ => linalg}/einsum_op_impl_int64.cc (94%) rename tensorflow/core/kernels/{ => linalg}/eye_functor.h (90%) rename tensorflow/core/kernels/{ => linalg}/eye_functor_gpu.cu.cc (97%) rename tensorflow/core/kernels/{ => linalg}/linalg_ops_common.cc (99%) create mode 100644 tensorflow/core/kernels/linalg/linalg_ops_common.h rename tensorflow/core/kernels/{ => linalg}/lu_op.cc (100%) rename tensorflow/core/kernels/{ => linalg}/lu_op_gpu.cu.cc (99%) rename tensorflow/core/kernels/{ => linalg}/matrix_band_part_op.cc (99%) rename tensorflow/core/kernels/{ => linalg}/matrix_band_part_op.h (86%) rename tensorflow/core/kernels/{ => linalg}/matrix_band_part_op_gpu.cu.cc (97%) rename tensorflow/core/kernels/{ => linalg}/matrix_diag_op.cc (99%) rename tensorflow/core/kernels/{ => linalg}/matrix_diag_op.h (94%) rename tensorflow/core/kernels/{ => linalg}/matrix_diag_op_gpu.cu.cc (99%) rename tensorflow/core/kernels/{ => linalg}/matrix_exponential_op.cc (97%) rename tensorflow/core/kernels/{ => linalg}/matrix_inverse_op.cc (98%) rename tensorflow/core/kernels/{ => linalg}/matrix_logarithm_op.cc (97%) rename tensorflow/core/kernels/{ => linalg}/matrix_set_diag_op.cc (99%) rename tensorflow/core/kernels/{ => linalg}/matrix_set_diag_op.h (89%) rename tensorflow/core/kernels/{ => linalg}/matrix_set_diag_op_gpu.cu.cc (99%) rename tensorflow/core/kernels/{ => linalg}/matrix_solve_ls_op_complex128.cc (92%) rename tensorflow/core/kernels/{ => linalg}/matrix_solve_ls_op_complex64.cc (92%) rename tensorflow/core/kernels/{ => linalg}/matrix_solve_ls_op_double.cc (92%) rename tensorflow/core/kernels/{ => linalg}/matrix_solve_ls_op_float.cc (92%) rename tensorflow/core/kernels/{ => linalg}/matrix_solve_ls_op_impl.h (96%) rename tensorflow/core/kernels/{ => linalg}/matrix_solve_op.cc (99%) rename tensorflow/core/kernels/{ => linalg}/matrix_square_root_op.cc (97%) rename tensorflow/core/kernels/{ => linalg}/matrix_triangular_solve_op_complex.cc (92%) rename tensorflow/core/kernels/{ => linalg}/matrix_triangular_solve_op_impl.h (97%) rename tensorflow/core/kernels/{ => linalg}/matrix_triangular_solve_op_real.cc (93%) rename tensorflow/core/kernels/{ => linalg}/matrix_triangular_solve_op_test.cc (100%) rename tensorflow/core/kernels/{ => linalg}/qr_op_complex128.cc (96%) rename tensorflow/core/kernels/{ => linalg}/qr_op_complex64.cc (95%) rename tensorflow/core/kernels/{ => linalg}/qr_op_double.cc (96%) rename tensorflow/core/kernels/{ => linalg}/qr_op_float.cc (96%) rename tensorflow/core/kernels/{ => linalg}/qr_op_impl.h (96%) rename tensorflow/core/kernels/{ => linalg}/self_adjoint_eig_op.cc (98%) rename tensorflow/core/kernels/{ => linalg}/self_adjoint_eig_v2_op_complex128.cc (93%) rename tensorflow/core/kernels/{ => linalg}/self_adjoint_eig_v2_op_complex64.cc (93%) rename tensorflow/core/kernels/{ => linalg}/self_adjoint_eig_v2_op_double.cc (92%) rename tensorflow/core/kernels/{ => linalg}/self_adjoint_eig_v2_op_float.cc (92%) rename tensorflow/core/kernels/{ => linalg}/self_adjoint_eig_v2_op_gpu.cc (99%) rename tensorflow/core/kernels/{ => linalg}/self_adjoint_eig_v2_op_impl.h (91%) rename tensorflow/core/kernels/{ => linalg}/svd_op_complex128.cc (93%) rename tensorflow/core/kernels/{ => linalg}/svd_op_complex64.cc (93%) rename tensorflow/core/kernels/{ => linalg}/svd_op_double.cc (93%) rename tensorflow/core/kernels/{ => linalg}/svd_op_float.cc (93%) rename tensorflow/core/kernels/{ => linalg}/svd_op_gpu.cu.cc (99%) rename tensorflow/core/kernels/{ => linalg}/svd_op_impl.h (95%) rename tensorflow/core/kernels/{ => linalg}/tridiagonal_matmul_op.cc (98%) rename tensorflow/core/kernels/{ => linalg}/tridiagonal_matmul_op_gpu.cu.cc (96%) rename tensorflow/core/kernels/{ => linalg}/tridiagonal_solve_op.cc (99%) rename tensorflow/core/kernels/{ => linalg}/tridiagonal_solve_op_gpu.cu.cc (99%) rename tensorflow/core/{kernels => util}/cuda_solvers.cc (99%) rename tensorflow/core/{kernels => util}/cuda_solvers.h (99%) rename tensorflow/core/{kernels => util}/cuda_sparse.cc (99%) rename tensorflow/core/{kernels => util}/cuda_sparse.h (93%) rename tensorflow/core/{kernels => util}/rocm_solvers.cc (99%) rename tensorflow/core/{kernels => util}/rocm_solvers.h (96%) rename tensorflow/core/{kernels => util}/rocm_sparse.cc (99%) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 86c9d1fc665..161a0a95856 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1010,7 +1010,7 @@ cc_library( "//tensorflow/core/kernels:histogram_op", "//tensorflow/core/kernels:image", "//tensorflow/core/kernels:io", - "//tensorflow/core/kernels:linalg", + "//tensorflow/core/kernels/linalg:linalg", "//tensorflow/core/kernels:lookup", "//tensorflow/core/kernels:logging", "//tensorflow/core/kernels:manip", diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 34a3ee800d8..12d4f1c5574 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -1039,9 +1039,6 @@ cc_library( ":immutable_constant_op", ":inplace_ops", ":listdiff_op", - ":matrix_band_part_op", - ":matrix_diag_op", - ":matrix_set_diag_op", ":mirror_pad_op", ":one_hot_op", ":pack_op", @@ -1174,26 +1171,6 @@ tf_kernel_library( deps = ARRAY_DEPS, ) -tf_kernel_library( - name = "matrix_band_part_op", - prefix = "matrix_band_part_op", - deps = if_cuda([ - ":cuda_solvers", - ]) + ARRAY_DEPS, -) - -tf_kernel_library( - name = "matrix_diag_op", - prefix = "matrix_diag_op", - deps = ARRAY_DEPS, -) - -tf_kernel_library( - name = "matrix_set_diag_op", - prefix = "matrix_set_diag_op", - deps = ARRAY_DEPS + [":matrix_diag_op"], -) - tf_kernel_library( name = "mirror_pad_op", prefix = "mirror_pad_op", @@ -1405,7 +1382,7 @@ tf_kernel_library( "where_op_gpu_impl_8.cu.cc", ], deps = if_cuda_or_rocm([ - ":cuda_solvers", + "//tensorflow/core/util:cuda_solvers", ]) + [":gpu_prim_hdrs"] + ARRAY_DEPS, ) @@ -2785,21 +2762,6 @@ tf_cuda_cc_tests( ], ) -tf_kernel_library( - name = "eye_functor", - hdrs = ["eye_functor.h"], - gpu_srcs = [ - "eye_functor_gpu.cu.cc", - "eye_functor.h", - ], - visibility = [":friends"], - deps = [ - "//tensorflow/core:framework", - "//third_party/eigen3", - ], - alwayslink = 0, -) - cc_library( name = "fifo_queue", srcs = ["fifo_queue.cc"], @@ -3558,289 +3520,6 @@ tf_cc_tests( ], ) -cc_library( - name = "linalg", - deps = [ - ":banded_triangular_solve_op", - ":cholesky_grad", - ":cholesky_op", - ":determinant_op", - ":eig_op", - ":einsum_op", - ":lu_op", - ":matrix_exponential_op", - ":matrix_inverse_op", - ":matrix_logarithm_op", - ":matrix_solve_ls_op", - ":matrix_solve_op", - ":matrix_square_root_op", - ":matrix_triangular_solve_op", - ":qr_op", - ":self_adjoint_eig_op", - ":self_adjoint_eig_v2_op", - ":svd_op", - ":tridiagonal_matmul_op", - ":tridiagonal_solve_op", - ], -) - -tf_kernel_library( - name = "cuda_solvers", - srcs = ["cuda_solvers.cc"], - hdrs = ["cuda_solvers.h"], - # @local_config_cuda//cuda:cusolver_static, //third_party/eigen3:blas, - # and //third_party/libf2c all contain various parts of BLAS, LAPACK, - # and f2c helper functions in global namespace. Tell the compiler to - # allow multiple definitions when linking this. - linkopts = select({ - "//tensorflow:macos": [], - "//tensorflow:windows": [], - "//conditions:default": ["-Wl,-z,muldefs"], - }), - visibility = [":friends"], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core/platform/default/build_config:cublas_plugin", - "//tensorflow/stream_executor/cuda:cublas_lib", - "//tensorflow/stream_executor/cuda:cusolver_lib", - ], -) - -tf_kernel_library( - name = "rocm_solvers", - srcs = ["rocm_solvers.cc"], - hdrs = ["rocm_solvers.h"], - visibility = [":friends"], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/stream_executor/lib", - "//tensorflow/stream_executor/platform:dso_loader", - "//tensorflow/stream_executor/rocm:rocblas_plugin", - "//tensorflow/stream_executor/rocm:rocm_gpu_executor", - ] + if_rocm([ - "@local_config_rocm//rocm:rocprim", - ]), -) - -tf_kernel_library( - name = "cuda_sparse", - srcs = if_cuda(["cuda_sparse.cc"]) + if_rocm(["rocm_sparse.cc"]), - hdrs = ["cuda_sparse.h"], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core/kernels:cuda_solvers", - ] + if_cuda([ - "//tensorflow/stream_executor/cuda:cusparse_lib", - "@cub_archive//:cub", - ]) + if_rocm([ - "@local_config_rocm//rocm:hipsparse", - ]), -) - -LINALG_DEPS = [ - ":linalg_ops_common", - "//third_party/eigen3", - "//tensorflow/core:framework", - "//tensorflow/core:lib", -] + if_cuda([ - ":cuda_solvers", - ":transpose_functor", -]) + if_rocm([ - ":rocm_solvers", -]) - -tf_kernel_library( - name = "cholesky_op", - prefix = "cholesky_op", - deps = if_cuda([ - ":matrix_band_part_op", - ]) + LINALG_DEPS, -) - -tf_kernel_library( - name = "cholesky_grad", - prefix = "cholesky_grad", - deps = LINALG_DEPS, -) - -tf_kernel_library( - name = "determinant_op", - prefix = "determinant_op", - deps = if_cuda([ - ":fill_functor", - ]) + LINALG_DEPS, -) - -tf_kernel_library( - name = "matrix_exponential_op", - prefix = "matrix_exponential_op", - deps = LINALG_DEPS, -) - -tf_kernel_library( - name = "matrix_logarithm_op", - prefix = "matrix_logarithm_op", - deps = LINALG_DEPS, -) - -tf_kernel_library( - name = "self_adjoint_eig_op", - prefix = "self_adjoint_eig_op", - deps = LINALG_DEPS + ["//tensorflow/core:lib_internal"], -) - -tf_kernel_library( - name = "self_adjoint_eig_v2_op", - prefix = "self_adjoint_eig_v2_op", - deps = LINALG_DEPS + ["//tensorflow/core:lib_internal"] + if_cuda([ - ":cast_op", - ":cwise_op", - ]), -) - -tf_kernel_library( - name = "eig_op", - prefix = "eig_op", - deps = LINALG_DEPS + ["//tensorflow/core:lib_internal"] + if_cuda([ - ":cast_op", - ":cwise_op", - ]), -) - -tf_kernel_library( - name = "matrix_inverse_op", - prefix = "matrix_inverse_op", - deps = LINALG_DEPS + if_cuda([":eye_functor"]), -) - -tf_kernel_library( - name = "matrix_solve_ls_op", - prefix = "matrix_solve_ls_op", - deps = LINALG_DEPS, -) - -tf_kernel_library( - name = "matrix_solve_op", - prefix = "matrix_solve_op", - deps = LINALG_DEPS, -) - -tf_kernel_library( - name = "matrix_square_root_op", - prefix = "matrix_square_root_op", - deps = LINALG_DEPS, -) - -tf_kernel_library( - name = "banded_triangular_solve_op", - prefix = "banded_triangular_solve_op", - deps = LINALG_DEPS + [":fill_functor"], -) - -tf_kernel_library( - name = "matrix_triangular_solve_op", - hdrs = ["matrix_triangular_solve_op_impl.h"], - prefix = "matrix_triangular_solve_op", - deps = [ - ":linalg_ops_common", - "//third_party/eigen3", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ":fill_functor", - "//tensorflow/core:stream_executor", - ] + if_cuda([ - "//tensorflow/core/platform/default/build_config:cublas_plugin", - ":cuda_solvers", - ]) + if_rocm([ - "@local_config_rocm//rocm:rocprim", - ":rocm_solvers", - ]) + if_cuda_or_rocm([ - ":transpose_functor", - ]), -) - -tf_kernel_library( - name = "tridiagonal_matmul_op", - srcs = ["tridiagonal_matmul_op.cc"], - gpu_srcs = ["tridiagonal_matmul_op_gpu.cu.cc"], - deps = LINALG_DEPS + if_cuda([ - ":cuda_sparse", - ]), -) - -tf_kernel_library( - name = "tridiagonal_solve_op", - srcs = ["tridiagonal_solve_op.cc"], - gpu_srcs = ["tridiagonal_solve_op_gpu.cu.cc"], - deps = LINALG_DEPS + if_cuda([ - ":cuda_sparse", - ]), -) - -tf_kernel_library( - name = "qr_op", - prefix = "qr_op", - deps = LINALG_DEPS + if_cuda([ - ":cwise_op", - ":eye_functor", - ":matrix_band_part_op", - ]), -) - -tf_kernel_library( - name = "svd_op", - prefix = "svd_op", - deps = LINALG_DEPS + if_cuda([ - ":eye_functor", - ]), -) - -tf_kernel_library( - name = "lu_op", - prefix = "lu_op", - deps = if_cuda([ - ":cuda_solvers", - ":transpose_functor", - ]) + [ - "//third_party/eigen3", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ], -) - -tf_kernel_library( - name = "einsum_op", - prefix = "einsum_op", - deps = [ - ":batch_matmul_op", - ":fill_functor", - ":reduction_ops", - ":transpose_functor", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core/profiler/lib:traceme", - "//third_party/eigen3", - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "linalg_ops_common", - srcs = ["linalg_ops_common.cc"], - hdrs = ["linalg_ops_common.h"], - visibility = ["//visibility:private"], - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], -) - cc_library( name = "logging", deps = [ @@ -4208,7 +3887,7 @@ tf_kernel_library( name = "segment_reduction_ops", prefix = "segment_reduction_ops", deps = MATH_DEPS + if_cuda_or_rocm([ - ":cuda_solvers", + "//tensorflow/core/util:cuda_solvers", ]), ) @@ -4405,45 +4084,6 @@ tf_cuda_cc_test( ], ) -tf_cuda_cc_test( - name = "banded_triangular_solve_op_test", - size = "small", - srcs = ["banded_triangular_solve_op_test.cc"], - deps = [ - ":banded_triangular_solve_op", - ":matrix_set_diag_op", - ":matrix_triangular_solve_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cuda_cc_test( - name = "matrix_triangular_solve_op_test", - size = "small", - srcs = ["matrix_triangular_solve_op_test.cc"], - deps = [ - ":broadcast_to_op", - ":matrix_triangular_solve_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - tf_cuda_cc_test( name = "scan_ops_test", size = "small", @@ -6672,10 +6312,7 @@ filegroup( "lookup_table_init_op.h", "lookup_table_op.h", "lookup_util.h", - "linalg_ops_common.h", "list_kernels.h", - "matrix_diag_op.h", - "matrix_set_diag_op.h", "maxpooling_op.h", "mfcc.h", "mfcc_dct.h", @@ -6723,6 +6360,9 @@ filegroup( "xent_op.h", ] + [ "//tensorflow/core/kernels/boosted_trees/quantiles:weighted_quantiles_hdrs", + "//tensorflow/core/kernels/linalg:linalg_ops_common.h", + "//tensorflow/core/kernels/linalg:matrix_diag_op.h", + "//tensorflow/core/kernels/linalg:matrix_set_diag_op.h", ], ) @@ -6823,16 +6463,6 @@ filegroup( "encode_wav_op.cc", "eigen_contraction_kernel.cc", "eigen_contraction_kernel.h", - "einsum_op_impl_half.cc", - "einsum_op_impl_bfloat16.cc", - "einsum_op_impl_int32.cc", - "einsum_op_impl_int64.cc", - "einsum_op_impl_float.cc", - "einsum_op_impl_double.cc", - "einsum_op_impl_complex64.cc", - "einsum_op_impl_complex128.cc", - "einsum_op_impl.h", - "einsum_op.h", "fake_quant_ops.cc", "fifo_queue.cc", "fifo_queue_op.cc", @@ -6844,6 +6474,17 @@ filegroup( "population_count_op.h", "winograd_transform.h", ":android_extended_ops_headers", + ] + [ + "//tensorflow/core/kernels/linalg:einsum_op_impl_half.cc", + "//tensorflow/core/kernels/linalg:einsum_op_impl_bfloat16.cc", + "//tensorflow/core/kernels/linalg:einsum_op_impl_int32.cc", + "//tensorflow/core/kernels/linalg:einsum_op_impl_int64.cc", + "//tensorflow/core/kernels/linalg:einsum_op_impl_float.cc", + "//tensorflow/core/kernels/linalg:einsum_op_impl_double.cc", + "//tensorflow/core/kernels/linalg:einsum_op_impl_complex64.cc", + "//tensorflow/core/kernels/linalg:einsum_op_impl_complex128.cc", + "//tensorflow/core/kernels/linalg:einsum_op_impl.h", + "//tensorflow/core/kernels/linalg:einsum_op.h", ] + select({ ":xsmm_convolutions": [ "xsmm_conv2d.h", @@ -6874,7 +6515,6 @@ filegroup( "in_topk_op.cc", "in_topk_op.h", "initializable_lookup_table.cc", - "linalg_ops_common.cc", "list_kernels.cc", "logging_ops.cc", "logging_ops.h", @@ -6882,9 +6522,6 @@ filegroup( "lookup_table_op.cc", "lookup_util.cc", "lrn_op.cc", - "matrix_diag_op.cc", - "matrix_inverse_op.cc", - "matrix_set_diag_op.cc", "maxpooling_op.cc", "mfcc.cc", "mfcc_dct.cc", @@ -7006,6 +6643,10 @@ filegroup( ":android_extended_ops_headers", ] + [ "//tensorflow/core/kernels/boosted_trees:quantile_ops.cc", + "//tensorflow/core/kernels/linalg:linalg_ops_common.cc", + "//tensorflow/core/kernels/linalg:matrix_diag_op.cc", + "//tensorflow/core/kernels/linalg:matrix_inverse_op.cc", + "//tensorflow/core/kernels/linalg:matrix_set_diag_op.cc", ], ) @@ -7059,6 +6700,7 @@ filegroup( srcs = [ "//tensorflow/c/kernels:android_all_op_kernels", "//tensorflow/core/kernels/data:android_all_op_kernels", + "//tensorflow/core/kernels/linalg:android_all_op_kernels", ] + glob( [ "*.cc", @@ -8827,3 +8469,15 @@ tf_kernel_library( "@sobol_data", ], ) + +# ---- temporary forwarding declaration for libraries in linalg +# TODO(b/160344057): Remove after updating dependencies. +tf_kernel_library( + name = "matrix_inverse_op", + deps = ["//tensorflow/core/kernels/linalg:matrix_inverse_op"], +) + +tf_kernel_library( + name = "einsum_op", + deps = ["//tensorflow/core/kernels/linalg:einsum_op"], +) diff --git a/tensorflow/core/kernels/linalg/BUILD b/tensorflow/core/kernels/linalg/BUILD new file mode 100644 index 00000000000..c735f58ae51 --- /dev/null +++ b/tensorflow/core/kernels/linalg/BUILD @@ -0,0 +1,376 @@ +load( + "//tensorflow:tensorflow.bzl", + "if_cuda_or_rocm", + "tf_kernel_library", +) +load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") +load( + "@local_config_rocm//rocm:build_defs.bzl", + "if_rocm", +) +load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test") + +# Description: +# Op kernel implementations for TensorFlow. +# +# Note: Any test that uses GPU support and which we would like to +# benchmark should be linked statically so that it can be executed +# from a py_binary or cuda_py_test test logger. For such a test, +# append "_gpu" to the test name to invoke the GPU benchmarks. Example: +# +# # for CPU tests +# $ bazel test --config opt //third_party/tensorflow/core/kernels:my_op_test +# # for GPU benchmarks +# $ bazel run --config opt --config=cuda //third_party/tensorflow/core/kernels:my_op_test_gpu -- --benchmarks=.. +# +package( + default_visibility = [ + "//tensorflow:__subpackages__", + "//tensorflow:internal", + ], + licenses = ["notice"], # Apache 2.0 +) + +# TODO(rmlarsen): Remove ASAP. +package_group( + name = "friends", + packages = ["//tensorflow/..."], +) + +# Export a few files for use on Android. +exports_files([ + "einsum_op_impl_half.cc", + "einsum_op_impl_bfloat16.cc", + "einsum_op_impl_int32.cc", + "einsum_op_impl_int64.cc", + "einsum_op_impl_float.cc", + "einsum_op_impl_double.cc", + "einsum_op_impl_complex64.cc", + "einsum_op_impl_complex128.cc", + "einsum_op_impl.h", + "einsum_op.h", + "linalg_ops_common.h", + "linalg_ops_common.cc", + "matrix_diag_op.h", + "matrix_diag_op.cc", + "matrix_inverse_op.cc", + "matrix_set_diag_op.h", + "matrix_set_diag_op.cc", +]) + +# Public support libraries ---------------------------------------------------- + +cc_library( + name = "linalg", + deps = [ + ":banded_triangular_solve_op", + ":cholesky_grad", + ":cholesky_op", + ":determinant_op", + ":eig_op", + ":einsum_op", + ":lu_op", + ":matrix_band_part_op", + ":matrix_diag_op", + ":matrix_exponential_op", + ":matrix_inverse_op", + ":matrix_logarithm_op", + ":matrix_set_diag_op", + ":matrix_solve_ls_op", + ":matrix_solve_op", + ":matrix_square_root_op", + ":matrix_triangular_solve_op", + ":qr_op", + ":self_adjoint_eig_op", + ":self_adjoint_eig_v2_op", + ":svd_op", + ":tridiagonal_matmul_op", + ":tridiagonal_solve_op", + ], +) + +LINALG_DEPS = [ + ":linalg_ops_common", + "//third_party/eigen3", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/kernels:cast_op", + "//tensorflow/core/kernels:fill_functor", +] + if_cuda([ + ":eye_functor", + "//tensorflow/core/util:cuda_solvers", + "//tensorflow/core/kernels:transpose_functor", +]) + if_rocm([ + "//tensorflow/core/util:rocm_solvers", +]) + +tf_kernel_library( + name = "matrix_band_part_op", + prefix = "matrix_band_part_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "matrix_diag_op", + prefix = "matrix_diag_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "matrix_set_diag_op", + prefix = "matrix_set_diag_op", + deps = LINALG_DEPS + [":matrix_diag_op"], +) + +tf_kernel_library( + name = "cholesky_op", + prefix = "cholesky_op", + deps = if_cuda([ + ":matrix_band_part_op", + ]) + LINALG_DEPS, +) + +tf_kernel_library( + name = "cholesky_grad", + prefix = "cholesky_grad", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "determinant_op", + prefix = "determinant_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "matrix_exponential_op", + prefix = "matrix_exponential_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "matrix_logarithm_op", + prefix = "matrix_logarithm_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "self_adjoint_eig_op", + prefix = "self_adjoint_eig_op", + deps = LINALG_DEPS + ["//tensorflow/core:lib_internal"], +) + +tf_kernel_library( + name = "self_adjoint_eig_v2_op", + prefix = "self_adjoint_eig_v2_op", + deps = LINALG_DEPS + ["//tensorflow/core:lib_internal"] + if_cuda([ + "//tensorflow/core/kernels:cwise_op", + ]), +) + +tf_kernel_library( + name = "eig_op", + prefix = "eig_op", + deps = LINALG_DEPS + ["//tensorflow/core:lib_internal"] + if_cuda([ + "//tensorflow/core/kernels:cwise_op", + ]), +) + +tf_kernel_library( + name = "matrix_inverse_op", + prefix = "matrix_inverse_op", + visibility = [":friends"], + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "matrix_solve_ls_op", + prefix = "matrix_solve_ls_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "matrix_solve_op", + prefix = "matrix_solve_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "matrix_square_root_op", + prefix = "matrix_square_root_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "banded_triangular_solve_op", + prefix = "banded_triangular_solve_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "matrix_triangular_solve_op", + hdrs = ["matrix_triangular_solve_op_impl.h"], + prefix = "matrix_triangular_solve_op", + deps = [ + ":linalg_ops_common", + "//third_party/eigen3", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/kernels:fill_functor", + "//tensorflow/core:stream_executor", + ] + if_cuda([ + "//tensorflow/core/platform/default/build_config:cublas_plugin", + "//tensorflow/core/util:cuda_solvers", + ]) + if_rocm([ + "@local_config_rocm//rocm:rocprim", + "//tensorflow/core/util:rocm_solvers", + ]) + if_cuda_or_rocm([ + "//tensorflow/core/kernels:transpose_functor", + ]), +) + +tf_kernel_library( + name = "tridiagonal_matmul_op", + srcs = ["tridiagonal_matmul_op.cc"], + gpu_srcs = ["tridiagonal_matmul_op_gpu.cu.cc"], + deps = LINALG_DEPS + if_cuda([ + "//tensorflow/core/util:cuda_sparse", + ]), +) + +tf_kernel_library( + name = "tridiagonal_solve_op", + srcs = ["tridiagonal_solve_op.cc"], + gpu_srcs = ["tridiagonal_solve_op_gpu.cu.cc"], + deps = LINALG_DEPS + if_cuda([ + "//tensorflow/core/util:cuda_sparse", + ]), +) + +tf_kernel_library( + name = "qr_op", + prefix = "qr_op", + deps = LINALG_DEPS + if_cuda([ + "//tensorflow/core/kernels:cwise_op", + ":matrix_band_part_op", + ]), +) + +tf_kernel_library( + name = "svd_op", + prefix = "svd_op", + deps = LINALG_DEPS, +) + +tf_kernel_library( + name = "lu_op", + prefix = "lu_op", + deps = if_cuda([ + "//tensorflow/core/util:cuda_solvers", + "//tensorflow/core/kernels:transpose_functor", + ]) + [ + "//third_party/eigen3", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + ], +) + +tf_kernel_library( + name = "einsum_op", + prefix = "einsum_op", + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/kernels:batch_matmul_op", + "//tensorflow/core/kernels:fill_functor", + "//tensorflow/core/kernels:reduction_ops", + "//tensorflow/core/kernels:transpose_functor", + "//tensorflow/core/profiler/lib:traceme", + "//third_party/eigen3", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/strings", + ], +) + +cc_library( + name = "linalg_ops_common", + srcs = ["linalg_ops_common.cc"], + hdrs = ["linalg_ops_common.h"], + visibility = ["//visibility:private"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//third_party/eigen3", + ], +) + +tf_cuda_cc_test( + name = "banded_triangular_solve_op_test", + size = "small", + srcs = ["banded_triangular_solve_op_test.cc"], + deps = [ + ":banded_triangular_solve_op", + ":matrix_set_diag_op", + ":matrix_triangular_solve_op", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "//tensorflow/core/kernels:ops_testutil", + "//tensorflow/core/kernels:ops_util", + ], +) + +tf_kernel_library( + name = "eye_functor", + hdrs = ["eye_functor.h"], + gpu_srcs = [ + "eye_functor_gpu.cu.cc", + "eye_functor.h", + ], + visibility = ["//tensorflow/core/kernels:friends"], + deps = [ + "//tensorflow/core:framework", + "//third_party/eigen3", + ], + alwayslink = 0, +) + +tf_cuda_cc_test( + name = "matrix_triangular_solve_op_test", + size = "small", + srcs = ["matrix_triangular_solve_op_test.cc"], + deps = [ + ":matrix_triangular_solve_op", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "//tensorflow/core/kernels:broadcast_to_op", + "//tensorflow/core/kernels:ops_testutil", + "//tensorflow/core/kernels:ops_util", + ], +) + +# A file group which contains all operators which are known to work on mobile. +filegroup( + name = "android_all_op_kernels", + srcs = glob( + [ + "*.cc", + "*.h", + ], + exclude = [ + "*test.cc", + "*test.h", + "*_test_*", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) diff --git a/tensorflow/core/kernels/banded_triangular_solve_op.cc b/tensorflow/core/kernels/linalg/banded_triangular_solve_op.cc similarity index 99% rename from tensorflow/core/kernels/banded_triangular_solve_op.cc rename to tensorflow/core/kernels/linalg/banded_triangular_solve_op.cc index d01a015502a..6758dcf5b8b 100644 --- a/tensorflow/core/kernels/banded_triangular_solve_op.cc +++ b/tensorflow/core/kernels/linalg/banded_triangular_solve_op.cc @@ -20,7 +20,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/fill_functor.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" diff --git a/tensorflow/core/kernels/banded_triangular_solve_op_test.cc b/tensorflow/core/kernels/linalg/banded_triangular_solve_op_test.cc similarity index 99% rename from tensorflow/core/kernels/banded_triangular_solve_op_test.cc rename to tensorflow/core/kernels/linalg/banded_triangular_solve_op_test.cc index 37e904a3e0e..7c20b88845f 100644 --- a/tensorflow/core/kernels/banded_triangular_solve_op_test.cc +++ b/tensorflow/core/kernels/linalg/banded_triangular_solve_op_test.cc @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/core/graph/graph.h" #include "tensorflow/core/graph/node_builder.h" #include "tensorflow/core/graph/testlib.h" -#include "tensorflow/core/kernels/matrix_set_diag_op.h" +#include "tensorflow/core/kernels/linalg/matrix_set_diag_op.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" diff --git a/tensorflow/core/kernels/cholesky_grad.cc b/tensorflow/core/kernels/linalg/cholesky_grad.cc similarity index 99% rename from tensorflow/core/kernels/cholesky_grad.cc rename to tensorflow/core/kernels/linalg/cholesky_grad.cc index eac66e580dd..31a5570cddf 100644 --- a/tensorflow/core/kernels/cholesky_grad.cc +++ b/tensorflow/core/kernels/linalg/cholesky_grad.cc @@ -18,7 +18,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/cholesky_op.cc b/tensorflow/core/kernels/linalg/cholesky_op.cc similarity index 98% rename from tensorflow/core/kernels/cholesky_op.cc rename to tensorflow/core/kernels/linalg/cholesky_op.cc index ff8fd08f228..eae09124b36 100644 --- a/tensorflow/core/kernels/cholesky_op.cc +++ b/tensorflow/core/kernels/linalg/cholesky_op.cc @@ -25,16 +25,16 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" #if GOOGLE_CUDA #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/matrix_band_part_op.h" +#include "tensorflow/core/kernels/linalg/matrix_band_part_op.h" #include "tensorflow/core/platform/stream_executor.h" +#include "tensorflow/core/util/cuda_solvers.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/determinant_op.cc b/tensorflow/core/kernels/linalg/determinant_op.cc similarity index 99% rename from tensorflow/core/kernels/determinant_op.cc rename to tensorflow/core/kernels/linalg/determinant_op.cc index b06f42384eb..8f0b0b618cf 100644 --- a/tensorflow/core/kernels/determinant_op.cc +++ b/tensorflow/core/kernels/linalg/determinant_op.cc @@ -20,7 +20,7 @@ limitations under the License. #if GOOGLE_CUDA #define EIGEN_USE_GPU #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/kernels/determinant_op.h" +#include "tensorflow/core/kernels/linalg/determinant_op.h" #endif #include "third_party/eigen3/Eigen/LU" @@ -28,14 +28,14 @@ limitations under the License. #include "tensorflow/core/framework/numeric_types.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" #if GOOGLE_CUDA -#include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/kernels/fill_functor.h" +#include "tensorflow/core/util/cuda_solvers.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/determinant_op.h b/tensorflow/core/kernels/linalg/determinant_op.h similarity index 90% rename from tensorflow/core/kernels/determinant_op.h rename to tensorflow/core/kernels/linalg/determinant_op.h index eefdfe0ae40..6ace1bef44b 100644 --- a/tensorflow/core/kernels/determinant_op.h +++ b/tensorflow/core/kernels/linalg/determinant_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_ -#define TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_DETERMINANT_OP_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_DETERMINANT_OP_H_ #include "tensorflow/core/framework/tensor_types.h" @@ -44,4 +44,4 @@ struct LogDeterminantFromPivotedLUFunctor { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_DETERMINANT_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_DETERMINANT_OP_H_ diff --git a/tensorflow/core/kernels/determinant_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/determinant_op_gpu.cu.cc similarity index 98% rename from tensorflow/core/kernels/determinant_op_gpu.cu.cc rename to tensorflow/core/kernels/linalg/determinant_op_gpu.cu.cc index 9aa64b3a7da..f6ab327bce0 100644 --- a/tensorflow/core/kernels/determinant_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/determinant_op_gpu.cu.cc @@ -21,8 +21,8 @@ limitations under the License. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/determinant_op.h" +#include "tensorflow/core/kernels/linalg/determinant_op.h" +#include "tensorflow/core/util/cuda_solvers.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/eig_op_complex128.cc b/tensorflow/core/kernels/linalg/eig_op_complex128.cc similarity index 93% rename from tensorflow/core/kernels/eig_op_complex128.cc rename to tensorflow/core/kernels/linalg/eig_op_complex128.cc index 988cc2f98d9..bd4b6fe36d0 100644 --- a/tensorflow/core/kernels/eig_op_complex128.cc +++ b/tensorflow/core/kernels/linalg/eig_op_complex128.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/eig_op_impl.h" +#include "tensorflow/core/kernels/linalg/eig_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/eig_op_complex64.cc b/tensorflow/core/kernels/linalg/eig_op_complex64.cc similarity index 93% rename from tensorflow/core/kernels/eig_op_complex64.cc rename to tensorflow/core/kernels/linalg/eig_op_complex64.cc index 6a3f7928715..b5b4a26ee85 100644 --- a/tensorflow/core/kernels/eig_op_complex64.cc +++ b/tensorflow/core/kernels/linalg/eig_op_complex64.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/eig_op_impl.h" +#include "tensorflow/core/kernels/linalg/eig_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/eig_op_double.cc b/tensorflow/core/kernels/linalg/eig_op_double.cc similarity index 93% rename from tensorflow/core/kernels/eig_op_double.cc rename to tensorflow/core/kernels/linalg/eig_op_double.cc index 2cd931cc135..c360637c84a 100644 --- a/tensorflow/core/kernels/eig_op_double.cc +++ b/tensorflow/core/kernels/linalg/eig_op_double.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/eig_op_impl.h" +#include "tensorflow/core/kernels/linalg/eig_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/eig_op_float.cc b/tensorflow/core/kernels/linalg/eig_op_float.cc similarity index 93% rename from tensorflow/core/kernels/eig_op_float.cc rename to tensorflow/core/kernels/linalg/eig_op_float.cc index a06f76e935f..18f576fcc19 100644 --- a/tensorflow/core/kernels/eig_op_float.cc +++ b/tensorflow/core/kernels/linalg/eig_op_float.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/eig_op_impl.h" +#include "tensorflow/core/kernels/linalg/eig_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/eig_op_impl.h b/tensorflow/core/kernels/linalg/eig_op_impl.h similarity index 93% rename from tensorflow/core/kernels/eig_op_impl.h rename to tensorflow/core/kernels/linalg/eig_op_impl.h index 4ebb6bde08b..a7aff7c2a5d 100644 --- a/tensorflow/core/kernels/eig_op_impl.h +++ b/tensorflow/core/kernels/linalg/eig_op_impl.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_EIG_OP_IMPL_H_ -#define TENSORFLOW_CORE_KERNELS_EIG_OP_IMPL_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_EIG_OP_IMPL_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_EIG_OP_IMPL_H_ // See docs in ../ops/linalg_ops.cc. @@ -23,7 +23,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/denormal.h" #include "tensorflow/core/platform/logging.h" @@ -95,4 +95,4 @@ class EigOp : public LinearAlgebraOp { } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_EIG_OP_IMPL_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_EIG_OP_IMPL_H_ diff --git a/tensorflow/core/kernels/einsum_op.h b/tensorflow/core/kernels/linalg/einsum_op.h similarity index 94% rename from tensorflow/core/kernels/einsum_op.h rename to tensorflow/core/kernels/linalg/einsum_op.h index 31d1109004c..f22f33c600a 100644 --- a/tensorflow/core/kernels/einsum_op.h +++ b/tensorflow/core/kernels/linalg/einsum_op.h @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_EINSUM_OP_H_ -#define TENSORFLOW_CORE_KERNELS_EINSUM_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_EINSUM_OP_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_EINSUM_OP_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/tensor_types.h" diff --git a/tensorflow/core/kernels/einsum_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/einsum_op_gpu.cu.cc similarity index 96% rename from tensorflow/core/kernels/einsum_op_gpu.cu.cc rename to tensorflow/core/kernels/linalg/einsum_op_gpu.cu.cc index 2935b7fd02a..5461e43e0ab 100644 --- a/tensorflow/core/kernels/einsum_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/einsum_op_gpu.cu.cc @@ -17,7 +17,7 @@ limitations under the License. #define EIGEN_USE_GPU #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/einsum_op.h" +#include "tensorflow/core/kernels/linalg/einsum_op.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/einsum_op_impl.h b/tensorflow/core/kernels/linalg/einsum_op_impl.h similarity index 99% rename from tensorflow/core/kernels/einsum_op_impl.h rename to tensorflow/core/kernels/linalg/einsum_op_impl.h index 312738442b8..b9b2d1f0eae 100644 --- a/tensorflow/core/kernels/einsum_op_impl.h +++ b/tensorflow/core/kernels/linalg/einsum_op_impl.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_EINSUM_OP_IMPL_H_ -#define TENSORFLOW_CORE_KERNELS_EINSUM_OP_IMPL_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_EINSUM_OP_IMPL_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_EINSUM_OP_IMPL_H_ #define EIGEN_USE_THREADS #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM @@ -31,8 +31,8 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/kernels/batch_matmul_op_impl.h" -#include "tensorflow/core/kernels/einsum_op.h" #include "tensorflow/core/kernels/fill_functor.h" +#include "tensorflow/core/kernels/linalg/einsum_op.h" #include "tensorflow/core/kernels/reduction_ops_common.h" #include "tensorflow/core/kernels/transpose_functor.h" #include "tensorflow/core/lib/core/errors.h" @@ -780,4 +780,4 @@ DECLARE_GPU_SPECS(complex128); } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_EINSUM_OP_IMPL_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_EINSUM_OP_IMPL_H_ diff --git a/tensorflow/core/kernels/einsum_op_impl_bfloat16.cc b/tensorflow/core/kernels/linalg/einsum_op_impl_bfloat16.cc similarity index 94% rename from tensorflow/core/kernels/einsum_op_impl_bfloat16.cc rename to tensorflow/core/kernels/linalg/einsum_op_impl_bfloat16.cc index 44508f86a5e..e2e13052df5 100644 --- a/tensorflow/core/kernels/einsum_op_impl_bfloat16.cc +++ b/tensorflow/core/kernels/linalg/einsum_op_impl_bfloat16.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/einsum_op_impl.h" +#include "tensorflow/core/kernels/linalg/einsum_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/einsum_op_impl_complex128.cc b/tensorflow/core/kernels/linalg/einsum_op_impl_complex128.cc similarity index 95% rename from tensorflow/core/kernels/einsum_op_impl_complex128.cc rename to tensorflow/core/kernels/linalg/einsum_op_impl_complex128.cc index 8473cbf545d..ff78d460acf 100644 --- a/tensorflow/core/kernels/einsum_op_impl_complex128.cc +++ b/tensorflow/core/kernels/linalg/einsum_op_impl_complex128.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/einsum_op_impl.h" +#include "tensorflow/core/kernels/linalg/einsum_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/einsum_op_impl_complex64.cc b/tensorflow/core/kernels/linalg/einsum_op_impl_complex64.cc similarity index 95% rename from tensorflow/core/kernels/einsum_op_impl_complex64.cc rename to tensorflow/core/kernels/linalg/einsum_op_impl_complex64.cc index bd506a04f5f..cd3788846b2 100644 --- a/tensorflow/core/kernels/einsum_op_impl_complex64.cc +++ b/tensorflow/core/kernels/linalg/einsum_op_impl_complex64.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/einsum_op_impl.h" +#include "tensorflow/core/kernels/linalg/einsum_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/einsum_op_impl_double.cc b/tensorflow/core/kernels/linalg/einsum_op_impl_double.cc similarity index 95% rename from tensorflow/core/kernels/einsum_op_impl_double.cc rename to tensorflow/core/kernels/linalg/einsum_op_impl_double.cc index f994590779b..e0c093fa4a9 100644 --- a/tensorflow/core/kernels/einsum_op_impl_double.cc +++ b/tensorflow/core/kernels/linalg/einsum_op_impl_double.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/einsum_op_impl.h" +#include "tensorflow/core/kernels/linalg/einsum_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/einsum_op_impl_float.cc b/tensorflow/core/kernels/linalg/einsum_op_impl_float.cc similarity index 95% rename from tensorflow/core/kernels/einsum_op_impl_float.cc rename to tensorflow/core/kernels/linalg/einsum_op_impl_float.cc index 1875310b687..ad9135c991c 100644 --- a/tensorflow/core/kernels/einsum_op_impl_float.cc +++ b/tensorflow/core/kernels/linalg/einsum_op_impl_float.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/einsum_op_impl.h" +#include "tensorflow/core/kernels/linalg/einsum_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/einsum_op_impl_half.cc b/tensorflow/core/kernels/linalg/einsum_op_impl_half.cc similarity index 95% rename from tensorflow/core/kernels/einsum_op_impl_half.cc rename to tensorflow/core/kernels/linalg/einsum_op_impl_half.cc index 0486b133e62..72a9f6bec4f 100644 --- a/tensorflow/core/kernels/einsum_op_impl_half.cc +++ b/tensorflow/core/kernels/linalg/einsum_op_impl_half.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/einsum_op_impl.h" +#include "tensorflow/core/kernels/linalg/einsum_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/einsum_op_impl_int32.cc b/tensorflow/core/kernels/linalg/einsum_op_impl_int32.cc similarity index 94% rename from tensorflow/core/kernels/einsum_op_impl_int32.cc rename to tensorflow/core/kernels/linalg/einsum_op_impl_int32.cc index db5169498d9..7569c979c59 100644 --- a/tensorflow/core/kernels/einsum_op_impl_int32.cc +++ b/tensorflow/core/kernels/linalg/einsum_op_impl_int32.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/einsum_op_impl.h" +#include "tensorflow/core/kernels/linalg/einsum_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/einsum_op_impl_int64.cc b/tensorflow/core/kernels/linalg/einsum_op_impl_int64.cc similarity index 94% rename from tensorflow/core/kernels/einsum_op_impl_int64.cc rename to tensorflow/core/kernels/linalg/einsum_op_impl_int64.cc index 7f1a1eac411..6ee0ebc9637 100644 --- a/tensorflow/core/kernels/einsum_op_impl_int64.cc +++ b/tensorflow/core/kernels/linalg/einsum_op_impl_int64.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/einsum_op_impl.h" +#include "tensorflow/core/kernels/linalg/einsum_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/eye_functor.h b/tensorflow/core/kernels/linalg/eye_functor.h similarity index 90% rename from tensorflow/core/kernels/eye_functor.h rename to tensorflow/core/kernels/linalg/eye_functor.h index 3799cfba9ae..c77372f089a 100644 --- a/tensorflow/core/kernels/eye_functor.h +++ b/tensorflow/core/kernels/linalg/eye_functor.h @@ -12,8 +12,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_ -#define TENSORFLOW_CORE_KERNELS_EYE_FUNCTOR_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_EYE_FUNCTOR_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_EYE_FUNCTOR_H_ #include "tensorflow/core/framework/tensor_types.h" diff --git a/tensorflow/core/kernels/eye_functor_gpu.cu.cc b/tensorflow/core/kernels/linalg/eye_functor_gpu.cu.cc similarity index 97% rename from tensorflow/core/kernels/eye_functor_gpu.cu.cc rename to tensorflow/core/kernels/linalg/eye_functor_gpu.cu.cc index 90df538dd2c..85865588f2c 100644 --- a/tensorflow/core/kernels/eye_functor_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/eye_functor_gpu.cu.cc @@ -20,7 +20,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/type_traits.h" -#include "tensorflow/core/kernels/eye_functor.h" +#include "tensorflow/core/kernels/linalg/eye_functor.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/linalg_ops_common.cc b/tensorflow/core/kernels/linalg/linalg_ops_common.cc similarity index 99% rename from tensorflow/core/kernels/linalg_ops_common.cc rename to tensorflow/core/kernels/linalg/linalg_ops_common.cc index 56a941fbd1f..c8d33e435c7 100644 --- a/tensorflow/core/kernels/linalg_ops_common.cc +++ b/tensorflow/core/kernels/linalg/linalg_ops_common.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include diff --git a/tensorflow/core/kernels/linalg/linalg_ops_common.h b/tensorflow/core/kernels/linalg/linalg_ops_common.h new file mode 100644 index 00000000000..3ab37480c90 --- /dev/null +++ b/tensorflow/core/kernels/linalg/linalg_ops_common.h @@ -0,0 +1,221 @@ +/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_LINALG_OPS_COMMON_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_LINALG_OPS_COMMON_H_ + +// Classes to support linear algebra functionality, similar to the numpy.linalg +// module. Supports batch computation on several matrices at once, sharding the +// computations across different threads if necessary. +#include + +#include "third_party/eigen3/Eigen/Core" +#include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_types.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/gtl/inlined_vector.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/work_sharder.h" + +namespace tensorflow { + +// Base class for linear algebra operators. +template +class LinearAlgebraOp : public OpKernel { + public: + explicit LinearAlgebraOp(OpKernelConstruction* context) : OpKernel(context) {} + + void Compute(OpKernelContext* context) override; + + protected: + using TensorShapes = gtl::InlinedVector; + // Returns the number of leading inputs that are to be treated as matrix + // inputs. By default this is all the inputs. Derived classes can override + // this to tell the base class to ignore one or more trailing inputs. + virtual int NumMatrixInputs(const OpKernelContext* context) const { + return context->num_inputs(); + } + + // Returns true if the number of inputs and their shapes are as expected. + // Many ops take a single square input matrix, so we provide that as a default + // implementation for convenience. + virtual void ValidateInputMatrixShapes( + OpKernelContext* context, const TensorShapes& input_matrix_shapes) const { + ValidateSingleSquareMatrix(context, input_matrix_shapes); + } + + // Convenience validators for common cases: + // + // Validate op taking a single matrix A. + static void ValidateSingleMatrix(OpKernelContext* context, + const TensorShapes& input_matrix_shapes); + // Validate op taking a single square matrix A. + static void ValidateSingleSquareMatrix( + OpKernelContext* context, const TensorShapes& input_matrix_shapes); + // Validate op taking two matrices A and B that have the same number of rows. + static void ValidateSolver(OpKernelContext* context, + const TensorShapes& input_matrix_shapes); + // Validate op taking two matrices A and B that have the same number of rows + // and A is square. + static void ValidateSquareSolver(OpKernelContext* context, + const TensorShapes& input_matrix_shapes); + + // Returns the output shapes of each individual matrix operation. Output + // matrices shapes must be rank 0, 1, or 2. Scalar outputs are rank 0. + // + // The derived class may return a number of shapes (N) less than + // context->num_outputs() (M) to indicate that a only leading subset of + // the outputs will be populated. In this case, a dummy scalar tensor with + // value zero will be return for the last M-N outputs. + // + // For many ops, the output dimensions are the same as the input dimensions, + // so we provide that as a default implementation for convenience. + virtual TensorShapes GetOutputMatrixShapes( + const TensorShapes& input_matrix_shapes) const { + return input_matrix_shapes; + } + + // Returns the cost per matrix operation. This is used to determine the + // number of threads to use for parallelizing calls to ComputeMatrix in + // batch mode. Cost per unit is assumed to be roughly 1ns, based on comments + // in core/util/work_sharder.cc. Many linear algebra ops take roughly max(m,n) + // * min(m,n)^2, where the first input matrix is m-by-n. We provide that as a + // default implementation for convenience. + virtual int64 GetCostPerUnit(const TensorShapes& input_matrix_shapes) const { + double m = static_cast(input_matrix_shapes[0].dim_size(0)); + double n = static_cast(input_matrix_shapes[0].dim_size(1)); + double cost = std::max(m, n) * std::min(m, n) * std::min(m, n); + return cost >= static_cast(kint64max) ? kint64max + : static_cast(cost); + } + + // Returns true if it is safe to forward (alias) input to output buffer + // and expect the kernel to perform the computation inplace. + virtual bool EnableInputForwarding() const { return true; } + + using InputMatrix = Eigen::Matrix; + using InputConstMatrixMap = Eigen::Map; + using InputMatrixMap = Eigen::Map; + using InputConstVectorMap = + Eigen::Map>; + using InputConstMatrixMaps = gtl::InlinedVector; + using InputMatrixMaps = gtl::InlinedVector; + using InputRealScalar = typename Eigen::NumTraits::Real; + + using OutputMatrix = Eigen::Matrix; + using OutputConstMatrixMap = Eigen::Map; + using OutputMatrixMap = Eigen::Map; + using OutputConstVectorMap = + Eigen::Map>; + using OutputConstMatrixMaps = gtl::InlinedVector; + using OutputMatrixMaps = gtl::InlinedVector; + using OutputRealScalar = typename Eigen::NumTraits::Real; + + // backward compatibility + using Scalar = OutputScalar; + using Matrix = + Eigen::Matrix; + using ConstMatrixMap = Eigen::Map; + using MatrixMap = Eigen::Map; + using ConstVectorMap = + Eigen::Map>; + using ConstMatrixMaps = gtl::InlinedVector; + using MatrixMaps = gtl::InlinedVector; + using RealScalar = typename Eigen::NumTraits::Real; + + // Performs a single matrix computation given input matrices, and + // stores the result in outputs. For batch operations, this will be called + // repeatedly for a single call to Compute() when multiple matrices exist in + // input Tensors with rank > 2. In this case the calls to ComputeMatrix are + // parallelized. The number of threads used is determined by a cost model from + // the value returned by GetCostPerUnit(). + virtual void ComputeMatrix(OpKernelContext* context, + const InputConstMatrixMaps& inputs, + OutputMatrixMaps* outputs) = 0; + + private: + using TensorInputs = gtl::InlinedVector; + using TensorOutputs = gtl::InlinedVector; + // This function maps 2-d slices (matrices) of the input and output tensors + // using Eigen::Map and calls ComputeMatrix implemented in terms of the + // Eigen::MatrixBase API by the derived class. + // + // The 'matrix_index' parameter specifies the index of the matrix to be used + // from each input tensor, and the index of the matrix to be written to each + // output tensor. The input matrices are in row major order, and located at + // the memory addresses + // inputs[i].flat().data() + + // matrix_index * input_matrix_shapes[i].num_elements() + // for i in 0...inputs.size()-1. + // The output matrices are in row major order, and located at the memory + // address + // outputs[i]->flat().data() + + // matrix_index * output_matrix_shapes[i].num_elements(). + // for i in 0...outputs.size()-1. + // + void ComputeTensorSlice(OpKernelContext* context, int64 matrix_index, + const TensorInputs& inputs, + const TensorShapes& input_matrix_shapes, + const TensorOutputs& outputs, + const TensorShapes& output_matrix_shapes); + + void AnalyzeInputs(OpKernelContext* context, TensorInputs* inputs, + TensorShapes* input_matrix_shapes, + TensorShape* batch_shape); + + void PrepareOutputs(OpKernelContext* context, + const TensorShapes& input_matrix_shapes, + const TensorShape& batch_shape, TensorOutputs* outputs, + TensorShapes* output_matrix_shapes); +}; + +// Declare LinearAlgebraOp, which is explicitly instantiated in +// linalg_ops_common.cc for float, double, complex64, and complex128. +extern template class LinearAlgebraOp; +extern template class LinearAlgebraOp; +extern template class LinearAlgebraOp; +extern template class LinearAlgebraOp; + +} // namespace tensorflow + +#define INHERIT_LINALG_TYPEDEFS(Scalar) \ + typedef LinearAlgebraOp Base; \ + using RealScalar = typename Eigen::NumTraits::Real; \ + using Matrix = typename Base::Matrix; \ + using MatrixMap = typename Base::MatrixMap; \ + using MatrixMaps = typename Base::MatrixMaps; \ + using ConstMatrixMap = typename Base::ConstMatrixMap; \ + using ConstMatrixMaps = typename Base::ConstMatrixMaps; \ + using ConstVectorMap = typename Base::ConstVectorMap; \ + using TensorShapes = typename Base::TensorShapes; + +#define REGISTER_LINALG_OP_CPU(OpName, OpClass, Scalar) \ + REGISTER_KERNEL_BUILDER( \ + Name(OpName).Device(DEVICE_CPU).TypeConstraint("T"), OpClass) + +#define REGISTER_LINALG_OP_GPU(OpName, OpClass, Scalar) \ + REGISTER_KERNEL_BUILDER( \ + Name(OpName).Device(DEVICE_GPU).TypeConstraint("T"), OpClass) + +// Deprecated, use one of the device-specific macros above. +#define REGISTER_LINALG_OP(OpName, OpClass, Scalar) \ + REGISTER_LINALG_OP_CPU(OpName, OpClass, Scalar) + +#endif // TENSORFLOW_CORE_KERNELS_LINALG_LINALG_OPS_COMMON_H_ diff --git a/tensorflow/core/kernels/lu_op.cc b/tensorflow/core/kernels/linalg/lu_op.cc similarity index 100% rename from tensorflow/core/kernels/lu_op.cc rename to tensorflow/core/kernels/linalg/lu_op.cc diff --git a/tensorflow/core/kernels/lu_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/lu_op_gpu.cu.cc similarity index 99% rename from tensorflow/core/kernels/lu_op_gpu.cu.cc rename to tensorflow/core/kernels/linalg/lu_op_gpu.cu.cc index 47b37ed7f7a..9d23a35057d 100644 --- a/tensorflow/core/kernels/lu_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/lu_op_gpu.cu.cc @@ -25,9 +25,9 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/kernels/transpose_functor.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_solvers.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_band_part_op.cc b/tensorflow/core/kernels/linalg/matrix_band_part_op.cc similarity index 99% rename from tensorflow/core/kernels/matrix_band_part_op.cc rename to tensorflow/core/kernels/linalg/matrix_band_part_op.cc index 4dcce5a8f58..23619bacc33 100644 --- a/tensorflow/core/kernels/matrix_band_part_op.cc +++ b/tensorflow/core/kernels/linalg/matrix_band_part_op.cc @@ -21,11 +21,12 @@ limitations under the License. #define EIGEN_USE_GPU #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/matrix_band_part_op.h" +#include "tensorflow/core/kernels/linalg/matrix_band_part_op.h" #include #include #include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" diff --git a/tensorflow/core/kernels/matrix_band_part_op.h b/tensorflow/core/kernels/linalg/matrix_band_part_op.h similarity index 86% rename from tensorflow/core/kernels/matrix_band_part_op.h rename to tensorflow/core/kernels/linalg/matrix_band_part_op.h index b04e36db8ed..2f68eba6dcd 100644 --- a/tensorflow/core/kernels/matrix_band_part_op.h +++ b/tensorflow/core/kernels/linalg/matrix_band_part_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MATRIX_BAND_PART_OP_H_ -#define TENSORFLOW_CORE_KERNELS_MATRIX_BAND_PART_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_BAND_PART_OP_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_BAND_PART_OP_H_ #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_types.h" @@ -34,4 +34,4 @@ struct MatrixBandPartFunctor { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_MATRIX_BAND_PART_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_BAND_PART_OP_H_ diff --git a/tensorflow/core/kernels/matrix_band_part_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/matrix_band_part_op_gpu.cu.cc similarity index 97% rename from tensorflow/core/kernels/matrix_band_part_op_gpu.cu.cc rename to tensorflow/core/kernels/linalg/matrix_band_part_op_gpu.cu.cc index 9eb3e4f72a2..9c734b7fd6e 100644 --- a/tensorflow/core/kernels/matrix_band_part_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/matrix_band_part_op_gpu.cu.cc @@ -21,7 +21,7 @@ limitations under the License. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/matrix_band_part_op.h" +#include "tensorflow/core/kernels/linalg/matrix_band_part_op.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_diag_op.cc b/tensorflow/core/kernels/linalg/matrix_diag_op.cc similarity index 99% rename from tensorflow/core/kernels/matrix_diag_op.cc rename to tensorflow/core/kernels/linalg/matrix_diag_op.cc index 05d7e4e6f86..69cc8170793 100644 --- a/tensorflow/core/kernels/matrix_diag_op.cc +++ b/tensorflow/core/kernels/linalg/matrix_diag_op.cc @@ -20,7 +20,7 @@ limitations under the License. #define EIGEN_USE_GPU #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/matrix_diag_op.h" +#include "tensorflow/core/kernels/linalg/matrix_diag_op.h" #include #include diff --git a/tensorflow/core/kernels/matrix_diag_op.h b/tensorflow/core/kernels/linalg/matrix_diag_op.h similarity index 94% rename from tensorflow/core/kernels/matrix_diag_op.h rename to tensorflow/core/kernels/linalg/matrix_diag_op.h index 707fd9b6c14..5758ba664cc 100644 --- a/tensorflow/core/kernels/matrix_diag_op.h +++ b/tensorflow/core/kernels/linalg/matrix_diag_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MATRIX_DIAG_OP_H_ -#define TENSORFLOW_CORE_KERNELS_MATRIX_DIAG_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_DIAG_OP_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_DIAG_OP_H_ // Generator definition for MatrixDiagOp, must be compilable by nvcc. @@ -69,4 +69,4 @@ struct MatrixDiag { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_MATRIX_DIAG_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_DIAG_OP_H_ diff --git a/tensorflow/core/kernels/matrix_diag_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/matrix_diag_op_gpu.cu.cc similarity index 99% rename from tensorflow/core/kernels/matrix_diag_op_gpu.cu.cc rename to tensorflow/core/kernels/linalg/matrix_diag_op_gpu.cu.cc index 76271798d5f..6b52e70716d 100644 --- a/tensorflow/core/kernels/matrix_diag_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/matrix_diag_op_gpu.cu.cc @@ -18,7 +18,7 @@ limitations under the License. #define EIGEN_USE_GPU #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/matrix_diag_op.h" +#include "tensorflow/core/kernels/linalg/matrix_diag_op.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_exponential_op.cc b/tensorflow/core/kernels/linalg/matrix_exponential_op.cc similarity index 97% rename from tensorflow/core/kernels/matrix_exponential_op.cc rename to tensorflow/core/kernels/linalg/matrix_exponential_op.cc index 01d4894438c..73407614955 100644 --- a/tensorflow/core/kernels/matrix_exponential_op.cc +++ b/tensorflow/core/kernels/linalg/matrix_exponential_op.cc @@ -20,7 +20,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" diff --git a/tensorflow/core/kernels/matrix_inverse_op.cc b/tensorflow/core/kernels/linalg/matrix_inverse_op.cc similarity index 98% rename from tensorflow/core/kernels/matrix_inverse_op.cc rename to tensorflow/core/kernels/linalg/matrix_inverse_op.cc index 52afdd15ba6..dc51776f2fe 100644 --- a/tensorflow/core/kernels/matrix_inverse_op.cc +++ b/tensorflow/core/kernels/linalg/matrix_inverse_op.cc @@ -24,7 +24,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" @@ -32,9 +32,9 @@ limitations under the License. #if GOOGLE_CUDA #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/eye_functor.h" +#include "tensorflow/core/kernels/linalg/eye_functor.h" #include "tensorflow/core/kernels/transpose_functor.h" +#include "tensorflow/core/util/cuda_solvers.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_logarithm_op.cc b/tensorflow/core/kernels/linalg/matrix_logarithm_op.cc similarity index 97% rename from tensorflow/core/kernels/matrix_logarithm_op.cc rename to tensorflow/core/kernels/linalg/matrix_logarithm_op.cc index 22ca094e243..79d5472f140 100644 --- a/tensorflow/core/kernels/matrix_logarithm_op.cc +++ b/tensorflow/core/kernels/linalg/matrix_logarithm_op.cc @@ -20,7 +20,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" diff --git a/tensorflow/core/kernels/matrix_set_diag_op.cc b/tensorflow/core/kernels/linalg/matrix_set_diag_op.cc similarity index 99% rename from tensorflow/core/kernels/matrix_set_diag_op.cc rename to tensorflow/core/kernels/linalg/matrix_set_diag_op.cc index bf98fd0d47d..df32228d0f2 100644 --- a/tensorflow/core/kernels/matrix_set_diag_op.cc +++ b/tensorflow/core/kernels/linalg/matrix_set_diag_op.cc @@ -21,7 +21,7 @@ limitations under the License. #define EIGEN_USE_GPU #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/matrix_set_diag_op.h" +#include "tensorflow/core/kernels/linalg/matrix_set_diag_op.h" #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" @@ -30,7 +30,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/matrix_diag_op.h" +#include "tensorflow/core/kernels/linalg/matrix_diag_op.h" #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" diff --git a/tensorflow/core/kernels/matrix_set_diag_op.h b/tensorflow/core/kernels/linalg/matrix_set_diag_op.h similarity index 89% rename from tensorflow/core/kernels/matrix_set_diag_op.h rename to tensorflow/core/kernels/linalg/matrix_set_diag_op.h index 04877cd34ca..449a3607ede 100644 --- a/tensorflow/core/kernels/matrix_set_diag_op.h +++ b/tensorflow/core/kernels/linalg/matrix_set_diag_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MATRIX_SET_DIAG_OP_H_ -#define TENSORFLOW_CORE_KERNELS_MATRIX_SET_DIAG_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_SET_DIAG_OP_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_SET_DIAG_OP_H_ #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_types.h" @@ -39,4 +39,4 @@ struct MatrixSetDiag { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_MATRIX_SET_DIAG_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_SET_DIAG_OP_H_ diff --git a/tensorflow/core/kernels/matrix_set_diag_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/matrix_set_diag_op_gpu.cu.cc similarity index 99% rename from tensorflow/core/kernels/matrix_set_diag_op_gpu.cu.cc rename to tensorflow/core/kernels/linalg/matrix_set_diag_op_gpu.cu.cc index 4e32f8a52e8..0cdb457db03 100644 --- a/tensorflow/core/kernels/matrix_set_diag_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/matrix_set_diag_op_gpu.cu.cc @@ -18,7 +18,7 @@ limitations under the License. #define EIGEN_USE_GPU #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/matrix_set_diag_op.h" +#include "tensorflow/core/kernels/linalg/matrix_set_diag_op.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_solve_ls_op_complex128.cc b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_complex128.cc similarity index 92% rename from tensorflow/core/kernels/matrix_solve_ls_op_complex128.cc rename to tensorflow/core/kernels/linalg/matrix_solve_ls_op_complex128.cc index 22274cc3daf..4e64eb42371 100644 --- a/tensorflow/core/kernels/matrix_solve_ls_op_complex128.cc +++ b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_complex128.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/matrix_solve_ls_op_impl.h" +#include "tensorflow/core/kernels/linalg/matrix_solve_ls_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_solve_ls_op_complex64.cc b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_complex64.cc similarity index 92% rename from tensorflow/core/kernels/matrix_solve_ls_op_complex64.cc rename to tensorflow/core/kernels/linalg/matrix_solve_ls_op_complex64.cc index c8421a3efba..719201f3f9e 100644 --- a/tensorflow/core/kernels/matrix_solve_ls_op_complex64.cc +++ b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_complex64.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/matrix_solve_ls_op_impl.h" +#include "tensorflow/core/kernels/linalg/matrix_solve_ls_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_solve_ls_op_double.cc b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_double.cc similarity index 92% rename from tensorflow/core/kernels/matrix_solve_ls_op_double.cc rename to tensorflow/core/kernels/linalg/matrix_solve_ls_op_double.cc index c7d03cb1052..614ecee4e23 100644 --- a/tensorflow/core/kernels/matrix_solve_ls_op_double.cc +++ b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_double.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/matrix_solve_ls_op_impl.h" +#include "tensorflow/core/kernels/linalg/matrix_solve_ls_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_solve_ls_op_float.cc b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_float.cc similarity index 92% rename from tensorflow/core/kernels/matrix_solve_ls_op_float.cc rename to tensorflow/core/kernels/linalg/matrix_solve_ls_op_float.cc index c98a84beded..809cff8148c 100644 --- a/tensorflow/core/kernels/matrix_solve_ls_op_float.cc +++ b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_float.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/matrix_solve_ls_op_impl.h" +#include "tensorflow/core/kernels/linalg/matrix_solve_ls_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_solve_ls_op_impl.h b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_impl.h similarity index 96% rename from tensorflow/core/kernels/matrix_solve_ls_op_impl.h rename to tensorflow/core/kernels/linalg/matrix_solve_ls_op_impl.h index 00a05a87a3a..1c8101a05b4 100644 --- a/tensorflow/core/kernels/matrix_solve_ls_op_impl.h +++ b/tensorflow/core/kernels/linalg/matrix_solve_ls_op_impl.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MATRIX_SOLVE_LS_OP_IMPL_H_ -#define TENSORFLOW_CORE_KERNELS_MATRIX_SOLVE_LS_OP_IMPL_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_SOLVE_LS_OP_IMPL_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_SOLVE_LS_OP_IMPL_H_ // See docs in ../ops/linalg_ops.cc. @@ -24,7 +24,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" @@ -163,4 +163,4 @@ class MatrixSolveLsOp : public LinearAlgebraOp { } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_MATRIX_SOLVE_LS_OP_IMPL_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_SOLVE_LS_OP_IMPL_H_ diff --git a/tensorflow/core/kernels/matrix_solve_op.cc b/tensorflow/core/kernels/linalg/matrix_solve_op.cc similarity index 99% rename from tensorflow/core/kernels/matrix_solve_op.cc rename to tensorflow/core/kernels/linalg/matrix_solve_op.cc index 3a75054f4ea..70f02bddf9b 100644 --- a/tensorflow/core/kernels/matrix_solve_op.cc +++ b/tensorflow/core/kernels/linalg/matrix_solve_op.cc @@ -25,7 +25,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" @@ -33,8 +33,8 @@ limitations under the License. #if GOOGLE_CUDA #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/kernels/transpose_functor.h" +#include "tensorflow/core/util/cuda_solvers.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_square_root_op.cc b/tensorflow/core/kernels/linalg/matrix_square_root_op.cc similarity index 97% rename from tensorflow/core/kernels/matrix_square_root_op.cc rename to tensorflow/core/kernels/linalg/matrix_square_root_op.cc index fe3d3043c26..ce43e358350 100644 --- a/tensorflow/core/kernels/matrix_square_root_op.cc +++ b/tensorflow/core/kernels/linalg/matrix_square_root_op.cc @@ -20,7 +20,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" diff --git a/tensorflow/core/kernels/matrix_triangular_solve_op_complex.cc b/tensorflow/core/kernels/linalg/matrix_triangular_solve_op_complex.cc similarity index 92% rename from tensorflow/core/kernels/matrix_triangular_solve_op_complex.cc rename to tensorflow/core/kernels/linalg/matrix_triangular_solve_op_complex.cc index ae3702078a0..27f3e77e29c 100644 --- a/tensorflow/core/kernels/matrix_triangular_solve_op_complex.cc +++ b/tensorflow/core/kernels/linalg/matrix_triangular_solve_op_complex.cc @@ -14,7 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/matrix_triangular_solve_op_impl.h" +#include "tensorflow/core/kernels/linalg/matrix_triangular_solve_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h b/tensorflow/core/kernels/linalg/matrix_triangular_solve_op_impl.h similarity index 97% rename from tensorflow/core/kernels/matrix_triangular_solve_op_impl.h rename to tensorflow/core/kernels/linalg/matrix_triangular_solve_op_impl.h index fb7e6f0f5ff..99249f792b6 100644 --- a/tensorflow/core/kernels/matrix_triangular_solve_op_impl.h +++ b/tensorflow/core/kernels/linalg/matrix_triangular_solve_op_impl.h @@ -15,8 +15,8 @@ limitations under the License. // See docs in ../ops/linalg_ops.cc. // -#ifndef TENSORFLOW_CORE_KERNELS_MATRIX_TRIANGULAR_SOLVE_OP_IMPL_H_ -#define TENSORFLOW_CORE_KERNELS_MATRIX_TRIANGULAR_SOLVE_OP_IMPL_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_TRIANGULAR_SOLVE_OP_IMPL_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_TRIANGULAR_SOLVE_OP_IMPL_H_ #include "third_party/eigen3/Eigen/Core" #include "tensorflow/core/framework/kernel_def_builder.h" @@ -24,7 +24,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/fill_functor.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" @@ -38,9 +38,9 @@ limitations under the License. #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM #if GOOGLE_CUDA -#include "tensorflow/core/kernels/cuda_solvers.h" +#include "tensorflow/core/util/cuda_solvers.h" #elif TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/rocm_solvers.h" +#include "tensorflow/core/util/rocm_solvers.h" #endif namespace tensorflow { @@ -434,4 +434,4 @@ struct LaunchBatchMatrixTriangularSolve { } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_MATRIX_TRIANGULAR_SOLVE_OP_IMPL_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_MATRIX_TRIANGULAR_SOLVE_OP_IMPL_H_ diff --git a/tensorflow/core/kernels/matrix_triangular_solve_op_real.cc b/tensorflow/core/kernels/linalg/matrix_triangular_solve_op_real.cc similarity index 93% rename from tensorflow/core/kernels/matrix_triangular_solve_op_real.cc rename to tensorflow/core/kernels/linalg/matrix_triangular_solve_op_real.cc index 0f92964dd72..71a62441dc4 100644 --- a/tensorflow/core/kernels/matrix_triangular_solve_op_real.cc +++ b/tensorflow/core/kernels/linalg/matrix_triangular_solve_op_real.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/matrix_triangular_solve_op_impl.h" +#include "tensorflow/core/kernels/linalg/matrix_triangular_solve_op_impl.h" #if GOOGLE_CUDA #include "third_party/gpus/cuda/include/cuda.h" diff --git a/tensorflow/core/kernels/matrix_triangular_solve_op_test.cc b/tensorflow/core/kernels/linalg/matrix_triangular_solve_op_test.cc similarity index 100% rename from tensorflow/core/kernels/matrix_triangular_solve_op_test.cc rename to tensorflow/core/kernels/linalg/matrix_triangular_solve_op_test.cc diff --git a/tensorflow/core/kernels/qr_op_complex128.cc b/tensorflow/core/kernels/linalg/qr_op_complex128.cc similarity index 96% rename from tensorflow/core/kernels/qr_op_complex128.cc rename to tensorflow/core/kernels/linalg/qr_op_complex128.cc index 8a3e3dc0a92..0c14c6d2818 100644 --- a/tensorflow/core/kernels/qr_op_complex128.cc +++ b/tensorflow/core/kernels/linalg/qr_op_complex128.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/qr_op_impl.h" +#include "tensorflow/core/kernels/linalg/qr_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/qr_op_complex64.cc b/tensorflow/core/kernels/linalg/qr_op_complex64.cc similarity index 95% rename from tensorflow/core/kernels/qr_op_complex64.cc rename to tensorflow/core/kernels/linalg/qr_op_complex64.cc index 467fa6c2d6a..fc0227ef7f9 100644 --- a/tensorflow/core/kernels/qr_op_complex64.cc +++ b/tensorflow/core/kernels/linalg/qr_op_complex64.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/qr_op_impl.h" +#include "tensorflow/core/kernels/linalg/qr_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/qr_op_double.cc b/tensorflow/core/kernels/linalg/qr_op_double.cc similarity index 96% rename from tensorflow/core/kernels/qr_op_double.cc rename to tensorflow/core/kernels/linalg/qr_op_double.cc index 05537a0eaa3..ae00b3e7921 100644 --- a/tensorflow/core/kernels/qr_op_double.cc +++ b/tensorflow/core/kernels/linalg/qr_op_double.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/qr_op_impl.h" +#include "tensorflow/core/kernels/linalg/qr_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/qr_op_float.cc b/tensorflow/core/kernels/linalg/qr_op_float.cc similarity index 96% rename from tensorflow/core/kernels/qr_op_float.cc rename to tensorflow/core/kernels/linalg/qr_op_float.cc index 6aebd981865..77b8eeb0286 100644 --- a/tensorflow/core/kernels/qr_op_float.cc +++ b/tensorflow/core/kernels/linalg/qr_op_float.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/qr_op_impl.h" +#include "tensorflow/core/kernels/linalg/qr_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/qr_op_impl.h b/tensorflow/core/kernels/linalg/qr_op_impl.h similarity index 96% rename from tensorflow/core/kernels/qr_op_impl.h rename to tensorflow/core/kernels/linalg/qr_op_impl.h index 535df9d160d..876594bc511 100644 --- a/tensorflow/core/kernels/qr_op_impl.h +++ b/tensorflow/core/kernels/linalg/qr_op_impl.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_QR_OP_IMPL_H_ -#define TENSORFLOW_CORE_KERNELS_QR_OP_IMPL_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_QR_OP_IMPL_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_QR_OP_IMPL_H_ // See docs in ../ops/linalg_ops.cc. // @@ -33,7 +33,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" @@ -41,11 +41,11 @@ limitations under the License. #if GOOGLE_CUDA #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" -#include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/kernels/cwise_ops.h" -#include "tensorflow/core/kernels/eye_functor.h" -#include "tensorflow/core/kernels/matrix_band_part_op.h" +#include "tensorflow/core/kernels/linalg/eye_functor.h" +#include "tensorflow/core/kernels/linalg/matrix_band_part_op.h" #include "tensorflow/core/kernels/transpose_functor.h" +#include "tensorflow/core/util/cuda_solvers.h" #endif namespace tensorflow { @@ -299,4 +299,4 @@ class QrOpGpu : public AsyncOpKernel { } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_QR_OP_IMPL_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_QR_OP_IMPL_H_ diff --git a/tensorflow/core/kernels/self_adjoint_eig_op.cc b/tensorflow/core/kernels/linalg/self_adjoint_eig_op.cc similarity index 98% rename from tensorflow/core/kernels/self_adjoint_eig_op.cc rename to tensorflow/core/kernels/linalg/self_adjoint_eig_op.cc index cea5883db7b..ebf1955b8ff 100644 --- a/tensorflow/core/kernels/self_adjoint_eig_op.cc +++ b/tensorflow/core/kernels/linalg/self_adjoint_eig_op.cc @@ -20,7 +20,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/denormal.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_complex128.cc b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_complex128.cc similarity index 93% rename from tensorflow/core/kernels/self_adjoint_eig_v2_op_complex128.cc rename to tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_complex128.cc index 4c7a391d56c..424c33a7ac1 100644 --- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_complex128.cc +++ b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_complex128.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h" +#include "tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_complex64.cc b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_complex64.cc similarity index 93% rename from tensorflow/core/kernels/self_adjoint_eig_v2_op_complex64.cc rename to tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_complex64.cc index 0ec5ec24dd1..bdd20998e3c 100644 --- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_complex64.cc +++ b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_complex64.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h" +#include "tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_double.cc b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_double.cc similarity index 92% rename from tensorflow/core/kernels/self_adjoint_eig_v2_op_double.cc rename to tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_double.cc index 7f81bb69021..afc50500d40 100644 --- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_double.cc +++ b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_double.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h" +#include "tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_float.cc b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_float.cc similarity index 92% rename from tensorflow/core/kernels/self_adjoint_eig_v2_op_float.cc rename to tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_float.cc index bf30952d1e7..1f795777a2e 100644 --- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_float.cc +++ b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_float.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h" +#include "tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_gpu.cc similarity index 99% rename from tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc rename to tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_gpu.cc index 3a84df07a9a..778c50ff408 100644 --- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_gpu.cc +++ b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_gpu.cc @@ -26,12 +26,12 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/kernels/cast_op.h" -#include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/kernels/cwise_ops.h" #include "tensorflow/core/kernels/transpose_functor.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_solvers.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_impl.h similarity index 91% rename from tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h rename to tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_impl.h index b5274f8788b..56f2936a66e 100644 --- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h +++ b/tensorflow/core/kernels/linalg/self_adjoint_eig_v2_op_impl.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_SELF_ADJOINT_EIG_V2_OP_IMPL_H_ -#define TENSORFLOW_CORE_KERNELS_SELF_ADJOINT_EIG_V2_OP_IMPL_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_SELF_ADJOINT_EIG_V2_OP_IMPL_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_SELF_ADJOINT_EIG_V2_OP_IMPL_H_ // See docs in ../ops/linalg_ops.cc. @@ -23,7 +23,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/denormal.h" #include "tensorflow/core/platform/logging.h" @@ -89,4 +89,4 @@ class SelfAdjointEigV2Op : public LinearAlgebraOp { } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_SELF_ADJOINT_EIG_V2_OP_IMPL_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_SELF_ADJOINT_EIG_V2_OP_IMPL_H_ diff --git a/tensorflow/core/kernels/svd_op_complex128.cc b/tensorflow/core/kernels/linalg/svd_op_complex128.cc similarity index 93% rename from tensorflow/core/kernels/svd_op_complex128.cc rename to tensorflow/core/kernels/linalg/svd_op_complex128.cc index a0f39418aca..36ac629e38a 100644 --- a/tensorflow/core/kernels/svd_op_complex128.cc +++ b/tensorflow/core/kernels/linalg/svd_op_complex128.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/svd_op_impl.h" +#include "tensorflow/core/kernels/linalg/svd_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/svd_op_complex64.cc b/tensorflow/core/kernels/linalg/svd_op_complex64.cc similarity index 93% rename from tensorflow/core/kernels/svd_op_complex64.cc rename to tensorflow/core/kernels/linalg/svd_op_complex64.cc index a8fd50c67d1..50d940b534a 100644 --- a/tensorflow/core/kernels/svd_op_complex64.cc +++ b/tensorflow/core/kernels/linalg/svd_op_complex64.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/svd_op_impl.h" +#include "tensorflow/core/kernels/linalg/svd_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/svd_op_double.cc b/tensorflow/core/kernels/linalg/svd_op_double.cc similarity index 93% rename from tensorflow/core/kernels/svd_op_double.cc rename to tensorflow/core/kernels/linalg/svd_op_double.cc index 539dae3a081..85bbe08d8c9 100644 --- a/tensorflow/core/kernels/svd_op_double.cc +++ b/tensorflow/core/kernels/linalg/svd_op_double.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/svd_op_impl.h" +#include "tensorflow/core/kernels/linalg/svd_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/svd_op_float.cc b/tensorflow/core/kernels/linalg/svd_op_float.cc similarity index 93% rename from tensorflow/core/kernels/svd_op_float.cc rename to tensorflow/core/kernels/linalg/svd_op_float.cc index 03839aa49c3..961d131293b 100644 --- a/tensorflow/core/kernels/svd_op_float.cc +++ b/tensorflow/core/kernels/linalg/svd_op_float.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/svd_op_impl.h" +#include "tensorflow/core/kernels/linalg/svd_op_impl.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/svd_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/svd_op_gpu.cu.cc similarity index 99% rename from tensorflow/core/kernels/svd_op_gpu.cu.cc rename to tensorflow/core/kernels/linalg/svd_op_gpu.cu.cc index 482fd057e4e..06d1efe6dd5 100644 --- a/tensorflow/core/kernels/svd_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/svd_op_gpu.cu.cc @@ -36,14 +36,14 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/eye_functor.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/eye_functor.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/kernels/transpose_functor.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/stream_executor.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_solvers.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/svd_op_impl.h b/tensorflow/core/kernels/linalg/svd_op_impl.h similarity index 95% rename from tensorflow/core/kernels/svd_op_impl.h rename to tensorflow/core/kernels/linalg/svd_op_impl.h index 675826a057c..c43aaaa4b7b 100644 --- a/tensorflow/core/kernels/svd_op_impl.h +++ b/tensorflow/core/kernels/linalg/svd_op_impl.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_SVD_OP_IMPL_H_ -#define TENSORFLOW_CORE_KERNELS_SVD_OP_IMPL_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_SVD_OP_IMPL_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_SVD_OP_IMPL_H_ // See docs in ../ops/linalg_ops.cc. // @@ -27,7 +27,7 @@ limitations under the License. #include "tensorflow/core/framework/kernel_def_builder.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" @@ -118,4 +118,4 @@ class SvdOp : public LinearAlgebraOp { } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_SVD_OP_IMPL_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_SVD_OP_IMPL_H_ diff --git a/tensorflow/core/kernels/tridiagonal_matmul_op.cc b/tensorflow/core/kernels/linalg/tridiagonal_matmul_op.cc similarity index 98% rename from tensorflow/core/kernels/tridiagonal_matmul_op.cc rename to tensorflow/core/kernels/linalg/tridiagonal_matmul_op.cc index 3ddf22012de..9d17c574148 100644 --- a/tensorflow/core/kernels/tridiagonal_matmul_op.cc +++ b/tensorflow/core/kernels/linalg/tridiagonal_matmul_op.cc @@ -19,7 +19,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/types.h" diff --git a/tensorflow/core/kernels/tridiagonal_matmul_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/tridiagonal_matmul_op_gpu.cu.cc similarity index 96% rename from tensorflow/core/kernels/tridiagonal_matmul_op_gpu.cu.cc rename to tensorflow/core/kernels/linalg/tridiagonal_matmul_op_gpu.cu.cc index 1c82cc18e32..a65db40d822 100644 --- a/tensorflow/core/kernels/tridiagonal_matmul_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/tridiagonal_matmul_op_gpu.cu.cc @@ -22,11 +22,11 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/kernels/transpose_functor.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #include "tensorflow/core/util/gpu_device_functions.h" #include "tensorflow/core/util/gpu_kernel_helper.h" #include "tensorflow/core/util/gpu_launch_config.h" diff --git a/tensorflow/core/kernels/tridiagonal_solve_op.cc b/tensorflow/core/kernels/linalg/tridiagonal_solve_op.cc similarity index 99% rename from tensorflow/core/kernels/tridiagonal_solve_op.cc rename to tensorflow/core/kernels/linalg/tridiagonal_solve_op.cc index 88931ff3e66..8fe04125f9a 100644 --- a/tensorflow/core/kernels/tridiagonal_solve_op.cc +++ b/tensorflow/core/kernels/linalg/tridiagonal_solve_op.cc @@ -19,7 +19,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/types.h" diff --git a/tensorflow/core/kernels/tridiagonal_solve_op_gpu.cu.cc b/tensorflow/core/kernels/linalg/tridiagonal_solve_op_gpu.cu.cc similarity index 99% rename from tensorflow/core/kernels/tridiagonal_solve_op_gpu.cu.cc rename to tensorflow/core/kernels/linalg/tridiagonal_solve_op_gpu.cu.cc index 089fa8c040f..86514cfb033 100644 --- a/tensorflow/core/kernels/tridiagonal_solve_op_gpu.cu.cc +++ b/tensorflow/core/kernels/linalg/tridiagonal_solve_op_gpu.cu.cc @@ -23,11 +23,11 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" -#include "tensorflow/core/kernels/linalg_ops_common.h" +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #include "tensorflow/core/kernels/transpose_functor.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #include "tensorflow/core/util/gpu_device_functions.h" #include "tensorflow/core/util/gpu_kernel_helper.h" #include "tensorflow/core/util/gpu_launch_config.h" diff --git a/tensorflow/core/kernels/linalg_ops_common.h b/tensorflow/core/kernels/linalg_ops_common.h index 65c2fb90f0e..0aa69801f19 100644 --- a/tensorflow/core/kernels/linalg_ops_common.h +++ b/tensorflow/core/kernels/linalg_ops_common.h @@ -12,211 +12,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ - #ifndef TENSORFLOW_CORE_KERNELS_LINALG_OPS_COMMON_H_ #define TENSORFLOW_CORE_KERNELS_LINALG_OPS_COMMON_H_ -// Classes to support linear algebra functionality, similar to the numpy.linalg -// module. Supports batch computation on several matrices at once, sharding the -// computations across different threads if necessary. -#include - -#include "third_party/eigen3/Eigen/Core" -#include "tensorflow/core/framework/kernel_def_builder.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/gtl/inlined_vector.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/work_sharder.h" - -namespace tensorflow { - -// Base class for linear algebra operators. -template -class LinearAlgebraOp : public OpKernel { - public: - explicit LinearAlgebraOp(OpKernelConstruction* context) : OpKernel(context) {} - - void Compute(OpKernelContext* context) override; - - protected: - using TensorShapes = gtl::InlinedVector; - // Returns the number of leading inputs that are to be treated as matrix - // inputs. By default this is all the inputs. Derived classes can override - // this to tell the base class to ignore one or more trailing inputs. - virtual int NumMatrixInputs(const OpKernelContext* context) const { - return context->num_inputs(); - } - - // Returns true if the number of inputs and their shapes are as expected. - // Many ops take a single square input matrix, so we provide that as a default - // implementation for convenience. - virtual void ValidateInputMatrixShapes( - OpKernelContext* context, const TensorShapes& input_matrix_shapes) const { - ValidateSingleSquareMatrix(context, input_matrix_shapes); - } - - // Convenience validators for common cases: - // - // Validate op taking a single matrix A. - static void ValidateSingleMatrix(OpKernelContext* context, - const TensorShapes& input_matrix_shapes); - // Validate op taking a single square matrix A. - static void ValidateSingleSquareMatrix( - OpKernelContext* context, const TensorShapes& input_matrix_shapes); - // Validate op taking two matrices A and B that have the same number of rows. - static void ValidateSolver(OpKernelContext* context, - const TensorShapes& input_matrix_shapes); - // Validate op taking two matrices A and B that have the same number of rows - // and A is square. - static void ValidateSquareSolver(OpKernelContext* context, - const TensorShapes& input_matrix_shapes); - - // Returns the output shapes of each individual matrix operation. Output - // matrices shapes must be rank 0, 1, or 2. Scalar outputs are rank 0. - // - // The derived class may return a number of shapes (N) less than - // context->num_outputs() (M) to indicate that a only leading subset of - // the outputs will be populated. In this case, a dummy scalar tensor with - // value zero will be return for the last M-N outputs. - // - // For many ops, the output dimensions are the same as the input dimensions, - // so we provide that as a default implementation for convenience. - virtual TensorShapes GetOutputMatrixShapes( - const TensorShapes& input_matrix_shapes) const { - return input_matrix_shapes; - } - - // Returns the cost per matrix operation. This is used to determine the - // number of threads to use for parallelizing calls to ComputeMatrix in - // batch mode. Cost per unit is assumed to be roughly 1ns, based on comments - // in core/util/work_sharder.cc. Many linear algebra ops take roughly max(m,n) - // * min(m,n)^2, where the first input matrix is m-by-n. We provide that as a - // default implementation for convenience. - virtual int64 GetCostPerUnit(const TensorShapes& input_matrix_shapes) const { - double m = static_cast(input_matrix_shapes[0].dim_size(0)); - double n = static_cast(input_matrix_shapes[0].dim_size(1)); - double cost = std::max(m, n) * std::min(m, n) * std::min(m, n); - return cost >= static_cast(kint64max) ? kint64max - : static_cast(cost); - } - - // Returns true if it is safe to forward (alias) input to output buffer - // and expect the kernel to perform the computation inplace. - virtual bool EnableInputForwarding() const { return true; } - - using InputMatrix = Eigen::Matrix; - using InputConstMatrixMap = Eigen::Map; - using InputMatrixMap = Eigen::Map; - using InputConstVectorMap = - Eigen::Map>; - using InputConstMatrixMaps = gtl::InlinedVector; - using InputMatrixMaps = gtl::InlinedVector; - using InputRealScalar = typename Eigen::NumTraits::Real; - - using OutputMatrix = Eigen::Matrix; - using OutputConstMatrixMap = Eigen::Map; - using OutputMatrixMap = Eigen::Map; - using OutputConstVectorMap = - Eigen::Map>; - using OutputConstMatrixMaps = gtl::InlinedVector; - using OutputMatrixMaps = gtl::InlinedVector; - using OutputRealScalar = typename Eigen::NumTraits::Real; - - // backward compatibility - using Scalar = OutputScalar; - using Matrix = - Eigen::Matrix; - using ConstMatrixMap = Eigen::Map; - using MatrixMap = Eigen::Map; - using ConstVectorMap = - Eigen::Map>; - using ConstMatrixMaps = gtl::InlinedVector; - using MatrixMaps = gtl::InlinedVector; - using RealScalar = typename Eigen::NumTraits::Real; - - // Performs a single matrix computation given input matrices, and - // stores the result in outputs. For batch operations, this will be called - // repeatedly for a single call to Compute() when multiple matrices exist in - // input Tensors with rank > 2. In this case the calls to ComputeMatrix are - // parallelized. The number of threads used is determined by a cost model from - // the value returned by GetCostPerUnit(). - virtual void ComputeMatrix(OpKernelContext* context, - const InputConstMatrixMaps& inputs, - OutputMatrixMaps* outputs) = 0; - - private: - using TensorInputs = gtl::InlinedVector; - using TensorOutputs = gtl::InlinedVector; - // This function maps 2-d slices (matrices) of the input and output tensors - // using Eigen::Map and calls ComputeMatrix implemented in terms of the - // Eigen::MatrixBase API by the derived class. - // - // The 'matrix_index' parameter specifies the index of the matrix to be used - // from each input tensor, and the index of the matrix to be written to each - // output tensor. The input matrices are in row major order, and located at - // the memory addresses - // inputs[i].flat().data() + - // matrix_index * input_matrix_shapes[i].num_elements() - // for i in 0...inputs.size()-1. - // The output matrices are in row major order, and located at the memory - // address - // outputs[i]->flat().data() + - // matrix_index * output_matrix_shapes[i].num_elements(). - // for i in 0...outputs.size()-1. - // - void ComputeTensorSlice(OpKernelContext* context, int64 matrix_index, - const TensorInputs& inputs, - const TensorShapes& input_matrix_shapes, - const TensorOutputs& outputs, - const TensorShapes& output_matrix_shapes); - - void AnalyzeInputs(OpKernelContext* context, TensorInputs* inputs, - TensorShapes* input_matrix_shapes, - TensorShape* batch_shape); - - void PrepareOutputs(OpKernelContext* context, - const TensorShapes& input_matrix_shapes, - const TensorShape& batch_shape, TensorOutputs* outputs, - TensorShapes* output_matrix_shapes); -}; - -// Declare LinearAlgebraOp, which is explicitly instantiated in -// linalg_ops_common.cc for float, double, complex64, and complex128. -extern template class LinearAlgebraOp; -extern template class LinearAlgebraOp; -extern template class LinearAlgebraOp; -extern template class LinearAlgebraOp; - -} // namespace tensorflow - -#define INHERIT_LINALG_TYPEDEFS(Scalar) \ - typedef LinearAlgebraOp Base; \ - using RealScalar = typename Eigen::NumTraits::Real; \ - using Matrix = typename Base::Matrix; \ - using MatrixMap = typename Base::MatrixMap; \ - using MatrixMaps = typename Base::MatrixMaps; \ - using ConstMatrixMap = typename Base::ConstMatrixMap; \ - using ConstMatrixMaps = typename Base::ConstMatrixMaps; \ - using ConstVectorMap = typename Base::ConstVectorMap; \ - using TensorShapes = typename Base::TensorShapes; - -#define REGISTER_LINALG_OP_CPU(OpName, OpClass, Scalar) \ - REGISTER_KERNEL_BUILDER( \ - Name(OpName).Device(DEVICE_CPU).TypeConstraint("T"), OpClass) - -#define REGISTER_LINALG_OP_GPU(OpName, OpClass, Scalar) \ - REGISTER_KERNEL_BUILDER( \ - Name(OpName).Device(DEVICE_GPU).TypeConstraint("T"), OpClass) - -// Deprecated, use one of the device-specific macros above. -#define REGISTER_LINALG_OP(OpName, OpClass, Scalar) \ - REGISTER_LINALG_OP_CPU(OpName, OpClass, Scalar) +// Temporary forwarding header. +#include "tensorflow/core/kernels/linalg/linalg_ops_common.h" #endif // TENSORFLOW_CORE_KERNELS_LINALG_OPS_COMMON_H_ diff --git a/tensorflow/core/kernels/segment_reduction_ops_impl.h b/tensorflow/core/kernels/segment_reduction_ops_impl.h index 6c3fad668ae..7cf15ef5b72 100644 --- a/tensorflow/core/kernels/segment_reduction_ops_impl.h +++ b/tensorflow/core/kernels/segment_reduction_ops_impl.h @@ -45,13 +45,13 @@ limitations under the License. #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM #if GOOGLE_CUDA -#include "tensorflow/core/kernels/cuda_solvers.h" +#include "tensorflow/core/util/cuda_solvers.h" #include "tensorflow/stream_executor/cuda/cuda_activation.h" using stream_executor::cuda::ScopedActivateExecutorContext; #elif TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/platform/rocm.h" +#include "tensorflow/core/util/cuda_solvers.h" using stream_executor::rocm::ScopedActivateExecutorContext; #endif // GOOGLE_CUDA diff --git a/tensorflow/core/kernels/sparse/BUILD b/tensorflow/core/kernels/sparse/BUILD index 1d281bc1d61..bfb6c4934bb 100644 --- a/tensorflow/core/kernels/sparse/BUILD +++ b/tensorflow/core/kernels/sparse/BUILD @@ -80,8 +80,8 @@ tf_kernel_library( "//tensorflow/core/kernels:transpose_functor", "//tensorflow/core/kernels:gpu_prim_hdrs", ] + if_cuda_or_rocm([ - "//tensorflow/core/kernels:cuda_solvers", - "//tensorflow/core/kernels:cuda_sparse", + "//tensorflow/core/util:cuda_solvers", + "//tensorflow/core/util:cuda_sparse", ]), alwayslink = 1, ) diff --git a/tensorflow/core/kernels/sparse/add_op.cc b/tensorflow/core/kernels/sparse/add_op.cc index b6265a1412c..06fe1cd042e 100644 --- a/tensorflow/core/kernels/sparse/add_op.cc +++ b/tensorflow/core/kernels/sparse/add_op.cc @@ -32,8 +32,8 @@ limitations under the License. #include "tensorflow/core/kernels/fill_functor.h" #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/conj_op.cc b/tensorflow/core/kernels/sparse/conj_op.cc index 7275262c1f0..147160fbe6c 100644 --- a/tensorflow/core/kernels/sparse/conj_op.cc +++ b/tensorflow/core/kernels/sparse/conj_op.cc @@ -32,8 +32,8 @@ limitations under the License. #include "tensorflow/core/kernels/sparse/sparse_matrix.h" #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/csr_sparse_matrix_to_dense_op.cc b/tensorflow/core/kernels/sparse/csr_sparse_matrix_to_dense_op.cc index 364c2c07bd8..2e5afbdcad7 100644 --- a/tensorflow/core/kernels/sparse/csr_sparse_matrix_to_dense_op.cc +++ b/tensorflow/core/kernels/sparse/csr_sparse_matrix_to_dense_op.cc @@ -34,8 +34,8 @@ limitations under the License. #include "tensorflow/core/util/work_sharder.h" #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/csr_sparse_matrix_to_sparse_tensor_op.cc b/tensorflow/core/kernels/sparse/csr_sparse_matrix_to_sparse_tensor_op.cc index 55ebfa4fc10..a81ccfa562e 100644 --- a/tensorflow/core/kernels/sparse/csr_sparse_matrix_to_sparse_tensor_op.cc +++ b/tensorflow/core/kernels/sparse/csr_sparse_matrix_to_sparse_tensor_op.cc @@ -32,8 +32,8 @@ limitations under the License. #include "tensorflow/core/util/work_sharder.h" #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/dense_to_csr_sparse_matrix_op.cc b/tensorflow/core/kernels/sparse/dense_to_csr_sparse_matrix_op.cc index 459bb219343..5c62a44f9ba 100644 --- a/tensorflow/core/kernels/sparse/dense_to_csr_sparse_matrix_op.cc +++ b/tensorflow/core/kernels/sparse/dense_to_csr_sparse_matrix_op.cc @@ -35,8 +35,8 @@ limitations under the License. #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h" -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif #if GOOGLE_CUDA diff --git a/tensorflow/core/kernels/sparse/kernels_gpu.cu.cc b/tensorflow/core/kernels/sparse/kernels_gpu.cu.cc index 1c014db3d0a..6b11e64307a 100644 --- a/tensorflow/core/kernels/sparse/kernels_gpu.cu.cc +++ b/tensorflow/core/kernels/sparse/kernels_gpu.cu.cc @@ -20,13 +20,13 @@ limitations under the License. #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/kernels/cuda_sparse.h" #include "tensorflow/core/kernels/gpu_device_array.h" #include "tensorflow/core/kernels/gpu_device_array_gpu.h" #include "tensorflow/core/kernels/gpu_prim.h" #include "tensorflow/core/kernels/sparse/kernels.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_sparse.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/mat_mul_op.cc b/tensorflow/core/kernels/sparse/mat_mul_op.cc index 50fa0ec88ea..bf9de570fbf 100644 --- a/tensorflow/core/kernels/sparse/mat_mul_op.cc +++ b/tensorflow/core/kernels/sparse/mat_mul_op.cc @@ -37,8 +37,8 @@ limitations under the License. #include "tensorflow/core/platform/threadpool.h" #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/mul_op.cc b/tensorflow/core/kernels/sparse/mul_op.cc index 33c3756ce58..d08f1568db1 100644 --- a/tensorflow/core/kernels/sparse/mul_op.cc +++ b/tensorflow/core/kernels/sparse/mul_op.cc @@ -29,7 +29,7 @@ limitations under the License. #include "tensorflow/core/kernels/sparse/sparse_matrix.h" #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/nnz_op.cc b/tensorflow/core/kernels/sparse/nnz_op.cc index ebc48c3e9a4..d67620443f0 100644 --- a/tensorflow/core/kernels/sparse/nnz_op.cc +++ b/tensorflow/core/kernels/sparse/nnz_op.cc @@ -29,8 +29,8 @@ limitations under the License. #include "tensorflow/core/kernels/sparse/sparse_matrix.h" #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/softmax_op.cc b/tensorflow/core/kernels/sparse/softmax_op.cc index 25025bfe2a6..f1a5db8d0f0 100644 --- a/tensorflow/core/kernels/sparse/softmax_op.cc +++ b/tensorflow/core/kernels/sparse/softmax_op.cc @@ -20,7 +20,7 @@ limitations under the License. #define EIGEN_USE_THREADS #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_sparse.h" #define EIGEN_USE_GPU #endif diff --git a/tensorflow/core/kernels/sparse/sparse_mat_mul_op.cc b/tensorflow/core/kernels/sparse/sparse_mat_mul_op.cc index fb652e13d15..fecee9e4555 100644 --- a/tensorflow/core/kernels/sparse/sparse_mat_mul_op.cc +++ b/tensorflow/core/kernels/sparse/sparse_mat_mul_op.cc @@ -36,8 +36,8 @@ limitations under the License. #include "tensorflow/core/util/work_sharder.h" #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/sparse_matrix_components_op.cc b/tensorflow/core/kernels/sparse/sparse_matrix_components_op.cc index 59540f63846..2eaf9bd5310 100644 --- a/tensorflow/core/kernels/sparse/sparse_matrix_components_op.cc +++ b/tensorflow/core/kernels/sparse/sparse_matrix_components_op.cc @@ -30,8 +30,8 @@ limitations under the License. #include "tensorflow/core/kernels/sparse/sparse_matrix.h" #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif namespace tensorflow { diff --git a/tensorflow/core/kernels/sparse/sparse_tensor_to_csr_sparse_matrix_op.cc b/tensorflow/core/kernels/sparse/sparse_tensor_to_csr_sparse_matrix_op.cc index e1a4b4194d2..2548ceaa57c 100644 --- a/tensorflow/core/kernels/sparse/sparse_tensor_to_csr_sparse_matrix_op.cc +++ b/tensorflow/core/kernels/sparse/sparse_tensor_to_csr_sparse_matrix_op.cc @@ -33,8 +33,8 @@ limitations under the License. #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h" -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" #endif #if GOOGLE_CUDA diff --git a/tensorflow/core/kernels/sparse/transpose_op.cc b/tensorflow/core/kernels/sparse/transpose_op.cc index 3158eb5016d..08d37fa1692 100644 --- a/tensorflow/core/kernels/sparse/transpose_op.cc +++ b/tensorflow/core/kernels/sparse/transpose_op.cc @@ -20,7 +20,7 @@ limitations under the License. #define EIGEN_USE_THREADS #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_sparse.h" #define EIGEN_USE_GPU #endif diff --git a/tensorflow/core/kernels/where_op.cc b/tensorflow/core/kernels/where_op.cc index 598cb526d77..d504ec9b2ed 100644 --- a/tensorflow/core/kernels/where_op.cc +++ b/tensorflow/core/kernels/where_op.cc @@ -39,7 +39,7 @@ limitations under the License. #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h" -#include "tensorflow/core/kernels/cuda_solvers.h" +#include "tensorflow/core/util/cuda_solvers.h" #if GOOGLE_CUDA #include "tensorflow/stream_executor/cuda/cuda_activation.h" using stream_executor::cuda::ScopedActivateExecutorContext; diff --git a/tensorflow/core/util/BUILD b/tensorflow/core/util/BUILD index bb2b9ff429e..dcb2787e309 100644 --- a/tensorflow/core/util/BUILD +++ b/tensorflow/core/util/BUILD @@ -14,6 +14,7 @@ load( "tf_copts", "tf_cuda_library", "tf_cuda_only_cc_test", + "tf_kernel_library", ) load("//tensorflow:tensorflow.bzl", "tf_version_info_genrule") load( @@ -24,6 +25,11 @@ load( "//tensorflow/core/platform:build_config_root.bzl", "if_static", ) +load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") +load( + "@local_config_rocm//rocm:build_defs.bzl", + "if_rocm", +) default_package_visibility = [ "//tensorflow/core:__subpackages__", @@ -567,6 +573,63 @@ cc_library( ], ) +tf_kernel_library( + name = "cuda_solvers", + srcs = ["cuda_solvers.cc"], + hdrs = ["cuda_solvers.h"], + # @local_config_cuda//cuda:cusolver_static, //third_party/eigen3:blas, + # and //third_party/libf2c all contain various parts of BLAS, LAPACK, + # and f2c helper functions in global namespace. Tell the compiler to + # allow multiple definitions when linking this. + linkopts = select({ + "//tensorflow:macos": [], + "//tensorflow:windows": [], + "//conditions:default": ["-Wl,-z,muldefs"], + }), + visibility = ["//tensorflow/core/kernels:friends"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/platform/default/build_config:cublas_plugin", + "//tensorflow/stream_executor/cuda:cublas_lib", + "//tensorflow/stream_executor/cuda:cusolver_lib", + ], +) + +tf_kernel_library( + name = "rocm_solvers", + srcs = ["rocm_solvers.cc"], + hdrs = ["rocm_solvers.h"], + visibility = ["//tensorflow/core/kernels:friends"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/stream_executor/lib", + "//tensorflow/stream_executor/platform:dso_loader", + "//tensorflow/stream_executor/rocm:rocblas_plugin", + "//tensorflow/stream_executor/rocm:rocm_gpu_executor", + ] + if_rocm([ + "@local_config_rocm//rocm:rocprim", + ]), +) + +tf_kernel_library( + name = "cuda_sparse", + srcs = if_cuda(["cuda_sparse.cc"]) + if_rocm(["rocm_sparse.cc"]), + hdrs = ["cuda_sparse.h"], + deps = [ + ":cuda_solvers", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + ] + if_cuda([ + "//tensorflow/stream_executor/cuda:cusparse_lib", + "@cub_archive//:cub", + ]) + if_rocm([ + "@local_config_rocm//rocm:hipsparse", + ]), +) + # Tests. tf_cc_test( diff --git a/tensorflow/core/kernels/cuda_solvers.cc b/tensorflow/core/util/cuda_solvers.cc similarity index 99% rename from tensorflow/core/kernels/cuda_solvers.cc rename to tensorflow/core/util/cuda_solvers.cc index f41ce2a5d27..3e4d2a05ac6 100644 --- a/tensorflow/core/kernels/cuda_solvers.cc +++ b/tensorflow/core/util/cuda_solvers.cc @@ -14,7 +14,7 @@ ============================================================================== */ #ifdef GOOGLE_CUDA -#include "tensorflow/core/kernels/cuda_solvers.h" +#include "tensorflow/core/util/cuda_solvers.h" #include #include diff --git a/tensorflow/core/kernels/cuda_solvers.h b/tensorflow/core/util/cuda_solvers.h similarity index 99% rename from tensorflow/core/kernels/cuda_solvers.h rename to tensorflow/core/util/cuda_solvers.h index eb1d5c8a200..79f45c9b0ea 100644 --- a/tensorflow/core/kernels/cuda_solvers.h +++ b/tensorflow/core/util/cuda_solvers.h @@ -14,8 +14,8 @@ limitations under the License. ============================================================================== */ -#ifndef TENSORFLOW_CORE_KERNELS_CUDA_SOLVERS_H_ -#define TENSORFLOW_CORE_KERNELS_CUDA_SOLVERS_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_CUDA_SOLVERS_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_CUDA_SOLVERS_H_ // This header declares the class CudaSolver, which contains wrappers of linear // algebra solvers in the cuBlas and cuSolverDN libraries for use in TensorFlow @@ -435,7 +435,7 @@ class HostLapackInfo : public ScratchSpace { public: HostLapackInfo(OpKernelContext* context, int64 size, const std::string& debug_info) - : ScratchSpace(context, size, debug_info, /* on_host */ true){}; + : ScratchSpace(context, size, debug_info, /* on_host */ true) {} }; class DeviceLapackInfo : public ScratchSpace { @@ -489,4 +489,4 @@ inline DeviceLapackInfo CudaSolver::GetDeviceLapackInfo( #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#endif // TENSORFLOW_CORE_KERNELS_CUDA_SOLVERS_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_CUDA_SOLVERS_H_ diff --git a/tensorflow/core/kernels/cuda_sparse.cc b/tensorflow/core/util/cuda_sparse.cc similarity index 99% rename from tensorflow/core/kernels/cuda_sparse.cc rename to tensorflow/core/util/cuda_sparse.cc index 141aae61571..47e018560e1 100644 --- a/tensorflow/core/kernels/cuda_sparse.cc +++ b/tensorflow/core/util/cuda_sparse.cc @@ -15,7 +15,7 @@ limitations under the License. #ifdef GOOGLE_CUDA -#include "tensorflow/core/kernels/cuda_sparse.h" +#include "tensorflow/core/util/cuda_sparse.h" #include #include @@ -28,7 +28,6 @@ limitations under the License. #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/cuda_solvers.h" #include "tensorflow/core/lib/core/blocking_counter.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/stringpiece.h" @@ -38,6 +37,7 @@ limitations under the License. #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/stream_executor.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_solvers.h" // TODO(rmlarsen,penporn): Investigate using newer kernels in CUDA 10.1+. diff --git a/tensorflow/core/kernels/cuda_sparse.h b/tensorflow/core/util/cuda_sparse.h similarity index 93% rename from tensorflow/core/kernels/cuda_sparse.h rename to tensorflow/core/util/cuda_sparse.h index 978bc9005ed..76580766d69 100644 --- a/tensorflow/core/kernels/cuda_sparse.h +++ b/tensorflow/core/util/cuda_sparse.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_CUDA_SPARSE_H_ -#define TENSORFLOW_CORE_KERNELS_CUDA_SPARSE_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_CUDA_SPARSE_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_CUDA_SPARSE_H_ // This header declares the class GpuSparse, which contains wrappers of // cuSparse libraries for use in TensorFlow kernels. @@ -75,8 +75,7 @@ using gpuStream_t = hipStream_t; namespace tensorflow { -inline std::string ConvertGPUSparseErrorToString( - const gpusparseStatus_t status) { +inline string ConvertGPUSparseErrorToString(const gpusparseStatus_t status) { switch (status) { #define STRINGIZE(q) #q #define RETURN_IF_STATUS(err) \ @@ -206,49 +205,49 @@ class GpuSparse { // Solves tridiagonal system of equations. // See: https://docs.nvidia.com/cuda/cusparse/index.html#gtsv2 template - Status Gtsv2(int m, int n, const Scalar *dl, const Scalar *d, - const Scalar *du, Scalar *B, int ldb, void *pBuffer) const; + Status Gtsv2(int m, int n, const Scalar* dl, const Scalar* d, + const Scalar* du, Scalar* B, int ldb, void* pBuffer) const; // Computes the size of a temporary buffer used by Gtsv2. // See: https://docs.nvidia.com/cuda/cusparse/index.html#gtsv2_bufferSize template - Status Gtsv2BufferSizeExt(int m, int n, const Scalar *dl, const Scalar *d, - const Scalar *du, const Scalar *B, int ldb, - size_t *bufferSizeInBytes) const; + Status Gtsv2BufferSizeExt(int m, int n, const Scalar* dl, const Scalar* d, + const Scalar* du, const Scalar* B, int ldb, + size_t* bufferSizeInBytes) const; // Solves tridiagonal system of equations without partial pivoting. // See: https://docs.nvidia.com/cuda/cusparse/index.html#gtsv2_nopivot template - Status Gtsv2NoPivot(int m, int n, const Scalar *dl, const Scalar *d, - const Scalar *du, Scalar *B, int ldb, - void *pBuffer) const; + Status Gtsv2NoPivot(int m, int n, const Scalar* dl, const Scalar* d, + const Scalar* du, Scalar* B, int ldb, + void* pBuffer) const; // Computes the size of a temporary buffer used by Gtsv2NoPivot. // See: // https://docs.nvidia.com/cuda/cusparse/index.html#gtsv2_nopivot_bufferSize template - Status Gtsv2NoPivotBufferSizeExt(int m, int n, const Scalar *dl, - const Scalar *d, const Scalar *du, - const Scalar *B, int ldb, - size_t *bufferSizeInBytes) const; + Status Gtsv2NoPivotBufferSizeExt(int m, int n, const Scalar* dl, + const Scalar* d, const Scalar* du, + const Scalar* B, int ldb, + size_t* bufferSizeInBytes) const; // Solves a batch of tridiagonal systems of equations. Doesn't support // multiple right-hand sides per each system. Doesn't do pivoting. // See: https://docs.nvidia.com/cuda/cusparse/index.html#gtsv2stridedbatch template - Status Gtsv2StridedBatch(int m, const Scalar *dl, const Scalar *d, - const Scalar *du, Scalar *x, int batchCount, - int batchStride, void *pBuffer) const; + Status Gtsv2StridedBatch(int m, const Scalar* dl, const Scalar* d, + const Scalar* du, Scalar* x, int batchCount, + int batchStride, void* pBuffer) const; // Computes the size of a temporary buffer used by Gtsv2StridedBatch. // See: // https://docs.nvidia.com/cuda/cusparse/index.html#gtsv2stridedbatch_bufferSize template - Status Gtsv2StridedBatchBufferSizeExt(int m, const Scalar *dl, - const Scalar *d, const Scalar *du, - const Scalar *x, int batchCount, + Status Gtsv2StridedBatchBufferSizeExt(int m, const Scalar* dl, + const Scalar* d, const Scalar* du, + const Scalar* x, int batchCount, int batchStride, - size_t *bufferSizeInBytes) const; + size_t* bufferSizeInBytes) const; // Compresses the indices of rows or columns. It can be interpreted as a // conversion from COO to CSR sparse storage format. See: @@ -449,7 +448,7 @@ class GpuSparse { private: bool initialized_; - OpKernelContext *context_; // not owned. + OpKernelContext* context_; // not owned. gpuStream_t gpu_stream_; gpusparseHandle_t* gpusparse_handle_; // not owned. @@ -585,4 +584,4 @@ class GpuSparseCsrSortingConversionInfo { #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#endif // TENSORFLOW_CORE_KERNELS_CUDA_SPARSE_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_CUDA_SPARSE_H_ diff --git a/tensorflow/core/kernels/rocm_solvers.cc b/tensorflow/core/util/rocm_solvers.cc similarity index 99% rename from tensorflow/core/kernels/rocm_solvers.cc rename to tensorflow/core/util/rocm_solvers.cc index 5faf718332e..13dadf602a7 100644 --- a/tensorflow/core/kernels/rocm_solvers.cc +++ b/tensorflow/core/util/rocm_solvers.cc @@ -14,7 +14,7 @@ ============================================================================== */ #if TENSORFLOW_USE_ROCM -#include "tensorflow/core/kernels/rocm_solvers.h" +#include "tensorflow/core/util/rocm_solvers.h" #include #include diff --git a/tensorflow/core/kernels/rocm_solvers.h b/tensorflow/core/util/rocm_solvers.h similarity index 96% rename from tensorflow/core/kernels/rocm_solvers.h rename to tensorflow/core/util/rocm_solvers.h index 94d3c82a497..afc8b936d05 100644 --- a/tensorflow/core/kernels/rocm_solvers.h +++ b/tensorflow/core/util/rocm_solvers.h @@ -14,8 +14,8 @@ limitations under the License. ============================================================================== */ -#ifndef TENSORFLOW_CORE_KERNELS_ROCM_SOLVERS_H_ -#define TENSORFLOW_CORE_KERNELS_ROCM_SOLVERS_H_ +#ifndef TENSORFLOW_CORE_KERNELS_LINALG_ROCM_SOLVERS_H_ +#define TENSORFLOW_CORE_KERNELS_LINALG_ROCM_SOLVERS_H_ // This header declares the class ROCmSolver, which contains wrappers of linear // algebra solvers in the cuBlas and cuSolverDN libraries for use in TensorFlow @@ -158,4 +158,4 @@ class ScratchSpace { #endif // TENSORFLOW_USE_ROCM -#endif // TENSORFLOW_CORE_KERNELS_ROCM_SOLVERS_H_ +#endif // TENSORFLOW_CORE_KERNELS_LINALG_ROCM_SOLVERS_H_ diff --git a/tensorflow/core/kernels/rocm_sparse.cc b/tensorflow/core/util/rocm_sparse.cc similarity index 99% rename from tensorflow/core/kernels/rocm_sparse.cc rename to tensorflow/core/util/rocm_sparse.cc index 97488692bc1..cc7b56fdc01 100644 --- a/tensorflow/core/kernels/rocm_sparse.cc +++ b/tensorflow/core/util/rocm_sparse.cc @@ -24,8 +24,6 @@ limitations under the License. #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/cuda_solvers.h" -#include "tensorflow/core/kernels/cuda_sparse.h" #include "tensorflow/core/lib/core/blocking_counter.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/stringpiece.h" @@ -35,6 +33,8 @@ limitations under the License. #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/stream_executor.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/cuda_solvers.h" +#include "tensorflow/core/util/cuda_sparse.h" namespace tensorflow { namespace { From 3f45c33ba54968010ef551f448a843ab2f1427fd Mon Sep 17 00:00:00 2001 From: Katherine Wu Date: Tue, 4 Aug 2020 17:20:06 -0700 Subject: [PATCH 0416/1017] Remove tracking of inbound nodes/outbound nodes. When building a Sequential model, clear_previously_created_nodes accidentally added tracking to the nodes attributes, which causes unnecessary warnings when loading a checkpoint. PiperOrigin-RevId: 324923805 Change-Id: I7ee4457b70b16bb1a3b410f41327bba269a128e5 --- tensorflow/python/keras/engine/base_layer.py | 22 +++++++++++++++++-- .../python/keras/engine/base_layer_v1.py | 22 +++++++++++++++++-- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/keras/engine/base_layer.py b/tensorflow/python/keras/engine/base_layer.py index 373e17a4004..c01c3d96aec 100644 --- a/tensorflow/python/keras/engine/base_layer.py +++ b/tensorflow/python/keras/engine/base_layer.py @@ -382,8 +382,8 @@ class Layer(module.Module, version_utils.LayerVersionSelector): # These lists will be filled via successive calls # to self._add_inbound_node(). # Used in symbolic mode only, only in conjunction with graph-networks - self._inbound_nodes = [] - self._outbound_nodes = [] + self._inbound_nodes_value = [] + self._outbound_nodes_value = [] self._init_call_fn_args() @@ -2268,6 +2268,24 @@ class Layer(module.Module, version_utils.LayerVersionSelector): # Methods & attributes below are all private and only used by the framework. # ############################################################################## + @property + def _inbound_nodes(self): + return self._inbound_nodes_value + + @_inbound_nodes.setter + @trackable.no_automatic_dependency_tracking + def _inbound_nodes(self, value): + self._inbound_nodes_value = value + + @property + def _outbound_nodes(self): + return self._outbound_nodes_value + + @_outbound_nodes.setter + @trackable.no_automatic_dependency_tracking + def _outbound_nodes(self, value): + self._outbound_nodes_value = value + def _set_dtype_policy(self, dtype): """Sets self._dtype_policy.""" if isinstance(dtype, policy.Policy): diff --git a/tensorflow/python/keras/engine/base_layer_v1.py b/tensorflow/python/keras/engine/base_layer_v1.py index e9ebc170b96..85d390f2360 100644 --- a/tensorflow/python/keras/engine/base_layer_v1.py +++ b/tensorflow/python/keras/engine/base_layer_v1.py @@ -217,8 +217,8 @@ class Layer(base_layer.Layer): # These lists will be filled via successive calls # to self._add_inbound_node(). # Used in symbolic mode only, only in conjunction with graph-networks - self._inbound_nodes = [] - self._outbound_nodes = [] + self._inbound_nodes_value = [] + self._outbound_nodes_value = [] self._init_call_fn_args() @@ -1740,6 +1740,24 @@ class Layer(base_layer.Layer): # Methods & attributes below are all private and only used by the framework. # ############################################################################## + @property + def _inbound_nodes(self): + return self._inbound_nodes_value + + @_inbound_nodes.setter + @trackable.no_automatic_dependency_tracking + def _inbound_nodes(self, value): + self._inbound_nodes_value = value + + @property + def _outbound_nodes(self): + return self._outbound_nodes_value + + @_outbound_nodes.setter + @trackable.no_automatic_dependency_tracking + def _outbound_nodes(self, value): + self._outbound_nodes_value = value + def _set_dtype_policy(self, dtype): """Sets self._dtype_policy.""" if isinstance(dtype, policy.Policy): From 92d021c1ae9bbcd500d13f24cd7c3c399ab82e6f Mon Sep 17 00:00:00 2001 From: ShengYang1 Date: Wed, 5 Aug 2020 08:28:55 +0800 Subject: [PATCH 0417/1017] Fix some error --- .../core/grappler/optimizers/remapper_test.cc | 79 +++++++++---------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/remapper_test.cc b/tensorflow/core/grappler/optimizers/remapper_test.cc index 417ecd6dd44..7b99c038c53 100644 --- a/tensorflow/core/grappler/optimizers/remapper_test.cc +++ b/tensorflow/core/grappler/optimizers/remapper_test.cc @@ -930,48 +930,45 @@ class FusedCmpAndCastTest : public GrapplerTest { template void TestFusedCmpAndCast() { using ::tensorflow::ops::Placeholder; - for (bool is_training : {true, false}) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - const int num_channels = 24; - TensorShape channel_shape({num_channels}); - TensorShape empty_shape({0}); - auto x = Placeholder(s.WithOpName("x"), TYPE, - ops::Placeholder::Shape({2, 8, 8, num_channels})); - auto y = Placeholder(s.WithOpName("y"), TYPE, - ops::Placeholder::Shape({2, 8, 8, num_channels})); - float epsilon = 0.1f; - auto comparator = ops::Equal(s.WithOpName("Equal"), x, y); - auto cast = ops::Cast(s.WithOpName("cast"), comparator.z, TYPE); - auto fetch = ops::Identity(s.WithOpName("fetch"), cast); - auto input1_t = GenerateRandomTensor({2, 8, 8, num_channels}); - auto input2_t = GenerateRandomTensor({2, 8, 8, num_channels}); - GrapplerItem item; - item.fetch = {"fetch"}; - item.feed = {{"x", input1_t}, {"y", input2_t}}; - TF_ASSERT_OK(s.ToGraphDef(&item.graph)); - for (int i = 0; i < item.graph.node_size(); ++i) { - item.graph.mutable_node(i)->set_device("/device:CPU:0"); - } - Remapper optimizer(RewriterConfig::AGGRESSIVE); - GraphDef output; - TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output)); - int found = 0; - for (const NodeDef& node : output.node()) { - if (node.name() == "cast") { - EXPECT_EQ(node.op(), "_EqualWithCast"); - ASSERT_EQ(node.input_size(), 2); - EXPECT_EQ(node.input(0), "x"); - EXPECT_EQ(node.input(1), "y"); - found++; - } - } - EXPECT_EQ(found, 1); - auto tensors_expected = EvaluateNodes(item.graph, item.fetch, item.feed); - ASSERT_EQ(tensors_expected.size(), 1); - auto tensors = EvaluateNodes(output, item.fetch, item.feed); - ASSERT_EQ(tensors.size(), 1); - test::ExpectClose(tensors[0], tensors_expected[0], 1e-2, 1e-2); + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + const int num_channels = 24; + TensorShape channel_shape({num_channels}); + TensorShape empty_shape({0}); + auto x = Placeholder(s.WithOpName("x"), TYPE, + ops::Placeholder::Shape({2, 8, 8, num_channels})); + auto y = Placeholder(s.WithOpName("y"), TYPE, + ops::Placeholder::Shape({2, 8, 8, num_channels})); + auto comparator = ops::Equal(s.WithOpName("Equal"), x, y); + auto cast = ops::Cast(s.WithOpName("cast"), comparator.z, TYPE); + auto fetch = ops::Identity(s.WithOpName("fetch"), cast); + auto input1_t = GenerateRandomTensor({2, 8, 8, num_channels}); + auto input2_t = GenerateRandomTensor({2, 8, 8, num_channels}); + GrapplerItem item; + item.fetch = {"fetch"}; + item.feed = {{"x", input1_t}, {"y", input2_t}}; + TF_ASSERT_OK(s.ToGraphDef(&item.graph)); + for (int i = 0; i < item.graph.node_size(); ++i) { + item.graph.mutable_node(i)->set_device("/device:CPU:0"); } + Remapper optimizer(RewriterConfig::AGGRESSIVE); + GraphDef output; + TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output)); + int found = 0; + for (const NodeDef& node : output.node()) { + if (node.name() == "cast") { + EXPECT_EQ(node.op(), "_EqualWithCast"); + ASSERT_EQ(node.input_size(), 2); + EXPECT_EQ(node.input(0), "x"); + EXPECT_EQ(node.input(1), "y"); + found++; + } + } + EXPECT_EQ(found, 1); + auto tensors_expected = EvaluateNodes(item.graph, item.fetch, item.feed); + ASSERT_EQ(tensors_expected.size(), 1); + auto tensors = EvaluateNodes(output, item.fetch, item.feed); + ASSERT_EQ(tensors.size(), 1); + test::ExpectClose(tensors[0], tensors_expected[0], 1e-2, 1e-2); } }; From f9bb6295251bdd44b1d533c2813b6997c829f591 Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Tue, 4 Aug 2020 17:46:36 -0700 Subject: [PATCH 0418/1017] [tf.data service] Use the journal to keep track of registered workers. As part of this change, we stop using integer worker ids, and instead use workers addresses as their identifiers. PiperOrigin-RevId: 324927652 Change-Id: If6ef5a08aac6bf32cc603108f9045887619488f1 --- tensorflow/core/data/service/common.proto | 9 ++ tensorflow/core/data/service/dispatcher.proto | 12 +-- .../core/data/service/dispatcher_impl.cc | 85 ++++++++++++------- .../core/data/service/dispatcher_impl.h | 23 ++--- .../core/data/service/dispatcher_state.cc | 31 +++++++ .../core/data/service/dispatcher_state.h | 36 +++++--- .../data/service/dispatcher_state_test.cc | 78 ++++++++++++++--- tensorflow/core/data/service/journal.cc | 2 +- tensorflow/core/data/service/journal.h | 4 +- tensorflow/core/data/service/journal.proto | 5 ++ tensorflow/core/data/service/worker_impl.cc | 3 - .../experimental/data_service_dataset_op.cc | 7 +- .../data/experimental/service_config.proto | 3 + 13 files changed, 206 insertions(+), 92 deletions(-) diff --git a/tensorflow/core/data/service/common.proto b/tensorflow/core/data/service/common.proto index 6d5398d9cd9..aeeb1371171 100644 --- a/tensorflow/core/data/service/common.proto +++ b/tensorflow/core/data/service/common.proto @@ -19,6 +19,15 @@ message TaskDef { int64 job_id = 4; } +message TaskInfo { + // The address of the worker processing the task. + string worker_address = 1; + // The task id. + int64 task_id = 2; + // The id of the job that the task is part of. + int64 job_id = 3; +} + enum ProcessingModeDef { // Each tf.data worker processes an entire epoch. PARALLEL_EPOCHS = 0; diff --git a/tensorflow/core/data/service/dispatcher.proto b/tensorflow/core/data/service/dispatcher.proto index 2a2d48ab93d..057fc58de52 100644 --- a/tensorflow/core/data/service/dispatcher.proto +++ b/tensorflow/core/data/service/dispatcher.proto @@ -10,8 +10,6 @@ message RegisterWorkerRequest { } message RegisterWorkerResponse { - // An id for the worker. - int64 worker_id = 1; // Tasks to begin processing. repeated TaskDef tasks = 2; } @@ -24,8 +22,7 @@ message TaskProgress { } message WorkerUpdateRequest { - // The worker id that the update is for. - int64 worker_id = 1; + string worker_address = 1; repeated TaskProgress updates = 2; } @@ -75,13 +72,6 @@ message GetTasksRequest { int64 job_id = 1; } -message TaskInfo { - // The address of the worker processing the task. - string worker_address = 1; - // The task id. - int64 id = 2; -} - message GetTasksResponse { // A list of all tasks for a job. repeated TaskInfo task_info = 1; diff --git a/tensorflow/core/data/service/dispatcher_impl.cc b/tensorflow/core/data/service/dispatcher_impl.cc index 77477df71e4..ffeae96c117 100644 --- a/tensorflow/core/data/service/dispatcher_impl.cc +++ b/tensorflow/core/data/service/dispatcher_impl.cc @@ -46,6 +46,7 @@ namespace { constexpr char kJournalDir[] = "journal"; using Dataset = DispatcherState::Dataset; +using Worker = DispatcherState::Worker; using NamedJobKey = DispatcherState::NamedJobKey; using Job = DispatcherState::Job; using Task = DispatcherState::Task; @@ -77,10 +78,16 @@ DataServiceDispatcherImpl::DataServiceDispatcherImpl( } Status DataServiceDispatcherImpl::Start() { - if (config_.work_dir().empty()) { + if (!config_.fault_tolerant_mode()) { + LOG(INFO) << "Running with fault_tolerant_mode=False. The dispatcher will " + "not be able to recover its state on restart."; return Status::OK(); } mutex_lock l(mu_); + if (config_.work_dir().empty()) { + return errors::InvalidArgument( + "fault_tolerant_mode is True, but no work_dir is configured."); + } Update update; bool end_of_journal = false; FileJournalReader reader(Env::Default(), JournalDir(config_.work_dir())); @@ -104,12 +111,16 @@ Status DataServiceDispatcherImpl::RegisterWorker( VLOG(3) << "Received register worker request"; mutex_lock l(mu_); std::string worker_address = request->worker_address(); - if (!workers_.contains(worker_address)) { - workers_[worker_address] = - std::make_shared(next_worker_id_++, worker_address); + std::shared_ptr worker; + Status s = state_.WorkerFromAddress(worker_address, &worker); + if (errors::IsNotFound(s)) { + Update update; + update.mutable_register_worker()->set_worker_address(worker_address); + TF_RETURN_IF_ERROR(Apply(update)); + } else if (!s.ok()) { + return s; } - int64 worker_id = workers_[worker_address]->worker_id; - response->set_worker_id(worker_id); + std::vector> jobs = state_.ListJobs(); // Allocate tasks to the worker. for (const auto& job : jobs) { @@ -127,8 +138,7 @@ Status DataServiceDispatcherImpl::RegisterWorker( task_def->set_task_id(task->task_id); } - VLOG(1) << "Registered worker at address " << request->worker_address() - << " with id " << worker_id; + VLOG(1) << "Registered worker at address " << request->worker_address(); return Status::OK(); } @@ -146,8 +156,7 @@ Status DataServiceDispatcherImpl::WorkerUpdate( continue; } Update update; - FinishTaskUpdate* finish_task = update.mutable_finish_task(); - finish_task->set_task_id(task_id); + update.mutable_finish_task()->set_task_id(task_id); TF_RETURN_IF_ERROR(Apply(update)); VLOG(3) << "Task " << task_id << " from job " << task->job_id << " completed"; @@ -310,10 +319,10 @@ Status DataServiceDispatcherImpl::CreateTasksForJob( std::shared_ptr job, std::vector>* tasks) EXCLUSIVE_LOCKS_REQUIRED(mu_) { + std::vector> workers = state_.ListWorkers(); tasks->clear(); - tasks->reserve(workers_.size()); - for (const auto& it : workers_) { - std::shared_ptr worker = it.second; + tasks->reserve(workers.size()); + for (const auto& worker : workers) { std::shared_ptr task; TF_RETURN_IF_ERROR(CreateTask(job, worker->address, &task)); tasks->push_back(task); @@ -345,10 +354,28 @@ Status DataServiceDispatcherImpl::AssignTasks( return Status::OK(); } -Status DataServiceDispatcherImpl::EnsureWorkerStubInitialized(Worker* worker) { - if (!worker->stub) { - TF_RETURN_IF_ERROR( - CreateWorkerStub(worker->address, config_.protocol(), &worker->stub)); +Status DataServiceDispatcherImpl::GetOrCreateWorkerStub( + const std::string& worker_address, WorkerService::Stub** out_stub) + LOCKS_EXCLUDED(mu_) { + { + mutex_lock l(mu_); + auto it = worker_stubs_.find(worker_address); + if (it != worker_stubs_.end()) { + *out_stub = it->second.get(); + return Status::OK(); + } + } + std::unique_ptr stub; + TF_RETURN_IF_ERROR( + CreateWorkerStub(worker_address, config_.protocol(), &stub)); + { + mutex_lock l(mu_); + // A concurrent call could have already created the stub. + auto& worker = worker_stubs_[worker_address]; + if (worker == nullptr) { + worker = std::move(stub); + } + *out_stub = worker.get(); } return Status::OK(); } @@ -359,25 +386,21 @@ Status DataServiceDispatcherImpl::AssignTask(std::shared_ptr task) ProcessTaskRequest req; TaskDef* task_def = req.mutable_task(); task_def->set_dataset_id(task->dataset_id); - std::shared_ptr worker; { mutex_lock l(mu_); - worker = workers_[task->worker_address]; std::shared_ptr dataset; TF_RETURN_IF_ERROR(state_.DatasetFromId(task->dataset_id, &dataset)); *task_def->mutable_dataset() = dataset->dataset_def; } - if (!worker) { - return errors::NotFound("No worker found for address ", - task->worker_address); - } task_def->set_task_id(task->task_id); ProcessTaskResponse resp; - TF_RETURN_IF_ERROR(EnsureWorkerStubInitialized(worker.get())); - grpc::Status s = worker->stub->ProcessTask(&client_ctx, req, &resp); + WorkerService::Stub* stub; + TF_RETURN_IF_ERROR(GetOrCreateWorkerStub(task->worker_address, &stub)); + grpc::Status s = stub->ProcessTask(&client_ctx, req, &resp); if (!s.ok()) { return grpc_util::WrapError( - absl::StrCat("Failed to submit task to worker ", worker->address), s); + absl::StrCat("Failed to submit task to worker ", task->worker_address), + s); } return Status::OK(); } @@ -391,7 +414,8 @@ Status DataServiceDispatcherImpl::GetTasks(const GetTasksRequest* request, for (const auto& task : tasks) { TaskInfo* task_info = response->mutable_task_info()->Add(); task_info->set_worker_address(task->worker_address); - task_info->set_id(task->task_id); + task_info->set_task_id(task->task_id); + task_info->set_job_id(task->job_id); } std::shared_ptr job; TF_RETURN_IF_ERROR(state_.JobFromId(request->job_id(), &job)); @@ -405,13 +429,12 @@ Status DataServiceDispatcherImpl::GetWorkers(const GetWorkersRequest* request, GetWorkersResponse* response) { mutex_lock l(mu_); VLOG(3) << "Enter GetWorkers"; - for (const auto& it : workers_) { - std::shared_ptr worker = it.second; + std::vector> workers = state_.ListWorkers(); + for (const auto& worker : workers) { WorkerInfo* info = response->add_workers(); info->set_address(worker->address); - info->set_id(worker->worker_id); } - VLOG(3) << "Returning list of " << workers_.size() + VLOG(3) << "Returning list of " << response->workers_size() << " workers from GetWorkers"; return Status::OK(); } diff --git a/tensorflow/core/data/service/dispatcher_impl.h b/tensorflow/core/data/service/dispatcher_impl.h index 6fa1815e9eb..f4cc6954fe8 100644 --- a/tensorflow/core/data/service/dispatcher_impl.h +++ b/tensorflow/core/data/service/dispatcher_impl.h @@ -71,21 +71,15 @@ class DataServiceDispatcherImpl { GetWorkersResponse* response); private: - struct Worker { - Worker(int64 worker_id, const std::string& address) - : worker_id(worker_id), address(address) {} - - const int64 worker_id; - const std::string address; - std::unique_ptr stub; - }; - // Registers a dataset with the given fingerprint, storing the new dataset's // id in `*dataset-id`. Status RegisterDataset(uint64 fingerprint, const DatasetDef& dataset, int64* dataset_id) EXCLUSIVE_LOCKS_REQUIRED(mu_); - // Initializes a workers stub, if it hasn't been initialized already. - Status EnsureWorkerStubInitialized(Worker* worker); + // Gets a worker's stub from `worker_stubs_`, or if none exists, creates a + // stub and stores it in `worker_stubs_`. + Status GetOrCreateWorkerStub(const std::string& worker_address, + WorkerService::Stub** out_stub) + LOCKS_EXCLUDED(mu_); // Creates a job and stores it in `*job`. This method updates the // dispatcher state with the new job, but does not assign tasks to workers. Status CreateJob(int64 dataset_id, ProcessingMode processing_mode, @@ -128,12 +122,11 @@ class DataServiceDispatcherImpl { mutex mu_; - int64 next_worker_id_ TF_GUARDED_BY(mu_) = 0; int64 next_task_id_ TF_GUARDED_BY(mu_) = 0; - // Registered workers, keyed by their addresses. - absl::flat_hash_map> workers_ - TF_GUARDED_BY(mu_); + // Cached worker stubs for communicating with workers. + absl::flat_hash_map> + worker_stubs_ TF_GUARDED_BY(mu_); absl::optional> journal_writer_ TF_GUARDED_BY(mu_); diff --git a/tensorflow/core/data/service/dispatcher_state.cc b/tensorflow/core/data/service/dispatcher_state.cc index 093457a55af..1e914b69e5b 100644 --- a/tensorflow/core/data/service/dispatcher_state.cc +++ b/tensorflow/core/data/service/dispatcher_state.cc @@ -30,6 +30,9 @@ Status DispatcherState::Apply(Update update) { case Update::kRegisterDataset: RegisterDataset(update.register_dataset()); break; + case Update::kRegisterWorker: + RegisterWorker(update.register_worker()); + break; case Update::kCreateJob: CreateJob(update.create_job()); break; @@ -59,6 +62,13 @@ void DispatcherState::RegisterDataset( next_available_dataset_id_ = std::max(next_available_dataset_id_, id + 1); } +void DispatcherState::RegisterWorker( + const RegisterWorkerUpdate& register_worker) { + std::string address = register_worker.worker_address(); + DCHECK(!workers_.contains(address)); + workers_[address] = std::make_shared(address); +} + void DispatcherState::CreateJob(const CreateJobUpdate& create_job) { int64 job_id = create_job.job_id(); absl::optional named_job_key; @@ -71,6 +81,7 @@ void DispatcherState::CreateJob(const CreateJobUpdate& create_job) { named_job_key); DCHECK(!jobs_.contains(job_id)); jobs_[job_id] = job; + tasks_by_job_[job_id] = std::vector>(); if (named_job_key.has_value()) { DCHECK(!named_jobs_.contains(named_job_key.value())); named_jobs_[named_job_key.value()] = job; @@ -129,6 +140,26 @@ Status DispatcherState::DatasetFromFingerprint( return Status::OK(); } +Status DispatcherState::WorkerFromAddress( + const std::string& address, std::shared_ptr* worker) const { + auto it = workers_.find(address); + if (it == workers_.end()) { + return errors::NotFound("Worker with address ", address, " not found."); + } + *worker = it->second; + return Status::OK(); +} + +std::vector> +DispatcherState::ListWorkers() const { + std::vector> workers; + workers.reserve(workers_.size()); + for (const auto& it : workers_) { + workers.push_back(it.second); + } + return workers; +} + std::vector> DispatcherState::ListJobs() { std::vector> jobs; diff --git a/tensorflow/core/data/service/dispatcher_state.h b/tensorflow/core/data/service/dispatcher_state.h index 7313274ae71..b1aa0aa3979 100644 --- a/tensorflow/core/data/service/dispatcher_state.h +++ b/tensorflow/core/data/service/dispatcher_state.h @@ -48,11 +48,6 @@ namespace data { // DispatcherImpl and for providing DispatcherImpl with read-only access to // the state. // -// Note that not all state needs to be journaled, and in general we journal -// as little state as possible. For example, worker and task state doesn't need -// to be journaled because we can recover that information from workers when -// they reconnect to a restarted dispatcher. -// // DispatcherState is thread-compatible but not thread-safe. class DispatcherState { public: @@ -65,7 +60,8 @@ class DispatcherState { // A dataset registered with the dispatcher. struct Dataset { - Dataset(int64 dataset_id, int64 fingerprint, const DatasetDef& dataset_def) + explicit Dataset(int64 dataset_id, int64 fingerprint, + const DatasetDef& dataset_def) : dataset_id(dataset_id), fingerprint(fingerprint), dataset_def(dataset_def) {} @@ -75,10 +71,17 @@ class DispatcherState { const DatasetDef dataset_def; }; + // A worker registered with the dispatcher. + struct Worker { + explicit Worker(const std::string& address) : address(address) {} + + const std::string address; + }; + // A key for identifying a named job. The key contains a user-specified name, // as well as an index describing which iteration of the job we are on. struct NamedJobKey { - NamedJobKey(absl::string_view name, int64 index) + explicit NamedJobKey(absl::string_view name, int64 index) : name(name), index(index) {} friend bool operator==(const NamedJobKey& lhs, const NamedJobKey& rhs) { @@ -96,8 +99,8 @@ class DispatcherState { // A job for processing a dataset. struct Job { - Job(int64 job_id, int64 dataset_id, ProcessingMode processing_mode, - absl::optional named_job_key) + explicit Job(int64 job_id, int64 dataset_id, ProcessingMode processing_mode, + absl::optional named_job_key) : job_id(job_id), dataset_id(dataset_id), processing_mode(processing_mode), @@ -111,8 +114,8 @@ class DispatcherState { }; struct Task { - Task(int64 task_id, int64 job_id, int64 dataset_id, - const std::string& worker_address) + explicit Task(int64 task_id, int64 job_id, int64 dataset_id, + const std::string& worker_address) : task_id(task_id), job_id(job_id), dataset_id(dataset_id), @@ -134,6 +137,12 @@ class DispatcherState { Status DatasetFromFingerprint(uint64 fingerprint, std::shared_ptr* dataset) const; + // Gets a worker by address. Returns NOT_FOUND if there is no such worker. + Status WorkerFromAddress(const std::string& address, + std::shared_ptr* worker) const; + // Lists all workers registered with the dispatcher. + std::vector> ListWorkers() const; + // Returns the next available job id. int64 NextAvailableJobId() const; // Returns a list of all jobs. @@ -153,8 +162,8 @@ class DispatcherState { std::vector>* tasks) const; private: - // Registers a dataset. The dataset must not already be registered. void RegisterDataset(const RegisterDatasetUpdate& register_dataset); + void RegisterWorker(const RegisterWorkerUpdate& register_worker); void CreateJob(const CreateJobUpdate& create_job); void CreateTask(const CreateTaskUpdate& create_task); void FinishTask(const FinishTaskUpdate& finish_task); @@ -166,6 +175,9 @@ class DispatcherState { absl::flat_hash_map> datasets_by_fingerprint_; + // Registered workers, keyed by address. + absl::flat_hash_map> workers_; + int64 next_available_job_id_ = 0; // Jobs, keyed by job ids. absl::flat_hash_map> jobs_; diff --git a/tensorflow/core/data/service/dispatcher_state_test.cc b/tensorflow/core/data/service/dispatcher_state_test.cc index 933d783d227..b5529951efb 100644 --- a/tensorflow/core/data/service/dispatcher_state_test.cc +++ b/tensorflow/core/data/service/dispatcher_state_test.cc @@ -14,6 +14,8 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/core/data/service/dispatcher_state.h" +#include + #include "tensorflow/core/data/service/common.pb.h" #include "tensorflow/core/data/service/journal.h" #include "tensorflow/core/data/service/journal.pb.h" @@ -27,9 +29,11 @@ namespace data { namespace { using Dataset = DispatcherState::Dataset; +using Worker = DispatcherState::Worker; using NamedJobKey = DispatcherState::NamedJobKey; using Job = DispatcherState::Job; using Task = DispatcherState::Task; +using ::testing::IsEmpty; using ::testing::SizeIs; Status RegisterDatasetWithIdAndFingerprint(int64 id, uint64 fingerprint, @@ -42,6 +46,13 @@ Status RegisterDatasetWithIdAndFingerprint(int64 id, uint64 fingerprint, return Status::OK(); } +Status RegisterWorker(std::string worker_address, DispatcherState* state) { + Update update; + update.mutable_register_worker()->set_worker_address(worker_address); + TF_RETURN_IF_ERROR(state->Apply(update)); + return Status::OK(); +} + Status CreateAnonymousJob(int64 job_id, int64 dataset_id, DispatcherState* state) { Update update; @@ -98,12 +109,12 @@ TEST(DispatcherState, RegisterDataset) { { std::shared_ptr dataset; TF_EXPECT_OK(state.DatasetFromFingerprint(fingerprint, &dataset)); - EXPECT_EQ(id, dataset->dataset_id); + EXPECT_EQ(dataset->dataset_id, id); } { std::shared_ptr dataset; TF_EXPECT_OK(state.DatasetFromId(id, &dataset)); - EXPECT_EQ(fingerprint, dataset->fingerprint); + EXPECT_EQ(dataset->fingerprint, fingerprint); } } @@ -126,10 +137,46 @@ TEST(DispatcherState, NextAvailableDatasetId) { int64 id = state.NextAvailableDatasetId(); uint64 fingerprint = 20; TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(id, fingerprint, &state)); - EXPECT_NE(id, state.NextAvailableDatasetId()); + EXPECT_NE(state.NextAvailableDatasetId(), id); EXPECT_EQ(state.NextAvailableDatasetId(), state.NextAvailableDatasetId()); } +TEST(DispatcherState, RegisterWorker) { + DispatcherState state; + std::string address = "test_worker_address"; + TF_EXPECT_OK(RegisterWorker(address, &state)); + std::shared_ptr worker; + TF_EXPECT_OK(state.WorkerFromAddress(address, &worker)); + EXPECT_EQ(worker->address, address); +} + +TEST(DispatcherState, ListWorkers) { + DispatcherState state; + std::string address_1 = "address_1"; + std::string address_2 = "address_2"; + { + std::vector> workers = state.ListWorkers(); + EXPECT_THAT(workers, IsEmpty()); + } + TF_EXPECT_OK(RegisterWorker(address_1, &state)); + { + std::vector> workers = state.ListWorkers(); + EXPECT_THAT(workers, SizeIs(1)); + } + TF_EXPECT_OK(RegisterWorker(address_2, &state)); + { + std::vector> workers = state.ListWorkers(); + EXPECT_THAT(workers, SizeIs(2)); + } +} + +TEST(DispatcherState, MissingWorker) { + DispatcherState state; + std::shared_ptr worker; + Status s = state.WorkerFromAddress("test_worker_address", &worker); + EXPECT_EQ(s.code(), error::NOT_FOUND); +} + TEST(DispatcherState, UnknownUpdate) { DispatcherState state; Update update; @@ -146,8 +193,11 @@ TEST(DispatcherState, AnonymousJob) { std::shared_ptr job; TF_EXPECT_OK(state.JobFromId(job_id, &job)); EXPECT_EQ(state.NextAvailableJobId(), job_id + 1); - EXPECT_EQ(dataset_id, job->dataset_id); - EXPECT_EQ(job_id, job->job_id); + EXPECT_EQ(job->dataset_id, dataset_id); + EXPECT_EQ(job->job_id, job_id); + std::vector> tasks; + TF_EXPECT_OK(state.TasksForJob(job_id, &tasks)); + EXPECT_THAT(tasks, IsEmpty()); EXPECT_FALSE(job->finished); } @@ -161,8 +211,8 @@ TEST(DispatcherState, NamedJob) { std::shared_ptr job; TF_EXPECT_OK(state.NamedJobByKey(named_job_key, &job)); EXPECT_EQ(state.NextAvailableJobId(), job_id + 1); - EXPECT_EQ(dataset_id, job->dataset_id); - EXPECT_EQ(job_id, job->job_id); + EXPECT_EQ(job->dataset_id, dataset_id); + EXPECT_EQ(job->job_id, job_id); EXPECT_FALSE(job->finished); } @@ -179,10 +229,10 @@ TEST(DispatcherState, CreateTask) { { std::shared_ptr task; TF_EXPECT_OK(state.TaskFromId(task_id, &task)); - EXPECT_EQ(task_id, task->task_id); - EXPECT_EQ(job_id, task->job_id); - EXPECT_EQ(dataset_id, task->dataset_id); - EXPECT_EQ(worker_address, task->worker_address); + EXPECT_EQ(task->task_id, task_id); + EXPECT_EQ(task->job_id, job_id); + EXPECT_EQ(task->dataset_id, dataset_id); + EXPECT_EQ(task->worker_address, worker_address); } { std::vector> tasks; @@ -207,7 +257,7 @@ TEST(DispatcherState, CreateTasksForSameJob) { { std::vector> tasks; TF_EXPECT_OK(state.TasksForJob(job_id, &tasks)); - EXPECT_EQ(2, tasks.size()); + EXPECT_THAT(tasks, SizeIs(2)); } } @@ -229,12 +279,12 @@ TEST(DispatcherState, CreateTasksForDifferentJobs) { { std::vector> tasks; TF_EXPECT_OK(state.TasksForJob(job_id_1, &tasks)); - EXPECT_EQ(1, tasks.size()); + EXPECT_THAT(tasks, SizeIs(1)); } { std::vector> tasks; TF_EXPECT_OK(state.TasksForJob(job_id_2, &tasks)); - EXPECT_EQ(1, tasks.size()); + EXPECT_THAT(tasks, SizeIs(1)); } } diff --git a/tensorflow/core/data/service/journal.cc b/tensorflow/core/data/service/journal.cc index a9aa43b9758..11952b0dfd9 100644 --- a/tensorflow/core/data/service/journal.cc +++ b/tensorflow/core/data/service/journal.cc @@ -48,7 +48,7 @@ Status FileJournalWriter::EnsureInitialized() { return Status::OK(); } -Status FileJournalWriter::Write(Update update) { +Status FileJournalWriter::Write(const Update& update) { TF_RETURN_IF_ERROR(EnsureInitialized()); std::string s = update.SerializeAsString(); if (s.empty()) { diff --git a/tensorflow/core/data/service/journal.h b/tensorflow/core/data/service/journal.h index 112c3b614be..c627c21756c 100644 --- a/tensorflow/core/data/service/journal.h +++ b/tensorflow/core/data/service/journal.h @@ -32,7 +32,7 @@ class JournalWriter { public: virtual ~JournalWriter() = default; // Writes and syncs an update to the journal. - virtual Status Write(Update update) = 0; + virtual Status Write(const Update& update) = 0; }; // FileJournalWriter is not thread-safe, requiring external synchronization when @@ -46,7 +46,7 @@ class FileJournalWriter : public JournalWriter { FileJournalWriter(const FileJournalWriter&) = delete; FileJournalWriter& operator=(const FileJournalWriter&) = delete; - Status Write(Update update) override; + Status Write(const Update& update) override; private: // Initializes the writer if it is not yet initialized. diff --git a/tensorflow/core/data/service/journal.proto b/tensorflow/core/data/service/journal.proto index fd4c5863ca9..725724a5cd5 100644 --- a/tensorflow/core/data/service/journal.proto +++ b/tensorflow/core/data/service/journal.proto @@ -10,6 +10,7 @@ import "tensorflow/core/data/service/common.proto"; message Update { oneof update_type { RegisterDatasetUpdate register_dataset = 1; + RegisterWorkerUpdate register_worker = 5; CreateJobUpdate create_job = 2; CreateTaskUpdate create_task = 3; FinishTaskUpdate finish_task = 4; @@ -22,6 +23,10 @@ message RegisterDatasetUpdate { uint64 fingerprint = 3; } +message RegisterWorkerUpdate { + string worker_address = 1; +} + message NamedJobKeyDef { string name = 1; int64 index = 2; diff --git a/tensorflow/core/data/service/worker_impl.cc b/tensorflow/core/data/service/worker_impl.cc index c6338d540f8..6326d65782b 100644 --- a/tensorflow/core/data/service/worker_impl.cc +++ b/tensorflow/core/data/service/worker_impl.cc @@ -197,8 +197,6 @@ Status DataServiceWorkerImpl::Register() EXCLUSIVE_LOCKS_REQUIRED(mu_) { for (const TaskDef& task : resp.tasks()) { TF_RETURN_IF_ERROR(ProcessTaskInternal(task)); } - worker_id_ = resp.worker_id(); - VLOG(3) << "Registered worker with id " << resp.worker_id(); return Status::OK(); } @@ -207,7 +205,6 @@ Status DataServiceWorkerImpl::SendTaskUpdate() EXCLUSIVE_LOCKS_REQUIRED(mu_) { << " task updates to dispatcher"; TF_RETURN_IF_ERROR(EnsureDispatcherStubInitialized()); WorkerUpdateRequest req; - req.set_worker_id(worker_id_); for (int task_id : pending_completed_tasks_) { TaskProgress* update = req.add_updates(); update->set_task_id(task_id); diff --git a/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc b/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc index 8e1713e2d77..233a61f440e 100644 --- a/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc @@ -338,7 +338,7 @@ class DataServiceDatasetOp::Dataset : public DatasetBase { } absl::flat_hash_map task_id_to_task; for (auto& task : tasks) { - task_id_to_task[task.id()] = task; + task_id_to_task[task.task_id()] = task; } mutex_lock l(mu_); job_finished_ = job_finished; @@ -371,8 +371,9 @@ class DataServiceDatasetOp::Dataset : public DatasetBase { get_next_cv_.notify_all(); continue; } - tasks_.push_back(std::make_shared( - task_info.id(), task_info.worker_address(), std::move(worker))); + tasks_.push_back(std::make_shared(task_info.task_id(), + task_info.worker_address(), + std::move(worker))); } if (dataset()->max_outstanding_requests_ == model::kAutotune) { // Adjust max_outstanding_requests to account for newly added tasks. diff --git a/tensorflow/core/protobuf/data/experimental/service_config.proto b/tensorflow/core/protobuf/data/experimental/service_config.proto index 872a47013eb..017aaa2a960 100644 --- a/tensorflow/core/protobuf/data/experimental/service_config.proto +++ b/tensorflow/core/protobuf/data/experimental/service_config.proto @@ -12,6 +12,9 @@ message DispatcherConfig { // An optional work directory to use for storing dispatcher state, and for // recovering during restarts. string work_dir = 3; + // Whether to run in fault tolerant mode, where dispatcher state is saved + // across restarts. + bool fault_tolerant_mode = 4; } // Configuration for a tf.data service WorkerServer. From 46a8319ee74337182c7aadf80acbeb7f01eb7ffd Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Tue, 4 Aug 2020 18:04:52 -0700 Subject: [PATCH 0419/1017] Moving rest of the filesystems to Transactional API --- .../platform/default/posix_file_system.cc | 59 +++++++------- .../core/platform/default/posix_file_system.h | 68 +++++++---------- .../platform/hadoop/hadoop_file_system.cc | 57 +++++++------- .../core/platform/hadoop/hadoop_file_system.h | 67 ++++++---------- .../platform/windows/windows_file_system.cc | 60 +++++++-------- .../platform/windows/windows_file_system.h | 76 +++++++------------ tensorflow/core/util/memmapped_file_system.cc | 62 +++++++-------- tensorflow/core/util/memmapped_file_system.h | 63 ++++++--------- .../asset_manager_filesystem.cc | 61 +++++++-------- .../asset_manager_filesystem.h | 66 +++++++--------- 10 files changed, 276 insertions(+), 363 deletions(-) diff --git a/tensorflow/core/platform/default/posix_file_system.cc b/tensorflow/core/platform/default/posix_file_system.cc index 8533e34fc3f..18fea3fe15d 100644 --- a/tensorflow/core/platform/default/posix_file_system.cc +++ b/tensorflow/core/platform/default/posix_file_system.cc @@ -178,8 +178,8 @@ class PosixReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { }; Status PosixFileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); Status s; int fd = open(translated_fname.c_str(), O_RDONLY); @@ -191,9 +191,9 @@ Status PosixFileSystem::NewRandomAccessFile( return s; } -Status PosixFileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { +Status PosixFileSystem::NewWritableFile(const string& fname, + TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); Status s; FILE* f = fopen(translated_fname.c_str(), "w"); @@ -206,8 +206,8 @@ Status PosixFileSystem::NewWritableFile( } Status PosixFileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); Status s; FILE* f = fopen(translated_fname.c_str(), "a"); @@ -220,8 +220,8 @@ Status PosixFileSystem::NewAppendableFile( } Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); Status s = Status::OK(); int fd = open(translated_fname.c_str(), O_RDONLY); @@ -244,17 +244,16 @@ Status PosixFileSystem::NewReadOnlyMemoryRegionFromFile( return s; } -Status PosixFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status PosixFileSystem::FileExists(const string& fname, + TransactionToken* token) { if (access(TranslateName(fname).c_str(), F_OK) == 0) { return Status::OK(); } return errors::NotFound(fname, " not found"); } -Status PosixFileSystem::GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token */) { +Status PosixFileSystem::GetChildren(const string& dir, TransactionToken* token, + std::vector* result) { string translated_dir = TranslateName(dir); result->clear(); DIR* d = opendir(translated_dir.c_str()); @@ -274,14 +273,14 @@ Status PosixFileSystem::GetChildren( return Status::OK(); } -Status PosixFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status PosixFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { return internal::GetMatchingPaths(this, Env::Default(), pattern, results); } -Status PosixFileSystem::DeleteFile( - const string& fname /*, TransactionToken* token */) { +Status PosixFileSystem::DeleteFile(const string& fname, + TransactionToken* token) { Status result; if (unlink(TranslateName(fname).c_str()) != 0) { result = IOError(fname, errno); @@ -289,8 +288,7 @@ Status PosixFileSystem::DeleteFile( return result; } -Status PosixFileSystem::CreateDir( - const string& name /*, TransactionToken* token */) { +Status PosixFileSystem::CreateDir(const string& name, TransactionToken* token) { string translated = TranslateName(name); if (translated.empty()) { return errors::AlreadyExists(name); @@ -301,8 +299,7 @@ Status PosixFileSystem::CreateDir( return Status::OK(); } -Status PosixFileSystem::DeleteDir( - const string& name /*, TransactionToken* token */) { +Status PosixFileSystem::DeleteDir(const string& name, TransactionToken* token) { Status result; if (rmdir(TranslateName(name).c_str()) != 0) { result = IOError(name, errno); @@ -310,8 +307,8 @@ Status PosixFileSystem::DeleteDir( return result; } -Status PosixFileSystem::GetFileSize( - const string& fname, uint64* size /*, TransactionToken* token */) { +Status PosixFileSystem::GetFileSize(const string& fname, + TransactionToken* token, uint64* size) { Status s; struct stat sbuf; if (stat(TranslateName(fname).c_str(), &sbuf) != 0) { @@ -323,8 +320,8 @@ Status PosixFileSystem::GetFileSize( return s; } -Status PosixFileSystem::Stat( - const string& fname, FileStatistics* stats /*, TransactionToken* token */) { +Status PosixFileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stats) { Status s; struct stat sbuf; if (stat(TranslateName(fname).c_str(), &sbuf) != 0) { @@ -337,8 +334,8 @@ Status PosixFileSystem::Stat( return s; } -Status PosixFileSystem::RenameFile( - const string& src, const string& target /*, TransactionToken* token */) { +Status PosixFileSystem::RenameFile(const string& src, const string& target, + TransactionToken* token) { Status result; if (rename(TranslateName(src).c_str(), TranslateName(target).c_str()) != 0) { result = IOError(src, errno); @@ -346,8 +343,8 @@ Status PosixFileSystem::RenameFile( return result; } -Status PosixFileSystem::CopyFile( - const string& src, const string& target /*, TransactionToken* token */) { +Status PosixFileSystem::CopyFile(const string& src, const string& target, + TransactionToken* token) { string translated_src = TranslateName(src); struct stat sbuf; if (stat(translated_src.c_str(), &sbuf) != 0) { diff --git a/tensorflow/core/platform/default/posix_file_system.h b/tensorflow/core/platform/default/posix_file_system.h index a1c6f34ad65..8e301c8b2e4 100644 --- a/tensorflow/core/platform/default/posix_file_system.h +++ b/tensorflow/core/platform/default/posix_file_system.h @@ -27,63 +27,47 @@ class PosixFileSystem : public FileSystem { ~PosixFileSystem() {} + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status FileExists(const string& fname, TransactionToken* token) override; - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override; - Status Stat( - const string& fname, - FileStatistics* stats /*, TransactionToken* token = nullptr */) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stats) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr */) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status DeleteFile(const string& fname, TransactionToken* token) override; - Status CreateDir( - const string& name /*, TransactionToken* token = nullptr */) override; + Status CreateDir(const string& name, TransactionToken* token) override; - Status DeleteDir( - const string& name /*, TransactionToken* token = nullptr */) override; + Status DeleteDir(const string& name, TransactionToken* token) override; - Status GetFileSize( - const string& fname, - uint64* size /*, TransactionToken* token = nullptr */) override; + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* size) override; - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override; + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override; - Status CopyFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override; + Status CopyFile(const string& src, const string& target, + TransactionToken* token) override; }; Status IOError(const string& context, int err_number); diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc index 5b2c5a76aae..f8ed61c3ac9 100644 --- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc +++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc @@ -280,8 +280,8 @@ class HDFSRandomAccessFile : public RandomAccessFile { }; Status HadoopFileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -373,8 +373,8 @@ class HDFSWritableFile : public WritableFile { }; Status HadoopFileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -388,8 +388,8 @@ Status HadoopFileSystem::NewWritableFile( } Status HadoopFileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -403,8 +403,8 @@ Status HadoopFileSystem::NewAppendableFile( } Status HadoopFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { // hadoopReadZero() technically supports this call with the following // caveats: // - It only works up to 2 GB. We'd have to Stat() the file to ensure that @@ -414,8 +414,8 @@ Status HadoopFileSystem::NewReadOnlyMemoryRegionFromFile( return errors::Unimplemented("HDFS does not support ReadOnlyMemoryRegion"); } -Status HadoopFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status HadoopFileSystem::FileExists(const string& fname, + TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); if (libhdfs()->hdfsExists(fs, TranslateName(fname).c_str()) == 0) { @@ -424,9 +424,8 @@ Status HadoopFileSystem::FileExists( return errors::NotFound(fname, " not found."); } -Status HadoopFileSystem::GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token */) { +Status HadoopFileSystem::GetChildren(const string& dir, TransactionToken* token, + std::vector* result) { result->clear(); hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(dir, &fs)); @@ -434,7 +433,7 @@ Status HadoopFileSystem::GetChildren( // hdfsListDirectory returns nullptr if the directory is empty. Do a separate // check to verify the directory exists first. FileStatistics stat; - TF_RETURN_IF_ERROR(Stat(dir, &stat)); + TF_RETURN_IF_ERROR(Stat(dir, token, &stat)); int entries = 0; hdfsFileInfo* info = @@ -453,14 +452,14 @@ Status HadoopFileSystem::GetChildren( return Status::OK(); } -Status HadoopFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status HadoopFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { return internal::GetMatchingPaths(this, Env::Default(), pattern, results); } -Status HadoopFileSystem::DeleteFile( - const string& fname /*, TransactionToken* token */) { +Status HadoopFileSystem::DeleteFile(const string& fname, + TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -471,8 +470,7 @@ Status HadoopFileSystem::DeleteFile( return Status::OK(); } -Status HadoopFileSystem::CreateDir( - const string& dir /*, TransactionToken* token */) { +Status HadoopFileSystem::CreateDir(const string& dir, TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(dir, &fs)); @@ -482,8 +480,7 @@ Status HadoopFileSystem::CreateDir( return Status::OK(); } -Status HadoopFileSystem::DeleteDir( - const string& dir /*, TransactionToken* token */) { +Status HadoopFileSystem::DeleteDir(const string& dir, TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(dir, &fs)); @@ -502,7 +499,7 @@ Status HadoopFileSystem::DeleteDir( // the call is actually successful. Check again by Stat. if (info == nullptr && errno != 0) { FileStatistics stat; - TF_RETURN_IF_ERROR(Stat(dir, &stat)); + TF_RETURN_IF_ERROR(Stat(dir, token, &stat)); } if (entries > 0) { @@ -515,8 +512,8 @@ Status HadoopFileSystem::DeleteDir( return Status::OK(); } -Status HadoopFileSystem::GetFileSize( - const string& fname, uint64* size /*, TransactionToken* token */) { +Status HadoopFileSystem::GetFileSize(const string& fname, + TransactionToken* token, uint64* size) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); @@ -530,8 +527,8 @@ Status HadoopFileSystem::GetFileSize( return Status::OK(); } -Status HadoopFileSystem::RenameFile( - const string& src, const string& target /*, TransactionToken* token */) { +Status HadoopFileSystem::RenameFile(const string& src, const string& target, + TransactionToken* token) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(src, &fs)); @@ -548,8 +545,8 @@ Status HadoopFileSystem::RenameFile( return Status::OK(); } -Status HadoopFileSystem::Stat( - const string& fname, FileStatistics* stats /*, TransactionToken* token */) { +Status HadoopFileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stats) { hdfsFS fs = nullptr; TF_RETURN_IF_ERROR(Connect(fname, &fs)); diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.h b/tensorflow/core/platform/hadoop/hadoop_file_system.h index 13abc067cd8..5e7233633a6 100644 --- a/tensorflow/core/platform/hadoop/hadoop_file_system.h +++ b/tensorflow/core/platform/hadoop/hadoop_file_system.h @@ -32,63 +32,46 @@ class HadoopFileSystem : public FileSystem { HadoopFileSystem(); ~HadoopFileSystem(); + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr*/) override; + Status FileExists(const string& fname, TransactionToken* token) override; - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr*/) - override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr*/) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr*/) override; + Status DeleteFile(const string& fname, TransactionToken* token) override; - Status CreateDir( - const string& name /*, TransactionToken* token = nullptr*/) override; + Status CreateDir(const string& name, TransactionToken* token) override; - Status DeleteDir( - const string& name /*, TransactionToken* token = nullptr*/) override; + Status DeleteDir(const string& name, TransactionToken* token) override; - Status GetFileSize( - const string& fname, - uint64* size /*, TransactionToken* token = nullptr*/) override; + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* size) override; - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr*/) override; + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override; - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr*/) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; - string TranslateName( - const string& name /*, TransactionToken* token = nullptr*/) - const override; + string TranslateName(const string& name) const override; private: Status Connect(StringPiece fname, hdfsFS* fs); diff --git a/tensorflow/core/platform/windows/windows_file_system.cc b/tensorflow/core/platform/windows/windows_file_system.cc index e1e8656bce5..475f8791144 100644 --- a/tensorflow/core/platform/windows/windows_file_system.cc +++ b/tensorflow/core/platform/windows/windows_file_system.cc @@ -261,8 +261,8 @@ class WinReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { } // namespace Status WindowsFileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); result->reset(); @@ -289,8 +289,8 @@ Status WindowsFileSystem::NewRandomAccessFile( } Status WindowsFileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); result->reset(); @@ -310,8 +310,8 @@ Status WindowsFileSystem::NewWritableFile( } Status WindowsFileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); result->reset(); @@ -341,8 +341,8 @@ Status WindowsFileSystem::NewAppendableFile( } Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string translated_fname = TranslateName(fname); std::wstring ws_translated_fname = Utf8ToWideChar(translated_fname); result->reset(); @@ -418,8 +418,8 @@ Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile( return s; } -Status WindowsFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status WindowsFileSystem::FileExists(const string& fname, + TransactionToken* token) { constexpr int kOk = 0; std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); if (_waccess(ws_translated_fname.c_str(), kOk) == 0) { @@ -428,9 +428,9 @@ Status WindowsFileSystem::FileExists( return errors::NotFound(fname, " not found"); } -Status WindowsFileSystem::GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token */) { +Status WindowsFileSystem::GetChildren(const string& dir, + TransactionToken* token, + std::vector* result) { string translated_dir = TranslateName(dir); std::wstring ws_translated_dir = Utf8ToWideChar(translated_dir); result->clear(); @@ -465,8 +465,8 @@ Status WindowsFileSystem::GetChildren( return Status::OK(); } -Status WindowsFileSystem::DeleteFile( - const string& fname /*, TransactionToken* token */) { +Status WindowsFileSystem::DeleteFile(const string& fname, + TransactionToken* token) { Status result; std::wstring file_name = Utf8ToWideChar(fname); if (_wunlink(file_name.c_str()) != 0) { @@ -475,8 +475,8 @@ Status WindowsFileSystem::DeleteFile( return result; } -Status WindowsFileSystem::CreateDir( - const string& name /*, TransactionToken* token */) { +Status WindowsFileSystem::CreateDir(const string& name, + TransactionToken* token) { Status result; std::wstring ws_name = Utf8ToWideChar(name); if (ws_name.empty()) { @@ -488,8 +488,8 @@ Status WindowsFileSystem::CreateDir( return result; } -Status WindowsFileSystem::DeleteDir( - const string& name /*, TransactionToken* token */) { +Status WindowsFileSystem::DeleteDir(const string& name, + TransactionToken* token) { Status result; std::wstring ws_name = Utf8ToWideChar(name); if (_wrmdir(ws_name.c_str()) != 0) { @@ -498,8 +498,8 @@ Status WindowsFileSystem::DeleteDir( return result; } -Status WindowsFileSystem::GetFileSize( - const string& fname, uint64* size /*, TransactionToken* token */) { +Status WindowsFileSystem::GetFileSize(const string& fname, + TransactionToken* token, uint64* size) { string translated_fname = TranslateName(fname); std::wstring ws_translated_dir = Utf8ToWideChar(translated_fname); Status result; @@ -517,8 +517,8 @@ Status WindowsFileSystem::GetFileSize( return result; } -Status WindowsFileSystem::IsDirectory( - const string& fname /*, TransactionToken* token */) { +Status WindowsFileSystem::IsDirectory(const string& fname, + TransactionToken* token) { TF_RETURN_IF_ERROR(FileExists(fname)); std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); if (PathIsDirectoryW(ws_translated_fname.c_str())) { @@ -527,8 +527,8 @@ Status WindowsFileSystem::IsDirectory( return Status(tensorflow::error::FAILED_PRECONDITION, "Not a directory"); } -Status WindowsFileSystem::RenameFile( - const string& src, const string& target /*, TransactionToken* token */) { +Status WindowsFileSystem::RenameFile(const string& src, const string& target, + TransactionToken* token) { Status result; // rename() is not capable of replacing the existing file as on Linux // so use OS API directly @@ -542,9 +542,9 @@ Status WindowsFileSystem::RenameFile( return result; } -Status WindowsFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status WindowsFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { // NOTE(mrry): The existing implementation of FileSystem::GetMatchingPaths() // does not handle Windows paths containing backslashes correctly. Since // Windows APIs will accept forward and backslashes equivalently, we @@ -567,8 +567,8 @@ bool WindowsFileSystem::Match(const string& filename, const string& pattern) { return PathMatchSpecW(ws_path.c_str(), ws_pattern.c_str()) == TRUE; } -Status WindowsFileSystem::Stat( - const string& fname, FileStatistics* stat /*, TransactionToken* token */) { +Status WindowsFileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) { Status result; struct _stat sbuf; std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname)); diff --git a/tensorflow/core/platform/windows/windows_file_system.h b/tensorflow/core/platform/windows/windows_file_system.h index 604cd141e40..8c550f53b84 100644 --- a/tensorflow/core/platform/windows/windows_file_system.h +++ b/tensorflow/core/platform/windows/windows_file_system.h @@ -32,72 +32,50 @@ class WindowsFileSystem : public FileSystem { ~WindowsFileSystem() {} + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status FileExists(const string& fname, TransactionToken* token) override; - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* result /*, TransactionToken* token = nullptr */) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* result) override; - bool Match( - const string& filename, - const string& pattern /*, TransactionToken* token = nullptr */) override; + bool Match(const string& filename, const string& pattern) override; - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status DeleteFile(const string& fname, TransactionToken* token) override; - Status CreateDir( - const string& name /*, TransactionToken* token = nullptr */) override; + Status CreateDir(const string& name, TransactionToken* token) override; - Status DeleteDir( - const string& name /*, TransactionToken* token = nullptr */) override; + Status DeleteDir(const string& name, TransactionToken* token) override; - Status GetFileSize( - const string& fname, - uint64* size /*, TransactionToken* token = nullptr */) override; + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* size) override; - Status IsDirectory( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status IsDirectory(const string& fname, TransactionToken* token) override; - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override; + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override; - string TranslateName( - const string& name /*, TransactionToken* token = nullptr */) - const override { - return name; - } + string TranslateName(const string& name) const override { return name; } char Separator() const override { return '\\'; }; }; diff --git a/tensorflow/core/util/memmapped_file_system.cc b/tensorflow/core/util/memmapped_file_system.cc index 1451d6350ce..c6bda8b07e9 100644 --- a/tensorflow/core/util/memmapped_file_system.cc +++ b/tensorflow/core/util/memmapped_file_system.cc @@ -86,8 +86,8 @@ class RandomAccessFileFromMemmapped : public RandomAccessFile { MemmappedFileSystem::MemmappedFileSystem() {} -Status MemmappedFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status MemmappedFileSystem::FileExists(const string& fname, + TransactionToken* token) { if (!mapped_memory_) { return errors::FailedPrecondition("MemmappedEnv is not initialized"); } @@ -99,8 +99,8 @@ Status MemmappedFileSystem::FileExists( } Status MemmappedFileSystem::NewRandomAccessFile( - const string& filename, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { if (!mapped_memory_) { return errors::FailedPrecondition("MemmappedEnv is not initialized"); } @@ -115,8 +115,8 @@ Status MemmappedFileSystem::NewRandomAccessFile( } Status MemmappedFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& filename, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { if (!mapped_memory_) { return errors::FailedPrecondition("MemmappedEnv is not initialized"); } @@ -130,8 +130,8 @@ Status MemmappedFileSystem::NewReadOnlyMemoryRegionFromFile( return Status::OK(); } -Status MemmappedFileSystem::GetFileSize( - const string& filename, uint64* size /*, TransactionToken* token */) { +Status MemmappedFileSystem::GetFileSize(const string& filename, + TransactionToken* token, uint64* size) { if (!mapped_memory_) { return errors::FailedPrecondition("MemmappedEnv is not initialized"); } @@ -143,59 +143,59 @@ Status MemmappedFileSystem::GetFileSize( return Status::OK(); } -Status MemmappedFileSystem::Stat( - const string& fname, FileStatistics* stat /*, TransactionToken* token */) { +Status MemmappedFileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) { uint64 size; - auto status = GetFileSize(fname, &size); + auto status = GetFileSize(fname, token, &size); if (status.ok()) { stat->length = size; } return status; } -Status MemmappedFileSystem::NewWritableFile( - const string& filename, - std::unique_ptr* wf /*, TransactionToken* token */) { +Status MemmappedFileSystem::NewWritableFile(const string& filename, + TransactionToken* token, + std::unique_ptr* wf) { return errors::Unimplemented("memmapped format doesn't support writing"); } Status MemmappedFileSystem::NewAppendableFile( - const string& filename, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { return errors::Unimplemented("memmapped format doesn't support writing"); } -Status MemmappedFileSystem::GetChildren( - const string& filename, - std::vector* strings /*, TransactionToken* token */) { +Status MemmappedFileSystem::GetChildren(const string& filename, + TransactionToken* token, + std::vector* strings) { return errors::Unimplemented("memmapped format doesn't support GetChildren"); } -Status MemmappedFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status MemmappedFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { return errors::Unimplemented( "memmapped format doesn't support GetMatchingPaths"); } -Status MemmappedFileSystem::DeleteFile( - const string& filename /*, TransactionToken* token */) { +Status MemmappedFileSystem::DeleteFile(const string& filename, + TransactionToken* token) { return errors::Unimplemented("memmapped format doesn't support DeleteFile"); } -Status MemmappedFileSystem::CreateDir( - const string& dirname /*, TransactionToken* token */) { +Status MemmappedFileSystem::CreateDir(const string& dirname, + TransactionToken* token) { return errors::Unimplemented("memmapped format doesn't support CreateDir"); } -Status MemmappedFileSystem::DeleteDir( - const string& dirname /*, TransactionToken* token */) { +Status MemmappedFileSystem::DeleteDir(const string& dirname, + TransactionToken* token) { return errors::Unimplemented("memmapped format doesn't support DeleteDir"); } -Status MemmappedFileSystem::RenameFile( - const string& filename_from, - const string& filename_to /*, TransactionToken* token */) { +Status MemmappedFileSystem::RenameFile(const string& filename_from, + const string& filename_to, + TransactionToken* token) { return errors::Unimplemented("memmapped format doesn't support RenameFile"); } diff --git a/tensorflow/core/util/memmapped_file_system.h b/tensorflow/core/util/memmapped_file_system.h index d8f19444454..27305a500f5 100644 --- a/tensorflow/core/util/memmapped_file_system.h +++ b/tensorflow/core/util/memmapped_file_system.h @@ -60,52 +60,39 @@ class MemmappedFileSystem : public FileSystem { MemmappedFileSystem(); ~MemmappedFileSystem() override = default; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override; + + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + + Status FileExists(const string& fname, TransactionToken* token) override; Status NewRandomAccessFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; // All these functions return Unimplemented error, the memmapped storage is // read only. - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; - Status GetChildren(const string& dir, - std::vector* - r /*, TransactionToken* token = nullptr */) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr */) - override; - Status DeleteFile( - const string& f /*, TransactionToken* token = nullptr */) override; - Status CreateDir( - const string& d /*, TransactionToken* token = nullptr */) override; - Status DeleteDir( - const string& d /*, TransactionToken* token = nullptr */) override; - Status RenameFile( - const string& s, - const string& t /*, TransactionToken* token = nullptr */) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* r) override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; + Status DeleteFile(const string& f, TransactionToken* token) override; + Status CreateDir(const string& d, TransactionToken* token) override; + Status DeleteDir(const string& d, TransactionToken* token) override; + Status RenameFile(const string& s, const string& t, + TransactionToken* token) override; // These functions are implemented. - Status GetFileSize( - const string& f, - uint64* s /*, TransactionToken* token = nullptr */) override; + Status GetFileSize(const string& f, TransactionToken* token, + uint64* s) override; // Currently just returns size. - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; // Initializes filesystem from a file in memmapped format. Status InitializeFromFile(Env* env, const string& filename); diff --git a/tensorflow/tools/android/inference_interface/asset_manager_filesystem.cc b/tensorflow/tools/android/inference_interface/asset_manager_filesystem.cc index 04d5774adb8..648affc9926 100644 --- a/tensorflow/tools/android/inference_interface/asset_manager_filesystem.cc +++ b/tensorflow/tools/android/inference_interface/asset_manager_filesystem.cc @@ -124,8 +124,8 @@ AssetManagerFileSystem::AssetManagerFileSystem(AAssetManager* asset_manager, const string& prefix) : asset_manager_(asset_manager), prefix_(prefix) {} -Status AssetManagerFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status AssetManagerFileSystem::FileExists(const string& fname, + TransactionToken* token) { string path = RemoveAssetPrefix(fname); auto asset = ScopedAsset( AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM)); @@ -136,8 +136,8 @@ Status AssetManagerFileSystem::FileExists( } Status AssetManagerFileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string path = RemoveAssetPrefix(fname); auto asset = ScopedAsset( AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_RANDOM)); @@ -149,8 +149,8 @@ Status AssetManagerFileSystem::NewRandomAccessFile( } Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string path = RemoveAssetPrefix(fname); auto asset = ScopedAsset( AAssetManager_open(asset_manager_, path.c_str(), AASSET_MODE_STREAMING)); @@ -186,9 +186,9 @@ Status AssetManagerFileSystem::NewReadOnlyMemoryRegionFromFile( return Status::OK(); } -Status AssetManagerFileSystem::GetChildren( - const string& prefixed_dir, - std::vector* r /*, TransactionToken* token */) { +Status AssetManagerFileSystem::GetChildren(const string& prefixed_dir, + TransactionToken* token, + std::vector* r) { std::string path = NormalizeDirectoryPath(prefixed_dir); auto dir = ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str())); @@ -203,8 +203,8 @@ Status AssetManagerFileSystem::GetChildren( return Status::OK(); } -Status AssetManagerFileSystem::GetFileSize( - const string& fname, uint64* s /*, TransactionToken* token */) { +Status AssetManagerFileSystem::GetFileSize(const string& fname, + TransactionToken* token, uint64* s) { // If fname corresponds to a directory, return early. It doesn't map to an // AAsset, and would otherwise return NotFound. if (DirectoryExists(fname)) { @@ -221,8 +221,9 @@ Status AssetManagerFileSystem::GetFileSize( return Status::OK(); } -Status AssetManagerFileSystem::Stat( - const string& fname, FileStatistics* stat /*, TransactionToken* token */) { +Status AssetManagerFileSystem::Stat(const string& fname, + TransactionToken* token, + FileStatistics* stat) { uint64 size; stat->is_directory = DirectoryExists(fname); TF_RETURN_IF_ERROR(GetFileSize(fname, &size)); @@ -240,8 +241,8 @@ string AssetManagerFileSystem::RemoveAssetPrefix(const string& name) { return string(piece); } -bool AssetManagerFileSystem::DirectoryExists( - const std::string& fname /*, TransactionToken* token */) { +bool AssetManagerFileSystem::DirectoryExists(const std::string& fname, + TransactionToken* token) { std::string path = NormalizeDirectoryPath(fname); auto dir = ScopedAssetDir(AAssetManager_openDir(asset_manager_, path.c_str())); @@ -250,36 +251,36 @@ bool AssetManagerFileSystem::DirectoryExists( return AAssetDir_getNextFileName(dir.get()) != NULL; } -Status AssetManagerFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status AssetManagerFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { return internal::GetMatchingPaths(this, Env::Default(), pattern, results); } Status AssetManagerFileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { return errors::Unimplemented("Asset storage is read only."); } Status AssetManagerFileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { return errors::Unimplemented("Asset storage is read only."); } -Status AssetManagerFileSystem::DeleteFile( - const string& f /*, TransactionToken* token */) { +Status AssetManagerFileSystem::DeleteFile(const string& f, + TransactionToken* token) { return errors::Unimplemented("Asset storage is read only."); } -Status AssetManagerFileSystem::CreateDir( - const string& d /*, TransactionToken* token */) { +Status AssetManagerFileSystem::CreateDir(const string& d, + TransactionToken* token) { return errors::Unimplemented("Asset storage is read only."); } -Status AssetManagerFileSystem::DeleteDir( - const string& d /*, TransactionToken* token */) { +Status AssetManagerFileSystem::DeleteDir(const string& d, + TransactionToken* token) { return errors::Unimplemented("Asset storage is read only."); } -Status AssetManagerFileSystem::RenameFile( - const string& s, const string& t /*, TransactionToken* token */) { +Status AssetManagerFileSystem::RenameFile(const string& s, const string& t, + TransactionToken* token) { return errors::Unimplemented("Asset storage is read only."); } diff --git a/tensorflow/tools/android/inference_interface/asset_manager_filesystem.h b/tensorflow/tools/android/inference_interface/asset_manager_filesystem.h index 329e55d6cc7..893d5ccb90a 100644 --- a/tensorflow/tools/android/inference_interface/asset_manager_filesystem.h +++ b/tensorflow/tools/android/inference_interface/asset_manager_filesystem.h @@ -42,52 +42,38 @@ class AssetManagerFileSystem : public FileSystem { AssetManagerFileSystem(AAssetManager* asset_manager, const string& prefix); ~AssetManagerFileSystem() override = default; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr*/) override; - Status NewRandomAccessFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; - Status NewReadOnlyMemoryRegionFromFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; - Status GetFileSize( - const string& f, - uint64* s /*, TransactionToken* token = nullptr*/) override; + Status FileExists(const string& fname, TransactionToken* token) override; + Status NewRandomAccessFile( + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; + Status NewReadOnlyMemoryRegionFromFile( + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; + + Status GetFileSize(const string& f, TransactionToken* token, + uint64* s) override; // Currently just returns size. - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr*/) override; - Status GetChildren( - const string& dir, - std::vector* r /*, TransactionToken* token = nullptr*/) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* r) override; // All these functions return Unimplemented error. Asset storage is // read only. - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr*/) override; - Status DeleteFile( - const string& f /*, TransactionToken* token = nullptr*/) override; - Status CreateDir( - const string& d /*, TransactionToken* token = nullptr*/) override; - Status DeleteDir( - const string& d /*, TransactionToken* token = nullptr*/) override; - Status RenameFile( - const string& s, - const string& t /*, TransactionToken* token = nullptr*/) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status DeleteFile(const string& f, TransactionToken* token) override; + Status CreateDir(const string& d, TransactionToken* token) override; + Status DeleteDir(const string& d, TransactionToken* token) override; + Status RenameFile(const string& s, const string& t, + TransactionToken* token) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr*/) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; private: string RemoveAssetPrefix(const string& name); From f05d6a01eefa65104fac428f6c23898694849dfa Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Tue, 4 Aug 2020 18:04:38 -0700 Subject: [PATCH 0420/1017] Fix passing StringPiece to proto string setter in dispatcher_state_test. PiperOrigin-RevId: 324930108 Change-Id: If7324887c5a8136cef4598150e97a701c83cfd1c --- tensorflow/core/data/service/dispatcher_state_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/data/service/dispatcher_state_test.cc b/tensorflow/core/data/service/dispatcher_state_test.cc index b5529951efb..78f507ec349 100644 --- a/tensorflow/core/data/service/dispatcher_state_test.cc +++ b/tensorflow/core/data/service/dispatcher_state_test.cc @@ -79,7 +79,7 @@ Status CreateNamedJob(int64 job_id, int64 dataset_id, NamedJobKey named_job_key, } Status CreateTask(int64 task_id, int64 job_id, int64 dataset_id, - StringPiece worker_address, DispatcherState* state) { + const std::string& worker_address, DispatcherState* state) { Update update; CreateTaskUpdate* create_task = update.mutable_create_task(); create_task->set_task_id(task_id); From 754dffaf7812d94233b6dbcba3abd2a10cb118e9 Mon Sep 17 00:00:00 2001 From: Rick Chao Date: Tue, 4 Aug 2020 18:17:22 -0700 Subject: [PATCH 0421/1017] PSv2: Attempt of a workaround for client_test's logging in windows/cpu_py38_full/nightly. PiperOrigin-RevId: 324931764 Change-Id: Id37db85a085c59a3c12382dd4eebeb0e5b288ec1 --- tensorflow/python/distribute/client/BUILD | 1 - tensorflow/python/distribute/client/client_test.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/distribute/client/BUILD b/tensorflow/python/distribute/client/BUILD index d37d855a390..35d8de95276 100644 --- a/tensorflow/python/distribute/client/BUILD +++ b/tensorflow/python/distribute/client/BUILD @@ -49,7 +49,6 @@ tf_py_test( srcs = ["client_test.py"], python_version = "PY3", shard_count = 12, - tags = ["no_windows"], # TODO(b/162751266) deps = [ ":client", "//tensorflow/python:client_testlib", diff --git a/tensorflow/python/distribute/client/client_test.py b/tensorflow/python/distribute/client/client_test.py index 459633aca2b..19deab26f63 100644 --- a/tensorflow/python/distribute/client/client_test.py +++ b/tensorflow/python/distribute/client/client_test.py @@ -80,10 +80,10 @@ class CoordinatedClosureQueueTest(test.TestCase): def get_func(label): def func(): - logging.info('Label: %s, before waiting 3 sec', label) + logging.info('Label: ' + label + ', before waiting 3 sec') # pylint: disable=logging-not-lazy time.sleep(3) processed_count[label] += 1 - logging.info('Label: %s, after waiting 3 sec', label) + logging.info('Label: ' + label + ', after waiting 3 sec') # pylint: disable=logging-not-lazy return func From deefe8cafb146ea7a41e47b85afbdeb886088573 Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Tue, 4 Aug 2020 18:17:40 -0700 Subject: [PATCH 0422/1017] [tf.data service] Only create journal writer when fault_tolerant_mode is enabled. PiperOrigin-RevId: 324931812 Change-Id: Ie0acbf5359d2db642118af886a3c754b3b07a6cd --- tensorflow/core/data/service/dispatcher_impl.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/data/service/dispatcher_impl.cc b/tensorflow/core/data/service/dispatcher_impl.cc index ffeae96c117..9e705d51ea8 100644 --- a/tensorflow/core/data/service/dispatcher_impl.cc +++ b/tensorflow/core/data/service/dispatcher_impl.cc @@ -71,23 +71,21 @@ Status CreateWorkerStub(const std::string& address, const std::string& protocol, DataServiceDispatcherImpl::DataServiceDispatcherImpl( const experimental::DispatcherConfig& config) : config_(config) { - if (!config_.work_dir().empty()) { - journal_writer_ = absl::make_unique( - Env::Default(), JournalDir(config_.work_dir())); - } } Status DataServiceDispatcherImpl::Start() { + mutex_lock l(mu_); if (!config_.fault_tolerant_mode()) { LOG(INFO) << "Running with fault_tolerant_mode=False. The dispatcher will " "not be able to recover its state on restart."; return Status::OK(); } - mutex_lock l(mu_); if (config_.work_dir().empty()) { return errors::InvalidArgument( "fault_tolerant_mode is True, but no work_dir is configured."); } + journal_writer_ = absl::make_unique( + Env::Default(), JournalDir(config_.work_dir())); Update update; bool end_of_journal = false; FileJournalReader reader(Env::Default(), JournalDir(config_.work_dir())); From e95a955af8045240333ea4599de7bd11deae18ae Mon Sep 17 00:00:00 2001 From: Tomer Kaftan Date: Tue, 4 Aug 2020 18:35:44 -0700 Subject: [PATCH 0423/1017] Fixed a bug with Functional model serialization when a layer that produces kwarg args of another layer had already been used to define a different functional model. PiperOrigin-RevId: 324934074 Change-Id: Ibcee3958120e50bc72eb3a3c95410f8e5e1135ea --- tensorflow/python/keras/engine/functional.py | 13 +++- .../python/keras/engine/functional_test.py | 67 ++++++++++++++++--- tensorflow/python/keras/engine/node.py | 32 ++++----- tensorflow/python/keras/models.py | 2 + 4 files changed, 88 insertions(+), 26 deletions(-) diff --git a/tensorflow/python/keras/engine/functional.py b/tensorflow/python/keras/engine/functional.py index 7c1fd4d1c72..42c706a923d 100644 --- a/tensorflow/python/keras/engine/functional.py +++ b/tensorflow/python/keras/engine/functional.py @@ -1129,7 +1129,18 @@ def reconstruct_from_config(config, custom_objects=None, created_layers=None): tensor_index = t[2] layer = layer_map[layer_name] - node = layer._inbound_nodes[get_node_index(layer, node_index)] + new_node_index = get_node_index(layer, node_index) + if new_node_index is None: + # The inbound node may not have been processed yet, + # (This can happen e.g. if it depends on a different set + # of inputs than those that have been processed already). + # raise an IndexError so that the current node puts itself + # back on the unprocessed queue. + # Caution: This may lead to infinite loops for malformed + # network configurations! (or when there is a bug in + # the network config loading code). + raise IndexError + node = layer._inbound_nodes[new_node_index] return nest.flatten(node.outputs)[tensor_index] return t diff --git a/tensorflow/python/keras/engine/functional_test.py b/tensorflow/python/keras/engine/functional_test.py index 1b6d15863e6..dc87098d71f 100644 --- a/tensorflow/python/keras/engine/functional_test.py +++ b/tensorflow/python/keras/engine/functional_test.py @@ -998,8 +998,11 @@ class NetworkConstructionTest(keras_parameterized.TestCase): # Check that second input was correctly added to first. self.assertEqual(history.history['loss'][0], 0.0) - @combinations.generate(combinations.keras_mode_combinations()) - def test_call_kwarg_derived_from_keras_layer(self): + @combinations.generate( + combinations.times( + combinations.keras_mode_combinations(), + combinations.combine(share_already_used_layer=[True, False]))) + def test_call_kwarg_derived_from_keras_layer(self, share_already_used_layer): class MaybeAdd(layers.Layer): @@ -1008,9 +1011,26 @@ class NetworkConstructionTest(keras_parameterized.TestCase): return x1 + x2 return x1 + class IdentityLayer(layers.Layer): + + def call(self, x): + return x + input1 = input_layer_lib.Input(10) input2 = input_layer_lib.Input(10) - outputs = MaybeAdd()(input1, x2=input2) + identity_layer = IdentityLayer() + + if share_already_used_layer: + # We have had model serialization/deserialization break in the past: + # when a layer was previously used to construct other functional models + # and had a non-empty list of inbound nodes before being used to define + # the model being serialized/deserialized. + # (The serialization/deserialization was not correctly adjusting + # the node_index serialization/deserialization). + # So, we explicitly test this case. + training_lib.Model([input1], identity_layer(input1)) + + outputs = MaybeAdd()(input1, x2=identity_layer(input2)) model = training_lib.Model([input1, input2], outputs) model.compile( 'sgd', @@ -1024,7 +1044,11 @@ class NetworkConstructionTest(keras_parameterized.TestCase): self.assertEqual(history.history['loss'][0], 0.0) model = training_lib.Model.from_config( - model.get_config(), custom_objects={'MaybeAdd': MaybeAdd}) + model.get_config(), + custom_objects={ + 'MaybeAdd': MaybeAdd, + 'IdentityLayer': IdentityLayer + }) model.compile( 'sgd', 'mse', @@ -1107,10 +1131,18 @@ class NetworkConstructionTest(keras_parameterized.TestCase): TypeError, 'Layer double was passed non-JSON-serializable arguments.'): model.get_config() - @combinations.generate(combinations.times( - combinations.keras_mode_combinations(), - combinations.keras_tensor_combinations())) - def test_call_kwarg_derived_from_keras_layer_and_first_arg_is_constant(self): + @combinations.generate( + combinations.times( + combinations.keras_mode_combinations(), + combinations.keras_tensor_combinations(), + combinations.combine(share_already_used_layer=[True, False]))) + def test_call_kwarg_derived_from_keras_layer_and_first_arg_is_constant( + self, share_already_used_layer): + + class IdentityLayer(layers.Layer): + + def call(self, x): + return x class MaybeAdd(layers.Layer): @@ -1120,7 +1152,18 @@ class NetworkConstructionTest(keras_parameterized.TestCase): return x1 input2 = input_layer_lib.Input(10) - outputs = MaybeAdd()(3., x2=input2) + identity_layer = IdentityLayer() + if share_already_used_layer: + # We have had model serialization/deserialization break in the past: + # when a layer was previously used to construct other functional models + # and had a non-empty list of inbound nodes before being used to define + # the model being serialized/deserialized. + # (The serialization/deserialization was not correctly adjusting + # the node_index serialization/deserialization). + # So, we explicitly test this case. + training_lib.Model([input2], identity_layer(input2)) + + outputs = MaybeAdd()(3., x2=identity_layer(input2)) model = training_lib.Model([input2], outputs) model.compile( 'sgd', @@ -1134,7 +1177,11 @@ class NetworkConstructionTest(keras_parameterized.TestCase): self.assertEqual(history.history['loss'][0], 0.0) model = training_lib.Model.from_config( - model.get_config(), custom_objects={'MaybeAdd': MaybeAdd}) + model.get_config(), + custom_objects={ + 'MaybeAdd': MaybeAdd, + 'IdentityLayer': IdentityLayer + }) model.compile( 'sgd', 'mse', diff --git a/tensorflow/python/keras/engine/node.py b/tensorflow/python/keras/engine/node.py index eb85bce7e75..2a35477eea2 100644 --- a/tensorflow/python/keras/engine/node.py +++ b/tensorflow/python/keras/engine/node.py @@ -169,6 +169,23 @@ class Node(object): arguments.update(kwargs) kwargs = arguments + def _serialize_keras_tensor(t): + """Serializes a single Tensor passed to `call`.""" + if hasattr(t, '_keras_history'): + kh = t._keras_history + node_index = kh.node_index + node_key = make_node_key(kh.layer.name, node_index) + new_node_index = node_conversion_map.get(node_key, 0) + return [kh.layer.name, new_node_index, kh.tensor_index] + + if isinstance(t, np.ndarray): + return t.tolist() + + if isinstance(t, ops.Tensor): + return backend.get_value(t).tolist() + + return t + kwargs = nest.map_structure(_serialize_keras_tensor, kwargs) try: json.dumps(kwargs, default=json_utils.get_json_type) @@ -273,18 +290,3 @@ class KerasHistory( def is_keras_tensor(obj): return hasattr(obj, '_keras_history') - - -def _serialize_keras_tensor(t): - """Serializes a single Tensor passed to `call`.""" - if hasattr(t, '_keras_history'): - kh = t._keras_history - return [kh.layer.name, kh.node_index, kh.tensor_index] - - if isinstance(t, np.ndarray): - return t.tolist() - - if isinstance(t, ops.Tensor): - return backend.get_value(t).tolist() - - return t diff --git a/tensorflow/python/keras/models.py b/tensorflow/python/keras/models.py index 37a3f01272f..76324621a8b 100644 --- a/tensorflow/python/keras/models.py +++ b/tensorflow/python/keras/models.py @@ -206,6 +206,8 @@ def _clone_functional_model(model, input_tensors=None, layer_fn=_clone_layer): ancillary_layers = [ layer for layer in created_layers.values() if layer not in model.layers ] + # TODO(b/162887610): This may need to adjust the inbound node index if the + # created layers had already been used to define other models. if ancillary_layers: new_nodes = nest.flatten([ layer.inbound_nodes[1:] From 6388aa43d7dc1bc1d87887c680f153caab30268f Mon Sep 17 00:00:00 2001 From: Yujing Zhang Date: Tue, 4 Aug 2020 19:09:39 -0700 Subject: [PATCH 0424/1017] Change the function output type, either a Tensor for a local output or a TensorShape for a remote output, preparing for the support of function outputs placed on remote workers. PiperOrigin-RevId: 324938354 Change-Id: I126822bd75bb284c917af7a72f2868601e798f09 --- .../common_runtime/eager/kernel_and_device.cc | 18 +++++- .../process_function_library_runtime.cc | 61 ++++++++++++++++--- .../process_function_library_runtime.h | 7 ++- .../process_function_library_runtime_test.cc | 25 ++++---- .../cluster_function_library_runtime.cc | 14 ++++- .../cluster_function_library_runtime.h | 2 +- .../eager/cluster_function_library_runtime.cc | 30 ++++++++- .../eager/cluster_function_library_runtime.h | 7 ++- .../eager/eager_service_impl_test.cc | 6 +- tensorflow/core/framework/function.h | 16 +++-- 10 files changed, 144 insertions(+), 42 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc index 1f506c318bc..5b7232f539a 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc +++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc @@ -395,13 +395,25 @@ void KernelAndDeviceFunc::RunAsync( }, profiler::ContextType::kTfExecutor, opts->step_id, profiler::TraceMeLevel::kInfo); - pflr_->Run(*opts, handle_, inputs, outputs, - [opts, rendezvous, local_cm, step_container, this, - done = std::move(done)](const Status& s) { + std::vector* function_rets = new std::vector; + pflr_->Run(*opts, handle_, inputs, function_rets, + [opts, outputs, function_rets, rendezvous, local_cm, + step_container, this, done = std::move(done)](const Status& s) { rendezvous->Unref(); if (step_container == nullptr) { this->step_container_.CleanUp(); } + if (s.ok()) { + // TODO(b/162618595): Change the type of `outputs` to + // support TensorShapes for remote outputs and remove the + // FunctionRet to Tensor conversion here. + for (const auto& ret : *function_rets) { + if (ret.index() == 0) { + outputs->push_back(absl::get(ret)); + } + } + } + delete function_rets; done(s); }); } diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc index aee482d92da..b31b2b78bf0 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc @@ -398,6 +398,21 @@ std::vector GetLocalArgs(gtl::ArraySlice args) { return tensors; } +// Update the done callback to push Tensors in `tensors` into `rets`. +FunctionLibraryRuntime::DoneCallback TensorsToFunctionRetsDoneCallback( + std::vector* rets, std::vector* tensors, + FunctionLibraryRuntime::DoneCallback done) { + return [rets, tensors, done = std::move(done)](const Status& s) { + if (s.ok()) { + for (const auto& t : *tensors) { + rets->push_back(t); + } + } + delete tensors; + done(s); + }; +} + } // anonymous namespace Status ProcessFunctionLibraryRuntime::PinArgsAndRets( @@ -1021,7 +1036,7 @@ Status ProcessFunctionLibraryRuntime::GetOutputDevices( void ProcessFunctionLibraryRuntime::RunMultiDevice( const FunctionLibraryRuntime::Options& opts, - FunctionLibraryRuntime::Handle handle, std::vector* rets, + FunctionLibraryRuntime::Handle handle, std::vector* rets, std::vector>* cleanup_items, FunctionLibraryRuntime::DoneCallback done, std::functionStartCancel(); continue; } - std::vector* comp_rets = new std::vector; + std::vector* comp_rets = new std::vector; rets->resize(data->num_outputs_); auto component_fn_callback = [comp_rets, rets, comp_data, refcounted_done, @@ -1136,8 +1151,11 @@ void ProcessFunctionLibraryRuntime::RunMultiDevice( << " with handle " << handle; VLOG(4) << " with " << opts_copy.DebugString(); - flr->Run(opts_copy, handle, GetLocalArgs(comp_args.args), comp_rets, - std::move(component_fn_callback)); + std::vector* comp_tensor_rets = new std::vector; + flr->Run( + opts_copy, handle, GetLocalArgs(comp_args.args), comp_tensor_rets, + TensorsToFunctionRetsDoneCallback(comp_rets, comp_tensor_rets, + std::move(component_fn_callback))); } else { opts_copy.remote_execution = true; @@ -1362,6 +1380,23 @@ void ProcessFunctionLibraryRuntime::Run( auto* cleanup_items = new std::vector>; done = ApplyCleanUpToDoneCallback(cleanup_items, std::move(done), new_opts.step_id, created_rendezvous); + std::vector* function_rets = new std::vector; + done = [rets, function_rets, done = std::move(done)](const Status& s) { + Status status = s; + if (status.ok()) { + for (const auto& ret : *function_rets) { + if (ret.index() == 0) { + rets->push_back(absl::get(ret)); + } else { + status.Update(errors::Internal( + "Expect a Tensor as a function output but got a TensorShape.")); + break; + } + } + } + delete function_rets; + done(status); + }; bool multi_device; { tf_shared_lock l(mu_); @@ -1392,21 +1427,21 @@ void ProcessFunctionLibraryRuntime::Run( } return Status::OK(); }; - return RunMultiDevice(new_opts, handle, rets, cleanup_items, + return RunMultiDevice(new_opts, handle, function_rets, cleanup_items, std::move(done), std::move(get_component_args)); } std::vector local_args; for (const auto& tensor : args) { local_args.push_back(tensor); } - RunInternal(new_opts, handle, local_args, rets, cleanup_items, + RunInternal(new_opts, handle, local_args, function_rets, cleanup_items, std::move(done)); } void ProcessFunctionLibraryRuntime::RunInternal( const FunctionLibraryRuntime::Options& opts, FunctionLibraryRuntime::Handle handle, gtl::ArraySlice args, - std::vector* rets, + std::vector* rets, std::vector>* cleanup_items, FunctionLibraryRuntime::DoneCallback done) const { FunctionLibraryRuntime* flr = nullptr; @@ -1475,10 +1510,13 @@ void ProcessFunctionLibraryRuntime::RunInternal( int64 num_returns = remote_rets->size(); delete remote_rets; // Now receive the return values from the target. + std::vector* recv_tensors = new std::vector; ReceiveTensorsAsync(target_device, source_device, "ret_", target_incarnation, num_returns, device_context, rets_alloc_attrs, rendezvous, - rets, std::move(done)); + recv_tensors, + TensorsToFunctionRetsDoneCallback( + rets, recv_tensors, std::move(done))); }); return; } @@ -1570,11 +1608,14 @@ Status ProcessFunctionLibraryRuntime::RunSync( void ProcessFunctionLibraryRuntime::Run( const FunctionLibraryRuntime::Options& opts, FunctionLibraryRuntime::Handle handle, const FunctionArgsInterface& args, - std::vector* rets, + std::vector* rets, FunctionLibraryRuntime::DoneCallback done) const { if (!args.HasRemoteOrPackedInputs()) { const std::vector local_inputs = args.GetLocalTensors(); - return Run(opts, handle, local_inputs, rets, std::move(done)); + std::vector* tensor_rets = new std::vector; + return Run( + opts, handle, local_inputs, tensor_rets, + TensorsToFunctionRetsDoneCallback(rets, tensor_rets, std::move(done))); } FunctionLibraryRuntime::Options new_opts = opts; diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.h b/tensorflow/core/common_runtime/process_function_library_runtime.h index 0bd85c62df5..3ba04f17880 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.h +++ b/tensorflow/core/common_runtime/process_function_library_runtime.h @@ -191,7 +191,7 @@ class ProcessFunctionLibraryRuntime { void Run(const FunctionLibraryRuntime::Options& opts, FunctionLibraryRuntime::Handle handle, - const FunctionArgsInterface& args, std::vector* rets, + const FunctionArgsInterface& args, std::vector* rets, FunctionLibraryRuntime::DoneCallback done) const; Status RunSync(const FunctionLibraryRuntime::Options& opts, @@ -304,7 +304,7 @@ class ProcessFunctionLibraryRuntime { void RunMultiDevice( const FunctionLibraryRuntime::Options& opts, - FunctionLibraryRuntime::Handle handle, std::vector* rets, + FunctionLibraryRuntime::Handle handle, std::vector* rets, std::vector>* cleanup_items, FunctionLibraryRuntime::DoneCallback done, std::function args, std::vector* rets, + gtl::ArraySlice args, + std::vector* rets, std::vector>* cleanup_items, FunctionLibraryRuntime::DoneCallback done) const; diff --git a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc index be279c84d1a..54c821d282a 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc @@ -72,7 +72,7 @@ class TestClusterFLR : public DistributedFunctionLibraryRuntime { void Run(const FunctionLibraryRuntime::Options& opts, FunctionLibraryRuntime::LocalHandle handle, - gtl::ArraySlice args, std::vector* rets, + gtl::ArraySlice args, std::vector* rets, FunctionLibraryRuntime::DoneCallback done) override {} void CleanUp(uint64 step_id, FunctionLibraryRuntime::LocalHandle handle, @@ -209,12 +209,12 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test { #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM } - template + template Status RunWithRuntime( const string& name, FunctionLibraryRuntime::Options opts, test::function::Attrs attrs, const FunctionLibraryRuntime::InstantiateOptions& instantiate_opts, - const T& args, std::vector rets, + const T& args, std::vector rets, ProcessFunctionLibraryRuntime* pflr) { FunctionLibraryRuntime::Handle handle; Status status = pflr->Instantiate(name, attrs, instantiate_opts, &handle); @@ -234,7 +234,7 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test { Notification done; opts.runner = &runner; - std::vector out; + std::vector out; pflr->Run(opts, handle, args, &out, [&status, &done](const Status& s) { status = s; done.Notify(); @@ -273,7 +273,7 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test { const FunctionLibraryRuntime::InstantiateOptions& instantiate_opts, const std::vector& args, std::vector rets, ProcessFunctionLibraryRuntime* pflr = nullptr) { - return RunWithRuntime>( + return RunWithRuntime, Tensor>( name, opts, attrs, instantiate_opts, args, rets, proc_flr_.get()); } @@ -281,9 +281,9 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test { const string& name, FunctionLibraryRuntime::Options opts, test::function::Attrs attrs, const FunctionLibraryRuntime::InstantiateOptions& instantiate_opts, - const FunctionArgsInterface& args, std::vector rets, + const FunctionArgsInterface& args, std::vector rets, ProcessFunctionLibraryRuntime* pflr = nullptr) { - return RunWithRuntime( + return RunWithRuntime( name, opts, attrs, instantiate_opts, args, rets, proc_flr_.get()); } @@ -879,10 +879,12 @@ TEST_F(ProcessFunctionLibraryRuntimeTest, MultiDevice_CompositeDevice) { handles.push_back(TensorValue(&resource_handle0)); handles.push_back(TensorValue(&resource_handle1)); TestFunctionPackedArgs args(0, std::move(handles)); - Tensor ret; + FunctionRet ret; TF_CHECK_OK(RunWithPackedArgs("AddVarAcrossDevices", opts, {{"T", DT_FLOAT}}, inst_opts, args, {&ret})); - test::ExpectTensorEqual(ret, test::AsTensor({40, 60})); + EXPECT_EQ(ret.index(), 0); + test::ExpectTensorEqual(absl::get(ret), + test::AsTensor({40, 60})); } // Packed Tensor @@ -1226,9 +1228,10 @@ TEST_F(ProcessFunctionLibraryRuntimeTest, SessionMetadataPresentAfterCloning) { instantiate_opts.target = "/job:a/replica:0/task:0/cpu:0"; const auto x = test::AsTensor({17}); Tensor y; - TF_CHECK_OK(RunWithRuntime>( + Status s = RunWithRuntime, Tensor>( "SessionMetadataReaderFn", opts, {}, instantiate_opts, {x}, {&y}, - cloned_proc_flr.get())); + cloned_proc_flr.get()); + TF_CHECK_OK(s); SessionMetadata read_metadata; ASSERT_TRUE(protobuf::TextFormat::ParseFromString(y.scalar()(), &read_metadata)); diff --git a/tensorflow/core/distributed_runtime/cluster_function_library_runtime.cc b/tensorflow/core/distributed_runtime/cluster_function_library_runtime.cc index 7ddba8811b4..3f7867200f8 100644 --- a/tensorflow/core/distributed_runtime/cluster_function_library_runtime.cc +++ b/tensorflow/core/distributed_runtime/cluster_function_library_runtime.cc @@ -333,7 +333,7 @@ void ClusterFunctionLibraryRuntime::Run( void ClusterFunctionLibraryRuntime::Run( const FunctionLibraryRuntime::Options& opts, FunctionLibraryRuntime::LocalHandle handle, - gtl::ArraySlice args, std::vector* rets, + gtl::ArraySlice args, std::vector* rets, FunctionLibraryRuntime::DoneCallback done) { std::vector tensors; for (const auto& arg : args) { @@ -346,7 +346,17 @@ void ClusterFunctionLibraryRuntime::Run( return; } } - return Run(opts, handle, tensors, rets, std::move(done)); + std::vector* ret_tensors = new std::vector; + return Run(opts, handle, tensors, ret_tensors, + [rets, ret_tensors, done = std::move(done)](const Status& s) { + if (s.ok()) { + for (const auto& t : *ret_tensors) { + rets->push_back(t); + } + } + delete ret_tensors; + done(s); + }); } void ClusterFunctionLibraryRuntime::CleanUp( diff --git a/tensorflow/core/distributed_runtime/cluster_function_library_runtime.h b/tensorflow/core/distributed_runtime/cluster_function_library_runtime.h index b720fe7ad6d..eb9ce64bcdb 100644 --- a/tensorflow/core/distributed_runtime/cluster_function_library_runtime.h +++ b/tensorflow/core/distributed_runtime/cluster_function_library_runtime.h @@ -49,7 +49,7 @@ class ClusterFunctionLibraryRuntime : public DistributedFunctionLibraryRuntime { void Run(const FunctionLibraryRuntime::Options& opts, FunctionLibraryRuntime::LocalHandle handle, - gtl::ArraySlice args, std::vector* rets, + gtl::ArraySlice args, std::vector* rets, FunctionLibraryRuntime::DoneCallback done) override; void CleanUp(uint64 step_id, FunctionLibraryRuntime::LocalHandle handle, diff --git a/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.cc b/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.cc index d8613e5f9b9..03944e12590 100644 --- a/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.cc +++ b/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.cc @@ -118,13 +118,31 @@ void EagerClusterFunctionLibraryRuntime::Run( for (const auto& tensor : args) { function_args.push_back(tensor); } - Run(opts, handle, function_args, rets, std::move(done)); + std::vector* function_rets = new std::vector; + Run(opts, handle, function_args, function_rets, + [rets, function_rets, done = std::move(done)](const Status& s) { + Status status = s; + if (status.ok()) { + for (const auto& t : *function_rets) { + if (t.index() == 0) { + rets->push_back(absl::get(t)); + } else { + status.Update( + errors::Internal("Expect a Tensor as a remote function " + "output but got a TensorShape.")); + break; + } + } + } + delete function_rets; + done(status); + }); } void EagerClusterFunctionLibraryRuntime::Run( const FunctionLibraryRuntime::Options& opts, FunctionLibraryRuntime::LocalHandle handle, - gtl::ArraySlice args, std::vector* rets, + gtl::ArraySlice args, std::vector* rets, FunctionLibraryRuntime::DoneCallback done) { FunctionData* function_data = nullptr; { @@ -204,6 +222,14 @@ void EagerClusterFunctionLibraryRuntime::Run( done(s); return; } + if (!response->shape().empty() && !response->tensor().empty()) { + done(errors::Internal( + "Both shape and tensor are specified in the same response")); + return; + } + for (const auto& shape : response->shape()) { + rets->push_back(shape); + } for (const auto& tensor_proto : response->tensor()) { Tensor t; if (t.FromProto(tensor_proto)) { diff --git a/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.h b/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.h index 9df9d1aecc1..6e60ee0b13d 100644 --- a/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.h +++ b/tensorflow/core/distributed_runtime/eager/cluster_function_library_runtime.h @@ -64,11 +64,12 @@ class EagerClusterFunctionLibraryRuntime gtl::ArraySlice args, std::vector* rets, FunctionLibraryRuntime::DoneCallback done) override; - // The component function inputs `args` can be RemoteTensorHandles, which will - // be lazily resolved remotely where the inputs are actually consumed. + // The component function inputs `args` and outputs `rets` may refer to remote + // tensors on a remote device, which will be lazily resolved remotely where + // the inputs/outputs are actually consumed. void Run(const FunctionLibraryRuntime::Options& opts, FunctionLibraryRuntime::LocalHandle handle, - gtl::ArraySlice args, std::vector* rets, + gtl::ArraySlice args, std::vector* rets, FunctionLibraryRuntime::DoneCallback done) override; void CleanUp(uint64 step_id, FunctionLibraryRuntime::LocalHandle handle, diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc index a2412eb9625..be81355cbc8 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc @@ -830,7 +830,7 @@ TEST_F(FunctionWithRemoteInputsTest, EagerPFLRTest) { input.set_op_device(local_device_); input.set_device(local_device_); std::vector inputs = {input}; - std::vector outputs; + std::vector outputs; gtl::InlinedVector tensor_args = {TensorValue()}; TestExecuteNodeArgs args( std::move(tensor_args), @@ -845,6 +845,10 @@ TEST_F(FunctionWithRemoteInputsTest, EagerPFLRTest) { }); done.WaitForNotification(); TF_ASSERT_OK(status); + EXPECT_EQ(outputs.size(), 1); + EXPECT_EQ(outputs.at(0).index(), 1); + const TensorShape& shape = absl::get(outputs.at(0)); + EXPECT_EQ(shape, TensorShape({2, 2})); CheckOutputsAndClose(op_id); } diff --git a/tensorflow/core/framework/function.h b/tensorflow/core/framework/function.h index 95f733d23a6..c7e6e2d158c 100644 --- a/tensorflow/core/framework/function.h +++ b/tensorflow/core/framework/function.h @@ -901,6 +901,9 @@ typedef FunctionArg; #endif +// Either a local tensor or the shape of a remote tensor. +typedef absl::variant FunctionRet; + // Used to instantiate and run functions in a distributed system. class DistributedFunctionLibraryRuntime { public: @@ -929,14 +932,15 @@ class DistributedFunctionLibraryRuntime { // Run an instantiated remote function (specified by `handle`) with a list of // input Tensors or RemoteTensorHandles as `args` and get its output Tensors - // in `rets`. When using RemoteTensorHandles as function inputs, the - // corresponding tensor data will be resolved on the remote worker, so it is - // not required to be locally available on the caller side. Using - // RemoteTensorHandle inputs is not supported in TensorFlow v1 runtime. - // TODO(yujingzhang): Support outputting tensors on remote devices. + // or TensorShapes in `rets`. When using RemoteTensorHandles as function + // inputs or TensorShapes as outputs, the corresponding tensor data will be + // resolved on the remote worker, so it is not required to be locally + // available on the caller side. Using RemoteTensorHandle inputs is not + // supported in TensorFlow v1 runtime. virtual void Run(const FunctionLibraryRuntime::Options& opts, FunctionLibraryRuntime::LocalHandle handle, - gtl::ArraySlice args, std::vector* rets, + gtl::ArraySlice args, + std::vector* rets, FunctionLibraryRuntime::DoneCallback done) = 0; // Clean up a previously instantiated function on remote worker. From eaa5235e003799784dcea14c528b570ee2634a55 Mon Sep 17 00:00:00 2001 From: Yuefeng Zhou Date: Tue, 4 Aug 2020 19:12:35 -0700 Subject: [PATCH 0425/1017] Improve docstring of strategy.run. PiperOrigin-RevId: 324938654 Change-Id: I3e13d90c026fad42657bb8094ccb32dc86e36b4b --- .../python/distribute/distribute_lib.py | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/tensorflow/python/distribute/distribute_lib.py b/tensorflow/python/distribute/distribute_lib.py index 43af23ab096..522849ac951 100644 --- a/tensorflow/python/distribute/distribute_lib.py +++ b/tensorflow/python/distribute/distribute_lib.py @@ -1146,10 +1146,11 @@ class StrategyBase(object): dataset_fn, options) def run(self, fn, args=(), kwargs=None, options=None): - """Run `fn` on each replica, with the given arguments. + """Invokes `fn` on each replica, with the given arguments. - Executes ops specified by `fn` on each replica. If `args` or `kwargs` have - `tf.distribute.DistributedValues`, such as those produced by a + This method is the primary way to distribute your computation with a + tf.distribute object. It invokes `fn` on each replica. If `args` or `kwargs` + have `tf.distribute.DistributedValues`, such as those produced by a `tf.distribute.DistributedDataset` from `tf.distribute.Strategy.experimental_distribute_dataset` or `tf.distribute.Strategy.experimental_distribute_datasets_from_function`, @@ -1157,20 +1158,27 @@ class StrategyBase(object): component of `tf.distribute.DistributedValues` that correspond to that replica. - `fn` may call `tf.distribute.get_replica_context()` to access members such - as `all_reduce`. + `fn` is invoked under a replica context. `fn` may call + `tf.distribute.get_replica_context()` to access members such as + `all_reduce`. Please see the module-level docstring of tf.distribute for the + concept of replica context. - All arguments in `args` or `kwargs` should either be nest of tensors or - `tf.distribute.DistributedValues` containing tensors or composite tensors. + All arguments in `args` or `kwargs` should either be Python values of a + nested structure of tensors, e.g. a list of tensors, in which case `args` + and `kwargs` will be passed to the `fn` invoked on each replica. Or `args` + or `kwargs` can be `tf.distribute.DistributedValues` containing tensors or + composite tensors, i.e. `tf.compat.v1.TensorInfo.CompositeTensor`, in which + case each `fn` call will get the component of a + `tf.distribute.DistributedValues` corresponding to its replica. IMPORTANT: Depending on the implementation of `tf.distribute.Strategy` and whether eager execution is enabled, `fn` may be called one or more times. If `fn` is annotated with `tf.function` or `tf.distribute.Strategy.run` is - called inside a `tf.function`, eager execution is disabled and `fn` is - called once (or once per replica, if you are using MirroredStrategy) to - generate a Tensorflow graph, which will then be reused for execution with - new inputs. Otherwise, if eager execution is enabled, `fn` will be called - every step just like regular python code. + called inside a `tf.function` (eager execution is disabled inside a + `tf.function` by default), `fn` is called once per replica to generate a + Tensorflow graph, which will then be reused for execution with new inputs. + Otherwise, if eager execution is enabled, `fn` will be called once per + replica every step just like regular python code. Example usage: @@ -1205,11 +1213,33 @@ class StrategyBase(object): >>> result + 3. Use `tf.distribute.ReplicaContext` to allreduce values. + + >>> strategy = tf.distribute.MirroredStrategy(["gpu:0", "gpu:1"]) + >>> @tf.function + ... def run(): + ... def value_fn(value_context): + ... return tf.constant(value_context.replica_id_in_sync_group) + ... distributed_values = ( + ... strategy.experimental_distribute_values_from_function( + ... value_fn)) + ... def replica_fn(input): + ... return tf.distribute.get_replica_context().all_reduce("sum", input) + ... return strategy.run(replica_fn, args=(distributed_values,)) + >>> result = run() + >>> result + PerReplica:{ + 0: , + 1: + } + Args: - fn: The function to run. The output must be a `tf.nest` of `Tensor`s. - args: (Optional) Positional arguments to `fn`. - kwargs: (Optional) Keyword arguments to `fn`. - options: (Optional) An instance of `tf.distribute.RunOptions` specifying + fn: The function to run on each replica. + args: Optional positional arguments to `fn`. Its element can be a Python + value, a tensor or a `tf.distribute.DistributedValues`. + kwargs: Optional keyword arguments to `fn`. Its element can be a Python + value, a tensor or a `tf.distribute.DistributedValues`. + options: An optional instance of `tf.distribute.RunOptions` specifying the options to run `fn`. Returns: From 77ee5e02721ba797fe01d47019e6017d2bb09ab7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 19:20:06 -0700 Subject: [PATCH 0426/1017] Rename LoadLibrary to avoid conflict with Windows macros PiperOrigin-RevId: 324939413 Change-Id: I2ad9f90c302f56ba4dfe847da44f9cd104457fbd --- tensorflow/c/c_api.cc | 7 +++---- tensorflow/c/env.cc | 4 ++-- .../c/experimental/filesystem/modular_filesystem.cc | 2 +- .../c/experimental/filesystem/modular_filesystem_test.cc | 1 - tensorflow/core/framework/load_library.cc | 6 +++--- tensorflow/core/framework/op_kernel.cc | 3 ++- tensorflow/core/platform/default/env.cc | 5 +++-- tensorflow/core/platform/default/load_library.cc | 2 +- tensorflow/core/platform/env.h | 8 +++++--- tensorflow/core/platform/hadoop/hadoop_file_system.cc | 2 +- tensorflow/core/platform/load_library.h | 2 +- tensorflow/core/platform/windows/env.cc | 6 +++--- tensorflow/core/platform/windows/load_library.cc | 3 +-- tensorflow/stream_executor/platform/default/dso_loader.cc | 2 +- third_party/eigen3/unsupported/Eigen/CXX11/Tensor | 1 - 15 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc index 36a08c8cfc9..2e1759ecea0 100644 --- a/tensorflow/c/c_api.cc +++ b/tensorflow/c/c_api.cc @@ -213,7 +213,6 @@ void TF_Reset(const TF_SessionOptions* opt, const char** containers, namespace tensorflow { - Status MessageToBuffer(const tensorflow::protobuf::MessageLite& in, TF_Buffer* out) { if (out->data != nullptr) { @@ -306,8 +305,8 @@ void TF_GraphSetOutputHandleShapesAndTypes(TF_Graph* graph, TF_Output output, } // Helpers for loading a TensorFlow plugin (a .so file). -Status LoadLibrary(const char* library_filename, void** result, - const void** buf, size_t* len); +Status LoadDynamicLibrary(const char* library_filename, void** result, + const void** buf, size_t* len); // TODO(josh11b,mrry): Change Session to be able to use a Graph* // directly, instead of requiring us to serialize to a GraphDef and @@ -552,7 +551,7 @@ void TF_PRun(TF_DeprecatedSession* s, const char* handle, TF_Library* TF_LoadLibrary(const char* library_filename, TF_Status* status) { TF_Library* lib_handle = new TF_Library; - status->status = tensorflow::LoadLibrary( + status->status = tensorflow::LoadDynamicLibrary( library_filename, &lib_handle->lib_handle, &lib_handle->op_list.data, &lib_handle->op_list.length); if (!status->status.ok()) { diff --git a/tensorflow/c/env.cc b/tensorflow/c/env.cc index e731c0659a7..fbde13dea5a 100644 --- a/tensorflow/c/env.cc +++ b/tensorflow/c/env.cc @@ -191,8 +191,8 @@ void* TF_LoadSharedLibrary(const char* library_filename, TF_Status* status) { void* handle = nullptr; TF_SetStatus(status, TF_OK, ""); ::tensorflow::Set_TF_Status_from_Status( - status, - ::tensorflow::Env::Default()->LoadLibrary(library_filename, &handle)); + status, ::tensorflow::Env::Default()->LoadDynamicLibrary(library_filename, + &handle)); return handle; } diff --git a/tensorflow/c/experimental/filesystem/modular_filesystem.cc b/tensorflow/c/experimental/filesystem/modular_filesystem.cc index 40258e43801..00a587521fd 100644 --- a/tensorflow/c/experimental/filesystem/modular_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/modular_filesystem.cc @@ -462,7 +462,7 @@ Status RegisterFilesystemPlugin(const std::string& dso_path) { // Step 1: Load plugin Env* env = Env::Default(); void* dso_handle; - TF_RETURN_IF_ERROR(env->LoadLibrary(dso_path.c_str(), &dso_handle)); + TF_RETURN_IF_ERROR(env->LoadDynamicLibrary(dso_path.c_str(), &dso_handle)); // Step 2: Load symbol for `TF_InitPlugin` void* dso_symbol; diff --git a/tensorflow/c/experimental/filesystem/modular_filesystem_test.cc b/tensorflow/c/experimental/filesystem/modular_filesystem_test.cc index 8ee47da01dd..7e0a95cc915 100644 --- a/tensorflow/c/experimental/filesystem/modular_filesystem_test.cc +++ b/tensorflow/c/experimental/filesystem/modular_filesystem_test.cc @@ -33,7 +33,6 @@ limitations under the License. // Windows defines the following macros to convert foo to fooA or fooW, // depending on the type of the string argument. We don't use these macros, so // undefine them here. -#undef LoadLibrary #undef CopyFile #undef DeleteFile #undef TranslateName diff --git a/tensorflow/core/framework/load_library.cc b/tensorflow/core/framework/load_library.cc index b9e33b148f7..34cd4b3386b 100644 --- a/tensorflow/core/framework/load_library.cc +++ b/tensorflow/core/framework/load_library.cc @@ -43,8 +43,8 @@ struct Library { // and OpList. Ops and kernels are registered as globals when a library is // loaded for the first time. Without caching, every subsequent load would not // perform initialization again, so the OpList would be empty. -Status LoadLibrary(const char* library_filename, void** result, - const void** buf, size_t* len) { +Status LoadDynamicLibrary(const char* library_filename, void** result, + const void** buf, size_t* len) { static mutex mu(LINKER_INITIALIZED); static std::unordered_map loaded_libs; Env* env = Env::Default(); @@ -76,7 +76,7 @@ Status LoadLibrary(const char* library_filename, void** result, return s; })); OpRegistry::Global()->DeferRegistrations(); - s = env->LoadLibrary(library_filename, &library.handle); + s = env->LoadDynamicLibrary(library_filename, &library.handle); if (s.ok()) { s = OpRegistry::Global()->ProcessRegistrations(); } diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc index 1930cc98da1..d9b679534ee 100644 --- a/tensorflow/core/framework/op_kernel.cc +++ b/tensorflow/core/framework/op_kernel.cc @@ -1211,7 +1211,8 @@ void LoadDynamicKernelsInternal() { if (s.ok() || override_abi_check) { // TODO(gunan): Store the handles to the opened files. void* unused_filehandle; - TF_CHECK_OK(env->LoadLibrary(fullpath.c_str(), &unused_filehandle)); + TF_CHECK_OK( + env->LoadDynamicLibrary(fullpath.c_str(), &unused_filehandle)); } else { LOG(WARNING) << "Not loading plugin library " << fullpath << ": " << s.error_message(); diff --git a/tensorflow/core/platform/default/env.cc b/tensorflow/core/platform/default/env.cc index 90e0ee97355..b933fa005a7 100644 --- a/tensorflow/core/platform/default/env.cc +++ b/tensorflow/core/platform/default/env.cc @@ -185,8 +185,9 @@ class PosixEnv : public Env { }); } - Status LoadLibrary(const char* library_filename, void** handle) override { - return tensorflow::internal::LoadLibrary(library_filename, handle); + Status LoadDynamicLibrary(const char* library_filename, + void** handle) override { + return tensorflow::internal::LoadDynamicLibrary(library_filename, handle); } Status GetSymbolFromLibrary(void* handle, const char* symbol_name, diff --git a/tensorflow/core/platform/default/load_library.cc b/tensorflow/core/platform/default/load_library.cc index ef9edcc4501..bbe5824acfa 100644 --- a/tensorflow/core/platform/default/load_library.cc +++ b/tensorflow/core/platform/default/load_library.cc @@ -23,7 +23,7 @@ namespace tensorflow { namespace internal { -Status LoadLibrary(const char* library_filename, void** handle) { +Status LoadDynamicLibrary(const char* library_filename, void** handle) { *handle = dlopen(library_filename, RTLD_NOW | RTLD_LOCAL); if (!*handle) { return errors::NotFound(dlerror()); diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h index 25544e87702..7b716798c28 100644 --- a/tensorflow/core/platform/env.h +++ b/tensorflow/core/platform/env.h @@ -334,7 +334,8 @@ class Env { // OK from the function. // Otherwise returns nullptr in "*handle" and an error status from the // function. - virtual Status LoadLibrary(const char* library_filename, void** handle) = 0; + virtual Status LoadDynamicLibrary(const char* library_filename, + void** handle) = 0; // \brief Get a pointer to a symbol from a dynamic library. // @@ -411,8 +412,9 @@ class EnvWrapper : public Env { void SchedClosureAfter(int64 micros, std::function closure) override { target_->SchedClosureAfter(micros, closure); } - Status LoadLibrary(const char* library_filename, void** handle) override { - return target_->LoadLibrary(library_filename, handle); + Status LoadDynamicLibrary(const char* library_filename, + void** handle) override { + return target_->LoadDynamicLibrary(library_filename, handle); } Status GetSymbolFromLibrary(void* handle, const char* symbol_name, void** symbol) override { diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc index 5b2c5a76aae..327f506665f 100644 --- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc +++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc @@ -70,7 +70,7 @@ class LibHDFS { private: void LoadAndBind() { auto TryLoadAndBind = [this](const char* name, void** handle) -> Status { - TF_RETURN_IF_ERROR(Env::Default()->LoadLibrary(name, handle)); + TF_RETURN_IF_ERROR(Env::Default()->LoadDynamicLibrary(name, handle)); #define BIND_HDFS_FUNC(function) \ TF_RETURN_IF_ERROR(BindFunc(*handle, #function, &function)); diff --git a/tensorflow/core/platform/load_library.h b/tensorflow/core/platform/load_library.h index 01efd4c1d01..60e84238487 100644 --- a/tensorflow/core/platform/load_library.h +++ b/tensorflow/core/platform/load_library.h @@ -22,7 +22,7 @@ namespace tensorflow { namespace internal { -Status LoadLibrary(const char* library_filename, void** handle); +Status LoadDynamicLibrary(const char* library_filename, void** handle); Status GetSymbolFromLibrary(void* handle, const char* symbol_name, void** symbol); string FormatLibraryFileName(const string& name, const string& version); diff --git a/tensorflow/core/platform/windows/env.cc b/tensorflow/core/platform/windows/env.cc index d75d2d5773d..ea6d1424529 100644 --- a/tensorflow/core/platform/windows/env.cc +++ b/tensorflow/core/platform/windows/env.cc @@ -22,7 +22,6 @@ limitations under the License. #include #include #include -#undef LoadLibrary #undef ERROR #include @@ -156,8 +155,9 @@ class WindowsEnv : public Env { SetThreadpoolTimer(timer, &FileDueTime, 0, 0); } - Status LoadLibrary(const char* library_filename, void** handle) override { - return tensorflow::internal::LoadLibrary(library_filename, handle); + Status LoadDynamicLibrary(const char* library_filename, + void** handle) override { + return tensorflow::internal::LoadDynamicLibrary(library_filename, handle); } Status GetSymbolFromLibrary(void* handle, const char* symbol_name, diff --git a/tensorflow/core/platform/windows/load_library.cc b/tensorflow/core/platform/windows/load_library.cc index f95e770cc6b..67fdffeca15 100644 --- a/tensorflow/core/platform/windows/load_library.cc +++ b/tensorflow/core/platform/windows/load_library.cc @@ -22,7 +22,6 @@ limitations under the License. #include #include #include -#undef LoadLibrary #undef ERROR #include "tensorflow/core/platform/errors.h" @@ -34,7 +33,7 @@ namespace tensorflow { namespace internal { -Status LoadLibrary(const char* library_filename, void** handle) { +Status LoadDynamicLibrary(const char* library_filename, void** handle) { string file_name = library_filename; std::replace(file_name.begin(), file_name.end(), '/', '\\'); diff --git a/tensorflow/stream_executor/platform/default/dso_loader.cc b/tensorflow/stream_executor/platform/default/dso_loader.cc index 01af4114536..6e0113ab05a 100644 --- a/tensorflow/stream_executor/platform/default/dso_loader.cc +++ b/tensorflow/stream_executor/platform/default/dso_loader.cc @@ -43,7 +43,7 @@ port::StatusOr GetDsoHandle(const string& name, const string& version) { auto filename = port::Env::Default()->FormatLibraryFileName(name, version); void* dso_handle; port::Status status = - port::Env::Default()->LoadLibrary(filename.c_str(), &dso_handle); + port::Env::Default()->LoadDynamicLibrary(filename.c_str(), &dso_handle); if (status.ok()) { LOG(INFO) << "Successfully opened dynamic library " << filename; return dso_handle; diff --git a/third_party/eigen3/unsupported/Eigen/CXX11/Tensor b/third_party/eigen3/unsupported/Eigen/CXX11/Tensor index 861a87b68bf..5bb7ca95db5 100644 --- a/third_party/eigen3/unsupported/Eigen/CXX11/Tensor +++ b/third_party/eigen3/unsupported/Eigen/CXX11/Tensor @@ -11,5 +11,4 @@ inline void sleep(unsigned int seconds) { Sleep(1000*seconds); } // prevent clashes. #undef DeleteFile #undef ERROR -#undef LoadLibrary #endif // _WIN32 From a778b8f95e5915573e26f8e3d62a4cedaed10a8c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 19:35:56 -0700 Subject: [PATCH 0427/1017] If an input-output pair is configured to be must-alias(off by default), they must be aliased at runtime. PiperOrigin-RevId: 324941010 Change-Id: I80995e25ce367ca17e5f884da55874252265f487 --- .../utils/compile_mlir_util_test.cc | 2 +- tensorflow/compiler/xla/client/xla_builder.cc | 2 +- tensorflow/compiler/xla/client/xla_builder.h | 17 ++---- .../xla/service/cpu/cpu_executable.cc | 6 -- .../xla/service/gpu/gpu_executable.cc | 6 -- tensorflow/compiler/xla/service/hlo.proto | 14 +---- .../service/hlo_input_output_alias_config.cc | 38 +++--------- .../service/hlo_input_output_alias_config.h | 32 +++------- tensorflow/compiler/xla/service/hlo_parser.cc | 59 ++++++++----------- .../compiler/xla/service/hlo_parser_test.cc | 41 +++++++++++-- .../xla/tests/buffer_donation_test.cc | 49 ++------------- .../tpu/tpu_executable_interface.cc | 18 ------ 12 files changed, 90 insertions(+), 194 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc index 8a07aab11e1..6ebf6897bb1 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc @@ -524,7 +524,7 @@ TEST(CompileGraphToXlaHlo, Resources) { ASSERT_TRUE(status_or_hlo_module.ok()); constexpr char expected_hlo_module_string[] = - R"(HloModule main.4, input_output_alias={ {0}: (1, {}, may_alias) } + R"(HloModule main.4, input_output_alias={ {0}: 1 } ENTRY %main.4 (Arg_0.1: f32[2], Arg_1.2: f32[2]) -> (f32[2]) { %Arg_1.2 = f32[2]{0} parameter(1) diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc index 484fb0aabe7..52f61408cbb 100644 --- a/tensorflow/compiler/xla/client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_builder.cc @@ -446,7 +446,7 @@ StatusOr XlaBuilder::Build(int64 root_id, alias.param_index.ToString().c_str()); } TF_RETURN_IF_ERROR(config.SetUpAlias(alias.output_index, alias.param_number, - alias.param_index, alias.kind)); + alias.param_index)); } *module->mutable_input_output_alias() = config.ToProto(); return Status::OK(); diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h index aa5074d28d9..1960d0c4632 100644 --- a/tensorflow/compiler/xla/client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_builder.h @@ -32,7 +32,6 @@ limitations under the License. #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" -#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -350,16 +349,12 @@ class XlaBuilder { // not available until the computation is built, and eventual error in the // arguments of this API will be detected only at computation Build() time. // - // Note: Except when 'must-alias' is true, alias is assumed to be 'may-alias' - // and only donated buffer at runtime will be aliased with output. If a buffer - // is not donated at runtime, a copy will be inserted by XLA to prevent buffer - // clobbering. + // Note: Aliasing API is 'may-alias' and only donated buffer at runtime will + // be aliased with output. If a buffer is not donated at runtime, a copy will + // be inserted by XLA to prevent buffer clobbering. void SetUpAlias(const ShapeIndex& output_index, int64 param_number, - const ShapeIndex& param_index, - HloInputOutputAliasConfig::AliasKind kind = - HloInputOutputAliasConfig::AliasKind::kMayAlias) { - input_output_aliases_.push_back( - {output_index, param_number, param_index, kind}); + const ShapeIndex& param_index) { + input_output_aliases_.push_back({output_index, param_number, param_index}); } // Describes an input/output alias as inserted by the SetUpAlias() API. @@ -370,8 +365,6 @@ class XlaBuilder { int64 param_number; // Specifies the index of the aliased buffer in the parameter ShapeIndex param_index; - // Specifies if the alias is a must alias or may alias. - HloInputOutputAliasConfig::AliasKind kind; }; // Looks up the HloInstruction and sets the frontend attribute "attribute" to diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc index 7431e829b8e..0abcc91a1d7 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc @@ -247,12 +247,6 @@ StatusOr CpuExecutable::CreateResultShapedBuffer( ExecutionInput& input = arguments[alias->parameter_number]; MaybeOwningDeviceMemory* maybe_owning_memory = input.MutableBuffer(alias->parameter_index); - if (alias->must_alias() && !maybe_owning_memory->HasOwnership()) { - return InvalidArgument( - "An input was configured to be must-alias at " - "compile time but not donated at runtime: %s", - alias->ToString()); - } if (absl::optional owning = maybe_owning_memory->Release()) { // If the caller passes the ownership of the device memory, reuse it diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc index 726f1963545..469f2919fba 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc @@ -480,12 +480,6 @@ StatusOr GpuExecutable::ExecuteAsyncOnStream( ExecutionInput& input = arguments[alias->parameter_number]; MaybeOwningDeviceMemory* maybe_owning_memory = input.MutableBuffer(alias->parameter_index); - if (alias->must_alias() && !maybe_owning_memory->HasOwnership()) { - return InvalidArgument( - "An input was configured to be must-alias at " - "compile time but not donated at runtime: %s", - alias->ToString()); - } if (absl::optional owning = maybe_owning_memory->Release()) { // If the caller passes the ownership of the device memory, reuse it diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto index e043216c17e..960f60fe882 100644 --- a/tensorflow/compiler/xla/service/hlo.proto +++ b/tensorflow/compiler/xla/service/hlo.proto @@ -283,16 +283,6 @@ message HloScheduleProto { map sequences = 1; } -enum Kind { - // Define a UNDEFINED_ALIAS equal to zero to get around the default-0 proto3 - // behavior and missing has_*() APIs. - UNDEFINED_ALIAS = 0; - // The buffers may or may not alias at runtime. - MAY_ALIAS = 1; - // The buffers must alias at runtime. - MUST_ALIAS = 2; -} - message HloInputOutputAliasProto { // The following proto describes a pair of aliased an input // (described by parameter number and a ShapeIndex of the parameter) @@ -314,8 +304,8 @@ message HloInputOutputAliasProto { int64 parameter_number = 2; // ShapeIndex of the parameter instruction. repeated int64 parameter_shape_index = 3; - // The kind of alias to be setup. - Kind kind = 4; + reserved 4; + reserved "kind"; } repeated AliasEntryProto entries = 1; diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc index 34bc30d641f..e123161720b 100644 --- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc +++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc @@ -15,7 +15,6 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h" -#include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_module.h" namespace xla { @@ -25,10 +24,9 @@ bool HloInputOutputAliasConfig::OutputHasAlias( return alias_.element(output_index).has_value(); } -Status HloInputOutputAliasConfig::SetUpAlias( - const ShapeIndex& output_index, int64 param_number, - const ShapeIndex& param_index, - HloInputOutputAliasConfig::AliasKind must_alias) { +Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index, + int64 param_number, + const ShapeIndex& param_index) { TF_RET_CHECK(ShapeUtil::IndexIsValid(alias_.shape(), output_index)) << "Trying to set up alias at " << output_index.ToString() << " which is an invalid index for shape " @@ -43,8 +41,7 @@ Status HloInputOutputAliasConfig::SetUpAlias( param_number, param_index.ToString(), output_index.ToString(), alias_.element(output_index)->parameter_number, alias_.element(output_index)->parameter_index.ToString()); - (*alias_.mutable_element(output_index)) = - Alias(param_number, param_index, must_alias); + (*alias_.mutable_element(output_index)) = Alias(param_number, param_index); VLOG(4) << "Set up alias between output index " << output_index.ToString() << " and parameter " << param_index << " at index " << param_index.ToString(); @@ -64,11 +61,6 @@ HloInputOutputAliasProto HloInputOutputAliasConfig::ToProto() const { for (int64 i : data->parameter_index) { entry.add_parameter_shape_index(i); } - if (data->must_alias()) { - entry.set_kind(Kind::MUST_ALIAS); - } else { - entry.set_kind(Kind::MAY_ALIAS); - } result.add_entries()->Swap(&entry); } }); @@ -85,9 +77,8 @@ StatusOr HloInputOutputAliasConfig::CreateFromProto( int64 param_number = entry.parameter_number(); ShapeIndex param_index(entry.parameter_shape_index().begin(), entry.parameter_shape_index().end()); - AliasKind kind = entry.kind() == Kind::MAY_ALIAS ? kMayAlias : kMustAlias; TF_RETURN_IF_ERROR( - result.SetUpAlias(output_index, param_number, param_index, kind)); + result.SetUpAlias(output_index, param_number, param_index)); } return result; } @@ -102,9 +93,9 @@ string HloInputOutputAliasConfig::ToString() const { ForEachAlias([&](const ShapeIndex& output_index, const Alias& alias) { pieces.push_back(absl::StrFormat( - " OutputIndex %s is %saliased with parameter %lld at %s:", - output_index.ToString(), alias.kind == kMustAlias ? "must-" : "may-", - alias.parameter_number, alias.parameter_index.ToString())); + " OutputIndex %s is aliased with parameter %lld at %s:", + output_index.ToString(), alias.parameter_number, + alias.parameter_index.ToString())); }); return absl::StrJoin(pieces, "\n"); } @@ -121,19 +112,6 @@ string HloInputOutputAliasConfig::ToShortString() const { return absl::StrJoin(pieces, ", "); } -bool HloInputOutputAliasConfig::ParameterMustAlias( - int64 param_number, const ShapeIndex& param_index) const { - bool result = false; - alias_.ForEachElement( - [&](const xla::ShapeIndex&, absl::optional alias) { - if (alias && alias->parameter_number == param_number && - alias->parameter_index == param_index && alias->must_alias()) { - result = true; - } - }); - return result; -} - absl::optional HloInputOutputAliasConfig::GetAliasedOutput( int64 param_number, const ShapeIndex& param_index) const { absl::optional output; diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h index 6b84bdb6a68..d5ca28e9387 100644 --- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h +++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h @@ -32,32 +32,22 @@ class HloModule; // parameter index in the entry computation. class HloInputOutputAliasConfig { public: - // The kind of aliases which can be set. A kMayAlias is one setup at - // compilation time by the user, and has to be respected. A kMustAlias one - // might be setup by the compiler, if it decides it is convenient to do so. - enum AliasKind { - kMayAlias, - kMustAlias, - }; // Defines the alias information for a given output buffer. A given output // buffer shape index can refer only to one parameter+index. struct Alias { - Alias(int64 parameter_number, ShapeIndex parameter_index, - AliasKind kind = kMayAlias) + Alias(int64 parameter_number, ShapeIndex parameter_index) : parameter_number(parameter_number), - parameter_index(std::move(parameter_index)), - kind(kind) {} + parameter_index(std::move(parameter_index)) {} int64 parameter_number; ShapeIndex parameter_index; - AliasKind kind; - - bool must_alias() const { return kind == kMustAlias; } std::string ToString() { - return absl::StrFormat("(%lld, %s, %s)", parameter_number, - parameter_index.ToString(), - kind == kMustAlias ? "must_alias" : "may_alias"); + if (parameter_index.empty()) { + return absl::StrCat(parameter_number); + } + return absl::StrFormat("(%lld, %s)", parameter_number, + parameter_index.ToString()); } }; @@ -71,8 +61,7 @@ class HloInputOutputAliasConfig { // Sets up alias config from `output_index` to `param_index` at // `param_number`. Status SetUpAlias(const ShapeIndex& output_index, int64 param_number, - const ShapeIndex& param_index, - AliasKind must_alias = kMayAlias); + const ShapeIndex& param_index); // Returns true if the given parameter is aliased with one of the output // buffers. @@ -103,11 +92,6 @@ class HloInputOutputAliasConfig { absl::optional GetAliasedParameter( const ShapeIndex& output_index) const; - // Returns if the parameter at the given parameter number and parameter - // index must-alias with an output. - bool ParameterMustAlias(int64 param_number, - const ShapeIndex& param_index) const; - using AliasFn = std::function; diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc index 31afe2a3673..0530062c43b 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.cc +++ b/tensorflow/compiler/xla/service/hlo_parser.cc @@ -552,37 +552,31 @@ bool HloParserImpl::ParseAliasing(AliasingData* data) { return false; } - if (!ParseToken(TokKind::kLparen, errmsg)) { - return false; - } - int64 param_num; - ParseInt64(¶m_num); - if (!ParseToken(TokKind::kComma, errmsg)) { - return false; - } - ShapeIndex param_idx; - if (!ParseShapeIndex(¶m_idx)) { - return false; - } - - HloInputOutputAliasConfig::AliasKind alias_kind = - HloInputOutputAliasConfig::kMayAlias; - if (EatIfPresent(TokKind::kComma)) { - std::string type; - ParseName(&type); - if (type == "must-alias") { - alias_kind = HloInputOutputAliasConfig::kMustAlias; - } else if (type == "may-alias") { - alias_kind = HloInputOutputAliasConfig::kMayAlias; - } else { - return TokenError("Unexpected aliasing kind; expected SYSTEM or USER"); + if (lexer_.GetKind() != TokKind::kLparen) { + // Short form: "{0}: 0", output index "{}" is assumed. + int64 param_num; + ParseInt64(¶m_num); + data->emplace(std::piecewise_construct, std::forward_as_tuple(out), + std::forward_as_tuple(param_num, ShapeIndex{})); + } else { + // Long form: "{0}: (0, {0})", output index is explicitly specified. + if (!ParseToken(TokKind::kLparen, errmsg)) { + return false; + } + int64 param_num; + ParseInt64(¶m_num); + if (!ParseToken(TokKind::kComma, errmsg)) { + return false; + } + ShapeIndex param_idx; + if (!ParseShapeIndex(¶m_idx)) { + return false; + } + data->emplace(std::piecewise_construct, std::forward_as_tuple(out), + std::forward_as_tuple(param_num, param_idx)); + if (!ParseToken(TokKind::kRparen, errmsg)) { + return false; } - } - - data->emplace(std::piecewise_construct, std::forward_as_tuple(out), - std::forward_as_tuple(param_num, param_idx, alias_kind)); - if (!ParseToken(TokKind::kRparen, errmsg)) { - return false; } if (!EatIfPresent(TokKind::kComma)) { @@ -630,9 +624,8 @@ bool HloParserImpl::ParseHloModule(HloModule* module) { if (aliasing_data) { HloInputOutputAliasConfig alias_config(module->result_shape()); for (auto& p : *aliasing_data) { - Status st = - alias_config.SetUpAlias(p.first, p.second.parameter_number, - p.second.parameter_index, p.second.kind); + Status st = alias_config.SetUpAlias(p.first, p.second.parameter_number, + p.second.parameter_index); if (!st.ok()) { return TokenError(st.error_message()); } diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc index 86b6b1bedd9..484578e5e0e 100644 --- a/tensorflow/compiler/xla/service/hlo_parser_test.cc +++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc @@ -2399,7 +2399,7 @@ ENTRY c2 { TEST_F(HloParserTest, SimpleAliasing) { const string original = R"( -HloModule Module, input_output_alias={ {0}: (0, {0}, must-alias), {1}: (0, {1}) } +HloModule Module, input_output_alias={ {0}: (0, {0}), {1}: (0, {1}) } ENTRY entry { %p = (f32[], f32[]) parameter(0) @@ -2413,13 +2413,42 @@ ENTRY entry { std::unique_ptr parsed_module = module.ConsumeValueOrDie(); EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(0, {0}), ShapeIndex{0}); - - EXPECT_TRUE( - parsed_module->input_output_alias_config().ParameterMustAlias(0, {0})); EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(0, {1}), ShapeIndex{1}); - EXPECT_FALSE( - parsed_module->input_output_alias_config().ParameterMustAlias(0, {1})); +} + +TEST_F(HloParserTest, SimpleAliasingShortForm) { + const string original = R"( +HloModule Module, input_output_alias={ {0}: 0, {1}: 1 } + +ENTRY entry { + %p0 = f32[] parameter(0) + %p1 = f32[] parameter(1) + ROOT %out = (f32[], f32[]) tuple(%p0, %p1) +} + )"; + auto module = ParseAndReturnVerifiedModule(original); + TF_ASSERT_OK(module.status()); + std::unique_ptr parsed_module = module.ConsumeValueOrDie(); + EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(0, {}), + ShapeIndex{0}); + EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(1, {}), + ShapeIndex{1}); +} + +TEST_F(HloParserTest, SimpleAliasingShortFormError) { + const string original = R"( +HloModule Module, input_output_alias={ {0}: A, {1}: 1 } + +ENTRY entry { + %p0 = f32[] parameter(0) + %p1 = f32[] parameter(1) + ROOT %out = (f32[], f32[]) tuple(%p0, %p1) +} + )"; + ExpectHasSubstr( + ParseAndReturnUnverifiedModule(original).status().error_message(), + "expects integer"); } TEST_F(HloParserTest, NestedAliasing) { diff --git a/tensorflow/compiler/xla/tests/buffer_donation_test.cc b/tensorflow/compiler/xla/tests/buffer_donation_test.cc index f78083fe2af..856ea7c9b44 100644 --- a/tensorflow/compiler/xla/tests/buffer_donation_test.cc +++ b/tensorflow/compiler/xla/tests/buffer_donation_test.cc @@ -61,7 +61,7 @@ class BufferDonationTest : public HloTestBase { absl::Span argument_literals, absl::Span donate_arguments, absl::Span expected_runtime_aliasing, - const Literal& expected, std::string expected_failure = "") { + const Literal& expected) { // Create a copy of the output shape because the HLO module is std::moved // into the compiler and may be deallocated. const Shape output_shape = hlo_module->result_shape(); @@ -123,19 +123,10 @@ class BufferDonationTest : public HloTestBase { ExecutionInput(std::move(owned_buffers), argument_literal.shape())); } - StatusOr output_status = + TF_ASSERT_OK_AND_ASSIGN( + ExecutionOutput output, executable->ExecuteAsyncOnStream(&service_run_options, std::move(args), - /*hlo_execution_profile=*/nullptr); - if (!expected_failure.empty()) { - ASSERT_FALSE(output_status.ok()); - ASSERT_TRUE(absl::StrContains(output_status.status().error_message(), - expected_failure)) - << "got: \n" - << output_status.status().error_message() << " \nvs want\n" - << expected_failure; - return; - } - ExecutionOutput output = output_status.ConsumeValueOrDie(); + /*hlo_execution_profile=*/nullptr)); se::DeviceMemoryBase result_root_buffer = output.Result().root_buffer(); LOG(INFO) << "result allocation = " << result_root_buffer.opaque() @@ -312,37 +303,5 @@ ENTRY entry { #endif } -TEST_F(BufferDonationTest, TestMustAliasNotDonated) { - HloModuleConfig config; - - StatusOr> module = - ParseAndReturnVerifiedModule(R"( -HloModule module - -ENTRY entry { - a = f32[] parameter(0) - b = f32[] parameter(1) - ROOT out = (f32[], f32[]) tuple(a, b) -} - )", - config); - - TF_ASSERT_OK(module->get()->input_output_alias_config().SetUpAlias( - {0}, 0, {}, HloInputOutputAliasConfig::kMustAlias)); - - std::vector args; - args.push_back(LiteralUtil::CreateR0(0.1)); - args.push_back(LiteralUtil::CreateR0(0.2)); - Literal expected = LiteralUtil::MakeTupleFromSlices( - {LiteralUtil::CreateR0(0.1), LiteralUtil::CreateR0(0.2)}); - -#ifndef XLA_TEST_BACKEND_INTERPRETER - RunAndCheck(std::move(*module), args, - /*donate_arguments=*/{false, false}, {true, false}, expected, - "An input was configured to be must-alias at " - "compile time but not donated at runtime:"); -#endif -} - } // namespace } // namespace xla diff --git a/tensorflow/stream_executor/tpu/tpu_executable_interface.cc b/tensorflow/stream_executor/tpu/tpu_executable_interface.cc index f260cc1631f..13f9db98e5d 100644 --- a/tensorflow/stream_executor/tpu/tpu_executable_interface.cc +++ b/tensorflow/stream_executor/tpu/tpu_executable_interface.cc @@ -62,24 +62,6 @@ TpuExecutableInterface::AllocateOutputMemoryWithInputReuse( << " host_shape = " << ShapeUtil::HumanStringWithLayout(host_shape); Shape device_shape = HostShapeToDeviceShape(host_shape); - TF_RETURN_IF_ERROR(alias_config.ForEachAliasWithStatus( - [&](const ShapeIndex& output_index, - absl::optional alias) { - if (alias && alias->must_alias()) { - VLOG(1) << alias->ToString(); - const MaybeOwningDeviceMemory& original_input = - (*arguments)[alias->parameter_number].Buffers().element( - alias->parameter_index); - if (!original_input.HasOwnership()) { - return InvalidArgument( - "An input was configured to be must-alias at " - "compile time but not donated at runtime: %s", - alias->ToString()); - } - } - return Status::OK(); - })); - if (VLOG_IS_ON(3)) { VLOG(3) << "AllocateOutputMemoryWithInputReuse, device = " << device_ordinal << " host_shape = " << ShapeUtil::HumanStringWithLayout(host_shape); From 8491e4ec502c7f4af64a6a3602552e7efb12c633 Mon Sep 17 00:00:00 2001 From: Rick Chao Date: Tue, 4 Aug 2020 19:36:00 -0700 Subject: [PATCH 0428/1017] Disable python/keras/distribute:collective_all_reduce_strategy_test on msan as it flakily fails. PiperOrigin-RevId: 324941016 Change-Id: Ic842c9b4f62f79f4cd2814718ff959b0e0b43594 --- tensorflow/python/keras/distribute/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/keras/distribute/BUILD b/tensorflow/python/keras/distribute/BUILD index 5a5cff01e33..56a6a9d0e1f 100644 --- a/tensorflow/python/keras/distribute/BUILD +++ b/tensorflow/python/keras/distribute/BUILD @@ -163,6 +163,7 @@ cuda_py_test( python_version = "PY3", tags = [ "multi_and_single_gpu", + "nomsan", # TODO(b/162894966) ], # b/155301154 broken with XLA:GPU xla_enable_strict_auto_jit = True, From 70c23b653fc258f5481474b920962a69b662de72 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 19:41:45 -0700 Subject: [PATCH 0429/1017] Enable clipnorm and clipvalue arguments in Optimizer with tf.distribute.Strategy. Apply gradient clipping after aggregation. CentralStorageStrategy is still not supported with these arguments. PiperOrigin-RevId: 324941564 Change-Id: If224a86efabbb28ffe589f14d3b4c15787ce735b --- RELEASE.md | 6 - .../distribute/distribute_strategy_test.py | 31 ----- tensorflow/python/keras/engine/training.py | 1 + .../python/keras/engine/training_eager.py | 1 + .../experimental/loss_scale_optimizer.py | 4 +- tensorflow/python/keras/optimizer_v2/BUILD | 1 - .../python/keras/optimizer_v2/optimizer_v2.py | 110 +++++++++--------- tensorflow/python/keras/optimizer_v2/utils.py | 38 ------ ...n.experimental.-loss-scale-optimizer.pbtxt | 8 -- ...ensorflow.keras.optimizers.-adadelta.pbtxt | 8 -- ...tensorflow.keras.optimizers.-adagrad.pbtxt | 8 -- .../tensorflow.keras.optimizers.-adam.pbtxt | 8 -- .../tensorflow.keras.optimizers.-adamax.pbtxt | 8 -- .../tensorflow.keras.optimizers.-ftrl.pbtxt | 8 -- .../tensorflow.keras.optimizers.-nadam.pbtxt | 8 -- ...nsorflow.keras.optimizers.-optimizer.pbtxt | 8 -- ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt | 8 -- .../tensorflow.keras.optimizers.-s-g-d.pbtxt | 8 -- ...n.experimental.-loss-scale-optimizer.pbtxt | 8 -- ...ensorflow.keras.optimizers.-adadelta.pbtxt | 8 -- ...tensorflow.keras.optimizers.-adagrad.pbtxt | 8 -- .../tensorflow.keras.optimizers.-adam.pbtxt | 8 -- .../tensorflow.keras.optimizers.-adamax.pbtxt | 8 -- .../tensorflow.keras.optimizers.-ftrl.pbtxt | 8 -- .../tensorflow.keras.optimizers.-nadam.pbtxt | 8 -- ...nsorflow.keras.optimizers.-optimizer.pbtxt | 8 -- ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt | 8 -- .../tensorflow.keras.optimizers.-s-g-d.pbtxt | 8 -- .../v2/tensorflow.optimizers.-adadelta.pbtxt | 8 -- .../v2/tensorflow.optimizers.-adagrad.pbtxt | 8 -- .../v2/tensorflow.optimizers.-adam.pbtxt | 8 -- .../v2/tensorflow.optimizers.-adamax.pbtxt | 8 -- .../v2/tensorflow.optimizers.-ftrl.pbtxt | 8 -- .../v2/tensorflow.optimizers.-nadam.pbtxt | 8 -- .../v2/tensorflow.optimizers.-optimizer.pbtxt | 8 -- .../v2/tensorflow.optimizers.-r-m-sprop.pbtxt | 8 -- .../v2/tensorflow.optimizers.-s-g-d.pbtxt | 8 -- 37 files changed, 58 insertions(+), 366 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index d7a345c7c76..b0c785c7d68 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -33,10 +33,6 @@ shape assumptions (note that you can pass shapes with `None` entries for axes that are meant to be dynamic). You can also disable the input checking entirely by setting `model.input_spec = None`. -* `tf.keras.optimizers.Optimizer.get_gradients` no longer performs gradient - clipping. Instead, gradient clipping is performed in - `tf.keras.optimizers.Optimizer.apply_gradients`, after the gradients on each - device have been aggregated. ## Known Caveats @@ -99,8 +95,6 @@ * Error messages when Functional API construction goes wrong (and when ops cannot be converted to Keras layers automatically) should be clearer and easier to understand. * `Optimizer.minimize` can now accept a loss `Tensor` and a `GradientTape` as an alternative to accepting a `callable` loss. - * `Optimizer` arguments `clipnorm` and `clipvalue` are now supported with - `tf.distribute.Strategy` (`CentralStorageStrategy` is not yet supported). * `tf.function` / AutoGraph: * Added `experimental_follow_type_hints` argument for `tf.function`. When True, the function may use type annotations to optimize the tracing diff --git a/tensorflow/python/keras/distribute/distribute_strategy_test.py b/tensorflow/python/keras/distribute/distribute_strategy_test.py index abcb5d1c0e8..4b6d3a80730 100644 --- a/tensorflow/python/keras/distribute/distribute_strategy_test.py +++ b/tensorflow/python/keras/distribute/distribute_strategy_test.py @@ -22,7 +22,6 @@ import numpy as np from tensorflow.python import keras from tensorflow.python.data.experimental.ops import cardinality from tensorflow.python.data.ops import dataset_ops -from tensorflow.python.distribute import central_storage_strategy from tensorflow.python.distribute import combinations from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import mirrored_strategy @@ -1864,36 +1863,6 @@ class TestDistributionStrategyWithKerasModels(test.TestCase, self.assertEqual(bc.predict_begin_batches, [0]) self.assertEqual(bc.predict_end_batches, [24]) - @combinations.generate( - combinations.combine(distribution=all_strategies, mode=['eager'])) - def test_gradient_clipping(self, distribution): - - class MyLayer(keras.layers.Layer): - - def build(self, _): - self.v1 = variables.Variable(1.) - self.v2 = variables.Variable(1.) - - def call(self, x): - return 3 * self.v1 - 3 * self.v2 - - x, y = np.ones((10, 1)), np.ones((10, 1)) - - with distribution.scope(): - layer = MyLayer() - model = keras.Sequential([layer]) - optimizer = gradient_descent_keras.SGD(1., clipnorm=2., clipvalue=2.) - model.compile(optimizer, 'mae') - - if isinstance(distribution, - central_storage_strategy.CentralStorageStrategy): - with self.assertRaisesRegex(ValueError, 'not supported'): - model.fit(x, y, batch_size=10, epochs=1) - else: - model.fit(x, y, batch_size=10, epochs=1) - self.assertAllClose(self.evaluate(layer.v1), 3.) - self.assertAllClose(self.evaluate(layer.v2), -1.) - @combinations.generate( combinations.times( all_strategy_combinations_minus_default())) diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index a1fb329feab..bf542129e5c 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -2744,6 +2744,7 @@ def _minimize(strategy, tape, optimizer, loss, trainable_variables): trainable_variables)) if isinstance(optimizer, lso.LossScaleOptimizer): gradients = optimizer.get_unscaled_gradients(gradients) + gradients = optimizer._clip_gradients(gradients) # pylint: disable=protected-access if trainable_variables: if aggregate_grads_outside_optimizer: optimizer.apply_gradients( diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py index b3ce3d13ed7..8064bf2a7ab 100644 --- a/tensorflow/python/keras/engine/training_eager.py +++ b/tensorflow/python/keras/engine/training_eager.py @@ -273,6 +273,7 @@ def _process_single_batch(model, if isinstance(model.optimizer, loss_scale_optimizer.LossScaleOptimizer): grads = model.optimizer.get_unscaled_gradients(grads) + grads = model.optimizer._clip_gradients(grads) model.optimizer.apply_gradients(zip(grads, trainable_weights)) else: logging.warning('The list of trainable weights is empty. Make sure that' diff --git a/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py b/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py index 59a49b03ad5..4a3f459de80 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py +++ b/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py @@ -258,8 +258,8 @@ class LossScaleOptimizer(_DelegatingTrackableMixin, optimizer_v2.OptimizerV2): 'clipvalue %s' % (optimizer, optimizer.clipvalue)) self._raise_if_strategy_unsupported() - self._clipnorm = None - self._clipvalue = None + self.clipnorm = None + self.clipvalue = None self._optimizer = optimizer self._loss_scale = keras_loss_scale_module.get(loss_scale) diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD index 9a317e5d114..b519ec7fb3d 100644 --- a/tensorflow/python/keras/optimizer_v2/BUILD +++ b/tensorflow/python/keras/optimizer_v2/BUILD @@ -40,7 +40,6 @@ py_library( "//tensorflow/python:state_ops", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", - "//tensorflow/python/distribute:central_storage_strategy", "//tensorflow/python/distribute:distribute_lib", "//tensorflow/python/distribute:parameter_server_strategy", "//tensorflow/python/distribute:reduce_util", diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py index 0ecca63a64f..18d94594542 100644 --- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py +++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py @@ -41,6 +41,7 @@ from tensorflow.python.keras.optimizer_v2 import utils as optimizer_utils from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import tf_utils from tensorflow.python.ops import array_ops +from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_resource_variable_ops from tensorflow.python.ops import gradients @@ -331,6 +332,15 @@ class OptimizerV2(trackable.Trackable): raise ValueError("decay cannot be less than 0: {}".format(decay)) self._initial_decay = decay + # Set the gradient clipping properties + self.clipnorm = kwargs.pop("clipnorm", None) + self.clipvalue = kwargs.pop("clipvalue", None) + if ((self.clipnorm is not None or self.clipvalue is not None) + and distribute_ctx.has_strategy()): + raise ValueError("Gradient clipping in the optimizer " + "(by setting clipnorm or clipvalue) is currently " + "unsupported when using a distribution strategy.") + self._hypers_created = False # Store the distribution strategy object if the optimizer is created inside @@ -340,33 +350,6 @@ class OptimizerV2(trackable.Trackable): else: self._distribution_strategy = None - # Set the gradient clipping properties - self._clipnorm = kwargs.pop("clipnorm", None) - self._clipvalue = kwargs.pop("clipvalue", None) - - # Configure gradient transforms. - self._transform_gradients_fns = [] - - if self._clipnorm is not None: - self._transform_gradients_fns.append( - optimizer_utils.make_gradient_clipnorm_fn(self._clipnorm)) - if self._clipvalue is not None: - self._transform_gradients_fns.append( - optimizer_utils.make_gradient_clipvalue_fn(self._clipvalue)) - - @property - def clipnorm(self): - """`float` or `None`. If set, clips gradients to this maximum norm.""" - return self._clipnorm - - @property - def clipvalue(self): - """`float` or `None`. - - If set, clips gradients to this maximum absolute value. - """ - return self._clipvalue - def minimize(self, loss, var_list, grad_loss=None, name=None, tape=None): """Minimize `loss` by updating `var_list`. @@ -402,6 +385,26 @@ class OptimizerV2(trackable.Trackable): loss, var_list=var_list, grad_loss=grad_loss, tape=tape) return self.apply_gradients(grads_and_vars, name=name) + def _clip_gradients(self, grads): + """Clip gradients according to the clipnorm and clipvalue attributes.""" + if self.clipnorm is not None: + if distribute_ctx.has_strategy(): + raise ValueError("Gradient clipping in the optimizer " + "(by setting clipnorm or clipvalue) is currently " + "unsupported when using a distribution strategy.") + grads = [None if g is None else clip_ops.clip_by_norm(g, self.clipnorm) + for g in grads] + if self.clipvalue is not None: + if distribute_ctx.has_strategy(): + raise ValueError("Gradient clipping in the optimizer " + "(by setting clipnorm or clipvalue) is currently " + "unsupported when using a distribution strategy.") + v = self.clipvalue + grads = [ + None if g is None else clip_ops.clip_by_value(g, -v, v) for g in grads + ] + return grads + def _compute_gradients(self, loss, var_list, grad_loss=None, tape=None): """Compute gradients of `loss` for the variables in `var_list`. @@ -451,6 +454,8 @@ class OptimizerV2(trackable.Trackable): var_list = nest.flatten(var_list) with ops.name_scope_v2(self._name + "/gradients"): grads = tape.gradient(loss, var_list, grad_loss) + # TODO(omalleyt): Move to post-aggregation. + grads = self._clip_gradients(grads) grads_and_vars = list(zip(grads, var_list)) self._assert_valid_dtypes([ @@ -460,12 +465,6 @@ class OptimizerV2(trackable.Trackable): return grads_and_vars - def _transform_gradients(self, grads_and_vars): - """Transformations to apply aggregated gradients.""" - for fn in self._transform_gradients_fns: - grads_and_vars = fn(grads_and_vars) - return grads_and_vars - def get_gradients(self, loss, params): """Returns gradients of `loss` with respect to `params`. @@ -484,15 +483,14 @@ class OptimizerV2(trackable.Trackable): with backend.get_graph().as_default(), backend.name_scope(self._name + "/gradients"): grads = gradients.gradients(loss, params) - grads_and_vars = list(zip(grads, params)) - for grad, param in grads_and_vars: + for grad, param in zip(grads, params): if grad is None: raise ValueError("Variable {} has `None` for gradient. " "Please make sure that all of your ops have a " "gradient defined (i.e. are differentiable). " "Common ops without gradient: " "K.argmax, K.round, K.eval.".format(param)) - grads = [g for g, _ in grads_and_vars] + grads = self._clip_gradients(grads) return grads def apply_gradients(self, @@ -536,23 +534,10 @@ class OptimizerV2(trackable.Trackable): ValueError: If none of the variables have gradients. RuntimeError: If called in a cross-replica context. """ - if distribute_ctx.in_cross_replica_context(): - raise RuntimeError( - "`apply_gradients() cannot be called in cross-replica context. " - "Use `tf.distribute.Strategy.run` to enter replica " - "context.") - - strategy = distribute_ctx.get_strategy() - if (not experimental_aggregate_gradients and strategy and - isinstance(strategy.extended, - parameter_server_strategy.ParameterServerStrategyExtended)): - raise NotImplementedError( - "`experimental_aggregate_gradients=False is not supported for " - "ParameterServerStrategy and CentralStorageStrategy") - grads_and_vars = optimizer_utils.filter_empty_gradients(grads_and_vars) - var_list = [v for _, v in grads_and_vars] - with ops.name_scope_v2(self._name): + var_list = [v for (_, v) in grads_and_vars] + + with backend.name_scope(self._name): # Create iteration if necessary. with ops.init_scope(): self._create_all_weights(var_list) @@ -562,12 +547,25 @@ class OptimizerV2(trackable.Trackable): # gradients return control_flow_ops.no_op() - if experimental_aggregate_gradients: - reduced_grads = self._aggregate_gradients(grads_and_vars) - grads_and_vars = list(zip(reduced_grads, var_list)) - grads_and_vars = self._transform_gradients(grads_and_vars) + if distribute_ctx.in_cross_replica_context(): + raise RuntimeError( + "`apply_gradients() cannot be called in cross-replica context. " + "Use `tf.distribute.Strategy.run` to enter replica " + "context.") + + strategy = distribute_ctx.get_strategy() + if (not experimental_aggregate_gradients and strategy and isinstance( + strategy.extended, + parameter_server_strategy.ParameterServerStrategyExtended)): + raise NotImplementedError( + "`experimental_aggregate_gradients=False is not supported for " + "ParameterServerStrategy and CentralStorageStrategy") apply_state = self._prepare(var_list) + if experimental_aggregate_gradients: + reduced_grads = self._aggregate_gradients(grads_and_vars) + var_list = [v for _, v in grads_and_vars] + grads_and_vars = list(zip(reduced_grads, var_list)) return distribute_ctx.get_replica_context().merge_call( functools.partial(self._distributed_apply, apply_state=apply_state), args=(grads_and_vars,), diff --git a/tensorflow/python/keras/optimizer_v2/utils.py b/tensorflow/python/keras/optimizer_v2/utils.py index f723c6d8b64..9f680e04dd6 100644 --- a/tensorflow/python/keras/optimizer_v2/utils.py +++ b/tensorflow/python/keras/optimizer_v2/utils.py @@ -18,10 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from tensorflow.python.distribute import central_storage_strategy from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx from tensorflow.python.distribute import reduce_util as ds_reduce_util -from tensorflow.python.ops import clip_ops from tensorflow.python.platform import tf_logging as logging @@ -59,42 +57,6 @@ def all_reduce_sum_gradients(grads_and_vars): return reduced_with_nones -def make_gradient_clipnorm_fn(clipnorm): - """Creates a gradient transformation function for clipping by norm.""" - - def gradient_clipnorm_fn(grads_and_vars): - - if isinstance(distribute_ctx.get_strategy(), - central_storage_strategy.CentralStorageStrategy): - raise ValueError( - "`clipnorm` is not supported with `CenteralStorageStrategy`") - - clipped_grads_and_vars = [ - (clip_ops.clip_by_norm(g, clipnorm), v) for g, v in grads_and_vars - ] - return clipped_grads_and_vars - - return gradient_clipnorm_fn - - -def make_gradient_clipvalue_fn(clipvalue): - """Creates a gradient transformation function for clipping by value.""" - - def gradient_clipvalue_fn(grads_and_vars): - - if isinstance(distribute_ctx.get_strategy(), - central_storage_strategy.CentralStorageStrategy): - raise ValueError( - "`clipvalue` is not supported with `CenteralStorageStrategy`") - - clipped_grads_and_vars = [(clip_ops.clip_by_value(g, -clipvalue, - clipvalue), v) - for g, v in grads_and_vars] - return clipped_grads_and_vars - - return gradient_clipvalue_fn - - def filter_empty_gradients(grads_and_vars): """Filter out `(grad, var)` pairs that have a gradient equal to `None`.""" grads_and_vars = tuple(grads_and_vars) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt index 58f8cf24495..dbab3abae8e 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt @@ -5,14 +5,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt index fb341cb24dd..af854e98013 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt index d8039ed21ef..e89cc5cef75 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt index 912f92f83a6..15414d7234f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt index 3abc6d39b3f..8b3c429e6b5 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt index 00880d3f73b..51ab675db74 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt index 2ce311d3504..342c0951bbe 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt index 2020de9fa5c..f007b4b971a 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt @@ -3,14 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt index 80a1449613c..d5bf6fa7f47 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt index 8acfe214256..df904f72511 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt index 58f8cf24495..dbab3abae8e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt @@ -5,14 +5,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt index fb341cb24dd..af854e98013 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt index d8039ed21ef..e89cc5cef75 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt index 912f92f83a6..15414d7234f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt index 3abc6d39b3f..8b3c429e6b5 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt index 00880d3f73b..51ab675db74 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt index 2ce311d3504..342c0951bbe 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt index 2020de9fa5c..f007b4b971a 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt @@ -3,14 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt index 80a1449613c..d5bf6fa7f47 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt index 8acfe214256..df904f72511 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt index 06212bdc95d..cb3d38246a7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt index 09fff0514d8..c7b2bca4b6b 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt index 195ba9e4f56..209c9fe6620 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt index 9859da430bd..12bbb14fb71 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt index a4ed911e39d..1482ed54eb9 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt index 128f223fdc7..2a422fa2340 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt index 5ea1ed521ef..e7021e02772 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt @@ -3,14 +3,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt index db89ecbabe7..6543f4023a4 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt index 0cb0205e65e..94ff8dfcdfc 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt @@ -4,14 +4,6 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" - member { - name: "clipnorm" - mtype: "" - } - member { - name: "clipvalue" - mtype: "" - } member { name: "iterations" mtype: "" From c0b2748f6bc35dcbdc775e77d00f7da4fab75cde Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Tue, 4 Aug 2020 19:51:27 -0700 Subject: [PATCH 0430/1017] Split GPU Compatibility Lib into two. PiperOrigin-RevId: 324942428 Change-Id: I8d5ac8aca0f2ec8d889822f9d4c2ed8c340a3ae0 --- .../delegates/gpu/java/src/main/native/BUILD | 2 +- .../java/src/main/native/gpu_delegate_jni.cc | 6 +- .../acceleration/compatibility/BUILD | 38 +------ .../compatibility/gpu_compatibility.cc | 9 +- .../compatibility/gpu_compatibility.h | 38 ++++--- .../gpu_compatibility_recommender.cc | 30 ------ .../gpu_compatibility_recommender.h | 64 ----------- .../gpu_compatibility_recommender_test.cc | 100 ------------------ .../compatibility/gpu_compatibility_test.cc | 61 ----------- 9 files changed, 38 insertions(+), 310 deletions(-) delete mode 100644 tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.cc delete mode 100644 tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h delete mode 100644 tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender_test.cc delete mode 100644 tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_test.cc diff --git a/tensorflow/lite/delegates/gpu/java/src/main/native/BUILD b/tensorflow/lite/delegates/gpu/java/src/main/native/BUILD index 7b340e20562..00b56bb0c06 100644 --- a/tensorflow/lite/delegates/gpu/java/src/main/native/BUILD +++ b/tensorflow/lite/delegates/gpu/java/src/main/native/BUILD @@ -30,7 +30,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/gl:egl_environment", "//tensorflow/lite/delegates/gpu/gl:request_gpu_info", "//tensorflow/lite/experimental/acceleration/compatibility:android_info", - "//tensorflow/lite/experimental/acceleration/compatibility:gpu_compatibility_recommender", + "//tensorflow/lite/experimental/acceleration/compatibility:gpu_compatibility", "//tensorflow/lite/java/jni", "@com_google_absl//absl/status", ], diff --git a/tensorflow/lite/delegates/gpu/java/src/main/native/gpu_delegate_jni.cc b/tensorflow/lite/delegates/gpu/java/src/main/native/gpu_delegate_jni.cc index c4571100818..d31d058b796 100644 --- a/tensorflow/lite/delegates/gpu/java/src/main/native/gpu_delegate_jni.cc +++ b/tensorflow/lite/delegates/gpu/java/src/main/native/gpu_delegate_jni.cc @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/egl_environment.h" #include "tensorflow/lite/delegates/gpu/gl/request_gpu_info.h" #include "tensorflow/lite/experimental/acceleration/compatibility/android_info.h" -#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h" +#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h" #ifdef __cplusplus extern "C" { @@ -74,13 +74,13 @@ class CompatibilityListHelper { } bool IsDelegateSupportedOnThisDevice() { - return compatibility_recommender_.Includes(android_info_, gpu_info_); + return compatibility_list_.Includes(android_info_, gpu_info_); } private: tflite::acceleration::AndroidInfo android_info_; tflite::gpu::GpuInfo gpu_info_; - tflite::acceleration::GPUCompatibilityRecommender compatibility_recommender_; + tflite::acceleration::GPUCompatibilityList compatibility_list_; }; } // namespace diff --git a/tensorflow/lite/experimental/acceleration/compatibility/BUILD b/tensorflow/lite/experimental/acceleration/compatibility/BUILD index 6adb6daaa6f..78a9d2eb8d8 100644 --- a/tensorflow/lite/experimental/acceleration/compatibility/BUILD +++ b/tensorflow/lite/experimental/acceleration/compatibility/BUILD @@ -152,6 +152,7 @@ cc_library( ":android_info", ":database_fbs", ":devicedb", + "//tensorflow/lite/delegates/gpu:delegate", "//tensorflow/lite/delegates/gpu/common:gpu_info", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", @@ -159,41 +160,4 @@ cc_library( ], ) -cc_test( - name = "gpu_compatibility_test", - srcs = ["gpu_compatibility_test.cc"], - deps = [ - ":gpu_compatibility", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], -) - -cc_library( - name = "gpu_compatibility_recommender", - srcs = [ - "gpu_compatibility_recommender.cc", - ], - hdrs = [ - "gpu_compatibility_recommender.h", - ], - deps = [ - ":android_info", - ":gpu_compatibility", - "//tensorflow/lite/delegates/gpu:delegate", - "//tensorflow/lite/delegates/gpu/common:gpu_info", - ], -) - -cc_test( - name = "gpu_compatibility_recommender_test", - srcs = ["gpu_compatibility_recommender_test.cc"], - tags = ["notap"], # Needs to be built with --copt=-DCL_DELEGATE_NO_GL - deps = [ - ":gpu_compatibility_recommender", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], -) - tflite_portable_test_suite() diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.cc b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.cc index 1911d26b8df..e04f5d18db4 100644 --- a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.cc +++ b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.cc @@ -89,8 +89,13 @@ bool GPUCompatibilityList::Includes( return variables[gpu::kStatus] == std::string(gpu::kStatusSupported); } -bool GPUCompatibilityList::IsDatabaseLoaded() const { - return database_ != nullptr; +TfLiteGpuDelegateOptionsV2 GPUCompatibilityList::GetBestOptionsFor( + const AndroidInfo& /* android_info */, + const ::tflite::gpu::GpuInfo& /* gpu_info */) const { + // This method is for forwards-compatibility: the list may later include + // information about which backend to choose (OpenGL/OpenCL/Vulkan) or other + // options. + return TfLiteGpuDelegateOptionsV2Default(); } } // namespace acceleration diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h index 873151dca66..f975fe04f22 100644 --- a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h +++ b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h @@ -19,6 +19,7 @@ limitations under the License. #include #include "tensorflow/lite/delegates/gpu/common/gpu_info.h" +#include "tensorflow/lite/delegates/gpu/delegate.h" #include "tensorflow/lite/experimental/acceleration/compatibility/android_info.h" #include "tensorflow/lite/experimental/acceleration/compatibility/devicedb.h" @@ -31,41 +32,54 @@ namespace acceleration { // Android version, OpenGL ES version, GPU chipset etc. The support is based on // measure stability, correctness and peformance. For more detail see README.md. // -// Reads from the flatbuffer. // Example usage: -// tflite::acceleration::GPUCompatibilityList list; +// tflite::Interpreter* interpreter = ... ; // tflite::acceleration::AndroidInfo android_info; // tflite::gpu::GpuInfo gpu_info; -// ... -// if(list.Includes(android_info, gpu_info)){ -// // SUPPORTED. -// } else{ -// // UNSUPPORTED. +// EXPECT_OK(tflite::acceleration::RequestAndroidInfo(&android_info)); +// EXPECT_OK(tflite::gpu::gl::EglEnvironment::NewEglEnvironment(&env)); +// EXPECT_OK(tflite::gpu::gl::RequestGpuInfo(&tflite_gpu_info)); +// tflite::acceleration::GPUCompatibilityList list; +// TfLiteDelegate* gpu_delegate = nullptr; +// TfLiteGpuDelegateOptions gpu_options; +// if (list.Includes(android_info, gpu_info)) { +// gpu_options = list.BestOptionsFor(android_info, gpu_info); +// gpu_delegate = TfLiteGpuDelegateCreate(&gpu_options); +// EXPECT_EQ(interpreter->ModifyGraphWithDelegate(gpu_delegate), TfLiteOk); +// } else { +// // Fallback path. // } class GPUCompatibilityList { public: // Construct list from bundled data. GPUCompatibilityList(); - // Constructs list from the given flatbuffer. - explicit GPUCompatibilityList( - const unsigned char* compatibility_list_flatbuffer); // Returns true if the provided device specs are supported by the database. bool Includes(const AndroidInfo& android_info, const ::tflite::gpu::GpuInfo& gpu_info) const; + + // Returns the best TfLiteGpuDelegateOptionsV2 for the provided device specs + // based on the database. The output can be modified as desired before passing + // to delegate creation. + TfLiteGpuDelegateOptionsV2 GetBestOptionsFor( + const AndroidInfo& android_info, + const ::tflite::gpu::GpuInfo& gpu_info) const; + // Convert android_info and gpu_info into a set of variables used for querying // the list, and update variables from list data. See variables.h // and devicedb.h for more information. std::map CalculateVariables( const AndroidInfo& android_info, const ::tflite::gpu::GpuInfo& gpu_info) const; + GPUCompatibilityList(const GPUCompatibilityList&) = delete; GPUCompatibilityList& operator=(const GPUCompatibilityList&) = delete; - // Indicates if the database is loaded. - bool IsDatabaseLoaded() const; protected: + explicit GPUCompatibilityList( + const unsigned char* compatibility_list_flatbuffer); const DeviceDatabase* database_; }; + } // namespace acceleration } // namespace tflite diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.cc b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.cc deleted file mode 100644 index 1b625913323..00000000000 --- a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.cc +++ /dev/null @@ -1,30 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h" - -namespace tflite { -namespace acceleration { - -TfLiteGpuDelegateOptionsV2 GPUCompatibilityRecommender::GetBestOptionsFor( - const AndroidInfo& /* android_info */, - const ::tflite::gpu::GpuInfo& /* gpu_info */) const { - // This method is for forwards-compatibility: the list may later include - // information about which backend to choose (OpenGL/OpenCL/Vulkan) or other - // options. - return TfLiteGpuDelegateOptionsV2Default(); -} - -} // namespace acceleration -} // namespace tflite diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h deleted file mode 100644 index 4443cfdf70f..00000000000 --- a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h +++ /dev/null @@ -1,64 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#ifndef TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_COMPATIBILITY_GPU_COMPATIBILITY_RECOMMENDER_H_ -#define TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_COMPATIBILITY_GPU_COMPATIBILITY_RECOMMENDER_H_ - -#include "tensorflow/lite/delegates/gpu/common/gpu_info.h" -#include "tensorflow/lite/delegates/gpu/delegate.h" -#include "tensorflow/lite/experimental/acceleration/compatibility/android_info.h" -#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h" - -namespace tflite { -namespace acceleration { - -// This class recommends best TfLiteGPU delegate options for Android devices. -// -// Example usage: -// tflite::Interpreter* interpreter = ... ; -// tflite::acceleration::AndroidInfo android_info; -// tflite::gpu::GpuInfo gpu_info; -// CHECK(tflite::acceleration::RequestAndroidInfo(&android_info)); -// CHECK(tflite::gpu::gl::EglEnvironment::NewEglEnvironment(&env)); -// CHECK(tflite::gpu::gl::RequestGpuInfo(&tflite_gpu_info)); -// tflite::acceleration::GPUCompatibilityRecommender recommender; -// TfLiteDelegate* gpu_delegate = nullptr; -// TfLiteGpuDelegateOptions gpu_options; -// if (list.Includes(android_info, gpu_info)) { -// gpu_options = recommender.BestOptionsFor(android_info, gpu_info); -// gpu_delegate = TfLiteGpuDelegateCreate(&gpu_options); -// CHECK_EQ(interpreter->ModifyGraphWithDelegate(gpu_delegate), TfLiteOk); -// } else { -// // Fallback path. -// } - -class GPUCompatibilityRecommender : public GPUCompatibilityList { - public: - GPUCompatibilityRecommender() {} - GPUCompatibilityRecommender(const GPUCompatibilityRecommender&) = delete; - GPUCompatibilityRecommender& operator=(const GPUCompatibilityRecommender&) = - delete; - - // Returns the best TfLiteGpuDelegateOptionsV2 for the provided device specs - // based on the database. The output can be modified as desired before passing - // to delegate creation. - TfLiteGpuDelegateOptionsV2 GetBestOptionsFor( - const AndroidInfo& android_info, - const ::tflite::gpu::GpuInfo& gpu_info) const; -}; - -} // namespace acceleration -} // namespace tflite - -#endif // TENSORFLOW_LITE_EXPERIMENTAL_ACCELERATION_COMPATIBILITY_GPU_COMPATIBILITY_RECOMMENDER_H_ diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender_test.cc b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender_test.cc deleted file mode 100644 index ebf793d5a94..00000000000 --- a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender_test.cc +++ /dev/null @@ -1,100 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_recommender.h" - -#include -#include - -namespace { - -class GPUCompatibilityRecommenderTest : public ::testing::Test { - protected: - GPUCompatibilityRecommenderTest() { - recommender_ = - absl::make_unique(); - } - - std::unique_ptr - recommender_; -}; - -TEST_F(GPUCompatibilityRecommenderTest, Load) { - EXPECT_TRUE(recommender_->IsDatabaseLoaded()); -} - -TEST_F(GPUCompatibilityRecommenderTest, ReturnsSupportedForFullMatch) { - tflite::acceleration::AndroidInfo android_info = { - .android_sdk_version = "28", - .model = "redmi_note_7G960F", - .device = "lavender", - .manufacturer = "xiaomi"}; - tflite::gpu::GpuInfo tflite_gpu_info = { - .renderer_name = "adreno_(tm)_512", - .major_version = 3, - .minor_version = 2, - }; - EXPECT_TRUE(recommender_->Includes(android_info, tflite_gpu_info)); -} - -TEST_F(GPUCompatibilityRecommenderTest, ReturnsUnsupported) { - tflite::acceleration::AndroidInfo android_info = {.android_sdk_version = "28", - .model = "sm_g960f", - .device = "starlte", - .manufacturer = "samsung"}; - tflite::gpu::GpuInfo tflite_gpu_info = { - .renderer_name = "mali_g72", - .major_version = 3, - .minor_version = 2, - }; - - EXPECT_FALSE(recommender_->Includes(android_info, tflite_gpu_info)); -} - -TEST_F(GPUCompatibilityRecommenderTest, MissingInfoReturnsUnsupported) { - tflite::acceleration::AndroidInfo android_info = {.android_sdk_version = "23", - .model = "sm_g532f", - .device = "grandpplte", - .manufacturer = "samsung"}; - tflite::gpu::GpuInfo tflite_gpu_info = { - .renderer_name = "mali_t720", - .major_version = 3, - .minor_version = 1, - }; - EXPECT_FALSE(recommender_->Includes(android_info, tflite_gpu_info)); -} - -TEST_F(GPUCompatibilityRecommenderTest, ReturnsDefaultOptions) { - tflite::acceleration::AndroidInfo android_info; - tflite::gpu::GpuInfo tflite_gpu_info; - auto default_options = TfLiteGpuDelegateOptionsV2Default(); - auto best_options = - recommender_->GetBestOptionsFor(android_info, tflite_gpu_info); - EXPECT_EQ(best_options.is_precision_loss_allowed, - default_options.is_precision_loss_allowed); - EXPECT_EQ(best_options.inference_preference, - default_options.inference_preference); - EXPECT_EQ(best_options.inference_priority1, - default_options.inference_priority1); - EXPECT_EQ(best_options.inference_priority2, - default_options.inference_priority2); - EXPECT_EQ(best_options.inference_priority3, - default_options.inference_priority3); - EXPECT_EQ(best_options.experimental_flags, - default_options.experimental_flags); - EXPECT_EQ(best_options.max_delegated_partitions, - default_options.max_delegated_partitions); -} - -} // namespace diff --git a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_test.cc b/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_test.cc deleted file mode 100644 index d300867a8b0..00000000000 --- a/tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility_test.cc +++ /dev/null @@ -1,61 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/lite/experimental/acceleration/compatibility/gpu_compatibility.h" - -#include - -#include -#include - -namespace { - -class GPUCompatibilityTest : public ::testing::Test { - protected: - GPUCompatibilityTest() { - list_ = absl::make_unique(); - } - - std::unique_ptr list_; -}; - -TEST_F(GPUCompatibilityTest, Load) { EXPECT_TRUE(list_->IsDatabaseLoaded()); } - -TEST_F(GPUCompatibilityTest, ReturnsSupportedForFullMatch) { - tflite::acceleration::AndroidInfo android_info = {.android_sdk_version = "27", - .model = "cph1803", - .device = "cph1803", - .manufacturer = "Oppo"}; - tflite::gpu::GpuInfo tflite_gpu_info = { - .renderer_name = "Adreno (TM) 506", - .major_version = 3, - .minor_version = 2, - }; - EXPECT_TRUE(list_->Includes(android_info, tflite_gpu_info)); -} - -TEST_F(GPUCompatibilityTest, ReturnsUnsupportedForFullMatch) { - tflite::acceleration::AndroidInfo android_info = {.android_sdk_version = "28", - .model = "SM-G960F", - .device = "starlte", - .manufacturer = "Samsung"}; - tflite::gpu::GpuInfo tflite_gpu_info = { - .renderer_name = "Mali-G72", - .major_version = 3, - .minor_version = 2, - }; - EXPECT_FALSE(list_->Includes(android_info, tflite_gpu_info)); -} - -} // namespace From d38f00677fff3e7060789de8fea5a07018cca7fe Mon Sep 17 00:00:00 2001 From: Pavithra Vijay Date: Tue, 4 Aug 2020 21:32:32 -0700 Subject: [PATCH 0431/1017] Add `is_control_flow_graph` property to `CondBranchFuncGraph`. PiperOrigin-RevId: 324952128 Change-Id: Ie75edbb09c366f20c913868834af3626bf403249 --- tensorflow/python/framework/func_graph.py | 1 + .../python/kernel_tests/cond_v2_test.py | 24 +++++++++++++++++++ .../python/kernel_tests/while_v2_test.py | 20 ++++++++++++++++ .../python/ops/control_flow_v2_func_graphs.py | 3 +++ 4 files changed, 48 insertions(+) diff --git a/tensorflow/python/framework/func_graph.py b/tensorflow/python/framework/func_graph.py index 55508c4803b..dbe0d57759b 100644 --- a/tensorflow/python/framework/func_graph.py +++ b/tensorflow/python/framework/func_graph.py @@ -192,6 +192,7 @@ class FuncGraph(ops.Graph): self.structured_outputs = None self._weak_variables = [] self._watched_variables = object_identity.ObjectIdentityWeakSet() + self.is_control_flow_graph = False outer_graph = ops.get_default_graph() self._weak_outer_graph = weakref.ref(outer_graph) diff --git a/tensorflow/python/kernel_tests/cond_v2_test.py b/tensorflow/python/kernel_tests/cond_v2_test.py index c64b608a253..b8829181747 100644 --- a/tensorflow/python/kernel_tests/cond_v2_test.py +++ b/tensorflow/python/kernel_tests/cond_v2_test.py @@ -1237,6 +1237,30 @@ class CondV2Test(test.TestCase): self.assertEqual(len(if_op.outputs), 1) # pylint: enable=g-deprecated-assert + def testIsControlFlowGraph(self): + x = constant_op.constant(1.0, name="x") + + @def_function.function + def f(c): + + def then_branch(): + i = x + 1 + self.assertTrue(i.graph.is_control_flow_graph) + return i + + def else_branch(): + i = x + 1 + self.assertTrue(i.graph.is_control_flow_graph) + return i + + return cond_v2.cond_v2(c, then_branch, else_branch) + + i = f(constant_op.constant(True)) + self.assertEqual(self.evaluate(i), 2.0) + + i = f(constant_op.constant(False)) + self.assertEqual(self.evaluate(i), 2.0) + class CondV2CollectionTest(test.TestCase): diff --git a/tensorflow/python/kernel_tests/while_v2_test.py b/tensorflow/python/kernel_tests/while_v2_test.py index e829edb0dfc..de2e8e3cc8d 100644 --- a/tensorflow/python/kernel_tests/while_v2_test.py +++ b/tensorflow/python/kernel_tests/while_v2_test.py @@ -1241,6 +1241,26 @@ class WhileV2Test(test.TestCase, parameterized.TestCase): config.experimental.executor_type = "SINGLE_THREADED_EXECUTOR" self._runBasicWithConfig(config) + def testIsControlFlowGraph(self): + x = constant_op.constant(0) + + @def_function.function + def F(c): + + def Cond(i): + self.assertTrue(i.graph.is_control_flow_graph) + return i < 2 + + def Body(i): + i = i + 1 + self.assertTrue(i.graph.is_control_flow_graph) + return i + + return while_loop_v2(Cond, Body, [c]) + + ret, = F(x) + self.assertEqual(2, self.evaluate(ret)) + def testImportFromSerializedWithFunctionInBody(self): serialized = """node { name: "Const" diff --git a/tensorflow/python/ops/control_flow_v2_func_graphs.py b/tensorflow/python/ops/control_flow_v2_func_graphs.py index 97e04f8d73d..23edd712797 100644 --- a/tensorflow/python/ops/control_flow_v2_func_graphs.py +++ b/tensorflow/python/ops/control_flow_v2_func_graphs.py @@ -30,6 +30,7 @@ class CondBranchFuncGraph(func_graph.FuncGraph): def __init__(self, *args, **kwargs): super(CondBranchFuncGraph, self).__init__(*args, **kwargs) + self.is_control_flow_graph = True if ops.executing_eagerly_outside_functions(): func_graph.override_func_graph_name_scope( self, self.outer_graph.get_name_scope()) @@ -43,6 +44,7 @@ class WhileCondFuncGraph(func_graph.FuncGraph): def __init__(self, *args, **kwargs): super(WhileCondFuncGraph, self).__init__(*args, **kwargs) + self.is_control_flow_graph = True if ops.executing_eagerly_outside_functions(): func_graph.override_func_graph_name_scope( self, self.outer_graph.get_name_scope()) @@ -56,6 +58,7 @@ class WhileBodyFuncGraph(func_graph.FuncGraph): def __init__(self, *args, **kwargs): super(WhileBodyFuncGraph, self).__init__(*args, **kwargs) + self.is_control_flow_graph = True if ops.executing_eagerly_outside_functions(): func_graph.override_func_graph_name_scope( self, self.outer_graph.get_name_scope()) From 54312c0e8fd3c2f2d2f3ee4812c8b0078fb24a8c Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 5 Aug 2020 05:06:49 +0000 Subject: [PATCH 0432/1017] fix build --- tensorflow/core/kernels/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 47c7d41d0fe..dd51d6fcc26 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2946,7 +2946,9 @@ cc_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", + "//tensorflow/core/framework:tensor", "//tensorflow/core/framework:tensor_shape_proto_cc", + "//tensorflow/core/framework:variant", "//tensorflow/core/lib/core:refcount", ], ) From 267aebacb555bf862bf1cb503f3af289cccd1d27 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Aug 2020 22:30:46 -0700 Subject: [PATCH 0433/1017] Integrate LLVM at llvm/llvm-project@28e322ea9393 Updates LLVM usage to match [28e322ea9393](https://github.com/llvm/llvm-project/commit/28e322ea9393) PiperOrigin-RevId: 324957510 Change-Id: I9199f451363a62e89a4237cfd6ff44e90ea9768c --- .../mlir/hlo/tests/lhlo-legalize-to-llvm.mlir | 8 +-- .../gpu/tests/execute_memzero_thunk.mlir | 6 +- .../service/mlir_gpu/tests/add_as_kernel.hlo | 56 +++++++++---------- tensorflow/workspace.bzl | 4 +- third_party/mlir/BUILD | 8 +++ third_party/mlir/test.BUILD | 27 +++++++++ 6 files changed, 72 insertions(+), 37 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/tests/lhlo-legalize-to-llvm.mlir b/tensorflow/compiler/mlir/hlo/tests/lhlo-legalize-to-llvm.mlir index a25a508b2d3..5bb1d475b24 100644 --- a/tensorflow/compiler/mlir/hlo/tests/lhlo-legalize-to-llvm.mlir +++ b/tensorflow/compiler/mlir/hlo/tests/lhlo-legalize-to-llvm.mlir @@ -11,11 +11,11 @@ func @static_memref_cast(%buf : memref<10x1x5xf32>) { // CHECK: %[[MEMREF_BLDR_0:.*]] = llvm.mlir.undef : [[DESCRIPTOR_TYPE_2D:!.*]] // CHECK: %[[IN_PTR:.*]] = llvm.extractvalue %[[INPUT_MEMREF:.*]][0] : [[DESCRIPTOR_TYPE_3D]] -// CHECK: %[[PTR:.*]] = llvm.bitcast %[[IN_PTR]] : !llvm<"float*"> to !llvm<"float*"> +// CHECK: %[[PTR:.*]] = llvm.bitcast %[[IN_PTR]] : !llvm.ptr to !llvm.ptr // CHECK: %[[MEMREF_BLDR_1:.*]] = llvm.insertvalue %[[PTR]], %[[MEMREF_BLDR_0]][0] : [[DESCRIPTOR_TYPE_2D]] // CHECK: %[[IN_ALIGNED_PTR:.*]] = llvm.extractvalue %[[INPUT_MEMREF]][1] : [[DESCRIPTOR_TYPE_3D]] -// CHECK: %[[ALIGNED_PTR:.*]] = llvm.bitcast %[[IN_ALIGNED_PTR]] : !llvm<"float*"> to !llvm<"float*"> +// CHECK: %[[ALIGNED_PTR:.*]] = llvm.bitcast %[[IN_ALIGNED_PTR]] : !llvm.ptr to !llvm.ptr // CHECK: %[[MEMREF_BLDR_2:.*]] = llvm.insertvalue %[[ALIGNED_PTR]], %[[MEMREF_BLDR_1]][1] : [[DESCRIPTOR_TYPE_2D]] // CHECK: %[[C2:.*]] = llvm.mlir.constant(2 : index) : !llvm.i64 @@ -50,11 +50,11 @@ func @dynamic_memref_cast(%buf : memref) { // CHECK: %[[MEMREF_BLDR_0:.*]] = llvm.mlir.undef : [[DESCRIPTOR_TYPE:!.*]] // CHECK: %[[IN_PTR:.*]] = llvm.extractvalue %[[INPUT_MEMREF:.*]][0] : [[DESCRIPTOR_TYPE]] -// CHECK: %[[PTR:.*]] = llvm.bitcast %[[IN_PTR]] : !llvm<"float*"> to !llvm<"float*"> +// CHECK: %[[PTR:.*]] = llvm.bitcast %[[IN_PTR]] : !llvm.ptr to !llvm.ptr // CHECK: %[[MEMREF_BLDR_1:.*]] = llvm.insertvalue %[[PTR]], %[[MEMREF_BLDR_0]][0] : [[DESCRIPTOR_TYPE]] // CHECK: %[[IN_ALIGNED_PTR:.*]] = llvm.extractvalue %[[INPUT_MEMREF]][1] : [[DESCRIPTOR_TYPE]] -// CHECK: %[[ALIGNED_PTR:.*]] = llvm.bitcast %[[IN_ALIGNED_PTR]] : !llvm<"float*"> to !llvm<"float*"> +// CHECK: %[[ALIGNED_PTR:.*]] = llvm.bitcast %[[IN_ALIGNED_PTR]] : !llvm.ptr to !llvm.ptr // CHECK: %[[MEMREF_BLDR_2:.*]] = llvm.insertvalue %[[ALIGNED_PTR]], %[[MEMREF_BLDR_1]][1] : [[DESCRIPTOR_TYPE]] // CHECK: %[[SRC_OFFSET:.*]] = llvm.extractvalue %[[INPUT_MEMREF]][2] : [[DESCRIPTOR_TYPE]] diff --git a/tensorflow/compiler/xla/service/gpu/tests/execute_memzero_thunk.mlir b/tensorflow/compiler/xla/service/gpu/tests/execute_memzero_thunk.mlir index 0a891833cd3..82f3f06db5c 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/execute_memzero_thunk.mlir +++ b/tensorflow/compiler/xla/service/gpu/tests/execute_memzero_thunk.mlir @@ -1,15 +1,15 @@ // RUN: xla-thunks-opt %s | FileCheck --color --dump-input=fail %s -func @main( %execute_params: !llvm<"i8*"> ) { +func @main( %execute_params: !llvm.ptr ) { // CHECK: "xla_thunks.execute_memzero_thunk" // CHECK-SAME: {allocation_index = 0 : i64, offset = 128 : i64, size = 1024 : i64} - // CHECK-SAME: (!llvm<"i8*">) -> (i1, !llvm<"i8*">) + // CHECK-SAME: (!llvm.ptr) -> (i1, !llvm.ptr) %ok, %error_message = "xla_thunks.execute_memzero_thunk"( %execute_params ) { allocation_slice = { allocation_index = 0 , offset = 128 , size = 1024 } } - : (!llvm<"i8*">) -> (i1, !llvm<"i8*">) + : (!llvm.ptr) -> (i1, !llvm.ptr) return } diff --git a/tensorflow/compiler/xla/service/mlir_gpu/tests/add_as_kernel.hlo b/tensorflow/compiler/xla/service/mlir_gpu/tests/add_as_kernel.hlo index 953eb2022f8..8d7930ea8c0 100644 --- a/tensorflow/compiler/xla/service/mlir_gpu/tests/add_as_kernel.hlo +++ b/tensorflow/compiler/xla/service/mlir_gpu/tests/add_as_kernel.hlo @@ -7,24 +7,24 @@ ENTRY %Add (x: f32[2,2], y: f32[2,2]) -> f32[2,2] { ROOT %add = f32[2,2]{1,0} add(f32[2,2]{1,0} %x, f32[2,2]{1,0} %y) } -// CHECK: func @add_kernel(%[[ARG0:.*]]: [[TYPE:!llvm<.*]], %[[ARG1:.*]]: [[TYPE]], %[[ARG2:.*]]: [[TYPE]] +// CHECK: func @add_kernel(%[[ARG0:.*]]: [[TYPE:!llvm\..*]], %[[ARG1:.*]]: [[TYPE]], %[[ARG2:.*]]: [[TYPE]] // // Check that relevant sizes and strides are emitted. // -// CHECK: %[[CAST0:.*]] = llvm.bitcast %[[ARG0:.*]] : !llvm<"i8*"> to !llvm<"float*"> +// CHECK: %[[CAST0:.*]] = llvm.bitcast %[[ARG0:.*]] : !llvm.ptr to !llvm.ptr // CHECK: %[[SIZE00:.*]] = llvm.mlir.constant(2 : i64) : !llvm.i64 // CHECK: %[[SIZE01:.*]] = llvm.mlir.constant(2 : i64) : !llvm.i64 // CHECK: %[[STRIDE01:.*]] = llvm.mlir.constant(1 : i64) : !llvm.i64 // CHECK: %[[STRIDE00:.*]] = llvm.mlir.constant(2 : i64) : !llvm.i64 -// CHECK: %[[CAST1:.*]] = llvm.bitcast %[[ARG1:.*]] : !llvm<"i8*"> to !llvm<"float*"> +// CHECK: %[[CAST1:.*]] = llvm.bitcast %[[ARG1:.*]] : !llvm.ptr to !llvm.ptr // CHECK: %[[SIZE10:.*]] = llvm.mlir.constant(2 : i64) : !llvm.i64 // CHECK: %[[SIZE11:.*]] = llvm.mlir.constant(2 : i64) : !llvm.i64 // CHECK: %[[STRIDE11:.*]] = llvm.mlir.constant(1 : i64) : !llvm.i64 // CHECK: %[[STRIDE10:.*]] = llvm.mlir.constant(2 : i64) : !llvm.i64 -// CHECK: %[[CAST2:.*]] = llvm.bitcast %[[ARG2:.*]] : !llvm<"i8*"> to !llvm<"float*"> +// CHECK: %[[CAST2:.*]] = llvm.bitcast %[[ARG2:.*]] : !llvm.ptr to !llvm.ptr // CHECK: %[[SIZE20:.*]] = llvm.mlir.constant(2 : i64) : !llvm.i64 // CHECK: %[[SIZE21:.*]] = llvm.mlir.constant(2 : i64) : !llvm.i64 // CHECK: %[[STRIDE21:.*]] = llvm.mlir.constant(1 : i64) : !llvm.i64 @@ -34,30 +34,30 @@ ENTRY %Add (x: f32[2,2], y: f32[2,2]) -> f32[2,2] { // Check that the emitted sizes and strides, as well the pointers to HLO buffers, // are inserted into the memref descriptors. // -// CHECK: %[[DESC0:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC01:.*]] = llvm.insertvalue %[[CAST0]], %[[DESC0]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC02:.*]] = llvm.insertvalue %[[CAST0]], %[[DESC01]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC03:.*]] = llvm.insertvalue %{{.*}}, %[[DESC02]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC04:.*]] = llvm.insertvalue %[[SIZE00]], %[[DESC03]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC05:.*]] = llvm.insertvalue %[[STRIDE00]], %[[DESC04]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC06:.*]] = llvm.insertvalue %[[SIZE01]], %[[DESC05]][3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %{{.*}} = llvm.insertvalue %[[STRIDE01]], %[[DESC06]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> +// CHECK: %[[DESC0:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC01:.*]] = llvm.insertvalue %[[CAST0]], %[[DESC0]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC02:.*]] = llvm.insertvalue %[[CAST0]], %[[DESC01]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC03:.*]] = llvm.insertvalue %{{.*}}, %[[DESC02]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC04:.*]] = llvm.insertvalue %[[SIZE00]], %[[DESC03]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC05:.*]] = llvm.insertvalue %[[STRIDE00]], %[[DESC04]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC06:.*]] = llvm.insertvalue %[[SIZE01]], %[[DESC05]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %{{.*}} = llvm.insertvalue %[[STRIDE01]], %[[DESC06]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC1:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC11:.*]] = llvm.insertvalue %[[CAST1]], %[[DESC1]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC12:.*]] = llvm.insertvalue %[[CAST1]], %[[DESC11]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC13:.*]] = llvm.insertvalue %{{.*}}, %[[DESC12]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC14:.*]] = llvm.insertvalue %[[SIZE10]], %[[DESC13]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC15:.*]] = llvm.insertvalue %[[STRIDE10]], %[[DESC14]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC16:.*]] = llvm.insertvalue %[[SIZE11]], %[[DESC15]][3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %{{.*}} = llvm.insertvalue %[[STRIDE11]], %[[DESC16]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> +// CHECK: %[[DESC1:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC11:.*]] = llvm.insertvalue %[[CAST1]], %[[DESC1]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC12:.*]] = llvm.insertvalue %[[CAST1]], %[[DESC11]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC13:.*]] = llvm.insertvalue %{{.*}}, %[[DESC12]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC14:.*]] = llvm.insertvalue %[[SIZE10]], %[[DESC13]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC15:.*]] = llvm.insertvalue %[[STRIDE10]], %[[DESC14]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC16:.*]] = llvm.insertvalue %[[SIZE11]], %[[DESC15]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %{{.*}} = llvm.insertvalue %[[STRIDE11]], %[[DESC16]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> -// CHECK: %[[DESC2:.*]] = llvm.mlir.undef : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC21:.*]] = llvm.insertvalue %[[CAST2]], %[[DESC2]][0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC22:.*]] = llvm.insertvalue %[[CAST2]], %[[DESC21]][1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC23:.*]] = llvm.insertvalue %{{.*}}, %[[DESC22]][2] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC24:.*]] = llvm.insertvalue %[[SIZE20]], %[[DESC23]][3, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC25:.*]] = llvm.insertvalue %[[STRIDE20]], %[[DESC24]][4, 0] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %[[DESC26:.*]] = llvm.insertvalue %[[SIZE21]], %[[DESC25]][3, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> -// CHECK: %{{.*}} = llvm.insertvalue %[[STRIDE21]], %[[DESC26]][4, 1] : !llvm<"{ float*, float*, i64, [2 x i64], [2 x i64] }"> +// CHECK: %[[DESC2:.*]] = llvm.mlir.undef : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC21:.*]] = llvm.insertvalue %[[CAST2]], %[[DESC2]][0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC22:.*]] = llvm.insertvalue %[[CAST2]], %[[DESC21]][1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC23:.*]] = llvm.insertvalue %{{.*}}, %[[DESC22]][2] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC24:.*]] = llvm.insertvalue %[[SIZE20]], %[[DESC23]][3, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC25:.*]] = llvm.insertvalue %[[STRIDE20]], %[[DESC24]][4, 0] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %[[DESC26:.*]] = llvm.insertvalue %[[SIZE21]], %[[DESC25]][3, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> +// CHECK: %{{.*}} = llvm.insertvalue %[[STRIDE21]], %[[DESC26]][4, 1] : !llvm.struct<(ptr, ptr, i64, array<2 x i64>, array<2 x i64>)> diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 29cba080fa1..b770dfeead5 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "b5059b7140232559ed123cb94d4e8f75ca9a44dc" - LLVM_SHA256 = "3075583f88b572da4afb1340281b0e170d51ef03ba6eb2965e7dc8288cbff153" + LLVM_COMMIT = "28e322ea9393e6b3841886006dd170ddd810fd9b" + LLVM_SHA256 = "438268a47b69687ea5e588a285a2255de414addc36e0405e1d70f7cb5208aa75" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), diff --git a/third_party/mlir/BUILD b/third_party/mlir/BUILD index 4f2873af3dd..3941375bc02 100644 --- a/third_party/mlir/BUILD +++ b/third_party/mlir/BUILD @@ -927,6 +927,7 @@ cc_library( ":DialectUtils", ":EDSC", ":IR", + ":LinalgOps", ":SCFDialect", ":SideEffectInterfaces", ":StandardOps", @@ -2368,6 +2369,7 @@ cc_library( ":ConversionPassIncGen", ":IR", ":LLVMDialect", + ":Parser", ":Pass", ":StandardOps", ":Support", @@ -2596,9 +2598,11 @@ cc_library( "lib/Target/LLVMIR/DebugTranslation.cpp", "lib/Target/LLVMIR/DebugTranslation.h", "lib/Target/LLVMIR/ModuleTranslation.cpp", + "lib/Target/LLVMIR/TypeTranslation.cpp", ], hdrs = [ "include/mlir/Target/LLVMIR/ModuleTranslation.h", + "include/mlir/Target/LLVMIR/TypeTranslation.h", ], includes = ["include"], deps = [ @@ -2771,6 +2775,7 @@ cc_library( "@llvm-project//mlir/test:TestAffine", "@llvm-project//mlir/test:TestDialect", "@llvm-project//mlir/test:TestIR", + "@llvm-project//mlir/test:TestLLVMIR", "@llvm-project//mlir/test:TestPass", "@llvm-project//mlir/test:TestReducer", "@llvm-project//mlir/test:TestSPIRV", @@ -2801,6 +2806,7 @@ cc_library( ":Support", ":Translation", "@llvm-project//llvm:Support", + "@llvm-project//mlir/test:TestLLVMTypeTranslation", ], ) @@ -2922,6 +2928,7 @@ cc_binary( "@llvm-project//mlir/test:TestAffine", "@llvm-project//mlir/test:TestDialect", "@llvm-project//mlir/test:TestIR", + "@llvm-project//mlir/test:TestLLVMIR", "@llvm-project//mlir/test:TestPass", "@llvm-project//mlir/test:TestReducer", "@llvm-project//mlir/test:TestSPIRV", @@ -3655,6 +3662,7 @@ cc_library( ":EDSC", ":IR", ":LLVMDialect", + ":LLVMIRModuleTranslation", ":Pass", ":StandardOps", ":StandardToLLVM", diff --git a/third_party/mlir/test.BUILD b/third_party/mlir/test.BUILD index f507842a639..6c4eeecc346 100644 --- a/third_party/mlir/test.BUILD +++ b/third_party/mlir/test.BUILD @@ -165,6 +165,20 @@ cc_library( ], ) +cc_library( + name = "TestLLVMTypeTranslation", + srcs = [ + "lib/Target/TestLLVMTypeTranslation.cpp", + ], + deps = [ + ":TestLLVMIR", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + "@llvm-project//mlir:LLVMIRModuleTranslation", + "@llvm-project//mlir:Translation", + ], +) + cc_library( name = "TestTransforms", srcs = glob(["lib/Transforms/*.cpp"]), @@ -216,6 +230,19 @@ cc_library( ], ) +cc_library( + name = "TestLLVMIR", + srcs = [ + "lib/Dialect/LLVMIR/LLVMTypeTestDialect.cpp", + ], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:Dialect", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:LLVMDialect", + ], +) + cc_library( name = "TestSPIRV", srcs = glob([ From 822eeba7ed45399ec4cdde8d584d5fc997d340bf Mon Sep 17 00:00:00 2001 From: Rick Chao Date: Tue, 4 Aug 2020 22:57:02 -0700 Subject: [PATCH 0434/1017] MultiProcessRunner: When subprocesses time out, prioritize error reporting from subprocesses over the fact that it times out. PiperOrigin-RevId: 324960220 Change-Id: Ic5258466053e01eee78d6021fb1d5680676433d9 --- .../python/distribute/multi_process_runner.py | 12 ++++++++---- .../distribute/multi_process_runner_test.py | 19 ++++++++++++++++++- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/distribute/multi_process_runner.py b/tensorflow/python/distribute/multi_process_runner.py index 4ded663e588..6d0854f18d7 100644 --- a/tensorflow/python/distribute/multi_process_runner.py +++ b/tensorflow/python/distribute/multi_process_runner.py @@ -524,6 +524,12 @@ class MultiProcessRunner(object): if all(p.exitcode is not None for p in self._processes.values()): return + def _reraise_if_subprocess_error(self, process_statuses): + for process_status in process_statuses.values(): + assert isinstance(process_status, _ProcessStatusInfo) + if not process_status.is_successful: + six.reraise(*process_status.exc_info) + def join(self, timeout=_DEFAULT_TIMEOUT_SEC): """Joins all the processes with timeout. @@ -587,6 +593,7 @@ class MultiProcessRunner(object): self.terminate_all() self._watchdog_thread.join() process_statuses = self._get_process_statuses() + self._reraise_if_subprocess_error(process_statuses) raise SubprocessTimeoutError('one or more subprocesses timed out.', self._get_mpr_result(process_statuses)) @@ -594,10 +601,7 @@ class MultiProcessRunner(object): logging.info('%s-%d exit code: %s', task_type, task_id, p.exitcode) process_statuses = self._get_process_statuses() - for process_status in process_statuses.values(): - assert isinstance(process_status, _ProcessStatusInfo) - if not process_status.is_successful: - six.reraise(*process_status.exc_info) + self._reraise_if_subprocess_error(process_statuses) # Checking all the processes that are expected to exit properly. for (task_type, task_id), p in self._processes.items(): diff --git a/tensorflow/python/distribute/multi_process_runner_test.py b/tensorflow/python/distribute/multi_process_runner_test.py index 0aa214d3ca4..b7d8acf55c6 100644 --- a/tensorflow/python/distribute/multi_process_runner_test.py +++ b/tensorflow/python/distribute/multi_process_runner_test.py @@ -414,7 +414,7 @@ class MultiProcessRunnerTest(test.TestCase): multi_worker_test_base.create_cluster_spec(num_workers=1), auto_restart=True) mpr.start() - with self.assertRaises(multi_process_runner.SubprocessTimeoutError): + with self.assertRaises(ValueError): mpr.join(timeout=10) def test_auto_restart_and_chief(self): @@ -478,6 +478,23 @@ class MultiProcessRunnerTest(test.TestCase): mpr.join(timeout=20) self.assertEqual(counter.value, 2) + def test_error_reporting_overrides_timeout_reporting(self): + + def proc_func(): + if self._worker_idx() == 1: + time.sleep(10000) + raise ValueError('Worker 0 errored') + + mpr = multi_process_runner.MultiProcessRunner( + proc_func, + multi_worker_test_base.create_cluster_spec(num_workers=2)) + mpr.start() + + with self.assertRaisesRegex( + ValueError, + 'Worker 0 errored'): + mpr.join(timeout=20) + class MultiProcessPoolRunnerTest(test.TestCase): From e2cba3e0a29c63dc7c255da4bbea034438c59736 Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Fri, 31 Jul 2020 14:22:53 -0700 Subject: [PATCH 0435/1017] Moving some filesystems to Transactional API --- .../filesystem/modular_filesystem.cc | 93 +++++++++-------- .../filesystem/modular_filesystem.h | 99 +++++++------------ .../common_runtime/constant_folding_test.cc | 2 +- .../kernels/immutable_constant_op_test.cc | 2 +- tensorflow/core/platform/env_test.cc | 8 +- tensorflow/core/platform/file_system.h | 8 +- tensorflow/core/platform/file_system_test.cc | 14 +-- tensorflow/core/platform/null_file_system.h | 61 +++++------- tensorflow/core/platform/ram_file_system.h | 67 +++++-------- .../core/platform/retrying_file_system.h | 88 +++++++---------- .../platform/retrying_file_system_test.cc | 73 ++++++-------- .../python/framework/test_file_system.cc | 6 +- 12 files changed, 215 insertions(+), 306 deletions(-) diff --git a/tensorflow/c/experimental/filesystem/modular_filesystem.cc b/tensorflow/c/experimental/filesystem/modular_filesystem.cc index 00a587521fd..9c8d3518800 100644 --- a/tensorflow/c/experimental/filesystem/modular_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/modular_filesystem.cc @@ -35,8 +35,8 @@ using UniquePtrTo_TF_Status = ::std::unique_ptr; Status ModularFileSystem::NewRandomAccessFile( - const std::string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const std::string& fname, TransactionToken* token, + std::unique_ptr* result) { if (ops_->new_random_access_file == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", fname, " does not support NewRandomAccessFile()")); @@ -55,8 +55,8 @@ Status ModularFileSystem::NewRandomAccessFile( } Status ModularFileSystem::NewWritableFile( - const std::string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const std::string& fname, TransactionToken* token, + std::unique_ptr* result) { if (ops_->new_writable_file == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", fname, " does not support NewWritableFile()")); @@ -75,8 +75,8 @@ Status ModularFileSystem::NewWritableFile( } Status ModularFileSystem::NewAppendableFile( - const std::string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const std::string& fname, TransactionToken* token, + std::unique_ptr* result) { if (ops_->new_appendable_file == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", fname, " does not support NewAppendableFile()")); @@ -95,8 +95,8 @@ Status ModularFileSystem::NewAppendableFile( } Status ModularFileSystem::NewReadOnlyMemoryRegionFromFile( - const std::string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const std::string& fname, TransactionToken* token, + std::unique_ptr* result) { if (ops_->new_read_only_memory_region_from_file == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", fname, @@ -116,8 +116,8 @@ Status ModularFileSystem::NewReadOnlyMemoryRegionFromFile( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::FileExists( - const std::string& fname /*, TransactionToken* token */) { +Status ModularFileSystem::FileExists(const std::string& fname, + TransactionToken* token) { if (ops_->path_exists == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", fname, " does not support FileExists()")); @@ -129,9 +129,9 @@ Status ModularFileSystem::FileExists( return StatusFromTF_Status(plugin_status.get()); } -bool ModularFileSystem::FilesExist( - const std::vector& files, - std::vector* status /*, TransactionToken* token */) { +bool ModularFileSystem::FilesExist(const std::vector& files, + TransactionToken* token, + std::vector* status) { if (ops_->paths_exist == nullptr) return FileSystem::FilesExist(files, status); @@ -162,9 +162,9 @@ bool ModularFileSystem::FilesExist( return result; } -Status ModularFileSystem::GetChildren( - const std::string& dir, - std::vector* result /*, TransactionToken* token */) { +Status ModularFileSystem::GetChildren(const std::string& dir, + TransactionToken* token, + std::vector* result) { if (ops_->get_children == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", dir, " does not support GetChildren()")); @@ -188,9 +188,9 @@ Status ModularFileSystem::GetChildren( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::GetMatchingPaths( - const std::string& pattern, - std::vector* result /*, TransactionToken* token */) { +Status ModularFileSystem::GetMatchingPaths(const std::string& pattern, + TransactionToken* token, + std::vector* result) { if (ops_->get_matching_paths == nullptr) return internal::GetMatchingPaths(this, Env::Default(), pattern, result); @@ -211,8 +211,8 @@ Status ModularFileSystem::GetMatchingPaths( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::DeleteFile( - const std::string& fname /*, TransactionToken* token */) { +Status ModularFileSystem::DeleteFile(const std::string& fname, + TransactionToken* token) { if (ops_->delete_file == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", fname, " does not support DeleteFile()")); @@ -224,9 +224,10 @@ Status ModularFileSystem::DeleteFile( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::DeleteRecursively( - const std::string& dirname, int64* undeleted_files, - int64* undeleted_dirs /*, TransactionToken* token */) { +Status ModularFileSystem::DeleteRecursively(const std::string& dirname, + TransactionToken* token, + int64* undeleted_files, + int64* undeleted_dirs) { if (undeleted_files == nullptr || undeleted_dirs == nullptr) return errors::FailedPrecondition( "DeleteRecursively must not be called with `undeleted_files` or " @@ -247,8 +248,8 @@ Status ModularFileSystem::DeleteRecursively( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::DeleteDir( - const std::string& dirname /*, TransactionToken* token */) { +Status ModularFileSystem::DeleteDir(const std::string& dirname, + TransactionToken* token) { if (ops_->delete_dir == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", dirname, " does not support DeleteDir()")); @@ -260,8 +261,8 @@ Status ModularFileSystem::DeleteDir( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::RecursivelyCreateDir( - const std::string& dirname /*, TransactionToken* token */) { +Status ModularFileSystem::RecursivelyCreateDir(const std::string& dirname, + TransactionToken* token) { if (ops_->recursively_create_dir == nullptr) return FileSystem::RecursivelyCreateDir(dirname); @@ -272,8 +273,8 @@ Status ModularFileSystem::RecursivelyCreateDir( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::CreateDir( - const std::string& dirname /*, TransactionToken* token */) { +Status ModularFileSystem::CreateDir(const std::string& dirname, + TransactionToken* token) { if (ops_->create_dir == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", dirname, " does not support CreateDir()")); @@ -285,9 +286,8 @@ Status ModularFileSystem::CreateDir( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::Stat( - const std::string& fname, - FileStatistics* stat /*, TransactionToken* token */) { +Status ModularFileSystem::Stat(const std::string& fname, + TransactionToken* token, FileStatistics* stat) { if (ops_->stat == nullptr) return errors::Unimplemented(tensorflow::strings::StrCat( "Filesystem for ", fname, " does not support Stat()")); @@ -310,8 +310,8 @@ Status ModularFileSystem::Stat( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::IsDirectory( - const std::string& name /*, TransactionToken* token */) { +Status ModularFileSystem::IsDirectory(const std::string& name, + TransactionToken* token) { if (ops_->is_directory == nullptr) return FileSystem::IsDirectory(name); UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus); @@ -321,9 +321,9 @@ Status ModularFileSystem::IsDirectory( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::GetFileSize( - const std::string& fname, - uint64* file_size /*, TransactionToken* token */) { +Status ModularFileSystem::GetFileSize(const std::string& fname, + TransactionToken* token, + uint64* file_size) { if (ops_->get_file_size == nullptr) { FileStatistics stat; Status status = Stat(fname, &stat); @@ -342,9 +342,9 @@ Status ModularFileSystem::GetFileSize( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::RenameFile( - const std::string& src, - const std::string& target /*, TransactionToken* token */) { +Status ModularFileSystem::RenameFile(const std::string& src, + const std::string& target, + TransactionToken* token) { if (ops_->rename_file == nullptr) { Status status = CopyFile(src, target); if (status.ok()) status = DeleteFile(src); @@ -359,9 +359,9 @@ Status ModularFileSystem::RenameFile( return StatusFromTF_Status(plugin_status.get()); } -Status ModularFileSystem::CopyFile( - const std::string& src, - const std::string& target /*, TransactionToken* token */) { +Status ModularFileSystem::CopyFile(const std::string& src, + const std::string& target, + TransactionToken* token) { if (ops_->copy_file == nullptr) return FileSystem::CopyFile(src, target); UniquePtrTo_TF_Status plugin_status(TF_NewStatus(), TF_DeleteStatus); @@ -372,8 +372,7 @@ Status ModularFileSystem::CopyFile( return StatusFromTF_Status(plugin_status.get()); } -std::string ModularFileSystem::TranslateName( - const std::string& name /*, TransactionToken* token */) const { +std::string ModularFileSystem::TranslateName(const std::string& name) const { if (ops_->translate_name == nullptr) return FileSystem::TranslateName(name); char* p = ops_->translate_name(filesystem_.get(), name.c_str()); @@ -385,7 +384,7 @@ std::string ModularFileSystem::TranslateName( return ret; } -void ModularFileSystem::FlushCaches(/*TransactionToken* token*/) { +void ModularFileSystem::FlushCaches(TransactionToken* token) { if (ops_->flush_caches != nullptr) ops_->flush_caches(filesystem_.get()); } diff --git a/tensorflow/c/experimental/filesystem/modular_filesystem.h b/tensorflow/c/experimental/filesystem/modular_filesystem.h index a2639152eff..6495d97ebf1 100644 --- a/tensorflow/c/experimental/filesystem/modular_filesystem.h +++ b/tensorflow/c/experimental/filesystem/modular_filesystem.h @@ -60,70 +60,45 @@ class ModularFileSystem final : public FileSystem { ~ModularFileSystem() override { ops_->cleanup(filesystem_.get()); } Status NewRandomAccessFile( - const std::string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; - Status NewWritableFile( - const std::string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; - Status NewAppendableFile( - const std::string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const std::string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status NewWritableFile(const std::string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status NewAppendableFile(const std::string& fname, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const std::string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; - Status FileExists( - const std::string& fname /*, TransactionToken* token = nullptr */) - override; + const std::string& fname, TransactionToken* token, + std::unique_ptr* result) override; + Status FileExists(const std::string& fname, TransactionToken* token) override; bool FilesExist(const std::vector& files, - std::vector* - status /*, TransactionToken* token = nullptr */) override; - Status GetChildren( - const std::string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override; - Status GetMatchingPaths( - const std::string& pattern, - std::vector* - results /*, TransactionToken* token = nullptr */) override; - Status DeleteFile( - const std::string& fname /*, TransactionToken* token = nullptr */) - override; - Status DeleteRecursively( - const std::string& dirname, int64* undeleted_files, - int64* undeleted_dirs /*, TransactionToken* token = nullptr */) override; - Status DeleteDir( - const std::string& dirname /*, TransactionToken* token = nullptr */) - override; - Status RecursivelyCreateDir( - const std::string& dirname /*, TransactionToken* token = nullptr */) - override; - Status CreateDir( - const std::string& dirname /*, TransactionToken* token = nullptr */) - override; - Status Stat( - const std::string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override; - Status IsDirectory( - const std::string& fname /*, TransactionToken* token = nullptr */) - override; - Status GetFileSize( - const std::string& fname, - uint64* file_size /*, TransactionToken* token = nullptr */) override; - Status RenameFile( - const std::string& src, - const std::string& target /*, TransactionToken* token = nullptr */) - override; - Status CopyFile(const std::string& src, - const std::string& - target /*, TransactionToken* token = nullptr */) override; - std::string TranslateName( - const std::string& name /*, TransactionToken* token = nullptr */) - const override; - void FlushCaches(/* TransactionToken* token=nullptr */) override; + TransactionToken* token, + std::vector* status) override; + Status GetChildren(const std::string& dir, TransactionToken* token, + std::vector* result) override; + Status GetMatchingPaths(const std::string& pattern, TransactionToken* token, + std::vector* results) override; + Status DeleteFile(const std::string& fname, TransactionToken* token) override; + Status DeleteRecursively(const std::string& dirname, TransactionToken* token, + int64* undeleted_files, + int64* undeleted_dirs) override; + Status DeleteDir(const std::string& dirname, + TransactionToken* token) override; + Status RecursivelyCreateDir(const std::string& dirname, + TransactionToken* token) override; + Status CreateDir(const std::string& dirname, + TransactionToken* token) override; + Status Stat(const std::string& fname, TransactionToken* token, + FileStatistics* stat) override; + Status IsDirectory(const std::string& fname, + TransactionToken* token) override; + Status GetFileSize(const std::string& fname, TransactionToken* token, + uint64* file_size) override; + Status RenameFile(const std::string& src, const std::string& target, + TransactionToken* token) override; + Status CopyFile(const std::string& src, const std::string& target, + TransactionToken* token) override; + std::string TranslateName(const std::string& name) const override; + void FlushCaches(TransactionToken* token) override; private: std::unique_ptr filesystem_; diff --git a/tensorflow/core/common_runtime/constant_folding_test.cc b/tensorflow/core/common_runtime/constant_folding_test.cc index e621b3b5006..b348117bb9e 100644 --- a/tensorflow/core/common_runtime/constant_folding_test.cc +++ b/tensorflow/core/common_runtime/constant_folding_test.cc @@ -688,7 +688,7 @@ class TestTFFileSystem : public ::tensorflow::NullFileSystem { data_tensor_(test::AsTensor({1., 2., 3., 4.}, {2, 2})) {} ::tensorflow::Status NewReadOnlyMemoryRegionFromFile( - const string& fname, + const string& fname, ::tensorflow::TransactionToken* token, std::unique_ptr<::tensorflow::ReadOnlyMemoryRegion>* result) override { if (fname != kTestMemRegionName) { return ::tensorflow::errors::Unimplemented( diff --git a/tensorflow/core/kernels/immutable_constant_op_test.cc b/tensorflow/core/kernels/immutable_constant_op_test.cc index 7eceba7ad8b..5c3f96a312d 100644 --- a/tensorflow/core/kernels/immutable_constant_op_test.cc +++ b/tensorflow/core/kernels/immutable_constant_op_test.cc @@ -61,7 +61,7 @@ class TestFileSystem : public NullFileSystem { public: ~TestFileSystem() override = default; Status NewReadOnlyMemoryRegionFromFile( - const string& fname, + const string& fname, TransactionToken* token, std::unique_ptr* result) override { float val = 0; StringPiece scheme, host, path; diff --git a/tensorflow/core/platform/env_test.cc b/tensorflow/core/platform/env_test.cc index f013aff9703..35374d65ee3 100644 --- a/tensorflow/core/platform/env_test.cc +++ b/tensorflow/core/platform/env_test.cc @@ -295,7 +295,7 @@ TEST_F(DefaultEnvTest, SleepForMicroseconds) { class TmpDirFileSystem : public NullFileSystem { public: - Status FileExists(const string& dir) override { + Status FileExists(const string& dir,TransactionToken* token) override { StringPiece scheme, host, path; io::ParseURI(dir, &scheme, &host, &path); if (path.empty()) return errors::NotFound(dir, " not found"); @@ -311,7 +311,7 @@ class TmpDirFileSystem : public NullFileSystem { return Env::Default()->FileExists(io::JoinPath(BaseDir(), path)); } - Status CreateDir(const string& dir) override { + Status CreateDir(const string& dir,TransactionToken* token) override { StringPiece scheme, host, path; io::ParseURI(dir, &scheme, &host, &path); if (scheme != "tmpdirfs") { @@ -328,7 +328,7 @@ class TmpDirFileSystem : public NullFileSystem { return status; } - Status IsDirectory(const string& dir) override { + Status IsDirectory(const string& dir,TransactionToken* token) override { StringPiece scheme, host, path; io::ParseURI(dir, &scheme, &host, &path); for (const auto& existing_dir : created_directories_) @@ -336,7 +336,7 @@ class TmpDirFileSystem : public NullFileSystem { return errors::NotFound(dir, " not found"); } - void FlushCaches() override { flushed_ = true; } + void FlushCaches(TransactionToken* token) override { flushed_ = true; } private: bool flushed_ = false; diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h index 4a8d9e63023..c4094d3a5a2 100644 --- a/tensorflow/core/platform/file_system.h +++ b/tensorflow/core/platform/file_system.h @@ -481,7 +481,7 @@ class FileSystem { /// \brief Starts a new transaction virtual tensorflow::Status StartTransaction(TransactionToken** token) { - token = nullptr; + *token = nullptr; return Status::OK(); } @@ -499,15 +499,15 @@ class FileSystem { /// \brief Get token for `path` or start a new transaction and add `path` to /// it. virtual tensorflow::Status GetTokenOrStartTransaction( - const std::string& path, TransactionToken** token) { - token = nullptr; + const string& path, TransactionToken** token) { + *token = nullptr; return Status::OK(); } /// \brief Return transaction for `path` or nullptr in `token` virtual tensorflow::Status GetTransactionForPath(const std::string& path, TransactionToken** token) { - token = nullptr; + *token = nullptr; return Status::OK(); } diff --git a/tensorflow/core/platform/file_system_test.cc b/tensorflow/core/platform/file_system_test.cc index 0af45185612..dd02da32073 100644 --- a/tensorflow/core/platform/file_system_test.cc +++ b/tensorflow/core/platform/file_system_test.cc @@ -32,7 +32,7 @@ static const char* const kPrefix = "ipfs://solarsystem"; // cannot have children further. class InterPlanetaryFileSystem : public NullFileSystem { public: - Status FileExists(const string& fname) override { + Status FileExists(const string& fname, TransactionToken* token) override { string parsed_path; ParsePath(fname, &parsed_path); if (BodyExists(parsed_path)) { @@ -42,7 +42,7 @@ class InterPlanetaryFileSystem : public NullFileSystem { } // Adds the dir to the parent's children list and creates an entry for itself. - Status CreateDir(const string& dirname) override { + Status CreateDir(const string& dirname, TransactionToken* token) override { string parsed_path; ParsePath(dirname, &parsed_path); // If the directory already exists, throw an error. @@ -88,7 +88,7 @@ class InterPlanetaryFileSystem : public NullFileSystem { return Status(tensorflow::error::FAILED_PRECONDITION, "Failed to create"); } - Status IsDirectory(const string& dirname) override { + Status IsDirectory(const string& dirname, TransactionToken* token) override { string parsed_path; ParsePath(dirname, &parsed_path); // Simulate evil_directory has bad permissions by throwing a LOG(FATAL) @@ -105,7 +105,8 @@ class InterPlanetaryFileSystem : public NullFileSystem { return Status(tensorflow::error::FAILED_PRECONDITION, "Not a dir"); } - Status GetChildren(const string& dir, std::vector* result) override { + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override { TF_RETURN_IF_ERROR(IsDirectory(dir)); string parsed_path; ParsePath(dir, &parsed_path); @@ -273,7 +274,7 @@ TEST(InterPlanetaryFileSystemTest, HasAtomicMove) { class TestFileSystem : public NullFileSystem { public: // Only allow for a single root directory. - Status IsDirectory(const string& dirname) override { + Status IsDirectory(const string& dirname, TransactionToken* token) override { if (dirname == "." || dirname.empty()) { return Status::OK(); } @@ -281,7 +282,8 @@ class TestFileSystem : public NullFileSystem { } // Simulating a FS with a root dir and a single file underneath it. - Status GetChildren(const string& dir, std::vector* result) override { + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override { if (dir == "." || dir.empty()) { result->push_back("test"); } diff --git a/tensorflow/core/platform/null_file_system.h b/tensorflow/core/platform/null_file_system.h index ef8879090e9..0af34258169 100644 --- a/tensorflow/core/platform/null_file_system.h +++ b/tensorflow/core/platform/null_file_system.h @@ -37,83 +37,66 @@ class NullFileSystem : public FileSystem { ~NullFileSystem() override = default; Status NewRandomAccessFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + const string& fname, TransactionToken* token, + std::unique_ptr* result) override { return errors::Unimplemented("NewRandomAccessFile unimplemented"); } - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override { return errors::Unimplemented("NewWritableFile unimplemented"); } - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override { return errors::Unimplemented("NewAppendableFile unimplemented"); } Status NewReadOnlyMemoryRegionFromFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + const string& fname, TransactionToken* token, + std::unique_ptr* result) override { return errors::Unimplemented( "NewReadOnlyMemoryRegionFromFile unimplemented"); } - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override { + Status FileExists(const string& fname, TransactionToken* token) override { return errors::Unimplemented("FileExists unimplemented"); } - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override { + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override { return errors::Unimplemented("GetChildren unimplemented"); } - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr */) - override { + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override { return internal::GetMatchingPaths(this, Env::Default(), pattern, results); } - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override { + Status DeleteFile(const string& fname, TransactionToken* token) override { return errors::Unimplemented("DeleteFile unimplemented"); } - Status CreateDir( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status CreateDir(const string& dirname, TransactionToken* token) override { return errors::Unimplemented("CreateDir unimplemented"); } - Status DeleteDir( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status DeleteDir(const string& dirname, TransactionToken* token) override { return errors::Unimplemented("DeleteDir unimplemented"); } - Status GetFileSize( - const string& fname, - uint64* file_size /*, TransactionToken* token = nullptr */) override { + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* file_size) override { return errors::Unimplemented("GetFileSize unimplemented"); } - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override { + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override { return errors::Unimplemented("RenameFile unimplemented"); } - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override { + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override { return errors::Unimplemented("Stat unimplemented"); } }; diff --git a/tensorflow/core/platform/ram_file_system.h b/tensorflow/core/platform/ram_file_system.h index 9437a7174a9..ba8bb2d7630 100644 --- a/tensorflow/core/platform/ram_file_system.h +++ b/tensorflow/core/platform/ram_file_system.h @@ -104,9 +104,8 @@ class RamRandomAccessFile : public RandomAccessFile, public WritableFile { class RamFileSystem : public FileSystem { public: Status NewRandomAccessFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + const string& fname, TransactionToken* token, + std::unique_ptr* result) override { mutex_lock m(mu_); if (fs_.find(fname) == fs_.end()) { return errors::NotFound(""); @@ -116,10 +115,8 @@ class RamFileSystem : public FileSystem { return Status::OK(); } - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override { mutex_lock m(mu_); if (fs_.find(fname) == fs_.end()) { fs_[fname] = std::make_shared(); @@ -128,10 +125,8 @@ class RamFileSystem : public FileSystem { new RamRandomAccessFile(fname, fs_[fname])); return Status::OK(); } - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override { mutex_lock m(mu_); if (fs_.find(fname) == fs_.end()) { fs_[fname] = std::make_shared(); @@ -142,22 +137,18 @@ class RamFileSystem : public FileSystem { } Status NewReadOnlyMemoryRegionFromFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + const string& fname, TransactionToken* token, + std::unique_ptr* result) override { return errors::Unimplemented(""); } - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override { + Status FileExists(const string& fname, TransactionToken* token) override { FileStatistics stat; - return Stat(fname, &stat); + return Stat(fname, token, &stat); } - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override { + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override { mutex_lock m(mu_); auto it = fs_.lower_bound(dir); while (it != fs_.end() && absl::StartsWith(it->first, dir)) { @@ -168,10 +159,8 @@ class RamFileSystem : public FileSystem { return Status::OK(); } - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr */) - override { + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override { mutex_lock m(mu_); Env* env = Env::Default(); for (auto it = fs_.begin(); it != fs_.end(); ++it) { @@ -182,9 +171,8 @@ class RamFileSystem : public FileSystem { return Status::OK(); } - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override { + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override { mutex_lock m(mu_); auto it = fs_.lower_bound(fname); if (it == fs_.end()) { @@ -204,8 +192,7 @@ class RamFileSystem : public FileSystem { return Status::OK(); } - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override { + Status DeleteFile(const string& fname, TransactionToken* token) override { mutex_lock m(mu_); if (fs_.find(fname) != fs_.end()) { fs_.erase(fname); @@ -215,24 +202,21 @@ class RamFileSystem : public FileSystem { return errors::NotFound(""); } - Status CreateDir( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status CreateDir(const string& dirname, TransactionToken* token) override { return Status::OK(); } - Status RecursivelyCreateDir( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status RecursivelyCreateDir(const string& dirname, + TransactionToken* token) override { return Status::OK(); } - Status DeleteDir( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status DeleteDir(const string& dirname, TransactionToken* token) override { return Status::OK(); } - Status GetFileSize( - const string& fname, - uint64* file_size /*, TransactionToken* token = nullptr */) override { + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* file_size) override { mutex_lock m(mu_); if (fs_.find(fname) != fs_.end()) { *file_size = fs_[fname]->size(); @@ -241,9 +225,8 @@ class RamFileSystem : public FileSystem { return errors::NotFound(""); } - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override { + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override { mutex_lock m(mu_); if (fs_.find(src) != fs_.end()) { fs_[target] = fs_[src]; diff --git a/tensorflow/core/platform/retrying_file_system.h b/tensorflow/core/platform/retrying_file_system.h index 3891ce7499f..2f50b6cd5df 100644 --- a/tensorflow/core/platform/retrying_file_system.h +++ b/tensorflow/core/platform/retrying_file_system.h @@ -39,36 +39,27 @@ class RetryingFileSystem : public FileSystem { retry_config_(retry_config) {} Status NewRandomAccessFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; - Status NewWritableFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewWritableFile(const string& filename, TransactionToken* token, + std::unique_ptr* result) override; - Status NewAppendableFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewAppendableFile(const string& filename, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& filename, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& filename, TransactionToken* token, + std::unique_ptr* result) override; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override { + Status FileExists(const string& fname, TransactionToken* token) override { return RetryingUtils::CallWithRetries( [this, &fname]() { return base_file_system_->FileExists(fname); }, retry_config_); } - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override { + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override { return RetryingUtils::CallWithRetries( [this, &dir, result]() { return base_file_system_->GetChildren(dir, result); @@ -76,10 +67,8 @@ class RetryingFileSystem : public FileSystem { retry_config_); } - Status GetMatchingPaths( - const string& pattern, - std::vector* result /*, TransactionToken* token = nullptr */) - override { + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* result) override { return RetryingUtils::CallWithRetries( [this, &pattern, result]() { return base_file_system_->GetMatchingPaths(pattern, result); @@ -87,38 +76,33 @@ class RetryingFileSystem : public FileSystem { retry_config_); } - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override { + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override { return RetryingUtils::CallWithRetries( [this, &fname, stat]() { return base_file_system_->Stat(fname, stat); }, retry_config_); } - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override { + Status DeleteFile(const string& fname, TransactionToken* token) override { return RetryingUtils::DeleteWithRetries( [this, &fname]() { return base_file_system_->DeleteFile(fname); }, retry_config_); } - Status CreateDir( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status CreateDir(const string& dirname, TransactionToken* token) override { return RetryingUtils::CallWithRetries( [this, &dirname]() { return base_file_system_->CreateDir(dirname); }, retry_config_); } - Status DeleteDir( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status DeleteDir(const string& dirname, TransactionToken* token) override { return RetryingUtils::DeleteWithRetries( [this, &dirname]() { return base_file_system_->DeleteDir(dirname); }, retry_config_); } - Status GetFileSize( - const string& fname, - uint64* file_size /*, TransactionToken* token = nullptr */) override { + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* file_size) override { return RetryingUtils::CallWithRetries( [this, &fname, file_size]() { return base_file_system_->GetFileSize(fname, file_size); @@ -126,9 +110,8 @@ class RetryingFileSystem : public FileSystem { retry_config_); } - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override { + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override { return RetryingUtils::CallWithRetries( [this, &src, &target]() { return base_file_system_->RenameFile(src, target); @@ -136,8 +119,7 @@ class RetryingFileSystem : public FileSystem { retry_config_); } - Status IsDirectory( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status IsDirectory(const string& dirname, TransactionToken* token) override { return RetryingUtils::CallWithRetries( [this, &dirname]() { return base_file_system_->IsDirectory(dirname); }, retry_config_); @@ -148,9 +130,9 @@ class RetryingFileSystem : public FileSystem { return base_file_system_->HasAtomicMove(path, has_atomic_move); } - Status DeleteRecursively( - const string& dirname, int64* undeleted_files, - int64* undeleted_dirs /*, TransactionToken* token = nullptr */) override { + Status DeleteRecursively(const string& dirname, TransactionToken* token, + int64* undeleted_files, + int64* undeleted_dirs) override { return RetryingUtils::DeleteWithRetries( [this, &dirname, undeleted_files, undeleted_dirs]() { return base_file_system_->DeleteRecursively(dirname, undeleted_files, @@ -159,7 +141,7 @@ class RetryingFileSystem : public FileSystem { retry_config_); } - void FlushCaches(/* TransactionToken* token=nullptr */) override { + void FlushCaches(TransactionToken* token) override { base_file_system_->FlushCaches(); } @@ -243,8 +225,8 @@ class RetryingWritableFile : public WritableFile { template Status RetryingFileSystem::NewRandomAccessFile( - const string& filename, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { std::unique_ptr base_file; TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries( [this, &filename, &base_file]() { @@ -258,8 +240,8 @@ Status RetryingFileSystem::NewRandomAccessFile( template Status RetryingFileSystem::NewWritableFile( - const string& filename, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { std::unique_ptr base_file; TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries( [this, &filename, &base_file]() { @@ -273,8 +255,8 @@ Status RetryingFileSystem::NewWritableFile( template Status RetryingFileSystem::NewAppendableFile( - const string& filename, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { std::unique_ptr base_file; TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries( [this, &filename, &base_file]() { @@ -288,8 +270,8 @@ Status RetryingFileSystem::NewAppendableFile( template Status RetryingFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& filename, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& filename, TransactionToken* token, + std::unique_ptr* result) { return RetryingUtils::CallWithRetries( [this, &filename, result]() { return base_file_system_->NewReadOnlyMemoryRegionFromFile(filename, diff --git a/tensorflow/core/platform/retrying_file_system_test.cc b/tensorflow/core/platform/retrying_file_system_test.cc index 439abd6f3ec..0cada5a5651 100644 --- a/tensorflow/core/platform/retrying_file_system_test.cc +++ b/tensorflow/core/platform/retrying_file_system_test.cc @@ -100,100 +100,83 @@ class MockFileSystem : public FileSystem { : calls_(calls), flushed_(flushed) {} Status NewRandomAccessFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + const string& fname, TransactionToken* token, + std::unique_ptr* result) override { *result = std::move(random_access_file_to_return); return calls_.ConsumeNextCall("NewRandomAccessFile"); } - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override { *result = std::move(writable_file_to_return); return calls_.ConsumeNextCall("NewWritableFile"); } - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override { *result = std::move(writable_file_to_return); return calls_.ConsumeNextCall("NewAppendableFile"); } Status NewReadOnlyMemoryRegionFromFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override { + const string& fname, TransactionToken* token, + std::unique_ptr* result) override { return calls_.ConsumeNextCall("NewReadOnlyMemoryRegionFromFile"); } - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override { + Status FileExists(const string& fname, TransactionToken* token) override { return calls_.ConsumeNextCall("FileExists"); } - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override { + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override { return calls_.ConsumeNextCall("GetChildren"); } - Status GetMatchingPaths( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override { + Status GetMatchingPaths(const string& dir, TransactionToken* token, + std::vector* result) override { return calls_.ConsumeNextCall("GetMatchingPaths"); } - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override { + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override { return calls_.ConsumeNextCall("Stat"); } - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override { + Status DeleteFile(const string& fname, TransactionToken* token) override { return calls_.ConsumeNextCall("DeleteFile"); } - Status CreateDir( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status CreateDir(const string& dirname, TransactionToken* token) override { return calls_.ConsumeNextCall("CreateDir"); } - Status DeleteDir( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status DeleteDir(const string& dirname, TransactionToken* token) override { return calls_.ConsumeNextCall("DeleteDir"); } - Status GetFileSize( - const string& fname, - uint64* file_size /*, TransactionToken* token = nullptr */) override { + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* file_size) override { return calls_.ConsumeNextCall("GetFileSize"); } - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override { + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override { return calls_.ConsumeNextCall("RenameFile"); } - Status IsDirectory( - const string& dirname /*, TransactionToken* token = nullptr */) override { + Status IsDirectory(const string& dirname, TransactionToken* token) override { return calls_.ConsumeNextCall("IsDirectory"); } - Status DeleteRecursively( - const string& dirname, int64* undeleted_files, - int64* undeleted_dirs /*, TransactionToken* token = nullptr */) override { + Status DeleteRecursively(const string& dirname, TransactionToken* token, + int64* undeleted_files, + int64* undeleted_dirs) override { return calls_.ConsumeNextCall("DeleteRecursively"); } - void FlushCaches(/* TransactionToken* token=nullptr */) override { + void FlushCaches( + TransactionToken* token) override { if (flushed_) { *flushed_ = true; } diff --git a/tensorflow/python/framework/test_file_system.cc b/tensorflow/python/framework/test_file_system.cc index 6e9915adbb6..ed0a66fbefd 100644 --- a/tensorflow/python/framework/test_file_system.cc +++ b/tensorflow/python/framework/test_file_system.cc @@ -39,12 +39,14 @@ class TestRandomAccessFile : public RandomAccessFile { class TestFileSystem : public NullFileSystem { public: Status NewRandomAccessFile( - const string& fname, std::unique_ptr* result) override { + const string& fname, TransactionToken* token, + std::unique_ptr* result) override { result->reset(new TestRandomAccessFile); return Status::OK(); } // Always return size of 10 - Status GetFileSize(const string& fname, uint64* file_size) override { + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* file_size) override { *file_size = 10; return Status::OK(); } From 18e4ffd5be8bd0f946fdbfb002d428d4cdb9bc1f Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Fri, 31 Jul 2020 18:48:50 -0700 Subject: [PATCH 0436/1017] Add all retrying file systems to PR --- .../core/platform/cloud/gcs_file_system.cc | 95 +++--- .../core/platform/cloud/gcs_file_system.h | 72 ++-- .../platform/cloud/gcs_file_system_test.cc | 320 ++++++++++-------- tensorflow/core/platform/file_system_test.cc | 46 +-- .../core/platform/retrying_file_system.h | 68 ++-- .../platform/retrying_file_system_test.cc | 80 ++--- tensorflow/core/platform/s3/s3_file_system.cc | 81 +++-- tensorflow/core/platform/s3/s3_file_system.h | 48 +-- 8 files changed, 421 insertions(+), 389 deletions(-) diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 63c601f2244..88e92f0f84c 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -648,7 +648,7 @@ class GcsWritableFile : public WritableFile { TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when composing to ", GetGcsPath()); TF_RETURN_WITH_CONTEXT_IF_ERROR( - filesystem_->DeleteFile(GetGcsPathWithObject(append_object)), + filesystem_->DeleteFile(GetGcsPathWithObject(append_object),nullptr), " when cleaning up."); return Status::OK(); }, @@ -929,8 +929,8 @@ GcsFileSystem::GcsFileSystem( additional_header_(additional_header) {} Status GcsFileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { string bucket, object; TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); TF_RETURN_IF_ERROR(CheckBucketLocationConstraint(bucket)); @@ -1231,9 +1231,9 @@ void GcsFileSystem::ClearFileCaches(const string& fname) { // MatchingPathsCache as well. } -Status GcsFileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { +Status GcsFileSystem::NewWritableFile(const string& fname, + TransactionToken* token, + std::unique_ptr* result) { string bucket, object; TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); @@ -1267,11 +1267,11 @@ Status GcsFileSystem::NewWritableFile( // Reads the file from GCS in chunks and stores it in a tmp file, // which is then passed to GcsWritableFile. -Status GcsFileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { +Status GcsFileSystem::NewAppendableFile(const string& fname, + TransactionToken* token, + std::unique_ptr* result) { std::unique_ptr reader; - TF_RETURN_IF_ERROR(NewRandomAccessFile(fname, &reader)); + TF_RETURN_IF_ERROR(NewRandomAccessFile(fname, token, &reader)); std::unique_ptr buffer(new char[kReadAppendableFileBufferSize]); Status status; uint64 offset = 0; @@ -1330,14 +1330,14 @@ Status GcsFileSystem::NewAppendableFile( } Status GcsFileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { uint64 size; - TF_RETURN_IF_ERROR(GetFileSize(fname, &size)); + TF_RETURN_IF_ERROR(GetFileSize(fname,token, &size)); std::unique_ptr data(new char[size]); std::unique_ptr file; - TF_RETURN_IF_ERROR(NewRandomAccessFile(fname, &file)); + TF_RETURN_IF_ERROR(NewRandomAccessFile(fname, token, &file)); StringPiece piece; TF_RETURN_IF_ERROR(file->Read(0, size, &piece, data.get())); @@ -1346,8 +1346,7 @@ Status GcsFileSystem::NewReadOnlyMemoryRegionFromFile( return Status::OK(); } -Status GcsFileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status GcsFileSystem::FileExists(const string& fname, TransactionToken* token) { string bucket, object; TF_RETURN_IF_ERROR(ParseGcsPath(fname, true, &bucket, &object)); if (object.empty()) { @@ -1561,17 +1560,17 @@ Status GcsFileSystem::FolderExists(const string& dirname, bool* result) { return s; } -Status GcsFileSystem::GetChildren( - const string& dirname, - std::vector* result /*, TransactionToken* token */) { +Status GcsFileSystem::GetChildren(const string& dirname, + TransactionToken* token, + std::vector* result) { return GetChildrenBounded(dirname, UINT64_MAX, result, false /* recursively */, false /* include_self_directory_marker */); } -Status GcsFileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status GcsFileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { MatchingPathsCache::ComputeFunc compute_func = [this](const string& pattern, std::vector* results) { results->clear(); @@ -1731,8 +1730,8 @@ Status GcsFileSystem::GetChildrenBounded(const string& dirname, } } -Status GcsFileSystem::Stat( - const string& fname, FileStatistics* stat /*, TransactionToken* token */) { +Status GcsFileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) { if (!stat) { return errors::Internal("'stat' cannot be nullptr."); } @@ -1766,8 +1765,7 @@ Status GcsFileSystem::Stat( return errors::NotFound("The specified path ", fname, " was not found."); } -Status GcsFileSystem::DeleteFile( - const string& fname /*, TransactionToken* token */) { +Status GcsFileSystem::DeleteFile(const string& fname, TransactionToken* token) { string bucket, object; TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); @@ -1783,8 +1781,8 @@ Status GcsFileSystem::DeleteFile( return Status::OK(); } -Status GcsFileSystem::CreateDir( - const string& dirname /*, TransactionToken* token */) { +Status GcsFileSystem::CreateDir(const string& dirname, + TransactionToken* token) { string dirname_with_slash = MaybeAppendSlash(dirname); VLOG(3) << "CreateDir: creating directory with dirname: " << dirname << " and dirname_with_slash: " << dirname_with_slash; @@ -1799,7 +1797,7 @@ Status GcsFileSystem::CreateDir( dirname_with_slash, " was not found."); } - if (FileExists(dirname_with_slash).ok()) { + if (FileExists(dirname_with_slash,token).ok()) { // Use the original name for a correct error here. VLOG(3) << "CreateDir: directory already exists, not uploading " << dirname; return errors::AlreadyExists(dirname); @@ -1833,8 +1831,8 @@ Status GcsFileSystem::CreateDir( // Checks that the directory is empty (i.e no objects with this prefix exist). // Deletes the GCS directory marker if it exists. -Status GcsFileSystem::DeleteDir( - const string& dirname /*, TransactionToken* token */) { +Status GcsFileSystem::DeleteDir(const string& dirname, + TransactionToken* token) { std::vector children; // A directory is considered empty either if there are no matching objects // with the corresponding name prefix or if there is exactly one matching @@ -1849,13 +1847,13 @@ Status GcsFileSystem::DeleteDir( } if (children.size() == 1 && children[0].empty()) { // This is the directory marker object. Delete it. - return DeleteFile(MaybeAppendSlash(dirname)); + return DeleteFile(MaybeAppendSlash(dirname),token); } return Status::OK(); } -Status GcsFileSystem::GetFileSize( - const string& fname, uint64* file_size /*, TransactionToken* token */) { +Status GcsFileSystem::GetFileSize(const string& fname, TransactionToken* token, + uint64* file_size) { if (!file_size) { return errors::Internal("'file_size' cannot be nullptr."); } @@ -1865,14 +1863,14 @@ Status GcsFileSystem::GetFileSize( TF_RETURN_IF_ERROR(ParseGcsPath(fname, false, &bucket, &object)); FileStatistics stat; - TF_RETURN_IF_ERROR(Stat(fname, &stat)); + TF_RETURN_IF_ERROR(Stat(fname,token, &stat)); *file_size = stat.length; return Status::OK(); } -Status GcsFileSystem::RenameFile( - const string& src, const string& target /*, TransactionToken* token */) { - if (!IsDirectory(src).ok()) { +Status GcsFileSystem::RenameFile(const string& src, const string& target, + TransactionToken* token) { + if (!IsDirectory(src,token).ok()) { return RenameObject(src, target); } // Rename all individual objects in the directory one by one. @@ -1930,11 +1928,11 @@ Status GcsFileSystem::RenameObject(const string& src, const string& target) { // on the server side, we can't just retry the whole RenameFile operation // because the source object is already gone. return RetryingUtils::DeleteWithRetries( - [this, &src]() { return DeleteFile(src); }, retry_config_); + [this, &src]() { return DeleteFile(src,nullptr); }, retry_config_); } -Status GcsFileSystem::IsDirectory( - const string& fname /*, TransactionToken* token */) { +Status GcsFileSystem::IsDirectory(const string& fname, + TransactionToken* token) { string bucket, object; TF_RETURN_IF_ERROR(ParseGcsPath(fname, true, &bucket, &object)); if (object.empty()) { @@ -1960,16 +1958,17 @@ Status GcsFileSystem::IsDirectory( return errors::NotFound("The specified path ", fname, " was not found."); } -Status GcsFileSystem::DeleteRecursively( - const string& dirname, int64* undeleted_files, - int64* undeleted_dirs /*, TransactionToken* token */) { +Status GcsFileSystem::DeleteRecursively(const string& dirname, + TransactionToken* token, + int64* undeleted_files, + int64* undeleted_dirs) { if (!undeleted_files || !undeleted_dirs) { return errors::Internal( "'undeleted_files' and 'undeleted_dirs' cannot be nullptr."); } *undeleted_files = 0; *undeleted_dirs = 0; - if (!IsDirectory(dirname).ok()) { + if (!IsDirectory(dirname,token).ok()) { *undeleted_dirs = 1; return Status( error::NOT_FOUND, @@ -1987,9 +1986,9 @@ Status GcsFileSystem::DeleteRecursively( // and therefore RetryingFileSystem won't pay attention to the failures, // we need to make sure these failures are properly retried. const auto& delete_file_status = RetryingUtils::DeleteWithRetries( - [this, &full_path]() { return DeleteFile(full_path); }, retry_config_); + [this, &full_path,token]() { return DeleteFile(full_path,token); }, retry_config_); if (!delete_file_status.ok()) { - if (IsDirectory(full_path).ok()) { + if (IsDirectory(full_path,token).ok()) { // The object is a directory marker. (*undeleted_dirs)++; } else { @@ -2003,7 +2002,7 @@ Status GcsFileSystem::DeleteRecursively( // Flushes all caches for filesystem metadata and file contents. Useful for // reclaiming memory once filesystem operations are done (e.g. model is loaded), // or for resetting the filesystem to a consistent state. -void GcsFileSystem::FlushCaches(/* TransactionToken* token */) { +void GcsFileSystem::FlushCaches(TransactionToken* token) { tf_shared_lock l(block_cache_lock_); file_block_cache_->Flush(); stat_cache_->Clear(); diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h index 6f0e9535bfe..0a27aba35c1 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.h +++ b/tensorflow/core/platform/cloud/gcs_file_system.h @@ -126,67 +126,49 @@ class GcsFileSystem : public FileSystem { bool compose_append); Status NewRandomAccessFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewWritableFile( - const string& fname, - std::unique_ptr* - result) /*, TransactionToken* token = nullptr */ override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewAppendableFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& fname, - std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status FileExists(const string& fname, TransactionToken* token) override; - Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; - Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) - override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override; - Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr */) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; - Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status DeleteFile(const string& fname, TransactionToken* token) override; - Status CreateDir( - const string& dirname /*, TransactionToken* token = nullptr */) override; + Status CreateDir(const string& dirname, TransactionToken* token) override; - Status DeleteDir( - const string& dirname /*, TransactionToken* token = nullptr */) override; + Status DeleteDir(const string& dirname, TransactionToken* token) override; - Status GetFileSize( - const string& fname, - uint64* file_size /*, TransactionToken* token = nullptr */) override; + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* file_size) override; - Status RenameFile( - const string& src, - const string& target /*, TransactionToken* token = nullptr */) override; + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override; - Status IsDirectory( - const string& fname /*, TransactionToken* token = nullptr */) override; + Status IsDirectory(const string& fname, TransactionToken* token) override; - Status DeleteRecursively( - const string& dirname, int64* undeleted_files, - int64* undeleted_dirs /*, TransactionToken* token = nullptr */) override; + Status DeleteRecursively(const string& dirname, TransactionToken* token, + int64* undeleted_files, + int64* undeleted_dirs) override; - void FlushCaches(/* TransactionToken* token = nullptr */) override; + void FlushCaches(TransactionToken* token) override; /// Set an object to collect runtime statistics from the GcsFilesystem. void SetStats(GcsStatsInterface* stats); diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc index c8e72487bbe..b216281d630 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc @@ -86,7 +86,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); StringPiece filename; TF_EXPECT_OK(file->Name(&filename)); @@ -133,7 +134,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); StringPiece filename; TF_EXPECT_OK(file->Name(&filename)); @@ -181,7 +183,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_Errors) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); StringPiece filename; TF_EXPECT_OK(file->Name(&filename)); @@ -228,7 +231,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_ReadAtEOF) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); StringPiece filename; TF_EXPECT_OK(file->Name(&filename)); @@ -269,7 +273,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_CachedOutOfRange) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); StringPiece filename; TF_EXPECT_OK(file->Name(&filename)); @@ -320,7 +325,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_CachedNotSequential) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); StringPiece filename; TF_EXPECT_OK(file->Name(&filename)); @@ -361,7 +367,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_Growing) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); StringPiece filename; TF_EXPECT_OK(file->Name(&filename)); @@ -408,7 +415,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_Buffered_ReadBackwards) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); StringPiece filename; TF_EXPECT_OK(file->Name(&filename)); @@ -450,7 +458,8 @@ TEST(GcsFileSystemTest, nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); } TEST(GcsFileSystemTest, NewRandomAccessFile_WithLocationConstraintCaching) { @@ -496,18 +505,18 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithLocationConstraintCaching) { string bucket = "gs://bucket/random_access.txt"; string another_bucket = "gs://anotherbucket/random_access.txt"; // Multiple calls should only cause one request to the location api. - TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file)); - TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file)); + TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file)); + TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file)); // A new bucket should have one cache miss - TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, &file)); + TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, nullptr, &file)); // And then future calls to both should be cached - TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file)); - TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, &file)); + TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file)); + TF_EXPECT_OK(fs.NewRandomAccessFile(another_bucket, nullptr, &file)); // Trigger a flush, should then require one more call - fs.FlushCaches(); - TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, &file)); + fs.FlushCaches(nullptr); + TF_EXPECT_OK(fs.NewRandomAccessFile(bucket, nullptr, &file)); } TEST(GcsFileSystemTest, @@ -533,10 +542,11 @@ TEST(GcsFileSystemTest, nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - EXPECT_EQ(tensorflow::errors::FailedPrecondition( - "Bucket 'bucket' is in 'barfoo' location, allowed locations " - "are: (us-east1)."), - fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + EXPECT_EQ( + tensorflow::errors::FailedPrecondition( + "Bucket 'bucket' is in 'barfoo' location, allowed locations " + "are: (us-east1)."), + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); } TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) { @@ -565,7 +575,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoBlockCache_DifferentN) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); char small_scratch[3]; StringPiece result; @@ -630,8 +641,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache) { // We are instantiating this in an enclosed scope to make sure after the // unique ptr goes out of scope, we can still access result. std::unique_ptr file; - TF_EXPECT_OK( - fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", + nullptr, &file)); // Read the first chunk. The cache will be populated with the first block of // 9 bytes. @@ -716,7 +727,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) { char scratch[100]; StringPiece result; std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); // Read the first chunk. The cache will be populated with the first block of // 9 bytes. scratch[5] = 'x'; @@ -725,7 +737,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_Flush) { EXPECT_EQ(scratch[5], 'x'); // Make sure we only copied 4 bytes. // Flush caches and read the second chunk. This will be a cache miss, and // the same block will be fetched again. - fs.FlushCaches(); + fs.FlushCaches(nullptr); TF_EXPECT_OK(file->Read(4, 4, &result, scratch)); EXPECT_EQ("4567", result); } @@ -772,8 +784,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_WithBlockCache_MaxStaleness) { // staleness of the filesystem is > 0, they will share the same blocks. std::unique_ptr file1; std::unique_ptr file2; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", &file1)); - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", &file2)); + TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", nullptr, &file1)); + TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/object", nullptr, &file2)); // Reading the first block from file1 should load it once. TF_EXPECT_OK(file1->Read(0, 8, &result, scratch)); EXPECT_EQ("01234567", result); @@ -834,7 +846,8 @@ TEST(GcsFileSystemTest, nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); char scratch[5]; StringPiece result; @@ -864,7 +877,7 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_NoObjectName) { std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.NewRandomAccessFile("gs://bucket/", &file).code()); + fs.NewRandomAccessFile("gs://bucket/", nullptr, &file).code()); } TEST(GcsFileSystemTest, NewRandomAccessFile_InconsistentRead) { @@ -897,10 +910,11 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_InconsistentRead) { // Stat the file first so that the file stats are cached. FileStatistics stat; - TF_ASSERT_OK(fs.Stat("gs://bucket/random_access.txt", &stat)); + TF_ASSERT_OK(fs.Stat("gs://bucket/random_access.txt", nullptr, &stat)); std::unique_ptr file; - TF_ASSERT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_ASSERT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); char scratch[6]; StringPiece result; @@ -964,14 +978,16 @@ TEST(GcsFileSystemTest, NewWritableFile) { // Read from the file first, to fill the block cache. std::unique_ptr rfile; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/writeable", &rfile)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/path/writeable", nullptr, &rfile)); char scratch[100]; StringPiece result; TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch)); EXPECT_EQ("0123", result); // Open the writable file. std::unique_ptr wfile; - TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable", &wfile)); + TF_EXPECT_OK( + fs.NewWritableFile("gs://bucket/path/writeable", nullptr, &wfile)); TF_EXPECT_OK(wfile->Append("content1,")); int64 pos; TF_EXPECT_OK(wfile->Tell(&pos)); @@ -1055,7 +1071,8 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceeds) { nullptr /* gcs additional header */, false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); + TF_EXPECT_OK( + fs.NewWritableFile("gs://bucket/path/writeable.txt", nullptr, &file)); TF_EXPECT_OK(file->Append("content1,")); TF_EXPECT_OK(file->Append("content2")); @@ -1127,7 +1144,8 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) { // Pull the file's first block into the cache. This will trigger the first // HTTP request to GCS. std::unique_ptr rfile; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/writeable", &rfile)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/path/writeable", nullptr, &rfile)); char scratch[100]; StringPiece result; TF_EXPECT_OK(rfile->Read(0, 4, &result, scratch)); @@ -1135,7 +1153,8 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadSucceedsOnGetStatus) { // Now write to the same file. Once the write succeeds, the cached block will // be flushed. std::unique_ptr wfile; - TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable", &wfile)); + TF_EXPECT_OK( + fs.NewWritableFile("gs://bucket/path/writeable", nullptr, &wfile)); TF_EXPECT_OK(wfile->Append("content1,")); TF_EXPECT_OK(wfile->Append("content2")); // Appending doesn't invalidate the read cache - only flushing does. This read @@ -1213,7 +1232,8 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) { false /* compose append */); std::unique_ptr file; - TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); + TF_EXPECT_OK( + fs.NewWritableFile("gs://bucket/path/writeable.txt", nullptr, &file)); TF_EXPECT_OK(file->Append("content1,")); TF_EXPECT_OK(file->Append("content2")); @@ -1277,7 +1297,8 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) { { std::unique_ptr file; - TF_EXPECT_OK(fs.NewWritableFile("gs://bucket/path/writeable.txt", &file)); + TF_EXPECT_OK( + fs.NewWritableFile("gs://bucket/path/writeable.txt", nullptr, &file)); TF_EXPECT_OK(file->Append("content1,")); TF_EXPECT_OK(file->Append("content2")); @@ -1317,7 +1338,7 @@ TEST(GcsFileSystemTest, NewWritableFile_NoObjectName) { std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.NewWritableFile("gs://bucket/", &file).code()); + fs.NewWritableFile("gs://bucket/", nullptr, &file).code()); } TEST(GcsFileSystemTest, NewAppendableFile) { @@ -1382,12 +1403,14 @@ TEST(GcsFileSystemTest, NewAppendableFile) { // Create an appendable file. This should read the file from GCS, and pull its // contents into the block cache. std::unique_ptr wfile; - TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/path/appendable", &wfile)); + TF_EXPECT_OK( + fs.NewAppendableFile("gs://bucket/path/appendable", nullptr, &wfile)); TF_EXPECT_OK(wfile->Append("content2")); // Verify that the file contents are in the block cache. This read should not // trigger an HTTP request to GCS. std::unique_ptr rfile; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/appendable", &rfile)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/path/appendable", nullptr, &rfile)); char scratch[100]; StringPiece result; TF_EXPECT_OK(rfile->Read(0, 8, &result, scratch)); @@ -1416,7 +1439,7 @@ TEST(GcsFileSystemTest, NewAppendableFile_NoObjectName) { std::unique_ptr file; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.NewAppendableFile("gs://bucket/", &file).code()); + fs.NewAppendableFile("gs://bucket/", nullptr, &file).code()); } TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { @@ -1450,7 +1473,7 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile) { std::unique_ptr region; TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile( - "gs://bucket/path/random_access.txt", ®ion)); + "gs://bucket/path/random_access.txt", nullptr, ®ion)); EXPECT_EQ(content, StringPiece(reinterpret_cast(region->data()), region->length())); @@ -1471,7 +1494,8 @@ TEST(GcsFileSystemTest, NewReadOnlyMemoryRegionFromFile_NoObjectName) { std::unique_ptr region; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.NewReadOnlyMemoryRegionFromFile("gs://bucket/", ®ion).code()); + fs.NewReadOnlyMemoryRegionFromFile("gs://bucket/", nullptr, ®ion) + .code()); } TEST(GcsFileSystemTest, FileExists_YesAsObject) { @@ -1493,7 +1517,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsObject) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt")); + TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt", nullptr)); } TEST(GcsFileSystemTest, FileExists_YesAsFolder) { @@ -1523,7 +1547,7 @@ TEST(GcsFileSystemTest, FileExists_YesAsFolder) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder")); + TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder", nullptr)); } TEST(GcsFileSystemTest, FileExists_YesAsBucket) { @@ -1549,8 +1573,8 @@ TEST(GcsFileSystemTest, FileExists_YesAsBucket) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.FileExists("gs://bucket1")); - TF_EXPECT_OK(fs.FileExists("gs://bucket1/")); + TF_EXPECT_OK(fs.FileExists("gs://bucket1", nullptr)); + TF_EXPECT_OK(fs.FileExists("gs://bucket1/", nullptr)); } TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) { @@ -1580,7 +1604,7 @@ TEST(GcsFileSystemTest, FileExists_NotAsObjectOrFolder) { nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(errors::Code::NOT_FOUND, - fs.FileExists("gs://bucket/path/file1.txt").code()); + fs.FileExists("gs://bucket/path/file1.txt", nullptr).code()); } TEST(GcsFileSystemTest, FileExists_NotAsBucket) { @@ -1606,9 +1630,9 @@ TEST(GcsFileSystemTest, FileExists_NotAsBucket) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.FileExists("gs://bucket2/").code()); + fs.FileExists("gs://bucket2/", nullptr).code()); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.FileExists("gs://bucket2").code()); + fs.FileExists("gs://bucket2", nullptr).code()); } TEST(GcsFileSystemTest, FileExists_StatCache) { @@ -1648,8 +1672,8 @@ TEST(GcsFileSystemTest, FileExists_StatCache) { // The stat cache will ensure that repeated lookups don't trigger additional // HTTP requests. for (int i = 0; i < 10; i++) { - TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt")); - TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder/")); + TF_EXPECT_OK(fs.FileExists("gs://bucket/path/file1.txt", nullptr)); + TF_EXPECT_OK(fs.FileExists("gs://bucket/path/subfolder/", nullptr)); } } @@ -1672,8 +1696,8 @@ TEST(GcsFileSystemTest, FileExists_DirectoryMark) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.FileExists("gs://bucket/dir/")); - TF_EXPECT_OK(fs.IsDirectory("gs://bucket/dir/")); + TF_EXPECT_OK(fs.FileExists("gs://bucket/dir/", nullptr)); + TF_EXPECT_OK(fs.IsDirectory("gs://bucket/dir/", nullptr)); } TEST(GcsFileSystemTest, GetChildren_NoItems) { @@ -1696,7 +1720,7 @@ TEST(GcsFileSystemTest, GetChildren_NoItems) { nullptr /* gcs additional header */, false /* compose append */); std::vector children; - TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); + TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children)); EXPECT_EQ(std::vector({"subpath/"}), children); } @@ -1724,7 +1748,7 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles) { nullptr /* gcs additional header */, false /* compose append */); std::vector children; - TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); + TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children)); EXPECT_EQ(std::vector({"file1.txt", "file3.txt", "subpath/"}), children); @@ -1753,7 +1777,7 @@ TEST(GcsFileSystemTest, GetChildren_SelfDirectoryMarker) { nullptr /* gcs additional header */, false /* compose append */); std::vector children; - TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); + TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children)); EXPECT_EQ(std::vector({"file3.txt", "subpath/"}), children); } @@ -1781,7 +1805,7 @@ TEST(GcsFileSystemTest, GetChildren_ThreeFiles_NoSlash) { nullptr /* gcs additional header */, false /* compose append */); std::vector children; - TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); + TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", nullptr, &children)); EXPECT_EQ(std::vector({"file1.txt", "file3.txt", "subpath/"}), children); @@ -1806,7 +1830,7 @@ TEST(GcsFileSystemTest, GetChildren_Root) { nullptr /* gcs additional header */, false /* compose append */); std::vector children; - TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", &children)); + TF_EXPECT_OK(fs.GetChildren("gs://bucket-a-b-c", nullptr, &children)); EXPECT_EQ(0, children.size()); } @@ -1831,7 +1855,7 @@ TEST(GcsFileSystemTest, GetChildren_Empty) { nullptr /* gcs additional header */, false /* compose append */); std::vector children; - TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", &children)); + TF_EXPECT_OK(fs.GetChildren("gs://bucket/path/", nullptr, &children)); EXPECT_EQ(0, children.size()); } @@ -1872,7 +1896,7 @@ TEST(GcsFileSystemTest, GetChildren_Pagination) { nullptr /* gcs additional header */, false /* compose append */); std::vector children; - TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", &children)); + TF_EXPECT_OK(fs.GetChildren("gs://bucket/path", nullptr, &children)); EXPECT_EQ(std::vector({"file1.txt", "file3.txt", "subpath/", "file4.txt", "file5.txt"}), @@ -1899,8 +1923,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_NoWildcard) { nullptr /* gcs additional header */, false /* compose append */); std::vector result; - TF_EXPECT_OK( - fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", + nullptr, &result)); EXPECT_EQ(std::vector({"gs://bucket/path/subpath/file2.txt"}), result); } @@ -1927,7 +1951,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_BucketAndWildcard) { nullptr /* gcs additional header */, false /* compose append */); std::vector result; - TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", nullptr, &result)); EXPECT_EQ(std::vector({"gs://bucket/path/file1.txt", "gs://bucket/path/file3.txt", "gs://bucket/path/subpath"}), @@ -1956,7 +1980,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_Matches) { nullptr /* gcs additional header */, false /* compose append */); std::vector result; - TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", &result)); + TF_EXPECT_OK( + fs.GetMatchingPaths("gs://bucket/path/*/file2.txt", nullptr, &result)); EXPECT_EQ(std::vector({"gs://bucket/path/subpath/file2.txt"}), result); } @@ -1982,7 +2007,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SelfDirectoryMarker) { nullptr /* gcs additional header */, false /* compose append */); std::vector result; - TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", nullptr, &result)); EXPECT_EQ(std::vector({"gs://bucket/path/file3.txt"}), result); } @@ -2007,7 +2032,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SlashInObjectName) { nullptr /* gcs additional header */, false /* compose append */); std::vector result; - TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*", nullptr, &result)); EXPECT_EQ(std::vector(), result); } @@ -2032,7 +2057,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_SlashInObjectNameEscaped) { nullptr /* gcs additional header */, false /* compose append */); std::vector result; - TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/\\/*", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/\\/*", nullptr, &result)); EXPECT_EQ(std::vector({"gs://bucket/path//foo.txt"}), result); } @@ -2058,7 +2083,8 @@ TEST(GcsFileSystemTest, GetMatchingPaths_FolderAndWildcard_NoMatches) { nullptr /* gcs additional header */, false /* compose append */); std::vector result; - TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", &result)); + TF_EXPECT_OK( + fs.GetMatchingPaths("gs://bucket/path/*/file3.txt", nullptr, &result)); EXPECT_EQ(std::vector(), result); } @@ -2077,7 +2103,7 @@ TEST(GcsFileSystemTest, GetMatchingPaths_OnlyWildcard) { std::vector result; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.GetMatchingPaths("gs://*", &result).code()); + fs.GetMatchingPaths("gs://*", nullptr, &result).code()); } TEST(GcsFileSystemTest, GetMatchingPaths_Cache) { @@ -2113,11 +2139,11 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache) { // any additional HTTP requests to GCS. for (int i = 0; i < 10; i++) { std::vector result; - TF_EXPECT_OK( - fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", + nullptr, &result)); EXPECT_EQ(std::vector({"gs://bucket/path/subpath/file2.txt"}), result); - TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/*/*", nullptr, &result)); EXPECT_EQ(std::vector({"gs://bucket/path/file1.txt", "gs://bucket/path/file3.txt", "gs://bucket/path/subpath"}), @@ -2155,17 +2181,17 @@ TEST(GcsFileSystemTest, GetMatchingPaths_Cache_Flush) { // This loop should trigger the first HTTP request to GCS. for (int i = 0; i < 10; i++) { std::vector result; - TF_EXPECT_OK( - fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", + nullptr, &result)); EXPECT_EQ(std::vector({"gs://bucket/path/subpath/file2.txt"}), result); } // After flushing caches, there should be another (identical) request to GCS. - fs.FlushCaches(); + fs.FlushCaches(nullptr); for (int i = 0; i < 10; i++) { std::vector result; - TF_EXPECT_OK( - fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://bucket/path/subpath/file2.txt", + nullptr, &result)); EXPECT_EQ(std::vector({"gs://bucket/path/subpath/file2.txt"}), result); } @@ -2220,11 +2246,12 @@ TEST(GcsFileSystemTest, DeleteFile) { char scratch[100]; StringPiece result; std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/file1.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/path/file1.txt", nullptr, &file)); TF_EXPECT_OK(file->Read(0, 8, &result, scratch)); EXPECT_EQ("01234567", result); // Deleting the file triggers the next HTTP request to GCS. - TF_EXPECT_OK(fs.DeleteFile("gs://bucket/path/file1.txt")); + TF_EXPECT_OK(fs.DeleteFile("gs://bucket/path/file1.txt", nullptr)); // Re-reading the file causes its contents to be reloaded from GCS and not // from the block cache. TF_EXPECT_OK(file->Read(0, 8, &result, scratch)); @@ -2245,7 +2272,7 @@ TEST(GcsFileSystemTest, DeleteFile_NoObjectName) { nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.DeleteFile("gs://bucket/").code()); + fs.DeleteFile("gs://bucket/", nullptr).code()); } TEST(GcsFileSystemTest, DeleteFile_StatCacheRemoved) { @@ -2289,14 +2316,15 @@ TEST(GcsFileSystemTest, DeleteFile_StatCacheRemoved) { // Stats the file first so the stat is cached. FileStatistics stat_before_deletion; - TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat_before_deletion)); + TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat_before_deletion)); EXPECT_EQ(1010, stat_before_deletion.length); - TF_EXPECT_OK(fs.DeleteFile("gs://bucket/file.txt")); + TF_EXPECT_OK(fs.DeleteFile("gs://bucket/file.txt", nullptr)); FileStatistics stat_after_deletion; - EXPECT_EQ(error::Code::NOT_FOUND, - fs.Stat("gs://bucket/file.txt", &stat_after_deletion).code()); + EXPECT_EQ( + error::Code::NOT_FOUND, + fs.Stat("gs://bucket/file.txt", nullptr, &stat_after_deletion).code()); } TEST(GcsFileSystemTest, DeleteDir_Empty) { @@ -2317,7 +2345,7 @@ TEST(GcsFileSystemTest, DeleteDir_Empty) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); + TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/", nullptr)); } TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { @@ -2346,7 +2374,7 @@ TEST(GcsFileSystemTest, DeleteDir_OnlyDirMarkerLeft) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/")); + TF_EXPECT_OK(fs.DeleteDir("gs://bucket/path/", nullptr)); } TEST(GcsFileSystemTest, DeleteDir_BucketOnly) { @@ -2366,7 +2394,7 @@ TEST(GcsFileSystemTest, DeleteDir_BucketOnly) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.DeleteDir("gs://bucket")); + TF_EXPECT_OK(fs.DeleteDir("gs://bucket", nullptr)); } TEST(GcsFileSystemTest, DeleteDir_NonEmpty) { @@ -2389,7 +2417,7 @@ TEST(GcsFileSystemTest, DeleteDir_NonEmpty) { nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, - fs.DeleteDir("gs://bucket/path/").code()); + fs.DeleteDir("gs://bucket/path/", nullptr).code()); } TEST(GcsFileSystemTest, GetFileSize) { @@ -2412,7 +2440,7 @@ TEST(GcsFileSystemTest, GetFileSize) { nullptr /* gcs additional header */, false /* compose append */); uint64 size; - TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", &size)); + TF_EXPECT_OK(fs.GetFileSize("gs://bucket/file.txt", nullptr, &size)); EXPECT_EQ(1010, size); } @@ -2431,7 +2459,7 @@ TEST(GcsFileSystemTest, GetFileSize_NoObjectName) { uint64 size; EXPECT_EQ(errors::Code::INVALID_ARGUMENT, - fs.GetFileSize("gs://bucket/", &size).code()); + fs.GetFileSize("gs://bucket/", nullptr, &size).code()); } TEST(GcsFileSystemTest, RenameFile_Folder) { @@ -2515,7 +2543,8 @@ TEST(GcsFileSystemTest, RenameFile_Folder) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/")); + TF_EXPECT_OK( + fs.RenameFile("gs://bucket/path1", "gs://bucket/path2/", nullptr)); } TEST(GcsFileSystemTest, RenameFile_Object) { @@ -2612,15 +2641,17 @@ TEST(GcsFileSystemTest, RenameFile_Object) { StringPiece result; std::unique_ptr src; std::unique_ptr dst; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/src.txt", &src)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/path/src.txt", nullptr, &src)); TF_EXPECT_OK(src->Read(0, 8, &result, scratch)); EXPECT_EQ("01234567", result); - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/path/dst.txt", &dst)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/path/dst.txt", nullptr, &dst)); TF_EXPECT_OK(dst->Read(0, 8, &result, scratch)); EXPECT_EQ("76543210", result); // Now rename src to dst. This should flush the block cache for both files. - TF_EXPECT_OK( - fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); + TF_EXPECT_OK(fs.RenameFile("gs://bucket/path/src.txt", + "gs://bucket/path/dst.txt", nullptr)); // Re-read both files. This should reload their contents from GCS. TF_EXPECT_OK(src->Read(0, 8, &result, scratch)); EXPECT_EQ("89abcdef", result); @@ -2690,14 +2721,16 @@ TEST(GcsFileSystemTest, RenameFile_Object_FlushTargetStatCache) { // Do an initial stat of the destination file to load their contents into the // stat cache. FileStatistics stat_before_renaming; - TF_EXPECT_OK(fs.Stat("gs://bucket/path/dst.txt", &stat_before_renaming)); + TF_EXPECT_OK( + fs.Stat("gs://bucket/path/dst.txt", nullptr, &stat_before_renaming)); EXPECT_EQ(1000, stat_before_renaming.length); - TF_EXPECT_OK( - fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); + TF_EXPECT_OK(fs.RenameFile("gs://bucket/path/src.txt", + "gs://bucket/path/dst.txt", nullptr)); FileStatistics stat_after_renaming; - TF_EXPECT_OK(fs.Stat("gs://bucket/path/dst.txt", &stat_after_renaming)); + TF_EXPECT_OK( + fs.Stat("gs://bucket/path/dst.txt", nullptr, &stat_after_renaming)); EXPECT_EQ(1010, stat_after_renaming.length); } @@ -2755,8 +2788,8 @@ TEST(GcsFileSystemTest, RenameFile_Object_DeletionRetried) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK( - fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt")); + TF_EXPECT_OK(fs.RenameFile("gs://bucket/path/src.txt", + "gs://bucket/path/dst.txt", nullptr)); } /// Tests the case when rewrite couldn't complete in one RPC. @@ -2797,10 +2830,10 @@ TEST(GcsFileSystemTest, RenameFile_Object_Incomplete) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - EXPECT_EQ( - errors::Code::UNIMPLEMENTED, - fs.RenameFile("gs://bucket/path/src.txt", "gs://bucket/path/dst.txt") - .code()); + EXPECT_EQ(errors::Code::UNIMPLEMENTED, + fs.RenameFile("gs://bucket/path/src.txt", + "gs://bucket/path/dst.txt", nullptr) + .code()); } TEST(GcsFileSystemTest, Stat_Object) { @@ -2823,7 +2856,7 @@ TEST(GcsFileSystemTest, Stat_Object) { nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; - TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); + TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat)); EXPECT_EQ(1010, stat.length); EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1); EXPECT_FALSE(stat.is_directory); @@ -2857,7 +2890,7 @@ TEST(GcsFileSystemTest, Stat_Folder) { nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; - TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", &stat)); + TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder", nullptr, &stat)); EXPECT_EQ(0, stat.length); EXPECT_EQ(0, stat.mtime_nsec); EXPECT_TRUE(stat.is_directory); @@ -2890,7 +2923,8 @@ TEST(GcsFileSystemTest, Stat_ObjectOrFolderNotFound) { nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; - EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/path", &stat).code()); + EXPECT_EQ(error::Code::NOT_FOUND, + fs.Stat("gs://bucket/path", nullptr, &stat).code()); } TEST(GcsFileSystemTest, Stat_Bucket) { @@ -2911,7 +2945,7 @@ TEST(GcsFileSystemTest, Stat_Bucket) { nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; - TF_EXPECT_OK(fs.Stat("gs://bucket/", &stat)); + TF_EXPECT_OK(fs.Stat("gs://bucket/", nullptr, &stat)); EXPECT_EQ(0, stat.length); EXPECT_EQ(0, stat.mtime_nsec); EXPECT_TRUE(stat.is_directory); @@ -2935,7 +2969,8 @@ TEST(GcsFileSystemTest, Stat_BucketNotFound) { nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; - EXPECT_EQ(error::Code::NOT_FOUND, fs.Stat("gs://bucket/", &stat).code()); + EXPECT_EQ(error::Code::NOT_FOUND, + fs.Stat("gs://bucket/", nullptr, &stat).code()); } TEST(GcsFileSystemTest, Stat_Cache) { @@ -2976,11 +3011,11 @@ TEST(GcsFileSystemTest, Stat_Cache) { // HTTP requests to GCS. for (int i = 0; i < 10; i++) { FileStatistics stat; - TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); + TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat)); EXPECT_EQ(1010, stat.length); EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1); EXPECT_FALSE(stat.is_directory); - TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder/", &stat)); + TF_EXPECT_OK(fs.Stat("gs://bucket/subfolder/", nullptr, &stat)); EXPECT_EQ(0, stat.length); EXPECT_EQ(0, stat.mtime_nsec); EXPECT_TRUE(stat.is_directory); @@ -3016,16 +3051,16 @@ TEST(GcsFileSystemTest, Stat_Cache_Flush) { // There should be a single HTTP request to GCS for fs.Stat in this loop. for (int i = 0; i < 10; i++) { FileStatistics stat; - TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); + TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat)); EXPECT_EQ(1010, stat.length); EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1); EXPECT_FALSE(stat.is_directory); } // After flushing caches, there should be a second request to GCS for fs.Stat. - fs.FlushCaches(); + fs.FlushCaches(nullptr); for (int i = 0; i < 10; i++) { FileStatistics stat; - TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); + TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat)); EXPECT_EQ(1010, stat.length); EXPECT_NEAR(1461971724896, stat.mtime_nsec / 1000 / 1000, 1); EXPECT_FALSE(stat.is_directory); @@ -3052,7 +3087,7 @@ TEST(GcsFileSystemTest, Stat_FilenameEndingWithSlash) { nullptr /* gcs additional header */, false /* compose append */); FileStatistics stat; - TF_EXPECT_OK(fs.Stat("gs://bucket/dir/", &stat)); + TF_EXPECT_OK(fs.Stat("gs://bucket/dir/", nullptr, &stat)); EXPECT_EQ(5, stat.length); EXPECT_TRUE(stat.is_directory); } @@ -3084,7 +3119,7 @@ TEST(GcsFileSystemTest, IsDirectory_NotFound) { nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(error::Code::NOT_FOUND, - fs.IsDirectory("gs://bucket/file.txt").code()); + fs.IsDirectory("gs://bucket/file.txt", nullptr).code()); } TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) { @@ -3115,7 +3150,7 @@ TEST(GcsFileSystemTest, IsDirectory_NotDirectoryButObject) { nullptr /* gcs additional header */, false /* compose append */); EXPECT_EQ(error::Code::FAILED_PRECONDITION, - fs.IsDirectory("gs://bucket/file.txt").code()); + fs.IsDirectory("gs://bucket/file.txt", nullptr).code()); } TEST(GcsFileSystemTest, IsDirectory_Yes) { @@ -3145,8 +3180,8 @@ TEST(GcsFileSystemTest, IsDirectory_Yes) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder")); - TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/")); + TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder", nullptr)); + TF_EXPECT_OK(fs.IsDirectory("gs://bucket/subfolder/", nullptr)); } TEST(GcsFileSystemTest, IsDirectory_Bucket) { @@ -3172,8 +3207,8 @@ TEST(GcsFileSystemTest, IsDirectory_Bucket) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.IsDirectory("gs://bucket")); - TF_EXPECT_OK(fs.IsDirectory("gs://bucket/")); + TF_EXPECT_OK(fs.IsDirectory("gs://bucket", nullptr)); + TF_EXPECT_OK(fs.IsDirectory("gs://bucket/", nullptr)); } TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) { @@ -3193,7 +3228,8 @@ TEST(GcsFileSystemTest, IsDirectory_BucketNotFound) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - EXPECT_EQ(error::Code::NOT_FOUND, fs.IsDirectory("gs://bucket/").code()); + EXPECT_EQ(error::Code::NOT_FOUND, + fs.IsDirectory("gs://bucket/", nullptr).code()); } TEST(GcsFileSystemTest, CreateDir_Folder) { @@ -3250,15 +3286,15 @@ TEST(GcsFileSystemTest, CreateDir_Folder) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath")); + TF_EXPECT_OK(fs.CreateDir("gs://bucket/subpath", nullptr)); // Check that when GCS returns the object already exists return that the // directory already exists. EXPECT_EQ(errors::AlreadyExists("gs://bucket/subpath"), - fs.CreateDir("gs://bucket/subpath")); + fs.CreateDir("gs://bucket/subpath", nullptr)); // Check that when GCS returns the object already has a version (failed // precondition) return directory already exists. EXPECT_EQ(errors::AlreadyExists("gs://bucket/subpath"), - fs.CreateDir("gs://bucket/subpath")); + fs.CreateDir("gs://bucket/subpath", nullptr)); } TEST(GcsFileSystemTest, CreateDir_Bucket) { @@ -3284,8 +3320,8 @@ TEST(GcsFileSystemTest, CreateDir_Bucket) { kTestTimeoutConfig, *kAllowedLocationsDefault, nullptr /* gcs additional header */, false /* compose append */); - TF_EXPECT_OK(fs.CreateDir("gs://bucket/")); - TF_EXPECT_OK(fs.CreateDir("gs://bucket")); + TF_EXPECT_OK(fs.CreateDir("gs://bucket/", nullptr)); + TF_EXPECT_OK(fs.CreateDir("gs://bucket", nullptr)); } TEST(GcsFileSystemTest, DeleteRecursively_Ok) { @@ -3357,8 +3393,8 @@ TEST(GcsFileSystemTest, DeleteRecursively_Ok) { nullptr /* gcs additional header */, false /* compose append */); int64 undeleted_files, undeleted_dirs; - TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, - &undeleted_dirs)); + TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", nullptr, + &undeleted_files, &undeleted_dirs)); EXPECT_EQ(0, undeleted_files); EXPECT_EQ(0, undeleted_dirs); } @@ -3450,8 +3486,8 @@ TEST(GcsFileSystemTest, DeleteRecursively_DeletionErrors) { nullptr /* gcs additional header */, false /* compose append */); int64 undeleted_files, undeleted_dirs; - TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", &undeleted_files, - &undeleted_dirs)); + TF_EXPECT_OK(fs.DeleteRecursively("gs://bucket/path", nullptr, + &undeleted_files, &undeleted_dirs)); EXPECT_EQ(1, undeleted_files); EXPECT_EQ(1, undeleted_dirs); } @@ -3486,7 +3522,7 @@ TEST(GcsFileSystemTest, DeleteRecursively_NotAFolder) { int64 undeleted_files, undeleted_dirs; EXPECT_EQ(error::Code::NOT_FOUND, - fs.DeleteRecursively("gs://bucket/path", &undeleted_files, + fs.DeleteRecursively("gs://bucket/path", nullptr, &undeleted_files, &undeleted_dirs) .code()); EXPECT_EQ(0, undeleted_files); @@ -3501,7 +3537,7 @@ TEST(GcsFileSystemTest, NoConstraintsEnvironmentVariableTest) { // Cover cache initialization code, any uninitialized cache will cause this to // fail - fs1.FlushCaches(); + fs1.FlushCaches(nullptr); } TEST(GcsFileSystemTest, BucketLocationConstraintEnvironmentVariableTest) { @@ -3715,7 +3751,7 @@ TEST(GcsFileSystemTest, Stat_StatsRecording) { EXPECT_EQ(stats.fs_, &fs); FileStatistics stat; - TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", &stat)); + TF_EXPECT_OK(fs.Stat("gs://bucket/file.txt", nullptr, &stat)); EXPECT_EQ(1, stats.stat_object_request_count_); } @@ -3742,7 +3778,8 @@ TEST(GcsFileSystemTest, NewRandomAccessFile_StatsRecording) { EXPECT_EQ(stats.fs_, &fs); std::unique_ptr file; - TF_EXPECT_OK(fs.NewRandomAccessFile("gs://bucket/random_access.txt", &file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("gs://bucket/random_access.txt", nullptr, &file)); char scratch[6]; StringPiece result; @@ -3883,8 +3920,8 @@ TEST(GcsFileSystemTest, NewAppendableFile_MultipleFlushesWithCompose) { // Create an appendable file. This should read the file from GCS, and pull its // contents into the block cache. std::unique_ptr wfile; - TF_EXPECT_OK( - fs.NewAppendableFile("gs://bucket/some/path/appendable", &wfile)); + TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/some/path/appendable", nullptr, + &wfile)); TF_EXPECT_OK(wfile->Append(contents[1])); TF_EXPECT_OK(wfile->Flush()); TF_EXPECT_OK(wfile->Append(contents[2])); @@ -3981,7 +4018,8 @@ TEST(GcsFileSystemTest, NewAppendableFile_MultipleFlushesWithoutCompose) { // Create an appendable file. This should read the file from GCS, and pull its // contents into the block cache. std::unique_ptr wfile; - TF_EXPECT_OK(fs.NewAppendableFile("gs://bucket/path/appendable", &wfile)); + TF_EXPECT_OK( + fs.NewAppendableFile("gs://bucket/path/appendable", nullptr, &wfile)); TF_EXPECT_OK(wfile->Append(contents[1])); TF_EXPECT_OK(wfile->Flush()); TF_EXPECT_OK(wfile->Append(contents[2])); diff --git a/tensorflow/core/platform/file_system_test.cc b/tensorflow/core/platform/file_system_test.cc index dd02da32073..1e23a2b853c 100644 --- a/tensorflow/core/platform/file_system_test.cc +++ b/tensorflow/core/platform/file_system_test.cc @@ -107,7 +107,7 @@ class InterPlanetaryFileSystem : public NullFileSystem { Status GetChildren(const string& dir, TransactionToken* token, std::vector* result) override { - TF_RETURN_IF_ERROR(IsDirectory(dir)); + TF_RETURN_IF_ERROR(IsDirectory(dir, nullptr)); string parsed_path; ParsePath(dir, &parsed_path); result->insert(result->begin(), celestial_bodies_[parsed_path].begin(), @@ -153,7 +153,7 @@ class InterPlanetaryFileSystem : public NullFileSystem { string Match(InterPlanetaryFileSystem* ipfs, const string& suffix_pattern) { std::vector results; Status s = - ipfs->GetMatchingPaths(ipfs->JoinPath(kPrefix, suffix_pattern), &results); + ipfs->GetMatchingPaths(ipfs->JoinPath(kPrefix, suffix_pattern), nullptr, &results); if (!s.ok()) { return s.ToString(); } else { @@ -180,18 +180,18 @@ TEST(InterPlanetaryFileSystemTest, IPFSMatch) { // Returns Jupiter's and Earth's moons. EXPECT_EQ(Match(&ipfs, "*/*"), "Earth/Moon,Jupiter/Europa,Jupiter/Ganymede,Jupiter/Io"); - TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "Planet0"))); - TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "Planet1"))); + TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "Planet0"), nullptr)); + TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "Planet1"), nullptr)); EXPECT_EQ(Match(&ipfs, "Planet[0-1]"), "Planet0,Planet1"); EXPECT_EQ(Match(&ipfs, "Planet?"), "Planet0,Planet1"); } TEST(InterPlanetaryFileSystemTest, MatchSimple) { InterPlanetaryFileSystem ipfs; - TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-00"))); - TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-0a"))); - TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-01"))); - TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-aaa"))); + TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-00"), nullptr)); + TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-0a"), nullptr)); + TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-01"), nullptr)); + TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "match-aaa"), nullptr)); EXPECT_EQ(Match(&ipfs, "match-*"), "match-00,match-01,match-0a,match-aaa"); EXPECT_EQ(Match(&ipfs, "match-0[0-9]"), "match-00,match-01"); @@ -204,8 +204,8 @@ TEST(InterPlanetaryFileSystemTest, MatchSimple) { // that evil_directory isn't accessed. TEST(InterPlanetaryFileSystemTest, MatchOnlyNeeded) { InterPlanetaryFileSystem ipfs; - TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "abcd"))); - TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "evil_directory"))); + TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "abcd"), nullptr)); + TF_EXPECT_OK(ipfs.CreateDir(ipfs.JoinPath(kPrefix, "evil_directory"), nullptr)); EXPECT_EQ(Match(&ipfs, "abcd"), "abcd"); } @@ -213,13 +213,13 @@ TEST(InterPlanetaryFileSystemTest, MatchOnlyNeeded) { TEST(InterPlanetaryFileSystemTest, MatchDirectory) { InterPlanetaryFileSystem ipfs; TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-00/abc/x"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-00/abc/x"), nullptr)); TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-0a/abc/x"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-0a/abc/x"), nullptr)); TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-01/abc/x"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-01/abc/x"), nullptr)); TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-aaa/abc/x"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-aaa/abc/x"), nullptr)); EXPECT_EQ(Match(&ipfs, "match-*/abc/x"), "match-00/abc/x,match-01/abc/x,match-0a/abc/x,match-aaa/abc/x"); @@ -234,19 +234,19 @@ TEST(InterPlanetaryFileSystemTest, MatchDirectory) { TEST(InterPlanetaryFileSystemTest, MatchMultipleWildcards) { InterPlanetaryFileSystem ipfs; TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-00/abc/00"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-00/abc/00"), nullptr)); TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-00/abc/01"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-00/abc/01"), nullptr)); TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-00/abc/09"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-00/abc/09"), nullptr)); TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-01/abc/00"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-01/abc/00"), nullptr)); TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-01/abc/04"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-01/abc/04"), nullptr)); TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-01/abc/10"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-01/abc/10"), nullptr)); TF_EXPECT_OK( - ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-02/abc/00"))); + ipfs.RecursivelyCreateDir(ipfs.JoinPath(kPrefix, "match-02/abc/00"), nullptr)); EXPECT_EQ(Match(&ipfs, "match-0[0-1]/abc/0[0-8]"), "match-00/abc/00,match-00/abc/01,match-01/abc/00,match-01/abc/04"); @@ -295,10 +295,10 @@ class TestFileSystem : public NullFileSystem { TEST(TestFileSystemTest, RootDirectory) { TestFileSystem fs; std::vector results; - auto ret = fs.GetMatchingPaths("./te*", &results); + auto ret = fs.GetMatchingPaths("./te*", nullptr, &results); EXPECT_EQ(1, results.size()); EXPECT_EQ("./test", results[0]); - ret = fs.GetMatchingPaths("te*", &results); + ret = fs.GetMatchingPaths("te*", nullptr, &results); EXPECT_EQ(1, results.size()); EXPECT_EQ("./test", results[0]); } diff --git a/tensorflow/core/platform/retrying_file_system.h b/tensorflow/core/platform/retrying_file_system.h index 2f50b6cd5df..ddbf255af2e 100644 --- a/tensorflow/core/platform/retrying_file_system.h +++ b/tensorflow/core/platform/retrying_file_system.h @@ -54,15 +54,17 @@ class RetryingFileSystem : public FileSystem { Status FileExists(const string& fname, TransactionToken* token) override { return RetryingUtils::CallWithRetries( - [this, &fname]() { return base_file_system_->FileExists(fname); }, + [this, &fname, token]() { + return base_file_system_->FileExists(fname, token); + }, retry_config_); } Status GetChildren(const string& dir, TransactionToken* token, std::vector* result) override { return RetryingUtils::CallWithRetries( - [this, &dir, result]() { - return base_file_system_->GetChildren(dir, result); + [this, &dir, result, token]() { + return base_file_system_->GetChildren(dir, token, result); }, retry_config_); } @@ -70,8 +72,8 @@ class RetryingFileSystem : public FileSystem { Status GetMatchingPaths(const string& pattern, TransactionToken* token, std::vector* result) override { return RetryingUtils::CallWithRetries( - [this, &pattern, result]() { - return base_file_system_->GetMatchingPaths(pattern, result); + [this, &pattern, result, token]() { + return base_file_system_->GetMatchingPaths(pattern, token, result); }, retry_config_); } @@ -79,33 +81,41 @@ class RetryingFileSystem : public FileSystem { Status Stat(const string& fname, TransactionToken* token, FileStatistics* stat) override { return RetryingUtils::CallWithRetries( - [this, &fname, stat]() { return base_file_system_->Stat(fname, stat); }, + [this, &fname, stat, token]() { + return base_file_system_->Stat(fname, token, stat); + }, retry_config_); } Status DeleteFile(const string& fname, TransactionToken* token) override { return RetryingUtils::DeleteWithRetries( - [this, &fname]() { return base_file_system_->DeleteFile(fname); }, + [this, &fname, token]() { + return base_file_system_->DeleteFile(fname, token); + }, retry_config_); } Status CreateDir(const string& dirname, TransactionToken* token) override { return RetryingUtils::CallWithRetries( - [this, &dirname]() { return base_file_system_->CreateDir(dirname); }, + [this, &dirname, token]() { + return base_file_system_->CreateDir(dirname, token); + }, retry_config_); } Status DeleteDir(const string& dirname, TransactionToken* token) override { return RetryingUtils::DeleteWithRetries( - [this, &dirname]() { return base_file_system_->DeleteDir(dirname); }, + [this, &dirname, token]() { + return base_file_system_->DeleteDir(dirname, token); + }, retry_config_); } Status GetFileSize(const string& fname, TransactionToken* token, uint64* file_size) override { return RetryingUtils::CallWithRetries( - [this, &fname, file_size]() { - return base_file_system_->GetFileSize(fname, file_size); + [this, &fname, file_size, token]() { + return base_file_system_->GetFileSize(fname, token, file_size); }, retry_config_); } @@ -113,15 +123,17 @@ class RetryingFileSystem : public FileSystem { Status RenameFile(const string& src, const string& target, TransactionToken* token) override { return RetryingUtils::CallWithRetries( - [this, &src, &target]() { - return base_file_system_->RenameFile(src, target); + [this, &src, &target, token]() { + return base_file_system_->RenameFile(src, target, token); }, retry_config_); } Status IsDirectory(const string& dirname, TransactionToken* token) override { return RetryingUtils::CallWithRetries( - [this, &dirname]() { return base_file_system_->IsDirectory(dirname); }, + [this, &dirname, token]() { + return base_file_system_->IsDirectory(dirname, token); + }, retry_config_); } @@ -134,15 +146,15 @@ class RetryingFileSystem : public FileSystem { int64* undeleted_files, int64* undeleted_dirs) override { return RetryingUtils::DeleteWithRetries( - [this, &dirname, undeleted_files, undeleted_dirs]() { - return base_file_system_->DeleteRecursively(dirname, undeleted_files, - undeleted_dirs); + [this, &dirname, token, undeleted_files, undeleted_dirs]() { + return base_file_system_->DeleteRecursively( + dirname, token, undeleted_files, undeleted_dirs); }, retry_config_); } void FlushCaches(TransactionToken* token) override { - base_file_system_->FlushCaches(); + base_file_system_->FlushCaches(token); } Underlying* underlying() const { return base_file_system_.get(); } @@ -229,8 +241,9 @@ Status RetryingFileSystem::NewRandomAccessFile( std::unique_ptr* result) { std::unique_ptr base_file; TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries( - [this, &filename, &base_file]() { - return base_file_system_->NewRandomAccessFile(filename, &base_file); + [this, &filename, &base_file, token]() { + return base_file_system_->NewRandomAccessFile(filename, token, + &base_file); }, retry_config_)); result->reset(new retrying_internals::RetryingRandomAccessFile( @@ -244,8 +257,8 @@ Status RetryingFileSystem::NewWritableFile( std::unique_ptr* result) { std::unique_ptr base_file; TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries( - [this, &filename, &base_file]() { - return base_file_system_->NewWritableFile(filename, &base_file); + [this, &filename, &base_file, token]() { + return base_file_system_->NewWritableFile(filename, token, &base_file); }, retry_config_)); result->reset(new retrying_internals::RetryingWritableFile( @@ -259,8 +272,9 @@ Status RetryingFileSystem::NewAppendableFile( std::unique_ptr* result) { std::unique_ptr base_file; TF_RETURN_IF_ERROR(RetryingUtils::CallWithRetries( - [this, &filename, &base_file]() { - return base_file_system_->NewAppendableFile(filename, &base_file); + [this, &filename, &base_file, token]() { + return base_file_system_->NewAppendableFile(filename, token, + &base_file); }, retry_config_)); result->reset(new retrying_internals::RetryingWritableFile( @@ -273,9 +287,9 @@ Status RetryingFileSystem::NewReadOnlyMemoryRegionFromFile( const string& filename, TransactionToken* token, std::unique_ptr* result) { return RetryingUtils::CallWithRetries( - [this, &filename, result]() { - return base_file_system_->NewReadOnlyMemoryRegionFromFile(filename, - result); + [this, &filename, result, token]() { + return base_file_system_->NewReadOnlyMemoryRegionFromFile( + filename, token, result); }, retry_config_); } diff --git a/tensorflow/core/platform/retrying_file_system_test.cc b/tensorflow/core/platform/retrying_file_system_test.cc index 0cada5a5651..8c8cafbeecd 100644 --- a/tensorflow/core/platform/retrying_file_system_test.cc +++ b/tensorflow/core/platform/retrying_file_system_test.cc @@ -175,8 +175,7 @@ class MockFileSystem : public FileSystem { return calls_.ConsumeNextCall("DeleteRecursively"); } - void FlushCaches( - TransactionToken* token) override { + void FlushCaches(TransactionToken* token) override { if (flushed_) { *flushed_ = true; } @@ -208,7 +207,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_ImmediateSuccess) { // Retrieve the wrapped random access file. std::unique_ptr random_access_file; - TF_EXPECT_OK(fs.NewRandomAccessFile("filename.txt", &random_access_file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("filename.txt", nullptr, &random_access_file)); // Use it and check the results. StringPiece result; @@ -239,7 +239,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_SuccessWith3rdTry) { // Retrieve the wrapped random access file. std::unique_ptr random_access_file; - TF_EXPECT_OK(fs.NewRandomAccessFile("filename.txt", &random_access_file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("filename.txt", nullptr, &random_access_file)); // Use it and check the results. StringPiece result; @@ -264,7 +265,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_AllRetriesFailed) { // Retrieve the wrapped random access file. std::unique_ptr random_access_file; - TF_EXPECT_OK(fs.NewRandomAccessFile("filename.txt", &random_access_file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("filename.txt", nullptr, &random_access_file)); // Use it and check the results. StringPiece result; @@ -294,7 +296,8 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_NoRetriesForSomeErrors) { // Retrieve the wrapped random access file. std::unique_ptr random_access_file; - TF_EXPECT_OK(fs.NewRandomAccessFile("filename.txt", &random_access_file)); + TF_EXPECT_OK( + fs.NewRandomAccessFile("filename.txt", nullptr, &random_access_file)); // Use it and check the results. StringPiece result; @@ -322,7 +325,7 @@ TEST(RetryingFileSystemTest, NewWritableFile_ImmediateSuccess) { // Retrieve the wrapped writable file. std::unique_ptr writable_file; - TF_EXPECT_OK(fs.NewWritableFile("filename.txt", &writable_file)); + TF_EXPECT_OK(fs.NewWritableFile("filename.txt", nullptr, &writable_file)); StringPiece result; TF_EXPECT_OK(writable_file->Name(&result)); @@ -353,7 +356,7 @@ TEST(RetryingFileSystemTest, NewWritableFile_SuccessWith3rdTry) { // Retrieve the wrapped writable file. std::unique_ptr writable_file; - TF_EXPECT_OK(fs.NewWritableFile("filename.txt", &writable_file)); + TF_EXPECT_OK(fs.NewWritableFile("filename.txt", nullptr, &writable_file)); // Use it and check the results. TF_EXPECT_OK(writable_file->Sync()); @@ -380,7 +383,7 @@ TEST(RetryingFileSystemTest, NewWritableFile_SuccessWith3rdTry_ViaDestructor) { // Retrieve the wrapped writable file. std::unique_ptr writable_file; - TF_EXPECT_OK(fs.NewWritableFile("filename.txt", &writable_file)); + TF_EXPECT_OK(fs.NewWritableFile("filename.txt", nullptr, &writable_file)); writable_file.reset(); // Trigger Close() via destructor. } @@ -406,7 +409,7 @@ TEST(RetryingFileSystemTest, NewAppendableFile_SuccessWith3rdTry) { // Retrieve the wrapped appendable file. std::unique_ptr writable_file; - TF_EXPECT_OK(fs.NewAppendableFile("filename.txt", &writable_file)); + TF_EXPECT_OK(fs.NewAppendableFile("filename.txt", nullptr, &writable_file)); // Use it and check the results. TF_EXPECT_OK(writable_file->Sync()); @@ -430,7 +433,7 @@ TEST(RetryingFileSystemTest, NewWritableFile_AllRetriesFailed) { // Retrieve the wrapped writable file. std::unique_ptr writable_file; - TF_EXPECT_OK(fs.NewWritableFile("filename.txt", &writable_file)); + TF_EXPECT_OK(fs.NewWritableFile("filename.txt", nullptr, &writable_file)); // Use it and check the results. const auto& status = writable_file->Sync(); @@ -450,7 +453,8 @@ TEST(RetryingFileSystemTest, std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::unique_ptr result; - TF_EXPECT_OK(fs.NewReadOnlyMemoryRegionFromFile("filename.txt", &result)); + TF_EXPECT_OK( + fs.NewReadOnlyMemoryRegionFromFile("filename.txt", nullptr, &result)); } TEST(RetryingFileSystemTest, NewReadOnlyMemoryRegionFromFile_AllRetriesFailed) { @@ -463,7 +467,7 @@ TEST(RetryingFileSystemTest, NewReadOnlyMemoryRegionFromFile_AllRetriesFailed) { std::unique_ptr result; const auto& status = - fs.NewReadOnlyMemoryRegionFromFile("filename.txt", &result); + fs.NewReadOnlyMemoryRegionFromFile("filename.txt", nullptr, &result); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -479,7 +483,7 @@ TEST(RetryingFileSystemTest, GetChildren_SuccessWith2ndTry) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; - TF_EXPECT_OK(fs.GetChildren("gs://path", &result)); + TF_EXPECT_OK(fs.GetChildren("gs://path", nullptr, &result)); } TEST(RetryingFileSystemTest, GetChildren_AllRetriesFailed) { @@ -490,7 +494,7 @@ TEST(RetryingFileSystemTest, GetChildren_AllRetriesFailed) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; - const auto& status = fs.GetChildren("gs://path", &result); + const auto& status = fs.GetChildren("gs://path", nullptr, &result); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -506,7 +510,7 @@ TEST(RetryingFileSystemTest, GetMatchingPaths_SuccessWith2ndTry) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; - TF_EXPECT_OK(fs.GetMatchingPaths("gs://path/dir", &result)); + TF_EXPECT_OK(fs.GetMatchingPaths("gs://path/dir", nullptr, &result)); } TEST(RetryingFileSystemTest, GetMatchingPaths_AllRetriesFailed) { @@ -518,7 +522,7 @@ TEST(RetryingFileSystemTest, GetMatchingPaths_AllRetriesFailed) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); std::vector result; - const auto& status = fs.GetMatchingPaths("gs://path/dir", &result); + const auto& status = fs.GetMatchingPaths("gs://path/dir", nullptr, &result); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -532,7 +536,7 @@ TEST(RetryingFileSystemTest, DeleteFile_SuccessWith2ndTry) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - TF_EXPECT_OK(fs.DeleteFile("gs://path/file.txt")); + TF_EXPECT_OK(fs.DeleteFile("gs://path/file.txt", nullptr)); } TEST(RetryingFileSystemTest, DeleteFile_AllRetriesFailed) { @@ -542,7 +546,7 @@ TEST(RetryingFileSystemTest, DeleteFile_AllRetriesFailed) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - const auto& status = fs.DeleteFile("gs://path/file.txt"); + const auto& status = fs.DeleteFile("gs://path/file.txt", nullptr); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -556,7 +560,7 @@ TEST(RetryingFileSystemTest, CreateDir_SuccessWith2ndTry) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - TF_EXPECT_OK(fs.CreateDir("gs://path/newdir")); + TF_EXPECT_OK(fs.CreateDir("gs://path/newdir", nullptr)); } TEST(RetryingFileSystemTest, CreateDir_AllRetriesFailed) { @@ -566,7 +570,7 @@ TEST(RetryingFileSystemTest, CreateDir_AllRetriesFailed) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - const auto& status = fs.CreateDir("gs://path/newdir"); + const auto& status = fs.CreateDir("gs://path/newdir", nullptr); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -580,7 +584,7 @@ TEST(RetryingFileSystemTest, DeleteDir_SuccessWith2ndTry) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - TF_EXPECT_OK(fs.DeleteDir("gs://path/dir")); + TF_EXPECT_OK(fs.DeleteDir("gs://path/dir", nullptr)); } TEST(RetryingFileSystemTest, DeleteDir_AllRetriesFailed) { @@ -590,7 +594,7 @@ TEST(RetryingFileSystemTest, DeleteDir_AllRetriesFailed) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - const auto& status = fs.DeleteDir("gs://path/dir"); + const auto& status = fs.DeleteDir("gs://path/dir", nullptr); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -606,7 +610,7 @@ TEST(RetryingFileSystemTest, GetFileSize_SuccessWith2ndTry) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); uint64 size; - TF_EXPECT_OK(fs.GetFileSize("gs://path/file.txt", &size)); + TF_EXPECT_OK(fs.GetFileSize("gs://path/file.txt", nullptr, &size)); } TEST(RetryingFileSystemTest, GetFileSize_AllRetriesFailed) { @@ -617,7 +621,7 @@ TEST(RetryingFileSystemTest, GetFileSize_AllRetriesFailed) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); uint64 size; - const auto& status = fs.GetFileSize("gs://path/file.txt", &size); + const auto& status = fs.GetFileSize("gs://path/file.txt", nullptr, &size); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -631,7 +635,7 @@ TEST(RetryingFileSystemTest, RenameFile_SuccessWith2ndTry) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - TF_EXPECT_OK(fs.RenameFile("old_name", "new_name")); + TF_EXPECT_OK(fs.RenameFile("old_name", "new_name", nullptr)); } TEST(RetryingFileSystemTest, RenameFile_AllRetriesFailed) { @@ -641,7 +645,7 @@ TEST(RetryingFileSystemTest, RenameFile_AllRetriesFailed) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - const auto& status = fs.RenameFile("old_name", "new_name"); + const auto& status = fs.RenameFile("old_name", "new_name", nullptr); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -656,7 +660,7 @@ TEST(RetryingFileSystemTest, Stat_SuccessWith2ndTry) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); FileStatistics stat; - TF_EXPECT_OK(fs.Stat("file_name", &stat)); + TF_EXPECT_OK(fs.Stat("file_name", nullptr, &stat)); } TEST(RetryingFileSystemTest, Stat_AllRetriesFailed) { @@ -667,7 +671,7 @@ TEST(RetryingFileSystemTest, Stat_AllRetriesFailed) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); FileStatistics stat; - const auto& status = fs.Stat("file_name", &stat); + const auto& status = fs.Stat("file_name", nullptr, &stat); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -679,7 +683,7 @@ TEST(RetryingFileSystemTest, FileExists_AllRetriesFailed) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - const auto& status = fs.FileExists("file_name"); + const auto& status = fs.FileExists("file_name", nullptr); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -693,7 +697,7 @@ TEST(RetryingFileSystemTest, FileExists_SuccessWith2ndTry) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - TF_EXPECT_OK(fs.FileExists("gs://path/dir")); + TF_EXPECT_OK(fs.FileExists("gs://path/dir", nullptr)); } TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) { @@ -706,7 +710,7 @@ TEST(RetryingFileSystemTest, IsDirectory_SuccessWith2ndTry) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - TF_EXPECT_OK(fs.IsDirectory("gs://path/dir")); + TF_EXPECT_OK(fs.IsDirectory("gs://path/dir", nullptr)); } TEST(RetryingFileSystemTest, IsDirectory_AllRetriesFailed) { @@ -716,7 +720,7 @@ TEST(RetryingFileSystemTest, IsDirectory_AllRetriesFailed) { RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - const auto& status = fs.IsDirectory("gs://path/dir"); + const auto& status = fs.IsDirectory("gs://path/dir", nullptr); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -732,8 +736,8 @@ TEST(RetryingFileSystemTest, DeleteRecursively_SuccessWith2ndTry) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); int64 undeleted_files, undeleted_dirs; - TF_EXPECT_OK( - fs.DeleteRecursively("gs://path/dir", &undeleted_files, &undeleted_dirs)); + TF_EXPECT_OK(fs.DeleteRecursively("gs://path/dir", nullptr, &undeleted_files, + &undeleted_dirs)); } TEST(RetryingFileSystemTest, DeleteRecursively_AllRetriesFailed) { @@ -745,8 +749,8 @@ TEST(RetryingFileSystemTest, DeleteRecursively_AllRetriesFailed) { std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); int64 undeleted_files, undeleted_dirs; - const auto& status = - fs.DeleteRecursively("gs://path/dir", &undeleted_files, &undeleted_dirs); + const auto& status = fs.DeleteRecursively("gs://path/dir", nullptr, + &undeleted_files, &undeleted_dirs); EXPECT_TRUE(absl::StrContains(status.error_message(), "Retriable error #10")) << status; } @@ -757,7 +761,7 @@ TEST(RetryingFileSystemTest, FlushCaches) { std::unique_ptr base_fs(new MockFileSystem(none, &flushed)); RetryingFileSystem fs( std::move(base_fs), RetryConfig(0 /* init_delay_time_us */)); - fs.FlushCaches(); + fs.FlushCaches(nullptr); EXPECT_TRUE(flushed); } diff --git a/tensorflow/core/platform/s3/s3_file_system.cc b/tensorflow/core/platform/s3/s3_file_system.cc index 8812424e89d..201694c994c 100644 --- a/tensorflow/core/platform/s3/s3_file_system.cc +++ b/tensorflow/core/platform/s3/s3_file_system.cc @@ -58,7 +58,7 @@ static const char* kS3TempFileTemplate = "/tmp/s3_filesystem_XXXXXX"; #endif static const char* kS3FileSystemAllocationTag = "S3FileSystemAllocation"; static const size_t kS3ReadAppendableFileBufferSize = 1024 * 1024; -static const int64 kS3TimeoutMsec = 300000; // 5 min +static const int64 kS3TimeoutMsec = 300000; // 5 min static const uint64 kS3MultiPartUploadChunkSize = 50 * 1024 * 1024; // 50 MB static const uint64 kS3MultiPartDownloadChunkSize = 2 * 1024 * 1024; // 50 MB static const int kS3GetChildrenMaxKeys = 100; @@ -568,14 +568,14 @@ S3FileSystem::GetExecutor() { } Status S3FileSystem::NewRandomAccessFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { - return NewRandomAccessFile(fname, result, true); + const string& fname, TransactionToken* token, + std::unique_ptr* result) { + return NewRandomAccessFile(fname, token, result, true); } Status S3FileSystem::NewRandomAccessFile( - const string& fname, std::unique_ptr* result, - bool use_multi_part_download /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result, bool use_multi_part_download) { string bucket, object; TF_RETURN_IF_ERROR(ParseS3Path(fname, false, &bucket, &object)); @@ -588,9 +588,9 @@ Status S3FileSystem::NewRandomAccessFile( return Status::OK(); } -Status S3FileSystem::NewWritableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { +Status S3FileSystem::NewWritableFile(const string& fname, + TransactionToken* token, + std::unique_ptr* result) { string bucket, object; TF_RETURN_IF_ERROR(ParseS3Path(fname, false, &bucket, &object)); result->reset(new S3WritableFile( @@ -601,11 +601,11 @@ Status S3FileSystem::NewWritableFile( return Status::OK(); } -Status S3FileSystem::NewAppendableFile( - const string& fname, - std::unique_ptr* result /*, TransactionToken* token */) { +Status S3FileSystem::NewAppendableFile(const string& fname, + TransactionToken* token, + std::unique_ptr* result) { std::unique_ptr reader; - TF_RETURN_IF_ERROR(NewRandomAccessFile(fname, &reader)); + TF_RETURN_IF_ERROR(NewRandomAccessFile(fname, token, &reader)); std::unique_ptr buffer(new char[kS3ReadAppendableFileBufferSize]); Status status; uint64 offset = 0; @@ -637,14 +637,14 @@ Status S3FileSystem::NewAppendableFile( } Status S3FileSystem::NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* - result /*, TransactionToken* token */) { + const string& fname, TransactionToken* token, + std::unique_ptr* result) { uint64 size; - TF_RETURN_IF_ERROR(GetFileSize(fname, &size)); + TF_RETURN_IF_ERROR(GetFileSize(fname, token, &size)); std::unique_ptr data(new char[size]); std::unique_ptr file; - TF_RETURN_IF_ERROR(NewRandomAccessFile(fname, &file)); + TF_RETURN_IF_ERROR(NewRandomAccessFile(fname, token, &file)); StringPiece piece; TF_RETURN_IF_ERROR(file->Read(0, size, &piece, data.get())); @@ -653,16 +653,14 @@ Status S3FileSystem::NewReadOnlyMemoryRegionFromFile( return Status::OK(); } -Status S3FileSystem::FileExists( - const string& fname /*, TransactionToken* token */) { +Status S3FileSystem::FileExists(const string& fname, TransactionToken* token) { FileStatistics stats; - TF_RETURN_IF_ERROR(this->Stat(fname, &stats)); + TF_RETURN_IF_ERROR(this->Stat(fname, token, &stats)); return Status::OK(); } -Status S3FileSystem::GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token */) { +Status S3FileSystem::GetChildren(const string& dir, TransactionToken* token, + std::vector* result) { VLOG(1) << "GetChildren for path: " << dir; string bucket, prefix; TF_RETURN_IF_ERROR(ParseS3Path(dir, true, &bucket, &prefix)); @@ -709,8 +707,8 @@ Status S3FileSystem::GetChildren( return Status::OK(); } -Status S3FileSystem::Stat( - const string& fname, FileStatistics* stats /*, TransactionToken* token */) { +Status S3FileSystem::Stat(const string& fname, TransactionToken* token, + FileStatistics* stats) { VLOG(1) << "Stat on path: " << fname; string bucket, object; TF_RETURN_IF_ERROR(ParseS3Path(fname, true, &bucket, &object)); @@ -772,14 +770,13 @@ Status S3FileSystem::Stat( return Status::OK(); } -Status S3FileSystem::GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token */) { +Status S3FileSystem::GetMatchingPaths(const string& pattern, + TransactionToken* token, + std::vector* results) { return internal::GetMatchingPaths(this, Env::Default(), pattern, results); } -Status S3FileSystem::DeleteFile( - const string& fname /*, TransactionToken* token */) { +Status S3FileSystem::DeleteFile(const string& fname, TransactionToken* token) { VLOG(1) << "DeleteFile: " << fname; string bucket, object; TF_RETURN_IF_ERROR(ParseS3Path(fname, false, &bucket, &object)); @@ -795,8 +792,7 @@ Status S3FileSystem::DeleteFile( return Status::OK(); } -Status S3FileSystem::CreateDir( - const string& dirname /*, TransactionToken* token */) { +Status S3FileSystem::CreateDir(const string& dirname, TransactionToken* token) { VLOG(1) << "CreateDir: " << dirname; string bucket, object; TF_RETURN_IF_ERROR(ParseS3Path(dirname, true, &bucket, &object)); @@ -815,16 +811,15 @@ Status S3FileSystem::CreateDir( if (filename.back() != '/') { filename.push_back('/'); } - if (!this->FileExists(filename).ok()) { + if (!this->FileExists(filename,token).ok()) { std::unique_ptr file; - TF_RETURN_IF_ERROR(NewWritableFile(filename, &file)); + TF_RETURN_IF_ERROR(NewWritableFile(filename,token, &file)); TF_RETURN_IF_ERROR(file->Close()); } return Status::OK(); } -Status S3FileSystem::DeleteDir( - const string& dirname /*, TransactionToken* token */) { +Status S3FileSystem::DeleteDir(const string& dirname, TransactionToken* token) { VLOG(1) << "DeleteDir: " << dirname; string bucket, object; TF_RETURN_IF_ERROR(ParseS3Path(dirname, false, &bucket, &object)); @@ -855,7 +850,7 @@ Status S3FileSystem::DeleteDir( if (filename.back() != '/') { filename.push_back('/'); } - return DeleteFile(filename); + return DeleteFile(filename,token); } } else { TF_RETURN_IF_ERROR(CheckForbiddenError(listObjectsOutcome.GetError())); @@ -863,10 +858,10 @@ Status S3FileSystem::DeleteDir( return Status::OK(); } -Status S3FileSystem::GetFileSize( - const string& fname, uint64* file_size /*, TransactionToken* token */) { +Status S3FileSystem::GetFileSize(const string& fname, TransactionToken* token, + uint64* file_size) { FileStatistics stats; - TF_RETURN_IF_ERROR(this->Stat(fname, &stats)); + TF_RETURN_IF_ERROR(this->Stat(fname, token, &stats)); *file_size = stats.length; return Status::OK(); } @@ -917,7 +912,7 @@ Status S3FileSystem::CopyFile(const Aws::String& source_bucket, Aws::String source_full_path = Aws::String("s3://") + source; uint64 file_length; TF_RETURN_IF_ERROR( - this->GetFileSize(string(source_full_path.c_str()), &file_length)); + this->GetFileSize(string(source_full_path.c_str()), nullptr, &file_length)); int num_parts; if (file_length <= multi_part_chunk_size_[Aws::Transfer::TransferDirection::UPLOAD]) { @@ -1135,8 +1130,8 @@ Status S3FileSystem::CompleteMultiPartCopy( return Status::OK(); } -Status S3FileSystem::RenameFile( - const string& src, const string& target /*, TransactionToken* token */) { +Status S3FileSystem::RenameFile(const string& src, const string& target, + TransactionToken* token) { VLOG(1) << "RenameFile from: " << src << " to: " << target; string src_bucket, src_object, target_bucket, target_object; TF_RETURN_IF_ERROR(ParseS3Path(src, false, &src_bucket, &src_object)); diff --git a/tensorflow/core/platform/s3/s3_file_system.h b/tensorflow/core/platform/s3/s3_file_system.h index 41a5195efec..e592a174183 100644 --- a/tensorflow/core/platform/s3/s3_file_system.h +++ b/tensorflow/core/platform/s3/s3_file_system.h @@ -50,66 +50,66 @@ class S3FileSystem : public FileSystem { ~S3FileSystem(); Status NewRandomAccessFile( - const string& fname, + const string& fname, TransactionToken * token, std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + result ) override; Status NewRandomAccessFile( - const string& fname, std::unique_ptr* result, - bool use_multi_part_download /*, TransactionToken* token = nullptr */); + const string& fname, TransactionToken * token,std::unique_ptr* result, + bool use_multi_part_download ); Status NewWritableFile( - const string& fname, + const string& fname,TransactionToken * token, std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + result ) override; Status NewAppendableFile( - const string& fname, + const string& fname,TransactionToken * token, std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + result ) override; Status NewReadOnlyMemoryRegionFromFile( - const string& fname, + const string& fname,TransactionToken * token, std::unique_ptr* - result /*, TransactionToken* token = nullptr */) override; + result ) override; Status FileExists( - const string& fname /*, TransactionToken* token = nullptr */) override; + const string& fname,TransactionToken * token ) override; Status GetChildren( - const string& dir, - std::vector* result /*, TransactionToken* token = nullptr */) + const string& dir,TransactionToken * token, + std::vector* result ) override; Status Stat( - const string& fname, - FileStatistics* stat /*, TransactionToken* token = nullptr */) override; + const string& fname,TransactionToken * token, + FileStatistics* stat ) override; Status GetMatchingPaths( - const string& pattern, - std::vector* results /*, TransactionToken* token = nullptr */) + const string& pattern,TransactionToken * token, + std::vector* results ) override; Status DeleteFile( - const string& fname /*, TransactionToken* token = nullptr */) override; + const string& fname,TransactionToken * token ) override; Status CreateDir( - const string& name /*, TransactionToken* token = nullptr */) override; + const string& name, TransactionToken * token) override; Status DeleteDir( - const string& name /*, TransactionToken* token = nullptr */) override; + const string& name,TransactionToken * token ) override; Status GetFileSize( - const string& fname, - uint64* size /*, TransactionToken* token = nullptr */) override; + const string& fname,TransactionToken * token, + uint64* size ) override; Status RenameFile( const string& src, - const string& target /*, TransactionToken* token = nullptr */) override; + const string& target,TransactionToken * token ) override; Status HasAtomicMove( const string& path, - bool* has_atomic_move /*, TransactionToken* token = nullptr */) override; + bool* has_atomic_move ) override; private: // Returns the member S3 client, initializing as-needed. From 8e75f3b993504c0090ab851c8b4313a8bbbd747a Mon Sep 17 00:00:00 2001 From: Sami Kama Date: Tue, 4 Aug 2020 23:20:15 -0700 Subject: [PATCH 0437/1017] Add TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT macro to modified classes --- .../filesystem/modular_filesystem.h | 2 + .../common_runtime/constant_folding_test.cc | 5 +- .../kernels/immutable_constant_op_test.cc | 4 + .../core/platform/cloud/gcs_file_system.h | 2 + tensorflow/core/platform/env_test.cc | 8 +- tensorflow/core/platform/file_system.h | 51 ++++++------- tensorflow/core/platform/file_system_test.cc | 2 + tensorflow/core/platform/null_file_system.h | 2 + tensorflow/core/platform/ram_file_system.h | 2 + .../core/platform/retrying_file_system.h | 2 + .../platform/retrying_file_system_test.cc | 2 + tensorflow/core/platform/s3/s3_file_system.cc | 10 +-- tensorflow/core/platform/s3/s3_file_system.h | 73 +++++++------------ 13 files changed, 84 insertions(+), 81 deletions(-) diff --git a/tensorflow/c/experimental/filesystem/modular_filesystem.h b/tensorflow/c/experimental/filesystem/modular_filesystem.h index 6495d97ebf1..061a1aa446b 100644 --- a/tensorflow/c/experimental/filesystem/modular_filesystem.h +++ b/tensorflow/c/experimental/filesystem/modular_filesystem.h @@ -59,6 +59,8 @@ class ModularFileSystem final : public FileSystem { ~ModularFileSystem() override { ops_->cleanup(filesystem_.get()); } + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( const std::string& fname, TransactionToken* token, std::unique_ptr* result) override; diff --git a/tensorflow/core/common_runtime/constant_folding_test.cc b/tensorflow/core/common_runtime/constant_folding_test.cc index b348117bb9e..2edc92eac5e 100644 --- a/tensorflow/core/common_runtime/constant_folding_test.cc +++ b/tensorflow/core/common_runtime/constant_folding_test.cc @@ -19,9 +19,6 @@ limitations under the License. #include #include "tensorflow/cc/ops/nn_ops.h" -#include "tensorflow/core/common_runtime/constant_folding.h" - -#include "tensorflow/cc/ops/array_ops_internal.h" #include "tensorflow/cc/ops/sendrecv_ops.h" #include "tensorflow/cc/ops/standard_ops.h" #include "tensorflow/core/common_runtime/device.h" @@ -687,6 +684,8 @@ class TestTFFileSystem : public ::tensorflow::NullFileSystem { : ::tensorflow::NullFileSystem(), data_tensor_(test::AsTensor({1., 2., 3., 4.}, {2, 2})) {} + using ::tensorflow::NullFileSystem::NewReadOnlyMemoryRegionFromFile; + ::tensorflow::Status NewReadOnlyMemoryRegionFromFile( const string& fname, ::tensorflow::TransactionToken* token, std::unique_ptr<::tensorflow::ReadOnlyMemoryRegion>* result) override { diff --git a/tensorflow/core/kernels/immutable_constant_op_test.cc b/tensorflow/core/kernels/immutable_constant_op_test.cc index 5c3f96a312d..d52a8b55a35 100644 --- a/tensorflow/core/kernels/immutable_constant_op_test.cc +++ b/tensorflow/core/kernels/immutable_constant_op_test.cc @@ -60,6 +60,10 @@ class TestReadOnlyMemoryRegion : public ReadOnlyMemoryRegion { class TestFileSystem : public NullFileSystem { public: ~TestFileSystem() override = default; + + // import non-transactional method from the base class + using NullFileSystem::NewReadOnlyMemoryRegionFromFile; + Status NewReadOnlyMemoryRegionFromFile( const string& fname, TransactionToken* token, std::unique_ptr* result) override { diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h index 0a27aba35c1..203c501ff4c 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.h +++ b/tensorflow/core/platform/cloud/gcs_file_system.h @@ -125,6 +125,8 @@ class GcsFileSystem : public FileSystem { std::pair* additional_header, bool compose_append); + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( const string& fname, TransactionToken* token, std::unique_ptr* result) override; diff --git a/tensorflow/core/platform/env_test.cc b/tensorflow/core/platform/env_test.cc index 35374d65ee3..79d793ee636 100644 --- a/tensorflow/core/platform/env_test.cc +++ b/tensorflow/core/platform/env_test.cc @@ -295,7 +295,9 @@ TEST_F(DefaultEnvTest, SleepForMicroseconds) { class TmpDirFileSystem : public NullFileSystem { public: - Status FileExists(const string& dir,TransactionToken* token) override { + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + + Status FileExists(const string& dir, TransactionToken* token) override { StringPiece scheme, host, path; io::ParseURI(dir, &scheme, &host, &path); if (path.empty()) return errors::NotFound(dir, " not found"); @@ -311,7 +313,7 @@ class TmpDirFileSystem : public NullFileSystem { return Env::Default()->FileExists(io::JoinPath(BaseDir(), path)); } - Status CreateDir(const string& dir,TransactionToken* token) override { + Status CreateDir(const string& dir, TransactionToken* token) override { StringPiece scheme, host, path; io::ParseURI(dir, &scheme, &host, &path); if (scheme != "tmpdirfs") { @@ -328,7 +330,7 @@ class TmpDirFileSystem : public NullFileSystem { return status; } - Status IsDirectory(const string& dir,TransactionToken* token) override { + Status IsDirectory(const string& dir, TransactionToken* token) override { StringPiece scheme, host, path; io::ParseURI(dir, &scheme, &host, &path); for (const auto& existing_dir : created_directories_) diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h index c4094d3a5a2..28d09c39db1 100644 --- a/tensorflow/core/platform/file_system.h +++ b/tensorflow/core/platform/file_system.h @@ -518,6 +518,30 @@ class FileSystem { virtual ~FileSystem() = default; }; +/// This macro adds forwarding methods from FileSystem class to +/// used class since name hiding will prevent these to be accessed from +/// derived classes and would require all use locations to migrate to +/// Transactional API. This is an interim solution until ModularFileSystem class +/// becomes a singleton. +// TODO(sami): Remove this macro when filesystem plugins migration is complete. +#define TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT \ + using FileSystem::NewRandomAccessFile; \ + using FileSystem::NewWritableFile; \ + using FileSystem::NewAppendableFile; \ + using FileSystem::NewReadOnlyMemoryRegionFromFile; \ + using FileSystem::FileExists; \ + using FileSystem::GetChildren; \ + using FileSystem::GetMatchingPaths; \ + using FileSystem::Stat; \ + using FileSystem::DeleteFile; \ + using FileSystem::RecursivelyCreateDir; \ + using FileSystem::DeleteDir; \ + using FileSystem::DeleteRecursively; \ + using FileSystem::GetFileSize; \ + using FileSystem::RenameFile; \ + using FileSystem::CopyFile; \ + using FileSystem::IsDirectory; \ + using FileSystem::FlushCaches /// A Wrapper class for Transactional FileSystem support. /// This provides means to make use of the transactions with minimal code change @@ -529,6 +553,8 @@ class FileSystem { /// transactional filesystem access with minimal code change. class WrappedFileSystem : public FileSystem { public: + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + tensorflow::Status NewRandomAccessFile( const std::string& fname, TransactionToken* token, std::unique_ptr* result) override { @@ -691,31 +717,6 @@ class WrappedFileSystem : public FileSystem { TransactionToken* token_; }; -/// This macro adds forwarding methods from FileSystem class to -/// used class since name hiding will prevent these to be accessed from -/// derived classes and would require all use locations to migrate to -/// Transactional API. This is an interim solution until ModularFileSystem class -/// becomes a singleton. -// TODO(sami): Remove this macro when filesystem plugins migration is complete. -#define TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT \ - using FileSystem::NewRandomAccessFile; \ - using FileSystem::NewWritableFile; \ - using FileSystem::NewAppendableFile; \ - using FileSystem::NewReadOnlyMemoryRegionFromFile; \ - using FileSystem::FileExists; \ - using FileSystem::GetChildren; \ - using FileSystem::GetMatchingPaths; \ - using FileSystem::Stat; \ - using FileSystem::DeleteFile; \ - using FileSystem::RecursivelyCreateDir; \ - using FileSystem::DeleteDir; \ - using FileSystem::DeleteRecursively; \ - using FileSystem::GetFileSize; \ - using FileSystem::RenameFile; \ - using FileSystem::CopyFile; \ - using FileSystem::IsDirectory; \ - using FileSystem::FlushCaches - /// A file abstraction for randomly reading the contents of a file. class RandomAccessFile { public: diff --git a/tensorflow/core/platform/file_system_test.cc b/tensorflow/core/platform/file_system_test.cc index 1e23a2b853c..1707fce4bb5 100644 --- a/tensorflow/core/platform/file_system_test.cc +++ b/tensorflow/core/platform/file_system_test.cc @@ -32,6 +32,8 @@ static const char* const kPrefix = "ipfs://solarsystem"; // cannot have children further. class InterPlanetaryFileSystem : public NullFileSystem { public: + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status FileExists(const string& fname, TransactionToken* token) override { string parsed_path; ParsePath(fname, &parsed_path); diff --git a/tensorflow/core/platform/null_file_system.h b/tensorflow/core/platform/null_file_system.h index 0af34258169..d7deca32da2 100644 --- a/tensorflow/core/platform/null_file_system.h +++ b/tensorflow/core/platform/null_file_system.h @@ -36,6 +36,8 @@ class NullFileSystem : public FileSystem { ~NullFileSystem() override = default; + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( const string& fname, TransactionToken* token, std::unique_ptr* result) override { diff --git a/tensorflow/core/platform/ram_file_system.h b/tensorflow/core/platform/ram_file_system.h index ba8bb2d7630..407bcb3ba0f 100644 --- a/tensorflow/core/platform/ram_file_system.h +++ b/tensorflow/core/platform/ram_file_system.h @@ -103,6 +103,8 @@ class RamRandomAccessFile : public RandomAccessFile, public WritableFile { class RamFileSystem : public FileSystem { public: + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( const string& fname, TransactionToken* token, std::unique_ptr* result) override { diff --git a/tensorflow/core/platform/retrying_file_system.h b/tensorflow/core/platform/retrying_file_system.h index ddbf255af2e..52e2caf8398 100644 --- a/tensorflow/core/platform/retrying_file_system.h +++ b/tensorflow/core/platform/retrying_file_system.h @@ -38,6 +38,8 @@ class RetryingFileSystem : public FileSystem { : base_file_system_(std::move(base_file_system)), retry_config_(retry_config) {} + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( const string& filename, TransactionToken* token, std::unique_ptr* result) override; diff --git a/tensorflow/core/platform/retrying_file_system_test.cc b/tensorflow/core/platform/retrying_file_system_test.cc index 8c8cafbeecd..093b85a1afc 100644 --- a/tensorflow/core/platform/retrying_file_system_test.cc +++ b/tensorflow/core/platform/retrying_file_system_test.cc @@ -99,6 +99,8 @@ class MockFileSystem : public FileSystem { explicit MockFileSystem(const ExpectedCalls& calls, bool* flushed = nullptr) : calls_(calls), flushed_(flushed) {} + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; + Status NewRandomAccessFile( const string& fname, TransactionToken* token, std::unique_ptr* result) override { diff --git a/tensorflow/core/platform/s3/s3_file_system.cc b/tensorflow/core/platform/s3/s3_file_system.cc index 201694c994c..8d74ea6aff6 100644 --- a/tensorflow/core/platform/s3/s3_file_system.cc +++ b/tensorflow/core/platform/s3/s3_file_system.cc @@ -811,9 +811,9 @@ Status S3FileSystem::CreateDir(const string& dirname, TransactionToken* token) { if (filename.back() != '/') { filename.push_back('/'); } - if (!this->FileExists(filename,token).ok()) { + if (!this->FileExists(filename, token).ok()) { std::unique_ptr file; - TF_RETURN_IF_ERROR(NewWritableFile(filename,token, &file)); + TF_RETURN_IF_ERROR(NewWritableFile(filename, token, &file)); TF_RETURN_IF_ERROR(file->Close()); } return Status::OK(); @@ -850,7 +850,7 @@ Status S3FileSystem::DeleteDir(const string& dirname, TransactionToken* token) { if (filename.back() != '/') { filename.push_back('/'); } - return DeleteFile(filename,token); + return DeleteFile(filename, token); } } else { TF_RETURN_IF_ERROR(CheckForbiddenError(listObjectsOutcome.GetError())); @@ -911,8 +911,8 @@ Status S3FileSystem::CopyFile(const Aws::String& source_bucket, Aws::String source = Aws::String((source_bucket + "/" + source_key).c_str()); Aws::String source_full_path = Aws::String("s3://") + source; uint64 file_length; - TF_RETURN_IF_ERROR( - this->GetFileSize(string(source_full_path.c_str()), nullptr, &file_length)); + TF_RETURN_IF_ERROR(this->GetFileSize(string(source_full_path.c_str()), + nullptr, &file_length)); int num_parts; if (file_length <= multi_part_chunk_size_[Aws::Transfer::TransferDirection::UPLOAD]) { diff --git a/tensorflow/core/platform/s3/s3_file_system.h b/tensorflow/core/platform/s3/s3_file_system.h index e592a174183..8da74c668d1 100644 --- a/tensorflow/core/platform/s3/s3_file_system.h +++ b/tensorflow/core/platform/s3/s3_file_system.h @@ -49,67 +49,50 @@ class S3FileSystem : public FileSystem { S3FileSystem(); ~S3FileSystem(); - Status NewRandomAccessFile( - const string& fname, TransactionToken * token, - std::unique_ptr* - result ) override; + TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; Status NewRandomAccessFile( - const string& fname, TransactionToken * token,std::unique_ptr* result, - bool use_multi_part_download ); + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status NewWritableFile( - const string& fname,TransactionToken * token, - std::unique_ptr* - result ) override; + Status NewRandomAccessFile(const string& fname, TransactionToken* token, + std::unique_ptr* result, + bool use_multi_part_download); - Status NewAppendableFile( - const string& fname,TransactionToken * token, - std::unique_ptr* - result ) override; + Status NewWritableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; + + Status NewAppendableFile(const string& fname, TransactionToken* token, + std::unique_ptr* result) override; Status NewReadOnlyMemoryRegionFromFile( - const string& fname,TransactionToken * token, - std::unique_ptr* - result ) override; + const string& fname, TransactionToken* token, + std::unique_ptr* result) override; - Status FileExists( - const string& fname,TransactionToken * token ) override; + Status FileExists(const string& fname, TransactionToken* token) override; - Status GetChildren( - const string& dir,TransactionToken * token, - std::vector* result ) - override; + Status GetChildren(const string& dir, TransactionToken* token, + std::vector* result) override; - Status Stat( - const string& fname,TransactionToken * token, - FileStatistics* stat ) override; + Status Stat(const string& fname, TransactionToken* token, + FileStatistics* stat) override; - Status GetMatchingPaths( - const string& pattern,TransactionToken * token, - std::vector* results ) - override; + Status GetMatchingPaths(const string& pattern, TransactionToken* token, + std::vector* results) override; - Status DeleteFile( - const string& fname,TransactionToken * token ) override; + Status DeleteFile(const string& fname, TransactionToken* token) override; - Status CreateDir( - const string& name, TransactionToken * token) override; + Status CreateDir(const string& name, TransactionToken* token) override; - Status DeleteDir( - const string& name,TransactionToken * token ) override; + Status DeleteDir(const string& name, TransactionToken* token) override; - Status GetFileSize( - const string& fname,TransactionToken * token, - uint64* size ) override; + Status GetFileSize(const string& fname, TransactionToken* token, + uint64* size) override; - Status RenameFile( - const string& src, - const string& target,TransactionToken * token ) override; + Status RenameFile(const string& src, const string& target, + TransactionToken* token) override; - Status HasAtomicMove( - const string& path, - bool* has_atomic_move ) override; + Status HasAtomicMove(const string& path, bool* has_atomic_move) override; private: // Returns the member S3 client, initializing as-needed. From 0b84561324672ee3045ee23112dbc68540b39cdc Mon Sep 17 00:00:00 2001 From: Eugene Zhulenev Date: Tue, 4 Aug 2020 23:19:26 -0700 Subject: [PATCH 0438/1017] [MLIR:TF] Fix a bug in binary out of concat hoisting PiperOrigin-RevId: 324962240 Change-Id: I77aa3559a3a7244ee92e952c384a86a655bf4e4b --- tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc index 791323ca992..5c19f9c3daa 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc @@ -682,8 +682,8 @@ HoistCwiseBinaryOutOfConcat::GetHoistParams(TF::ConcatV2Op op, // of `axis + 1` rank and axis dim has size `1`. auto is_all_tensors = [&](int operand_idx, int axis) -> bool { return llvm::all_of(op.values(), [&](Value arg) -> bool { - auto lhs = arg.getDefiningOp()->getOperand(operand_idx); - auto ranked = lhs.getType().dyn_cast(); + auto operand = arg.getDefiningOp()->getOperand(operand_idx); + auto ranked = operand.getType().dyn_cast(); return ranked && ranked.getRank() == (axis + 1) && ranked.getShape()[axis] == 1; }); @@ -692,13 +692,14 @@ HoistCwiseBinaryOutOfConcat::GetHoistParams(TF::ConcatV2Op op, // Returns true if all binary ops operands at `operand_idx` index are scalars. auto is_all_scalars = [&](int operand_idx) -> bool { return llvm::all_of(op.values(), [&](Value arg) -> bool { - auto lhs = arg.getDefiningOp()->getOperand(operand_idx); - auto ranked = lhs.getType().dyn_cast(); + auto operand = arg.getDefiningOp()->getOperand(operand_idx); + auto ranked = operand.getType().dyn_cast(); return ranked && ranked.hasRank() && ranked.getRank() == 0; }); }; - auto ranked = op.getType().cast(); + // Concat result type must be a ranked tensor. + auto ranked = op.getType().dyn_cast(); if (!ranked) return None; // TODO(ezhulenev): Add support for more valid concat patterns. From b89e12c5a3e4a95804f30bc1426e3e9b9d20570d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 5 Aug 2020 01:12:02 -0700 Subject: [PATCH 0439/1017] This is a near no-op. PiperOrigin-RevId: 324973737 Change-Id: Id8a50959b16d9e5f1eff5471d978a9568cace521 --- tensorflow/lite/g3doc/guide/codegen.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/lite/g3doc/guide/codegen.md b/tensorflow/lite/g3doc/guide/codegen.md index b74bfc5ed40..84dd2ffade9 100644 --- a/tensorflow/lite/g3doc/guide/codegen.md +++ b/tensorflow/lite/g3doc/guide/codegen.md @@ -171,7 +171,7 @@ generated by the Android Studio ML Model Binding. ## Read the metadata from models -The Metadata Extractor library is a convinient tool to read the metadata and +The Metadata Extractor library is a convenient tool to read the metadata and associated files from a models across different platforms (see the [Java version](https://github.com/tensorflow/tflite-support/tree/master/tensorflow_lite_support/metadata) and the C++ version is coming soon). Users can also build their own metadata @@ -198,7 +198,7 @@ information. As long as the file identifer is satisfied, the metadata extractor will not fail when reading metadata generated from an old or a future scheme due to the Flatbuffers forward and backwards compatibility mechanism. But fields from -future shcemas cannot be extracted by older metadata extractors. The +future schemas cannot be extracted by older metadata extractors. The [minimum necessary parser version](../convert/metadata.md#the-minimum-necessary-metadata-parser-version) of the metadata indicates the minimum version of metadata parser that can read the metadata Flatbuffers in full. You can use the following method to verify if From 5f6c13cb1081be062638fee8d32c3788afc940ba Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 5 Aug 2020 01:38:50 -0700 Subject: [PATCH 0440/1017] [XLA] Add TopK rewriter pass This pass pattern matches sort HLOs into a custom call. This will be useful for CPU. PiperOrigin-RevId: 324976268 Change-Id: I56224ad39e1cb2960bde9a366a7b47deffa9955f --- tensorflow/compiler/xla/service/BUILD | 31 +++ .../compiler/xla/service/topk_rewriter.cc | 187 ++++++++++++++++++ .../compiler/xla/service/topk_rewriter.h | 44 +++++ .../xla/service/topk_rewriter_test.cc | 153 ++++++++++++++ 4 files changed, 415 insertions(+) create mode 100644 tensorflow/compiler/xla/service/topk_rewriter.cc create mode 100644 tensorflow/compiler/xla/service/topk_rewriter.h create mode 100644 tensorflow/compiler/xla/service/topk_rewriter_test.cc diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 4d15bc432a2..49431b19a69 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -4985,3 +4985,34 @@ cc_library( "//tensorflow/stream_executor/lib", ], ) + +cc_library( + name = "topk_rewriter", + srcs = ["topk_rewriter.cc"], + hdrs = ["topk_rewriter.h"], + deps = [ + ":hlo", + ":hlo_casting_utils", + ":hlo_pass", + ":pattern_matcher", + "//tensorflow/compiler/xla:shape_util", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/types:optional", + ], +) + +tf_cc_test( + name = "topk_rewriter_test", + srcs = ["topk_rewriter_test.cc"], + deps = [ + ":hlo", + ":hlo_dce", + ":hlo_matchers", + ":topk_rewriter", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:test_macros_cpu", + "//tensorflow/compiler/xla/tests:test_utils", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/core:test", + ], +) diff --git a/tensorflow/compiler/xla/service/topk_rewriter.cc b/tensorflow/compiler/xla/service/topk_rewriter.cc new file mode 100644 index 00000000000..ae843760a8d --- /dev/null +++ b/tensorflow/compiler/xla/service/topk_rewriter.cc @@ -0,0 +1,187 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/topk_rewriter.h" + +#include "absl/algorithm/container.h" +#include "absl/types/optional.h" +#include "tensorflow/compiler/xla/service/hlo_casting_utils.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/shape_util.h" + +namespace xla { + +static bool IsNanSafeGt(HloComputation* comp) { + namespace m = match; + auto match_bitcast_f32 = [](int64 parameter_number) { + auto param = m::Parameter(parameter_number) + .WithShape(m::Shape().WithElementType(F32)); + auto param_s32 = + m::BitcastConvert(param).WithShape(m::Shape().WithElementType(S32)); + auto param_u32 = + m::BitcastConvert(param).WithShape(m::Shape().WithElementType(U32)); + return m::Select( + m::Lt(param_s32, m::ConstantScalar(0)), + m::BitcastConvert( + m::Subtract(m::ConstantScalar(std::numeric_limits::max()), + param_u32)) + .WithShape(m::Shape().WithElementType(S32)), + param_s32); + }; + auto match_bitcast_bf16 = [](int64 parameter_number) { + auto param = m::Convert(m::Parameter(parameter_number) + .WithShape(m::Shape().WithElementType(BF16))) + .WithShape(m::Shape().WithElementType(F32)); + auto param_s32 = + m::BitcastConvert(param).WithShape(m::Shape().WithElementType(S32)); + auto param_u32 = + m::BitcastConvert(param).WithShape(m::Shape().WithElementType(U32)); + return m::Select( + m::Lt(param_s32, m::ConstantScalar(0)), + m::BitcastConvert( + m::Subtract(m::ConstantScalar(std::numeric_limits::max()), + param_u32)) + .WithShape(m::Shape().WithElementType(S32)), + param_s32); + }; + return Match(comp->root_instruction(), + m::Gt(match_bitcast_f32(0), match_bitcast_f32(1))) || + Match(comp->root_instruction(), + m::Gt(match_bitcast_bf16(0), match_bitcast_bf16(1))); +} + +StatusOr TopkRewriter::Run(HloModule* module) { + bool changed = false; + for (HloComputation* comp : module->computations()) { + for (HloInstruction* inst : comp->MakeInstructionPostOrder()) { + HloSortInstruction* sort = DynCast(inst); + if (sort == nullptr || sort->operand_count() != 2) { + continue; + } + HloInstruction* data = sort->mutable_operand(0); + HloIotaInstruction* iota = + DynCast(sort->mutable_operand(1)); + const PrimitiveType element_type = data->shape().element_type(); + if (data->shape().rank() != 2 || + (element_type != F32 && element_type != BF16)) { + continue; + } + if (iota == nullptr || iota->shape().rank() != 2 || + iota->shape().element_type() != S32 || + iota->opcode() != HloOpcode::kIota || + iota->iota_dimension() != sort->sort_dimension()) { + continue; + } + if (!IsNanSafeGt(sort->to_apply())) { + continue; + } + const int64 sort_dim = sort->sort_dimension(); + const int64 batch_dim = sort_dim == 1 ? 0 : 1; + + bool supported = true; + absl::optional k; + for (HloInstruction* gte : sort->users()) { + if (gte->opcode() != HloOpcode::kGetTupleElement || + gte->user_count() != 1) { + supported = false; + break; + } + const HloInstruction* slice = gte->users()[0]; + if (slice->opcode() != HloOpcode::kSlice) { + // Non-slice user means we are not doing a TopK + supported = false; + break; + } + if (absl::c_any_of(slice->slice_starts(), + [](int x) { return x != 0; }) || + absl::c_any_of(slice->slice_strides(), + [](int x) { return x != 1; })) { + // Strided slice or slicing at the beginning isn't supported. + supported = false; + break; + } + if (slice->slice_limits(batch_dim) != + slice->operand(0)->shape().dimensions(batch_dim)) { + // Slicing along the batch dimension isn't supported. + supported = false; + break; + } + if (k == absl::nullopt) { + k = slice->slice_limits(sort_dim); + } else if (k != slice->slice_limits(sort_dim)) { + // Different k for the different operands isn't supported. + supported = false; + break; + } + } + if (k == absl::nullopt || !supported) { + continue; + } + + // Profitability check. + if (!is_profitable_to_convert_(sort, *k)) { + continue; + } + + const int64 batch_size = sort->operand(0)->shape().dimensions(batch_dim); + const int64 input_size = sort->operand(0)->shape().dimensions(sort_dim); + HloInstruction* input = sort->mutable_operand(0); + if (sort_dim == 0) { + input = comp->AddInstruction(HloInstruction::CreateTranspose( + ShapeUtil::MakeShape(element_type, {batch_size, input_size}), input, + {1, 0})); + } + + Shape topk_shape = ShapeUtil::MakeTupleShape( + {ShapeUtil::MakeShape(element_type, {batch_size, k.value()}), + ShapeUtil::MakeShape(S32, {batch_size, k.value()})}); + HloInstruction* topk = comp->AddInstruction( + HloInstruction::CreateCustomCall(topk_shape, {input}, "TopK")); + HloInstruction* value_gte = + comp->AddInstruction(HloInstruction::CreateGetTupleElement( + topk->shape().tuple_shapes(0), topk, 0)); + HloInstruction* index_gte = + comp->AddInstruction(HloInstruction::CreateGetTupleElement( + topk->shape().tuple_shapes(1), topk, 1)); + + if (sort_dim == 0) { + value_gte = comp->AddInstruction(HloInstruction::CreateTranspose( + ShapeUtil::MakeShape(element_type, {k.value(), batch_size}), + value_gte, {1, 0})); + index_gte = comp->AddInstruction(HloInstruction::CreateTranspose( + ShapeUtil::MakeShape(S32, {k.value(), batch_size}), index_gte, + {1, 0})); + } + + for (HloInstruction* gte : sort->users()) { + for (HloInstruction* slice : gte->users()) { + if (gte->tuple_index() == 0) { + TF_RETURN_IF_ERROR(slice->ReplaceAllUsesWith(value_gte)); + } else if (gte->tuple_index() == 1) { + TF_RETURN_IF_ERROR(slice->ReplaceAllUsesWith(index_gte)); + } else { + LOG(FATAL) << "Sort with more than 2 output isn't supported in " + "topk rewriter"; + } + } + } + changed = true; + } + } + return changed; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/topk_rewriter.h b/tensorflow/compiler/xla/service/topk_rewriter.h new file mode 100644 index 00000000000..68f8a8145e2 --- /dev/null +++ b/tensorflow/compiler/xla/service/topk_rewriter.h @@ -0,0 +1,44 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_TOPK_REWRITER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_TOPK_REWRITER_H_ + +#include "tensorflow/compiler/xla/service/hlo_instructions.h" +#include "tensorflow/compiler/xla/service/hlo_pass_interface.h" + +namespace xla { +// This pass pattern-matches soups of HLOs executing a TopK operation and +// replaces them with a TopK CustomCall when the given values are supported by +// the CustomCall and it is more efficient to use that implementation. +class TopkRewriter : public HloModulePass { + public: + explicit TopkRewriter(std::function + is_profitable_to_convert) + : is_profitable_to_convert_(std::move(is_profitable_to_convert)) {} + + absl::string_view name() const override { return "topk-rewriter"; } + + StatusOr Run(HloModule* module) override; + + private: + // Predicate that returns true if a sort instruction is profitable to be + // converted into a custom call. + std::function + is_profitable_to_convert_; +}; +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_TOPK_REWRITER_H_ diff --git a/tensorflow/compiler/xla/service/topk_rewriter_test.cc b/tensorflow/compiler/xla/service/topk_rewriter_test.cc new file mode 100644 index 00000000000..e440da5b163 --- /dev/null +++ b/tensorflow/compiler/xla/service/topk_rewriter_test.cc @@ -0,0 +1,153 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/topk_rewriter.h" + +#include "tensorflow/compiler/xla/service/hlo_dce.h" +#include "tensorflow/compiler/xla/service/hlo_matchers.h" +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/tests/test_macros.h" +#include "tensorflow/compiler/xla/tests/test_utils.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace op = xla::testing::opcode_matchers; + +namespace xla { +namespace { + +using TopkRewriterTest = HloTestBase; + +TEST_F(TopkRewriterTest, Rewrite) { + const char* const hlo_string = R"( +HloModule module + +%compare { + %p.1.lhs.8 = s32[] parameter(2) + %p.1.rhs.9 = s32[] parameter(3) + %p.0.lhs.6 = f32[] parameter(0) + %bitcast-convert.11 = s32[] bitcast-convert(%p.0.lhs.6) + %constant.15 = s32[] constant(0) + %compare.16 = pred[] compare(%bitcast-convert.11, %constant.15), direction=LT + %constant.10 = u32[] constant(2147483647) + %bitcast-convert.12 = u32[] bitcast-convert(%p.0.lhs.6) + %subtract.13 = u32[] subtract(%constant.10, %bitcast-convert.12) + %bitcast-convert.14 = s32[] bitcast-convert(%subtract.13) + %select.17 = s32[] select(%compare.16, %bitcast-convert.14, + %bitcast-convert.11) + %p.0.rhs.7 = f32[] parameter(1) + %bitcast-convert.19 = s32[] bitcast-convert(%p.0.rhs.7) + %constant.23 = s32[] constant(0) + %compare.24 = pred[] compare(%bitcast-convert.19, %constant.23), direction=LT + %constant.18 = u32[] constant(2147483647) + %bitcast-convert.20 = u32[] bitcast-convert(%p.0.rhs.7) + %subtract.21 = u32[] subtract(%constant.18, %bitcast-convert.20) + %bitcast-convert.22 = s32[] bitcast-convert(%subtract.21) + %select.25 = s32[] select(%compare.24, %bitcast-convert.22, + %bitcast-convert.19) + ROOT %compare.26 = pred[] compare(%select.17, %select.25), direction=GT +} + +ENTRY cluster { + %arg_tuple.1 = f32[8,1234567] parameter(0) + %iota.4 = s32[8,1234567] iota(), iota_dimension=1 + %sort.27 = (f32[8,1234567], s32[8,1234567]) sort(%arg_tuple.1, %iota.4), + dimensions={1}, is_stable=true, to_apply=%compare + %get-tuple-element.28 = f32[8,1234567] get-tuple-element(%sort.27), index=0 + %slice.29 = f32[8,5] slice(%get-tuple-element.28), slice={[0:8], [0:5]} + %get-tuple-element.30 = s32[8,1234567] get-tuple-element(%sort.27), index=1 + %slice.31 = s32[8,5] slice(%get-tuple-element.30), slice={[0:8], [0:5]} + ROOT %tuple.32 = (f32[8,5], s32[8,5]) tuple(%slice.29, %slice.31) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TopkRewriter rewriter([](const HloSortInstruction*, int64) { return true; }); + TF_ASSERT_OK_AND_ASSIGN(bool changed, rewriter.Run(module.get())); + TF_ASSERT_OK(HloDCE().Run(module.get()).status()); + EXPECT_TRUE(changed); + EXPECT_THAT( + module->entry_computation()->root_instruction(), + op::Tuple(op::GetTupleElement(op::CustomCall(op::Parameter(0)), 0), + op::GetTupleElement(op::CustomCall(op::Parameter(0)), 1))); + const HloInstruction* cc = + module->entry_computation()->root_instruction()->operand(0)->operand(0); + EXPECT_THAT(cc->custom_call_target(), "TopK"); +} + +TEST_F(TopkRewriterTest, RewriteTranspose) { + const char* const hlo_string = R"( +HloModule module + +%compare { + %p.1.lhs.8 = s32[] parameter(2) + %p.1.rhs.9 = s32[] parameter(3) + %p.0.lhs.6 = f32[] parameter(0) + %bitcast-convert.11 = s32[] bitcast-convert(%p.0.lhs.6) + %constant.15 = s32[] constant(0) + %compare.16 = pred[] compare(%bitcast-convert.11, %constant.15), direction=LT + %constant.10 = u32[] constant(2147483647) + %bitcast-convert.12 = u32[] bitcast-convert(%p.0.lhs.6) + %subtract.13 = u32[] subtract(%constant.10, %bitcast-convert.12) + %bitcast-convert.14 = s32[] bitcast-convert(%subtract.13) + %select.17 = s32[] select(%compare.16, %bitcast-convert.14, + %bitcast-convert.11) + %p.0.rhs.7 = f32[] parameter(1) + %bitcast-convert.19 = s32[] bitcast-convert(%p.0.rhs.7) + %constant.23 = s32[] constant(0) + %compare.24 = pred[] compare(%bitcast-convert.19, %constant.23), direction=LT + %constant.18 = u32[] constant(2147483647) + %bitcast-convert.20 = u32[] bitcast-convert(%p.0.rhs.7) + %subtract.21 = u32[] subtract(%constant.18, %bitcast-convert.20) + %bitcast-convert.22 = s32[] bitcast-convert(%subtract.21) + %select.25 = s32[] select(%compare.24, %bitcast-convert.22, + %bitcast-convert.19) + ROOT %compare.26 = pred[] compare(%select.17, %select.25), direction=GT +} + +ENTRY cluster { + %arg_tuple.1 = f32[1234567,8] parameter(0) + %iota.4 = s32[1234567,8] iota(), iota_dimension=0 + %sort.27 = (f32[1234567,8], s32[1234567,8]) sort(%arg_tuple.1, %iota.4), + dimensions={0}, is_stable=true, to_apply=%compare + %get-tuple-element.28 = f32[1234567,8] get-tuple-element(%sort.27), index=0 + %slice.29 = f32[5,8] slice(%get-tuple-element.28), slice={[0:5], [0:8]} + %get-tuple-element.30 = s32[1234567,8] get-tuple-element(%sort.27), index=1 + %slice.31 = s32[5,8] slice(%get-tuple-element.30), slice={[0:5], [0:8]} + ROOT %tuple.32 = (f32[5,8], s32[5,8]) tuple(%slice.29, %slice.31) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TopkRewriter rewriter([](const HloSortInstruction*, int64) { return true; }); + TF_ASSERT_OK_AND_ASSIGN(bool changed, rewriter.Run(module.get())); + TF_ASSERT_OK(HloDCE().Run(module.get()).status()); + EXPECT_TRUE(changed); + LOG(INFO) << module->entry_computation()->ToString(); + EXPECT_THAT( + module->entry_computation()->root_instruction(), + op::Tuple(op::Transpose(op::GetTupleElement( + op::CustomCall(op::Transpose(op::Parameter(0))), 0)), + op::Transpose(op::GetTupleElement( + op::CustomCall(op::Transpose(op::Parameter(0))), 1)))); + const HloInstruction* cc = module->entry_computation() + ->root_instruction() + ->operand(0) + ->operand(0) + ->operand(0); + EXPECT_THAT(cc->custom_call_target(), "TopK"); +} + +} // namespace +} // namespace xla From 1033515e9ede0996f0213da293d0a7cff6dc094c Mon Sep 17 00:00:00 2001 From: Tamas Bela Feher Date: Thu, 7 May 2020 18:17:52 +0200 Subject: [PATCH 0441/1017] Add TF-TRT converter for Shape op. The unit test includes test cases when the TensorRT network is called with no input tensor. The helper routines are adjusted to handle this case. --- .../tf2tensorrt/convert/convert_nodes.cc | 35 ++++++++++ .../tf2tensorrt/convert/convert_nodes_test.cc | 69 +++++++++++++++++-- .../utils/trt_shape_optimization_profiles.cc | 17 +++-- 3 files changed, 107 insertions(+), 14 deletions(-) diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc index 369b339d01a..6674081011f 100644 --- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc +++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes.cc @@ -2410,6 +2410,40 @@ Status ConvertTranspose(OpConverterParams* params) { return Status::OK(); } +Status ConvertShape(OpConverterParams* params) { + const auto& inputs = params->inputs; + TF_RETURN_IF_ERROR( + CheckInputsWeights(*params, {{"input", TrtInputArg::kBoth}})); + if (params->use_implicit_batch) { + return errors::Unimplemented( + "Shape is only supported for explicit batch mode."); + } + if (HasStaticShape(inputs.at(0).GetTrtDims())) { + if (params->validation_only) return Status::OK(); + nvinfer1::Dims input_dims = inputs.at(0).GetTrtDims(); + nvinfer1::Dims output_dims{1, {input_dims.nbDims}}; + // Create a const node with the values of output_dims + TRT_ShapedWeights weight = params->weight_store->GetTempWeights( + nvinfer1::DataType::kINT32, output_dims); + int32* values_ptr = static_cast(weight.GetValues()); + std::copy(input_dims.d, input_dims.d + input_dims.nbDims, values_ptr); + auto output = params->converter->CreateConstantLayer(weight, output_dims); + params->outputs->push_back(TRT_TensorOrWeights(output)); + return Status::OK(); + } +#if IS_TRT_VERSION_GE(6, 0, 0, 0) + if (params->validation_only) return Status::OK(); + nvinfer1::IShapeLayer* shape_layer = + params->converter->network()->addShape(*inputs.at(0).tensor()); + TFTRT_RETURN_ERROR_IF_NULLPTR(shape_layer, params->node_def.name()); + params->outputs->push_back(TRT_TensorOrWeights(shape_layer->getOutput(0))); + return Status::OK(); +#else + return errors::Unavailable( + "Shape op conversion requires TensorRT 6 or above"); +#endif +} + Status ConvertReshape(OpConverterParams* params) { const auto& inputs = params->inputs; TF_RETURN_IF_ERROR( @@ -5958,6 +5992,7 @@ static void RegisterValidatableOpConverters( (*registration)[pool_op_type] = ConvertPool3D; } #endif + (*registration)["Shape"] = ConvertShape; (*registration)["Rsqrt"] = ConvertRsqrt; (*registration)["Slice"] = ConvertSlice; (*registration)["Softmax"] = ConvertSoftmax; diff --git a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc index 52d05ff8225..9ca1c8c4c9f 100644 --- a/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc +++ b/tensorflow/compiler/tf2tensorrt/convert/convert_nodes_test.cc @@ -1781,7 +1781,8 @@ class ParameterizedOpConverterTestBase void BuildAndRun(const string& name, const std::vector>& expected_output_dims, const Status& expected_runtime_status, - const std::vector>>& matcher) { + const std::vector>>& matcher, + const std::vector& out_tf_types = {}) { TensorShape shape; const int n_output = expected_output_dims.size(); ASSERT_EQ(n_output, matcher.size()); @@ -1790,12 +1791,14 @@ class ParameterizedOpConverterTestBase TF_EXPECT_OK( TensorShapeUtils::MakeShape(expected_output_dims[i], &shape)); string out_name = (n_output == 1) ? name : StrCat(name, ":", i); - InputOutputData data{out_name, - ConstructTensor(shape.num_elements(), 0, tf_type)}; + DataType out_tf_type = + out_tf_types.size() > i ? out_tf_types[i] : tf_type; + InputOutputData data{ + out_name, ConstructTensor(shape.num_elements(), 0, out_tf_type)}; output_data.push_back(data); } - ASSERT_FALSE(input_data_.empty()); - const int batch_size = input_data_[0].tensor.shape().dim_size(0); + const int batch_size = + input_data_.empty() ? 1 : input_data_[0].tensor.shape().dim_size(0); Status stat = OpConverterTest::BuildAndRun(input_data_, &output_data, batch_size); ASSERT_EQ(expected_runtime_status.ok(), stat.ok()) @@ -1820,13 +1823,15 @@ class ParameterizedOpConverterTestBase const std::vector& expected_output_dims, const Status& expected_conversion_status, const Status& expected_runtime_status, - const Matcher>& matcher) { + const Matcher>& matcher, + const std::vector& out_tf_types = {}) { RunValidationAndConversion(node_def, expected_conversion_status, name.c_str(), expected_output_dims); if (expected_conversion_status.ok()) { BuildAndRun(name, std::vector>({expected_output_dims}), expected_runtime_status, - std::vector>>({matcher})); + std::vector>>({matcher}), + out_tf_types); } } @@ -2169,6 +2174,56 @@ TEST_F(OpConverterTest, ConvertReshape) { } } +TEST_P(OpConverterTest1, ConvertShape) { + // Get the NodeDef for Shape op. + Scope s = Scope::NewRootScope(); + auto input = ops::Placeholder(s.WithOpName("input"), tf_type); + auto shape = ops::Shape(s.WithOpName("my_shape"), input); + const NodeDef& node_def = shape.operation.node()->def(); + + Status conversion_status = + (trt_mode == TrtTestMode::kImplicitBatch) + ? errors::Unimplemented( + "Shape is only supported for explicit batch mode.") + : Status::OK(); + std::vector test_params = { + TestParamBase{{1, 2, 3}, {}, {3}, {}, conversion_status}, + // Add input as weight (we use non empty param ({1}) to trigger this). + TestParamBase{{1, 2, 3}, {}, {3}, {1}, conversion_status}, + }; + + auto input_is_weight = [](const TestParamBase p) { return !p.param.empty(); }; + for (auto p : test_params) { + SCOPED_TRACE(p); + Reset(); + // Number of elements of the input tensor. We leave it 0 in case we do + // not need to add an input tensor. This happens in explicit batch mode: the + // shape is known at conversion time and therefore the shape is added to the + // network as a constant layer. (In this case the single node network that + // we use for the unit test will have no actual input tensor when converted + // to a TensorRT network.) + int n_elements = 0; + // In explicit batch mode the shape is known at conversion time and + // therefore the shape is added to the network as a constant layer. As + // a result, the single node network that we use for this unit test will + // have no actual input tensor when converted to a TensorRT network. + if (input_is_weight(p) || trt_mode != TrtTestMode::kExplicitBatch) { + // Calculate the number of elements for adding input data. + n_elements = std::accumulate(p.input_dims.begin(), p.input_dims.end(), 1, + std::multiplies()); + } + std::vector input_val(n_elements, 1); + if (!input_is_weight(p)) { + AddTestTensor("input", p.input_dims, input_val); + } else { + AddTestWeights("input", p.input_dims, input_val, tf_type); + } + TestOpConverter("my_shape", node_def, p.expected_output_dims, p.status, + p.runtime_status, ElementsAreArray(p.input_dims), + {DT_INT32}); + } +} + // Helper function for testing MatMul and BatchMatMul // get_matmul corresponds to the function used to generate the node. It should // accept (DataType, transpose_a, transpose_b) as parameters. diff --git a/tensorflow/compiler/tf2tensorrt/utils/trt_shape_optimization_profiles.cc b/tensorflow/compiler/tf2tensorrt/utils/trt_shape_optimization_profiles.cc index 70a0a9a7b65..2f31865751f 100644 --- a/tensorflow/compiler/tf2tensorrt/utils/trt_shape_optimization_profiles.cc +++ b/tensorflow/compiler/tf2tensorrt/utils/trt_shape_optimization_profiles.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include "absl/algorithm/container.h" #include "tensorflow/compiler/tf2tensorrt/convert/utils.h" #if GOOGLE_CUDA && GOOGLE_TENSORRT @@ -35,14 +36,16 @@ void TrtShapeOptimizationProfile::InitProfiles() { << "for each input (min=opt=max)."; } for (auto& shape_vec : input_shapes_) { - std::vector dimvec; - for (auto& shape : shape_vec) { - dimvec.push_back(TensorShapeToTrtDims(shape, false)); + if (!shape_vec.empty()) { + std::vector dimvec(shape_vec.size()); + absl::c_transform(shape_vec, dimvec.begin(), [](TensorShape shape) { + return TensorShapeToTrtDims(shape, false); + }); + // Set min=opt=max. + OptimizationProfileConfig profConfig{dimvec, dimvec, dimvec}; + profiles_.push_back(std::move(profConfig)); + VLOG(1) << "Created profile " << profiles_.back().DebugString(); } - // We set min=opt=max. - OptimizationProfileConfig profConfig{dimvec, dimvec, dimvec}; - profiles_.push_back(std::move(profConfig)); - VLOG(1) << "Created profile " << profiles_.back().DebugString(); } } From ccbd5ff0e6a77c4463d76926bef9c8bdaad8826a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 5 Aug 2020 02:01:47 -0700 Subject: [PATCH 0442/1017] Update GraphDef version to 484. PiperOrigin-RevId: 324978474 Change-Id: Ib5d72c7e1955a8964dedf75e1a6a6ee6c582cfef --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 7febc640348..acaa48f251c 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 483 // Updated: 2020/8/4 +#define TF_GRAPH_DEF_VERSION 484 // Updated: 2020/8/5 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From f576f29e3d14c36dfe79f014372e49acd75ce406 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 5 Aug 2020 02:01:48 -0700 Subject: [PATCH 0443/1017] compat: Update forward compatibility horizon to 2020-08-05 PiperOrigin-RevId: 324978475 Change-Id: I3875bc841ac816ae7123acc82621acfdcb0fce17 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index c40337f00be..a274743d124 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 4) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 5) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From 926642ea7a1a8095cddf3b063c5d19d4478eaeda Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 5 Aug 2020 02:05:57 -0700 Subject: [PATCH 0444/1017] [XLA:CPU] Add a runtime function for F32 TopK and use TopkRewriter to target it This just delegates the hard work to std::partial_sort. PiperOrigin-RevId: 324979038 Change-Id: I16a7c4d840948f3744f4f920bac29d4e6d7333b3 --- tensorflow/compiler/tests/sort_ops_test.py | 59 +++++++------- tensorflow/compiler/xla/service/cpu/BUILD | 17 +++++ .../compiler/xla/service/cpu/cpu_compiler.cc | 4 + .../compiler/xla/service/cpu/cpu_runtime.cc | 1 + .../compiler/xla/service/cpu/cpu_runtime.h | 1 + .../compiler/xla/service/cpu/ir_emitter.cc | 38 ++++++++++ .../compiler/xla/service/cpu/ir_emitter.h | 1 + .../compiler/xla/service/cpu/runtime_topk.cc | 76 +++++++++++++++++++ .../compiler/xla/service/cpu/runtime_topk.h | 32 ++++++++ .../xla/service/cpu/simple_orc_jit.cc | 2 + .../compiler/xla/service/cpu/tests/BUILD | 16 ++++ .../xla/service/cpu/tests/cpu_topk_test.cc | 59 ++++++++++++++ 12 files changed, 273 insertions(+), 33 deletions(-) create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_topk.cc create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_topk.h create mode 100644 tensorflow/compiler/xla/service/cpu/tests/cpu_topk_test.cc diff --git a/tensorflow/compiler/tests/sort_ops_test.py b/tensorflow/compiler/tests/sort_ops_test.py index d50fdec7c63..838718aa1e3 100644 --- a/tensorflow/compiler/tests/sort_ops_test.py +++ b/tensorflow/compiler/tests/sort_ops_test.py @@ -129,42 +129,35 @@ class XlaSortOpTest(xla_test.XLATestCase): def testTopKZeros(self): """Tests that positive and negative zeros sort correctly.""" - # Only bfloat16 is implemented. - bfloat16 = dtypes.bfloat16.as_numpy_dtype - if bfloat16 not in self.numeric_types: - return - - with self.session() as sess: - p = array_ops.placeholder(dtypes.bfloat16) - with self.test_scope(): - topk = nn_ops.top_k(p, k=4) - results = sess.run( - topk, - {p: np.array([0., -0., 0., 3., -0., -4., 0., -0.], dtype=bfloat16)}) - self.assertAllEqual( - np.array([3., 0., 0., 0.], dtype=bfloat16), results[0]) - self.assertEqual(list([3, 0, 2, 6]), list(results[1])) + supported_types = set([dtypes.bfloat16.as_numpy_dtype, np.float32]) + for dtype in supported_types.intersection(self.numeric_types): + with self.session() as sess: + p = array_ops.placeholder(dtype) + with self.test_scope(): + topk = nn_ops.top_k(p, k=4) + results = sess.run( + topk, + {p: np.array([0., -0., 0., 3., -0., -4., 0., -0.], dtype=dtype)}) + self.assertAllEqual(np.array([3., 0., 0., 0.], dtype=dtype), results[0]) + self.assertEqual(list([3, 0, 2, 6]), list(results[1])) def testTopKInfinities(self): """Tests that positive and negative infinity sort correctly.""" - # Only bfloat16 is implemented. - bfloat16 = dtypes.bfloat16.as_numpy_dtype - if bfloat16 not in self.numeric_types: - return - - with self.session() as sess: - p = array_ops.placeholder(dtypes.bfloat16) - with self.test_scope(): - topk = nn_ops.top_k(p, k=6) - results = sess.run(topk, { - p: np.array( - [1, 2, float("inf"), -float("inf"), -1, -2], dtype=bfloat16) - }) - self.assertAllEqual( - np.array( - [float("inf"), 2.0, 1.0, -1.0, -2.0, -float("inf")], - dtype=bfloat16), results[0]) - self.assertEqual(list([2, 1, 0, 4, 5, 3]), list(results[1])) + supported_types = set([dtypes.bfloat16.as_numpy_dtype, np.float32]) + for dtype in supported_types.intersection(self.numeric_types): + with self.session() as sess: + p = array_ops.placeholder(dtype) + with self.test_scope(): + topk = nn_ops.top_k(p, k=6) + results = sess.run(topk, { + p: + np.array([1, 2, float("inf"), -float("inf"), -1, -2], + dtype=dtype) + }) + self.assertAllEqual( + np.array([float("inf"), 2.0, 1.0, -1.0, -2.0, -float("inf")], + dtype=dtype), results[0]) + self.assertEqual(list([2, 1, 0, 4, 5, 3]), list(results[1])) def testInTopK(self): supported_types = set([np.int32, np.int64]) diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index 782d08296f0..6eaf43902fe 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -49,6 +49,7 @@ filegroup( "runtime_single_threaded_conv2d.cc", "runtime_single_threaded_fft.cc", "runtime_single_threaded_matmul.cc", + "runtime_topk.cc", ], visibility = [":friends"], ) @@ -64,6 +65,7 @@ filegroup( "runtime_single_threaded_conv2d.h", "runtime_single_threaded_fft.h", "runtime_single_threaded_matmul.h", + "runtime_topk.h", ], visibility = [":friends"], ) @@ -134,6 +136,7 @@ cc_library( "//tensorflow/compiler/xla/service:copy_insertion", "//tensorflow/compiler/xla/service:hlo_casting_utils", "//tensorflow/compiler/xla/service:dump", + "//tensorflow/compiler/xla/service:topk_rewriter", "//tensorflow/compiler/xla/service:map_inliner", "//tensorflow/compiler/xla/service:rng_bit_generator_expander", "//tensorflow/compiler/xla/service:tree_reduction_rewriter", @@ -230,6 +233,7 @@ cc_library( ":runtime_fft", ":runtime_fork_join", ":runtime_key_value_sort", + ":runtime_topk", ":runtime_matmul", ":runtime_matmul_mkl", ":runtime_single_threaded_conv2d", @@ -759,6 +763,19 @@ cc_library( ], ) +cc_library( + name = "runtime_topk", + srcs = ["runtime_topk.cc"], + hdrs = ["runtime_topk.h"], + copts = runtime_copts(), + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/core/platform:dynamic_annotations", + "//tensorflow/core/platform:macros", + "//tensorflow/core/platform:types", + ], +) + cc_library( name = "runtime_fork_join", srcs = ["runtime_fork_join.cc"], diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 04d703fdd59..0826d7b8ce1 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -104,6 +104,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/slice_sinker.h" #include "tensorflow/compiler/xla/service/slow_operation_alarm.h" #include "tensorflow/compiler/xla/service/sort_simplifier.h" +#include "tensorflow/compiler/xla/service/topk_rewriter.h" #include "tensorflow/compiler/xla/service/transpose_folding.h" #include "tensorflow/compiler/xla/service/tree_reduction_rewriter.h" #include "tensorflow/compiler/xla/service/triangular_solve_expander.h" @@ -320,6 +321,9 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn( pass.AddPass(); pass.AddPass(); } + pipeline.AddPass([](const HloSortInstruction* sort, int64) { + return sort->operand(0)->shape().element_type() == F32; + }); pipeline.AddPass(); pipeline.AddPass( [&](const HloInstruction& dot, diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc index 2231ecfa1e8..5bee6049a5e 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc @@ -117,6 +117,7 @@ extern const char* const kParallelForkJoinSymbolName = "__xla_cpu_runtime_ParallelForkJoin"; extern const char* const kKeyValueSortSymbolName = "__xla_cpu_runtime_KeyValueSort"; +extern const char* const kTopKF32SymbolName = "__xla_cpu_runtime_TopKF32"; extern const char* const kTracingStartSymbolName = "__xla_cpu_runtime_TracingStart"; extern const char* const kTracingEndSymbolName = "__xla_cpu_runtime_TracingEnd"; diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h index ee75b97e4dc..eb24e0bc334 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h +++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h @@ -72,6 +72,7 @@ extern const char* const kAcquireOutfeedBufferForPopulationSymbolName; extern const char* const kReleaseOutfeedBufferAfterPopulationSymbolName; extern const char* const kParallelForkJoinSymbolName; extern const char* const kKeyValueSortSymbolName; +extern const char* const kTopKF32SymbolName; extern const char* const kAllReduceSymbolName; extern const char* const kCollectivePermuteSymbolName; extern const char* const kReplicaIdSymbolName; diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 278e6479e48..2688a7898af 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -2387,6 +2387,41 @@ Status IrEmitter::HandlePadToStatic(HloInstruction* hlo) { return Status::OK(); } +Status IrEmitter::HandleTopK(HloInstruction* hlo) { + TF_RETURN_IF_ERROR(EmitTargetAddressForOp(hlo)); + const HloInstruction* input = hlo->operand(0); + int64 k = hlo->shape().tuple_shapes(0).dimensions(1); + TF_RET_CHECK(input->shape().element_type() == F32); + TF_RET_CHECK(LayoutUtil::IsMonotonicWithDim0Major( + hlo->shape().tuple_shapes(0).layout())); + TF_RET_CHECK(LayoutUtil::IsMonotonicWithDim0Major( + hlo->shape().tuple_shapes(1).layout())); + TF_RET_CHECK( + LayoutUtil::IsMonotonicWithDim0Major(hlo->operand(0)->shape().layout())); + + TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice values_slice, + assignment_.GetUniqueSlice(hlo->operand(0), {})); + TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice out_values_slice, + assignment_.GetUniqueSlice(hlo, {0})); + TF_ASSIGN_OR_RETURN(const BufferAllocation::Slice out_indices_slice, + assignment_.GetUniqueSlice(hlo, {1})); + llvm::Value* values_ptr = + EmitBufferPointer(values_slice, hlo->operand(0)->shape()); + llvm::Value* out_values_ptr = + EmitBufferPointer(out_values_slice, hlo->shape().tuple_shapes(0)); + llvm::Value* out_indices_ptr = + EmitBufferPointer(out_indices_slice, hlo->shape().tuple_shapes(1)); + EmitCallToFunc(runtime::kTopKF32SymbolName, + {b_.getInt64(input->shape().dimensions(0)), + b_.getInt64(input->shape().dimensions(1)), b_.getInt64(k), + values_ptr, out_values_ptr, out_indices_ptr}, + b_.getVoidTy()); + + llvm_ir::EmitTuple(GetIrArrayFor(hlo), {out_values_ptr, out_indices_ptr}, + &b_); + return Status::OK(); +} + Status IrEmitter::HandleCustomCall(HloInstruction* custom_call) { if (custom_call->custom_call_target() == "PadToStatic") { return HandlePadToStatic(custom_call); @@ -2394,6 +2429,9 @@ Status IrEmitter::HandleCustomCall(HloInstruction* custom_call) { if (custom_call->custom_call_target() == "SliceToDynamic") { return HandleSliceToDynamic(custom_call); } + if (custom_call->custom_call_target() == "TopK") { + return HandleTopK(custom_call); + } absl::Span operands(custom_call->operands()); llvm::Type* i8_ptr_type = b_.getInt8PtrTy(); llvm::AllocaInst* operands_alloca = diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.h b/tensorflow/compiler/xla/service/cpu/ir_emitter.h index 3955deefbea..f136e3470e5 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.h +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.h @@ -190,6 +190,7 @@ class IrEmitter : public DfsHloVisitorWithDefault, private: Status HandleSliceToDynamic(HloInstruction* hlo); Status HandlePadToStatic(HloInstruction* hlo); + Status HandleTopK(HloInstruction* hlo); Status HandleAllReduceSingleReplica(HloInstruction* crs); Status HandleAllReduceMultipleReplica(HloInstruction* crs); diff --git a/tensorflow/compiler/xla/service/cpu/runtime_topk.cc b/tensorflow/compiler/xla/service/cpu/runtime_topk.cc new file mode 100644 index 00000000000..5174a3329fb --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/runtime_topk.cc @@ -0,0 +1,76 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/cpu/runtime_topk.h" + +#include +#include +#include +#include + +#include "tensorflow/core/platform/dynamic_annotations.h" +#include "tensorflow/core/platform/macros.h" + +template +static void TopK(tensorflow::int64 batch_size, tensorflow::int64 input_size, + tensorflow::int64 k, const T* values, T* out_values, + tensorflow::int32* out_indices) { + // 'values' is managed by the JIT code, so msan can't tell they are + // initialized. + TF_ANNOTATE_MEMORY_IS_INITIALIZED(values, + input_size * batch_size * sizeof(T)); + + std::vector temp_indices(input_size); + for (tensorflow::int64 batch = 0; batch != batch_size; ++batch) { + std::iota(temp_indices.begin(), temp_indices.end(), 0); + + const T* values_batch = values + batch * input_size; + + auto convert_to_int = [](T value) { + tensorflow::uint32 x; + std::memcpy(&x, &value, sizeof(x)); + return static_cast(x) < 0 + ? std::numeric_limits::max() - x + : x; + }; + + auto kth_element = temp_indices.begin() + k; + std::partial_sort(temp_indices.begin(), kth_element, temp_indices.end(), + [&](size_t i1, size_t i2) { + // Do the comparison in integers to enforce a total + // order of -NaN < -Inf < -0 < +0 < +Inf < +NaN. + tensorflow::int32 v1 = convert_to_int(values_batch[i1]); + tensorflow::int32 v2 = convert_to_int(values_batch[i2]); + if (v1 == v2) { + return i1 < i2; // Stabilize sorting. + } + return v1 > v2; + }); + + T* out_values_batch = out_values + batch * k; + tensorflow::int32* out_indices_batch = out_indices + batch * k; + std::copy(temp_indices.begin(), kth_element, out_indices_batch); + for (tensorflow::int64 i = 0; i < k; i++) { + out_values_batch[i] = values_batch[temp_indices[i]]; + } + } +} + +TF_ATTRIBUTE_NO_SANITIZE_MEMORY void __xla_cpu_runtime_TopKF32( + tensorflow::int64 batch_size, tensorflow::int64 input_size, + tensorflow::int64 k, const float* values, float* out_values, + tensorflow::int32* out_indices) { + TopK(batch_size, input_size, k, values, out_values, out_indices); +} diff --git a/tensorflow/compiler/xla/service/cpu/runtime_topk.h b/tensorflow/compiler/xla/service/cpu/runtime_topk.h new file mode 100644 index 00000000000..de69c0603e3 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/runtime_topk.h @@ -0,0 +1,32 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_TOPK_H +#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_TOPK_H + +#include "tensorflow/core/platform/types.h" + +extern "C" { + +// Calculates `batch_size` topk operations with `input_size` inputs each. The +// outputs are written to `out_values` and `out_indices`. +extern void __xla_cpu_runtime_TopKF32(tensorflow::int64 batch_size, + tensorflow::int64 input_size, + tensorflow::int64 k, const float* values, + float* out_values, + tensorflow::int32* out_indices); +} + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_TOPK_H diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc index 631c6985b03..28508bde4cd 100644 --- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc +++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc @@ -44,6 +44,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h" #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_fft.h" #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h" +#include "tensorflow/compiler/xla/service/cpu/runtime_topk.h" #include "tensorflow/compiler/xla/service/cpu/windows_compatibility.h" #include "tensorflow/compiler/xla/service/custom_call_target_registry.h" #include "tensorflow/compiler/xla/types.h" @@ -270,6 +271,7 @@ bool RegisterKnownJITSymbols() { REGISTER_CPU_RUNTIME_SYMBOL(ReleaseInfeedBufferAfterDequeue); REGISTER_CPU_RUNTIME_SYMBOL(ReleaseOutfeedBufferAfterPopulation); REGISTER_CPU_RUNTIME_SYMBOL(KeyValueSort); + REGISTER_CPU_RUNTIME_SYMBOL(TopKF32); REGISTER_CPU_RUNTIME_SYMBOL(TracingStart); REGISTER_CPU_RUNTIME_SYMBOL(TracingEnd); diff --git a/tensorflow/compiler/xla/service/cpu/tests/BUILD b/tensorflow/compiler/xla/service/cpu/tests/BUILD index d7c50dce3ca..527071d5f31 100644 --- a/tensorflow/compiler/xla/service/cpu/tests/BUILD +++ b/tensorflow/compiler/xla/service/cpu/tests/BUILD @@ -253,6 +253,22 @@ tf_cc_test( ], ) +tf_cc_test( + name = "cpu_topk_test", + srcs = ["cpu_topk_test.cc"], + deps = [ + ":cpu_codegen_test", + "//tensorflow/compiler/xla/client:xla_builder", + "//tensorflow/compiler/xla/client/lib:sorting", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/compiler/xla/service/cpu:cpu_compiler", + "//tensorflow/compiler/xla/service/cpu:test_header_helper", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + tf_cc_test( name = "cpu_vectorization_test", srcs = ["cpu_vectorization_test.cc"], diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_topk_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_topk_test.cc new file mode 100644 index 00000000000..a4c74cfb8a2 --- /dev/null +++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_topk_test.cc @@ -0,0 +1,59 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/xla/client/lib/sorting.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/service/cpu/cpu_compiler.h" +#include "tensorflow/compiler/xla/service/cpu/test_target_triple_helper.h" +#include "tensorflow/compiler/xla/service/cpu/tests/cpu_codegen_test.h" + +namespace xla { +namespace cpu { +namespace { + +using CpuTopKTest = CpuCodegenTest; + +TEST_F(CpuTopKTest, CallRuntime) { + XlaBuilder builder(TestName()); + XlaOp input = + Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {5, 100}), "input"); + TopK(input, 10); + TF_ASSERT_OK_AND_ASSIGN(XlaComputation xla_computation, builder.Build()); + + TF_ASSERT_OK_AND_ASSIGN(ProgramShape program_shape, + xla_computation.GetProgramShape()); + HloModuleConfig config(program_shape); + TF_ASSERT_OK_AND_ASSIGN( + auto module, HloModule::CreateFromProto(xla_computation.proto(), config)); + + constexpr char filecheck_pattern[] = R"( + CHECK: call void @__xla_cpu_runtime_TopKF32(i64 5, i64 100, i64 10, + )"; + + CpuAotCompilationOptions options{ + /*triple=*/kTargetTripleForHost, /*cpu_name=*/kTargetCpuForHost, + /*features=*/"", + /*entry_point_name=*/"entry", + /*relocation_model=*/CpuAotCompilationOptions::RelocationModel::Static}; + + CompileAheadOfTimeAndVerifyIr(std::move(module), options, filecheck_pattern, + /*match_optimized_ir=*/true); +} + +} // namespace +} // namespace cpu +} // namespace xla From 2d2523c8c71f1c4aec8ca25fc7ece5a7f2ede9a6 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 5 Aug 2020 02:37:26 -0700 Subject: [PATCH 0445/1017] [XLA] Rewrite 1d sort to TopK PiperOrigin-RevId: 324982391 Change-Id: Ia324c70137647117154ca21db3fc640c3e29606f --- .../compiler/xla/service/cpu/ir_emitter.cc | 9 +-- .../xla/service/cpu/tests/cpu_topk_test.cc | 29 +++++++- .../compiler/xla/service/topk_rewriter.cc | 27 +++++--- .../xla/service/topk_rewriter_test.cc | 69 ++++++++++--------- 4 files changed, 88 insertions(+), 46 deletions(-) diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 2688a7898af..72f4d5369c8 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -2390,7 +2390,8 @@ Status IrEmitter::HandlePadToStatic(HloInstruction* hlo) { Status IrEmitter::HandleTopK(HloInstruction* hlo) { TF_RETURN_IF_ERROR(EmitTargetAddressForOp(hlo)); const HloInstruction* input = hlo->operand(0); - int64 k = hlo->shape().tuple_shapes(0).dimensions(1); + const int64 k = hlo->shape().tuple_shapes(0).dimensions().back(); + const bool has_batch = hlo->shape().tuple_shapes(0).dimensions_size() == 2; TF_RET_CHECK(input->shape().element_type() == F32); TF_RET_CHECK(LayoutUtil::IsMonotonicWithDim0Major( hlo->shape().tuple_shapes(0).layout())); @@ -2412,9 +2413,9 @@ Status IrEmitter::HandleTopK(HloInstruction* hlo) { llvm::Value* out_indices_ptr = EmitBufferPointer(out_indices_slice, hlo->shape().tuple_shapes(1)); EmitCallToFunc(runtime::kTopKF32SymbolName, - {b_.getInt64(input->shape().dimensions(0)), - b_.getInt64(input->shape().dimensions(1)), b_.getInt64(k), - values_ptr, out_values_ptr, out_indices_ptr}, + {b_.getInt64(has_batch ? input->shape().dimensions(0) : 1), + b_.getInt64(input->shape().dimensions().back()), + b_.getInt64(k), values_ptr, out_values_ptr, out_indices_ptr}, b_.getVoidTy()); llvm_ir::EmitTuple(GetIrArrayFor(hlo), {out_values_ptr, out_indices_ptr}, diff --git a/tensorflow/compiler/xla/service/cpu/tests/cpu_topk_test.cc b/tensorflow/compiler/xla/service/cpu/tests/cpu_topk_test.cc index a4c74cfb8a2..b7647fb4b16 100644 --- a/tensorflow/compiler/xla/service/cpu/tests/cpu_topk_test.cc +++ b/tensorflow/compiler/xla/service/cpu/tests/cpu_topk_test.cc @@ -27,7 +27,34 @@ namespace { using CpuTopKTest = CpuCodegenTest; -TEST_F(CpuTopKTest, CallRuntime) { +TEST_F(CpuTopKTest, CallRuntimeUnbatched) { + XlaBuilder builder(TestName()); + XlaOp input = + Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {100}), "input"); + TopK(input, 10); + TF_ASSERT_OK_AND_ASSIGN(XlaComputation xla_computation, builder.Build()); + + TF_ASSERT_OK_AND_ASSIGN(ProgramShape program_shape, + xla_computation.GetProgramShape()); + HloModuleConfig config(program_shape); + TF_ASSERT_OK_AND_ASSIGN( + auto module, HloModule::CreateFromProto(xla_computation.proto(), config)); + + constexpr char filecheck_pattern[] = R"( + CHECK: call void @__xla_cpu_runtime_TopKF32(i64 1, i64 100, i64 10, + )"; + + CpuAotCompilationOptions options{ + /*triple=*/kTargetTripleForHost, /*cpu_name=*/kTargetCpuForHost, + /*features=*/"", + /*entry_point_name=*/"entry", + /*relocation_model=*/CpuAotCompilationOptions::RelocationModel::Static}; + + CompileAheadOfTimeAndVerifyIr(std::move(module), options, filecheck_pattern, + /*match_optimized_ir=*/true); +} + +TEST_F(CpuTopKTest, CallRuntimeBatched) { XlaBuilder builder(TestName()); XlaOp input = Parameter(&builder, 0, ShapeUtil::MakeShape(F32, {5, 100}), "input"); diff --git a/tensorflow/compiler/xla/service/topk_rewriter.cc b/tensorflow/compiler/xla/service/topk_rewriter.cc index ae843760a8d..000b1e94ece 100644 --- a/tensorflow/compiler/xla/service/topk_rewriter.cc +++ b/tensorflow/compiler/xla/service/topk_rewriter.cc @@ -75,11 +75,11 @@ StatusOr TopkRewriter::Run(HloModule* module) { HloIotaInstruction* iota = DynCast(sort->mutable_operand(1)); const PrimitiveType element_type = data->shape().element_type(); - if (data->shape().rank() != 2 || + if ((data->shape().rank() != 1 && data->shape().rank() != 2) || (element_type != F32 && element_type != BF16)) { continue; } - if (iota == nullptr || iota->shape().rank() != 2 || + if (iota == nullptr || iota->shape().rank() != data->shape().rank() || iota->shape().element_type() != S32 || iota->opcode() != HloOpcode::kIota || iota->iota_dimension() != sort->sort_dimension()) { @@ -90,6 +90,7 @@ StatusOr TopkRewriter::Run(HloModule* module) { } const int64 sort_dim = sort->sort_dimension(); const int64 batch_dim = sort_dim == 1 ? 0 : 1; + const bool has_batch = data->shape().rank() == 2; bool supported = true; absl::optional k; @@ -113,8 +114,8 @@ StatusOr TopkRewriter::Run(HloModule* module) { supported = false; break; } - if (slice->slice_limits(batch_dim) != - slice->operand(0)->shape().dimensions(batch_dim)) { + if (has_batch && slice->slice_limits(batch_dim) != + slice->operand(0)->shape().dimensions(batch_dim)) { // Slicing along the batch dimension isn't supported. supported = false; break; @@ -136,18 +137,24 @@ StatusOr TopkRewriter::Run(HloModule* module) { continue; } - const int64 batch_size = sort->operand(0)->shape().dimensions(batch_dim); + const int64 batch_size = + has_batch ? sort->operand(0)->shape().dimensions(batch_dim) : 1; const int64 input_size = sort->operand(0)->shape().dimensions(sort_dim); HloInstruction* input = sort->mutable_operand(0); - if (sort_dim == 0) { + if (has_batch && sort_dim == 0) { input = comp->AddInstruction(HloInstruction::CreateTranspose( ShapeUtil::MakeShape(element_type, {batch_size, input_size}), input, {1, 0})); } - Shape topk_shape = ShapeUtil::MakeTupleShape( - {ShapeUtil::MakeShape(element_type, {batch_size, k.value()}), - ShapeUtil::MakeShape(S32, {batch_size, k.value()})}); + Shape topk_shape = + has_batch ? ShapeUtil::MakeTupleShape( + {ShapeUtil::MakeShape(element_type, + {batch_size, k.value()}), + ShapeUtil::MakeShape(S32, {batch_size, k.value()})}) + : ShapeUtil::MakeTupleShape( + {ShapeUtil::MakeShape(element_type, {k.value()}), + ShapeUtil::MakeShape(S32, {k.value()})}); HloInstruction* topk = comp->AddInstruction( HloInstruction::CreateCustomCall(topk_shape, {input}, "TopK")); HloInstruction* value_gte = @@ -157,7 +164,7 @@ StatusOr TopkRewriter::Run(HloModule* module) { comp->AddInstruction(HloInstruction::CreateGetTupleElement( topk->shape().tuple_shapes(1), topk, 1)); - if (sort_dim == 0) { + if (has_batch && sort_dim == 0) { value_gte = comp->AddInstruction(HloInstruction::CreateTranspose( ShapeUtil::MakeShape(element_type, {k.value(), batch_size}), value_gte, {1, 0})); diff --git a/tensorflow/compiler/xla/service/topk_rewriter_test.cc b/tensorflow/compiler/xla/service/topk_rewriter_test.cc index e440da5b163..ec5b34b1c0a 100644 --- a/tensorflow/compiler/xla/service/topk_rewriter_test.cc +++ b/tensorflow/compiler/xla/service/topk_rewriter_test.cc @@ -31,10 +31,8 @@ namespace { using TopkRewriterTest = HloTestBase; -TEST_F(TopkRewriterTest, Rewrite) { - const char* const hlo_string = R"( -HloModule module - +std::string getComparator() { + return R"( %compare { %p.1.lhs.8 = s32[] parameter(2) %p.1.rhs.9 = s32[] parameter(3) @@ -59,8 +57,13 @@ HloModule module %select.25 = s32[] select(%compare.24, %bitcast-convert.22, %bitcast-convert.19) ROOT %compare.26 = pred[] compare(%select.17, %select.25), direction=GT +})"; } +TEST_F(TopkRewriterTest, Rewrite) { + const std::string hlo_string = R"( +HloModule module +)" + getComparator() + R"( ENTRY cluster { %arg_tuple.1 = f32[8,1234567] parameter(0) %iota.4 = s32[8,1234567] iota(), iota_dimension=1 @@ -87,36 +90,40 @@ ENTRY cluster { EXPECT_THAT(cc->custom_call_target(), "TopK"); } -TEST_F(TopkRewriterTest, RewriteTranspose) { - const char* const hlo_string = R"( +TEST_F(TopkRewriterTest, RewriteUnbatched) { + const std::string hlo_string = R"( HloModule module - -%compare { - %p.1.lhs.8 = s32[] parameter(2) - %p.1.rhs.9 = s32[] parameter(3) - %p.0.lhs.6 = f32[] parameter(0) - %bitcast-convert.11 = s32[] bitcast-convert(%p.0.lhs.6) - %constant.15 = s32[] constant(0) - %compare.16 = pred[] compare(%bitcast-convert.11, %constant.15), direction=LT - %constant.10 = u32[] constant(2147483647) - %bitcast-convert.12 = u32[] bitcast-convert(%p.0.lhs.6) - %subtract.13 = u32[] subtract(%constant.10, %bitcast-convert.12) - %bitcast-convert.14 = s32[] bitcast-convert(%subtract.13) - %select.17 = s32[] select(%compare.16, %bitcast-convert.14, - %bitcast-convert.11) - %p.0.rhs.7 = f32[] parameter(1) - %bitcast-convert.19 = s32[] bitcast-convert(%p.0.rhs.7) - %constant.23 = s32[] constant(0) - %compare.24 = pred[] compare(%bitcast-convert.19, %constant.23), direction=LT - %constant.18 = u32[] constant(2147483647) - %bitcast-convert.20 = u32[] bitcast-convert(%p.0.rhs.7) - %subtract.21 = u32[] subtract(%constant.18, %bitcast-convert.20) - %bitcast-convert.22 = s32[] bitcast-convert(%subtract.21) - %select.25 = s32[] select(%compare.24, %bitcast-convert.22, - %bitcast-convert.19) - ROOT %compare.26 = pred[] compare(%select.17, %select.25), direction=GT +)" + getComparator() + R"( +ENTRY cluster { + %arg_tuple.1 = f32[1234567] parameter(0) + %iota.4 = s32[1234567] iota(), iota_dimension=0 + %sort.27 = (f32[1234567], s32[1234567]) sort(%arg_tuple.1, %iota.4), + dimensions={0}, is_stable=true, to_apply=%compare + %get-tuple-element.28 = f32[1234567] get-tuple-element(%sort.27), index=0 + %slice.29 = f32[5] slice(%get-tuple-element.28), slice={[0:5]} + %get-tuple-element.30 = s32[1234567] get-tuple-element(%sort.27), index=1 + %slice.31 = s32[5] slice(%get-tuple-element.30), slice={[0:5]} + ROOT %tuple.32 = (f32[5], s32[5]) tuple(%slice.29, %slice.31) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TopkRewriter rewriter([](const HloSortInstruction*, int64) { return true; }); + TF_ASSERT_OK_AND_ASSIGN(bool changed, rewriter.Run(module.get())); + TF_ASSERT_OK(HloDCE().Run(module.get()).status()); + EXPECT_TRUE(changed); + EXPECT_THAT( + module->entry_computation()->root_instruction(), + op::Tuple(op::GetTupleElement(op::CustomCall(op::Parameter(0)), 0), + op::GetTupleElement(op::CustomCall(op::Parameter(0)), 1))); + const HloInstruction* cc = + module->entry_computation()->root_instruction()->operand(0)->operand(0); + EXPECT_THAT(cc->custom_call_target(), "TopK"); } +TEST_F(TopkRewriterTest, RewriteTranspose) { + const std::string hlo_string = R"( +HloModule module +)" + getComparator() + R"( ENTRY cluster { %arg_tuple.1 = f32[1234567,8] parameter(0) %iota.4 = s32[1234567,8] iota(), iota_dimension=0 From a833385e49b0c79d5f73cd505de217bab85a8afc Mon Sep 17 00:00:00 2001 From: Thai Nguyen Date: Wed, 5 Aug 2020 03:23:12 -0700 Subject: [PATCH 0446/1017] Fix flex delegate selective build in OSS PiperOrigin-RevId: 324987504 Change-Id: I833b4f62f10f16f072cd776225379cded9dea775 --- tensorflow/lite/delegates/flex/build_def.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/lite/delegates/flex/build_def.bzl b/tensorflow/lite/delegates/flex/build_def.bzl index b4965d1076e..9b9f1b2c4cb 100644 --- a/tensorflow/lite/delegates/flex/build_def.bzl +++ b/tensorflow/lite/delegates/flex/build_def.bzl @@ -136,6 +136,7 @@ def tflite_flex_cc_library( "@com_google_absl//absl/types:optional", "@gemmlowp", "//tensorflow/core:protos_all_cc", + "@icu//:common", "//tensorflow/core:portable_tensorflow_lib_lite", "//tensorflow/core/platform:strong_hash", ], From 3f07f84a0e8e09cbd7261fb2eb2b51a31f20d51a Mon Sep 17 00:00:00 2001 From: Stephan Herhut Date: Wed, 5 Aug 2020 03:44:37 -0700 Subject: [PATCH 0447/1017] Add bufferization pass that transforms hlo and some standard ops. This is good enough to do a tanh operation. PiperOrigin-RevId: 324989254 Change-Id: Ief17856bd17dc9d21feba4ed909d7499a54bdc9d --- .../mlir/tools/kernel_gen/transforms/BUILD | 23 +++- .../tools/kernel_gen/transforms/bufferize.cc | 110 ++++++++++++++++++ .../kernel_gen/transforms/bufferize_pass.cc | 107 +++++++++++++++++ .../mlir/tools/kernel_gen/transforms/passes.h | 6 +- .../tools/kernel_gen/transforms/passes.td | 5 + .../tools/kernel_gen/transforms/rewriters.h | 11 ++ .../transforms/shape_to_descriptors_pass.cc | 1 + 7 files changed, 260 insertions(+), 3 deletions(-) create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize.cc create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize_pass.cc diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD index 0d346da9956..66a378d5990 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD @@ -20,6 +20,20 @@ cc_library( ], ) +cc_library( + name = "bufferize", + srcs = ["bufferize.cc"], + hdrs = ["rewriters.h"], + deps = [ + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:StandardOps", + "@llvm-project//mlir:Support", + "@llvm-project//mlir:Transforms", + ], +) + cc_library( name = "embed_tf_framework", srcs = ["embed_tf_framework.cc"], @@ -36,7 +50,7 @@ cc_library( ) gentbl( - name = "tf_framework_passes_inc_gen", + name = "kernel_gen_passes_inc_gen", tbl_outs = [("-gen-pass-decls -name KernelGen", "kernel_gen_passes.h.inc")], tblgen = "@llvm-project//mlir:mlir-tblgen", td_file = "passes.td", @@ -46,15 +60,20 @@ gentbl( cc_library( name = "passes", srcs = [ + "bufferize_pass.cc", "embed_tf_framework_pass.cc", "shape_to_descriptors_pass.cc", "tf_framework_legalize_to_llvm_pass.cc", ], hdrs = ["passes.h"], deps = [ + ":bufferize", ":embed_tf_framework", + ":kernel_gen_passes_inc_gen", ":tf_framework_legalize_to_llvm", - ":tf_framework_passes_inc_gen", + "//tensorflow/compiler/mlir/hlo", + "//tensorflow/compiler/mlir/hlo:hlo_legalize_to_lhlo", + "//tensorflow/compiler/mlir/hlo:lhlo", "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_ops", "@llvm-project//llvm:Support", "@llvm-project//mlir:IR", diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize.cc new file mode 100644 index 00000000000..3d5c820e6dd --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize.cc @@ -0,0 +1,110 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file implements logic for translating mixed IR to buffer form. + +#include +#include + +#include "llvm/ADT/STLExtras.h" +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/Function.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/StandardTypes.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Transforms/BufferPlacement.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project + +namespace mlir { +namespace kernel_gen { +namespace transforms { + +namespace { + +class TensorFromElementsOpConverter + : public BufferAssignmentOpConversionPattern { + public: + using BufferAssignmentOpConversionPattern< + TensorFromElementsOp>::BufferAssignmentOpConversionPattern; + + LogicalResult matchAndRewrite( + TensorFromElementsOp op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + Location loc = op.getLoc(); + ShapedType result_type = op.getType().cast(); + int number_of_elements = op.elements().size(); + MemRefType memref_type = + MemRefType::get({number_of_elements}, result_type.getElementType()); + Value result = rewriter.create(loc, memref_type); + for (auto operand : llvm::enumerate(operands)) { + Value index = rewriter.create(loc, operand.index()); + rewriter.create(loc, operand.value(), result, index); + } + rewriter.replaceOp(op, {result}); + return success(); + } +}; + +class TensorLoadOpConversion + : public BufferAssignmentOpConversionPattern { + public: + using BufferAssignmentOpConversionPattern< + TensorLoadOp>::BufferAssignmentOpConversionPattern; + + LogicalResult matchAndRewrite( + TensorLoadOp op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + TensorLoadOpAdaptor adaptor(operands); + rewriter.replaceOp(op, {adaptor.memref()}); + return success(); + } +}; + +class ExtractElementOpConversion + : public BufferAssignmentOpConversionPattern { + public: + using BufferAssignmentOpConversionPattern< + ExtractElementOp>::BufferAssignmentOpConversionPattern; + + LogicalResult matchAndRewrite( + ExtractElementOp op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const final { + ExtractElementOpAdaptor adaptor(operands); + + if (!adaptor.aggregate().getType().isa()) { + return failure(); + } + + rewriter.replaceOpWithNewOp(op, adaptor.aggregate(), + adaptor.indices()); + return success(); + } +}; + +} // namespace + +void populateStandardBufferizePattern(MLIRContext *context, + BufferAssignmentPlacer *bufferAssignment, + TypeConverter *converter, + OwningRewritePatternList *patterns) { + patterns->insert(context, bufferAssignment, + converter); +} + +} // namespace transforms +} // namespace kernel_gen +} // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize_pass.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize_pass.cc new file mode 100644 index 00000000000..ebbc92f64c7 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize_pass.cc @@ -0,0 +1,107 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file implements logic for translating mixed IR to buffer form. +// Currently it supports MHLO and some operations from the Standard dialect. + +#include + +#include "mlir/Dialect/SCF/SCF.h" // from @llvm-project +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/Function.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/StandardTypes.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Transforms/BufferPlacement.h" // from @llvm-project +#include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.h" +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/passes.h" +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/rewriters.h" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h" + +namespace mlir { +namespace kernel_gen { +namespace transforms { +namespace { + +#define GEN_PASS_CLASSES +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/kernel_gen_passes.h.inc" + +struct BufferizePass : public BufferizePassBase { + public: + void runOnOperation() override { + OwningRewritePatternList patterns; + auto& context = getContext(); + ConversionTarget target(context); + target.addLegalDialect(); + target.addLegalDialect(); + target.addLegalDialect(); + target.addLegalOp(); + target.addLegalOp(); + target.addIllegalDialect(); + target.addIllegalOp(); + target.addIllegalOp(); + target.addIllegalOp(); + + BufferAssignmentTypeConverter converter; + auto typesAreLegal = [&converter](Operation* op) { + return converter.isLegal(op->getOperandTypes()) && + converter.isLegal(op->getResultTypes()); + }; + target.addDynamicallyLegalOp([&](FuncOp op) { + auto inputs = op.getType().getInputs(); + auto results = op.getType().getResults(); + return converter.isLegal(inputs) && converter.isLegal(results) && + converter.isLegal(&op.getBody()); + }); + target.addDynamicallyLegalOp(typesAreLegal); + target.addDynamicallyLegalOp(typesAreLegal); + + auto module = getOperation(); + WalkResult result = module.walk([&](FuncOp func) -> WalkResult { + BufferAssignmentPlacer bufferAssignment(func); + OwningRewritePatternList patterns; + mhlo::populateHLOToLHLOConversionPattern( + func.getContext(), &bufferAssignment, &converter, &patterns); + populateWithBufferAssignmentOpConversionPatterns< + ReturnOp, ReturnOp, lmhlo::CopyOp, + /*allowMemrefFunctionResults=*/true>(&context, &bufferAssignment, + &converter, &patterns); + populateStandardBufferizePattern(func.getContext(), &bufferAssignment, + &converter, &patterns); + + return applyFullConversion(func, target, patterns); + }); + module.dump(); + if (result.wasInterrupted()) { + signalPassFailure(); + } + } +}; + +} // namespace + +std::unique_ptr > CreateBufferizePass() { + return std::make_unique(); +} + +} // namespace transforms +} // namespace kernel_gen +} // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h index 13f367c9fe4..e65d8402fb2 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h @@ -41,7 +41,11 @@ namespace transforms { // Pass to tranform shape computations in shape dialect to standard and scf // using memref descriptors. -std::unique_ptr CreateShapeToDescriptorsPass(); +std::unique_ptr > CreateShapeToDescriptorsPass(); + +// Pass to tranform computations on values to their corresponding parts on +// buffers. +std::unique_ptr > CreateBufferizePass(); } // namespace transforms diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td index 61720674926..6a0e328f212 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.td @@ -34,4 +34,9 @@ def ShapeToDescriptorsPass : Pass<"test-shape-to-descriptors", "ModuleOp"> { let constructor = "transforms::CreateShapeToDescriptorsPass()"; } +def BufferizePass : Pass<"test-bufferize", "ModuleOp"> { + let summary = "Pass to transform operations on values to buffer based ones"; + let constructor = "transforms::CreateBufferizePass()"; +} + #endif // TF_FRAMEWORK_PASSES diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h index 257e84b4a21..4efc1e95bc8 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h @@ -20,6 +20,7 @@ limitations under the License. namespace mlir { +class BufferAssignmentPlacer; class LLVMTypeConverter; class MLIRContext; class OwningRewritePatternList; @@ -37,6 +38,16 @@ void PopulateEmbedTFFrameworkConversionPatterns( MLIRContext *context, OwningRewritePatternList *patterns); } // namespace tf_framework + +namespace transforms { + +/// Collects a set of patterns that bufferize operations from the standard +/// dialect. +void populateStandardBufferizePattern(MLIRContext *context, + BufferAssignmentPlacer *bufferAssignment, + TypeConverter *converter, + OwningRewritePatternList *patterns); +} // namespace transforms } // namespace kernel_gen } // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors_pass.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors_pass.cc index 9c1b434b9b2..28d3647bb63 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors_pass.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/shape_to_descriptors_pass.cc @@ -26,6 +26,7 @@ limitations under the License. #include "mlir/IR/PatternMatch.h" // from @llvm-project #include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Transforms/DialectConversion.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" namespace mlir { namespace kernel_gen { From da84c4fd367d157578f111e36dd69a821cd5179e Mon Sep 17 00:00:00 2001 From: Stephan Herhut Date: Wed, 5 Aug 2020 03:52:27 -0700 Subject: [PATCH 0448/1017] Remove optional static registration for hlo dialects again. Instead, we invoke multiple test tools in a row in end to end tests now. For hlo dialects and passes, we use mlir-hlo-opt explicitly. PiperOrigin-RevId: 324989884 Change-Id: I2601dee460075d05cf0befe250abb91967317f1b --- tensorflow/compiler/mlir/hlo/BUILD | 7 ----- .../mhlo/transforms/register_all_passes.cc | 28 ------------------- 2 files changed, 35 deletions(-) delete mode 100644 tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/register_all_passes.cc diff --git a/tensorflow/compiler/mlir/hlo/BUILD b/tensorflow/compiler/mlir/hlo/BUILD index 9eee39894e4..3bbe628cccd 100644 --- a/tensorflow/compiler/mlir/hlo/BUILD +++ b/tensorflow/compiler/mlir/hlo/BUILD @@ -807,13 +807,6 @@ cc_library( ], ) -cc_library( - name = "register_all_passes", - srcs = ["lib/Dialect/mhlo/transforms/register_all_passes.cc"], - deps = [":all_passes"], - alwayslink = 1, -) - cc_binary( name = "mlir-hlo-opt", srcs = [ diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/register_all_passes.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/register_all_passes.cc deleted file mode 100644 index 9349bee041e..00000000000 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/register_all_passes.cc +++ /dev/null @@ -1,28 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "mlir-hlo/Dialect/mhlo/transforms/register_passes.h" - -namespace mlir { - -namespace { - -bool register_all_passes = ([] { - mhlo::registerAllMhloPasses(); - lmhlo::registerAllLmhloPasses(); -}(), true); - -} // namespace -} // namespace mlir From 671844cd4186d04de4aaec2df4458a9753d0dac5 Mon Sep 17 00:00:00 2001 From: Chao Mei Date: Wed, 5 Aug 2020 04:08:12 -0700 Subject: [PATCH 0449/1017] Support to generate zip tests (i.e. cc_test) against a particular delegate, like xnnpack delegate. PiperOrigin-RevId: 324991483 Change-Id: I1876d23363a854791ddc8e24d398e9f49c625be3 --- tensorflow/lite/build_def.bzl | 28 ++++++++++++++++++++++++++-- tensorflow/lite/testing/BUILD | 21 ++++++++++++--------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/tensorflow/lite/build_def.bzl b/tensorflow/lite/build_def.bzl index 001b2fc791e..4de0be7c3fa 100644 --- a/tensorflow/lite/build_def.bzl +++ b/tensorflow/lite/build_def.bzl @@ -578,7 +578,14 @@ def flags_for_merged_test_models(test_name, conversion_mode): tests_csv = tests_csv[:-1] # Remove trailing comma. return " --no_tests_limit --test_sets=%s" % tests_csv -def gen_zip_test(name, test_name, conversion_mode, **kwargs): +def gen_zip_test( + name, + test_name, + conversion_mode, + test_tags, + test_args, + additional_test_args = {}, + **kwargs): """Generate a zipped-example test and its dependent zip files. Args: @@ -586,6 +593,11 @@ def gen_zip_test(name, test_name, conversion_mode, **kwargs): test_name: str. Test targets this model. Comes from the list above. conversion_mode: str. Which conversion mode to run with. Comes from the list above. + test_tags: tags for the generated cc_test. + test_args: the basic cc_test args to be used. + additional_test_args: a dictionary of additional args to be used together + with test_args. The key is an identifier to be used in test tag, and + the value is a list of additional test args to be used. **kwargs: tf_cc_test kwargs """ toco = "//tensorflow/lite/toco:toco" @@ -603,7 +615,19 @@ def gen_zip_test(name, test_name, conversion_mode, **kwargs): toco = toco, flags = flags + " --save_graphdefs", ) - tf_cc_test(name, **kwargs) + tf_cc_test( + name, + args = test_args, + tags = test_tags + ["gen_zip_test"], + **kwargs + ) + for key, value in additional_test_args.items(): + tf_cc_test( + name = "%s_%s" % (name, key), + args = test_args + value, + tags = test_tags + ["gen_zip_test_%s" % key], + **kwargs + ) def gen_zipped_test_file(name, file, toco, flags): """Generate a zip file of tests by using :generate_examples. diff --git a/tensorflow/lite/testing/BUILD b/tensorflow/lite/testing/BUILD index 3d4527e926e..4bfc17dc509 100644 --- a/tensorflow/lite/testing/BUILD +++ b/tensorflow/lite/testing/BUILD @@ -35,7 +35,16 @@ exports_files([ name = "zip_test_%s" % test_name, size = "medium", srcs = ["generated_examples_zip_test.cc"], - args = args + select({ + additional_test_args = { + # TODO(b/162696268): uncomment once the bug is fixed. + # "xnnpack": ["--use_xnnpack=true"], + }, + conversion_mode = conversion_mode, + data = [ + ":zip_%s" % test_name, + ], + shard_count = 20, + test_args = args + select({ "//tensorflow:android": [], "//conditions:default": [ "--zip_file_path=$(location :zip_%s)" % test_name, @@ -44,18 +53,12 @@ exports_files([ "--unzip_binary_path=/usr/bin/unzip", ], }), - conversion_mode = conversion_mode, - data = [ - ":zip_%s" % test_name, - ], - shard_count = 20, - tags = tags + [ - "gen_zip_test", + test_name = test_name, + test_tags = tags + [ "no_gpu", # Executing with TF GPU configurations is redundant. "no_oss", "tflite_not_portable_intentional", ], - test_name = test_name, deps = [ ":parse_testdata_lib", ":tflite_driver", From 4a64fa3df8455d06a81c126a28262f64db60d15b Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 5 Aug 2020 04:20:39 -0700 Subject: [PATCH 0450/1017] [XLA:CPU] Fix a crash in topk emission by canonicalizing pointers If there are multiple topk of different shapes in the same module, the signature of our runtime function will contain the shapes of the first instance. Subsequent instances clash with that signature. Canonicalize the types so all signatures become identical. PiperOrigin-RevId: 324992669 Change-Id: Ibbbfdd671dedfcdfdb85706e3cffdf8d64859da6 --- tensorflow/compiler/xla/service/cpu/ir_emitter.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc index 72f4d5369c8..242f3c6ceb7 100644 --- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc @@ -2412,11 +2412,14 @@ Status IrEmitter::HandleTopK(HloInstruction* hlo) { EmitBufferPointer(out_values_slice, hlo->shape().tuple_shapes(0)); llvm::Value* out_indices_ptr = EmitBufferPointer(out_indices_slice, hlo->shape().tuple_shapes(1)); - EmitCallToFunc(runtime::kTopKF32SymbolName, - {b_.getInt64(has_batch ? input->shape().dimensions(0) : 1), - b_.getInt64(input->shape().dimensions().back()), - b_.getInt64(k), values_ptr, out_values_ptr, out_indices_ptr}, - b_.getVoidTy()); + EmitCallToFunc( + runtime::kTopKF32SymbolName, + {b_.getInt64(has_batch ? input->shape().dimensions(0) : 1), + b_.getInt64(input->shape().dimensions().back()), b_.getInt64(k), + BitCast(values_ptr, b_.getFloatTy()->getPointerTo()), + BitCast(out_values_ptr, b_.getFloatTy()->getPointerTo()), + BitCast(out_indices_ptr, b_.getInt32Ty()->getPointerTo())}, + b_.getVoidTy()); llvm_ir::EmitTuple(GetIrArrayFor(hlo), {out_values_ptr, out_indices_ptr}, &b_); From 3c945050395eb6f283e8838257256ab4574b7e4a Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Wed, 5 Aug 2020 04:30:28 -0700 Subject: [PATCH 0451/1017] Integrate LLVM at llvm/llvm-project@c558c22cab9a Updates LLVM usage to match [c558c22cab9a](https://github.com/llvm/llvm-project/commit/c558c22cab9a) PiperOrigin-RevId: 324993578 Change-Id: I5de7a4aa5c53170f2a749b93e7038f49b9d0721c --- tensorflow/workspace.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index b770dfeead5..ffe1e95beff 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "28e322ea9393e6b3841886006dd170ddd810fd9b" - LLVM_SHA256 = "438268a47b69687ea5e588a285a2255de414addc36e0405e1d70f7cb5208aa75" + LLVM_COMMIT = "c558c22cab9a555d2e521102b775759381e9727f" + LLVM_SHA256 = "b3651e78f4f3b372273c71cb58e0d0767b61e7d9c93b79fd399065c1148089f5" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), From 428d74a880b481665c1a5b47153f9ea4d21db08f Mon Sep 17 00:00:00 2001 From: Alexander Belyaev Date: Wed, 5 Aug 2020 05:16:47 -0700 Subject: [PATCH 0452/1017] [MLIR][KERNEL_GEN] Add tf framework C interface. All alloc and dealloc ops in the kernel have to use TF OpKernelContext. std.alloc and std.dealloc are converted to tf_framework.alloc_raw and tf_framework.dealloc_raw which are lowered to external calls. This PR adds a C++ library with the external functions and also adds a fake implementation for testing. PiperOrigin-RevId: 324998567 Change-Id: I8528ea3fdf68d0b653cc7b185563ffb50eca8571 --- tensorflow/compiler/mlir/hlo/BUILD | 1 + .../compiler/mlir/tools/kernel_gen/BUILD | 17 ++++++- .../tools/kernel_gen/ir/tf_framework_ops.cc | 13 ++--- .../tools/kernel_gen/ir/tf_framework_ops.td | 13 ++++- .../kernel_gen/tf_framework_c_interface.cc | 49 +++++++++++++++++++ .../kernel_gen/tf_framework_c_interface.h | 35 +++++++++++++ .../mlir/tools/kernel_gen/transforms/BUILD | 1 + .../tf_framework_legalize_to_llvm.cc | 14 ++++++ .../tf_framework_legalize_to_llvm_pass.cc | 2 + 9 files changed, 137 insertions(+), 8 deletions(-) create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tf_framework_c_interface.cc create mode 100644 tensorflow/compiler/mlir/tools/kernel_gen/tf_framework_c_interface.h diff --git a/tensorflow/compiler/mlir/hlo/BUILD b/tensorflow/compiler/mlir/hlo/BUILD index 3bbe628cccd..dd63b68b890 100644 --- a/tensorflow/compiler/mlir/hlo/BUILD +++ b/tensorflow/compiler/mlir/hlo/BUILD @@ -404,6 +404,7 @@ cc_library( cc_library( name = "lhlo_legalize_to_llvm", srcs = ["lib/Dialect/mhlo/transforms/lhlo_legalize_to_llvm.cc"], + hdrs = ["include/mlir-hlo/Dialect/mhlo/transforms/rewriters.h"], deps = [ ":lhlo", "@llvm-project//mlir:IR", diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/BUILD index 066ca221d5d..5befdcdc513 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/BUILD @@ -1,7 +1,10 @@ load("//tensorflow:tensorflow.bzl", "tf_cc_binary") load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") -licenses(["notice"]) +package( + default_visibility = [":friends"], + licenses = ["notice"], # Apache 2.0 +) package_group( name = "friends", @@ -74,3 +77,15 @@ tf_cc_binary( "@llvm-project//mlir:Support", ], ) + +exports_files(["tf_framework_c_interface.h"]) + +cc_library( + name = "tf_framework_c_interface", + srcs = ["tf_framework_c_interface.cc"], + hdrs = ["tf_framework_c_interface.h"], + deps = [ + "//tensorflow/core:framework", + "@llvm-project//mlir:mlir_runner_utils", + ], +) diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc index e67b5fd7f85..f85f1229fe8 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc @@ -58,10 +58,16 @@ void TFFrameworkDialect::printType(Type type, DialectAsmPrinter &os) const { } } +template +LogicalResult Verify(OpTy op) { + return success(); +} + //===----------------------------------------------------------------------===// // AllocRawOp //===----------------------------------------------------------------------===// -static LogicalResult Verify(AllocRawOp op) { +template <> +LogicalResult Verify(AllocRawOp op) { // Check that the total number of operands matches the number of dynamic // dimensions specified in the memref type. unsigned result_dyn_dims = op.getType().getNumDynamicDims(); @@ -74,11 +80,6 @@ static LogicalResult Verify(AllocRawOp op) { return success(); } -//===----------------------------------------------------------------------===// -// DeallocRawOp -//===----------------------------------------------------------------------===// -static LogicalResult Verify(DeallocRawOp op) { return success(); } - #define GET_OP_CLASSES #include "tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc.inc" diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.td b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.td index 65481ad377f..bc390a5aaa5 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.td +++ b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.td @@ -45,7 +45,7 @@ def TFFramework_OpKernelContextType : DialectType traits = []> : Op { - let verifier = "return Verify(*this);"; + let verifier = "return Verify<$cppClass>(*this);"; } //===----------------------------------------------------------------------===// @@ -111,4 +111,15 @@ def TFFramework_DeallocRawOp : TFFramework_Op<"dealloc_raw", let assemblyFormat = "`(` $ctx `,` $memref `)` attr-dict `:` type($memref)"; } +//===----------------------------------------------------------------------===// +// NullContextOp +//===----------------------------------------------------------------------===// +def TFFramework_NullContextOp : TFFramework_Op<"null_context", + [NoSideEffect]> { + let summary = "Creates a fake TF context that will be lowered to nullptr"; + let description = [{Needed for testing}]; + let results = (outs TFFramework_OpKernelContextType:$result); + let assemblyFormat = "`(` `)` attr-dict `:` type($result)"; +} + #endif // TF_FRAMEWORK_OPS diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tf_framework_c_interface.cc b/tensorflow/compiler/mlir/tools/kernel_gen/tf_framework_c_interface.cc new file mode 100644 index 00000000000..e75db59d885 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/tf_framework_c_interface.cc @@ -0,0 +1,49 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/tools/kernel_gen/tf_framework_c_interface.h" + +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/op_kernel.h" + +namespace mlir { +namespace kernel_gen { +namespace tf_framework { +namespace { + +using tensorflow::Allocator; + +Allocator* GetAllocator(void* op_kernel_ctx) { + auto* ctx = static_cast(op_kernel_ctx); + // TODO(pifon): Figure out how to set AllocatorAttributes correctly. + tensorflow::AllocatorAttributes attrs; + return ctx->get_allocator(attrs); +} + +} // namespace + +extern "C" void* _mlir_ciface_tf_alloc_raw(void* op_kernel_ctx, + size_t num_bytes) { + return GetAllocator(op_kernel_ctx) + ->AllocateRaw(Allocator::kAllocatorAlignment, num_bytes); +} + +extern "C" void _mlir_ciface_tf_dealloc_raw(void* op_kernel_ctx, void* ptr) { + GetAllocator(op_kernel_ctx)->DeallocateRaw(ptr); +} + +} // namespace tf_framework +} // namespace kernel_gen +} // namespace mlir diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/tf_framework_c_interface.h b/tensorflow/compiler/mlir/tools/kernel_gen/tf_framework_c_interface.h new file mode 100644 index 00000000000..143ebc95932 --- /dev/null +++ b/tensorflow/compiler/mlir/tools/kernel_gen/tf_framework_c_interface.h @@ -0,0 +1,35 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TESTS_TF_FRAMEWORK_C_INTERFACE_H_ +#define TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TESTS_TF_FRAMEWORK_C_INTERFACE_H_ + +#include "mlir/ExecutionEngine/RunnerUtils.h" // from @llvm-project + +namespace mlir { +namespace kernel_gen { +namespace tf_framework { + +extern "C" MLIR_RUNNERUTILS_EXPORT void* _mlir_ciface_tf_alloc_raw( + void* op_kernel_ctx, size_t num_bytes); + +extern "C" MLIR_RUNNERUTILS_EXPORT void _mlir_ciface_tf_dealloc_raw( + void* op_kernel_ctx, void* ptr); + +} // namespace tf_framework +} // namespace kernel_gen +} // namespace mlir + +#endif // TENSORFLOW_COMPILER_MLIR_TOOLS_KERNEL_GEN_TESTS_TF_FRAMEWORK_C_INTERFACE_H_ diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD index 66a378d5990..b0f22b40f5b 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/BUILD @@ -74,6 +74,7 @@ cc_library( "//tensorflow/compiler/mlir/hlo", "//tensorflow/compiler/mlir/hlo:hlo_legalize_to_lhlo", "//tensorflow/compiler/mlir/hlo:lhlo", + "//tensorflow/compiler/mlir/hlo:lhlo_legalize_to_llvm", "//tensorflow/compiler/mlir/tools/kernel_gen/ir:tf_framework_ops", "@llvm-project//llvm:Support", "@llvm-project//mlir:IR", diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm.cc index 2edcaabd7b4..3ce111ff3ff 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm.cc @@ -101,6 +101,7 @@ class AllocRawOpConverter : public ConvertToLLVMCallOpPattern { protected: StringRef GetFuncName() const override { return kCInterfaceAlloc; } + LLVMType GetFuncType() const override { LLVMType llvm_void_ptr_type = getVoidPtrType(); return LLVM::LLVMType::getFunctionTy( @@ -175,10 +176,23 @@ class DeallocRawOpConverter : public ConvertToLLVMCallOpPattern { } }; +class NullContextOpConverter : public ConvertOpToLLVMPattern { + public: + using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern; + + LogicalResult matchAndRewrite( + Operation *op, ArrayRef operands, + ConversionPatternRewriter &rewriter) const override { + rewriter.replaceOpWithNewOp(op, getVoidPtrType()); + return success(); + } +}; + } // namespace void PopulateTFFrameworkToLLVMConversionPatterns( LLVMTypeConverter *converter, OwningRewritePatternList *patterns) { + patterns->insert(*converter); patterns->insert(*converter); } diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc index 916eedb55de..41b38bb574f 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc @@ -18,6 +18,7 @@ limitations under the License. #include "mlir/Dialect/LLVMIR/LLVMDialect.h" // from @llvm-project #include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project #include "mlir/Pass/Pass.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/rewriters.h" #include "tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h" #include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/passes.h" #include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/rewriters.h" @@ -46,6 +47,7 @@ class TestTFFrameworkToLLVMPass OwningRewritePatternList patterns; populateStdToLLVMConversionPatterns(type_converter, patterns); PopulateTFFrameworkToLLVMConversionPatterns(&type_converter, &patterns); + lmhlo::PopulateLhloToLLVMConversionPatterns(&type_converter, &patterns); // Set target. ConversionTarget target(getContext()); From 43cd816657f12403cf58ebc61c53164cee3db484 Mon Sep 17 00:00:00 2001 From: Steenu Johnson Date: Wed, 5 Aug 2020 18:27:47 +0530 Subject: [PATCH 0453/1017] Making the log_warning option as an attribute in IgnoreErrorsDataset instead of defining a new op. Hence this commit also removes IgnoreErrorsDatasetV2 as it is no longer required. Signed-off-by: Steenu Johnson --- .../api_def_IgnoreErrorsDatasetV2.pbtxt | 7 ---- .../experimental/ignore_errors_dataset_op.cc | 37 ++++++------------- .../core/ops/experimental_dataset_ops.cc | 10 +---- .../python/data/experimental/ops/error_ops.py | 17 +++------ .../api/golden/v1/tensorflow.raw_ops.pbtxt | 8 +--- .../api/golden/v2/tensorflow.raw_ops.pbtxt | 8 +--- 6 files changed, 23 insertions(+), 64 deletions(-) delete mode 100644 tensorflow/core/api_def/base_api/api_def_IgnoreErrorsDatasetV2.pbtxt diff --git a/tensorflow/core/api_def/base_api/api_def_IgnoreErrorsDatasetV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_IgnoreErrorsDatasetV2.pbtxt deleted file mode 100644 index 2d8f8470869..00000000000 --- a/tensorflow/core/api_def/base_api/api_def_IgnoreErrorsDatasetV2.pbtxt +++ /dev/null @@ -1,7 +0,0 @@ -op { - graph_op_name: "IgnoreErrorsDatasetV2" - summary: <

+ return target.split("://")[1] -def _make_distributed_dataset(dataset, service, job_name=None): - """Creates a distributed dataset with a short task refresh interval.""" +def _make_distributed_dataset(dataset, + dispatcher, + job_name=None, + max_outstanding_requests=None): return dataset.apply( data_service_ops._distribute( "parallel_epochs", - service, + dispatcher.target, job_name=job_name, + max_outstanding_requests=max_outstanding_requests, task_refresh_interval_hint_ms=20)) +def _make_distributed_range_dataset(num_elements, + dispatcher, + job_name=None, + max_outstanding_requests=None): + """Creates a distributed dataset. + + Args: + num_elements: The number of elements in the range dataset that will be + distributed. + dispatcher: The dispatcher to distribute to. + job_name: Optional job name for the distributed dataset. + max_outstanding_requests: Optional limit on the number of outstanding + requests. + + Returns: + The created dataset. + """ + dataset = dataset_ops.Dataset.range(num_elements) + return _make_distributed_dataset(dataset, dispatcher, job_name, + max_outstanding_requests) + + class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): - def create_cluster(self, num_workers): + def start_dispatch_server(self, port=0): + work_dir = os.path.join(self.get_temp_dir(), "work_dir") + return server_lib.DispatchServer( + port=port, + protocol=server_lib.DEFAULT_PROTOCOL, + work_dir=work_dir, + fault_tolerant_mode=True) + + def start_worker_server(self, dispatcher, port=0): + return server_lib.WorkerServer( + port=port, + dispatcher_address=_address_from_target(dispatcher.target), + protocol=server_lib.DEFAULT_PROTOCOL) + + def restart_dispatcher(self, dispatcher): + """Stops `dispatcher` and returns a new dispatcher with the same port.""" + port = int(_address_from_target(dispatcher.target).split(":")[1]) + dispatcher._stop() + return self.start_dispatch_server(port=port) + + def start_cluster(self, num_workers): """Creates a cluster of tf.data service servers. Args: num_workers: The number of workers in the cluster. Returns: - A string for connecting to the tf.data service. + A tuple of (dispatcher, list_of_workers). """ - self._dispatcher = server_lib.DispatchServer(port=0, protocol=PROTOCOL) - self._servers = [] - for _ in range(num_workers): - self._servers.append( - server_lib.WorkerServer( - port=0, - dispatcher_address=self._dispatcher._address, - protocol=PROTOCOL)) - - return "{0}://{1}".format(PROTOCOL, self._dispatcher._address) + dispatcher = self.start_dispatch_server() + servers = [self.start_worker_server(dispatcher) for _ in range(num_workers)] + return dispatcher, servers @combinations.generate(test_base.eager_only_combinations()) def testDistributeBasic(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 10 - service = self.create_cluster(1) - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_range_dataset(10, dispatcher) results = [elem.numpy() for elem in ds] self.assertEqual(list(range(num_elements)), results) @combinations.generate(test_base.eager_only_combinations()) - def testDispatcherPreemption(self): - self._dispatcher = server_lib.DispatchServer(port=0, protocol=PROTOCOL) - self._worker = server_lib.WorkerServer( - port=0, dispatcher_address=self._dispatcher._address, protocol=PROTOCOL) + def testDispatcherStop(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 100 - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset( - ds, "{}://{}".format(PROTOCOL, self._dispatcher._address)) + ds = _make_distributed_range_dataset(num_elements, dispatcher) iterator = iter(ds) results = [] results.append(next(iterator).numpy()) - self._dispatcher._stop() + dispatcher._stop() # After the dispatcher dies, the worker should continue providing the rest # of the dataset's elements. for _ in range(num_elements - 1): results.append(next(iterator).numpy()) self.assertEqual(results, list(range(num_elements))) + @combinations.generate(test_base.eager_only_combinations()) + def testDispatcherRestartBeforeReading(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + num_elements = 100 + ds = _make_distributed_range_dataset(num_elements, dispatcher) + dispatcher = self.restart_dispatcher(dispatcher) + + self.assertDatasetProduces(ds, list(range(num_elements))) + + @combinations.generate(test_base.eager_only_combinations()) + def testDispatcherRestartDuringReading(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + num_elements = 100 + ds = _make_distributed_range_dataset(num_elements, dispatcher) + iterator = iter(ds) + results = [] + for _ in range(num_elements // 2): + results.append(next(iterator).numpy()) + dispatcher = self.restart_dispatcher(dispatcher) + for elem in iterator: + results.append(elem.numpy()) + + self.assertEqual(list(range(num_elements)), results) + + @combinations.generate(test_base.eager_only_combinations()) + def testDispatcherRestartBetweenIterations(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + num_elements = 100 + ds = _make_distributed_range_dataset(100, dispatcher) + self.assertDatasetProduces(ds, list(range(num_elements))) + dispatcher = self.restart_dispatcher(dispatcher) + self.assertDatasetProduces(ds, list(range(num_elements))) + @combinations.generate(test_base.eager_only_combinations()) def testDistributeSparse(self): - service = self.create_cluster(1) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable element = sparse_tensor.SparseTensor( indices=[[0]], values=constant_op.constant([0], dtype=dtypes.int32), dense_shape=[1]) ds = dataset_ops.Dataset.from_tensors(element) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_dataset(ds, dispatcher) results = [sparse_ops.sparse_tensor_to_dense(elem) for elem in ds] self.assertAllEqual(results, [[0]]) @combinations.generate(test_base.eager_only_combinations()) def testDistributeRagged(self): - service = self.create_cluster(1) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable ds = dataset_ops.Dataset.from_tensor_slices([1, 5, 3, 2, 8]) ds = ds.map(math_ops.range) ds = ds.apply(batching.dense_to_ragged_batch(2)) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_dataset(ds, dispatcher) results = [elem.to_tensor() for elem in ds] self.assertAllEqual(results[0], [[0, 0, 0, 0, 0], [0, 1, 2, 3, 4]]) self.assertAllEqual(results[1], [[0, 1, 2], [0, 1, 0]]) @@ -134,10 +204,10 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): def testDifferentShuffleOrders(self): random_seed.set_random_seed(None) num_elements = 100 - service = self.create_cluster(2) + dispatcher, workers = self.start_cluster(2) # to avoid gcing workers, pylint: disable=unused-variable ds = dataset_ops.Dataset.range(num_elements) ds = ds.shuffle(num_elements) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_dataset(ds, dispatcher) output = [elem.numpy() for elem in ds] # The output will be two sequences of range(num_elements) @@ -154,34 +224,31 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testMultipleEpochs(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 3 - service = self.create_cluster(1) - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_range_dataset(num_elements, dispatcher) for _ in range(10): self.assertEqual(list(range(num_elements)), [elem.numpy() for elem in ds]) @combinations.generate(test_base.eager_only_combinations()) def testRepeatedDataset(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 10 num_repetitions = 5 - service = self.create_cluster(1) - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_range_dataset(num_elements, dispatcher) ds = ds.repeat(num_repetitions) self.assertDatasetProduces( ds, expected_output=num_repetitions * list(range(num_elements))) @combinations.generate(test_base.eager_only_combinations()) def testConcurrentEpoch(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 10 num_datasets = 3 - service = self.create_cluster(1) iterators = [] results = [] for _ in range(num_datasets): - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_range_dataset(num_elements, dispatcher) iterators.append(iter(ds)) results.append([]) @@ -195,11 +262,10 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testSharedEpoch(self): self.skipTest("Not yet implemented") + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 10 num_iterators = 3 - service = self.create_cluster(1) - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_range_dataset(num_elements, dispatcher) result = [] iterators = [] for _ in range(num_iterators): @@ -220,10 +286,9 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testMultiWorker(self): num_workers = 3 + dispatcher, workers = self.start_cluster(num_workers) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 10 - service = self.create_cluster(num_workers) - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_range_dataset(num_elements, dispatcher) results = [elem.numpy() for elem in ds] self.assertCountEqual(num_workers * list(range(num_elements)), results) @@ -237,12 +302,10 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): except: raise self.skipTest("Flakes in portpicker library do not represent " "TensorFlow errors.") - dispatcher = server_lib.DispatchServer( - port=dispatcher_port, protocol=PROTOCOL, start=False) + dispatcher = server_lib.DispatchServer(port=dispatcher_port, start=False) worker = server_lib.WorkerServer( port=0, - dispatcher_address=dispatcher._address, - protocol=PROTOCOL, + dispatcher_address=_address_from_target(dispatcher.target), start=False) def start_servers(): @@ -254,33 +317,25 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): start_servers_thread.start() num_elements = 10 - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset( - ds, "{}://{}".format(PROTOCOL, dispatcher._address)) + ds = _make_distributed_range_dataset(num_elements, dispatcher) results = [elem.numpy() for elem in ds] self.assertEqual(list(range(num_elements)), results) start_servers_thread.join() @combinations.generate(test_base.eager_only_combinations()) def testAddWorkerMidJob(self): - self._dispatcher = server_lib.DispatchServer(port=0, protocol=PROTOCOL) - self._worker = server_lib.WorkerServer( - port=0, dispatcher_address=self._dispatcher._address, protocol=PROTOCOL) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 100 - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset( - ds, "{}://{}".format(PROTOCOL, self._dispatcher._address)) + ds = _make_distributed_range_dataset(num_elements, dispatcher) iterator = iter(ds) results = [] # Read halfway through the dataset. for _ in range(num_elements // 2): results.append(next(iterator).numpy()) - self._new_worker = server_lib.WorkerServer( - port=0, dispatcher_address=self._dispatcher._address, protocol=PROTOCOL) - + new_worker = self.start_worker_server(dispatcher) # to avoid gcing workers, pylint: disable=unused-variable # Wait for the new worker to register with the dispatcher. - while self._dispatcher._num_workers() < 2: + while dispatcher._num_workers() < 2: time.sleep(10 / 1000) # 10ms for elem in iterator: @@ -292,13 +347,9 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): combinations.times(test_base.eager_only_combinations(), combinations.combine(use_same_port=[True, False]))) def testRestartWorker(self, use_same_port): - self._dispatcher = server_lib.DispatchServer(port=0, protocol=PROTOCOL) - self._worker = server_lib.WorkerServer( - port=0, dispatcher_address=self._dispatcher._address, protocol=PROTOCOL) + dispatcher, [worker] = self.start_cluster(1) num_elements = 100 - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset( - ds, "{}://{}".format(PROTOCOL, self._dispatcher._address)) + ds = _make_distributed_range_dataset(num_elements, dispatcher) iterator = iter(ds) # Read halfway through the dataset. midpoint = num_elements // 2 @@ -308,12 +359,9 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): # Stop the original worker and start a new one. port = 0 if use_same_port: - port = int(self._worker._address.split(":")[1]) - self._worker._stop() - self._new_worker = server_lib.WorkerServer( - port=port, - dispatcher_address=self._dispatcher._address, - protocol=PROTOCOL) + port = int(worker._address.split(":")[1]) + worker._stop() + new_worker = self.start_worker_server(dispatcher, port=port) # to avoid gcing workers, pylint: disable=unused-variable # There may have been some elements prefetched from the first worker # before it was stopped. @@ -331,29 +379,23 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testMaxOutstandingRequests(self): - num_elements = 10 num_workers = 3 - service = self.create_cluster(num_workers) - ds = dataset_ops.Dataset.range(num_elements) - ds = ds.apply( - data_service_ops._distribute( - "parallel_epochs", - service, - max_outstanding_requests=1, - task_refresh_interval_hint_ms=20)) + dispatcher, workers = self.start_cluster(num_workers) # to avoid gcing workers, pylint: disable=unused-variable + num_elements = 10 + ds = _make_distributed_range_dataset( + num_elements, dispatcher, max_outstanding_requests=1) self.assertCountEqual(num_workers * list(range(num_elements)), self.getDatasetOutput(ds)) @combinations.generate(test_base.eager_only_combinations()) def testInsideFunction(self): num_workers = 3 + dispatcher, workers = self.start_cluster(num_workers) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 10 - service = self.create_cluster(num_workers) @def_function.function def f(): - ds = dataset_ops.Dataset.range(num_elements) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_range_dataset(num_elements, dispatcher) result = tensor_array_ops.TensorArray( dtypes.int64, size=num_workers * num_elements, dynamic_size=True) i = 0 @@ -367,11 +409,11 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testSharedJobName(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 100 - service = self.create_cluster(1) ds = dataset_ops.Dataset.range(num_elements) - ds1 = _make_distributed_dataset(ds, service, job_name="job_name") - ds2 = _make_distributed_dataset(ds, service, job_name="job_name") + ds1 = _make_distributed_dataset(ds, dispatcher, job_name="job_name") + ds2 = _make_distributed_dataset(ds, dispatcher, job_name="job_name") iter1 = iter(ds1) iter2 = iter(ds2) results = [] @@ -386,21 +428,21 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testDifferentJobNames(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 10 - service = self.create_cluster(1) ds = dataset_ops.Dataset.range(num_elements) - ds1 = _make_distributed_dataset(ds, service, job_name="job_name1") - ds2 = _make_distributed_dataset(ds, service, job_name="job_name2") + ds1 = _make_distributed_dataset(ds, dispatcher, job_name="job_name1") + ds2 = _make_distributed_dataset(ds, dispatcher, job_name="job_name2") self.assertDatasetProduces(ds1, list(range(num_elements))) self.assertDatasetProduces(ds2, list(range(num_elements))) @combinations.generate(test_base.eager_only_combinations()) def testSharedJobNameMultiIteration(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 10 - service = self.create_cluster(1) ds = dataset_ops.Dataset.range(num_elements) - ds1 = _make_distributed_dataset(ds, service, job_name="job_name") - ds2 = _make_distributed_dataset(ds, service, job_name="job_name") + ds1 = _make_distributed_dataset(ds, dispatcher, job_name="job_name") + ds2 = _make_distributed_dataset(ds, dispatcher, job_name="job_name") # iteration 1 self.assertDatasetProduces(ds1, list(range(num_elements))) self.assertDatasetProduces(ds2, []) @@ -410,13 +452,13 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testSharedJobNameRepeat(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable num_elements = 100 num_repetitions = 3 - service = self.create_cluster(1) ds = dataset_ops.Dataset.range(num_elements) - ds1 = _make_distributed_dataset(ds, service, job_name="job_name") + ds1 = _make_distributed_dataset(ds, dispatcher, job_name="job_name") ds1 = ds1.repeat(num_repetitions) - ds2 = _make_distributed_dataset(ds, service, job_name="job_name") + ds2 = _make_distributed_dataset(ds, dispatcher, job_name="job_name") ds2 = ds2.repeat(num_repetitions) results = [] iter1 = iter(ds1) @@ -434,7 +476,7 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testApplyDeterminismOption(self): elements = list(range(10)) - service = self.create_cluster(1) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable def dataset_fn(delay_ms): @@ -451,7 +493,7 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): opts = dataset_ops.Options() opts.experimental_deterministic = False ds = ds.with_options(opts) - ds = _make_distributed_dataset(ds, service) + ds = _make_distributed_dataset(ds, dispatcher) return ds self.checkDeterminism( @@ -468,8 +510,8 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): options.experimental_external_state_policy = external_state_policy ds = ds.with_options(options) - service = self.create_cluster(3) - ds = _make_distributed_dataset(ds, service) + dispatcher, workers = self.start_cluster(3) # to avoid gcing workers, pylint: disable=unused-variable + ds = _make_distributed_dataset(ds, dispatcher) next(iter(ds)) @combinations.generate( @@ -489,13 +531,13 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testDistributeFromInterleave(self): - service = self.create_cluster(1) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable ds = dataset_ops.Dataset.range(2) def interleave_fn(_): - ds = dataset_ops.Dataset.range(2) - _make_distributed_dataset(ds, service) - return ds + dataset = dataset_ops.Dataset.range(2) + _make_distributed_dataset(dataset, dispatcher) + return dataset with self.assertRaisesRegex( errors.InvalidArgumentError, r"The `.distribute\(...\)` dataset " @@ -530,25 +572,25 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testFromDatasetId(self): - num_elements = 10 - service = self.create_cluster(1) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + num_elements = 10 ds = dataset_ops.Dataset.range(num_elements) - dataset_id = data_service_ops.register_dataset(service, ds) + dataset_id = data_service_ops.register_dataset(dispatcher.target, ds) from_dataset_id_ds = data_service_ops.from_dataset_id( - "parallel_epochs", service, dataset_id, ds.element_spec) + "parallel_epochs", dispatcher.target, dataset_id, ds.element_spec) self.assertDatasetProduces(from_dataset_id_ds, list(range(num_elements))) @combinations.generate(test_base.eager_only_combinations()) def testFromDatasetIdMultipleComponents(self): - num_elements = 10 - service = self.create_cluster(1) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + num_elements = 10 ds = dataset_ops.Dataset.range(num_elements) ds = dataset_ops.Dataset.zip({"a": (ds, ds), "b": ds}) - dataset_id = data_service_ops.register_dataset(service, ds) + dataset_id = data_service_ops.register_dataset(dispatcher.target, ds) from_dataset_id_ds = data_service_ops.from_dataset_id( - "parallel_epochs", service, dataset_id, ds.element_spec) + "parallel_epochs", dispatcher.target, dataset_id, ds.element_spec) output = self.getDatasetOutput(from_dataset_id_ds) for i in range(num_elements): self.assertEqual(i, output[i]["a"][0]) @@ -557,26 +599,26 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): @combinations.generate(test_base.eager_only_combinations()) def testFromDatasetIdWrongElementSpec(self): - num_elements = 10 - service = self.create_cluster(1) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + num_elements = 10 ds = dataset_ops.Dataset.range(num_elements) - dataset_id = data_service_ops.register_dataset(service, ds) + dataset_id = data_service_ops.register_dataset(dispatcher.target, ds) wrong_spec = tensor_spec.TensorSpec(shape=(), dtype=dtypes.variant) from_dataset_id_ds = data_service_ops.from_dataset_id( - "parallel_epochs", service, dataset_id, wrong_spec) + "parallel_epochs", dispatcher.target, dataset_id, wrong_spec) with self.assertRaisesRegex(errors.FailedPreconditionError, "Expected a tensor of type variant"): self.evaluate(self.getNext(from_dataset_id_ds)()) @combinations.generate(test_base.eager_only_combinations()) def testFromDatasetIdNotRegistered(self): - service = self.create_cluster(1) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable dataset_id = 0 element_spec = tensor_spec.TensorSpec(shape=(), dtype=dtypes.variant) from_dataset_id_ds = data_service_ops.from_dataset_id( - "parallel_epochs", service, dataset_id, element_spec) + "parallel_epochs", dispatcher.target, dataset_id, element_spec) with self.assertRaisesRegex(errors.NotFoundError, "Dataset id"): self.evaluate(self.getNext(from_dataset_id_ds)()) @@ -585,17 +627,14 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): self.skipTest("b/162521601") sleep_microseconds = int(1e6) * 1000 - self._dispatcher = server_lib.DispatchServer(port=0, protocol=PROTOCOL) - self._worker = server_lib.WorkerServer( - port=0, dispatcher_address=self._dispatcher._address, protocol=PROTOCOL) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable # Create a dataset which produces the first element quickly, and the second # element slowly. Fetching the first element triggers prefetching of the # second element, which we should be able to cancel. slow = dataset_ops.Dataset.range(1) slow = slow.apply(testing.sleep(sleep_microseconds)) ds = dataset_ops.Dataset.range(1).concatenate(slow) - ds = _make_distributed_dataset( - ds, "{}://{}".format(PROTOCOL, self._dispatcher._address)) + ds = _make_distributed_dataset(ds, dispatcher) ds = ds.prefetch(1) get_next = self.getNext(ds, requires_initialization=True) self.assertEqual(0, self.evaluate(get_next())) @@ -606,18 +645,18 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): def testRegisterEquivalentDatasets(self): ds_1 = dataset_ops.Dataset.range(10) ds_2 = dataset_ops.Dataset.range(10) - service = self.create_cluster(1) - id_1 = data_service_ops.register_dataset(service, ds_1) - id_2 = data_service_ops.register_dataset(service, ds_2) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + id_1 = data_service_ops.register_dataset(dispatcher.target, ds_1) + id_2 = data_service_ops.register_dataset(dispatcher.target, ds_2) self.assertEqual(id_1.numpy(), id_2.numpy()) @combinations.generate(test_base.eager_only_combinations()) def testRegisterDifferentDatasets(self): ds_1 = dataset_ops.Dataset.range(10) ds_2 = dataset_ops.Dataset.range(20) - service = self.create_cluster(1) - id_1 = data_service_ops.register_dataset(service, ds_1) - id_2 = data_service_ops.register_dataset(service, ds_2) + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + id_1 = data_service_ops.register_dataset(dispatcher.target, ds_1) + id_2 = data_service_ops.register_dataset(dispatcher.target, ds_2) self.assertNotEqual(id_1.numpy(), id_2.numpy()) diff --git a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.service.-dispatch-server.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.service.-dispatch-server.pbtxt index 86efaf268e0..522cc00448a 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.service.-dispatch-server.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.data.experimental.service.-dispatch-server.pbtxt @@ -8,7 +8,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'port\', \'protocol\', \'start\'], varargs=None, keywords=None, defaults=[\'None\', \'True\'], " + argspec: "args=[\'self\', \'port\', \'protocol\', \'work_dir\', \'fault_tolerant_mode\', \'start\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], " } member_method { name: "join" From cf9518e745cf7fccc53dcdcc856970216d0468fd Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Thu, 6 Aug 2020 14:11:14 -0700 Subject: [PATCH 0583/1017] [tf.data service] Avoid holding locks while calling RPCs from worker. This CL also improves some naming and changes WorkerImpl::Start to return Status, similar to DispatcherImpl::Start. PiperOrigin-RevId: 325307263 Change-Id: I8b1fc9416908acd3c6e971e0c6ee2b3cf23cfd4f --- .../core/data/service/grpc_worker_impl.cc | 15 +- .../core/data/service/grpc_worker_impl.h | 2 +- tensorflow/core/data/service/server_lib.cc | 2 +- tensorflow/core/data/service/worker_impl.cc | 132 ++++++++++-------- tensorflow/core/data/service/worker_impl.h | 37 ++--- 5 files changed, 101 insertions(+), 87 deletions(-) diff --git a/tensorflow/core/data/service/grpc_worker_impl.cc b/tensorflow/core/data/service/grpc_worker_impl.cc index c76e1062753..5e3183d61b8 100644 --- a/tensorflow/core/data/service/grpc_worker_impl.cc +++ b/tensorflow/core/data/service/grpc_worker_impl.cc @@ -23,7 +23,6 @@ namespace data { using ::grpc::ServerBuilder; using ::grpc::ServerContext; -using ::grpc::Status; GrpcWorkerImpl::GrpcWorkerImpl(ServerBuilder* server_builder, const experimental::WorkerConfig& config) @@ -32,15 +31,15 @@ GrpcWorkerImpl::GrpcWorkerImpl(ServerBuilder* server_builder, VLOG(1) << "Registered data service worker"; } -void GrpcWorkerImpl::Start(const std::string& worker_address) { - impl_.Start(worker_address); +Status GrpcWorkerImpl::Start(const std::string& worker_address) { + return impl_.Start(worker_address); } -#define HANDLER(method) \ - Status GrpcWorkerImpl::method(ServerContext* context, \ - const method##Request* request, \ - method##Response* response) { \ - return ToGrpcStatus(impl_.method(request, response)); \ +#define HANDLER(method) \ + grpc::Status GrpcWorkerImpl::method(ServerContext* context, \ + const method##Request* request, \ + method##Response* response) { \ + return ToGrpcStatus(impl_.method(request, response)); \ } HANDLER(ProcessTask); HANDLER(GetElement); diff --git a/tensorflow/core/data/service/grpc_worker_impl.h b/tensorflow/core/data/service/grpc_worker_impl.h index b0881143a57..49caab246ac 100644 --- a/tensorflow/core/data/service/grpc_worker_impl.h +++ b/tensorflow/core/data/service/grpc_worker_impl.h @@ -39,7 +39,7 @@ class GrpcWorkerImpl : public WorkerService::Service { const experimental::WorkerConfig& config); ~GrpcWorkerImpl() override {} - void Start(const std::string& worker_address); + Status Start(const std::string& worker_address); #define HANDLER(method) \ grpc::Status method(grpc::ServerContext* context, \ diff --git a/tensorflow/core/data/service/server_lib.cc b/tensorflow/core/data/service/server_lib.cc index 7f698f8669b..98157f6b232 100644 --- a/tensorflow/core/data/service/server_lib.cc +++ b/tensorflow/core/data/service/server_lib.cc @@ -116,7 +116,7 @@ Status WorkerGrpcDataServer::StartServiceInternal() { std::string resolved_address = str_util::StringReplace( worker_address, kPortPlaceholder, absl::StrCat(bound_port()), /*replace_all=*/false); - service_->Start(resolved_address); + TF_RETURN_IF_ERROR(service_->Start(resolved_address)); return Status::OK(); } diff --git a/tensorflow/core/data/service/worker_impl.cc b/tensorflow/core/data/service/worker_impl.cc index d2c75bbc719..0e955e136d2 100644 --- a/tensorflow/core/data/service/worker_impl.cc +++ b/tensorflow/core/data/service/worker_impl.cc @@ -36,7 +36,7 @@ limitations under the License. namespace tensorflow { namespace data { -const constexpr uint64 kHeartbeatIntervalMicros = 5ull * 1000 * 1000; +const constexpr uint64 kRetryIntervalMicros = 5ull * 1000 * 1000; namespace { auto* tf_data_service_created = @@ -54,24 +54,30 @@ DataServiceWorkerImpl::DataServiceWorkerImpl( DataServiceWorkerImpl::~DataServiceWorkerImpl() { mutex_lock l(mu_); cancelled_ = true; - heartbeat_cv_.notify_one(); + background_cv_.notify_one(); } -void DataServiceWorkerImpl::Start(const std::string& worker_address) { +Status DataServiceWorkerImpl::Start(const std::string& worker_address) { VLOG(3) << "Starting tf.data service worker at address " << worker_address; - mutex_lock l(mu_); worker_address_ = worker_address; - Thread* thread = Env::Default()->StartThread( - {}, "data-service-worker-heartbeat", [this]() { HeartbeatThread(); }); - heartbeat_thread_.reset(thread); - Status s = Register(); + std::unique_ptr dispatcher; + TF_RETURN_IF_ERROR(MakeDispatcherStub(&dispatcher)); + + Status s = Register(dispatcher.get()); while (!s.ok()) { LOG(WARNING) << "Failed to register with dispatcher at " << config_.dispatcher_address() << ": " << s; - Env::Default()->SleepForMicroseconds(kHeartbeatIntervalMicros); - s = Register(); + Env::Default()->SleepForMicroseconds(kRetryIntervalMicros); + s = Register(dispatcher.get()); } + Thread* thread = + Env::Default()->StartThread({}, "data-service-worker-background", + [this, dispatcher = dispatcher.release()]() { + BackgroundThread(dispatcher); + }); + background_thread_.reset(thread); + return Status::OK(); } Status DataServiceWorkerImpl::ProcessTask(const ProcessTaskRequest* request, @@ -98,7 +104,7 @@ Status DataServiceWorkerImpl::ProcessTaskInternal(const TaskDef& task_def) " already exists."); } Task& task = tasks_[task_def.task_id()]; - task.id = task_def.task_id(); + task.task_id = task_def.task_id(); task.dataset = std::move(dataset); task.iterator = std::move(iterator); VLOG(3) << "Began processing for task " << task_def.task_id(); @@ -128,8 +134,8 @@ Status DataServiceWorkerImpl::GetElement(const GetElementRequest* request, VLOG(3) << "Reached end_of_sequence for task " << request->task_id(); // Release iterator memory and leave a null entry as a tombstone. iter.reset(); - pending_completed_tasks_.push_back(request->task_id()); - heartbeat_cv_.notify_one(); + pending_completed_tasks_.insert(request->task_id()); + background_cv_.notify_one(); } } @@ -168,80 +174,88 @@ Status DataServiceWorkerImpl::GetElement(const GetElementRequest* request, return Status::OK(); } -Status DataServiceWorkerImpl::EnsureDispatcherStubInitialized() - EXCLUSIVE_LOCKS_REQUIRED(mu_) { - if (!dispatcher_stub_) { - ::grpc::ChannelArguments args; - std::shared_ptr<::grpc::ChannelCredentials> credentials; - TF_RETURN_IF_ERROR(CredentialsFactory::CreateClientCredentials( - config_.protocol(), &credentials)); - auto channel = ::grpc::CreateCustomChannel(config_.dispatcher_address(), - credentials, args); - dispatcher_stub_ = DispatcherService::NewStub(channel); - } +Status DataServiceWorkerImpl::MakeDispatcherStub( + std::unique_ptr* stub) { + ::grpc::ChannelArguments args; + std::shared_ptr<::grpc::ChannelCredentials> credentials; + TF_RETURN_IF_ERROR(CredentialsFactory::CreateClientCredentials( + config_.protocol(), &credentials)); + auto channel = ::grpc::CreateCustomChannel(config_.dispatcher_address(), + credentials, args); + *stub = DispatcherService::NewStub(channel); return Status::OK(); } -Status DataServiceWorkerImpl::Register() EXCLUSIVE_LOCKS_REQUIRED(mu_) { +Status DataServiceWorkerImpl::Register(DispatcherService::Stub* dispatcher_stub) + LOCKS_EXCLUDED(mu_) { VLOG(3) << "Registering with dispatcher at " << config_.dispatcher_address(); - TF_RETURN_IF_ERROR(EnsureDispatcherStubInitialized()); RegisterWorkerRequest req; req.set_worker_address(worker_address_); RegisterWorkerResponse resp; - grpc::ClientContext ctx; - grpc::Status s = dispatcher_stub_->RegisterWorker(&ctx, req, &resp); + grpc::Status s = dispatcher_stub->RegisterWorker(&ctx, req, &resp); if (!s.ok()) { return grpc_util::WrapError("Failed to register worker", s); } for (const TaskDef& task : resp.tasks()) { + mutex_lock l(mu_); TF_RETURN_IF_ERROR(ProcessTaskInternal(task)); } + VLOG(3) << "Registered worker with address " << worker_address_; return Status::OK(); } -Status DataServiceWorkerImpl::SendTaskUpdate() EXCLUSIVE_LOCKS_REQUIRED(mu_) { - VLOG(3) << "Sending " << pending_completed_tasks_.size() - << " task updates to dispatcher"; - TF_RETURN_IF_ERROR(EnsureDispatcherStubInitialized()); +void DataServiceWorkerImpl::BackgroundThread( + DispatcherService::Stub* dispatcher_ptr) LOCKS_EXCLUDED(mu_) { + std::unique_ptr dispatcher = + absl::WrapUnique(dispatcher_ptr); + while (true) { + { + mutex_lock l(mu_); + while (!cancelled_ && pending_completed_tasks_.empty()) { + background_cv_.wait(l); + } + if (cancelled_) { + VLOG(3) << "Background thread shutting down"; + return; + } + } + Status s = SendTaskUpdates(dispatcher.get()); + if (!s.ok()) { + LOG(WARNING) << "Failed to send task updates to dispatcher: " << s; + Env::Default()->SleepForMicroseconds(kRetryIntervalMicros); + } + } +} + +Status DataServiceWorkerImpl::SendTaskUpdates( + DispatcherService::Stub* dispatcher) LOCKS_EXCLUDED(mu_) { WorkerUpdateRequest req; - for (int task_id : pending_completed_tasks_) { - TaskProgress* update = req.add_updates(); - update->set_task_id(task_id); - update->set_completed(true); + { + mutex_lock l(mu_); + VLOG(3) << "Sending " << pending_completed_tasks_.size() + << " task updates to dispatcher"; + req.set_worker_address(worker_address_); + for (int task_id : pending_completed_tasks_) { + TaskProgress* update = req.add_updates(); + update->set_task_id(task_id); + update->set_completed(true); + } } WorkerUpdateResponse resp; grpc::ClientContext ctx; - grpc::Status s = dispatcher_stub_->WorkerUpdate(&ctx, req, &resp); + grpc::Status s = dispatcher->WorkerUpdate(&ctx, req, &resp); if (!s.ok()) { return grpc_util::WrapError("Failed to send task updates", s); } - pending_completed_tasks_.clear(); + mutex_lock l(mu_); + for (const auto& update : req.updates()) { + pending_completed_tasks_.erase(update.task_id()); + } VLOG(3) << "Sent " << req.updates().size() << " task updates "; return Status::OK(); } -void DataServiceWorkerImpl::HeartbeatThread() { - while (true) { - Status s; - { - mutex_lock l(mu_); - while (!cancelled_ && pending_completed_tasks_.empty()) { - heartbeat_cv_.wait(l); - } - if (cancelled_) { - VLOG(3) << "Heartbeat thread shutting down"; - return; - } - s = SendTaskUpdate(); - } - if (!s.ok()) { - LOG(WARNING) << "Failed to send task updates to dispatcher: " << s; - Env::Default()->SleepForMicroseconds(kHeartbeatIntervalMicros); - } - } -} - } // namespace data } // namespace tensorflow diff --git a/tensorflow/core/data/service/worker_impl.h b/tensorflow/core/data/service/worker_impl.h index 6961312ee34..8353d11efdc 100644 --- a/tensorflow/core/data/service/worker_impl.h +++ b/tensorflow/core/data/service/worker_impl.h @@ -38,7 +38,7 @@ class DataServiceWorkerImpl { // constructor because the worker may be binding to port `0`, in which case // the address isn't known until the worker has started and decided which port // to bind to. - void Start(const std::string& worker_address); + Status Start(const std::string& worker_address); // See worker.proto for API documentation. @@ -51,19 +51,23 @@ class DataServiceWorkerImpl { GetElementResponse* response); private: - // Sets dispatcher_stub_ if it isn't already set. - Status EnsureDispatcherStubInitialized(); + Status MakeDispatcherStub(std::unique_ptr* stub); // Registers the worker with the dispatcher. - Status Register(); - // Sends task status to the dispatcher. - Status SendTaskUpdate(); + Status Register(DispatcherService::Stub* dispatcher) LOCKS_EXCLUDED(mu_); + // Sends task status to the dispatcher and checks for dispatcher commands. + Status SendTaskUpdates(DispatcherService::Stub* dispatcher) + LOCKS_EXCLUDED(mu_); // Creates an iterator to process a task. - Status ProcessTaskInternal(const TaskDef& task); - // A thread for updating the dispatcher with worker status. - void HeartbeatThread(); + Status ProcessTaskInternal(const TaskDef& task) EXCLUSIVE_LOCKS_REQUIRED(mu_); + // A thread for doing async background processing not associated with a + // specific RPC, such as reporting finished tasks. The thread takes + // ownership of the passed dispatcher_ptr. We use a raw pointer instead of + // unique_ptr since unique_ptr cannot be passed to std::function. + void BackgroundThread(DispatcherService::Stub* dispatcher_ptr) + LOCKS_EXCLUDED(mu_); typedef struct Task { - int64 id; + int64 task_id; // TODO(aaudibert): Have standalone::Iterator own a reference to // standalone::Dataset so that we don't need to store the dataset here. std::unique_ptr dataset; @@ -75,17 +79,14 @@ class DataServiceWorkerImpl { std::string worker_address_; mutex mu_; - int64 worker_id_ TF_GUARDED_BY(mu_); - std::unique_ptr dispatcher_stub_ TF_GUARDED_BY(mu_); // Information about tasks, keyed by task ids. absl::flat_hash_map tasks_ TF_GUARDED_BY(mu_); - // List of completed tasks which haven't yet been communicated to the - // dispatcher. - std::vector pending_completed_tasks_ TF_GUARDED_BY(mu_); + // Completed tasks which haven't yet been communicated to the dispatcher. + absl::flat_hash_set pending_completed_tasks_ TF_GUARDED_BY(mu_); bool cancelled_ TF_GUARDED_BY(mu_) = false; - // Condition variable for notifying the heartbeat thread. - condition_variable heartbeat_cv_ TF_GUARDED_BY(mu_); - std::unique_ptr heartbeat_thread_; + // Condition variable for notifying the background thread. + condition_variable background_cv_ TF_GUARDED_BY(mu_); + std::unique_ptr background_thread_; TF_DISALLOW_COPY_AND_ASSIGN(DataServiceWorkerImpl); }; From fffd56ade18e5546599ec98f0156081db799fce4 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Thu, 6 Aug 2020 14:27:51 -0700 Subject: [PATCH 0584/1017] Add support for `mhlo.if` in LegalizeTFCommunication pass. TF/XLA communication ops in `mhlo.if` are now legalized, and `mhlo.if` parents/ancestors are rewritten to receive and emit `mhlo.token` generated from these communication op legalizations. PiperOrigin-RevId: 325310956 Change-Id: Id4c9a426b01d44b9191dd364458fc2eb8227383a --- .../xla/tests/legalize-tf-communication.mlir | 496 ++++++++++++++++ .../transforms/legalize_tf_communication.cc | 538 +++++++++++++++--- 2 files changed, 963 insertions(+), 71 deletions(-) diff --git a/tensorflow/compiler/mlir/xla/tests/legalize-tf-communication.mlir b/tensorflow/compiler/mlir/xla/tests/legalize-tf-communication.mlir index f84a2f28a23..d01ab38bd6b 100644 --- a/tensorflow/compiler/mlir/xla/tests/legalize-tf-communication.mlir +++ b/tensorflow/compiler/mlir/xla/tests/legalize-tf-communication.mlir @@ -407,6 +407,502 @@ func @callee2() attributes {sym_visibility = "private"} { // ----- +// Test cloned function rewrite also checks transitive function calls to +// TF/XLA communication ops. + +// CHECK: func @callee3() +func @callee3() { + // CHECK: [[CALLEE3_INIT_TOKEN:%.*]] = "mhlo.create_token" + + // CHECK: call @callee4{{.+}}([[CALLEE3_INIT_TOKEN]]) + call @callee4() : () -> () + return +} + +// CHECK: func @callee4() +func @callee4() { + // CHECK: [[CALLEE4_INIT_TOKEN:%.*]] = "mhlo.create_token" + + // CHECK: [[CALL_5:%.*]] = call @callee5([[CALLEE4_INIT_TOKEN]]) + call @callee5() : () -> () + + // CHECK: return + return +} + +// CHECK: func @callee5([[CALLEE5_ARG0:%.*]]: !mhlo.token) -> !mhlo.token +func @callee5() attributes {sym_visibility = "private"} { + // CHECK-NOT: "mhlo.create_token" + + // CHECK: [[RECV_TUPLE:%.*]] = "mhlo.recv"([[CALLEE5_ARG0]]) + // CHECK: [[RECV_VAL:%.*]] = "mhlo.get_tuple_element"([[RECV_TUPLE]]) + // CHECK-SAME: index = 0 + // CHECK: [[RECV_TOKEN:%.*]] = "mhlo.get_tuple_element"([[RECV_TUPLE]]) + // CHECK-SAME: index = 1 + %0 = "tf.XlaRecvFromHost"() {key = "recv_key", shape = #tf.shape<>} : () -> tensor + + // CHECK: return [[RECV_TOKEN]] + return +} + +// CHECK: func @callee4{{.+}}([[CALLEE4_ARG0:%.*]]: !mhlo.token) -> !mhlo.token attributes {sym_visibility = "private"} +// CHECK-NOT: "mhlo.create_token" +// CHECK: [[CALL_5:%.*]] = call @callee5([[CALLEE4_ARG0]]) +// CHECK: return [[CALL_5]] + +// ----- + +// Tests `mhlo.if` with branches populated with TF/XLA communication ops. + +// CHECK-LABEL: func @if_both_branches +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor, [[ARG2:%.*]]: tensor) +func @if_both_branches(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = "mhlo.create_token" + // CHECK: [[TRUE_TUPLE:%.*]] = "mhlo.tuple"([[ARG1]], [[INIT_TOKEN]]) + // CHECK: [[FALSE_TUPLE:%.*]] = "mhlo.tuple"([[ARG2]], [[INIT_TOKEN]]) + + // CHECK: [[IF_TUPLE:%.*]] = "mhlo.if"([[ARG0]], [[TRUE_TUPLE]], [[FALSE_TUPLE]]) + %0 = "mhlo.if"(%arg0, %arg1, %arg2) ( { + // CHECK: ^bb0([[TRUE_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg3: tensor): + // CHECK-DAG: [[TRUE_REGION_ARG_VALUE:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[TRUE_REGION_ARG_TOKEN:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 1 + + // CHECK: [[TRUE_SEND_TOKEN:%.*]] = "mhlo.send"([[TRUE_REGION_ARG_VALUE]], [[TRUE_REGION_ARG_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 1 : i64, type = 2 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "send_if_true_dtoh_0"} + + // CHECK: [[TRUE_RECV_TUPLE:%.*]] = "mhlo.recv"([[TRUE_SEND_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 2 : i64, type = 3 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "recv_if_true_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg3) {recv_key = "recv_if_true", send_key = "send_if_true", tpu_core = 0 : i64} : (tensor) -> tensor + + // CHECK-DAG: [[TRUE_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[TRUE_RECV_TUPLE]]) {index = 0 + // CHECK-DAG: [[TRUE_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[TRUE_RECV_TUPLE]]) {index = 1 + // CHECK: [[TRUE_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[TRUE_GET_TUPLE_ELEMENT0]], [[TRUE_GET_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[TRUE_RETURN_TUPLE]]) + "mhlo.return"(%1) : (tensor) -> () + }, { + // CHECK: ^bb0([[FALSE_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg3: tensor): + // CHECK-DAG: [[FALSE_REGION_ARG_VALUE:%.*]] = "mhlo.get_tuple_element"([[FALSE_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[FALSE_REGION_ARG_TOKEN:%.*]] = "mhlo.get_tuple_element"([[FALSE_REGION_ARG]]) {index = 1 + + // CHECK: [[FALSE_SEND_TOKEN:%.*]] = "mhlo.send"([[FALSE_REGION_ARG_VALUE]], [[FALSE_REGION_ARG_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 3 : i64, type = 2 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "send_if_false_dtoh_0"} + + // CHECK: [[FALSE_RECV_TUPLE:%.*]] = "mhlo.recv"([[FALSE_SEND_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 4 : i64, type = 3 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "recv_if_false_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg3) {recv_key = "recv_if_false", send_key = "send_if_false", tpu_core = 0 : i64} : (tensor) -> tensor + + // CHECK-DAG: [[FALSE_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[FALSE_RECV_TUPLE]]) {index = 0 + // CHECK-DAG: [[FALSE_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[FALSE_RECV_TUPLE]]) {index = 1 + // CHECK: [[FALSE_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[FALSE_GET_TUPLE_ELEMENT0]], [[FALSE_GET_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[FALSE_RETURN_TUPLE]]) + "mhlo.return"(%1) : (tensor) -> () + + // CHECK: (tensor, tuple, !mhlo.token>, tuple, !mhlo.token>) -> tuple, !mhlo.token> + }) : (tensor, tensor, tensor) -> tensor + + // CHECK: [[IF_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[IF_TUPLE]]) + // CHECK-SAME: index = 0 + // CHECK: return [[IF_TUPLE_ELEMENT0]] + return %0 : tensor +} + +// ----- + +// Tests `mhlo.if` with only the `true` branch populated with TF/XLA +// communication ops. + +// CHECK-LABEL: func @if_true_branch +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor, [[ARG2:%.*]]: tensor) +func @if_true_branch(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = "mhlo.create_token" + // CHECK: [[TRUE_TUPLE:%.*]] = "mhlo.tuple"([[ARG1]], [[INIT_TOKEN]]) + // CHECK: [[FALSE_TUPLE:%.*]] = "mhlo.tuple"([[ARG2]], [[INIT_TOKEN]]) + + // CHECK: [[IF_TUPLE:%.*]] = "mhlo.if"([[ARG0]], [[TRUE_TUPLE]], [[FALSE_TUPLE]]) + %0 = "mhlo.if"(%arg0, %arg1, %arg2) ( { + // CHECK: ^bb0([[TRUE_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg3: tensor): + // CHECK-DAG: [[TRUE_REGION_ARG_VALUE:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[TRUE_REGION_ARG_TOKEN:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 1 + + // CHECK: [[TRUE_SEND_TOKEN:%.*]] = "mhlo.send"([[TRUE_REGION_ARG_VALUE]], [[TRUE_REGION_ARG_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 1 : i64, type = 2 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "send_if_true_dtoh_0"} + + // CHECK: [[TRUE_RECV_TUPLE:%.*]] = "mhlo.recv"([[TRUE_SEND_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 2 : i64, type = 3 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "recv_if_true_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg3) {recv_key = "recv_if_true", send_key = "send_if_true", tpu_core = 0 : i64} : (tensor) -> tensor + + // CHECK-DAG: [[TRUE_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[TRUE_RECV_TUPLE]]) {index = 0 + // CHECK-DAG: [[TRUE_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[TRUE_RECV_TUPLE]]) {index = 1 + // CHECK: [[TRUE_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[TRUE_GET_TUPLE_ELEMENT0]], [[TRUE_GET_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[TRUE_RETURN_TUPLE]]) + "mhlo.return"(%1) : (tensor) -> () + }, { + // CHECK: ^bb0([[FALSE_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg3: tensor): + // CHECK-DAG: [[FALSE_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[FALSE_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[FALSE_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[FALSE_REGION_ARG]]) {index = 1 + // CHECK: [[FALSE_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[FALSE_GET_TUPLE_ELEMENT0]], [[FALSE_GET_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[FALSE_RETURN_TUPLE]]) + "mhlo.return"(%arg3) : (tensor) -> () + + // CHECK: (tensor, tuple, !mhlo.token>, tuple, !mhlo.token>) -> tuple, !mhlo.token> + }) : (tensor, tensor, tensor) -> tensor + + // CHECK: [[IF_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[IF_TUPLE]]) + // CHECK-SAME: index = 0 + // CHECK: return [[IF_TUPLE_ELEMENT0]] + return %0 : tensor +} + +// ----- + +// Tests `mhlo.if` with only the `false` branch populated with TF/XLA +// communication ops. + +// CHECK-LABEL: func @if_false_branch +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor, [[ARG2:%.*]]: tensor) +func @if_false_branch(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = "mhlo.create_token" + // CHECK: [[TRUE_TUPLE:%.*]] = "mhlo.tuple"([[ARG1]], [[INIT_TOKEN]]) + // CHECK: [[FALSE_TUPLE:%.*]] = "mhlo.tuple"([[ARG2]], [[INIT_TOKEN]]) + + // CHECK: [[IF_TUPLE:%.*]] = "mhlo.if"([[ARG0]], [[TRUE_TUPLE]], [[FALSE_TUPLE]]) + %0 = "mhlo.if"(%arg0, %arg1, %arg2) ( { + // CHECK: ^bb0([[TRUE_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg3: tensor): + // CHECK-DAG: [[TRUE_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[TRUE_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 1 + // CHECK: [[TRUE_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[TRUE_GET_TUPLE_ELEMENT0]], [[TRUE_GET_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[TRUE_RETURN_TUPLE]]) + "mhlo.return"(%arg3) : (tensor) -> () + }, { + // CHECK: ^bb0([[FALSE_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg3: tensor): + // CHECK-DAG: [[FALSE_REGION_ARG_VALUE:%.*]] = "mhlo.get_tuple_element"([[FALSE_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[FALSE_REGION_ARG_TOKEN:%.*]] = "mhlo.get_tuple_element"([[FALSE_REGION_ARG]]) {index = 1 + + // CHECK: [[FALSE_SEND_TOKEN:%.*]] = "mhlo.send"([[FALSE_REGION_ARG_VALUE]], [[FALSE_REGION_ARG_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 1 : i64, type = 2 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "send_if_false_dtoh_0"} + + // CHECK: [[FALSE_RECV_TUPLE:%.*]] = "mhlo.recv"([[FALSE_SEND_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 2 : i64, type = 3 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "recv_if_false_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg3) {recv_key = "recv_if_false", send_key = "send_if_false", tpu_core = 0 : i64} : (tensor) -> tensor + + // CHECK-DAG: [[FALSE_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[FALSE_RECV_TUPLE]]) {index = 0 + // CHECK-DAG: [[FALSE_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[FALSE_RECV_TUPLE]]) {index = 1 + // CHECK: [[FALSE_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[FALSE_GET_TUPLE_ELEMENT0]], [[FALSE_GET_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[FALSE_RETURN_TUPLE]]) + "mhlo.return"(%1) : (tensor) -> () + + // CHECK: (tensor, tuple, !mhlo.token>, tuple, !mhlo.token>) -> tuple, !mhlo.token> + }) : (tensor, tensor, tensor) -> tensor + + // CHECK: [[IF_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[IF_TUPLE]]) + // CHECK-SAME: index = 0 + // CHECK: return [[IF_TUPLE_ELEMENT0]] + return %0 : tensor +} + +// ----- + +// Tests `mhlo.if` with tuple arg from a `mhlo.tuple` only used by `mhlo.if` is +// replaced. + +// CHECK-LABEL: func @if_replace_tuple_arg +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor, [[ARG2:%.*]]: tensor) +func @if_replace_tuple_arg(%arg0: tensor, %arg1: tensor, %arg2: tensor) -> tensor { + // CHECK-NOT: "mhlo.tuple"([[ARG1]], [[ARG2]]) + // CHECK: [[INIT_TOKEN:%.*]] = "mhlo.create_token" + // CHECK: [[IF_ARG_TUPLE:%.*]] = "mhlo.tuple"([[ARG1]], [[ARG2]], [[INIT_TOKEN]]) + %0 = "mhlo.tuple"(%arg1, %arg2) : (tensor, tensor) -> tuple, tensor> + + // CHECK: [[IF_TUPLE:%.*]] = "mhlo.if"([[ARG0]], [[IF_ARG_TUPLE]], [[IF_ARG_TUPLE]]) + %1 = "mhlo.if"(%arg0, %0, %0) ( { + ^bb0(%arg3: tuple, tensor>): + %2 = "mhlo.get_tuple_element"(%arg3) {index = 0 : i32} : (tuple, tensor>) -> tensor + "tf.XlaSendToHost"(%2) {key = "send_key"} : (tensor) -> () + "mhlo.return"(%2) : (tensor) -> () + }, { + ^bb0(%arg3: tuple, tensor>): + %2 = "mhlo.get_tuple_element"(%arg3) {index = 0 : i32} : (tuple, tensor>) -> tensor + "mhlo.return"(%2) : (tensor) -> () + }) : (tensor, tuple, tensor>, tuple, tensor>) -> tensor + return %1 : tensor +} + +// ----- + +// Tests `mhlo.if` with tuple arg not from a `mhlo.tuple` is unpacked. + +// CHECK-LABEL: func @if_unpack_tuple_arg +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tuple, tensor>) +func @if_unpack_tuple_arg(%arg0: tensor, %arg1: tuple, tensor>) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = "mhlo.create_token" + // CHECK-DAG: [[IF_ARG_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[ARG1]]) {index = 0 + // CHECK-DAG: [[IF_ARG_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[ARG1]]) {index = 1 + // CHECK: [[IF_ARG_TUPLE:%.*]] = "mhlo.tuple"([[IF_ARG_ELEMENT0]], [[IF_ARG_ELEMENT1]], [[INIT_TOKEN]]) + + // CHECK: [[IF_TUPLE:%.*]] = "mhlo.if"([[ARG0]], [[IF_ARG_TUPLE]], [[IF_ARG_TUPLE]]) + %0 = "mhlo.if"(%arg0, %arg1, %arg1) ( { + ^bb0(%arg2: tuple, tensor>): + %1 = "mhlo.get_tuple_element"(%arg2) {index = 0 : i32} : (tuple, tensor>) -> tensor + "tf.XlaSendToHost"(%1) {key = "send_key"} : (tensor) -> () + "mhlo.return"(%1) : (tensor) -> () + }, { + ^bb0(%arg2: tuple, tensor>): + %1 = "mhlo.get_tuple_element"(%arg2) {index = 0 : i32} : (tuple, tensor>) -> tensor + "mhlo.return"(%1) : (tensor) -> () + }) : (tensor, tuple, tensor>, tuple, tensor>) -> tensor + return %0 : tensor +} + +// ----- + +// Tests `mhlo.if` tuple result is extended with a `mhlo.token`. + +// CHECK-LABEL: func @if_extend_tuple_result +func @if_extend_tuple_result(%arg0: tensor, %arg1: tuple, tensor>) -> tuple, tensor> { + // CHECK: [[IF_TUPLE:%.*]] = "mhlo.if" + %0 = "mhlo.if"(%arg0, %arg1, %arg1) ( { + ^bb0(%arg2: tuple, tensor>): + %1 = "mhlo.get_tuple_element"(%arg2) {index = 0 : i32} : (tuple, tensor>) -> tensor + "tf.XlaSendToHost"(%1) {key = "send_key"} : (tensor) -> () + "mhlo.return"(%arg2) : (tuple, tensor>) -> () + }, { + ^bb0(%arg2: tuple, tensor>): + "mhlo.return"(%arg2) : (tuple, tensor>) -> () + // CHECK: (tensor, tuple, tensor, !mhlo.token>, tuple, tensor, !mhlo.token>) -> tuple, tensor, !mhlo.token> + }) : (tensor, tuple, tensor>, tuple, tensor>) -> tuple, tensor> + + // CHECK-DAG: [[IF_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[IF_TUPLE]]) {index = 0 + // CHECK-DAG: [[IF_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[IF_TUPLE]]) {index = 1 + // CHECK: [[IF_SUBTUPLE_RESULT:%.*]] = "mhlo.tuple"([[IF_TUPLE_ELEMENT0]], [[IF_TUPLE_ELEMENT1]]) + // CHECK: return [[IF_SUBTUPLE_RESULT]] + return %0 : tuple, tensor> +} + +// ----- + +// Tests nested `mhlo.if` containing TF/XLA communication ops. + +// CHECK-LABEL: func @if_nested +// CHECK-SAME: ([[ARG0:%.*]]: tensor, [[ARG1:%.*]]: tensor) +func @if_nested(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = "mhlo.create_token" + // CHECK: [[OUTER_IF_ARG_TUPLE:%.*]] = "mhlo.tuple"([[ARG1]], [[INIT_TOKEN]]) + + // CHECK: "mhlo.if"([[ARG0]], [[OUTER_IF_ARG_TUPLE]], [[OUTER_IF_ARG_TUPLE]]) + %0 = "mhlo.if"(%arg0, %arg1, %arg1) ( { + // CHECK-NEXT: ^bb0([[OUTER_IF_TRUE_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg2: tensor): + // CHECK-DAG: [[OUTER_IF_TRUE_ARG_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[OUTER_IF_TRUE_ARG]]) {index = 0 + // CHECK-DAG: [[OUTER_IF_TRUE_ARG_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[OUTER_IF_TRUE_ARG]]) {index = 1 + // CHECK: [[INNER_IF_ARG_TUPLE:%.*]] = "mhlo.tuple"([[OUTER_IF_TRUE_ARG_ELEMENT0]], [[OUTER_IF_TRUE_ARG_ELEMENT1]]) + + %1 = mhlo.constant dense : tensor + + // CHECK: [[INNER_IF_TUPLE:%.*]] = "mhlo.if"({{%.*}}, [[INNER_IF_ARG_TUPLE]], [[INNER_IF_ARG_TUPLE]]) + %2 = "mhlo.if"(%1, %arg2, %arg2) ( { + // CHECK-NEXT: ^bb0([[INNER_IF_TRUE_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg3: tensor): + // CHECK-DAG: [[INNER_IF_TRUE_ARG_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[INNER_IF_TRUE_ARG]]) {index = 0 + // CHECK-DAG: [[INNER_IF_TRUE_ARG_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[INNER_IF_TRUE_ARG]]) {index = 1 + + // CHECK: [[SEND_TOKEN:%.*]] = "mhlo.send"([[INNER_IF_TRUE_ARG_ELEMENT0]], [[INNER_IF_TRUE_ARG_ELEMENT1]]) + "tf.XlaSendToHost"(%arg3) {key = "send_key"} : (tensor) -> () + + // CHECK: [[INNER_IF_TRUE_RESULT:%.*]] = "mhlo.tuple"([[INNER_IF_TRUE_ARG_ELEMENT0]], [[SEND_TOKEN]]) + // CHECK: "mhlo.return"([[INNER_IF_TRUE_RESULT]]) + "mhlo.return"(%arg3) : (tensor) -> () + + // CHECK-NEXT: }, { + }, { + + // CHECK-NEXT: ^bb0([[INNER_IF_FALSE_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg3: tensor): + // CHECK-DAG: [[INNER_IF_FALSE_ARG_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[INNER_IF_FALSE_ARG]]) {index = 0 + // CHECK-DAG: [[INNER_IF_FALSE_ARG_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[INNER_IF_FALSE_ARG]]) {index = 1 + // CHECK: [[INNER_IF_FALSE_RESULT:%.*]] = "mhlo.tuple"([[INNER_IF_FALSE_ARG_ELEMENT0]], [[INNER_IF_FALSE_ARG_ELEMENT1]]) + // CHECK: "mhlo.return"([[INNER_IF_FALSE_RESULT]]) + "mhlo.return"(%arg3) : (tensor) -> () + // CHECK-NEXT: (tensor, tuple, !mhlo.token>, tuple, !mhlo.token>) -> tuple, !mhlo.token> + }) : (tensor, tensor, tensor) -> tensor + + // CHECK-DAG: [[INNER_IF_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[INNER_IF_TUPLE]]) {index = 1 + // CHECK: [[OUTER_IF_TRUE_RESULT:%.*]] = "mhlo.tuple"([[OUTER_IF_TRUE_ARG_ELEMENT0]], [[INNER_IF_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[OUTER_IF_TRUE_RESULT]]) + "mhlo.return"(%arg2) : (tensor) -> () + + // CHECK-NEXT: }, { + }, { + + // CHECK-NEXT: ^bb0([[OUTER_IF_FALSE_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg2: tensor): + // CHECK-DAG: [[OUTER_IF_FALSE_ARG_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[OUTER_IF_FALSE_ARG]]) {index = 0 + // CHECK-DAG: [[OUTER_IF_FALSE_ARG_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[OUTER_IF_FALSE_ARG]]) {index = 1 + // CHECK: [[OUTER_IF_FALSE_RESULT:%.*]] = "mhlo.tuple"([[OUTER_IF_FALSE_ARG_ELEMENT0]], [[OUTER_IF_FALSE_ARG_ELEMENT1]]) + // CHECK: "mhlo.return"([[OUTER_IF_FALSE_RESULT]]) + "mhlo.return"(%arg2) : (tensor) -> () + // CHECK-NEXT: (tensor, tuple, !mhlo.token>, tuple, !mhlo.token>) -> tuple, !mhlo.token> + }) : (tensor, tensor, tensor) -> tensor + return %0 : tensor +} + +// ----- + +// Tests `mhlo.if` containing a function call to TF/XLA communication ops. + +// CHECK-LABEL: func @if_function_call +func @if_function_call(%arg0: tensor, %arg1: tensor) -> tensor { + // CHECK: "mhlo.if" + %0 = "mhlo.if"(%arg0, %arg1, %arg1) ( { + // CHECK: ^bb0([[TRUE_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg2: tensor): + // CHECK-DAG: [[TRUE_REGION_ARG_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[TRUE_REGION_ARG_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 1 + // CHECK: [[CALL_TOKEN:%.*]] = call @callee([[TRUE_REGION_ARG_ELEMENT0]], [[TRUE_REGION_ARG_ELEMENT1]]) + call @callee(%arg2) : (tensor) -> () + + // CHECK: [[TRUE_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[TRUE_REGION_ARG_ELEMENT0]], [[CALL_TOKEN]]) + // CHECK: "mhlo.return"([[TRUE_RETURN_TUPLE]]) + "mhlo.return"(%arg2) : (tensor) -> () + }, { + ^bb0(%arg2: tensor): + "mhlo.return"(%arg2) : (tensor) -> () + }) : (tensor, tensor, tensor) -> tensor + return %0 : tensor +} + +// CHECK-LABEL: func @callee +// CHECK-SAME: ([[CALLEE_ARG0:%.*]]: tensor, [[CALLEE_ARG1:%.*]]: !mhlo.token) -> !mhlo.token +func @callee(%arg0: tensor) attributes {sym_visibility = "private"} { + // CHECK: [[SEND_TOKEN:%.*]] = "mhlo.send" + "tf.XlaSendToHost"(%arg0) {key = "send_key"} : (tensor) -> () + + // CHECK: return [[SEND_TOKEN]] + return +} + +// ----- + +// Tests `mhlo.if` containing multiple TF/XLA communication ops. + +// CHECK-LABEL: func @if_region_multiple_ops +func @if_region_multiple_ops(%arg0: tensor, %arg1: tensor) { + // CHECK: "mhlo.if" + %0 = "mhlo.if"(%arg0, %arg1, %arg1) ( { + // CHECK: ^bb0([[TRUE_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg2: tensor): + // CHECK: [[TRUE_REGION_ARG_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 0 + // CHECK: [[TRUE_REGION_ARG_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[TRUE_REGION_ARG]]) {index = 1 + + // CHECK: [[SEND0_TOKEN:%.*]] = "mhlo.send"([[TRUE_REGION_ARG_ELEMENT0]], [[TRUE_REGION_ARG_ELEMENT1]]) + "tf.XlaSendToHost"(%arg2) {key = "send_key0"} : (tensor) -> () + + // CHECK: [[SEND1_TOKEN:%.*]] = "mhlo.send"([[TRUE_REGION_ARG_ELEMENT0]], [[SEND0_TOKEN]]) + "tf.XlaSendToHost"(%arg2) {key = "send_key1"} : (tensor) -> () + + // CHECK: [[TRUE_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[TRUE_REGION_ARG_ELEMENT0]], [[SEND1_TOKEN]]) + // CHECK: "mhlo.return"([[TRUE_RETURN_TUPLE]]) + "mhlo.return"(%arg2) : (tensor) -> () + }, { + ^bb0(%arg2: tensor): + "mhlo.return"(%arg2) : (tensor) -> () + }) : (tensor, tensor, tensor) -> tensor + return +} + +// ----- + +// Tests `mhlo.if` containing TF/XLA communication ops followed by other TF/XLA +// communication ops. + +func @if_followed_by_communication_op(%arg0: tensor, %arg1: tensor) { + // CHECK: [[IF_TUPLE:%.*]] = "mhlo.if" + %0 = "mhlo.if"(%arg0, %arg1, %arg1) ( { + ^bb0(%arg2: tensor): + "tf.XlaSendToHost"(%arg2) {key = "send_key0"} : (tensor) -> () + "mhlo.return"(%arg2) : (tensor) -> () + }, { + ^bb0(%arg2: tensor): + "mhlo.return"(%arg2) : (tensor) -> () + }) : (tensor, tensor, tensor) -> tensor + + // CHECK: [[IF_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[IF_TUPLE]]) {index = 1 + + // CHECK: "mhlo.send"({{.*}}, [[IF_TUPLE_ELEMENT1]]) + "tf.XlaSendToHost"(%arg1) {key = "send_key1"} : (tensor) -> () + return +} + +// ----- + +// Tests unsupported parent of TF/XLA communication op. + +func @unsupported_ancestor(%arg0: tensor, %arg1: tensor) { + %0 = "mhlo.reduce"(%arg0, %arg1) ( { + ^bb0(%arg2: tensor, %arg3: tensor): + %1 = mhlo.add %arg2, %arg3 : tensor + // expected-error@+1 {{expects ancestor(s) to be of ['mhlo.if', 'func']}} + "tf._XlaHostComputeMlir"() {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", tpu_core = 0 : i64} : () -> () + "mhlo.return"(%1) : (tensor) -> () + }) {dimensions = dense<[1]> : tensor<1xi64>} : (tensor, tensor) -> tensor + return +} + +// ----- + +// Tests transitive unsupported parent of TF/XLA communication op. + +func @unsupported_ancestor(%arg0: tensor, %arg1: tensor) { + %0 = "mhlo.reduce"(%arg0, %arg1) ( { + ^bb0(%arg2: tensor, %arg3: tensor): + %1 = mhlo.add %arg2, %arg3 : tensor + // expected-error@+1 {{expects ancestor(s) to be of ['mhlo.if', 'func']}} + call @callee() : () -> () + "mhlo.return"(%1) : (tensor) -> () + }) {dimensions = dense<[1]> : tensor<1xi64>} : (tensor, tensor) -> tensor + return +} + +func @callee() attributes {sym_visibility = "private"} { + "tf._XlaHostComputeMlir"() {recv_key = "host_compute_channel_recv", send_key = "host_compute_channel_send", tpu_core = 0 : i64} : () -> () + return +} + +// ----- + +// Tests unsupported `mhlo.if` with region of more than one block and contains a +// TF/XLA communication op. + +func @if_multiple_blocks(%arg0: tensor, %arg1: tensor) { + %0 = "mhlo.if"(%arg0, %arg1, %arg1) ( { + ^bb0(%arg2: tensor): + br ^bb1(%arg2 : tensor) + ^bb1(%arg3: tensor): + // expected-error@+1 {{expects single block region ancestor(s)}} + "tf.XlaSendToHost"(%arg3) {key = "send_key0"} : (tensor) -> () + "mhlo.return"(%arg3) : (tensor) -> () + }, { + ^bb0(%arg2: tensor): + "mhlo.return"(%arg2) : (tensor) -> () + }) : (tensor, tensor, tensor) -> tensor + return +} + +// ----- + // Tests function with more than one block that is to be rewritten emits an // error instead. diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc index 588e31ab669..b4e4f5c4f5c 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc @@ -22,15 +22,20 @@ limitations under the License. #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Sequence.h" +#include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FormatVariadic.h" #include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project #include "mlir/IR/Attributes.h" // from @llvm-project #include "mlir/IR/Builders.h" // from @llvm-project #include "mlir/IR/Module.h" // from @llvm-project +#include "mlir/IR/StandardTypes.h" // from @llvm-project #include "mlir/IR/TypeUtilities.h" // from @llvm-project +#include "mlir/IR/Value.h" // from @llvm-project #include "mlir/IR/Visitors.h" // from @llvm-project #include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project #include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.h" #include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" #include "tensorflow/compiler/mlir/xla/type_to_shape.h" @@ -49,45 +54,100 @@ const char kXlaHostTransferOriginalTypeAttr[] = "_xla_host_transfer_original_type"; // A pass that legalizes TF/XLA communication ops, propagate their respective -// tokens (for ordering), and rewrite their respective functions when necessary. +// tokens (for ordering), and rewrite their respective functions and control +// flow ops when necessary. // Note, this currently does not handle nested modules/functions or region based -// ops (e.g. control flow). +// ops other than certain control flow ops (`mhlo.if`). class LegalizeTFCommunication : public PassWrapper> { public: void runOnOperation() override; }; -// Checks if a function has any communication ops. -bool HasCommunicationOps(FuncOp func) { - auto result = func.walk([](Operation* op) { - if (isa(op)) +// Checks if an op is a TF/XLA communication op. +bool IsCommunicationOp(Operation* op) { + return isa(op); +} + +// Checks if an op is a supported HLO control flow op. +bool IsControlFlowOp(Operation* op) { return isa(op); } + +// Collects control flow op ancestors of a given op, up until FuncOp. If any +// ancestor is not a control flow op or a FuncOp, or of a single block region, +// an error will be returned. +LogicalResult GetControlFlowAncestors( + Operation* op, llvm::SmallPtrSetImpl& control_flow_ops, + llvm::SmallPtrSetImpl& control_flow_blocks) { + Block* block = op->getBlock(); + Operation* parent = block->getParentOp(); + while (block && parent && !isa(parent)) { + if (!IsControlFlowOp(parent)) + return op->emitOpError() + << "expects ancestor(s) to be of ['" << IfOp::getOperationName() + << "', '" << FuncOp::getOperationName() << "']"; + + if (!llvm::hasSingleElement(block->getParent()->getBlocks())) + return op->emitOpError() << "expects single block region ancestor(s)"; + + control_flow_ops.insert(parent); + control_flow_blocks.insert(block); + + parent = block->getParentOp(); + block = parent->getBlock(); + } + return success(); +} + +// Finds communication ops in a function. `control_flow_ops` and +// `control_flow_blocks` will be populated with control flow op ancestors for +// every communication op. +LogicalResult FindCommunicationOps( + FuncOp func, llvm::SmallPtrSetImpl& control_flow_ops, + llvm::SmallPtrSetImpl& control_flow_blocks, + bool& has_communication_ops) { + auto result = func.walk([&](Operation* op) { + if (!IsCommunicationOp(op)) return WalkResult::advance(); + has_communication_ops = true; + if (failed( + GetControlFlowAncestors(op, control_flow_ops, control_flow_blocks))) return WalkResult::interrupt(); return WalkResult::advance(); }); - return result.wasInterrupted(); + return failure(result.wasInterrupted()); } -// Helper struct holding a function and optional cloned version. If `clone` is -// set, function calls to `original` will be replaced with `clone`. -struct FuncAndClone { +// Helper struct holding a function to be rewritten, it's control flow ops that +// lead to a communication op or function call with a communication op +// (transitively), and an optional clone of itself. If `clone` is set, function +// calls to `original` will be replaced with `clone`. +struct FuncToRewrite { FuncOp original; + llvm::SmallPtrSet control_flow_ops; + llvm::SmallPtrSet control_flow_blocks; FuncOp clone; }; // Finds all functions that need to be rewritten with communication ops and // and associated tokens. -llvm::SmallDenseMap GetFunctionsToRewrite( - ModuleOp module) { +LogicalResult GetFunctionsToRewrite( + ModuleOp module, + llvm::SmallDenseMap& funcs_to_rewrite) { // Find functions containing communication ops. - llvm::SmallDenseMap funcs; SmallVector funcs_to_visit; for (FuncOp func : module.getOps()) { - if (HasCommunicationOps(func)) { - funcs.insert({func.getName(), {func, /*clone=*/nullptr}}); - funcs_to_visit.push_back(func); - } + FuncToRewrite func_to_rewrite{/*original=*/func, /*control_flow_ops=*/{}, + /*control_flow_blocks=*/{}, + /*clone=*/nullptr}; + bool has_communication_ops = false; + if (failed(FindCommunicationOps(func, func_to_rewrite.control_flow_ops, + func_to_rewrite.control_flow_blocks, + has_communication_ops))) + return failure(); + + if (!has_communication_ops) continue; + funcs_to_rewrite.insert({func.getName(), func_to_rewrite}); + funcs_to_visit.push_back(func); } // Find functions that call functions with communication ops, transitively. @@ -100,13 +160,30 @@ llvm::SmallDenseMap GetFunctionsToRewrite( // Only `mlir::CallOp` is supported as this requires knowing how to // rewrite arguments and results to a function. if (!isa(use.getUser())) continue; - auto caller_func = use.getUser()->getParentOfType(); - if (!caller_func) continue; - if (funcs - .insert( - {caller_func.getName(), {caller_func, /*clone=*/nullptr}}) - .second) - new_funcs_to_visit.push_back(caller_func); + auto caller_parent_func = use.getUser()->getParentOfType(); + if (!caller_parent_func) continue; + + FuncToRewrite func_to_rewrite{/*original=*/caller_parent_func, + /*control_flow_ops=*/{}, + /*control_flow_blocks=*/{}, + /*clone=*/nullptr}; + if (failed(GetControlFlowAncestors( + use.getUser(), func_to_rewrite.control_flow_ops, + func_to_rewrite.control_flow_blocks))) + return failure(); + + auto it = funcs_to_rewrite.insert( + {caller_parent_func.getName(), func_to_rewrite}); + if (it.second) { + new_funcs_to_visit.push_back(caller_parent_func); + } else { + it.first->getSecond().control_flow_ops.insert( + func_to_rewrite.control_flow_ops.begin(), + func_to_rewrite.control_flow_ops.end()); + it.first->getSecond().control_flow_blocks.insert( + func_to_rewrite.control_flow_blocks.begin(), + func_to_rewrite.control_flow_blocks.end()); + } } } @@ -116,8 +193,9 @@ llvm::SmallDenseMap GetFunctionsToRewrite( // Clone public functions that need to be rewritten. Function calls to this // function will be replaced with the cloned function. SymbolTable symbol_table(module); - for (auto& func : funcs) { - if (func.getSecond().original.isPublic()) { + for (auto& func : funcs_to_rewrite) { + if (func.getSecond().original.isPublic() && + !func.getSecond().original.symbolKnownUseEmpty(module)) { auto clone = func.getSecond().original.clone(); clone.setVisibility(SymbolTable::Visibility::Private); symbol_table.insert(clone); @@ -125,7 +203,7 @@ llvm::SmallDenseMap GetFunctionsToRewrite( } } - return funcs; + return success(); } // Assigns op sharding to an op for a given device core. @@ -329,94 +407,412 @@ Value RewriteCallOp(OpBuilder& builder, CallOp call, return new_call.getResults().back(); } -// Updates function terminator and type if a token is to be emitted by the -// function. -void RewriteFunctionTerminatorAndUpdateType(OpBuilder& builder, FuncOp func, - Block& func_body, Value token) { - // If the function signature is changed, update to emit a token and update - // the function type. - Operation* terminator = func_body.getTerminator(); - auto new_results = llvm::to_vector<4>(terminator->getOperands()); - new_results.push_back(token); - builder.setInsertionPoint(terminator); - auto new_return = - builder.create(terminator->getLoc(), new_results); - terminator->erase(); +// Helper struct holding state of which op to visit to next. If `op` is in a +// control flow op region, `region_idx` will be set with the respective region +// index. `token` will be current token from the last communication op/control +// flow op transitive communication ops. +struct OpVisitorState { + Optional region_idx; + Value token; + Operation* op; +}; +// Creates a tuple from a sequence of values. +Value CreateTuple(OpBuilder& builder, Location loc, ArrayRef operands) { + return builder.create(loc, operands).getResult(); +} + +// Replaces a value `value` with a new value but the token attached. If `value` +// is not a tuple, a new tuple is formed with `token`. If `value` is a tuple, +// `value` is extended instead. New tuple values created are cached. +Value GetValueWithToken(OpBuilder& builder, Value value, Value token, + llvm::SmallDenseMap& rewritten_values) { + // If value with token already exists, reuse it. + auto it = rewritten_values.find(value); + if (it != rewritten_values.end()) return it->getSecond(); + + auto create_tuple = [&](ArrayRef operands) { + auto new_result = CreateTuple(builder, value.getLoc(), operands); + rewritten_values.insert({value, new_result}); + return new_result; + }; + + auto tuple_type = value.getType().dyn_cast(); + // `value` is not a tuple, create a new tuple. + if (!tuple_type) return create_tuple({value, token}); + + // Extend tuple if `value` is a tuple. + // If `value` is an op result and the owner is a `mhlo.tuple`, simply unpack + // the tuple. + if (auto tuple_op = value.getDefiningOp()) { + auto tuple_operands = llvm::to_vector<4>(tuple_op.getOperands()); + tuple_operands.push_back(token); + return create_tuple(tuple_operands); + } + + // `value` is not created via a `mhlo.tuple` directly, unpack individual + // elements directly with `mhlo.get_tuple_element`. + SmallVector tuple_operands; + for (auto idx : llvm::seq(0, tuple_type.getTypes().size())) + tuple_operands.push_back( + builder.create(value.getLoc(), value, idx) + .getResult()); + + tuple_operands.push_back(token); + return create_tuple(tuple_operands); +} + +// Extends a type to include a `mhlo.token` type. If `type` is not a tuple type, +// a new tuple type with `type` and `mhlo.token` type is created instead. +TupleType GetTypeWithToken(OpBuilder& builder, Type type) { + auto token_type = TokenType::get(builder.getContext()); + if (auto tuple_type = type.dyn_cast()) { + auto result_types = llvm::to_vector<4>(tuple_type.getTypes()); + result_types.push_back(token_type); + return builder.getTupleType(result_types); + } + + return builder.getTupleType({type, token_type}); +} + +// Creates a slice of a tuple `value` with `mhlo.get_tuple_element` from index 0 +// to `end`, exclusive. +Value CreateSubTuple(OpBuilder& builder, Value value, size_t end) { + SmallVector tuple_operands; + for (auto idx : llvm::seq(0, end)) + tuple_operands.push_back( + builder.create(value.getLoc(), value, idx) + .getResult()); + + return CreateTuple(builder, value.getLoc(), tuple_operands); +} + +// Replaces uses of `value` with `replacement`. If `value` is not a tuple type, +// an explicit `mhlo.get_tuple_element` is created to unpack the tuple and +// return the first element. Otherwise, `mhlo.get_tuple_element` users are +// simply updated with `replacement`, and all other users are updated with a +// slice of `replacement`. +void ReplaceWithTupleResult(OpBuilder& builder, Value value, + Value replacement) { + auto tuple_type = value.getType().dyn_cast(); + if (!tuple_type) { + if (!value.use_empty()) { + auto new_element = builder.create(replacement.getLoc(), + replacement, 0); + value.replaceAllUsesWith(new_element.getResult()); + } + return; + } + + Value sub_tuple; + for (auto& use : llvm::make_early_inc_range(value.getUses())) { + if (isa(use.getOwner())) { + use.set(replacement); + continue; + } + + if (!sub_tuple) + sub_tuple = CreateSubTuple(builder, replacement, tuple_type.size()); + + use.set(sub_tuple); + } +} + +// Replaces control flow op block single block argument with new block argument +// of type `new_type` (tuple type). The last element of the new block argument +// (token) is returned. +Value UpdateControlFlowBlockArgWithToken(OpBuilder& builder, Block& block, + Type token_type) { + assert(block.getNumArguments() == 1); + builder.setInsertionPointToStart(&block); + auto new_arg = block.addArgument(token_type); + ReplaceWithTupleResult(builder, block.getArgument(0), new_arg); + block.eraseArgument(0); + return builder + .create(new_arg.getLoc(), new_arg, + token_type.cast().size() - 1) + .getResult(); +} + +// Updates control flow op terminator with an extra element `token`. If the +// original return value is not a tuple, a new tuple is formed. Otherwise the +// tuple is extended. +void RewriteControlFlowTerminator(OpBuilder& builder, Operation* terminator, + Value token) { + assert(terminator->getNumOperands() == 1); + assert(terminator->getBlock()->getNumArguments() == 1); + + builder.setInsertionPoint(terminator); + llvm::SmallDenseMap rewritten_operands; + Value new_result = GetValueWithToken(builder, terminator->getOperand(0), + token, rewritten_operands); + terminator->setOperand(0, new_result); +} + +// Rewrites a `mhlo.if` op to receive and forward a `mhlo.token`. Operands to +// the op for all of its regions are extended to have an extra operand `token`. +void RewriteRegionIfOp(OpBuilder& builder, IfOp region_if, + SmallVectorImpl& ops_to_visit, + Value token) { + SmallVector new_branch_operands; + llvm::SmallDenseMap rewritten_operands; + auto old_branch_operands = llvm::drop_begin(region_if.getOperands(), 1); + + // Rewrite all region operands to have an extra operand `token`. + for (Value operand : old_branch_operands) + new_branch_operands.push_back( + GetValueWithToken(builder, operand, token, rewritten_operands)); + + auto new_result_type = GetTypeWithToken(builder, region_if.getType()); + + // Create new `mhlo.if` op with extra token operands and result. + auto new_if = builder.create(region_if.getLoc(), new_result_type, + region_if.pred(), new_branch_operands[0], + new_branch_operands[1]); + + // Move all regions from the old `mhlo.if` op to its replacement. + for (auto& region_and_idx : llvm::enumerate(region_if.getRegions())) + new_if.getRegion(region_and_idx.index()).takeBody(*region_and_idx.value()); + + // Forward result from old `mhlo.if` with replacement, and unpack result when + // necessary. + ReplaceWithTupleResult(builder, region_if.getResult(), new_if.getResult()); + + auto new_token = builder.create( + new_if.getLoc(), new_if.getResult(), + new_if.getResult().getType().cast().size() - 1); + + region_if.erase(); + + // Remove leftover operands to old `mhlo.if` if they have no uses. + for (auto& rewritten_operand : rewritten_operands) + if (auto tuple_op = rewritten_operand.getFirst().getDefiningOp()) + if (tuple_op.use_empty()) tuple_op.erase(); + + // Next op to visit. The replacement is visited but at its first region. The + // token result of the new region if is propagated. + ops_to_visit.push_back({/*region_idx=*/0, new_token, new_if}); +} + +// Rewrites a `mhlo.if` region to receive and forward a `mhlo.token`. The block +// argument is updated to have an extra `mhlo.token` element. If the region +// block is to be rewritten, the next op to visit is set to the first op in the +// block. Otherwise the terminator is updated to forward `token`. +void RewriteRegionIfRegion( + OpBuilder& builder, IfOp region_if, unsigned region_idx, + SmallVectorImpl& ops_to_visit, + const llvm::SmallPtrSetImpl& control_flow_blocks, Value token) { + ops_to_visit.push_back({region_idx + 1, token, region_if}); + + Region& region = region_if.getRegion(region_idx); + assert(llvm::hasSingleElement(region)); + + auto block_token = UpdateControlFlowBlockArgWithToken( + builder, region.front(), region_if.getOperand(region_idx + 1).getType()); + + if (control_flow_blocks.contains(®ion.front())) { + ops_to_visit.push_back({/*region_idx=*/llvm::None, block_token, + block_token.getDefiningOp()->getNextNode()}); + return; + } + + RewriteControlFlowTerminator(builder, region.front().getTerminator(), + block_token); +} + +// Rewrites an `mhlo.if` op or its region. If `region_idx` is not set, the op +// operands and results rewritten. If `region_idx` is set, region `region_idx` +// is rewritten to take in and return an additional token. Returns true if op +// is still being rewritten. +bool ProcessRegionIfOp(OpBuilder& builder, IfOp region_if, + Optional region_idx, + SmallVectorImpl& ops_to_visit, + const llvm::SmallPtrSetImpl& control_flow_blocks, + Value token) { + builder.setInsertionPoint(region_if); + + if (!region_idx) { + RewriteRegionIfOp(builder, region_if, ops_to_visit, token); + return true; + } + + if (*region_idx < region_if.getNumRegions()) { + RewriteRegionIfRegion(builder, region_if, *region_idx, ops_to_visit, + control_flow_blocks, token); + return true; + } + + return false; +} + +// Updates function type based on current function body block arguments and +// terminator operand types. +void UpdateFunctionType(OpBuilder& builder, FuncOp func, Block& func_body) { auto new_argument_types = llvm::to_vector<4>(func_body.getArgumentTypes()); - auto new_result_types = llvm::to_vector<4>(new_return.getOperandTypes()); + auto new_result_types = + llvm::to_vector<4>(func_body.getTerminator()->getOperandTypes()); func.setType(FunctionType::get(new_argument_types, new_result_types, builder.getContext())); } -// Rewrites a function body and communication ops inside. The function may -// either be rewritten to create a token or take in and return a token, -// depending on its visibility and if there are any callers. +// Replaces a function terminator `return` with another `return` that has an +// extra `mhlo.token` operand. +void RewriteFunctionTerminator(OpBuilder& builder, mlir::ReturnOp terminator, + Value token) { + auto new_results = llvm::to_vector<4>(terminator.getOperands()); + new_results.push_back(token); + builder.setInsertionPoint(terminator); + builder.create(terminator.getLoc(), new_results); + terminator.erase(); +} + +// Rewrites a function body and communication ops inside. Region control flow +// are updated when necessary, to propagate tokens. The function may either be +// rewritten to create a token or take in and return a token, depending on its +// visibility and if there are any callers. LogicalResult RewriteFunction( OpBuilder& builder, int64_t& channel_id, ModuleOp module, FuncOp func, - const llvm::SmallDenseMap& funcs) { + const llvm::SmallDenseMap& funcs, + const llvm::SmallPtrSetImpl& control_flow_ops, + const llvm::SmallPtrSetImpl& control_flow_blocks, bool is_clone) { MLIRContext* context = module.getContext(); if (!llvm::hasSingleElement(func.getBody())) return func.emitError() << "'" << FuncOp::getOperationName() << "' ops with more than one block are not supported"; - bool rewrite_block = !func.isPublic() && !func.symbolKnownUseEmpty(module); + bool rewrite_block = + is_clone || (!func.isPublic() && !func.symbolKnownUseEmpty(module)); Block& func_body = func.front(); builder.setInsertionPointToStart(&func_body); - auto token_type = mlir::mhlo::TokenType::get(context); + auto token_type = TokenType::get(context); // If a function is public, it's signature should not be modified, and instead // a token will be created. Otherwise a token block argument is inserted. - Value token = rewrite_block - ? func_body.addArgument(token_type) + Value init_token = + rewrite_block ? func_body.addArgument(token_type) : builder.create(func.getLoc(), token_type) .getResult(); - for (Operation& op : llvm::make_early_inc_range(func_body)) { - if (auto host_compute = dyn_cast(op)) { + // Stack to keep track of region based control flow op nesting and current + // op to visit. + SmallVector ops_to_visit{ + {/*region_idx=*/llvm::None, init_token, &func_body.front()}}; + + while (!ops_to_visit.empty()) { + OpVisitorState op_to_visit = ops_to_visit.pop_back_val(); + Operation* curr_op = op_to_visit.op; + + Value token = op_to_visit.token; + // Ops may be removed, so the next op is kept track of beforehand. + Operation* next_op = curr_op->getNextNode(); + + if (auto host_compute = dyn_cast(curr_op)) { token = RewriteHostComputeOp(builder, channel_id, host_compute, token); - } else if (auto send_to_host = dyn_cast(op)) { + } else if (auto send_to_host = dyn_cast(curr_op)) { token = RewriteSendToHostOp(builder, channel_id, send_to_host, token); - } else if (auto recv_from_host = dyn_cast(op)) { + } else if (auto recv_from_host = dyn_cast(curr_op)) { token = RewriteRecvFromHostOp(builder, channel_id, recv_from_host, token); - } else if (auto call = dyn_cast(op)) { + } else if (auto call = dyn_cast(curr_op)) { // Only `mlir::CallOp` is supported as this requires knowing how to // rewrite arguments and results to a function. auto it = funcs.find(call.getCallee()); - if (it == funcs.end()) continue; - FuncOp clone = it->getSecond().clone; - Optional symbol_name = - clone ? Optional(clone.getName()) : llvm::None; - // If the function being called is to be cloned, update the call to also - // point to the cloned function. - token = RewriteCallOp(builder, call, symbol_name, token); + if (it != funcs.end()) { + FuncOp clone = it->getSecond().clone; + Optional symbol_name = + clone ? Optional(clone.getName()) : llvm::None; + // If the function being called is to be cloned, update the call to also + // point to the cloned function. + token = RewriteCallOp(builder, call, symbol_name, token); + } + } else if (auto region_if = dyn_cast(curr_op)) { + if (op_to_visit.region_idx || control_flow_ops.contains(region_if)) + if (ProcessRegionIfOp(builder, region_if, op_to_visit.region_idx, + ops_to_visit, control_flow_blocks, token)) + continue; + } else if (auto region_terminator = dyn_cast(curr_op)) { + RewriteControlFlowTerminator(builder, region_terminator, token); + // There is no next op afer the control flow op terminator, simply let + // stack have one less element. + continue; + } else if (auto func_terminator = dyn_cast(curr_op)) { + if (rewrite_block) + RewriteFunctionTerminator(builder, func_terminator, token); + + // There is no next op afer the function terminator, simply let stack have + // one less element/be empty. + continue; } + + // Visit next op. + ops_to_visit.push_back({/*region_idx=*/llvm::None, token, next_op}); } - if (rewrite_block) - RewriteFunctionTerminatorAndUpdateType(builder, func, func_body, token); + if (rewrite_block) UpdateFunctionType(builder, func, func_body); return success(); } +// Checks if a function call is pointing to a function with communication ops. +bool IsFunctionCallWithCommunication( + Operation* op, + const llvm::SmallDenseMap& funcs_to_rewrite) { + if (auto call = dyn_cast(op)) + return funcs_to_rewrite.count(call.callee()); + + return false; +} + +// Collects all control flow op ancestors of communication ops or function calls +// with communication ops (transitively). +void GetCommunicationControlFlowOps( + FuncOp func, + const llvm::SmallDenseMap& funcs_to_rewrite, + llvm::SmallPtrSetImpl& control_flow_ops, + llvm::SmallPtrSetImpl& control_flow_blocks) { + func.walk([&](Operation* op) { + if (IsCommunicationOp(op) || + IsFunctionCallWithCommunication(op, funcs_to_rewrite)) + if (failed(GetControlFlowAncestors(op, control_flow_ops, + control_flow_blocks))) + llvm_unreachable( + "checking original function for control flow ancestors should have " + "errored first"); + }); +} + void LegalizeTFCommunication::runOnOperation() { auto module = getOperation(); - llvm::SmallDenseMap funcs = - GetFunctionsToRewrite(module); + llvm::SmallDenseMap funcs_to_rewrite; + if (failed(GetFunctionsToRewrite(module, funcs_to_rewrite))) + return signalPassFailure(); // Module level counter to make sure Channel Id's are unique. int64_t channel_id = 1; OpBuilder builder(&getContext()); - for (const auto& func_and_name : funcs) { - FuncOp func = func_and_name.getSecond().original; - if (failed(RewriteFunction(builder, channel_id, module, func, funcs))) + for (const auto& func_and_name : funcs_to_rewrite) { + const auto& func_to_rewrite = func_and_name.getSecond(); + FuncOp func = func_to_rewrite.original; + if (failed(RewriteFunction(builder, channel_id, module, func, + funcs_to_rewrite, + func_to_rewrite.control_flow_ops, + func_to_rewrite.control_flow_blocks, + /*is_clone=*/false))) return signalPassFailure(); FuncOp clone = func_and_name.getSecond().clone; if (!clone) continue; - if (failed(RewriteFunction(builder, channel_id, module, clone, funcs))) - return signalPassFailure(); + llvm::SmallPtrSet clone_control_flow_ops; + llvm::SmallPtrSet clone_control_flow_blocks; + GetCommunicationControlFlowOps(clone, funcs_to_rewrite, + clone_control_flow_ops, + clone_control_flow_blocks); + if (failed(RewriteFunction(builder, channel_id, module, clone, + funcs_to_rewrite, clone_control_flow_ops, + clone_control_flow_blocks, + /*is_clone=*/true))) + llvm_unreachable( + "rewriting of original function should have errored first"); } } From 8296bf5a55e75d246b27e1cfa431f6e15dfef49d Mon Sep 17 00:00:00 2001 From: Brian Zhao Date: Thu, 6 Aug 2020 14:32:47 -0700 Subject: [PATCH 0585/1017] Add a utility to flatten StructuredValues of SavedConcreteFunctions' input and output signatures. This can be re-used for the upcoming tensorflow::Argument, as well as the current input/output size validation logic. PiperOrigin-RevId: 325312214 Change-Id: I2a043d869dfb4dca082a322587f7c65cbb8cdf01 --- .../c/experimental/saved_model/core/BUILD | 17 +++ .../saved_model/core/saved_model_utils.cc | 125 ++++++++-------- .../saved_model/core/saved_model_utils.h | 16 ++- .../core/signature_flattening_test.cc | 133 ++++++++++++++++++ 4 files changed, 231 insertions(+), 60 deletions(-) create mode 100644 tensorflow/c/experimental/saved_model/core/signature_flattening_test.cc diff --git a/tensorflow/c/experimental/saved_model/core/BUILD b/tensorflow/c/experimental/saved_model/core/BUILD index 8078758328c..b2e432782de 100644 --- a/tensorflow/c/experimental/saved_model/core/BUILD +++ b/tensorflow/c/experimental/saved_model/core/BUILD @@ -216,6 +216,23 @@ tf_cc_test( ], ) +tf_cc_test( + name = "signature_flattening_test", + srcs = [ + "signature_flattening_test.cc", + ], + deps = [ + ":saved_model_utils", + "//tensorflow/c/experimental/saved_model/core:tf_concrete_function_test_protos", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core/common_runtime/eager:core", + ], +) + tf_cc_test( name = "tf_concrete_function_loading_test", srcs = [ diff --git a/tensorflow/c/experimental/saved_model/core/saved_model_utils.cc b/tensorflow/c/experimental/saved_model/core/saved_model_utils.cc index 2037c4886de..0d97741d7f0 100644 --- a/tensorflow/c/experimental/saved_model/core/saved_model_utils.cc +++ b/tensorflow/c/experimental/saved_model/core/saved_model_utils.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/platform/stringpiece.h" #include "tensorflow/core/protobuf/saved_object_graph.pb.h" #include "tensorflow/core/protobuf/struct.pb.h" @@ -36,52 +37,8 @@ namespace tensorflow { namespace internal { namespace { -// This returns the size of `tf.nest.flatten(value)`, on values that are -// used in tf.function's input_signatures. -int FlattenedSize(const tensorflow::StructuredValue& value, Status* status) { - // This follows the logic from - // https://github.com/tensorflow/tensorflow/blob/1c064ab76064c58e54261b805027474885a1534d/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc#L2775 - switch (value.kind_case()) { - case StructuredValue::kDictValue: { - const DictValue& dict = value.dict_value(); - int size = 0; - for (const auto& field : dict.fields()) { - size += FlattenedSize(field.second, status); - } - return size; - } - case StructuredValue::kTupleValue: { - const TupleValue& tuple = value.tuple_value(); - int size = 0; - for (const StructuredValue& value : tuple.values()) { - size += FlattenedSize(value, status); - } - return size; - } - case StructuredValue::kListValue: { - const ListValue& list = value.list_value(); - int size = 0; - for (const StructuredValue& value : list.values()) { - size += FlattenedSize(value, status); - } - return size; - } - case StructuredValue::kTensorSpecValue: { - return 1; - } - case StructuredValue::kNoneValue: { - // Base case: do nothing. - // This arises, for example, as the top-level object of an output - // signature when there are no return values. - return 0; - } - default: { - status->Update(errors::Internal("Unhandled structured value kind ", - value.kind_case())); - return 0; - } - } -} +using StructuredValueDictEntry = + protobuf::MapPair; // Perform some basic sanity checks on SavedConcreteFunction's input and // output signatures with respect to the corresponding FunctionDef's input @@ -111,34 +68,34 @@ Status ValidateSavedFunctionCompatibleWithFunctionDef( // https://github.com/tensorflow/tensorflow/blob/1c064ab76064c58e54261b805027474885a1534d/tensorflow/python/eager/function.py#L1974-L1979 const std::string& name = function_def->signature().name(); + const StructuredValue& input_signature = saved_concrete_function.canonicalized_input_signature(); - Status status; - int input_signature_size = FlattenedSize(input_signature, &status); - TF_RETURN_IF_ERROR(status); - if (input_signature_size + saved_concrete_function.bound_inputs_size() != + std::vector input_specs; + TF_RETURN_IF_ERROR(FlattenSignature(input_signature, &input_specs)); + if (input_specs.size() + saved_concrete_function.bound_inputs_size() != function_def->signature().input_arg_size()) { return errors::FailedPrecondition( "FunctionDef ", name, " has ", function_def->signature().input_arg_size(), - " inputs, but the SavedConcreteFunction has ", input_signature_size, + " inputs, but the SavedConcreteFunction has ", input_specs.size(), " flattened user inputs and ", saved_concrete_function.bound_inputs_size(), " captured inputs."); } const StructuredValue& output_signature = saved_concrete_function.output_signature(); - int output_signature_size = FlattenedSize(output_signature, &status); - TF_RETURN_IF_ERROR(status); - if (output_signature_size != function_def->signature().output_arg_size()) { + std::vector output_specs; + TF_RETURN_IF_ERROR(FlattenSignature(output_signature, &output_specs)); + if (output_specs.size() != function_def->signature().output_arg_size()) { return errors::FailedPrecondition( "FunctionDef ", name, " has ", function_def->signature().output_arg_size(), - " outputs, but the SavedConcreteFunction has ", output_signature_size, + " outputs, but the SavedConcreteFunction has ", output_specs.size(), " flattened outputs."); } - return status; + return Status(); } } // namespace @@ -197,6 +154,62 @@ Status LoadTFConcreteFunction( out); } +Status FlattenSignature(const StructuredValue& signature, + std::vector* flattened_specs) { + // This follows the logic from + // https://github.com/tensorflow/tensorflow/blob/1c064ab76064c58e54261b805027474885a1534d/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc#L2775 + switch (signature.kind_case()) { + case StructuredValue::kDictValue: { + // Dictionaries must be sorted in order of keys + const DictValue& dict = signature.dict_value(); + std::vector entries; + entries.reserve(dict.fields_size()); + for (const auto& field : dict.fields()) { + entries.push_back(&field); + } + + std::sort(entries.begin(), entries.end(), + [](const StructuredValueDictEntry* x, + const StructuredValueDictEntry* y) { + return x->first < y->first; + }); + + for (const auto& entry : entries) { + TF_RETURN_IF_ERROR(FlattenSignature(entry->second, flattened_specs)); + } + return Status(); + } + case StructuredValue::kTupleValue: { + const TupleValue& tuple = signature.tuple_value(); + for (const StructuredValue& value : tuple.values()) { + TF_RETURN_IF_ERROR(FlattenSignature(value, flattened_specs)); + } + return Status(); + } + case StructuredValue::kListValue: { + const ListValue& list = signature.list_value(); + for (const StructuredValue& value : list.values()) { + TF_RETURN_IF_ERROR(FlattenSignature(value, flattened_specs)); + } + return Status(); + } + case StructuredValue::kTensorSpecValue: { + flattened_specs->push_back(&signature.tensor_spec_value()); + return Status(); + } + case StructuredValue::kNoneValue: { + // Base case: do nothing. + // This arises, for example, as the top-level object of an output + // signature when there are no return values. + return Status(); + } + default: { + return errors::Internal("Unhandled structured value kind ", + signature.kind_case()); + } + } +} + const SavedObject* FindNodeAtPath(StringPiece path, const SavedObjectGraph& object_graph) { const auto& nodes = object_graph.nodes(); diff --git a/tensorflow/c/experimental/saved_model/core/saved_model_utils.h b/tensorflow/c/experimental/saved_model/core/saved_model_utils.h index 57f30afa91b..68bfbe32222 100644 --- a/tensorflow/c/experimental/saved_model/core/saved_model_utils.h +++ b/tensorflow/c/experimental/saved_model/core/saved_model_utils.h @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/core/platform/status.h" #include "tensorflow/core/platform/stringpiece.h" #include "tensorflow/core/protobuf/saved_object_graph.pb.h" +#include "tensorflow/core/protobuf/struct.pb.h" namespace tensorflow { namespace internal { @@ -59,10 +60,17 @@ Status LoadTFConcreteFunction( captured_objects, ImmediateExecutionContext* ctx, std::unique_ptr* out); -// Find the SavedObject in `object_graph` at location `path`. `path` must be a -// dot-delimited string of object names relative to the root object. If no -// object is found, returns nullptr. Callers must ensure `object_graph` outlives -// the returned pointer. +// Flattens `signature` into a vector of TensorSpecProto pointers back into +// `signature`. `signature` must outlive flattened_specs. `signature` must also +// be the input or output signature of a SavedConcreteFunction (i.e. "nested +// structures of tensorspecs"). +Status FlattenSignature(const StructuredValue& signature, + std::vector* flattened_specs); + +// Find the SavedObject in `object_graph` at location `path`. `path` must be +// a dot-delimited string of object names relative to the root object. If no +// object is found, returns nullptr. Callers must ensure `object_graph` +// outlives the returned pointer. const SavedObject* FindNodeAtPath(StringPiece path, const SavedObjectGraph& object_graph); diff --git a/tensorflow/c/experimental/saved_model/core/signature_flattening_test.cc b/tensorflow/c/experimental/saved_model/core/signature_flattening_test.cc new file mode 100644 index 00000000000..9ee495f524a --- /dev/null +++ b/tensorflow/c/experimental/saved_model/core/signature_flattening_test.cc @@ -0,0 +1,133 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/c/experimental/saved_model/core/saved_model_utils.h" +#include "tensorflow/c/experimental/saved_model/core/tf_concrete_function_test_protos.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/struct.pb.h" + +namespace tensorflow { +namespace { + +// Validates names, shapes, and dtypes of two tensorspecprotos are equivalent. +bool TensorSpecsAreEqual(const TensorSpecProto& spec, + const std::string& expected_name, + const PartialTensorShape& expected_shape, + DataType expected_dtype) { + return spec.name() == expected_name && + PartialTensorShape(spec.shape()).IsIdenticalTo(expected_shape) && + spec.dtype() == expected_dtype; +} + +// This tests the common case for a tf.function w/o inputs. This ends up +// being serialized as a tuple of an empty tuple + empty dictionary +// (corresponding to the args, kwargs) of the function. +TEST(SignatureFlatteningTest, ZeroArgInputSignature) { + std::vector flattened; + StructuredValue value = testing::ZeroArgInputSignature(); + TF_EXPECT_OK(internal::FlattenSignature(value, &flattened)); + EXPECT_EQ(flattened.size(), 0); +} + +// This tests the common case for a tf.function w/o outputs. This ends up +// being serialized as a "NoneValue". +TEST(SignatureFlatteningTest, ZeroRetOutputSignature) { + std::vector flattened; + StructuredValue value = testing::ZeroReturnOutputSignature(); + TF_EXPECT_OK(internal::FlattenSignature(value, &flattened)); + EXPECT_EQ(flattened.size(), 0); +} + +TEST(SignatureFlatteningTest, SingleArgInputSignature) { + std::vector flattened; + StructuredValue value = testing::SingleArgInputSignature(); + TF_EXPECT_OK(internal::FlattenSignature(value, &flattened)); + EXPECT_EQ(flattened.size(), 1); + EXPECT_TRUE(TensorSpecsAreEqual(*flattened[0], + /* expected_name = */ "x", + /* expected_shape = */ {1, 10}, + /* expected_dtype = */ DT_FLOAT)) + << "Expected " << flattened[0]->DebugString(); +} + +TEST(SignatureFlatteningTest, SingleReturnOutputSignature) { + std::vector flattened; + StructuredValue value = testing::SingleReturnOutputSignature(); + TF_EXPECT_OK(internal::FlattenSignature(value, &flattened)); + EXPECT_EQ(flattened.size(), 1); + EXPECT_TRUE(TensorSpecsAreEqual(*flattened[0], + /* expected_name = */ "", + /* expected_shape = */ {1}, + /* expected_dtype = */ DT_FLOAT)) + << "Expected " << flattened[0]->DebugString(); +} + +TEST(SignatureFlatteningTest, ThreeArgInputSignature) { + std::vector flattened; + StructuredValue value = testing::ThreeArgInputSignature(); + TF_EXPECT_OK(internal::FlattenSignature(value, &flattened)); + EXPECT_EQ(flattened.size(), 3); + EXPECT_TRUE(TensorSpecsAreEqual(*flattened[0], + /* expected_name = */ "x", + /* expected_shape = */ {1}, + /* expected_dtype = */ DT_FLOAT)) + << "Expected " << flattened[0]->DebugString(); + + EXPECT_TRUE(TensorSpecsAreEqual(*flattened[1], + /* expected_name = */ "y", + /* expected_shape = */ {1}, + /* expected_dtype = */ DT_FLOAT)) + << "Expected " << flattened[1]->DebugString(); + + EXPECT_TRUE(TensorSpecsAreEqual(*flattened[2], + /* expected_name = */ "z", + /* expected_shape = */ {1}, + /* expected_dtype = */ DT_FLOAT)) + << "Expected " << flattened[2]->DebugString(); +} + +// This test has an exotic outputsignature of tuple of a +// dictionary, tensor +TEST(SignatureFlatteningTest, ThreeReturnOutputSignature) { + std::vector flattened; + StructuredValue value = testing::ThreeReturnOutputSignature(); + TF_EXPECT_OK(internal::FlattenSignature(value, &flattened)); + EXPECT_EQ(flattened.size(), 3); + EXPECT_TRUE(TensorSpecsAreEqual(*flattened[0], + /* expected_name = */ "0/a", + /* expected_shape = */ {1}, + /* expected_dtype = */ DT_FLOAT)) + << "Expected " << flattened[0]->DebugString(); + + EXPECT_TRUE(TensorSpecsAreEqual(*flattened[1], + /* expected_name = */ "0/b", + /* expected_shape = */ {1}, + /* expected_dtype = */ DT_FLOAT)) + << "Expected " << flattened[1]->DebugString(); + + EXPECT_TRUE(TensorSpecsAreEqual(*flattened[2], + /* expected_name = */ "1", + /* expected_shape = */ {1}, + /* expected_dtype = */ DT_FLOAT)) + << "Expected " << flattened[2]->DebugString(); +} + +} // namespace +} // namespace tensorflow From 1da0eb2f4781a68383f71c2082e4d753872fb953 Mon Sep 17 00:00:00 2001 From: Krzysztof Laskowski Date: Thu, 6 Aug 2020 23:42:39 +0200 Subject: [PATCH 0586/1017] Extend MemoryTypesForNode test Add verification of "_input_hostmem" and "_output_hostmem" attributes. --- .../core/framework/memory_types_test.cc | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/tensorflow/core/framework/memory_types_test.cc b/tensorflow/core/framework/memory_types_test.cc index 3126ea8e5f8..5228dbafc9b 100644 --- a/tensorflow/core/framework/memory_types_test.cc +++ b/tensorflow/core/framework/memory_types_test.cc @@ -33,12 +33,14 @@ class DummyKernel : public OpKernel { REGISTER_OP("HostMemoryTest") .Input("a: float") - .Input("b: T") - .Input("c: N * string") - .Input("d: Tlist") - .Input("e: Rlist") + .Input("b: float") + .Input("c: T") + .Input("d: N * string") + .Input("e: Tlist") + .Input("f: Rlist") .Output("o: N * T") - .Output("p: Tlist") + .Output("p: N * T") + .Output("r: Tlist") .Attr("T: type") .Attr("N: int") .Attr("Tlist: list(type)") @@ -46,21 +48,25 @@ REGISTER_OP("HostMemoryTest") REGISTER_KERNEL_BUILDER(Name("HostMemoryTest").Device(DEVICE_CPU), DummyKernel); REGISTER_KERNEL_BUILDER(Name("HostMemoryTest") .Device(DEVICE_GPU) - .HostMemory("a") - .HostMemory("c") + .HostMemory("b") .HostMemory("d") - .HostMemory("o"), + .HostMemory("e") + .HostMemory("p"), DummyKernel); TEST(MemoryTypesForNode, Simple) { NodeDef node_def; TF_ASSERT_OK(NodeDefBuilder("test", "HostMemoryTest") + .Input(FakeInput()) .Input(FakeInput()) .Input(FakeInput(DT_BOOL)) .Input(FakeInput(3)) .Input(FakeInput({DT_INT32, DT_FLOAT, DT_INT32})) .Input(FakeInput({DT_RESOURCE, DT_STRING, DT_RESOURCE})) .Finalize(&node_def)); + AddNodeAttr("_input_hostmem", {0}, &node_def); + AddNodeAttr("_output_hostmem", {6, 7}, &node_def); + MemoryTypeVector input, output; TF_EXPECT_OK(MemoryTypesForNode(OpRegistry::Global(), DEVICE_CPU, node_def, @@ -68,24 +74,26 @@ TEST(MemoryTypesForNode, Simple) { // a:float, b:bool, c:3*string, d:(int32, float, int32), // e:(resource, string, resource) EXPECT_EQ( - MemoryTypeVector({DEVICE_MEMORY, DEVICE_MEMORY, HOST_MEMORY, HOST_MEMORY, - HOST_MEMORY, DEVICE_MEMORY, DEVICE_MEMORY, + MemoryTypeVector({HOST_MEMORY, DEVICE_MEMORY, DEVICE_MEMORY, HOST_MEMORY, + HOST_MEMORY, HOST_MEMORY, DEVICE_MEMORY, DEVICE_MEMORY, DEVICE_MEMORY, HOST_MEMORY, HOST_MEMORY, HOST_MEMORY}), input); // o:3*bool, p:(int32, float, int32) EXPECT_EQ(MemoryTypeVector({DEVICE_MEMORY, DEVICE_MEMORY, DEVICE_MEMORY, - DEVICE_MEMORY, DEVICE_MEMORY, DEVICE_MEMORY}), + DEVICE_MEMORY, DEVICE_MEMORY, DEVICE_MEMORY, + HOST_MEMORY, HOST_MEMORY, DEVICE_MEMORY}), output); TF_EXPECT_OK(MemoryTypesForNode(OpRegistry::Global(), DEVICE_GPU, node_def, &input, &output)); EXPECT_EQ( - MemoryTypeVector({HOST_MEMORY, DEVICE_MEMORY, HOST_MEMORY, HOST_MEMORY, + MemoryTypeVector({HOST_MEMORY, HOST_MEMORY, DEVICE_MEMORY, HOST_MEMORY, HOST_MEMORY, HOST_MEMORY, HOST_MEMORY, HOST_MEMORY, - HOST_MEMORY, HOST_MEMORY, HOST_MEMORY}), + HOST_MEMORY, HOST_MEMORY, HOST_MEMORY, HOST_MEMORY}), input); - EXPECT_EQ(MemoryTypeVector({HOST_MEMORY, HOST_MEMORY, HOST_MEMORY, - DEVICE_MEMORY, DEVICE_MEMORY, DEVICE_MEMORY}), + EXPECT_EQ(MemoryTypeVector({DEVICE_MEMORY, DEVICE_MEMORY, DEVICE_MEMORY, + HOST_MEMORY, HOST_MEMORY, HOST_MEMORY, + HOST_MEMORY, HOST_MEMORY, DEVICE_MEMORY}), output); } From a0aee5ed2c89d83019e6b724c3bf593074456e1c Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Thu, 6 Aug 2020 14:43:38 -0700 Subject: [PATCH 0587/1017] [XLA] Implement S8,S16,U16 support for Literal::GetIntegralAsS64 PiperOrigin-RevId: 325314732 Change-Id: Ia89c4153d2a70564f46c880f25112c3b74a44b2d --- tensorflow/compiler/xla/literal.cc | 10 ++++++++-- tensorflow/compiler/xla/service/hlo_evaluator.cc | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/xla/literal.cc b/tensorflow/compiler/xla/literal.cc index 3807e6d3a56..d26e0881c53 100644 --- a/tensorflow/compiler/xla/literal.cc +++ b/tensorflow/compiler/xla/literal.cc @@ -1004,14 +1004,20 @@ absl::optional LiteralBase::GetIntegralAsS64( switch (shape().element_type()) { case PRED: return Get(multi_index); + case S8: + return Get(multi_index); case U8: return Get(multi_index); + case S16: + return Get(multi_index); + case U16: + return Get(multi_index); case S32: return Get(multi_index); - case S64: - return Get(multi_index); case U32: return Get(multi_index); + case S64: + return Get(multi_index); case U64: return Get(multi_index); default: diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc index 66e9e01fc38..acccf7aac9a 100644 --- a/tensorflow/compiler/xla/service/hlo_evaluator.cc +++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc @@ -1573,9 +1573,9 @@ class OutputBatchIndexToInputIndex { int64 index_vector_dim = dim_numbers_.index_vector_dim(); for (int64 i = 0, e = index_vector_.size(); i < e; i++) { index_vector_index_[index_vector_dim] = i; - // TODO(george): OK what should happen here? - // seems OK to crash though. - index_vector_[i] = *start_indices_.GetIntegralAsS64(index_vector_index_); + auto start_index = start_indices_.GetIntegralAsS64(index_vector_index_); + TF_RET_CHECK(start_index.has_value()); + index_vector_[i] = *start_index; } return Status::OK(); } From 1fb7fbe3563fc177c2d9d19239e3f3f3687cffa0 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Thu, 6 Aug 2020 14:54:56 -0700 Subject: [PATCH 0588/1017] Fix. --- tensorflow/BUILD | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 8a6c1048078..484e45eb11d 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -882,8 +882,8 @@ genrule( visibility = ["//visibility:public"], ) -# The interface library (tensorflow_framework.dll.if.lib) for linking tensorflow DLL library -# (tensorflow_framework.dll) on Windows. +# The interface library (tensorflow_framework.dll.if.lib) for linking tensorflow DLL +# library (tensorflow_framework.dll) on Windows. # To learn more about import library (called interface library in Bazel): # https://docs.microsoft.com/en-us/cpp/build/linking-an-executable-to-a-dll?view=vs-2017#linking-implicitly filegroup( @@ -893,8 +893,8 @@ filegroup( visibility = ["//visibility:public"], ) -# Rename the import library for tensorflow_framework.dll from tensorflow_framework.dll.if.lib to -# tensorflow_framework.lib +# Rename the import library for tensorflow_framework.dll from +# tensorflow_framework.dll.if.lib to tensorflow_framework.lib genrule( name = "tensorflow_framework_dll_import_lib", srcs = [":get_tensorflow_framework_dll_import_lib"], From 544771ff261d80e9ca831d9b18d1690d4e00bf7d Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Thu, 6 Aug 2020 14:46:43 -0700 Subject: [PATCH 0589/1017] Add a static `getDialectNamespace()` method on MLIR Dialect This is now a requirement from the framework. PiperOrigin-RevId: 325315389 Change-Id: Ia8b8641d208caabd861c7ef1f63a99cbbd4dac8e --- tensorflow/compiler/mlir/tensorflow/ir/tf_device.h | 1 + tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h | 1 + 2 files changed, 2 insertions(+) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_device.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_device.h index d1ca07d85a7..688c8ca5715 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_device.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_device.h @@ -36,6 +36,7 @@ namespace tf_device { // XlaRun. class TensorFlowDeviceDialect : public Dialect { public: + static StringRef getDialectNamespace() { return "tf_device"; } // Constructing TensorFlowDevice dialect under an non-null MLIRContext. explicit TensorFlowDeviceDialect(MLIRContext* context); }; diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h index 3bb30f16c3d..61358172d6d 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h @@ -35,6 +35,7 @@ namespace tf_executor { class TensorFlowExecutorDialect : public Dialect { public: + static StringRef getDialectNamespace() { return "tf_executor"; } explicit TensorFlowExecutorDialect(MLIRContext *context); // Parses a type registered to this dialect. From e75098f34e92a3538876e0b636785e2f413e472d Mon Sep 17 00:00:00 2001 From: Mehmet Deveci Date: Thu, 6 Aug 2020 14:48:07 -0700 Subject: [PATCH 0590/1017] Adding a file name suffix option to event file names to prevent event file name collusions. PiperOrigin-RevId: 325315695 Change-Id: I9020e0ab52ea8e86b1f2dd852ff285c8a27a3ddb --- tensorflow/python/tpu/tensor_tracer.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/tpu/tensor_tracer.py b/tensorflow/python/tpu/tensor_tracer.py index c0536d84182..3f8f7530a8d 100644 --- a/tensorflow/python/tpu/tensor_tracer.py +++ b/tensorflow/python/tpu/tensor_tracer.py @@ -44,6 +44,7 @@ from tensorflow.python.ops import logging_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import nn_impl from tensorflow.python.ops import state_ops +from tensorflow.python.ops import string_ops from tensorflow.python.ops import summary_ops_v2 as summary from tensorflow.python.ops import variable_scope from tensorflow.python.platform import analytics @@ -1643,11 +1644,12 @@ class TensorTracer(object): raise ValueError('Provide a trace_dir for tensor tracer in summary mode. ' '--trace_dir=/model/dir') - def _write_cache(step, **kwargs): + def _write_cache(step, event_file_suffix=None, **kwargs): """Writes the given caches as tensor summary. Args: step: Step tensor with dimension [num_cores]. + event_file_suffix: Event filename suffix tensor. **kwargs: The dictionary of tensors that needs to be written as summaries. Key and value pairs within kwargs correspond to the tag name, and tensor content that will be written using summary.write. @@ -1664,16 +1666,20 @@ class TensorTracer(object): Raises: RuntimeError: if there is no aggregate function defined for a signature. """ - + file_suffix = _TT_EVENT_FILE_SUFFIX + if event_file_suffix is not None: + file_suffix = string_ops.string_join([file_suffix, event_file_suffix], + separator='.') # TODO(deveci): Parametrize max_queue, so that flushing op can be called # less frequently. # Setting max_queue to 100 appears to be safe even when the number of # iterations are much lower, as the destructor of the writer flushes it. summary_write_ops = [] - with summary.create_file_writer_v2( + summary_writer = summary.create_file_writer_v2( self._parameters.trace_dir, - filename_suffix=_TT_EVENT_FILE_SUFFIX, - max_queue=_TT_SUMMARY_MAX_QUEUE).as_default(): + filename_suffix=file_suffix, + max_queue=_TT_SUMMARY_MAX_QUEUE) + with summary_writer.as_default(): summary_metadata = summary_pb2.SummaryMetadata( plugin_data=summary_pb2.SummaryMetadata.PluginData( plugin_name=_TT_TENSORBOARD_PLUGIN_NAME)) @@ -1688,8 +1694,7 @@ class TensorTracer(object): if key == _TT_SUMMARY_TAG and value.shape.as_list()[0] != 1: value = self.aggregate_global_cache(value) - with ops.control_dependencies( - summary.summary_writer_initializer_op()): + with ops.control_dependencies([summary_writer.init()]): summary_write_ops.append(summary.write( _TT_SUMMARY_TAG + '/' + key, value, metadata=summary_metadata, step=step[0])) From aca9c898732ab1ef08aaebbacce4f1a82cda2fcb Mon Sep 17 00:00:00 2001 From: Ayush Dubey Date: Thu, 6 Aug 2020 14:55:57 -0700 Subject: [PATCH 0591/1017] Return `errors::Aborted` from ScopedAllocatorOptimizer when it cannot optimize a valid graph. PiperOrigin-RevId: 325317272 Change-Id: I97c9d51cffbf08cb44f8078357530b83a8061c0e --- .../optimizers/scoped_allocator_optimizer.cc | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc index 6fb62019806..3f33ff50f6c 100644 --- a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc @@ -104,20 +104,20 @@ Status CheckTypesAndGetShapes(const GraphProperties& graph_properties, << shapes->size(); if (!graph_properties.HasOutputProperties(n->name())) { LOG(ERROR) << "Node " << n->DebugString() << " lacks output shape."; - return errors::Internal("Node ", n->name(), " lacks output shape."); + return errors::Aborted("Node ", n->name(), " lacks output shape."); } const std::vector& prop_list = graph_properties.GetOutputProperties(n->name()); if (prop_list.size() != 1) { - return errors::Internal("Node ", n->name(), - " does not have exactly one output as expected " - "by ScopedAllocatorOptimizer"); + return errors::Aborted("Node ", n->name(), + " does not have exactly one output as expected " + "by ScopedAllocatorOptimizer"); } const OpInfo::TensorProperties& props = prop_list[0]; if (shapes->empty()) { *type = props.dtype(); } else if (*type != props.dtype()) { - return errors::Internal("Group ops don't all have same type"); + return errors::Aborted("Group ops don't all have same type"); } if (*type != dtype) { return errors::Internal( @@ -128,7 +128,7 @@ Status CheckTypesAndGetShapes(const GraphProperties& graph_properties, // TensorShape::IsValid may return true if unknown_rank is True, i.e. // number of dimensions is unknown. But for ScopedAllocatorOptimizer we // need to know the shape fully. - return errors::Internal("Complete shape not known for ", n->name()); + return errors::Aborted("Complete shape not known for ", n->name()); } VLOG(2) << "Adding shape " << props.shape().DebugString(); shapes->push_back(TensorShape(props.shape())); @@ -301,8 +301,8 @@ Status GetInputs(ScopedAllocatorOptimizer* sa_opti, int64 invocation_count, GetOutputDataType(inode_output_props, output_index, &inode_dtype)); } if (inode_dtype != dtype) { - return errors::Internal("ScopedAllocatorOptimizer expected input type ", - dtype, " but found ", inode_dtype); + return errors::Aborted("ScopedAllocatorOptimizer expected input type ", + dtype, " but found ", inode_dtype); } inputs->emplace_back(inode, output_index, n); } @@ -393,7 +393,7 @@ class UnaryElementwiseRewriter : public ScopedAllocatorOptimizer::Rewriter { LOG(INFO) << "Abandoning ScopedAllocatorOptimizer because input " << nd.from_node_def->name() << " output " << scope_ids[0] << " is already assigned to scope_id " << scope_ids[1]; - return errors::Internal( + return errors::Aborted( "Abandoning ScopedAllocatorOptimizer because input ", nd.from_node_def->name(), " output ", scope_ids[0], " is already ", "assigned to scope_id ", scope_ids[1]); @@ -408,10 +408,10 @@ class UnaryElementwiseRewriter : public ScopedAllocatorOptimizer::Rewriter { for (const InputDesc& nd : inputs) { if (op_set.find(nd.from_node_def->name()) != op_set.end()) { if (nd.output_slot != tensorflow::Graph::kControlSlot) { - return errors::Internal("Data edge exists between ", - nd.from_node_def->name(), - " and another " - "node in the set"); + return errors::Aborted("Data edge exists between ", + nd.from_node_def->name(), + " and another " + "node in the set"); } } } @@ -539,7 +539,7 @@ class UnaryElementwiseRewriter : public ScopedAllocatorOptimizer::Rewriter { for (int i = 0, end = inputs.size(); i < end; ++i) { auto& nd = inputs[i]; if (IsArg(*nd.from_node_def)) { - return errors::Internal( + return errors::Aborted( "ScopedAllocatorOptimizer does not work well when the op inputs " "are _Arg ops; skipping this optimizer for this function"); } @@ -619,9 +619,9 @@ class UnaryElementwiseRewriter : public ScopedAllocatorOptimizer::Rewriter { if (op_instance_names.find(old_op_input) != op_instance_names.end()) { LOG(ERROR) << "Data edge between " << old_op_input << " and " << old_op->name() << " cannot build ScopedAllocator."; - return errors::Internal("Data edge between ", old_op_input, " and ", - old_op->name(), - " cannot build ScopedAllocator."); + return errors::Aborted("Data edge between ", old_op_input, " and ", + old_op->name(), + " cannot build ScopedAllocator."); } sac_inputs->push_back( NodeDefBuilder::NodeOut(old_op_input, 0, dtype)); @@ -952,7 +952,7 @@ int ScopedAllocatorOptimizer::NewScopedAllocatorId(int num_fields) { Status ScopedAllocatorOptimizer::NewIdentityId(int* id) { *id = next_identity_id_++; if (next_identity_id_ < 0) { - return errors::Internal("NewIdentityId overflow"); + return errors::Aborted("NewIdentityId overflow"); } return Status::OK(); } From 50145eeeb12f5023efab8fea27e6f94953d44299 Mon Sep 17 00:00:00 2001 From: Sachin Joglekar Date: Thu, 6 Aug 2020 15:07:26 -0700 Subject: [PATCH 0592/1017] Remove NodeInfoDelegate from calibrator & inspect Interpreter directly. It breaks the contract that delegates must outlive Interpreter. PiperOrigin-RevId: 325319661 Change-Id: Ia33c0e67721e35347ae859f82d8e3bf0d17d1c2d --- .../lite/tools/optimize/calibration/BUILD | 36 ---- .../tools/optimize/calibration/calibrator.cc | 25 +-- .../calibration/node_info_delegate.cc | 69 ------- .../optimize/calibration/node_info_delegate.h | 67 ------- .../calibration/node_info_delegate_test.cc | 178 ------------------ 5 files changed, 13 insertions(+), 362 deletions(-) delete mode 100644 tensorflow/lite/tools/optimize/calibration/node_info_delegate.cc delete mode 100644 tensorflow/lite/tools/optimize/calibration/node_info_delegate.h delete mode 100644 tensorflow/lite/tools/optimize/calibration/node_info_delegate_test.cc diff --git a/tensorflow/lite/tools/optimize/calibration/BUILD b/tensorflow/lite/tools/optimize/calibration/BUILD index 06183353e44..674ef0ae4f6 100644 --- a/tensorflow/lite/tools/optimize/calibration/BUILD +++ b/tensorflow/lite/tools/optimize/calibration/BUILD @@ -41,7 +41,6 @@ cc_library( ":calibration_reader", ":logging_op", ":logging_op_resolver", - ":node_info_delegate", "//tensorflow/lite:framework", "//tensorflow/lite:string_util", "//tensorflow/lite/c:common", @@ -156,39 +155,4 @@ cc_library( ], ) -cc_library( - name = "node_info_delegate", - srcs = ["node_info_delegate.cc"], - hdrs = ["node_info_delegate.h"], - copts = tflite_copts(), - deps = [ - ":calibration_common", - "//tensorflow/lite:framework", - ], -) - -tf_cc_test( - name = "node_info_delegate_test", - srcs = ["node_info_delegate_test.cc"], - args = [ - "--test_model_file=$(location //tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin)", - ], - data = [ - "//tensorflow/lite/tools/optimize:testdata/single_conv_weights_min_0_max_plus_10.bin", - ], - tags = [ - "tflite_not_portable_android", - "tflite_not_portable_ios", - ], - deps = [ - ":node_info_delegate", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/lite:framework", - "//tensorflow/lite/kernels:builtin_ops", - "//tensorflow/lite/tools/optimize:test_util", - "@com_google_googletest//:gtest", - ], -) - tflite_portable_test_suite() diff --git a/tensorflow/lite/tools/optimize/calibration/calibrator.cc b/tensorflow/lite/tools/optimize/calibration/calibrator.cc index fb1677fda99..c82057ec207 100644 --- a/tensorflow/lite/tools/optimize/calibration/calibrator.cc +++ b/tensorflow/lite/tools/optimize/calibration/calibrator.cc @@ -39,7 +39,6 @@ limitations under the License. #include "tensorflow/lite/tools/optimize/calibration/calibration_reader.h" #include "tensorflow/lite/tools/optimize/calibration/logging_op.h" #include "tensorflow/lite/tools/optimize/calibration/logging_op_resolver.h" -#include "tensorflow/lite/tools/optimize/calibration/node_info_delegate.h" namespace tflite { namespace optimize { @@ -267,18 +266,20 @@ TfLiteStatus GetNodeOpInfoMapAndContext( const std::unordered_map& node_to_opinfo, tflite::Interpreter* const interpreter, std::unordered_map* node_ptr_opinfo_map, - const TfLiteContext** context) { - NodeInfoDelegateObserver delegate_observer(node_to_opinfo, - node_ptr_opinfo_map); - NodeInfoDelegateParams delegate_params; - delegate_params.delegate_observer = &delegate_observer; - TfLiteDelegate logging_delegate = CreateNodeInfoDelegate(&delegate_params); + TfLiteContext** context) { + *context = interpreter->primary_subgraph().context(); - auto modify_status = interpreter->ModifyGraphWithDelegate(&logging_delegate); - if (modify_status != kTfLiteOk) { - return kTfLiteError; + // Since we only consider the primary subgraph while populating + // node_to_opinfo, do the same here. + TF_LITE_ENSURE_EQ(*context, interpreter->execution_plan().size(), + node_to_opinfo.size()); + for (const auto op_index : interpreter->execution_plan()) { + const auto* node_and_reg = interpreter->node_and_registration(op_index); + + auto op_info = node_to_opinfo.at(op_index); + op_info.registration = &node_and_reg->second; + node_ptr_opinfo_map->insert({&node_and_reg->first, op_info}); } - *context = delegate_observer.GetContext(); return kTfLiteOk; } @@ -391,7 +392,7 @@ TfLiteStatus BuildLoggingInterpreter( // Compute the mapping between runtime and static graph structure, i.e. // (TfLiteContext, TfLiteNode) -> OperatorInfo std::unordered_map node_ptr_opinfo_map; - const TfLiteContext* context = nullptr; + TfLiteContext* context = nullptr; GetNodeOpInfoMapAndContext(node_to_opinfo, interpreter->get(), &node_ptr_opinfo_map, &context); diff --git a/tensorflow/lite/tools/optimize/calibration/node_info_delegate.cc b/tensorflow/lite/tools/optimize/calibration/node_info_delegate.cc deleted file mode 100644 index 84031761b30..00000000000 --- a/tensorflow/lite/tools/optimize/calibration/node_info_delegate.cc +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include "tensorflow/lite/tools/optimize/calibration/node_info_delegate.h" - -namespace tflite { -namespace optimize { -namespace calibration { - -namespace { -// The prepare function for delegate that forwards the prepare call to the -// delegate observer in node info delegate params. -// The function simply calls a delegate observer OnDelegatePrepareMethod. -TfLiteStatus NodeInfoDelegatePrepare(TfLiteContext* context, - TfLiteDelegate* delegate) { - if (delegate == nullptr) return TfLiteStatus::kTfLiteError; - - NodeInfoDelegateParams* params = - reinterpret_cast(delegate->data_); - return params->delegate_observer->OnDelegatePrepareCalled(context); -} -} // namespace - -TfLiteDelegate CreateNodeInfoDelegate(NodeInfoDelegateParams* params) { - auto delegate = TfLiteDelegateCreate(); - delegate.data_ = params; - delegate.Prepare = NodeInfoDelegatePrepare; - delegate.CopyFromBufferHandle = nullptr; - delegate.CopyToBufferHandle = nullptr; - delegate.FreeBufferHandle = nullptr; - delegate.flags = kTfLiteDelegateFlagsAllowDynamicTensors; - return delegate; -} - -TfLiteStatus NodeInfoDelegateObserver::OnDelegatePrepareCalled( - TfLiteContext* context) { - context_ = context; - const size_t num_nodes = node_index_opinfo_map_.size(); - for (size_t node_index = 0; node_index < num_nodes; node_index++) { - TfLiteNode* node = nullptr; - TfLiteRegistration* reg = nullptr; - TF_LITE_ENSURE_STATUS( - context->GetNodeAndRegistration(context, node_index, &node, ®)); - auto op_info = node_index_opinfo_map_.at(node_index); - op_info.registration = reg; - node_ptr_opinfo_map_->insert({node, op_info}); - } - - if (node_ptr_opinfo_map_->size() != node_index_opinfo_map_.size()) { - // Something wrong. - return kTfLiteError; - } - return kTfLiteOk; -} - -} // namespace calibration -} // namespace optimize -} // namespace tflite diff --git a/tensorflow/lite/tools/optimize/calibration/node_info_delegate.h b/tensorflow/lite/tools/optimize/calibration/node_info_delegate.h deleted file mode 100644 index 56f6141f21d..00000000000 --- a/tensorflow/lite/tools/optimize/calibration/node_info_delegate.h +++ /dev/null @@ -1,67 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#ifndef TENSORFLOW_LITE_TOOLS_OPTIMIZE_NODE_INFO_DELEGATE_H_ -#define TENSORFLOW_LITE_TOOLS_OPTIMIZE_NODE_INFO_DELEGATE_H_ - -#include - -#include "tensorflow/lite/context.h" -#include "tensorflow/lite/tools/optimize/calibration/calibration_common.h" - -namespace tflite { -namespace optimize { -namespace calibration { - -// An interface for delegate observer that can listen to TfLiteDelegate::Prepare -// calls. -class DelegateObserver { - public: - virtual TfLiteStatus OnDelegatePrepareCalled(TfLiteContext* context) = 0; - virtual ~DelegateObserver() {} -}; - -// The parameters for the node info delegate. -struct NodeInfoDelegateParams { - DelegateObserver* delegate_observer; -}; - -// Creates a delegate with the given |params|. -TfLiteDelegate CreateNodeInfoDelegate(NodeInfoDelegateParams* params); - -// A delegate observer that can construct the map from TfLiteNode* -> -// OperatorInfo. -class NodeInfoDelegateObserver : public DelegateObserver { - public: - NodeInfoDelegateObserver( - const std::unordered_map& node_index_to_op, - std::unordered_map* node_ptr_opinfo_map) - : node_index_opinfo_map_(node_index_to_op), - node_ptr_opinfo_map_(node_ptr_opinfo_map) {} - - TfLiteStatus OnDelegatePrepareCalled(TfLiteContext* context) override; - - // Returns the context that was used to called the prepare method. - const TfLiteContext* GetContext() const { return context_; } - - private: - const TfLiteContext* context_ = nullptr; - const std::unordered_map& node_index_opinfo_map_; - std::unordered_map* node_ptr_opinfo_map_; -}; - -} // namespace calibration -} // namespace optimize -} // namespace tflite -#endif // TENSORFLOW_LITE_TOOLS_OPTIMIZE_NODE_INFO_DELEGATE_H_ diff --git a/tensorflow/lite/tools/optimize/calibration/node_info_delegate_test.cc b/tensorflow/lite/tools/optimize/calibration/node_info_delegate_test.cc deleted file mode 100644 index 722bdbdbb39..00000000000 --- a/tensorflow/lite/tools/optimize/calibration/node_info_delegate_test.cc +++ /dev/null @@ -1,178 +0,0 @@ -/* Copyright 2018 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ -#include - -#include -#include -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/util/command_line_flags.h" -#include "tensorflow/lite/kernels/register.h" -#include "tensorflow/lite/model.h" -#include "tensorflow/lite/tools/optimize/calibration/node_info_delegate.h" -#include "tensorflow/lite/tools/optimize/test_util.h" - -namespace { -tensorflow::string* g_test_model_dir = nullptr; -} // namespace - -namespace tflite { -namespace optimize { -namespace calibration { -namespace { - -std::unique_ptr ReadModel(const char* model) { - auto model_path = tensorflow::io::JoinPath(*g_test_model_dir, model); - return FlatBufferModel::BuildFromFile(model_path.c_str()); -} - -std::unique_ptr ReadModel() { - return ReadModel(internal::kConvModelWith0Plus10Weights); -} - -class TestDelegateObserver : public DelegateObserver { - public: - explicit TestDelegateObserver(TfLiteStatus status_to_return) - : status_to_return_(status_to_return) {} - - TfLiteStatus OnDelegatePrepareCalled(TfLiteContext* context) override { - num_times_called_++; - return status_to_return_; - } - int num_times_called() { return num_times_called_; } - - private: - int num_times_called_ = 0; - TfLiteStatus status_to_return_; -}; - -TEST(NodeInfoDelegateTest, DelegateObserverIsCalled) { - TestDelegateObserver observer(kTfLiteOk); - NodeInfoDelegateParams params; - params.delegate_observer = &observer; - auto model = ReadModel(); - ASSERT_TRUE(model); - std::unique_ptr interpreter; - ASSERT_EQ(InterpreterBuilder(*model, - ops::builtin::BuiltinOpResolver{})(&interpreter), - kTfLiteOk); - ASSERT_TRUE(interpreter); - EXPECT_EQ(0, observer.num_times_called()); - TfLiteDelegate delegate = CreateNodeInfoDelegate(¶ms); - - auto status = interpreter->ModifyGraphWithDelegate(&delegate); - EXPECT_EQ(kTfLiteOk, status); - EXPECT_EQ(1, observer.num_times_called()); -} - -TEST(NodeInfoDelegateTest, ObserverErrorCausesModifyGraphFailure) { - // Observer returns error - TestDelegateObserver observer(kTfLiteError); - NodeInfoDelegateParams params; - params.delegate_observer = &observer; - auto model = ReadModel(); - ASSERT_TRUE(model); - std::unique_ptr interpreter; - ASSERT_EQ(InterpreterBuilder(*model, - ops::builtin::BuiltinOpResolver{})(&interpreter), - kTfLiteOk); - ASSERT_TRUE(interpreter); - TfLiteDelegate delegate = CreateNodeInfoDelegate(¶ms); - - auto status = interpreter->ModifyGraphWithDelegate(&delegate); - EXPECT_EQ(kTfLiteDelegateError, status); -} - -TEST(NodeInfoDelegateTest, NodeInfoDelegateObserver) { - auto model = ReadModel(); - ASSERT_TRUE(model); - - std::unordered_map index_to_opinfo; - auto primary_subgraph = model->GetModel()->subgraphs()->Get(0); - auto operators = primary_subgraph->operators(); - auto subgraph_tensors = primary_subgraph->tensors(); - for (size_t i = 0; i < operators->size(); i++) { - OperatorInfo info; - auto op_inputs = operators->Get(i)->inputs(); - auto op_outputs = operators->Get(i)->outputs(); - info.inputs = std::vector(op_inputs->begin(), op_inputs->end()); - info.outputs = std::vector(op_outputs->begin(), op_outputs->end()); - index_to_opinfo[i] = info; - } - - std::unordered_map node_to_opinfo; - NodeInfoDelegateObserver observer(index_to_opinfo, &node_to_opinfo); - NodeInfoDelegateParams params; - params.delegate_observer = &observer; - std::unique_ptr interpreter; - ASSERT_EQ(InterpreterBuilder(*model, - ops::builtin::BuiltinOpResolver{})(&interpreter), - kTfLiteOk); - ASSERT_TRUE(interpreter); - - TfLiteDelegate delegate = CreateNodeInfoDelegate(¶ms); - - auto status = interpreter->ModifyGraphWithDelegate(&delegate); - EXPECT_EQ(kTfLiteOk, status); - EXPECT_EQ(index_to_opinfo.size(), node_to_opinfo.size()); - EXPECT_EQ(interpreter->nodes_size(), node_to_opinfo.size()); - - for (const auto& node_and_opinfo : node_to_opinfo) { - const TfLiteNode* tflite_node = node_and_opinfo.first; - const OperatorInfo& info = node_and_opinfo.second; - ASSERT_EQ(tflite_node->inputs->size, info.inputs.size()); - ASSERT_EQ(tflite_node->outputs->size, info.outputs.size()); - - for (size_t input_index = 0; input_index < info.inputs.size(); - input_index++) { - const TfLiteTensor* tflite_tensor = - interpreter->tensor(tflite_node->inputs->data[input_index]); - EXPECT_EQ(tflite_tensor->name, - subgraph_tensors->Get(info.inputs[input_index])->name()->str()); - } - - for (size_t output_index = 0; output_index < info.outputs.size(); - output_index++) { - const TfLiteTensor* tflite_tensor = - interpreter->tensor(tflite_node->outputs->data[output_index]); - EXPECT_EQ( - tflite_tensor->name, - subgraph_tensors->Get(info.outputs[output_index])->name()->str()); - } - } -} - -} // namespace -} // namespace calibration -} // namespace optimize -} // namespace tflite - -int main(int argc, char** argv) { - tensorflow::string model_file; - const std::vector flag_list = { - tensorflow::Flag("test_model_file", &model_file, - "Path to test tflite model file."), - }; - - const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); - if (!parse_result) { - std::cerr << "Required test_model_file\n"; - std::abort(); - } - g_test_model_dir = - new tensorflow::string(tensorflow::io::Dirname(model_file)); - ::tensorflow::port::InitMain(argv[0], &argc, &argv); - return RUN_ALL_TESTS(); -} From 9c8581efc5c58f09924d86e4ede497223f854d1e Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Thu, 6 Aug 2020 22:45:17 +0000 Subject: [PATCH 0593/1017] fix build --- tensorflow/core/framework/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/framework/BUILD b/tensorflow/core/framework/BUILD index e09022d5235..f0dfa267d77 100644 --- a/tensorflow/core/framework/BUILD +++ b/tensorflow/core/framework/BUILD @@ -291,6 +291,7 @@ filegroup( "resource_handle.h", "tensor.cc", "tensor.h", + "tensor_key.h", "tensor_shape.cc", "tensor_shape.h", "tensor_types.h", From c7e51d1866fa0f7ca43a0eb83d4072a200ab7946 Mon Sep 17 00:00:00 2001 From: Penporn Koanantakool Date: Thu, 6 Aug 2020 15:48:50 -0700 Subject: [PATCH 0594/1017] Refactor meta_support out of two targets ("quantized_ops" and "cwise_lib") that could possibly be included in the same binary and cause multiple definition errors. meta_support.h will be removed from all MKL targets in a later commit. PiperOrigin-RevId: 325327446 Change-Id: I89c25be8f9cb587b005cd68d32d3078ad9b8829c --- tensorflow/core/kernels/BUILD | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index dfe9f35701c..e5e2ad38d9b 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -6096,6 +6096,19 @@ cc_library( ], ) +cc_library( + name = "meta_support", + srcs = ["meta_support.cc"], + hdrs = ["meta_support.h"], + deps = [ + ":quantization_utils", + "//tensorflow/core:framework", + "//tensorflow/core/platform:logging", + "//tensorflow/core/platform:mutex", + "@gemmlowp", + ], +) + # Android libraries ----------------------------------------------------------- # Changes to the Android srcs here should be replicated in @@ -6867,7 +6880,6 @@ tf_kernel_library( name = "quantized_ops", srcs = [ "dequantize_op.cc", - "meta_support.cc", "quantize_down_and_shrink_range.cc", "quantize_op.cc", "quantized_activation_ops.cc", @@ -6886,16 +6898,14 @@ tf_kernel_library( "requantize.cc", "reshape_op.h", ], - hdrs = [ - "meta_support.h", - "reference_gemm.h", - ], + hdrs = ["reference_gemm.h"], deps = [ ":concat_lib_hdrs", ":conv_ops", ":cwise_op", ":eigen_helpers", ":image_resizer_state", + ":meta_support", ":ops_util", ":pooling_ops", ":quantization_utils", @@ -8264,10 +8274,7 @@ tf_kernel_library( # should not be linked by projects that also link the cwise_op library. cc_library( name = "cwise_lib", - srcs = [ - "cwise_ops_common.cc", - "meta_support.cc", - ], + srcs = ["cwise_ops_common.cc"], hdrs = [ "cwise_ops.h", "cwise_ops_common.h", @@ -8275,10 +8282,10 @@ cc_library( "cwise_ops_gpu_gradients.cu.h", "cwise_ops_gradients.h", "fill_functor.h", - "meta_support.h", ], deps = [ ":bounds_check", + ":meta_support", ":quantization_utils", "//tensorflow/core:framework", "//tensorflow/core:lib", From 41622e7754d0103e18b12cd3756e35c7d7953d96 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Thu, 6 Aug 2020 15:49:03 -0700 Subject: [PATCH 0595/1017] Clone nightly jobs for testing CUDA 11 PiperOrigin-RevId: 325327481 Change-Id: Iba28b227125f83527ee8b4bffd947b54fd9f2006 --- .../rel/ubuntu_cuda11/cpu_libtensorflow.sh | 40 ++++++++++++ .../rel/ubuntu_cuda11/cpu_py35_nonpip.sh | 48 +++++++++++++++ .../rel/ubuntu_cuda11/cpu_py35_pip.sh | 47 ++++++++++++++ .../rel/ubuntu_cuda11/cpu_py36_nonpip.sh | 48 +++++++++++++++ .../rel/ubuntu_cuda11/cpu_py36_pip.sh | 47 ++++++++++++++ .../rel/ubuntu_cuda11/cpu_py37_nonpip.sh | 48 +++++++++++++++ .../rel/ubuntu_cuda11/cpu_py37_pip.sh | 47 ++++++++++++++ .../rel/ubuntu_cuda11/cpu_py38_nonpip.sh | 48 +++++++++++++++ .../rel/ubuntu_cuda11/cpu_py38_pip.sh | 47 ++++++++++++++ .../rel/ubuntu_cuda11/gpu_libtensorflow.sh | 40 ++++++++++++ .../rel/ubuntu_cuda11/gpu_pip_on_cpu.sh | 61 +++++++++++++++++++ .../rel/ubuntu_cuda11/gpu_py35_nonpip.sh | 60 ++++++++++++++++++ .../rel/ubuntu_cuda11/gpu_py35_pip.sh | 55 +++++++++++++++++ .../rel/ubuntu_cuda11/gpu_py36_nonpip.sh | 60 ++++++++++++++++++ .../rel/ubuntu_cuda11/gpu_py36_pip.sh | 55 +++++++++++++++++ .../rel/ubuntu_cuda11/gpu_py37_nonpip.sh | 60 ++++++++++++++++++ .../rel/ubuntu_cuda11/gpu_py37_pip.sh | 55 +++++++++++++++++ .../rel/ubuntu_cuda11/gpu_py38_nonpip.sh | 60 ++++++++++++++++++ .../rel/ubuntu_cuda11/gpu_py38_pip.sh | 55 +++++++++++++++++ .../ci_build/rel/ubuntu_cuda11/sanity.sh | 36 +++++++++++ 20 files changed, 1017 insertions(+) create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_libtensorflow.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py35_nonpip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py35_pip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py36_nonpip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py36_pip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py37_nonpip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py37_pip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py38_nonpip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py38_pip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_libtensorflow.sh create mode 100755 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_pip_on_cpu.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_pip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py36_nonpip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py36_pip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_nonpip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py38_nonpip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py38_pip.sh create mode 100644 tensorflow/tools/ci_build/rel/ubuntu_cuda11/sanity.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_libtensorflow.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_libtensorflow.sh new file mode 100644 index 00000000000..a0e3a7f4594 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_libtensorflow.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e + +# Source the external common scripts. +source tensorflow/tools/ci_build/release/common.sh + + +# Install latest bazel +install_bazelisk +which bazel + +# Install realpath +sudo apt-get install realpath + +# Update the version string to nightly +if [ -n "${IS_NIGHTLY_BUILD}" ]; then + ./tensorflow/tools/ci_build/update_version.py --nightly +fi + +./tensorflow/tools/ci_build/linux/libtensorflow.sh + +# Copy the nightly version update script +if [ -n "${IS_NIGHTLY_BUILD}" ]; then + cp tensorflow/tools/ci_build/builds/libtensorflow_nightly_symlink.sh lib_package +fi + diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py35_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py35_nonpip.sh new file mode 100644 index 00000000000..fee64f0beb1 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py35_nonpip.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.5) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py35,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Run tests +set +e +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py35_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py35_pip.sh new file mode 100644 index 00000000000..bdbb7f15e34 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py35_pip.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.5' + +# Run configure. +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=release_cpu_linux" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py35,-v1only' +#export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo. +export TF_PROJECT_NAME="tensorflow_cpu" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py36_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py36_nonpip.sh new file mode 100644 index 00000000000..6b05141f00f --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py36_nonpip.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.6) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py36,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Run tests +set +e +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py36_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py36_pip.sh new file mode 100644 index 00000000000..6277291043c --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py36_pip.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.6' + +# Run configure. +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=release_cpu_linux" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py36,-v1only' +#export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo. +export TF_PROJECT_NAME="tensorflow_cpu" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py37_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py37_nonpip.sh new file mode 100644 index 00000000000..db0c6056b6c --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py37_nonpip.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.7) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py37,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Run tests +set +e +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py37_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py37_pip.sh new file mode 100644 index 00000000000..ff88ae46f39 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py37_pip.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.7' + +# Run configure. +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=release_cpu_linux" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py37,-v1only' +#export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo. +export TF_PROJECT_NAME="tensorflow_cpu" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py38_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py38_nonpip.sh new file mode 100644 index 00000000000..36da30167d0 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py38_nonpip.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.8 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.8) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py38,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Run tests +set +e +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py38_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py38_pip.sh new file mode 100644 index 00000000000..52872cfd0a6 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/cpu_py38_pip.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.8 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.8' + +# Run configure. +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=release_cpu_linux" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py38,-v1only' +#export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo. +export TF_PROJECT_NAME="tensorflow_cpu" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_libtensorflow.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_libtensorflow.sh new file mode 100644 index 00000000000..d294311d1ff --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_libtensorflow.sh @@ -0,0 +1,40 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e + +# Source the external common scripts. +source tensorflow/tools/ci_build/release/common.sh + + +# Install latest bazel +install_bazelisk +which bazel + +# Install realpath +sudo apt-get install realpath + +export TF_NEED_CUDA=1 + +# Update the version string to nightly +if [ -n "${IS_NIGHTLY_BUILD}" ]; then + ./tensorflow/tools/ci_build/update_version.py --nightly +fi + +./tensorflow/tools/ci_build/linux/libtensorflow.sh + +# Copy the nightly version update script +if [ -n "${IS_NIGHTLY_BUILD}" ]; then + cp tensorflow/tools/ci_build/builds/libtensorflow_nightly_symlink.sh lib_package +fi diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_pip_on_cpu.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_pip_on_cpu.sh new file mode 100755 index 00000000000..6e67bf20730 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_pip_on_cpu.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update Bazel to the desired version +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.6) +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=sm_35,sm_37,sm_52,sm_60,sm_61,compute_70 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +######################## +## Build GPU pip package +######################## +bazel build --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + tensorflow/tools/pip_package:build_pip_package + +# Set TF nightly flag so we get the proper version of estimator +if [[ "$IS_NIGHTLY" == 1 ]]; then + NIGHTLY_FLAG="--nightly_flag" +fi + +PIP_WHL_DIR=whl +mkdir -p ${PIP_WHL_DIR} +PIP_WHL_DIR=$(readlink -f ${PIP_WHL_DIR}) # Get absolute path +bazel-bin/tensorflow/tools/pip_package/build_pip_package "${PIP_WHL_DIR}" "${NIGHTLY_FLAG}" +WHL_PATH=$(ls "${PIP_WHL_DIR}"/*.whl) + +cp "${WHL_PATH}" "$(pwd)"/. +chmod +x tensorflow/tools/ci_build/builds/docker_cpu_pip.sh +docker run -e "BAZEL_VERSION=${BAZEL_VERSION}" -e "CI_BUILD_USER=$(id -u -n)" -e "CI_BUILD_UID=$(id -u)" -e "CI_BUILD_GROUP=$(id -g -n)" -e "CI_BUILD_GID=$(id -g)" -e "CI_BUILD_HOME=/bazel_pip" -v "$(pwd)":/bazel_pip tensorflow/tensorflow:devel "./bazel_pip/tensorflow/tools/ci_build/builds/with_the_same_user" "./bazel_pip/tensorflow/tools/ci_build/builds/docker_cpu_pip.sh" diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh new file mode 100644 index 00000000000..47ed3c4fd2a --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.5) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=sm_35,sm_37,sm_52,sm_60,sm_61,compute_70 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py35" + +set +e +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --test_tag_filters=${tag_filters} \ + --build_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_pip.sh new file mode 100644 index 00000000000..2a5c550890b --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_pip.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.5' + +# Run configure. +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py35' +export TF_BUILD_FLAGS="--config=release_gpu_linux " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION=10 --action_env=TF_CUDNN_VERSION=7 --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +#export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo. +export TF_PROJECT_NAME="tensorflow_gpu" +export TF_PIP_TEST_ROOT="pip_test" + +# To build both tensorflow and tensorflow-gpu pip packages +export TF_BUILD_BOTH_GPU_PACKAGES=1 + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py36_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py36_nonpip.sh new file mode 100644 index 00000000000..70038a8d875 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py36_nonpip.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.6) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=sm_35,sm_37,sm_52,sm_60,sm_61,compute_70 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py36" + +set +e +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --test_tag_filters=${tag_filters} \ + --build_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py36_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py36_pip.sh new file mode 100644 index 00000000000..9aa724c27b9 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py36_pip.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.6' + +# Run configure. +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py36' +export TF_BUILD_FLAGS="--config=release_gpu_linux " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION=10 --action_env=TF_CUDNN_VERSION=7 --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +#export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo. +export TF_PROJECT_NAME=="tensorflow_gpu" +export TF_PIP_TEST_ROOT="pip_test" + +# To build both tensorflow and tensorflow-gpu pip packages +export TF_BUILD_BOTH_GPU_PACKAGES=1 + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_nonpip.sh new file mode 100644 index 00000000000..225b2cf4b7b --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_nonpip.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +install_bazelisk + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.7) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=sm_35,sm_37,sm_52,sm_60,sm_61,compute_70 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py37" + +set +e +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --build_tag_filters=${tag_filters} \ + --test_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh new file mode 100644 index 00000000000..9bfc6608a0b --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +install_bazelisk + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.7' + +# Run configure. +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py37' +export TF_BUILD_FLAGS="--config=release_gpu_linux " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION=10 --action_env=TF_CUDNN_VERSION=7 --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +#export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo. +export TF_PROJECT_NAME=="tensorflow_gpu" +export TF_PIP_TEST_ROOT="pip_test" + +# To build both tensorflow and tensorflow-gpu pip packages +export TF_BUILD_BOTH_GPU_PACKAGES=1 + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py38_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py38_nonpip.sh new file mode 100644 index 00000000000..f7678b7436f --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py38_nonpip.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.8 +# Update bazel +update_bazel_linux + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.8) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=sm_35,sm_37,sm_52,sm_60,sm_61,compute_70 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py38" + +test +e +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --build_tag_filters=${tag_filters} \ + --test_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... +test_xml_summary_exit diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py38_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py38_pip.sh new file mode 100644 index 00000000000..d8838e7704a --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py38_pip.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.8 +# Update bazel +update_bazel_linux + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.8' + +# Run configure. +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py38' +export TF_BUILD_FLAGS="--config=release_gpu_linux " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION=10 --action_env=TF_CUDNN_VERSION=7 --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +#export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo. +export TF_PROJECT_NAME=="tensorflow_gpu" +export TF_PIP_TEST_ROOT="pip_test" + +# To build both tensorflow and tensorflow-gpu pip packages +export TF_BUILD_BOTH_GPU_PACKAGES=1 + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/sanity.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/sanity.sh new file mode 100644 index 00000000000..4fc600de867 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/sanity.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e + +# Install latest bazel +source tensorflow/tools/ci_build/release/common.sh +install_bazelisk +which bazel + +# We need py3 lint +sudo pip3 install pep8 + +# TODO(gunan): figure out why we get stuck with later versions of pylint. +# Install pylint. +sudo python3 -m pip install setuptools --upgrade +sudo python2 -m pip install pylint==1.6.4 +sudo python3 -m pip install pylint==1.6.4 + +# TODO(yifeif): print pylint version for debug. remove later. +python3 -m pylint --version + +# Run tensorflow sanity checks. +tensorflow/tools/ci_build/ci_sanity.sh From a684b992889cc93fb4dc461934de894a4f18268b Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Thu, 6 Aug 2020 16:06:25 -0700 Subject: [PATCH 0596/1017] Removed duplicated fields. PiperOrigin-RevId: 325330615 Change-Id: I9e5af33fb2dce06fff8f187a0a024b0c80b9cc3c --- .../lite/delegates/gpu/cl/kernels/conv_3d.cc | 47 +++++------ .../lite/delegates/gpu/cl/kernels/conv_3d.h | 5 +- .../gpu/cl/kernels/conv_buffer_1x1.cc | 7 +- .../gpu/cl/kernels/conv_buffer_1x1.h | 2 - .../delegates/gpu/cl/kernels/conv_powervr.cc | 82 +++++++++---------- .../delegates/gpu/cl/kernels/conv_powervr.h | 11 ++- 6 files changed, 68 insertions(+), 86 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.cc index b1e1e39327c..727cd488694 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.cc @@ -237,15 +237,12 @@ int3 Conv3D::GetGridSize() const { DivideRoundUp(dst_[0]->Slices(), conv_params_.block_size.w) * DivideRoundUp(dst_[0]->Depth(), conv_params_.block_size.z); int3 wg; - wg.x = DivideRoundUp(grid_x, conv_params_.work_group_size.x); - wg.y = DivideRoundUp(grid_y, conv_params_.work_group_size.y); - wg.z = DivideRoundUp(grid_z, conv_params_.work_group_size.z); - return int3(wg[conv_params_.work_group_launch_order[0]] * - conv_params_.work_group_size.x, - wg[conv_params_.work_group_launch_order[1]] * - conv_params_.work_group_size.y, - wg[conv_params_.work_group_launch_order[2]] * - conv_params_.work_group_size.z); + wg.x = DivideRoundUp(grid_x, work_group_size_.x); + wg.y = DivideRoundUp(grid_y, work_group_size_.y); + wg.z = DivideRoundUp(grid_z, work_group_size_.z); + return int3(wg[conv_params_.work_group_launch_order[0]] * work_group_size_.x, + wg[conv_params_.work_group_launch_order[1]] * work_group_size_.y, + wg[conv_params_.work_group_launch_order[2]] * work_group_size_.z); } absl::Status Conv3D::Tune(const TuningParameters& params) { @@ -259,9 +256,8 @@ absl::Status Conv3D::Tune(const TuningParameters& params) { conv_params_.work_group_launch_order[1] == 1 && conv_params_.work_group_launch_order[2] == 2) { RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - RETURN_IF_ERROR(GetBestWorkGroupConv(params, kernel_, grid_size_, - &conv_params_.work_group_size)); - work_group_size_ = conv_params_.work_group_size; + RETURN_IF_ERROR( + GetBestWorkGroupConv(params, kernel_, grid_size_, &work_group_size_)); } return absl::OkStatus(); } @@ -328,14 +324,13 @@ std::string Conv3D::GenerateConv3D(const OperationDef& op_def, conv_params.weights_upload_type == Conv3D::WeightsUploadType::LOCAL_MEM_ASYNC_SUBGROUP; - const int3 work_group_size = conv_params.work_group_size; const int4 block_size = conv_params.block_size; std::string c = GetCommonDefines(op_def.precision); if (need_local_mem) { // we use fixed workgroup size when use local mem c += "__attribute__((reqd_work_group_size(" + - std::to_string(work_group_size.x) + ", " + - std::to_string(work_group_size.y) + ", " + - std::to_string(work_group_size.z) + ")))\n"; + std::to_string(work_group_size_.x) + ", " + + std::to_string(work_group_size_.y) + ", " + + std::to_string(work_group_size_.z) + ")))\n"; } c += "__kernel void main_function(\n"; c += "$0) {\n"; @@ -348,7 +343,7 @@ std::string Conv3D::GenerateConv3D(const OperationDef& op_def, } if (conv_params.weights_upload_type == Conv3D::WeightsUploadType::LOCAL_MEM_BY_THREADS) { - c += " int lid = get_local_id(1) * " + std::to_string(work_group_size.x) + + c += " int lid = get_local_id(1) * " + std::to_string(work_group_size_.x) + " + get_local_id(0);\n"; } for (int s = 0; s < block_size.w; ++s) { @@ -608,7 +603,7 @@ std::string Conv3D::GenerateConv3D(const OperationDef& op_def, declare_src(); c += " do {\n"; const int total_work_items = - work_group_size.x * work_group_size.y * work_group_size.z; + work_group_size_.x * work_group_size_.y * work_group_size_.z; if (conv_params.weights_upload_type == Conv3D::WeightsUploadType::LOCAL_MEM_ASYNC_SUBGROUP) { c += @@ -731,14 +726,14 @@ Conv3D::ConvParams Conv3D::GuessBestParams(const CLDevice& device, int src_slices, int dst_slices, bool x_kernel_is_1, bool y_kernel_is_1, - bool z_kernel_is_1) const { + bool z_kernel_is_1) { ConvParams conv_params; conv_params.x_kernel_is_1 = x_kernel_is_1; conv_params.y_kernel_is_1 = y_kernel_is_1; conv_params.z_kernel_is_1 = z_kernel_is_1; if (device.IsNvidia()) { conv_params.block_size = int4(1, 1, 1, 4); - conv_params.work_group_size = int3(8, 4, 1); + work_group_size_ = int3(8, 4, 1); conv_params.work_group_launch_order = int3(2, 0, 1); conv_params.src_depth_loop_size = 1; conv_params.weights_upload_type = WeightsUploadType::LOCAL_MEM_BY_THREADS; @@ -757,7 +752,7 @@ Conv3D::ConvParams Conv3D::GuessBestParams(const CLDevice& device, } } else if (device.IsPowerVR()) { conv_params.block_size = int4(1, 1, 1, 4); - conv_params.work_group_size = int3(8, 4, 1); + work_group_size_ = int3(8, 4, 1); conv_params.work_group_launch_order = int3(2, 0, 1); conv_params.src_depth_loop_size = 1; conv_params.weights_upload_type = @@ -791,17 +786,17 @@ Conv3D::ConvParams Conv3D::GuessBestParams(const CLDevice& device, } } conv_params.block_size.x = 2; - conv_params.work_group_size = int3(4, 8, 1); + work_group_size_ = int3(4, 8, 1); } } else if (device.IsAdreno()) { conv_params.block_size = int4(2, 2, 1, 2); - conv_params.work_group_size = int3(8, 4, 1); + work_group_size_ = int3(8, 4, 1); conv_params.work_group_launch_order = int3(0, 1, 2); conv_params.src_depth_loop_size = 1; conv_params.weights_upload_type = WeightsUploadType::TEXTURES_MEM; } else if (device.IsMali()) { conv_params.block_size = int4(1, 1, 1, 4); - conv_params.work_group_size = int3(8, 4, 1); + work_group_size_ = int3(8, 4, 1); conv_params.work_group_launch_order = int3(0, 1, 2); conv_params.src_depth_loop_size = 1; conv_params.weights_upload_type = WeightsUploadType::GLOBAL_MEM; @@ -820,7 +815,7 @@ Conv3D::ConvParams Conv3D::GuessBestParams(const CLDevice& device, } } else { conv_params.block_size = int4(2, 2, 1, 2); - conv_params.work_group_size = int3(8, 4, 1); + work_group_size_ = int3(8, 4, 1); conv_params.work_group_launch_order = int3(0, 1, 2); conv_params.src_depth_loop_size = 1; conv_params.weights_upload_type = WeightsUploadType::TEXTURES_MEM; @@ -831,7 +826,7 @@ Conv3D::ConvParams Conv3D::GuessBestParams(const CLDevice& device, Conv3D::ConvParams Conv3D::GuessBestParams( const CLDevice& device, const OperationDef& definition, - const Convolution3DAttributes& attr) const { + const Convolution3DAttributes& attr) { const int dst_slices = DivideRoundUp(attr.weights.shape.o, 4); const int src_slices = DivideRoundUp(attr.weights.shape.i, 4); const bool x_kernel_is_1 = attr.weights.shape.w == 1 && attr.strides.w == 1 && diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.h b/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.h index ce2d7794411..ffa269d1629 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.h @@ -59,7 +59,6 @@ class Conv3D : public GPUOperation { struct ConvParams { int4 block_size; // WHDS - int3 work_group_size; int3 work_group_launch_order; int src_depth_loop_size; WeightsUploadType weights_upload_type; @@ -98,12 +97,12 @@ class Conv3D : public GPUOperation { ConvParams GuessBestParams(const CLDevice& device, const OperationDef& definition, - const Convolution3DAttributes& attr) const; + const Convolution3DAttributes& attr); ConvParams GuessBestParams(const CLDevice& device, const OperationDef& definition, int src_slices, int dst_slices, bool x_kernel_is_1, - bool y_kernel_is_1, bool z_kernel_is_1) const; + bool y_kernel_is_1, bool z_kernel_is_1); std::string GenerateConv3D(const OperationDef& op_def, bool stride_correction, const Conv3D::ConvParams& conv_params); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc index 949651c1f87..de6021aa5fe 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc @@ -153,7 +153,7 @@ ConvBuffer1x1::ConvBuffer1x1(const OperationDef& definition, const ConvParams& conv_params) : GPUOperation(definition), conv_params_(conv_params) { code_ = GenerateConvBuffer1x1(definition_, conv_params_, &args_); - work_group_size_ = conv_params_.work_group_size; + work_group_size_ = int3(2, 4, 1); } ConvBuffer1x1::ConvBuffer1x1(ConvBuffer1x1&& operation) @@ -317,9 +317,8 @@ int3 ConvBuffer1x1::GetGridSize() const { absl::Status ConvBuffer1x1::Tune(const TuningParameters& params) { RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - RETURN_IF_ERROR(GetBestWorkGroupConv(params, kernel_, grid_size_, - &conv_params_.work_group_size)); - work_group_size_ = conv_params_.work_group_size; + RETURN_IF_ERROR( + GetBestWorkGroupConv(params, kernel_, grid_size_, &work_group_size_)); return absl::OkStatus(); } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.h b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.h index 90df8f2f9ad..94b7cbd1b37 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.h @@ -65,8 +65,6 @@ class ConvBuffer1x1 : public GPUOperation { // some cases we need separate weights for H dimension and convolution // kernel requires very small modifications to support it. bool different_weights_for_height = false; - - int3 work_group_size = int3(2, 4, 1); }; private: diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc index c4e26725f74..f69368d1083 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc @@ -184,7 +184,6 @@ void ConvPowerVR::GenerateCode(const DeviceInfo& device_info) { definition_.IsBatchSupported() && stride_padding_.x != 1; code_ = GenerateConv(device_info, definition_, stride_correction, conv_params_); - work_group_size_ = conv_params_.work_group_size; if (definition_.precision == CalculationsPrecision::F16 && device_info.IsPowerVR()) { compiler_options_.push_back(CompilerOptions::POWERVR_FP16); @@ -225,23 +224,19 @@ int3 ConvPowerVR::GetGridSize() const { int3 wg; if (conv_params_.linear_hw) { - wg.x = DivideRoundUp(grid_x * grid_y, conv_params_.work_group_size.x); - wg.y = DivideRoundUp(grid_z, conv_params_.work_group_size.y); - return int3(wg[conv_params_.work_group_launch_order[0]] * - conv_params_.work_group_size.x, - wg[conv_params_.work_group_launch_order[1]] * - conv_params_.work_group_size.y, - 1); + wg.x = DivideRoundUp(grid_x * grid_y, work_group_size_.x); + wg.y = DivideRoundUp(grid_z, work_group_size_.y); + return int3( + wg[conv_params_.work_group_launch_order[0]] * work_group_size_.x, + wg[conv_params_.work_group_launch_order[1]] * work_group_size_.y, 1); } else { - wg.x = DivideRoundUp(grid_x, conv_params_.work_group_size.x); - wg.y = DivideRoundUp(grid_y, conv_params_.work_group_size.y); - wg.z = DivideRoundUp(grid_z, conv_params_.work_group_size.z); - return int3(wg[conv_params_.work_group_launch_order[0]] * - conv_params_.work_group_size.x, - wg[conv_params_.work_group_launch_order[1]] * - conv_params_.work_group_size.y, - wg[conv_params_.work_group_launch_order[2]] * - conv_params_.work_group_size.z); + wg.x = DivideRoundUp(grid_x, work_group_size_.x); + wg.y = DivideRoundUp(grid_y, work_group_size_.y); + wg.z = DivideRoundUp(grid_z, work_group_size_.z); + return int3( + wg[conv_params_.work_group_launch_order[0]] * work_group_size_.x, + wg[conv_params_.work_group_launch_order[1]] * work_group_size_.y, + wg[conv_params_.work_group_launch_order[2]] * work_group_size_.z); } } @@ -257,9 +252,8 @@ absl::Status ConvPowerVR::Tune(const TuningParameters& params) { conv_params_.work_group_launch_order[1] == 1 && conv_params_.work_group_launch_order[2] == 2) { RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - RETURN_IF_ERROR(GetBestWorkGroupConv(params, kernel_, grid_size_, - &conv_params_.work_group_size)); - work_group_size_ = conv_params_.work_group_size; + RETURN_IF_ERROR( + GetBestWorkGroupConv(params, kernel_, grid_size_, &work_group_size_)); } return absl::OkStatus(); } @@ -345,14 +339,12 @@ std::string ConvPowerVR::GenerateConv(const DeviceInfo& device_info, c += "#pragma OPENCL EXTENSION cl_khr_subgroups : enable\n"; } } - - const int3 work_group_size = conv_params.work_group_size; const int3 block_size = conv_params.block_size; if (conv_params.fixed_work_group_size) { c += "__attribute__((reqd_work_group_size(" + - std::to_string(work_group_size.x) + ", " + - std::to_string(work_group_size.y) + ", " + - std::to_string(work_group_size.z) + ")))\n"; + std::to_string(work_group_size_.x) + ", " + + std::to_string(work_group_size_.y) + ", " + + std::to_string(work_group_size_.z) + ")))\n"; } if (use_simd_broadcast && device_info.IsIntel()) { c += "__attribute__((intel_reqd_sub_group_size(" + @@ -383,7 +375,7 @@ std::string ConvPowerVR::GenerateConv(const DeviceInfo& device_info, c += " int lid = get_local_id(0);\n"; } else { c += " int lid = get_local_id(1) * " + - std::to_string(work_group_size.x) + " + get_local_id(0);\n"; + std::to_string(work_group_size_.x) + " + get_local_id(0);\n"; } } if (use_simd_broadcast) { @@ -590,7 +582,7 @@ std::string ConvPowerVR::GenerateConv(const DeviceInfo& device_info, c += " do {\n"; declare_src(); const int total_work_items = - work_group_size.x * work_group_size.y * work_group_size.z; + work_group_size_.x * work_group_size_.y * work_group_size_.z; if (conv_params.weights_upload_type == ConvPowerVR::WeightsUploadType::LOCAL_MEM_ASYNC_SUBGROUP) { c += GenerateAsyncUpload("weights_cache", "filters_loc", @@ -694,7 +686,7 @@ std::string ConvPowerVR::GenerateConv(const DeviceInfo& device_info, ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( const CLDevice& device, const OperationDef& definition, int src_depth, int dst_depth, bool x_kernel_is_1, bool y_kernel_is_1, - bool different_weights_for_height, const BHWC* dst_shape) const { + bool different_weights_for_height, const BHWC* dst_shape) { ConvParams conv_params; conv_params.linear_hw = false; conv_params.weights_data_type = @@ -704,12 +696,12 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( conv_params.different_weights_for_height = different_weights_for_height; if (device.IsNvidia()) { if (different_weights_for_height) { - conv_params.work_group_size = int3(32, 1, 1); + work_group_size_ = int3(32, 1, 1); conv_params.work_group_launch_order = int3(2, 0, 1); conv_params.fixed_work_group_size = true; } else { conv_params.linear_hw = true; - conv_params.work_group_size = int3(32, 1, 1); + work_group_size_ = int3(32, 1, 1); conv_params.work_group_launch_order = int3(1, 0, 2); conv_params.fixed_work_group_size = true; } @@ -749,12 +741,12 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( } } else if (device.IsPowerVR()) { if (different_weights_for_height) { - conv_params.work_group_size = int3(32, 1, 1); + work_group_size_ = int3(32, 1, 1); conv_params.work_group_launch_order = int3(2, 0, 1); conv_params.fixed_work_group_size = true; } else { conv_params.linear_hw = true; - conv_params.work_group_size = int3(32, 1, 1); + work_group_size_ = int3(32, 1, 1); conv_params.work_group_launch_order = int3(1, 0, 2); conv_params.fixed_work_group_size = true; } @@ -797,11 +789,11 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( } } else if (device.IsAMD()) { if (different_weights_for_height) { - conv_params.work_group_size = int3(32, 1, 1); + work_group_size_ = int3(32, 1, 1); conv_params.work_group_launch_order = int3(2, 0, 1); conv_params.fixed_work_group_size = true; } else { - conv_params.work_group_size = int3(8, 4, 1); + work_group_size_ = int3(8, 4, 1); conv_params.work_group_launch_order = int3(2, 0, 1); conv_params.fixed_work_group_size = true; } @@ -860,25 +852,25 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( definition.precision == CalculationsPrecision::F16) { conv_params.src_depth_loop_size = 4; } - conv_params.work_group_size = int3(4, 4, 1); + work_group_size_ = int3(4, 4, 1); conv_params.work_group_launch_order = int3(0, 1, 2); conv_params.fixed_work_group_size = false; conv_params.weights_upload_type = WeightsUploadType::GLOBAL_MEM; } else if (device.IsAdreno()) { conv_params.block_size = int3(2, 2, 1); - conv_params.work_group_size = int3(8, 2, 1); + work_group_size_ = int3(8, 2, 1); conv_params.work_group_launch_order = int3(0, 1, 2); conv_params.fixed_work_group_size = false; conv_params.src_depth_loop_size = 1; conv_params.weights_upload_type = WeightsUploadType::GLOBAL_MEM; } else if (device.IsIntel()) { if (different_weights_for_height) { - conv_params.work_group_size = int3(16, 1, 1); + work_group_size_ = int3(16, 1, 1); conv_params.work_group_launch_order = int3(0, 1, 2); conv_params.fixed_work_group_size = true; } else { conv_params.linear_hw = true; - conv_params.work_group_size = int3(16, 1, 1); + work_group_size_ = int3(16, 1, 1); conv_params.work_group_launch_order = int3(0, 1, 2); conv_params.fixed_work_group_size = true; } @@ -908,7 +900,7 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( } } else { conv_params.block_size = int3(1, 1, 4); - conv_params.work_group_size = int3(8, 2, 1); + work_group_size_ = int3(8, 2, 1); conv_params.work_group_launch_order = int3(0, 1, 2); conv_params.fixed_work_group_size = false; conv_params.src_depth_loop_size = 1; @@ -933,7 +925,7 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( const CLDevice& device, const OperationDef& definition, - const Convolution2DAttributes& attr, const BHWC* dst_shape) const { + const Convolution2DAttributes& attr, const BHWC* dst_shape) { const int dst_depth = DivideRoundUp(attr.weights.shape.o, 4); const int src_depth = DivideRoundUp(attr.weights.shape.i, 4); const bool x_kernel_is_1 = attr.weights.shape.w == 1 && attr.strides.w == 1 && @@ -951,7 +943,7 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( const CLDevice& device, const OperationDef& definition, const Convolution2DAttributes& attr, const BHWC& weights_shape, - const BHWC* dst_shape) const { + const BHWC* dst_shape) { const int dst_depth = DivideRoundUp(weights_shape.b, 4); const int src_depth = DivideRoundUp(weights_shape.c, 4); const bool x_kernel_is_1 = @@ -966,13 +958,13 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( const CLDevice& device, const OperationDef& definition, - const FullyConnectedAttributes& attr, const BHWC* dst_shape) const { + const FullyConnectedAttributes& attr, const BHWC* dst_shape) { const int dst_depth = DivideRoundUp(attr.weights.shape.o, 4); const int src_depth = DivideRoundUp(attr.weights.shape.i, 4); ConvPowerVR::ConvParams params = GuessBestParams( device, definition, src_depth, dst_depth, true, true, false, dst_shape); - params.work_group_size.x *= params.work_group_size.y; - params.work_group_size.y = 1; + work_group_size_.x *= work_group_size_.y; + work_group_size_.y = 1; params.block_size.x *= params.block_size.y; params.block_size.y = 1; return params; @@ -980,7 +972,7 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( ConvPowerVR::ConvParams ConvPowerVR::GuessBestParamsWinograd( const CLDevice& device, const OperationDef& definition, - const Convolution2DAttributes& attr, const BHWC* dst_shape) const { + const Convolution2DAttributes& attr, const BHWC* dst_shape) { const int dst_depth = DivideRoundUp(attr.weights.shape.o, 4); const int src_depth = DivideRoundUp(attr.weights.shape.i, 4); ConvPowerVR::ConvParams params = GuessBestParams( diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.h b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.h index 148dad38708..e61d4c14ce7 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.h @@ -81,7 +81,6 @@ class ConvPowerVR : public GPUOperation { // F32_F16 precision mode DataType weights_data_type; // used for weights and biases int3 block_size; - int3 work_group_size; int3 work_group_launch_order; bool fixed_work_group_size; bool linear_hw; @@ -180,26 +179,26 @@ class ConvPowerVR : public GPUOperation { ConvParams GuessBestParams(const CLDevice& device, const OperationDef& definition, const Convolution2DAttributes& attr, - const BHWC* dst_shape = nullptr) const; + const BHWC* dst_shape = nullptr); ConvParams GuessBestParams(const CLDevice& device, const OperationDef& definition, const Convolution2DAttributes& attr, const BHWC& weights_shape, - const BHWC* dst_shape = nullptr) const; + const BHWC* dst_shape = nullptr); ConvParams GuessBestParams(const CLDevice& device, const OperationDef& definition, const FullyConnectedAttributes& attr, - const BHWC* dst_shape = nullptr) const; + const BHWC* dst_shape = nullptr); ConvParams GuessBestParamsWinograd(const CLDevice& device, const OperationDef& definition, const Convolution2DAttributes& attr, - const BHWC* dst_shape = nullptr) const; + const BHWC* dst_shape = nullptr); ConvParams GuessBestParams(const CLDevice& device, const OperationDef& definition, int src_depth, int dst_depth, bool x_kernel_is_1, bool y_kernel_is_1, bool different_weights_for_height, - const BHWC* dst_shape = nullptr) const; + const BHWC* dst_shape = nullptr); std::string GenerateConv(const DeviceInfo& device_info, const OperationDef& op_def, bool stride_correction, From eaaccbe0dd81533e17ec5975553e70aedf48d302 Mon Sep 17 00:00:00 2001 From: Anthony Platanios Date: Thu, 6 Aug 2020 16:15:47 -0700 Subject: [PATCH 0597/1017] Fix. --- tensorflow/BUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 484e45eb11d..6745b3e54fa 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -882,7 +882,7 @@ genrule( visibility = ["//visibility:public"], ) -# The interface library (tensorflow_framework.dll.if.lib) for linking tensorflow DLL +# The interface library (tensorflow_framework.dll.if.lib) for linking tensorflow DLL # library (tensorflow_framework.dll) on Windows. # To learn more about import library (called interface library in Bazel): # https://docs.microsoft.com/en-us/cpp/build/linking-an-executable-to-a-dll?view=vs-2017#linking-implicitly @@ -893,7 +893,7 @@ filegroup( visibility = ["//visibility:public"], ) -# Rename the import library for tensorflow_framework.dll from +# Rename the import library for tensorflow_framework.dll from # tensorflow_framework.dll.if.lib to tensorflow_framework.lib genrule( name = "tensorflow_framework_dll_import_lib", From dc76cd3e09ef7bc1b3d3f3f1108df3f5cd22276c Mon Sep 17 00:00:00 2001 From: Katherine Wu Date: Thu, 6 Aug 2020 16:11:23 -0700 Subject: [PATCH 0598/1017] Add structured input signature to functions loaded from V1 SavedModel. PiperOrigin-RevId: 325331549 Change-Id: I030c5ce8a54372a0f5350a22b981721c787615d5 --- tensorflow/python/saved_model/BUILD | 1 + tensorflow/python/saved_model/load_v1_in_v2.py | 9 +++++++++ tensorflow/python/saved_model/load_v1_in_v2_test.py | 10 ++++++++++ 3 files changed, 20 insertions(+) diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD index 27e0e984f5f..45ee73de51c 100644 --- a/tensorflow/python/saved_model/BUILD +++ b/tensorflow/python/saved_model/BUILD @@ -445,6 +445,7 @@ py_strict_library( "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", + "//tensorflow/python:func_graph", "//tensorflow/python:platform", "//tensorflow/python:saver", "//tensorflow/python:sparse_tensor", diff --git a/tensorflow/python/saved_model/load_v1_in_v2.py b/tensorflow/python/saved_model/load_v1_in_v2.py index ede91da168c..add3b4e6320 100644 --- a/tensorflow/python/saved_model/load_v1_in_v2.py +++ b/tensorflow/python/saved_model/load_v1_in_v2.py @@ -25,6 +25,7 @@ from tensorflow.python.eager import lift_to_graph from tensorflow.python.eager import wrap_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes +from tensorflow.python.framework import func_graph from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops @@ -143,6 +144,7 @@ class _EagerSavedModelLoader(loader_impl.SavedModelLoader): for input_spec in input_specs ] input_names = [] + input_tensors = [] for original_input_name, feed in zip(original_input_names, feeds): if isinstance(feed, sparse_tensor.SparseTensor): # We have to give explicit name for SparseTensor arguments, because @@ -151,8 +153,10 @@ class _EagerSavedModelLoader(loader_impl.SavedModelLoader): values_name = "%s_values" % original_input_name dense_shape_name = "%s_dense_shape" % original_input_name input_names.extend([indices_name, values_name, dense_shape_name]) + input_tensors.extend([feed.indices, feed.values, feed.dense_shape]) else: input_names.append(original_input_name) + input_tensors.append(feed) fetches = {name: out for name, out in signature_def.outputs.items()} try: signature_fn = wrapped.prune(feeds=feeds, fetches=fetches) @@ -173,6 +177,11 @@ class _EagerSavedModelLoader(loader_impl.SavedModelLoader): raise # pylint: disable=protected-access signature_fn._arg_keywords = input_names + signature_fn._func_graph.structured_input_signature = ( + (), + func_graph.convert_structure_to_signature( + dict(zip(input_names, input_tensors)))) + if len(input_names) == 1: # Allowing positional arguments does not create any ambiguity if there's # only one. diff --git a/tensorflow/python/saved_model/load_v1_in_v2_test.py b/tensorflow/python/saved_model/load_v1_in_v2_test.py index bafeea128ed..806a4db6fba 100644 --- a/tensorflow/python/saved_model/load_v1_in_v2_test.py +++ b/tensorflow/python/saved_model/load_v1_in_v2_test.py @@ -32,6 +32,7 @@ from tensorflow.python.framework import function as framework_function from tensorflow.python.framework import ops from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import test_util from tensorflow.python.framework import versions from tensorflow.python.lib.io import file_io @@ -630,6 +631,15 @@ class LoadTest(test.TestCase): imported.signatures["serving_default"](constant_op.constant(2.))), {"y": [10, 8, 6, 4, 2, 0]}) + def test_structured_input_signature(self): + path = self._v1_single_metagraph_saved_model(False) + imported = load.load(path) + args, kwargs = ( + imported.signatures["serving_default"].structured_input_signature) + self.assertEqual(args, ()) + self.assertAllEqual( + kwargs, {"start": tensor_spec.TensorSpec(shape=None, name="start")}) + if __name__ == "__main__": test.main() From 0c3334857dd912a6b7a31c85b056721afe6af324 Mon Sep 17 00:00:00 2001 From: Advait Jain Date: Thu, 6 Aug 2020 16:15:44 -0700 Subject: [PATCH 0599/1017] Use selects.with_or to avoid duplication of deps. PiperOrigin-RevId: 325332323 Change-Id: I3b0b35abbde47adc1d2fcbdea2e21e7e608cfa43 --- tensorflow/lite/kernels/internal/BUILD | 91 ++++++++------------------ 1 file changed, 29 insertions(+), 62 deletions(-) diff --git a/tensorflow/lite/kernels/internal/BUILD b/tensorflow/lite/kernels/internal/BUILD index 2707871df16..ad11c06eb37 100644 --- a/tensorflow/lite/kernels/internal/BUILD +++ b/tensorflow/lite/kernels/internal/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//lib:selects.bzl", "selects") load("//tensorflow:tensorflow.bzl", "transitive_hdrs") load("//tensorflow/lite:build_def.bzl", "tflite_copts") load("//tensorflow/lite/micro:build_def.bzl", "micro_copts") @@ -735,70 +736,36 @@ cc_library( ":cpu_check", "//third_party/eigen3", "//tensorflow/lite/c:common", - ] + select({ - ":aarch64": [ - ":neon_tensor_utils", - ], - ":arm": [ - ":neon_tensor_utils", - ], - ":arm64-v8a": [ - ":neon_tensor_utils", - ], - ":armeabi-v7a": [ - ":neon_tensor_utils", - ], - ":armhf": [ - ":neon_tensor_utils", - ], - ":armv7a": [ - ":neon_tensor_utils", - ], - ":haswell": [ + ] + selects.with_or({ + ( + ":aarch64", + ":arm", + ":arm64-v8a", + ":armeabi-v7a", + ":armhf", + ":armv7a", + ":ios_armv7", + ":ios_arm64", + ":ios_arm64e", + ":raspberry_pi_with_neon", + ): [":neon_tensor_utils"], + ( + ":darwin", + ":darwin_x86_64", + ":freebsd", + ":haswell", + ":ios_x86_64", + ":x86_64", + ":x86", + ":k8", + ":windows", + ): [ ":sse_tensor_utils", ], - ":ios_armv7": [ - ":neon_tensor_utils", - ], - ":ios_arm64": [ - ":neon_tensor_utils", - ], - ":ios_arm64e": [ - ":neon_tensor_utils", - ], - ":raspberry_pi_with_neon": [ - ":neon_tensor_utils", - ], - ":ios_x86_64": [ - ":sse_tensor_utils", - ], - ":x86_64": [ - ":sse_tensor_utils", - ], - ":x86": [ - ":sse_tensor_utils", - ], - ":k8": [ - ":sse_tensor_utils", - ], - ":darwin": [ - ":sse_tensor_utils", - ], - ":darwin_x86_64": [ - ":sse_tensor_utils", - ], - ":freebsd": [ - ":sse_tensor_utils", - ], - ":windows": [ - ":sse_tensor_utils", - ], - ":tf_lite_static_memory": [ - ":portable_tensor_utils", - ], - "//conditions:default": [ - ":portable_tensor_utils", - ], + ( + ":tf_lite_static_memory", + "//conditions:default", + ): [":portable_tensor_utils"], }), ) From 0a0a9eeb6bf42ae062ebafa68e4d87ae0c62d7e5 Mon Sep 17 00:00:00 2001 From: Robert David Date: Thu, 6 Aug 2020 16:16:29 -0700 Subject: [PATCH 0600/1017] Change unordered containers to Swiss table. PiperOrigin-RevId: 325332451 Change-Id: I5349d9b9e9227b62752f21e0b2c777bfcc59d3eb --- tensorflow/lite/delegates/gpu/BUILD | 1 + tensorflow/lite/delegates/gpu/cl/BUILD | 3 +++ .../lite/delegates/gpu/cl/inference_context.cc | 9 +++++---- .../lite/delegates/gpu/cl/inference_context.h | 4 ++-- tensorflow/lite/delegates/gpu/cl/program_cache.h | 6 +++--- tensorflow/lite/delegates/gpu/common/BUILD | 5 +++++ .../lite/delegates/gpu/common/model_builder.cc | 12 ++++++------ .../lite/delegates/gpu/common/model_builder.h | 6 +++--- .../delegates/gpu/common/model_transformer.h | 4 ++-- .../lite/delegates/gpu/common/object_reader.cc | 6 +++--- .../lite/delegates/gpu/common/object_reader.h | 14 +++++++------- .../lite/delegates/gpu/common/operations.cc | 4 ++-- .../delegates/gpu/common/quantization_util.cc | 16 +++++++++------- .../delegates/gpu/common/quantization_util.h | 10 +++++----- .../gpu/common/quantization_util_test.cc | 8 ++++---- tensorflow/lite/delegates/gpu/delegate.cc | 4 ++-- tensorflow/lite/delegates/gpu/gl/BUILD | 3 +++ tensorflow/lite/delegates/gpu/gl/api.cc | 12 ++++++------ tensorflow/lite/delegates/gpu/gl/api.h | 2 +- tensorflow/lite/delegates/gpu/gl/api2.cc | 4 ++-- tensorflow/lite/delegates/gpu/gl/compiler.cc | 10 ++++++---- tensorflow/lite/delegates/gpu/gl/compiler.h | 7 ++++--- tensorflow/lite/delegates/gpu/gl/compiler/BUILD | 5 +++++ .../delegates/gpu/gl/compiler/compiled_node.cc | 5 ++--- .../delegates/gpu/gl/compiler/fuse_auto_input.cc | 4 ++-- .../delegates/gpu/gl/compiler/object_accessor.h | 4 ++-- .../lite/delegates/gpu/gl/compiler/rename.cc | 6 +++--- .../gpu/gl/compiler/variable_accessor.h | 6 +++--- tensorflow/lite/delegates/gpu/gl/kernels/BUILD | 6 +++++- .../delegates/gpu/gl/kernels/custom_registry.cc | 5 +++-- .../delegates/gpu/gl/kernels/custom_registry.h | 4 ++-- .../lite/delegates/gpu/gl/kernels/registry.cc | 4 ++-- .../lite/delegates/gpu/gl/kernels/test_util.cc | 12 ++++++------ tensorflow/lite/delegates/gpu/gl/runtime.cc | 1 - .../gl/workgroups/calculator_from_metadata.cc | 4 ++-- tensorflow/lite/delegates/gpu/gl_delegate.cc | 2 +- .../delegates/gpu/metal/kernels/elementwise.cc | 6 +++--- tensorflow/lite/delegates/gpu/metal_delegate.mm | 3 ++- 38 files changed, 127 insertions(+), 100 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/BUILD b/tensorflow/lite/delegates/gpu/BUILD index 4113d34b3f8..d69bed4c03a 100644 --- a/tensorflow/lite/delegates/gpu/BUILD +++ b/tensorflow/lite/delegates/gpu/BUILD @@ -251,6 +251,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/gl:api2", ], }) + [ + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", "@com_google_absl//absl/types:span", "//tensorflow/lite:kernel_api", diff --git a/tensorflow/lite/delegates/gpu/cl/BUILD b/tensorflow/lite/delegates/gpu/cl/BUILD index ebfb2cff41b..66bcbc826ea 100644 --- a/tensorflow/lite/delegates/gpu/cl/BUILD +++ b/tensorflow/lite/delegates/gpu/cl/BUILD @@ -388,6 +388,8 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:util", "//tensorflow/lite/delegates/gpu/common/transformations:add_bias", "//tensorflow/lite/delegates/gpu/common/transformations:merge_padding_with", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", ], ) @@ -454,6 +456,7 @@ cc_library( ":compiled_program_cache_cc_fbs", ":util", "//tensorflow/lite/delegates/gpu/common:status", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/types:span", "@farmhash_archive//:farmhash", "@flatbuffers", diff --git a/tensorflow/lite/delegates/gpu/cl/inference_context.cc b/tensorflow/lite/delegates/gpu/cl/inference_context.cc index 8e23eb1bcee..689b511bb5e 100644 --- a/tensorflow/lite/delegates/gpu/cl/inference_context.cc +++ b/tensorflow/lite/delegates/gpu/cl/inference_context.cc @@ -21,9 +21,10 @@ limitations under the License. #include #include #include -#include #include +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "tensorflow/lite/delegates/gpu/cl/buffer.h" #include "tensorflow/lite/delegates/gpu/cl/cl_device.h" #include "tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h" @@ -49,7 +50,7 @@ namespace gpu { namespace cl { namespace { -bool IsReady(const std::unordered_set& ready_tensors, +bool IsReady(const absl::flat_hash_set& ready_tensors, const CLNode& node) { for (const ValueId in_id : node.inputs) { if (ready_tensors.find(in_id) == ready_tensors.end()) { @@ -325,7 +326,7 @@ absl::Status InferenceContext::ConvertOperations( inputs, outputs, node, &gpu_subgraph)); } - std::unordered_map mapping_to_global_ids; + absl::flat_hash_map mapping_to_global_ids; for (int j = 0; j < gpu_subgraph.new_tensors.size(); ++j) { const auto& t = gpu_subgraph.new_tensors[j]; auto global_id = tensor_reserver_.Add({t.first, t.second}); @@ -364,7 +365,7 @@ absl::Status InferenceContext::ConvertOperations( } void InferenceContext::Merge() { - std::unordered_set ready_tensors; + absl::flat_hash_set ready_tensors; for (const auto& input_id : input_ids_) { ready_tensors.insert(input_id); } diff --git a/tensorflow/lite/delegates/gpu/cl/inference_context.h b/tensorflow/lite/delegates/gpu/cl/inference_context.h index 3f05026b795..e26cb170228 100644 --- a/tensorflow/lite/delegates/gpu/cl/inference_context.h +++ b/tensorflow/lite/delegates/gpu/cl/inference_context.h @@ -20,9 +20,9 @@ limitations under the License. #include #include #include -#include #include +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/delegates/gpu/cl/buffer.h" #include "tensorflow/lite/delegates/gpu/cl/cl_command_queue.h" #include "tensorflow/lite/delegates/gpu/cl/environment.h" @@ -160,7 +160,7 @@ class InferenceContext { DummyTensor Get(ValueId id) { return reservations_[id]; } private: - std::unordered_map reservations_; + absl::flat_hash_map reservations_; ValueId next_; }; TensorReserver tensor_reserver_; diff --git a/tensorflow/lite/delegates/gpu/cl/program_cache.h b/tensorflow/lite/delegates/gpu/cl/program_cache.h index 21f9583a59a..81649d677f7 100644 --- a/tensorflow/lite/delegates/gpu/cl/program_cache.h +++ b/tensorflow/lite/delegates/gpu/cl/program_cache.h @@ -18,9 +18,9 @@ limitations under the License. #include #include -#include #include +#include "absl/container/flat_hash_map.h" #include "absl/types/span.h" #include "tensorflow/lite/delegates/gpu/cl/cl_context.h" #include "tensorflow/lite/delegates/gpu/cl/cl_device.h" @@ -93,8 +93,8 @@ class ProgramCache { // There is a low probability of a hash collision when cache is deserialized // because only fingerprints are serialized instead of full source code. bool use_fingerprints_ = false; - std::unordered_map + absl::flat_hash_map programs_; }; diff --git a/tensorflow/lite/delegates/gpu/common/BUILD b/tensorflow/lite/delegates/gpu/common/BUILD index ab2d5d033f7..3caee09ca7e 100644 --- a/tensorflow/lite/delegates/gpu/common/BUILD +++ b/tensorflow/lite/delegates/gpu/common/BUILD @@ -114,6 +114,7 @@ cc_library( ":shape", ":status", ":tensor", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", "//tensorflow/lite/delegates:utils", "//tensorflow/lite:context", @@ -169,6 +170,7 @@ cc_library( hdrs = ["model_transformer.h"], deps = [ ":model", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings", ], ) @@ -186,6 +188,7 @@ cc_library( "//tensorflow/lite/c:common", "//tensorflow/lite/delegates:utils", "//tensorflow/lite/kernels:kernel_util", + "@com_google_absl//absl/container:flat_hash_map", ], ) @@ -198,6 +201,7 @@ cc_library( ":model", ":shape", ":status", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/types:variant", ], ) @@ -212,6 +216,7 @@ cc_library( "//tensorflow/lite/c:common", "//tensorflow/lite/kernels/internal:optimized_base", "//tensorflow/lite/kernels/internal:types", + "@com_google_absl//absl/container:flat_hash_map", ], ) diff --git a/tensorflow/lite/delegates/gpu/common/model_builder.cc b/tensorflow/lite/delegates/gpu/common/model_builder.cc index 4c0fd827834..84622cdc294 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder.cc +++ b/tensorflow/lite/delegates/gpu/common/model_builder.cc @@ -22,10 +22,10 @@ limitations under the License. #include #include #include -#include #include #include +#include "absl/container/flat_hash_map.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" @@ -2884,8 +2884,8 @@ TfLiteIntArray* GetOpsToReplace(TfLiteContext* context, bool allow_quant_ops, // guarantee that the order will match the source model tensors order. absl::Status PrecreateIOTensors( TfLiteContext* context, GraphFloat32* graph, TfLiteIntArray* io_tensors, - std::unordered_map* quant_conversion_map, - std::unordered_map* tensor_to_value) { + absl::flat_hash_map* quant_conversion_map, + absl::flat_hash_map* tensor_to_value) { for (int i = 0; i < io_tensors->size; ++i) { const int tensor_index = io_tensors->data[i]; const TfLiteTensor& tflite_tensor = context->tensors[tensor_index]; @@ -2899,7 +2899,7 @@ absl::Status PrecreateIOTensors( absl::Status BuildModel(TfLiteContext* context, const TfLiteDelegateParams* delegate_params, GraphFloat32* graph, - std::unordered_map* quant_conversion_map) { + absl::flat_hash_map* quant_conversion_map) { std::vector> operations; std::vector tflite_nodes; for (int i = 0; i < delegate_params->nodes_to_replace->size; ++i) { @@ -2925,7 +2925,7 @@ absl::Status BuildModel(TfLiteContext* context, operations.push_back(std::move(op_parser)); tflite_nodes.push_back(i); } - std::unordered_map tensor_to_value; + absl::flat_hash_map tensor_to_value; RETURN_IF_ERROR(PrecreateIOTensors(context, graph, delegate_params->input_tensors, quant_conversion_map, &tensor_to_value)); @@ -2952,7 +2952,7 @@ absl::Status BuildModel(TfLiteContext* context, absl::Status BuildFinalModel( TfLiteContext* context, const TfLiteDelegateParams* delegate_params, - GraphFloat32* graph, std::unordered_map* quant_conversion_map) { + GraphFloat32* graph, absl::flat_hash_map* quant_conversion_map) { RETURN_IF_ERROR( BuildModel(context, delegate_params, graph, quant_conversion_map)); diff --git a/tensorflow/lite/delegates/gpu/common/model_builder.h b/tensorflow/lite/delegates/gpu/common/model_builder.h index 1e5016d86b6..9d80e9636f0 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder.h +++ b/tensorflow/lite/delegates/gpu/common/model_builder.h @@ -18,8 +18,8 @@ limitations under the License. #include #include -#include +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/context.h" #include "tensorflow/lite/delegates/gpu/common/model.h" #include "tensorflow/lite/delegates/gpu/common/status.h" @@ -48,7 +48,7 @@ TfLiteIntArray* GetOpsToReplace(TfLiteContext* context, absl::Status BuildModel( TfLiteContext* context, const TfLiteDelegateParams* delegate_params, GraphFloat32* graph, - std::unordered_map* quant_conversion_map = nullptr); + absl::flat_hash_map* quant_conversion_map = nullptr); // Same as above but also apply all transformations on the final graph. // Prefer using this method instead of BuildModel. @@ -62,7 +62,7 @@ absl::Status BuildModel( absl::Status BuildFinalModel( TfLiteContext* context, const TfLiteDelegateParams* delegate_params, GraphFloat32* graph, - std::unordered_map* quant_conversion_map = nullptr); + absl::flat_hash_map* quant_conversion_map = nullptr); // Module-internal converter, exposed for unit testing purpose only. absl::Status ConvertTfLiteTensorToTensorRef(const TfLiteTensor& tflite_tensor, diff --git a/tensorflow/lite/delegates/gpu/common/model_transformer.h b/tensorflow/lite/delegates/gpu/common/model_transformer.h index d82a6a687ca..fd2667390f3 100644 --- a/tensorflow/lite/delegates/gpu/common/model_transformer.h +++ b/tensorflow/lite/delegates/gpu/common/model_transformer.h @@ -18,9 +18,9 @@ limitations under the License. #include #include -#include #include +#include "absl/container/flat_hash_set.h" #include "tensorflow/lite/delegates/gpu/common/model.h" namespace tflite { @@ -126,7 +126,7 @@ class ModelTransformer { TransformationReporter* reporter_; std::deque to_process_; - std::unordered_set processed_; + absl::flat_hash_set processed_; }; class NullTransformationReporter : public TransformationReporter { diff --git a/tensorflow/lite/delegates/gpu/common/object_reader.cc b/tensorflow/lite/delegates/gpu/common/object_reader.cc index 41f3ef8ff19..c837fa061c0 100644 --- a/tensorflow/lite/delegates/gpu/common/object_reader.cc +++ b/tensorflow/lite/delegates/gpu/common/object_reader.cc @@ -16,8 +16,8 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/object_reader.h" #include -#include +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/delegates/gpu/common/model.h" #include "tensorflow/lite/delegates/gpu/common/model_builder_helper.h" @@ -28,8 +28,8 @@ namespace tflite { namespace gpu { absl::Status ObjectReader::ReadNonConstantTensor( - TfLiteContext* context, std::unordered_map* tensor_to_value, - std::unordered_map* quant_conversion_map, GraphFloat32* graph, + TfLiteContext* context, absl::flat_hash_map* tensor_to_value, + absl::flat_hash_map* quant_conversion_map, GraphFloat32* graph, uint32_t tensor_idx, Value** value) { if (tensor_idx >= context->tensors_size) { return absl::OutOfRangeError( diff --git a/tensorflow/lite/delegates/gpu/common/object_reader.h b/tensorflow/lite/delegates/gpu/common/object_reader.h index be9a89e1b4e..246bc71f9c5 100644 --- a/tensorflow/lite/delegates/gpu/common/object_reader.h +++ b/tensorflow/lite/delegates/gpu/common/object_reader.h @@ -17,8 +17,8 @@ limitations under the License. #define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_OBJECT_READER_H_ #include -#include +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/delegates/gpu/common/model.h" #include "tensorflow/lite/delegates/gpu/common/model_builder_helper.h" @@ -34,14 +34,14 @@ namespace gpu { class ObjectReader { public: static absl::Status ReadNonConstantTensor( - TfLiteContext* context, std::unordered_map* tensor_to_value, - std::unordered_map* quant_conversion_map, GraphFloat32* graph, + TfLiteContext* context, absl::flat_hash_map* tensor_to_value, + absl::flat_hash_map* quant_conversion_map, GraphFloat32* graph, uint32_t tensor_idx, Value** value = nullptr); ObjectReader(GraphFloat32* graph, TfLiteContext* context, const TfLiteNode* node, - std::unordered_map* tensor_to_value, - std::unordered_map* quant_conversion_map = nullptr) + absl::flat_hash_map* tensor_to_value, + absl::flat_hash_map* quant_conversion_map = nullptr) : graph_(graph), context_(context), node_(node), @@ -98,8 +98,8 @@ class ObjectReader { GraphFloat32* graph_; TfLiteContext* context_; const TfLiteNode* node_; - std::unordered_map* tensor_to_value_; - std::unordered_map* quant_conversion_map_; + absl::flat_hash_map* tensor_to_value_; + absl::flat_hash_map* quant_conversion_map_; }; } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/common/operations.cc b/tensorflow/lite/delegates/gpu/common/operations.cc index 245a5a80639..fbffe9d65ff 100644 --- a/tensorflow/lite/delegates/gpu/common/operations.cc +++ b/tensorflow/lite/delegates/gpu/common/operations.cc @@ -16,8 +16,8 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/operations.h" #include -#include +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/delegates/gpu/common/shape.h" #include "tensorflow/lite/delegates/gpu/common/status.h" @@ -165,7 +165,7 @@ std::string ToString(enum OperationType op) { OperationType OperationTypeFromString(const std::string& name) { static const auto operations = - new std::unordered_map({ + new absl::flat_hash_map({ {"abs", OperationType::ABS}, {"add", OperationType::ADD}, {"batch_normalization", OperationType::BATCH_NORMALIZATION}, diff --git a/tensorflow/lite/delegates/gpu/common/quantization_util.cc b/tensorflow/lite/delegates/gpu/common/quantization_util.cc index 9584d1d98ec..fe92989a3ae 100644 --- a/tensorflow/lite/delegates/gpu/common/quantization_util.cc +++ b/tensorflow/lite/delegates/gpu/common/quantization_util.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/quantization_util.h" +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/kernels/internal/optimized/optimized_ops.h" #include "tensorflow/lite/kernels/internal/types.h" @@ -22,8 +23,9 @@ limitations under the License. namespace tflite { namespace gpu { namespace { -void DequantizeInput(TfLiteContext* context, int input_index, - const std::unordered_map& quant_conversion_map) { +void DequantizeInput( + TfLiteContext* context, int input_index, + const absl::flat_hash_map& quant_conversion_map) { if (quant_conversion_map.find(input_index) == quant_conversion_map.end()) { return; } @@ -50,7 +52,7 @@ void DequantizeInput(TfLiteContext* context, int input_index, } void QuantizeOutput(TfLiteContext* context, int output_index, - const std::unordered_map& quant_conversion_map) { + const absl::flat_hash_map& quant_conversion_map) { if (quant_conversion_map.find(output_index) == quant_conversion_map.end()) { return; } @@ -80,7 +82,7 @@ void QuantizeOutput(TfLiteContext* context, int output_index, absl::Status DequantizeInputs( TfLiteContext* context, const std::vector& input_indices, - const std::unordered_map& quant_conversion_map) { + const absl::flat_hash_map& quant_conversion_map) { for (auto index : input_indices) { DequantizeInput(context, static_cast(index), quant_conversion_map); } @@ -89,7 +91,7 @@ absl::Status DequantizeInputs( absl::Status DequantizeInputs( TfLiteContext* context, const std::vector& input_indices, - const std::unordered_map& quant_conversion_map) { + const absl::flat_hash_map& quant_conversion_map) { for (auto index : input_indices) { DequantizeInput(context, static_cast(index), quant_conversion_map); } @@ -98,7 +100,7 @@ absl::Status DequantizeInputs( absl::Status QuantizeOutputs( TfLiteContext* context, const std::vector& output_indices, - const std::unordered_map& quant_conversion_map) { + const absl::flat_hash_map& quant_conversion_map) { for (auto index : output_indices) { QuantizeOutput(context, static_cast(index), quant_conversion_map); } @@ -108,7 +110,7 @@ absl::Status QuantizeOutputs( absl::Status QuantizeOutputs( TfLiteContext* context, const std::vector& output_indices, - const std::unordered_map& quant_conversion_map) { + const absl::flat_hash_map& quant_conversion_map) { for (auto index : output_indices) { QuantizeOutput(context, static_cast(index), quant_conversion_map); } diff --git a/tensorflow/lite/delegates/gpu/common/quantization_util.h b/tensorflow/lite/delegates/gpu/common/quantization_util.h index 26512531f29..fc01d612d6f 100644 --- a/tensorflow/lite/delegates/gpu/common/quantization_util.h +++ b/tensorflow/lite/delegates/gpu/common/quantization_util.h @@ -16,9 +16,9 @@ limitations under the License. #ifndef TENSORFLOW_LITE_DELEGATES_GPU_COMMON_QUANTIZATION_UTIL_H_ #define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_QUANTIZATION_UTIL_H_ -#include #include +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/delegates/gpu/common/status.h" @@ -32,11 +32,11 @@ namespace gpu { // tensor and its original quantized one. absl::Status DequantizeInputs( TfLiteContext* context, const std::vector& input_indices, - const std::unordered_map& quant_conversion_map); + const absl::flat_hash_map& quant_conversion_map); absl::Status DequantizeInputs( TfLiteContext* context, const std::vector& input_indices, - const std::unordered_map& quant_conversion_map); + const absl::flat_hash_map& quant_conversion_map); // Quantizes output tensors post-inference, leaving float tensors intact. // output_indices contains (fp32) inputs to be quantized, which are outputs of @@ -45,11 +45,11 @@ absl::Status DequantizeInputs( // tensor and its original quantized one. absl::Status QuantizeOutputs( TfLiteContext* context, const std::vector& output_indices, - const std::unordered_map& quant_conversion_map); + const absl::flat_hash_map& quant_conversion_map); absl::Status QuantizeOutputs( TfLiteContext* context, const std::vector& output_indices, - const std::unordered_map& quant_conversion_map); + const absl::flat_hash_map& quant_conversion_map); } // namespace gpu } // namespace tflite diff --git a/tensorflow/lite/delegates/gpu/common/quantization_util_test.cc b/tensorflow/lite/delegates/gpu/common/quantization_util_test.cc index 064a2a2e6b2..b5cdaec91e0 100644 --- a/tensorflow/lite/delegates/gpu/common/quantization_util_test.cc +++ b/tensorflow/lite/delegates/gpu/common/quantization_util_test.cc @@ -151,7 +151,7 @@ TEST(DequantizeInputs, Int8) { PopulateContext(tensors, context); std::vector input_indices = {1}; - std::unordered_map quant_conversion_map = {{1, 0}}; + absl::flat_hash_map quant_conversion_map = {{1, 0}}; auto status = DequantizeInputs(&context, input_indices, quant_conversion_map); EXPECT_TRUE(status.ok()); @@ -176,7 +176,7 @@ TEST(DequantizeInputs, UInt8) { PopulateContext(tensors, context); std::vector input_indices = {1}; - std::unordered_map quant_conversion_map = {{1, 0}}; + absl::flat_hash_map quant_conversion_map = {{1, 0}}; auto status = DequantizeInputs(&context, input_indices, quant_conversion_map); EXPECT_TRUE(status.ok()); @@ -199,7 +199,7 @@ TEST(QuantizeOutputs, Int8) { PopulateContext(tensors, context); std::vector output_indices = {0}; - std::unordered_map quant_conversion_map = {{0, 1}}; + absl::flat_hash_map quant_conversion_map = {{0, 1}}; auto status = QuantizeOutputs(&context, output_indices, quant_conversion_map); EXPECT_TRUE(status.ok()); @@ -221,7 +221,7 @@ TEST(QuantizeOutputs, UInt8) { PopulateContext(tensors, context); std::vector output_indices = {0}; - std::unordered_map quant_conversion_map = {{0, 1}}; + absl::flat_hash_map quant_conversion_map = {{0, 1}}; auto status = QuantizeOutputs(&context, output_indices, quant_conversion_map); EXPECT_TRUE(status.ok()); diff --git a/tensorflow/lite/delegates/gpu/delegate.cc b/tensorflow/lite/delegates/gpu/delegate.cc index 0f2d9811633..bfc2b7f08c4 100644 --- a/tensorflow/lite/delegates/gpu/delegate.cc +++ b/tensorflow/lite/delegates/gpu/delegate.cc @@ -18,9 +18,9 @@ limitations under the License. #include #include #include // NOLINT(build/c++11) -#include #include +#include "absl/container/flat_hash_map.h" #include "absl/memory/memory.h" #include "absl/types/span.h" #include "tensorflow/lite/builtin_ops.h" @@ -350,7 +350,7 @@ class DelegateKernel { // Whenever quantized inference is enabled, this maps the tensor index of each // originally quantized (8-bit) tensor to its float version added in // model_builder - and vice versa. - std::unordered_map quant_conversion_map_; + absl::flat_hash_map quant_conversion_map_; std::thread::id thread_id_prepare_; // thread id used for Prapare() bool enforce_same_thread_ = false; // flag to enforce same thread for Invoke }; diff --git a/tensorflow/lite/delegates/gpu/gl/BUILD b/tensorflow/lite/delegates/gpu/gl/BUILD index 91472261d04..d39f5e3c34a 100644 --- a/tensorflow/lite/delegates/gpu/gl/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/BUILD @@ -29,6 +29,7 @@ cc_library( ":runtime_options", ":stats", ":variable", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "//tensorflow/lite/delegates/gpu/common:model", @@ -66,6 +67,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/gl/kernels:converter", "//tensorflow/lite/delegates/gpu/gl/kernels:registry", "//tensorflow/lite/delegates/gpu/gl/workgroups:default_calculator", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", "@com_google_absl//absl/types:span", ], @@ -125,6 +127,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/gl/compiler:fuse_inplace", "//tensorflow/lite/delegates/gpu/gl/compiler:shader_code", "//tensorflow/lite/delegates/gpu/gl/compiler:shader_codegen", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", "@com_google_absl//absl/types:any", ], diff --git a/tensorflow/lite/delegates/gpu/gl/api.cc b/tensorflow/lite/delegates/gpu/gl/api.cc index 0240a5cfbed..f50b8cb5d5c 100644 --- a/tensorflow/lite/delegates/gpu/gl/api.cc +++ b/tensorflow/lite/delegates/gpu/gl/api.cc @@ -19,10 +19,10 @@ limitations under the License. #include #include #include // NOLINT -#include #include #include +#include "absl/container/flat_hash_map.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "tensorflow/lite/delegates/gpu/common/model.h" @@ -46,7 +46,7 @@ namespace gpu { namespace gl { namespace { -using ObjectsSizes = std::unordered_map; +using ObjectsSizes = absl::flat_hash_map; enum class InferenceContextState { NOT_STARTED, @@ -313,7 +313,7 @@ class CompiledModelImpl full_shaders[shader.second] = shader.first; } - std::unordered_map partial_shader_to_index; + absl::flat_hash_map partial_shader_to_index; std::vector partial_shaders; for (const auto& program : programs_) { // Remove a header from a shader. @@ -366,16 +366,16 @@ class CompiledModelImpl std::vector shaders_; // Shaders are serialized in order of their indices. - std::unordered_map shader_to_index_; + absl::flat_hash_map shader_to_index_; std::deque programs_; - std::unordered_map object_sizes_; + absl::flat_hash_map object_sizes_; CompilerStats stats_; }; } // namespace absl::Status Compile(const CompilationOptions& options, const GraphFloat32& model, - const std::unordered_set& tflite_graph_io, + const std::unordered_set& tflite_graph_io, // NOLINT const NodeShader& node_shader, const WorkgroupsCalculator& workgroup_calculator, std::unique_ptr* compiled_model) { diff --git a/tensorflow/lite/delegates/gpu/gl/api.h b/tensorflow/lite/delegates/gpu/gl/api.h index c37eb9b7772..11498243757 100644 --- a/tensorflow/lite/delegates/gpu/gl/api.h +++ b/tensorflow/lite/delegates/gpu/gl/api.h @@ -67,7 +67,7 @@ class CompiledModel { // Turns the given model into "compiled" form that is suitable for inference. absl::Status Compile(const CompilationOptions& options, const GraphFloat32& model, - const std::unordered_set& tflite_graph_io, + const std::unordered_set& tflite_graph_io, // NOLINT const NodeShader& node_shader, const WorkgroupsCalculator& workgroup_calculator, std::unique_ptr* compiled_model); diff --git a/tensorflow/lite/delegates/gpu/gl/api2.cc b/tensorflow/lite/delegates/gpu/gl/api2.cc index c8bf6dd063a..c12463800a9 100644 --- a/tensorflow/lite/delegates/gpu/gl/api2.cc +++ b/tensorflow/lite/delegates/gpu/gl/api2.cc @@ -18,10 +18,10 @@ limitations under the License. #include #include #include -#include #include #include +#include "absl/container/flat_hash_map.h" #include "absl/memory/memory.h" #include "absl/types/span.h" #include "tensorflow/lite/delegates/gpu/common/data_type.h" @@ -542,7 +542,7 @@ class InferenceBuilderImpl : public InferenceBuilder { auto workgroup_calculator = NewDefaultWorkgroupsCalculator(*gpu_info_); auto external_objects = absl::make_unique(); std::vector shaders; - std::unordered_map shader_to_index; + absl::flat_hash_map shader_to_index; RuntimeOptions runtime_options; auto runtime = absl::make_unique(runtime_options, *gpu_info_, diff --git a/tensorflow/lite/delegates/gpu/gl/compiler.cc b/tensorflow/lite/delegates/gpu/gl/compiler.cc index d316505a0e0..eba25171ca3 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler.cc @@ -21,6 +21,7 @@ limitations under the License. #include #include +#include "absl/container/flat_hash_map.h" #include "absl/memory/memory.h" #include "absl/types/any.h" #include "tensorflow/lite/delegates/gpu/common/data_type.h" @@ -102,9 +103,10 @@ class CompilerImpl : public Compiler { } } - absl::Status Compile(const GraphFloat32& graph, - const std::unordered_set& tflite_graph_io, - const ShaderCodeCallback& callback) final { + absl::Status Compile( + const GraphFloat32& graph, + const std::unordered_set& tflite_graph_io, // NOLINT + const ShaderCodeCallback& callback) final { // It is important to have ids in a compiled graph identical to the given // graph. RETURN_IF_ERROR(graph.MakeExactCopy(&compiled_graph_)); @@ -158,7 +160,7 @@ class CompilerImpl : public Compiler { } // Prepare internal objects. - std::unordered_map objects; + absl::flat_hash_map objects; for (auto value : compiled_graph_.values()) { Object object = MakePHWC4Ref(value->id, value->tensor.shape); object.data_type = value->tensor.type; diff --git a/tensorflow/lite/delegates/gpu/gl/compiler.h b/tensorflow/lite/delegates/gpu/gl/compiler.h index 7769890b769..03ea3dd2a90 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler.h +++ b/tensorflow/lite/delegates/gpu/gl/compiler.h @@ -40,9 +40,10 @@ class Compiler { // Goes over a graph and generates OpenGL shaders for the given graph. // Callback is called for every generated shader. Callback may execute shaders // as they come or store them elsewhere to execute later. - virtual absl::Status Compile(const GraphFloat32& graph, - const std::unordered_set& tflite_graph_io, - const ShaderCodeCallback& callback) = 0; + virtual absl::Status Compile( + const GraphFloat32& graph, + const std::unordered_set& tflite_graph_io, // NOLINT + const ShaderCodeCallback& callback) = 0; }; std::unique_ptr NewCompiler( diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/BUILD b/tensorflow/lite/delegates/gpu/gl/compiler/BUILD index 601e809fffa..f62f48750bd 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/compiler/BUILD @@ -38,6 +38,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:data_type", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:object", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:variant", @@ -101,6 +102,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/gl:node_shader", "//tensorflow/lite/delegates/gpu/gl:object", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings", ], ) @@ -150,6 +152,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/gl:node_shader", "//tensorflow/lite/delegates/gpu/gl:object", "//tensorflow/lite/delegates/gpu/gl:variable", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", ], ) @@ -164,6 +167,7 @@ cc_library( "//tensorflow/lite/delegates/gpu/common:model_transformer", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:types", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:any", "@com_google_absl//absl/types:variant", @@ -193,6 +197,7 @@ cc_library( ":preprocessor", "//tensorflow/lite/delegates/gpu/common:types", "//tensorflow/lite/delegates/gpu/gl:variable", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:variant", diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/compiled_node.cc b/tensorflow/lite/delegates/gpu/gl/compiler/compiled_node.cc index 4048a07d087..035fce56d31 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/compiled_node.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler/compiled_node.cc @@ -15,8 +15,7 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/compiler/compiled_node.h" -#include - +#include "absl/container/flat_hash_set.h" #include "absl/strings/str_cat.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/gl/compiler/rename.h" @@ -28,7 +27,7 @@ namespace gl { absl::Status MergeCode(CompiledNodeAttributes* attr, CompiledNodeAttributes* merged_attr) { // build a map of known names. - std::unordered_set known_names; + absl::flat_hash_set known_names; for (const auto& parameter : merged_attr->code.parameters) { known_names.insert(parameter.name); } diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/fuse_auto_input.cc b/tensorflow/lite/delegates/gpu/gl/compiler/fuse_auto_input.cc index d0408c6a7be..36d8fa8c1c7 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/fuse_auto_input.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler/fuse_auto_input.cc @@ -16,9 +16,9 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/compiler/fuse_auto_input.h" #include -#include #include +#include "absl/container/flat_hash_set.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_replace.h" #include "absl/types/any.h" @@ -102,7 +102,7 @@ TransformResult FuseAutoInput::ApplyToNode(Node* node, GraphFloat32* graph) { // Skip fusions which will result in duplicate inputs, e.g. diamond shapes. { - std::unordered_set all_inputs; + absl::flat_hash_set all_inputs; for (const auto& node_to_fuse : nodes_to_fuse) { for (const auto& input : graph->FindInputs(node_to_fuse.first->id)) { if (all_inputs.find(input->id) != all_inputs.end()) { diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor.h b/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor.h index 78e7a2f1e17..5c4de49c44b 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor.h +++ b/tensorflow/lite/delegates/gpu/gl/compiler/object_accessor.h @@ -17,9 +17,9 @@ limitations under the License. #define TENSORFLOW_LITE_DELEGATES_GPU_GL_COMPILER_OBJECT_ACCESSOR_H_ #include -#include #include +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/delegates/gpu/gl/compiler/preprocessor.h" #include "tensorflow/lite/delegates/gpu/gl/compiler/variable_accessor.h" #include "tensorflow/lite/delegates/gpu/gl/object.h" @@ -85,7 +85,7 @@ class ObjectAccessor : public InlineRewrite { RewriteStatus RewriteWrite(absl::string_view location, absl::string_view value, std::string* output); - std::unordered_map name_to_object_; + absl::flat_hash_map name_to_object_; const bool is_mali_; const bool sampler_textures_; diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/rename.cc b/tensorflow/lite/delegates/gpu/gl/compiler/rename.cc index 956f6afae28..b41ba473b85 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/rename.cc +++ b/tensorflow/lite/delegates/gpu/gl/compiler/rename.cc @@ -16,10 +16,10 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/gl/compiler/rename.h" #include -#include #include #include +#include "absl/container/flat_hash_map.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" @@ -86,7 +86,7 @@ class VariableRewriter : public InlineRewrite { const std::string inline_delimiter_; const NameFunctor name_func_; - std::unordered_map name_to_variable_; + absl::flat_hash_map name_to_variable_; }; // Rewrites names of all objects according to returned values from the @@ -168,7 +168,7 @@ class ObjectRewriter : public InlineRewrite { const std::string inline_delimiter_; const NameFunctor name_func_; - std::unordered_map> + absl::flat_hash_map> name_to_object_; }; diff --git a/tensorflow/lite/delegates/gpu/gl/compiler/variable_accessor.h b/tensorflow/lite/delegates/gpu/gl/compiler/variable_accessor.h index c9946a00395..db4b031548b 100644 --- a/tensorflow/lite/delegates/gpu/gl/compiler/variable_accessor.h +++ b/tensorflow/lite/delegates/gpu/gl/compiler/variable_accessor.h @@ -16,11 +16,11 @@ limitations under the License. #ifndef TENSORFLOW_LITE_DELEGATES_GPU_GL_COMPILER_VARIABLE_ACCESSOR_H_ #define TENSORFLOW_LITE_DELEGATES_GPU_GL_COMPILER_VARIABLE_ACCESSOR_H_ -#include -#include #include +#include #include +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/delegates/gpu/gl/compiler/preprocessor.h" #include "tensorflow/lite/delegates/gpu/gl/variable.h" @@ -72,7 +72,7 @@ class VariableAccessor : public InlineRewrite { private: const bool inline_values_; const bool vulkan_support_; - std::unordered_map name_to_variable_; + absl::flat_hash_map name_to_variable_; std::set shared_variables_; std::set uniform_parameters_; }; diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/BUILD b/tensorflow/lite/delegates/gpu/gl/kernels/BUILD index a367a60ba41..774b6755014 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/kernels/BUILD @@ -155,7 +155,10 @@ cc_library( name = "custom_registry", srcs = ["custom_registry.cc"], hdrs = ["custom_registry.h"], - deps = ["//tensorflow/lite/delegates/gpu/gl:node_shader"], + deps = [ + "//tensorflow/lite/delegates/gpu/gl:node_shader", + "@com_google_absl//absl/container:flat_hash_map", + ], ) cc_library( @@ -774,6 +777,7 @@ cc_library( "//conditions:default": NON_TFLITE_GPU_BINARY_RELEASE_OPERATORS, }) + [ ":custom_registry", + "@com_google_absl//absl/container:flat_hash_map", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/gl:node_shader", diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/custom_registry.cc b/tensorflow/lite/delegates/gpu/gl/kernels/custom_registry.cc index f5c5429e867..a01e885adef 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/custom_registry.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/custom_registry.cc @@ -17,15 +17,16 @@ limitations under the License. #include #include -#include #include +#include "absl/container/flat_hash_map.h" + namespace tflite { namespace gpu { namespace gl { void RegisterCustomOps( - std::unordered_map>>* + absl::flat_hash_map>>* shaders) {} } // namespace gl diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/custom_registry.h b/tensorflow/lite/delegates/gpu/gl/kernels/custom_registry.h index 9a979a982db..7b2a841bca9 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/custom_registry.h +++ b/tensorflow/lite/delegates/gpu/gl/kernels/custom_registry.h @@ -18,9 +18,9 @@ limitations under the License. #include #include -#include #include +#include "absl/container/flat_hash_map.h" #include "tensorflow/lite/delegates/gpu/gl/node_shader.h" namespace tflite { @@ -29,7 +29,7 @@ namespace gl { // Registers custom operations. void RegisterCustomOps( - std::unordered_map>>* + absl::flat_hash_map>>* shaders_); } // namespace gl diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc b/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc index da6aad720a2..645e5b6c728 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/registry.cc @@ -18,10 +18,10 @@ limitations under the License. #include #include #include -#include #include #include +#include "absl/container/flat_hash_map.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" @@ -139,7 +139,7 @@ class Registry : public NodeShader { } private: - std::unordered_map>> + absl::flat_hash_map>> shaders_; }; diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/test_util.cc b/tensorflow/lite/delegates/gpu/gl/kernels/test_util.cc index e9abec7eec6..21a53acd9c9 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/test_util.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/test_util.cc @@ -17,10 +17,10 @@ limitations under the License. #include #include -#include -#include #include +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "tensorflow/lite/delegates/gpu/common/model.h" #include "tensorflow/lite/delegates/gpu/common/operations.h" #include "tensorflow/lite/delegates/gpu/common/status.h" @@ -78,7 +78,7 @@ absl::Status SingleOpModel::Invoke(const CompilationOptions& compile_options, // Create buffers for input tensors. { - std::unordered_map tensor_to_id; + absl::flat_hash_map tensor_to_id; for (const auto* input : graph_.inputs()) { tensor_to_id[input->tensor.ref] = input->id; } @@ -101,9 +101,9 @@ absl::Status SingleOpModel::Invoke(const CompilationOptions& compile_options, GpuInfo gpu_info; RETURN_IF_ERROR(RequestGpuInfo(&gpu_info)); std::unique_ptr compiled_model; - RETURN_IF_ERROR(Compile( - compile_options, graph_, /*tflite_graph_io=*/std::unordered_set(), - shader, *NewDefaultWorkgroupsCalculator(gpu_info), &compiled_model)); + RETURN_IF_ERROR(Compile(compile_options, graph_, /*tflite_graph_io=*/{}, + shader, *NewDefaultWorkgroupsCalculator(gpu_info), + &compiled_model)); // Get inference context. auto command_queue = NewCommandQueue(gpu_info); diff --git a/tensorflow/lite/delegates/gpu/gl/runtime.cc b/tensorflow/lite/delegates/gpu/gl/runtime.cc index b7e01a33570..7f0cbe0284b 100644 --- a/tensorflow/lite/delegates/gpu/gl/runtime.cc +++ b/tensorflow/lite/delegates/gpu/gl/runtime.cc @@ -17,7 +17,6 @@ limitations under the License. #include #include -#include #include #include "absl/strings/str_cat.h" diff --git a/tensorflow/lite/delegates/gpu/gl/workgroups/calculator_from_metadata.cc b/tensorflow/lite/delegates/gpu/gl/workgroups/calculator_from_metadata.cc index 7976fd54ed0..8a269e7cf25 100644 --- a/tensorflow/lite/delegates/gpu/gl/workgroups/calculator_from_metadata.cc +++ b/tensorflow/lite/delegates/gpu/gl/workgroups/calculator_from_metadata.cc @@ -18,8 +18,8 @@ limitations under the License. #ifndef TFLITE_GPU_BINARY_RELEASE #include -#include +#include "absl/container/flat_hash_map.h" #include "absl/memory/memory.h" #include "flatbuffers/flatbuffers.h" // from @flatbuffers #include "tensorflow/lite/delegates/gpu/common/gpu_info.h" @@ -62,7 +62,7 @@ class WorkgroupsCalculatorFromMetadata : public WorkgroupsCalculator { } private: - std::unordered_map workgroups_; + absl::flat_hash_map workgroups_; std::unique_ptr default_calculator_; }; diff --git a/tensorflow/lite/delegates/gpu/gl_delegate.cc b/tensorflow/lite/delegates/gpu/gl_delegate.cc index 0587cb4f3a3..2f25539802a 100644 --- a/tensorflow/lite/delegates/gpu/gl_delegate.cc +++ b/tensorflow/lite/delegates/gpu/gl_delegate.cc @@ -160,7 +160,7 @@ class Delegate { tensors_[value->id] = {value->tensor.shape, 0}; } - std::unordered_set tflite_graph_io; + std::unordered_set tflite_graph_io; // NOLINT // Prepare graph inputs. // diff --git a/tensorflow/lite/delegates/gpu/metal/kernels/elementwise.cc b/tensorflow/lite/delegates/gpu/metal/kernels/elementwise.cc index 53c1c5b38dd..9edfc884638 100644 --- a/tensorflow/lite/delegates/gpu/metal/kernels/elementwise.cc +++ b/tensorflow/lite/delegates/gpu/metal/kernels/elementwise.cc @@ -16,9 +16,9 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/metal/kernels/elementwise.h" #include -#include #include +#include "absl/container/flat_hash_map.h" #include "absl/strings/substitute.h" #include "tensorflow/lite/delegates/gpu/common/convert.h" #include "tensorflow/lite/delegates/gpu/common/operations.h" @@ -32,7 +32,7 @@ namespace metal { namespace { std::string OneInputFunctor(OperationType op_type, const std::string& value) { - const std::unordered_map functors{ + const absl::flat_hash_map functors{ {OperationType::ABS, "abs($0)"}, {OperationType::SIN, "sin($0)"}, {OperationType::HARD_SWISH, @@ -62,7 +62,7 @@ std::string OneInputFunctor(OperationType op_type, const std::string& value) { std::string TwoInputFunctor(OperationType op_type, const std::string& value0, const std::string& value1) { - const std::unordered_map functors{ + const absl::flat_hash_map functors{ {OperationType::ADD, "$0 + $1"}, {OperationType::DIV, "$0 / $1"}, {OperationType::MAXIMUM, "max($0, $1)"}, diff --git a/tensorflow/lite/delegates/gpu/metal_delegate.mm b/tensorflow/lite/delegates/gpu/metal_delegate.mm index 45bfe1f3b2f..c2e5289c604 100644 --- a/tensorflow/lite/delegates/gpu/metal_delegate.mm +++ b/tensorflow/lite/delegates/gpu/metal_delegate.mm @@ -26,6 +26,7 @@ limitations under the License. #include #include +#include "absl/container/flat_hash_set.h" #include "absl/types/span.h" #include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/c/common.h" @@ -613,7 +614,7 @@ class Delegate { // Whenever quantized inference is enabled, this maps the tensor index of each // originally quantized (8-bit) tensor to its float version added in // model_builder - and vice versa. - std::unordered_map quant_conversion_map_; + absl::flat_hash_map quant_conversion_map_; TFLInferenceContext* inference_context_; // input and output buffers are passed into Metal inference engine From c0b693f16476a32c8933501acea8261e4a81aca7 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Thu, 6 Aug 2020 23:33:31 +0000 Subject: [PATCH 0601/1017] clean up --- tensorflow/c/kernels/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 8909a32935a..9e774edeb5a 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -79,7 +79,7 @@ tf_gen_op_libs( "//tensorflow/c:ops", "//tensorflow/c:tf_status", "//tensorflow/core:lib", - ] + ], ) tf_cc_test( From 2e4bf2c0bcd506a1ee20388dab498e75b2ca0826 Mon Sep 17 00:00:00 2001 From: YoungSeok Yoon Date: Thu, 6 Aug 2020 16:32:28 -0700 Subject: [PATCH 0602/1017] Prepend "TensorFlowLiteC/" to common.h path in delegate subspecs PiperOrigin-RevId: 325335212 Change-Id: I6a932d12d84d308c0083db97aed23817c40e7f30 --- tensorflow/lite/experimental/ios/BUILD.apple | 13 +++++++++---- tensorflow/lite/experimental/ios/ios.bzl | 7 ++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tensorflow/lite/experimental/ios/BUILD.apple b/tensorflow/lite/experimental/ios/BUILD.apple index 99ea2f8acbb..e1e3be2bcde 100644 --- a/tensorflow/lite/experimental/ios/BUILD.apple +++ b/tensorflow/lite/experimental/ios/BUILD.apple @@ -24,13 +24,20 @@ sh_binary( ) strip_common_include_path_prefix( - name = "strip_common_include_path", + name = "strip_common_include_path_core", hdr_labels = [ "//tensorflow/lite/c:c_api.h", - "//tensorflow/lite/delegates/gpu:metal_delegate.h", "//tensorflow/lite/delegates/xnnpack:xnnpack_delegate.h", + ], +) + +strip_common_include_path_prefix( + name = "strip_common_include_path_subspecs", + hdr_labels = [ + "//tensorflow/lite/delegates/gpu:metal_delegate.h", "//tensorflow/lite/experimental/delegates/coreml:coreml_delegate.h", ], + prefix = "TensorFlowLiteC/", ) # bazel build -c opt --config=ios_fat //tensorflow/lite/experimental/ios:TensorFlowLiteC_framework @@ -77,7 +84,6 @@ tflite_ios_static_framework( name = "TensorFlowLiteCCoreML_framework", hdrs = [ ":coreml_delegate.h", - "//tensorflow/lite/c:common.h", ], allowlist_symbols_file = ":allowlist_TensorFlowLiteCCoreML.txt", bundle_name = "TensorFlowLiteCCoreML", @@ -97,7 +103,6 @@ tflite_ios_static_framework( name = "TensorFlowLiteCMetal_framework", hdrs = [ ":metal_delegate.h", - "//tensorflow/lite/c:common.h", ], allowlist_symbols_file = ":allowlist_TensorFlowLiteCMetal.txt", bundle_name = "TensorFlowLiteCMetal", diff --git a/tensorflow/lite/experimental/ios/ios.bzl b/tensorflow/lite/experimental/ios/ios.bzl index 43ca6ec6010..63747eb8d1a 100644 --- a/tensorflow/lite/experimental/ios/ios.bzl +++ b/tensorflow/lite/experimental/ios/ios.bzl @@ -76,13 +76,14 @@ def tflite_ios_static_framework( # to the "Headers" directory with no header path prefixes. This auxiliary rule # is used for stripping the path prefix to the "common.h" file included by the # "c_api.h" header. -def strip_common_include_path_prefix(name, hdr_labels): +def strip_common_include_path_prefix(name, hdr_labels, prefix = ""): """Create modified header files with the common.h include path stripped out. Args: name: The name to be used as a prefix to the generated genrules. hdr_labels: List of header labels to strip out the include path. Each label must end with a colon followed by the header file name. + prefix: Optional prefix path to prepend to the common.h inclusion path. """ for hdr_label in hdr_labels: @@ -94,8 +95,8 @@ def strip_common_include_path_prefix(name, hdr_labels): srcs = [hdr_label], outs = [hdr_filename], cmd = """ - sed 's|#include ".*common.h"|#include "common.h"|'\ + sed 's|#include ".*common.h"|#include "{}common.h"|'\ "$(location {})"\ > "$@" - """.format(hdr_label), + """.format(prefix, hdr_label), ) From 1db5e0f3233230cd99b4877d8a11fb96f16aac36 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Thu, 6 Aug 2020 16:57:28 -0700 Subject: [PATCH 0603/1017] Update ReplicateToIslandPass in preparation for cloning functions (NFC). Pass is now converted to a module pass and some pointers are replaced with references. Modification to replicate variant ops are now under one function. PiperOrigin-RevId: 325339815 Change-Id: I1419ea05c808b471701189698ceb965556b81ae8 --- .../mlir/tensorflow/transforms/bridge.cc | 3 +- .../mlir/tensorflow/transforms/passes.h | 2 +- .../transforms/replicate_to_island.cc | 172 ++++++++++-------- 3 files changed, 95 insertions(+), 82 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc b/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc index ed0528ae054..2a5c8a05ef3 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc @@ -47,7 +47,8 @@ void AddGraphExportLoweringPasses(OpPassManager &pm) { pm.addNestedPass(CreateFunctionalToExecutorDialectConversionPass()); add_pass(TFDevice::CreateParallelizeEmbeddingParamsOpsPass()); - add_pass(TFDevice::CreateReplicateToIslandPass()); + pm.addPass(TFDevice::CreateReplicateToIslandPass()); + pm.addPass(CreateBreakUpIslandsPass()); add_pass(TFDevice::CreateParallelExecuteToIslandsPass()); add_pass(TFDevice::CreateLaunchToDeviceAttributePass()); } diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/passes.h b/tensorflow/compiler/mlir/tensorflow/transforms/passes.h index 3afadd2b06d..3be6c9e1a70 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/passes.h +++ b/tensorflow/compiler/mlir/tensorflow/transforms/passes.h @@ -239,7 +239,7 @@ std::unique_ptr> CreateReplicateInvariantOpHoistingPass(); // Creates a pass that forms replica `tf_executor.island` from a single // `tf_device.replicate` island. -std::unique_ptr> CreateReplicateToIslandPass(); +std::unique_ptr> CreateReplicateToIslandPass(); // Creates a pass that creates `tf_executor.island` from a single // `tf_device.parallel_execute` island. diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc b/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc index fcf0bb98a61..e7f2977dbcd 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc @@ -49,8 +49,8 @@ constexpr char kReplicaIdAttr[] = "_xla_replica_id"; constexpr char kDeviceOrdinalAttr[] = "device_ordinal"; struct ReplicateToIslandPass - : public PassWrapper { - void runOnFunction() override; + : public PassWrapper> { + void runOnOperation() override; }; // Returns whether op requires `_xla_replica_id` attribute. @@ -64,29 +64,62 @@ bool RequiresDeviceOrdinalAttribute(Operation* op) { llvm::isa(op); } -// Adds integer attribute that represents replica id for replicated ops that -// require replica id attribute. -void AddReplicaIdToOpsInReplicatedRegion(OpBuilder* builder, Region* region, - const int replica_id) { - region->walk([&](Operation* replicated_op) { - if (RequiresReplicaIDAttribute(replicated_op)) - replicated_op->setAttr(kReplicaIdAttr, - builder->getI32IntegerAttr(replica_id)); +// Updates replica variant ops in a region based on replica `replica_id`. +// TODO(b/157624749): Replace this with better abstraction to differentiate ops +// for different replicas. Some ops, such as XlaHostCompute op or TPU Embedding +// ops, require replica id to be added as an op attribute to be used during +// execution. Handle such ops separately and add an integer attribute that +// represents replica id. +LogicalResult UpdateRegionReplicateVariantOps( + OpBuilder& builder, Location loc, Region& region, int replica_id, + const llvm::Optional& devices) { + int64_t device_ordinal = -1; + const bool has_devices = devices.hasValue(); + if (has_devices) { + if (auto tpu_replica_0 = devices.getValue().get("TPU_REPLICATED_CORE_0")) { + llvm::StringRef tpu_device = tpu_replica_0.cast()[replica_id] + .cast() + .getValue(); + if (failed(tensorflow::GetDeviceOrdinalFromDeviceString( + loc, tpu_device, &device_ordinal))) { + return failure(); + } + } + } + + region.walk([&](Operation* op) { + // Add replica id. + if (RequiresReplicaIDAttribute(op)) + op->setAttr(kReplicaIdAttr, builder.getI32IntegerAttr(replica_id)); + + if (!has_devices) return; + + // Map aliased devices to explicit devices based on replica. + if (auto launch = dyn_cast(op)) + if (auto device_by_replica = devices.getValue().get(launch.device())) + launch.setAttr( + kDeviceAttr, + device_by_replica.cast()[replica_id].cast()); + + // Add device ordinal. + if (device_ordinal >= 0 && RequiresDeviceOrdinalAttribute(op)) + op->setAttr(kDeviceOrdinalAttr, + builder.getI64IntegerAttr(device_ordinal)); }); + + return success(); } // Creates islands per replica from `tf_device.replicate` region. If for a // `tf_device.launch` op the device is an aliased device of the // `tf_device.replicate`, the device will be remapped to an explicit device // for the associated replica island. -llvm::SmallVector ExpandReplicateIntoReplicas( - const Dialect* tf_dialect, OpBuilder* builder, +LogicalResult ExpandReplicateIntoReplicas( + const Dialect* tf_dialect, OpBuilder& builder, tf_executor::IslandOp island_op, tf_device::ReplicateOp replicate_op, - int num_replicas) { - auto devices = replicate_op.devices(); - const bool has_devices = devices.hasValue(); - llvm::SmallVector replicas; + int num_replicas, llvm::SmallVectorImpl& replicas) { replicas.reserve(num_replicas); + auto devices = replicate_op.devices(); // Collect result types and operands. Operation& terminator = replicate_op.GetBody().back(); @@ -95,16 +128,16 @@ llvm::SmallVector ExpandReplicateIntoReplicas( llvm::SmallVector replica_inputs(island_op.controlInputs()); // Replace replicate terminator with YieldOp. - builder->setInsertionPoint(&terminator); - builder->create(terminator.getLoc(), - terminator.getOperands()); + builder.setInsertionPoint(&terminator); + builder.create(terminator.getLoc(), + terminator.getOperands()); terminator.erase(); - builder->setInsertionPoint(island_op); + builder.setInsertionPoint(island_op); BlockAndValueMapping mapping; for (int i : llvm::seq(0, num_replicas)) { // Create new island for replica. - auto replica = builder->create( + auto replica = builder.create( island_op.getLoc(), output_types, control_type, replica_inputs); // Map block arg to replica arg. @@ -116,42 +149,15 @@ llvm::SmallVector ExpandReplicateIntoReplicas( // Copy over replicate region into replica island. replicate_op.body().cloneInto(&replica.body(), mapping); - // TODO(b/157624749): Replace this with better abstraction to - // differentiate ops for different replicas. - // Some ops, such as XlaHostCompute op or TPU Embedding ops, require - // replica id to be added as an op attribute to be used during - // execution. Handle such ops separately and add an integer attribute - // that represents replica id. - AddReplicaIdToOpsInReplicatedRegion(builder, &replica.body(), i); - - // Map aliased devices to explicit devices based on replica. - if (has_devices) { - replica.walk([&](tf_device::LaunchOp launch) { - if (auto device_by_replica = devices.getValue().get(launch.device())) - launch.setAttr( - kDeviceAttr, - device_by_replica.cast()[i].cast()); - }); - - if (auto tpu_replica_0 = - devices.getValue().get("TPU_REPLICATED_CORE_0")) { - int64_t device_ordinal = 0; - tensorflow::GetDeviceOrdinalFromDeviceString( - replicate_op.getLoc(), - tpu_replica_0.cast()[i].cast().getValue(), - &device_ordinal); - replica.walk([&](Operation* op) { - if (RequiresDeviceOrdinalAttribute(op)) - op->setAttr(kDeviceOrdinalAttr, - builder->getI64IntegerAttr(device_ordinal)); - }); - } - } + if (failed(UpdateRegionReplicateVariantOps(builder, replicate_op.getLoc(), + replica.body(), /*replica_id=*/i, + devices))) + return failure(); replicas.push_back(replica); } - return replicas; + return success(); } // Creates islands per replica from `tf_device.replicate` region and remap @@ -204,17 +210,18 @@ llvm::SmallVector ExpandReplicateIntoReplicas( // }) {device = "/DEVICE:3"} : () -> tensor // tf_executor.yield %a1, %b1 : tensor, tensor // } -void CreateIslandsFromReplicate(const Dialect* tf_dialect, - tf_executor::GraphOp graph_op, - tf_executor::IslandOp island_op, - tf_device::ReplicateOp replicate_op) { +LogicalResult CreateIslandsFromReplicate(const Dialect* tf_dialect, + tf_executor::GraphOp graph_op, + tf_executor::IslandOp island_op, + tf_device::ReplicateOp replicate_op) { OpBuilder builder(island_op); const int num_replicas = replicate_op.n().getLimitedValue(); // Create islands per replica. - llvm::SmallVector replicas = - ExpandReplicateIntoReplicas(tf_dialect, &builder, island_op, replicate_op, - num_replicas); + llvm::SmallVector replicas; + if (failed(ExpandReplicateIntoReplicas(tf_dialect, builder, island_op, + replicate_op, num_replicas, replicas))) + return failure(); // Collect all replica results. llvm::SmallVector replicas_outputs(replicate_op.getNumResults(), @@ -265,36 +272,41 @@ void CreateIslandsFromReplicate(const Dialect* tf_dialect, } island_op.erase(); + return success(); } -// Finds islands with a single `tf_device.replicate` and create individual -// islands per replica of the replicate. -void LowerSingleIslandReplicateToIslands(const Dialect* tf_dialect, - tf_executor::GraphOp graph_op, - tf_executor::IslandOp island_op) { - if (!island_op.WrapsSingleOp()) return; - - if (auto replicate_op = - llvm::dyn_cast(&island_op.GetBody().front())) - CreateIslandsFromReplicate(tf_dialect, graph_op, island_op, replicate_op); -} - -void ReplicateToIslandPass::runOnFunction() { +void ReplicateToIslandPass::runOnOperation() { + auto module = getOperation(); const Dialect* tf_dialect = getContext().getRegisteredDialect("tf"); if (!tf_dialect) { - signalPassFailure(); - getFunction().emitError() << "'tf' dialect is not registered"; + module.emitError() << "'tf' dialect is not registered"; + return signalPassFailure(); } - getFunction().walk([&](tf_executor::GraphOp graph_op) { - for (auto island_op : - llvm::make_early_inc_range(graph_op.getOps())) - LowerSingleIslandReplicateToIslands(tf_dialect, graph_op, island_op); + // Find islands with a single `tf_device.replicate` and create individual + // islands per replica of the replicate. + llvm::SmallVector replicate_op_islands; + module.walk([&](tf_executor::GraphOp graph_op) { + for (auto island_op : graph_op.getOps()) { + if (!island_op.WrapsSingleOp()) continue; + + if (isa(&island_op.GetBody().front())) + replicate_op_islands.push_back(island_op); + } }); + + for (tf_executor::IslandOp island_op : replicate_op_islands) { + auto graph_op = island_op.getParentOfType(); + auto replicate_op = + cast(island_op.GetBody().front()); + if (failed(CreateIslandsFromReplicate(tf_dialect, graph_op, island_op, + replicate_op))) + return signalPassFailure(); + } } } // anonymous namespace -std::unique_ptr> CreateReplicateToIslandPass() { +std::unique_ptr> CreateReplicateToIslandPass() { return std::make_unique(); } From 3fd7bac6ed8fc9edac3be94c4fde0fc72a630663 Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Thu, 6 Aug 2020 17:09:55 -0700 Subject: [PATCH 0604/1017] Fix tsan failure. Instead of creating a new threadpool in ReadElementsParallel, we can reuse the existing thread_pool_ field. This solves the tsan failure because now the destructor will block until the threads created in ReadElementsParallel exit. PiperOrigin-RevId: 325341982 Change-Id: I1107bde215a5384ded98633f5f46a2dde3ff7e23 --- tensorflow/core/kernels/data/BUILD | 1 - .../kernels/data/parallel_interleave_dataset_op.cc | 10 +++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD index 94cc31a8cb6..1365f8a1d31 100644 --- a/tensorflow/core/kernels/data/BUILD +++ b/tensorflow/core/kernels/data/BUILD @@ -623,7 +623,6 @@ tf_cc_test( name = "parallel_interleave_dataset_op_test", size = "small", srcs = ["parallel_interleave_dataset_op_test.cc"], - tags = ["notsan"], # TODO(b/147147071): Remove this tag once bug fix lands. deps = [ ":captured_function", ":dataset_test_base", diff --git a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc index 54ad8886a95..90dd5337c1d 100644 --- a/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc +++ b/tensorflow/core/kernels/data/parallel_interleave_dataset_op.cc @@ -41,6 +41,7 @@ limitations under the License. #include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/blocking_counter.h" #include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/stringprintf.h" #include "tensorflow/core/profiler/lib/traceme.h" #include "tensorflow/core/profiler/lib/traceme_encode.h" @@ -1342,12 +1343,10 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { IteratorContext* ctx, IteratorStateReader* reader, int64 size, const string& name, std::vector>* elements) { elements->resize(size); - std::unique_ptr threadpool = - ctx->CreateThreadPool(absl::StrCat("read_", name), size); Status s = Status::OK(); BlockingCounter counter(size); for (int idx = 0; idx < size; ++idx) { - threadpool->Schedule( + thread_pool_->Schedule( [this, ctx, reader, idx, name, &s, &counter, elements] { RecordStart(ctx); auto cleanup = gtl::MakeCleanup([this, ctx, &counter]() { @@ -1357,6 +1356,11 @@ class ParallelInterleaveDatasetOp::Dataset : public DatasetBase { std::shared_ptr elem; Status ret_status = ReadElement(ctx, reader, idx, name, &elem); mutex_lock l(*mu_); + if (cancelled_) { + s.Update( + errors::Cancelled("Cancelled in ReadElementsParallel")); + return; + } if (!ret_status.ok()) { s.Update(ret_status); return; From 0573f8cb6976c592eee660da9e4ce58e0c1eb0c0 Mon Sep 17 00:00:00 2001 From: bhack Date: Thu, 6 Aug 2020 23:32:37 +0000 Subject: [PATCH 0605/1017] Check input and axis params --- tensorflow/python/ops/array_ops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 5d68deb7ac1..8724ecebbfe 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -5316,12 +5316,12 @@ def quantize_and_dequantize( A `Tensor`. Each element is the result of quantizing and dequantizing the corresponding element of `input`. """ + with ops.name_scope(name, "quantize_and_dequantize", [input]) as name: + if not tensor_util.is_tensor(input): + input = ops.convert_to_tensor(input) if axis is None: axis = -1 - elif axis < 0: - if input.shape.ndims is None: - raise ValueError("input should have known rank to use negative axis.") - axis %= input.shape.ndims + axis = get_positive_axis(axis, input.shape.ndims) return gen_array_ops.quantize_and_dequantize_v2( input, From bcf052b88c480ec87b14fd1430f89f6bedfd6e7d Mon Sep 17 00:00:00 2001 From: Lucy Fox Date: Thu, 6 Aug 2020 17:29:29 -0700 Subject: [PATCH 0606/1017] Relax DynamicBroadcastInDim verifier when dimensions are dynamic. For input and output dimensions which must match, we shouldn't fail in the case where one dim is dynamic and the other is static. This is insufficient information to conclude a dimension mismatch. PiperOrigin-RevId: 325344738 Change-Id: Ifb7b95219aa97244a08c053d70cb82020afc4c48 --- .../mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc | 8 ++++--- tensorflow/compiler/mlir/hlo/tests/ops.mlir | 24 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc index de3f950c300..a1f0480f4fe 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc @@ -748,10 +748,12 @@ static LogicalResult Verify(DynamicBroadcastInDimOp op) { auto dimSize = operandType.getDimSize(i); auto resultDimSize = resultType.getDimSize(dimIndex); - if (dimSize != 1 && dimSize != resultDimSize) { + // Note: verifyCompatibleShapes doesn't consider size-1 broadcasting, so we + // add a manual check for this. + if (dimSize != 1 && failed(verifyCompatibleShape(dimSize, resultDimSize))) { return op.emitOpError( - llvm::formatv("size of operand dimension {0} ({1}) is not equal to " - "1 or size of result dimension {2} ({3})", + llvm::formatv("size of operand dimension {0} ({1}) is not compatible " + "with size of result dimension {2} ({3})", i, dimSize, dimIndex, resultDimSize)); } } diff --git a/tensorflow/compiler/mlir/hlo/tests/ops.mlir b/tensorflow/compiler/mlir/hlo/tests/ops.mlir index 212e79432b1..3443f21bc84 100644 --- a/tensorflow/compiler/mlir/hlo/tests/ops.mlir +++ b/tensorflow/compiler/mlir/hlo/tests/ops.mlir @@ -116,6 +116,30 @@ func @dynamic_broadcast_in_dim(%arg0: tensor, %shape: tensor<3xi64>) -> // ----- +// CHECK-LABEL: func @dynamic_broadcast_in_dim_unknown_dim +func @dynamic_broadcast_in_dim_unknown_dim(%arg0: tensor<32xf32>, %shape: tensor<3xi64>) -> tensor { + %0 = "mhlo.dynamic_broadcast_in_dim"(%arg0, %shape) {broadcast_dimensions = dense<[2]> : tensor<1xi64>} : (tensor<32xf32>, tensor<3xi64>) -> tensor + return %0 : tensor +} + +// ----- + +// CHECK-LABEL: func @dynamic_broadcast_in_dim_ok_dim +func @dynamic_broadcast_in_dim_ok_dim(%arg0: tensor<1xf32>, %shape: tensor<3xi64>) -> tensor<7x8x9xf32> { + %0 = "mhlo.dynamic_broadcast_in_dim"(%arg0, %shape) {broadcast_dimensions = dense<[2]> : tensor<1xi64>} : (tensor<1xf32>, tensor<3xi64>) -> tensor<7x8x9xf32> + return %0 : tensor<7x8x9xf32> +} + +// ----- + +func @dynamic_broadcast_in_dim_shape_mismatch(%arg0: tensor<32xf32>, %shape: tensor<3xi64>) -> tensor<7x8x9xf32> { + // expected-error@+1 {{size of operand dimension 0 (32) is not compatible with size of result dimension 2 (9)}} + %0 = "mhlo.dynamic_broadcast_in_dim"(%arg0, %shape) {broadcast_dimensions = dense<[2]> : tensor<1xi64>} : (tensor<32xf32>, tensor<3xi64>) -> tensor<7x8x9xf32> + return %0 : tensor<7x8x9xf32> +} + +// ----- + func @broadcast_in_dim_bad_dimension_rank(%arg0: tensor<1x2xi32>) -> tensor<1x2x3xi32> { // expected-error@+1 {{broadcast_dimensions has rank 2 instead of rank 1}} %0 = "mhlo.broadcast_in_dim"(%arg0) {broadcast_dimensions = dense<[[1,1],[1,1]]> : tensor<2x2xi64>} : (tensor<1x2xi32>) -> tensor<1x2x3xi32> From d9ea5051104b3580fee2d49c94be2ec45012672f Mon Sep 17 00:00:00 2001 From: Jay Shi Date: Thu, 6 Aug 2020 17:32:29 -0700 Subject: [PATCH 0607/1017] [tf.data] Record the number of times tf.data experiment applied to input pipelines. PiperOrigin-RevId: 325345128 Change-Id: I8de30c47f681a6f41e25e5ade4460f20a9d7bb5d --- tensorflow/core/framework/metrics.cc | 9 +++++++++ tensorflow/core/framework/metrics.h | 3 +++ tensorflow/core/kernels/data/dataset_utils.cc | 11 ----------- .../core/kernels/data/optimize_dataset_op.cc | 15 +++++++++++++++ 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/framework/metrics.cc b/tensorflow/core/framework/metrics.cc index 8cbfcd5342a..f5aff3a4e11 100644 --- a/tensorflow/core/framework/metrics.cc +++ b/tensorflow/core/framework/metrics.cc @@ -80,6 +80,11 @@ auto* tf_data_bytes_fetched_counter = monitoring::Counter<0>::New( auto* tf_data_elements_counter = monitoring::Counter<1>::New( "/tensorflow/data/elements", "tf.data elements", "name"); +auto* tf_data_experiment_counter = monitoring::Counter<1>::New( + "/tensorflow/data/experiment", + "The number of times tf.data experiment is applied to input pipelines.", + "name"); + auto* tf_data_fingerprint_counter = monitoring::Counter<1>::New( "/tensorflow/data/fingerprint", "tf.data fingerprint", "name"); @@ -179,6 +184,10 @@ void RecordTFDataBytesFetched(int64 num_bytes) { tf_data_bytes_fetched_counter->GetCell()->IncrementBy(num_bytes); } +void RecordTFDataExperiment(const string& name) { + tf_data_experiment_counter->GetCell(name)->IncrementBy(1); +} + void RecordTFDataFingerprint(const string& name) { tf_data_fingerprint_counter->GetCell(name)->IncrementBy(1); } diff --git a/tensorflow/core/framework/metrics.h b/tensorflow/core/framework/metrics.h index 7bc9a1bda0b..f7c90ce593e 100644 --- a/tensorflow/core/framework/metrics.h +++ b/tensorflow/core/framework/metrics.h @@ -56,6 +56,9 @@ monitoring::CounterCell* GetTFDataElementsCounter(const string& name); // Records the number of bytes fetched from tf.data.Dataset iterator. void RecordTFDataBytesFetched(int64 num_bytes); +// Records the number of times tf.data experiment is applied to input pipelines. +void RecordTFDataExperiment(const string& name); + // Records the time spent in ItertatorResource::GetNext() in microseconds. void RecordTFDataGetNextDuration(uint64 duration_us); diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc index 4151442d747..66de482467d 100644 --- a/tensorflow/core/kernels/data/dataset_utils.cc +++ b/tensorflow/core/kernels/data/dataset_utils.cc @@ -1018,17 +1018,6 @@ std::vector SelectOptimizations( } } - // Log the experiments that will be applied. - if (VLOG_IS_ON(1)) { - for (auto& pair : live_experiments) { - string experiment = pair.first; - if (std::find(optimizations_set.begin(), optimizations_set.end(), - experiment) != optimizations_set.end()) { - VLOG(1) << "The experiment \"" << experiment << "\" is applied."; - } - } - } - std::vector optimizations; optimizations.insert(optimizations.end(), optimizations_set.begin(), optimizations_set.end()); diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc index a0101435794..a566693ec3d 100644 --- a/tensorflow/core/kernels/data/optimize_dataset_op.cc +++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc @@ -101,6 +101,21 @@ void OptimizeDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input, job_name, opt_ins_raw, opt_outs_raw, live_experiments, optimizations_enabled, optimizations_disabled, optimizations_default, hash_func); + + // Log the experiments that will be applied. + if (!live_experiments.empty() && VLOG_IS_ON(1)) { + VLOG(1) << "The input pipeline is subject to tf.data experiment. " + "Please see `go/tf-data-experiments` for more details."; + + for (auto& pair : live_experiments) { + string experiment = pair.first; + if (std::find(optimizations.begin(), optimizations.end(), + experiment) != optimizations.end()) { + VLOG(1) << "The experiment \"" << experiment << "\" is applied."; + metrics::RecordTFDataExperiment(experiment); + } + } + } } } From 235dbc2dc26a00ebf6d2b1f3cba37cba9d548ffc Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Thu, 6 Aug 2020 17:49:37 -0700 Subject: [PATCH 0608/1017] Added info about supported image formats to DeviceInfo. storage_type_util cleaned from OpenCL API calls/structs. PiperOrigin-RevId: 325347475 Change-Id: I096636e4dd837ef9754df70caf37842c605c24f5 --- tensorflow/lite/delegates/gpu/cl/BUILD | 5 +- .../lite/delegates/gpu/cl/cl_context.cc | 39 +++++++++++++ tensorflow/lite/delegates/gpu/cl/cl_device.cc | 16 ++--- tensorflow/lite/delegates/gpu/cl/cl_device.h | 8 +-- .../lite/delegates/gpu/cl/cl_program.cc | 4 +- .../lite/delegates/gpu/cl/device_info.cc | 22 +++++++ .../lite/delegates/gpu/cl/device_info.h | 17 ++++++ .../lite/delegates/gpu/cl/environment.cc | 4 +- .../delegates/gpu/cl/inference_context.cc | 7 +-- .../gpu/cl/kernels/conv_buffer_1x1.cc | 4 +- .../gpu/cl/kernels/conv_constants.cc | 4 +- .../delegates/gpu/cl/kernels/conv_powervr.cc | 12 ++-- .../delegates/gpu/cl/kernels/conv_texture.cc | 6 +- .../delegates/gpu/cl/kernels/converter.cc | 8 +-- .../gpu/cl/kernels/convolution_transposed.cc | 4 +- .../cl/kernels/convolution_transposed_thin.cc | 2 +- .../gpu/cl/kernels/depthwise_conv_3x3.cc | 2 +- .../delegates/gpu/cl/kernels/elementwise.cc | 14 ++--- .../gpu/cl/kernels/fully_connected.cc | 2 +- .../delegates/gpu/cl/kernels/gpu_operation.cc | 6 +- .../lite/delegates/gpu/cl/kernels/util.cc | 4 +- .../lite/delegates/gpu/cl/kernels/winograd.cc | 4 +- .../gpu/cl/selectors/convolution_selector.cc | 6 +- .../convolution_transposed_selector.cc | 2 +- .../cl/selectors/dw_convolution_selector.cc | 4 +- .../cl/selectors/fully_connected_selector.cc | 2 +- .../gpu/cl/selectors/operation_selector.cc | 11 ++-- .../delegates/gpu/cl/storage_type_util.cc | 58 +++++++++---------- .../lite/delegates/gpu/cl/storage_type_util.h | 12 ++-- 29 files changed, 177 insertions(+), 112 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/BUILD b/tensorflow/lite/delegates/gpu/cl/BUILD index 66bcbc826ea..d6076e221bd 100644 --- a/tensorflow/lite/delegates/gpu/cl/BUILD +++ b/tensorflow/lite/delegates/gpu/cl/BUILD @@ -257,6 +257,7 @@ cc_library( srcs = ["device_info.cc"], hdrs = ["device_info.h"], deps = [ + "//tensorflow/lite/delegates/gpu/common:data_type", "@com_google_absl//absl/strings", ], ) @@ -468,11 +469,11 @@ cc_library( srcs = ["storage_type_util.cc"], hdrs = ["storage_type_util.h"], deps = [ - ":cl_context", - ":cl_device", + ":device_info", ":tensor_type", "//tensorflow/lite/delegates/gpu/common:data_type", "//tensorflow/lite/delegates/gpu/common:shape", + "//tensorflow/lite/delegates/gpu/common:util", ], ) diff --git a/tensorflow/lite/delegates/gpu/cl/cl_context.cc b/tensorflow/lite/delegates/gpu/cl/cl_context.cc index e697c78b692..9a8f404c46e 100644 --- a/tensorflow/lite/delegates/gpu/cl/cl_context.cc +++ b/tensorflow/lite/delegates/gpu/cl/cl_context.cc @@ -43,6 +43,44 @@ std::vector GetSupportedImage2DFormats(cl_context context, return result; } +bool IsEqualToImageFormat(cl_image_format image_format, DataType data_type, + int num_channels) { + return image_format.image_channel_data_type == + ToImageChannelType(data_type) && + image_format.image_channel_order == ToChannelOrder(num_channels); +} + +void AddSupportedImageFormats(cl_context context, DeviceInfo* info) { + auto supported_formats = + GetSupportedImage2DFormats(context, CL_MEM_READ_WRITE); + for (auto format : supported_formats) { + info->supports_r_f16_tex2d = + info->supports_r_f16_tex2d || + IsEqualToImageFormat(format, DataType::FLOAT16, 1); + info->supports_rg_f16_tex2d = + info->supports_rg_f16_tex2d || + IsEqualToImageFormat(format, DataType::FLOAT16, 2); + info->supports_rgb_f16_tex2d = + info->supports_rgb_f16_tex2d || + IsEqualToImageFormat(format, DataType::FLOAT16, 3); + info->supports_rgba_f16_tex2d = + info->supports_rgba_f16_tex2d || + IsEqualToImageFormat(format, DataType::FLOAT16, 4); + info->supports_r_f32_tex2d = + info->supports_r_f32_tex2d || + IsEqualToImageFormat(format, DataType::FLOAT32, 1); + info->supports_rg_f32_tex2d = + info->supports_rg_f32_tex2d || + IsEqualToImageFormat(format, DataType::FLOAT32, 2); + info->supports_rgb_f32_tex2d = + info->supports_rgb_f32_tex2d || + IsEqualToImageFormat(format, DataType::FLOAT32, 3); + info->supports_rgba_f32_tex2d = + info->supports_rgba_f32_tex2d || + IsEqualToImageFormat(format, DataType::FLOAT32, 4); + } +} + absl::Status CreateCLContext(const CLDevice& device, cl_context_properties* properties, CLContext* result) { @@ -55,6 +93,7 @@ absl::Status CreateCLContext(const CLDevice& device, absl::StrCat("Failed to create a compute context - ", CLErrorCodeToString(error_code))); } + AddSupportedImageFormats(context, &device.info_); *result = CLContext(context, true); return absl::OkStatus(); diff --git a/tensorflow/lite/delegates/gpu/cl/cl_device.cc b/tensorflow/lite/delegates/gpu/cl/cl_device.cc index b93bfb25ad1..16f5ce217e9 100644 --- a/tensorflow/lite/delegates/gpu/cl/cl_device.cc +++ b/tensorflow/lite/delegates/gpu/cl/cl_device.cc @@ -248,24 +248,24 @@ DeviceInfo DeviceInfoFromDeviceID(cl_device_id id) { } CLDevice::CLDevice(cl_device_id id, cl_platform_id platform_id) - : id_(id), platform_id_(platform_id), info_(DeviceInfoFromDeviceID(id)) {} + : info_(DeviceInfoFromDeviceID(id)), id_(id), platform_id_(platform_id) {} CLDevice::CLDevice(const CLDevice& device) - : id_(device.id_), platform_id_(device.platform_id_), info_(device.info_) {} + : info_(device.info_), id_(device.id_), platform_id_(device.platform_id_) {} CLDevice& CLDevice::operator=(const CLDevice& device) { if (this != &device) { + info_ = device.info_; id_ = device.id_; platform_id_ = device.platform_id_; - info_ = device.info_; } return *this; } CLDevice::CLDevice(CLDevice&& device) - : id_(device.id_), - platform_id_(device.platform_id_), - info_(std::move(device.info_)) { + : info_(std::move(device.info_)), + id_(device.id_), + platform_id_(device.platform_id_) { device.id_ = nullptr; device.platform_id_ = nullptr; } @@ -274,9 +274,9 @@ CLDevice& CLDevice::operator=(CLDevice&& device) { if (this != &device) { id_ = nullptr; platform_id_ = nullptr; + info_ = std::move(device.info_); std::swap(id_, device.id_); std::swap(platform_id_, device.platform_id_); - info_ = std::move(device.info_); } return *this; } @@ -368,7 +368,7 @@ bool CLDevice::IsAMD() const { return info_.IsAMD(); } bool CLDevice::IsIntel() const { return info_.IsIntel(); } bool CLDevice::SupportsOneLayerTextureArray() const { - return !IsAdreno() || info_.adreno_info.support_one_layer_texture_array; + return info_.SupportsOneLayerTextureArray(); } void CLDevice::DisableOneLayerTextureArray() { diff --git a/tensorflow/lite/delegates/gpu/cl/cl_device.h b/tensorflow/lite/delegates/gpu/cl/cl_device.h index 7e4792b0a53..e7cd274661d 100644 --- a/tensorflow/lite/delegates/gpu/cl/cl_device.h +++ b/tensorflow/lite/delegates/gpu/cl/cl_device.h @@ -46,9 +46,6 @@ class CLDevice { cl_platform_id platform() const { return platform_id_; } std::string GetPlatformVersion() const; - const DeviceInfo& GetInfo() const { return info_; } - const DeviceInfo* GetInfoPtr() const { return &info_; } - Vendor vendor() const { return info_.vendor; } OpenCLVersion cl_version() const { return info_.cl_version; } bool SupportsFP16() const; @@ -76,10 +73,13 @@ class CLDevice { bool SupportsOneLayerTextureArray() const; void DisableOneLayerTextureArray(); + // We update device info during context creation, so as supported texture + // formats can be requested from context only. + mutable DeviceInfo info_; + private: cl_device_id id_ = nullptr; cl_platform_id platform_id_ = nullptr; - DeviceInfo info_; }; absl::Status CreateDefaultGPUDevice(CLDevice* result); diff --git a/tensorflow/lite/delegates/gpu/cl/cl_program.cc b/tensorflow/lite/delegates/gpu/cl/cl_program.cc index 3b821dc3a5d..fd29ebec2d7 100644 --- a/tensorflow/lite/delegates/gpu/cl/cl_program.cc +++ b/tensorflow/lite/delegates/gpu/cl/cl_program.cc @@ -78,13 +78,13 @@ std::string CompilerOptionToString(const CLDevice& device, CompilerOptions option) { switch (option) { case CompilerOptions::ADRENO_FULL_SIMD_LINE: - if (device.GetInfo().adreno_info.gpu_version < 500) { + if (device.info_.adreno_info.gpu_version < 500) { return "-qcom-accelerate-16-bit"; } else { return "-qcom-accelerate-16-bit=true"; } case CompilerOptions::ADRENO_MORE_WAVES: - if (device.GetInfo().adreno_info.gpu_version >= 500) { + if (device.info_.adreno_info.gpu_version >= 500) { return "-qcom-accelerate-16-bit=false"; } else { return ""; diff --git a/tensorflow/lite/delegates/gpu/cl/device_info.cc b/tensorflow/lite/delegates/gpu/cl/device_info.cc index 7e0acb87ab7..d1ed69aa100 100644 --- a/tensorflow/lite/delegates/gpu/cl/device_info.cc +++ b/tensorflow/lite/delegates/gpu/cl/device_info.cc @@ -231,6 +231,28 @@ bool DeviceInfo::SupportsImage3D() const { return supports_image3d_writes; } +bool DeviceInfo::SupportsFloatImage2D(DataType data_type, int channels) const { + if (channels == 1) { + return data_type == DataType::FLOAT32 ? supports_r_f32_tex2d + : supports_r_f16_tex2d; + } else if (channels == 2) { + return data_type == DataType::FLOAT32 ? supports_rg_f32_tex2d + : supports_rg_f16_tex2d; + } else if (channels == 3) { + return data_type == DataType::FLOAT32 ? supports_rgb_f32_tex2d + : supports_rgb_f16_tex2d; + } else if (channels == 4) { + return data_type == DataType::FLOAT32 ? supports_rgba_f32_tex2d + : supports_rgba_f16_tex2d; + } else { + return false; + } +} + +bool DeviceInfo::SupportsOneLayerTextureArray() const { + return !IsAdreno() || adreno_info.support_one_layer_texture_array; +} + bool DeviceInfo::IsAdreno() const { return vendor == Vendor::kQualcomm; } bool DeviceInfo::IsAdreno3xx() const { diff --git a/tensorflow/lite/delegates/gpu/cl/device_info.h b/tensorflow/lite/delegates/gpu/cl/device_info.h index b13fe3df846..7123891ecf4 100644 --- a/tensorflow/lite/delegates/gpu/cl/device_info.h +++ b/tensorflow/lite/delegates/gpu/cl/device_info.h @@ -19,6 +19,8 @@ limitations under the License. #include #include +#include "tensorflow/lite/delegates/gpu/common/data_type.h" + // for use only in device_info.cc, but keep here to make tests int GetAdrenoGPUVersion(const std::string& gpu_version); @@ -131,6 +133,11 @@ struct DeviceInfo { bool SupportsImageBuffer() const; bool SupportsImage3D() const; + bool SupportsFloatImage2D(DataType data_type, int channels) const; + + // To track bug on some Adreno. b/131099086 + bool SupportsOneLayerTextureArray() const; + std::vector extensions; bool supports_fp16; bool supports_image3d_writes; @@ -157,6 +164,16 @@ struct DeviceInfo { bool supports_fp32_rtn; bool supports_fp16_rtn; + bool supports_r_f16_tex2d = false; + bool supports_rg_f16_tex2d = false; + bool supports_rgb_f16_tex2d = false; + bool supports_rgba_f16_tex2d = false; + + bool supports_r_f32_tex2d = false; + bool supports_rg_f32_tex2d = false; + bool supports_rgb_f32_tex2d = false; + bool supports_rgba_f32_tex2d = false; + AdrenoInfo adreno_info; MaliInfo mali_info; }; diff --git a/tensorflow/lite/delegates/gpu/cl/environment.cc b/tensorflow/lite/delegates/gpu/cl/environment.cc index c8b0b56978c..3d5546a8ebb 100644 --- a/tensorflow/lite/delegates/gpu/cl/environment.cc +++ b/tensorflow/lite/delegates/gpu/cl/environment.cc @@ -47,7 +47,7 @@ __kernel void main_function(__write_only image2d_array_t dst) { absl::Status CheckKernelSupportOfOneLayerTextureArray(Environment* env, bool* result) { // No bug on Adreno 6xx - if (env->device().GetInfo().adreno_info.gpu_version >= 600) { + if (env->device().info_.adreno_info.gpu_version >= 600) { *result = true; return absl::OkStatus(); } @@ -242,7 +242,7 @@ TensorStorageType GetFastestStorageType(const CLDevice& gpu) { } else if (gpu.IsPowerVR()) { return TensorStorageType::TEXTURE_2D; } else if (gpu.IsMali()) { - const MaliInfo mali_info = gpu.GetInfo().mali_info; + const MaliInfo mali_info = gpu.info_.mali_info; if (mali_info.IsMaliT8xx() || mali_info.IsBifrostGen3() || mali_info.IsValhall()) { return TensorStorageType::TEXTURE_2D; diff --git a/tensorflow/lite/delegates/gpu/cl/inference_context.cc b/tensorflow/lite/delegates/gpu/cl/inference_context.cc index 689b511bb5e..7802024302b 100644 --- a/tensorflow/lite/delegates/gpu/cl/inference_context.cc +++ b/tensorflow/lite/delegates/gpu/cl/inference_context.cc @@ -203,7 +203,7 @@ absl::Status InferenceContext::InitFromGraph( TuningParameters tuning_parameters; tuning_parameters.queue = env->profiling_queue(); - tuning_parameters.info = env->device().GetInfoPtr(); + tuning_parameters.info = &env->device().info_; if (create_info.hints.Check(ModelHints::kFastTuning)) { tuning_parameters.tuning_type = TuningType::FAST; } @@ -244,14 +244,13 @@ void InferenceContext::ReserveGraphTensors( if (graph.IsGraphInput(t->id) || graph.IsGraphOutput(t->id)) { if (shape.c < 4 && CanCreateTensorWithShape( - *creation_context.context, *creation_context.device, shape, + creation_context.device->info_, shape, TensorDescriptor{data_type, TensorStorageType::SINGLE_TEXTURE_2D, layout})) { storage_type = TensorStorageType::SINGLE_TEXTURE_2D; } } - storage_type = SelectBestStorageType(*creation_context.context, - *creation_context.device, shape, + storage_type = SelectBestStorageType(creation_context.device->info_, shape, storage_type, data_type, layout); tensor_reserver_.Add( t->id, {shape, TensorDescriptor{data_type, storage_type, layout}}); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc index de6021aa5fe..3216e2ef246 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc @@ -93,7 +93,7 @@ ConvBuffer1x1::ConvParams GetBestParams(const CLDevice& device, } bool can_use_flt8 = (shape.w * shape.b) % 2 == 0 && definition.precision != CalculationsPrecision::F32; - bool is_midgard = device.IsMali() && device.GetInfo().mali_info.IsMidgard(); + bool is_midgard = device.IsMali() && device.info_.mali_info.IsMidgard(); if (is_midgard) { if (can_use_flt8) { conv_params.element_size = 8; @@ -141,7 +141,7 @@ ConvBuffer1x1::ConvParams GetBestParams(const CLDevice& device, conv_params.element_size = 4; conv_params.block_size = int3(1, 1, 1); if (device.IsMali() && definition.precision == CalculationsPrecision::F16 && - device.GetInfo().compute_units_count <= 4) { + device.info_.compute_units_count <= 4) { conv_params.block_size.x *= 2; } return conv_params; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc index d5a2a56c19c..1ed900a2080 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_constants.cc @@ -271,7 +271,7 @@ bool IsConvConstantsSupported(const CLDevice& device, ? sizeof(float) : sizeof(half); const int filters_buffer_size = filters_count * float_size; - const int kConstantMaxSize = GetOptimalMaxConstantSize(device.GetInfo()); + const int kConstantMaxSize = GetOptimalMaxConstantSize(device.info_); const int flt4_registers = DivideRoundUp(w_shape.o, 4); return filters_buffer_size <= kConstantMaxSize && flt4_registers <= 8; } @@ -283,7 +283,7 @@ absl::Status CreateConvConstants(const CreationContext& creation_context, if (!IsConvConstantsSupported(*creation_context.device, definition, attr)) { return absl::InvalidArgumentError("ConvConstants doesn't supported"); } - *result = ConvConstants(definition, attr, creation_context.device->GetInfo()); + *result = ConvConstants(definition, attr, creation_context.device->info_); RETURN_IF_ERROR( result->UploadWeights(attr.weights, creation_context.context)); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc index f69368d1083..d65595d068c 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc @@ -718,7 +718,7 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( if (dst_shape) { int task_size = dst_shape->w * dst_shape->b * dst_shape->h * dst_depth; float task_size_per_cu = - static_cast(task_size) / device.GetInfo().compute_units_count; + static_cast(task_size) / device.info_.compute_units_count; int block_size = conv_params.block_size.x * conv_params.block_size.y * conv_params.block_size.z; float threads_per_cu = task_size_per_cu / block_size; @@ -844,7 +844,7 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( conv_params.block_size = int3(1, 1, 1); } conv_params.src_depth_loop_size = 1; - MaliInfo mali_info = device.GetInfo().mali_info; + MaliInfo mali_info = device.info_.mali_info; if (src_depth % 2 == 0 && block_size <= 2 && !mali_info.IsMidgard()) { conv_params.src_depth_loop_size = 2; } @@ -987,7 +987,7 @@ absl::Status CreateConvPowerVR(const CreationContext& creation_context, const Convolution2DAttributes& attr, ConvPowerVR* result, const BHWC* dst_shape) { *result = ConvPowerVR(definition, attr, *creation_context.device, dst_shape); - result->GenerateCode(creation_context.device->GetInfo()); + result->GenerateCode(creation_context.device->info_); return result->UploadData(attr.weights, attr.bias, creation_context.context); } @@ -996,7 +996,7 @@ absl::Status CreateConvPowerVR(const CreationContext& creation_context, const FullyConnectedAttributes& attr, ConvPowerVR* result, const BHWC* dst_shape) { *result = ConvPowerVR(definition, attr, *creation_context.device, dst_shape); - result->GenerateCode(creation_context.device->GetInfo()); + result->GenerateCode(creation_context.device->info_); return result->UploadData(attr.weights, attr.bias, creation_context.context); } @@ -1006,7 +1006,7 @@ absl::Status CreateConvPowerVRDynamicWeights( ConvPowerVR* result, const BHWC* dst_shape) { *result = ConvPowerVR(definition, attr, weights_shape, *creation_context.device, dst_shape); - result->GenerateCode(creation_context.device->GetInfo()); + result->GenerateCode(creation_context.device->info_); return result->UploadBias(attr.bias, creation_context.context); } @@ -1017,7 +1017,7 @@ absl::Status CreateConvPowerVRWino4x4To6x6( *result = ConvPowerVR(definition); result->conv_params_ = result->GuessBestParamsWinograd( *creation_context.device, definition, attr, dst_shape); - result->GenerateCode(creation_context.device->GetInfo()); + result->GenerateCode(creation_context.device->info_); return result->UploadDataForWinograd4x4To6x6( attr.weights, *creation_context.device, creation_context.context); } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc index 88035556c86..581c8056ced 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc @@ -430,7 +430,7 @@ absl::Status CreateConvTexture(const CreationContext& creation_context, const Convolution2DAttributes& attr, ConvTexture* result) { *result = ConvTexture(definition, attr); - result->GenerateCode(creation_context.device->GetInfo()); + result->GenerateCode(creation_context.device->info_); return result->UploadData(attr.weights, attr.bias, creation_context.context); } @@ -439,7 +439,7 @@ absl::Status CreateConvTexture(const CreationContext& creation_context, const FullyConnectedAttributes& attr, ConvTexture* result) { *result = ConvTexture(definition); - result->GenerateCode(creation_context.device->GetInfo()); + result->GenerateCode(creation_context.device->info_); return result->UploadData(attr.weights, attr.bias, creation_context.context); } @@ -449,7 +449,7 @@ absl::Status CreateConvTextureWino4x4To6x6( *result = ConvTexture(definition); result->different_weights_for_height_ = true; result->block_size_ = {4, 1, 2}; - result->GenerateCode(creation_context.device->GetInfo()); + result->GenerateCode(creation_context.device->info_); return result->UploadDataForWinograd4x4To6x6( attr.weights, *creation_context.device, creation_context.context); } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/converter.cc b/tensorflow/lite/delegates/gpu/cl/kernels/converter.cc index bd5aaed8bc3..d52efb43a08 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/converter.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/converter.cc @@ -152,8 +152,8 @@ __kernel void from_tensor()" + context_ = &environment->context(); shape_ = BHWC(input_def.dimensions.b, input_def.dimensions.h, input_def.dimensions.w, input_def.dimensions.c); - RETURN_IF_ERROR(args_.TransformToCLCode(environment->device().GetInfo(), {}, - &shader_src)); + RETURN_IF_ERROR( + args_.TransformToCLCode(environment->device().info_, {}, &shader_src)); return environment->program_cache()->GetOrCreateCLKernel( shader_src, "from_tensor", environment->context(), environment->device(), &kernel_); @@ -272,8 +272,8 @@ __kernel void to_tensor()" + context_ = &environment->context(); shape_ = BHWC(output_def.dimensions.b, output_def.dimensions.h, output_def.dimensions.w, output_def.dimensions.c); - RETURN_IF_ERROR(args_.TransformToCLCode(environment->device().GetInfo(), {}, - &shader_src)); + RETURN_IF_ERROR( + args_.TransformToCLCode(environment->device().info_, {}, &shader_src)); return environment->program_cache()->GetOrCreateCLKernel( shader_src, "to_tensor", environment->context(), environment->device(), &kernel_); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc index a139b3affc9..c6eba691306 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc @@ -360,8 +360,8 @@ absl::Status CreateConvolutionTransposed( const CreationContext& creation_context, const OperationDef& definition, const ConvolutionTransposedAttributes& attr, ConvolutionTransposed* result) { - *result = ConvolutionTransposed(definition, attr, - creation_context.device->GetInfo()); + *result = + ConvolutionTransposed(definition, attr, creation_context.device->info_); RETURN_IF_ERROR( result->UploadWeights(attr.weights, creation_context.context)); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_thin.cc b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_thin.cc index 2268313a867..54fd5396869 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_thin.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_thin.cc @@ -175,7 +175,7 @@ absl::Status CreateConvolutionTransposedThin( "ConvolutionTransposedThin doesn't support this attributes"); } *result = ConvolutionTransposedThin(definition, attr, - creation_context.device->GetInfo()); + creation_context.device->info_); RETURN_IF_ERROR( result->UploadData(attr.weights, attr.bias, creation_context.context)); return absl::OkStatus(); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc index e171231fc0a..f0213cda805 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc @@ -330,7 +330,7 @@ absl::Status CreateDepthwiseConv3x3( bool local_mem_uploads = weights_are_buffer && creation_context.device->IsPowerVR(); *result = DepthwiseConv3x3(definition, weights_are_buffer, local_mem_uploads, - creation_context.device->GetInfo()); + creation_context.device->info_); return result->UploadWeightsAndBiases(attr.weights, attr.bias, creation_context.context); } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc index f735f1aa047..7d46ae4a109 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/elementwise.cc @@ -166,10 +166,9 @@ absl::Status CreateElementwiseTwoInput( const tflite::gpu::Tensor& constant_tensor, bool swap_inputs, GPUOperation* result) { const BHWC shape = BHWC(1, 1, 1, constant_tensor.shape.v); - TensorStorageType storage_type = - SelectBestStorageType(*creation_context.context, *creation_context.device, - shape, definition.GetPrimaryStorageType(), - definition.GetDataType(), Layout::HWC); + TensorStorageType storage_type = SelectBestStorageType( + creation_context.device->info_, shape, definition.GetPrimaryStorageType(), + definition.GetDataType(), Layout::HWC); TensorDescriptor desc{definition.GetDataType(), storage_type, Layout::HWC}; Tensor gpu_tensor; RETURN_IF_ERROR(CreateTensor(*creation_context.context, @@ -205,10 +204,9 @@ absl::Status CreateElementwiseTwoInput( bool swap_inputs, GPUOperation* result) { const BHWC shape = BHWC(1, constant_tensor.shape.h, constant_tensor.shape.w, constant_tensor.shape.c); - TensorStorageType storage_type = - SelectBestStorageType(*creation_context.context, *creation_context.device, - shape, definition.GetPrimaryStorageType(), - definition.GetDataType(), Layout::HWC); + TensorStorageType storage_type = SelectBestStorageType( + creation_context.device->info_, shape, definition.GetPrimaryStorageType(), + definition.GetDataType(), Layout::HWC); TensorDescriptor desc{definition.GetDataType(), storage_type, Layout::HWC}; Tensor gpu_tensor; RETURN_IF_ERROR(CreateTensor(*creation_context.context, diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/fully_connected.cc b/tensorflow/lite/delegates/gpu/cl/kernels/fully_connected.cc index 2ab0284febe..ec18fa9f6e2 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/fully_connected.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/fully_connected.cc @@ -114,7 +114,7 @@ absl::Status CreateFullyConnected(const CreationContext& creation_context, const OperationDef& definition, const FullyConnectedAttributes& attr, FullyConnected* result) { - *result = FullyConnected(definition, creation_context.device->GetInfo()); + *result = FullyConnected(definition, creation_context.device->info_); RETURN_IF_ERROR( result->UploadWeights(attr.weights, creation_context.context)); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc index 7260048c6d3..97c72c1269d 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc @@ -227,7 +227,7 @@ absl::Status GPUOperation::Compile(const CreationContext& creation_context) { RETURN_IF_ERROR( MergeOperations(linked_operations_, &args_, &element_wise_code)); RETURN_IF_ERROR(args_.TransformToCLCode( - creation_context.device->GetInfo(), + creation_context.device->info_, {{dst_tensors_names_[0], element_wise_code}}, &code)); code = absl::Substitute(code, args_.GetListOfArgs()); RETURN_IF_ERROR(creation_context.cache->GetOrCreateCLKernel( @@ -238,13 +238,13 @@ absl::Status GPUOperation::Compile(const CreationContext& creation_context) { RETURN_IF_ERROR( MergeOperations(linked_operations_, &args_, &element_wise_code)); RETURN_IF_ERROR(args_.TransformToCLCode( - creation_context.device->GetInfo(), + creation_context.device->info_, {{dst_tensors_names_[0], element_wise_code}}, &code_)); RETURN_IF_ERROR(creation_context.cache->GetOrCreateCLKernel( code_, "main_function", compiler_options_, *creation_context.context, *creation_context.device, &kernel_)); } - return PostCompileCheck(creation_context.device->GetInfo()); + return PostCompileCheck(creation_context.device->info_); } int3 GPUOperation::GetGridSize() const { diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/util.cc b/tensorflow/lite/delegates/gpu/cl/kernels/util.cc index 3fe4ffb4acd..d907c0210b7 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/util.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/util.cc @@ -117,7 +117,7 @@ int GetRecommendedBlockSizeForConv(const CLDevice& device, CalculationsPrecision precision, int task_size) { const float task_size_per_cu = - task_size / static_cast(device.GetInfo().compute_units_count); + task_size / static_cast(device.info_.compute_units_count); int block_size = 1; float threshold_1 = FLT_MAX; float threshold_2 = FLT_MAX; @@ -125,7 +125,7 @@ int GetRecommendedBlockSizeForConv(const CLDevice& device, if (!device.IsMali()) { return 1; } - MaliInfo mali_info = device.GetInfo().mali_info; + MaliInfo mali_info = device.info_.mali_info; switch (precision) { case CalculationsPrecision::F16: if (mali_info.IsBifrostGen1()) { diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc b/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc index 4c3e8ddba05..698599a5bbd 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc @@ -303,7 +303,7 @@ absl::Status CreateWinograd4x4To36(const CreationContext& creation_context, const Padding2D& padding, Winograd4x4To36* result) { *result = - Winograd4x4To36(definition, padding, creation_context.device->GetInfo()); + Winograd4x4To36(definition, padding, creation_context.device->info_); return result->UploadBt(creation_context.context); } @@ -502,7 +502,7 @@ absl::Status CreateWinograd36To4x4( const CreationContext& creation_context, const OperationDef& definition, const tflite::gpu::Tensor& biases, Winograd36To4x4* result) { - *result = Winograd36To4x4(definition, creation_context.device->GetInfo()); + *result = Winograd36To4x4(definition, creation_context.device->info_); TensorLinearDescriptor desc; desc.storage_type = LinearStorageType::TEXTURE_2D; desc.element_type = definition.GetDataType(); diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/convolution_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/convolution_selector.cc index b577757057e..4a97bdddd09 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/convolution_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/convolution_selector.cc @@ -167,7 +167,7 @@ absl::Status SelectConvolution(const Convolution2DAttributes& attr, const CreationContext& creation_context, const OperationDef& op_def, ModelHints hints, std::unique_ptr* ptr) { - const auto& device_info = creation_context.device->GetInfo(); + const auto& device_info = creation_context.device->info_; if (device_info.IsAdreno()) { return SelectConvolutionAdreno(attr, dst_shape, creation_context, op_def, hints, ptr); @@ -190,7 +190,7 @@ absl::Status SelectConvolutionForWinograd( const Convolution2DAttributes& attr, const BHWC& dst_shape, const CreationContext& creation_context, const OperationDef& op_def, ModelHints hints, std::unique_ptr* ptr) { - const auto& device_info = creation_context.device->GetInfo(); + const auto& device_info = creation_context.device->info_; if (device_info.IsAdreno()) { return SelectConvolutionWinogradAdreno(attr, dst_shape, creation_context, op_def, hints, ptr); @@ -215,7 +215,7 @@ absl::Status SelectConvolutionWithDynamicWeights( const BHWC& dst_shape, const CreationContext& creation_context, const OperationDef& op_def, ModelHints hints, std::unique_ptr* ptr, ConvWeightsDescription* weights_desc) { - const auto& device_info = creation_context.device->GetInfo(); + const auto& device_info = creation_context.device->info_; if (device_info.IsAdreno()) { return SelectConvolutionDynamicWeightsAdreno(attr, weights_shape, dst_shape, creation_context, op_def, diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/convolution_transposed_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/convolution_transposed_selector.cc index 56864f2c575..c00d9392702 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/convolution_transposed_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/convolution_transposed_selector.cc @@ -105,7 +105,7 @@ absl::Status SelectConvolutionTransposed( const ConvolutionTransposedAttributes& attr, const CreationContext& creation_context, const OperationDef& op_def, std::unique_ptr* ptr) { - const auto& device_info = creation_context.device->GetInfo(); + const auto& device_info = creation_context.device->info_; if (device_info.IsAdreno()) { return SelectConvolutionTransposedAdreno(attr, creation_context, op_def, ptr); diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/dw_convolution_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/dw_convolution_selector.cc index fafd9078f6f..b89f271365f 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/dw_convolution_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/dw_convolution_selector.cc @@ -69,7 +69,7 @@ absl::Status SelectDWConvolutionMali( const auto storage_type = op_def.src_tensors[0].storage_type; bool buffer_type = storage_type == TensorStorageType::BUFFER || storage_type == TensorStorageType::IMAGE_BUFFER; - MaliInfo mali_info = creation_context.device->GetInfo().mali_info; + MaliInfo mali_info = creation_context.device->info_.mali_info; if (IsDepthwiseConv3x3Supported(attr) && !mali_info.IsMidgard() && !buffer_type && op_def.precision != CalculationsPrecision::F32) { DepthwiseConv3x3 dw_conv; @@ -90,7 +90,7 @@ absl::Status SelectDWConvolution(const DepthwiseConvolution2DAttributes& attr, const CreationContext& creation_context, const OperationDef& op_def, std::unique_ptr* ptr) { - const auto& device_info = creation_context.device->GetInfo(); + const auto& device_info = creation_context.device->info_; if (device_info.IsAdreno()) { return SelectDWConvolutionAdreno(attr, creation_context, op_def, ptr); } else if (device_info.IsPowerVR()) { diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/fully_connected_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/fully_connected_selector.cc index cb967e45b52..0df8e243da3 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/fully_connected_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/fully_connected_selector.cc @@ -104,7 +104,7 @@ absl::Status SelectFullyConnected(const FullyConnectedAttributes& attr, const CreationContext& creation_context, const OperationDef& op_def, int batch_size, std::unique_ptr* ptr) { - const auto& device_info = creation_context.device->GetInfo(); + const auto& device_info = creation_context.device->info_; if (device_info.IsAdreno()) { return SelectFullyConnectedAdreno(attr, creation_context, op_def, batch_size, ptr); diff --git a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc index 5661c3d0a37..b257e5a85da 100644 --- a/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc +++ b/tensorflow/lite/delegates/gpu/cl/selectors/operation_selector.cc @@ -75,14 +75,14 @@ absl::Status WinogradFromNode(const CreationContext& creation_context, const BHWC shape_1{input_shape.b, 36, tiles_x * tiles_y, output_shape.c}; TensorDescriptor td_0; td_0.storage_type = SelectBestStorageType( - *creation_context.context, *creation_context.device, shape_0, + creation_context.device->info_, shape_0, op_def.src_tensors[0].storage_type, op_def.src_tensors[0].data_type, op_def.src_tensors[0].layout); td_0.data_type = op_def.src_tensors[0].data_type; td_0.layout = op_def.src_tensors[0].layout; TensorDescriptor td_1; td_1.storage_type = SelectBestStorageType( - *creation_context.context, *creation_context.device, shape_1, + creation_context.device->info_, shape_1, op_def.src_tensors[0].storage_type, op_def.src_tensors[0].data_type, op_def.src_tensors[0].layout); td_1.data_type = op_def.src_tensors[0].data_type; @@ -175,7 +175,7 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, channels[i] = inputs[i]->tensor.shape.c; } return SelectConcat(attr, channels, op_def, - creation_context.device->GetInfo(), gpu_op); + creation_context.device->info_, gpu_op); } case OperationType::CONVOLUTION_2D: { auto attr = @@ -248,7 +248,7 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, inputs[0]->tensor.shape.b, gpu_op); } case OperationType::LSTM: { - SelectLSTM(op_def, creation_context.device->GetInfo(), gpu_op); + SelectLSTM(op_def, creation_context.device->info_, gpu_op); return absl::OkStatus(); } case OperationType::MAX_UNPOOLING_2D: { @@ -259,8 +259,7 @@ absl::Status GPUOperationFromNode(const CreationContext& creation_context, } case OperationType::MEAN: { auto attr = absl::any_cast(node.operation.attributes); - return SelectMean(attr, op_def, creation_context.device->GetInfo(), - gpu_op); + return SelectMean(attr, op_def, creation_context.device->info_, gpu_op); } case OperationType::MEAN_STDDEV_NORMALIZATION: { MeanStdDevNormalization operation = CreateMeanStdDevNormalization(op_def); diff --git a/tensorflow/lite/delegates/gpu/cl/storage_type_util.cc b/tensorflow/lite/delegates/gpu/cl/storage_type_util.cc index 755da0c7619..ddcb65e07f9 100644 --- a/tensorflow/lite/delegates/gpu/cl/storage_type_util.cc +++ b/tensorflow/lite/delegates/gpu/cl/storage_type_util.cc @@ -15,18 +15,16 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/cl/storage_type_util.h" -#include "tensorflow/lite/delegates/gpu/cl/cl_context.h" -#include "tensorflow/lite/delegates/gpu/cl/cl_device.h" #include "tensorflow/lite/delegates/gpu/cl/tensor_type.h" #include "tensorflow/lite/delegates/gpu/common/data_type.h" #include "tensorflow/lite/delegates/gpu/common/shape.h" +#include "tensorflow/lite/delegates/gpu/common/util.h" namespace tflite { namespace gpu { namespace cl { -bool CanCreateTensorWithShape(const CLContext& context, const CLDevice& device, - const BHWDC& shape, +bool CanCreateTensorWithShape(const DeviceInfo& device_info, const BHWDC& shape, const TensorDescriptor& descriptor) { const int slices = DivideRoundUp(shape.c, 4); switch (descriptor.storage_type) { @@ -35,64 +33,60 @@ bool CanCreateTensorWithShape(const CLContext& context, const CLDevice& device, 4 * (descriptor.data_type == DataType::FLOAT32 ? 4 : 2); const int buffer_size = shape.b * shape.w * shape.h * shape.d * slices * flt4_size; - return buffer_size <= device.GetInfo().buffer_max_size; + return buffer_size <= device_info.buffer_max_size; } case TensorStorageType::IMAGE_BUFFER: return shape.b * shape.w * shape.h * shape.d * slices <= - device.GetInfo().image_buffer_max_size; + device_info.image_buffer_max_size; case TensorStorageType::TEXTURE_3D: - if (device.cl_version() < OpenCLVersion::CL_1_2 && slices == 1) { + if (device_info.cl_version < OpenCLVersion::CL_1_2 && slices == 1) { // clCreateImage3D (that used in CL 1.0/1.1) can not create image with // depth = 1 by specification; return false; } - return shape.w * shape.b <= device.GetInfo().image3d_max_width && - shape.h <= device.GetInfo().image3d_max_height && - slices * shape.d <= device.GetInfo().image3d_max_depth; + return shape.w * shape.b <= device_info.image3d_max_width && + shape.h <= device_info.image3d_max_height && + slices * shape.d <= device_info.image3d_max_depth; case TensorStorageType::TEXTURE_ARRAY: // Bug on some Adreno. b/131099086 - if (slices == 1 && !device.SupportsOneLayerTextureArray()) { + if (slices == 1 && !device_info.SupportsOneLayerTextureArray()) { return false; } - return shape.w * shape.b <= device.GetInfo().image2d_max_width && - shape.h <= device.GetInfo().image2d_max_height && - slices * shape.d <= device.GetInfo().image_array_max_layers; + return shape.w * shape.b <= device_info.image2d_max_width && + shape.h <= device_info.image2d_max_height && + slices * shape.d <= device_info.image_array_max_layers; case TensorStorageType::TEXTURE_2D: - return shape.w * shape.b * shape.d <= - device.GetInfo().image2d_max_width && - shape.h * slices <= device.GetInfo().image2d_max_height; + return shape.w * shape.b * shape.d <= device_info.image2d_max_width && + shape.h * slices <= device_info.image2d_max_height; case TensorStorageType::SINGLE_TEXTURE_2D: return shape.c <= 4 && - context.IsFloatTexture2DSupported(shape.c, descriptor.data_type) && - shape.w * shape.b * shape.d <= - device.GetInfo().image2d_max_width && - shape.h <= device.GetInfo().image2d_max_height; + device_info.SupportsFloatImage2D(descriptor.data_type, shape.c) && + shape.w * shape.b * shape.d <= device_info.image2d_max_width && + shape.h <= device_info.image2d_max_height; default: return false; } } -bool CanCreateTensorWithShape(const CLContext& context, const CLDevice& device, - const BHWC& shape, +bool CanCreateTensorWithShape(const DeviceInfo& device_info, const BHWC& shape, const TensorDescriptor& descriptor) { const BHWDC shape5D(shape.b, shape.h, shape.w, 1, shape.c); - return CanCreateTensorWithShape(context, device, shape5D, descriptor); + return CanCreateTensorWithShape(device_info, shape5D, descriptor); } -TensorStorageType SelectBestStorageType(const CLContext& context, - const CLDevice& device, +TensorStorageType SelectBestStorageType(const DeviceInfo& device_info, const BHWC& shape, const TensorStorageType& desired, const DataType& data_type, const Layout& layout) { - if (CanCreateTensorWithShape(context, device, shape, + if (CanCreateTensorWithShape(device_info, shape, TensorDescriptor{data_type, desired, layout})) { return desired; } auto GetBestTypeAfterTextureArray = [&]() { - if (device.SupportsImageBuffer() && + if (device_info.SupportsImageBuffer() && CanCreateTensorWithShape( - context, device, shape, + device_info, shape, TensorDescriptor{data_type, TensorStorageType::IMAGE_BUFFER, layout})) { return TensorStorageType::IMAGE_BUFFER; @@ -101,9 +95,9 @@ TensorStorageType SelectBestStorageType(const CLContext& context, } }; auto GetBestTypeAfterTexture2D = [&]() { - if (device.SupportsTextureArray() && + if (device_info.SupportsTextureArray() && CanCreateTensorWithShape( - context, device, shape, + device_info, shape, TensorDescriptor{data_type, TensorStorageType::TEXTURE_ARRAY, layout})) { return TensorStorageType::TEXTURE_ARRAY; @@ -113,7 +107,7 @@ TensorStorageType SelectBestStorageType(const CLContext& context, }; auto GetBestTypeAfterTexture3D = [&]() { if (CanCreateTensorWithShape( - context, device, shape, + device_info, shape, TensorDescriptor{data_type, TensorStorageType::TEXTURE_2D, layout})) { return TensorStorageType::TEXTURE_2D; diff --git a/tensorflow/lite/delegates/gpu/cl/storage_type_util.h b/tensorflow/lite/delegates/gpu/cl/storage_type_util.h index 87fc2206e81..a8a82008461 100644 --- a/tensorflow/lite/delegates/gpu/cl/storage_type_util.h +++ b/tensorflow/lite/delegates/gpu/cl/storage_type_util.h @@ -16,8 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_LITE_DELEGATES_GPU_CL_STORAGE_TYPE_UTIL_H_ #define TENSORFLOW_LITE_DELEGATES_GPU_CL_STORAGE_TYPE_UTIL_H_ -#include "tensorflow/lite/delegates/gpu/cl/cl_context.h" -#include "tensorflow/lite/delegates/gpu/cl/cl_device.h" +#include "tensorflow/lite/delegates/gpu/cl/device_info.h" #include "tensorflow/lite/delegates/gpu/cl/tensor_type.h" #include "tensorflow/lite/delegates/gpu/common/data_type.h" #include "tensorflow/lite/delegates/gpu/common/shape.h" @@ -26,16 +25,13 @@ namespace tflite { namespace gpu { namespace cl { -bool CanCreateTensorWithShape(const CLContext& context, const CLDevice& device, - const BHWDC& shape, +bool CanCreateTensorWithShape(const DeviceInfo& device_info, const BHWDC& shape, const TensorDescriptor& descriptor); -bool CanCreateTensorWithShape(const CLContext& context, const CLDevice& device, - const BHWC& shape, +bool CanCreateTensorWithShape(const DeviceInfo& device_info, const BHWC& shape, const TensorDescriptor& descriptor); -TensorStorageType SelectBestStorageType(const CLContext& context, - const CLDevice& device, +TensorStorageType SelectBestStorageType(const DeviceInfo& device_info, const BHWC& shape, const TensorStorageType& desired, const DataType& data_type, From 3e5a78fa1b3854e536587a94514ac42b8b621225 Mon Sep 17 00:00:00 2001 From: bhack Date: Fri, 7 Aug 2020 02:59:42 +0200 Subject: [PATCH 0609/1017] Else fix --- tensorflow/python/ops/array_ops.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 8724ecebbfe..9875342730c 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -5321,7 +5321,8 @@ def quantize_and_dequantize( input = ops.convert_to_tensor(input) if axis is None: axis = -1 - axis = get_positive_axis(axis, input.shape.ndims) + else: + axis = get_positive_axis(axis, input.shape.ndims) return gen_array_ops.quantize_and_dequantize_v2( input, From cfb79f71512a9e79e470948fa25c62da985de43a Mon Sep 17 00:00:00 2001 From: Advait Jain Date: Thu, 6 Aug 2020 17:50:20 -0700 Subject: [PATCH 0610/1017] Rollback of 0c2b2a3063dc4334b6 to allow for backwards compatibility for some more time. Note that the kernel tests will still fail unless the kernel implementations are updated to use TfLiteEvalTensors. PiperOrigin-RevId: 325347554 Change-Id: I0cb002257f0b8a807accb50b365d727eda579f3c --- tensorflow/lite/micro/BUILD | 1 - tensorflow/lite/micro/kernels/BUILD | 5 ---- .../micro/kernels/xtensa_hifimini/svdf.cc | 6 ++-- tensorflow/lite/micro/micro_interpreter.cc | 20 ++++++++----- tensorflow/lite/micro/test_helpers.cc | 30 ++++++++----------- 5 files changed, 29 insertions(+), 33 deletions(-) diff --git a/tensorflow/lite/micro/BUILD b/tensorflow/lite/micro/BUILD index a8fec96c3e3..9b3d0d623cc 100644 --- a/tensorflow/lite/micro/BUILD +++ b/tensorflow/lite/micro/BUILD @@ -87,7 +87,6 @@ cc_library( "//tensorflow/lite/kernels:kernel_util", "//tensorflow/lite/kernels/internal:compatibility", "//tensorflow/lite/kernels/internal:tensor", - "//tensorflow/lite/micro/kernels:kernel_util", "//tensorflow/lite/schema:schema_fbs", "@flatbuffers//:runtime_cc", ], diff --git a/tensorflow/lite/micro/kernels/BUILD b/tensorflow/lite/micro/kernels/BUILD index 76ad03991e3..dcf2337aa24 100644 --- a/tensorflow/lite/micro/kernels/BUILD +++ b/tensorflow/lite/micro/kernels/BUILD @@ -518,11 +518,6 @@ cc_library( "kernel_util.cc", ], hdrs = ["kernel_util.h"], - visibility = [ - # Needed for micro:test_helpers but visibility can not be finer-grained - # than a package. - ":micro_top_level", - ], deps = [ "//tensorflow/lite/c:common", "//tensorflow/lite/kernels/internal:compatibility", diff --git a/tensorflow/lite/micro/kernels/xtensa_hifimini/svdf.cc b/tensorflow/lite/micro/kernels/xtensa_hifimini/svdf.cc index 00ee9b2e809..545e91bab3d 100644 --- a/tensorflow/lite/micro/kernels/xtensa_hifimini/svdf.cc +++ b/tensorflow/lite/micro/kernels/xtensa_hifimini/svdf.cc @@ -343,7 +343,7 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { TF_LITE_ENSURE_EQ(context, activation_state->dims->data[1], memory_size * num_filters); - TF_LITE_ENSURE_EQ(context, NumInputs(node), 5); + TF_LITE_ENSURE_EQ(context, node->inputs->size, 5); TF_LITE_ENSURE_EQ(context, weights_feature->type, kTfLiteInt8); TF_LITE_ENSURE_EQ(context, weights_time->type, kTfLiteInt16); TF_LITE_ENSURE_EQ(context, activation_state->type, kTfLiteInt16); @@ -398,7 +398,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { const TfLiteEvalTensor* weights_time = tflite::micro::GetEvalInput(context, node, kWeightsTimeTensor); const TfLiteEvalTensor* bias = - tflite::micro::GetEvalInput(context, node, kBiasTensor); + (NumInputs(node) == 5) + ? tflite::micro::GetEvalInput(context, node, kBiasTensor) + : nullptr; TfLiteEvalTensor* activation_state = tflite::micro::GetMutableEvalInput( context, node, kInputActivationStateTensor); TfLiteEvalTensor* output = diff --git a/tensorflow/lite/micro/micro_interpreter.cc b/tensorflow/lite/micro/micro_interpreter.cc index 1c6ebd5953e..8c2f8e031d8 100644 --- a/tensorflow/lite/micro/micro_interpreter.cc +++ b/tensorflow/lite/micro/micro_interpreter.cc @@ -166,7 +166,9 @@ void MicroInterpreter::Init(tflite::Profiler* profiler) { context_.impl_ = static_cast(&context_helper_); context_.ReportError = context_helper_.ReportOpError; + context_.GetTensor = context_helper_.GetTensor; context_.GetEvalTensor = context_helper_.GetEvalTensor; + context_.recommended_num_threads = 1; context_.profiler = profiler; initialization_status_ = kTfLiteOk; @@ -275,12 +277,10 @@ TfLiteStatus MicroInterpreter::AllocateTensors() { } context_helper_.SetNodeIndex(-1); - // RequestScratchBufferInArena and GetTensor (with associated TempAllocation) - // are also available in Prepare stage. - context_.GetTensor = context_helper_.GetTensor; + // Both AllocatePersistentBuffer and RequestScratchBufferInArena is + // available in Prepare stage. context_.RequestScratchBufferInArena = context_helper_.RequestScratchBufferInArena; - for (size_t i = 0; i < subgraph_->operators()->size(); ++i) { // Set node idx to annotate the lifetime for scratch buffers. context_helper_.SetNodeIndex(i); @@ -300,13 +300,11 @@ TfLiteStatus MicroInterpreter::AllocateTensors() { } context_helper_.SetNodeIndex(-1); - // Prepare is done, we're ready for Invoke. Memory allocation and full - // TfLiteTensors (via GetTensor) are no longer allowed. Kernels can only fetch - // scratch buffers via GetScratchBuffer. + // Prepare is done, we're ready for Invoke. Memory allocation is no longer + // allowed. Kernels can only fetch scratch buffers via GetScratchBuffer. context_.AllocatePersistentBuffer = nullptr; context_.RequestScratchBufferInArena = nullptr; context_.GetScratchBuffer = context_helper_.GetScratchBuffer; - context_.GetTensor = nullptr; TF_LITE_ENSURE_OK(&context_, allocator_.FinishModelAllocation(model_, eval_tensors_)); @@ -345,6 +343,12 @@ TfLiteStatus MicroInterpreter::Invoke() { #endif invoke_status = registration->invoke(&context_, node); + // All TfLiteTensor structs used in the kernel are allocated from temp + // memory in the allocator. This creates a chain of allocations in the + // temp section. The call below resets the chain of allocations to + // prepare for the next call. + allocator_.ResetTempAllocations(); + if (invoke_status == kTfLiteError) { TF_LITE_REPORT_ERROR( error_reporter_, diff --git a/tensorflow/lite/micro/test_helpers.cc b/tensorflow/lite/micro/test_helpers.cc index a4f716fca06..23c7ca96408 100644 --- a/tensorflow/lite/micro/test_helpers.cc +++ b/tensorflow/lite/micro/test_helpers.cc @@ -28,7 +28,6 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" #include "tensorflow/lite/micro/all_ops_resolver.h" -#include "tensorflow/lite/micro/kernels/kernel_util.h" #include "tensorflow/lite/micro/micro_utils.h" #include "tensorflow/lite/schema/schema_generated.h" @@ -602,9 +601,8 @@ TfLiteStatus SimpleStatefulOp::Invoke(TfLiteContext* context, OpData* data = reinterpret_cast(node->user_data); data->invoke_count += 1; - const TfLiteEvalTensor* input = - tflite::micro::GetEvalInput(context, node, kInputTensor); - const uint8_t* input_data = tflite::micro::GetTensorData(input); + const TfLiteTensor* input = GetInput(context, node, kInputTensor); + const uint8_t* input_data = GetTensorData(input); int size = NumElements(input->dims); uint8_t* sorting_buffer = reinterpret_cast( @@ -622,13 +620,10 @@ TfLiteStatus SimpleStatefulOp::Invoke(TfLiteContext* context, } } - TfLiteEvalTensor* median = - tflite::micro::GetEvalOutput(context, node, kMedianTensor); - uint8_t* median_data = tflite::micro::GetTensorData(median); - TfLiteEvalTensor* invoke_count = - tflite::micro::GetEvalOutput(context, node, kInvokeCount); - int32_t* invoke_count_data = - tflite::micro::GetTensorData(invoke_count); + TfLiteTensor* median = GetOutput(context, node, kMedianTensor); + uint8_t* median_data = GetTensorData(median); + TfLiteTensor* invoke_count = GetOutput(context, node, kInvokeCount); + int32_t* invoke_count_data = GetTensorData(invoke_count); median_data[0] = sorting_buffer[size / 2]; invoke_count_data[0] = data->invoke_count; @@ -665,13 +660,14 @@ TfLiteStatus MockCustom::Prepare(TfLiteContext* context, TfLiteNode* node) { } TfLiteStatus MockCustom::Invoke(TfLiteContext* context, TfLiteNode* node) { - const TfLiteEvalTensor* input = tflite::micro::GetEvalInput(context, node, 0); - const int32_t* input_data = tflite::micro::GetTensorData(input); - const TfLiteEvalTensor* weight = - tflite::micro::GetEvalInput(context, node, 1); + const TfLiteTensor* input = tflite::GetInput(context, node, 0); + const int32_t* input_data = input->data.i32; + const TfLiteTensor* weight = tflite::GetInput(context, node, 1); const uint8_t* weight_data = weight->data.uint8; - TfLiteEvalTensor* output = tflite::micro::GetEvalOutput(context, node, 0); - int32_t* output_data = tflite::micro::GetTensorData(output); + TfLiteTensor* output = GetOutput(context, node, 0); + int32_t* output_data = output->data.i32; + output_data[0] = + 0; // Catch output tensor sharing memory with an input tensor output_data[0] = input_data[0] + weight_data[0]; return kTfLiteOk; } From 016646b73291f95625baf526881c5cc7b3a5c74d Mon Sep 17 00:00:00 2001 From: Blake Hechtman Date: Thu, 6 Aug 2020 17:54:14 -0700 Subject: [PATCH 0611/1017] [XLA] Make FillRandomDouble method that can allow for better creation of random bool arrays. PiperOrigin-RevId: 325348068 Change-Id: Idadb60f5a34f140c4affc5585216539460554f8e --- tensorflow/compiler/xla/array.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h index 392cd9bd359..0f31d4c27f5 100644 --- a/tensorflow/compiler/xla/array.h +++ b/tensorflow/compiler/xla/array.h @@ -289,13 +289,19 @@ class Array { } // Fills the array with random normal variables with the specified mean. - void FillRandom(const T& stddev, const double mean = 0.0, - const int seed = 12345) { + void FillRandom(const T& stddev, double mean = 0.0, int seed = 12345) { + FillRandomDouble(static_cast(stddev), mean, seed); + } + + void FillRandomDouble(double stddev, double mean = 0.0, int seed = 12345) { std::mt19937 g(seed); - std::normal_distribution distribution(mean, - static_cast(stddev)); + std::normal_distribution distribution(mean, stddev); for (int64 i = 0; i < num_elements(); ++i) { - values_[i] = static_cast(distribution(g)); + if constexpr (std::is_same()) { + values_[i] = distribution(g) > 0.0; + } else { + values_[i] = static_cast(distribution(g)); + } } } From 5f392207bb31c4fb381900bc4239b30a39a326ef Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Thu, 6 Aug 2020 17:57:23 -0700 Subject: [PATCH 0612/1017] Disable flaky test PiperOrigin-RevId: 325348505 Change-Id: I534aedd5f980915aa41f00b333d6ae25826319d8 --- tensorflow/python/data/kernel_tests/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD index 210b6f59681..639c07bac01 100644 --- a/tensorflow/python/data/kernel_tests/BUILD +++ b/tensorflow/python/data/kernel_tests/BUILD @@ -94,6 +94,7 @@ tf_py_test( name = "data_service_ops_test", size = "medium", srcs = ["data_service_ops_test.py"], + tags = ["notap"], # "b/163085430" deps = [ "//tensorflow:tensorflow_py", "//tensorflow/python:client_testlib", From 2a7de5bd6c1cd3f12ccfbbdc92e12d3e4a455d60 Mon Sep 17 00:00:00 2001 From: Yuanzhong Xu Date: Thu, 6 Aug 2020 18:23:40 -0700 Subject: [PATCH 0613/1017] [XLA] Merge partial sharding in elementwise ops' sharding propagation PiperOrigin-RevId: 325352068 Change-Id: I0e60eaaaad23d8091e5db686b7990e219a95d061 --- .../compiler/xla/service/hlo_sharding.cc | 18 +- .../compiler/xla/service/hlo_sharding.h | 10 + .../xla/service/sharding_propagation.cc | 265 ++++++++++++++---- .../xla/service/sharding_propagation_test.cc | 65 ++++- 4 files changed, 293 insertions(+), 65 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc index d522fc8bd14..ba1fc0d0450 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding.cc @@ -51,7 +51,12 @@ HloSharding HloSharding::PartialTile( int64 group = group_tile_assignment(group_index); *device = replication_groups[group][indices.back()]; }); - return HloSharding(new_tile_assignment, + return PartialTile(new_tile_assignment); +} + +HloSharding HloSharding::PartialTile( + const Array& tile_assignment_last_dim_replicate) { + return HloSharding(tile_assignment_last_dim_replicate, /*replicate_on_last_tile_dim=*/true); } @@ -494,6 +499,17 @@ Shape HloSharding::TileShape(const Shape& shape, int64 device) const { return result_shape; } +int64 HloSharding::NumTiles() const { + if (IsTileMaximal()) { + return 1; + } + if (ReplicateOnLastTileDim()) { + return tile_assignment().num_elements() / + tile_assignment().dimensions().back(); + } + return tile_assignment().num_elements(); +} + HloSharding HloSharding::GetSubSharding(const Shape& shape, const ShapeIndex& index) const { CHECK(IsTuple()); diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h index af28df56e68..1b827efff2d 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding.h +++ b/tensorflow/compiler/xla/service/hlo_sharding.h @@ -61,6 +61,12 @@ class HloSharding { const Array& group_tile_assignment, absl::Span> replication_groups); + // Creates a partially replicated tiled sharding with device-level tile + // assignment, where the last dimension is the additional replication + // dimension. + static HloSharding PartialTile( + const Array& tile_assignment_last_dim_replicate); + // Creates a new sharding which splits a one-dimensional input shape into // `num_tiles` tiles. static HloSharding Tile1D(const Shape& input_shape, int64 num_tiles); @@ -237,6 +243,10 @@ class HloSharding { // REQUIRES: !IsTuple() Shape TileShape(const Shape& shape, int64 device) const; + // Gets the number of tiles. If it has partial replication, this will not + // equal the device count. + int64 NumTiles() const; + private: HloSharding() : replicated_(true), diff --git a/tensorflow/compiler/xla/service/sharding_propagation.cc b/tensorflow/compiler/xla/service/sharding_propagation.cc index 5d85fb5189c..0e4b0568134 100644 --- a/tensorflow/compiler/xla/service/sharding_propagation.cc +++ b/tensorflow/compiler/xla/service/sharding_propagation.cc @@ -91,9 +91,7 @@ bool IsShardingMoreSpecific(const HloSharding& lhs, const HloSharding& rhs) { return is_better; } if (!rhs.IsTileMaximal()) { - // If we already have a non-tile-maximal sharding then we can't improve - // that. - return false; + return lhs.NumTiles() > rhs.NumTiles(); } else if (!rhs.IsReplicated()) { // If we are not replicated then only tiled (not tile maximal) shardings // can improve us. @@ -124,9 +122,12 @@ HloSharding MergeForMoreSpecificSharding(const HloSharding& a, // Updates the sharding of the specified instruction with the specified sharding // if it is better than the current one and returns true if a new sharding have -// been applied. +// been applied. If may_combine_partial_sharding is true, this may combine the +// new and existing sharding if they are both partial tiling partial +// replication. bool MaybeImproveInstructionSharding(const HloSharding& sharding, - HloInstruction* instruction) { + HloInstruction* instruction, + bool may_combine_partial_sharding) { // We don't want to propagate tile maximal shardings. if (!IsSpatiallyPartitioned(sharding)) { return false; @@ -136,6 +137,101 @@ bool MaybeImproveInstructionSharding(const HloSharding& sharding, instruction->set_sharding(sharding); return true; } + if (may_combine_partial_sharding && sharding.ReplicateOnLastTileDim() && + instruction->sharding().ReplicateOnLastTileDim()) { + if (sharding.tile_assignment().num_elements() == + instruction->sharding().tile_assignment().num_elements()) { + // Combine the tile dimension sizes from new and old. + int64 num_devices = sharding.tile_assignment().num_elements(); + std::vector new_tile_dims; + bool compatible = true; + new_tile_dims.reserve(sharding.tile_assignment().num_dimensions()); + for (int64 i = 0; i < sharding.tile_assignment().num_dimensions() - 1; + ++i) { + int64 new_dim = sharding.tile_assignment().dim(i); + int64 old_dim = instruction->sharding().tile_assignment().dim(i); + if (new_dim == 1) { + new_tile_dims.push_back(old_dim); + } else if (old_dim == 1) { + new_tile_dims.push_back(new_dim); + } else if (new_dim == old_dim) { + new_tile_dims.push_back(new_dim); + } else { + compatible = false; + break; + } + } + int64 replication = num_devices / Product(new_tile_dims); + if (compatible && num_devices % Product(new_tile_dims) == 0 && + replication < + instruction->sharding().tile_assignment().dimensions().back()) { + new_tile_dims.push_back(replication); + Array new_tile(new_tile_dims); + // Maps from replication group ID to sorted members. + absl::flat_hash_map> old_group_members; + absl::flat_hash_map> new_group_members; + auto get_group_index = [&](absl::Span tile_indices, + const HloSharding& sharding) { + int64 group_id = 0; + for (int64 i = 0; i < tile_indices.size() - 1; ++i) { + group_id *= sharding.tile_assignment().dim(i); + group_id += tile_indices[i]; + } + return group_id; + }; + instruction->sharding().tile_assignment().Each( + [&](absl::Span indices, int64 device) { + old_group_members[get_group_index(indices, + instruction->sharding())] + .insert(device); + }); + sharding.tile_assignment().Each([&](absl::Span indices, + int64 device) { + new_group_members[get_group_index(indices, sharding)].insert(device); + }); + // Try to find the intersection of old and new replication groups, in + // order to determine the merged tile assignment. + new_tile.Each([&](absl::Span indices, int64* device) { + if (!compatible) { + return; + } + std::vector old_index(indices.begin(), indices.end()); + std::vector new_index = old_index; + for (int64 i = 0; i < indices.size() - 1; ++i) { + if (instruction->sharding().tile_assignment().dim(i) == 1) { + old_index[i] = 0; + } + if (sharding.tile_assignment().dim(i) == 1) { + new_index[i] = 0; + } + } + int64 old_group_id = + get_group_index(old_index, instruction->sharding()); + int64 new_group_id = get_group_index(new_index, sharding); + if (old_group_members[old_group_id].empty() || + new_group_members[new_group_id].empty() || + *old_group_members[old_group_id].begin() != + *new_group_members[new_group_id].begin()) { + compatible = false; + return; + } + *device = *old_group_members[old_group_id].begin(); + old_group_members[old_group_id].erase(*device); + new_group_members[new_group_id].erase(*device); + }); + if (compatible) { + if (replication == 1) { + new_tile_dims.pop_back(); + new_tile.Reshape(new_tile_dims); + instruction->set_sharding(HloSharding::Tile(new_tile)); + } else { + instruction->set_sharding(HloSharding::PartialTile(new_tile)); + } + return true; + } + } + } + } if (IsShardingMoreSpecific(sharding, instruction->sharding())) { instruction->set_sharding(sharding); return true; @@ -363,7 +459,8 @@ bool SupportSpatialPartitioning(const HloInstruction* instruction, // Convolution handling for InferShardingFromOperands(). bool InferConvolutionShardingFromOperands(HloInstruction* instruction, - bool aggressive_prop) { + bool aggressive_prop, + bool may_combine_partial_sharding) { const auto& dnums = instruction->convolution_dimension_numbers(); const HloInstruction* lhs = instruction->operand(0); const HloInstruction* rhs = instruction->operand(1); @@ -430,13 +527,15 @@ bool InferConvolutionShardingFromOperands(HloInstruction* instruction, partitioned_only_along_non_trivial_dims(lhs->sharding(), dot_dims->batch_dims, 0)) { return MaybeImproveInstructionSharding(get_tiled_sharding_based_on_lhs(), - instruction); + instruction, + may_combine_partial_sharding); } if (IsSpatiallyPartitioned(rhs) && partitioned_only_along_non_trivial_dims(rhs->sharding(), dot_dims->batch_dims, 1)) { return MaybeImproveInstructionSharding(get_tiled_sharding_based_on_rhs(), - instruction); + instruction, + may_combine_partial_sharding); } if (aggressive_prop) { // If LHS/RHS is partitioned only along the non-contracting @@ -455,19 +554,23 @@ bool InferConvolutionShardingFromOperands(HloInstruction* instruction, if (Product(lhs->shape().dimensions()) >= Product(rhs->shape().dimensions())) { return MaybeImproveInstructionSharding( - get_tiled_sharding_based_on_lhs(), instruction); + get_tiled_sharding_based_on_lhs(), instruction, + may_combine_partial_sharding); } else { return MaybeImproveInstructionSharding( - get_tiled_sharding_based_on_rhs(), instruction); + get_tiled_sharding_based_on_rhs(), instruction, + may_combine_partial_sharding); } } if (can_propagate_from_lhs) { return MaybeImproveInstructionSharding( - get_tiled_sharding_based_on_lhs(), instruction); + get_tiled_sharding_based_on_lhs(), instruction, + may_combine_partial_sharding); } if (can_propagate_from_rhs) { return MaybeImproveInstructionSharding( - get_tiled_sharding_based_on_rhs(), instruction); + get_tiled_sharding_based_on_rhs(), instruction, + may_combine_partial_sharding); } } } @@ -476,8 +579,8 @@ bool InferConvolutionShardingFromOperands(HloInstruction* instruction, return false; } if (lhs->sharding().IsReplicated()) { - return MaybeImproveInstructionSharding(HloSharding::Replicate(), - instruction); + return MaybeImproveInstructionSharding( + HloSharding::Replicate(), instruction, may_combine_partial_sharding); } if (IsConvolutionKernelSmall(instruction)) { @@ -488,11 +591,13 @@ bool InferConvolutionShardingFromOperands(HloInstruction* instruction, return false; } return MaybeImproveInstructionSharding(get_tiled_sharding_based_on_lhs(), - instruction); + instruction, + may_combine_partial_sharding); } // If the kernel is large (e.g backward convolution) then we only support // replicated output. - return MaybeImproveInstructionSharding(HloSharding::Replicate(), instruction); + return MaybeImproveInstructionSharding(HloSharding::Replicate(), instruction, + may_combine_partial_sharding); } // Tries to update the sharding of the specified instruction based on its @@ -512,8 +617,9 @@ bool InferShardingFromOperands(HloInstruction* instruction, if (absl::c_any_of(instruction->operands(), [](const HloInstruction* op) { return op->has_sharding() && op->sharding().IsReplicated(); })) { - return MaybeImproveInstructionSharding(HloSharding::Replicate(), - instruction); + return MaybeImproveInstructionSharding( + HloSharding::Replicate(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } return false; } @@ -526,7 +632,8 @@ bool InferShardingFromOperands(HloInstruction* instruction, } HloSharding new_sharding = operand->sharding().GetSubSharding( operand->shape(), {instruction->tuple_index()}); - return MaybeImproveInstructionSharding(new_sharding, instruction); + return MaybeImproveInstructionSharding( + new_sharding, instruction, /*may_combine_partial_sharding=*/is_spmd); } case HloOpcode::kTuple: { if (absl::c_none_of(instruction->operands(), @@ -601,7 +708,8 @@ bool InferShardingFromOperands(HloInstruction* instruction, }; if (operand->sharding().IsReplicated()) { changed |= MaybeImproveInstructionSharding( - get_maybe_tuple_sharding(HloSharding::Replicate()), instruction); + get_maybe_tuple_sharding(HloSharding::Replicate()), instruction, + /*may_combine_partial_sharding=*/is_spmd); continue; } if (absl::c_any_of(instruction->dimensions(), [operand](int64 dim) { @@ -610,7 +718,8 @@ bool InferShardingFromOperands(HloInstruction* instruction, // We are reducing along one of the sharded dimensions. We don't // support tiled sharding in this case. changed |= MaybeImproveInstructionSharding( - get_maybe_tuple_sharding(HloSharding::Replicate()), instruction); + get_maybe_tuple_sharding(HloSharding::Replicate()), instruction, + /*may_combine_partial_sharding=*/is_spmd); } else { // We are reducing along some of the non-sharded dimensions. The // result sharding should be the same as the operand sharding with the @@ -631,7 +740,9 @@ bool InferShardingFromOperands(HloInstruction* instruction, // of the same reduce instruction. HloSharding new_sharding = get_maybe_tuple_sharding(HloSharding::Tile(new_tile_assignment)); - changed |= MaybeImproveInstructionSharding(new_sharding, instruction); + changed |= MaybeImproveInstructionSharding( + new_sharding, instruction, + /*may_combine_partial_sharding=*/is_spmd); } } return changed; @@ -665,10 +776,13 @@ bool InferShardingFromOperands(HloInstruction* instruction, Array new_tile_assignment = op->sharding().tile_assignment(); new_tile_assignment.Reshape(target_tile_assignment_dimensions); HloSharding new_sharding = HloSharding::Tile(new_tile_assignment); - return MaybeImproveInstructionSharding(new_sharding, instruction); + return MaybeImproveInstructionSharding( + new_sharding, instruction, /*may_combine_partial_sharding=*/is_spmd); } case HloOpcode::kConvolution: - return InferConvolutionShardingFromOperands(instruction, aggressive_prop); + return InferConvolutionShardingFromOperands( + instruction, aggressive_prop, + /*may_combine_partial_sharding=*/is_spmd); case HloOpcode::kTranspose: { const HloInstruction* input = instruction->operand(0); if (!IsSpatiallyPartitioned(input)) { @@ -676,7 +790,8 @@ bool InferShardingFromOperands(HloInstruction* instruction, } HloSharding sharding = hlo_sharding_util::TransposeSharding( input->sharding(), instruction->dimensions()); - return MaybeImproveInstructionSharding(sharding, instruction); + return MaybeImproveInstructionSharding( + sharding, instruction, /*may_combine_partial_sharding=*/is_spmd); } case HloOpcode::kReduceWindow: { const HloInstruction* lhs = instruction->operand(0); @@ -694,7 +809,9 @@ bool InferShardingFromOperands(HloInstruction* instruction, << instruction->ToString(); return false; } - return MaybeImproveInstructionSharding(lhs->sharding(), instruction); + return MaybeImproveInstructionSharding( + lhs->sharding(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } case HloOpcode::kSelectAndScatter: { // Shard according to first operand, as output keeps the same shape. @@ -713,7 +830,9 @@ bool InferShardingFromOperands(HloInstruction* instruction, << instruction->ToString(); return false; } - return MaybeImproveInstructionSharding(lhs->sharding(), instruction); + return MaybeImproveInstructionSharding( + lhs->sharding(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } case HloOpcode::kReshape: { if (!IsSpatiallyPartitioned(instruction->operand(0))) { @@ -724,8 +843,9 @@ bool InferShardingFromOperands(HloInstruction* instruction, instruction->operand(0)->shape(), instruction->shape(), instruction->operand(0)->sharding()); if (new_sharding.has_value()) { - return MaybeImproveInstructionSharding(new_sharding.value(), - instruction); + return MaybeImproveInstructionSharding( + new_sharding.value(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } return false; } @@ -736,7 +856,7 @@ bool InferShardingFromOperands(HloInstruction* instruction, return MaybeImproveInstructionSharding( hlo_sharding_util::ReverseSharding( instruction->operand(0)->sharding(), instruction->dimensions()), - instruction); + instruction, /*may_combine_partial_sharding=*/is_spmd); } case HloOpcode::kDot: { auto& dot_dim_numbs = instruction->dot_dimension_numbers(); @@ -765,8 +885,9 @@ bool InferShardingFromOperands(HloInstruction* instruction, } else if (ops_sharding[0]->IsReplicated() && ops_sharding[1]->IsReplicated()) { // Both replicated -> replicate - return MaybeImproveInstructionSharding(HloSharding::Replicate(), - instruction); + return MaybeImproveInstructionSharding( + HloSharding::Replicate(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } else if (!ops_sharding[0]->IsReplicated() && !ops_sharding[1]->IsReplicated()) { // Both tile sharded. The dot spatial partitioning implementation @@ -785,8 +906,9 @@ bool InferShardingFromOperands(HloInstruction* instruction, } if (ops_sharding[representative_op]->IsReplicated()) { - return MaybeImproveInstructionSharding(HloSharding::Replicate(), - instruction); + return MaybeImproveInstructionSharding( + HloSharding::Replicate(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } else { // Tile-shard instruction according to representative op. auto sharding = *ops_sharding[representative_op]; @@ -811,7 +933,8 @@ bool InferShardingFromOperands(HloInstruction* instruction, tile_assignment.Reshape(dimensions); sharding = HloSharding::Tile(tile_assignment); } - return MaybeImproveInstructionSharding(sharding, instruction); + return MaybeImproveInstructionSharding( + sharding, instruction, /*may_combine_partial_sharding=*/is_spmd); } } case HloOpcode::kParameter: { @@ -826,7 +949,8 @@ bool InferShardingFromOperands(HloInstruction* instruction, if (parent->called_computations()[i - 1] == instruction->parent()) { if (parent->operand(i)->has_sharding()) { return MaybeImproveInstructionSharding( - parent->operand(i)->sharding(), instruction); + parent->operand(i)->sharding(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } return false; } @@ -853,15 +977,16 @@ bool InferShardingFromOperands(HloInstruction* instruction, if (instruction->shape().IsTuple()) { return MaybeImproveInstructionSharding( HloSharding::SingleTuple(instruction->shape(), operand->sharding()), - instruction); + instruction, /*may_combine_partial_sharding=*/is_spmd); } else { - return MaybeImproveInstructionSharding(operand->sharding(), - instruction); + return MaybeImproveInstructionSharding( + operand->sharding(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } } case HloOpcode::kDynamicSlice: case HloOpcode::kDynamicUpdateSlice: { - auto propagate_slicing = [instruction]() { + auto propagate_slicing = [instruction, is_spmd]() { const HloInstruction* operand = instruction->opcode() == HloOpcode::kDynamicSlice ? instruction->operand(0) @@ -871,8 +996,9 @@ bool InferShardingFromOperands(HloInstruction* instruction, } if (operand->sharding().IsReplicated()) { - return MaybeImproveInstructionSharding(HloSharding::Replicate(), - instruction); + return MaybeImproveInstructionSharding( + HloSharding::Replicate(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } const auto& tile_assignment = operand->sharding().tile_assignment(); @@ -883,10 +1009,11 @@ bool InferShardingFromOperands(HloInstruction* instruction, return false; } } - return MaybeImproveInstructionSharding(operand->sharding(), - instruction); + return MaybeImproveInstructionSharding( + operand->sharding(), instruction, + /*may_combine_partial_sharding=*/is_spmd); }; - auto propagate_base = [instruction]() { + auto propagate_base = [instruction, is_spmd]() { if (instruction->opcode() != HloOpcode::kDynamicUpdateSlice) { return false; } @@ -894,7 +1021,8 @@ bool InferShardingFromOperands(HloInstruction* instruction, return false; } return MaybeImproveInstructionSharding( - instruction->operand(0)->sharding(), instruction); + instruction->operand(0)->sharding(), instruction, + /*may_combine_partial_sharding=*/is_spmd); }; return propagate_slicing() || propagate_base(); } @@ -903,15 +1031,18 @@ bool InferShardingFromOperands(HloInstruction* instruction, if (IsSpatiallyPartitioned(instruction->operand(1))) { HloSharding new_sharding = hlo_sharding_util::GatherOutputSharding( instruction->operand(1)->sharding(), instruction); - changed |= MaybeImproveInstructionSharding(new_sharding, instruction); + changed |= MaybeImproveInstructionSharding( + new_sharding, instruction, + /*may_combine_partial_sharding=*/is_spmd); } if (is_spmd && IsSpatiallyPartitioned(instruction->operand(0))) { auto maybe_from_data = hlo_sharding_util::GatherOutputShardingFromDataOperand( instruction->operand(0)->sharding(), *instruction); if (maybe_from_data) { - changed |= - MaybeImproveInstructionSharding(*maybe_from_data, instruction); + changed |= MaybeImproveInstructionSharding( + *maybe_from_data, instruction, + /*may_combine_partial_sharding=*/is_spmd); } } return changed; @@ -920,7 +1051,8 @@ bool InferShardingFromOperands(HloInstruction* instruction, bool changed = false; if (is_spmd && IsSpatiallyPartitioned(instruction->operand(0))) { changed |= MaybeImproveInstructionSharding( - instruction->operand(0)->sharding(), instruction); + instruction->operand(0)->sharding(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } if (!IsSpatiallyPartitioned(instruction->operand(1)) && !IsSpatiallyPartitioned(instruction->operand(2))) { @@ -931,12 +1063,14 @@ bool InferShardingFromOperands(HloInstruction* instruction, hlo_sharding_util::ScatterOutputShardingFromUpdate( instruction->operand(2)->sharding(), *instruction); if (maybe_from_update) { - changed |= - MaybeImproveInstructionSharding(*maybe_from_update, instruction); + changed |= MaybeImproveInstructionSharding( + *maybe_from_update, instruction, + /*may_combine_partial_sharding=*/is_spmd); } } - changed |= MaybeImproveInstructionSharding(HloSharding::Replicate(), - instruction); + changed |= MaybeImproveInstructionSharding( + HloSharding::Replicate(), instruction, + /*may_combine_partial_sharding=*/is_spmd); return changed; } case HloOpcode::kWhile: { @@ -948,14 +1082,28 @@ bool InferShardingFromOperands(HloInstruction* instruction, sharding = MergeForMoreSpecificSharding(sharding, instruction->sharding()); } - return MaybeImproveInstructionSharding(sharding, instruction); + return MaybeImproveInstructionSharding( + sharding, instruction, /*may_combine_partial_sharding=*/is_spmd); } default: { + if (instruction->IsElementwise() && is_spmd) { + bool changed = false; + for (auto operand : instruction->operands()) { + if (IsSpatiallyPartitioned(operand)) { + changed |= MaybeImproveInstructionSharding( + operand->sharding(), instruction, + /*may_combine_partial_sharding=*/is_spmd); + } + } + return changed; + } const HloInstruction* operand = PickRepresentativeOperand(instruction); if (!operand || !IsSpatiallyPartitioned(operand)) { return false; } - return MaybeImproveInstructionSharding(operand->sharding(), instruction); + return MaybeImproveInstructionSharding( + operand->sharding(), instruction, + /*may_combine_partial_sharding=*/is_spmd); } } return false; @@ -1308,8 +1456,9 @@ bool InferShardingFromUsers(HloInstruction* instruction, absl::optional user_sharding = GetShardingFromUser(*instruction, *user, aggressive_prop, is_spmd); if (user_sharding) { - improved_sharding |= - MaybeImproveInstructionSharding(*user_sharding, instruction); + improved_sharding |= MaybeImproveInstructionSharding( + *user_sharding, instruction, + /*may_combine_partial_sharding=*/is_spmd); } } return improved_sharding; diff --git a/tensorflow/compiler/xla/service/sharding_propagation_test.cc b/tensorflow/compiler/xla/service/sharding_propagation_test.cc index 594130daf0b..8aa10b67ed8 100644 --- a/tensorflow/compiler/xla/service/sharding_propagation_test.cc +++ b/tensorflow/compiler/xla/service/sharding_propagation_test.cc @@ -1149,21 +1149,21 @@ ENTRY entry { ShardingPropagation().Run(module.get())); EXPECT_TRUE(changed); EXPECT_THAT(FindInstruction(module.get(), "tp"), - op::Sharding("{{devices=[1,2]0,1}}")); + op::Sharding("{{devices=[3,1]0,1,2}}")); EXPECT_THAT(FindInstruction(module.get(), "tgte"), - op::Sharding("{devices=[1,2]0,1}")); + op::Sharding("{devices=[3,1]0,1,2}")); EXPECT_THAT(FindInstruction(module.get(), "ttr"), - op::Sharding("{devices=[2,1]0,1}")); + op::Sharding("{devices=[1,3]0,1,2}")); EXPECT_THAT(FindInstruction(module.get(), "tr"), - op::Sharding("{{devices=[2,1]0,1}}")); + op::Sharding("{{devices=[1,3]0,1,2}}")); EXPECT_THAT(FindInstruction(module.get(), "fp"), op::Sharding("{{devices=[1,3]0,1,2}}")); EXPECT_THAT(FindInstruction(module.get(), "fgte"), op::Sharding("{devices=[1,3]0,1,2}")); EXPECT_THAT(FindInstruction(module.get(), "fr"), - op::Sharding("{{devices=[2,1]0,1}}")); + op::Sharding("{{devices=[1,3]0,1,2}}")); EXPECT_THAT(FindInstruction(module.get(), "conditional"), - op::Sharding("{{devices=[2,1]0,1}}")); + op::Sharding("{{devices=[1,3]0,1,2}}")); } TEST_F(ShardingPropagationTest, TupleFromUser) { @@ -1764,5 +1764,58 @@ ENTRY entry { op::Sharding("{devices=[2,1]0,1}")); } +TEST_F(ShardingPropagationTest, PartialShardingOnElementwise) { + const char* const hlo_string = R"( +HloModule module + +ENTRY entry { + %p0 = f32[2,9] parameter(0), sharding={devices=[1,2,2]0,1,2,3 last_tile_dim_replicate} + %p1 = f32[2,9] parameter(1), sharding={devices=[2,1,2]0,2,1,3 last_tile_dim_replicate} + %lhs = f32[2,9] copy(%p0) + %rhs = f32[2,9] copy(%p1) + %add = f32[2,9] add(%lhs, %rhs) + ROOT %copy = f32[2,9] copy(%add) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT(FindInstruction(module.get(), "lhs"), + op::Sharding("{devices=[2,2]0,2,1,3}")); + EXPECT_THAT(FindInstruction(module.get(), "rhs"), + op::Sharding("{devices=[2,2]0,2,1,3}")); + EXPECT_THAT(FindInstruction(module.get(), "add"), + op::Sharding("{devices=[2,2]0,2,1,3}")); +} + +TEST_F(ShardingPropagationTest, PartialShardingOnElementwise2) { + const char* const hlo_string = R"( +HloModule module + +ENTRY entry { + %p0 = f32[2,9] parameter(0), sharding={devices=[1,2,4]0,1,2,3,4,5,6,7 last_tile_dim_replicate} + %p1 = f32[2,9] parameter(1), sharding={devices=[2,1,4]0,1,4,5,2,3,6,7 last_tile_dim_replicate} + %lhs = f32[2,9] copy(%p0) + %rhs = f32[2,9] copy(%p1) + %add = f32[2,9] add(%lhs, %rhs) + ROOT %copy = f32[2,9] copy(%add) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT( + FindInstruction(module.get(), "lhs"), + op::Sharding("{devices=[2,2,2]0,1,4,5,2,3,6,7 last_tile_dim_replicate}")); + EXPECT_THAT( + FindInstruction(module.get(), "rhs"), + op::Sharding("{devices=[2,2,2]0,1,4,5,2,3,6,7 last_tile_dim_replicate}")); + EXPECT_THAT( + FindInstruction(module.get(), "add"), + op::Sharding("{devices=[2,2,2]0,1,4,5,2,3,6,7 last_tile_dim_replicate}")); +} + } // namespace } // namespace xla From 8c0c1e173093db7993544c46187d3be34a9a8e0b Mon Sep 17 00:00:00 2001 From: Jay Shi Date: Thu, 6 Aug 2020 18:26:11 -0700 Subject: [PATCH 0614/1017] [tf.data] Add the optimization to disable intra op parallelism. PiperOrigin-RevId: 325352372 Change-Id: I7f31e249f788316ff21263dc1b08882029da6ab9 --- .../core/grappler/optimizers/data/BUILD | 35 ++++++ .../data/disable_intra_op_parallelism.cc | 99 +++++++++++++++ .../data/disable_intra_op_parallelism.h | 50 ++++++++ .../data/disable_intra_op_parallelism_test.cc | 117 ++++++++++++++++++ .../optimizers/data/inject_prefetch.cc | 2 +- .../optimizers/data/meta_optimizer.cc | 3 +- 6 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.cc create mode 100644 tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.h create mode 100644 tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism_test.cc diff --git a/tensorflow/core/grappler/optimizers/data/BUILD b/tensorflow/core/grappler/optimizers/data/BUILD index 1daf7e9b76e..860cbd7c35e 100644 --- a/tensorflow/core/grappler/optimizers/data/BUILD +++ b/tensorflow/core/grappler/optimizers/data/BUILD @@ -57,6 +57,41 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "disable_intra_op_parallelism", + srcs = ["disable_intra_op_parallelism.cc"], + hdrs = ["disable_intra_op_parallelism.h"], + deps = [ + ":graph_utils", + ":optimizer_base", + "//tensorflow/core/grappler:mutable_graph_view", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core/grappler:grappler_item", + "//tensorflow/core/grappler:op_types", + "//tensorflow/core/grappler:utils", + "//tensorflow/core/grappler/clusters:cluster", + "//tensorflow/core/grappler/optimizers:custom_graph_optimizer_registry", + "//tensorflow/core:lib_internal", + ] + tf_protos_all(), + alwayslink = 1, +) + +tf_cc_test( + name = "disable_intra_op_parallelism_test", + srcs = ["disable_intra_op_parallelism_test.cc"], + deps = [ + ":disable_intra_op_parallelism", + ":graph_test_utils", + ":graph_utils", + "//tensorflow/core:framework", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "//tensorflow/core/grappler:grappler_item", + ], +) + cc_library( name = "filter_fusion", srcs = ["filter_fusion.cc"], diff --git a/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.cc b/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.cc new file mode 100644 index 00000000000..4b6d6ac1bfa --- /dev/null +++ b/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.cc @@ -0,0 +1,99 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.h" + +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/grappler/clusters/cluster.h" +#include "tensorflow/core/grappler/grappler_item.h" +#include "tensorflow/core/grappler/mutable_graph_view.h" +#include "tensorflow/core/grappler/op_types.h" +#include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h" +#include "tensorflow/core/grappler/optimizers/data/graph_utils.h" +#include "tensorflow/core/grappler/utils.h" +#include "tensorflow/core/platform/protobuf.h" + +namespace tensorflow { +namespace grappler { +namespace { + +constexpr char kMaxIntraOpParallelismDataset[] = "MaxIntraOpParallelismDataset"; + +constexpr std::array kMaxIntraOpParallelismDatasetOps = { + "MaxIntraOpParallelismDataset", + "ExperimentalMaxIntraOpParallelismDataset", +}; + +} // namespace + +Status DisableIntraOpParallelism::OptimizeAndCollectStats( + Cluster* cluster, const GrapplerItem& item, GraphDef* output, + OptimizationStats* stats) { + *output = item.graph; + MutableGraphView graph(output); + + const NodeDef* sink_node; + for (const NodeDef& node : item.graph.node()) { + for (const auto& target_dataset_op : kMaxIntraOpParallelismDatasetOps) { + if (node.op() == target_dataset_op) { + // If parallelism is set by the user, we keep the user setting instead + // of disabling it. + return Status::OK(); + } + } + if (node.name() == "Sink") { + sink_node = &node; + } + } + + NodeDef* last_node = graph_utils::GetInputNode(*sink_node, graph); + + // Add a const node with value 1 + NodeDef* max_parallelism_value = graph_utils::AddScalarConstNode(1LL, &graph); + + NodeDef insert_node; + graph_utils::SetUniqueGraphNodeName("intra_op_parallelism", graph.graph(), + &insert_node); + insert_node.set_op(kMaxIntraOpParallelismDataset); + + // `input_dataset` input + *insert_node.mutable_input()->Add() = last_node->name(); + // `max_intra_op_parallelism` input + *insert_node.mutable_input()->Add() = max_parallelism_value->name(); + + for (const auto& attr_name : {"output_types", "output_shapes"}) { + graph_utils::CopyAttribute(attr_name, *last_node, &insert_node); + } + + auto* added_node = graph.AddNode(std::move(insert_node)); + TF_RETURN_IF_ERROR( + graph.UpdateFanouts(last_node->name(), added_node->name())); + + stats->num_changes++; + return Status::OK(); +} + +void DisableIntraOpParallelism::Feedback(Cluster* cluster, + const GrapplerItem& item, + const GraphDef& optimize_output, + double result) { + // no-op +} + +REGISTER_GRAPH_OPTIMIZER_AS(DisableIntraOpParallelism, + "disable_intra_op_parallelism"); + +} // namespace grappler +} // namespace tensorflow diff --git a/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.h b/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.h new file mode 100644 index 00000000000..d2355eb8766 --- /dev/null +++ b/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.h @@ -0,0 +1,50 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_DISABLE_INTRA_OP_PARALLELISM_H_ +#define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_DISABLE_INTRA_OP_PARALLELISM_H_ + +#include "tensorflow/core/grappler/optimizers/data/optimizer_base.h" + +namespace tensorflow { +namespace grappler { + +// This optimization sets intra-op parallelism to be 1. +class DisableIntraOpParallelism : public TFDataOptimizerBase { + public: + DisableIntraOpParallelism() = default; + ~DisableIntraOpParallelism() override = default; + + string name() const override { return "disable_intra_op_parallelism"; }; + + bool UsesFunctionLibrary() const override { return false; } + + Status Init( + const tensorflow::RewriterConfig_CustomGraphOptimizer* config) override { + return Status::OK(); + } + + Status OptimizeAndCollectStats(Cluster* cluster, const GrapplerItem& item, + GraphDef* output, + OptimizationStats* stats) override; + + void Feedback(Cluster* cluster, const GrapplerItem& item, + const GraphDef& optimize_output, double result) override; +}; + +} // namespace grappler +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DATA_DISABLE_INTRA_OP_PARALLELISM_H_ diff --git a/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism_test.cc b/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism_test.cc new file mode 100644 index 00000000000..76d6b46fb4e --- /dev/null +++ b/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism_test.cc @@ -0,0 +1,117 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism.h" + +#include "tensorflow/core/framework/attr_value_util.h" +#include "tensorflow/core/framework/function_testlib.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/grappler/grappler_item.h" +#include "tensorflow/core/grappler/optimizers/data/graph_test_utils.h" +#include "tensorflow/core/grappler/optimizers/data/graph_utils.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace grappler { +namespace { + +using test::function::NDef; + +// If the user manually sets intra op parallelism, we don't insert the op. +class IntraOpAlreadySetTest + : public ::testing::TestWithParam> {}; + +TEST_P(IntraOpAlreadySetTest, IntraOpParallelism) { + const string op = std::get<0>(GetParam()); + const int64 value = std::get<1>(GetParam()); + + GrapplerItem item; + MutableGraphView graph(&item.graph); + + NodeDef *start_val = graph_utils::AddScalarConstNode(0, &graph); + NodeDef *stop_val = graph_utils::AddScalarConstNode(10, &graph); + NodeDef *step_val = graph_utils::AddScalarConstNode(1, &graph); + std::vector range_inputs(3); + range_inputs[0] = start_val->name(); + range_inputs[1] = stop_val->name(); + range_inputs[2] = step_val->name(); + std::vector> range_attrs; + NodeDef *range_node = graph_utils::AddNode("", "RangeDataset", range_inputs, + range_attrs, &graph); + + NodeDef *max_parallelism_val = + graph_utils::AddScalarConstNode(value, &graph); + std::vector parallelism_inputs(2); + parallelism_inputs[0] = range_node->name(); + parallelism_inputs[1] = max_parallelism_val->name(); + std::vector> parallelism_attrs; + graph_utils::AddNode("", op, parallelism_inputs, parallelism_attrs, &graph); + + EXPECT_TRUE(graph_utils::ContainsNodeWithOp(op, item.graph)); + EXPECT_EQ(item.graph.node_size(), 6); + EXPECT_EQ(max_parallelism_val->attr().at("value").tensor().int64_val(0), + value); + + DisableIntraOpParallelism optimizer; + GraphDef output; + TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output)); + EXPECT_EQ(output.node_size(), 6); + EXPECT_TRUE(graph_utils::ContainsNodeWithOp(op, output)); + NodeDef test_node = output.node(graph_utils::FindGraphNodeWithOp(op, output)); + NodeDef test_val = output.node( + graph_utils::FindGraphNodeWithName(test_node.input(1), output)); + EXPECT_EQ(test_val.attr().at("value").tensor().int64_val(0), value); +} + +INSTANTIATE_TEST_SUITE_P( + Test, IntraOpAlreadySetTest, + ::testing::Combine( + ::testing::Values("MaxIntraOpParallelismDataset", + "ExperimentalMaxIntraOpParallelismDataset"), + ::testing::Values(1, 5))); + +// If the user hasn't set intra op parallelism, we insert the op to disable it. +TEST(IntraOpNotSetTest, IntraOpParallelism) { + GrapplerItem item; + + item.graph = test::function::GDef( + {NDef("start", "Const", {}, {{"value", 0}, {"dtype", DT_INT32}}), + NDef("stop", "Const", {}, {{"value", 10}, {"dtype", DT_INT32}}), + NDef("step", "Const", {}, {{"value", 1}, {"dtype", DT_INT32}}), + NDef("range", "RangeDataset", {"start", "stop", "step"}, + {{"output_shapes", gtl::ArraySlice{}}, + {"output_types", gtl::ArraySlice{}}}), + NDef("Sink", "Identity", {"range"}, {})}); + EXPECT_FALSE(graph_utils::ContainsNodeWithOp("MaxIntraOpParallelismDataset", + item.graph)); + EXPECT_EQ(item.graph.node_size(), 5); + + DisableIntraOpParallelism optimizer; + GraphDef output; + TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output)); + EXPECT_EQ(output.node_size(), 7); + EXPECT_TRUE( + graph_utils::ContainsNodeWithOp("MaxIntraOpParallelismDataset", output)); + NodeDef test_node = output.node( + graph_utils::FindGraphNodeWithOp("MaxIntraOpParallelismDataset", output)); + NodeDef test_val = output.node( + graph_utils::FindGraphNodeWithName(test_node.input(1), output)); + EXPECT_EQ(test_val.attr().at("value").tensor().int64_val(0), 1); +} + +} // namespace +} // namespace grappler +} // namespace tensorflow diff --git a/tensorflow/core/grappler/optimizers/data/inject_prefetch.cc b/tensorflow/core/grappler/optimizers/data/inject_prefetch.cc index eae8d294247..ed202c151ae 100644 --- a/tensorflow/core/grappler/optimizers/data/inject_prefetch.cc +++ b/tensorflow/core/grappler/optimizers/data/inject_prefetch.cc @@ -70,7 +70,7 @@ Status InjectPrefetch::OptimizeAndCollectStats(Cluster* cluster, graph_utils::SetUniqueGraphNodeName( strings::StrCat("inject/prefetch_", async_dataset_node->name()), graph.graph(), &prefetch_node); - prefetch_node.set_op("PrefetchDataset"); + prefetch_node.set_op(kPrefetchDataset); // `input_dataset` input *prefetch_node.mutable_input()->Add() = async_dataset_node->name(); // `buffer_size` input diff --git a/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc index bd7e18b807c..8d50a0409df 100644 --- a/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/data/meta_optimizer.cc @@ -35,8 +35,9 @@ using ConfigMap = std::map; // tf.data optimizations, in the order we want to perform them. -constexpr std::array kTFDataOptimizations = { +constexpr std::array kTFDataOptimizations = { "noop_elimination", + "disable_intra_op_parallelism", "shuffle_and_repeat_fusion", "map_fusion", "filter_fusion", From 3b47c2bdeadec041c62f2f56593e0054b2eb6743 Mon Sep 17 00:00:00 2001 From: Rahul Joshi Date: Thu, 6 Aug 2020 18:31:33 -0700 Subject: [PATCH 0615/1017] [MLIR][NFC] Adopt FuncOp/Region argument API's. - Use FuncOp::getArguments() and Region::getArguments() and friends where possible instead of going through the front() block. PiperOrigin-RevId: 325352975 Change-Id: Ib3dcfed692c0e04c554120a748f82e9efe009b89 --- .../hlo/lib/Dialect/mhlo/transforms/lhlo_legalize_to_gpu.cc | 6 +++--- .../compiler/mlir/lite/transforms/while_loop_outline.cc | 5 ++--- .../compiler/mlir/tensorflow/translate/import_model.cc | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/lhlo_legalize_to_gpu.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/lhlo_legalize_to_gpu.cc index 0d0b8b0ab6e..cffb58b37de 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/lhlo_legalize_to_gpu.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/lhlo_legalize_to_gpu.cc @@ -147,9 +147,9 @@ class LhloReduceToGPULaunchConverter : public OpConversionPattern { // Now copy over the actual body of the reduction, leaving out the // terminator. BlockAndValueMapping mapping; - mapping.map(reduce_op.body().front().getArgument(0), accumulator); - mapping.map(reduce_op.body().front().getArgument(1), rhs); - mapping.map(reduce_op.body().front().getArgument(2), accumulator); + mapping.map(reduce_op.body().getArgument(0), accumulator); + mapping.map(reduce_op.body().getArgument(1), rhs); + mapping.map(reduce_op.body().getArgument(2), accumulator); for (auto& nested : reduce_op.body().front().without_terminator()) { auto clone = rewriter.clone(nested, mapping); for (auto pair : llvm::zip(nested.getResults(), clone->getResults())) { diff --git a/tensorflow/compiler/mlir/lite/transforms/while_loop_outline.cc b/tensorflow/compiler/mlir/lite/transforms/while_loop_outline.cc index 3342981b75f..56b38ec58d8 100644 --- a/tensorflow/compiler/mlir/lite/transforms/while_loop_outline.cc +++ b/tensorflow/compiler/mlir/lite/transforms/while_loop_outline.cc @@ -80,7 +80,7 @@ void WhileOutlinePass::OutlineWhile(WhileOp while_op) { // The basic block arguments correspond to values that are loop carried, while // all those post are loop independent. Initialize extern_values with while_op // not loop carried operands. - auto num_loop_carried = while_op.cond().front().getNumArguments(); + auto num_loop_carried = while_op.cond().getNumArguments(); auto not_carried_operands = while_op.getOperands().drop_front(num_loop_carried); extern_values.insert(not_carried_operands.begin(), @@ -124,8 +124,7 @@ void WhileOutlinePass::OutlineWhile(WhileOp while_op) { // Collect new types. SmallVector types; types.reserve(extra_operands.size() + while_op.getNumOperands()); - for (BlockArgument ba : while_op.cond().front().getArguments()) - types.push_back(ba.getType()); + for (Type type : while_op.cond().getArgumentTypes()) types.push_back(type); for (Value operand : extern_values) types.push_back(operand.getType()); // Create outline function from region. Optional pass extra arguments through diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc b/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc index 27385e81262..ef0087c4310 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc @@ -2873,7 +2873,7 @@ void AdjustBoundInputArgTypes(mlir::ModuleOp module) { mlir::OpBuilder builder(func.getBody()); llvm::SmallVector new_input_types; for (int i = 0, e = func.getNumArguments(); i < e; i++) { - auto arg = func.front().getArgument(i); + auto arg = func.getArgument(i); auto global_tensor = mlir::tf_saved_model::LookupBoundInputOfType< mlir::tf_saved_model::GlobalTensorOp>(func, i, symbol_table); if (global_tensor) { From 39ea1f2706b7fad1905346f107e39259e8af7b7d Mon Sep 17 00:00:00 2001 From: Rick Chao Date: Thu, 6 Aug 2020 18:37:45 -0700 Subject: [PATCH 0616/1017] PSv2: Add distribute/client:parameter_server_client dep to tensorflow/python/distribute py_library so it gets built as part of tensorflow. PiperOrigin-RevId: 325353821 Change-Id: Ifba7e355c092dcdc138ef9f3ac5660d9c0e6e012 --- tensorflow/python/distribute/BUILD | 3 ++- tensorflow/python/distribute/__init__.py | 1 + tensorflow/python/distribute/client/BUILD | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD index 7965be1d90f..6bb0570d77b 100644 --- a/tensorflow/python/distribute/BUILD +++ b/tensorflow/python/distribute/BUILD @@ -146,6 +146,7 @@ py_library( ":mirrored_strategy", ":one_device_strategy", ":sharded_variable", + "//tensorflow/python/distribute/client:parameter_server_client", "//tensorflow/python/distribute/experimental", ], ) @@ -1775,7 +1776,7 @@ distribute_py_test( py_library( name = "parameter_server_strategy_v2", srcs = ["parameter_server_strategy_v2.py"], - srcs_version = "PY3", + srcs_version = "PY2AND3", deps = [ ":parameter_server_strategy", "//tensorflow/python:constant_op", diff --git a/tensorflow/python/distribute/__init__.py b/tensorflow/python/distribute/__init__.py index f9d0a95ea58..acb3c112226 100644 --- a/tensorflow/python/distribute/__init__.py +++ b/tensorflow/python/distribute/__init__.py @@ -25,6 +25,7 @@ from tensorflow.python.distribute import distribute_lib from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import mirrored_strategy from tensorflow.python.distribute import one_device_strategy +from tensorflow.python.distribute.client import parameter_server_client from tensorflow.python.distribute.experimental import collective_all_reduce_strategy from tensorflow.python.distribute.experimental import parameter_server_strategy # pylint: enable=unused-import diff --git a/tensorflow/python/distribute/client/BUILD b/tensorflow/python/distribute/client/BUILD index 35d8de95276..d0d8d3af4ec 100644 --- a/tensorflow/python/distribute/client/BUILD +++ b/tensorflow/python/distribute/client/BUILD @@ -10,7 +10,7 @@ exports_files(["LICENSE"]) py_library( name = "parameter_server_client", srcs = ["parameter_server_client.py"], - srcs_version = "PY3", + srcs_version = "PY2AND3", deps = [ ":client", "//tensorflow/python/distribute:parameter_server_strategy_v2", @@ -20,7 +20,7 @@ py_library( py_library( name = "client", srcs = ["client.py"], - srcs_version = "PY3", + srcs_version = "PY2AND3", deps = [ ":metric_utils", "//tensorflow/python:errors", @@ -84,7 +84,7 @@ tf_py_test( py_library( name = "metric_utils", srcs = ["metric_utils.py"], - srcs_version = "PY3", + srcs_version = "PY2AND3", deps = [ "//tensorflow/python/eager:monitoring", ], From ab66003a02ead8f8e8620cc18f2d7c07ed7d04f7 Mon Sep 17 00:00:00 2001 From: Chenkai Kuang Date: Thu, 6 Aug 2020 18:39:07 -0700 Subject: [PATCH 0617/1017] Disable collective_all_reduce_strategy_test msan test. PiperOrigin-RevId: 325354007 Change-Id: I38d72f6fd0bfb8111338a9d31c4ef5ac8a5d921c --- tensorflow/python/distribute/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD index 6bb0570d77b..f67f306706f 100644 --- a/tensorflow/python/distribute/BUILD +++ b/tensorflow/python/distribute/BUILD @@ -1497,6 +1497,7 @@ cuda_py_test( python_version = "PY3", tags = [ "multi_and_single_gpu", + "nomsan", # b/154224457: Re-enable when fixed. ], # b/155301154 broken with XLA:GPU xla_enable_strict_auto_jit = True, From a8108923832fcf7f43b2f551912951fa50bed066 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 6 Aug 2020 18:41:46 -0700 Subject: [PATCH 0618/1017] Integrate LLVM at llvm/llvm-project@9dbdaea9a0e6 Updates LLVM usage to match [9dbdaea9a0e6](https://github.com/llvm/llvm-project/commit/9dbdaea9a0e6) PiperOrigin-RevId: 325354353 Change-Id: Icb539b494c5dfa096c8ed605907ab9341c0f670f --- .../lib/Dialect/mhlo/transforms/lhlo_legalize_to_llvm.cc | 5 ++--- .../transforms/tf_framework_legalize_to_llvm_pass.cc | 2 +- .../compiler/xla/service/mlir_gpu/mlir_compiler_impl.cc | 8 ++++---- tensorflow/workspace.bzl | 4 ++-- third_party/mlir/BUILD | 1 + 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/lhlo_legalize_to_llvm.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/lhlo_legalize_to_llvm.cc index af64c448ad9..42b71543543 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/lhlo_legalize_to_llvm.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/lhlo_legalize_to_llvm.cc @@ -217,8 +217,7 @@ struct ReshapeMemRefCastOpConverter SmallVector sizes; UnrankedMemRefDescriptor::computeSizes(rewriter, loc, typeConverter, {target_desc}, sizes); - auto void_ptr_type = - LLVM::LLVMType::getInt8PtrTy(typeConverter.getDialect()); + auto void_ptr_type = LLVM::LLVMType::getInt8PtrTy(rewriter.getContext()); Value ranked_desc_mem = rewriter.create( loc, void_ptr_type, sizes.front(), llvm::None); target_desc.setMemRefDescPtr(rewriter, loc, ranked_desc_mem); @@ -282,7 +281,7 @@ struct ReshapeMemRefCastOpConverter auto index_arg = cond_block->addArgument(typeConverter.getIndexType()); auto stride_arg = cond_block->addArgument(typeConverter.getIndexType()); auto pred = rewriter.create( - loc, LLVM::LLVMType::getInt1Ty(typeConverter.getDialect()), + loc, LLVM::LLVMType::getInt1Ty(rewriter.getContext()), LLVM::ICmpPredicate::sge, index_arg, zero_index); Block *body_block = diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc index 41b38bb574f..42e89433dff 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/tf_framework_legalize_to_llvm_pass.cc @@ -40,7 +40,7 @@ class TestTFFrameworkToLLVMPass // Populate type conversions. LLVMTypeConverter type_converter(m.getContext()); type_converter.addConversion([&](tf_framework::OpKernelContextType type) { - return LLVM::LLVMType::getInt8PtrTy(type_converter.getDialect()); + return LLVM::LLVMType::getInt8PtrTy(m.getContext()); }); // Populate patterns. diff --git a/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler_impl.cc b/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler_impl.cc index 2c2076bbd97..25a35a89cb4 100644 --- a/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler_impl.cc +++ b/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler_impl.cc @@ -292,10 +292,10 @@ Status InsertBufferLoadPreduleIntoKernel( BufferAssignment* assignment, const std::vector& buffers) { mlir::OpBuilder builder(kernel.getBody()); - auto llvm_dialect = kernel.getContext()->getRegisteredDialect(); - auto offset_type = LLVMType::getInt64Ty(llvm_dialect); - auto ptr_type = LLVMType::getInt8PtrTy(llvm_dialect); - auto void_type = LLVMType::getVoidTy(llvm_dialect); + auto* context = kernel.getContext(); + auto offset_type = LLVMType::getInt64Ty(context); + auto ptr_type = LLVMType::getInt8PtrTy(context); + auto void_type = LLVMType::getVoidTy(context); auto loc = kernel.getLoc(); auto num_original_args = kernel.getNumArguments(); diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 440aa1b23ec..07b9950bca2 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "633e3dacf27ea4950b7067803502490597ba96e0" - LLVM_SHA256 = "585299b33c32ea3a39b0cfb70e5dd431f3ab064d9f96baa4787693b3c66af21e" + LLVM_COMMIT = "9dbdaea9a0e6f58417b5bd8980e7ea6723fd1783" + LLVM_SHA256 = "1ae491e33bb35777cf5f38acd183ce3ca2aff255c15254ae97084bcbd2e4aa56" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), diff --git a/third_party/mlir/BUILD b/third_party/mlir/BUILD index 872d04194f6..eeb78e0544b 100644 --- a/third_party/mlir/BUILD +++ b/third_party/mlir/BUILD @@ -1491,6 +1491,7 @@ cc_library( ":IR", ":LLVMDialect", ":Pass", + ":StandardToLLVM", ":Support", ":TargetNVVMIR", "@llvm-project//llvm:Core", From d51d619a0e7e9bb97c1a6bc1754d09db1c37f1b1 Mon Sep 17 00:00:00 2001 From: Ran Chen Date: Thu, 6 Aug 2020 18:55:08 -0700 Subject: [PATCH 0619/1017] Clean up is_saving_non_distributed() options here is never None. It's confusing to return False when it's None, since the default variable policy is NONE, not EXPAND_DISTRIBUTED_VARIABLE PiperOrigin-RevId: 325355974 Change-Id: Idfcb9498e3efcc20729e6b1309e47c7f80f35862 --- tensorflow/python/distribute/values_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/distribute/values_util.py b/tensorflow/python/distribute/values_util.py index 67b5234d82c..535351e6563 100644 --- a/tensorflow/python/distribute/values_util.py +++ b/tensorflow/python/distribute/values_util.py @@ -270,5 +270,5 @@ def is_saving_non_distributed(): if not save_context.in_save_context(): return False options = save_context.get_save_options() - return (options is not None and options.experimental_variable_policy != + return (options.experimental_variable_policy != save_options.VariablePolicy.EXPAND_DISTRIBUTED_VARIABLES) From 78606da47fb80ef2b16ad81b5c5f1129857040cd Mon Sep 17 00:00:00 2001 From: Robert David Date: Thu, 6 Aug 2020 19:23:10 -0700 Subject: [PATCH 0620/1017] Add include-what-you-use pragma so IWYU does not try to include these files again. PiperOrigin-RevId: 325359254 Change-Id: Ibeb53b70736036ab22ed59858f31adb0b55c65a7 --- tensorflow/lite/delegates/gpu/common/status.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/common/status.h b/tensorflow/lite/delegates/gpu/common/status.h index d6b5dd8a94a..22dcc11d57f 100644 --- a/tensorflow/lite/delegates/gpu/common/status.h +++ b/tensorflow/lite/delegates/gpu/common/status.h @@ -16,7 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_LITE_DELEGATES_GPU_COMMON_STATUS_H_ #define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_STATUS_H_ -#include "absl/status/status.h" -#define RETURN_IF_ERROR(s) {auto c=(s);if(!c.ok())return c;} +#include "absl/status/status.h" // IWYU pragma: export +#define RETURN_IF_ERROR(s) {auto c=(s);if(!c.ok())return c;} // IWYU pragma: export #endif // TENSORFLOW_LITE_DELEGATES_GPU_COMMON_STATUS_H_ From 74a3400d3f609b604cb44c48c3cd2bd7582eec5c Mon Sep 17 00:00:00 2001 From: Ran Chen Date: Thu, 6 Aug 2020 19:36:38 -0700 Subject: [PATCH 0621/1017] Add a test of some known deadlocks after peer failures PiperOrigin-RevId: 325360404 Change-Id: I003fac1bb797e8bacbaaafa02be6e8cc41e35f66 --- .../python/distribute/integration_test/BUILD | 19 ++ .../mwms_peer_failure_test.py | 167 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 tensorflow/python/distribute/integration_test/mwms_peer_failure_test.py diff --git a/tensorflow/python/distribute/integration_test/BUILD b/tensorflow/python/distribute/integration_test/BUILD index 156699fad7b..307f2580996 100644 --- a/tensorflow/python/distribute/integration_test/BUILD +++ b/tensorflow/python/distribute/integration_test/BUILD @@ -1,4 +1,5 @@ load("//tensorflow/core/platform/default:distribute.bzl", "distribute_py_test") +load("//tensorflow:tensorflow.bzl", "cuda_py_test") package( default_visibility = ["//tensorflow:internal"], @@ -19,3 +20,21 @@ distribute_py_test( "@absl_py//absl/testing:parameterized", ], ) + +cuda_py_test( + name = "mwms_peer_failure_test", + size = "medium", + srcs = ["mwms_peer_failure_test.py"], + python_version = "PY3", + shard_count = 2, + tags = [ + "multi_and_single_gpu", + ], + deps = [ + "//tensorflow:tensorflow_py", + "//tensorflow/python/distribute:combinations", + "//tensorflow/python/distribute:multi_process_runner", + "//tensorflow/python/distribute:multi_worker_test_base", + "//tensorflow/python/eager:test", + ], +) diff --git a/tensorflow/python/distribute/integration_test/mwms_peer_failure_test.py b/tensorflow/python/distribute/integration_test/mwms_peer_failure_test.py new file mode 100644 index 00000000000..c247be1c280 --- /dev/null +++ b/tensorflow/python/distribute/integration_test/mwms_peer_failure_test.py @@ -0,0 +1,167 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""This file contains tests that simulate peer failures. + +When a peer fails during MultiWorkerMirroredStrategy training. All workers +should get Unavailable error. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os + +import tensorflow as tf + +from tensorflow.python.distribute import combinations +from tensorflow.python.distribute import multi_process_runner +from tensorflow.python.distribute import multi_worker_test_base +from tensorflow.python.eager import test + + +def get_attempt(strategy, attempts): + task_type = strategy.cluster_resolver.task_type + task_id = strategy.cluster_resolver.task_id + attempts[(task_type, task_id)] = attempts.get((task_type, task_id), 0) + 1 + return task_id, attempts[(task_type, task_id)] + + +quick_exit = os._exit # pylint: disable=protected-access + + +class PeerFailureTest(test.TestCase): + # Note that all the tests use auto_restart=True. Currently we rely on the + # assumption that an external system restarts failed tasks. If the assumption + # is not true, the remaining tasks may still hang instead of fail. + # + # In these tests we leverage the auto restart feature of MultiProcessRunner. + # Failed workers are restarted automatically. In reality there needs to be + # some job management system that does the restart, e.g. Kubernetes. + # + # Worker failures may cause problems if there're more than one collective, and + # the failure happens after the first collective. In this case the recovered + # worker will be running a different collective with the rest, which causes a + # deadlock. Note that collectives are common, e.g. when creating variables the + # initial values are broadcasted from the first worker. + # + # We use a multiprocessing.Manager().dict() object to track the attempts of + # each worker. We take different actions in different attempts to simuate the + # events in real world. E.g. some tests make a worker fail on the first + # attempt only, and asserts that it should recovery. + + def test_creating_variable_broken(self): + # This test simulates the case when a worker fails before or during creating + # a variable. Creating variables involve broadcasting the initial value from + # the first replica to all replicas. + + def worker_fn(attempts): + strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() + task_id, attempt = get_attempt(strategy, attempts) + with strategy.scope(): + tf.Variable(1.) + # worker-1 dies here. + if attempt == 1 and task_id == 1: + quick_exit(1) + v = tf.Variable(tf.random.uniform(())) + return v.read_value().numpy() + + cluster_spec = multi_worker_test_base.create_cluster_spec(num_workers=2) + attempts = multi_process_runner.manager().dict() + mpr = multi_process_runner.MultiProcessRunner( + worker_fn, cluster_spec, args=(attempts,), auto_restart=True) + mpr.start() + # TODO(b/151232436): worker-0 should raises Unavailable instead of hanging. + # Now after worker-1 fails, worker-0 waits on the second variable creation; + # after worker-1 recovers, worker-1 waits on the first variable creation. + with self.assertRaises(multi_process_runner.SubprocessTimeoutError): + mpr.join(timeout=30) + + def test_reduce_small_tensor_broken(self): + # This test simulates the case when a worker fails before or during reducing + # a small tensors, e.g. reading a metric. + # + # Note that this is a rather corner case and only happens when all of the + # following conditions are met: + # - There're two workers. + # - They're reducing a small tensor. The definition of small varies + # per platform. + # - They're reducing a single tensor. Batched all-reduce are not affected. + # - It must be worker-1 that fails. + + def worker_fn(attempts): + strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() + task_id, attempt = get_attempt(strategy, attempts) + value = tf.identity([1.]) + strategy.reduce("sum", value, axis=None) + # worker-1 dies here. + if attempt == 1 and task_id == 1: + quick_exit(1) + strategy.reduce("sum", value, axis=None) + + cluster_spec = multi_worker_test_base.create_cluster_spec(num_workers=2) + attempts = multi_process_runner.manager().dict() + mpr = multi_process_runner.MultiProcessRunner( + worker_fn, cluster_spec, args=(attempts,), auto_restart=True) + mpr.start() + # TODO(b/151232436): worker-0 should raises Unavailable instead of hanging. + # Now after worker-1 fails, worker-0 waits on the second reduce; after + # worker-1 recovers, worker-1 waits on the first reduce. + with self.assertRaises(multi_process_runner.SubprocessTimeoutError): + mpr.join(timeout=30) + + def test_quick_recover(self): + # This test simulates the case when a worker fails but recovers quickly + # before the next collective. + # + # It's not guaranteed that the cluster only restarts once when one worker + # fails. The external job management system is expected to keep restarting + # failed workers. + + def worker_fn(attempts): + strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() + task_id, attempt = get_attempt(strategy, attempts) + + if attempt == 2 and task_id == 1: + multi_process_runner.barrier().wait() + + @tf.function + def replica_fn(): + ctx = tf.distribute.get_replica_context() + # Use a large tensor because small tensor may hang regardless when the + # worker recovers. + value = tf.ones((64, 64)) + ctx.all_reduce(tf.distribute.ReduceOp.SUM, [value, value]) + + strategy.run(replica_fn) + # worker-1 dies here. + if attempt == 1 and task_id == 1: + quick_exit(1) + # Make worker-0 waits for worker-1 to restart before entering the next + # collective to simulate a quick recovery of worker-1. + if attempt == 1 and task_id == 0: + multi_process_runner.barrier().wait() + strategy.run(replica_fn) + + cluster_spec = multi_worker_test_base.create_cluster_spec(num_workers=2) + attempts = multi_process_runner.manager().dict() + mpr = multi_process_runner.MultiProcessRunner( + worker_fn, cluster_spec, args=(attempts,), auto_restart=True) + mpr.start() + mpr.join(timeout=90) + + +if __name__ == "__main__": + combinations.main() From d547659011b22058e99ba97f941166954b69b29d Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Thu, 6 Aug 2020 20:34:30 -0700 Subject: [PATCH 0622/1017] Fix mlir-hlo CMakefiles after internal change PiperOrigin-RevId: 325366142 Change-Id: I17a38ef6ffd23052f1d112355caa4798bf269a27 --- .../include/mlir-hlo/Dialect/mhlo/transforms/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/CMakeLists.txt b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/CMakeLists.txt index 6fbc5306a8f..6de6851b8d7 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/CMakeLists.txt +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/CMakeLists.txt @@ -15,9 +15,9 @@ # set(LLVM_TARGET_DEFINITIONS mhlo_passes.td) -mlir_tablegen(mhlo_passes.h.inc -gen-pass-decls) +mlir_tablegen(mhlo_passes.h.inc -gen-pass-decls -name MHLO) add_public_tablegen_target(MLIRMhloPassIncGen) set(LLVM_TARGET_DEFINITIONS lmhlo_passes.td) -mlir_tablegen(lmhlo_passes.h.inc -gen-pass-decls) +mlir_tablegen(lmhlo_passes.h.inc -gen-pass-decls -name LMHLO) add_public_tablegen_target(MLIRLmhloPassIncGen) From 0a764ff2fe38274592d721690cc29c5612010967 Mon Sep 17 00:00:00 2001 From: Yuqi Li Date: Thu, 6 Aug 2020 21:00:05 -0700 Subject: [PATCH 0623/1017] Add homepage for TFLite Model Maker under "guide" PiperOrigin-RevId: 325368667 Change-Id: I3f4d1162684564373a9abbda59c520928448a0cf --- tensorflow/lite/g3doc/_book.yaml | 5 ++ tensorflow/lite/g3doc/guide/get_started.md | 4 ++ tensorflow/lite/g3doc/guide/model_maker.md | 60 ++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 tensorflow/lite/g3doc/guide/model_maker.md diff --git a/tensorflow/lite/g3doc/_book.yaml b/tensorflow/lite/g3doc/_book.yaml index 45be4737fd5..96ec7363ab1 100644 --- a/tensorflow/lite/g3doc/_book.yaml +++ b/tensorflow/lite/g3doc/_book.yaml @@ -93,6 +93,11 @@ upper_tabs: - title: "1.x compatibility" path: /lite/convert/1x_compatibility + - heading: "Create a model" + - title: "TensorFlow Lite Model Maker" + status: experimental + path: /lite/guide/model_maker + - heading: "Inference" - title: "Overview" path: /lite/guide/inference diff --git a/tensorflow/lite/g3doc/guide/get_started.md b/tensorflow/lite/g3doc/guide/get_started.md index c9543c7f553..df206e73416 100644 --- a/tensorflow/lite/g3doc/guide/get_started.md +++ b/tensorflow/lite/g3doc/guide/get_started.md @@ -67,6 +67,10 @@ If you have designed and trained your own TensorFlow model, or you have trained a model obtained from another source, you must [convert it to the TensorFlow Lite format](#2_convert_the_model_format). +You can also try [The TensorFlow Lite Model Maker library](model_maker.md) which +simplifies the process of training a TensorFlow Lite model using custom +datasets. + ## 2. Convert the model diff --git a/tensorflow/lite/g3doc/guide/model_maker.md b/tensorflow/lite/g3doc/guide/model_maker.md new file mode 100644 index 00000000000..824fb1a3fd6 --- /dev/null +++ b/tensorflow/lite/g3doc/guide/model_maker.md @@ -0,0 +1,60 @@ +# TensorFlow Lite Model Maker + +## Overview + +The TensorFlow Lite Model Maker library simplifies the process of training a +TensorFlow Lite model using custom dataset. It uses transfer learning to reduce +the amount of training data required and shorten the training time. + +## Supported Tasks + +The Model Maker library currently supports the following ML tasks. Click the +links below for guides on how to train the model. + +Supported Tasks | Task Utility +-------------------------------------------------------------------------------------------------------- | ------------ +Image Classification [guide](https://www.tensorflow.org/lite/tutorials/model_maker_image_classification) | Classify images into predefined categories. +Text Classification [guide](https://www.tensorflow.org/lite/tutorials/model_maker_text_classification) | Classify text into predefined categories. +Question Answer [guide](https://www.tensorflow.org/lite/tutorials/model_maker_question_answer) | Find the answer in a certain context for a given question. + +## End-to-End Example + +Model Maker allows you to train a TensorFlow Lite model using custom datasets in +just a few lines of code. For example, here are the steps to train an image +classification model. + +```python +# Load input data specific to an on-device ML app. +data = ImageClassifierDataLoader.from_folder('flower_photos/') +train_data, test_data = data.split(0.9) + +# Customize the TensorFlow model. +model = image_classifier.create(data) + +# Evaluate the model. +loss, accuracy = model.evaluate(test_data) + +# Export to Tensorflow Lite model and label file in `export_dir`. +model.export(export_dir='/tmp/') +``` + +For more details, see the +[image classification guide](https://www.tensorflow.org/lite/tutorials/model_maker_image_classification). + +## Installation + +There are two ways to install Model Maker. + +* Install a prebuilt pip package + +```shell +pip install tflite-model-maker +``` + +* Clone the source code from GitHub and install. + +```shell +git clone https://github.com/tensorflow/examples +cd examples +pip install .[model_maker] +``` From 982d7edbfa023bb271a5e7592c747b63919705bf Mon Sep 17 00:00:00 2001 From: Haoliang Zhang Date: Thu, 6 Aug 2020 23:12:36 -0700 Subject: [PATCH 0624/1017] Internal cleanup on minor namespace issues. PiperOrigin-RevId: 325383289 Change-Id: I76247774f4a1bf06de4547f0b70a35c77c3af68f --- tensorflow/compiler/mlir/tensorflow/utils/error_util.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/tensorflow/utils/error_util.h b/tensorflow/compiler/mlir/tensorflow/utils/error_util.h index 4feb3837357..b5f2acc581d 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/error_util.h +++ b/tensorflow/compiler/mlir/tensorflow/utils/error_util.h @@ -27,7 +27,7 @@ limitations under the License. namespace mlir { // TensorFlow's Status is used for error reporting back to callers. -using tensorflow::Status; +using ::tensorflow::Status; // Diagnostic handler that collects all the diagnostics reported and can produce // a Status to return to callers. This is for the case where MLIR functions are From d50776efab854ae3af2bfbfaef0f54c836a09fe9 Mon Sep 17 00:00:00 2001 From: Yuqi Li Date: Thu, 6 Aug 2020 23:27:23 -0700 Subject: [PATCH 0625/1017] Remove install from github directly for tflite model maker. PiperOrigin-RevId: 325384550 Change-Id: Iab4572a8ec9a105f8b80a5b2a9aae64f4558379a --- tensorflow/lite/g3doc/guide/model_maker.md | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tensorflow/lite/g3doc/guide/model_maker.md b/tensorflow/lite/g3doc/guide/model_maker.md index 824fb1a3fd6..76b32eac75e 100644 --- a/tensorflow/lite/g3doc/guide/model_maker.md +++ b/tensorflow/lite/g3doc/guide/model_maker.md @@ -43,18 +43,8 @@ For more details, see the ## Installation -There are two ways to install Model Maker. - -* Install a prebuilt pip package +Install a prebuilt pip package. ```shell pip install tflite-model-maker ``` - -* Clone the source code from GitHub and install. - -```shell -git clone https://github.com/tensorflow/examples -cd examples -pip install .[model_maker] -``` From bf3b427a6763f7109b0a08124108e3a4836ef549 Mon Sep 17 00:00:00 2001 From: Chao Mei Date: Thu, 6 Aug 2020 23:44:12 -0700 Subject: [PATCH 0626/1017] Add the flat_hash_map explicitly to the dep. PiperOrigin-RevId: 325385963 Change-Id: I953e4b0c6f26d60b1df7e54c9f7246b234300c89 --- tensorflow/lite/delegates/gpu/gl/workgroups/BUILD | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/gl/workgroups/BUILD b/tensorflow/lite/delegates/gpu/gl/workgroups/BUILD index 52fdb7435f9..1048912d754 100644 --- a/tensorflow/lite/delegates/gpu/gl/workgroups/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/workgroups/BUILD @@ -32,15 +32,16 @@ cc_library( deps = select({ "//tensorflow/lite/delegates/gpu:tflite_gpu_binary_release": [], "//conditions:default": [ - ":default_calculator", - "//tensorflow/lite/delegates/gpu/gl:common_cc_fbs", - "//tensorflow/lite/delegates/gpu/gl:workgroups_cc_fbs", - "//tensorflow/lite/delegates/gpu/common:gpu_info", - "//tensorflow/lite/delegates/gpu/gl:metadata_cc_fbs", ":calculator", + ":default_calculator", + "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", "@flatbuffers", + "//tensorflow/lite/delegates/gpu/common:gpu_info", "//tensorflow/lite/delegates/gpu/common:types", + "//tensorflow/lite/delegates/gpu/gl:common_cc_fbs", + "//tensorflow/lite/delegates/gpu/gl:metadata_cc_fbs", + "//tensorflow/lite/delegates/gpu/gl:workgroups_cc_fbs", ], }), ) From 5e0ed38eb746f3a86463f19bcf7138a959ddb2d4 Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Fri, 7 Aug 2020 00:37:37 -0700 Subject: [PATCH 0627/1017] Use string_view type for FormatFromString and FilterFormatFromString arguments PiperOrigin-RevId: 325391279 Change-Id: If834446e5eac71840b92e6efcd6eac5170579769 --- tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc | 9 ++++++--- tensorflow/core/util/BUILD | 1 + tensorflow/core/util/tensor_format.cc | 4 ++-- tensorflow/core/util/tensor_format.h | 5 +++-- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc index 39dae59ecb4..1a730a38618 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_a_m.cc @@ -339,7 +339,8 @@ void BatchToSpaceOp::getCanonicalizationPatterns( // are not unknown. // static LogicalResult Verify(BiasAddOp op) { - std::string data_format = op.data_format().str(); + absl::string_view data_format(op.data_format().data(), + op.data_format().size()); tensorflow::TensorFormat format; bool is_valid = FormatFromString(data_format, &format); DCHECK(is_valid) << data_format; @@ -385,7 +386,8 @@ static LogicalResult Verify(BiasAddOp op) { // * the out_backprop operands have valid ranks or are unranked. // static LogicalResult Verify(BiasAddGradOp op) { - std::string data_format = op.data_format().str(); + absl::string_view data_format(op.data_format().data(), + op.data_format().size()); tensorflow::TensorFormat format; bool is_valid = FormatFromString(data_format, &format); DCHECK(is_valid) << data_format; @@ -995,7 +997,8 @@ static LogicalResult Verify(OpT op) { int64_t input_channels = -1; if (auto ty = op.input().getType().template dyn_cast()) { - std::string data_format = op.data_format().str(); + absl::string_view data_format(op.data_format().data(), + op.data_format().size()); tensorflow::TensorFormat format; auto is_valid = FormatFromString(data_format, &format); DCHECK(is_valid) << data_format; diff --git a/tensorflow/core/util/BUILD b/tensorflow/core/util/BUILD index dcb2787e309..634a937d1c4 100644 --- a/tensorflow/core/util/BUILD +++ b/tensorflow/core/util/BUILD @@ -519,6 +519,7 @@ cc_library( "//tensorflow/core/lib/gtl:array_slice", "//tensorflow/core/lib/gtl:inlined_vector", "//tensorflow/core/platform:types", + "@com_google_absl//absl/strings", ], ) diff --git a/tensorflow/core/util/tensor_format.cc b/tensorflow/core/util/tensor_format.cc index 5dbd8ef318f..008c4d45200 100644 --- a/tensorflow/core/util/tensor_format.cc +++ b/tensorflow/core/util/tensor_format.cc @@ -73,7 +73,7 @@ string ToString(FilterTensorFormat format) { } } -bool FormatFromString(const string& format_str, TensorFormat* format) { +bool FormatFromString(absl::string_view format_str, TensorFormat* format) { if (format_str == "NHWC" || format_str == "NDHWC") { *format = FORMAT_NHWC; return true; @@ -101,7 +101,7 @@ bool FormatFromString(const string& format_str, TensorFormat* format) { return false; } -bool FilterFormatFromString(const string& format_str, +bool FilterFormatFromString(absl::string_view format_str, FilterTensorFormat* format) { if (format_str == "HWIO" || format_str == "DHWIO") { *format = FORMAT_HWIO; diff --git a/tensorflow/core/util/tensor_format.h b/tensorflow/core/util/tensor_format.h index d2d7b9e58de..07762f84300 100644 --- a/tensorflow/core/util/tensor_format.h +++ b/tensorflow/core/util/tensor_format.h @@ -19,6 +19,7 @@ limitations under the License. #include #include +#include "absl/strings/string_view.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/lib/gtl/array_slice.h" #include "tensorflow/core/lib/gtl/inlined_vector.h" @@ -97,11 +98,11 @@ enum FilterTensorFormat { // Parse tensor format from the given string. // Return true if the parsing succeeds, and false if it fails. -bool FormatFromString(const std::string& format_str, TensorFormat* format); +bool FormatFromString(absl::string_view format_str, TensorFormat* format); // Parse tensor format from the given string. // Return true if the parsing succeeds, and false if it fails. -bool FilterFormatFromString(const std::string& format_str, +bool FilterFormatFromString(absl::string_view format_str, FilterTensorFormat* format); // Convert a tensor format into string. From 155e561f2dae8ffef6570cf36d86e2d18a7eb0ce Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 02:01:30 -0700 Subject: [PATCH 0628/1017] Update GraphDef version to 486. PiperOrigin-RevId: 325399599 Change-Id: I40193d624f2af8aa0e8dd47ce17c9bf49545e948 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 431784a5a1a..ee9be29958d 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 485 // Updated: 2020/8/6 +#define TF_GRAPH_DEF_VERSION 486 // Updated: 2020/8/7 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From a8230634dddbbc1a097a8c24295d46680b897e4c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 02:01:33 -0700 Subject: [PATCH 0629/1017] compat: Update forward compatibility horizon to 2020-08-07 PiperOrigin-RevId: 325399607 Change-Id: I61d99514eb9400fd92e02762d19fd8011a856a0d --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index 65bb633855a..bef47619972 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 6) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 7) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From 542871f6a3a0346d2dc74715480a403edcd73c09 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Thu, 2 Jul 2020 01:03:17 +0200 Subject: [PATCH 0630/1017] Revert rollback of #40564 This reverts commit 8535dafb37ec4ce5c7272ffa4b8b4c491d44e999. --- .../experimental/autocast_variable.py | 103 +++++++++--------- .../experimental/autocast_variable_test.py | 31 ++++-- 2 files changed, 77 insertions(+), 57 deletions(-) diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py index caad08ce066..d44ff5f53e5 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py @@ -188,61 +188,87 @@ class AutoCastVariable(variables.Variable, core.Tensor): def constraint(self): return self._variable.constraint + def _apply_assign_update(self, + update_fn, + value, + use_locking=None, + name=None, + read_value=True): + if ops.executing_eagerly_outside_functions(): + assign_op = update_fn(value, use_locking, name, False) + return self if read_value else assign_op + + # Fallback to wrapping the returned variable in graph mode if possible + assign_var = update_fn(value, use_locking, name, read_value) + if read_value and resource_variable_ops.is_resource_variable(assign_var): + return create_autocast_variable(assign_var) + return assign_var + + def _apply_update(self, update_fn, *args, **kwargs): + update_var = update_fn(*args, **kwargs) + if ops.executing_eagerly_outside_functions(): + return self + + # Fallback to wrapping the returned variable in graph mode if possible + if resource_variable_ops.is_resource_variable(update_var): + return create_autocast_variable(update_var) + return update_var + def assign(self, value, use_locking=None, name=None, read_value=True): - assign_op = self._variable.assign(value, use_locking, name, read_value) - return _maybe_wrap(assign_op, wrap=read_value) + return self._apply_assign_update(self._variable.assign, value, use_locking, + name, read_value) def assign_add(self, delta, use_locking=None, name=None, read_value=True): - assign_op = self._variable.assign_add(delta, use_locking, name, read_value) - return _maybe_wrap(assign_op, wrap=read_value) + return self._apply_assign_update(self._variable.assign_add, delta, + use_locking, name, read_value) def assign_sub(self, delta, use_locking=None, name=None, read_value=True): - assign_op = self._variable.assign_sub(delta, use_locking, name, read_value) - return _maybe_wrap(assign_op, wrap=read_value) + return self._apply_assign_update(self._variable.assign_sub, delta, + use_locking, name, read_value) def scatter_sub(self, sparse_delta, use_locking=False, name=None): - var = self._variable.scatter_sub(sparse_delta, use_locking, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_sub, sparse_delta, + use_locking, name) def scatter_add(self, sparse_delta, use_locking=False, name=None): - var = self._variable.scatter_add(sparse_delta, use_locking, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_add, sparse_delta, + use_locking, name) def scatter_max(self, sparse_delta, use_locking=False, name=None): - var = self._variable.scatter_max(sparse_delta, use_locking, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_max, sparse_delta, + use_locking, name) def scatter_min(self, sparse_delta, use_locking=False, name=None): - var = self._variable.scatter_min(sparse_delta, use_locking, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_min, sparse_delta, + use_locking, name) def scatter_mul(self, sparse_delta, use_locking=False, name=None): - var = self._variable.scatter_mul(sparse_delta, use_locking, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_mul, sparse_delta, + use_locking, name) def scatter_div(self, sparse_delta, use_locking=False, name=None): - var = self._variable.scatter_div(sparse_delta, use_locking, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_div, sparse_delta, + use_locking, name) def scatter_update(self, sparse_delta, use_locking=False, name=None): - var = self._variable.scatter_update(sparse_delta, use_locking, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_update, sparse_delta, + use_locking, name) def batch_scatter_update(self, sparse_delta, use_locking=False, name=None): - var = self._variable.batch_scatter_update(sparse_delta, use_locking, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.batch_scatter_update, sparse_delta, + use_locking, name) def scatter_nd_sub(self, indices, updates, name=None): - var = self._variable.scatter_nd_sub(indices, updates, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_nd_sub, indices, updates, + name) def scatter_nd_add(self, indices, updates, name=None): - var = self._variable.scatter_nd_add(indices, updates, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_nd_add, indices, updates, + name) def scatter_nd_update(self, indices, updates, name=None): - var = self._variable.scatter_nd_update(indices, updates, name) - return _maybe_wrap(var) + return self._apply_update(self._variable.scatter_nd_update, indices, + updates, name) def load(self, value, session=None): return self._variable.load(value, session) @@ -469,24 +495,3 @@ def create_autocast_variable(variable): # pylint: enable=missing-format-attribute return AutoCastDistributedVariable(variable) - - -def _maybe_wrap(variable, wrap=True): - """Creates an AutoCastVariable that wraps another variable if applicable. - - This function is used to wrap the return value of AutoCastVariable.assign. - Unfortunately MirroredVariable.assign will (incorrectly) return a Mirrored - value instead of a MirroredVariable. So we cannot properly wrap it in an - AutoCastVariable. We return the original variable in that case. - - Args: - variable: A tf.Variable or op. - wrap: A boolean to define whether to wrap the variable in an - AutoCastVariable or not. - - Returns: - An AutoCastVariable if wrap is True and variable is a resource variable. - """ - if wrap and resource_variable_ops.is_resource_variable(variable): - return create_autocast_variable(variable) - return variable diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py index c3162e0e80a..c2dd69fc0ec 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py @@ -305,8 +305,8 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): self.assertAllClose(3., self.evaluate(x.assign_sub(3.))) # Assign multiple times - # This currently only works if no strategy is used - if not ds_context.has_strategy(): + # This currently doesn't work in graph mode if a strategy is used + if not ds_context.has_strategy() or context.executing_eagerly(): assign = x.assign(1.) self.assertAllClose(1., self.evaluate(assign)) self.assertAllClose(0., self.evaluate(assign.assign(0.))) @@ -344,6 +344,23 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): # assign still expect float32 value even if in float16 scope run_and_check() + @combinations.generate(maybe_distribute) + def test_assign_tf_function(self, distribution): + if not context.executing_eagerly(): + self.skipTest('Test is not compatible with graph mode') + + with distribution.scope(): + x = get_var(0., dtypes.float32) + x = autocast_variable.create_autocast_variable(x) + + @def_function.function + def run_assign(): + return x.assign(1.).assign_add(3.).assign_add(3.).assign_sub(2.) + + with ops.get_default_graph()._enable_auto_casting_variables( + dtypes.float16): + self.assertAllClose(5., self.evaluate(run_assign())) + @combinations.generate(maybe_distribute) def test_assign_stays_in_true_dtype(self, distribution): with distribution.scope(): @@ -358,18 +375,16 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): dtypes.float16): # Variable should be increased, despite it appearing to be the same # float16 value. - self.assertEqual(1. + small_val, - self.evaluate(x.assign(1. + small_tensor))) + self.evaluate(x.assign(1. + small_tensor)) self.assertEqual(1., self.evaluate(x.value())) - self.assertEqual(1. + small_val, self.evaluate(x.value())) + self.assertEqual(1. + small_val, self.evaluate(x)) self.evaluate(x.assign(1.)) with ops.get_default_graph()._enable_auto_casting_variables( dtypes.float16): - self.assertEqual(1. + small_val, - self.evaluate(x.assign_add(small_tensor))) + self.evaluate(x.assign_add(small_tensor)) self.assertEqual(1., self.evaluate(x.value())) - self.assertEqual(1. + small_val, self.evaluate(x.value())) + self.assertEqual(1. + small_val, self.evaluate(x)) @combinations.generate(maybe_distribute) def test_checkpoint(self, distribution): From 0b2fec54e14f7823bf17b67522804535597d1f04 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Thu, 2 Jul 2020 01:11:30 +0200 Subject: [PATCH 0631/1017] Add test for control deps of AutoCastDistributedVariable in tf.function --- .../experimental/autocast_variable_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py index c2dd69fc0ec..7b4d821aaff 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py @@ -361,6 +361,24 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): dtypes.float16): self.assertAllClose(5., self.evaluate(run_assign())) + @combinations.generate(maybe_distribute) + def test_tf_function_control_dependencies(self, distribution): + if not context.executing_eagerly(): + self.skipTest('Test is not compatible with graph mode') + + with distribution.scope(): + x = get_var(0., dtypes.float32) + x = autocast_variable.create_autocast_variable(x) + + @def_function.function + def func(): + update = x.assign_add(1.) + with ops.control_dependencies([update]): + x.assign_add(1.) + + func() + self.assertAllClose(2., self.evaluate(x)) + @combinations.generate(maybe_distribute) def test_assign_stays_in_true_dtype(self, distribution): with distribution.scope(): From 3c73355a9ad930e14a1471834de4957c5dd94daf Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Fri, 7 Aug 2020 13:45:01 +0200 Subject: [PATCH 0632/1017] Add test to verify assign().op doesn't throw --- .../experimental/autocast_variable_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py index 7b4d821aaff..92c98bd96da 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable_test.py @@ -361,6 +361,19 @@ class AutoCastVariableTest(test.TestCase, parameterized.TestCase): dtypes.float16): self.assertAllClose(5., self.evaluate(run_assign())) + @combinations.generate(maybe_distribute) + def test_assign_op(self, distribution): + with distribution.scope(): + x = get_var(0., dtypes.float32) + x = autocast_variable.create_autocast_variable(x) + + @def_function.function + def func(): + self.assertIsNotNone(x.assign(1.0).op) + self.assertIsNotNone(x.assign_add(1.0).op) + self.assertIsNotNone(x.assign_sub(1.0).op) + func() + @combinations.generate(maybe_distribute) def test_tf_function_control_dependencies(self, distribution): if not context.executing_eagerly(): From 3951b3666fd69d7112f125d0ca62e1dc6d61a3c5 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Fri, 7 Aug 2020 13:53:16 +0200 Subject: [PATCH 0633/1017] Set self._op when before returning AutoCastVariable after assign --- .../experimental/autocast_variable.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py index d44ff5f53e5..48ed905e67e 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py @@ -69,6 +69,7 @@ class AutoCastVariable(variables.Variable, core.Tensor): raise ValueError('variable must be a floating point variable but has ' 'type: %s' % variable.dtype.name) self._variable = variable + self._op = None def _should_cast(self): """Returns True if this variable should be casted when accessed.""" @@ -196,7 +197,10 @@ class AutoCastVariable(variables.Variable, core.Tensor): read_value=True): if ops.executing_eagerly_outside_functions(): assign_op = update_fn(value, use_locking, name, False) - return self if read_value else assign_op + if read_value: + self._op = assign_op + return self + return assign_op # Fallback to wrapping the returned variable in graph mode if possible assign_var = update_fn(value, use_locking, name, read_value) @@ -291,8 +295,16 @@ class AutoCastVariable(variables.Variable, core.Tensor): @property def op(self): + if self._op is not None: + return self._op return self._variable.op + def _as_graph_element(self): + graph_element = self._variable._as_graph_element() # pylint:disable=protected-access + if graph_element is None: + return self._op + return graph_element + @property def graph(self): return self._variable.graph From f068418748981d4e3a4a01c1a98b888eb4d76a67 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Fri, 7 Aug 2020 20:35:05 +0700 Subject: [PATCH 0634/1017] Gcs Filesystem refactor part 1 --- .../filesystem/plugins/gcs/gcs_filesystem.cc | 192 ++++++++---------- 1 file changed, 87 insertions(+), 105 deletions(-) diff --git a/tensorflow/c/experimental/filesystem/plugins/gcs/gcs_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/gcs/gcs_filesystem.cc index 039bc4fd236..d170e51e3b1 100644 --- a/tensorflow/c/experimental/filesystem/plugins/gcs/gcs_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/gcs/gcs_filesystem.cc @@ -776,28 +776,65 @@ static bool FolderExists(GCSFile* gcs_file, std::string dir, return true; } -void CreateDir(const TF_Filesystem* filesystem, const char* path, - TF_Status* status) { +static void ClearFileCaches(GCSFile* gcs_file, const std::string& path) { + absl::ReaderMutexLock l(&gcs_file->block_cache_lock); + gcs_file->file_block_cache->RemoveFile(path); + gcs_file->stat_cache->Delete(path); +} + +void PathExists(const TF_Filesystem* filesystem, const char* path, + TF_Status* status) { std::string bucket, object; ParseGCSPath(path, true, &bucket, &object, status); if (TF_GetCode(status) != TF_OK) return; + auto gcs_file = static_cast(filesystem->plugin_filesystem); if (object.empty()) { - auto bucket_metadata = gcs_file->gcs_client.GetBucketMetadata(bucket); - TF_SetStatusFromGCSStatus(bucket_metadata.status(), status); + bool result = BucketExists(gcs_file, bucket, status); + if (result) return TF_SetStatus(status, TF_OK, ""); + } + + GcsFileStat stat; + StatForObject(gcs_file, path, bucket, object, &stat, status); + if (TF_GetCode(status) != TF_NOT_FOUND) return; + + bool result = FolderExists(gcs_file, path, status); + if (TF_GetCode(status) != TF_OK || (TF_GetCode(status) == TF_OK && result)) + return; + return TF_SetStatus( + status, TF_NOT_FOUND, + absl::StrCat("The path ", path, " does not exist.").c_str()); +} + +void CreateDir(const TF_Filesystem* filesystem, const char* path, + TF_Status* status) { + std::string dir = path; + MaybeAppendSlash(&dir); + std::string bucket, object; + ParseGCSPath(dir, true, &bucket, &object, status); + if (TF_GetCode(status) != TF_OK) return; + auto gcs_file = static_cast(filesystem->plugin_filesystem); + if (object.empty()) { + bool is_directory = BucketExists(gcs_file, bucket, status); + if (TF_GetCode(status) != TF_OK) return; + if (!is_directory) + TF_SetStatus(status, TF_NOT_FOUND, + ("The specified bucket " + dir + " was not found.").c_str()); return; } - MaybeAppendSlash(&object); - auto object_metadata = gcs_file->gcs_client.GetObjectMetadata(bucket, object); - TF_SetStatusFromGCSStatus(object_metadata.status(), status); - if (TF_GetCode(status) == TF_NOT_FOUND) { - auto insert_metadata = - gcs_file->gcs_client.InsertObject(bucket, object, ""); - TF_SetStatusFromGCSStatus(insert_metadata.status(), status); - } else if (TF_GetCode(status) == TF_OK) { + PathExists(filesystem, dir.c_str(), status); + if (TF_GetCode(status) == TF_OK) + return TF_SetStatus(status, TF_ALREADY_EXISTS, path); + + auto metadata = gcs_file->gcs_client.InsertObject( + bucket, object, "", + // Adding this parameter means HTTP_CODE_PRECONDITION_FAILED + // will be returned if the object already exists, so avoid reuploading. + gcs::IfGenerationMatch(0)); + TF_SetStatusFromGCSStatus(metadata.status(), status); + if (TF_GetCode(status) == TF_FAILED_PRECONDITION) TF_SetStatus(status, TF_ALREADY_EXISTS, path); - } } // TODO(vnvo2409): `RecursivelyCreateDir` should use `CreateDir` instead of the @@ -813,53 +850,31 @@ void DeleteFile(const TF_Filesystem* filesystem, const char* path, auto gcs_file = static_cast(filesystem->plugin_filesystem); auto gcs_status = gcs_file->gcs_client.DeleteObject(bucket, object); TF_SetStatusFromGCSStatus(gcs_status, status); + if (TF_GetCode(status) == TF_OK) ClearFileCaches(gcs_file, path); } +// Checks that the directory is empty (i.e no objects with this prefix exist). +// Deletes the GCS directory marker if it exists. void DeleteDir(const TF_Filesystem* filesystem, const char* path, TF_Status* status) { - std::string bucket, object; - ParseGCSPath(path, false, &bucket, &object, status); - if (TF_GetCode(status) != TF_OK) return; - MaybeAppendSlash(&object); + // A directory is considered empty either if there are no matching objects + // with the corresponding name prefix or if there is exactly one matching + // object and it is the directory marker. Therefore we need to retrieve + // at most two children for the prefix to detect if a directory is empty. auto gcs_file = static_cast(filesystem->plugin_filesystem); - int object_count = 0; - for (auto&& metadata : - gcs_file->gcs_client.ListObjects(bucket, gcs::Prefix(object))) { - if (!metadata) { - TF_SetStatusFromGCSStatus(metadata.status(), status); - return; - } - ++object_count; - // We consider a path is a non-empty directory in two cases: - // - There are more than two objects whose keys start with the name of this - // directory. - // - There is one object whose key contains the name of this directory ( but - // not equal ). - if (object_count > 1 || metadata->name() != object) { - TF_SetStatus(status, TF_FAILED_PRECONDITION, - "Cannot delete a non-empty directory."); - return; - } + auto childrens = GetChildrenBounded(gcs_file, path, 2, true, true, status); + if (TF_GetCode(status) != TF_OK) return; + if (childrens.size() > 1 || (childrens.size() == 1 && !childrens[0].empty())) + return TF_SetStatus(status, TF_FAILED_PRECONDITION, + "Cannot delete a non-empty directory."); + if (childrens.size() == 1 && childrens[0].empty()) { + // This is the directory marker object. Delete it. + std::string dir = path; + MaybeAppendSlash(&dir); + DeleteFile(filesystem, dir.c_str(), status); + return; } - auto gcs_status = gcs_file->gcs_client.DeleteObject(bucket, object); - TF_SetStatusFromGCSStatus(gcs_status, status); -} - -// TODO(vnvo2409): `DeleteRecursively` needs `GetChildrens` but there will be -// some differents compared to the default implementation. Will be refactored. -static void DeleteRecursively(const TF_Filesystem* filesystem, const char* path, - uint64_t* undeleted_files, - uint64_t* undeleted_dirs, TF_Status* status) { - std::string bucket, object; - ParseGCSPath(path, false, &bucket, &object, status); - if (TF_GetCode(status) != TF_OK) return; - - auto gcs_file = static_cast(filesystem->plugin_filesystem); - auto gcs_status = gcs::DeleteByPrefix(gcs_file->gcs_client, bucket, object); - TF_SetStatusFromGCSStatus(gcs_status, status); - if (TF_GetCode(status) != TF_OK) return; - *undeleted_dirs = 0; - *undeleted_files = 0; + TF_SetStatus(status, TF_OK, ""); } // TODO(vnvo2409): `RewriteObjectBlocking` will set `status` to `TF_NOT_FOUND` @@ -904,31 +919,6 @@ void CopyFile(const TF_Filesystem* filesystem, const char* src, const char* dst, TF_SetStatusFromGCSStatus(metadata.status(), status); } -// TODO(vnvo2409): This approach can cause a problem when our path is -// `path/to/dir` and there is an object with key `path/to/directory`. Will be -// fixed when refactoring. -void PathExists(const TF_Filesystem* filesystem, const char* path, - TF_Status* status) { - std::string bucket, object; - ParseGCSPath(path, true, &bucket, &object, status); - if (TF_GetCode(status) != TF_OK) return; - - auto gcs_file = static_cast(filesystem->plugin_filesystem); - for (auto&& metadata : - gcs_file->gcs_client.ListObjects(bucket, gcs::Prefix(object))) { - if (!metadata) { - TF_SetStatusFromGCSStatus(metadata.status(), status); - return; - } - // We consider a path exists if there is at least one object whose key - // contains the path. - return TF_SetStatus(status, TF_OK, ""); - } - return TF_SetStatus( - status, TF_NOT_FOUND, - absl::StrCat("The path ", path, " does not exist.").c_str()); -} - bool IsDirectory(const TF_Filesystem* filesystem, const char* path, TF_Status* status) { std::string bucket, object; @@ -937,35 +927,27 @@ bool IsDirectory(const TF_Filesystem* filesystem, const char* path, auto gcs_file = static_cast(filesystem->plugin_filesystem); if (object.empty()) { - auto bucket_metadata = gcs_file->gcs_client.GetBucketMetadata(bucket); - TF_SetStatusFromGCSStatus(bucket_metadata.status(), status); - if (TF_GetCode(status) == TF_OK) - return true; - else - return false; + bool result = BucketExists(gcs_file, bucket, status); + if (TF_GetCode(status) != TF_OK) return false; + if (!result) + TF_SetStatus( + status, TF_NOT_FOUND, + ("The specified bucket gs://" + bucket + " was not found.").c_str()); + return result; } - // We check if there is an object with this key on the GCS server. - auto metadata = gcs_file->gcs_client.GetObjectMetadata(bucket, object); - if (metadata) { - TF_SetStatus(status, TF_OK, ""); - if (metadata->name().back() == '/') - return true; - else - return false; - } + bool is_folder = FolderExists(gcs_file, path, status); + if (TF_GetCode(status) != TF_OK) return false; + if (is_folder) return true; - // If there is no object with this key on the GCS server. We check if there is - // any object whose key contains that path. - MaybeAppendSlash(&object); - for (auto&& metadata : - gcs_file->gcs_client.ListObjects(bucket, gcs::Prefix(object))) { - if (!metadata) { - TF_SetStatusFromGCSStatus(metadata.status(), status); - return false; - } - TF_SetStatus(status, TF_OK, ""); - return true; + bool is_object = ObjectExists(gcs_file, path, bucket, object, status); + if (TF_GetCode(status) != TF_OK) return false; + if (is_object) { + TF_SetStatus( + status, TF_FAILED_PRECONDITION, + absl::StrCat("The specified path ", path, " is not a directory.") + .c_str()); + return false; } TF_SetStatus(status, TF_NOT_FOUND, absl::StrCat("The path ", path, " does not exist.").c_str()); From 0996281285b5d905062f56a80f996cacad9dfc79 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 07:53:36 -0700 Subject: [PATCH 0635/1017] Support 'exclusive' and 'reverse', in tf.CumSum legalization. PiperOrigin-RevId: 325434770 Change-Id: I48914d39f347ed68d7c730d220ea54c4c3f857fe --- .../compiler/mlir/xla/tests/legalize-tf.mlir | 52 +++++++++++++++++- .../mlir/xla/transforms/legalize_tf.cc | 55 ++++++++++++++----- 2 files changed, 91 insertions(+), 16 deletions(-) diff --git a/tensorflow/compiler/mlir/xla/tests/legalize-tf.mlir b/tensorflow/compiler/mlir/xla/tests/legalize-tf.mlir index 3b4efc388eb..bad9c1ef279 100644 --- a/tensorflow/compiler/mlir/xla/tests/legalize-tf.mlir +++ b/tensorflow/compiler/mlir/xla/tests/legalize-tf.mlir @@ -4581,21 +4581,65 @@ func @cumsum_static(%arg0: tensor<4xf32>) -> tensor<4xf32> { } // CHECK-LABEL: func @cumsum_exclusive +// CHECK-SAME: [[X:%.*]]: tensor<4xf32> func @cumsum_exclusive(%arg0: tensor<4xf32>) -> tensor<4xf32> { - // CHECK: "tf.Cumsum" + // CHECK: [[AXIS:%.*]] = mhlo.constant dense<0> : tensor + // CHECK: [[CONVERT_X:%.*]] = "mhlo.convert"([[X]]) : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: [[INIT:%.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK: [[REDUCE:%.*]] = "mhlo.reduce_window"([[CONVERT_X]], [[INIT]]) ( { + // CHECK: ^bb0([[A:%.*]]: tensor, [[B:%.*]]: tensor): + // CHECK: [[SUM:%.*]] = mhlo.add [[A]], [[B]] : tensor + // CHECK: "mhlo.return"([[SUM]]) : (tensor) -> () + // CHECK: }) {padding = dense<{{\[\[}}3, 0]]> : tensor<1x2xi64>, window_dimensions = dense<4> : tensor<1xi64>, window_strides = dense<1> : tensor<1xi64>} : (tensor<4xf32>, tensor) -> tensor<4xf32> + // CHECK: [[PAD:%.*]] = "mhlo.pad"([[REDUCE]], %{{.*}}) {edge_padding_high = dense<-1> : tensor<1xi64>, edge_padding_low = dense<1> : tensor<1xi64>, interior_padding = dense<0> : tensor<1xi64>} : (tensor<4xf32>, tensor) -> tensor<4xf32> + // CHECK: [[CONVERT_REDUCE:%.*]] = "mhlo.convert"([[PAD]]) : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: return [[CONVERT_REDUCE]] %0 = "tf.Const"() {_output_shapes = ["tfshape$"], device = "", dtype = i32, value = dense<0> : tensor} : () -> tensor %1 = "tf.Cumsum"(%arg0, %0) {exclusive = true, reverse = false} : (tensor<4xf32>, tensor) -> tensor<4xf32> return %1 : tensor<4xf32> } // CHECK-LABEL: func @cumsum_reverse +// CHECK-SAME: [[X:%.*]]: tensor<4xf32> func @cumsum_reverse(%arg0: tensor<4xf32>) -> tensor<4xf32> { - // CHECK: "tf.Cumsum" + // CHECK: [[AXIS:%.*]] = mhlo.constant dense<0> : tensor + // CHECK: [[REVERSE1:%.*]] = "mhlo.reverse"([[X]]) {dimensions = dense<0> : tensor<1xi64>} : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: [[CONVERT_X:%.*]] = "mhlo.convert"([[REVERSE1]]) : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: [[INIT:%.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK: [[REDUCE:%.*]] = "mhlo.reduce_window"([[CONVERT_X]], [[INIT]]) ( { + // CHECK: ^bb0([[A:%.*]]: tensor, [[B:%.*]]: tensor): + // CHECK: [[SUM:%.*]] = mhlo.add [[A]], [[B]] : tensor + // CHECK: "mhlo.return"([[SUM]]) : (tensor) -> () + // CHECK: }) {padding = dense<{{\[\[}}3, 0]]> : tensor<1x2xi64>, window_dimensions = dense<4> : tensor<1xi64>, window_strides = dense<1> : tensor<1xi64>} : (tensor<4xf32>, tensor) -> tensor<4xf32> + // CHECK: [[CONVERT_REDUCE:%.*]] = "mhlo.convert"([[REDUCE]]) : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: [[REVERSE_BACK:%.*]] = "mhlo.reverse"([[CONVERT_REDUCE]]) {dimensions = dense<0> : tensor<1xi64>} : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: return [[REVERSE_BACK]] %0 = "tf.Const"() {_output_shapes = ["tfshape$"], device = "", dtype = i32, value = dense<0> : tensor} : () -> tensor %1 = "tf.Cumsum"(%arg0, %0) {exclusive = false, reverse = true} : (tensor<4xf32>, tensor) -> tensor<4xf32> return %1 : tensor<4xf32> } +// CHECK-LABEL: func @cumsum_exclusive_reverse +// CHECK-SAME: [[X:%.*]]: tensor<4xf32> +func @cumsum_exclusive_reverse(%arg0: tensor<4xf32>) -> tensor<4xf32> { + // CHECK: [[AXIS:%.*]] = mhlo.constant dense<0> : tensor + // CHECK: [[REVERSE1:%.*]] = "mhlo.reverse"([[X]]) {dimensions = dense<0> : tensor<1xi64>} : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: [[CONVERT_X:%.*]] = "mhlo.convert"([[REVERSE1]]) : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: [[INIT:%.*]] = mhlo.constant dense<0.000000e+00> : tensor + // CHECK: [[REDUCE:%.*]] = "mhlo.reduce_window"([[CONVERT_X]], [[INIT]]) ( { + // CHECK: ^bb0([[A:%.*]]: tensor, [[B:%.*]]: tensor): + // CHECK: [[SUM:%.*]] = mhlo.add [[A]], [[B]] : tensor + // CHECK: "mhlo.return"([[SUM]]) : (tensor) -> () + // CHECK: }) {padding = dense<{{\[\[}}3, 0]]> : tensor<1x2xi64>, window_dimensions = dense<4> : tensor<1xi64>, window_strides = dense<1> : tensor<1xi64>} : (tensor<4xf32>, tensor) -> tensor<4xf32> + // CHECK: [[PAD:%.*]] = "mhlo.pad"([[REDUCE]], %{{.*}}) {edge_padding_high = dense<-1> : tensor<1xi64>, edge_padding_low = dense<1> : tensor<1xi64>, interior_padding = dense<0> : tensor<1xi64>} : (tensor<4xf32>, tensor) -> tensor<4xf32> + // CHECK: [[CONVERT_REDUCE:%.*]] = "mhlo.convert"([[PAD]]) : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: [[REVERSE_BACK:%.*]] = "mhlo.reverse"([[CONVERT_REDUCE]]) {dimensions = dense<0> : tensor<1xi64>} : (tensor<4xf32>) -> tensor<4xf32> + // CHECK: return [[REVERSE_BACK]] + %0 = "tf.Const"() {_output_shapes = ["tfshape$"], device = "", dtype = i32, value = dense<0> : tensor} : () -> tensor + %1 = "tf.Cumsum"(%arg0, %0) {exclusive = true, reverse = true} : (tensor<4xf32>, tensor) -> tensor<4xf32> + return %1 : tensor<4xf32> +} + // CHECK-LABEL: func @cumsum_dynamic func @cumsum_dynamic(%arg0: tensor, %arg1: tensor) -> tensor { // CHECK: "tf.Cumsum" @@ -4603,6 +4647,10 @@ func @cumsum_dynamic(%arg0: tensor, %arg1: tensor) -> tensor return %0 : tensor } +//===----------------------------------------------------------------------===// +// Qr op legalization +//===----------------------------------------------------------------------===// + // CHECK: func @qr([[VAL_0:%.*]]: tensor<500x100x75xf32>) -> (tensor<500x100x75xf32>, tensor<500x75x75xf32>) func @qr(%arg0: tensor<500x100x75xf32>) -> (tensor<500x100x75xf32>, tensor<500x75x75xf32>) { // The tf.Qr lowering is a full algorithm that is not effective to verify with diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc index 0b420fff785..f2b3822188c 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc @@ -5086,11 +5086,8 @@ class ConvertCumsumOp : public OpRewritePattern { return failure(); } - // TODO(jennik): Add support for the optional 'exclusive' and 'reverse' - // arguments. - if (op.exclusive() || op.reverse()) { - return failure(); - } + ArrayRef input_shape = input_type.getShape(); + int64_t rank = input_shape.size(); // We can only match when the axis is a constant scalar. DenseIntElementsAttr axis_attr; @@ -5098,15 +5095,6 @@ class ConvertCumsumOp : public OpRewritePattern { return failure(); } - // Convert if we need to enlarge the element type's bitwidth to avoid - // precision loss. - Type input_element_type = input_type.getElementType(); - Type sum_element_type = GetSumAccumulationType(input_element_type); - input = rewriter.create(op.getLoc(), input, sum_element_type); - - ArrayRef input_shape = input_type.getShape(); - int64_t rank = input_shape.size(); - // Get the dimension to apply the reduction on, and offset properly if it is // negative. int64_t axis = (*axis_attr.begin()).getSExtValue(); @@ -5114,6 +5102,21 @@ class ConvertCumsumOp : public OpRewritePattern { axis += rank; } + // If we're supposed to sum things up in the reverse direction, we reverse + // the input and then later reverse the output. + if (op.reverse()) { + llvm::SmallVector dims_to_reverse({axis}); + input = rewriter.create( + op.getLoc(), op.getType(), input, + GetI64ElementsAttr(dims_to_reverse, &rewriter)); + } + + // Convert if we need to enlarge the element type's bitwidth to avoid + // precision loss. + Type input_element_type = input_type.getElementType(); + Type sum_element_type = GetSumAccumulationType(input_element_type); + input = rewriter.create(op.getLoc(), input, sum_element_type); + SmallVector window_dims(rank, 1); SmallVector window_strides(rank, 1); window_dims[axis] = input_shape[axis]; @@ -5136,10 +5139,34 @@ class ConvertCumsumOp : public OpRewritePattern { BuildReduceBody(sum_element_type, &reduce.body(), &rewriter); Value result = reduce.getResult(); + if (op.exclusive()) { + // In "exclusive" operation, the output will start with the "init" (0) + // values. There is no way to express that as a ReduceWindowOp, so run the + // normal operation, and then use a PadOp to add the 0 "column" on the + // left and cut away the last column on the right. + llvm::SmallVector low_padding(rank, 0); + llvm::SmallVector high_padding(rank, 0); + llvm::SmallVector interior_padding(rank, 0); + low_padding[axis] = 1; + high_padding[axis] = -1; + result = rewriter.create( + op.getLoc(), op.getType(), result, init, + GetI64ElementsAttr(low_padding, &rewriter), + GetI64ElementsAttr(high_padding, &rewriter), + GetI64ElementsAttr(interior_padding, &rewriter)); + } + // Convert back if we enlarged the element type's bitwidth. result = rewriter.create(op.getLoc(), result, input_element_type); + if (op.reverse()) { + llvm::SmallVector dims_to_reverse({axis}); + result = rewriter.create( + op.getLoc(), op.getType(), result, + GetI64ElementsAttr(dims_to_reverse, &rewriter)); + } + rewriter.replaceOp(op, result); return success(); } From fb837585264e6abe4b0488e3a9dd5c5507e69bf6 Mon Sep 17 00:00:00 2001 From: Rick Chao Date: Fri, 7 Aug 2020 08:40:16 -0700 Subject: [PATCH 0636/1017] PSv2: Fix asan test for various targets. PiperOrigin-RevId: 325441069 Change-Id: I1fa1b2b10670f34739323292eab623d5b538142e --- tensorflow/python/distribute/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/distribute/__init__.py b/tensorflow/python/distribute/__init__.py index acb3c112226..f9d0a95ea58 100644 --- a/tensorflow/python/distribute/__init__.py +++ b/tensorflow/python/distribute/__init__.py @@ -25,7 +25,6 @@ from tensorflow.python.distribute import distribute_lib from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import mirrored_strategy from tensorflow.python.distribute import one_device_strategy -from tensorflow.python.distribute.client import parameter_server_client from tensorflow.python.distribute.experimental import collective_all_reduce_strategy from tensorflow.python.distribute.experimental import parameter_server_strategy # pylint: enable=unused-import From f63877d6371aa5e47391e520a29a6aa1ef2e0199 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 09:23:36 -0700 Subject: [PATCH 0637/1017] Update ops-related pbtxt files. PiperOrigin-RevId: 325447902 Change-Id: Ie19ab04da2b7d678efc2b87c52e15a88f0150c8c --- .../ops_history_v2/EmptyTensorMap.pbtxt | 7 ++ .../ops_history_v2/TensorMapErase.pbtxt | 27 +++++ .../ops_history_v2/TensorMapHasKey.pbtxt | 19 +++ .../ops_history_v2/TensorMapInsert.pbtxt | 27 +++++ .../ops_history_v2/TensorMapLookup.pbtxt | 23 ++++ .../compat/ops_history_v2/TensorMapSize.pbtxt | 11 ++ tensorflow/core/ops/ops.pbtxt | 114 ++++++++++++++++++ 7 files changed, 228 insertions(+) create mode 100644 tensorflow/core/ops/compat/ops_history_v2/EmptyTensorMap.pbtxt create mode 100644 tensorflow/core/ops/compat/ops_history_v2/TensorMapErase.pbtxt create mode 100644 tensorflow/core/ops/compat/ops_history_v2/TensorMapHasKey.pbtxt create mode 100644 tensorflow/core/ops/compat/ops_history_v2/TensorMapInsert.pbtxt create mode 100644 tensorflow/core/ops/compat/ops_history_v2/TensorMapLookup.pbtxt create mode 100644 tensorflow/core/ops/compat/ops_history_v2/TensorMapSize.pbtxt diff --git a/tensorflow/core/ops/compat/ops_history_v2/EmptyTensorMap.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/EmptyTensorMap.pbtxt new file mode 100644 index 00000000000..25327b4e1e8 --- /dev/null +++ b/tensorflow/core/ops/compat/ops_history_v2/EmptyTensorMap.pbtxt @@ -0,0 +1,7 @@ +op { + name: "EmptyTensorMap" + output_arg { + name: "handle" + type: DT_VARIANT + } +} diff --git a/tensorflow/core/ops/compat/ops_history_v2/TensorMapErase.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/TensorMapErase.pbtxt new file mode 100644 index 00000000000..8b6c16005b5 --- /dev/null +++ b/tensorflow/core/ops/compat/ops_history_v2/TensorMapErase.pbtxt @@ -0,0 +1,27 @@ +op { + name: "TensorMapErase" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + input_arg { + name: "key" + type_attr: "key_dtype" + } + output_arg { + name: "output_handle" + type: DT_VARIANT + } + output_arg { + name: "value" + type_attr: "value_dtype" + } + attr { + name: "key_dtype" + type: "type" + } + attr { + name: "value_dtype" + type: "type" + } +} diff --git a/tensorflow/core/ops/compat/ops_history_v2/TensorMapHasKey.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/TensorMapHasKey.pbtxt new file mode 100644 index 00000000000..437822797af --- /dev/null +++ b/tensorflow/core/ops/compat/ops_history_v2/TensorMapHasKey.pbtxt @@ -0,0 +1,19 @@ +op { + name: "TensorMapHasKey" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + input_arg { + name: "key" + type_attr: "element_dtype" + } + output_arg { + name: "has_key" + type: DT_BOOL + } + attr { + name: "element_dtype" + type: "type" + } +} diff --git a/tensorflow/core/ops/compat/ops_history_v2/TensorMapInsert.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/TensorMapInsert.pbtxt new file mode 100644 index 00000000000..10061ea1cde --- /dev/null +++ b/tensorflow/core/ops/compat/ops_history_v2/TensorMapInsert.pbtxt @@ -0,0 +1,27 @@ +op { + name: "TensorMapInsert" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + input_arg { + name: "key" + type_attr: "key_dtype" + } + input_arg { + name: "value" + type_attr: "value_dtype" + } + output_arg { + name: "output_handle" + type: DT_VARIANT + } + attr { + name: "key_dtype" + type: "type" + } + attr { + name: "value_dtype" + type: "type" + } +} diff --git a/tensorflow/core/ops/compat/ops_history_v2/TensorMapLookup.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/TensorMapLookup.pbtxt new file mode 100644 index 00000000000..b48fda8ac46 --- /dev/null +++ b/tensorflow/core/ops/compat/ops_history_v2/TensorMapLookup.pbtxt @@ -0,0 +1,23 @@ +op { + name: "TensorMapLookup" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + input_arg { + name: "key" + type_attr: "key_dtype" + } + output_arg { + name: "value" + type_attr: "value_dtype" + } + attr { + name: "key_dtype" + type: "type" + } + attr { + name: "value_dtype" + type: "type" + } +} diff --git a/tensorflow/core/ops/compat/ops_history_v2/TensorMapSize.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/TensorMapSize.pbtxt new file mode 100644 index 00000000000..dd8ade84414 --- /dev/null +++ b/tensorflow/core/ops/compat/ops_history_v2/TensorMapSize.pbtxt @@ -0,0 +1,11 @@ +op { + name: "TensorMapSize" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + output_arg { + name: "size" + type: DT_INT32 + } +} diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 931208fdb4a..7e138923a8d 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -13481,6 +13481,13 @@ op { } } } +op { + name: "EmptyTensorMap" + output_arg { + name: "handle" + type: DT_VARIANT + } +} op { name: "EncodeBase64" input_arg { @@ -53010,6 +53017,113 @@ op { } } } +op { + name: "TensorMapErase" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + input_arg { + name: "key" + type_attr: "key_dtype" + } + output_arg { + name: "output_handle" + type: DT_VARIANT + } + output_arg { + name: "value" + type_attr: "value_dtype" + } + attr { + name: "key_dtype" + type: "type" + } + attr { + name: "value_dtype" + type: "type" + } +} +op { + name: "TensorMapHasKey" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + input_arg { + name: "key" + type_attr: "element_dtype" + } + output_arg { + name: "has_key" + type: DT_BOOL + } + attr { + name: "element_dtype" + type: "type" + } +} +op { + name: "TensorMapInsert" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + input_arg { + name: "key" + type_attr: "key_dtype" + } + input_arg { + name: "value" + type_attr: "value_dtype" + } + output_arg { + name: "output_handle" + type: DT_VARIANT + } + attr { + name: "key_dtype" + type: "type" + } + attr { + name: "value_dtype" + type: "type" + } +} +op { + name: "TensorMapLookup" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + input_arg { + name: "key" + type_attr: "key_dtype" + } + output_arg { + name: "value" + type_attr: "value_dtype" + } + attr { + name: "key_dtype" + type: "type" + } + attr { + name: "value_dtype" + type: "type" + } +} +op { + name: "TensorMapSize" + input_arg { + name: "input_handle" + type: DT_VARIANT + } + output_arg { + name: "size" + type: DT_INT32 + } +} op { name: "TensorScatterAdd" input_arg { From c27f93d4b14c47f7bde086a0afb1ff657fe37031 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 09:45:54 -0700 Subject: [PATCH 0638/1017] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 325451755 Change-Id: Ic8ff17cf02922c9b68143e71ba2980d3205f278e --- tensorflow/go/op/wrappers.go | 206 +++++++++++++++++++++++++++-------- 1 file changed, 160 insertions(+), 46 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 34ff57636ca..cd6284aab05 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -13369,6 +13369,24 @@ func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...Resi return op.Output(0) } +// Returns the number of work units this Reader has finished processing. +// +// Arguments: +// reader_handle: Handle to a Reader. +func ReaderNumWorkUnitsCompletedV2(scope *Scope, reader_handle tf.Output) (units_completed tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "ReaderNumWorkUnitsCompletedV2", + Input: []tf.Input{ + reader_handle, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Returns up to `num_records` (key, value) pairs produced by a Reader. // // Will dequeue from the input queue if necessary (e.g. when the @@ -15101,6 +15119,94 @@ func TensorListLength(scope *Scope, input_handle tf.Output) (length tf.Output) { return op.Output(0) } +// Returns whether the given key exists in the map. +// +// input_handle: the input map +// key: the key to check +// has_key: whether the key is already in the map or not +func TensorMapHasKey(scope *Scope, input_handle tf.Output, key tf.Output) (has_key tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "TensorMapHasKey", + Input: []tf.Input{ + input_handle, key, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Returns the value from a given key in a tensor map. +// +// input_handle: the input map +// key: the key to be looked up +// value: the value found from the given key +func TensorMapLookup(scope *Scope, input_handle tf.Output, key tf.Output, value_dtype tf.DataType) (value tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"value_dtype": value_dtype} + opspec := tf.OpSpec{ + Type: "TensorMapLookup", + Input: []tf.Input{ + input_handle, key, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Inverse 3D fast Fourier transform. +// +// Computes the inverse 3-dimensional discrete Fourier transform over the +// inner-most 3 dimensions of `input`. +// +// Arguments: +// input: A complex tensor. +// +// Returns A complex tensor of the same shape as `input`. The inner-most 3 +// dimensions of `input` are replaced with their inverse 3D Fourier transform. +// +// @compatibility(numpy) +// Equivalent to np.fft.ifftn with 3 dimensions. +// @end_compatibility +func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "IFFT3D", + Input: []tf.Input{ + input, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + +// Returns a map that is the 'input_handle' with the given key-value pair inserted. +// +// input_handle: the original map +// output_handle: the map with key and value inserted +// key: the key to be inserted +// value: the value to be inserted +func TensorMapInsert(scope *Scope, input_handle tf.Output, key tf.Output, value tf.Output) (output_handle tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "TensorMapInsert", + Input: []tf.Input{ + input_handle, key, value, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Merges summaries. // // This op creates a @@ -20036,6 +20142,28 @@ func Polygamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) { return op.Output(0) } +// Returns a tensor map with item from given key erased. +// +// input_handle: the original map +// output_handle: the map with value from given key removed +// key: the key of the value to be erased +// value: the value that was erased +func TensorMapErase(scope *Scope, input_handle tf.Output, key tf.Output, value_dtype tf.DataType) (output_handle tf.Output, value tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{"value_dtype": value_dtype} + opspec := tf.OpSpec{ + Type: "TensorMapErase", + Input: []tf.Input{ + input_handle, key, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1) +} + // Shuffle dimensions of x according to a permutation. // // The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy: @@ -24997,6 +25125,24 @@ func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []i return op.Output(0), op.Output(1) } +// Returns the number of tensors in the input tensor map. +// +// input_handle: the input map +// size: the number of tensors in the map +func TensorMapSize(scope *Scope, input_handle tf.Output) (size tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "TensorMapSize", + Input: []tf.Input{ + input_handle, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // MaxPoolGradGradAttr is an optional argument to MaxPoolGradGrad. type MaxPoolGradGradAttr func(optionalAttr) @@ -35440,34 +35586,6 @@ func DecodeRaw(scope *Scope, bytes tf.Output, out_type tf.DataType, optional ... return op.Output(0) } -// Inverse 3D fast Fourier transform. -// -// Computes the inverse 3-dimensional discrete Fourier transform over the -// inner-most 3 dimensions of `input`. -// -// Arguments: -// input: A complex tensor. -// -// Returns A complex tensor of the same shape as `input`. The inner-most 3 -// dimensions of `input` are replaced with their inverse 3D Fourier transform. -// -// @compatibility(numpy) -// Equivalent to np.fft.ifftn with 3 dimensions. -// @end_compatibility -func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "IFFT3D", - Input: []tf.Input{ - input, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // QueueDequeueUpToV2Attr is an optional argument to QueueDequeueUpToV2. type QueueDequeueUpToV2Attr func(optionalAttr) @@ -39085,24 +39203,6 @@ func ResourceApplyAddSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Outpu return scope.AddOperation(opspec) } -// Returns the number of work units this Reader has finished processing. -// -// Arguments: -// reader_handle: Handle to a Reader. -func ReaderNumWorkUnitsCompletedV2(scope *Scope, reader_handle tf.Output) (units_completed tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "ReaderNumWorkUnitsCompletedV2", - Input: []tf.Input{ - reader_handle, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // FractionalMaxPoolAttr is an optional argument to FractionalMaxPool. type FractionalMaxPoolAttr func(optionalAttr) @@ -39425,6 +39525,20 @@ func RemoteFusedGraphExecute(scope *Scope, inputs []tf.Output, Toutputs []tf.Dat return outputs } +// Creates and returns an empty tensor map. +// +// handle: an empty tensor map +func EmptyTensorMap(scope *Scope) (handle tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "EmptyTensorMap", + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // DatasetToGraphAttr is an optional argument to DatasetToGraph. type DatasetToGraphAttr func(optionalAttr) From 17cdd71cc9fd30a7a38c08dc46c4f821c0685ea2 Mon Sep 17 00:00:00 2001 From: Edward Loper Date: Fri, 7 Aug 2020 09:58:24 -0700 Subject: [PATCH 0639/1017] Add c++ utility function that converts AttrDef values to the expected types. This will replace methods such as `make_bool` (in `eager/execute.py`) and `_MakeBool` (in `op_def_library.py`). PiperOrigin-RevId: 325453895 Change-Id: I3a454a0365b08c545944ef7528c357b5d8bf2c02 --- tensorflow/python/BUILD | 49 ++++ tensorflow/python/framework/op_def_library.py | 12 + tensorflow/python/framework/op_def_util.cc | 270 ++++++++++++++++++ tensorflow/python/framework/op_def_util.h | 77 +++++ .../python/framework/op_def_util_pybind.cc | 41 +++ .../python/framework/op_def_util_test.py | 97 +++++++ 6 files changed, 546 insertions(+) create mode 100644 tensorflow/python/framework/op_def_util.cc create mode 100644 tensorflow/python/framework/op_def_util.h create mode 100644 tensorflow/python/framework/op_def_util_pybind.cc create mode 100644 tensorflow/python/framework/op_def_util_test.py diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 5166d5b891d..039fc945eca 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1617,11 +1617,60 @@ py_library( ], ) +cc_library( + name = "op_def_util_cc", + srcs = ["framework/op_def_util.cc"], + hdrs = ["framework/op_def_util.h"], + deps = [ + ":cpp_python_util", + ":safe_ptr", + "//tensorflow/core:protos_all_cc", + "@com_google_absl//absl/strings", + ], +) + +# Note: this target is only used for op_def_util_test. It includes op_def_util.cc +# directly in its srcs (rather than depending on the `op_def_util_cc` target) because +# depending on that target adds dependencies that register objects; and since the +# extension is built as a shared object in some kokoro tests, this causes those objects +# to get registered multiple times (which fails). +tf_python_pybind_extension( + name = "_op_def_util", + srcs = [ + "framework/op_def_util.cc", + "framework/op_def_util_pybind.cc", + ], + hdrs = [ + "framework/op_def_util.h", + "lib/core/safe_ptr.h", + "util/util.h", + "//tensorflow/c:headers", + "//tensorflow/c/eager:headers", + ], + module_name = "_op_def_util", + deps = [ + ":pybind11_status", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/platform:status", + "//third_party/python_runtime:headers", + "@com_google_absl//absl/strings", + "@pybind11", + ], +) + +tf_py_test( + name = "op_def_util_test", + srcs = ["framework/op_def_util_test.py"], + python_version = "PY3", + tags = ["no_pip"], +) + py_library( name = "framework_ops", # "ops" is already the name of a deprecated target srcs = ["framework/ops.py"], srcs_version = "PY2AND3", deps = [ + ":_op_def_util", ":c_api_util", ":control_flow_util", ":device", diff --git a/tensorflow/python/framework/op_def_library.py b/tensorflow/python/framework/op_def_library.py index 17e06b79f74..53d092787f6 100644 --- a/tensorflow/python/framework/op_def_library.py +++ b/tensorflow/python/framework/op_def_library.py @@ -21,10 +21,12 @@ from __future__ import print_function import six +from google.protobuf import text_format from tensorflow.core.framework import attr_value_pb2 from tensorflow.core.framework import tensor_pb2 from tensorflow.core.framework import tensor_shape_pb2 from tensorflow.core.framework import types_pb2 +from tensorflow.python import _pywrap_utils from tensorflow.python.framework import dtypes from tensorflow.python.framework import op_callbacks from tensorflow.python.framework import op_def_registry @@ -788,3 +790,13 @@ def _apply_op_helper(op_type_name, name=None, **keywords): # pylint: disable=in outputs = callback_outputs return output_structure, op_def.is_stateful, op, outputs + + +# The following symbols are used by op_def_util.cc. +_pywrap_utils.RegisterPyObject("tf.dtypes.DType", dtypes.DType) +_pywrap_utils.RegisterPyObject("tf.dtypes.as_dtype", dtypes.as_dtype) +_pywrap_utils.RegisterPyObject("tf.TensorShape", tensor_shape.TensorShape) +_pywrap_utils.RegisterPyObject("tf.as_shape", tensor_shape.as_shape) +_pywrap_utils.RegisterPyObject("tf.TensorProto", tensor_pb2.TensorProto) +_pywrap_utils.RegisterPyObject("text_format.Parse", text_format.Parse) +_pywrap_utils.RegisterPyObject("tf.convert_to_tensor", ops.convert_to_tensor) diff --git a/tensorflow/python/framework/op_def_util.cc b/tensorflow/python/framework/op_def_util.cc new file mode 100644 index 00000000000..4f56c62317c --- /dev/null +++ b/tensorflow/python/framework/op_def_util.cc @@ -0,0 +1,270 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/python/framework/op_def_util.h" + +#include + +#include "absl/strings/str_cat.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/python/util/util.h" + +using ::tensorflow::swig::GetRegisteredPyObject; + +#if PY_MAJOR_VERSION < 3 +#define PY_STRING_CHECK(x) (PyString_Check(x) || PyUnicode_Check(x)) +#define PY_INT_CHECK(x) (PyInt_Check(x)) +#define PY_INT_TYPE PyInt_Type +#else +#define PY_STRING_CHECK(x) (PyBytes_Check(x) || PyUnicode_Check(x)) +#define PY_INT_CHECK(x) (PyLong_Check(x)) +#define PY_INT_TYPE PyLong_Type +#endif + +namespace tensorflow { + +namespace { + +const std::map* AttributeTypeNameMap() { + static auto* type_map = new std::map( + {{"any", AttributeType::ANY}, + {"float", AttributeType::FLOAT}, + {"int", AttributeType::INT}, + {"string", AttributeType::STRING}, + {"bool", AttributeType::BOOL}, + {"shape", AttributeType::SHAPE}, + {"type", AttributeType::DTYPE}, + {"tensor", AttributeType::TENSOR}, + {"list(any)", AttributeType::LIST_ANY}, + {"list(float)", AttributeType::LIST_FLOAT}, + {"list(int)", AttributeType::LIST_INT}, + {"list(string)", AttributeType::LIST_STRING}, + {"list(bool)", AttributeType::LIST_BOOL}, + {"list(type)", AttributeType::LIST_DTYPE}, + {"list(shape)", AttributeType::LIST_SHAPE}, + {"list(tensor)", AttributeType::LIST_TENSOR}}); + return type_map; +} + +// Note: we define functors for converting value types (rather than simple +// functions) so we can define a generic ConvertListAttr method. These +// functors all return a new reference on success, or nullptr on failure. +// They do not (necessarily) call PyErr_SetString. + +struct ConvertAnyFunctor { + Safe_PyObjectPtr operator()(PyObject* value) { + Py_INCREF(value); + return Safe_PyObjectPtr(value); + } +}; + +struct ConvertFloatFunctor { + Safe_PyObjectPtr operator()(PyObject* value) { + Safe_PyObjectPtr result; + if (PyFloat_Check(value)) { + Py_INCREF(value); + result.reset(value); + } else if (!PY_STRING_CHECK(value)) { + result.reset(PyObject_CallFunctionObjArgs( + reinterpret_cast(&PyFloat_Type), value, nullptr)); + } + return result; + } +}; + +struct ConvertIntFunctor { + Safe_PyObjectPtr operator()(PyObject* value) { + Safe_PyObjectPtr result; + if (PY_INT_CHECK(value)) { + Py_INCREF(value); + result.reset(value); + } else if (!PY_STRING_CHECK(value)) { + result.reset(PyObject_CallFunctionObjArgs( + reinterpret_cast(&PY_INT_TYPE), value, nullptr)); + } + return result; + } +}; + +struct ConvertStringFunctor { + Safe_PyObjectPtr operator()(PyObject* value) { + Safe_PyObjectPtr result; + if (PY_STRING_CHECK(value)) { + Py_INCREF(value); + result.reset(value); + } + return result; + } +}; + +// TODO(edloper): Should we allow ints (or any other values) to be converted +// to booleans? Currently, TensorFlow does not do this conversion for attribute +// values in _MakeBool or make_bool. +struct ConvertBoolFunctor { + Safe_PyObjectPtr operator()(PyObject* value) { + Safe_PyObjectPtr result; + if (PyBool_Check(value)) { + Py_INCREF(value); + result.reset(value); + } + return result; + } +}; + +struct ConvertDTypeFunctor { + Safe_PyObjectPtr operator()(PyObject* value) { + Safe_PyObjectPtr result; + // The following symbols are registered in op_def_library.py + static PyObject* dtype = GetRegisteredPyObject("tf.dtypes.DType"); + static PyObject* as_dtype = GetRegisteredPyObject("tf.dtypes.as_dtype"); + if (reinterpret_cast(value->ob_type) == dtype) { + Py_INCREF(value); + result.reset(value); + } else { + result.reset(PyObject_CallFunctionObjArgs(as_dtype, value, nullptr)); + } + return result; + } +}; + +struct ConvertTensorShapeFunctor { + Safe_PyObjectPtr operator()(PyObject* value) { + Safe_PyObjectPtr result; + // The following symbols are registered in op_def_library.py + static PyObject* shape = GetRegisteredPyObject("tf.TensorShape"); + static PyObject* as_shape = GetRegisteredPyObject("tf.as_shape"); + if (reinterpret_cast(value->ob_type) == shape) { + Py_INCREF(value); + result.reset(value); + } else { + result.reset(PyObject_CallFunctionObjArgs(as_shape, value, nullptr)); + } + return result; + } +}; + +struct ConvertTensorProtoFunctor { + Safe_PyObjectPtr operator()(PyObject* value) { + Safe_PyObjectPtr result; + // The following symbols are registered in op_def_library.py + static PyObject* tensor_proto = GetRegisteredPyObject("tf.TensorProto"); + static PyObject* text_format_parse = + GetRegisteredPyObject("text_format.Parse"); + if (reinterpret_cast(value->ob_type) == tensor_proto) { + Py_INCREF(value); + result.reset(value); + } else if (PY_STRING_CHECK(value)) { + result.reset(PyObject_CallObject(tensor_proto, nullptr)); + if (result) { + PyObject_CallFunctionObjArgs(text_format_parse, value, result.get(), + nullptr); + } + } + return result; + } +}; + +// Converts `value` to a list of elements with the same type, using +// `convert_functor` to convert each element. +template +Safe_PyObjectPtr ConvertListAttr(PyObject* value, T convert_functor) { + // Copy the list. + Safe_PyObjectPtr result(PySequence_List(value)); + if (!result) return nullptr; + + // Check the type of each item in the list. + Py_ssize_t len = PySequence_Fast_GET_SIZE(result.get()); + PyObject** items = PySequence_Fast_ITEMS(result.get()); + for (Py_ssize_t i = 0; i < len; ++i) { + if (!PyFloat_Check(value)) { + Safe_PyObjectPtr item = convert_functor(items[i]); + if (!item) return nullptr; + PySequence_SetItem(result.get(), i, item.get()); + } + } + return result; +} + +// Returns the given `value` value, converted to the indicated type. +// Returns nullptr if `value` is not convertible. +Safe_PyObjectPtr ConvertAttrOrNull(PyObject* value, AttributeType attr_type) { + switch (attr_type) { + case AttributeType::ANY: + return ConvertAnyFunctor()(value); + case AttributeType::FLOAT: + return ConvertFloatFunctor()(value); + case AttributeType::INT: + return ConvertIntFunctor()(value); + case AttributeType::STRING: + return ConvertStringFunctor()(value); + case AttributeType::BOOL: + return ConvertBoolFunctor()(value); + case AttributeType::DTYPE: + return ConvertDTypeFunctor()(value); + case AttributeType::SHAPE: + return ConvertTensorShapeFunctor()(value); + case AttributeType::TENSOR: + return ConvertTensorProtoFunctor()(value); + case AttributeType::LIST_ANY: + return ConvertListAttr(value, ConvertAnyFunctor()); + case AttributeType::LIST_FLOAT: + return ConvertListAttr(value, ConvertFloatFunctor()); + case AttributeType::LIST_INT: + return ConvertListAttr(value, ConvertIntFunctor()); + case AttributeType::LIST_STRING: + return ConvertListAttr(value, ConvertStringFunctor()); + case AttributeType::LIST_BOOL: + return ConvertListAttr(value, ConvertBoolFunctor()); + case AttributeType::LIST_DTYPE: + return ConvertListAttr(value, ConvertDTypeFunctor()); + case AttributeType::LIST_SHAPE: + return ConvertListAttr(value, ConvertTensorShapeFunctor()); + case AttributeType::LIST_TENSOR: + return ConvertListAttr(value, ConvertTensorProtoFunctor()); + default: + return nullptr; + } +} + +} // namespace + +AttributeType AttributeTypeFromName(const std::string& type_name) { + const auto* type_map = AttributeTypeNameMap(); + auto it = type_map->find(type_name); + return it != type_map->end() ? it->second : AttributeType::UNKNOWN; +} + +std::string AttributeTypeToName(AttributeType attr_type) { + for (const auto& pair : *AttributeTypeNameMap()) { + if (pair.second == attr_type) { + return pair.first; + } + } + return ""; +} + +Safe_PyObjectPtr ConvertPyObjectToAttributeType(PyObject* value, + AttributeType type) { + Safe_PyObjectPtr result = ConvertAttrOrNull(value, type); + if (!result) { + auto err = absl::StrCat("Failed to convert value of type '", + value->ob_type->tp_name, "' to type '", + AttributeTypeToName(type), "'."); + PyErr_SetString(PyExc_TypeError, err.c_str()); + } + + return result; +} + +} // namespace tensorflow diff --git a/tensorflow/python/framework/op_def_util.h b/tensorflow/python/framework/op_def_util.h new file mode 100644 index 00000000000..ef5e64e68fa --- /dev/null +++ b/tensorflow/python/framework/op_def_util.h @@ -0,0 +1,77 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_PYTHON_FRAMEWORK_OP_DEF_UTIL_H_ +#define TENSORFLOW_PYTHON_FRAMEWORK_OP_DEF_UTIL_H_ + +#include + +#include "tensorflow/python/lib/core/safe_ptr.h" + +namespace tensorflow { + +// Enumerated type corresponding with string values in AttrDef::type. +enum class AttributeType { + UNKNOWN, + ANY, // "any" + FLOAT, // "float" + INT, // "int" + STRING, // "string" + BOOL, // "bool" + DTYPE, // "type" (tf.dtypes.DType) + SHAPE, // "shape" (tf.TensorShape) + TENSOR, // "tensor" (tf.TensorProto) + LIST_ANY, // "list(any)" + LIST_FLOAT, // "list(float)" + LIST_INT, // "list(int)" + LIST_STRING, // "list(string)" + LIST_BOOL, // "list(bool)" + LIST_DTYPE, // "list(dtype)" + LIST_SHAPE, // "list(shape)" + LIST_TENSOR // "list(tensor)" +}; + +// Returns the enumerated value corresponding to a given string (e.g. +// "string" or "list(string)". +AttributeType AttributeTypeFromName(const std::string& type_name); + +// Returns the string corresponding to a given enumerated value. +std::string AttributeTypeToName(AttributeType attr_type); + +// Converts `value` to the specified type and returns a new reference to the +// converted value (if possible); or sets a Python exception and returns +// nullptr. This function is optimized to be fast if `value` already has the +// desired type. +// +// * 'any' values are returned as-is. +// * 'float' values are converted by calling float(value). +// * 'int' values are converted by calling int(value). +// * 'string' values are returned as-is if they are (bytes, unicode); +// otherwise, an exception is raised. +// * 'bool' values are returned as-is if they are boolean; otherwise, an +// exception is raised. +// * 'dtype' values are converted using `dtypes.as_dtype`. +// * 'shape' values are converted using `tensor_shape.as_shape`. +// * 'tensor' values are returned as-is if they are a `TensorProto`; or are +// parsed into `TensorProto` using `textformat.merge` if they are a string. +// Otherwise, an exception is raised. +// * 'list(*)' values are copied to a new list, and then each element is +// converted (in-place) as described above. (If the value is not iterable, +// or if conversion fails for any item, then an exception is raised.) +Safe_PyObjectPtr ConvertPyObjectToAttributeType(PyObject* value, + AttributeType type); + +} // namespace tensorflow + +#endif // TENSORFLOW_PYTHON_FRAMEWORK_OP_DEF_UTIL_H_ diff --git a/tensorflow/python/framework/op_def_util_pybind.cc b/tensorflow/python/framework/op_def_util_pybind.cc new file mode 100644 index 00000000000..d13f605b599 --- /dev/null +++ b/tensorflow/python/framework/op_def_util_pybind.cc @@ -0,0 +1,41 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "pybind11/pybind11.h" +#include "tensorflow/python/framework/op_def_util.h" + +namespace py = pybind11; + +namespace { + +py::handle ConvertAttr(py::handle value, std::string attr_type) { + tensorflow::Safe_PyObjectPtr result = + ::tensorflow::ConvertPyObjectToAttributeType( + value.ptr(), ::tensorflow::AttributeTypeFromName(attr_type)); + if (!result) { + throw py::error_already_set(); + } + Py_INCREF(result.get()); + return result.release(); +} + +} // namespace + +// Expose ConvertPyObjectToAttributeType via Python. Note: this is done to +// simplify testing; ConvertPyObjectToAttributeType is expected to be called +// directly from c++. +PYBIND11_MODULE(_op_def_util, m) { + m.def("ConvertPyObjectToAttributeType", ConvertAttr, py::arg("value"), + py::arg("attr_type_enum")); +} diff --git a/tensorflow/python/framework/op_def_util_test.py b/tensorflow/python/framework/op_def_util_test.py new file mode 100644 index 00000000000..74cd6046f68 --- /dev/null +++ b/tensorflow/python/framework/op_def_util_test.py @@ -0,0 +1,97 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Tests for tensorflow.python.ops.op_def_library.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized + +import numpy as np + +from tensorflow.core.framework import tensor_pb2 +from tensorflow.core.framework import types_pb2 +from tensorflow.python import _op_def_util +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import tensor_shape +from tensorflow.python.framework import test_util +from tensorflow.python.platform import googletest + + +class OpDefUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): + + @parameterized.parameters([ + ("any", "Foo", "Foo"), + ("any", 12, 12), + ("any", {2: 3}, {2: 3}), + ("string", "Foo", "Foo"), + ("string", b"Foo", b"Foo"), + ("int", 12, 12), + ("int", 12.3, 12), + ("float", 12, 12.0), + ("float", 12.3, 12.3), + ("bool", True, True), + ("shape", tensor_shape.TensorShape([3]), tensor_shape.TensorShape([3])), + ("shape", [3], tensor_shape.TensorShape([3])), + ("type", dtypes.int32, dtypes.int32), + ("type", np.int32, dtypes.int32), + ("type", "int32", dtypes.int32), + ("tensor", tensor_pb2.TensorProto(dtype=types_pb2.DataType.DT_FLOAT), + tensor_pb2.TensorProto(dtype=types_pb2.DataType.DT_FLOAT)), + ("tensor", "dtype: DT_FLOAT", + tensor_pb2.TensorProto(dtype=types_pb2.DataType.DT_FLOAT)), + ("list(any)", [1, "foo", 7.3, dtypes.int32], + [1, "foo", 7.3, dtypes.int32]), + ("list(any)", (1, "foo"), [1, "foo"]), + ("list(string)", ["foo", "bar"], ["foo", "bar"]), + ("list(string)", ("foo", "bar"), ["foo", "bar"]), + ("list(string)", iter("abcd"), ["a", "b", "c", "d"]), + ("list(int)", (1, 2.3), [1, 2]), + ("list(float)", (1, 2.3), [1.0, 2.3]), + ("list(bool)", [True, False], [True, False]), + ]) + def testConvert(self, attr_type, value, expected): + result = _op_def_util.ConvertPyObjectToAttributeType(value, attr_type) + + # Check that we get the expected value(s). + self.assertEqual(expected, result) + + # Check that we get the expected type(s). + self.assertEqual(type(expected), type(result)) + if isinstance(result, list): + for expected_item, result_item in zip(expected, result): + self.assertEqual(type(expected_item), type(result_item)) + + @parameterized.parameters([ + ("string", 12), + ("int", "foo"), + ("float", "foo"), + ("bool", 1), + ("dtype", None), + ("shape", 12.0), + ("tensor", [1, 2, 3]), + ("list(any)", 12), + ("list(int)", [1, "two"]), + ("list(string)", [1, "two"]), + ]) + def testConvertError(self, attr_type, value): + with self.assertRaisesRegex(TypeError, "Failed to convert value"): + _op_def_util.ConvertPyObjectToAttributeType(value, attr_type) + +if __name__ == "__main__": + googletest.main() + From be8b52212a31735463b49f2ced43c8f656e23ab4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 10:09:26 -0700 Subject: [PATCH 0640/1017] The code block in TF_BatchFunctionOp documentation was missing its end quote. Now it has it. PiperOrigin-RevId: 325456158 Change-Id: I7d988643100c21451a22f190a2559cb81f54dab7 --- tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td | 1 + tensorflow/core/api_def/base_api/api_def_BatchFunction.pbtxt | 1 + 2 files changed, 2 insertions(+) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td index 1e99675d938..376b7933b47 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td @@ -1295,6 +1295,7 @@ So, for example, in the following code batch_timeout_micros=100000, # 100ms allowed_batch_sizes=[3, 10], batching_queue="") + ``` If more than one session.run call is simultaneously trying to compute `b` the values of `a` will be gathered, non-deterministically concatenated diff --git a/tensorflow/core/api_def/base_api/api_def_BatchFunction.pbtxt b/tensorflow/core/api_def/base_api/api_def_BatchFunction.pbtxt index a7792dc9bf2..b2cace5c3bc 100644 --- a/tensorflow/core/api_def/base_api/api_def_BatchFunction.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_BatchFunction.pbtxt @@ -117,6 +117,7 @@ So, for example, in the following code batch_timeout_micros=100000, # 100ms allowed_batch_sizes=[3, 10], batching_queue="") + ``` If more than one session.run call is simultaneously trying to compute `b` the values of `a` will be gathered, non-deterministically concatenated From a919022737c0510d8f3f461e3b98f621ab0ba3bd Mon Sep 17 00:00:00 2001 From: Ken Franko Date: Fri, 7 Aug 2020 10:10:39 -0700 Subject: [PATCH 0641/1017] Mark ops within WhileRegion control flow for outside compilation when unsupported. Additionally marks WhileRegion for outside compilation if there are captured string arguments. PiperOrigin-RevId: 325456398 Change-Id: I21c83df43cbf475a61efa548224cc6c02cd2367b --- .../mark_ops_for_outside_compilation.mlir | 66 +++++++++++++++++++ .../mark_ops_for_outside_compilation.cc | 29 ++++---- 2 files changed, 79 insertions(+), 16 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir b/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir index d0a4c101bdf..0bb37e4c3cd 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir @@ -86,6 +86,7 @@ func @op_string_operand_string_result(%arg0: tensor) -> tensor } // Test that a tf.IfRegion op with a captured string operand is marked for outside compilation. + // CHECK-LABEL: func @if_region_captured_string func @if_region_captured_string(%arg0: tensor, %arg1: tensor) -> tensor { %0 = "tf_device.cluster"() ( { @@ -175,3 +176,68 @@ func @nested_if_region_string_op(%arg0: tensor, %arg1: tensor) -> ten }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } + +// Test that a tf.WhileRegion op with a captured string operand is marked for outside compilation. + +// CHECK-LABEL: func @while_region_captured_string +func @while_region_captured_string(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf_device.cluster"() ( { + // CHECK: "tf.Const"() {value = dense<1.000000e+00> : tensor} + // CHECK-NOT: _xla_outside_compilation + // CHECK: "tf.WhileRegion" + // CHECK: "tf.StringToNumber" + // CHECK: _xla_outside_compilation = "auto", is_stateless = true + %1 = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + %2:2 = "tf.WhileRegion"(%1, %arg0) ( { + ^bb0(%carg0: tensor, %carg1: tensor): + %limit = constant dense<5> : tensor + %cond = "tf.NotEqual"(%carg1, %limit) : (tensor, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg0: tensor, %barg1: tensor): + %one = constant dense<1> : tensor + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + %3 = "tf.StringToNumber"(%arg1) {out_type = f32} : (tensor) -> tensor + "tf.Yield"(%3, %sub) : (tensor, tensor) -> () + }) {is_stateless = true} : (tensor, tensor) -> (tensor, tensor) + // CHECK: "tf.Identity" + // CHECK-NOT: _xla_outside_compilation + %5 = "tf.Identity"(%2#0) : (tensor) -> (tensor) + tf_device.return %5 : tensor + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + return %0 : tensor +} + +// Test that an unsupported op within a tf.WhileRegion is marked for outside compilation. + +// CHECK-LABEL: func @while_region_unsupported_op +func @while_region_unsupported_op(%arg0: tensor, %arg1: tensor) -> tensor { + %0 = "tf_device.cluster"() ( { + // CHECK: "tf.Const"() {value = dense<1.000000e+00> : tensor} + // CHECK-NOT: _xla_outside_compilation + // CHECK: "tf.WhileRegion" + %1 = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + %2:2 = "tf.WhileRegion"(%1, %arg0) ( { + ^bb0(%carg0: tensor, %carg1: tensor): + %limit = constant dense<5> : tensor + %cond = "tf.NotEqual"(%carg1, %limit) : (tensor, tensor) -> tensor + "tf.Yield"(%cond) : (tensor) -> () + }, { + ^bb0(%barg0: tensor, %barg1: tensor): + %one = constant dense<1> : tensor + %sub = "tf.Sub"(%barg1, %one) : (tensor, tensor) -> tensor + // CHECK: "tf.UnsupportedOp" + // CHECK-SAME: _xla_outside_compilation + %3 = "tf.UnsupportedOp"() {value = dense<1> : tensor} : () -> tensor + // CHECK: "tf.Const"() {value = dense<1.000000e+00> : tensor} + %4 = "tf.Const"() {value = dense<1.0> : tensor} : () -> tensor + "tf.Yield"(%4, %sub) : (tensor, tensor) -> () + // CHECK: {is_stateless = true + }) {is_stateless = true} : (tensor, tensor) -> (tensor, tensor) + // CHECK: "tf.Identity" + // CHECK-NOT: _xla_outside_compilation + %5 = "tf.Identity"(%2#0) : (tensor) -> (tensor) + tf_device.return %5 : tensor + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + return %0 : tensor +} diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc b/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc index 6c93a9eb9cc..c0889affb30 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc @@ -49,6 +49,7 @@ struct MarkOpsForOutsideCompilation void AddSupportedControlFlowOps(MLIRContext* context, llvm::DenseSet* supported_ops) { supported_ops->insert(OperationName("tf.IfRegion", context)); + supported_ops->insert(OperationName("tf.WhileRegion", context)); supported_ops->insert(OperationName("tf.Yield", context)); } @@ -81,21 +82,17 @@ bool IsSupportedOp(Operation& op, mhlo::IsOpAllowedTf2XlaFallback(&op)); } -bool HasCapturedStringOperand(TF::IfRegionOp* if_op) { +// Checks all regions of `op` for captured string operands. +bool HasCapturedStringOperand(Operation* op) { bool string_operand = false; - mlir::visitUsedValuesDefinedAbove( - if_op->then_branch(), if_op->then_branch(), - [&](mlir::OpOperand* operand) { - if (getElementTypeOrSelf(operand->get()).isa()) - string_operand = true; - }); - if (string_operand) return string_operand; - mlir::visitUsedValuesDefinedAbove( - if_op->else_branch(), if_op->else_branch(), - [&](mlir::OpOperand* operand) { - if (getElementTypeOrSelf(operand->get()).isa()) - string_operand = true; - }); + for (auto& region : op->getRegions()) { + mlir::visitUsedValuesDefinedAbove( + region, region, [&](mlir::OpOperand* operand) { + if (getElementTypeOrSelf(operand->get()).isa()) + string_operand = true; + }); + if (string_operand) return string_operand; + } return string_operand; } @@ -106,8 +103,8 @@ LogicalResult MarkUncompilableOps( op->setAttr(kXlaOutsideCompilationAttr, StringAttr::get("auto", op->getContext())); } - if (auto if_op = llvm::dyn_cast(op)) { - if (HasCapturedStringOperand(&if_op)) { + if (llvm::isa(op)) { + if (HasCapturedStringOperand(op)) { op->setAttr(kXlaOutsideCompilationAttr, StringAttr::get("auto", op->getContext())); } From cc38583a9fc80f83482a73e88396ce6f89e8ca08 Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Fri, 7 Aug 2020 10:12:08 -0700 Subject: [PATCH 0642/1017] Fix finished task handling for restarted workers. This reflects the recent change where workers now receive their previous tasks on restart, as opposed to being assigned new tasks. PiperOrigin-RevId: 325456693 Change-Id: I2c99998c1310983ecc70e57f9c7c0a362d42c9d6 --- tensorflow/core/data/service/worker_impl.cc | 11 +++++++++++ tensorflow/core/data/service/worker_impl.h | 2 ++ .../data/experimental/data_service_dataset_op.cc | 14 ++++---------- tensorflow/python/data/kernel_tests/BUILD | 1 - 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/data/service/worker_impl.cc b/tensorflow/core/data/service/worker_impl.cc index 0e955e136d2..d17acffb941 100644 --- a/tensorflow/core/data/service/worker_impl.cc +++ b/tensorflow/core/data/service/worker_impl.cc @@ -76,7 +76,11 @@ Status DataServiceWorkerImpl::Start(const std::string& worker_address) { [this, dispatcher = dispatcher.release()]() { BackgroundThread(dispatcher); }); + LOG(INFO) << "Worker registered with dispatcher running at " + << config_.dispatcher_address(); background_thread_.reset(thread); + mutex_lock l(mu_); + registered_ = true; return Status::OK(); } @@ -118,6 +122,13 @@ Status DataServiceWorkerImpl::GetElement(const GetElementRequest* request, std::vector outputs; { mutex_lock l(mu_); + if (!registered_) { + // We need to reject requests until the worker has registered with the + // dispatcher, so that we don't return NOT_FOUND for tasks that the worker + // had before preemption. + return errors::Unavailable( + "Worker has not yet registered with dispatcher."); + } auto it = tasks_.find(request->task_id()); if (it == tasks_.end()) { return errors::NotFound("DataServiceWorkerImpl::GetElement failed. ", diff --git a/tensorflow/core/data/service/worker_impl.h b/tensorflow/core/data/service/worker_impl.h index 8353d11efdc..36edbe5ce74 100644 --- a/tensorflow/core/data/service/worker_impl.h +++ b/tensorflow/core/data/service/worker_impl.h @@ -84,6 +84,8 @@ class DataServiceWorkerImpl { // Completed tasks which haven't yet been communicated to the dispatcher. absl::flat_hash_set pending_completed_tasks_ TF_GUARDED_BY(mu_); bool cancelled_ TF_GUARDED_BY(mu_) = false; + // Whether the worker has registered with the dispatcher yet. + bool registered_ TF_GUARDED_BY(mu_) = false; // Condition variable for notifying the background thread. condition_variable background_cv_ TF_GUARDED_BY(mu_); std::unique_ptr background_thread_; diff --git a/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc b/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc index ca73799bd24..8a160aa8502 100644 --- a/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/data_service_dataset_op.cc @@ -401,7 +401,6 @@ class DataServiceDatasetOp::Dataset : public DatasetBase { mutex_lock l(mu_); num_running_worker_threads_--; outstanding_requests_--; - VLOG(3) << "Exiting worker thread"; }; worker_threads_.push_back(ctx->StartThread( "tf-data-service-task_thread", [this, done = std::move(done)]() { @@ -437,10 +436,10 @@ class DataServiceDatasetOp::Dataset : public DatasetBase { } worker_thread_cv_.wait(l); } + outstanding_requests_++; if (cancelled_) { return; } - outstanding_requests_++; // Search for a task to update. int num_tasks = tasks_.size(); for (int i = 0; i < num_tasks; ++i) { @@ -461,6 +460,9 @@ class DataServiceDatasetOp::Dataset : public DatasetBase { Status s = GetElement(task_to_process.get(), deadline_micros); if (!s.ok()) { mutex_lock l(mu_); + VLOG(1) << "Failed to get element for task " + << task_to_process->task_id << ": " << s; + task_to_process->in_use = false; status_ = s; get_next_cv_.notify_all(); return; @@ -486,14 +488,6 @@ class DataServiceDatasetOp::Dataset : public DatasetBase { if (s.ok()) { break; } - if (errors::IsNotFound(s)) { - // This indicates that the worker was restarted. The restarted worker - // will get a new task, and the old task is lost. - mutex_lock l(mu_); - finished_tasks_++; - task->end_of_sequence = true; - return Status::OK(); - } // Retry all errors that could indicate preemption. if (!errors::IsUnavailable(s) && !errors::IsCancelled(s) && !errors::IsAborted(s)) { diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD index 639c07bac01..210b6f59681 100644 --- a/tensorflow/python/data/kernel_tests/BUILD +++ b/tensorflow/python/data/kernel_tests/BUILD @@ -94,7 +94,6 @@ tf_py_test( name = "data_service_ops_test", size = "medium", srcs = ["data_service_ops_test.py"], - tags = ["notap"], # "b/163085430" deps = [ "//tensorflow:tensorflow_py", "//tensorflow/python:client_testlib", From 88109c5078286ddda4aad4e10b6bddff096554ba Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 10:19:36 -0700 Subject: [PATCH 0643/1017] Check if environment variables are null before assigning to string PiperOrigin-RevId: 325458185 Change-Id: I38905998dab7bfb5802ad94689282db4c08e271e --- tensorflow/core/kernels/data/optimize_dataset_op.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc index a566693ec3d..74468e71241 100644 --- a/tensorflow/core/kernels/data/optimize_dataset_op.cc +++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc @@ -94,8 +94,16 @@ void OptimizeDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input, // This is currently empty; we have no live experiments yet. absl::flat_hash_map live_experiments; - const string opt_ins_raw = std::getenv("TF_DATA_EXPERIMENT_OPT_IN"); - const string opt_outs_raw = std::getenv("TF_DATA_EXPERIMENT_OPT_OUT"); + const char* opt_ins_raw_cs = std::getenv("TF_DATA_EXPERIMENT_OPT_IN"); + const char* opt_outs_raw_cs = std::getenv("TF_DATA_EXPERIMENT_OPT_OUT"); + string opt_ins_raw; + if (opt_ins_raw_cs != nullptr) { + opt_ins_raw = string(opt_ins_raw_cs); + } + string opt_outs_raw; + if (opt_outs_raw_cs != nullptr) { + opt_outs_raw = string(opt_outs_raw_cs); + } auto hash_func = [](const string& str) { return Hash64(str); }; optimizations = SelectOptimizations( job_name, opt_ins_raw, opt_outs_raw, live_experiments, From 194ff7b835fef5aceb9495ae298e4fa05b8ad23e Mon Sep 17 00:00:00 2001 From: Lucy Fox Date: Fri, 7 Aug 2020 10:46:06 -0700 Subject: [PATCH 0644/1017] Verify that MHLO DynamicUpdateSlice start indices have matching element types. HLO requires that the element types match for all start index parameters. Right now we don't catch this invalid case until export, so adding a check in the verifier so that we catch this sooner. This also requires a small tweak to the TF InplaceUpdate op lowering. PiperOrigin-RevId: 325463796 Change-Id: I71b5ff347a87bb63138d228796ffa1b115d74aba --- .../mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc | 27 +++++++++++++++++++ tensorflow/compiler/mlir/hlo/tests/ops.mlir | 8 ++++++ .../mlir/xla/transforms/legalize_tf.cc | 7 ++++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc index a1f0480f4fe..6f453d1a167 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc @@ -340,6 +340,33 @@ void DynamicIotaOp::getCanonicalizationPatterns( results.insert(context); } +//===----------------------------------------------------------------------===// +// DynamicUpdateSliceOp +//===----------------------------------------------------------------------===// + +static LogicalResult Verify(DynamicUpdateSliceOp op) { + OperandRange indices = op.start_indices(); + if (indices.size() <= 1) return success(); + + // Note: start_indices is constrained to Variadic, so it + // is OK to cast indices to ShapedType here. + auto idx_tensor = indices.take_front().front().getType().cast(); + Type first_elem_ty = idx_tensor.getElementType(); + Type elem_ty; + + for (auto idx : llvm::drop_begin(indices, 1)) { + idx_tensor = idx.getType().cast(); + elem_ty = idx_tensor.getElementType(); + + if (first_elem_ty != elem_ty) { + return op.emitOpError() << "start indices must have same element type " + "(encountered mismatch: " + << first_elem_ty << " vs " << elem_ty << ")"; + } + } + return success(); +} + //===----------------------------------------------------------------------===// // AbsOp //===----------------------------------------------------------------------===// diff --git a/tensorflow/compiler/mlir/hlo/tests/ops.mlir b/tensorflow/compiler/mlir/hlo/tests/ops.mlir index 3443f21bc84..25c7d6aee61 100644 --- a/tensorflow/compiler/mlir/hlo/tests/ops.mlir +++ b/tensorflow/compiler/mlir/hlo/tests/ops.mlir @@ -754,6 +754,14 @@ func @dynamic_update_slice_invalid_start(%input: tensor<3x4xi64>, %update: tenso // ----- +func @dynamic_update_slice_mismatched_start(%input: tensor<11x3x4xi32>, %update: tensor<1x3x4xi32>, %start1: tensor, %start2: tensor, %start3: tensor) -> tensor<11x3x4xi32> { + // expected-error@+1 {{start indices must have same element type (encountered mismatch: 'i32' vs 'i64')}} + %0 = "mhlo.dynamic-update-slice"(%input, %update, %start1, %start2, %start3) : (tensor<11x3x4xi32>, tensor<1x3x4xi32>, tensor, tensor, tensor) -> tensor<11x3x4xi32> + return %0 : tensor<11x3x4xi32> +} + +// ----- + // CHECK-LABEL: func @transpose func @transpose(%arg0: tensor<1x2x3x4xi32>) -> tensor<2x1x4x3xi32> { %0 = "mhlo.transpose"(%arg0) {permutation = dense<[1, 0, 3, 2]> : tensor<4xi64>} : (tensor<1x2x3x4xi32>) -> tensor<2x1x4x3xi32> diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc index f2b3822188c..6d99e714fc2 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc @@ -5007,7 +5007,12 @@ class ConvertInplaceUpdateOp : public OpRewritePattern { SmallVector unpacked_indices_type( indices_type.getDimSize(0), RankedTensorType::get({}, indices_type.getElementType())); - auto zero_attr = IntegerAttr::get(rewriter.getIntegerType(64), 0); + // Note on zero_attr integer type: DynamicUpdateSlice op start_indices are + // required to have matching types. This rewrite rule creates + // DynamicUpdateSlice ops where the first "start index" is always i32 and + // subsequent ones are constructed based on zero_attr. Thus the type + // for zero_attr needs to be i32 as well. + auto zero_attr = IntegerAttr::get(rewriter.getIntegerType(32), 0); auto unpacked_indices = rewriter.create( op.getLoc(), unpacked_indices_type, indices, zero_attr); From 472576cae52279e72a4e5ddf6c9e767af98c4668 Mon Sep 17 00:00:00 2001 From: Peng Wang Date: Fri, 7 Aug 2020 10:46:11 -0700 Subject: [PATCH 0645/1017] Changed stateless_random_ops_test to use parameterized tests. Test time came down from ~600s to ~60s. PiperOrigin-RevId: 325463819 Change-Id: Ida4b4210ef6be21470d7b00688709edd6407e76c --- tensorflow/python/kernel_tests/random/BUILD | 3 - .../random/stateless_random_ops_test.py | 317 +++++++++++------- 2 files changed, 188 insertions(+), 132 deletions(-) diff --git a/tensorflow/python/kernel_tests/random/BUILD b/tensorflow/python/kernel_tests/random/BUILD index 31e0417102d..06360fc2095 100644 --- a/tensorflow/python/kernel_tests/random/BUILD +++ b/tensorflow/python/kernel_tests/random/BUILD @@ -120,9 +120,6 @@ cuda_py_test( size = "medium", srcs = ["stateless_random_ops_test.py"], shard_count = 10, - tags = [ - "notap", # b/162112278 - ], tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", diff --git a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py index 27b10ea2258..f3949f30c03 100644 --- a/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py +++ b/tensorflow/python/kernel_tests/random/stateless_random_ops_test.py @@ -52,161 +52,220 @@ def invert_philox(key, value): return np.array(value) -class StatelessOpsTest(test.TestCase, parameterized.TestCase): +SEEDS = ((7, 17), (11, 5), (2, 3)) +SEED_TYPES = [dtypes.int32, dtypes.int64] - def _test_match(self, cases): - # Stateless ops should be the same as stateful ops on the first call - # after seed scrambling. - cases = tuple(cases) - key = 0x3ec8f720, 0x02461e29 - for seed in (7, 17), (11, 5), (2, 3): - preseed = invert_philox(key, (seed[0], 0, seed[1], 0)).astype(np.uint64) - preseed = preseed[::2] | preseed[1::2] << 32 - random_seed.set_random_seed(seed[0]) - with test_util.use_gpu(): - for stateless_op, stateful_op in cases: - if context.executing_eagerly(): - # Call set_random_seed in order to clear kernel cache, to prevent - # kernel reusing for the stateful op - random_seed.set_random_seed(seed[0]) - stateful = stateful_op(seed=seed[1]) - pure = stateless_op(seed=preseed) - self.assertAllEqual(stateful, pure) - def _test_determinism(self, cases): - # Stateless values should be equal iff the seeds are equal (roughly) - cases = tuple(cases) - seeds = [(x, y) for x in range(5) for y in range(5)] * 3 - with self.test_session(use_gpu=True), test_util.use_gpu(): - for seed_type in [dtypes.int32, dtypes.int64]: - for stateless_op, _ in cases: - if context.executing_eagerly(): - values = [ - (seed, stateless_op(seed=constant_op.constant(seed, seed_type))) - for seed in seeds] - else: - # Have this branch because the above branch is too slow in graph - # mode - seed_t = array_ops.placeholder(seed_type, shape=[2]) - pure = stateless_op(seed=seed_t) - values = [ - (seed, pure.eval(feed_dict={seed_t: seed})) for seed in seeds - ] - for s0, v0 in values: - for s1, v1 in values: - self.assertEqual(s0 == s1, np.all(v0 == v1)) - - def _float_cases(self, shape_dtypes=(None,)): - float_cases = ( - # Uniform distribution, with and without range - (stateless.stateless_random_uniform, random_ops.random_uniform, {}), - (stateless.stateless_random_uniform, random_ops.random_uniform, - dict(minval=2.2, maxval=7.1)), - # Normal distribution, with and without mean+stddev - (stateless.stateless_random_normal, random_ops.random_normal, {}), - (stateless.stateless_random_normal, random_ops.random_normal, - dict(mean=2, stddev=3)), - # Truncated normal distribution, with and without mean+stddev - (stateless.stateless_truncated_normal, random_ops.truncated_normal, {}), - (stateless.stateless_truncated_normal, random_ops.truncated_normal, - dict(mean=3, stddev=4)), - ) - for dtype in dtypes.float16, dtypes.float32, dtypes.float64: - for shape_dtype in shape_dtypes: - for shape in (), (3,), (2, 5): - if shape_dtype is not None: - shape = constant_op.constant(shape, dtype=shape_dtype) - for stateless_op, stateful_op, kwds in float_cases: - kwds = dict(shape=shape, dtype=dtype, **kwds) - yield (functools.partial(stateless_op, **kwds), - functools.partial(stateful_op, **kwds)) - - def _int_cases(self, shape_dtypes=(None,)): +def float_cases(shape_dtypes=(None,)): + cases = ( + # Uniform distribution, with and without range + (stateless.stateless_random_uniform, random_ops.random_uniform, {}), + (stateless.stateless_random_uniform, random_ops.random_uniform, + dict(minval=2.2, maxval=7.1)), + # Normal distribution, with and without mean+stddev + (stateless.stateless_random_normal, random_ops.random_normal, {}), + (stateless.stateless_random_normal, random_ops.random_normal, + dict(mean=2, stddev=3)), + # Truncated normal distribution, with and without mean+stddev + (stateless.stateless_truncated_normal, random_ops.truncated_normal, {}), + (stateless.stateless_truncated_normal, random_ops.truncated_normal, + dict(mean=3, stddev=4)), + ) + # Explicitly passing in params because capturing cell variable from loop is + # problematic in Python + def wrap(op, dtype, shape, shape_dtype, kwds, seed): + shape_ = (constant_op.constant(shape, dtype=shape_dtype) + if shape_dtype is not None else shape) + return op(seed=seed, shape=shape_, dtype=dtype, **kwds) + for dtype in dtypes.float16, dtypes.float32, dtypes.float64: for shape_dtype in shape_dtypes: for shape in (), (3,), (2, 5): - if shape_dtype is not None: - shape = constant_op.constant(shape, dtype=shape_dtype) - for dtype in dtypes.int32, dtypes.int64: - kwds = dict(minval=2, maxval=11111, dtype=dtype, shape=shape) - yield (functools.partial(stateless.stateless_random_uniform, **kwds), - functools.partial(random_ops.random_uniform, **kwds)) + for stateless_op, stateful_op, kwds in cases: + yield (functools.partial(wrap, stateless_op, dtype, shape, + shape_dtype, kwds), + functools.partial(wrap, stateful_op, dtype, shape, + shape_dtype, kwds)) - def _multinomial_cases(self): - num_samples = 10 - for logits_dtype in np.float16, np.float32, np.float64: - for output_dtype in dtypes.int32, dtypes.int64: - for logits in ([[0.1, 0.25, 0.5, 0.15]], [[0.5, 0.5], [0.8, 0.2], - [0.25, 0.75]]): - kwds = dict( + +def int_cases(shape_dtypes=(None,)): + def wrap(op, shape, shape_dtype, dtype, seed): + shape_ = (constant_op.constant(shape, dtype=shape_dtype) + if shape_dtype is not None else shape) + return op(seed=seed, shape=shape_, minval=2, maxval=11111, + dtype=dtype) + for shape_dtype in shape_dtypes: + for shape in (), (3,), (2, 5): + for dtype in dtypes.int32, dtypes.int64: + yield (functools.partial(wrap, stateless.stateless_random_uniform, + shape, shape_dtype, dtype), + functools.partial(wrap, random_ops.random_uniform, + shape, shape_dtype, dtype)) + + +def multinomial_cases(): + num_samples = 10 + def wrap(op, logits, logits_dtype, output_dtype, seed): + return op(seed=seed, logits=constant_op.constant(logits, dtype=logits_dtype), - num_samples=num_samples, - output_dtype=output_dtype) - yield (functools.partial(stateless.stateless_multinomial, **kwds), - functools.partial(random_ops.multinomial, **kwds)) + num_samples=num_samples, output_dtype=output_dtype) + for logits_dtype in np.float16, np.float32, np.float64: + for output_dtype in dtypes.int32, dtypes.int64: + for logits in ([[0.1, 0.25, 0.5, 0.15]], [[0.5, 0.5], [0.8, 0.2], + [0.25, 0.75]]): + yield (functools.partial(wrap, stateless.stateless_multinomial, logits, + logits_dtype, output_dtype), + functools.partial(wrap, random_ops.multinomial, logits, + logits_dtype, output_dtype)) - def _gamma_cases(self): - for dtype in np.float16, np.float32, np.float64: - for alpha in ([[.5, 1., 2.]], [[0.5, 0.5], [0.8, 0.2], [0.25, 0.75]]): - kwds = dict(alpha=constant_op.constant(alpha, dtype=dtype), dtype=dtype) - yield ( - functools.partial(stateless.stateless_random_gamma, - shape=(10,) + tuple(np.shape(alpha)), **kwds), - functools.partial(random_ops.random_gamma, shape=(10,), **kwds)) - def _poisson_cases(self): - for lam_dtype in np.float16, np.float32, np.float64, np.int32, np.int64: - for out_dtype in np.float16, np.float32, np.float64, np.int32, np.int64: - for lam in ([[5.5, 1., 2.]], [[7.5, 10.5], [3.8, 8.2], [1.25, 9.75]]): - kwds = dict( +def gamma_cases(): + def wrap(op, alpha, dtype, shape, seed): + return op(seed=seed, shape=shape, + alpha=constant_op.constant(alpha, dtype=dtype), dtype=dtype) + for dtype in np.float16, np.float32, np.float64: + for alpha in ([[.5, 1., 2.]], [[0.5, 0.5], [0.8, 0.2], [0.25, 0.75]]): + yield (functools.partial(wrap, stateless.stateless_random_gamma, alpha, + dtype, (10,) + tuple(np.shape(alpha))), + functools.partial(wrap, random_ops.random_gamma, alpha, + dtype, (10,))) + + +def poisson_cases(): + def wrap(op, lam, lam_dtype, out_dtype, shape, seed): + return op(seed=seed, shape=shape, lam=constant_op.constant(lam_dtype(lam), dtype=lam_dtype), dtype=out_dtype) - yield ( - functools.partial(stateless.stateless_random_poisson, - shape=(10,) + tuple(np.shape(lam)), - **kwds), - functools.partial(random_ops.random_poisson, shape=(10,), **kwds)) + for lam_dtype in np.float16, np.float32, np.float64, np.int32, np.int64: + for out_dtype in np.float16, np.float32, np.float64, np.int32, np.int64: + for lam in ([[5.5, 1., 2.]], [[7.5, 10.5], [3.8, 8.2], [1.25, 9.75]]): + yield (functools.partial(wrap, stateless.stateless_random_poisson, lam, + lam_dtype, out_dtype, + (10,) + tuple(np.shape(lam))), + functools.partial(wrap, random_ops.random_poisson, lam, + lam_dtype, out_dtype, (10,))) - @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testMatchFloat(self): - self._test_match(self._float_cases()) - @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testMatchInt(self): - self._test_match(self._int_cases()) +class StatelessOpsTest(test.TestCase, parameterized.TestCase): - @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testMatchMultinomial(self): - self._test_match(self._multinomial_cases()) + def _test_match(self, case, seed): + # Stateless ops should be the same as stateful ops on the first call + # after seed scrambling. + key = 0x3ec8f720, 0x02461e29 + preseed = invert_philox(key, (seed[0], 0, seed[1], 0)).astype(np.uint64) + preseed = preseed[::2] | preseed[1::2] << 32 + random_seed.set_random_seed(seed[0]) + with test_util.use_gpu(): + stateless_op, stateful_op = case + if context.executing_eagerly(): + # Call set_random_seed in order to clear kernel cache, to prevent + # kernel reusing for the stateful op + random_seed.set_random_seed(seed[0]) + stateful = stateful_op(seed=seed[1]) + pure = stateless_op(seed=preseed) + self.assertAllEqual(stateful, pure) - @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testMatchGamma(self): - self._test_match(self._gamma_cases()) + def _test_determinism(self, case, seed_type): + # Stateless values should be equal iff the seeds are equal (roughly) + seeds = [(x, y) for x in range(5) for y in range(5)] * 3 # pylint: disable=g-complex-comprehension + with self.test_session(use_gpu=True), test_util.use_gpu(): + stateless_op, _ = case + if context.executing_eagerly(): + values = [ + (seed, stateless_op(seed=constant_op.constant(seed, seed_type))) + for seed in seeds] + else: + # Have this branch because the above branch is too slow in graph + # mode + seed_t = array_ops.placeholder(seed_type, shape=[2]) + pure = stateless_op(seed=seed_t) + values = [ + (seed, pure.eval(feed_dict={seed_t: seed})) for seed in seeds + ] + for s0, v0 in values: + for s1, v1 in values: + self.assertEqual(s0 == s1, np.all(v0 == v1)) + @parameterized.named_parameters( + ('_%s_%s' % (case_id, seed_id), case, seed) # pylint: disable=g-complex-comprehension + for seed_id, seed in enumerate(SEEDS) + for case_id, case in enumerate(float_cases())) @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testMatchPoisson(self): - self._test_match(self._poisson_cases()) + def testMatchFloat(self, case, seed): + self._test_match(case, seed) + @parameterized.named_parameters( + ('_%s_%s' % (case_id, seed_id), case, seed) # pylint: disable=g-complex-comprehension + for seed_id, seed in enumerate(SEEDS) + for case_id, case in enumerate(int_cases())) @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testDeterminismFloat(self): - self._test_determinism( - self._float_cases(shape_dtypes=(dtypes.int32, dtypes.int64))) + def testMatchInt(self, case, seed): + self._test_match(case, seed) + @parameterized.named_parameters( + ('_%s_%s' % (case_id, seed_id), case, seed) # pylint: disable=g-complex-comprehension + for seed_id, seed in enumerate(SEEDS) + for case_id, case in enumerate(multinomial_cases())) @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testDeterminismInt(self): - self._test_determinism( - self._int_cases(shape_dtypes=(dtypes.int32, dtypes.int64))) + def testMatchMultinomial(self, case, seed): + self._test_match(case, seed) + @parameterized.named_parameters( + ('_%s_%s' % (case_id, seed_id), case, seed) # pylint: disable=g-complex-comprehension + for seed_id, seed in enumerate(SEEDS) + for case_id, case in enumerate(gamma_cases())) @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testDeterminismMultinomial(self): - self._test_determinism(self._multinomial_cases()) + def testMatchGamma(self, case, seed): + self._test_match(case, seed) + @parameterized.named_parameters( + ('_%s_%s' % (case_id, seed_id), case, seed) # pylint: disable=g-complex-comprehension + for seed_id, seed in enumerate(SEEDS) + for case_id, case in enumerate(poisson_cases())) @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testDeterminismGamma(self): - self._test_determinism(self._gamma_cases()) + def testMatchPoisson(self, case, seed): + self._test_match(case, seed) + @parameterized.named_parameters( + ('_%s_%s' % (case_id, type_id), case, seed_type) # pylint: disable=g-complex-comprehension + for type_id, seed_type in enumerate(SEED_TYPES) + for case_id, case in enumerate(float_cases( + shape_dtypes=(dtypes.int32, dtypes.int64)))) @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') - def testDeterminismPoisson(self): - self._test_determinism(self._poisson_cases()) + def testDeterminismFloat(self, case, seed_type): + self._test_determinism(case, seed_type) + + @parameterized.named_parameters( + ('_%s_%s' % (case_id, type_id), case, seed_type) # pylint: disable=g-complex-comprehension + for type_id, seed_type in enumerate(SEED_TYPES) + for case_id, case in enumerate(int_cases( + shape_dtypes=(dtypes.int32, dtypes.int64)))) + @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') + def testDeterminismInt(self, case, seed_type): + self._test_determinism(case, seed_type) + + @parameterized.named_parameters( + ('_%s_%s' % (case_id, type_id), case, seed_type) # pylint: disable=g-complex-comprehension + for type_id, seed_type in enumerate(SEED_TYPES) + for case_id, case in enumerate(multinomial_cases())) + @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') + def testDeterminismMultinomial(self, case, seed_type): + self._test_determinism(case, seed_type) + + @parameterized.named_parameters( + ('_%s_%s' % (case_id, type_id), case, seed_type) # pylint: disable=g-complex-comprehension + for type_id, seed_type in enumerate(SEED_TYPES) + for case_id, case in enumerate(gamma_cases())) + @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') + def testDeterminismGamma(self, case, seed_type): + self._test_determinism(case, seed_type) + + @parameterized.named_parameters( + ('_%s_%s' % (case_id, type_id), case, seed_type) # pylint: disable=g-complex-comprehension + for type_id, seed_type in enumerate(SEED_TYPES) + for case_id, case in enumerate(poisson_cases())) + @test_util.disable_tfrt('tensorflow::DirectSession::Run crashes. b/156187396') + def testDeterminismPoisson(self, case, seed_type): + self._test_determinism(case, seed_type) def assertDTypeEqual(self, a, b): self.assertEqual(dtypes.as_dtype(a), dtypes.as_dtype(b)) From 595776085e1baadbf67ca62a81e6fdb54e61e4d9 Mon Sep 17 00:00:00 2001 From: Jiho Choi Date: Fri, 7 Aug 2020 11:11:46 -0700 Subject: [PATCH 0646/1017] Add helper to identify JAX op name and type. PiperOrigin-RevId: 325469642 Change-Id: I796eac9ab961385dd0b7b99670a3883780604d1e --- tensorflow/core/profiler/utils/tf_op_utils.cc | 13 +++++++++++-- tensorflow/core/profiler/utils/tf_op_utils.h | 3 +++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/profiler/utils/tf_op_utils.cc b/tensorflow/core/profiler/utils/tf_op_utils.cc index e58ccba445b..941676079b9 100644 --- a/tensorflow/core/profiler/utils/tf_op_utils.cc +++ b/tensorflow/core/profiler/utils/tf_op_utils.cc @@ -32,6 +32,7 @@ namespace { const absl::string_view kIterator = "Iterator"; const absl::string_view kSeparator = "::"; +constexpr char kNameScopeSeparator = '/'; } // namespace @@ -51,10 +52,17 @@ bool IsTfOpType(absl::string_view op_type) { } bool IsJaxOpType(absl::string_view op_type) { - static const LazyRE2 kJaxOpTypeRegEx = {"[a-z_]*"}; + static const LazyRE2 kJaxOpTypeRegEx = {"[a-z_][a-z_]*"}; return RE2::FullMatch(op_type, *kJaxOpTypeRegEx); } +bool IsJaxOpNameAndType(absl::string_view op_name, absl::string_view op_type) { + if (op_name.empty() || !IsJaxOpType(op_type)) return false; + std::vector split_result = + absl::StrSplit(op_name, kNameScopeSeparator); + return absl::StrContains(split_result.back(), op_type); +} + TfOp ParseTfOpFullname(absl::string_view tf_op_fullname) { // TF Op names have the format "name:type". TfOp tf_op = {Category::kUnknown, tf_op_fullname, kUnknownOp}; @@ -85,7 +93,8 @@ TfOp ParseTfOpFullname(absl::string_view tf_op_fullname) { } std::vector ParseTfNameScopes(const TfOp& tf_op) { - std::vector name_scopes = absl::StrSplit(tf_op.name, '/'); + std::vector name_scopes = + absl::StrSplit(tf_op.name, kNameScopeSeparator); // The last element is an op name not TF name scope. if (!name_scopes.empty()) name_scopes.pop_back(); return name_scopes; diff --git a/tensorflow/core/profiler/utils/tf_op_utils.h b/tensorflow/core/profiler/utils/tf_op_utils.h index f0668190a07..76e6256164b 100644 --- a/tensorflow/core/profiler/utils/tf_op_utils.h +++ b/tensorflow/core/profiler/utils/tf_op_utils.h @@ -104,6 +104,9 @@ bool IsTfOpType(absl::string_view op_type); // Returns true if the given string matches JAX pattern. bool IsJaxOpType(absl::string_view op_type); +// Returns true if the given strings match JAX pattern. +bool IsJaxOpNameAndType(absl::string_view op_name, absl::string_view op_type); + } // namespace profiler } // namespace tensorflow From dbb961df7fcaabb3c504e31b541b338d6360bec0 Mon Sep 17 00:00:00 2001 From: Jay Shi Date: Fri, 7 Aug 2020 11:36:44 -0700 Subject: [PATCH 0647/1017] [tf.data] Add more unit test to check the correctness of `disable_intra_op_parallelism` optimization. PiperOrigin-RevId: 325475258 Change-Id: I3a8480a15c0828add3fa37e7a7afe095a20a73e2 --- .../data/disable_intra_op_parallelism_test.cc | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism_test.cc b/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism_test.cc index 76d6b46fb4e..b1c886594ec 100644 --- a/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism_test.cc +++ b/tensorflow/core/grappler/optimizers/data/disable_intra_op_parallelism_test.cc @@ -70,10 +70,11 @@ TEST_P(IntraOpAlreadySetTest, IntraOpParallelism) { TF_ASSERT_OK(optimizer.Optimize(nullptr, item, &output)); EXPECT_EQ(output.node_size(), 6); EXPECT_TRUE(graph_utils::ContainsNodeWithOp(op, output)); - NodeDef test_node = output.node(graph_utils::FindGraphNodeWithOp(op, output)); - NodeDef test_val = output.node( - graph_utils::FindGraphNodeWithName(test_node.input(1), output)); - EXPECT_EQ(test_val.attr().at("value").tensor().int64_val(0), value); + NodeDef parallelism_node = + output.node(graph_utils::FindGraphNodeWithOp(op, output)); + NodeDef parallelism_val = output.node( + graph_utils::FindGraphNodeWithName(parallelism_node.input(1), output)); + EXPECT_EQ(parallelism_val.attr().at("value").tensor().int64_val(0), value); } INSTANTIATE_TEST_SUITE_P( @@ -105,11 +106,19 @@ TEST(IntraOpNotSetTest, IntraOpParallelism) { EXPECT_EQ(output.node_size(), 7); EXPECT_TRUE( graph_utils::ContainsNodeWithOp("MaxIntraOpParallelismDataset", output)); - NodeDef test_node = output.node( - graph_utils::FindGraphNodeWithOp("MaxIntraOpParallelismDataset", output)); - NodeDef test_val = output.node( - graph_utils::FindGraphNodeWithName(test_node.input(1), output)); - EXPECT_EQ(test_val.attr().at("value").tensor().int64_val(0), 1); + NodeDef sink_node = + output.node(graph_utils::FindGraphNodeWithName("Sink", output)); + EXPECT_EQ(sink_node.input_size(), 1); + NodeDef parallelism_node = output.node( + graph_utils::FindGraphNodeWithName(sink_node.input(0), output)); + EXPECT_EQ(parallelism_node.op(), "MaxIntraOpParallelismDataset"); + EXPECT_EQ(parallelism_node.input_size(), 2); + NodeDef range_node = output.node( + graph_utils::FindGraphNodeWithName(parallelism_node.input(0), output)); + EXPECT_EQ(range_node.name(), "range"); + NodeDef parallelism_val = output.node( + graph_utils::FindGraphNodeWithName(parallelism_node.input(1), output)); + EXPECT_EQ(parallelism_val.attr().at("value").tensor().int64_val(0), 1); } } // namespace From 2af416baef3c5dd9927950515ca357f5d28d483b Mon Sep 17 00:00:00 2001 From: Kaixi Hou Date: Fri, 7 Aug 2020 11:51:08 -0700 Subject: [PATCH 0648/1017] Fix a logic issue --- tensorflow/core/grappler/optimizers/generic_layout_optimizer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/grappler/optimizers/generic_layout_optimizer.cc b/tensorflow/core/grappler/optimizers/generic_layout_optimizer.cc index 6053f96ae08..9e3a09b5d79 100644 --- a/tensorflow/core/grappler/optimizers/generic_layout_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/generic_layout_optimizer.cc @@ -77,7 +77,7 @@ inline bool NumConvOnDeviceWithDataTypeOverThreshold( for (const auto& node : context.graph_view->GetNodes()) { const auto* node_def = node.node(); - if (!IsConv2D(*node_def) or !IsConv3D(*node_def)) { + if (!IsConv2D(*node_def) and !IsConv3D(*node_def)) { continue; } const string& device_name = From 49aae376828f823a5b0edc561b8cc7f0d48fb09f Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Fri, 7 Aug 2020 11:56:32 -0700 Subject: [PATCH 0649/1017] Changed signature of GenerateWorkGroupSizesAlignedToGrid to return void instead of Status. This function returned status::ok always. PiperOrigin-RevId: 325479195 Change-Id: Ib2d4e51265d3e226d60be7ae5313a8d5d9d82b2e --- .../delegates/gpu/cl/kernels/work_group_picking.cc | 4 ++-- .../lite/delegates/gpu/common/workgroup_selection.cc | 12 ++++++------ .../lite/delegates/gpu/common/workgroup_selection.h | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc index 3771a5b033a..e85e20761e3 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc @@ -84,8 +84,8 @@ absl::Status GetBestWorkGroupAlignedToGrid(const TuningParameters& params, max_wg_size.x = params.info->max_work_group_size_x; max_wg_size.y = params.info->max_work_group_size_y; max_wg_size.z = params.info->max_work_group_size_z; - RETURN_IF_ERROR(GenerateWorkGroupSizesAlignedToGrid( - grid, max_wg_size, kernel.GetMaxWorkGroupSize(), &work_groups)); + GenerateWorkGroupSizesAlignedToGrid( + grid, max_wg_size, kernel.GetMaxWorkGroupSize(), &work_groups); int best_work_group_index; RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( kernel, *params.info, grid, work_groups, &best_work_group_index)); diff --git a/tensorflow/lite/delegates/gpu/common/workgroup_selection.cc b/tensorflow/lite/delegates/gpu/common/workgroup_selection.cc index 3abab71829f..5ae2a53f449 100644 --- a/tensorflow/lite/delegates/gpu/common/workgroup_selection.cc +++ b/tensorflow/lite/delegates/gpu/common/workgroup_selection.cc @@ -184,9 +184,10 @@ template std::vector GenerateWorkGroupSizes( WorkGroupSizeAlignment z_alignment); template -absl::Status GenerateWorkGroupSizesAlignedToGrid( - const T& grid, const T& max_work_group_size, - const int max_work_group_invocations, std::vector* work_groups) { +void GenerateWorkGroupSizesAlignedToGrid(const T& grid, + const T& max_work_group_size, + const int max_work_group_invocations, + std::vector* work_groups) { auto alignment = WorkGroupSizeAlignment::PRECISE; *work_groups = GenerateWorkGroupSizes( grid, /*min_work_group_total_size = */ 32, max_work_group_invocations, @@ -196,16 +197,15 @@ absl::Status GenerateWorkGroupSizesAlignedToGrid( AddCornerCases(grid, max_work_group_invocations, max_work_group_size, alignment, alignment, alignment, work_groups); } - return absl::OkStatus(); } // Specializations of GenerateWorkGroupSizesAlignedToGrid for int3 and uint3 -template absl::Status GenerateWorkGroupSizesAlignedToGrid( +template void GenerateWorkGroupSizesAlignedToGrid( const int3& grid, const int3& max_work_group_size, const int max_work_group_invocations, std::vector* work_groups); -template absl::Status GenerateWorkGroupSizesAlignedToGrid( +template void GenerateWorkGroupSizesAlignedToGrid( const uint3& grid, const uint3& max_work_group_size, const int max_work_group_invocations, std::vector* work_groups); diff --git a/tensorflow/lite/delegates/gpu/common/workgroup_selection.h b/tensorflow/lite/delegates/gpu/common/workgroup_selection.h index 75967cb04df..a08bfce991a 100644 --- a/tensorflow/lite/delegates/gpu/common/workgroup_selection.h +++ b/tensorflow/lite/delegates/gpu/common/workgroup_selection.h @@ -42,9 +42,10 @@ std::vector GenerateWorkGroupSizes( WorkGroupSizeAlignment y_alignment, WorkGroupSizeAlignment z_alignment); template -absl::Status GenerateWorkGroupSizesAlignedToGrid( - const T& grid, const T& max_work_group_size, - const int max_work_group_invocations, std::vector* work_groups); +void GenerateWorkGroupSizesAlignedToGrid(const T& grid, + const T& max_work_group_size, + const int max_work_group_invocations, + std::vector* work_groups); } // namespace gpu } // namespace tflite From f037d18d239a0d6c78b62c43c5b894beb8b980eb Mon Sep 17 00:00:00 2001 From: Amit Patankar Date: Fri, 7 Aug 2020 12:03:42 -0700 Subject: [PATCH 0650/1017] Fix the missing symbol Windows breakage. PiperOrigin-RevId: 325480645 Change-Id: I6bd669446b2c74ad7f4bdb5eb4813c0ad0fecba8 --- tensorflow/tools/def_file_filter/symbols_pybind.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/def_file_filter/symbols_pybind.txt b/tensorflow/tools/def_file_filter/symbols_pybind.txt index 55db32e8d6f..b2546582418 100644 --- a/tensorflow/tools/def_file_filter/symbols_pybind.txt +++ b/tensorflow/tools/def_file_filter/symbols_pybind.txt @@ -20,6 +20,7 @@ tensorflow::swig::AssertSameStructureForData tensorflow::swig::RegisterPyObject tensorflow::swig::RegisterType tensorflow::swig::IsEagerTensorSlow +tensorflow::swig::GetRegisteredPyObject [util_port] # util_port tensorflow::IsGoogleCudaEnabled From 60dd10d03fb74d285dd7b3cad0acf350d542b620 Mon Sep 17 00:00:00 2001 From: Jonathan Chu Date: Fri, 7 Aug 2020 19:18:50 +0000 Subject: [PATCH 0651/1017] Remove TODO --- tensorflow/python/eager/def_function.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/eager/def_function.py b/tensorflow/python/eager/def_function.py index 8447245b524..a333c5e468b 100644 --- a/tensorflow/python/eager/def_function.py +++ b/tensorflow/python/eager/def_function.py @@ -917,7 +917,6 @@ class Function(object): canon_args, canon_kwds, flat_args, flat_kwds = \ self._stateful_fn._function_spec.canonicalize_function_inputs( # pylint: disable=protected-access *args, **kwds) - # TODO(jlchu): verify that modification to fn_with_cond works return function_lib.defun(fn_with_cond)(canon_args, canon_kwds, flat_args, flat_kwds) From 486ee6c6c6f037e63a5aa69057b5986d7a8ce145 Mon Sep 17 00:00:00 2001 From: Robert David Date: Fri, 7 Aug 2020 12:33:22 -0700 Subject: [PATCH 0652/1017] Move IsActivationSupported and MaybeFuseActivation to model_builder_helper.h/.cc PiperOrigin-RevId: 325486463 Change-Id: I25d035ffecffe2514f15a5f34ecfc16cc5741e4a --- tensorflow/lite/delegates/gpu/common/BUILD | 3 +- .../delegates/gpu/common/model_builder.cc | 97 +------------------ .../gpu/common/model_builder_helper.cc | 90 +++++++++++++++++ .../gpu/common/model_builder_helper.h | 12 +++ 4 files changed, 109 insertions(+), 93 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/common/BUILD b/tensorflow/lite/delegates/gpu/common/BUILD index 3caee09ca7e..60a0fda422c 100644 --- a/tensorflow/lite/delegates/gpu/common/BUILD +++ b/tensorflow/lite/delegates/gpu/common/BUILD @@ -109,6 +109,7 @@ cc_library( ":data_type", ":model", ":model_builder_helper", + ":model_transformer", ":object_reader", ":operations", ":shape", @@ -125,7 +126,6 @@ cc_library( "//tensorflow/lite/kernels:kernel_util", "//tensorflow/lite/kernels/internal:reference_base", "//tensorflow/lite/kernels/internal:tensor", - "//tensorflow/lite/schema:schema_fbs", ] + tf_platform_alias("custom_parsers", "//tensorflow/lite/delegates/gpu/common/"), ) @@ -148,6 +148,7 @@ cc_library( deps = [ ":data_type", ":model", + ":operations", ":shape", ":status", ":tensor", diff --git a/tensorflow/lite/delegates/gpu/common/model_builder.cc b/tensorflow/lite/delegates/gpu/common/model_builder.cc index 84622cdc294..426d4d2436a 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder.cc +++ b/tensorflow/lite/delegates/gpu/common/model_builder.cc @@ -16,29 +16,29 @@ limitations under the License. #include "tensorflow/lite/delegates/gpu/common/model_builder.h" #include +#include #include -#include -#include +#include #include +#include #include #include #include +#include #include #include "absl/container/flat_hash_map.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" -#include "tensorflow/lite/builtin_op_data.h" #include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" -#include "tensorflow/lite/context.h" -#include "tensorflow/lite/context_util.h" #include "tensorflow/lite/delegates/gpu/common/custom_parsers.h" #include "tensorflow/lite/delegates/gpu/common/data_type.h" #include "tensorflow/lite/delegates/gpu/common/model.h" #include "tensorflow/lite/delegates/gpu/common/model_builder_helper.h" +#include "tensorflow/lite/delegates/gpu/common/model_transformer.h" #include "tensorflow/lite/delegates/gpu/common/object_reader.h" #include "tensorflow/lite/delegates/gpu/common/operations.h" #include "tensorflow/lite/delegates/gpu/common/shape.h" @@ -49,34 +49,12 @@ limitations under the License. #include "tensorflow/lite/kernels/internal/reference/dequantize.h" #include "tensorflow/lite/kernels/internal/tensor_ctypes.h" #include "tensorflow/lite/kernels/kernel_util.h" -#include "tensorflow/lite/schema/schema_generated.h" #include "tensorflow/lite/util.h" namespace tflite { namespace gpu { namespace { -// Creates a node that consumes output from the given node. Because output need -// to stay the same, newly created node will inherit the output from the given -// node, which will in turn get newly created copy of output. This is necessary -// to preserve reference consistency if another node was pointing at that -// output: -// node(output) -// will turn into: -// node(copy(output)) <- passthrough_node(output) -absl::Status NewPassthroughNode(GraphFloat32* graph, Node* node, - const Value* output, Node** passthru_node) { - *passthru_node = graph->NewNode(); - // Make copies for every output in the original node. - RETURN_IF_ERROR(graph->SetProducer((*passthru_node)->id, output->id)); - Value* copy_output = graph->NewValue(); - RETURN_IF_ERROR(graph->SetProducer(node->id, copy_output->id)); - RETURN_IF_ERROR(graph->AddConsumer((*passthru_node)->id, copy_output->id)); - copy_output->tensor = output->tensor; - copy_output->tensor.ref = -1; - return absl::OkStatus(); -} - absl::Status CheckTensorIsAvailable(const TfLiteContext* context, const TfLiteNode* tflite_node, int idx) { // If tensor id is in range, it's guaranteed that it'll be available. @@ -105,71 +83,6 @@ class TFLiteOperationParser { const TfLiteRegistration* registration) = 0; }; -absl::Status IsActivationSupported(TfLiteFusedActivation fused_activation) { - switch (fused_activation) { - case kTfLiteActNone: - case kTfLiteActRelu: - case kTfLiteActReluN1To1: - case kTfLiteActRelu6: - case kTfLiteActTanh: - case kTfLiteActSigmoid: - return absl::OkStatus(); - case kTfLiteActSignBit: - return absl::UnimplementedError( - "TfLiteFusedActivation.kTfLiteActSignBit"); - - // Do not add default; we want compilation error rather than run-time - // error. - } -} - -// If there is fused activation present, then there will be another node created -// that will have identical output as the given node. New operation node will -// depend on the given node output. -absl::Status MaybeFuseActivation(TfLiteFusedActivation fused_activation, - GraphFloat32* graph, Node* node) { - const auto outputs = graph->FindOutputs(node->id); - if (outputs.size() != 1) { - return absl::InternalError("Number of outputs != 1"); - } - switch (fused_activation) { - case kTfLiteActNone: - // Nothing to do here - return absl::OkStatus(); - case kTfLiteActRelu: - case kTfLiteActReluN1To1: - case kTfLiteActRelu6: { - ReLUAttributes attr; - attr.clip = fused_activation == kTfLiteActRelu - ? 0.0f - : (fused_activation == kTfLiteActReluN1To1 ? 1.0f : 6.0f); - Node* activation_node; - RETURN_IF_ERROR( - NewPassthroughNode(graph, node, outputs[0], &activation_node)); - activation_node->operation.type = ToString(OperationType::RELU); - activation_node->operation.attributes = attr; - return absl::OkStatus(); - } - case kTfLiteActTanh: { - Node* activation_node; - RETURN_IF_ERROR( - NewPassthroughNode(graph, node, outputs[0], &activation_node)); - activation_node->operation.type = ToString(OperationType::TANH); - return absl::OkStatus(); - } - case kTfLiteActSigmoid: { - Node* activation_node; - RETURN_IF_ERROR( - NewPassthroughNode(graph, node, outputs[0], &activation_node)); - activation_node->operation.type = ToString(OperationType::SIGMOID); - return absl::OkStatus(); - } break; - default: - return absl::NotFoundError( - absl::StrCat("Unsupported fused activation: ", fused_activation)); - } -} - HW ToHW(int32_t h, int32_t w) { return HW(h > 0 ? h : 1, w > 0 ? w : 1); } template diff --git a/tensorflow/lite/delegates/gpu/common/model_builder_helper.cc b/tensorflow/lite/delegates/gpu/common/model_builder_helper.cc index 453e33ec916..b030fb7e700 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder_helper.cc +++ b/tensorflow/lite/delegates/gpu/common/model_builder_helper.cc @@ -25,12 +25,37 @@ limitations under the License. #include "tensorflow/lite/context.h" #include "tensorflow/lite/context_util.h" #include "tensorflow/lite/delegates/gpu/common/model.h" +#include "tensorflow/lite/delegates/gpu/common/operations.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/utils.h" #include "tensorflow/lite/kernels/kernel_util.h" namespace tflite { namespace gpu { +namespace { + +// Creates a node that consumes output from the given node. Because output need +// to stay the same, newly created node will inherit the output from the given +// node, which will in turn get newly created copy of output. This is necessary +// to preserve reference consistency if another node was pointing at that +// output: +// node(output) +// will turn into: +// node(copy(output)) <- passthrough_node(output) +absl::Status NewPassthroughNode(GraphFloat32* graph, Node* node, + const Value* output, Node** passthru_node) { + *passthru_node = graph->NewNode(); + // Make copies for every output in the original node. + RETURN_IF_ERROR(graph->SetProducer((*passthru_node)->id, output->id)); + Value* copy_output = graph->NewValue(); + RETURN_IF_ERROR(graph->SetProducer(node->id, copy_output->id)); + RETURN_IF_ERROR(graph->AddConsumer((*passthru_node)->id, copy_output->id)); + copy_output->tensor = output->tensor; + copy_output->tensor.ref = -1; + return absl::OkStatus(); +} + +} // namespace absl::Status GetNodeAndRegistration(TfLiteContext* context, int node_id, TfLiteNode** tflite_node, @@ -307,5 +332,70 @@ absl::Status SetAllDimensions(const TfLiteIntArray* dimensions, BHWC* shape) { return absl::OkStatus(); } +absl::Status IsActivationSupported(TfLiteFusedActivation fused_activation) { + switch (fused_activation) { + case kTfLiteActNone: + case kTfLiteActRelu: + case kTfLiteActReluN1To1: + case kTfLiteActRelu6: + case kTfLiteActTanh: + case kTfLiteActSigmoid: + return absl::OkStatus(); + case kTfLiteActSignBit: + return absl::UnimplementedError( + "TfLiteFusedActivation.kTfLiteActSignBit"); + + // Do not add default; we want compilation error rather than run-time + // error. + } +} + +// If there is fused activation present, then there will be another node created +// that will have identical output as the given node. New operation node will +// depend on the given node output. +absl::Status MaybeFuseActivation(TfLiteFusedActivation fused_activation, + GraphFloat32* graph, Node* node) { + const auto outputs = graph->FindOutputs(node->id); + if (outputs.size() != 1) { + return absl::InternalError("Number of outputs != 1"); + } + switch (fused_activation) { + case kTfLiteActNone: + // Nothing to do here + return absl::OkStatus(); + case kTfLiteActRelu: + case kTfLiteActReluN1To1: + case kTfLiteActRelu6: { + ReLUAttributes attr; + attr.clip = fused_activation == kTfLiteActRelu + ? 0.0f + : (fused_activation == kTfLiteActReluN1To1 ? 1.0f : 6.0f); + Node* activation_node; + RETURN_IF_ERROR( + NewPassthroughNode(graph, node, outputs[0], &activation_node)); + activation_node->operation.type = ToString(OperationType::RELU); + activation_node->operation.attributes = attr; + return absl::OkStatus(); + } + case kTfLiteActTanh: { + Node* activation_node; + RETURN_IF_ERROR( + NewPassthroughNode(graph, node, outputs[0], &activation_node)); + activation_node->operation.type = ToString(OperationType::TANH); + return absl::OkStatus(); + } + case kTfLiteActSigmoid: { + Node* activation_node; + RETURN_IF_ERROR( + NewPassthroughNode(graph, node, outputs[0], &activation_node)); + activation_node->operation.type = ToString(OperationType::SIGMOID); + return absl::OkStatus(); + } break; + default: + return absl::NotFoundError( + absl::StrCat("Unsupported fused activation: ", fused_activation)); + } +} + } // namespace gpu } // namespace tflite diff --git a/tensorflow/lite/delegates/gpu/common/model_builder_helper.h b/tensorflow/lite/delegates/gpu/common/model_builder_helper.h index 064c42ae9ed..849ef049683 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder_helper.h +++ b/tensorflow/lite/delegates/gpu/common/model_builder_helper.h @@ -16,6 +16,10 @@ limitations under the License. #ifndef TENSORFLOW_LITE_DELEGATES_GPU_COMMON_MODEL_BUILDER_HELPER_H_ #define TENSORFLOW_LITE_DELEGATES_GPU_COMMON_MODEL_BUILDER_HELPER_H_ +#include + +#include "absl/strings/str_cat.h" +#include "tensorflow/lite/c/builtin_op_data.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/delegates/gpu/common/data_type.h" #include "tensorflow/lite/delegates/gpu/common/model.h" @@ -118,6 +122,14 @@ absl::Status SetAllDimensions(const TfLiteIntArray* dimensions, OHWI* shape); absl::Status SetAllDimensions(const TfLiteIntArray* dimensions, BHWC* shape); +absl::Status IsActivationSupported(TfLiteFusedActivation fused_activation); + +// If there is fused activation present, then there will be another node created +// that will have identical output as the given node. New operation node will +// depend on the given node output. +absl::Status MaybeFuseActivation(TfLiteFusedActivation fused_activation, + GraphFloat32* graph, Node* node); + } // namespace gpu } // namespace tflite From a947442bbbf2f32d499d2e14052ec82ed9a55dc2 Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Fri, 7 Aug 2020 12:39:22 -0700 Subject: [PATCH 0653/1017] Removed virtual for Compile, so as method generic. PiperOrigin-RevId: 325487679 Change-Id: Ie4ac95290e4e77e763e49cbb08eab958d753b74b --- tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h index 620883f26f4..6fc9a47f075 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h @@ -98,7 +98,7 @@ class GPUOperation { return GetBestWorkGroup(params, kernel_, grid_size_, &work_group_size_); } - virtual absl::Status Compile(const CreationContext& creation_context); + absl::Status Compile(const CreationContext& creation_context); virtual absl::Status PostCompileCheck(const DeviceInfo& device_info) { return absl::OkStatus(); From 51ecfb3061d981bc8b9530e25f7323029029710c Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Fri, 7 Aug 2020 12:45:04 -0700 Subject: [PATCH 0654/1017] Properly inherit closure types in local functions. Add partial support for resolving local functions based on their type annotations. Propagate types into Expr nodes (although these are roots in expression trees). PiperOrigin-RevId: 325488762 Change-Id: Id1754d65bf15b47ca0ef991959d6491c7ebdc118 --- tensorflow/python/autograph/pyct/anno.py | 4 + .../pyct/static_analysis/type_inference.py | 101 ++++++++++++++---- .../static_analysis/type_inference_test.py | 52 ++++++++- 3 files changed, 134 insertions(+), 23 deletions(-) diff --git a/tensorflow/python/autograph/pyct/anno.py b/tensorflow/python/autograph/pyct/anno.py index 90535ffd903..3abee325084 100644 --- a/tensorflow/python/autograph/pyct/anno.py +++ b/tensorflow/python/autograph/pyct/anno.py @@ -35,10 +35,14 @@ import gast class NoValue(enum.Enum): + """Base class for different types of AST annotations.""" def of(self, node, default=None): return getanno(node, self, default=default) + def add_to(self, node, value): + setanno(node, self, value) + def exists(self, node): return hasanno(node, self) diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_inference.py b/tensorflow/python/autograph/pyct/static_analysis/type_inference.py index 755b6c32c64..a5ed40a1e53 100644 --- a/tensorflow/python/autograph/pyct/static_analysis/type_inference.py +++ b/tensorflow/python/autograph/pyct/static_analysis/type_inference.py @@ -31,7 +31,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from typing import Tuple +from typing import Any, Callable, Tuple import gast @@ -187,16 +187,13 @@ class StmtInferrer(gast.NodeVisitor): def visit(self, node): types = super().visit(node) + if __debug__: + self._check_set(types) if types is not None: # TODO(mdan): Normalize by removing subtypes. anno.setanno(node, anno.Static.TYPES, tuple(types)) return types - def visit_FunctionDef(self, node): - # Skip local function definitions. They are analyzed separately. - # TODO(mdan): Don't skip. Analyze side effects instead. - return None - def _check_set(self, value): if value is not None and not isinstance(value, set): raise ValueError('{} method expected to return set, got {}'.format( @@ -300,21 +297,73 @@ class StmtInferrer(gast.NodeVisitor): return types + def visit_FunctionDef(self, node): + f_name = qual_names.QN(node.name) + + if node.decorator_list: + raise NotImplementedError('decorators: {}'.format(node.decorator_list)) + + # TODO(mdan): Use args. + + ret_types = None + if node.returns: + ret_types, _ = self.resolver.res_name( + self.namespace, self.types_in.types, anno.Basic.QN.of(node.returns)) + if __debug__: + self._check_set(ret_types) + + if ret_types is None: + ret_types = {Any} + + fn_types = set() + for rt in ret_types: + fn_types.add(Callable[[Any], rt]) + + self.new_symbols[f_name] = fn_types + # The definition of a function is an expression, hence has no return value. + return None + + def _resolve_typed_callable(self, fn_types, arg_types, keyword_types): + ret_types = set() + for t in fn_types: + + if isinstance(t, Callable): + # Note: these are undocummented - may be version-specific! + # Callable[[x], y]: __args__ are (x, y) + args = t.__args__ + if args: + ret_types.add(args[-1]) + else: + ret_types.add(Any) + else: + raise NotImplementedError('callable type {}'.format(type(t))) + + # Side effects can not be inferred based on type alone. + side_effects = None + return ret_types, side_effects + def visit_Call(self, node): self.visit(node.func) - f_name = anno.getanno(node.func, anno.Basic.QN) - if f_name in self.scope.bound: - # Don't attempt external resolution of local functions. - # TODO(mdan): Use type annotations of the local definition. - return None - + f_name = anno.Basic.QN.of(node.func) arg_types = [self.visit(a) for a in node.args] keyword_types = [self.visit(kw.value) for kw in node.keywords] - ret_type, side_effects = self.resolver.res_call(self.namespace, - self.types_in.types, node, - arg_types, keyword_types) + if f_name in self.scope.bound: + # Local function, use local type definitions, if available. + fn_type = self.types_in.types.get(f_name, None) + if fn_type is None: + # No static type info available, nothing more to do. + ret_type, side_effects = None, None + else: + ret_type, side_effects = self._resolve_typed_callable( + self.types_in.types.get(f_name), arg_types, keyword_types) + + else: + # Nonlocal function, resolve externally. + ret_type, side_effects = self.resolver.res_call(self.namespace, + self.types_in.types, node, + arg_types, keyword_types) if __debug__: self._check_set(ret_type) if side_effects: @@ -330,6 +379,9 @@ class StmtInferrer(gast.NodeVisitor): self.new_symbols.update(side_effects) return ret_type + def visit_Expr(self, node): + return self.visit(node.value) + def visit_Index(self, node): return self.visit(node.value) @@ -406,15 +458,24 @@ class Analyzer(cfg.GraphVisitor): self.scope = scope self.closure_types = closure_types + context_types = { + n: t for n, t in closure_types.items() if n not in scope.bound + } + if context_types: + self.context_types = _SymbolTable() + self.context_types.types = context_types + else: + self.context_types = None + def init_state(self, _): return _SymbolTable() def _update_closure_types(self, ast_node, types): - existing_types = anno.getanno(ast_node, anno.Static.CLOSURE_TYPES, None) + existing_types = anno.Static.CLOSURE_TYPES.of(ast_node, None) if existing_types is None: existing_types = {} - anno.setanno(ast_node, anno.Static.CLOSURE_TYPES, existing_types) + anno.Static.CLOSURE_TYPES.add_to(ast_node, existing_types) for k, v in types.types.items(): if k in existing_types: @@ -428,6 +489,8 @@ class Analyzer(cfg.GraphVisitor): types_in = _SymbolTable() for n in node.prev: types_in |= self.out[n] + if (self.context_types is not None) and (node is self.graph.entry): + types_in |= self.context_types types_out = _SymbolTable(types_in) ast_node = node.ast_node @@ -437,8 +500,8 @@ class Analyzer(cfg.GraphVisitor): inferrer.visit(ast_node) types_out.types.update(inferrer.new_symbols) - reaching_fndefs = anno.getanno(ast_node, anno.Static.DEFINED_FNS_IN) - node_scope = anno.getanno(ast_node, anno.Static.SCOPE, None) + reaching_fndefs = anno.Static.DEFINED_FNS_IN.of(ast_node) + node_scope = anno.Static.SCOPE.of(ast_node, None) if node_scope is not None: # TODO(mdan): Check that it's actually safe to skip nodes without scope. reads = {str(qn) for qn in node_scope.read} diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_inference_test.py b/tensorflow/python/autograph/pyct/static_analysis/type_inference_test.py index de71854d4fe..ae54cd98b25 100644 --- a/tensorflow/python/autograph/pyct/static_analysis/type_inference_test.py +++ b/tensorflow/python/autograph/pyct/static_analysis/type_inference_test.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from typing import Any, Callable + from tensorflow.python.autograph.pyct import anno from tensorflow.python.autograph.pyct import cfg from tensorflow.python.autograph.pyct import qual_names @@ -33,7 +35,10 @@ class BasicTestResolver(type_inference.Resolver): """A very basic resolver for testing.""" def res_name(self, ns, types_ns, name): - return {type(ns[str(name)])}, ns[str(name)] + str_name = str(name) + if str_name == 'int': + return {int}, int + return {type(ns[str_name])}, ns[str_name] def res_value(self, ns, value): return {type(value)} @@ -72,7 +77,9 @@ class TypeInferenceAnalyzerTest(test.TestCase): def assertClosureTypes(self, node, expected): actual = anno.getanno(node, anno.Static.CLOSURE_TYPES) actual = {str(k): v for k, v in actual.items()} - self.assertDictEqual(actual, expected) + for k, v in expected.items(): + self.assertIn(k, actual) + self.assertEqual(actual[k], v) def test_no_inference_on_unknown_operand_types(self): @@ -188,10 +195,11 @@ class TypeInferenceAnalyzerTest(test.TestCase): node, _ = TestTranspiler(Resolver).transform(test_fn, None) fn_body = node.body - self.assertTypes(fn_body[0].value, int) - self.assertTypes(fn_body[0].value.func, str) self.assertEqual( anno.getanno(fn_body[0].value.func, anno.Static.VALUE), tc.a) + self.assertTypes(fn_body[0].value.func, str) + self.assertTypes(fn_body[0].value, int) + self.assertTypes(fn_body[0], int) def test_assign_overwriting(self): @@ -463,6 +471,26 @@ class TypeInferenceAnalyzerTest(test.TestCase): self.assertTypes(fn_body[0].body[0].value, 'int') self.assertClosureTypes(fn_body[0], {'x': {'int'}}) + def test_local_function_closure_nested(self): + + def test_fn(x: int): + + def foo(): + + def bar(): + return x + + bar() + + foo() + + node, _ = TestTranspiler(BasicTestResolver).transform(test_fn, None) + fn_body = node.body + + self.assertTypes(fn_body[0].body[0].body[0].value, 'int') + self.assertClosureTypes(fn_body[0], {'x': {'int'}}) + self.assertClosureTypes(fn_body[0].body[0], {'x': {'int'}}) + def test_local_function_closure_mutable_var(self): def test_fn(x: int): @@ -512,6 +540,22 @@ class TypeInferenceAnalyzerTest(test.TestCase): self.assertTypes(fn_body[1].targets[0], float) self.assertClosureTypes(fn_body[0], {'x': {float}}) + def test_local_function_type(self): + + def test_fn(x: int): + + def foo() -> int: + return x + + foo() + + node, _ = TestTranspiler(BasicTestResolver).transform(test_fn, None) + fn_body = node.body + + self.assertTypes(fn_body[1].value.func, Callable[[Any], int]) + self.assertTypes(fn_body[1].value, int) + self.assertTypes(fn_body[1], int) + def test_side_effects_on_arg_function_closure(self): test_self = self From cee1115ed5cd52a3c58fef0fdf21042e1195ff3a Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Fri, 7 Aug 2020 12:54:35 -0700 Subject: [PATCH 0655/1017] Internal change PiperOrigin-RevId: 325490761 Change-Id: Ice7ba20990a41d04271253e037002a379fc01335 --- .../gpu/cl/kernels/mean_stddev_normalization.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc index bf2ae33ec6d..ec775861da7 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.cc @@ -86,8 +86,11 @@ std::string MeanStdDevNormalization::GetNormalizationCode() { std::string c = GetCommonDefines(definition_.precision); c += GetVectorReduceCode(); c += GetReduceCode(work_group_size_.x, work_group_size_.y); - c += R"(__attribute__((reqd_work_group_size(128, 1, 1))) -__kernel void main_function( + c += "__attribute__((reqd_work_group_size(" + + std::to_string(work_group_size_.x) + ", " + + std::to_string(work_group_size_.y) + ", " + + std::to_string(work_group_size_.z) + ")))\n"; + c += R"(__kernel void main_function( $0) { #ifndef __opencl_c_work_group_collective_functions __local float tmp[)" + @@ -130,7 +133,7 @@ $0) { const float variance = sum_diff_sq / args.src_tensor.Channels(); const float stddev_inv = rsqrt(variance + 1.0e-8f); // Calculate (t-mean)/stddev for each element - for (int S = 0; S < args.src_tensor.Slices(); ++S) { + for (int S = get_local_id(0); S < args.src_tensor.Slices(); S += get_local_size(0)) { const float4 t = args.src_tensor.Read(0, 0, S, B); FLT4 result = TO_FLT4((t - mean) * stddev_inv); args.dst_tensor.Write(result, 0, 0, S, B); From 0a7a6220981cedb1cdaf858a563e73aeae90543b Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Fri, 7 Aug 2020 12:58:48 -0700 Subject: [PATCH 0656/1017] KernelInfo separated from kernel. GPUOperation::PostCompileCheck don't need CLKernel(API specific) now and uses KernelInfo(API neutral). PiperOrigin-RevId: 325491634 Change-Id: I6eb07614d7d2044dce34d61adaf63d606581fc21 --- tensorflow/lite/delegates/gpu/cl/cl_kernel.cc | 10 +-- tensorflow/lite/delegates/gpu/cl/cl_kernel.h | 12 +-- .../delegates/gpu/cl/kernels/gpu_operation.cc | 2 +- .../delegates/gpu/cl/kernels/gpu_operation.h | 3 +- .../lite/delegates/gpu/cl/kernels/winograd.cc | 4 +- .../gpu/cl/kernels/work_group_picking.cc | 78 +++++++++++-------- 6 files changed, 60 insertions(+), 49 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/cl_kernel.cc b/tensorflow/lite/delegates/gpu/cl/cl_kernel.cc index c498c14dfe8..7a8aaf6102f 100644 --- a/tensorflow/lite/delegates/gpu/cl/cl_kernel.cc +++ b/tensorflow/lite/delegates/gpu/cl/cl_kernel.cc @@ -58,8 +58,7 @@ absl::Status GetKernelPrivateMemorySize(cl_kernel kernel, } // namespace CLKernel::CLKernel(CLKernel&& kernel) - : private_memory_size_(kernel.private_memory_size_), - max_work_group_size_(kernel.max_work_group_size_), + : info_(kernel.info_), binding_counter_(kernel.binding_counter_), function_name_(std::move(kernel.function_name_)), program_(kernel.program_), @@ -70,8 +69,7 @@ CLKernel::CLKernel(CLKernel&& kernel) CLKernel& CLKernel::operator=(CLKernel&& kernel) { if (this != &kernel) { Release(); - std::swap(private_memory_size_, kernel.private_memory_size_); - std::swap(max_work_group_size_, kernel.max_work_group_size_); + std::swap(info_, kernel.info_); std::swap(binding_counter_, kernel.binding_counter_); function_name_ = std::move(kernel.function_name_); std::swap(program_, kernel.program_); @@ -119,9 +117,9 @@ absl::Status CLKernel::CreateFromProgram(const CLProgram& program, clRetainProgram(program_); RETURN_IF_ERROR(GetKernelPrivateMemorySize(kernel_, program.GetDeviceId(), - &private_memory_size_)); + &info_.private_memory_size)); RETURN_IF_ERROR(GetKernelMaxWorkGroupSize(kernel_, program.GetDeviceId(), - &max_work_group_size_)); + &info_.max_work_group_size)); return absl::OkStatus(); } diff --git a/tensorflow/lite/delegates/gpu/cl/cl_kernel.h b/tensorflow/lite/delegates/gpu/cl/cl_kernel.h index 81a777ed822..0af8052f738 100644 --- a/tensorflow/lite/delegates/gpu/cl/cl_kernel.h +++ b/tensorflow/lite/delegates/gpu/cl/cl_kernel.h @@ -28,6 +28,11 @@ namespace tflite { namespace gpu { namespace cl { +struct KernelInfo { + int private_memory_size; + int max_work_group_size; +}; + // Arguments binding to CLKernel can be manual or automatic // In manual you specify binding index explicitly // In automatic binding, index auto-incremented with every binding call @@ -61,9 +66,6 @@ class CLKernel { return SetBytesAuto(static_cast(&value), sizeof(T)); } - int GetPrivateMemorySize() const { return private_memory_size_; } - int GetMaxWorkGroupSize() const { return max_work_group_size_; } - int GetBindingCounter() const { return binding_counter_; } void ResetBindingCounter() { binding_counter_ = 0; } @@ -71,13 +73,13 @@ class CLKernel { // workaround for Mali memory leak absl::Status ReInit() const; + KernelInfo info_; + private: void Release(); absl::Status SetBytes(int index, const void* ptr, int length) const; absl::Status SetBytesAuto(const void* ptr, int length); - int private_memory_size_; - int max_work_group_size_; int binding_counter_ = -1; std::string function_name_; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc index 97c72c1269d..0aa1842791f 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc @@ -244,7 +244,7 @@ absl::Status GPUOperation::Compile(const CreationContext& creation_context) { code_, "main_function", compiler_options_, *creation_context.context, *creation_context.device, &kernel_)); } - return PostCompileCheck(creation_context.device->info_); + return PostCompileCheck(creation_context.device->info_, kernel_.info_); } int3 GPUOperation::GetGridSize() const { diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h index 6fc9a47f075..ba266f8dcc9 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h @@ -100,7 +100,8 @@ class GPUOperation { absl::Status Compile(const CreationContext& creation_context); - virtual absl::Status PostCompileCheck(const DeviceInfo& device_info) { + virtual absl::Status PostCompileCheck(const DeviceInfo& device_info, + const KernelInfo& kernel_info) { return absl::OkStatus(); } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc b/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc index 698599a5bbd..c77c805a712 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc @@ -263,7 +263,7 @@ int3 Winograd4x4To36::SelectBestWorkGroup() { const std::vector wgs = {{8, 6, 4}, {8, 6, 2}, {4, 6, 2}, {4, 6, 2}, {2, 6, 2}, {2, 6, 1}, {1, 6, 1}, {1, 3, 1}, {1, 1, 1}}; - return GetFirstSuitableWorkGroup(wgs, kernel_.GetMaxWorkGroupSize()); + return GetFirstSuitableWorkGroup(wgs, kernel_.info_.max_work_group_size); } absl::Status Winograd4x4To36::BindArguments() { @@ -465,7 +465,7 @@ int3 Winograd36To4x4::SelectBestWorkGroup() { const std::vector wgs = {{32, 4, 2}, {16, 4, 2}, {16, 4, 1}, {8, 4, 1}, {4, 4, 1}, {2, 4, 1}, {1, 4, 1}, {1, 2, 1}, {1, 1, 1}}; - return GetFirstSuitableWorkGroup(wgs, kernel_.GetMaxWorkGroupSize()); + return GetFirstSuitableWorkGroup(wgs, kernel_.info_.max_work_group_size); } absl::Status Winograd36To4x4::BindArguments() { diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc index e85e20761e3..9a1a24895bf 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc @@ -34,21 +34,22 @@ std::vector Get2DWorkgroupsEqualTo128() { } std::vector GenerateWorkGroupSizesXY128( - int3 grid, int max_work_group_size, WorkGroupSizeAlignment z_alignment) { + int3 grid, const KernelInfo& kernel_info, + WorkGroupSizeAlignment z_alignment) { std::vector work_groups; work_groups.reserve(32); std::vector possible_z_sizes = GetPossibleSizes(grid.z, z_alignment); - for (int x = 1; x <= max_work_group_size; x *= 2) { - for (int y = 1; y <= max_work_group_size; y *= 2) { + for (int x = 1; x <= kernel_info.max_work_group_size; x *= 2) { + for (int y = 1; y <= kernel_info.max_work_group_size; y *= 2) { int work_group_size_xy = x * y; if (work_group_size_xy % 128 != 0 || - work_group_size_xy > max_work_group_size) { + work_group_size_xy > kernel_info.max_work_group_size) { continue; } for (auto z : possible_z_sizes) { - if (work_group_size_xy * z > max_work_group_size) { + if (work_group_size_xy * z > kernel_info.max_work_group_size) { continue; } work_groups.push_back({x, y, z}); @@ -59,15 +60,17 @@ std::vector GenerateWorkGroupSizesXY128( } std::vector GenerateWorkGroupSizesXY128Linear( - int3 grid, int max_work_group_size, WorkGroupSizeAlignment z_alignment) { + int3 grid, const KernelInfo& kernel_info, + WorkGroupSizeAlignment z_alignment) { std::vector work_groups; work_groups.reserve(32); std::vector possible_z_sizes = GetPossibleSizes(grid.z, z_alignment); - for (int x = 128; x <= max_work_group_size && x < grid.x + 128; x += 128) { + for (int x = 128; x <= kernel_info.max_work_group_size && x < grid.x + 128; + x += 128) { for (auto z : possible_z_sizes) { - if (x * z <= max_work_group_size) { + if (x * z <= kernel_info.max_work_group_size) { work_groups.push_back({x, 1, z}); } } @@ -75,22 +78,15 @@ std::vector GenerateWorkGroupSizesXY128Linear( return work_groups; } -absl::Status GetBestWorkGroupAlignedToGrid(const TuningParameters& params, - const CLKernel& kernel, - const int3& grid, - int3* best_work_group) { - std::vector work_groups; +void GetWorkGroupsAlignedToGrid(const DeviceInfo& device_info, + const KernelInfo& kernel_info, const int3& grid, + std::vector* work_groups) { int3 max_wg_size; - max_wg_size.x = params.info->max_work_group_size_x; - max_wg_size.y = params.info->max_work_group_size_y; - max_wg_size.z = params.info->max_work_group_size_z; + max_wg_size.x = device_info.max_work_group_size_x; + max_wg_size.y = device_info.max_work_group_size_y; + max_wg_size.z = device_info.max_work_group_size_z; GenerateWorkGroupSizesAlignedToGrid( - grid, max_wg_size, kernel.GetMaxWorkGroupSize(), &work_groups); - int best_work_group_index; - RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( - kernel, *params.info, grid, work_groups, &best_work_group_index)); - *best_work_group = work_groups[best_work_group_index]; - return absl::OkStatus(); + grid, max_wg_size, kernel_info.max_work_group_size, work_groups); } int GetPenalty(int grid_size, int group_size) { @@ -210,8 +206,8 @@ absl::Status GetBestWorkGroupXY128(const TuningParameters& params, const CLKernel& kernel, const int3& grid, WorkGroupSizeAlignment z_alignment, int3* best_work_group) { - std::vector work_groups = GenerateWorkGroupSizesXY128( - grid, kernel.GetMaxWorkGroupSize(), z_alignment); + std::vector work_groups = + GenerateWorkGroupSizesXY128(grid, kernel.info_, z_alignment); int best_work_group_index; RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( kernel, *params.info, grid, work_groups, &best_work_group_index)); @@ -224,8 +220,8 @@ absl::Status GetBestWorkGroupXY128Linear(const TuningParameters& params, const int3& grid, WorkGroupSizeAlignment z_alignment, int3* best_work_group) { - std::vector work_groups = GenerateWorkGroupSizesXY128Linear( - grid, kernel.GetMaxWorkGroupSize(), z_alignment); + std::vector work_groups = + GenerateWorkGroupSizesXY128Linear(grid, kernel.info_, z_alignment); int best_work_group_index; RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( kernel, *params.info, grid, work_groups, &best_work_group_index)); @@ -254,11 +250,18 @@ absl::Status GetBestWorkGroup(const TuningParameters& params, int3* best_work_group) { switch (params.tuning_type) { case TuningType::FAST: - *best_work_group = GetWorkGroup(grid, kernel.GetMaxWorkGroupSize()); + *best_work_group = GetWorkGroup(grid, kernel.info_.max_work_group_size); return absl::OkStatus(); - case TuningType::EXHAUSTIVE: - return GetBestWorkGroupAlignedToGrid(params, kernel, grid, - best_work_group); + case TuningType::EXHAUSTIVE: { + std::vector work_groups; + GetWorkGroupsAlignedToGrid(*params.info, kernel.info_, grid, + &work_groups); + int best_work_group_index; + RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( + kernel, *params.info, grid, work_groups, &best_work_group_index)); + *best_work_group = work_groups[best_work_group_index]; + return absl::OkStatus(); + } default: *best_work_group = {8, 4, 1}; return absl::OkStatus(); @@ -276,12 +279,19 @@ absl::Status GetBestWorkGroupConv(const TuningParameters& params, } max_z_size = std::min(max_z_size, params.info->max_work_group_size_z); *best_work_group = - GetWorkGroupConv(grid, kernel.GetMaxWorkGroupSize(), max_z_size); + GetWorkGroupConv(grid, kernel.info_.max_work_group_size, max_z_size); + return absl::OkStatus(); + } + case TuningType::EXHAUSTIVE: { + std::vector work_groups; + GetWorkGroupsAlignedToGrid(*params.info, kernel.info_, grid, + &work_groups); + int best_work_group_index; + RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( + kernel, *params.info, grid, work_groups, &best_work_group_index)); + *best_work_group = work_groups[best_work_group_index]; return absl::OkStatus(); } - case TuningType::EXHAUSTIVE: - return GetBestWorkGroupAlignedToGrid(params, kernel, grid, - best_work_group); default: *best_work_group = {8, 4, 1}; return absl::OkStatus(); From 001ec7efbed18e9581e859513c5acc76e5aabbe9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 13:14:43 -0700 Subject: [PATCH 0657/1017] Changes are excluded via copybara PiperOrigin-RevId: 325494931 Change-Id: I69c55142e00f691e6ce1b77557173e28022d4b83 --- tensorflow/g3doc/README.txt | 46 ------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 tensorflow/g3doc/README.txt diff --git a/tensorflow/g3doc/README.txt b/tensorflow/g3doc/README.txt deleted file mode 100644 index 515a9e9a025..00000000000 --- a/tensorflow/g3doc/README.txt +++ /dev/null @@ -1,46 +0,0 @@ -Docs have moved! If you just want to view TensorFlow documentation, -go to: - - https://www.tensorflow.org/ - -Documentation (on Github, tensorflow.org, and anywhere else we decide to -serve it from) is now generated from the files in -tensorflow/docs_src/ (for tutorials and other guides) and -TensorFlow source code (for the API reference pages). If you see a problem with -API reference, edit the code comments in the appropriate language. If you see a -problem with our other docs, edit the files in docs_src. - -To preview the results of your changes, or generate an offline copy of -the docs, run: - - bazel run -- tensorflow/tools/docs:generate \ - --src_dir=/path/to/tensorflow/docs_src/ \ - --output_dir=/tmp/tfdocs/ - -`src_dir` must be absolute path to documentation source. -When authoring docs, note that we have some new syntax for references -- -at least for docs coming from Python docstrings or -tensorflow/docs_src/. Use: - -* `tf.symbol` to make a link to the reference page for a Python - symbol. Note that class members don't get their own page, but the - syntax still works, since `tf.MyClass.method` links to the right - part of the tf.MyClass page. - -* `tensorflow::symbol` to make a link to the reference page for a C++ - symbol. (This only works for a few symbols but will work for more soon.) - -* @{$doc_page} to make a link to another (not an API reference) doc - page. To link to - - red/green/blue/index.md use @{$blue} or @{$green/blue}, - - foo/bar/baz.md use @{$baz} or @{$bar/baz}. - The shorter one is preferred, so we can move pages around without - breaking these references. The main exception is that the Python API - guides should probably be referred to using @{$python/} - to avoid ambiguity. To link to an anchor in that doc and use - different link text (by default it uses the title of the target - page) use: - @{$doc_page#anchor-tag$link-text} - (You can skip #anchor-tag if you just want to override the link text). - -Thanks! From 874db4d37159c8d67a1df5cc978dda2b783c6476 Mon Sep 17 00:00:00 2001 From: Cesar Crusius Date: Fri, 7 Aug 2020 13:15:49 -0700 Subject: [PATCH 0658/1017] Allows experimental loading of a model in C++ with unsupported features. PiperOrigin-RevId: 325495136 Change-Id: Ie6fd62d6826abeb6d8fa3291c666dba37fc4ca72 --- .../c/experimental/saved_model/core/tf_saved_model_api.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow/c/experimental/saved_model/core/tf_saved_model_api.cc b/tensorflow/c/experimental/saved_model/core/tf_saved_model_api.cc index c22f8d86174..0f0102be857 100644 --- a/tensorflow/c/experimental/saved_model/core/tf_saved_model_api.cc +++ b/tensorflow/c/experimental/saved_model/core/tf_saved_model_api.cc @@ -47,6 +47,7 @@ limitations under the License. #include "tensorflow/core/lib/hash/hash.h" #include "tensorflow/core/platform/casts.h" #include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/path.h" #include "tensorflow/core/platform/stringpiece.h" @@ -241,8 +242,11 @@ Status RestoreCheckpoint(SavedModelV2Bundle* bundle, // TODO(bmzhao): This requires using the newly added Save/Restore // functions from // https://github.com/tensorflow/tensorflow/commit/df6b21c13c82b5d0981642cfe18f10e60f78ea5c - return errors::Unimplemented( - "Restoring non-variable objects has not been implemented yet. "); + LOG(WARNING) << "Restoring non-variable objects has not been " + "implemented yet. (Kind=" + << bundle->saved_object_graph().nodes(node).kind_case() + << ")"; + return Status::OK(); } Variable* variable = From bf6f488c1102a88252960c4c8b90f6122e966b14 Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Fri, 7 Aug 2020 13:22:02 -0700 Subject: [PATCH 0659/1017] Mark Interpreter:UseNNAPI(bool) deprecated Prefer using the NnApiDelegate() API directly. The current API is unreliable and causes issues with the first inference execution. PiperOrigin-RevId: 325496322 Change-Id: I44c8fc04bcd08ce5b92e22cd170e075c0abbaecf --- RELEASE.md | 2 ++ tensorflow/lite/c/BUILD | 1 + tensorflow/lite/c/c_api.cc | 10 ++++++++-- .../gpu/common/testing/tflite_model_reader.cc | 1 - tensorflow/lite/interpreter.cc | 7 ++++++- tensorflow/lite/interpreter.h | 2 ++ 6 files changed, 19 insertions(+), 4 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index bb4d29f9020..f0c590710cf 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -111,6 +111,8 @@ string to be joined is empty. * `TFLiteConverter`: * Support optional flags `inference_input_type` and `inference_output_type` for full integer quantized models. This allows users to modify the model input and output type to integer types (`tf.int8`, `tf.uint8`) instead of defaulting to float type (`tf.float32`). + * Deprecate `Interpreter::UseNNAPI(bool)` C++ API + * Prefer using `NnApiDelegate()` and related delegate configuration methods directly. * * `tf.random`: * diff --git a/tensorflow/lite/c/BUILD b/tensorflow/lite/c/BUILD index bdf86d7904f..5ac6d7881ac 100644 --- a/tensorflow/lite/c/BUILD +++ b/tensorflow/lite/c/BUILD @@ -62,6 +62,7 @@ cc_library( "//tensorflow/lite:framework", "//tensorflow/lite:version", "//tensorflow/lite/core/api", + "//tensorflow/lite/delegates/nnapi:nnapi_delegate", "//tensorflow/lite/kernels:builtin_ops", ], alwayslink = 1, diff --git a/tensorflow/lite/c/c_api.cc b/tensorflow/lite/c/c_api.cc index aa93a10302c..4afd413ba9c 100644 --- a/tensorflow/lite/c/c_api.cc +++ b/tensorflow/lite/c/c_api.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include "tensorflow/lite/c/c_api_internal.h" +#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h" #include "tensorflow/lite/error_reporter.h" #include "tensorflow/lite/interpreter.h" #include "tensorflow/lite/kernels/register.h" @@ -123,13 +124,18 @@ TfLiteInterpreter* TfLiteInterpreterCreate( } if (optional_options) { - interpreter->UseNNAPI(optional_options->use_nnapi); - if (optional_options->num_threads != TfLiteInterpreterOptions::kDefaultNumThreads) { interpreter->SetNumThreads(optional_options->num_threads); } + if (optional_options->use_nnapi) { + if (interpreter->ModifyGraphWithDelegate(tflite::NnApiDelegate()) != + kTfLiteOk) { + return nullptr; + } + } + for (auto* delegate : optional_options->delegates) { if (interpreter->ModifyGraphWithDelegate(delegate) != kTfLiteOk) { return nullptr; diff --git a/tensorflow/lite/delegates/gpu/common/testing/tflite_model_reader.cc b/tensorflow/lite/delegates/gpu/common/testing/tflite_model_reader.cc index 0faa621f72f..a67602cf245 100644 --- a/tensorflow/lite/delegates/gpu/common/testing/tflite_model_reader.cc +++ b/tensorflow/lite/delegates/gpu/common/testing/tflite_model_reader.cc @@ -79,7 +79,6 @@ absl::Status BuildFromFlatBuffer(const tflite::FlatBufferModel& flatbuffer, if (interpreter_builder(&interpreter) != kTfLiteOk || !interpreter) { return absl::InternalError("Unable to prepare TfLite interpreter."); } - interpreter->UseNNAPI(false); TfLiteDelegate delegate; delegate.data_ = graph; delegate.flags = kTfLiteDelegateFlagsNone; diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc index 88dcb37898a..62de5896d84 100644 --- a/tensorflow/lite/interpreter.cc +++ b/tensorflow/lite/interpreter.cc @@ -300,7 +300,12 @@ TfLiteStatus Interpreter::SetExecutionPlan(const std::vector& new_plan) { return primary_subgraph().SetExecutionPlan(new_plan); } -void Interpreter::UseNNAPI(bool enable) { primary_subgraph().UseNNAPI(enable); } +void Interpreter::UseNNAPI(bool enable) { + TFLITE_LOG_PROD_ONCE(TFLITE_LOG_INFO, + "Interpreter::UseNNAPI() is deprecated. Use " + "tflite::NnApiDelegate() directly instead."); + primary_subgraph().UseNNAPI(enable); +} TfLiteStatus Interpreter::SetNumThreads(int num_threads) { if (num_threads < -1) { diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h index 653283bc234..b9e2045cd96 100644 --- a/tensorflow/lite/interpreter.h +++ b/tensorflow/lite/interpreter.h @@ -365,6 +365,8 @@ class Interpreter { TfLiteStatus Invoke(); /// Enable or disable the NN API (true to enable) + /// NOTE: This API is deprecated, prefer using the NNAPI delegate directly. + /// This method will be removed in a future release. void UseNNAPI(bool enable); /// Set the number of threads available to the interpreter. From 7197362d5ae9a8b7461e93064ce646b40c1eb9e5 Mon Sep 17 00:00:00 2001 From: Jose Baiocchi Date: Fri, 7 Aug 2020 13:28:37 -0700 Subject: [PATCH 0660/1017] Avoid a copy in AnnotatedTraceMe PiperOrigin-RevId: 325497419 Change-Id: I2d6bee06b6037089ab388331b8942bcd9a96addc --- tensorflow/core/profiler/lib/annotated_traceme.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/profiler/lib/annotated_traceme.h b/tensorflow/core/profiler/lib/annotated_traceme.h index 636b901e226..eb75a896107 100644 --- a/tensorflow/core/profiler/lib/annotated_traceme.h +++ b/tensorflow/core/profiler/lib/annotated_traceme.h @@ -43,7 +43,7 @@ class AnnotatedTraceMe { scoped_annotation_.emplace(absl::string_view(name)); } if (TF_PREDICT_TRUE(traceme_enabled)) { - trace_me_.emplace([name = std::move(name)] { return name; }, level); + trace_me_.emplace([&name] { return std::move(name); }, level); } } } From 247e9bd050e68fa4b055fe6c99144def3fde4e81 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Fri, 7 Aug 2020 13:32:59 -0700 Subject: [PATCH 0661/1017] Treat Case similar to If/While wrt stateless variant On import dedup to tf.Case and on export expand to either Case or StatelessCase depending on variant. Kept it mechanical to the other two control flow ops here. PiperOrigin-RevId: 325498204 Change-Id: Icf5f6f580510908d7dd7c043ac287b19862eaa02 --- .../mlir/tensorflow/ir/tf_generated_ops.td | 42 --- .../compiler/mlir/tensorflow/ir/tf_ops.td | 45 +++ .../tests/graphdef2mlir/case_op.pbtxt | 261 ++++++++++++++++++ .../tensorflow/tests/mlir2graphdef/case.mlir | 38 +++ .../mlir/tensorflow/translate/import_model.cc | 24 +- .../mlir/tensorflow/utils/export_utils.cc | 25 +- 6 files changed, 361 insertions(+), 74 deletions(-) create mode 100644 tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/case_op.pbtxt create mode 100644 tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/case.mlir diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index 081903d13cf..bf8d7015b46 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -1350,48 +1350,6 @@ then the output will be TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<0>; } -def TF_CaseOp : TF_Op<"Case", []> { - let summary = [{ -An n-way switch statement which calls a single branch function. - }]; - - let description = [{ -An n-way switch statement, implementing the following: - ``` - switch (branch_index) { - case 0: - output = branches[0](input); - break; - case 1: - output = branches[1](input); - break; - ... - case [[nbranches-1]]: - default: - output = branches[nbranches-1](input); - break; - } - ``` - }]; - - let arguments = (ins - I32Tensor:$branch_index, - Variadic:$input, - - Confined]>:$branches, - DefaultValuedAttr:$output_shapes - ); - - let results = (outs - Variadic:$output - ); - - TF_DerivedOperandTypeListAttr Tin = TF_DerivedOperandTypeListAttr<1>; - TF_DerivedResultTypeListAttr Tout = TF_DerivedResultTypeListAttr<0>; - - let hasCanonicalizer = 1; -} - def TF_CastOp : TF_Op<"Cast", [NoSideEffect, SameOperandsAndResultShape]> { let summary = "Cast x of type SrcT to y of DstT."; diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td index 376b7933b47..5269bb82239 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.td @@ -68,6 +68,51 @@ class TF_TensorListInitOp : TF_Op { }]; } +def TF_CaseOp : TF_Op<"Case", []> { + let summary = [{ +An n-way switch statement which calls a single branch function. + }]; + + let description = [{ +An n-way switch statement, implementing the following: + ``` + switch (branch_index) { + case 0: + output = branches[0](input); + break; + case 1: + output = branches[1](input); + break; + ... + case [[nbranches-1]]: + default: + output = branches[nbranches-1](input); + break; + } + ``` + }]; + + let arguments = (ins + I32Tensor:$branch_index, + Variadic:$input, + + Confined]>:$branches, + DefaultValuedAttr:$output_shapes, + + // Used to map StatelessCase and Case to a common op. + DefaultValuedAttr:$is_stateless + ); + + let results = (outs + Variadic:$output + ); + + TF_DerivedOperandTypeListAttr Tin = TF_DerivedOperandTypeListAttr<1>; + TF_DerivedResultTypeListAttr Tout = TF_DerivedResultTypeListAttr<0>; + + let hasCanonicalizer = 1; +} + // In MLIR, the TensorFlow tensor value is represented as an ElementsAttr, with // its type encoding the tensor's shape and data type. def TF_ConstOp : TF_Op<"Const", [ConstantLike, NoSideEffect, diff --git a/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/case_op.pbtxt b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/case_op.pbtxt new file mode 100644 index 00000000000..1372ad71283 --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/tests/graphdef2mlir/case_op.pbtxt @@ -0,0 +1,261 @@ +# RUN: tf-mlir-translate -graphdef-to-splatted-mlir %s -o - | FileCheck %s + +node { + name: "Const" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } +} +node { + name: "Const_1" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 0 + } + } + } +} +node { + name: "indexed_case" + op: "StatelessCase" + input: "Const_1" + input: "Const" + attr { + key: "Tin" + value { + list { + type: DT_INT32 + } + } + } + attr { + key: "Tout" + value { + list { + type: DT_INT32 + } + } + } + attr { + key: "_lower_using_switch_merge" + value { + b: true + } + } + attr { + key: "_read_only_resource_inputs" + value { + list { + } + } + } + attr { + key: "branches" + value { + list { + func { + name: "indexed_case_branch0_4" + } + func { + name: "indexed_case_branch1_5" + } + } + } + } + attr { + key: "output_shapes" + value { + list { + shape { + } + } + } + } +} +node { + name: "indexed_case/Identity" + op: "Identity" + input: "indexed_case" + attr { + key: "T" + value { + type: DT_INT32 + } + } +} +library { + function { + signature { + name: "indexed_case_branch0_4" + input_arg { + name: "add_const" + type: DT_INT32 + } + output_arg { + name: "add" + type: DT_INT32 + } + } + node_def { + name: "add/y" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 1 + } + } + } + experimental_debug_info { + original_node_names: "add/y" + } + } + node_def { + name: "add_0" + op: "AddV2" + input: "add_const" + input: "add/y:output:0" + attr { + key: "T" + value { + type: DT_INT32 + } + } + experimental_debug_info { + original_node_names: "add" + } + } + ret { + key: "add" + value: "add_0:z:0" + } + arg_attr { + key: 0 + value { + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + } + } + function { + signature { + name: "indexed_case_branch1_5" + input_arg { + name: "add_const" + type: DT_INT32 + } + output_arg { + name: "add" + type: DT_INT32 + } + } + node_def { + name: "add/y" + op: "Const" + attr { + key: "dtype" + value { + type: DT_INT32 + } + } + attr { + key: "value" + value { + tensor { + dtype: DT_INT32 + tensor_shape { + } + int_val: 2 + } + } + } + experimental_debug_info { + original_node_names: "add/y" + } + } + node_def { + name: "add_0" + op: "AddV2" + input: "add_const" + input: "add/y:output:0" + attr { + key: "T" + value { + type: DT_INT32 + } + } + experimental_debug_info { + original_node_names: "add" + } + } + ret { + key: "add" + value: "add_0:z:0" + } + arg_attr { + key: 0 + value { + attr { + key: "_output_shapes" + value { + list { + shape { + } + } + } + } + } + } + } +} +versions { + producer: 486 + min_consumer: 12 +} + +# CHECK: tf.Case +# CHECK-SAME: is_stateless diff --git a/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/case.mlir b/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/case.mlir new file mode 100644 index 00000000000..2f2ee6f1286 --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/tests/mlir2graphdef/case.mlir @@ -0,0 +1,38 @@ +// RUN: tf-mlir-translate -mlir-to-graphdef %s -o - | FileCheck %s + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 12 : i32, producer = 486 : i32}} { + func @main() { + tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.Const"() {device = "", value = dense<0> : tensor} : () -> tensor + %outputs_2, %control_3 = tf_executor.island wraps "tf.Case"(%outputs_0, %outputs) {Tin = [i32], Tout = [i32], _lower_using_switch_merge = true, _read_only_resource_inputs = [], branches = [@indexed_case_branch0_40, @indexed_case_branch1_50], device = "", is_stateless = true, output_shapes = [#tf.shape<>]} : (tensor, tensor) -> tensor<*xi32> loc("stateless_case") + %outputs_4, %control_5 = tf_executor.island wraps "tf.Identity"(%outputs_2) {device = ""} : (tensor<*xi32>) -> tensor<*xi32> + %outputs_6, %control_7 = tf_executor.island wraps "tf.Case"(%outputs_0, %outputs) {Tin = [i32], Tout = [i32], _lower_using_switch_merge = true, _read_only_resource_inputs = [], branches = [@indexed_case_branch0_40, @indexed_case_branch1_50], device = "", is_stateless = false, output_shapes = [#tf.shape<>]} : (tensor, tensor) -> tensor<*xi32> loc("regular_case") + tf_executor.fetch + } + return + } + + func @indexed_case_branch0_40(%arg0: tensor) -> tensor<*xi32> attributes {sym_visibility = "private"} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<1> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.AddV2"(%arg0, %outputs) {device = ""} : (tensor, tensor) -> tensor<*xi32> + tf_executor.fetch %outputs_0 : tensor<*xi32> + } + return %0 : tensor<*xi32> + } + + func @indexed_case_branch1_50(%arg0: tensor) -> tensor<*xi32> attributes {sym_visibility = "private"} { + %0 = tf_executor.graph { + %outputs, %control = tf_executor.island wraps "tf.Const"() {device = "", value = dense<2> : tensor} : () -> tensor + %outputs_0, %control_1 = tf_executor.island wraps "tf.AddV2"(%arg0, %outputs) {device = ""} : (tensor, tensor) -> tensor<*xi32> + tf_executor.fetch %outputs_0 : tensor<*xi32> + } + return %0 : tensor<*xi32> + } +} + +// CHECK: name: "stateless_case" +// CHECK-NEXT: "StatelessCase" +// CHECK: name: "regular_case" +// CHECK-NEXT: "Case" diff --git a/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc b/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc index ef0087c4310..94ddf76736e 100644 --- a/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc +++ b/tensorflow/compiler/mlir/tensorflow/translate/import_model.cc @@ -1934,22 +1934,18 @@ Status ImporterBase::ConvertNode(const Node& node) { } } - // Map If and StatelessIf op in TensorFlow to the common If op in MLIR and add - // the differentiating attribute. - if (node.IsIfNode()) { - result.name = mlir::OperationName(get_full_op_name("If"), context_); - mlir::BoolAttr val = builder_.getBoolAttr(node_type_name == "StatelessIf"); + auto composite_control_flow_op = [&](const std::string& name) { + result.name = mlir::OperationName(get_full_op_name(name), context_); + bool stateless = absl::StartsWith(node_type_name, "Stateless"); + mlir::BoolAttr val = builder_.getBoolAttr(stateless); result.attributes.push_back(builder_.getNamedAttr("is_stateless", val)); - } + }; - // Map While and StatelessWhile op in TensorFlow to the common While op in - // MLIR and add the differentiating attribute. - if (node.IsWhileNode()) { - result.name = mlir::OperationName(get_full_op_name("While"), context_); - mlir::BoolAttr val = - builder_.getBoolAttr(node_type_name == "StatelessWhile"); - result.attributes.push_back(builder_.getNamedAttr("is_stateless", val)); - } + // Map Case/If/While and StatelessCase/If/While op in TensorFlow to the common + // Case/If/While op in MLIR and add the differentiating attribute. + if (node.IsCaseNode()) composite_control_flow_op("Case"); + if (node.IsIfNode()) composite_control_flow_op("If"); + if (node.IsWhileNode()) composite_control_flow_op("While"); // Register the mapping between the TF node and the newly created operation. node_values_[node.id()] = diff --git a/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc b/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc index 0364b935b92..ad9ddb277d7 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/export_utils.cc @@ -227,25 +227,13 @@ Status ConvertAttribute(const mlir::ArrayAttr& attr, AttrValue* value) { return Status::OK(); } -// Updates NodeDef constructed out of an MLIR If op to map it to either -// TensorFlow StatelessIf or If op depending on the additional attribute. -void UpdateCompositeIfOp(NodeDef* node_def) { +// Updates NodeDef constructed out of an MLIR Case/IfW/While op to map it to +// either TensorFlow StatelessX or X op depending on the additional attribute. +void UpdateCompositeOp(NodeDef* node_def) { auto it = node_def->mutable_attr()->find("is_stateless"); if (it != node_def->attr().end()) { if (it->second.b()) { - *node_def->mutable_op() = "StatelessIf"; - } - node_def->mutable_attr()->erase(it); - } -} - -// Updates NodeDef constructed out of an MLIR While op to map it to either -// TensorFlow StatelessWhile or While op depending on the additional attribute. -void UpdateCompositeWhileOp(NodeDef* node_def) { - auto it = node_def->mutable_attr()->find("is_stateless"); - if (it != node_def->attr().end()) { - if (it->second.b()) { - *node_def->mutable_op() = "StatelessWhile"; + *node_def->mutable_op() = "Stateless" + node_def->op(); } node_def->mutable_attr()->erase(it); } @@ -352,8 +340,9 @@ StatusOr> GetOperationNodeDef( TF_RETURN_IF_ERROR(ConvertLocation( inst->getLoc(), node_def->mutable_experimental_debug_info())); - if (node_def->op() == "If") UpdateCompositeIfOp(node_def.get()); - if (node_def->op() == "While") UpdateCompositeWhileOp(node_def.get()); + if (node_def->op() == "Case") UpdateCompositeOp(node_def.get()); + if (node_def->op() == "If") UpdateCompositeOp(node_def.get()); + if (node_def->op() == "While") UpdateCompositeOp(node_def.get()); return node_def; } From 5296ad4ffdb2c4ec6fb3a413e91591052d5f8684 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Fri, 7 Aug 2020 13:59:33 -0700 Subject: [PATCH 0662/1017] [Resubmit] If an input-output pair is configured to be must-alias(off by default), they must be aliased at runtime. PiperOrigin-RevId: 325503193 Change-Id: Ida4e46531052c40eebce5f0dff4c50914cc1f3f4 --- .../utils/compile_mlir_util_test.cc | 2 +- tensorflow/compiler/xla/client/xla_builder.cc | 2 +- tensorflow/compiler/xla/client/xla_builder.h | 17 ++++-- .../xla/service/cpu/cpu_executable.cc | 6 ++ .../xla/service/gpu/gpu_executable.cc | 6 ++ tensorflow/compiler/xla/service/hlo.proto | 14 ++++- .../service/hlo_input_output_alias_config.cc | 38 +++++++++--- .../service/hlo_input_output_alias_config.h | 32 +++++++--- tensorflow/compiler/xla/service/hlo_parser.cc | 59 +++++++++++-------- .../compiler/xla/service/hlo_parser_test.cc | 41 ++----------- .../xla/tests/buffer_donation_test.cc | 49 +++++++++++++-- .../tpu/tpu_executable_interface.cc | 18 ++++++ 12 files changed, 194 insertions(+), 90 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc index 6ebf6897bb1..8a07aab11e1 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc @@ -524,7 +524,7 @@ TEST(CompileGraphToXlaHlo, Resources) { ASSERT_TRUE(status_or_hlo_module.ok()); constexpr char expected_hlo_module_string[] = - R"(HloModule main.4, input_output_alias={ {0}: 1 } + R"(HloModule main.4, input_output_alias={ {0}: (1, {}, may_alias) } ENTRY %main.4 (Arg_0.1: f32[2], Arg_1.2: f32[2]) -> (f32[2]) { %Arg_1.2 = f32[2]{0} parameter(1) diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc index 52f61408cbb..484fb0aabe7 100644 --- a/tensorflow/compiler/xla/client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_builder.cc @@ -446,7 +446,7 @@ StatusOr XlaBuilder::Build(int64 root_id, alias.param_index.ToString().c_str()); } TF_RETURN_IF_ERROR(config.SetUpAlias(alias.output_index, alias.param_number, - alias.param_index)); + alias.param_index, alias.kind)); } *module->mutable_input_output_alias() = config.ToProto(); return Status::OK(); diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h index 1960d0c4632..aa5074d28d9 100644 --- a/tensorflow/compiler/xla/client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_builder.h @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/compiler/xla/literal_util.h" #include "tensorflow/compiler/xla/service/dynamic_parameter_binding.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" +#include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/status_macros.h" @@ -349,12 +350,16 @@ class XlaBuilder { // not available until the computation is built, and eventual error in the // arguments of this API will be detected only at computation Build() time. // - // Note: Aliasing API is 'may-alias' and only donated buffer at runtime will - // be aliased with output. If a buffer is not donated at runtime, a copy will - // be inserted by XLA to prevent buffer clobbering. + // Note: Except when 'must-alias' is true, alias is assumed to be 'may-alias' + // and only donated buffer at runtime will be aliased with output. If a buffer + // is not donated at runtime, a copy will be inserted by XLA to prevent buffer + // clobbering. void SetUpAlias(const ShapeIndex& output_index, int64 param_number, - const ShapeIndex& param_index) { - input_output_aliases_.push_back({output_index, param_number, param_index}); + const ShapeIndex& param_index, + HloInputOutputAliasConfig::AliasKind kind = + HloInputOutputAliasConfig::AliasKind::kMayAlias) { + input_output_aliases_.push_back( + {output_index, param_number, param_index, kind}); } // Describes an input/output alias as inserted by the SetUpAlias() API. @@ -365,6 +370,8 @@ class XlaBuilder { int64 param_number; // Specifies the index of the aliased buffer in the parameter ShapeIndex param_index; + // Specifies if the alias is a must alias or may alias. + HloInputOutputAliasConfig::AliasKind kind; }; // Looks up the HloInstruction and sets the frontend attribute "attribute" to diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc index 0abcc91a1d7..7431e829b8e 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_executable.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.cc @@ -247,6 +247,12 @@ StatusOr CpuExecutable::CreateResultShapedBuffer( ExecutionInput& input = arguments[alias->parameter_number]; MaybeOwningDeviceMemory* maybe_owning_memory = input.MutableBuffer(alias->parameter_index); + if (alias->must_alias() && !maybe_owning_memory->HasOwnership()) { + return InvalidArgument( + "An input was configured to be must-alias at " + "compile time but not donated at runtime: %s", + alias->ToString()); + } if (absl::optional owning = maybe_owning_memory->Release()) { // If the caller passes the ownership of the device memory, reuse it diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc index 469f2919fba..726f1963545 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc @@ -480,6 +480,12 @@ StatusOr GpuExecutable::ExecuteAsyncOnStream( ExecutionInput& input = arguments[alias->parameter_number]; MaybeOwningDeviceMemory* maybe_owning_memory = input.MutableBuffer(alias->parameter_index); + if (alias->must_alias() && !maybe_owning_memory->HasOwnership()) { + return InvalidArgument( + "An input was configured to be must-alias at " + "compile time but not donated at runtime: %s", + alias->ToString()); + } if (absl::optional owning = maybe_owning_memory->Release()) { // If the caller passes the ownership of the device memory, reuse it diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto index 960f60fe882..e043216c17e 100644 --- a/tensorflow/compiler/xla/service/hlo.proto +++ b/tensorflow/compiler/xla/service/hlo.proto @@ -283,6 +283,16 @@ message HloScheduleProto { map sequences = 1; } +enum Kind { + // Define a UNDEFINED_ALIAS equal to zero to get around the default-0 proto3 + // behavior and missing has_*() APIs. + UNDEFINED_ALIAS = 0; + // The buffers may or may not alias at runtime. + MAY_ALIAS = 1; + // The buffers must alias at runtime. + MUST_ALIAS = 2; +} + message HloInputOutputAliasProto { // The following proto describes a pair of aliased an input // (described by parameter number and a ShapeIndex of the parameter) @@ -304,8 +314,8 @@ message HloInputOutputAliasProto { int64 parameter_number = 2; // ShapeIndex of the parameter instruction. repeated int64 parameter_shape_index = 3; - reserved 4; - reserved "kind"; + // The kind of alias to be setup. + Kind kind = 4; } repeated AliasEntryProto entries = 1; diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc index e123161720b..34bc30d641f 100644 --- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc +++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h" +#include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_module.h" namespace xla { @@ -24,9 +25,10 @@ bool HloInputOutputAliasConfig::OutputHasAlias( return alias_.element(output_index).has_value(); } -Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index, - int64 param_number, - const ShapeIndex& param_index) { +Status HloInputOutputAliasConfig::SetUpAlias( + const ShapeIndex& output_index, int64 param_number, + const ShapeIndex& param_index, + HloInputOutputAliasConfig::AliasKind must_alias) { TF_RET_CHECK(ShapeUtil::IndexIsValid(alias_.shape(), output_index)) << "Trying to set up alias at " << output_index.ToString() << " which is an invalid index for shape " @@ -41,7 +43,8 @@ Status HloInputOutputAliasConfig::SetUpAlias(const ShapeIndex& output_index, param_number, param_index.ToString(), output_index.ToString(), alias_.element(output_index)->parameter_number, alias_.element(output_index)->parameter_index.ToString()); - (*alias_.mutable_element(output_index)) = Alias(param_number, param_index); + (*alias_.mutable_element(output_index)) = + Alias(param_number, param_index, must_alias); VLOG(4) << "Set up alias between output index " << output_index.ToString() << " and parameter " << param_index << " at index " << param_index.ToString(); @@ -61,6 +64,11 @@ HloInputOutputAliasProto HloInputOutputAliasConfig::ToProto() const { for (int64 i : data->parameter_index) { entry.add_parameter_shape_index(i); } + if (data->must_alias()) { + entry.set_kind(Kind::MUST_ALIAS); + } else { + entry.set_kind(Kind::MAY_ALIAS); + } result.add_entries()->Swap(&entry); } }); @@ -77,8 +85,9 @@ StatusOr HloInputOutputAliasConfig::CreateFromProto( int64 param_number = entry.parameter_number(); ShapeIndex param_index(entry.parameter_shape_index().begin(), entry.parameter_shape_index().end()); + AliasKind kind = entry.kind() == Kind::MAY_ALIAS ? kMayAlias : kMustAlias; TF_RETURN_IF_ERROR( - result.SetUpAlias(output_index, param_number, param_index)); + result.SetUpAlias(output_index, param_number, param_index, kind)); } return result; } @@ -93,9 +102,9 @@ string HloInputOutputAliasConfig::ToString() const { ForEachAlias([&](const ShapeIndex& output_index, const Alias& alias) { pieces.push_back(absl::StrFormat( - " OutputIndex %s is aliased with parameter %lld at %s:", - output_index.ToString(), alias.parameter_number, - alias.parameter_index.ToString())); + " OutputIndex %s is %saliased with parameter %lld at %s:", + output_index.ToString(), alias.kind == kMustAlias ? "must-" : "may-", + alias.parameter_number, alias.parameter_index.ToString())); }); return absl::StrJoin(pieces, "\n"); } @@ -112,6 +121,19 @@ string HloInputOutputAliasConfig::ToShortString() const { return absl::StrJoin(pieces, ", "); } +bool HloInputOutputAliasConfig::ParameterMustAlias( + int64 param_number, const ShapeIndex& param_index) const { + bool result = false; + alias_.ForEachElement( + [&](const xla::ShapeIndex&, absl::optional alias) { + if (alias && alias->parameter_number == param_number && + alias->parameter_index == param_index && alias->must_alias()) { + result = true; + } + }); + return result; +} + absl::optional HloInputOutputAliasConfig::GetAliasedOutput( int64 param_number, const ShapeIndex& param_index) const { absl::optional output; diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h index d5ca28e9387..6b84bdb6a68 100644 --- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h +++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h @@ -32,22 +32,32 @@ class HloModule; // parameter index in the entry computation. class HloInputOutputAliasConfig { public: + // The kind of aliases which can be set. A kMayAlias is one setup at + // compilation time by the user, and has to be respected. A kMustAlias one + // might be setup by the compiler, if it decides it is convenient to do so. + enum AliasKind { + kMayAlias, + kMustAlias, + }; // Defines the alias information for a given output buffer. A given output // buffer shape index can refer only to one parameter+index. struct Alias { - Alias(int64 parameter_number, ShapeIndex parameter_index) + Alias(int64 parameter_number, ShapeIndex parameter_index, + AliasKind kind = kMayAlias) : parameter_number(parameter_number), - parameter_index(std::move(parameter_index)) {} + parameter_index(std::move(parameter_index)), + kind(kind) {} int64 parameter_number; ShapeIndex parameter_index; + AliasKind kind; + + bool must_alias() const { return kind == kMustAlias; } std::string ToString() { - if (parameter_index.empty()) { - return absl::StrCat(parameter_number); - } - return absl::StrFormat("(%lld, %s)", parameter_number, - parameter_index.ToString()); + return absl::StrFormat("(%lld, %s, %s)", parameter_number, + parameter_index.ToString(), + kind == kMustAlias ? "must_alias" : "may_alias"); } }; @@ -61,7 +71,8 @@ class HloInputOutputAliasConfig { // Sets up alias config from `output_index` to `param_index` at // `param_number`. Status SetUpAlias(const ShapeIndex& output_index, int64 param_number, - const ShapeIndex& param_index); + const ShapeIndex& param_index, + AliasKind must_alias = kMayAlias); // Returns true if the given parameter is aliased with one of the output // buffers. @@ -92,6 +103,11 @@ class HloInputOutputAliasConfig { absl::optional GetAliasedParameter( const ShapeIndex& output_index) const; + // Returns if the parameter at the given parameter number and parameter + // index must-alias with an output. + bool ParameterMustAlias(int64 param_number, + const ShapeIndex& param_index) const; + using AliasFn = std::function; diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc index b12779e65ce..a093a9d0f52 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.cc +++ b/tensorflow/compiler/xla/service/hlo_parser.cc @@ -552,33 +552,39 @@ bool HloParserImpl::ParseAliasing(AliasingData* data) { return false; } - if (lexer_.GetKind() != TokKind::kLparen) { - // Short form: "{0}: 0", output index "{}" is assumed. - int64 param_num; - ParseInt64(¶m_num); - data->emplace(std::piecewise_construct, std::forward_as_tuple(out), - std::forward_as_tuple(param_num, ShapeIndex{})); - } else { - // Long form: "{0}: (0, {0})", output index is explicitly specified. - if (!ParseToken(TokKind::kLparen, errmsg)) { - return false; - } - int64 param_num; - ParseInt64(¶m_num); - if (!ParseToken(TokKind::kComma, errmsg)) { - return false; - } - ShapeIndex param_idx; - if (!ParseShapeIndex(¶m_idx)) { - return false; - } - data->emplace(std::piecewise_construct, std::forward_as_tuple(out), - std::forward_as_tuple(param_num, param_idx)); - if (!ParseToken(TokKind::kRparen, errmsg)) { - return false; + if (!ParseToken(TokKind::kLparen, errmsg)) { + return false; + } + int64 param_num; + ParseInt64(¶m_num); + if (!ParseToken(TokKind::kComma, errmsg)) { + return false; + } + ShapeIndex param_idx; + if (!ParseShapeIndex(¶m_idx)) { + return false; + } + + HloInputOutputAliasConfig::AliasKind alias_kind = + HloInputOutputAliasConfig::kMayAlias; + if (EatIfPresent(TokKind::kComma)) { + std::string type; + ParseName(&type); + if (type == "must-alias") { + alias_kind = HloInputOutputAliasConfig::kMustAlias; + } else if (type == "may-alias") { + alias_kind = HloInputOutputAliasConfig::kMayAlias; + } else { + return TokenError("Unexpected aliasing kind; expected SYSTEM or USER"); } } + data->emplace(std::piecewise_construct, std::forward_as_tuple(out), + std::forward_as_tuple(param_num, param_idx, alias_kind)); + if (!ParseToken(TokKind::kRparen, errmsg)) { + return false; + } + if (!EatIfPresent(TokKind::kComma)) { break; } @@ -624,8 +630,9 @@ bool HloParserImpl::ParseHloModule(HloModule* module) { if (aliasing_data) { HloInputOutputAliasConfig alias_config(module->result_shape()); for (auto& p : *aliasing_data) { - Status st = alias_config.SetUpAlias(p.first, p.second.parameter_number, - p.second.parameter_index); + Status st = + alias_config.SetUpAlias(p.first, p.second.parameter_number, + p.second.parameter_index, p.second.kind); if (!st.ok()) { return TokenError(st.error_message()); } diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc index 1b33cf2f4c3..7880075dcbe 100644 --- a/tensorflow/compiler/xla/service/hlo_parser_test.cc +++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc @@ -2399,7 +2399,7 @@ ENTRY c2 { TEST_F(HloParserTest, SimpleAliasing) { const string original = R"( -HloModule Module, input_output_alias={ {0}: (0, {0}), {1}: (0, {1}) } +HloModule Module, input_output_alias={ {0}: (0, {0}, must-alias), {1}: (0, {1}) } ENTRY entry { %p = (f32[], f32[]) parameter(0) @@ -2413,42 +2413,13 @@ ENTRY entry { std::unique_ptr parsed_module = module.ConsumeValueOrDie(); EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(0, {0}), ShapeIndex{0}); + + EXPECT_TRUE( + parsed_module->input_output_alias_config().ParameterMustAlias(0, {0})); EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(0, {1}), ShapeIndex{1}); -} - -TEST_F(HloParserTest, SimpleAliasingShortForm) { - const string original = R"( -HloModule Module, input_output_alias={ {0}: 0, {1}: 1 } - -ENTRY entry { - %p0 = f32[] parameter(0) - %p1 = f32[] parameter(1) - ROOT %out = (f32[], f32[]) tuple(%p0, %p1) -} - )"; - auto module = ParseAndReturnVerifiedModule(original); - TF_ASSERT_OK(module.status()); - std::unique_ptr parsed_module = module.ConsumeValueOrDie(); - EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(0, {}), - ShapeIndex{0}); - EXPECT_EQ(parsed_module->input_output_alias_config().GetAliasedOutput(1, {}), - ShapeIndex{1}); -} - -TEST_F(HloParserTest, SimpleAliasingShortFormError) { - const string original = R"( -HloModule Module, input_output_alias={ {0}: A, {1}: 1 } - -ENTRY entry { - %p0 = f32[] parameter(0) - %p1 = f32[] parameter(1) - ROOT %out = (f32[], f32[]) tuple(%p0, %p1) -} - )"; - ExpectHasSubstr( - ParseAndReturnUnverifiedModule(original).status().error_message(), - "expects integer"); + EXPECT_FALSE( + parsed_module->input_output_alias_config().ParameterMustAlias(0, {1})); } TEST_F(HloParserTest, NestedAliasing) { diff --git a/tensorflow/compiler/xla/tests/buffer_donation_test.cc b/tensorflow/compiler/xla/tests/buffer_donation_test.cc index 856ea7c9b44..f78083fe2af 100644 --- a/tensorflow/compiler/xla/tests/buffer_donation_test.cc +++ b/tensorflow/compiler/xla/tests/buffer_donation_test.cc @@ -61,7 +61,7 @@ class BufferDonationTest : public HloTestBase { absl::Span argument_literals, absl::Span donate_arguments, absl::Span expected_runtime_aliasing, - const Literal& expected) { + const Literal& expected, std::string expected_failure = "") { // Create a copy of the output shape because the HLO module is std::moved // into the compiler and may be deallocated. const Shape output_shape = hlo_module->result_shape(); @@ -123,10 +123,19 @@ class BufferDonationTest : public HloTestBase { ExecutionInput(std::move(owned_buffers), argument_literal.shape())); } - TF_ASSERT_OK_AND_ASSIGN( - ExecutionOutput output, + StatusOr output_status = executable->ExecuteAsyncOnStream(&service_run_options, std::move(args), - /*hlo_execution_profile=*/nullptr)); + /*hlo_execution_profile=*/nullptr); + if (!expected_failure.empty()) { + ASSERT_FALSE(output_status.ok()); + ASSERT_TRUE(absl::StrContains(output_status.status().error_message(), + expected_failure)) + << "got: \n" + << output_status.status().error_message() << " \nvs want\n" + << expected_failure; + return; + } + ExecutionOutput output = output_status.ConsumeValueOrDie(); se::DeviceMemoryBase result_root_buffer = output.Result().root_buffer(); LOG(INFO) << "result allocation = " << result_root_buffer.opaque() @@ -303,5 +312,37 @@ ENTRY entry { #endif } +TEST_F(BufferDonationTest, TestMustAliasNotDonated) { + HloModuleConfig config; + + StatusOr> module = + ParseAndReturnVerifiedModule(R"( +HloModule module + +ENTRY entry { + a = f32[] parameter(0) + b = f32[] parameter(1) + ROOT out = (f32[], f32[]) tuple(a, b) +} + )", + config); + + TF_ASSERT_OK(module->get()->input_output_alias_config().SetUpAlias( + {0}, 0, {}, HloInputOutputAliasConfig::kMustAlias)); + + std::vector args; + args.push_back(LiteralUtil::CreateR0(0.1)); + args.push_back(LiteralUtil::CreateR0(0.2)); + Literal expected = LiteralUtil::MakeTupleFromSlices( + {LiteralUtil::CreateR0(0.1), LiteralUtil::CreateR0(0.2)}); + +#ifndef XLA_TEST_BACKEND_INTERPRETER + RunAndCheck(std::move(*module), args, + /*donate_arguments=*/{false, false}, {true, false}, expected, + "An input was configured to be must-alias at " + "compile time but not donated at runtime:"); +#endif +} + } // namespace } // namespace xla diff --git a/tensorflow/stream_executor/tpu/tpu_executable_interface.cc b/tensorflow/stream_executor/tpu/tpu_executable_interface.cc index 13f9db98e5d..f260cc1631f 100644 --- a/tensorflow/stream_executor/tpu/tpu_executable_interface.cc +++ b/tensorflow/stream_executor/tpu/tpu_executable_interface.cc @@ -62,6 +62,24 @@ TpuExecutableInterface::AllocateOutputMemoryWithInputReuse( << " host_shape = " << ShapeUtil::HumanStringWithLayout(host_shape); Shape device_shape = HostShapeToDeviceShape(host_shape); + TF_RETURN_IF_ERROR(alias_config.ForEachAliasWithStatus( + [&](const ShapeIndex& output_index, + absl::optional alias) { + if (alias && alias->must_alias()) { + VLOG(1) << alias->ToString(); + const MaybeOwningDeviceMemory& original_input = + (*arguments)[alias->parameter_number].Buffers().element( + alias->parameter_index); + if (!original_input.HasOwnership()) { + return InvalidArgument( + "An input was configured to be must-alias at " + "compile time but not donated at runtime: %s", + alias->ToString()); + } + } + return Status::OK(); + })); + if (VLOG_IS_ON(3)) { VLOG(3) << "AllocateOutputMemoryWithInputReuse, device = " << device_ordinal << " host_shape = " << ShapeUtil::HumanStringWithLayout(host_shape); From 4e9d0b23aad82817527167cdfd8613567ca64d9c Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Thu, 30 Jul 2020 17:27:50 -0700 Subject: [PATCH 0663/1017] Add EuclideanNorm kernel Update allowlist Update allowlist Change ProcessData to ProcessInput Change ProcessInput to PreprocessInput Pass input to finalizer Change op name --- .../compiler/jit/mark_for_compilation_pass.cc | 1 + tensorflow/compiler/tests/reduce_ops_test.py | 24 ++++++++++- .../compiler/tf2xla/kernels/reduction_ops.cc | 42 +++++++++++++++++++ .../compiler/tf2xla/kernels/reduction_ops.h | 4 ++ .../tf2xla/kernels/reduction_ops_common.cc | 14 +++++-- 5 files changed, 81 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc index 19eb61b6f72..43619eca1fa 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc @@ -1892,6 +1892,7 @@ absl::flat_hash_set GetKnownXLAAllowlistOp() { "Einsum", "EmptyTensorList", "EnsureShape", + "EuclideanNorm", "ExtractImagePatches", "Igamma", "IgammaGradA", diff --git a/tensorflow/compiler/tests/reduce_ops_test.py b/tensorflow/compiler/tests/reduce_ops_test.py index eb46c536e07..5199cb972ee 100644 --- a/tensorflow/compiler/tests/reduce_ops_test.py +++ b/tensorflow/compiler/tests/reduce_ops_test.py @@ -25,6 +25,7 @@ from absl.testing import parameterized import numpy as np from tensorflow.compiler.tests import xla_test +from tensorflow.python.eager import def_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.ops import array_ops @@ -50,7 +51,8 @@ class ReduceOpsTest(xla_test.XLATestCase, parameterized.TestCase): with self.test_scope(): a = array_ops.placeholder(dtype) index = array_ops.placeholder(index_dtype) - out = tf_reduce_fn(a, index) + out = def_function.function(experimental_compile=True)( + tf_reduce_fn)(a, index) result = sess.run(out, {a: test_input, index: [0]}) self.assertAllClose( result, np_reduce_fn(test_input, axis=0), rtol=rtol, atol=atol) @@ -179,6 +181,26 @@ class ReduceOpsTest(xla_test.XLATestCase, parameterized.TestCase): 'Axes contains duplicate dimension'): sess.run(out, {a: [10, 20, 30], index: [0, 0]}) + def testReduceEuclideanNorm(self, index_dtype): + def reference_euclidean_norm(dtype, inp, axis): + inp = inp.astype(dtype) + return np.sqrt(np.sum(inp * np.conj(inp), axis)).astype(dtype) + + for real_dtype in [np.int32, np.int64, np.float16, + np.float32, np.float64]: + self._testReduction(math_ops.reduce_euclidean_norm, + functools.partial( + reference_euclidean_norm, real_dtype), + real_dtype, + self.REAL_DATA, index_dtype) + + for complex_dtype in [np.complex64, np.complex128]: + self._testReduction(math_ops.reduce_euclidean_norm, + functools.partial( + reference_euclidean_norm, complex_dtype), + complex_dtype, + self.COMPLEX_DATA, index_dtype) + class ReduceOpPrecisionTest(xla_test.XLATestCase): diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc index 4f63c0d1b66..f95d58fd96a 100644 --- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc @@ -16,12 +16,15 @@ limitations under the License. // XLA-specific reduction Ops. #include "tensorflow/compiler/tf2xla/kernels/reduction_ops.h" + #include "tensorflow/compiler/tf2xla/type_util.h" #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/lib/math.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal.h" +#include "tensorflow/compiler/xla/primitive_util.h" #include "tensorflow/core/framework/kernel_def_builder.h" namespace tensorflow { @@ -184,5 +187,44 @@ class AnyOp : public XlaReductionOp { REGISTER_XLA_OP(Name("Any").CompileTimeConstantInput("reduction_indices"), AnyOp); +class EuclideanNormOp : public XlaReductionOp { + public: + explicit EuclideanNormOp(OpKernelConstruction* ctx) + : XlaReductionOp(ctx, + XlaHelpers::SumAccumulationType(ctx->input_type(0))) {} + xla::XlaOp InitialValue(xla::XlaBuilder* builder) override { + return xla::Zero(builder, xla_reduction_type_); + } + + xla::XlaOp PreprocessInput(xla::XlaBuilder* /*builder*/, + const xla::XlaOp& data) override { + return xla::Mul(data, MaybeConjugate(data, true)); + } + + void BuildReducer(xla::XlaBuilder* builder, const xla::XlaOp& scalar_lhs, + const xla::XlaOp& scalar_rhs) override { + xla::Add(scalar_lhs, scalar_rhs); + } + + xla::XlaOp BuildFinalizer( + xla::XlaBuilder* /*builder*/, const xla::XlaOp& input, + const xla::XlaOp& reduce_output, + const std::vector& dimensions_to_reduce) override { + if (xla::primitive_util::IsIntegralType(xla_reduction_type_)) { + // XLA only supports float and complex sqrt. + // Thus, cast integral type to F32 for computation. + return XlaHelpers::ConvertElementType( + xla::Sqrt(xla::ConvertElementType(reduce_output, xla::F32)), + input_type(0)); + } + return XlaHelpers::ConvertElementType(xla::Sqrt(reduce_output), + input_type(0)); + } +}; + +REGISTER_XLA_OP( + Name("EuclideanNorm").CompileTimeConstantInput("reduction_indices"), + EuclideanNormOp); + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h index af716eab798..2091b496ddb 100644 --- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h +++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h @@ -39,6 +39,10 @@ class XlaReductionOp : public XlaOpKernel { // Return the base case for the reduction. virtual xla::XlaOp InitialValue(xla::XlaBuilder* builder) = 0; + // Preprocesses input before reduction. + virtual xla::XlaOp PreprocessInput(xla::XlaBuilder* builder, + const xla::XlaOp& data); + // Implement the (scalar,scalar)->scalar lambda that should be // applied to each pair of elements to be reduced. The desired // computation should be added to 'builder' and diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc index b4284a5498c..58d53dfea58 100644 --- a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc +++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc @@ -35,6 +35,12 @@ XlaReductionOp::XlaReductionOp(OpKernelConstruction* ctx, ctx, DataTypeToPrimitiveType(reduction_type_, &xla_reduction_type_)); } +// The default pre-processor directly returns the data. This can be overridden. +xla::XlaOp XlaReductionOp::PreprocessInput(xla::XlaBuilder* /*builder*/, + const xla::XlaOp& data) { + return data; +} + // The default finalizer converts the results back into the input type. This can // be overridden. xla::XlaOp XlaReductionOp::BuildFinalizer( @@ -111,7 +117,8 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) { xla::PrimitiveType type; TF_CHECK_OK(DataTypeToPrimitiveType(reduction_type_, &type)); - auto data = xla::ConvertElementType(ctx->Input(0), type); + auto converted_input = xla::ConvertElementType(ctx->Input(0), type); + auto processed_input = PreprocessInput(b, converted_input); // Call virtual method to get the initial value. auto initial = xla::ConvertElementType(InitialValue(b), type); // Make two scalar parameters of the desired type for the lambda. @@ -121,8 +128,9 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) { BuildReducer(&r, rx, ry); xla::XlaComputation reduction_computation = r.Build().ConsumeValueOrDie(); - auto reduce = xla::Reduce(data, initial, reduction_computation, xla_axes); - auto finalized = BuildFinalizer(b, data, reduce, xla_axes); + auto reduce = + xla::Reduce(processed_input, initial, reduction_computation, xla_axes); + auto finalized = BuildFinalizer(b, converted_input, reduce, xla_axes); auto result = keep_dims_ ? xla::Reshape(finalized, final_shape) : finalized; ctx->SetOutput(0, result); } From 4684e40f18bcc5af6ef5a5605d9e9901d430621f Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Fri, 7 Aug 2020 14:18:53 -0700 Subject: [PATCH 0664/1017] Clarify default Interpreter NNAPI behavior PiperOrigin-RevId: 325507160 Change-Id: Id2063801d286ebda56e0105c806aaed52a930e72 --- tensorflow/lite/interpreter.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h index b9e2045cd96..ab2d1250513 100644 --- a/tensorflow/lite/interpreter.h +++ b/tensorflow/lite/interpreter.h @@ -364,7 +364,11 @@ class Interpreter { /// Returns status of success or failure. TfLiteStatus Invoke(); - /// Enable or disable the NN API (true to enable) + /// Enable or disable NNAPI (true to enable). Disabled by default. + /// + /// WARNING: NNAPI cannot be disabled after the graph has been prepared + /// (via `AllocateTensors`) with NNAPI enabled. + /// /// NOTE: This API is deprecated, prefer using the NNAPI delegate directly. /// This method will be removed in a future release. void UseNNAPI(bool enable); From 201d45cea27c1792a86b3fc7eb688fb2dd1d0df1 Mon Sep 17 00:00:00 2001 From: Reed Wanderman-Milne Date: Fri, 7 Aug 2020 14:27:14 -0700 Subject: [PATCH 0665/1017] Improve performance of fp16 DepthwiseConv2DBackpropFilter. When cuDNN is not used, performance and numeric stability is improved by casting inputs to float32 and outputs back to float16. The original implementation does accumulation in float16 and is slow for unknown reasons. Running the benchmark in [this comment](https://github.com/tensorflow/tensorflow/issues/41715#issuecomment-664705080) on my machine with a Titan V, I get the following numbers. All numbers are in seconds. ``` bench before after float16 NHWC backprop_filter 7.6379 0.0098 float16 NCHW backprop_filter 4.1965 0.0449 float32 NHWC backprop_filter 0.0094 0.0094 float32 NCHW backprop_filter 0.0449 0.0444 ``` Fixes https://github.com/tensorflow/tensorflow/issues/41715. PiperOrigin-RevId: 325508729 Change-Id: I694a62dcdd8731bc90e98d2a09486160d8740b5f --- tensorflow/core/kernels/BUILD | 1 + .../core/kernels/depthwise_conv_grad_op.cc | 42 +++++++++++++++++-- .../kernel_tests/depthwise_conv_op_test.py | 2 +- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index f98b510b96f..99970a9558c 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -4422,6 +4422,7 @@ tf_kernel_library( "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:lib", + ":cast_op", ] + if_cuda([ "@local_config_cuda//cuda:cudnn_header", ]), diff --git a/tensorflow/core/kernels/depthwise_conv_grad_op.cc b/tensorflow/core/kernels/depthwise_conv_grad_op.cc index 310bd73ba65..b809e1d1065 100644 --- a/tensorflow/core/kernels/depthwise_conv_grad_op.cc +++ b/tensorflow/core/kernels/depthwise_conv_grad_op.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/cast_op.h" #include "tensorflow/core/kernels/conv_grad_ops.h" #include "tensorflow/core/kernels/depthwise_conv_op.h" #include "tensorflow/core/lib/core/status.h" @@ -1180,12 +1181,45 @@ class DepthwiseConv2dNativeBackpropFilterOp : public OpKernel { return; } - auto out_backprop_ptr = out_backprop.template flat().data(); - auto input_ptr = input.template flat().data(); - auto filter_backprop_ptr = filter_backprop->template flat().data(); - LaunchDepthwiseConvBackpropFilterOp()( + // For GPU inputs with type half, we cast inputs to float and outputs back + // to half, as half implementation is slow and does not use full precision + // accumulation in some cases. + constexpr bool cast_to_float = std::is_same::value && + std::is_same::value; + using U = typename std::conditional::type; + Tensor casted_out_backprop = out_backprop; + Tensor casted_input = input; + Tensor casted_filter_backprop = *filter_backprop; + const Device& device = context->template eigen_device(); + if (cast_to_float) { + functor::CastFunctor cast; + OP_REQUIRES_OK(context, + context->allocate_temp(DT_FLOAT, out_backprop.shape(), + &casted_out_backprop)); + cast(device, casted_out_backprop.template flat(), + out_backprop.template flat()); + OP_REQUIRES_OK(context, context->allocate_temp(DT_FLOAT, input.shape(), + &casted_input)); + cast(device, casted_input.template flat(), + input.template flat()); + OP_REQUIRES_OK(context, + context->allocate_temp(DT_FLOAT, filter_backprop->shape(), + &casted_filter_backprop)); + } + + auto out_backprop_ptr = casted_out_backprop.template flat().data(); + auto input_ptr = casted_input.template flat().data(); + auto filter_backprop_ptr = casted_filter_backprop.template flat().data(); + LaunchDepthwiseConvBackpropFilterOp()( context, args, out_backprop_ptr, input_ptr, filter_backprop_ptr, data_format_); + + if (cast_to_float) { + functor::CastFunctor cast; + const Tensor& casted_filter_backprop_const = casted_filter_backprop; + cast(device, filter_backprop->template flat(), + casted_filter_backprop_const.template flat()); + } } protected: diff --git a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py index 093de720b53..266a0f8d0fb 100644 --- a/tensorflow/python/kernel_tests/depthwise_conv_op_test.py +++ b/tensorflow/python/kernel_tests/depthwise_conv_op_test.py @@ -832,7 +832,7 @@ class DepthwiseConv2DTest(test.TestCase): # double datatype is currently not supported for convolution ops # on the ROCm platform optional_float64 = [] if test.is_built_with_rocm() else [dtypes.float64] - for data_type in ([dtypes.float32] + optional_float64): + for data_type in ([dtypes.float16, dtypes.float32] + optional_float64): self._ConstructAndTestGradient( input_size, filter_size, From a5f56f8c1166cd4a45b2a4fc10d07fa1dd3f1e6f Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Fri, 7 Aug 2020 14:29:41 -0700 Subject: [PATCH 0666/1017] Enable creating variables in loops. PiperOrigin-RevId: 325509160 Change-Id: I1d73baf75d0be1b3707b1cfceb97e8b9a32162e4 --- RELEASE.md | 14 ++++++++++++++ .../python/autograph/operators/control_flow.py | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index f0c590710cf..5f0553c2a94 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -106,6 +106,20 @@ True, the function may use type annotations to optimize the tracing performance. * Added support for `iter(DistributedDataset)` in AutoGraph `for` loops. + * AutoGraph now allows creating new symbols inside a TensorFLow loop, if + the values of these symbols at an iteration does not depend on the previous + iteration. These types of loops must run at least one iteration, and will + raise a runtime error otherwise. + + Example: + + ``` + for batch in data: + outputs = train_step(batch) + tf.print('final outputs', outputs) + ``` + See tensorflow/python/autograph/g3doc/reference/limitations.md for more + info. * `tf.lite`: * `DynamicBuffer::AddJoinedString()` will now add a separator if the first string to be joined is empty. diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py index 3418450e813..0106efda5dd 100644 --- a/tensorflow/python/autograph/operators/control_flow.py +++ b/tensorflow/python/autograph/operators/control_flow.py @@ -973,7 +973,8 @@ def _try_handling_undefineds( """ state_modified = False - if not os.getenv('AUTOGRAPH_CREATE_SYMBOLS_IN_LOOPS', ''): + # TODO(mdan): Remove once the default option is stable. + if os.getenv('AUTOGRAPH_CREATE_SYMBOLS_IN_LOOPS', '1') == '0': _verify_loop_init_vars(init_vars, symbol_names) return False, init_vars From 1e6fa32dfe9d3f1631ed6218985426d3f6ca4860 Mon Sep 17 00:00:00 2001 From: Revan Sopher Date: Fri, 7 Aug 2020 14:36:54 -0700 Subject: [PATCH 0667/1017] Exception grammar fix. PiperOrigin-RevId: 325510739 Change-Id: Idc535c7c62629b53e51919e8cc417e89e7168175 --- tensorflow/python/tpu/tpu_embedding_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/tpu/tpu_embedding_v2.py b/tensorflow/python/tpu/tpu_embedding_v2.py index 5e316d35aa4..412c7eb03d3 100644 --- a/tensorflow/python/tpu/tpu_embedding_v2.py +++ b/tensorflow/python/tpu/tpu_embedding_v2.py @@ -1501,7 +1501,7 @@ def make_sharded_variable_creator(hosts): if isinstance(initial_value, base.CheckpointInitialValue) and num_hosts > 1: raise RuntimeError("Delayed restoration of variables not available when " "there are multiple TPU hosts, please ensure that the " - "api object is build before you restore.") + "api object has been built before you restore.") for i, p in enumerate(partitions): with ops.device(hosts[i]): From 7a93fd22f78231a64abb212dffa15d9749da1281 Mon Sep 17 00:00:00 2001 From: Sachin Joglekar Date: Fri, 7 Aug 2020 14:39:18 -0700 Subject: [PATCH 0668/1017] Adds API for users to provide custom allocations for TFLite tensors PiperOrigin-RevId: 325511199 Change-Id: Ia8c0550375d508db3fa75b6b5df5a70088b7470b --- tensorflow/lite/BUILD | 2 + tensorflow/lite/c/common.h | 12 + tensorflow/lite/core/subgraph.cc | 57 ++++- tensorflow/lite/core/subgraph.h | 26 ++ tensorflow/lite/interpreter.cc | 6 + tensorflow/lite/interpreter.h | 23 ++ tensorflow/lite/interpreter_test.cc | 241 ++++++++++++++++++ .../lite/micro/micro_optional_debug_tools.cc | 2 + tensorflow/lite/optional_debug_tools.cc | 3 + .../benchmark/experimental/c/c_api_types.h | 12 + 10 files changed, 383 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD index fac85181231..7007a847d83 100644 --- a/tensorflow/lite/BUILD +++ b/tensorflow/lite/BUILD @@ -412,9 +412,11 @@ cc_test( "tflite_smoke_test", ], deps = [ + ":builtin_op_data", ":external_cpu_backend_context", ":framework", ":string_util", + ":util", ":version", "//tensorflow/lite/core/api", "//tensorflow/lite/kernels:builtin_ops", diff --git a/tensorflow/lite/c/common.h b/tensorflow/lite/c/common.h index 7ef173c78d2..23eb528f4c9 100644 --- a/tensorflow/lite/c/common.h +++ b/tensorflow/lite/c/common.h @@ -358,6 +358,8 @@ typedef union TfLitePtrUnion { // * kTfLitePersistentRo: Allocated and populated during prepare. This is // useful for tensors that can be computed during prepare and treated // as constant inputs for downstream ops (also in prepare). +// * kTfLiteCustom: Custom memory allocation provided by the user. See +// TfLiteCustomAllocation below. typedef enum TfLiteAllocationType { kTfLiteMemNone = 0, kTfLiteMmapRo, @@ -365,6 +367,7 @@ typedef enum TfLiteAllocationType { kTfLiteArenaRwPersistent, kTfLiteDynamic, kTfLitePersistentRo, + kTfLiteCustom, } TfLiteAllocationType; // The delegates should use zero or positive integers to represent handles. @@ -397,6 +400,15 @@ typedef struct TfLiteSparsity { int dim_metadata_size; } TfLiteSparsity; +// Defines a custom memory allocation not owned by the runtime. +// `data` should be aligned to kDefaultTensorAlignment defined in +// lite/util.h. (Currently 64 bytes) +// NOTE: See Interpreter.SetCustomAllocationForTensor for details on usage. +typedef struct TfLiteCustomAllocation { + void* data; + size_t bytes; +} TfLiteCustomAllocation; + // An tensor in the interpreter system which is a wrapper around a buffer of // data including a dimensionality (or NULL if not currently defined). #ifndef TF_LITE_STATIC_MEMORY diff --git a/tensorflow/lite/core/subgraph.cc b/tensorflow/lite/core/subgraph.cc index beedbe6c5ea..15b8a0bcc57 100644 --- a/tensorflow/lite/core/subgraph.cc +++ b/tensorflow/lite/core/subgraph.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/lite/core/subgraph.h" #include +#include #include "tensorflow/lite/arena_planner.h" #include "tensorflow/lite/c/common.h" @@ -140,6 +141,17 @@ const char* GetTFLiteOpName(const TfLiteRegistration& op_reg) { return tflite::EnumNamesBuiltinOperator()[op_reg.builtin_code]; } +TfLiteStatus ValidateCustomAllocationForTensor( + TfLiteContext* context, const TfLiteTensor* tensor, + const TfLiteCustomAllocation& allocation) { + TF_LITE_ENSURE(context, allocation.data != nullptr); + TF_LITE_ENSURE(context, allocation.bytes >= tensor->bytes); + // Ensure provided memory is aligned to what TFLite requires. + const intptr_t data_ptr_value = reinterpret_cast(allocation.data); + TF_LITE_ENSURE(context, data_ptr_value % kDefaultTensorAlignment == 0); + return kTfLiteOk; +} + } // namespace // A trivial implementation of GraphInfo around the Interpreter. @@ -898,9 +910,24 @@ TfLiteStatus Subgraph::PrepareOpsAndTensors() { execution_plan_, &last_exec_plan_index_prepared)); next_execution_plan_index_to_prepare_ = last_exec_plan_index_prepared + 1; + // Execute arena allocations. TF_LITE_ENSURE_STATUS(memory_planner_->ExecuteAllocations( next_execution_plan_index_to_plan_allocation_, last_exec_plan_index_prepared)); + + // Ensure custom allocations are still valid for applicable tensors. + // This causes some extra validations for cases with dynamic tensors, but the + // overhead should be minimal since the number of custom-allocated tensors + // will typically be low. + for (int i = 0; i < custom_allocations_.size(); ++i) { + auto idx_and_alloc = custom_allocations_[i]; + auto& tensor = tensors()[idx_and_alloc.first]; + const auto& alloc = idx_and_alloc.second; + TF_LITE_ENSURE(context(), tensor.allocation_type == kTfLiteCustom); + TF_LITE_ENSURE_STATUS( + ValidateCustomAllocationForTensor(context(), &tensor, alloc)); + } + next_execution_plan_index_to_plan_allocation_ = last_exec_plan_index_prepared + 1; @@ -1218,7 +1245,8 @@ TfLiteStatus Subgraph::ResizeTensorImpl(TfLiteTensor* tensor, if (tensor->allocation_type == kTfLiteArenaRw || tensor->allocation_type == kTfLiteDynamic || tensor->allocation_type == kTfLiteArenaRwPersistent || - tensor->allocation_type == kTfLitePersistentRo) { + tensor->allocation_type == kTfLitePersistentRo || + tensor->allocation_type == kTfLiteCustom) { tensor_resized_since_op_invoke_ |= TfLiteIntArrayEqual(tensor->dims, new_size) == 0; if (tensor->type != kTfLiteString) { @@ -1455,6 +1483,33 @@ TfLiteStatus Subgraph::ModifyGraphWithDelegate(TfLiteDelegate* delegate) { return status; } +TfLiteStatus Subgraph::SetCustomAllocationForTensor( + int tensor_index, const TfLiteCustomAllocation& allocation) { + TfLiteTensor* tensor = &context_.tensors[tensor_index]; + TF_LITE_ENSURE(context(), tensor->allocation_type == kTfLiteArenaRw || + tensor->allocation_type == kTfLiteCustom); + TF_LITE_ENSURE_STATUS( + ValidateCustomAllocationForTensor(context(), tensor, allocation)); + + // If tensor already has a custom alloc, just reassign. + const auto alloc_it = std::find_if( + custom_allocations_.begin(), custom_allocations_.end(), + [tensor_index]( + const std::pair& existing_alloc) { + return existing_alloc.first == tensor_index; + }); + if (alloc_it == custom_allocations_.end()) { + custom_allocations_.emplace_back(tensor_index, allocation); + } else { + alloc_it->second = allocation; + } + + tensor->allocation_type = kTfLiteCustom; + tensor->data.data = allocation.data; + + return kTfLiteOk; +} + } // namespace impl } // namespace tflite diff --git a/tensorflow/lite/core/subgraph.h b/tensorflow/lite/core/subgraph.h index 5058273667a..1fe1c7e4391 100644 --- a/tensorflow/lite/core/subgraph.h +++ b/tensorflow/lite/core/subgraph.h @@ -332,6 +332,29 @@ class Subgraph { // Before `AllocateTensors` is called, this will always return true; bool HasDynamicTensors() { return has_dynamic_tensors_; } + // Assigns (or reassigns) a custom memory allocation for the given tensor. + // If AllocateTensors() is called after this, the runtime does not consider + // the tensor during internal memory planning and will continue using the + // provided allocation for the tensor (assuming it satisfies the expected + // tensor byte length). + // The runtime does NOT take ownership of the underlying memory. + // Note that while this function can be called again to set a new allocation + // for the tensor, it can no longer be reset to the TFLite arena memory. + // + // Parameters should satisfy the following conditions: + // 1. tensor->allocation_type == kTfLiteArenaRw + // In general, this is true for all non-constants such as I/O tensors. + // 2. allocation->data has the appropriate permissions for runtime access + // (Read-only for inputs, Read-Write for others), and outlives Interpreter. + // 3. allocation->bytes >= tensor->bytes. + // This condition is checked again if any tensors are resized. + // 4. allocation->data should be aligned to kDefaultTensorAlignment + // defined in lite/util.h. (Currently 64 bytes) + // + // WARNING: This is an experimental interface that is subject to change. + TfLiteStatus SetCustomAllocationForTensor( + int tensor_index, const TfLiteCustomAllocation& allocation); + private: // SubgraphAwareProfiler wraps an actual TFLite profiler, such as a // BufferedProfiler instance, and takes care of event profiling/tracing in a @@ -680,6 +703,9 @@ class Subgraph { std::unique_ptr memory_planner_; + // Contains pairs for all applicable tensors. + std::vector> custom_allocations_; + // Tracking bit for whether a tensor was resized in the course of an op // invocation. This is a useful hint to ensure that dynamic tensor outputs // trigger downstream reallocation after op invocation. diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc index 62de5896d84..7a5f4df5155 100644 --- a/tensorflow/lite/interpreter.cc +++ b/tensorflow/lite/interpreter.cc @@ -163,6 +163,12 @@ void Interpreter::SetExternalContext(TfLiteExternalContextType type, primary_subgraph().SetExternalContext(type, ctx); } +TfLiteStatus Interpreter::SetCustomAllocationForTensor( + int tensor_index, const TfLiteCustomAllocation& allocation) { + return primary_subgraph().SetCustomAllocationForTensor(tensor_index, + allocation); +} + TfLiteStatus Interpreter::SetInputs(std::vector inputs) { return primary_subgraph().SetInputs(std::move(inputs)); } diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h index ab2d1250513..5c354d63dd5 100644 --- a/tensorflow/lite/interpreter.h +++ b/tensorflow/lite/interpreter.h @@ -504,6 +504,29 @@ class Interpreter { void SetExternalContext(TfLiteExternalContextType type, TfLiteExternalContext* ctx); + // Assigns (or reassigns) a custom memory allocation for the given tensor. + // If AllocateTensors() is called after this, the runtime does not consider + // the tensor during internal memory planning and will continue using the + // provided allocation for the tensor (assuming it satisfies the expected + // tensor byte length). + // The runtime does NOT take ownership of the underlying memory. + // Note that while this function can be called again to set a new allocation + // for the tensor, it can no longer be reset to the TFLite arena memory. + // + // Parameters should satisfy the following conditions: + // 1. tensor->allocation_type == kTfLiteArenaRw + // In general, this is true for all non-constants such as I/O tensors. + // 2. allocation->data has the appropriate permissions for runtime access + // (Read-only for inputs, Read-Write for others), and outlives Interpreter. + // 3. allocation->bytes >= tensor->bytes. + // This condition is checked again if any tensors are resized. + // 4. allocation->data should be aligned to kDefaultTensorAlignment + // defined in lite/util.h. (Currently 64 bytes) + // + // WARNING: This is an experimental interface that is subject to change. + TfLiteStatus SetCustomAllocationForTensor( + int tensor_index, const TfLiteCustomAllocation& allocation); + #ifndef DOXYGEN_SKIP /// Adds `subgraphs_to_add` subgraphs, preserving pre-existing Subgraph /// entries. The value pointed to by `first_new_subgraph_index` will be set to diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc index 899811b3fea..bf40843876c 100644 --- a/tensorflow/lite/interpreter_test.cc +++ b/tensorflow/lite/interpreter_test.cc @@ -22,8 +22,10 @@ limitations under the License. #include #include #include "third_party/eigen3/Eigen/Core" +#include "tensorflow/lite/builtin_op_data.h" #include "tensorflow/lite/core/api/error_reporter.h" #include "tensorflow/lite/external_cpu_backend_context.h" +#include "tensorflow/lite/kernels/builtin_op_kernels.h" #include "tensorflow/lite/kernels/cpu_backend_context.h" #include "tensorflow/lite/kernels/internal/compatibility.h" #include "tensorflow/lite/kernels/kernel_util.h" @@ -1480,6 +1482,245 @@ TEST_F(CancellationTest, CancelDuringInvoke) { ASSERT_EQ(invoke_error_code, kTfLiteError); } +// Tests functionality related to custom memory allocations in TFLite. +class TestCustomAllocation : public ::testing::Test { + protected: + void SetUp() override { + // Simple model with two custom ops that add 2 float tensors each. + interpreter_.reset(new Interpreter); + interpreter_->AddTensors(5); + interpreter_->SetInputs({0, 1}); + interpreter_->SetOutputs({3, 4}); + TfLiteQuantizationParams quant; + interpreter_->SetTensorParametersReadWrite(0, kTfLiteFloat32, "", {3}, + quant); + interpreter_->SetTensorParametersReadWrite(1, kTfLiteFloat32, "", {3}, + quant); + interpreter_->SetTensorParametersReadWrite(2, kTfLiteFloat32, "", {3}, + quant); + interpreter_->SetTensorParametersReadWrite(3, kTfLiteFloat32, "", {3}, + quant); + interpreter_->SetTensorParametersReadWrite(4, kTfLiteFloat32, "", {3}, + quant); + auto* add_reg = ops::builtin::Register_ADD(); + TfLiteAddParams* builtin_data0 = + reinterpret_cast(malloc(sizeof(TfLiteAddParams))); + TfLiteAddParams* builtin_data1 = + reinterpret_cast(malloc(sizeof(TfLiteAddParams))); + TfLiteAddParams* builtin_data2 = + reinterpret_cast(malloc(sizeof(TfLiteAddParams))); + builtin_data0->activation = kTfLiteActNone; + builtin_data1->activation = kTfLiteActNone; + builtin_data2->activation = kTfLiteActNone; + interpreter_->AddNodeWithParameters({0, 0}, {2}, nullptr, 0, builtin_data0, + add_reg); + interpreter_->AddNodeWithParameters({1, 1}, {3}, nullptr, 0, builtin_data1, + add_reg); + interpreter_->AddNodeWithParameters({2, 1}, {4}, nullptr, 0, builtin_data2, + add_reg); + } + + void AssignCustomAllocForTensor(int tensor_idx, int required_alignment) { + const TfLiteTensor* tensor = interpreter_->tensor(tensor_idx); + auto tensor_alloc = NewCustomAlloc(tensor->bytes, required_alignment); + ASSERT_EQ( + interpreter_->SetCustomAllocationForTensor(tensor_idx, tensor_alloc), + kTfLiteOk); + } + + void VerifyInvoke() { + std::vector input = {1.0f, 2.0f, 3.0f}; + std::vector expected_output = {2.0f, 4.0f, 6.0f}; + TfLiteTensor* tensor = interpreter_->tensor(interpreter_->outputs()[0]); + + // typed_tensor<...> should work irrespective of custom alloc, since it + // accesses tensor.data. + memcpy(interpreter_->typed_tensor(0), input.data(), + 3 * sizeof(float)); + memcpy(interpreter_->typed_tensor(1), input.data(), + 3 * sizeof(float)); + ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk); + for (int i = 0; i < 3; ++i) { + EXPECT_EQ(tensor->data.f[i], expected_output[i]) << i; + } + } + + // Actual initialized allocation is more than num_bytes, to account for + // required_allocation. + TfLiteCustomAllocation NewCustomAlloc(size_t num_bytes, + int required_alignment) { + // Extra memory to ensure alignment. + char* new_alloc = new char[num_bytes + required_alignment]; + char* new_underlying_buffer_aligned_ptr = reinterpret_cast( + AlignTo(required_alignment, reinterpret_cast(new_alloc))); + custom_alloc_buffers_.emplace_back(new_alloc); + + return TfLiteCustomAllocation( + {new_underlying_buffer_aligned_ptr, num_bytes}); + } + + intptr_t AlignTo(size_t alignment, intptr_t offset) { + return offset % alignment == 0 ? offset + : offset + (alignment - offset % alignment); + } + + void TearDown() override { + interpreter_.reset(); + custom_alloc_buffers_.clear(); + } + + protected: + TfLiteAddParams add_params_; + std::unique_ptr interpreter_; + std::vector> custom_alloc_buffers_; +}; + +TEST_F(TestCustomAllocation, InvalidAlignment) { + const TfLiteTensor* input_tensor = + interpreter_->tensor(interpreter_->inputs()[0]); + auto input_alloc = + NewCustomAlloc(input_tensor->bytes, kDefaultTensorAlignment - 1); + ASSERT_EQ(interpreter_->SetCustomAllocationForTensor( + interpreter_->inputs()[0], input_alloc), + kTfLiteError); + + // Allocate tensors & Invoke should still work. + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + VerifyInvoke(); +} + +TEST_F(TestCustomAllocation, InsufficientBytes) { + auto input_alloc = NewCustomAlloc(4, kDefaultTensorAlignment); + ASSERT_EQ(interpreter_->SetCustomAllocationForTensor( + interpreter_->inputs()[0], input_alloc), + kTfLiteError); + + // Allocate tensors & Invoke should still work. + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + VerifyInvoke(); +} + +TEST_F(TestCustomAllocation, CustomInputAlloc) { + // Set custom allocation for one input tensor. + AssignCustomAllocForTensor(interpreter_->inputs()[0], + /*required_alignment=*/kDefaultTensorAlignment); + + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + VerifyInvoke(); +} + +TEST_F(TestCustomAllocation, CustomInputAlloc_MultipleAssigns) { + // Set custom allocation for one input tensor. + AssignCustomAllocForTensor(interpreter_->inputs()[0], + /*required_alignment=*/kDefaultTensorAlignment); + + AssignCustomAllocForTensor(interpreter_->inputs()[0], + /*required_alignment=*/kDefaultTensorAlignment); + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + VerifyInvoke(); + + AssignCustomAllocForTensor(interpreter_->inputs()[0], + /*required_alignment=*/kDefaultTensorAlignment); + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + VerifyInvoke(); +} + +TEST_F(TestCustomAllocation, CustomInputAlloc_AllocateTensorsBefore) { + // Allocate tensors. + // Allocating now will cause TFLite to reserve some extra memory, but nothing + // should break. + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + + AssignCustomAllocForTensor(interpreter_->inputs()[0], + /*required_alignment=*/kDefaultTensorAlignment); + + VerifyInvoke(); +} + +TEST_F(TestCustomAllocation, CustomInputAndOutputAllocs) { + // Set custom allocations for all IO tensors. + AssignCustomAllocForTensor(interpreter_->inputs()[0], + /*required_alignment=*/kDefaultTensorAlignment); + AssignCustomAllocForTensor(interpreter_->inputs()[1], + /*required_alignment=*/kDefaultTensorAlignment); + AssignCustomAllocForTensor(interpreter_->outputs()[0], + /*required_alignment=*/kDefaultTensorAlignment); + AssignCustomAllocForTensor(interpreter_->outputs()[1], + /*required_alignment=*/kDefaultTensorAlignment); + + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + VerifyInvoke(); +} + +TEST_F(TestCustomAllocation, ResizeTensorsWithoutEnoughMemory) { + // Set custom allocations for all input tensors. + AssignCustomAllocForTensor(interpreter_->inputs()[0], + /*required_alignment=*/kDefaultTensorAlignment); + AssignCustomAllocForTensor(interpreter_->inputs()[1], + /*required_alignment=*/kDefaultTensorAlignment); + + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + + // Now resize tensors to double the size. + ASSERT_EQ(interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {2, 3}), + kTfLiteOk); + ASSERT_EQ(interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {2, 3}), + kTfLiteOk); + + // Since the custom memory previously allocated isn't enough, + // AllocateTensors() will fail. + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteError); + // Interpreter should no longer be in invokable state, so expect failure. + ASSERT_EQ(interpreter_->Invoke(), kTfLiteError); +} + +TEST_F(TestCustomAllocation, ResizeTensorsWithEnoughMemory) { + // Set custom allocations for all input tensors, with double the required + // memory. + const TfLiteTensor* input0_tensor = + interpreter_->tensor(interpreter_->inputs()[0]); + auto input0_alloc = + NewCustomAlloc(2 * input0_tensor->bytes, kDefaultTensorAlignment); + ASSERT_EQ(interpreter_->SetCustomAllocationForTensor( + interpreter_->inputs()[0], input0_alloc), + kTfLiteOk); + const TfLiteTensor* input1_tensor = + interpreter_->tensor(interpreter_->inputs()[1]); + auto input1_alloc = + NewCustomAlloc(2 * input1_tensor->bytes, kDefaultTensorAlignment); + ASSERT_EQ(interpreter_->SetCustomAllocationForTensor( + interpreter_->inputs()[1], input1_alloc), + kTfLiteOk); + + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + + // Now resize tensors to double the size. + ASSERT_EQ(interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {6, 1}), + kTfLiteOk); + ASSERT_EQ(interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {6, 1}), + kTfLiteOk); + + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + + std::vector input = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}; + std::vector expected_output = {2.0f, 4.0f, 6.0f, 8.0f, 10.0f, 12.0f}; + TfLiteTensor* tensor = interpreter_->tensor(interpreter_->outputs()[0]); + memcpy(interpreter_->typed_tensor(0), input.data(), 6 * sizeof(float)); + memcpy(interpreter_->typed_tensor(1), input.data(), 6 * sizeof(float)); + ASSERT_EQ(interpreter_->Invoke(), kTfLiteOk); + for (int i = 0; i < 6; ++i) { + EXPECT_EQ(tensor->data.f[i], expected_output[i]) << i; + } + + ASSERT_EQ(interpreter_->ResizeInputTensor(interpreter_->inputs()[0], {3, 1}), + kTfLiteOk); + ASSERT_EQ(interpreter_->ResizeInputTensor(interpreter_->inputs()[1], {3, 1}), + kTfLiteOk); + + ASSERT_EQ(interpreter_->AllocateTensors(), kTfLiteOk); + VerifyInvoke(); +} + } // namespace } // namespace tflite diff --git a/tensorflow/lite/micro/micro_optional_debug_tools.cc b/tensorflow/lite/micro/micro_optional_debug_tools.cc index 4617b3d9825..e7aee576351 100644 --- a/tensorflow/lite/micro/micro_optional_debug_tools.cc +++ b/tensorflow/lite/micro/micro_optional_debug_tools.cc @@ -109,6 +109,8 @@ const char* AllocTypeName(TfLiteAllocationType type) { return "kTfLiteArenaRwPersistent"; case kTfLitePersistentRo: return "kTfLitePersistentRo"; + case kTfLiteCustom: + return "kTfLiteCustom"; } return "(invalid)"; } diff --git a/tensorflow/lite/optional_debug_tools.cc b/tensorflow/lite/optional_debug_tools.cc index 8ee5c3b3f56..ef4ee1cb4e3 100644 --- a/tensorflow/lite/optional_debug_tools.cc +++ b/tensorflow/lite/optional_debug_tools.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow/lite/optional_debug_tools.h" +#include "tensorflow/lite/c/common.h" #include "tensorflow/lite/schema/schema_generated.h" namespace tflite { @@ -81,6 +82,8 @@ const char* AllocTypeName(TfLiteAllocationType type) { return "kTfLiteArenaRwPersistent"; case kTfLitePersistentRo: return "kTfLitePersistentRo"; + case kTfLiteCustom: + return "kTfLiteCustom"; } return "(invalid)"; } diff --git a/tensorflow/lite/tools/benchmark/experimental/c/c_api_types.h b/tensorflow/lite/tools/benchmark/experimental/c/c_api_types.h index 7ef173c78d2..23eb528f4c9 100644 --- a/tensorflow/lite/tools/benchmark/experimental/c/c_api_types.h +++ b/tensorflow/lite/tools/benchmark/experimental/c/c_api_types.h @@ -358,6 +358,8 @@ typedef union TfLitePtrUnion { // * kTfLitePersistentRo: Allocated and populated during prepare. This is // useful for tensors that can be computed during prepare and treated // as constant inputs for downstream ops (also in prepare). +// * kTfLiteCustom: Custom memory allocation provided by the user. See +// TfLiteCustomAllocation below. typedef enum TfLiteAllocationType { kTfLiteMemNone = 0, kTfLiteMmapRo, @@ -365,6 +367,7 @@ typedef enum TfLiteAllocationType { kTfLiteArenaRwPersistent, kTfLiteDynamic, kTfLitePersistentRo, + kTfLiteCustom, } TfLiteAllocationType; // The delegates should use zero or positive integers to represent handles. @@ -397,6 +400,15 @@ typedef struct TfLiteSparsity { int dim_metadata_size; } TfLiteSparsity; +// Defines a custom memory allocation not owned by the runtime. +// `data` should be aligned to kDefaultTensorAlignment defined in +// lite/util.h. (Currently 64 bytes) +// NOTE: See Interpreter.SetCustomAllocationForTensor for details on usage. +typedef struct TfLiteCustomAllocation { + void* data; + size_t bytes; +} TfLiteCustomAllocation; + // An tensor in the interpreter system which is a wrapper around a buffer of // data including a dimensionality (or NULL if not currently defined). #ifndef TF_LITE_STATIC_MEMORY From 271f2bad32147951000fbdfac341fb91436622f0 Mon Sep 17 00:00:00 2001 From: Geoffrey Martin-Noble Date: Fri, 7 Aug 2020 15:03:00 -0700 Subject: [PATCH 0669/1017] Add license header to lit.site.cfg.py.in PiperOrigin-RevId: 325515949 Change-Id: I5d507dedfcc56b241b79025ed170be67e15cec11 --- .../compiler/mlir/hlo/tests/lit.site.cfg.py.in | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tensorflow/compiler/mlir/hlo/tests/lit.site.cfg.py.in b/tensorflow/compiler/mlir/hlo/tests/lit.site.cfg.py.in index 17b99e983f6..1555d314df0 100644 --- a/tensorflow/compiler/mlir/hlo/tests/lit.site.cfg.py.in +++ b/tensorflow/compiler/mlir/hlo/tests/lit.site.cfg.py.in @@ -1,3 +1,16 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + @LIT_SITE_CFG_IN_HEADER@ import sys From ba1341974543e05eeede31927c2662193b28951b Mon Sep 17 00:00:00 2001 From: Yanhua Sun Date: Fri, 7 Aug 2020 15:06:36 -0700 Subject: [PATCH 0670/1017] add defun benchmark with relaxed shape option PiperOrigin-RevId: 325516758 Change-Id: I9b284d5faea62862775de86e617967acd0ea8315 --- tensorflow/python/eager/benchmarks_test.py | 41 ++++++++++++++++++---- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py index 93766d809f2..22110e1ae71 100644 --- a/tensorflow/python/eager/benchmarks_test.py +++ b/tensorflow/python/eager/benchmarks_test.py @@ -481,14 +481,26 @@ class MicroBenchmarks(benchmarks_test_base.MicroBenchmarksBase): num_iters, execution_mode=None): - def func_matmul(m): + @def_function.function( + input_signature=[tensor_spec.TensorSpec([2, 2], dtypes.float32)]) + def defun_matmul(m): return math_ops.matmul(m, m) - f = function.defun( - func_matmul, - input_signature=[tensor_spec.TensorSpec([2, 2], dtypes.float32)]) + func = lambda: defun_matmul(m) + self._run(func, num_iters, execution_mode=execution_mode) - func = lambda: f(m) + def _benchmark_defun_matmul_relaxed_shape(self, + m, + num_iters, + execution_mode=None): + + @def_function.function(experimental_relax_shapes=True) + def defun_matmul(m): + return math_ops.matmul(m, m) + + m_3_by_3 = random_ops.random_uniform((3, 3)) + defun_matmul(m_3_by_3) + func = lambda: defun_matmul(m) self._run(func, num_iters, execution_mode=execution_mode) def _benchmark_defun_args_matmul(self, m, num_iters, execution_mode=None): @@ -591,12 +603,18 @@ class MicroBenchmarks(benchmarks_test_base.MicroBenchmarksBase): self._benchmark_defun_matmul( m, transpose_b=False, num_iters=self._num_iters_2_by_2) - def benchmark_defun_matmul_2_by_2_CPU_with_signature(self): + def benchmark_defun_matmul_2_by_2_with_signature_CPU(self): with context.device(CPU): m = self._m_2_by_2.cpu() self._benchmark_defun_matmul_with_signature( m, num_iters=self._num_iters_2_by_2) + def benchmark_defun_matmul_2_by_2_relaxed_shape_CPU(self): + with context.device(CPU): + m = self._m_2_by_2.cpu() + self._benchmark_defun_matmul_relaxed_shape( + m, num_iters=self._num_iters_2_by_2) + @test_util.disable_tfrt("Graph is not supported yet. b/156187905") def benchmark_defun_args_matmul_2_by_2_CPU(self): with context.device(CPU): @@ -678,7 +696,7 @@ class MicroBenchmarks(benchmarks_test_base.MicroBenchmarksBase): m, transpose_b=False, num_iters=self._num_iters_2_by_2) @test_util.disable_tfrt("copy to GPU not supported") - def benchmark_defun_matmul_2_by_2_GPU_with_signature(self): + def benchmark_defun_matmul_2_by_2_with_signature_GPU(self): if not context.num_gpus(): return with context.device(GPU): @@ -686,6 +704,15 @@ class MicroBenchmarks(benchmarks_test_base.MicroBenchmarksBase): self._benchmark_defun_matmul_with_signature( m, num_iters=self._num_iters_2_by_2) + @test_util.disable_tfrt("copy to GPU not supported") + def benchmark_defun_matmul_2_by_2_relaxed_shape_GPU(self): + if not context.num_gpus(): + return + with context.device(GPU): + m = self._m_2_by_2.gpu() + self._benchmark_defun_matmul_relaxed_shape( + m, num_iters=self._num_iters_2_by_2) + @test_util.disable_tfrt("Graph is not supported yet. b/156187905") def benchmark_defun_args_matmul_2_by_2_GPU(self): if not context.num_gpus(): From 7e16538d56497f0465b8a020221da0543f52454c Mon Sep 17 00:00:00 2001 From: Chuan He Date: Fri, 7 Aug 2020 15:08:16 -0700 Subject: [PATCH 0671/1017] Eliminating NOP GatherND or ScatterND if the indices value are from 0 to n-1, n is the first dimension of input tensor. PiperOrigin-RevId: 325517081 Change-Id: I45ec755f0678c913c8be82956b2de0c22474d675 --- .../compiler/mlir/lite/tests/optimize.mlir | 30 +++++++++++++++++++ .../compiler/mlir/lite/transforms/optimize.cc | 25 ++++++++++++++++ .../mlir/lite/transforms/optimize_patterns.td | 16 ++++++++++ 3 files changed, 71 insertions(+) diff --git a/tensorflow/compiler/mlir/lite/tests/optimize.mlir b/tensorflow/compiler/mlir/lite/tests/optimize.mlir index cf7fe07d729..7923c82ba92 100644 --- a/tensorflow/compiler/mlir/lite/tests/optimize.mlir +++ b/tensorflow/compiler/mlir/lite/tests/optimize.mlir @@ -1085,3 +1085,33 @@ func @ConvertPow2ToSquare(%arg0: tensor<2x2xf32>) -> tensor<2x2xf32> { // CHECK: return %[[RESULT]] } +func @ConvertIdentityGatherNdOp(%arg0: tensor<4x3xf32>) -> tensor<4x3xf32> { + %cst = constant dense<[[0], [1], [2], [3]]> : tensor<4x1xi32> + %0 = "tfl.gather_nd"(%arg0, %cst) : (tensor<4x3xf32>, tensor<4x1xi32>) -> tensor<4x3xf32> + return %0 : tensor<4x3xf32> + +// CHECK-LABEL: ConvertIdentityGatherNdOp +// CHECK-SAME: (%[[ARG:.*]]: tensor<4x3xf32>) -> tensor<4x3xf32> +// CHECK-NEXT: return %[[ARG]] : tensor<4x3xf32> +} + +func @ConvertIdentityGatherNdOp3D(%arg0: tensor<4x3x4xf32>) -> tensor<4x3x4xf32> { + %cst = constant dense<[[0], [1], [2], [3]]> : tensor<4x1xi32> + %0 = "tfl.gather_nd"(%arg0, %cst) : (tensor<4x3x4xf32>, tensor<4x1xi32>) -> tensor<4x3x4xf32> + return %0 : tensor<4x3x4xf32> + +// CHECK-LABEL: ConvertIdentityGatherNdOp3D +// CHECK-SAME: (%[[ARG:.*]]: tensor<4x3x4xf32>) -> tensor<4x3x4xf32> +// CHECK-NEXT: return %[[ARG]] : tensor<4x3x4xf32> +} + +func @ConvertIdentityScatterNd(%arg0: tensor<4x3xf32>) -> tensor<4x3xf32> { + %cst = constant dense<[[0], [1], [2], [3]]> : tensor<4x1xi32> + %shape = constant dense<[4, 3]> : tensor<2xi32> + %0 = "tfl.scatter_nd"(%cst, %arg0, %shape) : (tensor<4x1xi32>, tensor<4x3xf32>, tensor<2xi32>) -> tensor<4x3xf32> + return %0 : tensor<4x3xf32> + +// CHECK-LABEL: ConvertIdentityScatterNd +// CHECK-SAME: (%[[ARG:.*]]: tensor<4x3xf32>) -> tensor<4x3xf32> +// CHECK-NEXT: return %[[ARG]] : tensor<4x3xf32> +} diff --git a/tensorflow/compiler/mlir/lite/transforms/optimize.cc b/tensorflow/compiler/mlir/lite/transforms/optimize.cc index 6de6187d81a..eeecfac67cf 100644 --- a/tensorflow/compiler/mlir/lite/transforms/optimize.cc +++ b/tensorflow/compiler/mlir/lite/transforms/optimize.cc @@ -160,6 +160,31 @@ bool CanFuseConvOrDepthwiseConv(Attribute filter, Attribute val, return false; } +// Retuns true if we can eliminate the GatherNdOp or ScatterNdOp. When the value +// of `indices` are from 0 to n-1, the output tensor are identical to the +// `params`. +bool CanOptimizeIdentityGatherNdOrScatterNdOp(Value params, + DenseIntElementsAttr indices) { + auto params_type = params.getType().dyn_cast(); + auto indices_type = indices.getType().dyn_cast(); + // Checks the shape of `params` is [n, ...], shape of `indices` is [n, 1]. 2D + // `indices` means it gets the first row of `params`. As long as indices + // iterate the first row of `params`, the output is identical to input. + if (!params_type || !indices_type || indices_type.getRank() != 2 || + indices_type.getDimSize(0) != params_type.getDimSize(0) || + indices_type.getDimSize(1) != 1) + return false; + + // Checks the value in `indices` is from 0 to n-1. + int cur_value = 0; + for (const auto &v : indices.getValues()) { + if (v.getSExtValue() != cur_value) return false; + ++cur_value; + } + + return true; +} + // Expand Attribute 'a' to 4D with all 1s except 1 dimension. // Which dimension depends on 'is_depthwise' is true or false. ElementsAttr ExpandTo4DForConvImpl(Attribute a, bool is_depthwise) { diff --git a/tensorflow/compiler/mlir/lite/transforms/optimize_patterns.td b/tensorflow/compiler/mlir/lite/transforms/optimize_patterns.td index ef6706875c9..3c5fc7a0c5e 100644 --- a/tensorflow/compiler/mlir/lite/transforms/optimize_patterns.td +++ b/tensorflow/compiler/mlir/lite/transforms/optimize_patterns.td @@ -520,3 +520,19 @@ def OptimizePow2ToSquare : Pat< (TFL_PowOp $input, (ConstantOp ConstantAttr, "2.0f">)), (TFL_MulOp $input, $input, TFL_AF_None)>; + +def CanOptimizeIdentityGatherNdOrScatterNdOp : Constraint())">>; + +def OptimizeIdentityGatherNdOp : Pat< + (TFL_GatherNdOp $params, (ConstantOp I32ElementsAttr: $indices)), + (replaceWithValue $params), + [(CanOptimizeIdentityGatherNdOrScatterNdOp $params, $indices)]>; + +def OptimizeIdentityScatterNdOp : Pat< + (TFL_ScatterNdOp (ConstantOp I32ElementsAttr: $indices), $params, $ignored), + (replaceWithValue $params), + [(CanOptimizeIdentityGatherNdOrScatterNdOp $params, $indices)]>; + + From cf8e65f21206ed48f8c87e5bb17d2326d4414c7d Mon Sep 17 00:00:00 2001 From: Xiao Yu Date: Fri, 7 Aug 2020 15:23:18 -0700 Subject: [PATCH 0672/1017] Update the namespaces of TF-TFRT integration. All TF-TFRT integration code should under namespace tfrt::tf. PiperOrigin-RevId: 325519742 Change-Id: I71a2abd0d209956dfd3ecb6a146769b29048b14f --- tensorflow/c/eager/c_api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc index 76d603694e3..fefa753c608 100644 --- a/tensorflow/c/eager/c_api.cc +++ b/tensorflow/c/eager/c_api.cc @@ -724,7 +724,7 @@ void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; } TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) { if (opts->use_tfrt) { #ifdef PLATFORM_GOOGLE - return tensorflow::wrap(new tfrt::ContextInterface(opts->async)); + return tensorflow::wrap(new tfrt::tf::ContextInterface(opts->async)); #else status->status = tensorflow::errors::Unimplemented("TFRT is not supported"); return nullptr; From 9091c70c38aad8adb1ee434e5b3c9fe61fdfd130 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 15:27:00 -0700 Subject: [PATCH 0673/1017] Added Softmax to the estimator. PiperOrigin-RevId: 325520405 Change-Id: I6d783f69cec11d0e8c82ee9fd385f57d6b46eb39 --- .../costs/analytical_cost_estimator_test.cc | 3 +- .../grappler/costs/op_level_cost_estimator.cc | 28 +++++++++++++++++++ .../grappler/costs/op_level_cost_estimator.h | 1 + .../costs/op_level_cost_estimator_test.cc | 9 +++--- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc index e558558d00a..b23b657308d 100644 --- a/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc +++ b/tensorflow/core/grappler/costs/analytical_cost_estimator_test.cc @@ -102,14 +102,13 @@ TEST_F(AnalyticalCostEstimatorTest, SimpleTest) { Costs summary; TF_ASSERT_OK(estimator.PredictCosts(item.graph, &run_metadata, &summary)); - EXPECT_EQ(Costs::NanoSeconds(9157), summary.execution_time); + EXPECT_EQ(Costs::NanoSeconds(9158), summary.execution_time); // Note there are totally 17 nodes (RandomUniform creates 2 nodes), but // grappler will not process "label", therefore we have 15 here instead EXPECT_EQ(15, summary.num_ops_total); // Make this estimate accurate: // TODO(http://b/70031255): Accurate estimator for RandomUniform op needed - // TODO(http://b/70031363): Accurate estimator for Softmax needed // // Change to EXPECT_FALSE when the above TODOs are done: EXPECT_TRUE(summary.inaccurate); diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc index d76ff4359c1..e148f6a61c8 100644 --- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc +++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc @@ -95,6 +95,7 @@ constexpr char kFusedBatchNormGrad[] = "FusedBatchNormGrad"; constexpr char kQuantizedMatMul[] = "QuantizedMatMul"; constexpr char kQuantizedMatMulV2[] = "QuantizedMatMulV2"; constexpr char kUnpack[] = "Unpack"; +constexpr char kSoftmax[] = "Softmax"; // Dynamic control flow ops. constexpr char kSwitch[] = "Switch"; constexpr char kMerge[] = "Merge"; @@ -503,6 +504,8 @@ OpLevelCostEstimator::OpLevelCostEstimator() { device_cost_impl_.emplace( kFusedBatchNormGrad, wrap(&OpLevelCostEstimator::PredictFusedBatchNormGrad)); + device_cost_impl_.emplace(kSoftmax, + wrap(&OpLevelCostEstimator::PredictSoftmax)); device_cost_impl_.emplace( kAssignVariableOp, wrap(&OpLevelCostEstimator::PredictAssignVariableOps)); device_cost_impl_.emplace( @@ -2287,5 +2290,30 @@ Costs OpLevelCostEstimator::PredictNaryOp(const OpContext& op_context) const { costs.num_ops_with_unknown_shapes = found_unknown_shapes; return costs; } + +// softmax[i, j] = exp(logits[i, j]) / sum_j(exp(logits[i, j])) +Costs OpLevelCostEstimator::PredictSoftmax(const OpContext& op_context) const { + bool found_unknown_shapes = false; + const int64 logits_size = CalculateTensorElementCount( + op_context.op_info.inputs(0), &found_unknown_shapes); + TensorShapeProto logits_shape = MaybeGetMinimumShape( + op_context.op_info.inputs(0).shape(), 2, &found_unknown_shapes); + +#define EIGEN_COST(X) Eigen::internal::functor_traits::Cost + + // Every element of will be exponentiated, have that result included + // in a sum across j, and also have that result multiplied by the reciprocal + // of the sum_j. In addition, we'll compute 1/sum_j for every i. + auto ops = + (EIGEN_COST(scalar_exp_op) + EIGEN_COST(scalar_sum_op) + + EIGEN_COST(scalar_product_op)) * + logits_size + + EIGEN_COST(scalar_inverse_op) * logits_shape.dim(0).size(); + +#undef EIGEN_COST + + return PredictOpCountBasedCost(ops, op_context.op_info); +} + } // end namespace grappler } // end namespace tensorflow diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.h b/tensorflow/core/grappler/costs/op_level_cost_estimator.h index f44f4ee19e5..be0d7f76621 100644 --- a/tensorflow/core/grappler/costs/op_level_cost_estimator.h +++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.h @@ -88,6 +88,7 @@ class OpLevelCostEstimator { Costs PredictEinsum(const OpContext& op_context) const; Costs PredictAssignVariableOps(const OpContext& op_context) const; Costs PredictPureMemoryOp(const OpContext& op_context) const; + Costs PredictSoftmax(const OpContext& op_context) const; // Generic cost prediction method for fused operations. Costs PredictFusedOp(const OpContext& op_context, diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc index c5209753a90..5ddefdc9602 100644 --- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc +++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc @@ -966,8 +966,9 @@ TEST_F(OpLevelCostEstimatorTest, SquaredDifferenceExecutionTime) { TEST_F(OpLevelCostEstimatorTest, UnaryOpExecutionTime) { std::vector> unary_ops = { - {"All", 1}, {"ArgMax", 1}, {"Cast", 1}, {"Max", 1}, {"Min", 1}, - {"Prod", 1}, {"Relu", 1}, {"Relu6", 1}, {"Sum", 1}, {"TopKV2", 1}}; + {"All", 1}, {"ArgMax", 1}, {"Cast", 1}, {"Max", 1}, + {"Min", 1}, {"Prod", 1}, {"Relu", 1}, {"Relu6", 1}, + {"Softmax", 43}, {"Sum", 1}, {"TopKV2", 1}}; const int kTensorSize = 1000; for (auto unary_op : unary_ops) { @@ -980,7 +981,8 @@ TEST_F(OpLevelCostEstimatorTest, UnaryOpExecutionTime) { auto cost = PredictCosts(op_context); EXPECT_EQ(cost.memory_time, Costs::Duration(kExpectedMemoryTime)); - EXPECT_EQ(cost.compute_time, Costs::Duration(expected_compute_time)); + EXPECT_EQ(cost.compute_time, Costs::Duration(expected_compute_time)) + << unary_op.first; EXPECT_EQ(cost.execution_time, Costs::Duration(expected_compute_time + kExpectedMemoryTime)); EXPECT_EQ(cost.num_ops_total, 1); @@ -1972,6 +1974,5 @@ TEST_F(OpLevelCostEstimatorTest, PureMemoryOpExecutionTime) { EXPECT_EQ(0, cost.num_ops_with_unknown_shapes); } } - } // end namespace grappler } // end namespace tensorflow From 6848f640974107c40ce0d53a233dffee122b2a85 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Fri, 7 Aug 2020 15:41:10 -0700 Subject: [PATCH 0674/1017] Run DimensionSizeRewriter and ZeroSizedHloElimination as part of the GPU simplification fixed-point pass pipeline. Looks like there was a typo here and we were running these two passes only once, rather than as part of the fixed-point pass pipeline. While we're here, fix a comment in algebraic_simplifier as well. PiperOrigin-RevId: 325522950 Change-Id: Ic716b4831c7ac5b96234d496b0890c3b312528b7 --- tensorflow/compiler/xla/service/algebraic_simplifier.cc | 4 ++-- tensorflow/compiler/xla/service/gpu/gpu_compiler.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc index c793c4958a2..fa4d0e47a5d 100755 --- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc +++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc @@ -4161,8 +4161,8 @@ Status AlgebraicSimplifierVisitor::HandleDynamicSlice( return ReplaceWithNewInstruction(dynamic_slice, std::move(new_broadcast)); } - // Convert a dynamic slice into a slice if all offsets are constant and the - // operand is not constant. If ev + // Convert a dynamic slice into a slice if all offsets are constant and the + // operand is not constant. if (operand->opcode() != HloOpcode::kConstant && absl::c_all_of(absl::MakeSpan(dynamic_slice->operands().begin() + 1, dynamic_slice->operands().end()), diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index f2d29b5d11f..6d441903b25 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -190,11 +190,11 @@ Status GpuCompiler::OptimizeHloModule( /*layout_sensitive=*/false, /*allow_mixed_precision=*/false); - pipeline.AddPass(); + pass.AddPass(); // BatchNormExpander can create zero-sized ops, so zero-sized HLO // elimination has to come after that pass. - pipeline.AddPass(); + pass.AddPass(); AlgebraicSimplifierOptions options; // When transposes appear in a fusion node, we can easily adjust the From f6cfee3dff8fbc2ab94667cf54545fadab378f0f Mon Sep 17 00:00:00 2001 From: Reed Wanderman-Milne Date: Fri, 7 Aug 2020 15:41:34 -0700 Subject: [PATCH 0675/1017] Fix Windows cwise_ops_test.py bfloat16 failure. Test was broken by https://github.com/tensorflow/tensorflow/commit/696a4a76cebdd9150ede235a6edfd223a64e7129 PiperOrigin-RevId: 325523032 Change-Id: Ieabe77e2cf53e4e4e81530ededbb55a4b3120c6a --- tensorflow/python/kernel_tests/cwise_ops_test.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py index 8d628d448db..a7d8f841401 100644 --- a/tensorflow/python/kernel_tests/cwise_ops_test.py +++ b/tensorflow/python/kernel_tests/cwise_ops_test.py @@ -840,14 +840,16 @@ class MathOpsOverloadTest(test.TestCase): return self.evaluate(z) def _compareBinary(self, x, y, dtype, np_func, tf_func): - np_ans = np_func(x, y).astype(dtype.as_numpy_dtype) - if dtype == dtypes_lib.bfloat16: - # assertAllClose does not properly handle bfloat16 values - np_ans = np_ans.astype(np.float32) + # astype and assertAllClose do not properly handle bfloat16 values + np_ans = np_func(x, y).astype(np.float32 if dtype == dtypes_lib.bfloat16 + else dtype.as_numpy_dtype) + rtol = 1e-2 if dtype == dtypes_lib.bfloat16 else 1e-6 self.assertAllClose(np_ans, - self._computeTensorAndLiteral(x, y, dtype, tf_func)) + self._computeTensorAndLiteral(x, y, dtype, tf_func), + rtol=rtol) self.assertAllClose(np_ans, - self._computeLiteralAndTensor(x, y, dtype, tf_func)) + self._computeLiteralAndTensor(x, y, dtype, tf_func), + rtol=rtol) def _compareUnary(self, x, dtype, np_func, tf_func): np_ans = np_func(x).astype(dtype.as_numpy_dtype) From 0d2f665129f15e45384dc455afecdaf2ac9b256b Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 7 Aug 2020 15:58:52 -0700 Subject: [PATCH 0676/1017] Add APIs for Buffer Linearization Support in TPUs PiperOrigin-RevId: 325525811 Change-Id: I964daa0ba93bba5707c4714ac8de3564bba58c06 --- tensorflow/core/tpu/tpu_library_init_fns.inc | 2 ++ tensorflow/stream_executor/tpu/tpu_executor_c_api.h | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/tensorflow/core/tpu/tpu_library_init_fns.inc b/tensorflow/core/tpu/tpu_library_init_fns.inc index bc93b737eb5..be9d594685e 100644 --- a/tensorflow/core/tpu/tpu_library_init_fns.inc +++ b/tensorflow/core/tpu/tpu_library_init_fns.inc @@ -161,6 +161,8 @@ tensorflow::Status SetExecutorStructFn(void* library_handle) { TFTPU_SET_FN(executor_fn, TpuTransferManager_TransferLiteralFromDevice); TFTPU_SET_FN(executor_fn, TpuTransferManager_GetByteSizeRequirement); TFTPU_SET_FN(executor_fn, TpuTransferManager_WriteSingleTupleIndexTable); + TFTPU_SET_FN(executor_fn, TpuTransferManager_LinearizeToBuffers); + TFTPU_SET_FN(executor_fn, TpuTransferManager_FreeBuffers); TFTPU_SET_FN(executor_fn, TpuComputationPlacer_New); TFTPU_SET_FN(executor_fn, TpuComputationPlacer_Free); diff --git a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h index a67fc9ddf61..2b66c2ce4c5 100644 --- a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h +++ b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h @@ -182,6 +182,11 @@ void TpuTransferManager_WriteSingleTupleIndexTable( XLA_TransferManager* manager, SE_Stream* stream, SE_DeviceMemoryBase* elements, size_t elements_len, XLA_Shape* shape, SE_DeviceMemoryBase* region, SE_Status* status); +void TpuTransferManager_LinearizeToBuffers( + XLA_TransferManager* manager, XLA_Literal* c_literal, char*** buffers_array, + int64_t** buffers_size, int64_t* buffers_array_size, SE_Status* status); +void TpuTransferManager_FreeBuffers(char** buffers_array, int64_t* buffers_size, + int64_t buffers_array_size); XLA_ComputationPlacer* TpuComputationPlacer_New(); void TpuComputationPlacer_Free(XLA_ComputationPlacer* placer); @@ -336,6 +341,8 @@ struct TfTpu_ExecutorApiFn { TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_TransferLiteralFromDevice); TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_GetByteSizeRequirement); TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_WriteSingleTupleIndexTable); + TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_LinearizeToBuffers); + TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_FreeBuffers); TFTPU_ADD_FN_IN_STRUCT(TpuComputationPlacer_New); TFTPU_ADD_FN_IN_STRUCT(TpuComputationPlacer_Free); From 6e909825ed44655636170c739d43b6030c742201 Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Fri, 7 Aug 2020 16:01:44 -0700 Subject: [PATCH 0677/1017] Expand simple gathers into dynamic-slice. Especially for (GPU) fusion, XLA prefers to call a slice a slice. PiperOrigin-RevId: 325526316 Change-Id: I12b98756eca017d520a9a40d03dc291a42b9eaa3 --- tensorflow/compiler/xla/service/BUILD | 1 + tensorflow/compiler/xla/service/cpu/BUILD | 1 + .../compiler/xla/service/cpu/cpu_compiler.cc | 2 + .../compiler/xla/service/gather_expander.cc | 31 +++++++--- .../compiler/xla/service/gather_expander.h | 27 +++++++- .../xla/service/gather_expander_test.cc | 62 ++++++++++++++++++- tensorflow/compiler/xla/service/gpu/BUILD | 1 + .../compiler/xla/service/gpu/gpu_compiler.cc | 3 + 8 files changed, 114 insertions(+), 14 deletions(-) diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index 49431b19a69..bfcdf6fae34 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -2259,6 +2259,7 @@ tf_cc_test( srcs = ["gather_expander_test.cc"], deps = [ ":gather_expander", + ":hlo_query", "//tensorflow/compiler/xla:test", "//tensorflow/compiler/xla/tests:hlo_test_base", "//tensorflow/compiler/xla/tests:test_macros_header", diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index 6eaf43902fe..e0317574e59 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -183,6 +183,7 @@ cc_library( "//tensorflow/compiler/xla/service:hlo_verifier", "//tensorflow/compiler/xla/service:indexed_array_analysis", "//tensorflow/compiler/xla/service:llvm_compiler", + "//tensorflow/compiler/xla/service:gather_expander", "//tensorflow/compiler/xla/service:reshape_mover", "//tensorflow/compiler/xla/service:rng_expander", "//tensorflow/compiler/xla/service:sort_simplifier", diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index 0826d7b8ce1..eb5d9e704f5 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -77,6 +77,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/dynamic_index_splitter.h" #include "tensorflow/compiler/xla/service/dynamic_padder.h" #include "tensorflow/compiler/xla/service/flatten_call_graph.h" +#include "tensorflow/compiler/xla/service/gather_expander.h" #include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_computation.h" #include "tensorflow/compiler/xla/service/hlo_constant_folding.h" @@ -303,6 +304,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn( pass.AddPass(options); pass.AddPass(); pass.AddPass(); + pass.AddPass(GatherExpander::kEliminateSimpleGathers); // BatchNormExpander can create zero-sized ops, so zero-sized HLO // elimination has to come after that pass. diff --git a/tensorflow/compiler/xla/service/gather_expander.cc b/tensorflow/compiler/xla/service/gather_expander.cc index 1838f65e6ea..d38873a501d 100644 --- a/tensorflow/compiler/xla/service/gather_expander.cc +++ b/tensorflow/compiler/xla/service/gather_expander.cc @@ -269,6 +269,22 @@ static StatusOr PermuteBatchAndOffsetDims( return MakeTransposeHlo(accumulator, permutation); } +// Computes how many trips a loop implementing this gather op would take. +static int64 GatherLoopTripCount(HloInstruction* gather_instr) { + HloInstruction* start_indices = gather_instr->mutable_operand(1); + const Shape& start_indices_shape = start_indices->shape(); + const GatherDimensionNumbers& dim_numbers = + gather_instr->gather_dimension_numbers(); + + int64 trip_count = 1; + for (int64 i = 0, e = start_indices_shape.dimensions_size(); i < e; i++) { + if (i != dim_numbers.index_vector_dim()) { + trip_count *= start_indices_shape.dimensions(i); + } + } + return trip_count; +} + // High Level Algorithm // // We follow the following steps in sequence: @@ -311,20 +327,13 @@ StatusOr GatherExpander::ExpandInstruction( HloComputation* computation = gather_instr->parent(); HloInstruction* operand = gather_instr->mutable_operand(0); HloInstruction* start_indices = gather_instr->mutable_operand(1); - const Shape& start_indices_shape = start_indices->shape(); const Shape& output_shape = gather_instr->shape(); int64 output_rank = output_shape.dimensions_size(); const GatherDimensionNumbers& dim_numbers = gather_instr->gather_dimension_numbers(); - int64 gather_loop_trip_count = 1; - for (int64 i = 0, e = start_indices_shape.dimensions_size(); i < e; i++) { - if (i != dim_numbers.index_vector_dim()) { - gather_loop_trip_count *= start_indices_shape.dimensions(i); - } - } - + int64 gather_loop_trip_count = GatherLoopTripCount(gather_instr); if (!IsInt32(gather_loop_trip_count)) { return Unimplemented( "Gather operations with more than 2147483647 gather indices are not " @@ -373,7 +382,11 @@ bool GatherExpander::InstructionMatchesPattern(HloInstruction* inst) { return inst->opcode() == HloOpcode::kGather && // Avoid expanding gather ops that produce zero sized tensors, // instead punt these to ZeroSizedHloElimination. - !ShapeUtil::IsZeroElementArray(inst->shape()); + !ShapeUtil::IsZeroElementArray(inst->shape()) && + // In kEliminateSimpleGathers mode, we only simplify instructions + // which can be represented without a loop -- i.e. we only simplify + // gathers which have a trip count of 1. + (mode_ == kEliminateAllGathers || GatherLoopTripCount(inst) == 1); } } // namespace xla diff --git a/tensorflow/compiler/xla/service/gather_expander.h b/tensorflow/compiler/xla/service/gather_expander.h index 5625a37cb46..e665fcd713c 100644 --- a/tensorflow/compiler/xla/service/gather_expander.h +++ b/tensorflow/compiler/xla/service/gather_expander.h @@ -21,10 +21,30 @@ limitations under the License. namespace xla { // This pass rewrites gather operations into (roughly) while loops of dynamic -// slices. This lets backends that don't support gather directly to -// nevertheless have a minimum level of support. +// slices. +// +// This pass can be used two ways: +// +// - kEliminateAllGathers: For backends that don't support gather, this pass +// can convert every gather to a loop. +// +// - kEliminateSimpleGathers: For backends that *do* support gather, this pass +// can strength-reduce "simple" gathers -- specifically, gathers that can be +// represented without a loop -- to dyanmic-slices. +// +// Note that even in kEliminateSimpleGathers mode, this pass may still expand a +// gather into a loop (with a trip-count of 1). It's up to other simplification +// passes to remove the loop. +// class GatherExpander : public OpExpanderPass { public: + enum Mode { + kEliminateAllGathers, + kEliminateSimpleGathers, + }; + + explicit GatherExpander(Mode m) : mode_(m) {} + absl::string_view name() const override { return "gather_expander"; } protected: @@ -32,6 +52,9 @@ class GatherExpander : public OpExpanderPass { StatusOr ExpandInstruction( HloInstruction* gather_inst) override; + + private: + Mode mode_; }; } // namespace xla diff --git a/tensorflow/compiler/xla/service/gather_expander_test.cc b/tensorflow/compiler/xla/service/gather_expander_test.cc index 706327091d9..4b0808e9aaf 100644 --- a/tensorflow/compiler/xla/service/gather_expander_test.cc +++ b/tensorflow/compiler/xla/service/gather_expander_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/gather_expander.h" +#include "tensorflow/compiler/xla/service/hlo_query.h" #include "tensorflow/compiler/xla/test.h" #include "tensorflow/compiler/xla/tests/hlo_test_base.h" #include "tensorflow/compiler/xla/tests/test_macros.h" @@ -42,7 +43,9 @@ ENTRY main { TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, ParseAndReturnVerifiedModule(hlo_text)); - Status status = GatherExpander{}.Run(module.get()).status(); + Status status = GatherExpander{GatherExpander::kEliminateAllGathers} + .Run(module.get()) + .status(); EXPECT_EQ(status.code(), tensorflow::error::UNIMPLEMENTED); ASSERT_THAT( @@ -68,7 +71,9 @@ ENTRY main { )"; TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, ParseAndReturnVerifiedModule(hlo_text)); - TF_ASSERT_OK_AND_ASSIGN(bool changed, GatherExpander{}.Run(module.get())); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, + GatherExpander{GatherExpander::kEliminateAllGathers}.Run(module.get())); ASSERT_TRUE(changed); HloInstruction* while_instr = nullptr; @@ -129,7 +134,9 @@ ENTRY main { OpMetadata metadata; metadata.set_op_name("Gather"); module->entry_computation()->root_instruction()->set_metadata(metadata); - TF_ASSERT_OK_AND_ASSIGN(bool changed, GatherExpander{}.Run(module.get())); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, + GatherExpander{GatherExpander::kEliminateAllGathers}.Run(module.get())); ASSERT_TRUE(changed); HloInstruction* while_instr = nullptr; @@ -147,5 +154,54 @@ ENTRY main { "after gather expansion"; EXPECT_EQ(while_instr->metadata().op_name(), "Gather"); } + +TEST_F(GatherExpanderTest, EliminateSimpleGathersSkipsNontrivialGather) { + const string hlo_text = R"( +HloModule TensorFlowGatherV1 + +ENTRY main { + operand = s32[3,3] parameter(0) + indices = s32[2] parameter(1) + ROOT gather = s32[2,3] gather(operand, indices), + offset_dims={1}, + collapsed_slice_dims={0}, + start_index_map={0}, + index_vector_dim=1, + slice_sizes={1, 3} +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(hlo_text)); + GatherExpander pass(GatherExpander::kEliminateSimpleGathers); + TF_ASSERT_OK_AND_ASSIGN(bool changed, RunHloPass(&pass, module.get())); + ASSERT_FALSE(changed); +} + +TEST_F(GatherExpanderTest, EliminateSimpleGathersRewritesTrivialGather) { + const string hlo_text = R"( +HloModule test + +ENTRY main { + operand = s32[100] parameter(0) + indices = s32[1] parameter(1) + ROOT gather = s32[10] gather(operand, indices), + offset_dims={0}, + collapsed_slice_dims={}, + start_index_map={0}, + index_vector_dim=0, + slice_sizes={10} +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr module, + ParseAndReturnVerifiedModule(hlo_text)); + GatherExpander pass(GatherExpander::kEliminateAllGathers); + TF_ASSERT_OK_AND_ASSIGN(bool changed, RunHloPass(&pass, module.get())); + ASSERT_TRUE(changed); + ASSERT_FALSE(hlo_query::ContainsInstrWithOpcode(module->entry_computation(), + {HloOpcode::kGather})); +} + } // namespace } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 8dfd73e9a6a..47af5756f87 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -1177,6 +1177,7 @@ cc_library( "//tensorflow/compiler/xla/service:dynamic_padder", "//tensorflow/compiler/xla/service:executable", "//tensorflow/compiler/xla/service:flatten_call_graph", + "//tensorflow/compiler/xla/service:gather_expander", "//tensorflow/compiler/xla/service:hlo", "//tensorflow/compiler/xla/service:hlo_constant_folding", "//tensorflow/compiler/xla/service:hlo_cse", diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index 6d441903b25..225fa328f3d 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -43,6 +43,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/dynamic_index_splitter.h" #include "tensorflow/compiler/xla/service/dynamic_padder.h" #include "tensorflow/compiler/xla/service/flatten_call_graph.h" +#include "tensorflow/compiler/xla/service/gather_expander.h" #include "tensorflow/compiler/xla/service/gpu/alias_passthrough_params.h" #include "tensorflow/compiler/xla/service/gpu/cudnn_batchnorm_rewriter.h" #include "tensorflow/compiler/xla/service/gpu/fusion_merger.h" @@ -196,6 +197,8 @@ Status GpuCompiler::OptimizeHloModule( // elimination has to come after that pass. pass.AddPass(); + pass.AddPass(GatherExpander::kEliminateSimpleGathers); + AlgebraicSimplifierOptions options; // When transposes appear in a fusion node, we can easily adjust the // multi-dimensional index to create the one needed for the operand. This From a8dad139b6a7ecce56cfad1a2838e31d4870de1e Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Fri, 7 Aug 2020 16:02:09 -0700 Subject: [PATCH 0678/1017] Avoid setting the leading dimension in TensorList slice to dynamic. The first dimension (which is always 1) has to be static. PiperOrigin-RevId: 325526392 Change-Id: I073d01b35267b9c5f3e7fbd4986cd5bdd3151d67 --- tensorflow/compiler/tf2xla/kernels/tensor_list_utils.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.cc index aa71e4d4364..0e367e10ec4 100644 --- a/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.cc +++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.cc @@ -504,7 +504,9 @@ Status ExecuteTensorListGetItem(xla::XlaOp list, xla::XlaOp index, xla::XlaOp list_part = xla::GetTupleElement(list, 0); xla::XlaOp read = xla::DynamicSlice(list_part, start_indices, slice_shape); - for (int64 i = 0; i < buffer_shape.dimensions_size(); ++i) { + // Propagate dynamic dimensions from buffer to the sliced buffer, except for + // leading dimension (which is always static 1). + for (int64 i = 1; i < buffer_shape.dimensions_size(); ++i) { if (buffer_shape.is_dynamic_dimension(i)) { auto buffer = xla::GetTupleElement(list, 0); auto gds = xla::GetDimensionSize(buffer, i); From d1e617ded2a7b172c89529b65a646eb7c9a86d37 Mon Sep 17 00:00:00 2001 From: Kuangyuan Chen Date: Fri, 7 Aug 2020 16:14:53 -0700 Subject: [PATCH 0679/1017] Pass `upgrade_legacy` cmd line flag to signaturedef importer in tf-mlir-translate. PiperOrigin-RevId: 325528613 Change-Id: I43585f35c9a7038bd0c094d3ffb68df8103a3a9f --- tensorflow/compiler/mlir/tf_mlir_translate_main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/tf_mlir_translate_main.cc b/tensorflow/compiler/mlir/tf_mlir_translate_main.cc index 8cfdfd01120..caac8ea1eeb 100644 --- a/tensorflow/compiler/mlir/tf_mlir_translate_main.cc +++ b/tensorflow/compiler/mlir/tf_mlir_translate_main.cc @@ -121,7 +121,7 @@ int main(int argc, char** argv) { mlir::MLIRContext context; auto module_or = tensorflow::SavedModelSignatureDefsToMlirImport( - input_filename, tags, exported_names, &context); + input_filename, tags, exported_names, &context, upgrade_legacy); if (!module_or.status().ok()) return 1; module_or.ConsumeValueOrDie()->print(output->os()); From 83d7587a56179bdf70d2cf3cfa21fefa887d0dc5 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Fri, 7 Aug 2020 23:20:23 +0000 Subject: [PATCH 0680/1017] resolve map_kernel conflicts --- tensorflow/core/kernels/map_kernels.cc | 3 ++- tensorflow/core/kernels/map_kernels.h | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index 26f67c576e5..19b79a242b1 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -52,4 +52,5 @@ REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_CPU, REGISTER_UNARY_VARIANT_UNARY_OP_FUNCTION(ZEROS_LIKE_VARIANT_UNARY_OP, DEVICE_CPU, TensorMap, TensorMapZerosLike); -} + +} // namespace tensorflow diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 5031ad969f4..4e2618bd08c 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -20,8 +20,6 @@ limitations under the License. #include "tensorflow/core/framework/variant_encode_decode.h" #include "tensorflow/core/util/tensor_ops_util.h" -#include - namespace tensorflow { @@ -214,6 +212,7 @@ Status TensorMapZerosLike(OpKernelContext* c, const TensorMap& x, TensorMap* y) return Status::OK(); } + } // namespace tensorflow #endif // TENSORFLOW_CORE_KERNELS_MAP_KERNELS_H_ From ff457d4d01479627a443c4999b0726dc8bbf9d7f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 16:29:36 -0700 Subject: [PATCH 0681/1017] Add bzl_library rules for .bzl files without one. PiperOrigin-RevId: 325530970 Change-Id: Ia5f631fc056830ec36de176143e1ee58a6284571 --- tensorflow/lite/micro/BUILD | 10 +++++++++- tensorflow/lite/micro/testing/BUILD | 8 ++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/micro/BUILD b/tensorflow/lite/micro/BUILD index 9b3d0d623cc..7cec8584413 100644 --- a/tensorflow/lite/micro/BUILD +++ b/tensorflow/lite/micro/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load( "//tensorflow/lite/micro/testing:micro_test.bzl", "tflite_micro_cc_test", @@ -43,13 +44,13 @@ cc_library( deps = [ ":memory_helpers", ":micro_compatibility", + ":micro_profiler", ":op_resolvers", "//tensorflow/lite:type_to_tflitetype", "//tensorflow/lite/c:common", "//tensorflow/lite/core/api", "//tensorflow/lite/kernels/internal:compatibility", "//tensorflow/lite/kernels/internal:tensor", - "//tensorflow/lite/micro:micro_profiler", "//tensorflow/lite/micro/memory_planner", "//tensorflow/lite/micro/memory_planner:greedy_memory_planner", "//tensorflow/lite/schema:schema_fbs", @@ -379,3 +380,10 @@ tflite_micro_cc_test( "//tensorflow/lite/micro/testing:test_conv_model", ], ) + +bzl_library( + name = "build_def_bzl", + srcs = ["build_def.bzl"], + visibility = [":micro"], + deps = ["//tensorflow:tensorflow_bzl"], +) diff --git a/tensorflow/lite/micro/testing/BUILD b/tensorflow/lite/micro/testing/BUILD index 6f4b2502f4a..207d500c53d 100644 --- a/tensorflow/lite/micro/testing/BUILD +++ b/tensorflow/lite/micro/testing/BUILD @@ -1,3 +1,4 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load( "//tensorflow/lite/micro/testing:micro_test.bzl", "tflite_micro_cc_test", @@ -78,3 +79,10 @@ py_binary( "@absl_py//absl:app", ], ) + +bzl_library( + name = "micro_test_bzl", + srcs = ["micro_test.bzl"], + visibility = ["//visibility:private"], + deps = ["//tensorflow/lite/micro:build_def_bzl"], +) From 2e3e2bb33559bef5b76cb1e5bc745a75488efaff Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Fri, 7 Aug 2020 16:36:02 -0700 Subject: [PATCH 0682/1017] [XLA] Use dynamism inference to infer dynamic dimensions for reshape. - Introduce dynamism inference function in xla builder, which tells if a value is dynamic or static. - Use dynamism inference to infer whether an input to reshape's dimensions is dynamic. - This removes the "-1" hack I made before in the bridge, makes the code cleaner. Plus it can support more complex cases dynamic reshape when the dimension comes from a series of transformations. PiperOrigin-RevId: 325532056 Change-Id: Icc5bad39a857be77537e4736dd6863b833e2fe9d --- .../compiler/tf2xla/kernels/reshape_op.cc | 38 +-- tensorflow/compiler/tf2xla/xla_expression.cc | 42 ++++ tensorflow/compiler/tf2xla/xla_expression.h | 4 + tensorflow/compiler/tf2xla/xla_op_kernel.cc | 42 ++++ tensorflow/compiler/tf2xla/xla_op_kernel.h | 3 + tensorflow/compiler/xla/client/xla_builder.cc | 238 ++++++++++++++++++ tensorflow/compiler/xla/client/xla_builder.h | 25 ++ .../service/dynamic_dimension_inference.cc | 3 +- tensorflow/compiler/xla/shape_util.cc | 15 +- tensorflow/compiler/xla/tests/BUILD | 25 ++ .../xla/tests/dynamism_inference_test.cc | 215 ++++++++++++++++ 11 files changed, 630 insertions(+), 20 deletions(-) create mode 100644 tensorflow/compiler/xla/tests/dynamism_inference_test.cc diff --git a/tensorflow/compiler/tf2xla/kernels/reshape_op.cc b/tensorflow/compiler/tf2xla/kernels/reshape_op.cc index bf9a9150ea6..a85ba547179 100644 --- a/tensorflow/compiler/tf2xla/kernels/reshape_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/reshape_op.cc @@ -109,27 +109,33 @@ class ReshapeOp : public XlaOpKernel { VLOG(2) << "Reshape from " << input_shape.DebugString() << " to " << shape.DebugString() << ", unknown_index=" << unknown_index; - shape_input.clear(); - // Run get input again, this time with dynamic dimension represented as - // "-1" - ctx->set_dynamic_dimension_is_minus_one(true); - OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(1, &shape_input)); - int dynamic_dimension = -1; - - for (int d = 0; d < num_dims; ++d) { - const int32 size = shape_input[d]; - if (size == -1) { - if (dynamic_dimension == -1) { + if (ctx->InputXlaShape(0)->is_dynamic()) { + std::vector dynamic_dims; + OP_REQUIRES_OK(ctx, + ctx->ResolveInputDynamismIntoPredVector(1, &dynamic_dims)); + for (int d = 0; d < num_dims; ++d) { + const bool dim_is_dynamic = dynamic_dims[d]; + if (dim_is_dynamic) { dynamic_dimension = d; - } else { - if (unknown_index != d) { - dynamic_dimension = d; - } } } - } + // When reshaping from dynamic dimension, unkwown index is considered + // dynamic. E.g., + // [<=10] + // | + // Reshape + // | + // [2, -1] + // The second dimension is dynamic. + if (dynamic_dimension == -1) { + dynamic_dimension = unknown_index; + } + VLOG(2) << "Reshape from " << ctx->InputXlaShape(0)->ToString() << " to " + << xla::VectorString(shape.dim_sizes()) + << ", dynamic_dim=" << dynamic_dimension; + } // Pass unknown_index to Xla::Reshape as a hint for dynamic shape inference // in XLA to know which output dimension is dynamic. ctx->SetOutput(0, xla::ReshapeWithInferredDimension( diff --git a/tensorflow/compiler/tf2xla/xla_expression.cc b/tensorflow/compiler/tf2xla/xla_expression.cc index 34e108bb6bf..f0cc8d26709 100644 --- a/tensorflow/compiler/tf2xla/xla_expression.cc +++ b/tensorflow/compiler/tf2xla/xla_expression.cc @@ -101,6 +101,48 @@ xla::XlaOp XlaExpression::AsXlaOp(xla::XlaBuilder* builder) const { }); } +xla::StatusOr XlaExpression::ResolveDynamism( + xla::Client* client) const { + switch (kind()) { + case Kind::kConstant: { + // Constant values are considered static. + Tensor constant_false(DT_BOOL, constant_value().shape()); + auto flat = constant_false.flat(); + for (int64 i = 0; i < flat.size(); ++i) flat(i) = false; + return constant_false; + } + case Kind::kXlaOp: + break; + case Kind::kTensorList: + TF_FALLTHROUGH_INTENDED; + case Kind::kResource: + TF_FALLTHROUGH_INTENDED; + case Kind::kInvalid: + return errors::InvalidArgument( + "ResolveDynamism called on unsupported XlaExpression: ", + HumanString()); + } + + if (!client) + return errors::InvalidArgument("client is required to resolve constant"); + + TF_ASSIGN_OR_RETURN(xla::XlaComputation constant_graph, + handle().builder()->BuildDynamicInferenceGraph(handle())); + + TF_ASSIGN_OR_RETURN(TensorShape shape, GetShape()); + + // The XLA layout is specified minor to major, and TensorFlow uses a major to + // minor order. + std::vector layout_indices(shape.dims()); + std::iota(layout_indices.rbegin(), layout_indices.rend(), 0); + xla::Layout layout = xla::LayoutUtil::MakeLayout(layout_indices); + TF_ASSIGN_OR_RETURN(xla::Literal literal, + client->ComputeConstant(constant_graph, &layout)); + Tensor tensor(DT_BOOL); + TF_RETURN_IF_ERROR(LiteralToHostTensor(literal, DT_BOOL, &tensor)); + return tensor; +} + xla::StatusOr> XlaExpression::ResolveConstant( xla::Client* client, bool dynamic_dimension_is_minus_one) const { switch (kind()) { diff --git a/tensorflow/compiler/tf2xla/xla_expression.h b/tensorflow/compiler/tf2xla/xla_expression.h index 3010964c5b7..3546368ff7b 100644 --- a/tensorflow/compiler/tf2xla/xla_expression.h +++ b/tensorflow/compiler/tf2xla/xla_expression.h @@ -99,6 +99,10 @@ class XlaExpression { xla::StatusOr> ResolveConstant( xla::Client* client, bool dynamic_dimension_is_minus_one = false) const; + // ResolveDynamism computes where a value inside this op is dynamic or can be + // inferred at compile time. + xla::StatusOr ResolveDynamism(xla::Client* client) const; + // Returns the shape of the tensor. // The shape of a resource is the shape of a resource handle (i.e., a scalar), // not the shape of the resource's value. diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc index 735a6c7291e..07537546d52 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc @@ -243,6 +243,48 @@ Status XlaOpKernelContext::ConstantInputAsFloatScalar(int index, double* out) { return LiteralToFloat64Scalar(literal, out); } +static Status LiteralToPredVector(const xla::LiteralSlice& literal, + std::vector* out) { + if (literal.shape().rank() != 1) { + return errors::InvalidArgument("value is not 1D, rank: ", + literal.shape().rank()); + } + int64 size = xla::ShapeUtil::ElementsIn(literal.shape()); + if (literal.shape().element_type() != xla::PRED) { + return errors::InvalidArgument("value is not PRED"); + } + for (int64 i = 0; i < size; ++i) { + out->push_back(literal.Get({i})); + } + return Status::OK(); +} + +Status XlaOpKernelContext::ResolveInputDynamismIntoPredVector( + int index, std::vector* out) { + xla::Literal literal; + XlaExpression e = InputExpression(index); + auto* client = compiler() ? compiler()->client() : nullptr; + xla::StatusOr dynamism_or_status = e.ResolveDynamism(client); + if (!dynamism_or_status.ok()) { + Status status = dynamism_or_status.status(); + errors::AppendToMessage(&status, "while evaluating input dynamism", index, + " of ", context_->op_kernel().type_string()); + return status; + } + Tensor dynamism = dynamism_or_status.ValueOrDie(); + + Tensor temp(dynamism.dtype()); + TensorShape tensor_shape({InputShape(index).num_elements()}); + if (!temp.CopyFrom(dynamism, tensor_shape)) { + return errors::InvalidArgument( + context_->op_kernel().name(), " input ", index, " has shape ", + dynamism.shape().DebugString(), " which is not a R1 ", tensor_shape); + } + + TF_ASSIGN_OR_RETURN(literal, HostTensorToLiteral(temp)); + return LiteralToPredVector(literal, out); +} + // Converts an int32 or int64 1D literal to an int64 vector. static Status LiteralToInt64Vector(const xla::LiteralSlice& literal, std::vector* out) { diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.h b/tensorflow/compiler/tf2xla/xla_op_kernel.h index 3cf51e6ec6f..75c3e60171a 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.h +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.h @@ -116,6 +116,9 @@ class XlaOpKernelContext { // returns a one-element list. Status InputList(absl::string_view name, std::vector* handles, std::vector* shapes); + // Evaluates input and returns their dynamism vector in a vector of + // predicates. + Status ResolveInputDynamismIntoPredVector(int index, std::vector* out); // Helper methods for constant inputs. diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc index 484fb0aabe7..8de8216c005 100644 --- a/tensorflow/compiler/xla/client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_builder.cc @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow/compiler/xla/client/xla_computation.h" #include "tensorflow/compiler/xla/comparison_util.h" #include "tensorflow/compiler/xla/execution_options_util.h" +#include "tensorflow/compiler/xla/service/hlo.pb.h" #include "tensorflow/compiler/xla/service/hlo_input_output_alias_config.h" #include "tensorflow/compiler/xla/service/hlo_instruction.h" #include "tensorflow/compiler/xla/service/hlo_opcode.h" @@ -39,6 +40,7 @@ limitations under the License. #include "tensorflow/compiler/xla/status_macros.h" #include "tensorflow/compiler/xla/util.h" #include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/platform/errors.h" namespace xla { @@ -71,6 +73,52 @@ void SetProtoIdAndName(T* entry, const string& base_name, char separator, entry->set_id(id); entry->set_name(GetFullName(base_name, separator, id)); } + +ShapeProto ConvertShapeProtoToPred(const ShapeProto& shape_proto) { + return ShapeUtil::ChangeElementType(Shape(shape_proto), PRED).ToProto(); +} + +HloInstructionProto CreateConstantInstruction(int64 id, const Shape& shape, + bool pred) { + HloInstructionProto const_instr; + Literal literal = LiteralUtil::CreateR0(pred); + Literal literal_broadcast = literal.Broadcast(shape, {}).ValueOrDie(); + *const_instr.mutable_shape() = shape.ToProto(); + *const_instr.mutable_literal() = literal_broadcast.ToProto(); + *const_instr.mutable_opcode() = HloOpcodeString(HloOpcode::kConstant); + const_instr.set_id(id); + return const_instr; +} + +// Converts a HloComputation into ReducerOr with predicate types. +HloComputationProto CreateReduceOr(int64 reducer_id, + HloComputationProto* original_reducer) { + HloComputationProto reducer; + SetProtoIdAndName(&reducer, StrCat("reduce_or"), kNameSeparator, reducer_id); + std::vector operands_id; + for (auto& inst : original_reducer->instructions()) { + // Copy params. + if (StringToHloOpcode(inst.opcode()).ValueOrDie() == + HloOpcode::kParameter) { + HloInstructionProto* new_param = reducer.add_instructions(); + *new_param = inst; + *new_param->mutable_shape() = ConvertShapeProtoToPred(inst.shape()); + operands_id.push_back(inst.id()); + } + if (inst.id() == original_reducer->root_id()) { + HloInstructionProto* new_root = reducer.add_instructions(); + *new_root = inst; + *new_root->mutable_shape() = ConvertShapeProtoToPred(inst.shape()); + *new_root->mutable_opcode() = HloOpcodeString(HloOpcode::kOr); + new_root->clear_operand_ids(); + for (int64 operand_id : operands_id) { + new_root->add_operand_ids(operand_id); + } + reducer.set_root_id(inst.id()); + } + } + return reducer; +} } // namespace namespace internal { @@ -2842,6 +2890,196 @@ StatusOr XlaBuilder::IsConstant(XlaOp operand) const { return is_constant; } +StatusOr XlaBuilder::BuildDynamicInferenceGraph(XlaOp root_op) { + TF_ASSIGN_OR_RETURN(const HloInstructionProto* root, + LookUpInstruction(root_op)); + + HloComputationProto entry; + SetProtoIdAndName(&entry, StrCat(name_, "_dynamic_inference"), kNameSeparator, + GetNextId()); + ProgramShapeProto* program_shape = entry.mutable_program_shape(); + *program_shape->mutable_result() = + ShapeUtil::ChangeElementType(Shape(root->shape()), PRED).ToProto(); + + std::set seen; + struct WorkItem { + explicit WorkItem(int64 handle, bool need_rewrite) + : handle(handle), need_rewrite(need_rewrite) {} + int64 handle; + // If need_rewrite is true, the instruction will be copied and rewrite into + // a pred instruction indicating if each value is dynamic. If need_rewrite + // is false, simply copy the instruction to the output graph. + // E.g., + // For select(P, A, B), we need to rewrite A and B into predicates, but + // don't need to rewrite P. + bool need_rewrite; + }; + std::queue worklist; + worklist.push(WorkItem(root->id(), true)); + entry.set_root_id(root->id()); + std::vector called_computatons; + // Rewritre instruction with id "from" into the new graph. + // Returns more work items that need to finish. + auto rewrite_instruction = + [&](int64 from, bool need_rewrite) -> StatusOr> { + // Rewrite the instruction with following rules: + // - Unary ops: Convert into bitcast (identity) with type Pred. + // - Binary ops: Convert into binary or. + // - Select: Convert into binary or with its two data operands. + // - Concat / Tuple/ GTE / Bitcast: Copy. + // - Param: Convert to constant True. + // - GetDimensionSize: Convert to constant True if dimension is dynamic, + // contant False if dimension is static. + // - Reduce: Convert to reduce or. + // - Constant: Convert to constant False. + // - Other ops: Not supported. + // Create the instruction for the new handle. + TF_ASSIGN_OR_RETURN(const HloInstructionProto* instr_proto, + LookUpInstructionByHandle(from)); + + TF_ASSIGN_OR_RETURN(HloOpcode opcode, + StringToHloOpcode(instr_proto->opcode())); + std::vector operands_todo; + auto* new_instr = entry.add_instructions(); + *new_instr = *instr_proto; + for (auto operand_id : new_instr->operand_ids()) { + operands_todo.emplace_back(operand_id, need_rewrite); + } + + if (!need_rewrite) { + *new_instr->mutable_name() = + GetFullName(instr_proto->opcode(), kNameSeparator, instr_proto->id()); + return operands_todo; + } + *new_instr->mutable_shape() = ConvertShapeProtoToPred(instr_proto->shape()); + Shape new_shape(new_instr->shape()); + switch (opcode) { + case HloOpcode::kAbs: + case HloOpcode::kRoundNearestAfz: + case HloOpcode::kBitcast: + case HloOpcode::kCeil: + case HloOpcode::kCollectivePermuteDone: + case HloOpcode::kCos: + case HloOpcode::kClz: + case HloOpcode::kExp: + case HloOpcode::kExpm1: + case HloOpcode::kFloor: + case HloOpcode::kImag: + case HloOpcode::kIsFinite: + case HloOpcode::kLog: + case HloOpcode::kLog1p: + case HloOpcode::kNot: + case HloOpcode::kNegate: + case HloOpcode::kPopulationCount: + case HloOpcode::kReal: + case HloOpcode::kRsqrt: + case HloOpcode::kLogistic: + case HloOpcode::kSign: + case HloOpcode::kSin: + case HloOpcode::kConvert: + case HloOpcode::kSqrt: + case HloOpcode::kCbrt: + case HloOpcode::kTanh: + CHECK_EQ(instr_proto->operand_ids_size(), 1); + *new_instr->mutable_opcode() = HloOpcodeString(HloOpcode::kBitcast); + break; + case HloOpcode::kAdd: + case HloOpcode::kAtan2: + case HloOpcode::kDivide: + case HloOpcode::kComplex: + case HloOpcode::kMaximum: + case HloOpcode::kMinimum: + case HloOpcode::kMultiply: + case HloOpcode::kPower: + case HloOpcode::kRemainder: + case HloOpcode::kSubtract: + case HloOpcode::kCompare: + case HloOpcode::kAnd: + case HloOpcode::kOr: + case HloOpcode::kXor: + case HloOpcode::kShiftLeft: + case HloOpcode::kShiftRightArithmetic: + case HloOpcode::kShiftRightLogical: + CHECK_EQ(instr_proto->operand_ids_size(), 2); + *new_instr->mutable_opcode() = HloOpcodeString(HloOpcode::kOr); + break; + case HloOpcode::kSelect: + operands_todo[0].need_rewrite = false; + break; + case HloOpcode::kGather: + operands_todo[1].need_rewrite = false; + break; + case HloOpcode::kReduce: { + int64 reducer_id = new_instr->called_computation_ids(0); + called_computatons.push_back( + CreateReduceOr(reducer_id, &embedded_[reducer_id])); + break; + } + case HloOpcode::kTuple: + case HloOpcode::kTranspose: + case HloOpcode::kGetTupleElement: + case HloOpcode::kSlice: + case HloOpcode::kBroadcast: + case HloOpcode::kConcatenate: + case HloOpcode::kReshape: + break; + case HloOpcode::kGetDimensionSize: { + int64 dimension = instr_proto->dimensions(0); + int64 operand_handle = instr_proto->operand_ids(0); + TF_ASSIGN_OR_RETURN(const HloInstructionProto* operand_proto, + LookUpInstructionByHandle(operand_handle)); + + *new_instr = CreateConstantInstruction( + from, new_shape, + operand_proto->shape().is_dynamic_dimension(dimension)); + operands_todo.clear(); + break; + } + case HloOpcode::kConstant: + *new_instr = CreateConstantInstruction(from, new_shape, false); + break; + case HloOpcode::kParameter: + *new_instr = CreateConstantInstruction(from, new_shape, true); + break; + default: + return InvalidArgument("Dynamic inferencing %s is not supported", + instr_proto->DebugString()); + } + *new_instr->mutable_name() = + GetFullName(instr_proto->opcode(), kNameSeparator, instr_proto->id()); + return operands_todo; + }; + + while (!worklist.empty()) { + WorkItem item = worklist.front(); + worklist.pop(); + if (!seen.insert(item.handle).second) { + continue; + } + TF_ASSIGN_OR_RETURN(auto todos, + rewrite_instruction(item.handle, item.need_rewrite)); + for (WorkItem& todo : todos) { + worklist.push(todo); + } + } + absl::c_sort(*entry.mutable_instructions(), + [](const HloInstructionProto& p1, + const HloInstructionProto& p2) { return p1.id() < p2.id(); }); + XlaComputation computation(entry.id()); + HloModuleProto* module = computation.mutable_proto(); + module->set_name(entry.name()); + module->set_id(entry.id()); + module->set_entry_computation_name(entry.name()); + module->set_entry_computation_id(entry.id()); + *module->mutable_host_program_shape() = *program_shape; + for (auto& called_comp : called_computatons) { + *module->add_computations() = called_comp; + } + *module->add_computations() = std::move(entry); + XLA_VLOG_LINES(3, module->DebugString()); + return std::move(computation); +} + StatusOr XlaBuilder::BuildConstantSubGraph( XlaOp root_op, bool dynamic_dimension_is_minus_one) { TF_ASSIGN_OR_RETURN(bool is_constant, IsConstant(root_op)); diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h index aa5074d28d9..6753b6dd919 100644 --- a/tensorflow/compiler/xla/client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_builder.h @@ -278,6 +278,31 @@ class XlaBuilder { StatusOr BuildConstantSubGraph( XlaOp root_op, bool dynamic_dimension_is_uint_max = false); + // Similar to BuildConstantSubGraph, but with root element type changed to + // boolean. A true value in the root indicates that the value is dynamic while + // false value indicates that the value is a constant. This will copy the + // needed ops/computations to the subgraph. + // + // E.g., + // Compuptation { + // a = 3 + // b = param(0) + // ROOT Tuple(a + b, a + 1, b + 1) + // } + // Calling BuildDynamicInferenceGraph on root will produce the following + // graph: + // + // Compuptation { + // a = False + // b = True + // ROOT Tuple(a | b, a, b) + // } + // + // The result, which is (True, False, True) after evaluation, can be + // interpreted as "First element is dynamic; Second element is static; Third + // element is dynamic". + StatusOr BuildDynamicInferenceGraph(XlaOp root_op); + // Returns the first error that was encountered while building the // computation. When an error is encountered, by default we return a vacuous // XlaOp and inform the user of the error that occurred while diff --git a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc index 2f2456863e9..36429d3d755 100644 --- a/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc +++ b/tensorflow/compiler/xla/service/dynamic_dimension_inference.cc @@ -805,7 +805,8 @@ Status DynamicDimensionInferenceVisitor::HandleReshape(HloInstruction* hlo) { } if (input_dim_size > output_dim_size) { - TF_RET_CHECK(input_dim_size % output_dim_size == 0); + TF_RET_CHECK(input_dim_size % output_dim_size == 0) + << reshape->ToString(); const int64 divisor = input_dim_size / output_dim_size; HloInstruction* divisor_hlo = hlo->parent()->AddInstruction(HloInstruction::CreateConstant( diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc index 02fcaafd19d..0833919b124 100644 --- a/tensorflow/compiler/xla/shape_util.cc +++ b/tensorflow/compiler/xla/shape_util.cc @@ -783,9 +783,18 @@ ShapeUtil::MakeShapeWithDescendingLayoutAndSamePhysicalLayout( /* static */ Shape ShapeUtil::ChangeElementType(const Shape& original, PrimitiveType type) { - Shape new_shape = original; - new_shape.set_element_type(type); - return new_shape; + if (original.IsTuple()) { + std::vector new_operands; + new_operands.reserve(original.tuple_shapes_size()); + for (const Shape& operand : original.tuple_shapes()) { + new_operands.push_back(ChangeElementType(operand, type)); + } + return MakeTupleShape(new_operands); + } else { + Shape new_shape = original; + new_shape.set_element_type(type); + return new_shape; + } } /* static */ bool ShapeUtil::IndexIsValid(const Shape& shape, diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 927f9d14883..17444c042e7 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -2088,6 +2088,31 @@ xla_test( ], ) +xla_test( + name = "dynamism_inference_test", + srcs = ["dynamism_inference_test.cc"], + deps = [ + ":test_macros_header", + "//tensorflow/compiler/xla:literal", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:status_macros", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:test", + "//tensorflow/compiler/xla:xla_data_proto_cc", + "//tensorflow/compiler/xla/client:client_library", + "//tensorflow/compiler/xla/client:global_data", + "//tensorflow/compiler/xla/client:xla_builder", + "//tensorflow/compiler/xla/client:xla_computation", + "//tensorflow/compiler/xla/client/lib:prng", + "//tensorflow/compiler/xla/tests:literal_test_util", + "//tensorflow/compiler/xla/tests:test_utils", + "//tensorflow/compiler/xla/tests:xla_internal_test_main", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "@com_google_absl//absl/strings", + ], +) + xla_test( name = "compute_constant_test", srcs = ["compute_constant_test.cc"], diff --git a/tensorflow/compiler/xla/tests/dynamism_inference_test.cc b/tensorflow/compiler/xla/tests/dynamism_inference_test.cc new file mode 100644 index 00000000000..ba4092def16 --- /dev/null +++ b/tensorflow/compiler/xla/tests/dynamism_inference_test.cc @@ -0,0 +1,215 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include + +#include "absl/strings/match.h" +#include "tensorflow/compiler/xla/client/client_library.h" +#include "tensorflow/compiler/xla/client/global_data.h" +#include "tensorflow/compiler/xla/client/lib/prng.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/client/xla_computation.h" +#include "tensorflow/compiler/xla/layout_util.h" +#include "tensorflow/compiler/xla/literal.h" +#include "tensorflow/compiler/xla/shape_util.h" +#include "tensorflow/compiler/xla/status_macros.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/test.h" +#include "tensorflow/compiler/xla/tests/literal_test_util.h" +#include "tensorflow/compiler/xla/tests/test_macros.h" +#include "tensorflow/compiler/xla/tests/test_utils.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/types.h" + +namespace xla { +namespace { + +// An enumerator for the client types that we want to iterate over in +// the various tests. +enum class ClientType { kLocal, kCompileOnly }; +ClientType client_types[] = {ClientType::kLocal, ClientType::kCompileOnly}; + +class DynamismInferenceTest : public ::testing::Test { + public: + explicit DynamismInferenceTest(se::Platform* platform = nullptr) + : platform_(platform) {} + + string TestName() const { + return ::testing::UnitTest::GetInstance()->current_test_info()->name(); + } + + Client* ClientOrDie(se::Platform* platform, ClientType client_type) { + if (client_type == ClientType::kLocal) { + StatusOr result = + ClientLibrary::GetOrCreateLocalClient(platform); + TF_CHECK_OK(result.status()) + << "could not create LocalClient for testing"; + return result.ValueOrDie(); + } else if (client_type == ClientType::kCompileOnly) { + StatusOr result = + ClientLibrary::GetOrCreateCompileOnlyClient(platform); + TF_CHECK_OK(result.status()) + << "could not create CompileOnlyClient for testing"; + return result.ValueOrDie(); + } + LOG(FATAL) << "invalid client_type value"; + } + + StatusOr ComputeDynamismLiteral(Client* client, XlaOp operand, + XlaBuilder* builder, + Layout* output_layout = nullptr) { + TF_ASSIGN_OR_RETURN(auto subgraph, + builder->BuildDynamicInferenceGraph(operand)); + TF_ASSIGN_OR_RETURN(auto computed, + client->ComputeConstant(subgraph, output_layout)); + return std::move(computed); + } + + StatusOr ComputeDynamismScalar(Client* client, XlaOp operand, + XlaBuilder* builder, + ShapeIndex index = {}) { + TF_ASSIGN_OR_RETURN(auto literal, ComputeDynamismLiteral(client, operand, + builder, nullptr)); + return literal.Get({}, index); + } + + se::Platform* platform_; +}; + +TEST_F(DynamismInferenceTest, ScalarInt32Literal) { + for (ClientType client_type : client_types) { + Client* client = ClientOrDie(platform_, client_type); + XlaBuilder b(TestName()); + auto computation = ConstantR0(&b, 42); + + auto value = ComputeDynamismScalar(client, computation, &b); + ASSERT_TRUE(value.ok()) << value.status(); + // A constant is not dynamic. + EXPECT_EQ(value.ValueOrDie(), false); + } +} + +TEST_F(DynamismInferenceTest, TupleGteKeepsDynamism) { + for (ClientType client_type : client_types) { + Client* client = ClientOrDie(platform_, client_type); + XlaBuilder b(TestName()); + auto c = ConstantR0(&b, 42); + auto p = Parameter(&b, 0, ShapeUtil::MakeScalarShape(S32), "0"); + + auto tuple = Tuple(&b, {c, p}); + auto gte0 = GetTupleElement(tuple, 0); + auto gte1 = GetTupleElement(tuple, 1); + auto tuple_2 = Tuple(&b, {gte0, gte1}); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {0}).ValueOrDie(), + false); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {1}).ValueOrDie(), + true); + } +} + +TEST_F(DynamismInferenceTest, ConcatSliceReshapeKeepsDynamism) { + for (ClientType client_type : client_types) { + Client* client = ClientOrDie(platform_, client_type); + XlaBuilder b(TestName()); + auto c = ConstantR0(&b, 42); + auto p = Parameter(&b, 0, ShapeUtil::MakeScalarShape(S32), "0"); + + auto concat = ConcatScalars(&b, {c, p}); + auto slice0 = SliceInDim(concat, 0, 1, 1, 0); + auto reshape0 = Reshape(slice0, {}); + auto slice1 = SliceInDim(concat, 1, 2, 1, 0); + auto reshape1 = Reshape(slice1, {}); + auto tuple_2 = Tuple(&b, {reshape0, reshape1}); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {0}).ValueOrDie(), + false); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {1}).ValueOrDie(), + true); + } +} + +TEST_F(DynamismInferenceTest, ParameterIsDynamic) { + for (ClientType client_type : client_types) { + Client* client = ClientOrDie(platform_, client_type); + XlaBuilder b(TestName()); + auto computation = Parameter(&b, 0, ShapeUtil::MakeScalarShape(S32), "0"); + + auto value = ComputeDynamismScalar(client, computation, &b); + ASSERT_TRUE(value.ok()) << value.status(); + // A parameter is considered dynamic. + EXPECT_EQ(value.ValueOrDie(), true); + } +} + +TEST_F(DynamismInferenceTest, UnaryOpKeepsDynamism) { + for (ClientType client_type : client_types) { + Client* client = ClientOrDie(platform_, client_type); + XlaBuilder b(TestName()); + auto c = ConstantR0(&b, 42); + auto p = Parameter(&b, 0, ShapeUtil::MakeScalarShape(S32), "0"); + + auto neg0 = Neg(c); + auto neg1 = Neg(p); + auto tuple_2 = Tuple(&b, {neg0, neg1}); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {0}).ValueOrDie(), + false); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {1}).ValueOrDie(), + true); + } +} + +TEST_F(DynamismInferenceTest, BinaryOpsOrsDynamism) { + for (ClientType client_type : client_types) { + Client* client = ClientOrDie(platform_, client_type); + XlaBuilder b(TestName()); + auto c = ConstantR0(&b, 42); + auto p = Parameter(&b, 0, ShapeUtil::MakeScalarShape(S32), "0"); + + // Static value + static value = static + auto add1 = Add(c, c); + // Dynamic value + dynamic value = dynamic + auto add2 = Add(p, c); + auto tuple_2 = Tuple(&b, {add1, add2}); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {0}).ValueOrDie(), + false); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {1}).ValueOrDie(), + true); + } +} + +TEST_F(DynamismInferenceTest, GetDimensionSize) { + for (ClientType client_type : client_types) { + Client* client = ClientOrDie(platform_, client_type); + XlaBuilder b(TestName()); + // param = Param([<=2, 3]) + // get_dimension_size(param, 0) is dynamic + // get_dimension_size(param, 1) is static + auto p = + Parameter(&b, 0, ShapeUtil::MakeShape(S32, {2, 3}, {true, false}), "0"); + + auto gds0 = GetDimensionSize(p, 0); + auto gds1 = GetDimensionSize(p, 1); + auto tuple_2 = Tuple(&b, {gds0, gds1}); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {0}).ValueOrDie(), + true); + EXPECT_EQ(ComputeDynamismScalar(client, tuple_2, &b, {1}).ValueOrDie(), + false); + } +} + +} // namespace +} // namespace xla From 0df7db5b43f19a3ecfb10b740026f215d57d07b7 Mon Sep 17 00:00:00 2001 From: Ken Franko Date: Fri, 7 Aug 2020 16:37:47 -0700 Subject: [PATCH 0683/1017] Only mark ops in the `tf` dialect for outside compilation. There are ops from other dialects like tf_device that will be rewritten in other steps and will never to compiled to HLO. PiperOrigin-RevId: 325532336 Change-Id: Ic763dc7bd2de667935f597f24bd3e043ca94d25b --- .../mark_ops_for_outside_compilation.cc | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc b/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc index c0889affb30..cd34525f2af 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc @@ -72,14 +72,17 @@ bool MatchesPattern(Operation& op, return (supported_ops.contains(op.getName())); } -// Checks if the op is supported inside of a device cluster. +// Checks if the op is supported inside of a device cluster. Ops not +// in `tf_dialect` are considered supported. bool IsSupportedOp(Operation& op, - const llvm::DenseSet& supported_ops) { - // TODO(b/161726307): Check the allowed ops list in LegalizeTfWithTf2XlaPass - // as well. - return !HasStringOperand(op) && !HasStringResult(op) && - (MatchesPattern(op, supported_ops) || - mhlo::IsOpAllowedTf2XlaFallback(&op)); + const llvm::DenseSet& supported_ops, + const Dialect* tf_dialect) { + if (op.getDialect() != tf_dialect) + return true; + else + return !HasStringOperand(op) && !HasStringResult(op) && + (MatchesPattern(op, supported_ops) || + mhlo::IsOpAllowedTf2XlaFallback(&op)); } // Checks all regions of `op` for captured string operands. @@ -96,10 +99,12 @@ bool HasCapturedStringOperand(Operation* op) { return string_operand; } +// Marks uncompilable ops that are in `tf_dialect` for outside compilation. LogicalResult MarkUncompilableOps( - Block* block, llvm::DenseSet& supported_ops) { + const Dialect* tf_dialect, Block* block, + llvm::DenseSet& supported_ops) { block->walk([&](Operation* op) { - if (!IsSupportedOp(*op, supported_ops)) { + if (!IsSupportedOp(*op, supported_ops, tf_dialect)) { op->setAttr(kXlaOutsideCompilationAttr, StringAttr::get("auto", op->getContext())); } @@ -115,6 +120,11 @@ LogicalResult MarkUncompilableOps( void MarkOpsForOutsideCompilation::runOnOperation() { auto module = getOperation(); + const Dialect* tf_dialect = getContext().getRegisteredDialect("tf"); + if (!tf_dialect) { + getOperation().emitError() << "'tf' dialect is not registered"; + return signalPassFailure(); + } OwningRewritePatternList patterns; mhlo::PopulateLegalizeTfPatterns(module.getContext(), &patterns); @@ -129,7 +139,8 @@ void MarkOpsForOutsideCompilation::runOnOperation() { AddSupportedControlFlowOps(module.getContext(), &supported_ops); auto result = module.walk([&](tf_device::ClusterOp cluster) { - if (failed(MarkUncompilableOps(&cluster.GetBody(), supported_ops))) + if (failed( + MarkUncompilableOps(tf_dialect, &cluster.GetBody(), supported_ops))) return WalkResult::interrupt(); return WalkResult::advance(); From 48c12a5abaf802d6a194df46f9ff95cd1557540d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 16:39:26 -0700 Subject: [PATCH 0684/1017] Calculate TensorCore utilization PiperOrigin-RevId: 325532659 Change-Id: I0450e1ec72e22d9d9c31a26d2e16e0950c13276a --- tensorflow/core/profiler/convert/BUILD | 2 + .../profiler/convert/op_stats_to_tf_stats.cc | 35 +++++--- .../convert/op_stats_to_tf_stats_test.cc | 79 ++++++++++++++++--- .../core/profiler/protobuf/tf_stats.proto | 3 + 4 files changed, 98 insertions(+), 21 deletions(-) diff --git a/tensorflow/core/profiler/convert/BUILD b/tensorflow/core/profiler/convert/BUILD index 3261d918e04..66e027ed8ac 100644 --- a/tensorflow/core/profiler/convert/BUILD +++ b/tensorflow/core/profiler/convert/BUILD @@ -153,6 +153,8 @@ cc_library( "//tensorflow/core/profiler/protobuf:op_metrics_proto_cc", "//tensorflow/core/profiler/protobuf:op_stats_proto_cc", "//tensorflow/core/profiler/protobuf:tf_stats_proto_cc", + "//tensorflow/core/profiler/utils:kernel_stats_utils", + "//tensorflow/core/profiler/utils:math_utils", "//tensorflow/core/profiler/utils:op_metrics_db_utils", "//tensorflow/core/profiler/utils:time_utils", ], diff --git a/tensorflow/core/profiler/convert/op_stats_to_tf_stats.cc b/tensorflow/core/profiler/convert/op_stats_to_tf_stats.cc index e23813a5b5d..67024809e61 100644 --- a/tensorflow/core/profiler/convert/op_stats_to_tf_stats.cc +++ b/tensorflow/core/profiler/convert/op_stats_to_tf_stats.cc @@ -20,6 +20,8 @@ limitations under the License. #include "tensorflow/core/profiler/protobuf/op_metrics.pb.h" #include "tensorflow/core/profiler/protobuf/op_stats.pb.h" #include "tensorflow/core/profiler/protobuf/tf_stats.pb.h" +#include "tensorflow/core/profiler/utils/kernel_stats_utils.h" +#include "tensorflow/core/profiler/utils/math_utils.h" #include "tensorflow/core/profiler/utils/op_metrics_db_utils.h" #include "tensorflow/core/profiler/utils/time_utils.h" @@ -40,9 +42,11 @@ TfStatsRecord ConvertOpMetricsToTfStatsRecord( return record; } -TfStatsTable GenerateTfStatsTable(const OpMetricsDb& host_tf_metrics_db, - const OpMetricsDb& device_tf_metrics_db, - double ridge_point, bool exclude_idle) { +TfStatsTable GenerateTfStatsTable( + const OpMetricsDb& host_tf_metrics_db, + const OpMetricsDb& device_tf_metrics_db, + const KernelStatsByOpName& kernel_stats_by_op_name, double ridge_point, + bool exclude_idle) { TfStatsTable tf_stats_table; TfStatsRecord sentinel; sentinel.set_rank(0); @@ -61,6 +65,15 @@ TfStatsTable GenerateTfStatsTable(const OpMetricsDb& host_tf_metrics_db, TfStatsRecord* record = tf_stats_table.add_tf_stats_record(); *record = ConvertOpMetricsToTfStatsRecord( /*on_device=*/true, *metrics, ridge_point); + // Compute TensorCore utilization only on device side. + auto iter = kernel_stats_by_op_name.find(record->op_name()); + if (iter != kernel_stats_by_op_name.end()) { + record->set_gpu_tensorcore_utilization( + SafeDivide(iter->second.tensor_core_duration_ns, + iter->second.total_duration_ns)); + } else { + record->set_gpu_tensorcore_utilization(0.0); + } SetRankAndDeviceTimeFractions(total_device_time_us, *prev_record, record); prev_record = record; } @@ -77,6 +90,8 @@ TfStatsTable GenerateTfStatsTable(const OpMetricsDb& host_tf_metrics_db, TfStatsRecord* record = tf_stats_table.add_tf_stats_record(); *record = ConvertOpMetricsToTfStatsRecord( /*on_device=*/false, *metrics, ridge_point); + // Host side TensorCore utilization is always 0.0 + record->set_gpu_tensorcore_utilization(0.0); SetRankAndHostTimeFractions(total_host_time_us, *prev_record, record); prev_record = record; } @@ -90,13 +105,15 @@ TfStatsDatabase ConvertOpStatsToTfStats(const OpStats& op_stats) { OpMetricsDb device_tf_metrics_db = CreateTfMetricsDbFromDeviceOpMetricsDb(op_stats.device_op_metrics_db()); double ridge_point = op_stats.perf_env().ridge_point(); + KernelStatsByOpName kernel_stats_by_op_name = + GroupKernelReportsByOpName(op_stats.kernel_stats_db()); TfStatsDatabase tf_stats_db; - *tf_stats_db.mutable_with_idle() = - GenerateTfStatsTable(host_tf_metrics_db, device_tf_metrics_db, - ridge_point, /*exclude_idle=*/false); - *tf_stats_db.mutable_without_idle() = - GenerateTfStatsTable(host_tf_metrics_db, device_tf_metrics_db, - ridge_point, /*exclude_idle=*/true); + *tf_stats_db.mutable_with_idle() = GenerateTfStatsTable( + host_tf_metrics_db, device_tf_metrics_db, kernel_stats_by_op_name, + ridge_point, /*exclude_idle=*/false); + *tf_stats_db.mutable_without_idle() = GenerateTfStatsTable( + host_tf_metrics_db, device_tf_metrics_db, kernel_stats_by_op_name, + ridge_point, /*exclude_idle=*/true); return tf_stats_db; } diff --git a/tensorflow/core/profiler/convert/op_stats_to_tf_stats_test.cc b/tensorflow/core/profiler/convert/op_stats_to_tf_stats_test.cc index 4abd210705b..5cf2847ea0d 100644 --- a/tensorflow/core/profiler/convert/op_stats_to_tf_stats_test.cc +++ b/tensorflow/core/profiler/convert/op_stats_to_tf_stats_test.cc @@ -32,32 +32,69 @@ namespace tensorflow { namespace profiler { namespace { -void AddTensorFlowOpEvent(absl::string_view tf_op_fullname, - int64 start_timestamp_ns, int64 duration_ns, - bool on_device, absl::string_view kernel_name, - XPlaneBuilder* plane, XLineBuilder* line) { +XEventBuilder AddTensorFlowOpEvent(absl::string_view tf_op_fullname, + int64 start_timestamp_ns, int64 duration_ns, + bool on_device, + absl::string_view kernel_name, + XPlaneBuilder* plane, XLineBuilder* line) { absl::string_view name = on_device ? kernel_name : tf_op_fullname; XEventBuilder event = line->AddEvent(*plane->GetOrCreateEventMetadata(name)); event.SetTimestampNs(start_timestamp_ns); event.SetDurationNs(duration_ns); - if (!on_device) return; + if (!on_device) return event; event.ParseAndAddStatValue(*plane->GetOrCreateStatMetadata("level 0"), tf_op_fullname); + return event; +} + +void AddTensorFlowOpEventWithKernelDetails(absl::string_view tf_op_fullname, + int64 start_timestamp_ns, + int64 duration_ns, bool on_device, + absl::string_view kernel_name, + absl::string_view kernel_details, + XPlaneBuilder* plane, + XLineBuilder* line) { + XEventBuilder event = + AddTensorFlowOpEvent(tf_op_fullname, start_timestamp_ns, duration_ns, + on_device, kernel_name, plane, line); + if (!on_device) return; + event.ParseAndAddStatValue(*plane->GetOrCreateStatMetadata("kernel_details"), + kernel_details); } TEST(OpStatsToTfStats, GpuTfStats) { - // TfOp1 has kernel1 and kernel2; TfOp2 has kernel3. + // TfOp1 has kernel1 and kernel2; TfOp2 has kernel3; + // TfOp3 has kernel4 and kernel5 and is TensorCore eligible. static constexpr char kTfOp1[] = "TfOp1"; static constexpr char kTfOp2[] = "TfOp2"; + static constexpr char kTfOp3[] = "Conv2D"; static constexpr char kKernel1[] = "kernel1"; static constexpr char kKernel2[] = "kernel2"; static constexpr char kKernel3[] = "kernel3"; + // Kernel4 is a kernel using TensorCore + static constexpr char kKernel4[] = "volta_fp16_s884gemm"; + static constexpr char kKernel5[] = "kernel5"; constexpr int64 kKernel1StartNs = 100000; constexpr int64 kKernel1DurationNs = 8000; constexpr int64 kKernel2StartNs = 110000; constexpr int64 kKernel2DurationNs = 10000; constexpr int64 kKernel3StartNs = 120000; constexpr int64 kKernel3DurationNs = 10000; + constexpr int64 kKernel4StartNs = 130000; + constexpr int64 kKernel4DurationNs = 10000; + constexpr int64 kKernel5StartNs = 150000; + constexpr int64 kKernel5DurationNs = 10000; + + // Mock kernel details for both kernel4 and kernel5. + const std::string kKernelDetails = R"MULTI(registers_per_thread:32 +static_shared_memory_usage:0 +dynamic_shared_memory_usage:16384 +grid_x:2 +grid_y:1 +grid_z:1 +block_x:32 +block_y:1 +block_z:1)MULTI"; XSpace space; XPlaneBuilder device_plane( @@ -79,12 +116,19 @@ TEST(OpStatsToTfStats, GpuTfStats) { AddTensorFlowOpEvent(absl::StrCat(kTfOp2, ":", kTfOp2), kKernel3StartNs, kKernel3DurationNs, /*on_device=*/true, kKernel3, &device_plane, &stream2); + AddTensorFlowOpEventWithKernelDetails( + absl::StrCat(kTfOp3, ":", kTfOp3), kKernel4StartNs, kKernel4DurationNs, + /*on_device=*/true, kKernel4, kKernelDetails, &device_plane, &stream2); + AddTensorFlowOpEventWithKernelDetails( + absl::StrCat(kTfOp3, ":", kTfOp3), kKernel5StartNs, kKernel5DurationNs, + /*on_device=*/true, kKernel5, kKernelDetails, &device_plane, &stream2); - const OpStats op_stats = ConvertXSpaceToOpStats(space, {OP_METRICS_DB}); + const OpStats op_stats = + ConvertXSpaceToOpStats(space, {OP_METRICS_DB, KERNEL_STATS_DB}); const TfStatsDatabase tf_stats = ConvertOpStatsToTfStats(op_stats); - // TfOp1, TfOp2, Idle - EXPECT_EQ(3, tf_stats.with_idle().tf_stats_record_size()); + // TfOp1, TfOp3, TfOp2, Idle + EXPECT_EQ(4, tf_stats.with_idle().tf_stats_record_size()); const TfStatsRecord& record_0 = tf_stats.with_idle().tf_stats_record(0); EXPECT_EQ(kTfOp1, record_0.op_name()); @@ -95,11 +139,22 @@ TEST(OpStatsToTfStats, GpuTfStats) { record_0.total_self_time_in_us()); const TfStatsRecord& record_1 = tf_stats.with_idle().tf_stats_record(1); - EXPECT_EQ(kTfOp2, record_1.op_name()); - EXPECT_EQ(kTfOp2, record_1.op_type()); + EXPECT_EQ(kTfOp3, record_1.op_name()); + EXPECT_EQ(kTfOp3, record_1.op_type()); EXPECT_EQ(1, record_1.occurrences()); + EXPECT_EQ( + NanosToMicros(kKernel4DurationNs) + NanosToMicros(kKernel5DurationNs), + record_1.total_self_time_in_us()); + // GPU TensorCore utilization is 0.5 because kernel4 is using TensorCore and + // kernel5 is not using TensorCore, and they have the same duration. + EXPECT_DOUBLE_EQ(0.5, record_1.gpu_tensorcore_utilization()); + + const TfStatsRecord& record_2 = tf_stats.with_idle().tf_stats_record(2); + EXPECT_EQ(kTfOp2, record_2.op_name()); + EXPECT_EQ(kTfOp2, record_2.op_type()); + EXPECT_EQ(1, record_2.occurrences()); EXPECT_EQ(NanosToMicros(kKernel3DurationNs), - record_1.total_self_time_in_us()); + record_2.total_self_time_in_us()); } } // namespace diff --git a/tensorflow/core/profiler/protobuf/tf_stats.proto b/tensorflow/core/profiler/protobuf/tf_stats.proto index 2dae6230f50..099d8478831 100644 --- a/tensorflow/core/profiler/protobuf/tf_stats.proto +++ b/tensorflow/core/profiler/protobuf/tf_stats.proto @@ -71,4 +71,7 @@ message TfStatsRecord { string bound_by = 17; // Whether this TF-op is eagerly executed. bool is_eager = 18; + // Fraction of kernel time that utilizes GPU TensorCore. + // It is 0.0 if this op does not run on a GPU device. + double gpu_tensorcore_utilization = 19; } From 6b65afa4209a8743d819693ab6c50b5df4db8af9 Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Fri, 7 Aug 2020 16:46:02 -0700 Subject: [PATCH 0685/1017] [TF2XLA] Remove XLA:Interpreter device PiperOrigin-RevId: 325533768 Change-Id: I6ebc60da947a5ba1e9b0782c7e18c996234875ce --- tensorflow/compiler/jit/BUILD | 16 --- .../compiler/jit/xla_interpreter_device.cc | 106 ------------------ 2 files changed, 122 deletions(-) delete mode 100644 tensorflow/compiler/jit/xla_interpreter_device.cc diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index 01b02ad3580..63f985935fb 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -128,22 +128,6 @@ cc_library( alwayslink = 1, ) -cc_library( - name = "xla_interpreter_device", - srcs = ["xla_interpreter_device.cc"], - visibility = [":friends"], - deps = [ - ":jit_compilation_passes", - ":xla_device", - "//tensorflow/compiler/jit/kernels:xla_ops", - "//tensorflow/compiler/tf2xla:xla_compiler", - "//tensorflow/compiler/tf2xla/kernels:xla_ops", - "//tensorflow/compiler/xla/service:interpreter_plugin", # buildcleaner: keep - "@com_google_absl//absl/memory", - ], - alwayslink = 1, -) - cc_library( name = "xla_tensor", srcs = ["xla_tensor.cc"], diff --git a/tensorflow/compiler/jit/xla_interpreter_device.cc b/tensorflow/compiler/jit/xla_interpreter_device.cc deleted file mode 100644 index f720183e196..00000000000 --- a/tensorflow/compiler/jit/xla_interpreter_device.cc +++ /dev/null @@ -1,106 +0,0 @@ -/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Registers the XLA_INTERPRETER device which exposes the XLA Interpreter. - -#include "absl/memory/memory.h" -#include "tensorflow/compiler/jit/kernels/xla_ops.h" -#include "tensorflow/compiler/jit/xla_device.h" -#include "tensorflow/compiler/jit/xla_device_ops.h" -#include "tensorflow/compiler/tf2xla/xla_op_registry.h" - -namespace tensorflow { - -const char* const DEVICE_XLA_INTERPRETER = "XLA_INTERPRETER"; -const char* const DEVICE_INTERPRETER_XLA_JIT = "XLA_INTERPRETER_JIT"; - -constexpr std::array kExecAllTypes = { - {DT_INT8, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, - DT_COMPLEX128, DT_BOOL, DT_BFLOAT16}}; - -class XlaInterpreterDeviceFactory : public DeviceFactory { - public: - Status ListPhysicalDevices(std::vector* devices) override; - Status CreateDevices(const SessionOptions& options, const string& name_prefix, - std::vector>* devices) override; -}; - -Status XlaInterpreterDeviceFactory::ListPhysicalDevices( - std::vector* devices) { - devices->push_back( - absl::StrCat("/physical_device:", DEVICE_XLA_INTERPRETER, ":0")); - - return Status::OK(); -} - -Status XlaInterpreterDeviceFactory::CreateDevices( - const SessionOptions& session_options, const string& name_prefix, - std::vector>* devices) { - static XlaDeviceOpRegistrations* registrations = RegisterXlaDeviceKernels( - DEVICE_XLA_INTERPRETER, DEVICE_INTERPRETER_XLA_JIT); - (void)registrations; - - XlaOpRegistry::DeviceRegistration registration; - registration.compilation_device_name = DEVICE_INTERPRETER_XLA_JIT; - registration.autoclustering_policy = - XlaOpRegistry::AutoclusteringPolicy::kAlways; - registration.cluster_resource_variable_ops_unsafely = true; - registration.cluster_stack_ops = false; - registration.cluster_tensor_array_ops = true; - registration.cluster_stateful_rng_ops = true; - registration.cluster_control_trigger = true; - registration.elide_assert_and_checknumerics = true; - registration.cluster_variant_ops = true; - registration.cluster_slow_ops = true; - registration.cluster_inaccurate_ops = true; - XlaOpRegistry::RegisterCompilationDevice(DEVICE_XLA_INTERPRETER, - registration); - - TF_ASSIGN_OR_RETURN( - auto platform, se::MultiPlatformManager::PlatformWithName("Interpreter")); - - XlaDevice::Options options; - options.platform = platform; - options.device_name_prefix = name_prefix; - options.device_name = DEVICE_XLA_INTERPRETER; - options.device_ordinal = 0; - options.compilation_device_name = DEVICE_INTERPRETER_XLA_JIT; - options.use_multiple_streams = false; - devices->push_back(absl::make_unique(session_options, options)); - - return Status::OK(); -} - -// Set priority to be below the default priority (50), so that Interpreter is -// not selected as a high priority device over other default devices. See -// constructor comments for Registrar in -// tensorflow/core/common_runtime/device_factory.h for a list of priority for -// devices. -REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_INTERPRETER, - XlaInterpreterDeviceFactory, 40); - -// Kernel registrations -static bool OpFilter(KernelDef* kdef) { return true; } - -REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_INTERPRETER, XlaLocalLaunchOp, - kExecAllTypes); -REGISTER_XLA_COMPILE_KERNEL(DEVICE_XLA_INTERPRETER, XlaCompileOp, - kExecAllTypes); -REGISTER_XLA_RUN_KERNEL(DEVICE_XLA_INTERPRETER, XlaRunOp, kExecAllTypes); - -REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_INTERPRETER, kExecAllTypes); -REGISTER_XLA_BACKEND(DEVICE_INTERPRETER_XLA_JIT, kExecAllTypes, OpFilter); - -} // namespace tensorflow From 00dbf072dbe69521ae2170a9fac4052187d187d6 Mon Sep 17 00:00:00 2001 From: Michael Gester Date: Fri, 7 Aug 2020 17:02:03 -0700 Subject: [PATCH 0686/1017] Add TF2XLA fallback patterns to LegalizeTF pass Now LegalizeTF pass can optionally apply TF2XLA fallback patterns. ConvertMLIRToXlaComputation now uses this instead of a separate TF2XLA fallback pass which has following advantages: - ops which need TF -> TF lowering before a fallback pattern can be applied can now be legalized, previously they couldn't - saves intermediate canonicalization and shape inference passes - more flexible control over order in which patterns should be applied PiperOrigin-RevId: 325536262 Change-Id: I6f5d42fe889e5d9404ac558b23ca2e4d6277226f --- .../tensorflow/utils/compile_mlir_util.cc | 16 ++--- tensorflow/compiler/mlir/xla/BUILD | 1 + .../legalize-tf-include-tf2xla-fallback.mlir | 50 +++++++++++++++ .../mlir/xla/transforms/legalize_tf.cc | 61 ++++++++++++++----- .../xla/transforms/legalize_tf_with_tf2xla.cc | 3 +- .../compiler/mlir/xla/transforms/passes.h | 17 +++++- 6 files changed, 119 insertions(+), 29 deletions(-) create mode 100644 tensorflow/compiler/mlir/xla/tests/legalize-tf-include-tf2xla-fallback.mlir diff --git a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc index f06fe1280f0..78d621cbe75 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc @@ -312,29 +312,25 @@ Status ConvertMLIRToXlaComputation( // inside PromoteResourcesToArgs. tf2xla.addPass(mlir::mhlo::createLegalizeTFControlFlowPass()); - tf2xla.addNestedPass(mlir::mhlo::createLegalizeTFPass(true)); + tf2xla.addNestedPass(mlir::mhlo::createLegalizeTFPass( + /*allow_partial_conversion=*/true, /*legalize_chlo=*/true, + /*tf2xla_fallback_device_type=*/device_type)); for (auto& target_pass : custom_legalization_passes) { tf2xla.addNestedPass(std::move(target_pass)); } tf2xla.addNestedPass(mlir::createCanonicalizerPass()); - tf2xla.addPass(mlir::TF::CreateTFShapeInferencePass()); - - // Leverage tf2xla kernels for ops that didn't get lowered in the previous - // legalization pass. - tf2xla.addPass(mlir::mhlo::createLegalizeTfWithTf2XlaPass(device_type)); - tf2xla.addNestedPass(mlir::createCanonicalizerPass()); - // Run shape inference pass to propagate shapes through tensor_cast operations // from static to dynamic shapes. This could be generated if the shape // inference was originally missing in a TF op but the corresponding HLO op // had static shape after lowering. tf2xla.addPass(mlir::TF::CreateTFShapeInferencePass()); - // Run LegalizeTFPass again because the previous legalization passes can // expose more graph pruning and canonicalization opportunities that are // necessary for the second LegalizeTFPass(allow_partial_conversion=false) // invocation. - tf2xla.addNestedPass(mlir::mhlo::createLegalizeTFPass(false)); + tf2xla.addNestedPass(mlir::mhlo::createLegalizeTFPass( + /*allow_partial_conversion=*/false, /*legalize_chlo=*/true, + /*tf2xla_fallback_device_type=*/device_type)); // In order to export to XLA, we must sink constants to control flow regions, // since XLA uses functional control flow. tf2xla.addNestedPass( diff --git a/tensorflow/compiler/mlir/xla/BUILD b/tensorflow/compiler/mlir/xla/BUILD index ada81634567..71e18af498b 100644 --- a/tensorflow/compiler/mlir/xla/BUILD +++ b/tensorflow/compiler/mlir/xla/BUILD @@ -56,6 +56,7 @@ cc_library( ], deps = [ ":type_to_shape", + ":xla_legalize_tf_with_tf2xla", "//tensorflow/compiler/mlir/hlo", "//tensorflow/compiler/mlir/hlo:chlo_legalize_to_hlo", "//tensorflow/compiler/mlir/hlo:convert_op_folder", diff --git a/tensorflow/compiler/mlir/xla/tests/legalize-tf-include-tf2xla-fallback.mlir b/tensorflow/compiler/mlir/xla/tests/legalize-tf-include-tf2xla-fallback.mlir new file mode 100644 index 00000000000..9f72820d15b --- /dev/null +++ b/tensorflow/compiler/mlir/xla/tests/legalize-tf-include-tf2xla-fallback.mlir @@ -0,0 +1,50 @@ +// RUN: tf-opt "-xla-legalize-tf=allow-partial-conversion use-tf2xla-fallback=false" -verify-diagnostics %s | FileCheck --check-prefix NO_FALLBACK %s +// RUN: tf-opt "-xla-legalize-tf=use-tf2xla-fallback=true device-type=XLA_CPU_JIT" -verify-diagnostics %s | FileCheck --check-prefix SUPPORTED_FALLBACK_DEVICE %s +// RUN: tf-opt "-xla-legalize-tf=allow-partial-conversion use-tf2xla-fallback=true" %s | FileCheck --check-prefix UNSPECIFIED_FALLBACK_DEVICE %s +// RUN: tf-opt "-xla-legalize-tf=allow-partial-conversion use-tf2xla-fallback=true device-type=INVALID_DEVICE_TYPE" %s | FileCheck --check-prefix UNSUPPORTED_FALLBACK_DEVICE %s + +// We run this test four times: +// 1) Legalize without using TF2XLA fallback (ops cannot be legalized). +// 2) Use fallback with a device that supports all ops (ops can be legalized). +// 3) Use fallback with unspecified device (ops cannot be legalized). +// 4) Use fallback with specified but unsupported device (ops cannot be legalized). +// +// Note: For 3) and 4) we do not use `-verify-diagnostics` because these cases +// produce remarks that don't occur for 1) and 2) and there is no way to check +// the remarks only for 3) and 4) (except using two files). + +module attributes {tf.versions = {bad_consumers = [], min_consumer = 0 : i32, producer = 268 : i32}} { + +// CHECK-LABEL: non_max_suppression_v4 +func @non_max_suppression_v4(%arg0: tensor<3x4xf32>, %arg1: tensor<3xf32>, %arg2: tensor, %arg3: tensor) -> tensor<2xi32> { + %max_size = mhlo.constant dense<2> : tensor + // NO_FALLBACK: tf.NonMaxSuppressionV4 + // SUPPORTED_FALLBACK_DEVICE-NOT: tf.NonMaxSuppressionV4 + // UNSPECIFIED_FALLBACK_DEVICE: tf.NonMaxSuppressionV4 + // UNSUPPORTED_FALLBACK_DEVICE: tf.NonMaxSuppressionV4 + %0:2 = "tf.NonMaxSuppressionV4"(%arg0, %arg1, %max_size, %arg2, %arg3) {pad_to_max_output_size = true}: (tensor<3x4xf32>, tensor<3xf32>, tensor, tensor, tensor) -> (tensor<2xi32>, tensor) + return %0#0 : tensor<2xi32> +} + +// CHECK-LABEL: mirror_pad +func @mirror_pad(%arg0: tensor<2x3xcomplex>) -> tensor<4x7xcomplex> { + %0 = mhlo.constant dense<[[1, 1], [2, 2]]> : tensor<2x2xi32> + // NO_FALLBACK: tf.MirrorPad + // SUPPORTED_FALLBACK_DEVICE-NOT: tf.MirrorPad + // UNSPECIFIED_FALLBACK_DEVICE: tf.MirrorPad + // UNSUPPORTED_FALLBACK_DEVICE: tf.MirrorPad + %1 = "tf.MirrorPad"(%arg0, %0) {mode = "SYMMETRIC"} : (tensor<2x3xcomplex>, tensor<2x2xi32>) -> tensor<4x7xcomplex> + return %1 : tensor<4x7xcomplex> +} + +// CHECK-LABEL: atan2 +func @atan2(%arg0: tensor<4x1xf32>, %arg1: tensor<4x1x4xf32>) -> tensor<4x4x4xf32> { + // NO_FALLBACK: tf.Atan2 + // SUPPORTED_FALLBACK_DEVICE-NOT: tf.Atan2 + // UNSPECIFIED_FALLBACK_DEVICE: tf.Atan2 + // UNSUPPORTED_FALLBACK_DEVICE: tf.Atan2 + %0 = "tf.Atan2"(%arg0, %arg1) : (tensor<4x1xf32>, tensor<4x1x4xf32>) -> tensor<4x4x4xf32> + return %0: tensor<4x4x4xf32> +} + +} \ No newline at end of file diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc index 6d99e714fc2..aa6f25570a1 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc @@ -71,9 +71,14 @@ class LegalizeTF : public PassWrapper { public: LegalizeTF() = default; LegalizeTF(const LegalizeTF &) {} - explicit LegalizeTF(bool allow_partial_conversion, bool legalize_chlo) { + explicit LegalizeTF(bool allow_partial_conversion, bool legalize_chlo, + llvm::Optional tf2xla_fallback_device_type) { allow_partial_conversion_ = allow_partial_conversion; legalize_chlo_ = legalize_chlo; + use_tf2xla_fallback_ = tf2xla_fallback_device_type.hasValue(); + if (tf2xla_fallback_device_type.hasValue()) { + device_type_ = tf2xla_fallback_device_type.getValue().str(); + } } /// Performs the lowering to XLA dialect. @@ -89,6 +94,17 @@ class LegalizeTF : public PassWrapper { llvm::cl::desc( "Also legalizes intermediate chlo ops to hlo (default true)"), llvm::cl::init(true)}; + Option use_tf2xla_fallback_{ + *this, "use-tf2xla-fallback", + llvm::cl::desc( + "Also use TF2XLA fallback for legalization (default false)"), + llvm::cl::init(false)}; + Option device_type_{ + *this, "device-type", + llvm::cl::desc( + "The device type used by TF2XLA fallback. Must be specified if " + "use-tf2xla-fallback is true, otherwise not used."), + llvm::cl::init("INVALID_DEVICE_TYPE")}; }; /// Returns if the given TF data format string is the default format. @@ -5746,9 +5762,14 @@ void EmitLegalizationErrors(Operation *op, // Performs the lowering to XLA dialect. void LegalizeTF::runOnFunction() { - if (failed( - legalizeTF(getFunction(), allow_partial_conversion_, legalize_chlo_))) + llvm::Optional tf2xla_fallback_device_type = llvm::None; + if (use_tf2xla_fallback_) { + tf2xla_fallback_device_type = device_type_; + } + if (failed(legalizeTF(getFunction(), allow_partial_conversion_, + legalize_chlo_, tf2xla_fallback_device_type))) { signalPassFailure(); + } } static PassRegistration pass( @@ -5758,14 +5779,29 @@ static PassRegistration pass( #include "tensorflow/compiler/mlir/xla/transforms/generated_legalize_tf.inc" -LogicalResult legalizeTF(Operation *op, bool allow_partial_conversion, - bool legalize_chlo) { +LogicalResult legalizeTF( + Operation *op, bool allow_partial_conversion, bool legalize_chlo, + llvm::Optional tf2xla_fallback_device_type) { MLIRContext *context = op->getContext(); - - // Add lowering patterns to the list. OwningRewritePatternList patterns; + // Note that the `OperationConverter` orders patterns lexicographically by: + // 1) Ascending legalization depth (i.e., minimum number of patterns necessary + // to arrive at conversion target). + // 2) Descending pattern benefit. + // 3) Order of patterns in `OwningRewritePatternList`. + + // Add TF->HLO legalization patterns. PopulateLegalizeTfPatterns(context, &patterns); + // Add TF->HLO legalization patterns via TF2XLA fallback. + if (tf2xla_fallback_device_type.hasValue()) { + PopulateLegalizeTfWithTf2XlaPatterns(tf2xla_fallback_device_type.getValue(), + patterns); + } + + // Add TF->TF lowering patterns. + TF::PopulateLoweringTFPatterns(context, &patterns); + // Populate with CHLO->HLO lowerings to account for TF ops legalized to // CHLO first. if (legalize_chlo) { @@ -5805,11 +5841,6 @@ LogicalResult legalizeTF(Operation *op, bool allow_partial_conversion, void PopulateLegalizeTfPatterns(MLIRContext *context, OwningRewritePatternList *patterns) { populateWithGenerated(context, patterns); - - // Add patterns that lower some of the high level TensorFlow ops to lower - // level TensorFlow ops. So, we don't have to target all the TensorFlow ops - // here for lowering to HLO. - TF::PopulateLoweringTFPatterns(context, patterns); patterns->insert< ConvertAllOp, ConvertAnyOp, ConvertArgMaxOp, ConvertBatchMatMulV2Op, ConvertBiasAddOp, ConvertBroadcastToOp, ConvertBF16FloorDivOp, @@ -5838,8 +5869,10 @@ void PopulateLegalizeTfPatterns(MLIRContext *context, } std::unique_ptr> createLegalizeTFPass( - bool allow_partial_conversion, bool legalize_chlo) { - return std::make_unique(allow_partial_conversion, legalize_chlo); + bool allow_partial_conversion, bool legalize_chlo, + llvm::Optional tf2xla_fallback_device_type) { + return std::make_unique(allow_partial_conversion, legalize_chlo, + tf2xla_fallback_device_type); } } // end namespace mhlo diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc index c63e77f2a47..f04f1653505 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc @@ -528,8 +528,7 @@ class LegalizeTF : public PassWrapper { // global device type for all TensorFlow ops. Option device_type_{ *this, "device-type", - llvm::cl::desc("XLA device type for execution of TensorFlow ops. " - "Supports XLA_CPU_JIT and XLA_TPU_JIT for now.")}; + llvm::cl::desc("XLA device type for execution of TensorFlow ops.")}; }; static PassRegistration pass( diff --git a/tensorflow/compiler/mlir/xla/transforms/passes.h b/tensorflow/compiler/mlir/xla/transforms/passes.h index 85bdaaa0e31..45166941620 100644 --- a/tensorflow/compiler/mlir/xla/transforms/passes.h +++ b/tensorflow/compiler/mlir/xla/transforms/passes.h @@ -36,8 +36,13 @@ namespace mhlo { /// Lowers from TF dialect to HLO dialect. When allow_partial_conversion is /// false, emits an error if there is any operation that can't be legalized. +/// When `tf2xla_fallback_device_type` is not `None`, also uses legalization +/// patterns from TF2XLA fallback for provided device type (see +/// legalize_tf_with_tf2xla.cc for details). By default, TF2XLA fallback is not +/// used. std::unique_ptr> createLegalizeTFPass( - bool allow_partial_conversion = false, bool legalize_chlo = true); + bool allow_partial_conversion = false, bool legalize_chlo = true, + llvm::Optional tf2xla_fallback_device_type = llvm::None); /// Lowers from TF dialect to HLO dialect using tf2xla op kernels for the /// specified device type. @@ -63,8 +68,14 @@ std::unique_ptr> createLegalizeTFControlFlowPass(); /// dialect using the conversion patterns registered by the HLO dialect. When /// allow_partial_conversion is false, emits an error if there is any operation /// that can't be legalized. -LogicalResult legalizeTF(Operation* op, bool allow_partial_conversion = false, - bool legalize_chlo = true); +/// When `tf2xla_fallback_device_type` is not `None`, also uses legalization +/// patterns from TF2XLA fallback for provided device type (see +/// legalize_tf_with_tf2xla.cc for details). By default, TF2XLA fallback is not +/// used. +LogicalResult legalizeTF( + Operation* op, bool allow_partial_conversion = false, + bool legalize_chlo = true, + llvm::Optional tf2xla_fallback_device_type = llvm::None); // Legalizes TF/XLA communication ops (TF dialect) to HLO dialect communication // ops. From 4bbdc6ce8e2011e6ca3ae41240b32eef3ed69e9a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 17:05:58 -0700 Subject: [PATCH 0687/1017] Break up core/kernels/BUILD (part 2 of N): Move MKL kernels to subdirectory tensorflow/core/kernels/mkl with its own BUILD file. PiperOrigin-RevId: 325536890 Change-Id: Ia98ea1a3b2c0aa52e0c611e6ae38315802b20261 --- tensorflow/core/BUILD | 86 +-- tensorflow/core/common_runtime/BUILD | 46 +- tensorflow/core/kernels/BUILD | 568 +----------------- tensorflow/core/kernels/mkl/BUILD | 428 +++++++++++++ .../kernels/{ => mkl}/mkl_aggregate_ops.cc | 0 .../kernels/{ => mkl}/mkl_avgpooling_op.cc | 2 +- .../kernels/{ => mkl}/mkl_batch_matmul_op.cc | 2 +- .../core/kernels/{ => mkl}/mkl_concat_op.cc | 0 .../{ => mkl}/mkl_conv_grad_filter_ops.cc | 2 +- .../{ => mkl}/mkl_conv_grad_input_ops.cc | 2 +- .../core/kernels/{ => mkl}/mkl_conv_ops.cc | 4 +- .../core/kernels/{ => mkl}/mkl_conv_ops.h | 8 +- .../kernels/{ => mkl}/mkl_conv_ops_test.cc | 0 .../kernels/{ => mkl}/mkl_cwise_ops_common.cc | 0 .../kernels/{ => mkl}/mkl_dequantize_op.cc | 0 .../{ => mkl}/mkl_dequantize_op_test.cc | 20 +- .../{ => mkl}/mkl_fused_batch_norm_op.cc | 0 .../{ => mkl}/mkl_fused_batch_norm_op_test.cc | 0 .../kernels/{ => mkl}/mkl_fused_ops_test.cc | 0 .../core/kernels/{ => mkl}/mkl_identity_op.cc | 0 .../{ => mkl}/mkl_input_conversion_op.cc | 8 +- .../core/kernels/{ => mkl}/mkl_lrn_op.cc | 0 .../core/kernels/{ => mkl}/mkl_matmul_op.cc | 2 +- .../kernels/{ => mkl}/mkl_matmul_op_fused.cc | 2 +- .../kernels/{ => mkl}/mkl_matmul_ops_common.h | 7 +- .../kernels/{ => mkl}/mkl_maxpooling_op.cc | 2 +- .../{ => mkl}/mkl_pooling_ops_common.cc | 2 +- .../{ => mkl}/mkl_pooling_ops_common.h | 6 +- .../core/kernels/{ => mkl}/mkl_qmatmul_op.cc | 4 +- .../kernels/{ => mkl}/mkl_qmatmul_op_test.cc | 0 .../core/kernels/{ => mkl}/mkl_quantize_op.cc | 0 .../kernels/{ => mkl}/mkl_quantize_op_test.cc | 0 .../{ => mkl}/mkl_quantized_concat_op_test.cc | 0 .../{ => mkl}/mkl_quantized_conv_ops.h | 6 +- .../mkl_quantized_conv_ops_perchannel_test.cc | 0 .../{ => mkl}/mkl_quantized_conv_ops_test.cc | 0 .../mkl_quantized_pooling_ops_test.cc | 0 .../core/kernels/{ => mkl}/mkl_relu_op.cc | 0 .../kernels/{ => mkl}/mkl_relu_op_test.cc | 0 ...mkl_requantization_range_per_channel_op.cc | 0 .../{ => mkl}/mkl_requantize_ops_test.cc | 0 .../mkl_requantize_per_channel_op.cc | 0 .../core/kernels/{ => mkl}/mkl_reshape_op.cc | 0 .../core/kernels/{ => mkl}/mkl_slice_op.cc | 0 .../core/kernels/{ => mkl}/mkl_softmax_op.cc | 0 .../core/kernels/{ => mkl}/mkl_tfconv_op.h | 6 +- .../kernels/{ => mkl}/mkl_tmp_bf16_ops.cc | 0 .../kernels/{ => mkl}/mkl_transpose_op.cc | 0 48 files changed, 533 insertions(+), 680 deletions(-) create mode 100644 tensorflow/core/kernels/mkl/BUILD rename tensorflow/core/kernels/{ => mkl}/mkl_aggregate_ops.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_avgpooling_op.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_batch_matmul_op.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_concat_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_conv_grad_filter_ops.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_conv_grad_input_ops.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_conv_ops.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_conv_ops.h (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_conv_ops_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_cwise_ops_common.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_dequantize_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_dequantize_op_test.cc (88%) rename tensorflow/core/kernels/{ => mkl}/mkl_fused_batch_norm_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_fused_batch_norm_op_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_fused_ops_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_identity_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_input_conversion_op.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_lrn_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_matmul_op.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_matmul_op_fused.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_matmul_ops_common.h (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_maxpooling_op.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_pooling_ops_common.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_pooling_ops_common.h (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_qmatmul_op.cc (99%) rename tensorflow/core/kernels/{ => mkl}/mkl_qmatmul_op_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_quantize_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_quantize_op_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_quantized_concat_op_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_quantized_conv_ops.h (95%) rename tensorflow/core/kernels/{ => mkl}/mkl_quantized_conv_ops_perchannel_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_quantized_conv_ops_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_quantized_pooling_ops_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_relu_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_relu_op_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_requantization_range_per_channel_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_requantize_ops_test.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_requantize_per_channel_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_reshape_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_slice_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_softmax_op.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_tfconv_op.h (97%) rename tensorflow/core/kernels/{ => mkl}/mkl_tmp_bf16_ops.cc (100%) rename tensorflow/core/kernels/{ => mkl}/mkl_transpose_op.cc (100%) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index b449ae1f484..67e0a160c4f 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1050,28 +1050,28 @@ cc_library( ] + if_not_windows([ "//tensorflow/core/kernels/neon:neon_depthwise_conv_op", ]) + if_mkl([ - "//tensorflow/core/kernels:mkl_aggregate_ops", - "//tensorflow/core/kernels:mkl_concat_op", - "//tensorflow/core/kernels:mkl_dequantize_op", - "//tensorflow/core/kernels:mkl_conv_op", - "//tensorflow/core/kernels:mkl_cwise_ops_common", - "//tensorflow/core/kernels:mkl_fused_batch_norm_op", - "//tensorflow/core/kernels:mkl_identity_op", - "//tensorflow/core/kernels:mkl_input_conversion_op", - "//tensorflow/core/kernels:mkl_lrn_op", - "//tensorflow/core/kernels:mkl_pooling_ops", - "//tensorflow/core/kernels:mkl_qmatmul_op", - "//tensorflow/core/kernels:mkl_requantize_ops", - "//tensorflow/core/kernels:mkl_quantize_op", - "//tensorflow/core/kernels:mkl_relu_op", - "//tensorflow/core/kernels:mkl_reshape_op", - "//tensorflow/core/kernels:mkl_slice_op", - "//tensorflow/core/kernels:mkl_softmax_op", - "//tensorflow/core/kernels:mkl_transpose_op", - "//tensorflow/core/kernels:mkl_batch_matmul_op", - "//tensorflow/core/kernels:mkl_matmul_op", - "//tensorflow/core/kernels:mkl_tfconv_op", - "//tensorflow/core/kernels:mkl_tmp_bf16_ops", + "//tensorflow/core/kernels/mkl:mkl_aggregate_ops", + "//tensorflow/core/kernels/mkl:mkl_concat_op", + "//tensorflow/core/kernels/mkl:mkl_dequantize_op", + "//tensorflow/core/kernels/mkl:mkl_conv_op", + "//tensorflow/core/kernels/mkl:mkl_cwise_ops_common", + "//tensorflow/core/kernels/mkl:mkl_fused_batch_norm_op", + "//tensorflow/core/kernels/mkl:mkl_identity_op", + "//tensorflow/core/kernels/mkl:mkl_input_conversion_op", + "//tensorflow/core/kernels/mkl:mkl_lrn_op", + "//tensorflow/core/kernels/mkl:mkl_pooling_ops", + "//tensorflow/core/kernels/mkl:mkl_qmatmul_op", + "//tensorflow/core/kernels/mkl:mkl_requantize_ops", + "//tensorflow/core/kernels/mkl:mkl_quantize_op", + "//tensorflow/core/kernels/mkl:mkl_relu_op", + "//tensorflow/core/kernels/mkl:mkl_reshape_op", + "//tensorflow/core/kernels/mkl:mkl_slice_op", + "//tensorflow/core/kernels/mkl:mkl_softmax_op", + "//tensorflow/core/kernels/mkl:mkl_transpose_op", + "//tensorflow/core/kernels/mkl:mkl_batch_matmul_op", + "//tensorflow/core/kernels/mkl:mkl_matmul_op", + "//tensorflow/core/kernels/mkl:mkl_tfconv_op", + "//tensorflow/core/kernels/mkl:mkl_tmp_bf16_ops", ]) + if_cuda_or_rocm([ "//tensorflow/core/kernels:cudnn_rnn_kernels", ]) + if_cuda([ @@ -2697,27 +2697,27 @@ tf_cc_test_mkl( "//tensorflow/core/kernels:ops_util", "//third_party/eigen3", ] + if_mkl([ - "//tensorflow/core/kernels:mkl_aggregate_ops", - "//tensorflow/core/kernels:mkl_batch_matmul_op", - "//tensorflow/core/kernels:mkl_concat_op", - "//tensorflow/core/kernels:mkl_conv_op", - "//tensorflow/core/kernels:mkl_cwise_ops_common", - "//tensorflow/core/kernels:mkl_dequantize_op", - "//tensorflow/core/kernels:mkl_fused_batch_norm_op", - "//tensorflow/core/kernels:mkl_identity_op", - "//tensorflow/core/kernels:mkl_input_conversion_op", - "//tensorflow/core/kernels:mkl_lrn_op", - "//tensorflow/core/kernels:mkl_matmul_op", - "//tensorflow/core/kernels:mkl_pooling_ops", - "//tensorflow/core/kernels:mkl_qmatmul_op", - "//tensorflow/core/kernels:mkl_quantize_op", - "//tensorflow/core/kernels:mkl_relu_op", - "//tensorflow/core/kernels:mkl_reshape_op", - "//tensorflow/core/kernels:mkl_slice_op", - "//tensorflow/core/kernels:mkl_softmax_op", - "//tensorflow/core/kernels:mkl_tfconv_op", - "//tensorflow/core/kernels:mkl_transpose_op", - "//tensorflow/core/kernels:mkl_tmp_bf16_ops", + "//tensorflow/core/kernels/mkl:mkl_aggregate_ops", + "//tensorflow/core/kernels/mkl:mkl_batch_matmul_op", + "//tensorflow/core/kernels/mkl:mkl_concat_op", + "//tensorflow/core/kernels/mkl:mkl_conv_op", + "//tensorflow/core/kernels/mkl:mkl_cwise_ops_common", + "//tensorflow/core/kernels/mkl:mkl_dequantize_op", + "//tensorflow/core/kernels/mkl:mkl_fused_batch_norm_op", + "//tensorflow/core/kernels/mkl:mkl_identity_op", + "//tensorflow/core/kernels/mkl:mkl_input_conversion_op", + "//tensorflow/core/kernels/mkl:mkl_lrn_op", + "//tensorflow/core/kernels/mkl:mkl_matmul_op", + "//tensorflow/core/kernels/mkl:mkl_pooling_ops", + "//tensorflow/core/kernels/mkl:mkl_qmatmul_op", + "//tensorflow/core/kernels/mkl:mkl_quantize_op", + "//tensorflow/core/kernels/mkl:mkl_relu_op", + "//tensorflow/core/kernels/mkl:mkl_reshape_op", + "//tensorflow/core/kernels/mkl:mkl_slice_op", + "//tensorflow/core/kernels/mkl:mkl_softmax_op", + "//tensorflow/core/kernels/mkl:mkl_tfconv_op", + "//tensorflow/core/kernels/mkl:mkl_transpose_op", + "//tensorflow/core/kernels/mkl:mkl_tmp_bf16_ops", ]), ) diff --git a/tensorflow/core/common_runtime/BUILD b/tensorflow/core/common_runtime/BUILD index 2dbcfdbee38..b46efe01474 100644 --- a/tensorflow/core/common_runtime/BUILD +++ b/tensorflow/core/common_runtime/BUILD @@ -1050,10 +1050,13 @@ cc_library( deps = [ ":function", ":optimization_registry", + "@com_google_absl//absl/base", "//tensorflow/core:framework", "//tensorflow/core:framework_internal", "//tensorflow/core:graph", "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", ] + mkl_deps(), alwayslink = 1, ) @@ -1073,6 +1076,7 @@ cc_library( "//tensorflow/core:framework_internal", "//tensorflow/core:graph", "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", ] + mkl_deps(), alwayslink = 1, ) @@ -2597,27 +2601,27 @@ tf_cc_test_mkl( "//tensorflow/core/kernels:ops_util", "//third_party/eigen3", ] + if_mkl([ - "//tensorflow/core/kernels:mkl_aggregate_ops", - "//tensorflow/core/kernels:mkl_batch_matmul_op", - "//tensorflow/core/kernels:mkl_concat_op", - "//tensorflow/core/kernels:mkl_conv_op", - "//tensorflow/core/kernels:mkl_cwise_ops_common", - "//tensorflow/core/kernels:mkl_dequantize_op", - "//tensorflow/core/kernels:mkl_fused_batch_norm_op", - "//tensorflow/core/kernels:mkl_identity_op", - "//tensorflow/core/kernels:mkl_input_conversion_op", - "//tensorflow/core/kernels:mkl_lrn_op", - "//tensorflow/core/kernels:mkl_matmul_op", - "//tensorflow/core/kernels:mkl_pooling_ops", - "//tensorflow/core/kernels:mkl_qmatmul_op", - "//tensorflow/core/kernels:mkl_quantize_op", - "//tensorflow/core/kernels:mkl_relu_op", - "//tensorflow/core/kernels:mkl_reshape_op", - "//tensorflow/core/kernels:mkl_slice_op", - "//tensorflow/core/kernels:mkl_softmax_op", - "//tensorflow/core/kernels:mkl_tfconv_op", - "//tensorflow/core/kernels:mkl_transpose_op", - "//tensorflow/core/kernels:mkl_tmp_bf16_ops", + "//tensorflow/core/kernels/mkl:mkl_aggregate_ops", + "//tensorflow/core/kernels/mkl:mkl_batch_matmul_op", + "//tensorflow/core/kernels/mkl:mkl_concat_op", + "//tensorflow/core/kernels/mkl:mkl_conv_op", + "//tensorflow/core/kernels/mkl:mkl_cwise_ops_common", + "//tensorflow/core/kernels/mkl:mkl_dequantize_op", + "//tensorflow/core/kernels/mkl:mkl_fused_batch_norm_op", + "//tensorflow/core/kernels/mkl:mkl_identity_op", + "//tensorflow/core/kernels/mkl:mkl_input_conversion_op", + "//tensorflow/core/kernels/mkl:mkl_lrn_op", + "//tensorflow/core/kernels/mkl:mkl_matmul_op", + "//tensorflow/core/kernels/mkl:mkl_pooling_ops", + "//tensorflow/core/kernels/mkl:mkl_qmatmul_op", + "//tensorflow/core/kernels/mkl:mkl_quantize_op", + "//tensorflow/core/kernels/mkl:mkl_relu_op", + "//tensorflow/core/kernels/mkl:mkl_reshape_op", + "//tensorflow/core/kernels/mkl:mkl_slice_op", + "//tensorflow/core/kernels/mkl:mkl_softmax_op", + "//tensorflow/core/kernels/mkl:mkl_tfconv_op", + "//tensorflow/core/kernels/mkl:mkl_transpose_op", + "//tensorflow/core/kernels/mkl:mkl_tmp_bf16_ops", ]), ) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 99970a9558c..9be043c3907 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -9,12 +9,10 @@ load( "tf_cc_binary", "tf_cc_shared_object", "tf_cc_test", - "tf_cc_test_mkl", "tf_cc_tests", "tf_copts", "tf_cuda_library", "tf_kernel_library", - "tf_mkl_kernel_library", "tf_opts_nortti_if_lite_protos", ) load("@local_config_sycl//sycl:build_defs.bzl", "if_sycl") @@ -654,6 +652,7 @@ cc_library( cc_library( name = "batch_kernels", srcs = ["batch_kernels.cc"], + hdrs = ["batch_matmul_op_impl.h"], deps = [ ":ops_util_hdrs", "//tensorflow/core:framework", @@ -939,7 +938,7 @@ cc_library( cc_library( name = "image_resizer_state", hdrs = ["image_resizer_state.h"], - visibility = ["//visibility:private"], + visibility = ["//tensorflow:__subpackages__"], deps = [ ":bounds_check", "//tensorflow/core:framework", @@ -3818,16 +3817,6 @@ tf_kernel_library( ]), ) -tf_mkl_kernel_library( - name = "mkl_batch_matmul_op", - srcs = ["mkl_batch_matmul_op.cc"], - hdrs = [ - "batch_matmul_op_impl.h", - "mkl_matmul_ops_common.h", - ], - deps = MATH_DEPS + mkl_deps(), -) - tf_kernel_library( name = "betainc_op", prefix = "betainc_op", @@ -3909,16 +3898,6 @@ tf_kernel_library( ]) + if_cuda_or_rocm([":gpu_utils"]), ) -tf_mkl_kernel_library( - name = "mkl_matmul_op", - srcs = [ - "mkl_matmul_op.cc", - "mkl_matmul_op_fused.cc", - ], - hdrs = ["mkl_matmul_ops_common.h"], - deps = MATH_DEPS + mkl_deps(), -) - tf_kernel_library( name = "reduction_ops", gpu_srcs = ["reduction_gpu_kernels.cu.h"], @@ -6824,7 +6803,6 @@ filegroup( "unicode_ops.cc", "unicode_script_op.cc", # Ops that are inherently incompatible with Android (e.g. tied to x86 platform). - "mkl_*", "xsmm_*", "cwise_ops_sycl_common.h", "nextafter_op.cc", @@ -7260,50 +7238,6 @@ tf_cc_test( ], ) -tf_cc_test_mkl( - name = "mkl_quantized_conv_ops_perchannel_test", - size = "small", - srcs = ["mkl_quantized_conv_ops_perchannel_test.cc"], - deps = [ - ":mkl_conv_op", - ":mkl_input_conversion_op", - ":ops_testutil", - ":ops_util", - ":quantization_utils", - ":quantized_ops", - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:nn_ops_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cc_test_mkl( - name = "mkl_quantized_conv_ops_test", - size = "small", - srcs = ["mkl_quantized_conv_ops_test.cc"], - deps = [ - ":mkl_conv_op", - ":mkl_input_conversion_op", - ":ops_testutil", - ":ops_util", - ":quantization_utils", - ":quantized_ops", - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:nn_ops_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - tf_cc_test( name = "quantize_op_test", size = "small", @@ -7344,28 +7278,6 @@ tf_cc_test( ], ) -tf_cc_test_mkl( - name = "mkl_qmatmul_op_test", - size = "small", - srcs = ["mkl_qmatmul_op_test.cc"], - deps = [ - ":mkl_input_conversion_op", - ":mkl_qmatmul_op", - ":ops_testutil", - ":ops_util", - ":quantization_utils", - ":quantized_ops", - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:nn_ops_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - # Android-only test for quantized multiply. cc_binary( name = "quantized_mul_op_test_android_only", @@ -7446,66 +7358,6 @@ tf_cc_test( ], ) -tf_mkl_kernel_library( - name = "mkl_quantize_op", - srcs = ["mkl_quantize_op.cc"], - hdrs = [ - "meta_support.h", - "reference_gemm.h", - ], - deps = [ - ":bounds_check", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:mkl_graph_util", - "@gemmlowp", - ] + mkl_deps(), -) - -tf_cc_test_mkl( - name = "mkl_quantize_op_test", - size = "small", - srcs = ["mkl_quantize_op_test.cc"], - deps = [ - ":mkl_quantize_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:nn_ops_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cc_test_mkl( - name = "mkl_quantized_pooling_ops_test", - size = "small", - srcs = ["mkl_quantized_pooling_ops_test.cc"], - deps = [ - ":mkl_input_conversion_op", - ":mkl_pooling_ops", - ":ops_testutil", - ":ops_util", - ":quantization_utils", - ":quantized_ops", - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:nn_ops_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - tf_cc_test( name = "quantized_reshape_op_test", size = "small", @@ -7545,30 +7397,6 @@ tf_cc_test( ], ) -tf_cc_test_mkl( - name = "mkl_quantized_concat_op_test", - size = "small", - srcs = ["mkl_quantized_concat_op_test.cc"], - deps = [ - ":mkl_concat_op", - ":ops_testutil", - ":ops_util", - ":quantization_utils", - ":quantized_ops", - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:mkl_array_ops_op_lib", - "//tensorflow/core:nn_ops_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - tf_cc_test( name = "quantized_batch_norm_op_test", size = "small", @@ -7792,50 +7620,6 @@ tf_cc_test( ], ) -tf_mkl_kernel_library( - name = "mkl_qmatmul_op", - srcs = ["mkl_qmatmul_op.cc"], - hdrs = [ - "mkl_matmul_ops_common.h", - "mkl_quantized_conv_ops.h", - "no_op.h", - ], - deps = [ - ":bounds_check", - ":fill_functor", - ":matmul_op", - ":ops_util", - "//third_party/eigen3", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:mkl_nn_ops_op_lib", - "//tensorflow/core:nn_ops_op_lib", - ] + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_conv_op", - hdrs = [ - "mkl_quantized_conv_ops.h", - "no_op.h", - ], - prefix = "mkl_conv", - deps = [ - ":bounds_check", - ":conv_ops", - ":ops_util", - "@com_google_absl//absl/strings", - "//third_party/eigen3", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - ] + mkl_deps(), -) - tf_cc_test( name = "bias_op_test", size = "small", @@ -7850,354 +7634,6 @@ tf_cc_test( ], ) -tf_cc_test_mkl( - name = "mkl_conv_ops_test", - size = "small", - srcs = ["mkl_conv_ops_test.cc"], - linkstatic = 1, # Fixes dyld error on MacOS. - deps = [ - ":ops_testutil", - ":ops_util", - "//tensorflow/cc:cc_ops", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cc_test_mkl( - name = "mkl_relu_op_test", - size = "small", - srcs = ["mkl_relu_op_test.cc"], - linkstatic = 1, # Fixes dyld error on MacOS. - deps = [ - ":ops_testutil", - ":ops_util", - "//tensorflow/cc:cc_ops", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_mkl_kernel_library( - name = "mkl_tfconv_op", - prefix = "mkl_tfconv", - deps = [ - ":bounds_check", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - ] + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_input_conversion_op", - hdrs = ["mkl_tfconv_op.h"], - prefix = "mkl_input_conversion", - deps = [ - ":bounds_check", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - ] + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_pooling_ops", - srcs = [ - "mkl_avgpooling_op.cc", - "mkl_maxpooling_op.cc", - "mkl_pooling_ops_common.cc", - ], - hdrs = ["mkl_pooling_ops_common.h"], - deps = [ - ":bounds_check", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - ] + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_dequantize_op", - srcs = ["mkl_dequantize_op.cc"], - hdrs = [ - "meta_support.h", - "reference_gemm.h", - ], - deps = [ - ":concat_lib_hdrs", - ":conv_ops", - ":cwise_op", - ":eigen_helpers", - ":image_resizer_state", - ":ops_util", - ":pooling_ops", - ":quantization_utils", - ":quantized_ops", - ":transpose_functor", - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:mkl_graph_util", - "//tensorflow/core:nn_ops_op_lib", - "//third_party/eigen3", - "@gemmlowp", - ] + mkl_deps(), -) - -tf_cc_test_mkl( - name = "mkl_dequantize_op_test", - size = "small", - srcs = ["mkl_dequantize_op_test.cc"], - # TODO(b/149940073): Re-enable. - tags = [ - "no_oss", - "notap", - ], - deps = [ - ":mkl_dequantize_op", - ":mkl_tfconv_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:framework", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:mkl_array_ops_op_lib", - "//tensorflow/core:nn_ops_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_mkl_kernel_library( - name = "mkl_relu_op", - prefix = "mkl_relu", - deps = [ - ":bounds_check", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//third_party/eigen3", - ] + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_softmax_op", - prefix = "mkl_softmax", - deps = [ - ":bounds_check", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//third_party/eigen3", - ] + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_tmp_bf16_ops", - prefix = "mkl_tmp_bf16_ops", - deps = [ - ":no_op", - ] + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_fused_batch_norm_op", - srcs = ["mkl_fused_batch_norm_op.cc"], - deps = NN_DEPS + [ - ":fused_batch_norm_op", - ":no_op", - ] + mkl_deps(), -) - -tf_cc_test_mkl( - name = "mkl_fused_batch_norm_op_test", - size = "small", - srcs = ["mkl_fused_batch_norm_op_test.cc"], - linkstatic = 1, - deps = [ - ":mkl_fused_batch_norm_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/cc:cc_ops", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_mkl_kernel_library( - name = "mkl_aggregate_ops", - prefix = "mkl_aggregate_ops", - deps = MATH_DEPS + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_concat_op", - prefix = "mkl_concat_op", - deps = [":quantization_utils"] + ARRAY_DEPS + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_reshape_op", - prefix = "mkl_reshape_op", - deps = ARRAY_DEPS + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_slice_op", - prefix = "mkl_slice_op", - deps = ARRAY_DEPS + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_identity_op", - prefix = "mkl_identity_op", - deps = ARRAY_DEPS + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_lrn_op", - prefix = "mkl_lrn_op", - deps = NN_DEPS + mkl_deps(), -) - -tf_mkl_kernel_library( - name = "mkl_cwise_ops_common", - hdrs = [ - "cwise_ops.h", - "cwise_ops_common.h", - "cwise_ops_gradients.h", - ], - prefix = "mkl_cwise_ops_common", - deps = NN_DEPS + mkl_deps() + [":cwise_op"], -) - -tf_mkl_kernel_library( - name = "mkl_requantize_ops", - srcs = [ - "mkl_requantization_range_per_channel_op.cc", - "mkl_requantize_per_channel_op.cc", - ], - hdrs = [ - "meta_support.h", - "no_op.h", - "reference_gemm.h", - ], - deps = [ - ":concat_lib_hdrs", - ":conv_ops", - ":cwise_op", - ":eigen_helpers", - ":image_resizer_state", - ":ops_util", - ":pooling_ops", - ":quantization_utils", - ":transpose_functor", - "//third_party/eigen3", - "@gemmlowp", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - ] + mkl_deps(), -) - -tf_cc_test_mkl( - name = "mkl_requantize_ops_test", - size = "small", - srcs = ["mkl_requantize_ops_test.cc"], - linkstatic = 1, # Fixes dyld error on MacOS. - deps = [ - ":mkl_requantize_ops", - ":ops_testutil", - ":ops_util", - ":quantization_utils", - ":quantized_ops", - "//tensorflow/cc:cc_ops", - "//tensorflow/core:array_ops_op_lib", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:math_ops_op_lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cc_test_mkl( - name = "mkl_fused_ops_test", - size = "small", - srcs = ["mkl_fused_ops_test.cc"], - linkstatic = 1, - deps = [ - ":conv_ops", - ":image", - ":mkl_conv_op", - ":mkl_matmul_op", - ":mkl_tfconv_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/cc:cc_ops", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_mkl_kernel_library( - name = "mkl_transpose_op", - srcs = [ - "mkl_transpose_op.cc", - ], - hdrs = ["transpose_op.h"], - deps = ARRAY_DEPS + mkl_deps() + [":transpose_op"], -) - # NOTE(lespeholt): This rule is deprecated, please use: # tensorflow/core/util/batch_util.h cc_library( diff --git a/tensorflow/core/kernels/mkl/BUILD b/tensorflow/core/kernels/mkl/BUILD new file mode 100644 index 00000000000..4abeee20e30 --- /dev/null +++ b/tensorflow/core/kernels/mkl/BUILD @@ -0,0 +1,428 @@ +load( + "//tensorflow:tensorflow.bzl", + "tf_cc_test_mkl", + "tf_mkl_kernel_library", +) +load( + "//third_party/mkl:build_defs.bzl", + "mkl_deps", +) + +package( + default_visibility = [ + "//tensorflow:__subpackages__", + "//tensorflow:internal", + ], + licenses = ["notice"], # Apache 2.0 +) + +# Public support libraries ---------------------------------------------------- +MKL_SHORT_DEPS = [ + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core/kernels:bounds_check", + "//tensorflow/core/kernels:ops_util", +] + mkl_deps() + +MKL_DEPS = MKL_SHORT_DEPS + [ + "//third_party/eigen3", + "//tensorflow/core:array_grad", + "//tensorflow/core:math_grad", + "//tensorflow/core:nn_grad", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/kernels:concat_lib", + "//tensorflow/core/kernels:conv_2d", + "//tensorflow/core/kernels:eigen_contraction_kernel", + "//tensorflow/core/kernels:fill_functor", + "//tensorflow/core/kernels:gather_functor", + "//tensorflow/core/kernels:transpose_functor", +] + +MKL_TEST_DEPS = [ + ":mkl_input_conversion_op", + "//tensorflow/cc:cc_ops", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "//tensorflow/core/kernels:ops_testutil", + "//tensorflow/core/kernels:ops_util", +] + +tf_mkl_kernel_library( + name = "mkl_batch_matmul_op", + srcs = ["mkl_batch_matmul_op.cc"], + hdrs = [ + "mkl_matmul_ops_common.h", + ], + deps = ["//tensorflow/core/kernels:batch_matmul_op"] + MKL_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_matmul_op", + srcs = [ + "mkl_matmul_op.cc", + "mkl_matmul_op_fused.cc", + ], + hdrs = ["mkl_matmul_ops_common.h"], + deps = MKL_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_quantized_conv_ops_perchannel_test", + size = "small", + srcs = ["mkl_quantized_conv_ops_perchannel_test.cc"], + deps = [ + ":mkl_conv_op", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core:nn_ops_op_lib", + "//tensorflow/core/kernels:quantization_utils", + "//tensorflow/core/kernels:quantized_ops", + ] + MKL_TEST_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_quantized_conv_ops_test", + size = "small", + srcs = ["mkl_quantized_conv_ops_test.cc"], + deps = [ + ":mkl_conv_op", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core:nn_ops_op_lib", + "//tensorflow/core/kernels:quantization_utils", + "//tensorflow/core/kernels:quantized_ops", + ] + MKL_TEST_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_qmatmul_op_test", + size = "small", + srcs = ["mkl_qmatmul_op_test.cc"], + deps = [ + ":mkl_qmatmul_op", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core:nn_ops_op_lib", + "//tensorflow/core/framework:fake_input", + "//tensorflow/core/framework:tensor_testutil", + "//tensorflow/core/kernels:quantization_utils", + "//tensorflow/core/kernels:quantized_ops", + ] + MKL_TEST_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_quantize_op", + srcs = ["mkl_quantize_op.cc"], + deps = [ + "//tensorflow/core/kernels:quantized_ops", + "//tensorflow/core:mkl_graph_util", + "@gemmlowp", + ] + MKL_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_quantize_op_test", + size = "small", + srcs = ["mkl_quantize_op_test.cc"], + deps = [ + ":mkl_quantize_op", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core:nn_ops_op_lib", + ] + MKL_TEST_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_quantized_pooling_ops_test", + size = "small", + srcs = ["mkl_quantized_pooling_ops_test.cc"], + deps = [ + ":mkl_pooling_ops", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core:nn_ops_op_lib", + "//tensorflow/core/kernels:quantization_utils", + "//tensorflow/core/kernels:quantized_ops", + ] + MKL_TEST_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_quantized_concat_op_test", + size = "small", + srcs = ["mkl_quantized_concat_op_test.cc"], + deps = [ + ":mkl_concat_op", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core:mkl_array_ops_op_lib", + "//tensorflow/core:nn_ops_op_lib", + "//tensorflow/core/kernels:quantization_utils", + "//tensorflow/core/kernels:quantized_ops", + ] + MKL_TEST_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_qmatmul_op", + srcs = ["mkl_qmatmul_op.cc"], + hdrs = [ + "mkl_matmul_ops_common.h", + "mkl_quantized_conv_ops.h", + ], + deps = [ + "//tensorflow/core/kernels:matmul_op", + "//tensorflow/core/kernels:no_op", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core:mkl_nn_ops_op_lib", + "//tensorflow/core:nn_ops_op_lib", + ] + MKL_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_conv_op", + hdrs = [ + "mkl_quantized_conv_ops.h", + ], + prefix = "mkl_conv", + deps = [ + "@com_google_absl//absl/strings", + "//tensorflow/core/kernels:conv_ops", + "//tensorflow/core/kernels:no_op", + ] + MKL_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_conv_ops_test", + size = "small", + srcs = ["mkl_conv_ops_test.cc"], + linkstatic = 1, # Fixes dyld error on MacOS. + deps = MKL_TEST_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_relu_op_test", + size = "small", + srcs = ["mkl_relu_op_test.cc"], + linkstatic = 1, # Fixes dyld error on MacOS. + deps = MKL_TEST_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_tfconv_op", + prefix = "mkl_tfconv", + deps = MKL_SHORT_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_input_conversion_op", + hdrs = ["mkl_tfconv_op.h"], + prefix = "mkl_input_conversion", + deps = MKL_SHORT_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_pooling_ops", + srcs = [ + "mkl_avgpooling_op.cc", + "mkl_maxpooling_op.cc", + "mkl_pooling_ops_common.cc", + ], + hdrs = ["mkl_pooling_ops_common.h"], + deps = MKL_SHORT_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_dequantize_op", + srcs = ["mkl_dequantize_op.cc"], + deps = [ + "//tensorflow/core/kernels:concat_lib_hdrs", + "//tensorflow/core/kernels:conv_ops", + "//tensorflow/core/kernels:cwise_op", + "//tensorflow/core/kernels:eigen_helpers", + "//tensorflow/core/kernels:image_resizer_state", + "//tensorflow/core/kernels:ops_util", + "//tensorflow/core/kernels:pooling_ops", + "//tensorflow/core/kernels:quantization_utils", + "//tensorflow/core/kernels:quantized_ops", + "//tensorflow/core/kernels:transpose_functor", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core:mkl_graph_util", + "//tensorflow/core:nn_ops_op_lib", + "//third_party/eigen3", + "@gemmlowp", + ] + mkl_deps(), +) + +tf_cc_test_mkl( + name = "mkl_dequantize_op_test", + size = "small", + srcs = ["mkl_dequantize_op_test.cc"], + # TODO(b/149940073): Re-enable. + tags = [ + "no_oss", + "notap", + ], + deps = [ + ":mkl_dequantize_op", + ":mkl_tfconv_op", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core:mkl_array_ops_op_lib", + "//tensorflow/core:nn_ops_op_lib", + ] + MKL_TEST_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_relu_op", + prefix = "mkl_relu", + deps = MKL_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_softmax_op", + prefix = "mkl_softmax", + deps = MKL_SHORT_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_tmp_bf16_ops", + prefix = "mkl_tmp_bf16_ops", + deps = MKL_DEPS + [ + "//tensorflow/core/kernels:no_op", + ], +) + +tf_mkl_kernel_library( + name = "mkl_fused_batch_norm_op", + srcs = ["mkl_fused_batch_norm_op.cc"], + deps = [ + "//tensorflow/core/kernels:fused_batch_norm_op", + "//tensorflow/core/kernels:no_op", + ] + mkl_deps(), +) + +tf_cc_test_mkl( + name = "mkl_fused_batch_norm_op_test", + size = "small", + srcs = ["mkl_fused_batch_norm_op_test.cc"], + linkstatic = 1, + deps = [ + ":mkl_fused_batch_norm_op", + "//tensorflow/core:direct_session", + "//tensorflow/core/kernels:conv_ops_gpu_hdrs", + ] + MKL_TEST_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_aggregate_ops", + prefix = "mkl_aggregate_ops", + deps = MKL_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_concat_op", + prefix = "mkl_concat_op", + deps = ["//tensorflow/core/kernels:quantization_utils"] + MKL_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_reshape_op", + prefix = "mkl_reshape_op", + deps = MKL_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_slice_op", + prefix = "mkl_slice_op", + deps = MKL_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_identity_op", + prefix = "mkl_identity_op", + deps = MKL_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_lrn_op", + prefix = "mkl_lrn_op", + deps = MKL_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_cwise_ops_common", + prefix = "mkl_cwise_ops_common", + deps = MKL_DEPS + ["//tensorflow/core/kernels:cwise_op"], +) + +tf_mkl_kernel_library( + name = "mkl_requantize_ops", + srcs = [ + "mkl_requantization_range_per_channel_op.cc", + "mkl_requantize_per_channel_op.cc", + ], + deps = [ + "//tensorflow/core/kernels:concat_lib_hdrs", + "//tensorflow/core/kernels:conv_ops", + "//tensorflow/core/kernels:eigen_helpers", + "//tensorflow/core/kernels:image_resizer_state", + "//tensorflow/core/kernels:meta_support", + "//tensorflow/core/kernels:no_op", + "//tensorflow/core/kernels:pooling_ops", + "//tensorflow/core/kernels:quantization_utils", + "@gemmlowp", + ] + MKL_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_requantize_ops_test", + size = "small", + srcs = ["mkl_requantize_ops_test.cc"], + linkstatic = 1, # Fixes dyld error on MacOS. + deps = [ + ":mkl_requantize_ops", + "//tensorflow/core:array_ops_op_lib", + "//tensorflow/core:math_ops_op_lib", + "//tensorflow/core/kernels:quantization_utils", + "//tensorflow/core/kernels:quantized_ops", + ] + MKL_TEST_DEPS, +) + +tf_cc_test_mkl( + name = "mkl_fused_ops_test", + size = "small", + srcs = ["mkl_fused_ops_test.cc"], + linkstatic = 1, + deps = [ + ":mkl_conv_op", + ":mkl_matmul_op", + ":mkl_tfconv_op", + "//tensorflow/core:direct_session", + "//tensorflow/core/kernels:bias_op", + "//tensorflow/core/kernels:conv_ops", + "//tensorflow/core/kernels:depthwise_conv_op", + "//tensorflow/core/kernels:image", + "//tensorflow/core/kernels:matmul_op", + "//tensorflow/core/kernels:pad_op", + "//tensorflow/core/kernels:relu_op", + ] + MKL_TEST_DEPS, +) + +tf_mkl_kernel_library( + name = "mkl_transpose_op", + srcs = [ + "mkl_transpose_op.cc", + ], + deps = MKL_DEPS + ["//tensorflow/core/kernels:transpose_op"], +) diff --git a/tensorflow/core/kernels/mkl_aggregate_ops.cc b/tensorflow/core/kernels/mkl/mkl_aggregate_ops.cc similarity index 100% rename from tensorflow/core/kernels/mkl_aggregate_ops.cc rename to tensorflow/core/kernels/mkl/mkl_aggregate_ops.cc diff --git a/tensorflow/core/kernels/mkl_avgpooling_op.cc b/tensorflow/core/kernels/mkl/mkl_avgpooling_op.cc similarity index 99% rename from tensorflow/core/kernels/mkl_avgpooling_op.cc rename to tensorflow/core/kernels/mkl/mkl_avgpooling_op.cc index a238f51860b..754156c860a 100644 --- a/tensorflow/core/kernels/mkl_avgpooling_op.cc +++ b/tensorflow/core/kernels/mkl/mkl_avgpooling_op.cc @@ -21,7 +21,7 @@ #include "tensorflow/core/framework/common_shape_fns.h" #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/mkl_pooling_ops_common.h" +#include "tensorflow/core/kernels/mkl/mkl_pooling_ops_common.h" #include "tensorflow/core/util/mkl_types.h" #include "tensorflow/core/util/mkl_util.h" diff --git a/tensorflow/core/kernels/mkl_batch_matmul_op.cc b/tensorflow/core/kernels/mkl/mkl_batch_matmul_op.cc similarity index 99% rename from tensorflow/core/kernels/mkl_batch_matmul_op.cc rename to tensorflow/core/kernels/mkl/mkl_batch_matmul_op.cc index b65c70566b5..da5a239c224 100644 --- a/tensorflow/core/kernels/mkl_batch_matmul_op.cc +++ b/tensorflow/core/kernels/mkl/mkl_batch_matmul_op.cc @@ -45,7 +45,7 @@ limitations under the License. #include "tensorflow/core/framework/types.h" #include "tensorflow/core/kernels/batch_matmul_op_impl.h" #include "tensorflow/core/kernels/fill_functor.h" -#include "tensorflow/core/kernels/mkl_matmul_ops_common.h" +#include "tensorflow/core/kernels/mkl/mkl_matmul_ops_common.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/matmul_bcast.h" diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl/mkl_concat_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_concat_op.cc rename to tensorflow/core/kernels/mkl/mkl_concat_op.cc diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl/mkl_conv_grad_filter_ops.cc similarity index 99% rename from tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc rename to tensorflow/core/kernels/mkl/mkl_conv_grad_filter_ops.cc index 12581d0bfa5..339ab938cca 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc +++ b/tensorflow/core/kernels/mkl/mkl_conv_grad_filter_ops.cc @@ -28,7 +28,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_slice.h" #include "tensorflow/core/kernels/conv_grad_ops.h" -#include "tensorflow/core/kernels/mkl_conv_ops.h" +#include "tensorflow/core/kernels/mkl/mkl_conv_ops.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/gtl/array_slice.h" diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl/mkl_conv_grad_input_ops.cc similarity index 99% rename from tensorflow/core/kernels/mkl_conv_grad_input_ops.cc rename to tensorflow/core/kernels/mkl/mkl_conv_grad_input_ops.cc index 7177431029a..2e700d0a627 100644 --- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc +++ b/tensorflow/core/kernels/mkl/mkl_conv_grad_input_ops.cc @@ -35,7 +35,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_util.h" #include "tensorflow/core/kernels/conv_grad_ops.h" #include "tensorflow/core/kernels/conv_grad_shape_utils.h" -#include "tensorflow/core/kernels/mkl_conv_ops.h" +#include "tensorflow/core/kernels/mkl/mkl_conv_ops.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/gtl/array_slice.h" diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl/mkl_conv_ops.cc similarity index 99% rename from tensorflow/core/kernels/mkl_conv_ops.cc rename to tensorflow/core/kernels/mkl/mkl_conv_ops.cc index 210044436aa..84fa20ed221 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.cc +++ b/tensorflow/core/kernels/mkl/mkl_conv_ops.cc @@ -16,7 +16,7 @@ limitations under the License. // See docs in ../ops/nn_ops.cc. #ifdef INTEL_MKL -#include "tensorflow/core/kernels/mkl_conv_ops.h" +#include "tensorflow/core/kernels/mkl/mkl_conv_ops.h" #include #include @@ -33,7 +33,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_slice.h" -#include "tensorflow/core/kernels/mkl_quantized_conv_ops.h" +#include "tensorflow/core/kernels/mkl/mkl_quantized_conv_ops.h" #include "tensorflow/core/kernels/no_op.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/errors.h" diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl/mkl_conv_ops.h similarity index 99% rename from tensorflow/core/kernels/mkl_conv_ops.h rename to tensorflow/core/kernels/mkl/mkl_conv_ops.h index 2ee2a621067..c4a4942e877 100644 --- a/tensorflow/core/kernels/mkl_conv_ops.h +++ b/tensorflow/core/kernels/mkl/mkl_conv_ops.h @@ -13,9 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MKL_CONV_OPS_H_ -#define TENSORFLOW_CORE_KERNELS_MKL_CONV_OPS_H_ +#ifndef TENSORFLOW_CORE_KERNELS_MKL_MKL_CONV_OPS_H_ +#define TENSORFLOW_CORE_KERNELS_MKL_MKL_CONV_OPS_H_ +#ifdef INTEL_MKL #include #include #include @@ -640,4 +641,5 @@ class MklDummyOp : public OpKernel { } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_MKL_CONV_OPS_H_ +#endif // INTEL_MKL +#endif // TENSORFLOW_CORE_KERNELS_MKL_MKL_CONV_OPS_H_ diff --git a/tensorflow/core/kernels/mkl_conv_ops_test.cc b/tensorflow/core/kernels/mkl/mkl_conv_ops_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_conv_ops_test.cc rename to tensorflow/core/kernels/mkl/mkl_conv_ops_test.cc diff --git a/tensorflow/core/kernels/mkl_cwise_ops_common.cc b/tensorflow/core/kernels/mkl/mkl_cwise_ops_common.cc similarity index 100% rename from tensorflow/core/kernels/mkl_cwise_ops_common.cc rename to tensorflow/core/kernels/mkl/mkl_cwise_ops_common.cc diff --git a/tensorflow/core/kernels/mkl_dequantize_op.cc b/tensorflow/core/kernels/mkl/mkl_dequantize_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_dequantize_op.cc rename to tensorflow/core/kernels/mkl/mkl_dequantize_op.cc diff --git a/tensorflow/core/kernels/mkl_dequantize_op_test.cc b/tensorflow/core/kernels/mkl/mkl_dequantize_op_test.cc similarity index 88% rename from tensorflow/core/kernels/mkl_dequantize_op_test.cc rename to tensorflow/core/kernels/mkl/mkl_dequantize_op_test.cc index b400fb761cb..564c2829e99 100644 --- a/tensorflow/core/kernels/mkl_dequantize_op_test.cc +++ b/tensorflow/core/kernels/mkl/mkl_dequantize_op_test.cc @@ -62,23 +62,6 @@ TEST_F(MklDequantizeOpTest, small) { test::ExpectTensorNear(expected, output, 0.1); } -Tensor CreateMklInput() { - MklDnnShape mkl_shape; - memory::desc md = - memory::desc({1, 2, 2, 2}, MklDnnType(), memory::format::nhwc); - mkl_shape.SetMklTensor(true); - mkl_shape.SetMklLayout(&md); - mkl_shape.SetElemType(MklDnnType()); - mkl_shape.SetTfLayout(4, {1, 2, 2, 2}, memory::format::nhwc); - - DataType dtype = DataTypeToEnum::v(); - Tensor mkl_tensor(dtype, {mkl_shape.GetSerializeBufferSize()}); - mkl_shape.SerializeMklDnnShape( - mkl_tensor.flat().data(), - mkl_tensor.flat().size() * sizeof(uint8)); - return mkl_tensor; -} - template class CommonTestUtilities : public OpsTestBase { public: @@ -129,8 +112,7 @@ TEST_F(MklDequantizeOpTest, MKLInput) { AddInputFromArray(TensorShape({1}), {0}); // max_range = 200 AddInputFromArray(TensorShape({1}), {200.0f}); - auto mkl_tensor = CreateMklInput(); - AddInputFromArray(mkl_tensor.shape(), mkl_tensor.flat()); + AddInputFromArray(dummy_shape, dummy_tensor); AddInputFromArray(dummy_shape, dummy_tensor); AddInputFromArray(dummy_shape, dummy_tensor); TF_ASSERT_OK(RunOpKernel()); diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl/mkl_fused_batch_norm_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_fused_batch_norm_op.cc rename to tensorflow/core/kernels/mkl/mkl_fused_batch_norm_op.cc diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op_test.cc b/tensorflow/core/kernels/mkl/mkl_fused_batch_norm_op_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_fused_batch_norm_op_test.cc rename to tensorflow/core/kernels/mkl/mkl_fused_batch_norm_op_test.cc diff --git a/tensorflow/core/kernels/mkl_fused_ops_test.cc b/tensorflow/core/kernels/mkl/mkl_fused_ops_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_fused_ops_test.cc rename to tensorflow/core/kernels/mkl/mkl_fused_ops_test.cc diff --git a/tensorflow/core/kernels/mkl_identity_op.cc b/tensorflow/core/kernels/mkl/mkl_identity_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_identity_op.cc rename to tensorflow/core/kernels/mkl/mkl_identity_op.cc diff --git a/tensorflow/core/kernels/mkl_input_conversion_op.cc b/tensorflow/core/kernels/mkl/mkl_input_conversion_op.cc similarity index 99% rename from tensorflow/core/kernels/mkl_input_conversion_op.cc rename to tensorflow/core/kernels/mkl/mkl_input_conversion_op.cc index f7866cbcea6..ae130700a8d 100644 --- a/tensorflow/core/kernels/mkl_input_conversion_op.cc +++ b/tensorflow/core/kernels/mkl/mkl_input_conversion_op.cc @@ -17,21 +17,21 @@ limitations under the License. #include #include + +#include "mkldnn.hpp" #include "tensorflow/core/framework/numeric_op.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/kernels/mkl/mkl_tfconv_op.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/platform/byte_order.h" #include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/util/tensor_format.h" - -#include "mkldnn.hpp" -#include "tensorflow/core/kernels/mkl_tfconv_op.h" #include "tensorflow/core/util/mkl_util.h" +#include "tensorflow/core/util/tensor_format.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/mkl_lrn_op.cc b/tensorflow/core/kernels/mkl/mkl_lrn_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_lrn_op.cc rename to tensorflow/core/kernels/mkl/mkl_lrn_op.cc diff --git a/tensorflow/core/kernels/mkl_matmul_op.cc b/tensorflow/core/kernels/mkl/mkl_matmul_op.cc similarity index 99% rename from tensorflow/core/kernels/mkl_matmul_op.cc rename to tensorflow/core/kernels/mkl/mkl_matmul_op.cc index c92fceb415c..81339489223 100644 --- a/tensorflow/core/kernels/mkl_matmul_op.cc +++ b/tensorflow/core/kernels/mkl/mkl_matmul_op.cc @@ -34,7 +34,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/kernels/fill_functor.h" -#include "tensorflow/core/kernels/mkl_matmul_ops_common.h" +#include "tensorflow/core/kernels/mkl/mkl_matmul_ops_common.h" #include "tensorflow/core/util/mkl_util.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/mkl_matmul_op_fused.cc b/tensorflow/core/kernels/mkl/mkl_matmul_op_fused.cc similarity index 99% rename from tensorflow/core/kernels/mkl_matmul_op_fused.cc rename to tensorflow/core/kernels/mkl/mkl_matmul_op_fused.cc index 9e05d3c0cfe..4dd7e3f8c6e 100644 --- a/tensorflow/core/kernels/mkl_matmul_op_fused.cc +++ b/tensorflow/core/kernels/mkl/mkl_matmul_op_fused.cc @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/kernels/fill_functor.h" -#include "tensorflow/core/kernels/mkl_matmul_ops_common.h" +#include "tensorflow/core/kernels/mkl/mkl_matmul_ops_common.h" #include "tensorflow/core/lib/core/errors.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/mkl_matmul_ops_common.h b/tensorflow/core/kernels/mkl/mkl_matmul_ops_common.h similarity index 99% rename from tensorflow/core/kernels/mkl_matmul_ops_common.h rename to tensorflow/core/kernels/mkl/mkl_matmul_ops_common.h index f8242d06fa6..fc03374a414 100644 --- a/tensorflow/core/kernels/mkl_matmul_ops_common.h +++ b/tensorflow/core/kernels/mkl/mkl_matmul_ops_common.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MKL_MATMUL_OPS_COMMON_H_ -#define TENSORFLOW_CORE_KERNELS_MKL_MATMUL_OPS_COMMON_H_ +#ifndef TENSORFLOW_CORE_KERNELS_MKL_MKL_MATMUL_OPS_COMMON_H_ +#define TENSORFLOW_CORE_KERNELS_MKL_MKL_MATMUL_OPS_COMMON_H_ #ifdef INTEL_MKL #include @@ -41,6 +41,7 @@ typedef Eigen::ThreadPoolDevice CPUDevice; typedef enum { CblasRowMajor, CblasColumnMajor } CBLAS_LAYOUT; #define MKL_INT int #endif + // This structure aggregates multiple inputs to MklDnnMatMul* methods. struct MklDnnMatMulFwdParams { memory::dims src_dims; @@ -817,4 +818,4 @@ void dnnl_gemm(char transa, char transb, int64_t m, int64_t n, int64_t k, } // namespace tensorflow #endif // INTEL_MKL -#endif // TENSORFLOW_CORE_KERNELS_MKL_MATMUL_OPS_COMMON_H_ +#endif // TENSORFLOW_CORE_KERNELS_MKL_MKL_MATMUL_OPS_COMMON_H_ diff --git a/tensorflow/core/kernels/mkl_maxpooling_op.cc b/tensorflow/core/kernels/mkl/mkl_maxpooling_op.cc similarity index 99% rename from tensorflow/core/kernels/mkl_maxpooling_op.cc rename to tensorflow/core/kernels/mkl/mkl_maxpooling_op.cc index 3ed6b9d02a2..ca7ebd7fd12 100644 --- a/tensorflow/core/kernels/mkl_maxpooling_op.cc +++ b/tensorflow/core/kernels/mkl/mkl_maxpooling_op.cc @@ -22,7 +22,7 @@ limitations under the License. #include "mkldnn.hpp" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/mkl_pooling_ops_common.h" +#include "tensorflow/core/kernels/mkl/mkl_pooling_ops_common.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/util/mkl_types.h" #include "tensorflow/core/util/mkl_util.h" diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.cc b/tensorflow/core/kernels/mkl/mkl_pooling_ops_common.cc similarity index 99% rename from tensorflow/core/kernels/mkl_pooling_ops_common.cc rename to tensorflow/core/kernels/mkl/mkl_pooling_ops_common.cc index c7ad39ddb50..9824fabce0e 100644 --- a/tensorflow/core/kernels/mkl_pooling_ops_common.cc +++ b/tensorflow/core/kernels/mkl/mkl_pooling_ops_common.cc @@ -15,7 +15,7 @@ limitations under the License. #ifdef INTEL_MKL -#include "tensorflow/core/kernels/mkl_pooling_ops_common.h" +#include "tensorflow/core/kernels/mkl/mkl_pooling_ops_common.h" #include #include diff --git a/tensorflow/core/kernels/mkl_pooling_ops_common.h b/tensorflow/core/kernels/mkl/mkl_pooling_ops_common.h similarity index 99% rename from tensorflow/core/kernels/mkl_pooling_ops_common.h rename to tensorflow/core/kernels/mkl/mkl_pooling_ops_common.h index 3d5498ed77b..3a608a66c16 100644 --- a/tensorflow/core/kernels/mkl_pooling_ops_common.h +++ b/tensorflow/core/kernels/mkl/mkl_pooling_ops_common.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MKL_POOLING_OPS_COMMON_H_ -#define TENSORFLOW_CORE_KERNELS_MKL_POOLING_OPS_COMMON_H_ +#ifndef TENSORFLOW_CORE_KERNELS_MKL_MKL_POOLING_OPS_COMMON_H_ +#define TENSORFLOW_CORE_KERNELS_MKL_MKL_POOLING_OPS_COMMON_H_ #ifdef INTEL_MKL @@ -728,4 +728,4 @@ class MklPoolingBackwardOpBase : public MklPoolingOpBase { } // namespace tensorflow #endif // INTEL_MKL -#endif // TENSORFLOW_CORE_KERNELS_MKL_POOLING_OPS_COMMON_H_ +#endif // TENSORFLOW_CORE_KERNELS_MKL_MKL_POOLING_OPS_COMMON_H_ diff --git a/tensorflow/core/kernels/mkl_qmatmul_op.cc b/tensorflow/core/kernels/mkl/mkl_qmatmul_op.cc similarity index 99% rename from tensorflow/core/kernels/mkl_qmatmul_op.cc rename to tensorflow/core/kernels/mkl/mkl_qmatmul_op.cc index b59612433e6..1cc1945dd4b 100644 --- a/tensorflow/core/kernels/mkl_qmatmul_op.cc +++ b/tensorflow/core/kernels/mkl/mkl_qmatmul_op.cc @@ -94,8 +94,8 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/kernels/fill_functor.h" -#include "tensorflow/core/kernels/mkl_matmul_ops_common.h" -#include "tensorflow/core/kernels/mkl_quantized_conv_ops.h" +#include "tensorflow/core/kernels/mkl/mkl_matmul_ops_common.h" +#include "tensorflow/core/kernels/mkl/mkl_quantized_conv_ops.h" #include "tensorflow/core/kernels/no_op.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/util/mkl_threadpool.h" diff --git a/tensorflow/core/kernels/mkl_qmatmul_op_test.cc b/tensorflow/core/kernels/mkl/mkl_qmatmul_op_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_qmatmul_op_test.cc rename to tensorflow/core/kernels/mkl/mkl_qmatmul_op_test.cc diff --git a/tensorflow/core/kernels/mkl_quantize_op.cc b/tensorflow/core/kernels/mkl/mkl_quantize_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_quantize_op.cc rename to tensorflow/core/kernels/mkl/mkl_quantize_op.cc diff --git a/tensorflow/core/kernels/mkl_quantize_op_test.cc b/tensorflow/core/kernels/mkl/mkl_quantize_op_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_quantize_op_test.cc rename to tensorflow/core/kernels/mkl/mkl_quantize_op_test.cc diff --git a/tensorflow/core/kernels/mkl_quantized_concat_op_test.cc b/tensorflow/core/kernels/mkl/mkl_quantized_concat_op_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_quantized_concat_op_test.cc rename to tensorflow/core/kernels/mkl/mkl_quantized_concat_op_test.cc diff --git a/tensorflow/core/kernels/mkl_quantized_conv_ops.h b/tensorflow/core/kernels/mkl/mkl_quantized_conv_ops.h similarity index 95% rename from tensorflow/core/kernels/mkl_quantized_conv_ops.h rename to tensorflow/core/kernels/mkl/mkl_quantized_conv_ops.h index 4121c88fb83..9fd699cf704 100644 --- a/tensorflow/core/kernels/mkl_quantized_conv_ops.h +++ b/tensorflow/core/kernels/mkl/mkl_quantized_conv_ops.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MKL_QUANTIZED_CONV_OPS_H_ -#define TENSORFLOW_CORE_KERNELS_MKL_QUANTIZED_CONV_OPS_H_ +#ifndef TENSORFLOW_CORE_KERNELS_MKL_MKL_QUANTIZED_CONV_OPS_H_ +#define TENSORFLOW_CORE_KERNELS_MKL_MKL_QUANTIZED_CONV_OPS_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/tensor.h" @@ -90,4 +90,4 @@ void MklQuantizationRangeForMultiplication(float min_a, float max_a, #endif // INTEL_MKL -#endif // TENSORFLOW_CORE_KERNELS_MKL_QUANTIZED_CONV_OPS_H_ +#endif // TENSORFLOW_CORE_KERNELS_MKL_MKL_QUANTIZED_CONV_OPS_H_ diff --git a/tensorflow/core/kernels/mkl_quantized_conv_ops_perchannel_test.cc b/tensorflow/core/kernels/mkl/mkl_quantized_conv_ops_perchannel_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_quantized_conv_ops_perchannel_test.cc rename to tensorflow/core/kernels/mkl/mkl_quantized_conv_ops_perchannel_test.cc diff --git a/tensorflow/core/kernels/mkl_quantized_conv_ops_test.cc b/tensorflow/core/kernels/mkl/mkl_quantized_conv_ops_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_quantized_conv_ops_test.cc rename to tensorflow/core/kernels/mkl/mkl_quantized_conv_ops_test.cc diff --git a/tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc b/tensorflow/core/kernels/mkl/mkl_quantized_pooling_ops_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_quantized_pooling_ops_test.cc rename to tensorflow/core/kernels/mkl/mkl_quantized_pooling_ops_test.cc diff --git a/tensorflow/core/kernels/mkl_relu_op.cc b/tensorflow/core/kernels/mkl/mkl_relu_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_relu_op.cc rename to tensorflow/core/kernels/mkl/mkl_relu_op.cc diff --git a/tensorflow/core/kernels/mkl_relu_op_test.cc b/tensorflow/core/kernels/mkl/mkl_relu_op_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_relu_op_test.cc rename to tensorflow/core/kernels/mkl/mkl_relu_op_test.cc diff --git a/tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc b/tensorflow/core/kernels/mkl/mkl_requantization_range_per_channel_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_requantization_range_per_channel_op.cc rename to tensorflow/core/kernels/mkl/mkl_requantization_range_per_channel_op.cc diff --git a/tensorflow/core/kernels/mkl_requantize_ops_test.cc b/tensorflow/core/kernels/mkl/mkl_requantize_ops_test.cc similarity index 100% rename from tensorflow/core/kernels/mkl_requantize_ops_test.cc rename to tensorflow/core/kernels/mkl/mkl_requantize_ops_test.cc diff --git a/tensorflow/core/kernels/mkl_requantize_per_channel_op.cc b/tensorflow/core/kernels/mkl/mkl_requantize_per_channel_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_requantize_per_channel_op.cc rename to tensorflow/core/kernels/mkl/mkl_requantize_per_channel_op.cc diff --git a/tensorflow/core/kernels/mkl_reshape_op.cc b/tensorflow/core/kernels/mkl/mkl_reshape_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_reshape_op.cc rename to tensorflow/core/kernels/mkl/mkl_reshape_op.cc diff --git a/tensorflow/core/kernels/mkl_slice_op.cc b/tensorflow/core/kernels/mkl/mkl_slice_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_slice_op.cc rename to tensorflow/core/kernels/mkl/mkl_slice_op.cc diff --git a/tensorflow/core/kernels/mkl_softmax_op.cc b/tensorflow/core/kernels/mkl/mkl_softmax_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_softmax_op.cc rename to tensorflow/core/kernels/mkl/mkl_softmax_op.cc diff --git a/tensorflow/core/kernels/mkl_tfconv_op.h b/tensorflow/core/kernels/mkl/mkl_tfconv_op.h similarity index 97% rename from tensorflow/core/kernels/mkl_tfconv_op.h rename to tensorflow/core/kernels/mkl/mkl_tfconv_op.h index f7aa4d2bebf..0a603ee2c12 100644 --- a/tensorflow/core/kernels/mkl_tfconv_op.h +++ b/tensorflow/core/kernels/mkl/mkl_tfconv_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MKL_TFCONV_OP_H_ -#define TENSORFLOW_CORE_KERNELS_MKL_TFCONV_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_MKL_MKL_TFCONV_OP_H_ +#define TENSORFLOW_CORE_KERNELS_MKL_MKL_TFCONV_OP_H_ #ifdef INTEL_MKL @@ -160,4 +160,4 @@ TF_CALL_QUANTIZED_TYPES(REGISTER_CPU); } // namespace tensorflow #endif // INTEL_MKL -#endif // TENSORFLOW_CORE_KERNELS_MKL_TFCONV_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_MKL_MKL_TFCONV_OP_H_ diff --git a/tensorflow/core/kernels/mkl_tmp_bf16_ops.cc b/tensorflow/core/kernels/mkl/mkl_tmp_bf16_ops.cc similarity index 100% rename from tensorflow/core/kernels/mkl_tmp_bf16_ops.cc rename to tensorflow/core/kernels/mkl/mkl_tmp_bf16_ops.cc diff --git a/tensorflow/core/kernels/mkl_transpose_op.cc b/tensorflow/core/kernels/mkl/mkl_transpose_op.cc similarity index 100% rename from tensorflow/core/kernels/mkl_transpose_op.cc rename to tensorflow/core/kernels/mkl/mkl_transpose_op.cc From 59affc4d61328c86baf680f2d818b43da13373cc Mon Sep 17 00:00:00 2001 From: Yanhui Liang Date: Fri, 7 Aug 2020 17:18:52 -0700 Subject: [PATCH 0688/1017] Refactor LSTM and GRU layer with `tf.function` and stateless Case op. PiperOrigin-RevId: 325538595 Change-Id: Iecc5789d33f9dbfc221aedc54500745c79230022 --- .../python/keras/layers/recurrent_v2.py | 306 ++++++++++++------ 1 file changed, 205 insertions(+), 101 deletions(-) diff --git a/tensorflow/python/keras/layers/recurrent_v2.py b/tensorflow/python/keras/layers/recurrent_v2.py index 878269dee5e..a2ed7141608 100644 --- a/tensorflow/python/keras/layers/recurrent_v2.py +++ b/tensorflow/python/keras/layers/recurrent_v2.py @@ -20,6 +20,7 @@ from __future__ import print_function import uuid +from tensorflow.python.compat import compat from tensorflow.python.eager import context from tensorflow.python.eager import function from tensorflow.python.framework import constant_op @@ -386,6 +387,20 @@ class GRU(recurrent.DropoutRNNCellMixin, recurrent.GRU): else: logging.warn(_CUDNN_NOT_AVAILABLE_MSG % self.name) + # TODO(b/162616551): Remove all compat statements after 08/20/2020. + # This follows b/161915509 and is mainly to test the stateless Case op. + if compat.forward_compatible(2020, 8, 20): + # The first two attributes are added to support TFLite use case. + supportive_attributes = { + 'time_major': time_major, + 'go_backwards': go_backwards, + _FUNCTION_API_NAME_ATTRIBUTE: 'gru_' + str(uuid.uuid4()) + } + self.defun_gru_with_backend_selection = function.defun_with_attributes( + gru_with_backend_selection, + attributes=supportive_attributes, + autograph=False) + def build(self, input_shape): super(GRU, self).build(input_shape) @@ -468,38 +483,54 @@ class GRU(recurrent.DropoutRNNCellMixin, recurrent.GRU): if dropout_mask is not None: inputs = inputs * dropout_mask[0] - gpu_gru_kwargs = { - 'inputs': inputs, - 'init_h': _read_variable_value(initial_state[0]), - 'kernel': _read_variable_value(self.cell.kernel), - 'recurrent_kernel': _read_variable_value(self.cell.recurrent_kernel), - 'bias': _read_variable_value(self.cell.bias), - 'mask': mask, - 'time_major': self.time_major, - 'go_backwards': self.go_backwards, - 'sequence_lengths': sequence_lengths - } - normal_gru_kwargs = gpu_gru_kwargs.copy() - normal_gru_kwargs.update({ - 'zero_output_for_mask': self.zero_output_for_mask, - }) - - if context.executing_eagerly(): - device_type = _get_context_device_type() - can_use_gpu = ( - # Either user specified GPU or unspecified but GPU is available. - (device_type == _GPU_DEVICE_NAME - or (device_type is None and context.num_gpus() > 0)) - and - (mask is None or is_cudnn_supported_inputs(mask, self.time_major))) - # Under eager context, check the device placement and prefer the - if can_use_gpu: - last_output, outputs, new_h, runtime = gpu_gru(**gpu_gru_kwargs) - else: - last_output, outputs, new_h, runtime = standard_gru(**normal_gru_kwargs) + if compat.forward_compatible(2020, 8, 20): + gru_kwargs = { + 'inputs': inputs, + 'init_h': _read_variable_value(initial_state[0]), + 'kernel': _read_variable_value(self.cell.kernel), + 'recurrent_kernel': _read_variable_value(self.cell.recurrent_kernel), + 'bias': _read_variable_value(self.cell.bias), + 'mask': mask, + 'time_major': self.time_major, + 'go_backwards': self.go_backwards, + 'sequence_lengths': sequence_lengths, + 'zero_output_for_mask': self.zero_output_for_mask + } + (last_output, outputs, new_h, + runtime) = self.defun_gru_with_backend_selection(**gru_kwargs) else: - last_output, outputs, new_h, runtime = gru_with_backend_selection( - **normal_gru_kwargs) + gpu_gru_kwargs = { + 'inputs': inputs, + 'init_h': _read_variable_value(initial_state[0]), + 'kernel': _read_variable_value(self.cell.kernel), + 'recurrent_kernel': _read_variable_value(self.cell.recurrent_kernel), + 'bias': _read_variable_value(self.cell.bias), + 'mask': mask, + 'time_major': self.time_major, + 'go_backwards': self.go_backwards, + 'sequence_lengths': sequence_lengths + } + normal_gru_kwargs = gpu_gru_kwargs.copy() + normal_gru_kwargs.update({ + 'zero_output_for_mask': self.zero_output_for_mask, + }) + + if context.executing_eagerly(): + device_type = _get_context_device_type() + can_use_gpu = ( + # Either user specified GPU or unspecified but GPU is available. + (device_type == _GPU_DEVICE_NAME or + (device_type is None and context.num_gpus() > 0)) and + (mask is None or is_cudnn_supported_inputs(mask, self.time_major))) + # Under eager context, check the device placement and prefer the + if can_use_gpu: + last_output, outputs, new_h, runtime = gpu_gru(**gpu_gru_kwargs) + else: + last_output, outputs, new_h, runtime = standard_gru( + **normal_gru_kwargs) + else: + last_output, outputs, new_h, runtime = gru_with_backend_selection( + **normal_gru_kwargs) states = [new_h] return last_output, outputs, runtime, states @@ -766,24 +797,36 @@ def gru_with_backend_selection(inputs, init_h, kernel, recurrent_kernel, bias, true_fn=cudnn_gru_fn, false_fn=standard_gru_fn) - # Each time a `tf.function` is called, we will give it a unique - # identifiable API name, so that Grappler won't get confused when it - # sees multiple GRU layers added into same graph, and it will be able - # to pair up the different implementations across them. - api_name = 'gru_' + str(uuid.uuid4()) - supportive_attribute = { - 'time_major': time_major, - 'go_backwards': go_backwards, - } - defun_standard_gru = _generate_defun_backend( - api_name, _CPU_DEVICE_NAME, standard_gru, supportive_attribute) - defun_gpu_gru = _generate_defun_backend( - api_name, _GPU_DEVICE_NAME, gpu_gru_with_fallback, supportive_attribute) + if compat.forward_compatible(2020, 8, 20): + # Chooses the implementation dynamicly based on the running device. + (last_output, outputs, new_h, + runtime) = control_flow_ops.execute_fn_for_device( + { + _CPU_DEVICE_NAME: lambda: standard_gru(**params), + _GPU_DEVICE_NAME: lambda: gpu_gru_with_fallback(**params) + }, lambda: standard_gru(**params)) + else: + # Each time a `tf.function` is called, we will give it a unique + # identifiable API name, so that Grappler won't get confused when it + # sees multiple GRU layers added into same graph, and it will be able + # to pair up the different implementations across them. + api_name = 'gru_' + str(uuid.uuid4()) + supportive_attribute = { + 'time_major': time_major, + 'go_backwards': go_backwards, + } + defun_standard_gru = _generate_defun_backend(api_name, _CPU_DEVICE_NAME, + standard_gru, + supportive_attribute) + defun_gpu_gru = _generate_defun_backend(api_name, _GPU_DEVICE_NAME, + gpu_gru_with_fallback, + supportive_attribute) + + # Call the normal GRU impl and register the CuDNN impl function. The + # grappler will kick in during session execution to optimize the graph. + last_output, outputs, new_h, runtime = defun_standard_gru(**params) + function.register(defun_gpu_gru, **params) - # Call the normal GRU impl and register the CuDNN impl function. The - # grappler will kick in during session execution to optimize the graph. - last_output, outputs, new_h, runtime = defun_standard_gru(**params) - function.register(defun_gpu_gru, **params) return last_output, outputs, new_h, runtime @@ -1098,6 +1141,19 @@ class LSTM(recurrent.DropoutRNNCellMixin, recurrent.LSTM): else: logging.warn(_CUDNN_NOT_AVAILABLE_MSG % self.name) + if compat.forward_compatible(2020, 8, 20): + # The first two attributes are added to support TFLite use case. + supportive_attributes = { + 'time_major': time_major, + 'go_backwards': go_backwards, + _FUNCTION_API_NAME_ATTRIBUTE: 'lstm_' + str(uuid.uuid4()) + } + + self.defun_lstm_with_backend_selection = function.defun_with_attributes( + lstm_with_backend_selection, + attributes=supportive_attributes, + autograph=False) + def call(self, inputs, mask=None, training=None, initial_state=None): # The input should be dense, padded with zeros. If a ragged input is fed # into the layer, it is padded and the row lengths are used for masking. @@ -1146,42 +1202,80 @@ class LSTM(recurrent.DropoutRNNCellMixin, recurrent.LSTM): dropout_mask = self.get_dropout_mask_for_cell(inputs, training, count=4) if dropout_mask is not None: inputs = inputs * dropout_mask[0] - gpu_lstm_kwargs = { - 'inputs': inputs, - 'init_h': _read_variable_value(initial_state[0]), - 'init_c': _read_variable_value(initial_state[1]), - 'kernel': _read_variable_value(self.cell.kernel), - 'recurrent_kernel': _read_variable_value(self.cell.recurrent_kernel), - 'bias': _read_variable_value(self.cell.bias), - 'mask': mask, - 'time_major': self.time_major, - 'go_backwards': self.go_backwards, - 'sequence_lengths': row_lengths - } - normal_lstm_kwargs = gpu_lstm_kwargs.copy() - normal_lstm_kwargs.update({ - 'zero_output_for_mask': self.zero_output_for_mask, - }) - - if context.executing_eagerly(): - device_type = _get_context_device_type() - can_use_gpu = ( - # Either user specified GPU or unspecified but GPU is available. - (device_type == _GPU_DEVICE_NAME - or (device_type is None and context.num_gpus() > 0)) - and - (mask is None or is_cudnn_supported_inputs(mask, self.time_major))) - # Under eager context, check the device placement and prefer the - # GPU implementation when GPU is available. - if can_use_gpu: - last_output, outputs, new_h, new_c, runtime = gpu_lstm( - **gpu_lstm_kwargs) - else: - last_output, outputs, new_h, new_c, runtime = standard_lstm( - **normal_lstm_kwargs) - else: + if compat.forward_compatible(2020, 8, 20): + lstm_kwargs = { + 'inputs': + inputs, + 'init_h': + _read_variable_value(initial_state[0]), + 'init_c': + _read_variable_value(initial_state[1]), + 'kernel': + _read_variable_value(self.cell.kernel), + 'recurrent_kernel': + _read_variable_value(self.cell.recurrent_kernel), + 'bias': + _read_variable_value(self.cell.bias), + 'mask': + mask, + 'time_major': + self.time_major, + 'go_backwards': + self.go_backwards, + 'sequence_lengths': + row_lengths, + 'zero_output_for_mask': + self.zero_output_for_mask, + } (last_output, outputs, new_h, new_c, - runtime) = lstm_with_backend_selection(**normal_lstm_kwargs) + runtime) = self.defun_lstm_with_backend_selection(**lstm_kwargs) + else: + gpu_lstm_kwargs = { + 'inputs': + inputs, + 'init_h': + _read_variable_value(initial_state[0]), + 'init_c': + _read_variable_value(initial_state[1]), + 'kernel': + _read_variable_value(self.cell.kernel), + 'recurrent_kernel': + _read_variable_value(self.cell.recurrent_kernel), + 'bias': + _read_variable_value(self.cell.bias), + 'mask': + mask, + 'time_major': + self.time_major, + 'go_backwards': + self.go_backwards, + 'sequence_lengths': + row_lengths + } + normal_lstm_kwargs = gpu_lstm_kwargs.copy() + normal_lstm_kwargs.update({ + 'zero_output_for_mask': self.zero_output_for_mask, + }) + + if context.executing_eagerly(): + device_type = _get_context_device_type() + can_use_gpu = ( + # Either user specified GPU or unspecified but GPU is available. + (device_type == _GPU_DEVICE_NAME or + (device_type is None and context.num_gpus() > 0)) and + (mask is None or + is_cudnn_supported_inputs(mask, self.time_major))) + # Under eager context, check the device placement and prefer the + # GPU implementation when GPU is available. + if can_use_gpu: + last_output, outputs, new_h, new_c, runtime = gpu_lstm( + **gpu_lstm_kwargs) + else: + last_output, outputs, new_h, new_c, runtime = standard_lstm( + **normal_lstm_kwargs) + else: + (last_output, outputs, new_h, new_c, + runtime) = lstm_with_backend_selection(**normal_lstm_kwargs) states = [new_h, new_c] @@ -1539,25 +1633,35 @@ def lstm_with_backend_selection(inputs, init_h, init_c, kernel, true_fn=cudnn_lstm_fn, false_fn=stardard_lstm_fn) - # Each time a `tf.function` is called, we will give it a unique - # identifiable API name, so that Grappler won't get confused when it - # sees multiple LSTM layers added into same graph, and it will be able - # to pair up the different implementations across them. - api_name = 'lstm_' + str(uuid.uuid4()) - supportive_attribute = { - 'time_major': time_major, - 'go_backwards': go_backwards, - } - defun_standard_lstm = _generate_defun_backend( - api_name, _CPU_DEVICE_NAME, standard_lstm, supportive_attribute) - defun_gpu_lstm = _generate_defun_backend( - api_name, _GPU_DEVICE_NAME, gpu_lstm_with_fallback, supportive_attribute) + if compat.forward_compatible(2020, 8, 20): + # Chooses the implementation dynamicly based on the running device. + (last_output, outputs, new_h, new_c, + runtime) = control_flow_ops.execute_fn_for_device( + { + _CPU_DEVICE_NAME: lambda: standard_lstm(**params), + _GPU_DEVICE_NAME: lambda: gpu_lstm_with_fallback(**params) + }, lambda: standard_lstm(**params)) + else: + # Each time a `tf.function` is called, we will give it a unique + # identifiable API name, so that Grappler won't get confused when it + # sees multiple LSTM layers added into same graph, and it will be able + # to pair up the different implementations across them. + api_name = 'lstm_' + str(uuid.uuid4()) + supportive_attribute = { + 'time_major': time_major, + 'go_backwards': go_backwards, + } + defun_standard_lstm = _generate_defun_backend(api_name, _CPU_DEVICE_NAME, + standard_lstm, + supportive_attribute) + defun_gpu_lstm = _generate_defun_backend(api_name, _GPU_DEVICE_NAME, + gpu_lstm_with_fallback, + supportive_attribute) - # Call the normal LSTM impl and register the CuDNN impl function. The - # grappler will kick in during session execution to optimize the graph. - last_output, outputs, new_h, new_c, runtime = defun_standard_lstm( - **params) - function.register(defun_gpu_lstm, **params) + # Call the normal LSTM impl and register the CuDNN impl function. The + # grappler will kick in during session execution to optimize the graph. + last_output, outputs, new_h, new_c, runtime = defun_standard_lstm(**params) + function.register(defun_gpu_lstm, **params) return last_output, outputs, new_h, new_c, runtime From 769155a21e632b89ffa96a6e0d27523f6b6a94b3 Mon Sep 17 00:00:00 2001 From: Ayush Dubey Date: Fri, 7 Aug 2020 17:46:31 -0700 Subject: [PATCH 0689/1017] Make ScopedAllocatorOptimizer compatible with Const input. In a previous change, we aborted ScopedAllocatorOptimizer when one of the inputs is a Const op. This change instead enables Const op to work with ScopedAllocatorOptimizer by introducing an Identity op after Const. Thus we change: Const -> CollectiveReduce to Const -> Identity -> CollectiveReduce The Identity becomes the real input to CollectiveReduce, and it will use the pre-allocated buffer slice for its output tensor when it invokes `set_output` by this logic: https://github.com/tensorflow/tensorflow/blob/6b65afa4209a8743d819693ab6c50b5df4db8af9/tensorflow/core/framework/op_kernel.cc#L903. This is similar to the approach in cl/259138773. PiperOrigin-RevId: 325541732 Change-Id: I6487685089520b73387197a31fef5780217a3a4b --- tensorflow/core/grappler/optimizers/BUILD | 1 + .../optimizers/scoped_allocator_optimizer.cc | 5 +-- .../scoped_allocator_optimizer_test.cc | 33 +++++++++++++++---- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD index d3db2f19596..2ce037178b9 100644 --- a/tensorflow/core/grappler/optimizers/BUILD +++ b/tensorflow/core/grappler/optimizers/BUILD @@ -993,6 +993,7 @@ tf_cc_test( "//tensorflow/core:test_main", "//tensorflow/core:testlib", "//tensorflow/core/grappler:grappler_item", + "//tensorflow/core/grappler:op_types", "//tensorflow/core/grappler:utils", "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder", "//tensorflow/core/grappler/utils:topological_sort", diff --git a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc index 3f33ff50f6c..11f95894ff9 100644 --- a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc +++ b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer.cc @@ -218,8 +218,9 @@ Status MaybeRewriteInput(ScopedAllocatorOptimizer* sa_opti, NodeDef* input, const string& edge_name, int output_index, NodeDef* op, NodeDef** new_input, int* new_output_index, bool* rewrite) { - *rewrite = IsExit(*input) || (sa_opti->repeated_outputs().find(edge_name) != - sa_opti->repeated_outputs().end()); + *rewrite = IsConstant(*input) || IsExit(*input) || + (sa_opti->repeated_outputs().find(edge_name) != + sa_opti->repeated_outputs().end()); if (!(*rewrite)) { *new_input = input; *new_output_index = output_index; diff --git a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer_test.cc b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer_test.cc index 905968b5fcb..4f7f4f582e4 100644 --- a/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/scoped_allocator_optimizer_test.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/graph/testlib.h" #include "tensorflow/core/grappler/grappler_item.h" +#include "tensorflow/core/grappler/op_types.h" #include "tensorflow/core/grappler/utils.h" #include "tensorflow/core/grappler/utils/topological_sort.h" #include "tensorflow/core/lib/core/status_test_util.h" @@ -241,8 +242,9 @@ class ScopedAllocatorOptimizerTest : public ::testing::Test { // Constructs the following graph. // // c1 and c2 are Const ops. a1 and a2 are Abs ops. - // We expect the optimizer to fail, because Const ops do not allocate their - // output on every Compute, and hence are not compatible with ScopedAllocator. + // We expect the optimizer to succeed and insert Identity between ci and ai. + // This will ensure that we will still be able use ScopedAllocator with Const + // inputs. /* c1 c2 | | @@ -559,7 +561,8 @@ TEST_F(ScopedAllocatorOptimizerTest, ControlEdgeRewire) { EXPECT_EQ(NumControlInputs(&node_map, "ctl4"), 1); } -// Test that the optimization fails when any input is a Const op. +// Test that the optimization succeeds when any input is a Const op, and that it +// inserts Identity op between Const and Abs. TEST_F(ScopedAllocatorOptimizerTest, ConstInput) { GrapplerItem item; BuildConstGraph(&item.graph, false); @@ -572,10 +575,26 @@ TEST_F(ScopedAllocatorOptimizerTest, ConstInput) { ons.insert("Abs"); GraphDef optimized_graph; - auto status = sao.Optimize(nullptr /*cluster*/, item, &optimized_graph); - EXPECT_EQ(status.code(), tensorflow::error::ABORTED); - EXPECT_TRUE(str_util::StrContains(status.error_message(), - "does not use AllocatorAttributes")); + TF_ASSERT_OK(sao.Optimize(nullptr /*cluster*/, item, &optimized_graph)); + + // Examine the resulting graphdef. + const NodeDef* sa_node = nullptr; + for (const NodeDef& node : optimized_graph.node()) { + if (node.op() == "_ScopedAllocator") { + sa_node = &node; + break; + } + } + ASSERT_NE(sa_node, nullptr); + int num_identity_ops = 0; + NodeMap node_map(&optimized_graph); + for (NodeDef* sa_output : node_map.GetOutputs(sa_node->name())) { + EXPECT_FALSE(IsConstant(*sa_output)); + if (IsIdentity(*sa_output)) { + ++num_identity_ops; + } + } + EXPECT_EQ(num_identity_ops, 2); } } // namespace From 3ba0deba916b52d1a8ee0b13b94352c60203072d Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Fri, 7 Aug 2020 17:51:06 -0700 Subject: [PATCH 0690/1017] Introduce additional TPU infeed and outfeed ops PiperOrigin-RevId: 325542225 Change-Id: Ie972e60d6c5639b71719837c500ecc716eda2ebd --- tensorflow/core/tpu/BUILD | 8 +- tensorflow/core/tpu/kernels/BUILD | 107 ++++ .../core/tpu/kernels/image_resize_ops.cc | 155 +++++ tensorflow/core/tpu/kernels/infeed_ops.cc | 529 ++++++++++++++++++ tensorflow/core/tpu/kernels/infeed_ops.h | 69 +++ tensorflow/core/tpu/kernels/outfeed_ops.cc | 116 ++++ tensorflow/core/tpu/kernels/outfeed_ops.h | 69 +++ .../core/tpu/kernels/replication_ops.cc | 27 + .../core/tpu/kernels/tpu_handle_to_key_op.cc | 62 ++ tensorflow/core/tpu/kernels/transfer_ops.cc | 98 ++++ tensorflow/core/tpu/kernels/transfer_ops.h | 56 ++ tensorflow/core/tpu/tpu_defs.cc | 18 + tensorflow/core/tpu/tpu_defs.h | 6 + tensorflow/core/tpu/tpu_library_init_fns.inc | 1 + tensorflow/stream_executor/tpu/BUILD | 2 + .../stream_executor/tpu/tpu_executor_c_api.h | 3 + .../tpu/tpu_transfer_manager.h | 6 + .../tpu/tpu_transfer_manager_interface.cc | 40 ++ .../tpu/tpu_transfer_manager_interface.h | 7 + 19 files changed, 1378 insertions(+), 1 deletion(-) create mode 100644 tensorflow/core/tpu/kernels/image_resize_ops.cc create mode 100644 tensorflow/core/tpu/kernels/infeed_ops.cc create mode 100644 tensorflow/core/tpu/kernels/infeed_ops.h create mode 100644 tensorflow/core/tpu/kernels/outfeed_ops.cc create mode 100644 tensorflow/core/tpu/kernels/outfeed_ops.h create mode 100644 tensorflow/core/tpu/kernels/replication_ops.cc create mode 100644 tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc create mode 100644 tensorflow/core/tpu/kernels/transfer_ops.cc create mode 100644 tensorflow/core/tpu/kernels/transfer_ops.h create mode 100644 tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.cc diff --git a/tensorflow/core/tpu/BUILD b/tensorflow/core/tpu/BUILD index 0a17ba3d408..15b2b93e46f 100644 --- a/tensorflow/core/tpu/BUILD +++ b/tensorflow/core/tpu/BUILD @@ -88,7 +88,13 @@ cc_library( name = "tpu_defs", srcs = ["tpu_defs.cc"], hdrs = ["tpu_defs.h"], - deps = ["//tensorflow/core:protos_all_cc"], + deps = [ + ":tpu_api", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/core:protos_all_cc", + "//tensorflow/stream_executor/tpu:c_api_conversions", + "//tensorflow/stream_executor/tpu:c_api_decl", + ], ) cc_library( diff --git a/tensorflow/core/tpu/kernels/BUILD b/tensorflow/core/tpu/kernels/BUILD index 1336f52ed34..157aeb3df58 100644 --- a/tensorflow/core/tpu/kernels/BUILD +++ b/tensorflow/core/tpu/kernels/BUILD @@ -28,10 +28,16 @@ tf_kernel_library( deps = [ ":cross_replica_ops", ":host_compute_ops", + ":image_resize_ops", + ":infeed_ops", + ":outfeed_ops", + ":replication_ops", ":topk_ops", ":tpu_compile_op", ":tpu_configuration_ops", ":tpu_execute_op", + ":tpu_handle_to_key_op", + ":transfer_ops", ], ) @@ -684,3 +690,104 @@ cc_library( ], alwayslink = 1, ) + +cc_library( + name = "infeed_ops", + srcs = ["infeed_ops.cc"], + hdrs = ["infeed_ops.h"], + visibility = ["//visibility:public"], + deps = [ + ":transfer_ops", + "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", + "//tensorflow/compiler/tf2xla:common", + "//tensorflow/compiler/xla:util", + "//tensorflow/core:framework", + "//tensorflow/core/common_runtime:dma_helper", + "//tensorflow/core/framework:protos_all_cc", + "//tensorflow/core/kernels:transpose_functor", + "//tensorflow/core/platform:status", + "//tensorflow/core/profiler/lib:traceme", + "//tensorflow/core/tpu:tpu_defs", + "//tensorflow/stream_executor:multi_platform_manager", + "//tensorflow/stream_executor/tpu:tpu_transfer_manager_base", + "//tensorflow/stream_executor/tpu:tpu_transfer_manager_interface", + ], + alwayslink = True, +) + +cc_library( + name = "transfer_ops", + srcs = ["transfer_ops.cc"], + hdrs = ["transfer_ops.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", + "//tensorflow/core:framework", + "//tensorflow/core:lib_internal", + "//tensorflow/core/kernels:ops_util", + "//tensorflow/core/profiler/lib:traceme", + "//tensorflow/stream_executor:multi_platform_manager", + "//tensorflow/stream_executor/tpu:tpu_node_context", + "//tensorflow/stream_executor/tpu:tpu_platform_interface", + "//tensorflow/stream_executor/tpu:tpu_transfer_manager_interface", + ], + alwayslink = True, +) + +cc_library( + name = "outfeed_ops", + srcs = ["outfeed_ops.cc"], + hdrs = ["outfeed_ops.h"], + visibility = ["//visibility:public"], + deps = [ + ":transfer_ops", + "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", + "//tensorflow/compiler/tf2xla:common", + "//tensorflow/core:framework", + "//tensorflow/core/framework:protos_all_cc", + "//tensorflow/core/tpu:tpu_defs", + "//tensorflow/stream_executor:multi_platform_manager", + ], + alwayslink = True, +) + +cc_library( + name = "image_resize_ops", + srcs = ["image_resize_ops.cc"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/tf2xla:common", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/xla/client:xla_builder", + "//tensorflow/compiler/xla/client/lib:constants", + "//tensorflow/core:framework", + "//tensorflow/core/tpu:tpu_defs", + "@com_google_absl//absl/strings", + ], + alwayslink = True, +) + +cc_library( + name = "replication_ops", + srcs = ["replication_ops.cc"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", + "//tensorflow/core:framework", + "//tensorflow/core/tpu:tpu_defs", + ], + alwayslink = True, +) + +cc_library( + name = "tpu_handle_to_key_op", + srcs = ["tpu_handle_to_key_op.cc"], + visibility = ["//visibility:public"], + deps = [ + ":tpu_compilation_cache_interface", + ":tpu_op_consts", + "//tensorflow/core:framework", + "//tensorflow/core/tpu:tpu_configuration", + ], + alwayslink = True, +) diff --git a/tensorflow/core/tpu/kernels/image_resize_ops.cc b/tensorflow/core/tpu/kernels/image_resize_ops.cc new file mode 100644 index 00000000000..fd0f5e4c7a6 --- /dev/null +++ b/tensorflow/core/tpu/kernels/image_resize_ops.cc @@ -0,0 +1,155 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/tpu/tpu_defs.h" + +namespace tensorflow { + +class TpuCustomResizeOp : public XlaOpKernel { + public: + explicit TpuCustomResizeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("align_corners", &align_corners_)); + OP_REQUIRES_OK(ctx, + ctx->GetAttr("half_pixel_centers", &half_pixel_centers_)); + } + + xla::Shape GetOutputShape(XlaOpKernelContext* ctx) const { + std::vector out_size; + auto status = ctx->ConstantInputAsIntVector(1, &out_size); + CHECK_EQ(out_size.size(), 2) << status.ToString(); + xla::Shape output_shape = + TensorShapeToXLAShape(ctx->output_xla_type(0), ctx->InputShape(0)); + output_shape.mutable_dimensions()[1] = out_size[0]; + output_shape.mutable_dimensions()[2] = out_size[1]; + return output_shape; + } + + string OpaqueField() const { + return absl::StrCat("\"", align_corners_, half_pixel_centers_, "\""); + } + + void CompileGrad(XlaOpKernelContext* ctx, const char* target, + const xla::Shape& output_shape) { + auto input_shape = + TensorShapeToXLAShape(ctx->output_xla_type(0), ctx->InputShape(0)); + if (ctx->InputShape(1).dim_sizes() == ctx->InputShape(0).dim_sizes()) { + ctx->SetOutput( + 0, xla::ConvertElementType(ctx->Input(0), ctx->output_xla_type(0))); + return; + } + // The gradient should be done in two phases for large resizes. + auto input = ctx->Input(0); + if (input_shape.dimensions(1) / output_shape.dimensions(1) > 3 && + input_shape.dimensions(2) / output_shape.dimensions(2) > 3) { + auto intermediate_shape = output_shape; + intermediate_shape.mutable_dimensions()[1] = input_shape.dimensions(1); + input = xla::CustomCall(ctx->builder(), target, {ctx->Input(0)}, + intermediate_shape, OpaqueField()); + } + ctx->SetOutput(0, xla::CustomCall(ctx->builder(), target, {input}, + output_shape, OpaqueField())); + } + + void CompileForward(XlaOpKernelContext* ctx, const char* target) { + auto output_shape = GetOutputShape(ctx); + if (ctx->InputShape(0).dim_size(1) == output_shape.dimensions(1) && + ctx->InputShape(0).dim_size(2) == output_shape.dimensions(2)) { + ctx->SetOutput( + 0, xla::ConvertElementType(ctx->Input(0), ctx->output_xla_type(0))); + return; + } + if (ctx->InputShape(0).dim_size(1) == 1 && + ctx->InputShape(0).dim_size(2) == 1) { + ctx->SetOutput(0, + ctx->Input(0) + xla::Zeros(ctx->builder(), output_shape)); + return; + } + ctx->SetOutput(0, xla::CustomCall(ctx->builder(), target, {ctx->Input(0)}, + output_shape, OpaqueField())); + } + + private: + bool align_corners_; + bool half_pixel_centers_; +}; + +class TpuResizeNearestNeighborOp : public TpuCustomResizeOp { + public: + explicit TpuResizeNearestNeighborOp(OpKernelConstruction* ctx) + : TpuCustomResizeOp(ctx) {} + void Compile(XlaOpKernelContext* ctx) override { + CompileForward(ctx, "ResizeNearest"); + } +}; + +class TpuResizeBilinearOp : public TpuCustomResizeOp { + public: + explicit TpuResizeBilinearOp(OpKernelConstruction* ctx) + : TpuCustomResizeOp(ctx) {} + void Compile(XlaOpKernelContext* ctx) override { + CompileForward(ctx, "ResizeBilinear"); + } +}; + +class TpuResizeNearestNeighborGradOp : public TpuCustomResizeOp { + public: + explicit TpuResizeNearestNeighborGradOp(OpKernelConstruction* ctx) + : TpuCustomResizeOp(ctx) {} + void Compile(XlaOpKernelContext* ctx) override { + CompileGrad(ctx, "ResizeNearestGrad", GetOutputShape(ctx)); + } +}; + +class TpuResizeBilinearGradOp : public TpuCustomResizeOp { + public: + explicit TpuResizeBilinearGradOp(OpKernelConstruction* ctx) + : TpuCustomResizeOp(ctx) {} + void Compile(XlaOpKernelContext* ctx) override { + auto output_shape = + TensorShapeToXLAShape(ctx->output_xla_type(0), ctx->InputShape(1)); + CompileGrad(ctx, "ResizeBilinearGrad", output_shape); + } +}; + +REGISTER_XLA_OP(Name("ResizeNearestNeighbor") + .CompileTimeConstantInput("size") + .Device(DEVICE_TPU_XLA_JIT), + TpuResizeNearestNeighborOp); + +REGISTER_XLA_OP(Name("ResizeNearestNeighborGrad") + .CompileTimeConstantInput("size") + .Device(DEVICE_TPU_XLA_JIT), + TpuResizeNearestNeighborGradOp); + +REGISTER_XLA_OP(Name("ResizeBilinear") + .CompileTimeConstantInput("size") + .Device(DEVICE_TPU_XLA_JIT), + TpuResizeBilinearOp); + +REGISTER_XLA_OP(Name("ResizeBilinearGrad").Device(DEVICE_TPU_XLA_JIT), + TpuResizeBilinearGradOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/infeed_ops.cc b/tensorflow/core/tpu/kernels/infeed_ops.cc new file mode 100644 index 00000000000..f3fbd16b6cc --- /dev/null +++ b/tensorflow/core/tpu/kernels/infeed_ops.cc @@ -0,0 +1,529 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/tpu/kernels/infeed_ops.h" + +#include +#include + +#include "tensorflow/compiler/jit/xla_device.h" +#include "tensorflow/compiler/tf2xla/literal_util.h" +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/common_runtime/dma_helper.h" +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/dataset.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function_handle_cache.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/variant.h" +#include "tensorflow/core/framework/variant_encode_decode.h" +#include "tensorflow/core/framework/variant_tensor_data.h" +#include "tensorflow/core/kernels/transpose_functor.h" +#include "tensorflow/core/profiler/lib/traceme.h" +#include "tensorflow/core/tpu/kernels/transfer_ops.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/stream_executor/multi_platform_manager.h" +#include "tensorflow/stream_executor/tpu/tpu_transfer_manager.h" +#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" + +namespace tensorflow { +namespace { + +typedef Eigen::ThreadPoolDevice CPUDevice; +typedef tensorflow::tpu::NoncopyableBuffer LinearizerBuffer; +typedef std::deque LinearizerBufferList; + +// Transposes the given tensor using the tensorflow C++ transpose implementation +// to obtain a XLA literal for the host tensor laid out as the given layout. The +// returned tensor is normalized to the dim0major layout -- F32[10,20,30]{2,0,1} +// is returned as F32[20,10,30]{2,1,0}. +xla::StatusOr TransposeTensor(OpKernelContext* ctx, + const Tensor& input_tensor, + const xla::Shape& xla_shape) { + profiler::TraceMe trace_me("TransposeTensor", /*level=*/2); + const int64 rank = xla_shape.rank(); + std::vector permutation(rank); + std::vector transposed_shapes(rank); + for (int64 i = 0; i < rank; ++i) { + permutation[i] = xla_shape.layout().minor_to_major(rank - 1 - i); + transposed_shapes[i] = xla_shape.dimensions(permutation[i]); + } + + Tensor transposed_tensor; + + // If this is a trivial transpose (i.e., bitcast), just create an aliased + // tensor with the transposed shape. + if (xla::LayoutUtil::IsMonotonicWithDim0Major( + xla::ShapeUtil::DropDegenerateDimensions(xla_shape).layout())) { + TensorShape shape; + TF_RETURN_IF_ERROR(TensorShapeUtils::MakeShape(transposed_shapes, &shape)); + TF_RETURN_IF_ERROR(transposed_tensor.BitcastFrom( + input_tensor, input_tensor.dtype(), shape)); + return transposed_tensor; + } + + AllocatorAttributes alloc_attr; + alloc_attr.set_on_host(true); + TF_RETURN_IF_ERROR(ctx->allocate_temp(input_tensor.dtype(), + TensorShape(transposed_shapes), + &transposed_tensor, alloc_attr)); + // Eigen Transpose fails with SIGFPE if there is a dimension of size 0. + if (input_tensor.NumElements() > 0) { + TF_RETURN_IF_ERROR(DoTranspose(ctx->eigen_device(), + input_tensor, permutation, + &transposed_tensor)); + } + return transposed_tensor; +} + +xla::StatusOr GetLayoutOverride(OpKernelConstruction* ctx, + const char* attrn_name, + std::vector* minor_to_major) { + if (!ctx->HasAttr(attrn_name)) { + return false; + } + TF_RETURN_IF_ERROR(ctx->GetAttr(attrn_name, minor_to_major)); + return !minor_to_major->empty(); +} + +Status GetInfeedShapeWithLayout(OpKernelConstruction* ctx, + const char* attrn_name, + const xla::Shape& input_shape, + xla::Shape* output_shape) { + std::vector minor_to_major; + TF_ASSIGN_OR_RETURN(bool has_override, + GetLayoutOverride(ctx, attrn_name, &minor_to_major)); + if (!has_override) { + *output_shape = input_shape; + if (output_shape->IsTuple()) { + int64 tuple_elements = xla::ShapeUtil::TupleElementCount(*output_shape); + for (int64 i = 0; i < tuple_elements; ++i) { + xla::Shape* sub_shape = + xla::ShapeUtil::GetMutableSubshape(output_shape, {i}); + *sub_shape->mutable_layout() = GetTPUInfeedLayout(*sub_shape).layout(); + } + } else { + *output_shape->mutable_layout() = + GetTPUInfeedLayout(*output_shape).layout(); + } + return Status::OK(); + } + + auto layout_func = [](const xla::Shape& shape) -> xla::Layout { + return GetTPUInfeedLayout(shape).layout(); + }; + return GetShapeWithLayout(input_shape, minor_to_major, layout_func, + output_shape); +} + +// LinearizedBuffersWrapper is an opaque C++ data structure for the outputs of +// PrelinearizeOp and PrelinearizeTupleOp. It holds the resultant linearized +// buffers and references to input tensors whose underlying storage are shared +// with linearized buffers. +// NOTE: This is not a feature-complete implementation of the DT_VARIANT +// specification. In particular, we cannot currently serialize an arbitrary +// `LinearizerBufferList` (aka `std::deque`) +// object, so the `Encode()` and `Decode()` methods are not implemented. +struct LinearizedBuffersWrapper { + explicit LinearizedBuffersWrapper() {} + explicit LinearizedBuffersWrapper(LinearizerBufferList bufs, + std::vector ts) + : buffers(std::move(bufs)), tensors(std::move(ts)) {} + LinearizedBuffersWrapper(const LinearizedBuffersWrapper& wrapper) { + // tensorflow::Variant requires this copy constructor to compile. + LOG(FATAL) << "LinearizedBuffersWrapper should not copy."; + } + LinearizedBuffersWrapper& operator=(const LinearizedBuffersWrapper& wrapper) = + delete; + LinearizedBuffersWrapper(LinearizedBuffersWrapper&&) = default; + LinearizedBuffersWrapper& operator=(LinearizedBuffersWrapper&&) = default; + ~LinearizedBuffersWrapper() = default; + + // These functions are tensorflow::Variant requirements. + string TypeName() const { return "(anonymous)::LinearizedBuffersWrapper"; } + void Encode(tensorflow::VariantTensorData* data) const { + LOG(ERROR) << "Encode() is not implemented for LinearizedBuffersWrapper " + "objects."; + } + bool Decode(const tensorflow::VariantTensorData& data) { + LOG(ERROR) << "Decode() is not implemented for LinearizedBuffersWrapper " + "objects."; + return false; + } + + LinearizerBufferList buffers; + // Save references on tensors whose underlying storage are shared with + // LiteralLinearizer::Buffer in `buffers`. + std::vector tensors; +}; + +Status AutoTransposeAndLinearize(OpKernelContext* ctx, + const Tensor& input_tensor, + const xla::Shape& shape, + LinearizerBufferList* linearized_buffers, + std::vector* saved_input_tensors) { + const Tensor* tensor = &input_tensor; + // If the given layout is not in dim0major layout, tranposes the tensor. + bool has_transposed = false; + Tensor transposed_tensor; + if (!xla::LayoutUtil::IsMonotonicWithDim0Major(shape.layout())) { + // If the given layout is not in dim0major layout, transpose the tensor. + TF_ASSIGN_OR_RETURN(transposed_tensor, + TransposeTensor(ctx, input_tensor, shape)); + tensor = &transposed_tensor; + has_transposed = true; + } + + xla::BorrowingLiteral literal; + TF_RETURN_IF_ERROR(HostTensorToBorrowingLiteral(*tensor, &literal)); + + TF_RETURN_IF_ERROR( + xla::TpuTransferManagerInterface::GetRegisteredTpuTransferManager() + ->LinearizeToBuffers(literal, linearized_buffers)); + + // The input tensor is ref-counted. Save a handle on the input tensor if + // its underlying storage is shared with linearized buffers to prevent + // input tensor from getting freed. + for (const auto& buffer : *linearized_buffers) { + if (!buffer.owns_data() && !has_transposed) { + // `buffer` is created from zero-copy fast path from the un-transposed + // input tensor so its underlying data is shared with input tensor. + // Save a handle to input tensor to increment its ref-count and avoid + // it getting deallocated after PrelinearizeTupleOp completes. + saved_input_tensors->push_back(*tensor); + // A literal can be linearized to zero to two buffers. If any of the + // linearized buffer shares storage with input tensor. We save exactly + // one handle on the input tensor. + break; + } + } + return Status::OK(); +} + +// PrelinearizeOp is used to linearize one tensor to the device format. +class PrelinearizeOp : public OpKernel { + public: + explicit PrelinearizeOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); + xla::Shape shape; + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &shape)); + OP_REQUIRES_OK(ctx, + GetInfeedShapeWithLayout(ctx, "layout", shape, &xla_shape_)); + } + + void Compute(OpKernelContext* ctx) override { + const Tensor& input_tensor = ctx->input(0); + // Validate input. + OP_REQUIRES( + ctx, input_tensor.dtype() == dtype_, + errors::InvalidArgument("Prelinearize dtype mismatch; expected ", + DataType_Name(dtype_), ", got ", + DataType_Name(input_tensor.dtype()))); + OP_REQUIRES( + ctx, input_tensor.shape() == shape_, + errors::InvalidArgument("Prelinearize shape mismatch; expected ", + shape_.DebugString(), ", got ", + input_tensor.shape().DebugString())); + + // Auto-transpose and prelinearize. + LinearizerBufferList linearized_buffers; + std::vector saved_input_tensors; + auto status = + AutoTransposeAndLinearize(ctx, input_tensor, xla_shape_, + &linearized_buffers, &saved_input_tensors); + OP_REQUIRES_OK(ctx, status); + + // Write to output. + tensorflow::Tensor* output; + OP_REQUIRES_OK(ctx, + ctx->allocate_output(0, tensorflow::TensorShape{}, &output)); + output->scalar()() = LinearizedBuffersWrapper{ + std::move(linearized_buffers), std::move(saved_input_tensors)}; + } + + bool IsExpensive() override { return true; } + + private: + TensorShape shape_; + DataType dtype_; + xla::Shape xla_shape_; + + // PrelinearizeOp is neither copyable nor movable. + PrelinearizeOp(const PrelinearizeOp&) = delete; + PrelinearizeOp& operator=(const PrelinearizeOp&) = delete; +}; + +// PrelinearizeTupleOp is used to linearize multiple tensors to the device +// format. +class PrelinearizeTupleOp : public OpKernel { + public: + explicit PrelinearizeTupleOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); + OP_REQUIRES( + ctx, shapes_.size() == dtypes_.size(), + errors::InvalidArgument( + "shapes and dtypes must be the same length. shapes length = ", + shapes_.size(), ", dtypes length = ", dtypes_.size())); + + std::vector xla_shapes; + for (int i = 0; i < shapes_.size(); i++) { + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, + TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); + xla_shapes.push_back(xla_shape); + } + OP_REQUIRES_OK( + ctx, GetInfeedShapeWithLayout( + ctx, "layouts", xla::ShapeUtil::MakeTupleShape(xla_shapes), + &tuple_shape_)); + } + + void Compute(OpKernelContext* ctx) override { + OpInputList values; + OP_REQUIRES_OK(ctx, ctx->input_list("inputs", &values)); + OP_REQUIRES(ctx, values.size() == shapes_.size(), + errors::InvalidArgument( + "Wrong number of inputs to PrelinearizeTuple.")); + + LinearizerBufferList all_linearized_buffers; + std::vector all_saved_input_tensors; + for (int i = 0; i < values.size(); i++) { + // Validate input. + const Tensor& input_tensor = values[i]; + OP_REQUIRES(ctx, input_tensor.dtype() == dtypes_[i], + errors::InvalidArgument( + "PrelinearizeTuple dtype mismatch at tuple element ", i, + "; expected ", DataType_Name(dtypes_[i]), ", got ", + DataType_Name(input_tensor.dtype()))); + OP_REQUIRES(ctx, input_tensor.shape() == shapes_[i], + errors::InvalidArgument( + "PrelinearizeTuple shape mismatch at tuple element ", i, + "; expected ", shapes_[i].DebugString(), ", got ", + input_tensor.shape().DebugString())); + + // Auto-transpose and prelinearize. + LinearizerBufferList linearized_buffers; + std::vector saved_input_tensors; + auto status = AutoTransposeAndLinearize( + ctx, input_tensor, tuple_shape_.tuple_shapes(i), &linearized_buffers, + &saved_input_tensors); + OP_REQUIRES_OK(ctx, status); + all_linearized_buffers.insert( + all_linearized_buffers.end(), + std::make_move_iterator(linearized_buffers.begin()), + std::make_move_iterator(linearized_buffers.end())); + all_saved_input_tensors.insert( + all_saved_input_tensors.end(), + std::make_move_iterator(saved_input_tensors.begin()), + std::make_move_iterator(saved_input_tensors.end())); + } + + tensorflow::Tensor* output; + OP_REQUIRES_OK(ctx, + ctx->allocate_output(0, tensorflow::TensorShape{}, &output)); + output->scalar()() = LinearizedBuffersWrapper{ + std::move(all_linearized_buffers), std::move(all_saved_input_tensors)}; + } + + bool IsExpensive() override { return true; } + + private: + std::vector shapes_; + DataTypeVector dtypes_; + xla::Shape tuple_shape_; + + // PrelinearizeTupleOp is neither copyable nor movable. + PrelinearizeTupleOp(const PrelinearizeTupleOp&) = delete; + PrelinearizeTupleOp& operator=(const PrelinearizeTupleOp&) = delete; +}; + +// The InfeedEnqueuePrelinearizedBufferOp op is used to transfer prelinearized +// buffers to the device infeed queue. +class InfeedEnqueuePrelinearizedBufferOp : public TpuTransferAsyncOpKernel { + public: + explicit InfeedEnqueuePrelinearizedBufferOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "prelinearized_buffers_to_infeed", 8) {} + + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override { + const Tensor& input_tensor = ctx->input(0); + const LinearizedBuffersWrapper* wrapper = + input_tensor.scalar()() + .get(); + TF_RETURN_IF_ERROR(transfer_manager->TransferBuffersToInfeed( + stream_executor, wrapper->buffers)); + + return Status::OK(); + } + + private: + // InfeedEnqueuePrelinearizedBufferOp is neither copyable nor movable. + InfeedEnqueuePrelinearizedBufferOp( + const InfeedEnqueuePrelinearizedBufferOp&) = delete; + InfeedEnqueuePrelinearizedBufferOp& operator=( + const InfeedEnqueuePrelinearizedBufferOp&) = delete; +}; + +} // anonymous namespace + +TpuInfeedEnqueueOp::TpuInfeedEnqueueOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "infeed_enqueue", 8) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); + xla::Shape shape; + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &shape)); + OP_REQUIRES_OK(ctx, + GetInfeedShapeWithLayout(ctx, "layout", shape, &xla_shape_)); +} + +Status TpuInfeedEnqueueOp::DoWork( + OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) { + const Tensor& input_tensor = ctx->input(0); + + // Validate runtime shape and fail if it doesn't match the contract. + if (input_tensor.dtype() != dtype_) { + return errors::InvalidArgument("Infeed dtype mismatch."); + } + if (input_tensor.shape() != shape_) { + return errors::InvalidArgument("Infeed shape mismatch; expected ", + shape_.DebugString(), ", got ", + input_tensor.shape().DebugString()); + } + + const Tensor* tensor = &input_tensor; + Tensor transposed_tensor; + if (!xla::LayoutUtil::IsMonotonicWithDim0Major(xla_shape_.layout())) { + // If the given layout is not in dim0major layout, transpose the tensor. + TF_ASSIGN_OR_RETURN(transposed_tensor, + TransposeTensor(ctx, input_tensor, xla_shape_)); + tensor = &transposed_tensor; + } + + xla::BorrowingLiteral literal; + TF_RETURN_IF_ERROR(HostTensorToBorrowingLiteral(*tensor, &literal)); + + // Transfer the given literal to the Infeed interface of the device. + TF_RETURN_IF_ERROR( + transfer_manager->TransferLiteralToInfeed(stream_executor, literal)); + return Status::OK(); +} + +TpuInfeedEnqueueTupleOp::TpuInfeedEnqueueTupleOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "infeed_enqueue", 8) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); + OP_REQUIRES( + ctx, shapes_.size() == dtypes_.size(), + errors::InvalidArgument("shapes and dtypes must be the same length.")); + + std::vector xla_shapes; + for (int i = 0; i < shapes_.size(); i++) { + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, + TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); + xla_shapes.push_back(xla_shape); + } + OP_REQUIRES_OK( + ctx, GetInfeedShapeWithLayout(ctx, "layouts", + xla::ShapeUtil::MakeTupleShape(xla_shapes), + &tuple_shape_)); +} + +Status TpuInfeedEnqueueTupleOp::DoWork( + OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) { + OpInputList values; + TF_RETURN_IF_ERROR(ctx->input_list("inputs", &values)); + if (values.size() != shapes_.size()) { + return errors::InvalidArgument( + "Wrong number of inputs to InfeedEnqueueTuple."); + } + + for (const auto& shapes : shapes_) { + VLOG(1) << "TransferLiteralToInfeed " << shapes.DebugString(); + } + + std::vector maybe_transposed_tensors; + maybe_transposed_tensors.reserve(values.size()); + for (int i = 0; i < values.size(); i++) { + // Validate runtime shapes and fail if it doesn't match the contract. + const Tensor* tensor = &values[i]; + if (tensor->shape() != shapes_[i]) { + return errors::InvalidArgument("Infeed shape mismatch for tuple element ", + i, "; expected ", shapes_[i].DebugString(), + ", got ", tensor->shape().DebugString()); + } + if (!xla::LayoutUtil::IsMonotonicWithDim0Major( + tuple_shape_.tuple_shapes(i).layout())) { + // If the given layout is not in dim0major layout, tranposes the given + // tensor. + TF_ASSIGN_OR_RETURN( + Tensor transposed_tensor, + TransposeTensor(ctx, *tensor, tuple_shape_.tuple_shapes(i))); + maybe_transposed_tensors.emplace_back(transposed_tensor); + } else { + maybe_transposed_tensors.emplace_back(*tensor); + } + } + + xla::BorrowingLiteral tuple; + TF_RETURN_IF_ERROR( + HostTensorsToBorrowingLiteralTuple(maybe_transposed_tensors, &tuple)); + + // Transfer the given literal to the Infeed interface of the device. + TF_RETURN_IF_ERROR( + transfer_manager->TransferLiteralToInfeed(stream_executor, tuple)); + + VLOG(1) << "TransferLiteralToInfeed complete."; + + return Status::OK(); +} + +// These ops execute on either the TPU device or the CPU device. When running on +// CPU they must specify a non-negative value for device_ordinal to indicate +// which TPU to send infeed to. +REGISTER_KERNEL_BUILDER( + Name("InfeedEnqueue").Device(DEVICE_TPU_NODE).HostMemory("input"), + TpuInfeedEnqueueOp); +REGISTER_KERNEL_BUILDER(Name("InfeedEnqueue").Device(DEVICE_CPU), + TpuInfeedEnqueueOp); + +REGISTER_KERNEL_BUILDER( + Name("InfeedEnqueueTuple").Device(DEVICE_TPU_NODE).HostMemory("inputs"), + TpuInfeedEnqueueTupleOp); +REGISTER_KERNEL_BUILDER(Name("InfeedEnqueueTuple").Device(DEVICE_CPU), + TpuInfeedEnqueueTupleOp); + +// Prelinearize ops run on CPU as part of tf.data input pipeline. +REGISTER_KERNEL_BUILDER(Name("Prelinearize").Device(DEVICE_CPU), + PrelinearizeOp); +REGISTER_KERNEL_BUILDER(Name("PrelinearizeTuple").Device(DEVICE_CPU), + PrelinearizeTupleOp); + +// InfeedEnqueuePrelinearizedBuffer op run on CPU and takes a device_ordinal to +// select the right device to infeed. +REGISTER_KERNEL_BUILDER( + Name("InfeedEnqueuePrelinearizedBuffer").Device(DEVICE_CPU), + InfeedEnqueuePrelinearizedBufferOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/infeed_ops.h b/tensorflow/core/tpu/kernels/infeed_ops.h new file mode 100644 index 00000000000..622583b6a73 --- /dev/null +++ b/tensorflow/core/tpu/kernels/infeed_ops.h @@ -0,0 +1,69 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_TPU_KERNELS_INFEED_OPS_H_ +#define TENSORFLOW_CORE_TPU_KERNELS_INFEED_OPS_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/tpu/kernels/transfer_ops.h" + +namespace tensorflow { + +// TODO(b/65200690): Rework this when there is a callback based infeed API to +// StreamExecutor. + +// The InfeedEnqueue op is used to deliver data to the device infeed queue. +class TpuInfeedEnqueueOp : public TpuTransferAsyncOpKernel { + public: + explicit TpuInfeedEnqueueOp(OpKernelConstruction* ctx); + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override; + + private: + TensorShape shape_; + DataType dtype_; + xla::Shape xla_shape_; + + // TpuInfeedEnqueueOp is neither copyable nor movable. + TpuInfeedEnqueueOp(const TpuInfeedEnqueueOp&) = delete; + TpuInfeedEnqueueOp& operator=(const TpuInfeedEnqueueOp&) = delete; +}; + +// The InfeedEnqueueTuple op is used on the host to deliver multiple tensors to +// the device infeed queue as an XLA tuple. +class TpuInfeedEnqueueTupleOp : public TpuTransferAsyncOpKernel { + public: + explicit TpuInfeedEnqueueTupleOp(OpKernelConstruction* ctx); + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override; + + private: + std::vector shapes_; + DataTypeVector dtypes_; + xla::Shape tuple_shape_; + + // TpuInfeedEnqueueTupleOp is neither copyable nor movable. + TpuInfeedEnqueueTupleOp(const TpuInfeedEnqueueTupleOp&) = delete; + TpuInfeedEnqueueTupleOp& operator=(const TpuInfeedEnqueueTupleOp&) = delete; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_TPU_KERNELS_INFEED_OPS_H_ diff --git a/tensorflow/core/tpu/kernels/outfeed_ops.cc b/tensorflow/core/tpu/kernels/outfeed_ops.cc new file mode 100644 index 00000000000..51a3a71a297 --- /dev/null +++ b/tensorflow/core/tpu/kernels/outfeed_ops.cc @@ -0,0 +1,116 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/tpu/kernels/outfeed_ops.h" + +#include "tensorflow/compiler/jit/xla_device.h" +#include "tensorflow/compiler/tf2xla/literal_util.h" +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/tpu/kernels/transfer_ops.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/stream_executor/multi_platform_manager.h" + +namespace tensorflow { + +TpuOutfeedDequeueOp::TpuOutfeedDequeueOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "outfeed_dequeue", 1) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &xla_shape_)); +} + +Status TpuOutfeedDequeueOp::DoWork( + OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) { + Tensor* output; + TF_RETURN_IF_ERROR(ctx->allocate_output(0, shape_, &output)); + + // Transfer from the outfeed interface of the device. + xla::MutableBorrowingLiteral literal; + TF_RETURN_IF_ERROR( + HostTensorToMutableBorrowingLiteral(xla_shape_, output, &literal)); + + VLOG(1) << "TransferLiteralFromOutfeed " + << xla::ShapeUtil::HumanStringWithLayout(xla_shape_); + + TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralFromOutfeed( + stream_executor, xla_shape_, literal)); + + VLOG(1) << "TransferLiteralFromOutfeed complete."; + + return Status::OK(); +} + +// The OutfeedDequeueTuple op is used to retrieve multiple tensors from the +// device outfeed queue. +TpuOutfeedDequeueTupleOp::TpuOutfeedDequeueTupleOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "outfeed_dequeue", 1) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); + OP_REQUIRES( + ctx, shapes_.size() == dtypes_.size(), + errors::InvalidArgument("shapes and dtypes must be the same length.")); + // The `dtypes` list is inferred from the supplied inputs, so it + // is always the correct length. + for (int i = 0; i < shapes_.size(); i++) { + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, + TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); + xla_shapes_.push_back(xla_shape); + } + tuple_shape_ = xla::ShapeUtil::MakeTupleShape(xla_shapes_); +} + +Status TpuOutfeedDequeueTupleOp::DoWork( + OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) { + VLOG(1) << "TransferLiteralFromOutfeed " + << xla::ShapeUtil::HumanStringWithLayout(tuple_shape_); + + for (int i = 0; i < shapes_.size(); ++i) { + Tensor* output; + TF_RETURN_IF_ERROR(ctx->allocate_output(i, shapes_[i], &output)); + + xla::MutableBorrowingLiteral literal; + TF_RETURN_IF_ERROR( + HostTensorToMutableBorrowingLiteral(xla_shapes_[i], output, &literal)); + TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralFromOutfeed( + stream_executor, xla_shapes_[i], literal)); + } + return Status::OK(); +} + +// These ops execute on either the TPU device or the CPU device. When +// running on CPU they must specify a non-negative value for +// device_ordinal to indicate which TPU to receive outfeed from. +REGISTER_KERNEL_BUILDER( + Name("OutfeedDequeue").Device(DEVICE_TPU_NODE).HostMemory("output"), + TpuOutfeedDequeueOp); +REGISTER_KERNEL_BUILDER(Name("OutfeedDequeue").Device(DEVICE_CPU), + TpuOutfeedDequeueOp); + +REGISTER_KERNEL_BUILDER( + Name("OutfeedDequeueTuple").Device(DEVICE_TPU_NODE).HostMemory("outputs"), + TpuOutfeedDequeueTupleOp); +REGISTER_KERNEL_BUILDER(Name("OutfeedDequeueTuple").Device(DEVICE_CPU), + TpuOutfeedDequeueTupleOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/outfeed_ops.h b/tensorflow/core/tpu/kernels/outfeed_ops.h new file mode 100644 index 00000000000..5e3ed87c04b --- /dev/null +++ b/tensorflow/core/tpu/kernels/outfeed_ops.h @@ -0,0 +1,69 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_TPU_KERNELS_OUTFEED_OPS_H_ +#define TENSORFLOW_CORE_TPU_KERNELS_OUTFEED_OPS_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/tpu/kernels/transfer_ops.h" + +namespace tensorflow { + +// The OutfeedDequeue op is used to retrieve a single tensor from the device +// outfeed queue. +class TpuOutfeedDequeueOp : public TpuTransferAsyncOpKernel { + public: + explicit TpuOutfeedDequeueOp(OpKernelConstruction* ctx); + + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override; + + private: + TensorShape shape_; + DataType dtype_; + xla::Shape xla_shape_; + + // OutfeedDequeueOp is neither copyable nor movable. + TpuOutfeedDequeueOp(const TpuOutfeedDequeueOp&) = delete; + TpuOutfeedDequeueOp& operator=(const TpuOutfeedDequeueOp&) = delete; +}; + +// The OutfeedDequeueTuple op is used to retrieve multiple tensors from the +// device outfeed queue. +class TpuOutfeedDequeueTupleOp : public TpuTransferAsyncOpKernel { + public: + explicit TpuOutfeedDequeueTupleOp(OpKernelConstruction* ctx); + + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override; + + private: + std::vector shapes_; + DataTypeVector dtypes_; + std::vector xla_shapes_; + xla::Shape tuple_shape_; + + // OutfeedDequeueTupleOp is neither copyable nor movable. + TpuOutfeedDequeueTupleOp(const TpuOutfeedDequeueTupleOp&) = delete; + TpuOutfeedDequeueTupleOp& operator=(const TpuOutfeedDequeueTupleOp&) = delete; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_TPU_KERNELS_OUTFEED_OPS_H_ diff --git a/tensorflow/core/tpu/kernels/replication_ops.cc b/tensorflow/core/tpu/kernels/replication_ops.cc new file mode 100644 index 00000000000..4c986e880e7 --- /dev/null +++ b/tensorflow/core/tpu/kernels/replication_ops.cc @@ -0,0 +1,27 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/jit/xla_device_ops.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/tpu/tpu_defs.h" + +namespace tensorflow { + +REGISTER_KERNEL_BUILDER(Name("_TPUReplicate").Device(DEVICE_TPU_SYSTEM), + XlaDeviceDummyOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc b/tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc new file mode 100644 index 00000000000..ec2ae91d3eb --- /dev/null +++ b/tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc @@ -0,0 +1,62 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h" +#include "tensorflow/core/tpu/kernels/tpu_op_consts.h" +#include "tensorflow/core/tpu/tpu_configuration.h" + +namespace tensorflow { + +class TpuHandleToProtoKeyOp : public OpKernel { + public: + explicit TpuHandleToProtoKeyOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + ~TpuHandleToProtoKeyOp() override = default; + TpuHandleToProtoKeyOp(const TpuHandleToProtoKeyOp&) = delete; + TpuHandleToProtoKeyOp& operator=(const TpuHandleToProtoKeyOp&) = delete; + + void Compute(OpKernelContext* ctx) override { + VLOG(1) << "TpuHandleToProtoKeyOp::Compute " << ctx->op_kernel().name() + << " on device " << ctx->op_kernel().requested_device(); + const Tensor& uid = ctx->input(0); + + ResourceMgr* rm = GetTPUConfigResourceMgr(); + tpu::TpuCompilationCacheInterface* cache; + OP_REQUIRES_OK(ctx, rm->Lookup( + rm->default_container(), + tpu::kCompilationCacheResourceName, &cache)); + core::ScopedUnref cache_unref(cache); + + std::vector keys; + OP_REQUIRES_OK(ctx, cache->GetKeysFromUid(uid.scalar()(), &keys)); + + TensorShape output_shape; + output_shape.AddDim(keys.size()); + Tensor* result = nullptr; + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, output_shape, &result)); + for (int i = 0; i < keys.size(); ++i) { + result->vec()(i) = keys[i]; + } + }; +}; + +REGISTER_KERNEL_BUILDER(Name("TpuHandleToProtoKey").Device(DEVICE_CPU), + TpuHandleToProtoKeyOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/transfer_ops.cc b/tensorflow/core/tpu/kernels/transfer_ops.cc new file mode 100644 index 00000000000..40b85e2cfbd --- /dev/null +++ b/tensorflow/core/tpu/kernels/transfer_ops.cc @@ -0,0 +1,98 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/tpu/kernels/transfer_ops.h" + +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/platform/tracing.h" +#include "tensorflow/core/profiler/lib/traceme.h" +#include "tensorflow/stream_executor/multi_platform_manager.h" +#include "tensorflow/stream_executor/tpu/tpu_node_context.h" +#include "tensorflow/stream_executor/tpu/tpu_platform_interface.h" +#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" + +namespace tensorflow { + +TpuTransferAsyncOpKernel::TpuTransferAsyncOpKernel(OpKernelConstruction* ctx, + const string& transfer_type, + int number_of_threads) + : AsyncOpKernel(ctx), + thread_pool_(new thread::ThreadPool( + ctx->env(), + strings::StrCat(transfer_type, "_thread_", + SanitizeThreadSuffix(def().name())), + /*num_threads=*/8)) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("device_ordinal", &device_ordinal_)); + if (ctx->device_type() == DeviceType(DEVICE_CPU)) { + OP_REQUIRES( + ctx, device_ordinal_ >= 0, + errors::InvalidArgument(transfer_type, + " ops must specify a device_ordinal when " + "placed on CPU.")); + } +} + +void TpuTransferAsyncOpKernel::ComputeAsync(OpKernelContext* ctx, + DoneCallback done) { + CancellationToken token = + ctx->cancellation_manager()->get_cancellation_token(); + bool already_cancelled; + { + // Only protect registering the cancellation callback as mu_ cannot be held + // at a point where `done` could be called. + mutex_lock lock(mu_); + already_cancelled = !ctx->cancellation_manager()->RegisterCallback( + token, [this]() { Cancel(); }); + } + OP_REQUIRES_ASYNC(ctx, !already_cancelled, + errors::Cancelled("Infeed was cancelled."), done); + thread_pool_->Schedule([this, ctx, done, token]() { + Status s = RunTransfer(ctx); + ctx->cancellation_manager()->DeregisterCallback(token); + OP_REQUIRES_OK_ASYNC(ctx, s, done); + done(); + }); +} + +Status TpuTransferAsyncOpKernel::RunTransfer(OpKernelContext* ctx) { + auto* tpu_platform = tpu::TpuPlatformInterface::GetRegisteredPlatform(); + + int real_device_ordinal = device_ordinal_; + if (real_device_ordinal < 0) { + const XlaDevice::Metadata* metadata; + TF_RETURN_IF_ERROR(XlaDevice::GetMetadata(ctx, &metadata)); + real_device_ordinal = metadata->device_ordinal(); + } + stream_executor::StreamExecutor* stream_executor = + tpu_platform->ExecutorForDevice(real_device_ordinal).ValueOrDie(); + + // When Xprof profiling is off (which is the default), constructing the + // activity is simple enough that its overhead is negligible. + profiler::TraceMe activity( + [this] { return profiler::TraceMeOp(name(), type_string()); }, + profiler::TraceMeLevel::kInfo); + return DoWork( + ctx, xla::TpuTransferManagerInterface::GetRegisteredTpuTransferManager(), + stream_executor); +} + +void TpuTransferAsyncOpKernel::Cancel() { + mutex_lock lock(mu_); + TF_CHECK_OK(tpu::TpuNodeContext::CloseTpuHost()); +} + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/transfer_ops.h b/tensorflow/core/tpu/kernels/transfer_ops.h new file mode 100644 index 00000000000..d98d743f569 --- /dev/null +++ b/tensorflow/core/tpu/kernels/transfer_ops.h @@ -0,0 +1,56 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_TPU_KERNELS_TRANSFER_OPS_H_ +#define TENSORFLOW_CORE_TPU_KERNELS_TRANSFER_OPS_H_ + +#include "tensorflow/compiler/jit/xla_device.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/util/stream_executor_util.h" +#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" + +namespace tensorflow { + +// Base class providing common functionality for async ops that transfer from +// host to TPU. +class TpuTransferAsyncOpKernel : public AsyncOpKernel { + public: + explicit TpuTransferAsyncOpKernel(OpKernelConstruction* ctx, + const string& transfer_type, + int number_of_threads); + + void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override; + + protected: + virtual Status DoWork(OpKernelContext* context, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) = 0; + + private: + Status RunTransfer(OpKernelContext* ctx); + void Cancel(); + + std::unique_ptr thread_pool_; + int device_ordinal_; + mutex mu_; + + // TpuTransferAsyncOpKernel is neither copyable nor movable. + TpuTransferAsyncOpKernel(const TpuTransferAsyncOpKernel&) = delete; + TpuTransferAsyncOpKernel& operator=(const TpuTransferAsyncOpKernel&) = delete; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_TPU_KERNELS_TRANSFER_OPS_H_ diff --git a/tensorflow/core/tpu/tpu_defs.cc b/tensorflow/core/tpu/tpu_defs.cc index 69669bfdb7b..69d4989773a 100644 --- a/tensorflow/core/tpu/tpu_defs.cc +++ b/tensorflow/core/tpu/tpu_defs.cc @@ -15,6 +15,10 @@ limitations under the License. #include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/core/tpu/tpu_api.h" +#include "tensorflow/stream_executor/tpu/c_api_conversions.h" +#include "tensorflow/stream_executor/tpu/c_api_decl.h" + namespace tensorflow { const char* const DEVICE_TPU_NODE = "TPU"; @@ -27,4 +31,18 @@ const char* const TPUREPLICATE_MIRRORED_VAR_INDICES_ATTR = const char* const kTPUReplicateAttr = "_tpu_replicate"; const char* const kOutsideCompilationAttr = "_xla_outside_compilation"; +xla::Shape GetTPUInfeedLayout(const xla::Shape& shape) { + XLA_Shape c_shape; + XLA_Shape c_infeed_shape; + + ApiConverter::ToC(shape, &c_shape); + + tpu::ExecutorApiFn()->TpuTransferManager_GetInfeedLayoutFn(&c_shape, + &c_infeed_shape); + xla::Shape infeed_shape = ApiConverter::FromC(&c_infeed_shape); + ApiConverter::Free(&c_shape); + ApiConverter::Free(&c_infeed_shape); + return infeed_shape; +} + } // namespace tensorflow diff --git a/tensorflow/core/tpu/tpu_defs.h b/tensorflow/core/tpu/tpu_defs.h index 008e386dde6..29954b2289f 100644 --- a/tensorflow/core/tpu/tpu_defs.h +++ b/tensorflow/core/tpu/tpu_defs.h @@ -20,6 +20,7 @@ limitations under the License. #include +#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/core/framework/types.pb.h" namespace tensorflow { @@ -56,6 +57,11 @@ static constexpr std::array kTpuAllTypes = { DT_COMPLEX64, DT_INT64, DT_UINT64, DT_QINT8, DT_QUINT8, DT_INT8, DT_UINT8, DT_INT16, DT_UINT16}}; +// For the given shape, chooses a layout for infeed on TPU. The returned shape +// has the same dimensions as the original shape, and only the layout is +// changed. +xla::Shape GetTPUInfeedLayout(const xla::Shape& shape); + } // namespace tensorflow #endif // TENSORFLOW_CORE_TPU_TPU_DEFS_H_ diff --git a/tensorflow/core/tpu/tpu_library_init_fns.inc b/tensorflow/core/tpu/tpu_library_init_fns.inc index be9d594685e..40130bd46dd 100644 --- a/tensorflow/core/tpu/tpu_library_init_fns.inc +++ b/tensorflow/core/tpu/tpu_library_init_fns.inc @@ -161,6 +161,7 @@ tensorflow::Status SetExecutorStructFn(void* library_handle) { TFTPU_SET_FN(executor_fn, TpuTransferManager_TransferLiteralFromDevice); TFTPU_SET_FN(executor_fn, TpuTransferManager_GetByteSizeRequirement); TFTPU_SET_FN(executor_fn, TpuTransferManager_WriteSingleTupleIndexTable); + TFTPU_SET_FN(executor_fn, TpuTransferManager_GetInfeedLayout); TFTPU_SET_FN(executor_fn, TpuTransferManager_LinearizeToBuffers); TFTPU_SET_FN(executor_fn, TpuTransferManager_FreeBuffers); diff --git a/tensorflow/stream_executor/tpu/BUILD b/tensorflow/stream_executor/tpu/BUILD index a52f9919e6e..a8178404dff 100644 --- a/tensorflow/stream_executor/tpu/BUILD +++ b/tensorflow/stream_executor/tpu/BUILD @@ -203,10 +203,12 @@ cc_library( cc_library( name = "tpu_transfer_manager_interface", + srcs = ["tpu_transfer_manager_interface.cc"], hdrs = ["tpu_transfer_manager_interface.h"], visibility = ["//visibility:public"], deps = [ ":noncopyable_buffer", + ":tpu_platform_interface", "//tensorflow/compiler/xla/service:transfer_manager", ], ) diff --git a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h index 2b66c2ce4c5..013e7fe4e0c 100644 --- a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h +++ b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h @@ -182,6 +182,8 @@ void TpuTransferManager_WriteSingleTupleIndexTable( XLA_TransferManager* manager, SE_Stream* stream, SE_DeviceMemoryBase* elements, size_t elements_len, XLA_Shape* shape, SE_DeviceMemoryBase* region, SE_Status* status); +void TpuTransferManager_GetInfeedLayout(XLA_Shape* shape, + XLA_Shape* infeed_shape); void TpuTransferManager_LinearizeToBuffers( XLA_TransferManager* manager, XLA_Literal* c_literal, char*** buffers_array, int64_t** buffers_size, int64_t* buffers_array_size, SE_Status* status); @@ -341,6 +343,7 @@ struct TfTpu_ExecutorApiFn { TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_TransferLiteralFromDevice); TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_GetByteSizeRequirement); TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_WriteSingleTupleIndexTable); + TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_GetInfeedLayout); TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_LinearizeToBuffers); TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_FreeBuffers); diff --git a/tensorflow/stream_executor/tpu/tpu_transfer_manager.h b/tensorflow/stream_executor/tpu/tpu_transfer_manager.h index c201d63d2d5..e758c702204 100644 --- a/tensorflow/stream_executor/tpu/tpu_transfer_manager.h +++ b/tensorflow/stream_executor/tpu/tpu_transfer_manager.h @@ -81,6 +81,12 @@ class TpuTransferManager : public xla::TpuTransferManagerInterface { const xla::Shape& shape, stream_executor::DeviceMemoryBase* region) override; + Status LinearizeToBuffers( + const xla::LiteralSlice& literal, + std::deque* buffers) override { + LOG(FATAL) << "Not yet implemented."; + } + private: XLA_TransferManager* manager_; }; diff --git a/tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.cc b/tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.cc new file mode 100644 index 00000000000..746093972a4 --- /dev/null +++ b/tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.cc @@ -0,0 +1,40 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" + +#include "tensorflow/stream_executor/tpu/tpu_platform_interface.h" + +namespace xla { + +/*static*/ TpuTransferManagerInterface* +TpuTransferManagerInterface::GetRegisteredTpuTransferManager() { + auto* platform = tensorflow::tpu::TpuPlatformInterface::GetRegisteredPlatform( + /*initialize_platform=*/false); + if (platform == nullptr) { + LOG(ERROR) << "Unable to retrieve registered TPU platform."; + return nullptr; + } + auto tm = xla::TransferManager::GetForPlatform(platform); + if (!tm.ok()) { + LOG(ERROR) << "Unable to retrieve TpuTransferManager. No TPU platform is " + "registered for platform " + << platform->Name() << " and ID " << platform->id(); + return nullptr; + } + return static_cast(tm.ValueOrDie()); +} + +} // namespace xla diff --git a/tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h b/tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h index 3f34ed8064d..b7e000b89ac 100644 --- a/tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h +++ b/tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h @@ -24,9 +24,16 @@ limitations under the License. namespace xla { class TpuTransferManagerInterface : public xla::TransferManager { + public: virtual Status TransferBuffersToInfeed( se::StreamExecutor* executor, const std::deque& buffers) = 0; + + virtual Status LinearizeToBuffers( + const LiteralSlice& literal, + std::deque* buffers) = 0; + + static TpuTransferManagerInterface* GetRegisteredTpuTransferManager(); }; } // namespace xla From 997eef7812d26d8b13bb53c0d24de9a86f968deb Mon Sep 17 00:00:00 2001 From: Karim Nosir Date: Fri, 7 Aug 2020 18:21:06 -0700 Subject: [PATCH 0691/1017] Add cache for const nodes in hexagon delegate. On few test models this reduces the size of const nodes by half, which will reduce graph preparation time. Bug fix for sometimes wrong casting. Remove some redundant const nodes. PiperOrigin-RevId: 325545512 Change-Id: I6918ab991b416c9a729fd8e7e303f543b331523e --- .../lite/delegates/hexagon/builders/BUILD | 1 + .../hexagon/builders/conv_2d_builder.cc | 8 +- .../hexagon/builders/conv_2d_builder.h | 2 - .../hexagon/builders/conv_2d_helpers.cc | 19 +-- .../hexagon/builders/min_max_builder.cc | 4 - .../delegates/hexagon/builders/op_builder.cc | 112 +++++++++++++++--- .../delegates/hexagon/builders/op_builder.h | 34 +++++- .../hexagon/builders/transpose_builder.cc | 10 +- .../builders/transpose_conv_2d_builder.cc | 22 +--- .../builders/transpose_conv_2d_builder.h | 2 +- .../hexagon/hexagon_delegate_kernel.cc | 5 +- 11 files changed, 152 insertions(+), 67 deletions(-) diff --git a/tensorflow/lite/delegates/hexagon/builders/BUILD b/tensorflow/lite/delegates/hexagon/builders/BUILD index 63ff274c7b7..ef4b0e957c1 100644 --- a/tensorflow/lite/delegates/hexagon/builders/BUILD +++ b/tensorflow/lite/delegates/hexagon/builders/BUILD @@ -85,6 +85,7 @@ cc_library( "//tensorflow/lite/kernels:padding", "//tensorflow/lite/kernels/internal:optimized_base", "//tensorflow/lite/kernels/internal:tensor", + "@farmhash_archive//:farmhash", "@hexagon_nn//:hexagon_nn_ops", ], ) diff --git a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc index cfddd2c2b97..c6d20004227 100644 --- a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc @@ -267,13 +267,13 @@ TfLiteStatus Conv2dOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, auto* conv_op = graph_builder_->AddNode(GetTFLiteNodeID()); conv_op->SetOpType(OP_DepthwiseSupernode_8x8p32to8); conv_op->AddInput(space_to_batch_op_out); - conv_op->AddInput(TensorID(weights_data_node_->GetID(), 0)); + conv_op->AddInput(graph_builder_->GetHexagonTensorId(inputs->data[1])); conv_op->AddInput(TensorID(data_min_const->GetID(), 0)); conv_op->AddInput(TensorID(data_max_const->GetID(), 0)); conv_op->AddInput(TensorID(weights_min_node_->GetID(), 0)); conv_op->AddInput(TensorID(weights_max_node_->GetID(), 0)); conv_op->AddInput(TensorID(stride_node->GetID(), 0)); - conv_op->AddInput(TensorID(bias_data_node_->GetID(), 0)); + conv_op->AddInput(graph_builder_->GetHexagonTensorId(inputs->data[2])); conv_op->AddInput(TensorID(bias_min_node_->GetID(), 0)); conv_op->AddInput(TensorID(bias_max_node_->GetID(), 0)); conv_op->AddInput(TensorID(conv_output_min_const->GetID(), 0)); @@ -330,13 +330,13 @@ TfLiteStatus Conv2dOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, } // Inputs AddInput(graph_builder_->GetHexagonTensorId(inputs->data[0])); - AddInput(TensorID(weights_data_node_->GetID(), 0)); + AddInput(graph_builder_->GetHexagonTensorId(inputs->data[1])); AddInput(TensorID(data_min_const->GetID(), 0)); AddInput(TensorID(data_max_const->GetID(), 0)); AddInput(TensorID(weights_min_node_->GetID(), 0)); AddInput(TensorID(weights_max_node_->GetID(), 0)); AddInput(TensorID(stride_node->GetID(), 0)); - AddInput(TensorID(bias_data_node_->GetID(), 0)); + AddInput(graph_builder_->GetHexagonTensorId(inputs->data[2])); AddInput(TensorID(bias_min_node_->GetID(), 0)); AddInput(TensorID(bias_max_node_->GetID(), 0)); AddInput(TensorID(conv_output_min_const->GetID(), 0)); diff --git a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h index 4980b294481..1407f06154b 100644 --- a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h +++ b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h @@ -62,10 +62,8 @@ class Conv2dOpBuilder : public OpBuilder { std::vector transposed_weights_; std::vector stride_shape_; std::vector weight_shape_; - OpBuilder* weights_data_node_ = nullptr; OpBuilder* weights_min_node_ = nullptr; OpBuilder* weights_max_node_ = nullptr; - OpBuilder* bias_data_node_ = nullptr; OpBuilder* bias_min_node_ = nullptr; OpBuilder* bias_max_node_ = nullptr; diff --git a/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc b/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc index bf68bbe5a25..b33e28f4e71 100644 --- a/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc +++ b/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc @@ -106,6 +106,7 @@ TfLiteStatus Conv2dOpBuilder::InitializeWeightsNodes( const bool is_per_channel_quant = weights_quant_params->scale->size > 1; // WEIGHTS DATA. + OpBuilder* weights_data_node = nullptr; if (op_node_.op_type == OP_Supernode_8x8p32to8) { // Hexagon lib expects the weight tensor in HWCN, TFLite uses NHWC. // Transpose NHWC -> HWCN @@ -137,7 +138,7 @@ TfLiteStatus Conv2dOpBuilder::InitializeWeightsNodes( weights_tensor.data.uint8, hwcn_shape, hwcn.data()); } - weights_data_node_ = graph_builder_->AddConstNodeWithData( + weights_data_node = graph_builder_->AddConstNodeWithData( weight_shape_.data(), reinterpret_cast(hwcn.data()), hwcn.size() * sizeof(hwcn[0])); } else if (op_node_.op_type == OP_DepthwiseSupernode_8x8p32to8) { @@ -156,17 +157,17 @@ TfLiteStatus Conv2dOpBuilder::InitializeWeightsNodes( for (int i = 0; i < converted_data.size(); ++i) { converted_data[i] = weights_tensor.data.int8[i] ^ k8BitSignFlipConstant; } - weights_data_node_ = graph_builder_->AddConstNodeWithData( + weights_data_node = graph_builder_->AddConstNodeWithData( weight_shape_.data(), reinterpret_cast(converted_data.data()), converted_data.size() * sizeof(converted_data[0])); } else { - weights_data_node_ = graph_builder_->AddConstNodeWithData( + weights_data_node = graph_builder_->AddConstNodeWithData( weight_shape_.data(), weights_tensor.data.raw, NumElements(&weights_tensor) * sizeof(weights_tensor.data.uint8[0])); } } - graph_builder_->AddTensorWithID(inputs->data[1], weights_data_node_->GetID(), - 0); + graph_builder_->AddTensorWithID(inputs->data[1], weights_data_node->GetID(), + 0, /*overwrite=*/true); // WEIGHTS QUANTIZATION. float weights_min = 0; @@ -229,9 +230,11 @@ TfLiteStatus Conv2dOpBuilder::ProcessPerChannelQuantizedBias( } // Add nodes for bias. const std::vector bias_shape = {1, 1, 1, bias_size}; - bias_data_node_ = graph_builder_->AddConstNodeWithData( + auto* bias_data_node = graph_builder_->AddConstNodeWithData( bias_shape.data(), reinterpret_cast(preprocessed_bias_data.data()), preprocessed_bias_data.size() * sizeof(preprocessed_bias_data[0])); + graph_builder_->AddTensorWithID(inputs->data[2], bias_data_node->GetID(), 0, + /*overwrite=*/true); return kTfLiteOk; } @@ -248,8 +251,10 @@ TfLiteStatus Conv2dOpBuilder::InitializeBiasNodes(const TfLiteIntArray* inputs, ProcessPerChannelQuantizedBias(inputs, outputs, context, &bias_min, &bias_max); } else { - bias_data_node_ = + auto* bias_data_node = graph_builder_->AddConstNodeWithData(inputs->data[2], bias_tensor); + graph_builder_->AddTensorWithID(inputs->data[2], bias_data_node->GetID(), 0, + /*overwrite=*/true); TF_LITE_ENSURE_STATUS( ComputeMinAndMaxQuantValues(bias_tensor, &bias_min, &bias_max)); } diff --git a/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc b/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc index bcfae6032c8..0c6dea2096d 100644 --- a/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc @@ -27,10 +27,6 @@ TfLiteStatus MinMaxOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, int b_tensor_id = inputs->data[1]; const auto& a_tensor = context->tensors[a_tensor_id]; const auto& b_tensor = context->tensors[b_tensor_id]; - if (a_tensor.allocation_type == kTfLiteMmapRo) - graph_builder_->AddConstNodeWithData(a_tensor_id, a_tensor); - if (b_tensor.allocation_type == kTfLiteMmapRo) - graph_builder_->AddConstNodeWithData(b_tensor_id, b_tensor); AddInput(graph_builder_->GetHexagonTensorId(a_tensor_id)); AddInput(graph_builder_->GetHexagonTensorId(b_tensor_id)); diff --git a/tensorflow/lite/delegates/hexagon/builders/op_builder.cc b/tensorflow/lite/delegates/hexagon/builders/op_builder.cc index 0f32a4de6e1..80aa4c8155c 100644 --- a/tensorflow/lite/delegates/hexagon/builders/op_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/op_builder.cc @@ -18,10 +18,59 @@ limitations under the License. #include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/delegates/hexagon/builders/op_factory.h" +#include namespace tflite { namespace delegates { namespace hexagon { +namespace { +// Farmhash Fingerprint +inline uint64_t CombineFingerprints(uint64_t l, uint64_t h) { + // Murmur-inspired hashing. + const uint64_t kMul = 0x9ddfea08eb382d69ULL; + uint64_t a = (l ^ h) * kMul; + a ^= (a >> 47); + uint64_t b = (h ^ a) * kMul; + b ^= (b >> 44); + b *= kMul; + b ^= (b >> 41); + b *= kMul; + return b; +} + +inline uint64_t ComputeHash(const int shape[], const char* data, + const int data_len) { + return CombineFingerprints( + ::util::Fingerprint64(data, data_len), + ::util::Fingerprint64(reinterpret_cast(shape), + sizeof(shape[0]) * 4)); +} + +inline uint64_t ComputeHash(const TfLiteTensor& tensor, const int shape[], + int int8_to_uint8) { + auto data_hash = ComputeHash(shape, tensor.data.raw_const, tensor.bytes); + auto int8_to_uint8_hash = ::util::Fingerprint64( + reinterpret_cast(&int8_to_uint8), sizeof(int8_to_uint8)); + return CombineFingerprints(data_hash, int8_to_uint8_hash); +} + +int GetElementSize(TfLiteType type) { + switch (type) { + case kTfLiteFloat32: + return sizeof(float); + case kTfLiteBool: + return sizeof(bool); + case kTfLiteInt32: + return sizeof(int32_t); + case kTfLiteInt8: + return sizeof(int8_t); + case kTfLiteUInt8: + return sizeof(uint8_t); + default: + return sizeof(int8_t); + } +} +} // namespace OpBuilder* GraphBuilder::CreateOpBuilderFromTfLiteOp(int op_type, TfLiteNode* node) { @@ -116,8 +165,20 @@ OpBuilder* GraphBuilder::CreateOpBuilderFromTfLiteOp(int op_type, } } +OpBuilder* GraphBuilder::LookupConstData(uint64_t cache_key) { + auto lookup_result = cache_.find(cache_key); + if (lookup_result != cache_.end()) return lookup_result->second; + return nullptr; +} + +void GraphBuilder::AddToCache(uint64_t cache_key, OpBuilder* value) { + cache_[cache_key] = value; +} + OpBuilder* GraphBuilder::AddConstNodeWithData(const int shape[], char* data, int data_size) { + auto cache_key = ComputeHash(shape, data, data_size); + if (auto lookup_result = LookupConstData(cache_key)) return lookup_result; builders_.emplace_back(new OpBuilder(this, OP_Const)); builders_.back()->SetConstNode(); builders_.back()->SetNodeId(builders_.size()); @@ -125,22 +186,36 @@ OpBuilder* GraphBuilder::AddConstNodeWithData(const int shape[], char* data, graph_id_, builders_.size(), shape[0], shape[1], shape[2], shape[3], reinterpret_cast(data), data_size); if (error != 0) { - context_->ReportError(context_, "Error adding const node with shape id: %d", - (int)builders_.size()); + TF_LITE_KERNEL_LOG(context_, "Error adding const node with shape id: %d", + static_cast(builders_.size())); return nullptr; } + AddToCache(cache_key, builders_.back().get()); return builders_.back().get(); } OpBuilder* GraphBuilder::AddConstNodeWithData(int tensor_id, const TfLiteTensor& tensor, bool int8_to_uint8) { + // Fetch shape of tensor and pad 1's so it is always 4D. + int batch_size, height_size, width_size, depth_size; + GetDims(&batch_size, &height_size, &width_size, &depth_size, tensor.dims); + const int shape[] = {batch_size, height_size, width_size, depth_size}; + + auto cache_key = ComputeHash(tensor, shape, int8_to_uint8 ? 1 : 0); + if (auto lookup_result = LookupConstData(cache_key)) { + // If tensor is cached but with no id, that can happen when the same + // data is added from a constant value (not tensor). We can cache the data + // and reuse it. + // We assign the tensor to this cached const node before returning. + if (!HasTensor(tensor_id)) + AddTensorWithID(tensor_id, lookup_result->GetID(), 0); + return lookup_result; + } builders_.emplace_back(new OpBuilder(this, OP_Const)); const int node_id = builders_.size(); builders_.back()->SetConstNode(); builders_.back()->SetNodeId(node_id); - int batch_size, height_size, width_size, depth_size; - GetDims(&batch_size, &height_size, &width_size, &depth_size, tensor.dims); int error = hexagon_nn_->hexagon_nn_append_const_node( graph_id_, node_id, batch_size, height_size, width_size, depth_size, reinterpret_cast(tensor.data.raw), tensor.bytes); @@ -150,19 +225,26 @@ OpBuilder* GraphBuilder::AddConstNodeWithData(int tensor_id, return nullptr; } AddTensorWithID(tensor_id, node_id, 0); + // We need to return the builder with result, so we can't rely + // on builders_.back() as it can change while casting, so we hold pointer + // and update with value from casting if needed. + OpBuilder* result_builder = builders_.back().get(); // Cast int8 to uint8 if requested. // This will add cast op to uint8 and update tensor map to point // to the casted tensor. if (int8_to_uint8 && tensor.type == kTfLiteInt8) { - AddCastOp(context_, OP_Quantized_CastInt8ToUInt8, tensor_id); + AddCastOp(context_, OP_Quantized_CastInt8ToUInt8, tensor_id, + &result_builder); } - return builders_.back().get(); + AddToCache(cache_key, result_builder); + return result_builder; } // TODO(b/154604279): Support these casting ops in Hexagon op profiling (which // seems to key tensors on a single op, which may not be the case now). TfLiteStatus GraphBuilder::AddCastOp(TfLiteContext* context, int op_type, - int tensor_id) { + int tensor_id, + OpBuilder** cast_op_builder) { // Create a new OpBuilder for casting the tensor. OpBuilder* cast_builder = CreateCastBuilder(this, op_type); builders_.emplace_back(cast_builder); @@ -177,6 +259,7 @@ TfLiteStatus GraphBuilder::AddCastOp(TfLiteContext* context, int op_type, TF_LITE_ENSURE_STATUS(cast_builder->RegisterOutputs(tensor_data, context)); TfLiteIntArrayFree(tensor_data); + if (cast_op_builder != nullptr) *cast_op_builder = cast_builder; return kTfLiteOk; } @@ -192,12 +275,12 @@ TfLiteStatus GraphBuilder::AddInputTensors(const TfLiteIntArray* input_tensors, const int tensor_id = input_tensors->data[i]; const auto& tensor = context->tensors[tensor_id]; if (tensor.allocation_type == kTfLiteMmapRo) continue; - input_op->AddOutput(tensor.dims); + input_op->AddOutput(tensor.dims, GetElementSize(tensor.type)); AddTensorWithID(tensor_id, input_op->GetID(), num_inputs); // If tensor is of type int8, add an op to cast it to uint8. if (tensor.type == kTfLiteInt8) { - TF_LITE_ENSURE_STATUS( - AddCastOp(context, OP_Quantized_CastInt8ToUInt8, tensor_id)); + TF_LITE_ENSURE_STATUS(AddCastOp(context, OP_Quantized_CastInt8ToUInt8, + tensor_id, /*cast_op_builder=*/nullptr)); } ++num_inputs; } @@ -215,8 +298,8 @@ TfLiteStatus GraphBuilder::AddOutputTensors( const auto& tensor = context->tensors[tensor_id]; // If tensor is of type int8, add an op to cast it to uint8. if (tensor.type == kTfLiteInt8) { - TF_LITE_ENSURE_STATUS( - AddCastOp(context, OP_Quantized_CastUInt8ToInt8, tensor_id)); + TF_LITE_ENSURE_STATUS(AddCastOp(context, OP_Quantized_CastUInt8ToInt8, + tensor_id, /*cast_op_builder=*/nullptr)); } hexagon_output_ids.push_back(GetHexagonTensorId(tensor_id)); } @@ -231,9 +314,10 @@ TfLiteStatus GraphBuilder::AddOutputTensors( return kTfLiteOk; } -OpBuilder::TensorID OpBuilder::AddOutput(const TfLiteIntArray* dims) { +OpBuilder::TensorID OpBuilder::AddOutput(const TfLiteIntArray* dims, + int element_size) { op_node_.outputs.push_back(hexagon_nn_output()); - op_node_.outputs.back().elementsize = sizeof(uint8_t); + op_node_.outputs.back().elementsize = element_size; op_node_.outputs.back().rank = 4; // TODO(karimnosseir): What is a good to estimate the max size ? int batch_size, height_size, width_size, depth_size; diff --git a/tensorflow/lite/delegates/hexagon/builders/op_builder.h b/tensorflow/lite/delegates/hexagon/builders/op_builder.h index 52b130c756f..c2a2889b142 100644 --- a/tensorflow/lite/delegates/hexagon/builders/op_builder.h +++ b/tensorflow/lite/delegates/hexagon/builders/op_builder.h @@ -16,6 +16,7 @@ limitations under the License. #define TENSORFLOW_LITE_DELEGATES_HEXAGON_BUILDERS_OP_BUILDER_H_ #include +#include #include #include #include @@ -131,9 +132,9 @@ class OpBuilder { void AddInput(const TensorID& tensor_id) { input_ids_.push_back(tensor_id); } // Adds Output to the current node, the output has shape defined in 'dims'. - // This assumes the data type is uint8. + // The size of each element is defined using 'element_size'. // Returns the TensorID identifying this output in the graph. - TensorID AddOutput(const TfLiteIntArray* dims); + TensorID AddOutput(const TfLiteIntArray* dims, int element_size); // Adds Output to the current node, each element in the output has // size 'elementsize' and rank 'rank' and for each dimension in the output @@ -316,11 +317,22 @@ class GraphBuilder { bool AddTensorWithID(int tflite_tensor_id, int hexagon_node_id, int hexagon_node_output_id, bool overwrite = false) { if (!overwrite && HasTensor(tflite_tensor_id)) { + TF_LITE_KERNEL_LOG( + context_, + "Trying to add duplicate tensor without overwrite, tflite_tensor_id " + "%d, hexagon_node_id %d, hexagon_node_output_id %d", + tflite_tensor_id, hexagon_node_id, hexagon_node_output_id); return false; } if (tensors_.size() <= tflite_tensor_id) { tensors_.resize(tflite_tensor_id + 1); } + if (hexagon_node_id == -1 || hexagon_node_output_id == -1) + TF_LITE_KERNEL_LOG(context_, + "Trying to add invalid id, tflite_tensor_id " + "%d, hexagon_node_id %d, hexagon_node_output_id %d", + tflite_tensor_id, hexagon_node_id, + hexagon_node_output_id); tensors_[tflite_tensor_id] = OpBuilder::TensorID(hexagon_node_id, hexagon_node_output_id); return true; @@ -348,6 +360,14 @@ class GraphBuilder { int GetMaxBatchSize() const { return max_size_for_batch_; } private: + // Lookup in cache if data with key 'cache_key' is present. + // Return OpBuilder* for the data if found, nullptr otherwise. + OpBuilder* LookupConstData(uint64_t cache_key); + + // Inserts 'value' in cache, with key equals 'cache_key'. + // If data in cache with same key then it will be overwritten. + void AddToCache(uint64_t cache_key, OpBuilder* value); + // Helper method to fetch dimensions. // TODO(karimnosseir): Move this method to shared place. void GetDims(int* batch_size, int* height_size, int* width_size, @@ -360,7 +380,10 @@ class GraphBuilder { } // Adds a Cast op to convert a tensor from int8 to uint8 (or vice versa). - TfLiteStatus AddCastOp(TfLiteContext* context, int op_type, int tensor_id); + // The builder which has the casting operator is filled in 'cast_op_builder' + // if not nullptr. + TfLiteStatus AddCastOp(TfLiteContext* context, int op_type, int tensor_id, + OpBuilder** cast_op_builder); const HexagonNN* hexagon_nn_ = nullptr; TfLiteContext* context_ = nullptr; @@ -373,6 +396,11 @@ class GraphBuilder { // If the graph being built supports dynamic batch, this represents // the maximum value for batch. int max_size_for_batch_ = -1; + + // Cache for const data in the graph. + // Key is hash of the data, value is pointer to the OpBuilder* for the added + // data. + std::map cache_; }; } // namespace hexagon diff --git a/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc b/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc index 4a7304d011e..eb0c2668edc 100644 --- a/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc @@ -29,15 +29,7 @@ TfLiteStatus TransposeOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, const auto& input_tensor = context->tensors[tensor_id]; AddInput(graph_builder_->GetHexagonTensorId(tensor_id)); // permutation tensor. - tensor_id = inputs->data[1]; - const auto& control_tensor = context->tensors[tensor_id]; - if (control_tensor.allocation_type == kTfLiteMmapRo) { - auto* const_control_tensor_node = - graph_builder_->AddConstNodeWithData(tensor_id, control_tensor); - AddInput(TensorID(const_control_tensor_node->GetID(), 0)); - } else { - AddInput(graph_builder_->GetHexagonTensorId(tensor_id)); - } + AddInput(graph_builder_->GetHexagonTensorId(inputs->data[1])); TF_LITE_ENSURE_STATUS(ComputeAndAddMinAndMax(context, input_tensor)); diff --git a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc index d2620f71007..3e852533394 100644 --- a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc @@ -97,8 +97,6 @@ TfLiteStatus TransposeConv2dOpBuilder::PopulateSubGraph( filter_depth_size; GetDims(&filter_batch_size, &filter_height_size, &filter_width_size, &filter_depth_size, weights_tensor.dims); - weight_shape_ = {filter_batch_size, filter_height_size, filter_width_size, - filter_depth_size}; // Weights tensor could be int8 even for per-tensor quantization. // Therefore, we look at the number of scale values to check if it is // per-channel quantized. @@ -106,25 +104,7 @@ TfLiteStatus TransposeConv2dOpBuilder::PopulateSubGraph( reinterpret_cast( weights_tensor.quantization.params); const bool is_per_channel_quant = weights_quant_params->scale->size > 1; - - OpBuilder* const_weights_node; - if (weights_tensor.type == kTfLiteInt8) { - std::vector weights_data(NumElements(&weights_tensor)); - const int8_t* original_data = weights_tensor.data.int8; - // Flip bits on the weight values so that the int8 values are treated - // as uint8. - for (int i = 0; i < NumElements(&weights_tensor); ++i) { - weights_data[i] = original_data[i] ^ k8BitSignFlipConstant; - } - const_weights_node = graph_builder_->AddConstNodeWithData( - weight_shape_.data(), reinterpret_cast(weights_data.data()), - weights_data.size() * sizeof(weights_data[0])); - } else { - const_weights_node = graph_builder_->AddConstNodeWithData( - weight_shape_.data(), weights_tensor.data.raw, weights_tensor.bytes); - } - graph_builder_->AddTensorWithID(tensor_id, const_weights_node->GetID(), 0); - AddInput(TensorID(const_weights_node->GetID(), 0)); + AddInput(graph_builder_->GetHexagonTensorId(tensor_id)); // Handle weights quantization. float weights_min = 0; diff --git a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h index 0a6a90a0297..4afab9894f0 100644 --- a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h +++ b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h @@ -47,7 +47,7 @@ class TransposeConv2dOpBuilder : public OpBuilder { TensorID node_output_; std::vector transposed_weights_; std::vector stride_shape_; - std::vector weight_shape_, bias_shape_; + std::vector bias_shape_; std::vector bias_data_; // Non-null only if node has per-channel quantized weights/biases. diff --git a/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc b/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc index cdf6b555929..83ebc15510e 100644 --- a/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc +++ b/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc @@ -264,8 +264,9 @@ TfLiteStatus HexagonDelegateKernel::BuildGraph( if (tensor_id == -1) continue; const auto& input_tensor = context->tensors[tensor_id]; if (input_tensor.allocation_type == kTfLiteMmapRo) { - builder_->AddConstNodeWithData(tensor_id, input_tensor, - /*int8_to_uint8*/ true); + builder_->AddConstNodeWithData( + tensor_id, input_tensor, + /*int8_to_uint8*/ (input_tensor.type == kTfLiteInt8)); } } auto* op_builder = From e38601bb2d8fde60ad333483ebf44a05dff87624 Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Fri, 7 Aug 2020 10:14:51 -0700 Subject: [PATCH 0692/1017] Parallel tf.AddN Clarify range --- .../mlir/tensorflow/tests/lower_tf.mlir | 18 ++++++-- .../mlir/tensorflow/transforms/lower_tf.cc | 45 ++++++++++++++++--- 2 files changed, 52 insertions(+), 11 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/lower_tf.mlir b/tensorflow/compiler/mlir/tensorflow/tests/lower_tf.mlir index e7e07845fcc..23fdddc0eb7 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/lower_tf.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/lower_tf.mlir @@ -353,15 +353,25 @@ func @ZerosLike_variant(%arg0: tensor>>) -> tensor>> } -// CHECK-LABEL: func @addN -func @addN(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<*xf32>) -> tensor<*xf32> { - // CHECK: %[[SUM0:.*]] = "tf.AddV2"(%arg0, %arg1) - // CHECK: %[[SUM1:.*]] = "tf.AddV2"(%[[SUM0]], %arg2) +// CHECK-LABEL: func @addN_3 +func @addN_3(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: %[[SUM0:.*]] = "tf.AddV2"(%arg1, %arg2) + // CHECK: %[[SUM1:.*]] = "tf.AddV2"(%arg0, %[[SUM0]]) // return %[[SUM1]] %0 = "tf.AddN"(%arg0, %arg1, %arg2) : (tensor<*xf32>, tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> return %0 : tensor<*xf32> } +// CHECK-LABEL: func @addN_4 +func @addN_4(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<*xf32>, %arg3: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: %[[SUM0:.*]] = "tf.AddV2"(%arg0, %arg1) + // CHECK: %[[SUM1:.*]] = "tf.AddV2"(%arg2, %arg3) + // CHECK: %[[SUM2:.*]] = "tf.AddV2"(%[[SUM0]], %[[SUM1]]) + // return %[[SUM2]] + %0 = "tf.AddN"(%arg0, %arg1, %arg2, %arg3) : (tensor<*xf32>, tensor<*xf32>, tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + return %0 : tensor<*xf32> +} + // CHECK-LABEL: func @addN_variant func @addN_variant(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>) -> tensor>> { // CHECK: tf.AddN diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc b/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc index d67739a739b..f853d8bd1fa 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc @@ -113,12 +113,27 @@ Type InferExpandDimsType(Type ty, int64_t axis, Builder *builder) { // Lowers AddN op to a sequence of AddV2 ops to accumulate operands. // +// Note that to improve the parallelism, the operands are split +// into two halves, and are accumulated first. +// +// Example: +// // %result = "tf.AddN"(%0, %1, %2) // // is lowered to: // -// %sum_0 = "tf.AddV2"(%0, %1) -// %result = "tf.AddV2"(%sum_0, %2) +// %sum_right = "tf.AddV2"(%1, %2) +// %result = "tf.AddV2"(%0, %sum_right) +// +// Or +// +// %result = "tf.AddN"(%0, %1, %2, %3) +// +// is lowered to: +// +// %sum_left = "tf.AddV2"(%0, %1) +// %sum_right = "tf.AddV2"(%2, %2) +// %result = "tf.AddV2"(%sum_left, %sum_right) // class LowerAddNOp : public OpRewritePattern { public: @@ -131,13 +146,29 @@ class LowerAddNOp : public OpRewritePattern { // support variant type so variant types require special handling. if (getElementTypeOrSelf(op.getType()).isa()) return failure(); - // TODO(hinsu): Improve parallelism by splitting operands in two halves and - // accumulating them first. - Value result = *op.inputs().begin(); - for (Value operand : llvm::drop_begin(op.inputs(), 1)) { - result = rewriter.create(op.getLoc(), result, operand); + auto begin = op.inputs().begin(); + // Return the only operand directly. + if (op.N() == 1) { + rewriter.replaceOp(op, *begin); + return success(); } + // Helper functor to accumulate from `begin` to `end` (exclusive). + auto accumulate_add = [&rewriter, &op] (auto begin, auto end) -> Value { + Value result = *begin; + ++begin; + for (auto operand = begin; operand != end; ++operand) { + result = rewriter.create(op.getLoc(), result, *operand); + } + return result; + }; + + // Accumulate range `[begin, half)` and `[half, end)`, + // and add the results of two halves. + auto half = begin + op.N() / 2; + Value left = accumulate_add(begin, half); + Value right = accumulate_add(half, op.inputs().end()); + Value result = rewriter.create(op.getLoc(), left, right); rewriter.replaceOp(op, result); return success(); } From be5728d9362a1259d9e5126146782b2d3dbde2ea Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Fri, 7 Aug 2020 16:46:32 -0700 Subject: [PATCH 0693/1017] Use tree-based reduction Fix comment Update index Remove trivial word Use only i to index --- .../mlir/tensorflow/tests/lower_tf.mlir | 23 +++++- .../mlir/tensorflow/transforms/lower_tf.cc | 70 +++++++++++-------- 2 files changed, 60 insertions(+), 33 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/lower_tf.mlir b/tensorflow/compiler/mlir/tensorflow/tests/lower_tf.mlir index 23fdddc0eb7..e11474c0755 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/lower_tf.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/lower_tf.mlir @@ -353,10 +353,18 @@ func @ZerosLike_variant(%arg0: tensor>>) -> tensor>> } +// CHECK-LABEL: func @addN_2 +func @addN_2(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: %[[SUM0:.*]] = "tf.AddV2"(%arg0, %arg1) + // return %[[SUM0]] + %0 = "tf.AddN"(%arg0, %arg1) : (tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + return %0 : tensor<*xf32> +} + // CHECK-LABEL: func @addN_3 func @addN_3(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<*xf32>) -> tensor<*xf32> { - // CHECK: %[[SUM0:.*]] = "tf.AddV2"(%arg1, %arg2) - // CHECK: %[[SUM1:.*]] = "tf.AddV2"(%arg0, %[[SUM0]]) + // CHECK: %[[SUM0:.*]] = "tf.AddV2"(%arg0, %arg1) + // CHECK: %[[SUM1:.*]] = "tf.AddV2"(%[[SUM0]], %arg2) // return %[[SUM1]] %0 = "tf.AddN"(%arg0, %arg1, %arg2) : (tensor<*xf32>, tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> return %0 : tensor<*xf32> @@ -372,6 +380,17 @@ func @addN_4(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<*xf32>, % return %0 : tensor<*xf32> } +// CHECK-LABEL: func @addN_5 +func @addN_5(%arg0: tensor<*xf32>, %arg1: tensor<*xf32>, %arg2: tensor<*xf32>, %arg3: tensor<*xf32>, %arg4: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: %[[SUM0:.*]] = "tf.AddV2"(%arg0, %arg1) + // CHECK: %[[SUM1:.*]] = "tf.AddV2"(%arg2, %arg3) + // CHECK: %[[SUM2:.*]] = "tf.AddV2"(%[[SUM0]], %[[SUM1]]) + // CHECK: %[[SUM3:.*]] = "tf.AddV2"(%[[SUM2]], %arg4) + // return %[[SUM3]] + %0 = "tf.AddN"(%arg0, %arg1, %arg2, %arg3, %arg4) : (tensor<*xf32>, tensor<*xf32>, tensor<*xf32>, tensor<*xf32>, tensor<*xf32>) -> tensor<*xf32> + return %0 : tensor<*xf32> +} + // CHECK-LABEL: func @addN_variant func @addN_variant(%arg0: tensor>>, %arg1: tensor>>, %arg2: tensor>>) -> tensor>> { // CHECK: tf.AddN diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc b/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc index f853d8bd1fa..483c84b3e80 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/lower_tf.cc @@ -113,8 +113,22 @@ Type InferExpandDimsType(Type ty, int64_t axis, Builder *builder) { // Lowers AddN op to a sequence of AddV2 ops to accumulate operands. // -// Note that to improve the parallelism, the operands are split -// into two halves, and are accumulated first. +// Note that to improve the parallelism, AddN op uses tree-based reduction. +// For example, tf.AddN([0, 1, 2, 3, 4]) behaves as follows: +// +// 0 1 2 3 4 +// | | | | | +// ------- ------- | +// | | | +// 5 6 | +// | | | +// ------------- | +// | | +// 7 | +// | | +// ---------------- +// | +// 8 // // Example: // @@ -122,18 +136,19 @@ Type InferExpandDimsType(Type ty, int64_t axis, Builder *builder) { // // is lowered to: // -// %sum_right = "tf.AddV2"(%1, %2) -// %result = "tf.AddV2"(%0, %sum_right) +// %sum0 = "tf.AddV2"(%0, %1) +// %result = "tf.AddV2"(%sum0, %2) // -// Or +// While // -// %result = "tf.AddN"(%0, %1, %2, %3) +// %result = "tf.AddN"(%0, %1, %2, %3, %4) // // is lowered to: // -// %sum_left = "tf.AddV2"(%0, %1) -// %sum_right = "tf.AddV2"(%2, %2) -// %result = "tf.AddV2"(%sum_left, %sum_right) +// %sum0 = "tf.AddV2"(%0, %1) +// %sum1 = "tf.AddV2"(%2, %3) +// %sum2 = "tf.AddV2"(%sum0, %sum1) +// %result = "tf.AddV2"(%sum2, %4) // class LowerAddNOp : public OpRewritePattern { public: @@ -146,30 +161,23 @@ class LowerAddNOp : public OpRewritePattern { // support variant type so variant types require special handling. if (getElementTypeOrSelf(op.getType()).isa()) return failure(); - auto begin = op.inputs().begin(); - // Return the only operand directly. - if (op.N() == 1) { - rewriter.replaceOp(op, *begin); - return success(); + llvm::SmallVector operands(op.inputs().begin(), + op.inputs().end()); + + int64_t n = operands.size(); + // Keep doing tree-based reduction when there are more than one operand. + while (n > 1) { + for (int64_t i = 0; i < n; i += 2) { + // Add two adjacent operands if applicable. + operands[i / 2] = (i + 1 < n) + ? rewriter.create( + op.getLoc(), operands[i], operands[i + 1]) + : operands[i]; + } + n = (n + 1) / 2; } - // Helper functor to accumulate from `begin` to `end` (exclusive). - auto accumulate_add = [&rewriter, &op] (auto begin, auto end) -> Value { - Value result = *begin; - ++begin; - for (auto operand = begin; operand != end; ++operand) { - result = rewriter.create(op.getLoc(), result, *operand); - } - return result; - }; - - // Accumulate range `[begin, half)` and `[half, end)`, - // and add the results of two halves. - auto half = begin + op.N() / 2; - Value left = accumulate_add(begin, half); - Value right = accumulate_add(half, op.inputs().end()); - Value result = rewriter.create(op.getLoc(), left, right); - rewriter.replaceOp(op, result); + rewriter.replaceOp(op, operands[0]); return success(); } }; From 902abbe41e198d58446896dacc4f00bc5e7bac7d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Aug 2020 20:12:44 -0700 Subject: [PATCH 0694/1017] Add cache for const nodes in hexagon delegate. On few test models this reduces the size of const nodes by half, which will reduce graph preparation time. Bug fix for sometimes wrong casting. Remove some redundant const nodes. PiperOrigin-RevId: 325554729 Change-Id: I805afc4151d99a1ed2f739accc7f86d6336b14b7 --- .../lite/delegates/hexagon/builders/BUILD | 1 - .../hexagon/builders/conv_2d_builder.cc | 8 +- .../hexagon/builders/conv_2d_builder.h | 2 + .../hexagon/builders/conv_2d_helpers.cc | 19 ++- .../hexagon/builders/min_max_builder.cc | 4 + .../delegates/hexagon/builders/op_builder.cc | 112 +++--------------- .../delegates/hexagon/builders/op_builder.h | 34 +----- .../hexagon/builders/transpose_builder.cc | 10 +- .../builders/transpose_conv_2d_builder.cc | 22 +++- .../builders/transpose_conv_2d_builder.h | 2 +- .../hexagon/hexagon_delegate_kernel.cc | 5 +- 11 files changed, 67 insertions(+), 152 deletions(-) diff --git a/tensorflow/lite/delegates/hexagon/builders/BUILD b/tensorflow/lite/delegates/hexagon/builders/BUILD index ef4b0e957c1..63ff274c7b7 100644 --- a/tensorflow/lite/delegates/hexagon/builders/BUILD +++ b/tensorflow/lite/delegates/hexagon/builders/BUILD @@ -85,7 +85,6 @@ cc_library( "//tensorflow/lite/kernels:padding", "//tensorflow/lite/kernels/internal:optimized_base", "//tensorflow/lite/kernels/internal:tensor", - "@farmhash_archive//:farmhash", "@hexagon_nn//:hexagon_nn_ops", ], ) diff --git a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc index c6d20004227..cfddd2c2b97 100644 --- a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc @@ -267,13 +267,13 @@ TfLiteStatus Conv2dOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, auto* conv_op = graph_builder_->AddNode(GetTFLiteNodeID()); conv_op->SetOpType(OP_DepthwiseSupernode_8x8p32to8); conv_op->AddInput(space_to_batch_op_out); - conv_op->AddInput(graph_builder_->GetHexagonTensorId(inputs->data[1])); + conv_op->AddInput(TensorID(weights_data_node_->GetID(), 0)); conv_op->AddInput(TensorID(data_min_const->GetID(), 0)); conv_op->AddInput(TensorID(data_max_const->GetID(), 0)); conv_op->AddInput(TensorID(weights_min_node_->GetID(), 0)); conv_op->AddInput(TensorID(weights_max_node_->GetID(), 0)); conv_op->AddInput(TensorID(stride_node->GetID(), 0)); - conv_op->AddInput(graph_builder_->GetHexagonTensorId(inputs->data[2])); + conv_op->AddInput(TensorID(bias_data_node_->GetID(), 0)); conv_op->AddInput(TensorID(bias_min_node_->GetID(), 0)); conv_op->AddInput(TensorID(bias_max_node_->GetID(), 0)); conv_op->AddInput(TensorID(conv_output_min_const->GetID(), 0)); @@ -330,13 +330,13 @@ TfLiteStatus Conv2dOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, } // Inputs AddInput(graph_builder_->GetHexagonTensorId(inputs->data[0])); - AddInput(graph_builder_->GetHexagonTensorId(inputs->data[1])); + AddInput(TensorID(weights_data_node_->GetID(), 0)); AddInput(TensorID(data_min_const->GetID(), 0)); AddInput(TensorID(data_max_const->GetID(), 0)); AddInput(TensorID(weights_min_node_->GetID(), 0)); AddInput(TensorID(weights_max_node_->GetID(), 0)); AddInput(TensorID(stride_node->GetID(), 0)); - AddInput(graph_builder_->GetHexagonTensorId(inputs->data[2])); + AddInput(TensorID(bias_data_node_->GetID(), 0)); AddInput(TensorID(bias_min_node_->GetID(), 0)); AddInput(TensorID(bias_max_node_->GetID(), 0)); AddInput(TensorID(conv_output_min_const->GetID(), 0)); diff --git a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h index 1407f06154b..4980b294481 100644 --- a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h +++ b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h @@ -62,8 +62,10 @@ class Conv2dOpBuilder : public OpBuilder { std::vector transposed_weights_; std::vector stride_shape_; std::vector weight_shape_; + OpBuilder* weights_data_node_ = nullptr; OpBuilder* weights_min_node_ = nullptr; OpBuilder* weights_max_node_ = nullptr; + OpBuilder* bias_data_node_ = nullptr; OpBuilder* bias_min_node_ = nullptr; OpBuilder* bias_max_node_ = nullptr; diff --git a/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc b/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc index b33e28f4e71..bf68bbe5a25 100644 --- a/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc +++ b/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc @@ -106,7 +106,6 @@ TfLiteStatus Conv2dOpBuilder::InitializeWeightsNodes( const bool is_per_channel_quant = weights_quant_params->scale->size > 1; // WEIGHTS DATA. - OpBuilder* weights_data_node = nullptr; if (op_node_.op_type == OP_Supernode_8x8p32to8) { // Hexagon lib expects the weight tensor in HWCN, TFLite uses NHWC. // Transpose NHWC -> HWCN @@ -138,7 +137,7 @@ TfLiteStatus Conv2dOpBuilder::InitializeWeightsNodes( weights_tensor.data.uint8, hwcn_shape, hwcn.data()); } - weights_data_node = graph_builder_->AddConstNodeWithData( + weights_data_node_ = graph_builder_->AddConstNodeWithData( weight_shape_.data(), reinterpret_cast(hwcn.data()), hwcn.size() * sizeof(hwcn[0])); } else if (op_node_.op_type == OP_DepthwiseSupernode_8x8p32to8) { @@ -157,17 +156,17 @@ TfLiteStatus Conv2dOpBuilder::InitializeWeightsNodes( for (int i = 0; i < converted_data.size(); ++i) { converted_data[i] = weights_tensor.data.int8[i] ^ k8BitSignFlipConstant; } - weights_data_node = graph_builder_->AddConstNodeWithData( + weights_data_node_ = graph_builder_->AddConstNodeWithData( weight_shape_.data(), reinterpret_cast(converted_data.data()), converted_data.size() * sizeof(converted_data[0])); } else { - weights_data_node = graph_builder_->AddConstNodeWithData( + weights_data_node_ = graph_builder_->AddConstNodeWithData( weight_shape_.data(), weights_tensor.data.raw, NumElements(&weights_tensor) * sizeof(weights_tensor.data.uint8[0])); } } - graph_builder_->AddTensorWithID(inputs->data[1], weights_data_node->GetID(), - 0, /*overwrite=*/true); + graph_builder_->AddTensorWithID(inputs->data[1], weights_data_node_->GetID(), + 0); // WEIGHTS QUANTIZATION. float weights_min = 0; @@ -230,11 +229,9 @@ TfLiteStatus Conv2dOpBuilder::ProcessPerChannelQuantizedBias( } // Add nodes for bias. const std::vector bias_shape = {1, 1, 1, bias_size}; - auto* bias_data_node = graph_builder_->AddConstNodeWithData( + bias_data_node_ = graph_builder_->AddConstNodeWithData( bias_shape.data(), reinterpret_cast(preprocessed_bias_data.data()), preprocessed_bias_data.size() * sizeof(preprocessed_bias_data[0])); - graph_builder_->AddTensorWithID(inputs->data[2], bias_data_node->GetID(), 0, - /*overwrite=*/true); return kTfLiteOk; } @@ -251,10 +248,8 @@ TfLiteStatus Conv2dOpBuilder::InitializeBiasNodes(const TfLiteIntArray* inputs, ProcessPerChannelQuantizedBias(inputs, outputs, context, &bias_min, &bias_max); } else { - auto* bias_data_node = + bias_data_node_ = graph_builder_->AddConstNodeWithData(inputs->data[2], bias_tensor); - graph_builder_->AddTensorWithID(inputs->data[2], bias_data_node->GetID(), 0, - /*overwrite=*/true); TF_LITE_ENSURE_STATUS( ComputeMinAndMaxQuantValues(bias_tensor, &bias_min, &bias_max)); } diff --git a/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc b/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc index 0c6dea2096d..bcfae6032c8 100644 --- a/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc @@ -27,6 +27,10 @@ TfLiteStatus MinMaxOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, int b_tensor_id = inputs->data[1]; const auto& a_tensor = context->tensors[a_tensor_id]; const auto& b_tensor = context->tensors[b_tensor_id]; + if (a_tensor.allocation_type == kTfLiteMmapRo) + graph_builder_->AddConstNodeWithData(a_tensor_id, a_tensor); + if (b_tensor.allocation_type == kTfLiteMmapRo) + graph_builder_->AddConstNodeWithData(b_tensor_id, b_tensor); AddInput(graph_builder_->GetHexagonTensorId(a_tensor_id)); AddInput(graph_builder_->GetHexagonTensorId(b_tensor_id)); diff --git a/tensorflow/lite/delegates/hexagon/builders/op_builder.cc b/tensorflow/lite/delegates/hexagon/builders/op_builder.cc index 80aa4c8155c..0f32a4de6e1 100644 --- a/tensorflow/lite/delegates/hexagon/builders/op_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/op_builder.cc @@ -18,59 +18,10 @@ limitations under the License. #include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/delegates/hexagon/builders/op_factory.h" -#include namespace tflite { namespace delegates { namespace hexagon { -namespace { -// Farmhash Fingerprint -inline uint64_t CombineFingerprints(uint64_t l, uint64_t h) { - // Murmur-inspired hashing. - const uint64_t kMul = 0x9ddfea08eb382d69ULL; - uint64_t a = (l ^ h) * kMul; - a ^= (a >> 47); - uint64_t b = (h ^ a) * kMul; - b ^= (b >> 44); - b *= kMul; - b ^= (b >> 41); - b *= kMul; - return b; -} - -inline uint64_t ComputeHash(const int shape[], const char* data, - const int data_len) { - return CombineFingerprints( - ::util::Fingerprint64(data, data_len), - ::util::Fingerprint64(reinterpret_cast(shape), - sizeof(shape[0]) * 4)); -} - -inline uint64_t ComputeHash(const TfLiteTensor& tensor, const int shape[], - int int8_to_uint8) { - auto data_hash = ComputeHash(shape, tensor.data.raw_const, tensor.bytes); - auto int8_to_uint8_hash = ::util::Fingerprint64( - reinterpret_cast(&int8_to_uint8), sizeof(int8_to_uint8)); - return CombineFingerprints(data_hash, int8_to_uint8_hash); -} - -int GetElementSize(TfLiteType type) { - switch (type) { - case kTfLiteFloat32: - return sizeof(float); - case kTfLiteBool: - return sizeof(bool); - case kTfLiteInt32: - return sizeof(int32_t); - case kTfLiteInt8: - return sizeof(int8_t); - case kTfLiteUInt8: - return sizeof(uint8_t); - default: - return sizeof(int8_t); - } -} -} // namespace OpBuilder* GraphBuilder::CreateOpBuilderFromTfLiteOp(int op_type, TfLiteNode* node) { @@ -165,20 +116,8 @@ OpBuilder* GraphBuilder::CreateOpBuilderFromTfLiteOp(int op_type, } } -OpBuilder* GraphBuilder::LookupConstData(uint64_t cache_key) { - auto lookup_result = cache_.find(cache_key); - if (lookup_result != cache_.end()) return lookup_result->second; - return nullptr; -} - -void GraphBuilder::AddToCache(uint64_t cache_key, OpBuilder* value) { - cache_[cache_key] = value; -} - OpBuilder* GraphBuilder::AddConstNodeWithData(const int shape[], char* data, int data_size) { - auto cache_key = ComputeHash(shape, data, data_size); - if (auto lookup_result = LookupConstData(cache_key)) return lookup_result; builders_.emplace_back(new OpBuilder(this, OP_Const)); builders_.back()->SetConstNode(); builders_.back()->SetNodeId(builders_.size()); @@ -186,36 +125,22 @@ OpBuilder* GraphBuilder::AddConstNodeWithData(const int shape[], char* data, graph_id_, builders_.size(), shape[0], shape[1], shape[2], shape[3], reinterpret_cast(data), data_size); if (error != 0) { - TF_LITE_KERNEL_LOG(context_, "Error adding const node with shape id: %d", - static_cast(builders_.size())); + context_->ReportError(context_, "Error adding const node with shape id: %d", + (int)builders_.size()); return nullptr; } - AddToCache(cache_key, builders_.back().get()); return builders_.back().get(); } OpBuilder* GraphBuilder::AddConstNodeWithData(int tensor_id, const TfLiteTensor& tensor, bool int8_to_uint8) { - // Fetch shape of tensor and pad 1's so it is always 4D. - int batch_size, height_size, width_size, depth_size; - GetDims(&batch_size, &height_size, &width_size, &depth_size, tensor.dims); - const int shape[] = {batch_size, height_size, width_size, depth_size}; - - auto cache_key = ComputeHash(tensor, shape, int8_to_uint8 ? 1 : 0); - if (auto lookup_result = LookupConstData(cache_key)) { - // If tensor is cached but with no id, that can happen when the same - // data is added from a constant value (not tensor). We can cache the data - // and reuse it. - // We assign the tensor to this cached const node before returning. - if (!HasTensor(tensor_id)) - AddTensorWithID(tensor_id, lookup_result->GetID(), 0); - return lookup_result; - } builders_.emplace_back(new OpBuilder(this, OP_Const)); const int node_id = builders_.size(); builders_.back()->SetConstNode(); builders_.back()->SetNodeId(node_id); + int batch_size, height_size, width_size, depth_size; + GetDims(&batch_size, &height_size, &width_size, &depth_size, tensor.dims); int error = hexagon_nn_->hexagon_nn_append_const_node( graph_id_, node_id, batch_size, height_size, width_size, depth_size, reinterpret_cast(tensor.data.raw), tensor.bytes); @@ -225,26 +150,19 @@ OpBuilder* GraphBuilder::AddConstNodeWithData(int tensor_id, return nullptr; } AddTensorWithID(tensor_id, node_id, 0); - // We need to return the builder with result, so we can't rely - // on builders_.back() as it can change while casting, so we hold pointer - // and update with value from casting if needed. - OpBuilder* result_builder = builders_.back().get(); // Cast int8 to uint8 if requested. // This will add cast op to uint8 and update tensor map to point // to the casted tensor. if (int8_to_uint8 && tensor.type == kTfLiteInt8) { - AddCastOp(context_, OP_Quantized_CastInt8ToUInt8, tensor_id, - &result_builder); + AddCastOp(context_, OP_Quantized_CastInt8ToUInt8, tensor_id); } - AddToCache(cache_key, result_builder); - return result_builder; + return builders_.back().get(); } // TODO(b/154604279): Support these casting ops in Hexagon op profiling (which // seems to key tensors on a single op, which may not be the case now). TfLiteStatus GraphBuilder::AddCastOp(TfLiteContext* context, int op_type, - int tensor_id, - OpBuilder** cast_op_builder) { + int tensor_id) { // Create a new OpBuilder for casting the tensor. OpBuilder* cast_builder = CreateCastBuilder(this, op_type); builders_.emplace_back(cast_builder); @@ -259,7 +177,6 @@ TfLiteStatus GraphBuilder::AddCastOp(TfLiteContext* context, int op_type, TF_LITE_ENSURE_STATUS(cast_builder->RegisterOutputs(tensor_data, context)); TfLiteIntArrayFree(tensor_data); - if (cast_op_builder != nullptr) *cast_op_builder = cast_builder; return kTfLiteOk; } @@ -275,12 +192,12 @@ TfLiteStatus GraphBuilder::AddInputTensors(const TfLiteIntArray* input_tensors, const int tensor_id = input_tensors->data[i]; const auto& tensor = context->tensors[tensor_id]; if (tensor.allocation_type == kTfLiteMmapRo) continue; - input_op->AddOutput(tensor.dims, GetElementSize(tensor.type)); + input_op->AddOutput(tensor.dims); AddTensorWithID(tensor_id, input_op->GetID(), num_inputs); // If tensor is of type int8, add an op to cast it to uint8. if (tensor.type == kTfLiteInt8) { - TF_LITE_ENSURE_STATUS(AddCastOp(context, OP_Quantized_CastInt8ToUInt8, - tensor_id, /*cast_op_builder=*/nullptr)); + TF_LITE_ENSURE_STATUS( + AddCastOp(context, OP_Quantized_CastInt8ToUInt8, tensor_id)); } ++num_inputs; } @@ -298,8 +215,8 @@ TfLiteStatus GraphBuilder::AddOutputTensors( const auto& tensor = context->tensors[tensor_id]; // If tensor is of type int8, add an op to cast it to uint8. if (tensor.type == kTfLiteInt8) { - TF_LITE_ENSURE_STATUS(AddCastOp(context, OP_Quantized_CastUInt8ToInt8, - tensor_id, /*cast_op_builder=*/nullptr)); + TF_LITE_ENSURE_STATUS( + AddCastOp(context, OP_Quantized_CastUInt8ToInt8, tensor_id)); } hexagon_output_ids.push_back(GetHexagonTensorId(tensor_id)); } @@ -314,10 +231,9 @@ TfLiteStatus GraphBuilder::AddOutputTensors( return kTfLiteOk; } -OpBuilder::TensorID OpBuilder::AddOutput(const TfLiteIntArray* dims, - int element_size) { +OpBuilder::TensorID OpBuilder::AddOutput(const TfLiteIntArray* dims) { op_node_.outputs.push_back(hexagon_nn_output()); - op_node_.outputs.back().elementsize = element_size; + op_node_.outputs.back().elementsize = sizeof(uint8_t); op_node_.outputs.back().rank = 4; // TODO(karimnosseir): What is a good to estimate the max size ? int batch_size, height_size, width_size, depth_size; diff --git a/tensorflow/lite/delegates/hexagon/builders/op_builder.h b/tensorflow/lite/delegates/hexagon/builders/op_builder.h index c2a2889b142..52b130c756f 100644 --- a/tensorflow/lite/delegates/hexagon/builders/op_builder.h +++ b/tensorflow/lite/delegates/hexagon/builders/op_builder.h @@ -16,7 +16,6 @@ limitations under the License. #define TENSORFLOW_LITE_DELEGATES_HEXAGON_BUILDERS_OP_BUILDER_H_ #include -#include #include #include #include @@ -132,9 +131,9 @@ class OpBuilder { void AddInput(const TensorID& tensor_id) { input_ids_.push_back(tensor_id); } // Adds Output to the current node, the output has shape defined in 'dims'. - // The size of each element is defined using 'element_size'. + // This assumes the data type is uint8. // Returns the TensorID identifying this output in the graph. - TensorID AddOutput(const TfLiteIntArray* dims, int element_size); + TensorID AddOutput(const TfLiteIntArray* dims); // Adds Output to the current node, each element in the output has // size 'elementsize' and rank 'rank' and for each dimension in the output @@ -317,22 +316,11 @@ class GraphBuilder { bool AddTensorWithID(int tflite_tensor_id, int hexagon_node_id, int hexagon_node_output_id, bool overwrite = false) { if (!overwrite && HasTensor(tflite_tensor_id)) { - TF_LITE_KERNEL_LOG( - context_, - "Trying to add duplicate tensor without overwrite, tflite_tensor_id " - "%d, hexagon_node_id %d, hexagon_node_output_id %d", - tflite_tensor_id, hexagon_node_id, hexagon_node_output_id); return false; } if (tensors_.size() <= tflite_tensor_id) { tensors_.resize(tflite_tensor_id + 1); } - if (hexagon_node_id == -1 || hexagon_node_output_id == -1) - TF_LITE_KERNEL_LOG(context_, - "Trying to add invalid id, tflite_tensor_id " - "%d, hexagon_node_id %d, hexagon_node_output_id %d", - tflite_tensor_id, hexagon_node_id, - hexagon_node_output_id); tensors_[tflite_tensor_id] = OpBuilder::TensorID(hexagon_node_id, hexagon_node_output_id); return true; @@ -360,14 +348,6 @@ class GraphBuilder { int GetMaxBatchSize() const { return max_size_for_batch_; } private: - // Lookup in cache if data with key 'cache_key' is present. - // Return OpBuilder* for the data if found, nullptr otherwise. - OpBuilder* LookupConstData(uint64_t cache_key); - - // Inserts 'value' in cache, with key equals 'cache_key'. - // If data in cache with same key then it will be overwritten. - void AddToCache(uint64_t cache_key, OpBuilder* value); - // Helper method to fetch dimensions. // TODO(karimnosseir): Move this method to shared place. void GetDims(int* batch_size, int* height_size, int* width_size, @@ -380,10 +360,7 @@ class GraphBuilder { } // Adds a Cast op to convert a tensor from int8 to uint8 (or vice versa). - // The builder which has the casting operator is filled in 'cast_op_builder' - // if not nullptr. - TfLiteStatus AddCastOp(TfLiteContext* context, int op_type, int tensor_id, - OpBuilder** cast_op_builder); + TfLiteStatus AddCastOp(TfLiteContext* context, int op_type, int tensor_id); const HexagonNN* hexagon_nn_ = nullptr; TfLiteContext* context_ = nullptr; @@ -396,11 +373,6 @@ class GraphBuilder { // If the graph being built supports dynamic batch, this represents // the maximum value for batch. int max_size_for_batch_ = -1; - - // Cache for const data in the graph. - // Key is hash of the data, value is pointer to the OpBuilder* for the added - // data. - std::map cache_; }; } // namespace hexagon diff --git a/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc b/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc index eb0c2668edc..4a7304d011e 100644 --- a/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc @@ -29,7 +29,15 @@ TfLiteStatus TransposeOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, const auto& input_tensor = context->tensors[tensor_id]; AddInput(graph_builder_->GetHexagonTensorId(tensor_id)); // permutation tensor. - AddInput(graph_builder_->GetHexagonTensorId(inputs->data[1])); + tensor_id = inputs->data[1]; + const auto& control_tensor = context->tensors[tensor_id]; + if (control_tensor.allocation_type == kTfLiteMmapRo) { + auto* const_control_tensor_node = + graph_builder_->AddConstNodeWithData(tensor_id, control_tensor); + AddInput(TensorID(const_control_tensor_node->GetID(), 0)); + } else { + AddInput(graph_builder_->GetHexagonTensorId(tensor_id)); + } TF_LITE_ENSURE_STATUS(ComputeAndAddMinAndMax(context, input_tensor)); diff --git a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc index 3e852533394..d2620f71007 100644 --- a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc @@ -97,6 +97,8 @@ TfLiteStatus TransposeConv2dOpBuilder::PopulateSubGraph( filter_depth_size; GetDims(&filter_batch_size, &filter_height_size, &filter_width_size, &filter_depth_size, weights_tensor.dims); + weight_shape_ = {filter_batch_size, filter_height_size, filter_width_size, + filter_depth_size}; // Weights tensor could be int8 even for per-tensor quantization. // Therefore, we look at the number of scale values to check if it is // per-channel quantized. @@ -104,7 +106,25 @@ TfLiteStatus TransposeConv2dOpBuilder::PopulateSubGraph( reinterpret_cast( weights_tensor.quantization.params); const bool is_per_channel_quant = weights_quant_params->scale->size > 1; - AddInput(graph_builder_->GetHexagonTensorId(tensor_id)); + + OpBuilder* const_weights_node; + if (weights_tensor.type == kTfLiteInt8) { + std::vector weights_data(NumElements(&weights_tensor)); + const int8_t* original_data = weights_tensor.data.int8; + // Flip bits on the weight values so that the int8 values are treated + // as uint8. + for (int i = 0; i < NumElements(&weights_tensor); ++i) { + weights_data[i] = original_data[i] ^ k8BitSignFlipConstant; + } + const_weights_node = graph_builder_->AddConstNodeWithData( + weight_shape_.data(), reinterpret_cast(weights_data.data()), + weights_data.size() * sizeof(weights_data[0])); + } else { + const_weights_node = graph_builder_->AddConstNodeWithData( + weight_shape_.data(), weights_tensor.data.raw, weights_tensor.bytes); + } + graph_builder_->AddTensorWithID(tensor_id, const_weights_node->GetID(), 0); + AddInput(TensorID(const_weights_node->GetID(), 0)); // Handle weights quantization. float weights_min = 0; diff --git a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h index 4afab9894f0..0a6a90a0297 100644 --- a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h +++ b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h @@ -47,7 +47,7 @@ class TransposeConv2dOpBuilder : public OpBuilder { TensorID node_output_; std::vector transposed_weights_; std::vector stride_shape_; - std::vector bias_shape_; + std::vector weight_shape_, bias_shape_; std::vector bias_data_; // Non-null only if node has per-channel quantized weights/biases. diff --git a/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc b/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc index 83ebc15510e..cdf6b555929 100644 --- a/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc +++ b/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc @@ -264,9 +264,8 @@ TfLiteStatus HexagonDelegateKernel::BuildGraph( if (tensor_id == -1) continue; const auto& input_tensor = context->tensors[tensor_id]; if (input_tensor.allocation_type == kTfLiteMmapRo) { - builder_->AddConstNodeWithData( - tensor_id, input_tensor, - /*int8_to_uint8*/ (input_tensor.type == kTfLiteInt8)); + builder_->AddConstNodeWithData(tensor_id, input_tensor, + /*int8_to_uint8*/ true); } } auto* op_builder = From 702ca66743d70882099d6cc4ac1c81646763b8f7 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Fri, 7 Aug 2020 20:30:32 -0700 Subject: [PATCH 0695/1017] Add support for replicating functions with replicate variant ops in ReplicateToIslandPass. Certain ops, while stateful, should be rewritten with different attributes depending on which replica they are in. This extends to such ops via function calls. PiperOrigin-RevId: 325555999 Change-Id: I9806184cc3b4dae49b22061ba1ab680c02ad9ba1 --- .../tensorflow/tests/replicate_to_island.mlir | 185 ++++++++++++++- .../transforms/replicate_to_island.cc | 211 +++++++++++++++--- 2 files changed, 370 insertions(+), 26 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island.mlir b/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island.mlir index ddcfde5cbcd..487234ce958 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/replicate_to_island.mlir @@ -1,4 +1,4 @@ -// RUN: tf-opt %s -tf-replicate-to-island | FileCheck %s +// RUN: tf-opt -split-input-file %s -tf-replicate-to-island | FileCheck %s // Tests per replica island has same control operands as island holding // replicate. @@ -256,3 +256,186 @@ func @device_ordinals(%arg0: tensor, %arg1: tensor<2x!tf.string>) { // CHECK: "tf._XlaSendFromHost" // CHECK-SAME: device_ordinal = 2 // CHECK: "tf.NoOp" + +// ----- + +// Tests functions with replica variant ops reachable from a replicate region +// is cloned and remapped. + +// CHECK-LABEL: func @call_with_replicate_variant_ops +func @call_with_replicate_variant_ops(%arg0: tensor, %arg1: tensor<2x!tf.string>) { + tf_executor.graph { + tf_executor.island { + tf_device.replicate([%arg0, %arg0] as %arg2: tensor) {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2"]}} { + "tf.StatefulPartitionedCall"(%arg1) {config = "", config_proto = "", executor_type = "", f = @send_recv} : (tensor<2x!tf.string>) -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + return +} + +// CHECK: "tf.StatefulPartitionedCall" +// CHECK-SAME: f = [[CALL_REPLICA_0:@[a-z0-9_]+]] +// CHECK: "tf.StatefulPartitionedCall" +// CHECK-SAME: f = [[CALL_REPLICA_1:@[a-z0-9_]+]] + +func @send_recv(%arg0: tensor<2x!tf.string>) { + %0 = "tf._XlaRecvAtHost"(%arg0) {_xla_has_host_transfer = true, device_ordinal = 0 : i64, key = "host_compute_channel_send_0"} : (tensor<2x!tf.string>) -> tensor + "tf._XlaSendFromHost"(%0, %arg0) {_xla_has_host_transfer = true, device_ordinal = 0 : i64, key = "host_compute_channel_recv_0"} : (tensor, tensor<2x!tf.string>) -> () + "tf.NoOp"() : () -> () + return +} + +// CHECK: func [[CALL_REPLICA_0]] +// CHECK: "tf._XlaRecvAtHost" +// CHECK-SAME: device_ordinal = 1 +// CHECK: "tf._XlaSendFromHost" +// CHECK-SAME: device_ordinal = 1 + +// CHECK: func [[CALL_REPLICA_1]] +// CHECK: "tf._XlaRecvAtHost" +// CHECK-SAME: device_ordinal = 2 +// CHECK: "tf._XlaSendFromHost" +// CHECK-SAME: device_ordinal = 2 + +// ----- + +// Tests transitive functions with replica variant ops reachable from a +// replicate region is cloned and remapped. + +// CHECK-LABEL: func @call_with_replicate_variant_ops +func @call_with_replicate_variant_ops(%arg0: tensor, %arg1: tensor<2x!tf.string>) { + tf_executor.graph { + tf_executor.island { + tf_device.replicate([%arg0, %arg0] as %arg2: tensor) {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2"]}} { + "tf.StatefulPartitionedCall"(%arg1) {config = "", config_proto = "", executor_type = "", f = @callee} : (tensor<2x!tf.string>) -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + return +} + +// CHECK: "tf.StatefulPartitionedCall" +// CHECK-SAME: f = [[CALLEE_REPLICA_0:@[a-z0-9_]+]] +// CHECK: "tf.StatefulPartitionedCall" +// CHECK-SAME: f = [[CALLEE_REPLICA_1:@[a-z0-9_]+]] + +func @callee(%arg0: tensor<2x!tf.string>) { + "tf.StatefulPartitionedCall"(%arg0) {config = "", config_proto = "", executor_type = "", f = @send_recv} : (tensor<2x!tf.string>) -> () + return +} + +func @send_recv(%arg0: tensor<2x!tf.string>) { + %0 = "tf._XlaRecvAtHost"(%arg0) {_xla_has_host_transfer = true, device_ordinal = 0 : i64, key = "host_compute_channel_send_0"} : (tensor<2x!tf.string>) -> tensor + "tf._XlaSendFromHost"(%0, %arg0) {_xla_has_host_transfer = true, device_ordinal = 0 : i64, key = "host_compute_channel_recv_0"} : (tensor, tensor<2x!tf.string>) -> () + "tf.NoOp"() : () -> () + return +} + +// CHECK: func [[CALLEE_REPLICA_0]] +// CHECK: "tf.StatefulPartitionedCall" +// CHECK-SAME: f = [[TRANSITIVE_CALLEE_REPLICA_0:@[a-z0-9_]+]] + +// CHECK: func [[TRANSITIVE_CALLEE_REPLICA_0]] +// CHECK: "tf._XlaRecvAtHost" +// CHECK-SAME: device_ordinal = 1 +// CHECK: "tf._XlaSendFromHost" +// CHECK-SAME: device_ordinal = 1 + +// CHECK: func [[CALLEE_REPLICA_1]] +// CHECK: "tf.StatefulPartitionedCall" +// CHECK-SAME: f = [[TRANSITIVE_CALLEE_REPLICA_1:@[a-z0-9_]+]] + +// CHECK: func [[TRANSITIVE_CALLEE_REPLICA_1]] +// CHECK: "tf._XlaRecvAtHost" +// CHECK-SAME: device_ordinal = 2 +// CHECK: "tf._XlaSendFromHost" +// CHECK-SAME: device_ordinal = 2 + +// ----- + +// Tests functional control flow functions with replica variant ops reachable +// from a replicate region is cloned and remapped. Only the branches reachable +// with replica variant ops are cloned. + +// CHECK-LABEL: func @control_flow_with_replicate_variant_ops +func @control_flow_with_replicate_variant_ops(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor<2x!tf.string>) { + tf_executor.graph { + tf_executor.island { + tf_device.replicate([%arg0, %arg0] as %arg4: tensor, [%arg1, %arg1] as %arg5: tensor, [%arg2, %arg2] as %arg6: tensor) {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2"]}} { + %0 = "tf.If"(%arg4, %arg5, %arg6, %arg3) {else_branch = @cond_false, is_stateless = true, then_branch = @cond_true} : (tensor, tensor, tensor, tensor<2x!tf.string>) -> tensor + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + return +} + +// CHECK: "tf.If" +// CHECK-SAME: else_branch = @cond_false +// CHECK-SAME: then_branch = [[COND_TRUE_REPLICA_0:@[a-z0-9_]+]] +// CHECK: "tf.If" +// CHECK-SAME: else_branch = @cond_false +// CHECK-SAME: then_branch = [[COND_TRUE_REPLICA_1:@[a-z0-9_]+]] + +func @cond_false(%arg0: tensor, %arg1: tensor, %arg2: tensor<2x!tf.string>) -> tensor { + return %arg0 : tensor +} + +// CHECK-NOT: func @cond_false.+( + +func @cond_true(%arg0: tensor, %arg1: tensor, %arg2: tensor<2x!tf.string>) -> tensor { + "tf._XlaSendFromHost"(%arg1, %arg2) {_xla_has_host_transfer = true, device_ordinal = 0 : i64, key = "host_compute_channel_recv_0"} : (tensor, tensor<2x!tf.string>) -> () + %0 = "tf._XlaRecvAtHost"(%arg2) {_xla_has_host_transfer = true, device_ordinal = 0 : i64, key = "host_compute_channel_send_0"} : (tensor<2x!tf.string>) -> tensor + return %0 : tensor +} + +// CHECK: func [[COND_TRUE_REPLICA_0]] +// CHECK: "tf._XlaSendFromHost" +// CHECK-SAME: device_ordinal = 1 +// CHECK: "tf._XlaRecvAtHost" +// CHECK-SAME: device_ordinal = 1 + +// CHECK: func [[COND_TRUE_REPLICA_1]] +// CHECK: "tf._XlaSendFromHost" +// CHECK-SAME: device_ordinal = 2 +// CHECK: "tf._XlaRecvAtHost" +// CHECK-SAME: device_ordinal = 2 + +// ----- + +// Tests function with no replica variant ops reachable from a replicate region +// is not cloned. + +// CHECK-LABEL: func @no_replicate_variant_ops +func @no_replicate_variant_ops(%arg0: tensor, %arg1: tensor<2x!tf.string>) { + tf_executor.graph { + tf_executor.island { + tf_device.replicate([%arg0, %arg0] as %arg2: tensor) {n = 2 : i32, devices = {TPU_REPLICATED_CORE_0 = ["/job:worker/replica:0/task:0/device:TPU:1", "/job:worker/replica:0/task:0/device:TPU:2"]}} { + "tf.StatefulPartitionedCall"(%arg1) {config = "", config_proto = "", executor_type = "", f = @send_recv} : (tensor<2x!tf.string>) -> () + tf_device.return + } + tf_executor.yield + } + tf_executor.fetch + } + return +} + +// CHECK: "tf.StatefulPartitionedCall" +// CHECK-SAME: f = @send_recv + +func @send_recv(%arg0: tensor<2x!tf.string>) { + "tf.NoOp"() : () -> () + return +} + +// CHECK-NOT: @send_recv.+( diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc b/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc index e7f2977dbcd..ef75f90d5c1 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/replicate_to_island.cc @@ -20,6 +20,7 @@ limitations under the License. #include #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Sequence.h" @@ -32,6 +33,7 @@ limitations under the License. #include "mlir/IR/Builders.h" // from @llvm-project #include "mlir/IR/Diagnostics.h" // from @llvm-project #include "mlir/IR/Dialect.h" // from @llvm-project +#include "mlir/IR/SymbolTable.h" // from @llvm-project #include "mlir/IR/Visitors.h" // from @llvm-project #include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Support/LogicalResult.h" // from @llvm-project @@ -64,6 +66,151 @@ bool RequiresDeviceOrdinalAttribute(Operation* op) { llvm::isa(op); } +// Checks if a region contains ops that are replica variant. +bool HasReplicaVariantOps(Region& region, + const llvm::Optional& devices) { + auto result = region.walk([&](Operation* op) { + if (RequiresReplicaIDAttribute(op) || + (devices.hasValue() && RequiresDeviceOrdinalAttribute(op))) + return WalkResult::interrupt(); + + if (auto launch = dyn_cast(op)) + if (devices.hasValue() && devices.getValue().get(launch.device())) + return WalkResult::interrupt(); + + return WalkResult::advance(); + }); + return result.wasInterrupted(); +} + +// Collects all functions reachable from a region, including transitive ones. +llvm::SmallPtrSet GetReachableFunctionsFromRegion(ModuleOp module, + Region& region) { + llvm::SmallPtrSet visited_functions; + + SymbolTable symbol_table(module); + auto symbol_uses = symbol_table.getSymbolUses(®ion); + if (!symbol_uses) return {}; + + for (auto& use : *symbol_uses) + if (auto func = + symbol_table.lookup(use.getSymbolRef().getRootReference())) + visited_functions.insert(func); + + llvm::SmallVector functions_to_visit(visited_functions.begin(), + visited_functions.end()); + while (!functions_to_visit.empty()) { + llvm::SmallVector new_functions_to_visit; + + for (FuncOp function_to_visit : functions_to_visit) { + auto func_symbol_uses = + symbol_table.getSymbolUses(function_to_visit.getCallableRegion()); + if (!func_symbol_uses) continue; + + for (auto& use : *func_symbol_uses) + if (auto func = symbol_table.lookup( + use.getSymbolRef().getRootReference())) + if (visited_functions.insert(func).second) + new_functions_to_visit.push_back(func); + } + + functions_to_visit.swap(new_functions_to_visit); + } + + return visited_functions; +} + +// Collects all functions and transitive functions reachable from region that +// contain replicate variant ops. +llvm::SmallDenseMap GetReachableFunctionsToClone( + ModuleOp module, Region& region, + const llvm::Optional& devices) { + llvm::SmallPtrSet reachable_functions = + GetReachableFunctionsFromRegion(module, region); + + llvm::SmallDenseMap functions_to_clone; + llvm::SmallVector functions_to_visit; + for (FuncOp func : reachable_functions) { + if (!func.getCallableRegion()) continue; + if (HasReplicaVariantOps(*func.getCallableRegion(), devices)) { + functions_to_clone.insert({func.getName(), func}); + functions_to_visit.push_back(func); + } + } + + while (!functions_to_visit.empty()) { + llvm::SmallVector new_functions_to_visit; + + for (FuncOp func_to_visit : functions_to_visit) { + auto func_uses = func_to_visit.getSymbolUses(module); + if (!func_uses) continue; + for (auto use : *func_uses) { + auto parent_func = use.getUser()->getParentOfType(); + if (!parent_func || !reachable_functions.contains(parent_func) || + !functions_to_clone.insert({parent_func.getName(), parent_func}) + .second) + continue; + new_functions_to_visit.push_back(parent_func); + } + } + + functions_to_visit.swap(new_functions_to_visit); + } + + return functions_to_clone; +} + +struct FuncOldNameAndClone { + StringRef old_name; + FuncOp clone; +}; + +// Replaces all symbol uses with cloned functions, for `region` and across the +// cloned functions themselves. +LogicalResult UpdateSymbolUsesWithClones( + SymbolTable& symbol_table, ModuleOp module, Region& region, + llvm::MutableArrayRef cloned_functions) { + llvm::SmallVector, 4> old_to_new_names; + old_to_new_names.reserve(cloned_functions.size()); + for (auto& cloned_function : cloned_functions) + old_to_new_names.push_back( + {cloned_function.old_name, cloned_function.clone.getName()}); + + for (const auto& old_to_new_name : old_to_new_names) { + if (failed(symbol_table.replaceAllSymbolUses( + old_to_new_name.first, old_to_new_name.second, ®ion))) + return failure(); + + for (auto& cloned_function : cloned_functions) + if (failed(symbol_table.replaceAllSymbolUses( + old_to_new_name.first, old_to_new_name.second, + cloned_function.clone.getCallableRegion()))) + return failure(); + } + return success(); +} + +// Collects TPU device ordinal for outside compilation communication ops. This +// currently assumes outside compilation only uses `TPU_REPLICATED_CORE_0` +// aliased device for the device computation. +llvm::Optional GetDeviceOrdinal( + const llvm::Optional& devices, Location loc, + unsigned replica_id) { + int64_t device_ordinal = 0; + if (devices.hasValue()) { + if (auto tpu_replica_0 = devices.getValue().get("TPU_REPLICATED_CORE_0")) { + llvm::StringRef tpu_device = tpu_replica_0.cast()[replica_id] + .cast() + .getValue(); + if (succeeded(tensorflow::GetDeviceOrdinalFromDeviceString( + loc, tpu_device, &device_ordinal))) { + return llvm::Optional(device_ordinal); + } + } + } + return llvm::None; +} + // Updates replica variant ops in a region based on replica `replica_id`. // TODO(b/157624749): Replace this with better abstraction to differentiate ops // for different replicas. Some ops, such as XlaHostCompute op or TPU Embedding @@ -72,27 +219,17 @@ bool RequiresDeviceOrdinalAttribute(Operation* op) { // represents replica id. LogicalResult UpdateRegionReplicateVariantOps( OpBuilder& builder, Location loc, Region& region, int replica_id, + llvm::MutableArrayRef cloned_functions, const llvm::Optional& devices) { - int64_t device_ordinal = -1; - const bool has_devices = devices.hasValue(); - if (has_devices) { - if (auto tpu_replica_0 = devices.getValue().get("TPU_REPLICATED_CORE_0")) { - llvm::StringRef tpu_device = tpu_replica_0.cast()[replica_id] - .cast() - .getValue(); - if (failed(tensorflow::GetDeviceOrdinalFromDeviceString( - loc, tpu_device, &device_ordinal))) { - return failure(); - } - } - } + llvm::Optional device_ordinal = + GetDeviceOrdinal(devices, loc, replica_id); - region.walk([&](Operation* op) { + auto update_replicate_variant_ops = [&](Operation* op) { // Add replica id. if (RequiresReplicaIDAttribute(op)) op->setAttr(kReplicaIdAttr, builder.getI32IntegerAttr(replica_id)); - if (!has_devices) return; + if (!devices.hasValue()) return; // Map aliased devices to explicit devices based on replica. if (auto launch = dyn_cast(op)) @@ -102,10 +239,15 @@ LogicalResult UpdateRegionReplicateVariantOps( device_by_replica.cast()[replica_id].cast()); // Add device ordinal. - if (device_ordinal >= 0 && RequiresDeviceOrdinalAttribute(op)) + if (device_ordinal && RequiresDeviceOrdinalAttribute(op)) op->setAttr(kDeviceOrdinalAttr, - builder.getI64IntegerAttr(device_ordinal)); - }); + builder.getI64IntegerAttr(*device_ordinal)); + }; + + region.walk(update_replicate_variant_ops); + for (auto& cloned_function : cloned_functions) + cloned_function.clone.getCallableRegion()->walk( + update_replicate_variant_ops); return success(); } @@ -115,7 +257,7 @@ LogicalResult UpdateRegionReplicateVariantOps( // `tf_device.replicate`, the device will be remapped to an explicit device // for the associated replica island. LogicalResult ExpandReplicateIntoReplicas( - const Dialect* tf_dialect, OpBuilder& builder, + const Dialect* tf_dialect, OpBuilder& builder, ModuleOp module, tf_executor::IslandOp island_op, tf_device::ReplicateOp replicate_op, int num_replicas, llvm::SmallVectorImpl& replicas) { replicas.reserve(num_replicas); @@ -133,9 +275,23 @@ LogicalResult ExpandReplicateIntoReplicas( terminator.getOperands()); terminator.erase(); + auto funcs_to_clone = + GetReachableFunctionsToClone(module, replicate_op.body(), devices); + SymbolTable symbol_table(module); + builder.setInsertionPoint(island_op); BlockAndValueMapping mapping; for (int i : llvm::seq(0, num_replicas)) { + // Clone reachable functions with replica variant ops. + llvm::SmallVector cloned_functions; + cloned_functions.reserve(funcs_to_clone.size()); + for (auto& func_to_clone : funcs_to_clone) { + auto cloned_function = func_to_clone.getSecond().clone(); + symbol_table.insert(cloned_function, module.end()); + cloned_functions.push_back( + {func_to_clone.getSecond().getName(), cloned_function}); + } + // Create new island for replica. auto replica = builder.create( island_op.getLoc(), output_types, control_type, replica_inputs); @@ -149,9 +305,13 @@ LogicalResult ExpandReplicateIntoReplicas( // Copy over replicate region into replica island. replicate_op.body().cloneInto(&replica.body(), mapping); - if (failed(UpdateRegionReplicateVariantOps(builder, replicate_op.getLoc(), - replica.body(), /*replica_id=*/i, - devices))) + if (failed(UpdateSymbolUsesWithClones(symbol_table, module, replica.body(), + cloned_functions))) + return failure(); + + if (failed(UpdateRegionReplicateVariantOps( + builder, replicate_op.getLoc(), replica.body(), + /*replica_id=*/i, cloned_functions, devices))) return failure(); replicas.push_back(replica); @@ -211,6 +371,7 @@ LogicalResult ExpandReplicateIntoReplicas( // tf_executor.yield %a1, %b1 : tensor, tensor // } LogicalResult CreateIslandsFromReplicate(const Dialect* tf_dialect, + ModuleOp module, tf_executor::GraphOp graph_op, tf_executor::IslandOp island_op, tf_device::ReplicateOp replicate_op) { @@ -219,7 +380,7 @@ LogicalResult CreateIslandsFromReplicate(const Dialect* tf_dialect, // Create islands per replica. llvm::SmallVector replicas; - if (failed(ExpandReplicateIntoReplicas(tf_dialect, builder, island_op, + if (failed(ExpandReplicateIntoReplicas(tf_dialect, builder, module, island_op, replicate_op, num_replicas, replicas))) return failure(); @@ -299,8 +460,8 @@ void ReplicateToIslandPass::runOnOperation() { auto graph_op = island_op.getParentOfType(); auto replicate_op = cast(island_op.GetBody().front()); - if (failed(CreateIslandsFromReplicate(tf_dialect, graph_op, island_op, - replicate_op))) + if (failed(CreateIslandsFromReplicate(tf_dialect, module, graph_op, + island_op, replicate_op))) return signalPassFailure(); } } From a144b1a24e06a173d700dcfe3d12f804f14acb75 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Fri, 7 Aug 2020 21:33:04 -0700 Subject: [PATCH 0696/1017] Update mhlo.constant to use a custom assembly format instead of a custom printer and parser (NFC). PiperOrigin-RevId: 325560779 Change-Id: I8ce3350d6af6986cdcb7e4e1ee308b1095fb120b --- .../mlir-hlo/Dialect/mhlo/IR/hlo_ops.td | 3 +- .../mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc | 31 ------------------- tensorflow/compiler/mlir/hlo/tests/ops.mlir | 10 +++--- 3 files changed, 6 insertions(+), 38 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td index 4c09c209bd1..b8b1926a0c9 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td @@ -67,8 +67,7 @@ def HLO_ConstOp : HLO_Op<"constant", "OpBuilder &builder, OperationState &result, Attribute value" >]; - let printer = [{ return Print(*this, &p); }]; - let parser = [{ return ParseConstOp(&parser, &result); }]; + let assemblyFormat = "attr-dict $value"; let hasFolder = 1; diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc index 6f453d1a167..eda10b0f187 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc @@ -112,37 +112,6 @@ DenseIntElementsAttr BuildSliceLimits(DenseIntElementsAttr start_indices, // ConstOp //===----------------------------------------------------------------------===// -static void Print(ConstOp op, OpAsmPrinter* printer) { - // Print op name. - *printer << op.getOperationName(); - - // Elide attribute value while printing the attribute dictionary. - SmallVector elided_attrs; - elided_attrs.push_back("value"); - printer->printOptionalAttrDict(op.getAttrs(), elided_attrs); - - *printer << ' ' << op.value(); -} - -static ParseResult ParseConstOp(OpAsmParser* parser, OperationState* result) { - if (parser->parseOptionalAttrDict(result->attributes)) return failure(); - - // If colon is not present after attribute dictionary, it should be short form - // and attribute 'value' is outside the dictionary. - if (failed(parser->parseOptionalColon())) { - Attribute value; - if (parser->parseAttribute(value, "value", result->attributes)) - return failure(); - return parser->addTypeToList(value.getType(), result->types); - } - - // Long form should have type of the result after colon. - Type ty; - if (parser->parseType(ty)) return failure(); - result->types.push_back(ty); - return success(); -} - OpFoldResult ConstOp::fold(ArrayRef operands) { assert(operands.empty() && "constant has no operands"); diff --git a/tensorflow/compiler/mlir/hlo/tests/ops.mlir b/tensorflow/compiler/mlir/hlo/tests/ops.mlir index 25c7d6aee61..a8f16c403ae 100644 --- a/tensorflow/compiler/mlir/hlo/tests/ops.mlir +++ b/tensorflow/compiler/mlir/hlo/tests/ops.mlir @@ -480,7 +480,7 @@ func @map_non_scalar_computation_operand(%arg0: tensor<4x5xf32>, %arg1: tensor<4 // expected-error@+1 {{computation arguments must be 0-rank tensor, but got: arg #1 of type 'tensor<5xf32>'}} %0 = "mhlo.map"(%arg0, %arg1) ( { ^bb0(%arg2: tensor, %arg3: tensor<5xf32>): - %1 = mhlo.constant {value = dense<2.0> : tensor} : tensor + %1 = mhlo.constant dense<2.0> : tensor "mhlo.return"(%1) : (tensor) -> () }) {dimensions = dense<[0, 1]> : tensor<2xi64>} : (tensor<4x5xf32>, tensor<4x5xf32>) -> tensor<4x5xf32> return %0 : tensor<4x5xf32> @@ -492,7 +492,7 @@ func @map_mismatch_operand_and_computation_args(%arg0: tensor<4x5xf32>, %arg1: t // expected-error@+1 {{element type of operands and computation arguments must match, but got: 'f32' and 'i32'}} %0 = "mhlo.map"(%arg0, %arg1) ( { ^bb0(%arg2: tensor, %arg3: tensor): - %1 = mhlo.constant {value = dense<2.0> : tensor} : tensor + %1 = mhlo.constant dense<2.0> : tensor "mhlo.return"(%1) : (tensor) -> () }) {dimensions = dense<[0, 1]> : tensor<2xi64>} : (tensor<4x5xf32>, tensor<4x5xf32>) -> tensor<4x5xf32> return %0 : tensor<4x5xf32> @@ -504,7 +504,7 @@ func @map_invalid_number_of_computation_output(%arg0: tensor<4x5xf32>, %arg1: te // expected-error@+1 {{computation must return single output, but got: 0}} %0 = "mhlo.map"(%arg0, %arg1) ( { ^bb0(%arg2: tensor, %arg3: tensor): - %1 = mhlo.constant {value = dense<2.0> : tensor} : tensor + %1 = mhlo.constant dense<2.0> : tensor "mhlo.return"() : () -> () }) {dimensions = dense<[0, 1]> : tensor<2xi64>} : (tensor<4x5xf32>, tensor<4x5xf32>) -> tensor<4x5xf32> return %0 : tensor<4x5xf32> @@ -516,7 +516,7 @@ func @main_non_scalar_computation_output(%arg0: tensor<4x5xf32>, %arg1: tensor<4 // expected-error@+1 {{computation must return 0-rank tensor, but got: 'tensor<5xf32>'}} %0 = "mhlo.map"(%arg0, %arg1) ( { ^bb0(%arg2: tensor, %arg3: tensor): - %1 = mhlo.constant {value = dense<2.0> : tensor} : tensor<5xf32> + %1 = mhlo.constant dense<2.0> : tensor<5xf32> "mhlo.return"(%1) : (tensor<5xf32>) -> () }) {dimensions = dense<[0, 1]> : tensor<2xi64>} : (tensor<4x5xf32>, tensor<4x5xf32>) -> tensor<4x5xf32> return %0 : tensor<4x5xf32> @@ -528,7 +528,7 @@ func @mismatch_computation_output_type(%arg0: tensor<4x5xf32>, %arg1: tensor<4x5 // expected-error@+1 {{element type of result and computation output must match, but got: 'f32' and 'i32'}} %0 = "mhlo.map"(%arg0, %arg1) ( { ^bb0(%arg2: tensor, %arg3: tensor): - %1 = mhlo.constant {value = dense<2> : tensor} : tensor + %1 = mhlo.constant dense<2> : tensor "mhlo.return"(%1) : (tensor) -> () }) {dimensions = dense<[0, 1]> : tensor<2xi64>} : (tensor<4x5xf32>, tensor<4x5xf32>) -> tensor<4x5xf32> return %0 : tensor<4x5xf32> From e7c59b4c5164552386f208de8766b31c787ff4f9 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Fri, 7 Aug 2020 21:46:10 -0700 Subject: [PATCH 0697/1017] Early terminate LaunchToDeviceAttributePass on missing 'tf' dialect registration (NFC). PiperOrigin-RevId: 325561680 Change-Id: Iafeee051275632bdfa7890ca44239e277d55feef --- .../mlir/tensorflow/transforms/launch_to_device_attribute.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/launch_to_device_attribute.cc b/tensorflow/compiler/mlir/tensorflow/transforms/launch_to_device_attribute.cc index bce18c0b4b7..9f67a3e7e71 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/launch_to_device_attribute.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/launch_to_device_attribute.cc @@ -106,8 +106,8 @@ LogicalResult HoistOpsAndAnnotateWithDevice(const Dialect* tf_dialect, void LaunchToDeviceAttributePass::runOnFunction() { const Dialect* tf_dialect = getContext().getRegisteredDialect("tf"); if (!tf_dialect) { - signalPassFailure(); getFunction().emitError() << "'tf' dialect is not registered"; + return signalPassFailure(); } auto result = getFunction().walk([&](tf_device::LaunchOp launch) { From 78686f49f50ec0bc69a864911c6e33adca56937e Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Fri, 7 Aug 2020 21:46:23 -0700 Subject: [PATCH 0698/1017] Update tf_device.return to use assembly format instead of a custom parser and printer (NFC). PiperOrigin-RevId: 325561693 Change-Id: Ie13a8006580c4477a793e745b7e71e06bb19bd1f --- .../compiler/mlir/tensorflow/ir/tf_device.cc | 25 ------------------- .../mlir/tensorflow/ir/tf_device_ops.td | 3 +-- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc index 77008b55672..9aa0a72f475 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc @@ -118,31 +118,6 @@ TensorFlowDeviceDialect::TensorFlowDeviceDialect(MLIRContext* context) // operation results are perfectly forwarded to the launch return. bool LaunchOp::WrapsSingleOp() { return BlockWrapsSingleOp(&GetBody()); } -//===----------------------------------------------------------------------===// -// tf_device.return -//===----------------------------------------------------------------------===// - -namespace { -ParseResult ParseReturnOp(OpAsmParser* parser, OperationState* state) { - llvm::SmallVector op_info; - llvm::SmallVector types; - llvm::SMLoc loc = parser->getCurrentLocation(); - return failure(parser->parseOperandList(op_info) || - (!op_info.empty() && parser->parseColonTypeList(types)) || - parser->resolveOperands(op_info, types, loc, state->operands)); -} - -void Print(ReturnOp op, OpAsmPrinter* p) { - *p << op.getOperationName(); - if (op.getNumOperands() > 0) { - *p << ' '; - p->printOperands(op.getOperands()); - *p << " : "; - interleaveComma(op.getOperandTypes(), *p); - } -} -} // anonymous namespace - //===----------------------------------------------------------------------===// // tf_device.parallel_execute //===----------------------------------------------------------------------===// diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_device_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_device_ops.td index 565be63a74f..d94a37d9b02 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_device_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_device_ops.td @@ -104,8 +104,7 @@ The `tf_device.return` operation terminates and returns values from a }]> ]; - let parser = [{ return Parse$cppClass(&parser, &result); }]; - let printer = [{ return Print(*this, &p); }]; + let assemblyFormat = "attr-dict ($results^ `:` type($results))?"; } def TfDevice_LaunchFuncOp : TfDevice_Op<"launch_func", []> { From 3be8012f0cb2003db4c28b33cf9078010ace8cf8 Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Fri, 7 Aug 2020 22:20:35 -0700 Subject: [PATCH 0699/1017] Disable shape inference while importing Graph to MLIR in the bridge Currently the shape inference causes a hard error during inference for ops that requires some of the inputs to be ranked. For example, SpaceToBatchND requires block_size to be of rank one. This mode is slated to be the default mode and we run shape inference pass in the bridge early in the pipeline so we don't need to infer shapes during the import. PiperOrigin-RevId: 325565060 Change-Id: Ic8933afd406757d5ca9bb0a1174101f38483f368 --- tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc | 6 ++++++ .../compiler/mlir/tensorflow/utils/compile_mlir_util.cc | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc b/tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc index 8e6d9042987..8be6facce38 100644 --- a/tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc +++ b/tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc @@ -132,6 +132,12 @@ Status MlirFunctionOptimizationPass::Run( import_config.graph_as_function = true; import_config.control_outputs = *control_ret_node_names; import_config.upgrade_legacy = true; + // Disable shape inference during import as some TensorFlow op fails during + // shape inference with dynamic shaped operands. This in turn causes the + // import to fail. Shape inference during import is going to be removed and + // the shape inference pass is run early in the pass pipeline, shape inference + // during import is not necessary. + import_config.enable_shape_inference = false; TF_ASSIGN_OR_RETURN(auto module_ref, ConvertGraphToMlir(**graph, debug_info, *flib_def, import_config, &context)); diff --git a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc index 78d621cbe75..eee2f0a560c 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc @@ -510,6 +510,12 @@ Status CompileGraphToXlaHlo( mlir::MLIRContext context; GraphImportConfig config; config.graph_as_function = true; + // Disable shape inference during import as some TensorFlow op fails during + // shape inference with dynamic shaped operands. This in turn causes the + // import to fail. Shape inference during import is going to be removed and + // the shape inference pass is run early in the pass pipeline, shape inference + // during import is not necessary. + config.enable_shape_inference = false; auto module_or = ConvertGraphToMlir(graph, debug_info, flib_def, config, &context); if (!module_or.ok()) return module_or.status(); From fb9db5d28cc97bb2c05305b671f2e5fe12a251e2 Mon Sep 17 00:00:00 2001 From: Jan Pfeiffer Date: Sat, 8 Aug 2020 01:24:44 -0700 Subject: [PATCH 0700/1017] Adds `with_updates` method to `StructuredTensor`. PiperOrigin-RevId: 325577753 Change-Id: Ie7c462c2ed468c54a68e43905da4885be1041267 --- .../ops/structured/structured_tensor.py | 267 +++++++++++++++++- .../ops/structured/structured_tensor_test.py | 175 +++++++++++- 2 files changed, 435 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/ops/structured/structured_tensor.py b/tensorflow/python/ops/structured/structured_tensor.py index 3c3bd03a06b..c09a38f1d21 100644 --- a/tensorflow/python/ops/structured/structured_tensor.py +++ b/tensorflow/python/ops/structured/structured_tensor.py @@ -1,3 +1,4 @@ +# Lint as python3 # Copyright 2019 The TensorFlow Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +20,7 @@ from __future__ import division from __future__ import print_function import re +from typing import Callable, Dict, List, Sequence, Tuple, Union import numpy as np @@ -85,6 +87,23 @@ class StructuredTensor(composite_tensor.CompositeTensor): field. """ + #============================================================================= + # Common Types + #============================================================================= + # pylint: disable=invalid-name + # Field names work as key, and they can be a sequence to refer to the + # sub-levels (embedded) StructuredTensor's. + FieldName = Union[str, Sequence[str]] + + # Each field may contain one of the following types of Tensors. + FieldValue = Union[ops.Tensor, ragged_tensor.RaggedTensor, 'StructuredTensor'] + + # Function that takes a FieldValue as input and returns the transformed + # FieldValue. + FieldFn = Callable[[FieldValue], FieldValue] + + # pylint: enable=invalid-name + #============================================================================= # Constructor & Factory Methods #============================================================================= @@ -252,6 +271,180 @@ class StructuredTensor(composite_tensor.CompositeTensor): row_partitions, internal=_structured_tensor_factory_key) + def with_updates(self, + updates: Dict[FieldName, Union[FieldValue, FieldFn, None]], + validate: bool = False) -> 'StructuredTensor': # pylint: disable=bad-whitespace + """Creates a new `StructuredTensor` with the updated fields. + + If this `StructuredTensor` is a scalar, and `k` is the `FieldName` being + updated and `v` the new value, then: + + ``` + result[k] = v # If (k, v) is in updates and v is a FieldValue + result[k] = f(self[k]) # If (k, f) is in updates and f is a FieldFn + result[k] = self[k] # If k is in self.field_names but not in updates + ``` + + If this `StructuredTensor` has rank `N` and shape `[D1...DN]`, then each + FieldValue `v` in `updates` must have shape `[D1...DN, ...]`, that is, + prefixed with the same shape as the `StructuredTensor`. Then the resulting + `StructuredTensor` will have: + + ``` + result[i1...iN][k] = v[i1...iN] # (k, v) in updates + result[i1...iN][k] = f(self.field_value(k))[i1...iN] # (k, f) in updates + result[i1...iN][k] = self[i1...iN][k] # k not in updates + ``` + + Note that `result.shape` is always equal to `self.shape` (but the shapes + of nested StructuredTensors may be changed if they are updated with new + values). + + Args: + updates: A dictionary mapping `FieldName` to either a `FieldValue` to be + used to update, or a `FieldFn` that will transform the value for the + given `FieldName`. `FieldName` can be a string for a direct field, or a + sequence of strings to refer to a nested sub-field. `FieldFn` is a + function that takes a `FieldValue` as input and should return a + `FieldValue`. All other fields are copied over to the new + `StructuredTensor`. New `FieldName` can be given (to add new fields), + but only to existing `StructuredTensor`, it won't automatically create + new nested structures -- but one can create a whole `StructureTensor` + sub-structure and set that into an existing structure. If the new value + is set to `None`, it is removed. + validate: If true, then add runtime validation ops that check that the + field values all have compatible shapes in the outer `shape.rank` + dimensions. + + Returns: + A `StructuredTensor`. + + Raises: + `ValueError`: If the any of the `FieldName` keys points to non-existent + sub-structures, if parent and child nodes are updated, if shapes + change, if a delete update is given for a non-existant field, or if a + `FieldFn` transforming function is given for a `FieldName` that doesn't + yet exist. + + Examples: + + >>> shoes_us = StructuredTensor.from_pyval([ + ... {"age": 12, "nicknames": ["Josaphine"], + ... "shoes": {"sizes": [8.0, 7.5, 7.5]}}, + ... {"age": 82, "nicknames": ["Bob", "Bobby"], + ... "shoes": {"sizes": [11.0, 11.5, 12.0]}}, + ... {"age": 42, "nicknames": ["Elmo"], + ... "shoes": {"sizes": [9.0, 9.5, 10.0]}}]) + >>> def us_to_europe(t): + ... return tf.round(t * 2.54 + 17.0) # Rough approximation. + >>> shoe_sizes_key = ("shoes", "sizes") + >>> shoes_eu = shoes_us.with_updates({shoe_sizes_key: us_to_europe}) + >>> shoes_eu.field_value(shoe_sizes_key) + + """ + updates_items = [(_normalize_field_name_to_tuple(name), value) + for name, value in updates.items()] + + # Sort by keys and check for updates of both parent and child nodes. + updates_items = sorted(updates_items) + for i in range(1, len(updates_items)): + # Parent of a node would precede node in the sorted order. + name = updates_items[i][0] # item[0] is the name, item[1] is the value. + prev_name = updates_items[i - 1][0] + if name[:len(prev_name)] == prev_name: + raise ValueError( + '`StructuredTensor.with_updates` does not allow both parent and ' + 'child nodes to be updated: parent={}, child={}. If needed you can ' + 'update child nodes in the parent update value.'.format( + prev_name, name)) + return self._with_updates_impl((), updates_items, validate) + + def _with_updates_impl(self, error_prefix: Tuple[str], # pylint: disable=invalid-sequence-index + updates: List[Tuple[FieldName, Union[FieldValue, # pylint: disable=invalid-sequence-index + FieldFn]]], + validate: bool) -> 'StructuredTensor': + """Recursive part of `with_updates` implementation.""" + # Get current fields. + new_fields = dict(self._fields) + + # Convert field name to string with full path for error messages. + def name_fullpath(name: Sequence[str]) -> str: + return str(error_prefix + (name,)) + + # Apply value if a function or the value itself. + def apply_value(name: str, value: Union['FieldValue', + 'FieldFn']) -> 'FieldValue': + if callable(value): + # `value` is actually a transforming function. + if name not in new_fields: + raise ValueError( + '`StructuredTensor.with_updates` cannot update the field {} ' + 'because a transforming function was given, but that field ' + 'does not already exist.'.format(name_fullpath(name))) + value = value(new_fields[name]) + return value + + # Merge updates. + for name, value in updates: + if not name or not name[0]: + raise ValueError( + '`StructuredTensor.with_updates` does not allow empty names ' + '{}.'.format(name_fullpath(name))) + + if len(name) == 1: + name = name[0] + if value is None: + if name not in new_fields: + raise ValueError( + '`StructuredTensor.with_updates` cannot delete field ' + '{} because it is not present.'.format(name_fullpath(name))) + new_fields.pop(name) + else: + new_fields[name] = apply_value(name, value) + else: + # Recursive + prefix = name[0] + suffix = name[1:] + if prefix not in new_fields: + raise ValueError( + '`StructuredTensor.with_updates` cannot create new sub-field ' + '{} if parent field {} is not set.'.format( + error_prefix + tuple(name), name_fullpath(prefix))) + current_value = new_fields[prefix] + if not isinstance(current_value, StructuredTensor): + raise ValueError( + '`StructuredTensor.with_updates` cannot create new sub-field ' + '{} if parent structure {} is not a `StructuredTensor` that ' + 'can contain sub-structures -- it is a `{}`.'.format( + error_prefix + tuple(name), name_fullpath(prefix), + type(current_value))) + one_update = [(suffix, value)] + + # Accessing protected member in recursion. + # FutureWork: optimize by aggregating the recursions, instead of + # calling one at a time. + # pylint: disable=protected-access + value = current_value._with_updates_impl(error_prefix + (prefix,), + one_update, validate) + # pylint: enable=protected-access + new_fields[prefix] = value + + # TODO(edloper): When validate=True, only validate the modified fields. + try: + return StructuredTensor.from_fields( + new_fields, + shape=self.shape, + row_partitions=self._row_partitions, + nrows=self._nrows, + validate=validate) + + except ValueError as e: + msg = '`StructuredTensor.with_updates` failed' + if error_prefix: + msg = '{} for field {}'.format(msg, error_prefix) + raise ValueError('{}: {}'.format(msg, e)) + #============================================================================= # Properties #============================================================================= @@ -279,22 +472,74 @@ class StructuredTensor(composite_tensor.CompositeTensor): def row_partitions(self): """A tuple of `RowPartition`s defining the shape of this `StructuredTensor`. - If this `StructuredTensor` has a ragged shape, then all fields will be - encoded as either `RaggedTensor`s or `StructuredTensor`s with these - `RowPartition`s used to define their outermost `self.rank` dimensions. + When `self.rank <= 1`, this tuple will be empty. - If this `StructuredTensor` has a uniform (non-ragged) shape, then these - row partitions will all be defined using `uniform_row_length`. + When `self.rank > 1`, these `RowPartitions` define the shape of the + `StructuredTensor` by describing how a flat (1D) list of structures can be + repeatedly partitioned to form a higher-dimensional object. In particular, + the flat list is first partitioned into sublists using `row_partitions[-1]`, + and then those sublists are further partitioned using `row_partitions[-2]`, + etc. The following examples show the row partitions used to describe + several different `StructuredTensor`, each of which contains 8 copies of + the same structure (`x`): + + >>> x = {'a': 1, 'b': ['foo', 'bar', 'baz']} # shape = [] (scalar) + + >>> s1 = [[x, x, x, x], [x, x, x, x]] # shape = [2, 4] + >>> StructuredTensor.from_pyval(s1).row_partitions + (tf.RowPartition(row_splits=tf.Tensor([0 4 8], shape=(3,), + dtype=int64)),) + + >>> s2 = [[x, x], [x, x], [x, x], [x, x]] # shape = [4, 2] + >>> StructuredTensor.from_pyval(s2).row_partitions + (tf.RowPartition(row_splits=tf.Tensor([0 2 4 6 8], shape=(5,), + dtype=int64)),) + + >>> s3 = [[x, x, x], [], [x, x, x, x], [x]] # shape = [2, None] + >>> StructuredTensor.from_pyval(s3).row_partitions + (tf.RowPartition(row_splits=tf.Tensor([0 3 3 7 8], shape=(5,), + dtype=int64)),) + + >>> s4 = [[[x, x], [x, x]], [[x, x], [x, x]]] # shape = [2, 2, 2] + >>> StructuredTensor.from_pyval(s4).row_partitions + (tf.RowPartition(row_splits=tf.Tensor([0 2 4], shape=(3,), dtype=int64)), + tf.RowPartition(row_splits=tf.Tensor([0 2 4 6 8], shape=(5,), + dtype=int64))) + + + >>> s5 = [[[x, x], [x]], [[x, x]], [[x, x], [x]]] # shape = [3, None, None] + >>> StructuredTensor.from_pyval(s5).row_partitions + (tf.RowPartition(row_splits=tf.Tensor([0 2 3 5], shape=(4,), dtype=int64)), + tf.RowPartition(row_splits=tf.Tensor([0 2 3 5 7 8], shape=(6,), + dtype=int64))) + + Note that shapes for nested fields (such as `x['b']` in the above example) + are not considered part of the shape of a `StructuredTensor`, and are not + included in `row_partitions`. + + If this `StructuredTensor` has a ragged shape (i.e., if any of the + `row_partitions` is not uniform in size), then all fields will be encoded + as either `RaggedTensor`s or `StructuredTensor`s with these `RowPartition`s + used to define their outermost `self.rank` dimensions. Returns: A `tuple` of `RowPartition` objects with length `self.rank - 1` - (or `0` if `self.rank < 2`). + (or `0` if `self.rank < 2`) + """ return self._row_partitions def nrows(self): """The number of rows in this StructuredTensor (if rank>0). + This means the length of the outer-most dimension of the StructuredTensor. + + Notice that if `self.rank > 1`, then this equals the number of rows + of the first row partition. That is, + `self.nrows() == self.row_partitions[0].nrows()`. + + Otherwise `self.nrows()` will be the first dimension of the field values. + Returns: A scalar integer `Tensor` (or `None` if `self.rank == 0`). """ @@ -1175,3 +1420,13 @@ def _merge_dims(value, outer_axis, inner_axis): _structured_tensor_factory_key = object() # unique private object + + +def _normalize_field_name_to_tuple(name: 'FieldName') -> Sequence[str]: + """FieldName can be given also as string, this normalizes it to a tuple.""" + if isinstance(name, str): + return (name,) + if isinstance(name, list): + return tuple(name) + assert isinstance(name, tuple) + return name diff --git a/tensorflow/python/ops/structured/structured_tensor_test.py b/tensorflow/python/ops/structured/structured_tensor_test.py index 75aa5a872a6..f4218042cc2 100644 --- a/tensorflow/python/ops/structured/structured_tensor_test.py +++ b/tensorflow/python/ops/structured/structured_tensor_test.py @@ -924,7 +924,7 @@ class StructuredTensorTest(test_util.TensorFlowTestCase, st = StructuredTensor.from_pyval({"a": 5, "b": {"c": [1, 2, 3]}}) self.assertAllEqual(st.field_value(("a",)), 5) self.assertAllEqual(st.field_value(("b", "c")), [1, 2, 3]) - expected = "Field path \(.*a.*,.*b.*\) not found in .*" + expected = r"Field path \(.*a.*,.*b.*\) not found in .*" with self.assertRaisesRegex(KeyError, expected): st.field_value(("a", "b")) @@ -961,6 +961,179 @@ class StructuredTensorTest(test_util.TensorFlowTestCase, r = result.field_value("r") self.assertAllEqual(r, [[[1, 2], [3, 4]]]) + @parameterized.parameters([ + # Simple example. + ( + {"a": 12, "b": 23}, + {"a": 7}, + ), + # New field. + ( + {"a": 12}, + {("b",): 13}, + ), + # Nested example. + ( + {"a": 12, "b": {"c": 23}}, + {("b", "c"): 7}, + ), + # Multipe updates. + ( + {"a": 12, "b": {"c": 23}}, + {"a": 3, ("b", "c"): 7}, + ), + # Deep updates. + ( + {"a": 12, "b": {"c": 23, "d": {"e": 11}}}, + {("b", "c"): 7, ("b", "d", "e"): 13}, + ), + # Multiple updates to the same substructure. + ( + {"a": 12, "b": {"c": 23, "d": {"e": 11}}}, + {("b", "c"): 7, ("b", "f"): 13}, + ), + # Scalar to non-scalar elements. Shape remains unchanged. + ( + {"a": 5}, + {"a": ragged_factory_ops.constant_value([[51, 52], [61, 62, 63]])}, + ), + # Non-scalar element to scalar. + ( + {"c": {"a": [5, 3], "b": 2}}, + {("c", "a"): 5}, + ), + # Rank-1 StructuredTensor: shape is preserved and an item is added. + ( + [{"a": 5}, {"a": 6}], + {"a": [15, 16], "b": np.array([0.9, 1.1])}, + ), + # Non-scalar ragged elements, within a rank-2 StructuredTensor: elements + # rows (inner dimensions) are changed, but StructuredTensor shape + # (outer dimensions) are preserved. + ( + [[{"a": [5]}], [{"a": [3, 4]}, {"a": [8]}]], + {"a": ragged_factory_ops.constant_value([[[50, 60]], [[30], []]])}, + ), + ]) # pyformat: disable + def testWithUpdatesValues(self, pyval, updates): + st = StructuredTensor.from_pyval(pyval) + updated_st = st.with_updates(updates, validate=False) + for key, value in updates.items(): + got = updated_st.field_value(key) + self.assertAllEqual( + value, got, "Update failed: key={}, value={}, got={}".format( + key, value, got)) + + def testWithUpdatesFunctions(self): + pyval = {"a": 12, "b": {"c": 23, "d": {"e": 11}}} + st = StructuredTensor.from_pyval(pyval) + st_updated = st.with_updates( + { + "a": lambda x: x + 1, + ("b", "d", "e"): lambda x: x + 7 + }, validate=True) + # Updated values. + self.assertAllEqual(st_updated.field_value("a"), 13) + self.assertAllEqual(st_updated.field_value(("b", "d", "e")), 18) + # Unchanged value. + self.assertAllEqual(st_updated.field_value(("b", "c")), 23) + + def testWithUpdatesChecks(self): + pyval = {"a": 12, "b": {"c": 23, "d": {"e": 11}}} + st = StructuredTensor.from_pyval(pyval) + + # Try to set non-existant sub-structure. + with self.assertRaisesRegex( + ValueError, r"cannot create new sub-field.*\('b', 'x'\).*is not set"): + st.with_updates({("b", "x", "e"): 5}) + + # Try to set with path to a non-sub-structure. + with self.assertRaisesRegex( + ValueError, r"cannot create new sub-field.*\('b', 'c'\).*is not a " + r"`StructuredTensor`"): + st.with_updates({("b", "c", "e"): 5}) + + # Try to apply function to non-existing value. + with self.assertRaisesRegex( + ValueError, r"cannot update.*\('b', 'd', 'x'\).*does not already " + r"exist"): + st.with_updates({("b", "d", "x"): lambda x: x + 1}) + + # Empty names not allowed. + with self.assertRaisesRegex(ValueError, r"does not allow empty names"): + st.with_updates({(): lambda x: x + 1}) + with self.assertRaisesRegex(ValueError, r"does not allow empty names"): + st.with_updates({("b", ""): lambda x: x + 1}) + + # Parent and child nodes cannot be updated simultaneously. + with self.assertRaisesRegex( + ValueError, r"does not allow both parent and child nodes.*" + r"parent=\('b'.*child=\('b', 'd'"): + st.with_updates({("b", "d"): lambda x: x + 1, "a": 3, "b": 10}) + + # Invalid shape change. + with self.assertRaisesRegex( + ValueError, r"\('c'.*incompatible with the shape that was specified"): + st_with_shape = StructuredTensor.from_pyval([[{ + "c": { + "a": 5, + "b": 2 + } + }], [{ + "c": { + "a": 3, + "b": 1 + } + }, { + "c": { + "a": 8, + "b": 18 + } + }]]) + st_with_shape.with_updates({("c", "a"): 3}) + + def testWithUpdatesDelete(self): + pyval = {"a": 12, "b": {"c": 23, "d": {"e": 11}}} + st = StructuredTensor.from_pyval(pyval) + updated_st = st.with_updates({("b", "c"): None}, validate=True) + self.assertNotIn("c", updated_st.field_value("b").field_names()) + with self.assertRaisesRegex(ValueError, + r"cannot delete.*\('b', 'x'\).*not present"): + st.with_updates({("b", "x"): None}, validate=True) + with self.assertRaisesRegex(ValueError, + r"cannot delete.*\'x'.*not present"): + st.with_updates({"x": None}, validate=False) + + # Test that nrows() and rowpartitions() is preserved after removal. + pyval = [[{"a": 1}, {"a": 2}], [{"a": 3}]] + st = StructuredTensor.from_pyval(pyval) + self.assertLen(st.row_partitions, 1) + self.assertAllEqual(st.nrows(), 2) + self.assertAllEqual(st.row_partitions[0].row_lengths(), [2, 1]) + updated_st = st.with_updates({("a",): None}, validate=True) + self.assertLen(updated_st.row_partitions, 1) + self.assertAllEqual(updated_st.nrows(), 2) + self.assertAllEqual(updated_st.row_partitions[0].row_lengths(), [2, 1]) + + # Test that it works also for rank-1 and rank-0 empty results. + pyval = [{"a": 1}, {"a": 2}] + st = StructuredTensor.from_pyval(pyval) + self.assertEqual(st.rank, 1) + updated_st = st.with_updates({("a",): None}, validate=True) + self.assertEqual(updated_st.rank, 1) + + # assertEqual won't work because nrows() returns a tensor, and + # assertEqual doesn't do the magic to convert them to numbers in a + # way that works in eager/non-eager mode. + self.assertAllEqual(updated_st.nrows(), 2) + pyval = {"a": [0, 1]} + st = StructuredTensor.from_pyval(pyval) + self.assertEqual(st.rank, 0) + updated_st = st.with_updates({("a",): None}, validate=True) + self.assertEqual(updated_st.rank, 0) + self.assertFalse(updated_st.row_partitions) + self.assertIsNone(updated_st.nrows()) + if __name__ == "__main__": googletest.main() From cdfd1114fd309abc86f4c2b5507abdd3578457fb Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 8 Aug 2020 02:01:43 -0700 Subject: [PATCH 0701/1017] compat: Update forward compatibility horizon to 2020-08-08 PiperOrigin-RevId: 325579714 Change-Id: Ia49ab74cdbebbc59dc4f919fb2b6a68c3507cc42 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index bef47619972..01ea900fd11 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 7) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 8) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From 1b8087726a5667d9e88e41dda310f25e1593b0f2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 8 Aug 2020 02:01:44 -0700 Subject: [PATCH 0702/1017] Update GraphDef version to 487. PiperOrigin-RevId: 325579715 Change-Id: Ie4b2d7cdd7672486494b057c5a1cdd35ab2b6f81 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index ee9be29958d..3512cb4c5b9 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 486 // Updated: 2020/8/7 +#define TF_GRAPH_DEF_VERSION 487 // Updated: 2020/8/8 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From c1a176930a9d5895bdd87b91ba0dde53a0aa1a35 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 8 Aug 2020 04:34:20 -0700 Subject: [PATCH 0703/1017] Integrate LLVM at llvm/llvm-project@b6d9add71b1a Updates LLVM usage to match [b6d9add71b1a](https://github.com/llvm/llvm-project/commit/b6d9add71b1a) PiperOrigin-RevId: 325589103 Change-Id: If80989dd59ceb82283256a4149cceb3062ec2c72 --- .../hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.h | 8 +++++++- .../mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc | 3 +-- .../mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc | 2 +- .../mlir/hlo/lib/Dialect/mhlo/IR/lhlo_ops.cc | 2 +- tensorflow/compiler/mlir/lite/ir/tfl_ops.cc | 2 +- tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc | 3 ++- .../compiler/mlir/tensorflow/ir/tf_executor.cc | 3 ++- tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc | 2 +- .../compiler/mlir/tensorflow/ir/tf_saved_model.cc | 3 ++- tensorflow/compiler/mlir/tfjs/ir/tfjs_ops.cc | 3 +-- .../compiler/mlir/tools/kernel_gen/cubin_creator.cc | 3 ++- .../mlir/tools/kernel_gen/ir/tf_framework_ops.cc | 3 +-- tensorflow/compiler/xla/service/cpu/cpu_compiler.cc | 13 +++++-------- tensorflow/compiler/xla/service/cpu/mlir_emitter.cc | 8 +++++--- .../compiler/xla/service/gpu/ir/xla_thunks_ops.cc | 2 +- tensorflow/compiler/xla/service/mlir_gpu/BUILD | 1 + .../xla/service/mlir_gpu/lhlo_dialect_emitter.cc | 2 +- .../compiler/xla/service/mlir_gpu/mlir_compiler.cc | 10 +++------- .../xla/service/mlir_gpu/mlir_compiler_impl.cc | 7 ++++++- tensorflow/workspace.bzl | 4 ++-- third_party/llvm/llvm.autogenerated.BUILD | 3 +++ 21 files changed, 49 insertions(+), 38 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.h b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.h index 14a22e92a74..4c92ef3de85 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.h +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.h @@ -32,8 +32,14 @@ namespace mlir { namespace chlo { class HloClientDialect : public Dialect { + void initialize(); + public: - explicit HloClientDialect(MLIRContext *context); + explicit HloClientDialect(MLIRContext *context) + : Dialect(getDialectNamespace(), context, + TypeID::get()) { + initialize(); + } static StringRef getDialectNamespace() { return "chlo"; } }; diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc index 81389c3be89..d43dd71e94b 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc @@ -266,8 +266,7 @@ BROADCAST_BINARY_OP_DEFS(BroadcastXorOp); // chlo Dialect Constructor //===----------------------------------------------------------------------===// -HloClientDialect::HloClientDialect(MLIRContext* context) - : Dialect(getDialectNamespace(), context) { +void HloClientDialect::initialize() { addOperations< #define GET_OP_LIST #include "mlir-hlo/Dialect/mhlo/IR/chlo_ops.cc.inc" diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc index eda10b0f187..f5deb94e3a4 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/hlo_ops.cc @@ -2188,7 +2188,7 @@ struct HLOInlinerInterface : public DialectInlinerInterface { //===----------------------------------------------------------------------===// MhloDialect::MhloDialect(MLIRContext* context) - : Dialect(getDialectNamespace(), context) { + : Dialect(getDialectNamespace(), context, TypeID::get()) { addOperations< #define GET_OP_LIST #include "mlir-hlo/Dialect/mhlo/IR/hlo_ops.cc.inc" diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/lhlo_ops.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/lhlo_ops.cc index bbb463cd1a9..f61a66397e7 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/lhlo_ops.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/lhlo_ops.cc @@ -49,7 +49,7 @@ namespace mlir { namespace lmhlo { LmhloDialect::LmhloDialect(MLIRContext *context) - : Dialect(getDialectNamespace(), context) { + : Dialect(getDialectNamespace(), context, TypeID::get()) { addOperations< #define GET_OP_LIST #include "mlir-hlo/Dialect/mhlo/IR/lhlo_ops.cc.inc" diff --git a/tensorflow/compiler/mlir/lite/ir/tfl_ops.cc b/tensorflow/compiler/mlir/lite/ir/tfl_ops.cc index ae1e3ebe5e6..b5fcd5e82e2 100644 --- a/tensorflow/compiler/mlir/lite/ir/tfl_ops.cc +++ b/tensorflow/compiler/mlir/lite/ir/tfl_ops.cc @@ -269,7 +269,7 @@ struct TensorFlowLiteOpFolderDialectInterface }; TensorFlowLiteDialect::TensorFlowLiteDialect(mlir::MLIRContext *context) - : Dialect(/*name=*/"tfl", context) { + : Dialect(/*name=*/"tfl", context, TypeID::get()) { addOperations< #define GET_OP_LIST #include "tensorflow/compiler/mlir/lite/ir/tfl_ops.cc.inc" diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc index 9aa0a72f475..5345000b4bd 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc @@ -101,7 +101,8 @@ bool BlockWrapsSingleOp(Block* block) { } // end anonymous namespace TensorFlowDeviceDialect::TensorFlowDeviceDialect(MLIRContext* context) - : Dialect(/*name=*/"tf_device", context) { + : Dialect(/*name=*/"tf_device", context, + TypeID::get()) { addOperations< #define GET_OP_LIST #include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.cc.inc" diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc index c18723b0982..9c2968fab37 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc @@ -92,7 +92,8 @@ struct TensorFlowExecutorOpFolderDialectInterface } // namespace TensorFlowExecutorDialect::TensorFlowExecutorDialect(MLIRContext *context) - : Dialect(/*name=*/"tf_executor", context) { + : Dialect(/*name=*/"tf_executor", context, + TypeID::get()) { addOperations< #define GET_OP_LIST #include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.cc.inc" diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc index abff4c21cf1..dbad613d909 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc @@ -188,7 +188,7 @@ std::vector new std::vector(); TensorFlowDialect::TensorFlowDialect(MLIRContext *context) - : Dialect(/*name=*/"tf", context) { + : Dialect(/*name=*/"tf", context, TypeID::get()) { addOperations< #define GET_OP_LIST #include "tensorflow/compiler/mlir/tensorflow/ir/tf_all_ops.cc.inc" diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc index 94a792ec3db..6883d0358ec 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc @@ -113,7 +113,8 @@ static LogicalResult Verify(SessionInitializerOp session_initializer) { //===----------------------------------------------------------------------===// TensorFlowSavedModelDialect::TensorFlowSavedModelDialect(MLIRContext *context) - : Dialect(/*name=*/"tf_saved_model", context) { + : Dialect(/*name=*/"tf_saved_model", context, + TypeID::get()) { addOperations< #define GET_OP_LIST #include "tensorflow/compiler/mlir/tensorflow/ir/tf_saved_model.cc.inc" diff --git a/tensorflow/compiler/mlir/tfjs/ir/tfjs_ops.cc b/tensorflow/compiler/mlir/tfjs/ir/tfjs_ops.cc index 9ba875cdce4..331bed09dce 100644 --- a/tensorflow/compiler/mlir/tfjs/ir/tfjs_ops.cc +++ b/tensorflow/compiler/mlir/tfjs/ir/tfjs_ops.cc @@ -25,8 +25,7 @@ namespace tfjs { // TFJSDialect //===----------------------------------------------------------------------===// -TFJSDialect::TFJSDialect(MLIRContext *context) - : Dialect(getDialectNamespace(), context) { +void TFJSDialect::initialize() { addOperations< #define GET_OP_LIST #include "tensorflow/compiler/mlir/tfjs/ir/tfjs_ops.cc.inc" diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/cubin_creator.cc b/tensorflow/compiler/mlir/tools/kernel_gen/cubin_creator.cc index 1f511e27d9e..82b0e613f90 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/cubin_creator.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/cubin_creator.cc @@ -278,7 +278,8 @@ StatusOr> tensorflow::kernel_gen::GenerateCubinForTfCode( mlir::OwningModuleRef kernel_module = xla::mlir_gpu::ExtractKernelModule(*module).ValueOrDie(); - auto llvmModule = mlir::translateModuleToNVVMIR(*kernel_module); + llvm::LLVMContext llvmContext; + auto llvmModule = mlir::translateModuleToNVVMIR(*kernel_module, llvmContext); if (!llvmModule) { return InternalError("Could not translate MLIR module to NVVM"); } diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc index f85f1229fe8..5b7a19a3eac 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc @@ -24,8 +24,7 @@ namespace mlir { namespace kernel_gen { namespace tf_framework { -TFFrameworkDialect::TFFrameworkDialect(MLIRContext *context) - : Dialect(getDialectNamespace(), context) { +void TFFrameworkDialect::initialize() { addOperations< #define GET_OP_LIST #include "tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.cc.inc" diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index eb5d9e704f5..aab13f6e8dd 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -622,10 +622,9 @@ StatusOr> CpuCompiler::RunBackend( // Compile must be thread-safe so create a new LLVM context for the module. mlir::MLIRContext mlir_context; - auto llvm_module = absl::make_unique( - "__compute_module", - mlir_context.getRegisteredDialect() - ->getLLVMContext()); + llvm::LLVMContext llvm_context; + auto llvm_module = + absl::make_unique("__compute_module", llvm_context); auto jit = absl::make_unique( CompilerTargetOptions(module->config()), @@ -834,10 +833,8 @@ CpuCompiler::CompileAheadOfTime(std::unique_ptr module_group, // Compile must be thread-safe so create a new LLVM context for the module. mlir::MLIRContext mlir_context; - llvm::Module llvm_module( - "__compute_module", - mlir_context.getRegisteredDialect() - ->getLLVMContext()); + llvm::LLVMContext llvm_context; + llvm::Module llvm_module("__compute_module", llvm_context); llvm_module.setDataLayout(target_machine->createDataLayout()); llvm_module.setTargetTriple(triple.getTriple()); if (pic_level != llvm::PICLevel::NotPIC) { diff --git a/tensorflow/compiler/xla/service/cpu/mlir_emitter.cc b/tensorflow/compiler/xla/service/cpu/mlir_emitter.cc index ff48f554ce6..ae23f224207 100644 --- a/tensorflow/compiler/xla/service/cpu/mlir_emitter.cc +++ b/tensorflow/compiler/xla/service/cpu/mlir_emitter.cc @@ -32,7 +32,8 @@ namespace cpu { namespace { // Lower an MLIR module to an LLVM module. -std::unique_ptr MakeLLVMModule(mlir::OwningModuleRef module) { +std::unique_ptr MakeLLVMModule(mlir::OwningModuleRef module, + llvm::LLVMContext *context) { // When set, the LLVM backend will be allowed to reassociate floating-point // reductions, which enables much more efficient "horizontal" SIMD // implementations. @@ -47,7 +48,7 @@ std::unique_ptr MakeLLVMModule(mlir::OwningModuleRef module) { mlir::LowerVectorToLLVMOptions().setReassociateFPReductions( kReassociateFPReductions))); CHECK(succeeded(manager.run(*module))); - return mlir::translateModuleToLLVMIR(*module); + return mlir::translateModuleToLLVMIR(*module, *context); } // Get arguments to pass a memref to an mlir function. @@ -114,7 +115,8 @@ Status EmitMlirFuncAndCall( emitter(&op_builder, function); // Now link it all into the main LLVM module. - auto mlir_llvm_module = MakeLLVMModule(std::move(mlir_module)); + auto mlir_llvm_module = + MakeLLVMModule(std::move(mlir_module), &b->getContext()); mlir_llvm_module->setDataLayout(llvm_module->getDataLayout()); llvm::Linker::linkModules( *llvm_module, std::move(mlir_llvm_module), llvm::Linker::None, diff --git a/tensorflow/compiler/xla/service/gpu/ir/xla_thunks_ops.cc b/tensorflow/compiler/xla/service/gpu/ir/xla_thunks_ops.cc index 4dbd3196ae6..154612824ef 100644 --- a/tensorflow/compiler/xla/service/gpu/ir/xla_thunks_ops.cc +++ b/tensorflow/compiler/xla/service/gpu/ir/xla_thunks_ops.cc @@ -28,7 +28,7 @@ namespace mlir { namespace xla_thunks { XLAThunksDialect::XLAThunksDialect(MLIRContext *context) - : Dialect(getDialectNamespace(), context) { + : Dialect(getDialectNamespace(), context, TypeID::get()) { addOperations< #define GET_OP_LIST #include "tensorflow/compiler/xla/service/gpu/ir/xla_thunks_ops.cc.inc" diff --git a/tensorflow/compiler/xla/service/mlir_gpu/BUILD b/tensorflow/compiler/xla/service/mlir_gpu/BUILD index 2bcf5fa7dae..43a6efe9e90 100644 --- a/tensorflow/compiler/xla/service/mlir_gpu/BUILD +++ b/tensorflow/compiler/xla/service/mlir_gpu/BUILD @@ -82,6 +82,7 @@ cc_library( ":kernel_lowering", ":lhlo_dialect_emitter", "@com_google_absl//absl/container:flat_hash_map", + "@llvm-project//llvm:Core", "@llvm-project//mlir:GPUDialect", "@llvm-project//mlir:AllPassesAndDialects", "@llvm-project//mlir:IR", diff --git a/tensorflow/compiler/xla/service/mlir_gpu/lhlo_dialect_emitter.cc b/tensorflow/compiler/xla/service/mlir_gpu/lhlo_dialect_emitter.cc index 194eb4618d3..e0d7456fbb8 100644 --- a/tensorflow/compiler/xla/service/mlir_gpu/lhlo_dialect_emitter.cc +++ b/tensorflow/compiler/xla/service/mlir_gpu/lhlo_dialect_emitter.cc @@ -205,7 +205,7 @@ LhloDialectEmitter::LhloDialectEmitter( platform_(platform) { LLVMDialect* llvmDialect = mlir_module.getContext()->getRegisteredDialect(); - pointer_size_ = llvmDialect->getLLVMModule().getDataLayout().getPointerSize(); + pointer_size_ = llvmDialect->getDataLayout().getPointerSize(); } void LhloDialectEmitter::AddThunkToThunkSequence(std::unique_ptr thunk) { diff --git a/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler.cc b/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler.cc index 458522f89e6..df2bd2e4c23 100644 --- a/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler.cc +++ b/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler.cc @@ -30,18 +30,14 @@ namespace { using ::mlir::MLIRContext; using ::mlir::LLVM::LLVMDialect; -int64 ConfigureLLVMModuleAndGetPointerSize(MLIRContext* context) { +int64 GetPointerSize(MLIRContext* context) { LLVMDialect* dialect = context->getRegisteredDialect(); - llvm::Module& module = dialect->getLLVMModule(); - module.setTargetTriple(gpu::nvptx::kTargetTriple); - module.setDataLayout(gpu::nvptx::kDataLayout); - return module.getDataLayout().getPointerSize(); + return dialect->getDataLayout().getPointerSize(); } } // namespace -MlirCompiler::MlirCompiler() - : pointer_size_(ConfigureLLVMModuleAndGetPointerSize(&context_)) {} +MlirCompiler::MlirCompiler() : pointer_size_(GetPointerSize(&context_)) {} se::Platform::Id MlirCompiler::PlatformId() const { return stream_executor::cuda::kCudaPlatformId; diff --git a/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler_impl.cc b/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler_impl.cc index 25a35a89cb4..4879c6b5099 100644 --- a/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler_impl.cc +++ b/tensorflow/compiler/xla/service/mlir_gpu/mlir_compiler_impl.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include "absl/container/flat_hash_map.h" +#include "llvm/IR/LLVMContext.h" #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h" // from @llvm-project #include "mlir/Dialect/GPU/GPUDialect.h" // from @llvm-project #include "mlir/Dialect/LLVMIR/LLVMDialect.h" // from @llvm-project @@ -543,7 +544,11 @@ StatusOr> MlirCompilerImpl::RunBackend( TF_RETURN_IF_ERROR( module_hook_.invoke(IRHook::LoweringStage::KERNEL, *kernel_module)); - auto llvmModule = mlir::translateModuleToNVVMIR(*kernel_module); + // Translate to LLVM IR in a fresh context. The module is further translated + // to textual PTX and a CUBIN blob so there is no need for the context to live + // longer than this function. + llvm::LLVMContext llvmContext; + auto llvmModule = mlir::translateModuleToNVVMIR(*kernel_module, llvmContext); if (!llvmModule) { return InternalError("Translation to LLVM failed"); diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 07b9950bca2..0b4898b2c35 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "9dbdaea9a0e6f58417b5bd8980e7ea6723fd1783" - LLVM_SHA256 = "1ae491e33bb35777cf5f38acd183ce3ca2aff255c15254ae97084bcbd2e4aa56" + LLVM_COMMIT = "b6d9add71b1a7bc77ce504ed09a43288ca67c0cd" + LLVM_SHA256 = "60160d35f22445819da148af4ac1119d9275f7c363a841748593e276c284fa20" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), diff --git a/third_party/llvm/llvm.autogenerated.BUILD b/third_party/llvm/llvm.autogenerated.BUILD index befc20c4fab..e3ae2c9e889 100644 --- a/third_party/llvm/llvm.autogenerated.BUILD +++ b/third_party/llvm/llvm.autogenerated.BUILD @@ -1556,7 +1556,9 @@ cc_library( ":BPFInfo", ":CodeGen", ":Core", + ":IPO", ":MC", + ":Scalar", ":SelectionDAG", ":Support", ":Target", @@ -1763,6 +1765,7 @@ cc_library( ":Core", ":Instrumentation", ":MC", + ":Passes", ":ProfileData", ":Scalar", ":Support", From 24bbe30d1ee97f7ae051b5c52d70c2a2b94a1bb4 Mon Sep 17 00:00:00 2001 From: Benjamin Kramer Date: Sat, 8 Aug 2020 05:03:25 -0700 Subject: [PATCH 0704/1017] Integrate LLVM at llvm/llvm-project@38537307e502 Updates LLVM usage to match [38537307e502](https://github.com/llvm/llvm-project/commit/38537307e502) PiperOrigin-RevId: 325590988 Change-Id: I8ee6b700411d243d6df8e044a54b72be8323a6c4 --- tensorflow/workspace.bzl | 4 ++-- third_party/llvm/llvm.autogenerated.BUILD | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 0b4898b2c35..a726bf642d1 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "b6d9add71b1a7bc77ce504ed09a43288ca67c0cd" - LLVM_SHA256 = "60160d35f22445819da148af4ac1119d9275f7c363a841748593e276c284fa20" + LLVM_COMMIT = "38537307e502c1ac9a09e6f75f9208db1327a0bf" + LLVM_SHA256 = "c801bf0f2ebfce86dbf7ad39c40ee371f422e8d07213f4ca67e5e46c7cb200ed" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), diff --git a/third_party/llvm/llvm.autogenerated.BUILD b/third_party/llvm/llvm.autogenerated.BUILD index e3ae2c9e889..13bc7bf2902 100644 --- a/third_party/llvm/llvm.autogenerated.BUILD +++ b/third_party/llvm/llvm.autogenerated.BUILD @@ -1765,7 +1765,6 @@ cc_library( ":Core", ":Instrumentation", ":MC", - ":Passes", ":ProfileData", ":Scalar", ":Support", From 11305e9e40f3a992fe4f9edc8f1acfcc024e4ca1 Mon Sep 17 00:00:00 2001 From: Rohan Jain Date: Sat, 8 Aug 2020 07:43:57 -0700 Subject: [PATCH 0705/1017] Removes most run_deprecated_v1 and run_v1_only from scatter_nd_ops_test. The others will be removed in another cl PiperOrigin-RevId: 325599122 Change-Id: Ic0d8827514b2feb79abf7b05cdbecfa901cad7cb --- tensorflow/python/framework/test_util.py | 4 +- .../kernel_tests/scatter_nd_ops_test.py | 331 +++++++++--------- .../python/kernel_tests/v1_compat_tests/BUILD | 12 + .../v1_compat_tests/scatter_nd_ops_test.py | 159 +++++++++ 4 files changed, 330 insertions(+), 176 deletions(-) create mode 100644 tensorflow/python/kernel_tests/v1_compat_tests/scatter_nd_ops_test.py diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py index 15f4507b5e2..4d7b7746b9c 100644 --- a/tensorflow/python/framework/test_util.py +++ b/tensorflow/python/framework/test_util.py @@ -3295,8 +3295,8 @@ def _fake_gradient_tape_context_manager(): def watch(self, x): pass - def gradient(self, y, x): - result = gradients_impl.gradients(y, x) + def gradient(self, y, x, grad_ys=None): + result = gradients_impl.gradients(y, x, grad_ys) # Unlike `tape.gradient()`, `tf.gradients()` returns a list for a single # element. So unpack if needed to match `tape.gradient()` behavior. diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py index d6768712d65..c5e5e549ee7 100644 --- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py +++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py @@ -20,18 +20,18 @@ from __future__ import print_function import functools +from absl.testing import parameterized import numpy as np -from tensorflow.python.client import session from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors +from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops -from tensorflow.python.ops import gradient_checker -from tensorflow.python.ops import gradients_impl +from tensorflow.python.ops import gradient_checker_v2 from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import state_ops from tensorflow.python.ops import variables @@ -115,7 +115,7 @@ class StatefulScatterNdTest(test.TestCase): np.random.seed(8) ref_shapes = [(3, 6), (3, 6), (3, 6, 9), (3, 6, 9), (3, 6, 9), (3, 6, 9)] indices_shapes = [(2,), (2, 2), (2,), (2, 2), (2, 3), (2, 3, 3)] - with self.cached_session(use_gpu=True): + with test_util.device(use_gpu=True): for ref_shape, indices_shape in zip(ref_shapes, indices_shapes): num_updates = indices_shape[0] ixdim = indices_shape[-1] @@ -151,7 +151,7 @@ class StatefulScatterNdTest(test.TestCase): # Scatter via tensorflow ref_var = variables.VariableV1(ref) self.evaluate(ref_var.initializer) - tf_scatter(ref_var, indices, updates).eval() + self.evaluate(tf_scatter(ref_var, indices, updates)) # Compare self.assertAllClose(new, self.evaluate(ref_var)) @@ -187,7 +187,6 @@ class StatefulScatterNdTest(test.TestCase): self.assertAllEqual(self.evaluate(update), [b"qq", b"cc", b"ee", b"dd", b"aa", b"", b"", b"bb"]) - @test_util.run_deprecated_v1 def testSimpleResource(self): indices = constant_op.constant([[4], [3], [1], [7]], dtype=dtypes.int32) updates = constant_op.constant([9, 10, 11, 12], dtype=dtypes.float32) @@ -195,10 +194,9 @@ class StatefulScatterNdTest(test.TestCase): [0, 0, 0, 0, 0, 0, 0, 0], dtype=dtypes.float32) expected = np.array([0, 11, 0, 10, 9, 0, 0, 12]) scatter = state_ops.scatter_nd_update(ref, indices, updates) - init = variables.global_variables_initializer() - with self.session(use_gpu=True) as sess: - self.evaluate(init) + with test_util.device(use_gpu=True): + self.evaluate(ref.initializer) self.evaluate(scatter) self.assertAllClose(ref, expected) @@ -230,15 +228,12 @@ class StatefulScatterNdTest(test.TestCase): result = self.evaluate(scatter) self.assertAllClose(result, expected) - @test_util.run_deprecated_v1 def testVariableRankUpdate(self): self._VariableRankTests(_NumpyUpdate, state_ops.scatter_nd_update) - @test_util.run_deprecated_v1 def testVariableRankAdd(self): self._VariableRankTests(_NumpyAdd, state_ops.scatter_nd_add) - @test_util.run_deprecated_v1 def testVariableRankSub(self): self._VariableRankTests(_NumpySub, state_ops.scatter_nd_sub) @@ -256,13 +251,10 @@ class StatefulScatterNdTest(test.TestCase): self._VariableRankTest( np_scatter, tf_scatter, vtype, itype, repeat_indices=True) - @test_util.run_v1_only("b/120545219") def testScatterRepeatIndices(self): """This tests scatter_add using indices that repeat.""" self._ScatterRepeatIndicesTest(_NumpyAdd, state_ops.scatter_nd_add) self._ScatterRepeatIndicesTest(_NumpySub, state_ops.scatter_nd_sub) - self._ScatterRepeatIndicesTest(_NumpyMin, state_ops.scatter_nd_min) - self._ScatterRepeatIndicesTest(_NumpyMax, state_ops.scatter_nd_max) # TODO(ebrevdo): Re-enable when we need ScatterNdMul and ScatterNdDiv. # self._ScatterRepeatIndicesTest(_NumpyMul, state_ops.scatter_nd_mul) # self._ScatterRepeatIndicesTest(_NumpyDiv, state_ops.scatter_nd_div) @@ -280,34 +272,32 @@ class StatefulScatterNdTest(test.TestCase): # session.run([update0, update1]) # self.assertAllEqual([False, True], self.evaluate(var)) - @test_util.run_v1_only("b/120545219") def testScatterOutOfRangeCpu(self): # TODO(simister): Re-enable once binary size increase due to # scatter_nd ops is under control. # tf.scatter_nd_mul, tf.scatter_nd_div, for op in (state_ops.scatter_nd_add, state_ops.scatter_nd_sub, - state_ops.scatter_nd_min, state_ops.scatter_nd_max, state_ops.scatter_nd_update): params = np.array([1, 2, 3, 4, 5, 6]).astype(np.float32) updates = np.array([-3, -4, -5]).astype(np.float32) - with self.cached_session(use_gpu=False): + with test_util.device(use_gpu=False): ref = variables.VariableV1(params) self.evaluate(ref.initializer) # Indices all in range, no problem. indices = np.array([[2], [0], [5]]) - op(ref, indices, updates).eval() + self.evaluate(op(ref, indices, updates)) # Test some out of range errors. indices = np.array([[-1], [0], [5]]) with self.assertRaisesOpError( r"indices\[0\] = \[-1\] does not index into shape \[6\]"): - op(ref, indices, updates).eval() + self.evaluate(op(ref, indices, updates)) indices = np.array([[2], [0], [6]]) with self.assertRaisesOpError( r"indices\[2\] = \[6\] does not index into shape \[6\]"): - op(ref, indices, updates).eval() + self.evaluate(op(ref, indices, updates)) def testRank3ValidShape(self): indices = array_ops.zeros([2, 2, 2], dtypes.int32) @@ -318,7 +308,6 @@ class StatefulScatterNdTest(test.TestCase): state_ops.scatter_nd_update(ref, indices, updates).get_shape().as_list(), shape) - @test_util.run_v1_only("b/120545219") @test_util.disable_xla("b/123337890") # Error messages differ def testResVarInvalidOutputShape(self): res = variables.Variable( @@ -329,7 +318,6 @@ class StatefulScatterNdTest(test.TestCase): with self.assertRaisesOpError("Output must be at least 1-D"): state_ops.scatter_nd_update(res, [[0]], [0.22]).eval() - @test_util.run_deprecated_v1 def testExtraIndicesDimensions(self): indices = array_ops.zeros([1, 1, 2], dtypes.int32) updates = array_ops.zeros([1, 1], dtypes.int32) @@ -363,7 +351,6 @@ class StatefulScatterNdTest(test.TestCase): ValueError, r"The inner \d+ dimensions of input\.shape="): state_ops.scatter_nd_update(ref, indices, updates) - @test_util.run_deprecated_v1 def testConcurrentUpdates(self): num_updates = 10000 update_values = np.random.rand(num_updates) @@ -377,10 +364,9 @@ class StatefulScatterNdTest(test.TestCase): scatter = state_ops.scatter_nd_add(ref, indices, updates) init = variables.global_variables_initializer() - with session.Session() as sess: - self.evaluate(init) - result = self.evaluate(scatter) - assert np.allclose(result, expected_result) + self.evaluate(init) + result = self.evaluate(scatter) + assert np.allclose(result, expected_result) # TODO(fpmc): Re-enable this test when gpu_pip test actually runs on a GPU. def _disabledTestScatterOutOfRangeGpu(self): @@ -410,7 +396,7 @@ class StatefulScatterNdTest(test.TestCase): op(ref, indices, updates).eval() -class ScatterNdTest(test.TestCase): +class ScatterNdTest(test.TestCase, parameterized.TestCase): non_aliasing_add_test = False def scatter_nd(self, indices, updates, shape, input_=None): @@ -492,7 +478,6 @@ class ScatterNdTest(test.TestCase): self.assertAllEqual( self.scatter_nd(indices, updates, shape).get_shape().as_list(), shape) - @test_util.run_deprecated_v1 def testExtraIndicesDimensions(self): indices = array_ops.zeros([1, 1, 2], dtypes.int32) updates = array_ops.zeros([1, 1], dtypes.int32) @@ -500,29 +485,31 @@ class ScatterNdTest(test.TestCase): scatter = self.scatter_nd(indices, updates, shape) self.assertAllEqual(scatter.get_shape().as_list(), shape) expected_result = np.zeros([2, 2], dtype=np.int32) - with self.cached_session(): - self.assertAllEqual(expected_result, self.evaluate(scatter)) + self.assertAllEqual(expected_result, self.evaluate(scatter)) - @test_util.run_deprecated_v1 def testUndefinedIndicesShape(self): - indices = array_ops.placeholder(dtypes.int32, shape=None) - updates = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) - shape = constant_op.constant([2, 2, 2], dtypes.int32) - self.scatter_nd(indices, updates, shape) + # Placeholders are only valid in Graph. + with ops.Graph().as_default(): + indices = array_ops.placeholder(dtypes.int32, shape=None) + updates = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) + shape = constant_op.constant([2, 2, 2], dtypes.int32) + self.scatter_nd(indices, updates, shape) - @test_util.run_deprecated_v1 def testUndefinedUpdatesShape(self): - indices = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) - updates = array_ops.placeholder(dtypes.int32, shape=None) - shape = constant_op.constant([2, 2, 2], dtypes.int32) - self.scatter_nd(indices, updates, shape) + # Placeholders are only valid in Graph. + with ops.Graph().as_default(): + indices = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) + updates = array_ops.placeholder(dtypes.int32, shape=None) + shape = constant_op.constant([2, 2, 2], dtypes.int32) + self.scatter_nd(indices, updates, shape) - @test_util.run_deprecated_v1 def testUndefinedOutputShape(self): - indices = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) - updates = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) - shape = array_ops.placeholder(dtypes.int32, shape=[None]) - self.scatter_nd(indices, updates, shape) + # Placeholders are only valid in Graph. + with ops.Graph().as_default(): + indices = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) + updates = array_ops.placeholder(dtypes.int32, shape=[2, 2, 2]) + shape = array_ops.placeholder(dtypes.int32, shape=[None]) + self.scatter_nd(indices, updates, shape) @test_util.run_deprecated_v1 def testEmptyOutputShape1(self): @@ -534,21 +521,21 @@ class ScatterNdTest(test.TestCase): ValueError, "Indices and updates specified for empty output shape"): self.scatter_nd(indices, updates, shape) - @test_util.run_v1_only("b/120545219") def testEmptyOutputShape2(self): - indices = array_ops.placeholder(dtypes.int32, shape=None) - updates = array_ops.placeholder(dtypes.int32, shape=None) - shape = constant_op.constant([0, 3, 2], dtypes.int32) + with ops.Graph().as_default(): + indices = array_ops.placeholder(dtypes.int32, shape=None) + updates = array_ops.placeholder(dtypes.int32, shape=None) + shape = constant_op.constant([0, 3, 2], dtypes.int32) - with self.cached_session(): - with self.assertRaisesOpError( - "Indices and updates specified for empty output"): - self.scatter_nd(indices, updates, shape).eval(feed_dict={ - indices: np.zeros([2, 2, 2], dtype=np.int32), - updates: np.zeros([2, 2, 2], dtype=np.int32) - }) + with self.cached_session(): + with self.assertRaisesOpError( + "Indices and updates specified for empty output"): + self.scatter_nd(indices, updates, shape).eval( + feed_dict={ + indices: np.zeros([2, 2, 2], dtype=np.int32), + updates: np.zeros([2, 2, 2], dtype=np.int32) + }) - @test_util.run_deprecated_v1 def testEmptyOutputShape3(self): indices = array_ops.zeros([0], dtypes.int32) updates = array_ops.zeros([0], dtypes.int32) @@ -576,139 +563,138 @@ class ScatterNdTest(test.TestCase): ValueError, r"The inner \d+ dimensions of (input|output)\.shape="): self.scatter_nd(indices, updates, shape) - @test_util.run_deprecated_v1 - def testGradientsRank2ElementUpdate(self): + @parameterized.parameters(set((True, context.executing_eagerly()))) + def testGradientsRank2ElementUpdate(self, use_tape): for dtype in GRADIENT_TESTS_DTYPES: - indices = constant_op.constant([[0, 0], [1, 1]], dtype=dtypes.int32) - updates = constant_op.constant([1, 4], dtype=dtype) - shape = constant_op.constant([2, 2], dtype=dtypes.int32) - input_ = array_ops.zeros(shape, dtype=dtype) - outputs = self.scatter_nd(indices, updates, shape, input_) + with test_util.AbstractGradientTape(use_tape=use_tape) as tape: + indices = constant_op.constant([[0, 0], [1, 1]], dtype=dtypes.int32) + updates = constant_op.constant([1, 4], dtype=dtype) + tape.watch(updates) + shape = constant_op.constant([2, 2], dtype=dtypes.int32) + input_ = array_ops.zeros(shape, dtype=dtype) + tape.watch(input_) + outputs = self.scatter_nd(indices, updates, shape, input_) - grad_vals = constant_op.constant([[1, 2], [3, 4]], dtype=dtype) - updates_grad, input_grad = gradients_impl.gradients( - [outputs], [updates, input_], [grad_vals]) + grad_vals = constant_op.constant([[1, 2], [3, 4]], dtype=dtype) + + updates_grad, input_grad = tape.gradient([outputs], [updates, input_], + [grad_vals]) expected_updates_grad = np.array([1, 4], dtype=dtype.as_numpy_dtype()) expected_input_grad = np.array([[1, 2], [3, 4]], dtype=dtype.as_numpy_dtype()) - with self.cached_session(): - self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) - if self.non_aliasing_add_test: - self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) + self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) + if self.non_aliasing_add_test: + self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) - @test_util.run_deprecated_v1 - def testGradientsRank2SliceUpdate(self): + @parameterized.parameters(set((True, context.executing_eagerly()))) + def testGradientsRank2SliceUpdate(self, use_tape): for dtype in GRADIENT_TESTS_DTYPES: - indices = constant_op.constant([[1], [0]], dtype=dtypes.int32) - updates = constant_op.constant([[3, 4], [1, 2]], dtype=dtype) - shape = constant_op.constant([2, 2], dtype=dtypes.int32) - input_ = array_ops.zeros(shape, dtype=dtype) - outputs = self.scatter_nd(indices, updates, shape, input_) + with test_util.AbstractGradientTape(use_tape=use_tape) as tape: + indices = constant_op.constant([[1], [0]], dtype=dtypes.int32) + updates = constant_op.constant([[3, 4], [1, 2]], dtype=dtype) + tape.watch(updates) + shape = constant_op.constant([2, 2], dtype=dtypes.int32) + input_ = array_ops.zeros(shape, dtype=dtype) + tape.watch(input_) + outputs = self.scatter_nd(indices, updates, shape, input_) - grad_vals = constant_op.constant([[3, 4], [1, 2]], dtype=dtype) - updates_grad, input_grad = gradients_impl.gradients( - [outputs], [updates, input_], [grad_vals]) + grad_vals = constant_op.constant([[3, 4], [1, 2]], dtype=dtype) + updates_grad, input_grad = tape.gradient([outputs], [updates, input_], + [grad_vals]) expected_updates_grad = np.array([[1, 2], [3, 4]], dtype=dtype.as_numpy_dtype()) expected_input_grad = np.array([[3, 4], [1, 2]], dtype=dtype.as_numpy_dtype()) - with self.cached_session(): - self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) - if self.non_aliasing_add_test: - self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) + self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) + if self.non_aliasing_add_test: + self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) - @test_util.run_deprecated_v1 - def testGradientsRank3SliceUpdate(self): + @parameterized.parameters(set((True, context.executing_eagerly()))) + def testGradientsRank3SliceUpdate(self, use_tape): for dtype in GRADIENT_TESTS_DTYPES: - indices = constant_op.constant([[[0, 1], [1, 0]], [[0, 0], [1, 1]]], - dtype=dtypes.int32) - updates = constant_op.constant([[[5, 7], [2, 4]], [[1, 3], [6, 8]]], - dtype=dtype) - shape = constant_op.constant([2, 2, 2], dtype=dtypes.int32) - input_ = array_ops.zeros(shape, dtype=dtype) - outputs = self.scatter_nd(indices, updates, shape, input_) - - grad_vals = constant_op.constant([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], + with test_util.AbstractGradientTape(use_tape=use_tape) as tape: + indices = constant_op.constant([[[0, 1], [1, 0]], [[0, 0], [1, 1]]], + dtype=dtypes.int32) + updates = constant_op.constant([[[5, 7], [2, 4]], [[1, 3], [6, 8]]], dtype=dtype) - updates_grad, input_grad = gradients_impl.gradients( - [outputs], [updates, input_], [grad_vals]) + tape.watch(updates) + shape = constant_op.constant([2, 2, 2], dtype=dtypes.int32) + input_ = array_ops.zeros(shape, dtype=dtype) + tape.watch(input_) + outputs = self.scatter_nd(indices, updates, shape, input_) + + grad_vals = constant_op.constant([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], + dtype=dtype) + updates_grad, input_grad = tape.gradient([outputs], [updates, input_], + [grad_vals]) expected_updates_grad = np.array([[[3, 4], [5, 6]], [[1, 2], [7, 8]]], dtype=dtype.as_numpy_dtype()) expected_input_grad = np.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]], dtype=dtype.as_numpy_dtype()) - with self.cached_session(): - self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) - if self.non_aliasing_add_test: - self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) + self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) + if self.non_aliasing_add_test: + self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) - @test_util.run_deprecated_v1 - def testGradientsRank7SliceUpdate(self): + @parameterized.parameters(set((True, context.executing_eagerly()))) + def testGradientsRank7SliceUpdate(self, use_tape): for dtype in GRADIENT_TESTS_DTYPES: - indices = constant_op.constant( - [[[[[[[0, 0, 0, 0, 0, 1], [0, 0, 1, 0, 0, 0]]]], - [[[[0, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 1]]]]]]], - dtype=dtypes.int32) - updates = constant_op.constant( - [[[[[[[5, 6], [2, 4]]]], [[[[1, 3], [6, 8]]]]]]], dtype=dtype) - shape = constant_op.constant([1, 1, 2, 1, 1, 2, 2], dtype=dtypes.int32) - input_ = array_ops.zeros(shape, dtype=dtype) - outputs = self.scatter_nd(indices, updates, shape, input_) + with test_util.AbstractGradientTape(use_tape=use_tape) as tape: + indices = constant_op.constant( + [[[[[[[0, 0, 0, 0, 0, 1], [0, 0, 1, 0, 0, 0]]]], + [[[[0, 0, 0, 0, 0, 0], [0, 0, 1, 0, 0, 1]]]]]]], + dtype=dtypes.int32) + updates = constant_op.constant( + [[[[[[[5, 6], [2, 4]]]], [[[[1, 3], [6, 8]]]]]]], dtype=dtype) + tape.watch(updates) + shape = constant_op.constant([1, 1, 2, 1, 1, 2, 2], dtype=dtypes.int32) + input_ = array_ops.zeros(shape, dtype=dtype) + tape.watch(input_) + outputs = self.scatter_nd(indices, updates, shape, input_) - grad_vals = constant_op.constant( - [[[[[[[1, 2], [3, 4]]]], [[[[5, 6], [7, 8]]]]]]], dtype=dtype) - updates_grad, input_grad = gradients_impl.gradients( - [outputs], [updates, input_], [grad_vals]) + grad_vals = constant_op.constant( + [[[[[[[1, 2], [3, 4]]]], [[[[5, 6], [7, 8]]]]]]], dtype=dtype) + updates_grad, input_grad = tape.gradient([outputs], [updates, input_], + [grad_vals]) expected_updates_grad = np.array( [[[[[[[3, 4], [5, 6]]]], [[[[1, 2], [7, 8]]]]]]], dtype=dtype.as_numpy_dtype()) expected_input_grad = np.array( [[[[[[[1, 2], [3, 4]]]], [[[[5, 6], [7, 8]]]]]]], dtype=dtype.as_numpy_dtype()) - with self.cached_session(): - self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) - if self.non_aliasing_add_test: - self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) + self.assertAllEqual(expected_updates_grad, self.evaluate(updates_grad)) + if self.non_aliasing_add_test: + self.assertAllEqual(expected_input_grad, self.evaluate(input_grad)) - @test_util.run_deprecated_v1 def testScatterNdRepeatedIndicesAdd(self): indices = array_ops.zeros([100000, 1], dtypes.int32) values = np.random.randn(100000) shape = [1] - with self.cached_session(): - val = self.scatter_nd(indices, values, shape).eval() + val = self.evaluate(self.scatter_nd(indices, values, shape)) self.assertAllClose([np.sum(values)], val) - @test_util.run_deprecated_v1 def testSmokeScatterNdBatch2DSliceDim2(self): - with self.cached_session(): - indices = array_ops.zeros([3, 5, 2], dtype=dtypes.int32) - values = array_ops.zeros([3, 5, 7]) - shape = [4, 6, 7] - self.scatter_nd(indices, values, shape).eval() + indices = array_ops.zeros([3, 5, 2], dtype=dtypes.int32) + values = array_ops.zeros([3, 5, 7]) + shape = [4, 6, 7] + self.evaluate(self.scatter_nd(indices, values, shape)) - @test_util.run_deprecated_v1 def testSmokeScatterNdBatch1DSliceDim2(self): - with self.cached_session(): - indices = array_ops.zeros([0, 2], dtype=dtypes.int32) - values = array_ops.zeros([0, 7]) - shape = [4, 6, 7] - self.scatter_nd(indices, values, shape).eval() + indices = array_ops.zeros([0, 2], dtype=dtypes.int32) + values = array_ops.zeros([0, 7]) + shape = [4, 6, 7] + self.evaluate(self.scatter_nd(indices, values, shape)) - @test_util.run_deprecated_v1 def testSmokeScatterNdBatch1DSliceDim3ShapeRank7(self): - with self.cached_session(): - indices = array_ops.zeros([1, 3], dtype=dtypes.int32) - values = array_ops.zeros([1, 6, 7, 8, 9]) - shape = [3, 4, 5, 6, 7, 8, 9] - self.scatter_nd(indices, values, shape).eval() + indices = array_ops.zeros([1, 3], dtype=dtypes.int32) + values = array_ops.zeros([1, 6, 7, 8, 9]) + shape = [3, 4, 5, 6, 7, 8, 9] + self.evaluate(self.scatter_nd(indices, values, shape)) - @test_util.run_deprecated_v1 def testSmokeScatterNdBatch2DSliceDim3ShapeRank7(self): - with self.cached_session(): - indices = array_ops.zeros([1, 2, 3], dtype=dtypes.int32) - values = array_ops.zeros([1, 2, 6, 7, 8, 9]) - shape = [3, 4, 5, 6, 7, 8, 9] - self.scatter_nd(indices, values, shape).eval() + indices = array_ops.zeros([1, 2, 3], dtype=dtypes.int32) + values = array_ops.zeros([1, 2, 6, 7, 8, 9]) + shape = [3, 4, 5, 6, 7, 8, 9] + self.evaluate(self.scatter_nd(indices, values, shape)) class ScatterNdNonAliasingAddTest(ScatterNdTest): @@ -742,37 +728,34 @@ class ScatterNdTensorTest(test.TestCase): self.assertAllEqual(subbed, constant_op.constant([1, -10, 1, -9, -8, 1, 1, -11])) - @test_util.run_v1_only("b/120545219") def testUpdateAddSubGradients(self): - with self.cached_session(): indices = constant_op.constant([[3], [1]]) updates = constant_op.constant([9, 10], dtype=dtypes.float32) x = array_ops.ones([4], dtype=dtypes.float32) - assigned = array_ops.tensor_scatter_update(x, indices, updates) - added = array_ops.tensor_scatter_add(x, indices, updates) - subbed = array_ops.tensor_scatter_sub(x, indices, updates) + theoretical, numerical = gradient_checker_v2.compute_gradient( + lambda x: array_ops.tensor_scatter_update(x, indices, updates), [x]) + self.assertAllClose(theoretical, numerical, 5e-4, 5e-4) + theoretical, numerical = gradient_checker_v2.compute_gradient( + lambda x: array_ops.tensor_scatter_add(x, indices, updates), [x]) + self.assertAllClose(theoretical, numerical, 5e-4, 5e-4) + theoretical, numerical = gradient_checker_v2.compute_gradient( + lambda x: array_ops.tensor_scatter_sub(x, indices, updates), [x]) + self.assertAllClose(theoretical, numerical, 5e-4, 5e-4) - err_assigned = gradient_checker.compute_gradient_error( - x, [4], assigned, [4]) - err_added = gradient_checker.compute_gradient_error(x, [4], added, [4]) - err_subbed = gradient_checker.compute_gradient_error(x, [4], subbed, [4]) - - self.assertLess(err_assigned, 2e-4) - self.assertLess(err_added, 2e-4) - self.assertLess(err_subbed, 2e-4) - - err_assigned_wrt_updates = gradient_checker.compute_gradient_error( - updates, [2], assigned, [4]) - err_added_wrt_updates = gradient_checker.compute_gradient_error( - updates, [2], added, [4]) - err_subbed_wrt_updates = gradient_checker.compute_gradient_error( - updates, [2], subbed, [4]) - - self.assertLess(err_assigned_wrt_updates, 2e-4) - self.assertLess(err_added_wrt_updates, 2e-4) - self.assertLess(err_subbed_wrt_updates, 2e-4) + theoretical, numerical = gradient_checker_v2.compute_gradient( + lambda updates: array_ops.tensor_scatter_update(x, indices, updates), + [updates]) + self.assertAllClose(theoretical, numerical, 5e-4, 5e-4) + theoretical, numerical = gradient_checker_v2.compute_gradient( + lambda updates: array_ops.tensor_scatter_add(x, indices, updates), + [updates]) + self.assertAllClose(theoretical, numerical, 5e-4, 5e-4) + theoretical, numerical = gradient_checker_v2.compute_gradient( + lambda updates: array_ops.tensor_scatter_sub(x, indices, updates), + [updates]) + self.assertAllClose(theoretical, numerical, 5e-4, 5e-4) @test_util.run_in_graph_and_eager_modes def testUpdateMinMax(self): diff --git a/tensorflow/python/kernel_tests/v1_compat_tests/BUILD b/tensorflow/python/kernel_tests/v1_compat_tests/BUILD index 9cd0f4df101..bd9c02d8101 100644 --- a/tensorflow/python/kernel_tests/v1_compat_tests/BUILD +++ b/tensorflow/python/kernel_tests/v1_compat_tests/BUILD @@ -19,6 +19,18 @@ tf_py_test( ], ) +cuda_py_test( + name = "scatter_nd_ops_test", + size = "small", + srcs = ["scatter_nd_ops_test.py"], + deps = [ + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:state_ops", + "//tensorflow/python:variables", + "//third_party/py/numpy", + ], +) + cuda_py_test( name = "session_ops_test", size = "small", diff --git a/tensorflow/python/kernel_tests/v1_compat_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/v1_compat_tests/scatter_nd_ops_test.py new file mode 100644 index 00000000000..6ee75649867 --- /dev/null +++ b/tensorflow/python/kernel_tests/v1_compat_tests/scatter_nd_ops_test.py @@ -0,0 +1,159 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for scatter_nd_ops that only work in V1.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import functools + +import numpy as np + +from tensorflow.python.framework import test_util +from tensorflow.python.ops import state_ops +from tensorflow.python.ops import variables +from tensorflow.python.platform import test + + +def _AsType(v, vtype): + return v.astype(vtype) if isinstance(v, np.ndarray) else vtype(v) + + +def _FlatInnerDims(tensor, ndims=2): + shape = list(tensor.shape) + return tensor.reshape( + [functools.reduce(lambda x, y: x * y, shape[:-ndims + 1], 1)] + + shape[-ndims + 1:]) + + +def _FlatOuterDims(tensor, ndims=2): + shape = list(tensor.shape) + return tensor.reshape( + shape[:ndims - 1] + + [functools.reduce(lambda x, y: x * y, shape[ndims - 1:], 1)]) + + +def _NumpyScatterNd(ref, indices, updates, op): + ixdim = indices.shape[-1] + num_updates = indices.size // ixdim + total_nd = len(ref.shape) + slice_size = 1 + for i in range(ixdim, total_nd): + slice_size *= ref.shape[i] + flat_indices = _FlatInnerDims(indices) + flat_updates = updates.reshape((num_updates, slice_size)) + output_flat = _FlatOuterDims(ref, ixdim + 1) + for ix_updates, ix_output in enumerate(flat_indices): + ix_output = tuple(ix_output) + output_flat[ix_output] = op(output_flat[ix_output], + flat_updates[ix_updates]) + return output_flat.reshape(ref.shape) + + +def _NumpyMin(ref, indices, updates): + return _NumpyScatterNd(ref, indices, updates, np.minimum) + + +def _NumpyMax(ref, indices, updates): + return _NumpyScatterNd(ref, indices, updates, np.maximum) + + +class StatefulScatterNdTest(test.TestCase): + + def _VariableRankTest(self, + np_scatter, + tf_scatter, + vtype, + itype, + repeat_indices=False): + np.random.seed(8) + ref_shapes = [(3, 6), (3, 6), (3, 6, 9), (3, 6, 9), (3, 6, 9), (3, 6, 9)] + indices_shapes = [(2,), (2, 2), (2,), (2, 2), (2, 3), (2, 3, 3)] + with test_util.device(use_gpu=True): + for ref_shape, indices_shape in zip(ref_shapes, indices_shapes): + num_updates = indices_shape[0] + ixdim = indices_shape[-1] + + indexable_area_shape = () + for i in range(ixdim): + indexable_area_shape += (ref_shape[i],) + all_indices = [ + list(coord) for coord, _ in np.ndenumerate( + np.empty(indexable_area_shape, vtype)) + ] + np.random.shuffle(all_indices) + indices = np.array(all_indices[:num_updates]) + + if num_updates > 1 and repeat_indices: + indices = indices[:num_updates // 2] + for _ in range(num_updates - num_updates // 2): + indices = np.append( + indices, [indices[np.random.randint(num_updates // 2)]], axis=0) + np.random.shuffle(indices) + indices = _AsType(indices[:num_updates], itype) + + updates_shape = (num_updates,) + for i in range(ixdim, len(ref_shape)): + updates_shape += (ref_shape[i],) + updates = _AsType(np.random.randn(*(updates_shape)), vtype) + ref = _AsType(np.random.randn(*(ref_shape)), vtype) + + # Scatter via numpy + new = ref.copy() + np_scatter(new, indices, updates) + # Scatter via tensorflow + ref_var = variables.VariableV1(ref) + self.evaluate(ref_var.initializer) + self.evaluate(tf_scatter(ref_var, indices, updates)) + + # Compare + self.assertAllClose(new, self.evaluate(ref_var)) + + def _ScatterRepeatIndicesTest(self, np_scatter, tf_scatter): + for vtype in (np.int32, np.float16, np.float32, np.float64): + for itype in (np.int32, np.int64): + self._VariableRankTest( + np_scatter, tf_scatter, vtype, itype, repeat_indices=True) + + @test_util.run_v1_only("Don't need to test VariableV1 in TF2") + def testScatterRepeatIndicesMinMax(self): + """This tests scatter_add using indices that repeat.""" + self._ScatterRepeatIndicesTest(_NumpyMin, state_ops.scatter_nd_min) + self._ScatterRepeatIndicesTest(_NumpyMax, state_ops.scatter_nd_max) + + @test_util.run_v1_only("Don't need to test VariableV1 in TF2") + def testScatterOutOfRangeCpu(self): + for op in (state_ops.scatter_nd_min, state_ops.scatter_nd_max): + params = np.array([1, 2, 3, 4, 5, 6]).astype(np.float32) + updates = np.array([-3, -4, -5]).astype(np.float32) + with self.cached_session(use_gpu=False): + ref = variables.VariableV1(params) + self.evaluate(ref.initializer) + + # Indices all in range, no problem. + indices = np.array([[2], [0], [5]]) + self.evaluate(op(ref, indices, updates)) + + # Test some out of range errors. + indices = np.array([[-1], [0], [5]]) + with self.assertRaisesOpError( + r"indices\[0\] = \[-1\] does not index into shape \[6\]"): + op(ref, indices, updates).eval() + + indices = np.array([[2], [0], [6]]) + with self.assertRaisesOpError( + r"indices\[2\] = \[6\] does not index into shape \[6\]"): + op(ref, indices, updates).eval() From 2f93ec916b844309b3544e278995fd57a1e049be Mon Sep 17 00:00:00 2001 From: Jose Baiocchi Date: Sat, 8 Aug 2020 08:17:04 -0700 Subject: [PATCH 0706/1017] Mark TraceMe(string&&) as deleted PiperOrigin-RevId: 325600691 Change-Id: Icde5cd4515e4feb31ab90465c8da15a370b6ad28 --- tensorflow/core/profiler/lib/traceme.h | 27 +++++++++----------------- 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/tensorflow/core/profiler/lib/traceme.h b/tensorflow/core/profiler/lib/traceme.h index 64103d95215..526f6d5104d 100644 --- a/tensorflow/core/profiler/lib/traceme.h +++ b/tensorflow/core/profiler/lib/traceme.h @@ -97,30 +97,21 @@ class TraceMe { #endif } - // string&& constructor to prevent an unnecessary string copy, e.g. when a - // TraceMe is constructed based on the result of a StrCat operation. - // Note: We can't take the string by value because a) it would make the - // overloads ambiguous, and b) we want lvalue strings to use the string_view - // constructor so we avoid copying them when tracing is disabled. - explicit TraceMe(std::string&& name, int level = 1) { - DCHECK_GE(level, 1); -#if !defined(IS_MOBILE_PLATFORM) - if (TF_PREDICT_FALSE(TraceMeRecorder::Active(level))) { - new (&no_init_.name) std::string(std::move(name)); - start_time_ = EnvTime::NowNanos(); - } -#endif - } + // Do not allow passing a temporary string as the overhead of generating that + // string should only be incurred when tracing is enabled. Wrap the temporary + // string generation (e.g., StrCat) in a lambda and use the name_generator + // template instead. + explicit TraceMe(std::string&& name, int level = 1) = delete; // Do not allow passing strings by reference or value since the caller // may unintentionally maintain ownership of the name. - // Explicitly std::move the name or wrap it in a string_view if - // you really wish to maintain ownership. + // Explicitly wrap the name in a string_view if you really wish to maintain + // ownership of a string already generated for other purposes. For temporary + // strings (e.g., result of StrCat) use the name_generator template. explicit TraceMe(const std::string& name, int level = 1) = delete; // This overload is necessary to make TraceMe's with string literals work. - // Otherwise, the string&& and the string_view constructor would be equally - // good overload candidates. + // Otherwise, the name_generator template would be used. explicit TraceMe(const char* raw, int level = 1) : TraceMe(absl::string_view(raw), level) {} From 79b77e8bfd95d614f5fabea72ad06d18afda284c Mon Sep 17 00:00:00 2001 From: David Majnemer Date: Sat, 8 Aug 2020 13:07:42 -0700 Subject: [PATCH 0707/1017] Fix C++14 build. PiperOrigin-RevId: 325618095 Change-Id: Ib9d47e20a8ab0e432948cf275211725936cec0ce --- tensorflow/compiler/xla/array.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h index 0f31d4c27f5..a85d551769c 100644 --- a/tensorflow/compiler/xla/array.h +++ b/tensorflow/compiler/xla/array.h @@ -297,8 +297,8 @@ class Array { std::mt19937 g(seed); std::normal_distribution distribution(mean, stddev); for (int64 i = 0; i < num_elements(); ++i) { - if constexpr (std::is_same()) { - values_[i] = distribution(g) > 0.0; + if (std::is_same()) { + values_[i] = static_cast(distribution(g) > 0.0); } else { values_[i] = static_cast(distribution(g)); } From 450ce2e30577daec1789d6aee944c8c7799a86ac Mon Sep 17 00:00:00 2001 From: Xingyu Long Date: Sat, 8 Aug 2020 16:51:30 -0400 Subject: [PATCH 0708/1017] Update the GPU description in setup part --- .../python/keras/benchmarks/keras_examples_benchmarks/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/README.md b/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/README.md index 3c34dbc68ab..b7a16c516c0 100644 --- a/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/README.md +++ b/tensorflow/python/keras/benchmarks/keras_examples_benchmarks/README.md @@ -49,7 +49,7 @@ These examples are implemented by Functional API and Sequential API. The listed benchmark results are obtained by running on Google Cloud Platform (GCP) with the following setup:
-- GPU: 2 x Tesla V100 (only for GPU test)
+- GPU: 2 x Tesla V100
- OS: Ubuntu 18.04
- CPU: 8 x vCPUs, 30 GB memory
- CUDA: 10.1
From ddba76d1f27f383bfc05c5ecea03a8ebea2f87d8 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Sat, 8 Aug 2020 18:02:41 -0700 Subject: [PATCH 0709/1017] C++17 build without linking libc++ Based on #23561 and #41710, trying to see if this would enable building on C++17 without also linking in `libc++` (which is a Clang lib, does not come from a default GCC install) --- .bazelrc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.bazelrc b/.bazelrc index ddeb2515d70..c15ce04fdfb 100644 --- a/.bazelrc +++ b/.bazelrc @@ -278,6 +278,8 @@ build:dynamic_kernels --copt=-DAUTOLOAD_DYNAMIC_KERNELS build:c++17 --cxxopt=-std=c++1z build:c++17 --cxxopt=-stdlib=libc++ build:c++1z --config=c++17 +build:c++17_gcc --cxxopt=-std=c++1z +build:c++1z_gcc --config=c++17_gcc # Enable using platform specific build settings, except when cross-compiling for # mobile platforms. From 9f62efeba92a72d6696fba46b1a55aa983f8338b Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Sat, 8 Aug 2020 18:06:25 -0700 Subject: [PATCH 0710/1017] Add documentation of the new options --- .bazelrc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index c15ce04fdfb..1b9f5e87c6b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -18,8 +18,10 @@ # # Compiler options: # cuda_clang: Use clang when building CUDA code. -# c++17: Build with C++17 options -# c++1z: Build with C++17 options +# c++17: Build with C++17 options (links with libc++) +# c++1z: Build with C++17 options (links with libc++) +# c++17_gcc: Build with C++17 options (links with stdlibc++) +# c++1z_gcc: Build with C++17 options (links with stdlibc++) # avx_linux: Build with avx instruction set on linux. # avx2_linux: Build with avx2 instruction set on linux. # native_arch_linux: Build with instruction sets available to the host machine on linux From cbe30de1d255ad46932aca35f51af30dfb3dd7ee Mon Sep 17 00:00:00 2001 From: Kibeom Kim Date: Sat, 8 Aug 2020 18:34:51 -0700 Subject: [PATCH 0711/1017] Enable more tfrt tests(311) that are passing. PiperOrigin-RevId: 325637415 Change-Id: Ied4cb5e4149f6ad60b5dd0eecdb4c34f87e70ca0 --- tensorflow/python/BUILD | 128 ++++++++++++++++++++ tensorflow/python/eager/BUILD | 8 ++ tensorflow/python/kernel_tests/BUILD | 175 +++++++++++++++++++++++++++ 3 files changed, 311 insertions(+) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 039fc945eca..4efe769c59d 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -347,6 +347,7 @@ tf_py_test( "no_pip", "no_windows", ], + tfrt_enabled = True, deps = [ ":platform", ":platform_test", @@ -365,6 +366,7 @@ tf_py_test( "no_pip", "no_windows", ], + tfrt_enabled = True, deps = [ ":platform", ":platform_test", @@ -376,6 +378,7 @@ tf_py_test( size = "small", srcs = ["platform/flags_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":platform", @@ -391,6 +394,7 @@ tf_py_test( "no_windows", "nomac", ], + tfrt_enabled = True, deps = [ ":client_testlib", ":platform", @@ -1148,6 +1152,7 @@ tf_py_test( name = "decorator_utils_test", srcs = ["util/decorator_utils_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":platform", @@ -1159,6 +1164,7 @@ tf_py_test( name = "deprecation_test", srcs = ["util/deprecation_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":platform", @@ -1170,6 +1176,7 @@ tf_py_test( name = "dispatch_test", srcs = ["util/dispatch_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":platform", @@ -1181,6 +1188,7 @@ tf_py_test( name = "keyword_args_test", srcs = ["util/keyword_args_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":util", @@ -1518,6 +1526,7 @@ tf_py_test( srcs = ["framework/function_def_to_graph_test.py"], python_version = "PY3", tags = ["no_pip"], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -1663,6 +1672,7 @@ tf_py_test( srcs = ["framework/op_def_util_test.py"], python_version = "PY3", tags = ["no_pip"], + tfrt_enabled = True, ) py_library( @@ -1875,6 +1885,7 @@ tf_py_test( size = "small", srcs = ["framework/smart_cond_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":constant_op", @@ -1947,6 +1958,7 @@ tf_py_test( srcs = ["framework/composite_tensor_utils_test.py"], main = "framework/composite_tensor_utils_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":composite_tensor", @@ -2204,6 +2216,7 @@ tf_py_test( srcs = ["framework/registry_test.py"], main = "framework/registry_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -2217,6 +2230,7 @@ tf_py_test( srcs = ["framework/errors_test.py"], main = "framework/errors_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":errors", @@ -2230,6 +2244,7 @@ tf_py_test( srcs = ["framework/error_interpolation_test.py"], main = "framework/error_interpolation_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":constant_op", @@ -2244,6 +2259,7 @@ tf_py_test( srcs = ["framework/subscribe_test.py"], main = "framework/subscribe_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework", ":framework_for_generated_wrappers", @@ -2286,6 +2302,7 @@ tf_py_test( tags = [ "no_pip", ], + tfrt_enabled = True, deps = [ ":client_testlib", ":platform", @@ -2298,6 +2315,7 @@ tf_py_test( srcs = ["framework/proto_test.py"], main = "framework/proto_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -2386,6 +2404,7 @@ tf_py_test( srcs = ["framework/versions_test.py"], main = "framework/versions_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -2398,6 +2417,7 @@ tf_py_test( srcs = ["framework/importer_test.py"], main = "framework/importer_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -2434,6 +2454,7 @@ tf_py_test( "no_pip", "no_windows", ], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -2457,6 +2478,7 @@ tf_py_test( srcs = ["framework/traceable_stack_test.py"], main = "framework/traceable_stack_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_test_lib", ":platform_test", @@ -2511,6 +2533,7 @@ tf_py_test( srcs = ["framework/common_shapes_test.py"], main = "framework/common_shapes_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework", ":framework_for_generated_wrappers", @@ -2557,6 +2580,7 @@ tf_py_test( srcs = ["framework/ops_enable_eager_test.py"], main = "framework/ops_enable_eager_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework", ":platform_test", @@ -2570,6 +2594,7 @@ tf_py_test( srcs = ["framework/tensor_shape_test.py"], main = "framework/tensor_shape_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -2585,6 +2610,7 @@ tf_py_test( srcs = ["framework/type_spec_test.py"], main = "framework/type_spec_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -2600,6 +2626,7 @@ tf_py_test( srcs = ["framework/tensor_spec_test.py"], main = "framework/tensor_spec_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -2632,6 +2659,7 @@ tf_py_test( srcs = ["framework/device_spec_test.py"], main = "framework/device_spec_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -2646,6 +2674,7 @@ tf_py_test( srcs = ["framework/device_test.py"], main = "framework/device_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -2660,6 +2689,7 @@ tf_py_test( srcs = ["framework/random_seed_test.py"], main = "framework/random_seed_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":framework", @@ -2672,6 +2702,7 @@ tf_py_test( srcs = ["framework/tensor_shape_div_test.py"], main = "framework/tensor_shape_div_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -2688,6 +2719,7 @@ tf_py_test( main = "framework/tensor_util_test.py", python_version = "PY3", tags = ["no_windows"], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -2707,6 +2739,7 @@ tf_py_test( main = "framework/test_util_test.py", python_version = "PY3", tags = ["no_windows"], + tfrt_enabled = True, deps = [ ":control_flow_ops", ":errors", @@ -2741,6 +2774,7 @@ tf_py_test( "nomsan", # TODO(b/149948895): Re-enable. "notsan", # TODO(b/149948895): Re-enable. ], + tfrt_enabled = True, deps = [ ":framework_test_lib", # TODO(kkb): Find more appropriate place to add `memory_checker` as deps @@ -2766,6 +2800,7 @@ tf_py_test( srcs = ["framework/dtypes_test.py"], main = "framework/dtypes_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -2781,6 +2816,7 @@ tf_py_test( size = "small", srcs = ["framework/op_def_library_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -2794,6 +2830,7 @@ tf_py_test( srcs = ["framework/kernels_test.py"], main = "framework/kernels_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_test_lib", ":kernels", @@ -3319,6 +3356,7 @@ tf_py_test( size = "small", srcs = ["ops/clip_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":clip_ops", @@ -3344,6 +3382,7 @@ tf_py_test( size = "medium", srcs = ["ops/clustering_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":clustering_ops", @@ -3368,6 +3407,7 @@ tf_py_test( srcs = ["ops/collective_ops_test.py"], python_version = "PY3", tags = ["no_rocm"], + tfrt_enabled = True, deps = [ ":client_testlib", ":collective_ops", @@ -3388,6 +3428,7 @@ tf_py_test( "no_windows", "nomac", ], + tfrt_enabled = True, xla_enable_strict_auto_jit = True, deps = [ ":client_testlib", @@ -3409,6 +3450,7 @@ cuda_py_test( "no_rocm", "no_windows", ], + tfrt_enabled = True, deps = [ ":client_testlib", ":collective_ops", @@ -3512,6 +3554,7 @@ tf_py_test( size = "small", srcs = ["ops/control_flow_v2_toggles_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":control_flow_util_v2", @@ -3525,6 +3568,7 @@ tf_py_test( size = "small", srcs = ["ops/control_flow_v2_enable_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":control_flow_util", @@ -3546,6 +3590,7 @@ tf_py_test( "no_oss", "no_pip", ], + tfrt_enabled = True, deps = [ ":client_testlib", ":control_flow_util", @@ -3627,6 +3672,7 @@ tf_py_test( size = "small", srcs = ["ops/bincount_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":bincount_ops", ":platform_test", @@ -4108,6 +4154,7 @@ cuda_py_test( size = "small", srcs = ["training/experimental/mixed_precision_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":mixed_precision", @@ -4426,6 +4473,7 @@ cuda_py_test( size = "medium", srcs = ["ops/stateful_random_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, xla_enable_strict_auto_jit = False, xla_enabled = True, deps = [ @@ -4597,6 +4645,7 @@ tf_py_test( name = "sort_ops_test", srcs = ["ops/sort_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -4696,6 +4745,7 @@ cuda_py_test( name = "rnn_grad_test", srcs = ["ops/rnn_grad_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -4978,6 +5028,7 @@ cuda_py_test( srcs = ["ops/bitwise_ops_test.py"], python_version = "PY3", tags = ["no_windows"], + tfrt_enabled = True, deps = [ ":bitwise_ops", ":constant_op", @@ -5089,6 +5140,7 @@ cuda_py_test( size = "small", srcs = ["ops/histogram_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -5105,6 +5157,7 @@ cuda_py_test( size = "medium", srcs = ["ops/image_grad_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -5146,6 +5199,7 @@ cuda_py_test( size = "small", srcs = ["ops/init_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_ops", @@ -5161,6 +5215,7 @@ cuda_py_test( size = "medium", srcs = ["ops/init_ops_v2_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -5231,6 +5286,7 @@ cuda_py_test( python_version = "PY3", shard_count = 4, tags = ["no_windows"], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -5250,6 +5306,7 @@ cuda_py_test( srcs = ["ops/nn_fused_batchnorm_test.py"], python_version = "PY3", shard_count = 24, + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -5300,6 +5357,7 @@ cuda_py_test( size = "medium", srcs = ["ops/nn_xent_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -5333,6 +5391,7 @@ cuda_py_test( "no_oss", # TODO(b/149565560) "no_windows_gpu", ], + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -5369,6 +5428,7 @@ tf_py_test( size = "small", srcs = ["ops/variable_spec_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":framework_for_generated_wrappers", ":framework_test_lib", @@ -5639,6 +5699,7 @@ tf_py_test( name = "tf_export_test", srcs = ["util/tf_export_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":platform", @@ -5696,6 +5757,7 @@ tf_py_test( name = "tf_stack_test", srcs = ["util/tf_stack_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":tf_export", @@ -5768,6 +5830,7 @@ tf_py_test( size = "small", srcs = ["util/object_identity_test.py"], python_version = "PY3", + tfrt_enabled = True, ) # Placeholder for intenal nest_test comments. @@ -5777,6 +5840,7 @@ tf_py_test( srcs = ["util/nest_test.py"], main = "util/nest_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [":util_nest_test_main_lib"], ) @@ -5802,6 +5866,7 @@ tf_py_test( srcs = ["util/serialization_test.py"], main = "util/serialization_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":util", @@ -5812,6 +5877,7 @@ tf_py_test( name = "function_utils_test", srcs = ["util/function_utils_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":util", @@ -5823,6 +5889,7 @@ tf_py_test( size = "small", srcs = ["util/tf_contextlib_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":util", @@ -5834,6 +5901,7 @@ tf_py_test( size = "small", srcs = ["util/tf_decorator_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":util", @@ -5857,6 +5925,7 @@ tf_py_test( size = "small", srcs = ["util/tf_should_use_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":tf_should_use", @@ -5868,6 +5937,7 @@ tf_py_test( size = "small", srcs = ["util/tf_inspect_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":util", @@ -5892,6 +5962,7 @@ tf_py_test( srcs = ["util/lock_util_test.py"], main = "util/lock_util_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":util", @@ -5904,6 +5975,7 @@ tf_py_test( size = "small", srcs = ["util/module_wrapper_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":util", @@ -5946,6 +6018,7 @@ tf_py_test( main = "util/protobuf/compare_test.py", python_version = "PY3", tags = ["no_pip"], # compare_test_pb2 proto is not available in pip. + tfrt_enabled = True, deps = [ ":compare_test_proto_py", ":platform_test", @@ -5960,6 +6033,7 @@ tf_py_test( srcs = ["util/example_parser_configuration_test.py"], main = "util/example_parser_configuration_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -5975,6 +6049,7 @@ tf_py_test( size = "small", srcs = ["client/events_writer_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":errors", ":framework_test_lib", @@ -6396,6 +6471,7 @@ tf_py_test( tags = [ "noasan", # TODO(b/161236904): flaky timeout in trying to start gRPC server ], + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -6417,6 +6493,7 @@ tf_py_test( srcs = ["training/server_lib_multiple_containers_test.py"], grpc_enabled = True, python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -6438,6 +6515,7 @@ tf_py_test( srcs = ["training/server_lib_same_variables_clear_container_test.py"], grpc_enabled = True, python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -6459,6 +6537,7 @@ tf_py_test( srcs = ["training/server_lib_same_variables_clear_test.py"], grpc_enabled = True, python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -6480,6 +6559,7 @@ tf_py_test( srcs = ["training/server_lib_same_variables_no_clear_test.py"], grpc_enabled = True, python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -6501,6 +6581,7 @@ tf_py_test( srcs = ["training/server_lib_sparse_job_test.py"], grpc_enabled = True, python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -6528,6 +6609,7 @@ cuda_py_test( "no_oss", # Test flaky due to port collisions. "oss_serial", ], + tfrt_enabled = True, deps = [ ":client", ":client_testlib", @@ -6554,6 +6636,7 @@ tf_py_test( "notsan", # data race due to b/62910646 "oss_serial", ], + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -6596,6 +6679,7 @@ tf_py_test( "no_pip_gpu", # testInteractivePlacePrunedGraph fails on invalid assumption about GPU ops. "no_windows", ], + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -6658,6 +6742,7 @@ tf_py_test( "no_pip_gpu", "notsan", # data race due to b/62910646 ], + tfrt_enabled = True, deps = [ ":client", ":framework", @@ -6677,6 +6762,7 @@ tf_py_test( "no_gpu", "no_windows", ], + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -6701,6 +6787,7 @@ cuda_py_test( "gpu_cupti", "no_gpu", # b/154742661 ], + tfrt_enabled = True, xla_enable_strict_auto_jit = False, # Graph structure is different with autojit deps = [ ":client", @@ -6720,6 +6807,7 @@ cuda_py_test( "no_gpu", # b/127386241 "no_windows_gpu", ], + tfrt_enabled = True, deps = [ ":client", ":client_testlib", @@ -6734,6 +6822,7 @@ tf_py_test( size = "small", srcs = ["framework/c_api_util_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":c_api_util", ":framework_test_lib", @@ -6746,6 +6835,7 @@ tf_py_test( size = "small", srcs = ["framework/graph_util_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client", ":client_testlib", @@ -6766,6 +6856,7 @@ tf_py_test( srcs = ["framework/convert_to_constants_test.py"], python_version = "PY3", tags = ["no_rocm"], + tfrt_enabled = True, deps = [ ":client_testlib", ":control_flow_v2_toggles", @@ -6780,6 +6871,7 @@ tf_py_test( size = "small", srcs = ["lib/core/bfloat16_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":lib", @@ -6796,6 +6888,7 @@ tf_py_test( "no_rocm", "no_windows", ], + tfrt_enabled = True, deps = [ ":client_testlib", ":errors", @@ -6808,6 +6901,7 @@ tf_py_test( size = "small", srcs = ["lib/io/tf_record_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":errors", @@ -6844,6 +6938,7 @@ cuda_py_test( "no_windows", # b/139083295: bfloat16 tests fail on Windows "notsan", ], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -7014,6 +7109,7 @@ tf_py_test( "noasan", # http://b/30379628 "notsan", # http://b/30379628 ], + tfrt_enabled = True, deps = [ ":client", ":client_testlib", @@ -7034,6 +7130,7 @@ tf_py_test( "noasan", # http://b/30782289 "notsan", # http://b/30782289 ], + tfrt_enabled = True, deps = [ ":client", ":client_testlib", @@ -7051,6 +7148,7 @@ cuda_py_test( grpc_enabled = True, main = "training/session_manager_test.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -7071,6 +7169,7 @@ tf_py_test( grpc_enabled = True, python_version = "PY3", tags = ["no_windows"], + tfrt_enabled = True, deps = [ ":array_ops", ":checkpoint_management", @@ -7099,6 +7198,7 @@ tf_py_test( "no_windows", "notsan", # intermittent races on a few percent of runs ], + tfrt_enabled = True, deps = [ ":client", ":client_testlib", @@ -7149,6 +7249,7 @@ tf_py_test( size = "small", srcs = ["training/checkpoint_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":checkpoint_ops_gen", ":client", @@ -7170,6 +7271,7 @@ tf_py_test( size = "medium", srcs = ["training/warm_starting_util_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -7191,6 +7293,7 @@ tf_py_test( "no_pip", "notsan", # b/67945581 ], + tfrt_enabled = True, deps = [ ":array_ops", ":checkpoint_management", @@ -7235,6 +7338,7 @@ tf_py_test( size = "small", srcs = ["training/training_util_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":framework", @@ -7250,6 +7354,7 @@ tf_py_test( size = "medium", srcs = ["training/input_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -7400,6 +7505,7 @@ tf_py_test( srcs = ["ops/dequantize_op_test.py"], python_version = "PY3", tags = ["no_windows"], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -7414,6 +7520,7 @@ tf_py_test( srcs = ["ops/quantized_ops_test.py"], python_version = "PY3", tags = ["no_windows"], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -7428,6 +7535,7 @@ tf_py_test( srcs = ["ops/quantized_conv_ops_test.py"], python_version = "PY3", tags = ["no_windows"], + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -7461,6 +7569,7 @@ cuda_py_test( main = "ops/accumulate_n_benchmark.py", python_version = "PY3", shard_count = 6, + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -7480,6 +7589,7 @@ cuda_py_test( srcs = ["ops/batch_norm_benchmark.py"], main = "ops/batch_norm_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -7501,6 +7611,7 @@ cuda_py_test( srcs = ["ops/collective_ops_benchmark.py"], main = "ops/collective_ops_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -7518,6 +7629,7 @@ cuda_py_test( srcs = ["ops/concat_benchmark.py"], main = "ops/concat_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -7536,6 +7648,7 @@ cuda_py_test( srcs = ["ops/control_flow_ops_benchmark.py"], main = "ops/control_flow_ops_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":constant_op", @@ -7551,6 +7664,7 @@ cuda_py_test( srcs = ["ops/conv2d_benchmark.py"], main = "ops/conv2d_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":client", ":client_testlib", @@ -7571,6 +7685,7 @@ cuda_py_test( srcs = ["ops/split_benchmark.py"], main = "ops/split_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -7591,6 +7706,7 @@ cuda_py_test( srcs = ["ops/transpose_benchmark.py"], main = "ops/transpose_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -7611,6 +7727,7 @@ cuda_py_test( srcs = ["ops/matmul_benchmark.py"], main = "ops/matmul_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [":matmul_benchmark_main_lib"], ) @@ -7640,6 +7757,7 @@ cuda_py_test( grpc_enabled = True, main = "client/session_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client", @@ -7658,6 +7776,7 @@ cuda_py_test( srcs = ["framework/graph_building_benchmark.py"], main = "framework/graph_building_benchmark.py", python_version = "PY3", + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -7673,6 +7792,7 @@ cuda_py_test( size = "medium", srcs = ["ops/nn_grad_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -7727,6 +7847,7 @@ tf_py_test( "grappler", "no_pip", # tf_optimizer is not available in pip. ], + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -7747,6 +7868,7 @@ tf_py_test( "grappler", "no_pip", # tf_optimizer is not available in pip. ], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -7865,6 +7987,7 @@ tf_py_test( "grappler", "no_pip", # tf_optimizer is not available in pip. ], + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -7886,6 +8009,7 @@ tf_py_test( tags = [ "grappler", ], + tfrt_enabled = True, deps = [ ":client_testlib", ":framework_for_generated_wrappers", @@ -8021,6 +8145,7 @@ tf_py_test( "no_pip", "no_windows", # TODO(b/151942037) ], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -8055,6 +8180,7 @@ tf_py_test( "grappler", "no_pip", ], + tfrt_enabled = True, deps = [ ":array_ops", ":client_testlib", @@ -8075,6 +8201,7 @@ cuda_py_test( ], python_version = "PY3", tags = ["grappler"], + tfrt_enabled = True, # This test analyzes the graph, but XLA changes the names of nodes. xla_enable_strict_auto_jit = False, deps = [ @@ -8330,6 +8457,7 @@ cuda_py_test( name = "raw_ops_test", srcs = ["ops/raw_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":client_testlib", ], diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 3c0c3894a64..358929dc870 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -144,6 +144,7 @@ cuda_py_test( size = "small", srcs = ["cancellation_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":cancellation", ":test", @@ -250,6 +251,7 @@ cuda_py_test( name = "monitoring_test", srcs = ["monitoring_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":monitoring", ":test", @@ -393,6 +395,7 @@ cuda_py_test( size = "medium", srcs = ["function_argument_naming_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":backprop", ":def_function", @@ -408,6 +411,7 @@ cuda_py_test( size = "medium", srcs = ["function_defun_collection_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":backprop", ":def_function", @@ -524,6 +528,7 @@ cuda_py_test( name = "graph_only_ops_test", srcs = ["graph_only_ops_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ "graph_only_ops", "//tensorflow/python:client_testlib", @@ -670,6 +675,7 @@ cuda_py_test( name = "remote_benchmarks_test", srcs = ["remote_benchmarks_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":backprop", ":benchmarks_test_base", @@ -695,6 +701,7 @@ tf_py_test( name = "tape_test", srcs = ["tape_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":backprop", ":context", @@ -803,6 +810,7 @@ tf_py_test( size = "medium", srcs = ["lift_to_graph_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ "lift_to_graph", "//tensorflow/python:framework_ops", diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 91d1cd4c4c9..e73f2ea29fc 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -20,6 +20,7 @@ tf_py_test( size = "small", srcs = ["as_string_op_test.py"], tags = ["no_windows"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -33,6 +34,7 @@ tf_py_test( name = "attention_ops_test", size = "small", srcs = ["attention_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -52,6 +54,7 @@ tf_py_test( "nomsan", # TODO(b/161902335): Re-enable. "notsan", # TODO(b/161829717): Re-enable. ], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:data_flow_ops", @@ -66,6 +69,7 @@ tf_py_test( size = "small", srcs = ["base64_ops_test.py"], tags = ["nomac"], # b/35468214 + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -80,6 +84,7 @@ tf_py_test( tf_py_test( name = "batch_scatter_ops_test", srcs = ["batch_scatter_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -100,6 +105,7 @@ tf_py_test( name = "bcast_ops_test", size = "small", srcs = ["bcast_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops_gen", "//tensorflow/python:client_testlib", @@ -157,6 +163,7 @@ cuda_py_test( size = "small", srcs = ["benchmark_test.py"], tags = ["no_windows"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client", "//tensorflow/python:client_testlib", @@ -169,6 +176,7 @@ cuda_py_test( cuda_py_test( name = "reduce_benchmark_test", srcs = ["reduce_benchmark_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -187,6 +195,7 @@ cuda_py_test( size = "small", srcs = ["bincount_op_test.py"], tags = ["no_windows_gpu"], + tfrt_enabled = True, deps = [ "//tensorflow/python:bincount_ops", "//tensorflow/python:client_testlib", @@ -198,6 +207,7 @@ tf_py_test( name = "candidate_sampler_ops_test", size = "small", srcs = ["candidate_sampler_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:candidate_sampling_ops", @@ -212,6 +222,7 @@ tf_py_test( name = "checkpoint_ops_test", size = "medium", srcs = ["checkpoint_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:checkpoint_ops_gen", @@ -259,6 +270,7 @@ tf_py_test( "no_gpu", # b/127001953 "no_windows", ], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:clip_ops", @@ -270,6 +282,7 @@ tf_py_test( name = "collective_ops_test", size = "small", srcs = ["collective_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:collective_ops_gen", @@ -281,6 +294,7 @@ tf_py_test( name = "conditional_accumulator_test", size = "small", srcs = ["conditional_accumulator_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -298,6 +312,7 @@ tf_py_test( name = "ctc_decoder_ops_test", size = "small", srcs = ["ctc_decoder_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -338,6 +353,7 @@ cuda_py_test( name = "cudnn_deterministic_ops_test", size = "small", srcs = ["cudnn_deterministic_ops_test.py"], + tfrt_enabled = True, xla_enable_strict_auto_jit = True, deps = [ ":cudnn_deterministic_base", @@ -348,6 +364,7 @@ cuda_py_test( name = "cudnn_deterministic_test", size = "small", srcs = ["cudnn_deterministic_test.py"], + tfrt_enabled = True, deps = [ ":cudnn_deterministic_base", ], @@ -357,6 +374,7 @@ cuda_py_test( name = "cumulative_logsumexp_test", size = "medium", srcs = ["cumulative_logsumexp_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -372,6 +390,7 @@ tf_py_test( name = "decode_csv_op_test", size = "small", srcs = ["decode_csv_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:errors", @@ -386,6 +405,7 @@ tf_py_test( name = "decode_png_op_test", size = "small", srcs = ["decode_png_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -399,6 +419,7 @@ tf_py_test( name = "decode_bmp_op_test", size = "small", srcs = ["decode_bmp_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -412,6 +433,7 @@ tf_py_test( name = "decode_jpeg_op_test", srcs = ["decode_jpeg_op_test.py"], data = ["//tensorflow/core:image_testdata"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -425,6 +447,7 @@ tf_py_test( size = "small", srcs = ["decode_image_op_test.py"], data = ["//tensorflow/core:image_testdata"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:errors", @@ -439,6 +462,7 @@ tf_py_test( name = "decode_raw_op_test", size = "small", srcs = ["decode_raw_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -452,6 +476,7 @@ tf_py_test( name = "decode_compressed_op_test", size = "small", srcs = ["decode_compressed_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -465,6 +490,7 @@ cuda_py_test( name = "determinant_op_test", size = "medium", srcs = ["determinant_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -477,6 +503,7 @@ tf_py_test( name = "draw_bounding_box_op_test", size = "small", srcs = ["draw_bounding_box_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -491,6 +518,7 @@ tf_py_test( name = "edit_distance_op_test", size = "small", srcs = ["edit_distance_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -522,6 +550,7 @@ tf_py_test( name = "fingerprint_op_test", size = "small", srcs = ["fingerprint_op_test.py"], + tfrt_enabled = True, deps = [ "//third_party/py/numpy", ], @@ -532,6 +561,7 @@ tf_py_test( size = "small", srcs = ["fractional_avg_pool_op_test.py"], shard_count = 5, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -548,6 +578,7 @@ tf_py_test( size = "small", srcs = ["fractional_max_pool_op_test.py"], shard_count = 5, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -563,6 +594,7 @@ tf_py_test( name = "identity_op_py_test", size = "small", srcs = ["identity_op_py_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:array_ops_gen", @@ -577,6 +609,7 @@ tf_py_test( name = "identity_n_op_py_test", size = "small", srcs = ["identity_n_op_py_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:array_ops_gen", @@ -591,6 +624,7 @@ cuda_py_test( name = "in_topk_op_test", size = "small", srcs = ["in_topk_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:errors", @@ -603,6 +637,7 @@ tf_py_test( name = "record_input_test", size = "medium", srcs = ["record_input_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:data_flow_ops", @@ -615,6 +650,7 @@ tf_py_test( name = "io_ops_test", size = "small", srcs = ["io_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:io_ops", @@ -626,6 +662,7 @@ tf_py_test( name = "listdiff_op_test", size = "small", srcs = ["listdiff_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -642,6 +679,7 @@ tf_py_test( tags = [ "no_windows", ], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -688,6 +726,7 @@ tf_py_test( name = "losses_test", size = "medium", srcs = ["losses_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -710,6 +749,7 @@ tf_py_test( srcs = ["matrix_exponential_op_test.py"], shard_count = 16, tags = ["no_windows_gpu"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -787,6 +827,7 @@ cuda_py_test( name = "banded_triangular_solve_op_test", size = "small", srcs = ["banded_triangular_solve_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:linalg_ops", @@ -799,6 +840,7 @@ cuda_py_test( size = "medium", srcs = ["matrix_triangular_solve_op_test.py"], shard_count = 3, + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:linalg_ops", @@ -827,6 +869,7 @@ tf_py_test( name = "parse_single_example_op_test", size = "small", srcs = ["parse_single_example_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -844,6 +887,7 @@ tf_py_test( name = "partitioned_variables_test", size = "small", srcs = ["partitioned_variables_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -861,6 +905,7 @@ tf_py_test( name = "priority_queue_test", size = "medium", srcs = ["priority_queue_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -896,6 +941,7 @@ tf_py_test( name = "regex_replace_op_test", size = "small", srcs = ["regex_replace_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -909,6 +955,7 @@ tf_py_test( name = "regex_full_match_op_test", size = "small", srcs = ["regex_full_match_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -970,6 +1017,7 @@ tf_py_test( name = "sparse_add_op_test", size = "small", srcs = ["sparse_add_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client", "//tensorflow/python:client_testlib", @@ -986,6 +1034,7 @@ tf_py_test( name = "sparse_concat_op_test", size = "small", srcs = ["sparse_concat_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1000,6 +1049,7 @@ tf_py_test( name = "sparse_conditional_accumulator_test", size = "small", srcs = ["sparse_conditional_accumulator_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1014,6 +1064,7 @@ tf_py_test( name = "sparse_reorder_op_test", size = "small", srcs = ["sparse_reorder_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1029,6 +1080,7 @@ tf_py_test( name = "sparse_reshape_op_test", size = "small", srcs = ["sparse_reshape_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1044,6 +1096,7 @@ tf_py_test( name = "sparse_split_op_test", size = "small", srcs = ["sparse_split_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework", @@ -1056,6 +1109,7 @@ tf_py_test( name = "sparse_slice_op_test", size = "small", srcs = ["sparse_slice_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework", @@ -1069,6 +1123,7 @@ tf_py_test( name = "sparse_to_dense_op_py_test", size = "small", srcs = ["sparse_to_dense_op_py_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1082,6 +1137,7 @@ tf_py_test( name = "sparsemask_op_test", size = "small", srcs = ["sparsemask_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1107,6 +1163,7 @@ tf_py_test( name = "string_join_op_test", size = "small", srcs = ["string_join_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:string_ops", @@ -1117,6 +1174,7 @@ tf_py_test( name = "string_split_op_test", size = "small", srcs = ["string_split_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1136,6 +1194,7 @@ tf_py_test( name = "string_bytes_split_op_test", size = "small", srcs = ["string_bytes_split_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1155,6 +1214,7 @@ tf_py_test( name = "string_length_op_test", size = "small", srcs = ["string_length_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -1166,6 +1226,7 @@ tf_py_test( name = "string_strip_op_test", size = "small", srcs = ["string_strip_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1180,6 +1241,7 @@ tf_py_test( name = "string_lower_op_test", size = "small", srcs = ["string_lower_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1194,6 +1256,7 @@ tf_py_test( name = "string_upper_op_test", size = "small", srcs = ["string_upper_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1208,6 +1271,7 @@ tf_py_test( name = "substr_op_test", size = "small", srcs = ["substr_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:errors", @@ -1246,6 +1310,7 @@ tf_py_test( name = "summary_v1_ops_test", size = "small", srcs = ["summary_v1_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:client_testlib", @@ -1259,6 +1324,7 @@ tf_py_test( name = "summary_v1_tensor_op_test", size = "small", srcs = ["summary_v1_tensor_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -1293,6 +1359,7 @@ cuda_py_test( name = "template_mirrored_strategy_test", size = "small", srcs = ["template_mirrored_strategy_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:init_ops", @@ -1312,6 +1379,7 @@ cuda_py_test( tags = [ "no_oss", # TODO(b/142818120): Re-enable. ], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -1324,6 +1392,7 @@ tf_py_test( name = "unicode_script_op_test", size = "small", srcs = ["unicode_script_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -1336,6 +1405,7 @@ cuda_py_test( name = "topk_op_test", size = "medium", srcs = ["topk_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1351,6 +1421,7 @@ cuda_py_test( name = "nth_element_op_test", size = "small", srcs = ["nth_element_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1366,6 +1437,7 @@ tf_py_test( name = "unicode_encode_op_test", size = "small", srcs = ["unicode_encode_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:constant_op", @@ -1384,6 +1456,7 @@ tf_py_test( name = "unicode_transcode_op_test", size = "small", srcs = ["unicode_transcode_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -1416,6 +1489,7 @@ tf_py_test( name = "unique_op_test", size = "small", srcs = ["unique_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1477,6 +1551,7 @@ cuda_py_test( name = "where_op_test", size = "medium", srcs = ["where_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1489,6 +1564,7 @@ cuda_py_test( name = "cast_op_test", size = "small", srcs = ["cast_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1505,6 +1581,7 @@ cuda_py_test( size = "small", srcs = ["dense_update_ops_no_tsan_test.py"], tags = ["notsan"], + tfrt_enabled = True, # TODO (b/140294007): the test fails with XLA. xla_enable_strict_auto_jit = False, deps = [ @@ -1523,6 +1600,7 @@ cuda_py_test( srcs = ["diag_op_test.py"], shard_count = 6, tags = ["no_windows_gpu"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1538,6 +1616,7 @@ tf_py_test( size = "small", srcs = ["reader_ops_test.py"], data = ["//tensorflow/core:lmdb_testdata"], + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:client_testlib", @@ -1556,6 +1635,7 @@ cuda_py_test( name = "aggregate_ops_test", size = "small", srcs = ["aggregate_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1569,6 +1649,7 @@ cuda_py_test( name = "argmax_op_test", size = "small", srcs = ["argmax_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:math_ops", @@ -1622,6 +1703,7 @@ cuda_py_test( size = "small", srcs = ["inplace_ops_test.py"], shard_count = 10, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1638,6 +1720,7 @@ cuda_py_test( size = "medium", srcs = ["batch_matmul_op_test.py"], shard_count = 20, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1651,6 +1734,7 @@ cuda_py_test( name = "batchtospace_op_test", size = "small", srcs = ["batchtospace_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:array_ops_gen", @@ -1664,6 +1748,7 @@ cuda_py_test( name = "betainc_op_test", size = "small", srcs = ["betainc_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1692,6 +1777,7 @@ cuda_py_test( name = "bias_op_deterministic_test", size = "medium", srcs = ["bias_op_deterministic_test.py"], + tfrt_enabled = True, deps = [ ":bias_op_base", ], @@ -1710,6 +1796,7 @@ cuda_py_test( name = "bitcast_op_test", size = "small", srcs = ["bitcast_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1740,6 +1827,7 @@ cuda_py_test( name = "constant_op_test", size = "small", srcs = ["constant_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1816,6 +1904,7 @@ tf_py_test( name = "control_flow_util_test", size = "small", srcs = ["control_flow_util_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", @@ -1843,6 +1932,7 @@ cuda_py_test( name = "conv1d_test", size = "small", srcs = ["conv1d_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1855,6 +1945,7 @@ cuda_py_test( name = "conv1d_transpose_test", size = "small", srcs = ["conv1d_transpose_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client", "//tensorflow/python:client_testlib", @@ -1868,6 +1959,7 @@ cuda_py_test( name = "conv2d_transpose_test", size = "small", srcs = ["conv2d_transpose_test.py"], + tfrt_enabled = True, # TODO(b/144432983): S32 convolutions should not be auto-clustered, only # crashes tests. @@ -1886,6 +1978,7 @@ cuda_py_test( name = "conv3d_backprop_filter_v2_grad_test", size = "small", srcs = ["conv3d_backprop_filter_v2_grad_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1900,6 +1993,7 @@ cuda_py_test( name = "cross_grad_test", size = "small", srcs = ["cross_grad_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1924,6 +2018,7 @@ cuda_py_test( name = "dense_update_ops_test", size = "small", srcs = ["dense_update_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1940,6 +2035,7 @@ cuda_py_test( size = "medium", srcs = ["depthtospace_op_test.py"], tags = ["no_windows_gpu"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1954,6 +2050,7 @@ cuda_py_test( size = "medium", srcs = ["division_past_test.py"], tags = ["manual"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -1965,6 +2062,7 @@ cuda_py_test( name = "dynamic_partition_op_test", size = "medium", srcs = ["dynamic_partition_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -1980,6 +2078,7 @@ cuda_py_test( name = "dynamic_stitch_op_test", size = "small", srcs = ["dynamic_stitch_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:data_flow_grad", @@ -1994,6 +2093,7 @@ cuda_py_test( name = "extract_image_patches_op_test", size = "small", srcs = ["extract_image_patches_op_test.py"], + tfrt_enabled = True, # TODO(b/144432983): S32 convolutions should not be auto-clustered. xla_enable_strict_auto_jit = False, deps = [ @@ -2008,6 +2108,7 @@ cuda_py_test( name = "extract_volume_patches_op_test", size = "small", srcs = ["extract_volume_patches_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2046,6 +2147,7 @@ cuda_py_test( name = "gather_nd_op_test", size = "small", srcs = ["gather_nd_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client", @@ -2061,6 +2163,7 @@ cuda_py_test( name = "gradient_correctness_test", size = "small", srcs = ["gradient_correctness_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2118,6 +2221,7 @@ cuda_py_test( name = "lrn_op_test", size = "medium", srcs = ["lrn_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2133,6 +2237,7 @@ cuda_py_test( name = "lu_op_test", size = "small", srcs = ["lu_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2167,6 +2272,7 @@ cuda_py_test( size = "small", srcs = ["manip_ops_test.py"], tags = ["no_windows_gpu"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2180,6 +2286,7 @@ cuda_py_test( size = "medium", srcs = ["matmul_op_test.py"], shard_count = 20, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2196,6 +2303,7 @@ cuda_py_test( name = "morphological_ops_test", size = "small", srcs = ["morphological_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2209,6 +2317,7 @@ cuda_py_test( name = "numerics_test", size = "small", srcs = ["numerics_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2225,6 +2334,7 @@ cuda_py_test( size = "small", srcs = ["one_hot_op_test.py"], tags = ["no_windows_gpu"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2237,6 +2347,7 @@ cuda_py_test( name = "stack_op_test", size = "small", srcs = ["stack_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2273,6 +2384,7 @@ cuda_py_test( name = "pad_op_test", size = "small", srcs = ["pad_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2285,6 +2397,7 @@ cuda_py_test( name = "padding_fifo_queue_test", size = "small", srcs = ["padding_fifo_queue_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2318,6 +2431,7 @@ cuda_py_test( name = "reduce_join_op_test", size = "small", srcs = ["reduce_join_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2348,6 +2462,7 @@ cuda_py_test( tags = [ "no_windows_gpu", ], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2367,6 +2482,7 @@ cuda_py_test( "no_gpu", "noguitar", ], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2380,6 +2496,7 @@ cuda_py_test( name = "relu_op_test", size = "small", srcs = ["relu_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2398,6 +2515,7 @@ cuda_py_test( name = "reshape_op_test", size = "small", srcs = ["reshape_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2410,6 +2528,7 @@ cuda_py_test( name = "reverse_sequence_op_test", size = "small", srcs = ["reverse_sequence_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2422,6 +2541,7 @@ cuda_py_test( name = "compare_and_bitpack_op_test", size = "small", srcs = ["compare_and_bitpack_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2434,6 +2554,7 @@ cuda_py_test( name = "scalar_test", size = "small", srcs = ["scalar_test.py"], + tfrt_enabled = True, # b/140221961: Invalid dims for operations xla_enable_strict_auto_jit = False, deps = [ @@ -2454,6 +2575,7 @@ cuda_py_test( name = "scan_ops_test", size = "medium", srcs = ["scan_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:errors", @@ -2467,6 +2589,7 @@ cuda_py_test( name = "shape_ops_test", size = "medium", srcs = ["shape_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -2484,6 +2607,7 @@ cuda_py_test( name = "softmax_op_test", size = "medium", srcs = ["softmax_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2498,6 +2622,7 @@ cuda_py_test( name = "softplus_op_test", size = "small", srcs = ["softplus_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2511,6 +2636,7 @@ cuda_py_test( name = "softsign_op_test", size = "small", srcs = ["softsign_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2524,6 +2650,7 @@ cuda_py_test( name = "spacetobatch_op_test", size = "small", srcs = ["spacetobatch_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:array_ops_gen", @@ -2543,6 +2670,7 @@ cuda_py_test( "no_windows", "no_windows_gpu", ], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2556,6 +2684,7 @@ tf_py_test( name = "sparse_serialization_ops_test", size = "small", srcs = ["sparse_serialization_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2570,6 +2699,7 @@ tf_py_test( name = "sparse_tensors_map_ops_test", size = "small", srcs = ["sparse_tensors_map_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client", @@ -2586,6 +2716,7 @@ cuda_py_test( name = "sparse_tensor_dense_matmul_grad_test", size = "small", srcs = ["sparse_tensor_dense_matmul_grad_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework", @@ -2600,6 +2731,7 @@ cuda_py_test( name = "sparse_xent_op_test", size = "small", srcs = ["sparse_xent_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -2638,6 +2770,7 @@ cuda_py_test( name = "stack_ops_test", size = "small", srcs = ["stack_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:control_flow_ops", @@ -2653,6 +2786,7 @@ cuda_py_test( name = "string_to_hash_bucket_op_test", size = "small", srcs = ["string_to_hash_bucket_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2665,6 +2799,7 @@ cuda_py_test( name = "string_to_number_op_test", size = "small", srcs = ["string_to_number_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2677,6 +2812,7 @@ cuda_py_test( name = "summary_v1_audio_op_test", size = "small", srcs = ["summary_v1_audio_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:client_testlib", @@ -2690,6 +2826,7 @@ cuda_py_test( name = "summary_v1_image_op_test", size = "small", srcs = ["summary_v1_image_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:client_testlib", @@ -2740,6 +2877,7 @@ cuda_py_test( size = "small", srcs = ["trace_op_test.py"], tags = ["no_windows_gpu"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:math_ops", @@ -2770,6 +2908,7 @@ cuda_py_test( name = "variable_ops_test", size = "small", srcs = ["variable_ops_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2787,6 +2926,7 @@ cuda_py_test( name = "xent_op_test", size = "small", srcs = ["xent_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2803,6 +2943,7 @@ cuda_py_test( name = "zero_division_test", size = "medium", srcs = ["zero_division_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:errors", @@ -2818,6 +2959,7 @@ cuda_py_test( tags = [ "no_gpu", # Flaky: b/80127739, b/127001953 ], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2834,6 +2976,7 @@ cuda_py_test( size = "medium", srcs = ["atrous_convolution_test.py"], tags = ["manual"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2848,6 +2991,7 @@ cuda_py_test( name = "pool_test", size = "medium", srcs = ["pool_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2879,6 +3023,7 @@ cuda_py_test( name = "conv3d_transpose_test", size = "medium", srcs = ["conv3d_transpose_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2922,6 +3067,7 @@ cuda_py_test( shard_count = 3, # TODO(b/118842098): Re-enable this test in Kokoro. tags = ["no_oss"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2938,6 +3084,7 @@ tf_py_test( size = "medium", srcs = ["neon_depthwise_conv_op_test.py"], tags = ["no_windows"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -2954,6 +3101,7 @@ cuda_py_test( size = "medium", srcs = ["division_future_test.py"], tags = ["manual"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2965,6 +3113,7 @@ cuda_py_test( name = "pooling_ops_3d_test", size = "medium", srcs = ["pooling_ops_3d_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -2979,6 +3128,7 @@ cuda_py_test( size = "medium", srcs = ["pooling_ops_test.py"], shard_count = 4, + tfrt_enabled = True, xla_enable_strict_auto_jit = False, # Flaky in XLA b/149568654 deps = [ "//tensorflow/python:array_ops", @@ -2999,6 +3149,7 @@ cuda_py_test( timeout = "long", srcs = ["rnn_test.py"], shard_count = 10, + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -3073,6 +3224,7 @@ cuda_py_test( tags = [ "no_oss", # Requires 4GB+ RAM ], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3087,6 +3239,7 @@ cuda_py_test( size = "medium", srcs = ["sparse_matmul_op_test.py"], tags = ["no_windows"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -3122,6 +3275,7 @@ cuda_py_test( name = "sparse_tensor_dense_matmul_op_test", size = "medium", srcs = ["sparse_tensor_dense_matmul_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/core:protos_all_py", "//tensorflow/python:array_ops", @@ -3176,6 +3330,7 @@ cuda_py_test( name = "stage_op_test", size = "medium", srcs = ["stage_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3191,6 +3346,7 @@ cuda_py_test( size = "medium", srcs = ["map_stage_op_test.py"], tags = ["no_oss"], # b/124474135 + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3206,6 +3362,7 @@ cuda_py_test( size = "medium", srcs = ["concat_op_test.py"], tags = ["no_windows"], # b/126916429 + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:array_ops_gen", @@ -3227,6 +3384,7 @@ cuda_py_test( "nomsan", "notsan", ], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3239,6 +3397,7 @@ cuda_py_test( size = "medium", srcs = ["conv_ops_3d_test.py"], shard_count = 30, + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -3252,6 +3411,7 @@ cuda_py_test( size = "medium", srcs = ["cwise_ops_test.py"], shard_count = 50, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3272,6 +3432,7 @@ cuda_py_test( size = "medium", srcs = ["cwise_ops_binary_test.py"], shard_count = 50, + tfrt_enabled = True, # b/140155647: Error just outside of tolerance xla_enable_strict_auto_jit = False, deps = [ @@ -3316,6 +3477,7 @@ cuda_py_test( size = "medium", srcs = ["embedding_ops_test.py"], shard_count = 20, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3358,6 +3520,7 @@ cuda_py_test( size = "medium", srcs = ["matrix_band_part_op_test.py"], shard_count = 20, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3375,6 +3538,7 @@ tf_py_test( tags = [ "no_windows", ], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3395,6 +3559,7 @@ cuda_py_test( tags = [ "no_windows", ], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3454,6 +3619,7 @@ cuda_py_test( "no_windows_gpu", "nomsan", ], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3515,6 +3681,7 @@ tf_py_test( name = "sets_test", size = "medium", srcs = ["sets_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:errors", "//tensorflow/python:framework", @@ -3533,6 +3700,7 @@ tf_py_test( size = "small", srcs = ["weights_broadcast_test.py"], shard_count = 3, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3552,6 +3720,7 @@ tf_py_test( srcs = ["metrics_test.py"], shard_count = 20, tags = ["no_windows_gpu"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3572,6 +3741,7 @@ tf_py_test( name = "confusion_matrix_test", size = "small", srcs = ["confusion_matrix_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3587,6 +3757,7 @@ cuda_py_test( name = "bucketize_op_test", size = "medium", srcs = ["bucketize_op_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -3600,6 +3771,7 @@ tf_py_test( size = "small", srcs = ["sparse_cross_op_test.py"], tags = ["no_windows"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", @@ -3612,6 +3784,7 @@ tf_py_test( name = "garbage_collection_test", size = "small", srcs = ["garbage_collection_test.py"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:dtypes", @@ -3693,6 +3866,7 @@ cuda_py_test( size = "medium", srcs = ["cond_v2_test.py"], grpc_enabled = True, + tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", "//tensorflow/python:client_testlib", @@ -3768,6 +3942,7 @@ cuda_py_test( srcs = ["tridiagonal_matmul_op_test.py"], shard_count = 10, tags = ["no_rocm"], + tfrt_enabled = True, deps = [ "//tensorflow/python:client_testlib", "//tensorflow/python:framework_for_generated_wrappers", From 12a806e96866296b154134b27ef4228f39f403cc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sat, 8 Aug 2020 19:12:31 -0700 Subject: [PATCH 0712/1017] PR #42109: Check input and axis param in quantize and dequantize Imported from GitHub PR https://github.com/tensorflow/tensorflow/pull/42109 Try to fix https://github.com/tensorflow/tensorflow/issues/42105 Copybara import of the project: -- 0573f8cb6976c592eee660da9e4ce58e0c1eb0c0 by bhack : Check input and axis params -- 3e5a78fa1b3854e536587a94514ac42b8b621225 by bhack : Else fix PiperOrigin-RevId: 325639857 Change-Id: Ifc18bc9686e9ff38839bb0c45fb3ef1d2ad9c208 --- tensorflow/python/ops/array_ops.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py index 9875342730c..5d68deb7ac1 100644 --- a/tensorflow/python/ops/array_ops.py +++ b/tensorflow/python/ops/array_ops.py @@ -5316,13 +5316,12 @@ def quantize_and_dequantize( A `Tensor`. Each element is the result of quantizing and dequantizing the corresponding element of `input`. """ - with ops.name_scope(name, "quantize_and_dequantize", [input]) as name: - if not tensor_util.is_tensor(input): - input = ops.convert_to_tensor(input) if axis is None: axis = -1 - else: - axis = get_positive_axis(axis, input.shape.ndims) + elif axis < 0: + if input.shape.ndims is None: + raise ValueError("input should have known rank to use negative axis.") + axis %= input.shape.ndims return gen_array_ops.quantize_and_dequantize_v2( input, From 8f35f8fd6be8e7dcba038fd87065f88035a5aa20 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 9 Aug 2020 02:01:33 -0700 Subject: [PATCH 0713/1017] Update GraphDef version to 488. PiperOrigin-RevId: 325663031 Change-Id: I5dbbba32c099c84f19dc20174929489b13b353ff --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 3512cb4c5b9..1813717e87a 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 487 // Updated: 2020/8/8 +#define TF_GRAPH_DEF_VERSION 488 // Updated: 2020/8/9 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From 4308605c8926e5f53550d22fde86d22573507d2c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 9 Aug 2020 02:01:47 -0700 Subject: [PATCH 0714/1017] compat: Update forward compatibility horizon to 2020-08-09 PiperOrigin-RevId: 325663050 Change-Id: I312308efc62cc443638895aaa32ee33b23f4ea05 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index 01ea900fd11..fefbf667704 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 8) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 9) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From 947b6c3a4bc8d51565e73e2f3d977b3298b9f64c Mon Sep 17 00:00:00 2001 From: Alexander Belyaev Date: Sun, 9 Aug 2020 02:36:32 -0700 Subject: [PATCH 0715/1017] [MLIR] Add e2e test for unranked unary TF op, lowered and run with CPU runner. PiperOrigin-RevId: 325665428 Change-Id: I3e8a1a3a9551ba470e858fb775a31ca894f47359 --- .../mhlo/transforms/transform_unranked_hlo.cc | 2 +- .../kernel_gen/transforms/bufferize_pass.cc | 30 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/transform_unranked_hlo.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/transform_unranked_hlo.cc index b6e55a9322f..7c985ea7535 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/transform_unranked_hlo.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/transform_unranked_hlo.cc @@ -170,7 +170,7 @@ struct TransformUnrankedHloPass PopulateTransformUnrankedHloPatterns(&ctx, &patterns); // Apply transformation. - if (failed(applyFullConversion(getFunction(), target, patterns))) + if (failed(applyPartialConversion(getFunction(), target, patterns))) return signalPassFailure(); } }; diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize_pass.cc b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize_pass.cc index 7d195c69c37..ef07c801bc4 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize_pass.cc +++ b/tensorflow/compiler/mlir/tools/kernel_gen/transforms/bufferize_pass.cc @@ -44,6 +44,28 @@ namespace { #define GEN_PASS_CLASSES #include "tensorflow/compiler/mlir/tools/kernel_gen/transforms/kernel_gen_passes.h.inc" +// TODO(herhut) : This could become a real pattern in bufferize pass. What we +// would need to do is insert a copy to model the semantics correctly. The same +// is true for the TensorLoad pattern that is already in there. Then buffer +// assignment free insertion and copy removal should clean this up for us. +// +// This patten erases `tensor_store(src_unranked_tensor, dst_unranked_memref)` +// op and replaces the result of the defining op produced `dst_unranked_memref` +// with the rewritten `src_unranked_tensor`. +class UnrankedTensorStoreTestOnlyPattern + : public OpConversionPattern { + public: + using OpConversionPattern::OpConversionPattern; + + LogicalResult matchAndRewrite( + mlir::TensorStoreOp op, ArrayRef operands, + ConversionPatternRewriter& rewriter) const final { + rewriter.replaceOp(op.memref().getDefiningOp(), op.tensor()); + rewriter.replaceOp(op, {}); + return success(); + } +}; + struct BufferizePass : public BufferizePassBase { public: void runOnOperation() override { @@ -57,8 +79,11 @@ struct BufferizePass : public BufferizePassBase { target.addLegalOp(); target.addIllegalDialect(); target.addIllegalOp(); - target.addIllegalOp(); target.addIllegalOp(); + target.addIllegalOp(); + target.addDynamicallyLegalOp([&](TensorStoreOp op) { + return !op.tensor().getType().isa(); + }); BufferAssignmentTypeConverter converter; auto typesAreLegal = [&converter](Operation* op) { @@ -86,8 +111,9 @@ struct BufferizePass : public BufferizePassBase { &converter, &patterns); populateStandardBufferizePattern(func.getContext(), &bufferAssignment, &converter, &patterns); + patterns.insert(func.getContext()); - return applyFullConversion(func, target, patterns); + return applyPartialConversion(func, target, patterns); }); if (result.wasInterrupted()) { signalPassFailure(); From 5676c66e9108dd6bb609b6e032860ba9c2e80078 Mon Sep 17 00:00:00 2001 From: Kibeom Kim Date: Sun, 9 Aug 2020 09:48:49 -0700 Subject: [PATCH 0716/1017] Add convert_to_tensor KPI and introduce @trace.trace_wrapper(...) API Note that this is a superset of tf.convert_to_tensor, which includes internal convert_to_tensor calls. Also, introduced @trace.trace_wrapper(...) API that's faster than `with trace.Trace(...):` API. Benchmark: `with trace.Trace(...):` time : 0.67 us `@trace.trace_wrapper(...)` time : 0.21 us Direct `if trace.enabled:` inlining time : 0.17 us PiperOrigin-RevId: 325690563 Change-Id: I3251e38b7543e3121be6fad0266706a1b3b5c389 --- tensorflow/python/BUILD | 1 + tensorflow/python/framework/ops.py | 2 ++ tensorflow/python/profiler/trace.py | 42 +++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 4efe769c59d..a8a70566ab7 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1702,6 +1702,7 @@ py_library( "//tensorflow/python/eager:core", "//tensorflow/python/eager:monitoring", "//tensorflow/python/eager:tape", + "//tensorflow/python/profiler:traceme", "@six_archive//:six", ], ) diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py index 75a36f83fc5..f07bca17061 100644 --- a/tensorflow/python/framework/ops.py +++ b/tensorflow/python/framework/ops.py @@ -62,6 +62,7 @@ from tensorflow.python.framework import versions from tensorflow.python.ops import control_flow_util from tensorflow.python.platform import app from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.profiler import trace from tensorflow.python.types import core as core_tf_types from tensorflow.python.types import internal from tensorflow.python.util import compat @@ -1472,6 +1473,7 @@ def pack_eager_tensors(tensors, ctx=None): return packed_tensor +@trace.trace_wrapper("convert_to_tensor") def convert_to_tensor(value, dtype=None, name=None, diff --git a/tensorflow/python/profiler/trace.py b/tensorflow/python/profiler/trace.py index 0591d90fa43..e4cf581bd25 100644 --- a/tensorflow/python/profiler/trace.py +++ b/tensorflow/python/profiler/trace.py @@ -18,6 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import functools + from tensorflow.python.profiler.internal import _pywrap_traceme from tensorflow.python.util.tf_export import tf_export @@ -123,3 +125,43 @@ class Trace(object): def __exit__(self, exc_type, exc_val, exc_tb): if self._traceme: self._traceme.Stop() + + +def trace_wrapper(trace_name, **trace_kwargs): + """Decorator alternative to `with Trace(): ...`. It's faster. + + Args: + trace_name: The name of the trace event. + **trace_kwargs: Keyword arguments added to the trace event. Both the key and + value are of types that can be converted to strings, which will be + interpreted by the profiler according to the traceme name. + + Returns: + A decorator that can wrap a function and apply `Trace` scope if needed. + + Example usage: + ```python + + @trace_wrapper('trace_name') + def func(x, y, z): + pass # code to execute and apply `Trace` if needed. + + # Equivalent to + # with Trace('trace_name'): + # func(1, 2, 3) + func(1, 2, 3) + ``` + """ + + def inner_wrapper(func): + + @functools.wraps(func) + def wrapped(*args, **kwargs): + if enabled: + with Trace(trace_name, **trace_kwargs): + return func(*args, **kwargs) + return func(*args, **kwargs) + + return wrapped + + return inner_wrapper From 11f952a84a07771e43c068c7094f1ee3674f7b20 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Sun, 9 Aug 2020 10:41:51 -0700 Subject: [PATCH 0717/1017] Disable tensor_map_test for now since it failed msan test. PiperOrigin-RevId: 325693925 Change-Id: I905ac70c5a6717d5da69b914a2093d357f82f67a --- tensorflow/core/kernels/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index 9be043c3907..ccb12d9b09d 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -2932,6 +2932,7 @@ tf_cc_tests( srcs = [ "tensor_map_test.cc", ], + tags = ["nomsan"], # b/163222155 deps = [ ":tensor_map", "//tensorflow/core:framework", From 11f5967d2ca3f73ecff4ac5c32bfc53a744a9871 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Sun, 9 Aug 2020 10:43:12 -0700 Subject: [PATCH 0718/1017] Disable op_level_cost_estimator_test for the moment. PiperOrigin-RevId: 325694016 Change-Id: I4eac77ca6bc9c3cd68418f8afdd72679b63827d3 --- tensorflow/core/grappler/costs/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD index edbdaffa1c8..02c69920b84 100644 --- a/tensorflow/core/grappler/costs/BUILD +++ b/tensorflow/core/grappler/costs/BUILD @@ -337,6 +337,7 @@ cc_library( tf_cc_test( name = "op_level_cost_estimator_test", srcs = ["op_level_cost_estimator_test.cc"], + tags = ["no_oss"], # b/163222310 deps = [ ":op_level_cost_estimator", "//tensorflow/core:framework", From e69bf2f9e30cefee81c7fdcee1ae71962b09d19c Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Sun, 9 Aug 2020 10:44:32 -0700 Subject: [PATCH 0719/1017] Disable cwise_ops_test on windows for now. PiperOrigin-RevId: 325694097 Change-Id: I7566db6043b246efd67f947ed36977af319e921d --- tensorflow/python/kernel_tests/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index e73f2ea29fc..2888730e2bb 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -3411,6 +3411,7 @@ cuda_py_test( size = "medium", srcs = ["cwise_ops_test.py"], shard_count = 50, + tags = ["no_windows"], # b/163222163 tfrt_enabled = True, deps = [ "//tensorflow/python:array_ops", From 1f3b5f79383f3f84957c5f98710ed9d24bdeba2c Mon Sep 17 00:00:00 2001 From: Yuanzhong Xu Date: Sun, 9 Aug 2020 17:55:57 -0700 Subject: [PATCH 0720/1017] [XLA:SPMD] Fix replicate to partial sharding PiperOrigin-RevId: 325721330 Change-Id: I53448aa275c4c3ef8790485994c0ebb6cc4394a7 --- tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc index 9db76a65486..813ccc46f32 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc @@ -302,14 +302,11 @@ PartitionedHlo PartitionedHlo::ReshardNoCache(const HloSharding& target) { 1); std::iota(group_dims.begin(), group_dims.end(), 0); auto target_grouped = GroupShardingOnDims(target, group_dims); - auto per_group_partitioner_state = CreatePerGroupPartitioningState( - state_, target_grouped.device_groups, state_.b); auto partially_sharded = PerGroupSliceFromReplicated( hlo_, state_.partition_id, target_grouped.device_groups, group_dims, target_grouped.group_dim_sizes, state_.b); partially_sharded->set_sharding(target); - return PartitionedHlo(partially_sharded, base_shape(), - per_group_partitioner_state); + return PartitionedHlo(partially_sharded, base_shape(), state_); } // 'Replicated' to 'Tiled'. From f4c0fae59291f9d5f92bdadc166cb616315a551e Mon Sep 17 00:00:00 2001 From: Thai Nguyen Date: Sun, 9 Aug 2020 18:01:47 -0700 Subject: [PATCH 0721/1017] Fix linking error of android_tensorflow_image_op The errors occurs due to duplicated definition in portable_tensorflow_lib_lite and tensorflow/core/platform:strcat (a dependency listed in android_gif_internal). PiperOrigin-RevId: 325721659 Change-Id: Ic5c0accd67999f901d0cb284467b808f293d4771 --- tensorflow/core/BUILD | 5 +---- tensorflow/core/lib/gif/gif_io.cc | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 67e0a160c4f..41eba6b5e28 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1977,6 +1977,7 @@ cc_library( ":lib", ":lib_internal", "//tensorflow/core/platform:gif", + "@com_google_absl//absl/strings", ], ) @@ -2084,13 +2085,9 @@ cc_library( copts = tf_copts(), linkopts = ["-ldl"], deps = [ - "//tensorflow/core/lib/strings:numbers", - "//tensorflow/core/lib/strings:strcat", "//tensorflow/core/platform:dynamic_annotations", "//tensorflow/core/platform:gif", "//tensorflow/core/platform:logging", - "//tensorflow/core/platform:numbers", - "//tensorflow/core/platform:strcat", "//tensorflow/core/platform:stringpiece", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/strings", diff --git a/tensorflow/core/lib/gif/gif_io.cc b/tensorflow/core/lib/gif/gif_io.cc index 32e2f6dfa52..659513d05ed 100644 --- a/tensorflow/core/lib/gif/gif_io.cc +++ b/tensorflow/core/lib/gif/gif_io.cc @@ -16,9 +16,11 @@ limitations under the License. // Functions to read images in GIF format. #include "tensorflow/core/lib/gif/gif_io.h" + #include + +#include "absl/strings/str_cat.h" #include "tensorflow/core/lib/gtl/cleanup.h" -#include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/gif.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/mem.h" @@ -68,17 +70,17 @@ uint8* Decode(const void* srcdata, int datasize, } }); if (error_code != D_GIF_SUCCEEDED) { - *error_string = strings::StrCat("failed to open gif file: ", - GifErrorStringNonNull(error_code)); + *error_string = absl::StrCat("failed to open gif file: ", + GifErrorStringNonNull(error_code)); return nullptr; } if (DGifSlurp(gif_file) != GIF_OK) { - *error_string = strings::StrCat("failed to slurp gif file: ", - GifErrorStringNonNull(gif_file->Error)); + *error_string = absl::StrCat("failed to slurp gif file: ", + GifErrorStringNonNull(gif_file->Error)); return nullptr; } if (gif_file->ImageCount <= 0) { - *error_string = strings::StrCat("gif file does not contain any image"); + *error_string = "gif file does not contain any image"; return nullptr; } @@ -118,8 +120,7 @@ uint8* Decode(const void* srcdata, int datasize, img_desc->Height != height) { // If the first frame does not fill the entire canvas then return error. if (k == 0) { - *error_string = - strings::StrCat("the first frame does not fill the canvas"); + *error_string = "the first frame does not fill the canvas"; return nullptr; } // Otherwise previous frame will be reused to fill the unoccupied canvas. @@ -144,7 +145,7 @@ uint8* Decode(const void* srcdata, int datasize, ? this_image->ImageDesc.ColorMap : gif_file->SColorMap; if (color_map == nullptr) { - *error_string = strings::StrCat("missing color map for frame ", k); + *error_string = absl::StrCat("missing color map for frame ", k); return nullptr; } @@ -156,9 +157,9 @@ uint8* Decode(const void* srcdata, int datasize, (j - img_desc->Left)]; if (color_index >= color_map->ColorCount) { - *error_string = strings::StrCat("found color index ", color_index, - " outside of color map range ", - color_map->ColorCount); + *error_string = absl::StrCat("found color index ", color_index, + " outside of color map range ", + color_map->ColorCount); return nullptr; } From 5f75a84642b6032c553362f5e48254c4bf0aa86c Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Sun, 9 Aug 2020 19:18:29 -0700 Subject: [PATCH 0722/1017] Disable gpu_compatibility_test on mac for now. PiperOrigin-RevId: 325727277 Change-Id: I55338ecf787c010d16410e37c804f0391b5dad75 --- tensorflow/lite/experimental/acceleration/compatibility/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/lite/experimental/acceleration/compatibility/BUILD b/tensorflow/lite/experimental/acceleration/compatibility/BUILD index 387f475fa17..97c903d561f 100644 --- a/tensorflow/lite/experimental/acceleration/compatibility/BUILD +++ b/tensorflow/lite/experimental/acceleration/compatibility/BUILD @@ -169,6 +169,7 @@ cc_library( cc_test( name = "gpu_compatibility_test", srcs = ["gpu_compatibility_test.cc"], + tags = ["no_mac"], # b/163222453 deps = [ ":devicedb_sample", ":gpu_compatibility", From 4e7127d73f2ddf14b85ef55e3c4c0717dbbcde1d Mon Sep 17 00:00:00 2001 From: Chao Mei Date: Sun, 9 Aug 2020 19:33:03 -0700 Subject: [PATCH 0723/1017] Add a method to add metric to the benchmark entry. PiperOrigin-RevId: 325728292 Change-Id: I1829091880d54a4e6692c0fd3d39b3f128a3a1a6 --- tensorflow/core/util/reporter.cc | 8 ++++++++ tensorflow/core/util/reporter.h | 3 +++ tensorflow/core/util/reporter_test.cc | 25 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/tensorflow/core/util/reporter.cc b/tensorflow/core/util/reporter.cc index 8e9d863b4c2..44465a58329 100644 --- a/tensorflow/core/util/reporter.cc +++ b/tensorflow/core/util/reporter.cc @@ -91,6 +91,14 @@ Status TestReporter::SetProperty(const string& name, double value) { return Status::OK(); } +Status TestReporter::AddMetric(const string& name, double value) { + if (report_file_.IsClosed()) return Status::OK(); + auto* metric = benchmark_entry_.add_metrics(); + metric->set_name(name); + metric->set_value(value); + return Status::OK(); +} + Status TestReporter::Initialize() { return report_file_.Initialize(); } } // namespace tensorflow diff --git a/tensorflow/core/util/reporter.h b/tensorflow/core/util/reporter.h index 51d7502701c..900fe40353e 100644 --- a/tensorflow/core/util/reporter.h +++ b/tensorflow/core/util/reporter.h @@ -111,6 +111,9 @@ class TestReporter { // Set property on Benchmark to the given value. Status SetProperty(const string& name, const string& value); + // Add the given value to the metrics on the Benchmark. + Status AddMetric(const string& name, double value); + // TODO(b/32704451): Don't just ignore the ::tensorflow::Status object! ~TestReporter() { Close().IgnoreError(); } // Autoclose in destructor. diff --git a/tensorflow/core/util/reporter_test.cc b/tensorflow/core/util/reporter_test.cc index 4c06560b852..77e7ed6467e 100644 --- a/tensorflow/core/util/reporter_test.cc +++ b/tensorflow/core/util/reporter_test.cc @@ -138,5 +138,30 @@ TEST(TestReporter, SetProperties) { EXPECT_EQ(4.0, extras.at("double_prop").double_value()); } +TEST(TestReporter, AddMetrics) { + string fname = + strings::StrCat(testing::TmpDir(), "/test_reporter_benchmarks_"); + TestReporter test_reporter(fname, "b3/4/5"); + TF_EXPECT_OK(test_reporter.Initialize()); + TF_EXPECT_OK(test_reporter.AddMetric("metric1", 2.0)); + TF_EXPECT_OK(test_reporter.AddMetric("metric2", 3.0)); + + TF_EXPECT_OK(test_reporter.Close()); + string expected_fname = strings::StrCat(fname, "b3__4__5"); + string read; + TF_EXPECT_OK(ReadFileToString(Env::Default(), expected_fname, &read)); + + BenchmarkEntries benchmark_entries; + ASSERT_TRUE(benchmark_entries.ParseFromString(read)); + ASSERT_EQ(1, benchmark_entries.entry_size()); + const BenchmarkEntry& benchmark_entry = benchmark_entries.entry(0); + const auto& metrics = benchmark_entry.metrics(); + ASSERT_EQ(2, metrics.size()); + EXPECT_EQ("metric1", metrics.at(0).name()); + EXPECT_EQ(2.0, metrics.at(0).value()); + EXPECT_EQ("metric2", metrics.at(1).name()); + EXPECT_EQ(3.0, metrics.at(1).value()); +} + } // namespace } // namespace tensorflow From 0720cbfdd3b1147f6c88343ee04997fc3085c5dd Mon Sep 17 00:00:00 2001 From: Chao Mei Date: Sun, 9 Aug 2020 20:09:37 -0700 Subject: [PATCH 0724/1017] Make it more clear about applying xnnpack delegate by default in TFLite's interpreter when a delegate is explicitly applied via ModifyGraphWithDelegate. This is to address https://github.com/tensorflow/tensorflow/issues/41977 PiperOrigin-RevId: 325731102 Change-Id: I6702088e5b10cae23f8de5b7b30d3fc53243fb86 --- tensorflow/lite/interpreter.cc | 11 ++++++++++- .../lite/tools/benchmark/benchmark_tflite_model.cc | 13 ++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc index 7a5f4df5155..307ede187b2 100644 --- a/tensorflow/lite/interpreter.cc +++ b/tensorflow/lite/interpreter.cc @@ -188,7 +188,16 @@ TfLiteStatus Interpreter::AllocateTensors() { // The execution will fall back to default implementation if the XNNPACK // delegate fails to be applied. Therefore, we ignore the return status // here and let it fall through the rest of the code. - ModifyGraphWithDelegate(std::move(lazy_delegate_provider_)); + auto status = ModifyGraphWithDelegate(std::move(lazy_delegate_provider_)); + if (status != kTfLiteOk) { + TF_LITE_REPORT_ERROR( + error_reporter_, + "Ignoring failed application of the default TensorFlow Lite " + "delegate."); + } else { + TFLITE_LOG(TFLITE_LOG_INFO, + "Successfully applied the default TensorFlow Lite delegate."); + } lazy_delegate_provider_.reset(); } diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc index 9da48badfbc..39ecded5484 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc +++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc @@ -669,18 +669,21 @@ TfLiteStatus BenchmarkTfLiteModel::Init() { return kTfLiteError; } if (fully_delegated) { - TFLITE_LOG(INFO) << "Applied " << delegate_provider->GetName() + TFLITE_LOG(INFO) << "Explicitly applied " + << delegate_provider->GetName() << " delegate, and the model graph will be completely" << " executed by the delegate."; } else if (num_delegated_kernels > 0) { - TFLITE_LOG(INFO) << "Applied " << delegate_provider->GetName() + TFLITE_LOG(INFO) << "Explicitly applied " + << delegate_provider->GetName() << " delegate, and the model graph will be partially" << " executed by the delegate w/ " << num_delegated_kernels << " delegate kernels."; } else { - TFLITE_LOG(INFO) << "Though " << delegate_provider->GetName() - << " delegate is applied, the model graph will not be" - << " executed by the delegate."; + TFLITE_LOG(INFO) + << "Though " << delegate_provider->GetName() + << " delegate is explicitly applied, the model graph will not be" + << " executed by the delegate."; } } owned_delegates_.emplace_back(std::move(delegate)); From ef88a7aad44756a99aa57deddc59dc756ac2b469 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Sun, 9 Aug 2020 20:54:59 -0700 Subject: [PATCH 0725/1017] Port the Mobilenet_v3 to keras/application. Fix https://github.com/tensorflow/tensorflow/issues/40217. The implementation is based on https://github.com/keras-team/keras-applications/blob/master/keras_applications/mobilenet_v3.py with a few modifications. 1. Updated to use TF backend only (theano related code is removed). 2. Remove all the *kwargs, and directly use tf.keras packages (disallow package injection). 3. Add 'classifier_activation' which is used by 'top' layer. This is aligned with v1/v2 implementation. 4. [Major] Changed the include_top implementation. The Conv2D layer with name "Conv_2" and its activation is moved to be base model structure, which means they are in the model even the include_top is False. This is based on comparing the implementation detail in original slim implementation in https://github.com/tensorflow/models/blob/a811a3b7e640722318ad868c99feddf3f3063e36/research/slim/nets/mobilenet/mobilenet_v3.py. If we can confirm this change is correct, then we should also fix it on the OSS keras_application as well. 5. [Major] Remove the first ZeroPadding2D layer right after the model input, and change the first conv2D layer to use "same" padding. This is aligned with original implementation in https://github.com/tensorflow/models/blob/692215511a27c49dadabd4fac1d83aef25bc840f/research/slim/nets/mobilenet/mobilenet.py#L155, where use_explicit_padding is False. 6. Added API for preprocess_input and decode_predictions, which aligns with v1 and v2 implementation. PiperOrigin-RevId: 325734579 Change-Id: I2ba6a9aa695baaa145d1a7cd3aeae86d48b823a2 --- RELEASE.md | 1 + tensorflow/python/keras/api/BUILD | 1 + tensorflow/python/keras/applications/BUILD | 17 + .../applications_load_weight_test.py | 3 + .../keras/applications/applications_test.py | 3 + .../python/keras/applications/mobilenet_v3.py | 567 ++++++++++++++++++ .../tools/api/generator/api_init_files.bzl | 1 + .../tools/api/generator/api_init_files_v1.bzl | 1 + ...flow.keras.applications.mobilenet_v3.pbtxt | 11 + .../v1/tensorflow.keras.applications.pbtxt | 12 + ...flow.keras.applications.mobilenet_v3.pbtxt | 11 + .../v2/tensorflow.keras.applications.pbtxt | 12 + 12 files changed, 640 insertions(+) create mode 100644 tensorflow/python/keras/applications/mobilenet_v3.py create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.applications.mobilenet_v3.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.applications.mobilenet_v3.pbtxt diff --git a/RELEASE.md b/RELEASE.md index 5f0553c2a94..191c18e5ddb 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -101,6 +101,7 @@ * `Optimizer.minimize` can now accept a loss `Tensor` and a `GradientTape` as an alternative to accepting a `callable` loss. * Added `beta` parameter to FTRL optimizer to match paper. + * Added `mobilenet_v3` to keras application model. * `tf.function` / AutoGraph: * Added `experimental_follow_type_hints` argument for `tf.function`. When True, the function may use type annotations to optimize the tracing diff --git a/tensorflow/python/keras/api/BUILD b/tensorflow/python/keras/api/BUILD index ff54400ae15..d69930b7455 100644 --- a/tensorflow/python/keras/api/BUILD +++ b/tensorflow/python/keras/api/BUILD @@ -23,6 +23,7 @@ keras_packages = [ "tensorflow.python.keras.applications.inception_v3", "tensorflow.python.keras.applications.mobilenet", "tensorflow.python.keras.applications.mobilenet_v2", + "tensorflow.python.keras.applications.mobilenet_v3", "tensorflow.python.keras.applications.nasnet", "tensorflow.python.keras.applications.resnet", "tensorflow.python.keras.applications.resnet_v2", diff --git a/tensorflow/python/keras/applications/BUILD b/tensorflow/python/keras/applications/BUILD index 0c566c6e6d5..a2c41dbe501 100644 --- a/tensorflow/python/keras/applications/BUILD +++ b/tensorflow/python/keras/applications/BUILD @@ -25,6 +25,7 @@ py_library( "inception_v3.py", "mobilenet.py", "mobilenet_v2.py", + "mobilenet_v3.py", "nasnet.py", "resnet.py", "resnet_v2.py", @@ -209,6 +210,22 @@ tf_py_test( ], ) +tf_py_test( + name = "applications_load_weight_test_mobilenet_v3", + srcs = ["applications_load_weight_test.py"], + args = ["--module=mobilenet_v3"], + main = "applications_load_weight_test.py", + tags = [ + "no_oss", + "no_pip", + ], + deps = [ + ":applications", + "//tensorflow/python:client_testlib", + "@absl_py//absl/testing:parameterized", + ], +) + tf_py_test( name = "applications_load_weight_test_densenet", size = "large", diff --git a/tensorflow/python/keras/applications/applications_load_weight_test.py b/tensorflow/python/keras/applications/applications_load_weight_test.py index 42146c66f97..aaafe9f984a 100644 --- a/tensorflow/python/keras/applications/applications_load_weight_test.py +++ b/tensorflow/python/keras/applications/applications_load_weight_test.py @@ -28,6 +28,7 @@ from tensorflow.python.keras.applications import inception_resnet_v2 from tensorflow.python.keras.applications import inception_v3 from tensorflow.python.keras.applications import mobilenet from tensorflow.python.keras.applications import mobilenet_v2 +from tensorflow.python.keras.applications import mobilenet_v3 from tensorflow.python.keras.applications import nasnet from tensorflow.python.keras.applications import resnet from tensorflow.python.keras.applications import resnet_v2 @@ -51,6 +52,8 @@ ARG_TO_MODEL = { [inception_resnet_v2.InceptionResNetV2]), 'mobilenet': (mobilenet, [mobilenet.MobileNet]), 'mobilenet_v2': (mobilenet_v2, [mobilenet_v2.MobileNetV2]), + 'mobilenet_v3': (mobilenet_v3, [mobilenet_v3.MobileNetV3Small, + mobilenet_v3.MobileNetV3Large]), 'densenet': (densenet, [densenet.DenseNet121, densenet.DenseNet169, densenet.DenseNet201]), 'nasnet_mobile': (nasnet, [nasnet.NASNetMobile]), diff --git a/tensorflow/python/keras/applications/applications_test.py b/tensorflow/python/keras/applications/applications_test.py index 198bebd904c..d92a2aaee7f 100644 --- a/tensorflow/python/keras/applications/applications_test.py +++ b/tensorflow/python/keras/applications/applications_test.py @@ -27,6 +27,7 @@ from tensorflow.python.keras.applications import inception_resnet_v2 from tensorflow.python.keras.applications import inception_v3 from tensorflow.python.keras.applications import mobilenet from tensorflow.python.keras.applications import mobilenet_v2 +from tensorflow.python.keras.applications import mobilenet_v3 from tensorflow.python.keras.applications import nasnet from tensorflow.python.keras.applications import resnet from tensorflow.python.keras.applications import resnet_v2 @@ -50,6 +51,8 @@ MODEL_LIST_NO_NASNET = [ (inception_resnet_v2.InceptionResNetV2, 1536), (mobilenet.MobileNet, 1024), (mobilenet_v2.MobileNetV2, 1280), + (mobilenet_v3.MobileNetV3Small, 1024), + (mobilenet_v3.MobileNetV3Large, 1280), (densenet.DenseNet121, 1024), (densenet.DenseNet169, 1664), (densenet.DenseNet201, 1920), diff --git a/tensorflow/python/keras/applications/mobilenet_v3.py b/tensorflow/python/keras/applications/mobilenet_v3.py new file mode 100644 index 00000000000..bdf2ca40142 --- /dev/null +++ b/tensorflow/python/keras/applications/mobilenet_v3.py @@ -0,0 +1,567 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# pylint: disable=invalid-name +# pylint: disable=missing-function-docstring +"""MobileNet v3 models for Keras.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + + +from tensorflow.python.keras import backend +from tensorflow.python.keras import models +from tensorflow.python.keras.applications import imagenet_utils +from tensorflow.python.keras.layers import VersionAwareLayers +from tensorflow.python.keras.utils import data_utils +from tensorflow.python.keras.utils import layer_utils +from tensorflow.python.lib.io import file_io +from tensorflow.python.platform import tf_logging as logging +from tensorflow.python.util.tf_export import keras_export + + +# TODO(scottzhu): Change this to the GCS path. +BASE_WEIGHT_PATH = ('https://storage.googleapis.com/tensorflow/' + 'keras-applications/mobilenet_v3/') +WEIGHTS_HASHES = { + 'large_224_0.75_float': ('765b44a33ad4005b3ac83185abf1d0eb', + 'e7b4d1071996dd51a2c2ca2424570e20'), + 'large_224_1.0_float': ('59e551e166be033d707958cf9e29a6a7', + '037116398e07f018c0005ffcb0406831'), + 'large_minimalistic_224_1.0_float': ('675e7b876c45c57e9e63e6d90a36599c', + 'a2c33aed672524d1d0b4431808177695'), + 'small_224_0.75_float': ('cb65d4e5be93758266aa0a7f2c6708b7', + '4d2fe46f1c1f38057392514b0df1d673'), + 'small_224_1.0_float': ('8768d4c2e7dee89b9d02b2d03d65d862', + 'be7100780f875c06bcab93d76641aa26'), + 'small_minimalistic_224_1.0_float': ('99cd97fb2fcdad2bf028eb838de69e37', + '20d4e357df3f7a6361f3a288857b1051'), +} + +layers = VersionAwareLayers() + + +BASE_DOCSTRING = """Instantiates the {name} architecture. + + Reference: + - [Searching for MobileNetV3]( + https://arxiv.org/pdf/1905.02244.pdf) (ICCV 2019) + + The following table describes the performance of MobileNets: + ------------------------------------------------------------------------ + MACs stands for Multiply Adds + + |Classification Checkpoint|MACs(M)|Parameters(M)|Top1 Accuracy|Pixel1|CPU(ms)| + | [mobilenet_v3_large_1.0_224] | 217 | 5.4 | 75.6 | 51.2 | + | [mobilenet_v3_large_0.75_224] | 155 | 4.0 | 73.3 | 39.8 | + | [mobilenet_v3_large_minimalistic_1.0_224] | 209 | 3.9 | 72.3 | 44.1 | + | [mobilenet_v3_small_1.0_224] | 66 | 2.9 | 68.1 | 15.8 | + | [mobilenet_v3_small_0.75_224] | 44 | 2.4 | 65.4 | 12.8 | + | [mobilenet_v3_small_minimalistic_1.0_224] | 65 | 2.0 | 61.9 | 12.2 | + + The weights for all 6 models are obtained and translated from the Tensorflow + checkpoints from TensorFlow checkpoints found [here] + (https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet/README.md). + + Optionally loads weights pre-trained on ImageNet. + + Caution: Be sure to properly pre-process your inputs to the application. + Please see `applications.mobilenet_v3.preprocess_input` for an example. + + Arguments: + input_shape: Optional shape tuple, to be specified if you would + like to use a model with an input image resolution that is not + (224, 224, 3). + It should have exactly 3 inputs channels (224, 224, 3). + You can also omit this option if you would like + to infer input_shape from an input_tensor. + If you choose to include both input_tensor and input_shape then + input_shape will be used if they match, if the shapes + do not match then we will throw an error. + E.g. `(160, 160, 3)` would be one valid value. + alpha: controls the width of the network. This is known as the + depth multiplier in the MobileNetV3 paper, but the name is kept for + consistency with MobileNetV1 in Keras. + - If `alpha` < 1.0, proportionally decreases the number + of filters in each layer. + - If `alpha` > 1.0, proportionally increases the number + of filters in each layer. + - If `alpha` = 1, default number of filters from the paper + are used at each layer. + minimalistic: In addition to large and small models this module also + contains so-called minimalistic models, these models have the same + per-layer dimensions characteristic as MobilenetV3 however, they don't + utilize any of the advanced blocks (squeeze-and-excite units, hard-swish, + and 5x5 convolutions). While these models are less efficient on CPU, they + are much more performant on GPU/DSP. + include_top: Boolean, whether to include the fully-connected + layer at the top of the network. Defaults to `True`. + weights: String, one of `None` (random initialization), + 'imagenet' (pre-training on ImageNet), + or the path to the weights file to be loaded. + input_tensor: Optional Keras tensor (i.e. output of + `layers.Input()`) + to use as image input for the model. + pooling: String, optional pooling mode for feature extraction + when `include_top` is `False`. + - `None` means that the output of the model + will be the 4D tensor output of the + last convolutional block. + - `avg` means that global average pooling + will be applied to the output of the + last convolutional block, and thus + the output of the model will be a + 2D tensor. + - `max` means that global max pooling will + be applied. + classes: Integer, optional number of classes to classify images + into, only to be specified if `include_top` is True, and + if no `weights` argument is specified. + dropout_rate: fraction of the input units to drop on the last layer. + classifier_activation: A `str` or callable. The activation function to use + on the "top" layer. Ignored unless `include_top=True`. Set + `classifier_activation=None` to return the logits of the "top" layer. + + Returns: + A `keras.Model` instance. + + Raises: + ValueError: in case of invalid argument for `weights`, + or invalid input shape or invalid alpha, rows when + weights='imagenet' + ValueError: if `classifier_activation` is not `softmax` or `None` when + using a pretrained top layer. +""" + + +def MobileNetV3(stack_fn, + last_point_ch, + input_shape=None, + alpha=1.0, + model_type='large', + minimalistic=False, + include_top=True, + weights='imagenet', + input_tensor=None, + classes=1000, + pooling=None, + dropout_rate=0.2, + classifier_activation='softmax'): + if not (weights in {'imagenet', None} or file_io.file_exists(weights)): + raise ValueError('The `weights` argument should be either ' + '`None` (random initialization), `imagenet` ' + '(pre-training on ImageNet), ' + 'or the path to the weights file to be loaded.') + + if weights == 'imagenet' and include_top and classes != 1000: + raise ValueError('If using `weights` as `"imagenet"` with `include_top` ' + 'as true, `classes` should be 1000') + + # Determine proper input shape and default size. + # If both input_shape and input_tensor are used, they should match + if input_shape is not None and input_tensor is not None: + try: + is_input_t_tensor = backend.is_keras_tensor(input_tensor) + except ValueError: + try: + is_input_t_tensor = backend.is_keras_tensor( + layer_utils.get_source_inputs(input_tensor)) + except ValueError: + raise ValueError('input_tensor: ', input_tensor, + 'is not type input_tensor') + if is_input_t_tensor: + if backend.image_data_format == 'channels_first': + if backend.int_shape(input_tensor)[1] != input_shape[1]: + raise ValueError('input_shape: ', input_shape, 'and input_tensor: ', + input_tensor, + 'do not meet the same shape requirements') + else: + if backend.int_shape(input_tensor)[2] != input_shape[1]: + raise ValueError('input_shape: ', input_shape, 'and input_tensor: ', + input_tensor, + 'do not meet the same shape requirements') + else: + raise ValueError('input_tensor specified: ', input_tensor, + 'is not a keras tensor') + + # If input_shape is None, infer shape from input_tensor + if input_shape is None and input_tensor is not None: + + try: + backend.is_keras_tensor(input_tensor) + except ValueError: + raise ValueError('input_tensor: ', input_tensor, 'is type: ', + type(input_tensor), 'which is not a valid type') + + if backend.is_keras_tensor(input_tensor): + if backend.image_data_format() == 'channels_first': + rows = backend.int_shape(input_tensor)[2] + cols = backend.int_shape(input_tensor)[3] + input_shape = (3, cols, rows) + else: + rows = backend.int_shape(input_tensor)[1] + cols = backend.int_shape(input_tensor)[2] + input_shape = (cols, rows, 3) + # If input_shape is None and input_tensor is None using standart shape + if input_shape is None and input_tensor is None: + input_shape = (None, None, 3) + + if backend.image_data_format() == 'channels_last': + row_axis, col_axis = (0, 1) + else: + row_axis, col_axis = (1, 2) + rows = input_shape[row_axis] + cols = input_shape[col_axis] + if rows and cols and (rows < 32 or cols < 32): + raise ValueError('Input size must be at least 32x32; got `input_shape=' + + str(input_shape) + '`') + if weights == 'imagenet': + if (not minimalistic and alpha not in [0.75, 1.0] + or minimalistic and alpha != 1.0): + raise ValueError('If imagenet weights are being loaded, ' + 'alpha can be one of `0.75`, `1.0` for non minimalistic' + ' or `1.0` for minimalistic only.') + + if rows != cols or rows != 224: + logging.warning('`input_shape` is undefined or non-square, ' + 'or `rows` is not 224.' + ' Weights for input shape (224, 224) will be' + ' loaded as the default.') + + if input_tensor is None: + img_input = layers.Input(shape=input_shape) + else: + if not backend.is_keras_tensor(input_tensor): + img_input = layers.Input(tensor=input_tensor, shape=input_shape) + else: + img_input = input_tensor + + channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 + + if minimalistic: + kernel = 3 + activation = relu + se_ratio = None + else: + kernel = 5 + activation = hard_swish + se_ratio = 0.25 + + x = img_input + x = layers.Rescaling(1. / 255.)(x) + x = layers.Conv2D( + 16, + kernel_size=3, + strides=(2, 2), + padding='same', + use_bias=False, + name='Conv')(x) + x = layers.BatchNormalization( + axis=channel_axis, epsilon=1e-3, + momentum=0.999, name='Conv/BatchNorm')(x) + x = activation(x) + + x = stack_fn(x, kernel, activation, se_ratio) + + last_conv_ch = _depth(backend.int_shape(x)[channel_axis] * 6) + + # if the width multiplier is greater than 1 we + # increase the number of output channels + if alpha > 1.0: + last_point_ch = _depth(last_point_ch * alpha) + x = layers.Conv2D( + last_conv_ch, + kernel_size=1, + padding='same', + use_bias=False, + name='Conv_1')(x) + x = layers.BatchNormalization( + axis=channel_axis, epsilon=1e-3, + momentum=0.999, name='Conv_1/BatchNorm')(x) + x = activation(x) + x = layers.Conv2D( + last_point_ch, + kernel_size=1, + padding='same', + use_bias=True, + name='Conv_2')(x) + x = activation(x) + + if include_top: + x = layers.GlobalAveragePooling2D()(x) + if channel_axis == 1: + x = layers.Reshape((last_point_ch, 1, 1))(x) + else: + x = layers.Reshape((1, 1, last_point_ch))(x) + if dropout_rate > 0: + x = layers.Dropout(dropout_rate)(x) + x = layers.Conv2D(classes, kernel_size=1, padding='same', name='Logits')(x) + x = layers.Flatten()(x) + imagenet_utils.validate_activation(classifier_activation, weights) + x = layers.Activation(activation=classifier_activation, + name='Predictions')(x) + else: + if pooling == 'avg': + x = layers.GlobalAveragePooling2D(name='avg_pool')(x) + elif pooling == 'max': + x = layers.GlobalMaxPooling2D(name='max_pool')(x) + # Ensure that the model takes into account + # any potential predecessors of `input_tensor`. + if input_tensor is not None: + inputs = layer_utils.get_source_inputs(input_tensor) + else: + inputs = img_input + + # Create model. + model = models.Model(inputs, x, name='MobilenetV3' + model_type) + + # Load weights. + if weights == 'imagenet': + model_name = '{}{}_224_{}_float'.format( + model_type, '_minimalistic' if minimalistic else '', str(alpha)) + if include_top: + file_name = 'weights_mobilenet_v3_' + model_name + '.h5' + file_hash = WEIGHTS_HASHES[model_name][0] + else: + file_name = 'weights_mobilenet_v3_' + model_name + '_no_top.h5' + file_hash = WEIGHTS_HASHES[model_name][1] + weights_path = data_utils.get_file( + file_name, + BASE_WEIGHT_PATH + file_name, + cache_subdir='models', + file_hash=file_hash) + model.load_weights(weights_path) + elif weights is not None: + model.load_weights(weights) + + return model + + +@keras_export('keras.applications.MobileNetV3Samll') +def MobileNetV3Small(input_shape=None, + alpha=1.0, + minimalistic=False, + include_top=True, + weights='imagenet', + input_tensor=None, + classes=1000, + pooling=None, + dropout_rate=0.2, + classifier_activation='softmax'): + + def stack_fn(x, kernel, activation, se_ratio): + + def depth(d): + return _depth(d * alpha) + + x = _inverted_res_block(x, 1, depth(16), 3, 2, se_ratio, relu, 0) + x = _inverted_res_block(x, 72. / 16, depth(24), 3, 2, None, relu, 1) + x = _inverted_res_block(x, 88. / 24, depth(24), 3, 1, None, relu, 2) + x = _inverted_res_block(x, 4, depth(40), kernel, 2, se_ratio, activation, 3) + x = _inverted_res_block(x, 6, depth(40), kernel, 1, se_ratio, activation, 4) + x = _inverted_res_block(x, 6, depth(40), kernel, 1, se_ratio, activation, 5) + x = _inverted_res_block(x, 3, depth(48), kernel, 1, se_ratio, activation, 6) + x = _inverted_res_block(x, 3, depth(48), kernel, 1, se_ratio, activation, 7) + x = _inverted_res_block(x, 6, depth(96), kernel, 2, se_ratio, activation, 8) + x = _inverted_res_block(x, 6, depth(96), kernel, 1, se_ratio, activation, 9) + x = _inverted_res_block(x, 6, depth(96), kernel, 1, se_ratio, activation, + 10) + return x + + return MobileNetV3(stack_fn, 1024, input_shape, alpha, 'small', minimalistic, + include_top, weights, input_tensor, classes, pooling, + dropout_rate, classifier_activation) + + +@keras_export('keras.applications.MobileNetV3Large') +def MobileNetV3Large(input_shape=None, + alpha=1.0, + minimalistic=False, + include_top=True, + weights='imagenet', + input_tensor=None, + classes=1000, + pooling=None, + dropout_rate=0.2, + classifier_activation='softmax'): + + def stack_fn(x, kernel, activation, se_ratio): + + def depth(d): + return _depth(d * alpha) + + x = _inverted_res_block(x, 1, depth(16), 3, 1, None, relu, 0) + x = _inverted_res_block(x, 4, depth(24), 3, 2, None, relu, 1) + x = _inverted_res_block(x, 3, depth(24), 3, 1, None, relu, 2) + x = _inverted_res_block(x, 3, depth(40), kernel, 2, se_ratio, relu, 3) + x = _inverted_res_block(x, 3, depth(40), kernel, 1, se_ratio, relu, 4) + x = _inverted_res_block(x, 3, depth(40), kernel, 1, se_ratio, relu, 5) + x = _inverted_res_block(x, 6, depth(80), 3, 2, None, activation, 6) + x = _inverted_res_block(x, 2.5, depth(80), 3, 1, None, activation, 7) + x = _inverted_res_block(x, 2.3, depth(80), 3, 1, None, activation, 8) + x = _inverted_res_block(x, 2.3, depth(80), 3, 1, None, activation, 9) + x = _inverted_res_block(x, 6, depth(112), 3, 1, se_ratio, activation, 10) + x = _inverted_res_block(x, 6, depth(112), 3, 1, se_ratio, activation, 11) + x = _inverted_res_block(x, 6, depth(160), kernel, 2, se_ratio, activation, + 12) + x = _inverted_res_block(x, 6, depth(160), kernel, 1, se_ratio, activation, + 13) + x = _inverted_res_block(x, 6, depth(160), kernel, 1, se_ratio, activation, + 14) + return x + + return MobileNetV3(stack_fn, 1280, input_shape, alpha, 'large', minimalistic, + include_top, weights, input_tensor, classes, pooling, + dropout_rate, classifier_activation) + + +MobileNetV3Small.__doc__ = BASE_DOCSTRING.format(name='MobileNetV3Small') +MobileNetV3Large.__doc__ = BASE_DOCSTRING.format(name='MobileNetV3Large') + + +def relu(x): + return layers.ReLU()(x) + + +def hard_sigmoid(x): + return layers.ReLU(6.)(x + 3.) * (1. / 6.) + + +def hard_swish(x): + return layers.Multiply()([hard_sigmoid(x), x]) + + +# This function is taken from the original tf repo. +# It ensures that all layers have a channel number that is divisible by 8 +# It can be seen here: +# https://github.com/tensorflow/models/blob/master/research/ +# slim/nets/mobilenet/mobilenet.py + + +def _depth(v, divisor=8, min_value=None): + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +def _se_block(inputs, filters, se_ratio, prefix): + x = layers.GlobalAveragePooling2D(name=prefix + 'squeeze_excite/AvgPool')( + inputs) + if backend.image_data_format() == 'channels_first': + x = layers.Reshape((filters, 1, 1))(x) + else: + x = layers.Reshape((1, 1, filters))(x) + x = layers.Conv2D( + _depth(filters * se_ratio), + kernel_size=1, + padding='same', + name=prefix + 'squeeze_excite/Conv')( + x) + x = layers.ReLU(name=prefix + 'squeeze_excite/Relu')(x) + x = layers.Conv2D( + filters, + kernel_size=1, + padding='same', + name=prefix + 'squeeze_excite/Conv_1')( + x) + x = hard_sigmoid(x) + x = layers.Multiply(name=prefix + 'squeeze_excite/Mul')([inputs, x]) + return x + + +def _inverted_res_block(x, expansion, filters, kernel_size, stride, se_ratio, + activation, block_id): + channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 + shortcut = x + prefix = 'expanded_conv/' + infilters = backend.int_shape(x)[channel_axis] + if block_id: + # Expand + prefix = 'expanded_conv_{}/'.format(block_id) + x = layers.Conv2D( + _depth(infilters * expansion), + kernel_size=1, + padding='same', + use_bias=False, + name=prefix + 'expand')( + x) + x = layers.BatchNormalization( + axis=channel_axis, + epsilon=1e-3, + momentum=0.999, + name=prefix + 'expand/BatchNorm')( + x) + x = activation(x) + + if stride == 2: + x = layers.ZeroPadding2D( + padding=imagenet_utils.correct_pad(x, kernel_size), + name=prefix + 'depthwise/pad')( + x) + x = layers.DepthwiseConv2D( + kernel_size, + strides=stride, + padding='same' if stride == 1 else 'valid', + use_bias=False, + name=prefix + 'depthwise')( + x) + x = layers.BatchNormalization( + axis=channel_axis, + epsilon=1e-3, + momentum=0.999, + name=prefix + 'depthwise/BatchNorm')( + x) + x = activation(x) + + if se_ratio: + x = _se_block(x, _depth(infilters * expansion), se_ratio, prefix) + + x = layers.Conv2D( + filters, + kernel_size=1, + padding='same', + use_bias=False, + name=prefix + 'project')( + x) + x = layers.BatchNormalization( + axis=channel_axis, + epsilon=1e-3, + momentum=0.999, + name=prefix + 'project/BatchNorm')( + x) + + if stride == 1 and infilters == filters: + x = layers.Add(name=prefix + 'Add')([shortcut, x]) + return x + + +@keras_export('keras.applications.mobilenet_v3.preprocess_input') +def preprocess_input(x, data_format=None): # pylint: disable=unused-argument + return x + + +@keras_export('keras.applications.mobilenet_v3.decode_predictions') +def decode_predictions(preds, top=5): + return imagenet_utils.decode_predictions(preds, top=top) + + +preprocess_input.__doc__ = imagenet_utils.PREPROCESS_INPUT_DOC.format( + mode='', + ret=imagenet_utils.PREPROCESS_INPUT_RET_DOC_TF, + error=imagenet_utils.PREPROCESS_INPUT_ERROR_DOC) +decode_predictions.__doc__ = imagenet_utils.decode_predictions.__doc__ diff --git a/tensorflow/python/tools/api/generator/api_init_files.bzl b/tensorflow/python/tools/api/generator/api_init_files.bzl index d2770e92b2e..c7a788450dc 100644 --- a/tensorflow/python/tools/api/generator/api_init_files.bzl +++ b/tensorflow/python/tools/api/generator/api_init_files.bzl @@ -97,6 +97,7 @@ KERAS_API_INIT_FILES = [ "keras/applications/inception_v3/__init__.py", "keras/applications/mobilenet/__init__.py", "keras/applications/mobilenet_v2/__init__.py", + "keras/applications/mobilenet_v3/__init__.py", "keras/applications/nasnet/__init__.py", "keras/applications/resnet/__init__.py", "keras/applications/resnet_v2/__init__.py", diff --git a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl index d1761b4d2bc..36593eff901 100644 --- a/tensorflow/python/tools/api/generator/api_init_files_v1.bzl +++ b/tensorflow/python/tools/api/generator/api_init_files_v1.bzl @@ -113,6 +113,7 @@ KERAS_API_INIT_FILES_V1 = [ "keras/applications/inception_v3/__init__.py", "keras/applications/mobilenet/__init__.py", "keras/applications/mobilenet_v2/__init__.py", + "keras/applications/mobilenet_v3/__init__.py", "keras/applications/nasnet/__init__.py", "keras/applications/resnet/__init__.py", "keras/applications/resnet_v2/__init__.py", diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.mobilenet_v3.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.mobilenet_v3.pbtxt new file mode 100644 index 00000000000..418ace0882f --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.mobilenet_v3.pbtxt @@ -0,0 +1,11 @@ +path: "tensorflow.keras.applications.mobilenet_v3" +tf_module { + member_method { + name: "decode_predictions" + argspec: "args=[\'preds\', \'top\'], varargs=None, keywords=None, defaults=[\'5\'], " + } + member_method { + name: "preprocess_input" + argspec: "args=[\'x\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.pbtxt index 900df849f45..9f367742398 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.pbtxt @@ -28,6 +28,10 @@ tf_module { name: "mobilenet_v2" mtype: "" } + member { + name: "mobilenet_v3" + mtype: "" + } member { name: "nasnet" mtype: "" @@ -116,6 +120,14 @@ tf_module { name: "MobileNetV2" argspec: "args=[\'input_shape\', \'alpha\', \'include_top\', \'weights\', \'input_tensor\', \'pooling\', \'classes\', \'classifier_activation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'1.0\', \'True\', \'imagenet\', \'None\', \'None\', \'1000\', \'softmax\'], " } + member_method { + name: "MobileNetV3Large" + argspec: "args=[\'input_shape\', \'alpha\', \'minimalistic\', \'include_top\', \'weights\', \'input_tensor\', \'classes\', \'pooling\', \'dropout_rate\', \'classifier_activation\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'False\', \'True\', \'imagenet\', \'None\', \'1000\', \'None\', \'0.2\', \'softmax\'], " + } + member_method { + name: "MobileNetV3Samll" + argspec: "args=[\'input_shape\', \'alpha\', \'minimalistic\', \'include_top\', \'weights\', \'input_tensor\', \'classes\', \'pooling\', \'dropout_rate\', \'classifier_activation\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'False\', \'True\', \'imagenet\', \'None\', \'1000\', \'None\', \'0.2\', \'softmax\'], " + } member_method { name: "NASNetLarge" argspec: "args=[\'input_shape\', \'include_top\', \'weights\', \'input_tensor\', \'pooling\', \'classes\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'imagenet\', \'None\', \'None\', \'1000\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.mobilenet_v3.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.mobilenet_v3.pbtxt new file mode 100644 index 00000000000..418ace0882f --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.mobilenet_v3.pbtxt @@ -0,0 +1,11 @@ +path: "tensorflow.keras.applications.mobilenet_v3" +tf_module { + member_method { + name: "decode_predictions" + argspec: "args=[\'preds\', \'top\'], varargs=None, keywords=None, defaults=[\'5\'], " + } + member_method { + name: "preprocess_input" + argspec: "args=[\'x\', \'data_format\'], varargs=None, keywords=None, defaults=[\'None\'], " + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.pbtxt index 900df849f45..9f367742398 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.pbtxt @@ -28,6 +28,10 @@ tf_module { name: "mobilenet_v2" mtype: "" } + member { + name: "mobilenet_v3" + mtype: "" + } member { name: "nasnet" mtype: "" @@ -116,6 +120,14 @@ tf_module { name: "MobileNetV2" argspec: "args=[\'input_shape\', \'alpha\', \'include_top\', \'weights\', \'input_tensor\', \'pooling\', \'classes\', \'classifier_activation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'1.0\', \'True\', \'imagenet\', \'None\', \'None\', \'1000\', \'softmax\'], " } + member_method { + name: "MobileNetV3Large" + argspec: "args=[\'input_shape\', \'alpha\', \'minimalistic\', \'include_top\', \'weights\', \'input_tensor\', \'classes\', \'pooling\', \'dropout_rate\', \'classifier_activation\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'False\', \'True\', \'imagenet\', \'None\', \'1000\', \'None\', \'0.2\', \'softmax\'], " + } + member_method { + name: "MobileNetV3Samll" + argspec: "args=[\'input_shape\', \'alpha\', \'minimalistic\', \'include_top\', \'weights\', \'input_tensor\', \'classes\', \'pooling\', \'dropout_rate\', \'classifier_activation\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'False\', \'True\', \'imagenet\', \'None\', \'1000\', \'None\', \'0.2\', \'softmax\'], " + } member_method { name: "NASNetLarge" argspec: "args=[\'input_shape\', \'include_top\', \'weights\', \'input_tensor\', \'pooling\', \'classes\'], varargs=None, keywords=None, defaults=[\'None\', \'True\', \'imagenet\', \'None\', \'None\', \'1000\'], " From 83d4de42ff4339f22cf08c9a707872e69415f85e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 9 Aug 2020 22:46:00 -0700 Subject: [PATCH 0726/1017] Add CompileTimeConstantInput to XlaGather. PiperOrigin-RevId: 325743288 Change-Id: Ia2629ba3562a17937decd381ec2cb28695422b86 --- tensorflow/compiler/tests/xla_ops_test.py | 19 +++++++++++++++++++ .../tf2xla/kernels/gather_scatter_ops.cc | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/tests/xla_ops_test.py b/tensorflow/compiler/tests/xla_ops_test.py index 0d6ae81ef6e..3e9f5e8c5dd 100644 --- a/tensorflow/compiler/tests/xla_ops_test.py +++ b/tensorflow/compiler/tests/xla_ops_test.py @@ -79,6 +79,25 @@ class XlaOpsNumericalTest(xla_test.XLATestCase, parameterized.TestCase): args=(v,), expected=np.tile(v, (7, 42, 1, 1))) + @test_util.disable_mlir_bridge('Not supported yet') + def testGather(self): + operand = np.arange(10, dtype=np.int32).reshape([2, 5]) + start_indices = np.array([2], np.int32) + slice_sizes = np.array([1, 3], np.int32) + + def gather(operand, start_indices): + dimension_numbers = xla_data_pb2.GatherDimensionNumbers() + dimension_numbers.offset_dims.extend([1]) + dimension_numbers.collapsed_slice_dims.extend([0]) + dimension_numbers.start_index_map.extend([0]) + dimension_numbers.index_vector_dim = 1 + return xla.gather(operand, start_indices, dimension_numbers, slice_sizes) + + self._assertOpOutputMatchesExpected( + gather, + args=(operand, start_indices), + expected=np.array([[5, 6, 7]])) + @test_util.disable_mlir_bridge('Dynamic result types not supported') def testShiftRightLogical(self): self._assertOpOutputMatchesExpected( diff --git a/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc b/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc index 19aa85f9d42..b4b18dd2b36 100644 --- a/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc @@ -49,7 +49,8 @@ class GatherOp : public XlaOpKernel { bool indices_are_sorted_; }; -REGISTER_XLA_OP(Name("XlaGather"), GatherOp); +REGISTER_XLA_OP(Name("XlaGather").CompileTimeConstantInput("slice_sizes"), + GatherOp); class ScatterOp : public XlaOpKernel { public: From 24e203fa08feee48c766b15eaa3afcc912324437 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Sun, 9 Aug 2020 23:04:38 -0700 Subject: [PATCH 0727/1017] Add CompileTimeConstantInput to XlaGather. PiperOrigin-RevId: 325744813 Change-Id: I26a4f02376493e0642a251a8a504bfcefc93c6ce --- tensorflow/compiler/tests/xla_ops_test.py | 19 ------------------- .../tf2xla/kernels/gather_scatter_ops.cc | 3 +-- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/tensorflow/compiler/tests/xla_ops_test.py b/tensorflow/compiler/tests/xla_ops_test.py index 3e9f5e8c5dd..0d6ae81ef6e 100644 --- a/tensorflow/compiler/tests/xla_ops_test.py +++ b/tensorflow/compiler/tests/xla_ops_test.py @@ -79,25 +79,6 @@ class XlaOpsNumericalTest(xla_test.XLATestCase, parameterized.TestCase): args=(v,), expected=np.tile(v, (7, 42, 1, 1))) - @test_util.disable_mlir_bridge('Not supported yet') - def testGather(self): - operand = np.arange(10, dtype=np.int32).reshape([2, 5]) - start_indices = np.array([2], np.int32) - slice_sizes = np.array([1, 3], np.int32) - - def gather(operand, start_indices): - dimension_numbers = xla_data_pb2.GatherDimensionNumbers() - dimension_numbers.offset_dims.extend([1]) - dimension_numbers.collapsed_slice_dims.extend([0]) - dimension_numbers.start_index_map.extend([0]) - dimension_numbers.index_vector_dim = 1 - return xla.gather(operand, start_indices, dimension_numbers, slice_sizes) - - self._assertOpOutputMatchesExpected( - gather, - args=(operand, start_indices), - expected=np.array([[5, 6, 7]])) - @test_util.disable_mlir_bridge('Dynamic result types not supported') def testShiftRightLogical(self): self._assertOpOutputMatchesExpected( diff --git a/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc b/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc index b4b18dd2b36..19aa85f9d42 100644 --- a/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc @@ -49,8 +49,7 @@ class GatherOp : public XlaOpKernel { bool indices_are_sorted_; }; -REGISTER_XLA_OP(Name("XlaGather").CompileTimeConstantInput("slice_sizes"), - GatherOp); +REGISTER_XLA_OP(Name("XlaGather"), GatherOp); class ScatterOp : public XlaOpKernel { public: From e800ce58624851b0c6cca284e2ab8a97e028bc69 Mon Sep 17 00:00:00 2001 From: Yuqi Li Date: Sun, 9 Aug 2020 23:35:43 -0700 Subject: [PATCH 0728/1017] add installation from github source. PiperOrigin-RevId: 325746904 Change-Id: I093322bb6fa72489864e23604c3911aad7744593 --- tensorflow/lite/g3doc/guide/model_maker.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tensorflow/lite/g3doc/guide/model_maker.md b/tensorflow/lite/g3doc/guide/model_maker.md index 76b32eac75e..3746dbd6c9f 100644 --- a/tensorflow/lite/g3doc/guide/model_maker.md +++ b/tensorflow/lite/g3doc/guide/model_maker.md @@ -43,8 +43,18 @@ For more details, see the ## Installation -Install a prebuilt pip package. +There are two ways to install Model Maker. + +* Install a prebuilt pip package. ```shell pip install tflite-model-maker ``` + +* Clone the source code from GitHub and install. + +```shell +git clone https://github.com/tensorflow/examples +cd examples/tensorflow_examples/lite/model_maker/pip_package +pip install -e . +``` From fc11ea4eec3a74dda5f7fd302ec5da07a12cdb82 Mon Sep 17 00:00:00 2001 From: Yuanzhong Xu Date: Sun, 9 Aug 2020 23:48:22 -0700 Subject: [PATCH 0729/1017] [XLA] Partial sharding and propagation for broadcast/reduce PiperOrigin-RevId: 325747712 Change-Id: I31b795ec0a2138b427a0818939331d7fe30bd457 --- .../compiler/xla/service/hlo_sharding.cc | 27 +++- .../compiler/xla/service/hlo_sharding.h | 4 +- .../compiler/xla/service/hlo_sharding_util.cc | 64 +++++++++ .../compiler/xla/service/hlo_sharding_util.h | 11 ++ .../xla/service/sharding_propagation.cc | 99 ++++++------- .../xla/service/sharding_propagation_test.cc | 110 +++++++++++++++ .../xla/service/spmd/spmd_partitioner.cc | 132 +++++------------- .../xla/service/spmd/spmd_partitioner_test.cc | 50 +++++++ .../xla/service/spmd/spmd_partitioner_util.cc | 4 +- 9 files changed, 350 insertions(+), 151 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc index ba1fc0d0450..07444aca82b 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding.cc @@ -56,8 +56,28 @@ HloSharding HloSharding::PartialTile( HloSharding HloSharding::PartialTile( const Array& tile_assignment_last_dim_replicate) { - return HloSharding(tile_assignment_last_dim_replicate, - /*replicate_on_last_tile_dim=*/true); + std::vector> sorted_groups( + tile_assignment_last_dim_replicate.num_elements() / + tile_assignment_last_dim_replicate.dimensions().back()); + auto get_group_id = [&](absl::Span indices) { + int64 group_id = 0; + for (int64 i = 0; i < indices.size() - 1; ++i) { + group_id *= tile_assignment_last_dim_replicate.dim(i); + group_id += indices[i]; + } + return group_id; + }; + tile_assignment_last_dim_replicate.Each( + [&](absl::Span indices, const int64 device) { + sorted_groups[get_group_id(indices)].insert(device); + }); + Array sorted_tile(tile_assignment_last_dim_replicate.dimensions()); + sorted_tile.Each([&](absl::Span indices, int64* device) { + auto begin = sorted_groups[get_group_id(indices)].begin(); + *device = *begin; + sorted_groups[get_group_id(indices)].erase(begin); + }); + return HloSharding(sorted_tile, /*replicate_on_last_tile_dim=*/true); } HloSharding HloSharding::Tuple(const ShapeTree& sub_shardings) { @@ -437,7 +457,8 @@ Status HloSharding::ValidateNonTuple(const Shape& shape, proto.tile_assignment_dimensions().end())); std::copy(proto.tile_assignment_devices().begin(), proto.tile_assignment_devices().end(), tile_assignment.begin()); - return HloSharding(tile_assignment, proto.replicate_on_last_tile_dim()); + return proto.replicate_on_last_tile_dim() ? PartialTile(tile_assignment) + : HloSharding(tile_assignment); } OpSharding HloSharding::ToProto() const { diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h index 1b827efff2d..e7ba2bc0680 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding.h +++ b/tensorflow/compiler/xla/service/hlo_sharding.h @@ -56,14 +56,14 @@ class HloSharding { // Creates a new sharding where data is replicated within each replication // group, and sharded across replication groups according to - // group_tile_assignment. + // group_tile_assignment. Replication group members will be sorted. static HloSharding PartialTile( const Array& group_tile_assignment, absl::Span> replication_groups); // Creates a partially replicated tiled sharding with device-level tile // assignment, where the last dimension is the additional replication - // dimension. + // dimension. Replication group members will be sorted. static HloSharding PartialTile( const Array& tile_assignment_last_dim_replicate); diff --git a/tensorflow/compiler/xla/service/hlo_sharding_util.cc b/tensorflow/compiler/xla/service/hlo_sharding_util.cc index 94c348cdeaa..65295a8e620 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding_util.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding_util.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_sharding_util.h" #include +#include #include "absl/algorithm/container.h" #include "tensorflow/compiler/xla/array.h" @@ -777,5 +778,68 @@ std::vector DevicesForSharding( return devices; } +HloSharding PartiallyReplicateTiledShardingOnDims( + const HloSharding& sharding, const std::vector& dims_to_replicate) { + if (sharding.IsTileMaximal()) { + return sharding; + } + int64 group_count = 1; + for (int64 dim : dims_to_replicate) { + if (sharding.ReplicateOnLastTileDim()) { + CHECK_LT(dim, sharding.tile_assignment().num_dimensions()); + } + group_count *= sharding.tile_assignment().dim(dim); + } + if (group_count == 1) { + return sharding; + } + if (group_count == sharding.NumTiles()) { + return HloSharding::Replicate(); + } + std::vector dim_permutation( + sharding.tile_assignment().num_dimensions()); + std::iota(dim_permutation.begin(), dim_permutation.end(), 0); + absl::c_sort(dim_permutation, [&](const int64 a, const int64 b) { + return absl::c_linear_search(dims_to_replicate, a) < + absl::c_linear_search(dims_to_replicate, b); + }); + auto transposed = TransposeSharding(sharding, dim_permutation); + auto new_tile = transposed.tile_assignment(); + std::vector new_tile_shape( + sharding.tile_assignment().dimensions().begin(), + sharding.tile_assignment().dimensions().end()); + for (int64 dim : dims_to_replicate) { + new_tile_shape[dim] = 1; + } + if (sharding.ReplicateOnLastTileDim()) { + new_tile_shape.back() *= group_count; + } else { + new_tile_shape.push_back(group_count); + } + new_tile.Reshape(new_tile_shape); + return HloSharding::PartialTile(new_tile); +} + +HloSharding RemoveShapeDimensions(const HloSharding& sharding, + const std::vector& dims_to_remove) { + if (sharding.IsTileMaximal() || dims_to_remove.empty()) { + return sharding; + } + std::vector new_tile_shape; + new_tile_shape.reserve(sharding.tile_assignment().num_dimensions() - + dims_to_remove.size()); + for (int64 i = 0; i < sharding.tile_assignment().num_dimensions(); ++i) { + if (absl::c_linear_search(dims_to_remove, i)) { + CHECK_EQ(sharding.tile_assignment().dim(i), 1); + } else { + new_tile_shape.push_back(sharding.tile_assignment().dim(i)); + } + } + auto new_tile = sharding.tile_assignment(); + new_tile.Reshape(new_tile_shape); + return sharding.ReplicateOnLastTileDim() ? HloSharding::PartialTile(new_tile) + : HloSharding::Tile(new_tile); +} + } // namespace hlo_sharding_util } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_sharding_util.h b/tensorflow/compiler/xla/service/hlo_sharding_util.h index cc4068121ae..ce19d8c7a19 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding_util.h +++ b/tensorflow/compiler/xla/service/hlo_sharding_util.h @@ -163,6 +163,17 @@ IdentityValueAndHloOpcodeForScatterReduceComputation( std::vector DevicesForSharding( const HloSharding& sharding, const std::vector& available_devices); +// Returns a sharding that replicates data across devices along the given +// dimensions in the original sharding. +HloSharding PartiallyReplicateTiledShardingOnDims( + const HloSharding& sharding, const std::vector& dims_to_replicate); + +// Returns a sharding the removes given tile dimensions. +// +// Precondition: if not tile maximal, the size of each tile dimension must be 1. +HloSharding RemoveShapeDimensions(const HloSharding& sharding, + const std::vector& dims_to_remove); + } // namespace hlo_sharding_util } // namespace xla diff --git a/tensorflow/compiler/xla/service/sharding_propagation.cc b/tensorflow/compiler/xla/service/sharding_propagation.cc index 0e4b0568134..4ff492047a3 100644 --- a/tensorflow/compiler/xla/service/sharding_propagation.cc +++ b/tensorflow/compiler/xla/service/sharding_propagation.cc @@ -706,44 +706,37 @@ bool InferShardingFromOperands(HloInstruction* instruction, sharding); return HloSharding::Tuple(instruction->shape(), tuple); }; - if (operand->sharding().IsReplicated()) { + if (operand->sharding().IsReplicated() || + (!is_spmd && + absl::c_any_of(instruction->dimensions(), [operand](int64 dim) { + return operand->sharding().tile_assignment().dim(dim) > 1; + }))) { + // We are reducing along one of the sharded dimensions. We only + // support this in SPMD. changed |= MaybeImproveInstructionSharding( get_maybe_tuple_sharding(HloSharding::Replicate()), instruction, /*may_combine_partial_sharding=*/is_spmd); continue; } - if (absl::c_any_of(instruction->dimensions(), [operand](int64 dim) { - return operand->sharding().tile_assignment().dim(dim) > 1; - })) { - // We are reducing along one of the sharded dimensions. We don't - // support tiled sharding in this case. + auto after_partial_replication = + operand->sharding().IsReplicated() + ? operand->sharding() + : hlo_sharding_util::PartiallyReplicateTiledShardingOnDims( + operand->sharding(), instruction->dimensions()); + if (after_partial_replication.IsReplicated()) { changed |= MaybeImproveInstructionSharding( get_maybe_tuple_sharding(HloSharding::Replicate()), instruction, /*may_combine_partial_sharding=*/is_spmd); - } else { - // We are reducing along some of the non-sharded dimensions. The - // result sharding should be the same as the operand sharding with the - // reduction dimensions removed as they are removed from the result - // shape. - std::vector target_tile_assignment_dimensions; - const auto& dimensions = instruction->dimensions(); - for (int64 i = 0; i < operand->shape().rank(); ++i) { - if (absl::c_find(dimensions, i) == dimensions.end()) { - target_tile_assignment_dimensions.push_back( - operand->sharding().tile_assignment().dim(i)); - } - } - Array new_tile_assignment = - operand->sharding().tile_assignment(); - new_tile_assignment.Reshape(target_tile_assignment_dimensions); - // Use the same sharding for all tuple elements, because they are part - // of the same reduce instruction. - HloSharding new_sharding = - get_maybe_tuple_sharding(HloSharding::Tile(new_tile_assignment)); - changed |= MaybeImproveInstructionSharding( - new_sharding, instruction, - /*may_combine_partial_sharding=*/is_spmd); + continue; } + // Use the same sharding for all tuple elements, because they are part + // of the same reduce instruction. + HloSharding new_sharding = + get_maybe_tuple_sharding(hlo_sharding_util::RemoveShapeDimensions( + after_partial_replication, instruction->dimensions())); + changed |= MaybeImproveInstructionSharding( + new_sharding, instruction, + /*may_combine_partial_sharding=*/is_spmd); } return changed; } @@ -773,9 +766,16 @@ bool InferShardingFromOperands(HloInstruction* instruction, op->sharding().tile_assignment().dim(source_dim)); } } + if (op->sharding().ReplicateOnLastTileDim()) { + target_tile_assignment_dimensions.push_back( + op->sharding().tile_assignment().dimensions().back()); + } Array new_tile_assignment = op->sharding().tile_assignment(); new_tile_assignment.Reshape(target_tile_assignment_dimensions); - HloSharding new_sharding = HloSharding::Tile(new_tile_assignment); + HloSharding new_sharding = + op->sharding().ReplicateOnLastTileDim() + ? HloSharding::PartialTile(new_tile_assignment) + : HloSharding::Tile(new_tile_assignment); return MaybeImproveInstructionSharding( new_sharding, instruction, /*may_combine_partial_sharding=*/is_spmd); } @@ -1121,25 +1121,25 @@ absl::optional GetShardingFromUser( if (user.sharding().IsReplicated()) { return user.sharding(); } - // Only support when none of the partitioned dimensions in the broadcast - // output belong to new dimensions. + std::vector dims_to_replicate; + bool needs_replication = false; for (int64 i = 0; i < user.shape().rank(); ++i) { - if (user.sharding().tile_assignment().dim(i) > 1 && - absl::c_count(user.dimensions(), i) == 0) { - return absl::nullopt; + if (absl::c_count(user.dimensions(), i) == 0) { + dims_to_replicate.push_back(i); + if (user.sharding().tile_assignment().dim(i) > 1) { + needs_replication = true; + } } } - - // The instruction (operand of broadcast) will be tiled the same way - // as the output. - std::vector target_tile_assignment_dimensions; - for (int64 output_dim : user.dimensions()) { - target_tile_assignment_dimensions.push_back( - user.sharding().tile_assignment().dim(output_dim)); + // If not SPMD, only support when none of the partitioned dimensions in + // the broadcast output belong to new dimensions. + if (!is_spmd && needs_replication) { + return absl::nullopt; } - Array new_tile_assignment = user.sharding().tile_assignment(); - new_tile_assignment.Reshape(target_tile_assignment_dimensions); - return HloSharding::Tile(new_tile_assignment); + return hlo_sharding_util::RemoveShapeDimensions( + hlo_sharding_util::PartiallyReplicateTiledShardingOnDims( + user.sharding(), dims_to_replicate), + dims_to_replicate); } case HloOpcode::kConcatenate: { if (user.sharding().IsReplicated()) { @@ -1364,10 +1364,11 @@ absl::optional GetShardingFromUser( return user_sharding; } std::vector target_tile_assignment_dimensions( - instruction.shape().rank()); + instruction.shape().rank() + + (user_sharding.ReplicateOnLastTileDim() ? 1 : 0)); const auto& dimensions = user.dimensions(); int64 next_output_dim = 0; - for (int64 i = 0; i < instruction.shape().rank(); ++i) { + for (int64 i = 0; i < target_tile_assignment_dimensions.size(); ++i) { if (absl::c_find(dimensions, i) == dimensions.end()) { target_tile_assignment_dimensions[i] = user_sharding.tile_assignment().dim(next_output_dim++); @@ -1377,7 +1378,9 @@ absl::optional GetShardingFromUser( } auto tile_assignment = user_sharding.tile_assignment(); tile_assignment.Reshape(target_tile_assignment_dimensions); - return HloSharding::Tile(tile_assignment); + return user_sharding.ReplicateOnLastTileDim() + ? HloSharding::PartialTile(tile_assignment) + : HloSharding::Tile(tile_assignment); } case HloOpcode::kSort: { if (user.sharding().IsTuple()) { diff --git a/tensorflow/compiler/xla/service/sharding_propagation_test.cc b/tensorflow/compiler/xla/service/sharding_propagation_test.cc index 8aa10b67ed8..a182af001c2 100644 --- a/tensorflow/compiler/xla/service/sharding_propagation_test.cc +++ b/tensorflow/compiler/xla/service/sharding_propagation_test.cc @@ -118,6 +118,25 @@ ENTRY %broadcast { op::Sharding("{devices=[1,2,2,1]0,1,2,3}")); } +TEST_F(ShardingPropagationTest, BroadcastForwardPartial) { + const char* const hlo_string = R"( +HloModule module +ENTRY %broadcast { + %param0 = f32[3,2048]parameter(0), + sharding={devices=[1,2,2]0,1,2,3 last_tile_dim_replicate} + %broadcast = f32[3,2048,3] broadcast(%param0), dimensions={0,1} + ROOT %copy = f32[3,2048,3] copy(%broadcast) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT( + FindInstruction(module.get(), "broadcast"), + op::Sharding("{devices=[1,2,1,2]0,1,2,3 last_tile_dim_replicate}")); +} + TEST_F(ShardingPropagationTest, BroadcastUser) { const char* const hlo_string = R"( HloModule module @@ -136,6 +155,25 @@ ENTRY %broadcast { op::Sharding("{devices=[2,4]0,1,2,3,4,5,6,7}")); } +TEST_F(ShardingPropagationTest, BroadcastUserPartial) { + const char* const hlo_string = R"( +HloModule module +ENTRY %broadcast { + %param0 = f32[24,8]{0,1} parameter(0) + %copy = f32[24,8]{0,1} copy(%param0) + ROOT %broadcast = f32[4,24,6,8] broadcast(%copy), dimensions={1,3}, + sharding={devices=[4,2,1,1]0,1,2,3,4,5,6,7} +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT( + FindInstruction(module.get(), "copy"), + op::Sharding("{devices=[2,1,4]0,2,4,6,1,3,5,7 last_tile_dim_replicate}")); +} + TEST_F(ShardingPropagationTest, MaximalReduceForwardPass) { const char* const hlo_string = R"( HloModule module @@ -184,6 +222,78 @@ ENTRY %reduce { op::Sharding("{devices=[2,2]0,1,2,3}")); } +TEST_F(ShardingPropagationTest, ReducePartiallyOnTiledDims) { + const char* const hlo_string = R"( +HloModule module +%add { + %lhs = f32[] parameter(0) + %rhs = f32[] parameter(1) + ROOT %add = f32[] add(%lhs, %rhs) +} +ENTRY %reduce { + %param0 = f32[8,8] parameter(0), sharding={devices=[2,2]0,1,2,3} + %init = f32[] parameter(1) + %reduce = f32[8] reduce(%param0, %init), dimensions={0}, to_apply=%add + ROOT %copy = f32[8] copy(%reduce) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT(FindInstruction(module.get(), "reduce"), + op::Sharding("{devices=[2,2]0,2,1,3 last_tile_dim_replicate}")); +} + +TEST_F(ShardingPropagationTest, ReducePartiallyOnTiledDims2) { + const char* const hlo_string = R"( +HloModule module +%add { + %lhs = f32[] parameter(0) + %rhs = f32[] parameter(1) + ROOT %add = f32[] add(%lhs, %rhs) +} +ENTRY %reduce { + %param0 = f32[8,8] parameter(0), sharding={devices=[2,2,2]0,1,2,3,4,5,6,7 last_tile_dim_replicate} + %init = f32[] parameter(1) + %reduce = f32[8] reduce(%param0, %init), dimensions={0}, to_apply=%add + ROOT %copy = f32[8] copy(%reduce) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT( + FindInstruction(module.get(), "reduce"), + op::Sharding("{devices=[2,4]0,1,4,5,2,3,6,7 last_tile_dim_replicate}")); +} + +TEST_F(ShardingPropagationTest, ReducePartiallyBackward) { + const char* const hlo_string = R"( +HloModule module +%add { + %lhs = f32[] parameter(0) + %rhs = f32[] parameter(1) + ROOT %add = f32[] add(%lhs, %rhs) +} +ENTRY %reduce { + %param0 = f32[8,8] parameter(0) + %input = f32[8,8] copy(%param0) + %init = f32[] parameter(1) + %reduce = f32[8] reduce(%input, %init), dimensions={0}, to_apply=%add, + sharding={devices=[2,2]0,1,2,3 last_tile_dim_replicate} + ROOT %copy = f32[8] copy(%reduce) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN(bool changed, + ShardingPropagation().Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT(FindInstruction(module.get(), "input"), + op::Sharding("{devices=[1,2,2]0,1,2,3 last_tile_dim_replicate}")); +} + TEST_F(ShardingPropagationTest, ShardedTupleReduceForwardAndBackwardPass) { const char* const hlo_string = R"( HloModule module diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc index 813ccc46f32..fc065bcdd72 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc @@ -826,8 +826,9 @@ PartitionedHlo PartitionedHlo::ReshardWithAllToAll( sharding().tile_assignment().dim(source_dim); temp_target_tile.Reshape(temp_target_tile_dims); } - auto temp_target = HloSharding::Tile(temp_target_tile); - + auto temp_target = target.ReplicateOnLastTileDim() + ? HloSharding::PartialTile(temp_target_tile) + : HloSharding::Tile(temp_target_tile); auto padded_shape = hlo_->shape(); padded_shape.set_dimensions( target_dim, @@ -1974,67 +1975,22 @@ Status SpmdPartitioningVisitor::HandleBroadcast(HloInstruction* hlo) { auto& operand = GetPartitionedHlo(hlo->operand(0)); // Tiled output. - std::vector wanted_input_tile_size(operand.base_shape().rank()); - std::vector sharded_new_dims; - for (int64 i = 0; i < operand.base_shape().rank(); ++i) { - wanted_input_tile_size[i] = - hlo->sharding().tile_assignment().dim(hlo->dimensions(i)); - } + std::vector new_dims; for (int64 i = 0; i < hlo->shape().rank(); ++i) { - if (!absl::c_linear_search(hlo->dimensions(), i) && - hlo->sharding().tile_assignment().dim(i) > 1) { - sharded_new_dims.push_back(i); + if (!absl::c_linear_search(hlo->dimensions(), i)) { + new_dims.push_back(i); } } - if (sharded_new_dims.empty()) { - // The new dimensions are replicated, so that we can do the adjustment on - // the input. - Array wanted_input_tile_assignment(wanted_input_tile_size); - wanted_input_tile_assignment.Each( - [&](absl::Span indices, int64* val) { - std::vector indices_in_broadcast(hlo->shape().rank(), 0); - for (int64 i = 0; i < operand.base_shape().rank(); ++i) { - indices_in_broadcast[hlo->dimensions(i)] = indices[i]; - } - *val = hlo->sharding().tile_assignment()(indices_in_broadcast); - }); - SetPartitionedHlo(hlo, [&] { - return b_.AddInstruction(hlo->CloneWithNewOperands( - MakePartitionedShape(hlo->shape(), hlo->sharding()), - {operand.Reshard(HloSharding::Tile(wanted_input_tile_assignment)) - .hlo()})); - }); - } else { - auto input = operand.Reshard(HloSharding::Replicate()).hlo(); - // We pad and shard the input first, then broadcast to the final shard - // shape. - auto output_offsets = - MakePartitionOffsets(hlo->shape(), hlo->sharding(), partition_id_, &b_); - std::vector input_offsets(operand.base_shape().rank()); - auto output_shard_shape = - MakePartitionedShape(hlo->shape(), hlo->sharding()); - auto input_shard_shape = input->shape(); - auto padded_input_shape = input->shape(); - for (int64 i = 0; i < input_offsets.size(); ++i) { - input_offsets[i] = output_offsets[hlo->dimensions(i)]; - input_shard_shape.set_dimensions( - i, output_shard_shape.dimensions(hlo->dimensions(i))); - padded_input_shape.set_dimensions( - i, hlo->sharding().tile_assignment().dim(hlo->dimensions(i)) * - input_shard_shape.dimensions(i)); - } - auto padded_input = PadToShape(input, padded_input_shape, &b_); - auto input_shard = - ShapeUtil::Compatible(input_shard_shape, padded_input->shape()) - ? padded_input - : b_.AddInstruction(HloInstruction::CreateDynamicSlice( - input_shard_shape, padded_input, input_offsets, - input_shard_shape.dimensions())); - SetPartitionedHlo(hlo, [&] { - return b_.AddInstruction( - hlo->CloneWithNewOperands(output_shard_shape, {input_shard})); - }); - } + auto desired_input_sharding = hlo_sharding_util::RemoveShapeDimensions( + hlo_sharding_util::PartiallyReplicateTiledShardingOnDims(hlo->sharding(), + new_dims), + new_dims); + auto input = operand.Reshard(desired_input_sharding).hlo(); + auto output_shard_shape = MakePartitionedShape(hlo->shape(), hlo->sharding()); + SetPartitionedHlo(hlo, [&] { + return b_.AddInstruction( + hlo->CloneWithNewOperands(output_shard_shape, {input})); + }); return Status::OK(); } @@ -2533,17 +2489,6 @@ Status SpmdPartitioningVisitor::HandleReduce(HloInstruction* hlo) { if (reduce_sharded_dimension && input_count > 1) { return DefaultAction(hlo); } - - // Currently we only support reducing all or none of the sharded - // dimensions. - if (reduce_sharded_dimension) { - for (int64 i = 0; i < inputs[0].base_shape().rank(); ++i) { - if (inputs[0].sharding().tile_assignment().dim(i) > 1 && - absl::c_count(hlo->dimensions(), i) == 0) { - return DefaultAction(hlo); - } - } - } } std::vector new_operand_shapes(input_count * 2); @@ -2556,7 +2501,6 @@ Status SpmdPartitioningVisitor::HandleReduce(HloInstruction* hlo) { auto reduce_shape, ShapeInference::InferReduceShape(new_operand_shapes, hlo->dimensions(), hlo->to_apply()->ComputeProgramShape())); - *reduce_shape.mutable_layout() = hlo->shape().layout(); std::vector input_hlos(input_count); for (int64 i = 0; i < input_count; ++i) { @@ -2567,36 +2511,30 @@ Status SpmdPartitioningVisitor::HandleReduce(HloInstruction* hlo) { local_reduce->set_metadata(hlo->metadata()); SetPartitionedHlo(hlo, [&]() { - HloInstruction* reduce; + HloInstruction* reduce = local_reduce; if (reduce_sharded_dimension) { CHECK(local_reduce->shape().IsArray()); - reduce = collective_ops_creator_.create_cross_partition_all_reduce( - &b_, local_reduce, hlo->to_apply(), {}, NewChannel()); - reduce->set_sharding(HloSharding::Replicate()); - } else { - reduce = local_reduce; - if (inputs[0].sharding().IsTileMaximal()) { - reduce->set_sharding(inputs[0].sharding()); - } else { - // Remove tile assignment dimensions that are reduced. - std::vector tile_dimensions; - for (int64 i = 0; i < input_hlos[0]->shape().rank(); ++i) { - if (absl::c_count(hlo->dimensions(), i) == 0) { - tile_dimensions.push_back( - inputs[0].sharding().tile_assignment().dim(i)); - } + std::vector preserved_dims; + for (int64 i = 0; i < inputs[0].base_shape().rank(); ++i) { + if (!absl::c_linear_search(hlo->dimensions(), i)) { + preserved_dims.push_back(i); } - Array new_tile = inputs[0].sharding().tile_assignment(); - new_tile.Reshape(tile_dimensions); - auto sharding = HloSharding::Tile(new_tile); - if (input_count > 1) { - std::vector tuple(input_count, sharding); - sharding = HloSharding::Tuple(hlo->shape(), tuple); - } - reduce->set_sharding(sharding); } + if (inputs[0].sharding().ReplicateOnLastTileDim()) { + preserved_dims.push_back(inputs[0].base_shape().rank()); + } + auto grouped = GroupShardingOnDims(inputs[0].sharding(), preserved_dims); + auto grouped_state = CreatePerGroupPartitioningState( + inputs[0].state(), grouped.device_groups, &b_); + reduce = grouped_state.collective_ops_creator + .create_cross_partition_all_reduce( + &b_, local_reduce, hlo->to_apply(), {}, NewChannel()); } - + auto sharding = hlo_sharding_util::RemoveShapeDimensions( + hlo_sharding_util::PartiallyReplicateTiledShardingOnDims( + inputs[0].sharding(), hlo->dimensions()), + hlo->dimensions()); + reduce->set_sharding(sharding); return PartitionedHlo(reduce, hlo->shape(), MakePartitioningState()) .Reshard(hlo->sharding()) .hlo(); diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc index 04641f2f463..386d634779b 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc @@ -564,6 +564,27 @@ ENTRY entry { op::Constant()))))); } +TEST_F(SpmdPartitioningTest, + BroadcastBothOldAndNewDimsShardedPartiallySharded) { + const char* const hlo_string = R"( +HloModule module + +ENTRY entry { + param = f32[4,3] parameter(0), + sharding={devices=[1,2,4]0,1,4,5,2,3,6,7 last_tile_dim_replicate} + ROOT broadcast = f32[4,4,3] broadcast(param), dimensions={1,2}, + sharding={devices=[2,1,2,2]0,1,2,3,4,5,6,7 last_tile_dim_replicate} +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + PartitionComputation(hlo_string, /*num_devices=*/8)); + VLOG(1) << module->ToString(); + HloInstruction* root = module->entry_computation()->root_instruction(); + EXPECT_THAT( + root, + AllOf(op::Shape("f32[2,4,2]"), + op::Broadcast(AllOf(op::Shape("f32[4,2]"), op::Parameter(0))))); +} + TEST_F(SpmdPartitioningTest, ConvWithParallelDimAndNonParallelSpatialDimPartitioned) { const char* const hlo_string = R"( @@ -2746,6 +2767,35 @@ ENTRY entry { AllOf(op::Reduce(param0, op::Constant()), op::Shape("f32[64]"))); } +TEST_F(SpmdPartitioningTest, PartialTiledToPartialTiledReduce) { + const char* const hlo_string = R"( +HloModule module + +sum { + a = f32[] parameter(0) + b = f32[] parameter(1) + ROOT add = f32[] add(a, b) +} + +ENTRY entry { + %param0 = f32[4,4] parameter(0), + sharding={devices=[2,2,2]0,1,2,3,4,5,6,7 last_tile_dim_replicate} + %constant.1 = f32[] constant(0), sharding={replicated} + ROOT %reduce = f32[4] reduce(%param0, %constant.1), dimensions={0}, + to_apply=%sum, + sharding={devices=[2,4]0,1,4,5,2,3,6,7 last_tile_dim_replicate} +})"; + + TF_ASSERT_OK_AND_ASSIGN(auto module, + PartitionComputation(hlo_string, /*num_devices=*/8)); + VLOG(1) << module->ToString(); + + auto root = module->entry_computation()->root_instruction(); + EXPECT_THAT(root, + AllOf(op::AllReduce(op::Reduce(op::Parameter(0), op::Constant())), + op::Shape("f32[2]"))); +} + TEST_F(SpmdPartitioningTest, TiledToTiledTupleReduce) { const char* const hlo_string = R"( HloModule module diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc index 8f94a90de8e..767bed2a21a 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc @@ -942,7 +942,8 @@ GetReshardAllToAllSourceTargetDims(const HloSharding& source, const HloSharding& target) { if (source.IsTileMaximal() || target.IsTileMaximal() || source.tile_assignment().num_dimensions() != - target.tile_assignment().num_dimensions()) { + target.tile_assignment().num_dimensions() || + source.NumTiles() != target.NumTiles()) { return absl::nullopt; } // Record partition count to index for indices that have different partition @@ -1027,6 +1028,7 @@ bool CanReshardWithCollectivePermute(const HloSharding& source, return !source.IsTileMaximal() && !target.IsTileMaximal() && source.tile_assignment().dimensions() == target.tile_assignment().dimensions() && + source.ReplicateOnLastTileDim() == target.ReplicateOnLastTileDim() && source.tile_assignment() != target.tile_assignment(); } From fe968502a9835afec951a669d64224e411746605 Mon Sep 17 00:00:00 2001 From: Chao Mei Date: Sun, 9 Aug 2020 23:59:01 -0700 Subject: [PATCH 0730/1017] Stop support of --use_legacy_nnapi in the benchmark tool as Interpreter:UseNNAPI(bool) is marked deprecated now. Instead, use "--use_nnapi" and other NNAPI options. PiperOrigin-RevId: 325748539 Change-Id: Ie2c1c4ac3054239e9355b85092020ed36f06f0ff --- tensorflow/lite/interpreter.cc | 2 +- tensorflow/lite/tools/benchmark/README.md | 10 +--------- .../lite/tools/benchmark/benchmark_tflite_model.cc | 7 ------- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc index 307ede187b2..1d702dd8397 100644 --- a/tensorflow/lite/interpreter.cc +++ b/tensorflow/lite/interpreter.cc @@ -113,7 +113,7 @@ Interpreter::Interpreter(ErrorReporter* error_reporter) external_contexts_[kTfLiteCpuBackendContext] = own_external_cpu_backend_context_.get(); - UseNNAPI(false); + primary_subgraph().UseNNAPI(false); } Interpreter::~Interpreter() { diff --git a/tensorflow/lite/tools/benchmark/README.md b/tensorflow/lite/tools/benchmark/README.md index 8d7e6643d79..68cc59dd371 100644 --- a/tensorflow/lite/tools/benchmark/README.md +++ b/tensorflow/lite/tools/benchmark/README.md @@ -34,13 +34,6 @@ and the following optional parameters: * `run_delay`: `float` (default=-1.0) \ The delay in seconds between subsequent benchmark runs. Non-positive values mean use no delay. -* `use_legacy_nnapi`: `bool` (default=false) \ - Whether to use the legacy - [Android NNAPI](https://developer.android.com/ndk/guides/neuralnetworks/) - TFLite path, which requires the graph to be fully compatible with NNAPI. - This is available on recent Android devices. Note that some Android P - devices will fail to use NNAPI for models in `/data/local/tmp/` and this - benchmark tool will not correctly use NNAPI. * `enable_op_profiling`: `bool` (default=false) \ Whether to enable per-operator profiling measurement. * `enable_platform_tracing`: `bool` (default=false) \ @@ -65,8 +58,7 @@ The following simply lists the names of these parameters and additional notes where applicable. For details about each parameter, please refer to [this page](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/lite/tools/delegates/README.md#tflite-delegate-registrar). #### Common parameters -* `max_delegated_partitions`: `int` (default=0) \ -Note when `use_legacy_nnapi` is selected, this parameter won't work. +* `max_delegated_partitions`: `int` (default=0) * `min_nodes_per_partition`:`int` (default=0) #### GPU delegate diff --git a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc index 39ecded5484..ef9742eaac7 100644 --- a/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc +++ b/tensorflow/lite/tools/benchmark/benchmark_tflite_model.cc @@ -259,8 +259,6 @@ BenchmarkParams BenchmarkTfLiteModel::DefaultParams() { BenchmarkParam::Create("")); default_params.AddParam("input_layer_value_files", BenchmarkParam::Create("")); - default_params.AddParam("use_legacy_nnapi", - BenchmarkParam::Create(false)); default_params.AddParam("allow_fp16", BenchmarkParam::Create(false)); default_params.AddParam("require_full_delegation", BenchmarkParam::Create(false)); @@ -324,7 +322,6 @@ std::vector BenchmarkTfLiteModel::GetFlags() { "input_layer_value_range of the input_name will be ignored. The file " "format is binary and it should be array format or null separated " "strings format."), - CreateFlag("use_legacy_nnapi", ¶ms_, "use legacy nnapi api"), CreateFlag("allow_fp16", ¶ms_, "allow fp16"), CreateFlag("require_full_delegation", ¶ms_, "require delegate to run the entire graph"), @@ -363,9 +360,6 @@ void BenchmarkTfLiteModel::LogParams() { LOG_BENCHMARK_PARAM(std::string, "input_layer_value_files", "Input value files", verbose); -#if defined(__ANDROID__) - LOG_BENCHMARK_PARAM(bool, "use_legacy_nnapi", "Use legacy nnapi", verbose); -#endif LOG_BENCHMARK_PARAM(bool, "allow_fp16", "Allow fp16", verbose); LOG_BENCHMARK_PARAM(bool, "require_full_delegation", "Require full delegation", verbose); @@ -635,7 +629,6 @@ TfLiteStatus BenchmarkTfLiteModel::Init() { profiling_listener_ = MayCreateProfilingListener(); if (profiling_listener_) AddListener(profiling_listener_.get()); - interpreter_->UseNNAPI(params_.Get("use_legacy_nnapi")); interpreter_->SetAllowFp16PrecisionForFp32(params_.Get("allow_fp16")); owned_delegates_.clear(); From 502b61114452d6bd6ceaba5d3b3578dab88af71c Mon Sep 17 00:00:00 2001 From: Balint Cristian Date: Mon, 10 Aug 2020 10:30:22 +0300 Subject: [PATCH 0731/1017] [EXT-SYSLIB] Add runtime_py for external flatbuffer. --- third_party/flatbuffers/BUILD.system | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/third_party/flatbuffers/BUILD.system b/third_party/flatbuffers/BUILD.system index 14fceada826..8fe4d7a5907 100644 --- a/third_party/flatbuffers/BUILD.system +++ b/third_party/flatbuffers/BUILD.system @@ -36,3 +36,8 @@ cc_library( name = "runtime_cc", visibility = ["//visibility:public"], ) + +py_library( + name = "runtime_py", + visibility = ["//visibility:public"], +) From c25b9687614443c27a6425a25276133bbf4ac06d Mon Sep 17 00:00:00 2001 From: Ben Barsdell Date: Mon, 10 Aug 2020 17:34:11 +1000 Subject: [PATCH 0732/1017] Skip fp16 depthwise conv test when cudnn < v8 --- tensorflow/python/grappler/auto_mixed_precision_test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow/python/grappler/auto_mixed_precision_test.py b/tensorflow/python/grappler/auto_mixed_precision_test.py index f7f3777f7a9..615e01ed668 100644 --- a/tensorflow/python/grappler/auto_mixed_precision_test.py +++ b/tensorflow/python/grappler/auto_mixed_precision_test.py @@ -46,6 +46,7 @@ from tensorflow.python.ops import random_ops from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops import variables from tensorflow.python.ops.losses import losses +from tensorflow.python.platform import sysconfig from tensorflow.python.platform import test from tensorflow.python.training import adam from tensorflow.python.training import gradient_descent @@ -578,6 +579,12 @@ class AutoMixedPrecisionTest(test.TestCase, parameterized.TestCase): def test_depthwise_conv2d(self, mode): """Test grad ops with depthwise convolution2d graph.""" self._maybe_skip(mode) + cudnn_version = tuple([ + int(x) for x in sysconfig.get_build_info()['cudnn_version'].split('.')]) + if cudnn_version < (8,): + # Depthwise conv2d ops are only enabled in auto_mixed_precision as of + # cuDNN v8. + self.skipTest('cuDNN version >= 8 required') random_seed.set_random_seed(0) x = _input([2, 8, 8, 1]) f = _weight([3, 3, 1, 4]) From d97f769ce41968108dccbac00309809a08b61e29 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 10 Aug 2020 01:01:23 -0700 Subject: [PATCH 0733/1017] Introduce additional TPU infeed and outfeed ops PiperOrigin-RevId: 325755345 Change-Id: Ia6ebc88985f3a0859e044f1145cf8689cfcfd387 --- tensorflow/core/tpu/BUILD | 8 +- tensorflow/core/tpu/kernels/BUILD | 107 ---- .../core/tpu/kernels/image_resize_ops.cc | 155 ----- tensorflow/core/tpu/kernels/infeed_ops.cc | 529 ------------------ tensorflow/core/tpu/kernels/infeed_ops.h | 69 --- tensorflow/core/tpu/kernels/outfeed_ops.cc | 116 ---- tensorflow/core/tpu/kernels/outfeed_ops.h | 69 --- .../core/tpu/kernels/replication_ops.cc | 27 - .../core/tpu/kernels/tpu_handle_to_key_op.cc | 62 -- tensorflow/core/tpu/kernels/transfer_ops.cc | 98 ---- tensorflow/core/tpu/kernels/transfer_ops.h | 56 -- tensorflow/core/tpu/tpu_defs.cc | 18 - tensorflow/core/tpu/tpu_defs.h | 6 - tensorflow/core/tpu/tpu_library_init_fns.inc | 1 - 14 files changed, 1 insertion(+), 1320 deletions(-) delete mode 100644 tensorflow/core/tpu/kernels/image_resize_ops.cc delete mode 100644 tensorflow/core/tpu/kernels/infeed_ops.cc delete mode 100644 tensorflow/core/tpu/kernels/infeed_ops.h delete mode 100644 tensorflow/core/tpu/kernels/outfeed_ops.cc delete mode 100644 tensorflow/core/tpu/kernels/outfeed_ops.h delete mode 100644 tensorflow/core/tpu/kernels/replication_ops.cc delete mode 100644 tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc delete mode 100644 tensorflow/core/tpu/kernels/transfer_ops.cc delete mode 100644 tensorflow/core/tpu/kernels/transfer_ops.h diff --git a/tensorflow/core/tpu/BUILD b/tensorflow/core/tpu/BUILD index 15b2b93e46f..0a17ba3d408 100644 --- a/tensorflow/core/tpu/BUILD +++ b/tensorflow/core/tpu/BUILD @@ -88,13 +88,7 @@ cc_library( name = "tpu_defs", srcs = ["tpu_defs.cc"], hdrs = ["tpu_defs.h"], - deps = [ - ":tpu_api", - "//tensorflow/compiler/xla:shape_util", - "//tensorflow/core:protos_all_cc", - "//tensorflow/stream_executor/tpu:c_api_conversions", - "//tensorflow/stream_executor/tpu:c_api_decl", - ], + deps = ["//tensorflow/core:protos_all_cc"], ) cc_library( diff --git a/tensorflow/core/tpu/kernels/BUILD b/tensorflow/core/tpu/kernels/BUILD index 157aeb3df58..1336f52ed34 100644 --- a/tensorflow/core/tpu/kernels/BUILD +++ b/tensorflow/core/tpu/kernels/BUILD @@ -28,16 +28,10 @@ tf_kernel_library( deps = [ ":cross_replica_ops", ":host_compute_ops", - ":image_resize_ops", - ":infeed_ops", - ":outfeed_ops", - ":replication_ops", ":topk_ops", ":tpu_compile_op", ":tpu_configuration_ops", ":tpu_execute_op", - ":tpu_handle_to_key_op", - ":transfer_ops", ], ) @@ -690,104 +684,3 @@ cc_library( ], alwayslink = 1, ) - -cc_library( - name = "infeed_ops", - srcs = ["infeed_ops.cc"], - hdrs = ["infeed_ops.h"], - visibility = ["//visibility:public"], - deps = [ - ":transfer_ops", - "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", - "//tensorflow/compiler/tf2xla:common", - "//tensorflow/compiler/xla:util", - "//tensorflow/core:framework", - "//tensorflow/core/common_runtime:dma_helper", - "//tensorflow/core/framework:protos_all_cc", - "//tensorflow/core/kernels:transpose_functor", - "//tensorflow/core/platform:status", - "//tensorflow/core/profiler/lib:traceme", - "//tensorflow/core/tpu:tpu_defs", - "//tensorflow/stream_executor:multi_platform_manager", - "//tensorflow/stream_executor/tpu:tpu_transfer_manager_base", - "//tensorflow/stream_executor/tpu:tpu_transfer_manager_interface", - ], - alwayslink = True, -) - -cc_library( - name = "transfer_ops", - srcs = ["transfer_ops.cc"], - hdrs = ["transfer_ops.h"], - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", - "//tensorflow/core:framework", - "//tensorflow/core:lib_internal", - "//tensorflow/core/kernels:ops_util", - "//tensorflow/core/profiler/lib:traceme", - "//tensorflow/stream_executor:multi_platform_manager", - "//tensorflow/stream_executor/tpu:tpu_node_context", - "//tensorflow/stream_executor/tpu:tpu_platform_interface", - "//tensorflow/stream_executor/tpu:tpu_transfer_manager_interface", - ], - alwayslink = True, -) - -cc_library( - name = "outfeed_ops", - srcs = ["outfeed_ops.cc"], - hdrs = ["outfeed_ops.h"], - visibility = ["//visibility:public"], - deps = [ - ":transfer_ops", - "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", - "//tensorflow/compiler/tf2xla:common", - "//tensorflow/core:framework", - "//tensorflow/core/framework:protos_all_cc", - "//tensorflow/core/tpu:tpu_defs", - "//tensorflow/stream_executor:multi_platform_manager", - ], - alwayslink = True, -) - -cc_library( - name = "image_resize_ops", - srcs = ["image_resize_ops.cc"], - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/compiler/tf2xla:common", - "//tensorflow/compiler/tf2xla:xla_compiler", - "//tensorflow/compiler/xla/client:xla_builder", - "//tensorflow/compiler/xla/client/lib:constants", - "//tensorflow/core:framework", - "//tensorflow/core/tpu:tpu_defs", - "@com_google_absl//absl/strings", - ], - alwayslink = True, -) - -cc_library( - name = "replication_ops", - srcs = ["replication_ops.cc"], - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", - "//tensorflow/core:framework", - "//tensorflow/core/tpu:tpu_defs", - ], - alwayslink = True, -) - -cc_library( - name = "tpu_handle_to_key_op", - srcs = ["tpu_handle_to_key_op.cc"], - visibility = ["//visibility:public"], - deps = [ - ":tpu_compilation_cache_interface", - ":tpu_op_consts", - "//tensorflow/core:framework", - "//tensorflow/core/tpu:tpu_configuration", - ], - alwayslink = True, -) diff --git a/tensorflow/core/tpu/kernels/image_resize_ops.cc b/tensorflow/core/tpu/kernels/image_resize_ops.cc deleted file mode 100644 index fd0f5e4c7a6..00000000000 --- a/tensorflow/core/tpu/kernels/image_resize_ops.cc +++ /dev/null @@ -1,155 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "absl/strings/match.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/string_view.h" -#include "tensorflow/compiler/tf2xla/shape_util.h" -#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" -#include "tensorflow/compiler/tf2xla/xla_op_registry.h" -#include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/xla_builder.h" -#include "tensorflow/core/framework/kernel_def_builder.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/tpu/tpu_defs.h" - -namespace tensorflow { - -class TpuCustomResizeOp : public XlaOpKernel { - public: - explicit TpuCustomResizeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("align_corners", &align_corners_)); - OP_REQUIRES_OK(ctx, - ctx->GetAttr("half_pixel_centers", &half_pixel_centers_)); - } - - xla::Shape GetOutputShape(XlaOpKernelContext* ctx) const { - std::vector out_size; - auto status = ctx->ConstantInputAsIntVector(1, &out_size); - CHECK_EQ(out_size.size(), 2) << status.ToString(); - xla::Shape output_shape = - TensorShapeToXLAShape(ctx->output_xla_type(0), ctx->InputShape(0)); - output_shape.mutable_dimensions()[1] = out_size[0]; - output_shape.mutable_dimensions()[2] = out_size[1]; - return output_shape; - } - - string OpaqueField() const { - return absl::StrCat("\"", align_corners_, half_pixel_centers_, "\""); - } - - void CompileGrad(XlaOpKernelContext* ctx, const char* target, - const xla::Shape& output_shape) { - auto input_shape = - TensorShapeToXLAShape(ctx->output_xla_type(0), ctx->InputShape(0)); - if (ctx->InputShape(1).dim_sizes() == ctx->InputShape(0).dim_sizes()) { - ctx->SetOutput( - 0, xla::ConvertElementType(ctx->Input(0), ctx->output_xla_type(0))); - return; - } - // The gradient should be done in two phases for large resizes. - auto input = ctx->Input(0); - if (input_shape.dimensions(1) / output_shape.dimensions(1) > 3 && - input_shape.dimensions(2) / output_shape.dimensions(2) > 3) { - auto intermediate_shape = output_shape; - intermediate_shape.mutable_dimensions()[1] = input_shape.dimensions(1); - input = xla::CustomCall(ctx->builder(), target, {ctx->Input(0)}, - intermediate_shape, OpaqueField()); - } - ctx->SetOutput(0, xla::CustomCall(ctx->builder(), target, {input}, - output_shape, OpaqueField())); - } - - void CompileForward(XlaOpKernelContext* ctx, const char* target) { - auto output_shape = GetOutputShape(ctx); - if (ctx->InputShape(0).dim_size(1) == output_shape.dimensions(1) && - ctx->InputShape(0).dim_size(2) == output_shape.dimensions(2)) { - ctx->SetOutput( - 0, xla::ConvertElementType(ctx->Input(0), ctx->output_xla_type(0))); - return; - } - if (ctx->InputShape(0).dim_size(1) == 1 && - ctx->InputShape(0).dim_size(2) == 1) { - ctx->SetOutput(0, - ctx->Input(0) + xla::Zeros(ctx->builder(), output_shape)); - return; - } - ctx->SetOutput(0, xla::CustomCall(ctx->builder(), target, {ctx->Input(0)}, - output_shape, OpaqueField())); - } - - private: - bool align_corners_; - bool half_pixel_centers_; -}; - -class TpuResizeNearestNeighborOp : public TpuCustomResizeOp { - public: - explicit TpuResizeNearestNeighborOp(OpKernelConstruction* ctx) - : TpuCustomResizeOp(ctx) {} - void Compile(XlaOpKernelContext* ctx) override { - CompileForward(ctx, "ResizeNearest"); - } -}; - -class TpuResizeBilinearOp : public TpuCustomResizeOp { - public: - explicit TpuResizeBilinearOp(OpKernelConstruction* ctx) - : TpuCustomResizeOp(ctx) {} - void Compile(XlaOpKernelContext* ctx) override { - CompileForward(ctx, "ResizeBilinear"); - } -}; - -class TpuResizeNearestNeighborGradOp : public TpuCustomResizeOp { - public: - explicit TpuResizeNearestNeighborGradOp(OpKernelConstruction* ctx) - : TpuCustomResizeOp(ctx) {} - void Compile(XlaOpKernelContext* ctx) override { - CompileGrad(ctx, "ResizeNearestGrad", GetOutputShape(ctx)); - } -}; - -class TpuResizeBilinearGradOp : public TpuCustomResizeOp { - public: - explicit TpuResizeBilinearGradOp(OpKernelConstruction* ctx) - : TpuCustomResizeOp(ctx) {} - void Compile(XlaOpKernelContext* ctx) override { - auto output_shape = - TensorShapeToXLAShape(ctx->output_xla_type(0), ctx->InputShape(1)); - CompileGrad(ctx, "ResizeBilinearGrad", output_shape); - } -}; - -REGISTER_XLA_OP(Name("ResizeNearestNeighbor") - .CompileTimeConstantInput("size") - .Device(DEVICE_TPU_XLA_JIT), - TpuResizeNearestNeighborOp); - -REGISTER_XLA_OP(Name("ResizeNearestNeighborGrad") - .CompileTimeConstantInput("size") - .Device(DEVICE_TPU_XLA_JIT), - TpuResizeNearestNeighborGradOp); - -REGISTER_XLA_OP(Name("ResizeBilinear") - .CompileTimeConstantInput("size") - .Device(DEVICE_TPU_XLA_JIT), - TpuResizeBilinearOp); - -REGISTER_XLA_OP(Name("ResizeBilinearGrad").Device(DEVICE_TPU_XLA_JIT), - TpuResizeBilinearGradOp); - -} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/infeed_ops.cc b/tensorflow/core/tpu/kernels/infeed_ops.cc deleted file mode 100644 index f3fbd16b6cc..00000000000 --- a/tensorflow/core/tpu/kernels/infeed_ops.cc +++ /dev/null @@ -1,529 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/tpu/kernels/infeed_ops.h" - -#include -#include - -#include "tensorflow/compiler/jit/xla_device.h" -#include "tensorflow/compiler/tf2xla/literal_util.h" -#include "tensorflow/compiler/tf2xla/shape_util.h" -#include "tensorflow/compiler/xla/util.h" -#include "tensorflow/core/common_runtime/dma_helper.h" -#include "tensorflow/core/framework/allocator.h" -#include "tensorflow/core/framework/dataset.h" -#include "tensorflow/core/framework/function.h" -#include "tensorflow/core/framework/function_handle_cache.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/variant.h" -#include "tensorflow/core/framework/variant_encode_decode.h" -#include "tensorflow/core/framework/variant_tensor_data.h" -#include "tensorflow/core/kernels/transpose_functor.h" -#include "tensorflow/core/profiler/lib/traceme.h" -#include "tensorflow/core/tpu/kernels/transfer_ops.h" -#include "tensorflow/core/tpu/tpu_defs.h" -#include "tensorflow/stream_executor/multi_platform_manager.h" -#include "tensorflow/stream_executor/tpu/tpu_transfer_manager.h" -#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" - -namespace tensorflow { -namespace { - -typedef Eigen::ThreadPoolDevice CPUDevice; -typedef tensorflow::tpu::NoncopyableBuffer LinearizerBuffer; -typedef std::deque LinearizerBufferList; - -// Transposes the given tensor using the tensorflow C++ transpose implementation -// to obtain a XLA literal for the host tensor laid out as the given layout. The -// returned tensor is normalized to the dim0major layout -- F32[10,20,30]{2,0,1} -// is returned as F32[20,10,30]{2,1,0}. -xla::StatusOr TransposeTensor(OpKernelContext* ctx, - const Tensor& input_tensor, - const xla::Shape& xla_shape) { - profiler::TraceMe trace_me("TransposeTensor", /*level=*/2); - const int64 rank = xla_shape.rank(); - std::vector permutation(rank); - std::vector transposed_shapes(rank); - for (int64 i = 0; i < rank; ++i) { - permutation[i] = xla_shape.layout().minor_to_major(rank - 1 - i); - transposed_shapes[i] = xla_shape.dimensions(permutation[i]); - } - - Tensor transposed_tensor; - - // If this is a trivial transpose (i.e., bitcast), just create an aliased - // tensor with the transposed shape. - if (xla::LayoutUtil::IsMonotonicWithDim0Major( - xla::ShapeUtil::DropDegenerateDimensions(xla_shape).layout())) { - TensorShape shape; - TF_RETURN_IF_ERROR(TensorShapeUtils::MakeShape(transposed_shapes, &shape)); - TF_RETURN_IF_ERROR(transposed_tensor.BitcastFrom( - input_tensor, input_tensor.dtype(), shape)); - return transposed_tensor; - } - - AllocatorAttributes alloc_attr; - alloc_attr.set_on_host(true); - TF_RETURN_IF_ERROR(ctx->allocate_temp(input_tensor.dtype(), - TensorShape(transposed_shapes), - &transposed_tensor, alloc_attr)); - // Eigen Transpose fails with SIGFPE if there is a dimension of size 0. - if (input_tensor.NumElements() > 0) { - TF_RETURN_IF_ERROR(DoTranspose(ctx->eigen_device(), - input_tensor, permutation, - &transposed_tensor)); - } - return transposed_tensor; -} - -xla::StatusOr GetLayoutOverride(OpKernelConstruction* ctx, - const char* attrn_name, - std::vector* minor_to_major) { - if (!ctx->HasAttr(attrn_name)) { - return false; - } - TF_RETURN_IF_ERROR(ctx->GetAttr(attrn_name, minor_to_major)); - return !minor_to_major->empty(); -} - -Status GetInfeedShapeWithLayout(OpKernelConstruction* ctx, - const char* attrn_name, - const xla::Shape& input_shape, - xla::Shape* output_shape) { - std::vector minor_to_major; - TF_ASSIGN_OR_RETURN(bool has_override, - GetLayoutOverride(ctx, attrn_name, &minor_to_major)); - if (!has_override) { - *output_shape = input_shape; - if (output_shape->IsTuple()) { - int64 tuple_elements = xla::ShapeUtil::TupleElementCount(*output_shape); - for (int64 i = 0; i < tuple_elements; ++i) { - xla::Shape* sub_shape = - xla::ShapeUtil::GetMutableSubshape(output_shape, {i}); - *sub_shape->mutable_layout() = GetTPUInfeedLayout(*sub_shape).layout(); - } - } else { - *output_shape->mutable_layout() = - GetTPUInfeedLayout(*output_shape).layout(); - } - return Status::OK(); - } - - auto layout_func = [](const xla::Shape& shape) -> xla::Layout { - return GetTPUInfeedLayout(shape).layout(); - }; - return GetShapeWithLayout(input_shape, minor_to_major, layout_func, - output_shape); -} - -// LinearizedBuffersWrapper is an opaque C++ data structure for the outputs of -// PrelinearizeOp and PrelinearizeTupleOp. It holds the resultant linearized -// buffers and references to input tensors whose underlying storage are shared -// with linearized buffers. -// NOTE: This is not a feature-complete implementation of the DT_VARIANT -// specification. In particular, we cannot currently serialize an arbitrary -// `LinearizerBufferList` (aka `std::deque`) -// object, so the `Encode()` and `Decode()` methods are not implemented. -struct LinearizedBuffersWrapper { - explicit LinearizedBuffersWrapper() {} - explicit LinearizedBuffersWrapper(LinearizerBufferList bufs, - std::vector ts) - : buffers(std::move(bufs)), tensors(std::move(ts)) {} - LinearizedBuffersWrapper(const LinearizedBuffersWrapper& wrapper) { - // tensorflow::Variant requires this copy constructor to compile. - LOG(FATAL) << "LinearizedBuffersWrapper should not copy."; - } - LinearizedBuffersWrapper& operator=(const LinearizedBuffersWrapper& wrapper) = - delete; - LinearizedBuffersWrapper(LinearizedBuffersWrapper&&) = default; - LinearizedBuffersWrapper& operator=(LinearizedBuffersWrapper&&) = default; - ~LinearizedBuffersWrapper() = default; - - // These functions are tensorflow::Variant requirements. - string TypeName() const { return "(anonymous)::LinearizedBuffersWrapper"; } - void Encode(tensorflow::VariantTensorData* data) const { - LOG(ERROR) << "Encode() is not implemented for LinearizedBuffersWrapper " - "objects."; - } - bool Decode(const tensorflow::VariantTensorData& data) { - LOG(ERROR) << "Decode() is not implemented for LinearizedBuffersWrapper " - "objects."; - return false; - } - - LinearizerBufferList buffers; - // Save references on tensors whose underlying storage are shared with - // LiteralLinearizer::Buffer in `buffers`. - std::vector tensors; -}; - -Status AutoTransposeAndLinearize(OpKernelContext* ctx, - const Tensor& input_tensor, - const xla::Shape& shape, - LinearizerBufferList* linearized_buffers, - std::vector* saved_input_tensors) { - const Tensor* tensor = &input_tensor; - // If the given layout is not in dim0major layout, tranposes the tensor. - bool has_transposed = false; - Tensor transposed_tensor; - if (!xla::LayoutUtil::IsMonotonicWithDim0Major(shape.layout())) { - // If the given layout is not in dim0major layout, transpose the tensor. - TF_ASSIGN_OR_RETURN(transposed_tensor, - TransposeTensor(ctx, input_tensor, shape)); - tensor = &transposed_tensor; - has_transposed = true; - } - - xla::BorrowingLiteral literal; - TF_RETURN_IF_ERROR(HostTensorToBorrowingLiteral(*tensor, &literal)); - - TF_RETURN_IF_ERROR( - xla::TpuTransferManagerInterface::GetRegisteredTpuTransferManager() - ->LinearizeToBuffers(literal, linearized_buffers)); - - // The input tensor is ref-counted. Save a handle on the input tensor if - // its underlying storage is shared with linearized buffers to prevent - // input tensor from getting freed. - for (const auto& buffer : *linearized_buffers) { - if (!buffer.owns_data() && !has_transposed) { - // `buffer` is created from zero-copy fast path from the un-transposed - // input tensor so its underlying data is shared with input tensor. - // Save a handle to input tensor to increment its ref-count and avoid - // it getting deallocated after PrelinearizeTupleOp completes. - saved_input_tensors->push_back(*tensor); - // A literal can be linearized to zero to two buffers. If any of the - // linearized buffer shares storage with input tensor. We save exactly - // one handle on the input tensor. - break; - } - } - return Status::OK(); -} - -// PrelinearizeOp is used to linearize one tensor to the device format. -class PrelinearizeOp : public OpKernel { - public: - explicit PrelinearizeOp(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); - xla::Shape shape; - OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &shape)); - OP_REQUIRES_OK(ctx, - GetInfeedShapeWithLayout(ctx, "layout", shape, &xla_shape_)); - } - - void Compute(OpKernelContext* ctx) override { - const Tensor& input_tensor = ctx->input(0); - // Validate input. - OP_REQUIRES( - ctx, input_tensor.dtype() == dtype_, - errors::InvalidArgument("Prelinearize dtype mismatch; expected ", - DataType_Name(dtype_), ", got ", - DataType_Name(input_tensor.dtype()))); - OP_REQUIRES( - ctx, input_tensor.shape() == shape_, - errors::InvalidArgument("Prelinearize shape mismatch; expected ", - shape_.DebugString(), ", got ", - input_tensor.shape().DebugString())); - - // Auto-transpose and prelinearize. - LinearizerBufferList linearized_buffers; - std::vector saved_input_tensors; - auto status = - AutoTransposeAndLinearize(ctx, input_tensor, xla_shape_, - &linearized_buffers, &saved_input_tensors); - OP_REQUIRES_OK(ctx, status); - - // Write to output. - tensorflow::Tensor* output; - OP_REQUIRES_OK(ctx, - ctx->allocate_output(0, tensorflow::TensorShape{}, &output)); - output->scalar()() = LinearizedBuffersWrapper{ - std::move(linearized_buffers), std::move(saved_input_tensors)}; - } - - bool IsExpensive() override { return true; } - - private: - TensorShape shape_; - DataType dtype_; - xla::Shape xla_shape_; - - // PrelinearizeOp is neither copyable nor movable. - PrelinearizeOp(const PrelinearizeOp&) = delete; - PrelinearizeOp& operator=(const PrelinearizeOp&) = delete; -}; - -// PrelinearizeTupleOp is used to linearize multiple tensors to the device -// format. -class PrelinearizeTupleOp : public OpKernel { - public: - explicit PrelinearizeTupleOp(OpKernelConstruction* ctx) : OpKernel(ctx) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); - OP_REQUIRES( - ctx, shapes_.size() == dtypes_.size(), - errors::InvalidArgument( - "shapes and dtypes must be the same length. shapes length = ", - shapes_.size(), ", dtypes length = ", dtypes_.size())); - - std::vector xla_shapes; - for (int i = 0; i < shapes_.size(); i++) { - xla::Shape xla_shape; - OP_REQUIRES_OK(ctx, - TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); - xla_shapes.push_back(xla_shape); - } - OP_REQUIRES_OK( - ctx, GetInfeedShapeWithLayout( - ctx, "layouts", xla::ShapeUtil::MakeTupleShape(xla_shapes), - &tuple_shape_)); - } - - void Compute(OpKernelContext* ctx) override { - OpInputList values; - OP_REQUIRES_OK(ctx, ctx->input_list("inputs", &values)); - OP_REQUIRES(ctx, values.size() == shapes_.size(), - errors::InvalidArgument( - "Wrong number of inputs to PrelinearizeTuple.")); - - LinearizerBufferList all_linearized_buffers; - std::vector all_saved_input_tensors; - for (int i = 0; i < values.size(); i++) { - // Validate input. - const Tensor& input_tensor = values[i]; - OP_REQUIRES(ctx, input_tensor.dtype() == dtypes_[i], - errors::InvalidArgument( - "PrelinearizeTuple dtype mismatch at tuple element ", i, - "; expected ", DataType_Name(dtypes_[i]), ", got ", - DataType_Name(input_tensor.dtype()))); - OP_REQUIRES(ctx, input_tensor.shape() == shapes_[i], - errors::InvalidArgument( - "PrelinearizeTuple shape mismatch at tuple element ", i, - "; expected ", shapes_[i].DebugString(), ", got ", - input_tensor.shape().DebugString())); - - // Auto-transpose and prelinearize. - LinearizerBufferList linearized_buffers; - std::vector saved_input_tensors; - auto status = AutoTransposeAndLinearize( - ctx, input_tensor, tuple_shape_.tuple_shapes(i), &linearized_buffers, - &saved_input_tensors); - OP_REQUIRES_OK(ctx, status); - all_linearized_buffers.insert( - all_linearized_buffers.end(), - std::make_move_iterator(linearized_buffers.begin()), - std::make_move_iterator(linearized_buffers.end())); - all_saved_input_tensors.insert( - all_saved_input_tensors.end(), - std::make_move_iterator(saved_input_tensors.begin()), - std::make_move_iterator(saved_input_tensors.end())); - } - - tensorflow::Tensor* output; - OP_REQUIRES_OK(ctx, - ctx->allocate_output(0, tensorflow::TensorShape{}, &output)); - output->scalar()() = LinearizedBuffersWrapper{ - std::move(all_linearized_buffers), std::move(all_saved_input_tensors)}; - } - - bool IsExpensive() override { return true; } - - private: - std::vector shapes_; - DataTypeVector dtypes_; - xla::Shape tuple_shape_; - - // PrelinearizeTupleOp is neither copyable nor movable. - PrelinearizeTupleOp(const PrelinearizeTupleOp&) = delete; - PrelinearizeTupleOp& operator=(const PrelinearizeTupleOp&) = delete; -}; - -// The InfeedEnqueuePrelinearizedBufferOp op is used to transfer prelinearized -// buffers to the device infeed queue. -class InfeedEnqueuePrelinearizedBufferOp : public TpuTransferAsyncOpKernel { - public: - explicit InfeedEnqueuePrelinearizedBufferOp(OpKernelConstruction* ctx) - : TpuTransferAsyncOpKernel(ctx, "prelinearized_buffers_to_infeed", 8) {} - - Status DoWork(OpKernelContext* ctx, - xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) override { - const Tensor& input_tensor = ctx->input(0); - const LinearizedBuffersWrapper* wrapper = - input_tensor.scalar()() - .get(); - TF_RETURN_IF_ERROR(transfer_manager->TransferBuffersToInfeed( - stream_executor, wrapper->buffers)); - - return Status::OK(); - } - - private: - // InfeedEnqueuePrelinearizedBufferOp is neither copyable nor movable. - InfeedEnqueuePrelinearizedBufferOp( - const InfeedEnqueuePrelinearizedBufferOp&) = delete; - InfeedEnqueuePrelinearizedBufferOp& operator=( - const InfeedEnqueuePrelinearizedBufferOp&) = delete; -}; - -} // anonymous namespace - -TpuInfeedEnqueueOp::TpuInfeedEnqueueOp(OpKernelConstruction* ctx) - : TpuTransferAsyncOpKernel(ctx, "infeed_enqueue", 8) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); - xla::Shape shape; - OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &shape)); - OP_REQUIRES_OK(ctx, - GetInfeedShapeWithLayout(ctx, "layout", shape, &xla_shape_)); -} - -Status TpuInfeedEnqueueOp::DoWork( - OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) { - const Tensor& input_tensor = ctx->input(0); - - // Validate runtime shape and fail if it doesn't match the contract. - if (input_tensor.dtype() != dtype_) { - return errors::InvalidArgument("Infeed dtype mismatch."); - } - if (input_tensor.shape() != shape_) { - return errors::InvalidArgument("Infeed shape mismatch; expected ", - shape_.DebugString(), ", got ", - input_tensor.shape().DebugString()); - } - - const Tensor* tensor = &input_tensor; - Tensor transposed_tensor; - if (!xla::LayoutUtil::IsMonotonicWithDim0Major(xla_shape_.layout())) { - // If the given layout is not in dim0major layout, transpose the tensor. - TF_ASSIGN_OR_RETURN(transposed_tensor, - TransposeTensor(ctx, input_tensor, xla_shape_)); - tensor = &transposed_tensor; - } - - xla::BorrowingLiteral literal; - TF_RETURN_IF_ERROR(HostTensorToBorrowingLiteral(*tensor, &literal)); - - // Transfer the given literal to the Infeed interface of the device. - TF_RETURN_IF_ERROR( - transfer_manager->TransferLiteralToInfeed(stream_executor, literal)); - return Status::OK(); -} - -TpuInfeedEnqueueTupleOp::TpuInfeedEnqueueTupleOp(OpKernelConstruction* ctx) - : TpuTransferAsyncOpKernel(ctx, "infeed_enqueue", 8) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); - OP_REQUIRES( - ctx, shapes_.size() == dtypes_.size(), - errors::InvalidArgument("shapes and dtypes must be the same length.")); - - std::vector xla_shapes; - for (int i = 0; i < shapes_.size(); i++) { - xla::Shape xla_shape; - OP_REQUIRES_OK(ctx, - TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); - xla_shapes.push_back(xla_shape); - } - OP_REQUIRES_OK( - ctx, GetInfeedShapeWithLayout(ctx, "layouts", - xla::ShapeUtil::MakeTupleShape(xla_shapes), - &tuple_shape_)); -} - -Status TpuInfeedEnqueueTupleOp::DoWork( - OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) { - OpInputList values; - TF_RETURN_IF_ERROR(ctx->input_list("inputs", &values)); - if (values.size() != shapes_.size()) { - return errors::InvalidArgument( - "Wrong number of inputs to InfeedEnqueueTuple."); - } - - for (const auto& shapes : shapes_) { - VLOG(1) << "TransferLiteralToInfeed " << shapes.DebugString(); - } - - std::vector maybe_transposed_tensors; - maybe_transposed_tensors.reserve(values.size()); - for (int i = 0; i < values.size(); i++) { - // Validate runtime shapes and fail if it doesn't match the contract. - const Tensor* tensor = &values[i]; - if (tensor->shape() != shapes_[i]) { - return errors::InvalidArgument("Infeed shape mismatch for tuple element ", - i, "; expected ", shapes_[i].DebugString(), - ", got ", tensor->shape().DebugString()); - } - if (!xla::LayoutUtil::IsMonotonicWithDim0Major( - tuple_shape_.tuple_shapes(i).layout())) { - // If the given layout is not in dim0major layout, tranposes the given - // tensor. - TF_ASSIGN_OR_RETURN( - Tensor transposed_tensor, - TransposeTensor(ctx, *tensor, tuple_shape_.tuple_shapes(i))); - maybe_transposed_tensors.emplace_back(transposed_tensor); - } else { - maybe_transposed_tensors.emplace_back(*tensor); - } - } - - xla::BorrowingLiteral tuple; - TF_RETURN_IF_ERROR( - HostTensorsToBorrowingLiteralTuple(maybe_transposed_tensors, &tuple)); - - // Transfer the given literal to the Infeed interface of the device. - TF_RETURN_IF_ERROR( - transfer_manager->TransferLiteralToInfeed(stream_executor, tuple)); - - VLOG(1) << "TransferLiteralToInfeed complete."; - - return Status::OK(); -} - -// These ops execute on either the TPU device or the CPU device. When running on -// CPU they must specify a non-negative value for device_ordinal to indicate -// which TPU to send infeed to. -REGISTER_KERNEL_BUILDER( - Name("InfeedEnqueue").Device(DEVICE_TPU_NODE).HostMemory("input"), - TpuInfeedEnqueueOp); -REGISTER_KERNEL_BUILDER(Name("InfeedEnqueue").Device(DEVICE_CPU), - TpuInfeedEnqueueOp); - -REGISTER_KERNEL_BUILDER( - Name("InfeedEnqueueTuple").Device(DEVICE_TPU_NODE).HostMemory("inputs"), - TpuInfeedEnqueueTupleOp); -REGISTER_KERNEL_BUILDER(Name("InfeedEnqueueTuple").Device(DEVICE_CPU), - TpuInfeedEnqueueTupleOp); - -// Prelinearize ops run on CPU as part of tf.data input pipeline. -REGISTER_KERNEL_BUILDER(Name("Prelinearize").Device(DEVICE_CPU), - PrelinearizeOp); -REGISTER_KERNEL_BUILDER(Name("PrelinearizeTuple").Device(DEVICE_CPU), - PrelinearizeTupleOp); - -// InfeedEnqueuePrelinearizedBuffer op run on CPU and takes a device_ordinal to -// select the right device to infeed. -REGISTER_KERNEL_BUILDER( - Name("InfeedEnqueuePrelinearizedBuffer").Device(DEVICE_CPU), - InfeedEnqueuePrelinearizedBufferOp); - -} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/infeed_ops.h b/tensorflow/core/tpu/kernels/infeed_ops.h deleted file mode 100644 index 622583b6a73..00000000000 --- a/tensorflow/core/tpu/kernels/infeed_ops.h +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CORE_TPU_KERNELS_INFEED_OPS_H_ -#define TENSORFLOW_CORE_TPU_KERNELS_INFEED_OPS_H_ - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/platform/status.h" -#include "tensorflow/core/tpu/kernels/transfer_ops.h" - -namespace tensorflow { - -// TODO(b/65200690): Rework this when there is a callback based infeed API to -// StreamExecutor. - -// The InfeedEnqueue op is used to deliver data to the device infeed queue. -class TpuInfeedEnqueueOp : public TpuTransferAsyncOpKernel { - public: - explicit TpuInfeedEnqueueOp(OpKernelConstruction* ctx); - Status DoWork(OpKernelContext* ctx, - xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) override; - - private: - TensorShape shape_; - DataType dtype_; - xla::Shape xla_shape_; - - // TpuInfeedEnqueueOp is neither copyable nor movable. - TpuInfeedEnqueueOp(const TpuInfeedEnqueueOp&) = delete; - TpuInfeedEnqueueOp& operator=(const TpuInfeedEnqueueOp&) = delete; -}; - -// The InfeedEnqueueTuple op is used on the host to deliver multiple tensors to -// the device infeed queue as an XLA tuple. -class TpuInfeedEnqueueTupleOp : public TpuTransferAsyncOpKernel { - public: - explicit TpuInfeedEnqueueTupleOp(OpKernelConstruction* ctx); - Status DoWork(OpKernelContext* ctx, - xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) override; - - private: - std::vector shapes_; - DataTypeVector dtypes_; - xla::Shape tuple_shape_; - - // TpuInfeedEnqueueTupleOp is neither copyable nor movable. - TpuInfeedEnqueueTupleOp(const TpuInfeedEnqueueTupleOp&) = delete; - TpuInfeedEnqueueTupleOp& operator=(const TpuInfeedEnqueueTupleOp&) = delete; -}; - -} // namespace tensorflow - -#endif // TENSORFLOW_CORE_TPU_KERNELS_INFEED_OPS_H_ diff --git a/tensorflow/core/tpu/kernels/outfeed_ops.cc b/tensorflow/core/tpu/kernels/outfeed_ops.cc deleted file mode 100644 index 51a3a71a297..00000000000 --- a/tensorflow/core/tpu/kernels/outfeed_ops.cc +++ /dev/null @@ -1,116 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/tpu/kernels/outfeed_ops.h" - -#include "tensorflow/compiler/jit/xla_device.h" -#include "tensorflow/compiler/tf2xla/literal_util.h" -#include "tensorflow/compiler/tf2xla/shape_util.h" -#include "tensorflow/compiler/tf2xla/type_util.h" -#include "tensorflow/core/framework/allocator.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/tpu/kernels/transfer_ops.h" -#include "tensorflow/core/tpu/tpu_defs.h" -#include "tensorflow/stream_executor/multi_platform_manager.h" - -namespace tensorflow { - -TpuOutfeedDequeueOp::TpuOutfeedDequeueOp(OpKernelConstruction* ctx) - : TpuTransferAsyncOpKernel(ctx, "outfeed_dequeue", 1) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); - OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &xla_shape_)); -} - -Status TpuOutfeedDequeueOp::DoWork( - OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) { - Tensor* output; - TF_RETURN_IF_ERROR(ctx->allocate_output(0, shape_, &output)); - - // Transfer from the outfeed interface of the device. - xla::MutableBorrowingLiteral literal; - TF_RETURN_IF_ERROR( - HostTensorToMutableBorrowingLiteral(xla_shape_, output, &literal)); - - VLOG(1) << "TransferLiteralFromOutfeed " - << xla::ShapeUtil::HumanStringWithLayout(xla_shape_); - - TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralFromOutfeed( - stream_executor, xla_shape_, literal)); - - VLOG(1) << "TransferLiteralFromOutfeed complete."; - - return Status::OK(); -} - -// The OutfeedDequeueTuple op is used to retrieve multiple tensors from the -// device outfeed queue. -TpuOutfeedDequeueTupleOp::TpuOutfeedDequeueTupleOp(OpKernelConstruction* ctx) - : TpuTransferAsyncOpKernel(ctx, "outfeed_dequeue", 1) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); - OP_REQUIRES( - ctx, shapes_.size() == dtypes_.size(), - errors::InvalidArgument("shapes and dtypes must be the same length.")); - // The `dtypes` list is inferred from the supplied inputs, so it - // is always the correct length. - for (int i = 0; i < shapes_.size(); i++) { - xla::Shape xla_shape; - OP_REQUIRES_OK(ctx, - TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); - xla_shapes_.push_back(xla_shape); - } - tuple_shape_ = xla::ShapeUtil::MakeTupleShape(xla_shapes_); -} - -Status TpuOutfeedDequeueTupleOp::DoWork( - OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) { - VLOG(1) << "TransferLiteralFromOutfeed " - << xla::ShapeUtil::HumanStringWithLayout(tuple_shape_); - - for (int i = 0; i < shapes_.size(); ++i) { - Tensor* output; - TF_RETURN_IF_ERROR(ctx->allocate_output(i, shapes_[i], &output)); - - xla::MutableBorrowingLiteral literal; - TF_RETURN_IF_ERROR( - HostTensorToMutableBorrowingLiteral(xla_shapes_[i], output, &literal)); - TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralFromOutfeed( - stream_executor, xla_shapes_[i], literal)); - } - return Status::OK(); -} - -// These ops execute on either the TPU device or the CPU device. When -// running on CPU they must specify a non-negative value for -// device_ordinal to indicate which TPU to receive outfeed from. -REGISTER_KERNEL_BUILDER( - Name("OutfeedDequeue").Device(DEVICE_TPU_NODE).HostMemory("output"), - TpuOutfeedDequeueOp); -REGISTER_KERNEL_BUILDER(Name("OutfeedDequeue").Device(DEVICE_CPU), - TpuOutfeedDequeueOp); - -REGISTER_KERNEL_BUILDER( - Name("OutfeedDequeueTuple").Device(DEVICE_TPU_NODE).HostMemory("outputs"), - TpuOutfeedDequeueTupleOp); -REGISTER_KERNEL_BUILDER(Name("OutfeedDequeueTuple").Device(DEVICE_CPU), - TpuOutfeedDequeueTupleOp); - -} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/outfeed_ops.h b/tensorflow/core/tpu/kernels/outfeed_ops.h deleted file mode 100644 index 5e3ed87c04b..00000000000 --- a/tensorflow/core/tpu/kernels/outfeed_ops.h +++ /dev/null @@ -1,69 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CORE_TPU_KERNELS_OUTFEED_OPS_H_ -#define TENSORFLOW_CORE_TPU_KERNELS_OUTFEED_OPS_H_ - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/tpu/kernels/transfer_ops.h" - -namespace tensorflow { - -// The OutfeedDequeue op is used to retrieve a single tensor from the device -// outfeed queue. -class TpuOutfeedDequeueOp : public TpuTransferAsyncOpKernel { - public: - explicit TpuOutfeedDequeueOp(OpKernelConstruction* ctx); - - Status DoWork(OpKernelContext* ctx, - xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) override; - - private: - TensorShape shape_; - DataType dtype_; - xla::Shape xla_shape_; - - // OutfeedDequeueOp is neither copyable nor movable. - TpuOutfeedDequeueOp(const TpuOutfeedDequeueOp&) = delete; - TpuOutfeedDequeueOp& operator=(const TpuOutfeedDequeueOp&) = delete; -}; - -// The OutfeedDequeueTuple op is used to retrieve multiple tensors from the -// device outfeed queue. -class TpuOutfeedDequeueTupleOp : public TpuTransferAsyncOpKernel { - public: - explicit TpuOutfeedDequeueTupleOp(OpKernelConstruction* ctx); - - Status DoWork(OpKernelContext* ctx, - xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) override; - - private: - std::vector shapes_; - DataTypeVector dtypes_; - std::vector xla_shapes_; - xla::Shape tuple_shape_; - - // OutfeedDequeueTupleOp is neither copyable nor movable. - TpuOutfeedDequeueTupleOp(const TpuOutfeedDequeueTupleOp&) = delete; - TpuOutfeedDequeueTupleOp& operator=(const TpuOutfeedDequeueTupleOp&) = delete; -}; - -} // namespace tensorflow - -#endif // TENSORFLOW_CORE_TPU_KERNELS_OUTFEED_OPS_H_ diff --git a/tensorflow/core/tpu/kernels/replication_ops.cc b/tensorflow/core/tpu/kernels/replication_ops.cc deleted file mode 100644 index 4c986e880e7..00000000000 --- a/tensorflow/core/tpu/kernels/replication_ops.cc +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/jit/xla_device_ops.h" -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/shape_inference.h" -#include "tensorflow/core/tpu/tpu_defs.h" - -namespace tensorflow { - -REGISTER_KERNEL_BUILDER(Name("_TPUReplicate").Device(DEVICE_TPU_SYSTEM), - XlaDeviceDummyOp); - -} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc b/tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc deleted file mode 100644 index ec2ae91d3eb..00000000000 --- a/tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc +++ /dev/null @@ -1,62 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include -#include - -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h" -#include "tensorflow/core/tpu/kernels/tpu_op_consts.h" -#include "tensorflow/core/tpu/tpu_configuration.h" - -namespace tensorflow { - -class TpuHandleToProtoKeyOp : public OpKernel { - public: - explicit TpuHandleToProtoKeyOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} - ~TpuHandleToProtoKeyOp() override = default; - TpuHandleToProtoKeyOp(const TpuHandleToProtoKeyOp&) = delete; - TpuHandleToProtoKeyOp& operator=(const TpuHandleToProtoKeyOp&) = delete; - - void Compute(OpKernelContext* ctx) override { - VLOG(1) << "TpuHandleToProtoKeyOp::Compute " << ctx->op_kernel().name() - << " on device " << ctx->op_kernel().requested_device(); - const Tensor& uid = ctx->input(0); - - ResourceMgr* rm = GetTPUConfigResourceMgr(); - tpu::TpuCompilationCacheInterface* cache; - OP_REQUIRES_OK(ctx, rm->Lookup( - rm->default_container(), - tpu::kCompilationCacheResourceName, &cache)); - core::ScopedUnref cache_unref(cache); - - std::vector keys; - OP_REQUIRES_OK(ctx, cache->GetKeysFromUid(uid.scalar()(), &keys)); - - TensorShape output_shape; - output_shape.AddDim(keys.size()); - Tensor* result = nullptr; - OP_REQUIRES_OK(ctx, ctx->allocate_output(0, output_shape, &result)); - for (int i = 0; i < keys.size(); ++i) { - result->vec()(i) = keys[i]; - } - }; -}; - -REGISTER_KERNEL_BUILDER(Name("TpuHandleToProtoKey").Device(DEVICE_CPU), - TpuHandleToProtoKeyOp); - -} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/transfer_ops.cc b/tensorflow/core/tpu/kernels/transfer_ops.cc deleted file mode 100644 index 40b85e2cfbd..00000000000 --- a/tensorflow/core/tpu/kernels/transfer_ops.cc +++ /dev/null @@ -1,98 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/core/tpu/kernels/transfer_ops.h" - -#include "tensorflow/core/framework/op.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/kernels/ops_util.h" -#include "tensorflow/core/platform/tracing.h" -#include "tensorflow/core/profiler/lib/traceme.h" -#include "tensorflow/stream_executor/multi_platform_manager.h" -#include "tensorflow/stream_executor/tpu/tpu_node_context.h" -#include "tensorflow/stream_executor/tpu/tpu_platform_interface.h" -#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" - -namespace tensorflow { - -TpuTransferAsyncOpKernel::TpuTransferAsyncOpKernel(OpKernelConstruction* ctx, - const string& transfer_type, - int number_of_threads) - : AsyncOpKernel(ctx), - thread_pool_(new thread::ThreadPool( - ctx->env(), - strings::StrCat(transfer_type, "_thread_", - SanitizeThreadSuffix(def().name())), - /*num_threads=*/8)) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("device_ordinal", &device_ordinal_)); - if (ctx->device_type() == DeviceType(DEVICE_CPU)) { - OP_REQUIRES( - ctx, device_ordinal_ >= 0, - errors::InvalidArgument(transfer_type, - " ops must specify a device_ordinal when " - "placed on CPU.")); - } -} - -void TpuTransferAsyncOpKernel::ComputeAsync(OpKernelContext* ctx, - DoneCallback done) { - CancellationToken token = - ctx->cancellation_manager()->get_cancellation_token(); - bool already_cancelled; - { - // Only protect registering the cancellation callback as mu_ cannot be held - // at a point where `done` could be called. - mutex_lock lock(mu_); - already_cancelled = !ctx->cancellation_manager()->RegisterCallback( - token, [this]() { Cancel(); }); - } - OP_REQUIRES_ASYNC(ctx, !already_cancelled, - errors::Cancelled("Infeed was cancelled."), done); - thread_pool_->Schedule([this, ctx, done, token]() { - Status s = RunTransfer(ctx); - ctx->cancellation_manager()->DeregisterCallback(token); - OP_REQUIRES_OK_ASYNC(ctx, s, done); - done(); - }); -} - -Status TpuTransferAsyncOpKernel::RunTransfer(OpKernelContext* ctx) { - auto* tpu_platform = tpu::TpuPlatformInterface::GetRegisteredPlatform(); - - int real_device_ordinal = device_ordinal_; - if (real_device_ordinal < 0) { - const XlaDevice::Metadata* metadata; - TF_RETURN_IF_ERROR(XlaDevice::GetMetadata(ctx, &metadata)); - real_device_ordinal = metadata->device_ordinal(); - } - stream_executor::StreamExecutor* stream_executor = - tpu_platform->ExecutorForDevice(real_device_ordinal).ValueOrDie(); - - // When Xprof profiling is off (which is the default), constructing the - // activity is simple enough that its overhead is negligible. - profiler::TraceMe activity( - [this] { return profiler::TraceMeOp(name(), type_string()); }, - profiler::TraceMeLevel::kInfo); - return DoWork( - ctx, xla::TpuTransferManagerInterface::GetRegisteredTpuTransferManager(), - stream_executor); -} - -void TpuTransferAsyncOpKernel::Cancel() { - mutex_lock lock(mu_); - TF_CHECK_OK(tpu::TpuNodeContext::CloseTpuHost()); -} - -} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/transfer_ops.h b/tensorflow/core/tpu/kernels/transfer_ops.h deleted file mode 100644 index d98d743f569..00000000000 --- a/tensorflow/core/tpu/kernels/transfer_ops.h +++ /dev/null @@ -1,56 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_CORE_TPU_KERNELS_TRANSFER_OPS_H_ -#define TENSORFLOW_CORE_TPU_KERNELS_TRANSFER_OPS_H_ - -#include "tensorflow/compiler/jit/xla_device.h" -#include "tensorflow/core/framework/op_kernel.h" -#include "tensorflow/core/util/stream_executor_util.h" -#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" - -namespace tensorflow { - -// Base class providing common functionality for async ops that transfer from -// host to TPU. -class TpuTransferAsyncOpKernel : public AsyncOpKernel { - public: - explicit TpuTransferAsyncOpKernel(OpKernelConstruction* ctx, - const string& transfer_type, - int number_of_threads); - - void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override; - - protected: - virtual Status DoWork(OpKernelContext* context, - xla::TpuTransferManagerInterface* transfer_manager, - stream_executor::StreamExecutor* stream_executor) = 0; - - private: - Status RunTransfer(OpKernelContext* ctx); - void Cancel(); - - std::unique_ptr thread_pool_; - int device_ordinal_; - mutex mu_; - - // TpuTransferAsyncOpKernel is neither copyable nor movable. - TpuTransferAsyncOpKernel(const TpuTransferAsyncOpKernel&) = delete; - TpuTransferAsyncOpKernel& operator=(const TpuTransferAsyncOpKernel&) = delete; -}; - -} // namespace tensorflow - -#endif // TENSORFLOW_CORE_TPU_KERNELS_TRANSFER_OPS_H_ diff --git a/tensorflow/core/tpu/tpu_defs.cc b/tensorflow/core/tpu/tpu_defs.cc index 69d4989773a..69669bfdb7b 100644 --- a/tensorflow/core/tpu/tpu_defs.cc +++ b/tensorflow/core/tpu/tpu_defs.cc @@ -15,10 +15,6 @@ limitations under the License. #include "tensorflow/core/tpu/tpu_defs.h" -#include "tensorflow/core/tpu/tpu_api.h" -#include "tensorflow/stream_executor/tpu/c_api_conversions.h" -#include "tensorflow/stream_executor/tpu/c_api_decl.h" - namespace tensorflow { const char* const DEVICE_TPU_NODE = "TPU"; @@ -31,18 +27,4 @@ const char* const TPUREPLICATE_MIRRORED_VAR_INDICES_ATTR = const char* const kTPUReplicateAttr = "_tpu_replicate"; const char* const kOutsideCompilationAttr = "_xla_outside_compilation"; -xla::Shape GetTPUInfeedLayout(const xla::Shape& shape) { - XLA_Shape c_shape; - XLA_Shape c_infeed_shape; - - ApiConverter::ToC(shape, &c_shape); - - tpu::ExecutorApiFn()->TpuTransferManager_GetInfeedLayoutFn(&c_shape, - &c_infeed_shape); - xla::Shape infeed_shape = ApiConverter::FromC(&c_infeed_shape); - ApiConverter::Free(&c_shape); - ApiConverter::Free(&c_infeed_shape); - return infeed_shape; -} - } // namespace tensorflow diff --git a/tensorflow/core/tpu/tpu_defs.h b/tensorflow/core/tpu/tpu_defs.h index 29954b2289f..008e386dde6 100644 --- a/tensorflow/core/tpu/tpu_defs.h +++ b/tensorflow/core/tpu/tpu_defs.h @@ -20,7 +20,6 @@ limitations under the License. #include -#include "tensorflow/compiler/xla/shape.h" #include "tensorflow/core/framework/types.pb.h" namespace tensorflow { @@ -57,11 +56,6 @@ static constexpr std::array kTpuAllTypes = { DT_COMPLEX64, DT_INT64, DT_UINT64, DT_QINT8, DT_QUINT8, DT_INT8, DT_UINT8, DT_INT16, DT_UINT16}}; -// For the given shape, chooses a layout for infeed on TPU. The returned shape -// has the same dimensions as the original shape, and only the layout is -// changed. -xla::Shape GetTPUInfeedLayout(const xla::Shape& shape); - } // namespace tensorflow #endif // TENSORFLOW_CORE_TPU_TPU_DEFS_H_ diff --git a/tensorflow/core/tpu/tpu_library_init_fns.inc b/tensorflow/core/tpu/tpu_library_init_fns.inc index 40130bd46dd..be9d594685e 100644 --- a/tensorflow/core/tpu/tpu_library_init_fns.inc +++ b/tensorflow/core/tpu/tpu_library_init_fns.inc @@ -161,7 +161,6 @@ tensorflow::Status SetExecutorStructFn(void* library_handle) { TFTPU_SET_FN(executor_fn, TpuTransferManager_TransferLiteralFromDevice); TFTPU_SET_FN(executor_fn, TpuTransferManager_GetByteSizeRequirement); TFTPU_SET_FN(executor_fn, TpuTransferManager_WriteSingleTupleIndexTable); - TFTPU_SET_FN(executor_fn, TpuTransferManager_GetInfeedLayout); TFTPU_SET_FN(executor_fn, TpuTransferManager_LinearizeToBuffers); TFTPU_SET_FN(executor_fn, TpuTransferManager_FreeBuffers); From 53c817161456f7922c832db1a3dbd84d994337ac Mon Sep 17 00:00:00 2001 From: Yuqi Li Date: Mon, 10 Aug 2020 01:07:34 -0700 Subject: [PATCH 0734/1017] update pip installation and import path in colabs for tflite model maker. PiperOrigin-RevId: 325756233 Change-Id: Ifb629e75408c620e5c11e00fa5eaef79633c858b --- .../model_maker_image_classification.ipynb | 61 +++---------------- .../model_maker_question_answer.ipynb | 12 ++-- .../model_maker_text_classification.ipynb | 12 ++-- 3 files changed, 22 insertions(+), 63 deletions(-) diff --git a/tensorflow/lite/g3doc/tutorials/model_maker_image_classification.ipynb b/tensorflow/lite/g3doc/tutorials/model_maker_image_classification.ipynb index 99ebb7087f2..e88c2e93519 100644 --- a/tensorflow/lite/g3doc/tutorials/model_maker_image_classification.ipynb +++ b/tensorflow/lite/g3doc/tutorials/model_maker_image_classification.ipynb @@ -101,7 +101,7 @@ }, "outputs": [], "source": [ - "!pip install git+https://github.com/tensorflow/examples.git#egg=tensorflow-examples[model_maker]" + "!pip install tflite-model-maker" ] }, { @@ -129,11 +129,10 @@ "import tensorflow as tf\n", "assert tf.__version__.startswith('2')\n", "\n", - "from tensorflow_examples.lite.model_maker.core.data_util.image_dataloader import ImageClassifierDataLoader\n", - "from tensorflow_examples.lite.model_maker.core.task import image_classifier\n", - "from tensorflow_examples.lite.model_maker.core.task.configs import QuantizationConfig\n", - "from tensorflow_examples.lite.model_maker.core.task.model_spec import mobilenet_v2_spec\n", - "from tensorflow_examples.lite.model_maker.core.task.model_spec import ImageModelSpec\n", + "from tflite_model_maker import configs\n", + "from tflite_model_maker import image_classifier\n", + "from tflite_model_maker import ImageClassifierDataLoader\n", + "from tflite_model_maker import model_spec\n", "\n", "import matplotlib.pyplot as plt" ] @@ -640,7 +639,7 @@ "id": "-4jQaxyT5_KV" }, "source": [ - "Here, we also demonstrate how to use the above files to run and evaluate the TensorFlow Lite model." + "You can also evalute the tflite model with the `evaluate_tflite` method." ] }, { @@ -653,47 +652,7 @@ }, "outputs": [], "source": [ - "# Read TensorFlow Lite model from TensorFlow Lite file.\n", - "with tf.io.gfile.GFile('model.tflite', 'rb') as f:\n", - " model_content = f.read()\n", - "\n", - "# Initialze TensorFlow Lite inpterpreter.\n", - "interpreter = tf.lite.Interpreter(model_content=model_content)\n", - "interpreter.allocate_tensors()\n", - "input_index = interpreter.get_input_details()[0]['index']\n", - "output = interpreter.tensor(interpreter.get_output_details()[0][\"index\"])\n", - "\n", - "# Run predictions on each test image data and calculate accuracy.\n", - "accurate_count = 0\n", - "for i, (image, label) in enumerate(test_data.dataset):\n", - " # Pre-processing should remain the same. Currently, just normalize each pixel value and resize image according to the model's specification.\n", - " image, _ = model.preprocess(image, label)\n", - " # Add batch dimension and convert to float32 to match with the model's input\n", - " # data format.\n", - " image = tf.expand_dims(image, 0).numpy()\n", - "\n", - " # Run inference.\n", - " interpreter.set_tensor(input_index, image)\n", - " interpreter.invoke()\n", - "\n", - " # Post-processing: remove batch dimension and find the label with highest\n", - " # probability.\n", - " predict_label = np.argmax(output()[0])\n", - "\n", - " accurate_count += (predict_label == label.numpy())\n", - "\n", - "accuracy = accurate_count * 1.0 / test_data.size\n", - "print('TensorFlow Lite model accuracy = %.4f' % accuracy)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "fuHB-NFqpKTD" - }, - "source": [ - "Note that preprocessing for inference should be the same as training. Currently, preprocessing contains normalizing each pixel value and resizing the image according to the model's specification. For EfficientNet-Lite0, input image should be normalized to `[0, 1]` and resized to `[224, 224, 3]`." + "model.evaluate_tflite('model.tflite', test_data)" ] }, { @@ -760,7 +719,7 @@ }, "outputs": [], "source": [ - "config = QuantizationConfig.create_full_integer_quantization(representative_data=test_data, is_integer_only=True)" + "config = configs.QuantizationConfig.create_full_integer_quantization(representative_data=test_data, is_integer_only=True)" ] }, { @@ -830,7 +789,7 @@ }, "outputs": [], "source": [ - "model = image_classifier.create(train_data, model_spec=mobilenet_v2_spec, validation_data=validation_data)" + "model = image_classifier.create(train_data, model_spec=model_spec.mobilenet_v2_spec, validation_data=validation_data)" ] }, { @@ -882,7 +841,7 @@ }, "outputs": [], "source": [ - "inception_v3_spec = ImageModelSpec(\n", + "inception_v3_spec = model_spec.ImageModelSpec(\n", " uri='https://tfhub.dev/google/imagenet/inception_v3/feature_vector/1')\n", "inception_v3_spec.input_image_shape = [299, 299]" ] diff --git a/tensorflow/lite/g3doc/tutorials/model_maker_question_answer.ipynb b/tensorflow/lite/g3doc/tutorials/model_maker_question_answer.ipynb index a1d11115a75..645be959d0e 100644 --- a/tensorflow/lite/g3doc/tutorials/model_maker_question_answer.ipynb +++ b/tensorflow/lite/g3doc/tutorials/model_maker_question_answer.ipynb @@ -188,7 +188,7 @@ }, "outputs": [], "source": [ - "!pip install git+https://github.com/tensorflow/examples.git#egg=tensorflow-examples[model_maker]" + "!pip install tflite-model-maker" ] }, { @@ -217,10 +217,10 @@ "import tensorflow as tf\n", "assert tf.__version__.startswith('2')\n", "\n", - "from tensorflow_examples.lite.model_maker.core.data_util.text_dataloader import QuestionAnswerDataLoader\n", - "from tensorflow_examples.lite.model_maker.core.task import model_spec\n", - "from tensorflow_examples.lite.model_maker.core.task import question_answer\n", - "from tensorflow_examples.lite.model_maker.core.task.configs import QuantizationConfig" + "from tflite_model_maker import configs\n", + "from tflite_model_maker import model_spec\n", + "from tflite_model_maker import question_answer\n", + "from tflite_model_maker import QuestionAnswerDataLoader" ] }, { @@ -448,7 +448,7 @@ }, "outputs": [], "source": [ - "config = QuantizationConfig.create_dynamic_range_quantization(optimizations=[tf.lite.Optimize.OPTIMIZE_FOR_LATENCY])\n", + "config = configs.QuantizationConfig.create_dynamic_range_quantization(optimizations=[tf.lite.Optimize.OPTIMIZE_FOR_LATENCY])\n", "config._experimental_new_quantizer = True" ] }, diff --git a/tensorflow/lite/g3doc/tutorials/model_maker_text_classification.ipynb b/tensorflow/lite/g3doc/tutorials/model_maker_text_classification.ipynb index 1a839d70e38..88cef93e761 100644 --- a/tensorflow/lite/g3doc/tutorials/model_maker_text_classification.ipynb +++ b/tensorflow/lite/g3doc/tutorials/model_maker_text_classification.ipynb @@ -110,7 +110,7 @@ }, "outputs": [], "source": [ - "!pip install git+https://github.com/tensorflow/examples.git#egg=tensorflow-examples[model_maker]" + "!pip install tflite-model-maker" ] }, { @@ -139,10 +139,10 @@ "import tensorflow as tf\n", "assert tf.__version__.startswith('2')\n", "\n", - "from tensorflow_examples.lite.model_maker.core.data_util.text_dataloader import TextClassifierDataLoader\n", - "from tensorflow_examples.lite.model_maker.core.task import model_spec\n", - "from tensorflow_examples.lite.model_maker.core.task import text_classifier\n", - "from tensorflow_examples.lite.model_maker.core.task.configs import QuantizationConfig" + "from tflite_model_maker import configs\n", + "from tflite_model_maker import model_spec\n", + "from tflite_model_maker import text_classifier\n", + "from tflite_model_maker import TextClassifierDataLoader" ] }, { @@ -344,7 +344,7 @@ }, "outputs": [], "source": [ - "config = QuantizationConfig.create_dynamic_range_quantization(optimizations=[tf.lite.Optimize.OPTIMIZE_FOR_LATENCY])\n", + "config = configs.QuantizationConfig.create_dynamic_range_quantization(optimizations=[tf.lite.Optimize.OPTIMIZE_FOR_LATENCY])\n", "config._experimental_new_quantizer = True" ] }, From c09f5c4f1aa857a2f6bd311d30695fe1b0b66f1b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 02:01:29 -0700 Subject: [PATCH 0735/1017] compat: Update forward compatibility horizon to 2020-08-10 PiperOrigin-RevId: 325761194 Change-Id: I5cb5e65aef07cd70a0331aa162c92b5f8d828ce8 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index fefbf667704..3d8f9c5490e 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 9) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 10) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From 8a4ffe2e1ae722cff5306778df0cfca8b7f503fe Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 02:01:30 -0700 Subject: [PATCH 0736/1017] Update GraphDef version to 489. PiperOrigin-RevId: 325761195 Change-Id: I1eb19d20d43ee9189b563472dbe0b3a65a5ef9f8 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 1813717e87a..fa0df24a7e7 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 488 // Updated: 2020/8/9 +#define TF_GRAPH_DEF_VERSION 489 // Updated: 2020/8/10 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From d5eaf2316406e774a7f38a6fac7826783cbf4e8c Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Mon, 10 Aug 2020 06:03:37 -0700 Subject: [PATCH 0737/1017] Disable logical_expressions_test for now since it failed tsan test. PiperOrigin-RevId: 325788797 Change-Id: I92a0141f570fc7b4c2d5ff182278f8bbc9ad7436 --- tensorflow/python/autograph/converters/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/autograph/converters/BUILD b/tensorflow/python/autograph/converters/BUILD index f584038978f..fd8ec1dbaa3 100644 --- a/tensorflow/python/autograph/converters/BUILD +++ b/tensorflow/python/autograph/converters/BUILD @@ -176,6 +176,7 @@ py_test( srcs = ["logical_expressions_test.py"], python_version = "PY3", srcs_version = "PY2AND3", + tags = ["notsan"], # b/163218460 deps = [ ":converters", "//tensorflow/python:client_testlib", From 7f5e4f3361f178f5c1109b4ff0b69185daf63aa4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 06:44:31 -0700 Subject: [PATCH 0738/1017] Return shared_ptr to allow passing them back to C++. PiperOrigin-RevId: 325793565 Change-Id: I71958cdb27da33471ea795d7e8a41085983bfb2b --- tensorflow/compiler/xla/python/py_client.cc | 6 ++++-- tensorflow/compiler/xla/python/py_client.h | 2 +- tensorflow/compiler/xla/python/xla.cc | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/python/py_client.cc b/tensorflow/compiler/xla/python/py_client.cc index 4224d69dc8f..9b95f8e03de 100644 --- a/tensorflow/compiler/xla/python/py_client.cc +++ b/tensorflow/compiler/xla/python/py_client.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow/compiler/xla/python/py_client.h" +#include + #include "absl/container/flat_hash_map.h" #include "tensorflow/compiler/xla/python/py_buffer.h" #include "tensorflow/compiler/xla/python/py_executable.h" @@ -120,7 +122,7 @@ StatusOr> PyClient::BufferFromPyval( std::move(traceback)); } -StatusOr> PyClient::Compile( +StatusOr> PyClient::Compile( const XlaComputation& computation, CompileOptions options) { std::unique_ptr executable; absl::optional fingerprint; @@ -133,7 +135,7 @@ StatusOr> PyClient::Compile( pjrt_client_->ExecutableFingerprint(*executable)); } auto traceback = Traceback::Get(); - return std::make_unique( + return std::make_shared( shared_from_this(), std::move(executable), std::move(traceback), std::move(fingerprint)); } diff --git a/tensorflow/compiler/xla/python/py_client.h b/tensorflow/compiler/xla/python/py_client.h index d33f3dadd7d..e41415c42f2 100644 --- a/tensorflow/compiler/xla/python/py_client.h +++ b/tensorflow/compiler/xla/python/py_client.h @@ -124,7 +124,7 @@ class PyClient : public std::enable_shared_from_this { const pybind11::object& argument, Device* device, bool force_copy, PjRtBuffer::HostBufferSemantics host_buffer_semantics); - StatusOr> Compile( + StatusOr> Compile( const XlaComputation& computation, CompileOptions options); pybind11::bytes HeapProfile(); diff --git a/tensorflow/compiler/xla/python/xla.cc b/tensorflow/compiler/xla/python/xla.cc index 9590c5d57c3..510175cebf6 100644 --- a/tensorflow/compiler/xla/python/xla.cc +++ b/tensorflow/compiler/xla/python/xla.cc @@ -654,7 +654,7 @@ PYBIND11_MODULE(xla_extension, m) { PyTypeObject* buffer_type = reinterpret_cast(buffer.ptr()); buffer_type->tp_as_buffer = PyBuffer::BufferProtocol(); - py::class_> executable( + py::class_> executable( m, "Executable"); executable.def_property_readonly("client", &PyExecutable::client) .def("local_logical_device_ids", &PyExecutable::local_logical_device_ids) From 2ec0a4987e807b18c52a6ec93e5aed74ea22117c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 07:05:41 -0700 Subject: [PATCH 0739/1017] Add a `PjRtExecute` function. PiperOrigin-RevId: 325796542 Change-Id: Iba3235bd650316f61b3ce37013b4437cafb205fa --- .../compiler/xla/python/py_executable.cc | 33 ++++++++++++------- .../compiler/xla/python/py_executable.h | 9 +++++ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/tensorflow/compiler/xla/python/py_executable.cc b/tensorflow/compiler/xla/python/py_executable.cc index b2cd2af56ea..ed524f1cb33 100644 --- a/tensorflow/compiler/xla/python/py_executable.cc +++ b/tensorflow/compiler/xla/python/py_executable.cc @@ -37,7 +37,9 @@ PyExecutable::PyExecutable(std::shared_ptr client, if (next_) { next_->prev_ = this; } + options_.untuple_result = true; if (fingerprint_) { + options_.launch_id = tensorflow::Fingerprint32(*fingerprint_); VLOG(1) << "Fingerprint for executable " << executable_->name() << ": " << *fingerprint_; } @@ -65,21 +67,33 @@ std::vector> PyExecutable::LocalDevices() const { return devices; } +StatusOr>> PyExecutable::PjRtExecute( + absl::Span args) { + std::vector> output_buffers; + { + py::gil_scoped_release gil_release; + TF_ASSIGN_OR_RETURN(output_buffers, executable_->Execute(args, options_)); + } + auto traceback = Traceback::Get(); + std::vector> outputs; + outputs.reserve(output_buffers.size()); + for (auto& buffer : output_buffers) { + outputs.push_back( + std::make_unique(client_, std::move(buffer), traceback)); + } + return outputs; +} + StatusOr>> PyExecutable::Execute( absl::Span args) { std::vector> output_buffers; { py::gil_scoped_release gil_release; - ExecuteOptions options; - options.untuple_result = true; - if (fingerprint_) { - options.launch_id = tensorflow::Fingerprint32(*fingerprint_); - } std::vector arg_buffers(args.size()); absl::c_transform(args, arg_buffers.begin(), [](PyBuffer* buf) { return buf->buffer(); }); TF_ASSIGN_OR_RETURN(output_buffers, - executable_->Execute(arg_buffers, options)); + executable_->Execute(arg_buffers, options_)); } auto traceback = Traceback::Get(); std::vector> outputs; @@ -97,11 +111,6 @@ PyExecutable::ExecuteOnLocalDevices( std::vector>> output_buffers; { py::gil_scoped_release gil_release; - ExecuteOptions options; - options.untuple_result = true; - if (fingerprint_) { - options.launch_id = tensorflow::Fingerprint32(*fingerprint_); - } std::vector> arg_buffers(args.size()); for (int computation = 0; computation < args.size(); ++computation) { arg_buffers[computation].resize(args[computation].size()); @@ -109,7 +118,7 @@ PyExecutable::ExecuteOnLocalDevices( [](PyBuffer* buf) { return buf->buffer(); }); } TF_ASSIGN_OR_RETURN(output_buffers, executable_->ExecuteOnLocalDevices( - arg_buffers, options)); + arg_buffers, options_)); } auto traceback = Traceback::Get(); std::vector>> outputs; diff --git a/tensorflow/compiler/xla/python/py_executable.h b/tensorflow/compiler/xla/python/py_executable.h index 1051d065335..24f177261e7 100644 --- a/tensorflow/compiler/xla/python/py_executable.h +++ b/tensorflow/compiler/xla/python/py_executable.h @@ -58,6 +58,10 @@ class PyExecutable { StatusOr>> Execute( absl::Span args); + // Same as above, but take as inputs `PjRtBuffer*`. Only targets C++ code. + StatusOr>> PjRtExecute( + absl::Span args); + StatusOr>>> ExecuteOnLocalDevices(absl::Span> args); @@ -65,6 +69,8 @@ class PyExecutable { Traceback* traceback() { return traceback_.get(); } + const PjRtExecutable& pjrt_executable() const { return *executable_; } + private: friend class PyClient; @@ -77,6 +83,9 @@ class PyExecutable { // aren't implemented. absl::optional fingerprint_; + // The options to pass to `executable_.Execute`. + ExecuteOptions options_; + // Doubly-linked list of all executables known to the client. Protected by the // GIL. PyExecutable* next_; From 21240f6f4086038a94c65c6cb527786cef17cc76 Mon Sep 17 00:00:00 2001 From: Tom Hennigan Date: Mon, 10 Aug 2020 08:09:30 -0700 Subject: [PATCH 0740/1017] Set connect timeout based on client RPC deadline. PiperOrigin-RevId: 325805342 Change-Id: I22ec18729cf027560a479ef27596fce54bc90606 --- tensorflow/compiler/xla/pjrt/distributed/client.cc | 2 ++ tensorflow/compiler/xla/pjrt/distributed/protocol.h | 2 +- tensorflow/compiler/xla/pjrt/distributed/protocol.proto | 1 + tensorflow/compiler/xla/pjrt/distributed/service.cc | 6 ++++-- tensorflow/compiler/xla/pjrt/distributed/service.h | 2 -- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tensorflow/compiler/xla/pjrt/distributed/client.cc b/tensorflow/compiler/xla/pjrt/distributed/client.cc index 55b02c6a09e..43c0c7b277d 100644 --- a/tensorflow/compiler/xla/pjrt/distributed/client.cc +++ b/tensorflow/compiler/xla/pjrt/distributed/client.cc @@ -17,6 +17,7 @@ limitations under the License. #include // NOLINT +#include "absl/time/time.h" #include "tensorflow/compiler/xla/pjrt/distributed/protocol.h" #include "tensorflow/compiler/xla/pjrt/distributed/util.h" @@ -36,6 +37,7 @@ xla::Status DistributedRuntimeClient::Connect( ctx.set_deadline(absl::ToChronoTime(absl::Now() + rpc_timeout_)); ConnectRequest request; request.set_protocol_version(kDistributedRuntimeProtocolVersion); + request.set_timeout_milliseconds(absl::ToInt64Milliseconds(rpc_timeout_)); *request.mutable_local_topology() = local_topology; VLOG(10) << "Connect: " << request.DebugString(); ConnectResponse response; diff --git a/tensorflow/compiler/xla/pjrt/distributed/protocol.h b/tensorflow/compiler/xla/pjrt/distributed/protocol.h index 4daa939ac8d..e8be43006f7 100644 --- a/tensorflow/compiler/xla/pjrt/distributed/protocol.h +++ b/tensorflow/compiler/xla/pjrt/distributed/protocol.h @@ -18,7 +18,7 @@ limitations under the License. namespace xla { -static constexpr int kDistributedRuntimeProtocolVersion = 1; +static constexpr int kDistributedRuntimeProtocolVersion = 2; } // namespace xla diff --git a/tensorflow/compiler/xla/pjrt/distributed/protocol.proto b/tensorflow/compiler/xla/pjrt/distributed/protocol.proto index 18bfa221110..c3bbb3a7f5d 100644 --- a/tensorflow/compiler/xla/pjrt/distributed/protocol.proto +++ b/tensorflow/compiler/xla/pjrt/distributed/protocol.proto @@ -61,6 +61,7 @@ message ConnectRequest { int32 protocol_version = 1; // Always 1 at present. LocalTopologyProto local_topology = 2; + int32 timeout_milliseconds = 3; } message ConnectResponse { diff --git a/tensorflow/compiler/xla/pjrt/distributed/service.cc b/tensorflow/compiler/xla/pjrt/distributed/service.cc index 3325fcd8319..868529637de 100644 --- a/tensorflow/compiler/xla/pjrt/distributed/service.cc +++ b/tensorflow/compiler/xla/pjrt/distributed/service.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow/compiler/xla/pjrt/distributed/service.h" +#include "absl/time/time.h" #include "tensorflow/compiler/xla/pjrt/distributed/protocol.h" #include "tensorflow/compiler/xla/pjrt/distributed/util.h" #include "tensorflow/compiler/xla/status.h" @@ -69,11 +70,12 @@ void BuildGlobalTopology(absl::Span local_topologies, mu_.AssertHeld(); return num_nodes_present_ == nodes_.size(); }; + auto connect_timeout = absl::Milliseconds(request->timeout_milliseconds()); if (!mu_.AwaitWithTimeout(absl::Condition(&all_nodes_present), - kConnectTimeout)) { + connect_timeout)) { return ToGrpcStatus(tensorflow::errors::DeadlineExceeded( "Timed out after %s waiting for all nodes to call Connect()", - absl::FormatDuration(kConnectTimeout))); + absl::FormatDuration(connect_timeout))); } if (node_id == 0) { diff --git a/tensorflow/compiler/xla/pjrt/distributed/service.h b/tensorflow/compiler/xla/pjrt/distributed/service.h index 9ecbdb3cc7c..fe323d9f3b2 100644 --- a/tensorflow/compiler/xla/pjrt/distributed/service.h +++ b/tensorflow/compiler/xla/pjrt/distributed/service.h @@ -50,8 +50,6 @@ class DistributedRuntimeServiceImpl final KeyValueSetResponse* response) override; private: - const absl::Duration kConnectTimeout = absl::Seconds(120); - absl::Mutex mu_; enum class State { kInitializing, kRunning }; State state_ ABSL_GUARDED_BY(mu_) = State::kInitializing; From 0e368fa3998ce7213159751f74bde3c98d72edc9 Mon Sep 17 00:00:00 2001 From: Pankaj Kanwar Date: Mon, 10 Aug 2020 08:40:33 -0700 Subject: [PATCH 0741/1017] Test build for testing cuda11 changes. PiperOrigin-RevId: 325810725 Change-Id: Id018e4512c074fba2af19b8d8919c3dbb6c7768b --- tensorflow/opensource_only.files | 2 + .../rel/ubuntu_cuda11/gpu_py35_nonpip.sh | 6 +- .../rel/ubuntu_cuda11/gpu_py37_pip.sh | 2 +- .../gcc7_manylinux2010-nvcc-cuda11/BUILD | 175 ++ .../cc_toolchain_config.bzl | 1516 +++++++++++++++++ .../bin/crosstool_wrapper_driver_is_not_gcc | 289 ++++ 6 files changed, 1986 insertions(+), 4 deletions(-) create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/BUILD create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/cc_toolchain_config.bzl create mode 100755 third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/clang/bin/crosstool_wrapper_driver_is_not_gcc diff --git a/tensorflow/opensource_only.files b/tensorflow/opensource_only.files index d46d9a27b24..faf097e85f9 100644 --- a/tensorflow/opensource_only.files +++ b/tensorflow/opensource_only.files @@ -245,6 +245,8 @@ tensorflow/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc- tensorflow/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0/cc_toolchain_config.bzl tensorflow/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1/BUILD tensorflow/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1/cc_toolchain_config.bzl +tensorflow/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/BUILD +tensorflow/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/cc_toolchain_config.bzl tensorflow/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010/BUILD tensorflow/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010/cc_toolchain_config.bzl tensorflow/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010/dummy_toolchain.bzl diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh index 47ed3c4fd2a..8a0796723b2 100644 --- a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh @@ -27,8 +27,8 @@ export TF_NEED_GCP=1 export TF_NEED_HDFS=1 export TF_NEED_S3=1 export TF_NEED_CUDA=1 -export TF_CUDA_VERSION=10 -export TF_CUDNN_VERSION=7 +export TF_CUDA_VERSION=11 +export TF_CUDNN_VERSION=8 export TF_NEED_TENSORRT=1 export TENSORRT_INSTALL_PATH=/usr/local/tensorrt export CC_OPT_FLAGS='-mavx' @@ -47,7 +47,7 @@ tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py35" set +e bazel test --config=cuda --config=opt \ - --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11:toolchain \ --linkopt=-lrt \ --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ --test_lang_filters=py \ diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh index 9bfc6608a0b..71d6f3e6401 100644 --- a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh @@ -39,7 +39,7 @@ export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss export TF_BUILD_FLAGS="--config=release_gpu_linux " export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ --distinct_host_configuration=false \ ---action_env=TF_CUDA_VERSION=10 --action_env=TF_CUDNN_VERSION=7 --test_env=TF2_BEHAVIOR=1 \ +--action_env=TF_CUDA_VERSION=11 --action_env=TF_CUDNN_VERSION=8 --test_env=TF2_BEHAVIOR=1 \ --config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ --verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/BUILD b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/BUILD new file mode 100755 index 00000000000..92305526c5c --- /dev/null +++ b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/BUILD @@ -0,0 +1,175 @@ +# This file is expanded from a template by cuda_configure.bzl +# Update cuda_configure.bzl#verify_build_defines when adding new variables. + +load(":cc_toolchain_config.bzl", "cc_toolchain_config") + +licenses(["restricted"]) + +package(default_visibility = ["//visibility:public"]) + +toolchain( + name = "toolchain-linux-x86_64", + exec_compatible_with = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], + target_compatible_with = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], + toolchain = ":cc-compiler-local", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +cc_toolchain_suite( + name = "toolchain", + toolchains = { + "local|compiler": ":cc-compiler-local", + "darwin|compiler": ":cc-compiler-darwin", + "x64_windows|msvc-cl": ":cc-compiler-windows", + "x64_windows": ":cc-compiler-windows", + "arm": ":cc-compiler-local", + "aarch64": ":cc-compiler-local", + "k8": ":cc-compiler-local", + "piii": ":cc-compiler-local", + "ppc": ":cc-compiler-local", + "darwin": ":cc-compiler-darwin", + }, +) + +cc_toolchain( + name = "cc-compiler-local", + all_files = ":crosstool_wrapper_driver_is_not_gcc", + ar_files = ":crosstool_wrapper_driver_is_not_gcc", + as_files = ":crosstool_wrapper_driver_is_not_gcc", + compiler_files = ":crosstool_wrapper_driver_is_not_gcc", + dwp_files = ":empty", + linker_files = ":crosstool_wrapper_driver_is_not_gcc", + objcopy_files = ":empty", + strip_files = ":empty", + # To support linker flags that need to go to the start of command line + # we need the toolchain to support parameter files. Parameter files are + # last on the command line and contain all shared libraries to link, so all + # regular options will be left of them. + supports_param_files = 1, + toolchain_config = ":cc-compiler-local-config", + toolchain_identifier = "local_linux", +) + +cc_toolchain_config( + name = "cc-compiler-local-config", + builtin_include_directories = [ + "/dt7/usr/include/c++/7", + "/dt7/usr/include/c++/7/x86_64-pc-linux-gnu", + "/dt7/usr/include/c++/7/backward", + "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include", + "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include-fixed", + "/dt7/usr/include", + "/usr/local/cuda11/targets/x86_64-linux/include", + "/usr/local/cuda11/include", + "/usr/local/cuda11/extras/CUPTI/include", + "/usr/include", + ], + builtin_sysroot = "", + cpu = "local", + cuda_path = "", + extra_no_canonical_prefixes_flags = ["-fno-canonical-system-headers"], + host_compiler_path = "clang/bin/crosstool_wrapper_driver_is_not_gcc", + host_compiler_prefix = "/usr/bin", + host_compiler_warnings = [], + host_unfiltered_compile_flags = [], + linker_bin_path = "/usr/bin", +) + +cc_toolchain( + name = "cc-compiler-darwin", + all_files = ":crosstool_wrapper_driver_is_not_gcc", + ar_files = ":crosstool_wrapper_driver_is_not_gcc", + as_files = ":crosstool_wrapper_driver_is_not_gcc", + compiler_files = ":crosstool_wrapper_driver_is_not_gcc", + dwp_files = ":empty", + linker_files = ":crosstool_wrapper_driver_is_not_gcc", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 0, + toolchain_config = ":cc-compiler-local-darwin", + toolchain_identifier = "local_darwin", +) + +cc_toolchain_config( + name = "cc-compiler-local-darwin", + builtin_include_directories = [ + "/dt7/usr/include/c++/7", + "/dt7/usr/include/c++/7/x86_64-pc-linux-gnu", + "/dt7/usr/include/c++/7/backward", + "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include", + "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include-fixed", + "/dt7/usr/include", + "/usr/local/cuda-11/targets/x86_64-linux/include", + "/usr/local/cuda-11/include", + "/usr/local/cuda-11/extras/CUPTI/include", + "/usr/include", + ], + cpu = "darwin", + extra_no_canonical_prefixes_flags = ["-fno-canonical-system-headers"], + host_compiler_path = "clang/bin/crosstool_wrapper_driver_is_not_gcc", + host_compiler_prefix = "/usr/bin", + host_compiler_warnings = [], + host_unfiltered_compile_flags = [], + linker_bin_path = "/usr/bin", +) + +cc_toolchain( + name = "cc-compiler-windows", + all_files = ":windows_msvc_wrapper_files", + ar_files = ":windows_msvc_wrapper_files", + as_files = ":windows_msvc_wrapper_files", + compiler_files = ":windows_msvc_wrapper_files", + dwp_files = ":empty", + linker_files = ":windows_msvc_wrapper_files", + objcopy_files = ":empty", + strip_files = ":empty", + supports_param_files = 1, + toolchain_config = ":cc-compiler-windows-config", + toolchain_identifier = "local_windows", +) + +cc_toolchain_config( + name = "cc-compiler-windows-config", + builtin_include_directories = [ + "/dt7/usr/include/c++/7", + "/dt7/usr/include/c++/7/x86_64-pc-linux-gnu", + "/dt7/usr/include/c++/7/backward", + "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include", + "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include-fixed", + "/dt7/usr/include", + "/usr/local/cuda-11/targets/x86_64-linux/include", + "/usr/local/cuda-11/include", + "/usr/local/cuda-11/extras/CUPTI/include", + "/usr/include", + ], + cpu = "x64_windows", + msvc_cl_path = "msvc_not_used", + msvc_env_include = "msvc_not_used", + msvc_env_lib = "msvc_not_used", + msvc_env_path = "msvc_not_used", + msvc_env_tmp = "msvc_not_used", + msvc_lib_path = "msvc_not_used", + msvc_link_path = "msvc_not_used", + msvc_ml_path = "msvc_not_used", +) + +filegroup( + name = "empty", + srcs = [], +) + +filegroup( + name = "crosstool_wrapper_driver_is_not_gcc", + srcs = ["clang/bin/crosstool_wrapper_driver_is_not_gcc"], +) + +filegroup( + name = "windows_msvc_wrapper_files", + srcs = glob(["windows/msvc_*"]), +) diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/cc_toolchain_config.bzl b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/cc_toolchain_config.bzl new file mode 100755 index 00000000000..70197628811 --- /dev/null +++ b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/cc_toolchain_config.bzl @@ -0,0 +1,1516 @@ +"""cc_toolchain_config rule for configuring CUDA toolchains on Linux, Mac, and Windows.""" + +load( + "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", + "action_config", + "env_entry", + "env_set", + "feature", + "feature_set", + "flag_group", + "flag_set", + "tool", + "tool_path", + "variable_with_value", +) +load( + "@bazel_tools//tools/build_defs/cc:action_names.bzl", + "ASSEMBLE_ACTION_NAME", + "CC_FLAGS_MAKE_VARIABLE_ACTION_NAME", + "CLIF_MATCH_ACTION_NAME", + "CPP_COMPILE_ACTION_NAME", + "CPP_HEADER_PARSING_ACTION_NAME", + "CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME", + "CPP_LINK_EXECUTABLE_ACTION_NAME", + "CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME", + "CPP_LINK_STATIC_LIBRARY_ACTION_NAME", + "CPP_MODULE_CODEGEN_ACTION_NAME", + "CPP_MODULE_COMPILE_ACTION_NAME", + "C_COMPILE_ACTION_NAME", + "LINKSTAMP_COMPILE_ACTION_NAME", + "LTO_BACKEND_ACTION_NAME", + "LTO_INDEXING_ACTION_NAME", + "OBJCPP_COMPILE_ACTION_NAME", + "OBJCPP_EXECUTABLE_ACTION_NAME", + "OBJC_ARCHIVE_ACTION_NAME", + "OBJC_COMPILE_ACTION_NAME", + "OBJC_EXECUTABLE_ACTION_NAME", + "OBJC_FULLY_LINK_ACTION_NAME", + "PREPROCESS_ASSEMBLE_ACTION_NAME", + "STRIP_ACTION_NAME", +) + +ACTION_NAMES = struct( + c_compile = C_COMPILE_ACTION_NAME, + cpp_compile = CPP_COMPILE_ACTION_NAME, + linkstamp_compile = LINKSTAMP_COMPILE_ACTION_NAME, + cc_flags_make_variable = CC_FLAGS_MAKE_VARIABLE_ACTION_NAME, + cpp_module_codegen = CPP_MODULE_CODEGEN_ACTION_NAME, + cpp_header_parsing = CPP_HEADER_PARSING_ACTION_NAME, + cpp_module_compile = CPP_MODULE_COMPILE_ACTION_NAME, + assemble = ASSEMBLE_ACTION_NAME, + preprocess_assemble = PREPROCESS_ASSEMBLE_ACTION_NAME, + lto_indexing = LTO_INDEXING_ACTION_NAME, + lto_backend = LTO_BACKEND_ACTION_NAME, + cpp_link_executable = CPP_LINK_EXECUTABLE_ACTION_NAME, + cpp_link_dynamic_library = CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME, + cpp_link_nodeps_dynamic_library = CPP_LINK_NODEPS_DYNAMIC_LIBRARY_ACTION_NAME, + cpp_link_static_library = CPP_LINK_STATIC_LIBRARY_ACTION_NAME, + strip = STRIP_ACTION_NAME, + objc_archive = OBJC_ARCHIVE_ACTION_NAME, + objc_compile = OBJC_COMPILE_ACTION_NAME, + objc_executable = OBJC_EXECUTABLE_ACTION_NAME, + objc_fully_link = OBJC_FULLY_LINK_ACTION_NAME, + objcpp_compile = OBJCPP_COMPILE_ACTION_NAME, + objcpp_executable = OBJCPP_EXECUTABLE_ACTION_NAME, + clif_match = CLIF_MATCH_ACTION_NAME, + objcopy_embed_data = "objcopy_embed_data", + ld_embed_data = "ld_embed_data", +) + +def _impl(ctx): + if (ctx.attr.cpu == "darwin"): + toolchain_identifier = "local_darwin" + elif (ctx.attr.cpu == "local"): + toolchain_identifier = "local_linux" + elif (ctx.attr.cpu == "x64_windows"): + toolchain_identifier = "local_windows" + else: + fail("Unreachable") + + host_system_name = "local" + + target_system_name = "local" + + if (ctx.attr.cpu == "darwin"): + target_cpu = "darwin" + elif (ctx.attr.cpu == "local"): + target_cpu = "local" + elif (ctx.attr.cpu == "x64_windows"): + target_cpu = "x64_windows" + else: + fail("Unreachable") + + if (ctx.attr.cpu == "local"): + target_libc = "local" + elif (ctx.attr.cpu == "darwin"): + target_libc = "macosx" + elif (ctx.attr.cpu == "x64_windows"): + target_libc = "msvcrt" + else: + fail("Unreachable") + + if (ctx.attr.cpu == "darwin" or + ctx.attr.cpu == "local"): + compiler = "compiler" + elif (ctx.attr.cpu == "x64_windows"): + compiler = "msvc-cl" + else: + fail("Unreachable") + + abi_version = "local" + + abi_libc_version = "local" + + cc_target_os = None + + builtin_sysroot = ctx.attr.builtin_sysroot + + all_link_actions = [ + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ] + + cpp_link_dynamic_library_action = action_config( + action_name = ACTION_NAMES.cpp_link_dynamic_library, + implies = [ + "nologo", + "shared_flag", + "linkstamps", + "output_execpath_flags", + "input_param_flags", + "user_link_flags", + "linker_subsystem_flag", + "linker_param_file", + "msvc_env", + "no_stripping", + "has_configured_linker_path", + "def_file", + ], + tools = [tool(path = ctx.attr.msvc_link_path)], + ) + + cpp_link_nodeps_dynamic_library_action = action_config( + action_name = ACTION_NAMES.cpp_link_nodeps_dynamic_library, + implies = [ + "nologo", + "shared_flag", + "linkstamps", + "output_execpath_flags", + "input_param_flags", + "user_link_flags", + "linker_subsystem_flag", + "linker_param_file", + "msvc_env", + "no_stripping", + "has_configured_linker_path", + "def_file", + ], + tools = [tool(path = ctx.attr.msvc_link_path)], + ) + + cpp_link_static_library_action = action_config( + action_name = ACTION_NAMES.cpp_link_static_library, + implies = [ + "nologo", + "archiver_flags", + "input_param_flags", + "linker_param_file", + "msvc_env", + ], + tools = [tool(path = ctx.attr.msvc_lib_path)], + ) + + assemble_action = action_config( + action_name = ACTION_NAMES.assemble, + implies = [ + "compiler_input_flags", + "compiler_output_flags", + "nologo", + "msvc_env", + "sysroot", + ], + tools = [tool(path = ctx.attr.msvc_ml_path)], + ) + + preprocess_assemble_action = action_config( + action_name = ACTION_NAMES.preprocess_assemble, + implies = [ + "compiler_input_flags", + "compiler_output_flags", + "nologo", + "msvc_env", + "sysroot", + ], + tools = [tool(path = ctx.attr.msvc_ml_path)], + ) + + c_compile_action = action_config( + action_name = ACTION_NAMES.c_compile, + implies = [ + "compiler_input_flags", + "compiler_output_flags", + "nologo", + "msvc_env", + "parse_showincludes", + "user_compile_flags", + "sysroot", + "unfiltered_compile_flags", + ], + tools = [tool(path = ctx.attr.msvc_cl_path)], + ) + + cpp_compile_action = action_config( + action_name = ACTION_NAMES.cpp_compile, + implies = [ + "compiler_input_flags", + "compiler_output_flags", + "nologo", + "msvc_env", + "parse_showincludes", + "user_compile_flags", + "sysroot", + "unfiltered_compile_flags", + ], + tools = [tool(path = ctx.attr.msvc_cl_path)], + ) + + cpp_link_executable_action = action_config( + action_name = ACTION_NAMES.cpp_link_executable, + implies = [ + "nologo", + "linkstamps", + "output_execpath_flags", + "input_param_flags", + "user_link_flags", + "linker_subsystem_flag", + "linker_param_file", + "msvc_env", + "no_stripping", + ], + tools = [tool(path = ctx.attr.msvc_link_path)], + ) + + if (ctx.attr.cpu == "darwin" or + ctx.attr.cpu == "local"): + action_configs = [] + elif (ctx.attr.cpu == "x64_windows"): + action_configs = [ + assemble_action, + preprocess_assemble_action, + c_compile_action, + cpp_compile_action, + cpp_link_executable_action, + cpp_link_dynamic_library_action, + cpp_link_nodeps_dynamic_library_action, + cpp_link_static_library_action, + ] + else: + fail("Unreachable") + + no_windows_export_all_symbols_feature = feature(name = "no_windows_export_all_symbols") + + pic_feature = feature( + name = "pic", + enabled = True, + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group(flags = ["-fPIC"], expand_if_available = "pic"), + flag_group( + flags = ["-fPIE"], + expand_if_not_available = "pic", + ), + ], + ), + ], + ) + + preprocessor_defines_feature = feature( + name = "preprocessor_defines", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = [ + flag_group( + flags = ["/D%{preprocessor_defines}"], + iterate_over = "preprocessor_defines", + ), + ], + ), + ], + ) + + generate_pdb_file_feature = feature( + name = "generate_pdb_file", + requires = [ + feature_set(features = ["dbg"]), + feature_set(features = ["fastbuild"]), + ], + ) + + linkstamps_feature = feature( + name = "linkstamps", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{linkstamp_paths}"], + iterate_over = "linkstamp_paths", + expand_if_available = "linkstamp_paths", + ), + ], + ), + ], + ) + + unfiltered_compile_flags_feature = feature( + name = "unfiltered_compile_flags", + flag_sets = ([ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ], + flag_groups = [ + flag_group( + flags = ctx.attr.host_unfiltered_compile_flags, + ), + ], + ), + ] if ctx.attr.host_unfiltered_compile_flags else []), + ) + + determinism_feature = feature( + name = "determinism", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = [ + "-Wno-builtin-macro-redefined", + "-D__DATE__=\"redacted\"", + "-D__TIMESTAMP__=\"redacted\"", + "-D__TIME__=\"redacted\"", + ], + ), + ], + ), + ], + ) + + nologo_feature = feature( + name = "nologo", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_static_library, + ], + flag_groups = [flag_group(flags = ["/nologo"])], + ), + ], + ) + + supports_pic_feature = feature(name = "supports_pic", enabled = True) + + output_execpath_flags_feature = feature( + name = "output_execpath_flags", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["/OUT:%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + ], + ) + + default_link_flags_feature = feature( + name = "default_link_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["/MACHINE:X64"])], + ), + ], + ) + + if (ctx.attr.cpu == "local"): + hardening_feature = feature( + name = "hardening", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = [ + "-U_FORTIFY_SOURCE", + "-D_FORTIFY_SOURCE=1", + "-fstack-protector", + ], + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [flag_group(flags = ["-Wl,-z,relro,-z,now"])], + ), + flag_set( + actions = [ACTION_NAMES.cpp_link_executable], + flag_groups = [flag_group(flags = ["-pie", "-Wl,-z,relro,-z,now"])], + ), + ], + ) + elif (ctx.attr.cpu == "darwin"): + hardening_feature = feature( + name = "hardening", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = [ + "-U_FORTIFY_SOURCE", + "-D_FORTIFY_SOURCE=1", + "-fstack-protector", + ], + ), + ], + ), + flag_set( + actions = [ACTION_NAMES.cpp_link_executable], + flag_groups = [flag_group(flags = ["-pie"])], + ), + ], + ) + else: + hardening_feature = None + + supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) + + targets_windows_feature = feature( + name = "targets_windows", + enabled = True, + implies = ["copy_dynamic_libraries_to_binary"], + ) + + msvc_env_feature = feature( + name = "msvc_env", + env_sets = [ + env_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_static_library, + ], + env_entries = [ + env_entry(key = "PATH", value = ctx.attr.msvc_env_path), + env_entry( + key = "INCLUDE", + value = ctx.attr.msvc_env_include, + ), + env_entry(key = "LIB", value = ctx.attr.msvc_env_lib), + env_entry(key = "TMP", value = ctx.attr.msvc_env_tmp), + env_entry(key = "TEMP", value = ctx.attr.msvc_env_tmp), + ], + ), + ], + ) + + linker_subsystem_flag_feature = feature( + name = "linker_subsystem_flag", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["/SUBSYSTEM:CONSOLE"])], + ), + ], + ) + + dynamic_link_msvcrt_no_debug_feature = feature( + name = "dynamic_link_msvcrt_no_debug", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["/MD"])], + ), + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["/DEFAULTLIB:msvcrt.lib"])], + ), + ], + requires = [ + feature_set(features = ["fastbuild"]), + feature_set(features = ["opt"]), + ], + ) + + warnings_feature = feature( + name = "warnings", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = ["-Wall"] + ctx.attr.host_compiler_warnings, + ), + ], + ), + ], + ) + + dynamic_link_msvcrt_debug_feature = feature( + name = "dynamic_link_msvcrt_debug", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["/MDd"])], + ), + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["/DEFAULTLIB:msvcrtd.lib"])], + ), + ], + requires = [feature_set(features = ["dbg"])], + ) + + compiler_output_flags_feature = feature( + name = "compiler_output_flags", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.assemble], + flag_groups = [ + flag_group( + flag_groups = [ + flag_group( + flags = ["/Fo%{output_file}", "/Zi"], + expand_if_not_available = "output_preprocess_file", + ), + ], + expand_if_available = "output_file", + expand_if_not_available = "output_assembly_file", + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ], + flag_groups = [ + flag_group( + flag_groups = [ + flag_group( + flags = ["/Fo%{output_file}"], + expand_if_not_available = "output_preprocess_file", + ), + ], + expand_if_available = "output_file", + expand_if_not_available = "output_assembly_file", + ), + flag_group( + flag_groups = [ + flag_group( + flags = ["/Fa%{output_file}"], + expand_if_available = "output_assembly_file", + ), + ], + expand_if_available = "output_file", + ), + flag_group( + flag_groups = [ + flag_group( + flags = ["/P", "/Fi%{output_file}"], + expand_if_available = "output_preprocess_file", + ), + ], + expand_if_available = "output_file", + ), + ], + ), + ], + ) + + default_compile_flags_feature = feature( + name = "default_compile_flags", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.linkstamp_compile, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.lto_backend, + ACTION_NAMES.clif_match, + ], + flag_groups = [ + flag_group( + flags = [ + "/DCOMPILER_MSVC", + "/DNOMINMAX", + "/D_WIN32_WINNT=0x0600", + "/D_CRT_SECURE_NO_DEPRECATE", + "/D_CRT_SECURE_NO_WARNINGS", + "/D_SILENCE_STDEXT_HASH_DEPRECATION_WARNINGS", + "/bigobj", + "/Zm500", + "/J", + "/Gy", + "/GF", + "/EHsc", + "/wd4351", + "/wd4291", + "/wd4250", + "/wd4996", + ], + ), + ], + ), + ], + ) + + static_link_msvcrt_debug_feature = feature( + name = "static_link_msvcrt_debug", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["/MTd"])], + ), + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["/DEFAULTLIB:libcmtd.lib"])], + ), + ], + requires = [feature_set(features = ["dbg"])], + ) + + static_link_msvcrt_feature = feature(name = "static_link_msvcrt") + + if (ctx.attr.cpu == "darwin" or + ctx.attr.cpu == "local"): + dbg_feature = feature( + name = "dbg", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["-g"])], + ), + ], + implies = ["common"], + ) + elif (ctx.attr.cpu == "x64_windows"): + dbg_feature = feature( + name = "dbg", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["/Od", "/Z7", "/DDEBUG"])], + ), + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["/DEBUG:FULL", "/INCREMENTAL:NO"])], + ), + ], + implies = ["generate_pdb_file"], + ) + else: + dbg_feature = None + + undefined_dynamic_feature = feature( + name = "undefined-dynamic", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [flag_group(flags = ["-undefined", "dynamic_lookup"])], + ), + ], + ) + + parse_showincludes_feature = feature( + name = "parse_showincludes", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_header_parsing, + ], + flag_groups = [flag_group(flags = ["/showIncludes"])], + ), + ], + ) + + linker_param_file_feature = feature( + name = "linker_param_file", + flag_sets = [ + flag_set( + actions = all_link_actions + + [ACTION_NAMES.cpp_link_static_library], + flag_groups = [ + flag_group( + flags = ["@%{linker_param_file}"], + expand_if_available = "linker_param_file", + ), + ], + ), + ], + ) + + static_link_msvcrt_no_debug_feature = feature( + name = "static_link_msvcrt_no_debug", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["/MT"])], + ), + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["/DEFAULTLIB:libcmt.lib"])], + ), + ], + requires = [ + feature_set(features = ["fastbuild"]), + feature_set(features = ["opt"]), + ], + ) + + supports_interface_shared_libraries_feature = feature( + name = "supports_interface_shared_libraries", + enabled = True, + ) + + disable_assertions_feature = feature( + name = "disable-assertions", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["-DNDEBUG"])], + ), + ], + ) + + if (ctx.attr.cpu == "x64_windows"): + fastbuild_feature = feature( + name = "fastbuild", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["/Od", "/Z7", "/DDEBUG"])], + ), + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group(flags = ["/DEBUG:FASTLINK", "/INCREMENTAL:NO"]), + ], + ), + ], + implies = ["generate_pdb_file"], + ) + elif (ctx.attr.cpu == "darwin" or + ctx.attr.cpu == "local"): + fastbuild_feature = feature(name = "fastbuild", implies = ["common"]) + else: + fastbuild_feature = None + + user_compile_flags_feature = feature( + name = "user_compile_flags", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ], + flag_groups = [ + flag_group( + flags = ["%{user_compile_flags}"], + iterate_over = "user_compile_flags", + expand_if_available = "user_compile_flags", + ), + ], + ), + ], + ) + + compiler_input_flags_feature = feature( + name = "compiler_input_flags", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ], + flag_groups = [ + flag_group( + flags = ["/c", "%{source_file}"], + expand_if_available = "source_file", + ), + ], + ), + ], + ) + + no_legacy_features_feature = feature(name = "no_legacy_features") + + archiver_flags_feature = feature( + name = "archiver_flags", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.cpp_link_static_library], + flag_groups = [ + flag_group( + flags = ["/OUT:%{output_execpath}"], + expand_if_available = "output_execpath", + ), + ], + ), + ], + ) + + redirector_feature = feature( + name = "redirector", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ], + flag_groups = [ + flag_group( + flags = [ + "-B", + "external/local_config_cuda/crosstool/windows/msvc_wrapper_for_nvcc.py", + ], + ), + ], + ), + ], + ) + + linker_bin_path_feature = feature( + name = "linker-bin-path", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["-B" + ctx.attr.linker_bin_path])], + ), + ], + ) + + if (ctx.attr.cpu == "local"): + opt_feature = feature( + name = "opt", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = ["-g0", "-O2", "-ffunction-sections", "-fdata-sections"], + ), + ], + ), + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [flag_group(flags = ["-Wl,--gc-sections"])], + ), + ], + implies = ["common", "disable-assertions"], + ) + elif (ctx.attr.cpu == "darwin"): + opt_feature = feature( + name = "opt", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [ + flag_group( + flags = ["-g0", "-O2", "-ffunction-sections", "-fdata-sections"], + ), + ], + ), + ], + implies = ["common", "disable-assertions"], + ) + elif (ctx.attr.cpu == "x64_windows"): + opt_feature = feature( + name = "opt", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["/O2", "/DNDEBUG"])], + ), + ], + ) + else: + opt_feature = None + + include_paths_feature = feature( + name = "include_paths", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ], + flag_groups = [ + flag_group( + flags = ["/I%{quote_include_paths}"], + iterate_over = "quote_include_paths", + ), + flag_group( + flags = ["/I%{include_paths}"], + iterate_over = "include_paths", + ), + flag_group( + flags = ["/I%{system_include_paths}"], + iterate_over = "system_include_paths", + ), + ], + ), + ], + ) + + shared_flag_feature = feature( + name = "shared_flag", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [flag_group(flags = ["/DLL"])], + ), + ], + ) + + windows_export_all_symbols_feature = feature(name = "windows_export_all_symbols") + + frame_pointer_feature = feature( + name = "frame-pointer", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["-fno-omit-frame-pointer"])], + ), + ], + ) + + build_id_feature = feature( + name = "build-id", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["-Wl,--build-id=md5", "-Wl,--hash-style=gnu"], + ), + ], + ), + ], + ) + + sysroot_feature = feature( + name = "sysroot", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = ["--sysroot=%{sysroot}"], + iterate_over = "sysroot", + expand_if_available = "sysroot", + ), + ], + ), + ], + ) + + cuda_path_feature = feature( + name = "cuda_path", + enabled = True, + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.assemble, + ACTION_NAMES.preprocess_assemble, + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_header_parsing, + ACTION_NAMES.cpp_module_compile, + ACTION_NAMES.cpp_module_codegen, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = ["--cuda-path=" + ctx.attr.cuda_path], + ), + ], + ), + ], + ) + + def_file_feature = feature( + name = "def_file", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["/DEF:%{def_file_path}", "/ignore:4070"], + expand_if_available = "def_file_path", + ), + ], + ), + ], + ) + + if (ctx.attr.cpu == "darwin"): + stdlib_feature = feature( + name = "stdlib", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["-lc++"])], + ), + ], + ) + elif (ctx.attr.cpu == "local"): + stdlib_feature = feature( + name = "stdlib", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [flag_group(flags = ["-lstdc++"])], + ), + ], + ) + else: + stdlib_feature = None + + no_stripping_feature = feature(name = "no_stripping") + + alwayslink_feature = feature( + name = "alwayslink", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ACTION_NAMES.cpp_link_executable, + ], + flag_groups = [flag_group(flags = ["-Wl,-no-as-needed"])], + ), + ], + ) + + input_param_flags_feature = feature( + name = "input_param_flags", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = ["/IMPLIB:%{interface_library_output_path}"], + expand_if_available = "interface_library_output_path", + ), + ], + ), + flag_set( + actions = all_link_actions + + [ACTION_NAMES.cpp_link_static_library], + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link", + flag_groups = [ + flag_group( + iterate_over = "libraries_to_link.object_files", + flag_groups = [flag_group(flags = ["%{libraries_to_link.object_files}"])], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file_group", + ), + ), + flag_group( + flag_groups = [flag_group(flags = ["%{libraries_to_link.name}"])], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "object_file", + ), + ), + flag_group( + flag_groups = [flag_group(flags = ["%{libraries_to_link.name}"])], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "interface_library", + ), + ), + flag_group( + flag_groups = [ + flag_group( + flags = ["%{libraries_to_link.name}"], + expand_if_false = "libraries_to_link.is_whole_archive", + ), + flag_group( + flags = ["/WHOLEARCHIVE:%{libraries_to_link.name}"], + expand_if_true = "libraries_to_link.is_whole_archive", + ), + ], + expand_if_equal = variable_with_value( + name = "libraries_to_link.type", + value = "static_library", + ), + ), + ], + expand_if_available = "libraries_to_link", + ), + ], + ), + ], + ) + + if (ctx.attr.cpu == "local"): + no_canonical_prefixes_feature = feature( + name = "no-canonical-prefixes", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [ + flag_group( + flags = [ + "-no-canonical-prefixes", + ] + ctx.attr.extra_no_canonical_prefixes_flags, + ), + ], + ), + ], + ) + elif (ctx.attr.cpu == "darwin"): + no_canonical_prefixes_feature = feature( + name = "no-canonical-prefixes", + flag_sets = [ + flag_set( + actions = [ + ACTION_NAMES.c_compile, + ACTION_NAMES.cpp_compile, + ACTION_NAMES.cpp_link_executable, + ACTION_NAMES.cpp_link_dynamic_library, + ACTION_NAMES.cpp_link_nodeps_dynamic_library, + ], + flag_groups = [flag_group(flags = ["-no-canonical-prefixes"])], + ), + ], + ) + else: + no_canonical_prefixes_feature = None + + has_configured_linker_path_feature = feature(name = "has_configured_linker_path") + + copy_dynamic_libraries_to_binary_feature = feature(name = "copy_dynamic_libraries_to_binary") + + user_link_flags_feature = feature( + name = "user_link_flags", + flag_sets = [ + flag_set( + actions = all_link_actions, + flag_groups = [ + flag_group( + flags = ["%{user_link_flags}"], + iterate_over = "user_link_flags", + expand_if_available = "user_link_flags", + ), + ], + ), + ], + ) + + cpp11_feature = feature( + name = "c++11", + flag_sets = [ + flag_set( + actions = [ACTION_NAMES.cpp_compile], + flag_groups = [flag_group(flags = ["-std=c++11"])], + ), + ], + ) + + if (ctx.attr.cpu == "local"): + common_feature = feature( + name = "common", + implies = [ + "stdlib", + "c++11", + "determinism", + "alwayslink", + "hardening", + "warnings", + "frame-pointer", + "build-id", + "no-canonical-prefixes", + "linker-bin-path", + ], + ) + elif (ctx.attr.cpu == "darwin"): + common_feature = feature( + name = "common", + implies = [ + "stdlib", + "c++11", + "determinism", + "hardening", + "warnings", + "frame-pointer", + "no-canonical-prefixes", + "linker-bin-path", + "undefined-dynamic", + ], + ) + else: + common_feature = None + + if (ctx.attr.cpu == "local"): + features = [ + cpp11_feature, + stdlib_feature, + determinism_feature, + alwayslink_feature, + pic_feature, + hardening_feature, + warnings_feature, + frame_pointer_feature, + build_id_feature, + no_canonical_prefixes_feature, + disable_assertions_feature, + linker_bin_path_feature, + common_feature, + opt_feature, + fastbuild_feature, + dbg_feature, + supports_dynamic_linker_feature, + supports_pic_feature, + ] + if ctx.attr.cuda_path: + features.append(cuda_path_feature) + elif (ctx.attr.cpu == "darwin"): + features = [ + cpp11_feature, + stdlib_feature, + determinism_feature, + pic_feature, + hardening_feature, + warnings_feature, + frame_pointer_feature, + no_canonical_prefixes_feature, + disable_assertions_feature, + linker_bin_path_feature, + undefined_dynamic_feature, + common_feature, + opt_feature, + fastbuild_feature, + dbg_feature, + supports_dynamic_linker_feature, + supports_pic_feature, + ] + elif (ctx.attr.cpu == "x64_windows"): + features = [ + no_legacy_features_feature, + redirector_feature, + nologo_feature, + has_configured_linker_path_feature, + no_stripping_feature, + targets_windows_feature, + copy_dynamic_libraries_to_binary_feature, + default_compile_flags_feature, + msvc_env_feature, + include_paths_feature, + preprocessor_defines_feature, + parse_showincludes_feature, + generate_pdb_file_feature, + shared_flag_feature, + linkstamps_feature, + output_execpath_flags_feature, + archiver_flags_feature, + input_param_flags_feature, + linker_subsystem_flag_feature, + user_link_flags_feature, + default_link_flags_feature, + linker_param_file_feature, + static_link_msvcrt_feature, + static_link_msvcrt_no_debug_feature, + dynamic_link_msvcrt_no_debug_feature, + static_link_msvcrt_debug_feature, + dynamic_link_msvcrt_debug_feature, + dbg_feature, + fastbuild_feature, + opt_feature, + user_compile_flags_feature, + sysroot_feature, + unfiltered_compile_flags_feature, + compiler_output_flags_feature, + compiler_input_flags_feature, + def_file_feature, + windows_export_all_symbols_feature, + no_windows_export_all_symbols_feature, + supports_dynamic_linker_feature, + supports_interface_shared_libraries_feature, + ] + else: + fail("Unreachable") + + cxx_builtin_include_directories = ctx.attr.builtin_include_directories + + if (ctx.attr.cpu == "x64_windows"): + tool_paths = [ + tool_path(name = "ar", path = ctx.attr.msvc_lib_path), + tool_path(name = "ml", path = ctx.attr.msvc_ml_path), + tool_path(name = "cpp", path = ctx.attr.msvc_cl_path), + tool_path(name = "gcc", path = ctx.attr.msvc_cl_path), + tool_path(name = "gcov", path = "wrapper/bin/msvc_nop.bat"), + tool_path(name = "ld", path = ctx.attr.msvc_link_path), + tool_path(name = "nm", path = "wrapper/bin/msvc_nop.bat"), + tool_path( + name = "objcopy", + path = "wrapper/bin/msvc_nop.bat", + ), + tool_path( + name = "objdump", + path = "wrapper/bin/msvc_nop.bat", + ), + tool_path( + name = "strip", + path = "wrapper/bin/msvc_nop.bat", + ), + ] + elif (ctx.attr.cpu == "local"): + tool_paths = [ + tool_path(name = "gcc", path = ctx.attr.host_compiler_path), + tool_path(name = "ar", path = ctx.attr.host_compiler_prefix + "/ar"), + tool_path(name = "compat-ld", path = ctx.attr.host_compiler_prefix + "/ld"), + tool_path(name = "cpp", path = ctx.attr.host_compiler_prefix + "/cpp"), + tool_path(name = "dwp", path = ctx.attr.host_compiler_prefix + "/dwp"), + tool_path(name = "gcov", path = ctx.attr.host_compiler_prefix + "/gcov"), + tool_path(name = "ld", path = ctx.attr.host_compiler_prefix + "/ld"), + tool_path(name = "nm", path = ctx.attr.host_compiler_prefix + "/nm"), + tool_path(name = "objcopy", path = ctx.attr.host_compiler_prefix + "/objcopy"), + tool_path(name = "objdump", path = ctx.attr.host_compiler_prefix + "/objdump"), + tool_path(name = "strip", path = ctx.attr.host_compiler_prefix + "/strip"), + ] + elif (ctx.attr.cpu == "darwin"): + tool_paths = [ + tool_path(name = "gcc", path = ctx.attr.host_compiler_path), + tool_path(name = "ar", path = ctx.attr.host_compiler_prefix + "/libtool"), + tool_path(name = "compat-ld", path = ctx.attr.host_compiler_prefix + "/ld"), + tool_path(name = "cpp", path = ctx.attr.host_compiler_prefix + "/cpp"), + tool_path(name = "dwp", path = ctx.attr.host_compiler_prefix + "/dwp"), + tool_path(name = "gcov", path = ctx.attr.host_compiler_prefix + "/gcov"), + tool_path(name = "ld", path = ctx.attr.host_compiler_prefix + "/ld"), + tool_path(name = "nm", path = ctx.attr.host_compiler_prefix + "/nm"), + tool_path(name = "objcopy", path = ctx.attr.host_compiler_prefix + "/objcopy"), + tool_path(name = "objdump", path = ctx.attr.host_compiler_prefix + "/objdump"), + tool_path(name = "strip", path = ctx.attr.host_compiler_prefix + "/strip"), + ] + else: + fail("Unreachable") + + out = ctx.actions.declare_file(ctx.label.name) + ctx.actions.write(out, "Fake executable") + return [ + cc_common.create_cc_toolchain_config_info( + ctx = ctx, + features = features, + action_configs = action_configs, + artifact_name_patterns = [], + cxx_builtin_include_directories = cxx_builtin_include_directories, + toolchain_identifier = toolchain_identifier, + host_system_name = host_system_name, + target_system_name = target_system_name, + target_cpu = target_cpu, + target_libc = target_libc, + compiler = compiler, + abi_version = abi_version, + abi_libc_version = abi_libc_version, + tool_paths = tool_paths, + make_variables = [], + builtin_sysroot = builtin_sysroot, + cc_target_os = cc_target_os, + ), + DefaultInfo( + executable = out, + ), + ] + +cc_toolchain_config = rule( + implementation = _impl, + attrs = { + "cpu": attr.string(mandatory = True, values = ["darwin", "local", "x64_windows"]), + "builtin_include_directories": attr.string_list(), + "extra_no_canonical_prefixes_flags": attr.string_list(), + "host_compiler_path": attr.string(), + "host_compiler_prefix": attr.string(), + "host_compiler_warnings": attr.string_list(), + "host_unfiltered_compile_flags": attr.string_list(), + "linker_bin_path": attr.string(), + "builtin_sysroot": attr.string(), + "cuda_path": attr.string(), + "msvc_cl_path": attr.string(default = "msvc_not_used"), + "msvc_env_include": attr.string(default = "msvc_not_used"), + "msvc_env_lib": attr.string(default = "msvc_not_used"), + "msvc_env_path": attr.string(default = "msvc_not_used"), + "msvc_env_tmp": attr.string(default = "msvc_not_used"), + "msvc_lib_path": attr.string(default = "msvc_not_used"), + "msvc_link_path": attr.string(default = "msvc_not_used"), + "msvc_ml_path": attr.string(default = "msvc_not_used"), + }, + provides = [CcToolchainConfigInfo], + executable = True, +) diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/clang/bin/crosstool_wrapper_driver_is_not_gcc b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/clang/bin/crosstool_wrapper_driver_is_not_gcc new file mode 100755 index 00000000000..07c85a38229 --- /dev/null +++ b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/clang/bin/crosstool_wrapper_driver_is_not_gcc @@ -0,0 +1,289 @@ +#!/usr/bin/env python +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Crosstool wrapper for compiling CUDA programs. + +SYNOPSIS: + crosstool_wrapper_is_not_gcc [options passed in by cc_library() + or cc_binary() rule] + +DESCRIPTION: + This script is expected to be called by the cc_library() or cc_binary() bazel + rules. When the option "-x cuda" is present in the list of arguments passed + to this script, it invokes the nvcc CUDA compiler. Most arguments are passed + as is as a string to --compiler-options of nvcc. When "-x cuda" is not + present, this wrapper invokes hybrid_driver_is_not_gcc with the input + arguments as is. + +NOTES: + Changes to the contents of this file must be propagated from + //third_party/gpus/crosstool/crosstool_wrapper_is_not_gcc to + //third_party/gpus/crosstool/v*/*/clang/bin/crosstool_wrapper_is_not_gcc +""" + +from __future__ import print_function + +__author__ = 'keveman@google.com (Manjunath Kudlur)' + +from argparse import ArgumentParser +import os +import subprocess +import re +import sys +import pipes + +# Template values set by cuda_autoconf. +CPU_COMPILER = ('/dt7/usr/bin/gcc') +GCC_HOST_COMPILER_PATH = ('/dt7/usr/bin/gcc') + +NVCC_PATH = '/usr/local/cuda-11.0/bin/nvcc' +PREFIX_DIR = os.path.dirname(GCC_HOST_COMPILER_PATH) +NVCC_VERSION = '10.1' + +def Log(s): + print('gpus/crosstool: {0}'.format(s)) + + +def GetOptionValue(argv, option): + """Extract the list of values for option from the argv list. + + Args: + argv: A list of strings, possibly the argv passed to main(). + option: The option whose value to extract, with the leading '-'. + + Returns: + A list of values, either directly following the option, + (eg., -opt val1 val2) or values collected from multiple occurrences of + the option (eg., -opt val1 -opt val2). + """ + + parser = ArgumentParser() + parser.add_argument(option, nargs='*', action='append') + option = option.lstrip('-').replace('-', '_') + args, _ = parser.parse_known_args(argv) + if not args or not vars(args)[option]: + return [] + else: + return sum(vars(args)[option], []) + + +def GetHostCompilerOptions(argv): + """Collect the -isystem, -iquote, and --sysroot option values from argv. + + Args: + argv: A list of strings, possibly the argv passed to main(). + + Returns: + The string that can be used as the --compiler-options to nvcc. + """ + + parser = ArgumentParser() + parser.add_argument('-isystem', nargs='*', action='append') + parser.add_argument('-iquote', nargs='*', action='append') + parser.add_argument('--sysroot', nargs=1) + parser.add_argument('-g', nargs='*', action='append') + parser.add_argument('-fno-canonical-system-headers', action='store_true') + parser.add_argument('-no-canonical-prefixes', action='store_true') + + args, _ = parser.parse_known_args(argv) + + opts = '' + + if args.isystem: + opts += ' -isystem ' + ' -isystem '.join(sum(args.isystem, [])) + if args.iquote: + opts += ' -iquote ' + ' -iquote '.join(sum(args.iquote, [])) + if args.g: + opts += ' -g' + ' -g'.join(sum(args.g, [])) + if args.fno_canonical_system_headers: + opts += ' -fno-canonical-system-headers' + if args.no_canonical_prefixes: + opts += ' -no-canonical-prefixes' + if args.sysroot: + opts += ' --sysroot ' + args.sysroot[0] + + return opts + +def _update_options(nvcc_options): + if NVCC_VERSION in ("7.0",): + return nvcc_options + + update_options = { "relaxed-constexpr" : "expt-relaxed-constexpr" } + return [ update_options[opt] if opt in update_options else opt + for opt in nvcc_options ] + +def GetNvccOptions(argv): + """Collect the -nvcc_options values from argv. + + Args: + argv: A list of strings, possibly the argv passed to main(). + + Returns: + The string that can be passed directly to nvcc. + """ + + parser = ArgumentParser() + parser.add_argument('-nvcc_options', nargs='*', action='append') + + args, _ = parser.parse_known_args(argv) + + if args.nvcc_options: + options = _update_options(sum(args.nvcc_options, [])) + return ' '.join(['--'+a for a in options]) + return '' + +def system(cmd): + """Invokes cmd with os.system(). + + Args: + cmd: The command. + + Returns: + The exit code if the process exited with exit() or -signal + if the process was terminated by a signal. + """ + retv = os.system(cmd) + if os.WIFEXITED(retv): + return os.WEXITSTATUS(retv) + else: + return -os.WTERMSIG(retv) + +def InvokeNvcc(argv, log=False): + """Call nvcc with arguments assembled from argv. + + Args: + argv: A list of strings, possibly the argv passed to main(). + log: True if logging is requested. + + Returns: + The return value of calling system('nvcc ' + args) + """ + + host_compiler_options = GetHostCompilerOptions(argv) + nvcc_compiler_options = GetNvccOptions(argv) + opt_option = GetOptionValue(argv, '-O') + m_options = GetOptionValue(argv, '-m') + m_options = ''.join([' -m' + m for m in m_options if m in ['32', '64']]) + include_options = GetOptionValue(argv, '-I') + out_file = GetOptionValue(argv, '-o') + depfiles = GetOptionValue(argv, '-MF') + defines = GetOptionValue(argv, '-D') + defines = ''.join([' -D' + define for define in defines]) + undefines = GetOptionValue(argv, '-U') + undefines = ''.join([' -U' + define for define in undefines]) + std_options = GetOptionValue(argv, '-std') + # Supported -std flags as of CUDA 9.0. Only keep last to mimic gcc/clang. + nvcc_allowed_std_options = ["c++03", "c++11", "c++14"] + std_options = ''.join([' -std=' + define + for define in std_options if define in nvcc_allowed_std_options][-1:]) + fatbin_options = ''.join([' --fatbin-options=' + option + for option in GetOptionValue(argv, '-Xcuda-fatbinary')]) + + # The list of source files get passed after the -c option. I don't know of + # any other reliable way to just get the list of source files to be compiled. + src_files = GetOptionValue(argv, '-c') + + # Pass -w through from host to nvcc, but don't do anything fancier with + # warnings-related flags, since they're not necessarily the same across + # compilers. + warning_options = ' -w' if '-w' in argv else '' + + if len(src_files) == 0: + return 1 + if len(out_file) != 1: + return 1 + + opt = (' -O2' if (len(opt_option) > 0 and int(opt_option[0]) > 0) + else ' -g') + + includes = (' -I ' + ' -I '.join(include_options) + if len(include_options) > 0 + else '') + + # Unfortunately, there are other options that have -c prefix too. + # So allowing only those look like C/C++ files. + src_files = [f for f in src_files if + re.search('\.cpp$|\.cc$|\.c$|\.cxx$|\.C$', f)] + srcs = ' '.join(src_files) + out = ' -o ' + out_file[0] + + nvccopts = '-D_FORCE_INLINES ' + for capability in GetOptionValue(argv, "--cuda-gpu-arch"): + capability = capability[len('sm_'):] + nvccopts += r'-gencode=arch=compute_%s,\"code=sm_%s\" ' % (capability, + capability) + for capability in GetOptionValue(argv, '--cuda-include-ptx'): + capability = capability[len('sm_'):] + nvccopts += r'-gencode=arch=compute_%s,\"code=compute_%s\" ' % (capability, + capability) + nvccopts += nvcc_compiler_options + nvccopts += undefines + nvccopts += defines + nvccopts += std_options + nvccopts += m_options + nvccopts += warning_options + nvccopts += fatbin_options + + if depfiles: + # Generate the dependency file + depfile = depfiles[0] + cmd = (NVCC_PATH + ' ' + nvccopts + + ' --compiler-options "' + host_compiler_options + '"' + + ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH + + ' -I .' + + ' -x cu ' + opt + includes + ' ' + srcs + ' -M -o ' + depfile) + if log: Log(cmd) + exit_status = system(cmd) + if exit_status != 0: + return exit_status + + cmd = (NVCC_PATH + ' ' + nvccopts + + ' --compiler-options "' + host_compiler_options + ' -fPIC"' + + ' --compiler-bindir=' + GCC_HOST_COMPILER_PATH + + ' -I .' + + ' -x cu ' + opt + includes + ' -c ' + srcs + out) + + # TODO(zhengxq): for some reason, 'gcc' needs this help to find 'as'. + # Need to investigate and fix. + cmd = 'PATH=' + PREFIX_DIR + ':$PATH ' + cmd + if log: Log(cmd) + return system(cmd) + + +def main(): + parser = ArgumentParser() + parser.add_argument('-x', nargs=1) + parser.add_argument('--cuda_log', action='store_true') + args, leftover = parser.parse_known_args(sys.argv[1:]) + + if args.x and args.x[0] == 'cuda': + if args.cuda_log: Log('-x cuda') + leftover = [pipes.quote(s) for s in leftover] + if args.cuda_log: Log('using nvcc') + return InvokeNvcc(leftover, log=args.cuda_log) + + # Strip our flags before passing through to the CPU compiler for files which + # are not -x cuda. We can't just pass 'leftover' because it also strips -x. + # We not only want to pass -x to the CPU compiler, but also keep it in its + # relative location in the argv list (the compiler is actually sensitive to + # this). + cpu_compiler_flags = [flag for flag in sys.argv[1:] + if not flag.startswith(('--cuda_log'))] + + return subprocess.call([CPU_COMPILER] + cpu_compiler_flags) + +if __name__ == '__main__': + sys.exit(main()) From 14ca6184d13822192c2126fbc2ebeaf9d87dcec7 Mon Sep 17 00:00:00 2001 From: Vo Van Nghia Date: Mon, 10 Aug 2020 22:48:53 +0700 Subject: [PATCH 0742/1017] Gcs refactor part 2 --- .../filesystem/plugins/gcs/gcs_filesystem.cc | 152 +++++++++++++++--- 1 file changed, 126 insertions(+), 26 deletions(-) diff --git a/tensorflow/c/experimental/filesystem/plugins/gcs/gcs_filesystem.cc b/tensorflow/c/experimental/filesystem/plugins/gcs/gcs_filesystem.cc index d170e51e3b1..f16f55f251f 100644 --- a/tensorflow/c/experimental/filesystem/plugins/gcs/gcs_filesystem.cc +++ b/tensorflow/c/experimental/filesystem/plugins/gcs/gcs_filesystem.cc @@ -877,32 +877,6 @@ void DeleteDir(const TF_Filesystem* filesystem, const char* path, TF_SetStatus(status, TF_OK, ""); } -// TODO(vnvo2409): `RewriteObjectBlocking` will set `status` to `TF_NOT_FOUND` -// if the object does not exist. In that case, we will have to check if the -// `src` is a directory or not to set the correspondent `status` (i.e -// `TF_NOT_FOUND` if path `src` does not exist, `TF_FAILED_PRECONDITION` if -// path `src` is a directory). -void RenameFile(const TF_Filesystem* filesystem, const char* src, - const char* dst, TF_Status* status) { - std::string bucket_src, object_src; - ParseGCSPath(src, false, &bucket_src, &object_src, status); - if (TF_GetCode(status) != TF_OK) return; - - std::string bucket_dst, object_dst; - ParseGCSPath(dst, false, &bucket_dst, &object_dst, status); - if (TF_GetCode(status) != TF_OK) return; - - auto gcs_file = static_cast(filesystem->plugin_filesystem); - auto metadata = gcs_file->gcs_client.RewriteObjectBlocking( - bucket_src, object_src, bucket_dst, object_dst); - if (!metadata) { - TF_SetStatusFromGCSStatus(metadata.status(), status); - return; - } - auto gcs_status = gcs_file->gcs_client.DeleteObject(bucket_src, object_src); - TF_SetStatusFromGCSStatus(gcs_status, status); -} - void CopyFile(const TF_Filesystem* filesystem, const char* src, const char* dst, TF_Status* status) { std::string bucket_src, object_src; @@ -954,6 +928,100 @@ bool IsDirectory(const TF_Filesystem* filesystem, const char* path, return false; } +static void RenameObject(const TF_Filesystem* filesystem, + const std::string& src, const std::string& dst, + TF_Status* status) { + std::string bucket_src, object_src; + ParseGCSPath(src, false, &bucket_src, &object_src, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string bucket_dst, object_dst; + ParseGCSPath(dst, false, &bucket_dst, &object_dst, status); + if (TF_GetCode(status) != TF_OK) return; + + auto gcs_file = static_cast(filesystem->plugin_filesystem); + auto metadata = gcs_file->gcs_client.RewriteObjectBlocking( + bucket_src, object_src, bucket_dst, object_dst); + TF_SetStatusFromGCSStatus(metadata.status(), status); + if (TF_GetCode(status) != TF_OK) return; + + ClearFileCaches(gcs_file, dst); + DeleteFile(filesystem, src.c_str(), status); +} + +void RenameFile(const TF_Filesystem* filesystem, const char* src, + const char* dst, TF_Status* status) { + if (!IsDirectory(filesystem, src, status)) { + if (TF_GetCode(status) == TF_FAILED_PRECONDITION) + RenameObject(filesystem, src, dst, status); + return; + } + + auto gcs_file = static_cast(filesystem->plugin_filesystem); + std::vector childrens = + GetChildrenBounded(gcs_file, src, UINT64_MAX, true, true, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string src_dir = src; + std::string dst_dir = dst; + MaybeAppendSlash(&src_dir); + MaybeAppendSlash(&dst_dir); + for (const std::string& children : childrens) { + RenameObject(filesystem, src_dir + children, dst_dir + children, status); + if (TF_GetCode(status) != TF_OK) return; + } + TF_SetStatus(status, TF_OK, ""); +} + +void DeleteRecursively(const TF_Filesystem* filesystem, const char* path, + uint64_t* undeleted_files, uint64_t* undeleted_dirs, + TF_Status* status) { + if (!undeleted_files || !undeleted_dirs) + return TF_SetStatus( + status, TF_INTERNAL, + "'undeleted_files' and 'undeleted_dirs' cannot be nullptr."); + *undeleted_files = 0; + *undeleted_dirs = 0; + if (!IsDirectory(filesystem, path, status)) { + *undeleted_dirs = 1; + return; + } + auto gcs_file = static_cast(filesystem->plugin_filesystem); + std::vector childrens = + GetChildrenBounded(gcs_file, path, UINT64_MAX, true, true, status); + if (TF_GetCode(status) != TF_OK) return; + + std::string dir = path; + MaybeAppendSlash(&dir); + for (const std::string& children : childrens) { + const std::string& full_path = dir + children; + DeleteFile(filesystem, full_path.c_str(), status); + if (TF_GetCode(status) != TF_OK) { + if (IsDirectory(filesystem, full_path.c_str(), status)) + // The object is a directory marker. + (*undeleted_dirs)++; + else + (*undeleted_files)++; + } + } +} + +int GetChildren(const TF_Filesystem* filesystem, const char* path, + char*** entries, TF_Status* status) { + auto gcs_file = static_cast(filesystem->plugin_filesystem); + std::vector childrens = + GetChildrenBounded(gcs_file, path, UINT64_MAX, false, false, status); + if (TF_GetCode(status) != TF_OK) return -1; + + int num_entries = childrens.size(); + *entries = static_cast( + plugin_memory_allocate(num_entries * sizeof((*entries)[0]))); + for (int i = 0; i < num_entries; i++) + (*entries)[i] = strdup(childrens[i].c_str()); + TF_SetStatus(status, TF_OK, ""); + return num_entries; +} + void Stat(const TF_Filesystem* filesystem, const char* path, TF_FileStatistics* stats, TF_Status* status) { std::string bucket, object; @@ -991,6 +1059,17 @@ void Stat(const TF_Filesystem* filesystem, const char* path, } } +static char* TranslateName(const TF_Filesystem* filesystem, const char* uri) { + return strdup(uri); +} + +static void FlushCaches(const TF_Filesystem* filesystem) { + auto gcs_file = static_cast(filesystem->plugin_filesystem); + absl::ReaderMutexLock l(&gcs_file->block_cache_lock); + gcs_file->file_block_cache->Flush(); + gcs_file->stat_cache->Clear(); +} + } // namespace tf_gcs_filesystem static void ProvideFilesystemSupportFor(TF_FilesystemPluginOps* ops, @@ -1007,6 +1086,13 @@ static void ProvideFilesystemSupportFor(TF_FilesystemPluginOps* ops, plugin_memory_allocate(TF_WRITABLE_FILE_OPS_SIZE)); ops->writable_file_ops->cleanup = tf_writable_file::Cleanup; + ops->read_only_memory_region_ops = static_cast( + plugin_memory_allocate(TF_READ_ONLY_MEMORY_REGION_OPS_SIZE)); + ops->read_only_memory_region_ops->cleanup = + tf_read_only_memory_region::Cleanup; + ops->read_only_memory_region_ops->data = tf_read_only_memory_region::Data; + ops->read_only_memory_region_ops->length = tf_read_only_memory_region::Length; + ops->filesystem_ops = static_cast( plugin_memory_allocate(TF_FILESYSTEM_OPS_SIZE)); ops->filesystem_ops->init = tf_gcs_filesystem::Init; @@ -1016,6 +1102,20 @@ static void ProvideFilesystemSupportFor(TF_FilesystemPluginOps* ops, ops->filesystem_ops->new_writable_file = tf_gcs_filesystem::NewWritableFile; ops->filesystem_ops->new_appendable_file = tf_gcs_filesystem::NewAppendableFile; + ops->filesystem_ops->new_read_only_memory_region_from_file = + tf_gcs_filesystem::NewReadOnlyMemoryRegionFromFile; + ops->filesystem_ops->create_dir = tf_gcs_filesystem::CreateDir; + ops->filesystem_ops->delete_file = tf_gcs_filesystem::DeleteFile; + ops->filesystem_ops->delete_dir = tf_gcs_filesystem::DeleteDir; + ops->filesystem_ops->delete_recursively = + tf_gcs_filesystem::DeleteRecursively; + ops->filesystem_ops->copy_file = tf_gcs_filesystem::CopyFile; + ops->filesystem_ops->path_exists = tf_gcs_filesystem::PathExists; + ops->filesystem_ops->is_directory = tf_gcs_filesystem::IsDirectory; + ops->filesystem_ops->stat = tf_gcs_filesystem::Stat; + ops->filesystem_ops->get_children = tf_gcs_filesystem::GetChildren; + ops->filesystem_ops->translate_name = tf_gcs_filesystem::TranslateName; + ops->filesystem_ops->flush_caches = tf_gcs_filesystem::FlushCaches; } void TF_InitPlugin(TF_FilesystemPluginInfo* info) { From 91736566223baa1a74cd729a8b869f95f895da61 Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Mon, 10 Aug 2020 08:59:35 -0700 Subject: [PATCH 0743/1017] Add HLO RngBitGenerator This adds the XlaBuilder RngBitGenerator to the MHLO dialect. The op is currently represented very directly using int attribute for random algorithm and direct import/export. PiperOrigin-RevId: 325814134 Change-Id: I640e3141b8e16d1e186cd99848273b245d1c8b3e --- .../include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td | 16 +++++++++++++++- .../mlir-hlo/Dialect/mhlo/IR/hlo_ops_base.td | 13 +++++++++++++ .../compiler/mlir/xla/hlo_function_importer.cc | 7 +++++++ tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.cc | 11 +++++++++++ .../mlir/xla/tests/translate/export.mlir | 12 ++++++++++++ .../mlir/xla/tests/translate/import.hlotxt | 9 +++++++++ 6 files changed, 67 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td index b8b1926a0c9..d0abbe043ea 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.td @@ -1329,8 +1329,9 @@ def HLO_TorchIndexSelectOp : HLO_Op<"torch_index_select", [NoSideEffect]> { } //===----------------------------------------------------------------------===// -// MHLO RngUniform Operator. +// MHLO RNG Operators. //===----------------------------------------------------------------------===// + def HLO_RngUniformOp : HLO_Op<"rng_uniform", []>, BASE_HLO_RngUniformOp { let arguments = (ins HLO_PredIntOrFpTensor:$a, @@ -1355,6 +1356,19 @@ def HLO_RngNormalOp : HLO_Op<"rng_normal", []>, BASE_HLO_RngNormalOp { let hasCustomHLOConverter = 1; } +def HLO_RngBitGeneratorOp : HLO_Op<"rng_bit_generator", [NoSideEffect]>, BASE_HLO_RngBitGeneratorOp { + let arguments = (ins + // TODO(jpienaar): This could be an enum instead. + I32Attr:$rng_algorithm, + HLO_IntOrFpTensor:$initial_state + ); + + let results = (outs HLO_TensorOrTuple:$result); + + // TODO(jpienaar): This should not be needed. + let hasCustomHLOConverter = 1; +} + //===----------------------------------------------------------------------===// // MHLO Quantize Operator. //===----------------------------------------------------------------------===// diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops_base.td b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops_base.td index 7f9784d7f11..2f80545ad19 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops_base.td +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops_base.td @@ -316,6 +316,19 @@ class BASE_HLO_RealOp { }]; } +class BASE_HLO_RngBitGeneratorOp { + string summary = "Uniform random number generator operator"; + + string description = [{ + Returns an output with a given shape filled with uniform random bits using + the specified algorithm (or backend default) and returns an updated state + (with the same shape as initial state) and the generated random data. + + See + https://www.tensorflow.org/xla/operation_semantics#rngbitgenerator. + }]; +} + class BASE_HLO_RoundOp { string summary = "Round operator"; diff --git a/tensorflow/compiler/mlir/xla/hlo_function_importer.cc b/tensorflow/compiler/mlir/xla/hlo_function_importer.cc index d366a36c212..a63fc12c285 100644 --- a/tensorflow/compiler/mlir/xla/hlo_function_importer.cc +++ b/tensorflow/compiler/mlir/xla/hlo_function_importer.cc @@ -521,6 +521,13 @@ StatusOr HloFunctionImporter::ImportInstruction( RandomDistributionToString(instruction->random_distribution()))); } } + case HloOpcode::kRngBitGenerator: { + auto rng_op = Cast(instruction); + auto op = func_builder->create( + loc, result_type, + func_builder->getI32IntegerAttr(rng_op->algorithm()), operands[0]); + return op.getOperation(); + } case HloOpcode::kWhile: { auto op = func_builder->create( loc, operands[0].getType(), operands[0]); diff --git a/tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.cc b/tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.cc index e6d0b8f8dd8..5398cd70777 100644 --- a/tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.cc +++ b/tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.cc @@ -882,6 +882,17 @@ LogicalResult ExportXlaOp(ReturnOp op, OpLoweringContext ctx) { return failure(); } +LogicalResult ExportXlaOp(RngBitGeneratorOp op, OpLoweringContext ctx) { + auto& value_map = *ctx.values; + auto result = op.getResult(); + auto xla_arg_1 = value_map[*op.getODSOperands(0).begin()]; + auto xla_result = xla::RngBitGenerator( + static_cast(op.rng_algorithm().getSExtValue()), + Unwrap(xla_arg_1), xla::TypeToShape(result.getType()).tuple_shapes(1)); + value_map[result] = xla_result; + return mlir::success(); +} + LogicalResult ExportXlaOp(RngNormalOp op, OpLoweringContext ctx) { auto& value_map = *ctx.values; xla::XlaOp mu, sigma; diff --git a/tensorflow/compiler/mlir/xla/tests/translate/export.mlir b/tensorflow/compiler/mlir/xla/tests/translate/export.mlir index 9929bd85b43..316eda4c4aa 100644 --- a/tensorflow/compiler/mlir/xla/tests/translate/export.mlir +++ b/tensorflow/compiler/mlir/xla/tests/translate/export.mlir @@ -1087,3 +1087,15 @@ func @main(%arg: tensor<3x4xf32>, %token: !mhlo.token) -> !mhlo.token { } // CHECK-NOT: frontend_attributes + +// ----- + +// Checks exporting rng-bit-generator. + +// CHECK: HloModule +func @main(%arg: tensor<3xui64>) -> tuple, tensor<2x2xui32>> { +// CHECK: %[[ARG0:.*]] = u64[3] parameter(0) +// CHECK: ROOT %[[RESULT:.*]] = (u64[3], u32[2,2]) rng-bit-generator(u64[3] %[[ARG0]]), algorithm=rng_philox + %0 = "mhlo.rng_bit_generator"(%arg) {rng_algorithm = 2 : i32} : (tensor<3xui64>) -> tuple, tensor<2x2xui32>> + return %0 : tuple, tensor<2x2xui32>> +} diff --git a/tensorflow/compiler/mlir/xla/tests/translate/import.hlotxt b/tensorflow/compiler/mlir/xla/tests/translate/import.hlotxt index d89b1fa44e1..4d4e0213da8 100644 --- a/tensorflow/compiler/mlir/xla/tests/translate/import.hlotxt +++ b/tensorflow/compiler/mlir/xla/tests/translate/import.hlotxt @@ -1005,3 +1005,12 @@ add { // CHECK: "mhlo.not"(%[[ARG0]]) {name = "{{.*}}"} : (tensor<4xui16>) -> tensor<4xui16> ROOT %not.2 = u16[4] not(u16[4] %Arg_0.1) } + +// CHECK-LABEL: func @rngbitgen +// CHECK-SAME: (%[[ARG0:.*]]: tensor<3xui64>) +%rngbitgen (Arg_0.1: u64[3]) -> (u64[3], u32[2,2]) { + %Arg_0.1 = u64[3] parameter(0) + // CHECK: "mhlo.rng_bit_generator"(%[[ARG0]]) {rng_algorithm = 2 : i32} : (tensor<3xui64>) -> tuple, tensor<2x2xui32>> + ROOT %rng-bit-generator.2 = (u64[3], u32[2,2]) rng-bit-generator(u64[3] %Arg_0.1), algorithm=rng_philox +} + From 2c6f7e24dd003aaf2c66747ea40f1280b011ffa7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 09:05:54 -0700 Subject: [PATCH 0744/1017] Get namedtuple _make method from instance instead of class. It is possible that v0 is a _TupleWrapper object wrapping a namedtuple instead of a namedtuple itself. The class _TupleWrapper does not have the class method _make, but it should have the instance method _make which calls the namedtuple class method. Instances of a namedtuple also have the instance method _make which calls the relevant class method. This will allow the code to work for both cases. PiperOrigin-RevId: 325815594 Change-Id: I209f8ec5a8617f72183e4c12937b4429321e7b4f --- tensorflow/python/distribute/BUILD | 1 + tensorflow/python/distribute/distribute_utils.py | 4 ++-- tensorflow/python/distribute/distribute_utils_test.py | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/distribute/BUILD b/tensorflow/python/distribute/BUILD index f67f306706f..8497c4da8a7 100644 --- a/tensorflow/python/distribute/BUILD +++ b/tensorflow/python/distribute/BUILD @@ -1197,6 +1197,7 @@ distribute_py_test( "//tensorflow/python/eager:test", "//tensorflow/python/saved_model/model_utils:mode_keys", "@absl_py//absl/testing:parameterized", + "@wrapt", ], ) diff --git a/tensorflow/python/distribute/distribute_utils.py b/tensorflow/python/distribute/distribute_utils.py index 916ebafd8ac..62f03c60224 100644 --- a/tensorflow/python/distribute/distribute_utils.py +++ b/tensorflow/python/distribute/distribute_utils.py @@ -63,8 +63,8 @@ def regroup(values, wrap_class=values_lib.PerReplica, always_wrap=False): if hasattr(v0, "_fields"): # This tuple is in fact a namedtuple! Create a new namedtuple instance # and initialize it with the regrouped values: - assert hasattr(type(v0), "_make") - return type(v0)._make(regrouped_tuple) + assert hasattr(v0, "_make") + return v0._make(regrouped_tuple) else: return regrouped_tuple diff --git a/tensorflow/python/distribute/distribute_utils_test.py b/tensorflow/python/distribute/distribute_utils_test.py index f91cad2db47..22ea6264d07 100644 --- a/tensorflow/python/distribute/distribute_utils_test.py +++ b/tensorflow/python/distribute/distribute_utils_test.py @@ -21,6 +21,7 @@ from __future__ import print_function import collections from absl.testing import parameterized +import wrapt from tensorflow.python.distribute import combinations from tensorflow.python.distribute import distribute_utils @@ -211,6 +212,15 @@ class RegroupAndSelectDeviceTest(test.TestCase, parameterized.TestCase): distribute_utils.select_replica( device_id, merged_estimator_spec)) + def testWrappedNamedTuple(self): + Point = collections.namedtuple("Point", ["x", "y"]) + point1 = Point(x=0, y=2) + point2 = Point(x=1, y=3) + wrapped1 = wrapt.ObjectProxy(point1) + wrapped2 = wrapt.ObjectProxy(point2) + result = distribute_utils.regroup([wrapped1, wrapped2]) + self.assertEqual(result.x.values, (0, 1)) + self.assertEqual(result.y.values, (2, 3)) if __name__ == "__main__": test.main() From 4ed4c14e4cbb2a6439cce8a62bd20957a00a75d0 Mon Sep 17 00:00:00 2001 From: Stefano Galarraga Date: Mon, 10 Aug 2020 09:09:59 -0700 Subject: [PATCH 0745/1017] Add NNAPI Delegation support for requantization use cases (transforming a quantized tensor into another quantized one with different quantization parameters) by converting the operation into a dequantize-quantize pair. PiperOrigin-RevId: 325816400 Change-Id: I55f8726f0478e9795c667a9cf4eddda084ed95a7 --- RELEASE.md | 1 + .../delegates/nnapi/acceleration_test_list.cc | 2 ++ .../lite/delegates/nnapi/nnapi_delegate.cc | 24 +++++++++++++++---- .../delegates/nnapi/nnapi_delegate_kernel.h | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 191c18e5ddb..525db3cade8 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -128,6 +128,7 @@ * Support optional flags `inference_input_type` and `inference_output_type` for full integer quantized models. This allows users to modify the model input and output type to integer types (`tf.int8`, `tf.uint8`) instead of defaulting to float type (`tf.float32`). * Deprecate `Interpreter::UseNNAPI(bool)` C++ API * Prefer using `NnApiDelegate()` and related delegate configuration methods directly. + * Add NNAPI Delegation support for requantization use cases by converting the operation into a dequantize-quantize pair. * * `tf.random`: * diff --git a/tensorflow/lite/delegates/nnapi/acceleration_test_list.cc b/tensorflow/lite/delegates/nnapi/acceleration_test_list.cc index 5183ab4b062..43f9c1b0953 100644 --- a/tensorflow/lite/delegates/nnapi/acceleration_test_list.cc +++ b/tensorflow/lite/delegates/nnapi/acceleration_test_list.cc @@ -309,6 +309,8 @@ QuantizedLstmTest/BasicQuantizedLstmTest/29 # quantize_test QuantizeOpTest/UINT8,29 +QuantizeOpTest/UInt8UInt8.+,29 +QuantizeOpTest/Int8Int8.+,30 QuantizeOpTest/INT8,30 # rank diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc index 2a33d764949..122ddc043b2 100644 --- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc +++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc @@ -2436,13 +2436,20 @@ bool NNAPIDelegateKernel::Validate( "Input should be Float32.", &val_ctx); } break; case kTfLiteBuiltinQuantize: { - ExpectOpVersion(version, 1, &val_ctx); + ExpectMaxOpVersion(version, 2, &val_ctx); ExpectMinAndroidSdkVersion(android_sdk_version, kMinSdkVersionForNNAPI12, &val_ctx); const auto value_type = context->tensors[node->inputs->data[0]].type; - Expect(value_type == kTfLiteFloat32, + Expect(value_type == kTfLiteFloat32 || IsQuantized(value_type), NNAPIValidationFailureType::kUnsupportedInputType, - "Value should be Float32.", &val_ctx); + "Value should be quantized or Float32.", &val_ctx); + if (IsQuantized(value_type)) { + const auto quantization_params = + context->tensors[node->inputs->data[0]].params; + Expect(quantization_params.scale > 0.f, + NNAPIValidationFailureType::kUnsupportedQuantizationParameters, + "Quantization scale should be > 0.", &val_ctx); + } const auto output_type = context->tensors[node->outputs->data[0]].type; if (android_sdk_version < kMinSdkVersionForNNAPI13) { Expect(output_type == kTfLiteUInt8, @@ -3284,6 +3291,15 @@ TfLiteStatus NNAPIDelegateKernel::Map( *nn_op_type = ANEURALNETWORKS_LOG_SOFTMAX; } break; case kTfLiteBuiltinQuantize: { + auto input_index = mapping_args.node->inputs->data[0]; + // NNAPI doesn't support requantization cases but only quantizations + // from float. Dequantizing our input adding a Dequantize node before + // this one. + if (IsQuantized(mapping_args.context->tensors[input_index].type)) { + mapping_args.builder->AddDequantize(0, input_index, kTfLiteFloat32, + mapping_args.node_index); + } + *nn_op_type = ANEURALNETWORKS_QUANTIZE; } break; case kTfLiteBuiltinReduceAny: { @@ -4254,7 +4270,7 @@ TfLiteStatus NNAPIDelegateKernel::AddOpsAndTensors( int nn_op_type; TF_LITE_ENSURE_STATUS( Map(context, reg->builtin_code, reg->version, target_sdk_version_, - {context, &builder, node, &model_state_outputs_, + {context, &builder, node, node_index, &model_state_outputs_, &model_state_tfl_inputs_, &feedback_loops_, nnapi_errno}, &nn_op_type)); diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate_kernel.h b/tensorflow/lite/delegates/nnapi/nnapi_delegate_kernel.h index 72a64d2404a..36c1dd32efb 100644 --- a/tensorflow/lite/delegates/nnapi/nnapi_delegate_kernel.h +++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate_kernel.h @@ -111,6 +111,7 @@ struct NNAPIOpMappingArgs { TfLiteContext* context; NNAPIOpBuilder* builder; TfLiteNode* node; + int node_index; std::vector* model_state_outputs; std::vector* model_state_tfl_inputs; std::vector>* feedback_loops; From 04d770b603809a9c89974e77576af238d422fde5 Mon Sep 17 00:00:00 2001 From: Alexandre Passos Date: Mon, 10 Aug 2020 09:27:19 -0700 Subject: [PATCH 0746/1017] Causes some windows tests to fail with ``` FAILED: //tensorflow/python/kernel_tests:cwise_ops_test (Summary) C:/tmp/hgi62d3r/execroot/org_tensorflow/bazel-out/x64_windows-opt/testlogs/tensorflow/python/kernel_tests/cwise_ops_test/shard_41_of_50/test.log C:/tmp/hgi62d3r/execroot/org_tensorflow/bazel-out/x64_windows-opt/testlogs/tensorflow/python/kernel_tests/cwise_ops_test/shard_41_of_50/test_attempts/attempt_1.log C:/tmp/hgi62d3r/execroot/org_tensorflow/bazel-out/x64_windows-opt/testlogs/tensorflow/python/kernel_tests/cwise_ops_test/shard_41_of_50/test_attempts/attempt_2.log INFO: From Testing //tensorflow/python/kernel_tests:cwise_ops_test (shard 41 of 50): ==================== Test output for //tensorflow/python/kernel_tests:cwise_ops_test (shard 41 of 50): Running tests under Python 3.8.3: c:\Python38\python.exe [ RUN ] MathOpsOverloadTest.testOverload 2020-08-07 18:06:36.702914: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX AVX2 To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. 2020-08-07 18:06:36.731707: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x1f14f681f00 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-08-07 18:06:36.732038: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version INFO:tensorflow:time(__main__.MathOpsOverloadTest.testOverload): 0.09s I0807 18:06:36.788563 9040 test_util.py:1974] time(__main__.MathOpsOverloadTest.testOverload): 0.09s No pending test case: __main__.MathOpsOverloadTest.testOverload ====================================================================== ERROR: testOverload (__main__.MathOpsOverloadTest) (dtype=tf.bfloat16, np_func=, tf_func= at 0x000001F152FA13A0>) testOverload (__main__.MathOpsOverloadTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_6r__ku2y\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 883, in testOverload self._compareBinary(10, 5, dtype, np_func, tf_func) File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_6r__ku2y\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 843, in _compareBinary np_ans = np_func(x, y).astype(dtype.as_numpy_dtype) ValueError: No cast function available. ====================================================================== ERROR: testOverload (__main__.MathOpsOverloadTest) (dtype=tf.bfloat16, np_func=, tf_func= at 0x000001F152FA14C0>) testOverload (__main__.MathOpsOverloadTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_6r__ku2y\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 883, in testOverload self._compareBinary(10, 5, dtype, np_func, tf_func) File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_6r__ku2y\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 843, in _compareBinary np_ans = np_func(x, y).astype(dtype.as_numpy_dtype) ValueError: No cast function available. ---------------------------------------------------------------------- Ran 1 test in 0.093s FAILED (errors=2) ================================================================================ ==================== Test output for //tensorflow/python/kernel_tests:cwise_ops_test (shard 41 of 50): Running tests under Python 3.8.3: c:\Python38\python.exe [ RUN ] MathOpsOverloadTest.testOverload 2020-08-07 18:07:02.741015: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX AVX2 To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. 2020-08-07 18:07:02.768942: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x177405cd170 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-08-07 18:07:02.769253: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version INFO:tensorflow:time(__main__.MathOpsOverloadTest.testOverload): 0.08s I0807 18:07:02.821412 3188 test_util.py:1974] time(__main__.MathOpsOverloadTest.testOverload): 0.08s No pending test case: __main__.MathOpsOverloadTest.testOverload ====================================================================== ERROR: testOverload (__main__.MathOpsOverloadTest) (dtype=tf.bfloat16, np_func=, tf_func= at 0x0000017741FFF3A0>) testOverload (__main__.MathOpsOverloadTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_1vkgaz40\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 883, in testOverload self._compareBinary(10, 5, dtype, np_func, tf_func) File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_1vkgaz40\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 843, in _compareBinary np_ans = np_func(x, y).astype(dtype.as_numpy_dtype) ValueError: No cast function available. ====================================================================== ERROR: testOverload (__main__.MathOpsOverloadTest) (dtype=tf.bfloat16, np_func=, tf_func= at 0x0000017741FFF4C0>) testOverload (__main__.MathOpsOverloadTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_1vkgaz40\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 883, in testOverload self._compareBinary(10, 5, dtype, np_func, tf_func) File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_1vkgaz40\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 843, in _compareBinary np_ans = np_func(x, y).astype(dtype.as_numpy_dtype) ValueError: No cast function available. ---------------------------------------------------------------------- Ran 1 test in 0.087s FAILED (errors=2) ================================================================================ ==================== Test output for //tensorflow/python/kernel_tests:cwise_ops_test (shard 41 of 50): Running tests under Python 3.8.3: c:\Python38\python.exe [ RUN ] MathOpsOverloadTest.testOverload 2020-08-07 18:07:22.498277: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX AVX2 To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. 2020-08-07 18:07:22.528039: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x18eee98a7e0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-08-07 18:07:22.528627: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version INFO:tensorflow:time(__main__.MathOpsOverloadTest.testOverload): 0.07s I0807 18:07:22.565276 5312 test_util.py:1974] time(__main__.MathOpsOverloadTest.testOverload): 0.07s No pending test case: __main__.MathOpsOverloadTest.testOverload ====================================================================== ERROR: testOverload (__main__.MathOpsOverloadTest) (dtype=tf.bfloat16, np_func=, tf_func= at 0x0000018EED9D13A0>) testOverload (__main__.MathOpsOverloadTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_ruka25hh\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 883, in testOverload self._compareBinary(10, 5, dtype, np_func, tf_func) File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_ruka25hh\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 843, in _compareBinary np_ans = np_func(x, y).astype(dtype.as_numpy_dtype) ValueError: No cast function available. ====================================================================== ERROR: testOverload (__main__.MathOpsOverloadTest) (dtype=tf.bfloat16, np_func=, tf_func= at 0x0000018EED9D14C0>) testOverload (__main__.MathOpsOverloadTest) ---------------------------------------------------------------------- Traceback (most recent call last): File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_ruka25hh\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 883, in testOverload self._compareBinary(10, 5, dtype, np_func, tf_func) File "\\?\C:\Users\ContainerAdministrator\AppData\Local\Temp\Bazel.runfiles_ruka25hh\runfiles\org_tensorflow\tensorflow\python\kernel_tests\cwise_ops_test.py", line 843, in _compareBinary np_ans = np_func(x, y).astype(dtype.as_numpy_dtype) ValueError: No cast function available. ---------------------------------------------------------------------- Ran 1 test in 0.074s ``` PiperOrigin-RevId: 325819709 Change-Id: I919270e31978573c652f88224a32beb55592aa84 --- tensorflow/BUILD | 24 ----------- .../core/util/tensor_bundle/tensor_bundle.cc | 29 ++++++------- .../core/util/tensor_bundle/tensor_bundle.h | 41 +++++++++---------- .../def_file_filter/def_file_filter.py.tpl | 3 -- 4 files changed, 33 insertions(+), 64 deletions(-) diff --git a/tensorflow/BUILD b/tensorflow/BUILD index 6745b3e54fa..d1c1d7dcdef 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -882,30 +882,6 @@ genrule( visibility = ["//visibility:public"], ) -# The interface library (tensorflow_framework.dll.if.lib) for linking tensorflow DLL -# library (tensorflow_framework.dll) on Windows. -# To learn more about import library (called interface library in Bazel): -# https://docs.microsoft.com/en-us/cpp/build/linking-an-executable-to-a-dll?view=vs-2017#linking-implicitly -filegroup( - name = "get_tensorflow_framework_dll_import_lib", - srcs = ["//tensorflow:tensorflow_framework.dll"], - output_group = "interface_library", - visibility = ["//visibility:public"], -) - -# Rename the import library for tensorflow_framework.dll from -# tensorflow_framework.dll.if.lib to tensorflow_framework.lib -genrule( - name = "tensorflow_framework_dll_import_lib", - srcs = [":get_tensorflow_framework_dll_import_lib"], - outs = ["tensorflow_framework.lib"], - cmd = select({ - "//tensorflow:windows": "cp -f $< $@", - "//conditions:default": "touch $@", # Just a placeholder for Unix platforms - }), - visibility = ["//visibility:public"], -) - # The interface library (tensorflow_cc.dll.if.lib) for linking tensorflow DLL library (tensorflow_cc.dll) on Windows. # To learn more about import library (called interface library in Bazel): # https://docs.microsoft.com/en-us/cpp/build/linking-an-executable-to-a-dll?view=vs-2017#linking-implicitly diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc index ae9e10b3f67..bb18000fcfe 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle.cc +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.cc @@ -741,7 +741,7 @@ Status MergeBundles(Env* env, gtl::ArraySlice prefixes, // Interface for reading a tensor bundle. -TF_EXPORT BundleReader::BundleReader(Env* env, StringPiece prefix) +BundleReader::BundleReader(Env* env, StringPiece prefix) : env_(env), prefix_(prefix), metadata_(nullptr), @@ -796,7 +796,7 @@ TF_EXPORT BundleReader::BundleReader(Env* env, StringPiece prefix) kTensorBundleMinProducer, "Checkpoint", "checkpoint"); } -TF_EXPORT BundleReader::~BundleReader() { +BundleReader::~BundleReader() { delete metadata_; delete iter_; delete table_; @@ -936,7 +936,7 @@ Status BundleReader::GetValue(const BundleEntryProto& entry, Tensor* val) { return Status::OK(); } -TF_EXPORT Status BundleReader::Lookup(StringPiece key, Tensor* val) { +Status BundleReader::Lookup(StringPiece key, Tensor* val) { CHECK(val != nullptr); BundleEntryProto entry; TF_RETURN_IF_ERROR(GetBundleEntryProto(key, &entry)); @@ -950,7 +950,7 @@ TF_EXPORT Status BundleReader::Lookup(StringPiece key, Tensor* val) { } } -TF_EXPORT Status BundleReader::ReadCurrent(Tensor* val) { +Status BundleReader::ReadCurrent(Tensor* val) { CHECK(val != nullptr); BundleEntryProto entry; TF_RETURN_IF_ERROR(ParseEntryProto(iter_->key(), iter_->value(), &entry)); @@ -968,8 +968,8 @@ TF_EXPORT Status BundleReader::ReadCurrent(Tensor* val) { } } -TF_EXPORT Status BundleReader::LookupTensorSlices( - StringPiece key, std::vector* slices) { +Status BundleReader::LookupTensorSlices(StringPiece key, + std::vector* slices) { slices->clear(); BundleEntryProto entry; TF_RETURN_IF_ERROR(GetBundleEntryProto(key, &entry)); @@ -980,9 +980,8 @@ TF_EXPORT Status BundleReader::LookupTensorSlices( return Status::OK(); } -TF_EXPORT Status BundleReader::LookupSlice(StringPiece full_tensor_key, - const TensorSlice& slice_spec, - Tensor* val) { +Status BundleReader::LookupSlice(StringPiece full_tensor_key, + const TensorSlice& slice_spec, Tensor* val) { CHECK(val != nullptr); BundleEntryProto entry; TF_RETURN_IF_ERROR(GetBundleEntryProto(full_tensor_key, &entry)); @@ -1104,14 +1103,13 @@ Status BundleReader::GetSliceValue(StringPiece full_tensor_key, return Status::OK(); } -TF_EXPORT bool BundleReader::Contains(StringPiece key) { +bool BundleReader::Contains(StringPiece key) { Seek(key); return Valid() && (this->key() == key); } -TF_EXPORT Status BundleReader::LookupDtypeAndShape(StringPiece key, - DataType* dtype, - TensorShape* shape) { +Status BundleReader::LookupDtypeAndShape(StringPiece key, DataType* dtype, + TensorShape* shape) { BundleEntryProto entry; TF_RETURN_IF_ERROR(GetBundleEntryProto(key, &entry)); *dtype = entry.dtype(); @@ -1119,13 +1117,12 @@ TF_EXPORT Status BundleReader::LookupDtypeAndShape(StringPiece key, return Status::OK(); } -TF_EXPORT Status BundleReader::LookupTensorShape(StringPiece key, - TensorShape* shape) { +Status BundleReader::LookupTensorShape(StringPiece key, TensorShape* shape) { DataType ignored; return LookupDtypeAndShape(key, &ignored, shape); } -TF_EXPORT string BundleReader::DebugString() { +string BundleReader::DebugString() { // Format used below emulates that of TensorSliceReader::DebugString(). string shape_str; BundleEntryProto entry; diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle.h b/tensorflow/core/util/tensor_bundle/tensor_bundle.h index 0ff0b2d8939..c441000e47d 100644 --- a/tensorflow/core/util/tensor_bundle/tensor_bundle.h +++ b/tensorflow/core/util/tensor_bundle/tensor_bundle.h @@ -182,28 +182,28 @@ Status MergeBundles(Env* env, gtl::ArraySlice prefixes, // All threads accessing the same BundleReader must synchronize. class BundleReader { public: - TF_EXPORT BundleReader(Env* const env, StringPiece prefix); - TF_EXPORT ~BundleReader(); + BundleReader(Env* const env, StringPiece prefix); + ~BundleReader(); // Is ok() iff the reader construction is successful (completed the read of // the metadata). - TF_EXPORT Status status() const { return status_; } + Status status() const { return status_; } // Queries whether the bundle contains an entry keyed by "key". Calls Seek() // internally, so this call invalidates the reader's current position. // REQUIRES: status().ok() - TF_EXPORT bool Contains(StringPiece key); + bool Contains(StringPiece key); // Looks up the dtype and the shape of the tensor keyed by "key". // REQUIRES: status().ok() - TF_EXPORT Status LookupDtypeAndShape(StringPiece key, DataType* dtype, - TensorShape* shape) TF_MUST_USE_RESULT; + Status LookupDtypeAndShape(StringPiece key, DataType* dtype, + TensorShape* shape) TF_MUST_USE_RESULT; // Looks up the shape of the tensor keyed by "key". // Clears "shape" if not found. // REQUIRES: status().ok() - TF_EXPORT Status LookupTensorShape(StringPiece key, - TensorShape* shape) TF_MUST_USE_RESULT; + Status LookupTensorShape(StringPiece key, + TensorShape* shape) TF_MUST_USE_RESULT; // Looks up the tensor keyed by "key". If "key" refers to a partitioned // tensor, attempts to look up the full contents using all stored slices. @@ -217,7 +217,7 @@ class BundleReader { // // Validates the stored crc32c checksum against the restored bytes. // REQUIRES: status().ok() - TF_EXPORT Status Lookup(StringPiece key, Tensor* val) TF_MUST_USE_RESULT; + Status Lookup(StringPiece key, Tensor* val) TF_MUST_USE_RESULT; // Looks up the tensor pointed to by the internal iterator. // @@ -225,7 +225,7 @@ class BundleReader { // // Validates the stored crc32c checksum against the restored bytes. // REQUIRES: status().ok() && Valid() - TF_EXPORT Status ReadCurrent(Tensor* val) TF_MUST_USE_RESULT; + Status ReadCurrent(Tensor* val) TF_MUST_USE_RESULT; // Looks up the slices of the tensor keyed by "key". On OK, "slices" // is non-empty if and only if the tensor is a partitioned tensor. @@ -234,35 +234,34 @@ class BundleReader { // a slice with a larger start index in some dimension could come before // another slice with a smaller start index in the same dimension. // REQUIRES: status().ok() - TF_EXPORT Status LookupTensorSlices( - StringPiece key, std::vector* slices) TF_MUST_USE_RESULT; + Status LookupTensorSlices(StringPiece key, std::vector* slices) + TF_MUST_USE_RESULT; // Looks up a specific slice of a partitioned tensor. // It is only required that the stored slices cover the requested slice, // namely "slice_spec" is a subset of the union of the stored slices. // REQUIRES: status().ok() - TF_EXPORT Status LookupSlice(StringPiece full_tensor_key, - const TensorSlice& slice_spec, - Tensor* val) TF_MUST_USE_RESULT; + Status LookupSlice(StringPiece full_tensor_key, const TensorSlice& slice_spec, + Tensor* val) TF_MUST_USE_RESULT; // Seeks to the first position in the bundle whose key is no less than "key". // REQUIRES: status().ok() - TF_EXPORT void Seek(StringPiece key) { return iter_->Seek(key); } + void Seek(StringPiece key) { return iter_->Seek(key); } // Moves to the next position in the bundle. // REQUIRES: status().ok() - TF_EXPORT void Next() const { iter_->Next(); } + void Next() const { iter_->Next(); } // Returns true iff the reader is positioned to a key/val pair. // REQUIRES: status().ok() - TF_EXPORT bool Valid() const { return iter_->Valid(); } + bool Valid() const { return iter_->Valid(); } // Returns the key at the current position. // REQUIRES: status().ok() && Valid() - TF_EXPORT StringPiece key() const { return iter_->key(); } + StringPiece key() const { return iter_->key(); } // Returns the raw value at the current position. // REQUIRES: status().ok() && Valid() - TF_EXPORT StringPiece value() const { return iter_->value(); } + StringPiece value() const { return iter_->value(); } - TF_EXPORT string DebugString(); + string DebugString(); private: // Seeks for "key" and reads the metadata proto. diff --git a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl index 8642a6d2e24..1049939c94b 100644 --- a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl +++ b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl @@ -51,11 +51,8 @@ INCLUDEPRE_RE = re.compile(r"google::protobuf::internal::ExplicitlyConstructed|" r"google::protobuf::internal::ArenaImpl::AddCleanup|" # for contrib/data/_prefetching_ops r"google::protobuf::internal::LogMessage|" # for contrib/data/_prefetching_ops r"google::protobuf::Arena::OnArenaAllocation|" # for contrib/data/_prefetching_ops - r"google::protobuf::Message::InitializationErrorString|" - r"google::protobuf::MessageLite::ParseFromArray|" r"absl::Mutex::ReaderLock|" # for //tensorflow/contrib/rnn:python/ops/_gru_ops.so and more ops r"absl::Mutex::ReaderUnlock|" # for //tensorflow/contrib/rnn:python/ops/_gru_ops.so and more ops - r"tensorflow::TensorShape|" r"tensorflow::internal::LogMessage|" r"tensorflow::internal::LogString|" r"tensorflow::internal::CheckOpMessageBuilder|" From fd87e24980083e179bf93430cb6a1f920b9839d2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 09:32:46 -0700 Subject: [PATCH 0747/1017] Adding total-order comparison support in proto and HloInstruction. Specifically a comparison type attribute is added to Hlo proto so that total order comparison can be explicitly specified. A comparison expander pass is added to all compilers to expand total order comparison into equivalent implementations through type conversion. PiperOrigin-RevId: 325820826 Change-Id: I7beceb2f751ddc0be7c6b7a74037e562e7580b62 --- tensorflow/compiler/xla/client/lib/BUILD | 6 +- .../compiler/xla/client/lib/comparators.cc | 90 ++---------- .../compiler/xla/client/lib/comparators.h | 7 +- tensorflow/compiler/xla/client/xla_builder.cc | 64 ++++++++- tensorflow/compiler/xla/client/xla_builder.h | 45 +++--- tensorflow/compiler/xla/comparison_util.cc | 57 +++++--- tensorflow/compiler/xla/comparison_util.h | 6 +- .../compiler/xla/g3doc/operation_semantics.md | 5 +- tensorflow/compiler/xla/primitive_util.cc | 15 ++ tensorflow/compiler/xla/primitive_util.h | 2 + tensorflow/compiler/xla/service/BUILD | 20 ++- .../xla/service/comparison_expander.cc | 133 ++++++++++++++++++ .../xla/service/comparison_expander.h | 47 +++++++ tensorflow/compiler/xla/service/cpu/BUILD | 1 + .../compiler/xla/service/cpu/cpu_compiler.cc | 2 + tensorflow/compiler/xla/service/gpu/BUILD | 1 + .../compiler/xla/service/gpu/gpu_compiler.cc | 4 + tensorflow/compiler/xla/service/hlo.proto | 5 +- .../compiler/xla/service/hlo_instruction.cc | 20 ++- .../compiler/xla/service/hlo_instruction.h | 3 +- .../compiler/xla/service/hlo_instructions.cc | 26 ++-- .../compiler/xla/service/hlo_instructions.h | 4 +- tensorflow/compiler/xla/service/hlo_parser.cc | 29 +++- .../compiler/xla/service/hlo_parser_test.cc | 4 +- .../compiler/xla/service/interpreter/BUILD | 1 + .../xla/service/interpreter/compiler.cc | 2 + .../xla/tests/array_elementwise_ops_test.cc | 22 +++ 27 files changed, 472 insertions(+), 149 deletions(-) create mode 100644 tensorflow/compiler/xla/service/comparison_expander.cc create mode 100644 tensorflow/compiler/xla/service/comparison_expander.h diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD index 06fd8ceeb2b..a3c7c39e3ff 100644 --- a/tensorflow/compiler/xla/client/lib/BUILD +++ b/tensorflow/compiler/xla/client/lib/BUILD @@ -55,9 +55,13 @@ xla_test( cc_library( name = "comparators", srcs = ["comparators.cc"], - hdrs = ["comparators.h"], + hdrs = [ + "comparators.h", + "//tensorflow/compiler/xla:literal_util", + ], deps = [ ":constants", + "//tensorflow/compiler/xla:literal_util", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:types", "//tensorflow/compiler/xla:xla_data_proto_cc", diff --git a/tensorflow/compiler/xla/client/lib/comparators.cc b/tensorflow/compiler/xla/client/lib/comparators.cc index 74e89b767cf..cd594a5cf39 100644 --- a/tensorflow/compiler/xla/client/lib/comparators.cc +++ b/tensorflow/compiler/xla/client/lib/comparators.cc @@ -32,85 +32,13 @@ limitations under the License. namespace xla { namespace { -using XlaOpGenerator = XlaOp (*)(XlaOp, XlaOp, absl::Span); - -XlaOp BitcastConvertFloatingPointToIntegral(const XlaOp& value, - int64 bit_width) { - PrimitiveType signed_type; - PrimitiveType unsigned_type; - XlaOp max_value; - switch (bit_width) { - case 16: - max_value = - ConstantR0(value.builder(), - static_cast(std::numeric_limits::max())); - signed_type = S16; - unsigned_type = U16; - break; - case 32: - max_value = - ConstantR0(value.builder(), - static_cast(std::numeric_limits::max())); - signed_type = S32; - unsigned_type = U32; - break; - case 64: - max_value = - ConstantR0(value.builder(), - static_cast(std::numeric_limits::max())); - signed_type = S64; - unsigned_type = U64; - break; - default: - return value.builder()->ReportError( - InvalidArgument("Invalid bit width %lld for Comparator floating " - "point parameter.", - bit_width)); - } - // Switch from a floating point value to a integer value in such a way that - // when using the integer value to compare, we get the same result for normal - // values, and -Nan is treated as the smallest value, and Nan is treated as - // the largest value. - // If f is a float, and - // x = bit_cast(f); - // y = x < 0 ? numeric_limits::max() - x : x; - // then y is ordered as an int32 such that finite values have the obvious - // order, -0 is ordered before 0, and -NaN and NaN appear at the beginning - // and end of the ordering. - // Note that in order to avoid -x to overflow, we calculate - // numeric_limits::max() - x as unsigned, and then convert back to - // signed. - auto signed_value = BitcastConvertType(value, signed_type); - auto unsigned_value = BitcastConvertType(value, unsigned_type); - auto flipped_value = - BitcastConvertType(Sub(max_value, unsigned_value), signed_type); - auto is_negative = Lt(signed_value, Zero(value.builder(), signed_type)); - return Select(is_negative, flipped_value, signed_value); -} - -void ConvertFloatingPoint(const PrimitiveType& operand_type, XlaOp* lhs_param, - XlaOp* rhs_param) { - if (primitive_util::IsFloatingPointType(operand_type)) { - PrimitiveType compare_type = operand_type; - // Special-case handling for BF16. We currently do not support direct - // comparisons with BF16, so we convert to F32 and then use the F32 - // comparison logic. - if (compare_type == BF16) { - compare_type = F32; - *lhs_param = ConvertElementType(*lhs_param, F32); - *rhs_param = ConvertElementType(*rhs_param, F32); - } - int64 bit_width = primitive_util::BitWidth(compare_type); - *lhs_param = BitcastConvertFloatingPointToIntegral(*lhs_param, bit_width); - *rhs_param = BitcastConvertFloatingPointToIntegral(*rhs_param, bit_width); - } -} +using XlaCompareOp = XlaOp (*)(XlaOp, XlaOp, absl::Span); XlaComputation CreateScalarComparisonComputation( const string& name, const std::vector& operand_types, - XlaBuilder* builder, XlaOpGenerator generator) { + XlaBuilder* builder, XlaCompareOp generator) { CHECK_NE(operand_types.size(), 0); - std::vector> generators(operand_types.size()); + std::vector> generators(operand_types.size()); generators[0] = generator; return CreateScalarComparisonComputation(name, operand_types, generators, builder); @@ -119,7 +47,7 @@ XlaComputation CreateScalarComparisonComputation( XlaComputation CreateScalarComparisonComputation( const string& name, const std::vector& operand_types, - const std::vector>& generators, + const std::vector>& generators, XlaBuilder* builder) { // Create a default computation where we compare only the first two // parameters of type 'operand_types[0]'. @@ -146,7 +74,6 @@ XlaComputation CreateScalarComparisonComputation( absl::StrCat("p.", parameter_count, ".lhs")); auto rhs_param = Parameter(b.get(), parameter_count * 2 + 1, scalar_shape, absl::StrCat("p.", parameter_count, ".rhs")); - ConvertFloatingPoint(operand_type, &lhs_param, &rhs_param); lhs_params.emplace_back(lhs_param); rhs_params.emplace_back(rhs_param); if (generators[parameter_count].has_value()) { @@ -169,7 +96,8 @@ XlaComputation CreateScalarComparisonComputation( generators[i].value()(lhs_params[i], rhs_params[i], {}), result); if (i != last_generator_index) { - param_equal = And(param_equal, Eq(lhs_params[i], rhs_params[i])); + param_equal = + And(param_equal, EqTotalOrder(lhs_params[i], rhs_params[i])); } } } @@ -181,14 +109,14 @@ XlaComputation CreateScalarComparisonComputation( XlaComputation CreateScalarLtComputation( const std::vector& operand_types, XlaBuilder* builder) { return CreateScalarComparisonComputation("compare-less-than", operand_types, - builder, Lt); + builder, LtTotalOrder); } // Creates a scalar greater-than computation and returns it. XlaComputation CreateScalarGtComputation( const std::vector& operand_types, XlaBuilder* builder) { - return CreateScalarComparisonComputation("compare-greater-than", - operand_types, builder, Gt); + return CreateScalarComparisonComputation( + "compare-greater-than", operand_types, builder, GtTotalOrder); } } // namespace xla diff --git a/tensorflow/compiler/xla/client/lib/comparators.h b/tensorflow/compiler/xla/client/lib/comparators.h index 25924d4a4f4..a82a84799aa 100644 --- a/tensorflow/compiler/xla/client/lib/comparators.h +++ b/tensorflow/compiler/xla/client/lib/comparators.h @@ -43,14 +43,13 @@ XlaComputation CreateScalarGtComputation( const std::vector& operand_types, XlaBuilder* builder); // Creates a scalar comparison computation and returns it. This function takes -// an std::vector> and compare the operands -// where the generator isn't nullopt with the specified comparator -// at that location. +// a vector of comparator functions to compare the operands where the function +// isn't nullopt with the specified comparator at that location. XlaComputation CreateScalarComparisonComputation( const string& name, const std::vector& operand_types, const std::vector< absl::optional)>>& - generators, + comparators, XlaBuilder* builder); } // namespace xla diff --git a/tensorflow/compiler/xla/client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_builder.cc index 8de8216c005..2b69c71042d 100644 --- a/tensorflow/compiler/xla/client/xla_builder.cc +++ b/tensorflow/compiler/xla/client/xla_builder.cc @@ -577,7 +577,8 @@ XlaOp XlaBuilder::UnaryOp(HloOpcode unop, XlaOp operand) { XlaOp XlaBuilder::BinaryOp(HloOpcode binop, XlaOp lhs, XlaOp rhs, absl::Span broadcast_dimensions, - absl::optional direction) { + absl::optional direction, + absl::optional type) { return ReportErrorOrReturn([&]() -> StatusOr { TF_ASSIGN_OR_RETURN(const Shape* lhs_shape, GetShapePtr(lhs)); TF_ASSIGN_OR_RETURN(const Shape* rhs_shape, GetShapePtr(rhs)); @@ -635,7 +636,11 @@ XlaOp XlaBuilder::BinaryOp(HloOpcode binop, XlaOp lhs, XlaOp rhs, return InvalidArgument( "kCompare expects a ComparisonDirection, but none provided."); } - return Compare(shape, updated_lhs, updated_rhs, *direction); + if (type == absl::nullopt) { + return Compare(shape, updated_lhs, updated_rhs, *direction); + } else { + return Compare(shape, updated_lhs, updated_rhs, *direction, *type); + } } if (direction.has_value()) { @@ -658,8 +663,16 @@ XlaOp XlaBuilder::BinaryOpNoBroadcast(HloOpcode binop, const Shape& shape, StatusOr XlaBuilder::Compare(const Shape& shape, XlaOp lhs, XlaOp rhs, ComparisonDirection direction) { + return Compare(shape, lhs, rhs, direction, + Comparison::DefaultComparisonType(shape.element_type())); +} + +StatusOr XlaBuilder::Compare(const Shape& shape, XlaOp lhs, XlaOp rhs, + ComparisonDirection direction, + Comparison::Type type) { HloInstructionProto instr; instr.set_comparison_direction(ComparisonDirectionToString(direction)); + instr.set_comparison_type(ComparisonTypeToString(type)); *instr.mutable_shape() = shape.ToProto(); return AddInstruction(std::move(instr), HloOpcode::kCompare, {lhs, rhs}); } @@ -3512,31 +3525,71 @@ XlaOp Eq(const XlaOp lhs, const XlaOp rhs, return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kEq); } +XlaOp EqTotalOrder(const XlaOp lhs, const XlaOp rhs, + absl::Span broadcast_dimensions) { + auto compare_type = Comparison::Type::kFloatTotalOrder; + return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kEq, + compare_type); +} + XlaOp Ne(const XlaOp lhs, const XlaOp rhs, absl::Span broadcast_dimensions) { return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kNe); } +XlaOp NeTotalOrder(const XlaOp lhs, const XlaOp rhs, + absl::Span broadcast_dimensions) { + auto compare_type = Comparison::Type::kFloatTotalOrder; + return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kNe, + compare_type); +} + XlaOp Ge(const XlaOp lhs, const XlaOp rhs, absl::Span broadcast_dimensions) { return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kGe); } +XlaOp GeTotalOrder(const XlaOp lhs, const XlaOp rhs, + absl::Span broadcast_dimensions) { + auto compare_type = Comparison::Type::kFloatTotalOrder; + return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kGe, + compare_type); +} + XlaOp Gt(const XlaOp lhs, const XlaOp rhs, absl::Span broadcast_dimensions) { return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kGt); } +XlaOp GtTotalOrder(const XlaOp lhs, const XlaOp rhs, + absl::Span broadcast_dimensions) { + auto compare_type = Comparison::Type::kFloatTotalOrder; + return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kGt, + compare_type); +} + XlaOp Le(const XlaOp lhs, const XlaOp rhs, absl::Span broadcast_dimensions) { return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kLe); } +XlaOp LeTotalOrder(const XlaOp lhs, const XlaOp rhs, + absl::Span broadcast_dimensions) { + auto compare_type = Comparison::Type::kFloatTotalOrder; + return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kLe, + compare_type); +} XlaOp Lt(const XlaOp lhs, const XlaOp rhs, absl::Span broadcast_dimensions) { return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kLt); } +XlaOp LtTotalOrder(const XlaOp lhs, const XlaOp rhs, + absl::Span broadcast_dimensions) { + return Compare(lhs, rhs, broadcast_dimensions, ComparisonDirection::kLt, + Comparison::Type::kFloatTotalOrder); +} + XlaOp Compare(const XlaOp lhs, const XlaOp rhs, absl::Span broadcast_dimensions, ComparisonDirection direction) { @@ -3544,6 +3597,13 @@ XlaOp Compare(const XlaOp lhs, const XlaOp rhs, broadcast_dimensions, direction); } +XlaOp Compare(const XlaOp lhs, const XlaOp rhs, + absl::Span broadcast_dimensions, + ComparisonDirection direction, Comparison::Type compare_type) { + return lhs.builder()->BinaryOp(HloOpcode::kCompare, lhs, rhs, + broadcast_dimensions, direction, compare_type); +} + XlaOp Compare(const XlaOp lhs, const XlaOp rhs, ComparisonDirection direction) { return Compare(lhs, rhs, {}, direction); } diff --git a/tensorflow/compiler/xla/client/xla_builder.h b/tensorflow/compiler/xla/client/xla_builder.h index 6753b6dd919..6d30195d3d0 100644 --- a/tensorflow/compiler/xla/client/xla_builder.h +++ b/tensorflow/compiler/xla/client/xla_builder.h @@ -792,14 +792,17 @@ class XlaBuilder { // broadcast_dimensions specifies which dimensions to use for broadcasting // when the operation is between tensors of different ranks. The direction is // only used if opcode is kCompare. - XlaOp BinaryOp( - HloOpcode binop, XlaOp lhs, XlaOp rhs, - absl::Span broadcast_dimensions, - absl::optional direction = absl::nullopt); + XlaOp BinaryOp(HloOpcode binop, XlaOp lhs, XlaOp rhs, + absl::Span broadcast_dimensions, + absl::optional direction = absl::nullopt, + absl::optional type = absl::nullopt); // Internal helper method for binary op compare without broadcast dimensions. virtual StatusOr Compare(const Shape& shape, XlaOp lhs, XlaOp rhs, - Comparison::Direction direction); + ComparisonDirection direction); + virtual StatusOr Compare(const Shape& shape, XlaOp lhs, XlaOp rhs, + ComparisonDirection direction, + Comparison::Type type); // Internal helper method that does the building for an arbitrary binary op // with same ranked operands that doesn't broadcast. @@ -965,22 +968,13 @@ class XlaBuilder { friend XlaOp Select(XlaOp pred, XlaOp on_true, XlaOp on_false); friend XlaOp Tuple(XlaBuilder* builder, absl::Span elements); friend XlaOp GetTupleElement(XlaOp tuple_data, int64 index); - friend XlaOp Eq(XlaOp lhs, XlaOp rhs, - absl::Span broadcast_dimensions); - friend XlaOp Ne(XlaOp lhs, XlaOp rhs, - absl::Span broadcast_dimensions); - friend XlaOp Ge(XlaOp lhs, XlaOp rhs, - absl::Span broadcast_dimensions); - friend XlaOp Gt(XlaOp lhs, XlaOp rhs, - absl::Span broadcast_dimensions); - friend XlaOp Lt(XlaOp lhs, XlaOp rhs, - absl::Span broadcast_dimensions); - friend XlaOp Le(XlaOp lhs, XlaOp rhs, - absl::Span broadcast_dimensions); friend XlaOp Compare(XlaOp lhs, XlaOp rhs, absl::Span broadcast_dimensions, ComparisonDirection direction); - friend XlaOp Compare(XlaOp lhs, XlaOp rhs, ComparisonDirection direction); + friend XlaOp Compare(XlaOp lhs, XlaOp rhs, + absl::Span broadcast_dimensions, + ComparisonDirection direction, + Comparison::Type compare_type); friend XlaOp Dot(XlaOp lhs, XlaOp rhs, const PrecisionConfig* precision_config); friend XlaOp DotGeneral(XlaOp lhs, XlaOp rhs, @@ -1574,29 +1568,44 @@ XlaOp GetTupleElement(XlaOp tuple_data, int64 index); // Enqueues an equal-to comparison instruction onto the computation. XlaOp Eq(XlaOp lhs, XlaOp rhs, absl::Span broadcast_dimensions = {}); +XlaOp EqTotalOrder(XlaOp lhs, XlaOp rhs, + absl::Span broadcast_dimensions = {}); // Enqueues a not-equal comparison instruction onto the computation. XlaOp Ne(XlaOp lhs, XlaOp rhs, absl::Span broadcast_dimensions = {}); +XlaOp NeTotalOrder(XlaOp lhs, XlaOp rhs, + absl::Span broadcast_dimensions = {}); // Enqueues a greater-or-equal comparison instruction onto the computation. XlaOp Ge(XlaOp lhs, XlaOp rhs, absl::Span broadcast_dimensions = {}); +XlaOp GeTotalOrder(XlaOp lhs, XlaOp rhs, + absl::Span broadcast_dimensions = {}); // Enqueues a greater-than comparison instruction onto the computation. XlaOp Gt(XlaOp lhs, XlaOp rhs, absl::Span broadcast_dimensions = {}); +XlaOp GtTotalOrder(XlaOp lhs, XlaOp rhs, + absl::Span broadcast_dimensions = {}); // Enqueues a less-than comparison instruction onto the computation. XlaOp Lt(XlaOp lhs, XlaOp rhs, absl::Span broadcast_dimensions = {}); +XlaOp LtTotalOrder(XlaOp lhs, XlaOp rhs, + absl::Span broadcast_dimensions = {}); // Enqueues a less-or-equal comparison instruction onto the computation. XlaOp Le(XlaOp lhs, XlaOp rhs, absl::Span broadcast_dimensions = {}); +XlaOp LeTotalOrder(XlaOp lhs, XlaOp rhs, + absl::Span broadcast_dimensions = {}); // Enqueues a comparison instruction onto the computation (optionally without // broadcast_dimensions for consistency with others). +XlaOp Compare(XlaOp lhs, XlaOp rhs, + absl::Span broadcast_dimensions, + ComparisonDirection direction, Comparison::Type compare_type); XlaOp Compare(XlaOp lhs, XlaOp rhs, absl::Span broadcast_dimensions, ComparisonDirection direction); diff --git a/tensorflow/compiler/xla/comparison_util.cc b/tensorflow/compiler/xla/comparison_util.cc index 47fb69e3bce..06dd9642cac 100644 --- a/tensorflow/compiler/xla/comparison_util.cc +++ b/tensorflow/compiler/xla/comparison_util.cc @@ -54,32 +54,59 @@ StatusOr StringToComparisonDirection( return it->second; } -Comparison::Comparison(Direction dir, PrimitiveType type) : dir_(dir) { +StatusOr StringToComparisonType( + absl::string_view compare_type_name) { + static auto* type_map = new absl::flat_hash_map({ + {"FLOAT", Comparison::Type::kFloat}, + {"TOTALORDER", Comparison::Type::kFloatTotalOrder}, + {"SIGNED", Comparison::Type::kSigned}, + {"UNSIGNED", Comparison::Type::kUnsigned}, + }); + auto it = type_map->find(compare_type_name); + if (it == type_map->end()) { + return InvalidArgument("Unknown comparison type: %s", compare_type_name); + } + return it->second; +} + +std::string ComparisonTypeToString(Comparison::Type type) { + switch (type) { + case Comparison::Type::kFloat: + return "FLOAT"; + case Comparison::Type::kFloatTotalOrder: + return "TOTALORDER"; + case Comparison::Type::kSigned: + return "SIGNED"; + case Comparison::Type::kUnsigned: + return "UNSIGNED"; + } +} + +Comparison::Comparison(Direction dir, PrimitiveType type) + : dir_(dir), type_(DefaultComparisonType(type)) {} + +Comparison::Type Comparison::DefaultComparisonType(PrimitiveType type) { switch (type) { case S8: case S16: case S32: case S64: - type_ = Type::kSigned; - break; + return Type::kSigned; case PRED: case U8: case U16: case U32: case U64: - type_ = Type::kUnsigned; - break; + return Type::kUnsigned; case F16: case F32: case BF16: case F64: case C64: case C128: - type_ = Type::kFloat; - break; + return Type::kFloat; default: LOG(FATAL) << "Unsupported comparison mode." - << ComparisonDirectionToString(dir) << ":" << PrimitiveType_Name(type) << "\n"; } } @@ -164,20 +191,6 @@ bool Comparison::IsAntireflexive() const { } } -/* static */ const char* Comparison::ComparisonTypeToString( - Comparison::Type type) { - switch (type) { - case Type::kFloat: - return "f"; - case Type::kFloatTotalOrder: - return "ft"; - case Type::kSigned: - return "s"; - case Type::kUnsigned: - return "u"; - } -} - std::string Comparison::ToString(std::string prefix1, std::string prefix2) const { return prefix1 + std::string(ComparisonDirectionToString(dir_)) + prefix2 + diff --git a/tensorflow/compiler/xla/comparison_util.h b/tensorflow/compiler/xla/comparison_util.h index 11335c6b5ba..33ae2c67106 100644 --- a/tensorflow/compiler/xla/comparison_util.h +++ b/tensorflow/compiler/xla/comparison_util.h @@ -103,11 +103,11 @@ class Comparison { bool Compare(const T a, const T b) const { return GetComparator()(a, b); } + static Type DefaultComparisonType(PrimitiveType t); private: static Direction Converse(Direction dir); static Direction Inverse(Direction dir); - static const char* ComparisonTypeToString(Type type); const Direction dir_; Type type_; @@ -117,10 +117,14 @@ inline std::ostream& operator<<(std::ostream& os, const Comparison& cmp) { return os << cmp.ToString(); } string ComparisonDirectionToString(Comparison::Direction direction); +std::string ComparisonTypeToString(Comparison::Type type); StatusOr StringToComparisonDirection( absl::string_view direction_name); +StatusOr StringToComparisonType( + absl::string_view compare_type_name); + using ComparisonDirection = Comparison::Direction; } // namespace xla diff --git a/tensorflow/compiler/xla/g3doc/operation_semantics.md b/tensorflow/compiler/xla/g3doc/operation_semantics.md index 3031bfbf2e2..051c1539f6b 100644 --- a/tensorflow/compiler/xla/g3doc/operation_semantics.md +++ b/tensorflow/compiler/xla/g3doc/operation_semantics.md @@ -1235,7 +1235,10 @@ floating-point types. Where `Op` is one of `Eq` (equal-to), `Ne` (not equal-to), `Ge` (greater-or-equal-than), `Gt` (greater-than), `Le` (less-or-equal-than), `Lt` -(less-than). +(less-than). Another set of operators, EqTotalOrder, NeTotalOrder, GeTotalOrder, +GtTotalOrder, LeTotalOrder, and LtTotalOrder, provide the same functionalities, +except that they additionally support a total order over the floating point +numbers, by enforcing -NaN < -Inf < -Finite < -0 < +0 < +Finite < +Inf < +NaN. Arguments | Type | Semantics --------- | ------- | ---------------------------------------- diff --git a/tensorflow/compiler/xla/primitive_util.cc b/tensorflow/compiler/xla/primitive_util.cc index 2143d1dfbe7..c932469c56a 100644 --- a/tensorflow/compiler/xla/primitive_util.cc +++ b/tensorflow/compiler/xla/primitive_util.cc @@ -112,6 +112,21 @@ xla::PrimitiveType UnsignedIntegralTypeForBitWidth(int64 src_bitwidth) { } } +xla::PrimitiveType SignedIntegralTypeForBitWidth(int64 src_bitwidth) { + switch (src_bitwidth) { + case 8: + return xla::S8; + case 16: + return xla::S16; + case 32: + return xla::S32; + case 64: + return xla::S64; + default: + return xla::PRIMITIVE_TYPE_INVALID; + } +} + PrimitiveType ComplexComponentType(PrimitiveType complex_type) { switch (complex_type) { case C64: diff --git a/tensorflow/compiler/xla/primitive_util.h b/tensorflow/compiler/xla/primitive_util.h index 034c14e8930..1228b4f9a32 100644 --- a/tensorflow/compiler/xla/primitive_util.h +++ b/tensorflow/compiler/xla/primitive_util.h @@ -153,6 +153,8 @@ int BitWidth(PrimitiveType type); PrimitiveType UnsignedIntegralTypeForBitWidth(int64 src_bitwidth); +PrimitiveType SignedIntegralTypeForBitWidth(int64 src_bitwidth); + // Returns the real, imag component type underlying the given complex type. // LOG(FATAL)'s if complex_type is not complex. PrimitiveType ComplexComponentType(PrimitiveType complex_type); diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index bfcdf6fae34..fa7b480cab6 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -1700,7 +1700,10 @@ cc_library( cc_library( name = "hlo_creation_utils", srcs = ["hlo_creation_utils.cc"], - hdrs = ["hlo_creation_utils.h"], + hdrs = [ + "hlo_creation_utils.h", + "//tensorflow/compiler/xla:literal_util", + ], deps = [ ":hlo", ":hlo_module_config", @@ -1816,6 +1819,21 @@ cc_library( ], ) +cc_library( + name = "comparison_expander", + srcs = ["comparison_expander.cc"], + hdrs = ["comparison_expander.h"], + deps = [ + ":hlo", + ":hlo_creation_utils", + ":hlo_pass", + ":op_expander_pass", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla/client/lib:comparators", + ], +) + cc_library( name = "scatter_expander", srcs = ["scatter_expander.cc"], diff --git a/tensorflow/compiler/xla/service/comparison_expander.cc b/tensorflow/compiler/xla/service/comparison_expander.cc new file mode 100644 index 00000000000..5c88ff8cae2 --- /dev/null +++ b/tensorflow/compiler/xla/service/comparison_expander.cc @@ -0,0 +1,133 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/comparison_expander.h" + +#include "tensorflow/compiler/xla/client/lib/comparators.h" +#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h" +#include "tensorflow/compiler/xla/service/hlo_computation.h" +#include "tensorflow/compiler/xla/service/hlo_creation_utils.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_instructions.h" +#include "tensorflow/compiler/xla/service/hlo_opcode.h" +#include "tensorflow/compiler/xla/util.h" + +namespace xla { + +HloInstruction* BitcastConvertFloatingPointToIntegral( + HloComputation* computation, HloInstruction* value, + const Shape& signed_shape, const Shape& unsigned_shape, + HloInstruction* zero, HloInstruction* max_value) { + // Switch from a floating point value to a integer value in such a way that + // when using the integer value to compare, we get the same result for normal + // values, and -Nan is treated as the smallest value, and Nan is treated as + // the largest value. + // If f is a float, and + // x = bit_cast(f); + // y = x < 0 ? numeric_limits::max() - x : x; + // then y is ordered as an int32 such that finite values have the obvious + // order, -0 is ordered before 0, and -NaN and NaN appear at the beginning + // and end of the ordering. + // Note that in order to avoid -x to overflow, we calculate + // numeric_limits::max() - x as unsigned, and then convert back to + // signed. + auto signed_value = computation->AddInstruction( + HloInstruction::CreateBitcastConvert(signed_shape, value)); + auto unsigned_value = computation->AddInstruction( + HloInstruction::CreateBitcastConvert(unsigned_shape, value)); + auto flipped_value = computation->AddInstruction(HloInstruction::CreateBinary( + unsigned_shape, HloOpcode::kSubtract, max_value, unsigned_value)); + flipped_value = computation->AddInstruction( + HloInstruction::CreateBitcastConvert(signed_shape, flipped_value)); + auto compare_shape = signed_shape; + compare_shape.set_element_type(PRED); + auto is_negative = computation->AddInstruction(HloInstruction::CreateCompare( + compare_shape, signed_value, zero, ComparisonDirection::kLt)); + return computation->AddInstruction( + HloInstruction::CreateTernary(signed_shape, HloOpcode::kSelect, + is_negative, flipped_value, signed_value)); +} + +bool ComparisonExpander::InstructionMatchesPattern( + HloInstruction* instruction) { + if (HloCompareInstruction* compare = + dynamic_cast(instruction)) { + HloInstruction* lhs = instruction->operands()[0]; + if (compare->type() == Comparison::Type::kFloatTotalOrder && + primitive_util::IsFloatingPointType(lhs->shape().element_type())) { + return true; + } + } + return false; +} + +StatusOr ComparisonExpander::ExpandInstruction( + HloInstruction* instruction) { + CHECK(instruction->opcode() == HloOpcode::kCompare); + HloCompareInstruction* compare = + static_cast(instruction); + CHECK(compare->type() == Comparison::Type::kFloatTotalOrder); + HloComputation* computation = instruction->parent(); + HloInstruction* lhs = instruction->operands()[0]; + HloInstruction* rhs = instruction->operands()[1]; + Shape compare_shape = lhs->shape(); + PrimitiveType compare_type = compare_shape.element_type(); + CHECK(primitive_util::IsFloatingPointType(compare_type)); + // Special-case handling for BF16. We currently do not support direct + // comparisons with BF16, so we convert to F32 and then use the F32 + // comparison logic. + if (compare_type == BF16) { + compare_type = F32; + compare_shape.set_element_type(compare_type); + lhs = computation->AddInstruction( + HloInstruction::CreateConvert(compare_shape, lhs)); + rhs = computation->AddInstruction( + HloInstruction::CreateConvert(compare_shape, rhs)); + } + + int64 bit_width = primitive_util::BitWidth(compare_type); + PrimitiveType signed_type = + primitive_util::SignedIntegralTypeForBitWidth(bit_width); + PrimitiveType unsigned_type = + primitive_util::UnsignedIntegralTypeForBitWidth(bit_width); + auto signed_shape = compare_shape; + signed_shape.set_element_type(signed_type); + auto unsigned_shape = compare_shape; + unsigned_shape.set_element_type(unsigned_type); + auto zero_value = computation->AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::Zero(signed_type))); + zero_value = computation->AddInstruction(HloInstruction::CreateBroadcast( + signed_shape, zero_value, zero_value->shape().dimensions())); + auto max_signed = computation->AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::MaxValue(signed_type))); + auto max_shape = max_signed->shape(); + max_shape.set_element_type(unsigned_type); + auto max_unsigned = computation->AddInstruction( + HloInstruction::CreateConvert(max_shape, max_signed)); + auto max_value = computation->AddInstruction(HloInstruction::CreateBroadcast( + unsigned_shape, max_unsigned, max_shape.dimensions())); + lhs = BitcastConvertFloatingPointToIntegral( + computation, lhs, signed_shape, unsigned_shape, zero_value, max_value); + rhs = BitcastConvertFloatingPointToIntegral( + computation, rhs, signed_shape, unsigned_shape, zero_value, max_value); + auto new_compare = computation->AddInstruction(HloInstruction::CreateCompare( + instruction->shape(), lhs, rhs, compare->direction(), + Comparison::Type::kSigned)); + VLOG(2) << "New comparison instruction for total order:" + << new_compare->ToString() << "\n"; + return new_compare; +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/service/comparison_expander.h b/tensorflow/compiler/xla/service/comparison_expander.h new file mode 100644 index 00000000000..df8b5dc0137 --- /dev/null +++ b/tensorflow/compiler/xla/service/comparison_expander.h @@ -0,0 +1,47 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_COMPARISON_EXPANDER_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_COMPARISON_EXPANDER_H_ + +#include + +#include "tensorflow/compiler/xla/service/hlo_module.h" +#include "tensorflow/compiler/xla/service/hlo_pass_interface.h" +#include "tensorflow/compiler/xla/service/op_expander_pass.h" + +namespace xla { + +// A pass which performs expansion of the comparison operator to support total +// order comparison of floating point numbers. +class ComparisonExpander : public OpExpanderPass { + public: + explicit ComparisonExpander() = default; + ~ComparisonExpander() override = default; + absl::string_view name() const override { return "comparison-expander"; } + + private: + // Returns `true` if `instruction` should be expanded by this pass. + bool InstructionMatchesPattern(HloInstruction* instruction) override; + // Returns a replacement for `instruction`, or nullptr if no replacement is + // needed (e.g. only the to_apply subcomputation of the instruction was + // modified). + StatusOr ExpandInstruction( + HloInstruction* instruction) override; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_COMPARISON_EXPANDER_H_ diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD index e0317574e59..7c362b2da44 100644 --- a/tensorflow/compiler/xla/service/cpu/BUILD +++ b/tensorflow/compiler/xla/service/cpu/BUILD @@ -145,6 +145,7 @@ cc_library( "//tensorflow/compiler/xla/service:conditional_to_select", "//tensorflow/compiler/xla/service:slow_operation_alarm", "//tensorflow/compiler/xla/service:scatter_expander", + "//tensorflow/compiler/xla/service:comparison_expander", "//tensorflow/compiler/xla/service:slice_sinker", "//tensorflow/compiler/xla:cpu_function_runtime", "//tensorflow/compiler/xla:literal", diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc index aab13f6e8dd..39d2b11ad37 100644 --- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc +++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc @@ -54,6 +54,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/call_inliner.h" #include "tensorflow/compiler/xla/service/cholesky_expander.h" +#include "tensorflow/compiler/xla/service/comparison_expander.h" #include "tensorflow/compiler/xla/service/conditional_canonicalizer.h" #include "tensorflow/compiler/xla/service/conditional_simplifier.h" #include "tensorflow/compiler/xla/service/conditional_to_select.h" @@ -261,6 +262,7 @@ Status CpuCompiler::RunHloPassesThroughLayoutAssn( pipeline.AddPass(); pipeline.AddPass(); + pipeline.AddPass(); pipeline.AddPass(); pipeline.AddPass(); diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 47af5756f87..b9ba2100293 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -1168,6 +1168,7 @@ cc_library( "//tensorflow/compiler/xla/service:batchnorm_expander", "//tensorflow/compiler/xla/service:buffer_assignment", "//tensorflow/compiler/xla/service:call_inliner", + "//tensorflow/compiler/xla/service:comparison_expander", "//tensorflow/compiler/xla/service:conditional_canonicalizer", "//tensorflow/compiler/xla/service:conditional_simplifier", "//tensorflow/compiler/xla/service:convolution_4d_expander", diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index 225fa328f3d..f5bf7476059 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -35,6 +35,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/batchnorm_expander.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/call_inliner.h" +#include "tensorflow/compiler/xla/service/comparison_expander.h" #include "tensorflow/compiler/xla/service/conditional_canonicalizer.h" #include "tensorflow/compiler/xla/service/conditional_simplifier.h" #include "tensorflow/compiler/xla/service/convolution_4d_expander.h" @@ -140,6 +141,9 @@ Status GpuCompiler::OptimizeHloModule( pipeline.AddPass(); pipeline.AddPass(RandomAlgorithm::RNG_PHILOX); + // Comparison total order expander + pipeline.AddPass(); + // Remove zero-sized HLO from the input so that other passes don't have to // handle it. pipeline.AddPass(); diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto index e043216c17e..17a7b18c84b 100644 --- a/tensorflow/compiler/xla/service/hlo.proto +++ b/tensorflow/compiler/xla/service/hlo.proto @@ -35,7 +35,7 @@ import "tensorflow/compiler/xla/xla_data.proto"; option cc_enable_arenas = true; // Serialization of HloInstruction. -// Next ID: 72 +// Next ID: 73 message HloInstructionProto { reserved 10; reserved "parameter_name"; @@ -248,6 +248,9 @@ message HloInstructionProto { // RNG algorithm used by kRngBitGenerator. xla.RandomAlgorithm rng_algorithm = 70; + + // The comparison type used for kCompare. + string comparison_type = 72; } // Serialization of HloComputation. diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc index 94d53ebe0b1..2ce3c12b4e9 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.cc +++ b/tensorflow/compiler/xla/service/hlo_instruction.cc @@ -174,8 +174,19 @@ StatusOr> HloInstruction::CreateFromProto( comparison_direction, StringToComparisonDirection(proto.comparison_direction())); } - instruction = - CreateCompare(shape, operands(0), operands(1), *comparison_direction); + auto comparison_type_str = proto.comparison_type(); + if (!comparison_type_str.empty()) { + // If a comparison type is specified, it *must* be valid. + TF_ASSIGN_OR_RETURN(auto comparison_type, + StringToComparisonType(comparison_type_str)); + instruction = CreateCompare(shape, operands(0), operands(1), + *comparison_direction, comparison_type); + } else { + // Allow the specify of comparison type to be optional. + // The comparison type will be determined by the types of the operands. + instruction = CreateCompare(shape, operands(0), operands(1), + *comparison_direction); + } break; } case HloOpcode::kTriangularSolve: { @@ -926,8 +937,9 @@ HloInstruction::CreateRngBitGenerator(const Shape& shape, HloInstruction* state, /* static */ std::unique_ptr HloInstruction::CreateCompare( const Shape& shape, HloInstruction* lhs, HloInstruction* rhs, - ComparisonDirection direction) { - return absl::make_unique(shape, lhs, rhs, direction); + ComparisonDirection direction, absl::optional type) { + return absl::make_unique(shape, lhs, rhs, direction, + type); } /* static */ std::unique_ptr diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h index e29323c25b4..bdd64c908f0 100644 --- a/tensorflow/compiler/xla/service/hlo_instruction.h +++ b/tensorflow/compiler/xla/service/hlo_instruction.h @@ -595,7 +595,8 @@ class HloInstruction { // Creates a compare op, performing the comparison specified in direction. static std::unique_ptr CreateCompare( const Shape& shape, HloInstruction* lhs, HloInstruction* rhs, - Comparison::Direction direction); + Comparison::Direction direction, + absl::optional type = absl::nullopt); static std::unique_ptr CreateTriangularSolve( const Shape& shape, HloInstruction* a, HloInstruction* b, diff --git a/tensorflow/compiler/xla/service/hlo_instructions.cc b/tensorflow/compiler/xla/service/hlo_instructions.cc index 3d34fa03a80..dbc1d85d1bb 100644 --- a/tensorflow/compiler/xla/service/hlo_instructions.cc +++ b/tensorflow/compiler/xla/service/hlo_instructions.cc @@ -204,12 +204,13 @@ std::unique_ptr HloFftInstruction::CloneWithNewOperandsImpl( fft_length_); } -HloCompareInstruction::HloCompareInstruction(const Shape& shape, - HloInstruction* lhs, - HloInstruction* rhs, - ComparisonDirection direction) +HloCompareInstruction::HloCompareInstruction( + const Shape& shape, HloInstruction* lhs, HloInstruction* rhs, + ComparisonDirection direction, absl::optional type) : HloInstruction(HloOpcode::kCompare, shape), - compare_(direction, lhs->shape().element_type()) { + compare_(direction, type ? (*type) + : Comparison::DefaultComparisonType( + lhs->shape().element_type())) { AppendOperand(lhs); AppendOperand(rhs); } @@ -218,12 +219,21 @@ HloInstructionProto HloCompareInstruction::ToProto() const { HloInstructionProto proto = HloInstruction::ToProto(); proto.set_comparison_direction( ComparisonDirectionToString(compare_.GetDirection())); + proto.set_comparison_type(ComparisonTypeToString(compare_.GetType())); return proto; } std::vector HloCompareInstruction::ExtraAttributesToStringImpl( const HloPrintOptions& options) const { - return {StrCat("direction=", ComparisonDirectionToString(direction()))}; + std::vector result; + result.push_back( + StrCat("direction=", ComparisonDirectionToString(direction()))); + if (compare_.GetType() != + Comparison::DefaultComparisonType(operand(0)->shape().element_type())) { + result.push_back( + StrCat("type=", ComparisonTypeToString(compare_.GetType()))); + } + return result; } bool HloCompareInstruction::IdenticalSlowPath( @@ -238,8 +248,8 @@ std::unique_ptr HloCompareInstruction::CloneWithNewOperandsImpl( const Shape& shape, absl::Span new_operands, HloCloneContext* context) const { CHECK_EQ(new_operands.size(), 2); - return absl::make_unique(shape, new_operands[0], - new_operands[1], direction()); + return absl::make_unique( + shape, new_operands[0], new_operands[1], direction(), type()); } namespace { diff --git a/tensorflow/compiler/xla/service/hlo_instructions.h b/tensorflow/compiler/xla/service/hlo_instructions.h index 51317b32bd0..3f92bb92f02 100644 --- a/tensorflow/compiler/xla/service/hlo_instructions.h +++ b/tensorflow/compiler/xla/service/hlo_instructions.h @@ -136,8 +136,10 @@ class HloCompareInstruction : public HloInstruction { public: explicit HloCompareInstruction(const Shape& shape, HloInstruction* lhs, HloInstruction* rhs, - ComparisonDirection direction); + ComparisonDirection direction, + absl::optional type); ComparisonDirection direction() const { return compare_.GetDirection(); } + Comparison::Type type() const { return compare_.GetType(); } HloInstructionProto ToProto() const override; private: diff --git a/tensorflow/compiler/xla/service/hlo_parser.cc b/tensorflow/compiler/xla/service/hlo_parser.cc index a093a9d0f52..2afa06a5df4 100644 --- a/tensorflow/compiler/xla/service/hlo_parser.cc +++ b/tensorflow/compiler/xla/service/hlo_parser.cc @@ -194,6 +194,7 @@ class HloParserImpl : public HloParser { kBracedHloComputationList, kFftType, kComparisonDirection, + kComparisonType, kWindow, kConvolutionDimensionNumbers, kSharding, @@ -327,6 +328,7 @@ class HloParserImpl : public HloParser { bool ParseOpcode(HloOpcode* result); bool ParseFftType(FftType* result); bool ParseComparisonDirection(ComparisonDirection* result); + bool ParseComparisonType(Comparison::Type* result); bool ParseFusionKind(HloInstruction::FusionKind* result); bool ParseRandomDistribution(RandomDistribution* result); bool ParseRandomAlgorithm(RandomAlgorithm* result); @@ -1362,14 +1364,16 @@ bool HloParserImpl::ParseInstructionRhs(HloComputation::Builder* builder, } case HloOpcode::kCompare: { optional direction; + optional type; attrs["direction"] = {/*required=*/true, AttrTy::kComparisonDirection, &direction}; + attrs["type"] = {/*required=*/false, AttrTy::kComparisonType, &type}; if (!ParseOperands(&operands, /*expected_size=*/2) || !ParseAttributes(attrs)) { return false; } instruction = builder->AddInstruction(HloInstruction::CreateCompare( - shape, operands[0], operands[1], *direction)); + shape, operands[0], operands[1], *direction, type)); break; } case HloOpcode::kCholesky: { @@ -3018,6 +3022,14 @@ bool HloParserImpl::ParseAttributeHelper( ->emplace(result); return true; } + case AttrTy::kComparisonType: { + Comparison::Type result; + if (!ParseComparisonType(&result)) { + return false; + } + static_cast*>(attr_out_ptr)->emplace(result); + return true; + } case AttrTy::kEnum: { if (lexer_.GetKind() != TokKind::kIdent) { return TokenError("expects an enumeration value"); @@ -4145,6 +4157,21 @@ bool HloParserImpl::ParseComparisonDirection(ComparisonDirection* result) { return true; } +bool HloParserImpl::ParseComparisonType(Comparison::Type* result) { + VLOG(1) << "ParseComparisonType"; + if (lexer_.GetKind() != TokKind::kIdent) { + return TokenError("expects comparison type"); + } + std::string val = lexer_.GetStrVal(); + auto status_or_result = StringToComparisonType(val); + if (!status_or_result.ok()) { + return TokenError(StrFormat("expects comparison type but sees: %s", val)); + } + *result = status_or_result.ValueOrDie(); + lexer_.Lex(); + return true; +} + bool HloParserImpl::ParseFusionKind(HloInstruction::FusionKind* result) { VLOG(3) << "ParseFusionKind"; if (lexer_.GetKind() != TokKind::kIdent) { diff --git a/tensorflow/compiler/xla/service/hlo_parser_test.cc b/tensorflow/compiler/xla/service/hlo_parser_test.cc index 7880075dcbe..aba6aeff999 100644 --- a/tensorflow/compiler/xla/service/hlo_parser_test.cc +++ b/tensorflow/compiler/xla/service/hlo_parser_test.cc @@ -230,7 +230,7 @@ R"(HloModule SelectR1F32WithCmpR1F32sFromParamsSmall_module ENTRY %SelectR1F32WithCmpR1F32sFromParamsSmall.v4 (v1: f32[4], v2: f32[4]) -> f32[4] { %v1 = f32[4]{0} parameter(0), sharding={maximal device=1} %v2 = f32[4]{0} parameter(1), sharding={maximal device=1} - %greater-than = pred[4]{0} compare(f32[4]{0} %v1, f32[4]{0} %v2), direction=GT, sharding={replicated} + %greater-than = pred[4]{0} compare(f32[4]{0} %v1, f32[4]{0} %v2), direction=GT, type=TOTALORDER, sharding={replicated} ROOT %select = f32[4]{0} select(pred[4]{0} %greater-than, f32[4]{0} %v1, f32[4]{0} %v2), sharding={} } @@ -512,7 +512,7 @@ R"(HloModule R4F32OverlapSmall_module %ge_F32.v3 (lhs: f32[], rhs: f32[]) -> pred[] { %lhs = f32[] parameter(0) %rhs = f32[] parameter(1) - ROOT %greater-than-or-equal-to = pred[] compare(f32[] %lhs, f32[] %rhs), direction=GE + ROOT %greater-than-or-equal-to = pred[] compare(f32[] %lhs, f32[] %rhs), direction=GE, type=TOTALORDER } %add_F32.v3 (lhs.1: f32[], rhs.1: f32[]) -> f32[] { diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD index 7a4eefc1ab6..3444d4cae42 100644 --- a/tensorflow/compiler/xla/service/interpreter/BUILD +++ b/tensorflow/compiler/xla/service/interpreter/BUILD @@ -34,6 +34,7 @@ cc_library( "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla/service:algebraic_simplifier", "//tensorflow/compiler/xla/service:cholesky_expander", + "//tensorflow/compiler/xla/service:comparison_expander", "//tensorflow/compiler/xla/service:compiler", "//tensorflow/compiler/xla/service:computation_placer", "//tensorflow/compiler/xla/service:custom_call_target_registry", diff --git a/tensorflow/compiler/xla/service/interpreter/compiler.cc b/tensorflow/compiler/xla/service/interpreter/compiler.cc index 1649be2ca8f..a059482d832 100644 --- a/tensorflow/compiler/xla/service/interpreter/compiler.cc +++ b/tensorflow/compiler/xla/service/interpreter/compiler.cc @@ -21,6 +21,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "tensorflow/compiler/xla/service/algebraic_simplifier.h" #include "tensorflow/compiler/xla/service/cholesky_expander.h" +#include "tensorflow/compiler/xla/service/comparison_expander.h" #include "tensorflow/compiler/xla/service/computation_placer.h" #include "tensorflow/compiler/xla/service/custom_call_target_registry.h" #include "tensorflow/compiler/xla/service/dynamic_index_splitter.h" @@ -81,6 +82,7 @@ Status InterpreterCompiler::RunHloOptimization(HloModule* hlo_module) { pipeline.AddPass(); pipeline.AddPass(); + pipeline.AddPass(); pipeline.AddPass(); pipeline.AddPass( hlo_module->mutable_entry_computation_layout(), diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc index a956b85a940..fdc679a61c6 100644 --- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc +++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc @@ -1203,6 +1203,16 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareEqF32s) { ComputeAndCompareR1(&builder, {false, false, true, false, false}, {}); } +XLA_TEST_F(ArrayElementwiseOpTest, CompareEqF32sTO) { + SetFastMathDisabled(true); + XlaBuilder builder(TestName()); + auto lhs = ConstantR1(&builder, {-2.5f, 25.5f, 2.25f, NAN, 6.0f}); + auto rhs = ConstantR1(&builder, {10.0f, 5.0f, 2.25f, NAN, NAN}); + EqTotalOrder(lhs, rhs); + + ComputeAndCompareR1(&builder, {false, false, true, true, false}, {}); +} + XLA_TEST_F(ArrayElementwiseOpTest, CompareEqZeroElementF32s) { XlaBuilder builder(TestName()); auto lhs = ConstantR1(&builder, {}); @@ -1222,6 +1232,18 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGeF32s) { ComputeAndCompareR1(&builder, {false, true, true, false, false}, {}); } +XLA_TEST_F(ArrayElementwiseOpTest, CompareGeF32sTO) { + SetFastMathDisabled(true); + XlaBuilder builder(TestName()); + auto lhs = + ConstantR1(&builder, {-2.5f, 25.5f, 2.25f, NAN, 6.0f, 6.0f}); + auto rhs = ConstantR1(&builder, {10.0f, 5.0f, 1.0f, 10.0f, NAN, -NAN}); + GeTotalOrder(lhs, rhs); + + ComputeAndCompareR1(&builder, {false, true, true, true, false, true}, + {}); +} + XLA_TEST_F(ArrayElementwiseOpTest, CompareGtF32s) { SetFastMathDisabled(true); XlaBuilder builder(TestName()); From 5161a45676dc8959a2d4aaeca5b8efa5335461f7 Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Mon, 10 Aug 2020 09:52:40 -0700 Subject: [PATCH 0748/1017] Added virtual destructors. PiperOrigin-RevId: 325824783 Change-Id: I06f8add05dc77dd93310738157c4cb83adf1e795 --- tensorflow/lite/delegates/gpu/cl/buffer.cc | 2 -- tensorflow/lite/delegates/gpu/cl/buffer.h | 2 +- tensorflow/lite/delegates/gpu/cl/linear_storage.h | 2 ++ tensorflow/lite/delegates/gpu/cl/texture2d.cc | 2 -- tensorflow/lite/delegates/gpu/cl/texture2d.h | 2 +- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/buffer.cc b/tensorflow/lite/delegates/gpu/cl/buffer.cc index 8639e8bbf18..31770fca47e 100644 --- a/tensorflow/lite/delegates/gpu/cl/buffer.cc +++ b/tensorflow/lite/delegates/gpu/cl/buffer.cc @@ -132,8 +132,6 @@ Buffer& Buffer::operator=(Buffer&& buffer) { return *this; } -Buffer::~Buffer() { Release(); } - void Buffer::Release() { if (buffer_) { clReleaseMemObject(buffer_); diff --git a/tensorflow/lite/delegates/gpu/cl/buffer.h b/tensorflow/lite/delegates/gpu/cl/buffer.h index dc5befebea2..dbc43463bc7 100644 --- a/tensorflow/lite/delegates/gpu/cl/buffer.h +++ b/tensorflow/lite/delegates/gpu/cl/buffer.h @@ -61,7 +61,7 @@ class Buffer : public GPUObject { Buffer(const Buffer&) = delete; Buffer& operator=(const Buffer&) = delete; - ~Buffer(); + virtual ~Buffer() { Release(); } // for profiling and memory statistics uint64_t GetMemorySizeInBytes() const { return size_; } diff --git a/tensorflow/lite/delegates/gpu/cl/linear_storage.h b/tensorflow/lite/delegates/gpu/cl/linear_storage.h index 1bc855f4205..b69f76b9c1a 100644 --- a/tensorflow/lite/delegates/gpu/cl/linear_storage.h +++ b/tensorflow/lite/delegates/gpu/cl/linear_storage.h @@ -61,6 +61,8 @@ class LinearStorage : public GPUObject { public: LinearStorage() {} + virtual ~LinearStorage() {} + // Move only LinearStorage(LinearStorage&& storage); LinearStorage& operator=(LinearStorage&& storage); diff --git a/tensorflow/lite/delegates/gpu/cl/texture2d.cc b/tensorflow/lite/delegates/gpu/cl/texture2d.cc index cbeafe04c05..5edf64e83e7 100644 --- a/tensorflow/lite/delegates/gpu/cl/texture2d.cc +++ b/tensorflow/lite/delegates/gpu/cl/texture2d.cc @@ -118,8 +118,6 @@ Texture2D& Texture2D::operator=(Texture2D&& texture) { return *this; } -Texture2D::~Texture2D() { Release(); } - void Texture2D::Release() { if (texture_) { clReleaseMemObject(texture_); diff --git a/tensorflow/lite/delegates/gpu/cl/texture2d.h b/tensorflow/lite/delegates/gpu/cl/texture2d.h index 54a2732fc90..0e972de8cd3 100644 --- a/tensorflow/lite/delegates/gpu/cl/texture2d.h +++ b/tensorflow/lite/delegates/gpu/cl/texture2d.h @@ -57,7 +57,7 @@ class Texture2D : public GPUObject { Texture2D(const Texture2D&) = delete; Texture2D& operator=(const Texture2D&) = delete; - ~Texture2D(); + virtual ~Texture2D() { Release(); } cl_mem GetMemoryPtr() const { return texture_; } From 078052d2a955fcfe662f97c6b1b225397ced06a9 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 10 Aug 2020 17:09:52 +0000 Subject: [PATCH 0749/1017] clean up --- tensorflow/c/kernels_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index 708c39690a8..e216b85c13f 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -587,4 +587,4 @@ void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, memcpy(data, values, tensor_size_bytes); #endif } -} // namespace tensorflow \ No newline at end of file +} // namespace tensorflow From 662291071f6c6271138e52c73a61a31308d5bc6e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 10:12:27 -0700 Subject: [PATCH 0750/1017] Integrate LLVM at llvm/llvm-project@54cb552b9620 Updates LLVM usage to match [54cb552b9620](https://github.com/llvm/llvm-project/commit/54cb552b9620) PiperOrigin-RevId: 325829818 Change-Id: If43fd8a2f7453600e07781bcf288b09d1cd8ffc1 --- tensorflow/workspace.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index a726bf642d1..73b3fb42a96 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "38537307e502c1ac9a09e6f75f9208db1327a0bf" - LLVM_SHA256 = "c801bf0f2ebfce86dbf7ad39c40ee371f422e8d07213f4ca67e5e46c7cb200ed" + LLVM_COMMIT = "54cb552b962097d0e3ef7306b69a3c82cc7fff37" + LLVM_SHA256 = "42a65541c62e8349cac068e7f44cb1d9d41addf0dbab0002cc7a7d7203bcb35b" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), From e1654c78feae0663e6cbf1b13d8a54832deb3129 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Mon, 10 Aug 2020 10:18:01 -0700 Subject: [PATCH 0751/1017] Retire temporary release toggle. PiperOrigin-RevId: 325831070 Change-Id: If723c0bd89255522c8cbfd02156c30db23ea53b8 --- tensorflow/python/autograph/core/converter_testing.py | 2 -- tensorflow/python/autograph/operators/control_flow.py | 6 ------ tensorflow/python/autograph/utils/testing.py | 2 -- 3 files changed, 10 deletions(-) diff --git a/tensorflow/python/autograph/core/converter_testing.py b/tensorflow/python/autograph/core/converter_testing.py index 2909cf3f8bc..9f2604dec94 100644 --- a/tensorflow/python/autograph/core/converter_testing.py +++ b/tensorflow/python/autograph/core/converter_testing.py @@ -21,7 +21,6 @@ from __future__ import print_function import contextlib import imp import inspect -import os import sys import six @@ -101,7 +100,6 @@ class TestCase(test.TestCase): def setUp(self): # AutoGraph tests must run in graph mode to properly test control flow. - os.environ['AUTOGRAPH_CREATE_SYMBOLS_IN_LOOPS'] = '1' self.graph = ops.Graph().as_default() self.graph.__enter__() diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py index 0106efda5dd..7b307ed5020 100644 --- a/tensorflow/python/autograph/operators/control_flow.py +++ b/tensorflow/python/autograph/operators/control_flow.py @@ -60,7 +60,6 @@ from __future__ import division from __future__ import print_function import functools -import os import traceback import numpy as np @@ -973,11 +972,6 @@ def _try_handling_undefineds( """ state_modified = False - # TODO(mdan): Remove once the default option is stable. - if os.getenv('AUTOGRAPH_CREATE_SYMBOLS_IN_LOOPS', '1') == '0': - _verify_loop_init_vars(init_vars, symbol_names) - return False, init_vars - try: # Stage an iteration of the loop body in a temporary graph. with func_graph.FuncGraph('tmp').as_default(): diff --git a/tensorflow/python/autograph/utils/testing.py b/tensorflow/python/autograph/utils/testing.py index 1da82db66c8..df60583bf85 100644 --- a/tensorflow/python/autograph/utils/testing.py +++ b/tensorflow/python/autograph/utils/testing.py @@ -18,7 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import os import re import types import unittest @@ -108,7 +107,6 @@ class AutoGraphTestCase(test.TestCase): def setUp(self): super().setUp() - os.environ['AUTOGRAPH_CREATE_SYMBOLS_IN_LOOPS'] = '1' self.variables = {} self.trace_log = [] op_callbacks.add_op_callback(self._op_callback) From 96e80d355547079f2bb8db0fddd68a924883c5d9 Mon Sep 17 00:00:00 2001 From: Yujing Zhang Date: Mon, 10 Aug 2020 10:29:54 -0700 Subject: [PATCH 0752/1017] Support local multi-device functions with outputs on remote devices. PiperOrigin-RevId: 325834034 Change-Id: Iee8c0c6694dfcee108b73ee9dec41658907cb187 --- tensorflow/c/eager/c_api_remote_test.cc | 88 +++++--- .../core/common_runtime/eager/execute.cc | 207 +++++++++++++----- .../common_runtime/eager/kernel_and_device.cc | 27 +-- .../common_runtime/eager/kernel_and_device.h | 22 +- .../eager/kernel_and_device_test.cc | 2 +- .../process_function_library_runtime.cc | 32 +-- .../eager/eager_service_impl_test.cc | 21 +- .../eager/remote_copy_node.cc | 17 +- 8 files changed, 268 insertions(+), 148 deletions(-) diff --git a/tensorflow/c/eager/c_api_remote_test.cc b/tensorflow/c/eager/c_api_remote_test.cc index 94c32cf3f30..e99f6d6e170 100644 --- a/tensorflow/c/eager/c_api_remote_test.cc +++ b/tensorflow/c/eager/c_api_remote_test.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "absl/strings/str_cat.h" #include "tensorflow/c/eager/c_api.h" #include "tensorflow/c/eager/c_api_experimental.h" #include "tensorflow/c/eager/c_api_internal.h" @@ -115,40 +116,42 @@ void TestRemoteExecute(bool async) { TEST(CAPI, RemoteExecute) { TestRemoteExecute(false); } TEST(CAPI, RemoteExecuteAsync) { TestRemoteExecute(true); } -string MatMulFunction() { +string MatMulFunction(const string& matmul_device) { tensorflow::FunctionDef def; CHECK(tensorflow::protobuf::TextFormat::ParseFromString( - " signature {" - " name: 'MatMulFunction'" - " input_arg {" - " name: 'a'" - " type: DT_FLOAT" - " }" - " input_arg {" - " name: 'b'" - " type: DT_FLOAT" - " }" - " output_arg {" - " name: 'm'" - " type: DT_FLOAT" - " }" - " }" - " node_def {" - " name: 'matmul'" - " op: 'MatMul'" - " input: 'a'" - " input: 'b'" - " attr {" - " key: 'T'" - " value {" - " type: DT_FLOAT" - " }" - " }" - " }" - " ret {" - " key: 'm'" - " value: 'matmul:product'" - " }", + absl::StrCat(" signature {" + " name: 'MatMulFunction'" + " input_arg {" + " name: 'a'" + " type: DT_FLOAT" + " }" + " input_arg {" + " name: 'b'" + " type: DT_FLOAT" + " }" + " output_arg {" + " name: 'm'" + " type: DT_FLOAT" + " }" + " }" + " node_def {" + " name: 'matmul'" + " op: 'MatMul'" + " input: 'a'" + " input: 'b'" + " device: '", + matmul_device, "'", + " attr {" + " key: 'T'" + " value {" + " type: DT_FLOAT" + " }" + " }" + " }" + " ret {" + " key: 'm'" + " value: 'matmul:product'" + " }"), &def)); return def.SerializeAsString(); } @@ -157,7 +160,8 @@ string MatMulFunction() { // which creates a remote remote input, to simulate a scenario that the remote // input is not ready when we start running an op or a function. void TestRemoteExecuteSilentCopies(bool async, bool remote, bool func, - bool heavy_load_on_streaming_rpc) { + bool heavy_load_on_streaming_rpc, + bool remote_func_outputs = false) { tensorflow::ServerDef server_def = GetServerDef(3); // This server def has the task index set to 0. @@ -214,7 +218,8 @@ void TestRemoteExecuteSilentCopies(bool async, bool remote, bool func, TFE_Op* matmul = nullptr; if (func) { - string function_def = MatMulFunction(); + const string matmul_device = remote_func_outputs ? task2_name : ""; + string function_def = MatMulFunction(matmul_device); TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(), status); CHECK_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); @@ -250,7 +255,7 @@ void TestRemoteExecuteSilentCopies(bool async, bool remote, bool func, EXPECT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); // TODO(gjn): Add support for waiting on async local mirrors - if (!remote && !async) { + if (!remote && !async && !remote_func_outputs) { auto remote_arg = tensorflow::TensorHandleFromInterface(tensorflow::unwrap(h1_task2)); // The input handles should never change since they have been mirrored. @@ -329,6 +334,19 @@ TEST(CAPI, RemoteExecuteSilentCopiesLocalAsyncFunc) { TestRemoteExecuteSilentCopies(/*async=*/true, /*remote=*/false, /*func=*/true, /*heavy_load_on_streaming_rpc=*/false); } +// TODO(b/162618595): Enable this test once we remove the check of remote +// outputs in ProcessFunctionLibraryRuntime. +TEST(CAPI, DISABLED_RemoteExecuteSilentCopiesLocalFuncRemoteOutputs) { + TestRemoteExecuteSilentCopies(/*async=*/false, /*remote=*/false, + /*func=*/true, + /*heavy_load_on_streaming_rpc=*/false, + /*remote_func_outputs=*/true); +} +TEST(CAPI, DISABLED_RemoteExecuteSilentCopiesLocalAsyncFuncRemoteOutputs) { + TestRemoteExecuteSilentCopies(/*async=*/true, /*remote=*/false, /*func=*/true, + /*heavy_load_on_streaming_rpc=*/false, + /*remote_func_outputs=*/true); +} TEST(CAPI, RemoteExecuteSilentCopiesLocalAsyncFuncOrdering) { // A remote input may be not ready when we start running a function. Test that // the function execution should wait until the remote input is ready. diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc index e51456eaa27..4bffd887750 100644 --- a/tensorflow/core/common_runtime/eager/execute.cc +++ b/tensorflow/core/common_runtime/eager/execute.cc @@ -584,6 +584,101 @@ Status GetOrCreateKernelAndDevice( return Status::OK(); } +Status CreateUnshapedOutput( + const KernelAndDevice& kernel, const int output_num, Device* output_device, + const DataType& output_dtype, + const absl::optional& remote_func_params, + EagerContext* ctx, TensorHandle** output) { +#if defined(IS_MOBILE_PLATFORM) + return errors::Unimplemented( + "Remote outputs are not available on mobile devices."); +#else // !IS_MOBILE_PLATFORM + int64 op_id; + if (remote_func_params.has_value()) { + op_id = remote_func_params.value().op_id; + } else { + return errors::InvalidArgument( + "Unable to find a remote op id for a remote output of ", kernel.name()); + } + string remote_task; + if (!DeviceNameUtils::GetTaskName(output_device->parsed_name(), + &remote_task)) { + return errors::InvalidArgument( + "Unable to find remote task corresponding to device ", + output_device->name()); + } + *output = TensorHandle::CreateUnshapedRemoteHandle( + op_id, output_num, remote_task, output_dtype, output_device, ctx); + return Status::OK(); +#endif // !IS_MOBILE_PLATFORM +} + +Status AddOrExecuteNode(core::RefCountPtr kernel, + EagerOperation* op, TensorHandle** retvals) { + EagerExecutor& executor = op->Executor(); + EagerContext& ctx = op->EagerContext(); + GraphCollector* graph_collector = nullptr; + if (ctx.ShouldStoreGraphs()) { + graph_collector = ctx.GetGraphCollector(); + } + const int num_outputs = kernel->num_outputs(); + absl::optional remote_func_params = + op->remote_func_params(); + if (kernel->IsCrossProcess() && !remote_func_params.has_value()) { + // Create an eager op id for a cross-process function if not exist. +#if defined(IS_MOBILE_PLATFORM) + return errors::Unimplemented( + "Cross-process functions are not supported on mobile devices."); +#else // !IS_MOBILE_PLATFORM + const int64 op_id = ctx.RemoteMgr()->NextOpId(); + remote_func_params = + EagerRemoteFunctionParams{op_id, /*step_id=*/absl::nullopt}; +#endif // !IS_MOBILE_PLATFORM + } + if (executor.Async()) { + const DataTypeVector& output_dtypes = kernel->output_dtypes(); + for (int i = 0, end = num_outputs; i < end; ++i) { + Device* output_device = ctx.CanonicalDevice(kernel->OutputDevice(i)); + if (output_device == nullptr || output_device->IsLocal()) { + retvals[i] = TensorHandle::CreateEmptyLocalHandle( + /* d= */ output_device, /* op_device= */ kernel->device(), + /* resource_device= */ kernel->OutputResourceDevice(i), + output_dtypes[i], &ctx); + } else { + TF_RETURN_IF_ERROR( + CreateUnshapedOutput(*kernel, i, output_device, output_dtypes[i], + remote_func_params, &ctx, &retvals[i])); + } + } + auto node = absl::make_unique( + &ctx, op->Inputs(), remote_func_params, std::move(kernel), + graph_collector, op->GetCancellationManager(), + absl::Span(retvals, num_outputs), op->GetStackTrace()); + // Release the inputs from the eager operation since the AsyncExecuteNode + // would have taken ownership. This allows the inputs to be forwarded if + // possible. + op->Clear(); + // For async mode, execution order will make sure that all + // input handles are ready before executing them. + // TODO(b/137118203): Consider executing "cheap" kernels inline for + // performance. + return executor.AddOrExecute(std::move(node)); + } else { + for (int i = 0, end = num_outputs; i < end; ++i) { + retvals[i] = nullptr; + } + ExecuteNode node(&ctx, op->Inputs(), remote_func_params, kernel, + graph_collector, op->GetCancellationManager(), + {retvals, static_cast(num_outputs)}); + Status s = executor.SyncExecute(&node); + // We release the inputs AFTER executing the operation in sync mode since + // ExecuteNode does not increment the reference count and thus does not have + // ownership of the inputs while executing. + op->Clear(); + return s; + } +} + // There are a lot of references to devices in this function and around. // Here is what they mean: // EagerOperation::Device(): The device on which the user requested the op @@ -626,47 +721,7 @@ Status EagerLocalExecute(EagerOperation* op, TensorHandle** retvals, } } - GraphCollector* graph_collector = nullptr; - if (ctx.ShouldStoreGraphs()) { - graph_collector = ctx.GetGraphCollector(); - } - - Status s; - if (executor.Async()) { - const DataTypeVector& output_dtypes = kernel->output_dtypes(); - for (int i = 0, end = num_outputs; i < end; ++i) { - retvals[i] = TensorHandle::CreateEmptyLocalHandle( - /* d= */ ctx.CanonicalDevice(kernel->OutputDevice(i)), - /* op_device= */ kernel->device(), - /* resource_device= */ kernel->OutputResourceDevice(i), - output_dtypes[i], &ctx); - } - auto node = absl::make_unique( - &ctx, op->Inputs(), op->remote_func_params(), std::move(kernel), - graph_collector, op->GetCancellationManager(), - absl::Span(retvals, num_outputs), op->GetStackTrace()); - // Release the inputs from the eager operation since the AsyncExecuteNode - // would have taken ownership. This allows the inputs to be forwarded if - // possible. - op->Clear(); - // For async mode, execution order will make sure that all - // input handles are ready before executing them. - // TODO(b/137118203): Consider executing "cheap" kernels inline for - // performance. - s = executor.AddOrExecute(std::move(node)); - } else { - for (int i = 0, end = num_outputs; i < end; ++i) { - retvals[i] = nullptr; - } - ExecuteNode node(&ctx, op->Inputs(), op->remote_func_params(), kernel, - graph_collector, op->GetCancellationManager(), - {retvals, static_cast(num_outputs)}); - s = executor.SyncExecute(&node); - // We release the inputs AFTER executing the operation in sync mode since - // ExecuteNode does not increment the reference count and thus does not have - // ownership of the inputs while executing. - op->Clear(); - } + Status s = AddOrExecuteNode(std::move(kernel), op, retvals); // Since the operation failed, we need to Unref any outputs if they were // allocated. if (!s.ok()) { @@ -917,18 +972,34 @@ Status EagerRemoteExecute(EagerOperation* op, TensorHandle** retvals, } #endif // IS_MOBILE_PLATFORM -Status GetKernelOutputs(std::vector* outputs, int num_outputs, - TensorHandle** retvals, EagerContext* ctx, - KernelAndDevice* kernel) { +Status GetKernelOutputs( + std::vector* outputs, int num_outputs, + TensorHandle** retvals, EagerContext* ctx, KernelAndDevice* kernel, + const absl::optional& remote_func_params) { for (int i = 0, end = num_outputs; i < end; ++i) { if (retvals[i] == nullptr) { - retvals[i] = TensorHandle::CreateLocalHandle( - std::move((*outputs)[i]), - /* d= */ ctx->CanonicalDevice(kernel->OutputDevice(i)), - /* op_device= */ kernel->device(), - /* resource_device= */ kernel->OutputResourceDevice(i), ctx); + EagerKernelRet& ret = (*outputs)[i]; + Device* output_device = ctx->CanonicalDevice(kernel->OutputDevice(i)); + if (ret.index() == 0) { + retvals[i] = TensorHandle::CreateLocalHandle( + std::move(absl::get(ret)), + /* d= */ output_device, + /* op_device= */ kernel->device(), + /* resource_device= */ kernel->OutputResourceDevice(i), ctx); + } else { + const DataTypeVector& output_dtypes = kernel->output_dtypes(); + TF_RETURN_IF_ERROR( + CreateUnshapedOutput(*kernel, i, output_device, output_dtypes[i], + remote_func_params, ctx, &retvals[i])); +#if !defined(IS_MOBILE_PLATFORM) + TF_RETURN_IF_ERROR( + retvals[i]->SetRemoteShape(absl::get(ret), + output_device, ctx->GetContextViewId())); +#endif // IS_MOBILE_PLATFORM + } } else { - if (TF_PREDICT_FALSE(kernel->device() != retvals[i]->op_device())) { + if (!kernel->IsFunction() && + TF_PREDICT_FALSE(kernel->device() != retvals[i]->op_device())) { return errors::Internal( "Kernel output tensor handle has a different op device than the " "kernel. This should never happen."); @@ -940,9 +1011,21 @@ Status GetKernelOutputs(std::vector* outputs, int num_outputs, "the specified kernel output device. This should never happen."); } - TF_RETURN_IF_ERROR( - retvals[i]->SetTensor(std::move((*outputs)[i]), - ctx->CanonicalDevice(kernel->OutputDevice(i)))); + EagerKernelRet& ret = (*outputs)[i]; + if (ret.index() == 0) { + TF_RETURN_IF_ERROR(retvals[i]->SetTensor( + std::move(absl::get(ret)), + ctx->CanonicalDevice(kernel->OutputDevice(i)))); + } else { +#if defined(IS_MOBILE_PLATFORM) + return errors::Unimplemented( + "Remote outputs are not available on mobile devices."); +#else // !IS_MOBILE_PLATFORM + TF_RETURN_IF_ERROR(retvals[i]->SetRemoteShape( + absl::get(ret), + absl::get(retvals[i]->device()), ctx->GetContextViewId())); +#endif // !IS_MOBILE_PLATFORM + } } } return Status::OK(); @@ -1022,7 +1105,7 @@ Status EagerKernelExecute( absl::Span retvals) { profiler::TraceMe activity("EagerKernelExecute", profiler::TraceMeLevel::kInfo); - std::vector outputs(1); + std::vector outputs(1); ExecuteNodeArgs inputs(op_inputs.size()); TF_RETURN_IF_ERROR(inputs.Init(ctx, op_inputs, kernel)); @@ -1047,7 +1130,7 @@ Status EagerKernelExecute( "happen. Please file a bug with the TensorFlow team."); } return GetKernelOutputs(&outputs, retvals.size(), retvals.data(), ctx, - kernel.get()); + kernel.get(), remote_func_params); } namespace { @@ -1229,7 +1312,7 @@ void EagerKernelExecuteAsync( GraphCollector* graph_collector, CancellationManager* cancellation_manager, TensorHandle** retvals, int num_outputs, StatusCallback done) { auto inputs = std::make_shared(op_inputs.size()); - auto outputs = std::make_shared>(1); + auto outputs = std::make_shared>(1); Status s = inputs->Init(ctx, op_inputs, kernel); if (!s.ok()) { @@ -1242,7 +1325,8 @@ void EagerKernelExecuteAsync( ctx->StepContainer(), *inputs, outputs.get(), cancellation_manager, remote_func_params, [retvals, inputs, outputs, num_outputs, ctx, graph_collector, - kernel_raw = kernel.get(), done = std::move(done)](const Status& s) { + remote_func_params, kernel_raw = kernel.get(), + done = std::move(done)](const Status& s) { auto wrapped_done = [&](const Status& s) { kernel_raw->Unref(); done(s); @@ -1256,7 +1340,7 @@ void EagerKernelExecuteAsync( } DCHECK_EQ(num_outputs, outputs->size()); wrapped_done(GetKernelOutputs(outputs.get(), num_outputs, retvals, ctx, - kernel_raw)); + kernel_raw, remote_func_params)); }); } } // namespace @@ -1316,7 +1400,12 @@ void EagerLocalExecuteAsync(EagerOperation* op, TensorHandle** retvals, } for (int i = 0, end = num_outputs; i < end; ++i) { - retvals[i] = nullptr; + const DataTypeVector& output_dtypes = kernel->output_dtypes(); + retvals[i] = TensorHandle::CreateEmptyLocalHandle( + /* d= */ ctx.CanonicalDevice(kernel->OutputDevice(i)), + /* op_device= */ kernel->device(), + /* resource_device= */ kernel->OutputResourceDevice(i), + output_dtypes[i], &ctx); } EagerKernelExecuteAsync( diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc index 46aea040295..00d832365e9 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device.cc +++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc @@ -239,7 +239,8 @@ struct OpExecutionState : public core::RefCounted { Status KernelAndDeviceOp::Run( ScopedStepContainer* step_container, const EagerKernelArgs& inputs, - std::vector* outputs, CancellationManager* cancellation_manager, + std::vector* outputs, + CancellationManager* cancellation_manager, const absl::optional& remote_func_params) { OpKernelContext::Params params; params.device = device_; @@ -316,7 +317,8 @@ Status KernelAndDeviceOp::Run( Status KernelAndDeviceFunc::Run( ScopedStepContainer* step_container, const EagerKernelArgs& inputs, - std::vector* outputs, CancellationManager* cancellation_manager, + std::vector* outputs, + CancellationManager* cancellation_manager, const absl::optional& remote_func_params) { Notification n; Status status; @@ -331,7 +333,8 @@ Status KernelAndDeviceFunc::Run( void KernelAndDeviceFunc::RunAsync( ScopedStepContainer* step_container, const EagerKernelArgs& inputs, - std::vector* outputs, CancellationManager* cancellation_manager, + std::vector* outputs, + CancellationManager* cancellation_manager, const absl::optional& remote_func_params, std::function done) { std::shared_ptr opts = nullptr; @@ -392,25 +395,13 @@ void KernelAndDeviceFunc::RunAsync( }, profiler::ContextType::kTfExecutor, opts->step_id, profiler::TraceMeLevel::kInfo); - std::vector* function_rets = new std::vector; - pflr_->Run(*opts, handle_, inputs, function_rets, - [opts, outputs, function_rets, rendezvous, local_cm, - step_container, this, done = std::move(done)](const Status& s) { + pflr_->Run(*opts, handle_, inputs, outputs, + [opts, rendezvous, local_cm, step_container, this, + done = std::move(done)](const Status& s) { rendezvous->Unref(); if (step_container == nullptr) { this->step_container_.CleanUp(); } - if (s.ok()) { - // TODO(b/162618595): Change the type of `outputs` to - // support TensorShapes for remote outputs and remove the - // FunctionRet to Tensor conversion here. - for (const auto& ret : *function_rets) { - if (ret.index() == 0) { - outputs->push_back(absl::get(ret)); - } - } - } - delete function_rets; done(s); }); } diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.h b/tensorflow/core/common_runtime/eager/kernel_and_device.h index 87c2d7a5510..7bf4afbaf24 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device.h +++ b/tensorflow/core/common_runtime/eager/kernel_and_device.h @@ -82,6 +82,8 @@ class EagerKernelArgs : public FunctionArgsInterface { gtl::InlinedVector tensor_args_; }; +typedef absl::variant EagerKernelRet; + // KernelAndDevice encapsulates the logic needed to run a computation eagerly. // The computation can be a single instantiated kernel (implemented by // KernelAndDeviceOp below) or a multi-device function (implemented by @@ -124,10 +126,13 @@ class KernelAndDevice : public core::RefCounted { virtual bool IsFunction() { return false; } + virtual bool IsCrossProcess() { return false; } + // TODO(ashankar): Handle list-valued inputs. virtual Status Run( ScopedStepContainer* step_container, const EagerKernelArgs& inputs, - std::vector* outputs, CancellationManager* cancellation_manager, + std::vector* outputs, + CancellationManager* cancellation_manager, const absl::optional& remote_func_params) = 0; // Execute kernel asynchronously when applicable. Different from `Run` which @@ -140,7 +145,8 @@ class KernelAndDevice : public core::RefCounted { // from sync execution. virtual void RunAsync( ScopedStepContainer* step_container, const EagerKernelArgs& inputs, - std::vector* outputs, CancellationManager* cancellation_manager, + std::vector* outputs, + CancellationManager* cancellation_manager, const absl::optional& remote_func_params, StatusCallback done) = 0; @@ -203,14 +209,15 @@ class KernelAndDeviceOp final : public KernelAndDevice { GraphCollector* graph_collector) override; Status Run(ScopedStepContainer* step_container, const EagerKernelArgs& inputs, - std::vector* outputs, + std::vector* outputs, CancellationManager* cancellation_manager, const absl::optional& remote_func_params) override; void RunAsync( ScopedStepContainer* step_container, const EagerKernelArgs& inputs, - std::vector* outputs, CancellationManager* cancellation_manager, + std::vector* outputs, + CancellationManager* cancellation_manager, const absl::optional& remote_func_params, StatusCallback done) override { // Trivial async implementation on top of the sync version @@ -288,6 +295,8 @@ class KernelAndDeviceFunc : public KernelAndDevice { bool IsFunction() override { return true; }; + bool IsCrossProcess() override { return is_cross_process_; } + Status InstantiateFunc(const Context& ctx, const NodeDef& ndef, GraphCollector* graph_collector); @@ -295,14 +304,15 @@ class KernelAndDeviceFunc : public KernelAndDevice { GraphCollector* graph_collector) override; Status Run(ScopedStepContainer* step_container, const EagerKernelArgs& inputs, - std::vector* outputs, + std::vector* outputs, CancellationManager* cancellation_manager, const absl::optional& remote_func_params) override; void RunAsync( ScopedStepContainer* step_container, const EagerKernelArgs& inputs, - std::vector* outputs, CancellationManager* cancellation_manager, + std::vector* outputs, + CancellationManager* cancellation_manager, const absl::optional& remote_func_params, StatusCallback done) override; diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc index a7aac4a8f6d..33e85b25fb4 100644 --- a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc +++ b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc @@ -133,7 +133,7 @@ void BM_KernelAndDeviceRun(int iters) { gtl::InlinedVector inputs; inputs.push_back(TensorValue(&t)); inputs.push_back(TensorValue(&t)); - std::vector outputs; + std::vector outputs; NodeDef ndef(AttrBuilder("MatMul") .Set("T", DT_FLOAT) .Set("transpose_a", false) diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.cc b/tensorflow/core/common_runtime/process_function_library_runtime.cc index b31b2b78bf0..3248d3f10a7 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime.cc @@ -999,30 +999,30 @@ Status ProcessFunctionLibraryRuntime::GetOutputDevices( for (const auto& pair : data->glue_) { const ComponentFunctionData& comp_data = pair.second; DCHECK(comp_data.ret_alloc_attrs.size() == comp_data.ret_indices.size()); + if (comp_data.ret_indices.empty()) { + continue; + } const string& target = pair.first; FunctionLibraryRuntime* target_flr = GetFLR(target); + Device* target_device = nullptr; if (target_flr == nullptr) { - if (!comp_data.ret_indices.empty()) { - return errors::Unimplemented( - "Currently, outputting tensors on remote devices is not supported. " - "The ", - comp_data.ret_indices[0], - "-th return value of the function outputs to target_device: ", - target, - " Please copy the tensor to local device explicitly using " - "tf.identity and return the new Tensor instead."); - } - continue; + // TODO(b/162618595): Remove this error once we support a remote + // multi-device function with remote outputs. + return errors::Unimplemented( + "Currently, outputting tensors on remote devices is not supported." + "The ", + comp_data.ret_indices[0], + "-th return value of the function outputs to target_device: ", target, + " Please copy the tensor to local device explicitly using " + "tf.identity and return the new Tensor instead."); + } else { + target_device = target_flr->device(); } - Device* target_device = target_flr->device(); - const FunctionBody* fbody = target_flr->GetFunctionBody(comp_data.handle); - DCHECK(fbody != nullptr); - output_devices->resize(data->num_outputs_); for (int j = 0; j < comp_data.ret_indices.size(); ++j) { int ret_index = comp_data.ret_indices[j]; - if (fbody->ret_types[j] == DT_RESOURCE) { + if (data->ret_types_[ret_index] == DT_RESOURCE) { (*output_devices)[ret_index] = target_device; } else { (*output_devices)[ret_index] = diff --git a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc index be81355cbc8..76fc12d1adc 100644 --- a/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc +++ b/tensorflow/core/distributed_runtime/eager/eager_service_impl_test.cc @@ -771,12 +771,17 @@ class FunctionWithRemoteInputsTest : public EagerServiceImplTest { &close_context_response)); } - void CheckOutputsAndClose(const int64 op_id) { + void CheckOutputsAndClose(const std::vector& outputs, + const int64 op_id) { const tensorflow::Tensor* t = nullptr; tensorflow::TensorHandle* tensor_handle; TF_ASSERT_OK(eager_service_impl_.GetTensorHandle( context_id_, RemoteTensorHandleInternal(2, 0), &tensor_handle)); TF_ASSERT_OK(tensor_handle->Tensor(&t)); + EXPECT_EQ(outputs.size(), 1); + EXPECT_EQ(outputs.at(0).index(), 1); + const TensorShape& shape = absl::get(outputs.at(0)); + EXPECT_EQ(shape, t->shape()); CheckOutputTensorAndClose(*t); } @@ -845,11 +850,7 @@ TEST_F(FunctionWithRemoteInputsTest, EagerPFLRTest) { }); done.WaitForNotification(); TF_ASSERT_OK(status); - EXPECT_EQ(outputs.size(), 1); - EXPECT_EQ(outputs.at(0).index(), 1); - const TensorShape& shape = absl::get(outputs.at(0)); - EXPECT_EQ(shape, TensorShape({2, 2})); - CheckOutputsAndClose(op_id); + CheckOutputsAndClose(outputs, op_id); } // Test executes a remote function with local input and output tensors. @@ -940,13 +941,13 @@ TEST_F(FunctionWithRemoteInputsTest, KernelAndDeviceFuncTest) { *handle = remote_handles.at(index); return Status::OK(); }); - std::vector outputs; + std::vector outputs; TF_ASSERT_OK(kernel->Run(/*step_container=*/nullptr, inputs, &outputs, /*cancellation_manager=*/nullptr, /*remote_func_params=*/absl::nullopt)); - CheckOutputsAndClose(op_id); + CheckOutputsAndClose(outputs, op_id); } // Test executes a remote function through KernelAndDeviceFunc::RunAsync. @@ -987,7 +988,7 @@ TEST_F(FunctionWithRemoteInputsTest, KernelAndDeviceFuncAsyncTest) { *handle = remote_handles.at(index); return Status::OK(); }); - std::vector outputs; + std::vector outputs; Status status; Notification n; @@ -1000,7 +1001,7 @@ TEST_F(FunctionWithRemoteInputsTest, KernelAndDeviceFuncAsyncTest) { }); n.WaitForNotification(); TF_ASSERT_OK(status); - CheckOutputsAndClose(op_id); + CheckOutputsAndClose(outputs, op_id); } // Test creates a context and attempts to send a tensor (using the RPC), and diff --git a/tensorflow/core/distributed_runtime/eager/remote_copy_node.cc b/tensorflow/core/distributed_runtime/eager/remote_copy_node.cc index d4b5fe38964..a1d0e09faf9 100644 --- a/tensorflow/core/distributed_runtime/eager/remote_copy_node.cc +++ b/tensorflow/core/distributed_runtime/eager/remote_copy_node.cc @@ -192,9 +192,20 @@ Status RemoteCopyNode::RunLocalRecv(EagerOperation* op, TF_RETURN_IF_ERROR(CreateUncachedKernelAndDeviceOp(op, &kernel)); EagerKernelArgs args; - return kernel->Run(/*step_container*/ nullptr, args, outputs, - captured_state_->recv_cancellation(), - /*remote_func_params=*/absl::nullopt); + std::vector rets; + TF_RETURN_IF_ERROR(kernel->Run(/*step_container*/ nullptr, args, &rets, + captured_state_->recv_cancellation(), + /*remote_func_params=*/absl::nullopt)); + outputs->clear(); + for (const auto& ret : rets) { + if (ret.index() == 0) { + outputs->push_back(absl::get(ret)); + } else { + return errors::Internal( + "Expect to receive a Tensor but got a TensorShape."); + } + } + return Status::OK(); } void RemoteCopyNode::RunRemoteRecv(EagerOperation* op, StatusCallback done) { From 79448c479679170b84354ff452ca31a5f34007f5 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Mon, 10 Aug 2020 10:53:55 -0700 Subject: [PATCH 0753/1017] Touch ~/.bigqueryrc when running bq commands "bq load" corrupts output on its first invocation if ~/.bigqueryrc doesn't exist. AFAIK this cannot be turned off. PiperOrigin-RevId: 325840422 Change-Id: Iefe81ac14b1b4679530fca152e7f336dc9e76f68 --- tensorflow/tools/ci_build/sizetrack_helper.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tensorflow/tools/ci_build/sizetrack_helper.py b/tensorflow/tools/ci_build/sizetrack_helper.py index ff5ff1bf60d..8377d733c56 100755 --- a/tensorflow/tools/ci_build/sizetrack_helper.py +++ b/tensorflow/tools/ci_build/sizetrack_helper.py @@ -54,9 +54,11 @@ import csv import datetime import os import os.path +import pathlib import platform import subprocess + parser = argparse.ArgumentParser( usage=__doc__, formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( @@ -231,6 +233,15 @@ def gcloud(tool, args, stdin=None): return ret.stdout.strip() +def bq(args, stdin=None): + """Helper for running bq, the BigQuery tool.""" + # bq prints extra messages to stdout if ~/.bigqueryrc doesn't exist + pathlib.Path(pathlib.Path.home() / ".bigqueryrc").touch() + return gcloud( + "bq", ["--project_id", FLAGS.project, "--headless", *args], + stdin=stdin) + + def get_all_tested_commits(): """Get details about the full commit range tested by this invocation.""" head_info = git_pretty("HEAD", PRETTY_HEAD_INFO, n=1) @@ -245,12 +256,8 @@ def get_all_tested_commits(): # --format=csv returns an empty string if no results, or else two lines: # commit # COMMIT_HASH - earliest_commit = gcloud( - "bq", [ - "--project_id", FLAGS.project, "--headless", "-q", "query", - "--format", "csv", "--nouse_legacy_sql" - ], - stdin=query_earliest_included_commit) + earliest_commit = bq(["query", "--format", "csv", "--nouse_legacy_sql"], + stdin=query_earliest_included_commit) # Compute the commit/CL range since the last test if earliest_commit: @@ -359,9 +366,8 @@ def main(): with open("data.tsv", "w") as tsvfile: writer = csv.writer(tsvfile, delimiter="\t", quoting=csv.QUOTE_MINIMAL) writer.writerow(next_tsv_row) - gcloud("bq", [ - "--project_id", FLAGS.project, "--headless", "-q", "load", - "--source_format", "CSV", "--field_delimiter", "tab", + bq([ + "load", "--source_format", "CSV", "--field_delimiter", "tab", PROJECT_LEVEL_TABLE_NAME, "data.tsv", SCHEMA ]) From 7c991932387ac05f727dec82c88329ae5f97b3f5 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Mon, 10 Aug 2020 10:57:19 -0700 Subject: [PATCH 0754/1017] Fix the typo in the API name for "MobileNetV3Samll" PiperOrigin-RevId: 325841242 Change-Id: Ie05542d29674c5e3d8ea8e51799094256e90ee52 --- tensorflow/python/keras/applications/mobilenet_v3.py | 2 +- .../tools/api/golden/v1/tensorflow.keras.applications.pbtxt | 2 +- .../tools/api/golden/v2/tensorflow.keras.applications.pbtxt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/applications/mobilenet_v3.py b/tensorflow/python/keras/applications/mobilenet_v3.py index bdf2ca40142..44ba6fd3a39 100644 --- a/tensorflow/python/keras/applications/mobilenet_v3.py +++ b/tensorflow/python/keras/applications/mobilenet_v3.py @@ -348,7 +348,7 @@ def MobileNetV3(stack_fn, return model -@keras_export('keras.applications.MobileNetV3Samll') +@keras_export('keras.applications.MobileNetV3Small') def MobileNetV3Small(input_shape=None, alpha=1.0, minimalistic=False, diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.pbtxt index 9f367742398..60c9b6d2909 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.applications.pbtxt @@ -125,7 +125,7 @@ tf_module { argspec: "args=[\'input_shape\', \'alpha\', \'minimalistic\', \'include_top\', \'weights\', \'input_tensor\', \'classes\', \'pooling\', \'dropout_rate\', \'classifier_activation\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'False\', \'True\', \'imagenet\', \'None\', \'1000\', \'None\', \'0.2\', \'softmax\'], " } member_method { - name: "MobileNetV3Samll" + name: "MobileNetV3Small" argspec: "args=[\'input_shape\', \'alpha\', \'minimalistic\', \'include_top\', \'weights\', \'input_tensor\', \'classes\', \'pooling\', \'dropout_rate\', \'classifier_activation\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'False\', \'True\', \'imagenet\', \'None\', \'1000\', \'None\', \'0.2\', \'softmax\'], " } member_method { diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.pbtxt index 9f367742398..60c9b6d2909 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.applications.pbtxt @@ -125,7 +125,7 @@ tf_module { argspec: "args=[\'input_shape\', \'alpha\', \'minimalistic\', \'include_top\', \'weights\', \'input_tensor\', \'classes\', \'pooling\', \'dropout_rate\', \'classifier_activation\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'False\', \'True\', \'imagenet\', \'None\', \'1000\', \'None\', \'0.2\', \'softmax\'], " } member_method { - name: "MobileNetV3Samll" + name: "MobileNetV3Small" argspec: "args=[\'input_shape\', \'alpha\', \'minimalistic\', \'include_top\', \'weights\', \'input_tensor\', \'classes\', \'pooling\', \'dropout_rate\', \'classifier_activation\'], varargs=None, keywords=None, defaults=[\'None\', \'1.0\', \'False\', \'True\', \'imagenet\', \'None\', \'1000\', \'None\', \'0.2\', \'softmax\'], " } member_method { From 4fcaeef9048077099bb20d5b40ced6edf68eaa2b Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 10 Aug 2020 11:04:33 -0700 Subject: [PATCH 0755/1017] Re-introduce some infeed and outfeed ops after a revert caused by failing tests. PiperOrigin-RevId: 325843303 Change-Id: I081e2ba0437fe9812ea61cf90d6bef62c50b3908 --- tensorflow/core/tpu/kernels/BUILD | 109 ++++ .../core/tpu/kernels/image_resize_ops.cc | 155 +++++ tensorflow/core/tpu/kernels/infeed_ops.cc | 548 ++++++++++++++++++ tensorflow/core/tpu/kernels/infeed_ops.h | 69 +++ tensorflow/core/tpu/kernels/outfeed_ops.cc | 116 ++++ tensorflow/core/tpu/kernels/outfeed_ops.h | 69 +++ .../core/tpu/kernels/replication_ops.cc | 27 + .../core/tpu/kernels/tpu_handle_to_key_op.cc | 62 ++ tensorflow/core/tpu/kernels/transfer_ops.cc | 98 ++++ tensorflow/core/tpu/kernels/transfer_ops.h | 56 ++ tensorflow/core/tpu/tpu_library_init_fns.inc | 1 + 11 files changed, 1310 insertions(+) create mode 100644 tensorflow/core/tpu/kernels/image_resize_ops.cc create mode 100644 tensorflow/core/tpu/kernels/infeed_ops.cc create mode 100644 tensorflow/core/tpu/kernels/infeed_ops.h create mode 100644 tensorflow/core/tpu/kernels/outfeed_ops.cc create mode 100644 tensorflow/core/tpu/kernels/outfeed_ops.h create mode 100644 tensorflow/core/tpu/kernels/replication_ops.cc create mode 100644 tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc create mode 100644 tensorflow/core/tpu/kernels/transfer_ops.cc create mode 100644 tensorflow/core/tpu/kernels/transfer_ops.h diff --git a/tensorflow/core/tpu/kernels/BUILD b/tensorflow/core/tpu/kernels/BUILD index 1336f52ed34..6d3369022ad 100644 --- a/tensorflow/core/tpu/kernels/BUILD +++ b/tensorflow/core/tpu/kernels/BUILD @@ -28,10 +28,16 @@ tf_kernel_library( deps = [ ":cross_replica_ops", ":host_compute_ops", + ":image_resize_ops", + ":infeed_ops", + ":outfeed_ops", + ":replication_ops", ":topk_ops", ":tpu_compile_op", ":tpu_configuration_ops", ":tpu_execute_op", + ":tpu_handle_to_key_op", + ":transfer_ops", ], ) @@ -684,3 +690,106 @@ cc_library( ], alwayslink = 1, ) + +cc_library( + name = "infeed_ops", + srcs = ["infeed_ops.cc"], + hdrs = ["infeed_ops.h"], + visibility = ["//visibility:public"], + deps = [ + ":transfer_ops", + "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", + "//tensorflow/compiler/tf2xla:common", + "//tensorflow/compiler/xla:util", + "//tensorflow/core:framework", + "//tensorflow/core/common_runtime:dma_helper", + "//tensorflow/core/framework:protos_all_cc", + "//tensorflow/core/kernels:transpose_functor", + "//tensorflow/core/platform:status", + "//tensorflow/core/profiler/lib:traceme", + "//tensorflow/core/tpu:tpu_api", + "//tensorflow/core/tpu:tpu_defs", + "//tensorflow/stream_executor:multi_platform_manager", + "//tensorflow/stream_executor/tpu:c_api_conversions", + "//tensorflow/stream_executor/tpu:tpu_transfer_manager_base", + "//tensorflow/stream_executor/tpu:tpu_transfer_manager_interface", + ], + alwayslink = True, +) + +cc_library( + name = "transfer_ops", + srcs = ["transfer_ops.cc"], + hdrs = ["transfer_ops.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", + "//tensorflow/core:framework", + "//tensorflow/core:lib_internal", + "//tensorflow/core/kernels:ops_util", + "//tensorflow/core/profiler/lib:traceme", + "//tensorflow/stream_executor:multi_platform_manager", + "//tensorflow/stream_executor/tpu:tpu_node_context", + "//tensorflow/stream_executor/tpu:tpu_platform_interface", + "//tensorflow/stream_executor/tpu:tpu_transfer_manager_interface", + ], + alwayslink = True, +) + +cc_library( + name = "outfeed_ops", + srcs = ["outfeed_ops.cc"], + hdrs = ["outfeed_ops.h"], + visibility = ["//visibility:public"], + deps = [ + ":transfer_ops", + "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", + "//tensorflow/compiler/tf2xla:common", + "//tensorflow/core:framework", + "//tensorflow/core/framework:protos_all_cc", + "//tensorflow/core/tpu:tpu_defs", + "//tensorflow/stream_executor:multi_platform_manager", + ], + alwayslink = True, +) + +cc_library( + name = "image_resize_ops", + srcs = ["image_resize_ops.cc"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/tf2xla:common", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/xla/client:xla_builder", + "//tensorflow/compiler/xla/client/lib:constants", + "//tensorflow/core:framework", + "//tensorflow/core/tpu:tpu_defs", + "@com_google_absl//absl/strings", + ], + alwayslink = True, +) + +cc_library( + name = "replication_ops", + srcs = ["replication_ops.cc"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/jit:xla_device_no_jit_rewrite_registration", + "//tensorflow/core:framework", + "//tensorflow/core/tpu:tpu_defs", + ], + alwayslink = True, +) + +cc_library( + name = "tpu_handle_to_key_op", + srcs = ["tpu_handle_to_key_op.cc"], + visibility = ["//visibility:public"], + deps = [ + ":tpu_compilation_cache_interface", + ":tpu_op_consts", + "//tensorflow/core:framework", + "//tensorflow/core/tpu:tpu_configuration", + ], + alwayslink = True, +) diff --git a/tensorflow/core/tpu/kernels/image_resize_ops.cc b/tensorflow/core/tpu/kernels/image_resize_ops.cc new file mode 100644 index 00000000000..fd0f5e4c7a6 --- /dev/null +++ b/tensorflow/core/tpu/kernels/image_resize_ops.cc @@ -0,0 +1,155 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/tpu/tpu_defs.h" + +namespace tensorflow { + +class TpuCustomResizeOp : public XlaOpKernel { + public: + explicit TpuCustomResizeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("align_corners", &align_corners_)); + OP_REQUIRES_OK(ctx, + ctx->GetAttr("half_pixel_centers", &half_pixel_centers_)); + } + + xla::Shape GetOutputShape(XlaOpKernelContext* ctx) const { + std::vector out_size; + auto status = ctx->ConstantInputAsIntVector(1, &out_size); + CHECK_EQ(out_size.size(), 2) << status.ToString(); + xla::Shape output_shape = + TensorShapeToXLAShape(ctx->output_xla_type(0), ctx->InputShape(0)); + output_shape.mutable_dimensions()[1] = out_size[0]; + output_shape.mutable_dimensions()[2] = out_size[1]; + return output_shape; + } + + string OpaqueField() const { + return absl::StrCat("\"", align_corners_, half_pixel_centers_, "\""); + } + + void CompileGrad(XlaOpKernelContext* ctx, const char* target, + const xla::Shape& output_shape) { + auto input_shape = + TensorShapeToXLAShape(ctx->output_xla_type(0), ctx->InputShape(0)); + if (ctx->InputShape(1).dim_sizes() == ctx->InputShape(0).dim_sizes()) { + ctx->SetOutput( + 0, xla::ConvertElementType(ctx->Input(0), ctx->output_xla_type(0))); + return; + } + // The gradient should be done in two phases for large resizes. + auto input = ctx->Input(0); + if (input_shape.dimensions(1) / output_shape.dimensions(1) > 3 && + input_shape.dimensions(2) / output_shape.dimensions(2) > 3) { + auto intermediate_shape = output_shape; + intermediate_shape.mutable_dimensions()[1] = input_shape.dimensions(1); + input = xla::CustomCall(ctx->builder(), target, {ctx->Input(0)}, + intermediate_shape, OpaqueField()); + } + ctx->SetOutput(0, xla::CustomCall(ctx->builder(), target, {input}, + output_shape, OpaqueField())); + } + + void CompileForward(XlaOpKernelContext* ctx, const char* target) { + auto output_shape = GetOutputShape(ctx); + if (ctx->InputShape(0).dim_size(1) == output_shape.dimensions(1) && + ctx->InputShape(0).dim_size(2) == output_shape.dimensions(2)) { + ctx->SetOutput( + 0, xla::ConvertElementType(ctx->Input(0), ctx->output_xla_type(0))); + return; + } + if (ctx->InputShape(0).dim_size(1) == 1 && + ctx->InputShape(0).dim_size(2) == 1) { + ctx->SetOutput(0, + ctx->Input(0) + xla::Zeros(ctx->builder(), output_shape)); + return; + } + ctx->SetOutput(0, xla::CustomCall(ctx->builder(), target, {ctx->Input(0)}, + output_shape, OpaqueField())); + } + + private: + bool align_corners_; + bool half_pixel_centers_; +}; + +class TpuResizeNearestNeighborOp : public TpuCustomResizeOp { + public: + explicit TpuResizeNearestNeighborOp(OpKernelConstruction* ctx) + : TpuCustomResizeOp(ctx) {} + void Compile(XlaOpKernelContext* ctx) override { + CompileForward(ctx, "ResizeNearest"); + } +}; + +class TpuResizeBilinearOp : public TpuCustomResizeOp { + public: + explicit TpuResizeBilinearOp(OpKernelConstruction* ctx) + : TpuCustomResizeOp(ctx) {} + void Compile(XlaOpKernelContext* ctx) override { + CompileForward(ctx, "ResizeBilinear"); + } +}; + +class TpuResizeNearestNeighborGradOp : public TpuCustomResizeOp { + public: + explicit TpuResizeNearestNeighborGradOp(OpKernelConstruction* ctx) + : TpuCustomResizeOp(ctx) {} + void Compile(XlaOpKernelContext* ctx) override { + CompileGrad(ctx, "ResizeNearestGrad", GetOutputShape(ctx)); + } +}; + +class TpuResizeBilinearGradOp : public TpuCustomResizeOp { + public: + explicit TpuResizeBilinearGradOp(OpKernelConstruction* ctx) + : TpuCustomResizeOp(ctx) {} + void Compile(XlaOpKernelContext* ctx) override { + auto output_shape = + TensorShapeToXLAShape(ctx->output_xla_type(0), ctx->InputShape(1)); + CompileGrad(ctx, "ResizeBilinearGrad", output_shape); + } +}; + +REGISTER_XLA_OP(Name("ResizeNearestNeighbor") + .CompileTimeConstantInput("size") + .Device(DEVICE_TPU_XLA_JIT), + TpuResizeNearestNeighborOp); + +REGISTER_XLA_OP(Name("ResizeNearestNeighborGrad") + .CompileTimeConstantInput("size") + .Device(DEVICE_TPU_XLA_JIT), + TpuResizeNearestNeighborGradOp); + +REGISTER_XLA_OP(Name("ResizeBilinear") + .CompileTimeConstantInput("size") + .Device(DEVICE_TPU_XLA_JIT), + TpuResizeBilinearOp); + +REGISTER_XLA_OP(Name("ResizeBilinearGrad").Device(DEVICE_TPU_XLA_JIT), + TpuResizeBilinearGradOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/infeed_ops.cc b/tensorflow/core/tpu/kernels/infeed_ops.cc new file mode 100644 index 00000000000..1d10667f2c2 --- /dev/null +++ b/tensorflow/core/tpu/kernels/infeed_ops.cc @@ -0,0 +1,548 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/tpu/kernels/infeed_ops.h" + +#include +#include + +#include "tensorflow/compiler/jit/xla_device.h" +#include "tensorflow/compiler/tf2xla/literal_util.h" +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/common_runtime/dma_helper.h" +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/dataset.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/function_handle_cache.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/variant.h" +#include "tensorflow/core/framework/variant_encode_decode.h" +#include "tensorflow/core/framework/variant_tensor_data.h" +#include "tensorflow/core/kernels/transpose_functor.h" +#include "tensorflow/core/profiler/lib/traceme.h" +#include "tensorflow/core/tpu/kernels/transfer_ops.h" +#include "tensorflow/core/tpu/tpu_api.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/stream_executor/tpu/c_api_conversions.h" +#include "tensorflow/stream_executor/multi_platform_manager.h" +#include "tensorflow/stream_executor/tpu/tpu_transfer_manager.h" +#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" + +namespace tensorflow { +namespace { + +typedef Eigen::ThreadPoolDevice CPUDevice; +typedef tensorflow::tpu::NoncopyableBuffer LinearizerBuffer; +typedef std::deque LinearizerBufferList; + +// For the given shape, chooses a layout for infeed on TPU. The returned shape +// has the same dimensions as the original shape, and only the layout is +// changed. +xla::Shape GetTPUInfeedLayout(const xla::Shape& shape) { + XLA_Shape c_shape; + XLA_Shape c_infeed_shape; + + ApiConverter::ToC(shape, &c_shape); + + tpu::ExecutorApiFn()->TpuTransferManager_GetInfeedLayoutFn(&c_shape, + &c_infeed_shape); + xla::Shape infeed_shape = ApiConverter::FromC(&c_infeed_shape); + ApiConverter::Free(&c_shape); + ApiConverter::Free(&c_infeed_shape); + return infeed_shape; +} + +// Transposes the given tensor using the tensorflow C++ transpose implementation +// to obtain a XLA literal for the host tensor laid out as the given layout. The +// returned tensor is normalized to the dim0major layout -- F32[10,20,30]{2,0,1} +// is returned as F32[20,10,30]{2,1,0}. +xla::StatusOr TransposeTensor(OpKernelContext* ctx, + const Tensor& input_tensor, + const xla::Shape& xla_shape) { + profiler::TraceMe trace_me("TransposeTensor", /*level=*/2); + const int64 rank = xla_shape.rank(); + std::vector permutation(rank); + std::vector transposed_shapes(rank); + for (int64 i = 0; i < rank; ++i) { + permutation[i] = xla_shape.layout().minor_to_major(rank - 1 - i); + transposed_shapes[i] = xla_shape.dimensions(permutation[i]); + } + + Tensor transposed_tensor; + + // If this is a trivial transpose (i.e., bitcast), just create an aliased + // tensor with the transposed shape. + if (xla::LayoutUtil::IsMonotonicWithDim0Major( + xla::ShapeUtil::DropDegenerateDimensions(xla_shape).layout())) { + TensorShape shape; + TF_RETURN_IF_ERROR(TensorShapeUtils::MakeShape(transposed_shapes, &shape)); + TF_RETURN_IF_ERROR(transposed_tensor.BitcastFrom( + input_tensor, input_tensor.dtype(), shape)); + return transposed_tensor; + } + + AllocatorAttributes alloc_attr; + alloc_attr.set_on_host(true); + TF_RETURN_IF_ERROR(ctx->allocate_temp(input_tensor.dtype(), + TensorShape(transposed_shapes), + &transposed_tensor, alloc_attr)); + // Eigen Transpose fails with SIGFPE if there is a dimension of size 0. + if (input_tensor.NumElements() > 0) { + TF_RETURN_IF_ERROR(DoTranspose(ctx->eigen_device(), + input_tensor, permutation, + &transposed_tensor)); + } + return transposed_tensor; +} + +xla::StatusOr GetLayoutOverride(OpKernelConstruction* ctx, + const char* attrn_name, + std::vector* minor_to_major) { + if (!ctx->HasAttr(attrn_name)) { + return false; + } + TF_RETURN_IF_ERROR(ctx->GetAttr(attrn_name, minor_to_major)); + return !minor_to_major->empty(); +} + +Status GetInfeedShapeWithLayout(OpKernelConstruction* ctx, + const char* attrn_name, + const xla::Shape& input_shape, + xla::Shape* output_shape) { + std::vector minor_to_major; + TF_ASSIGN_OR_RETURN(bool has_override, + GetLayoutOverride(ctx, attrn_name, &minor_to_major)); + if (!has_override) { + *output_shape = input_shape; + if (output_shape->IsTuple()) { + int64 tuple_elements = xla::ShapeUtil::TupleElementCount(*output_shape); + for (int64 i = 0; i < tuple_elements; ++i) { + xla::Shape* sub_shape = + xla::ShapeUtil::GetMutableSubshape(output_shape, {i}); + *sub_shape->mutable_layout() = GetTPUInfeedLayout(*sub_shape).layout(); + } + } else { + *output_shape->mutable_layout() = + GetTPUInfeedLayout(*output_shape).layout(); + } + return Status::OK(); + } + + auto layout_func = [](const xla::Shape& shape) -> xla::Layout { + return GetTPUInfeedLayout(shape).layout(); + }; + return GetShapeWithLayout(input_shape, minor_to_major, layout_func, + output_shape); +} + +// LinearizedBuffersWrapper is an opaque C++ data structure for the outputs of +// PrelinearizeOp and PrelinearizeTupleOp. It holds the resultant linearized +// buffers and references to input tensors whose underlying storage are shared +// with linearized buffers. +// NOTE: This is not a feature-complete implementation of the DT_VARIANT +// specification. In particular, we cannot currently serialize an arbitrary +// `LinearizerBufferList` (aka `std::deque`) +// object, so the `Encode()` and `Decode()` methods are not implemented. +struct LinearizedBuffersWrapper { + explicit LinearizedBuffersWrapper() {} + explicit LinearizedBuffersWrapper(LinearizerBufferList bufs, + std::vector ts) + : buffers(std::move(bufs)), tensors(std::move(ts)) {} + LinearizedBuffersWrapper(const LinearizedBuffersWrapper& wrapper) { + // tensorflow::Variant requires this copy constructor to compile. + LOG(FATAL) << "LinearizedBuffersWrapper should not copy."; + } + LinearizedBuffersWrapper& operator=(const LinearizedBuffersWrapper& wrapper) = + delete; + LinearizedBuffersWrapper(LinearizedBuffersWrapper&&) = default; + LinearizedBuffersWrapper& operator=(LinearizedBuffersWrapper&&) = default; + ~LinearizedBuffersWrapper() = default; + + // These functions are tensorflow::Variant requirements. + string TypeName() const { return "(anonymous)::LinearizedBuffersWrapper"; } + void Encode(tensorflow::VariantTensorData* data) const { + LOG(ERROR) << "Encode() is not implemented for LinearizedBuffersWrapper " + "objects."; + } + bool Decode(const tensorflow::VariantTensorData& data) { + LOG(ERROR) << "Decode() is not implemented for LinearizedBuffersWrapper " + "objects."; + return false; + } + + LinearizerBufferList buffers; + // Save references on tensors whose underlying storage are shared with + // LiteralLinearizer::Buffer in `buffers`. + std::vector tensors; +}; + +Status AutoTransposeAndLinearize(OpKernelContext* ctx, + const Tensor& input_tensor, + const xla::Shape& shape, + LinearizerBufferList* linearized_buffers, + std::vector* saved_input_tensors) { + const Tensor* tensor = &input_tensor; + // If the given layout is not in dim0major layout, tranposes the tensor. + bool has_transposed = false; + Tensor transposed_tensor; + if (!xla::LayoutUtil::IsMonotonicWithDim0Major(shape.layout())) { + // If the given layout is not in dim0major layout, transpose the tensor. + TF_ASSIGN_OR_RETURN(transposed_tensor, + TransposeTensor(ctx, input_tensor, shape)); + tensor = &transposed_tensor; + has_transposed = true; + } + + xla::BorrowingLiteral literal; + TF_RETURN_IF_ERROR(HostTensorToBorrowingLiteral(*tensor, &literal)); + + TF_RETURN_IF_ERROR( + xla::TpuTransferManagerInterface::GetRegisteredTpuTransferManager() + ->LinearizeToBuffers(literal, linearized_buffers)); + + // The input tensor is ref-counted. Save a handle on the input tensor if + // its underlying storage is shared with linearized buffers to prevent + // input tensor from getting freed. + for (const auto& buffer : *linearized_buffers) { + if (!buffer.owns_data() && !has_transposed) { + // `buffer` is created from zero-copy fast path from the un-transposed + // input tensor so its underlying data is shared with input tensor. + // Save a handle to input tensor to increment its ref-count and avoid + // it getting deallocated after PrelinearizeTupleOp completes. + saved_input_tensors->push_back(*tensor); + // A literal can be linearized to zero to two buffers. If any of the + // linearized buffer shares storage with input tensor. We save exactly + // one handle on the input tensor. + break; + } + } + return Status::OK(); +} + +// PrelinearizeOp is used to linearize one tensor to the device format. +class PrelinearizeOp : public OpKernel { + public: + explicit PrelinearizeOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); + xla::Shape shape; + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &shape)); + OP_REQUIRES_OK(ctx, + GetInfeedShapeWithLayout(ctx, "layout", shape, &xla_shape_)); + } + + void Compute(OpKernelContext* ctx) override { + const Tensor& input_tensor = ctx->input(0); + // Validate input. + OP_REQUIRES( + ctx, input_tensor.dtype() == dtype_, + errors::InvalidArgument("Prelinearize dtype mismatch; expected ", + DataType_Name(dtype_), ", got ", + DataType_Name(input_tensor.dtype()))); + OP_REQUIRES( + ctx, input_tensor.shape() == shape_, + errors::InvalidArgument("Prelinearize shape mismatch; expected ", + shape_.DebugString(), ", got ", + input_tensor.shape().DebugString())); + + // Auto-transpose and prelinearize. + LinearizerBufferList linearized_buffers; + std::vector saved_input_tensors; + auto status = + AutoTransposeAndLinearize(ctx, input_tensor, xla_shape_, + &linearized_buffers, &saved_input_tensors); + OP_REQUIRES_OK(ctx, status); + + // Write to output. + tensorflow::Tensor* output; + OP_REQUIRES_OK(ctx, + ctx->allocate_output(0, tensorflow::TensorShape{}, &output)); + output->scalar()() = LinearizedBuffersWrapper{ + std::move(linearized_buffers), std::move(saved_input_tensors)}; + } + + bool IsExpensive() override { return true; } + + private: + TensorShape shape_; + DataType dtype_; + xla::Shape xla_shape_; + + // PrelinearizeOp is neither copyable nor movable. + PrelinearizeOp(const PrelinearizeOp&) = delete; + PrelinearizeOp& operator=(const PrelinearizeOp&) = delete; +}; + +// PrelinearizeTupleOp is used to linearize multiple tensors to the device +// format. +class PrelinearizeTupleOp : public OpKernel { + public: + explicit PrelinearizeTupleOp(OpKernelConstruction* ctx) : OpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); + OP_REQUIRES( + ctx, shapes_.size() == dtypes_.size(), + errors::InvalidArgument( + "shapes and dtypes must be the same length. shapes length = ", + shapes_.size(), ", dtypes length = ", dtypes_.size())); + + std::vector xla_shapes; + for (int i = 0; i < shapes_.size(); i++) { + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, + TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); + xla_shapes.push_back(xla_shape); + } + OP_REQUIRES_OK( + ctx, GetInfeedShapeWithLayout( + ctx, "layouts", xla::ShapeUtil::MakeTupleShape(xla_shapes), + &tuple_shape_)); + } + + void Compute(OpKernelContext* ctx) override { + OpInputList values; + OP_REQUIRES_OK(ctx, ctx->input_list("inputs", &values)); + OP_REQUIRES(ctx, values.size() == shapes_.size(), + errors::InvalidArgument( + "Wrong number of inputs to PrelinearizeTuple.")); + + LinearizerBufferList all_linearized_buffers; + std::vector all_saved_input_tensors; + for (int i = 0; i < values.size(); i++) { + // Validate input. + const Tensor& input_tensor = values[i]; + OP_REQUIRES(ctx, input_tensor.dtype() == dtypes_[i], + errors::InvalidArgument( + "PrelinearizeTuple dtype mismatch at tuple element ", i, + "; expected ", DataType_Name(dtypes_[i]), ", got ", + DataType_Name(input_tensor.dtype()))); + OP_REQUIRES(ctx, input_tensor.shape() == shapes_[i], + errors::InvalidArgument( + "PrelinearizeTuple shape mismatch at tuple element ", i, + "; expected ", shapes_[i].DebugString(), ", got ", + input_tensor.shape().DebugString())); + + // Auto-transpose and prelinearize. + LinearizerBufferList linearized_buffers; + std::vector saved_input_tensors; + auto status = AutoTransposeAndLinearize( + ctx, input_tensor, tuple_shape_.tuple_shapes(i), &linearized_buffers, + &saved_input_tensors); + OP_REQUIRES_OK(ctx, status); + all_linearized_buffers.insert( + all_linearized_buffers.end(), + std::make_move_iterator(linearized_buffers.begin()), + std::make_move_iterator(linearized_buffers.end())); + all_saved_input_tensors.insert( + all_saved_input_tensors.end(), + std::make_move_iterator(saved_input_tensors.begin()), + std::make_move_iterator(saved_input_tensors.end())); + } + + tensorflow::Tensor* output; + OP_REQUIRES_OK(ctx, + ctx->allocate_output(0, tensorflow::TensorShape{}, &output)); + output->scalar()() = LinearizedBuffersWrapper{ + std::move(all_linearized_buffers), std::move(all_saved_input_tensors)}; + } + + bool IsExpensive() override { return true; } + + private: + std::vector shapes_; + DataTypeVector dtypes_; + xla::Shape tuple_shape_; + + // PrelinearizeTupleOp is neither copyable nor movable. + PrelinearizeTupleOp(const PrelinearizeTupleOp&) = delete; + PrelinearizeTupleOp& operator=(const PrelinearizeTupleOp&) = delete; +}; + +// The InfeedEnqueuePrelinearizedBufferOp op is used to transfer prelinearized +// buffers to the device infeed queue. +class InfeedEnqueuePrelinearizedBufferOp : public TpuTransferAsyncOpKernel { + public: + explicit InfeedEnqueuePrelinearizedBufferOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "prelinearized_buffers_to_infeed", 8) {} + + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override { + const Tensor& input_tensor = ctx->input(0); + const LinearizedBuffersWrapper* wrapper = + input_tensor.scalar()() + .get(); + TF_RETURN_IF_ERROR(transfer_manager->TransferBuffersToInfeed( + stream_executor, wrapper->buffers)); + + return Status::OK(); + } + + private: + // InfeedEnqueuePrelinearizedBufferOp is neither copyable nor movable. + InfeedEnqueuePrelinearizedBufferOp( + const InfeedEnqueuePrelinearizedBufferOp&) = delete; + InfeedEnqueuePrelinearizedBufferOp& operator=( + const InfeedEnqueuePrelinearizedBufferOp&) = delete; +}; + +} // anonymous namespace + +TpuInfeedEnqueueOp::TpuInfeedEnqueueOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "infeed_enqueue", 8) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); + xla::Shape shape; + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &shape)); + OP_REQUIRES_OK(ctx, + GetInfeedShapeWithLayout(ctx, "layout", shape, &xla_shape_)); +} + +Status TpuInfeedEnqueueOp::DoWork( + OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) { + const Tensor& input_tensor = ctx->input(0); + + // Validate runtime shape and fail if it doesn't match the contract. + if (input_tensor.dtype() != dtype_) { + return errors::InvalidArgument("Infeed dtype mismatch."); + } + if (input_tensor.shape() != shape_) { + return errors::InvalidArgument("Infeed shape mismatch; expected ", + shape_.DebugString(), ", got ", + input_tensor.shape().DebugString()); + } + + const Tensor* tensor = &input_tensor; + Tensor transposed_tensor; + if (!xla::LayoutUtil::IsMonotonicWithDim0Major(xla_shape_.layout())) { + // If the given layout is not in dim0major layout, transpose the tensor. + TF_ASSIGN_OR_RETURN(transposed_tensor, + TransposeTensor(ctx, input_tensor, xla_shape_)); + tensor = &transposed_tensor; + } + + xla::BorrowingLiteral literal; + TF_RETURN_IF_ERROR(HostTensorToBorrowingLiteral(*tensor, &literal)); + + // Transfer the given literal to the Infeed interface of the device. + TF_RETURN_IF_ERROR( + transfer_manager->TransferLiteralToInfeed(stream_executor, literal)); + return Status::OK(); +} + +TpuInfeedEnqueueTupleOp::TpuInfeedEnqueueTupleOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "infeed_enqueue", 8) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); + OP_REQUIRES( + ctx, shapes_.size() == dtypes_.size(), + errors::InvalidArgument("shapes and dtypes must be the same length.")); + + std::vector xla_shapes; + for (int i = 0; i < shapes_.size(); i++) { + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, + TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); + xla_shapes.push_back(xla_shape); + } + OP_REQUIRES_OK( + ctx, GetInfeedShapeWithLayout(ctx, "layouts", + xla::ShapeUtil::MakeTupleShape(xla_shapes), + &tuple_shape_)); +} + +Status TpuInfeedEnqueueTupleOp::DoWork( + OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) { + OpInputList values; + TF_RETURN_IF_ERROR(ctx->input_list("inputs", &values)); + if (values.size() != shapes_.size()) { + return errors::InvalidArgument( + "Wrong number of inputs to InfeedEnqueueTuple."); + } + + for (const auto& shapes : shapes_) { + VLOG(1) << "TransferLiteralToInfeed " << shapes.DebugString(); + } + + std::vector maybe_transposed_tensors; + maybe_transposed_tensors.reserve(values.size()); + for (int i = 0; i < values.size(); i++) { + // Validate runtime shapes and fail if it doesn't match the contract. + const Tensor* tensor = &values[i]; + if (tensor->shape() != shapes_[i]) { + return errors::InvalidArgument("Infeed shape mismatch for tuple element ", + i, "; expected ", shapes_[i].DebugString(), + ", got ", tensor->shape().DebugString()); + } + if (!xla::LayoutUtil::IsMonotonicWithDim0Major( + tuple_shape_.tuple_shapes(i).layout())) { + // If the given layout is not in dim0major layout, tranposes the given + // tensor. + TF_ASSIGN_OR_RETURN( + Tensor transposed_tensor, + TransposeTensor(ctx, *tensor, tuple_shape_.tuple_shapes(i))); + maybe_transposed_tensors.emplace_back(transposed_tensor); + } else { + maybe_transposed_tensors.emplace_back(*tensor); + } + } + + xla::BorrowingLiteral tuple; + TF_RETURN_IF_ERROR( + HostTensorsToBorrowingLiteralTuple(maybe_transposed_tensors, &tuple)); + + // Transfer the given literal to the Infeed interface of the device. + TF_RETURN_IF_ERROR( + transfer_manager->TransferLiteralToInfeed(stream_executor, tuple)); + + VLOG(1) << "TransferLiteralToInfeed complete."; + + return Status::OK(); +} + +// These ops execute on either the TPU device or the CPU device. When running on +// CPU they must specify a non-negative value for device_ordinal to indicate +// which TPU to send infeed to. +REGISTER_KERNEL_BUILDER( + Name("InfeedEnqueue").Device(DEVICE_TPU_NODE).HostMemory("input"), + TpuInfeedEnqueueOp); +REGISTER_KERNEL_BUILDER(Name("InfeedEnqueue").Device(DEVICE_CPU), + TpuInfeedEnqueueOp); + +REGISTER_KERNEL_BUILDER( + Name("InfeedEnqueueTuple").Device(DEVICE_TPU_NODE).HostMemory("inputs"), + TpuInfeedEnqueueTupleOp); +REGISTER_KERNEL_BUILDER(Name("InfeedEnqueueTuple").Device(DEVICE_CPU), + TpuInfeedEnqueueTupleOp); + +// Prelinearize ops run on CPU as part of tf.data input pipeline. +REGISTER_KERNEL_BUILDER(Name("Prelinearize").Device(DEVICE_CPU), + PrelinearizeOp); +REGISTER_KERNEL_BUILDER(Name("PrelinearizeTuple").Device(DEVICE_CPU), + PrelinearizeTupleOp); + +// InfeedEnqueuePrelinearizedBuffer op run on CPU and takes a device_ordinal to +// select the right device to infeed. +REGISTER_KERNEL_BUILDER( + Name("InfeedEnqueuePrelinearizedBuffer").Device(DEVICE_CPU), + InfeedEnqueuePrelinearizedBufferOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/infeed_ops.h b/tensorflow/core/tpu/kernels/infeed_ops.h new file mode 100644 index 00000000000..622583b6a73 --- /dev/null +++ b/tensorflow/core/tpu/kernels/infeed_ops.h @@ -0,0 +1,69 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_TPU_KERNELS_INFEED_OPS_H_ +#define TENSORFLOW_CORE_TPU_KERNELS_INFEED_OPS_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/tpu/kernels/transfer_ops.h" + +namespace tensorflow { + +// TODO(b/65200690): Rework this when there is a callback based infeed API to +// StreamExecutor. + +// The InfeedEnqueue op is used to deliver data to the device infeed queue. +class TpuInfeedEnqueueOp : public TpuTransferAsyncOpKernel { + public: + explicit TpuInfeedEnqueueOp(OpKernelConstruction* ctx); + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override; + + private: + TensorShape shape_; + DataType dtype_; + xla::Shape xla_shape_; + + // TpuInfeedEnqueueOp is neither copyable nor movable. + TpuInfeedEnqueueOp(const TpuInfeedEnqueueOp&) = delete; + TpuInfeedEnqueueOp& operator=(const TpuInfeedEnqueueOp&) = delete; +}; + +// The InfeedEnqueueTuple op is used on the host to deliver multiple tensors to +// the device infeed queue as an XLA tuple. +class TpuInfeedEnqueueTupleOp : public TpuTransferAsyncOpKernel { + public: + explicit TpuInfeedEnqueueTupleOp(OpKernelConstruction* ctx); + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override; + + private: + std::vector shapes_; + DataTypeVector dtypes_; + xla::Shape tuple_shape_; + + // TpuInfeedEnqueueTupleOp is neither copyable nor movable. + TpuInfeedEnqueueTupleOp(const TpuInfeedEnqueueTupleOp&) = delete; + TpuInfeedEnqueueTupleOp& operator=(const TpuInfeedEnqueueTupleOp&) = delete; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_TPU_KERNELS_INFEED_OPS_H_ diff --git a/tensorflow/core/tpu/kernels/outfeed_ops.cc b/tensorflow/core/tpu/kernels/outfeed_ops.cc new file mode 100644 index 00000000000..51a3a71a297 --- /dev/null +++ b/tensorflow/core/tpu/kernels/outfeed_ops.cc @@ -0,0 +1,116 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/tpu/kernels/outfeed_ops.h" + +#include "tensorflow/compiler/jit/xla_device.h" +#include "tensorflow/compiler/tf2xla/literal_util.h" +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/tpu/kernels/transfer_ops.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/stream_executor/multi_platform_manager.h" + +namespace tensorflow { + +TpuOutfeedDequeueOp::TpuOutfeedDequeueOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "outfeed_dequeue", 1) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &xla_shape_)); +} + +Status TpuOutfeedDequeueOp::DoWork( + OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) { + Tensor* output; + TF_RETURN_IF_ERROR(ctx->allocate_output(0, shape_, &output)); + + // Transfer from the outfeed interface of the device. + xla::MutableBorrowingLiteral literal; + TF_RETURN_IF_ERROR( + HostTensorToMutableBorrowingLiteral(xla_shape_, output, &literal)); + + VLOG(1) << "TransferLiteralFromOutfeed " + << xla::ShapeUtil::HumanStringWithLayout(xla_shape_); + + TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralFromOutfeed( + stream_executor, xla_shape_, literal)); + + VLOG(1) << "TransferLiteralFromOutfeed complete."; + + return Status::OK(); +} + +// The OutfeedDequeueTuple op is used to retrieve multiple tensors from the +// device outfeed queue. +TpuOutfeedDequeueTupleOp::TpuOutfeedDequeueTupleOp(OpKernelConstruction* ctx) + : TpuTransferAsyncOpKernel(ctx, "outfeed_dequeue", 1) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); + OP_REQUIRES( + ctx, shapes_.size() == dtypes_.size(), + errors::InvalidArgument("shapes and dtypes must be the same length.")); + // The `dtypes` list is inferred from the supplied inputs, so it + // is always the correct length. + for (int i = 0; i < shapes_.size(); i++) { + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, + TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); + xla_shapes_.push_back(xla_shape); + } + tuple_shape_ = xla::ShapeUtil::MakeTupleShape(xla_shapes_); +} + +Status TpuOutfeedDequeueTupleOp::DoWork( + OpKernelContext* ctx, xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) { + VLOG(1) << "TransferLiteralFromOutfeed " + << xla::ShapeUtil::HumanStringWithLayout(tuple_shape_); + + for (int i = 0; i < shapes_.size(); ++i) { + Tensor* output; + TF_RETURN_IF_ERROR(ctx->allocate_output(i, shapes_[i], &output)); + + xla::MutableBorrowingLiteral literal; + TF_RETURN_IF_ERROR( + HostTensorToMutableBorrowingLiteral(xla_shapes_[i], output, &literal)); + TF_RETURN_IF_ERROR(transfer_manager->TransferLiteralFromOutfeed( + stream_executor, xla_shapes_[i], literal)); + } + return Status::OK(); +} + +// These ops execute on either the TPU device or the CPU device. When +// running on CPU they must specify a non-negative value for +// device_ordinal to indicate which TPU to receive outfeed from. +REGISTER_KERNEL_BUILDER( + Name("OutfeedDequeue").Device(DEVICE_TPU_NODE).HostMemory("output"), + TpuOutfeedDequeueOp); +REGISTER_KERNEL_BUILDER(Name("OutfeedDequeue").Device(DEVICE_CPU), + TpuOutfeedDequeueOp); + +REGISTER_KERNEL_BUILDER( + Name("OutfeedDequeueTuple").Device(DEVICE_TPU_NODE).HostMemory("outputs"), + TpuOutfeedDequeueTupleOp); +REGISTER_KERNEL_BUILDER(Name("OutfeedDequeueTuple").Device(DEVICE_CPU), + TpuOutfeedDequeueTupleOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/outfeed_ops.h b/tensorflow/core/tpu/kernels/outfeed_ops.h new file mode 100644 index 00000000000..5e3ed87c04b --- /dev/null +++ b/tensorflow/core/tpu/kernels/outfeed_ops.h @@ -0,0 +1,69 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_TPU_KERNELS_OUTFEED_OPS_H_ +#define TENSORFLOW_CORE_TPU_KERNELS_OUTFEED_OPS_H_ + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/tpu/kernels/transfer_ops.h" + +namespace tensorflow { + +// The OutfeedDequeue op is used to retrieve a single tensor from the device +// outfeed queue. +class TpuOutfeedDequeueOp : public TpuTransferAsyncOpKernel { + public: + explicit TpuOutfeedDequeueOp(OpKernelConstruction* ctx); + + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override; + + private: + TensorShape shape_; + DataType dtype_; + xla::Shape xla_shape_; + + // OutfeedDequeueOp is neither copyable nor movable. + TpuOutfeedDequeueOp(const TpuOutfeedDequeueOp&) = delete; + TpuOutfeedDequeueOp& operator=(const TpuOutfeedDequeueOp&) = delete; +}; + +// The OutfeedDequeueTuple op is used to retrieve multiple tensors from the +// device outfeed queue. +class TpuOutfeedDequeueTupleOp : public TpuTransferAsyncOpKernel { + public: + explicit TpuOutfeedDequeueTupleOp(OpKernelConstruction* ctx); + + Status DoWork(OpKernelContext* ctx, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) override; + + private: + std::vector shapes_; + DataTypeVector dtypes_; + std::vector xla_shapes_; + xla::Shape tuple_shape_; + + // OutfeedDequeueTupleOp is neither copyable nor movable. + TpuOutfeedDequeueTupleOp(const TpuOutfeedDequeueTupleOp&) = delete; + TpuOutfeedDequeueTupleOp& operator=(const TpuOutfeedDequeueTupleOp&) = delete; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_TPU_KERNELS_OUTFEED_OPS_H_ diff --git a/tensorflow/core/tpu/kernels/replication_ops.cc b/tensorflow/core/tpu/kernels/replication_ops.cc new file mode 100644 index 00000000000..4c986e880e7 --- /dev/null +++ b/tensorflow/core/tpu/kernels/replication_ops.cc @@ -0,0 +1,27 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/jit/xla_device_ops.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/tpu/tpu_defs.h" + +namespace tensorflow { + +REGISTER_KERNEL_BUILDER(Name("_TPUReplicate").Device(DEVICE_TPU_SYSTEM), + XlaDeviceDummyOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc b/tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc new file mode 100644 index 00000000000..ec2ae91d3eb --- /dev/null +++ b/tensorflow/core/tpu/kernels/tpu_handle_to_key_op.cc @@ -0,0 +1,62 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/tpu/kernels/tpu_compilation_cache_interface.h" +#include "tensorflow/core/tpu/kernels/tpu_op_consts.h" +#include "tensorflow/core/tpu/tpu_configuration.h" + +namespace tensorflow { + +class TpuHandleToProtoKeyOp : public OpKernel { + public: + explicit TpuHandleToProtoKeyOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + ~TpuHandleToProtoKeyOp() override = default; + TpuHandleToProtoKeyOp(const TpuHandleToProtoKeyOp&) = delete; + TpuHandleToProtoKeyOp& operator=(const TpuHandleToProtoKeyOp&) = delete; + + void Compute(OpKernelContext* ctx) override { + VLOG(1) << "TpuHandleToProtoKeyOp::Compute " << ctx->op_kernel().name() + << " on device " << ctx->op_kernel().requested_device(); + const Tensor& uid = ctx->input(0); + + ResourceMgr* rm = GetTPUConfigResourceMgr(); + tpu::TpuCompilationCacheInterface* cache; + OP_REQUIRES_OK(ctx, rm->Lookup( + rm->default_container(), + tpu::kCompilationCacheResourceName, &cache)); + core::ScopedUnref cache_unref(cache); + + std::vector keys; + OP_REQUIRES_OK(ctx, cache->GetKeysFromUid(uid.scalar()(), &keys)); + + TensorShape output_shape; + output_shape.AddDim(keys.size()); + Tensor* result = nullptr; + OP_REQUIRES_OK(ctx, ctx->allocate_output(0, output_shape, &result)); + for (int i = 0; i < keys.size(); ++i) { + result->vec()(i) = keys[i]; + } + }; +}; + +REGISTER_KERNEL_BUILDER(Name("TpuHandleToProtoKey").Device(DEVICE_CPU), + TpuHandleToProtoKeyOp); + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/transfer_ops.cc b/tensorflow/core/tpu/kernels/transfer_ops.cc new file mode 100644 index 00000000000..40b85e2cfbd --- /dev/null +++ b/tensorflow/core/tpu/kernels/transfer_ops.cc @@ -0,0 +1,98 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/tpu/kernels/transfer_ops.h" + +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/platform/tracing.h" +#include "tensorflow/core/profiler/lib/traceme.h" +#include "tensorflow/stream_executor/multi_platform_manager.h" +#include "tensorflow/stream_executor/tpu/tpu_node_context.h" +#include "tensorflow/stream_executor/tpu/tpu_platform_interface.h" +#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" + +namespace tensorflow { + +TpuTransferAsyncOpKernel::TpuTransferAsyncOpKernel(OpKernelConstruction* ctx, + const string& transfer_type, + int number_of_threads) + : AsyncOpKernel(ctx), + thread_pool_(new thread::ThreadPool( + ctx->env(), + strings::StrCat(transfer_type, "_thread_", + SanitizeThreadSuffix(def().name())), + /*num_threads=*/8)) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("device_ordinal", &device_ordinal_)); + if (ctx->device_type() == DeviceType(DEVICE_CPU)) { + OP_REQUIRES( + ctx, device_ordinal_ >= 0, + errors::InvalidArgument(transfer_type, + " ops must specify a device_ordinal when " + "placed on CPU.")); + } +} + +void TpuTransferAsyncOpKernel::ComputeAsync(OpKernelContext* ctx, + DoneCallback done) { + CancellationToken token = + ctx->cancellation_manager()->get_cancellation_token(); + bool already_cancelled; + { + // Only protect registering the cancellation callback as mu_ cannot be held + // at a point where `done` could be called. + mutex_lock lock(mu_); + already_cancelled = !ctx->cancellation_manager()->RegisterCallback( + token, [this]() { Cancel(); }); + } + OP_REQUIRES_ASYNC(ctx, !already_cancelled, + errors::Cancelled("Infeed was cancelled."), done); + thread_pool_->Schedule([this, ctx, done, token]() { + Status s = RunTransfer(ctx); + ctx->cancellation_manager()->DeregisterCallback(token); + OP_REQUIRES_OK_ASYNC(ctx, s, done); + done(); + }); +} + +Status TpuTransferAsyncOpKernel::RunTransfer(OpKernelContext* ctx) { + auto* tpu_platform = tpu::TpuPlatformInterface::GetRegisteredPlatform(); + + int real_device_ordinal = device_ordinal_; + if (real_device_ordinal < 0) { + const XlaDevice::Metadata* metadata; + TF_RETURN_IF_ERROR(XlaDevice::GetMetadata(ctx, &metadata)); + real_device_ordinal = metadata->device_ordinal(); + } + stream_executor::StreamExecutor* stream_executor = + tpu_platform->ExecutorForDevice(real_device_ordinal).ValueOrDie(); + + // When Xprof profiling is off (which is the default), constructing the + // activity is simple enough that its overhead is negligible. + profiler::TraceMe activity( + [this] { return profiler::TraceMeOp(name(), type_string()); }, + profiler::TraceMeLevel::kInfo); + return DoWork( + ctx, xla::TpuTransferManagerInterface::GetRegisteredTpuTransferManager(), + stream_executor); +} + +void TpuTransferAsyncOpKernel::Cancel() { + mutex_lock lock(mu_); + TF_CHECK_OK(tpu::TpuNodeContext::CloseTpuHost()); +} + +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/transfer_ops.h b/tensorflow/core/tpu/kernels/transfer_ops.h new file mode 100644 index 00000000000..d98d743f569 --- /dev/null +++ b/tensorflow/core/tpu/kernels/transfer_ops.h @@ -0,0 +1,56 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_TPU_KERNELS_TRANSFER_OPS_H_ +#define TENSORFLOW_CORE_TPU_KERNELS_TRANSFER_OPS_H_ + +#include "tensorflow/compiler/jit/xla_device.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/util/stream_executor_util.h" +#include "tensorflow/stream_executor/tpu/tpu_transfer_manager_interface.h" + +namespace tensorflow { + +// Base class providing common functionality for async ops that transfer from +// host to TPU. +class TpuTransferAsyncOpKernel : public AsyncOpKernel { + public: + explicit TpuTransferAsyncOpKernel(OpKernelConstruction* ctx, + const string& transfer_type, + int number_of_threads); + + void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override; + + protected: + virtual Status DoWork(OpKernelContext* context, + xla::TpuTransferManagerInterface* transfer_manager, + stream_executor::StreamExecutor* stream_executor) = 0; + + private: + Status RunTransfer(OpKernelContext* ctx); + void Cancel(); + + std::unique_ptr thread_pool_; + int device_ordinal_; + mutex mu_; + + // TpuTransferAsyncOpKernel is neither copyable nor movable. + TpuTransferAsyncOpKernel(const TpuTransferAsyncOpKernel&) = delete; + TpuTransferAsyncOpKernel& operator=(const TpuTransferAsyncOpKernel&) = delete; +}; + +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_TPU_KERNELS_TRANSFER_OPS_H_ diff --git a/tensorflow/core/tpu/tpu_library_init_fns.inc b/tensorflow/core/tpu/tpu_library_init_fns.inc index be9d594685e..40130bd46dd 100644 --- a/tensorflow/core/tpu/tpu_library_init_fns.inc +++ b/tensorflow/core/tpu/tpu_library_init_fns.inc @@ -161,6 +161,7 @@ tensorflow::Status SetExecutorStructFn(void* library_handle) { TFTPU_SET_FN(executor_fn, TpuTransferManager_TransferLiteralFromDevice); TFTPU_SET_FN(executor_fn, TpuTransferManager_GetByteSizeRequirement); TFTPU_SET_FN(executor_fn, TpuTransferManager_WriteSingleTupleIndexTable); + TFTPU_SET_FN(executor_fn, TpuTransferManager_GetInfeedLayout); TFTPU_SET_FN(executor_fn, TpuTransferManager_LinearizeToBuffers); TFTPU_SET_FN(executor_fn, TpuTransferManager_FreeBuffers); From fb05951e41e6bcdaddf183dffb8cc91c434f5ff4 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Mon, 10 Aug 2020 11:04:50 -0700 Subject: [PATCH 0756/1017] Extend LegalizeTFCommunication to support legalizing TF/XLA communication ops in `mhlo.while` op regions. This supports nesting of `mhlo.while` and other control flow ops. `mhlo.while` ops are rewritten to propagate !mhlo.token values across. PiperOrigin-RevId: 325843393 Change-Id: I0e1af045f7c78256b1b0598094ade46c027e5f11 --- .../xla/tests/legalize-tf-communication.mlir | 192 ++++++++++++++++++ .../transforms/legalize_tf_communication.cc | 131 +++++++++--- 2 files changed, 296 insertions(+), 27 deletions(-) diff --git a/tensorflow/compiler/mlir/xla/tests/legalize-tf-communication.mlir b/tensorflow/compiler/mlir/xla/tests/legalize-tf-communication.mlir index d01ab38bd6b..550b2ba4da3 100644 --- a/tensorflow/compiler/mlir/xla/tests/legalize-tf-communication.mlir +++ b/tensorflow/compiler/mlir/xla/tests/legalize-tf-communication.mlir @@ -848,6 +848,198 @@ func @if_followed_by_communication_op(%arg0: tensor, %arg1: tensor) { // ----- +// Tests `mhlo.while` with cond and body populated with TF/XLA communication +// ops. + +// CHECK-LABEL: func @while_cond_body +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func @while_cond_body(%arg0: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = "mhlo.create_token" + // CHECK: [[ARG_TUPLE:%.*]] = "mhlo.tuple"([[ARG0]], [[INIT_TOKEN]]) + + // CHECK: [[WHILE_TUPLE:%.*]] = "mhlo.while"([[ARG_TUPLE]]) + %0 = "mhlo.while"(%arg0) ( { + // CHECK: ^bb0([[COND_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg1: tensor): + // CHECK-DAG: [[COND_REGION_ARG_VALUE:%.*]] = "mhlo.get_tuple_element"([[COND_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[COND_REGION_ARG_TOKEN:%.*]] = "mhlo.get_tuple_element"([[COND_REGION_ARG]]) {index = 1 + + // CHECK: [[COND_SEND_TOKEN:%.*]] = "mhlo.send"([[COND_REGION_ARG_VALUE]], [[COND_REGION_ARG_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 1 : i64, type = 2 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "send_while_cond_dtoh_0"} + + // CHECK: [[COND_RECV_TUPLE:%.*]] = "mhlo.recv"([[COND_SEND_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 2 : i64, type = 3 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "recv_while_cond_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_while_cond", send_key = "send_while_cond", tpu_core = 0 : i64} : (tensor) -> tensor + + // CHECK-DAG: [[COND_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[COND_RECV_TUPLE]]) {index = 0 + // CHECK-DAG: [[COND_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[COND_RECV_TUPLE]]) {index = 1 + + // CHECK: [[COND_COMPARE:%.*]] = "mhlo.compare"([[COND_GET_TUPLE_ELEMENT0]], [[COND_GET_TUPLE_ELEMENT0]]) + %2 = "mhlo.compare"(%1, %1) {comparison_direction = "LT"} : (tensor, tensor) -> tensor + + // CHECK: "mhlo.return"([[COND_COMPARE]]) + "mhlo.return"(%2) : (tensor) -> () + }, { + // CHECK: ^bb0([[BODY_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg1: tensor): + // CHECK-DAG: [[BODY_REGION_ARG_VALUE:%.*]] = "mhlo.get_tuple_element"([[BODY_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[BODY_REGION_ARG_TOKEN:%.*]] = "mhlo.get_tuple_element"([[BODY_REGION_ARG]]) {index = 1 + + // CHECK: [[BODY_SEND_TOKEN:%.*]] = "mhlo.send"([[BODY_REGION_ARG_VALUE]], [[BODY_REGION_ARG_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 3 : i64, type = 2 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "send_while_body_dtoh_0"} + + // CHECK: [[BODY_RECV_TUPLE:%.*]] = "mhlo.recv"([[BODY_SEND_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 4 : i64, type = 3 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "recv_while_body_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_while_body", send_key = "send_while_body", tpu_core = 0 : i64} : (tensor) -> tensor + + // CHECK-DAG: [[BODY_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[BODY_RECV_TUPLE]]) {index = 0 + // CHECK-DAG: [[BODY_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[BODY_RECV_TUPLE]]) {index = 1 + // CHECK: [[BODY_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[BODY_GET_TUPLE_ELEMENT0]], [[BODY_GET_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[BODY_RETURN_TUPLE]]) + "mhlo.return"(%1) : (tensor) -> () + // CHECK: (tuple, !mhlo.token>) -> tuple, !mhlo.token> + }) : (tensor) -> tensor + + // CHECK: [[WHILE_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[WHILE_TUPLE]]) + // CHECK-SAME: index = 0 + // CHECK: return [[WHILE_TUPLE_ELEMENT0]] + return %0 : tensor +} + +// ----- + +// Tests `mhlo.while` with only the `cond` region populated with TF/XLA +// communication ops. + +// CHECK-LABEL: func @while_cond +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func @while_cond(%arg0: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = "mhlo.create_token" + // CHECK: [[ARG_TUPLE:%.*]] = "mhlo.tuple"([[ARG0]], [[INIT_TOKEN]]) + + // CHECK: [[WHILE_TUPLE:%.*]] = "mhlo.while"([[ARG_TUPLE]]) + %0 = "mhlo.while"(%arg0) ( { + // CHECK: ^bb0([[COND_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg1: tensor): + // CHECK-DAG: [[COND_REGION_ARG_VALUE:%.*]] = "mhlo.get_tuple_element"([[COND_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[COND_REGION_ARG_TOKEN:%.*]] = "mhlo.get_tuple_element"([[COND_REGION_ARG]]) {index = 1 + + // CHECK: [[COND_SEND_TOKEN:%.*]] = "mhlo.send"([[COND_REGION_ARG_VALUE]], [[COND_REGION_ARG_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 1 : i64, type = 2 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "send_while_cond_dtoh_0"} + + // CHECK: [[COND_RECV_TUPLE:%.*]] = "mhlo.recv"([[COND_SEND_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 2 : i64, type = 3 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "recv_while_cond_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_while_cond", send_key = "send_while_cond", tpu_core = 0 : i64} : (tensor) -> tensor + + // CHECK-DAG: [[COND_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[COND_RECV_TUPLE]]) {index = 0 + // CHECK-DAG: [[COND_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[COND_RECV_TUPLE]]) {index = 1 + + // CHECK: [[COND_COMPARE:%.*]] = "mhlo.compare"([[COND_GET_TUPLE_ELEMENT0]], [[COND_GET_TUPLE_ELEMENT0]]) + %2 = "mhlo.compare"(%1, %1) {comparison_direction = "LT"} : (tensor, tensor) -> tensor + + // CHECK: "mhlo.return"([[COND_COMPARE]]) + "mhlo.return"(%2) : (tensor) -> () + }, { + // CHECK: ^bb0([[BODY_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg1: tensor): + // CHECK-DAG: [[BODY_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[BODY_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[BODY_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[BODY_REGION_ARG]]) {index = 1 + // CHECK: [[BODY_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[BODY_GET_TUPLE_ELEMENT0]], [[BODY_GET_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[BODY_RETURN_TUPLE]]) + "mhlo.return"(%arg1) : (tensor) -> () + // CHECK: (tuple, !mhlo.token>) -> tuple, !mhlo.token> + }) : (tensor) -> tensor + + // CHECK: [[WHILE_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[WHILE_TUPLE]]) + // CHECK-SAME: index = 0 + // CHECK: return [[WHILE_TUPLE_ELEMENT0]] + return %0 : tensor +} + +// ----- + +// Tests `mhlo.while` with only the `body` region populated with TF/XLA +// communication ops. + +// CHECK-LABEL: func @while_body +// CHECK-SAME: ([[ARG0:%.*]]: tensor) +func @while_body(%arg0: tensor) -> tensor { + // CHECK: [[INIT_TOKEN:%.*]] = "mhlo.create_token" + // CHECK: [[ARG_TUPLE:%.*]] = "mhlo.tuple"([[ARG0]], [[INIT_TOKEN]]) + + // CHECK: [[WHILE_TUPLE:%.*]] = "mhlo.while"([[ARG_TUPLE]]) + %0 = "mhlo.while"(%arg0) ( { + // CHECK: ^bb0([[COND_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg1: tensor): + // CHECK-DAG: [[COND_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[COND_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[COND_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[COND_REGION_ARG]]) {index = 1 + + // CHECK: [[COND_COMPARE:%.*]] = "mhlo.compare"([[COND_GET_TUPLE_ELEMENT0]], [[COND_GET_TUPLE_ELEMENT0]]) + %2 = "mhlo.compare"(%arg1, %arg1) {comparison_direction = "LT"} : (tensor, tensor) -> tensor + + // CHECK: "mhlo.return"([[COND_COMPARE]]) + "mhlo.return"(%2) : (tensor) -> () + }, { + // CHECK: ^bb0([[BODY_REGION_ARG:%.*]]: tuple, !mhlo.token>): + ^bb0(%arg1: tensor): + // CHECK-DAG: [[BODY_REGION_ARG_VALUE:%.*]] = "mhlo.get_tuple_element"([[BODY_REGION_ARG]]) {index = 0 + // CHECK-DAG: [[BODY_REGION_ARG_TOKEN:%.*]] = "mhlo.get_tuple_element"([[BODY_REGION_ARG]]) {index = 1 + + // CHECK: [[BODY_SEND_TOKEN:%.*]] = "mhlo.send"([[BODY_REGION_ARG_VALUE]], [[BODY_REGION_ARG_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 1 : i64, type = 2 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "send_while_body_dtoh_0"} + + // CHECK: [[BODY_RECV_TUPLE:%.*]] = "mhlo.recv"([[BODY_SEND_TOKEN]]) + // CHECK-SAME: channel_id = {handle = 2 : i64, type = 3 : i64} + // CHECK-SAME: mhlo.frontend_attributes = {_xla_host_transfer_original_type = "f32", _xla_host_transfer_rendezvous = "recv_while_body_htod_0"} + %1 = "tf._XlaHostComputeMlir"(%arg1) {recv_key = "recv_while_body", send_key = "send_while_body", tpu_core = 0 : i64} : (tensor) -> tensor + + // CHECK-DAG: [[BODY_GET_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[BODY_RECV_TUPLE]]) {index = 0 + // CHECK-DAG: [[BODY_GET_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[BODY_RECV_TUPLE]]) {index = 1 + // CHECK: [[BODY_RETURN_TUPLE:%.*]] = "mhlo.tuple"([[BODY_GET_TUPLE_ELEMENT0]], [[BODY_GET_TUPLE_ELEMENT1]]) + // CHECK: "mhlo.return"([[BODY_RETURN_TUPLE]]) + "mhlo.return"(%1) : (tensor) -> () + // CHECK: (tuple, !mhlo.token>) -> tuple, !mhlo.token> + }) : (tensor) -> tensor + + // CHECK: [[WHILE_TUPLE_ELEMENT0:%.*]] = "mhlo.get_tuple_element"([[WHILE_TUPLE]]) + // CHECK-SAME: index = 0 + // CHECK: return [[WHILE_TUPLE_ELEMENT0]] + return %0 : tensor +} + +// ----- + +// Tests `mhlo.while` containing TF/XLA communication ops followed by other +// TF/XLA communication ops. + +func @while_followed_by_communication_op(%arg0: tensor) { + // CHECK: [[WHILE_TUPLE:%.*]] = "mhlo.while" + %0 = "mhlo.while"(%arg0) ( { + ^bb0(%arg1: tensor): + "tf.XlaSendToHost"(%arg1) {key = "send_key0"} : (tensor) -> () + %1 = "mhlo.compare"(%arg1, %arg1) {comparison_direction = "LT"} : (tensor, tensor) -> tensor + "mhlo.return"(%1) : (tensor) -> () + }, { + ^bb0(%arg1: tensor): + "mhlo.return"(%arg1) : (tensor) -> () + }) : (tensor) -> tensor + + // CHECK: [[WHILE_TUPLE_ELEMENT1:%.*]] = "mhlo.get_tuple_element"([[WHILE_TUPLE]]) {index = 1 + + // CHECK: "mhlo.send"({{.*}}, [[WHILE_TUPLE_ELEMENT1]]) + "tf.XlaSendToHost"(%arg0) {key = "send_key1"} : (tensor) -> () + return +} + +// ----- + // Tests unsupported parent of TF/XLA communication op. func @unsupported_ancestor(%arg0: tensor, %arg1: tensor) { diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc index b4e4f5c4f5c..1d6ce36300f 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_communication.cc @@ -57,7 +57,7 @@ const char kXlaHostTransferOriginalTypeAttr[] = // tokens (for ordering), and rewrite their respective functions and control // flow ops when necessary. // Note, this currently does not handle nested modules/functions or region based -// ops other than certain control flow ops (`mhlo.if`). +// ops other than certain control flow ops (`mhlo.if`, `mhlo.while`). class LegalizeTFCommunication : public PassWrapper> { public: @@ -71,7 +71,7 @@ bool IsCommunicationOp(Operation* op) { } // Checks if an op is a supported HLO control flow op. -bool IsControlFlowOp(Operation* op) { return isa(op); } +bool IsControlFlowOp(Operation* op) { return isa(op); } // Collects control flow op ancestors of a given op, up until FuncOp. If any // ancestor is not a control flow op or a FuncOp, or of a single block region, @@ -541,6 +541,10 @@ void RewriteControlFlowTerminator(OpBuilder& builder, Operation* terminator, Value token) { assert(terminator->getNumOperands() == 1); assert(terminator->getBlock()->getNumArguments() == 1); + // `mhlo.while` cond terminator does not need to be rewritten as it always + // returns a tensor predicate value. + if (auto while_parent = dyn_cast_or_null(terminator->getParentOp())) + if (terminator->getParentRegion() == &while_parent.cond()) return; builder.setInsertionPoint(terminator); llvm::SmallDenseMap rewritten_operands; @@ -554,25 +558,24 @@ void RewriteControlFlowTerminator(OpBuilder& builder, Operation* terminator, void RewriteRegionIfOp(OpBuilder& builder, IfOp region_if, SmallVectorImpl& ops_to_visit, Value token) { - SmallVector new_branch_operands; llvm::SmallDenseMap rewritten_operands; - auto old_branch_operands = llvm::drop_begin(region_if.getOperands(), 1); // Rewrite all region operands to have an extra operand `token`. - for (Value operand : old_branch_operands) - new_branch_operands.push_back( - GetValueWithToken(builder, operand, token, rewritten_operands)); + Value new_true_operand = GetValueWithToken(builder, region_if.true_arg(), + token, rewritten_operands); + Value new_false_operand = GetValueWithToken(builder, region_if.false_arg(), + token, rewritten_operands); auto new_result_type = GetTypeWithToken(builder, region_if.getType()); // Create new `mhlo.if` op with extra token operands and result. auto new_if = builder.create(region_if.getLoc(), new_result_type, - region_if.pred(), new_branch_operands[0], - new_branch_operands[1]); + region_if.pred(), new_true_operand, + new_false_operand); // Move all regions from the old `mhlo.if` op to its replacement. - for (auto& region_and_idx : llvm::enumerate(region_if.getRegions())) - new_if.getRegion(region_and_idx.index()).takeBody(*region_and_idx.value()); + new_if.true_branch().takeBody(region_if.true_branch()); + new_if.false_branch().takeBody(region_if.false_branch()); // Forward result from old `mhlo.if` with replacement, and unpack result when // necessary. @@ -594,21 +597,22 @@ void RewriteRegionIfOp(OpBuilder& builder, IfOp region_if, ops_to_visit.push_back({/*region_idx=*/0, new_token, new_if}); } -// Rewrites a `mhlo.if` region to receive and forward a `mhlo.token`. The block -// argument is updated to have an extra `mhlo.token` element. If the region -// block is to be rewritten, the next op to visit is set to the first op in the -// block. Otherwise the terminator is updated to forward `token`. -void RewriteRegionIfRegion( - OpBuilder& builder, IfOp region_if, unsigned region_idx, - SmallVectorImpl& ops_to_visit, +// Rewrites a `mhlo.if`/`mhlo.while` region to receive and forward a +// `mhlo.token`. The block argument is updated to have an extra `mhlo.token` +// element. If the region block is to be rewritten, the next op to visit is set +// to the first op in the block. Otherwise the terminator is updated to forward +// `token`. +void RewriteControlFlowOpRegion( + OpBuilder& builder, Operation* region_op, unsigned region_idx, + Type block_arg_type, SmallVectorImpl& ops_to_visit, const llvm::SmallPtrSetImpl& control_flow_blocks, Value token) { - ops_to_visit.push_back({region_idx + 1, token, region_if}); + ops_to_visit.push_back({region_idx + 1, token, region_op}); - Region& region = region_if.getRegion(region_idx); + Region& region = region_op->getRegion(region_idx); assert(llvm::hasSingleElement(region)); - auto block_token = UpdateControlFlowBlockArgWithToken( - builder, region.front(), region_if.getOperand(region_idx + 1).getType()); + auto block_token = UpdateControlFlowBlockArgWithToken(builder, region.front(), + block_arg_type); if (control_flow_blocks.contains(®ion.front())) { ops_to_visit.push_back({/*region_idx=*/llvm::None, block_token, @@ -621,9 +625,9 @@ void RewriteRegionIfRegion( } // Rewrites an `mhlo.if` op or its region. If `region_idx` is not set, the op -// operands and results rewritten. If `region_idx` is set, region `region_idx` -// is rewritten to take in and return an additional token. Returns true if op -// is still being rewritten. +// operands and results are rewritten. If `region_idx` is set, region +// `region_idx` is rewritten to take in and return an additional token. Returns +// true if the op or its region was rewritten. bool ProcessRegionIfOp(OpBuilder& builder, IfOp region_if, Optional region_idx, SmallVectorImpl& ops_to_visit, @@ -637,8 +641,76 @@ bool ProcessRegionIfOp(OpBuilder& builder, IfOp region_if, } if (*region_idx < region_if.getNumRegions()) { - RewriteRegionIfRegion(builder, region_if, *region_idx, ops_to_visit, - control_flow_blocks, token); + RewriteControlFlowOpRegion(builder, region_if, *region_idx, + region_if.getOperand(*region_idx + 1).getType(), + ops_to_visit, control_flow_blocks, token); + return true; + } + + return false; +} + +// Rewrites a `mhlo.while` op to receive and forward a `mhlo.token`. Operands to +// the op for all of its regions are extended to have an extra operand `token`. +void RewriteRegionWhileOp(OpBuilder& builder, WhileOp region_while, + SmallVectorImpl& ops_to_visit, + Value token) { + llvm::SmallDenseMap rewritten_operands; + + // Rewrite region operand to have an extra operand `token`. + Value new_val_operand = + GetValueWithToken(builder, region_while.val(), token, rewritten_operands); + + auto new_result_type = GetTypeWithToken(builder, region_while.getType()); + + // Create new `mhlo.while` op with extra token operand and result. + auto new_while = builder.create(region_while.getLoc(), + new_result_type, new_val_operand); + + // Move all regions from the old `mhlo.while` op to its replacement. + new_while.cond().takeBody(region_while.cond()); + new_while.body().takeBody(region_while.body()); + + // Forward result from old `mhlo.while` with replacement, and unpack result + // when necessary. + ReplaceWithTupleResult(builder, region_while.getResult(), + new_while.getResult()); + + auto new_token = builder.create( + new_while.getLoc(), new_while.getResult(), + new_while.getResult().getType().cast().size() - 1); + + region_while.erase(); + + // Remove leftover operands to old `mhlo.while` if they have no uses. + for (auto& rewritten_operand : rewritten_operands) + if (auto tuple_op = rewritten_operand.getFirst().getDefiningOp()) + if (tuple_op.use_empty()) tuple_op.erase(); + + // Next op to visit. The replacement is visited but at its first region. The + // token result of the new region if is propagated. + ops_to_visit.push_back({/*region_idx=*/0, new_token, new_while}); +} + +// Rewrites an `mhlo.while` op or its region. If `region_idx` is not set, the op +// operands and results are rewritten. If `region_idx` is set, region +// `region_idx` is rewritten to take in and return an additional token. Returns +// true if the op or its region was rewritten. +bool ProcessRegionWhileOp( + OpBuilder& builder, WhileOp region_while, Optional region_idx, + SmallVectorImpl& ops_to_visit, + const llvm::SmallPtrSetImpl& control_flow_blocks, Value token) { + builder.setInsertionPoint(region_while); + + if (!region_idx) { + RewriteRegionWhileOp(builder, region_while, ops_to_visit, token); + return true; + } + + if (*region_idx < region_while.getNumRegions()) { + RewriteControlFlowOpRegion(builder, region_while, *region_idx, + region_while.val().getType(), ops_to_visit, + control_flow_blocks, token); return true; } @@ -730,6 +802,11 @@ LogicalResult RewriteFunction( if (ProcessRegionIfOp(builder, region_if, op_to_visit.region_idx, ops_to_visit, control_flow_blocks, token)) continue; + } else if (auto region_while = dyn_cast(curr_op)) { + if (op_to_visit.region_idx || control_flow_ops.contains(region_while)) + if (ProcessRegionWhileOp(builder, region_while, op_to_visit.region_idx, + ops_to_visit, control_flow_blocks, token)) + continue; } else if (auto region_terminator = dyn_cast(curr_op)) { RewriteControlFlowTerminator(builder, region_terminator, token); // There is no next op afer the control flow op terminator, simply let From 525e240a17eff24c1901b1468554c846028b665e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20K=C3=B6ppe?= Date: Mon, 10 Aug 2020 11:13:41 -0700 Subject: [PATCH 0757/1017] Remove using-declaration for absl::bit_cast. Using-declarations into absl:: are discouraged, and this removal disambiguates which overload of absl::bit_cast is actually selected, and thus helps us trace the remaining uses of the "bad" overload. PiperOrigin-RevId: 325845404 Change-Id: I6441a0be8ef048ca8af544047b36b1eefac94b85 --- tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc b/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc index ac8f657d20d..4bbe74c423a 100644 --- a/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc +++ b/tensorflow/core/lib/jpeg/jpeg_mem_unittest.cc @@ -34,7 +34,6 @@ namespace tensorflow { namespace jpeg { namespace { -using absl::bit_cast; const char kTestData[] = "tensorflow/core/lib/jpeg/testdata/"; int ComputeSumAbsoluteDifference(const uint8* a, const uint8* b, int width, @@ -60,7 +59,7 @@ void TestJPEG(Env* env, const string& jpegfile) { string jpeg; ReadFileToStringOrDie(env, jpegfile, &jpeg); const int fsize = jpeg.size(); - const uint8* const temp = bit_cast(jpeg.data()); + const uint8* const temp = absl::bit_cast(jpeg.data()); // Try partial decoding (half of the data) int w, h, c; @@ -102,7 +101,7 @@ void TestCropAndDecodeJpeg(Env* env, const string& jpegfile, string jpeg; ReadFileToStringOrDie(env, jpegfile, &jpeg); const int fsize = jpeg.size(); - auto temp = bit_cast(jpeg.data()); + const auto* temp = absl::bit_cast(jpeg.data()); // Decode the whole image. std::unique_ptr imgdata1; @@ -225,7 +224,7 @@ TEST(JpegMemTest, CropAndDecodeJpegWithStride) { string jpeg; ReadFileToStringOrDie(env, data_path + "jpeg_merge_test1.jpg", &jpeg); const int fsize = jpeg.size(); - auto temp = bit_cast(jpeg.data()); + const auto* temp = absl::bit_cast(jpeg.data()); int w, h, c; ASSERT_TRUE(GetImageInfo(temp, fsize, &w, &h, &c)); @@ -263,7 +262,7 @@ TEST(JpegMemTest, CropAndDecodeJpegWithInvalidCropWindow) { string jpeg; ReadFileToStringOrDie(env, data_path + "jpeg_merge_test1.jpg", &jpeg); const int fsize = jpeg.size(); - auto temp = bit_cast(jpeg.data()); + const auto* temp = absl::bit_cast(jpeg.data()); int w, h, c; ASSERT_TRUE(GetImageInfo(temp, fsize, &w, &h, &c)); From 0708e8260bed47c925a14c4db322ae716ee561ee Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Mon, 10 Aug 2020 16:57:26 +0000 Subject: [PATCH 0758/1017] clean up --- tensorflow/c/kernels/histogram_summary_op.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/c/kernels/histogram_summary_op.cc b/tensorflow/c/kernels/histogram_summary_op.cc index 0767658664b..fad7a35ccd7 100644 --- a/tensorflow/c/kernels/histogram_summary_op.cc +++ b/tensorflow/c/kernels/histogram_summary_op.cc @@ -87,7 +87,7 @@ void HistogramSummaryOp_Compute(void* kernel, TF_OpKernelContext* ctx) { // Cast values to array to access tensor elements by index auto values_array = static_cast(TF_TensorData(safe_values_ptr.get())); tensorflow::histogram::Histogram histo; - for (int i = 0; i < TF_TensorElementCount(safe_values_ptr.get()); ++i) { + for (int64_t i = 0; i < TF_TensorElementCount(safe_values_ptr.get()); ++i) { const double double_val = static_cast(values_array[i]); if (Eigen::numext::isnan(double_val)) { std::ostringstream err; @@ -163,4 +163,4 @@ TF_ATTRIBUTE_UNUSED static bool IsHistogramSummaryOpKernelRegistered = []() { } return true; }(); -} // namespace \ No newline at end of file +} // namespace From 52b64bc279923596d4f2d8b4c6355eb926f1b76d Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Mon, 10 Aug 2020 11:38:09 -0700 Subject: [PATCH 0759/1017] Make global NOW timestamp for upload times I forgot that most scripts don't execute within one nanosecond. PiperOrigin-RevId: 325850999 Change-Id: Ib05e4debd598299a4d17c603908a5625082e9331 --- tensorflow/tools/ci_build/sizetrack_helper.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/tools/ci_build/sizetrack_helper.py b/tensorflow/tools/ci_build/sizetrack_helper.py index 8377d733c56..eb3a6afda5e 100755 --- a/tensorflow/tools/ci_build/sizetrack_helper.py +++ b/tensorflow/tools/ci_build/sizetrack_helper.py @@ -114,6 +114,9 @@ size.add_argument( help="Manually set the recorded size instead of providing an artifact.") FLAGS = parser.parse_args() + +NOW = datetime.datetime.now( + datetime.timezone.utc).replace(microsecond=0).isoformat() TABLE_NAME = "{}.{}".format(FLAGS.dataset, FLAGS.table) PROJECT_LEVEL_TABLE_NAME = "{}:{}".format(FLAGS.project, TABLE_NAME) CL_TRAILER = "PiperOrigin-RevId" @@ -285,15 +288,13 @@ def get_upload_path(): """Generate URL for 'gsutil cp'.""" if FLAGS.upload and FLAGS.artifact: artifact_filename = os.path.basename(FLAGS.artifact.name) - ts = datetime.datetime.now( - datetime.timezone.utc).replace(microsecond=0).isoformat() # note: not os.path.join here, because gsutil is always linux-style # Using a timestamp prevents duplicate entries path = "{bucket}/{team}/{artifact_id}/{now}.{artifact_filename}".format( bucket=FLAGS.bucket, team=FLAGS.team, artifact_id=FLAGS.artifact_id, - now=ts, + now=NOW, artifact_filename=artifact_filename) return path else: From f7ca4c921e71d28257c88c6787dae911e3411f25 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 10 Aug 2020 11:58:28 -0700 Subject: [PATCH 0760/1017] Add TpuTopologyExternal::cores(). Also expose the underlying TpuCoreLocationExternal pointer so internal users can get at not-yet-exposed functionality. PiperOrigin-RevId: 325855758 Change-Id: I640e4d4996f60689cd533321434a5fec5f612c60 --- tensorflow/core/tpu/tpu_library_init_fns.inc | 2 ++ .../stream_executor/tpu/tpu_executor_c_api.h | 6 ++++++ tensorflow/stream_executor/tpu/tpu_topology.cc | 15 +++++++++++++++ tensorflow/stream_executor/tpu/tpu_topology.h | 5 +++++ 4 files changed, 28 insertions(+) diff --git a/tensorflow/core/tpu/tpu_library_init_fns.inc b/tensorflow/core/tpu/tpu_library_init_fns.inc index 40130bd46dd..a27dbea2388 100644 --- a/tensorflow/core/tpu/tpu_library_init_fns.inc +++ b/tensorflow/core/tpu/tpu_library_init_fns.inc @@ -175,6 +175,8 @@ tensorflow::Status SetExecutorStructFn(void* library_handle) { TFTPU_SET_FN(executor_fn, TpuTopology_ChipBounds_Z); TFTPU_SET_FN(executor_fn, TpuTopology_HasChip); TFTPU_SET_FN(executor_fn, TpuTopology_Core); + TFTPU_SET_FN(executor_fn, TpuTopology_NumCores); + TFTPU_SET_FN(executor_fn, TpuTopology_Cores); TFTPU_SET_FN(executor_fn, TpuTopology_IdForHost); TFTPU_SET_FN(executor_fn, TpuCoreLocation_ChipCoordinates); TFTPU_SET_FN(executor_fn, TpuCoreLocation_HostCoordinates); diff --git a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h index 013e7fe4e0c..1dcb3eaf244 100644 --- a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h +++ b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h @@ -203,6 +203,10 @@ int TpuTopology_ChipBounds_Z(void* tpu_topology); bool TpuTopology_HasChip(void* tpu_topology, int x, int y, int z); void* TpuTopology_Core(void* tpu_topology, int x, int y, int z, TpuCoreTypeEnum tpu_core_type, int index); +int TpuTopology_NumCores(void* tpu_topology, TpuCoreTypeEnum tpu_core_type); +// 'cores' should be a preallocated array of size TpuTopology_NumCores. +void TpuTopology_Cores(void* tpu_topology, TpuCoreTypeEnum tpu_core_type, + void** cores); int TpuTopology_IdForHost(void* tpu_topology, int x, int y, int z); void TpuCoreLocation_ChipCoordinates(void* tpu_core_location, int* x, int* y, int* z); @@ -357,6 +361,8 @@ struct TfTpu_ExecutorApiFn { TFTPU_ADD_FN_IN_STRUCT(TpuTopology_ChipBounds_Z); TFTPU_ADD_FN_IN_STRUCT(TpuTopology_HasChip); TFTPU_ADD_FN_IN_STRUCT(TpuTopology_Core); + TFTPU_ADD_FN_IN_STRUCT(TpuTopology_NumCores); + TFTPU_ADD_FN_IN_STRUCT(TpuTopology_Cores); TFTPU_ADD_FN_IN_STRUCT(TpuTopology_IdForHost); TFTPU_ADD_FN_IN_STRUCT(TpuCoreLocation_ChipCoordinates); diff --git a/tensorflow/stream_executor/tpu/tpu_topology.cc b/tensorflow/stream_executor/tpu/tpu_topology.cc index 74eb0aaf607..cfcea2dc944 100644 --- a/tensorflow/stream_executor/tpu/tpu_topology.cc +++ b/tensorflow/stream_executor/tpu/tpu_topology.cc @@ -75,6 +75,21 @@ TpuCoreLocationExternal TpuTopologyExternal::Core(int x, int y, int z, topology_, x, y, z, core_type, index)); } +std::vector TpuTopologyExternal::cores( + TpuCoreTypeEnum core_type) const { + int num_cores = + tpu::ExecutorApiFn()->TpuTopology_NumCoresFn(topology_, core_type); + std::vector core_ptrs(num_cores); + tpu::ExecutorApiFn()->TpuTopology_CoresFn(topology_, core_type, + core_ptrs.data()); + std::vector result; + result.reserve(num_cores); + for (void* ptr : core_ptrs) { + result.emplace_back(ptr); + } + return result; +} + int TpuTopologyExternal::IdForHost(TpuDimensionsExternal host) const { return tpu::ExecutorApiFn()->TpuTopology_IdForHostFn(topology_, host.x, host.y, host.z); diff --git a/tensorflow/stream_executor/tpu/tpu_topology.h b/tensorflow/stream_executor/tpu/tpu_topology.h index 6b64fb64985..3b0c4c5aa20 100644 --- a/tensorflow/stream_executor/tpu/tpu_topology.h +++ b/tensorflow/stream_executor/tpu/tpu_topology.h @@ -16,6 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_STREAM_EXECUTOR_TPU_TPU_TOPOLOGY_H_ #define TENSORFLOW_STREAM_EXECUTOR_TPU_TPU_TOPOLOGY_H_ +#include + #include "tensorflow/core/platform/types.h" #include "tensorflow/stream_executor/tpu/c_api_decl.h" @@ -38,6 +40,8 @@ class TpuCoreLocationExternal { int32 index() const; int32 Id() const; + void* impl() const { return core_location_; } + private: void* core_location_; }; @@ -67,6 +71,7 @@ class TpuTopologyExternal { bool HasChip(int x, int y, int z) const; TpuCoreLocationExternal Core(int x, int y, int z, TpuCoreTypeEnum core_type, int index) const; + std::vector cores(TpuCoreTypeEnum core_type) const; int IdForHost(TpuDimensionsExternal host) const; private: From 9908d143206dffc4b24b9b028ebfa0dda392e05d Mon Sep 17 00:00:00 2001 From: Jacques Pienaar Date: Mon, 10 Aug 2020 12:19:15 -0700 Subject: [PATCH 0761/1017] Add chlo.constant_like op which splats a constant to shape of operand This allows specifying a constant whose shape is only known when operand shape is. Also use it to update tf.Acos legalization. PiperOrigin-RevId: 325860604 Change-Id: I93317bd2c9d6935d527712b10b6ef312c4f8548f --- .../mlir-hlo/Dialect/mhlo/IR/chlo_ops.h | 14 ++++++ .../mlir-hlo/Dialect/mhlo/IR/chlo_ops.td | 18 ++++++++ .../mlir-hlo/Dialect/mhlo/IR/hlo_utils.td | 3 ++ .../lib/Dialect/mhlo/IR/chlo_canonicalize.td | 30 +++++++++++++ .../mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc | 44 +++++++++++++++++++ .../compiler/mlir/hlo/tests/canonicalize.mlir | 14 ++++++ .../compiler/mlir/xla/tests/legalize-tf.mlir | 27 ++++++++++-- .../mlir/xla/transforms/legalize_tf.cc | 10 +++++ .../xla/transforms/legalize_tf_patterns.td | 23 +++++----- 9 files changed, 170 insertions(+), 13 deletions(-) create mode 100644 tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_canonicalize.td diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.h b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.h index 4c92ef3de85..9704f34a4d6 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.h +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.h @@ -24,6 +24,7 @@ limitations under the License. #include "mlir/IR/OpDefinition.h" #include "mlir/IR/Operation.h" #include "mlir/IR/StandardTypes.h" +#include "mlir/IR/TypeUtilities.h" #include "mlir/IR/Types.h" #include "mlir/Interfaces/InferTypeOpInterface.h" #include "mlir/Interfaces/SideEffectInterfaces.h" @@ -46,6 +47,19 @@ class HloClientDialect : public Dialect { #define GET_OP_CLASSES #include "mlir-hlo/Dialect/mhlo/IR/chlo_ops.h.inc" +template +static Value getConstantLike(OpBuilder& b, T constant, Value val) { + Type ty = getElementTypeOrSelf(val.getType()); + + auto getAttr = [&]() -> Attribute { + if (ty.isa()) return b.getIntegerAttr(ty, constant); + if (ty.isa()) return b.getFloatAttr(ty, constant); + llvm_unreachable("unhandled element type"); + }; + // TODO(jpienaar): Add ability to pass loc via native call and update. + return b.create(b.getUnknownLoc(), getAttr(), val); +} + } // namespace chlo } // namespace mlir diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.td b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.td index d7cdd12d351..2f3bbefb5ab 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.td +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/chlo_ops.td @@ -364,6 +364,24 @@ def HLOClient_AcosOp: HLOClient_UnaryElementwiseOp<"acos", }]; } +def HLOClient_ConstantLikeOp: HLOClient_Op<"constant_like", + [NoSideEffect, SameOperandsAndResultShape, + InferTypeOpInterface, + DeclareOpInterfaceMethods, + NativeOpTrait<"InferTensorType">]> { + let summary = "Constant like operator"; + + let description = [{ + Returns a splat constant of the same shape as the operand. + }]; + + // TODO(jpienaar): value's type could be tightened. + let arguments = (ins AnyAttr:$value, HLO_Tensor:$operand); + let results = (outs HLO_Tensor); + + let hasCanonicalizer = 1; +} + //===----------------------------------------------------------------------===// // Broadcasting compare op //===----------------------------------------------------------------------===// diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_utils.td b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_utils.td index e1ae9e1fb89..c201aeff8ec 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_utils.td +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_utils.td @@ -27,6 +27,9 @@ def CastIntElementsAttr : NativeCodeCall<"$0.cast()">; class ConstantSplat : NativeCodeCall< "hlo::getSplat(&$_builder, $0, " # value # ")">; +class HLO_ConstantLike : NativeCodeCall< + "chlo::getConstantLike($_builder, " # value # ", $0)">; + def NullDenseIntElementsAttr : NativeCodeCall<"DenseIntElementsAttr()">; def BinBroadcastDimensions : NativeCodeCall< diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_canonicalize.td b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_canonicalize.td new file mode 100644 index 00000000000..eb92d9e0e46 --- /dev/null +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_canonicalize.td @@ -0,0 +1,30 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This is the canonicalize pattern definition file. + +include "mlir/IR/OpBase.td" +include "mlir-hlo/Dialect/mhlo/IR/hlo_ops.td" +include "mlir-hlo/Dialect/mhlo/IR/hlo_utils.td" + +def UnaryToBinaryEinsumEq : NativeCodeCall< + "$_builder.getStringAttr(\",\" + $0.getValue().str())">; + +// Convert UnaryEinsumOp to EinsumOp with two operands with redundant first +// operand. +def UnaryEinsumToEinsum : Pat< + (HLO_UnaryEinsumOp $operand, $equation), + (HLO_EinsumOp (HLO_ConstOp (GetScalarOfType<1> $operand)), + $operand, (UnaryToBinaryEinsumEq $equation))>; diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc index d43dd71e94b..b5eacd686bd 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/IR/chlo_ops.cc @@ -15,10 +15,12 @@ limitations under the License. #include "mlir-hlo/Dialect/mhlo/IR/chlo_ops.h" +#include "mlir-hlo/Dialect/mhlo/IR/hlo_ops.h" #include "mlir-hlo/utils/broadcast_utils.h" #include "mlir/IR/Attributes.h" #include "mlir/IR/Builders.h" #include "mlir/IR/Diagnostics.h" +#include "mlir/IR/PatternMatch.h" #include "mlir/IR/StandardTypes.h" #include "mlir/IR/TypeUtilities.h" @@ -259,6 +261,48 @@ BROADCAST_BINARY_OP_DEFS(BroadcastXorOp); #undef BROADCAST_INFER_SHAPE_TYPE_OP_DEFS #undef BROADCAST_BINARY_OP_DEFS +static LogicalResult Verify(ConstantLikeOp op) { + if (op.value().getType() != op.getType().cast().getElementType()) + return op.emitOpError() << "value's type doesn't match element return type"; + return success(); +} + +LogicalResult ConstantLikeOp::inferReturnTypeComponents( + MLIRContext* context, Optional location, ValueRange operands, + DictionaryAttr attributes, RegionRange regions, + SmallVectorImpl& inferedReturnShapes) { + ConstantLikeOp::Adaptor op(operands, attributes); + if (failed(op.verify(location.getValue()))) return failure(); + Type element_type = op.value().getType(); + Type operand_type = op.operand().getType(); + if (operand_type.isa()) { + inferedReturnShapes.emplace_back(element_type); + } else { + const auto& shape = operand_type.cast().getShape(); + inferedReturnShapes.emplace_back(shape, element_type); + } + return success(); +} + +struct ConstantLikeToConstant : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + + LogicalResult matchAndRewrite(ConstantLikeOp op, + PatternRewriter& rewriter) const override { + auto op_type = op.operand().getType().cast(); + if (!op_type.hasStaticShape()) return failure(); + auto type = RankedTensorType::get(op_type.getShape(), op.value().getType()); + ElementsAttr attr = DenseElementsAttr::get(type, op.value()); + rewriter.replaceOpWithNewOp(op.getOperation(), attr); + return success(); + } +}; + +void ConstantLikeOp::getCanonicalizationPatterns( + OwningRewritePatternList& results, MLIRContext* context) { + results.insert(context); +} + #define GET_OP_CLASSES #include "mlir-hlo/Dialect/mhlo/IR/chlo_ops.cc.inc" diff --git a/tensorflow/compiler/mlir/hlo/tests/canonicalize.mlir b/tensorflow/compiler/mlir/hlo/tests/canonicalize.mlir index e793e213e50..15b1a150fdd 100644 --- a/tensorflow/compiler/mlir/hlo/tests/canonicalize.mlir +++ b/tensorflow/compiler/mlir/hlo/tests/canonicalize.mlir @@ -191,6 +191,20 @@ func @concatenate_const_2D_horizontal() -> tensor<2x2xi32> { return %2 : tensor<2x2xi32> } +// CHECK-LABEL: constant_like_constant +func @constant_like_constant(%arg0: tensor<3x4xi32>) -> tensor<3x4xf32> { + // CHECK: mhlo.constant dense<3.200000e+00> + %0 = "chlo.constant_like"(%arg0) { value = 3.2 : f32 } : (tensor<3x4xi32>) -> tensor<3x4xf32> + return %0 : tensor<3x4xf32> +} + +// CHECK-LABEL: constant_like_constant_dynamic +func @constant_like_constant_dynamic(%arg0: tensor<*xi32>) -> tensor<*xf32> { + // CHECK: chlo.constant_like + %0 = "chlo.constant_like"(%arg0) { value = 3.2 : f32 } : (tensor<*xi32>) -> tensor<*xf32> + return %0 : tensor<*xf32> +} + // CHECK-LABEL: dynamic_slice_variable_start func @dynamic_slice_variable_start(%arg0: tensor<3x4xi32>, %arg1: tensor, %arg2: tensor) -> tensor<1x4xi32> { // CHECK: "mhlo.dynamic-slice" diff --git a/tensorflow/compiler/mlir/xla/tests/legalize-tf.mlir b/tensorflow/compiler/mlir/xla/tests/legalize-tf.mlir index bad9c1ef279..9b32fb97260 100644 --- a/tensorflow/compiler/mlir/xla/tests/legalize-tf.mlir +++ b/tensorflow/compiler/mlir/xla/tests/legalize-tf.mlir @@ -1,5 +1,5 @@ // RUN: tf-opt "-xla-legalize-tf=allow-partial-conversion legalize-chlo=false" %s | FILECHECK_OPTS="" FileCheck %s -// RUN: tf-opt "-xla-legalize-tf=allow-partial-conversion legalize-chlo=true" -verify-diagnostics %s | FileCheck %s --check-prefix CHLO +// RUN: tf-opt "-xla-legalize-tf=allow-partial-conversion legalize-chlo=true" -verify-diagnostics %s | FileCheck %s --check-prefix CHLO --dump-input-filter=all // This test runs twice: // 1. Through FILECHECK_OPTS="" FileCheck with chlo legalization disabled since verifying // that the chlo ops emit produces more useful tests. @@ -1854,14 +1854,14 @@ func @abs_unranked(%arg0: tensor<*xf32>) -> tensor<*xf32> { func @acos(%arg0: tensor<2xf32>) -> tensor<2xf32> { // CHECK: "chlo.acos"(%arg0) : (tensor<2xf32>) -> tensor<2xf32> // CHLO: %[[VAL_1:.*]] = "mhlo.compare"({{.*}}) {comparison_direction = "NE"} -// CHLO: %[[VAL_3:.*]] = mhlo.constant dense<2.000000e+00> -// CHLO: %[[VAL_4:.*]] = mhlo.constant dense<1.000000e+00> // CHLO: %[[VAL_5:.*]] = mhlo.multiply %arg0, %arg0 +// CHLO: %[[VAL_4:.*]] = mhlo.constant dense<1.000000e+00> // CHLO: %[[VAL_6:.*]] = mhlo.subtract %[[VAL_4]], %[[VAL_5]] // CHLO: %[[VAL_7:.*]] = "mhlo.sqrt"(%[[VAL_6]]) // CHLO: %[[VAL_8:.*]] = mhlo.constant dense<1.000000e+00> // CHLO: %[[VAL_9:.*]] = mhlo.add %[[VAL_8]], %arg0 // CHLO: %[[VAL_10:.*]] = mhlo.atan2 %[[VAL_7]], %[[VAL_9]] +// CHLO: %[[VAL_3:.*]] = mhlo.constant dense<2.000000e+00> // CHLO: %[[VAL_11:.*]] = mhlo.multiply %[[VAL_3]], %[[VAL_10]] // CHLO: %[[VAL_12:.*]] = mhlo.constant dense<3.14159274> // CHLO: %[[VAL_13:.*]] = "mhlo.select"(%[[VAL_1]], %[[VAL_11]], %[[VAL_12]]) @@ -1870,6 +1870,27 @@ func @acos(%arg0: tensor<2xf32>) -> tensor<2xf32> { return %0 : tensor<2xf32> } +// CHECK-LABEL: @acos_dynamic +// CHLO-LABEL: @acos_dynamic +func @acos_dynamic(%arg0: tensor<*xf32>) -> tensor<*xf32> { + // CHECK: "chlo.acos"(%arg0) : (tensor<*xf32>) -> tensor<*xf32> +// CHLO: %[[VAL_1:.*]] = "mhlo.compare"({{.*}}) {comparison_direction = "NE"} +// CHLO: %[[VAL_5:.*]] = mhlo.multiply %arg0, %arg0 +// CHLO: %[[VAL_4:.*]] = "chlo.constant_like"(%arg0) {value = 1.000000e+00 : f32} +// CHLO: %[[VAL_6:.*]] = mhlo.subtract %[[VAL_4]], %[[VAL_5]] +// CHLO: %[[VAL_7:.*]] = "mhlo.sqrt"(%[[VAL_6]]) +// CHLO: %[[VAL_8:.*]] = "chlo.constant_like"(%arg0) {value = 1.000000e+00 : f32} +// CHLO: %[[VAL_9:.*]] = mhlo.add %[[VAL_8]], %arg0 +// CHLO: %[[VAL_10:.*]] = mhlo.atan2 %[[VAL_7]], %[[VAL_9]] +// CHLO: %[[VAL_3:.*]] = "chlo.constant_like"(%arg0) {value = 2.000000e+00 : f32} +// CHLO: %[[VAL_11:.*]] = mhlo.multiply %[[VAL_3]], %[[VAL_10]] +// CHLO: %[[VAL_12:.*]] = "chlo.constant_like"(%arg0) {value = 3.14159274 : f32} +// CHLO: %[[VAL_13:.*]] = "mhlo.select"(%[[VAL_1]], %[[VAL_11]], %[[VAL_12]]) +// CHLO: return %[[VAL_13]] + %0 = "tf.Acos"(%arg0) : (tensor<*xf32>) -> tensor<*xf32> + return %0 : tensor<*xf32> +} + // CHECK-LABEL: func @cast_dynamic_i2f func @cast_dynamic_i2f(%arg0: tensor) -> tensor { // CHECK: "mhlo.convert"(%arg0) : (tensor) -> tensor diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc index aa6f25570a1..1f63f2a9396 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc @@ -5807,10 +5807,20 @@ LogicalResult legalizeTF( if (legalize_chlo) { chlo::PopulateLegalizeChloToHloPatterns(context, &patterns); } + // ConstantLike op is convenient to create splat constants, but is + // canonicalized to plain HLO constant if statically shaped. Add the + // canonicalization pattern to pattern list to enable multi-hop lowering. + chlo::ConstantLikeOp::getCanonicalizationPatterns(patterns, context); ConversionTarget target(*context); if (legalize_chlo) { target.addIllegalDialect(); + + // Mark ConstantLikeOp as dynamically legal only when it doesn't have a + // static result type so that it gets canonicalized to MHLO constant. + target.addDynamicallyLegalOp([](Operation *op) { + return !op->getResultTypes().front().cast().hasStaticShape(); + }); } else { target.addLegalDialect(); } diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_patterns.td b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_patterns.td index f0ad04c8246..1d4c9503afa 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_patterns.td +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_patterns.td @@ -548,17 +548,17 @@ foreach Mapping = [ } // Expand acos to MHLO dialect as follows: -// acos(x) = 2 * atan(sqrt(1 - x^2) / (1 + x)) if x != -1 -// = pi if x == -1 +// acos(x) = 2 * atan(sqrt(1 - x^2) / (1 + x)) if x != -1 +// = pi if x == -1 def : Pat<(HLOClient_AcosOp $input), (HLO_SelectOp - (HLO_CompareOp $input, (HLO_ConstOp (ConstantSplat<"0"> $input)), - HLO_COMPARISON_DIRECTION_NE), - (HLO_MulOp (HLO_ConstOp (ConstantSplat<"2"> $input)), - (HLO_Atan2Op (HLO_SqrtOp (HLO_SubOp - (HLO_ConstOp (ConstantSplat<"1"> $input)), - (HLO_MulOp $input, $input))), - (HLO_AddOp (HLO_ConstOp (ConstantSplat<"1"> $input)), $input))), - (HLO_ConstOp (ConstantSplat<"M_PI"> $input)))>; + (HLO_CompareOp $input, (HLO_ConstantLike<"0"> $input), + HLO_COMPARISON_DIRECTION_NE), + (HLO_MulOp (HLO_ConstantLike<"2.0f"> $input), + (HLO_Atan2Op + (HLO_SqrtOp (HLO_SubOp + (HLO_ConstantLike<"1"> $input), (HLO_MulOp $input, $input))), + (HLO_AddOp (HLO_ConstantLike<"1"> $input), $input))), + (HLO_ConstantLike<"M_PI"> $input))>; // TODO(bixia): Lower Cast with a Complex type source operand or with // Truncate=True for floating point value conversions. @@ -594,6 +594,9 @@ def : Pat<(TF_BitcastOp:$res HLO_Tensor:$arg), (HLO_BitcastConvertOp $arg), [(BothElementTypesSameWidthIntOrFloat $res, $arg)]>; +// TODO(jpienaar): Lower constant like to constant to broadcast if dynamic +// and going to MHLO. + //===----------------------------------------------------------------------===// // Random ops. //===----------------------------------------------------------------------===// From 672d92ce3e7eb45762d1de36bf34ea635ff0fef9 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 12:19:58 -0700 Subject: [PATCH 0762/1017] Implement AllPermute collective in TF2 AllPermute takes - a list of devices participating in the collective - a permutation as a list of integers. - a tensor The list of devices replaces the need for group_key and group_size. The number of inputs only scales with the number of devices within one group. The integers in the permutation are based on indices of the list of devices. E.g. devices = {"GPU:0", "GPU:1"} and permutation = {1,0} means - devices[0] sends to devices[permutation[0]] and - devices[1] sends to devices[permutation[1]]. PiperOrigin-RevId: 325860722 Change-Id: I919438fa1cd75d684e47f927289de389e8dcb3f0 --- tensorflow/core/common_runtime/BUILD | 51 ++ .../base_collective_executor.cc | 1 + tensorflow/core/common_runtime/permuter.cc | 118 ++++ tensorflow/core/common_runtime/permuter.h | 89 +++ .../core/common_runtime/permuter_test.cc | 507 ++++++++++++++++++ tensorflow/core/framework/collective.h | 4 + 6 files changed, 770 insertions(+) create mode 100644 tensorflow/core/common_runtime/permuter.cc create mode 100644 tensorflow/core/common_runtime/permuter.h create mode 100644 tensorflow/core/common_runtime/permuter_test.cc diff --git a/tensorflow/core/common_runtime/BUILD b/tensorflow/core/common_runtime/BUILD index b46efe01474..4978a613707 100644 --- a/tensorflow/core/common_runtime/BUILD +++ b/tensorflow/core/common_runtime/BUILD @@ -269,6 +269,7 @@ filegroup( "threadpool_device.h", "process_state.h", "pool_allocator.h", + "permuter.h", ] + if_mkl(["//tensorflow/core/graph:mkl_graph_util_header"]), ) @@ -1129,6 +1130,27 @@ cc_library( ], ) +cc_library( + name = "permuter", + srcs = ["permuter.cc"], + hdrs = ["permuter.h"], + copts = tf_copts(), + deps = [ + ":base_collective_executor", + ":collective_rma_local", + ":collective_util", + ":copy_tensor", + ":device", + ":device_mgr", + ":dma_helper", + ":process_util", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/profiler/lib:traceme", + ], + alwayslink = 1, +) + cc_library( name = "pool_allocator", srcs = ["pool_allocator.cc"], @@ -1585,6 +1607,7 @@ tf_cuda_library( ":parallel_concat_optimizer", ":partitioning_utils", ":pending_counts", + ":permuter", ":placer", ":pool_allocator", ":process_state", @@ -1973,6 +1996,34 @@ tf_cc_tests_gpu( ], ) +tf_cc_tests_gpu( + name = "permuter_test", + size = "medium", + srcs = [ + "permuter_test.cc", + ], + linkstatic = tf_kernel_tests_linkstatic(), + tags = ["no_cuda_on_cpu_tap"], + deps = [ + ":core", + ":core_cpu", + ":core_cpu_internal", + "//tensorflow/core:all_kernels", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:ops", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "//tensorflow/core/common_runtime/gpu:gpu_runtime", + "//tensorflow/core/util:protos_test_cc", + "@com_google_absl//absl/memory", + ], +) + tf_cc_test_mkl( name = "mkl_runtime_tests", size = "small", diff --git a/tensorflow/core/common_runtime/base_collective_executor.cc b/tensorflow/core/common_runtime/base_collective_executor.cc index 80820c9022c..754f8196d29 100644 --- a/tensorflow/core/common_runtime/base_collective_executor.cc +++ b/tensorflow/core/common_runtime/base_collective_executor.cc @@ -255,6 +255,7 @@ void BaseCollectiveExecutor::ExecuteAsync(OpKernelContext* ctx, Tensor* output = ctx->mutable_output(0); const Tensor* input = (col_params.instance.type == REDUCTION_COLLECTIVE || col_params.instance.type == GATHER_COLLECTIVE || + col_params.instance.type == PERMUTE_COLLECTIVE || (col_params.instance.type == BROADCAST_COLLECTIVE && col_params.is_source)) ? &ctx->input(0) diff --git a/tensorflow/core/common_runtime/permuter.cc b/tensorflow/core/common_runtime/permuter.cc new file mode 100644 index 00000000000..c3081d6bc61 --- /dev/null +++ b/tensorflow/core/common_runtime/permuter.cc @@ -0,0 +1,118 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/common_runtime/permuter.h" + +#include "tensorflow/core/common_runtime/collective_rma_local.h" +#include "tensorflow/core/common_runtime/collective_util.h" +#include "tensorflow/core/common_runtime/copy_tensor.h" +#include "tensorflow/core/common_runtime/device.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/common_runtime/dma_helper.h" +#include "tensorflow/core/common_runtime/process_util.h" +#include "tensorflow/core/framework/allocator.h" +#include "tensorflow/core/framework/device_base.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { + +Permuter::Permuter() + : col_ctx_(nullptr), col_params_(nullptr), done_(nullptr), counter_(0) {} + +bool Permuter::CheckCounter() { + mutex_lock lock(mu_counter_); + ++counter_; + if (counter_ == 2) return true; + return false; +} + +StatusCallback Permuter::HalfDone() { + return [this](const Status& s) { + status_.Update(s); + if (CheckCounter()) done_(status_); + }; +} + +Status Permuter::InitializeCollectiveContext( + std::shared_ptr col_ctx) { + DCHECK(col_ctx->dev_mgr); + col_ctx_ = col_ctx; + col_params_ = &col_ctx->col_params; + return collective_util::InitializeDeviceAndLocality( + col_ctx->dev_mgr, col_ctx->device_name, &col_ctx->device, + &col_ctx->device_locality); +} + +void Permuter::Run(StatusCallback done) { + done_ = std::move(done); + for (int i = 0; i < col_params_->instance.devices.size(); ++i) { + if (col_ctx_->device_name == col_params_->instance.devices[i]) { + DispatchSend(i, col_params_->instance.permutation[i], col_ctx_->input, + HalfDone()); + continue; + } + if (col_ctx_->device_name == + col_params_->instance.devices[col_params_->instance.permutation[i]]) { + DispatchRecv(i, col_params_->instance.permutation[i], col_ctx_->output, + HalfDone()); + } + } +} + +void Permuter::DispatchSend(int src_rank, int target_rank, const Tensor* tensor, + const StatusCallback& done) { + string send_buf_key = + strings::StrCat(col_ctx_->exec_key, src_rank, target_rank); + VLOG(1) << "DispatchSend " << send_buf_key << " from_device " + << col_ctx_->device_name << " to_device " + << col_params_->instance.devices[target_rank] + << " target_rank=" << target_rank << " src_rank=" << src_rank; + col_ctx_->col_exec->PostToPeer(col_params_->instance.devices[target_rank], + col_params_->instance.task_names[target_rank], + send_buf_key, col_ctx_->device, + col_ctx_->op_ctx->op_device_context(), + col_ctx_->op_ctx->output_alloc_attr(0), tensor, + col_ctx_->device_locality, done); +} + +void Permuter::DispatchRecv(int src_rank, int target_rank, Tensor* tensor, + const StatusCallback& done) { + string recv_buf_key = + strings::StrCat(col_ctx_->exec_key, src_rank, target_rank); + VLOG(1) << "DispatchRecv " << recv_buf_key << " to_device " + << col_ctx_->device_name << " from_device " + << col_params_->instance.devices[src_rank] + << " target_rank=" << target_rank << " src_rank=" << src_rank; + col_ctx_->col_exec->RecvFromPeer(col_params_->instance.devices[src_rank], + col_params_->instance.task_names[src_rank], + col_params_->task.is_local[src_rank], + recv_buf_key, col_ctx_->device, + col_ctx_->op_ctx->op_device_context(), + col_ctx_->op_ctx->output_alloc_attr(0), + tensor, col_ctx_->device_locality, 0, done); +} +namespace { +REGISTER_COLLECTIVE(Permute, Permuter); +} // namespace + +} // namespace tensorflow diff --git a/tensorflow/core/common_runtime/permuter.h b/tensorflow/core/common_runtime/permuter.h new file mode 100644 index 00000000000..245168b4b0d --- /dev/null +++ b/tensorflow/core/common_runtime/permuter.h @@ -0,0 +1,89 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_PERMUTER_H_ +#define TENSORFLOW_CORE_COMMON_RUNTIME_PERMUTER_H_ + +#include +#include +#include +#include + +#include "tensorflow/core/common_runtime/base_collective_executor.h" +#include "tensorflow/core/framework/collective.h" + +namespace tensorflow { +class Device; + +// Implementation of collective permute. +// +// Permute takes +// - a list of devices participating in the collective +// - a permutation as a list of integers. +// - a tensor +// +// The list of devices replaces the need for group_key and group_size. The +// number of inputs only scales with the number of devices within one group. +// +// The integers in the permutation are based on indices of the list of devices. +// E.g. devices = {"GPU:0", "GPU:1"} and permutation = {1,0} means +// - devices[0] sends to devices[permutation[0]] and +// - devices[1] sends to devices[permutation[1]]. +// +// Each device sends exactly one tensor and receives exactly one tensor. +class Permuter : public CollectiveImplementationInterface { + public: + Permuter(); + ~Permuter() override = default; + + void Run(StatusCallback done) override; + + Status InitializeCollectiveParams(CollectiveParams* col_params) override { + return Status::OK(); + } + + // Initializes members of CollectiveContext not yet initialized, i.e. device + // and device_locality. Also saves the CollectiveContext in this object. + Status InitializeCollectiveContext( + std::shared_ptr col_ctx) override; + + Status InitializeCollectiveGroupRuntimeDetails( + CollGroupRuntimeDetails*) override { + return Status::OK(); + } + + private: + std::shared_ptr col_ctx_; + const CollectiveParams* col_params_; // Not owned + StatusCallback done_; + Status status_; + mutex mu_counter_; + int counter_ TF_GUARDED_BY(mu_counter_); + + void DispatchSend(int src_rank, int target_rank, const Tensor* tensor, + const StatusCallback& done); + + void DispatchRecv(int src_rank, int target_rank, Tensor* tensor, + const StatusCallback& done); + + // Checks if counter_ reaches 2. + // Atomically increments counter_ by one for sending, one for receiving. + // The purpose of this check is to ensure that done_ is called only once. + bool CheckCounter(); + + StatusCallback HalfDone(); +}; + +} // namespace tensorflow +#endif // TENSORFLOW_CORE_COMMON_RUNTIME_PERMUTER_H_ diff --git a/tensorflow/core/common_runtime/permuter_test.cc b/tensorflow/core/common_runtime/permuter_test.cc new file mode 100644 index 00000000000..a5117322ffa --- /dev/null +++ b/tensorflow/core/common_runtime/permuter_test.cc @@ -0,0 +1,507 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/common_runtime/permuter.h" + +#include + +#include "absl/memory/memory.h" +#include "tensorflow/core/common_runtime/base_collective_executor.h" +#include "tensorflow/core/common_runtime/collective_rma_local.h" +#include "tensorflow/core/common_runtime/device_mgr.h" +#include "tensorflow/core/common_runtime/device_resolver_local.h" +#include "tensorflow/core/common_runtime/process_util.h" +#include "tensorflow/core/common_runtime/test_collective_executor_mgr.h" +#include "tensorflow/core/common_runtime/threadpool_device.h" +#include "tensorflow/core/framework/collective.h" +#include "tensorflow/core/framework/fake_input.h" +#include "tensorflow/core/framework/node_def.pb.h" +#include "tensorflow/core/framework/node_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/platform/unbounded_work_queue.h" +#include "tensorflow/core/public/session_options.h" +#include "tensorflow/core/public/version.h" + +namespace tensorflow { +namespace { + +static int64 kStepId = 123; + +// Wraps CollectiveRemoteAccessLocal with the ability to return an +// error status to the N'th action. +// TODO(b/113171733): factor out of this file and ring_reducer_test.cc +// into a single common source. +class FailTestRMA : public CollectiveRemoteAccessLocal { + public: + FailTestRMA(const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver, + std::shared_ptr work_queue, int64 step_id, + int fail_after) + : CollectiveRemoteAccessLocal(dev_mgr, dev_resolver, work_queue, step_id), + fail_after_(fail_after) {} + + bool MaybeFail(const StatusCallback& done) { + bool fail_now = false; + { + mutex_lock l(mu_); + if (fail_after_ > 0) { + fail_now = (--fail_after_ == 0); + } + } + if (fail_now) { + auto error = errors::Internal("Deliberate failure"); + LOG(INFO) << "triggering failure " << error; + SchedNonBlockingClosureAfter( + 1000, [this, error] { buf_rendezvous()->StartAbort(error); }); + done(error); + return true; + } + return false; + } + + void RecvFromPeer(const string& peer_device, const string& peer_task, + bool peer_is_local, const string& key, Device* to_device, + DeviceContext* to_device_ctx, + const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor, + const DeviceLocality& client_locality, int stream_index, + const StatusCallback& done) override { + if (MaybeFail(done)) return; + CollectiveRemoteAccessLocal::RecvFromPeer( + peer_device, peer_task, peer_is_local, key, to_device, to_device_ctx, + to_alloc_attr, to_tensor, client_locality, stream_index, done); + } + + void PostToPeer(const string& peer_device, const string& peer_task, + const string& key, Device* from_device, + DeviceContext* from_device_ctx, + const AllocatorAttributes& from_alloc_attr, + const Tensor* from_tensor, + const DeviceLocality& client_locality, + const StatusCallback& done) override { + if (MaybeFail(done)) return; + CollectiveRemoteAccessLocal::PostToPeer( + peer_device, peer_task, key, from_device, from_device_ctx, + from_alloc_attr, from_tensor, client_locality, done); + } + + mutex mu_; + int fail_after_ TF_GUARDED_BY(mu_); +}; + +class PermuterTest : public ::testing::Test { + protected: + PermuterTest() : device_type_(DEVICE_CPU) {} + + ~PermuterTest() override { + stop_ = true; + for (auto i : instances_) delete i; + if (col_exec_) col_exec_->Unref(); + } + +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + void InitGPUDevices() { + auto device_factory = DeviceFactory::GetFactory("GPU"); + CHECK(device_factory); + SessionOptions options; + Status s = device_factory->CreateDevices( + options, "/job:worker/replica:0/task:0", &gpu_devices_); + CHECK(s.ok()); + } +#endif + + void Init(int num_workers, int num_devices_per_worker, DataType dtype, + const DeviceType& device_type, int fail_after) { +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM + InitGPUDevices(); +#endif + device_type_ = device_type; + std::vector> local_devices; + SessionOptions sess_opts; + sess_opts.env = Env::Default(); + Bytes mem_limit(4 << 20); + DeviceLocality dev_locality; + for (int wi = 0; wi < num_workers; ++wi) { + for (int di = 0; di < num_devices_per_worker; ++di) { + if (device_type == DEVICE_CPU) { + string dev_name = strings::StrCat("/job:worker/replica:0/task:", wi, + "/device:CPU:", di); + local_devices.push_back(absl::make_unique( + sess_opts, dev_name, mem_limit, dev_locality, cpu_allocator())); + } else if (device_type == DEVICE_GPU && !gpu_devices_.empty()) { + int dev_idx = (wi * num_devices_per_worker) + di; + if (dev_idx >= static_cast(gpu_devices_.size())) { + LOG(INFO) << "dev_mgr has access to limited GPUs, reusing for more " + "than one ring node."; + } else { + local_devices.push_back(std::move(gpu_devices_[dev_idx])); + } + } else { + LOG(FATAL) << "Unsupported device_type " << device_type; + } + } + } + if (!dev_mgr_ || device_type == DEVICE_CPU) { + dev_mgr_ = absl::make_unique(std::move(local_devices)); + } + if (!gpu_ring_order_) { + gpu_ring_order_ = absl::make_unique(); + } + dev_resolver_ = absl::make_unique(dev_mgr_.get()); + work_queue_ = std::make_shared(Env::Default(), "test"); + rma_ = new FailTestRMA(dev_mgr_.get(), dev_resolver_.get(), work_queue_, + kStepId, fail_after); + col_exec_ = new BaseCollectiveExecutor( + &col_exec_mgr_, rma_, kStepId, dev_mgr_.get(), gpu_ring_order_.get()); + col_params_.name = "test_collective"; + col_params_.instance.data_type = dtype; + static const int kInstanceKey = 18; + col_params_.instance.instance_key = kInstanceKey; + col_params_.group.device_type = device_type; + col_params_.instance.type = PERMUTE_COLLECTIVE; + + // Set up all the fake device contexts. + for (int wi = 0; wi < num_workers; wi++) { + for (int di = 0; di < num_devices_per_worker; di++) { + string task_name = strings::StrCat("/job:worker/replica:0/task:", wi); + string dev_name; + if (device_type == DEVICE_GPU) { + dev_name = strings::StrCat(task_name, "/device:GPU:0"); + } else { + dev_name = strings::StrCat(task_name, "/device:CPU:", di); + } + col_params_.instance.device_names.push_back(dev_name); + col_params_.instance.devices.push_back(dev_name); + int default_rank = wi * num_devices_per_worker + di; + permutation_.push_back(default_rank); + col_params_.instance.task_names.push_back(task_name); + col_params_.task.is_local.push_back(true); + } + } + + // Generate a permutation by permuting every two instances. + // E.g. [0,1] becomes [1,0] + // [0,1,2,3] becomes [1,0,3,2] + for (int i = 0; i < permutation_.size(); i += 2) { + // If the total number of instances is odd, + // swap the last instance with the first. + // E.g. [0,1,2] becomes [2,0,1] + if (permutation_.size() == i + 1) { + std::swap(permutation_[i], permutation_[0]); + continue; + } + std::next_permutation(permutation_.begin() + i, + permutation_.begin() + i + 2); + } + col_params_.instance.permutation = permutation_; + + for (int wi = 0; wi < num_workers; wi++) { + for (int di = 0; di < num_devices_per_worker; di++) { + int default_rank = wi * num_devices_per_worker + di; + instances_.push_back(new DeviceInstance( + default_rank, col_params_.instance.device_names[default_rank], + device_type, this)); + } + } + } + + typedef std::function InitFunc; + + void Permute(int fail_after) { + std::atomic done(0); + for (auto di : instances_) { + SchedClosure([di, &done] { + di->DoPermute(); + ++done; + }); + if (fail_after > 0) { + // Stagger the op execution starts. + Env::Default()->SleepForMicroseconds(100); + } + } + while (done < instances_.size()) { + if (stop_) break; + Env::Default()->SleepForMicroseconds(1000); + } + } + + template + void RunTest(DataType dtype, const DeviceType& device_type, int num_workers, + int num_devices, int tensor_len, int fail_after) { + Init(num_workers, num_devices, dtype, device_type, fail_after); + std::vector expected(tensor_len * num_devices * num_workers, 0.0); + // Initialize each instance tensor with distinct values. + for (int di = 0; di < instances_.size(); ++di) { + DeviceInstance* instance = instances_[di]; + instance->InitTensor( + dtype, TensorShape({tensor_len}), + [this, &expected, di, tensor_len](Tensor* t) { + for (size_t i = 0; i < t->NumElements(); ++i) { + // The cast is necessary to prevent clang-tidy from insisting + // that a faster non-open source function be substituted. + float value = pow(10, static_cast(di)) * i; + t->flat()(i) = value; + expected[permutation_[di] * tensor_len + i] = value; + } + }); + } + + Permute(fail_after); + + // At this point all of the ops have terminated. + for (int di = 0; di < instances_.size(); ++di) { + if (!instances_[di]->status_.ok()) { + ASSERT_GT(fail_after, 0); + ASSERT_NE( + instances_[di]->status_.error_message().find("Deliberate failure"), + string::npos); + continue; + } + TF_EXPECT_OK(instances_[di]->status_); + Tensor* inst = &instances_[di]->tensor_output_; + Tensor actual(dtype, TensorShape({tensor_len})); + if (device_type_ == DEVICE_CPU) { + CHECK(actual.CopyFrom(*inst, inst->shape())); + } else if (device_type_ == DEVICE_GPU) { + Device* dev = instances_[di]->device_; + auto* dev_info = dev->tensorflow_gpu_device_info(); + CHECK(dev_info); + TF_CHECK_OK(dev_info->default_context->CopyDeviceTensorToCPUSync( + inst, "" /*tensor_name*/, dev, &actual)); + } + for (int i = 0; i < tensor_len; ++i) { + switch (dtype) { + case DT_FLOAT: + EXPECT_FLOAT_EQ(expected[(di * tensor_len) + i], + actual.template flat()(i)) + << "Mismatch at device " << di << " index " << i; + break; + case DT_DOUBLE: + EXPECT_DOUBLE_EQ(expected[(di * tensor_len) + i], + actual.template flat()(i)) + << "Mismatch at device " << di << " index " << i; + break; + case DT_INT32: + case DT_INT64: + EXPECT_EQ(expected[(di * tensor_len) + i], + actual.template flat()(i)) + << "Mismatch at device " << di << " index " << i; + break; + default: + LOG(FATAL) << "unimplemented"; + } + } + // } + } + } + + class DeviceInstance { + public: + DeviceInstance(int rank, const string& dev_name, + const DeviceType& device_type, PermuterTest* parent) + : parent_(parent), + dev_name_(dev_name), + device_type_(device_type), + rank_(rank) { + TF_CHECK_OK(parent_->dev_mgr_->LookupDevice(dev_name, &device_)); + col_params_.name = parent_->col_params_.name; + col_params_.instance.data_type = parent_->col_params_.instance.data_type; + col_params_.instance.instance_key = + parent_->col_params_.instance.instance_key; + col_params_.group.device_type = parent_->col_params_.group.device_type; + col_params_.instance.device_names = + parent_->col_params_.instance.device_names; + col_params_.instance.devices = parent_->col_params_.instance.devices; + col_params_.instance.permutation = + parent->col_params_.instance.permutation; + col_params_.instance.task_names = + parent_->col_params_.instance.task_names; + col_params_.task.is_local = parent_->col_params_.task.is_local; + CHECK_EQ(col_params_.instance.devices.size(), + col_params_.instance.device_names.size()); + // Default rank is order in device_names. + col_params_.default_rank = rank; + } + + void InitTensor(DataType dtype, const TensorShape& shape, + const InitFunc& f) { + tensor_input_ = + Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape); + tensor_output_ = + Tensor(device_->GetAllocator(AllocatorAttributes()), dtype, shape); + if (device_type_ == DEVICE_CPU) { + f(&tensor_input_); + } else if (device_type_ == DEVICE_GPU) { + Tensor cpu_tensor(dtype, shape); + f(&cpu_tensor); + // Notification notification; + auto* dev_info = device_->tensorflow_gpu_device_info(); + CHECK(dev_info); + TF_CHECK_OK(dev_info->default_context->CopyCPUTensorToDeviceSync( + &cpu_tensor, device_, &tensor_input_)); + } else { + LOG(FATAL) << "Unsupported device_type " << device_type_; + } + } + + void DoPermute() { + // Prepare an OpKernelContext. + OpKernelContext::Params op_params; + op_params.step_id = parent_->step_id_; + op_params.device = device_; + gtl::InlinedVector inputs; + inputs.push_back(TensorValue(&tensor_input_)); + op_params.inputs = &inputs; + gtl::InlinedVector input_aa( + {AllocatorAttributes()}); + op_params.input_alloc_attrs = &input_aa; + DeviceContext* dev_ctx = nullptr; + auto* dev_info = device_->tensorflow_gpu_device_info(); + if (dev_info) { + dev_ctx = dev_info->default_context; + dev_ctx->Ref(); + } else { + dev_ctx = new DeviceContext; + } + op_params.op_device_context = dev_ctx; + AllocatorAttributes generic_alloc_attr; + op_params.output_attr_array = &generic_alloc_attr; + OpKernelContext ctx(&op_params, 1); + + // Prepare a Permuter instance. + string exec_key = + strings::StrCat(col_params_.instance.instance_key, ":0:0"); + Permuter* permuter = new Permuter; + core::ScopedUnref unref(permuter); + auto col_ctx = std::make_shared( + parent_->col_exec_, parent_->dev_mgr_.get(), &ctx, &op_params, + col_params_, exec_key, kStepId, &tensor_input_, &tensor_output_); + TF_CHECK_OK(permuter->InitializeCollectiveContext(col_ctx)); + Notification note; + // Run the permute. + permuter->Run([this, ¬e](Status s) { + status_ = s; + note.Notify(); + }); + note.WaitForNotification(); + dev_ctx->Unref(); + } + + PermuterTest* parent_; + string dev_name_; + DeviceType device_type_ = DEVICE_CPU; + int rank_; + Tensor tensor_input_; + Tensor tensor_output_; + Device* device_; + CollectiveParams col_params_; + Status status_; + }; // class DeviceInstance + + bool stop_ = false; + int64 step_id_ = kStepId; + DeviceType device_type_; + TestCollectiveExecutorMgr col_exec_mgr_; + CollectiveExecutor* col_exec_ = nullptr; + CollectiveRemoteAccessLocal* rma_; + std::unique_ptr dev_resolver_; + std::shared_ptr work_queue_; + std::vector instances_; + CollectiveParams col_params_; + std::vector> gpu_devices_; + std::unique_ptr dev_mgr_; + std::unique_ptr gpu_ring_order_; + mutex mu_; + int permute_counter_ TF_GUARDED_BY(mu_) = 0; + std::vector permutation_; +}; + +// TODO(b/113171733): change to use TEST_P. +// Tests of full permute algorithm, with different device and +// data types. +// B = data element type +// T = device type +// W = number of workers +// D = number of devices per worker +// L = tensor length +// A = abort after count +#define DEF_TEST(B, T, W, D, L, A) \ + TEST_F(PermuterTest, \ + DaTy##B##_DevTy##T##_Wkr##W##_Dev##D##_Sdiv##S##_Len##L##_Abrt##A) { \ + DataType dtype = DT_##B; \ + switch (dtype) { \ + case DT_FLOAT: { \ + RunTest(dtype, DEVICE_##T, W, D, L, A); \ + } break; \ + case DT_DOUBLE: { \ + RunTest(dtype, DEVICE_##T, W, D, L, A); \ + } break; \ + case DT_INT32: { \ + RunTest(dtype, DEVICE_##T, W, D, L, A); \ + } break; \ + case DT_INT64: { \ + RunTest(dtype, DEVICE_##T, W, D, L, A); \ + } break; \ + default: \ + LOG(FATAL) << "Unimplemented"; \ + } \ + } + +#if !(GOOGLE_CUDA || TENSORFLOW_USE_ROCM) +// B T W D L A +DEF_TEST(FLOAT, CPU, 1, 2, 1, 0) +DEF_TEST(FLOAT, CPU, 1, 3, 3, 0) +DEF_TEST(FLOAT, CPU, 1, 7, 3, 0) +DEF_TEST(FLOAT, CPU, 1, 2, 1001, 0) +DEF_TEST(FLOAT, CPU, 2, 2, 3, 0) +DEF_TEST(FLOAT, CPU, 2, 1, 128, 0) +DEF_TEST(FLOAT, CPU, 2, 4, 128, 0) +DEF_TEST(FLOAT, CPU, 2, 8, 4095, 0) +DEF_TEST(FLOAT, CPU, 4, 4, 1045991, 0) + +DEF_TEST(DOUBLE, CPU, 2, 4, 128, 0) +DEF_TEST(INT32, CPU, 2, 4, 128, 0) +DEF_TEST(INT64, CPU, 2, 4, 128, 0) + +// Failure cases +DEF_TEST(FLOAT, CPU, 1, 2, 1, 1) +DEF_TEST(FLOAT, CPU, 2, 4, 128, 1) +DEF_TEST(FLOAT, CPU, 2, 4, 128, 5) +#endif + +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM +// Can only set W=1 for GPU tests. +// B T W D L A +DEF_TEST(FLOAT, GPU, 1, 2, 1, 0) +DEF_TEST(FLOAT, GPU, 1, 7, 3, 0) +DEF_TEST(FLOAT, GPU, 1, 2, 33, 0) +DEF_TEST(FLOAT, GPU, 1, 3, 64, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 1001, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 4095, 0) +DEF_TEST(FLOAT, GPU, 1, 8, 1045991, 0) + +DEF_TEST(BOOL, GPU, 1, 4, 1, 0) +DEF_TEST(BOOL, GPU, 1, 4, 1001, 0) + +DEF_TEST(DOUBLE, GPU, 1, 8, 1001, 0) +DEF_TEST(INT64, GPU, 1, 8, 1001, 0) + +// Failure cases +DEF_TEST(FLOAT, GPU, 1, 8, 128, 6) +#endif + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h index 94e83fa2f08..e7110d9512c 100644 --- a/tensorflow/core/framework/collective.h +++ b/tensorflow/core/framework/collective.h @@ -43,6 +43,7 @@ enum CollectiveType { REDUCTION_COLLECTIVE = 0, BROADCAST_COLLECTIVE, GATHER_COLLECTIVE, + PERMUTE_COLLECTIVE, UNDEFINED_COLLECTIVE, }; @@ -89,6 +90,7 @@ struct CollImplDetails { }; // Data common to all members of a collective instance. +// TODO(b/163171014) Refactor this struct to not be a union of all fields. struct CollInstanceParams { // Identifies all participating graph nodes. int32 instance_key = -1; @@ -109,6 +111,8 @@ struct CollInstanceParams { CollImplDetails impl_details; string ToString() const; CollInstanceParams& operator=(const struct CollInstanceParams& other); + std::vector devices; // all_permute only + std::vector permutation; // all_permute only }; // Data common to all instance members in the same task. From 413823fcb11f2658ba16219d0dce71ed09082557 Mon Sep 17 00:00:00 2001 From: Deven Desai Date: Sat, 8 Aug 2020 13:15:33 +0000 Subject: [PATCH 0763/1017] Enabling XLA specific subtests (for ROCm) within the test `def_function_xla_jit_test` --- .../python/eager/def_function_xla_jit_test.py | 31 +++++-------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/tensorflow/python/eager/def_function_xla_jit_test.py b/tensorflow/python/eager/def_function_xla_jit_test.py index 813c1377cd9..ba75aed5f1c 100644 --- a/tensorflow/python/eager/def_function_xla_jit_test.py +++ b/tensorflow/python/eager/def_function_xla_jit_test.py @@ -76,9 +76,7 @@ class DefFunctionTest(xla_test.XLATestCase): inputs = constant_op.constant([1, 2, 2, 3, 3]) self.assertAllClose([2, 3, 3, 4, 4], func(inputs, 1)) - if not test.is_built_with_rocm(): - # XLA support is not yet enabled for TF ROCm - self.assertAllClose([2, 3, 3, 4, 4], xla_func(inputs, 1)) + self.assertAllClose([2, 3, 3, 4, 4], xla_func(inputs, 1)) def testBasicInt32(self): with ops.device('device:{}:0'.format(self.device)): @@ -88,14 +86,10 @@ class DefFunctionTest(xla_test.XLATestCase): return x + a inputs = constant_op.constant([1, 2, 2, 3, 3], dtype=dtypes.int32) - if not test.is_built_with_rocm(): - # XLA support is not yet enabled for TF ROCm - self.assertAllClose([2, 3, 3, 4, 4], fn(inputs, 1)) + self.assertAllClose([2, 3, 3, 4, 4], fn(inputs, 1)) def testDerivative(self): with ops.device('device:{}:0'.format(self.device)): - if test.is_built_with_rocm(): - return def fn(x, a): return 2 * x + a @@ -135,9 +129,7 @@ class DefFunctionTest(xla_test.XLATestCase): return fn(x, a) inputs = constant_op.constant([1, 2, 2, 3, 3]) - if not test.is_built_with_rocm(): - # XLA support is not yet enabled for TF ROCm - self.assertAllClose([2, 3, 3, 4, 4], fn2(inputs, 1)) + self.assertAllClose([2, 3, 3, 4, 4], fn2(inputs, 1)) @test_util.disable_mlir_bridge('TODO(b/162272821): MLIR bridge returns' ' wrong status type') @@ -154,10 +146,9 @@ class DefFunctionTest(xla_test.XLATestCase): func = def_function.function(fn2, experimental_compile=False) inputs = constant_op.constant([1, 2, 2, 3, 3]) - if not test.is_built_with_rocm(): - with self.assertRaisesRegex(errors.InvalidArgumentError, - 'not compilable'): - func(inputs) + with self.assertRaisesRegex(errors.InvalidArgumentError, + 'not compilable'): + func(inputs) @test_util.disable_mlir_bridge('TODO(b/162272821): MLIR bridge returns' ' wrong status type') @@ -196,9 +187,7 @@ class DefFunctionTest(xla_test.XLATestCase): self.assertAllClose(3.0, dy) run_and_check(func) - if not test.is_built_with_rocm(): - # XLA support is not yet enabled for TF ROCm - run_and_check(xla_func) + run_and_check(xla_func) @test_util.disable_mlir_bridge('TODO(b/162521846): MLIR bridge fails' ' msan, function library not found') @@ -233,8 +222,6 @@ class DefFunctionTest(xla_test.XLATestCase): self.assertAllClose([40.0, 28.0], g(2.0)) def testMethodCompilation(self): - if test.is_built_with_rocm(): - return with ops.device('device:{}:0'.format(self.device)): @@ -251,8 +238,6 @@ class DefFunctionTest(xla_test.XLATestCase): @test_util.disable_mlir_bridge('TODO(b/162272821): MLIR bridge returns ' ' wrong status type') def testMethodCompilationUnsupportedFunc(self): - if test.is_built_with_rocm(): - return with ops.device('device:{}:0'.format(self.device)): @@ -273,8 +258,6 @@ class DefFunctionTest(xla_test.XLATestCase): self.skipTest('b/162799319: Cannot resolve constant on TPU') with ops.device('device:{}:0'.format(self.device)): - if test.is_built_with_rocm(): - return @def_function.function(experimental_compile=True) def f(): From 35475aae9b38d00129652a2f316730c8c17398fd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 13:30:12 -0700 Subject: [PATCH 0764/1017] [XLA:SPMD] Support partial replicate to tile resharding by dynamic slice. PiperOrigin-RevId: 325876094 Change-Id: I6a085904de8e3ab21f3e5107ce70c22c6a892cea --- .../compiler/xla/service/hlo_sharding.cc | 7 +- .../xla/service/spmd/spmd_partitioner.cc | 71 +++++++ .../xla/service/spmd/spmd_partitioner_test.cc | 47 +++++ .../xla/service/spmd/spmd_partitioner_util.cc | 198 +++++++++++++++++- .../xla/service/spmd/spmd_partitioner_util.h | 24 +++ 5 files changed, 343 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc index 07444aca82b..92270005ffd 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding.cc @@ -219,8 +219,11 @@ std::vector HloSharding::TileOffsetForDevice(const Shape& shape, if (maximal_) { return std::vector(shape.dimensions_size(), 0); } - - CHECK_EQ(shape.dimensions_size(), tile_assignment_.num_dimensions()); + if (replicate_on_last_tile_dim_) { + CHECK_EQ(shape.dimensions_size(), tile_assignment_.num_dimensions() - 1); + } else { + CHECK_EQ(shape.dimensions_size(), tile_assignment_.num_dimensions()); + } std::vector index = TileIndexForDevice(device); for (int64 i = 0; i < index.size(); ++i) { const int64 shape_dim = shape.dimensions(i); diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc index fc065bcdd72..8006e47d90d 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc @@ -282,6 +282,77 @@ PartitionedHlo PartitionedHlo::ReshardNoCache(const HloSharding& target) { return ReshardWithAllToAll(target, *src_tgt_dims); } + // Partial replicated to tiled. + if (sharding().ReplicateOnLastTileDim() && !target.ReplicateOnLastTileDim() && + !target.IsTileMaximal()) { + // Get the temp sharding target from partial replicate to target tile dims. + // target_compatible_sharding has the same tile_assignment dimensions + // as the target and can reshard to target by collective permute. + // target_compatible_sharding could have different device assignment as + // targe. sharding() can reshard to target_compatible_sharding by + // dynamic slice. + auto target_compatible_sharding = PartialReplicateToTileCompatibleSharding( + sharding(), target.tile_assignment().dimensions()); + // Reshard to target_compatible_sharding by dynamic slice. + if (target_compatible_sharding.has_value()) { + std::vector expand_tile_dims; + std::vector tiling_dim_factors; + int64 rank = shape.rank(); + tiling_dim_factors.reserve(rank); + auto temp_target_sharding = target_compatible_sharding.value(); + for (int64 dim = 0; dim < rank; dim++) { + if (temp_target_sharding.tile_assignment().dim(dim) > + sharding().tile_assignment().dim(dim)) { + expand_tile_dims.push_back(dim); + } + tiling_dim_factors.emplace_back( + temp_target_sharding.tile_assignment().dim(dim) / + sharding().tile_assignment().dim(dim)); + } + + // Get per_group partitioner state. + std::vector group_dims( + sharding().tile_assignment().num_dimensions() - 1); + std::iota(group_dims.begin(), group_dims.end(), 0); + auto sharding_grouped = GroupShardingOnDims(sharding(), group_dims); + auto per_group_partitioner_state = CreatePerGroupPartitioningState( + state_, sharding_grouped.device_groups, state_.b); + // 2. Get the padded_hlo, do right halo exchange if needed. + auto padded_hlo = PadFromPartialReplicateShape( + hlo_, base_shape_, sharding(), temp_target_sharding, expand_tile_dims, + state_.collective_ops_creator, state_.next_channel_id, + state_.partition_id, state_.b); + if (padded_hlo.has_value()) { + // 3. Slice out the tile from replicate ones. + auto shard_shape = + MakePartitionedShape(base_shape_, temp_target_sharding); + // device assignment within each group is sorted in + // HloSharding::PartialTile, thus partiton_id within each group can be + // matched with the order in tile_assignment. + Array tiling_assignment(tiling_dim_factors); + tiling_assignment.FillIota(0); + auto slice = + state_.b->AddInstruction(HloInstruction::CreateDynamicSlice( + shard_shape, padded_hlo.value(), + MakePartitionOffsets(padded_hlo.value()->shape(), + HloSharding::Tile(tiling_assignment), + per_group_partitioner_state.partition_id, + per_group_partitioner_state.b), + shard_shape.dimensions())); + slice->set_sharding(temp_target_sharding); + auto result = PartitionedHlo(slice, base_shape_, state_); + // If temp_target_sharding's device assignment is different from target, + // use collective permute to reshard. + if (CanReshardWithCollectivePermute(temp_target_sharding, target)) { + return result.ReshardWithCollectivePermute(target); + } + // If device assignment in temp_target_sharding and target are the same, + // return result directly. + return result; + } + } + } + // If not replicated yet, first replicate and then reshard to use one of the // two implementations below. if (!sharding().IsReplicated()) { diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc index 386d634779b..3ffe2954d61 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc @@ -4348,6 +4348,53 @@ ENTRY entry { EXPECT_THAT(root, AllOf(op::Shape("f32[4,4,12,32]"), op::Reshape(xpose))); } +TEST_F(SpmdPartitioningTest, + ElementwiseTest_PartialReplicateToTiledHaloExchange) { + const char* const hlo_string = R"( +HloModule module + +ENTRY entry { + constant = f32[6,3]{1,0} + constant({{1,3,7},{5,1,4},{1,2,8},{2,3,7},{5,2,4},{2,2,8}}), + sharding={replicated} + constant.1 = f32[6,3]{1,0} + constant({{2,7,2},{2,9,2},{2,6,2},{3,7,2},{2,9,3},{2,3,2}}), + sharding={replicated} + multiply = f32[6,3]{1,0} multiply(constant, constant.1), + sharding={devices=[2,1,2]0,1,2,3 last_tile_dim_replicate} + ROOT add = f32[6,3]{1,0} add(multiply, constant.1), + sharding={devices=[4,1]0,1,2,3} +} +)"; + + TF_ASSERT_OK_AND_ASSIGN(auto module, + PartitionComputation(hlo_string, /*num_devices=*/4)); + VLOG(1) << module->ToString(); + auto partial_replicate_lhs = + AllOf(op::Shape("f32[3,3]"), + op::DynamicSlice(op::Constant(), op::Reshape(), op::Constant())); + auto partial_replicate_rhs = + AllOf(op::Shape("f32[3,3]"), + op::DynamicSlice(op::Constant(), op::Reshape(), op::Constant())); + auto multiply = + AllOf(op::Shape("f32[3,3]"), + op::Multiply(partial_replicate_lhs, partial_replicate_rhs)); + auto right_halo = + AllOf(op::Shape("f32[1,3]"), op::CollectivePermute(op::Slice(multiply))); + auto add_lhs = AllOf( + op::Shape("f32[2,3]"), + op::DynamicSlice( + op::DynamicSlice( + op::Pad(op::Concatenate(multiply, right_halo), op::Constant()), + op::Reshape(), op::Constant()), + op::Reshape(), op::Constant())); + auto add_rhs = AllOf(op::Shape("f32[2,3]"), + op::DynamicSlice(op::Pad(op::Constant(), op::Constant()), + op::Reshape(), op::Constant())); + auto root = module->entry_computation()->root_instruction(); + EXPECT_THAT(root, AllOf(op::Shape("f32[2,3]"), op::Add(add_lhs, add_rhs))); +} + } // namespace } // namespace spmd } // namespace xla diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc index 767bed2a21a..3443c6e013d 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc @@ -32,9 +32,11 @@ limitations under the License. #include "tensorflow/compiler/xla/service/hlo_sharding.h" #include "tensorflow/compiler/xla/service/hlo_sharding_util.h" #include "tensorflow/compiler/xla/service/pattern_matcher.h" +#include "tensorflow/compiler/xla/service/shape_inference.h" #include "tensorflow/compiler/xla/service/spmd/spmd_partitioner.h" #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/util.h" +#include "tensorflow/compiler/xla/window_util.h" #include "tensorflow/compiler/xla/xla_data.pb.h" namespace xla { @@ -229,8 +231,11 @@ std::vector MakePartitionOffsets( std::vector MakeTiledPartitionOrdinals( const HloSharding& sharding, HloInstruction* partition_id, SpmdBuilder* b) { CHECK(!sharding.IsTileMaximal()); - auto table_shape = - ShapeUtil::MakeShape(S32, sharding.tile_assignment().dimensions()); + auto dimensions = sharding.tile_assignment().dimensions(); + if (sharding.ReplicateOnLastTileDim()) { + dimensions.pop_back(); + } + auto table_shape = ShapeUtil::MakeShape(S32, dimensions); return MakePartitionOffsets(table_shape, sharding, partition_id, b); } @@ -287,6 +292,195 @@ HloInstruction* PadBaseShapeBeforeUnevenTiledSharding( return PadToShape(hlo, padded_base_shape, b); } +// TODO(wangtao): generize this function when target is partial replicate. +absl::optional PartialReplicateToTileCompatibleSharding( + const HloSharding& partial_sharding, + const std::vector& target_tile_dims) { + if (!partial_sharding.ReplicateOnLastTileDim()) { + return absl::nullopt; + } + int64 rank = partial_sharding.tile_assignment().num_dimensions() - 1; + if (target_tile_dims.size() < rank) { + return absl::nullopt; + } + // A dimension is expanded when target_tile_size > partial_tile_size and + // target_tile_size % partial_tile_size == 0. + // expand_tile_dims_positions is the index of the expand_dim. + std::vector expand_tile_dims_indices(rank, -1); + // expand_tile_size = target_tile_size / partial_tile_size. + std::vector expand_tile_sizes; + int num_expand_dims = 0; + for (int64 dim = 0; dim < rank; dim++) { + int64 partial_tile_size = partial_sharding.tile_assignment().dim(dim); + int64 target_tile_size = target_tile_dims[dim]; + if (target_tile_size % partial_tile_size != 0 || + target_tile_size < partial_tile_size) { + return absl::nullopt; + } + + if (target_tile_size > partial_tile_size) { + expand_tile_dims_indices[dim] = num_expand_dims++; + expand_tile_sizes.emplace_back(target_tile_size / partial_tile_size); + } + } + + // Reshape the partial replicate tile_dimensions. + auto reshape_dimensions = partial_sharding.tile_assignment().dimensions(); + int64 num_replication = reshape_dimensions.back(); + if (num_replication != Product(expand_tile_sizes)) { + return absl::nullopt; + } + reshape_dimensions.pop_back(); + reshape_dimensions.insert(reshape_dimensions.end(), expand_tile_sizes.begin(), + expand_tile_sizes.end()); + auto reshape_tile_assignment = partial_sharding.tile_assignment(); + + // Transpose. + std::vector perm; + perm.reserve(rank); + for (int64 dim = 0; dim < rank; dim++) { + perm.emplace_back(dim); + if (expand_tile_dims_indices[dim] > -1) { + perm.emplace_back(expand_tile_dims_indices[dim] + rank); + } + } + auto transpose_sharding = hlo_sharding_util::TransposeSharding( + HloSharding::Tile(reshape_tile_assignment), perm); + + // Reshape to target shape + auto transpose_tile_assignment = transpose_sharding.tile_assignment(); + transpose_tile_assignment.Reshape(target_tile_dims); + + return HloSharding::Tile(transpose_tile_assignment); +} + +absl::optional PadFromPartialReplicateShape( + HloInstruction* hlo, const Shape& base_shape, + const HloSharding& src_sharding, const HloSharding& dst_sharding, + const std::vector& expand_tile_dims, + const SPMDCollectiveOpsCreator& collective_ops_creator, + int64* next_channel_id, HloInstruction* partition_id, SpmdBuilder* b) { + auto padded_src_shape = + GetPaddedShapeForUnevenPartitioning(base_shape, src_sharding); + auto padded_dst_shape = + GetPaddedShapeForUnevenPartitioning(base_shape, dst_sharding); + if (ShapeUtil::Compatible(padded_dst_shape, hlo->shape())) { + return hlo; + } + + auto partition_ordinals = + MakeTiledPartitionOrdinals(src_sharding, partition_id, b); + + HloInstruction* result = hlo; + auto zero = b->AddInstruction(HloInstruction::CreateConstant( + LiteralUtil::Zero(hlo->shape().element_type()))); + std::vector expand_dims_without_halo_exchange; + // Pad the dimensions needs halo exchange and record the padded dims that + // won't need halo exchange. + for (auto dim : expand_tile_dims) { + int64 src_shard_count = src_sharding.tile_assignment().dim(dim); + int64 src_per_shard_size = + padded_src_shape.dimensions(dim) / src_shard_count; + // Calculate per shard size using the sharding to compare if dst_sharding + // needs more padding at the end. + int64 dst_per_shard_size = + padded_dst_shape.dimensions(dim) / src_shard_count; + + // If dst_sharding doesn't need more padding at the end. + if (src_per_shard_size >= dst_per_shard_size) { + continue; + } + // If src sharding at this dimension is not partitoned, simply pad to + // the desired shape. + if (src_shard_count == 1) { + expand_dims_without_halo_exchange.emplace_back(dim); + continue; + } + + // If dst_padding needs more padding at the end, need to re-distribute the + // data between each shard using collective permute. + // For example, if dimension size is 6 and shard 2 ways in the src but + // needs to shard 4 ways in the dst. 4 ways needs padding 2 0s at the end + // and has 2 elements at each shard, while 2 way sharding has 3 elements + // in each shard, re-distribution is needed. + // + // 1. Calculate left_halo size. + // left-halo size is 0 + OffsetCalculation left_halo_size_function = + OffsetCalculation(MultiplyAddDivideOffsetCalculation(0, 0, 1)); + + // 2. Calculate right_halo size. + // right-halo size is D * (i + 1) - S * (i + 1) = (D - S) * i + (D - S) + OffsetCalculation right_halo_size_function = + OffsetCalculation(MultiplyAddDivideOffsetCalculation( + dst_per_shard_size - src_per_shard_size, + dst_per_shard_size - src_per_shard_size, 1)); + + auto concat = result; + // 3. Halo exchange. + auto halo_exchange_result = ExchangeHalo( + result, left_halo_size_function, right_halo_size_function, dim, + src_sharding, collective_ops_creator, next_channel_id, b); + + if (halo_exchange_result.has_value()) { + concat = halo_exchange_result.value(); + } else { + return absl::nullopt; + } + + // 4. Pad. + std::vector zero_padding(concat->shape().rank()); + PaddingConfig pad_config = window_util::MakeSymmetricPadding(zero_padding); + pad_config.mutable_dimensions(dim)->set_edge_padding_low(0); + int64 max_right_halo_size = + right_halo_size_function.MaxInRange(0, src_shard_count - 1); + pad_config.mutable_dimensions(dim)->set_edge_padding_high(std::max( + 0LL, padded_dst_shape.dimensions(dim) - + padded_src_shape.dimensions(dim) - max_right_halo_size)); + auto padded_concat_shape = ShapeInference::InferPadShape( + concat->shape(), zero->shape(), pad_config) + .ValueOrDie(); + concat = b->AddInstruction(HloInstruction::CreatePad( + padded_concat_shape, concat, zero, pad_config)); + + // 5. Slice the valid result. + // Slice offset is (D-S) * i + auto zero_s32 = b->AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::Zero(S32))); + OffsetCalculation start_offset_on_padded_concat_calculation = + OffsetCalculation(MultiplyAddDivideOffsetCalculation( + dst_per_shard_size - src_per_shard_size, 0, 1)); + auto slice_shape = concat->shape(); + slice_shape.set_dimensions(dim, dst_per_shard_size); + std::vector slice_offsets(concat->shape().rank(), + zero_s32); + slice_offsets[dim] = start_offset_on_padded_concat_calculation.Calculate( + partition_ordinals[dim], b); + result = b->AddInstruction(HloInstruction::CreateDynamicSlice( + slice_shape, concat, slice_offsets, slice_shape.dimensions())); + } + + // Pad other dimensions that won't need halo exchange with a single pad. + if (!expand_dims_without_halo_exchange.empty()) { + std::vector zero_padding(result->shape().rank()); + PaddingConfig pad_config = window_util::MakeSymmetricPadding(zero_padding); + + auto padded_shape = result->shape(); + for (auto dim : expand_dims_without_halo_exchange) { + pad_config.mutable_dimensions(dim)->set_edge_padding_low(0); + pad_config.mutable_dimensions(dim)->set_edge_padding_high( + padded_dst_shape.dimensions(dim) - padded_src_shape.dimensions(dim)); + padded_shape.set_dimensions(dim, result->shape().dimensions(dim) + + padded_dst_shape.dimensions(dim) - + padded_src_shape.dimensions(dim)); + } + result = b->AddInstruction( + HloInstruction::CreatePad(padded_shape, result, zero, pad_config)); + } + + return result; +} + absl::optional UniqueTiledDim(const HloSharding& sharding) { if (sharding.IsTileMaximal()) { return absl::nullopt; diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h index e8b51567359..6906b52ca79 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h @@ -347,6 +347,30 @@ absl::optional TransposeShardingWithCollapsedDims( absl::optional ParseReductionComputation( const HloComputation* reduction_comp); +// Pad the shape from partial replicate shape for `dst_sharding`. +// If dst_sharding needs more padding and per_shard_size increased in +// dst_sharding, halo exchange on the right side is needed. +absl::optional PadFromPartialReplicateShape( + HloInstruction* hlo, const Shape& base_shape, + const HloSharding& src_sharding, const HloSharding& dst_sharding, + const std::vector& expand_tile_dims, + const SPMDCollectiveOpsCreator& collective_ops_creator, + int64* next_channel_id, HloInstruction* partition_id, SpmdBuilder* b); + +// Get the compatible sharding from a partial replicate sharding to a given +// target tile dimensions. +// Compatible means replicate sharding can transform to the target tile +// dimensions by dynamic slice. +// For example, if partial_sharding is +// {devices=[1,2,2]0,1,2,3 last_tile_dim_replicate} +// Target tile dims is {2, 2}, the returned compatible sharding will be +// sharding={devices=[1,2,2]0,2,1,3 last_tile_dim_replicate}. +// If patial replicate sharding is not partial replicate or can't reshard to +// target_tile_dims by dynamic slice, return absl::nullopt. +absl::optional PartialReplicateToTileCompatibleSharding( + const HloSharding& partial_sharding, + const std::vector& target_tile_dims); + } // namespace spmd } // namespace xla From 7ffd29e060f005a92f089dca9cec1eccf76a79af Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Mon, 10 Aug 2020 13:34:30 -0700 Subject: [PATCH 0765/1017] [tf.data service] Write new journal files instead of appending. Some filesystems don't support appending to an existing file. Better to begin a new journal file each time the dispatcher is restarted. PiperOrigin-RevId: 325876992 Change-Id: I0e1046a67e85241d7542b368c8b5b7a5b8955556 --- tensorflow/core/data/service/BUILD | 1 + .../core/data/service/dispatcher_impl.cc | 20 +++-- tensorflow/core/data/service/journal.cc | 80 ++++++++++++++----- tensorflow/core/data/service/journal.h | 32 ++++++-- tensorflow/core/data/service/journal_test.cc | 6 +- tensorflow/core/data/service/server_lib.cc | 17 ++-- tensorflow/core/data/service/server_lib.h | 8 +- .../kernel_tests/data_service_ops_test.py | 43 ++++++++-- 8 files changed, 159 insertions(+), 48 deletions(-) diff --git a/tensorflow/core/data/service/BUILD b/tensorflow/core/data/service/BUILD index aed402fb3b9..13034eb4354 100644 --- a/tensorflow/core/data/service/BUILD +++ b/tensorflow/core/data/service/BUILD @@ -173,6 +173,7 @@ cc_library( deps = [ ":journal_proto_cc", "//tensorflow/core:lib", + "//tensorflow/core/platform:regexp", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ], diff --git a/tensorflow/core/data/service/dispatcher_impl.cc b/tensorflow/core/data/service/dispatcher_impl.cc index b440e9be905..5dbcece7b49 100644 --- a/tensorflow/core/data/service/dispatcher_impl.cc +++ b/tensorflow/core/data/service/dispatcher_impl.cc @@ -87,21 +87,25 @@ Status DataServiceDispatcherImpl::Start() { } journal_writer_ = absl::make_unique( Env::Default(), JournalDir(config_.work_dir())); + LOG(INFO) << "Restoring dispatcher state from journal in " + << JournalDir(config_.work_dir()); Update update; bool end_of_journal = false; FileJournalReader reader(Env::Default(), JournalDir(config_.work_dir())); Status s = reader.Read(&update, &end_of_journal); if (errors::IsNotFound(s)) { LOG(INFO) << "No journal found. Starting dispatcher from new state."; - return Status::OK(); - } - TF_RETURN_IF_ERROR(s); - LOG(INFO) << "Restoring dispatcher state from journal in " - << JournalDir(config_.work_dir()); - while (!end_of_journal) { - TF_RETURN_IF_ERROR(ApplyWithoutJournaling(update)); - TF_RETURN_IF_ERROR(reader.Read(&update, &end_of_journal)); + } else if (!s.ok()) { + return s; + } else { + while (!end_of_journal) { + TF_RETURN_IF_ERROR(ApplyWithoutJournaling(update)); + TF_RETURN_IF_ERROR(reader.Read(&update, &end_of_journal)); + } } + // Initialize the journal writer in `Start` so that we fail fast in case it + // can't be initialized. + TF_RETURN_IF_ERROR(journal_writer_.value()->EnsureInitialized()); return Status::OK(); } diff --git a/tensorflow/core/data/service/journal.cc b/tensorflow/core/data/service/journal.cc index 11952b0dfd9..b0ce0876c69 100644 --- a/tensorflow/core/data/service/journal.cc +++ b/tensorflow/core/data/service/journal.cc @@ -22,29 +22,51 @@ limitations under the License. #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/errors.h" #include "tensorflow/core/platform/path.h" +#include "tensorflow/core/platform/regexp.h" namespace tensorflow { namespace data { namespace { constexpr StringPiece kJournal = "journal"; + +Status ParseSequenceNumber(const std::string& journal_file, + int64* sequence_number) { + if (!RE2::FullMatch(journal_file, ".*_(\\d+)", sequence_number)) { + return errors::InvalidArgument("Failed to parse journal file name: ", + journal_file); + } + return Status::OK(); +} } // namespace -std::string DataServiceJournalFile(StringPiece journal_dir) { - return io::JoinPath(journal_dir, kJournal); +std::string DataServiceJournalFile(const std::string& journal_dir, + int64 sequence_number) { + return io::JoinPath(journal_dir, + absl::StrCat(kJournal, "_", sequence_number)); } -FileJournalWriter::FileJournalWriter(Env* env, StringPiece journal_dir) +FileJournalWriter::FileJournalWriter(Env* env, const std::string& journal_dir) : env_(env), journal_dir_(journal_dir) {} Status FileJournalWriter::EnsureInitialized() { if (writer_) { return Status::OK(); } + std::vector journal_files; TF_RETURN_IF_ERROR(env_->RecursivelyCreateDir(journal_dir_)); - TF_RETURN_IF_ERROR( - env_->NewAppendableFile(DataServiceJournalFile(journal_dir_), &file_)); + TF_RETURN_IF_ERROR(env_->GetChildren(journal_dir_, &journal_files)); + int64 latest_sequence_number = -1; + for (const auto& file : journal_files) { + int64 sequence_number; + TF_RETURN_IF_ERROR(ParseSequenceNumber(file, &sequence_number)); + latest_sequence_number = std::max(latest_sequence_number, sequence_number); + } + std::string journal_file = + DataServiceJournalFile(journal_dir_, latest_sequence_number + 1); + TF_RETURN_IF_ERROR(env_->NewAppendableFile(journal_file, &file_)); writer_ = absl::make_unique(file_.get()); + VLOG(1) << "Created journal writer to write to " << journal_file; return Status::OK(); } @@ -58,6 +80,9 @@ Status FileJournalWriter::Write(const Update& update) { TF_RETURN_IF_ERROR(writer_->WriteRecord(s)); TF_RETURN_IF_ERROR(writer_->Flush()); TF_RETURN_IF_ERROR(file_->Sync()); + if (VLOG_IS_ON(4)) { + VLOG(4) << "Wrote journal entry: " << update.DebugString(); + } return Status::OK(); } @@ -68,25 +93,44 @@ Status FileJournalReader::EnsureInitialized() { if (reader_) { return Status::OK(); } - TF_RETURN_IF_ERROR( - env_->NewRandomAccessFile(DataServiceJournalFile(journal_dir_), &file_)); - reader_ = absl::make_unique(file_.get()); - return Status::OK(); + return UpdateFile(DataServiceJournalFile(journal_dir_, 0)); } Status FileJournalReader::Read(Update* update, bool* end_of_journal) { TF_RETURN_IF_ERROR(EnsureInitialized()); - tstring record; - Status s = reader_->ReadRecord(&offset_, &record); - if (errors::IsOutOfRange(s)) { - *end_of_journal = true; + while (true) { + tstring record; + Status s = reader_->ReadRecord(&offset_, &record); + if (errors::IsOutOfRange(s)) { + sequence_number_++; + std::string next_journal_file = + DataServiceJournalFile(journal_dir_, sequence_number_); + if (errors::IsNotFound(env_->FileExists(next_journal_file))) { + VLOG(3) << "Next journal file " << next_journal_file + << " does not exist. End of journal reached."; + *end_of_journal = true; + return Status::OK(); + } + TF_RETURN_IF_ERROR(UpdateFile(next_journal_file)); + continue; + } + TF_RETURN_IF_ERROR(s); + if (!update->ParseFromString(record)) { + return errors::DataLoss("Failed to parse journal record."); + } + if (VLOG_IS_ON(4)) { + VLOG(4) << "Read journal entry: " << update->DebugString(); + } + *end_of_journal = false; return Status::OK(); } - TF_RETURN_IF_ERROR(s); - if (!update->ParseFromString(record)) { - return errors::DataLoss("Failed to parse journal record."); - } - *end_of_journal = false; +} + +Status FileJournalReader::UpdateFile(const std::string& filename) { + VLOG(1) << "Reading from journal file " << filename; + TF_RETURN_IF_ERROR(env_->NewRandomAccessFile(filename, &file_)); + reader_ = absl::make_unique(file_.get()); + offset_ = 0; return Status::OK(); } diff --git a/tensorflow/core/data/service/journal.h b/tensorflow/core/data/service/journal.h index c627c21756c..3483497705e 100644 --- a/tensorflow/core/data/service/journal.h +++ b/tensorflow/core/data/service/journal.h @@ -25,7 +25,8 @@ namespace tensorflow { namespace data { // Returns the location of the journal file within the journal directory. -std::string DataServiceJournalFile(StringPiece journal_dir); +std::string DataServiceJournalFile(const std::string& journal_dir, + int64 sequence_number); // Interface for writing to a journal. class JournalWriter { @@ -33,25 +34,39 @@ class JournalWriter { virtual ~JournalWriter() = default; // Writes and syncs an update to the journal. virtual Status Write(const Update& update) = 0; + // Initializes the writer if it is not yet initialized. + virtual Status EnsureInitialized() = 0; }; // FileJournalWriter is not thread-safe, requiring external synchronization when // used by multiple threads. +// +// FileJournalWriter writes journal files to a configured journal directory. The +// directory is laid out in the following format: +// +// journal_dir/ +// journal_0 +// journal_1 +// ... +// +// When the writer is created, it lists the directory to find the next available +// journal file name. For example, if the journal directory contains +// "journal_0", "journal_1", and "journal_2", the writer will write to +// "journal_3". The writer will flush updates as they are written, so that they +// can be stored durably in case of machine failure. class FileJournalWriter : public JournalWriter { public: // Creates a journal writer to write to the given journal directory. // If there is already journal data there, the journal writer will append to // the existing journal. - explicit FileJournalWriter(Env* env, StringPiece journal_dir); + explicit FileJournalWriter(Env* env, const std::string& journal_dir); FileJournalWriter(const FileJournalWriter&) = delete; FileJournalWriter& operator=(const FileJournalWriter&) = delete; Status Write(const Update& update) override; + Status EnsureInitialized() override; private: - // Initializes the writer if it is not yet initialized. - Status EnsureInitialized(); - Env* env_; const std::string journal_dir_; std::unique_ptr file_; @@ -69,6 +84,9 @@ class JournalReader { // JournalReader is not thread-safe, requiring external synchronization when // used by multiple threads. +// +// The journal reader reads through all journal files in the configured journal +// directory, in order of their sequence numbers. See FileJournalWriter above. class FileJournalReader : public JournalReader { public: explicit FileJournalReader(Env* env, StringPiece journal_dir); @@ -80,9 +98,13 @@ class FileJournalReader : public JournalReader { private: // Initializes the reader if it is not yet initialized. Status EnsureInitialized(); + // Updates the `FileJournalReader` to read from a new file. + Status UpdateFile(const std::string& filename); Env* env_; const std::string journal_dir_; + // Sequence number of current journal file. + int64 sequence_number_ = 0; // Current offset into `file_`. uint64 offset_ = 0; std::unique_ptr file_; diff --git a/tensorflow/core/data/service/journal_test.cc b/tensorflow/core/data/service/journal_test.cc index 169e58ed048..313b216fe76 100644 --- a/tensorflow/core/data/service/journal_test.cc +++ b/tensorflow/core/data/service/journal_test.cc @@ -95,7 +95,7 @@ TEST(Journal, RoundTripMultiple) { TF_EXPECT_OK(CheckJournalContent(journal_dir, updates)); } -TEST(Journal, AppendExistingFile) { +TEST(Journal, AppendExistingJournal) { std::string journal_dir; EXPECT_TRUE(NewJournalDir(&journal_dir)); std::vector updates = {MakeCreateJobUpdate(), @@ -127,7 +127,7 @@ TEST(Journal, NonRecordData) { { std::unique_ptr file; TF_ASSERT_OK(Env::Default()->NewAppendableFile( - DataServiceJournalFile(journal_dir), &file)); + DataServiceJournalFile(journal_dir, /*sequence_number=*/0), &file)); TF_ASSERT_OK(file->Append("not record data")); } @@ -147,7 +147,7 @@ TEST(Journal, InvalidRecordData) { { std::unique_ptr file; TF_ASSERT_OK(Env::Default()->NewAppendableFile( - DataServiceJournalFile(journal_dir), &file)); + DataServiceJournalFile(journal_dir, /*sequence_number=*/0), &file)); auto writer = absl::make_unique(file.get()); TF_ASSERT_OK(writer->WriteRecord("not serializd proto")); } diff --git a/tensorflow/core/data/service/server_lib.cc b/tensorflow/core/data/service/server_lib.cc index 98157f6b232..fb33319db29 100644 --- a/tensorflow/core/data/service/server_lib.cc +++ b/tensorflow/core/data/service/server_lib.cc @@ -28,8 +28,12 @@ namespace { constexpr char kPortPlaceholder[] = "%port%"; } -GrpcDataServerBase::GrpcDataServerBase(int port, const std::string& protocol) - : requested_port_(port), protocol_(protocol), bound_port_(port) {} +GrpcDataServerBase::GrpcDataServerBase(int port, const std::string& protocol, + const std::string server_type) + : requested_port_(port), + protocol_(protocol), + server_type_(server_type), + bound_port_(port) {} Status GrpcDataServerBase::Start() { if (stopped_) { @@ -56,7 +60,8 @@ Status GrpcDataServerBase::Start() { TF_RETURN_IF_ERROR(StartServiceInternal()); started_ = true; - VLOG(1) << "Started tf.data service running at 0.0.0.0:" << BoundPort(); + LOG(INFO) << "Started tf.data " << server_type_ + << " running at 0.0.0.0:" << BoundPort(); return Status::OK(); } @@ -74,7 +79,8 @@ int GrpcDataServerBase::BoundPort() { return bound_port(); } DispatchGrpcDataServer::DispatchGrpcDataServer( const experimental::DispatcherConfig& config) - : GrpcDataServerBase(config.port(), config.protocol()), config_(config) {} + : GrpcDataServerBase(config.port(), config.protocol(), "DispatchServer"), + config_(config) {} DispatchGrpcDataServer::~DispatchGrpcDataServer() { delete service_; } @@ -100,7 +106,8 @@ Status DispatchGrpcDataServer::NumWorkers(int* num_workers) { WorkerGrpcDataServer::WorkerGrpcDataServer( const experimental::WorkerConfig& config) - : GrpcDataServerBase(config.port(), config.protocol()), config_(config) {} + : GrpcDataServerBase(config.port(), config.protocol(), "WorkerServer"), + config_(config) {} WorkerGrpcDataServer::~WorkerGrpcDataServer() { delete service_; } diff --git a/tensorflow/core/data/service/server_lib.h b/tensorflow/core/data/service/server_lib.h index 2c300947f63..62662e61c8a 100644 --- a/tensorflow/core/data/service/server_lib.h +++ b/tensorflow/core/data/service/server_lib.h @@ -34,10 +34,9 @@ class GrpcDataServerBase { public: // Constructs a tf.data server with the specified port. If the port is 0, the // server will find an available port in `Start()`. The chosen port can be - // found in the output of `Target()`. - // - // dispatcher_address is only needed for worker data servers. - GrpcDataServerBase(int requested_port, const std::string& protocol); + // found by calling `BoundPort()`. + GrpcDataServerBase(int requested_port, const std::string& protocol, + const std::string server_type); virtual ~GrpcDataServerBase() {} // Starts the server running asynchronously. @@ -62,6 +61,7 @@ class GrpcDataServerBase { const int requested_port_; const std::string protocol_; + const std::string server_type_; private: int bound_port_; diff --git a/tensorflow/python/data/kernel_tests/data_service_ops_test.py b/tensorflow/python/data/kernel_tests/data_service_ops_test.py index 49cf1772661..6ef9293ddd7 100644 --- a/tensorflow/python/data/kernel_tests/data_service_ops_test.py +++ b/tensorflow/python/data/kernel_tests/data_service_ops_test.py @@ -107,6 +107,14 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): dispatcher._stop() return self.start_dispatch_server(port=port) + def restart_worker(self, worker, dispatcher, use_same_port=True): + """Stops `worker` and returns a new worker.""" + port = 0 + if use_same_port: + port = int(worker._address.split(":")[1]) + worker._stop() + return self.start_worker_server(dispatcher, port) + def start_cluster(self, num_workers): """Creates a cluster of tf.data service servers. @@ -176,6 +184,35 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): dispatcher = self.restart_dispatcher(dispatcher) self.assertDatasetProduces(ds, list(range(num_elements))) + @combinations.generate(test_base.eager_only_combinations()) + def testDispatcherManyRestarts(self): + dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + num_elements_start = 10 + num_elements_end = 15 + datasets = [] + for num_elements in range(num_elements_start, num_elements_end): + datasets.append(_make_distributed_range_dataset(num_elements, dispatcher)) + dispatcher = self.restart_dispatcher(dispatcher) + for ds, num_elements in zip(datasets, + range(num_elements_start, num_elements_end)): + self.assertDatasetProduces(ds, list(range(num_elements))) + + @combinations.generate(test_base.eager_only_combinations()) + def testDispatcherAndWorkerRestart(self): + dispatcher, [worker] = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable + num_elements = 100 + ds = dataset_ops.Dataset.range(num_elements) + + def restart(): + return (self.restart_dispatcher(dispatcher), + self.restart_worker(worker, dispatcher)) + + ds = _make_distributed_dataset(ds, dispatcher) + dispatcher, worker = restart() + self.assertDatasetProduces(ds, list(range(num_elements))) + dispatcher, worker = restart() + self.assertDatasetProduces(ds, list(range(num_elements))) + @combinations.generate(test_base.eager_only_combinations()) def testDistributeSparse(self): dispatcher, workers = self.start_cluster(1) # to avoid gcing workers, pylint: disable=unused-variable @@ -357,11 +394,7 @@ class DataServiceOpsTest(test_base.DatasetTestBase, parameterized.TestCase): self.assertEqual(i, next(iterator).numpy()) # Stop the original worker and start a new one. - port = 0 - if use_same_port: - port = int(worker._address.split(":")[1]) - worker._stop() - new_worker = self.start_worker_server(dispatcher, port=port) # to avoid gcing workers, pylint: disable=unused-variable + worker = self.restart_worker(worker, dispatcher, use_same_port) # There may have been some elements prefetched from the first worker # before it was stopped. From 72aa724a2860b34945698dc84e199bce3f05084a Mon Sep 17 00:00:00 2001 From: Justin Lebar Date: Mon, 10 Aug 2020 13:46:08 -0700 Subject: [PATCH 0766/1017] Add VLOGs to XLA:GPU sort emitter. These give us a sense of what it's doing, which is particularly important because all of this is hidden within --xla_hlo_profile (as it's just one HLO). PiperOrigin-RevId: 325879218 Change-Id: I7b177ac6c636e35f0a178502098d9c1317a68448 --- tensorflow/compiler/xla/service/gpu/BUILD | 1 + .../xla/service/gpu/ir_emitter_unnested.cc | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index b9ba2100293..074fbd92b27 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -286,6 +286,7 @@ cc_library( "@com_google_absl//absl/container:inlined_vector", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:optional", "@com_google_absl//absl/types:span", "@llvm-project//llvm:Core", diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 34cdfb4ecf0..61b78b6004d 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -27,6 +27,7 @@ limitations under the License. #include "absl/container/inlined_vector.h" #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" #include "absl/types/optional.h" #include "absl/types/span.h" #include "llvm/ADT/StringRef.h" @@ -1284,6 +1285,7 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { if (destination_buffer != source_address) { // TODO(b/26783907): Figure out why we never seem to share buffers for // key/value sort. + VLOG(2) << sort->name() << " requires initial D2D copy for operand " << i; thunks.push_back(absl::make_unique( Thunk::ThunkInfo(), /*source_address=*/source_address, @@ -1294,6 +1296,7 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { uint64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort); int64 num_stages = tensorflow::Log2Ceiling(dimension_to_sort_bound); + VLOG(2) << sort->name() << " requires " << num_stages << " stages."; CHECK_GE(1ULL << num_stages, dimension_to_sort_bound); CHECK_LT(1ULL << (num_stages - 1), dimension_to_sort_bound); @@ -1368,11 +1371,27 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { ir_emitter_context_->gpu_device_info().threads_per_block_limit || total_shared_memory_needed > ir_emitter_context_->gpu_device_info().shared_memory_per_block; + VLOG(2) << absl::StreamFormat( + "%s %s use tiling. No tiling if any of the following is true: " + "kTileSize=%d < 128, " + "kThreadsPerBlock=%d > threads_per_block_limit=%d, " + "total_shared_memory_needed=%d > shared_memory_per_block=%d", + sort->name(), (no_tiling ? "won't" : "will"), kTileSize, kThreadsPerBlock, + ir_emitter_context_->gpu_device_info().threads_per_block_limit, + total_shared_memory_needed, + ir_emitter_context_->gpu_device_info().shared_memory_per_block); uint64 num_blocks = CeilOfRatio(num_iterations, kThreadsPerBlock); LaunchDimensions tiled_launch_dimensions(num_blocks, kThreadsPerBlock); + VLOG(2) << absl::StreamFormat("%s launch dims: %d blocks, %d threads/block", + sort->name(), num_blocks, kThreadsPerBlock); auto emit_kernel = [&](absl::Span xor_masks) { + VLOG(2) << absl::StreamFormat( + "%s uses kernel for xor masks [%s]", sort->name(), + absl::StrJoin(xor_masks, ", ", [](std::string* out, int64 xor_mask) { + absl::StrAppendFormat(out, "0x%x", xor_mask); + })); thunks.push_back( BuildKernelThunk(sort, /*implements_whole_instruction=*/false)); LaunchDimensions launch_dimensions = xor_masks.size() > 1 @@ -1421,6 +1440,9 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { if (!xor_masks.empty()) { TF_RETURN_IF_ERROR(emit_kernel(xor_masks)); } + VLOG(2) << absl::StreamFormat( + "%s requires %d thunks (including any D2D copies)", sort->name(), + thunks.size()); AddThunkToThunkSequence(absl::make_unique( GetThunkInfo(sort), std::move(thunks))); From 83b86b6333a2e1a8cdb7b399378b91927caded00 Mon Sep 17 00:00:00 2001 From: Berkin Ilbeyi Date: Mon, 10 Aug 2020 13:54:37 -0700 Subject: [PATCH 0767/1017] [XLA] NFC: Refactor allocation success/failure logic in memory space assignment. This CL is in preparation for repacking the allocations in case we run out of alternate memory due to fragmentation. This CL introduces a Result enum to capture success of an allocation or failure due to multiple different reasons. Moved the logic around to perform the decision to Finalize or Uncommit (and possibly retry) the allocations based on the Result. We can further use this Result to repack the assigned offsets to help with fragmentation. I also modified the function signatures that I touched to use non-const references instead of non-const pointers due to go/totw/178. PiperOrigin-RevId: 325881001 Change-Id: Icdbf1892d75208ec3eba8d2e084097106fb07dde --- .../xla/service/memory_space_assignment.cc | 208 ++++++++++-------- .../xla/service/memory_space_assignment.h | 130 ++++++++--- 2 files changed, 219 insertions(+), 119 deletions(-) diff --git a/tensorflow/compiler/xla/service/memory_space_assignment.cc b/tensorflow/compiler/xla/service/memory_space_assignment.cc index 377c84eaf6b..4131a0199bf 100644 --- a/tensorflow/compiler/xla/service/memory_space_assignment.cc +++ b/tensorflow/compiler/xla/service/memory_space_assignment.cc @@ -610,7 +610,9 @@ std::string MemorySpaceAssignment::AllocationValue::ToShortString() const { } void AlternateMemoryBestFitHeap::CreateAllocationValues( - const HloValue* value, std::vector* allocation_values) { + const AlternateMemoryBestFitHeap::BufferInterval& buffer_interval, + std::vector& allocation_values) const { + const HloValue* value = buffer_interval.buffer; VLOG(3) << "Creating AllocationValues for: " << value->ToString(); // Find and sort all non-trivial (excluding GTE, Tuple, and bitcast) @@ -638,10 +640,10 @@ void AlternateMemoryBestFitHeap::CreateAllocationValues( // Create an AllocationValue for each non-trivial position. absl::flat_hash_set computations; - int beginning_idx = allocation_values->size(); + int beginning_idx = allocation_values.size(); for (int i = 0; i < positions.size(); ++i) { const HloPosition& position = positions.at(i); - allocation_values->emplace_back(value, position); + allocation_values.emplace_back(value, position, buffer_interval.size); } std::vector uses(value->uses()); @@ -662,8 +664,8 @@ void AlternateMemoryBestFitHeap::CreateAllocationValues( HloComputation* use_computation = use.instruction->parent(); AllocationValue* last_allocation_value = nullptr; - for (int i = beginning_idx; i < allocation_values->size(); ++i) { - AllocationValue* allocation_value = &allocation_values->at(i); + for (int i = beginning_idx; i < allocation_values.size(); ++i) { + AllocationValue* allocation_value = &allocation_values.at(i); if (allocation_value->computation() == use_computation && instruction_schedule.at( allocation_value->defining_position().instruction) < use_time) { @@ -674,9 +676,9 @@ void AlternateMemoryBestFitHeap::CreateAllocationValues( last_allocation_value->AddUse(use, use_time); } - for (int i = beginning_idx; i < allocation_values->size(); ++i) { + for (int i = beginning_idx; i < allocation_values.size(); ++i) { VLOG(3) << "Created allocation value: " - << allocation_values->at(i).ToString(); + << allocation_values.at(i).ToString(); } } @@ -920,27 +922,27 @@ void AlternateMemoryBestFitHeap::AppendBufferInfoDebugString( } void AlternateMemoryBestFitHeap::AppendAllocationInfoDebugString( - const GlobalDecreasingSizeBestFitHeap::BufferInterval& interval, + const AllocationValue& value, const MemorySpaceAssignment::Allocation& allocation, - std::string* debug_str) const { + std::string& debug_str) const { // Columns in allocation information: // buffer_id: int. This value can be used the match with buffer info. // size: int. In bytes. // offset: int. In bytes. // start_time: int. Logical start time of the allocation. // end_time: int. Logical end time of the allocation. - if (debug_str->empty()) { + if (debug_str.empty()) { // Append the column names. - absl::StrAppend(debug_str, "buffer_id,size,offset,start_time,end_time\n"); + absl::StrAppend(&debug_str, "buffer_id,size,offset,start_time,end_time\n"); } if (allocation.memory_space() == MemorySpace::kAlternate) { const HloBuffer& buffer = - alias_analysis_.GetBufferContainingValue(*interval.buffer); - absl::StrAppend(debug_str, buffer.id(), ","); - absl::StrAppend(debug_str, interval.size, ","); - absl::StrAppend(debug_str, allocation.chunk().offset, ","); - absl::StrAppend(debug_str, allocation.start_time(), ","); - absl::StrAppend(debug_str, allocation.end_time(), "\n"); + alias_analysis_.GetBufferContainingValue(*value.value()); + absl::StrAppend(&debug_str, buffer.id(), ","); + absl::StrAppend(&debug_str, value.size(), ","); + absl::StrAppend(&debug_str, allocation.chunk().offset, ","); + absl::StrAppend(&debug_str, allocation.start_time(), ","); + absl::StrAppend(&debug_str, allocation.end_time(), "\n"); } } @@ -1044,16 +1046,25 @@ HeapSimulator::Result AlternateMemoryBestFitHeap::Finish() { AppendBufferInfoDebugString(interval, &buffer_info_str_); + std::vector allocation_values; + CreateAllocationValuesFromColocatedIntervals(colocated_intervals, + allocation_values); + // Retry allocating this value with larger limits if allocation fails. for (int retry_number = 0; retry_number < options_.max_retries; retry_number++) { - final_retry_ = (retry_number == options_.max_retries - 1); + bool final_retry = (retry_number == options_.max_retries - 1); options_.prefetch_interval_picker->SetRetryNumber(retry_number); - bool success = AllocateColocatedIntervals(colocated_intervals); - if (success) { + Result result = + AllocateAllocationValues(absl::MakeSpan(allocation_values)); + if (result_requires_uncommit(result) || + (!final_retry && result_failed_because_of_async_copy(result))) { + UncommitPendingChunks(absl::MakeSpan(allocation_values)); + VLOG(2) << "Couldn't allocate. Retry number " << retry_number; + } else { + FinalizeAllocations(absl::MakeSpan(allocation_values)); break; } - VLOG(2) << "Couldn't allocate. Retry number " << retry_number; } } @@ -1066,9 +1077,10 @@ HeapSimulator::Result AlternateMemoryBestFitHeap::Finish() { return result_; } -bool AlternateMemoryBestFitHeap::AllocateColocatedIntervals( - const std::vector& - colocated_intervals) { +void AlternateMemoryBestFitHeap::CreateAllocationValuesFromColocatedIntervals( + absl::Span + colocated_intervals, + std::vector& allocation_values) { // TODO(berkin): For now, place the phi values due to conditionals in // default memory. for (const BufferInterval* colocated_interval : colocated_intervals) { @@ -1089,11 +1101,15 @@ bool AlternateMemoryBestFitHeap::AllocateColocatedIntervals( } // Create AllocationValues for all the colocated intervals. - std::vector allocation_values; for (const auto& colocated_interval : colocated_intervals) { - CreateAllocationValues(colocated_interval->buffer, &allocation_values); + CreateAllocationValues(*colocated_interval, allocation_values); } FindAliases(&allocation_values); +} + +AlternateMemoryBestFitHeap::Result +AlternateMemoryBestFitHeap::AllocateAllocationValues( + absl::Span allocation_values) { const auto& instruction_schedule = hlo_live_range_.instruction_schedule(); // Data structure to contain the preferred offset for a given computation. @@ -1102,8 +1118,8 @@ bool AlternateMemoryBestFitHeap::AllocateColocatedIntervals( absl::flat_hash_map preferred_offset_for_computation; - bool allocation_success = true; - for (auto& allocation_value : allocation_values) { + Result result = Result::kSuccess; + for (AllocationValue& allocation_value : allocation_values) { int64 definition_time = instruction_schedule.at(allocation_value.defining_instruction()); @@ -1217,20 +1233,19 @@ bool AlternateMemoryBestFitHeap::AllocateColocatedIntervals( request.start_time = std::min(definition_time, use_time); request.end_time = use_time; request.latest_prefetch_time = latest_prefetch_time; - request.size = colocated_intervals[0]->size; + request.size = allocation_value.size(); request.allow_no_copy_alternate_mem_allocation = allow_no_copy_alternate_mem_allocation; request.earliest_prefetch_time = earliest_prefetch_time; request.preferred_offset = preferred_offset; request.use = &use; request.allocation_value = &allocation_value; - if (!AllocateSegment(request)) { + result_mark(AllocateSegment(request), result); + if (result_requires_uncommit(result)) { // If the allocation finding failed (e.g., due to running out of // asynchronous copies), then fall back to allocating the buffer // entirely in the default memory. - UncommitPendingChunks(); - allocation_success = false; - break; + return result; } // If there are multiple uses, they can try using the memory allocation @@ -1256,24 +1271,8 @@ bool AlternateMemoryBestFitHeap::AllocateColocatedIntervals( aliased_allocation->chunk().offset; } } - if (!allocation_success) { - break; - } } - if (allocation_success) { - for (AllocationValue& allocation_value : allocation_values) { - for (auto& allocation : *allocation_value.allocation_sequence()) { - AppendAllocationInfoDebugString(*colocated_intervals[0], *allocation, - &allocation_info_str_); - allocations_->push_back(std::move(allocation)); - } - } - } - - pending_chunks_.clear(); - pending_async_copies_.clear(); - pending_required_assignments_.clear(); - return allocation_success; + return result; } bool operator<(const AsynchronousCopy& a, const AsynchronousCopy& b) { @@ -1365,9 +1364,7 @@ void AlternateMemoryBestFitHeap::AllocateCrossProgramPrefetchBuffer( allocations_->push_back(std::move(allocation)); } - pending_chunks_.clear(); - pending_async_copies_.clear(); - pending_required_assignments_.clear(); + ClearPendingChunks(); } absl::optional @@ -1544,7 +1541,13 @@ bool AlternateMemoryBestFitHeap::AreIntervalsReservedInAlternateMemory( return false; } -void AlternateMemoryBestFitHeap::UncommitPendingChunks() { +void AlternateMemoryBestFitHeap::UncommitPendingChunks( + absl::Span allocation_values) { + // Clear the allocation sequence of the allocation values so that in case we + // retry allocation after uncommitting. + for (AllocationValue& allocation_value : allocation_values) { + allocation_value.allocation_sequence()->clear(); + } for (const auto& interval_and_chunk : pending_chunks_) { const BufferInterval& interval = interval_and_chunk.first; const Chunk& chunk = interval_and_chunk.second.chunk; @@ -1583,6 +1586,22 @@ void AlternateMemoryBestFitHeap::UncommitPendingChunks() { } } } + ClearPendingChunks(); +} + +void AlternateMemoryBestFitHeap::FinalizeAllocations( + absl::Span allocation_values) { + for (AllocationValue& allocation_value : allocation_values) { + for (auto& allocation : *allocation_value.allocation_sequence()) { + AppendAllocationInfoDebugString(allocation_value, *allocation, + allocation_info_str_); + allocations_->push_back(std::move(allocation)); + } + } + ClearPendingChunks(); +} + +void AlternateMemoryBestFitHeap::ClearPendingChunks() { pending_chunks_.clear(); pending_async_copies_.clear(); pending_required_assignments_.clear(); @@ -1598,7 +1617,7 @@ void AlternateMemoryBestFitHeap::AddToPendingChunks( CommitChunk(buffer_interval, chunk_candidate); } -bool AlternateMemoryBestFitHeap::AllocateSegment( +AlternateMemoryBestFitHeap::Result AlternateMemoryBestFitHeap::AllocateSegment( const AllocationRequest& request) { auto allocation_sequence = request.allocation_value->allocation_sequence(); // start_time == end_time is a special case where the value is consumed @@ -1609,7 +1628,7 @@ bool AlternateMemoryBestFitHeap::AllocateSegment( GetLiveAllocationAt(*allocation_sequence, request.end_time); CHECK_NE(allocation, nullptr); allocation->AddUse(request.use->hlo_use); - return true; + return Result::kSuccess; } const HloPosition& defining_position = @@ -1673,12 +1692,15 @@ bool AlternateMemoryBestFitHeap::AllocateSegment( } } + Result allocation_result = Result::kSuccess; // First try keeping the allocation entirely in the alternate memory. if (required_memory_space_at_start != MemorySpace::kDefault && required_memory_space_at_end != MemorySpace::kDefault && - request.allow_no_copy_alternate_mem_allocation && - AllocateInAlternateMemoryNoCopy(request)) { - return true; + request.allow_no_copy_alternate_mem_allocation) { + allocation_result = AllocateInAlternateMemoryNoCopy(request); + if (allocation_result == Result::kSuccess) { + return Result::kSuccess; + } } auto prev_allocation_it = allocation_sequence->rbegin(); @@ -1697,8 +1719,10 @@ bool AlternateMemoryBestFitHeap::AllocateSegment( (*prev_allocation_it)->defining_position() == defining_position) { // If there was an allocation for this HloValue that was in the alternate // memory space, we also need to perform an eviction. - if (!Evict(request)) { - return false; + Result eviction_result = Evict(request); + if (eviction_result != Result::kSuccess) { + // A non-success eviction requires us to uncommit previous allocations. + return result_mark(Result::kFailRequiresUncommit, eviction_result); } prev_allocation_in_default_mem_it = allocation_sequence->rbegin(); } else if (prev_allocation_in_default_mem_it == allocation_sequence->rend()) { @@ -1719,31 +1743,28 @@ bool AlternateMemoryBestFitHeap::AllocateSegment( << "Not trying to prefetch because use requires buffer in default mem."; (*prev_allocation_in_default_mem_it)->Extend(request.end_time); (*prev_allocation_in_default_mem_it)->AddUse(request.use->hlo_use); - return true; + return Result::kSuccess; } // Finally, try to prefetch the buffer into alternate memory. - if (Prefetch(request, **prev_allocation_in_default_mem_it)) { - return true; - } - if (!final_retry_ && prefetch_failed_due_to_async_copy_) { - // If prefetching failed due to asynchronous copy and we're not in our final - // try, return false (failure) so that we can retry this interval with - // larger limits. - return false; + Result prefetch_result = + Prefetch(request, **prev_allocation_in_default_mem_it); + if (prefetch_result == Result::kSuccess) { + return Result::kSuccess; } + result_mark(prefetch_result, allocation_result); // If the end assignment was required to be in alternate memory but that // wasn't possible, then this allocation is invalid. if (required_memory_space_at_end == MemorySpace::kAlternate) { - return false; + return result_mark(Result::kFailRequiresUncommit, allocation_result); } // If a copy wasn't inserted, then add this use to the latest allocation in // default memory. (*prev_allocation_in_default_mem_it)->Extend(request.end_time); (*prev_allocation_in_default_mem_it)->AddUse(request.use->hlo_use); - return true; + return allocation_result; } void AlternateMemoryBestFitHeap::AddAsyncCopy( @@ -1810,7 +1831,8 @@ AlternateMemoryBestFitHeap::ViolatesAsyncCopyOrdering(int64 start_time, return async_copy_ordering_.ViolatesOrdering(start_time, end_time); } -bool AlternateMemoryBestFitHeap::AllocateInAlternateMemoryNoCopy( +AlternateMemoryBestFitHeap::Result +AlternateMemoryBestFitHeap::AllocateInAlternateMemoryNoCopy( const AllocationRequest& request) { MemorySpaceAssignment::Allocation* prev_allocation = nullptr; bool can_eliminate_copy = false; @@ -1829,7 +1851,7 @@ bool AlternateMemoryBestFitHeap::AllocateInAlternateMemoryNoCopy( } if (!can_eliminate_copy) { - return false; + return Result::kFailPrevAllocationNotInAlternateMem; } const HloPosition& defining_position = @@ -1837,7 +1859,7 @@ bool AlternateMemoryBestFitHeap::AllocateInAlternateMemoryNoCopy( if (!options_.prefetch_interval_picker->CanAllocateInAlternateMemoryNoCopy( defining_position.shape(), request.start_time + 1, request.end_time)) { - return false; + return Result::kFailLiveRangeTooLong; } BufferInterval alternate_mem_interval; @@ -1916,12 +1938,13 @@ bool AlternateMemoryBestFitHeap::AllocateInAlternateMemoryNoCopy( } request.allocation_value->allocation_sequence()->back()->AddUse( request.use->hlo_use); - return true; + return Result::kSuccess; } - return false; + return Result::kFailOutOfMemory; } -bool AlternateMemoryBestFitHeap::Evict(const AllocationRequest& request) { +AlternateMemoryBestFitHeap::Result AlternateMemoryBestFitHeap::Evict( + const AllocationRequest& request) { CHECK_GT(request.allocation_value->allocation_sequence()->size(), 0); MemorySpaceAssignment::Allocation* prev_allocation = request.allocation_value->allocation_sequence()->back().get(); @@ -2010,10 +2033,12 @@ bool AlternateMemoryBestFitHeap::Evict(const AllocationRequest& request) { << " and " << hlo_live_range_.flattened_instruction_sequence() .instructions()[eviction_end_time]; - return false; + // return false; + return Result::kFailOutOfAsyncCopies; } } - return true; + // return true; + return Result::kSuccess; } int64 AlternateMemoryBestFitHeap::FindPrefetchEndTime( @@ -2063,7 +2088,7 @@ int64 AlternateMemoryBestFitHeap::FindPrefetchEndTime( return prefetch_end_time; } -bool AlternateMemoryBestFitHeap::Prefetch( +AlternateMemoryBestFitHeap::Result AlternateMemoryBestFitHeap::Prefetch( const AllocationRequest& request, const MemorySpaceAssignment::Allocation& prev_allocation_in_default_mem) { // Try partially placing the buffer in the alternate space. The time that is @@ -2097,15 +2122,12 @@ bool AlternateMemoryBestFitHeap::Prefetch( BufferInterval alternate_mem_interval; alternate_mem_interval.buffer = request.allocation_value->value(); alternate_mem_interval.size = request.size; - // If any of the prefetch intervals couldn't be used due to number of - // outstanding async copy limit or async copy ordering, set - // prefetch_failed_due_to_async_copy_. - prefetch_failed_due_to_async_copy_ = false; // While uses might be allowed to have additional outstanding prefetches. int64 extra_async_copy_limit = request.use->hlo_use.instruction->opcode() == HloOpcode::kWhile ? options_.while_use_extra_outstanding_prefetch_limit : 0; + Result result = Result::kSuccess; while (!options_.prefetch_interval_picker->Done()) { alternate_mem_interval.start = options_.prefetch_interval_picker->Next(); CHECK_LT(alternate_mem_interval.start, prefetch_end_time); @@ -2116,14 +2138,14 @@ bool AlternateMemoryBestFitHeap::Prefetch( if (ViolatesAsyncCopyOrdering(alternate_mem_interval.start, prefetch_end_time)) { VLOG(4) << "This would violate asynchronous copy ordering."; - prefetch_failed_due_to_async_copy_ = true; + result_mark(Result::kFailViolatesAsyncCopyOrdering, result); continue; } if (ViolatesMaximumOutstandingAsyncCopies( alternate_mem_interval.start, prefetch_end_time, /*is_prefetch=*/true, extra_async_copy_limit)) { VLOG(4) << "This would violate the outstanding async copy limit."; - prefetch_failed_due_to_async_copy_ = true; + result_mark(Result::kFailOutOfAsyncCopies, result); continue; } @@ -2147,11 +2169,17 @@ bool AlternateMemoryBestFitHeap::Prefetch( request.allocation_value->allocation_sequence()->back()->AddUse( request.use->hlo_use); - prefetch_failed_due_to_async_copy_ = false; - return true; + return Result::kSuccess; } + result_mark(Result::kFailOutOfMemory, result); + } + // If we didn't consider any prefetch intervals, then the live range was too + // short. + if (result == Result::kSuccess) { + return Result::kFailLiveRangeTooShort; + } else { + return result; } - return false; } absl::optional diff --git a/tensorflow/compiler/xla/service/memory_space_assignment.h b/tensorflow/compiler/xla/service/memory_space_assignment.h index 87f7dd2ddae..d530a57d257 100644 --- a/tensorflow/compiler/xla/service/memory_space_assignment.h +++ b/tensorflow/compiler/xla/service/memory_space_assignment.h @@ -687,13 +687,15 @@ class MemorySpaceAssignment { std::vector aliases; }; - AllocationValue(const HloValue* value, const HloPosition& position) - : value_(value), defining_position_(position) {} + AllocationValue(const HloValue* value, const HloPosition& position, + int64 size) + : value_(value), defining_position_(position), size_(size) {} const HloPosition& defining_position() const { return defining_position_; } const HloInstruction* defining_instruction() const { return defining_position().instruction; } + int64 size() const { return size_; } const std::vector& uses() const { return uses_; } std::vector& uses() { return uses_; } const HloValue* value() const { return value_; } @@ -712,6 +714,7 @@ class MemorySpaceAssignment { private: const HloValue* value_; HloPosition defining_position_; + int64 size_; std::vector uses_; AllocationSequence allocation_sequence_; }; @@ -958,6 +961,62 @@ class AlternateMemoryBestFitHeap : public GlobalDecreasingSizeBestFitHeap { MemorySpaceAssignment::AllocationValue* allocation_value; }; + // Result of an allocation, prefetch, eviction etc. request. The result is + // either kSuccess or a bitwise OR of one or more failures. The values are + // unique powers of two. To check if a result contains a particular failure, + // use the result_is method. To add a new failure to a result, use the + // result_mark method. + enum class Result { + // Successful allocation. + kSuccess = 0, + // Allocation failed because we ran out of alternate memory. + kFailOutOfMemory = 1, + // A no-copy allocation couldn't be performed because the previous + // allocation wasn't in the alternate memory space. + kFailPrevAllocationNotInAlternateMem = 2, + // A no-copy allocation couldn't be performed because the live range was too + // long. + kFailLiveRangeTooLong = 4, + // A prefetching couldn't be performed because the live range was too short. + kFailLiveRangeTooShort = 8, + // Ran out of outstanding asynchronous copy limit either during prefetching + // or eviction. + kFailOutOfAsyncCopies = 16, + // A prefetching couldn't be performed because the asynchronous copy + // ordering was violated. + kFailViolatesAsyncCopyOrdering = 32, + // An allocation failure happened that requires uncommitting all the pending + // allocations. Usually this is due to a situation requiring an eviction but + // the eviction couldn't be performed. + kFailRequiresUncommit = 64 + }; + + // Return true if the result belongs to a failure. + static bool result_is(Result result, Result failure) { + return static_cast(result) & static_cast(failure); + } + + // Mark (bitwise OR) a failure to the result. + static Result result_mark(Result failure, Result& result) { + result = static_cast(static_cast(result) | + static_cast(failure)); + return result; + } + + // Return true if the result is a failure that requires us to uncommit pending + // chunks. + static bool result_requires_uncommit(Result result) { + return result_is(result, Result::kFailRequiresUncommit); + } + + // Return true if the result is a failure either due to running out of + // outstanding asynchronous copies or due to violating asynchronous copy + // ordering. + static bool result_failed_because_of_async_copy(Result result) { + return result_is(result, Result::kFailOutOfAsyncCopies) || + result_is(result, Result::kFailViolatesAsyncCopyOrdering); + } + // Given an allocation sequence, returns the live allocation at time with a // preference towards allocations in alternate memory. Returns nullptr if no // allocation is alive at that time. @@ -968,17 +1027,24 @@ class AlternateMemoryBestFitHeap : public GlobalDecreasingSizeBestFitHeap { bool IsUseAllowedInAlternateMemory(const AllocationValue& value, const HloUse& use) const; - // Given an HloValue, creates AllocationValue objects and corresponding + // Given a BufferInterval, creates AllocationValue objects and corresponding // AllocationSequences and appends them into allocation_sequence_list_. - void CreateAllocationValues(const HloValue* value, - std::vector* allocation_values); + void CreateAllocationValues( + const BufferInterval& buffer_interval, + std::vector& allocation_values) const; - // Finds allocations for colocated intervals. Colocated intervals consist of - // one or more BufferIntervals, each with a different HloValue. All of the - // intervals within colocated intervals have a must-alias relationship with - // each other. Returns true if allocation succeeded. - bool AllocateColocatedIntervals( - const std::vector& colocated_intervals); + // Given colocated intervals, populates allocation_values with the + // corresponding AllocationValue objects. + void CreateAllocationValuesFromColocatedIntervals( + absl::Span colocated_intervals, + std::vector& allocation_values); + + // Finds allocations for allocation values generated from colocated intervals. + // All of the allocation values have a must-alias relationship with each + // other. Returns either kSuccess if all of the sites could be placed in the + // alternate memory or a bitwise OR of failure reasons why they couldn't + Result AllocateAllocationValues( + absl::Span allocation_values); // Go through all the uses in the AllocationValues and find the aliasing // positions. @@ -996,24 +1062,26 @@ class AlternateMemoryBestFitHeap : public GlobalDecreasingSizeBestFitHeap { // if there is enough space and if the prefetch interval picker allows. // // If an eviction (2) was requested and was unsuccessful, this method returns - // false. This means we could not find a suitable allocation, so all previous - // allocations for this buffer must be removed and allocated in the default - // memory. Otherwise, this method returns true. - bool AllocateSegment(const AllocationRequest& request); + // Result::kFailRequiresUncommit. This means we could not find a suitable + // allocation, so all previous allocations for this buffer must be removed and + // allocated in the default memory. Otherwise, this method may return + // Result::kSuccess if the buffer could be placed in alternate memory or some + // other Result with an OR of reasons why the buffer couldn't be placed in + // alternate memory. + Result AllocateSegment(const AllocationRequest& request); - // Try allocating in alternate memory without any copies. Returns true if - // successful. - bool AllocateInAlternateMemoryNoCopy(const AllocationRequest& request); + // Try allocating in alternate memory without any copies. + Result AllocateInAlternateMemoryNoCopy(const AllocationRequest& request); - // Try evicting to default memory space. Returns true if successful. - bool Evict(const AllocationRequest& request); + // Try evicting to default memory space. + Result Evict(const AllocationRequest& request); // Returns the time a copy done of a prefetch should be scheduled. int64 FindPrefetchEndTime(const AllocationRequest& request, int64 earliest_prefetch_time) const; - // Try prefetching to alternate memory space. Returns true if successful. - bool Prefetch( + // Try prefetching to alternate memory space. + Result Prefetch( const AllocationRequest& request, const MemorySpaceAssignment::Allocation& prev_allocation_in_default_mem); @@ -1095,17 +1163,24 @@ class AlternateMemoryBestFitHeap : public GlobalDecreasingSizeBestFitHeap { const ChunkCandidate& chunk_candidate); // If we need to remove the allocations for this allocation sequence, this // removes pending chunks and asynchronous copies in the respective pending - // buffers from the interval trees. - void UncommitPendingChunks(); + // buffers from the interval trees. If an allocation request returns + // kFailRequiresUncommit, this method must be called. + void UncommitPendingChunks(absl::Span allocation_values); + + // Finalizes the allocations where they can no longer be uncommitted. + void FinalizeAllocations(absl::Span allocation_values); + + // Clears all pending chunks and asynchronous copies. + void ClearPendingChunks(); // Append buffer and allocation infos for debugging and dump it into a file, // if enabled. void AppendBufferInfoDebugString(const BufferInterval& interval, std::string* debug_str) const; void AppendAllocationInfoDebugString( - const BufferInterval& interval, + const AllocationValue& value, const MemorySpaceAssignment::Allocation& allocation, - std::string* debug_str) const; + std::string& debug_str) const; void DumpDebugStringsIfEnabled() const; // Returns the available heap size in the alternate memory. @@ -1132,9 +1207,6 @@ class AlternateMemoryBestFitHeap : public GlobalDecreasingSizeBestFitHeap { required_assignments_; // Number of bytes reserved in alternate memory space. int64 reserved_in_bytes_ = 0; - // Variables to control allocation retries. - bool final_retry_; - bool prefetch_failed_due_to_async_copy_; // Debug strings. std::string buffer_info_str_; std::string allocation_info_str_; From 45d4e5624be1ec842299216833fd91717db34e21 Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Mon, 10 Aug 2020 14:03:39 -0700 Subject: [PATCH 0768/1017] [tf.data service] Rename dispatcher journal directory. PiperOrigin-RevId: 325882898 Change-Id: I4ce77ee6f7cbeb5ed8cfd7caed96a505aea327aa --- tensorflow/core/data/service/dispatcher_impl.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/data/service/dispatcher_impl.cc b/tensorflow/core/data/service/dispatcher_impl.cc index 5dbcece7b49..a30de89ccea 100644 --- a/tensorflow/core/data/service/dispatcher_impl.cc +++ b/tensorflow/core/data/service/dispatcher_impl.cc @@ -44,7 +44,7 @@ namespace data { namespace { // The name of the journal directory inside the dispatcher's working directory. -constexpr char kJournalDir[] = "journal"; +constexpr char kJournalDir[] = "tf_data_dispatcher_journal"; using Dataset = DispatcherState::Dataset; using Worker = DispatcherState::Worker; From 642360f24eaa774abd8a48a2e94e08265da07038 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 14:22:32 -0700 Subject: [PATCH 0769/1017] Let CreateNewUploadSession support non-resumable writes by returning whether a resumable session was created via the out param "bool* resumable". PiperOrigin-RevId: 325886630 Change-Id: I97d81cb636d75c8593370676e052f40ab10d9a0c --- .../core/platform/cloud/gcs_file_system.cc | 44 +++++++++++-------- .../core/platform/cloud/gcs_file_system.h | 13 ++++-- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc index 31b0c790f50..f0d2138b379 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.cc +++ b/tensorflow/core/platform/cloud/gcs_file_system.cc @@ -389,7 +389,7 @@ class BufferedGcsRandomAccessFile : public RandomAccessFile { typedef std::function + UploadSessionHandle* session_handle)> SessionCreator; // Function object declaration with params needed to upload objects. @@ -542,7 +542,7 @@ class GcsWritableFile : public WritableFile { return errors::Internal( "Could not write to the internal temporary file."); } - string session_uri; + UploadSessionHandle session_handle; uint64 start_offset = 0; string object_to_upload = object_; bool should_compose = false; @@ -556,17 +556,21 @@ class GcsWritableFile : public WritableFile { io::Basename(object_), ".", start_offset_); } } - TF_RETURN_IF_ERROR( - CreateNewUploadSession(start_offset, object_to_upload, &session_uri)); + TF_RETURN_IF_ERROR(CreateNewUploadSession(start_offset, object_to_upload, + &session_handle)); uint64 already_uploaded = 0; bool first_attempt = true; const Status upload_status = RetryingUtils::CallWithRetries( - [&first_attempt, &already_uploaded, &session_uri, &start_offset, + [&first_attempt, &already_uploaded, &session_handle, &start_offset, this]() { - if (!first_attempt) { + if (session_handle.resumable && !first_attempt) { bool completed; TF_RETURN_IF_ERROR(RequestUploadSessionStatus( - session_uri, &completed, &already_uploaded)); + session_handle.session_uri, &completed, &already_uploaded)); + LOG(INFO) << "### RequestUploadSessionStatus: completed = " + << completed + << ", already_uploaded = " << already_uploaded + << ", file = " << GetGcsPath(); if (completed) { // Erase the file from the file cache on every successful write. file_cache_erase_(); @@ -577,7 +581,8 @@ class GcsWritableFile : public WritableFile { } } first_attempt = false; - return UploadToSession(session_uri, start_offset, already_uploaded); + return UploadToSession(session_handle.session_uri, start_offset, + already_uploaded); }, retry_config_); if (upload_status.code() == errors::Code::NOT_FOUND) { @@ -617,11 +622,11 @@ class GcsWritableFile : public WritableFile { /// Initiates a new resumable upload session. Status CreateNewUploadSession(uint64 start_offset, std::string object_to_upload, - std::string* session_uri) { + UploadSessionHandle* session_handle) { uint64 file_size; TF_RETURN_IF_ERROR(GetCurrentFileSize(&file_size)); return session_creator_(start_offset, object_to_upload, bucket_, file_size, - GetGcsPath(), session_uri); + GetGcsPath(), session_handle); } /// Appends the data of append_object to the original object and deletes @@ -913,6 +918,7 @@ GcsFileSystem::GcsFileSystem( std::pair* additional_header, bool compose_append) : timeouts_(timeouts), + retry_config_(retry_config), auth_provider_(std::move(auth_provider)), http_request_factory_(std::move(http_request_factory)), zone_provider_(std::move(zone_provider)), @@ -926,7 +932,6 @@ GcsFileSystem::GcsFileSystem( kCacheNeverExpire, kBucketLocationCacheMaxEntries)), allowed_locations_(allowed_locations), compose_append_(compose_append), - retry_config_(retry_config), additional_header_(additional_header) {} Status GcsFileSystem::NewRandomAccessFile( @@ -1080,7 +1085,7 @@ Status GcsFileSystem::LoadBufferFromGCS(const string& fname, size_t offset, Status GcsFileSystem::CreateNewUploadSession( uint64 start_offset, const std::string& object_to_upload, const std::string& bucket, uint64 file_size, const std::string& gcs_path, - std::string* session_uri) { + UploadSessionHandle* session_handle) { std::vector output_buffer; std::unique_ptr request; TF_RETURN_IF_ERROR(CreateHttpRequest(&request)); @@ -1096,9 +1101,10 @@ Status GcsFileSystem::CreateNewUploadSession( request->SetTimeouts(timeouts_.connect, timeouts_.idle, timeouts_.metadata); TF_RETURN_WITH_CONTEXT_IF_ERROR(request->Send(), " when initiating an upload to ", gcs_path); - if (session_uri != nullptr) { - *session_uri = request->GetResponseHeader("Location"); - if (session_uri->empty()) { + if (session_handle != nullptr) { + session_handle->resumable = true; + session_handle->session_uri = request->GetResponseHeader("Location"); + if (session_handle->session_uri.empty()) { return errors::Internal("Unexpected response from GCS when writing to ", gcs_path, ": 'Location' header not returned."); } @@ -1241,9 +1247,9 @@ Status GcsFileSystem::NewWritableFile(const string& fname, auto session_creator = [this](uint64 start_offset, const std::string& object_to_upload, const std::string& bucket, uint64 file_size, - const std::string& gcs_path, std::string* session_uri) { + const std::string& gcs_path, UploadSessionHandle* session_handle) { return CreateNewUploadSession(start_offset, object_to_upload, bucket, - file_size, gcs_path, session_uri); + file_size, gcs_path, session_handle); }; auto object_uploader = [this](const std::string& session_uri, uint64 start_offset, @@ -1301,9 +1307,9 @@ Status GcsFileSystem::NewAppendableFile(const string& fname, auto session_creator = [this](uint64 start_offset, const std::string& object_to_upload, const std::string& bucket, uint64 file_size, - const std::string& gcs_path, std::string* session_uri) { + const std::string& gcs_path, UploadSessionHandle* session_handle) { return CreateNewUploadSession(start_offset, object_to_upload, bucket, - file_size, gcs_path, session_uri); + file_size, gcs_path, session_handle); }; auto object_uploader = [this](const std::string& session_uri, uint64 start_offset, diff --git a/tensorflow/core/platform/cloud/gcs_file_system.h b/tensorflow/core/platform/cloud/gcs_file_system.h index 203c501ff4c..eceb76970fb 100644 --- a/tensorflow/core/platform/cloud/gcs_file_system.h +++ b/tensorflow/core/platform/cloud/gcs_file_system.h @@ -101,6 +101,11 @@ class GcsStatsInterface { virtual ~GcsStatsInterface() = default; }; +struct UploadSessionHandle { + std::string session_uri; + bool resumable; +}; + /// Google Cloud Storage implementation of a file system. /// /// The clients should use RetryingGcsFileSystem defined below, @@ -281,7 +286,7 @@ class GcsFileSystem : public FileSystem { const std::string& bucket, uint64 file_size, const std::string& gcs_path, - std::string* session_uri); + UploadSessionHandle* session_handle); // Uploads object data to session. virtual Status UploadToSession(const std::string& session_uri, @@ -318,6 +323,9 @@ class GcsFileSystem : public FileSystem { // Used by a subclass. TimeoutConfig timeouts_; + /// The retry configuration used for retrying failed calls. + RetryConfig retry_config_; + private: // GCS file statistics. struct GcsFileStat { @@ -416,9 +424,6 @@ class GcsFileSystem : public FileSystem { GcsStatsInterface* stats_ = nullptr; // Not owned. - /// The initial delay for exponential backoffs when retrying failed calls. - RetryConfig retry_config_; - // Additional header material to be transmitted with all GCS requests std::unique_ptr> additional_header_; From 9b4f994681a00f741bf6dc1f033b588099064fc0 Mon Sep 17 00:00:00 2001 From: Edward Loper Date: Mon, 10 Aug 2020 14:25:56 -0700 Subject: [PATCH 0770/1017] Improve error message when tf.ragged.map_flat_values is called with a function that doesn't preserve the outer dimension size of ragged inputs. PiperOrigin-RevId: 325887317 Change-Id: Ibdc85269a9ff8b842844b1f22c4bd005e554bf52 --- .../ops/ragged/ragged_functional_ops.py | 44 ++++++++++++++++--- .../ragged/ragged_map_flat_values_op_test.py | 31 +++++++++---- 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/tensorflow/python/ops/ragged/ragged_functional_ops.py b/tensorflow/python/ops/ragged/ragged_functional_ops.py index 00b5ced6170..22625077e56 100644 --- a/tensorflow/python/ops/ragged/ragged_functional_ops.py +++ b/tensorflow/python/ops/ragged/ragged_functional_ops.py @@ -20,6 +20,7 @@ from __future__ import print_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_shape from tensorflow.python.ops import math_ops from tensorflow.python.ops.ragged import ragged_config from tensorflow.python.ops.ragged import ragged_tensor @@ -70,10 +71,22 @@ def map_flat_values(op, *args, **kwargs): # Replace RaggedTensors with their values; and collect the splits tensors # from each RaggedTensor. nested_splits_lists = [] - inner_args = _replace_ragged_with_flat_values(args, nested_splits_lists) - inner_kwargs = _replace_ragged_with_flat_values(kwargs, nested_splits_lists) + flat_values_nrows = [] + inner_args = _replace_ragged_with_flat_values(args, nested_splits_lists, + flat_values_nrows) + inner_kwargs = _replace_ragged_with_flat_values(kwargs, nested_splits_lists, + flat_values_nrows) if not nested_splits_lists: return op(*args, **kwargs) + if flat_values_nrows: + flat_values_nrows = set(flat_values_nrows) + if len(flat_values_nrows) != 1: + raise ValueError("Input RaggedTensors' flat_values must all have the " + "same outer-dimension size. Got sizes: %s" % + flat_values_nrows) + flat_values_nrows = flat_values_nrows.pop() # Get the single element + else: + flat_values_nrows = None split_dtypes = set(splits[0].dtype for splits in nested_splits_lists) if len(split_dtypes) > 1: @@ -88,13 +101,23 @@ def map_flat_values(op, *args, **kwargs): with ops.control_dependencies( ragged_util.assert_splits_match(nested_splits_lists)): - # Delegate to op, and then compose the result from the transformed values - # and the splits. + # Delegate to `op` + op_output = op(*inner_args, **inner_kwargs) + # Check that the result has the expected shape (if known). + if flat_values_nrows is not None: + if not op_output.shape[:1].is_compatible_with([flat_values_nrows]): + raise ValueError( + "tf.ragged.map_flat_values requires that the output of `op` have " + "the same outer-dimension size as flat_values of any ragged " + "inputs. (output shape: %s; expected outer dimension size: %s)" % + (op_output.shape, flat_values_nrows)) + # Compose the result from the transformed values and the splits. return ragged_tensor.RaggedTensor.from_nested_row_splits( - op(*inner_args, **inner_kwargs), nested_splits_lists[0], validate=False) + op_output, nested_splits_lists[0], validate=False) -def _replace_ragged_with_flat_values(value, nested_splits_lists): +def _replace_ragged_with_flat_values(value, nested_splits_lists, + flat_values_nrows): """Replace RaggedTensors with their flat_values, and record their splits. Returns a copy of `value`, with any nested `RaggedTensor`s replaced by their @@ -106,6 +129,9 @@ def _replace_ragged_with_flat_values(value, nested_splits_lists): value: The value that should be transformed by replacing `RaggedTensors`. nested_splits_lists: An output parameter used to record the `nested_splits` for any `RaggedTensors` that were replaced. + flat_values_nrows: An output parameter used to record the outer dimension + size for each replacement `flat_values` (when known). Contains a list of + int. Returns: A copy of `value` with nested `RaggedTensors` replaced by their `values`. @@ -114,11 +140,15 @@ def _replace_ragged_with_flat_values(value, nested_splits_lists): if ragged_tensor.is_ragged(value): value = ragged_tensor.convert_to_tensor_or_ragged_tensor(value) nested_splits_lists.append(value.nested_row_splits) + nrows = tensor_shape.dimension_at_index(value.flat_values.shape, 0).value + if nrows is not None: + flat_values_nrows.append(nrows) return value.flat_values # Recursion cases def recurse(v): - return _replace_ragged_with_flat_values(v, nested_splits_lists) + return _replace_ragged_with_flat_values(v, nested_splits_lists, + flat_values_nrows) if isinstance(value, list): return [recurse(v) for v in value] diff --git a/tensorflow/python/ops/ragged/ragged_map_flat_values_op_test.py b/tensorflow/python/ops/ragged/ragged_map_flat_values_op_test.py index 588a5473741..e65c877aa68 100644 --- a/tensorflow/python/ops/ragged/ragged_map_flat_values_op_test.py +++ b/tensorflow/python/ops/ragged/ragged_map_flat_values_op_test.py @@ -178,18 +178,33 @@ class RaggedMapInnerValuesOpTest(test_util.TensorFlowTestCase): def testRaggedTensorSplitsRaggedRankMismatchError(self): x = ragged_factory_ops.constant([[3, 1, 4], [], [1, 5]]) y = ragged_factory_ops.constant([[[3, 1, 4], []], [], [[1, 5]]]) - self.assertRaisesRegex(ValueError, - r'Inputs must have identical ragged splits.*', - ragged_functional_ops.map_flat_values, math_ops.add, - x, y) + with self.assertRaisesRegex(ValueError, + r'Inputs must have identical ragged splits.*'): + ragged_functional_ops.map_flat_values(math_ops.add, x, y) def testRaggedTensorSplitsValueMismatchError(self): x = ragged_factory_ops.constant([[3, 1, 4], [], [1, 5]]) y = ragged_factory_ops.constant([[1], [2, 3], [4, 5]]) - self.assertRaisesRegex(errors.InvalidArgumentError, - r'Inputs must have identical ragged splits.*', - ragged_functional_ops.map_flat_values, math_ops.add, - x, y) + with self.assertRaisesRegex(errors.InvalidArgumentError, + r'Inputs must have identical ragged splits.*'): + ragged_functional_ops.map_flat_values(math_ops.add, x, y) + + z_splits = array_ops.placeholder_with_default( + constant_op.constant([0, 3], dtypes.int64), None) + z = ragged_tensor.RaggedTensor.from_row_splits([0, 1, 2], z_splits) + with self.assertRaisesRegex( + ValueError, + r"Input RaggedTensors' flat_values must all have the same " + r'outer-dimension size. Got sizes: \{3, 5\}'): + ragged_functional_ops.map_flat_values(math_ops.add, x, z) + + def testRaggedTensorShapeMismatchError(self): + x = ragged_factory_ops.constant([[1, 2, 3], [4, 5]]) + with self.assertRaisesRegex( + ValueError, r'tf.ragged.map_flat_values requires that the output of ' + '`op` have the same outer-dimension size as flat_values of any ragged ' + r'inputs. \(output shape: \(\); expected outer dimension size: 5\)'): + ragged_functional_ops.map_flat_values(math_ops.argmax, x) def testRaggedTensorSplitsMismatchErrorAtRuntime(self): splits1 = array_ops.placeholder_with_default( From c69cd441212e35a3157d067b263314bcb9b34c0e Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Mon, 10 Aug 2020 14:26:29 -0700 Subject: [PATCH 0771/1017] Replace CLDevice with DeviceInfo in util.h. PiperOrigin-RevId: 325887446 Change-Id: Icbe7decfd7e1f3c851a088a37af92d8e2d30c4b2 --- tensorflow/lite/delegates/gpu/cl/kernels/BUILD | 2 +- .../gpu/cl/kernels/conv_buffer_1x1.cc | 4 ++-- .../delegates/gpu/cl/kernels/conv_powervr.cc | 4 ++-- .../lite/delegates/gpu/cl/kernels/util.cc | 18 ++++-------------- .../lite/delegates/gpu/cl/kernels/util.h | 4 ++-- 5 files changed, 11 insertions(+), 21 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/BUILD b/tensorflow/lite/delegates/gpu/cl/kernels/BUILD index b89e7d7252a..27e12b5981f 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/cl/kernels/BUILD @@ -1296,7 +1296,7 @@ cc_library( srcs = ["util.cc"], hdrs = ["util.h"], deps = [ - "//tensorflow/lite/delegates/gpu/cl:cl_device", + "//tensorflow/lite/delegates/gpu/cl:device_info", "//tensorflow/lite/delegates/gpu/cl:precision", "//tensorflow/lite/delegates/gpu/cl:tensor_type", "//tensorflow/lite/delegates/gpu/common:access_type", diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc index 3216e2ef246..38e04e221f2 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc @@ -105,8 +105,8 @@ ConvBuffer1x1::ConvParams GetBestParams(const CLDevice& device, } int task_size = shape.w * shape.b * shape.h * dst_depth; - int block_size = - GetRecommendedBlockSizeForConv(device, definition.precision, task_size); + int block_size = GetRecommendedBlockSizeForConv( + device.info_, definition.precision, task_size); if (!can_use_flt8 && block_size > 4) { block_size = 4; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc index d65595d068c..f04102d25d6 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc @@ -820,8 +820,8 @@ ConvPowerVR::ConvParams ConvPowerVR::GuessBestParams( int block_size = 2; if (dst_shape) { int task_size = dst_shape->w * dst_shape->b * dst_shape->h * dst_depth; - block_size = GetRecommendedBlockSizeForConv(device, definition.precision, - task_size); + block_size = GetRecommendedBlockSizeForConv( + device.info_, definition.precision, task_size); } if (!x_kernel_is_1 || !y_kernel_is_1) { block_size = std::min(block_size, 4); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/util.cc b/tensorflow/lite/delegates/gpu/cl/kernels/util.cc index d907c0210b7..b7cfa5f013e 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/util.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/util.cc @@ -84,16 +84,6 @@ std::string GetXStrideCorrected(const std::string& src_x, batch_size, stride_x, padding_x); } -TextureAddressMode GetFastestZeroMode(const CLDevice& device) { - return device.IsAdreno3xx() ? TextureAddressMode::DONT_CARE - : TextureAddressMode::ZERO; -} - -TextureAddressMode GetFastestZeroMode(const DeviceInfo& device_info) { - return device_info.IsAdreno3xx() ? TextureAddressMode::DONT_CARE - : TextureAddressMode::ZERO; -} - float4 GetMaskForLastPlane(int channels) { float4 mask = float4(0.0f); const int reminder = channels % 4 == 0 ? 4 : channels % 4; @@ -113,19 +103,19 @@ int3 GetFirstSuitableWorkGroup(const std::vector& wgs, int max_wg_size) { return {1, 1, 1}; } -int GetRecommendedBlockSizeForConv(const CLDevice& device, +int GetRecommendedBlockSizeForConv(const DeviceInfo& device_info, CalculationsPrecision precision, int task_size) { const float task_size_per_cu = - task_size / static_cast(device.info_.compute_units_count); + task_size / static_cast(device_info.compute_units_count); int block_size = 1; float threshold_1 = FLT_MAX; float threshold_2 = FLT_MAX; float threshold_4 = FLT_MAX; - if (!device.IsMali()) { + if (!device_info.IsMali()) { return 1; } - MaliInfo mali_info = device.info_.mali_info; + MaliInfo mali_info = device_info.mali_info; switch (precision) { case CalculationsPrecision::F16: if (mali_info.IsBifrostGen1()) { diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/util.h b/tensorflow/lite/delegates/gpu/cl/kernels/util.h index 173a4d43072..b1dd4fe8c57 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/util.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/util.h @@ -19,7 +19,7 @@ limitations under the License. #include #include "absl/types/span.h" -#include "tensorflow/lite/delegates/gpu/cl/cl_device.h" +#include "tensorflow/lite/delegates/gpu/cl/device_info.h" #include "tensorflow/lite/delegates/gpu/cl/precision.h" #include "tensorflow/lite/delegates/gpu/cl/tensor_type.h" #include "tensorflow/lite/delegates/gpu/common/access_type.h" @@ -95,7 +95,7 @@ float4 GetMaskForLastPlane(int channels); int3 GetFirstSuitableWorkGroup(const std::vector& wgs, int max_wg_size); // task_size as amount of FLT4 processed elements. -int GetRecommendedBlockSizeForConv(const CLDevice& device, +int GetRecommendedBlockSizeForConv(const DeviceInfo& device, CalculationsPrecision precision, int task_size); } // namespace cl From 354722afd88e95971ec3cb5f2f6ed01e7f538557 Mon Sep 17 00:00:00 2001 From: Lukas Geiger Date: Mon, 10 Aug 2020 23:38:27 +0200 Subject: [PATCH 0772/1017] Return new AutoCastVariable after assignment --- .../experimental/autocast_variable.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py index 48ed905e67e..20770061639 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py +++ b/tensorflow/python/keras/mixed_precision/experimental/autocast_variable.py @@ -53,11 +53,12 @@ class AutoCastVariable(variables.Variable, core.Tensor): called. """ - def __init__(self, variable): + def __init__(self, variable, op=None): """Creates an AutoCastVariable instance. Args: variable: A floating-point resource variable to wrap. + op: Optional operation of this variable. Raises: ValueError: If `variable` is not a floating-point resource variable @@ -69,7 +70,7 @@ class AutoCastVariable(variables.Variable, core.Tensor): raise ValueError('variable must be a floating point variable but has ' 'type: %s' % variable.dtype.name) self._variable = variable - self._op = None + self._op = op def _should_cast(self): """Returns True if this variable should be casted when accessed.""" @@ -198,8 +199,7 @@ class AutoCastVariable(variables.Variable, core.Tensor): if ops.executing_eagerly_outside_functions(): assign_op = update_fn(value, use_locking, name, False) if read_value: - self._op = assign_op - return self + return create_autocast_variable(self._variable, op=assign_op) return assign_op # Fallback to wrapping the returned variable in graph mode if possible @@ -466,7 +466,7 @@ ops.register_tensor_conversion_function(AutoCastVariable, AutoCastVariable._dense_var_to_tensor) # pylint:disable=protected-access -def create_autocast_variable(variable): +def create_autocast_variable(variable, op=None): """Creates an AutoCastVariable that wraps another variable. This typically just returns `AutoCastVariable(variable)`. But, if the variable @@ -478,13 +478,14 @@ def create_autocast_variable(variable): Args: variable: A floating-point resource variable to wrap. + op: Optional operation of this variable. Returns: An AutoCastVariable that wraps the variable. """ if not isinstance(variable, (distribute_values.DistributedVariable, ps_distribute_values.AggregatingVariable)): - return AutoCastVariable(variable) + return AutoCastVariable(variable, op=op) class AutoCastDistributedVariable(AutoCastVariable, variable.__class__): """An AutoCastVariable that also subclasses from variable.__class__. @@ -506,4 +507,4 @@ def create_autocast_variable(variable): ).format(v=self) # pylint: enable=missing-format-attribute - return AutoCastDistributedVariable(variable) + return AutoCastDistributedVariable(variable, op=op) From 7a9511547798bff3d48ecc8598ec3acc3b7269f7 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 10 Aug 2020 14:29:41 -0700 Subject: [PATCH 0773/1017] Introduce local TPU support for TPUClusterResolver PiperOrigin-RevId: 325888141 Change-Id: Iea19b39a6c5a6a56dec7cd32eae8b5b12ce849a7 --- .../tpu/tpu_cluster_resolver.py | 98 ++++++++++++------- .../tpu/tpu_cluster_resolver_test.py | 4 + 2 files changed, 66 insertions(+), 36 deletions(-) diff --git a/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py index e42420ec644..d400e7aeed4 100644 --- a/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py +++ b/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py @@ -22,6 +22,7 @@ import collections import re from tensorflow.python.distribute.cluster_resolver import cluster_resolver +from tensorflow.python.framework import config as framework_config from tensorflow.python.framework import errors from tensorflow.python.platform import tf_logging as logging from tensorflow.python.tpu import tpu_system_metadata as tpu_system_metadata_lib @@ -155,7 +156,8 @@ class TPUClusterResolver(cluster_resolver.ClusterResolver): Args: tpu: A string corresponding to the TPU to use. It can be the TPU name or TPU worker gRPC address. If not set, it will try automatically resolve - the TPU address on Cloud TPUs. + the TPU address on Cloud TPUs. If set to "local", it will assume that + the TPU is directly connected to the VM instead of over the network. zone: Zone where the TPUs are located. If omitted or empty, we will assume that the zone of the TPU is the same as the zone of the GCE VM, which we will try to discover from the GCE metadata service. @@ -187,15 +189,21 @@ class TPUClusterResolver(cluster_resolver.ClusterResolver): Google Cloud environment. """ - self._cloud_tpu_client = client.Client( - tpu=tpu, - zone=zone, - project=project, - credentials=credentials, - service=service, - discovery_url=discovery_url) + if tpu != 'local': + # Default Cloud environment + self._cloud_tpu_client = client.Client( + tpu=tpu, + zone=zone, + project=project, + credentials=credentials, + service=service, + discovery_url=discovery_url) + self._tpu = self._cloud_tpu_client.name() + else: + # Directly connected TPU environment + self._cloud_tpu_client = None + self._tpu = 'local' - self._tpu = self._cloud_tpu_client.name() # By default the task_type is 'worker` and the task_id is 0 (which is the # first worker in the task). self.task_type = job_name @@ -238,20 +246,23 @@ class TPUClusterResolver(cluster_resolver.ClusterResolver): ValueError: If none of the TPUs specified exists. """ - cluster_spec = self.cluster_spec() - if task_type is not None and task_id is not None: - # task_type and task_id is from the function parameter - master = cluster_spec.task_address(task_type, task_id) - elif self.task_type is not None and self.task_id is not None: - # task_type and task_id is from the object - master = cluster_spec.task_address(self.task_type, self.task_id) + if self._tpu != 'local': + cluster_spec = self.cluster_spec() + if task_type is not None and task_id is not None: + # task_type and task_id is from the function parameter + master = cluster_spec.task_address(task_type, task_id) + elif self.task_type is not None and self.task_id is not None: + # task_type and task_id is from the object + master = cluster_spec.task_address(self.task_type, self.task_id) + else: + # by default we take the first item in the cluster with the right name + job_tasks = cluster_spec.job_tasks(self.task_type) + if not job_tasks: + raise ValueError('No TPUs with the specified names exist.') + master = job_tasks[0] + return cluster_resolver.format_master_url(master, 'grpc') else: - # by default we take the first item in the cluster with the right name - job_tasks = cluster_spec.job_tasks(self.task_type) - if not job_tasks: - raise ValueError('No TPUs with the specified names exist.') - master = job_tasks[0] - return cluster_resolver.format_master_url(master, 'grpc') + return '' def get_master(self): return self.master() @@ -298,7 +309,8 @@ class TPUClusterResolver(cluster_resolver.ClusterResolver): RuntimeError: If the provided TPU is not healthy. """ ############################################################################ - # There are 5 potential cases this code must handle: + # There are 6 potential cases this code must handle: + # 0. [Local case.] When a TPU is connected directly to the VM. # 1. [Normal case.] We should resolve the TPU name to a set of tasks, and # a. Create a ClusterSpec that includes the coordinator job # b. Create a ClusterSpec without the coordinator job. @@ -308,17 +320,19 @@ class TPUClusterResolver(cluster_resolver.ClusterResolver): # b. Create a ClusterSpec without the coordinator ############################################################################ - network_endpoints = self._cloud_tpu_client.network_endpoints() - worker_list = [ - '%s:%s' % (endpoint['ipAddress'], endpoint['port']) - for endpoint in network_endpoints - ] - cluster_spec = {self.task_type: worker_list} - if self._coordinator_address: - # {1, 2}.a - cluster_spec[self._coordinator_name] = [self._coordinator_address] - - return server_lib.ClusterSpec(cluster_spec) + if self._tpu != 'local': + network_endpoints = self._cloud_tpu_client.network_endpoints() + worker_list = [ + '%s:%s' % (endpoint['ipAddress'], endpoint['port']) + for endpoint in network_endpoints + ] + cluster_spec = {self.task_type: worker_list} + if self._coordinator_address: + # {1, 2}.a + cluster_spec[self._coordinator_name] = [self._coordinator_address] + return server_lib.ClusterSpec(cluster_spec) + else: + return server_lib.ClusterSpec({}) def num_accelerators(self, task_type=None, @@ -340,6 +354,15 @@ class TPUClusterResolver(cluster_resolver.ClusterResolver): RuntimeError: If we cannot talk to a TPU worker after retrying or if the number of TPU devices per host is different. """ + if self._tpu == 'local': + return { + 'TPU': + len([ + d for d in framework_config.list_logical_devices() + if d.device_type == 'TPU' + ]) + } + retry_count = 1 # TODO(b/120564445): Replace with standard library for retries. while True: @@ -360,8 +383,11 @@ class TPUClusterResolver(cluster_resolver.ClusterResolver): raise RuntimeError(error_message) if device_details.total_cores: - return {'TPU': TPUClusterResolver._verify_and_return_same_core_count( - device_details.device_map)} + return { + 'TPU': + TPUClusterResolver._verify_and_return_same_core_count( + device_details.device_map) + } return {'TPU': 0} @property diff --git a/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver_test.py b/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver_test.py index 51abc850bb2..155410f9668 100644 --- a/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver_test.py +++ b/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver_test.py @@ -706,6 +706,10 @@ class TPUClusterResolverTest(test.TestCase): with self.assertRaises(RuntimeError): cluster_resolver.num_accelerators() + def testLocalTpuResolver(self): + cr = resolver.TPUClusterResolver(tpu='local') + self.assertEqual(cr.get_master(), '') + if __name__ == '__main__': test.main() From 31ea2b852ae121ffc1b1958ea7ed0f0edab10ae7 Mon Sep 17 00:00:00 2001 From: Smit Hinsu Date: Mon, 10 Aug 2020 14:29:48 -0700 Subject: [PATCH 0774/1017] Legalize TensorFlow ops related to Matrix diag, space and batch conversion in the fallback path These ops are: BatchToSpaceNDOp BatchToSpaceOp SpaceToBatchNDOp SpaceToBatchOp MatrixDiagV3Op MatrixSetDiagV3Op PiperOrigin-RevId: 325888162 Change-Id: Ifab8fa68c87f745f91a8d6faeb765e40c2d07d94 --- .../compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc | 6 ++++++ tensorflow/compiler/tests/BUILD | 2 ++ 2 files changed, 8 insertions(+) diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc index f04f1653505..6e651df5075 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc @@ -100,6 +100,8 @@ bool IsOpAllowedTf2XlaFallback(Operation* op) { TypeID::get(), TypeID::get(), TypeID::get(), + TypeID::get(), + TypeID::get(), TypeID::get(), TypeID::get(), TypeID::get(), @@ -152,6 +154,8 @@ bool IsOpAllowedTf2XlaFallback(Operation* op) { TypeID::get(), TypeID::get(), TypeID::get(), + TypeID::get(), + TypeID::get(), TypeID::get(), TypeID::get(), TypeID::get(), @@ -184,6 +188,8 @@ bool IsOpAllowedTf2XlaFallback(Operation* op) { TypeID::get(), TypeID::get(), TypeID::get(), + TypeID::get(), + TypeID::get(), TypeID::get(), TypeID::get(), TypeID::get(), diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index a3134fc1c94..924834fc0fc 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -858,6 +858,7 @@ tf_xla_py_test( size = "medium", timeout = "long", srcs = ["matrix_diag_ops_test.py"], + enable_mlir_bridge = True, python_version = "PY3", tags = [ "no_pip", # TODO(b/149738646): fix pip install so these tests run on kokoro pip @@ -1204,6 +1205,7 @@ tf_xla_py_test( name = "spacetobatch_op_test", size = "medium", srcs = ["spacetobatch_op_test.py"], + enable_mlir_bridge = True, python_version = "PY3", shard_count = 3, tags = [ From c042a0c82aa3949745eb92e6b9c82350ab710625 Mon Sep 17 00:00:00 2001 From: Bruce Fontaine Date: Mon, 10 Aug 2020 14:33:35 -0700 Subject: [PATCH 0775/1017] Add support for the multiply_linear_by_lr, beta and allow_zero_accumulator options to the FTRL optimizer for TPUEmbedding. Increase TPU initialization timeout in TPUEstimator. PiperOrigin-RevId: 325889098 Change-Id: I826f3bd8a7fbfb9b74ece7359b778d7fee604b10 --- tensorflow/python/tpu/tpu_embedding.py | 25 ++++++++++++++++++- ...ow.tpu.experimental.-ftrl-parameters.pbtxt | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/tpu/tpu_embedding.py b/tensorflow/python/tpu/tpu_embedding.py index 13afe1a2147..e9d1a3be6b6 100644 --- a/tensorflow/python/tpu/tpu_embedding.py +++ b/tensorflow/python/tpu/tpu_embedding.py @@ -577,9 +577,15 @@ class FtrlParameters(_OptimizationParameters): clip_weight_min=None, clip_weight_max=None, weight_decay_factor=None, - multiply_weight_decay_factor_by_learning_rate=None): + multiply_weight_decay_factor_by_learning_rate=None, + multiply_linear_by_learning_rate=False, + beta=0, + allow_zero_accumulator=False): """Optimization parameters for Ftrl. + Implements FTRL as described in the following [paper]( + https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/41159.pdf) + Args: learning_rate: a floating point value. The learning rate. learning_rate_power: A float value, must be less or equal to zero. @@ -602,6 +608,14 @@ class FtrlParameters(_OptimizationParameters): weights are not decayed. multiply_weight_decay_factor_by_learning_rate: if true, `weight_decay_factor` is multiplied by the current learning rate. + multiply_linear_by_learning_rate: When true, multiplies the usages of the + linear slot in the weight update by the learning rate. This is useful + when ramping up learning rate from 0 (which would normally produce + NaNs). + beta: The beta parameter for FTRL. + allow_zero_accumulator: Changes the implementation of the square root to + allow for the case of initial_accumulator_value being zero. This will + cause a slight performance drop. """ super(FtrlParameters, self).__init__(learning_rate, use_gradient_accumulation, @@ -628,6 +642,9 @@ class FtrlParameters(_OptimizationParameters): self.initial_linear_value = 0.0 self.l1_regularization_strength = l1_regularization_strength self.l2_regularization_strength = l2_regularization_strength + self.multiply_linear_by_learning_rate = multiply_linear_by_learning_rate + self.beta = beta + self.allow_zero_accumulator = allow_zero_accumulator class ProximalYogiParameters(_OptimizationParameters): @@ -1896,6 +1913,12 @@ class _FtrlHandler(_OptimizerHandler): self._optimization_parameters.l1_regularization_strength) table_descriptor.optimization_parameters.ftrl.l2 = ( self._optimization_parameters.l2_regularization_strength) + table_descriptor.optimization_parameters.ftrl.multiply_linear_by_lr = ( + self._optimization_parameters.multiply_linear_by_learning_rate) + table_descriptor.optimization_parameters.ftrl.beta = ( + self._optimization_parameters.beta) + table_descriptor.optimization_parameters.ftrl.allow_zero_accumulator = ( + self._optimization_parameters.allow_zero_accumulator) def get_default_slot_variable_names(self, table): # These match the default slot variable names created by diff --git a/tensorflow/tools/api/golden/v1/tensorflow.tpu.experimental.-ftrl-parameters.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.tpu.experimental.-ftrl-parameters.pbtxt index 9e435cc0e8f..450015c3695 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.tpu.experimental.-ftrl-parameters.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.tpu.experimental.-ftrl-parameters.pbtxt @@ -5,6 +5,6 @@ tf_class { is_instance: "" member_method { name: "__init__" - argspec: "args=[\'self\', \'learning_rate\', \'learning_rate_power\', \'initial_accumulator_value\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_gradient_accumulation\', \'clip_weight_min\', \'clip_weight_max\', \'weight_decay_factor\', \'multiply_weight_decay_factor_by_learning_rate\'], varargs=None, keywords=None, defaults=[\'-0.5\', \'0.1\', \'0.0\', \'0.0\', \'True\', \'None\', \'None\', \'None\', \'None\'], " + argspec: "args=[\'self\', \'learning_rate\', \'learning_rate_power\', \'initial_accumulator_value\', \'l1_regularization_strength\', \'l2_regularization_strength\', \'use_gradient_accumulation\', \'clip_weight_min\', \'clip_weight_max\', \'weight_decay_factor\', \'multiply_weight_decay_factor_by_learning_rate\', \'multiply_linear_by_learning_rate\', \'beta\', \'allow_zero_accumulator\'], varargs=None, keywords=None, defaults=[\'-0.5\', \'0.1\', \'0.0\', \'0.0\', \'True\', \'None\', \'None\', \'None\', \'None\', \'False\', \'0\', \'False\'], " } } From 2949d74a7885e6d440bd64ccd0ce839e1e6c9a87 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Mon, 10 Aug 2020 21:54:41 +0000 Subject: [PATCH 0776/1017] fix typo --- tensorflow/core/kernels/map_kernels.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.cc b/tensorflow/core/kernels/map_kernels.cc index 19b79a242b1..03e5e05e9fd 100644 --- a/tensorflow/core/kernels/map_kernels.cc +++ b/tensorflow/core/kernels/map_kernels.cc @@ -41,9 +41,9 @@ REGISTER_KERNEL_BUILDER(Name("TensorMapErase").Device(DEVICE_CPU), REGISTER_KERNEL_BUILDER(Name("TensorMapHasKey").Device(DEVICE_CPU), TensorMapHasKey); -#undef REGISTER_TENSOR_LIST_OPS_CPU +#undef REGISTER_TENSOR_MAP_OPS_CPU -#define REGISTER_TENSOR_LIST_OPS_CPU(T) +#define REGISTER_TENSOR_MAP_OPS_CPU(T) REGISTER_UNARY_VARIANT_BINARY_OP_FUNCTION(ADD_VARIANT_BINARY_OP, DEVICE_CPU, TensorMap, From 6411114cbd94777be68d78b6005cb44fad8e4983 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Mon, 10 Aug 2020 14:47:36 -0700 Subject: [PATCH 0777/1017] Fix CSV writer on Windows PiperOrigin-RevId: 325892185 Change-Id: I96053328785323a40450e4b33704d873a6c6f38b --- tensorflow/tools/ci_build/sizetrack_helper.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/ci_build/sizetrack_helper.py b/tensorflow/tools/ci_build/sizetrack_helper.py index eb3a6afda5e..03a04e36588 100755 --- a/tensorflow/tools/ci_build/sizetrack_helper.py +++ b/tensorflow/tools/ci_build/sizetrack_helper.py @@ -364,8 +364,9 @@ def main(): print("DRY RUN: Generated this TSV row:") print("\t".join(map(str, next_tsv_row))) else: - with open("data.tsv", "w") as tsvfile: - writer = csv.writer(tsvfile, delimiter="\t", quoting=csv.QUOTE_MINIMAL) + with open("data.tsv", "w", newline="") as tsvfile: + writer = csv.writer(tsvfile, delimiter="\t", quoting=csv.QUOTE_MINIMAL, + lineterminator=os.linesep) writer.writerow(next_tsv_row) bq([ "load", "--source_format", "CSV", "--field_delimiter", "tab", From dd2ee4e8cc917e878d566b7f5ca28d8866247492 Mon Sep 17 00:00:00 2001 From: Edward Loper Date: Mon, 10 Aug 2020 14:48:44 -0700 Subject: [PATCH 0778/1017] Fix bug in ConvertPyObjectToAttributeType when type="tensor" and value is a string that's not a Tensor textproto. PiperOrigin-RevId: 325892441 Change-Id: I4a29954ec32b84fef75859ac36f4e694a0317688 --- tensorflow/python/framework/op_def_util.cc | 6 ++++-- tensorflow/python/framework/op_def_util_test.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/framework/op_def_util.cc b/tensorflow/python/framework/op_def_util.cc index 4f56c62317c..c915c494be9 100644 --- a/tensorflow/python/framework/op_def_util.cc +++ b/tensorflow/python/framework/op_def_util.cc @@ -167,8 +167,10 @@ struct ConvertTensorProtoFunctor { } else if (PY_STRING_CHECK(value)) { result.reset(PyObject_CallObject(tensor_proto, nullptr)); if (result) { - PyObject_CallFunctionObjArgs(text_format_parse, value, result.get(), - nullptr); + if (!PyObject_CallFunctionObjArgs(text_format_parse, value, + result.get(), nullptr)) { + return nullptr; + } } } return result; diff --git a/tensorflow/python/framework/op_def_util_test.py b/tensorflow/python/framework/op_def_util_test.py index 74cd6046f68..69aaffbf19f 100644 --- a/tensorflow/python/framework/op_def_util_test.py +++ b/tensorflow/python/framework/op_def_util_test.py @@ -87,6 +87,7 @@ class OpDefUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): ("list(any)", 12), ("list(int)", [1, "two"]), ("list(string)", [1, "two"]), + ("tensor", "string that is not a text-formatted TensorProto"), ]) def testConvertError(self, attr_type, value): with self.assertRaisesRegex(TypeError, "Failed to convert value"): From 1e336d3ef0706792772b605a1fed0135f5cc5cfe Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Mon, 10 Aug 2020 15:19:24 -0700 Subject: [PATCH 0779/1017] [SE] Don't assume that the CUDA context has not changed in the outermost ScopedActivationContext. Will fix https://github.com/google/jax/issues/3802 when incorporated into JAX. PiperOrigin-RevId: 325899237 Change-Id: I1f2bf59d982da16db138229d8fa155f41a7e094a --- tensorflow/stream_executor/cuda/BUILD | 12 +++ .../stream_executor/cuda/cuda_driver.cc | 18 ++++- .../stream_executor/cuda/cuda_driver_test.cc | 76 +++++++++++++++++++ 3 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 tensorflow/stream_executor/cuda/cuda_driver_test.cc diff --git a/tensorflow/stream_executor/cuda/BUILD b/tensorflow/stream_executor/cuda/BUILD index f3cffc04465..bd545f097cf 100644 --- a/tensorflow/stream_executor/cuda/BUILD +++ b/tensorflow/stream_executor/cuda/BUILD @@ -130,6 +130,18 @@ cc_library( ], ) +tf_cuda_cc_test( + name = "cuda_driver_test", + srcs = ["cuda_driver_test.cc"], + tags = tf_cuda_tests_tags(), + deps = [ + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/stream_executor/lib", + "@local_config_cuda//cuda:cuda_headers", + ], +) + tf_cuda_cc_test( name = "memcpy_test", srcs = ["memcpy_test.cc"], diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc index e30eb549a9c..67fd72d52f3 100644 --- a/tensorflow/stream_executor/cuda/cuda_driver.cc +++ b/tensorflow/stream_executor/cuda/cuda_driver.cc @@ -200,6 +200,21 @@ ScopedActivateContext::ScopedActivateContext(GpuContext* cuda_context) { if (FLAGS_gpuexec_cuda_sync_around_driver_calls) SynchronizeOrDie(); auto* tls = &tls_data.get(); + + // If this is an outermost scope, we must not assume that the CUDA context has + // been left in the same state we left it. Other code may have run on this + // thread and altered the context. + if (tls->depth == 0) { + VLOG(3) << "ScopedActivateContext switching to " << cuda_context->id(); + FAIL_IF_CUDA_RES_ERROR(cuCtxSetCurrent(cuda_context->context()), + "Failed setting context"); + tls->depth = 1; + tls->id = cuda_context->id(); + tls->context = cuda_context; + to_restore_ = nullptr; + return; + } + tls->depth++; if (tls->id == cuda_context->id()) { if (kVerifyGpuContext) { @@ -212,8 +227,7 @@ ScopedActivateContext::ScopedActivateContext(GpuContext* cuda_context) { VLOG(3) << "ScopedActivateContext switching context from " << tls->id << " to " << cuda_context->id(); - to_restore_ = (tls->depth == 1 ? nullptr : tls->context); - + to_restore_ = tls->context; // Set the context and update thread local. FAIL_IF_CUDA_RES_ERROR(cuCtxSetCurrent(cuda_context->context()), "Failed setting context"); diff --git a/tensorflow/stream_executor/cuda/cuda_driver_test.cc b/tensorflow/stream_executor/cuda/cuda_driver_test.cc new file mode 100644 index 00000000000..5b173f96d85 --- /dev/null +++ b/tensorflow/stream_executor/cuda/cuda_driver_test.cc @@ -0,0 +1,76 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#if GOOGLE_CUDA +#include "tensorflow/stream_executor/cuda/cuda_driver.h" + +#include "absl/memory/memory.h" +#include "third_party/gpus/cuda/include/cuda_runtime_api.h" +#include "tensorflow/core/platform/test.h" + +namespace stream_executor { +namespace gpu { + +void CheckCuda(CUresult result, const char* file, int line) { + if (result == CUDA_SUCCESS) { + return; + } + const char* name; + cuGetErrorName(result, &name); + const char* message; + cuGetErrorString(result, &message); + LOG(FATAL) << file << "(" << line << "): " << name << ", " << message; +} + +void CheckCuda(cudaError_t result, const char* file, int line) { + if (result == cudaSuccess) { + return; + } + const char* name = cudaGetErrorName(result); + const char* message = cudaGetErrorString(result); + LOG(FATAL) << file << "(" << line << "): " << name << ", " << message; +} + +#define CHECK_CUDA(result) CheckCuda(result, __FILE__, __LINE__) + +TEST(CudaDriverTest, ScopedActivateContextTest) { + CHECK_CUDA(cuInit(0)); + CUdevice device; + CHECK_CUDA(cuDeviceGet(&device, 0)); + CUcontext context0, context1; + CHECK_CUDA(cuCtxCreate(&context0, 0, device)); + CHECK_CUDA(cuCtxCreate(&context1, 0, device)); + GpuContext se_context1(context1, /*id=*/101); + { + ScopedActivateContext scope(&se_context1); + CUcontext c; + CHECK_CUDA(cuCtxGetCurrent(&c)); + EXPECT_EQ(c, context1); + } + CHECK_CUDA(cuCtxSetCurrent(context0)); + // ScopedActivateContext must correctly set the CUDA context even if some + // other code changes the context between the two scopes. + { + ScopedActivateContext scope(&se_context1); + CUcontext c; + CHECK_CUDA(cuCtxGetCurrent(&c)); + EXPECT_EQ(c, context1); + } +} + +} // namespace gpu +} // namespace stream_executor + +#endif // GOOGLE_CUDA From ba06a75f1661be297f741feba705808d97612717 Mon Sep 17 00:00:00 2001 From: Jay Shi Date: Mon, 10 Aug 2020 15:40:19 -0700 Subject: [PATCH 0780/1017] [tf.data] Add unit test to test the environment variable settings in `ObtainOptimizations` function. PiperOrigin-RevId: 325903283 Change-Id: I45742135cf4afe7c45c29afd9585b0445c07dd0a --- tensorflow/core/kernels/data/dataset_utils.cc | 30 +++- tensorflow/core/kernels/data/dataset_utils.h | 7 +- .../core/kernels/data/dataset_utils_test.cc | 141 ++++++++++++++---- .../core/kernels/data/optimize_dataset_op.cc | 59 +++----- 4 files changed, 160 insertions(+), 77 deletions(-) diff --git a/tensorflow/core/kernels/data/dataset_utils.cc b/tensorflow/core/kernels/data/dataset_utils.cc index 66de482467d..d79288b86d3 100644 --- a/tensorflow/core/kernels/data/dataset_utils.cc +++ b/tensorflow/core/kernels/data/dataset_utils.cc @@ -906,13 +906,38 @@ bool MatchesAnyVersionRE(StringPiece op_prefix, StringPiece op_to_match) { } std::vector SelectOptimizations( - const string& job_name, const string& opt_ins_raw, - const string& opt_outs_raw, + const string& job_name, const absl::flat_hash_map& live_experiments, const std::vector& optimizations_enabled, const std::vector& optimizations_disabled, const std::vector& optimizations_default, std::function hash_func) { + std::vector optimizations; + if (job_name.empty()) { + // If `job_name` is empty, apply the enabled and default optimizations + // directly. + optimizations.insert(optimizations.end(), optimizations_enabled.begin(), + optimizations_enabled.end()); + optimizations.insert(optimizations.end(), optimizations_default.begin(), + optimizations_default.end()); + return optimizations; + } + + // If `job_name` is non-empty, we determine which optimizations to apply to + // this job based on the enable/disable settings from tf.data.Options, the + // opt in/out settings from environment variables, and rollout condition from + // `live_experiments`. + const char* opt_ins_raw_cs = std::getenv("TF_DATA_EXPERIMENT_OPT_IN"); + const char* opt_outs_raw_cs = std::getenv("TF_DATA_EXPERIMENT_OPT_OUT"); + string opt_ins_raw; + if (opt_ins_raw_cs != nullptr) { + opt_ins_raw = string(opt_ins_raw_cs); + } + string opt_outs_raw; + if (opt_outs_raw_cs != nullptr) { + opt_outs_raw = string(opt_outs_raw_cs); + } + // Creates a set of optimizations. absl::flat_hash_set optimizations_set; @@ -1018,7 +1043,6 @@ std::vector SelectOptimizations( } } - std::vector optimizations; optimizations.insert(optimizations.end(), optimizations_set.begin(), optimizations_set.end()); return optimizations; diff --git a/tensorflow/core/kernels/data/dataset_utils.h b/tensorflow/core/kernels/data/dataset_utils.h index 0fe3618f34b..7f9ea923b98 100644 --- a/tensorflow/core/kernels/data/dataset_utils.h +++ b/tensorflow/core/kernels/data/dataset_utils.h @@ -304,12 +304,11 @@ class DummyResourceOp : public OpKernel { // MatchesAnyVersionRE("PaddedBatchDataset", "BatchDataset") == false bool MatchesAnyVersionRE(StringPiece op_prefix, StringPiece op_to_match); -// Based on `optimizations_enabled`, `optimizations_disabled`, and -// `optimizations_disabled`, returns the list of optimizations that will be +// Based on `job_name`, `optimizations_enabled`, `optimizations_disabled` and +// `optimizations_default`, returns the list of optimizations that will be // applied. std::vector SelectOptimizations( - const string& job_name, const string& opt_ins_raw, - const string& opt_outs_raw, + const string& job_name, const absl::flat_hash_map& live_experiments, const std::vector& optimizations_enabled, const std::vector& optimizations_disabled, diff --git a/tensorflow/core/kernels/data/dataset_utils_test.cc b/tensorflow/core/kernels/data/dataset_utils_test.cc index a1f624faeb6..85019e3f8da 100644 --- a/tensorflow/core/kernels/data/dataset_utils_test.cc +++ b/tensorflow/core/kernels/data/dataset_utils_test.cc @@ -1138,18 +1138,15 @@ class SelectOptimizationsHashTest : public ::testing::TestWithParam {}; TEST_P(SelectOptimizationsHashTest, DatasetUtils) { const uint64 hash_result = GetParam(); string job_name = "job"; - const string opt_ins_raw = ""; - const string opt_outs_raw = ""; auto hash_func = [hash_result](const string& str) { return hash_result; }; absl::flat_hash_map live_experiments = { {"exp1", 0}, {"exp2", 20}, {"exp3", 33}, {"exp4", 45}, {"exp5", 67}, {"exp6", 88}, {"exp7", 100}}; std::vector optimizations_enabled, optimizations_disabled, optimizations_default; - std::vector optimizations = - SelectOptimizations(job_name, opt_ins_raw, opt_outs_raw, live_experiments, - optimizations_enabled, optimizations_disabled, - optimizations_default, hash_func); + std::vector optimizations = SelectOptimizations( + job_name, live_experiments, optimizations_enabled, optimizations_disabled, + optimizations_default, hash_func); int tested_times = 0; switch (hash_result) { @@ -1182,48 +1179,60 @@ class SelectOptimizationsOptTest : public ::testing::TestWithParam> {}; TEST_P(SelectOptimizationsOptTest, DatasetUtils) { + const string opt_ins = std::get<0>(GetParam()); + const string opt_outs = std::get<1>(GetParam()); + if (!opt_ins.empty()) { + setenv("TF_DATA_EXPERIMENT_OPT_IN", opt_ins.c_str(), 1); + } + if (!opt_outs.empty()) { + setenv("TF_DATA_EXPERIMENT_OPT_OUT", opt_outs.c_str(), 1); + } string job_name = "job"; - const string opt_ins_raw = std::get<0>(GetParam()); - const string opt_outs_raw = std::get<1>(GetParam()); auto hash_func = [](const string& str) { return 50; }; absl::flat_hash_map live_experiments = { {"exp1", 0}, {"exp2", 25}, {"exp3", 50}, {"exp4", 75}, {"exp5", 100}}; std::vector optimizations_enabled, optimizations_disabled, optimizations_default; - std::vector optimizations = - SelectOptimizations(job_name, opt_ins_raw, opt_outs_raw, live_experiments, - optimizations_enabled, optimizations_disabled, - optimizations_default, hash_func); + std::vector optimizations = SelectOptimizations( + job_name, live_experiments, optimizations_enabled, optimizations_disabled, + optimizations_default, hash_func); int tested_times = 0; - if (opt_outs_raw == "all") { + if (opt_outs == "all") { EXPECT_THAT(optimizations, UnorderedElementsAre()); tested_times++; - } else if (opt_outs_raw.empty()) { - if (opt_ins_raw == "all") { + } else if (opt_outs.empty()) { + if (opt_ins == "all") { EXPECT_THAT(optimizations, UnorderedElementsAre("exp1", "exp2", "exp3", "exp4", "exp5")); tested_times++; - } else if (opt_ins_raw.empty()) { + } else if (opt_ins.empty()) { EXPECT_THAT(optimizations, UnorderedElementsAre("exp4", "exp5")); tested_times++; - } else if (opt_ins_raw == "exp2,exp4") { + } else if (opt_ins == "exp2,exp4") { EXPECT_THAT(optimizations, UnorderedElementsAre("exp2", "exp4", "exp5")); tested_times++; } - } else if (opt_outs_raw == "exp1,exp5") { - if (opt_ins_raw == "all") { + } else if (opt_outs == "exp1,exp5") { + if (opt_ins == "all") { EXPECT_THAT(optimizations, UnorderedElementsAre("exp2", "exp3", "exp4")); tested_times++; - } else if (opt_ins_raw.empty()) { + } else if (opt_ins.empty()) { EXPECT_THAT(optimizations, UnorderedElementsAre("exp4")); tested_times++; - } else if (opt_ins_raw == "exp2,exp4") { + } else if (opt_ins == "exp2,exp4") { EXPECT_THAT(optimizations, UnorderedElementsAre("exp2", "exp4")); tested_times++; } } EXPECT_EQ(tested_times, 1); + + if (!opt_ins.empty()) { + unsetenv("TF_DATA_EXPERIMENT_OPT_IN"); + } + if (!opt_outs.empty()) { + unsetenv("TF_DATA_EXPERIMENT_OPT_OUT"); + } } INSTANTIATE_TEST_SUITE_P( @@ -1235,10 +1244,16 @@ class SelectOptimizationsConflictTest : public ::testing::TestWithParam> {}; TEST_P(SelectOptimizationsConflictTest, DatasetUtils) { - string job_name = "job"; - const string opt_ins_raw = std::get<0>(GetParam()); - const string opt_outs_raw = std::get<1>(GetParam()); + const string opt_ins = std::get<0>(GetParam()); + const string opt_outs = std::get<1>(GetParam()); const uint64 hash_result = std::get<2>(GetParam()); + if (!opt_ins.empty()) { + setenv("TF_DATA_EXPERIMENT_OPT_IN", opt_ins.c_str(), 1); + } + if (!opt_outs.empty()) { + setenv("TF_DATA_EXPERIMENT_OPT_OUT", opt_outs.c_str(), 1); + } + string job_name = "job"; auto hash_func = [hash_result](const string& str) { return hash_result; }; absl::flat_hash_map live_experiments = { {"exp1", 20}, {"exp2", 30}, {"exp3", 40}, @@ -1246,21 +1261,27 @@ TEST_P(SelectOptimizationsConflictTest, DatasetUtils) { std::vector optimizations_enabled = {"exp1", "exp4"}, optimizations_disabled = {"exp2", "exp5"}, optimizations_default = {"exp3", "exp6"}; - std::vector optimizations = - SelectOptimizations(job_name, opt_ins_raw, opt_outs_raw, live_experiments, - optimizations_enabled, optimizations_disabled, - optimizations_default, hash_func); + std::vector optimizations = SelectOptimizations( + job_name, live_experiments, optimizations_enabled, optimizations_disabled, + optimizations_default, hash_func); int tested_times = 0; - if (opt_outs_raw.empty()) { + if (opt_outs.empty()) { EXPECT_THAT(optimizations, UnorderedElementsAre("exp1", "exp3", "exp4", "exp6")); tested_times++; - } else if (opt_outs_raw == "exp1,exp3") { + } else if (opt_outs == "exp1,exp3") { EXPECT_THAT(optimizations, UnorderedElementsAre("exp1", "exp4", "exp6")); tested_times++; } EXPECT_EQ(tested_times, 1); + + if (!opt_ins.empty()) { + unsetenv("TF_DATA_EXPERIMENT_OPT_IN"); + } + if (!opt_outs.empty()) { + unsetenv("TF_DATA_EXPERIMENT_OPT_OUT"); + } } INSTANTIATE_TEST_SUITE_P(Test, SelectOptimizationsConflictTest, @@ -1268,6 +1289,66 @@ INSTANTIATE_TEST_SUITE_P(Test, SelectOptimizationsConflictTest, ::testing::Values("", "exp1,exp3"), ::testing::Values(10, 50, 90))); +class SelectOptimizationsJobTest + : public ::testing::TestWithParam> {}; + +TEST_P(SelectOptimizationsJobTest, DatasetUtils) { + const string job_name = std::get<0>(GetParam()); + const string opt_ins = std::get<1>(GetParam()); + const string opt_outs = std::get<2>(GetParam()); + if (!opt_ins.empty()) { + setenv("TF_DATA_EXPERIMENT_OPT_IN", opt_ins.c_str(), 1); + } + if (!opt_outs.empty()) { + setenv("TF_DATA_EXPERIMENT_OPT_OUT", opt_outs.c_str(), 1); + } + std::vector optimizations_enabled = {"exp4"}, optimizations_disabled, + optimizations_default = {"exp2"}; + absl::flat_hash_map live_experiments = { + {"exp1", 0}, {"exp2", 100}, {"exp3", 100}}; + auto hash_func = [](const string& str) { return Hash64(str); }; + std::vector optimizations = SelectOptimizations( + job_name, live_experiments, optimizations_enabled, optimizations_disabled, + optimizations_default, hash_func); + + int tested_times = 0; + if (job_name.empty()) { + EXPECT_THAT(optimizations, UnorderedElementsAre("exp2", "exp4")); + tested_times++; + } else if (opt_ins.empty()) { + if (opt_outs.empty()) { + EXPECT_THAT(optimizations, UnorderedElementsAre("exp2", "exp3", "exp4")); + tested_times++; + } else if (opt_outs == "exp2,exp3") { + EXPECT_THAT(optimizations, UnorderedElementsAre("exp4")); + tested_times++; + } + } else if (opt_ins == "exp1") { + if (opt_outs.empty()) { + EXPECT_THAT(optimizations, + UnorderedElementsAre("exp1", "exp2", "exp3", "exp4")); + tested_times++; + } else if (opt_outs == "exp2,exp3") { + EXPECT_THAT(optimizations, UnorderedElementsAre("exp1", "exp4")); + tested_times++; + } + } + EXPECT_EQ(tested_times, 1); + + if (!opt_ins.empty()) { + unsetenv("TF_DATA_EXPERIMENT_OPT_IN"); + } + if (!opt_outs.empty()) { + unsetenv("TF_DATA_EXPERIMENT_OPT_OUT"); + } +} + +INSTANTIATE_TEST_SUITE_P(Test, SelectOptimizationsJobTest, + ::testing::Combine(::testing::Values("", "job"), + ::testing::Values("", "exp1"), + ::testing::Values("", + "exp2,exp3"))); + } // namespace } // namespace data } // namespace tensorflow diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc index 74468e71241..f1fa96d9ac3 100644 --- a/tensorflow/core/kernels/data/optimize_dataset_op.cc +++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc @@ -80,48 +80,27 @@ void OptimizeDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input, &optimizations_default)); string job_name = port::JobName(); - if (job_name.empty()) { - // If `job_name` is empty, apply the enabled and default optimizations - // directly. - optimizations.insert(optimizations.end(), optimizations_enabled.begin(), - optimizations_enabled.end()); - optimizations.insert(optimizations.end(), optimizations_default.begin(), - optimizations_default.end()); - } else { - // The map that stores the experiment names and for how much percentage - // of the jobs, the experiments will be randomly turned on. - // - // This is currently empty; we have no live experiments yet. - absl::flat_hash_map live_experiments; + // The map that stores the experiment names and for how much percentage + // of the jobs, the experiments will be randomly turned on. + // + // This is currently empty; we have no live experiments yet. + absl::flat_hash_map live_experiments; + auto hash_func = [](const string& str) { return Hash64(str); }; + optimizations = SelectOptimizations( + job_name, live_experiments, optimizations_enabled, + optimizations_disabled, optimizations_default, hash_func); - const char* opt_ins_raw_cs = std::getenv("TF_DATA_EXPERIMENT_OPT_IN"); - const char* opt_outs_raw_cs = std::getenv("TF_DATA_EXPERIMENT_OPT_OUT"); - string opt_ins_raw; - if (opt_ins_raw_cs != nullptr) { - opt_ins_raw = string(opt_ins_raw_cs); - } - string opt_outs_raw; - if (opt_outs_raw_cs != nullptr) { - opt_outs_raw = string(opt_outs_raw_cs); - } - auto hash_func = [](const string& str) { return Hash64(str); }; - optimizations = SelectOptimizations( - job_name, opt_ins_raw, opt_outs_raw, live_experiments, - optimizations_enabled, optimizations_disabled, optimizations_default, - hash_func); + // Log and record the experiments that will be applied. + if (!job_name.empty() && !live_experiments.empty()) { + VLOG(1) << "The input pipeline is subject to tf.data experiment. " + "Please see `go/tf-data-experiments` for more details."; - // Log the experiments that will be applied. - if (!live_experiments.empty() && VLOG_IS_ON(1)) { - VLOG(1) << "The input pipeline is subject to tf.data experiment. " - "Please see `go/tf-data-experiments` for more details."; - - for (auto& pair : live_experiments) { - string experiment = pair.first; - if (std::find(optimizations.begin(), optimizations.end(), - experiment) != optimizations.end()) { - VLOG(1) << "The experiment \"" << experiment << "\" is applied."; - metrics::RecordTFDataExperiment(experiment); - } + for (auto& pair : live_experiments) { + string experiment = pair.first; + if (std::find(optimizations.begin(), optimizations.end(), experiment) != + optimizations.end()) { + VLOG(1) << "The experiment \"" << experiment << "\" is applied."; + metrics::RecordTFDataExperiment(experiment); } } } From 242632c70980bd5426d3ccd331cdf82f950b5c71 Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Mon, 10 Aug 2020 00:08:31 -0700 Subject: [PATCH 0781/1017] Canonicalize ShapeNOp with partial static input shape Simplify variable --- .../mlir/tensorflow/ir/tf_generated_ops.td | 2 ++ .../compiler/mlir/tensorflow/ir/tf_ops_n_z.cc | 35 ++++++++++++++++++- .../mlir/tensorflow/tests/constant-fold.mlir | 14 ++++---- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index 081903d13cf..56baed107bd 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -8712,6 +8712,8 @@ This operation returns N 1-D integer tensors representing shape of `input[i]s`. return Verify(*this); }]; + let hasCanonicalizer = 1; + let hasFolder = 1; } diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc index ffedcb47f7e..f1a4415902d 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc @@ -947,9 +947,42 @@ LogicalResult ShapeNOp::fold(ArrayRef operands, return success(); } -// TODO(hinsu): Add canonicalization pattern for ShapeN ops that don't have all +namespace { +// Canonicalization pattern for ShapeNOp that don't have all // static input shapes. Replacing output values corresponding to static input // types may enable optimizations in users of the values. +class ShapeNPartialStaticInputShape : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(ShapeNOp op, + PatternRewriter &rewriter) const override { + if (op.getNumOperands() == 0) return success(); + int width = op.getType(0) + .cast() + .getElementType() + .getIntOrFloatBitWidth(); + BoolAttr use32Bit = BoolAttr::get(width == 32, op.getContext()); + + SmallVector results; + for (Value input : op.getOperands()) { + Value shape; + if (OpFoldResult result = ConvertShapeToAttr(input.getType(), width)) { + shape = + rewriter.create(op.getLoc(), result.get()); + } else { + shape = rewriter.create(op.getLoc(), input, use32Bit); + } + results.push_back(shape); + } + rewriter.replaceOp(op, results); + return success(); + } +}; +} // namespace + +void ShapeNOp::getCanonicalizationPatterns(OwningRewritePatternList &results, + MLIRContext *context) { + results.insert(context); +} //===----------------------------------------------------------------------===// // SizeOp diff --git a/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir b/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir index b86815dbe57..ce80dab266d 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir @@ -89,16 +89,18 @@ func @testEmptybf16() -> (tensor<5xbf16>) { } // CHECK-LABEL: func @testShapeN -func @testShapeN(%arg0: tensor, %arg1: tensor<1x32x32x16xf32>, %arg2: tensor<*xf32>) -> (tensor<0xi64>, tensor<4xi64>, tensor<4xi64>, tensor) { +func @testShapeN(%arg0: tensor, %arg1: tensor<1x32x32x16xf32>, %arg2: tensor<*xf32>, %arg3: tensor<1x32x32xf32>) -> (tensor<0xi64>, tensor<4xi64>, tensor<4xi64>, tensor, tensor<3xi64>) { - // CHECK: "tf.Const"() {value = dense<> : tensor<0xi64> - // CHECK: "tf.Const"() {value = dense<[1, 32, 32, 16]> : tensor<4xi64>} + // CHECK: %[[SHAPE0:.*]] = "tf.Const"() {value = dense<> : tensor<0xi64>} + // CHECK: %[[SHAPE1:.*]] = "tf.Const"() {value = dense<[1, 32, 32, 16]> : tensor<4xi64>} %0:2 = "tf.ShapeN"(%arg0, %arg1) : (tensor, tensor<1x32x32x16xf32>) -> (tensor<0xi64>, tensor<4xi64>) - // CHECK: tf.ShapeN - %1:2 = "tf.ShapeN"(%arg1, %arg2) : (tensor<1x32x32x16xf32>, tensor<*xf32>) -> (tensor<4xi64>, tensor) + // CHECK: %[[SHAPE3:.*]] = "tf.Const"() {value = dense<[1, 32, 32]> : tensor<3xi64>} + // CHECK: %[[SHAPE2:.*]] = "tf.Shape"(%arg2) : (tensor<*xf32>) -> tensor + %1:3 = "tf.ShapeN"(%arg1, %arg2, %arg3) : (tensor<1x32x32x16xf32>, tensor<*xf32>, tensor<1x32x32xf32>) -> (tensor<4xi64>, tensor, tensor<3xi64>) - return %0#0, %0#1, %1#0, %1#1 : tensor<0xi64>, tensor<4xi64>, tensor<4xi64>, tensor + // CHECK: return %[[SHAPE0]], %[[SHAPE1]], %[[SHAPE1]], %[[SHAPE2]], %[[SHAPE3]] + return %0#0, %0#1, %1#0, %1#1, %1#2 : tensor<0xi64>, tensor<4xi64>, tensor<4xi64>, tensor, tensor<3xi64> } // CHECK-LABEL: func @testLeakyRelu From 7eb63dd2642842d622ec4d5a2656e4f14014c263 Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Mon, 10 Aug 2020 14:34:43 -0700 Subject: [PATCH 0782/1017] Create a ShapeNOp instead of multiple ShapeOps Use getType directly Create ShapeOp instead of ShapeNOp when there is only one dynamic input Run clang-format Use plural variable name --- .../compiler/mlir/tensorflow/ir/tf_ops_n_z.cc | 41 +++++++++++++++---- .../mlir/tensorflow/tests/constant-fold.mlir | 30 +++++++++++--- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc index f1a4415902d..d726894c6ba 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc @@ -955,24 +955,47 @@ class ShapeNPartialStaticInputShape : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(ShapeNOp op, PatternRewriter &rewriter) const override { + // ShapeNOp::fold handles this case. if (op.getNumOperands() == 0) return success(); int width = op.getType(0) .cast() .getElementType() .getIntOrFloatBitWidth(); - BoolAttr use32Bit = BoolAttr::get(width == 32, op.getContext()); - SmallVector results; - for (Value input : op.getOperands()) { - Value shape; - if (OpFoldResult result = ConvertShapeToAttr(input.getType(), width)) { - shape = - rewriter.create(op.getLoc(), result.get()); + SmallVector results(op.getNumOperands()); + SmallVector dynamic_indices; + SmallVector dynamic_inputs; + SmallVector result_types; + for (auto e : llvm::enumerate(op.getOperands())) { + if (Attribute result = ConvertShapeToAttr(e.value().getType(), width)) { + results[e.index()] = + rewriter.create(op.getLoc(), result); } else { - shape = rewriter.create(op.getLoc(), input, use32Bit); + dynamic_indices.push_back(e.index()); + dynamic_inputs.push_back(e.value()); + result_types.push_back(op.getType(e.index())); } - results.push_back(shape); } + + if (dynamic_inputs.size() == op.getNumOperands()) { + // Cannot canonicalize ShapeN if all inputs are dynamic. + return failure(); + } + + // Create a ShapeOp when there is only one dynamic input. + // Or create a ShapeNOp when there are two or more dynamic inputs. + if (dynamic_inputs.size() == 1) { + results[dynamic_indices[0]] = rewriter.create( + op.getLoc(), result_types[0], dynamic_inputs[0]); + } else if (dynamic_inputs.size() >= 2) { + auto dynamic_shape_n = rewriter.create( + op.getLoc(), result_types, dynamic_inputs); + for (auto index_result : + llvm::zip(dynamic_indices, dynamic_shape_n.getResults())) { + results[std::get<0>(index_result)] = std::get<1>(index_result); + } + } + rewriter.replaceOp(op, results); return success(); } diff --git a/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir b/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir index ce80dab266d..0e0ba409f76 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir @@ -89,18 +89,36 @@ func @testEmptybf16() -> (tensor<5xbf16>) { } // CHECK-LABEL: func @testShapeN -func @testShapeN(%arg0: tensor, %arg1: tensor<1x32x32x16xf32>, %arg2: tensor<*xf32>, %arg3: tensor<1x32x32xf32>) -> (tensor<0xi64>, tensor<4xi64>, tensor<4xi64>, tensor, tensor<3xi64>) { +func @testShapeN(%arg0: tensor, %arg1: tensor<1x32x32x16xf32>) -> (tensor<0xi64>, tensor<4xi64>) { // CHECK: %[[SHAPE0:.*]] = "tf.Const"() {value = dense<> : tensor<0xi64>} // CHECK: %[[SHAPE1:.*]] = "tf.Const"() {value = dense<[1, 32, 32, 16]> : tensor<4xi64>} %0:2 = "tf.ShapeN"(%arg0, %arg1) : (tensor, tensor<1x32x32x16xf32>) -> (tensor<0xi64>, tensor<4xi64>) - // CHECK: %[[SHAPE3:.*]] = "tf.Const"() {value = dense<[1, 32, 32]> : tensor<3xi64>} - // CHECK: %[[SHAPE2:.*]] = "tf.Shape"(%arg2) : (tensor<*xf32>) -> tensor - %1:3 = "tf.ShapeN"(%arg1, %arg2, %arg3) : (tensor<1x32x32x16xf32>, tensor<*xf32>, tensor<1x32x32xf32>) -> (tensor<4xi64>, tensor, tensor<3xi64>) + // CHECK: return %[[SHAPE0]], %[[SHAPE1]] + return %0#0, %0#1 : tensor<0xi64>, tensor<4xi64> +} - // CHECK: return %[[SHAPE0]], %[[SHAPE1]], %[[SHAPE1]], %[[SHAPE2]], %[[SHAPE3]] - return %0#0, %0#1, %1#0, %1#1, %1#2 : tensor<0xi64>, tensor<4xi64>, tensor<4xi64>, tensor, tensor<3xi64> +// CHECK-LABEL: func @testShapeNPartialStatic +func @testShapeNPartialStatic(%arg0: tensor, %arg1: tensor<2x?x3xf32>, %arg2: tensor<1x32x32x16xf32>, %arg3: tensor<*xf32>) -> (tensor<0xi64>, tensor<3xi64>, tensor<4xi64>, tensor) { + // CHECK: %[[SHAPE0:.*]] = "tf.Const"() {value = dense<> : tensor<0xi64>} + // CHECK: %[[SHAPE2:.*]] = "tf.Const"() {value = dense<[1, 32, 32, 16]> : tensor<4xi64>} + // CHECK: %[[SHAPE13:.*]]:2 = "tf.ShapeN"(%arg1, %arg3) : (tensor<2x?x3xf32>, tensor<*xf32>) -> (tensor<3xi64>, tensor) + %0:4 = "tf.ShapeN"(%arg0, %arg1, %arg2, %arg3) : (tensor, tensor<2x?x3xf32>, tensor<1x32x32x16xf32>, tensor<*xf32>) -> (tensor<0xi64>, tensor<3xi64>, tensor<4xi64>, tensor) + + // CHECK: return %[[SHAPE0]], %[[SHAPE13]]#0, %[[SHAPE2]], %[[SHAPE13]]#1 + return %0#0, %0#1, %0#2, %0#3 : tensor<0xi64>, tensor<3xi64>, tensor<4xi64>, tensor +} + +// CHECK-LABEL: func @testShapeNOneDynamic +func @testShapeNOneDynamic(%arg0: tensor, %arg1: tensor<1x32x32x16xf32>, %arg2: tensor<*xf32>) -> (tensor<0xi64>, tensor<4xi64>, tensor) { + // CHECK: %[[SHAPE0:.*]] = "tf.Const"() {value = dense<> : tensor<0xi64>} + // CHECK: %[[SHAPE1:.*]] = "tf.Const"() {value = dense<[1, 32, 32, 16]> : tensor<4xi64>} + // CHECK: %[[SHAPE2:.*]] = "tf.Shape"(%arg2) : (tensor<*xf32>) -> tensor + %0:3 = "tf.ShapeN"(%arg0, %arg1, %arg2) : (tensor, tensor<1x32x32x16xf32>, tensor<*xf32>) -> (tensor<0xi64>, tensor<4xi64>, tensor) + + // CHECK: return %[[SHAPE0]], %[[SHAPE1]], %[[SHAPE2]] + return %0#0, %0#1, %0#2 : tensor<0xi64>, tensor<4xi64>, tensor } // CHECK-LABEL: func @testLeakyRelu From b9c5f8084357a5d4072c7fa8404b060150a3239f Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Mon, 10 Aug 2020 15:27:45 -0700 Subject: [PATCH 0783/1017] Separate ShapeNToShape canonicalization pattern Remove folder Add tests Use getElementTypeOrSelf utility --- .../mlir/tensorflow/ir/tf_generated_ops.td | 2 - .../compiler/mlir/tensorflow/ir/tf_ops_n_z.cc | 52 +++++++++---------- .../mlir/tensorflow/tests/constant-fold.mlir | 9 ++++ 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index 56baed107bd..ce69118d7f8 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -8713,8 +8713,6 @@ This operation returns N 1-D integer tensors representing shape of `input[i]s`. }]; let hasCanonicalizer = 1; - - let hasFolder = 1; } def TF_ShardedFilenameOp : TF_Op<"ShardedFilename", [NoSideEffect]> { diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc index d726894c6ba..c5da5e0b07b 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops_n_z.cc @@ -932,21 +932,6 @@ static LogicalResult Verify(ShapeNOp op) { return success(); } -LogicalResult ShapeNOp::fold(ArrayRef operands, - SmallVectorImpl &results) { - if (getNumOperands() == 0) return success(); - int width = - getType(0).cast().getElementType().getIntOrFloatBitWidth(); - - for (Type input_ty : getOperandTypes()) { - OpFoldResult result = ConvertShapeToAttr(input_ty, width); - if (!result) return failure(); - - results.push_back(result); - } - return success(); -} - namespace { // Canonicalization pattern for ShapeNOp that don't have all // static input shapes. Replacing output values corresponding to static input @@ -955,12 +940,12 @@ class ShapeNPartialStaticInputShape : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; LogicalResult matchAndRewrite(ShapeNOp op, PatternRewriter &rewriter) const override { - // ShapeNOp::fold handles this case. - if (op.getNumOperands() == 0) return success(); - int width = op.getType(0) - .cast() - .getElementType() - .getIntOrFloatBitWidth(); + if (op.getNumOperands() == 0) { + rewriter.eraseOp(op); + return success(); + } + + int width = getElementTypeOrSelf(op.getType(0)).getIntOrFloatBitWidth(); SmallVector results(op.getNumOperands()); SmallVector dynamic_indices; @@ -982,12 +967,8 @@ class ShapeNPartialStaticInputShape : public OpRewritePattern { return failure(); } - // Create a ShapeOp when there is only one dynamic input. - // Or create a ShapeNOp when there are two or more dynamic inputs. - if (dynamic_inputs.size() == 1) { - results[dynamic_indices[0]] = rewriter.create( - op.getLoc(), result_types[0], dynamic_inputs[0]); - } else if (dynamic_inputs.size() >= 2) { + // Create a ShapeNOp for all dynamic inputs. + if (!dynamic_inputs.empty()) { auto dynamic_shape_n = rewriter.create( op.getLoc(), result_types, dynamic_inputs); for (auto index_result : @@ -1000,11 +981,26 @@ class ShapeNPartialStaticInputShape : public OpRewritePattern { return success(); } }; + +// Canonicalize ShapeNOp to ShapeOp if there is only one operand. +class ShapeNToShape : public OpRewritePattern { + using OpRewritePattern::OpRewritePattern; + LogicalResult matchAndRewrite(ShapeNOp op, + PatternRewriter &rewriter) const override { + if (op.getNumOperands() != 1) { + return failure(); + } + auto shape = rewriter.create( + op.getLoc(), op.getType(0), op.getOperand(0)); + rewriter.replaceOp(op, {shape}); + return success(); + } +}; } // namespace void ShapeNOp::getCanonicalizationPatterns(OwningRewritePatternList &results, MLIRContext *context) { - results.insert(context); + results.insert(context); } //===----------------------------------------------------------------------===// diff --git a/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir b/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir index 0e0ba409f76..f114d1724f2 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/constant-fold.mlir @@ -121,6 +121,15 @@ func @testShapeNOneDynamic(%arg0: tensor, %arg1: tensor<1x32x32x16xf32>, %a return %0#0, %0#1, %0#2 : tensor<0xi64>, tensor<4xi64>, tensor } +// CHECK-LABEL: func @testShapeNToShape +func @testShapeNToShape(%arg0: tensor<*xf32>) -> tensor { + // CHECK: %[[SHAPE0:.*]] = "tf.Shape"(%arg0) : (tensor<*xf32>) -> tensor + %0:1 = "tf.ShapeN"(%arg0) : (tensor<*xf32>) -> tensor + + // CHECK: return %[[SHAPE0]] + return %0#0 : tensor +} + // CHECK-LABEL: func @testLeakyRelu func @testLeakyRelu(%arg0 : tensor<16xf32>) -> (tensor<16xf32>, tensor, tensor, tensor<16xf32>) { %pos = constant dense<5.0> : tensor From 8daae0f3dab01119fe8dbf994852d20a80bace39 Mon Sep 17 00:00:00 2001 From: Mihai Maruseac Date: Mon, 10 Aug 2020 16:04:30 -0700 Subject: [PATCH 0784/1017] TEsting CUDA 11 nightly build PiperOrigin-RevId: 325907860 Change-Id: Ieff7d7964818490300c019143afde201b6491156 --- .../tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh index 8a0796723b2..3e91bf787a9 100644 --- a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py35_nonpip.sh @@ -46,7 +46,8 @@ source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh tag_filters="gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py35" set +e -bazel test --config=cuda --config=opt \ +ls /usr/include/cud* +bazel test --config=cuda --config=opt -s \ --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11:toolchain \ --linkopt=-lrt \ --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ @@ -54,7 +55,7 @@ bazel test --config=cuda --config=opt \ --test_tag_filters=${tag_filters} \ --build_tag_filters=${tag_filters} \ --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ - --test_output=errors --verbose_failures=true --keep_going \ + --test_output=errors --verbose_failures=true \ --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... test_xml_summary_exit From 14f6926300639fb589a10a750b44b03d32997765 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 16:05:20 -0700 Subject: [PATCH 0785/1017] Implement extraction of outside compilation of ops nested inside tf.IfRegion op. PiperOrigin-RevId: 325908017 Change-Id: Ib45dce2d7e9c61a9f1ca9e9d90663f9b902d44c4 --- .../tpu_extract_outside_compilation.mlir | 232 ++++++++++++++ .../tpu_extract_outside_compilation.cc | 289 +++++++++++++++--- 2 files changed, 474 insertions(+), 47 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tpu_extract_outside_compilation.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tpu_extract_outside_compilation.mlir index 732e34fce90..1f516a25824 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/tpu_extract_outside_compilation.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/tpu_extract_outside_compilation.mlir @@ -456,4 +456,236 @@ module attributes {tf.versions = {producer = 888 : i32}, tf.devices = ["/job:wor } return %1 : tensor } + + // Tests extraction of a single outside compiled cluster inside a tf.IfRegion op. + + // CHECK-LABEL: func @outside_compiled_ops_inside_tf_if + func @outside_compiled_ops_inside_tf_if(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + + // CHECK: %[[REPLICATE:[0-9]*]]:2 = tf_device.replicate + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]] = "tf_device.parallel_execute" + // CHECK-NEXT: "tf_device.launch" + // CHECK-NEXT: %[[PLACEHOLDER_KEY:[0-9]*]] = "tf._TPUCompileMlirPlaceholderProgramKey"() + // CHECK-NEXT: %[[PREDICATE_RECV_OUTPUT:[0-9]*]] = "tf._XlaRecvAtHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "if_predicate_channel_cluster1_0" + // CHECK-NEXT: tf.IfRegion"(%[[PREDICATE_RECV_OUTPUT]]) + // CHECK-NEXT: %[[ARG_RECV_OUTPUT:[0-9]*]]:2 = "tf._XlaRecvAtHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "host_compute_channel_cluster1_args" + // CHECK: "tf.D"(%[[ARG_RECV_OUTPUT]]#0, %[[ARG_RECV_OUTPUT]]#1) + // CHECK: "tf._XlaSendFromHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "host_compute_channel_cluster1_retvals" + // CHECK-NEXT: "tf.Yield"() : () -> () + // CHECK: "tf_device.cluster" + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + // CHECK: %[[B_OUTPUT:[0-9]*]] = "tf.B" + // CHECK: %[[G_OUTPUT:[0-9]*]] = "tf.G" + // CHECK: "tf.XlaSendToHost"(%6) {key = "if_predicate_channel_cluster1_0"} + // CHECK-NEXT: tf.IfRegion"(%[[G_OUTPUT]]) + // CHECK: "tf._XlaHostComputeMlir"(%[[B_OUTPUT]], %[[A_OUTPUT]]) + // CHECK-SAME: recv_key = "host_compute_channel_cluster1_retvals" + // CHECK-SAME: send_key = "host_compute_channel_cluster1_args" + // CHECK-SAME: tpu_core = 0 + // CHECK-NEXT: "tf.Yield"() : () -> () + %1:2 = tf_device.replicate([%0, %arg0] as %ri_0: tensor) {n = 2 : i32} { + %2 = "tf_device.cluster"() ( { + %3 = "tf.A"() : () -> (tensor) + %4 = "tf.B"() : () -> (tensor) + %6 = "tf.G"() : () -> (tensor) + + "tf.IfRegion"(%6) ({ + "tf.D"(%4, %3) {_xla_outside_compilation = "cluster1"} : (tensor, tensor) -> () + "tf.Yield"() : () -> () + }, { + "tf.Yield"() : () -> () + }) { is_stateless = false} : (tensor) -> () + + %5 = "tf.E"() : () -> tensor + tf_device.return %5 : tensor + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + tf_device.return %2 : tensor + } + + return %1 : tensor + } + + // Tests extraction of a single outside compiled cluster inside a tf.IfRegion + // op with return values. + + // CHECK-LABEL: func @outside_compiled_ops_inside_tf_if_with_return_values + func @outside_compiled_ops_inside_tf_if_with_return_values( + %arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + + // CHECK: %[[REPLICATE:[0-9]*]]:2 = tf_device.replicate + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]] = "tf_device.parallel_execute" + // CHECK-NEXT: "tf_device.launch" + // CHECK-NEXT: %[[PLACEHOLDER_KEY:[0-9]*]] = "tf._TPUCompileMlirPlaceholderProgramKey"() + // CHECK-NEXT: %[[PREDICATE_RECV_OUTPUT:[0-9]*]] = "tf._XlaRecvAtHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "if_predicate_channel_cluster1_0" + // CHECK-NEXT: tf.IfRegion"(%[[PREDICATE_RECV_OUTPUT]]) + // CHECK-NEXT: %[[ARG_RECV_OUTPUT:[0-9]*]]:2 = "tf._XlaRecvAtHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "host_compute_channel_cluster1_args" + // CHECK: %[[D_OUTPUT:[0-9]*]] = "tf.D"(%[[ARG_RECV_OUTPUT]]#0, %[[ARG_RECV_OUTPUT]]#1) + // CHECK: "tf._XlaSendFromHost"(%[[D_OUTPUT]], %[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "host_compute_channel_cluster1_retvals" + // CHECK-NEXT: "tf.Yield"() : () -> () + // CHECK: "tf_device.cluster" + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + // CHECK: %[[B_OUTPUT:[0-9]*]] = "tf.B" + // CHECK: %[[G_OUTPUT:[0-9]*]] = "tf.G" + // CHECK: "tf.XlaSendToHost"(%6) {key = "if_predicate_channel_cluster1_0"} + // CHECK-NEXT: tf.IfRegion"(%[[G_OUTPUT]]) + // CHECK: %[[HOST_COMPUTE_OUT:[0-9]*]] = "tf._XlaHostComputeMlir"(%[[B_OUTPUT]], %[[A_OUTPUT]]) + // CHECK-SAME: recv_key = "host_compute_channel_cluster1_retvals" + // CHECK-SAME: send_key = "host_compute_channel_cluster1_args" + // CHECK-SAME: tpu_core = 0 + // CHECK-NEXT: "tf.Yield"(%[[HOST_COMPUTE_OUT]]) + %1:2 = tf_device.replicate([%0, %arg0] as %ri_0: tensor) {n = 2 : i32} { + %2 = "tf_device.cluster"() ( { + %3 = "tf.A"() : () -> (tensor) + %4 = "tf.B"() : () -> (tensor) + %6 = "tf.G"() : () -> (tensor) + + "tf.IfRegion"(%6) ({ + %7 = "tf.D"(%4, %3) {_xla_outside_compilation = "cluster1"} : (tensor, tensor) -> (tensor) + "tf.Yield"(%7) : (tensor) -> () + }, { + + %8 = "tf.F"() : () -> (tensor) + "tf.Yield"(%8) : (tensor) -> () + }) { is_stateless = false} : (tensor) -> (tensor) + + %5 = "tf.E"() : () -> tensor + tf_device.return %5 : tensor + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + tf_device.return %2 : tensor + } + + return %1 : tensor + } + + // Tests extraction of a single outside compiled cluster inside a tf.IfRegion op without external inputs/outputs + + // CHECK-LABEL: func @outside_compiled_ops_inside_tf_if_without_input_outputs + func @outside_compiled_ops_inside_tf_if_without_input_outputs( + %arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[REPLICATE:[0-9]*]]:2 = tf_device.replicate + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]] = "tf_device.parallel_execute" + // CHECK-NEXT: "tf_device.launch" + // CHECK-NEXT: %[[PLACEHOLDER_KEY:[0-9]*]] = "tf._TPUCompileMlirPlaceholderProgramKey"() + // CHECK-NEXT: %[[PREDICATE_RECV_OUTPUT:[0-9]*]] = "tf._XlaRecvAtHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "if_predicate_channel_cluster1_0" + // CHECK-NEXT: tf.IfRegion"(%[[PREDICATE_RECV_OUTPUT]]) + // CHECK: "tf.D" + // CHECK-NEXT: "tf.Yield"() : () -> () + // CHECK: "tf_device.cluster" + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + // CHECK: %[[B_OUTPUT:[0-9]*]] = "tf.B" + // CHECK: %[[G_OUTPUT:[0-9]*]] = "tf.G" + // CHECK: "tf.XlaSendToHost"(%6) {key = "if_predicate_channel_cluster1_0"} + // CHECK-NEXT: tf.IfRegion"(%[[G_OUTPUT]]) + // CHECK-NEXT: "tf.Yield"() : () -> () + %1:2 = tf_device.replicate([%0, %arg0] as %ri_0: tensor) {n = 2 : i32} { + %2 = "tf_device.cluster"() ( { + %3 = "tf.A"() : () -> (tensor) + %4 = "tf.B"() : () -> (tensor) + %6 = "tf.G"() : () -> (tensor) + + "tf.IfRegion"(%6) ({ + "tf.D"() {_xla_outside_compilation = "cluster1"} : () -> () + "tf.Yield"() : () -> () + }, { + "tf.Yield"() : () -> () + }) { is_stateless = false} : (tensor) -> () + + %5 = "tf.E"() : () -> tensor + tf_device.return %5 : tensor + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + tf_device.return %2 : tensor + } + + return %1 : tensor + } + + // Tests extraction of a single outside compiled cluster inside a nested + // tf.IfRegion op. + + // CHECK-LABEL: func @outside_compiled_ops_inside_nested_if + func @outside_compiled_ops_inside_nested_if(%arg0: tensor) -> tensor { + %0 = "tf.A"(%arg0) : (tensor) -> tensor + // CHECK: %[[REPLICATE:[0-9]*]]:2 = tf_device.replicate + // CHECK: %[[PARALLEL_EXECUTE_OUTPUT:[0-9]*]] = "tf_device.parallel_execute" + // CHECK-NEXT: "tf_device.launch" + // CHECK-NEXT: %[[PLACEHOLDER_KEY:[0-9]*]] = "tf._TPUCompileMlirPlaceholderProgramKey"() + // CHECK-NEXT: %[[PREDICATE_RECV_OUTPUT:[0-9]*]] = "tf._XlaRecvAtHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "if_predicate_channel_cluster1_0" + // CHECK-NEXT: tf.IfRegion"(%[[PREDICATE_RECV_OUTPUT]]) + // CHECK-NEXT: %[[PREDICATE2_RECV_OUTPUT:[0-9]*]] = "tf._XlaRecvAtHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "if_predicate_channel_cluster1_1" + // CHECK-NEXT: tf.IfRegion"(%[[PREDICATE2_RECV_OUTPUT]]) + // CHECK-NEXT: "tf.Yield"() : () -> () + // CHECK: %[[ARG_RECV_OUTPUT:[0-9]*]] = "tf._XlaRecvAtHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "host_compute_channel_cluster1_args" + // CHECK: "tf.D"(%[[ARG_RECV_OUTPUT]]) + // CHECK: "tf._XlaSendFromHost"(%[[PLACEHOLDER_KEY]]) + // CHECK-SAME: device_ordinal = 0 + // CHECK-SAME: key = "host_compute_channel_cluster1_retvals" + // CHECK-NEXT: "tf.Yield"() : () -> () + + // CHECK: "tf_device.cluster" + // CHECK: %[[A_OUTPUT:[0-9]*]] = "tf.A" + // CHECK: %[[B_OUTPUT:[0-9]*]] = "tf.B" + // CHECK: %[[G_OUTPUT:[0-9]*]] = "tf.G" + // CHECK: "tf.XlaSendToHost"(%[[G_OUTPUT]]) {key = "if_predicate_channel_cluster1_0"} + // CHECK-NEXT: tf.IfRegion"(%[[G_OUTPUT]]) + // CHECK: %[[H_OUTPUT:[0-9]*]] = "tf.H"(%[[B_OUTPUT]]) + // CHECK: "tf.XlaSendToHost"(%[[H_OUTPUT]]) {key = "if_predicate_channel_cluster1_1"} + // CHECK-NEXT: tf.IfRegion"(%[[H_OUTPUT]]) + // CHECK-NEXT: "tf.Yield"() : () -> () + // CHECK: %[[I_OUTPUT:[0-9]*]] = "tf.I"(%[[H_OUTPUT]]) + // CHECK: "tf._XlaHostComputeMlir"(%[[I_OUTPUT]]) + // CHECK-NEXT: "tf.Yield"() : () -> () + %1:2 = tf_device.replicate([%0, %arg0] as %ri_0: tensor) {n = 2 : i32} { + %2 = "tf_device.cluster"() ( { + %3 = "tf.A"() : () -> (tensor) + %4 = "tf.B"() : () -> (tensor) + %6 = "tf.G"() : () -> (tensor) + + "tf.IfRegion"(%6) ({ + %7 = "tf.H"(%4) : (tensor) -> (tensor) + + "tf.IfRegion"(%7)({ + "tf.Yield"() : () -> () + }, + { + %8 = "tf.I"(%7) : (tensor) -> (tensor) + "tf.D"(%8) {_xla_outside_compilation = "cluster1"} : (tensor) -> () + "tf.Yield"() : () -> () + }) { is_stateless = false} : (tensor) -> () + + "tf.Yield"() : () -> () + }, { + "tf.Yield"() : () -> () + }) { is_stateless = false} : (tensor) -> () + + %5 = "tf.E"() : () -> tensor + tf_device.return %5 : tensor + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + tf_device.return %2 : tensor + } + + return %1 : tensor + } } diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc b/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc index cbea4ae6544..9365807663a 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc @@ -17,11 +17,21 @@ limitations under the License. #include #include +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/FormatVariadic.h" +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Function.h" // from @llvm-project +#include "mlir/IR/MLIRContext.h" // from @llvm-project +#include "mlir/IR/Module.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/StandardTypes.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project #include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Pass/PassRegistry.h" // from @llvm-project +#include "mlir/Support/LogicalResult.h" // from @llvm-project #include "mlir/Transforms/RegionUtils.h" // from @llvm-project #include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" #include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" @@ -77,31 +87,203 @@ struct TPUExtractOutsideCompilation void runOnOperation() override; }; -// Collects and clusters ops in `block` with the same `_xla_outside_compilation` -// attribute into `clusters` This returns an error if a -// `_xla_outside_compilation` attribute of an op is empty. -LogicalResult CollectAndGroupOutsideClusterOps(Block* block, - OutsideClusterMap* clusters) { - for (Operation& op : *block) { - if (auto attr = op.getAttrOfType(kXlaOutsideCompilationAttr)) { - if (attr.getValue().empty()) - return op.emitError() - << "attribute '" << kXlaOutsideCompilationAttr << "' is empty"; +// Holds information about control flow operations that wrap outside compiled +// op. Currently only tf.If op is supported. +class ControlFlowStackInfo { + public: + enum ControlFlowBranchType { kIfThen, kIfElse }; - auto it = clusters->try_emplace(attr.getValue()); - it.first->getSecond().push_back(&op); + explicit ControlFlowStackInfo(Operation* wrapping_op, Operation* nested_op) + : callsite_op_(wrapping_op) { + // Only tf.IfRegion op is supported for now. + auto control_flow_op = llvm::cast(callsite_op_); + assert(control_flow_op); + + auto parent_region = nested_op->getParentRegion(); + if (&control_flow_op.then_branch() == parent_region) { + type_ = ControlFlowBranchType::kIfThen; + } else { + type_ = ControlFlowBranchType::kIfElse; } } - return success(); + Value GetIfPredicateValue() { + auto if_op = llvm::cast(callsite_op_); + return if_op.cond(); + } + + ControlFlowBranchType GetBranchType() const { return type_; } + + Operation* GetCallSiteOp() const { return callsite_op_; } + + private: + ControlFlowBranchType type_; + + // `this` does not hold ownership of `callsite_op_`. + Operation* callsite_op_; +}; + +// Returns a list of ControlFlowStackInfo that represents a stack of control +// flow operations that wraps `op`. +llvm::SmallVector GetControlFlowStackForOp( + tf_device::ClusterOp tpu_cluster, Operation* op) { + assert(tpu_cluster.getOperation()->isProperAncestor(op)); + + llvm::SmallVector controlflow_stack; + Operation* op_in_stack = op; + while (op_in_stack != tpu_cluster.getOperation()) { + auto parent_op = op_in_stack->getParentOp(); + if (llvm::isa(parent_op)) { + controlflow_stack.insert(controlflow_stack.begin(), + ControlFlowStackInfo(parent_op, op_in_stack)); + } + op_in_stack = parent_op; + } + + return controlflow_stack; } -// Moves `cluster_ops` to associated `launch_op` body. -void MoveOutsideClusterOpsToLaunchOp(tf_device::LaunchOp launch_op, - llvm::ArrayRef cluster_ops) { - MLIRContext* context = launch_op.getContext(); - Operation* terminator = launch_op.GetBody().getTerminator(); +// Creates a IfRegionOp with `predicate` and then/else region with yield op and +// an empty block. +TF::IfRegionOp CloneEmptyIfWithPredicate(Value predicate, bool is_stateless, + Location loc, OpBuilder* builder) { + auto host_side_if = builder->create( + loc, llvm::SmallVector{}, predicate, is_stateless); + // Create empty then branch region. + auto& then_branch = host_side_if.then_branch(); + builder->setInsertionPoint(&then_branch.front(), then_branch.front().begin()); + builder->createBlock(&then_branch); + builder->create(loc, llvm::SmallVector({})); + + // Create empty else branch region. + auto& else_branch = host_side_if.else_branch(); + builder->setInsertionPoint(&else_branch.front(), else_branch.front().begin()); + builder->createBlock(&else_branch); + builder->create(loc, llvm::SmallVector({})); + return host_side_if; +} + +// Replicates tf.IfRegion op to host side computation. +Operation* ReplicateIf(const ControlFlowStackInfo& controlflow_info, + llvm::StringRef outside_cluster_name, ModuleOp module, + Value compilation_key, OpBuilder* builder, + int* send_recv_counter) { + // Create XlaSendToHostOp to send predicate value from device to host. + OpBuilder::InsertPoint insert_point = builder->saveInsertionPoint(); + auto if_callsite_op = + llvm::cast(controlflow_info.GetCallSiteOp()); + builder->setInsertionPoint(if_callsite_op); + + const auto predicate_send_recv_key = + llvm::formatv("if_predicate_channel_{0}_{1}", outside_cluster_name, + *send_recv_counter) + .str(); + *send_recv_counter += 1; + + auto predicate = if_callsite_op.cond(); + auto predicate_shape = predicate.getType(); + builder->create(if_callsite_op.getLoc(), predicate, + predicate_send_recv_key); + + // Create XlaRecvAtHostOp to receive predicate value from host. + builder->restoreInsertionPoint(insert_point); + auto recv_predicate_at_host = builder->create( + if_callsite_op.getLoc(), llvm::ArrayRef{predicate_shape}, + /*dynamic_key=*/compilation_key, + builder->getStringAttr(predicate_send_recv_key), + /*device_ordinal=*/builder->getI64IntegerAttr(0)); + + // Create host side if op. + return CloneEmptyIfWithPredicate(recv_predicate_at_host.getResult(0), + if_callsite_op.is_stateless(), + if_callsite_op.getLoc(), builder); +} + +// TODO(b/157054714): Use a better abstraction instead of +// _TPUCompileMlirOp and _XlaRecvAtHostOp and _XlaSendFromHostOp. +// Creates a compilation key as placeholder. A placeholder compilation cache key +// is created because it is a required input to _XlaRecvAtHost and +// _XlaSendFromHost but the _TPUCompileMlir has not yet been created for the TPU +// cluster that contains the outside compiled ops. This placeholder should be +// replaced by the TPU cluster _TPUCompileMlir in a subsequent pass. +Value CreateCompilationKeyPlaceholder(Location loc, OpBuilder* builder) { + auto result_type = + RankedTensorType::get({2}, builder->getType()); + return builder->create( + loc, /*program=*/result_type, llvm::ArrayRef{}); +} + +// Replicates the control flow operations that wraps outside compiled ops to +// `destination_block`. +Block* ReplicateControlFlowStack( + llvm::StringRef outside_cluster_name, + const llvm::SmallVectorImpl& stack_info, + tf_device::ClusterOp tpu_cluster, ModuleOp module, Value compilation_key, + Block* destination_block, int* send_recv_counter) { + assert(stack_info.size()); + OpBuilder builder = OpBuilder::atBlockTerminator(destination_block); + Operation* previous_replicated_controlflow_op = nullptr; + for (const auto& controlflow_stack_info : stack_info) { + // Create control flow op given provided insertion point and + // ControlFlowStackInfo. + previous_replicated_controlflow_op = + ReplicateIf(controlflow_stack_info, outside_cluster_name, module, + compilation_key, &builder, send_recv_counter); + auto if_op = llvm::cast(previous_replicated_controlflow_op); + auto type = controlflow_stack_info.GetBranchType(); + + // Update the insertion point to proper region inside the newly created + // control flow op. + if (type == ControlFlowStackInfo::kIfThen) { + builder.setInsertionPoint(&if_op.then_branch().front().front()); + } else { + builder.setInsertionPoint(&if_op.else_branch().front().front()); + } + } + + // Return the inner most branch at which outside compiled op is located. + // This block will later be used as insertion point to create send/recv ops. + auto inner_most_controlflow_stack = stack_info.back(); + auto inner_most_if = + llvm::cast(previous_replicated_controlflow_op); + if (inner_most_controlflow_stack.GetBranchType() == + ControlFlowStackInfo::kIfThen) { + return &inner_most_if.then_branch().front(); + } else { + return &inner_most_if.else_branch().front(); + } +} + +// Collects and clusters ops in `block` with the same `_xla_outside_compilation` +// attribute into `clusters` This returns an error if a +// `_xla_outside_compilation` attribute of an op is empty. +// TODO(b/163141763): Make sure ops inside control flow regions are not outside +// compiled if the entire control flow op is marked as outside compiled. +LogicalResult CollectAndGroupOutsideClusterOps(Block* block, + OutsideClusterMap* clusters) { + auto walk_result = block->walk([&](Operation* op) { + if (auto attr = op->getAttrOfType(kXlaOutsideCompilationAttr)) { + if (attr.getValue().empty()) { + op->emitError() << "attribute '" << kXlaOutsideCompilationAttr + << "' is empty"; + return WalkResult::interrupt(); + } + + auto it = clusters->try_emplace(attr.getValue()); + it.first->getSecond().push_back(op); + } + return WalkResult::advance(); + }); + + return failure(walk_result.wasInterrupted()); +} + +// Moves `cluster_ops` to associated `block`. +void MoveOutsideClusterOpsToBlock(Block& block, + llvm::ArrayRef cluster_ops, + MLIRContext* context) { + Operation* terminator = block.getTerminator(); for (Operation* cluster_op : cluster_ops) { // Remove `_xla_outside_compilation` and `device` attribute from ops in the // cluster as that information will be present in the `launch_op`. @@ -112,7 +294,7 @@ void MoveOutsideClusterOpsToLaunchOp(tf_device::LaunchOp launch_op, } } -// Creates a `tf_device::LaunchOp` to wrap cluster ops. +// Creates a `tf_device.launch` to wrap cluster ops. tf_device::LaunchOp CreateLaunchOpForOutsideCluster( OpBuilder* builder, Operation* last_cluster_op, llvm::StringRef host_device) { @@ -212,34 +394,43 @@ TF::_XlaHostComputeMlirOp CreateHostCompute( } void MoveOutsideCompiledOps( - tf_device::ClusterOp tpu_cluster, llvm::StringRef outside_cluster_name, - tf_device::LaunchOp host_launch_op, llvm::ArrayRef cluster_ops, + ModuleOp module, tf_device::ClusterOp tpu_cluster, + llvm::StringRef outside_cluster_name, tf_device::LaunchOp host_launch_op, + llvm::ArrayRef cluster_ops, const llvm::SmallSetVector& external_inputs, llvm::ArrayRef external_outputs) { + // Since ops in `cluster_ops` do not cross function/control flow boundary, it + // is sufficient to identify the control flow that wraps `cluster_ops` by + // looking at any arbitary op inside `cluster_ops`. + auto controlflow_stack = + GetControlFlowStackForOp(tpu_cluster, cluster_ops.front()); + + Value compilation_key; + if (!controlflow_stack.empty() || !external_inputs.empty() || + !external_outputs.empty()) { + OpBuilder builder(&host_launch_op.GetBody().front()); + compilation_key = + CreateCompilationKeyPlaceholder(tpu_cluster.getLoc(), &builder); + } + + Block* block_to_move_host_cluster = nullptr; + if (controlflow_stack.empty()) { + block_to_move_host_cluster = &host_launch_op.GetBody(); + } else { + int send_recv_counter = 0; + block_to_move_host_cluster = ReplicateControlFlowStack( + outside_cluster_name, controlflow_stack, tpu_cluster, module, + compilation_key, &host_launch_op.GetBody(), &send_recv_counter); + } + + MLIRContext* context = host_launch_op.getContext(); if (external_inputs.empty() && external_outputs.empty()) { - MoveOutsideClusterOpsToLaunchOp(host_launch_op, cluster_ops); + MoveOutsideClusterOpsToBlock(*block_to_move_host_cluster, cluster_ops, + context); return; } - OpBuilder builder(host_launch_op.GetBody().getTerminator()); - auto result_type = - RankedTensorType::get({}, builder.getType()); - - std::string txt_metadata; - std::string txt_module; - // TODO(b/157054714): Use a better abstraction instead of _TPUCompileMlirOp - // and _XlaRecvAtHostOp and _XlaSendFromHostOp. - - // A placeholder compilation cache key is created because it is a required - // input to _XlaRecvAtHost and _XlaSendFromHost but the _TPUCompileMlir has - // not yet been created for the TPU cluster that contains the outside compiled - // ops. This placeholder should be replaced by the TPU cluster _TPUCompileMlir - // in a subsequent pass. - auto compilation_key = - builder.create( - tpu_cluster.getLoc(), /*program=*/result_type, - llvm::ArrayRef{}); - + OpBuilder builder(block_to_move_host_cluster->getTerminator()); llvm::SmallVector host_output_types; for (const auto& external_input : external_inputs) host_output_types.push_back(external_input.getType()); @@ -250,6 +441,7 @@ void MoveOutsideCompiledOps( std::string retvals_communication_key = llvm::formatv("host_compute_channel_{0}_retvals", outside_cluster_name) .str(); + auto recv_at_host = builder.create( tpu_cluster.getLoc(), host_output_types, /*dynamic_key=*/compilation_key, @@ -259,9 +451,10 @@ void MoveOutsideCompiledOps( auto host_compute = CreateHostCompute( &builder, tpu_cluster, cluster_ops, external_inputs, external_outputs, args_communication_key, retvals_communication_key); - MoveOutsideClusterOpsToLaunchOp(host_launch_op, cluster_ops); + MoveOutsideClusterOpsToBlock(*block_to_move_host_cluster, cluster_ops, + context); - builder.setInsertionPoint(host_launch_op.GetBody().getTerminator()); + builder.setInsertionPoint(block_to_move_host_cluster->getTerminator()); builder.create( tpu_cluster.getLoc(), external_outputs, /*dynamic_key=*/compilation_key, @@ -279,7 +472,8 @@ void MoveOutsideCompiledOps( // Creates a `parallel_execute` op in place of launch with 'clusters` and // 'launch` as regions. -void CreateParallelExecuteFromOutsideClusters(tf_device::ClusterOp tpu_cluster, +void CreateParallelExecuteFromOutsideClusters(ModuleOp module, + tf_device::ClusterOp tpu_cluster, const OutsideClusterMap& clusters, llvm::StringRef host_device) { OpBuilder builder(tpu_cluster); @@ -295,6 +489,7 @@ void CreateParallelExecuteFromOutsideClusters(tf_device::ClusterOp tpu_cluster, Block& outside_block = parallel_execute_op.GetRegionBlockWithIndex(cluster.index()); + builder.setInsertionPointToEnd(&outside_block); tf_device::LaunchOp host_launch_op = CreateLaunchOpForOutsideCluster( &builder, cluster_ops.back(), host_device); @@ -303,10 +498,9 @@ void CreateParallelExecuteFromOutsideClusters(tf_device::ClusterOp tpu_cluster, auto external_inputs = GetExternalOperands(cluster_ops); auto external_outputs = GetExternalOutputs(cluster_ops); - MoveOutsideCompiledOps(tpu_cluster, cluster.value().getFirst(), + MoveOutsideCompiledOps(module, tpu_cluster, cluster.value().getFirst(), host_launch_op, cluster_ops, external_inputs, external_outputs); - builder.setInsertionPointToEnd(&outside_block); builder.create(tpu_cluster.getLoc(), ArrayRef{}); @@ -352,7 +546,8 @@ void TPUExtractOutsideCompilation::runOnOperation() { std::string host_device; tensorflow::GetHostDeviceOutsideComputation(devices, tpu_cluster, &host_device); - CreateParallelExecuteFromOutsideClusters(tpu_cluster, clusters, + + CreateParallelExecuteFromOutsideClusters(module, tpu_cluster, clusters, host_device); return WalkResult::advance(); From 53f24479118a9a0a358a86c2bd93654b776f0870 Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Mon, 10 Aug 2020 16:17:30 -0700 Subject: [PATCH 0786/1017] [TF2XLA] [NFC] Allow using XlaCompileOnDemandOp without XlaDeviceMetadata PiperOrigin-RevId: 325910208 Change-Id: I24f6b14fa24c614b0994ee2efdd077e5ef2fe55e --- tensorflow/compiler/jit/BUILD | 5 +- tensorflow/compiler/jit/kernels/xla_ops.cc | 146 +--------------- tensorflow/compiler/jit/kernels/xla_ops.h | 56 +------ .../compiler/jit/xla_compile_on_demand_op.cc | 57 +++---- .../compiler/jit/xla_compile_on_demand_op.h | 12 +- tensorflow/compiler/jit/xla_platform_info.cc | 158 ++++++++++++++++++ tensorflow/compiler/jit/xla_platform_info.h | 108 ++++++++++++ 7 files changed, 315 insertions(+), 227 deletions(-) create mode 100644 tensorflow/compiler/jit/xla_platform_info.cc create mode 100644 tensorflow/compiler/jit/xla_platform_info.h diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index 63f985935fb..d05bb8264c3 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -195,6 +195,7 @@ XLA_DEVICE_DEPS = [ "//tensorflow/core/kernels/data:optional_ops", "//tensorflow/core/kernels/data:prefetch_dataset_op", "//tensorflow/core/profiler/lib:traceme", + "//tensorflow/stream_executor:tf_allocator_adapter", "//tensorflow/stream_executor/platform", ] @@ -205,16 +206,18 @@ cc_library( "xla_device.cc", "xla_device_context.cc", "xla_device_ops.cc", + "xla_platform_info.cc", ], hdrs = [ "xla_compile_on_demand_op.h", "xla_device.h", "xla_device_context.h", "xla_device_ops.h", + "xla_platform_info.h", ], # Public visibility is needed for external TF/XLA backends. visibility = ["//visibility:public"], - deps = XLA_DEVICE_DEPS, + deps = XLA_DEVICE_DEPS + [":xla_compilation_cache"], ) cc_library( diff --git a/tensorflow/compiler/jit/kernels/xla_ops.cc b/tensorflow/compiler/jit/kernels/xla_ops.cc index 38e33a60657..9cee4b9af28 100644 --- a/tensorflow/compiler/jit/kernels/xla_ops.cc +++ b/tensorflow/compiler/jit/kernels/xla_ops.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/jit/xla_activity_listener.h" #include "tensorflow/compiler/jit/xla_cluster_util.h" +#include "tensorflow/compiler/jit/xla_platform_info.h" #include "tensorflow/compiler/tf2xla/shape_util.h" #include "tensorflow/compiler/tf2xla/tf2xla_util.h" #include "tensorflow/compiler/tf2xla/xla_compiler.h" @@ -63,38 +64,6 @@ namespace tensorflow { namespace { -XlaPlatformInfo PlatformInfoFromContext(OpKernelConstruction* ctx) { - DeviceType device_type = ctx->device_type(); - se::Platform::Id platform_id = nullptr; - const XlaDevice::Metadata* xla_device_metadata = nullptr; - se::DeviceMemoryAllocator* custom_allocator = nullptr; - - if (ctx->device_type() == DeviceType(DEVICE_CPU)) { - platform_id = se::host::kHostPlatformId; - } else if (ctx->device_type() == DeviceType(DEVICE_GPU)) { - platform_id = ctx->device() - ->tensorflow_gpu_device_info() - ->stream->parent() - ->platform() - ->id(); - } else if (XlaDevice::GetMetadata(ctx, &xla_device_metadata).ok()) { - // If we are on an XlaDevice, use the underlying XLA platform's allocator - // directly. We could use the StreamExecutor's allocator which may - // theoretically be more correct, but XLA returns a nice OOM message in a - // Status and StreamExecutor does not. - // - // Importantly we can't use ctx->device()->GetAllocator() as the allocator - // (which xla_allocator above uses) as on an XlaDevice, this is a dummy - // allocator that returns XlaTensor objects. The XlaCompiler needs a real - // allocator to allocate real buffers. - platform_id = xla_device_metadata->platform()->id(); - custom_allocator = - xla_device_metadata->client()->backend().memory_allocator(); - } - - return XlaPlatformInfo(device_type, platform_id, xla_device_metadata, - custom_allocator); -} // A closure describing how to run a compiled version of a TensorFlow function. // @@ -178,31 +147,6 @@ class XlaExecutableClosureStore { TF_DISALLOW_COPY_AND_ASSIGN(XlaExecutableClosureStore); }; -// Return allocator from platform info if non-null, or populate and return a -// pointer to the allocator adapter with allocator from context. -// -// This is necessary because for XLA devices the underlying TF allocator returns -// dummy tensors. -se::DeviceMemoryAllocator* GetAllocator( - absl::optional* tf_allocator_adapter, - OpKernelContext* ctx, const XlaPlatformInfo& platform_info) { - if (platform_info.custom_allocator()) { - return platform_info.custom_allocator(); - } - if (!ctx->op_device_context()) { - // Stream is not set for the host platform. - se::Platform* platform = - se::MultiPlatformManager::PlatformWithId(platform_info.platform_id()) - .ValueOrDie(); - tf_allocator_adapter->emplace(ctx->device()->GetAllocator({}), platform); - return &tf_allocator_adapter->value(); - } - // platform_info. - tf_allocator_adapter->emplace(ctx->device()->GetAllocator({}), - ctx->op_device_context()->stream()); - return &tf_allocator_adapter->value(); -} - } // namespace XlaLocalLaunchBase::XlaLocalLaunchBase(OpKernelConstruction* ctx, @@ -214,65 +158,9 @@ XlaLocalLaunchBase::XlaLocalLaunchBase(OpKernelConstruction* ctx, constants_(constants), resources_(resources), function_(function), - platform_info_(PlatformInfoFromContext(ctx)), + platform_info_(XlaPlatformInfoFromContext(ctx)), has_ref_vars_(has_ref_vars) {} -static Status BuildCompilationCache(OpKernelContext* ctx, - const XlaPlatformInfo& platform_info, - XlaCompilationCache** cache) { - if (platform_info.xla_device_metadata()) { - *cache = new XlaCompilationCache( - platform_info.xla_device_metadata()->client(), - platform_info.xla_device_metadata()->jit_device_type()); - return Status::OK(); - } - - auto platform = - se::MultiPlatformManager::PlatformWithId(platform_info.platform_id()); - if (!platform.ok()) { - return platform.status(); - } - - xla::StatusOr compiler_for_platform = - xla::Compiler::GetForPlatform(platform.ValueOrDie()); - if (!compiler_for_platform.ok()) { - // In some rare cases (usually in unit tests with very small clusters) we - // may end up transforming an XLA cluster with at least one GPU operation - // (which would normally force the cluster to be compiled using XLA:GPU) - // into an XLA cluster with no GPU operations (i.e. containing only CPU - // operations). Such a cluster can fail compilation (in way that - // MarkForCompilation could not have detected) if the CPU JIT is not linked - // in. - // - // So bail out of _XlaCompile in this case, and let the executor handle the - // situation for us. - const Status& status = compiler_for_platform.status(); - if (status.code() == error::NOT_FOUND) { - return errors::Unimplemented("Could not find compiler for platform ", - platform.ValueOrDie()->Name(), ": ", - status.ToString()); - } - } - - xla::LocalClientOptions client_options; - client_options.set_platform(platform.ValueOrDie()); - client_options.set_intra_op_parallelism_threads( - ctx->device()->tensorflow_cpu_worker_threads()->num_threads); - auto client = xla::ClientLibrary::GetOrCreateLocalClient(client_options); - if (!client.ok()) { - return client.status(); - } - const XlaOpRegistry::DeviceRegistration* registration; - if (!XlaOpRegistry::GetCompilationDevice(platform_info.device_type().type(), - ®istration)) { - return errors::InvalidArgument("No JIT device registered for ", - platform_info.device_type().type()); - } - *cache = new XlaCompilationCache( - client.ValueOrDie(), DeviceType(registration->compilation_device_name)); - return Status::OK(); -} - static Status CompileToLocalExecutable( OpKernelContext* ctx, const NameAttrList& function, bool has_ref_vars, const XlaPlatformInfo& platform_info, @@ -292,7 +180,7 @@ static Status CompileToLocalExecutable( TF_RETURN_IF_ERROR(rm->LookupOrCreate( rm->default_container(), "xla_cache", &cache, [&](XlaCompilationCache** cache) { - return BuildCompilationCache(ctx, platform_info, cache); + return BuildXlaCompilationCache(ctx, platform_info, cache); })); // Hold the reference to the JIT during evaluation. (We could probably // free it sooner because the ResourceMgr will retain a reference, but @@ -302,32 +190,14 @@ static Status CompileToLocalExecutable( *client = static_cast(cache->client()); absl::optional tf_allocator_adapter; - XlaCompiler::Options options; - options.client = *client; - if (ctx->op_device_context() != nullptr) { - options.device_ordinal = - ctx->op_device_context()->stream()->parent()->device_ordinal(); - } - options.device_type = cache->device_type(); - options.flib_def = ctx->function_library()->GetFunctionLibraryDefinition(); - options.graph_def_version = ctx->function_library()->graph_def_version(); - options.allow_cpu_custom_calls = - (platform_info.platform_id() == se::host::kHostPlatformId); - options.device_allocator = - GetAllocator(&tf_allocator_adapter, ctx, platform_info); - if (platform_info.xla_device_metadata()) { - options.shape_representation_fn = - platform_info.xla_device_metadata()->shape_representation_fn(); - } - // If reference variables are not present in the graph, we can safely alias - // passthrough parameters without performing a copy. - options.alias_passthrough_params = - !has_ref_vars && !platform_info.is_on_xla_device(); + XlaCompiler::Options options = GenerateCompilerOptions( + cache, ctx, platform_info, has_ref_vars, &tf_allocator_adapter); std::map constant_args; for (int i : constants) { constant_args.insert({i, ctx->input(i)}); } + XlaCompiler::CompileOptions compile_options; compile_options.is_entry_computation = true; // Optimization: where possible, have the computation return a naked array @@ -503,7 +373,7 @@ XlaCompileOp::XlaCompileOp(OpKernelConstruction* ctx) constants_(ConstantsVector(ctx)), resources_(ResourcesVector(ctx)), function_(FunctionAttr(ctx)), - platform_info_(PlatformInfoFromContext(ctx)), + platform_info_(XlaPlatformInfoFromContext(ctx)), must_compile_(MustCompileAttr(ctx)), has_ref_vars_(HasRefVars(ctx)) {} @@ -591,7 +461,7 @@ void XlaCompileOp::Compute(OpKernelContext* ctx) { } XlaRunOp::XlaRunOp(OpKernelConstruction* ctx) - : OpKernel(ctx), platform_info_(PlatformInfoFromContext(ctx)) {} + : OpKernel(ctx), platform_info_(XlaPlatformInfoFromContext(ctx)) {} void XlaRunOp::Compute(OpKernelContext* ctx) { VLOG(3) << "XlaRunOp " << def().name(); diff --git a/tensorflow/compiler/jit/kernels/xla_ops.h b/tensorflow/compiler/jit/kernels/xla_ops.h index 112408226a8..78707c8126d 100644 --- a/tensorflow/compiler/jit/kernels/xla_ops.h +++ b/tensorflow/compiler/jit/kernels/xla_ops.h @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/compiler/jit/xla_compilation_cache.h" #include "tensorflow/compiler/jit/xla_device.h" #include "tensorflow/compiler/jit/xla_launch_util.h" +#include "tensorflow/compiler/jit/xla_platform_info.h" #include "tensorflow/core/framework/allocator.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" @@ -31,61 +32,6 @@ limitations under the License. namespace tensorflow { -// Holds some information about the platform on which an -// XlaLaunch/_XlaCompile/_XlaRun op must run on. -class XlaPlatformInfo { - public: - XlaPlatformInfo() : device_type_("") {} - XlaPlatformInfo(XlaPlatformInfo&&) = default; - explicit XlaPlatformInfo(const DeviceType device_type, - se::Platform::Id platform_id, - const XlaDevice::Metadata* xla_device_metadata, - se::DeviceMemoryAllocator* device_allocator) - : device_type_(device_type), - platform_id_(platform_id), - xla_device_metadata_(xla_device_metadata), - device_allocator_(device_allocator) {} - - XlaPlatformInfo& operator=(XlaPlatformInfo&& other) = default; - - bool UseMultipleStreams() const { - return xla_device_metadata_ && xla_device_metadata_->UseMultipleStreams(); - } - - // Non-null only when run on an XLA device. - se::DeviceMemoryAllocator* custom_allocator() const { - return device_allocator_; - } - - DeviceType device_type() const { return device_type_; } - - // This is equal to xla_device_metadata()->platform()->id() if - // xla_device_metadata() is not nullptr. - se::Platform::Id platform_id() const { return platform_id_; } - - // This may be null if the op this XlaPlatformInfo is for was not placed on an - // XLA device. - const XlaDevice::Metadata* xla_device_metadata() const { - return xla_device_metadata_; - } - bool is_on_xla_device() const { return xla_device_metadata() != nullptr; } - - private: - DeviceType device_type_; - se::Platform::Id platform_id_; - - // xla_device_metadata_ lives in the tensorflow::DeviceBase in which the - // XlaLaunch/_XlaCompile/_XlaRun op is placed and thus does not die before the - // XlaLaunch/_XlaCompile/_XlaRun OpKernel. - const XlaDevice::Metadata* xla_device_metadata_; - - // If the op associated with this XlaPlatformInfo is placed on an XLA device - // then device_allocator_ is the xla::Backend's memory allocator. If the op - // is placed on a regular CPU or GPU device then device_allocator_ is null. - se::DeviceMemoryAllocator* device_allocator_; - - TF_DISALLOW_COPY_AND_ASSIGN(XlaPlatformInfo); -}; // XlaLocalLaunchBase is almost the same as XlaLocalLaunchOp. // The only difference is that it does not require arguments to follow diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc index 50813859603..d1ea9083796 100644 --- a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc +++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc @@ -20,6 +20,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "tensorflow/compiler/jit/xla_device.h" #include "tensorflow/compiler/jit/xla_launch_util.h" +#include "tensorflow/compiler/jit/xla_platform_info.h" #include "tensorflow/compiler/tf2xla/const_analysis.h" #include "tensorflow/compiler/tf2xla/tf2xla_util.h" #include "tensorflow/compiler/tf2xla/xla_compiler.h" @@ -41,18 +42,19 @@ static std::vector GetResourceVariableIndices(OpKernelContext* ctx) { } Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, - const XlaDevice::Metadata& metadata, + XlaCompilationCache* cache, const XlaCompiler::CompilationResult* result, xla::LocalExecutable* executable, const ResourceVarsSnapshot& variable_args) { - xla::LocalClient* client = metadata.client(); + xla::LocalClient* client = static_cast(cache->client()); - // Builds an XLA allocator for the device. XlaComputationLaunchContext launch_context( client, client->backend().memory_allocator(), client->default_device_ordinal(), - /*allocate_xla_tensors=*/true, - /*use_multiple_streams=*/metadata.UseMultipleStreams()); + /*allocate_xla_tensors=*/platform_info_.xla_device_metadata() != nullptr, + platform_info_.xla_device_metadata() + ? platform_info_.xla_device_metadata()->UseMultipleStreams() + : false); std::map snapshot_ptrs; for (auto& p : variable_args) { @@ -70,7 +72,6 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, se::Stream* stream = ctx->op_device_context() ? ctx->op_device_context()->stream() : nullptr; - TF_RET_CHECK(stream); VLOG(2) << "Executing computation: " << name(); xla::ExecutableRunOptions run_options; @@ -116,9 +117,9 @@ Status XlaCompileOnDemandOp::ShouldArgumentBeConstant( } Status XlaCompileOnDemandOp::Compile( - OpKernelContext* ctx, const XlaDevice::Metadata& metadata, - const XlaCompiler::CompilationResult** result, - ResourceVarsSnapshot* variable_args, xla::LocalExecutable** executable) { + OpKernelContext* ctx, const XlaCompiler::CompilationResult** result, + XlaCompilationCache** cache, ResourceVarsSnapshot* variable_args, + xla::LocalExecutable** executable) { std::map constant_arguments; for (int64 i = 0; i < ctx->num_inputs(); ++i) { const Tensor& device_tensor = ctx->input(i); @@ -168,24 +169,16 @@ Status XlaCompileOnDemandOp::Compile( ResourceMgr* rm = ctx->resource_manager(); CHECK(rm); - XlaCompilationCache* cache; TF_RETURN_IF_ERROR(rm->LookupOrCreate( - rm->default_container(), "xla_cache", &cache, - [&](XlaCompilationCache** cache) { - *cache = new XlaCompilationCache(metadata.client(), - metadata.jit_device_type()); - return Status::OK(); + rm->default_container(), "xla_cache", cache, + [&](XlaCompilationCache** write_into_cache) { + return BuildXlaCompilationCache(ctx, platform_info_, write_into_cache); })); - // Hold the reference to the JIT during evaluation. (We could probably - // free it sooner because the ResourceMgr will retain a reference, but - // this is more obviously correct.) - core::ScopedUnref cache_ref(cache); - XlaCompiler::Options options; - options.device_type = metadata.jit_device_type(); - options.client = metadata.client(); - options.flib_def = ctx->function_library()->GetFunctionLibraryDefinition(); - options.shape_representation_fn = metadata.shape_representation_fn(); + absl::optional tf_allocator_adapter; + XlaCompiler::Options options = + GenerateCompilerOptions(*cache, ctx, platform_info_, + /*has_ref_vars=*/true, &tf_allocator_adapter); XlaCompiler::CompileOptions compile_options; compile_options.is_entry_computation = true; @@ -206,19 +199,23 @@ Status XlaCompileOnDemandOp::Compile( constant_arguments, variable_infos, ctx, &args)); } - return cache->CompileSingleOp(options, args, ctx, compile_options, result, - executable); + return (*cache)->CompileSingleOp(options, args, ctx, compile_options, result, + executable); } void XlaCompileOnDemandOp::Compute(OpKernelContext* ctx) { const XlaCompiler::CompilationResult* result; xla::LocalExecutable* executable; - const XlaDevice::Metadata* metadata; - OP_REQUIRES_OK(ctx, XlaDevice::GetMetadata(ctx, &metadata)); ResourceVarsSnapshot variable_args; + XlaCompilationCache* cache; OP_REQUIRES_OK(ctx, - Compile(ctx, *metadata, &result, &variable_args, &executable)); - OP_REQUIRES_OK(ctx, Run(ctx, *metadata, result, executable, variable_args)); + Compile(ctx, &result, &cache, &variable_args, &executable)); + + // Hold the reference to the JIT during evaluation. (We could probably + // free it sooner because the ResourceMgr will retain a reference, but + // this is more obviously correct.) + core::ScopedUnref cache_ref(cache); + OP_REQUIRES_OK(ctx, Run(ctx, cache, result, executable, variable_args)); } } // namespace tensorflow diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.h b/tensorflow/compiler/jit/xla_compile_on_demand_op.h index cc5f2f1e42f..a3fb60febd7 100644 --- a/tensorflow/compiler/jit/xla_compile_on_demand_op.h +++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.h @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/compiler/jit/xla_device.h" #include "tensorflow/compiler/jit/xla_launch_util.h" +#include "tensorflow/compiler/jit/xla_platform_info.h" #include "tensorflow/compiler/tf2xla/xla_compiler.h" #include "tensorflow/compiler/xla/client/local_client.h" #include "tensorflow/core/framework/function.h" @@ -35,7 +36,8 @@ namespace tensorflow { // vanilla TensorFlow op as long as the bridge supports it. class XlaCompileOnDemandOp : public OpKernel { public: - explicit XlaCompileOnDemandOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + explicit XlaCompileOnDemandOp(OpKernelConstruction* ctx) + : OpKernel(ctx), platform_info_(XlaPlatformInfoFromContext(ctx)) {} void Compute(OpKernelContext* ctx) override; private: @@ -46,14 +48,18 @@ class XlaCompileOnDemandOp : public OpKernel { Status MustArgumentBeConstant(const OpKernel* op_kernel, int64 argument_idx, FunctionLibraryRuntime* flib_runtime, bool* result); - Status Compile(OpKernelContext* ctx, const XlaDevice::Metadata& metadata, + Status Compile(OpKernelContext* ctx, const XlaCompiler::CompilationResult** result, + XlaCompilationCache** cache, ResourceVarsSnapshot* variable_args, xla::LocalExecutable** executable); - Status Run(OpKernelContext* ctx, const XlaDevice::Metadata& metadata, + + Status Run(OpKernelContext* ctx, XlaCompilationCache* cache, const XlaCompiler::CompilationResult* result, xla::LocalExecutable* executable, const ResourceVarsSnapshot& variable_args); + + const XlaPlatformInfo platform_info_; }; } // namespace tensorflow diff --git a/tensorflow/compiler/jit/xla_platform_info.cc b/tensorflow/compiler/jit/xla_platform_info.cc new file mode 100644 index 00000000000..e2a89353055 --- /dev/null +++ b/tensorflow/compiler/jit/xla_platform_info.cc @@ -0,0 +1,158 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/jit/xla_platform_info.h" + +#include "tensorflow/compiler/xla/client/client_library.h" + +namespace tensorflow { + +Status BuildXlaCompilationCache(OpKernelContext* ctx, + const XlaPlatformInfo& platform_info, + XlaCompilationCache** cache) { + if (platform_info.xla_device_metadata()) { + *cache = new XlaCompilationCache( + platform_info.xla_device_metadata()->client(), + platform_info.xla_device_metadata()->jit_device_type()); + return Status::OK(); + } + + auto platform = + se::MultiPlatformManager::PlatformWithId(platform_info.platform_id()); + if (!platform.ok()) { + return platform.status(); + } + + xla::StatusOr compiler_for_platform = + xla::Compiler::GetForPlatform(platform.ValueOrDie()); + if (!compiler_for_platform.ok()) { + // In some rare cases (usually in unit tests with very small clusters) we + // may end up transforming an XLA cluster with at least one GPU operation + // (which would normally force the cluster to be compiled using XLA:GPU) + // into an XLA cluster with no GPU operations (i.e. containing only CPU + // operations). Such a cluster can fail compilation (in way that + // MarkForCompilation could not have detected) if the CPU JIT is not linked + // in. + // + // So bail out of _XlaCompile in this case, and let the executor handle the + // situation for us. + const Status& status = compiler_for_platform.status(); + if (status.code() == error::NOT_FOUND) { + return errors::Unimplemented("Could not find compiler for platform ", + platform.ValueOrDie()->Name(), ": ", + status.ToString()); + } + } + + xla::LocalClientOptions client_options; + client_options.set_platform(platform.ValueOrDie()); + client_options.set_intra_op_parallelism_threads( + ctx->device()->tensorflow_cpu_worker_threads()->num_threads); + auto client = xla::ClientLibrary::GetOrCreateLocalClient(client_options); + if (!client.ok()) { + return client.status(); + } + const XlaOpRegistry::DeviceRegistration* registration; + if (!XlaOpRegistry::GetCompilationDevice(platform_info.device_type().type(), + ®istration)) { + return errors::InvalidArgument("No JIT device registered for ", + platform_info.device_type().type()); + } + *cache = new XlaCompilationCache( + client.ValueOrDie(), DeviceType(registration->compilation_device_name)); + return Status::OK(); +} + +XlaPlatformInfo XlaPlatformInfoFromContext(OpKernelConstruction* ctx) { + DeviceType device_type = ctx->device_type(); + se::Platform::Id platform_id = nullptr; + const XlaDevice::Metadata* xla_device_metadata = nullptr; + se::DeviceMemoryAllocator* custom_allocator = nullptr; + + if (ctx->device_type() == DeviceType(DEVICE_CPU)) { + platform_id = se::host::kHostPlatformId; + } else if (ctx->device_type() == DeviceType(DEVICE_GPU)) { + platform_id = ctx->device() + ->tensorflow_gpu_device_info() + ->stream->parent() + ->platform() + ->id(); + } else if (XlaDevice::GetMetadata(ctx, &xla_device_metadata).ok()) { + // If we are on an XlaDevice, use the underlying XLA platform's allocator + // directly. We could use the StreamExecutor's allocator which may + // theoretically be more correct, but XLA returns a nice OOM message in a + // Status and StreamExecutor does not. + // + // Importantly we can't use ctx->device()->GetAllocator() as the allocator + // (which xla_allocator above uses) as on an XlaDevice, this is a dummy + // allocator that returns XlaTensor objects. The XlaCompiler needs a real + // allocator to allocate real buffers. + platform_id = xla_device_metadata->platform()->id(); + custom_allocator = + xla_device_metadata->client()->backend().memory_allocator(); + } + + return XlaPlatformInfo(device_type, platform_id, xla_device_metadata, + custom_allocator); +} + +se::DeviceMemoryAllocator* GetAllocator( + absl::optional* tf_allocator_adapter, + OpKernelContext* ctx, const XlaPlatformInfo& platform_info) { + if (platform_info.custom_allocator()) { + return platform_info.custom_allocator(); + } + if (!ctx->op_device_context()) { + // Stream is not set for the host platform. + se::Platform* platform = + se::MultiPlatformManager::PlatformWithId(platform_info.platform_id()) + .ValueOrDie(); + tf_allocator_adapter->emplace(ctx->device()->GetAllocator({}), platform); + return &tf_allocator_adapter->value(); + } + tf_allocator_adapter->emplace(ctx->device()->GetAllocator({}), + ctx->op_device_context()->stream()); + return &tf_allocator_adapter->value(); +} + +XlaCompiler::Options GenerateCompilerOptions( + XlaCompilationCache* cache, OpKernelContext* ctx, + const XlaPlatformInfo& platform_info, bool has_ref_vars, + absl::optional* tf_allocator_adapter) { + XlaCompiler::Options options; + options.client = static_cast(cache->client()); + if (ctx->op_device_context() != nullptr) { + options.device_ordinal = + ctx->op_device_context()->stream()->parent()->device_ordinal(); + } + options.device_type = cache->device_type(); + options.flib_def = ctx->function_library()->GetFunctionLibraryDefinition(); + options.graph_def_version = ctx->function_library()->graph_def_version(); + options.allow_cpu_custom_calls = + (platform_info.platform_id() == se::host::kHostPlatformId); + options.device_allocator = + GetAllocator(tf_allocator_adapter, ctx, platform_info); + if (platform_info.xla_device_metadata()) { + options.shape_representation_fn = + platform_info.xla_device_metadata()->shape_representation_fn(); + } + // If reference variables are not present in the graph, we can safely alias + // passthrough parameters without performing a copy. + options.alias_passthrough_params = + !has_ref_vars && !platform_info.is_on_xla_device(); + return options; +} + +} // namespace tensorflow diff --git a/tensorflow/compiler/jit/xla_platform_info.h b/tensorflow/compiler/jit/xla_platform_info.h new file mode 100644 index 00000000000..dac45529ac9 --- /dev/null +++ b/tensorflow/compiler/jit/xla_platform_info.h @@ -0,0 +1,108 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_JIT_XLA_PLATFORM_INFO_H_ +#define TENSORFLOW_COMPILER_JIT_XLA_PLATFORM_INFO_H_ + +#include "tensorflow/compiler/jit/xla_compilation_cache.h" +#include "tensorflow/compiler/jit/xla_device.h" +#include "tensorflow/stream_executor/tf_allocator_adapter.h" + +namespace tensorflow { + +// Holds some information about the platform on which an +// XlaLaunch/_XlaCompile/_XlaRun op must run on. Provides a common layer of +// abstraction for normal and XLA devices. +class XlaPlatformInfo { + public: + XlaPlatformInfo() : device_type_("") {} + XlaPlatformInfo(XlaPlatformInfo&&) = default; + explicit XlaPlatformInfo(const DeviceType device_type, + se::Platform::Id platform_id, + const XlaDevice::Metadata* xla_device_metadata, + se::DeviceMemoryAllocator* device_allocator) + : device_type_(device_type), + platform_id_(platform_id), + xla_device_metadata_(xla_device_metadata), + device_allocator_(device_allocator) {} + + XlaPlatformInfo& operator=(XlaPlatformInfo&& other) = default; + + bool UseMultipleStreams() const { + return xla_device_metadata_ && xla_device_metadata_->UseMultipleStreams(); + } + + // Non-null only when run on an XLA device. + se::DeviceMemoryAllocator* custom_allocator() const { + return device_allocator_; + } + + DeviceType device_type() const { return device_type_; } + + // This is equal to xla_device_metadata()->platform()->id() if + // xla_device_metadata() is not nullptr. + se::Platform::Id platform_id() const { return platform_id_; } + + // This may be null if the op this XlaPlatformInfo is for was not placed on an + // XLA device. + const XlaDevice::Metadata* xla_device_metadata() const { + return xla_device_metadata_; + } + bool is_on_xla_device() const { return xla_device_metadata() != nullptr; } + + private: + DeviceType device_type_; + se::Platform::Id platform_id_; + + // xla_device_metadata_ lives in the tensorflow::DeviceBase in which the + // XlaLaunch/_XlaCompile/_XlaRun op is placed and thus does not die before the + // XlaLaunch/_XlaCompile/_XlaRun OpKernel. + const XlaDevice::Metadata* xla_device_metadata_; + + // If the op associated with this XlaPlatformInfo is placed on an XLA device + // then device_allocator_ is the xla::Backend's memory allocator. If the op + // is placed on a regular CPU or GPU device then device_allocator_ is null. + se::DeviceMemoryAllocator* device_allocator_; + + TF_DISALLOW_COPY_AND_ASSIGN(XlaPlatformInfo); +}; + +// Returns created XLA compilation cache. +Status BuildXlaCompilationCache(OpKernelContext* ctx, + const XlaPlatformInfo& platform_info, + XlaCompilationCache** cache); + +// Returns information about the platform from kernel context. +XlaPlatformInfo XlaPlatformInfoFromContext(OpKernelConstruction* ctx); + +// Returns allocator from platform info if non-null, or populate and return a +// pointer to the allocator adapter with allocator from context. +// +// This is necessary because for XLA devices the underlying TF allocator returns +// dummy tensors. +se::DeviceMemoryAllocator* GetAllocator( + absl::optional* tf_allocator_adapter, + OpKernelContext* ctx, const XlaPlatformInfo& platform_info); + +// Returns created options for the XLA compiler, and writes the used allocator +// into `tf_allocator_adapter`. +XlaCompiler::Options GenerateCompilerOptions( + XlaCompilationCache* cache, OpKernelContext* ctx, + const XlaPlatformInfo& platform_info, bool has_ref_vars, + absl::optional* tf_allocator_adapter); + +} // namespace tensorflow + +#endif // TENSORFLOW_COMPILER_JIT_XLA_PLATFORM_INFO_H_ From dda5175d39285f53379ebb0627bdc1d575fea69a Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Mon, 10 Aug 2020 16:22:41 -0700 Subject: [PATCH 0787/1017] [TF2XLA] Simplify logic for finding constant arguments in XlaCompileOnDemandOp PiperOrigin-RevId: 325911135 Change-Id: I7664ed7d19edb5aba52b2c07443e9786761823bd --- .../compiler/jit/xla_compile_on_demand_op.cc | 42 ++++--------------- .../compiler/jit/xla_compile_on_demand_op.h | 6 --- 2 files changed, 9 insertions(+), 39 deletions(-) diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc index d1ea9083796..73c512bfa6f 100644 --- a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc +++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc @@ -95,53 +95,29 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, return Status::OK(); } -Status XlaCompileOnDemandOp::MustArgumentBeConstant( - const OpKernel* op_kernel, int64 argument_idx, - FunctionLibraryRuntime* flib_runtime, bool* result) { - *result = false; - - // TODO(jmolloy): This could be expensive, so memoize. - std::vector constant_input_indices; - TF_RETURN_IF_ERROR(GetCompileTimeConstInputs( - op_kernel, &constant_input_indices, flib_runtime)); - *result = absl::c_binary_search(constant_input_indices, argument_idx); - return Status::OK(); -} - -// TODO(ycao): Remove the need to call ShouldArgumentBeConstant. Its benefit is -// not clear yet and it causes heavy constant analysis to run twice. -Status XlaCompileOnDemandOp::ShouldArgumentBeConstant( - const OpKernel* op_kernel, int64 argument_idx, - FunctionLibraryRuntime* flib_runtime, bool* result) { - return MustArgumentBeConstant(op_kernel, argument_idx, flib_runtime, result); -} - Status XlaCompileOnDemandOp::Compile( OpKernelContext* ctx, const XlaCompiler::CompilationResult** result, XlaCompilationCache** cache, ResourceVarsSnapshot* variable_args, xla::LocalExecutable** executable) { std::map constant_arguments; + + std::vector constant_input_indices; + TF_RETURN_IF_ERROR(GetCompileTimeConstInputs( + &ctx->op_kernel(), &constant_input_indices, ctx->function_library())); + CHECK(absl::c_is_sorted(constant_input_indices)); + for (int64 i = 0; i < ctx->num_inputs(); ++i) { const Tensor& device_tensor = ctx->input(i); if (const XlaTensor* xla_tensor = XlaTensor::FromTensor(&device_tensor)) { if (xla_tensor->has_host_tensor()) { - bool should_arg_be_const; - TF_RETURN_IF_ERROR(ShouldArgumentBeConstant(&ctx->op_kernel(), i, - ctx->function_library(), - &should_arg_be_const)); - if (should_arg_be_const) { + if (absl::c_binary_search(constant_input_indices, i)) { constant_arguments[i] = xla_tensor->host_tensor(); } } } - if (constant_arguments.count(i) == 0) { - bool must_argument_be_const; - TF_RETURN_IF_ERROR(MustArgumentBeConstant(&ctx->op_kernel(), i, - ctx->function_library(), - &must_argument_be_const)); - - if (must_argument_be_const) { + if (!constant_arguments.count(i)) { + if (absl::c_binary_search(constant_input_indices, i)) { // Slow path; the argument is not available as a host constant so we // must fetch it synchronously. Tensor host_tensor; diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.h b/tensorflow/compiler/jit/xla_compile_on_demand_op.h index a3fb60febd7..095d3427d41 100644 --- a/tensorflow/compiler/jit/xla_compile_on_demand_op.h +++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.h @@ -42,12 +42,6 @@ class XlaCompileOnDemandOp : public OpKernel { private: XlaCompiler::Argument CreateCompilerArgument(OpKernelContext* ctx, int64 i); - Status ShouldArgumentBeConstant(const OpKernel* op_kernel, int64 argument_idx, - FunctionLibraryRuntime* flib_runtime, - bool* result); - Status MustArgumentBeConstant(const OpKernel* op_kernel, int64 argument_idx, - FunctionLibraryRuntime* flib_runtime, - bool* result); Status Compile(OpKernelContext* ctx, const XlaCompiler::CompilationResult** result, XlaCompilationCache** cache, From 5db822426c82d63f88e7423bd6d0e5734a70e41a Mon Sep 17 00:00:00 2001 From: Tim Shen Date: Mon, 10 Aug 2020 16:22:44 -0700 Subject: [PATCH 0788/1017] Roll forward XLA/GPU LHLO sort emitter PiperOrigin-RevId: 325911150 Change-Id: Idf8f73f2840377592c4f2eaa439a07c1700236fd --- tensorflow/compiler/mlir/xla/hlo_utils.cc | 3 + .../non_identity_layouts.hlotxt | 2 +- .../xla/transforms/mhlo_to_lhlo_with_xla.cc | 11 +- .../xla/transforms/mhlo_to_lhlo_with_xla.h | 3 +- tensorflow/compiler/xla/service/gpu/BUILD | 10 + .../compiler/xla/service/gpu/gpu_compiler.cc | 24 +- .../xla/service/gpu/hlo_to_ir_bindings.cc | 20 +- .../xla/service/gpu/hlo_to_ir_bindings.h | 4 + .../xla/service/gpu/ir_emitter_context.h | 7 +- .../xla/service/gpu/ir_emitter_unnested.cc | 416 +++++++++++---- .../xla/service/gpu/ir_emitter_unnested.h | 82 ++- .../compiler/xla/service/gpu/tests/BUILD | 29 + .../xla/service/gpu/tests/sorting.hlo | 504 +++++++++--------- .../xla/service/gpu/tests/sorting_test.cc | 71 +++ .../compiler/xla/service/llvm_ir/llvm_util.cc | 7 +- .../compiler/xla/service/llvm_ir/llvm_util.h | 2 +- 16 files changed, 792 insertions(+), 403 deletions(-) create mode 100644 tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc diff --git a/tensorflow/compiler/mlir/xla/hlo_utils.cc b/tensorflow/compiler/mlir/xla/hlo_utils.cc index cf78c81908d..18b4265d786 100644 --- a/tensorflow/compiler/mlir/xla/hlo_utils.cc +++ b/tensorflow/compiler/mlir/xla/hlo_utils.cc @@ -83,6 +83,9 @@ StatusOr> GetPermutationIfAvailable( strides[dim] = accumulated_stride; accumulated_stride *= shape.dimensions(dim); } + if (accumulated_stride == 0) { + return llvm::SmallVector{}; + } return llvm::SmallVector{ makeStridedLinearLayoutMap(strides, /*offset=*/0, builder.getContext())}; } diff --git a/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt b/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt index 3630d2d45e4..a83e36cff64 100644 --- a/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt +++ b/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt @@ -8,6 +8,6 @@ HloModule TestModule ENTRY TestComputation { x = f32[3, 2]{1,0} parameter(0) - // CHECK: "lmhlo.copy"(%{{.*}}, %{{.*}}) : (memref<3x2xf32>, memref<3x2xf32, #[[MAP]]>) -> () + // CHECK: "lmhlo.copy"(%{{.*}}, %{{.*}}) {name = "copy.1"} : (memref<3x2xf32>, memref<3x2xf32, #[[MAP]]>) -> () ROOT x.copy = f32[3, 2]{0,1} copy(x) } diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc index 832bad2dcc8..6ce91599fb1 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc @@ -34,7 +34,6 @@ limitations under the License. #include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Pass/PassOptions.h" // from @llvm-project #include "mlir/Translation.h" // from @llvm-project -#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" #include "tensorflow/compiler/mlir/xla/hlo_function_importer.h" #include "tensorflow/compiler/mlir/xla/hlo_utils.h" #include "tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.h" @@ -182,7 +181,10 @@ template StatusOr LhloDialectEmitter::CreateOpWithoutAttrs( HloInstruction* instr) { Location loc = getLocation(instr); - ArrayRef> attrs; + std::pair attrs[] = { + {Identifier::get("name", builder_.getContext()), + builder_.getStringAttr(instr->name())}, + }; ArrayRef rets{}; llvm::SmallVector operands; @@ -252,15 +254,14 @@ Status LhloDialectEmitter::DefaultAction(HloInstruction* instr) { return Status::OK(); } -StatusOr LhloDialectEmitter::EmitSortOp( - HloInstruction* instr) { +StatusOr LhloDialectEmitter::EmitSortOp(HloInstruction* instr) { TF_ASSIGN_OR_RETURN(auto sort, CreateOpWithoutAttrs(instr)); auto* sort_instr = ::xla::Cast<::xla::HloSortInstruction>(instr); sort.dimensionAttr(builder_.getI64IntegerAttr(sort_instr->sort_dimension())); sort.is_stableAttr(builder_.getBoolAttr(sort_instr->is_stable())); TF_RETURN_IF_ERROR(::xla::HloFunctionImporter::ImportAsRegion( *sort_instr->called_computations()[0], &sort.comparator(), &builder_)); - return sort.getOperation(); + return sort; } Status LhloDialectEmitter::HandleSort(HloInstruction* instr) { diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h index bdc977616b1..4000fa01970 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h @@ -19,6 +19,7 @@ limitations under the License. #include "mlir/IR/Builders.h" // from @llvm-project #include "mlir/IR/Module.h" // from @llvm-project #include "mlir/IR/StandardTypes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/hlo_module.h" @@ -41,7 +42,7 @@ class LhloDialectEmitter : public ::xla::DfsHloVisitorWithDefault { builder_(module.getContext()), i8_type_(builder_.getIntegerType(8)) {} - ::xla::StatusOr EmitSortOp(::xla::HloInstruction* instr); + ::xla::StatusOr EmitSortOp(::xla::HloInstruction* instr); private: template diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 074fbd92b27..a19f9965fc7 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -254,6 +254,11 @@ cc_library( ":target_util", ":thunk", ":thunk_emitter", + "//tensorflow/compiler/mlir/hlo:lhlo", + "//tensorflow/compiler/mlir/xla:hlo_utils", + "//tensorflow/compiler/mlir/xla:mhlo_to_lhlo_with_xla", + "//tensorflow/compiler/mlir/xla:mlir_hlo_to_hlo", + "//tensorflow/compiler/mlir/xla:type_to_shape", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", @@ -291,6 +296,8 @@ cc_library( "@com_google_absl//absl/types:span", "@llvm-project//llvm:Core", "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:StandardOps", ], ) @@ -1159,6 +1166,7 @@ cc_library( ":target_constants", ":tree_reduction_rewriter", ":variadic_op_splitter", + "//tensorflow/compiler/mlir/xla:mhlo_to_lhlo_with_xla", "//tensorflow/compiler/xla:protobuf_util", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", @@ -1217,6 +1225,8 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@llvm-project//llvm:Core", + "@llvm-project//mlir:AllPassesAndDialectsNoRegistration", + "@llvm-project//mlir:IR", ], ) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index f5bf7476059..b796737e601 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -29,6 +29,8 @@ limitations under the License. #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" +#include "mlir/IR/Module.h" // from @llvm-project +#include "mlir/InitAllDialects.h" // from @llvm-project #include "tensorflow/compiler/xla/protobuf_util.h" #include "tensorflow/compiler/xla/service/algebraic_simplifier.h" #include "tensorflow/compiler/xla/service/all_reduce_combiner.h" @@ -516,15 +518,22 @@ static Status CompileModuleToLlvmIrImpl( DumpHloModuleIfEnabled(*hlo_module, **buffer_assignment, "after_optimizations"); + mlir::registerAllDialects(); + mlir::MLIRContext mlir_context; + IrEmitterContext ir_emitter_context( hlo_module, buffer_assignment->get(), platform_name, gpu_device_info, - cuda_compute_capability, profile_index_map, llvm_module->get()); + cuda_compute_capability, profile_index_map, &mlir_context, + llvm_module->get()); HloComputation* entry_computation = hlo_module->entry_computation(); - IrEmitterUnnested ir_emitter(hlo_module->config(), entry_computation, - &ir_emitter_context); - TF_RETURN_IF_ERROR(ir_emitter.EmitConstantGlobals()); + TF_ASSIGN_OR_RETURN( + auto ir_emitter, + IrEmitterUnnested::Create(hlo_module->config(), entry_computation, + &ir_emitter_context)); + + TF_RETURN_IF_ERROR(ir_emitter->EmitConstantGlobals()); { XLA_SCOPED_LOGGING_TIMER("GpuCompiler::RunBackend - IR emission"); @@ -533,9 +542,10 @@ static Status CompileModuleToLlvmIrImpl( ThunkSequence thunk_sequence; absl::Span order = hlo_schedule->ThunkLaunchOrder(); for (HloInstruction* instruction : order) { - TF_RETURN_IF_ERROR(instruction->Visit(&ir_emitter)); - TF_RETURN_IF_ERROR(ir_emitter.Postprocess(instruction)); - std::unique_ptr thunks = ir_emitter.ConsumeThunkSequence(); + TF_RETURN_IF_ERROR(instruction->Visit(ir_emitter.get())); + TF_RETURN_IF_ERROR(ir_emitter->Postprocess(instruction)); + std::unique_ptr thunks = + ir_emitter->ConsumeThunkSequence(); // The invariants between each input HloInstruction* and output Thunk* are // not all explicitly checked, but at least we can document them here: diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc index 5d38d1b727c..332db83b6ad 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc @@ -117,11 +117,11 @@ static bool HasMeaningfulName(llvm::Value* value) { return false; } -llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, - ShapeIndexView shape_index, - llvm::Value* ir_value) { - llvm::Type* pointee_type = llvm_ir::ShapeToIrType( - ShapeUtil::GetSubshape(hlo.shape(), shape_index), module_); +llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, + llvm::IRBuilder<>* b) { + llvm::Type* pointee_type = + llvm_ir::ShapeToIrType(shape, b->GetInsertBlock()->getModule()); + llvm::Type* dest_type = pointee_type->getPointerTo(); llvm::Value* typed_ir_value; @@ -129,9 +129,17 @@ llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, typed_ir_value = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast( llvm::cast(ir_value), dest_type); } else { - typed_ir_value = b_->CreatePointerBitCastOrAddrSpaceCast( + typed_ir_value = b->CreatePointerBitCastOrAddrSpaceCast( ir_value, pointee_type->getPointerTo()); } + return typed_ir_value; +} + +llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, + ShapeIndexView shape_index, + llvm::Value* ir_value) { + auto typed_ir_value = CastToTypedValue( + ShapeUtil::GetSubshape(hlo.shape(), shape_index), ir_value, b_); if (!HasMeaningfulName(ir_value)) { ir_value->setName(llvm_ir::IrName(&hlo, "raw")); } diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h index 5eef6727801..3813ec6c949 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h @@ -116,6 +116,10 @@ class HloToIrBindings { llvm::Value* temp_buffer_base_ = nullptr; }; +// Converts `ir_value` with type i8* to a typed LLVM Value* based on `shape`. +llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, + llvm::IRBuilder<>* b); + } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h index 9c43f80dc60..7d5a8d032e6 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_CONTEXT_H_ #include "llvm/IR/Module.h" +#include "mlir/IR/MLIRContext.h" // from @llvm-project #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/gpu/launch_dimensions.h" #include "tensorflow/compiler/xla/service/hlo_execution_profile.h" @@ -34,13 +35,15 @@ class IrEmitterContext { const HloModule* hlo_module, const BufferAssignment* buffer_assignment, std::string platform_name, GpuDeviceInfo gpu_device_info, absl::optional cuda_compute_capability, - const HloProfileIndexMap* profile_index_map, llvm::Module* llvm_module) + const HloProfileIndexMap* profile_index_map, + mlir::MLIRContext* mlir_context, llvm::Module* llvm_module) : hlo_module_(hlo_module), buffer_assignment_(buffer_assignment), platform_name_(std::move(platform_name)), gpu_device_info_(gpu_device_info), cuda_compute_capability_(cuda_compute_capability), profile_index_map_(profile_index_map), + mlir_context_(mlir_context), llvm_module_(llvm_module) {} // Disallow copy and assign. IrEmitterContext(const IrEmitterContext&) = delete; @@ -57,6 +60,7 @@ class IrEmitterContext { return cuda_compute_capability_; } const HloProfileIndexMap* profile_index_map() { return profile_index_map_; } + mlir::MLIRContext* mlir_context() { return mlir_context_; } llvm::Module* llvm_module() { return llvm_module_; } NameUniquer* name_uniquer() { return &name_uniquer_; } @@ -67,6 +71,7 @@ class IrEmitterContext { GpuDeviceInfo gpu_device_info_; absl::optional cuda_compute_capability_; const HloProfileIndexMap* profile_index_map_; + mlir::MLIRContext* mlir_context_; llvm::Module* llvm_module_; NameUniquer name_uniquer_; }; diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 61b78b6004d..f88c70b1a33 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -37,6 +37,13 @@ limitations under the License. #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Function.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" +#include "tensorflow/compiler/mlir/xla/hlo_utils.h" +#include "tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.h" +#include "tensorflow/compiler/mlir/xla/type_to_shape.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" @@ -144,13 +151,86 @@ void UpdateLaunchDimensions(const LaunchDimensions& launch_dims, Thunk* thunk, llvm::ConstantAsMetadata::get(threads_per_block_ir_value)})); } +const BufferAllocation* GetAllocation( + mlir::BlockArgument func_arg, const BufferAssignment& buffer_assignment) { + auto func_op = + mlir::cast(func_arg.getParentRegion()->getParentOp()); + int64 allocation_index = func_op + .getArgAttrOfType( + func_arg.getArgNumber(), "lmhlo.alloc") + .getValue() + .getSExtValue(); + return &buffer_assignment.GetAllocation(allocation_index); +} + +StatusOr GetAllocationSliceForMlir( + mlir::Value v, const BufferAssignment& buffer_assignment) { + int64 size = v.getType().cast().getSizeInBits() / 8; + + if (auto arg = v.dyn_cast()) { + return BufferAllocation::Slice(GetAllocation(arg, buffer_assignment), 0, + size); + } + + // We match two patterns here: + // * v = ViewOp(arg); + // * v = StaticMemRefCastOp(ViewOp(arg)); + if (mlir::Operation* op = v.getDefiningOp()) { + if (auto cast = mlir::dyn_cast(op)) { + mlir::Value source = cast.getViewSource(); + op = source.getDefiningOp(); + if (!op) { + return Unimplemented("StaticMemRefCastOp has to wrap an op"); + } + } + if (auto view = mlir::dyn_cast(op)) { + return BufferAllocation::Slice( + GetAllocation(view.source().cast(), + buffer_assignment), + mlir::cast(view.byte_shift().getDefiningOp()) + .value() + .cast() + .getValue() + .getSExtValue(), + size); + } + return Unimplemented("StaticMemRefCastOp has to wrap a ViewOp"); + } + + return Unimplemented( + "Operand has to be in the form of ViewOp(arg) or " + "StaticMemRefCastOp(ViewOp(arg))"); +} + +absl::string_view GetHloName(mlir::Operation* op) { + if (auto attr = op->getAttrOfType("name")) { + auto ref = attr.getValue(); + return absl::string_view(ref.data(), ref.size()); + } + return ""; +} + } // namespace IrEmitterUnnested::IrEmitterUnnested(const HloModuleConfig& hlo_module_config, const HloComputation* hlo_computation, IrEmitterContext* ir_emitter_context) : IrEmitter(hlo_module_config, ir_emitter_context, /*is_nested=*/false), - hlo_computation_(hlo_computation) {} + hlo_computation_(hlo_computation), + mlir_scratch_module_(mlir::ModuleOp::create( + mlir::Builder(ir_emitter_context->mlir_context()).getUnknownLoc())), + lhlo_scratch_emitter_(ir_emitter_context_->buffer_assignment(), + *hlo_computation, mlir_scratch_module_.get()) {} + +StatusOr> IrEmitterUnnested::Create( + const HloModuleConfig& hlo_module_config, + const HloComputation* hlo_computation, + IrEmitterContext* ir_emitter_context) { + auto emitter = std::unique_ptr(new IrEmitterUnnested( + hlo_module_config, hlo_computation, ir_emitter_context)); + TF_RETURN_IF_ERROR(emitter->lhlo_scratch_emitter_.Initialize()); + return std::move(emitter); +} Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { bindings_.UnbindAllLocalIrValues(); @@ -158,12 +238,11 @@ Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { } llvm::Function* IrEmitterUnnested::BuildKernelPrototype( - const HloInstruction& inst, - absl::Span args) { + absl::string_view name, absl::Span args) { // Compute the kernel name. The opcode string may contain "-" which cannot be // in a PTX function name, so sanitize the name before uniquifying it. string kernel_name = ir_emitter_context_->name_uniquer()->GetUniqueName( - llvm_ir::SanitizeFunctionName(inst.name())); + llvm_ir::SanitizeFunctionName(std::string(name))); // Create the kernel and add it to the module. llvm::Module* module = ir_emitter_context_->llvm_module(); @@ -359,7 +438,8 @@ Status IrEmitterUnnested::HandleDot(HloInstruction* dot) { } Status IrEmitterUnnested::HandleConditional(HloInstruction* conditional) { - AddThunkToThunkSequence(BuildConditionalThunk(conditional)); + TF_ASSIGN_OR_RETURN(auto thunk, BuildConditionalThunk(conditional)); + AddThunkToThunkSequence(std::move(thunk)); return Status::OK(); } @@ -1038,10 +1118,13 @@ Status IrEmitterUnnested::HandleWhile(HloInstruction* xla_while) { // Build ForThunk for conformant while loops, otherwise build WhileThunk. auto config = xla_while->backend_config(); if (config.ok() && config.ValueOrDie().has_known_trip_count()) { - AddThunkToThunkSequence( + TF_ASSIGN_OR_RETURN( + auto thunk, BuildForThunk(xla_while, config.ValueOrDie().known_trip_count().n())); + AddThunkToThunkSequence(std::move(thunk)); } else { - AddThunkToThunkSequence(BuildWhileThunk(xla_while)); + TF_ASSIGN_OR_RETURN(auto thunk, BuildWhileThunk(xla_while)); + AddThunkToThunkSequence(std::move(thunk)); } return Status::OK(); } @@ -1264,39 +1347,109 @@ Status IrEmitterUnnested::HandleSelect(HloInstruction* select) { return IrEmitter::HandleSelect(select); } +StatusOr +IrEmitterUnnested::GetOrCreateSubComputationFromRegion(mlir::Region* region) { + std::unique_ptr& module = scratch_nested_computations_[region]; + if (module == nullptr) { + xla::XlaComputation xla_computation; + TF_RETURN_IF_ERROR(ConvertRegionToComputation(region, &xla_computation)); + TF_ASSIGN_OR_RETURN(auto program_shape, xla_computation.GetProgramShape()); + TF_ASSIGN_OR_RETURN( + module, HloModule::CreateFromProto(xla_computation.proto(), + HloModuleConfig(program_shape))); + } + return module->entry_computation(); +} + Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { + MlirEmitterInput result; + + TF_ASSIGN_OR_RETURN(auto sort_op, lhlo_scratch_emitter_.EmitSortOp(sort)); + result.op = sort_op; + result.name = GetHloName(sort_op); + // The name in sort op has no semantics, and it's for debug only. If the name + // doesn't exist, we should use a namer (e.g. count-based). + // TODO(timshen): use a namer instead of relying on the HloInstruction names. + if (result.name.empty()) { + result.name = sort->name(); + } + const auto& buffer_assignment = ir_emitter_context_->buffer_assignment(); + auto& slice = result.extra_slice; + TF_ASSIGN_OR_RETURN(slice.buffer_slice, + buffer_assignment.GetUniqueSlice(sort, {})); + slice.written = true; + slice.shape = sort->shape(); + + result.thunk_info = GetThunkInfo(sort); + + return EmitMlirSort(result); +} + +Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { + const auto& buffer_assignment = ir_emitter_context_->buffer_assignment(); + auto sort_op = mlir::cast(input.op); + + int operand_count = sort_op.operands().size(); + std::vector operand_shapes(operand_count); + std::vector slices; + std::vector output_shapes(sort_op.output().size()); + + for (int i = 0; i < operand_count; i++) { + operand_shapes[i] = + TypeToShape(sort_op.operands()[i].getType().cast()); + } + + // Craft n + 1 slices, where the first n are output parameters, and the last + // is the on-device tuple storage. We don't need n operands because sorting + // kernels are always in-place. + for (int i = 0; i < operand_count; i++) { + output_shapes[i] = + TypeToShape(sort_op.output()[i].getType().cast()); + MlirBufferSlice slice; + TF_ASSIGN_OR_RETURN( + slice.buffer_slice, + GetAllocationSliceForMlir(sort_op.output()[i], buffer_assignment)); + slice.written = true; + slice.shape = operand_shapes[i]; + slices.push_back(slice); + } + slices.push_back(input.extra_slice); + std::vector> thunks; - Shape keys_shape = sort->operand(0)->shape(); - int64 dimension_to_sort = sort->dimensions(0); - for (int64 i = 0; i < sort->operand_count(); ++i) { - ShapeIndex shape_index = - sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({}); + + Shape keys_shape = operand_shapes[0]; + int64 dimension_to_sort = sort_op.dimension().getSExtValue(); + for (int64 i = 0; i < operand_count; ++i) { // We assume that the layout of all involved operands and outputs is the // same. - TF_RET_CHECK(LayoutUtil::LayoutsInShapesEqual(keys_shape, - sort->operand(i)->shape())); - TF_RET_CHECK(LayoutUtil::LayoutsInShapesEqual( - keys_shape, ShapeUtil::GetSubshape(sort->shape(), shape_index))); + TF_RET_CHECK( + LayoutUtil::LayoutsInShapesEqual(keys_shape, operand_shapes[i])); + TF_RET_CHECK( + LayoutUtil::LayoutsInShapesEqual(keys_shape, output_shapes[i])); // If possible, we share buffers. If that is not possible, we need to copy // the values, because the emitter does the sorting in-place. - auto destination_buffer = GetAllocationSlice(*sort, shape_index); - auto source_address = GetAllocationSlice(*sort->operand(i)); + TF_ASSIGN_OR_RETURN( + auto destination_buffer, + GetAllocationSliceForMlir(sort_op.output()[i], buffer_assignment)); + TF_ASSIGN_OR_RETURN( + auto source_address, + GetAllocationSliceForMlir(sort_op.operands()[i], buffer_assignment)); if (destination_buffer != source_address) { // TODO(b/26783907): Figure out why we never seem to share buffers for // key/value sort. - VLOG(2) << sort->name() << " requires initial D2D copy for operand " << i; + VLOG(2) << input.name << " requires initial D2D copy for operand " << i; thunks.push_back(absl::make_unique( Thunk::ThunkInfo(), /*source_address=*/source_address, /*destination_buffer=*/destination_buffer, - /*mem_size=*/ShapeUtil::ByteSizeOf(sort->operand(i)->shape()))); + /*mem_size=*/ShapeUtil::ByteSizeOf(operand_shapes[i]))); } } uint64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort); int64 num_stages = tensorflow::Log2Ceiling(dimension_to_sort_bound); - VLOG(2) << sort->name() << " requires " << num_stages << " stages."; + VLOG(2) << input.name << " requires " << num_stages << " stages."; CHECK_GE(1ULL << num_stages, dimension_to_sort_bound); CHECK_LT(1ULL << (num_stages - 1), dimension_to_sort_bound); @@ -1360,10 +1513,10 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { // we have not enough threads, or not enough shared memory. Also it does not // give a speedup if the tile size is < 128. int64 total_shared_memory_needed = 0; - for (int64 i = 0; i < sort->operand_count(); ++i) { + for (int64 i = 0; i < operand_count; ++i) { total_shared_memory_needed += - kTileSize * ShapeUtil::ByteSizeOfPrimitiveType( - sort->operand(i)->shape().element_type()); + kTileSize * + ShapeUtil::ByteSizeOfPrimitiveType(operand_shapes[i].element_type()); } bool no_tiling = kTileSize < 128 || @@ -1376,7 +1529,7 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { "kTileSize=%d < 128, " "kThreadsPerBlock=%d > threads_per_block_limit=%d, " "total_shared_memory_needed=%d > shared_memory_per_block=%d", - sort->name(), (no_tiling ? "won't" : "will"), kTileSize, kThreadsPerBlock, + input.name, (no_tiling ? "won't" : "will"), kTileSize, kThreadsPerBlock, ir_emitter_context_->gpu_device_info().threads_per_block_limit, total_shared_memory_needed, ir_emitter_context_->gpu_device_info().shared_memory_per_block); @@ -1384,37 +1537,38 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { uint64 num_blocks = CeilOfRatio(num_iterations, kThreadsPerBlock); LaunchDimensions tiled_launch_dimensions(num_blocks, kThreadsPerBlock); VLOG(2) << absl::StreamFormat("%s launch dims: %d blocks, %d threads/block", - sort->name(), num_blocks, kThreadsPerBlock); + input.name, num_blocks, kThreadsPerBlock); + std::vector ir_arrays; auto emit_kernel = [&](absl::Span xor_masks) { VLOG(2) << absl::StreamFormat( - "%s uses kernel for xor masks [%s]", sort->name(), + "%s uses kernel for xor masks [%s]", input.name, absl::StrJoin(xor_masks, ", ", [](std::string* out, int64 xor_mask) { absl::StrAppendFormat(out, "0x%x", xor_mask); })); - thunks.push_back( - BuildKernelThunk(sort, /*implements_whole_instruction=*/false)); + thunks.push_back(BuildKernelThunkForMlir(input.name, Thunk::ThunkInfo(), + slices, &ir_arrays)); LaunchDimensions launch_dimensions = xor_masks.size() > 1 ? tiled_launch_dimensions : standard_launch_dimensions; UpdateLaunchDimensions(launch_dimensions, thunks.back().get(), ir_emitter_context_->llvm_module()); std::vector values_arrays; - values_arrays.reserve(sort->operand_count()); - for (int64 i = 0; i < sort->operand_count(); ++i) { - ShapeIndex shape_index = - sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({}); - values_arrays.push_back(GetIrArray(*sort, *sort, shape_index)); + values_arrays.reserve(operand_count); + for (int64 i = 0; i < operand_count; ++i) { + values_arrays.push_back(ir_arrays[i]); } + TF_ASSIGN_OR_RETURN( + const HloComputation* comparator, + GetOrCreateSubComputationFromRegion(&sort_op.comparator())); return llvm_ir::EmitSortInPlace( - dimension_to_sort, values_arrays, IrName(sort), xor_masks, &b_, + dimension_to_sort, values_arrays, IrName(input.name), xor_masks, &b_, launch_dimensions, xor_masks.size() > 1 ? num_iterations_in_sort_dim : standard_num_iterations_in_sort_dim, kTileSize, [&](absl::Span operands, llvm::Value* output) { - return EmitCallToNestedComputation(*sort->to_apply(), operands, - output); + return EmitCallToNestedComputation(*comparator, operands, output); }); }; std::vector xor_masks; @@ -1441,17 +1595,18 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { TF_RETURN_IF_ERROR(emit_kernel(xor_masks)); } VLOG(2) << absl::StreamFormat( - "%s requires %d thunks (including any D2D copies)", sort->name(), + "%s requires %d thunks (including any D2D copies)", input.name, thunks.size()); - AddThunkToThunkSequence(absl::make_unique( - GetThunkInfo(sort), std::move(thunks))); - if (sort->operand_count() > 1) { + AddThunkToThunkSequence( + absl::make_unique(input.thunk_info, std::move(thunks))); + if (operand_count > 1) { // Emit the tuple as part of the last stage of sorting. // We are currently in the block sorted.in_bounds.after. b_.SetInsertPoint(b_.GetInsertBlock()->getTerminator()); - llvm_ir::EmitTuple(GetIrArray(*sort, *sort), - ConstructIrArrayForOutputs(*sort), &b_); + llvm_ir::EmitTuple( + ir_arrays[operand_count], + absl::MakeSpan(ir_arrays).subspan(0, ir_arrays.size() - 1), &b_); } return Status::OK(); } @@ -1589,24 +1744,6 @@ Status IrEmitterUnnested::HandleAfterAll(HloInstruction* after_all) { return Status::OK(); } -// Describes how to access a particular subshape for an HLO. For instance if -// `.hlo_index` is {1} and `.gte_index` is {3, 4} then buffer for `.instr` at -// ShapeIndex {1} (i.e. the buffer for the second tuple element of hlo) is found -// at `.buffer_slice`[3][4]. That is, `.slice` is a void***, which we -// dereference twice -- first at index 3, and then at index 4 -- to get the -// address of our buffer. -struct HloBufferSlice { - const HloInstruction* instr; - ShapeIndex hlo_index; - - // The root buffer to look at. - BufferAllocation::Slice buffer_slice; - - // Describes how to dereference starting at that buffer to get to the buffer - // in question. - ShapeIndex gte_index; -}; - // Figures out how to access the buffers for all subshapes of hlo's operands and // for hlo itself (i.e. all the buffers produced by HLO). // @@ -1715,22 +1852,22 @@ static std::vector GetHloBufferSlices( return result; } -std::unique_ptr IrEmitterUnnested::BuildKernelThunk( - const HloInstruction* inst, bool implements_whole_instruction) { - const BufferAssignment& buffer_assn = - ir_emitter_context_->buffer_assignment(); - - std::vector hlo_slices = - GetHloBufferSlices(inst, buffer_assn); +std::unique_ptr +IrEmitterUnnested::BuildKernelThunkFromBufferSlices( + absl::string_view name, Thunk::ThunkInfo thunk_info, + absl::Span slices, + std::function + bind_slice_to_ir_value) { + const auto& buffer_assn = ir_emitter_context_->buffer_assignment(); // Figure out which buffer allocations need to be passed as arguments to our - // kernel. This is simply all of the allocations referenced in hlo_slices, + // kernel. This is simply all of the allocations referenced in slices, // plus the XLA temp buffer (if we have it). We always include the temp // buffer because even if the kernel itself doesn't use it, a nested // subcomputation within the kernel (e.g. a kMap's computation) might. std::unordered_set buffers_needed; - for (const auto& hlo_buffer_slice : hlo_slices) { - buffers_needed.insert(hlo_buffer_slice.buffer_slice.allocation()); + for (auto* slice : slices) { + buffers_needed.insert(slice->buffer_slice.allocation()); } absl::optional temp_buffer; for (const BufferAllocation& alloc : buffer_assn.Allocations()) { @@ -1759,7 +1896,7 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( return a->index() < b->index(); }); - llvm::Function* kernel = BuildKernelPrototype(*inst, non_constant_buffers); + llvm::Function* kernel = BuildKernelPrototype(name, non_constant_buffers); // Build a map from a BufferAllocation to the corresponding argument in our // kernel. @@ -1793,24 +1930,19 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( // For each buffer our kernel might want to touch, bind it to a value derived // from our kernel args. - for (const auto& hlo_buffer_slice : hlo_slices) { - const HloInstruction* instr = hlo_buffer_slice.instr; - const ShapeIndex& index = hlo_buffer_slice.hlo_index; - const BufferAllocation::Slice& slice = hlo_buffer_slice.buffer_slice; - const ShapeIndex& gte_index = hlo_buffer_slice.gte_index; - - VLOG(3) << "Buffer for " << instr->ToString() << " at " << index.ToString() - << " is found in slice " << slice.ToString() << " at GTE index " - << gte_index.ToString(); + for (auto* slice : slices) { + const BufferAllocation::Slice& buffer_slice = slice->buffer_slice; + const ShapeIndex& gte_index = slice->gte_index; llvm::Value* loc; - if (slice.allocation()->is_constant()) { + if (buffer_slice.allocation()->is_constant()) { loc = ir_emitter_context_->llvm_module()->getGlobalVariable( - llvm_ir::ConstantBufferAllocationToGlobalName(*slice.allocation())); + llvm_ir::ConstantBufferAllocationToGlobalName( + *buffer_slice.allocation())); CHECK_NE(loc, nullptr); } else { - loc = InBoundsGEP(kernel_args.at(slice.allocation()), - {b_.getInt64(slice.offset())}); + loc = InBoundsGEP(kernel_args.at(buffer_slice.allocation()), + {b_.getInt64(buffer_slice.offset())}); } // If gte_index is nonempty, we have to dereference `loc` to get to the @@ -1822,7 +1954,7 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( loc = Load(InBoundsGEP(loc, {b_.getInt64(idx)})); } - bindings_.BindHloToIrValue(*instr, loc, index); + bind_slice_to_ir_value(slice, loc); } // Bind the temp buffer so that nested subcomputations can find it if they @@ -1834,9 +1966,66 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( llvm::ConstantPointerNull::get(b_.getInt8PtrTy())); } - return absl::make_unique( + return absl::make_unique(thunk_info, non_constant_buffers, + std::string(kernel->getName())); +} + +std::unique_ptr IrEmitterUnnested::BuildKernelThunk( + const HloInstruction* inst, bool implements_whole_instruction) { + std::vector hlo_slices = + GetHloBufferSlices(inst, ir_emitter_context_->buffer_assignment()); + + std::vector slice_ptrs; + slice_ptrs.reserve(hlo_slices.size()); + for (auto& slice : hlo_slices) { + slice_ptrs.push_back(&slice); + } + + return BuildKernelThunkFromBufferSlices( + inst->name(), implements_whole_instruction ? GetThunkInfo(inst) : Thunk::ThunkInfo(), - non_constant_buffers, std::string(kernel->getName())); + slice_ptrs, [this](const BufferSlice* slice, llvm::Value* value) { + const HloBufferSlice* hlo_buffer_slice = + static_cast(slice); + const HloInstruction* instr = hlo_buffer_slice->instr; + const ShapeIndex& index = hlo_buffer_slice->hlo_index; + VLOG(3) << "Buffer for " << instr->ToString() << " at " + << index.ToString() << " is found in slice " + << hlo_buffer_slice->buffer_slice.ToString() << " at GTE index " + << hlo_buffer_slice->gte_index.ToString(); + + bindings_.BindHloToIrValue(*instr, value, index); + }); +} + +std::unique_ptr IrEmitterUnnested::BuildKernelThunkForMlir( + absl::string_view name, Thunk::ThunkInfo thunk_info, + absl::Span slices, + std::vector* ir_arrays) { + absl::flat_hash_set buffers_written; + std::vector slice_ptrs; + slice_ptrs.reserve(slices.size()); + for (auto& slice : slices) { + slice_ptrs.push_back(&slice); + if (slice.written) { + buffers_written.insert(slice.buffer_slice); + } + } + + ir_arrays->clear(); + return BuildKernelThunkFromBufferSlices( + name, thunk_info, slice_ptrs, + [&](const BufferSlice* slice, llvm::Value* value) { + const auto& mlir_slice = static_cast(*slice); + + llvm_ir::IrArray ir_array( + CastToTypedValue(mlir_slice.shape, value, &b_), mlir_slice.shape); + if (!buffers_written.contains(slice->buffer_slice)) { + ir_array.MarkInvariantOverWholeProgram(&value->getContext()); + } + + ir_arrays->push_back(ir_array); + }); } StatusOr> IrEmitterUnnested::BuildInitializerThunk( @@ -2043,7 +2232,7 @@ Status CheckConditionalBuffersShareAllocation( } // namespace -std::unique_ptr IrEmitterUnnested::BuildWhileThunk( +StatusOr> IrEmitterUnnested::BuildWhileThunk( const HloInstruction* hlo) { // Check that all while-related buffers share an allocation. TF_CHECK_OK(CheckWhileBuffersShareAllocation( @@ -2051,24 +2240,26 @@ std::unique_ptr IrEmitterUnnested::BuildWhileThunk( // Generate thunk sequence for while 'condition'. HloComputation* condition = hlo->while_condition(); - IrEmitterUnnested ir_emitter_condition(hlo_module_config_, condition, - ir_emitter_context_); - TF_CHECK_OK(condition->Accept(&ir_emitter_condition)); + TF_ASSIGN_OR_RETURN(auto ir_emitter_condition, + IrEmitterUnnested::Create(hlo_module_config_, condition, + ir_emitter_context_)); + TF_RETURN_IF_ERROR(condition->Accept(ir_emitter_condition.get())); // Generate thunk sequence for while 'body'. HloComputation* body = hlo->while_body(); - IrEmitterUnnested ir_emitter_body(hlo_module_config_, body, - ir_emitter_context_); - TF_CHECK_OK(body->Accept(&ir_emitter_body)); + TF_ASSIGN_OR_RETURN( + auto ir_emitter_body, + IrEmitterUnnested::Create(hlo_module_config_, body, ir_emitter_context_)); + TF_RETURN_IF_ERROR(body->Accept(ir_emitter_body.get())); - return absl::make_unique( + return std::unique_ptr(new WhileThunk( GetThunkInfo(hlo), GetAllocationSlice(*condition->root_instruction()), // cond result - ir_emitter_condition.ConsumeThunkSequence(), - ir_emitter_body.ConsumeThunkSequence()); + ir_emitter_condition->ConsumeThunkSequence(), + ir_emitter_body->ConsumeThunkSequence())); } -std::unique_ptr IrEmitterUnnested::BuildForThunk( +StatusOr> IrEmitterUnnested::BuildForThunk( const HloInstruction* hlo, const int64 loop_limit) { // Check that all while-related buffers share an allocation. TF_CHECK_OK(CheckWhileBuffersShareAllocation( @@ -2076,15 +2267,16 @@ std::unique_ptr IrEmitterUnnested::BuildForThunk( // Generate thunk sequence for while 'body' (will be used a For loop body). HloComputation* body = hlo->while_body(); - IrEmitterUnnested ir_emitter_body(hlo_module_config_, body, - ir_emitter_context_); - TF_CHECK_OK(body->Accept(&ir_emitter_body)); + TF_ASSIGN_OR_RETURN( + auto ir_emitter_body, + IrEmitterUnnested::Create(hlo_module_config_, body, ir_emitter_context_)); + TF_RETURN_IF_ERROR(body->Accept(ir_emitter_body.get())); - return absl::make_unique(GetThunkInfo(hlo), loop_limit, - ir_emitter_body.ConsumeThunkSequence()); + return std::unique_ptr(new ForThunk( + GetThunkInfo(hlo), loop_limit, ir_emitter_body->ConsumeThunkSequence())); } -std::unique_ptr IrEmitterUnnested::BuildConditionalThunk( +StatusOr> IrEmitterUnnested::BuildConditionalThunk( const HloInstruction* hlo) { // Check that the buffers used in conditional are shared with the operands and // result appropriately. @@ -2096,15 +2288,17 @@ std::unique_ptr IrEmitterUnnested::BuildConditionalThunk( for (int j = 0; j < hlo->branch_count(); ++j) { branch_operands.emplace_back(GetAllocationSlice(*hlo->operand(j + 1))); HloComputation* branch_computation = hlo->branch_computation(j); - IrEmitterUnnested ir_emitter(hlo_module_config_, branch_computation, - ir_emitter_context_); - TF_CHECK_OK(branch_computation->Accept(&ir_emitter)); - branch_thunks.push_back(std::move(*ir_emitter.ConsumeThunkSequence())); + TF_ASSIGN_OR_RETURN( + auto ir_emitter, + IrEmitterUnnested::Create(hlo_module_config_, branch_computation, + ir_emitter_context_)); + TF_CHECK_OK(branch_computation->Accept(ir_emitter.get())); + branch_thunks.push_back(std::move(*ir_emitter->ConsumeThunkSequence())); } - return absl::make_unique( + return std::unique_ptr(new ConditionalThunk( GetThunkInfo(hlo), GetAllocationSlice(*hlo->operand(0)), branch_operands, - std::move(branch_thunks)); + std::move(branch_thunks))); } Status IrEmitterUnnested::EmitTargetElementLoopInThunk( diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h index 019fcdf21db..b9146dd8fae 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_ #include "absl/container/inlined_vector.h" +#include "tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h" #include "tensorflow/compiler/xla/service/gpu/ir_emitter.h" #include "tensorflow/compiler/xla/service/gpu/kernel_mapping_scheme.h" #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h" @@ -28,6 +29,40 @@ limitations under the License. namespace xla { namespace gpu { +struct BufferSlice { + // The root buffer to look at. + BufferAllocation::Slice buffer_slice; + + // Describes how to dereference starting at that buffer to get to the buffer + // in question. + ShapeIndex gte_index; +}; + +// Describes how to access a particular subshape for an HLO. For instance if +// `.hlo_index` is {1} and `.gte_index` is {3, 4} then buffer for `.instr` at +// ShapeIndex {1} (i.e. the buffer for the second tuple element of hlo) is +// found at `.buffer_slice`[3][4]. That is, `.slice` is a void***, which we +// dereference twice -- first at index 3, and then at index 4 -- to get the +// address of our buffer. +struct HloBufferSlice : public BufferSlice { + const HloInstruction* instr; + ShapeIndex hlo_index; +}; + +struct MlirBufferSlice : public BufferSlice { + // The buffer is modified by the kernel. + bool written; + + Shape shape; +}; + +struct MlirEmitterInput { + mlir::Operation* op; + absl::string_view name; + Thunk::ThunkInfo thunk_info; + MlirBufferSlice extra_slice; +}; + // Emits LLVM IR for an "unnested computation". // // An unnested computation is an HloComputation which you run by executing one @@ -89,12 +124,14 @@ class IrEmitterUnnested : public IrEmitter, const string& loop_name, llvm::Value* tile_height, llvm::Value* tile_width, KernelSupportLibrary* ksl)>; - IrEmitterUnnested(const HloModuleConfig& hlo_module_config, - const HloComputation* hlo_computation, - IrEmitterContext* ir_emitter_context); IrEmitterUnnested(const IrEmitterUnnested&) = delete; IrEmitterUnnested& operator=(const IrEmitterUnnested&) = delete; + static StatusOr> Create( + const HloModuleConfig& hlo_module_config, + const HloComputation* hlo_computation, + IrEmitterContext* ir_emitter_context); + // Transfers the ownship of thunk_sequence_ out. std::unique_ptr ConsumeThunkSequence() { return std::make_unique(std::move(thunk_sequence_)); @@ -124,6 +161,7 @@ class IrEmitterUnnested : public IrEmitter, Status HandleScatter(HloInstruction* scatter) override; Status HandleSelect(HloInstruction* select) override; Status HandleSort(HloInstruction* sort) override; + Status EmitMlirSort(MlirEmitterInput input); Status HandleTriangularSolve(HloInstruction* hlo) override; Status HandleTupleSelect(HloInstruction* tuple_select) override; Status HandleAllReduce(HloInstruction* crs) override; @@ -148,6 +186,10 @@ class IrEmitterUnnested : public IrEmitter, Status Postprocess(HloInstruction* hlo) override; private: + IrEmitterUnnested(const HloModuleConfig& hlo_module_config, + const HloComputation* hlo_computation, + IrEmitterContext* ir_emitter_context); + // Add a owning Thunk object to the thunk sequence. void AddThunkToThunkSequence(std::unique_ptr thunk) override { thunk_sequence_.emplace_back(std::move(thunk)); @@ -264,8 +306,7 @@ class IrEmitterUnnested : public IrEmitter, // Builds the prototype of the IR kernel for `inst` and adds it to the module. // This kernel takes as arguments pointers to the given buffer allocations. llvm::Function* BuildKernelPrototype( - const HloInstruction& inst, - absl::Span args); + absl::string_view name, absl::Span args); // Helper for writing extra outputs from inside a reduce kernel. Status EmitExtraOutputsForReduce( @@ -490,6 +531,12 @@ class IrEmitterUnnested : public IrEmitter, HloComputation* reducer, llvm::Type* element_type, llvm::Value* partial_result_address); + std::unique_ptr BuildKernelThunkFromBufferSlices( + absl::string_view name, Thunk::ThunkInfo thunk_info, + absl::Span slices, + std::function + bind_slice_to_ir_value); + // Returns a KernelThunk that invokes the kernel emitted for `inst`. The // caller needs to make sure `inst` outlives the lifetime of the returned // Thunk object. 'implements_whole_instruction' specifies whether this @@ -498,6 +545,11 @@ class IrEmitterUnnested : public IrEmitter, std::unique_ptr BuildKernelThunk( const HloInstruction* inst, bool implements_whole_instruction); + std::unique_ptr BuildKernelThunkForMlir( + absl::string_view name, Thunk::ThunkInfo thunk_info, + absl::Span slices, + std::vector* ir_arrays); + // Returns a thunk that, given a reduce or select-and-scatter op, // initializes its memory to the appropriate initial value. StatusOr> BuildInitializerThunk( @@ -505,17 +557,18 @@ class IrEmitterUnnested : public IrEmitter, // Returns a WhileThunk that invokes thunk sequences for 'condition' and // 'body' sub-computations of while instruction 'hlo'. - std::unique_ptr BuildWhileThunk(const HloInstruction* hlo); + StatusOr> BuildWhileThunk(const HloInstruction* hlo); // Returns a ForThunk which executes 'loop_limit' invocations of a thunk // sequence from the 'body' sub-computation of the while instruction 'hlo'. - std::unique_ptr BuildForThunk(const HloInstruction* hlo, - const int64 loop_limit); + StatusOr> BuildForThunk(const HloInstruction* hlo, + const int64 loop_limit); // Returns a ConditionalThunk which executes the thunk sequence for the // 'branch_computation' corresponding to the predicate/branch_index of the // given conditional instruction. - std::unique_ptr BuildConditionalThunk(const HloInstruction* hlo); + StatusOr> BuildConditionalThunk( + const HloInstruction* hlo); // Emits current thread id with the given type. // @@ -545,6 +598,9 @@ class IrEmitterUnnested : public IrEmitter, absl::optional thread_id_filter = absl::nullopt, absl::optional block_id_filter = absl::nullopt); + StatusOr GetOrCreateSubComputationFromRegion( + mlir::Region* region); + // Returns the last generated thunk. Thunk* LastThunk() const { return thunk_sequence_.back().get(); } @@ -555,6 +611,14 @@ class IrEmitterUnnested : public IrEmitter, // The HloComputation that this IrEmitter emits code for. const HloComputation* hlo_computation_; + + mlir::OwningModuleRef mlir_scratch_module_; + + // This is for cache-purpose only. It has no significant semantics. + mlir::LhloDialectEmitter lhlo_scratch_emitter_; + + absl::flat_hash_map> + scratch_nested_computations_; }; } // namespace gpu diff --git a/tensorflow/compiler/xla/service/gpu/tests/BUILD b/tensorflow/compiler/xla/service/gpu/tests/BUILD index a2bddd2d0d7..809b277317f 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/BUILD +++ b/tensorflow/compiler/xla/service/gpu/tests/BUILD @@ -458,6 +458,35 @@ xla_test( ], ) +tf_cc_test( + name = "sorting_test", + srcs = [ + "sorting_test.cc", + ], + tags = tf_cuda_tests_tags() + [ + "no_rocm", + ], + deps = [ + ":gpu_codegen_test", + "//tensorflow/compiler/xla:debug_options_flags", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:xla_proto_cc", + "//tensorflow/compiler/xla/service:gpu_plugin", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/compiler/xla/service:hlo_module_config", + "//tensorflow/compiler/xla/service:hlo_parser", + "//tensorflow/compiler/xla/service/gpu:gpu_executable", + "//tensorflow/compiler/xla/tests:filecheck", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:llvm_irgen_test_base", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/stream_executor/lib", + "@com_google_absl//absl/memory", + ], +) + tf_cc_binary( name = "hlo_to_llvm_ir", srcs = ["hlo_to_llvm_ir.cc"], diff --git a/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo b/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo index 272c9a25769..4d29a8df116 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo +++ b/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo @@ -8,162 +8,162 @@ compare { ROOT lt = pred[] compare(p.0.lhs, p.0.rhs), direction=LT } -// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) +// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 +// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 +// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 -// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] -// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 -// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] -// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 +// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] +// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 +// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] +// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: call void @compare(float* [[TMP12]], float* [[TMP13]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP14:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP14]], 0 +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: call void @region_0_4(float* [[TMP16]], float* [[TMP17]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP18]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP16:%.*]] = load float, float* [[TMP13]], align 4 -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: store float [[TMP16]], float* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] +// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define internal void @compare(float* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) +// CHECK: define internal void @region_0_4(float* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) // CHECK-NEXT: entry: -// CHECK-NEXT: [[LT_TYPED:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[P_0_LHS_TYPED]], align 4 -// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[P_0_RHS_TYPED]], align 4 +// CHECK-NEXT: [[COMPARE_3_TYPED:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[ARG_0_1_TYPED:%.*]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[ARG_1_2_TYPED:%.*]], align 4 // CHECK-NEXT: [[TMP2:%.*]] = fcmp olt float [[TMP0]], [[TMP1]] // CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i8 -// CHECK-NEXT: store i8 [[TMP3]], i8* [[LT_TYPED]], align 1 -// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[LT_TYPED]], align 1 -// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG]], align 1 +// CHECK-NEXT: store i8 [[TMP3]], i8* [[COMPARE_3_TYPED]], align 1 +// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[COMPARE_3_TYPED]], align 1 +// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG:%.*]], align 1 // CHECK-NEXT: ret void -// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) { +// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) { // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 +// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 +// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = xor i64 [[TMP4]], 3 -// CHECK-NEXT: [[TMP8:%.*]] = icmp slt i64 [[TMP4]], [[TMP7]] -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], 3 -// CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP8]], [[TMP9]] -// CHECK-NEXT: br i1 [[TMP10]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP11:%.*]] = xor i64 [[TMP8]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = icmp slt i64 [[TMP8]], [[TMP11]] +// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], 3 +// CHECK-NEXT: [[TMP14:%.*]] = and i1 [[TMP12]], [[TMP13]] +// CHECK-NEXT: br i1 [[TMP14]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: call void @compare(float* [[TMP11]], float* [[TMP12]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP13:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP13]], 0 +// CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP8]] +// CHECK-NEXT: call void @region_0_4(float* [[TMP15]], float* [[TMP16]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP17:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP17]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP14:%.*]] = load float, float* [[TMP11]], align 4 -// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: store float [[TMP14]], float* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = load float, float* [[TMP15]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP8]] +// CHECK-NEXT: store float [[TMP18]], float* [[TMP20]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) { +// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) { // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 +// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 +// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 -// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] -// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 -// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] -// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 +// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] +// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 +// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] +// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: call void @compare(float* [[TMP12]], float* [[TMP13]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP14:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP14]], 0 +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: call void @region_0_4(float* [[TMP16]], float* [[TMP17]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP18]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP16:%.*]] = load float, float* [[TMP13]], align 4 -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: store float [[TMP16]], float* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] +// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] ENTRY main { x = f32[2, 3] parameter(0) @@ -182,210 +182,198 @@ compare { ROOT lt = pred[] compare(p.1.lhs, p.1.rhs), direction=LT } -// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* -// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 -// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3]], i64 0 -// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* +// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 +// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 +// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 +// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 -// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] -// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 -// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] -// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP10]], 2 +// CHECK-NEXT: [[TMP14:%.*]] = xor i64 [[TMP13]], 1 +// CHECK-NEXT: [[TMP15:%.*]] = icmp slt i64 [[TMP13]], [[TMP14]] +// CHECK-NEXT: [[TMP16:%.*]] = icmp slt i64 [[TMP14]], 3 +// CHECK-NEXT: [[TMP17:%.*]] = and i1 [[TMP15]], [[TMP16]] +// CHECK-NEXT: br i1 [[TMP17]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: call void @compare(i32* [[TMP12]], i32* [[TMP13]], float* [[TMP14]], float* [[TMP15]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP16:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP16]], 0 +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP14]] +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP14]] +// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: call void @region_0_6(i32* [[TMP18]], i32* [[TMP19]], float* [[TMP20]], float* [[TMP21]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP22:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP22]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP13]], align 4 -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: store i32 [[TMP18]], i32* [[TMP20]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = load float, float* [[TMP14]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = load float, float* [[TMP15]], align 4 -// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP21]], float* [[TMP23]], align 4 -// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: store float [[TMP22]], float* [[TMP24]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: store i32 [[TMP23]], i32* [[TMP25]], align 4 +// CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP14]] +// CHECK-NEXT: store i32 [[TMP24]], i32* [[TMP26]], align 4 +// CHECK-NEXT: [[TMP27:%.*]] = load float, float* [[TMP20]], align 4 +// CHECK-NEXT: [[TMP28:%.*]] = load float, float* [[TMP21]], align 4 +// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: store float [[TMP27]], float* [[TMP29]], align 4 +// CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP14]] +// CHECK-NEXT: store float [[TMP28]], float* [[TMP30]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define internal void @compare(i32* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], i32* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) +// CHECK: define internal void @region_0_6(i32* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], i32* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) // CHECK-NEXT: entry: -// CHECK-NEXT: [[LT_TYPED:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[P_1_LHS_TYPED]], align 4 -// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[P_1_RHS_TYPED]], align 4 +// CHECK-NEXT: [[COMPARE_5_TYPED:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[ARG_2_3_TYPED:%.*]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[ARG_3_4_TYPED:%.*]], align 4 // CHECK-NEXT: [[TMP2:%.*]] = fcmp olt float [[TMP0]], [[TMP1]] // CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i8 -// CHECK-NEXT: store i8 [[TMP3]], i8* [[LT_TYPED]], align 1 -// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[LT_TYPED]], align 1 -// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG]], align 1 +// CHECK-NEXT: store i8 [[TMP3]], i8* [[COMPARE_5_TYPED]], align 1 +// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[COMPARE_5_TYPED]], align 1 +// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG:%.*]], align 1 // CHECK-NEXT: ret void -// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* -// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2:%.*]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3:%.*]], i64 0 -// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* +// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 +// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 +// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 +// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = xor i64 [[TMP4]], 3 -// CHECK-NEXT: [[TMP8:%.*]] = icmp slt i64 [[TMP4]], [[TMP7]] -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], 3 -// CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP8]], [[TMP9]] -// CHECK-NEXT: br i1 [[TMP10]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP13:%.*]] = xor i64 [[TMP10]], 3 +// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP10]], [[TMP13]] +// CHECK-NEXT: [[TMP15:%.*]] = icmp slt i64 [[TMP13]], 3 +// CHECK-NEXT: [[TMP16:%.*]] = and i1 [[TMP14]], [[TMP15]] +// CHECK-NEXT: br i1 [[TMP16]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: call void @compare(i32* [[TMP11]], i32* [[TMP12]], float* [[TMP13]], float* [[TMP14]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP15:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP15]], 0 +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP10]] +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP10]] +// CHECK-NEXT: call void @region_0_6(i32* [[TMP17]], i32* [[TMP18]], float* [[TMP19]], float* [[TMP20]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP21:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP21]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP11]], align 4 -// CHECK-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: store i32 [[TMP16]], i32* [[TMP18]], align 4 -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP13]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = load float, float* [[TMP14]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 -// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP21]], float* [[TMP23]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP10]] +// CHECK-NEXT: store i32 [[TMP22]], i32* [[TMP24]], align 4 +// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: store i32 [[TMP23]], i32* [[TMP25]], align 4 +// CHECK-NEXT: [[TMP26:%.*]] = load float, float* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP27:%.*]] = load float, float* [[TMP20]], align 4 +// CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP10]] +// CHECK-NEXT: store float [[TMP26]], float* [[TMP28]], align 4 +// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: store float [[TMP27]], float* [[TMP29]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* -// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2:%.*]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3:%.*]], i64 0 -// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* +// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 +// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 +// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 +// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: -// CHECK-NEXT: [[TMP7:%.*]] = bitcast [2 x [3 x i32]]* [[SORT_TYPED2]] to i8* -// CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[SORT_TYPED]], i64 0, i64 0 -// CHECK-NEXT: store i8* [[TMP7]], i8** [[TMP8]], align 8 -// CHECK-NEXT: [[TMP9:%.*]] = bitcast [2 x [3 x float]]* [[SORT_TYPED4]] to i8* -// CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[SORT_TYPED]], i64 0, i64 1 -// CHECK-NEXT: store i8* [[TMP9]], i8** [[TMP10]], align 8 +// CHECK-NEXT: [[TMP13:%.*]] = bitcast [2 x [3 x i32]]* [[TMP1]] to i8* +// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[TMP5]], i64 0, i64 0 +// CHECK-NEXT: store i8* [[TMP13]], i8** [[TMP14]], align 8 +// CHECK-NEXT: [[TMP15:%.*]] = bitcast [2 x [3 x float]]* [[TMP3]] to i8* +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[TMP5]], i64 0, i64 1 +// CHECK-NEXT: store i8* [[TMP15]], i8** [[TMP16]], align 8 // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP4]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 -// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] -// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 -// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] -// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP10]], 2 +// CHECK-NEXT: [[TMP18:%.*]] = xor i64 [[TMP17]], 1 +// CHECK-NEXT: [[TMP19:%.*]] = icmp slt i64 [[TMP17]], [[TMP18]] +// CHECK-NEXT: [[TMP20:%.*]] = icmp slt i64 [[TMP18]], 3 +// CHECK-NEXT: [[TMP21:%.*]] = and i1 [[TMP19]], [[TMP20]] +// CHECK-NEXT: br i1 [[TMP21]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP12]] -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP11]] -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP12]] -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP11]] -// CHECK-NEXT: call void @compare(i32* [[TMP16]], i32* [[TMP17]], float* [[TMP18]], float* [[TMP19]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP20:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP20]], 0 +// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP18]] +// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP17]] +// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP18]] +// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP17]] +// CHECK-NEXT: call void @region_0_6(i32* [[TMP22]], i32* [[TMP23]], float* [[TMP24]], float* [[TMP25]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP26:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP26]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP21:%.*]] = load i32, i32* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP11]] -// CHECK-NEXT: store i32 [[TMP21]], i32* [[TMP23]], align 4 -// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP12]] -// CHECK-NEXT: store i32 [[TMP22]], i32* [[TMP24]], align 4 -// CHECK-NEXT: [[TMP25:%.*]] = load float, float* [[TMP18]], align 4 -// CHECK-NEXT: [[TMP26:%.*]] = load float, float* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP11]] -// CHECK-NEXT: store float [[TMP25]], float* [[TMP27]], align 4 -// CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP12]] -// CHECK-NEXT: store float [[TMP26]], float* [[TMP28]], align 4 +// CHECK-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP22]], align 4 +// CHECK-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP23]], align 4 +// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP17]] +// CHECK-NEXT: store i32 [[TMP27]], i32* [[TMP29]], align 4 +// CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP18]] +// CHECK-NEXT: store i32 [[TMP28]], i32* [[TMP30]], align 4 +// CHECK-NEXT: [[TMP31:%.*]] = load float, float* [[TMP24]], align 4 +// CHECK-NEXT: [[TMP32:%.*]] = load float, float* [[TMP25]], align 4 +// CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP17]] +// CHECK-NEXT: store float [[TMP31]], float* [[TMP33]], align 4 +// CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP18]] +// CHECK-NEXT: store float [[TMP32]], float* [[TMP34]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] ENTRY main { x = s32[2, 3] parameter(0) diff --git a/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc b/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc new file mode 100644 index 00000000000..197a0c6cfeb --- /dev/null +++ b/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc @@ -0,0 +1,71 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/xla/service/gpu/gpu_executable.h" +#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_module_config.h" +#include "tensorflow/compiler/xla/service/hlo_parser.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/tests/filecheck.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/xla.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/stream_executor/lib/statusor.h" + +namespace xla { +namespace gpu { + +namespace { + +class SortingTest : public GpuCodegenTest { + protected: + HloModuleConfig ConfigWithoutLayoutAssignment() { + HloModuleConfig config; + auto debug_options = HloTestBase::GetDebugOptionsForTest(); + // Disable layout_assignment to use the preassigned layouts. + debug_options.add_xla_disable_hlo_passes("layout-assignment"); + config.set_debug_options(debug_options); + return config; + } +}; + +TEST_F(SortingTest, Regression1) { + const char* hlo_text = R"( +HloModule TestModule + +compare { + p.0.lhs = f32[] parameter(0) + p.0.rhs = f32[] parameter(1) + ROOT lt = pred[] compare(p.0.lhs, p.0.rhs), direction=LT +} + +ENTRY TestComputation { + x = f32[3, 2]{1, 0} parameter(0) + x.copy = f32[3, 2]{0, 1} copy(x) + ROOT sort = f32[3, 2]{0, 1} sort(x.copy), dimensions={1}, to_apply=compare +} + +)"; + + EXPECT_TRUE(RunAndCompareNoHloPasses(hlo_text, ErrorSpec{1e-5, 1e-5})); +} + +} // namespace +} // namespace gpu +} // namespace xla diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc index b01ae2efe43..2963d546380 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc @@ -415,9 +415,10 @@ llvm::Instruction* AddRangeMetadata(int64 lower, int64 upper, return inst; } -string IrName(string a) { - a.erase(std::remove(a.begin(), a.end(), '%'), a.end()); - return a; +string IrName(absl::string_view a) { + std::string s(a); + s.erase(std::remove(s.begin(), s.end(), '%'), s.end()); + return s; } string IrName(absl::string_view a, absl::string_view b) { diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h index 642965b6470..c0a55e4da33 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h @@ -87,7 +87,7 @@ string DumpModuleToString(const llvm::Module& module); // - joining all of the nonempty inputs by '.', and then // - removing all '%'s. // -string IrName(string a); +string IrName(absl::string_view a); string IrName(absl::string_view a, absl::string_view b); string IrName(const HloInstruction* a, absl::string_view b = ""); From d10c814ce1e58831c8c5a9869555eb326d67fd2f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 16:25:19 -0700 Subject: [PATCH 0789/1017] TF NumPy: point to guide from API documentation. PiperOrigin-RevId: 325911592 Change-Id: I64f9c34badb5b5484be4527a86df218bf910a79f --- tensorflow/python/ops/numpy_ops/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow/python/ops/numpy_ops/__init__.py b/tensorflow/python/ops/numpy_ops/__init__.py index 5cc5cf5ac85..633b74b4a78 100644 --- a/tensorflow/python/ops/numpy_ops/__init__.py +++ b/tensorflow/python/ops/numpy_ops/__init__.py @@ -24,6 +24,9 @@ NumPy" section. ## Getting Started +Please also see [TensorFlow NumPy Guide]( +https://www.tensorflow.org/guide/tf_numpy). + In the code snippets below, we will assume that `tf.experimental.numpy` is imported as `tnp` and NumPy is imported as `np` From 82c043fee561dccc4ede0e44e2a09c9c744a5b9d Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Mon, 10 Aug 2020 17:06:51 -0700 Subject: [PATCH 0790/1017] Create dummy topology types for libtpu C API. This gives us better typechecking than just using void*. PiperOrigin-RevId: 325919251 Change-Id: Idc375ed151a2b0b2cf7d62b7be9cf2fafe48e934 --- tensorflow/stream_executor/tpu/BUILD | 1 + tensorflow/stream_executor/tpu/c_api_decl.h | 4 ++ tensorflow/stream_executor/tpu/c_api_defn.h | 4 +- .../stream_executor/tpu/tpu_executor_c_api.h | 45 ++++++++++--------- .../tpu/tpu_platform_interface.h | 5 ++- .../stream_executor/tpu/tpu_topology.cc | 4 +- tensorflow/stream_executor/tpu/tpu_topology.h | 15 ++++--- 7 files changed, 46 insertions(+), 32 deletions(-) diff --git a/tensorflow/stream_executor/tpu/BUILD b/tensorflow/stream_executor/tpu/BUILD index a8178404dff..207984e0c89 100644 --- a/tensorflow/stream_executor/tpu/BUILD +++ b/tensorflow/stream_executor/tpu/BUILD @@ -294,6 +294,7 @@ cc_library( hdrs = ["tpu_platform_interface.h"], visibility = ["//visibility:public"], deps = [ + ":c_api_decl", ":tpu_topology_external", "//tensorflow/core:lib", "//tensorflow/stream_executor", diff --git a/tensorflow/stream_executor/tpu/c_api_decl.h b/tensorflow/stream_executor/tpu/c_api_decl.h index c42423c232f..bca5f254ad1 100644 --- a/tensorflow/stream_executor/tpu/c_api_decl.h +++ b/tensorflow/stream_executor/tpu/c_api_decl.h @@ -253,6 +253,10 @@ typedef struct XLA_ComputationPlacer XLA_ComputationPlacer; typedef void (*XLA_CallbackFn)(void*); typedef void (*XLA_StatusCallbackFn)(void*, SE_Status*); + +typedef struct SE_TpuTopology SE_TpuTopology; +typedef struct SE_TpuTopology_Core SE_TpuTopology_Core; +typedef struct SE_TpuTopology_Core SE_TpuTopology_Host; } #endif // TENSORFLOW_STREAM_EXECUTOR_TPU_C_API_DECL_H_ diff --git a/tensorflow/stream_executor/tpu/c_api_defn.h b/tensorflow/stream_executor/tpu/c_api_defn.h index 1599f1f266a..62c02e2de48 100644 --- a/tensorflow/stream_executor/tpu/c_api_defn.h +++ b/tensorflow/stream_executor/tpu/c_api_defn.h @@ -63,8 +63,10 @@ struct SE_DeviceOptions { stream_executor::DeviceOptions options; }; +// Ignored -- these are just used to enforce the interface types struct XLA_TransferManager {}; - struct XLA_ComputationPlacer {}; +struct SE_TpuTopology {}; +struct SE_TpuTopology_Core {}; #endif // TENSORFLOW_STREAM_EXECUTOR_TPU_C_API_DEFN_H_ diff --git a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h index 1dcb3eaf244..c498244cc6e 100644 --- a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h +++ b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h @@ -39,8 +39,8 @@ SE_PlatformId TpuPlatform_Id(SE_Platform* platform); int64_t TpuPlatform_VisibleDeviceCount(SE_Platform* platform); int64_t TpuPlatform_TpuMemoryLimit(SE_Platform* platform); bool TpuPlatform_ShouldRegisterTpuDeviceToDeviceCopy(SE_Platform* platform); -void* TpuPlatform_GetTopologyPtr(SE_Platform* platform); -void* TpuPlatform_GetHostLocation(SE_Platform* platform); +SE_TpuTopology* TpuPlatform_GetTopologyPtr(SE_Platform* platform); +SE_TpuTopology_Host* TpuPlatform_GetHostLocation(SE_Platform* platform); void TpuExecutor_Init(SE_StreamExecutor* executor, int device_ordinal, SE_DeviceOptions* device_options, SE_Status* status); @@ -193,29 +193,32 @@ void TpuTransferManager_FreeBuffers(char** buffers_array, int64_t* buffers_size, XLA_ComputationPlacer* TpuComputationPlacer_New(); void TpuComputationPlacer_Free(XLA_ComputationPlacer* placer); -int TpuTopology_LogicalDevicesPerHost(void* tpu_topology, +int TpuTopology_LogicalDevicesPerHost(SE_TpuTopology* tpu_topology, TpuCoreTypeEnum tpu_core_type); -int TpuTopology_LogicalDevicesPerChip(void* tpu_topology, +int TpuTopology_LogicalDevicesPerChip(SE_TpuTopology* tpu_topology, TpuCoreTypeEnum tpu_core_type); -int TpuTopology_ChipBounds_X(void* tpu_topology); -int TpuTopology_ChipBounds_Y(void* tpu_topology); -int TpuTopology_ChipBounds_Z(void* tpu_topology); -bool TpuTopology_HasChip(void* tpu_topology, int x, int y, int z); -void* TpuTopology_Core(void* tpu_topology, int x, int y, int z, - TpuCoreTypeEnum tpu_core_type, int index); -int TpuTopology_NumCores(void* tpu_topology, TpuCoreTypeEnum tpu_core_type); +int TpuTopology_ChipBounds_X(SE_TpuTopology* tpu_topology); +int TpuTopology_ChipBounds_Y(SE_TpuTopology* tpu_topology); +int TpuTopology_ChipBounds_Z(SE_TpuTopology* tpu_topology); +bool TpuTopology_HasChip(SE_TpuTopology* tpu_topology, int x, int y, int z); +SE_TpuTopology_Core* TpuTopology_Core(SE_TpuTopology* tpu_topology, int x, + int y, int z, + TpuCoreTypeEnum tpu_core_type, int index); +int TpuTopology_NumCores(SE_TpuTopology* tpu_topology, + TpuCoreTypeEnum tpu_core_type); // 'cores' should be a preallocated array of size TpuTopology_NumCores. -void TpuTopology_Cores(void* tpu_topology, TpuCoreTypeEnum tpu_core_type, - void** cores); -int TpuTopology_IdForHost(void* tpu_topology, int x, int y, int z); -void TpuCoreLocation_ChipCoordinates(void* tpu_core_location, int* x, int* y, - int* z); -void TpuCoreLocation_HostCoordinates(void* tpu_core_location, int* x, int* y, - int* z); -int TpuCoreLocation_Index(void* tpu_core_location); -int TpuCoreLocation_Id(void* tpu_core_location); +void TpuTopology_Cores(SE_TpuTopology* tpu_topology, + TpuCoreTypeEnum tpu_core_type, + SE_TpuTopology_Core** cores); +int TpuTopology_IdForHost(SE_TpuTopology* tpu_topology, int x, int y, int z); +void TpuCoreLocation_ChipCoordinates(SE_TpuTopology_Core* tpu_core_location, + int* x, int* y, int* z); +void TpuCoreLocation_HostCoordinates(SE_TpuTopology_Core* tpu_core_location, + int* x, int* y, int* z); +int TpuCoreLocation_Index(SE_TpuTopology_Core* tpu_core_location); +int TpuCoreLocation_Id(SE_TpuTopology_Core* tpu_core_location); -int TpuHostLocation_Id(void* tpu_host_location); +int TpuHostLocation_Id(SE_TpuTopology_Host* tpu_host_location); // C API for XLA::Compiler interface diff --git a/tensorflow/stream_executor/tpu/tpu_platform_interface.h b/tensorflow/stream_executor/tpu/tpu_platform_interface.h index a0a3b444550..936de8d5c34 100644 --- a/tensorflow/stream_executor/tpu/tpu_platform_interface.h +++ b/tensorflow/stream_executor/tpu/tpu_platform_interface.h @@ -18,12 +18,15 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow/stream_executor/platform.h" +#include "tensorflow/stream_executor/tpu/c_api_decl.h" #include "tensorflow/stream_executor/tpu/tpu_topology.h" namespace tensorflow { namespace tpu { -typedef void* TpuTopologyPtr; +// TODO(skyewm): get rid of TpuTopologyPtr and either use SE_TpuTopology* or +// return a TpuTopologyExternal. +typedef SE_TpuTopology* TpuTopologyPtr; class TpuPlatformInterface : public stream_executor::Platform { public: diff --git a/tensorflow/stream_executor/tpu/tpu_topology.cc b/tensorflow/stream_executor/tpu/tpu_topology.cc index cfcea2dc944..6c885b229ec 100644 --- a/tensorflow/stream_executor/tpu/tpu_topology.cc +++ b/tensorflow/stream_executor/tpu/tpu_topology.cc @@ -79,12 +79,12 @@ std::vector TpuTopologyExternal::cores( TpuCoreTypeEnum core_type) const { int num_cores = tpu::ExecutorApiFn()->TpuTopology_NumCoresFn(topology_, core_type); - std::vector core_ptrs(num_cores); + std::vector core_ptrs(num_cores); tpu::ExecutorApiFn()->TpuTopology_CoresFn(topology_, core_type, core_ptrs.data()); std::vector result; result.reserve(num_cores); - for (void* ptr : core_ptrs) { + for (SE_TpuTopology_Core* ptr : core_ptrs) { result.emplace_back(ptr); } return result; diff --git a/tensorflow/stream_executor/tpu/tpu_topology.h b/tensorflow/stream_executor/tpu/tpu_topology.h index 3b0c4c5aa20..07e9afc7d81 100644 --- a/tensorflow/stream_executor/tpu/tpu_topology.h +++ b/tensorflow/stream_executor/tpu/tpu_topology.h @@ -33,27 +33,27 @@ struct TpuDimensionsExternal { class TpuCoreLocationExternal { public: TpuCoreLocationExternal() : core_location_(nullptr) {} - explicit TpuCoreLocationExternal(void* core_location) + explicit TpuCoreLocationExternal(SE_TpuTopology_Core* core_location) : core_location_(core_location) {} TpuDimensionsExternal chip_coordinates() const; TpuDimensionsExternal host_coordinates() const; int32 index() const; int32 Id() const; - void* impl() const { return core_location_; } + SE_TpuTopology_Core* impl() const { return core_location_; } private: - void* core_location_; + SE_TpuTopology_Core* core_location_; }; class TpuHostLocationExternal { public: - explicit TpuHostLocationExternal(void* host_location) + explicit TpuHostLocationExternal(SE_TpuTopology_Host* host_location) : host_location_(host_location) {} int32 Id() const; private: - void* host_location_; + SE_TpuTopology_Host* host_location_; }; struct TpuTopologyChipBoundsExternal { @@ -64,7 +64,8 @@ struct TpuTopologyChipBoundsExternal { class TpuTopologyExternal { public: - explicit TpuTopologyExternal(void* topology) : topology_(topology) {} + explicit TpuTopologyExternal(SE_TpuTopology* topology) + : topology_(topology) {} int32 LogicalDevicesPerHost(TpuCoreTypeEnum core_type) const; int32 LogicalDevicesPerChip(TpuCoreTypeEnum core_type) const; TpuTopologyChipBoundsExternal chip_bounds() const; @@ -75,7 +76,7 @@ class TpuTopologyExternal { int IdForHost(TpuDimensionsExternal host) const; private: - void* topology_; + SE_TpuTopology* topology_; }; } // namespace tpu From 956c8578c338f85ad0ababee2274b3d7db1a777c Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Mon, 10 Aug 2020 17:14:19 -0700 Subject: [PATCH 0791/1017] Clone Windows job nightly job suite for CUDA11 testing PiperOrigin-RevId: 325920384 Change-Id: I136dfc361c01a9efa3e4ef2f87fca2d9c041314a --- .../rel/windows_cuda11/cpu_libtensorflow.bat | 20 ++++++++++++++++ .../ci_build/rel/windows_cuda11/cpu_py35.bat | 20 ++++++++++++++++ .../ci_build/rel/windows_cuda11/cpu_py36.bat | 20 ++++++++++++++++ .../ci_build/rel/windows_cuda11/cpu_py37.bat | 20 ++++++++++++++++ .../ci_build/rel/windows_cuda11/cpu_py38.bat | 21 +++++++++++++++++ .../rel/windows_cuda11/gpu_libtensorflow.bat | 20 ++++++++++++++++ .../rel/windows_cuda11/gpu_pip_on_cpu.bat | 21 +++++++++++++++++ .../ci_build/rel/windows_cuda11/gpu_py35.bat | 23 +++++++++++++++++++ .../ci_build/rel/windows_cuda11/gpu_py36.bat | 23 +++++++++++++++++++ .../ci_build/rel/windows_cuda11/gpu_py37.bat | 23 +++++++++++++++++++ .../ci_build/rel/windows_cuda11/gpu_py38.bat | 23 +++++++++++++++++++ 11 files changed, 234 insertions(+) create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat new file mode 100644 index 00000000000..67941234b15 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\bazel\run_libtensorflow.bat || exit /b 1 + +copy lib_package %TF_ARTIFACTS_DIR%\lib_package diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat new file mode 100644 index 00000000000..175917d7cad --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python35 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat new file mode 100644 index 00000000000..85b75053eff --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python36 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat new file mode 100644 index 00000000000..d8a6673ba4c --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python37 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat new file mode 100644 index 00000000000..86adcda0bb9 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat @@ -0,0 +1,21 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python38 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" + diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat new file mode 100644 index 00000000000..8ab78bef3ca --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\bazel\run_libtensorflow.bat || exit /b + +copy lib_package %TF_ARTIFACTS_DIR%\lib_package diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat new file mode 100644 index 00000000000..213de532069 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat @@ -0,0 +1,21 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python36 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\integration\gpu_pip_on_cpu\run.bat + diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat new file mode 100644 index 00000000000..86c118b2f83 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat @@ -0,0 +1,23 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python35 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" +bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat new file mode 100644 index 00000000000..cc4f84afbee --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat @@ -0,0 +1,23 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python36 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" +bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh \ No newline at end of file diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat new file mode 100644 index 00000000000..5fa798e3eb8 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat @@ -0,0 +1,23 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python37 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" +bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh \ No newline at end of file diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat new file mode 100644 index 00000000000..fa1fc131145 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat @@ -0,0 +1,23 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python38 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" + +for %%a in ("%~dp0\.") do set "PARENT_DIR=%%~nxa" +bash -l tensorflow\tools\ci_build\release\windows\%PARENT_DIR%\release_pip_rename.sh From f621eebcfff1b506ac48716c763988196ef7d881 Mon Sep 17 00:00:00 2001 From: Steve Chien Date: Mon, 10 Aug 2020 17:19:05 -0700 Subject: [PATCH 0792/1017] Add DP-enabled binary-class head and multi-class heads for Estimator. PiperOrigin-RevId: 325921076 Change-Id: I958d492afb9b0d53300559d1880372b27400154e --- tensorflow/python/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index a8a70566ab7..745fed375b8 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -102,6 +102,7 @@ py_library( "//tensorflow/tools/api/tests:__pkg__", "//tensorflow/tools/compatibility/update:__pkg__", "//tensorflow_estimator:__subpackages__", + "//third_party/py/tensorflow_privacy:__subpackages__", # TODO(b/163395075): remove when fixed ], deps = [ ":layers", From 36d55f1c562e89d0e1eddc0a7d4d79049cabb466 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 17:23:57 -0700 Subject: [PATCH 0793/1017] [Profiler] Fix a bug in the memory profiler that caused the disappearance of the TF ops in the memory breakdown table. PiperOrigin-RevId: 325921811 Change-Id: I22877b935fa9edcbd66be5489c0b7da29d0276fc --- tensorflow/core/profiler/convert/BUILD | 1 + .../convert/xplane_to_memory_profile.cc | 94 ++++++++++++++++--- 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/tensorflow/core/profiler/convert/BUILD b/tensorflow/core/profiler/convert/BUILD index 66e027ed8ac..2274a227f4d 100644 --- a/tensorflow/core/profiler/convert/BUILD +++ b/tensorflow/core/profiler/convert/BUILD @@ -514,6 +514,7 @@ cc_library( "//tensorflow/core/profiler/utils:xplane_visitor", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:optional", diff --git a/tensorflow/core/profiler/convert/xplane_to_memory_profile.cc b/tensorflow/core/profiler/convert/xplane_to_memory_profile.cc index 9a5130f63be..3b67124ef27 100644 --- a/tensorflow/core/profiler/convert/xplane_to_memory_profile.cc +++ b/tensorflow/core/profiler/convert/xplane_to_memory_profile.cc @@ -24,11 +24,13 @@ limitations under the License. #include "absl/algorithm/container.h" #include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/profiler/protobuf/memory_profile.pb.h" @@ -424,23 +426,86 @@ void ProcessActiveAllocations(int64 peak_bytes_profile_step_id, << memory_profile->active_allocations_size(); } +struct Sample { + int64 orig_index; // original index to the snapshot. + MemoryProfileSnapshot* snapshot; +}; + +// This function samples max_num_snapshots from snapshots. We first keep the +// snapshots referenced by active_allocations in the samples. After this, if +// there is still room for more samples, we pick more from snapshots into the +// samples. Then, we sort the samples in time (so that they can be correctly +// displayed on the timeline). Finally, we need to adjust the original indices +// (to snapshots) in active_allocations to the new indices in the samples. void SampleSnapshots( int64 max_num_snapshots, - protobuf::RepeatedPtrField* snapshots) { + protobuf::RepeatedPtrField* snapshots, + protobuf::RepeatedPtrField* active_allocations) { if (snapshots->size() <= max_num_snapshots) return; - absl::c_partial_sort( - *snapshots, snapshots->begin() + max_num_snapshots, - [](const MemoryProfileSnapshot& a, const MemoryProfileSnapshot& b) { - return a.aggregation_stats().free_memory_bytes() < - b.aggregation_stats().free_memory_bytes(); - }); - snapshots->erase(snapshots->begin() + max_num_snapshots, snapshots->end()); - // Sort the memory_profile_snapshots by time_offset_ps (ascending) after - // sampling. - absl::c_sort(*snapshots, [](const MemoryProfileSnapshot& a, - const MemoryProfileSnapshot& b) { - return a.time_offset_ps() < b.time_offset_ps(); + + std::vector samples; + + // First, puts the snapshots referenced by active_allocations in samples[]. + absl::flat_hash_set allocation_snapshot_indices; + for (const auto& allocation : *active_allocations) { + auto orig_index = allocation.snapshot_index(); + if (orig_index < 0) continue; + allocation_snapshot_indices.insert(orig_index); + samples.push_back({orig_index, &(*snapshots)[orig_index]}); + if (allocation_snapshot_indices.size() >= max_num_snapshots) break; + } + + // Second, extracts remaining samples from snapshots. + int64 num_samples_remained = + max_num_snapshots - allocation_snapshot_indices.size(); + if (num_samples_remained > 0) { + std::vector remaining; + for (int64 i = 0; i < snapshots->size(); i++) { + if (allocation_snapshot_indices.contains(i)) continue; + // snapshots[i] is not yet sampled; put it in remaining[] for further + // consideration. + remaining.push_back({i, &(*snapshots)[i]}); + } + // Moves the num_samples_remained snapshots with least free bytes to the + // beginning of remaining[]. + absl::c_partial_sort( + remaining, remaining.begin() + num_samples_remained, + [](const Sample& a, const Sample& b) { + return a.snapshot->aggregation_stats().free_memory_bytes() < + b.snapshot->aggregation_stats().free_memory_bytes(); + }); + // Copies the first num_samples_remained in remaining[] to samples[]. + for (int64 i = 0; i < num_samples_remained; i++) + samples.push_back(remaining[i]); + } + + // Third, sorts samples[] in ascending order of time_offset_ps. + absl::c_sort(samples, [](const Sample& a, const Sample& b) { + return a.snapshot->time_offset_ps() < b.snapshot->time_offset_ps(); }); + + // Fourth, constructs a map from the original snapshot index to samples index. + absl::flat_hash_map index_map; + for (int64 i = 0; i < samples.size(); i++) { + index_map[samples[i].orig_index] = i; + } + + // Fifth, changes the original snapshot indices in active_allocations to the + // sample indices. + for (auto& allocation : *active_allocations) { + auto orig_index = allocation.snapshot_index(); + if (orig_index < 0) continue; + auto new_index = gtl::FindWithDefault(index_map, orig_index, -1); + allocation.set_snapshot_index(new_index); + } + + // Sixth, replaces *snapshot by samples[] + protobuf::RepeatedPtrField new_snapshots; + new_snapshots.Reserve(samples.size()); + for (const auto& sample : samples) { + *new_snapshots.Add() = std::move(*sample.snapshot); + } + *snapshots = std::move(new_snapshots); } // Post-process the memory profile to correctly update proto fields, and break @@ -478,7 +543,8 @@ void ProcessMemoryProfileProto(int64 max_num_snapshots, .peak_bytes_in_use(), allocator_memory_profile); ProcessActiveAllocations(peak_step_id, allocator_memory_profile); - SampleSnapshots(max_num_snapshots, snapshots); + SampleSnapshots(max_num_snapshots, snapshots, + allocator_memory_profile->mutable_active_allocations()); } } From 6fb229b3e7ab5f0f45397fccabf5105266b74f68 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 17:24:25 -0700 Subject: [PATCH 0794/1017] Re-apply github tensorflow/pull/22264/commits/51d72a7d7f74784b68916819edd04e890b36f957 PiperOrigin-RevId: 325921879 Change-Id: I703edc9e0f381d64784027eb9457bc10f5e5aef8 --- tensorflow/python/saved_model/model_utils/BUILD | 2 ++ .../python/saved_model/model_utils/export_output.py | 11 ++++++----- .../saved_model/model_utils/export_output_test.py | 9 ++++++++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/saved_model/model_utils/BUILD b/tensorflow/python/saved_model/model_utils/BUILD index 775d81a86bc..8e41a613b64 100644 --- a/tensorflow/python/saved_model/model_utils/BUILD +++ b/tensorflow/python/saved_model/model_utils/BUILD @@ -48,6 +48,7 @@ py_strict_library( "//tensorflow/python:constant_op", "//tensorflow/python:dtypes", "//tensorflow/python:framework_ops", + "//tensorflow/python:tensor_util", "//tensorflow/python/saved_model:signature_def_utils", "@six_archive//:six", ], @@ -69,6 +70,7 @@ py_strict_test( "//tensorflow/python:framework_ops", "//tensorflow/python:metrics", "//tensorflow/python:sparse_tensor", + "//tensorflow/python:variables", "//tensorflow/python/eager:context", "//tensorflow/python/saved_model:signature_constants", ], diff --git a/tensorflow/python/saved_model/model_utils/export_output.py b/tensorflow/python/saved_model/model_utils/export_output.py index b571bad067e..9b3ce04e071 100644 --- a/tensorflow/python/saved_model/model_utils/export_output.py +++ b/tensorflow/python/saved_model/model_utils/export_output.py @@ -26,6 +26,7 @@ import six from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops +from tensorflow.python.framework import tensor_util from tensorflow.python.saved_model import signature_def_utils @@ -342,16 +343,16 @@ class _SupervisedOutput(ExportOutput): raise ValueError( '{} output value must be a Tensor; got {}.'.format( key, metric_val)) - if (not isinstance(metric_op, ops.Tensor) and - not isinstance(metric_op, ops.Operation)): + if not (tensor_util.is_tensor(metric_op) or + isinstance(metric_op, ops.Operation)): raise ValueError( '{} update_op must be a Tensor or Operation; got {}.'.format( key, metric_op)) - # We must wrap any ops in a Tensor before export, as the SignatureDef - # proto expects tensors only. See b/109740581 + # We must wrap any ops (or variables) in a Tensor before export, as the + # SignatureDef proto expects tensors only. See b/109740581 metric_op_tensor = metric_op - if isinstance(metric_op, ops.Operation): + if not isinstance(metric_op, ops.Tensor): with ops.control_dependencies([metric_op]): metric_op_tensor = constant_op.constant([], name='metric_op_wrapper') diff --git a/tensorflow/python/saved_model/model_utils/export_output_test.py b/tensorflow/python/saved_model/model_utils/export_output_test.py index 8a3f107ce6c..8fd13b3d72e 100644 --- a/tensorflow/python/saved_model/model_utils/export_output_test.py +++ b/tensorflow/python/saved_model/model_utils/export_output_test.py @@ -29,6 +29,7 @@ from tensorflow.python.framework import sparse_tensor from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import metrics as metrics_module +from tensorflow.python.ops import variables from tensorflow.python.platform import test from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model.model_utils import export_output as export_output_lib @@ -373,10 +374,16 @@ class SupervisedOutputTest(test.TestCase): mean, update_op = metrics_module.mean_tensor(constant_op.constant([0])) metrics = { 'metrics_1': (mean, update_op), - 'metrics_2': (constant_op.constant([0]), control_flow_ops.no_op()) + 'metrics_2': (constant_op.constant([0]), control_flow_ops.no_op()), + # Keras metric's update_state() could return a Variable, rather than + # an Operation or Tensor. + 'keras_1': (constant_op.constant([0.5]), + variables.Variable(1.0, name='AssignAddVariableOp_3')) } outputter = MockSupervisedOutput(loss, predictions, metrics) + # If we get there, it means constructor succeeded; which is sufficient + # for testing the constructor. self.assertTrue(outputter.metrics['metrics_1/update_op'].name.startswith( 'mean/update_op')) From da05b9c999fad870bfd4107de0d96e6f0733e8bb Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Mon, 10 Aug 2020 17:33:01 -0700 Subject: [PATCH 0795/1017] Disable failed tests. PiperOrigin-RevId: 325923288 Change-Id: I1b9ece631c3bfec26e234eb26ee1f2fd74930373 --- tensorflow/core/common_runtime/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/common_runtime/BUILD b/tensorflow/core/common_runtime/BUILD index 4978a613707..a2b9867f132 100644 --- a/tensorflow/core/common_runtime/BUILD +++ b/tensorflow/core/common_runtime/BUILD @@ -2003,7 +2003,7 @@ tf_cc_tests_gpu( "permuter_test.cc", ], linkstatic = tf_kernel_tests_linkstatic(), - tags = ["no_cuda_on_cpu_tap"], + tags = ["notap"], # b/163417734 deps = [ ":core", ":core_cpu", From 4ef3bf9474e8b13c277d8fe2651ba46c6df23500 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Mon, 10 Aug 2020 17:52:52 -0700 Subject: [PATCH 0796/1017] Return None, not ndarray wrapping a None from tape gradient PiperOrigin-RevId: 325926529 Change-Id: Ic0326b07b9d8d396c48aa2c16d5c5e3f8889c434 --- tensorflow/python/eager/backprop.py | 6 +++++- tensorflow/python/ops/numpy_ops/np_interop_test.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py index 7cb3abf4e07..71b1303ecf4 100644 --- a/tensorflow/python/eager/backprop.py +++ b/tensorflow/python/eager/backprop.py @@ -1091,7 +1091,11 @@ class GradientTape(object): self._tape = None if rewrap_as_ndarray: - flat_grad = nest.map_structure(np_arrays.tensor_to_ndarray, flat_grad) + def _tensor_to_ndarray(x): + if x is not None: + return np_arrays.tensor_to_ndarray(x) + return None + flat_grad = nest.map_structure(_tensor_to_ndarray, flat_grad) grad = nest.pack_sequence_as(sources, flat_grad) return grad diff --git a/tensorflow/python/ops/numpy_ops/np_interop_test.py b/tensorflow/python/ops/numpy_ops/np_interop_test.py index 3eb7bebd767..0b474035edd 100644 --- a/tensorflow/python/ops/numpy_ops/np_interop_test.py +++ b/tensorflow/python/ops/numpy_ops/np_interop_test.py @@ -98,6 +98,18 @@ class InteropTest(tf.test.TestCase): self.assertAllClose(dx, 2.0) self.assertAllClose(dy, 3.0) + def testGradientTapeNoneGradients(self): + y = np.asarray(2.0) + + with tf.GradientTape() as t: + x = np.asarray(3.0) + t.watch([x]) + z = 2 * x + + dz = t.gradient(z, y) + + self.assertIsNone(dz) + def testCondInterop(self): x = np.asarray(3.0) From 11e82b2a6cd87f4fa47bdd8baa4032e2d29d898b Mon Sep 17 00:00:00 2001 From: Jose Baiocchi Date: Mon, 10 Aug 2020 17:57:55 -0700 Subject: [PATCH 0797/1017] Expose CopyOpMetricsMetadata via header file PiperOrigin-RevId: 325927125 Change-Id: I69b70e48ac41c275ac9ad131a29a8f27cb29efac --- .../convert/op_metrics_db_combiner.cc | 19 +++++++++---------- .../profiler/convert/op_metrics_db_combiner.h | 5 ++++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tensorflow/core/profiler/convert/op_metrics_db_combiner.cc b/tensorflow/core/profiler/convert/op_metrics_db_combiner.cc index ad1d4bf380a..425bf0077c3 100644 --- a/tensorflow/core/profiler/convert/op_metrics_db_combiner.cc +++ b/tensorflow/core/profiler/convert/op_metrics_db_combiner.cc @@ -25,8 +25,14 @@ namespace { using OperationType = OpMetrics::MemoryAccessed::OperationType; -// Copies OpMetrics symbol data from src to dst. -void CopyOpMetricsSymbolData(const OpMetrics& src, OpMetrics* dst) { +void CombinePrecisionStats(const PrecisionStats& src, PrecisionStats* dst) { + dst->set_compute_16bit_ps(src.compute_16bit_ps() + dst->compute_16bit_ps()); + dst->set_compute_32bit_ps(src.compute_32bit_ps() + dst->compute_32bit_ps()); +} + +} // namespace + +void CopyOpMetricsMetadata(const OpMetrics& src, OpMetrics* dst) { DCHECK(dst != nullptr); DCHECK_EQ(src.hlo_module_id(), dst->hlo_module_id()); DCHECK_EQ(src.name(), dst->name()); @@ -47,13 +53,6 @@ void CopyOpMetricsSymbolData(const OpMetrics& src, OpMetrics* dst) { } } -void CombinePrecisionStats(const PrecisionStats& src, PrecisionStats* dst) { - dst->set_compute_16bit_ps(src.compute_16bit_ps() + dst->compute_16bit_ps()); - dst->set_compute_32bit_ps(src.compute_32bit_ps() + dst->compute_32bit_ps()); -} - -} // namespace - void CombineOpMetrics(const OpMetrics& src, OpMetrics* dst) { DCHECK(dst != nullptr); if (dst->occurrences() == 0) { @@ -115,7 +114,7 @@ void OpMetricsDbCombiner::Combine(const OpMetricsDb& src) { for (const auto& src_metrics : src.metrics_db()) { auto* dst_metrics = LookupOrInsertNewOpMetrics(src_metrics.hlo_module_id(), src_metrics.name()); - CopyOpMetricsSymbolData(src_metrics, dst_metrics); + CopyOpMetricsMetadata(src_metrics, dst_metrics); CombineOpMetrics(src_metrics, dst_metrics); } } diff --git a/tensorflow/core/profiler/convert/op_metrics_db_combiner.h b/tensorflow/core/profiler/convert/op_metrics_db_combiner.h index a87a2b53500..5c1490d2e8b 100644 --- a/tensorflow/core/profiler/convert/op_metrics_db_combiner.h +++ b/tensorflow/core/profiler/convert/op_metrics_db_combiner.h @@ -23,7 +23,10 @@ limitations under the License. namespace tensorflow { namespace profiler { -// Combines the src OpMetrics into the dst OpMetrics. +// Copies OpMetrics metadata (e.g., category, provenance) from src to dst. +void CopyOpMetricsMetadata(const OpMetrics& src, OpMetrics* dst); + +// Combines OpMetrics data (e.g., occurrences, time) from src into dst. void CombineOpMetrics(const OpMetrics& src, OpMetrics* dst); // Combines the memory access breakdown. From 6ea0d3d925a6588f9b48283c6043b14752593cc6 Mon Sep 17 00:00:00 2001 From: Jian Li Date: Mon, 10 Aug 2020 18:54:05 -0700 Subject: [PATCH 0798/1017] Remove duplicated comments. PiperOrigin-RevId: 325934154 Change-Id: I26296779d852b5a6a7b9bc17f06ecc9adc6b3dc8 --- tensorflow/lite/kernels/lstm_eval.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/lite/kernels/lstm_eval.cc b/tensorflow/lite/kernels/lstm_eval.cc index 9087bbeada9..e11a7c5a026 100644 --- a/tensorflow/lite/kernels/lstm_eval.cc +++ b/tensorflow/lite/kernels/lstm_eval.cc @@ -673,7 +673,7 @@ void CalculateLstmGateInteger8x8_8( tensor_utils::ApplyLayerNormFloat( gate, layer_norm_gate_weight, layer_norm_gate_scale_a, layer_norm_gate_scale_b, gate_bias, n_batch, n_cell, gate); - // Apply activation. // Apply activation + // Apply activation. switch (activation) { case kTfLiteActSigmoid: tensor_utils::ApplySigmoidFloat(gate, n_batch, n_cell, gate); From ef20eb2110fc3e1584849b8641aacd7846576b28 Mon Sep 17 00:00:00 2001 From: Hye Soo Yang Date: Mon, 10 Aug 2020 19:26:24 -0700 Subject: [PATCH 0799/1017] Add `stateless_random_crop` to tf.image API; it is a deterministic version of `tf.image.random_crop`. Given the same seed, `stateless_random_crop` guarantees the same results independent of how many times it is called, and independent of global seed settings. PiperOrigin-RevId: 325938094 Change-Id: Iad3132e097d71513193304d8aad45a5585656c53 --- tensorflow/core/kernels/random_crop_op.cc | 5 -- .../kernel_tests/random/random_crop_test.py | 85 ++++++++++++++++++- tensorflow/python/ops/random_ops.py | 57 ++++++++++++- .../api/golden/v2/tensorflow.image.pbtxt | 4 + 4 files changed, 142 insertions(+), 9 deletions(-) diff --git a/tensorflow/core/kernels/random_crop_op.cc b/tensorflow/core/kernels/random_crop_op.cc index b89bda4769d..eb7980fa58e 100644 --- a/tensorflow/core/kernels/random_crop_op.cc +++ b/tensorflow/core/kernels/random_crop_op.cc @@ -63,11 +63,6 @@ class RandomCropOp : public OpKernel { if ((target_height == height) && (target_width == width)) { *output = context->input(0); } - - // TODO(shlens): Implement edge case to guarantee output size dimensions. - // Edge case. The target dimensions are larger then the image, so - // zero-pad the image. This guarantees that the image will *always* - // be [target_height, target_width] in size. OP_REQUIRES(context, width >= target_width, errors::FailedPrecondition( "width must be >= target_width: width = ", width, diff --git a/tensorflow/python/kernel_tests/random/random_crop_test.py b/tensorflow/python/kernel_tests/random/random_crop_test.py index 724bee07157..f8effa0ee7b 100644 --- a/tensorflow/python/kernel_tests/random/random_crop_test.py +++ b/tensorflow/python/kernel_tests/random/random_crop_test.py @@ -77,5 +77,88 @@ class RandomCropTest(test.TestCase): self.assertAllClose(counts, mean, atol=four_stddev) -if __name__ == '__main__': +class StatelessRandomCropTest(test.TestCase): + + def testNoOp(self): + # No random cropping is performed since the size is value.shape. + for shape in (2, 1, 1), (2, 1, 3), (4, 5, 3): + value = np.arange(0, np.prod(shape), dtype=np.int32).reshape(shape) + crop = random_ops.stateless_random_crop(value, shape, seed=(1, 2)) + self.evaluate(crop) + self.assertAllEqual(crop, value) + + def testContains(self): + with test_util.use_gpu(): + shape = (3, 5, 7) + target = (2, 3, 4) + value = np.random.randint(1000000, size=shape) + iterations = 10 + value_set = set( + tuple(value[i:i + 2, j:j + 3, k:k + 4].ravel()) # pylint: disable=g-complex-comprehension + for i in range(2) for j in range(3) for k in range(4)) + test_seeds = [ + tuple(map(lambda x, i=i: x + 1 * i, t)) + for (i, t) in enumerate((1, 2) for _ in range(iterations)) + ] + + # Check that the result is valid by making sure that it is one of all + # possible values for randomly cropping `value` with `target` shape. + for seed in test_seeds: + crop = random_ops.stateless_random_crop(value, size=target, seed=seed) + y = self.evaluate(crop) + self.assertAllEqual(y.shape, target) + self.assertIn(tuple(y.ravel()), value_set) + + # TODO(b/162345082): stateless random op generates different random number + # with xla_gpu. Update tests such that there is a single ground truth result + # to test against. + def testRandomization(self): + with test_util.use_gpu(): + shape = [5, 4, 1] + size = np.prod(shape) + single = [1, 1, 1] + value = np.arange(size).reshape(shape) + iterations = 5 + num_samples = 5 + + # Test that the same result is returned given the same seed is provided + # for each round. + test_seed = (1, 2) + observations = [[] for _ in range(iterations)] + for observation in observations: + crop = random_ops.stateless_random_crop(value, single, seed=test_seed) + counts = np.zeros(size, dtype=np.int32) + for _ in range(num_samples): + y = self.evaluate(crop) + self.assertAllEqual(y.shape, single) + counts[y] += 1 + + observation.append(counts) + + for i in range(1, iterations): + self.assertAllEqual(observations[0], observations[i]) + + # Test that the same sequence of results are returned given the same + # sequence of seeds provided. + test_seeds = [ + tuple(map(lambda x, i=i: x + 1 * i, t)) + for (i, t) in enumerate((1, 2) for _ in range(iterations)) + ] + observations = [[] for _ in range(iterations)] + for observation in observations: + counts = np.zeros(size, dtype=np.int32) + for seed in test_seeds: + crop = random_ops.stateless_random_crop( + value, single, seed=seed) + y = self.evaluate(crop) + self.assertAllEqual(y.shape, single) + counts[y] += 1 + + observation.append(counts) + + for i in range(1, iterations): + self.assertAllEqual(observations[0], observations[i]) + + +if __name__ == "__main__": test.main() diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py index 0bb4b78c29f..46a1e321093 100644 --- a/tensorflow/python/ops/random_ops.py +++ b/tensorflow/python/ops/random_ops.py @@ -29,6 +29,7 @@ from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_random_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import stateless_random_ops # go/tf-wildcard-import # pylint: disable=wildcard-import @@ -373,9 +374,6 @@ def random_crop(value, size, seed=None, name=None): Returns: A cropped tensor of the same rank as `value` and shape `size`. """ - # TODO(shlens): Implement edge case to guarantee output size dimensions. - # If size > value.shape, zero pad the result so that it always has shape - # exactly size. with ops.name_scope(name, "random_crop", [value, size]) as name: value = ops.convert_to_tensor(value, name="value") size = ops.convert_to_tensor(size, dtype=dtypes.int32, name="size") @@ -394,6 +392,59 @@ def random_crop(value, size, seed=None, name=None): return array_ops.slice(value, offset, size, name=name) +@tf_export("image.stateless_random_crop", v1=[]) +@dispatch.add_dispatch_support +def stateless_random_crop(value, size, seed, name=None): + """Randomly crops a tensor to a given size in a deterministic manner. + + Slices a shape `size` portion out of `value` at a uniformly chosen offset. + Requires `value.shape >= size`. + + If a dimension should not be cropped, pass the full size of that dimension. + For example, RGB images can be cropped with + `size = [crop_height, crop_width, 3]`. + + Guarantees the same results given the same `seed` independent of how many + times the function is called, and independent of global seed settings (e.g. + `tf.random.set_seed`). + + Usage Example: + + >>> image = [[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]] + >>> seed = (1, 2) + >>> tf.image.stateless_random_crop(value=image, size=(1, 2, 3), seed=seed) + + + Args: + value: Input tensor to crop. + size: 1-D tensor with size the rank of `value`. + seed: A shape [2] Tensor, the seed to the random number generator. Must have + dtype `int32` or `int64`. (When using XLA, only `int32` is allowed.) + name: A name for this operation (optional). + + Returns: + A cropped tensor of the same rank as `value` and shape `size`. + """ + with ops.name_scope(name, "random_crop", [value, size]) as name: + value = ops.convert_to_tensor(value, name="value") + size = ops.convert_to_tensor(size, dtype=dtypes.int32, name="size") + shape = array_ops.shape(value) + check = control_flow_ops.Assert( + math_ops.reduce_all(shape >= size), + ["Need value.shape >= size, got ", shape, size], + summarize=1000) + shape = control_flow_ops.with_dependencies([check], shape) + limit = shape - size + 1 + offset = stateless_random_ops.stateless_random_uniform( + array_ops.shape(shape), + dtype=size.dtype, + maxval=size.dtype.max, + seed=seed) % limit + return array_ops.slice(value, offset, size, name=name) + + @tf_export(v1=["random.multinomial", "multinomial"]) @dispatch.add_dispatch_support @deprecation.deprecated( diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt index 8bca192e1c1..fd3cf2988da 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt @@ -240,6 +240,10 @@ tf_module { name: "stateless_random_contrast" argspec: "args=[\'image\', \'lower\', \'upper\', \'seed\'], varargs=None, keywords=None, defaults=None" } + member_method { + name: "stateless_random_crop" + argspec: "args=[\'value\', \'size\', \'seed\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " + } member_method { name: "stateless_random_flip_left_right" argspec: "args=[\'image\', \'seed\'], varargs=None, keywords=None, defaults=None" From a1e5b65f08babdbe5163a05da161abbd5ddc676f Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 11 Aug 2020 02:46:32 +0000 Subject: [PATCH 0800/1017] merge conflicts with master --- tensorflow/core/kernels/map_kernels.h | 6 ------ tensorflow/python/kernel_tests/map_ops_test.py | 3 --- 2 files changed, 9 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index a4ee00d739e..9e436bc5fcd 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -176,8 +176,6 @@ class TensorMapHasKey : public OpKernel { } }; -<<<<<<< HEAD - template Status TensorMapBinaryAdd(OpKernelContext* c, const TensorMap& a, const TensorMap& b, TensorMap* out) { @@ -196,7 +194,6 @@ Status TensorMapBinaryAdd(OpKernelContext* c, const TensorMap& a, return Status::OK(); } - template Status TensorMapZerosLike(OpKernelContext* c, const TensorMap& x, TensorMap* y) { for (const std::pair& p : x.tensors()) { @@ -207,9 +204,6 @@ Status TensorMapZerosLike(OpKernelContext* c, const TensorMap& x, TensorMap* y) return Status::OK(); } - -======= ->>>>>>> master } // namespace tensorflow #endif // TENSORFLOW_CORE_KERNELS_MAP_KERNELS_H_ diff --git a/tensorflow/python/kernel_tests/map_ops_test.py b/tensorflow/python/kernel_tests/map_ops_test.py index 623484b77eb..ba15c0d3c4c 100644 --- a/tensorflow/python/kernel_tests/map_ops_test.py +++ b/tensorflow/python/kernel_tests/map_ops_test.py @@ -207,7 +207,6 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): self.assertAllClose(g3, 7) del tape -<<<<<<< HEAD def testDiffKeySameValueGrad(self): with backprop.GradientTape(persistent=True) as tape: m = map_ops.empty_tensor_map() @@ -395,8 +394,6 @@ class MapOpsTest(test_util.TensorFlowTestCase, parameterized.TestCase): s = map_ops.tensor_map_size(m) self.assertAllEqual(s, 0) self.assertAllClose(e, v) -======= ->>>>>>> master if __name__ == "__main__": test.main() From f3e8f1f3f1792796b2e369d6d3b69fbfcaa92968 Mon Sep 17 00:00:00 2001 From: Hye Soo Yang Date: Mon, 10 Aug 2020 20:23:11 -0700 Subject: [PATCH 0801/1017] Add `stateless_sample_distorted_bounding_box` op which is deterministic; it guarantees the same results independent of how many times the they are called, and independent of global seed settings. PiperOrigin-RevId: 325943656 Change-Id: Ia1c3e6f95862c175dcc55be672fd878a4130f3c4 --- RELEASE.md | 8 +- ..._StatelessSampleDistortedBoundingBox.pbtxt | 144 ++++++++++++++++++ ..._StatelessSampleDistortedBoundingBox.pbtxt | 4 + tensorflow/core/kernels/BUILD | 2 +- .../sample_distorted_bounding_box_op.cc | 90 ++++++++--- tensorflow/core/ops/image_ops.cc | 38 +++++ tensorflow/python/ops/image_ops_impl.py | 124 +++++++++++++++ tensorflow/python/ops/image_ops_test.py | 143 +++++++++++++++++ .../api/golden/v1/tensorflow.raw_ops.pbtxt | 4 + .../api/golden/v2/tensorflow.image.pbtxt | 4 + .../api/golden/v2/tensorflow.raw_ops.pbtxt | 4 + 11 files changed, 542 insertions(+), 23 deletions(-) create mode 100644 tensorflow/core/api_def/base_api/api_def_StatelessSampleDistortedBoundingBox.pbtxt create mode 100644 tensorflow/core/api_def/python_api/api_def_StatelessSampleDistortedBoundingBox.pbtxt diff --git a/RELEASE.md b/RELEASE.md index 525db3cade8..430e1b83885 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -86,9 +86,11 @@ option. * `tf.image`: * Added deterministic `tf.image.stateless_random_*` functions for each - `tf.image.random_*` function. Given the same seed, the stateless functions - produce the same results independent of how many times the function is - called, and independent of global seed settings. + `tf.image.random_*` function. Added a new op + `stateless_sample_distorted_bounding_box` which is a determinstic + version of `sample_distorted_bounding_box` op. Given the same seed, these + stateless functions/ops produce the same results independent of how many + times the function is called, and independent of global seed settings. * `tf.distribute`: * * `tf.keras`: diff --git a/tensorflow/core/api_def/base_api/api_def_StatelessSampleDistortedBoundingBox.pbtxt b/tensorflow/core/api_def/base_api/api_def_StatelessSampleDistortedBoundingBox.pbtxt new file mode 100644 index 00000000000..2c5e32a0c1e --- /dev/null +++ b/tensorflow/core/api_def/base_api/api_def_StatelessSampleDistortedBoundingBox.pbtxt @@ -0,0 +1,144 @@ +op { + graph_op_name: "StatelessSampleDistortedBoundingBox" + in_arg { + name: "image_size" + description: <>> image = np.array([[[1], [2], [3]], [[4], [5], [6]], [[7], [8], [9]]]) +>>> bbox = tf.constant( +... [0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]) +>>> seed = (1, 2) +>>> # Generate a single distorted bounding box. +>>> bbox_begin, bbox_size, bbox_draw = ( +... tf.image.stateless_sample_distorted_bounding_box( +... tf.shape(image), bounding_boxes=bbox, seed=seed)) +>>> # Employ the bounding box to distort the image. +>>> tf.slice(image, bbox_begin, bbox_size) + +>>> # Draw the bounding box in an image summary. +>>> colors = np.array([[1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]) +>>> tf.image.draw_bounding_boxes( +... tf.expand_dims(tf.cast(image, tf.float32),0), bbox_draw, colors) + + +Note that if no bounding box information is available, setting +`use_image_if_no_bounding_boxes = true` will assume there is a single implicit +bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is +false and no bounding boxes are supplied, an error is raised. +END +} diff --git a/tensorflow/core/api_def/python_api/api_def_StatelessSampleDistortedBoundingBox.pbtxt b/tensorflow/core/api_def/python_api/api_def_StatelessSampleDistortedBoundingBox.pbtxt new file mode 100644 index 00000000000..2ee453ee2f5 --- /dev/null +++ b/tensorflow/core/api_def/python_api/api_def_StatelessSampleDistortedBoundingBox.pbtxt @@ -0,0 +1,4 @@ +op { + graph_op_name: "StatelessSampleDistortedBoundingBox" + visibility: HIDDEN +} diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index ccb12d9b09d..bfb192023a1 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -3166,7 +3166,7 @@ tf_kernel_library( tf_kernel_library( name = "sample_distorted_bounding_box_op", prefix = "sample_distorted_bounding_box_op", - deps = IMAGE_DEPS, + deps = IMAGE_DEPS + [":stateless_random_ops"], ) tf_kernel_library( diff --git a/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc b/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc index 2936856ec29..3b1cc3d27f0 100644 --- a/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc +++ b/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc @@ -14,12 +14,16 @@ limitations under the License. ==============================================================================*/ // See docs in ../ops/image_ops.cc. #include + #include + #include "tensorflow/core/framework/bounds_check.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/kernels/stateless_random_ops.h" +#include "tensorflow/core/lib/random/philox_random.h" #include "tensorflow/core/lib/random/simple_philox.h" #include "tensorflow/core/util/guarded_philox_random.h" @@ -201,12 +205,10 @@ bool GenerateRandomCrop(int original_width, int original_height, } // namespace template -class SampleDistortedBoundingBoxV2Op : public OpKernel { +class SampleDistortedBoundingBoxBaseOp : public OpKernel { public: - explicit SampleDistortedBoundingBoxV2Op(OpKernelConstruction* context) + explicit SampleDistortedBoundingBoxBaseOp(OpKernelConstruction* context) : OpKernel(context) { - OP_REQUIRES_OK(context, generator_.Init(context)); - if (context->num_inputs() == 2) { OP_REQUIRES_OK(context, context->GetAttr("min_object_covered", &min_object_covered_)); @@ -252,7 +254,7 @@ class SampleDistortedBoundingBoxV2Op : public OpKernel { max_attempts_)); } - void Compute(OpKernelContext* context) override { + void DoCompute(OpKernelContext* context, const random::PhiloxRandom& rng) { const Tensor& image_size = context->input(0); OP_REQUIRES(context, image_size.dims() == 1, @@ -287,7 +289,11 @@ class SampleDistortedBoundingBoxV2Op : public OpKernel { input_boxes.shape().DebugString())); float min_object_covered_val = 0.0; - if (context->num_inputs() == 3) { + // `SampleDistortedBoundingBox` op accepts 2 inputs and has + // `min_object_covered` as an attribute (handled in the constructor). + // `SampleDistortedBoundingBoxV2` and `StatelessSampleDistortedBoundingBox` + // ops accept 3+ inputs, including `min_object_covered`. + if (context->num_inputs() >= 3) { const Tensor& min_object_covered = context->input(2); OP_REQUIRES( @@ -342,8 +348,8 @@ class SampleDistortedBoundingBoxV2Op : public OpKernel { const float min_sample_aspect_ratio = aspect_ratio_range_[0]; const float max_sample_aspect_ratio = aspect_ratio_range_[1]; - auto local_gen = generator_.ReserveSamples32(4 * max_attempts_); - random::SimplePhilox random(&local_gen); + auto local_rng = rng; + random::SimplePhilox random(&local_rng); Rectangle crop_rect; bool sample_generated = false; @@ -420,8 +426,7 @@ class SampleDistortedBoundingBoxV2Op : public OpKernel { size_data(2) = T(-1); } - private: - GuardedPhiloxRandom generator_; + protected: int32 max_attempts_; std::vector area_range_; std::vector aspect_ratio_range_; @@ -429,15 +434,62 @@ class SampleDistortedBoundingBoxV2Op : public OpKernel { bool use_image_if_no_bounding_boxes_; }; -#define REGISTER_KERNELS(type) \ - REGISTER_KERNEL_BUILDER(Name("SampleDistortedBoundingBox") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T"), \ - SampleDistortedBoundingBoxV2Op) \ - REGISTER_KERNEL_BUILDER(Name("SampleDistortedBoundingBoxV2") \ - .Device(DEVICE_CPU) \ - .TypeConstraint("T"), \ - SampleDistortedBoundingBoxV2Op) +template +class StatefulSampleDistortedBoundingBoxOp + : public SampleDistortedBoundingBoxBaseOp { + public: + explicit StatefulSampleDistortedBoundingBoxOp(OpKernelConstruction* context) + : SampleDistortedBoundingBoxBaseOp(context) { + OP_REQUIRES_OK(context, generator_.Init(context)); + } + + void Compute(OpKernelContext* context) override { + // Need to reserve samples since `generator_` is shared. + this->DoCompute(context, + generator_.ReserveSamples32(4 * this->max_attempts_)); + } + + private: + GuardedPhiloxRandom generator_; +}; + +template +class StatelessSampleDistortedBoundingBoxOp + : public SampleDistortedBoundingBoxBaseOp { + public: + explicit StatelessSampleDistortedBoundingBoxOp(OpKernelConstruction* context) + : SampleDistortedBoundingBoxBaseOp(context) {} + + void Compute(OpKernelContext* context) override { + const Tensor& seed_t = context->input(3); + OP_REQUIRES(context, seed_t.dims() == 1 && seed_t.dim_size(0) == 2, + errors::InvalidArgument("seed must have shape [2], not ", + seed_t.shape().DebugString())); + + // Create and initialize stateless random number generator (rng). + // There is no need to `Skip` (or reserve) samples since the scope of this + // rng is local. + random::PhiloxRandom::Key key; + random::PhiloxRandom::ResultType counter; + OP_REQUIRES_OK(context, GenerateKey(seed_t, &key, &counter)); + + this->DoCompute(context, random::PhiloxRandom(counter, key)); + } +}; + +#define REGISTER_KERNELS(type) \ + REGISTER_KERNEL_BUILDER(Name("SampleDistortedBoundingBox") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T"), \ + StatefulSampleDistortedBoundingBoxOp) \ + REGISTER_KERNEL_BUILDER(Name("SampleDistortedBoundingBoxV2") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T"), \ + StatefulSampleDistortedBoundingBoxOp) \ + REGISTER_KERNEL_BUILDER(Name("StatelessSampleDistortedBoundingBox") \ + .Device(DEVICE_CPU) \ + .TypeConstraint("T"), \ + StatelessSampleDistortedBoundingBoxOp) TF_CALL_INTEGRAL_TYPES(REGISTER_KERNELS); #undef REGISTER_KERNELS diff --git a/tensorflow/core/ops/image_ops.cc b/tensorflow/core/ops/image_ops.cc index 43ee65c4ab4..8dfc67f22d3 100644 --- a/tensorflow/core/ops/image_ops.cc +++ b/tensorflow/core/ops/image_ops.cc @@ -758,6 +758,44 @@ REGISTER_OP("SampleDistortedBoundingBoxV2") return Status::OK(); }); +REGISTER_OP("StatelessSampleDistortedBoundingBox") + .Input("image_size: T") + .Input("bounding_boxes: float") + .Input("min_object_covered: float") + .Input("seed: Tseed") + .Output("begin: T") + .Output("size: T") + .Output("bboxes: float") + .Attr("T: {uint8, int8, int16, int32, int64}") + .Attr("Tseed: {int32, int64}") + .Attr("aspect_ratio_range: list(float) = [0.75, 1.33]") + .Attr("area_range: list(float) = [0.05, 1.0]") + .Attr("max_attempts: int = 100") + .Attr("use_image_if_no_bounding_boxes: bool = false") + .SetShapeFn([](InferenceContext* c) { + // Get inputs and validate ranks. + ShapeHandle image_size; + TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &image_size)); + ShapeHandle bounding_boxes; + TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 3, &bounding_boxes)); + ShapeHandle min_object_covered; + TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &min_object_covered)); + ShapeHandle seed; + TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 1, &seed)); + // image_size: 1-D with [height, width, channels] + // bounding_boxes: 3-D with shape [batch, N, 4] + DimensionHandle unused; + TF_RETURN_IF_ERROR(c->WithValue(c->Dim(image_size, 0), 3, &unused)); + TF_RETURN_IF_ERROR(c->WithValue(c->Dim(bounding_boxes, 2), 4, &unused)); + TF_RETURN_IF_ERROR(c->WithValue(c->Dim(seed, 0), 2, &unused)); + + c->set_output(0, c->Vector(3)); + c->set_output(1, c->Vector(3)); + c->set_output(2, c->MakeShape({1, 1, 4})); + + return Status::OK(); + }); + // -------------------------------------------------------------------------- // glimpse = extract_glimpse(input, size, offsets) extract the glimpse diff --git a/tensorflow/python/ops/image_ops_impl.py b/tensorflow/python/ops/image_ops_impl.py index 8d542b4eaaa..e728da34117 100644 --- a/tensorflow/python/ops/image_ops_impl.py +++ b/tensorflow/python/ops/image_ops_impl.py @@ -3256,6 +3256,130 @@ def sample_distorted_bounding_box_v2(image_size, name=name) +@tf_export('image.stateless_sample_distorted_bounding_box', v1=[]) +@dispatch.add_dispatch_support +def stateless_sample_distorted_bounding_box(image_size, + bounding_boxes, + seed, + min_object_covered=0.1, + aspect_ratio_range=None, + area_range=None, + max_attempts=None, + use_image_if_no_bounding_boxes=None, + name=None): + """Generate a randomly distorted bounding box for an image deterministically. + + Bounding box annotations are often supplied in addition to ground-truth labels + in image recognition or object localization tasks. A common technique for + training such a system is to randomly distort an image while preserving + its content, i.e. *data augmentation*. This Op, given the same `seed`, + deterministically outputs a randomly distorted localization of an object, i.e. + bounding box, given an `image_size`, `bounding_boxes` and a series of + constraints. + + The output of this Op is a single bounding box that may be used to crop the + original image. The output is returned as 3 tensors: `begin`, `size` and + `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the + image. The latter may be supplied to `tf.image.draw_bounding_boxes` to + visualize what the bounding box looks like. + + Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. + The bounding box coordinates are floats in `[0.0, 1.0]` relative to the width + and the height of the underlying image. + + The output of this Op is guaranteed to be the same given the same `seed` and + is independent of how many times the function is called, and independent of + global seed settings (e.g. `tf.random.set_seed`). + + Example usage: + + >>> image = np.array([[[1], [2], [3]], [[4], [5], [6]], [[7], [8], [9]]]) + >>> bbox = tf.constant( + ... [0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]) + >>> seed = (1, 2) + >>> # Generate a single distorted bounding box. + >>> bbox_begin, bbox_size, bbox_draw = ( + ... tf.image.stateless_sample_distorted_bounding_box( + ... tf.shape(image), bounding_boxes=bbox, seed=seed)) + >>> # Employ the bounding box to distort the image. + >>> tf.slice(image, bbox_begin, bbox_size) + + >>> # Draw the bounding box in an image summary. + >>> colors = np.array([[1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]) + >>> tf.image.draw_bounding_boxes( + ... tf.expand_dims(tf.cast(image, tf.float32),0), bbox_draw, colors) + + + Note that if no bounding box information is available, setting + `use_image_if_no_bounding_boxes = true` will assume there is a single implicit + bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is + false and no bounding boxes are supplied, an error is raised. + + Args: + image_size: A `Tensor`. Must be one of the following types: `uint8`, `int8`, + `int16`, `int32`, `int64`. 1-D, containing `[height, width, channels]`. + bounding_boxes: A `Tensor` of type `float32`. 3-D with shape `[batch, N, 4]` + describing the N bounding boxes associated with the image. + seed: A shape [2] Tensor, the seed to the random number generator. Must have + dtype `int32` or `int64`. (When using XLA, only `int32` is allowed.) + min_object_covered: A Tensor of type `float32`. Defaults to `0.1`. The + cropped area of the image must contain at least this fraction of any + bounding box supplied. The value of this parameter should be non-negative. + In the case of 0, the cropped area does not need to overlap any of the + bounding boxes supplied. + aspect_ratio_range: An optional list of `floats`. Defaults to `[0.75, + 1.33]`. The cropped area of the image must have an aspect `ratio = width / + height` within this range. + area_range: An optional list of `floats`. Defaults to `[0.05, 1]`. The + cropped area of the image must contain a fraction of the supplied image + within this range. + max_attempts: An optional `int`. Defaults to `100`. Number of attempts at + generating a cropped region of the image of the specified constraints. + After `max_attempts` failures, return the entire image. + use_image_if_no_bounding_boxes: An optional `bool`. Defaults to `False`. + Controls behavior if no bounding boxes supplied. If true, assume an + implicit bounding box covering the whole input. If false, raise an error. + name: A name for the operation (optional). + + Returns: + A tuple of `Tensor` objects (begin, size, bboxes). + + begin: A `Tensor`. Has the same type as `image_size`. 1-D, containing + `[offset_height, offset_width, 0]`. Provide as input to + `tf.slice`. + size: A `Tensor`. Has the same type as `image_size`. 1-D, containing + `[target_height, target_width, -1]`. Provide as input to + `tf.slice`. + bboxes: A `Tensor` of type `float32`. 3-D with shape `[1, 1, 4]` containing + the distorted bounding box. + Provide as input to `tf.image.draw_bounding_boxes`. + """ + with ops.name_scope(name, 'stateless_sample_distorted_bounding_box'): + return gen_image_ops.stateless_sample_distorted_bounding_box( + image_size=image_size, + bounding_boxes=bounding_boxes, + seed=seed, + min_object_covered=min_object_covered, + aspect_ratio_range=aspect_ratio_range, + area_range=area_range, + max_attempts=max_attempts, + use_image_if_no_bounding_boxes=use_image_if_no_bounding_boxes, + name=name) + + @tf_export(v1=['image.sample_distorted_bounding_box']) @dispatch.add_dispatch_support @deprecation.deprecated( diff --git a/tensorflow/python/ops/image_ops_test.py b/tensorflow/python/ops/image_ops_test.py index 1adece3474b..210b6c6e65d 100644 --- a/tensorflow/python/ops/image_ops_test.py +++ b/tensorflow/python/ops/image_ops_test.py @@ -2418,6 +2418,149 @@ class SelectDistortedCropBoxTest(test_util.TensorFlowTestCase): end = self.evaluate(end) bbox_for_drawing = self.evaluate(bbox_for_drawing) + def _testStatelessSampleDistortedBoundingBox(self, image, bounding_box, + min_object_covered, + aspect_ratio_range, area_range): + with test_util.use_gpu(): + original_area = float(np.prod(image.shape)) + bounding_box_area = float((bounding_box[3] - bounding_box[1]) * + (bounding_box[2] - bounding_box[0])) + + image_size_np = np.array(image.shape, dtype=np.int32) + bounding_box_np = ( + np.array(bounding_box, dtype=np.float32).reshape([1, 1, 4])) + + iterations = 2 + test_seeds = [(1, 2), (3, 4), (5, 6)] + + for seed in test_seeds: + aspect_ratios = [] + area_ratios = [] + fraction_object_covered = [] + for _ in range(iterations): + image_tf = constant_op.constant(image, shape=image.shape) + image_size_tf = constant_op.constant( + image_size_np, shape=image_size_np.shape) + bounding_box_tf = constant_op.constant(bounding_box_np, + dtype=dtypes.float32, + shape=bounding_box_np.shape) + begin, size, _ = image_ops.stateless_sample_distorted_bounding_box( + image_size=image_size_tf, + bounding_boxes=bounding_box_tf, + seed=seed, + min_object_covered=min_object_covered, + aspect_ratio_range=aspect_ratio_range, + area_range=area_range) + y = array_ops.strided_slice(image_tf, begin, begin + size) + y_tf = self.evaluate(y) + crop_height = y_tf.shape[0] + crop_width = y_tf.shape[1] + aspect_ratio = float(crop_width) / float(crop_height) + area = float(crop_width * crop_height) + aspect_ratios.append(aspect_ratio) + area_ratio = area / original_area + area_ratios.append(area_ratio) + fraction_object_covered.append( + float(np.sum(y_tf)) / bounding_box_area) + + # Check that `area_ratio` is within valid range. + self.assertLessEqual(area_ratio, area_range[1]) + self.assertGreaterEqual(area_ratio, area_range[0]) + + # Each array should consist of one value just repeated `iteration` times + # because the same seed is used. + self.assertEqual(len(set(aspect_ratios)), 1) + self.assertEqual(len(set(area_ratios)), 1) + self.assertEqual(len(set(fraction_object_covered)), 1) + + # TODO(b/162345082): stateless random op generates different random number + # with xla_gpu. Update tests such that there is a single ground truth result + # to test against. + def testWholeImageBoundingBoxStateless(self): + height = 40 + width = 50 + image_size = [height, width, 1] + bounding_box = [0.0, 0.0, 1.0, 1.0] + image = np.arange( + 0, np.prod(image_size), dtype=np.int32).reshape(image_size) + for min_obj_covered in [0.1, constant_op.constant(0.1)]: + self._testStatelessSampleDistortedBoundingBox( + image, + bounding_box, + min_object_covered=min_obj_covered, + aspect_ratio_range=(0.75, 1.33), + area_range=(0.05, 1.0)) + + # TODO(b/162345082): stateless random op generates different random number + # with xla_gpu. Update tests such that there is a single ground truth result + # to test against. + def testWithBoundingBoxStateless(self): + height = 40 + width = 50 + x_shape = [height, width, 1] + image = np.zeros(x_shape, dtype=np.int32) + + xmin = 2 + ymin = 3 + xmax = 12 + ymax = 13 + for x in np.arange(xmin, xmax + 1, 1): + for y in np.arange(ymin, ymax + 1, 1): + image[x, y] = 1 + + # Bounding box is specified as (ymin, xmin, ymax, xmax) in + # relative coordinates. + bounding_box = (float(ymin) / height, float(xmin) / width, + float(ymax) / height, float(xmax) / width) + + # Test both scalar and tensor input for `min_object_covered`. + for min_obj_covered in [0.1, constant_op.constant(0.1)]: + self._testStatelessSampleDistortedBoundingBox( + image, + bounding_box=bounding_box, + min_object_covered=min_obj_covered, + aspect_ratio_range=(0.75, 1.33), + area_range=(0.05, 1.0)) + + def testSampleDistortedBoundingBoxShapeStateless(self): + with test_util.use_gpu(): + image_size = constant_op.constant( + [40, 50, 1], shape=[3], dtype=dtypes.int32) + bounding_box = constant_op.constant( + [[[0.0, 0.0, 1.0, 1.0]]], + shape=[1, 1, 4], + dtype=dtypes.float32, + ) + + bbox_func = functools.partial( + image_ops.stateless_sample_distorted_bounding_box, + image_size=image_size, + bounding_boxes=bounding_box, + min_object_covered=0.1, + aspect_ratio_range=(0.75, 1.33), + area_range=(0.05, 1.0)) + + # Check error is raised with wrong seed shapes. + for seed in [1, (1, 2, 3)]: + with self.assertRaises((ValueError, errors.InvalidArgumentError)): + begin, end, bbox_for_drawing = bbox_func(seed=seed) + + test_seed = (1, 2) + begin, end, bbox_for_drawing = bbox_func(seed=test_seed) + + # Test that the shapes are correct. + self.assertAllEqual([3], begin.get_shape().as_list()) + self.assertAllEqual([3], end.get_shape().as_list()) + self.assertAllEqual([1, 1, 4], bbox_for_drawing.get_shape().as_list()) + + # Actual run to make sure shape is correct inside Compute(). + begin = self.evaluate(begin) + end = self.evaluate(end) + bbox_for_drawing = self.evaluate(bbox_for_drawing) + self.assertAllEqual([3], begin.shape) + self.assertAllEqual([3], end.shape) + self.assertAllEqual([1, 1, 4], bbox_for_drawing.shape) + class ResizeImagesV2Test(test_util.TensorFlowTestCase): diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt index 1d74f47508a..0a2843431f2 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt @@ -4552,6 +4552,10 @@ tf_module { name: "StatelessRandomUniformInt" argspec: "args=[\'shape\', \'seed\', \'minval\', \'maxval\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "StatelessSampleDistortedBoundingBox" + argspec: "args=[\'image_size\', \'bounding_boxes\', \'min_object_covered\', \'seed\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\', \'name\'], varargs=None, keywords=None, defaults=[\'[0.75, 1.33]\', \'[0.05, 1]\', \'100\', \'False\', \'None\'], " + } member_method { name: "StatelessTruncatedNormal" argspec: "args=[\'shape\', \'seed\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt index fd3cf2988da..941d811f435 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.image.pbtxt @@ -264,6 +264,10 @@ tf_module { name: "stateless_random_saturation" argspec: "args=[\'image\', \'lower\', \'upper\', \'seed\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "stateless_sample_distorted_bounding_box" + argspec: "args=[\'image_size\', \'bounding_boxes\', \'seed\', \'min_object_covered\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\', \'name\'], varargs=None, keywords=None, defaults=[\'0.1\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } member_method { name: "total_variation" argspec: "args=[\'images\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt index 1d74f47508a..0a2843431f2 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt @@ -4552,6 +4552,10 @@ tf_module { name: "StatelessRandomUniformInt" argspec: "args=[\'shape\', \'seed\', \'minval\', \'maxval\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], " } + member_method { + name: "StatelessSampleDistortedBoundingBox" + argspec: "args=[\'image_size\', \'bounding_boxes\', \'min_object_covered\', \'seed\', \'aspect_ratio_range\', \'area_range\', \'max_attempts\', \'use_image_if_no_bounding_boxes\', \'name\'], varargs=None, keywords=None, defaults=[\'[0.75, 1.33]\', \'[0.05, 1]\', \'100\', \'False\', \'None\'], " + } member_method { name: "StatelessTruncatedNormal" argspec: "args=[\'shape\', \'seed\', \'dtype\', \'name\'], varargs=None, keywords=None, defaults=[\"\", \'None\'], " From 4b901e2a7ea0b849ad1d1cea311cd131bc089ebe Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Mon, 10 Aug 2020 20:33:26 -0700 Subject: [PATCH 0802/1017] Disable the failed array_elementwise_ops_test for OSS build PiperOrigin-RevId: 325944589 Change-Id: Id94df83d6f35acbe912f09bae9f36763f1b90290 --- tensorflow/compiler/xla/tests/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 17444c042e7..3dac381ae7d 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -728,6 +728,7 @@ xla_test( name = "array_elementwise_ops_test", srcs = ["array_elementwise_ops_test.cc"], shard_count = 25, + tags = ["no_oss"], # b/163416869 deps = [ ":test_macros_header", "//tensorflow/compiler/xla:array2d", From 3812bce0ef02a3fc23f2beb55e633da74b2958d3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 21:18:02 -0700 Subject: [PATCH 0803/1017] Update ops-related pbtxt files. PiperOrigin-RevId: 325949809 Change-Id: I7ce980f2d0b6a8f392d05b3aa54777bfea7c9d3f --- .../StatelessSampleDistortedBoundingBox.pbtxt | 88 +++++++++++++++++++ tensorflow/core/ops/ops.pbtxt | 88 +++++++++++++++++++ 2 files changed, 176 insertions(+) create mode 100644 tensorflow/core/ops/compat/ops_history_v2/StatelessSampleDistortedBoundingBox.pbtxt diff --git a/tensorflow/core/ops/compat/ops_history_v2/StatelessSampleDistortedBoundingBox.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/StatelessSampleDistortedBoundingBox.pbtxt new file mode 100644 index 00000000000..6858a110cf4 --- /dev/null +++ b/tensorflow/core/ops/compat/ops_history_v2/StatelessSampleDistortedBoundingBox.pbtxt @@ -0,0 +1,88 @@ +op { + name: "StatelessSampleDistortedBoundingBox" + input_arg { + name: "image_size" + type_attr: "T" + } + input_arg { + name: "bounding_boxes" + type: DT_FLOAT + } + input_arg { + name: "min_object_covered" + type: DT_FLOAT + } + input_arg { + name: "seed" + type_attr: "Tseed" + } + output_arg { + name: "begin" + type_attr: "T" + } + output_arg { + name: "size" + type_attr: "T" + } + output_arg { + name: "bboxes" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + } + } + } + attr { + name: "Tseed" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + attr { + name: "aspect_ratio_range" + type: "list(float)" + default_value { + list { + f: 0.75 + f: 1.33 + } + } + } + attr { + name: "area_range" + type: "list(float)" + default_value { + list { + f: 0.05 + f: 1 + } + } + } + attr { + name: "max_attempts" + type: "int" + default_value { + i: 100 + } + } + attr { + name: "use_image_if_no_bounding_boxes" + type: "bool" + default_value { + b: false + } + } +} diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 7e138923a8d..7610619019d 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -49730,6 +49730,94 @@ op { } } } +op { + name: "StatelessSampleDistortedBoundingBox" + input_arg { + name: "image_size" + type_attr: "T" + } + input_arg { + name: "bounding_boxes" + type: DT_FLOAT + } + input_arg { + name: "min_object_covered" + type: DT_FLOAT + } + input_arg { + name: "seed" + type_attr: "Tseed" + } + output_arg { + name: "begin" + type_attr: "T" + } + output_arg { + name: "size" + type_attr: "T" + } + output_arg { + name: "bboxes" + type: DT_FLOAT + } + attr { + name: "T" + type: "type" + allowed_values { + list { + type: DT_UINT8 + type: DT_INT8 + type: DT_INT16 + type: DT_INT32 + type: DT_INT64 + } + } + } + attr { + name: "Tseed" + type: "type" + allowed_values { + list { + type: DT_INT32 + type: DT_INT64 + } + } + } + attr { + name: "aspect_ratio_range" + type: "list(float)" + default_value { + list { + f: 0.75 + f: 1.33 + } + } + } + attr { + name: "area_range" + type: "list(float)" + default_value { + list { + f: 0.05 + f: 1 + } + } + } + attr { + name: "max_attempts" + type: "int" + default_value { + i: 100 + } + } + attr { + name: "use_image_if_no_bounding_boxes" + type: "bool" + default_value { + b: false + } + } +} op { name: "StatelessTruncatedNormal" input_arg { From 1731f0719a3c11e1f5122d8f81534a580b298fe0 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Mon, 10 Aug 2020 21:24:51 -0700 Subject: [PATCH 0804/1017] Provide an implementation for LinearizeToBuffers in TpuTransferManager PiperOrigin-RevId: 325950544 Change-Id: I600856e01f357bcfe3c8f2071ebc42f18a809678 --- tensorflow/stream_executor/tpu/BUILD | 1 + .../tpu/tpu_transfer_manager.cc | 28 +++++++++++++++++++ .../tpu/tpu_transfer_manager.h | 4 +-- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/tensorflow/stream_executor/tpu/BUILD b/tensorflow/stream_executor/tpu/BUILD index 207984e0c89..93998a4aefc 100644 --- a/tensorflow/stream_executor/tpu/BUILD +++ b/tensorflow/stream_executor/tpu/BUILD @@ -230,6 +230,7 @@ cc_library( hdrs = ["tpu_transfer_manager.h"], deps = [ ":c_api_conversions", + ":noncopyable_buffer", ":proto_helper", ":status_helper", ":tpu_executor_base", diff --git a/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc b/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc index a7288003f8d..9b268a6d8c9 100644 --- a/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc +++ b/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/tpu/tpu_api.h" #include "tensorflow/stream_executor/device_memory.h" #include "tensorflow/stream_executor/tpu/c_api_conversions.h" +#include "tensorflow/stream_executor/tpu/noncopyable_buffer.h" #include "tensorflow/stream_executor/tpu/proto_helper.h" #include "tensorflow/stream_executor/tpu/status_helper.h" #include "tensorflow/stream_executor/tpu/tpu_executor_c_api.h" @@ -168,4 +169,31 @@ Status TpuTransferManager::WriteSingleTupleIndexTable( return status.status(); } +Status TpuTransferManager::LinearizeToBuffers( + const xla::LiteralSlice& literal, + std::deque* buffers) { + XLA_Literal c_literal; + ApiConverter::ToC(literal, &c_literal); + + char** buffers_array; + int64_t* buffers_size; + int64_t buffers_array_size; + StatusHelper status; + + tpu::ExecutorApiFn()->TpuTransferManager_LinearizeToBuffersFn( + manager_, &c_literal, &buffers_array, &buffers_size, &buffers_array_size, + status.c_status); + + for (int64_t i = 0; i < buffers_array_size; ++i) { + tpu::NoncopyableBuffer buf(buffers_size[i]); + memcpy(buf.mutable_data().data(), buffers_array[i], buffers_size[i]); + buffers->push_back(std::move(buf)); + } + + tpu::ExecutorApiFn()->TpuTransferManager_FreeBuffersFn( + buffers_array, buffers_size, buffers_array_size); + + return status.status(); +} + } // namespace tensorflow diff --git a/tensorflow/stream_executor/tpu/tpu_transfer_manager.h b/tensorflow/stream_executor/tpu/tpu_transfer_manager.h index e758c702204..558a5106d86 100644 --- a/tensorflow/stream_executor/tpu/tpu_transfer_manager.h +++ b/tensorflow/stream_executor/tpu/tpu_transfer_manager.h @@ -83,9 +83,7 @@ class TpuTransferManager : public xla::TpuTransferManagerInterface { Status LinearizeToBuffers( const xla::LiteralSlice& literal, - std::deque* buffers) override { - LOG(FATAL) << "Not yet implemented."; - } + std::deque* buffers) override; private: XLA_TransferManager* manager_; From acbfab2b0191ec7a845eb967b39f1cb08d3a3c3a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 10 Aug 2020 21:47:22 -0700 Subject: [PATCH 0805/1017] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 325952753 Change-Id: Ib8753517fc99d292c911071948e591a8ba0d4b05 --- tensorflow/go/op/wrappers.go | 147 +++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index cd6284aab05..4d39ab20deb 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -12077,6 +12077,153 @@ func ExtractGlimpse(scope *Scope, input tf.Output, size tf.Output, offsets tf.Ou return op.Output(0) } +// StatelessSampleDistortedBoundingBoxAttr is an optional argument to StatelessSampleDistortedBoundingBox. +type StatelessSampleDistortedBoundingBoxAttr func(optionalAttr) + +// StatelessSampleDistortedBoundingBoxAspectRatioRange sets the optional aspect_ratio_range attribute to value. +// +// value: The cropped area of the image must have an aspect ratio = +// width / height within this range. +// If not specified, defaults to +func StatelessSampleDistortedBoundingBoxAspectRatioRange(value []float32) StatelessSampleDistortedBoundingBoxAttr { + return func(m optionalAttr) { + m["aspect_ratio_range"] = value + } +} + +// StatelessSampleDistortedBoundingBoxAreaRange sets the optional area_range attribute to value. +// +// value: The cropped area of the image must contain a fraction of the +// supplied image within this range. +// If not specified, defaults to +func StatelessSampleDistortedBoundingBoxAreaRange(value []float32) StatelessSampleDistortedBoundingBoxAttr { + return func(m optionalAttr) { + m["area_range"] = value + } +} + +// StatelessSampleDistortedBoundingBoxMaxAttempts sets the optional max_attempts attribute to value. +// +// value: Number of attempts at generating a cropped region of the image +// of the specified constraints. After `max_attempts` failures, return the entire +// image. +// If not specified, defaults to 100 +func StatelessSampleDistortedBoundingBoxMaxAttempts(value int64) StatelessSampleDistortedBoundingBoxAttr { + return func(m optionalAttr) { + m["max_attempts"] = value + } +} + +// StatelessSampleDistortedBoundingBoxUseImageIfNoBoundingBoxes sets the optional use_image_if_no_bounding_boxes attribute to value. +// +// value: Controls behavior if no bounding boxes supplied. +// If true, assume an implicit bounding box covering the whole input. If false, +// raise an error. +// If not specified, defaults to false +func StatelessSampleDistortedBoundingBoxUseImageIfNoBoundingBoxes(value bool) StatelessSampleDistortedBoundingBoxAttr { + return func(m optionalAttr) { + m["use_image_if_no_bounding_boxes"] = value + } +} + +// Generate a randomly distorted bounding box for an image deterministically. +// +// Bounding box annotations are often supplied in addition to ground-truth labels +// in image recognition or object localization tasks. A common technique for +// training such a system is to randomly distort an image while preserving its +// content, i.e. *data augmentation*. This Op, given the same `seed`, +// deterministically outputs a randomly distorted localization of an object, i.e. +// bounding box, given an `image_size`, `bounding_boxes` and a series of +// constraints. +// +// The output of this Op is a single bounding box that may be used to crop the +// original image. The output is returned as 3 tensors: `begin`, `size` and +// `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the +// image. The latter may be supplied to `tf.image.draw_bounding_boxes` to visualize +// what the bounding box looks like. +// +// Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. The +// bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and +// the height of the underlying image. +// +// The output of this Op is guaranteed to be the same given the same `seed` and is +// independent of how many times the function is called, and independent of global +// seed settings (e.g. `tf.random.set_seed`). +// +// Example usage: +// +// >>> image = np.array([[[1], [2], [3]], [[4], [5], [6]], [[7], [8], [9]]]) +// >>> bbox = tf.constant( +// ... [0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]) +// >>> seed = (1, 2) +// >>> # Generate a single distorted bounding box. +// >>> bbox_begin, bbox_size, bbox_draw = ( +// ... tf.image.stateless_sample_distorted_bounding_box( +// ... tf.shape(image), bounding_boxes=bbox, seed=seed)) +// >>> # Employ the bounding box to distort the image. +// >>> tf.slice(image, bbox_begin, bbox_size) +// +// >>> # Draw the bounding box in an image summary. +// >>> colors = np.array([[1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]) +// >>> tf.image.draw_bounding_boxes( +// ... tf.expand_dims(tf.cast(image, tf.float32),0), bbox_draw, colors) +// +// +// Note that if no bounding box information is available, setting +// `use_image_if_no_bounding_boxes = true` will assume there is a single implicit +// bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is +// false and no bounding boxes are supplied, an error is raised. +// +// Arguments: +// image_size: 1-D, containing `[height, width, channels]`. +// bounding_boxes: 3-D with shape `[batch, N, 4]` describing the N bounding boxes +// associated with the image. +// min_object_covered: The cropped area of the image must contain at least this +// fraction of any bounding box supplied. The value of this parameter should be +// non-negative. In the case of 0, the cropped area does not need to overlap +// any of the bounding boxes supplied. +// seed: 1-D with shape `[2]`. The seed to the random number generator. Must have dtype +// `int32` or `int64`. (When using XLA, only `int32` is allowed.) +// +// Returns: +// begin: 1-D, containing `[offset_height, offset_width, 0]`. Provide as input to +// `tf.slice`. +// size: 1-D, containing `[target_height, target_width, -1]`. Provide as input to +// `tf.slice`. +// bboxes: 3-D with shape `[1, 1, 4]` containing the distorted bounding box. +// Provide as input to `tf.image.draw_bounding_boxes`. +func StatelessSampleDistortedBoundingBox(scope *Scope, image_size tf.Output, bounding_boxes tf.Output, min_object_covered tf.Output, seed tf.Output, optional ...StatelessSampleDistortedBoundingBoxAttr) (begin tf.Output, size tf.Output, bboxes tf.Output) { + if scope.Err() != nil { + return + } + attrs := map[string]interface{}{} + for _, a := range optional { + a(attrs) + } + opspec := tf.OpSpec{ + Type: "StatelessSampleDistortedBoundingBox", + Input: []tf.Input{ + image_size, bounding_boxes, min_object_covered, seed, + }, + Attrs: attrs, + } + op := scope.AddOperation(opspec) + return op.Output(0), op.Output(1), op.Output(2) +} + // SampleDistortedBoundingBoxAttr is an optional argument to SampleDistortedBoundingBox. type SampleDistortedBoundingBoxAttr func(optionalAttr) From 403258a15c64b0a6fb8d0d40996702bd304446af Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Mon, 10 Aug 2020 22:26:41 -0700 Subject: [PATCH 0806/1017] [TF2XLA] Do not enable XLA devices by default XLA:CPU and XLA:GPU devices are a perennial source of confusion and hard-to-trace bugs due to: 1) Unexpected appearence in the device list 2) Soft placement on these devices Unfortunately, we can't permanently remove them because of a) Test coverage we need for tf2xla bridge when running on XLA devices in compiler/tests b) XRT using XLA:CPU/XLA:GPU devices to insert their hooks Moreover, long term XLA:TPU device is still useful so we can't remove the corresponding codepath either. This patch disables registration of XLA:CPU and XLA:GPU devices by default, unless the option `tf_xla_enable_xla_devices` is set to `True` before the device initialization. This option can be set using one of these ways: i) When running the process: run inside the environment `TF_XLA_FLAGS=--tf_xla_enable_xla_devices` ii) In Python: call `context.context().enable_xla_devices()` (should not be called outside a very specific number of tests) iii) In C++: set `GetXlaDeviceFlags()->tf_xla_enable_xla_devices` to `true` TBD: I. Good testing story: what is a good place to check that XLA devices are not registered? II. Removing special casing XLA devices inside TF: do we still need it for those tests which do actually use XLA devices? Probably most of it can be still removed. PiperOrigin-RevId: 325956767 Change-Id: Ic1bb050d1fe10b9af5a32071b8da9e39d35d9104 --- RELEASE.md | 3 + tensorflow/compiler/jit/BUILD | 1 + tensorflow/compiler/jit/flags.cc | 2 +- tensorflow/compiler/jit/kernels/xla_ops.cc | 2 +- .../jit/mark_for_compilation_pass_test.cc | 5 ++ .../jit/partially_decluster_pass_test.cc | 31 ------- .../compiler/jit/xla_compile_on_demand_op.cc | 53 ++++++----- tensorflow/compiler/jit/xla_device.cc | 29 +++--- tensorflow/compiler/jit/xla_device.h | 2 + .../compiler/jit/xla_ops_regular_devices.cc | 89 +++++++++++++++++++ tensorflow/compiler/jit/xla_platform_info.cc | 7 +- tensorflow/compiler/jit/xla_platform_info.h | 2 +- tensorflow/compiler/tests/BUILD | 1 + .../tests/unary_ops_composition_test.cc | 6 ++ .../compiler/tests/xla_device_gpu_test.py | 5 ++ tensorflow/compiler/tests/xla_test.py | 2 + tensorflow/compiler/tf2xla/BUILD | 2 + .../compiler/tf2xla/const_analysis_test.cc | 6 ++ .../fused_batchnorm_reserve_space_test.cc | 7 ++ tensorflow/compiler/xrt/BUILD | 1 + tensorflow/compiler/xrt/ops/xrt_state_ops.cc | 6 ++ .../optimizers/pin_to_host_optimizer_test.cc | 20 ----- tensorflow/python/eager/context.py | 14 +-- tensorflow/python/framework/config_test.py | 9 -- .../parallel_for/xla_control_flow_ops_test.py | 5 ++ tensorflow/python/tfe_wrapper.cc | 3 + 26 files changed, 206 insertions(+), 107 deletions(-) create mode 100644 tensorflow/compiler/jit/xla_ops_regular_devices.cc diff --git a/RELEASE.md b/RELEASE.md index 430e1b83885..241a5077251 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -33,6 +33,9 @@ shape assumptions (note that you can pass shapes with `None` entries for axes that are meant to be dynamic). You can also disable the input checking entirely by setting `model.input_spec = None`. +* XLA:CPU and XLA:GPU devices are no longer registered by default. Use + `TF_XLA_FLAGS=--tf_xla_enable_xla_devices` if you really need them (to be + removed). ## Known Caveats diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index d05bb8264c3..55400c7d3b7 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -206,6 +206,7 @@ cc_library( "xla_device.cc", "xla_device_context.cc", "xla_device_ops.cc", + "xla_ops_regular_devices.cc", "xla_platform_info.cc", ], hdrs = [ diff --git a/tensorflow/compiler/jit/flags.cc b/tensorflow/compiler/jit/flags.cc index ff085c854c6..a4a750bae0d 100644 --- a/tensorflow/compiler/jit/flags.cc +++ b/tensorflow/compiler/jit/flags.cc @@ -159,7 +159,7 @@ void AllocateAndParseFlags() { device_flags = new XlaDeviceFlags; device_flags->tf_xla_compile_on_demand = false; - device_flags->tf_xla_enable_xla_devices = true; + device_flags->tf_xla_enable_xla_devices = false; ops_flags = new XlaOpsCommonFlags; ops_flags->tf_xla_always_defer_compilation = false; diff --git a/tensorflow/compiler/jit/kernels/xla_ops.cc b/tensorflow/compiler/jit/kernels/xla_ops.cc index 9cee4b9af28..de462928c46 100644 --- a/tensorflow/compiler/jit/kernels/xla_ops.cc +++ b/tensorflow/compiler/jit/kernels/xla_ops.cc @@ -191,7 +191,7 @@ static Status CompileToLocalExecutable( absl::optional tf_allocator_adapter; XlaCompiler::Options options = GenerateCompilerOptions( - cache, ctx, platform_info, has_ref_vars, &tf_allocator_adapter); + *cache, ctx, platform_info, has_ref_vars, &tf_allocator_adapter); std::map constant_args; for (int i : constants) { diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc index e88319bb732..1be3e5ba9e7 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc @@ -44,6 +44,11 @@ using ::tensorflow::testing::FindNodeByName; namespace tensorflow { namespace { +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + REGISTER_OP("UncompilableNullary").Output("o: float"); REGISTER_OP("UncompilableUnary").Input("a: float").Output("o: float"); diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc index 7378d17f88d..87c9fbf0af7 100644 --- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc +++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc @@ -406,37 +406,6 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) { EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0"); } -TEST(PartiallyDeclusterPassTest, DontDeclusterNonTensorFlowOps) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - Output dynamic_slice_operand = - ops::Placeholder(s.WithOpName("dynamic_slice_operand"), DT_INT32, - ops::Placeholder::Attrs{}); - Output dynamic_slice_begin = ops::Placeholder( - s.WithOpName("dynamic_slice_begin"), DT_INT32, ops::Placeholder::Attrs{}); - Output dynamic_slice_size = ops::Placeholder( - s.WithOpName("dynamic_slice_size"), DT_INT32, ops::Placeholder::Attrs{}); - Output dynamic_slice = - ops::XlaDynamicSlice(s.WithOpName("dynamic_slice"), dynamic_slice_operand, - dynamic_slice_begin, dynamic_slice_size); - - Output reshape_input = ops::Placeholder(s.WithOpName("reshape_input"), - DT_FLOAT, ops::Placeholder::Attrs{}); - Output reshape = - ops::Reshape(s.WithOpName("reshape"), reshape_input, dynamic_slice); - - AddToCluster({dynamic_slice.node(), reshape.node()}, "cluster_0"); - - std::unique_ptr graph = absl::make_unique(OpRegistry::Global()); - TF_ASSERT_OK(s.ToGraph(graph.get())); - - Node* n = FindNodeByName(*graph, "dynamic_slice"); - ASSERT_NE(n, nullptr); - - TF_ASSERT_OK(PartiallyDecluster(&graph)); - - EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0"); -} - TEST(PartiallyDeclusterPassTest, EliminatedUnusedNodes) { const char* const kClusteredProducer0Name = "ClusteredProducer0"; const char* const kClusteredProducer1Name = "ClusteredProducer1"; diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc index 73c512bfa6f..da251c2c8f3 100644 --- a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc +++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc @@ -48,9 +48,11 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, const ResourceVarsSnapshot& variable_args) { xla::LocalClient* client = static_cast(cache->client()); + absl::optional tf_allocator_adapter; + se::DeviceMemoryAllocator* allocator = + GetAllocator(&tf_allocator_adapter, ctx, platform_info_); XlaComputationLaunchContext launch_context( - client, client->backend().memory_allocator(), - client->default_device_ordinal(), + client, allocator, client->default_device_ordinal(), /*allocate_xla_tensors=*/platform_info_.xla_device_metadata() != nullptr, platform_info_.xla_device_metadata() ? platform_info_.xla_device_metadata()->UseMultipleStreams() @@ -76,7 +78,7 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, VLOG(2) << "Executing computation: " << name(); xla::ExecutableRunOptions run_options; run_options.set_stream(stream); - run_options.set_allocator(client->backend().memory_allocator()); + run_options.set_allocator(allocator); run_options.set_intra_op_thread_pool(&ctx->eigen_cpu_device()); run_options.set_rng_seed(GetXLARandomSeed()); @@ -108,6 +110,7 @@ Status XlaCompileOnDemandOp::Compile( for (int64 i = 0; i < ctx->num_inputs(); ++i) { const Tensor& device_tensor = ctx->input(i); + if (const XlaTensor* xla_tensor = XlaTensor::FromTensor(&device_tensor)) { if (xla_tensor->has_host_tensor()) { if (absl::c_binary_search(constant_input_indices, i)) { @@ -118,24 +121,30 @@ Status XlaCompileOnDemandOp::Compile( if (!constant_arguments.count(i)) { if (absl::c_binary_search(constant_input_indices, i)) { - // Slow path; the argument is not available as a host constant so we - // must fetch it synchronously. - Tensor host_tensor; - AllocatorAttributes attrs; - attrs.set_on_host(true); - TF_RETURN_IF_ERROR(ctx->allocate_temp( - device_tensor.dtype(), device_tensor.shape(), &host_tensor, attrs)); - Status status = ctx->op_device_context()->CopyDeviceTensorToCPUSync( - &device_tensor, "ConstantArgument", - reinterpret_cast(ctx->device()), &host_tensor); - if (!status.ok()) { - LOG(ERROR) << "Copying tensor of shape " - << device_tensor.shape().DebugString() << " from " - << ctx->device()->name() << "to CPU failed with " - << status.ToString(); - return status; + if (ctx->input_memory_type(i) != HOST_MEMORY && + ctx->op_device_context()) { + // Slow path; the argument is not available as a host constant so we + // must fetch it synchronously. + Tensor host_tensor; + AllocatorAttributes attrs; + attrs.set_on_host(true); + TF_RETURN_IF_ERROR(ctx->allocate_temp(device_tensor.dtype(), + device_tensor.shape(), + &host_tensor, attrs)); + Status status = ctx->op_device_context()->CopyDeviceTensorToCPUSync( + &device_tensor, "ConstantArgument", + reinterpret_cast(ctx->device()), &host_tensor); + if (!status.ok()) { + LOG(ERROR) << "Copying tensor of shape " + << device_tensor.shape().DebugString() << " from " + << ctx->device()->name() << "to CPU failed with " + << status.ToString(); + return status; + } + constant_arguments[i] = host_tensor; + } else { + constant_arguments[i] = device_tensor; } - constant_arguments[i] = host_tensor; } } } @@ -153,7 +162,7 @@ Status XlaCompileOnDemandOp::Compile( absl::optional tf_allocator_adapter; XlaCompiler::Options options = - GenerateCompilerOptions(*cache, ctx, platform_info_, + GenerateCompilerOptions(**cache, ctx, platform_info_, /*has_ref_vars=*/true, &tf_allocator_adapter); XlaCompiler::CompileOptions compile_options; @@ -184,6 +193,8 @@ void XlaCompileOnDemandOp::Compute(OpKernelContext* ctx) { xla::LocalExecutable* executable; ResourceVarsSnapshot variable_args; XlaCompilationCache* cache; + OP_REQUIRES(ctx, ctx->function_library(), + errors::Internal("Function library missing")); OP_REQUIRES_OK(ctx, Compile(ctx, &result, &cache, &variable_args, &executable)); diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc index 7842513331d..c47c9a29c1a 100644 --- a/tensorflow/compiler/jit/xla_device.cc +++ b/tensorflow/compiler/jit/xla_device.cc @@ -61,6 +61,21 @@ limitations under the License. namespace tensorflow { +// Default PaddedShapeFn implementation that simply returns the unpadded +// on-device shape. This is accurate for CPU and GPU devices that neither +// transpose nor pad tensors. +Status DefaultPaddedShapeFn(const Tensor& tensor, xla::Shape* shape) { + const tensorflow::XlaTensor* xla_tensor = + tensorflow::XlaTensor::FromTensor(&tensor); + if (xla_tensor == nullptr) { + return TensorShapeToXLAShape(tensor.dtype(), tensor.shape(), shape); + } + + const xla::ShapedBuffer& shaped_buffer = xla_tensor->shaped_buffer(); + *shape = shaped_buffer.on_device_shape(); + return Status::OK(); +} + // Caches a XlaDeviceAllocator per pair. A // XlaDeviceAllocator is created on demand and is associated with a // XlaDevice. It outlives the device itself (for instance, the buffer @@ -116,20 +131,6 @@ XlaDeviceAllocator* XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator( namespace { -// Default PaddedShapeFn implementation that simply returns the unpadded -// on-device shape. This is accurate for CPU and GPU devices that neither -// transpose nor pad tensors. -Status DefaultPaddedShapeFn(const Tensor& tensor, xla::Shape* shape) { - const tensorflow::XlaTensor* xla_tensor = - tensorflow::XlaTensor::FromTensor(&tensor); - if (xla_tensor == nullptr) { - return TensorShapeToXLAShape(tensor.dtype(), tensor.shape(), shape); - } - - const xla::ShapedBuffer& shaped_buffer = xla_tensor->shaped_buffer(); - *shape = shaped_buffer.on_device_shape(); - return Status::OK(); -} static DeviceAttributes BuildXlaDeviceAttributes(const string& name_prefix, const string& device_name, diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h index 30f9a99e36a..f7e7ee9cf95 100644 --- a/tensorflow/compiler/jit/xla_device.h +++ b/tensorflow/compiler/jit/xla_device.h @@ -280,6 +280,8 @@ struct XlaDeviceOpRegistrations { XlaDeviceOpRegistrations* RegisterXlaDeviceKernels(const char* device, const char* jit_device); +Status DefaultPaddedShapeFn(const Tensor& tensor, xla::Shape* shape); + } // namespace tensorflow #endif // TENSORFLOW_COMPILER_JIT_XLA_DEVICE_H_ diff --git a/tensorflow/compiler/jit/xla_ops_regular_devices.cc b/tensorflow/compiler/jit/xla_ops_regular_devices.cc new file mode 100644 index 00000000000..82510a4926b --- /dev/null +++ b/tensorflow/compiler/jit/xla_ops_regular_devices.cc @@ -0,0 +1,89 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Register XlaXXX operations on regular CPU/GPU devices using +// `XlaCompileOnDemandOp`. +#include "tensorflow/compiler/jit/xla_compile_on_demand_op.h" +#include "tensorflow/core/framework/op_kernel.h" + +namespace tensorflow { + +#define REGISTER_XLA_OPS_ON_DEVICE(DEVICE) \ + REGISTER_KERNEL_BUILDER(Name("XlaConv") \ + .HostMemory("window_strides") \ + .HostMemory("padding") \ + .HostMemory("lhs_dilation") \ + .HostMemory("rhs_dilation") \ + .HostMemory("feature_group_count") \ + .Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("XlaBroadcastHelper").HostMemory("broadcast_dims").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSelfAdjointEig").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSvd").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaDot").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaDynamicSlice").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaDynamicUpdateSlice").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaIf").Device(DEVICE), XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaPad").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaRecv").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaReduce").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaReduceWindow").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSelectAndScatter") \ + .HostMemory("window_dimensions") \ + .HostMemory("window_strides") \ + .HostMemory("padding") \ + .Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSend").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSort").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaKeyValueSort").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaWhile").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaDequantize").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaEinsum").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSpmdShardToFullShape").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSharding").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaReplicaId").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaGather") \ + .HostMemory("start_indices") \ + .HostMemory("slice_sizes") \ + .Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaScatter").Device(DEVICE), \ + XlaCompileOnDemandOp); + +REGISTER_XLA_OPS_ON_DEVICE(DEVICE_CPU); +REGISTER_XLA_OPS_ON_DEVICE(DEVICE_GPU); + +} // namespace tensorflow diff --git a/tensorflow/compiler/jit/xla_platform_info.cc b/tensorflow/compiler/jit/xla_platform_info.cc index e2a89353055..a5e12b37563 100644 --- a/tensorflow/compiler/jit/xla_platform_info.cc +++ b/tensorflow/compiler/jit/xla_platform_info.cc @@ -128,16 +128,17 @@ se::DeviceMemoryAllocator* GetAllocator( } XlaCompiler::Options GenerateCompilerOptions( - XlaCompilationCache* cache, OpKernelContext* ctx, + const XlaCompilationCache& cache, OpKernelContext* ctx, const XlaPlatformInfo& platform_info, bool has_ref_vars, absl::optional* tf_allocator_adapter) { + CHECK(ctx->function_library()); XlaCompiler::Options options; - options.client = static_cast(cache->client()); + options.client = static_cast(cache.client()); if (ctx->op_device_context() != nullptr) { options.device_ordinal = ctx->op_device_context()->stream()->parent()->device_ordinal(); } - options.device_type = cache->device_type(); + options.device_type = cache.device_type(); options.flib_def = ctx->function_library()->GetFunctionLibraryDefinition(); options.graph_def_version = ctx->function_library()->graph_def_version(); options.allow_cpu_custom_calls = diff --git a/tensorflow/compiler/jit/xla_platform_info.h b/tensorflow/compiler/jit/xla_platform_info.h index dac45529ac9..d58b32a996f 100644 --- a/tensorflow/compiler/jit/xla_platform_info.h +++ b/tensorflow/compiler/jit/xla_platform_info.h @@ -99,7 +99,7 @@ se::DeviceMemoryAllocator* GetAllocator( // Returns created options for the XLA compiler, and writes the used allocator // into `tf_allocator_adapter`. XlaCompiler::Options GenerateCompilerOptions( - XlaCompilationCache* cache, OpKernelContext* ctx, + const XlaCompilationCache& cache, OpKernelContext* ctx, const XlaPlatformInfo& platform_info, bool has_ref_vars, absl::optional* tf_allocator_adapter); diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index 924834fc0fc..7f099540f39 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -1687,6 +1687,7 @@ tf_cuda_cc_test( deps = [ "//tensorflow/cc:cc_ops", "//tensorflow/compiler/jit", + "//tensorflow/compiler/jit:flags", "//tensorflow/compiler/jit:xla_kernel_creator", "//tensorflow/compiler/tf2xla:xla_compiler", "//tensorflow/core:core_cpu", diff --git a/tensorflow/compiler/tests/unary_ops_composition_test.cc b/tensorflow/compiler/tests/unary_ops_composition_test.cc index 569261de094..0e40c497c24 100644 --- a/tensorflow/compiler/tests/unary_ops_composition_test.cc +++ b/tensorflow/compiler/tests/unary_ops_composition_test.cc @@ -20,6 +20,7 @@ limitations under the License. #include #include "absl/synchronization/notification.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/common_runtime/device_factory.h" @@ -43,6 +44,11 @@ limitations under the License. namespace tensorflow { namespace { +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + class UnaryOpsCompositionTest : public OpsTestBase { protected: template diff --git a/tensorflow/compiler/tests/xla_device_gpu_test.py b/tensorflow/compiler/tests/xla_device_gpu_test.py index 1e30ebd55d0..304405c82ce 100644 --- a/tensorflow/compiler/tests/xla_device_gpu_test.py +++ b/tensorflow/compiler/tests/xla_device_gpu_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.client import session as session_lib +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -27,6 +28,10 @@ from tensorflow.python.platform import test class XlaDeviceGpuTest(test.TestCase): + def __init__(self, method_name="runTest"): + super(XlaDeviceGpuTest, self).__init__(method_name) + context.context().enable_xla_devices() + def testCopiesToAndFromGpuWork(self): """Tests that copies between GPU and XLA devices work.""" if not test.is_gpu_available(): diff --git a/tensorflow/compiler/tests/xla_test.py b/tensorflow/compiler/tests/xla_test.py index 3b057ed8b17..8c31629c234 100644 --- a/tensorflow/compiler/tests/xla_test.py +++ b/tensorflow/compiler/tests/xla_test.py @@ -83,6 +83,8 @@ class XLATestCase(test.TestCase): def __init__(self, method_name='runTest'): super(XLATestCase, self).__init__(method_name) + if 'XLA' in FLAGS.test_device: + context.context().enable_xla_devices() context.context().enable_mlir_bridge = test_util.is_mlir_bridge_enabled() self.device = FLAGS.test_device diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD index 1e57c11b2cf..ac999d875de 100644 --- a/tensorflow/compiler/tf2xla/BUILD +++ b/tensorflow/compiler/tf2xla/BUILD @@ -787,6 +787,7 @@ tf_cc_test( "//tensorflow/cc:function_ops", "//tensorflow/cc:functional_ops", "//tensorflow/cc:ops", + "//tensorflow/compiler/jit:flags", "//tensorflow/compiler/jit:xla_cluster_util", "//tensorflow/compiler/tf2xla/kernels:xla_ops", "//tensorflow/core:core_cpu_internal", @@ -1087,6 +1088,7 @@ tf_cuda_cc_test( "//tensorflow/cc:ops", "//tensorflow/cc:scope", "//tensorflow/compiler/jit", + "//tensorflow/compiler/jit:flags", "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:framework_internal", diff --git a/tensorflow/compiler/tf2xla/const_analysis_test.cc b/tensorflow/compiler/tf2xla/const_analysis_test.cc index 936b74f7b33..c7c8702b49b 100644 --- a/tensorflow/compiler/tf2xla/const_analysis_test.cc +++ b/tensorflow/compiler/tf2xla/const_analysis_test.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/cc/ops/function_ops.h" #include "tensorflow/cc/ops/functional_ops.h" #include "tensorflow/cc/ops/standard_ops.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/jit/xla_cluster_util.h" #include "tensorflow/core/common_runtime/process_function_library_runtime.h" #include "tensorflow/core/graph/algorithm.h" @@ -217,5 +218,10 @@ TEST(ConstAnalysisTest, RespectExplicitAttr_1) { EXPECT_EQ(const_args, std::vector({true})); } +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc b/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc index 1a26f974989..02f178f9acf 100644 --- a/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc +++ b/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/cc/ops/array_ops.h" #include "tensorflow/cc/ops/const_op.h" #include "tensorflow/cc/ops/nn_ops.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/framework/device_attributes.pb.h" #include "tensorflow/core/framework/graph.pb.h" #include "tensorflow/core/framework/tensor.h" @@ -139,5 +140,11 @@ TEST(FusedBatchnormReserveSpaceTest, Test) { test::ExpectClose(results[0], results[1], /*atol=*/1e-4); test::ExpectClose(results[2], results[3], /*atol=*/1e-4); } + +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/xrt/BUILD b/tensorflow/compiler/xrt/BUILD index 6a704be4adb..172a970d207 100644 --- a/tensorflow/compiler/xrt/BUILD +++ b/tensorflow/compiler/xrt/BUILD @@ -96,6 +96,7 @@ tf_gen_op_libs( "xrt_execute_op", ], deps = [ + "//tensorflow/compiler/jit:flags", "//tensorflow/core:lib", ], ) diff --git a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc index a4be39b96c6..321d7409103 100644 --- a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc +++ b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/framework/common_shape_fns.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/shape_inference.h" @@ -20,6 +21,11 @@ limitations under the License. namespace tensorflow { +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + REGISTER_OP("XRTAllocate") .Input("allocation: string") .Output("handle: int64") diff --git a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc index 641b5b4ef31..4fe8e6a6c3f 100644 --- a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc @@ -44,26 +44,6 @@ TEST_F(PinToHostOptimizerTest, TryFindHostDeviceCpuXlaGpu) { "/device:CPU:0"); } -TEST_F(PinToHostOptimizerTest, TryFindHostDeviceXlaCpuXlaGpu) { - gtl::FlatSet devices = {"/device:XLA_CPU:0", "/device:XLA_GPU:0"}; - - EXPECT_EQ(internal::TryFindHostDevice(devices, false, ""), ""); - EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:0"), - "/device:XLA_CPU:0"); - EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:*"), - "/device:XLA_CPU:0"); -} - -TEST_F(PinToHostOptimizerTest, TryFindHostDeviceXlaGpu) { - gtl::FlatSet devices = {"/device:XLA_GPU:0"}; - - EXPECT_EQ(internal::TryFindHostDevice(devices, false, ""), ""); - EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:0"), - ""); - EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:*"), - ""); -} - TEST_F(PinToHostOptimizerTest, OptimizeSmallOpsToHost) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); Output a = ops::Const(s.WithOpName("a"), 1, {1024, 1024}); diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 765c77af7cd..9c939fe0a76 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -42,6 +42,7 @@ from tensorflow.python.framework import device as pydev from tensorflow.python.util import compat from tensorflow.python.util import is_in_graph_mode from tensorflow.python.util import tf_contextlib +from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export GRAPH_MODE = 0 @@ -1254,12 +1255,7 @@ class Context(object): p: i for i, p in enumerate(self._physical_devices) } - # Construct the visible device list from all physical devices but ignore - # XLA devices - self._visible_device_list = [ - d for d in self._physical_devices - if not d.device_type.startswith("XLA") - ] + self._visible_device_list = list(self._physical_devices) self._memory_growth_map = { d: None for d in self._physical_devices if d.device_type == "GPU" } @@ -1493,6 +1489,12 @@ class Context(object): self._virtual_device_map[dev] = virtual_devices + @deprecated( + None, "XLA:CPU and XLA:GPU devices are deprecated", warn_once=True) + def enable_xla_devices(self): + """Enables XLA:CPU and XLA:GPU devices registration.""" + pywrap_tfe.TF_EnableXlaDevices() + @property def enable_mlir_bridge(self): return pywrap_tfe.TF_IsMlirBridgeEnabled() diff --git a/tensorflow/python/framework/config_test.py b/tensorflow/python/framework/config_test.py index 70857ef4b83..ee7e111f6b0 100644 --- a/tensorflow/python/framework/config_test.py +++ b/tensorflow/python/framework/config_test.py @@ -435,9 +435,6 @@ class DeviceTest(test.TestCase): self.assertEqual(len(config.get_visible_devices('CPU')), 1) self.assertGreater(len(config.get_visible_devices('GPU')), 0) - # get_visible_devices filters out XLA_* devices. list_logical_devices does - # not, but we can't call it here because it initializes the devices and - # calling set_visible_devices after that is disallowed. self.assertEqual(len(config.get_visible_devices('XLA_GPU')), 0) config.set_visible_devices(cpus[0]) @@ -451,12 +448,6 @@ class DeviceTest(test.TestCase): a = array_ops.identity(1.0) self.evaluate(a) - with self.assertRaisesRegex(errors.InvalidArgumentError, - 'Could not satisfy'): - with ops.device('/device:XLA_GPU:0'): - a = array_ops.identity(1.0) - self.evaluate(a) - # Modifying the visible devices is not supported with self.assertRaisesRegex(RuntimeError, 'cannot be modified'): config.set_visible_devices(gpus) diff --git a/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py index 33f0d7b76ae..188df3f9b87 100644 --- a/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py +++ b/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py @@ -22,6 +22,7 @@ from __future__ import print_function from tensorflow.compiler.tf2xla.python import xla as xla_ops from tensorflow.python.compiler.xla import jit from tensorflow.python.compiler.xla import xla +from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import test_util @@ -39,6 +40,10 @@ from tensorflow.python.platform import test @test_util.run_all_in_graph_and_eager_modes class PForTest(PForTestCase): + def __init__(self, method_name="runTest"): + super(PForTest, self).__init__(method_name) + context.context().enable_xla_devices() + def test_xla_einsum(self): num_loop = 10 x_series = random_ops.random_uniform([num_loop, 9, 9]) diff --git a/tensorflow/python/tfe_wrapper.cc b/tensorflow/python/tfe_wrapper.cc index c66397036c0..0afd05e94cb 100644 --- a/tensorflow/python/tfe_wrapper.cc +++ b/tensorflow/python/tfe_wrapper.cc @@ -444,6 +444,9 @@ PYBIND11_MODULE(_pywrap_tfe, m) { m.def("TF_EnableMlirBridge", [](bool enabled) { tensorflow::GetMlirCommonFlags()->tf_mlir_enable_mlir_bridge = enabled; }); + m.def("TF_EnableXlaDevices", [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + }); // // TFE_Context Logic m.def( From 61173a23ce65e9d9541d6a1bc84bfd17e3dcfd23 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 00:16:38 -0700 Subject: [PATCH 0807/1017] Internal change PiperOrigin-RevId: 325968539 Change-Id: Ie240a4ab7389e5b54923738ce290ca1a3b0bafa4 --- RELEASE.md | 3 - tensorflow/compiler/jit/BUILD | 1 - tensorflow/compiler/jit/flags.cc | 2 +- tensorflow/compiler/jit/kernels/xla_ops.cc | 2 +- .../jit/mark_for_compilation_pass_test.cc | 5 -- .../jit/partially_decluster_pass_test.cc | 31 +++++++ .../compiler/jit/xla_compile_on_demand_op.cc | 53 +++++------ tensorflow/compiler/jit/xla_device.cc | 29 +++--- tensorflow/compiler/jit/xla_device.h | 2 - .../compiler/jit/xla_ops_regular_devices.cc | 89 ------------------- tensorflow/compiler/jit/xla_platform_info.cc | 7 +- tensorflow/compiler/jit/xla_platform_info.h | 2 +- tensorflow/compiler/tests/BUILD | 1 - .../tests/unary_ops_composition_test.cc | 6 -- .../compiler/tests/xla_device_gpu_test.py | 5 -- tensorflow/compiler/tests/xla_test.py | 2 - tensorflow/compiler/tf2xla/BUILD | 2 - .../compiler/tf2xla/const_analysis_test.cc | 6 -- .../fused_batchnorm_reserve_space_test.cc | 7 -- tensorflow/compiler/xrt/BUILD | 1 - tensorflow/compiler/xrt/ops/xrt_state_ops.cc | 6 -- .../optimizers/pin_to_host_optimizer_test.cc | 20 +++++ tensorflow/python/eager/context.py | 14 ++- tensorflow/python/framework/config_test.py | 9 ++ .../parallel_for/xla_control_flow_ops_test.py | 5 -- tensorflow/python/tfe_wrapper.cc | 3 - 26 files changed, 107 insertions(+), 206 deletions(-) delete mode 100644 tensorflow/compiler/jit/xla_ops_regular_devices.cc diff --git a/RELEASE.md b/RELEASE.md index 241a5077251..430e1b83885 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -33,9 +33,6 @@ shape assumptions (note that you can pass shapes with `None` entries for axes that are meant to be dynamic). You can also disable the input checking entirely by setting `model.input_spec = None`. -* XLA:CPU and XLA:GPU devices are no longer registered by default. Use - `TF_XLA_FLAGS=--tf_xla_enable_xla_devices` if you really need them (to be - removed). ## Known Caveats diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index 55400c7d3b7..d05bb8264c3 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -206,7 +206,6 @@ cc_library( "xla_device.cc", "xla_device_context.cc", "xla_device_ops.cc", - "xla_ops_regular_devices.cc", "xla_platform_info.cc", ], hdrs = [ diff --git a/tensorflow/compiler/jit/flags.cc b/tensorflow/compiler/jit/flags.cc index a4a750bae0d..ff085c854c6 100644 --- a/tensorflow/compiler/jit/flags.cc +++ b/tensorflow/compiler/jit/flags.cc @@ -159,7 +159,7 @@ void AllocateAndParseFlags() { device_flags = new XlaDeviceFlags; device_flags->tf_xla_compile_on_demand = false; - device_flags->tf_xla_enable_xla_devices = false; + device_flags->tf_xla_enable_xla_devices = true; ops_flags = new XlaOpsCommonFlags; ops_flags->tf_xla_always_defer_compilation = false; diff --git a/tensorflow/compiler/jit/kernels/xla_ops.cc b/tensorflow/compiler/jit/kernels/xla_ops.cc index de462928c46..9cee4b9af28 100644 --- a/tensorflow/compiler/jit/kernels/xla_ops.cc +++ b/tensorflow/compiler/jit/kernels/xla_ops.cc @@ -191,7 +191,7 @@ static Status CompileToLocalExecutable( absl::optional tf_allocator_adapter; XlaCompiler::Options options = GenerateCompilerOptions( - *cache, ctx, platform_info, has_ref_vars, &tf_allocator_adapter); + cache, ctx, platform_info, has_ref_vars, &tf_allocator_adapter); std::map constant_args; for (int i : constants) { diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc index 1be3e5ba9e7..e88319bb732 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc @@ -44,11 +44,6 @@ using ::tensorflow::testing::FindNodeByName; namespace tensorflow { namespace { -static bool Initialized = [] { - tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; - return true; -}(); - REGISTER_OP("UncompilableNullary").Output("o: float"); REGISTER_OP("UncompilableUnary").Input("a: float").Output("o: float"); diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc index 87c9fbf0af7..7378d17f88d 100644 --- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc +++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc @@ -406,6 +406,37 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) { EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0"); } +TEST(PartiallyDeclusterPassTest, DontDeclusterNonTensorFlowOps) { + tensorflow::Scope s = tensorflow::Scope::NewRootScope(); + Output dynamic_slice_operand = + ops::Placeholder(s.WithOpName("dynamic_slice_operand"), DT_INT32, + ops::Placeholder::Attrs{}); + Output dynamic_slice_begin = ops::Placeholder( + s.WithOpName("dynamic_slice_begin"), DT_INT32, ops::Placeholder::Attrs{}); + Output dynamic_slice_size = ops::Placeholder( + s.WithOpName("dynamic_slice_size"), DT_INT32, ops::Placeholder::Attrs{}); + Output dynamic_slice = + ops::XlaDynamicSlice(s.WithOpName("dynamic_slice"), dynamic_slice_operand, + dynamic_slice_begin, dynamic_slice_size); + + Output reshape_input = ops::Placeholder(s.WithOpName("reshape_input"), + DT_FLOAT, ops::Placeholder::Attrs{}); + Output reshape = + ops::Reshape(s.WithOpName("reshape"), reshape_input, dynamic_slice); + + AddToCluster({dynamic_slice.node(), reshape.node()}, "cluster_0"); + + std::unique_ptr graph = absl::make_unique(OpRegistry::Global()); + TF_ASSERT_OK(s.ToGraph(graph.get())); + + Node* n = FindNodeByName(*graph, "dynamic_slice"); + ASSERT_NE(n, nullptr); + + TF_ASSERT_OK(PartiallyDecluster(&graph)); + + EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0"); +} + TEST(PartiallyDeclusterPassTest, EliminatedUnusedNodes) { const char* const kClusteredProducer0Name = "ClusteredProducer0"; const char* const kClusteredProducer1Name = "ClusteredProducer1"; diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc index da251c2c8f3..73c512bfa6f 100644 --- a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc +++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc @@ -48,11 +48,9 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, const ResourceVarsSnapshot& variable_args) { xla::LocalClient* client = static_cast(cache->client()); - absl::optional tf_allocator_adapter; - se::DeviceMemoryAllocator* allocator = - GetAllocator(&tf_allocator_adapter, ctx, platform_info_); XlaComputationLaunchContext launch_context( - client, allocator, client->default_device_ordinal(), + client, client->backend().memory_allocator(), + client->default_device_ordinal(), /*allocate_xla_tensors=*/platform_info_.xla_device_metadata() != nullptr, platform_info_.xla_device_metadata() ? platform_info_.xla_device_metadata()->UseMultipleStreams() @@ -78,7 +76,7 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, VLOG(2) << "Executing computation: " << name(); xla::ExecutableRunOptions run_options; run_options.set_stream(stream); - run_options.set_allocator(allocator); + run_options.set_allocator(client->backend().memory_allocator()); run_options.set_intra_op_thread_pool(&ctx->eigen_cpu_device()); run_options.set_rng_seed(GetXLARandomSeed()); @@ -110,7 +108,6 @@ Status XlaCompileOnDemandOp::Compile( for (int64 i = 0; i < ctx->num_inputs(); ++i) { const Tensor& device_tensor = ctx->input(i); - if (const XlaTensor* xla_tensor = XlaTensor::FromTensor(&device_tensor)) { if (xla_tensor->has_host_tensor()) { if (absl::c_binary_search(constant_input_indices, i)) { @@ -121,30 +118,24 @@ Status XlaCompileOnDemandOp::Compile( if (!constant_arguments.count(i)) { if (absl::c_binary_search(constant_input_indices, i)) { - if (ctx->input_memory_type(i) != HOST_MEMORY && - ctx->op_device_context()) { - // Slow path; the argument is not available as a host constant so we - // must fetch it synchronously. - Tensor host_tensor; - AllocatorAttributes attrs; - attrs.set_on_host(true); - TF_RETURN_IF_ERROR(ctx->allocate_temp(device_tensor.dtype(), - device_tensor.shape(), - &host_tensor, attrs)); - Status status = ctx->op_device_context()->CopyDeviceTensorToCPUSync( - &device_tensor, "ConstantArgument", - reinterpret_cast(ctx->device()), &host_tensor); - if (!status.ok()) { - LOG(ERROR) << "Copying tensor of shape " - << device_tensor.shape().DebugString() << " from " - << ctx->device()->name() << "to CPU failed with " - << status.ToString(); - return status; - } - constant_arguments[i] = host_tensor; - } else { - constant_arguments[i] = device_tensor; + // Slow path; the argument is not available as a host constant so we + // must fetch it synchronously. + Tensor host_tensor; + AllocatorAttributes attrs; + attrs.set_on_host(true); + TF_RETURN_IF_ERROR(ctx->allocate_temp( + device_tensor.dtype(), device_tensor.shape(), &host_tensor, attrs)); + Status status = ctx->op_device_context()->CopyDeviceTensorToCPUSync( + &device_tensor, "ConstantArgument", + reinterpret_cast(ctx->device()), &host_tensor); + if (!status.ok()) { + LOG(ERROR) << "Copying tensor of shape " + << device_tensor.shape().DebugString() << " from " + << ctx->device()->name() << "to CPU failed with " + << status.ToString(); + return status; } + constant_arguments[i] = host_tensor; } } } @@ -162,7 +153,7 @@ Status XlaCompileOnDemandOp::Compile( absl::optional tf_allocator_adapter; XlaCompiler::Options options = - GenerateCompilerOptions(**cache, ctx, platform_info_, + GenerateCompilerOptions(*cache, ctx, platform_info_, /*has_ref_vars=*/true, &tf_allocator_adapter); XlaCompiler::CompileOptions compile_options; @@ -193,8 +184,6 @@ void XlaCompileOnDemandOp::Compute(OpKernelContext* ctx) { xla::LocalExecutable* executable; ResourceVarsSnapshot variable_args; XlaCompilationCache* cache; - OP_REQUIRES(ctx, ctx->function_library(), - errors::Internal("Function library missing")); OP_REQUIRES_OK(ctx, Compile(ctx, &result, &cache, &variable_args, &executable)); diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc index c47c9a29c1a..7842513331d 100644 --- a/tensorflow/compiler/jit/xla_device.cc +++ b/tensorflow/compiler/jit/xla_device.cc @@ -61,21 +61,6 @@ limitations under the License. namespace tensorflow { -// Default PaddedShapeFn implementation that simply returns the unpadded -// on-device shape. This is accurate for CPU and GPU devices that neither -// transpose nor pad tensors. -Status DefaultPaddedShapeFn(const Tensor& tensor, xla::Shape* shape) { - const tensorflow::XlaTensor* xla_tensor = - tensorflow::XlaTensor::FromTensor(&tensor); - if (xla_tensor == nullptr) { - return TensorShapeToXLAShape(tensor.dtype(), tensor.shape(), shape); - } - - const xla::ShapedBuffer& shaped_buffer = xla_tensor->shaped_buffer(); - *shape = shaped_buffer.on_device_shape(); - return Status::OK(); -} - // Caches a XlaDeviceAllocator per pair. A // XlaDeviceAllocator is created on demand and is associated with a // XlaDevice. It outlives the device itself (for instance, the buffer @@ -131,6 +116,20 @@ XlaDeviceAllocator* XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator( namespace { +// Default PaddedShapeFn implementation that simply returns the unpadded +// on-device shape. This is accurate for CPU and GPU devices that neither +// transpose nor pad tensors. +Status DefaultPaddedShapeFn(const Tensor& tensor, xla::Shape* shape) { + const tensorflow::XlaTensor* xla_tensor = + tensorflow::XlaTensor::FromTensor(&tensor); + if (xla_tensor == nullptr) { + return TensorShapeToXLAShape(tensor.dtype(), tensor.shape(), shape); + } + + const xla::ShapedBuffer& shaped_buffer = xla_tensor->shaped_buffer(); + *shape = shaped_buffer.on_device_shape(); + return Status::OK(); +} static DeviceAttributes BuildXlaDeviceAttributes(const string& name_prefix, const string& device_name, diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h index f7e7ee9cf95..30f9a99e36a 100644 --- a/tensorflow/compiler/jit/xla_device.h +++ b/tensorflow/compiler/jit/xla_device.h @@ -280,8 +280,6 @@ struct XlaDeviceOpRegistrations { XlaDeviceOpRegistrations* RegisterXlaDeviceKernels(const char* device, const char* jit_device); -Status DefaultPaddedShapeFn(const Tensor& tensor, xla::Shape* shape); - } // namespace tensorflow #endif // TENSORFLOW_COMPILER_JIT_XLA_DEVICE_H_ diff --git a/tensorflow/compiler/jit/xla_ops_regular_devices.cc b/tensorflow/compiler/jit/xla_ops_regular_devices.cc deleted file mode 100644 index 82510a4926b..00000000000 --- a/tensorflow/compiler/jit/xla_ops_regular_devices.cc +++ /dev/null @@ -1,89 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Register XlaXXX operations on regular CPU/GPU devices using -// `XlaCompileOnDemandOp`. -#include "tensorflow/compiler/jit/xla_compile_on_demand_op.h" -#include "tensorflow/core/framework/op_kernel.h" - -namespace tensorflow { - -#define REGISTER_XLA_OPS_ON_DEVICE(DEVICE) \ - REGISTER_KERNEL_BUILDER(Name("XlaConv") \ - .HostMemory("window_strides") \ - .HostMemory("padding") \ - .HostMemory("lhs_dilation") \ - .HostMemory("rhs_dilation") \ - .HostMemory("feature_group_count") \ - .Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER( \ - Name("XlaBroadcastHelper").HostMemory("broadcast_dims").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaSelfAdjointEig").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaSvd").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaDot").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaDynamicSlice").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaDynamicUpdateSlice").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaIf").Device(DEVICE), XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaPad").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaRecv").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaReduce").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaReduceWindow").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaSelectAndScatter") \ - .HostMemory("window_dimensions") \ - .HostMemory("window_strides") \ - .HostMemory("padding") \ - .Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaSend").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaSort").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaKeyValueSort").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaWhile").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaDequantize").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaEinsum").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaSpmdShardToFullShape").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaSharding").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaReplicaId").Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaGather") \ - .HostMemory("start_indices") \ - .HostMemory("slice_sizes") \ - .Device(DEVICE), \ - XlaCompileOnDemandOp); \ - REGISTER_KERNEL_BUILDER(Name("XlaScatter").Device(DEVICE), \ - XlaCompileOnDemandOp); - -REGISTER_XLA_OPS_ON_DEVICE(DEVICE_CPU); -REGISTER_XLA_OPS_ON_DEVICE(DEVICE_GPU); - -} // namespace tensorflow diff --git a/tensorflow/compiler/jit/xla_platform_info.cc b/tensorflow/compiler/jit/xla_platform_info.cc index a5e12b37563..e2a89353055 100644 --- a/tensorflow/compiler/jit/xla_platform_info.cc +++ b/tensorflow/compiler/jit/xla_platform_info.cc @@ -128,17 +128,16 @@ se::DeviceMemoryAllocator* GetAllocator( } XlaCompiler::Options GenerateCompilerOptions( - const XlaCompilationCache& cache, OpKernelContext* ctx, + XlaCompilationCache* cache, OpKernelContext* ctx, const XlaPlatformInfo& platform_info, bool has_ref_vars, absl::optional* tf_allocator_adapter) { - CHECK(ctx->function_library()); XlaCompiler::Options options; - options.client = static_cast(cache.client()); + options.client = static_cast(cache->client()); if (ctx->op_device_context() != nullptr) { options.device_ordinal = ctx->op_device_context()->stream()->parent()->device_ordinal(); } - options.device_type = cache.device_type(); + options.device_type = cache->device_type(); options.flib_def = ctx->function_library()->GetFunctionLibraryDefinition(); options.graph_def_version = ctx->function_library()->graph_def_version(); options.allow_cpu_custom_calls = diff --git a/tensorflow/compiler/jit/xla_platform_info.h b/tensorflow/compiler/jit/xla_platform_info.h index d58b32a996f..dac45529ac9 100644 --- a/tensorflow/compiler/jit/xla_platform_info.h +++ b/tensorflow/compiler/jit/xla_platform_info.h @@ -99,7 +99,7 @@ se::DeviceMemoryAllocator* GetAllocator( // Returns created options for the XLA compiler, and writes the used allocator // into `tf_allocator_adapter`. XlaCompiler::Options GenerateCompilerOptions( - const XlaCompilationCache& cache, OpKernelContext* ctx, + XlaCompilationCache* cache, OpKernelContext* ctx, const XlaPlatformInfo& platform_info, bool has_ref_vars, absl::optional* tf_allocator_adapter); diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index 7f099540f39..924834fc0fc 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -1687,7 +1687,6 @@ tf_cuda_cc_test( deps = [ "//tensorflow/cc:cc_ops", "//tensorflow/compiler/jit", - "//tensorflow/compiler/jit:flags", "//tensorflow/compiler/jit:xla_kernel_creator", "//tensorflow/compiler/tf2xla:xla_compiler", "//tensorflow/core:core_cpu", diff --git a/tensorflow/compiler/tests/unary_ops_composition_test.cc b/tensorflow/compiler/tests/unary_ops_composition_test.cc index 0e40c497c24..569261de094 100644 --- a/tensorflow/compiler/tests/unary_ops_composition_test.cc +++ b/tensorflow/compiler/tests/unary_ops_composition_test.cc @@ -20,7 +20,6 @@ limitations under the License. #include #include "absl/synchronization/notification.h" -#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/common_runtime/device_factory.h" @@ -44,11 +43,6 @@ limitations under the License. namespace tensorflow { namespace { -static bool Initialized = [] { - tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; - return true; -}(); - class UnaryOpsCompositionTest : public OpsTestBase { protected: template diff --git a/tensorflow/compiler/tests/xla_device_gpu_test.py b/tensorflow/compiler/tests/xla_device_gpu_test.py index 304405c82ce..1e30ebd55d0 100644 --- a/tensorflow/compiler/tests/xla_device_gpu_test.py +++ b/tensorflow/compiler/tests/xla_device_gpu_test.py @@ -19,7 +19,6 @@ from __future__ import division from __future__ import print_function from tensorflow.python.client import session as session_lib -from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -28,10 +27,6 @@ from tensorflow.python.platform import test class XlaDeviceGpuTest(test.TestCase): - def __init__(self, method_name="runTest"): - super(XlaDeviceGpuTest, self).__init__(method_name) - context.context().enable_xla_devices() - def testCopiesToAndFromGpuWork(self): """Tests that copies between GPU and XLA devices work.""" if not test.is_gpu_available(): diff --git a/tensorflow/compiler/tests/xla_test.py b/tensorflow/compiler/tests/xla_test.py index 8c31629c234..3b057ed8b17 100644 --- a/tensorflow/compiler/tests/xla_test.py +++ b/tensorflow/compiler/tests/xla_test.py @@ -83,8 +83,6 @@ class XLATestCase(test.TestCase): def __init__(self, method_name='runTest'): super(XLATestCase, self).__init__(method_name) - if 'XLA' in FLAGS.test_device: - context.context().enable_xla_devices() context.context().enable_mlir_bridge = test_util.is_mlir_bridge_enabled() self.device = FLAGS.test_device diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD index ac999d875de..1e57c11b2cf 100644 --- a/tensorflow/compiler/tf2xla/BUILD +++ b/tensorflow/compiler/tf2xla/BUILD @@ -787,7 +787,6 @@ tf_cc_test( "//tensorflow/cc:function_ops", "//tensorflow/cc:functional_ops", "//tensorflow/cc:ops", - "//tensorflow/compiler/jit:flags", "//tensorflow/compiler/jit:xla_cluster_util", "//tensorflow/compiler/tf2xla/kernels:xla_ops", "//tensorflow/core:core_cpu_internal", @@ -1088,7 +1087,6 @@ tf_cuda_cc_test( "//tensorflow/cc:ops", "//tensorflow/cc:scope", "//tensorflow/compiler/jit", - "//tensorflow/compiler/jit:flags", "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:framework_internal", diff --git a/tensorflow/compiler/tf2xla/const_analysis_test.cc b/tensorflow/compiler/tf2xla/const_analysis_test.cc index c7c8702b49b..936b74f7b33 100644 --- a/tensorflow/compiler/tf2xla/const_analysis_test.cc +++ b/tensorflow/compiler/tf2xla/const_analysis_test.cc @@ -21,7 +21,6 @@ limitations under the License. #include "tensorflow/cc/ops/function_ops.h" #include "tensorflow/cc/ops/functional_ops.h" #include "tensorflow/cc/ops/standard_ops.h" -#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/jit/xla_cluster_util.h" #include "tensorflow/core/common_runtime/process_function_library_runtime.h" #include "tensorflow/core/graph/algorithm.h" @@ -218,10 +217,5 @@ TEST(ConstAnalysisTest, RespectExplicitAttr_1) { EXPECT_EQ(const_args, std::vector({true})); } -static bool Initialized = [] { - tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; - return true; -}(); - } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc b/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc index 02f178f9acf..1a26f974989 100644 --- a/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc +++ b/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc @@ -26,7 +26,6 @@ limitations under the License. #include "tensorflow/cc/ops/array_ops.h" #include "tensorflow/cc/ops/const_op.h" #include "tensorflow/cc/ops/nn_ops.h" -#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/framework/device_attributes.pb.h" #include "tensorflow/core/framework/graph.pb.h" #include "tensorflow/core/framework/tensor.h" @@ -140,11 +139,5 @@ TEST(FusedBatchnormReserveSpaceTest, Test) { test::ExpectClose(results[0], results[1], /*atol=*/1e-4); test::ExpectClose(results[2], results[3], /*atol=*/1e-4); } - -static bool Initialized = [] { - tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; - return true; -}(); - } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/xrt/BUILD b/tensorflow/compiler/xrt/BUILD index 172a970d207..6a704be4adb 100644 --- a/tensorflow/compiler/xrt/BUILD +++ b/tensorflow/compiler/xrt/BUILD @@ -96,7 +96,6 @@ tf_gen_op_libs( "xrt_execute_op", ], deps = [ - "//tensorflow/compiler/jit:flags", "//tensorflow/core:lib", ], ) diff --git a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc index 321d7409103..a4be39b96c6 100644 --- a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc +++ b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/framework/common_shape_fns.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/shape_inference.h" @@ -21,11 +20,6 @@ limitations under the License. namespace tensorflow { -static bool Initialized = [] { - tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; - return true; -}(); - REGISTER_OP("XRTAllocate") .Input("allocation: string") .Output("handle: int64") diff --git a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc index 4fe8e6a6c3f..641b5b4ef31 100644 --- a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc @@ -44,6 +44,26 @@ TEST_F(PinToHostOptimizerTest, TryFindHostDeviceCpuXlaGpu) { "/device:CPU:0"); } +TEST_F(PinToHostOptimizerTest, TryFindHostDeviceXlaCpuXlaGpu) { + gtl::FlatSet devices = {"/device:XLA_CPU:0", "/device:XLA_GPU:0"}; + + EXPECT_EQ(internal::TryFindHostDevice(devices, false, ""), ""); + EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:0"), + "/device:XLA_CPU:0"); + EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:*"), + "/device:XLA_CPU:0"); +} + +TEST_F(PinToHostOptimizerTest, TryFindHostDeviceXlaGpu) { + gtl::FlatSet devices = {"/device:XLA_GPU:0"}; + + EXPECT_EQ(internal::TryFindHostDevice(devices, false, ""), ""); + EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:0"), + ""); + EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:*"), + ""); +} + TEST_F(PinToHostOptimizerTest, OptimizeSmallOpsToHost) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); Output a = ops::Const(s.WithOpName("a"), 1, {1024, 1024}); diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 9c939fe0a76..765c77af7cd 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -42,7 +42,6 @@ from tensorflow.python.framework import device as pydev from tensorflow.python.util import compat from tensorflow.python.util import is_in_graph_mode from tensorflow.python.util import tf_contextlib -from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export GRAPH_MODE = 0 @@ -1255,7 +1254,12 @@ class Context(object): p: i for i, p in enumerate(self._physical_devices) } - self._visible_device_list = list(self._physical_devices) + # Construct the visible device list from all physical devices but ignore + # XLA devices + self._visible_device_list = [ + d for d in self._physical_devices + if not d.device_type.startswith("XLA") + ] self._memory_growth_map = { d: None for d in self._physical_devices if d.device_type == "GPU" } @@ -1489,12 +1493,6 @@ class Context(object): self._virtual_device_map[dev] = virtual_devices - @deprecated( - None, "XLA:CPU and XLA:GPU devices are deprecated", warn_once=True) - def enable_xla_devices(self): - """Enables XLA:CPU and XLA:GPU devices registration.""" - pywrap_tfe.TF_EnableXlaDevices() - @property def enable_mlir_bridge(self): return pywrap_tfe.TF_IsMlirBridgeEnabled() diff --git a/tensorflow/python/framework/config_test.py b/tensorflow/python/framework/config_test.py index ee7e111f6b0..70857ef4b83 100644 --- a/tensorflow/python/framework/config_test.py +++ b/tensorflow/python/framework/config_test.py @@ -435,6 +435,9 @@ class DeviceTest(test.TestCase): self.assertEqual(len(config.get_visible_devices('CPU')), 1) self.assertGreater(len(config.get_visible_devices('GPU')), 0) + # get_visible_devices filters out XLA_* devices. list_logical_devices does + # not, but we can't call it here because it initializes the devices and + # calling set_visible_devices after that is disallowed. self.assertEqual(len(config.get_visible_devices('XLA_GPU')), 0) config.set_visible_devices(cpus[0]) @@ -448,6 +451,12 @@ class DeviceTest(test.TestCase): a = array_ops.identity(1.0) self.evaluate(a) + with self.assertRaisesRegex(errors.InvalidArgumentError, + 'Could not satisfy'): + with ops.device('/device:XLA_GPU:0'): + a = array_ops.identity(1.0) + self.evaluate(a) + # Modifying the visible devices is not supported with self.assertRaisesRegex(RuntimeError, 'cannot be modified'): config.set_visible_devices(gpus) diff --git a/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py index 188df3f9b87..33f0d7b76ae 100644 --- a/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py +++ b/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py @@ -22,7 +22,6 @@ from __future__ import print_function from tensorflow.compiler.tf2xla.python import xla as xla_ops from tensorflow.python.compiler.xla import jit from tensorflow.python.compiler.xla import xla -from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import test_util @@ -40,10 +39,6 @@ from tensorflow.python.platform import test @test_util.run_all_in_graph_and_eager_modes class PForTest(PForTestCase): - def __init__(self, method_name="runTest"): - super(PForTest, self).__init__(method_name) - context.context().enable_xla_devices() - def test_xla_einsum(self): num_loop = 10 x_series = random_ops.random_uniform([num_loop, 9, 9]) diff --git a/tensorflow/python/tfe_wrapper.cc b/tensorflow/python/tfe_wrapper.cc index 0afd05e94cb..c66397036c0 100644 --- a/tensorflow/python/tfe_wrapper.cc +++ b/tensorflow/python/tfe_wrapper.cc @@ -444,9 +444,6 @@ PYBIND11_MODULE(_pywrap_tfe, m) { m.def("TF_EnableMlirBridge", [](bool enabled) { tensorflow::GetMlirCommonFlags()->tf_mlir_enable_mlir_bridge = enabled; }); - m.def("TF_EnableXlaDevices", [] { - tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; - }); // // TFE_Context Logic m.def( From 56548c2425ffb735cf85c4a147ccd71aadf79f4d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 00:34:59 -0700 Subject: [PATCH 0808/1017] Add std:: qualifications to all references to std::string and std::basic_string. PiperOrigin-RevId: 325970529 Change-Id: Id596be41afa3a19116d00197773b8a89dd98ebdc --- tensorflow/core/platform/file_system.h | 34 ++++++++++++++------------ 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h index 28d09c39db1..ca9f0fd5145 100644 --- a/tensorflow/core/platform/file_system.h +++ b/tensorflow/core/platform/file_system.h @@ -68,7 +68,7 @@ class FileSystem { /// The ownership of the returned RandomAccessFile is passed to the caller /// and the object should be deleted when is not used. virtual tensorflow::Status NewRandomAccessFile( - const string& fname, std::unique_ptr* result) { + const std::string& fname, std::unique_ptr* result) { return NewRandomAccessFile(fname, nullptr, result); }; @@ -93,7 +93,7 @@ class FileSystem { /// The ownership of the returned WritableFile is passed to the caller /// and the object should be deleted when is not used. virtual tensorflow::Status NewWritableFile( - const string& fname, std::unique_ptr* result) { + const std::string& fname, std::unique_ptr* result) { return NewWritableFile(fname, nullptr, result); }; @@ -115,7 +115,7 @@ class FileSystem { /// The ownership of the returned WritableFile is passed to the caller /// and the object should be deleted when is not used. virtual tensorflow::Status NewAppendableFile( - const string& fname, std::unique_ptr* result) { + const std::string& fname, std::unique_ptr* result) { return NewAppendableFile(fname, nullptr, result); }; @@ -136,7 +136,7 @@ class FileSystem { /// The ownership of the returned ReadOnlyMemoryRegion is passed to the caller /// and the object should be deleted when is not used. virtual tensorflow::Status NewReadOnlyMemoryRegionFromFile( - const string& fname, std::unique_ptr* result) { + const std::string& fname, std::unique_ptr* result) { return NewReadOnlyMemoryRegionFromFile(fname, nullptr, result); } @@ -147,7 +147,7 @@ class FileSystem { } /// Returns OK if the named path exists and NOT_FOUND otherwise. - virtual tensorflow::Status FileExists(const string& fname) { + virtual tensorflow::Status FileExists(const std::string& fname) { return FileExists(fname, nullptr); }; @@ -222,7 +222,8 @@ class FileSystem { virtual bool Match(const std::string& filename, const std::string& pattern); /// \brief Obtains statistics for the given path. - virtual tensorflow::Status Stat(const string& fname, FileStatistics* stat) { + virtual tensorflow::Status Stat(const std::string& fname, + FileStatistics* stat) { return Stat(fname, nullptr, stat); } @@ -233,7 +234,7 @@ class FileSystem { } /// \brief Deletes the named file. - virtual tensorflow::Status DeleteFile(const string& fname) { + virtual tensorflow::Status DeleteFile(const std::string& fname) { return DeleteFile(fname, nullptr); } @@ -247,7 +248,7 @@ class FileSystem { /// * OK - successfully created the directory. /// * ALREADY_EXISTS - directory with name dirname already exists. /// * PERMISSION_DENIED - dirname is not writable. - virtual tensorflow::Status CreateDir(const string& dirname) { + virtual tensorflow::Status CreateDir(const std::string& dirname) { return CreateDir(dirname, nullptr); } @@ -262,7 +263,7 @@ class FileSystem { /// * OK - successfully created the directory and sub directories, even if /// they were already created. /// * PERMISSION_DENIED - dirname or some subdirectory is not writable. - virtual tensorflow::Status RecursivelyCreateDir(const string& dirname) { + virtual tensorflow::Status RecursivelyCreateDir(const std::string& dirname) { return RecursivelyCreateDir(dirname, nullptr); } @@ -270,7 +271,7 @@ class FileSystem { TransactionToken* token); /// \brief Deletes the specified directory. - virtual tensorflow::Status DeleteDir(const string& dirname) { + virtual tensorflow::Status DeleteDir(const std::string& dirname) { return DeleteDir(dirname, nullptr); }; @@ -309,7 +310,7 @@ class FileSystem { return DeleteRecursively(dirname, nullptr, undeleted_files, undeleted_dirs); } - virtual tensorflow::Status DeleteRecursively(const string& dirname, + virtual tensorflow::Status DeleteRecursively(const std::string& dirname, TransactionToken* token, int64* undeleted_files, int64* undeleted_dirs); @@ -327,8 +328,8 @@ class FileSystem { } /// \brief Overwrites the target if it exists. - virtual tensorflow::Status RenameFile(const string& src, - const string& target) { + virtual tensorflow::Status RenameFile(const std::string& src, + const std::string& target) { return RenameFile(src, target, nullptr); } @@ -339,7 +340,8 @@ class FileSystem { } /// \brief Copy the src to target. - virtual tensorflow::Status CopyFile(const string& src, const string& target) { + virtual tensorflow::Status CopyFile(const std::string& src, + const std::string& target) { return CopyFile(src, target, nullptr); } @@ -365,7 +367,7 @@ class FileSystem { /// * NOT_FOUND - The path entry does not exist. /// * PERMISSION_DENIED - Insufficient permissions. /// * UNIMPLEMENTED - The file factory doesn't support directories. - virtual tensorflow::Status IsDirectory(const string& fname) { + virtual tensorflow::Status IsDirectory(const std::string& fname) { return IsDirectory(fname, nullptr); } @@ -631,7 +633,7 @@ class WrappedFileSystem : public FileSystem { return fs_->DeleteDir(dirname, (token ? token : token_)); } - tensorflow::Status DeleteRecursively(const string& dirname, + tensorflow::Status DeleteRecursively(const std::string& dirname, TransactionToken* token, int64* undeleted_files, int64* undeleted_dirs) override { From b4ab032959b2fb68022b2cf70de46e9564a50d6f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 01:40:26 -0700 Subject: [PATCH 0809/1017] Integrate LLVM at llvm/llvm-project@0de60b550b72 Updates LLVM usage to match [0de60b550b72](https://github.com/llvm/llvm-project/commit/0de60b550b72) PiperOrigin-RevId: 325977375 Change-Id: I0f24ce1139fd51196e24adcbc5e7b8027cf3bb53 --- tensorflow/workspace.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 73b3fb42a96..6493fe87836 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "54cb552b962097d0e3ef7306b69a3c82cc7fff37" - LLVM_SHA256 = "42a65541c62e8349cac068e7f44cb1d9d41addf0dbab0002cc7a7d7203bcb35b" + LLVM_COMMIT = "0de60b550b727fa3a0202a9ab5ca30520e291dd5" + LLVM_SHA256 = "d23a64cca502c32aa3990b5252f19cb2ad59534084384df6d7a355e6f23fac62" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), From 59bcb3ab288a303fbd7225d6b9622b6945b038e4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 02:01:47 -0700 Subject: [PATCH 0810/1017] Update GraphDef version to 490. PiperOrigin-RevId: 325979480 Change-Id: Id7dc33982c8f45b76f8b1d53a2d1f7f0a446ad8d --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index fa0df24a7e7..34b358612ad 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 489 // Updated: 2020/8/10 +#define TF_GRAPH_DEF_VERSION 490 // Updated: 2020/8/11 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From 7169ab7935248f4cd2ceb8cbe1b874849b07d346 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 02:01:48 -0700 Subject: [PATCH 0811/1017] compat: Update forward compatibility horizon to 2020-08-11 PiperOrigin-RevId: 325979481 Change-Id: If52dfd419928bf71c132207368e54f07a96348eb --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index 3d8f9c5490e..3e91a8b3f45 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 10) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 11) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From cb09f04f33362e40147643a151005a731798f77a Mon Sep 17 00:00:00 2001 From: Thai Nguyen Date: Tue, 11 Aug 2020 02:08:22 -0700 Subject: [PATCH 0812/1017] Add script to generate trimmed tflite aar files The script can be used as follows: bash tensorflow/lite/java/build_customized_aar_for_models.sh \ --customize_for_models=model1,model2 \ --target_archs=x86,x86_64,arm64-v8a,armeabi-v7a it will generate the tensorflow-lite.aar and tensorflow-lite-select-tf-ops.aar if needed. PiperOrigin-RevId: 325980230 Change-Id: I9c5216108ae87ac22a69cbb88b37bb473bd4d37f --- tensorflow/lite/g3doc/guide/android.md | 15 +- tensorflow/lite/tools/BUILD | 21 ++ tensorflow/lite/tools/build_aar.sh | 214 ++++++++++++++++++ .../lite/tools/build_aar_with_docker.sh | 115 ++++++++++ tensorflow/lite/tools/list_flex_ops.h | 2 +- .../lite/tools/list_flex_ops_no_kernel.cc | 61 +++++ 6 files changed, 425 insertions(+), 3 deletions(-) create mode 100755 tensorflow/lite/tools/build_aar.sh create mode 100755 tensorflow/lite/tools/build_aar_with_docker.sh create mode 100644 tensorflow/lite/tools/list_flex_ops_no_kernel.cc diff --git a/tensorflow/lite/g3doc/guide/android.md b/tensorflow/lite/g3doc/guide/android.md index a1493090588..72eb07aa34b 100644 --- a/tensorflow/lite/g3doc/guide/android.md +++ b/tensorflow/lite/g3doc/guide/android.md @@ -205,8 +205,19 @@ bazel build -c opt --fat_apk_cpu=x86,x86_64,arm64-v8a,armeabi-v7a \ This will generate an AAR file in `bazel-bin/tensorflow/lite/java/`. Note that this builds a "fat" AAR with several different architectures; if you don't need all of them, use the subset appropriate for your deployment environment. -From there, there are several approaches you can take to use the .aar in your -Android Studio project. + +Caution: Following feature is experimental and only available at HEAD. You can +build smaller AAR files targeting only a set of models as follows: + +```sh +bash tensorflow/lite/tools/build_aar.sh \ + --input_models=model1,model2 \ + --target_archs=x86,x86_64,arm64-v8a,armeabi-v7a +``` + +Above script will generate the `tensorflow-lite.aar` file and optionally the +`tensorflow-lite-select-tf-ops.aar` file if one of the models is using +Tensorflow ops. ##### Add AAR directly to project diff --git a/tensorflow/lite/tools/BUILD b/tensorflow/lite/tools/BUILD index 1f57cad7f7a..ad19cd2b519 100644 --- a/tensorflow/lite/tools/BUILD +++ b/tensorflow/lite/tools/BUILD @@ -296,6 +296,27 @@ cc_library( ], ) +cc_library( + name = "list_flex_ops_no_kernel", + srcs = ["list_flex_ops_no_kernel.cc"], + hdrs = ["list_flex_ops.h"], + deps = [ + "//tensorflow/lite:framework", + "@com_google_absl//absl/strings", + ], +) + +tf_cc_binary( + name = "list_flex_ops_no_kernel_main", + srcs = ["list_flex_ops_main.cc"], + visibility = ["//visibility:public"], + deps = [ + ":list_flex_ops_no_kernel", + "//tensorflow/lite/tools:command_line_flags", + "@com_google_absl//absl/strings", + ], +) + tf_cc_test( name = "list_flex_ops_test", srcs = ["list_flex_ops_test.cc"], diff --git a/tensorflow/lite/tools/build_aar.sh b/tensorflow/lite/tools/build_aar.sh new file mode 100755 index 00000000000..6d84d5b35b1 --- /dev/null +++ b/tensorflow/lite/tools/build_aar.sh @@ -0,0 +1,214 @@ +#!/bin/bash +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/../../../" && pwd)" + +function print_usage { + echo "Usage:" + echo " $(basename ${BASH_SOURCE}) \\" + echo " --input_models=model1.tflite,model2.tflite \\" + echo " --target_archs=x86,x86_64,arm64-v8a,armeabi-v7a \\" + echo " --tflite_custom_ops_srcs=file1.cc,file2.h \\" + echo " --tflite_custom_ops_deps=dep1,dep2" + echo "" + echo "Where: " + echo " --input_models: Supported TFLite models. " + echo " --target_archs: Supported arches included in the aar file." + echo " --tflite_custom_ops_srcs: The src files for building additional TFLite custom ops if any." + echo " --tflite_custom_ops_deps: Dependencies for building additional TFLite custom ops if any." + echo "" + exit 1 +} + +function generate_list_field { + local name="$1" + local list_string="$2" + local list=(${list_string//,/ }) + + local message+=("$name=[") + for item in "${list[@]}" + do + message+=("\"$item\",") + done + message+=('],') + printf '%s' "${message[@]}" +} + +function print_output { + echo "Output can be found here:" + for i in "$@" + do + # Check if the file exist. + ls -1a ${ROOT_DIR}/$i + done +} + +function generate_tflite_aar { + pushd ${TMP_DIR} > /dev/null + # Generate the BUILD file. + message=( + 'load("//tensorflow/lite:build_def.bzl", "tflite_custom_android_library")' + 'load("//tensorflow/lite/java:aar_with_jni.bzl", "aar_with_jni")' + '' + 'tflite_custom_android_library(' + ' name = "custom_tensorflowlite",' + ) + message+=(' '$(generate_list_field "models" $MODEL_NAMES)) + message+=(' '$(generate_list_field "srcs" $TFLITE_OPS_SRCS)) + message+=(' '$(generate_list_field "deps" $FLAG_TFLITE_OPS_DEPS)) + message+=( + ')' + '' + 'aar_with_jni(' + ' name = "tensorflow-lite",' + ' android_library = ":custom_tensorflowlite",' + ')' + '' + ) + printf '%s\n' "${message[@]}" >> BUILD + + # Build the aar package. + popd > /dev/null + bazel build -c opt --cxxopt='--std=c++14' \ + --fat_apk_cpu=${TARGET_ARCHS} \ + --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ + //tmp:tensorflow-lite + + OUT_FILES="${OUT_FILES} bazel-bin/tmp/tensorflow-lite.aar" +} + +function generate_flex_aar { + pushd ${TMP_DIR} + # Generating the BUILD file. + message=( + 'load("//tensorflow/lite/delegates/flex:build_def.bzl", "tflite_flex_android_library")' + 'load("//tensorflow/lite/java:aar_with_jni.bzl", "aar_with_jni")' + '' + 'tflite_flex_android_library(' + ' name = "custom_tensorflowlite_flex",' + ) + message+=(' '$(generate_list_field "models" $MODEL_NAMES)) + message+=( + ')' + '' + 'aar_with_jni(' + ' name = "tensorflow-lite-select-tf-ops",' + ' android_library = ":custom_tensorflowlite_flex",' + ')' + ) + printf '%s\n' "${message[@]}" >> BUILD + + cp ${ROOT_DIR}/tensorflow/lite/java/AndroidManifest.xml . + cp ${ROOT_DIR}/tensorflow/lite/java/proguard.flags . + popd + + # Build the aar package. + bazel build -c opt --cxxopt='--std=c++14' \ + --fat_apk_cpu=${TARGET_ARCHS} \ + --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ + //tmp:tensorflow-lite-select-tf-ops + + OUT_FILES="${OUT_FILES} bazel-bin/tmp/tensorflow-lite-select-tf-ops.aar" +} + +# Check command line flags. +TARGET_ARCHS=x86,x86_64,arm64-v8a,armeabi-v7a + +if [ "$#" -gt 4 ]; then + echo "ERROR: Too many arguments." + print_usage +fi + +for i in "$@" +do +case $i in + --input_models=*) + FLAG_MODELS="${i#*=}" + shift;; + --target_archs=*) + TARGET_ARCHS="${i#*=}" + shift;; + --tflite_custom_ops_srcs=*) + FLAG_TFLITE_OPS_SRCS="${i#*=}" + shift;; + --tflite_custom_ops_deps=*) + FLAG_TFLITE_OPS_DEPS="${i#*=}" + shift;; + *) + echo "ERROR: Unrecognized argument: ${i}" + print_usage;; +esac +done + +# Check if users already run configure +cd $ROOT_DIR +if [ ! -f "$ROOT_DIR/.tf_configure.bazelrc" ]; then + echo "ERROR: Please run ./configure first." + exit 1 +else + if ! grep -q ANDROID_SDK_HOME "$ROOT_DIR/.tf_configure.bazelrc"; then + echo "ERROR: Please run ./configure with Android config." + exit 1 + fi +fi + +# Build the standard aar package of no models provided. +if [ -z ${FLAG_MODELS} ]; then + bazel build -c opt --cxxopt='--std=c++14' \ + --fat_apk_cpu=${TARGET_ARCHS} \ + --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \ + //tensorflow/lite/java:tensorflow-lite + + print_output bazel-bin/tensorflow/lite/java/tensorflow-lite.aar + exit 0 +fi + +# Prepare the tmp directory. +TMP_DIR="${ROOT_DIR}/tmp/" +rm -rf ${TMP_DIR} && mkdir -p ${TMP_DIR} + +# Copy models to tmp directory. +MODEL_NAMES="" +for model in $(echo ${FLAG_MODELS} | sed "s/,/ /g") +do + cp ${model} ${TMP_DIR} + MODEL_NAMES="${MODEL_NAMES},$(basename ${model})" +done + +# Copy srcs of additional tflite ops to tmp directory. +TFLITE_OPS_SRCS="" +for src_file in $(echo ${FLAG_TFLITE_OPS_SRCS} | sed "s/,/ /g") +do + cp ${src_file} ${TMP_DIR} + TFLITE_OPS_SRCS="${TFLITE_OPS_SRCS},$(basename ${src_file})" +done + +# Build the custom aar package. +generate_tflite_aar + +# Build flex aar if one of the models contain flex ops. +bazel build -c opt --config=monolithic //tensorflow/lite/tools:list_flex_ops_no_kernel_main +bazel-bin/tensorflow/lite/tools/list_flex_ops_no_kernel_main --graphs=${FLAG_MODELS} > ${TMP_DIR}/ops_list.txt +if [[ `cat ${TMP_DIR}/ops_list.txt` != "[]" ]]; then + generate_flex_aar +fi + +# List the output files. +rm -rf ${TMP_DIR} +print_output ${OUT_FILES} diff --git a/tensorflow/lite/tools/build_aar_with_docker.sh b/tensorflow/lite/tools/build_aar_with_docker.sh new file mode 100755 index 00000000000..2af4787c35c --- /dev/null +++ b/tensorflow/lite/tools/build_aar_with_docker.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +function print_usage { + echo "Usage:" + echo " $(basename ${BASH_SOURCE}) \\" + echo " --input_models=model1.tflite,model2.tflite \\" + echo " --target_archs=x86,x86_64,arm64-v8a,armeabi-v7a \\" + echo " --checkpoint=master" + echo "" + echo "Where: " + echo " --input_models: Supported TFLite models. " + echo " --target_archs: Supported arches included in the aar file." + echo " --checkpoint: Checkpoint of the github repo, could be a branch, a commit or a tag. Default: master" + echo "" + exit 1 +} + +# Check command line flags. +ARGUMENTS=$@ +TARGET_ARCHS=x86,x86_64,arm64-v8a,armeabi-v7a +FLAG_CHECKPOINT="master" + +if [ "$#" -gt 3 ]; then + echo "ERROR: Too many arguments." + print_usage +fi + +for i in "$@" +do +case $i in + --input_models=*) + FLAG_MODELS="${i#*=}" + shift;; + --target_archs=*) + TARGET_ARCHS="${i#*=}" + shift;; + --checkpoint=*) + FLAG_CHECKPOINT="${i#*=}" + shift;; + *) + echo "ERROR: Unrecognized argument: ${i}" + print_usage;; +esac +done + +if [ ! -d /tensorflow_src ]; then + # Running on host. + for model in $(echo ${FLAG_MODELS} | sed "s/,/ /g") + do + FLAG_DIR="${FLAG_DIR} -v ${model}:${model}" + done + docker run --rm -it -v $PWD:/tmp -v ${SCRIPT_DIR}:/script_dir ${FLAG_DIR} \ + --entrypoint /script_dir/build_aar_with_docker.sh tflite-builder \ + ${ARGUMENTS} + exit 0 +else + # Running inside docker container, download the SDK first. + android update sdk --no-ui -a \ + --filter tools,platform-tools,android-${ANDROID_API_LEVEL},build-tools-${ANDROID_BUILD_TOOLS_VERSION} + + cd /tensorflow_src + + # Run configure. + configs=( + '/usr/bin/python3' + '/usr/lib/python3/dist-packages' + 'N' + 'N' + 'N' + 'N' + '-march=native -Wno-sign-compare' + 'y' + '/android/sdk' + ) + printf '%s\n' "${configs[@]}" | ./configure + + # Pull the latest code from tensorflow. + git pull -a + git checkout ${FLAG_CHECKPOINT} + + # Building with bazel. + bash /tensorflow_src/tensorflow/lite/tools/build_aar.sh ${ARGUMENTS} + + # Copy the output files from docker container. + clear + OUT_FILES="/tensorflow_src/bazel-bin/tmp/tensorflow-lite.aar" + OUT_FILES="${OUT_FILES} /tensorflow_src/bazel-bin/tmp/tensorflow-lite-select-tf-ops.aar" + echo "Output can be found here:" + for i in ${OUT_FILES} + do + if [ -f $i ]; then + cp $i /tmp + basename $i + fi + done +fi + diff --git a/tensorflow/lite/tools/list_flex_ops.h b/tensorflow/lite/tools/list_flex_ops.h index 070da2d9b3d..f9bc7b952df 100644 --- a/tensorflow/lite/tools/list_flex_ops.h +++ b/tensorflow/lite/tools/list_flex_ops.h @@ -42,7 +42,7 @@ struct OpKernelCompare { using OpKernelSet = std::set; // Find flex ops and its kernel classes inside a TFLite model and add them to -// the map flex_ops. The map stores +// the map flex_ops. void AddFlexOpsFromModel(const tflite::Model* model, OpKernelSet* flex_ops); // Serialize the list op of to a json string. If flex_ops is empty, return an diff --git a/tensorflow/lite/tools/list_flex_ops_no_kernel.cc b/tensorflow/lite/tools/list_flex_ops_no_kernel.cc new file mode 100644 index 00000000000..11a9f39dbfd --- /dev/null +++ b/tensorflow/lite/tools/list_flex_ops_no_kernel.cc @@ -0,0 +1,61 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "tensorflow/lite/tools/list_flex_ops.h" + +namespace tflite { +namespace flex { + +std::string OpListToJSONString(const OpKernelSet& flex_ops) { + return absl::StrCat("[", + absl::StrJoin(flex_ops, ",\n", + [](std::string* out, const OpKernel& op) { + absl::StrAppend(out, "\"", op.op_name, + "\""); + }), + "]"); +} + +void AddFlexOpsFromModel(const tflite::Model* model, OpKernelSet* flex_ops) { + auto* subgraphs = model->subgraphs(); + if (!subgraphs) return; + + for (int subgraph_index = 0; subgraph_index < subgraphs->size(); + ++subgraph_index) { + const tflite::SubGraph* subgraph = subgraphs->Get(subgraph_index); + auto* operators = subgraph->operators(); + auto* opcodes = model->operator_codes(); + if (!operators || !opcodes) continue; + + for (int i = 0; i < operators->size(); ++i) { + const tflite::Operator* op = operators->Get(i); + const tflite::OperatorCode* opcode = opcodes->Get(op->opcode_index()); + if (opcode->builtin_code() != tflite::BuiltinOperator_CUSTOM || + !tflite::IsFlexOp(opcode->custom_code()->c_str())) { + continue; + } + + // Remove the "Flex" prefix from op name. + std::string flex_op_name(opcode->custom_code()->c_str()); + std::string tf_op_name = + flex_op_name.substr(strlen(tflite::kFlexCustomCodePrefix)); + + flex_ops->insert({tf_op_name, ""}); + } + } +} +} // namespace flex +} // namespace tflite From f4fe3510740131d0a0ab08f8f4349387c1804223 Mon Sep 17 00:00:00 2001 From: Myung-Hyun Kim Date: Tue, 11 Aug 2020 18:27:42 +0900 Subject: [PATCH 0813/1017] Update configure.cmd Resolved an error that can not run configure.py if the env %configure_dir% contains spaces. --- configure.cmd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.cmd b/configure.cmd index 021afdbbea1..738e106da18 100644 --- a/configure.cmd +++ b/configure.cmd @@ -16,5 +16,5 @@ set configure_dir=%~dp0 set configure_dir=%configure_dir:~0,-1% -python %configure_dir%\configure.py %* || ( exit /b ) +python "%configure_dir%\configure.py" %* || ( exit /b ) echo Configuration finished From 78a451a5bc954891f6db9abc2a68508c21781e4a Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Tue, 11 Aug 2020 02:47:42 -0700 Subject: [PATCH 0814/1017] Add a warning handler for StatusScopedDiagnosticHandler. PiperOrigin-RevId: 325984749 Change-Id: I3a6b2a7dd043148b8e2dc5cdf2a2929341e699bc --- .../compiler/mlir/lite/tf_to_tfl_flatbuffer.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc index 414a0de0118..c158f3a8e21 100644 --- a/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc +++ b/tensorflow/compiler/mlir/lite/tf_to_tfl_flatbuffer.cc @@ -129,6 +129,18 @@ Status ConvertTFExecutorToTFLOrFlatbuffer( bool emit_select_tf_ops, bool emit_custom_ops, const mlir::TFL::QuantizationSpecs& quant_specs, std::string* result, mlir::PassManager* pass_manager) { + // Register a warning handler only log to std out. + mlir::ScopedDiagnosticHandler s( + module.getContext(), [](mlir::Diagnostic& diag) { + if (diag.getSeverity() == mlir::DiagnosticSeverity::Warning) { + for (auto& note : diag.getNotes()) { + std::cout << note.str() << "\n"; + LOG(WARNING) << note.str() << "\n"; + } + } + return mlir::failure(); + }); + mlir::StatusScopedDiagnosticHandler statusHandler(module.getContext(), /*propagate=*/true); From af5df07df41a2a9dcb9c73b594b5e7ffc853e88a Mon Sep 17 00:00:00 2001 From: Alexander Belyaev Date: Tue, 11 Aug 2020 04:00:17 -0700 Subject: [PATCH 0815/1017] [MLIR:GPU] Expose aux passes in kernel_lowering.cc. Also clean up dependencies in kernel_lowering. PiperOrigin-RevId: 325992384 Change-Id: I420b4ef7c8058bc573cc2b525961283df4927f99 --- .../compiler/xla/service/mlir_gpu/BUILD | 24 +- .../xla/service/mlir_gpu/kernel_lowering.cc | 412 +---------------- .../compiler/xla/service/mlir_gpu/passes.cc | 423 ++++++++++++++++++ .../compiler/xla/service/mlir_gpu/passes.h | 66 +++ 4 files changed, 519 insertions(+), 406 deletions(-) create mode 100644 tensorflow/compiler/xla/service/mlir_gpu/passes.cc create mode 100644 tensorflow/compiler/xla/service/mlir_gpu/passes.h diff --git a/tensorflow/compiler/xla/service/mlir_gpu/BUILD b/tensorflow/compiler/xla/service/mlir_gpu/BUILD index 43a6efe9e90..31cf36dee85 100644 --- a/tensorflow/compiler/xla/service/mlir_gpu/BUILD +++ b/tensorflow/compiler/xla/service/mlir_gpu/BUILD @@ -155,11 +155,31 @@ cc_library( ], ) +cc_library( + name = "passes", + srcs = ["passes.cc"], + hdrs = ["passes.h"], + deps = [ + "//tensorflow/compiler/mlir/hlo:lhlo", + "@com_google_absl//absl/memory", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:GPUDialect", + "@llvm-project//mlir:GPUTransforms", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:Pass", + "@llvm-project//mlir:SCFDialect", + "@llvm-project//mlir:SCFTransforms", + "@llvm-project//mlir:StandardOps", + "@llvm-project//mlir:Transforms", + ], +) + cc_library( name = "kernel_lowering", srcs = ["kernel_lowering.cc"], hdrs = ["kernel_lowering.h"], deps = [ + ":passes", "//tensorflow/compiler/mlir/hlo", "//tensorflow/compiler/mlir/hlo:hlo_dialect_force_registration", "//tensorflow/compiler/mlir/hlo:hlo_legalize_to_lhlo", @@ -173,9 +193,7 @@ cc_library( "//tensorflow/compiler/xla:status", "//tensorflow/compiler/xla:statusor", "//tensorflow/compiler/xla:util", - "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/memory", - "@llvm-project//mlir:Affine", "@llvm-project//mlir:AffineToStandardTransforms", "@llvm-project//mlir:CFGTransforms", "@llvm-project//mlir:GPUDialect", @@ -184,7 +202,6 @@ cc_library( "@llvm-project//mlir:IR", "@llvm-project//mlir:LLVMDialect", "@llvm-project//mlir:LLVMTransforms", - "@llvm-project//mlir:LinalgOps", "@llvm-project//mlir:LinalgToLLVM", "@llvm-project//mlir:LinalgTransforms", "@llvm-project//mlir:NVVMDialect", @@ -193,7 +210,6 @@ cc_library( "@llvm-project//mlir:SCFToGPUPass", "@llvm-project//mlir:SCFTransforms", "@llvm-project//mlir:StandardOps", - "@llvm-project//mlir:Support", "@llvm-project//mlir:Transforms", ], ) diff --git a/tensorflow/compiler/xla/service/mlir_gpu/kernel_lowering.cc b/tensorflow/compiler/xla/service/mlir_gpu/kernel_lowering.cc index 2e3fa00ca86..ae99cc9ba63 100644 --- a/tensorflow/compiler/xla/service/mlir_gpu/kernel_lowering.cc +++ b/tensorflow/compiler/xla/service/mlir_gpu/kernel_lowering.cc @@ -22,419 +22,26 @@ limitations under the License. #include "mlir/Conversion/SCFToGPU/SCFToGPUPass.h" // from @llvm-project #include "mlir/Conversion/SCFToStandard/SCFToStandard.h" // from @llvm-project #include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVM.h" // from @llvm-project -#include "mlir/Conversion/StandardToLLVM/ConvertStandardToLLVMPass.h" // from @llvm-project -#include "mlir/Dialect/Affine/IR/AffineOps.h" // from @llvm-project #include "mlir/Dialect/GPU/GPUDialect.h" // from @llvm-project -#include "mlir/Dialect/GPU/ParallelLoopMapper.h" // from @llvm-project #include "mlir/Dialect/GPU/Passes.h" // from @llvm-project #include "mlir/Dialect/LLVMIR/LLVMDialect.h" // from @llvm-project #include "mlir/Dialect/LLVMIR/NVVMDialect.h" // from @llvm-project -#include "mlir/Dialect/Linalg/IR/LinalgOps.h" // from @llvm-project #include "mlir/Dialect/Linalg/Passes.h" // from @llvm-project #include "mlir/Dialect/SCF/Passes.h" // from @llvm-project -#include "mlir/Dialect/SCF/SCF.h" // from @llvm-project #include "mlir/Dialect/SCF/Transforms.h" // from @llvm-project -#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project -#include "mlir/IR/Attributes.h" // from @llvm-project -#include "mlir/IR/BlockAndValueMapping.h" // from @llvm-project #include "mlir/IR/Builders.h" // from @llvm-project -#include "mlir/IR/Function.h" // from @llvm-project -#include "mlir/IR/Module.h" // from @llvm-project -#include "mlir/IR/OperationSupport.h" // from @llvm-project -#include "mlir/IR/PatternMatch.h" // from @llvm-project -#include "mlir/IR/Region.h" // from @llvm-project -#include "mlir/IR/Value.h" // from @llvm-project #include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Pass/PassManager.h" // from @llvm-project #include "mlir/Transforms/BufferPlacement.h" // from @llvm-project -#include "mlir/Transforms/DialectConversion.h" // from @llvm-project -#include "mlir/Transforms/LoopUtils.h" // from @llvm-project #include "mlir/Transforms/Passes.h" // from @llvm-project -#include "mlir/Transforms/RegionUtils.h" // from @llvm-project #include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" #include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/passes.h" #include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/transforms/rewriters.h" +#include "tensorflow/compiler/xla/service/mlir_gpu/passes.h" #include "tensorflow/compiler/xla/util.h" namespace xla { namespace mlir_gpu { -namespace { - -using ::mlir::lmhlo::FusionOp; - -// Replaces a FusionOp by the operations contained in its region. -struct FusionOpRemover - : public mlir::PassWrapper { - void runOnFunction() override { - getFunction().walk([&](FusionOp op) { - mlir::OpBuilder builder(op); - // FusionOp has a single region with a single block, so we can just walk - // over it and clone operations to the outside. - mlir::BlockAndValueMapping mapping; - for (auto& nested_op : op.region().front().without_terminator()) { - auto clone = builder.clone(nested_op, mapping); - for (auto pair : - llvm::zip(nested_op.getResults(), clone->getResults())) { - mapping.map(std::get<0>(pair), std::get<1>(pair)); - } - } - op.erase(); - }); - } -}; - -// Simple pass that replaces a load that immediately follows a store to the -// same address with the stored value. This needs generalization. -struct StoreForwardingPass - : mlir::PassWrapper { - mlir::StoreOp findStore(mlir::Operation* op, - std::function matches) { - // Search from op upwards in the current block. - mlir::Block* block = op->getBlock(); - auto startFromIt = - std::find_if(block->rbegin(), block->rend(), - [op](mlir::Operation& other) { return &other == op; }); - for (auto storeOpIt = startFromIt; storeOpIt != block->rend(); - ++storeOpIt) { - auto storeOp = llvm::dyn_cast(&*(storeOpIt)); - if (!storeOp || !matches(storeOp)) { - continue; - } - - return storeOp; - } - // No store operation found. Continue search outside of the parallel - // loop if block is in a parallel loop. - if (auto parallelOp = - llvm::dyn_cast(block->getParentOp())) { - return findStore(parallelOp.getOperation(), matches); - } - return {}; - } - - // Recursively search defining ops for AllocOp. Return either AllocOp if it is - // found or nullptr. - mlir::Operation* SearchAllocOp(mlir::Value memref) { - mlir::Operation* defOp = memref.getDefiningOp(); - while (auto subviewOp = mlir::dyn_cast_or_null(defOp)) { - defOp = subviewOp.source().getDefiningOp(); - } - if (auto allocOp = mlir::dyn_cast_or_null(defOp)) { - return allocOp.getOperation(); - } - return nullptr; - } - - // Retrieves AllocOp from the cache or actually looks for it. - mlir::Operation* GetAllocOp( - mlir::Value memref, - llvm::DenseMap* memrefToAllocOp) { - auto allocOpIt = memrefToAllocOp->find(memref); - if (allocOpIt != memrefToAllocOp->end()) { - return allocOpIt->second; - } - auto allocOp = SearchAllocOp(memref); - memrefToAllocOp->insert({memref, allocOp}); - return allocOp; - } - - void runOnFunction() override { - llvm::DenseMap memrefToAllocOp; - - getFunction().walk([&](mlir::LoadOp loadOp) { - auto storeOp = findStore(loadOp, [&](mlir::StoreOp storeOp) { - mlir::Operation* storeOpAlloc = - GetAllocOp(storeOp.memref(), &memrefToAllocOp); - mlir::Operation* loadOpAlloc = - GetAllocOp(loadOp.memref(), &memrefToAllocOp); - return storeOpAlloc && loadOpAlloc && (storeOpAlloc == loadOpAlloc); - }); - if (!storeOp) { - return; - } - auto storeIndices = storeOp.getIndices(); - auto loadIndices = loadOp.getIndices(); - if (!std::equal(storeIndices.begin(), storeIndices.end(), - loadIndices.begin(), loadIndices.end())) { - return; - } - loadOp.replaceAllUsesWith(storeOp.getValueToStore()); - loadOp.erase(); - }); - } -}; - -// Simple pass that removes temporary buffers that are only written to but -// never read from or that are read but the read value is not used. -// Needs an analysis that proves that loads and stores are side-effect free -// (in bounds, no aliasing, etc.). -struct DeadTempBufferRemoval - : mlir::PassWrapper { - bool operationConsideredDead(mlir::Operation* op) { - for (auto result : op->getResults()) { - if (!llvm::all_of(result.getUsers(), [&](mlir::Operation* op) { - // Store and Dealloc is OK. - if (llvm::isa(op)) { - return true; - } - // Load without uses is also ok. - if (auto loadOp = llvm::dyn_cast(op)) { - return loadOp.use_empty(); - } - // Subview is ok if it is dead itself. - if (llvm::isa(op)) { - return operationConsideredDead(op); - } - return false; - })) { - return false; - } - } - return true; - } - - void recursiveErase(mlir::Operation* op, - llvm::SmallVectorImpl* erase_list) { - for (auto result : op->getResults()) { - for (auto user : llvm::make_early_inc_range(result.getUsers())) { - recursiveErase(user, erase_list); - } - } - erase_list->push_back(op); - } - - void runOnFunction() override { - llvm::SmallVector dead_ops; - getFunction().walk([&](mlir::AllocOp allocOp) { - if (!operationConsideredDead(allocOp)) { - return; - } - - // TODO(herhut): There should be a generic helper for this. - recursiveErase(allocOp, &dead_ops); - }); - for (auto op : dead_ops) { - op->erase(); - } - } -}; - -// TODO(herhut): Move this to MLIR core. -struct MoveScalarComputationsIntoGpuLaunch - : mlir::PassWrapper { - static bool isInliningBeneficiary(mlir::Operation* op) { - return llvm::isa(op); - } - - static bool extractBeneficiaryOps( - mlir::Operation* op, llvm::SmallVectorImpl* ops, - llvm::SetVector args) { - if (!isInliningBeneficiary(op)) { - return false; - } - - ops->push_back(op); - for (auto operand : op->getOperands()) { - // It is an existing arg, keep going. - if (args.count(operand)) { - continue; - } - mlir::Operation* definingOp = operand.getDefiningOp(); - if (!definingOp || !extractBeneficiaryOps(definingOp, ops, args)) { - return false; - } - } - return true; - } - - static void inlineOperationsIntoLaunch(mlir::gpu::LaunchOp launch) { - llvm::SetVector used_above; - mlir::getUsedValuesDefinedAbove(launch.body(), used_above); - mlir::BlockAndValueMapping inlined_map; - for (mlir::Value v : used_above) { - llvm::SmallVector ops_to_move; - mlir::Operation* definingOp = v.getDefiningOp(); - if (definingOp && - extractBeneficiaryOps(definingOp, &ops_to_move, used_above)) { - mlir::OpBuilder b(launch.body()); - for (mlir::Operation* op : llvm::reverse(ops_to_move)) { - auto result = b.clone(*op, inlined_map); - for (auto pair : llvm::zip(op->getResults(), result->getResults())) { - mlir::replaceAllUsesInRegionWith(std::get<0>(pair), - std::get<1>(pair), launch.body()); - } - inlined_map.map(op->getResults(), result->getResults()); - } - } - } - } - - void runOnFunction() override { - mlir::FuncOp fun = getFunction(); - fun.walk( - [](mlir::gpu::LaunchOp launch) { inlineOperationsIntoLaunch(launch); }); - } -}; - -// Sort the operands to the kernel for a deterministic order. First operands -// that are defined by function arguments, followed by operands that are -// returned from the function. This only works for simple functions without -// control flow and can be used in cases where the kernel is extracted and used -// independently of the host-side code. -struct RewriteKernelSignature - : mlir::PassWrapper { - void runOnFunction() override { - mlir::FuncOp func = getFunction(); - mlir::ModuleOp module = func.getParentOfType(); - getFunction().walk([&](mlir::gpu::LaunchFuncOp launchOp) { - mlir::gpu::GPUFuncOp kernel = - module.lookupSymbol(launchOp.kernel()); - - if (kernel.getNumFuncArguments() != - func.getNumArguments() + func.getNumResults()) { - kernel.emitError() - << "number of kernel arguments does not match number" - << "of arguments and results of surrounding function"; - signalPassFailure(); - return; - } - if (!llvm::hasSingleElement(func)) { - func.emitError() << "surrounding function has more than one block"; - signalPassFailure(); - return; - } - - // Compute a map from function arguments to kernel function operands. - mlir::BlockAndValueMapping func_to_kernel; - for (mlir::BlockArgument arg : func.getArguments()) { - for (int i = 0, e = launchOp.getNumKernelOperands(); i < e; ++i) { - if (launchOp.getKernelOperand(i) == arg) { - func_to_kernel.map(arg, kernel.getArgument(i)); - break; - } - } - } - // Also add function results that are computed by the launch. - mlir::Operation* returnOp = func.getBody().back().getTerminator(); - for (mlir::Value result : returnOp->getOperands()) { - for (int i = 0, e = launchOp.getNumKernelOperands(); i < e; ++i) { - if (launchOp.getKernelOperand(i) == result) { - func_to_kernel.map(result, kernel.getArgument(i)); - break; - } - } - } - - // Create a new kernel function with modified signature. It will have the - // parameters and result types of the original funcion as its parameter - // type and otherwise will be void. - auto gpu_module = kernel.getParentOfType(); - mlir::OpBuilder kernel_builder(gpu_module.body()); - auto operand_types = llvm::to_vector<4>(llvm::concat( - func.getType().getInputs(), func.getType().getResults())); - auto new_kernel = kernel_builder.create( - kernel.getLoc(), kernel.getName(), - kernel_builder.getFunctionType(operand_types, {})); - new_kernel.setAttr(mlir::gpu::GPUDialect::getKernelFuncAttrName(), - kernel_builder.getUnitAttr()); - - // Create a map from old kernel argument to new one. - mlir::BlockAndValueMapping old_kernel_to_new; - for (int i = 0, e = func.getNumArguments(); i < e; ++i) { - mlir::Value func_arg = func.getArgument(i); - mlir::Value new_kernel_arg = new_kernel.getArgument(i); - mlir::Value old_kernel_arg = func_to_kernel.lookupOrNull(func_arg); - if (!old_kernel_arg) { - kernel.emitOpError() - << "argument " << i - << " to containing function is not an argument to the kernel"; - signalPassFailure(); - return; - } - old_kernel_to_new.map(old_kernel_arg, new_kernel_arg); - } - for (int i = 0, e = returnOp->getNumOperands(); i < e; ++i) { - mlir::Value ret_op = returnOp->getOperand(i); - mlir::Value new_kernel_arg = - new_kernel.getArgument(func.getNumArguments() + i); - mlir::Value old_kernel_arg = func_to_kernel.lookupOrNull(ret_op); - if (!old_kernel_arg) { - kernel.emitOpError() - << "result " << i - << " of containing function is not an argument to the kernel"; - signalPassFailure(); - return; - } - old_kernel_to_new.map(old_kernel_arg, new_kernel_arg); - } - // Steal the body by appending the blocks and inserting a branch. - kernel.body().cloneInto(&new_kernel.getBody(), old_kernel_to_new); - kernel_builder.setInsertionPointToEnd(&new_kernel.body().front()); - kernel_builder.create( - new_kernel.getLoc(), &*std::next(new_kernel.body().begin())); - // Now create a new launchOp calling the new kernel. We need to forward - // the arguments of the surrounding function and operands to the return. - mlir::SmallVector new_operands; - new_operands.reserve(new_kernel.getNumFuncArguments()); - new_operands.append(func.args_begin(), func.args_end()); - new_operands.append(returnOp->operand_begin(), returnOp->operand_end()); - mlir::OpBuilder launch_builder(launchOp); - launch_builder.create( - launchOp.getLoc(), new_kernel, launchOp.getGridSizeOperandValues(), - launchOp.getBlockSizeOperandValues(), new_operands); - // Launch does not have results, so we can just erase it. And the kernel - // also needs to go. - launchOp.erase(); - kernel.erase(); - }); - } -}; - -// Extract_element(mhlo_scalars_to_dimension_tensor(v_i), i) -> v_i -// -// We need to direct fusion to the inner loops. This cannot be done with -// a passmanager alone ATM, as nested pass managers require operations to -// be closed from above. -struct MapParallelLoops - : public mlir::PassWrapper { - void runOnFunction() override { - mlir::greedilyMapParallelSCFToGPU(getFunction().getBody()); - } -}; - -// We need to direct fusion to the inner loops. This cannot be done with -// a passmanager alone ATM, as nested pass managers require operations to -// be closed from above. -struct FuseInnerParallelLoops - : public mlir::PassWrapper { - void runOnFunction() override { - getFunction().walk([](mlir::scf::ParallelOp op) { - mlir::scf::naivelyFuseParallelOps(op.region()); - }); - } -}; - -// Collapse all loop dimension into the first one. -struct ParallelLoopCollapsingToFirstDim - : public mlir::PassWrapper> { - void runOnOperation() override { - mlir::Operation* module = getOperation(); - - module->walk([&](mlir::scf::ParallelOp op) { - unsigned num_loops = op.getNumLoops(); - std::vector combinedLoops; - combinedLoops.reserve(num_loops); - for (unsigned i = 0; i < num_loops; ++i) { - combinedLoops.push_back(i); - } - mlir::collapseParallelLoops(op, {combinedLoops}); - }); - } -}; -} // namespace Status LowerLHLOToGPU(mlir::ModuleOp module, LowerLHLOToGPUOptions options) { mlir::PassManager pm(module.getContext()); @@ -461,7 +68,7 @@ Status LowerLHLOToGPU(mlir::ModuleOp module, LowerLHLOToGPUOptions options) { // Moving `AllocOp`s and inserting missing `DeallocOp`s pm.addPass(::mlir::createBufferPlacementPass()); // Next, we can strip the outer fusion operation. - pm.addPass(absl::make_unique()); + pm.addPass(createFusionOpRemoverPass()); // Remove unnecessary LHLO copies. pm.addPass(::mlir::lmhlo::createLhloCopyRemovalPass()); // Transform LHLO operations to LinAlg. @@ -479,26 +86,26 @@ Status LowerLHLOToGPU(mlir::ModuleOp module, LowerLHLOToGPUOptions options) { pm.addNestedPass<::mlir::FuncOp>(::mlir::createCanonicalizerPass()); pm.addNestedPass<::mlir::FuncOp>(::mlir::createCSEPass()); // Fuse the inner-most loops. - pm.addPass(absl::make_unique()); + pm.addPass(createFuseInnerParallelLoopsPass()); // Run CSE to ensure that loads and stores to the same subview get // recognized as such. pm.addNestedPass<::mlir::FuncOp>(::mlir::createCSEPass()); // Forward stores to buffers to loads. - pm.addPass(absl::make_unique()); + pm.addPass(createStoreForwardingPass()); // Remove now unused temporary buffers. - pm.addPass(absl::make_unique()); + pm.addPass(createDeadTempBufferRemovalPass()); if (!options.unroll_factors.empty()) { pm.addPass(::mlir::createParallelLoopTilingPass(as_int64)); } // Project all loop dimensions to X if necessary. if (options.collapse_parallel_loops) { - pm.addPass(absl::make_unique()); + pm.addPass(createParallelLoopCollapsingToFirstDimPass()); } // Some basic cleanup. pm.addNestedPass<::mlir::FuncOp>(::mlir::createCanonicalizerPass()); pm.addNestedPass<::mlir::FuncOp>(::mlir::createCSEPass()); // Greedily map the remaining loop to GPU hardware dimensions. - pm.addPass(absl::make_unique()); + pm.addPass(createMapParallelLoopsPass()); // Apply the mapping. pm.addPass(mlir::createParallelLoopToGpuPass()); // Some basic cleanup. @@ -515,13 +122,13 @@ Status LowerLHLOToGPU(mlir::ModuleOp module, LowerLHLOToGPUOptions options) { ::mlir::mhlo::createLegalizeTanhToApproximationPass()); } // Move scalar operations into the launch to ensure smaller signatures. - pm.addPass(absl::make_unique()); + pm.addPass(createMoveScalarComputationsIntoGpuLaunchPass()); // Take launches to launches with kernels. pm.addPass(::mlir::createGpuKernelOutliningPass()); // Make sure the kernel signature resembled the original function's // signature if (options.rewrite_signature) { - pm.addPass(absl::make_unique()); + pm.addPass(createRewriteKernelSignaturePass()); } if (failed(pm.run(module))) { return InternalError("Lowering to GPU kernels failed."); @@ -595,5 +202,6 @@ StatusOr ExtractKernelModule(mlir::ModuleOp module) { }); return kernelModule; } + } // namespace mlir_gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/mlir_gpu/passes.cc b/tensorflow/compiler/xla/service/mlir_gpu/passes.cc new file mode 100644 index 00000000000..887f14e90d9 --- /dev/null +++ b/tensorflow/compiler/xla/service/mlir_gpu/passes.cc @@ -0,0 +1,423 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/xla/service/mlir_gpu/passes.h" + +#include "absl/memory/memory.h" +#include "llvm/ADT/SetVector.h" +#include "mlir/Dialect/GPU/GPUDialect.h" // from @llvm-project +#include "mlir/Dialect/GPU/ParallelLoopMapper.h" // from @llvm-project +#include "mlir/Dialect/SCF/SCF.h" // from @llvm-project +#include "mlir/Dialect/SCF/Transforms.h" // from @llvm-project +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/BlockAndValueMapping.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/Transforms/LoopUtils.h" // from @llvm-project +#include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" + +namespace xla { +namespace mlir_gpu { +namespace { + +struct FusionOpRemoverPass + : public mlir::PassWrapper { + void runOnFunction() override { + getFunction().walk([&](mlir::lmhlo::FusionOp op) { + mlir::OpBuilder builder(op); + // FusionOp has a single region with a single block, so we can just walk + // over it and clone operations to the outside. + mlir::BlockAndValueMapping mapping; + for (auto& nested_op : op.region().front().without_terminator()) { + auto clone = builder.clone(nested_op, mapping); + for (auto pair : + llvm::zip(nested_op.getResults(), clone->getResults())) { + mapping.map(std::get<0>(pair), std::get<1>(pair)); + } + } + op.erase(); + }); + } +}; + +struct StoreForwardingPass + : mlir::PassWrapper { + mlir::StoreOp findStore(mlir::Operation* op, + std::function matches) { + // Search from op upwards in the current block. + mlir::Block* block = op->getBlock(); + auto startFromIt = + std::find_if(block->rbegin(), block->rend(), + [op](mlir::Operation& other) { return &other == op; }); + for (auto storeOpIt = startFromIt; storeOpIt != block->rend(); + ++storeOpIt) { + auto storeOp = llvm::dyn_cast(&*(storeOpIt)); + if (!storeOp || !matches(storeOp)) { + continue; + } + + return storeOp; + } + // No store operation found. Continue search outside of the parallel + // loop if block is in a parallel loop. + if (auto parallelOp = + llvm::dyn_cast(block->getParentOp())) { + return findStore(parallelOp.getOperation(), matches); + } + return {}; + } + + // Recursively search defining ops for AllocOp. Return either AllocOp if it is + // found or nullptr. + mlir::Operation* SearchAllocOp(mlir::Value memref) { + mlir::Operation* defOp = memref.getDefiningOp(); + while (auto subviewOp = mlir::dyn_cast_or_null(defOp)) { + defOp = subviewOp.source().getDefiningOp(); + } + if (auto allocOp = mlir::dyn_cast_or_null(defOp)) { + return allocOp.getOperation(); + } + return nullptr; + } + + // Retrieves AllocOp from the cache or actually looks for it. + mlir::Operation* GetAllocOp( + mlir::Value memref, + llvm::DenseMap* memrefToAllocOp) { + auto allocOpIt = memrefToAllocOp->find(memref); + if (allocOpIt != memrefToAllocOp->end()) { + return allocOpIt->second; + } + auto allocOp = SearchAllocOp(memref); + memrefToAllocOp->insert({memref, allocOp}); + return allocOp; + } + + void runOnFunction() override { + llvm::DenseMap memrefToAllocOp; + + getFunction().walk([&](mlir::LoadOp loadOp) { + auto storeOp = findStore(loadOp, [&](mlir::StoreOp storeOp) { + mlir::Operation* storeOpAlloc = + GetAllocOp(storeOp.memref(), &memrefToAllocOp); + mlir::Operation* loadOpAlloc = + GetAllocOp(loadOp.memref(), &memrefToAllocOp); + return storeOpAlloc && loadOpAlloc && (storeOpAlloc == loadOpAlloc); + }); + if (!storeOp) { + return; + } + auto storeIndices = storeOp.getIndices(); + auto loadIndices = loadOp.getIndices(); + if (!std::equal(storeIndices.begin(), storeIndices.end(), + loadIndices.begin(), loadIndices.end())) { + return; + } + loadOp.replaceAllUsesWith(storeOp.getValueToStore()); + loadOp.erase(); + }); + } +}; + +struct DeadTempBufferRemovalPass + : mlir::PassWrapper { + bool operationConsideredDead(mlir::Operation* op) { + for (auto result : op->getResults()) { + if (!llvm::all_of(result.getUsers(), [&](mlir::Operation* op) { + // Store and Dealloc is OK. + if (llvm::isa(op)) { + return true; + } + // Load without uses is also ok. + if (auto loadOp = llvm::dyn_cast(op)) { + return loadOp.use_empty(); + } + // Subview is ok if it is dead itself. + if (llvm::isa(op)) { + return operationConsideredDead(op); + } + return false; + })) { + return false; + } + } + return true; + } + + void recursiveErase(mlir::Operation* op, + llvm::SmallVectorImpl* erase_list) { + for (auto result : op->getResults()) { + for (auto user : llvm::make_early_inc_range(result.getUsers())) { + recursiveErase(user, erase_list); + } + } + erase_list->push_back(op); + } + + void runOnFunction() override { + llvm::SmallVector dead_ops; + getFunction().walk([&](mlir::AllocOp allocOp) { + if (!operationConsideredDead(allocOp)) { + return; + } + + // TODO(herhut): There should be a generic helper for this. + recursiveErase(allocOp, &dead_ops); + }); + for (auto op : dead_ops) { + op->erase(); + } + } +}; + +struct MoveScalarComputationsIntoGpuLaunchPass + : mlir::PassWrapper { + static bool isInliningBeneficiary(mlir::Operation* op) { + return llvm::isa(op); + } + + static bool extractBeneficiaryOps( + mlir::Operation* op, llvm::SmallVectorImpl* ops, + llvm::SetVector args) { + if (!isInliningBeneficiary(op)) { + return false; + } + + ops->push_back(op); + for (auto operand : op->getOperands()) { + // It is an existing arg, keep going. + if (args.count(operand)) { + continue; + } + mlir::Operation* definingOp = operand.getDefiningOp(); + if (!definingOp || !extractBeneficiaryOps(definingOp, ops, args)) { + return false; + } + } + return true; + } + + static void inlineOperationsIntoLaunch(mlir::gpu::LaunchOp launch) { + llvm::SetVector used_above; + mlir::getUsedValuesDefinedAbove(launch.body(), used_above); + mlir::BlockAndValueMapping inlined_map; + for (mlir::Value v : used_above) { + llvm::SmallVector ops_to_move; + mlir::Operation* definingOp = v.getDefiningOp(); + if (definingOp && + extractBeneficiaryOps(definingOp, &ops_to_move, used_above)) { + mlir::OpBuilder b(launch.body()); + for (mlir::Operation* op : llvm::reverse(ops_to_move)) { + auto result = b.clone(*op, inlined_map); + for (auto pair : llvm::zip(op->getResults(), result->getResults())) { + mlir::replaceAllUsesInRegionWith(std::get<0>(pair), + std::get<1>(pair), launch.body()); + } + inlined_map.map(op->getResults(), result->getResults()); + } + } + } + } + + void runOnFunction() override { + mlir::FuncOp fun = getFunction(); + fun.walk( + [](mlir::gpu::LaunchOp launch) { inlineOperationsIntoLaunch(launch); }); + } +}; + +struct RewriteKernelSignaturePass + : mlir::PassWrapper { + void runOnFunction() override { + mlir::FuncOp func = getFunction(); + mlir::ModuleOp module = func.getParentOfType(); + getFunction().walk([&](mlir::gpu::LaunchFuncOp launchOp) { + mlir::gpu::GPUFuncOp kernel = + module.lookupSymbol(launchOp.kernel()); + + if (kernel.getNumFuncArguments() != + func.getNumArguments() + func.getNumResults()) { + kernel.emitError() + << "number of kernel arguments does not match number" + << "of arguments and results of surrounding function"; + signalPassFailure(); + return; + } + if (!llvm::hasSingleElement(func)) { + func.emitError() << "surrounding function has more than one block"; + signalPassFailure(); + return; + } + + // Compute a map from function arguments to kernel function operands. + mlir::BlockAndValueMapping func_to_kernel; + for (mlir::BlockArgument arg : func.getArguments()) { + for (int i = 0, e = launchOp.getNumKernelOperands(); i < e; ++i) { + if (launchOp.getKernelOperand(i) == arg) { + func_to_kernel.map(arg, kernel.getArgument(i)); + break; + } + } + } + // Also add function results that are computed by the launch. + mlir::Operation* returnOp = func.getBody().back().getTerminator(); + for (mlir::Value result : returnOp->getOperands()) { + for (int i = 0, e = launchOp.getNumKernelOperands(); i < e; ++i) { + if (launchOp.getKernelOperand(i) == result) { + func_to_kernel.map(result, kernel.getArgument(i)); + break; + } + } + } + + // Create a new kernel function with modified signature. It will have the + // parameters and result types of the original funcion as its parameter + // type and otherwise will be void. + auto gpu_module = kernel.getParentOfType(); + mlir::OpBuilder kernel_builder(gpu_module.body()); + auto operand_types = llvm::to_vector<4>(llvm::concat( + func.getType().getInputs(), func.getType().getResults())); + auto new_kernel = kernel_builder.create( + kernel.getLoc(), kernel.getName(), + kernel_builder.getFunctionType(operand_types, {})); + new_kernel.setAttr(mlir::gpu::GPUDialect::getKernelFuncAttrName(), + kernel_builder.getUnitAttr()); + + // Create a map from old kernel argument to new one. + mlir::BlockAndValueMapping old_kernel_to_new; + for (int i = 0, e = func.getNumArguments(); i < e; ++i) { + mlir::Value func_arg = func.getArgument(i); + mlir::Value new_kernel_arg = new_kernel.getArgument(i); + mlir::Value old_kernel_arg = func_to_kernel.lookupOrNull(func_arg); + if (!old_kernel_arg) { + kernel.emitOpError() + << "argument " << i + << " to containing function is not an argument to the kernel"; + signalPassFailure(); + return; + } + old_kernel_to_new.map(old_kernel_arg, new_kernel_arg); + } + for (int i = 0, e = returnOp->getNumOperands(); i < e; ++i) { + mlir::Value ret_op = returnOp->getOperand(i); + mlir::Value new_kernel_arg = + new_kernel.getArgument(func.getNumArguments() + i); + mlir::Value old_kernel_arg = func_to_kernel.lookupOrNull(ret_op); + if (!old_kernel_arg) { + kernel.emitOpError() + << "result " << i + << " of containing function is not an argument to the kernel"; + signalPassFailure(); + return; + } + old_kernel_to_new.map(old_kernel_arg, new_kernel_arg); + } + // Steal the body by appending the blocks and inserting a branch. + kernel.body().cloneInto(&new_kernel.getBody(), old_kernel_to_new); + kernel_builder.setInsertionPointToEnd(&new_kernel.body().front()); + kernel_builder.create( + new_kernel.getLoc(), &*std::next(new_kernel.body().begin())); + // Now create a new launchOp calling the new kernel. We need to forward + // the arguments of the surrounding function and operands to the return. + mlir::SmallVector new_operands; + new_operands.reserve(new_kernel.getNumFuncArguments()); + new_operands.append(func.args_begin(), func.args_end()); + new_operands.append(returnOp->operand_begin(), returnOp->operand_end()); + mlir::OpBuilder launch_builder(launchOp); + launch_builder.create( + launchOp.getLoc(), new_kernel, launchOp.getGridSizeOperandValues(), + launchOp.getBlockSizeOperandValues(), new_operands); + // Launch does not have results, so we can just erase it. And the kernel + // also needs to go. + launchOp.erase(); + kernel.erase(); + }); + } +}; + +struct MapParallelLoopsPass + : public mlir::PassWrapper { + void runOnFunction() override { + mlir::greedilyMapParallelSCFToGPU(getFunction().getBody()); + } +}; + +struct FuseInnerParallelLoopsPass + : public mlir::PassWrapper { + void runOnFunction() override { + getFunction().walk([](mlir::scf::ParallelOp op) { + mlir::scf::naivelyFuseParallelOps(op.region()); + }); + } +}; + +struct ParallelLoopCollapsingToFirstDimPass + : public mlir::PassWrapper> { + void runOnOperation() override { + mlir::Operation* module = getOperation(); + + module->walk([&](mlir::scf::ParallelOp op) { + unsigned num_loops = op.getNumLoops(); + std::vector combinedLoops; + combinedLoops.reserve(num_loops); + for (unsigned i = 0; i < num_loops; ++i) { + combinedLoops.push_back(i); + } + mlir::collapseParallelLoops(op, {combinedLoops}); + }); + } +}; + +} // namespace + +std::unique_ptr createFusionOpRemoverPass() { + return absl::make_unique(); +} + +std::unique_ptr createStoreForwardingPass() { + return absl::make_unique(); +} + +std::unique_ptr createDeadTempBufferRemovalPass() { + return absl::make_unique(); +} + +std::unique_ptr +createMoveScalarComputationsIntoGpuLaunchPass() { + return absl::make_unique(); +} + +std::unique_ptr createRewriteKernelSignaturePass() { + return absl::make_unique(); +} + +std::unique_ptr createFuseInnerParallelLoopsPass() { + return absl::make_unique(); +} + +std::unique_ptr createMapParallelLoopsPass() { + return absl::make_unique(); +} + +std::unique_ptr> +createParallelLoopCollapsingToFirstDimPass() { + return absl::make_unique(); +} + +} // namespace mlir_gpu +} // namespace xla diff --git a/tensorflow/compiler/xla/service/mlir_gpu/passes.h b/tensorflow/compiler/xla/service/mlir_gpu/passes.h new file mode 100644 index 00000000000..e3840628a2e --- /dev/null +++ b/tensorflow/compiler/xla/service/mlir_gpu/passes.h @@ -0,0 +1,66 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_MLIR_GPU_PASSES_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_MLIR_GPU_PASSES_H_ + +#include + +#include "mlir/Pass/Pass.h" // from @llvm-project + +namespace xla { +namespace mlir_gpu { + +// TODO(herhut, pifon): Move these passes to MLIR Core. + +/// Replaces a FusionOp by the operations contained in its region. +std::unique_ptr createFusionOpRemoverPass(); + +/// Replaces a load that immediately follows a store to the same address with +/// the stored value. This needs generalization. +std::unique_ptr createStoreForwardingPass(); + +/// Removes temporary buffers that are only written to but never read from or +/// that are read but the read value is not used. Needs an analysis that proves +/// that loads and stores are side-effect free (in bounds, no aliasing, etc.). +std::unique_ptr createDeadTempBufferRemovalPass(); + +/// Moves scalar computations to the GPULaunchOp body. +std::unique_ptr +createMoveScalarComputationsIntoGpuLaunchPass(); + +/// Sorts the operands to the kernel for a deterministic order. First operands +/// that are defined by function arguments, followed by operands that are +/// returned from the function. This only works for simple functions without +/// control flow and can be used in cases where the kernel is extracted and used +/// independently of the host-side code. +std::unique_ptr createRewriteKernelSignaturePass(); + +/// We need to direct fusion to the inner loops. This cannot be done with +/// a passmanager alone ATM, as nested pass managers require operations to +/// be closed from above. +std::unique_ptr createFuseInnerParallelLoopsPass(); + +/// Greedily maps loops to GPU hardware dimensions. +std::unique_ptr createMapParallelLoopsPass(); + +/// Collapses all loop dimension into the first one. +std::unique_ptr> +createParallelLoopCollapsingToFirstDimPass(); + +} // namespace mlir_gpu +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_MLIR_GPU_PASSES_H_ From d9f7377784a85bc619e66a74138f9d8fe300446a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 05:38:25 -0700 Subject: [PATCH 0816/1017] Integrate LLVM at llvm/llvm-project@b2b7dbb47aa9 Updates LLVM usage to match [b2b7dbb47aa9](https://github.com/llvm/llvm-project/commit/b2b7dbb47aa9) PiperOrigin-RevId: 326003266 Change-Id: Id69725d2af6a92b2f55046438a6ec8e7c025581e --- tensorflow/workspace.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 6493fe87836..7552eb2dd5a 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "0de60b550b727fa3a0202a9ab5ca30520e291dd5" - LLVM_SHA256 = "d23a64cca502c32aa3990b5252f19cb2ad59534084384df6d7a355e6f23fac62" + LLVM_COMMIT = "b2b7dbb47aa9aff1252d4440bb9986df5a7e67cb" + LLVM_SHA256 = "1e26635eb360b81f75304d172a6eea50ea5a55a42fd21a35b973321b32df69e9" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), From 4507fa1347f0671c50123ba2fa0a100776562efe Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Tue, 11 Aug 2020 05:48:36 -0700 Subject: [PATCH 0817/1017] Fix bug in occurring handling when composite symbols enter a undefined into a TF loop. PiperOrigin-RevId: 326004271 Change-Id: I1a8641ca4d71bd9396fd2b5d73dc49c0c8fb9612 --- .../autograph/operators/control_flow.py | 8 ++++-- .../autograph/operators/control_flow_test.py | 28 +++++++++++++++++++ tensorflow/python/autograph/utils/testing.py | 3 ++ 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py index 7b307ed5020..ef9c3ae6427 100644 --- a/tensorflow/python/autograph/operators/control_flow.py +++ b/tensorflow/python/autograph/operators/control_flow.py @@ -934,7 +934,7 @@ def _shape_invariants_mapping_to_positional_list(mapping, keys): LEGAL_LOOP_TYPES = 'Tensor, int, float, bool or a list, tuple or dict thereof' -def _placeholder_value(like, original): +def _placeholder_value(like, original=None): if isinstance(like, (variables.Undefined, variables.UndefinedReturnValue)): return original if isinstance(like, (int, float, bool)): @@ -1079,8 +1079,10 @@ def _tf_while_stmt(test, body, get_state, set_state, symbol_names, opts): _runtime_zero_iterations_errmsg(symbol_names, nulls, orig_init_vars) ]) ]): - final_loop_vars = tuple( - array_ops.identity(v) for v in final_loop_vars[1:]) + final_loop_vars = nest.map_structure( + lambda v: (array_ops.identity(v) if tensor_util.is_tensor(v) else v), + final_loop_vars[1:], + ) set_state(final_loop_vars) diff --git a/tensorflow/python/autograph/operators/control_flow_test.py b/tensorflow/python/autograph/operators/control_flow_test.py index 553643956f6..32b36a29797 100644 --- a/tensorflow/python/autograph/operators/control_flow_test.py +++ b/tensorflow/python/autograph/operators/control_flow_test.py @@ -626,6 +626,34 @@ class WhileLoopTest(testing.AutoGraphTestCase): # Node naming is inconsistent between V1 and V2. self.assertGraphContains(r'(while/)?pow$', 1) + def test_tensor_creating_complex_variable(self): + + def body(): + nonlocal i, s + i = {'a': constant_op.constant(2), 'b': {'c': constant_op.constant(1)}} + s = i['a'] ** 5 + + def set_state(loop_vars): + nonlocal i, s + i, s = loop_vars + + i = variable_operators.Undefined('i') + s = constant_op.constant(0) + control_flow.while_stmt( + test=lambda: math_ops.equal(s, 0), + body=body, + get_state=lambda: (i, s), + set_state=set_state, + symbol_names=('i', 's'), + opts={}) + + self.assertDictEqual(i, {'a': 2, 'b': {'c': 1}}) + self.assertEqual(s, 32) + self.assertOpCreated('StatelessWhile') + # Check that the temporary staging of the body did not create extra ops. + # Node naming is inconsistent between V1 and V2. + self.assertGraphContains(r'(while/)?pow$', 1) + def test_tensor_with_side_effecting_condition(self): v = self.variable('v', 0, dtypes.int32) diff --git a/tensorflow/python/autograph/utils/testing.py b/tensorflow/python/autograph/utils/testing.py index df60583bf85..bec6966e7cb 100644 --- a/tensorflow/python/autograph/utils/testing.py +++ b/tensorflow/python/autograph/utils/testing.py @@ -142,3 +142,6 @@ class AutoGraphTestCase(test.TestCase): def assertEqual(self, *args): self.assertions.append((super().assertEqual, list(args))) + + def assertDictEqual(self, *args): + self.assertions.append((super().assertDictEqual, list(args))) From 7fd83a73f00efda181def4bee99e07413038dbe5 Mon Sep 17 00:00:00 2001 From: Pankaj Kanwar Date: Tue, 11 Aug 2020 08:06:11 -0700 Subject: [PATCH 0818/1017] update cuda paths for cuda11 PiperOrigin-RevId: 326022147 Change-Id: I7e0dff924c3d4ad5e3b50cdcd6551572ca93de93 --- .../gcc7_manylinux2010-nvcc-cuda11/BUILD | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/BUILD b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/BUILD index 92305526c5c..358af09fbdd 100755 --- a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/BUILD +++ b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11/BUILD @@ -65,9 +65,9 @@ cc_toolchain_config( "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include", "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include-fixed", "/dt7/usr/include", - "/usr/local/cuda11/targets/x86_64-linux/include", - "/usr/local/cuda11/include", - "/usr/local/cuda11/extras/CUPTI/include", + "/usr/local/cuda-11.0/targets/x86_64-linux/include", + "/usr/local/cuda-11.0/include", + "/usr/local/cuda-11.0/extras/CUPTI/include", "/usr/include", ], builtin_sysroot = "", @@ -105,9 +105,9 @@ cc_toolchain_config( "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include", "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include-fixed", "/dt7/usr/include", - "/usr/local/cuda-11/targets/x86_64-linux/include", - "/usr/local/cuda-11/include", - "/usr/local/cuda-11/extras/CUPTI/include", + "/usr/local/cuda-11.0/targets/x86_64-linux/include", + "/usr/local/cuda-11.0/include", + "/usr/local/cuda-11.0/extras/CUPTI/include", "/usr/include", ], cpu = "darwin", @@ -143,9 +143,9 @@ cc_toolchain_config( "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include", "/dt7/usr/lib/gcc/x86_64-pc-linux-gnu/7/include-fixed", "/dt7/usr/include", - "/usr/local/cuda-11/targets/x86_64-linux/include", - "/usr/local/cuda-11/include", - "/usr/local/cuda-11/extras/CUPTI/include", + "/usr/local/cuda-11.0/targets/x86_64-linux/include", + "/usr/local/cuda-11.0/include", + "/usr/local/cuda-11.0/extras/CUPTI/include", "/usr/include", ], cpu = "x64_windows", From 0dc091cb46ad7778fbe19fd7519eb41cadba777d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 08:47:31 -0700 Subject: [PATCH 0819/1017] Integrate LLVM at llvm/llvm-project@950f1bf976b3 Updates LLVM usage to match [950f1bf976b3](https://github.com/llvm/llvm-project/commit/950f1bf976b3) PiperOrigin-RevId: 326028888 Change-Id: Ia11d7606673f92f9de38703cc961ed8bc3416e75 --- tensorflow/workspace.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 7552eb2dd5a..95b9c91a62f 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "b2b7dbb47aa9aff1252d4440bb9986df5a7e67cb" - LLVM_SHA256 = "1e26635eb360b81f75304d172a6eea50ea5a55a42fd21a35b973321b32df69e9" + LLVM_COMMIT = "950f1bf976b332eca60267b25bf759e2ad564e0c" + LLVM_SHA256 = "89b0e1e5d0cd56adfbe061fc42804088eaed6773e8ff9f1d597137b474055096" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), From 289c617c2ce0f5961da742ba4ad3e0ea23cbdb94 Mon Sep 17 00:00:00 2001 From: Michael Gester Date: Tue, 11 Aug 2020 09:16:38 -0700 Subject: [PATCH 0820/1017] Change benefit of TF2XLA fallback pattern to 0 This pattern should only be used as a last resort so we use the minimum benefit for it. PiperOrigin-RevId: 326034560 Change-Id: I1e11cae4807aad82dc0919ea32cac7eb1a59cca2 --- tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc | 6 +++--- .../compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc index 1f63f2a9396..5fe933ee635 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf.cc @@ -5793,15 +5793,15 @@ LogicalResult legalizeTF( // Add TF->HLO legalization patterns. PopulateLegalizeTfPatterns(context, &patterns); + // Add TF->TF lowering patterns. + TF::PopulateLoweringTFPatterns(context, &patterns); + // Add TF->HLO legalization patterns via TF2XLA fallback. if (tf2xla_fallback_device_type.hasValue()) { PopulateLegalizeTfWithTf2XlaPatterns(tf2xla_fallback_device_type.getValue(), patterns); } - // Add TF->TF lowering patterns. - TF::PopulateLoweringTFPatterns(context, &patterns); - // Populate with CHLO->HLO lowerings to account for TF ops legalized to // CHLO first. if (legalize_chlo) { diff --git a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc index 6e651df5075..904b80e05b1 100644 --- a/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc +++ b/tensorflow/compiler/mlir/xla/transforms/legalize_tf_with_tf2xla.cc @@ -499,8 +499,10 @@ tensorflow::XlaExpression Tf2XlaRewriter::GetExprForOperand(Value operand, class Tf2XlaRewritePattern : public RewritePattern { public: + // Set benefit to 0 (= least benefit) so this pattern is only used as a + // fallback. explicit Tf2XlaRewritePattern(const std::string& device_type) - : RewritePattern(1, MatchAnyOpTypeTag()), device_type_(device_type) {} + : RewritePattern(0, MatchAnyOpTypeTag()), device_type_(device_type) {} LogicalResult matchAndRewrite(Operation* op, PatternRewriter& rewriter) const override { From ee2c2d17814c015477041dcafed0c9c7f1f00162 Mon Sep 17 00:00:00 2001 From: Raman Sarokin Date: Tue, 11 Aug 2020 09:32:56 -0700 Subject: [PATCH 0821/1017] Virtual Tune splitted to Virtual GetPossibleKernelWorkGroups(API neutral) and non-virtual generic Tune(API specific). PiperOrigin-RevId: 326037556 Change-Id: Ie330fe2b17316046c3a4cbb7d509d8cb6ab42fd4 --- .../lite/delegates/gpu/cl/kernels/BUILD | 2 + .../delegates/gpu/cl/kernels/concat_test.cc | 4 +- .../lite/delegates/gpu/cl/kernels/conv_3d.cc | 16 +- .../lite/delegates/gpu/cl/kernels/conv_3d.h | 5 +- .../gpu/cl/kernels/conv_buffer_1x1.cc | 10 +- .../gpu/cl/kernels/conv_buffer_1x1.h | 5 +- .../delegates/gpu/cl/kernels/conv_powervr.cc | 15 +- .../delegates/gpu/cl/kernels/conv_powervr.h | 5 +- .../delegates/gpu/cl/kernels/conv_texture.cc | 8 +- .../delegates/gpu/cl/kernels/conv_texture.h | 5 +- .../gpu/cl/kernels/convolution_transposed.cc | 8 +- .../gpu/cl/kernels/convolution_transposed.h | 5 +- .../cl/kernels/convolution_transposed_3d.cc | 12 +- .../cl/kernels/convolution_transposed_3d.h | 5 +- .../cl/kernels/convolution_transposed_3x3.h | 7 +- .../cl/kernels/convolution_transposed_4x4.h | 7 +- .../gpu/cl/kernels/depthwise_conv_3x3.cc | 11 +- .../gpu/cl/kernels/depthwise_conv_3x3.h | 5 +- .../gpu/cl/kernels/fully_connected.h | 7 +- .../delegates/gpu/cl/kernels/gpu_operation.cc | 29 ++++ .../delegates/gpu/cl/kernels/gpu_operation.h | 9 +- .../delegates/gpu/cl/kernels/lstm_test.cc | 2 +- .../lite/delegates/gpu/cl/kernels/mean.h | 7 +- .../cl/kernels/mean_stddev_normalization.h | 7 +- .../delegates/gpu/cl/kernels/mean_test.cc | 2 +- .../delegates/gpu/cl/kernels/softmax1x1.h | 7 +- .../lite/delegates/gpu/cl/kernels/winograd.cc | 39 ++--- .../lite/delegates/gpu/cl/kernels/winograd.h | 14 +- .../gpu/cl/kernels/work_group_picking.cc | 140 +++++++++--------- .../gpu/cl/kernels/work_group_picking.h | 40 ++--- 30 files changed, 267 insertions(+), 171 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/BUILD b/tensorflow/lite/delegates/gpu/cl/kernels/BUILD index 27e12b5981f..c8351304188 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/cl/kernels/BUILD @@ -1388,6 +1388,8 @@ test_suite( "fully_connected_test", "lstm_test", "max_unpooling_test", + "mean_stddev_normalization_test", + "mean_test", "padding_test", "pooling_test", "prelu_test", diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/concat_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/concat_test.cc index b2e2e23b6f9..d6889af7717 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/concat_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/concat_test.cc @@ -118,7 +118,7 @@ TEST_F(OpenCLOperationTest, ConcatChannels) { op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; ConcatZ operation = - CreateConcatZ(op_def, {1, 2, 3}, env_.GetDevicePtr()->GetInfo()); + CreateConcatZ(op_def, {1, 2, 3}, env_.GetDevicePtr()->info_); ASSERT_OK(ExecuteGPUOperation({src0, src1, src2}, creation_context_, &operation, BHWC(1, 2, 1, 6), &dst_tensor)); EXPECT_THAT(dst_tensor.data, @@ -152,7 +152,7 @@ TEST_F(OpenCLOperationTest, ConcatChannelsAlignedx4) { op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; ConcatZ operation = - CreateConcatZ(op_def, {4, 4}, env_.GetDevicePtr()->GetInfo()); + CreateConcatZ(op_def, {4, 4}, env_.GetDevicePtr()->info_); ASSERT_OK(ExecuteGPUOperation({src0, src1}, creation_context_, &operation, BHWC(1, 2, 1, 8), &dst_tensor)); EXPECT_THAT( diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.cc index 727cd488694..4b898378c2d 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.cc @@ -245,21 +245,25 @@ int3 Conv3D::GetGridSize() const { wg[conv_params_.work_group_launch_order[2]] * work_group_size_.z); } -absl::Status Conv3D::Tune(const TuningParameters& params) { +void Conv3D::GetPossibleKernelWorkGroups(TuningType tuning_type, + const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const { if (conv_params_.weights_upload_type == WeightsUploadType::LOCAL_MEM_ASYNC_SUBGROUP || conv_params_.weights_upload_type == WeightsUploadType::LOCAL_MEM_BY_THREADS) { - return absl::OkStatus(); + work_groups->push_back(work_group_size_); + return; } if (conv_params_.work_group_launch_order[0] == 0 && conv_params_.work_group_launch_order[1] == 1 && conv_params_.work_group_launch_order[2] == 2) { - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - RETURN_IF_ERROR( - GetBestWorkGroupConv(params, kernel_, grid_size_, &work_group_size_)); + GetPossibleWorkGroupsConv(tuning_type, device_info, kernel_info, grid_size_, + work_groups); + } else { + work_groups->push_back(work_group_size_); } - return absl::OkStatus(); } std::string Conv3D::GenerateConv3D(const OperationDef& op_def, diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.h b/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.h index ffa269d1629..e53c9c8a6d0 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_3d.h @@ -39,7 +39,10 @@ namespace cl { class Conv3D : public GPUOperation { public: Conv3D() = default; - absl::Status Tune(const TuningParameters& params) override; + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override; absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc index 38e04e221f2..e75fe02df7a 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.cc @@ -315,11 +315,11 @@ int3 ConvBuffer1x1::GetGridSize() const { return int3(grid_x, grid_y, grid_z); } -absl::Status ConvBuffer1x1::Tune(const TuningParameters& params) { - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - RETURN_IF_ERROR( - GetBestWorkGroupConv(params, kernel_, grid_size_, &work_group_size_)); - return absl::OkStatus(); +void ConvBuffer1x1::GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const { + GetPossibleWorkGroupsConv(tuning_type, device_info, kernel_info, grid_size_, + work_groups); } bool IsConvBuffer1x1Supported(const OperationDef& definition, diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.h b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.h index 94b7cbd1b37..530aec70a17 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_buffer_1x1.h @@ -47,7 +47,10 @@ class ConvBuffer1x1 : public GPUOperation { ConvBuffer1x1(const ConvBuffer1x1&) = delete; ConvBuffer1x1& operator=(const ConvBuffer1x1&) = delete; - absl::Status Tune(const TuningParameters& params) override; + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override; int3 GetGridSize() const override; ConvWeightsDescription GetConvWeightsDescription() const { diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc index f04102d25d6..eb5baa8a6ba 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.cc @@ -240,22 +240,25 @@ int3 ConvPowerVR::GetGridSize() const { } } -absl::Status ConvPowerVR::Tune(const TuningParameters& params) { +void ConvPowerVR::GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const { if (conv_params_.weights_upload_type == WeightsUploadType::LOCAL_MEM_ASYNC_SUBGROUP || conv_params_.weights_upload_type == WeightsUploadType::LOCAL_MEM_BY_THREADS || conv_params_.fixed_work_group_size) { - return absl::OkStatus(); + work_groups->push_back(work_group_size_); + return; } if (conv_params_.work_group_launch_order[0] == 0 && conv_params_.work_group_launch_order[1] == 1 && conv_params_.work_group_launch_order[2] == 2) { - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - RETURN_IF_ERROR( - GetBestWorkGroupConv(params, kernel_, grid_size_, &work_group_size_)); + GetPossibleWorkGroupsConv(tuning_type, device_info, kernel_info, grid_size_, + work_groups); + } else { + work_groups->push_back(work_group_size_); } - return absl::OkStatus(); } std::string ConvPowerVR::GenerateConv(const DeviceInfo& device_info, diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.h b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.h index e61d4c14ce7..1ff6db43cbc 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_powervr.h @@ -41,7 +41,10 @@ namespace cl { class ConvPowerVR : public GPUOperation { public: ConvPowerVR() = default; - absl::Status Tune(const TuningParameters& params) override; + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override; absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc index 581c8056ced..7f987cc724c 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.cc @@ -420,9 +420,11 @@ int3 ConvTexture::GetGridSize() const { return int3(grid_x, grid_y, grid_z); } -absl::Status ConvTexture::Tune(const TuningParameters& params) { - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - return GetBestWorkGroupConv(params, kernel_, grid_size_, &work_group_size_); +void ConvTexture::GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const { + GetPossibleWorkGroupsConv(tuning_type, device_info, kernel_info, grid_size_, + work_groups); } absl::Status CreateConvTexture(const CreationContext& creation_context, diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.h b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.h index 10efc23a044..8406918fe80 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/conv_texture.h @@ -42,7 +42,10 @@ namespace cl { class ConvTexture : public GPUOperation { public: ConvTexture() = default; - absl::Status Tune(const TuningParameters& params) override; + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override; absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc index c6eba691306..314d0b20499 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.cc @@ -351,9 +351,11 @@ int3 ConvolutionTransposed::GetGridSize() const { return int3(grid_x, grid_y, grid_z); } -absl::Status ConvolutionTransposed::Tune(const TuningParameters& params) { - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - return GetBestWorkGroupConv(params, kernel_, grid_size_, &work_group_size_); +void ConvolutionTransposed::GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const { + GetPossibleWorkGroupsConv(tuning_type, device_info, kernel_info, grid_size_, + work_groups); } absl::Status CreateConvolutionTransposed( diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.h b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.h index 44e1c942925..9f865f8f0b7 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed.h @@ -38,7 +38,10 @@ namespace cl { class ConvolutionTransposed : public GPUOperation { public: ConvolutionTransposed() = default; - absl::Status Tune(const TuningParameters& params) override; + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override; absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.cc b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.cc index eeb3ae15e51..2b35080b1ab 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.cc @@ -394,17 +394,19 @@ int3 ConvolutionTransposed3D::GetGridSize() const { return int3(grid_x, grid_y, grid_z); } -absl::Status ConvolutionTransposed3D::Tune(const TuningParameters& params) { - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - return GetBestWorkGroupConv(params, kernel_, grid_size_, &work_group_size_); +void ConvolutionTransposed3D::GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const { + GetPossibleWorkGroupsConv(tuning_type, device_info, kernel_info, grid_size_, + work_groups); } absl::Status CreateConvolutionTransposed3D( const CreationContext& creation_context, const OperationDef& definition, const ConvolutionTransposed3DAttributes& attr, ConvolutionTransposed3D* result) { - *result = ConvolutionTransposed3D(definition, attr, - creation_context.device->GetInfo()); + *result = + ConvolutionTransposed3D(definition, attr, creation_context.device->info_); RETURN_IF_ERROR( result->UploadWeights(attr.weights, creation_context.context)); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.h b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.h index 0025d9da7b6..919181bceab 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3d.h @@ -38,7 +38,10 @@ namespace cl { class ConvolutionTransposed3D : public GPUOperation { public: ConvolutionTransposed3D() = default; - absl::Status Tune(const TuningParameters& params) override; + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override; absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3.h b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3.h index 9addfe11984..0f4022b6eb6 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_3x3.h @@ -37,8 +37,11 @@ namespace cl { class ConvolutionTransposed3x3 : public GPUOperation { public: ConvolutionTransposed3x3() = default; - absl::Status Tune(const TuningParameters& params) override { - return absl::OkStatus(); + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override { + work_groups->push_back(work_group_size_); } absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_4x4.h b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_4x4.h index 21ec8c3e293..6344ca39bc0 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_4x4.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/convolution_transposed_4x4.h @@ -37,8 +37,11 @@ namespace cl { class ConvolutionTransposed4x4 : public GPUOperation { public: ConvolutionTransposed4x4() = default; - absl::Status Tune(const TuningParameters& params) override { - return absl::OkStatus(); + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override { + work_groups->push_back(work_group_size_); } absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc index f0213cda805..bb1b409482f 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.cc @@ -301,12 +301,15 @@ int3 DepthwiseConv3x3::GetGridSize() const { return int3(grid_x, grid_y, grid_z); } -absl::Status DepthwiseConv3x3::Tune(const TuningParameters& params) { +void DepthwiseConv3x3::GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const { if (local_mem_uploads_) { - return absl::OkStatus(); + work_groups->push_back(work_group_size_); + } else { + GetPossibleWorkGroups(tuning_type, device_info, kernel_info, grid_size_, + work_groups); } - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - return GetBestWorkGroup(params, kernel_, GetGridSize(), &work_group_size_); } bool IsDepthwiseConv3x3Supported(const DepthwiseConvolution2DAttributes& attr) { diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.h b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.h index dedc9b530bb..b324b039f2b 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/depthwise_conv_3x3.h @@ -38,7 +38,10 @@ namespace cl { class DepthwiseConv3x3 : public GPUOperation { public: DepthwiseConv3x3() = default; - absl::Status Tune(const TuningParameters& params) override; + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override; absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/fully_connected.h b/tensorflow/lite/delegates/gpu/cl/kernels/fully_connected.h index ced3913ead7..8543c3defc0 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/fully_connected.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/fully_connected.h @@ -89,8 +89,11 @@ void RearrangeFCWeightsToIOO4I4(const tflite::gpu::Tensor& weights, class FullyConnected : public GPUOperation { public: FullyConnected() = default; - absl::Status Tune(const TuningParameters& params) override { - return absl::OkStatus(); + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override { + work_groups->push_back(work_group_size_); } int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc index 0aa1842791f..29f6c038f77 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.cc @@ -247,6 +247,35 @@ absl::Status GPUOperation::Compile(const CreationContext& creation_context) { return PostCompileCheck(creation_context.device->info_, kernel_.info_); } +void GPUOperation::GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const { + GetPossibleWorkGroups(tuning_type, device_info, kernel_info, grid_size_, + work_groups); +} + +absl::Status GPUOperation::Tune(const TuningParameters& params) { + std::vector possible_work_groups; + GetPossibleKernelWorkGroups(params.tuning_type, *params.info, kernel_.info_, + &possible_work_groups); + if (possible_work_groups.empty()) { + return absl::NotFoundError( + "Can not found work_group size to launch kernel"); + } + if (possible_work_groups.size() == 1) { + work_group_size_ = possible_work_groups[0]; + return absl::OkStatus(); + } else { + RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); + int best_work_group_index; + RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( + kernel_, *params.info, grid_size_, possible_work_groups, + &best_work_group_index)); + work_group_size_ = possible_work_groups[best_work_group_index]; + return absl::OkStatus(); + } +} + int3 GPUOperation::GetGridSize() const { if (elementwise_) { const int grid_x = dst_[0]->Width() * dst_[0]->Batch(); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h index ba266f8dcc9..80f2eb3c950 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/gpu_operation.h @@ -93,10 +93,11 @@ class GPUOperation { return queue->DispatchImplicit(kernel_, grid_size_, work_group_size_); } - virtual absl::Status Tune(const TuningParameters& params) { - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - return GetBestWorkGroup(params, kernel_, grid_size_, &work_group_size_); - } + virtual void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const; + + absl::Status Tune(const TuningParameters& params); absl::Status Compile(const CreationContext& creation_context); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/lstm_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/lstm_test.cc index d7ea3ee6474..52e9b4cba4c 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/lstm_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/lstm_test.cc @@ -67,7 +67,7 @@ TEST_F(OpenCLOperationTest, LSTM) { op_def.dst_tensors.push_back({data_type, storage, Layout::BHWC}); TensorFloat32 new_state; TensorFloat32 new_activ; - LSTM operation = CreateLSTM(op_def, env_.GetDevicePtr()->GetInfo()); + LSTM operation = CreateLSTM(op_def, env_.GetDevicePtr()->info_); ASSERT_OK(ExecuteGPUOperation( {src_tensor, prev_state}, creation_context_, &operation, {BHWC(1, 1, 1, 4), BHWC(1, 1, 1, 4)}, {&new_state, &new_activ})); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/mean.h b/tensorflow/lite/delegates/gpu/cl/kernels/mean.h index cfdd7be53d3..12735c0b916 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/mean.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/mean.h @@ -31,8 +31,11 @@ class Mean : public GPUOperation { Mean() = default; Mean(const OperationDef& definition, const DeviceInfo& device_info); - absl::Status Tune(const TuningParameters& params) override { - return absl::OkStatus(); + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override { + work_groups->push_back(work_group_size_); } absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.h b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.h index 7dd45fcb86a..47cc7ff46d1 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/mean_stddev_normalization.h @@ -30,8 +30,11 @@ class MeanStdDevNormalization : public GPUOperation { public: explicit MeanStdDevNormalization(const OperationDef& definition); - absl::Status Tune(const TuningParameters& params) override { - return absl::OkStatus(); + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override { + work_groups->push_back(work_group_size_); } int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/mean_test.cc b/tensorflow/lite/delegates/gpu/cl/kernels/mean_test.cc index dbb70127317..b1ae1d354eb 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/mean_test.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/mean_test.cc @@ -47,7 +47,7 @@ TEST_F(OpenCLOperationTest, Mean) { op_def.src_tensors.push_back({data_type, storage, Layout::HWC}); op_def.dst_tensors.push_back({data_type, storage, Layout::HWC}); TensorFloat32 dst_tensor; - Mean operation = CreateMean(op_def, env_.GetDevicePtr()->GetInfo()); + Mean operation = CreateMean(op_def, env_.GetDevicePtr()->info_); ASSERT_OK(ExecuteGPUOperation(src_tensor, creation_context_, &operation, BHWC(1, 1, 1, 1), &dst_tensor)); EXPECT_THAT(dst_tensor.data, Pointwise(FloatNear(eps), {2.5f})); diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/softmax1x1.h b/tensorflow/lite/delegates/gpu/cl/kernels/softmax1x1.h index 42cbbabe799..5bc9278d612 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/softmax1x1.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/softmax1x1.h @@ -29,8 +29,11 @@ class Softmax1x1 : public GPUOperation { public: Softmax1x1() = default; explicit Softmax1x1(const OperationDef& definition); - absl::Status Tune(const TuningParameters& params) override { - return absl::OkStatus(); + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override { + work_groups->push_back(work_group_size_); } absl::Status BindArguments() override; int3 GetGridSize() const override; diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc b/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc index c77c805a712..3af4c658ce2 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/winograd.cc @@ -259,11 +259,11 @@ absl::Status Winograd4x4To36::UploadBt(CLContext* context) { return absl::OkStatus(); } -int3 Winograd4x4To36::SelectBestWorkGroup() { +int3 Winograd4x4To36::SelectBestWorkGroup(const KernelInfo& kernel_info) const { const std::vector wgs = {{8, 6, 4}, {8, 6, 2}, {4, 6, 2}, {4, 6, 2}, {2, 6, 2}, {2, 6, 1}, {1, 6, 1}, {1, 3, 1}, {1, 1, 1}}; - return GetFirstSuitableWorkGroup(wgs, kernel_.info_.max_work_group_size); + return GetFirstSuitableWorkGroup(wgs, kernel_info.max_work_group_size); } absl::Status Winograd4x4To36::BindArguments() { @@ -286,15 +286,18 @@ int3 Winograd4x4To36::GetGridSize() const { return int3(grid_x, grid_y, grid_z); } -absl::Status Winograd4x4To36::Tune(const TuningParameters& params) { - switch (params.tuning_type) { +void Winograd4x4To36::GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const { + switch (tuning_type) { case TuningType::EXHAUSTIVE: - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - return GetBestWorkGroup(params, kernel_, grid_size_, &work_group_size_); + GetPossibleWorkGroups(tuning_type, device_info, kernel_info, grid_size_, + work_groups); + return; case TuningType::FAST: default: - work_group_size_ = SelectBestWorkGroup(); - return absl::OkStatus(); + work_groups->push_back(SelectBestWorkGroup(kernel_info)); + return; } } @@ -461,11 +464,11 @@ absl::Status Winograd36To4x4::UploadAt(CLContext* context) { return absl::OkStatus(); } -int3 Winograd36To4x4::SelectBestWorkGroup() { +int3 Winograd36To4x4::SelectBestWorkGroup(const KernelInfo& kernel_info) const { const std::vector wgs = {{32, 4, 2}, {16, 4, 2}, {16, 4, 1}, {8, 4, 1}, {4, 4, 1}, {2, 4, 1}, {1, 4, 1}, {1, 2, 1}, {1, 1, 1}}; - return GetFirstSuitableWorkGroup(wgs, kernel_.info_.max_work_group_size); + return GetFirstSuitableWorkGroup(wgs, kernel_info.max_work_group_size); } absl::Status Winograd36To4x4::BindArguments() { @@ -485,19 +488,21 @@ int3 Winograd36To4x4::GetGridSize() const { return int3(grid_x, grid_y, grid_z); } -absl::Status Winograd36To4x4::Tune(const TuningParameters& params) { - switch (params.tuning_type) { +void Winograd36To4x4::GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, std::vector* work_groups) const { + switch (tuning_type) { case TuningType::EXHAUSTIVE: - RETURN_IF_ERROR(args_.Bind(kernel_.kernel())); - return GetBestWorkGroup(params, kernel_, grid_size_, &work_group_size_); + GetPossibleWorkGroups(tuning_type, device_info, kernel_info, grid_size_, + work_groups); + return; case TuningType::FAST: default: - work_group_size_ = SelectBestWorkGroup(); - return absl::OkStatus(); + work_groups->push_back(SelectBestWorkGroup(kernel_info)); + return; } } - absl::Status CreateWinograd36To4x4( const CreationContext& creation_context, const OperationDef& definition, const tflite::gpu::Tensor& biases, diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/winograd.h b/tensorflow/lite/delegates/gpu/cl/kernels/winograd.h index ddc1155e0b5..08153f1d8aa 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/winograd.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/winograd.h @@ -38,7 +38,10 @@ class Winograd4x4To36 : public GPUOperation { const DeviceInfo& device_info); absl::Status BindArguments() override; int3 GetGridSize() const override; - absl::Status Tune(const TuningParameters& params) override; + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override; // Move only Winograd4x4To36(Winograd4x4To36&& operation); @@ -56,7 +59,7 @@ class Winograd4x4To36 : public GPUOperation { std::string GetWinograd4x4To36Code(const OperationDef& op_def); // Must be called after kernel compilation - int3 SelectBestWorkGroup(); + int3 SelectBestWorkGroup(const KernelInfo& kernel_info) const; Padding2D padding_; }; @@ -73,7 +76,10 @@ class Winograd36To4x4 : public GPUOperation { const DeviceInfo& device_info); absl::Status BindArguments() override; int3 GetGridSize() const override; - absl::Status Tune(const TuningParameters& params) override; + void GetPossibleKernelWorkGroups( + TuningType tuning_type, const DeviceInfo& device_info, + const KernelInfo& kernel_info, + std::vector* work_groups) const override; // Move only Winograd36To4x4(Winograd36To4x4&& operation); @@ -92,7 +98,7 @@ class Winograd36To4x4 : public GPUOperation { std::string GetWinograd36To4x4Code(const OperationDef& op_def); // Must be called after kernel compilation - int3 SelectBestWorkGroup(); + int3 SelectBestWorkGroup(const KernelInfo& kernel_info) const; }; absl::Status CreateWinograd36To4x4( diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc index 9a1a24895bf..4c0cbc06985 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc +++ b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.cc @@ -33,9 +33,9 @@ std::vector Get2DWorkgroupsEqualTo128() { {8, 16}, {4, 32}, {2, 64}, {1, 128}}; } -std::vector GenerateWorkGroupSizesXY128( - int3 grid, const KernelInfo& kernel_info, - WorkGroupSizeAlignment z_alignment) { +std::vector GenerateWorkGroupSizesXYMultipleOf( + int multiplier, int3 grid, const KernelInfo& kernel_info, + const DeviceInfo& device_info, WorkGroupSizeAlignment z_alignment) { std::vector work_groups; work_groups.reserve(32); @@ -44,7 +44,7 @@ std::vector GenerateWorkGroupSizesXY128( for (int x = 1; x <= kernel_info.max_work_group_size; x *= 2) { for (int y = 1; y <= kernel_info.max_work_group_size; y *= 2) { int work_group_size_xy = x * y; - if (work_group_size_xy % 128 != 0 || + if (work_group_size_xy % multiplier != 0 || work_group_size_xy > kernel_info.max_work_group_size) { continue; } @@ -52,26 +52,38 @@ std::vector GenerateWorkGroupSizesXY128( if (work_group_size_xy * z > kernel_info.max_work_group_size) { continue; } - work_groups.push_back({x, y, z}); + if (x <= device_info.max_work_group_size_x && + y <= device_info.max_work_group_size_y && + z <= device_info.max_work_group_size_z) { + work_groups.push_back({x, y, z}); + } } } } return work_groups; } -std::vector GenerateWorkGroupSizesXY128Linear( - int3 grid, const KernelInfo& kernel_info, - WorkGroupSizeAlignment z_alignment) { +std::vector GenerateWorkGroupSizesXMultipleOf( + int multiplier, int3 grid, const KernelInfo& kernel_info, + const DeviceInfo& device_info, WorkGroupSizeAlignment z_alignment) { std::vector work_groups; work_groups.reserve(32); std::vector possible_z_sizes = GetPossibleSizes(grid.z, z_alignment); + std::vector possible_y_sizes = + GetPossibleSizes(grid.y, WorkGroupSizeAlignment::PRECISE); - for (int x = 128; x <= kernel_info.max_work_group_size && x < grid.x + 128; - x += 128) { - for (auto z : possible_z_sizes) { - if (x * z <= kernel_info.max_work_group_size) { - work_groups.push_back({x, 1, z}); + for (int x = multiplier; + x <= kernel_info.max_work_group_size && x < grid.x + multiplier; + x += multiplier) { + for (auto y : possible_y_sizes) { + for (auto z : possible_z_sizes) { + if (x <= device_info.max_work_group_size_x && + y <= device_info.max_work_group_size_y && + z <= device_info.max_work_group_size_z && + x * y * z <= kernel_info.max_work_group_size) { + work_groups.push_back({x, y, z}); + } } } } @@ -202,31 +214,24 @@ int3 GetWorkGroupConv(const int3& grid, int max_size, int max_z_size) { return int3(wg_x, wg_y, wg_z); } -absl::Status GetBestWorkGroupXY128(const TuningParameters& params, - const CLKernel& kernel, const int3& grid, - WorkGroupSizeAlignment z_alignment, - int3* best_work_group) { - std::vector work_groups = - GenerateWorkGroupSizesXY128(grid, kernel.info_, z_alignment); - int best_work_group_index; - RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( - kernel, *params.info, grid, work_groups, &best_work_group_index)); - *best_work_group = work_groups[best_work_group_index]; - return absl::OkStatus(); +void GetPossibleWorkGroupsXYMultipleOf(int multiplier, + const DeviceInfo& device_info, + const KernelInfo& kernel_info, + const int3& grid, + WorkGroupSizeAlignment z_alignment, + std::vector* work_groups) { + *work_groups = GenerateWorkGroupSizesXYMultipleOf( + multiplier, grid, kernel_info, device_info, z_alignment); } -absl::Status GetBestWorkGroupXY128Linear(const TuningParameters& params, - const CLKernel& kernel, - const int3& grid, - WorkGroupSizeAlignment z_alignment, - int3* best_work_group) { - std::vector work_groups = - GenerateWorkGroupSizesXY128Linear(grid, kernel.info_, z_alignment); - int best_work_group_index; - RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( - kernel, *params.info, grid, work_groups, &best_work_group_index)); - *best_work_group = work_groups[best_work_group_index]; - return absl::OkStatus(); +void GetPossibleWorkGroupsXMultipleOf(int multiplier, + const DeviceInfo& device_info, + const KernelInfo& kernel_info, + const int3& grid, + WorkGroupSizeAlignment z_alignment, + std::vector* work_groups) { + *work_groups = GenerateWorkGroupSizesXMultipleOf( + multiplier, grid, kernel_info, device_info, z_alignment); } bool XY128RequiresMoreWorkGroupsThenXY128Linear(int width, int height) { @@ -245,56 +250,47 @@ bool XY128RequiresMoreWorkGroupsThenXY128Linear(int width, int height) { return !have_equal_work_groups; } -absl::Status GetBestWorkGroup(const TuningParameters& params, - const CLKernel& kernel, const int3& grid, - int3* best_work_group) { - switch (params.tuning_type) { +void GetPossibleWorkGroups(TuningType tuning_type, + const DeviceInfo& device_info, + const KernelInfo& kernel_info, const int3& grid, + std::vector* work_groups) { + switch (tuning_type) { case TuningType::FAST: - *best_work_group = GetWorkGroup(grid, kernel.info_.max_work_group_size); - return absl::OkStatus(); + work_groups->push_back( + GetWorkGroup(grid, kernel_info.max_work_group_size)); + return; case TuningType::EXHAUSTIVE: { - std::vector work_groups; - GetWorkGroupsAlignedToGrid(*params.info, kernel.info_, grid, - &work_groups); - int best_work_group_index; - RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( - kernel, *params.info, grid, work_groups, &best_work_group_index)); - *best_work_group = work_groups[best_work_group_index]; - return absl::OkStatus(); + GetWorkGroupsAlignedToGrid(device_info, kernel_info, grid, work_groups); + return; } default: - *best_work_group = {8, 4, 1}; - return absl::OkStatus(); + work_groups->push_back({8, 4, 1}); + return; } } -absl::Status GetBestWorkGroupConv(const TuningParameters& params, - const CLKernel& kernel, const int3& grid, - int3* best_work_group) { - switch (params.tuning_type) { +void GetPossibleWorkGroupsConv(TuningType tuning_type, + const DeviceInfo& device_info, + const KernelInfo& kernel_info, const int3& grid, + std::vector* work_groups) { + switch (tuning_type) { case TuningType::FAST: { int max_z_size = 16; - if (params.info->IsAdreno()) { - max_z_size = params.info->adreno_info.gpu_version < 400 ? 16 : 64; + if (device_info.IsAdreno()) { + max_z_size = device_info.IsAdreno3xx() ? 16 : 64; } - max_z_size = std::min(max_z_size, params.info->max_work_group_size_z); - *best_work_group = - GetWorkGroupConv(grid, kernel.info_.max_work_group_size, max_z_size); - return absl::OkStatus(); + max_z_size = std::min(max_z_size, device_info.max_work_group_size_z); + work_groups->push_back( + GetWorkGroupConv(grid, kernel_info.max_work_group_size, max_z_size)); + return; } case TuningType::EXHAUSTIVE: { - std::vector work_groups; - GetWorkGroupsAlignedToGrid(*params.info, kernel.info_, grid, - &work_groups); - int best_work_group_index; - RETURN_IF_ERROR(params.queue->GetBestWorkGroupIndex( - kernel, *params.info, grid, work_groups, &best_work_group_index)); - *best_work_group = work_groups[best_work_group_index]; - return absl::OkStatus(); + GetWorkGroupsAlignedToGrid(device_info, kernel_info, grid, work_groups); + return; } default: - *best_work_group = {8, 4, 1}; - return absl::OkStatus(); + work_groups->push_back({8, 4, 1}); + return; } } diff --git a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.h b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.h index 7cc60f4723f..0c1be10782e 100644 --- a/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.h +++ b/tensorflow/lite/delegates/gpu/cl/kernels/work_group_picking.h @@ -27,20 +27,20 @@ namespace tflite { namespace gpu { namespace cl { -// writes best_work_group if successful -// Here and later you can find XY128, this is because 128 is SIMD width of A6xx -// And XY128 means that work_group_size.x * work_group_size.y % 128 = 0 -// We need it to correctly work with constants uploading on A6xx -absl::Status GetBestWorkGroupXY128(const TuningParameters& params, - const CLKernel& kernel, const int3& grid, - WorkGroupSizeAlignment z_alignment, - int3* best_work_group); +// multiplier can be power of two only +void GetPossibleWorkGroupsXYMultipleOf(int multiplier, + const DeviceInfo& device_info, + const KernelInfo& kernel_info, + const int3& grid, + WorkGroupSizeAlignment z_alignment, + std::vector* work_groups); -absl::Status GetBestWorkGroupXY128Linear(const TuningParameters& params, - const CLKernel& kernel, - const int3& grid, - WorkGroupSizeAlignment z_alignment, - int3* best_work_group); +void GetPossibleWorkGroupsXMultipleOf(int multiplier, + const DeviceInfo& device_info, + const KernelInfo& kernel_info, + const int3& grid, + WorkGroupSizeAlignment z_alignment, + std::vector* work_groups); int3 GetWorkGroupXY128ConvLinear(const int3& grid); @@ -49,13 +49,15 @@ int3 GetWorkGroupXY128Conv(const int3& grid); bool XY128RequiresMoreWorkGroupsThenXY128Linear(int width, int height); -absl::Status GetBestWorkGroup(const TuningParameters& params, - const CLKernel& kernel, const int3& grid, - int3* best_work_group); +void GetPossibleWorkGroups(TuningType tuning_type, + const DeviceInfo& device_info, + const KernelInfo& kernel_info, const int3& grid, + std::vector* work_groups); -absl::Status GetBestWorkGroupConv(const TuningParameters& params, - const CLKernel& kernel, const int3& grid, - int3* best_work_group); +void GetPossibleWorkGroupsConv(TuningType tuning_type, + const DeviceInfo& device_info, + const KernelInfo& kernel_info, const int3& grid, + std::vector* work_groups); } // namespace cl } // namespace gpu From b4297ce0a82e4dc513fb819ab55e86379ac46e7f Mon Sep 17 00:00:00 2001 From: Paul Wankadia Date: Tue, 11 Aug 2020 09:39:30 -0700 Subject: [PATCH 0822/1017] Make RegexFullMatchOp and RegexReplaceOp cache RE2 objects. PiperOrigin-RevId: 326038690 Change-Id: Ia3b0177e38a40c1514c08d8821b992aca60cd0ac --- .../core/kernels/regex_full_match_op.cc | 36 +++++++++++++-- tensorflow/core/kernels/regex_replace_op.cc | 44 +++++++++++++++---- 2 files changed, 67 insertions(+), 13 deletions(-) diff --git a/tensorflow/core/kernels/regex_full_match_op.cc b/tensorflow/core/kernels/regex_full_match_op.cc index 04da969df12..f00e971c0bc 100644 --- a/tensorflow/core/kernels/regex_full_match_op.cc +++ b/tensorflow/core/kernels/regex_full_match_op.cc @@ -20,6 +20,8 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/thread_annotations.h" #include "tensorflow/core/util/ptr_util.h" namespace tensorflow { @@ -28,6 +30,8 @@ class RegexFullMatchOp : public OpKernel { public: explicit RegexFullMatchOp(OpKernelConstruction* ctx) : OpKernel(ctx) {} + ~RegexFullMatchOp() override {} + void Compute(OpKernelContext* ctx) override { const Tensor* input_tensor; OP_REQUIRES_OK(ctx, ctx->input("input", &input_tensor)); @@ -39,19 +43,43 @@ class RegexFullMatchOp : public OpKernel { errors::InvalidArgument("Pattern must be scalar, but received ", pattern_tensor->shape().DebugString())); const string pattern = pattern_tensor->flat()(0); - const RE2 match(pattern); - OP_REQUIRES(ctx, match.ok(), + std::shared_ptr regex = CachedRE2(pattern); + OP_REQUIRES(ctx, regex->ok(), errors::InvalidArgument("Invalid pattern: ", pattern, - ", error: ", match.error())); + ", error: ", regex->error())); Tensor* output_tensor = nullptr; OP_REQUIRES_OK(ctx, ctx->allocate_output("output", input_tensor->shape(), &output_tensor)); auto output_flat = output_tensor->flat(); for (size_t i = 0; i < input_flat.size(); ++i) { - output_flat(i) = RE2::FullMatch(input_flat(i), match); + output_flat(i) = RE2::FullMatch(input_flat(i), *regex); } } + + private: + std::shared_ptr CachedRE2(const string& pattern) { + { + tf_shared_lock l(mu_); + if (regex_ != nullptr && regex_->pattern() == pattern) { + return regex_; + } + } + // Construct the new RE2 object before acquiring the lock. + auto regex = std::make_shared(pattern); + { + mutex_lock l(mu_); + // Swap instead of assigning so that we destruct the old + // RE2 object (when necessary) after releasing the lock. + regex_.swap(regex); + return regex_; + } + } + + mutex mu_; + std::shared_ptr regex_ TF_GUARDED_BY(mu_); + + TF_DISALLOW_COPY_AND_ASSIGN(RegexFullMatchOp); }; REGISTER_KERNEL_BUILDER(Name("RegexFullMatch").Device(DEVICE_CPU), diff --git a/tensorflow/core/kernels/regex_replace_op.cc b/tensorflow/core/kernels/regex_replace_op.cc index 4eb83c5fe0d..5e464e0a13a 100644 --- a/tensorflow/core/kernels/regex_replace_op.cc +++ b/tensorflow/core/kernels/regex_replace_op.cc @@ -20,6 +20,8 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/thread_annotations.h" #include "tensorflow/core/util/ptr_util.h" namespace tensorflow { @@ -29,7 +31,7 @@ namespace { // Context requirements: // - "input" string Tensor at input_index=0 // - "output" string Tensor at output_index=0 -Status InternalCompute(const RE2& match, const string& rewrite, +Status InternalCompute(const RE2& regex, const string& rewrite, const bool replace_global, OpKernelContext* ctx) { const Tensor* input_tensor; TF_RETURN_IF_ERROR(ctx->input("input", &input_tensor)); @@ -52,9 +54,9 @@ Status InternalCompute(const RE2& match, const string& rewrite, // accept std::string. string buf = output_flat(i); if (replace_global) { - RE2::GlobalReplace(&buf, match, rewrite); + RE2::GlobalReplace(&buf, regex, rewrite); } else { - RE2::Replace(&buf, match, rewrite); + RE2::Replace(&buf, regex, rewrite); } output_flat(i) = std::move(buf); } @@ -68,6 +70,8 @@ class RegexReplaceOp : public OpKernel { OP_REQUIRES_OK(ctx, ctx->GetAttr("replace_global", &replace_global_)); } + ~RegexReplaceOp() override {} + void Compute(OpKernelContext* ctx) override { const Tensor* pattern_tensor; OP_REQUIRES_OK(ctx, ctx->input("pattern", &pattern_tensor)); @@ -75,10 +79,10 @@ class RegexReplaceOp : public OpKernel { errors::InvalidArgument("Pattern must be scalar, but received ", pattern_tensor->shape().DebugString())); const string& pattern = pattern_tensor->scalar()(); - const RE2 match(pattern); - OP_REQUIRES(ctx, match.ok(), + std::shared_ptr regex = CachedRE2(pattern); + OP_REQUIRES(ctx, regex->ok(), errors::InvalidArgument("Invalid pattern: ", pattern, - ", error: ", match.error())); + ", error: ", regex->error())); const Tensor* rewrite_tensor; OP_REQUIRES_OK(ctx, ctx->input("rewrite", &rewrite_tensor)); @@ -86,11 +90,33 @@ class RegexReplaceOp : public OpKernel { errors::InvalidArgument("Rewrite must be scalar, but received ", rewrite_tensor->shape().DebugString())); const string& rewrite = rewrite_tensor->scalar()(); - OP_REQUIRES_OK(ctx, InternalCompute(match, rewrite, replace_global_, ctx)); + OP_REQUIRES_OK(ctx, InternalCompute(*regex, rewrite, replace_global_, ctx)); } private: + std::shared_ptr CachedRE2(const string& pattern) { + { + tf_shared_lock l(mu_); + if (regex_ != nullptr && regex_->pattern() == pattern) { + return regex_; + } + } + // Construct the new RE2 object before acquiring the lock. + auto regex = std::make_shared(pattern); + { + mutex_lock l(mu_); + // Swap instead of assigning so that we destruct the old + // RE2 object (when necessary) after releasing the lock. + regex_.swap(regex); + return regex_; + } + } + bool replace_global_; + mutex mu_; + std::shared_ptr regex_ TF_GUARDED_BY(mu_); + + TF_DISALLOW_COPY_AND_ASSIGN(RegexReplaceOp); }; REGISTER_KERNEL_BUILDER(Name("RegexReplace").Device(DEVICE_CPU), @@ -101,11 +127,11 @@ class StaticRegexReplaceOp : public OpKernel { explicit StaticRegexReplaceOp(OpKernelConstruction* ctx) : OpKernel(ctx) { string pattern; OP_REQUIRES_OK(ctx, ctx->GetAttr("pattern", &pattern)); - OP_REQUIRES_OK(ctx, ctx->GetAttr("rewrite", &rewrite_str_)); re_ = MakeUnique(pattern); OP_REQUIRES(ctx, re_->ok(), errors::InvalidArgument("Invalid pattern: ", pattern, ", error: ", re_->error())); + OP_REQUIRES_OK(ctx, ctx->GetAttr("rewrite", &rewrite_str_)); OP_REQUIRES_OK(ctx, ctx->GetAttr("replace_global", &replace_global_)); } @@ -115,8 +141,8 @@ class StaticRegexReplaceOp : public OpKernel { } private: - string rewrite_str_; std::unique_ptr re_; + string rewrite_str_; bool replace_global_; }; From 0f35ef2abc9535fbbf2e2b490867c8bd9212f03d Mon Sep 17 00:00:00 2001 From: Berkin Ilbeyi Date: Tue, 11 Aug 2020 09:47:25 -0700 Subject: [PATCH 0823/1017] [XLA] Implement mechanism to repack allocations to reduce fragmentation. This CL defines an interface for allocation repackers. A repacker can be specified in MemorySpaceAssignment::Options. If an HloValue couldn't be allocated due to running out of alternate memory, we now export the allocations done so far to the repacker, run the repacker, and import the new offsets back into memory space assignment for better memory packing. PiperOrigin-RevId: 326040207 Change-Id: Icc518874781eb74e38701514d8f6fb20d23a4124 --- tensorflow/compiler/xla/service/BUILD | 10 + .../xla/service/memory_space_assignment.cc | 71 +++++++ .../xla/service/memory_space_assignment.h | 32 +++ .../memory_space_assignment_repacking.h | 57 ++++++ .../service/memory_space_assignment_test.cc | 183 +++++++++++++++++- 5 files changed, 347 insertions(+), 6 deletions(-) create mode 100644 tensorflow/compiler/xla/service/memory_space_assignment_repacking.h diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD index fa7b480cab6..f5618b95c3e 100644 --- a/tensorflow/compiler/xla/service/BUILD +++ b/tensorflow/compiler/xla/service/BUILD @@ -3426,6 +3426,15 @@ cc_library( ], ) +cc_library( + name = "memory_space_assignment_repacking", + hdrs = ["memory_space_assignment_repacking.h"], + deps = [ + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:types", + ], +) + cc_library( name = "memory_space_assignment", srcs = ["memory_space_assignment.cc"], @@ -3433,6 +3442,7 @@ cc_library( deps = [ ":heap_simulator", ":hlo_cost_analysis", + ":memory_space_assignment_repacking", ":memory_space_assignment_utils", "//tensorflow/compiler/xla:debug_options_flags", "//tensorflow/core/lib/math:math_util", diff --git a/tensorflow/compiler/xla/service/memory_space_assignment.cc b/tensorflow/compiler/xla/service/memory_space_assignment.cc index 4131a0199bf..c5ae0573bed 100644 --- a/tensorflow/compiler/xla/service/memory_space_assignment.cc +++ b/tensorflow/compiler/xla/service/memory_space_assignment.cc @@ -1057,10 +1057,28 @@ HeapSimulator::Result AlternateMemoryBestFitHeap::Finish() { options_.prefetch_interval_picker->SetRetryNumber(retry_number); Result result = AllocateAllocationValues(absl::MakeSpan(allocation_values)); + VLOG(2) << "Allocation result = " + << absl::StrFormat("%x", static_cast(result)); if (result_requires_uncommit(result) || (!final_retry && result_failed_because_of_async_copy(result))) { UncommitPendingChunks(absl::MakeSpan(allocation_values)); VLOG(2) << "Couldn't allocate. Retry number " << retry_number; + } else if (result_is(result, Result::kFailOutOfMemory) && + num_repacks_ < options_.max_repacks) { + UncommitPendingChunks(absl::MakeSpan(allocation_values)); + ++num_repacks_; + CHECK_NE(options_.repacker, nullptr); + std::vector repack_allocation_blocks; + ExportAllocationsForRepacking(repack_allocation_blocks); + VLOG(2) << "Repacking."; + auto repack_status = + options_.repacker->Repack(absl::MakeSpan(repack_allocation_blocks)); + CHECK_EQ(repack_status.status(), Status::OK()); + VLOG(2) << "Repack complete. Modified = " << *repack_status; + if (*repack_status) { + ImportRepackedAllocations(absl::MakeSpan(repack_allocation_blocks)); + --retry_number; + } } else { FinalizeAllocations(absl::MakeSpan(allocation_values)); break; @@ -1541,6 +1559,33 @@ bool AlternateMemoryBestFitHeap::AreIntervalsReservedInAlternateMemory( return false; } +void AlternateMemoryBestFitHeap::ExportAllocationsForRepacking( + std::vector& + allocations) { + for (RepackAllocationBlock& allocation_block : repack_allocation_blocks_) { + allocations.push_back(&allocation_block); + } +} + +void AlternateMemoryBestFitHeap::ImportRepackedAllocations( + absl::Span + repacked_allocations) { + interval_tree_ = {}; + for (RepackAllocationBlock* allocation_block : repacked_allocations) { + MemorySpaceAssignment::Allocation* allocation = allocation_block->opaque; + VLOG(3) << "Moved " << allocation->ToString() << ", size " + << allocation->chunk().size << " from " + << allocation_block->initial_offset << " to " + << allocation_block->offset; + allocation_block->opaque->mutable_chunk()->offset = + allocation_block->offset; + interval_tree_.Add(allocation_block->start_time, allocation_block->end_time, + {allocation_block->offset, allocation_block->size}); + allocation_block->initial_offset = allocation_block->offset; + allocation_block->offset = -1; + } +} + void AlternateMemoryBestFitHeap::UncommitPendingChunks( absl::Span allocation_values) { // Clear the allocation sequence of the allocation values so that in case we @@ -1591,11 +1636,37 @@ void AlternateMemoryBestFitHeap::UncommitPendingChunks( void AlternateMemoryBestFitHeap::FinalizeAllocations( absl::Span allocation_values) { + absl::flat_hash_map> + colocation_map; for (AllocationValue& allocation_value : allocation_values) { for (auto& allocation : *allocation_value.allocation_sequence()) { AppendAllocationInfoDebugString(allocation_value, *allocation, allocation_info_str_); allocations_->push_back(std::move(allocation)); + MemorySpaceAssignment::Allocation* inserted_allocation = + allocations_->back().get(); + if (inserted_allocation->memory_space() == MemorySpace::kAlternate) { + colocation_map[inserted_allocation->chunk().offset].push_back( + inserted_allocation); + } + } + } + // Assume allocations that received the same offset need to be colocated. + // Export these to repack_allocation_blocks_ so that we can repack them to + // reduce fragmentation. + for (auto& colocation : colocation_map) { + std::vector colocations; + for (MemorySpaceAssignment::Allocation* colocated_allocation : + colocation.second) { + repack_allocation_blocks_.push_back( + {colocated_allocation->start_time(), colocated_allocation->end_time(), + colocated_allocation->chunk().size, /*offset=*/-1, + colocated_allocation->chunk().offset, /*colocations=*/{}, + colocated_allocation}); + colocations.push_back(&repack_allocation_blocks_.back()); + } + for (RepackAllocationBlock* repack_block : colocations) { + repack_block->colocations = colocations; } } ClearPendingChunks(); diff --git a/tensorflow/compiler/xla/service/memory_space_assignment.h b/tensorflow/compiler/xla/service/memory_space_assignment.h index d530a57d257..d366c06a599 100644 --- a/tensorflow/compiler/xla/service/memory_space_assignment.h +++ b/tensorflow/compiler/xla/service/memory_space_assignment.h @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/compiler/xla/service/heap_simulator.h" #include "tensorflow/compiler/xla/service/hlo_cost_analysis.h" +#include "tensorflow/compiler/xla/service/memory_space_assignment_repacking.h" namespace xla { @@ -379,6 +380,9 @@ class MemorySpaceAssignment { // space and a fast and small alternate memory space. enum class MemorySpace { kDefault, kAlternate }; + // Forward declaration for Allocation. + class Allocation; + // The different options to be passed to the Run() API. struct Options { // Backend-specific integer value that describes the alternate memory. @@ -424,6 +428,15 @@ class MemorySpaceAssignment { // copies or asynchronous copy ordering. int64 max_retries = 1; + // The maximum number of repacks that we are willing to perform in case we + // can't allocate a buffer due to running out of memory. If this value is + // greater than 0, repacker must be non-nullptr. + int64 max_repacks = 0; + + // The repacking algorithm to reduce fragmentation. Must be non-null if + // max_repacks is greater than 0. + MemorySpaceAssignmentRepacker* repacker = nullptr; + // If true, tries allocating buffers across (e.g., before and inside a while // loop body) sequential calls (kWhile, kCall, and kConditional). bool allocate_across_sequential_calls = false; @@ -511,6 +524,7 @@ class MemorySpaceAssignment { const std::vector& uses() const { return uses_; } MemorySpace memory_space() const { return memory_space_; } Chunk chunk() const { return *chunk_; } + Chunk* mutable_chunk() { return &*chunk_; } void set_start_time(int64 start_time) { start_time_ = start_time; } int64 start_time() const { return start_time_; } int64 end_time() const { return end_time_; } @@ -929,6 +943,9 @@ class AlternateMemoryBestFitHeap : public GlobalDecreasingSizeBestFitHeap { HeapSimulator::Result Finish() override; private: + using RepackAllocationBlock = MemorySpaceAssignmentRepacker< + MemorySpaceAssignment::Allocation*>::AllocationBlock; + // An allocation request for a use segment. A use segment is the time segment // between the definition and the first use, and the time segment between the // uses of a buffer. For example, the time between the definition and Use1, is @@ -1149,6 +1166,16 @@ class AlternateMemoryBestFitHeap : public GlobalDecreasingSizeBestFitHeap { absl::optional ViolatesAsyncCopyOrdering( int64 start_time, int64 end_time) const; + // Exports the allocations for repacking and puts them into the vector in the + // parameter. + void ExportAllocationsForRepacking( + std::vector& allocations); + + // Imports repacked allocations and updates the internal data structures + // consistent with the new packing. + void ImportRepackedAllocations( + absl::Span repacked_allocations); + // Adds an asynchronous copy to the allocations. void AddAsyncCopy(const MemorySpaceAssignment::Allocation& prev_allocation, MemorySpace memory_space, absl::optional chunk, @@ -1197,6 +1224,11 @@ class AlternateMemoryBestFitHeap : public GlobalDecreasingSizeBestFitHeap { BufferIntervalTree prefetch_interval_tree_; BufferIntervalTree eviction_interval_tree_; AsynchronousCopyOrdering async_copy_ordering_; + // A list of RepackAllocationBlock objects that mirrors allocation sequences, + // used for repacking. We use a list here because we need pointer stability + // for aliased allocations. + std::list repack_allocation_blocks_; + int64 num_repacks_ = 0; std::vector> pending_chunks_; std::vector pending_async_copies_; std::vector> diff --git a/tensorflow/compiler/xla/service/memory_space_assignment_repacking.h b/tensorflow/compiler/xla/service/memory_space_assignment_repacking.h new file mode 100644 index 00000000000..fcfdfc797fb --- /dev/null +++ b/tensorflow/compiler/xla/service/memory_space_assignment_repacking.h @@ -0,0 +1,57 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_MEMORY_SPACE_ASSIGNMENT_REPACKING_H_ +#define TENSORFLOW_COMPILER_XLA_SERVICE_MEMORY_SPACE_ASSIGNMENT_REPACKING_H_ + +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/types.h" + +namespace xla { + +// An interface to define allocation repacking algorithms. +template +class MemorySpaceAssignmentRepacker { + public: + MemorySpaceAssignmentRepacker() = default; + virtual ~MemorySpaceAssignmentRepacker() = default; + + // A contiguous block of allocation consisting of start and end (logical) + // times, size, and the initial offset. After repacking, if the repacking was + // successful and the allocations were modified, the offset field holds the + // new offset. To support aliased allocations, AllocationBlock also includes a + // vector of AllocationBlock pointers, called colocations. All AllocationBlock + // objects within the colocations must get the same offset. The opaque field + // is used by the MemorySpaceAssignment pass and should not be accessed by the + // repacking algorithm. + struct AllocationBlock { + int64 start_time; + int64 end_time; + int64 size; + int64 offset; + int64 initial_offset; + std::vector colocations; + O opaque; + }; + + // Repack the AllocationBlocks provided in the parameter. Returns true if + // allocations have been modified and false if not. Returns a non-ok status if + // there was an error. + virtual StatusOr Repack(absl::Span allocations) = 0; +}; + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_SERVICE_MEMORY_SPACE_ASSIGNMENT_REPACKING_H_ diff --git a/tensorflow/compiler/xla/service/memory_space_assignment_test.cc b/tensorflow/compiler/xla/service/memory_space_assignment_test.cc index a3f8024bca8..464cfb502be 100644 --- a/tensorflow/compiler/xla/service/memory_space_assignment_test.cc +++ b/tensorflow/compiler/xla/service/memory_space_assignment_test.cc @@ -71,19 +71,22 @@ class MemorySpaceAssignmentTest : public HloTestBase, std::unique_ptr AssignMemorySpace( HloModule* module, int64 max_outstanding_async_copies = -1, - int64 max_prefetch_interval = 10, int64 min_prefetch_interval = 2) { + int64 max_prefetch_interval = 10, int64 min_prefetch_interval = 2, + absl::optional options = absl::nullopt) { InstructionCountPrefetchIntervalPicker prefetch_interval_picker( min_prefetch_interval, max_prefetch_interval); return AssignMemorySpace(module, max_outstanding_async_copies, /*buffer_interval_compare=*/{}, - &prefetch_interval_picker); + &prefetch_interval_picker, options); } std::unique_ptr AssignMemorySpace( HloModule* module, int64 max_outstanding_async_copies, absl::optional buffer_interval_compare, - PrefetchIntervalPicker* prefetch_interval_picker) { + PrefetchIntervalPicker* prefetch_interval_picker, + absl::optional + memory_space_assignment_options = absl::nullopt) { auto size_fn = [](const BufferValue& buffer) { return ShapeUtil::ByteSizeOf(buffer.shape(), /*pointer_size=*/8); }; @@ -117,9 +120,15 @@ class MemorySpaceAssignmentTest : public HloTestBase, } MemorySpaceAssignment::Options options; + if (memory_space_assignment_options) { + options = *memory_space_assignment_options; + } else { + options.max_size_in_bytes = 128; + options.alignment_in_bytes = 8; + options.verify = true; + } + options.alternate_memory_space = kAlternateMemorySpace; - options.max_size_in_bytes = 128; - options.alignment_in_bytes = 8; options.buffer_interval_compare = buffer_interval_compare; options.prefetch_interval_picker = prefetch_interval_picker; options.size_fn = size_fn; @@ -127,7 +136,6 @@ class MemorySpaceAssignmentTest : public HloTestBase, options.max_outstanding_prefetches = max_outstanding_async_copies; options.max_outstanding_evictions = max_outstanding_async_copies; options.allocate_across_sequential_calls = GetParam(); - options.verify = true; auto alias_analysis = HloAliasAnalysis::Run(module).ValueOrDie(); std::unique_ptr hlo_live_range = @@ -4058,6 +4066,169 @@ TEST_P(MemorySpaceAssignmentTest, MoveCopyDoneEarlier) { find_schedule_index(cos->operand(0))); } +// A mock MemorySpaceAssignmentRepacker class that accepst a map of +// (start_time,offset) -> new_offset values. Using this map, the repacker +// repacks the allocations to the new_offset. +class FakeMemorySpaceAssignmentRepacker + : public MemorySpaceAssignmentRepacker { + public: + FakeMemorySpaceAssignmentRepacker( + absl::flat_hash_map, int64>& repack_map) + : repack_map_(repack_map) {} + + StatusOr Repack(absl::Span allocations) override { + bool modified = false; + for (AllocationBlock* block : allocations) { + VLOG(1) << "Alloc time: [" << block->start_time << ", " << block->end_time + << "] size: " << block->size + << " init offset: " << block->initial_offset; + auto it = repack_map_.find({block->start_time, block->initial_offset}); + if (it != repack_map_.end()) { + modified = true; + block->offset = it->second; + } else { + block->offset = block->initial_offset; + } + for (AllocationBlock* colocation : block->colocations) { + VLOG(1) << " [" << colocation->start_time << ", " + << colocation->end_time << "]"; + if (it != repack_map_.end()) { + colocation->offset = it->second; + } else { + colocation->offset = colocation->initial_offset; + } + } + } + + return modified; + } + + private: + // A map from (start_time, offset) to new_offset. + absl::flat_hash_map, int64> repack_map_; +}; + +TEST_P(MemorySpaceAssignmentTest, Repack) { + // We initially perform the following allocations at these offsets. + // + // Max memory + // ------------------------------------------- + // + // + // + // + // +------------+ + // | b | + // +------------+ + // +-------+ +------------+ + // | a | | n | + // +-------+ +------------+ + // ------------------------------------------- + // Min memory time -> + // + // Next up, we try to allocate the prefetch for m. However due to + // fragmentation, this won't be possible: + // + // Max memory + // ------------------------------------------- + // + // + // + // +---------+ + // +------------+ | + // | b | | | + // +------------+ | + // +-------+ | | +------------+ + // | a | | d | | n | + // +-------+ +---------+ +------------+ + // ------------------------------------------- + // Min memory time -> + // + // We then call repack to repack the existing allocations which allows us to + // allocate the prefetch for m: + // + // Max memory + // ------------------------------------------- + // +---------+ + // | | + // | | + // | | + // +-------+ | | + // | a | | d | + // +-------+ +---------+ + // +------------+ +------------+ + // | b | | n | + // +------------+ +------------+ + // ------------------------------------------- + // Min memory time -> + absl::string_view hlo_string = R"( + HloModule bug, is_scheduled=true + + ENTRY Entry { + param0 = f32[8,3] parameter(0) + param1 = f32[2,4] parameter(1) + a = f32[2,4] sine(param1) + b = f32[2,4] cosine(param1) + c = f32[8,3] negate(param0) + j = f32[2,4] negate(a) + d = f32[8,3] tanh(param0) + k = f32[2,4] negate(j) + l = f32[2,4] add(b, k) + m = f32[8,3] negate(d) + n = f32[2,4] sine(l) + o = f32[8,3] negate(m) + p = f32[2,4] negate(n) + q = f32[8,3] negate(m) + ROOT tuple = (f32[2,4], f32[8,3], f32[8,3]) tuple(p, q, o) + } + )"; + + MemorySpaceAssignment::BufferIntervalCompare buffer_interval_compare = + [](const MemorySpaceAssignment::BufferInterval& a, + const MemorySpaceAssignment::BufferInterval& b) { + auto get_opcode_priority = [](const HloOpcode& opcode) { + switch (opcode) { + case HloOpcode::kSin: + return 0; + case HloOpcode::kCos: + return 1; + case HloOpcode::kTanh: + return 2; + default: + return 3; + } + }; + + return get_opcode_priority(a.buffer->defining_instruction()->opcode()) < + get_opcode_priority(b.buffer->defining_instruction()->opcode()); + }; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + + InstructionCountPrefetchIntervalPicker prefetch_interval_picker(2, 10); + absl::flat_hash_map, int64> repack_map; + // Move "a" from offset 0 to 32. + repack_map[{2, 0}] = 32; + // Move "b" from offset 32 to 0. + repack_map[{3, 32}] = 0; + FakeMemorySpaceAssignmentRepacker repacker = + FakeMemorySpaceAssignmentRepacker(repack_map); + MemorySpaceAssignment::Options options; + options.max_size_in_bytes = 128; + options.alignment_in_bytes = 8; + options.verify = true; + options.max_repacks = 1; + options.repacker = &repacker; + AssignMemorySpace(module.get(), /*max_outstanding_async_copies=*/-1, + buffer_interval_compare, &prefetch_interval_picker, + options); + + // If repacking succeeds, we should find the buffer for d in alternate memory. + const HloInstruction* d = + module->entry_computation()->GetInstructionWithName("d"); + EXPECT_EQ(d->shape().layout().memory_space(), kAlternateMemorySpace); +} + TEST_P(MemorySpaceAssignmentTest, Determinism) { // Run memory space assignment a few times to make sure every time it compiles // to the same thing. From 6a2f00362eed16cfcc792d8fc716c19fa9108ea4 Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Tue, 11 Aug 2020 09:50:22 -0700 Subject: [PATCH 0824/1017] Add TpuTopologyExternal::version() and TpuVersionEnumToString(). PiperOrigin-RevId: 326040796 Change-Id: I3033d21271db465e4d6c57f5a3fe3c0c8913d1aa --- tensorflow/core/tpu/tpu_library_init_fns.inc | 2 ++ tensorflow/stream_executor/tpu/c_api_decl.h | 6 ++++++ .../stream_executor/tpu/tpu_executor_c_api.h | 2 ++ tensorflow/stream_executor/tpu/tpu_topology.cc | 15 +++++++++++++++ tensorflow/stream_executor/tpu/tpu_topology.h | 3 +++ 5 files changed, 28 insertions(+) diff --git a/tensorflow/core/tpu/tpu_library_init_fns.inc b/tensorflow/core/tpu/tpu_library_init_fns.inc index a27dbea2388..9ac4fb9ec6d 100644 --- a/tensorflow/core/tpu/tpu_library_init_fns.inc +++ b/tensorflow/core/tpu/tpu_library_init_fns.inc @@ -178,6 +178,8 @@ tensorflow::Status SetExecutorStructFn(void* library_handle) { TFTPU_SET_FN(executor_fn, TpuTopology_NumCores); TFTPU_SET_FN(executor_fn, TpuTopology_Cores); TFTPU_SET_FN(executor_fn, TpuTopology_IdForHost); + TFTPU_SET_FN(executor_fn, TpuTopology_Version); + TFTPU_SET_FN(executor_fn, TpuCoreLocation_ChipCoordinates); TFTPU_SET_FN(executor_fn, TpuCoreLocation_HostCoordinates); TFTPU_SET_FN(executor_fn, TpuCoreLocation_Index); diff --git a/tensorflow/stream_executor/tpu/c_api_decl.h b/tensorflow/stream_executor/tpu/c_api_decl.h index bca5f254ad1..a7b4c372e18 100644 --- a/tensorflow/stream_executor/tpu/c_api_decl.h +++ b/tensorflow/stream_executor/tpu/c_api_decl.h @@ -31,6 +31,12 @@ enum TpuCoreTypeEnum { kEmbeddingV2, }; +enum TpuVersionEnum { + kUnknownTpuVersion, + kTpuV2, + kTpuV3, +}; + typedef struct SE_Status SE_Status; typedef struct SE_Platform SE_Platform; diff --git a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h index c498244cc6e..149a00615a9 100644 --- a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h +++ b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h @@ -211,6 +211,7 @@ void TpuTopology_Cores(SE_TpuTopology* tpu_topology, TpuCoreTypeEnum tpu_core_type, SE_TpuTopology_Core** cores); int TpuTopology_IdForHost(SE_TpuTopology* tpu_topology, int x, int y, int z); +TpuVersionEnum TpuTopology_Version(SE_TpuTopology* tpu_topology); void TpuCoreLocation_ChipCoordinates(SE_TpuTopology_Core* tpu_core_location, int* x, int* y, int* z); void TpuCoreLocation_HostCoordinates(SE_TpuTopology_Core* tpu_core_location, @@ -367,6 +368,7 @@ struct TfTpu_ExecutorApiFn { TFTPU_ADD_FN_IN_STRUCT(TpuTopology_NumCores); TFTPU_ADD_FN_IN_STRUCT(TpuTopology_Cores); TFTPU_ADD_FN_IN_STRUCT(TpuTopology_IdForHost); + TFTPU_ADD_FN_IN_STRUCT(TpuTopology_Version); TFTPU_ADD_FN_IN_STRUCT(TpuCoreLocation_ChipCoordinates); TFTPU_ADD_FN_IN_STRUCT(TpuCoreLocation_HostCoordinates); diff --git a/tensorflow/stream_executor/tpu/tpu_topology.cc b/tensorflow/stream_executor/tpu/tpu_topology.cc index 6c885b229ec..c86b399b34e 100644 --- a/tensorflow/stream_executor/tpu/tpu_topology.cc +++ b/tensorflow/stream_executor/tpu/tpu_topology.cc @@ -95,5 +95,20 @@ int TpuTopologyExternal::IdForHost(TpuDimensionsExternal host) const { host.y, host.z); } +TpuVersionEnum TpuTopologyExternal::version() const { + return tpu::ExecutorApiFn()->TpuTopology_VersionFn(topology_); +} + +std::string TpuVersionEnumToString(TpuVersionEnum version) { + switch (version) { + case kUnknownTpuVersion: + return "Unknown TPU version"; + case kTpuV2: + return "TPU v2"; + case kTpuV3: + return "TPU v3"; + } +} + } // namespace tpu } // namespace tensorflow diff --git a/tensorflow/stream_executor/tpu/tpu_topology.h b/tensorflow/stream_executor/tpu/tpu_topology.h index 07e9afc7d81..5219ba7017b 100644 --- a/tensorflow/stream_executor/tpu/tpu_topology.h +++ b/tensorflow/stream_executor/tpu/tpu_topology.h @@ -74,11 +74,14 @@ class TpuTopologyExternal { int index) const; std::vector cores(TpuCoreTypeEnum core_type) const; int IdForHost(TpuDimensionsExternal host) const; + TpuVersionEnum version() const; private: SE_TpuTopology* topology_; }; +std::string TpuVersionEnumToString(TpuVersionEnum version); + } // namespace tpu } // namespace tensorflow From aa9d2d80f4921c9d352aaa6ab3872cc14c4028e4 Mon Sep 17 00:00:00 2001 From: Ruoxin Sang Date: Tue, 11 Aug 2020 10:03:07 -0700 Subject: [PATCH 0825/1017] Add a test for model.predict with tf.keras.layers.ConvLSTM2D layer when XLA dynamic padder is enabled. PiperOrigin-RevId: 326043413 Change-Id: I38837b69a1bb09190f0c85b31d6d26ede4963e11 --- .../custom_training_loop_models_test.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tensorflow/python/keras/distribute/custom_training_loop_models_test.py b/tensorflow/python/keras/distribute/custom_training_loop_models_test.py index 5a9384bb7e0..b680960429c 100644 --- a/tensorflow/python/keras/distribute/custom_training_loop_models_test.py +++ b/tensorflow/python/keras/distribute/custom_training_loop_models_test.py @@ -251,6 +251,33 @@ class KerasModelsTest(test.TestCase, parameterized.TestCase): train_step(input_iterator) + @combinations.generate( + combinations.combine( + distribution=strategy_combinations.all_strategies, + mode=["eager"])) + def test_model_predict_with_dynamic_batch(self, distribution): + input_data = np.random.random([1, 32, 64, 64, 3]) + input_shape = tuple(input_data.shape[1:]) + + def build_model(): + model = keras.models.Sequential() + model.add( + keras.layers.ConvLSTM2D( + 4, + kernel_size=(4, 4), + activation="sigmoid", + padding="same", + input_shape=input_shape)) + model.add(keras.layers.GlobalMaxPooling2D()) + model.add(keras.layers.Dense(2, activation="sigmoid")) + return model + + with distribution.scope(): + model = build_model() + model.compile(loss="binary_crossentropy", optimizer="adam") + result = model.predict(input_data) + self.assertEqual(result.shape, (1, 2)) + @combinations.generate( combinations.combine( distribution=strategy_combinations.all_strategies, From 0d4f0584efdfd828335b594cd68827d4ddf6cd0d Mon Sep 17 00:00:00 2001 From: Karim Nosir Date: Tue, 11 Aug 2020 10:18:01 -0700 Subject: [PATCH 0826/1017] Add cache for const nodes in hexagon delegate. On few test models this reduces the size of const nodes by half, which will reduce graph preparation time. Bug fix for sometimes wrong casting. Remove some redundant const nodes. PiperOrigin-RevId: 326046789 Change-Id: I462dd6702e0e02953c43ab47dd53589a653b3531 --- .../lite/delegates/hexagon/builders/BUILD | 1 + .../hexagon/builders/conv_2d_builder.cc | 8 +- .../hexagon/builders/conv_2d_builder.h | 2 - .../hexagon/builders/conv_2d_helpers.cc | 19 +-- .../hexagon/builders/min_max_builder.cc | 4 - .../delegates/hexagon/builders/op_builder.cc | 112 +++++++++++++++--- .../delegates/hexagon/builders/op_builder.h | 34 +++++- .../hexagon/builders/transpose_builder.cc | 10 +- .../builders/transpose_conv_2d_builder.cc | 22 +--- .../builders/transpose_conv_2d_builder.h | 2 +- .../hexagon/hexagon_delegate_kernel.cc | 5 +- 11 files changed, 152 insertions(+), 67 deletions(-) diff --git a/tensorflow/lite/delegates/hexagon/builders/BUILD b/tensorflow/lite/delegates/hexagon/builders/BUILD index 63ff274c7b7..ef4b0e957c1 100644 --- a/tensorflow/lite/delegates/hexagon/builders/BUILD +++ b/tensorflow/lite/delegates/hexagon/builders/BUILD @@ -85,6 +85,7 @@ cc_library( "//tensorflow/lite/kernels:padding", "//tensorflow/lite/kernels/internal:optimized_base", "//tensorflow/lite/kernels/internal:tensor", + "@farmhash_archive//:farmhash", "@hexagon_nn//:hexagon_nn_ops", ], ) diff --git a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc index cfddd2c2b97..c6d20004227 100644 --- a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.cc @@ -267,13 +267,13 @@ TfLiteStatus Conv2dOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, auto* conv_op = graph_builder_->AddNode(GetTFLiteNodeID()); conv_op->SetOpType(OP_DepthwiseSupernode_8x8p32to8); conv_op->AddInput(space_to_batch_op_out); - conv_op->AddInput(TensorID(weights_data_node_->GetID(), 0)); + conv_op->AddInput(graph_builder_->GetHexagonTensorId(inputs->data[1])); conv_op->AddInput(TensorID(data_min_const->GetID(), 0)); conv_op->AddInput(TensorID(data_max_const->GetID(), 0)); conv_op->AddInput(TensorID(weights_min_node_->GetID(), 0)); conv_op->AddInput(TensorID(weights_max_node_->GetID(), 0)); conv_op->AddInput(TensorID(stride_node->GetID(), 0)); - conv_op->AddInput(TensorID(bias_data_node_->GetID(), 0)); + conv_op->AddInput(graph_builder_->GetHexagonTensorId(inputs->data[2])); conv_op->AddInput(TensorID(bias_min_node_->GetID(), 0)); conv_op->AddInput(TensorID(bias_max_node_->GetID(), 0)); conv_op->AddInput(TensorID(conv_output_min_const->GetID(), 0)); @@ -330,13 +330,13 @@ TfLiteStatus Conv2dOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, } // Inputs AddInput(graph_builder_->GetHexagonTensorId(inputs->data[0])); - AddInput(TensorID(weights_data_node_->GetID(), 0)); + AddInput(graph_builder_->GetHexagonTensorId(inputs->data[1])); AddInput(TensorID(data_min_const->GetID(), 0)); AddInput(TensorID(data_max_const->GetID(), 0)); AddInput(TensorID(weights_min_node_->GetID(), 0)); AddInput(TensorID(weights_max_node_->GetID(), 0)); AddInput(TensorID(stride_node->GetID(), 0)); - AddInput(TensorID(bias_data_node_->GetID(), 0)); + AddInput(graph_builder_->GetHexagonTensorId(inputs->data[2])); AddInput(TensorID(bias_min_node_->GetID(), 0)); AddInput(TensorID(bias_max_node_->GetID(), 0)); AddInput(TensorID(conv_output_min_const->GetID(), 0)); diff --git a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h index 4980b294481..1407f06154b 100644 --- a/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h +++ b/tensorflow/lite/delegates/hexagon/builders/conv_2d_builder.h @@ -62,10 +62,8 @@ class Conv2dOpBuilder : public OpBuilder { std::vector transposed_weights_; std::vector stride_shape_; std::vector weight_shape_; - OpBuilder* weights_data_node_ = nullptr; OpBuilder* weights_min_node_ = nullptr; OpBuilder* weights_max_node_ = nullptr; - OpBuilder* bias_data_node_ = nullptr; OpBuilder* bias_min_node_ = nullptr; OpBuilder* bias_max_node_ = nullptr; diff --git a/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc b/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc index bf68bbe5a25..b33e28f4e71 100644 --- a/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc +++ b/tensorflow/lite/delegates/hexagon/builders/conv_2d_helpers.cc @@ -106,6 +106,7 @@ TfLiteStatus Conv2dOpBuilder::InitializeWeightsNodes( const bool is_per_channel_quant = weights_quant_params->scale->size > 1; // WEIGHTS DATA. + OpBuilder* weights_data_node = nullptr; if (op_node_.op_type == OP_Supernode_8x8p32to8) { // Hexagon lib expects the weight tensor in HWCN, TFLite uses NHWC. // Transpose NHWC -> HWCN @@ -137,7 +138,7 @@ TfLiteStatus Conv2dOpBuilder::InitializeWeightsNodes( weights_tensor.data.uint8, hwcn_shape, hwcn.data()); } - weights_data_node_ = graph_builder_->AddConstNodeWithData( + weights_data_node = graph_builder_->AddConstNodeWithData( weight_shape_.data(), reinterpret_cast(hwcn.data()), hwcn.size() * sizeof(hwcn[0])); } else if (op_node_.op_type == OP_DepthwiseSupernode_8x8p32to8) { @@ -156,17 +157,17 @@ TfLiteStatus Conv2dOpBuilder::InitializeWeightsNodes( for (int i = 0; i < converted_data.size(); ++i) { converted_data[i] = weights_tensor.data.int8[i] ^ k8BitSignFlipConstant; } - weights_data_node_ = graph_builder_->AddConstNodeWithData( + weights_data_node = graph_builder_->AddConstNodeWithData( weight_shape_.data(), reinterpret_cast(converted_data.data()), converted_data.size() * sizeof(converted_data[0])); } else { - weights_data_node_ = graph_builder_->AddConstNodeWithData( + weights_data_node = graph_builder_->AddConstNodeWithData( weight_shape_.data(), weights_tensor.data.raw, NumElements(&weights_tensor) * sizeof(weights_tensor.data.uint8[0])); } } - graph_builder_->AddTensorWithID(inputs->data[1], weights_data_node_->GetID(), - 0); + graph_builder_->AddTensorWithID(inputs->data[1], weights_data_node->GetID(), + 0, /*overwrite=*/true); // WEIGHTS QUANTIZATION. float weights_min = 0; @@ -229,9 +230,11 @@ TfLiteStatus Conv2dOpBuilder::ProcessPerChannelQuantizedBias( } // Add nodes for bias. const std::vector bias_shape = {1, 1, 1, bias_size}; - bias_data_node_ = graph_builder_->AddConstNodeWithData( + auto* bias_data_node = graph_builder_->AddConstNodeWithData( bias_shape.data(), reinterpret_cast(preprocessed_bias_data.data()), preprocessed_bias_data.size() * sizeof(preprocessed_bias_data[0])); + graph_builder_->AddTensorWithID(inputs->data[2], bias_data_node->GetID(), 0, + /*overwrite=*/true); return kTfLiteOk; } @@ -248,8 +251,10 @@ TfLiteStatus Conv2dOpBuilder::InitializeBiasNodes(const TfLiteIntArray* inputs, ProcessPerChannelQuantizedBias(inputs, outputs, context, &bias_min, &bias_max); } else { - bias_data_node_ = + auto* bias_data_node = graph_builder_->AddConstNodeWithData(inputs->data[2], bias_tensor); + graph_builder_->AddTensorWithID(inputs->data[2], bias_data_node->GetID(), 0, + /*overwrite=*/true); TF_LITE_ENSURE_STATUS( ComputeMinAndMaxQuantValues(bias_tensor, &bias_min, &bias_max)); } diff --git a/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc b/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc index bcfae6032c8..0c6dea2096d 100644 --- a/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/min_max_builder.cc @@ -27,10 +27,6 @@ TfLiteStatus MinMaxOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, int b_tensor_id = inputs->data[1]; const auto& a_tensor = context->tensors[a_tensor_id]; const auto& b_tensor = context->tensors[b_tensor_id]; - if (a_tensor.allocation_type == kTfLiteMmapRo) - graph_builder_->AddConstNodeWithData(a_tensor_id, a_tensor); - if (b_tensor.allocation_type == kTfLiteMmapRo) - graph_builder_->AddConstNodeWithData(b_tensor_id, b_tensor); AddInput(graph_builder_->GetHexagonTensorId(a_tensor_id)); AddInput(graph_builder_->GetHexagonTensorId(b_tensor_id)); diff --git a/tensorflow/lite/delegates/hexagon/builders/op_builder.cc b/tensorflow/lite/delegates/hexagon/builders/op_builder.cc index 0f32a4de6e1..80aa4c8155c 100644 --- a/tensorflow/lite/delegates/hexagon/builders/op_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/op_builder.cc @@ -18,10 +18,59 @@ limitations under the License. #include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/c/common.h" #include "tensorflow/lite/delegates/hexagon/builders/op_factory.h" +#include namespace tflite { namespace delegates { namespace hexagon { +namespace { +// Farmhash Fingerprint +inline uint64_t CombineFingerprints(uint64_t l, uint64_t h) { + // Murmur-inspired hashing. + const uint64_t kMul = 0x9ddfea08eb382d69ULL; + uint64_t a = (l ^ h) * kMul; + a ^= (a >> 47); + uint64_t b = (h ^ a) * kMul; + b ^= (b >> 44); + b *= kMul; + b ^= (b >> 41); + b *= kMul; + return b; +} + +inline uint64_t ComputeHash(const int shape[], const char* data, + const int data_len) { + return CombineFingerprints( + ::util::Fingerprint64(data, data_len), + ::util::Fingerprint64(reinterpret_cast(shape), + sizeof(shape[0]) * 4)); +} + +inline uint64_t ComputeHash(const TfLiteTensor& tensor, const int shape[], + int int8_to_uint8) { + auto data_hash = ComputeHash(shape, tensor.data.raw_const, tensor.bytes); + auto int8_to_uint8_hash = ::util::Fingerprint64( + reinterpret_cast(&int8_to_uint8), sizeof(int8_to_uint8)); + return CombineFingerprints(data_hash, int8_to_uint8_hash); +} + +int GetElementSize(TfLiteType type) { + switch (type) { + case kTfLiteFloat32: + return sizeof(float); + case kTfLiteBool: + return sizeof(bool); + case kTfLiteInt32: + return sizeof(int32_t); + case kTfLiteInt8: + return sizeof(int8_t); + case kTfLiteUInt8: + return sizeof(uint8_t); + default: + return sizeof(int8_t); + } +} +} // namespace OpBuilder* GraphBuilder::CreateOpBuilderFromTfLiteOp(int op_type, TfLiteNode* node) { @@ -116,8 +165,20 @@ OpBuilder* GraphBuilder::CreateOpBuilderFromTfLiteOp(int op_type, } } +OpBuilder* GraphBuilder::LookupConstData(uint64_t cache_key) { + auto lookup_result = cache_.find(cache_key); + if (lookup_result != cache_.end()) return lookup_result->second; + return nullptr; +} + +void GraphBuilder::AddToCache(uint64_t cache_key, OpBuilder* value) { + cache_[cache_key] = value; +} + OpBuilder* GraphBuilder::AddConstNodeWithData(const int shape[], char* data, int data_size) { + auto cache_key = ComputeHash(shape, data, data_size); + if (auto lookup_result = LookupConstData(cache_key)) return lookup_result; builders_.emplace_back(new OpBuilder(this, OP_Const)); builders_.back()->SetConstNode(); builders_.back()->SetNodeId(builders_.size()); @@ -125,22 +186,36 @@ OpBuilder* GraphBuilder::AddConstNodeWithData(const int shape[], char* data, graph_id_, builders_.size(), shape[0], shape[1], shape[2], shape[3], reinterpret_cast(data), data_size); if (error != 0) { - context_->ReportError(context_, "Error adding const node with shape id: %d", - (int)builders_.size()); + TF_LITE_KERNEL_LOG(context_, "Error adding const node with shape id: %d", + static_cast(builders_.size())); return nullptr; } + AddToCache(cache_key, builders_.back().get()); return builders_.back().get(); } OpBuilder* GraphBuilder::AddConstNodeWithData(int tensor_id, const TfLiteTensor& tensor, bool int8_to_uint8) { + // Fetch shape of tensor and pad 1's so it is always 4D. + int batch_size, height_size, width_size, depth_size; + GetDims(&batch_size, &height_size, &width_size, &depth_size, tensor.dims); + const int shape[] = {batch_size, height_size, width_size, depth_size}; + + auto cache_key = ComputeHash(tensor, shape, int8_to_uint8 ? 1 : 0); + if (auto lookup_result = LookupConstData(cache_key)) { + // If tensor is cached but with no id, that can happen when the same + // data is added from a constant value (not tensor). We can cache the data + // and reuse it. + // We assign the tensor to this cached const node before returning. + if (!HasTensor(tensor_id)) + AddTensorWithID(tensor_id, lookup_result->GetID(), 0); + return lookup_result; + } builders_.emplace_back(new OpBuilder(this, OP_Const)); const int node_id = builders_.size(); builders_.back()->SetConstNode(); builders_.back()->SetNodeId(node_id); - int batch_size, height_size, width_size, depth_size; - GetDims(&batch_size, &height_size, &width_size, &depth_size, tensor.dims); int error = hexagon_nn_->hexagon_nn_append_const_node( graph_id_, node_id, batch_size, height_size, width_size, depth_size, reinterpret_cast(tensor.data.raw), tensor.bytes); @@ -150,19 +225,26 @@ OpBuilder* GraphBuilder::AddConstNodeWithData(int tensor_id, return nullptr; } AddTensorWithID(tensor_id, node_id, 0); + // We need to return the builder with result, so we can't rely + // on builders_.back() as it can change while casting, so we hold pointer + // and update with value from casting if needed. + OpBuilder* result_builder = builders_.back().get(); // Cast int8 to uint8 if requested. // This will add cast op to uint8 and update tensor map to point // to the casted tensor. if (int8_to_uint8 && tensor.type == kTfLiteInt8) { - AddCastOp(context_, OP_Quantized_CastInt8ToUInt8, tensor_id); + AddCastOp(context_, OP_Quantized_CastInt8ToUInt8, tensor_id, + &result_builder); } - return builders_.back().get(); + AddToCache(cache_key, result_builder); + return result_builder; } // TODO(b/154604279): Support these casting ops in Hexagon op profiling (which // seems to key tensors on a single op, which may not be the case now). TfLiteStatus GraphBuilder::AddCastOp(TfLiteContext* context, int op_type, - int tensor_id) { + int tensor_id, + OpBuilder** cast_op_builder) { // Create a new OpBuilder for casting the tensor. OpBuilder* cast_builder = CreateCastBuilder(this, op_type); builders_.emplace_back(cast_builder); @@ -177,6 +259,7 @@ TfLiteStatus GraphBuilder::AddCastOp(TfLiteContext* context, int op_type, TF_LITE_ENSURE_STATUS(cast_builder->RegisterOutputs(tensor_data, context)); TfLiteIntArrayFree(tensor_data); + if (cast_op_builder != nullptr) *cast_op_builder = cast_builder; return kTfLiteOk; } @@ -192,12 +275,12 @@ TfLiteStatus GraphBuilder::AddInputTensors(const TfLiteIntArray* input_tensors, const int tensor_id = input_tensors->data[i]; const auto& tensor = context->tensors[tensor_id]; if (tensor.allocation_type == kTfLiteMmapRo) continue; - input_op->AddOutput(tensor.dims); + input_op->AddOutput(tensor.dims, GetElementSize(tensor.type)); AddTensorWithID(tensor_id, input_op->GetID(), num_inputs); // If tensor is of type int8, add an op to cast it to uint8. if (tensor.type == kTfLiteInt8) { - TF_LITE_ENSURE_STATUS( - AddCastOp(context, OP_Quantized_CastInt8ToUInt8, tensor_id)); + TF_LITE_ENSURE_STATUS(AddCastOp(context, OP_Quantized_CastInt8ToUInt8, + tensor_id, /*cast_op_builder=*/nullptr)); } ++num_inputs; } @@ -215,8 +298,8 @@ TfLiteStatus GraphBuilder::AddOutputTensors( const auto& tensor = context->tensors[tensor_id]; // If tensor is of type int8, add an op to cast it to uint8. if (tensor.type == kTfLiteInt8) { - TF_LITE_ENSURE_STATUS( - AddCastOp(context, OP_Quantized_CastUInt8ToInt8, tensor_id)); + TF_LITE_ENSURE_STATUS(AddCastOp(context, OP_Quantized_CastUInt8ToInt8, + tensor_id, /*cast_op_builder=*/nullptr)); } hexagon_output_ids.push_back(GetHexagonTensorId(tensor_id)); } @@ -231,9 +314,10 @@ TfLiteStatus GraphBuilder::AddOutputTensors( return kTfLiteOk; } -OpBuilder::TensorID OpBuilder::AddOutput(const TfLiteIntArray* dims) { +OpBuilder::TensorID OpBuilder::AddOutput(const TfLiteIntArray* dims, + int element_size) { op_node_.outputs.push_back(hexagon_nn_output()); - op_node_.outputs.back().elementsize = sizeof(uint8_t); + op_node_.outputs.back().elementsize = element_size; op_node_.outputs.back().rank = 4; // TODO(karimnosseir): What is a good to estimate the max size ? int batch_size, height_size, width_size, depth_size; diff --git a/tensorflow/lite/delegates/hexagon/builders/op_builder.h b/tensorflow/lite/delegates/hexagon/builders/op_builder.h index 52b130c756f..c2a2889b142 100644 --- a/tensorflow/lite/delegates/hexagon/builders/op_builder.h +++ b/tensorflow/lite/delegates/hexagon/builders/op_builder.h @@ -16,6 +16,7 @@ limitations under the License. #define TENSORFLOW_LITE_DELEGATES_HEXAGON_BUILDERS_OP_BUILDER_H_ #include +#include #include #include #include @@ -131,9 +132,9 @@ class OpBuilder { void AddInput(const TensorID& tensor_id) { input_ids_.push_back(tensor_id); } // Adds Output to the current node, the output has shape defined in 'dims'. - // This assumes the data type is uint8. + // The size of each element is defined using 'element_size'. // Returns the TensorID identifying this output in the graph. - TensorID AddOutput(const TfLiteIntArray* dims); + TensorID AddOutput(const TfLiteIntArray* dims, int element_size); // Adds Output to the current node, each element in the output has // size 'elementsize' and rank 'rank' and for each dimension in the output @@ -316,11 +317,22 @@ class GraphBuilder { bool AddTensorWithID(int tflite_tensor_id, int hexagon_node_id, int hexagon_node_output_id, bool overwrite = false) { if (!overwrite && HasTensor(tflite_tensor_id)) { + TF_LITE_KERNEL_LOG( + context_, + "Trying to add duplicate tensor without overwrite, tflite_tensor_id " + "%d, hexagon_node_id %d, hexagon_node_output_id %d", + tflite_tensor_id, hexagon_node_id, hexagon_node_output_id); return false; } if (tensors_.size() <= tflite_tensor_id) { tensors_.resize(tflite_tensor_id + 1); } + if (hexagon_node_id == -1 || hexagon_node_output_id == -1) + TF_LITE_KERNEL_LOG(context_, + "Trying to add invalid id, tflite_tensor_id " + "%d, hexagon_node_id %d, hexagon_node_output_id %d", + tflite_tensor_id, hexagon_node_id, + hexagon_node_output_id); tensors_[tflite_tensor_id] = OpBuilder::TensorID(hexagon_node_id, hexagon_node_output_id); return true; @@ -348,6 +360,14 @@ class GraphBuilder { int GetMaxBatchSize() const { return max_size_for_batch_; } private: + // Lookup in cache if data with key 'cache_key' is present. + // Return OpBuilder* for the data if found, nullptr otherwise. + OpBuilder* LookupConstData(uint64_t cache_key); + + // Inserts 'value' in cache, with key equals 'cache_key'. + // If data in cache with same key then it will be overwritten. + void AddToCache(uint64_t cache_key, OpBuilder* value); + // Helper method to fetch dimensions. // TODO(karimnosseir): Move this method to shared place. void GetDims(int* batch_size, int* height_size, int* width_size, @@ -360,7 +380,10 @@ class GraphBuilder { } // Adds a Cast op to convert a tensor from int8 to uint8 (or vice versa). - TfLiteStatus AddCastOp(TfLiteContext* context, int op_type, int tensor_id); + // The builder which has the casting operator is filled in 'cast_op_builder' + // if not nullptr. + TfLiteStatus AddCastOp(TfLiteContext* context, int op_type, int tensor_id, + OpBuilder** cast_op_builder); const HexagonNN* hexagon_nn_ = nullptr; TfLiteContext* context_ = nullptr; @@ -373,6 +396,11 @@ class GraphBuilder { // If the graph being built supports dynamic batch, this represents // the maximum value for batch. int max_size_for_batch_ = -1; + + // Cache for const data in the graph. + // Key is hash of the data, value is pointer to the OpBuilder* for the added + // data. + std::map cache_; }; } // namespace hexagon diff --git a/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc b/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc index 4a7304d011e..eb0c2668edc 100644 --- a/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/transpose_builder.cc @@ -29,15 +29,7 @@ TfLiteStatus TransposeOpBuilder::PopulateSubGraph(const TfLiteIntArray* inputs, const auto& input_tensor = context->tensors[tensor_id]; AddInput(graph_builder_->GetHexagonTensorId(tensor_id)); // permutation tensor. - tensor_id = inputs->data[1]; - const auto& control_tensor = context->tensors[tensor_id]; - if (control_tensor.allocation_type == kTfLiteMmapRo) { - auto* const_control_tensor_node = - graph_builder_->AddConstNodeWithData(tensor_id, control_tensor); - AddInput(TensorID(const_control_tensor_node->GetID(), 0)); - } else { - AddInput(graph_builder_->GetHexagonTensorId(tensor_id)); - } + AddInput(graph_builder_->GetHexagonTensorId(inputs->data[1])); TF_LITE_ENSURE_STATUS(ComputeAndAddMinAndMax(context, input_tensor)); diff --git a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc index d2620f71007..3e852533394 100644 --- a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc +++ b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.cc @@ -97,8 +97,6 @@ TfLiteStatus TransposeConv2dOpBuilder::PopulateSubGraph( filter_depth_size; GetDims(&filter_batch_size, &filter_height_size, &filter_width_size, &filter_depth_size, weights_tensor.dims); - weight_shape_ = {filter_batch_size, filter_height_size, filter_width_size, - filter_depth_size}; // Weights tensor could be int8 even for per-tensor quantization. // Therefore, we look at the number of scale values to check if it is // per-channel quantized. @@ -106,25 +104,7 @@ TfLiteStatus TransposeConv2dOpBuilder::PopulateSubGraph( reinterpret_cast( weights_tensor.quantization.params); const bool is_per_channel_quant = weights_quant_params->scale->size > 1; - - OpBuilder* const_weights_node; - if (weights_tensor.type == kTfLiteInt8) { - std::vector weights_data(NumElements(&weights_tensor)); - const int8_t* original_data = weights_tensor.data.int8; - // Flip bits on the weight values so that the int8 values are treated - // as uint8. - for (int i = 0; i < NumElements(&weights_tensor); ++i) { - weights_data[i] = original_data[i] ^ k8BitSignFlipConstant; - } - const_weights_node = graph_builder_->AddConstNodeWithData( - weight_shape_.data(), reinterpret_cast(weights_data.data()), - weights_data.size() * sizeof(weights_data[0])); - } else { - const_weights_node = graph_builder_->AddConstNodeWithData( - weight_shape_.data(), weights_tensor.data.raw, weights_tensor.bytes); - } - graph_builder_->AddTensorWithID(tensor_id, const_weights_node->GetID(), 0); - AddInput(TensorID(const_weights_node->GetID(), 0)); + AddInput(graph_builder_->GetHexagonTensorId(tensor_id)); // Handle weights quantization. float weights_min = 0; diff --git a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h index 0a6a90a0297..4afab9894f0 100644 --- a/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h +++ b/tensorflow/lite/delegates/hexagon/builders/transpose_conv_2d_builder.h @@ -47,7 +47,7 @@ class TransposeConv2dOpBuilder : public OpBuilder { TensorID node_output_; std::vector transposed_weights_; std::vector stride_shape_; - std::vector weight_shape_, bias_shape_; + std::vector bias_shape_; std::vector bias_data_; // Non-null only if node has per-channel quantized weights/biases. diff --git a/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc b/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc index cdf6b555929..83ebc15510e 100644 --- a/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc +++ b/tensorflow/lite/delegates/hexagon/hexagon_delegate_kernel.cc @@ -264,8 +264,9 @@ TfLiteStatus HexagonDelegateKernel::BuildGraph( if (tensor_id == -1) continue; const auto& input_tensor = context->tensors[tensor_id]; if (input_tensor.allocation_type == kTfLiteMmapRo) { - builder_->AddConstNodeWithData(tensor_id, input_tensor, - /*int8_to_uint8*/ true); + builder_->AddConstNodeWithData( + tensor_id, input_tensor, + /*int8_to_uint8*/ (input_tensor.type == kTfLiteInt8)); } } auto* op_builder = From a879f1c1fb59d9c741a5eef5f614343f1d794eef Mon Sep 17 00:00:00 2001 From: Rahul Joshi Date: Tue, 11 Aug 2020 10:26:24 -0700 Subject: [PATCH 0827/1017] [MLIR] Extend ResourceAliasAnalysis to handle calls and region based control flow - Extend the analysis to handle IfRegion, WhileRegion, and calls, propagating aliases through passthrough values. - Fix input -> output propagation to use AddValueUniqueIDMapping() so that both maps get correctly updated. - Fix GetResourceAliases() to add all unknown resource values to the set of aliases for any given value. - Add a new transform to annotate the IR with resource alias analysis info and use this pass to add a new unit test for resource alias analysis. PiperOrigin-RevId: 326048726 Change-Id: I92f6962dcece6ee55df236507fac20aee795157f --- tensorflow/compiler/mlir/tensorflow/BUILD | 1 + .../analysis/resource_alias_analysis.cc | 190 +++++++++----- .../analysis/resource_alias_analysis.h | 14 +- .../tests/resource-alias-analysis-test.mlir | 234 ++++++++++++++++++ .../test_resource_alias_analysis.cc | 111 +++++++++ 5 files changed, 483 insertions(+), 67 deletions(-) create mode 100644 tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir create mode 100644 tensorflow/compiler/mlir/tensorflow/transforms/test_resource_alias_analysis.cc diff --git a/tensorflow/compiler/mlir/tensorflow/BUILD b/tensorflow/compiler/mlir/tensorflow/BUILD index 3ee591ce46a..d2e57f72774 100644 --- a/tensorflow/compiler/mlir/tensorflow/BUILD +++ b/tensorflow/compiler/mlir/tensorflow/BUILD @@ -776,6 +776,7 @@ cc_library( "transforms/stack_ops_decomposition.cc", "transforms/tensor_array_ops_decomposition.cc", "transforms/tensor_list_ops_decomposition.cc", + "transforms/test_resource_alias_analysis.cc", "transforms/test_side_effect_analysis.cc", "transforms/tf_data_optimization_pass.cc", "transforms/tf_device_assignment.cc", diff --git a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc index 3278c06fabe..53de595eef2 100644 --- a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc +++ b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc @@ -34,12 +34,12 @@ limitations under the License. #include "mlir/IR/Operation.h" // from @llvm-project #include "mlir/IR/StandardTypes.h" // from @llvm-project #include "mlir/IR/Value.h" // from @llvm-project +#include "mlir/IR/Visitors.h" // from @llvm-project #include "mlir/Support/LLVM.h" // from @llvm-project #include "mlir/Support/LogicalResult.h" // from @llvm-project #include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" #include "tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h" #include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" -#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" #include "tensorflow/compiler/tf2xla/resource_operation_table.h" #include "tensorflow/core/framework/resource_mgr.h" @@ -161,7 +161,7 @@ Value BacktrackAnalysis::BacktrackValue(Value value) { // in the Island body. if (value == island.control()) break; value = island.GetYield().getOperand(res_index); - } else if (isa(op)) { + } else if (isa(op)) { value = op->getOperand(res_index); } else { break; @@ -196,12 +196,12 @@ constexpr char kResourceArgUniqueIdAttr[] = "tf._resource_arg_unique_id"; // Returns if a VarHandleOp is anonymous, which means it always creates a new // variable. -bool IsResourceHandleAnonymous(TF::VarHandleOp handle) { +bool IsResourceHandleAnonymous(VarHandleOp handle) { return handle.shared_name() == tensorflow::ResourceHandle::ANONYMOUS_NAME; } // Returns a string unique identifier for a non-anonymous VarHandleOp. -std::string GetVarHandleStringId(TF::VarHandleOp handle) { +std::string GetVarHandleStringId(VarHandleOp handle) { auto device = handle.getAttrOfType("device"); return absl::StrCat(handle.container().str(), "/", handle.shared_name().str(), "/", device ? device.getValue().str() : std::string("")); @@ -210,7 +210,7 @@ std::string GetVarHandleStringId(TF::VarHandleOp handle) { // Finds a unique ID for a VarHandleOp's output. If it is anonymous, always // creates a new ID; otherwise, tries to reuse the existing ID for the // referenced variable if it exists, or creates a new one if not. -int64_t GetOrCreateIdForVarHandle(TF::VarHandleOp handle, int64_t* next_id, +int64_t GetOrCreateIdForVarHandle(VarHandleOp handle, int64_t* next_id, llvm::StringMap* name_id_map) { // Always create a new ID for anonymous handle. if (IsResourceHandleAnonymous(handle)) return (*next_id)++; @@ -234,121 +234,173 @@ ResourceAliasAnalysisInfo::ResourceAliasAnalysisInfo( FuncOp func_op, const detail::BacktrackAnalysis& backtrack_analysis) { // This function populates resource_value_to_ids_ and id_to_resource_values_. + int64_t next_unique_id = 0; + + // Helper to assign new unique id for all resources in the given list of + // values. + auto assign_unique_id_to_all = [&](ValueRange values) { + for (Value value : filter_resources(values)) { + AddValueUniqueIDMapping(value, next_unique_id++); + } + }; + + // Helper to assign new unknown id for all resources in the given list of + // values. + auto assign_unknown_id_to_all = [&](ValueRange values) { + for (Value value : filter_resources(values)) { + AddValueUniqueIDMapping(value, kUnknownResourceId); + } + }; + // If the "tf.resource_arg_unique_id" argument attributes are present for // resource-type arguments, respect them when choosing IDs; otherwise, they // must not alias. - int64_t next_unique_id = 0; const bool has_arg_unique_id_attrs = llvm::any_of(func_op.getArguments(), [&](const BlockArgument& arg) { return func_op.getArgAttr(arg.getArgNumber(), kResourceArgUniqueIdAttr); }); // Maps the kResourceArgUniqueIdAttr attribute value to the internal integer // ID used by this pass. - llvm::SmallDenseMap attr_id_to_internal_id; - for (auto arg : func_op.getArguments()) { - if (!mlir::getElementTypeOrSelf(arg.getType()).isa()) - continue; - if (has_arg_unique_id_attrs) { + if (has_arg_unique_id_attrs) { + llvm::SmallDenseMap attr_id_to_internal_id; + for (auto arg : filter_resources(func_op.getArguments())) { auto id_attr = func_op.getArgAttrOfType( arg.getArgNumber(), kResourceArgUniqueIdAttr); assert(id_attr && - "tf.resource_arg_unique_id attribute should exist on either none " - "or all arguments."); + "tf.resource_arg_unique_id attribute should exist on either " + "none or all arguments."); auto emplace_res = attr_id_to_internal_id.try_emplace(id_attr.getInt(), next_unique_id++); AddValueUniqueIDMapping(arg, emplace_res.first->getSecond()); - } else { - AddValueUniqueIDMapping(arg, next_unique_id++); } + } else { + assign_unique_id_to_all(func_op.getArguments()); } + + // Since this analysis is neither inter-procedural nor inter-regional, + // each region attached to Op's within a function is analyzed independently. + // Seed this analysis for each such region by mapping all resource arguments + // for such regions to a new unique-id. This is required because walk() walks + // the attached regions first before visiting the op, so there is no + // opportunity during the walk to seed region arguments. Also note that walk + // eventually also visits the Op on which the walk() is called, so make sure + // we do not overwrite the function argument mapping here. + func_op.walk([&](Operation* op) { + if (op == func_op) return; + for (Region& region : op->getRegions()) { + assign_unique_id_to_all(region.getArguments()); + } + }); + llvm::StringMap var_handle_name_id_map; auto forward_input_to_output = [&](const Value& operand, - const Value& result) { - if (!mlir::getElementTypeOrSelf(result.getType()).isa()) - return; - auto& result_ids = resource_value_to_ids_[result]; + const OpResult& result) { auto operand_it = resource_value_to_ids_.find(operand); assert(operand_it != resource_value_to_ids_.end() && "A resource-type output does not have the corresponding " "resource-type input."); - result_ids.insert(operand_it->getSecond().begin(), - operand_it->getSecond().end()); + for (int64_t id : operand_it->second) AddValueUniqueIDMapping(result, id); }; func_op.walk([&](Operation* op) { - if (auto var_handle = llvm::dyn_cast(op)) { + if (auto var_handle = dyn_cast(op)) { AddValueUniqueIDMapping( var_handle.resource(), GetOrCreateIdForVarHandle(var_handle, &next_unique_id, &var_handle_name_id_map)); - } else if (llvm::isa(op)) { - for (auto operand_and_result : - llvm::zip(op->getOperands(), op->getResults())) { - forward_input_to_output(std::get<0>(operand_and_result), - std::get<1>(operand_and_result)); - } - } else if (auto replicate = llvm::dyn_cast(op)) { - // The nested block for ReplicateOp is handled separately in side-effect - // analysis. Inside that block, we can still treat its block arguments as - // different resources. - for (auto arg : replicate.GetBody().getArguments()) { - if (mlir::getElementTypeOrSelf(arg.getType()).isa()) { - AddValueUniqueIDMapping(arg, next_unique_id++); - } - } - } else if (auto while_op = llvm::dyn_cast(op)) { + } else if (llvm::isa(op)) { + for (auto result : filter_resources(op->getResults())) + forward_input_to_output(op->getOperand(result.getResultNumber()), + result); + } else if (auto while_op = dyn_cast(op)) { const auto& body_info = backtrack_analysis.GetAnalysisForFunc(while_op.body_func()); // If a result is a passthrough of the body input, use the corresponding // operand's resource IDs. - for (auto result : llvm::enumerate(while_op.getResults())) { - if (!mlir::getElementTypeOrSelf(result.value().getType()) - .isa()) { - continue; - } - auto passthrough_arg = body_info.GetArg(result.index()); + for (auto result : filter_resources(while_op.getResults())) { + auto passthrough_arg = body_info.GetArg(result.getResultNumber()); if (passthrough_arg) { forward_input_to_output( - while_op.getOperand(passthrough_arg.getValue()), result.value()); + while_op.getOperand(passthrough_arg.getValue()), result); } else { - AddValueUniqueIDMapping(result.value(), kUnknownResourceId); + AddValueUniqueIDMapping(result, kUnknownResourceId); } } - } else if (auto if_op = llvm::dyn_cast(op)) { + } else if (auto while_region = dyn_cast(op)) { + const auto& body_info = + backtrack_analysis.GetAnalysisForRegion(while_region.body()); + // If a result is a passthrough of the body input, use the corresponding + // operand's resource IDs. + for (auto result : filter_resources(while_region.getResults())) { + auto passthrough_arg = body_info.GetArg(result.getResultNumber()); + if (passthrough_arg) { + forward_input_to_output( + while_region.getOperand(passthrough_arg.getValue()), result); + } else { + AddValueUniqueIDMapping(result, kUnknownResourceId); + } + } + } else if (auto if_op = dyn_cast(op)) { const auto& then_info = backtrack_analysis.GetAnalysisForFunc(if_op.then_func()); const auto& else_info = backtrack_analysis.GetAnalysisForFunc(if_op.else_func()); // If a result is a passthrough of both branches' inputs, merge the // resource IDs of corresponding operands for the two inputs. - for (auto result : llvm::enumerate(if_op.getResults())) { - if (!mlir::getElementTypeOrSelf(result.value().getType()) - .isa()) { - continue; - } - auto passthrough_then_arg = then_info.GetArg(result.index()); - auto passthrough_else_arg = else_info.GetArg(result.index()); + for (auto result : filter_resources(if_op.getResults())) { + auto passthrough_then_arg = then_info.GetArg(result.getResultNumber()); + auto passthrough_else_arg = else_info.GetArg(result.getResultNumber()); if (passthrough_then_arg && passthrough_else_arg) { Value then_operand = if_op.input()[passthrough_then_arg.getValue()]; Value else_operand = if_op.input()[passthrough_else_arg.getValue()]; - forward_input_to_output(then_operand, result.value()); - forward_input_to_output(else_operand, result.value()); + forward_input_to_output(then_operand, result); + forward_input_to_output(else_operand, result); } else { - AddValueUniqueIDMapping(result.value(), kUnknownResourceId); + AddValueUniqueIDMapping(result, kUnknownResourceId); + } + } + } else if (auto if_region = dyn_cast(op)) { + const auto& then_info = + backtrack_analysis.GetAnalysisForRegion(if_region.then_branch()); + const auto& else_info = + backtrack_analysis.GetAnalysisForRegion(if_region.else_branch()); + for (auto result : filter_resources(if_region.getResults())) { + Value then_result = then_info.GetValue(result.getResultNumber()); + Value else_result = else_info.GetValue(result.getResultNumber()); + // For IfRegion, the walk would have visited the else and then regions + // before visiting the IfRegion op. Backtracking of the then and else + // results will either give a value computed within these regions, + // or a region capture. If its a region capture, computed before this + // IfRegion, it will have been visited earlier and a mapping would + // exist for that value. If its computed within the region, then again + // a mapping would exist. + forward_input_to_output(then_result, result); + forward_input_to_output(else_result, result); + } + } else if (auto call = dyn_cast(op)) { + FuncOp func = dyn_cast(call.resolveCallable()); + if (!func) { + assign_unknown_id_to_all(op->getResults()); + return WalkResult::advance(); + } + const auto& func_info = backtrack_analysis.GetAnalysisForFunc(func); + for (auto result : filter_resources(op->getResults())) { + auto passthrough_arg = func_info.GetArg(result.getResultNumber()); + if (passthrough_arg) { + forward_input_to_output( + call.getArgOperands()[passthrough_arg.getValue()], result); + } else { + AddValueUniqueIDMapping(result, kUnknownResourceId); } } } else { - for (auto result : op->getResults()) { - if (!mlir::getElementTypeOrSelf(result.getType()) - .isa()) - continue; - AddValueUniqueIDMapping(result, kUnknownResourceId); - } + assign_unknown_id_to_all(op->getResults()); } + return WalkResult::advance(); }); } -bool ResourceAliasAnalysisInfo::IsUnknownResource(const Value resource) const { +bool ResourceAliasAnalysisInfo::IsUnknownResource(Value resource) const { auto it = resource_value_to_ids_.find(resource); assert(it != resource_value_to_ids_.end() && !it->getSecond().empty()); // The set is sorted so we only need to check the first element since @@ -360,6 +412,7 @@ bool ResourceAliasAnalysisInfo::IsUnknownResource(const Value resource) const { const llvm::SmallSet& ResourceAliasAnalysisInfo::GetResourceUniqueIds(Value resource) const { + assert(!IsUnknownResource(resource)); auto it = resource_value_to_ids_.find(resource); assert(it != resource_value_to_ids_.end() && "Unseen resource was queried"); return it->getSecond(); @@ -373,14 +426,19 @@ ResourceAliasAnalysisInfo::GetUniqueIdResources(const int64_t id) const { } llvm::SmallSetVector ResourceAliasAnalysisInfo::GetResourceAliases( - const Value resource) const { - assert(!IsUnknownResource(resource) && "Unseen resource was queried"); + Value resource) const { + assert(!IsUnknownResource(resource) && "Unknown resource was queried"); llvm::SmallSetVector aliases; for (int64_t id : GetResourceUniqueIds(resource)) { const llvm::SmallSetVector& resources_aliasing_id = GetUniqueIdResources(id); aliases.insert(resources_aliasing_id.begin(), resources_aliasing_id.end()); } + // If there are resources that were marked as unknown, they alias with all + // other resources. + auto it = id_to_resource_values_.find(kUnknownResourceId); + if (it != id_to_resource_values_.end()) + aliases.insert(it->getSecond().begin(), it->getSecond().end()); return aliases; } diff --git a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h index 5a514a7fb64..d9fd693042f 100644 --- a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h +++ b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h @@ -20,13 +20,16 @@ limitations under the License. #include #include +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "mlir/IR/Operation.h" // from @llvm-project #include "mlir/IR/Region.h" // from @llvm-project +#include "mlir/IR/TypeUtilities.h" // from @llvm-project #include "tensorflow/compiler/mlir/tensorflow/analysis/per_function_aggregate_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_types.h" namespace mlir { namespace TF { @@ -43,7 +46,7 @@ class ResourceAliasAnalysisInfo { ResourceAliasAnalysisInfo(ResourceAliasAnalysisInfo&&) = default; // Returns if the analysis fails to resolve a resource-type value. - bool IsUnknownResource(const Value resource) const; + bool IsUnknownResource(Value resource) const; // Returns the set unique IDs which `resource` could alias. Requires that // IsUnknownResource(resource) == false. @@ -91,6 +94,15 @@ class ResourceAliasAnalysis : public detail::PerFunctionAggregateAnalysis< explicit ResourceAliasAnalysis(Operation* op); }; +// Returns a range with just resource type values from the input range +// preserved. +template +auto filter_resources(RangeT&& range) { + return llvm::make_filter_range(std::forward(range), [](Value val) { + return getElementTypeOrSelf(val.getType()).isa(); + }); +} + } // namespace TF } // namespace mlir diff --git a/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir b/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir new file mode 100644 index 00000000000..af63f3312bc --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir @@ -0,0 +1,234 @@ +// RUN: tf-opt -split-input-file -tf-test-resource-alias-analysis -verify-diagnostics %s | FileCheck %s + +// Test 2 resources that do not alias. + +!tf_res = type tensor<*x!tf.resource>> +// CHECK-LABEL: func @non_aliasing_reads_writes +// expected-remark@below {{Region #0, Arg #0, ID 1 : 1}} +// expected-remark@below {{Region #0, Arg #1, ID 2 : 2}} +func @non_aliasing_reads_writes( + %arg0: !tf_res, + %arg1: !tf_res, + %arg2: tensor<32xf32>) -> (tensor<32xf32>) { + %graph = tf_executor.graph { + // CHECK: tf_executor.island + %island:2 = tf_executor.island { + %read0 = "tf.ReadVariableOp"(%arg0) : (!tf_res) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %arg2) : (!tf_res, tensor<32xf32>) -> () + %read1 = "tf.ReadVariableOp"(%arg1) : (!tf_res) -> tensor<32xf32> + // expected-remark@below {{Result #0, ID 0 : 0}} + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + %read2 = "tf.ReadVariableOp"(%var_handle) : (!tf_res) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg1, %read0) : (!tf_res, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg0, %read2) : (!tf_res, tensor<32xf32>) -> () + %read3 = "tf.ReadVariableOp"(%arg0) : (!tf_res) -> tensor<32xf32> + tf_executor.yield %read3 : tensor<32xf32> + } + tf_executor.fetch %island#0 : tensor<32xf32> + } + return %graph : tensor<32xf32> +} + +// ----- +// Tests aliasing of the two resource handles that refer to the same variable. + +!tf_res = type tensor<*x!tf.resource>> +// CHECK-LABEL: func @aliasing_reads_writes +func @aliasing_reads_writes(%arg0: tensor<32xf32>) -> () { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 2}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 0, 1, 2}} + %vh1 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 2 : 0, 1, 2}} + %vh1_id:2 = "tf.IdentityN"(%vh1, %arg0) : (!tf_res, tensor<32xf32>) -> (!tf_res, tensor<32xf32>) + %read0 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor<32xf32> + "tf.AssignVariableOp"(%vh1_id#0, %arg0) : (!tf_res, tensor<32xf32>) -> () + %read1 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor<32xf32> + %read2 = "tf.ReadVariableOp"(%vh1) : (!tf_res) -> tensor<32xf32> + "tf.AssignVariableOp"(%vh0, %read2) : (!tf_res, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%vh1_id#0, %read1) : (!tf_res, tensor<32xf32>) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + return +} + +// ----- +// Test an unknown op that has a resource result is marked unknown + +!tf_res = type tensor<*x!tf.resource>> +// CHECK-LABEL: func @unknown_resource_op +func @unknown_resource_op(%arg0: tensor<32xf32>) -> () { + // expected-remark@below {{Result #0, ID 0 : Unknown}} + %0 = "tf.UnknownVarHandleOp"() : () -> !tf_res +} + +// ----- +// Test aliasing through IfOp + +!tf_res = type tensor<*x!tf.resource>> + +// CHECK-LABEL: func @if_op_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 4 : 1, 4}} +// expected-remark@below {{Region #0, Arg #1, ID 5 : 1, 2, 3, 5}} +func @if_op_aliasing(%arg0: !tf_res, %arg1: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + %read0 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor<32xf32> + // expected-remark@below {{Result #0, ID 1 : Unknown}} + // expected-remark@below {{Result #1, ID 2 : 1, 2, 3, 5}} + // expected-remark@below {{Result #2, ID 3 : 0, 1, 2, 3, 5}} + %if:3 = "tf.If"(%read0, %arg1, %vh0) { + then_branch = @if_then, else_branch = @if_else, is_stateless = true + } : (tensor<32xf32>, !tf_res, !tf_res) -> (!tf_res, !tf_res, !tf_res) + return +} + +// expected-remark@below {{Region #0, Arg #0, ID 2 : 0, 1, 2}} +// expected-remark@below {{Region #0, Arg #1, ID 3 : 0, 3}} +func @if_then(%arg0: !tf_res, %arg1: !tf_res) -> (!tf_res, !tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 0, 1, 2}} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + return %u0, %id0, %id0 : !tf_res, !tf_res, !tf_res +} + +// expected-remark@below {{Region #0, Arg #0, ID 1 : 0, 1}} +// expected-remark@below {{Region #0, Arg #1, ID 2 : 2}} +func @if_else(%arg0: !tf_res, %arg1: !tf_res) -> (!tf_res, !tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0, 1}} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + return %id0, %id0, %arg1 : !tf_res, !tf_res, !tf_res +} + +// ----- +// Test aliasing through WhileOp +!tf_res = type tensor<*x!tf.resource>> + +// CHECK-LABEL: func @while_op_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 4 : 1, 4}} +// expected-remark@below {{Region #0, Arg #1, ID 5 : 1, 3, 5}} +// expected-remark@below {{Region #0, Arg #2, ID 6 : 1, 2, 6}} +func @while_op_aliasing(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : Unknown}} + // expected-remark@below {{Result #1, ID 2 : 1, 2, 6}} + // expected-remark@below {{Result #2, ID 3 : 1, 3, 5}} + %w:3 = "tf.While"(%arg0, %arg1, %arg2) { + body = @while_body, cond = @while_cond, is_stateless = false + } : (!tf_res, !tf_res, !tf_res) -> (!tf_res, !tf_res, !tf_res) + return +} + +// CHECK-LABEL: func @while_body +// Return 0 : new unknown resource +// Return 1 : arg2 +// Return 2 : arg1 +// expected-remark@below {{Region #0, Arg #0, ID 1 : 0, 1}} +// expected-remark@below {{Region #0, Arg #1, ID 2 : 0, 2}} +// expected-remark@below {{Region #0, Arg #2, ID 3 : 0, 3}} +func @while_body(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) -> (!tf_res, !tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + return %u0, %arg2, %arg1 : !tf_res, !tf_res, !tf_res +} + +// CHECK-LABEL: func @while_cond +// expected-remark@below {{Region #0, Arg #0, ID 0 : 0}} +// expected-remark@below {{Region #0, Arg #1, ID 1 : 1}} +// expected-remark@below {{Region #0, Arg #2, ID 2 : 2}} +func @while_cond(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) -> tensor { + %0 = constant dense : tensor + return %0 : tensor +} + +// ----- +// Test alias propagation through calls. +!tf_res = type tensor<*x!tf.resource>> +// CHECK-LABEL: func @aliasing_through_calls +func @aliasing_through_calls(%arg0: tensor<32xf32>) -> () { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 2, 3}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 1 : 0, 1, 2, 3}} + %vh1 = "tf.Identity"(%vh0) : (!tf_res) -> (!tf_res) + // expected-remark@below {{Result #0, ID 2 : Unknown}} + // expected-remark@below {{Result #1, ID 3 : 0, 1, 2, 3}} + %c:2 = call @passthru(%vh1) : (!tf_res) -> (!tf_res, !tf_res) + return +} + +// expected-remark@below {{Region #0, Arg #0, ID 1 : 1}} +func @passthru(%arg0: !tf_res) -> (!tf_res, !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0}} + %vx = "tf.VarHandleOp"() {container = "cf", shared_name = "vx"} : () -> !tf_res + return %vx, %arg0 : !tf_res, !tf_res +} + +// ----- +// Test aliasing through IfRegion + +!tf_res = type tensor<*x!tf.resource>> + +// CHECK-LABEL: func @if_region_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 7 : 1, 4, 6, 7}} +// expected-remark@below {{Region #0, Arg #1, ID 8 : 1, 2, 4, 5, 6, 8}} +func @if_region_aliasing(%arg0: !tf_res, %arg1: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 3, 4, 5}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + %read0 = "tf.ReadVariableOp"(%vh0) : (!tf_res) -> tensor<32xf32> + // expected-remark@below {{Result #0, ID 4 : Unknown}} + // expected-remark@below {{Result #1, ID 5 : 0, 1, 2, 3, 4, 5, 6, 8}} + // expected-remark@below {{Result #2, ID 6 : 1, 2, 4, 5, 6, 7, 8}} + %if:3 = "tf.IfRegion"(%read0) ({ + // expected-remark@below {{Result #0, ID 1 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + // expected-remark@below {{Result #0, ID 2 : 1, 2, 4, 5, 6, 8}} + %id0 = "tf.Identity"(%arg1) : (!tf_res) -> !tf_res + "tf.Yield"(%u0, %id0, %id0) : (!tf_res, !tf_res, !tf_res) -> () + }, { + // expected-remark@below {{Result #0, ID 3 : 0, 1, 3, 4, 5}} + %id0 = "tf.Identity"(%vh0) : (!tf_res) -> !tf_res + "tf.Yield"(%id0, %id0, %arg0) : (!tf_res, !tf_res, !tf_res) -> () + }) {is_stateless = true} : (tensor<32xf32>) -> (!tf_res, !tf_res, !tf_res) + return +} + +// ----- +// Test aliasing through WhileRegion +!tf_res = type tensor<*x!tf.resource>> + +// CHECK-LABEL: func @while_region_aliasing +// expected-remark@below {{Region #0, Arg #0, ID 11 : 1, 8, 11}} +// expected-remark@below {{Region #0, Arg #1, ID 12 : 1, 8, 10, 12}} +// expected-remark@below {{Region #0, Arg #2, ID 13 : 1, 8, 9, 13}} +func @while_region_aliasing(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) { + // expected-remark@below {{Result #0, ID 0 : 0, 1, 8}} + %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res + // expected-remark@below {{Result #0, ID 8 : Unknown}} + // expected-remark@below {{Result #1, ID 9 : 1, 8, 9, 13}} + // expected-remark@below {{Result #2, ID 10 : 1, 8, 10, 12}} + // expected-remark@below {{Region #0, Arg #0, ID 2 : 1, 2, 8}} + // expected-remark@below {{Region #0, Arg #1, ID 3 : 1, 3, 8}} + // expected-remark@below {{Region #0, Arg #2, ID 4 : 1, 4, 8}} + // expected-remark@below {{Region #1, Arg #0, ID 5 : 1, 5, 8}} + // expected-remark@below {{Region #1, Arg #1, ID 6 : 1, 6, 8}} + // expected-remark@below {{Region #1, Arg #2, ID 7 : 1, 7, 8}} + %w:3 = "tf.WhileRegion"(%arg0, %arg1, %arg2) ({ + ^bb0(%carg0: !tf_res, %carg1: !tf_res, %carg2: !tf_res): + %0 = constant dense : tensor + "tf.Yield"(%0) : (tensor) -> () + },{ + ^bb0(%barg0: !tf_res, %barg1: !tf_res, %barg2: !tf_res): + // expected-remark@below {{Result #0, ID 1 : Unknown}} + %u0 = "tf._UnknownSideEffectingOp_"() : () -> !tf_res + "tf.Yield"(%u0, %barg2, %barg1) : (!tf_res, !tf_res, !tf_res) -> () + }) {is_stateless = false} : (!tf_res, !tf_res, !tf_res) -> (!tf_res, !tf_res, !tf_res) + return +} + diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/test_resource_alias_analysis.cc b/tensorflow/compiler/mlir/tensorflow/transforms/test_resource_alias_analysis.cc new file mode 100644 index 00000000000..920b2024c0f --- /dev/null +++ b/tensorflow/compiler/mlir/tensorflow/transforms/test_resource_alias_analysis.cc @@ -0,0 +1,111 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/Debug.h" +#include "mlir/Pass/Pass.h" // from @llvm-project +#include "mlir/Pass/PassManager.h" // from @llvm-project +#include "mlir/Support/LLVM.h" // from @llvm-project +#include "mlir/Transforms/Passes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h" +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" + +namespace mlir { +namespace TF { +namespace { + +// A pass that annotates each operation with a resource type result with the +// aliasing values for each such result. Each value is assigned a unique ID, and +// that ID is used to annotate the operations. +struct TestResourceAliasAnalysis + : public TF::PerFunctionAggregateAnalysisConsumerPass< + TestResourceAliasAnalysis, TF::ResourceAliasAnalysis> { + void runOnFunction(FuncOp func, + const TF::ResourceAliasAnalysis::Info& analysis) { + int64_t next_id = 0; + llvm::SmallDenseMap ids; + + auto assign_id = [&](Value value) { + if (ids.find(value) == ids.end()) ids.insert({value, next_id++}); + }; + + auto get_id = [&](Value value) -> int64_t { + auto it = ids.find(value); + assert(it != ids.end()); + return it->second; + }; + + auto print_aliases = [&](InFlightDiagnostic& diag, Value value) { + diag << ", ID " << get_id(value) << " : "; + if (analysis.IsUnknownResource(value)) { + diag << "Unknown"; + } else { + auto aliases = llvm::to_vector<4>(analysis.GetResourceAliases(value)); + llvm::sort(aliases, + [&](Value v1, Value v2) { return get_id(v1) < get_id(v2); }); + llvm::interleaveComma(aliases, diag, + [&](Value v) { diag << get_id(v); }); + } + }; + + // Assign a unique ID to each value seen in this function. + func.walk([&](Operation* op) { + // For all attached regions, assign ID to the region arguments. + for (Region& region : op->getRegions()) { + for (auto region_arg : filter_resources(region.getArguments())) + assign_id(region_arg); + } + + // Assign ID for all results. + for (auto result : filter_resources(op->getResults())) assign_id(result); + }); + + // Now walk each operation, and annotate it wil remarks for aliases for + // each resource type result + func.walk([&](Operation* op) { + // For all attached regions, assign ID to the region arguments. + for (Region& region : op->getRegions()) { + for (auto region_arg : filter_resources(region.getArguments())) { + InFlightDiagnostic diag = op->emitRemark("Region #") + << region.getRegionNumber() << ", Arg #" + << region_arg.getArgNumber(); + print_aliases(diag, region_arg); + } + } + + for (auto result : filter_resources(op->getResults())) { + InFlightDiagnostic diag = op->emitRemark("Result #") + << result.getResultNumber(); + print_aliases(diag, result); + } + }); + } +}; + +static mlir::PassRegistration pass( + "tf-test-resource-alias-analysis", + "Add remarks based on resource alias analysis result, for testing " + "purpose."); + +} // anonymous namespace +} // namespace TF +} // namespace mlir From e8598ce0454c440fca64e4ebc4aeedfa7afd5c97 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 10:57:58 -0700 Subject: [PATCH 0828/1017] Break up tensorflow/core/kernels/BUILD (part 3 of N): Move image processing kernels to subdirectory tensorflow/core/kernels/image with its own BUILD file. PiperOrigin-RevId: 326056277 Change-Id: Ibe8813eeb36432c6220bdd14de40898fdef195f2 --- tensorflow/core/BUILD | 4 +- tensorflow/core/kernels/BUILD | 679 +++--------------- .../kernels/conv_ops_fused_image_transform.cc | 2 +- .../core/kernels/conv_ops_using_gemm.cc | 2 +- tensorflow/core/kernels/image/BUILD | 449 ++++++++++++ .../kernels/{ => image}/adjust_contrast_op.cc | 4 +- .../kernels/{ => image}/adjust_contrast_op.h | 6 +- .../adjust_contrast_op_benchmark_test.cc | 0 .../{ => image}/adjust_contrast_op_gpu.cu.cc | 3 +- .../{ => image}/adjust_contrast_op_test.cc | 0 .../kernels/{ => image}/adjust_hsv_gpu.cu.h | 6 +- .../core/kernels/{ => image}/adjust_hue_op.cc | 4 +- .../core/kernels/{ => image}/adjust_hue_op.h | 6 +- .../{ => image}/adjust_hue_op_gpu.cu.cc | 4 +- .../{ => image}/adjust_saturation_op.cc | 4 +- .../{ => image}/adjust_saturation_op.h | 6 +- .../adjust_saturation_op_gpu.cu.cc | 4 +- .../core/kernels/{ => image}/attention_ops.cc | 0 .../core/kernels/{ => image}/colorspace_op.cc | 3 +- .../core/kernels/{ => image}/colorspace_op.h | 6 +- .../{ => image}/colorspace_op_gpu.cu.cc | 2 +- .../kernels/{ => image}/colorspace_op_test.cc | 0 .../kernels/{ => image}/crop_and_resize_op.cc | 2 +- .../kernels/{ => image}/crop_and_resize_op.h | 6 +- .../crop_and_resize_op_benchmark_test.cc | 0 .../{ => image}/crop_and_resize_op_gpu.cu.cc | 2 +- .../{ => image}/crop_and_resize_op_test.cc | 0 .../kernels/{ => image}/decode_image_op.cc | 0 .../{ => image}/draw_bounding_box_op.cc | 0 .../kernels/{ => image}/encode_jpeg_op.cc | 0 .../{ => image}/encode_jpeg_op_test.cc | 0 .../core/kernels/{ => image}/encode_png_op.cc | 0 .../{ => image}/extract_image_patches_op.cc | 2 +- .../{ => image}/extract_image_patches_op.h | 6 +- .../extract_image_patches_op_gpu.cu.cc | 2 +- .../{ => image}/extract_jpeg_shape_op.cc | 0 .../{ => image}/extract_volume_patches_op.cc | 2 +- .../{ => image}/extract_volume_patches_op.h | 6 +- .../extract_volume_patches_op_gpu.cu.cc | 2 +- .../generate_box_proposals_op.cu.cc | 2 +- .../core/kernels/{ => image}/image_ops.cc | 2 +- .../core/kernels/{ => image}/image_ops.h | 6 +- .../kernels/{ => image}/image_ops_gpu.cu.cc | 2 +- .../core/kernels/{ => image}/mirror_pad_op.cc | 4 +- .../core/kernels/{ => image}/mirror_pad_op.h | 6 +- .../mirror_pad_op_benchmark_test.cc | 0 .../{ => image}/mirror_pad_op_cpu_impl.h | 11 +- .../{ => image}/mirror_pad_op_cpu_impl_1.cc | 2 +- .../{ => image}/mirror_pad_op_cpu_impl_2.cc | 2 +- .../{ => image}/mirror_pad_op_cpu_impl_3.cc | 2 +- .../{ => image}/mirror_pad_op_cpu_impl_4.cc | 2 +- .../{ => image}/mirror_pad_op_cpu_impl_5.cc | 2 +- .../{ => image}/mirror_pad_op_gpu.cu.cc | 3 +- .../kernels/{ => image}/mirror_pad_op_test.cc | 0 .../{ => image}/non_max_suppression_op.cc | 2 +- .../{ => image}/non_max_suppression_op.cu.cc | 2 +- .../{ => image}/non_max_suppression_op.h | 6 +- .../non_max_suppression_op_benchmark_test.cc | 0 .../non_max_suppression_op_gpu_test.cc | 0 .../non_max_suppression_op_test.cc | 0 .../kernels/{ => image}/random_crop_op.cc | 1 + .../kernels/{ => image}/resize_area_op.cc | 3 +- .../{ => image}/resize_area_op_test.cc | 0 .../kernels/{ => image}/resize_bicubic_op.cc | 3 +- .../{ => image}/resize_bicubic_op_test.cc | 0 .../kernels/{ => image}/resize_bilinear_op.cc | 5 +- .../kernels/{ => image}/resize_bilinear_op.h | 6 +- .../{ => image}/resize_bilinear_op_gpu.cu.cc | 2 +- .../{ => image}/resize_bilinear_op_test.cc | 0 .../{ => image}/resize_nearest_neighbor_op.cc | 4 +- .../{ => image}/resize_nearest_neighbor_op.h | 6 +- .../resize_nearest_neighbor_op_gpu.cu.cc | 2 +- .../resize_nearest_neighbor_op_test.cc | 0 .../{ => image}/resize_op_benchmark_test.cc | 0 .../sample_distorted_bounding_box_op.cc | 0 .../kernels/{ => image}/sampling_kernels.cc | 4 +- .../kernels/{ => image}/sampling_kernels.h | 0 .../{ => image}/sampling_kernels_test.cc | 2 +- .../{ => image}/scale_and_translate_op.cc | 5 +- .../{ => image}/scale_and_translate_op.h | 8 +- .../scale_and_translate_op_test.cc | 2 +- tensorflow/core/kernels/linalg/BUILD | 13 - tensorflow/core/kernels/mkl/BUILD | 30 +- .../kernels/quantized_resize_bilinear_op.cc | 2 +- tensorflow/core/util/BUILD | 15 +- .../{kernels => util}/image_resizer_state.h | 8 +- tensorflow/examples/label_image/BUILD | 2 +- 87 files changed, 709 insertions(+), 694 deletions(-) create mode 100644 tensorflow/core/kernels/image/BUILD rename tensorflow/core/kernels/{ => image}/adjust_contrast_op.cc (99%) rename tensorflow/core/kernels/{ => image}/adjust_contrast_op.h (97%) rename tensorflow/core/kernels/{ => image}/adjust_contrast_op_benchmark_test.cc (100%) rename tensorflow/core/kernels/{ => image}/adjust_contrast_op_gpu.cu.cc (96%) rename tensorflow/core/kernels/{ => image}/adjust_contrast_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/adjust_hsv_gpu.cu.h (96%) rename tensorflow/core/kernels/{ => image}/adjust_hue_op.cc (99%) rename tensorflow/core/kernels/{ => image}/adjust_hue_op.h (88%) rename tensorflow/core/kernels/{ => image}/adjust_hue_op_gpu.cu.cc (93%) rename tensorflow/core/kernels/{ => image}/adjust_saturation_op.cc (99%) rename tensorflow/core/kernels/{ => image}/adjust_saturation_op.h (87%) rename tensorflow/core/kernels/{ => image}/adjust_saturation_op_gpu.cu.cc (93%) rename tensorflow/core/kernels/{ => image}/attention_ops.cc (100%) rename tensorflow/core/kernels/{ => image}/colorspace_op.cc (99%) rename tensorflow/core/kernels/{ => image}/colorspace_op.h (95%) rename tensorflow/core/kernels/{ => image}/colorspace_op_gpu.cu.cc (95%) rename tensorflow/core/kernels/{ => image}/colorspace_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/crop_and_resize_op.cc (99%) rename tensorflow/core/kernels/{ => image}/crop_and_resize_op.h (93%) rename tensorflow/core/kernels/{ => image}/crop_and_resize_op_benchmark_test.cc (100%) rename tensorflow/core/kernels/{ => image}/crop_and_resize_op_gpu.cu.cc (99%) rename tensorflow/core/kernels/{ => image}/crop_and_resize_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/decode_image_op.cc (100%) rename tensorflow/core/kernels/{ => image}/draw_bounding_box_op.cc (100%) rename tensorflow/core/kernels/{ => image}/encode_jpeg_op.cc (100%) rename tensorflow/core/kernels/{ => image}/encode_jpeg_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/encode_png_op.cc (100%) rename tensorflow/core/kernels/{ => image}/extract_image_patches_op.cc (98%) rename tensorflow/core/kernels/{ => image}/extract_image_patches_op.h (91%) rename tensorflow/core/kernels/{ => image}/extract_image_patches_op_gpu.cu.cc (94%) rename tensorflow/core/kernels/{ => image}/extract_jpeg_shape_op.cc (100%) rename tensorflow/core/kernels/{ => image}/extract_volume_patches_op.cc (99%) rename tensorflow/core/kernels/{ => image}/extract_volume_patches_op.h (92%) rename tensorflow/core/kernels/{ => image}/extract_volume_patches_op_gpu.cu.cc (94%) rename tensorflow/core/kernels/{ => image}/generate_box_proposals_op.cu.cc (99%) rename tensorflow/core/kernels/{ => image}/image_ops.cc (99%) rename tensorflow/core/kernels/{ => image}/image_ops.h (98%) rename tensorflow/core/kernels/{ => image}/image_ops_gpu.cu.cc (96%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op.cc (99%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op.h (99%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op_benchmark_test.cc (100%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op_cpu_impl.h (83%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op_cpu_impl_1.cc (91%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op_cpu_impl_2.cc (91%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op_cpu_impl_3.cc (91%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op_cpu_impl_4.cc (91%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op_cpu_impl_5.cc (91%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op_gpu.cu.cc (97%) rename tensorflow/core/kernels/{ => image}/mirror_pad_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/non_max_suppression_op.cc (99%) rename tensorflow/core/kernels/{ => image}/non_max_suppression_op.cu.cc (99%) rename tensorflow/core/kernels/{ => image}/non_max_suppression_op.h (92%) rename tensorflow/core/kernels/{ => image}/non_max_suppression_op_benchmark_test.cc (100%) rename tensorflow/core/kernels/{ => image}/non_max_suppression_op_gpu_test.cc (100%) rename tensorflow/core/kernels/{ => image}/non_max_suppression_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/random_crop_op.cc (99%) rename tensorflow/core/kernels/{ => image}/resize_area_op.cc (99%) rename tensorflow/core/kernels/{ => image}/resize_area_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/resize_bicubic_op.cc (99%) rename tensorflow/core/kernels/{ => image}/resize_bicubic_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/resize_bilinear_op.cc (99%) rename tensorflow/core/kernels/{ => image}/resize_bilinear_op.h (90%) rename tensorflow/core/kernels/{ => image}/resize_bilinear_op_gpu.cu.cc (99%) rename tensorflow/core/kernels/{ => image}/resize_bilinear_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/resize_nearest_neighbor_op.cc (99%) rename tensorflow/core/kernels/{ => image}/resize_nearest_neighbor_op.h (88%) rename tensorflow/core/kernels/{ => image}/resize_nearest_neighbor_op_gpu.cu.cc (99%) rename tensorflow/core/kernels/{ => image}/resize_nearest_neighbor_op_test.cc (100%) rename tensorflow/core/kernels/{ => image}/resize_op_benchmark_test.cc (100%) rename tensorflow/core/kernels/{ => image}/sample_distorted_bounding_box_op.cc (100%) rename tensorflow/core/kernels/{ => image}/sampling_kernels.cc (96%) rename tensorflow/core/kernels/{ => image}/sampling_kernels.h (100%) rename tensorflow/core/kernels/{ => image}/sampling_kernels_test.cc (98%) rename tensorflow/core/kernels/{ => image}/scale_and_translate_op.cc (99%) rename tensorflow/core/kernels/{ => image}/scale_and_translate_op.h (92%) rename tensorflow/core/kernels/{ => image}/scale_and_translate_op_test.cc (99%) rename tensorflow/core/{kernels => util}/image_resizer_state.h (98%) diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 41eba6b5e28..12e143e7933 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -1010,9 +1010,7 @@ cc_library( "//tensorflow/core/kernels:functional_ops", "//tensorflow/core/kernels:grappler", "//tensorflow/core/kernels:histogram_op", - "//tensorflow/core/kernels:image", "//tensorflow/core/kernels:io", - "//tensorflow/core/kernels/linalg:linalg", "//tensorflow/core/kernels:lookup", "//tensorflow/core/kernels:logging", "//tensorflow/core/kernels:manip", @@ -1046,6 +1044,8 @@ cc_library( "//tensorflow/core/kernels:summary_kernels", "//tensorflow/core/kernels:training_ops", "//tensorflow/core/kernels:word2vec_kernels", + "//tensorflow/core/kernels/linalg:linalg", + "//tensorflow/core/kernels/image:image", "//tensorflow/core/kernels/sparse:kernels", ] + if_not_windows([ "//tensorflow/core/kernels/neon:neon_depthwise_conv_op", diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD index bfb192023a1..14e8d691d98 100644 --- a/tensorflow/core/kernels/BUILD +++ b/tensorflow/core/kernels/BUILD @@ -321,7 +321,6 @@ tf_kernel_library( deps = [ ":eigen_helpers", ":fill_functor", - ":image_resizer_state", ":ops_util", "//third_party/eigen3", "//tensorflow/core:core_cpu", @@ -341,32 +340,6 @@ cc_library( ], ) -tf_kernel_library( - name = "extract_image_patches_op", - prefix = "extract_image_patches_op", - deps = [ - ":bounds_check", - ":eigen_helpers", - ":ops_util", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], -) - -tf_kernel_library( - name = "extract_volume_patches_op", - prefix = "extract_volume_patches_op", - deps = [ - ":bounds_check", - ":eigen_helpers", - ":ops_util", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], -) - cc_library( name = "conv_3d", hdrs = ["conv_3d.h"], @@ -652,7 +625,6 @@ cc_library( cc_library( name = "batch_kernels", srcs = ["batch_kernels.cc"], - hdrs = ["batch_matmul_op_impl.h"], deps = [ ":ops_util_hdrs", "//tensorflow/core:framework", @@ -935,43 +907,6 @@ cc_library( ], ) -cc_library( - name = "image_resizer_state", - hdrs = ["image_resizer_state.h"], - visibility = ["//tensorflow:__subpackages__"], - deps = [ - ":bounds_check", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//third_party/eigen3", - ], -) - -cc_header_only_library( - name = "image_resizer_state_lib", - deps = [":image_resizer_state"], -) - -cc_library( - name = "sampling_kernels", - srcs = ["sampling_kernels.cc"], - hdrs = ["sampling_kernels.h"], - visibility = ["//visibility:private"], - deps = ["//tensorflow/core:lib"], -) - -tf_cc_test( - name = "sampling_kernels_test", - srcs = ["sampling_kernels_test.cc"], - deps = [ - ":sampling_kernels", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - "@com_google_absl//absl/strings", - ], -) - # OpKernel libraries ---------------------------------------------------------- ARRAY_DEPS = [ @@ -1026,8 +961,6 @@ cc_library( ":depth_space_ops", ":diag_op", ":edit_distance_op", - ":extract_image_patches_op", - ":extract_volume_patches_op", ":fingerprint_op", ":gather_nd_op", ":gather_op", @@ -1038,7 +971,6 @@ cc_library( ":immutable_constant_op", ":inplace_ops", ":listdiff_op", - ":mirror_pad_op", ":one_hot_op", ":pack_op", ":pad_op", @@ -1170,12 +1102,6 @@ tf_kernel_library( deps = ARRAY_DEPS, ) -tf_kernel_library( - name = "mirror_pad_op", - prefix = "mirror_pad_op", - deps = ARRAY_DEPS, -) - tf_kernel_library( name = "one_hot_op", prefix = "one_hot_op", @@ -1730,7 +1656,6 @@ tf_cuda_cc_test( tags = ["no_cuda11"], # b/159664089 deps = [ ":conv_ops", - ":image", ":ops_testutil", ":ops_util", "//tensorflow/cc:cc_ops", @@ -1743,6 +1668,7 @@ tf_cuda_cc_test( "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core:testlib", + "//tensorflow/core/kernels/image", "@com_google_absl//absl/algorithm:container", ], ) @@ -1828,7 +1754,6 @@ tf_cuda_cc_test( tags = tf_cuda_tests_tags(), deps = [ ":conv_ops", - ":image", ":ops_testutil", ":ops_util", "//tensorflow/cc:cc_ops", @@ -1841,69 +1766,7 @@ tf_cuda_cc_test( "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core:testlib", - ], -) - -tf_cc_test( - name = "decode_wav_op_test", - size = "small", - srcs = ["decode_wav_op_test.cc"], - deps = [ - ":decode_wav_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/cc:cc_ops", - "//tensorflow/cc:client_session", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cc_test( - name = "encode_jpeg_op_test", - size = "small", - srcs = ["encode_jpeg_op_test.cc"], - deps = [ - ":encode_jpeg_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cc_test( - name = "encode_wav_op_test", - size = "small", - srcs = ["encode_wav_op_test.cc"], - deps = [ - ":decode_wav_op", - ":encode_wav_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/cc:cc_ops", - "//tensorflow/cc:client_session", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:framework_internal", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:tensorflow", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", + "//tensorflow/core/kernels/image", ], ) @@ -2999,205 +2862,6 @@ tf_kernel_library( ], ) -cc_library( - name = "image", - deps = [ - ":adjust_contrast_op", - ":adjust_hue_op", - ":adjust_saturation_op", - ":attention_ops", - ":colorspace_op", - ":crop_and_resize_op", - ":decode_image_op", - ":draw_bounding_box_op", - ":encode_jpeg_op", - ":encode_png_op", - ":extract_jpeg_shape_op", - ":generate_box_proposals_op", - ":image_ops", - ":non_max_suppression_op", - ":random_crop_op", - ":resize_area_op", - ":resize_bicubic_op", - ":resize_bilinear_op", - ":resize_nearest_neighbor_op", - ":sample_distorted_bounding_box_op", - ":scale_and_translate_op", - ], -) - -IMAGE_DEPS = [ - ":bounds_check", - ":eigen_helpers", - ":image_resizer_state", - "//third_party/eigen3", - "//tensorflow/core:framework", - "//tensorflow/core:gif_internal", - "//tensorflow/core:jpeg_internal", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:png_internal", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core/util/tensor_bundle", -] - -tf_kernel_library( - name = "adjust_contrast_op", - prefix = "adjust_contrast_op", - deps = IMAGE_DEPS, -) - -cc_library( - name = "adjust_hsv_gpu_lib", - hdrs = ["adjust_hsv_gpu.cu.h"], - deps = ["//tensorflow/core:framework"], -) - -tf_kernel_library( - name = "adjust_hue_op", - prefix = "adjust_hue_op", - deps = IMAGE_DEPS + [":adjust_hsv_gpu_lib"], -) - -tf_kernel_library( - name = "adjust_saturation_op", - prefix = "adjust_saturation_op", - deps = IMAGE_DEPS + [":adjust_hsv_gpu_lib"], -) - -tf_kernel_library( - name = "attention_ops", - prefix = "attention_ops", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "colorspace_op", - prefix = "colorspace_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "crop_and_resize_op", - prefix = "crop_and_resize_op", - deps = IMAGE_DEPS + ["//tensorflow/core:framework_internal"], -) - -tf_kernel_library( - name = "decode_image_op", - prefix = "decode_image_op", - deps = IMAGE_DEPS + ["@com_google_absl//absl/strings"], -) - -tf_kernel_library( - name = "draw_bounding_box_op", - prefix = "draw_bounding_box_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "encode_jpeg_op", - prefix = "encode_jpeg_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "encode_png_op", - prefix = "encode_png_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "extract_jpeg_shape_op", - prefix = "extract_jpeg_shape_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "generate_box_proposals_op", - gpu_srcs = ["generate_box_proposals_op.cu.cc"], - deps = [":gpu_prim_hdrs"] + if_cuda([ - ":non_max_suppression_op_gpu", - ]), -) - -tf_kernel_library( - name = "non_max_suppression_op", - prefix = "non_max_suppression_op", - deps = IMAGE_DEPS + [":gpu_prim_hdrs"], -) - -tf_kernel_library( - name = "scale_and_translate_op", - prefix = "scale_and_translate_op", - deps = IMAGE_DEPS + [":sampling_kernels"], -) - -tf_kernel_library( - name = "random_crop_op", - prefix = "random_crop_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "resize_area_op", - prefix = "resize_area_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "resize_bicubic_op", - prefix = "resize_bicubic_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "resize_bilinear_op", - prefix = "resize_bilinear_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "resize_nearest_neighbor_op", - prefix = "resize_nearest_neighbor_op", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "sample_distorted_bounding_box_op", - prefix = "sample_distorted_bounding_box_op", - deps = IMAGE_DEPS + [":stateless_random_ops"], -) - -tf_kernel_library( - name = "image_ops", - prefix = "image_ops", - deps = IMAGE_DEPS, -) - -tf_kernel_library( - name = "encode_wav_op", - prefix = "encode_wav_op", - deps = [ - ":bounds_check", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:protos_all_cc", - ], -) - -tf_kernel_library( - name = "decode_wav_op", - prefix = "decode_wav_op", - deps = [ - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:protos_all_cc", - ], -) - tf_cc_tests( name = "eigen_test", size = "small", @@ -3282,158 +2946,6 @@ tf_cc_tests( ], ) -tf_cc_tests( - name = "bonus_tests", - srcs = [ - "adjust_contrast_op_test.cc", - "colorspace_op_test.cc", - "crop_and_resize_op_test.cc", - "mirror_pad_op_test.cc", - "non_max_suppression_op_test.cc", - "resize_area_op_test.cc", - "resize_bicubic_op_test.cc", - "resize_nearest_neighbor_op_test.cc", - "scale_and_translate_op_test.cc", - ], - linkopts = select({ - "//tensorflow:macos": ["-headerpad_max_install_names"], - "//conditions:default": [], - }), - deps = [ - ":image", - ":mirror_pad_op", - ":ops_testutil", - ":ops_util", - ":sampling_kernels", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cc_test( - name = "non_max_suppression_op_benchmark_test", - srcs = ["non_max_suppression_op_benchmark_test.cc"], - deps = [ - ":image", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cuda_cc_test( - name = "resize_bilinear_op_test", - srcs = ["resize_bilinear_op_test.cc"], - tags = ["no_cuda_on_cpu_tap"], - deps = [ - ":image", - ":ops_testutil", - ":ops_util", - ":sampling_kernels", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:lib_internal", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cuda_cc_test( - name = "adjust_contrast_op_benchmark_test", - srcs = ["adjust_contrast_op_benchmark_test.cc"], - deps = [ - ":image", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cuda_cc_test( - name = "crop_and_resize_op_benchmark_test", - srcs = ["crop_and_resize_op_benchmark_test.cc"], - deps = [ - ":image", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cuda_cc_test( - name = "mirror_pad_op_benchmark_test", - srcs = ["mirror_pad_op_benchmark_test.cc"], - deps = [ - ":mirror_pad_op", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - -tf_cuda_cc_test( - name = "non_max_suppression_op_gpu_test", - srcs = ["non_max_suppression_op_gpu_test.cc"], - tags = tf_cuda_tests_tags() + ["no_cuda_on_cpu_tap"], - deps = [ - ":image", - ":ops_testutil", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:lib", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - "@com_google_absl//absl/strings", - ], -) - -tf_cuda_cc_test( - name = "resize_benchmark_test", - srcs = ["resize_op_benchmark_test.cc"], - deps = [ - ":image", - ":ops_testutil", - ":ops_util", - "//tensorflow/core:core_cpu", - "//tensorflow/core:framework", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/core:testlib", - ], -) - cc_library( name = "io", deps = [ @@ -4312,7 +3824,6 @@ tf_kernel_library( ":conv_2d", ":conv_3d", ":eigen_contraction_kernel", - ":image_resizer_state", ":fill_functor", ":fused_eigen_output_kernels", ":ops_util", @@ -4323,6 +3834,7 @@ tf_kernel_library( "//tensorflow/core:framework", "//tensorflow/core:lib", "//tensorflow/core:lib_internal", + "//tensorflow/core/util:image_resizer_state", "//tensorflow/core/util/proto:proto_utils", "//tensorflow/stream_executor/gpu:gpu_asm_opts", ] + select({ @@ -5894,6 +5406,74 @@ tf_kernel_library( ], ) +tf_kernel_library( + name = "encode_wav_op", + prefix = "encode_wav_op", + deps = [ + ":bounds_check", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + ], +) + +tf_cc_test( + name = "encode_wav_op_test", + size = "small", + srcs = ["encode_wav_op_test.cc"], + deps = [ + ":decode_wav_op", + ":encode_wav_op", + ":ops_testutil", + ":ops_util", + "//tensorflow/cc:cc_ops", + "//tensorflow/cc:client_session", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:tensorflow", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + +tf_kernel_library( + name = "decode_wav_op", + prefix = "decode_wav_op", + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + ], +) + +tf_cc_test( + name = "decode_wav_op_test", + size = "small", + srcs = ["decode_wav_op_test.cc"], + deps = [ + ":decode_wav_op", + ":ops_testutil", + ":ops_util", + "//tensorflow/cc:cc_ops", + "//tensorflow/cc:client_session", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:framework_internal", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:tensorflow", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + ], +) + filegroup( name = "spectrogram_test_data", srcs = [ @@ -6247,8 +5827,6 @@ filegroup( "matmul_op.h", "no_op.cc", "no_op.h", - "non_max_suppression_op.cc", - "non_max_suppression_op.h", "one_hot_op.cc", "one_hot_op.h", "ops_util.h", @@ -6296,7 +5874,10 @@ filegroup( "unpack_op.cc", "variable_ops.cc", "variable_ops.h", + ] + [ "//tensorflow/c/kernels:android_all_op_kernels", + "//tensorflow/core/kernels/image:non_max_suppression_op.cc", + "//tensorflow/core/kernels/image:non_max_suppression_op.h", ], ) @@ -6320,9 +5901,6 @@ filegroup( filegroup( name = "android_extended_ops_headers", srcs = [ - "adjust_contrast_op.h", - "adjust_hue_op.h", - "adjust_saturation_op.h", "argmax_op.h", "avgpooling_op.h", "batch_matmul_op_impl.h", @@ -6340,12 +5918,9 @@ filegroup( "depthwise_conv_op.h", "diag_op.h", "dilation_ops.h", - "extract_image_patches_op.h", "fake_quant_ops_functor.h", "fused_batch_norm_op.h", "gemm_functors.h", - "image_ops.h", - "image_resizer_state.h", "initializable_lookup_table.h", "inplace_ops.cc", "inplace_ops_functor.h", @@ -6357,8 +5932,6 @@ filegroup( "mfcc.h", "mfcc_dct.h", "mfcc_mel_filterbank.h", - "mirror_pad_op.h", - "mirror_pad_op_cpu_impl.h", "multinomial_op.h", "pad_op.h", "pooling_ops_3d.h", @@ -6369,8 +5942,6 @@ filegroup( "relu_op.h", "relu_op_functor.h", "reshape_util.h", - "resize_bilinear_op.h", - "resize_nearest_neighbor_op.h", "reverse_op.h", "save_restore_tensor.h", "scan_ops.h", @@ -6400,15 +5971,26 @@ filegroup( "xent_op.h", ] + [ "//tensorflow/core/kernels/boosted_trees/quantiles:weighted_quantiles_hdrs", + "//tensorflow/core/kernels/image:adjust_contrast_op.h", + "//tensorflow/core/kernels/image:adjust_hue_op.h", + "//tensorflow/core/kernels/image:adjust_saturation_op.h", + "//tensorflow/core/kernels/image:extract_image_patches_op.h", + "//tensorflow/core/kernels/image:image_ops.h", + "//tensorflow/core/kernels/image:mirror_pad_op.h", + "//tensorflow/core/kernels/image:mirror_pad_op_cpu_impl.h", + "//tensorflow/core/kernels/image:resize_bilinear_op.h", + "//tensorflow/core/kernels/image:resize_nearest_neighbor_op.h", "//tensorflow/core/kernels/linalg:linalg_ops_common.h", "//tensorflow/core/kernels/linalg:matrix_diag_op.h", "//tensorflow/core/kernels/linalg:matrix_set_diag_op.h", + "//tensorflow/core/util:image_resizer_state.h", ], ) filegroup( name = "android_extended_ops_group1", srcs = [ + ":android_extended_ops_headers", "argmax_op.cc", "avgpooling_op.cc", "batch_matmul_op_real.cc", @@ -6421,18 +6003,16 @@ filegroup( "conv_grad_input_ops.cc", "conv_grad_ops.h", "conv_grad_ops_3d.cc", - "conv_grad_shape_utils.h", "conv_grad_shape_utils.cc", + "conv_grad_shape_utils.h", "conv_ops.cc", "conv_ops_3d.cc", "conv_ops_fused_double.cc", "conv_ops_fused_float.cc", "conv_ops_fused_half.cc", - "conv_ops_fused_impl.h", "conv_ops_fused_image_transform.cc", + "conv_ops_fused_impl.h", "conv_ops_using_gemm.cc", - "crop_and_resize_op.cc", - "crop_and_resize_op.h", "cwise_op_abs.cc", "cwise_op_add_1.cc", "cwise_op_add_2.cc", @@ -6448,8 +6028,6 @@ filegroup( "cwise_op_div.cc", "cwise_op_equal_to_1.cc", "cwise_op_equal_to_2.cc", - "cwise_op_not_equal_to_1.cc", - "cwise_op_not_equal_to_2.cc", "cwise_op_erf.cc", "cwise_op_exp.cc", "cwise_op_floor.cc", @@ -6474,6 +6052,8 @@ filegroup( "cwise_op_mul_2.cc", "cwise_op_neg_1.cc", "cwise_op_neg_2.cc", + "cwise_op_not_equal_to_1.cc", + "cwise_op_not_equal_to_2.cc", "cwise_op_pow.cc", "cwise_op_real.cc", "cwise_op_reciprocal.cc", @@ -6491,9 +6071,9 @@ filegroup( "cwise_op_sub.cc", "cwise_op_tan.cc", "cwise_op_tanh.cc", - "cwise_op_xlogy.cc", - "cwise_op_xlog1py.cc", "cwise_op_xdivy.cc", + "cwise_op_xlog1py.cc", + "cwise_op_xlogy.cc", "data_format_ops.cc", "decode_raw_op.cc", "decode_wav_op.cc", @@ -6501,9 +6081,9 @@ filegroup( "deep_conv2d.h", "depthwise_conv_op.cc", "dynamic_partition_op.cc", - "encode_wav_op.cc", "eigen_contraction_kernel.cc", "eigen_contraction_kernel.h", + "encode_wav_op.cc", "fake_quant_ops.cc", "fifo_queue.cc", "fifo_queue_op.cc", @@ -6514,8 +6094,9 @@ filegroup( "population_count_op.cc", "population_count_op.h", "winograd_transform.h", - ":android_extended_ops_headers", ] + [ + "//tensorflow/core/kernels/image:crop_and_resize_op.cc", + "//tensorflow/core/kernels/image:crop_and_resize_op.h", "//tensorflow/core/kernels/linalg:einsum_op_impl_half.cc", "//tensorflow/core/kernels/linalg:einsum_op_impl_bfloat16.cc", "//tensorflow/core/kernels/linalg:einsum_op_impl_int32.cc", @@ -6538,9 +6119,7 @@ filegroup( filegroup( name = "android_extended_ops_group2", srcs = [ - "adjust_contrast_op.cc", - "adjust_hue_op.cc", - "adjust_saturation_op.cc", + ":android_extended_ops_headers", "base64_ops.cc", "batchtospace_op.cc", "broadcast_to_op.cc", @@ -6550,9 +6129,7 @@ filegroup( "diag_op.cc", "dilation_ops.cc", "dynamic_stitch_op.cc", - "extract_image_patches_op.cc", "fft_ops.cc", - "image_ops.cc", "in_topk_op.cc", "in_topk_op.h", "initializable_lookup_table.cc", @@ -6568,12 +6145,6 @@ filegroup( "mfcc_dct.cc", "mfcc_mel_filterbank.cc", "mfcc_op.cc", - "mirror_pad_op.cc", - "mirror_pad_op_cpu_impl_1.cc", - "mirror_pad_op_cpu_impl_2.cc", - "mirror_pad_op_cpu_impl_3.cc", - "mirror_pad_op_cpu_impl_4.cc", - "mirror_pad_op_cpu_impl_5.cc", "multinomial_op.cc", "pad_op.cc", "padding_fifo_queue.cc", @@ -6600,11 +6171,8 @@ filegroup( "regex_replace_op.cc", "relu_op.cc", "reshape_util.cc", - "resize_bilinear_op.cc", - "resize_nearest_neighbor_op.cc", "restore_op.cc", "reverse_op.cc", - "sample_distorted_bounding_box_op.cc", "save_op.cc", "save_restore_tensor.cc", "save_restore_v2_ops.cc", @@ -6682,9 +6250,22 @@ filegroup( "unique_op.cc", "where_op.cc", "xent_op.cc", - ":android_extended_ops_headers", ] + [ "//tensorflow/core/kernels/boosted_trees:quantile_ops.cc", + "//tensorflow/core/kernels/image:adjust_contrast_op.cc", + "//tensorflow/core/kernels/image:adjust_hue_op.cc", + "//tensorflow/core/kernels/image:adjust_saturation_op.cc", + "//tensorflow/core/kernels/image:extract_image_patches_op.cc", + "//tensorflow/core/kernels/image:image_ops.cc", + "//tensorflow/core/kernels/image:mirror_pad_op.cc", + "//tensorflow/core/kernels/image:mirror_pad_op_cpu_impl_1.cc", + "//tensorflow/core/kernels/image:mirror_pad_op_cpu_impl_2.cc", + "//tensorflow/core/kernels/image:mirror_pad_op_cpu_impl_3.cc", + "//tensorflow/core/kernels/image:mirror_pad_op_cpu_impl_4.cc", + "//tensorflow/core/kernels/image:mirror_pad_op_cpu_impl_5.cc", + "//tensorflow/core/kernels/image:resize_bilinear_op.cc", + "//tensorflow/core/kernels/image:resize_nearest_neighbor_op.cc", + "//tensorflow/core/kernels/image:sample_distorted_bounding_box_op.cc", "//tensorflow/core/kernels/linalg:linalg_ops_common.cc", "//tensorflow/core/kernels/linalg:matrix_diag_op.cc", "//tensorflow/core/kernels/linalg:matrix_inverse_op.cc", @@ -6727,7 +6308,6 @@ ANDROID_TEXTUAL_HDRS = [ "eigen_spatial_convolutions-inl.h", "gather_nd_op_cpu_impl.h", "gemm_functors.h", - "mirror_pad_op_cpu_impl.h", "scatter_nd_op_cpu_impl.h", "slice_op_cpu_impl.h", "strided_slice_op_impl.h", @@ -6742,6 +6322,7 @@ filegroup( srcs = [ "//tensorflow/c/kernels:android_all_op_kernels", "//tensorflow/core/kernels/data:android_all_op_kernels", + "//tensorflow/core/kernels/image:android_all_op_kernels", "//tensorflow/core/kernels/linalg:android_all_op_kernels", ] + glob( [ @@ -6773,13 +6354,6 @@ filegroup( "sparse_cross_op.*", "text_line_reader_op.*", "summary_image_op.*", - "decode_image_op.*", - "encode_png_op.*", - "encode_jpeg_op.*", - "extract_jpeg_shape_op.*", - "decode_jpeg_op.*", - "decode_and_crop_jpeg_op.*", - "decode_gif_op.*", "identity_reader_op.*", "remote_fused_graph_execute_op.*", "remote_fused_graph_rewriter_transform.*", @@ -6824,7 +6398,10 @@ filegroup( filegroup( name = "android_all_ops_textual_hdrs", - srcs = ANDROID_TEXTUAL_HDRS, + srcs = ANDROID_TEXTUAL_HDRS + [ + "//tensorflow/core/kernels/image:android_all_ops_textual_hdrs", + "//tensorflow/core/util:image_resizer_state.h", + ], visibility = ["//visibility:public"], ) # LINT.ThenChange(//tensorflow/contrib/makefile/tf_op_files.txt) @@ -6868,26 +6445,6 @@ build_test( targets = [":portable_tensorflow_kernels"], ) -cc_library( - name = "android_tensorflow_image_op", - srcs = if_android(["decode_image_op.cc"]), - copts = tf_copts(), - linkopts = ["-ldl"], - visibility = ["//visibility:public"], - deps = [ - "//tensorflow/core:android_gif_internal", - "//tensorflow/core:android_jpeg_internal", - "//tensorflow/core:android_png_internal", - "//tensorflow/core:portable_tensorflow_lib_lite", - ], - alwayslink = 1, -) - -build_test( - name = "android_tensorflow_image_op_build_test", - targets = [":android_tensorflow_image_op"], -) - cc_library( name = "android_whole_file_read_ops", srcs = if_android(["whole_file_read_ops.cc"]), @@ -6930,7 +6487,6 @@ tf_kernel_library( ":conv_ops", ":cwise_op", ":eigen_helpers", - ":image_resizer_state", ":meta_support", ":ops_util", ":pooling_ops", @@ -6938,6 +6494,7 @@ tf_kernel_library( "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:lib", + "//tensorflow/core/util:image_resizer_state", "//third_party/eigen3", "@gemmlowp", ], @@ -7831,7 +7388,6 @@ test_suite( ":cast_op_test", ":concat_op_test", ":control_flow_ops_test", - ":crop_and_resize_op_test", ":cwise_ops_test", ":deep_conv2d_test", ":dequantize_op_test", @@ -7844,7 +7400,6 @@ test_suite( ":mfcc_test", ":multinomial_op_test", ":nn_ops_test", - ":non_max_suppression_op_test", ":quantization_utils_test", ":quantize_and_dequantize_op_test", ":quantize_op_test", @@ -7858,7 +7413,6 @@ test_suite( ":random_poisson_op_test", ":reduction_ops_test", ":requantization_range_op_test", - ":resize_bilinear_op_test", ":scatter_op_test", ":segment_reduction_ops_test", ":slice_op_test", @@ -7868,6 +7422,9 @@ test_suite( ":strided_slice_op_test", ":unique_op_test", ":variable_ops_test", + "//tensorflow/core/kernels/image:crop_and_resize_op_test", + "//tensorflow/core/kernels/image:non_max_suppression_op_test", + "//tensorflow/core/kernels/image:resize_bilinear_op_test", ], ) diff --git a/tensorflow/core/kernels/conv_ops_fused_image_transform.cc b/tensorflow/core/kernels/conv_ops_fused_image_transform.cc index 9055639aaaf..091e483b2ca 100644 --- a/tensorflow/core/kernels/conv_ops_fused_image_transform.cc +++ b/tensorflow/core/kernels/conv_ops_fused_image_transform.cc @@ -34,9 +34,9 @@ limitations under the License. #include "tensorflow/core/kernels/conv_2d.h" #include "tensorflow/core/kernels/conv_ops.h" #include "tensorflow/core/kernels/gemm_functors.h" -#include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/kernels/ops_util.h" #include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/util/image_resizer_state.h" #include "tensorflow/core/util/mirror_pad_mode.h" #include "tensorflow/core/util/padding.h" #include "tensorflow/core/util/tensor_format.h" diff --git a/tensorflow/core/kernels/conv_ops_using_gemm.cc b/tensorflow/core/kernels/conv_ops_using_gemm.cc index dff1a533ee0..71eda28899e 100644 --- a/tensorflow/core/kernels/conv_ops_using_gemm.cc +++ b/tensorflow/core/kernels/conv_ops_using_gemm.cc @@ -62,7 +62,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_slice.h" #include "tensorflow/core/kernels/conv_ops.h" #include "tensorflow/core/kernels/gemm_functors.h" -#include "tensorflow/core/kernels/image_resizer_state.h" +#include "tensorflow/core/util/image_resizer_state.h" #include "tensorflow/core/util/mirror_pad_mode.h" #include "tensorflow/core/util/padding.h" #include "tensorflow/core/util/tensor_format.h" diff --git a/tensorflow/core/kernels/image/BUILD b/tensorflow/core/kernels/image/BUILD new file mode 100644 index 00000000000..f7ad9ab0371 --- /dev/null +++ b/tensorflow/core/kernels/image/BUILD @@ -0,0 +1,449 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load( + "//tensorflow:tensorflow.bzl", + "if_android", + "tf_cc_test", + "tf_cc_tests", + "tf_copts", + "tf_kernel_library", +) +load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") +load( + "//tensorflow/core/platform:build_config_root.bzl", + "tf_cuda_tests_tags", +) +load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test") + +# TODO(rmlarsen): Remove ASAP. +package_group( + name = "friends", + packages = [ + "//tensorflow/...", + "//tensorflow_text/...", + ], +) + +package( + default_visibility = [ + ":friends", + "//tensorflow:__subpackages__", + "//tensorflow:internal", + ], + licenses = ["notice"], # Apache 2.0 +) + +# Export a few files for use on Android. +exports_files([ + "adjust_contrast_op.cc", + "adjust_contrast_op.h", + "adjust_hue_op.cc", + "adjust_hue_op.h", + "adjust_saturation_op.cc", + "adjust_saturation_op.h", + "crop_and_resize_op.cc", + "crop_and_resize_op.h", + "extract_image_patches_op.cc", + "extract_image_patches_op.h", + "image_ops.h", + "image_ops.cc", + "mirror_pad_op.cc", + "mirror_pad_op.h", + "mirror_pad_op_cpu_impl.h", + "mirror_pad_op_cpu_impl_1.cc", + "mirror_pad_op_cpu_impl_2.cc", + "mirror_pad_op_cpu_impl_3.cc", + "mirror_pad_op_cpu_impl_4.cc", + "mirror_pad_op_cpu_impl_5.cc", + "non_max_suppression_op.cc", + "non_max_suppression_op.h", + "resize_bilinear_op.cc", + "resize_bilinear_op.h", + "resize_nearest_neighbor_op.cc", + "resize_nearest_neighbor_op.h", + "sample_distorted_bounding_box_op.cc", +]) + +# Private support libraries --------------------------------------------------- +cc_library( + name = "sampling_kernels", + srcs = ["sampling_kernels.cc"], + hdrs = ["sampling_kernels.h"], + visibility = ["//visibility:private"], + deps = ["//tensorflow/core:lib"], +) + +tf_cc_test( + name = "sampling_kernels_test", + srcs = ["sampling_kernels_test.cc"], + deps = [ + ":sampling_kernels", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "@com_google_absl//absl/strings", + ], +) + +# Public support libraries ----------------------------------------------------< +cc_library( + name = "image", + deps = [ + ":adjust_contrast_op", + ":adjust_hue_op", + ":adjust_saturation_op", + ":attention_ops", + ":colorspace_op", + ":crop_and_resize_op", + ":decode_image_op", + ":draw_bounding_box_op", + ":encode_jpeg_op", + ":encode_png_op", + ":extract_image_patches_op", + ":extract_jpeg_shape_op", + ":extract_volume_patches_op", + ":generate_box_proposals_op", + ":image_ops", + ":mirror_pad_op", + ":non_max_suppression_op", + ":random_crop_op", + ":resize_area_op", + ":resize_bicubic_op", + ":resize_bilinear_op", + ":resize_nearest_neighbor_op", + ":sample_distorted_bounding_box_op", + ":scale_and_translate_op", + ], +) + +IMAGE_DEPS = [ + "//third_party/eigen3", + "//tensorflow/core:framework", + "//tensorflow/core:gif_internal", + "//tensorflow/core:jpeg_internal", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:png_internal", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/kernels:bounds_check", + "//tensorflow/core/kernels:eigen_helpers", + "//tensorflow/core/util/tensor_bundle", + "//tensorflow/core/util:image_resizer_state", +] + +IMAGE_TEST_DEPS = [ + "//tensorflow/core/kernels:ops_testutil", + "//tensorflow/core/kernels:ops_util", + "//tensorflow/core:core_cpu", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:lib_internal", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", +] + +tf_kernel_library( + name = "adjust_contrast_op", + prefix = "adjust_contrast_op", + deps = IMAGE_DEPS, +) + +cc_library( + name = "adjust_hsv_gpu_lib", + hdrs = ["adjust_hsv_gpu.cu.h"], + deps = ["//tensorflow/core:framework"], +) + +tf_kernel_library( + name = "adjust_hue_op", + prefix = "adjust_hue_op", + deps = IMAGE_DEPS + [":adjust_hsv_gpu_lib"], +) + +tf_kernel_library( + name = "adjust_saturation_op", + prefix = "adjust_saturation_op", + deps = IMAGE_DEPS + [":adjust_hsv_gpu_lib"], +) + +tf_kernel_library( + name = "attention_ops", + prefix = "attention_ops", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "colorspace_op", + prefix = "colorspace_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "crop_and_resize_op", + prefix = "crop_and_resize_op", + deps = IMAGE_DEPS + ["//tensorflow/core:framework_internal"], +) + +tf_kernel_library( + name = "decode_image_op", + prefix = "decode_image_op", + deps = IMAGE_DEPS + ["@com_google_absl//absl/strings"], +) + +tf_kernel_library( + name = "draw_bounding_box_op", + prefix = "draw_bounding_box_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "encode_jpeg_op", + prefix = "encode_jpeg_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "encode_png_op", + prefix = "encode_png_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "extract_jpeg_shape_op", + prefix = "extract_jpeg_shape_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "extract_image_patches_op", + prefix = "extract_image_patches_op", + deps = [ + "//tensorflow/core/kernels:ops_util", + ] + IMAGE_DEPS, +) + +tf_kernel_library( + name = "extract_volume_patches_op", + prefix = "extract_volume_patches_op", + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/kernels:eigen_helpers", + "//tensorflow/core/kernels:ops_util", + "//third_party/eigen3", + ], +) + +tf_kernel_library( + name = "generate_box_proposals_op", + gpu_srcs = ["generate_box_proposals_op.cu.cc"], + deps = ["//tensorflow/core/kernels:gpu_prim_hdrs"] + if_cuda([ + ":non_max_suppression_op_gpu", + ]), +) + +tf_kernel_library( + name = "non_max_suppression_op", + prefix = "non_max_suppression_op", + deps = IMAGE_DEPS + ["//tensorflow/core/kernels:gpu_prim_hdrs"], +) + +tf_kernel_library( + name = "scale_and_translate_op", + prefix = "scale_and_translate_op", + deps = IMAGE_DEPS + [":sampling_kernels"], +) + +tf_kernel_library( + name = "random_crop_op", + prefix = "random_crop_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "resize_area_op", + prefix = "resize_area_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "resize_bicubic_op", + prefix = "resize_bicubic_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "resize_bilinear_op", + prefix = "resize_bilinear_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "resize_nearest_neighbor_op", + prefix = "resize_nearest_neighbor_op", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "sample_distorted_bounding_box_op", + prefix = "sample_distorted_bounding_box_op", + deps = IMAGE_DEPS + ["//tensorflow/core/kernels:stateless_random_ops"], +) + +tf_kernel_library( + name = "image_ops", + prefix = "image_ops", + deps = IMAGE_DEPS, +) + +tf_kernel_library( + name = "mirror_pad_op", + prefix = "mirror_pad_op", + deps = IMAGE_DEPS, +) + +# Tests ------------------------ + +tf_cc_tests( + name = "bonus_tests", + srcs = [ + "adjust_contrast_op_test.cc", + "colorspace_op_test.cc", + "crop_and_resize_op_test.cc", + "mirror_pad_op_test.cc", + "non_max_suppression_op_test.cc", + "resize_area_op_test.cc", + "resize_bicubic_op_test.cc", + "resize_nearest_neighbor_op_test.cc", + "scale_and_translate_op_test.cc", + ], + linkopts = select({ + "//tensorflow:macos": ["-headerpad_max_install_names"], + "//conditions:default": [], + }), + deps = [ + ":image", + ":sampling_kernels", + ":mirror_pad_op", + ] + IMAGE_TEST_DEPS, +) + +tf_cc_test( + name = "non_max_suppression_op_benchmark_test", + srcs = ["non_max_suppression_op_benchmark_test.cc"], + deps = [ + ":image", + ] + IMAGE_TEST_DEPS, +) + +tf_cuda_cc_test( + name = "resize_bilinear_op_test", + srcs = ["resize_bilinear_op_test.cc"], + tags = ["no_cuda_on_cpu_tap"], + deps = [ + ":image", + ":sampling_kernels", + ] + IMAGE_TEST_DEPS, +) + +tf_cuda_cc_test( + name = "adjust_contrast_op_benchmark_test", + srcs = ["adjust_contrast_op_benchmark_test.cc"], + deps = [ + ":image", + ] + IMAGE_TEST_DEPS, +) + +tf_cuda_cc_test( + name = "crop_and_resize_op_benchmark_test", + srcs = ["crop_and_resize_op_benchmark_test.cc"], + deps = [ + ":image", + ] + IMAGE_TEST_DEPS, +) + +tf_cuda_cc_test( + name = "mirror_pad_op_benchmark_test", + srcs = ["mirror_pad_op_benchmark_test.cc"], + deps = [ + ":mirror_pad_op", + ] + IMAGE_TEST_DEPS, +) + +tf_cuda_cc_test( + name = "non_max_suppression_op_gpu_test", + srcs = ["non_max_suppression_op_gpu_test.cc"], + tags = tf_cuda_tests_tags() + ["no_cuda_on_cpu_tap"], + deps = [ + ":image", + "@com_google_absl//absl/strings", + ] + IMAGE_TEST_DEPS, +) + +tf_cuda_cc_test( + name = "resize_benchmark_test", + srcs = ["resize_op_benchmark_test.cc"], + deps = [ + ":image", + ] + IMAGE_TEST_DEPS, +) + +tf_cc_test( + name = "encode_jpeg_op_test", + size = "small", + srcs = ["encode_jpeg_op_test.cc"], + deps = [ + ":encode_jpeg_op", + ] + IMAGE_TEST_DEPS, +) + +cc_library( + name = "android_tensorflow_image_op", + srcs = if_android(["decode_image_op.cc"]), + copts = tf_copts(), + linkopts = ["-ldl"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/core:android_gif_internal", + "//tensorflow/core:android_jpeg_internal", + "//tensorflow/core:android_png_internal", + "//tensorflow/core:portable_tensorflow_lib_lite", + ], + alwayslink = 1, +) + +build_test( + name = "android_tensorflow_image_op_build_test", + targets = [":android_tensorflow_image_op"], +) + +# A file group which contains all operators which are known to work on mobile. +filegroup( + name = "android_all_op_kernels", + srcs = glob( + [ + "*.cc", + "*.h", + ], + exclude = [ + "*test.cc", + "*test.h", + "*_test_*", + "decode_image_op.*", + "encode_png_op.*", + "encode_jpeg_op.*", + "extract_jpeg_shape_op.*", + "decode_jpeg_op.*", + "decode_and_crop_jpeg_op.*", + "decode_gif_op.*", + ], + ), + visibility = ["//tensorflow:__subpackages__"], +) + +filegroup( + name = "android_all_ops_textual_hdrs", + srcs = ["mirror_pad_op.h"], + visibility = ["//visibility:public"], +) diff --git a/tensorflow/core/kernels/adjust_contrast_op.cc b/tensorflow/core/kernels/image/adjust_contrast_op.cc similarity index 99% rename from tensorflow/core/kernels/adjust_contrast_op.cc rename to tensorflow/core/kernels/image/adjust_contrast_op.cc index c13619e0e5f..6853465d9db 100644 --- a/tensorflow/core/kernels/adjust_contrast_op.cc +++ b/tensorflow/core/kernels/image/adjust_contrast_op.cc @@ -16,8 +16,10 @@ limitations under the License. // See docs in ../ops/image_ops.cc #define EIGEN_USE_THREADS -#include "tensorflow/core/kernels/adjust_contrast_op.h" +#include "tensorflow/core/kernels/image/adjust_contrast_op.h" + #include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" diff --git a/tensorflow/core/kernels/adjust_contrast_op.h b/tensorflow/core/kernels/image/adjust_contrast_op.h similarity index 97% rename from tensorflow/core/kernels/adjust_contrast_op.h rename to tensorflow/core/kernels/image/adjust_contrast_op.h index 3e501bccee3..4bff5f73a63 100644 --- a/tensorflow/core/kernels/adjust_contrast_op.h +++ b/tensorflow/core/kernels/image/adjust_contrast_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_CONTRAST_OP_H_ -#define TENSORFLOW_CORE_KERNELS_ADJUST_CONTRAST_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGEADJUST_CONTRAST_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGEADJUST_CONTRAST_OP_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/tensor_types.h" @@ -157,4 +157,4 @@ struct AdjustContrastv2 { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_ADJUST_CONTRAST_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGEADJUST_CONTRAST_OP_H_ diff --git a/tensorflow/core/kernels/adjust_contrast_op_benchmark_test.cc b/tensorflow/core/kernels/image/adjust_contrast_op_benchmark_test.cc similarity index 100% rename from tensorflow/core/kernels/adjust_contrast_op_benchmark_test.cc rename to tensorflow/core/kernels/image/adjust_contrast_op_benchmark_test.cc diff --git a/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc b/tensorflow/core/kernels/image/adjust_contrast_op_gpu.cu.cc similarity index 96% rename from tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc rename to tensorflow/core/kernels/image/adjust_contrast_op_gpu.cu.cc index e072dc46f5f..147700c1574 100644 --- a/tensorflow/core/kernels/adjust_contrast_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/adjust_contrast_op_gpu.cu.cc @@ -18,9 +18,8 @@ limitations under the License. #define EIGEN_USE_GPU -#include "tensorflow/core/kernels/adjust_contrast_op.h" - #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/kernels/image/adjust_contrast_op.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/adjust_contrast_op_test.cc b/tensorflow/core/kernels/image/adjust_contrast_op_test.cc similarity index 100% rename from tensorflow/core/kernels/adjust_contrast_op_test.cc rename to tensorflow/core/kernels/image/adjust_contrast_op_test.cc diff --git a/tensorflow/core/kernels/adjust_hsv_gpu.cu.h b/tensorflow/core/kernels/image/adjust_hsv_gpu.cu.h similarity index 96% rename from tensorflow/core/kernels/adjust_hsv_gpu.cu.h rename to tensorflow/core/kernels/image/adjust_hsv_gpu.cu.h index ba4427ffb9d..42511f249bb 100644 --- a/tensorflow/core/kernels/adjust_hsv_gpu.cu.h +++ b/tensorflow/core/kernels/image/adjust_hsv_gpu.cu.h @@ -11,8 +11,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_HSV_GPU_CU_H_ -#define TENSORFLOW_CORE_KERNELS_ADJUST_HSV_GPU_CU_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGEADJUST_HSV_GPU_CU_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGEADJUST_HSV_GPU_CU_H_ #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM @@ -142,4 +142,4 @@ __global__ void adjust_hsv_nhwc( } // namespace tensorflow #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#endif // TENSORFLOW_CORE_KERNELS_ADJUST_HSV_GPU_CU_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGEADJUST_HSV_GPU_CU_H_ diff --git a/tensorflow/core/kernels/adjust_hue_op.cc b/tensorflow/core/kernels/image/adjust_hue_op.cc similarity index 99% rename from tensorflow/core/kernels/adjust_hue_op.cc rename to tensorflow/core/kernels/image/adjust_hue_op.cc index c1993029ac6..764665be48e 100644 --- a/tensorflow/core/kernels/adjust_hue_op.cc +++ b/tensorflow/core/kernels/image/adjust_hue_op.cc @@ -17,9 +17,9 @@ limitations under the License. #define EIGEN_USE_GPU #endif -#include +#include "tensorflow/core/kernels/image/adjust_hue_op.h" -#include "tensorflow/core/kernels/adjust_hue_op.h" +#include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" diff --git a/tensorflow/core/kernels/adjust_hue_op.h b/tensorflow/core/kernels/image/adjust_hue_op.h similarity index 88% rename from tensorflow/core/kernels/adjust_hue_op.h rename to tensorflow/core/kernels/image/adjust_hue_op.h index edaf7f538e3..6a5758a44fb 100644 --- a/tensorflow/core/kernels/adjust_hue_op.h +++ b/tensorflow/core/kernels/image/adjust_hue_op.h @@ -11,8 +11,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_HUE_OP_H_ -#define TENSORFLOW_CORE_KERNELS_ADJUST_HUE_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGEADJUST_HUE_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGEADJUST_HUE_OP_H_ #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -38,4 +38,4 @@ struct AdjustHueGPU { } // namespace tensorflow #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#endif // TENSORFLOW_CORE_KERNELS_ADJUST_HUE_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGEADJUST_HUE_OP_H_ diff --git a/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc b/tensorflow/core/kernels/image/adjust_hue_op_gpu.cu.cc similarity index 93% rename from tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc rename to tensorflow/core/kernels/image/adjust_hue_op_gpu.cu.cc index 174ca0002af..10c1ddb6aaf 100644 --- a/tensorflow/core/kernels/adjust_hue_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/adjust_hue_op_gpu.cu.cc @@ -16,8 +16,8 @@ limitations under the License. #define EIGEN_USE_GPU -#include "tensorflow/core/kernels/adjust_hsv_gpu.cu.h" -#include "tensorflow/core/kernels/adjust_hue_op.h" +#include "tensorflow/core/kernels/image/adjust_hsv_gpu.cu.h" +#include "tensorflow/core/kernels/image/adjust_hue_op.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/adjust_saturation_op.cc b/tensorflow/core/kernels/image/adjust_saturation_op.cc similarity index 99% rename from tensorflow/core/kernels/adjust_saturation_op.cc rename to tensorflow/core/kernels/image/adjust_saturation_op.cc index d1fc9d349be..41b0988cc50 100644 --- a/tensorflow/core/kernels/adjust_saturation_op.cc +++ b/tensorflow/core/kernels/image/adjust_saturation_op.cc @@ -18,8 +18,10 @@ limitations under the License. #define EIGEN_USE_GPU #endif -#include "tensorflow/core/kernels/adjust_saturation_op.h" +#include "tensorflow/core/kernels/image/adjust_saturation_op.h" + #include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" diff --git a/tensorflow/core/kernels/adjust_saturation_op.h b/tensorflow/core/kernels/image/adjust_saturation_op.h similarity index 87% rename from tensorflow/core/kernels/adjust_saturation_op.h rename to tensorflow/core/kernels/image/adjust_saturation_op.h index 0117a48ead8..4a1a619e1fd 100644 --- a/tensorflow/core/kernels/adjust_saturation_op.h +++ b/tensorflow/core/kernels/image/adjust_saturation_op.h @@ -11,8 +11,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_ADJUST_SATURATION_OP_H_ -#define TENSORFLOW_CORE_KERNELS_ADJUST_SATURATION_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGEADJUST_SATURATION_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGEADJUST_SATURATION_OP_H_ #if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -38,4 +38,4 @@ struct AdjustSaturationGPU { } // namespace tensorflow #endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM -#endif // TENSORFLOW_CORE_KERNELS_ADJUST_SATURATION_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGEADJUST_SATURATION_OP_H_ diff --git a/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc b/tensorflow/core/kernels/image/adjust_saturation_op_gpu.cu.cc similarity index 93% rename from tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc rename to tensorflow/core/kernels/image/adjust_saturation_op_gpu.cu.cc index c2ef9a4d273..59541e41b46 100644 --- a/tensorflow/core/kernels/adjust_saturation_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/adjust_saturation_op_gpu.cu.cc @@ -16,8 +16,8 @@ limitations under the License. #define EIGEN_USE_GPU -#include "tensorflow/core/kernels/adjust_hsv_gpu.cu.h" -#include "tensorflow/core/kernels/adjust_saturation_op.h" +#include "tensorflow/core/kernels/image/adjust_hsv_gpu.cu.h" +#include "tensorflow/core/kernels/image/adjust_saturation_op.h" #include "tensorflow/core/util/gpu_kernel_helper.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/attention_ops.cc b/tensorflow/core/kernels/image/attention_ops.cc similarity index 100% rename from tensorflow/core/kernels/attention_ops.cc rename to tensorflow/core/kernels/image/attention_ops.cc diff --git a/tensorflow/core/kernels/colorspace_op.cc b/tensorflow/core/kernels/image/colorspace_op.cc similarity index 99% rename from tensorflow/core/kernels/colorspace_op.cc rename to tensorflow/core/kernels/image/colorspace_op.cc index 6c817f73058..a3164bb582d 100644 --- a/tensorflow/core/kernels/colorspace_op.cc +++ b/tensorflow/core/kernels/image/colorspace_op.cc @@ -16,6 +16,8 @@ limitations under the License. // See docs in ../ops/array_ops.cc. #define EIGEN_USE_THREADS +#include "tensorflow/core/kernels/image/colorspace_op.h" + #include #include @@ -26,7 +28,6 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/colorspace_op.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/types.h" diff --git a/tensorflow/core/kernels/colorspace_op.h b/tensorflow/core/kernels/image/colorspace_op.h similarity index 95% rename from tensorflow/core/kernels/colorspace_op.h rename to tensorflow/core/kernels/image/colorspace_op.h index 4de14bc3391..486aa1f5dca 100644 --- a/tensorflow/core/kernels/colorspace_op.h +++ b/tensorflow/core/kernels/image/colorspace_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_COLORSPACE_OP_H_ -#define TENSORFLOW_CORE_KERNELS_COLORSPACE_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGECOLORSPACE_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGECOLORSPACE_OP_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/tensor_shape.h" @@ -91,4 +91,4 @@ struct HSVToRGB { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_COLORSPACE_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGECOLORSPACE_OP_H_ diff --git a/tensorflow/core/kernels/colorspace_op_gpu.cu.cc b/tensorflow/core/kernels/image/colorspace_op_gpu.cu.cc similarity index 95% rename from tensorflow/core/kernels/colorspace_op_gpu.cu.cc rename to tensorflow/core/kernels/image/colorspace_op_gpu.cu.cc index 227490a2056..c49698e4c04 100644 --- a/tensorflow/core/kernels/colorspace_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/colorspace_op_gpu.cu.cc @@ -19,7 +19,7 @@ limitations under the License. #define EIGEN_USE_GPU #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/colorspace_op.h" +#include "tensorflow/core/kernels/image/colorspace_op.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/colorspace_op_test.cc b/tensorflow/core/kernels/image/colorspace_op_test.cc similarity index 100% rename from tensorflow/core/kernels/colorspace_op_test.cc rename to tensorflow/core/kernels/image/colorspace_op_test.cc diff --git a/tensorflow/core/kernels/crop_and_resize_op.cc b/tensorflow/core/kernels/image/crop_and_resize_op.cc similarity index 99% rename from tensorflow/core/kernels/crop_and_resize_op.cc rename to tensorflow/core/kernels/image/crop_and_resize_op.cc index 23058788a4b..1979b0514c6 100644 --- a/tensorflow/core/kernels/crop_and_resize_op.cc +++ b/tensorflow/core/kernels/image/crop_and_resize_op.cc @@ -17,7 +17,7 @@ limitations under the License. #define EIGEN_USE_THREADS -#include "tensorflow/core/kernels/crop_and_resize_op.h" +#include "tensorflow/core/kernels/image/crop_and_resize_op.h" #include #include diff --git a/tensorflow/core/kernels/crop_and_resize_op.h b/tensorflow/core/kernels/image/crop_and_resize_op.h similarity index 93% rename from tensorflow/core/kernels/crop_and_resize_op.h rename to tensorflow/core/kernels/image/crop_and_resize_op.h index 66ff695d9ce..c26380e395c 100644 --- a/tensorflow/core/kernels/crop_and_resize_op.h +++ b/tensorflow/core/kernels/image/crop_and_resize_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_CROP_AND_RESIZE_OP_H_ -#define TENSORFLOW_CORE_KERNELS_CROP_AND_RESIZE_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGECROP_AND_RESIZE_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGECROP_AND_RESIZE_OP_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_types.h" @@ -69,4 +69,4 @@ struct CheckValidBoxIndexHelper { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_CROP_AND_RESIZE_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGECROP_AND_RESIZE_OP_H_ diff --git a/tensorflow/core/kernels/crop_and_resize_op_benchmark_test.cc b/tensorflow/core/kernels/image/crop_and_resize_op_benchmark_test.cc similarity index 100% rename from tensorflow/core/kernels/crop_and_resize_op_benchmark_test.cc rename to tensorflow/core/kernels/image/crop_and_resize_op_benchmark_test.cc diff --git a/tensorflow/core/kernels/crop_and_resize_op_gpu.cu.cc b/tensorflow/core/kernels/image/crop_and_resize_op_gpu.cu.cc similarity index 99% rename from tensorflow/core/kernels/crop_and_resize_op_gpu.cu.cc rename to tensorflow/core/kernels/image/crop_and_resize_op_gpu.cu.cc index e64a055503f..e4bbbfa108a 100644 --- a/tensorflow/core/kernels/crop_and_resize_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/crop_and_resize_op_gpu.cu.cc @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/kernels/crop_and_resize_op.h" +#include "tensorflow/core/kernels/image/crop_and_resize_op.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/gpu_kernel_helper.h" diff --git a/tensorflow/core/kernels/crop_and_resize_op_test.cc b/tensorflow/core/kernels/image/crop_and_resize_op_test.cc similarity index 100% rename from tensorflow/core/kernels/crop_and_resize_op_test.cc rename to tensorflow/core/kernels/image/crop_and_resize_op_test.cc diff --git a/tensorflow/core/kernels/decode_image_op.cc b/tensorflow/core/kernels/image/decode_image_op.cc similarity index 100% rename from tensorflow/core/kernels/decode_image_op.cc rename to tensorflow/core/kernels/image/decode_image_op.cc diff --git a/tensorflow/core/kernels/draw_bounding_box_op.cc b/tensorflow/core/kernels/image/draw_bounding_box_op.cc similarity index 100% rename from tensorflow/core/kernels/draw_bounding_box_op.cc rename to tensorflow/core/kernels/image/draw_bounding_box_op.cc diff --git a/tensorflow/core/kernels/encode_jpeg_op.cc b/tensorflow/core/kernels/image/encode_jpeg_op.cc similarity index 100% rename from tensorflow/core/kernels/encode_jpeg_op.cc rename to tensorflow/core/kernels/image/encode_jpeg_op.cc diff --git a/tensorflow/core/kernels/encode_jpeg_op_test.cc b/tensorflow/core/kernels/image/encode_jpeg_op_test.cc similarity index 100% rename from tensorflow/core/kernels/encode_jpeg_op_test.cc rename to tensorflow/core/kernels/image/encode_jpeg_op_test.cc diff --git a/tensorflow/core/kernels/encode_png_op.cc b/tensorflow/core/kernels/image/encode_png_op.cc similarity index 100% rename from tensorflow/core/kernels/encode_png_op.cc rename to tensorflow/core/kernels/image/encode_png_op.cc diff --git a/tensorflow/core/kernels/extract_image_patches_op.cc b/tensorflow/core/kernels/image/extract_image_patches_op.cc similarity index 98% rename from tensorflow/core/kernels/extract_image_patches_op.cc rename to tensorflow/core/kernels/image/extract_image_patches_op.cc index 4e87dfc93a4..a7890090acb 100644 --- a/tensorflow/core/kernels/extract_image_patches_op.cc +++ b/tensorflow/core/kernels/image/extract_image_patches_op.cc @@ -18,7 +18,7 @@ limitations under the License. #define USE_EIGEN_TENSOR #define EIGEN_USE_THREADS -#include "tensorflow/core/kernels/extract_image_patches_op.h" +#include "tensorflow/core/kernels/image/extract_image_patches_op.h" #include diff --git a/tensorflow/core/kernels/extract_image_patches_op.h b/tensorflow/core/kernels/image/extract_image_patches_op.h similarity index 91% rename from tensorflow/core/kernels/extract_image_patches_op.h rename to tensorflow/core/kernels/image/extract_image_patches_op.h index 64b8c0338bd..ba952275c3e 100644 --- a/tensorflow/core/kernels/extract_image_patches_op.h +++ b/tensorflow/core/kernels/image/extract_image_patches_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_EXTRACT_IMAGE_PATCHES_OP_H_ -#define TENSORFLOW_CORE_KERNELS_EXTRACT_IMAGE_PATCHES_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGEEXTRACT_IMAGE_PATCHES_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGEEXTRACT_IMAGE_PATCHES_OP_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/tensor_shape.h" @@ -53,4 +53,4 @@ struct ExtractImagePatchesForward { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_EXTRACT_IMAGE_PATCHES_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGEEXTRACT_IMAGE_PATCHES_OP_H_ diff --git a/tensorflow/core/kernels/extract_image_patches_op_gpu.cu.cc b/tensorflow/core/kernels/image/extract_image_patches_op_gpu.cu.cc similarity index 94% rename from tensorflow/core/kernels/extract_image_patches_op_gpu.cu.cc rename to tensorflow/core/kernels/image/extract_image_patches_op_gpu.cu.cc index e6a49da7fd2..37b9c9bda32 100644 --- a/tensorflow/core/kernels/extract_image_patches_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/extract_image_patches_op_gpu.cu.cc @@ -19,7 +19,7 @@ limitations under the License. #define EIGEN_USE_GPU #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/extract_image_patches_op.h" +#include "tensorflow/core/kernels/image/extract_image_patches_op.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/extract_jpeg_shape_op.cc b/tensorflow/core/kernels/image/extract_jpeg_shape_op.cc similarity index 100% rename from tensorflow/core/kernels/extract_jpeg_shape_op.cc rename to tensorflow/core/kernels/image/extract_jpeg_shape_op.cc diff --git a/tensorflow/core/kernels/extract_volume_patches_op.cc b/tensorflow/core/kernels/image/extract_volume_patches_op.cc similarity index 99% rename from tensorflow/core/kernels/extract_volume_patches_op.cc rename to tensorflow/core/kernels/image/extract_volume_patches_op.cc index 3f003b6f7f6..e48e7602afa 100644 --- a/tensorflow/core/kernels/extract_volume_patches_op.cc +++ b/tensorflow/core/kernels/image/extract_volume_patches_op.cc @@ -24,7 +24,7 @@ when rates are to be added. #define USE_EIGEN_TENSOR #define EIGEN_USE_THREADS -#include "tensorflow/core/kernels/extract_volume_patches_op.h" +#include "tensorflow/core/kernels/image/extract_volume_patches_op.h" #include diff --git a/tensorflow/core/kernels/extract_volume_patches_op.h b/tensorflow/core/kernels/image/extract_volume_patches_op.h similarity index 92% rename from tensorflow/core/kernels/extract_volume_patches_op.h rename to tensorflow/core/kernels/image/extract_volume_patches_op.h index 7e0502b7707..f20ee6a6ade 100644 --- a/tensorflow/core/kernels/extract_volume_patches_op.h +++ b/tensorflow/core/kernels/image/extract_volume_patches_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_KERNELS_EXTRACT_VOLUME_PATCHES_OP_H_ -#define TENSORFLOW_KERNELS_EXTRACT_VOLUME_PATCHES_OP_H_ +#ifndef TENSORFLOW_KERNELS_IMAGE_EXTRACT_VOLUME_PATCHES_OP_H_ +#define TENSORFLOW_KERNELS_IMAGE_EXTRACT_VOLUME_PATCHES_OP_H_ #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_types.h" @@ -55,4 +55,4 @@ struct ExtractVolumePatchesForward { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_KERNELS_EXTRACT_VOLUME_PATCHES_OP_H_ +#endif // TENSORFLOW_KERNELS_IMAGE_EXTRACT_VOLUME_PATCHES_OP_H_ diff --git a/tensorflow/core/kernels/extract_volume_patches_op_gpu.cu.cc b/tensorflow/core/kernels/image/extract_volume_patches_op_gpu.cu.cc similarity index 94% rename from tensorflow/core/kernels/extract_volume_patches_op_gpu.cu.cc rename to tensorflow/core/kernels/image/extract_volume_patches_op_gpu.cu.cc index df8b6f8bfa2..379907712a8 100644 --- a/tensorflow/core/kernels/extract_volume_patches_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/extract_volume_patches_op_gpu.cu.cc @@ -17,8 +17,8 @@ limitations under the License. #define EIGEN_USE_GPU -#include "tensorflow/core/kernels/extract_volume_patches_op.h" #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/kernels/image/extract_volume_patches_op.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/generate_box_proposals_op.cu.cc b/tensorflow/core/kernels/image/generate_box_proposals_op.cu.cc similarity index 99% rename from tensorflow/core/kernels/generate_box_proposals_op.cu.cc rename to tensorflow/core/kernels/image/generate_box_proposals_op.cu.cc index b862c42d299..721d190fa22 100644 --- a/tensorflow/core/kernels/generate_box_proposals_op.cu.cc +++ b/tensorflow/core/kernels/image/generate_box_proposals_op.cu.cc @@ -24,7 +24,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/kernels/gpu_prim.h" -#include "tensorflow/core/kernels/non_max_suppression_op.h" +#include "tensorflow/core/kernels/image/non_max_suppression_op.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/stream_executor.h" diff --git a/tensorflow/core/kernels/image_ops.cc b/tensorflow/core/kernels/image/image_ops.cc similarity index 99% rename from tensorflow/core/kernels/image_ops.cc rename to tensorflow/core/kernels/image/image_ops.cc index 8792372b6ff..f121fb81654 100644 --- a/tensorflow/core/kernels/image_ops.cc +++ b/tensorflow/core/kernels/image/image_ops.cc @@ -19,7 +19,7 @@ limitations under the License. #define EIGEN_USE_GPU #endif // GOOGLE_CUDA -#include "tensorflow/core/kernels/image_ops.h" +#include "tensorflow/core/kernels/image/image_ops.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" diff --git a/tensorflow/core/kernels/image_ops.h b/tensorflow/core/kernels/image/image_ops.h similarity index 98% rename from tensorflow/core/kernels/image_ops.h rename to tensorflow/core/kernels/image/image_ops.h index e77fcbbd56a..70b47e181df 100644 --- a/tensorflow/core/kernels/image_ops.h +++ b/tensorflow/core/kernels/image/image_ops.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_IMAGE_OPS_H_ -#define TENSORFLOW_CORE_KERNELS_IMAGE_OPS_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGE_IMAGE_OPS_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGE_IMAGE_OPS_H_ // See docs in ../ops/image_ops.cc. @@ -255,4 +255,4 @@ struct FillProjectiveTransform { } // end namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_IMAGE_OPS_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGE_IMAGE_OPS_H_ diff --git a/tensorflow/core/kernels/image_ops_gpu.cu.cc b/tensorflow/core/kernels/image/image_ops_gpu.cu.cc similarity index 96% rename from tensorflow/core/kernels/image_ops_gpu.cu.cc rename to tensorflow/core/kernels/image/image_ops_gpu.cu.cc index 827fb493e4c..dd94559ffd7 100644 --- a/tensorflow/core/kernels/image_ops_gpu.cu.cc +++ b/tensorflow/core/kernels/image/image_ops_gpu.cu.cc @@ -19,7 +19,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/image_ops.h" +#include "tensorflow/core/kernels/image/image_ops.h" #include "tensorflow/core/platform/types.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/mirror_pad_op.cc b/tensorflow/core/kernels/image/mirror_pad_op.cc similarity index 99% rename from tensorflow/core/kernels/mirror_pad_op.cc rename to tensorflow/core/kernels/image/mirror_pad_op.cc index 20211c88c8b..e22b1f1adbf 100644 --- a/tensorflow/core/kernels/mirror_pad_op.cc +++ b/tensorflow/core/kernels/image/mirror_pad_op.cc @@ -17,11 +17,11 @@ limitations under the License. #define EIGEN_USE_THREADS -#include "tensorflow/core/kernels/mirror_pad_op.h" +#include "tensorflow/core/kernels/image/mirror_pad_op.h" + #include #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" - #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" diff --git a/tensorflow/core/kernels/mirror_pad_op.h b/tensorflow/core/kernels/image/mirror_pad_op.h similarity index 99% rename from tensorflow/core/kernels/mirror_pad_op.h rename to tensorflow/core/kernels/image/mirror_pad_op.h index 23ab574b8b6..8a8f84b7a64 100644 --- a/tensorflow/core/kernels/mirror_pad_op.h +++ b/tensorflow/core/kernels/image/mirror_pad_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MIRROR_PAD_OP_H_ -#define TENSORFLOW_CORE_KERNELS_MIRROR_PAD_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGE_MIRROR_PAD_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGE_MIRROR_PAD_OP_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/tensor_types.h" @@ -444,4 +444,4 @@ struct MirrorPadGrad { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_MIRROR_PAD_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGE_MIRROR_PAD_OP_H_ diff --git a/tensorflow/core/kernels/mirror_pad_op_benchmark_test.cc b/tensorflow/core/kernels/image/mirror_pad_op_benchmark_test.cc similarity index 100% rename from tensorflow/core/kernels/mirror_pad_op_benchmark_test.cc rename to tensorflow/core/kernels/image/mirror_pad_op_benchmark_test.cc diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl.h similarity index 83% rename from tensorflow/core/kernels/mirror_pad_op_cpu_impl.h rename to tensorflow/core/kernels/image/mirror_pad_op_cpu_impl.h index 45e6676e5a6..7a7c263c526 100644 --- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl.h +++ b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl.h @@ -13,13 +13,14 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_MIRROR_PAD_OP_CPU_IMPL_H_ -#define TENSORFLOW_CORE_KERNELS_MIRROR_PAD_OP_CPU_IMPL_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGE_MIRROR_PAD_OP_CPU_IMPL_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGE_MIRROR_PAD_OP_CPU_IMPL_H_ +#if CPU_PROVIDED_IXDIM #define EIGEN_USE_THREADS #include "tensorflow/core/framework/register_types.h" -#include "tensorflow/core/kernels/mirror_pad_op.h" +#include "tensorflow/core/kernels/image/mirror_pad_op.h" namespace tensorflow { @@ -39,7 +40,7 @@ TF_CALL_tstring(DEFINE_CPU_SPECS); CPU_PROVIDED_IXDIM>; TF_CALL_NUMBER_TYPES(DEFINE_CPU_SPECS); #undef DEFINE_CPU_SPECS - } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_MIRROR_PAD_OP_CPU_IMPL_H_ +#endif // CPU_PROVIDED_IXDIM +#endif // TENSORFLOW_CORE_KERNELS_IMAGE_MIRROR_PAD_OP_CPU_IMPL_H_ diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_1.cc b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_1.cc similarity index 91% rename from tensorflow/core/kernels/mirror_pad_op_cpu_impl_1.cc rename to tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_1.cc index 140c487221f..ad64170aa0f 100644 --- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_1.cc +++ b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_1.cc @@ -14,5 +14,5 @@ limitations under the License. ==============================================================================*/ #define CPU_PROVIDED_IXDIM 1 -#include "tensorflow/core/kernels/mirror_pad_op_cpu_impl.h" +#include "tensorflow/core/kernels/image/mirror_pad_op_cpu_impl.h" #undef CPU_PROVIDED_IXDIM diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_2.cc b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_2.cc similarity index 91% rename from tensorflow/core/kernels/mirror_pad_op_cpu_impl_2.cc rename to tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_2.cc index d67f7754e1d..76096f78030 100644 --- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_2.cc +++ b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_2.cc @@ -14,5 +14,5 @@ limitations under the License. ==============================================================================*/ #define CPU_PROVIDED_IXDIM 2 -#include "tensorflow/core/kernels/mirror_pad_op_cpu_impl.h" +#include "tensorflow/core/kernels/image/mirror_pad_op_cpu_impl.h" #undef CPU_PROVIDED_IXDIM diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_3.cc b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_3.cc similarity index 91% rename from tensorflow/core/kernels/mirror_pad_op_cpu_impl_3.cc rename to tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_3.cc index 096547f1f9c..3c29e87bc45 100644 --- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_3.cc +++ b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_3.cc @@ -14,5 +14,5 @@ limitations under the License. ==============================================================================*/ #define CPU_PROVIDED_IXDIM 3 -#include "tensorflow/core/kernels/mirror_pad_op_cpu_impl.h" +#include "tensorflow/core/kernels/image/mirror_pad_op_cpu_impl.h" #undef CPU_PROVIDED_IXDIM diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_4.cc b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_4.cc similarity index 91% rename from tensorflow/core/kernels/mirror_pad_op_cpu_impl_4.cc rename to tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_4.cc index 5a7455f3c07..5d1a3400054 100644 --- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_4.cc +++ b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_4.cc @@ -14,5 +14,5 @@ limitations under the License. ==============================================================================*/ #define CPU_PROVIDED_IXDIM 4 -#include "tensorflow/core/kernels/mirror_pad_op_cpu_impl.h" +#include "tensorflow/core/kernels/image/mirror_pad_op_cpu_impl.h" #undef CPU_PROVIDED_IXDIM diff --git a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_5.cc b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_5.cc similarity index 91% rename from tensorflow/core/kernels/mirror_pad_op_cpu_impl_5.cc rename to tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_5.cc index ed2db03a8f5..71a6c9307c6 100644 --- a/tensorflow/core/kernels/mirror_pad_op_cpu_impl_5.cc +++ b/tensorflow/core/kernels/image/mirror_pad_op_cpu_impl_5.cc @@ -14,5 +14,5 @@ limitations under the License. ==============================================================================*/ #define CPU_PROVIDED_IXDIM 5 -#include "tensorflow/core/kernels/mirror_pad_op_cpu_impl.h" +#include "tensorflow/core/kernels/image/mirror_pad_op_cpu_impl.h" #undef CPU_PROVIDED_IXDIM diff --git a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc b/tensorflow/core/kernels/image/mirror_pad_op_gpu.cu.cc similarity index 97% rename from tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc rename to tensorflow/core/kernels/image/mirror_pad_op_gpu.cu.cc index ac89599714d..f0afc707fc6 100644 --- a/tensorflow/core/kernels/mirror_pad_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/mirror_pad_op_gpu.cu.cc @@ -17,9 +17,8 @@ limitations under the License. #define EIGEN_USE_GPU -#include "tensorflow/core/kernels/mirror_pad_op.h" - #include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/kernels/image/mirror_pad_op.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/mirror_pad_op_test.cc b/tensorflow/core/kernels/image/mirror_pad_op_test.cc similarity index 100% rename from tensorflow/core/kernels/mirror_pad_op_test.cc rename to tensorflow/core/kernels/image/mirror_pad_op_test.cc diff --git a/tensorflow/core/kernels/non_max_suppression_op.cc b/tensorflow/core/kernels/image/non_max_suppression_op.cc similarity index 99% rename from tensorflow/core/kernels/non_max_suppression_op.cc rename to tensorflow/core/kernels/image/non_max_suppression_op.cc index 20ae3a2e0d0..701753a81d6 100644 --- a/tensorflow/core/kernels/non_max_suppression_op.cc +++ b/tensorflow/core/kernels/image/non_max_suppression_op.cc @@ -17,7 +17,7 @@ limitations under the License. #define EIGEN_USE_THREADS -#include "tensorflow/core/kernels/non_max_suppression_op.h" +#include "tensorflow/core/kernels/image/non_max_suppression_op.h" #include #include diff --git a/tensorflow/core/kernels/non_max_suppression_op.cu.cc b/tensorflow/core/kernels/image/non_max_suppression_op.cu.cc similarity index 99% rename from tensorflow/core/kernels/non_max_suppression_op.cu.cc rename to tensorflow/core/kernels/image/non_max_suppression_op.cu.cc index 8ec26ba13d7..37d7d42e438 100644 --- a/tensorflow/core/kernels/non_max_suppression_op.cu.cc +++ b/tensorflow/core/kernels/image/non_max_suppression_op.cu.cc @@ -23,7 +23,7 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/kernels/gpu_prim.h" -#include "tensorflow/core/kernels/non_max_suppression_op.h" +#include "tensorflow/core/kernels/image/non_max_suppression_op.h" #include "tensorflow/core/util/gpu_kernel_helper.h" #include "tensorflow/core/util/gpu_launch_config.h" #include "tensorflow/stream_executor/stream_executor.h" diff --git a/tensorflow/core/kernels/non_max_suppression_op.h b/tensorflow/core/kernels/image/non_max_suppression_op.h similarity index 92% rename from tensorflow/core/kernels/non_max_suppression_op.h rename to tensorflow/core/kernels/image/non_max_suppression_op.h index 24957c2bbed..d6d3b68b099 100644 --- a/tensorflow/core/kernels/non_max_suppression_op.h +++ b/tensorflow/core/kernels/image/non_max_suppression_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_NON_MAX_SUPPRESSION_OP_H_ -#define TENSORFLOW_CORE_KERNELS_NON_MAX_SUPPRESSION_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGENON_MAX_SUPPRESSION_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGENON_MAX_SUPPRESSION_OP_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_types.h" @@ -59,4 +59,4 @@ Status NmsGpu(const float* d_sorted_boxes_float_ptr, const int num_boxes, } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_NON_MAX_SUPPRESSION_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGENON_MAX_SUPPRESSION_OP_H_ diff --git a/tensorflow/core/kernels/non_max_suppression_op_benchmark_test.cc b/tensorflow/core/kernels/image/non_max_suppression_op_benchmark_test.cc similarity index 100% rename from tensorflow/core/kernels/non_max_suppression_op_benchmark_test.cc rename to tensorflow/core/kernels/image/non_max_suppression_op_benchmark_test.cc diff --git a/tensorflow/core/kernels/non_max_suppression_op_gpu_test.cc b/tensorflow/core/kernels/image/non_max_suppression_op_gpu_test.cc similarity index 100% rename from tensorflow/core/kernels/non_max_suppression_op_gpu_test.cc rename to tensorflow/core/kernels/image/non_max_suppression_op_gpu_test.cc diff --git a/tensorflow/core/kernels/non_max_suppression_op_test.cc b/tensorflow/core/kernels/image/non_max_suppression_op_test.cc similarity index 100% rename from tensorflow/core/kernels/non_max_suppression_op_test.cc rename to tensorflow/core/kernels/image/non_max_suppression_op_test.cc diff --git a/tensorflow/core/kernels/random_crop_op.cc b/tensorflow/core/kernels/image/random_crop_op.cc similarity index 99% rename from tensorflow/core/kernels/random_crop_op.cc rename to tensorflow/core/kernels/image/random_crop_op.cc index eb7980fa58e..7da97466636 100644 --- a/tensorflow/core/kernels/random_crop_op.cc +++ b/tensorflow/core/kernels/image/random_crop_op.cc @@ -63,6 +63,7 @@ class RandomCropOp : public OpKernel { if ((target_height == height) && (target_width == width)) { *output = context->input(0); } + OP_REQUIRES(context, width >= target_width, errors::FailedPrecondition( "width must be >= target_width: width = ", width, diff --git a/tensorflow/core/kernels/resize_area_op.cc b/tensorflow/core/kernels/image/resize_area_op.cc similarity index 99% rename from tensorflow/core/kernels/resize_area_op.cc rename to tensorflow/core/kernels/image/resize_area_op.cc index 325c5ccade1..00691ae46b0 100644 --- a/tensorflow/core/kernels/resize_area_op.cc +++ b/tensorflow/core/kernels/image/resize_area_op.cc @@ -18,15 +18,16 @@ limitations under the License. #include #include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/image_resizer_state.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/resize_area_op_test.cc b/tensorflow/core/kernels/image/resize_area_op_test.cc similarity index 100% rename from tensorflow/core/kernels/resize_area_op_test.cc rename to tensorflow/core/kernels/image/resize_area_op_test.cc diff --git a/tensorflow/core/kernels/resize_bicubic_op.cc b/tensorflow/core/kernels/image/resize_bicubic_op.cc similarity index 99% rename from tensorflow/core/kernels/resize_bicubic_op.cc rename to tensorflow/core/kernels/image/resize_bicubic_op.cc index 48bd1986b7b..89f34cb80f0 100644 --- a/tensorflow/core/kernels/resize_bicubic_op.cc +++ b/tensorflow/core/kernels/image/resize_bicubic_op.cc @@ -17,6 +17,7 @@ limitations under the License. #define EIGEN_USE_THREADS #include + #include #include @@ -26,9 +27,9 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/image_resizer_state.h" namespace tensorflow { namespace { diff --git a/tensorflow/core/kernels/resize_bicubic_op_test.cc b/tensorflow/core/kernels/image/resize_bicubic_op_test.cc similarity index 100% rename from tensorflow/core/kernels/resize_bicubic_op_test.cc rename to tensorflow/core/kernels/image/resize_bicubic_op_test.cc diff --git a/tensorflow/core/kernels/resize_bilinear_op.cc b/tensorflow/core/kernels/image/resize_bilinear_op.cc similarity index 99% rename from tensorflow/core/kernels/resize_bilinear_op.cc rename to tensorflow/core/kernels/image/resize_bilinear_op.cc index a0673fea73d..b9eb650c029 100644 --- a/tensorflow/core/kernels/resize_bilinear_op.cc +++ b/tensorflow/core/kernels/image/resize_bilinear_op.cc @@ -16,22 +16,23 @@ limitations under the License. // See docs in ../ops/image_ops.cc #define EIGEN_USE_THREADS -#include "tensorflow/core/kernels/resize_bilinear_op.h" +#include "tensorflow/core/kernels/image/resize_bilinear_op.h" #ifdef __SSE4_1__ #include #endif #include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/image_resizer_state.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/resize_bilinear_op.h b/tensorflow/core/kernels/image/resize_bilinear_op.h similarity index 90% rename from tensorflow/core/kernels/resize_bilinear_op.h rename to tensorflow/core/kernels/image/resize_bilinear_op.h index b4d0066d4f3..34a6b320251 100644 --- a/tensorflow/core/kernels/resize_bilinear_op.h +++ b/tensorflow/core/kernels/image/resize_bilinear_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_RESIZE_BILINEAR_OP_H_ -#define TENSORFLOW_CORE_KERNELS_RESIZE_BILINEAR_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGERESIZE_BILINEAR_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGERESIZE_BILINEAR_OP_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_types.h" @@ -43,4 +43,4 @@ struct ResizeBilinearGrad { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_RESIZE_BILINEAR_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGERESIZE_BILINEAR_OP_H_ diff --git a/tensorflow/core/kernels/resize_bilinear_op_gpu.cu.cc b/tensorflow/core/kernels/image/resize_bilinear_op_gpu.cu.cc similarity index 99% rename from tensorflow/core/kernels/resize_bilinear_op_gpu.cu.cc rename to tensorflow/core/kernels/image/resize_bilinear_op_gpu.cu.cc index 42a3daae116..aa475a4a3af 100644 --- a/tensorflow/core/kernels/resize_bilinear_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/resize_bilinear_op_gpu.cu.cc @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/kernels/resize_bilinear_op.h" +#include "tensorflow/core/kernels/image/resize_bilinear_op.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/gpu_kernel_helper.h" diff --git a/tensorflow/core/kernels/resize_bilinear_op_test.cc b/tensorflow/core/kernels/image/resize_bilinear_op_test.cc similarity index 100% rename from tensorflow/core/kernels/resize_bilinear_op_test.cc rename to tensorflow/core/kernels/image/resize_bilinear_op_test.cc diff --git a/tensorflow/core/kernels/resize_nearest_neighbor_op.cc b/tensorflow/core/kernels/image/resize_nearest_neighbor_op.cc similarity index 99% rename from tensorflow/core/kernels/resize_nearest_neighbor_op.cc rename to tensorflow/core/kernels/image/resize_nearest_neighbor_op.cc index 4a357333957..a3c6a69a692 100644 --- a/tensorflow/core/kernels/resize_nearest_neighbor_op.cc +++ b/tensorflow/core/kernels/image/resize_nearest_neighbor_op.cc @@ -16,7 +16,7 @@ limitations under the License. // See docs in ../ops/image_ops.cc #define EIGEN_USE_THREADS -#include "tensorflow/core/kernels/resize_nearest_neighbor_op.h" +#include "tensorflow/core/kernels/image/resize_nearest_neighbor_op.h" #include @@ -26,9 +26,9 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" +#include "tensorflow/core/util/image_resizer_state.h" namespace tensorflow { diff --git a/tensorflow/core/kernels/resize_nearest_neighbor_op.h b/tensorflow/core/kernels/image/resize_nearest_neighbor_op.h similarity index 88% rename from tensorflow/core/kernels/resize_nearest_neighbor_op.h rename to tensorflow/core/kernels/image/resize_nearest_neighbor_op.h index d6b053180ce..db0276477eb 100644 --- a/tensorflow/core/kernels/resize_nearest_neighbor_op.h +++ b/tensorflow/core/kernels/image/resize_nearest_neighbor_op.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_RESIZE_NEAREST_NEIGHBOR_OP_H_ -#define TENSORFLOW_CORE_KERNELS_RESIZE_NEAREST_NEIGHBOR_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGERESIZE_NEAREST_NEIGHBOR_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGERESIZE_NEAREST_NEIGHBOR_OP_H_ #include "tensorflow/core/framework/tensor_types.h" #include "tensorflow/core/platform/types.h" @@ -42,4 +42,4 @@ struct ResizeNearestNeighborGrad { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_RESIZE_NEAREST_NEIGHBOR_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGERESIZE_NEAREST_NEIGHBOR_OP_H_ diff --git a/tensorflow/core/kernels/resize_nearest_neighbor_op_gpu.cu.cc b/tensorflow/core/kernels/image/resize_nearest_neighbor_op_gpu.cu.cc similarity index 99% rename from tensorflow/core/kernels/resize_nearest_neighbor_op_gpu.cu.cc rename to tensorflow/core/kernels/image/resize_nearest_neighbor_op_gpu.cu.cc index b6a9c77ba13..50066d5b653 100644 --- a/tensorflow/core/kernels/resize_nearest_neighbor_op_gpu.cu.cc +++ b/tensorflow/core/kernels/image/resize_nearest_neighbor_op_gpu.cu.cc @@ -21,7 +21,7 @@ limitations under the License. #include "tensorflow/core/framework/register_types.h" #include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/kernels/resize_nearest_neighbor_op.h" +#include "tensorflow/core/kernels/image/resize_nearest_neighbor_op.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/gpu_kernel_helper.h" diff --git a/tensorflow/core/kernels/resize_nearest_neighbor_op_test.cc b/tensorflow/core/kernels/image/resize_nearest_neighbor_op_test.cc similarity index 100% rename from tensorflow/core/kernels/resize_nearest_neighbor_op_test.cc rename to tensorflow/core/kernels/image/resize_nearest_neighbor_op_test.cc diff --git a/tensorflow/core/kernels/resize_op_benchmark_test.cc b/tensorflow/core/kernels/image/resize_op_benchmark_test.cc similarity index 100% rename from tensorflow/core/kernels/resize_op_benchmark_test.cc rename to tensorflow/core/kernels/image/resize_op_benchmark_test.cc diff --git a/tensorflow/core/kernels/sample_distorted_bounding_box_op.cc b/tensorflow/core/kernels/image/sample_distorted_bounding_box_op.cc similarity index 100% rename from tensorflow/core/kernels/sample_distorted_bounding_box_op.cc rename to tensorflow/core/kernels/image/sample_distorted_bounding_box_op.cc diff --git a/tensorflow/core/kernels/sampling_kernels.cc b/tensorflow/core/kernels/image/sampling_kernels.cc similarity index 96% rename from tensorflow/core/kernels/sampling_kernels.cc rename to tensorflow/core/kernels/image/sampling_kernels.cc index 306b8d6a390..ae62a1b2e3d 100644 --- a/tensorflow/core/kernels/sampling_kernels.cc +++ b/tensorflow/core/kernels/image/sampling_kernels.cc @@ -13,8 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/sampling_kernels.h" +#include "tensorflow/core/kernels/image/sampling_kernels.h" + #include + #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/strings/str_util.h" diff --git a/tensorflow/core/kernels/sampling_kernels.h b/tensorflow/core/kernels/image/sampling_kernels.h similarity index 100% rename from tensorflow/core/kernels/sampling_kernels.h rename to tensorflow/core/kernels/image/sampling_kernels.h diff --git a/tensorflow/core/kernels/sampling_kernels_test.cc b/tensorflow/core/kernels/image/sampling_kernels_test.cc similarity index 98% rename from tensorflow/core/kernels/sampling_kernels_test.cc rename to tensorflow/core/kernels/image/sampling_kernels_test.cc index 37c2edc14a3..039a785063f 100644 --- a/tensorflow/core/kernels/sampling_kernels_test.cc +++ b/tensorflow/core/kernels/image/sampling_kernels_test.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow/core/kernels/sampling_kernels.h" +#include "tensorflow/core/kernels/image/sampling_kernels.h" #include "tensorflow/core/platform/test.h" diff --git a/tensorflow/core/kernels/scale_and_translate_op.cc b/tensorflow/core/kernels/image/scale_and_translate_op.cc similarity index 99% rename from tensorflow/core/kernels/scale_and_translate_op.cc rename to tensorflow/core/kernels/image/scale_and_translate_op.cc index fff457e55c7..1011af7d19e 100644 --- a/tensorflow/core/kernels/scale_and_translate_op.cc +++ b/tensorflow/core/kernels/image/scale_and_translate_op.cc @@ -16,9 +16,10 @@ limitations under the License. // See docs in ../ops/image_ops.cc #define EIGEN_USE_THREADS -#include "tensorflow/core/kernels/scale_and_translate_op.h" +#include "tensorflow/core/kernels/image/scale_and_translate_op.h" #include + #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/bounds_check.h" #include "tensorflow/core/framework/op_kernel.h" @@ -27,7 +28,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/kernels/sampling_kernels.h" +#include "tensorflow/core/kernels/image/sampling_kernels.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/logging.h" diff --git a/tensorflow/core/kernels/scale_and_translate_op.h b/tensorflow/core/kernels/image/scale_and_translate_op.h similarity index 92% rename from tensorflow/core/kernels/scale_and_translate_op.h rename to tensorflow/core/kernels/image/scale_and_translate_op.h index 74bc87ecc7a..9c0650a4c26 100644 --- a/tensorflow/core/kernels/scale_and_translate_op.h +++ b/tensorflow/core/kernels/image/scale_and_translate_op.h @@ -13,14 +13,14 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_CORE_KERNELS_SCALE_AND_TRANSLATE_OP_H_ -#define TENSORFLOW_CORE_KERNELS_SCALE_AND_TRANSLATE_OP_H_ +#ifndef TENSORFLOW_CORE_KERNELS_IMAGESCALE_AND_TRANSLATE_OP_H_ +#define TENSORFLOW_CORE_KERNELS_IMAGESCALE_AND_TRANSLATE_OP_H_ #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/numeric_types.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/kernels/sampling_kernels.h" +#include "tensorflow/core/kernels/image/sampling_kernels.h" namespace tensorflow { namespace functor { @@ -72,4 +72,4 @@ struct GatherSpans { } // namespace functor } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_SCALE_AND_TRANSLATE_OP_H_ +#endif // TENSORFLOW_CORE_KERNELS_IMAGESCALE_AND_TRANSLATE_OP_H_ diff --git a/tensorflow/core/kernels/scale_and_translate_op_test.cc b/tensorflow/core/kernels/image/scale_and_translate_op_test.cc similarity index 99% rename from tensorflow/core/kernels/scale_and_translate_op_test.cc rename to tensorflow/core/kernels/image/scale_and_translate_op_test.cc index 412a1012686..2959f93a266 100644 --- a/tensorflow/core/kernels/scale_and_translate_op_test.cc +++ b/tensorflow/core/kernels/image/scale_and_translate_op_test.cc @@ -21,9 +21,9 @@ limitations under the License. #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/framework/types.h" #include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/core/kernels/image/sampling_kernels.h" #include "tensorflow/core/kernels/ops_testutil.h" #include "tensorflow/core/kernels/ops_util.h" -#include "tensorflow/core/kernels/sampling_kernels.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/random/random.h" #include "tensorflow/core/lib/random/simple_philox.h" diff --git a/tensorflow/core/kernels/linalg/BUILD b/tensorflow/core/kernels/linalg/BUILD index c735f58ae51..ab25fad3ec3 100644 --- a/tensorflow/core/kernels/linalg/BUILD +++ b/tensorflow/core/kernels/linalg/BUILD @@ -10,19 +10,6 @@ load( ) load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test") -# Description: -# Op kernel implementations for TensorFlow. -# -# Note: Any test that uses GPU support and which we would like to -# benchmark should be linked statically so that it can be executed -# from a py_binary or cuda_py_test test logger. For such a test, -# append "_gpu" to the test name to invoke the GPU benchmarks. Example: -# -# # for CPU tests -# $ bazel test --config opt //third_party/tensorflow/core/kernels:my_op_test -# # for GPU benchmarks -# $ bazel run --config opt --config=cuda //third_party/tensorflow/core/kernels:my_op_test_gpu -- --benchmarks=.. -# package( default_visibility = [ "//tensorflow:__subpackages__", diff --git a/tensorflow/core/kernels/mkl/BUILD b/tensorflow/core/kernels/mkl/BUILD index 4abeee20e30..16180a5b7bd 100644 --- a/tensorflow/core/kernels/mkl/BUILD +++ b/tensorflow/core/kernels/mkl/BUILD @@ -242,16 +242,8 @@ tf_mkl_kernel_library( name = "mkl_dequantize_op", srcs = ["mkl_dequantize_op.cc"], deps = [ - "//tensorflow/core/kernels:concat_lib_hdrs", - "//tensorflow/core/kernels:conv_ops", - "//tensorflow/core/kernels:cwise_op", - "//tensorflow/core/kernels:eigen_helpers", - "//tensorflow/core/kernels:image_resizer_state", - "//tensorflow/core/kernels:ops_util", - "//tensorflow/core/kernels:pooling_ops", - "//tensorflow/core/kernels:quantization_utils", - "//tensorflow/core/kernels:quantized_ops", - "//tensorflow/core/kernels:transpose_functor", + "//third_party/eigen3", + "@gemmlowp", "//tensorflow/core:array_ops_op_lib", "//tensorflow/core:core_cpu", "//tensorflow/core:framework", @@ -259,8 +251,16 @@ tf_mkl_kernel_library( "//tensorflow/core:math_ops_op_lib", "//tensorflow/core:mkl_graph_util", "//tensorflow/core:nn_ops_op_lib", - "//third_party/eigen3", - "@gemmlowp", + "//tensorflow/core/kernels:concat_lib_hdrs", + "//tensorflow/core/kernels:conv_ops", + "//tensorflow/core/kernels:cwise_op", + "//tensorflow/core/kernels:eigen_helpers", + "//tensorflow/core/kernels:ops_util", + "//tensorflow/core/kernels:pooling_ops", + "//tensorflow/core/kernels:quantization_utils", + "//tensorflow/core/kernels:quantized_ops", + "//tensorflow/core/kernels:transpose_functor", + "//tensorflow/core/util:image_resizer_state", ] + mkl_deps(), ) @@ -373,15 +373,15 @@ tf_mkl_kernel_library( "mkl_requantize_per_channel_op.cc", ], deps = [ + "@gemmlowp", "//tensorflow/core/kernels:concat_lib_hdrs", "//tensorflow/core/kernels:conv_ops", "//tensorflow/core/kernels:eigen_helpers", - "//tensorflow/core/kernels:image_resizer_state", "//tensorflow/core/kernels:meta_support", "//tensorflow/core/kernels:no_op", "//tensorflow/core/kernels:pooling_ops", "//tensorflow/core/kernels:quantization_utils", - "@gemmlowp", + "//tensorflow/core/util:image_resizer_state", ] + MKL_DEPS, ) @@ -412,10 +412,10 @@ tf_cc_test_mkl( "//tensorflow/core/kernels:bias_op", "//tensorflow/core/kernels:conv_ops", "//tensorflow/core/kernels:depthwise_conv_op", - "//tensorflow/core/kernels:image", "//tensorflow/core/kernels:matmul_op", "//tensorflow/core/kernels:pad_op", "//tensorflow/core/kernels:relu_op", + "//tensorflow/core/kernels/image:image", ] + MKL_TEST_DEPS, ) diff --git a/tensorflow/core/kernels/quantized_resize_bilinear_op.cc b/tensorflow/core/kernels/quantized_resize_bilinear_op.cc index 4da56cde547..da0a35a6554 100644 --- a/tensorflow/core/kernels/quantized_resize_bilinear_op.cc +++ b/tensorflow/core/kernels/quantized_resize_bilinear_op.cc @@ -25,9 +25,9 @@ limitations under the License. #include "tensorflow/core/framework/op_kernel.h" #include "tensorflow/core/framework/types.h" -#include "tensorflow/core/kernels/image_resizer_state.h" #include "tensorflow/core/kernels/quantization_utils.h" #include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/util/image_resizer_state.h" namespace tensorflow { diff --git a/tensorflow/core/util/BUILD b/tensorflow/core/util/BUILD index 634a937d1c4..4d2ff9a8058 100644 --- a/tensorflow/core/util/BUILD +++ b/tensorflow/core/util/BUILD @@ -40,11 +40,12 @@ package( licenses = ["notice"], # Apache 2.0 ) -# List of exported proto source files. +# List of exported source files. exports_files( srcs = [ "event.proto", "example_proto_fast_parsing_test.proto", + "image_resizer_state.h", "memmapped_file_system.proto", "saved_tensor_slice.proto", ], @@ -631,8 +632,18 @@ tf_kernel_library( ]), ) -# Tests. +cc_library( + name = "image_resizer_state", + hdrs = ["image_resizer_state.h"], + deps = [ + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core/kernels:bounds_check", + "//third_party/eigen3", + ], +) +# Tests. tf_cc_test( name = "overflow_test", size = "small", diff --git a/tensorflow/core/kernels/image_resizer_state.h b/tensorflow/core/util/image_resizer_state.h similarity index 98% rename from tensorflow/core/kernels/image_resizer_state.h rename to tensorflow/core/util/image_resizer_state.h index 1b1550fd47a..b302021918d 100644 --- a/tensorflow/core/kernels/image_resizer_state.h +++ b/tensorflow/core/util/image_resizer_state.h @@ -18,12 +18,12 @@ limitations under the License. // reduce code duplication and ensure consistency across the different // resizers, it performs the input validation. -#ifndef TENSORFLOW_CORE_KERNELS_IMAGE_RESIZER_STATE_H_ -#define TENSORFLOW_CORE_KERNELS_IMAGE_RESIZER_STATE_H_ +#ifndef TENSORFLOW_CORE_KERNELS_UTIL_IMAGE_RESIZER_STATE_H_ +#define TENSORFLOW_CORE_KERNELS_UTIL_IMAGE_RESIZER_STATE_H_ #define EIGEN_USE_THREADS - #include + #include #include @@ -228,4 +228,4 @@ struct ImageResizerGradientState { } // namespace tensorflow -#endif // TENSORFLOW_CORE_KERNELS_IMAGE_RESIZER_STATE_H_ +#endif // TENSORFLOW_CORE_KERNELS_UTIL_IMAGE_RESIZER_STATE_H_ diff --git a/tensorflow/examples/label_image/BUILD b/tensorflow/examples/label_image/BUILD index a0e5005d45a..7c3a6dca1b2 100644 --- a/tensorflow/examples/label_image/BUILD +++ b/tensorflow/examples/label_image/BUILD @@ -38,7 +38,7 @@ tf_cc_binary( "//tensorflow/core:portable_tensorflow_lib", # cc:android_tensorflow_image_op is for including jpeg/gif/png # decoder to enable real-image evaluation on Android - "//tensorflow/core/kernels:android_tensorflow_image_op", + "//tensorflow/core/kernels/image:android_tensorflow_image_op", ], "//conditions:default": [ "//tensorflow/cc:cc_ops", From a392a47994310363af835146586eb9c3b8f57466 Mon Sep 17 00:00:00 2001 From: Rahul Joshi Date: Tue, 11 Aug 2020 10:59:27 -0700 Subject: [PATCH 0829/1017] [MLIR] Handle while loops correctly in ResourceAliasAnalysis - When arguments pass through the while loop body to results, the existing code was not handling the loop back branch correctly. - Fixed the code to handle loops correctly by propagating resource id's along the loop back edge till the results converge. PiperOrigin-RevId: 326056635 Change-Id: I3dde7f7c9b63be15d47c00ce2996d15e614ecc6c --- .../analysis/resource_alias_analysis.cc | 145 ++++++++++++------ .../analysis/resource_alias_analysis.h | 17 +- .../tests/resource-alias-analysis-test.mlir | 16 +- 3 files changed, 116 insertions(+), 62 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc index 53de595eef2..256217b6542 100644 --- a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc +++ b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc @@ -45,8 +45,8 @@ limitations under the License. namespace mlir { namespace TF { +namespace detail { -namespace { //===----------------------------------------------------------------------===// // BacktrackAnalysisInfo //===----------------------------------------------------------------------===// @@ -86,9 +86,6 @@ class BacktrackAnalysisInfo { // Backtracked values indexed by the result number. llvm::SmallVector backtracked_values_; }; -} // namespace - -namespace detail { //===----------------------------------------------------------------------===// // BacktrackAnalysis @@ -169,9 +166,6 @@ Value BacktrackAnalysis::BacktrackValue(Value value) { } return value; } -} // namespace detail - -namespace { // Analyze the region. BacktrackAnalysisInfo::BacktrackAnalysisInfo( @@ -188,6 +182,8 @@ BacktrackAnalysisInfo::BacktrackAnalysisInfo( backtracked_values_.push_back(backtrack_analysis.BacktrackValue(result)); } +namespace { + //===----------------------------------------------------------------------===// // ResourceAliasAnalysisInfo helper functions. //===----------------------------------------------------------------------===// @@ -224,14 +220,13 @@ int64_t GetOrCreateIdForVarHandle(VarHandleOp handle, int64_t* next_id, } // namespace -namespace detail { //===----------------------------------------------------------------------===// // ResourceAliasAnalysisInfo //===----------------------------------------------------------------------===// // Constructs the analysis info by analyzing the given function. ResourceAliasAnalysisInfo::ResourceAliasAnalysisInfo( - FuncOp func_op, const detail::BacktrackAnalysis& backtrack_analysis) { + FuncOp func_op, const BacktrackAnalysis& backtrack_analysis) { // This function populates resource_value_to_ids_ and id_to_resource_values_. int64_t next_unique_id = 0; @@ -293,15 +288,6 @@ ResourceAliasAnalysisInfo::ResourceAliasAnalysisInfo( }); llvm::StringMap var_handle_name_id_map; - auto forward_input_to_output = [&](const Value& operand, - const OpResult& result) { - auto operand_it = resource_value_to_ids_.find(operand); - assert(operand_it != resource_value_to_ids_.end() && - "A resource-type output does not have the corresponding " - "resource-type input."); - for (int64_t id : operand_it->second) AddValueUniqueIDMapping(result, id); - }; - func_op.walk([&](Operation* op) { if (auto var_handle = dyn_cast(op)) { AddValueUniqueIDMapping( @@ -310,36 +296,14 @@ ResourceAliasAnalysisInfo::ResourceAliasAnalysisInfo( &var_handle_name_id_map)); } else if (llvm::isa(op)) { for (auto result : filter_resources(op->getResults())) - forward_input_to_output(op->getOperand(result.getResultNumber()), - result); + PropagateInputToOutput(op->getOperand(result.getResultNumber()), + result); } else if (auto while_op = dyn_cast(op)) { - const auto& body_info = - backtrack_analysis.GetAnalysisForFunc(while_op.body_func()); - // If a result is a passthrough of the body input, use the corresponding - // operand's resource IDs. - for (auto result : filter_resources(while_op.getResults())) { - auto passthrough_arg = body_info.GetArg(result.getResultNumber()); - if (passthrough_arg) { - forward_input_to_output( - while_op.getOperand(passthrough_arg.getValue()), result); - } else { - AddValueUniqueIDMapping(result, kUnknownResourceId); - } - } + AnalyzeWhileLoop(while_op, backtrack_analysis.GetAnalysisForFunc( + while_op.body_func())); } else if (auto while_region = dyn_cast(op)) { - const auto& body_info = - backtrack_analysis.GetAnalysisForRegion(while_region.body()); - // If a result is a passthrough of the body input, use the corresponding - // operand's resource IDs. - for (auto result : filter_resources(while_region.getResults())) { - auto passthrough_arg = body_info.GetArg(result.getResultNumber()); - if (passthrough_arg) { - forward_input_to_output( - while_region.getOperand(passthrough_arg.getValue()), result); - } else { - AddValueUniqueIDMapping(result, kUnknownResourceId); - } - } + AnalyzeWhileLoop(while_region, backtrack_analysis.GetAnalysisForRegion( + while_region.body())); } else if (auto if_op = dyn_cast(op)) { const auto& then_info = backtrack_analysis.GetAnalysisForFunc(if_op.then_func()); @@ -353,8 +317,8 @@ ResourceAliasAnalysisInfo::ResourceAliasAnalysisInfo( if (passthrough_then_arg && passthrough_else_arg) { Value then_operand = if_op.input()[passthrough_then_arg.getValue()]; Value else_operand = if_op.input()[passthrough_else_arg.getValue()]; - forward_input_to_output(then_operand, result); - forward_input_to_output(else_operand, result); + PropagateInputToOutput(then_operand, result); + PropagateInputToOutput(else_operand, result); } else { AddValueUniqueIDMapping(result, kUnknownResourceId); } @@ -374,8 +338,8 @@ ResourceAliasAnalysisInfo::ResourceAliasAnalysisInfo( // IfRegion, it will have been visited earlier and a mapping would // exist for that value. If its computed within the region, then again // a mapping would exist. - forward_input_to_output(then_result, result); - forward_input_to_output(else_result, result); + PropagateInputToOutput(then_result, result); + PropagateInputToOutput(else_result, result); } } else if (auto call = dyn_cast(op)) { FuncOp func = dyn_cast(call.resolveCallable()); @@ -387,7 +351,7 @@ ResourceAliasAnalysisInfo::ResourceAliasAnalysisInfo( for (auto result : filter_resources(op->getResults())) { auto passthrough_arg = func_info.GetArg(result.getResultNumber()); if (passthrough_arg) { - forward_input_to_output( + PropagateInputToOutput( call.getArgOperands()[passthrough_arg.getValue()], result); } else { AddValueUniqueIDMapping(result, kUnknownResourceId); @@ -400,6 +364,85 @@ ResourceAliasAnalysisInfo::ResourceAliasAnalysisInfo( }); } +// Propagates the resource ID's from an input operand to a result. Returns true +// if the mapping changed. +bool ResourceAliasAnalysisInfo::PropagateInputToOutput(const Value& operand, + const OpResult& result) { + auto operand_it = resource_value_to_ids_.find(operand); + assert(operand_it != resource_value_to_ids_.end() && + "A resource-type output does not have the corresponding " + "resource-type input."); + bool change = false; + for (int64_t id : operand_it->second) + change = AddValueUniqueIDMapping(result, id) || change; + return change; +} + +// Analyzes while loops to compute resourceIDs for the loop results. +// +// (1) The base case for the analysis is that if the loop body does not execute +// at all, the resource IDs for each result is the same as the resource IDs +// of the corresponding input. +// (2) If the loop does execute one or more times, then we need to account for +// data flow through the body of the while loop. If result #r is the same +// as arg #a of the loop body (pass through argument), then we can reason +// further, else if the result is not a passthrough, we mark it as unknown. +// (3) For passthrough results, if result #r is the same as arg #a of the loop +// body, after one iteration, result #r = arg #a, so we need to also +// propagate arg #a to result #r. After another iteration, arg #a of the +// loop body will be result #a of the previous iteration. So then we need +// propagate from result #a to result #r. Generalizing, the resource ID +// propagation (for results which are passthrough) looks like: +// +// for r in (0, num_results) : result[r] = arg[r]; +// repeat till no change { +// a = passthrough arg for result #r; +// result[r] += result[a]; +// } +// +void ResourceAliasAnalysisInfo::AnalyzeWhileLoop( + Operation* while_op, const BacktrackAnalysisInfo& body_info) { + // Seed the resource ID's for the results using either the resource ID of the + // passthrough arg, or unknown. We need to perform further analysis if we + // find a passthrough arg which is not the same as corresponding the result #. + llvm::SmallVector, 4> passthrough_args( + while_op->getNumResults()); + bool need_analysis = false; + for (auto result : filter_resources(while_op->getResults())) { + int result_index = result.getResultNumber(); + passthrough_args[result_index] = body_info.GetArg(result_index); + if (passthrough_args[result_index]) { + int passthru_index = passthrough_args[result_index].getValue(); + PropagateInputToOutput(while_op->getOperand(passthru_index), result); + need_analysis |= + !IsUnknownResource(result) && passthru_index != result_index; + } else { + AddValueUniqueIDMapping(result, kUnknownResourceId); + } + } + + if (!need_analysis) return; + + // We found a result that is not unknown and whose passthrough operand index + // is not the same as the result index, which means there is "crosstalk" + // between 2 or more operands. In that case, we do an iterative propagation + // of resource ID's till the results converge. + bool change = true; + while (change) { + change = false; + for (auto result : filter_resources(while_op->getResults())) { + if (IsUnknownResource(result)) continue; + // If this result has a valid passthrough arg, propagate resource ID's + // from the result of the passthrough arg + int result_index = result.getResultNumber(); + int passthru_index = passthrough_args[result_index].getValue(); + change = + PropagateInputToOutput(while_op->getResult(passthru_index), result) || + change; + } + } +} + bool ResourceAliasAnalysisInfo::IsUnknownResource(Value resource) const { auto it = resource_value_to_ids_.find(resource); assert(it != resource_value_to_ids_.end() && !it->getSecond().empty()); diff --git a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h index d9fd693042f..c965b5d7602 100644 --- a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h +++ b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h @@ -35,6 +35,7 @@ namespace mlir { namespace TF { namespace detail { class BacktrackAnalysis; +class BacktrackAnalysisInfo; // Resource alias analysis information for a single function. class ResourceAliasAnalysisInfo { @@ -57,15 +58,25 @@ class ResourceAliasAnalysisInfo { llvm::SmallSetVector GetResourceAliases(Value resource) const; private: - // Maps resource value to unique ID and vice-versa. - void AddValueUniqueIDMapping(Value value, int64_t id) { + // Maps resource value to unique ID and vice-versa. Returns true of the + // mapping has changed. + bool AddValueUniqueIDMapping(Value value, int64_t id) { resource_value_to_ids_[value].insert(id); - id_to_resource_values_[id].insert(value); + return id_to_resource_values_[id].insert(value); } // Returns the set unique Values which map to `id`. const llvm::SmallSetVector& GetUniqueIdResources(int64_t id) const; + // Propagates the resource ID's from an input operand to a result. Returns + // true of the mapping has changed. + bool PropagateInputToOutput(const Value& operand, const OpResult& result); + + // Analyzes while loops to compute resourceID's for the loop results. + // `body_info` is the backtrack analysis info for the loop body. + void AnalyzeWhileLoop(Operation* while_op, + const BacktrackAnalysisInfo& body_info); + // Maps each resource-type value to a set of unique IDs that it could alias. llvm::SmallDenseMap, 8> resource_value_to_ids_; diff --git a/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir b/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir index af63f3312bc..87da399b726 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/resource-alias-analysis-test.mlir @@ -112,14 +112,14 @@ func @if_else(%arg0: !tf_res, %arg1: !tf_res) -> (!tf_res, !tf_res, !tf_res) { // CHECK-LABEL: func @while_op_aliasing // expected-remark@below {{Region #0, Arg #0, ID 4 : 1, 4}} -// expected-remark@below {{Region #0, Arg #1, ID 5 : 1, 3, 5}} -// expected-remark@below {{Region #0, Arg #2, ID 6 : 1, 2, 6}} +// expected-remark@below {{Region #0, Arg #1, ID 5 : 1, 2, 3, 5}} +// expected-remark@below {{Region #0, Arg #2, ID 6 : 1, 2, 3, 6}} func @while_op_aliasing(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) { // expected-remark@below {{Result #0, ID 0 : 0}} %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res // expected-remark@below {{Result #0, ID 1 : Unknown}} - // expected-remark@below {{Result #1, ID 2 : 1, 2, 6}} - // expected-remark@below {{Result #2, ID 3 : 1, 3, 5}} + // expected-remark@below {{Result #1, ID 2 : 1, 2, 3, 5, 6}} + // expected-remark@below {{Result #2, ID 3 : 1, 2, 3, 5, 6}} %w:3 = "tf.While"(%arg0, %arg1, %arg2) { body = @while_body, cond = @while_cond, is_stateless = false } : (!tf_res, !tf_res, !tf_res) -> (!tf_res, !tf_res, !tf_res) @@ -205,14 +205,14 @@ func @if_region_aliasing(%arg0: !tf_res, %arg1: !tf_res) { // CHECK-LABEL: func @while_region_aliasing // expected-remark@below {{Region #0, Arg #0, ID 11 : 1, 8, 11}} -// expected-remark@below {{Region #0, Arg #1, ID 12 : 1, 8, 10, 12}} -// expected-remark@below {{Region #0, Arg #2, ID 13 : 1, 8, 9, 13}} +// expected-remark@below {{Region #0, Arg #1, ID 12 : 1, 8, 9, 10, 12}} +// expected-remark@below {{Region #0, Arg #2, ID 13 : 1, 8, 9, 10, 13}} func @while_region_aliasing(%arg0: !tf_res, %arg1: !tf_res, %arg2: !tf_res) { // expected-remark@below {{Result #0, ID 0 : 0, 1, 8}} %vh0 = "tf.VarHandleOp"() {container = "c", shared_name = "v0"} : () -> !tf_res // expected-remark@below {{Result #0, ID 8 : Unknown}} - // expected-remark@below {{Result #1, ID 9 : 1, 8, 9, 13}} - // expected-remark@below {{Result #2, ID 10 : 1, 8, 10, 12}} + // expected-remark@below {{Result #1, ID 9 : 1, 8, 9, 10, 12, 13}} + // expected-remark@below {{Result #2, ID 10 : 1, 8, 9, 10, 12, 13}} // expected-remark@below {{Region #0, Arg #0, ID 2 : 1, 2, 8}} // expected-remark@below {{Region #0, Arg #1, ID 3 : 1, 3, 8}} // expected-remark@below {{Region #0, Arg #2, ID 4 : 1, 4, 8}} From f6944b7c59924f49cd58620985abef6bd1777756 Mon Sep 17 00:00:00 2001 From: Yimei Sun Date: Tue, 11 Aug 2020 11:10:42 -0700 Subject: [PATCH 0830/1017] [Intel MKL] Enable eager mode support for MKL Conv2D BF16 --- .../eager/mkl_eager_op_rewrite.cc | 2 +- .../eager/mkl_eager_op_rewrite_test.cc | 115 ++++++++---------- 2 files changed, 55 insertions(+), 62 deletions(-) diff --git a/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite.cc b/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite.cc index f2339806814..df4423de392 100644 --- a/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite.cc +++ b/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite.cc @@ -207,7 +207,7 @@ bool MklEagerOpRewrite::SlowCheckIfKernelRegistered(string op_name, DataType dt) { // Find if the eager op_name exists in mkl_eager_ops_ list. auto element = mkl_eager_ops_.find(op_name); - if (element != mkl_eager_ops_.end() && dt == DT_FLOAT) { + if (element != mkl_eager_ops_.end()) { // Eager Op exists. So verify registry and return registered or not. return (mkl_op_registry::IsMklNameChangeOp( mkl_op_registry::GetMklEagerOpName(op_name), dt) || diff --git a/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite_test.cc b/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite_test.cc index 91ca800cbac..bcae4e6765f 100644 --- a/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite_test.cc +++ b/tensorflow/core/common_runtime/eager/mkl_eager_op_rewrite_test.cc @@ -19,10 +19,11 @@ limitations under the License. #include "tensorflow/core/common_runtime/eager/eager_op_rewrite_registry.h" #include "tensorflow/core/framework/rendezvous.h" #include "tensorflow/core/platform/test.h" +#include "tensorflow/core/util/mkl_util.h" namespace tensorflow { -class EagerOpRewriteTest { +class EagerOpRewriteTest : public ::testing::Test { public: EagerOpRewriteTest() {} @@ -68,71 +69,63 @@ class EagerOpRewriteTest { } }; -TEST(EagerOpRewriteTest, Conv2D) { - const string orig_op_name = "Conv2D"; - std::unique_ptr orig_op = - EagerOpRewriteTest::CreateOp(orig_op_name); +#define REGISTER_TEST(NAME, T, INPUT) \ + TEST_F(EagerOpRewriteTest, NAME##_##T) { \ + auto orig_op = CreateOp("Conv2D"); \ + orig_op->MutableAttrs()->Set("T", T); \ + orig_op->MutableAttrs()->Set("padding", "VALID"); \ + CheckRewrite(orig_op.get(), "_MklEagerConv2D"); \ + } +REGISTER_TEST_ALL_TYPES(Conv2D); +#undef REGISTER_TEST - orig_op->MutableAttrs()->Set("T", DT_FLOAT); - orig_op->MutableAttrs()->Set("padding", "VALID"); +#define REGISTER_TEST(NAME, T, INPUT) \ + TEST_F(EagerOpRewriteTest, NAME##_##T) { \ + auto orig_op = CreateOp("Conv2D"); \ + orig_op->MutableAttrs()->Set("T", T); \ + orig_op->MutableAttrs()->Set("padding", "EXPLICIT"); \ + CheckRewrite(orig_op.get(), "Conv2D"); \ + } +REGISTER_TEST_ALL_TYPES(Conv2D_Explicit_Padding); +#undef REGISTER_TEST - EagerOpRewriteTest::CheckRewrite(orig_op.get(), "_MklEagerConv2D"); -} +#define REGISTER_TEST(NAME, T, INPUT) \ + TEST_F(EagerOpRewriteTest, NAME##_##T) { \ + auto orig_op = CreateOp("Conv2DBackpropInput"); \ + orig_op->MutableAttrs()->Set("T", T); \ + orig_op->MutableAttrs()->Set("padding", "VALID"); \ + CheckRewrite(orig_op.get(), "_MklEagerConv2DBackpropInput"); \ + } +REGISTER_TEST_ALL_TYPES(Conv2DBackpropInput); +#undef REGISTER_TEST -TEST(EagerOpRewriteTest, Conv2D_Explicit_Padding) { - const string orig_op_name = "Conv2D"; - std::unique_ptr orig_op = - EagerOpRewriteTest::CreateOp(orig_op_name); +#define REGISTER_TEST(NAME, T, INPUT) \ + TEST_F(EagerOpRewriteTest, NAME##_##T) { \ + auto orig_op = CreateOp("Conv2DBackpropFilter"); \ + orig_op->MutableAttrs()->Set("T", T); \ + orig_op->MutableAttrs()->Set("padding", "VALID"); \ + CheckRewrite(orig_op.get(), "_MklEagerConv2DBackpropFilter"); \ + } +REGISTER_TEST_ALL_TYPES(Conv2DBackpropFilter); +#undef REGISTER_TEST - orig_op->MutableAttrs()->Set("T", DT_FLOAT); - orig_op->MutableAttrs()->Set("padding", "EXPLICIT"); +#define REGISTER_TEST(NAME, T, INPUT) \ + TEST_F(EagerOpRewriteTest, NAME##_##T) { \ + auto orig_op = CreateOp("BatchMatMul"); \ + orig_op->MutableAttrs()->Set("T", T); \ + CheckRewrite(orig_op.get(), "_MklBatchMatMul"); \ + } +REGISTER_TEST_ALL_TYPES(BatchMatMul); +#undef REGISTER_TEST - EagerOpRewriteTest::CheckRewrite(orig_op.get(), "Conv2D"); -} - -TEST(EagerOpRewriteTest, Conv2DBackpropInput) { - const string orig_op_name = "Conv2DBackpropInput"; - std::unique_ptr orig_op = - EagerOpRewriteTest::CreateOp(orig_op_name); - - orig_op->MutableAttrs()->Set("T", DT_FLOAT); - orig_op->MutableAttrs()->Set("padding", "VALID"); - - EagerOpRewriteTest::CheckRewrite(orig_op.get(), - "_MklEagerConv2DBackpropInput"); -} - -TEST(EagerOpRewriteTest, Conv2DBackpropFilter) { - const string orig_op_name = "Conv2DBackpropFilter"; - std::unique_ptr orig_op = - EagerOpRewriteTest::CreateOp(orig_op_name); - - orig_op->MutableAttrs()->Set("T", DT_FLOAT); - orig_op->MutableAttrs()->Set("padding", "VALID"); - - EagerOpRewriteTest::CheckRewrite(orig_op.get(), - "_MklEagerConv2DBackpropFilter"); -} - -TEST(EagerOpRewriteTest, BatchMatMul) { - const string orig_op_name = "BatchMatMul"; - std::unique_ptr orig_op = - EagerOpRewriteTest::CreateOp(orig_op_name); - - orig_op->MutableAttrs()->Set("T", DT_FLOAT); - - EagerOpRewriteTest::CheckRewrite(orig_op.get(), "_MklBatchMatMul"); -} - -TEST(EagerOpRewriteTest, MatMul) { - const string orig_op_name = "MatMul"; - std::unique_ptr orig_op = - EagerOpRewriteTest::CreateOp(orig_op_name); - - orig_op->MutableAttrs()->Set("T", DT_FLOAT); - - EagerOpRewriteTest::CheckRewrite(orig_op.get(), "_MklMatMul"); -} +#define REGISTER_TEST(NAME, T, INPUT) \ + TEST_F(EagerOpRewriteTest, NAME##_##T) { \ + auto orig_op = CreateOp("MatMul"); \ + orig_op->MutableAttrs()->Set("T", T); \ + CheckRewrite(orig_op.get(), "_MklMatMul"); \ + } +REGISTER_TEST_ALL_TYPES(MatMul); +#undef REGISTER_TEST } // namespace tensorflow From ede136a3ba5c4dad4a6f58cf408ffe8206e1677d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 10:59:38 -0700 Subject: [PATCH 0831/1017] Add a workaround for slowness of tf.while_loop in the default executor when maximum_iterations is set. Fixes #40517. PiperOrigin-RevId: 326056684 Change-Id: I81854d6731a9134b695c704b0f28786091f8239e --- .../python/autograph/operators/control_flow.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py index ef9c3ae6427..f194c446dc0 100644 --- a/tensorflow/python/autograph/operators/control_flow.py +++ b/tensorflow/python/autograph/operators/control_flow.py @@ -81,7 +81,6 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops -from tensorflow.python.ops import control_flow_util from tensorflow.python.ops import math_ops from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops.ragged import ragged_tensor @@ -479,9 +478,7 @@ def _known_len_tf_for_stmt( return control_flow_ops.cond(main_test, extra_test, lambda: False) return main_test - # TODO(b/159186914): Remove. - if not control_flow_util.GraphOrParentsInXlaContext(ops.get_default_graph()): - opts['maximum_iterations'] = n + opts['maximum_iterations'] = n _tf_while_stmt( aug_test, @@ -527,9 +524,7 @@ def _tf_ragged_for_stmt( return control_flow_ops.cond(main_test, extra_test, lambda: False) return main_test - # TODO(b/159186914): Remove. - if not control_flow_util.GraphOrParentsInXlaContext(ops.get_default_graph()): - opts['maximum_iterations'] = n + opts['maximum_iterations'] = n _tf_while_stmt( aug_test, @@ -587,10 +582,8 @@ def _tf_range_for_stmt( main_test = control_flow_ops.cond(main_test, extra_test, lambda: False) return main_test - # TODO(b/134181679): Remove. - if not control_flow_util.GraphOrParentsInXlaContext(ops.get_default_graph()): - opts['maximum_iterations'] = math_ops.cast( - misc.get_range_len(start, limit, delta), dtypes.int32) + opts['maximum_iterations'] = math_ops.cast( + misc.get_range_len(start, limit, delta), dtypes.int32) _tf_while_stmt( aug_test, From b7685e0b65c6056da80122cf053942690b487831 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 11:16:13 -0700 Subject: [PATCH 0832/1017] No external changes. PiperOrigin-RevId: 326060668 Change-Id: I44be7b76ffebddf584b13f6baab1769a98c27321 --- tensorflow/compiler/xla/pjrt/BUILD | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow/compiler/xla/pjrt/BUILD b/tensorflow/compiler/xla/pjrt/BUILD index 6e61e0600a0..5b3b75eb352 100644 --- a/tensorflow/compiler/xla/pjrt/BUILD +++ b/tensorflow/compiler/xla/pjrt/BUILD @@ -59,6 +59,10 @@ cc_library( name = "tracked_device_buffer", srcs = ["tracked_device_buffer.cc"], hdrs = ["tracked_device_buffer.h"], + visibility = [ + "//learning/pathways/data_parallel:__pkg__", + "//tensorflow:internal", + ], deps = [ ":event_pool", ":local_device_state", From 47bd06cf2132cee053bfaa1e8d925f12f0c31c75 Mon Sep 17 00:00:00 2001 From: Ken Franko Date: Tue, 11 Aug 2020 11:16:44 -0700 Subject: [PATCH 0833/1017] Don't extract Embedding ops for outside compilation. These ops are rewritten as part of the compile op and shouldn't be marked for outside compilation. PiperOrigin-RevId: 326060767 Change-Id: I92b7d9500c05a8dad011b5a19df0a654346edeca --- .../mark_ops_for_outside_compilation.mlir | 14 ++++++++++++++ .../mark_ops_for_outside_compilation.cc | 19 ++++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir b/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir index 0bb37e4c3cd..2d86889e35b 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir @@ -32,6 +32,20 @@ func @tf2xla_fallback_op() -> tensor { return %0 : tensor } +// CHECK-LABEL: func @ignore_embedding_ops +func @ignore_embedding_ops() -> () { + "tf_device.cluster"() ( { + // CHECK: "tf.RecvTPUEmbeddingActivations" + // CHECK-NOT: _xla_outside_compilation + // CHECK: "tf.SendTPUEmbeddingGradients" + // CHECK-NOT: _xla_outside_compilation + %2:2 = "tf.RecvTPUEmbeddingActivations"() {_tpu_embedding_layer = "call1", config = "\0A\0B\0C\0D"} : () -> (tensor<2x2xf32>, tensor<4x4xf32>) + "tf.SendTPUEmbeddingGradients"(%2#0, %2#1) {_tpu_embedding_layer = "call1", config = "\0A\0B\0C\0D", operand_segment_sizes = dense<[2, 0]> : vector<2xi32>} : (tensor<2x2xf32>, tensor<4x4xf32>) -> () + tf_device.return + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> () + return +} + // CHECK-LABEL: func @op_string_result func @op_string_result() -> tensor { %0 = "tf_device.cluster"() ( { diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc b/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc index cd34525f2af..ece26dca416 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc @@ -48,9 +48,21 @@ struct MarkOpsForOutsideCompilation // added. void AddSupportedControlFlowOps(MLIRContext* context, llvm::DenseSet* supported_ops) { - supported_ops->insert(OperationName("tf.IfRegion", context)); - supported_ops->insert(OperationName("tf.WhileRegion", context)); - supported_ops->insert(OperationName("tf.Yield", context)); + supported_ops->insert( + OperationName(TF::IfRegionOp::getOperationName(), context)); + supported_ops->insert( + OperationName(TF::WhileRegionOp::getOperationName(), context)); + supported_ops->insert( + OperationName(TF::YieldOp::getOperationName(), context)); +} + +// These embedding ops are rewritten when running TPUCompileOp. +void AddRewrittenEmbeddingOps(MLIRContext* context, + llvm::DenseSet* supported_ops) { + supported_ops->insert(OperationName( + TF::RecvTPUEmbeddingActivationsOp::getOperationName(), context)); + supported_ops->insert(OperationName( + TF::SendTPUEmbeddingGradientsOp::getOperationName(), context)); } bool HasStringOperand(Operation& op) { @@ -137,6 +149,7 @@ void MarkOpsForOutsideCompilation::runOnOperation() { supported_ops.insert(*pattern->getRootKind()); } AddSupportedControlFlowOps(module.getContext(), &supported_ops); + AddRewrittenEmbeddingOps(module.getContext(), &supported_ops); auto result = module.walk([&](tf_device::ClusterOp cluster) { if (failed( From 2940b356dfaf06ffb421968ba9cb163f0edde013 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 11:50:51 -0700 Subject: [PATCH 0834/1017] Fix typos in the doc. PiperOrigin-RevId: 326068303 Change-Id: I7ffc0560b3ba23055e6e5fe29166747748fbc1aa --- tensorflow/python/keras/metrics.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/python/keras/metrics.py b/tensorflow/python/keras/metrics.py index 6b53a02ce05..b3f391c7897 100644 --- a/tensorflow/python/keras/metrics.py +++ b/tensorflow/python/keras/metrics.py @@ -644,7 +644,7 @@ class MeanMetricWrapper(Mean): @keras_export('keras.metrics.Accuracy') class Accuracy(MeanMetricWrapper): - """Calculates how often predictions equals labels. + """Calculates how often predictions equal labels. This metric creates two local variables, `total` and `count` that are used to compute the frequency with which `y_pred` matches `y_true`. This frequency is @@ -686,7 +686,7 @@ class Accuracy(MeanMetricWrapper): @keras_export('keras.metrics.BinaryAccuracy') class BinaryAccuracy(MeanMetricWrapper): - """Calculates how often predictions matches binary labels. + """Calculates how often predictions match binary labels. This metric creates two local variables, `total` and `count` that are used to compute the frequency with which `y_pred` matches `y_true`. This frequency is From 9bc641d16c132adb74b955fa09067630bcc93dae Mon Sep 17 00:00:00 2001 From: Chenkai Kuang Date: Tue, 11 Aug 2020 12:00:53 -0700 Subject: [PATCH 0835/1017] Fix an error when a concrete function is passed to client.schedule. PiperOrigin-RevId: 326070549 Change-Id: I620e3feb329a436121377e144d89f9d88e6bf442 --- tensorflow/python/distribute/client/client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/python/distribute/client/client.py b/tensorflow/python/distribute/client/client.py index 7bef5e2385c..37f000d4a87 100644 --- a/tensorflow/python/distribute/client/client.py +++ b/tensorflow/python/distribute/client/client.py @@ -293,8 +293,7 @@ class Closure(object): self._output_remote_values = nest.map_structure( lambda x: RemoteValue(self, x), concrete_function.structured_outputs) elif isinstance(function, tf_function.ConcreteFunction): - self._function = cancellation_mgr.get_cancelable_function( - concrete_function) + self._function = cancellation_mgr.get_cancelable_function(function) self._output_remote_values = nest.map_structure( lambda x: RemoteValue(self, x), function.structured_outputs) else: From 0572b205b847917062091e4377110f5431c60d2b Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Tue, 11 Aug 2020 12:08:22 -0700 Subject: [PATCH 0836/1017] Roll back XLA/GPU LHLO sort emitter again It breaks an internal msan enabled test. PiperOrigin-RevId: 326072372 Change-Id: I245525cefa4da88097725662c75ccb213a328f19 --- tensorflow/compiler/mlir/xla/hlo_utils.cc | 3 - .../non_identity_layouts.hlotxt | 2 +- .../xla/transforms/mhlo_to_lhlo_with_xla.cc | 11 +- .../xla/transforms/mhlo_to_lhlo_with_xla.h | 3 +- tensorflow/compiler/xla/service/gpu/BUILD | 10 - .../compiler/xla/service/gpu/gpu_compiler.cc | 24 +- .../xla/service/gpu/hlo_to_ir_bindings.cc | 20 +- .../xla/service/gpu/hlo_to_ir_bindings.h | 4 - .../xla/service/gpu/ir_emitter_context.h | 7 +- .../xla/service/gpu/ir_emitter_unnested.cc | 416 ++++----------- .../xla/service/gpu/ir_emitter_unnested.h | 82 +-- .../compiler/xla/service/gpu/tests/BUILD | 29 - .../xla/service/gpu/tests/sorting.hlo | 504 +++++++++--------- .../xla/service/gpu/tests/sorting_test.cc | 71 --- .../compiler/xla/service/llvm_ir/llvm_util.cc | 7 +- .../compiler/xla/service/llvm_ir/llvm_util.h | 2 +- 16 files changed, 403 insertions(+), 792 deletions(-) delete mode 100644 tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc diff --git a/tensorflow/compiler/mlir/xla/hlo_utils.cc b/tensorflow/compiler/mlir/xla/hlo_utils.cc index 18b4265d786..cf78c81908d 100644 --- a/tensorflow/compiler/mlir/xla/hlo_utils.cc +++ b/tensorflow/compiler/mlir/xla/hlo_utils.cc @@ -83,9 +83,6 @@ StatusOr> GetPermutationIfAvailable( strides[dim] = accumulated_stride; accumulated_stride *= shape.dimensions(dim); } - if (accumulated_stride == 0) { - return llvm::SmallVector{}; - } return llvm::SmallVector{ makeStridedLinearLayoutMap(strides, /*offset=*/0, builder.getContext())}; } diff --git a/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt b/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt index a83e36cff64..3630d2d45e4 100644 --- a/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt +++ b/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt @@ -8,6 +8,6 @@ HloModule TestModule ENTRY TestComputation { x = f32[3, 2]{1,0} parameter(0) - // CHECK: "lmhlo.copy"(%{{.*}}, %{{.*}}) {name = "copy.1"} : (memref<3x2xf32>, memref<3x2xf32, #[[MAP]]>) -> () + // CHECK: "lmhlo.copy"(%{{.*}}, %{{.*}}) : (memref<3x2xf32>, memref<3x2xf32, #[[MAP]]>) -> () ROOT x.copy = f32[3, 2]{0,1} copy(x) } diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc index 6ce91599fb1..832bad2dcc8 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc @@ -34,6 +34,7 @@ limitations under the License. #include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Pass/PassOptions.h" // from @llvm-project #include "mlir/Translation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" #include "tensorflow/compiler/mlir/xla/hlo_function_importer.h" #include "tensorflow/compiler/mlir/xla/hlo_utils.h" #include "tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.h" @@ -181,10 +182,7 @@ template StatusOr LhloDialectEmitter::CreateOpWithoutAttrs( HloInstruction* instr) { Location loc = getLocation(instr); - std::pair attrs[] = { - {Identifier::get("name", builder_.getContext()), - builder_.getStringAttr(instr->name())}, - }; + ArrayRef> attrs; ArrayRef rets{}; llvm::SmallVector operands; @@ -254,14 +252,15 @@ Status LhloDialectEmitter::DefaultAction(HloInstruction* instr) { return Status::OK(); } -StatusOr LhloDialectEmitter::EmitSortOp(HloInstruction* instr) { +StatusOr LhloDialectEmitter::EmitSortOp( + HloInstruction* instr) { TF_ASSIGN_OR_RETURN(auto sort, CreateOpWithoutAttrs(instr)); auto* sort_instr = ::xla::Cast<::xla::HloSortInstruction>(instr); sort.dimensionAttr(builder_.getI64IntegerAttr(sort_instr->sort_dimension())); sort.is_stableAttr(builder_.getBoolAttr(sort_instr->is_stable())); TF_RETURN_IF_ERROR(::xla::HloFunctionImporter::ImportAsRegion( *sort_instr->called_computations()[0], &sort.comparator(), &builder_)); - return sort; + return sort.getOperation(); } Status LhloDialectEmitter::HandleSort(HloInstruction* instr) { diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h index 4000fa01970..bdc977616b1 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h @@ -19,7 +19,6 @@ limitations under the License. #include "mlir/IR/Builders.h" // from @llvm-project #include "mlir/IR/Module.h" // from @llvm-project #include "mlir/IR/StandardTypes.h" // from @llvm-project -#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/hlo_module.h" @@ -42,7 +41,7 @@ class LhloDialectEmitter : public ::xla::DfsHloVisitorWithDefault { builder_(module.getContext()), i8_type_(builder_.getIntegerType(8)) {} - ::xla::StatusOr EmitSortOp(::xla::HloInstruction* instr); + ::xla::StatusOr EmitSortOp(::xla::HloInstruction* instr); private: template diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index a19f9965fc7..074fbd92b27 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -254,11 +254,6 @@ cc_library( ":target_util", ":thunk", ":thunk_emitter", - "//tensorflow/compiler/mlir/hlo:lhlo", - "//tensorflow/compiler/mlir/xla:hlo_utils", - "//tensorflow/compiler/mlir/xla:mhlo_to_lhlo_with_xla", - "//tensorflow/compiler/mlir/xla:mlir_hlo_to_hlo", - "//tensorflow/compiler/mlir/xla:type_to_shape", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", @@ -296,8 +291,6 @@ cc_library( "@com_google_absl//absl/types:span", "@llvm-project//llvm:Core", "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:StandardOps", ], ) @@ -1166,7 +1159,6 @@ cc_library( ":target_constants", ":tree_reduction_rewriter", ":variadic_op_splitter", - "//tensorflow/compiler/mlir/xla:mhlo_to_lhlo_with_xla", "//tensorflow/compiler/xla:protobuf_util", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", @@ -1225,8 +1217,6 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@llvm-project//llvm:Core", - "@llvm-project//mlir:AllPassesAndDialectsNoRegistration", - "@llvm-project//mlir:IR", ], ) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index b796737e601..f5bf7476059 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -29,8 +29,6 @@ limitations under the License. #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" -#include "mlir/IR/Module.h" // from @llvm-project -#include "mlir/InitAllDialects.h" // from @llvm-project #include "tensorflow/compiler/xla/protobuf_util.h" #include "tensorflow/compiler/xla/service/algebraic_simplifier.h" #include "tensorflow/compiler/xla/service/all_reduce_combiner.h" @@ -518,22 +516,15 @@ static Status CompileModuleToLlvmIrImpl( DumpHloModuleIfEnabled(*hlo_module, **buffer_assignment, "after_optimizations"); - mlir::registerAllDialects(); - mlir::MLIRContext mlir_context; - IrEmitterContext ir_emitter_context( hlo_module, buffer_assignment->get(), platform_name, gpu_device_info, - cuda_compute_capability, profile_index_map, &mlir_context, - llvm_module->get()); + cuda_compute_capability, profile_index_map, llvm_module->get()); HloComputation* entry_computation = hlo_module->entry_computation(); + IrEmitterUnnested ir_emitter(hlo_module->config(), entry_computation, + &ir_emitter_context); - TF_ASSIGN_OR_RETURN( - auto ir_emitter, - IrEmitterUnnested::Create(hlo_module->config(), entry_computation, - &ir_emitter_context)); - - TF_RETURN_IF_ERROR(ir_emitter->EmitConstantGlobals()); + TF_RETURN_IF_ERROR(ir_emitter.EmitConstantGlobals()); { XLA_SCOPED_LOGGING_TIMER("GpuCompiler::RunBackend - IR emission"); @@ -542,10 +533,9 @@ static Status CompileModuleToLlvmIrImpl( ThunkSequence thunk_sequence; absl::Span order = hlo_schedule->ThunkLaunchOrder(); for (HloInstruction* instruction : order) { - TF_RETURN_IF_ERROR(instruction->Visit(ir_emitter.get())); - TF_RETURN_IF_ERROR(ir_emitter->Postprocess(instruction)); - std::unique_ptr thunks = - ir_emitter->ConsumeThunkSequence(); + TF_RETURN_IF_ERROR(instruction->Visit(&ir_emitter)); + TF_RETURN_IF_ERROR(ir_emitter.Postprocess(instruction)); + std::unique_ptr thunks = ir_emitter.ConsumeThunkSequence(); // The invariants between each input HloInstruction* and output Thunk* are // not all explicitly checked, but at least we can document them here: diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc index 332db83b6ad..5d38d1b727c 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc @@ -117,11 +117,11 @@ static bool HasMeaningfulName(llvm::Value* value) { return false; } -llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, - llvm::IRBuilder<>* b) { - llvm::Type* pointee_type = - llvm_ir::ShapeToIrType(shape, b->GetInsertBlock()->getModule()); - +llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, + ShapeIndexView shape_index, + llvm::Value* ir_value) { + llvm::Type* pointee_type = llvm_ir::ShapeToIrType( + ShapeUtil::GetSubshape(hlo.shape(), shape_index), module_); llvm::Type* dest_type = pointee_type->getPointerTo(); llvm::Value* typed_ir_value; @@ -129,17 +129,9 @@ llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, typed_ir_value = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast( llvm::cast(ir_value), dest_type); } else { - typed_ir_value = b->CreatePointerBitCastOrAddrSpaceCast( + typed_ir_value = b_->CreatePointerBitCastOrAddrSpaceCast( ir_value, pointee_type->getPointerTo()); } - return typed_ir_value; -} - -llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, - ShapeIndexView shape_index, - llvm::Value* ir_value) { - auto typed_ir_value = CastToTypedValue( - ShapeUtil::GetSubshape(hlo.shape(), shape_index), ir_value, b_); if (!HasMeaningfulName(ir_value)) { ir_value->setName(llvm_ir::IrName(&hlo, "raw")); } diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h index 3813ec6c949..5eef6727801 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h @@ -116,10 +116,6 @@ class HloToIrBindings { llvm::Value* temp_buffer_base_ = nullptr; }; -// Converts `ir_value` with type i8* to a typed LLVM Value* based on `shape`. -llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, - llvm::IRBuilder<>* b); - } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h index 7d5a8d032e6..9c43f80dc60 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h @@ -17,7 +17,6 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_CONTEXT_H_ #include "llvm/IR/Module.h" -#include "mlir/IR/MLIRContext.h" // from @llvm-project #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/gpu/launch_dimensions.h" #include "tensorflow/compiler/xla/service/hlo_execution_profile.h" @@ -35,15 +34,13 @@ class IrEmitterContext { const HloModule* hlo_module, const BufferAssignment* buffer_assignment, std::string platform_name, GpuDeviceInfo gpu_device_info, absl::optional cuda_compute_capability, - const HloProfileIndexMap* profile_index_map, - mlir::MLIRContext* mlir_context, llvm::Module* llvm_module) + const HloProfileIndexMap* profile_index_map, llvm::Module* llvm_module) : hlo_module_(hlo_module), buffer_assignment_(buffer_assignment), platform_name_(std::move(platform_name)), gpu_device_info_(gpu_device_info), cuda_compute_capability_(cuda_compute_capability), profile_index_map_(profile_index_map), - mlir_context_(mlir_context), llvm_module_(llvm_module) {} // Disallow copy and assign. IrEmitterContext(const IrEmitterContext&) = delete; @@ -60,7 +57,6 @@ class IrEmitterContext { return cuda_compute_capability_; } const HloProfileIndexMap* profile_index_map() { return profile_index_map_; } - mlir::MLIRContext* mlir_context() { return mlir_context_; } llvm::Module* llvm_module() { return llvm_module_; } NameUniquer* name_uniquer() { return &name_uniquer_; } @@ -71,7 +67,6 @@ class IrEmitterContext { GpuDeviceInfo gpu_device_info_; absl::optional cuda_compute_capability_; const HloProfileIndexMap* profile_index_map_; - mlir::MLIRContext* mlir_context_; llvm::Module* llvm_module_; NameUniquer name_uniquer_; }; diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index f88c70b1a33..61b78b6004d 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -37,13 +37,6 @@ limitations under the License. #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" -#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project -#include "mlir/IR/Builders.h" // from @llvm-project -#include "mlir/IR/Function.h" // from @llvm-project -#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" -#include "tensorflow/compiler/mlir/xla/hlo_utils.h" -#include "tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.h" -#include "tensorflow/compiler/mlir/xla/type_to_shape.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" @@ -151,86 +144,13 @@ void UpdateLaunchDimensions(const LaunchDimensions& launch_dims, Thunk* thunk, llvm::ConstantAsMetadata::get(threads_per_block_ir_value)})); } -const BufferAllocation* GetAllocation( - mlir::BlockArgument func_arg, const BufferAssignment& buffer_assignment) { - auto func_op = - mlir::cast(func_arg.getParentRegion()->getParentOp()); - int64 allocation_index = func_op - .getArgAttrOfType( - func_arg.getArgNumber(), "lmhlo.alloc") - .getValue() - .getSExtValue(); - return &buffer_assignment.GetAllocation(allocation_index); -} - -StatusOr GetAllocationSliceForMlir( - mlir::Value v, const BufferAssignment& buffer_assignment) { - int64 size = v.getType().cast().getSizeInBits() / 8; - - if (auto arg = v.dyn_cast()) { - return BufferAllocation::Slice(GetAllocation(arg, buffer_assignment), 0, - size); - } - - // We match two patterns here: - // * v = ViewOp(arg); - // * v = StaticMemRefCastOp(ViewOp(arg)); - if (mlir::Operation* op = v.getDefiningOp()) { - if (auto cast = mlir::dyn_cast(op)) { - mlir::Value source = cast.getViewSource(); - op = source.getDefiningOp(); - if (!op) { - return Unimplemented("StaticMemRefCastOp has to wrap an op"); - } - } - if (auto view = mlir::dyn_cast(op)) { - return BufferAllocation::Slice( - GetAllocation(view.source().cast(), - buffer_assignment), - mlir::cast(view.byte_shift().getDefiningOp()) - .value() - .cast() - .getValue() - .getSExtValue(), - size); - } - return Unimplemented("StaticMemRefCastOp has to wrap a ViewOp"); - } - - return Unimplemented( - "Operand has to be in the form of ViewOp(arg) or " - "StaticMemRefCastOp(ViewOp(arg))"); -} - -absl::string_view GetHloName(mlir::Operation* op) { - if (auto attr = op->getAttrOfType("name")) { - auto ref = attr.getValue(); - return absl::string_view(ref.data(), ref.size()); - } - return ""; -} - } // namespace IrEmitterUnnested::IrEmitterUnnested(const HloModuleConfig& hlo_module_config, const HloComputation* hlo_computation, IrEmitterContext* ir_emitter_context) : IrEmitter(hlo_module_config, ir_emitter_context, /*is_nested=*/false), - hlo_computation_(hlo_computation), - mlir_scratch_module_(mlir::ModuleOp::create( - mlir::Builder(ir_emitter_context->mlir_context()).getUnknownLoc())), - lhlo_scratch_emitter_(ir_emitter_context_->buffer_assignment(), - *hlo_computation, mlir_scratch_module_.get()) {} - -StatusOr> IrEmitterUnnested::Create( - const HloModuleConfig& hlo_module_config, - const HloComputation* hlo_computation, - IrEmitterContext* ir_emitter_context) { - auto emitter = std::unique_ptr(new IrEmitterUnnested( - hlo_module_config, hlo_computation, ir_emitter_context)); - TF_RETURN_IF_ERROR(emitter->lhlo_scratch_emitter_.Initialize()); - return std::move(emitter); -} + hlo_computation_(hlo_computation) {} Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { bindings_.UnbindAllLocalIrValues(); @@ -238,11 +158,12 @@ Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { } llvm::Function* IrEmitterUnnested::BuildKernelPrototype( - absl::string_view name, absl::Span args) { + const HloInstruction& inst, + absl::Span args) { // Compute the kernel name. The opcode string may contain "-" which cannot be // in a PTX function name, so sanitize the name before uniquifying it. string kernel_name = ir_emitter_context_->name_uniquer()->GetUniqueName( - llvm_ir::SanitizeFunctionName(std::string(name))); + llvm_ir::SanitizeFunctionName(inst.name())); // Create the kernel and add it to the module. llvm::Module* module = ir_emitter_context_->llvm_module(); @@ -438,8 +359,7 @@ Status IrEmitterUnnested::HandleDot(HloInstruction* dot) { } Status IrEmitterUnnested::HandleConditional(HloInstruction* conditional) { - TF_ASSIGN_OR_RETURN(auto thunk, BuildConditionalThunk(conditional)); - AddThunkToThunkSequence(std::move(thunk)); + AddThunkToThunkSequence(BuildConditionalThunk(conditional)); return Status::OK(); } @@ -1118,13 +1038,10 @@ Status IrEmitterUnnested::HandleWhile(HloInstruction* xla_while) { // Build ForThunk for conformant while loops, otherwise build WhileThunk. auto config = xla_while->backend_config(); if (config.ok() && config.ValueOrDie().has_known_trip_count()) { - TF_ASSIGN_OR_RETURN( - auto thunk, + AddThunkToThunkSequence( BuildForThunk(xla_while, config.ValueOrDie().known_trip_count().n())); - AddThunkToThunkSequence(std::move(thunk)); } else { - TF_ASSIGN_OR_RETURN(auto thunk, BuildWhileThunk(xla_while)); - AddThunkToThunkSequence(std::move(thunk)); + AddThunkToThunkSequence(BuildWhileThunk(xla_while)); } return Status::OK(); } @@ -1347,109 +1264,39 @@ Status IrEmitterUnnested::HandleSelect(HloInstruction* select) { return IrEmitter::HandleSelect(select); } -StatusOr -IrEmitterUnnested::GetOrCreateSubComputationFromRegion(mlir::Region* region) { - std::unique_ptr& module = scratch_nested_computations_[region]; - if (module == nullptr) { - xla::XlaComputation xla_computation; - TF_RETURN_IF_ERROR(ConvertRegionToComputation(region, &xla_computation)); - TF_ASSIGN_OR_RETURN(auto program_shape, xla_computation.GetProgramShape()); - TF_ASSIGN_OR_RETURN( - module, HloModule::CreateFromProto(xla_computation.proto(), - HloModuleConfig(program_shape))); - } - return module->entry_computation(); -} - Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { - MlirEmitterInput result; - - TF_ASSIGN_OR_RETURN(auto sort_op, lhlo_scratch_emitter_.EmitSortOp(sort)); - result.op = sort_op; - result.name = GetHloName(sort_op); - // The name in sort op has no semantics, and it's for debug only. If the name - // doesn't exist, we should use a namer (e.g. count-based). - // TODO(timshen): use a namer instead of relying on the HloInstruction names. - if (result.name.empty()) { - result.name = sort->name(); - } - const auto& buffer_assignment = ir_emitter_context_->buffer_assignment(); - auto& slice = result.extra_slice; - TF_ASSIGN_OR_RETURN(slice.buffer_slice, - buffer_assignment.GetUniqueSlice(sort, {})); - slice.written = true; - slice.shape = sort->shape(); - - result.thunk_info = GetThunkInfo(sort); - - return EmitMlirSort(result); -} - -Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { - const auto& buffer_assignment = ir_emitter_context_->buffer_assignment(); - auto sort_op = mlir::cast(input.op); - - int operand_count = sort_op.operands().size(); - std::vector operand_shapes(operand_count); - std::vector slices; - std::vector output_shapes(sort_op.output().size()); - - for (int i = 0; i < operand_count; i++) { - operand_shapes[i] = - TypeToShape(sort_op.operands()[i].getType().cast()); - } - - // Craft n + 1 slices, where the first n are output parameters, and the last - // is the on-device tuple storage. We don't need n operands because sorting - // kernels are always in-place. - for (int i = 0; i < operand_count; i++) { - output_shapes[i] = - TypeToShape(sort_op.output()[i].getType().cast()); - MlirBufferSlice slice; - TF_ASSIGN_OR_RETURN( - slice.buffer_slice, - GetAllocationSliceForMlir(sort_op.output()[i], buffer_assignment)); - slice.written = true; - slice.shape = operand_shapes[i]; - slices.push_back(slice); - } - slices.push_back(input.extra_slice); - std::vector> thunks; - - Shape keys_shape = operand_shapes[0]; - int64 dimension_to_sort = sort_op.dimension().getSExtValue(); - for (int64 i = 0; i < operand_count; ++i) { + Shape keys_shape = sort->operand(0)->shape(); + int64 dimension_to_sort = sort->dimensions(0); + for (int64 i = 0; i < sort->operand_count(); ++i) { + ShapeIndex shape_index = + sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({}); // We assume that the layout of all involved operands and outputs is the // same. - TF_RET_CHECK( - LayoutUtil::LayoutsInShapesEqual(keys_shape, operand_shapes[i])); - TF_RET_CHECK( - LayoutUtil::LayoutsInShapesEqual(keys_shape, output_shapes[i])); + TF_RET_CHECK(LayoutUtil::LayoutsInShapesEqual(keys_shape, + sort->operand(i)->shape())); + TF_RET_CHECK(LayoutUtil::LayoutsInShapesEqual( + keys_shape, ShapeUtil::GetSubshape(sort->shape(), shape_index))); // If possible, we share buffers. If that is not possible, we need to copy // the values, because the emitter does the sorting in-place. - TF_ASSIGN_OR_RETURN( - auto destination_buffer, - GetAllocationSliceForMlir(sort_op.output()[i], buffer_assignment)); - TF_ASSIGN_OR_RETURN( - auto source_address, - GetAllocationSliceForMlir(sort_op.operands()[i], buffer_assignment)); + auto destination_buffer = GetAllocationSlice(*sort, shape_index); + auto source_address = GetAllocationSlice(*sort->operand(i)); if (destination_buffer != source_address) { // TODO(b/26783907): Figure out why we never seem to share buffers for // key/value sort. - VLOG(2) << input.name << " requires initial D2D copy for operand " << i; + VLOG(2) << sort->name() << " requires initial D2D copy for operand " << i; thunks.push_back(absl::make_unique( Thunk::ThunkInfo(), /*source_address=*/source_address, /*destination_buffer=*/destination_buffer, - /*mem_size=*/ShapeUtil::ByteSizeOf(operand_shapes[i]))); + /*mem_size=*/ShapeUtil::ByteSizeOf(sort->operand(i)->shape()))); } } uint64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort); int64 num_stages = tensorflow::Log2Ceiling(dimension_to_sort_bound); - VLOG(2) << input.name << " requires " << num_stages << " stages."; + VLOG(2) << sort->name() << " requires " << num_stages << " stages."; CHECK_GE(1ULL << num_stages, dimension_to_sort_bound); CHECK_LT(1ULL << (num_stages - 1), dimension_to_sort_bound); @@ -1513,10 +1360,10 @@ Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { // we have not enough threads, or not enough shared memory. Also it does not // give a speedup if the tile size is < 128. int64 total_shared_memory_needed = 0; - for (int64 i = 0; i < operand_count; ++i) { + for (int64 i = 0; i < sort->operand_count(); ++i) { total_shared_memory_needed += - kTileSize * - ShapeUtil::ByteSizeOfPrimitiveType(operand_shapes[i].element_type()); + kTileSize * ShapeUtil::ByteSizeOfPrimitiveType( + sort->operand(i)->shape().element_type()); } bool no_tiling = kTileSize < 128 || @@ -1529,7 +1376,7 @@ Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { "kTileSize=%d < 128, " "kThreadsPerBlock=%d > threads_per_block_limit=%d, " "total_shared_memory_needed=%d > shared_memory_per_block=%d", - input.name, (no_tiling ? "won't" : "will"), kTileSize, kThreadsPerBlock, + sort->name(), (no_tiling ? "won't" : "will"), kTileSize, kThreadsPerBlock, ir_emitter_context_->gpu_device_info().threads_per_block_limit, total_shared_memory_needed, ir_emitter_context_->gpu_device_info().shared_memory_per_block); @@ -1537,38 +1384,37 @@ Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { uint64 num_blocks = CeilOfRatio(num_iterations, kThreadsPerBlock); LaunchDimensions tiled_launch_dimensions(num_blocks, kThreadsPerBlock); VLOG(2) << absl::StreamFormat("%s launch dims: %d blocks, %d threads/block", - input.name, num_blocks, kThreadsPerBlock); + sort->name(), num_blocks, kThreadsPerBlock); - std::vector ir_arrays; auto emit_kernel = [&](absl::Span xor_masks) { VLOG(2) << absl::StreamFormat( - "%s uses kernel for xor masks [%s]", input.name, + "%s uses kernel for xor masks [%s]", sort->name(), absl::StrJoin(xor_masks, ", ", [](std::string* out, int64 xor_mask) { absl::StrAppendFormat(out, "0x%x", xor_mask); })); - thunks.push_back(BuildKernelThunkForMlir(input.name, Thunk::ThunkInfo(), - slices, &ir_arrays)); + thunks.push_back( + BuildKernelThunk(sort, /*implements_whole_instruction=*/false)); LaunchDimensions launch_dimensions = xor_masks.size() > 1 ? tiled_launch_dimensions : standard_launch_dimensions; UpdateLaunchDimensions(launch_dimensions, thunks.back().get(), ir_emitter_context_->llvm_module()); std::vector values_arrays; - values_arrays.reserve(operand_count); - for (int64 i = 0; i < operand_count; ++i) { - values_arrays.push_back(ir_arrays[i]); + values_arrays.reserve(sort->operand_count()); + for (int64 i = 0; i < sort->operand_count(); ++i) { + ShapeIndex shape_index = + sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({}); + values_arrays.push_back(GetIrArray(*sort, *sort, shape_index)); } - TF_ASSIGN_OR_RETURN( - const HloComputation* comparator, - GetOrCreateSubComputationFromRegion(&sort_op.comparator())); return llvm_ir::EmitSortInPlace( - dimension_to_sort, values_arrays, IrName(input.name), xor_masks, &b_, + dimension_to_sort, values_arrays, IrName(sort), xor_masks, &b_, launch_dimensions, xor_masks.size() > 1 ? num_iterations_in_sort_dim : standard_num_iterations_in_sort_dim, kTileSize, [&](absl::Span operands, llvm::Value* output) { - return EmitCallToNestedComputation(*comparator, operands, output); + return EmitCallToNestedComputation(*sort->to_apply(), operands, + output); }); }; std::vector xor_masks; @@ -1595,18 +1441,17 @@ Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { TF_RETURN_IF_ERROR(emit_kernel(xor_masks)); } VLOG(2) << absl::StreamFormat( - "%s requires %d thunks (including any D2D copies)", input.name, + "%s requires %d thunks (including any D2D copies)", sort->name(), thunks.size()); - AddThunkToThunkSequence( - absl::make_unique(input.thunk_info, std::move(thunks))); - if (operand_count > 1) { + AddThunkToThunkSequence(absl::make_unique( + GetThunkInfo(sort), std::move(thunks))); + if (sort->operand_count() > 1) { // Emit the tuple as part of the last stage of sorting. // We are currently in the block sorted.in_bounds.after. b_.SetInsertPoint(b_.GetInsertBlock()->getTerminator()); - llvm_ir::EmitTuple( - ir_arrays[operand_count], - absl::MakeSpan(ir_arrays).subspan(0, ir_arrays.size() - 1), &b_); + llvm_ir::EmitTuple(GetIrArray(*sort, *sort), + ConstructIrArrayForOutputs(*sort), &b_); } return Status::OK(); } @@ -1744,6 +1589,24 @@ Status IrEmitterUnnested::HandleAfterAll(HloInstruction* after_all) { return Status::OK(); } +// Describes how to access a particular subshape for an HLO. For instance if +// `.hlo_index` is {1} and `.gte_index` is {3, 4} then buffer for `.instr` at +// ShapeIndex {1} (i.e. the buffer for the second tuple element of hlo) is found +// at `.buffer_slice`[3][4]. That is, `.slice` is a void***, which we +// dereference twice -- first at index 3, and then at index 4 -- to get the +// address of our buffer. +struct HloBufferSlice { + const HloInstruction* instr; + ShapeIndex hlo_index; + + // The root buffer to look at. + BufferAllocation::Slice buffer_slice; + + // Describes how to dereference starting at that buffer to get to the buffer + // in question. + ShapeIndex gte_index; +}; + // Figures out how to access the buffers for all subshapes of hlo's operands and // for hlo itself (i.e. all the buffers produced by HLO). // @@ -1852,22 +1715,22 @@ static std::vector GetHloBufferSlices( return result; } -std::unique_ptr -IrEmitterUnnested::BuildKernelThunkFromBufferSlices( - absl::string_view name, Thunk::ThunkInfo thunk_info, - absl::Span slices, - std::function - bind_slice_to_ir_value) { - const auto& buffer_assn = ir_emitter_context_->buffer_assignment(); +std::unique_ptr IrEmitterUnnested::BuildKernelThunk( + const HloInstruction* inst, bool implements_whole_instruction) { + const BufferAssignment& buffer_assn = + ir_emitter_context_->buffer_assignment(); + + std::vector hlo_slices = + GetHloBufferSlices(inst, buffer_assn); // Figure out which buffer allocations need to be passed as arguments to our - // kernel. This is simply all of the allocations referenced in slices, + // kernel. This is simply all of the allocations referenced in hlo_slices, // plus the XLA temp buffer (if we have it). We always include the temp // buffer because even if the kernel itself doesn't use it, a nested // subcomputation within the kernel (e.g. a kMap's computation) might. std::unordered_set buffers_needed; - for (auto* slice : slices) { - buffers_needed.insert(slice->buffer_slice.allocation()); + for (const auto& hlo_buffer_slice : hlo_slices) { + buffers_needed.insert(hlo_buffer_slice.buffer_slice.allocation()); } absl::optional temp_buffer; for (const BufferAllocation& alloc : buffer_assn.Allocations()) { @@ -1896,7 +1759,7 @@ IrEmitterUnnested::BuildKernelThunkFromBufferSlices( return a->index() < b->index(); }); - llvm::Function* kernel = BuildKernelPrototype(name, non_constant_buffers); + llvm::Function* kernel = BuildKernelPrototype(*inst, non_constant_buffers); // Build a map from a BufferAllocation to the corresponding argument in our // kernel. @@ -1930,19 +1793,24 @@ IrEmitterUnnested::BuildKernelThunkFromBufferSlices( // For each buffer our kernel might want to touch, bind it to a value derived // from our kernel args. - for (auto* slice : slices) { - const BufferAllocation::Slice& buffer_slice = slice->buffer_slice; - const ShapeIndex& gte_index = slice->gte_index; + for (const auto& hlo_buffer_slice : hlo_slices) { + const HloInstruction* instr = hlo_buffer_slice.instr; + const ShapeIndex& index = hlo_buffer_slice.hlo_index; + const BufferAllocation::Slice& slice = hlo_buffer_slice.buffer_slice; + const ShapeIndex& gte_index = hlo_buffer_slice.gte_index; + + VLOG(3) << "Buffer for " << instr->ToString() << " at " << index.ToString() + << " is found in slice " << slice.ToString() << " at GTE index " + << gte_index.ToString(); llvm::Value* loc; - if (buffer_slice.allocation()->is_constant()) { + if (slice.allocation()->is_constant()) { loc = ir_emitter_context_->llvm_module()->getGlobalVariable( - llvm_ir::ConstantBufferAllocationToGlobalName( - *buffer_slice.allocation())); + llvm_ir::ConstantBufferAllocationToGlobalName(*slice.allocation())); CHECK_NE(loc, nullptr); } else { - loc = InBoundsGEP(kernel_args.at(buffer_slice.allocation()), - {b_.getInt64(buffer_slice.offset())}); + loc = InBoundsGEP(kernel_args.at(slice.allocation()), + {b_.getInt64(slice.offset())}); } // If gte_index is nonempty, we have to dereference `loc` to get to the @@ -1954,7 +1822,7 @@ IrEmitterUnnested::BuildKernelThunkFromBufferSlices( loc = Load(InBoundsGEP(loc, {b_.getInt64(idx)})); } - bind_slice_to_ir_value(slice, loc); + bindings_.BindHloToIrValue(*instr, loc, index); } // Bind the temp buffer so that nested subcomputations can find it if they @@ -1966,66 +1834,9 @@ IrEmitterUnnested::BuildKernelThunkFromBufferSlices( llvm::ConstantPointerNull::get(b_.getInt8PtrTy())); } - return absl::make_unique(thunk_info, non_constant_buffers, - std::string(kernel->getName())); -} - -std::unique_ptr IrEmitterUnnested::BuildKernelThunk( - const HloInstruction* inst, bool implements_whole_instruction) { - std::vector hlo_slices = - GetHloBufferSlices(inst, ir_emitter_context_->buffer_assignment()); - - std::vector slice_ptrs; - slice_ptrs.reserve(hlo_slices.size()); - for (auto& slice : hlo_slices) { - slice_ptrs.push_back(&slice); - } - - return BuildKernelThunkFromBufferSlices( - inst->name(), + return absl::make_unique( implements_whole_instruction ? GetThunkInfo(inst) : Thunk::ThunkInfo(), - slice_ptrs, [this](const BufferSlice* slice, llvm::Value* value) { - const HloBufferSlice* hlo_buffer_slice = - static_cast(slice); - const HloInstruction* instr = hlo_buffer_slice->instr; - const ShapeIndex& index = hlo_buffer_slice->hlo_index; - VLOG(3) << "Buffer for " << instr->ToString() << " at " - << index.ToString() << " is found in slice " - << hlo_buffer_slice->buffer_slice.ToString() << " at GTE index " - << hlo_buffer_slice->gte_index.ToString(); - - bindings_.BindHloToIrValue(*instr, value, index); - }); -} - -std::unique_ptr IrEmitterUnnested::BuildKernelThunkForMlir( - absl::string_view name, Thunk::ThunkInfo thunk_info, - absl::Span slices, - std::vector* ir_arrays) { - absl::flat_hash_set buffers_written; - std::vector slice_ptrs; - slice_ptrs.reserve(slices.size()); - for (auto& slice : slices) { - slice_ptrs.push_back(&slice); - if (slice.written) { - buffers_written.insert(slice.buffer_slice); - } - } - - ir_arrays->clear(); - return BuildKernelThunkFromBufferSlices( - name, thunk_info, slice_ptrs, - [&](const BufferSlice* slice, llvm::Value* value) { - const auto& mlir_slice = static_cast(*slice); - - llvm_ir::IrArray ir_array( - CastToTypedValue(mlir_slice.shape, value, &b_), mlir_slice.shape); - if (!buffers_written.contains(slice->buffer_slice)) { - ir_array.MarkInvariantOverWholeProgram(&value->getContext()); - } - - ir_arrays->push_back(ir_array); - }); + non_constant_buffers, std::string(kernel->getName())); } StatusOr> IrEmitterUnnested::BuildInitializerThunk( @@ -2232,7 +2043,7 @@ Status CheckConditionalBuffersShareAllocation( } // namespace -StatusOr> IrEmitterUnnested::BuildWhileThunk( +std::unique_ptr IrEmitterUnnested::BuildWhileThunk( const HloInstruction* hlo) { // Check that all while-related buffers share an allocation. TF_CHECK_OK(CheckWhileBuffersShareAllocation( @@ -2240,26 +2051,24 @@ StatusOr> IrEmitterUnnested::BuildWhileThunk( // Generate thunk sequence for while 'condition'. HloComputation* condition = hlo->while_condition(); - TF_ASSIGN_OR_RETURN(auto ir_emitter_condition, - IrEmitterUnnested::Create(hlo_module_config_, condition, - ir_emitter_context_)); - TF_RETURN_IF_ERROR(condition->Accept(ir_emitter_condition.get())); + IrEmitterUnnested ir_emitter_condition(hlo_module_config_, condition, + ir_emitter_context_); + TF_CHECK_OK(condition->Accept(&ir_emitter_condition)); // Generate thunk sequence for while 'body'. HloComputation* body = hlo->while_body(); - TF_ASSIGN_OR_RETURN( - auto ir_emitter_body, - IrEmitterUnnested::Create(hlo_module_config_, body, ir_emitter_context_)); - TF_RETURN_IF_ERROR(body->Accept(ir_emitter_body.get())); + IrEmitterUnnested ir_emitter_body(hlo_module_config_, body, + ir_emitter_context_); + TF_CHECK_OK(body->Accept(&ir_emitter_body)); - return std::unique_ptr(new WhileThunk( + return absl::make_unique( GetThunkInfo(hlo), GetAllocationSlice(*condition->root_instruction()), // cond result - ir_emitter_condition->ConsumeThunkSequence(), - ir_emitter_body->ConsumeThunkSequence())); + ir_emitter_condition.ConsumeThunkSequence(), + ir_emitter_body.ConsumeThunkSequence()); } -StatusOr> IrEmitterUnnested::BuildForThunk( +std::unique_ptr IrEmitterUnnested::BuildForThunk( const HloInstruction* hlo, const int64 loop_limit) { // Check that all while-related buffers share an allocation. TF_CHECK_OK(CheckWhileBuffersShareAllocation( @@ -2267,16 +2076,15 @@ StatusOr> IrEmitterUnnested::BuildForThunk( // Generate thunk sequence for while 'body' (will be used a For loop body). HloComputation* body = hlo->while_body(); - TF_ASSIGN_OR_RETURN( - auto ir_emitter_body, - IrEmitterUnnested::Create(hlo_module_config_, body, ir_emitter_context_)); - TF_RETURN_IF_ERROR(body->Accept(ir_emitter_body.get())); + IrEmitterUnnested ir_emitter_body(hlo_module_config_, body, + ir_emitter_context_); + TF_CHECK_OK(body->Accept(&ir_emitter_body)); - return std::unique_ptr(new ForThunk( - GetThunkInfo(hlo), loop_limit, ir_emitter_body->ConsumeThunkSequence())); + return absl::make_unique(GetThunkInfo(hlo), loop_limit, + ir_emitter_body.ConsumeThunkSequence()); } -StatusOr> IrEmitterUnnested::BuildConditionalThunk( +std::unique_ptr IrEmitterUnnested::BuildConditionalThunk( const HloInstruction* hlo) { // Check that the buffers used in conditional are shared with the operands and // result appropriately. @@ -2288,17 +2096,15 @@ StatusOr> IrEmitterUnnested::BuildConditionalThunk( for (int j = 0; j < hlo->branch_count(); ++j) { branch_operands.emplace_back(GetAllocationSlice(*hlo->operand(j + 1))); HloComputation* branch_computation = hlo->branch_computation(j); - TF_ASSIGN_OR_RETURN( - auto ir_emitter, - IrEmitterUnnested::Create(hlo_module_config_, branch_computation, - ir_emitter_context_)); - TF_CHECK_OK(branch_computation->Accept(ir_emitter.get())); - branch_thunks.push_back(std::move(*ir_emitter->ConsumeThunkSequence())); + IrEmitterUnnested ir_emitter(hlo_module_config_, branch_computation, + ir_emitter_context_); + TF_CHECK_OK(branch_computation->Accept(&ir_emitter)); + branch_thunks.push_back(std::move(*ir_emitter.ConsumeThunkSequence())); } - return std::unique_ptr(new ConditionalThunk( + return absl::make_unique( GetThunkInfo(hlo), GetAllocationSlice(*hlo->operand(0)), branch_operands, - std::move(branch_thunks))); + std::move(branch_thunks)); } Status IrEmitterUnnested::EmitTargetElementLoopInThunk( diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h index b9146dd8fae..019fcdf21db 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h @@ -17,7 +17,6 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_ #include "absl/container/inlined_vector.h" -#include "tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h" #include "tensorflow/compiler/xla/service/gpu/ir_emitter.h" #include "tensorflow/compiler/xla/service/gpu/kernel_mapping_scheme.h" #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h" @@ -29,40 +28,6 @@ limitations under the License. namespace xla { namespace gpu { -struct BufferSlice { - // The root buffer to look at. - BufferAllocation::Slice buffer_slice; - - // Describes how to dereference starting at that buffer to get to the buffer - // in question. - ShapeIndex gte_index; -}; - -// Describes how to access a particular subshape for an HLO. For instance if -// `.hlo_index` is {1} and `.gte_index` is {3, 4} then buffer for `.instr` at -// ShapeIndex {1} (i.e. the buffer for the second tuple element of hlo) is -// found at `.buffer_slice`[3][4]. That is, `.slice` is a void***, which we -// dereference twice -- first at index 3, and then at index 4 -- to get the -// address of our buffer. -struct HloBufferSlice : public BufferSlice { - const HloInstruction* instr; - ShapeIndex hlo_index; -}; - -struct MlirBufferSlice : public BufferSlice { - // The buffer is modified by the kernel. - bool written; - - Shape shape; -}; - -struct MlirEmitterInput { - mlir::Operation* op; - absl::string_view name; - Thunk::ThunkInfo thunk_info; - MlirBufferSlice extra_slice; -}; - // Emits LLVM IR for an "unnested computation". // // An unnested computation is an HloComputation which you run by executing one @@ -124,14 +89,12 @@ class IrEmitterUnnested : public IrEmitter, const string& loop_name, llvm::Value* tile_height, llvm::Value* tile_width, KernelSupportLibrary* ksl)>; + IrEmitterUnnested(const HloModuleConfig& hlo_module_config, + const HloComputation* hlo_computation, + IrEmitterContext* ir_emitter_context); IrEmitterUnnested(const IrEmitterUnnested&) = delete; IrEmitterUnnested& operator=(const IrEmitterUnnested&) = delete; - static StatusOr> Create( - const HloModuleConfig& hlo_module_config, - const HloComputation* hlo_computation, - IrEmitterContext* ir_emitter_context); - // Transfers the ownship of thunk_sequence_ out. std::unique_ptr ConsumeThunkSequence() { return std::make_unique(std::move(thunk_sequence_)); @@ -161,7 +124,6 @@ class IrEmitterUnnested : public IrEmitter, Status HandleScatter(HloInstruction* scatter) override; Status HandleSelect(HloInstruction* select) override; Status HandleSort(HloInstruction* sort) override; - Status EmitMlirSort(MlirEmitterInput input); Status HandleTriangularSolve(HloInstruction* hlo) override; Status HandleTupleSelect(HloInstruction* tuple_select) override; Status HandleAllReduce(HloInstruction* crs) override; @@ -186,10 +148,6 @@ class IrEmitterUnnested : public IrEmitter, Status Postprocess(HloInstruction* hlo) override; private: - IrEmitterUnnested(const HloModuleConfig& hlo_module_config, - const HloComputation* hlo_computation, - IrEmitterContext* ir_emitter_context); - // Add a owning Thunk object to the thunk sequence. void AddThunkToThunkSequence(std::unique_ptr thunk) override { thunk_sequence_.emplace_back(std::move(thunk)); @@ -306,7 +264,8 @@ class IrEmitterUnnested : public IrEmitter, // Builds the prototype of the IR kernel for `inst` and adds it to the module. // This kernel takes as arguments pointers to the given buffer allocations. llvm::Function* BuildKernelPrototype( - absl::string_view name, absl::Span args); + const HloInstruction& inst, + absl::Span args); // Helper for writing extra outputs from inside a reduce kernel. Status EmitExtraOutputsForReduce( @@ -531,12 +490,6 @@ class IrEmitterUnnested : public IrEmitter, HloComputation* reducer, llvm::Type* element_type, llvm::Value* partial_result_address); - std::unique_ptr BuildKernelThunkFromBufferSlices( - absl::string_view name, Thunk::ThunkInfo thunk_info, - absl::Span slices, - std::function - bind_slice_to_ir_value); - // Returns a KernelThunk that invokes the kernel emitted for `inst`. The // caller needs to make sure `inst` outlives the lifetime of the returned // Thunk object. 'implements_whole_instruction' specifies whether this @@ -545,11 +498,6 @@ class IrEmitterUnnested : public IrEmitter, std::unique_ptr BuildKernelThunk( const HloInstruction* inst, bool implements_whole_instruction); - std::unique_ptr BuildKernelThunkForMlir( - absl::string_view name, Thunk::ThunkInfo thunk_info, - absl::Span slices, - std::vector* ir_arrays); - // Returns a thunk that, given a reduce or select-and-scatter op, // initializes its memory to the appropriate initial value. StatusOr> BuildInitializerThunk( @@ -557,18 +505,17 @@ class IrEmitterUnnested : public IrEmitter, // Returns a WhileThunk that invokes thunk sequences for 'condition' and // 'body' sub-computations of while instruction 'hlo'. - StatusOr> BuildWhileThunk(const HloInstruction* hlo); + std::unique_ptr BuildWhileThunk(const HloInstruction* hlo); // Returns a ForThunk which executes 'loop_limit' invocations of a thunk // sequence from the 'body' sub-computation of the while instruction 'hlo'. - StatusOr> BuildForThunk(const HloInstruction* hlo, - const int64 loop_limit); + std::unique_ptr BuildForThunk(const HloInstruction* hlo, + const int64 loop_limit); // Returns a ConditionalThunk which executes the thunk sequence for the // 'branch_computation' corresponding to the predicate/branch_index of the // given conditional instruction. - StatusOr> BuildConditionalThunk( - const HloInstruction* hlo); + std::unique_ptr BuildConditionalThunk(const HloInstruction* hlo); // Emits current thread id with the given type. // @@ -598,9 +545,6 @@ class IrEmitterUnnested : public IrEmitter, absl::optional thread_id_filter = absl::nullopt, absl::optional block_id_filter = absl::nullopt); - StatusOr GetOrCreateSubComputationFromRegion( - mlir::Region* region); - // Returns the last generated thunk. Thunk* LastThunk() const { return thunk_sequence_.back().get(); } @@ -611,14 +555,6 @@ class IrEmitterUnnested : public IrEmitter, // The HloComputation that this IrEmitter emits code for. const HloComputation* hlo_computation_; - - mlir::OwningModuleRef mlir_scratch_module_; - - // This is for cache-purpose only. It has no significant semantics. - mlir::LhloDialectEmitter lhlo_scratch_emitter_; - - absl::flat_hash_map> - scratch_nested_computations_; }; } // namespace gpu diff --git a/tensorflow/compiler/xla/service/gpu/tests/BUILD b/tensorflow/compiler/xla/service/gpu/tests/BUILD index 809b277317f..a2bddd2d0d7 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/BUILD +++ b/tensorflow/compiler/xla/service/gpu/tests/BUILD @@ -458,35 +458,6 @@ xla_test( ], ) -tf_cc_test( - name = "sorting_test", - srcs = [ - "sorting_test.cc", - ], - tags = tf_cuda_tests_tags() + [ - "no_rocm", - ], - deps = [ - ":gpu_codegen_test", - "//tensorflow/compiler/xla:debug_options_flags", - "//tensorflow/compiler/xla:statusor", - "//tensorflow/compiler/xla:xla_proto_cc", - "//tensorflow/compiler/xla/service:gpu_plugin", - "//tensorflow/compiler/xla/service:hlo", - "//tensorflow/compiler/xla/service:hlo_module_config", - "//tensorflow/compiler/xla/service:hlo_parser", - "//tensorflow/compiler/xla/service/gpu:gpu_executable", - "//tensorflow/compiler/xla/tests:filecheck", - "//tensorflow/compiler/xla/tests:hlo_test_base", - "//tensorflow/compiler/xla/tests:llvm_irgen_test_base", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/stream_executor/lib", - "@com_google_absl//absl/memory", - ], -) - tf_cc_binary( name = "hlo_to_llvm_ir", srcs = ["hlo_to_llvm_ir.cc"], diff --git a/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo b/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo index 4d29a8df116..272c9a25769 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo +++ b/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo @@ -8,162 +8,162 @@ compare { ROOT lt = pred[] compare(p.0.lhs, p.0.rhs), direction=LT } -// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) +// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 -// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 -// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 -// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] -// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 -// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] -// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 +// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] +// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 +// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] +// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: call void @region_0_4(float* [[TMP16]], float* [[TMP17]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP18]], 0 +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: call void @compare(float* [[TMP12]], float* [[TMP13]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP14:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP14]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] -// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 +// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = load float, float* [[TMP13]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: store float [[TMP16]], float* [[TMP18]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define internal void @region_0_4(float* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) +// CHECK: define internal void @compare(float* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) // CHECK-NEXT: entry: -// CHECK-NEXT: [[COMPARE_3_TYPED:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[ARG_0_1_TYPED:%.*]], align 4 -// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[ARG_1_2_TYPED:%.*]], align 4 +// CHECK-NEXT: [[LT_TYPED:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[P_0_LHS_TYPED]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[P_0_RHS_TYPED]], align 4 // CHECK-NEXT: [[TMP2:%.*]] = fcmp olt float [[TMP0]], [[TMP1]] // CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i8 -// CHECK-NEXT: store i8 [[TMP3]], i8* [[COMPARE_3_TYPED]], align 1 -// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[COMPARE_3_TYPED]], align 1 -// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG:%.*]], align 1 +// CHECK-NEXT: store i8 [[TMP3]], i8* [[LT_TYPED]], align 1 +// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[LT_TYPED]], align 1 +// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG]], align 1 // CHECK-NEXT: ret void -// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) { +// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) { // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 -// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 -// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP11:%.*]] = xor i64 [[TMP8]], 3 -// CHECK-NEXT: [[TMP12:%.*]] = icmp slt i64 [[TMP8]], [[TMP11]] -// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], 3 -// CHECK-NEXT: [[TMP14:%.*]] = and i1 [[TMP12]], [[TMP13]] -// CHECK-NEXT: br i1 [[TMP14]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = xor i64 [[TMP4]], 3 +// CHECK-NEXT: [[TMP8:%.*]] = icmp slt i64 [[TMP4]], [[TMP7]] +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], 3 +// CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP8]], [[TMP9]] +// CHECK-NEXT: br i1 [[TMP10]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP8]] -// CHECK-NEXT: call void @region_0_4(float* [[TMP15]], float* [[TMP16]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP17:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP17]], 0 +// CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: call void @compare(float* [[TMP11]], float* [[TMP12]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP13:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP13]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP18:%.*]] = load float, float* [[TMP15]], align 4 -// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP8]] -// CHECK-NEXT: store float [[TMP18]], float* [[TMP20]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 +// CHECK-NEXT: [[TMP14:%.*]] = load float, float* [[TMP11]], align 4 +// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: store float [[TMP14]], float* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) { +// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) { // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 -// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 -// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 -// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] -// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 -// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] -// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 +// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] +// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 +// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] +// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: call void @region_0_4(float* [[TMP16]], float* [[TMP17]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP18]], 0 +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: call void @compare(float* [[TMP12]], float* [[TMP13]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP14:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP14]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] -// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 +// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = load float, float* [[TMP13]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: store float [[TMP16]], float* [[TMP18]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] ENTRY main { x = f32[2, 3] parameter(0) @@ -182,198 +182,210 @@ compare { ROOT lt = pred[] compare(p.1.lhs, p.1.rhs), direction=LT } -// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* -// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 -// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 -// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* +// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 +// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3]], i64 0 +// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 -// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP10]], 2 -// CHECK-NEXT: [[TMP14:%.*]] = xor i64 [[TMP13]], 1 -// CHECK-NEXT: [[TMP15:%.*]] = icmp slt i64 [[TMP13]], [[TMP14]] -// CHECK-NEXT: [[TMP16:%.*]] = icmp slt i64 [[TMP14]], 3 -// CHECK-NEXT: [[TMP17:%.*]] = and i1 [[TMP15]], [[TMP16]] -// CHECK-NEXT: br i1 [[TMP17]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 +// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] +// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 +// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] +// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP14]] -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP14]] -// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: call void @region_0_6(i32* [[TMP18]], i32* [[TMP19]], float* [[TMP20]], float* [[TMP21]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP22:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP22]], 0 +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: call void @compare(i32* [[TMP12]], i32* [[TMP13]], float* [[TMP14]], float* [[TMP15]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP16:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP16]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP18]], align 4 -// CHECK-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: store i32 [[TMP23]], i32* [[TMP25]], align 4 -// CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP14]] -// CHECK-NEXT: store i32 [[TMP24]], i32* [[TMP26]], align 4 -// CHECK-NEXT: [[TMP27:%.*]] = load float, float* [[TMP20]], align 4 -// CHECK-NEXT: [[TMP28:%.*]] = load float, float* [[TMP21]], align 4 -// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: store float [[TMP27]], float* [[TMP29]], align 4 -// CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP14]] -// CHECK-NEXT: store float [[TMP28]], float* [[TMP30]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP13]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: store i32 [[TMP18]], i32* [[TMP20]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = load float, float* [[TMP14]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = load float, float* [[TMP15]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP21]], float* [[TMP23]], align 4 +// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: store float [[TMP22]], float* [[TMP24]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define internal void @region_0_6(i32* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], i32* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) +// CHECK: define internal void @compare(i32* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], i32* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) // CHECK-NEXT: entry: -// CHECK-NEXT: [[COMPARE_5_TYPED:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[ARG_2_3_TYPED:%.*]], align 4 -// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[ARG_3_4_TYPED:%.*]], align 4 +// CHECK-NEXT: [[LT_TYPED:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[P_1_LHS_TYPED]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[P_1_RHS_TYPED]], align 4 // CHECK-NEXT: [[TMP2:%.*]] = fcmp olt float [[TMP0]], [[TMP1]] // CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i8 -// CHECK-NEXT: store i8 [[TMP3]], i8* [[COMPARE_5_TYPED]], align 1 -// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[COMPARE_5_TYPED]], align 1 -// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG:%.*]], align 1 +// CHECK-NEXT: store i8 [[TMP3]], i8* [[LT_TYPED]], align 1 +// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[LT_TYPED]], align 1 +// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG]], align 1 // CHECK-NEXT: ret void -// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* -// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 -// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 -// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* +// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2:%.*]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3:%.*]], i64 0 +// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 -// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP13:%.*]] = xor i64 [[TMP10]], 3 -// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP10]], [[TMP13]] -// CHECK-NEXT: [[TMP15:%.*]] = icmp slt i64 [[TMP13]], 3 -// CHECK-NEXT: [[TMP16:%.*]] = and i1 [[TMP14]], [[TMP15]] -// CHECK-NEXT: br i1 [[TMP16]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = xor i64 [[TMP4]], 3 +// CHECK-NEXT: [[TMP8:%.*]] = icmp slt i64 [[TMP4]], [[TMP7]] +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], 3 +// CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP8]], [[TMP9]] +// CHECK-NEXT: br i1 [[TMP10]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP10]] -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP10]] -// CHECK-NEXT: call void @region_0_6(i32* [[TMP17]], i32* [[TMP18]], float* [[TMP19]], float* [[TMP20]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP21:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP21]], 0 +// CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: call void @compare(i32* [[TMP11]], i32* [[TMP12]], float* [[TMP13]], float* [[TMP14]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP15:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP15]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP18]], align 4 -// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP10]] -// CHECK-NEXT: store i32 [[TMP22]], i32* [[TMP24]], align 4 -// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: store i32 [[TMP23]], i32* [[TMP25]], align 4 -// CHECK-NEXT: [[TMP26:%.*]] = load float, float* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP27:%.*]] = load float, float* [[TMP20]], align 4 -// CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP10]] -// CHECK-NEXT: store float [[TMP26]], float* [[TMP28]], align 4 -// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: store float [[TMP27]], float* [[TMP29]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP11]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: store i32 [[TMP16]], i32* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP13]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = load float, float* [[TMP14]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP21]], float* [[TMP23]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* -// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 -// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 -// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* +// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2:%.*]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3:%.*]], i64 0 +// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 -// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: -// CHECK-NEXT: [[TMP13:%.*]] = bitcast [2 x [3 x i32]]* [[TMP1]] to i8* -// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[TMP5]], i64 0, i64 0 -// CHECK-NEXT: store i8* [[TMP13]], i8** [[TMP14]], align 8 -// CHECK-NEXT: [[TMP15:%.*]] = bitcast [2 x [3 x float]]* [[TMP3]] to i8* -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[TMP5]], i64 0, i64 1 -// CHECK-NEXT: store i8* [[TMP15]], i8** [[TMP16]], align 8 +// CHECK-NEXT: [[TMP7:%.*]] = bitcast [2 x [3 x i32]]* [[SORT_TYPED2]] to i8* +// CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[SORT_TYPED]], i64 0, i64 0 +// CHECK-NEXT: store i8* [[TMP7]], i8** [[TMP8]], align 8 +// CHECK-NEXT: [[TMP9:%.*]] = bitcast [2 x [3 x float]]* [[SORT_TYPED4]] to i8* +// CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[SORT_TYPED]], i64 0, i64 1 +// CHECK-NEXT: store i8* [[TMP9]], i8** [[TMP10]], align 8 // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP10]], 2 -// CHECK-NEXT: [[TMP18:%.*]] = xor i64 [[TMP17]], 1 -// CHECK-NEXT: [[TMP19:%.*]] = icmp slt i64 [[TMP17]], [[TMP18]] -// CHECK-NEXT: [[TMP20:%.*]] = icmp slt i64 [[TMP18]], 3 -// CHECK-NEXT: [[TMP21:%.*]] = and i1 [[TMP19]], [[TMP20]] -// CHECK-NEXT: br i1 [[TMP21]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP4]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 +// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] +// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 +// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] +// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP18]] -// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP17]] -// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP18]] -// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP17]] -// CHECK-NEXT: call void @region_0_6(i32* [[TMP22]], i32* [[TMP23]], float* [[TMP24]], float* [[TMP25]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP26:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP26]], 0 +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP12]] +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP11]] +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP12]] +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP11]] +// CHECK-NEXT: call void @compare(i32* [[TMP16]], i32* [[TMP17]], float* [[TMP18]], float* [[TMP19]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP20:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP20]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP22]], align 4 -// CHECK-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP23]], align 4 -// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP17]] -// CHECK-NEXT: store i32 [[TMP27]], i32* [[TMP29]], align 4 -// CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP18]] -// CHECK-NEXT: store i32 [[TMP28]], i32* [[TMP30]], align 4 -// CHECK-NEXT: [[TMP31:%.*]] = load float, float* [[TMP24]], align 4 -// CHECK-NEXT: [[TMP32:%.*]] = load float, float* [[TMP25]], align 4 -// CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP17]] -// CHECK-NEXT: store float [[TMP31]], float* [[TMP33]], align 4 -// CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP18]] -// CHECK-NEXT: store float [[TMP32]], float* [[TMP34]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = load i32, i32* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP11]] +// CHECK-NEXT: store i32 [[TMP21]], i32* [[TMP23]], align 4 +// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP12]] +// CHECK-NEXT: store i32 [[TMP22]], i32* [[TMP24]], align 4 +// CHECK-NEXT: [[TMP25:%.*]] = load float, float* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP26:%.*]] = load float, float* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP11]] +// CHECK-NEXT: store float [[TMP25]], float* [[TMP27]], align 4 +// CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP12]] +// CHECK-NEXT: store float [[TMP26]], float* [[TMP28]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] ENTRY main { x = s32[2, 3] parameter(0) diff --git a/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc b/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc deleted file mode 100644 index 197a0c6cfeb..00000000000 --- a/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include - -#include "tensorflow/compiler/xla/service/gpu/gpu_executable.h" -#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h" -#include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/hlo_module_config.h" -#include "tensorflow/compiler/xla/service/hlo_parser.h" -#include "tensorflow/compiler/xla/statusor.h" -#include "tensorflow/compiler/xla/tests/filecheck.h" -#include "tensorflow/compiler/xla/tests/hlo_test_base.h" -#include "tensorflow/compiler/xla/xla.pb.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/stream_executor/lib/statusor.h" - -namespace xla { -namespace gpu { - -namespace { - -class SortingTest : public GpuCodegenTest { - protected: - HloModuleConfig ConfigWithoutLayoutAssignment() { - HloModuleConfig config; - auto debug_options = HloTestBase::GetDebugOptionsForTest(); - // Disable layout_assignment to use the preassigned layouts. - debug_options.add_xla_disable_hlo_passes("layout-assignment"); - config.set_debug_options(debug_options); - return config; - } -}; - -TEST_F(SortingTest, Regression1) { - const char* hlo_text = R"( -HloModule TestModule - -compare { - p.0.lhs = f32[] parameter(0) - p.0.rhs = f32[] parameter(1) - ROOT lt = pred[] compare(p.0.lhs, p.0.rhs), direction=LT -} - -ENTRY TestComputation { - x = f32[3, 2]{1, 0} parameter(0) - x.copy = f32[3, 2]{0, 1} copy(x) - ROOT sort = f32[3, 2]{0, 1} sort(x.copy), dimensions={1}, to_apply=compare -} - -)"; - - EXPECT_TRUE(RunAndCompareNoHloPasses(hlo_text, ErrorSpec{1e-5, 1e-5})); -} - -} // namespace -} // namespace gpu -} // namespace xla diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc index 2963d546380..b01ae2efe43 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc @@ -415,10 +415,9 @@ llvm::Instruction* AddRangeMetadata(int64 lower, int64 upper, return inst; } -string IrName(absl::string_view a) { - std::string s(a); - s.erase(std::remove(s.begin(), s.end(), '%'), s.end()); - return s; +string IrName(string a) { + a.erase(std::remove(a.begin(), a.end(), '%'), a.end()); + return a; } string IrName(absl::string_view a, absl::string_view b) { diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h index c0a55e4da33..642965b6470 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h @@ -87,7 +87,7 @@ string DumpModuleToString(const llvm::Module& module); // - joining all of the nonempty inputs by '.', and then // - removing all '%'s. // -string IrName(absl::string_view a); +string IrName(string a); string IrName(absl::string_view a, absl::string_view b); string IrName(const HloInstruction* a, absl::string_view b = ""); From d9d7f3711880bb7c6505685796968c00e9ea86a0 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Tue, 11 Aug 2020 12:33:11 -0700 Subject: [PATCH 0837/1017] Add build instructions for the standalone mlir-hlo compiler in the Readme.md + a title PiperOrigin-RevId: 326077782 Change-Id: I6fb6c95a47de332cbe1543a51b093f829e6b4ce8 --- tensorflow/compiler/mlir/hlo/.gitignore | 4 +++ tensorflow/compiler/mlir/hlo/README.md | 37 +++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 tensorflow/compiler/mlir/hlo/.gitignore diff --git a/tensorflow/compiler/mlir/hlo/.gitignore b/tensorflow/compiler/mlir/hlo/.gitignore new file mode 100644 index 00000000000..cc1696bf575 --- /dev/null +++ b/tensorflow/compiler/mlir/hlo/.gitignore @@ -0,0 +1,4 @@ +build +llvm-project +llvm-build + diff --git a/tensorflow/compiler/mlir/hlo/README.md b/tensorflow/compiler/mlir/hlo/README.md index 1be6fb29d13..9eaa14031fd 100644 --- a/tensorflow/compiler/mlir/hlo/README.md +++ b/tensorflow/compiler/mlir/hlo/README.md @@ -1,4 +1,4 @@ -# MLIR-HLO +# MLIR-HLO: A Standalone "HLO" MLIR-based Compiler The code here exists in two places: @@ -22,10 +22,43 @@ upstream. ## QuickStart: building and testing -TODO +These instructions work on Linux, you may have to adjust for your plaform. + +To build the code in this repository, you need a clone of the LLVM/MLIR git +repository: + + $ git clone https://github.com/llvm/llvm-project.git + + +You need to make sure you have the right commit checked out in the LLVM +repository (you need to do this every time you pull from this repo): + + $ (cd llvm-project && git checkout $(cat build_tools/llvm_version.txt)) + +We provide a script to configure and build LLVM/MLIR: + + $ build_tools/build_mlir.sh ${PWD}/llvm-project/ ${PWD}/llvm-build + +Again this is something to do every time you pull from this repository and the +LLVM revision changes. + +Finally you can build and test this repository: + + $ mkdir build && cd build + $ cmake .. -GNinja \ + -DLLVM_ENABLE_LLD=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_ENABLE_ASSERTIONS=On \ + -DMLIR_DIR=${PWD}/../llvm-build/lib/cmake/mlir + $ ninja check-mlir-hlo + ## Overview +MLIR-HLO aims to provide an end-to-end compiler for CPU and GPU, as well as +building reusable blocks for other accelerators. This is heavily inspired by the +success of XLA. + [XLA](https://www.tensorflow.org/xla/) (Accelerated Linear Algebra) is a domain-specific compiler framework and execution environment for linear algebra, which powers code-generation for ML frameworks like TensorFlow, JAX, and others. From 68016e2697820763096254bfb139478f40c79442 Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Tue, 11 Aug 2020 13:02:39 -0700 Subject: [PATCH 0838/1017] Rollback of rollback of disabling XLA:CPU/GPU devices by default. PiperOrigin-RevId: 326084520 Change-Id: Id537bc29ac9d4c4c3c8fe7533e8d4151adb4cadf --- RELEASE.md | 3 + tensorflow/compiler/jit/BUILD | 1 + tensorflow/compiler/jit/flags.cc | 2 +- tensorflow/compiler/jit/kernels/xla_ops.cc | 2 +- .../jit/mark_for_compilation_pass_test.cc | 5 ++ .../jit/partially_decluster_pass_test.cc | 31 ------- .../compiler/jit/xla_compile_on_demand_op.cc | 53 ++++++----- tensorflow/compiler/jit/xla_device.cc | 29 +++--- tensorflow/compiler/jit/xla_device.h | 2 + .../jit/xla_ops_on_regular_devices.cc | 89 +++++++++++++++++++ tensorflow/compiler/jit/xla_platform_info.cc | 7 +- tensorflow/compiler/jit/xla_platform_info.h | 2 +- tensorflow/compiler/tests/BUILD | 1 + .../tests/unary_ops_composition_test.cc | 6 ++ .../compiler/tests/xla_device_gpu_test.py | 5 ++ tensorflow/compiler/tests/xla_test.py | 2 + tensorflow/compiler/tf2xla/BUILD | 2 + .../compiler/tf2xla/const_analysis_test.cc | 6 ++ .../fused_batchnorm_reserve_space_test.cc | 7 ++ tensorflow/compiler/xrt/BUILD | 1 + tensorflow/compiler/xrt/ops/xrt_state_ops.cc | 6 ++ .../optimizers/pin_to_host_optimizer_test.cc | 20 ----- tensorflow/python/eager/context.py | 14 +-- tensorflow/python/framework/config_test.py | 9 -- .../parallel_for/xla_control_flow_ops_test.py | 5 ++ tensorflow/python/tfe_wrapper.cc | 3 + 26 files changed, 206 insertions(+), 107 deletions(-) create mode 100644 tensorflow/compiler/jit/xla_ops_on_regular_devices.cc diff --git a/RELEASE.md b/RELEASE.md index 430e1b83885..241a5077251 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -33,6 +33,9 @@ shape assumptions (note that you can pass shapes with `None` entries for axes that are meant to be dynamic). You can also disable the input checking entirely by setting `model.input_spec = None`. +* XLA:CPU and XLA:GPU devices are no longer registered by default. Use + `TF_XLA_FLAGS=--tf_xla_enable_xla_devices` if you really need them (to be + removed). ## Known Caveats diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index d05bb8264c3..35c6a8b0357 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -206,6 +206,7 @@ cc_library( "xla_device.cc", "xla_device_context.cc", "xla_device_ops.cc", + "xla_ops_on_regular_devices.cc", "xla_platform_info.cc", ], hdrs = [ diff --git a/tensorflow/compiler/jit/flags.cc b/tensorflow/compiler/jit/flags.cc index ff085c854c6..a4a750bae0d 100644 --- a/tensorflow/compiler/jit/flags.cc +++ b/tensorflow/compiler/jit/flags.cc @@ -159,7 +159,7 @@ void AllocateAndParseFlags() { device_flags = new XlaDeviceFlags; device_flags->tf_xla_compile_on_demand = false; - device_flags->tf_xla_enable_xla_devices = true; + device_flags->tf_xla_enable_xla_devices = false; ops_flags = new XlaOpsCommonFlags; ops_flags->tf_xla_always_defer_compilation = false; diff --git a/tensorflow/compiler/jit/kernels/xla_ops.cc b/tensorflow/compiler/jit/kernels/xla_ops.cc index 9cee4b9af28..de462928c46 100644 --- a/tensorflow/compiler/jit/kernels/xla_ops.cc +++ b/tensorflow/compiler/jit/kernels/xla_ops.cc @@ -191,7 +191,7 @@ static Status CompileToLocalExecutable( absl::optional tf_allocator_adapter; XlaCompiler::Options options = GenerateCompilerOptions( - cache, ctx, platform_info, has_ref_vars, &tf_allocator_adapter); + *cache, ctx, platform_info, has_ref_vars, &tf_allocator_adapter); std::map constant_args; for (int i : constants) { diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc index e88319bb732..1be3e5ba9e7 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc @@ -44,6 +44,11 @@ using ::tensorflow::testing::FindNodeByName; namespace tensorflow { namespace { +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + REGISTER_OP("UncompilableNullary").Output("o: float"); REGISTER_OP("UncompilableUnary").Input("a: float").Output("o: float"); diff --git a/tensorflow/compiler/jit/partially_decluster_pass_test.cc b/tensorflow/compiler/jit/partially_decluster_pass_test.cc index 7378d17f88d..87c9fbf0af7 100644 --- a/tensorflow/compiler/jit/partially_decluster_pass_test.cc +++ b/tensorflow/compiler/jit/partially_decluster_pass_test.cc @@ -406,37 +406,6 @@ TEST(PartiallyDeclusterPassTest, DontDeclusterXlaDeviceOps) { EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0"); } -TEST(PartiallyDeclusterPassTest, DontDeclusterNonTensorFlowOps) { - tensorflow::Scope s = tensorflow::Scope::NewRootScope(); - Output dynamic_slice_operand = - ops::Placeholder(s.WithOpName("dynamic_slice_operand"), DT_INT32, - ops::Placeholder::Attrs{}); - Output dynamic_slice_begin = ops::Placeholder( - s.WithOpName("dynamic_slice_begin"), DT_INT32, ops::Placeholder::Attrs{}); - Output dynamic_slice_size = ops::Placeholder( - s.WithOpName("dynamic_slice_size"), DT_INT32, ops::Placeholder::Attrs{}); - Output dynamic_slice = - ops::XlaDynamicSlice(s.WithOpName("dynamic_slice"), dynamic_slice_operand, - dynamic_slice_begin, dynamic_slice_size); - - Output reshape_input = ops::Placeholder(s.WithOpName("reshape_input"), - DT_FLOAT, ops::Placeholder::Attrs{}); - Output reshape = - ops::Reshape(s.WithOpName("reshape"), reshape_input, dynamic_slice); - - AddToCluster({dynamic_slice.node(), reshape.node()}, "cluster_0"); - - std::unique_ptr graph = absl::make_unique(OpRegistry::Global()); - TF_ASSERT_OK(s.ToGraph(graph.get())); - - Node* n = FindNodeByName(*graph, "dynamic_slice"); - ASSERT_NE(n, nullptr); - - TF_ASSERT_OK(PartiallyDecluster(&graph)); - - EXPECT_EQ(GetXlaClusterForNode(*n), "cluster_0"); -} - TEST(PartiallyDeclusterPassTest, EliminatedUnusedNodes) { const char* const kClusteredProducer0Name = "ClusteredProducer0"; const char* const kClusteredProducer1Name = "ClusteredProducer1"; diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc index 73c512bfa6f..da251c2c8f3 100644 --- a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc +++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc @@ -48,9 +48,11 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, const ResourceVarsSnapshot& variable_args) { xla::LocalClient* client = static_cast(cache->client()); + absl::optional tf_allocator_adapter; + se::DeviceMemoryAllocator* allocator = + GetAllocator(&tf_allocator_adapter, ctx, platform_info_); XlaComputationLaunchContext launch_context( - client, client->backend().memory_allocator(), - client->default_device_ordinal(), + client, allocator, client->default_device_ordinal(), /*allocate_xla_tensors=*/platform_info_.xla_device_metadata() != nullptr, platform_info_.xla_device_metadata() ? platform_info_.xla_device_metadata()->UseMultipleStreams() @@ -76,7 +78,7 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx, VLOG(2) << "Executing computation: " << name(); xla::ExecutableRunOptions run_options; run_options.set_stream(stream); - run_options.set_allocator(client->backend().memory_allocator()); + run_options.set_allocator(allocator); run_options.set_intra_op_thread_pool(&ctx->eigen_cpu_device()); run_options.set_rng_seed(GetXLARandomSeed()); @@ -108,6 +110,7 @@ Status XlaCompileOnDemandOp::Compile( for (int64 i = 0; i < ctx->num_inputs(); ++i) { const Tensor& device_tensor = ctx->input(i); + if (const XlaTensor* xla_tensor = XlaTensor::FromTensor(&device_tensor)) { if (xla_tensor->has_host_tensor()) { if (absl::c_binary_search(constant_input_indices, i)) { @@ -118,24 +121,30 @@ Status XlaCompileOnDemandOp::Compile( if (!constant_arguments.count(i)) { if (absl::c_binary_search(constant_input_indices, i)) { - // Slow path; the argument is not available as a host constant so we - // must fetch it synchronously. - Tensor host_tensor; - AllocatorAttributes attrs; - attrs.set_on_host(true); - TF_RETURN_IF_ERROR(ctx->allocate_temp( - device_tensor.dtype(), device_tensor.shape(), &host_tensor, attrs)); - Status status = ctx->op_device_context()->CopyDeviceTensorToCPUSync( - &device_tensor, "ConstantArgument", - reinterpret_cast(ctx->device()), &host_tensor); - if (!status.ok()) { - LOG(ERROR) << "Copying tensor of shape " - << device_tensor.shape().DebugString() << " from " - << ctx->device()->name() << "to CPU failed with " - << status.ToString(); - return status; + if (ctx->input_memory_type(i) != HOST_MEMORY && + ctx->op_device_context()) { + // Slow path; the argument is not available as a host constant so we + // must fetch it synchronously. + Tensor host_tensor; + AllocatorAttributes attrs; + attrs.set_on_host(true); + TF_RETURN_IF_ERROR(ctx->allocate_temp(device_tensor.dtype(), + device_tensor.shape(), + &host_tensor, attrs)); + Status status = ctx->op_device_context()->CopyDeviceTensorToCPUSync( + &device_tensor, "ConstantArgument", + reinterpret_cast(ctx->device()), &host_tensor); + if (!status.ok()) { + LOG(ERROR) << "Copying tensor of shape " + << device_tensor.shape().DebugString() << " from " + << ctx->device()->name() << "to CPU failed with " + << status.ToString(); + return status; + } + constant_arguments[i] = host_tensor; + } else { + constant_arguments[i] = device_tensor; } - constant_arguments[i] = host_tensor; } } } @@ -153,7 +162,7 @@ Status XlaCompileOnDemandOp::Compile( absl::optional tf_allocator_adapter; XlaCompiler::Options options = - GenerateCompilerOptions(*cache, ctx, platform_info_, + GenerateCompilerOptions(**cache, ctx, platform_info_, /*has_ref_vars=*/true, &tf_allocator_adapter); XlaCompiler::CompileOptions compile_options; @@ -184,6 +193,8 @@ void XlaCompileOnDemandOp::Compute(OpKernelContext* ctx) { xla::LocalExecutable* executable; ResourceVarsSnapshot variable_args; XlaCompilationCache* cache; + OP_REQUIRES(ctx, ctx->function_library(), + errors::Internal("Function library missing")); OP_REQUIRES_OK(ctx, Compile(ctx, &result, &cache, &variable_args, &executable)); diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc index 7842513331d..c47c9a29c1a 100644 --- a/tensorflow/compiler/jit/xla_device.cc +++ b/tensorflow/compiler/jit/xla_device.cc @@ -61,6 +61,21 @@ limitations under the License. namespace tensorflow { +// Default PaddedShapeFn implementation that simply returns the unpadded +// on-device shape. This is accurate for CPU and GPU devices that neither +// transpose nor pad tensors. +Status DefaultPaddedShapeFn(const Tensor& tensor, xla::Shape* shape) { + const tensorflow::XlaTensor* xla_tensor = + tensorflow::XlaTensor::FromTensor(&tensor); + if (xla_tensor == nullptr) { + return TensorShapeToXLAShape(tensor.dtype(), tensor.shape(), shape); + } + + const xla::ShapedBuffer& shaped_buffer = xla_tensor->shaped_buffer(); + *shape = shaped_buffer.on_device_shape(); + return Status::OK(); +} + // Caches a XlaDeviceAllocator per pair. A // XlaDeviceAllocator is created on demand and is associated with a // XlaDevice. It outlives the device itself (for instance, the buffer @@ -116,20 +131,6 @@ XlaDeviceAllocator* XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator( namespace { -// Default PaddedShapeFn implementation that simply returns the unpadded -// on-device shape. This is accurate for CPU and GPU devices that neither -// transpose nor pad tensors. -Status DefaultPaddedShapeFn(const Tensor& tensor, xla::Shape* shape) { - const tensorflow::XlaTensor* xla_tensor = - tensorflow::XlaTensor::FromTensor(&tensor); - if (xla_tensor == nullptr) { - return TensorShapeToXLAShape(tensor.dtype(), tensor.shape(), shape); - } - - const xla::ShapedBuffer& shaped_buffer = xla_tensor->shaped_buffer(); - *shape = shaped_buffer.on_device_shape(); - return Status::OK(); -} static DeviceAttributes BuildXlaDeviceAttributes(const string& name_prefix, const string& device_name, diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h index 30f9a99e36a..f7e7ee9cf95 100644 --- a/tensorflow/compiler/jit/xla_device.h +++ b/tensorflow/compiler/jit/xla_device.h @@ -280,6 +280,8 @@ struct XlaDeviceOpRegistrations { XlaDeviceOpRegistrations* RegisterXlaDeviceKernels(const char* device, const char* jit_device); +Status DefaultPaddedShapeFn(const Tensor& tensor, xla::Shape* shape); + } // namespace tensorflow #endif // TENSORFLOW_COMPILER_JIT_XLA_DEVICE_H_ diff --git a/tensorflow/compiler/jit/xla_ops_on_regular_devices.cc b/tensorflow/compiler/jit/xla_ops_on_regular_devices.cc new file mode 100644 index 00000000000..82510a4926b --- /dev/null +++ b/tensorflow/compiler/jit/xla_ops_on_regular_devices.cc @@ -0,0 +1,89 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Register XlaXXX operations on regular CPU/GPU devices using +// `XlaCompileOnDemandOp`. +#include "tensorflow/compiler/jit/xla_compile_on_demand_op.h" +#include "tensorflow/core/framework/op_kernel.h" + +namespace tensorflow { + +#define REGISTER_XLA_OPS_ON_DEVICE(DEVICE) \ + REGISTER_KERNEL_BUILDER(Name("XlaConv") \ + .HostMemory("window_strides") \ + .HostMemory("padding") \ + .HostMemory("lhs_dilation") \ + .HostMemory("rhs_dilation") \ + .HostMemory("feature_group_count") \ + .Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER( \ + Name("XlaBroadcastHelper").HostMemory("broadcast_dims").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSelfAdjointEig").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSvd").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaDot").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaDynamicSlice").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaDynamicUpdateSlice").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaIf").Device(DEVICE), XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaPad").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaRecv").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaReduce").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaReduceWindow").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSelectAndScatter") \ + .HostMemory("window_dimensions") \ + .HostMemory("window_strides") \ + .HostMemory("padding") \ + .Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSend").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSort").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaKeyValueSort").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaWhile").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaDequantize").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaEinsum").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSpmdShardToFullShape").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaSharding").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaReplicaId").Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaGather") \ + .HostMemory("start_indices") \ + .HostMemory("slice_sizes") \ + .Device(DEVICE), \ + XlaCompileOnDemandOp); \ + REGISTER_KERNEL_BUILDER(Name("XlaScatter").Device(DEVICE), \ + XlaCompileOnDemandOp); + +REGISTER_XLA_OPS_ON_DEVICE(DEVICE_CPU); +REGISTER_XLA_OPS_ON_DEVICE(DEVICE_GPU); + +} // namespace tensorflow diff --git a/tensorflow/compiler/jit/xla_platform_info.cc b/tensorflow/compiler/jit/xla_platform_info.cc index e2a89353055..a5e12b37563 100644 --- a/tensorflow/compiler/jit/xla_platform_info.cc +++ b/tensorflow/compiler/jit/xla_platform_info.cc @@ -128,16 +128,17 @@ se::DeviceMemoryAllocator* GetAllocator( } XlaCompiler::Options GenerateCompilerOptions( - XlaCompilationCache* cache, OpKernelContext* ctx, + const XlaCompilationCache& cache, OpKernelContext* ctx, const XlaPlatformInfo& platform_info, bool has_ref_vars, absl::optional* tf_allocator_adapter) { + CHECK(ctx->function_library()); XlaCompiler::Options options; - options.client = static_cast(cache->client()); + options.client = static_cast(cache.client()); if (ctx->op_device_context() != nullptr) { options.device_ordinal = ctx->op_device_context()->stream()->parent()->device_ordinal(); } - options.device_type = cache->device_type(); + options.device_type = cache.device_type(); options.flib_def = ctx->function_library()->GetFunctionLibraryDefinition(); options.graph_def_version = ctx->function_library()->graph_def_version(); options.allow_cpu_custom_calls = diff --git a/tensorflow/compiler/jit/xla_platform_info.h b/tensorflow/compiler/jit/xla_platform_info.h index dac45529ac9..d58b32a996f 100644 --- a/tensorflow/compiler/jit/xla_platform_info.h +++ b/tensorflow/compiler/jit/xla_platform_info.h @@ -99,7 +99,7 @@ se::DeviceMemoryAllocator* GetAllocator( // Returns created options for the XLA compiler, and writes the used allocator // into `tf_allocator_adapter`. XlaCompiler::Options GenerateCompilerOptions( - XlaCompilationCache* cache, OpKernelContext* ctx, + const XlaCompilationCache& cache, OpKernelContext* ctx, const XlaPlatformInfo& platform_info, bool has_ref_vars, absl::optional* tf_allocator_adapter); diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD index 924834fc0fc..7f099540f39 100644 --- a/tensorflow/compiler/tests/BUILD +++ b/tensorflow/compiler/tests/BUILD @@ -1687,6 +1687,7 @@ tf_cuda_cc_test( deps = [ "//tensorflow/cc:cc_ops", "//tensorflow/compiler/jit", + "//tensorflow/compiler/jit:flags", "//tensorflow/compiler/jit:xla_kernel_creator", "//tensorflow/compiler/tf2xla:xla_compiler", "//tensorflow/core:core_cpu", diff --git a/tensorflow/compiler/tests/unary_ops_composition_test.cc b/tensorflow/compiler/tests/unary_ops_composition_test.cc index 569261de094..0e40c497c24 100644 --- a/tensorflow/compiler/tests/unary_ops_composition_test.cc +++ b/tensorflow/compiler/tests/unary_ops_composition_test.cc @@ -20,6 +20,7 @@ limitations under the License. #include #include "absl/synchronization/notification.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/core/common_runtime/device.h" #include "tensorflow/core/common_runtime/device_factory.h" @@ -43,6 +44,11 @@ limitations under the License. namespace tensorflow { namespace { +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + class UnaryOpsCompositionTest : public OpsTestBase { protected: template diff --git a/tensorflow/compiler/tests/xla_device_gpu_test.py b/tensorflow/compiler/tests/xla_device_gpu_test.py index 1e30ebd55d0..304405c82ce 100644 --- a/tensorflow/compiler/tests/xla_device_gpu_test.py +++ b/tensorflow/compiler/tests/xla_device_gpu_test.py @@ -19,6 +19,7 @@ from __future__ import division from __future__ import print_function from tensorflow.python.client import session as session_lib +from tensorflow.python.eager import context from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops @@ -27,6 +28,10 @@ from tensorflow.python.platform import test class XlaDeviceGpuTest(test.TestCase): + def __init__(self, method_name="runTest"): + super(XlaDeviceGpuTest, self).__init__(method_name) + context.context().enable_xla_devices() + def testCopiesToAndFromGpuWork(self): """Tests that copies between GPU and XLA devices work.""" if not test.is_gpu_available(): diff --git a/tensorflow/compiler/tests/xla_test.py b/tensorflow/compiler/tests/xla_test.py index 3b057ed8b17..8c31629c234 100644 --- a/tensorflow/compiler/tests/xla_test.py +++ b/tensorflow/compiler/tests/xla_test.py @@ -83,6 +83,8 @@ class XLATestCase(test.TestCase): def __init__(self, method_name='runTest'): super(XLATestCase, self).__init__(method_name) + if 'XLA' in FLAGS.test_device: + context.context().enable_xla_devices() context.context().enable_mlir_bridge = test_util.is_mlir_bridge_enabled() self.device = FLAGS.test_device diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD index 1e57c11b2cf..ac999d875de 100644 --- a/tensorflow/compiler/tf2xla/BUILD +++ b/tensorflow/compiler/tf2xla/BUILD @@ -787,6 +787,7 @@ tf_cc_test( "//tensorflow/cc:function_ops", "//tensorflow/cc:functional_ops", "//tensorflow/cc:ops", + "//tensorflow/compiler/jit:flags", "//tensorflow/compiler/jit:xla_cluster_util", "//tensorflow/compiler/tf2xla/kernels:xla_ops", "//tensorflow/core:core_cpu_internal", @@ -1087,6 +1088,7 @@ tf_cuda_cc_test( "//tensorflow/cc:ops", "//tensorflow/cc:scope", "//tensorflow/compiler/jit", + "//tensorflow/compiler/jit:flags", "//tensorflow/core:core_cpu", "//tensorflow/core:framework", "//tensorflow/core:framework_internal", diff --git a/tensorflow/compiler/tf2xla/const_analysis_test.cc b/tensorflow/compiler/tf2xla/const_analysis_test.cc index 936b74f7b33..c7c8702b49b 100644 --- a/tensorflow/compiler/tf2xla/const_analysis_test.cc +++ b/tensorflow/compiler/tf2xla/const_analysis_test.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/cc/ops/function_ops.h" #include "tensorflow/cc/ops/functional_ops.h" #include "tensorflow/cc/ops/standard_ops.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/compiler/jit/xla_cluster_util.h" #include "tensorflow/core/common_runtime/process_function_library_runtime.h" #include "tensorflow/core/graph/algorithm.h" @@ -217,5 +218,10 @@ TEST(ConstAnalysisTest, RespectExplicitAttr_1) { EXPECT_EQ(const_args, std::vector({true})); } +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc b/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc index 1a26f974989..02f178f9acf 100644 --- a/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc +++ b/tensorflow/compiler/tf2xla/fused_batchnorm_reserve_space_test.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/cc/ops/array_ops.h" #include "tensorflow/cc/ops/const_op.h" #include "tensorflow/cc/ops/nn_ops.h" +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/framework/device_attributes.pb.h" #include "tensorflow/core/framework/graph.pb.h" #include "tensorflow/core/framework/tensor.h" @@ -139,5 +140,11 @@ TEST(FusedBatchnormReserveSpaceTest, Test) { test::ExpectClose(results[0], results[1], /*atol=*/1e-4); test::ExpectClose(results[2], results[3], /*atol=*/1e-4); } + +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/xrt/BUILD b/tensorflow/compiler/xrt/BUILD index 6a704be4adb..172a970d207 100644 --- a/tensorflow/compiler/xrt/BUILD +++ b/tensorflow/compiler/xrt/BUILD @@ -96,6 +96,7 @@ tf_gen_op_libs( "xrt_execute_op", ], deps = [ + "//tensorflow/compiler/jit:flags", "//tensorflow/core:lib", ], ) diff --git a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc index a4be39b96c6..321d7409103 100644 --- a/tensorflow/compiler/xrt/ops/xrt_state_ops.cc +++ b/tensorflow/compiler/xrt/ops/xrt_state_ops.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/framework/common_shape_fns.h" #include "tensorflow/core/framework/op.h" #include "tensorflow/core/framework/shape_inference.h" @@ -20,6 +21,11 @@ limitations under the License. namespace tensorflow { +static bool Initialized = [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + return true; +}(); + REGISTER_OP("XRTAllocate") .Input("allocation: string") .Output("handle: int64") diff --git a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc index 641b5b4ef31..4fe8e6a6c3f 100644 --- a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc @@ -44,26 +44,6 @@ TEST_F(PinToHostOptimizerTest, TryFindHostDeviceCpuXlaGpu) { "/device:CPU:0"); } -TEST_F(PinToHostOptimizerTest, TryFindHostDeviceXlaCpuXlaGpu) { - gtl::FlatSet devices = {"/device:XLA_CPU:0", "/device:XLA_GPU:0"}; - - EXPECT_EQ(internal::TryFindHostDevice(devices, false, ""), ""); - EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:0"), - "/device:XLA_CPU:0"); - EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:*"), - "/device:XLA_CPU:0"); -} - -TEST_F(PinToHostOptimizerTest, TryFindHostDeviceXlaGpu) { - gtl::FlatSet devices = {"/device:XLA_GPU:0"}; - - EXPECT_EQ(internal::TryFindHostDevice(devices, false, ""), ""); - EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:0"), - ""); - EXPECT_EQ(internal::TryFindHostDevice(devices, false, "/device:XLA_GPU:*"), - ""); -} - TEST_F(PinToHostOptimizerTest, OptimizeSmallOpsToHost) { tensorflow::Scope s = tensorflow::Scope::NewRootScope(); Output a = ops::Const(s.WithOpName("a"), 1, {1024, 1024}); diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py index 765c77af7cd..9c939fe0a76 100644 --- a/tensorflow/python/eager/context.py +++ b/tensorflow/python/eager/context.py @@ -42,6 +42,7 @@ from tensorflow.python.framework import device as pydev from tensorflow.python.util import compat from tensorflow.python.util import is_in_graph_mode from tensorflow.python.util import tf_contextlib +from tensorflow.python.util.deprecation import deprecated from tensorflow.python.util.tf_export import tf_export GRAPH_MODE = 0 @@ -1254,12 +1255,7 @@ class Context(object): p: i for i, p in enumerate(self._physical_devices) } - # Construct the visible device list from all physical devices but ignore - # XLA devices - self._visible_device_list = [ - d for d in self._physical_devices - if not d.device_type.startswith("XLA") - ] + self._visible_device_list = list(self._physical_devices) self._memory_growth_map = { d: None for d in self._physical_devices if d.device_type == "GPU" } @@ -1493,6 +1489,12 @@ class Context(object): self._virtual_device_map[dev] = virtual_devices + @deprecated( + None, "XLA:CPU and XLA:GPU devices are deprecated", warn_once=True) + def enable_xla_devices(self): + """Enables XLA:CPU and XLA:GPU devices registration.""" + pywrap_tfe.TF_EnableXlaDevices() + @property def enable_mlir_bridge(self): return pywrap_tfe.TF_IsMlirBridgeEnabled() diff --git a/tensorflow/python/framework/config_test.py b/tensorflow/python/framework/config_test.py index 70857ef4b83..ee7e111f6b0 100644 --- a/tensorflow/python/framework/config_test.py +++ b/tensorflow/python/framework/config_test.py @@ -435,9 +435,6 @@ class DeviceTest(test.TestCase): self.assertEqual(len(config.get_visible_devices('CPU')), 1) self.assertGreater(len(config.get_visible_devices('GPU')), 0) - # get_visible_devices filters out XLA_* devices. list_logical_devices does - # not, but we can't call it here because it initializes the devices and - # calling set_visible_devices after that is disallowed. self.assertEqual(len(config.get_visible_devices('XLA_GPU')), 0) config.set_visible_devices(cpus[0]) @@ -451,12 +448,6 @@ class DeviceTest(test.TestCase): a = array_ops.identity(1.0) self.evaluate(a) - with self.assertRaisesRegex(errors.InvalidArgumentError, - 'Could not satisfy'): - with ops.device('/device:XLA_GPU:0'): - a = array_ops.identity(1.0) - self.evaluate(a) - # Modifying the visible devices is not supported with self.assertRaisesRegex(RuntimeError, 'cannot be modified'): config.set_visible_devices(gpus) diff --git a/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py b/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py index 33f0d7b76ae..188df3f9b87 100644 --- a/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py +++ b/tensorflow/python/ops/parallel_for/xla_control_flow_ops_test.py @@ -22,6 +22,7 @@ from __future__ import print_function from tensorflow.compiler.tf2xla.python import xla as xla_ops from tensorflow.python.compiler.xla import jit from tensorflow.python.compiler.xla import xla +from tensorflow.python.eager import context from tensorflow.python.eager import def_function from tensorflow.python.framework import constant_op from tensorflow.python.framework import test_util @@ -39,6 +40,10 @@ from tensorflow.python.platform import test @test_util.run_all_in_graph_and_eager_modes class PForTest(PForTestCase): + def __init__(self, method_name="runTest"): + super(PForTest, self).__init__(method_name) + context.context().enable_xla_devices() + def test_xla_einsum(self): num_loop = 10 x_series = random_ops.random_uniform([num_loop, 9, 9]) diff --git a/tensorflow/python/tfe_wrapper.cc b/tensorflow/python/tfe_wrapper.cc index c66397036c0..0afd05e94cb 100644 --- a/tensorflow/python/tfe_wrapper.cc +++ b/tensorflow/python/tfe_wrapper.cc @@ -444,6 +444,9 @@ PYBIND11_MODULE(_pywrap_tfe, m) { m.def("TF_EnableMlirBridge", [](bool enabled) { tensorflow::GetMlirCommonFlags()->tf_mlir_enable_mlir_bridge = enabled; }); + m.def("TF_EnableXlaDevices", [] { + tensorflow::GetXlaDeviceFlags()->tf_xla_enable_xla_devices = true; + }); // // TFE_Context Logic m.def( From d6076ef731d0c9ff731736a0fd92f0c6fb8a0cc6 Mon Sep 17 00:00:00 2001 From: Tomer Kaftan Date: Tue, 11 Aug 2020 13:07:05 -0700 Subject: [PATCH 0839/1017] Add release note specifying gradients cannot be taken w.r.t KerasTensors PiperOrigin-RevId: 326085518 Change-Id: I4cc22d2cfb369bf3d14b508bb4e3c39a20b2655b --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 241a5077251..2f1a890e60b 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -22,6 +22,7 @@ * Code that uses `tf.map_fn`/`tf.cond`/`tf.while_loop`/control flow as op layers and happens to work before TF 2.4. These will explicitly be unsupported now. Converting these ops to Functional API op layers was unreliable before TF 2.4, and prone to erroring incomprehensibly or being silently buggy. * Code that directly asserts on a Keras symbolic value in cases where ops like `tf.rank` used to return a static or symbolic value depending on if the input had a fully static shape or not. Now these ops always return symbolic values. * Code already susceptible to leaking tensors outside of graphs becomes slightly more likely to do so now. + * Code that tries directly getting gradients with respect to symbolic Keras inputs/outputs. Use GradientTape on the actual Tensors passed to the already-constructed model instead. * Code that requires very tricky shape manipulation via converted op layers in order to work, where the Keras symbolic shape inference proves insufficient. * Code that tries manually walking a `tf.keras.Model` layer by layer and assumes layers only ever have one positional argument. This assumption doesn't hold true before TF 2.4 either, but is more likely to cause issues know. * Code that manually enters `keras.backend.get_graph()` before building a functional model. This is no longer needed. From 2434678ef7f2130c13a48678b8475e821cac688b Mon Sep 17 00:00:00 2001 From: Rahul Joshi Date: Tue, 11 Aug 2020 13:11:19 -0700 Subject: [PATCH 0840/1017] [MLIR] Add MergeSummary to TF generated ops. - This will allow the op to be correctly inferred as having no side effects. PiperOrigin-RevId: 326086345 Change-Id: I4a01662b35f170a5baa0b789d8701f72b9f0ed2b --- .../mlir/tensorflow/ir/tf_generated_ops.td | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index bf8d7015b46..0c227ee49cc 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -5818,6 +5818,30 @@ def TF_MaximumOp : TF_Op<"Maximum", [NoSideEffect, ResultsBroadcastableShape, TF TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<0>; } +def TF_MergeSummaryOp : TF_Op<"MergeSummary", [NoSideEffect, SameOperandsAndResultType]> { + let summary = "Merges summaries."; + + let description = [{ +This op creates a +[`Summary`](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto) +protocol buffer that contains the union of all the values in the input +summaries. + +When the Op is run, it reports an `InvalidArgument` error if multiple values +in the summaries to merge use the same tag. + }]; + + let arguments = (ins + Variadic:$inputs + ); + + let results = (outs + TF_StrTensor:$summary + ); + + TF_DerivedOperandSizeAttr N = TF_DerivedOperandSizeAttr<0>; +} + def TF_MergeV2CheckpointsOp : TF_Op<"MergeV2Checkpoints", []> { let summary = [{ V2 format specific: merges the metadata files of sharded checkpoints. The From a5d7f8181004d585838a1e047c397e2e7ad99f0e Mon Sep 17 00:00:00 2001 From: Pankaj Kanwar Date: Tue, 11 Aug 2020 13:17:40 -0700 Subject: [PATCH 0841/1017] Change test file for cuda11 for pip build. PiperOrigin-RevId: 326087677 Change-Id: I40cd3e172128431f881912565d20c68cd9665167 --- .../ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh index 71d6f3e6401..d884a484167 100644 --- a/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh +++ b/tensorflow/tools/ci_build/rel/ubuntu_cuda11/gpu_py37_pip.sh @@ -36,13 +36,23 @@ source tensorflow/tools/ci_build/build_scripts/DEFAULT_TEST_TARGETS.sh # Export optional variables for running pip.sh export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-no_oss,-oss_serial,-no_oss_py37' -export TF_BUILD_FLAGS="--config=release_gpu_linux " +# TODO (pkanwar): Revert this CL (cl/326069644) once the cuda 11 migration is complete. +export TF_BUILD_FLAGS="--config=release_common " export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ --distinct_host_configuration=false \ --action_env=TF_CUDA_VERSION=11 --action_env=TF_CUDNN_VERSION=8 --test_env=TF2_BEHAVIOR=1 \ --config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ --verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ ---run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ +--config=cuda \ +--config=tensorrt \ +--action_env=CUDA_TOOLKIT_PATH=/usr/local/cuda-11.0 --action_env=TF_NEED_TENSORRT=1 \ +--action_env=TF_CUDA_COMPUTE_CAPABILITIES=sm_35,sm_37,sm_52,sm_60,sm_61,compute_70 \ +--action_env=TENSORRT_INSTALL_PATH=/usr/local/tensorrt \ +--action_env=LD_LIBRARY_PATH=/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:/usr/local/tensorrt/lib \ +--action_env=GCC_HOST_COMPILER_PATH=/usr/bin/gcc-5 \ +--config=avx_linux \ +--crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda11:toolchain" export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" #export IS_NIGHTLY=0 # Not nightly; uncomment if building from tf repo. From deeeffefe317a0728b70262b9177712cadd77df1 Mon Sep 17 00:00:00 2001 From: Jin Young Sohn Date: Tue, 11 Aug 2020 13:30:40 -0700 Subject: [PATCH 0842/1017] Add constants defined headers to the .cc files as builds currently fail in debug mode. PiperOrigin-RevId: 326090231 Change-Id: I3412ce95706aca8a9b5643c31b5ab398a60e4700 --- tensorflow/core/kernels/data/experimental/io_ops.cc | 5 +++++ .../data/experimental/snapshot_dataset_op.cc | 13 ++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/kernels/data/experimental/io_ops.cc b/tensorflow/core/kernels/data/experimental/io_ops.cc index 112a58a7e9a..903088ff481 100644 --- a/tensorflow/core/kernels/data/experimental/io_ops.cc +++ b/tensorflow/core/kernels/data/experimental/io_ops.cc @@ -28,6 +28,11 @@ namespace tensorflow { namespace data { namespace experimental { +/* static */ constexpr const int SaveDatasetOp::kFileFormatVersion; +/* static */ constexpr const char* const LoadDatasetOp::kCompression; +/* static */ constexpr const char* const LoadDatasetOp::kReaderFunc; +/* static */ constexpr const char* const LoadDatasetOp::kReaderFuncTarguments; + SaveDatasetOp::SaveDatasetOp(OpKernelConstruction* ctx) : HybridAsyncOpKernel(ctx, "tf_data_save_dataset") { OP_REQUIRES_OK(ctx, ctx->GetAttr(kCompression, &compression_)); diff --git a/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc b/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc index ec6cf02e02e..8b1c5fb7453 100644 --- a/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/snapshot_dataset_op.cc @@ -63,6 +63,15 @@ namespace tensorflow { namespace data { namespace experimental { +/* static */ constexpr const char* const SnapshotDatasetV2Op::kCompression; +/* static */ constexpr const char* const SnapshotDatasetV2Op::kReaderFunc; +/* static */ constexpr const char* const SnapshotDatasetV2Op::kShardFunc; +/* static */ constexpr const char* const + SnapshotDatasetV2Op::kReaderFuncTarguments; +/* static */ constexpr const char* const + SnapshotDatasetV2Op::kShardFuncTarguments; +/* static */ constexpr const int SnapshotDatasetV2Op::kFileFormatVersion; + // ==== Snapshot Implementation ==== /* The current snapshot on-disk layout is as follows: @@ -1122,9 +1131,7 @@ class SnapshotDatasetOp : public UnaryDatasetOpKernel { // Initialize at first and at that point we don't know which iterator // (Reader / Writer / Passthrough) we need to restore as this info is part // of the checkpoint. - Status Initialize(IteratorContext* ctx) override { - return Status::OK(); - } + Status Initialize(IteratorContext* ctx) override { return Status::OK(); } Status GetNextInternal(IteratorContext* ctx, std::vector* out_tensors, From 6245590976ba310e3cc4f7bbe83114ccfcd52f09 Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Tue, 11 Aug 2020 13:44:34 -0700 Subject: [PATCH 0843/1017] Fix priting of input/output must/may aliasing. PiperOrigin-RevId: 326092933 Change-Id: Ic17b4b6922846c2e0acf6e7d64cd7fe4627b7a3a --- .../compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc | 2 +- tensorflow/compiler/xla/service/hlo_input_output_alias_config.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc index 8a07aab11e1..80e2c1132fd 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util_test.cc @@ -524,7 +524,7 @@ TEST(CompileGraphToXlaHlo, Resources) { ASSERT_TRUE(status_or_hlo_module.ok()); constexpr char expected_hlo_module_string[] = - R"(HloModule main.4, input_output_alias={ {0}: (1, {}, may_alias) } + R"(HloModule main.4, input_output_alias={ {0}: (1, {}, may-alias) } ENTRY %main.4 (Arg_0.1: f32[2], Arg_1.2: f32[2]) -> (f32[2]) { %Arg_1.2 = f32[2]{0} parameter(1) diff --git a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h index 6b84bdb6a68..d5630467783 100644 --- a/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h +++ b/tensorflow/compiler/xla/service/hlo_input_output_alias_config.h @@ -57,7 +57,7 @@ class HloInputOutputAliasConfig { std::string ToString() { return absl::StrFormat("(%lld, %s, %s)", parameter_number, parameter_index.ToString(), - kind == kMustAlias ? "must_alias" : "may_alias"); + kind == kMustAlias ? "must-alias" : "may-alias"); } }; From 6337a6498a062dab136dfbd4fff4f907dabedc05 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Tue, 11 Aug 2020 13:50:37 -0700 Subject: [PATCH 0844/1017] Internal change PiperOrigin-RevId: 326094154 Change-Id: I841055526472333219f42c62fca33fb4c5cb5217 --- tensorflow/python/keras/engine/base_layer_v1.py | 2 +- tensorflow/python/keras/engine/training_v1.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/keras/engine/base_layer_v1.py b/tensorflow/python/keras/engine/base_layer_v1.py index 85d390f2360..f047d84d16a 100644 --- a/tensorflow/python/keras/engine/base_layer_v1.py +++ b/tensorflow/python/keras/engine/base_layer_v1.py @@ -153,7 +153,7 @@ class Layer(base_layer.Layer): @trackable.no_automatic_dependency_tracking def __init__(self, trainable=True, name=None, dtype=None, dynamic=False, **kwargs): - base_layer.keras_api_gauge.get_cell('layer v1').set(True) + base_layer.keras_api_gauge.get_cell('layer').set(True) base_layer.keras_layers_gauge.get_cell(self.__class__.__name__).set(True) # These properties should be set by the user via keyword arguments. # note that 'dtype', 'input_shape' and 'batch_input_shape' diff --git a/tensorflow/python/keras/engine/training_v1.py b/tensorflow/python/keras/engine/training_v1.py index 2ac3337948a..2c38de92666 100644 --- a/tensorflow/python/keras/engine/training_v1.py +++ b/tensorflow/python/keras/engine/training_v1.py @@ -138,7 +138,6 @@ class Model(training_lib.Model): def __init__(self, *args, **kwargs): super(Model, self).__init__(*args, **kwargs) - base_layer.keras_api_gauge.get_cell('model v1').set(True) # initializing _distribution_strategy here since it is possible to call # predict on a model without compiling it. self._distribution_strategy = None @@ -409,7 +408,7 @@ class Model(training_lib.Model): # time the model gets called on training data. return self._is_compiled = True - base_layer.keras_api_gauge.get_cell('compile_v1').set(True) + base_layer.keras_api_gauge.get_cell('compile').set(True) # Prepare list of loss functions, same size of model outputs. self.loss_functions = training_utils.prepare_loss_functions( @@ -770,7 +769,7 @@ class Model(training_lib.Model): and what the model expects. """ self._assert_built_as_v1() - base_layer.keras_api_gauge.get_cell('fit_v1').set(True) + base_layer.keras_api_gauge.get_cell('fit').set(True) # Legacy support if 'nb_epoch' in kwargs: logging.warning( @@ -891,7 +890,7 @@ class Model(training_lib.Model): ValueError: in case of invalid arguments. """ self._assert_built_as_v1() - base_layer.keras_api_gauge.get_cell('evaluate_v1').set(True) + base_layer.keras_api_gauge.get_cell('evaluate').set(True) self._assert_compile_was_called() self._check_call_args('evaluate') @@ -971,7 +970,7 @@ class Model(training_lib.Model): that is not a multiple of the batch size. """ self._assert_built_as_v1() - base_layer.keras_api_gauge.get_cell('predict_v1').set(True) + base_layer.keras_api_gauge.get_cell('predict').set(True) self._check_call_args('predict') func = self._select_training_loop(x) From c717e051fc04fd3adf8c63b2f3a6ad4e717e0624 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Tue, 11 Aug 2020 13:51:17 -0700 Subject: [PATCH 0845/1017] Update mobilenet v1 and v2 to align with the original implementation. Remove the first ZeroPadding2D layer and update the conv2D with padding="same". This is align with original model garden in https://github.com/tensorflow/models/blob/692215511a27c49dadabd4fac1d83aef25bc840f/research/slim/nets/mobilenet/mobilenet.py#L155, where use_explicit_padding = False. PiperOrigin-RevId: 326094277 Change-Id: I063af2d80c5c6905b9b25a8ad8c78ef3986d8fca --- tensorflow/python/keras/applications/mobilenet.py | 6 ++---- tensorflow/python/keras/applications/mobilenet_v2.py | 8 ++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tensorflow/python/keras/applications/mobilenet.py b/tensorflow/python/keras/applications/mobilenet.py index c59246fb8ef..f2949170077 100644 --- a/tensorflow/python/keras/applications/mobilenet.py +++ b/tensorflow/python/keras/applications/mobilenet.py @@ -349,15 +349,13 @@ def _conv_block(inputs, filters, alpha, kernel=(3, 3), strides=(1, 1)): """ channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 filters = int(filters * alpha) - x = layers.ZeroPadding2D(padding=((0, 1), (0, 1)), name='conv1_pad')(inputs) x = layers.Conv2D( filters, kernel, - padding='valid', + padding='same', use_bias=False, strides=strides, - name='conv1')( - x) + name='conv1')(inputs) x = layers.BatchNormalization(axis=channel_axis, name='conv1_bn')(x) return layers.ReLU(6., name='conv1_relu')(x) diff --git a/tensorflow/python/keras/applications/mobilenet_v2.py b/tensorflow/python/keras/applications/mobilenet_v2.py index 1891c3ff19f..77392d487bd 100644 --- a/tensorflow/python/keras/applications/mobilenet_v2.py +++ b/tensorflow/python/keras/applications/mobilenet_v2.py @@ -298,17 +298,13 @@ def MobileNetV2(input_shape=None, channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 first_block_filters = _make_divisible(32 * alpha, 8) - x = layers.ZeroPadding2D( - padding=imagenet_utils.correct_pad(img_input, 3), - name='Conv1_pad')(img_input) x = layers.Conv2D( first_block_filters, kernel_size=3, strides=(2, 2), - padding='valid', + padding='same', use_bias=False, - name='Conv1')( - x) + name='Conv1')(img_input) x = layers.BatchNormalization( axis=channel_axis, epsilon=1e-3, momentum=0.999, name='bn_Conv1')( x) From 45e89479c96e63e53f96cf27730e2be256bf1d9b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 14:09:23 -0700 Subject: [PATCH 0846/1017] Make the semantics of repository_root more consistent. PiperOrigin-RevId: 326098252 Change-Id: I036f3943b1e35aa0543906128e0d78f59ccf1735 --- .../convert/xplane_to_profile_response.cc | 4 +- .../core/profiler/profiler_analysis.proto | 3 +- .../core/profiler/profiler_service.proto | 3 +- tensorflow/core/profiler/rpc/client/BUILD | 2 + .../profiler/rpc/client/capture_profile.cc | 89 ++++++++++++------- .../profiler/rpc/client/capture_profile.h | 8 +- .../core/profiler/rpc/client/save_profile.cc | 86 ++++++++---------- .../core/profiler/rpc/client/save_profile.h | 25 +++--- tensorflow/python/profiler/internal/BUILD | 1 - .../profiler/internal/profiler_wrapper.cc | 20 +---- 10 files changed, 114 insertions(+), 127 deletions(-) diff --git a/tensorflow/core/profiler/convert/xplane_to_profile_response.cc b/tensorflow/core/profiler/convert/xplane_to_profile_response.cc index 54e9a8b2a10..b7e28b23758 100644 --- a/tensorflow/core/profiler/convert/xplane_to_profile_response.cc +++ b/tensorflow/core/profiler/convert/xplane_to_profile_response.cc @@ -81,7 +81,7 @@ Status ConvertXSpaceToProfileResponse(const XSpace& xspace, response->set_empty_trace(true); return Status::OK(); } - TF_RETURN_IF_ERROR(SaveGzippedToolDataToTensorboardProfile( + TF_RETURN_IF_ERROR(SaveGzippedToolData( req.repository_root(), req.session_id(), req.host_name(), ToolName(kTraceViewer), TraceEventsToJson(trace))); // Trace viewer is the only tool, skip OpStats conversion. @@ -110,7 +110,7 @@ Status ConvertXSpaceToProfileResponse(const XSpace& xspace, if (tools.contains(kMemoryProfile)) { std::string json_output; TF_RETURN_IF_ERROR(ConvertXSpaceToMemoryProfileJson(xspace, &json_output)); - TF_RETURN_IF_ERROR(SaveGzippedToolDataToTensorboardProfile( + TF_RETURN_IF_ERROR(SaveGzippedToolData( req.repository_root(), req.session_id(), req.host_name(), ToolName(kMemoryProfile), json_output)); } diff --git a/tensorflow/core/profiler/profiler_analysis.proto b/tensorflow/core/profiler/profiler_analysis.proto index 8f7bd01920a..5f2c5aa86fd 100644 --- a/tensorflow/core/profiler/profiler_analysis.proto +++ b/tensorflow/core/profiler/profiler_analysis.proto @@ -7,8 +7,7 @@ import "tensorflow/core/profiler/profiler_service.proto"; message NewProfileSessionRequest { ProfileRequest request = 1; // The place where we will dump profile data. We will normally use - // MODEL_DIR as the repository root. The data will be saved under - // MODEL_DIR/plugins/profile/. + // MODEL_DIR/plugins/profile as the repository root. string repository_root = 2; repeated string hosts = 3; string session_id = 4; diff --git a/tensorflow/core/profiler/profiler_service.proto b/tensorflow/core/profiler/profiler_service.proto index 649a5be56bf..69ccf04e304 100644 --- a/tensorflow/core/profiler/profiler_service.proto +++ b/tensorflow/core/profiler/profiler_service.proto @@ -48,8 +48,7 @@ message ProfileRequest { ProfileOptions opts = 4; // The place where we will dump profile data. We will normally use - // MODEL_DIR as the repository root. The data will be saved under - // MODEL_DIR/plugins/profile/. + // MODEL_DIR/plugins/profile/ as the repository root. string repository_root = 5; // The user provided profile session identifier. diff --git a/tensorflow/core/profiler/rpc/client/BUILD b/tensorflow/core/profiler/rpc/client/BUILD index 68b382eecee..72820ee4d6c 100644 --- a/tensorflow/core/profiler/rpc/client/BUILD +++ b/tensorflow/core/profiler/rpc/client/BUILD @@ -26,6 +26,8 @@ cc_library( "//tensorflow/core/profiler:profiler_options_proto_cc", "//tensorflow/core/profiler:profiler_service_proto_cc", "//tensorflow/core/profiler/convert:xplane_to_profile_response", + "//tensorflow/core/profiler/protobuf:xplane_proto_cc", + "//tensorflow/core:lib_internal", ] + tf_profiler_client_deps(), ) diff --git a/tensorflow/core/profiler/rpc/client/capture_profile.cc b/tensorflow/core/profiler/rpc/client/capture_profile.cc index bd82ba64db2..d7707aff5c2 100644 --- a/tensorflow/core/profiler/rpc/client/capture_profile.cc +++ b/tensorflow/core/profiler/rpc/client/capture_profile.cc @@ -22,6 +22,7 @@ limitations under the License. #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" #include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/host_info.h" #include "tensorflow/core/platform/status.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/profiler/convert/xplane_to_profile_response.h" @@ -47,6 +48,30 @@ MonitorRequest PopulateMonitorRequest(int duration_ms, int monitoring_level, return request; } +ProfileRequest PopulateProfileRequest(int duration_ms, + const std::string& repository_root, + const std::string& session_id, + const std::string& host_name, + const ProfileOptions& opts) { + ProfileRequest request; + request.set_duration_ms(duration_ms); + request.set_max_events(kMaxEvents); + request.set_repository_root(repository_root); + request.set_session_id(session_id); + request.set_host_name(host_name); + request.add_tools("trace_viewer"); + request.add_tools("op_profile"); + request.add_tools("input_pipeline"); + request.add_tools("kernel_stats"); + request.add_tools("memory_viewer"); + request.add_tools("memory_profile"); + request.add_tools("overview_page"); + request.add_tools("pod_viewer"); + request.add_tools("tensorflow_stats"); + *request.mutable_opts() = opts; + return request; +} + NewProfileSessionRequest PopulateNewProfileSessionRequest( const std::string& service_addr, const std::string& repository_root, const std::vector& hostnames, int duration_ms, @@ -87,20 +112,20 @@ Status ConvertXSpaceToToolsInProfileResponse(const ProfileRequest& request, return Status::OK(); } -Status Profile(const std::string& service_addr, const std::string& logdir, - int duration_ms, const std::string& session_id, - const ProfileOptions& opts) { +Status Profile(const std::string& service_addr, + const std::string& repository_root, int duration_ms, + const std::string& session_id, const ProfileOptions& opts) { std::vector parts = absl::StrSplit(service_addr, ':'); - ProfileRequest request = - PopulateProfileRequest(duration_ms, logdir, session_id, parts[0], opts); + ProfileRequest request = PopulateProfileRequest(duration_ms, repository_root, + session_id, parts[0], opts); ProfileResponse response; TF_RETURN_IF_ERROR(ProfileGrpc(service_addr, request, &response)); if (!response.empty_trace()) { TF_RETURN_IF_ERROR( ConvertXSpaceToToolsInProfileResponse(request, &response)); - TF_RETURN_IF_ERROR(SaveTensorboardProfile( - logdir, session_id, request.host_name(), response, &std::cout)); + TF_RETURN_IF_ERROR(SaveProfile(repository_root, session_id, + request.host_name(), response, &std::cout)); // Print this at the end so that it's not buried in irrelevant LOG messages. std::cout << "NOTE: using the trace duration " << duration_ms << "ms.\n" @@ -138,30 +163,6 @@ Status NewSession(const std::string& service_addr, } // namespace -ProfileRequest PopulateProfileRequest(int duration_ms, - const std::string& repository_root, - const std::string& session_id, - const std::string& host_name, - const ProfileOptions& opts) { - ProfileRequest request; - request.set_duration_ms(duration_ms); - request.set_max_events(kMaxEvents); - request.set_repository_root(repository_root); - request.set_session_id(session_id); - request.set_host_name(host_name); - request.add_tools("trace_viewer"); - request.add_tools("op_profile"); - request.add_tools("input_pipeline"); - request.add_tools("kernel_stats"); - request.add_tools("memory_viewer"); - request.add_tools("memory_profile"); - request.add_tools("overview_page"); - request.add_tools("pod_viewer"); - request.add_tools("tensorflow_stats"); - *request.mutable_opts() = opts; - return request; -} - // Starts tracing on a single or multiple hosts and saves the result in the // given logdir. If no trace was collected, retries tracing for // num_tracing_attempts. @@ -174,6 +175,9 @@ Status Trace(const std::string& service_addr, const std::string& logdir, if (!workers_list.empty()) { hostnames = absl::StrSplit(workers_list, ','); } + TF_RETURN_IF_ERROR(MaybeCreateEmptyEventFile(logdir)); + std::string repository_root = + profiler::GetTensorBoardProfilePluginDir(logdir); Status status = Status::OK(); int remaining_attempts = num_tracing_attempts; @@ -181,9 +185,10 @@ Status Trace(const std::string& service_addr, const std::string& logdir, std::cout << "Starting to trace for " << duration_ms << " ms. " << "Remaining attempt(s): " << --remaining_attempts << std::endl; if (hostnames.empty()) { - status = Profile(service_addr, logdir, duration_ms, session_id, opts); + status = + Profile(service_addr, repository_root, duration_ms, session_id, opts); } else { - status = NewSession(service_addr, logdir, hostnames, duration_ms, + status = NewSession(service_addr, repository_root, hostnames, duration_ms, session_id, opts); } if (remaining_attempts <= 0 || status.ok() || !ShouldRetryTracing(status)) @@ -213,5 +218,23 @@ Status Monitor(const std::string& service_addr, int duration_ms, return Status::OK(); } +Status ExportToTensorBoard(const XSpace& xspace, const std::string& logdir) { + TF_RETURN_IF_ERROR(MaybeCreateEmptyEventFile(logdir)); + + ProfileResponse response; + ProfileRequest request = PopulateProfileRequest( + /*duration_ms=*/0, GetTensorBoardProfilePluginDir(logdir), + GetCurrentTimeStampAsString(), port::Hostname(), /*opts=*/{}); + TF_RETURN_IF_ERROR( + ConvertXSpaceToProfileResponse(xspace, request, &response)); + + std::stringstream ss; // Record LOG messages. + TF_RETURN_IF_ERROR(SaveProfile(request.repository_root(), + request.session_id(), request.host_name(), + response, &ss)); + LOG(INFO) << ss.str(); + return Status::OK(); +} + } // namespace profiler } // namespace tensorflow diff --git a/tensorflow/core/profiler/rpc/client/capture_profile.h b/tensorflow/core/profiler/rpc/client/capture_profile.h index 5745f24cbfa..771f1fee722 100644 --- a/tensorflow/core/profiler/rpc/client/capture_profile.h +++ b/tensorflow/core/profiler/rpc/client/capture_profile.h @@ -22,15 +22,13 @@ limitations under the License. #include "tensorflow/core/platform/status.h" #include "tensorflow/core/profiler/profiler_options.pb.h" #include "tensorflow/core/profiler/profiler_service.pb.h" +#include "tensorflow/core/profiler/protobuf/xplane.pb.h" namespace tensorflow { namespace profiler { -ProfileRequest PopulateProfileRequest(int duration_ms, - const std::string& repository_root, - const std::string& session_id, - const std::string& host_name, - const ProfileOptions& opts); +// Convert XSpace to tool data and saves under /plugins/profile/. +Status ExportToTensorBoard(const XSpace& xspace, const std::string& logdir); // Collects one sample of monitoring profile and shows user-friendly metrics. // If timestamp flag is true, timestamp will be displayed in "%H:%M:%S" format. diff --git a/tensorflow/core/profiler/rpc/client/save_profile.cc b/tensorflow/core/profiler/rpc/client/save_profile.cc index 9cf2e291692..9c24c78a5d8 100644 --- a/tensorflow/core/profiler/rpc/client/save_profile.cc +++ b/tensorflow/core/profiler/rpc/client/save_profile.cc @@ -82,10 +82,8 @@ string ProfilerJoinPath(const T&... args) { constexpr char kProtoTraceFileName[] = "trace"; constexpr char kTfStatsHelperSuffix[] = "tf_stats_helper_result"; -Status DumpToolDataToLogDirectory(absl::string_view run_dir, - absl::string_view host, - const ProfileToolData& tool, - std::ostream* os) { +Status DumpToolData(absl::string_view run_dir, absl::string_view host, + const ProfileToolData& tool, std::ostream* os) { // Don't save the intermediate results for combining the per host tool data. if (absl::EndsWith(tool.name(), kTfStatsHelperSuffix)) return Status::OK(); string host_prefix = host.empty() ? "" : absl::StrCat(host, "."); @@ -99,23 +97,6 @@ Status DumpToolDataToLogDirectory(absl::string_view run_dir, return Status::OK(); } -// Creates an empty event file if not already exists, which indicates that we -// have a plugins/profile/ directory in the current logdir. -Status MaybeCreateEmptyEventFile(const string& logdir) { - // Suffix for an empty event file. it should be kept in sync with - // _EVENT_FILE_SUFFIX in tensorflow/python/eager/profiler.py. - constexpr char kProfileEmptySuffix[] = ".profile-empty"; - std::vector children; - TF_RETURN_IF_ERROR(Env::Default()->GetChildren(logdir, &children)); - for (const string& child : children) { - if (absl::EndsWith(child, kProfileEmptySuffix)) { - return Status::OK(); - } - } - EventsWriter event_writer(ProfilerJoinPath(logdir, "events")); - return event_writer.InitWithSuffix(kProfileEmptySuffix); -} - Status WriteGzippedDataToFile(const string& filepath, const string& data) { std::unique_ptr file; TF_RETURN_IF_ERROR(Env::Default()->NewWritableFile(filepath, &file)); @@ -129,20 +110,14 @@ Status WriteGzippedDataToFile(const string& filepath, const string& data) { return Status::OK(); } -Status GetOrCreateProfileRunDir(const string& logdir, const string& run, - string* profile_run_dir, std::ostream* os) { - // Dumps profile data to /plugins/profile//. - *profile_run_dir = - ProfilerJoinPath(GetTensorBoardProfilePluginDir(logdir), run); - *os << "Creating directory: " << *profile_run_dir; - TF_RETURN_IF_ERROR(Env::Default()->RecursivelyCreateDir(*profile_run_dir)); - - // Creates an empty event file so that TensorBoard plugin logic can find - // the logdir. - TF_RETURN_IF_ERROR(MaybeCreateEmptyEventFile(logdir)); +Status GetOrCreateRunDir(const string& repository_root, const string& run, + string* run_dir, std::ostream* os) { + // Dumps profile data to //. + *run_dir = ProfilerJoinPath(repository_root, run); + *os << "Creating directory: " << *run_dir; + TF_RETURN_IF_ERROR(Env::Default()->RecursivelyCreateDir(*run_dir)); return Status::OK(); } - } // namespace string GetTensorBoardProfilePluginDir(const string& logdir) { @@ -151,33 +126,42 @@ string GetTensorBoardProfilePluginDir(const string& logdir) { return ProfilerJoinPath(logdir, kPluginName, kProfileName); } -Status SaveTensorboardProfile(const string& logdir, const string& run, - const string& host, - const ProfileResponse& response, - std::ostream* os) { - string profile_run_dir; - TF_RETURN_IF_ERROR( - GetOrCreateProfileRunDir(logdir, run, &profile_run_dir, os)); +Status MaybeCreateEmptyEventFile(const string& logdir) { + // Suffix for an empty event file. it should be kept in sync with + // _EVENT_FILE_SUFFIX in tensorflow/python/eager/profiler.py. + constexpr char kProfileEmptySuffix[] = ".profile-empty"; + std::vector children; + TF_RETURN_IF_ERROR(Env::Default()->GetChildren(logdir, &children)); + for (const string& child : children) { + if (absl::EndsWith(child, kProfileEmptySuffix)) { + return Status::OK(); + } + } + EventsWriter event_writer(ProfilerJoinPath(logdir, "events")); + return event_writer.InitWithSuffix(kProfileEmptySuffix); +} + +Status SaveProfile(const string& repository_root, const string& run, + const string& host, const ProfileResponse& response, + std::ostream* os) { + string run_dir; + TF_RETURN_IF_ERROR(GetOrCreateRunDir(repository_root, run, &run_dir, os)); for (const auto& tool_data : response.tool_data()) { - TF_RETURN_IF_ERROR( - DumpToolDataToLogDirectory(profile_run_dir, host, tool_data, os)); + TF_RETURN_IF_ERROR(DumpToolData(run_dir, host, tool_data, os)); } return Status::OK(); } -Status SaveGzippedToolDataToTensorboardProfile(const string& logdir, - const string& run, - const string& host, - const string& tool_name, - const string& data) { - string profile_run_dir; +Status SaveGzippedToolData(const string& repository_root, const string& run, + const string& host, const string& tool_name, + const string& data) { + string run_dir; std::stringstream ss; - Status status = GetOrCreateProfileRunDir(logdir, run, &profile_run_dir, &ss); + Status status = GetOrCreateRunDir(repository_root, run, &run_dir, &ss); LOG(INFO) << ss.str(); TF_RETURN_IF_ERROR(status); string host_prefix = host.empty() ? "" : absl::StrCat(host, "."); - string path = - ProfilerJoinPath(profile_run_dir, absl::StrCat(host_prefix, tool_name)); + string path = ProfilerJoinPath(run_dir, absl::StrCat(host_prefix, tool_name)); TF_RETURN_IF_ERROR(WriteGzippedDataToFile(path, data)); LOG(INFO) << "Dumped gzipped tool data for " << tool_name << " to " << path; return Status::OK(); diff --git a/tensorflow/core/profiler/rpc/client/save_profile.h b/tensorflow/core/profiler/rpc/client/save_profile.h index 2e8fc96390a..c155502fb60 100644 --- a/tensorflow/core/profiler/rpc/client/save_profile.h +++ b/tensorflow/core/profiler/rpc/client/save_profile.h @@ -30,21 +30,22 @@ string GetCurrentTimeStampAsString(); // Returns the profile plugin directory given a logdir to TensorBoard. string GetTensorBoardProfilePluginDir(const string& logdir); -// Saves all profiling tool data in a profile to a TensorBoard log directory -// with the given run name. This writes user-facing log messages to `os`. +// Creates an empty event file if not already exists, which indicates that we +// have a plugins/profile/ directory in the current logdir. +Status MaybeCreateEmptyEventFile(const string& logdir); + +// Saves all profiling tool data in a profile to //. +// This writes user-facing log messages to `os`. // Note: this function creates a directory even when all fields in // ProfileResponse are unset/empty. -Status SaveTensorboardProfile(const string& logdir, const string& run, - const string& host, - const ProfileResponse& response, - std::ostream* os); +Status SaveProfile(const string& repository_root, const string& run, + const string& host, const ProfileResponse& response, + std::ostream* os); -// Gzip the data and save to the specified filepath. -Status SaveGzippedToolDataToTensorboardProfile(const string& logdir, - const string& run, - const string& host, - const string& tool_name, - const string& data); +// Gzip the data and save to //. +Status SaveGzippedToolData(const string& repository_root, const string& run, + const string& host, const string& tool_name, + const string& data); } // namespace profiler } // namespace tensorflow diff --git a/tensorflow/python/profiler/internal/BUILD b/tensorflow/python/profiler/internal/BUILD index eead8061e14..98e93367c8f 100644 --- a/tensorflow/python/profiler/internal/BUILD +++ b/tensorflow/python/profiler/internal/BUILD @@ -120,7 +120,6 @@ tf_python_pybind_extension( "//tensorflow/core:lib", "//tensorflow/core/profiler/convert:op_stats_to_tf_stats", "//tensorflow/core/profiler/convert:xplane_to_op_stats", - "//tensorflow/core/profiler/convert:xplane_to_profile_response", "//tensorflow/core/profiler/convert:xplane_to_trace_events", "//tensorflow/core/profiler/lib:profiler_session_headers", "//tensorflow/core/profiler/rpc:profiler_server_headers", diff --git a/tensorflow/python/profiler/internal/profiler_wrapper.cc b/tensorflow/python/profiler/internal/profiler_wrapper.cc index 0984a8b45c5..1ac294e720e 100644 --- a/tensorflow/python/profiler/internal/profiler_wrapper.cc +++ b/tensorflow/python/profiler/internal/profiler_wrapper.cc @@ -20,7 +20,6 @@ limitations under the License. #include "pybind11/pybind11.h" #include "pybind11/pytypes.h" #include "tensorflow/core/platform/errors.h" -#include "tensorflow/core/platform/host_info.h" #include "tensorflow/core/platform/status.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/profiler/convert/op_stats_to_input_pipeline_analysis.h" @@ -28,7 +27,6 @@ limitations under the License. #include "tensorflow/core/profiler/convert/op_stats_to_tf_stats.h" #include "tensorflow/core/profiler/convert/xplane_to_memory_profile.h" #include "tensorflow/core/profiler/convert/xplane_to_op_stats.h" -#include "tensorflow/core/profiler/convert/xplane_to_profile_response.h" #include "tensorflow/core/profiler/convert/xplane_to_trace_events.h" #include "tensorflow/core/profiler/lib/profiler_session.h" #include "tensorflow/core/profiler/protobuf/input_pipeline.pb.h" @@ -105,23 +103,7 @@ class ProfilerSessionWrapper { tensorflow::Status status; status = session_->CollectData(&xspace); session_.reset(); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - - tensorflow::ProfileResponse response; - tensorflow::ProfileRequest request = - tensorflow::profiler::PopulateProfileRequest( - /*duration_ms=*/0, logdir_, - tensorflow::profiler::GetCurrentTimeStampAsString(), - tensorflow::port::Hostname(), /*opts=*/{}); - status = tensorflow::profiler::ConvertXSpaceToProfileResponse( - xspace, request, &response); - tensorflow::MaybeRaiseRegisteredFromStatus(status); - - std::stringstream ss; // Record LOG messages. - status = tensorflow::profiler::SaveTensorboardProfile( - request.repository_root(), request.session_id(), request.host_name(), - response, &ss); - LOG(INFO) << ss.str(); + status = tensorflow::profiler::ExportToTensorBoard(xspace, logdir_); tensorflow::MaybeRaiseRegisteredFromStatus(status); } From 0cc126fee50ebb0d34a67c9e7848cf8fe1fd4d72 Mon Sep 17 00:00:00 2001 From: Rahul Joshi Date: Tue, 11 Aug 2020 14:39:39 -0700 Subject: [PATCH 0847/1017] [MLIR] Update SideEffectAnalysis to handle region based control flow - Introduce a TensorFlowDialect::CanHaveSideEffects() function to determine which ops can have side effects. Also move CanDuplicate into the TensorFlowDialect class. - Update OpIsKnownToHaveNoSideEffect() to rely on a TF dialect function to determine whether an op can have side effects. - Update the side effect analysis unit test to test region based control flow. - Since terminators in TF dialect are now inferred as not having side effects, several control dependencies that were added earlier are no longer added. Update the unit test to reflect the new reduced dependencies. PiperOrigin-RevId: 326104678 Change-Id: I9a1352c1ad4355145912e1af14763fd6aadc183e --- .../analysis/side_effect_analysis.cc | 47 +- .../compiler/mlir/tensorflow/ir/tf_ops.cc | 55 ++- .../compiler/mlir/tensorflow/ir/tf_ops.h | 6 + .../tests/side-effect-analysis-test.mlir | 459 +++++++++++++++++- 4 files changed, 505 insertions(+), 62 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.cc b/tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.cc index 9e78b90debc..e382bdb28c6 100644 --- a/tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.cc +++ b/tensorflow/compiler/mlir/tensorflow/analysis/side_effect_analysis.cc @@ -67,16 +67,12 @@ llvm::SmallDenseSet FindAccessedResources( Operation* op, const ResourceAliasAnalysis::Info& alias_analysis) { llvm::SmallDenseSet resources; - for (auto operand : op->getOperands()) { - if (!mlir::getElementTypeOrSelf(operand.getType()).isa()) - continue; + for (auto operand : filter_resources(op->getOperands())) { if (alias_analysis.IsUnknownResource(operand)) return UnknownResourceSet(); const auto& ids = alias_analysis.GetResourceUniqueIds(operand); resources.insert(ids.begin(), ids.end()); } - for (auto result : op->getResults()) { - if (!mlir::getElementTypeOrSelf(result.getType()).isa()) - continue; + for (auto result : filter_resources(op->getResults())) { if (alias_analysis.IsUnknownResource(result)) return UnknownResourceSet(); const auto& ids = alias_analysis.GetResourceUniqueIds(result); resources.insert(ids.begin(), ids.end()); @@ -117,34 +113,19 @@ bool OpIsDeclaration(Operation* op, // Returns if `op` is know to not have any side effect. bool OpIsKnownToHaveNoSideEffect(Operation* op) { - // TODO(riverriddle) We shouldn't treat all terminator operations as having - // side effects, this should be relaxed. - // TODO(riverriddle) Properly handle region side effects. - if (MemoryEffectOpInterface::hasNoEffect(op) && op->isKnownNonTerminator() && - op->getNumRegions() == 0) { - return true; - } - if (auto if_op = llvm::dyn_cast(op)) { - return if_op.is_stateless(); - } - if (auto while_op = llvm::dyn_cast(op)) { - return while_op.is_stateless(); - } + // Note: Identity op is really side-effect free, but it is not marked as such + // in the TF dialect (see comments in definition of Identity op in tf_ops.td) + // However, for adding control dependencies, its safe to assume + // that the Identity op is side-effect free. + if (isa(op)) return true; - // Try to get the statefulness flag from the registry. - // - // TODO(yuanzx): Remove this after all ops are defined in the dialect. - if (op->getName().getDialect() != - TF::TensorFlowDialect::getDialectNamespace()) { - return false; - } - StringRef op_name = op->getName().getStringRef(); - // Drop the `tf.` prefix to query TF registry. - auto node_name = - op_name.drop_front(TensorFlowDialect::getDialectNamespace().size() + 1); - const tensorflow::OpRegistrationData* op_reg_data = - tensorflow::OpRegistry::Global()->LookUp(node_name.data()); - return op_reg_data && !op_reg_data->op_def.is_stateful(); + // For op's in the Tensorflow dialect, query the dialect. + if (op->getName().getDialect() == + TF::TensorFlowDialect::getDialectNamespace()) + return !TensorFlowDialect::CanHaveSideEffects(op); + + // Otherwise, conservatively assume that there can be side effects. + return false; } } // namespace diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc index dbad613d909..e35e5dc40a8 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.cc @@ -74,23 +74,6 @@ namespace TF { //===----------------------------------------------------------------------===// namespace { -// Returns true if the op can be duplicated. -bool CanDuplicate(Operation *op) { - // If the op is marked with the cannot duplicate trait, it cannot be - // duplicated. - if (op->hasTrait()) return false; - - // If the op has no memory side effects, it can be duplicated. - if (MemoryEffectOpInterface::hasNoEffect(op)) return true; - - // If the op is marked stateless using the `is_stateless` attribute, that - // attribute determines if the op can be duplicated. - if (auto is_stateless = op->getAttrOfType("is_stateless")) - return is_stateless.getValue(); - - // Otherwise, assume ops can be duplicated by default. - return true; -} // Returns true of the given function has a single uses (within the scope // of the module containing it and all parent modules). @@ -156,7 +139,7 @@ struct TFInlinerInterface : public DialectInlinerInterface { // post inlining, the function will be dead and eliminated from the IR. // So there won't be any code duplication. FuncOp func = op->getParentOfType(); - return !func || CanDuplicate(op) || HasSingleUse(func); + return !func || TensorFlowDialect::CanDuplicate(op) || HasSingleUse(func); } //===--------------------------------------------------------------------===// @@ -183,6 +166,42 @@ struct TFInlinerInterface : public DialectInlinerInterface { // TF Dialect //===----------------------------------------------------------------------===// +// Returns true if the op can be duplicated. +bool TensorFlowDialect::CanDuplicate(Operation *op) { + // If the op is marked with the cannot duplicate trait, it cannot be + // duplicated. + if (op->hasTrait()) return false; + + // If the op has no memory side effects, it can be duplicated. + if (MemoryEffectOpInterface::hasNoEffect(op)) return true; + + // If the op is marked stateless using the `is_stateless` attribute, that + // attribute determines if the op can be duplicated. + if (auto is_stateless = op->getAttrOfType("is_stateless")) + return is_stateless.getValue(); + + // Otherwise, assume ops can be duplicated by default if its registered, else + // it cannot be for unknown ops. + return op->isRegistered(); +} + +// Returns true if the op can have side effects. +bool TensorFlowDialect::CanHaveSideEffects(Operation *op) { + // If the op has no memory side effects, it has no side effects + if (MemoryEffectOpInterface::hasNoEffect(op)) return false; + + // If the op is marked stateless using the `is_stateless` attribute, then + // it has no side effects. + if (auto is_stateless = op->getAttrOfType("is_stateless")) + return !is_stateless.getValue(); + + // Terminators defined in the TF dialect do not have side effects. + if (op->isKnownTerminator()) return false; + + // Otherwise assume that the op can have side effects. + return true; +} + std::vector *TensorFlowDialect::additional_operation_hooks_ = new std::vector(); diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h index 039ed1bc3a8..bbcce4ee177 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h @@ -63,6 +63,12 @@ class TensorFlowDialect : public Dialect { // Returns the string description of stateful attribute. static StringRef GetStatefulAttrName() { return "tf.signature.is_stateful"; } + // Returns true if the op can be duplicated during transformations. + static bool CanDuplicate(Operation *op); + + // Returns true if the op can have side effects. + static bool CanHaveSideEffects(Operation *op); + Attribute parseAttribute(DialectAsmParser &parser, Type type) const override; void printAttribute(Attribute attr, DialectAsmPrinter &os) const override; diff --git a/tensorflow/compiler/mlir/tensorflow/tests/side-effect-analysis-test.mlir b/tensorflow/compiler/mlir/tensorflow/tests/side-effect-analysis-test.mlir index d79e028ba9e..5eacbdea180 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/side-effect-analysis-test.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/side-effect-analysis-test.mlir @@ -277,7 +277,7 @@ func @with_replicate( // ----- -// Tests that the pass does not add control dependencies a stateless if op. +// Tests that the pass does not add control dependencies for a stateless if op. // CHECK-LABEL: func @stateless_if_op func @stateless_if_op( @@ -361,6 +361,83 @@ func @if_else(%arg0: tensor) -> tensor { // ----- +// Tests that the pass does not add control dependencies for a stateless +// IfRegion op. + +// CHECK-LABEL: func @stateless_ifregion_op +func @stateless_ifregion_op( + // expected-remark@above {{ID: 18}} + %arg0: tensor<*x!tf.resource>>, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 16}} + // expected-remark@above {{Successors: {17}}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 14}} + // expected-remark@above {{Successors: {15}}} + + %r0 = "tf.ReadVariableOp"(%arg0) : + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {12}}} + (tensor<*x!tf.resource>>) -> tensor<32xf32> + + %if = "tf.IfRegion"(%arg1) ( + // expected-remark@above {{ID: 11}} + { // Then region. + %graph = tf_executor.graph { + // expected-remark@above {{ID: 4}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield %arg1 : tensor + // expected-remark@above {{ID: 1}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 5}} + }, { // Else region + %graph = tf_executor.graph { + // expected-remark@above {{ID: 9}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Successors: {8}}} + tf_executor.yield %arg1 : tensor + // expected-remark@above {{ID: 6}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Predecessors: {7}}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 10}} + } + ) { is_stateless = true} : (tensor) -> tensor + + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {13}}} + (tensor<*x!tf.resource>>, tensor<32xf32>) -> () + + tf_executor.yield + // expected-remark@above {{ID: 13}} + // expected-remark@above {{Predecessors: {12}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 15}} + // expected-remark@above {{Predecessors: {14}}} + } + return + // expected-remark@above {{ID: 17}} + // expected-remark@above {{Predecessors: {16}}} +} + +// ----- + // Tests that the pass does not add control dependencies a stateless while op. // CHECK-LABEL: func @stateless_if_op @@ -379,7 +456,7 @@ func @stateless_if_op( // expected-remark@above {{ID: 0}} // expected-remark@above {{Successors: {2}}} (tensor<*x!tf.resource>>) -> tensor<32xf32> - %if = "tf.While"(%arg1) { + %while = "tf.While"(%arg1) { // expected-remark@above {{ID: 1}} body = @while_body, cond = @while_cond, is_stateless = true} : (tensor) -> tensor @@ -445,9 +522,98 @@ func @while_cond(%arg0: tensor) -> tensor { // ----- +// Tests that the pass does not add control dependencies a stateless WhileRegion +// op. + +// CHECK-LABEL: func @stateless_whileregion_op +func @stateless_whileregion_op( + // expected-remark@above {{ID: 18}} + %arg0: tensor<*x!tf.resource>>, + %arg1: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 16}} + // expected-remark@above {{Successors: {17}}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 14}} + // expected-remark@above {{Successors: {15}}} + %r0 = "tf.ReadVariableOp"(%arg0) : + // expected-remark@above {{ID: 0}} + // expected-remark@above {{Successors: {12}}} + (tensor<*x!tf.resource>>) -> tensor<32xf32> + + %while = "tf.WhileRegion"(%arg1) ( + // expected-remark@above {{ID: 11}} + { + ^bb0(%carg: tensor): + %graph = tf_executor.graph { + // expected-remark@above {{ID: 4}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 2}} + // expected-remark@above {{Successors: {3}}} + tf_executor.yield %carg : tensor + // expected-remark@above {{ID: 1}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {2}}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 5}} + }, { + ^bb0(%barg: tensor): + %graph = tf_executor.graph { + // expected-remark@above {{ID: 9}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 7}} + // expected-remark@above {{Successors: {8}}} + tf_executor.yield %barg : tensor + // expected-remark@above {{ID: 6}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 8}} + // expected-remark@above {{Predecessors: {7}}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 10}} + } + ) {is_stateless = true} : (tensor) -> tensor + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Predecessors: {0}}} + // expected-remark@above {{Successors: {13}}} + (tensor<*x!tf.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 13}} + // expected-remark@above {{Predecessors: {12}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 15}} + // expected-remark@above {{Predecessors: {14}}} + } + return + // expected-remark@above {{ID: 17}} + // expected-remark@above {{Predecessors: {16}}} +} + +// ----- + // Tests that the pass tracks control dependencies for variables from an if op's // output. +// In this test, the resources computed and used are as follows: +// (* = unknown resource id which aliases with everything else) +// id0 = arg0 +// if-then-branch: [u0, arg0, arg0] +// if-else-branch: [arg0, arg0, arg1] +// => first result is unknown, second and third is passthrough +// if results : [*, arg0, {arg0, arg1}[ +// ID #2: read (unknown) -> succ {5, 6) +// ID #3: read (arg0) -> succ {5} +// ID #4: read({arg0,arg1}) -> succ {5,6} +// ID #5: write(arg0) +// ID #6: write(arg1) + // CHECK-LABEL: func @output_of_if_op func @output_of_if_op( // expected-remark@above {{ID: 12}} @@ -597,9 +763,151 @@ func @if_else( // ----- +// Tests that the pass tracks control dependencies for variables from an +// IfRegion op's output. + +// CHECK-LABEL: func @output_of_ifregion_op +func @output_of_ifregion_op( + // expected-remark@above {{ID: 26}} + %arg0: tensor<*x!tf.resource>>, + %arg1: tensor<*x!tf.resource>>, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 24}} + // expected-remark@above {{Successors: {25}}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 22}} + // expected-remark@above {{Successors: {23}}} + %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf.resource>>) + // expected-remark@above {{ID: 0}} + -> tensor<*x!tf.resource>> + %if:3 = "tf.IfRegion"(%arg2) ( + // expected-remark@above {{ID: 15}} + // expected-remark@above {{Successors: {16,17,18}}} + { + %graph:3 = tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island:4 = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %u0 = "tf._UnknownSideEffectingOp_"() : () + // expected-remark@above {{ID: 1}} + // expected-remark@above {{Successors: {3}}} + -> tensor<*x!tf.resource>> + %iid0 = "tf.Identity"(%id0) : (tensor<*x!tf.resource>>) + // expected-remark@above {{ID: 2}} + -> tensor<*x!tf.resource>> + tf_executor.yield %u0, %iid0, %iid0 : + // expected-remark@above {{ID: 3}} + // expected-remark@above {{Predecessors: {1}}} + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>> + } + tf_executor.fetch %island#0, %island#1, %island#2 : + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>> + } + "tf.Yield"(%graph#0, %graph#1, %graph#2) : + // expected-remark@above {{ID: 7}} + (tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>) -> () + }, + { + %graph:3 = tf_executor.graph { + // expected-remark@above {{ID: 13}} + %island:4 = tf_executor.island { + // expected-remark@above {{ID: 11}} + // expected-remark@above {{Successors: {12}}} + %iid0 = "tf.Identity"(%id0) : (tensor<*x!tf.resource>>) + // expected-remark@above {{ID: 8}} + -> tensor<*x!tf.resource>> + %iid1 = "tf.Identity"(%arg1) : (tensor<*x!tf.resource>>) + // expected-remark@above {{ID: 9}} + -> tensor<*x!tf.resource>> + tf_executor.yield %iid0, %iid0, %iid1 : + // expected-remark@above {{ID: 10}} + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>> + } + tf_executor.fetch %island#0, %island#1, %island#2 : + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Predecessors: {11}}} + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>> + } + "tf.Yield"(%graph#0, %graph#1, %graph#2) : + // expected-remark@above {{ID: 14}} + (tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>) -> () + }) { is_stateless = false} + : (tensor) -> + (tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>) + %r0 = "tf.ReadVariableOp"(%if#0) : + // expected-remark@above {{ID: 16}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19,20}}} + (tensor<*x!tf.resource>>) -> tensor<32xf32> + %r1 = "tf.ReadVariableOp"(%if#1) : + // expected-remark@above {{ID: 17}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19}}} + (tensor<*x!tf.resource>>) -> tensor<32xf32> + %r2 = "tf.ReadVariableOp"(%if#2) : + // expected-remark@above {{ID: 18}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19,20}}} + (tensor<*x!tf.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 19}} + // expected-remark@above {{Predecessors: {16,17,18}}} + // expected-remark@above {{Successors: {21}}} + (tensor<*x!tf.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg1, %r0) : + // expected-remark@above {{ID: 20}} + // expected-remark@above {{Predecessors: {16,18}}} + // expected-remark@above {{Successors: {21}}} + (tensor<*x!tf.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 21}} + // expected-remark@above {{Predecessors: {19,20}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 23}} + // expected-remark@above {{Predecessors: {22}}} + } + return + // expected-remark@above {{ID: 25}} + // expected-remark@above {{Predecessors: {24}}} +} + +// ----- + // Tests that the pass tracks control dependencies for variables from a while // op's output. +// Here: +// id0 = arg0 +// while-inputs = (id0/arg0, arg1, arg1) +// while body pass through first and second arg, not last one +// while-results = (arg0, arg1, Unknown) +// #ID 2: read(arg0) -> succ{5} +// #ID 3: read(arg1) -> succ{6} +// #ID 4: read(unknown) -> succ{5,6} +// #ID 5 : write(arg0) +// #ID 6 : write(arg1) + + // CHECK-LABEL: func @output_of_while_op func @output_of_while_op( // expected-remark@above {{ID: 12}} @@ -631,24 +939,24 @@ func @output_of_while_op( // expected-remark@above {{Predecessors: {1}}} // expected-remark@above {{Successors: {5}}} (tensor<*x!tf.resource>>) -> tensor<32xf32> - %r1 = "tf.ReadVariableOp"(%while#1) : + %r1 = "tf.ReadVariableOp"(%while#2) : // expected-remark@above {{ID: 3}} // expected-remark@above {{Predecessors: {1}}} - // expected-remark@above {{Successors: {5}}} - (tensor<*x!tf.resource>>) -> tensor<32xf32> - %r2 = "tf.ReadVariableOp"(%while#2) : - // expected-remark@above {{ID: 4}} - // expected-remark@above {{Predecessors: {1}}} // expected-remark@above {{Successors: {6}}} (tensor<*x!tf.resource>>) -> tensor<32xf32> + %r2 = "tf.ReadVariableOp"(%while#3) : + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Predecessors: {1}}} + // expected-remark@above {{Successors: {5,6}}} + (tensor<*x!tf.resource>>) -> tensor<32xf32> "tf.AssignVariableOp"(%arg0, %r0) : // expected-remark@above {{ID: 5}} - // expected-remark@above {{Predecessors: {2,3}}} + // expected-remark@above {{Predecessors: {2,4}}} // expected-remark@above {{Successors: {7}}} (tensor<*x!tf.resource>>, tensor<32xf32>) -> () "tf.AssignVariableOp"(%arg1, %r0) : // expected-remark@above {{ID: 6}} - // expected-remark@above {{Predecessors: {4}}} + // expected-remark@above {{Predecessors: {3,4}}} // expected-remark@above {{Successors: {7}}} (tensor<*x!tf.resource>>, tensor<32xf32>) -> () tf_executor.yield @@ -740,6 +1048,136 @@ func @while_cond( // ----- +// Tests that the pass tracks control dependencies for variables from a +// WhileRegion op's output. + +// CHECK-LABEL: func @output_of_whileregion_op +func @output_of_whileregion_op( + // expected-remark@above {{ID: 26}} + %arg0: tensor<*x!tf.resource>>, + %arg1: tensor<*x!tf.resource>>, + %arg2: tensor) { + tf_executor.graph { + // expected-remark@above {{ID: 24}} + // expected-remark@above {{Successors: {25}}} + // CHECK: tf_executor.island + %island = tf_executor.island { + // expected-remark@above {{ID: 22}} + // expected-remark@above {{Successors: {23}}} + %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf.resource>>) + // expected-remark@above {{ID: 0}} + -> tensor<*x!tf.resource>> + %while:4 = "tf.WhileRegion"(%arg2, %id0, %arg1, %arg1) ( + // expected-remark@above {{ID: 15}} + // expected-remark@above {{Successors: {16,17,18}}} + { + ^bb0(%pred: tensor, + %carg1: tensor<*x!tf.resource>>, + %carg2: tensor<*x!tf.resource>>, + %carg3: tensor<*x!tf.resource>>): + %graph = tf_executor.graph { + // expected-remark@above {{ID: 6}} + %island:2 = tf_executor.island { + // expected-remark@above {{ID: 4}} + // expected-remark@above {{Successors: {5}}} + %const = "tf.Const"() { value = dense<0> : tensor } : () -> tensor + // expected-remark@above {{ID: 1}} + %eq = "tf.Equal"(%pred, %const) : (tensor, tensor) -> tensor + // expected-remark@above {{ID: 2}} + tf_executor.yield %eq : tensor + // expected-remark@above {{ID: 3}} + } + tf_executor.fetch %island#0 : tensor + // expected-remark@above {{ID: 5}} + // expected-remark@above {{Predecessors: {4}}} + } + "tf.Yield"(%graph) : (tensor) -> () + // expected-remark@above {{ID: 7}} + }, + { + ^bb0(%pred: tensor, + %barg0: tensor<*x!tf.resource>>, + %barg1: tensor<*x!tf.resource>>, + %barg2: tensor<*x!tf.resource>>): + %graph:4 = tf_executor.graph { + // expected-remark@above {{ID: 13}} + %island:5 = tf_executor.island { + // expected-remark@above {{ID: 11}} + // expected-remark@above {{Successors: {12}}} + %iid0 = "tf.Identity"(%barg0) : (tensor<*x!tf.resource>>) + // expected-remark@above {{ID: 8}} + -> tensor<*x!tf.resource>> + %u0 = "tf._UnknownSideEffectingOp_"() : () + // expected-remark@above {{ID: 9}} + // expected-remark@above {{Successors: {10}}} + -> tensor<*x!tf.resource>> + tf_executor.yield %pred, %iid0, %barg1, %u0 : + // expected-remark@above {{ID: 10}} + // expected-remark@above {{Predecessors: {9}}} + tensor, tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>> + } + tf_executor.fetch %island#0, %island#1, %island#2, %island#3 : + // expected-remark@above {{ID: 12}} + // expected-remark@above {{Predecessors: {11}}} + tensor, tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>> + } + "tf.Yield"(%graph#0, %graph#1, %graph#2, %graph#3) : + // expected-remark@above {{ID: 14}} + (tensor, tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>) -> () + } + ) {is_stateless = false} + : (tensor, tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>) -> + (tensor, tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>, + tensor<*x!tf.resource>>) + %r0 = "tf.ReadVariableOp"(%while#1) : + // expected-remark@above {{ID: 16}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19}}} + (tensor<*x!tf.resource>>) -> tensor<32xf32> + %r1 = "tf.ReadVariableOp"(%while#2) : + // expected-remark@above {{ID: 17}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {20}}} + (tensor<*x!tf.resource>>) -> tensor<32xf32> + %r2 = "tf.ReadVariableOp"(%while#3) : + // expected-remark@above {{ID: 18}} + // expected-remark@above {{Predecessors: {15}}} + // expected-remark@above {{Successors: {19,20}}} + (tensor<*x!tf.resource>>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %r0) : + // expected-remark@above {{ID: 19}} + // expected-remark@above {{Predecessors: {16,18}}} + // expected-remark@above {{Successors: {21}}} + (tensor<*x!tf.resource>>, tensor<32xf32>) -> () + "tf.AssignVariableOp"(%arg1, %r0) : + // expected-remark@above {{ID: 20}} + // expected-remark@above {{Predecessors: {17,18}}} + // expected-remark@above {{Successors: {21}}} + (tensor<*x!tf.resource>>, tensor<32xf32>) -> () + tf_executor.yield + // expected-remark@above {{ID: 21}} + // expected-remark@above {{Predecessors: {19,20}}} + } + tf_executor.fetch %island : !tf_executor.control + // expected-remark@above {{ID: 23}} + // expected-remark@above {{Predecessors: {22}}} + } + return + // expected-remark@above {{ID: 25}} + // expected-remark@above {{Predecessors: {24}}} +} + +// ----- + // Tests that the pass tracks control dependencies based on TF op registry // statefulness flag, for ops not yet defined in ODS. @@ -824,4 +1262,3 @@ func @arguments_with_unique_ids( // expected-remark@above {{ID: 8}} // expected-remark@above {{Predecessors: {7}}} } - From 9662a54d49aa6a114a8950d7ed5a92ed9f1750ff Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Tue, 11 Aug 2020 14:47:17 -0700 Subject: [PATCH 0848/1017] Change the cloned windows builds to use CUDA11 1. Change the pool to the CUDA11 pool. 2. Set TF_CUDA_VERSION and TF_CUDNN_VERSION PiperOrigin-RevId: 326106265 Change-Id: I4618c38a54ef784cb3533ae225f2570b65f48a22 --- .../rel/windows_cuda11/common_win_cuda11.bat | 24 +++++++++++++++++++ .../rel/windows_cuda11/cpu_libtensorflow.bat | 2 +- .../ci_build/rel/windows_cuda11/cpu_py35.bat | 2 +- .../ci_build/rel/windows_cuda11/cpu_py36.bat | 2 +- .../ci_build/rel/windows_cuda11/cpu_py37.bat | 2 +- .../ci_build/rel/windows_cuda11/cpu_py38.bat | 2 +- .../rel/windows_cuda11/gpu_libtensorflow.bat | 2 +- .../rel/windows_cuda11/gpu_pip_on_cpu.bat | 2 +- .../ci_build/rel/windows_cuda11/gpu_py35.bat | 2 +- .../ci_build/rel/windows_cuda11/gpu_py36.bat | 2 +- .../ci_build/rel/windows_cuda11/gpu_py37.bat | 2 +- .../ci_build/rel/windows_cuda11/gpu_py38.bat | 2 +- .../tools/ci_build/release/common_win.bat | 4 +++- 13 files changed, 38 insertions(+), 12 deletions(-) create mode 100644 tensorflow/tools/ci_build/rel/windows_cuda11/common_win_cuda11.bat diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/common_win_cuda11.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/common_win_cuda11.bat new file mode 100644 index 00000000000..dea3a791277 --- /dev/null +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/common_win_cuda11.bat @@ -0,0 +1,24 @@ +:: Copyright 2020 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +echo on + +SET TF_CUDA_VERSION=11 +SET TF_CUDNN_VERSION=8 + +REM TODO(sanjoy): This script should be removed once common_win.bat +REM defaults to CUDA 11. + +CALL tensorflow\tools\ci_build\release\common_win.bat diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat index 67941234b15..9d7c98dcc91 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat @@ -13,7 +13,7 @@ :: limitations under the License. :: ============================================================================= -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\bazel\run_libtensorflow.bat || exit /b 1 diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat index 175917d7cad..487ce7592fa 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat @@ -15,6 +15,6 @@ SET PYTHON_DIRECTORY=Python35 -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat index 85b75053eff..56bcd9403b1 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat @@ -15,6 +15,6 @@ SET PYTHON_DIRECTORY=Python36 -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat index d8a6673ba4c..a326a0c7aa8 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat @@ -15,6 +15,6 @@ SET PYTHON_DIRECTORY=Python37 -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat index 86adcda0bb9..46bfcc351c2 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python38 -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat index 8ab78bef3ca..b1db68586aa 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat @@ -13,7 +13,7 @@ :: limitations under the License. :: ============================================================================= -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\bazel\run_libtensorflow.bat || exit /b diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat index 213de532069..65d1f420292 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python36 -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\integration\gpu_pip_on_cpu\run.bat diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat index 86c118b2f83..66cdfd09141 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python35 -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat index cc4f84afbee..6931c08160d 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python36 -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat index 5fa798e3eb8..dcaf588b912 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python37 -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat index fa1fc131145..6305c544c76 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python38 -CALL tensorflow\tools\ci_build\release\common_win.bat +CALL common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/release/common_win.bat b/tensorflow/tools/ci_build/release/common_win.bat index e460ec8b0e1..6b9b533e25c 100644 --- a/tensorflow/tools/ci_build/release/common_win.bat +++ b/tensorflow/tools/ci_build/release/common_win.bat @@ -62,7 +62,9 @@ IF "%PYTHON_DIRECTORY%"=="Python37" ( IF NOT DEFINED TF_CUDA_VERSION ( SET TF_CUDA_VERSION=10.1 ) -SET TF_CUDNN_VERSION=7 +IF NOT DEFINED TF_CUDNN_VERSION ( + SET TF_CUDNN_VERSION=7 +) SET TF_CUDA_COMPUTE_CAPABILITIES=sm_35,sm_37,sm_52,sm_60,sm_61,compute_70 SET CUDA_TOOLKIT_PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v%TF_CUDA_VERSION% SET CUDNN_INSTALL_PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v%TF_CUDA_VERSION% From a5a42397ef32593df3050bdf27c1874191fcb81d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 14:55:32 -0700 Subject: [PATCH 0849/1017] PR #42031: Enable depthwise convs in auto_mixed_precision Imported from GitHub PR https://github.com/tensorflow/tensorflow/pull/42031 These are well-supported as of CUDNN v8. Also adds a Python test. PiperOrigin-RevId: 326107903 Change-Id: I736a576f0420e69237da03e8af9f08d22b0af861 --- .../optimizers/auto_mixed_precision_lists.h | 10 ++--- .../grappler/auto_mixed_precision_test.py | 43 ------------------- 2 files changed, 5 insertions(+), 48 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h b/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h index 7902700fb0f..805a7de9225 100644 --- a/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h +++ b/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h @@ -127,6 +127,11 @@ class AutoMixedPrecisionListsCuda : public AutoMixedPrecisionLists { "GRUBlockCellGrad", "LSTMBlockCell", "LSTMBlockCellGrad", + // TODO(benbarsdell): Enable these when fast and safe fp16 kernels are + // available for depthwise convolutions. + // "DepthwiseConv2dNative", + // "DepthwiseConv2dNativeBackpropFilter", + // "DepthwiseConv2dNativeBackpropInput", "MatMul", }; if (cuda_version_ >= 9010) { @@ -142,11 +147,6 @@ class AutoMixedPrecisionListsCuda : public AutoMixedPrecisionLists { list.insert("Conv3DBackpropInput"); list.insert("Conv3DBackpropInputV2"); } - if (cudnn_version_ >= 8000) { - list.insert("DepthwiseConv2dNative"); - list.insert("DepthwiseConv2dNativeBackpropFilter"); - list.insert("DepthwiseConv2dNativeBackpropInput"); - } UpdateList("ALLOWLIST", &list); // For backwards compatibility, keeping the original env variable here. // TODO(reedwm): This should be removed if we don't have active users. diff --git a/tensorflow/python/grappler/auto_mixed_precision_test.py b/tensorflow/python/grappler/auto_mixed_precision_test.py index f196c15bd7f..567ff8c000d 100644 --- a/tensorflow/python/grappler/auto_mixed_precision_test.py +++ b/tensorflow/python/grappler/auto_mixed_precision_test.py @@ -46,7 +46,6 @@ from tensorflow.python.ops import random_ops from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops import variables from tensorflow.python.ops.losses import losses -from tensorflow.python.platform import sysconfig from tensorflow.python.platform import test from tensorflow.python.training import adam from tensorflow.python.training import gradient_descent @@ -139,11 +138,6 @@ def _conv_pool(x): return h_pool2 -def _depthwise_conv2d(x, w): - """Returns a 2d depthwise convolution layer with full stride.""" - return nn.depthwise_conv2d(x, w, strides=[1, 1, 1, 1], padding='SAME') - - def _simple_loop(x, functor): """Simple loop whose body is provided by the functor.""" init = (constant_op.constant(0), x) @@ -572,43 +566,6 @@ class AutoMixedPrecisionTest(test.TestCase, parameterized.TestCase): tol = 5e-3 if mode == 'mkl' else 1e-3 self.assertAllClose(output_val_ref, output_val, atol=tol, rtol=tol) - # TODO(benbarsdell): This test has not been tried with MKL. - @parameterized.parameters(['cuda']) - @test_util.run_deprecated_v1 - @test_util.disable_xla('This test does not pass with XLA') - def test_depthwise_conv2d(self, mode): - """Test grad ops with depthwise convolution2d graph.""" - self._maybe_skip(mode) - cudnn_version = tuple([ - int(x) for x in sysconfig.get_build_info()['cudnn_version'].split('.') - ]) - if cudnn_version < (8,): - # Depthwise conv2d ops are only enabled in auto_mixed_precision as of - # cuDNN v8. - self.skipTest('cuDNN version >= 8 required') - random_seed.set_random_seed(0) - x = _input([2, 8, 8, 1]) - f = _weight([3, 3, 1, 4]) - y = _depthwise_conv2d(x, f) - y = array_ops.identity(y) - optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.01) - g = optimizer.compute_gradients(y, [x, f]) - output = (y, g) - - output_val_ref, output_val, cost_graph = self._run(mode, output) - node_map = _build_node_map(cost_graph.node) - self._assert_output_f16(mode, node_map, 'depthwise') - self._assert_output_f16( - mode, node_map, - 'gradients/depthwise_grad/DepthwiseConv2dNativeBackpropInput') - self._assert_output_f16( - mode, node_map, - 'gradients/depthwise_grad/DepthwiseConv2dNativeBackpropFilter') - - output_val_ref, output_val, cost_graph = self._run(mode, output) - tol = 2e-3 - self.assertAllClose(output_val_ref, output_val, atol=tol, rtol=tol) - @parameterized.parameters(['cuda', 'mkl']) @test_util.run_v1_only('b/138749235') @test_util.disable_xla('This test does not pass with XLA') From 61d63b37643568c47f0fc293924d89837cd60c0a Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Tue, 11 Aug 2020 22:27:04 +0000 Subject: [PATCH 0850/1017] binary add and zeros like update --- tensorflow/core/kernels/map_kernels.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 9e436bc5fcd..6b38fe7aee6 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -178,7 +178,8 @@ class TensorMapHasKey : public OpKernel { template Status TensorMapBinaryAdd(OpKernelContext* c, const TensorMap& a, - const TensorMap& b, TensorMap* out) { + const TensorMap& b, TensorMap* out) { + // binary add returns a union of keys, values with keys in the intersection are added out->tensors() = a.tensors(); for (const std::pair& p : b.tensors()) { absl::flat_hash_map::iterator it = out->tensors().find(p.first); @@ -196,11 +197,7 @@ Status TensorMapBinaryAdd(OpKernelContext* c, const TensorMap& a, template Status TensorMapZerosLike(OpKernelContext* c, const TensorMap& x, TensorMap* y) { - for (const std::pair& p : x.tensors()) { - Tensor val; - TF_RETURN_IF_ERROR(ZerosLikeTensor(c, p.second, &val)); - y->tensors().emplace(p.first, val); - } + // zeros like returns an empty map return Status::OK(); } From 6861b3f69d8db26045316d8b4db10d4d5ce20c65 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 15:23:25 -0700 Subject: [PATCH 0851/1017] Add device_type to TfStatsDatabase. PiperOrigin-RevId: 326113527 Change-Id: I794cd6fa3724c8189971990abf65e3180c0b82dc --- tensorflow/core/profiler/convert/op_stats_to_tf_stats.cc | 1 + tensorflow/core/profiler/convert/op_stats_to_tf_stats_test.cc | 2 ++ tensorflow/core/profiler/protobuf/tf_stats.proto | 2 ++ 3 files changed, 5 insertions(+) diff --git a/tensorflow/core/profiler/convert/op_stats_to_tf_stats.cc b/tensorflow/core/profiler/convert/op_stats_to_tf_stats.cc index 67024809e61..a5e127a45d0 100644 --- a/tensorflow/core/profiler/convert/op_stats_to_tf_stats.cc +++ b/tensorflow/core/profiler/convert/op_stats_to_tf_stats.cc @@ -114,6 +114,7 @@ TfStatsDatabase ConvertOpStatsToTfStats(const OpStats& op_stats) { *tf_stats_db.mutable_without_idle() = GenerateTfStatsTable( host_tf_metrics_db, device_tf_metrics_db, kernel_stats_by_op_name, ridge_point, /*exclude_idle=*/true); + tf_stats_db.set_device_type(op_stats.run_environment().device_type()); return tf_stats_db; } diff --git a/tensorflow/core/profiler/convert/op_stats_to_tf_stats_test.cc b/tensorflow/core/profiler/convert/op_stats_to_tf_stats_test.cc index 5cf2847ea0d..2b255352916 100644 --- a/tensorflow/core/profiler/convert/op_stats_to_tf_stats_test.cc +++ b/tensorflow/core/profiler/convert/op_stats_to_tf_stats_test.cc @@ -127,6 +127,8 @@ block_z:1)MULTI"; ConvertXSpaceToOpStats(space, {OP_METRICS_DB, KERNEL_STATS_DB}); const TfStatsDatabase tf_stats = ConvertOpStatsToTfStats(op_stats); + EXPECT_EQ(tf_stats.device_type(), op_stats.run_environment().device_type()); + // TfOp1, TfOp3, TfOp2, Idle EXPECT_EQ(4, tf_stats.with_idle().tf_stats_record_size()); diff --git a/tensorflow/core/profiler/protobuf/tf_stats.proto b/tensorflow/core/profiler/protobuf/tf_stats.proto index 099d8478831..6e09057e557 100644 --- a/tensorflow/core/profiler/protobuf/tf_stats.proto +++ b/tensorflow/core/profiler/protobuf/tf_stats.proto @@ -10,6 +10,8 @@ message TfStatsDatabase { TfStatsTable with_idle = 4; // The table that excludes IDLE time. TfStatsTable without_idle = 5; + // The type of device used. + string device_type = 6; reserved 1, 2, 3; } From bd62ee5bfa8e855ecfda67e3a28095ecb31fb531 Mon Sep 17 00:00:00 2001 From: Jared Duke Date: Tue, 11 Aug 2020 15:44:18 -0700 Subject: [PATCH 0852/1017] Fix several narrowing issues on 32-bit devices PiperOrigin-RevId: 326117351 Change-Id: Iaa094b1e7664ec0bc9f0c91e6cd972fcae19b5b9 --- .../core/kernels/batching_util/concat_split_util.h | 7 +++++-- .../core/kernels/sparse/sparse_matrix_components_op.cc | 5 +++-- tensorflow/core/kernels/tensor_list.h | 3 ++- tensorflow/core/kernels/tensor_map.h | 3 ++- tensorflow/core/kernels/unravel_index_op.cc | 10 ++++++---- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tensorflow/core/kernels/batching_util/concat_split_util.h b/tensorflow/core/kernels/batching_util/concat_split_util.h index 50ffc664452..fcd3b6ef0bb 100644 --- a/tensorflow/core/kernels/batching_util/concat_split_util.h +++ b/tensorflow/core/kernels/batching_util/concat_split_util.h @@ -174,8 +174,11 @@ Status SplitCPU(OpKernelContext* context, const Tensor& input, context->allocate_temp(input.dtype(), output_shape, &output)); auto output_shaped = output.shaped({size, suffix_dim_size}); - Eigen::DSizes slice_indices{position, 0}; - Eigen::DSizes slice_sizes{size, suffix_dim_size}; + Eigen::DSizes slice_indices{ + static_cast(position), 0}; + Eigen::DSizes slice_sizes{ + static_cast(size), + static_cast(suffix_dim_size)}; functor::Split()(context->eigen_device(), output_shaped, input_reshaped, slice_indices, slice_sizes); diff --git a/tensorflow/core/kernels/sparse/sparse_matrix_components_op.cc b/tensorflow/core/kernels/sparse/sparse_matrix_components_op.cc index 2eaf9bd5310..2832d66ee9f 100644 --- a/tensorflow/core/kernels/sparse/sparse_matrix_components_op.cc +++ b/tensorflow/core/kernels/sparse/sparse_matrix_components_op.cc @@ -89,8 +89,9 @@ class CSRSparseMatrixComponentsOp : public OpKernel { slice_int(d, /*output*/ row_ptrs, /*input*/ csr_sparse_matrix->row_pointers().vec(), - /*slice_indices*/ EVec{index * (rows + 1)}, - /*slice_sizes*/ EVec{rows + 1}); + /*slice_indices*/ + EVec{static_cast(index * (rows + 1))}, + /*slice_sizes*/ EVec{static_cast(rows + 1)}); slice_int(d, /*output*/ col_inds, /*input*/ csr_sparse_matrix->col_indices().vec(), diff --git a/tensorflow/core/kernels/tensor_list.h b/tensorflow/core/kernels/tensor_list.h index b67157d4c65..5d3921cffe9 100644 --- a/tensorflow/core/kernels/tensor_list.h +++ b/tensorflow/core/kernels/tensor_list.h @@ -151,7 +151,8 @@ class TensorList { #if defined(PLATFORM_GOOGLE) // TODO(ebrevdo): Identify why Variant inline size is smaller on mobile devices. -static_assert(Variant::CanInlineType(), +// For 32-bit devices, it's acceptable not to inline. +static_assert(Variant::CanInlineType() || sizeof(void*) < 8, "Must be able to inline TensorList into a Variant"); #endif } // namespace tensorflow diff --git a/tensorflow/core/kernels/tensor_map.h b/tensorflow/core/kernels/tensor_map.h index 859d51b47dd..33b53e7c72f 100644 --- a/tensorflow/core/kernels/tensor_map.h +++ b/tensorflow/core/kernels/tensor_map.h @@ -179,7 +179,8 @@ class TensorMap { #if defined(PLATFORM_GOOGLE) // TODO(ebrevdo): Identify why Variant inline size is smaller on mobile devices. -static_assert(Variant::CanInlineType(), +// For 32-bit devices, it's acceptable not to inline. +static_assert(Variant::CanInlineType() || sizeof(void*) < 8, "Must be able to inline TensorMap into a Variant"); #endif } // namespace tensorflow diff --git a/tensorflow/core/kernels/unravel_index_op.cc b/tensorflow/core/kernels/unravel_index_op.cc index b45ff5e5b85..11d9dac70f7 100644 --- a/tensorflow/core/kernels/unravel_index_op.cc +++ b/tensorflow/core/kernels/unravel_index_op.cc @@ -107,12 +107,14 @@ class UnravelIndexOp : public OpKernel { auto output = output_tensor->matrix(); - Eigen::array reshape{{dims_tensor.NumElements(), 1}}; - Eigen::array bcast({1, indices_tensor.NumElements()}); + Eigen::array reshape{ + {static_cast(dims_tensor.NumElements()), 1}}; + Eigen::array bcast( + {1, static_cast(indices_tensor.NumElements())}); Eigen::array indices_reshape{ - {1, indices_tensor.NumElements()}}; + {1, static_cast(indices_tensor.NumElements())}}; Eigen::array indices_bcast( - {dims_tensor.NumElements(), 1}); + {static_cast(dims_tensor.NumElements()), 1}); output = indices_tensor.vec() .reshape(indices_reshape) From c09eabcd28667c81e86a2cbafb5a68c03186e038 Mon Sep 17 00:00:00 2001 From: Yixing Fu Date: Tue, 11 Aug 2020 22:51:35 +0000 Subject: [PATCH 0853/1017] changes according to comment --- tensorflow/python/keras/saving/hdf5_format_test.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/keras/saving/hdf5_format_test.py b/tensorflow/python/keras/saving/hdf5_format_test.py index 3d5c5c1f762..e4676c83e7d 100644 --- a/tensorflow/python/keras/saving/hdf5_format_test.py +++ b/tensorflow/python/keras/saving/hdf5_format_test.py @@ -731,9 +731,9 @@ class TestWholeModelSaving(keras_parameterized.TestCase): os.remove(fname) def test_model_saving_to_new_dir_path(self): - saved_model_dir = self._save_model_dir() - saved_model_dir = os.path.join(saved_model_dir, 'newdir') - saved_model_dir = os.path.join(saved_model_dir, 'saved_model') + saved_model_dir = os.path.join(self._save_model_dir(), + 'newdir', + 'saved_model') save_format = testing_utils.get_save_format() with self.cached_session(): @@ -766,9 +766,8 @@ class TestWholeModelSaving(keras_parameterized.TestCase): model.add(keras.layers.RepeatVector(3)) model.add(keras.layers.TimeDistributed(keras.layers.Dense(3))) - x = np.random.random((1, 3)) - with self.assertRaises(OSError): - with h5py.File(saved_model_path, 'w') as f: + with self.assertRaisesRegex(OSError, 'Unable to create file'): + with h5py.File(saved_model_path, 'w'): keras.models.save_model(model, saved_model_path) From c604d85364963b8d80eb009f01ed4e70177dc785 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Tue, 11 Aug 2020 15:48:16 -0700 Subject: [PATCH 0854/1017] Parallel device: hide device.name, move scoping to the ParallelDevice object Naming is an implementation detail that is expected to change, and having enter/exit on the object itself is a better API. PiperOrigin-RevId: 326118080 Change-Id: Ica7cf901960631c9896a4efcad435bf3aaddde67 --- .../parallel_device/parallel_device.py | 48 ++++++++++++------- .../parallel_device/parallel_device_test.py | 32 ++++++------- 2 files changed, 46 insertions(+), 34 deletions(-) diff --git a/tensorflow/python/distribute/parallel_device/parallel_device.py b/tensorflow/python/distribute/parallel_device/parallel_device.py index 2dbdc653a64..57942652af6 100644 --- a/tensorflow/python/distribute/parallel_device/parallel_device.py +++ b/tensorflow/python/distribute/parallel_device/parallel_device.py @@ -18,7 +18,6 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import contextlib import threading from tensorflow.python import _pywrap_parallel_device @@ -58,14 +57,16 @@ class ParallelDevice(object): with _next_device_number_lock: # TODO(allenl): Better names for parallel devices (right now "CUSTOM" is # special-cased). - self.name = "{}/device:CUSTOM:{}".format( - ctx.host_address_space(), _next_device_number) + self._name = "{}/device:CUSTOM:{}".format(ctx.host_address_space(), + _next_device_number) _next_device_number += 1 device, device_info = _pywrap_parallel_device.GetParallelDeviceCapsules( - self.name, self.components) - context.register_custom_device(device, self.name, device_info) - with ops.device(self.name): + self._name, self.components) + context.register_custom_device(device, self._name, device_info) + with ops.device(self._name): self._device_ids = gen_parallel_device_ops.device_id() + self._device_scope = None + self._saving_scope = None def pack(self, tensors): """Create a tensor on the parallel device from a sequence of tensors. @@ -74,21 +75,21 @@ class ParallelDevice(object): tensors: A flat list of tensors, one per device in `self.components`. Returns: - A single tensor placed on `self.name`. + A single tensor placed on the ParallelDevice. """ - with ops.device(self.name): + with ops.device(self._name): return tpu_ops.tpu_replicated_input(inputs=tensors) def unpack(self, parallel_tensor): """Unpack a parallel tensor into its components. Args: - parallel_tensor: A tensor placed on `self.name`. + parallel_tensor: A tensor placed on the ParallelDevice. Returns: A flat list of tensors, one per `self.components`. """ - with ops.device(self.name): + with ops.device(self._name): return tpu_ops.tpu_replicated_output( parallel_tensor, num_replicas=len(self.components)) @@ -104,12 +105,23 @@ class ParallelDevice(object): """ return self._device_ids - # TODO(allenl): Fixing saving in Python is a bit odd. One alternative would be - # to provide a hook for the custom device to create save specs/etc., then call - # that hook from the default variable implementation if the variable is on a - # custom device. We'll likely want similar hooks for repr() and such. - @contextlib.contextmanager - def scope(self): + def __enter__(self): """Runs ops in parallel, makes variables which save independent buffers.""" - with ops.device(self.name), saving.independent_buffers(self): - yield + if (self._device_scope is not None or self._saving_scope is not None): + raise AssertionError( + "Re-entered a ParallelDevice scope without first exiting it.") + self._device_scope = ops.device(self._name) + self._saving_scope = saving.independent_buffers(self) + self._device_scope.__enter__() + # TODO(allenl): Fixing saving in Python is a bit odd. One alternative would + # be to provide a hook for the custom device to create save specs/etc., then + # call that hook from the default variable implementation if the variable is + # on a custom device. We'll likely want similar hooks for repr() and such. + self._saving_scope.__enter__() + return self + + def __exit__(self, typ, exc, tb): + self._device_scope.__exit__(typ, exc, tb) + self._saving_scope.__exit__(typ, exc, tb) + self._device_scope = None + self._saving_scope = None diff --git a/tensorflow/python/distribute/parallel_device/parallel_device_test.py b/tensorflow/python/distribute/parallel_device/parallel_device_test.py index 780aca993d9..7239dcb2b09 100644 --- a/tensorflow/python/distribute/parallel_device/parallel_device_test.py +++ b/tensorflow/python/distribute/parallel_device/parallel_device_test.py @@ -112,7 +112,7 @@ class _VirtualDeviceTestCase(test.TestCase): class ParallelDeviceTests(_VirtualDeviceTestCase): def test_register_parallel_device(self): - with ops.device(self.device.name): + with self.device: c = constant_op.constant(1.) d = constant_op.constant(2.) e = c + d @@ -129,7 +129,7 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): self.assertIn(self.device.components[1], device_ids[1].backing_device) def test_collective_reduce(self): - with ops.device(self.device.name): + with self.device: x = self.device.pack( [constant_op.constant(-1.5), constant_op.constant(3.5)]) @@ -142,7 +142,7 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): def test_collective_reduce_async_scope(self): # Note that ops on the parallel device currently don't execute # asynchronously. The test is just that we don't get deadlocks. - with context.async_scope(), ops.device(self.device.name): + with context.async_scope(), self.device: x = self.device.pack( [constant_op.constant(-1.5), constant_op.constant(3.5)]) @@ -160,7 +160,7 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): self.setUp() # Note that ops on the parallel device currently don't execute # asynchronously. The test is just that we don't get deadlocks. - with ops.device(self.device.name): + with self.device: x = self.device.pack( [constant_op.constant(-1.5), constant_op.constant(3.5)]) @@ -195,7 +195,7 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): return control_flow_ops.switch_case( device_id, branch_fns={0: send, 1: recv}) - with ops.device(self.device.name): + with self.device: result = broadcast_send_recv(self.device.device_ids) self.assertAllClose([[2], [6]], self.device.unpack(result)) @@ -203,23 +203,23 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): self.skipTest( "Disable saving until SaveableObject's methods are traceable.") prefix = os.path.join(self.get_temp_dir(), "ckpt") - with self.device.scope(): + with self.device: different_values = self.device.pack( [constant_op.constant(-1.), constant_op.constant(3.)]) v = variables.Variable(different_values) checkpoint = tracking.Checkpoint(v=v) save_path = checkpoint.save(prefix) - with ops.device(self.device.name): + with self.device: v.assign(constant_op.constant(0.)) checkpoint.restore(save_path).assert_consumed() - with ops.device(self.device.name): + with self.device: outputs = self.device.unpack(v) self.assertAllClose([-1., 3.], outputs) def _assert_close_to_non_parallel(self, computation): """Asserts that replication of `computation` works and is equivalent.""" - with ops.device(self.device.name): + with self.device: parallel_result = computation() non_parallel_result = computation() # The computations should have the same number and structure of Tensor @@ -230,8 +230,8 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): parallel_flat = nest.flatten(parallel_result) self.assertGreater(len(parallel_flat), 0) for non_parallel, parallel in zip(non_parallel_flat, parallel_flat): - self.assertEqual(self.device.name, parallel.device) - self.assertNotEqual(self.device.name, non_parallel.device) + self.assertEqual(self.device._name, parallel.device) + self.assertNotEqual(self.device._name, non_parallel.device) for parallel_component in self.device.unpack(parallel): self.assertAllClose(non_parallel, parallel_component) @@ -257,7 +257,7 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): class LayerTests(_VirtualDeviceTestCase): def test_layer_forward(self): - with ops.device(self.device.name): + with self.device: layer = _Dense(5) x = constant_op.constant([[2.]]) y = layer(x) @@ -268,7 +268,7 @@ class LayerTests(_VirtualDeviceTestCase): self.assertIn(self.device.components[1], outputs[1].backing_device) # With different Layer inputs we get different outputs - with ops.device(self.device.name): + with self.device: x = self.device.pack( [constant_op.constant([[-0.5]]), constant_op.constant([[0.5]])]) @@ -280,7 +280,7 @@ class LayerTests(_VirtualDeviceTestCase): self.assertIn(self.device.components[1], outputs[1].backing_device) def test_layer_sync_training(self): - with ops.device(self.device.name): + with self.device: layer = _Dense(5) with backprop.GradientTape() as tape: @@ -305,7 +305,7 @@ class LayerTests(_VirtualDeviceTestCase): self.assertIn(self.device.components[1], final_kernels[1].backing_device) def test_layer_divergent_buffer_training(self): - with ops.device(self.device.name): + with self.device: layer = _Dense(5) with backprop.GradientTape() as tape: @@ -339,7 +339,7 @@ class LayerTests(_VirtualDeviceTestCase): manager.restore_or_initialize() for _ in range(10): - with self.device.scope(): + with self.device: with backprop.GradientTape() as tape: x = self.device.pack( [constant_op.constant([[-0.5]]), From 29784fade22771cbe97f6d2bf236fd11a9e8116f Mon Sep 17 00:00:00 2001 From: Yunxing Dai Date: Tue, 11 Aug 2020 15:49:12 -0700 Subject: [PATCH 0855/1017] Support dynamic leading dimension for tensorlist. PiperOrigin-RevId: 326118256 Change-Id: Icea13ee03e23aaac56f17dc5f58bd9182a4ba02a --- .../tf2xla/kernels/tensor_list_ops.cc | 58 +++++++++++++------ .../tf2xla/kernels/tensor_list_utils.cc | 41 +++++++++---- .../tf2xla/kernels/tensor_list_utils.h | 15 +++-- .../compiler/tf2xla/kernels/while_op.cc | 20 ++++++- tensorflow/compiler/tf2xla/xla_op_kernel.cc | 26 +++++++++ tensorflow/compiler/tf2xla/xla_op_kernel.h | 2 +- 6 files changed, 124 insertions(+), 38 deletions(-) diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc index 976ff91f6ce..1ea0e797675 100644 --- a/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_ops.cc @@ -45,22 +45,32 @@ namespace tensorflow { namespace { // GetTensorListDynamicDims collects the dynamic dimensions that a tensorlist -// may carry and returns them in a 2D vector: int64[ElementSize][DimSize]. If a -// dimension is static, a constant dimension is returned. +// may carry and returns them in a 2D vector: XlaOp[ElementSize][DimSize]. If a +// dimension is static, a constant dimension is returned. If a dim is dynamic, a +// dynamic XlaOp representing the dynamic size is returned. xla::StatusOr>> GetTensorListDynamicDims( XlaOpKernelContext* ctx, const xla::Shape& element_shape, const xla::Shape& list_shape, int64 num_elements) { std::vector dynamic_sizes; - ctx->set_dynamic_dimension_is_minus_one(true); // The multiplier can be a dynamic value. TF_RETURN_IF_ERROR(ctx->ConstantInputAsIntVector(0, &dynamic_sizes)); + std::vector dims_are_dynamic; + TF_RETURN_IF_ERROR( + ctx->ResolveInputDynamismIntoPredVector(0, &dims_are_dynamic)); + bool leading_dim_is_dynamic; + TF_RETURN_IF_ERROR( + ctx->ResolveInputDynamismIntoPred(1, &leading_dim_is_dynamic)); std::vector> list_dynamic_dims; // Set dynamic dimension size to 0 for initialization value. std::vector dynamic_dims; - // Leading dim is a static dimension. - dynamic_dims.push_back(xla::ConstantR0(ctx->builder(), num_elements)); + if (leading_dim_is_dynamic) { + dynamic_dims.push_back(ctx->Input(1)); + } else { + dynamic_dims.push_back( + xla::ConstantR0(ctx->builder(), num_elements)); + } for (int64 dim = 0; dim < element_shape.dimensions_size(); ++dim) { - if (ctx->is_dynamic_dimension(dynamic_sizes[dim])) { + if (dims_are_dynamic[dim]) { auto dynamic_dim_size = xla::Slice(ctx->Input(0), {dim}, {dim + 1}, {1}); dynamic_dim_size = xla::Reshape(dynamic_dim_size, {}); dynamic_dim_size = xla::ConvertElementType(dynamic_dim_size, xla::S32); @@ -80,11 +90,12 @@ class TensorListLengthOp : public XlaOpKernel { void Compile(XlaOpKernelContext* ctx) override { int64 leading_dim; - OP_REQUIRES_OK(ctx, - GetLeadingDimForTensorList(ctx->Input(0), &leading_dim)); - Tensor length_tensor(DT_INT32, {}); - length_tensor.scalar()() = static_cast(leading_dim); - ctx->SetConstantOutput(0, length_tensor); + xla::XlaOp leading_dim_size; + bool leading_dim_is_dynamic; + OP_REQUIRES_OK(ctx, GetLeadingDimForTensorList(ctx->Input(0), &leading_dim, + &leading_dim_is_dynamic, + &leading_dim_size)); + ctx->SetOutput(0, leading_dim_size); } private: @@ -134,6 +145,9 @@ class TensorListReserveOp : public XlaOpKernel { void Compile(XlaOpKernelContext* ctx) override { int64 num_elements; OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(1, &num_elements)); + bool num_element_is_dynamic; + OP_REQUIRES_OK( + ctx, ctx->ResolveInputDynamismIntoPred(1, &num_element_is_dynamic)); OP_REQUIRES( ctx, num_elements >= 0, errors::InvalidArgument( @@ -156,7 +170,8 @@ class TensorListReserveOp : public XlaOpKernel { if (got_shape) { xla::Shape list_shape; OP_REQUIRES_OK(ctx, GetTensorListShapeFromElementShape( - element_shape, num_elements, &list_shape)); + element_shape, num_elements, + num_element_is_dynamic, &list_shape)); // Set up dynamic dimension sizes to create the zero tensor. auto list_dynamic_dims_or = GetTensorListDynamicDims( ctx, element_shape, list_shape, num_elements); @@ -175,8 +190,8 @@ class TensorListReserveOp : public XlaOpKernel { return; } - xla::XlaOp result = - BuildUninitializedTensorList(ctx->builder(), num_elements); + xla::XlaOp result = BuildUninitializedTensorList( + ctx->builder(), num_elements, num_element_is_dynamic, ctx->Input(1)); ctx->SetTensorListOutput(0, result); } @@ -200,6 +215,9 @@ class EmptyTensorListOp : public XlaOpKernel { void Compile(XlaOpKernelContext* ctx) override { int64 max_num_elements; OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(1, &max_num_elements)); + bool num_element_is_dynamic; + OP_REQUIRES_OK( + ctx, ctx->ResolveInputDynamismIntoPred(1, &num_element_is_dynamic)); OP_REQUIRES(ctx, max_num_elements >= 0, errors::InvalidArgument( "XLA compilation requires a fixed tensor list size. Set " @@ -210,9 +228,9 @@ class EmptyTensorListOp : public XlaOpKernel { if (dtype_ != DT_VARIANT) { // We are creating a non-nested TensorList. - // If element shape is compile time constant and it's not "unknown rank" - // shape (-1), create an initialized TensorList. Otherwise create an - // uninitialized TensorList. + // If element shape is compile time constant and it's not "unknown + // rank" shape (-1), create an initialized TensorList. Otherwise + // create an uninitialized TensorList. xla::XlaOp element_shape_handle = ctx->Input(0); xla::PrimitiveType type; OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(dtype_, &type)); @@ -224,7 +242,8 @@ class EmptyTensorListOp : public XlaOpKernel { if (got_shape) { xla::Shape list_shape; OP_REQUIRES_OK(ctx, GetTensorListShapeFromElementShape( - element_shape, max_num_elements, &list_shape)); + element_shape, max_num_elements, + num_element_is_dynamic, &list_shape)); // Set up dynamic dimension sizes to create the zero tensor. auto list_dynamic_dims_or = GetTensorListDynamicDims( ctx, element_shape, list_shape, max_num_elements); @@ -243,7 +262,8 @@ class EmptyTensorListOp : public XlaOpKernel { // We are creating a nested TensorList or a non-nested TensorList with // unknown shape. Just create an uninitialized TensorList. xla::XlaOp result = - BuildUninitializedTensorList(ctx->builder(), max_num_elements); + BuildUninitializedTensorList(ctx->builder(), max_num_elements, + num_element_is_dynamic, ctx->Input(1)); ctx->SetTensorListOutput(0, result); } diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.cc b/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.cc index 0e367e10ec4..156f9bfea40 100644 --- a/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.cc +++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.cc @@ -189,28 +189,42 @@ Status SetTensorListPushIndex(xla::XlaOp list, xla::XlaOp push_index, } xla::XlaOp BuildUninitializedTensorList(xla::XlaBuilder* b, - int64 leading_dimension) { + int64 leading_dimension, + bool leading_size_is_dynamic, + xla::XlaOp leading_dim_size) { auto zero = xla::ConstantLiteral(b, xla::LiteralUtil::Zero(xla::PrimitiveType::S32)); - return xla::Broadcast(zero, std::vector{leading_dimension}); + auto broadcast = xla::Broadcast(zero, std::vector{leading_dimension}); + if (leading_size_is_dynamic) { + return xla::SetDimensionSize(broadcast, leading_dim_size, 0); + } else { + return broadcast; + } } -Status GetLeadingDimForTensorList(xla::XlaOp list, int64* leading_dim) { +Status GetLeadingDimForTensorList(xla::XlaOp list, int64* leading_dim, + bool* leading_dim_is_dynamic, + xla::XlaOp* leading_dim_dynamic_size) { bool is_initialized; TF_RETURN_IF_ERROR(IsTensorListInitialized(list, &is_initialized)); TF_ASSIGN_OR_RETURN(xla::Shape list_shape, list.builder()->GetShape(list)); if (is_initialized) { auto buffer_shape = xla::ShapeUtil::GetTupleElementShape(list_shape, 0); + *leading_dim_is_dynamic = buffer_shape.is_dynamic_dimension(0); + auto buffer = xla::GetTupleElement(list, 0); *leading_dim = buffer_shape.dimensions(0); + *leading_dim_dynamic_size = xla::GetDimensionSize(buffer, 0); } else { + *leading_dim_is_dynamic = list_shape.is_dynamic_dimension(0); *leading_dim = list_shape.dimensions(0); + *leading_dim_dynamic_size = xla::GetDimensionSize(list, 0); } return Status::OK(); } Status GetTensorListShapeFromElementTensorListShape( const xla::Shape& element_tensor_list_shape, int64 leading_dim, - xla::Shape* tensor_list_shape) { + bool leading_dim_is_dynamic, xla::Shape* tensor_list_shape) { std::vector shapes; int tuple_size = xla::ShapeUtil::TupleElementCount(element_tensor_list_shape); for (int i = 0; i < tuple_size; i++) { @@ -220,6 +234,9 @@ Status GetTensorListShapeFromElementTensorListShape( dimensions.insert(dimensions.begin(), leading_dim); shapes.push_back( xla::ShapeUtil::MakeShape(shape.element_type(), dimensions)); + if (leading_dim_is_dynamic) { + shapes.back().set_dynamic_dimension(0, true); + } } shapes.push_back( xla::ShapeUtil::MakeShape(xla::PrimitiveType::S32, std::vector{})); @@ -229,6 +246,7 @@ Status GetTensorListShapeFromElementTensorListShape( Status GetTensorListShapeFromElementShape(const xla::Shape& element_shape, int64 leading_dim, + bool leading_dim_is_dynamic, xla::Shape* tensor_list_shape) { if (!element_shape.IsArray()) { return errors::InvalidArgument( @@ -236,12 +254,12 @@ Status GetTensorListShapeFromElementShape(const xla::Shape& element_shape, "shape. But element shape is ", element_shape.DebugString()); } - std::vector shapes; std::vector dimensions = xla::SpanToVector(element_shape.dimensions()); dimensions.insert(dimensions.begin(), leading_dim); shapes.push_back( xla::ShapeUtil::MakeShape(element_shape.element_type(), dimensions)); + shapes.back().set_dynamic_dimension(0, leading_dim_is_dynamic); shapes.push_back( xla::ShapeUtil::MakeShape(xla::PrimitiveType::S32, std::vector{})); *tensor_list_shape = xla::ShapeUtil::MakeTupleShape(shapes); @@ -279,7 +297,10 @@ Status GetInitializedTensorListForElement(xla::XlaOp list, xla::XlaOp element, bool element_is_tensor_list, xla::XlaOp* initialized_list) { int64 leading_dim; - TF_RETURN_IF_ERROR(GetLeadingDimForTensorList(list, &leading_dim)); + xla::XlaOp leading_dim_dynamic_size; + bool leading_dim_is_dynamic; + TF_RETURN_IF_ERROR(GetLeadingDimForTensorList( + list, &leading_dim, &leading_dim_is_dynamic, &leading_dim_dynamic_size)); xla::XlaBuilder* b = list.builder(); xla::Shape list_shape; @@ -287,12 +308,11 @@ Status GetInitializedTensorListForElement(xla::XlaOp list, xla::XlaOp element, if (element_is_tensor_list) { TF_RETURN_IF_ERROR(GetTensorListShapeFromElementTensorListShape( - element_shape, leading_dim, &list_shape)); + element_shape, leading_dim, leading_dim_is_dynamic, &list_shape)); } else { TF_RETURN_IF_ERROR(GetTensorListShapeFromElementShape( - element_shape, leading_dim, &list_shape)); + element_shape, leading_dim, leading_dim_is_dynamic, &list_shape)); } - bool is_initialized; TF_RETURN_IF_ERROR(IsTensorListInitialized(list, &is_initialized)); if (is_initialized) { @@ -312,8 +332,7 @@ Status GetInitializedTensorListForElement(xla::XlaOp list, xla::XlaOp element, for (int64 i = 0; i < list_shape.tuple_shapes_size() - 1; ++i) { std::vector dynamic_dims; const xla::Shape& shape = list_shape.tuple_shapes(i); - // Leading dim is a static dimension. - dynamic_dims.push_back(xla::ConstantR0(b, leading_dim)); + dynamic_dims.push_back(leading_dim_dynamic_size); xla::XlaOp sub_element; if (element_is_tensor_list) { sub_element = xla::GetTupleElement(element, i); diff --git a/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.h b/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.h index ef3c8badf71..549ccd5aece 100644 --- a/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.h +++ b/tensorflow/compiler/tf2xla/kernels/tensor_list_utils.h @@ -60,17 +60,22 @@ Status SetTensorListPushIndex(xla::XlaOp list, xla::XlaOp push_index, // Returns an uninitialized TensorList. xla::XlaOp BuildUninitializedTensorList(xla::XlaBuilder* b, - int64 leading_dimension); + int64 leading_dimension, + bool leading_size_is_dynamic, + xla::XlaOp leading_dim_size); -// Returns leading dimension for the TensorList. -// Input can be initialized or uninitialized TensorList. -// Non-nested and nested TensorLists are both supported. -Status GetLeadingDimForTensorList(xla::XlaOp list, int64* leading_dim); +// Returns leading dimension for the TensorList as well as a dynamic op +// representing the dynamic size. Input can be initialized or uninitialized +// TensorList. Non-nested and nested TensorLists are both supported. +Status GetLeadingDimForTensorList(xla::XlaOp list, int64* leading_dim, + bool* leading_dim_is_dynamic, + xla::XlaOp* leading_dim_dynamic_size); // Returns TensorList shape for the element shape. // Element shape must be a normal tensor shape. Status GetTensorListShapeFromElementShape(const xla::Shape& element_shape, int64 leading_dim, + bool leading_dim_is_dynamic, xla::Shape* tensor_list_shape); // Returns a TensorList filled by zeros with the given shape. diff --git a/tensorflow/compiler/tf2xla/kernels/while_op.cc b/tensorflow/compiler/tf2xla/kernels/while_op.cc index fe7a5898011..a94411f1b30 100644 --- a/tensorflow/compiler/tf2xla/kernels/while_op.cc +++ b/tensorflow/compiler/tf2xla/kernels/while_op.cc @@ -513,10 +513,26 @@ void XlaWhileOp::Compile(XlaOpKernelContext* ctx) { // Prepare dynamic dimensions for element shapes. std::vector> list_dynamic_dims; for (int64 i = 0; i < list_shape.tuple_shapes_size() - 1; ++i) { - // Set dynamic dimension size to 0 for initilization value. std::vector dynamic_dims; + const xla::Shape& shape = list_shape.tuple_shapes(i); - for (int64 dim = 0; dim < shape.dimensions_size(); ++dim) { + + // We already have the dynamic size of leading dimension outside of + // the while loop without initializing the TensorList inside the while + // loop. + if (shape.is_dynamic_dimension(0)) { + xla::XlaOp leading_dim_size = xla::GetDimensionSize(input, 0); + dynamic_dims.push_back(leading_dim_size); + } else { + int32 dim_size = shape.dimensions(0); + dynamic_dims.push_back( + xla::ConstantR0(ctx->builder(), dim_size)); + } + + // Set dynamic dimension size to 0 for element value. Inside the while + // loop, TensorlistSetItem will properly set the element shape's + // dynamic diemnsion. + for (int64 dim = 1; dim < shape.dimensions_size(); ++dim) { int32 dim_size = shape.dimensions(dim); if (shape.is_dynamic_dimension(dim)) { dim_size = 0; diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.cc b/tensorflow/compiler/tf2xla/xla_op_kernel.cc index 07537546d52..c2d1906e47a 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.cc +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.cc @@ -259,6 +259,32 @@ static Status LiteralToPredVector(const xla::LiteralSlice& literal, return Status::OK(); } +Status XlaOpKernelContext::ResolveInputDynamismIntoPred(int index, bool* out) { + xla::Literal literal; + XlaExpression e = InputExpression(index); + auto* client = compiler() ? compiler()->client() : nullptr; + xla::StatusOr dynamism_or_status = e.ResolveDynamism(client); + if (!dynamism_or_status.ok()) { + Status status = dynamism_or_status.status(); + errors::AppendToMessage(&status, "while evaluating input dynamism", index, + " of ", context_->op_kernel().type_string()); + return status; + } + Tensor dynamism = dynamism_or_status.ValueOrDie(); + + Tensor temp(dynamism.dtype()); + TensorShape tensor_shape({}); + if (!temp.CopyFrom(dynamism, tensor_shape)) { + return errors::InvalidArgument( + context_->op_kernel().name(), " input ", index, " has shape ", + dynamism.shape().DebugString(), " which is not a R0 ", tensor_shape); + } + + TF_ASSIGN_OR_RETURN(literal, HostTensorToLiteral(temp)); + *out = literal.Get({}); + return Status::OK(); +} + Status XlaOpKernelContext::ResolveInputDynamismIntoPredVector( int index, std::vector* out) { xla::Literal literal; diff --git a/tensorflow/compiler/tf2xla/xla_op_kernel.h b/tensorflow/compiler/tf2xla/xla_op_kernel.h index 75c3e60171a..1ed343ba20f 100644 --- a/tensorflow/compiler/tf2xla/xla_op_kernel.h +++ b/tensorflow/compiler/tf2xla/xla_op_kernel.h @@ -119,7 +119,7 @@ class XlaOpKernelContext { // Evaluates input and returns their dynamism vector in a vector of // predicates. Status ResolveInputDynamismIntoPredVector(int index, std::vector* out); - + Status ResolveInputDynamismIntoPred(int index, bool* out); // Helper methods for constant inputs. // Evaluates input `index` and stores it in `*constant_literal`. If the From a7e4df924652a151677b3e0b95609db27cb04631 Mon Sep 17 00:00:00 2001 From: Daniel Nguyen Date: Tue, 11 Aug 2020 21:01:09 +0000 Subject: [PATCH 0856/1017] switched from unsigned char to TF_Bool. Added input checking for TF_AllocatorAttributes --- tensorflow/c/BUILD | 1 + tensorflow/c/c_api_macros.h | 6 ++++++ tensorflow/c/kernels.cc | 14 ++++++++++---- tensorflow/c/kernels.h | 2 +- tensorflow/c/kernels_test.cc | 7 +++---- tensorflow/c/tf_tensor.h | 2 +- 6 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD index c4c0420aaec..7b78fdb4cf6 100644 --- a/tensorflow/c/BUILD +++ b/tensorflow/c/BUILD @@ -81,6 +81,7 @@ tf_cuda_library( hdrs = [ "c_api.h", "c_api_internal.h", + "c_api_macros.h", "tf_datatype.h", "tf_tensor.h", "tf_tstring.h", diff --git a/tensorflow/c/c_api_macros.h b/tensorflow/c/c_api_macros.h index ce24e4d8cbd..58547a57b6e 100644 --- a/tensorflow/c/c_api_macros.h +++ b/tensorflow/c/c_api_macros.h @@ -30,6 +30,12 @@ limitations under the License. #endif // _WIN32 #endif // SWIG +// TF_Bool is the C API typedef for unsigned char, while TF_BOOL is +// the datatype for boolean tensors. +#ifndef TF_BOOL_DEFINED +#define TF_Bool unsigned char +#endif // TF_BOOL_DEFINED + // Macro used to calculate struct size for maintaining ABI stability across // different struct implementations. #ifndef TF_OFFSET_OF_END diff --git a/tensorflow/c/kernels.cc b/tensorflow/c/kernels.cc index 6d9d7d57517..cce6f2c494d 100644 --- a/tensorflow/c/kernels.cc +++ b/tensorflow/c/kernels.cc @@ -261,7 +261,6 @@ TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, int index, size_t len, TF_Status* status) { TF_SetStatus(status, TF_OK, ""); auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); - static_assert(sizeof(int64_t) == sizeof(tensorflow::int64), "64-bit int types should match in size"); tensorflow::gtl::ArraySlice dimarray( @@ -286,22 +285,29 @@ TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, TF_AllocatorAttributes* attributes, TF_Status* status) { auto* cc_ctx = reinterpret_cast<::tensorflow::OpKernelContext*>(context); - TF_SetStatus(status, TF_OK, ""); + TF_SetStatus(status, TF_OK, ""); + static_assert(sizeof(int64_t) == sizeof(tensorflow::int64), + "64-bit int types should match in size"); tensorflow::gtl::ArraySlice dimarray( reinterpret_cast(dims), num_dims); + if (attributes && !attributes->struct_size) { + TF_SetStatus(status, TF_INVALID_ARGUMENT, "TF_AllocatorAttributes struct " + "size member must be set to TF_ALLOCATOR_ATTRIBUTES_STRUCT_SIZE"); + return nullptr; + } tensorflow::AllocatorAttributes allocator_attr; - if (attributes->on_host) { + if (attributes && attributes->on_host) { allocator_attr.set_on_host(true); } tensorflow::Status s; tensorflow::Tensor tensor; - TF_Tensor* tf_tensor; s = cc_ctx->allocate_temp(static_cast(dtype), tensorflow::TensorShape(dimarray), &tensor, allocator_attr); if (!s.ok()) { ::tensorflow::Set_TF_Status_from_Status(status, s); return nullptr; } + TF_Tensor* tf_tensor; tf_tensor = TF_TensorFromTensor(tensor, &s); if (!s.ok()) { ::tensorflow::Set_TF_Status_from_Status(status, s); diff --git a/tensorflow/c/kernels.h b/tensorflow/c/kernels.h index ee865613b6e..0be31b76839 100644 --- a/tensorflow/c/kernels.h +++ b/tensorflow/c/kernels.h @@ -203,7 +203,7 @@ TF_CAPI_EXPORT TF_Tensor* TF_AllocateOutput(TF_OpKernelContext* context, // Allocates a temporary Tensor of the specified type and shape. The // Tensor must not be used after kernel construction is // complete. - +// // num_dims must equal the size of array dims TF_CAPI_EXPORT extern TF_Tensor* TF_AllocateTemp(TF_OpKernelContext* context, TF_DataType dtype, int64_t* dims, int num_dims, TF_AllocatorAttributes* diff --git a/tensorflow/c/kernels_test.cc b/tensorflow/c/kernels_test.cc index e216b85c13f..7663a3570eb 100644 --- a/tensorflow/c/kernels_test.cc +++ b/tensorflow/c/kernels_test.cc @@ -368,13 +368,12 @@ class DeviceKernelOpTest : public OpsTestBase { #endif }; -// Helper function for tests that validates that the tensor has -// shape and type corresponding to dims and dtype. +// Validates that the tensor has shape and type corresponding to +// dims and dtype. void validate_tensor(TF_Tensor* tensor, int64_t* dims, int64_t num_dims, TF_DataType dtype); -// Helper function for tests that copies data of length -// tensor_size_bytes from values to tensor +// Copies data of length tensor_size_bytes from values to tensor. template void set_tensor_data(TF_Tensor* tensor, T* values, size_t tensor_size_bytes, TF_OpKernelContext* ctx); diff --git a/tensorflow/c/tf_tensor.h b/tensorflow/c/tf_tensor.h index ced57df77d4..1faa368f25a 100644 --- a/tensorflow/c/tf_tensor.h +++ b/tensorflow/c/tf_tensor.h @@ -50,7 +50,7 @@ extern "C" { typedef struct TF_AllocatorAttributes { size_t struct_size; // Set boolean to 1 for CPU allocation, else 0. - unsigned char on_host; + TF_Bool on_host; } TF_AllocatorAttributes; From b297140e1a1f0e916d0f60e743e7763745608b45 Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Tue, 11 Aug 2020 16:05:01 -0700 Subject: [PATCH 0857/1017] Allow NdarraySpec to be written in saved model. PiperOrigin-RevId: 326121293 Change-Id: I7a4351a9ab3e0381ff5616f67d0e61880f3bb649 --- tensorflow/core/protobuf/struct.proto | 1 + tensorflow/python/saved_model/BUILD | 1 + tensorflow/python/saved_model/load_test.py | 30 +++++++++++++++++++ .../saved_model/nested_structure_coder.py | 3 ++ .../nested_structure_coder_test.py | 9 ++++++ 5 files changed, 44 insertions(+) diff --git a/tensorflow/core/protobuf/struct.proto b/tensorflow/core/protobuf/struct.proto index ee0f089f2a3..c99eab5dd88 100644 --- a/tensorflow/core/protobuf/struct.proto +++ b/tensorflow/core/protobuf/struct.proto @@ -136,6 +136,7 @@ message TypeSpecProto { PER_REPLICA_SPEC = 8; // PerReplicaSpec from distribute/values.py VARIABLE_SPEC = 9; // tf.VariableSpec ROW_PARTITION_SPEC = 10; // RowPartitionSpec from ragged/row_partition.py + NDARRAY_SPEC = 11; // TF Numpy NDarray spec } TypeSpecClass type_spec_class = 1; diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD index 45ee73de51c..4507118c17c 100644 --- a/tensorflow/python/saved_model/BUILD +++ b/tensorflow/python/saved_model/BUILD @@ -587,6 +587,7 @@ py_strict_library( "//tensorflow/python/data/ops:iterator_ops", "//tensorflow/python/data/ops:optional_ops", "//tensorflow/python/distribute:values", + "//tensorflow/python/ops/numpy_ops:numpy", "//tensorflow/python/ops/ragged:ragged_tensor", "//tensorflow/python/ops/ragged:row_partition", "@six_archive//:six", diff --git a/tensorflow/python/saved_model/load_test.py b/tensorflow/python/saved_model/load_test.py index 320182385f8..54124df6eba 100644 --- a/tensorflow/python/saved_model/load_test.py +++ b/tensorflow/python/saved_model/load_test.py @@ -50,9 +50,11 @@ from tensorflow.python.ops import cond_v2 from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import lookup_ops from tensorflow.python.ops import math_ops +from tensorflow.python.ops import numpy_ops as tnp from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope from tensorflow.python.ops import variables +from tensorflow.python.ops.numpy_ops import np_arrays from tensorflow.python.ops.ragged import ragged_factory_ops from tensorflow.python.ops.ragged import ragged_tensor from tensorflow.python.saved_model import load @@ -1810,6 +1812,34 @@ class LoadTest(test.TestCase, parameterized.TestCase): self.assertEqual(self.evaluate(imported.lookup("foo")), 15) self.assertEqual(self.evaluate(imported.lookup("idk")), -1) + def test_saving_ndarray_specs(self, cycles): + class NdarrayModule(module.Module): + + @def_function.function + def plain(self, x): + return tnp.add(x, 1) + + @def_function.function(input_signature=[ + np_arrays.NdarraySpec(tensor_spec.TensorSpec([], dtypes.float32))]) + def with_signature(self, x): + return tnp.add(x, 1) + + m = NdarrayModule() + c = tnp.asarray(3.0, tnp.float32) + output_plain, output_with_signature = m.plain(c), m.with_signature(c) + + loaded_m = cycle(m, cycles) + + load_output_plain, load_output_with_signature = ( + loaded_m.plain(c), loaded_m.with_signature(c)) + + self.assertIsInstance(output_plain, tnp.ndarray) + self.assertIsInstance(load_output_plain, tnp.ndarray) + self.assertIsInstance(output_with_signature, tnp.ndarray) + self.assertIsInstance(load_output_with_signature, tnp.ndarray) + self.assertAllClose(output_plain, load_output_plain) + self.assertAllClose(output_with_signature, load_output_with_signature) + class SingleCycleTests(test.TestCase, parameterized.TestCase): diff --git a/tensorflow/python/saved_model/nested_structure_coder.py b/tensorflow/python/saved_model/nested_structure_coder.py index 9c71b853675..a7e5548ee06 100644 --- a/tensorflow/python/saved_model/nested_structure_coder.py +++ b/tensorflow/python/saved_model/nested_structure_coder.py @@ -48,6 +48,7 @@ from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import tensor_util from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import tensor_array_ops +from tensorflow.python.ops.numpy_ops import np_arrays from tensorflow.python.ops.ragged import ragged_tensor from tensorflow.python.ops.ragged import row_partition from tensorflow.python.util import compat @@ -516,6 +517,8 @@ class _TypeSpecCodec(object): resource_variable_ops.VariableSpec, struct_pb2.TypeSpecProto.ROW_PARTITION_SPEC: row_partition.RowPartitionSpec, + struct_pb2.TypeSpecProto.NDARRAY_SPEC: + np_arrays.NdarraySpec, } # Mapping from type (TypeSpec subclass) to enum value. diff --git a/tensorflow/python/saved_model/nested_structure_coder_test.py b/tensorflow/python/saved_model/nested_structure_coder_test.py index 9951ea64a49..fb074f76eb0 100644 --- a/tensorflow/python/saved_model/nested_structure_coder_test.py +++ b/tensorflow/python/saved_model/nested_structure_coder_test.py @@ -28,6 +28,7 @@ from tensorflow.python.framework import sparse_tensor from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import tensor_util +from tensorflow.python.ops.numpy_ops import np_arrays from tensorflow.python.ops.ragged import ragged_tensor from tensorflow.python.platform import test from tensorflow.python.saved_model import nested_structure_coder @@ -331,6 +332,14 @@ class NestedStructureTest(test.TestCase): decoded = self._coder.decode_proto(encoded) self.assertEqual(structure, decoded) + def testEncodeDecodeNdarraySpec(self): + structure = [np_arrays.NdarraySpec( + tensor_spec.TensorSpec([4, 2], dtypes.float32))] + self.assertTrue(self._coder.can_encode(structure)) + encoded = self._coder.encode_structure(structure) + decoded = self._coder.decode_proto(encoded) + self.assertEqual(structure, decoded) + def testNotEncodable(self): class NotEncodable(object): From 672041b9524a30f6c567b98b2ebf12f032b8a7fd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 16:07:52 -0700 Subject: [PATCH 0858/1017] [XLA:SPMD] Reshard tile to partial replicate. PiperOrigin-RevId: 326121818 Change-Id: I5e26a7c93d5bb54841df0b7da31708e754f8e4a2 --- .../xla/service/spmd/spmd_partitioner.cc | 61 ++++++++- .../xla/service/spmd/spmd_partitioner_util.cc | 118 +++++++++++++++++- .../xla/service/spmd/spmd_partitioner_util.h | 14 +++ 3 files changed, 186 insertions(+), 7 deletions(-) diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc index 8006e47d90d..343239983b0 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc @@ -353,6 +353,65 @@ PartitionedHlo PartitionedHlo::ReshardNoCache(const HloSharding& target) { } } + // Tiled to partial replicate + if (!sharding().ReplicateOnLastTileDim() && !sharding().IsTileMaximal() && + target.ReplicateOnLastTileDim()) { + // Get the comptible sharding to target with resharding by all reduce. + auto compatible_sharding = PartialReplicateToTileCompatibleSharding( + target, sharding().tile_assignment().dimensions()); + if (compatible_sharding.has_value()) { + auto temp_sharding = compatible_sharding.value(); + auto partitioned_hlo = *this; + // Use collective permute to adjust device assignment if needed. + if (CanReshardWithCollectivePermute(sharding(), temp_sharding)) { + partitioned_hlo = + partitioned_hlo.ReshardWithCollectivePermute(temp_sharding); + } + + // Get replicate dims and replicate factor of each dimensions. + int64 rank = hlo_->shape().rank(); + std::vector replicate_dims; + std::vector replicate_factors; + for (int64 dim = 0; dim < rank; dim++) { + int64 replicate_factor = temp_sharding.tile_assignment().dim(dim) / + target.tile_assignment().dim(dim); + if (replicate_factor > 1) { + replicate_dims.emplace_back(dim); + replicate_factors.emplace_back(replicate_factor); + } + } + + // Do left halo exchange if all-reduce directly will remove useful data + // from the source. + auto halo_exchange = TileToPartialReplicateHaloExchange( + partitioned_hlo.hlo_, base_shape_, temp_sharding, target, + replicate_dims, partitioned_hlo.state().collective_ops_creator, + partitioned_hlo.state().next_channel_id, + partitioned_hlo.state().partition_id, partitioned_hlo.state().b); + if (halo_exchange.has_value()) { + auto halo_exchange_hlo = halo_exchange.value(); + // Grouped on replicate dimensions. + auto sharding_grouped = GroupShardingOnDims( + temp_sharding, replicate_dims, replicate_factors); + auto per_group_partitioner_state = CreatePerGroupPartitioningState( + partitioned_hlo.state(), sharding_grouped.device_groups, + partitioned_hlo.state().b); + auto base_shape = MakePartitionedShape(base_shape_, target); + // It's possible that halo_exchange_hlo == hlo.hlo(). + // Record the sharding of hlo here, and reset it before return. + auto original_sharding = partitioned_hlo.sharding(); + halo_exchange_hlo->set_sharding(sharding_grouped.sharding); + auto partial_replicate_hlo = PartitionedHlo( + halo_exchange_hlo, base_shape, per_group_partitioner_state); + HloInstruction* result = + partial_replicate_hlo.ReplicatePartial(replicate_dims); + partitioned_hlo.hlo()->set_sharding(original_sharding); + result->set_sharding(target); + return PartitionedHlo(result, base_shape_, partitioned_hlo.state()); + } + } + } + // If not replicated yet, first replicate and then reshard to use one of the // two implementations below. if (!sharding().IsReplicated()) { @@ -808,7 +867,7 @@ HloInstruction* PartitionedHlo::ReplicatePartial(absl::Span dims) { std::vector strides(target_shape.rank(), 1); result = state_.b->AddInstruction( HloInstruction::CreateSlice(target_shape, result, start_indices, - base_shape_.dimensions(), strides)); + target_shape.dimensions(), strides)); } return result; } diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc index 3443c6e013d..68e486afa83 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc @@ -334,10 +334,11 @@ absl::optional PartialReplicateToTileCompatibleSharding( reshape_dimensions.insert(reshape_dimensions.end(), expand_tile_sizes.begin(), expand_tile_sizes.end()); auto reshape_tile_assignment = partial_sharding.tile_assignment(); + reshape_tile_assignment.Reshape(reshape_dimensions); // Transpose. std::vector perm; - perm.reserve(rank); + perm.reserve(rank + expand_tile_sizes.size()); for (int64 dim = 0; dim < rank; dim++) { perm.emplace_back(dim); if (expand_tile_dims_indices[dim] > -1) { @@ -354,6 +355,100 @@ absl::optional PartialReplicateToTileCompatibleSharding( return HloSharding::Tile(transpose_tile_assignment); } +absl::optional TileToPartialReplicateHaloExchange( + HloInstruction* hlo, const Shape& base_shape, + const HloSharding& src_sharding, const HloSharding& dst_sharding, + const std::vector& replicate_dims, + const SPMDCollectiveOpsCreator& collective_ops_creator, + int64* next_channel_id, HloInstruction* partition_id, SpmdBuilder* b) { + // Source is tile sharding. + auto padded_src_shape = + GetPaddedShapeForUnevenPartitioning(base_shape, src_sharding); + // Target is partial replicate. + auto padded_dst_shape = + GetPaddedShapeForUnevenPartitioning(base_shape, dst_sharding); + if (ShapeUtil::Compatible(padded_dst_shape, hlo->shape())) { + return hlo; + } + + auto partition_ordinals = + MakeTiledPartitionOrdinals(dst_sharding, partition_id, b); + + auto result = hlo; + auto hlo_shape = hlo->shape(); + for (auto dim : replicate_dims) { + int64 dst_shard_count = dst_sharding.tile_assignment().dim(dim); + int64 src_per_shard_size = + padded_src_shape.dimensions(dim) / dst_shard_count; + // Calculate per shard size using the sharding to compare if dst_sharding + // needs more padding at the end. + int64 dst_per_shard_size = + padded_dst_shape.dimensions(dim) / dst_shard_count; + + // If src per shard doesn't have redudant data. + if (src_per_shard_size <= dst_per_shard_size || dst_shard_count == 1) { + continue; + } + + // If src_per_shard * replicate_factor > dst_per_shard , need to + // re-distribute the data between each shard using collective permute. For + // example, if dimension size is 6 and shard 4 ways in the src but needs to + // shard 2 ways in the dst. 4 way sharding has 2 element in each shard, + // while 2 way sharding has 3 elements, the last element in the first shard + // will be sliced out. re-distribution is needed. + // + // 1. Calculate left_halo size. + // left-halo size is + // (src_per_shard_size - dst_per_shard_size) * i / replicate_factor + int64 replicate_factor = src_sharding.tile_assignment().dim(dim) / + dst_sharding.tile_assignment().dim(dim); + OffsetCalculation left_halo_size_function = + OffsetCalculation(MultiplyAddDivideOffsetCalculation( + src_per_shard_size - dst_per_shard_size, 0, replicate_factor)); + + // 2. Calculate right_halo size. + // right-halo size is 0 + OffsetCalculation right_halo_size_function = + OffsetCalculation(MultiplyAddDivideOffsetCalculation(0, 0, 1)); + + auto concat = result; + // 3. Halo exchange. + auto halo_exchange_result = ExchangeHalo( + result, left_halo_size_function, right_halo_size_function, dim, + src_sharding, collective_ops_creator, next_channel_id, b); + + if (halo_exchange_result.has_value()) { + concat = halo_exchange_result.value(); + } else { + return absl::nullopt; + } + + // 4. Slice the valid result. + // Slice offset is + // (dst_shard_count - i - 1) * + // (src_per_shard_size - dst_per_shard_size) + // i is the index in dst_sharindg. + auto zero_s32 = b->AddInstruction( + HloInstruction::CreateConstant(LiteralUtil::Zero(S32))); + OffsetCalculation start_offset_on_padded_concat_calculation = + OffsetCalculation(MultiplyAddDivideOffsetCalculation( + dst_per_shard_size - src_per_shard_size, + (src_per_shard_size - dst_per_shard_size) * (dst_shard_count - 1), + 1)); + auto slice_shape = concat->shape(); + slice_shape.set_dimensions(dim, + padded_src_shape.dimensions(dim) / + src_sharding.tile_assignment().dim(dim)); + std::vector slice_offsets(concat->shape().rank(), + zero_s32); + slice_offsets[dim] = start_offset_on_padded_concat_calculation.Calculate( + partition_ordinals[dim], b); + result = b->AddInstruction(HloInstruction::CreateDynamicSlice( + slice_shape, concat, slice_offsets, slice_shape.dimensions())); + } + return result; +} + absl::optional PadFromPartialReplicateShape( HloInstruction* hlo, const Shape& base_shape, const HloSharding& src_sharding, const HloSharding& dst_sharding, @@ -1228,21 +1323,32 @@ bool CanReshardWithCollectivePermute(const HloSharding& source, GroupedSharding GroupShardingOnDims(const HloSharding& sharding, absl::Span group_dims) { + std::vector group_dim_shards(group_dims.size(), 1); + return GroupShardingOnDims(sharding, group_dims, group_dim_shards); +} + +GroupedSharding GroupShardingOnDims(const HloSharding& sharding, + absl::Span group_dims, + absl::Span group_dim_shards) { CHECK(!sharding.IsTileMaximal()); std::vector grouped_tiling_dims = sharding.tile_assignment().dimensions(); std::vector group_dim_sizes(group_dims.size()); for (int64 i = 0; i < group_dims.size(); ++i) { - group_dim_sizes[i] = grouped_tiling_dims[group_dims[i]]; - grouped_tiling_dims[group_dims[i]] = 1; + CHECK_GE(grouped_tiling_dims[group_dims[i]], group_dim_shards[i]); + group_dim_sizes[i] = + grouped_tiling_dims[group_dims[i]] / group_dim_shards[i]; + grouped_tiling_dims[group_dims[i]] = group_dim_shards[i]; } + std::vector> device_groups(Product(group_dim_sizes)); sharding.tile_assignment().Each( [&](absl::Span indices, int64 device) { int64 group_id = 0; - for (int64 dim : group_dims) { - group_id *= sharding.tile_assignment().dim(dim); - group_id += indices[dim]; + for (int64 i = 0; i < group_dims.size(); ++i) { + group_id *= sharding.tile_assignment().dim(group_dims[i]) / + group_dim_shards[i]; + group_id += indices[group_dims[i]] / group_dim_shards[i]; } device_groups[group_id].push_back(device); }); diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h index 6906b52ca79..fab573c5111 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h @@ -304,6 +304,11 @@ struct GroupedSharding { HloSharding sharding; }; +// Creates a GroupedSharding for a tiled sharding with group dim shard sizes. +GroupedSharding GroupShardingOnDims(const HloSharding& sharding, + absl::Span group_dims, + absl::Span group_dim_shards); + // Creates a GroupedSharding for a tiled sharding. GroupedSharding GroupShardingOnDims(const HloSharding& sharding, absl::Span group_dims); @@ -371,6 +376,15 @@ absl::optional PartialReplicateToTileCompatibleSharding( const HloSharding& partial_sharding, const std::vector& target_tile_dims); +// Do left halo exchange if all-reduce directly from tile sharding to partial +// replicate sharding will remove useful data from the source. +absl::optional TileToPartialReplicateHaloExchange( + HloInstruction* hlo, const Shape& base_shape, + const HloSharding& src_sharding, const HloSharding& dst_sharding, + const std::vector& replicate_dims, + const SPMDCollectiveOpsCreator& collective_ops_creator, + int64* next_channel_id, HloInstruction* partition_id, SpmdBuilder* b); + } // namespace spmd } // namespace xla From 518d3339772433c790bff0bb70fb8b57d0fdc082 Mon Sep 17 00:00:00 2001 From: Jose Baiocchi Date: Tue, 11 Aug 2020 16:08:06 -0700 Subject: [PATCH 0859/1017] Add XStats to XEventMetadata PiperOrigin-RevId: 326121854 Change-Id: I0c00f7ea095dab412afeaa9a1213c2ee4c44235f --- tensorflow/core/profiler/protobuf/xplane.proto | 15 ++++++++++++--- tensorflow/core/profiler/utils/xplane_visitor.h | 7 +++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/profiler/protobuf/xplane.proto b/tensorflow/core/profiler/protobuf/xplane.proto index 48aa38dafff..dd34c2f40b1 100644 --- a/tensorflow/core/profiler/protobuf/xplane.proto +++ b/tensorflow/core/profiler/protobuf/xplane.proto @@ -36,6 +36,7 @@ message XPlane { map stat_metadata = 5; // XStats associated with this plane, e.g. device capabilities. + // Each of these XStats should have a different metadata_id. repeated XStat stats = 6; } @@ -89,6 +90,7 @@ message XEvent { int64 duration_ps = 3; // XStats associated with the event. + // Each of these XStats should have a different metadata_id. repeated XStat stats = 4; } @@ -112,8 +114,9 @@ message XStat { } } -// Metadata for an XEvent, shared by all instances of the same event. -// Next ID: 5 +// Metadata for an XEvent, corresponds to an event type and is shared by +// all XEvents with the same metadata_id. +// Next ID: 6 message XEventMetadata { // XPlane.event_metadata map key. int64 id = 1; @@ -126,15 +129,21 @@ message XEventMetadata { // Additional metadata in serialized format. bytes metadata = 3; + + // XStats that are constant for all XEvents with the same metadata_id. + // Each of these XStats should have a different metadata_id. + repeated XStat stats = 5; } -// Metadata for an XStat, shared by all instances of the same stat. +// Metadata for an XStat, corresponds to a stat type and is shared by all +// XStats with the same metadata_id. // Next ID: 4 message XStatMetadata { // XPlane.stat_metadata map key. int64 id = 1; // Name of the stat (should be short). + // Two XStatMetadata with different id should have different names. string name = 2; // Description of the stat (might be long). diff --git a/tensorflow/core/profiler/utils/xplane_visitor.h b/tensorflow/core/profiler/utils/xplane_visitor.h index 6605bdf5658..f0e4204e4d3 100644 --- a/tensorflow/core/profiler/utils/xplane_visitor.h +++ b/tensorflow/core/profiler/utils/xplane_visitor.h @@ -101,11 +101,18 @@ class XStatsOwner { const XPlaneVisitor* metadata_; }; +using XEventMetadataVisitor = XStatsOwner; + class XEventVisitor : public XStatsOwner { public: // REQUIRED: plane, line and event cannot be nullptr. XEventVisitor(const XPlaneVisitor* plane, const XLine* line, const XEvent* event); + + XEventMetadataVisitor MetadataStats() const { + return XEventMetadataVisitor(plane_, metadata_); + } + int64 Id() const { return event_->metadata_id(); } absl::string_view Name() const { return metadata_->name(); } From 24d9188b68a82f602205e5116b8a54725e023377 Mon Sep 17 00:00:00 2001 From: Yanhua Sun Date: Tue, 11 Aug 2020 16:25:39 -0700 Subject: [PATCH 0860/1017] avoid one extra function call to optimize functions. Also, this is more consistent with everywhere else on how the checks are done PiperOrigin-RevId: 326125083 Change-Id: I5636d8160a83f5fc810526c9f5993080bbd7f697 --- tensorflow/python/eager/function.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index c6f472a4071..f0004f0aa65 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -54,7 +54,6 @@ from tensorflow.python.framework import func_graph as func_graph_module from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_shape from tensorflow.python.framework import tensor_spec -from tensorflow.python.framework import tensor_util from tensorflow.python.framework import type_spec from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops @@ -2713,8 +2712,10 @@ def _is_ndarray(value): """Tests whether the given value is an ndarray (and not a TF tensor/var).""" # TODO(tomhennigan) Support __array_interface__ too. return hasattr(value, "__array__") and not ( - resource_variable_ops.is_resource_variable(value) - or tensor_util.is_tensor(value) + isinstance(value, ops.Tensor) + or isinstance(value, resource_variable_ops.BaseResourceVariable) + or hasattr(value, "_should_act_as_resource_variable") + # For legacy reasons we do not automatically promote Numpy strings. or isinstance(value, np.str_) # NumPy dtypes have __array__ as unbound methods. From db04ee6238f95c6ac23664eada064b15636b69d8 Mon Sep 17 00:00:00 2001 From: Rahul Joshi Date: Tue, 11 Aug 2020 16:45:18 -0700 Subject: [PATCH 0861/1017] [MLIR] Extend ResourceDeviceInference to handle calls. - Handle calls by propagating device attribute to callee arguments, similar to functional If and While. - Also extend the unit test to exercise calls and IfRegion (which works without any code changes in this pass). PiperOrigin-RevId: 326128697 Change-Id: I65188acc5ee453fdb3d4151279869f0669ffe71d --- .../tests/resource-device-inference.mlir | 320 ++++++++++++++---- .../transforms/resource_device_inference.cc | 148 ++++---- 2 files changed, 318 insertions(+), 150 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/resource-device-inference.mlir b/tensorflow/compiler/mlir/tensorflow/tests/resource-device-inference.mlir index a4a7c1dad2e..dd622e565c0 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/resource-device-inference.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/resource-device-inference.mlir @@ -1,31 +1,33 @@ // RUN: tf-opt -split-input-file -verify-diagnostics -tf-resource-device-inference %s | FileCheck %s +!tf_res = type tensor<*x!tf.resource>> + // Tests that the pass can correctly propagate device attributes inside the same // function. // CHECK-LABEL: func @propagate_in_function func @propagate_in_function( - %arg0: tensor<*x!tf.resource>> {tf.device = "/TPU:0"}, - %arg1: tensor<*x!tf.resource>> {tf.device = "/TPU:1"}) { + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: !tf_res {tf.device = "/TPU:1"}) { tf_executor.graph { // CHECK: tf_executor.island %island = tf_executor.island { // CHECK-NEXT: "tf.VarHandleOp" %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/CPU:0"} - : () -> tensor<*x!tf.resource>> + : () -> !tf_res // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:0"} - %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:0"} - %id1 = "tf.Identity"(%id0) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id1 = "tf.Identity"(%id0) : (!tf_res) + -> !tf_res // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/CPU:0"} - %id2 = "tf.Identity"(%var_handle) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> - %read = "tf.ReadVariableOp"(%id2) : (tensor<*x!tf.resource>>) -> tensor<32xf32> + %id2 = "tf.Identity"(%var_handle) : (!tf_res) + -> !tf_res + %read = "tf.ReadVariableOp"(%id2) : (!tf_res) -> tensor<32xf32> %id3 = "tf.Identity"(%read) : (tensor<32xf32>) -> tensor<32xf32> tf_executor.yield } @@ -35,30 +37,31 @@ func @propagate_in_function( } // ----- +!tf_res = type tensor<*x!tf.resource>> // Tesets that the pass can propagate through tf.If's branches. // CHECK-LABEL: func @propagate_if_op func @propagate_if_op( - %arg0: tensor<*x!tf.resource>> {tf.device = "/TPU:0"}, + %arg0: !tf_res {tf.device = "/TPU:0"}, %arg1: tensor) { tf_executor.graph { // CHECK: tf_executor.island %island = tf_executor.island { // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:0"} - %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res // CHECK-NEXT: "tf.VarHandleOp" %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} - : () -> tensor<*x!tf.resource>> + : () -> !tf_res // CHECK-NEXT: "tf.If" "tf.If"(%arg1, %id0, %var_handle) { then_branch = @if_then, else_branch = @if_else, is_stateless = false} - : (tensor, tensor<*x!tf.resource>>, - tensor<*x!tf.resource>>) -> () + : (tensor, !tf_res, + !tf_res) -> () tf_executor.yield } tf_executor.fetch %island : !tf_executor.control @@ -68,19 +71,19 @@ func @propagate_if_op( // CHECK-LABEL: func @if_then func @if_then( - %arg0: tensor<*x!tf.resource>>, - %arg1: tensor<*x!tf.resource>>) { + %arg0: !tf_res, + %arg1: !tf_res) { tf_executor.graph { // CHECK: tf_executor.island %island = tf_executor.island { // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:0"} - %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:1"} - %id1 = "tf.Identity"(%arg1) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id1 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res tf_executor.yield } tf_executor.fetch %island : !tf_executor.control @@ -90,15 +93,15 @@ func @if_then( // CHECK-LABEL: func @if_else func @if_else( - %arg0: tensor<*x!tf.resource>>, - %arg1: tensor<*x!tf.resource>>) { + %arg0: !tf_res, + %arg1: !tf_res) { tf_executor.graph { // CHECK: tf_executor.island %island = tf_executor.island { // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:0"} - %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res tf_executor.yield } tf_executor.fetch %island : !tf_executor.control @@ -108,31 +111,31 @@ func @if_else( // ----- +!tf_res = type tensor<*x!tf.resource>> // Tesets that the pass can propagate through tf.While's branches. - // CHECK-LABEL: func @propagate_while_op func @propagate_while_op( - %arg0: tensor<*x!tf.resource>> {tf.device = "/TPU:0"}, + %arg0: !tf_res {tf.device = "/TPU:0"}, %arg1: tensor) { tf_executor.graph { // CHECK: tf_executor.island %island = tf_executor.island { // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:0"} - %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res // CHECK-NEXT: "tf.VarHandleOp" %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} - : () -> tensor<*x!tf.resource>> + : () -> !tf_res // CHECK-NEXT: "tf.While" "tf.While"(%arg1, %id0, %var_handle) { body = @while_body, cond = @while_cond, is_stateless = false} - : (tensor, tensor<*x!tf.resource>>, - tensor<*x!tf.resource>>) -> - (tensor, tensor<*x!tf.resource>>, - tensor<*x!tf.resource>>) + : (tensor, !tf_res, + !tf_res) -> + (tensor, !tf_res, + !tf_res) tf_executor.yield } tf_executor.fetch %island : !tf_executor.control @@ -143,48 +146,48 @@ func @propagate_while_op( // CHECK-LABEL: func @while_body func @while_body( %arg0: tensor, - %arg1: tensor<*x!tf.resource>>, - %arg2: tensor<*x!tf.resource>>) -> - (tensor, tensor<*x!tf.resource>>, - tensor<*x!tf.resource>>) { + %arg1: !tf_res, + %arg2: !tf_res) -> + (tensor, !tf_res, + !tf_res) { %graph:3 = tf_executor.graph { // CHECK: tf_executor.island %island:4 = tf_executor.island { // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:0"} - %id0 = "tf.Identity"(%arg1) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id0 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:1"} - %id1 = "tf.Identity"(%arg2) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id1 = "tf.Identity"(%arg2) : (!tf_res) + -> !tf_res tf_executor.yield %arg0, %id0, %id1 - : tensor, tensor<*x!tf.resource>>, - tensor<*x!tf.resource>> + : tensor, !tf_res, + !tf_res } tf_executor.fetch %island#0, %island#1, %island#2 - : tensor, tensor<*x!tf.resource>>, - tensor<*x!tf.resource>> + : tensor, !tf_res, + !tf_res } return %graph#0, %graph#1, %graph#2 - : tensor, tensor<*x!tf.resource>>, - tensor<*x!tf.resource>> + : tensor, !tf_res, + !tf_res } // CHECK-LABEL: func @while_cond func @while_cond( %arg0: tensor, - %arg1: tensor<*x!tf.resource>>, - %arg2: tensor<*x!tf.resource>>) -> tensor<32xf32> { + %arg1: !tf_res, + %arg2: !tf_res) -> tensor<32xf32> { %graph = tf_executor.graph { // CHECK: tf_executor.island %island:2 = tf_executor.island { // CHECK-NEXT: "tf.Identity" // CHECK-SAME: {device = "/TPU:0"} - %id0 = "tf.Identity"(%arg1) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id0 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res %read = "tf.ReadVariableOp"(%id0) - : (tensor<*x!tf.resource>>) -> tensor<32xf32> + : (!tf_res) -> tensor<32xf32> tf_executor.yield %read : tensor<32xf32> } tf_executor.fetch %island#0 : tensor<32xf32> @@ -193,31 +196,32 @@ func @while_cond( } // ----- +!tf_res = type tensor<*x!tf.resource>> // Tesets that the pass reports error on conflicting assignments from multiple // callers. func @error_on_conflict_multiple_callers( - %arg0: tensor<*x!tf.resource>> {tf.device = "/TPU:0"}, + %arg0: !tf_res {tf.device = "/TPU:0"}, %arg1: tensor) { tf_executor.graph { %island = tf_executor.island { - %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} - : () -> tensor<*x!tf.resource>> + : () -> !tf_res "tf.If"(%arg1, %id0, %var_handle) { then_branch = @if_then_and_else, else_branch = @if_then_and_else, is_stateless = false} - : (tensor, tensor<*x!tf.resource>>, - tensor<*x!tf.resource>>) -> () + : (tensor, !tf_res, + !tf_res) -> () "tf.If"(%arg1, %var_handle, %id0) { // expected-error@above {{Conflicting device assignment for resource}} then_branch = @if_then_and_else, else_branch = @if_then_and_else, is_stateless = false} - : (tensor, tensor<*x!tf.resource>>, - tensor<*x!tf.resource>>) -> () + : (tensor, !tf_res, + !tf_res) -> () tf_executor.yield } tf_executor.fetch %island : !tf_executor.control @@ -226,14 +230,194 @@ func @error_on_conflict_multiple_callers( } func @if_then_and_else( - %arg0: tensor<*x!tf.resource>>, - %arg1: tensor<*x!tf.resource>>) { + %arg0: !tf_res, + %arg1: !tf_res) { tf_executor.graph { %island = tf_executor.island { - %id0 = "tf.Identity"(%arg0) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> - %id1 = "tf.Identity"(%arg1) : (tensor<*x!tf.resource>>) - -> tensor<*x!tf.resource>> + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + %id1 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + return +} + +// ----- + +// Test that the pass can propagate through calls +!tf_res = type tensor<*x!tf.resource>> + +// CHECK-LABEL: func @test_function +// CHECK-SAME: {tf.device = "/TPU:0"} +func @test_function(%arg0: !tf_res) { + // CHECK: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) -> !tf_res + %read = "tf.ReadVariableOp"(%id0) : (!tf_res) -> tensor<32xf32> + %cst = constant dense<3.0> : tensor<32xf32> + %add = "tf.AddV2"(%read, %cst) : (tensor<32xf32>, tensor<32xf32>) -> tensor<32xf32> + "tf.AssignVariableOp"(%arg0, %add) : (!tf_res, tensor<32xf32>) -> () + return +} + +// CHECK-LABEL: func @propagate_through_calls +func @propagate_through_calls( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: !tf_res {tf.device = "/TPU:1"}) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/CPU:0"} + : () -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id1 = "tf.Identity"(%id0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/CPU:0"} + %id2 = "tf.Identity"(%var_handle) : (!tf_res) + -> !tf_res + %read = "tf.ReadVariableOp"(%id2) : (!tf_res) -> tensor<32xf32> + %id3 = "tf.Identity"(%read) : (tensor<32xf32>) -> tensor<32xf32> + call @test_function(%id1) : (!tf_res) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + return +} + +// Test propagation through IfRegion (with non-inlined calls) +// CHECK-LABEL: func @propagate_if_region +func @propagate_if_region( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} + : () -> !tf_res + // CHECK-NEXT: "tf.IfRegion" + "tf.IfRegion"(%arg1) ({ + call @ifregion_then(%id0, %var_handle) : (!tf_res, !tf_res) -> () + "tf.Yield"() : () -> () + }, { + call @ifregion_else(%id0, %var_handle) : (!tf_res, !tf_res) -> () + "tf.Yield"() : () -> () + }) {is_stateless = false} : (tensor) -> () + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + return +} + +// CHECK-LABEL: func @ifregion_then +// CHECK-SAME: (%arg0: {{.+}} {tf.device = "/TPU:0"}, %arg1: {{.+}} {tf.device = "/TPU:1"} +func @ifregion_then( + %arg0: !tf_res, + %arg1: !tf_res) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id1 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + return +} + +// CHECK-LABEL: func @ifregion_else +// CHECK-SAME: (%arg0: {{.+}} {tf.device = "/TPU:0"}, %arg1: {{.+}} {tf.device = "/TPU:1"} +func @ifregion_else( + %arg0: !tf_res, + %arg1: !tf_res) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id1 = "tf.Identity"(%arg1) : (!tf_res) + -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + return +} + +// Test progagation through IfRegion (inlined calls) +// CHECK-LABEL: func @propagate_if_region_inlined +func @propagate_if_region_inlined( + %arg0: !tf_res {tf.device = "/TPU:0"}, + %arg1: tensor) { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id0 = "tf.Identity"(%arg0) : (!tf_res) + -> !tf_res + // CHECK-NEXT: "tf.VarHandleOp" + %var_handle = "tf.VarHandleOp"() {container = "c", shared_name = "v0", device = "/TPU:1"} + : () -> !tf_res + // CHECK-NEXT: "tf.IfRegion" + "tf.IfRegion"(%arg1) ({ + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id1 = "tf.Identity"(%id0) : (!tf_res) -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id2 = "tf.Identity"(%var_handle) : (!tf_res) -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + "tf.Yield"() : () -> () + }, { + tf_executor.graph { + // CHECK: tf_executor.island + %island = tf_executor.island { + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:0"} + %id1 = "tf.Identity"(%id0) : (!tf_res) -> !tf_res + // CHECK-NEXT: "tf.Identity" + // CHECK-SAME: {device = "/TPU:1"} + %id2 = "tf.Identity"(%var_handle) : (!tf_res) -> !tf_res + tf_executor.yield + } + tf_executor.fetch %island : !tf_executor.control + } + "tf.Yield"() : () -> () + }) {is_stateless = false} : (tensor) -> () tf_executor.yield } tf_executor.fetch %island : !tf_executor.control diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/resource_device_inference.cc b/tensorflow/compiler/mlir/tensorflow/transforms/resource_device_inference.cc index 7e8e9ee30c8..bd0e8a94a61 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/resource_device_inference.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/resource_device_inference.cc @@ -66,22 +66,18 @@ class PerFunctionResult { : alias_analysis_(alias_analysis) {} // Returns the recorded device assignment for a resource, if any. - llvm::Optional DeviceForResource( - const Value resource) const { - llvm::Optional result; - if (alias_analysis_.IsUnknownResource(resource)) return result; + Optional DeviceForResource(Value resource) const { + Optional result; + if (alias_analysis_.IsUnknownResource(resource)) return llvm::None; for (int64_t id : alias_analysis_.GetResourceUniqueIds(resource)) { auto it = resource_id_to_device_.find(id); if (it == resource_id_to_device_.end()) continue; - if (!result) { + if (!result || result == it->second) { result = it->getSecond(); continue; } - if (result != it->getSecond()) { - // Got conflicting assignments, clear the result. - result.reset(); - return result; - } + // Got conflicting assignments + return llvm::None; } return result; } @@ -90,7 +86,7 @@ class PerFunctionResult { // conflicts with an existing one, returns an error. // // If `changed` is provided, assign *changed to true if anything is modified. - LogicalResult AddResourceDevice(const Value resource, llvm::StringRef device, + LogicalResult AddResourceDevice(Value resource, StringRef device, bool* changed = nullptr) { if (alias_analysis_.IsUnknownResource(resource)) return success(); for (int64_t id : alias_analysis_.GetResourceUniqueIds(resource)) { @@ -106,13 +102,12 @@ class PerFunctionResult { } private: - llvm::SmallDenseMap resource_id_to_device_; + llvm::SmallDenseMap resource_id_to_device_; const TF::ResourceAliasAnalysis::Info& alias_analysis_; }; // Tries to record device assignment for a resource. -LogicalResult AddResourceDeviceAndEmitError(const Value resource, - llvm::StringRef device, +LogicalResult AddResourceDeviceAndEmitError(Value resource, StringRef device, Operation* error_reporting_op, PerFunctionResult* result, bool* changed = nullptr) { @@ -124,18 +119,27 @@ LogicalResult AddResourceDeviceAndEmitError(const Value resource, return res; } +// Extracts and canonicalizes the device attribute. +inline StringRef GetDeviceAttr(FuncOp func, int arg_no) { + auto device_attr = + func.getArgAttrOfType(arg_no, kFuncDeviceAttr); + return device_attr ? device_attr.getValue() : ""; +} + +// Extracts and canonicalizes the device attribute. +inline StringRef GetDeviceAttr(Operation* op) { + auto device_attr = op->getAttrOfType(kDeviceAttr); + return device_attr ? device_attr.getValue() : ""; +} + // Propagates device assignment inside a function. LogicalResult ComputeResourceDevicesInComputation(FuncOp func_op, PerFunctionResult* result) { OpBuilder builder(func_op); // Function arguments. - for (auto arg : func_op.getArguments()) { - if (!mlir::getElementTypeOrSelf(arg.getType()).isa()) { - continue; - } - auto device_attr = func_op.getArgAttrOfType( - arg.getArgNumber(), kFuncDeviceAttr); - if (!device_attr || device_attr.getValue() == "") { + for (auto arg : filter_resources(func_op.getArguments())) { + StringRef device_attr = GetDeviceAttr(func_op, arg.getArgNumber()); + if (device_attr.empty()) { // If device_attr does not exist, try to construct it from any recorded // assignment. if (auto device = result->DeviceForResource(arg)) { @@ -145,49 +149,28 @@ LogicalResult ComputeResourceDevicesInComputation(FuncOp func_op, continue; } // Record the attribute. - auto res = AddResourceDeviceAndEmitError(arg, device_attr.getValue(), - func_op, result); + auto res = AddResourceDeviceAndEmitError(arg, device_attr, func_op, result); if (failed(res)) return res; } + auto walk_res = func_op.walk([&](Operation* op) { - if (auto var_handle = llvm::dyn_cast(op)) { + if (auto var_handle = dyn_cast(op)) { // Record VarHandleOp's device attribute. - auto device_attr = - var_handle.getAttrOfType(kDeviceAttr); - if (!device_attr || device_attr.getValue().empty()) { - return WalkResult::advance(); - } - auto res = AddResourceDeviceAndEmitError( - var_handle.resource(), device_attr.getValue(), op, result); + StringRef device_attr = GetDeviceAttr(op); + if (device_attr.empty()) return WalkResult::advance(); + auto res = AddResourceDeviceAndEmitError(var_handle.resource(), + device_attr, op, result); if (failed(res)) return WalkResult::interrupt(); } - if (auto identity = llvm::dyn_cast(op)) { + if (auto identity = dyn_cast(op)) { // Try to construct IdentityOp's attribute from recorded assignment. - if (!mlir::getElementTypeOrSelf(identity.output().getType()) - .isa()) { - return WalkResult::advance(); - } - if (auto device = result->DeviceForResource(identity.output())) { - auto device_attr = - identity.getAttrOfType(kDeviceAttr); - if (!device_attr || device_attr.getValue().empty()) { + if (!GetDeviceAttr(op).empty()) return WalkResult::advance(); + for (auto output : filter_resources(op->getResults())) { + if (auto device = result->DeviceForResource(output)) identity.setAttr(kDeviceAttr, builder.getStringAttr(*device)); - } } return WalkResult::advance(); } - // Propagate and record output device assignment for other ops based on - // existing recording. E.g., IdentityN. - for (auto output : op->getResults()) { - if (!mlir::getElementTypeOrSelf(output.getType()) - .isa()) { - continue; - } - if (auto device = result->DeviceForResource(output)) { - auto res = AddResourceDeviceAndEmitError(output, *device, op, result); - if (failed(res)) return WalkResult::interrupt(); - } - } return WalkResult::advance(); }); return failure(walk_res.wasInterrupted()); @@ -198,13 +181,13 @@ void ResourceDeviceInference::runOnOperation() { const auto& resource_alias_analysis = getAnalysis(); - llvm::SmallDenseMap per_function_results; + llvm::SmallDenseMap per_function_results; llvm::SetVector worklist; - module.walk([&](FuncOp func_op) { + for (auto func_op : module.getOps()) { worklist.insert(func_op); per_function_results.try_emplace( func_op, func_op, resource_alias_analysis.GetAnalysisForFunc(func_op)); - }); + } // Helper that propagates an op's recorded operand device assignments to its // called function's arguments. auto propagate_operands_to_callee_arguments = @@ -214,51 +197,52 @@ void ResourceDeviceInference::runOnOperation() { assert(callee); auto& callee_res = per_function_results.find(callee)->getSecond(); bool callee_needs_recompute = false; - for (auto operand_and_argument : - llvm::zip(caller_operands, callee.getArguments())) { - if (!mlir::getElementTypeOrSelf( - std::get<0>(operand_and_argument).getType()) - .isa()) { - continue; - } - auto device = - caller_res.DeviceForResource(std::get<0>(operand_and_argument)); + for (BlockArgument arg : filter_resources(callee.getArguments())) { + Value arg_operand = caller_operands[arg.getArgNumber()]; + auto device = caller_res.DeviceForResource(arg_operand); if (!device) continue; - if (failed(AddResourceDeviceAndEmitError( - std::get<1>(operand_and_argument), *device, caller, - &callee_res, &callee_needs_recompute))) { + if (failed(AddResourceDeviceAndEmitError(arg, *device, caller, + &callee_res, + &callee_needs_recompute))) return failure(); - } } // If the callee recording is modified, make sure that it will be // reprocessed. - if (callee_needs_recompute) { - worklist.insert(callee); - } + if (callee_needs_recompute) worklist.insert(callee); } return success(); }; while (!worklist.empty()) { - auto func_op = worklist.back(); - worklist.pop_back(); + auto func_op = worklist.pop_back_val(); auto& func_res = per_function_results.find(func_op)->getSecond(); // In-function propagation. - if (failed(ComputeResourceDevicesInComputation(func_op, &func_res))) { + if (failed(ComputeResourceDevicesInComputation(func_op, &func_res))) return signalPassFailure(); - } + // Propagation to callees. auto walk_res = func_op.walk([&](Operation* op) { - if (auto while_op = llvm::dyn_cast(op)) { + if (auto while_op = dyn_cast(op)) { if (failed(propagate_operands_to_callee_arguments( while_op, while_op.getOperands(), {while_op.body_func(), while_op.cond_func()}, func_res))) return WalkResult::interrupt(); - } else if (auto if_op = llvm::dyn_cast(op)) { + } else if (auto if_op = dyn_cast(op)) { if (failed(propagate_operands_to_callee_arguments( if_op, if_op.input(), {if_op.then_func(), if_op.else_func()}, func_res))) return WalkResult::interrupt(); + } else if (auto call = dyn_cast(op)) { + auto func = dyn_cast(call.resolveCallable()); + if (!func) { + op->emitError( + "Cannot propagate device attribute to callee: Unable to resolve " + "call"); + return WalkResult::interrupt(); + } + if (failed(propagate_operands_to_callee_arguments( + call, call.getArgOperands(), {func}, func_res))) + return WalkResult::interrupt(); } return WalkResult::advance(); }); @@ -266,15 +250,15 @@ void ResourceDeviceInference::runOnOperation() { } } +PassRegistration pass( + "tf-resource-device-inference", + "Propagates the device attribute on resources from callers to callees."); + } // namespace std::unique_ptr> CreateResourceDeviceInferencePass() { return std::make_unique(); } -static PassRegistration pass( - "tf-resource-device-inference", - "Propagates the device attribute on resources from callers to callees."); - } // namespace TF } // namespace mlir From 85cfe48f62bb433bd996297ee41b052cd644d006 Mon Sep 17 00:00:00 2001 From: Tim Shen Date: Tue, 11 Aug 2020 16:51:12 -0700 Subject: [PATCH 0862/1017] [PJRT] Disable MSan for two tests. PiperOrigin-RevId: 326129785 Change-Id: Ib3912bc4fa6847f4ac4d2ccd40ec9f925ca96e88 --- tensorflow/compiler/xla/pjrt/distributed/BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/compiler/xla/pjrt/distributed/BUILD b/tensorflow/compiler/xla/pjrt/distributed/BUILD index 5cada95390c..175b4268dda 100644 --- a/tensorflow/compiler/xla/pjrt/distributed/BUILD +++ b/tensorflow/compiler/xla/pjrt/distributed/BUILD @@ -52,6 +52,9 @@ cc_library( tf_cc_test( name = "service_test", srcs = ["service_test.cc"], + tags = [ + "nomsan", # b/163629207 + ], deps = [ ":protocol_proto_cc", ":service", @@ -106,6 +109,9 @@ cc_library( tf_cc_test( name = "client_server_test", srcs = ["client_server_test.cc"], + tags = [ + "nomsan", # b/163629207 + ], deps = [ ":client", ":protocol_proto_cc", From 497d94e9acc1c062791a62e9d8d339a16f223c47 Mon Sep 17 00:00:00 2001 From: Yujing Zhang Date: Tue, 11 Aug 2020 17:08:40 -0700 Subject: [PATCH 0863/1017] Separate function tests from c_api_remote_test PiperOrigin-RevId: 326133427 Change-Id: Ia611cd513ef84d393f0f32ba9410c037f339fe19 --- tensorflow/c/eager/BUILD | 41 +++ .../c/eager/c_api_remote_function_test.cc | 56 +++++ tensorflow/c/eager/c_api_remote_test.cc | 237 +----------------- tensorflow/c/eager/c_api_remote_test_util.cc | 215 ++++++++++++++++ tensorflow/c/eager/c_api_remote_test_util.h | 26 ++ 5 files changed, 348 insertions(+), 227 deletions(-) create mode 100644 tensorflow/c/eager/c_api_remote_function_test.cc create mode 100644 tensorflow/c/eager/c_api_remote_test_util.cc create mode 100644 tensorflow/c/eager/c_api_remote_test_util.h diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD index 61701bc8b21..d542eae8238 100644 --- a/tensorflow/c/eager/BUILD +++ b/tensorflow/c/eager/BUILD @@ -508,6 +508,27 @@ tf_cuda_cc_test( ], ) +tf_cuda_library( + name = "c_api_remote_test_util", + testonly = 1, + srcs = ["c_api_remote_test_util.cc"], + hdrs = ["c_api_remote_test_util.h"], + visibility = ["//tensorflow:__subpackages__"], + deps = [ + ":c_api", + ":c_api_internal", + ":c_api_test_util", + ":tfe_tensorhandle_internal", + "//tensorflow/core:framework", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core:test", + "//tensorflow/core/common_runtime/eager:tensor_handle", + "//tensorflow/core/distributed_runtime/rpc:grpc_server_lib", + "@com_google_absl//absl/strings", + ], +) + tf_cuda_cc_test( name = "c_api_remote_test", size = "small", @@ -524,6 +545,7 @@ tf_cuda_cc_test( ":c_api", ":c_api_experimental", ":c_api_internal", + ":c_api_remote_test_util", ":c_api_test_util", ":tfe_tensorhandle_internal", "//tensorflow/c:c_test_util", @@ -540,6 +562,25 @@ tf_cuda_cc_test( ], ) +tf_cuda_cc_test( + name = "c_api_remote_function_test", + size = "small", + srcs = [ + "c_api_remote_function_test.cc", + ], + # TODO(b/136478427): Figure out how to correctly shut the server down + args = ["--heap_check=local"], + extra_copts = tfe_xla_copts(), + tags = [ + "no_windows", + ], + deps = [ + ":c_api_remote_test_util", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + ], +) + tf_cuda_cc_test( name = "c_api_distributed_test", size = "small", diff --git a/tensorflow/c/eager/c_api_remote_function_test.cc b/tensorflow/c/eager/c_api_remote_function_test.cc new file mode 100644 index 00000000000..d3f9826635c --- /dev/null +++ b/tensorflow/c/eager/c_api_remote_function_test.cc @@ -0,0 +1,56 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/eager/c_api_remote_test_util.h" +#include "tensorflow/core/platform/test.h" + +namespace { + +void TestRemoteExecuteSilentCopiesFunc(bool async, bool remote, + bool heavy_load_on_streaming_rpc, + bool remote_func_outputs = false) { + return TestRemoteExecuteSilentCopies(async, remote, /*func=*/true, + heavy_load_on_streaming_rpc, + remote_func_outputs); +} + +TEST(CAPI, RemoteExecuteSilentCopiesAsyncFunc) { + TestRemoteExecuteSilentCopiesFunc(/*async=*/true, /*remote=*/true, + /*heavy_load_on_streaming_rpc=*/false); +} +TEST(CAPI, RemoteExecuteSilentCopiesLocalAsyncFunc) { + TestRemoteExecuteSilentCopiesFunc(/*async=*/true, /*remote=*/false, + /*heavy_load_on_streaming_rpc=*/false); +} +// TODO(b/162618595): Enable this test once we remove the check of remote +// outputs in ProcessFunctionLibraryRuntime. +TEST(CAPI, DISABLED_RemoteExecuteSilentCopiesLocalFuncRemoteOutputs) { + TestRemoteExecuteSilentCopiesFunc(/*async=*/false, /*remote=*/false, + /*heavy_load_on_streaming_rpc=*/false, + /*remote_func_outputs=*/true); +} +TEST(CAPI, DISABLED_RemoteExecuteSilentCopiesLocalAsyncFuncRemoteOutputs) { + TestRemoteExecuteSilentCopiesFunc(/*async=*/true, /*remote=*/false, + /*heavy_load_on_streaming_rpc=*/false, + /*remote_func_outputs=*/true); +} +TEST(CAPI, RemoteExecuteSilentCopiesLocalAsyncFuncOrdering) { + // A remote input may be not ready when we start running a function. Test that + // the function execution should wait until the remote input is ready. + TestRemoteExecuteSilentCopiesFunc(/*async=*/true, /*remote=*/false, + /*heavy_load_on_streaming_rpc=*/true); +} + +} // namespace diff --git a/tensorflow/c/eager/c_api_remote_test.cc b/tensorflow/c/eager/c_api_remote_test.cc index e99f6d6e170..e68e15ba560 100644 --- a/tensorflow/c/eager/c_api_remote_test.cc +++ b/tensorflow/c/eager/c_api_remote_test.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/c/eager/c_api.h" #include "tensorflow/c/eager/c_api_experimental.h" #include "tensorflow/c/eager/c_api_internal.h" +#include "tensorflow/c/eager/c_api_remote_test_util.h" #include "tensorflow/c/eager/c_api_test_util.h" #include "tensorflow/c/eager/tfe_tensorhandle_internal.h" #include "tensorflow/core/common_runtime/eager/eager_operation.h" @@ -116,242 +117,24 @@ void TestRemoteExecute(bool async) { TEST(CAPI, RemoteExecute) { TestRemoteExecute(false); } TEST(CAPI, RemoteExecuteAsync) { TestRemoteExecute(true); } -string MatMulFunction(const string& matmul_device) { - tensorflow::FunctionDef def; - CHECK(tensorflow::protobuf::TextFormat::ParseFromString( - absl::StrCat(" signature {" - " name: 'MatMulFunction'" - " input_arg {" - " name: 'a'" - " type: DT_FLOAT" - " }" - " input_arg {" - " name: 'b'" - " type: DT_FLOAT" - " }" - " output_arg {" - " name: 'm'" - " type: DT_FLOAT" - " }" - " }" - " node_def {" - " name: 'matmul'" - " op: 'MatMul'" - " input: 'a'" - " input: 'b'" - " device: '", - matmul_device, "'", - " attr {" - " key: 'T'" - " value {" - " type: DT_FLOAT" - " }" - " }" - " }" - " ret {" - " key: 'm'" - " value: 'matmul:product'" - " }"), - &def)); - return def.SerializeAsString(); -} - -// If heavy_load_on_streaming_rpc is true, send some rpc reqeusts before the one -// which creates a remote remote input, to simulate a scenario that the remote -// input is not ready when we start running an op or a function. -void TestRemoteExecuteSilentCopies(bool async, bool remote, bool func, - bool heavy_load_on_streaming_rpc, - bool remote_func_outputs = false) { - tensorflow::ServerDef server_def = GetServerDef(3); - - // This server def has the task index set to 0. - string serialized = server_def.SerializeAsString(); - - server_def.set_task_index(1); - std::unique_ptr worker_server1; - ASSERT_TRUE(tensorflow::GrpcServer::Create( - server_def, tensorflow::Env::Default(), &worker_server1) - .ok()); - ASSERT_TRUE(worker_server1->Start().ok()); - - server_def.set_task_index(2); - std::unique_ptr worker_server2; - ASSERT_TRUE(tensorflow::GrpcServer::Create( - server_def, tensorflow::Env::Default(), &worker_server2) - .ok()); - ASSERT_TRUE(worker_server2->Start().ok()); - - TF_Status* status = TF_NewStatus(); - TFE_ContextOptions* opts = TFE_NewContextOptions(); - TFE_ContextOptionsSetAsync(opts, static_cast(async)); - TFE_ContextOptionsSetDevicePlacementPolicy(opts, TFE_DEVICE_PLACEMENT_SILENT); - TFE_Context* ctx = TFE_NewContext(opts, status); - EXPECT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - TFE_DeleteContextOptions(opts); - - TFE_ContextSetServerDef(ctx, 0, serialized.data(), serialized.size(), status); - EXPECT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - - TFE_TensorHandle* h0_task0 = TestMatrixTensorHandle(ctx); - TFE_TensorHandle* h1_task0 = TestMatrixTensorHandle(ctx); - std::vector handles_task0; - if (heavy_load_on_streaming_rpc) { - // Send 50 tensor copy requests to simulate that there have been some RPC - // requests been enqueued. - for (int i = 0; i < 50; ++i) { - handles_task0.push_back(TestMatrixTensorHandle(ctx)); - } - } - const char task1_name[] = "/job:localhost/replica:0/task:1/device:CPU:0"; - const char task2_name[] = "/job:localhost/replica:0/task:2/device:CPU:0"; - - std::vector handles_task2; - for (auto* h_task0 : handles_task0) { - handles_task2.push_back( - TFE_TensorHandleCopyToDevice(h_task0, ctx, task2_name, status)); - ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - } - - auto* h1_task2 = - TFE_TensorHandleCopyToDevice(h1_task0, ctx, task2_name, status); - ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - - TFE_Op* matmul = nullptr; - if (func) { - const string matmul_device = remote_func_outputs ? task2_name : ""; - string function_def = MatMulFunction(matmul_device); - TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(), - status); - CHECK_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - - matmul = TFE_NewOp(ctx, "MatMulFunction", status); - ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - TFE_OpAddInput(matmul, h0_task0, status); - ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - TFE_OpAddInput(matmul, h1_task2, status); - ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - } else { - // Handles are on task0 (local), and task2, but op is on task1. - matmul = MatMulOp(ctx, h0_task0, h1_task2); - } - if (remote) { - TFE_OpSetDevice(matmul, task1_name, status); - ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - } else if (!async) { - // Set the local device to CPU to easily validate mirroring - string cpu_device_name; - ASSERT_TRUE(GetDeviceName(ctx, &cpu_device_name, "CPU")); - TFE_OpSetDevice(matmul, cpu_device_name.c_str(), status); - EXPECT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - auto remote_arg = - tensorflow::TensorHandleFromInterface(tensorflow::unwrap(h1_task2)); - // The input handles should never change since they have been mirrored. - ASSERT_FALSE(remote_arg->HasLocalMirror(nullptr)); - } - - TFE_TensorHandle* retvals[1]; - int num_retvals = 1; - TFE_Execute(matmul, &retvals[0], &num_retvals, status); - EXPECT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - - // TODO(gjn): Add support for waiting on async local mirrors - if (!remote && !async && !remote_func_outputs) { - auto remote_arg = - tensorflow::TensorHandleFromInterface(tensorflow::unwrap(h1_task2)); - // The input handles should never change since they have been mirrored. - ASSERT_TRUE(remote_arg->HasLocalMirror(nullptr)); - } - - auto* retval_task0 = TFE_TensorHandleCopyToDevice( - retvals[0], ctx, "/job:localhost/replica:0/task:0/device:CPU:0", status); - ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - - TF_Tensor* t = TFE_TensorHandleResolve(retval_task0, status); - ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - TFE_DeleteTensorHandle(retval_task0); - float product[4] = {0}; - EXPECT_EQ(sizeof(product), TF_TensorByteSize(t)); - memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t)); - TF_DeleteTensor(t); - EXPECT_EQ(7, product[0]); - EXPECT_EQ(10, product[1]); - EXPECT_EQ(15, product[2]); - EXPECT_EQ(22, product[3]); - - TFE_DeleteTensorHandle(h0_task0); - TFE_DeleteTensorHandle(h1_task0); - TFE_DeleteTensorHandle(h1_task2); - TFE_DeleteTensorHandle(retvals[0]); - for (auto* h : handles_task0) { - TFE_DeleteTensorHandle(h); - } - for (auto* h : handles_task2) { - TFE_DeleteTensorHandle(h); - } - - TFE_DeleteOp(matmul); - - TFE_Executor* executor = TFE_ContextGetExecutorForThread(ctx); - TFE_ExecutorWaitForAllPendingNodes(executor, status); - ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); - TFE_DeleteExecutor(executor); - if (func) { - TFE_ContextRemoveFunction(ctx, "MatMulFunction", status); - } - TFE_DeleteContext(ctx); - - TF_DeleteStatus(status); - - // TODO(b/136478427): Figure out how to correctly shut the server down. - worker_server1.release(); - worker_server2.release(); +void TestRemoteExecuteSilentCopiesOp(bool async, bool remote, + bool remote_func_outputs = false) { + return TestRemoteExecuteSilentCopies(async, remote, /*func=*/false, + /*heavy_load_on_streaming_rpc=*/false, + remote_func_outputs); } TEST(CAPI, RemoteExecuteSilentCopies) { - TestRemoteExecuteSilentCopies(/*async=*/false, /*remote=*/true, - /*func=*/false, - /*heavy_load_on_streaming_rpc=*/false); + TestRemoteExecuteSilentCopiesOp(/*async=*/false, /*remote=*/true); } TEST(CAPI, RemoteExecuteSilentCopiesAsync) { - TestRemoteExecuteSilentCopies(/*async=*/true, /*remote=*/true, /*func=*/false, - /*heavy_load_on_streaming_rpc=*/false); -} -TEST(CAPI, RemoteExecuteSilentCopiesAsyncFunc) { - TestRemoteExecuteSilentCopies(/*async=*/true, /*remote=*/true, /*func=*/true, - /*heavy_load_on_streaming_rpc=*/false); + TestRemoteExecuteSilentCopiesOp(/*async=*/true, /*remote=*/true); } TEST(CAPI, RemoteExecuteSilentCopiesLocal) { - TestRemoteExecuteSilentCopies(/*async=*/false, /*remote=*/false, - /*func=*/false, - /*heavy_load_on_streaming_rpc=*/false); + TestRemoteExecuteSilentCopiesOp(/*async=*/false, /*remote=*/false); } TEST(CAPI, RemoteExecuteSilentCopiesLocalAsync) { - TestRemoteExecuteSilentCopies(/*async=*/true, /*remote=*/false, - /*func=*/false, - /*heavy_load_on_streaming_rpc=*/false); -} -TEST(CAPI, RemoteExecuteSilentCopiesLocalAsyncFunc) { - TestRemoteExecuteSilentCopies(/*async=*/true, /*remote=*/false, /*func=*/true, - /*heavy_load_on_streaming_rpc=*/false); -} -// TODO(b/162618595): Enable this test once we remove the check of remote -// outputs in ProcessFunctionLibraryRuntime. -TEST(CAPI, DISABLED_RemoteExecuteSilentCopiesLocalFuncRemoteOutputs) { - TestRemoteExecuteSilentCopies(/*async=*/false, /*remote=*/false, - /*func=*/true, - /*heavy_load_on_streaming_rpc=*/false, - /*remote_func_outputs=*/true); -} -TEST(CAPI, DISABLED_RemoteExecuteSilentCopiesLocalAsyncFuncRemoteOutputs) { - TestRemoteExecuteSilentCopies(/*async=*/true, /*remote=*/false, /*func=*/true, - /*heavy_load_on_streaming_rpc=*/false, - /*remote_func_outputs=*/true); -} -TEST(CAPI, RemoteExecuteSilentCopiesLocalAsyncFuncOrdering) { - // A remote input may be not ready when we start running a function. Test that - // the function execution should wait until the remote input is ready. - TestRemoteExecuteSilentCopies(/*async=*/true, /*remote=*/false, /*func=*/true, - /*heavy_load_on_streaming_rpc=*/true); + TestRemoteExecuteSilentCopiesOp(/*async=*/true, /*remote=*/false); } } // namespace diff --git a/tensorflow/c/eager/c_api_remote_test_util.cc b/tensorflow/c/eager/c_api_remote_test_util.cc new file mode 100644 index 00000000000..0ae5b74553a --- /dev/null +++ b/tensorflow/c/eager/c_api_remote_test_util.cc @@ -0,0 +1,215 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/c/eager/c_api_remote_test_util.h" + +#include "absl/strings/str_cat.h" +#include "tensorflow/c/eager/c_api_internal.h" +#include "tensorflow/c/eager/c_api_test_util.h" +#include "tensorflow/c/eager/tfe_tensorhandle_internal.h" +#include "tensorflow/core/common_runtime/eager/tensor_handle.h" +#include "tensorflow/core/distributed_runtime/rpc/grpc_server_lib.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/tensorflow_server.pb.h" + +using ::tensorflow::string; + +string MatMulFunction(const string& matmul_device) { + tensorflow::FunctionDef def; + CHECK(tensorflow::protobuf::TextFormat::ParseFromString( + absl::StrCat(" signature {" + " name: 'MatMulFunction'" + " input_arg {" + " name: 'a'" + " type: DT_FLOAT" + " }" + " input_arg {" + " name: 'b'" + " type: DT_FLOAT" + " }" + " output_arg {" + " name: 'm'" + " type: DT_FLOAT" + " }" + " }" + " node_def {" + " name: 'matmul'" + " op: 'MatMul'" + " input: 'a'" + " input: 'b'" + " device: '", + matmul_device, "'", + " attr {" + " key: 'T'" + " value {" + " type: DT_FLOAT" + " }" + " }" + " }" + " ret {" + " key: 'm'" + " value: 'matmul:product'" + " }"), + &def)); + return def.SerializeAsString(); +} + +void TestRemoteExecuteSilentCopies(bool async, bool remote, bool func, + bool heavy_load_on_streaming_rpc, + bool remote_func_outputs) { + tensorflow::ServerDef server_def = GetServerDef(3); + + // This server def has the task index set to 0. + string serialized = server_def.SerializeAsString(); + + server_def.set_task_index(1); + std::unique_ptr worker_server1; + ASSERT_TRUE(tensorflow::GrpcServer::Create( + server_def, tensorflow::Env::Default(), &worker_server1) + .ok()); + ASSERT_TRUE(worker_server1->Start().ok()); + + server_def.set_task_index(2); + std::unique_ptr worker_server2; + ASSERT_TRUE(tensorflow::GrpcServer::Create( + server_def, tensorflow::Env::Default(), &worker_server2) + .ok()); + ASSERT_TRUE(worker_server2->Start().ok()); + + TF_Status* status = TF_NewStatus(); + TFE_ContextOptions* opts = TFE_NewContextOptions(); + TFE_ContextOptionsSetAsync(opts, static_cast(async)); + TFE_ContextOptionsSetDevicePlacementPolicy(opts, TFE_DEVICE_PLACEMENT_SILENT); + TFE_Context* ctx = TFE_NewContext(opts, status); + EXPECT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + TFE_DeleteContextOptions(opts); + + TFE_ContextSetServerDef(ctx, 0, serialized.data(), serialized.size(), status); + EXPECT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + + TFE_TensorHandle* h0_task0 = TestMatrixTensorHandle(ctx); + TFE_TensorHandle* h1_task0 = TestMatrixTensorHandle(ctx); + std::vector handles_task0; + if (heavy_load_on_streaming_rpc) { + // Send 50 tensor copy requests to simulate that there have been some RPC + // requests been enqueued. + for (int i = 0; i < 50; ++i) { + handles_task0.push_back(TestMatrixTensorHandle(ctx)); + } + } + const char task1_name[] = "/job:localhost/replica:0/task:1/device:CPU:0"; + const char task2_name[] = "/job:localhost/replica:0/task:2/device:CPU:0"; + + std::vector handles_task2; + for (auto* h_task0 : handles_task0) { + handles_task2.push_back( + TFE_TensorHandleCopyToDevice(h_task0, ctx, task2_name, status)); + ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + } + + auto* h1_task2 = + TFE_TensorHandleCopyToDevice(h1_task0, ctx, task2_name, status); + ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + + TFE_Op* matmul = nullptr; + if (func) { + const string matmul_device = remote_func_outputs ? task2_name : ""; + string function_def = MatMulFunction(matmul_device); + TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(), + status); + CHECK_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + + matmul = TFE_NewOp(ctx, "MatMulFunction", status); + ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + TFE_OpAddInput(matmul, h0_task0, status); + ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + TFE_OpAddInput(matmul, h1_task2, status); + ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + } else { + // Handles are on task0 (local), and task2, but op is on task1. + matmul = MatMulOp(ctx, h0_task0, h1_task2); + } + if (remote) { + TFE_OpSetDevice(matmul, task1_name, status); + ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + } else if (!async) { + // Set the local device to CPU to easily validate mirroring + string cpu_device_name; + ASSERT_TRUE(GetDeviceName(ctx, &cpu_device_name, "CPU")); + TFE_OpSetDevice(matmul, cpu_device_name.c_str(), status); + EXPECT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + auto remote_arg = + tensorflow::TensorHandleFromInterface(tensorflow::unwrap(h1_task2)); + // The input handles should never change since they have been mirrored. + ASSERT_FALSE(remote_arg->HasLocalMirror(nullptr)); + } + + TFE_TensorHandle* retvals[1]; + int num_retvals = 1; + TFE_Execute(matmul, &retvals[0], &num_retvals, status); + EXPECT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + + // TODO(gjn): Add support for waiting on async local mirrors + if (!remote && !async && !remote_func_outputs) { + auto remote_arg = + tensorflow::TensorHandleFromInterface(tensorflow::unwrap(h1_task2)); + // The input handles should never change since they have been mirrored. + ASSERT_TRUE(remote_arg->HasLocalMirror(nullptr)); + } + + auto* retval_task0 = TFE_TensorHandleCopyToDevice( + retvals[0], ctx, "/job:localhost/replica:0/task:0/device:CPU:0", status); + ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + + TF_Tensor* t = TFE_TensorHandleResolve(retval_task0, status); + ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + TFE_DeleteTensorHandle(retval_task0); + float product[4] = {0}; + EXPECT_EQ(sizeof(product), TF_TensorByteSize(t)); + memcpy(&product[0], TF_TensorData(t), TF_TensorByteSize(t)); + TF_DeleteTensor(t); + EXPECT_EQ(7, product[0]); + EXPECT_EQ(10, product[1]); + EXPECT_EQ(15, product[2]); + EXPECT_EQ(22, product[3]); + + TFE_DeleteTensorHandle(h0_task0); + TFE_DeleteTensorHandle(h1_task0); + TFE_DeleteTensorHandle(h1_task2); + TFE_DeleteTensorHandle(retvals[0]); + for (auto* h : handles_task0) { + TFE_DeleteTensorHandle(h); + } + for (auto* h : handles_task2) { + TFE_DeleteTensorHandle(h); + } + + TFE_DeleteOp(matmul); + + TFE_Executor* executor = TFE_ContextGetExecutorForThread(ctx); + TFE_ExecutorWaitForAllPendingNodes(executor, status); + ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); + TFE_DeleteExecutor(executor); + if (func) { + TFE_ContextRemoveFunction(ctx, "MatMulFunction", status); + } + TFE_DeleteContext(ctx); + + TF_DeleteStatus(status); + + // TODO(b/136478427): Figure out how to correctly shut the server down. + worker_server1.release(); + worker_server2.release(); +} diff --git a/tensorflow/c/eager/c_api_remote_test_util.h b/tensorflow/c/eager/c_api_remote_test_util.h new file mode 100644 index 00000000000..08633689402 --- /dev/null +++ b/tensorflow/c/eager/c_api_remote_test_util.h @@ -0,0 +1,26 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_C_EAGER_C_API_REMOTE_TEST_UTIL_H_ +#define TENSORFLOW_C_EAGER_C_API_REMOTE_TEST_UTIL_H_ + +// Run a function containing a MatMul op and check its output. +// If heavy_load_on_streaming_rpc is true, send some rpc reqeusts before the one +// which creates a remote remote input, to simulate a scenario that the remote +// input is not ready when we start running an op or a function. +void TestRemoteExecuteSilentCopies(bool async, bool remote, bool func, + bool heavy_load_on_streaming_rpc, + bool remote_func_outputs = false); + +#endif // TENSORFLOW_C_EAGER_C_API_REMOTE_TEST_UTIL_H_ From cee5f17479966a40125a842fba3bfdd050faf079 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Tue, 11 Aug 2020 17:17:20 -0700 Subject: [PATCH 0864/1017] [tf.data] Split RangeBenchmark into two different benchmark methods PiperOrigin-RevId: 326135090 Change-Id: I5e7735468f953e5155fcbbb2c61a6084332096c0 --- .../python/data/benchmarks/range_benchmark.py | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tensorflow/python/data/benchmarks/range_benchmark.py b/tensorflow/python/data/benchmarks/range_benchmark.py index 4eb7c94b564..81e554252df 100644 --- a/tensorflow/python/data/benchmarks/range_benchmark.py +++ b/tensorflow/python/data/benchmarks/range_benchmark.py @@ -24,18 +24,22 @@ from tensorflow.python.data.ops import dataset_ops class RangeBenchmark(benchmark_base.DatasetBenchmarkBase): """Benchmarks for `tf.data.Dataset.range()`.""" - def benchmark_range(self): - for modeling_enabled in [False, True]: - num_elements = 10000000 if modeling_enabled else 50000000 - options = dataset_ops.Options() - options.experimental_optimization.autotune = modeling_enabled - dataset = dataset_ops.Dataset.range(num_elements) - dataset = dataset.with_options(options) + def _benchmark_range(self, num_elements, modeling_enabled): + options = dataset_ops.Options() + options.experimental_optimization.autotune = modeling_enabled + dataset = dataset_ops.Dataset.range(num_elements) + dataset = dataset.with_options(options) - self.run_and_report_benchmark( - dataset, - num_elements=num_elements, - name="modeling_%s" % ("on" if modeling_enabled else "off")) + self.run_and_report_benchmark( + dataset, + num_elements=num_elements, + name="modeling_%s" % ("on" if modeling_enabled else "off")) + + def benchmark_range_with_modeling(self): + self._benchmark_range(num_elements=10000000, modeling_enabled=True) + + def benchmark_range_without_modeling(self): + self._benchmark_range(num_elements=50000000, modeling_enabled=False) if __name__ == "__main__": From 68984541641975e9e80a80eb739c25e3fab9c92f Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Tue, 11 Aug 2020 17:17:31 -0700 Subject: [PATCH 0865/1017] Use the correct path for common_win_cuda11.bat PiperOrigin-RevId: 326135121 Change-Id: I51aeebde1bbbfb0c5be0575d49f4e3d940988fcf --- .../tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat | 2 +- .../tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat index 9d7c98dcc91..fc64acea607 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat @@ -13,7 +13,7 @@ :: limitations under the License. :: ============================================================================= -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\bazel\run_libtensorflow.bat || exit /b 1 diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat index 487ce7592fa..975de5bab54 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat @@ -15,6 +15,6 @@ SET PYTHON_DIRECTORY=Python35 -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat index 56bcd9403b1..16f2df362f9 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat @@ -15,6 +15,6 @@ SET PYTHON_DIRECTORY=Python36 -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat index a326a0c7aa8..2050d2050b1 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat @@ -15,6 +15,6 @@ SET PYTHON_DIRECTORY=Python37 -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat index 46bfcc351c2..145f5a17c7b 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python38 -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat index b1db68586aa..c7e7376e63f 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat @@ -13,7 +13,7 @@ :: limitations under the License. :: ============================================================================= -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\bazel\run_libtensorflow.bat || exit /b diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat index 65d1f420292..c6ca873607e 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python36 -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\integration\gpu_pip_on_cpu\run.bat diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat index 66cdfd09141..3ac22d3e7da 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python35 -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat index 6931c08160d..6a590d1dd87 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python36 -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat index dcaf588b912..34ec6c26124 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python37 -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat index 6305c544c76..a7dd094c811 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python38 -CALL common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" From ebfe3af239f8ee3bd45148ad2da9ff9e9db4edfa Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 17:18:05 -0700 Subject: [PATCH 0866/1017] Raise an error in tf.linalg.expm if the L1 norm of the matrix is infinite or NaN instead of looping infinitely. PiperOrigin-RevId: 326135199 Change-Id: I740d0a5502f14aa3b45fff9496b3f7c307304909 --- .../python/kernel_tests/matrix_exponential_op_test.py | 8 ++++++++ tensorflow/python/ops/linalg/linalg_impl.py | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py index 149ce0d631e..4744d88c34d 100644 --- a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py @@ -24,6 +24,7 @@ import numpy as np from tensorflow.python.client import session from tensorflow.python.framework import constant_op +from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops @@ -137,6 +138,13 @@ class ExponentialOpTest(test.TestCase): with self.assertRaises(ValueError): linalg_impl.matrix_exponential(tensor3) + def testInfinite(self): + # Check that the op does not loop forever on infinite inputs. (b/158433036) + in_tensor = np.random.rand(100, 100).astype(np.float) + in_tensor[0][0] = np.inf + with self.assertRaises(errors_impl.InvalidArgumentError): + self.evaluate(linalg_impl.matrix_exponential(in_tensor)) + def testEmpty(self): self._verifyExponentialReal(np.empty([0, 2, 2])) self._verifyExponentialReal(np.empty([2, 0, 0])) diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py index 9ddf7b5e8b8..cdef22695e9 100644 --- a/tensorflow/python/ops/linalg/linalg_impl.py +++ b/tensorflow/python/ops/linalg/linalg_impl.py @@ -31,6 +31,7 @@ from tensorflow.python.ops import gen_linalg_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import map_fn from tensorflow.python.ops import math_ops +from tensorflow.python.ops import numerics from tensorflow.python.ops import special_math_ops from tensorflow.python.util import dispatch from tensorflow.python.util.tf_export import tf_export @@ -276,6 +277,9 @@ def matrix_exponential(input, name=None): # pylint: disable=redefined-builtin math_ops.abs(matrix), axis=array_ops.size(array_ops.shape(matrix)) - 2), axis=-1)[..., array_ops.newaxis, array_ops.newaxis] + l1_norm = numerics.verify_tensor_all_finite( + l1_norm, 'l1 norm of matrix is Inf or NaN.') + const = lambda x: constant_op.constant(x, l1_norm.dtype) def _nest_where(vals, cases): From 66a07f25095c0963e9f86c6e9ecab46a45ad9c58 Mon Sep 17 00:00:00 2001 From: Yuanzhong Xu Date: Tue, 11 Aug 2020 17:25:03 -0700 Subject: [PATCH 0867/1017] Enable XLA SPMD by default for TF1 PiperOrigin-RevId: 326136232 Change-Id: I9e5272f4eff444b9291fce517b4d7faae04e5d0d --- tensorflow/python/tpu/tpu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/tpu/tpu.py b/tensorflow/python/tpu/tpu.py index 5a2f7ba4454..d90ad4aa60e 100644 --- a/tensorflow/python/tpu/tpu.py +++ b/tensorflow/python/tpu/tpu.py @@ -825,7 +825,7 @@ class XLAOptions( requested. """ - def __new__(cls, use_spmd_for_xla_partitioning=False): + def __new__(cls, use_spmd_for_xla_partitioning=True): return super(XLAOptions, cls).__new__(cls, use_spmd_for_xla_partitioning) From 930274bb8ab10379f4c76618cccc604c9fe27996 Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Tue, 11 Aug 2020 17:48:47 -0700 Subject: [PATCH 0868/1017] [TF2XLA] Update the test to match removed XLA:CPU device PiperOrigin-RevId: 326139696 Change-Id: I65a0341859ea0c3711b7fe7d8b6f47a3d47f6c17 --- tensorflow/python/eager/remote_cloud_tpu_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/eager/remote_cloud_tpu_test.py b/tensorflow/python/eager/remote_cloud_tpu_test.py index 8ba11a3e6ac..3a485b54783 100644 --- a/tensorflow/python/eager/remote_cloud_tpu_test.py +++ b/tensorflow/python/eager/remote_cloud_tpu_test.py @@ -36,7 +36,6 @@ DEVICES_PER_TASK = 8 EXPECTED_DEVICES_PRE_CONNECT = [ '/device:CPU:0', - '/device:XLA_CPU:0', ] EXPECTED_NEW_DEVICES_AFTER_CONNECT_TEMPLATES = [ '/job:worker/replica:0/task:{task}/device:CPU:0', From 9636571807c847dfdfc4bf386154cdf72228bfee Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Tue, 11 Aug 2020 19:27:48 -0700 Subject: [PATCH 0869/1017] [TF2XLA] Fix Windows build: export missing symbol PiperOrigin-RevId: 326151875 Change-Id: I4a39f43652e4bfb7500c3eff01ab4cb180e6ce61 --- tensorflow/tools/def_file_filter/symbols_pybind.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/tools/def_file_filter/symbols_pybind.txt b/tensorflow/tools/def_file_filter/symbols_pybind.txt index b2546582418..93305a0707c 100644 --- a/tensorflow/tools/def_file_filter/symbols_pybind.txt +++ b/tensorflow/tools/def_file_filter/symbols_pybind.txt @@ -379,6 +379,7 @@ tensorflow::grappler::CostAnalyzer::GenerateReport [flags] # tfe tensorflow::IsXlaEnabled tensorflow::GetMlirCommonFlags +tensorflow::GetXlaDeviceFlags [tf32_utils] # tf32 tensorflow::allow_tf32_execution From 642db2faf55e3ca7acd06ea236e9d47f63190718 Mon Sep 17 00:00:00 2001 From: Anna R Date: Tue, 11 Aug 2020 19:56:45 -0700 Subject: [PATCH 0870/1017] Remove SharedMemoryConfig since it is not used anywhere. PiperOrigin-RevId: 326154532 Change-Id: I13be21f577226c48c7e0d5bfc7efb787f0422e85 --- .../xla/service/interpreter/executor.h | 10 -- tensorflow/stream_executor/BUILD | 12 --- .../stream_executor/cuda/cuda_gpu_executor.cc | 98 ++++++------------- tensorflow/stream_executor/gpu/gpu_executor.h | 4 - .../stream_executor/host/host_gpu_executor.h | 14 --- .../stream_executor/rocm/rocm_gpu_executor.cc | 71 +++----------- .../stream_executor/shared_memory_config.h | 34 ------- .../stream_executor_internal.h | 4 - .../stream_executor/stream_executor_pimpl.cc | 19 +--- .../stream_executor/stream_executor_pimpl.h | 23 ++--- tensorflow/stream_executor/tpu/tpu_executor.h | 12 +-- 11 files changed, 54 insertions(+), 247 deletions(-) delete mode 100644 tensorflow/stream_executor/shared_memory_config.h diff --git a/tensorflow/compiler/xla/service/interpreter/executor.h b/tensorflow/compiler/xla/service/interpreter/executor.h index 9e4bdeb2b2d..9416b11a07e 100644 --- a/tensorflow/compiler/xla/service/interpreter/executor.h +++ b/tensorflow/compiler/xla/service/interpreter/executor.h @@ -38,7 +38,6 @@ limitations under the License. #include "tensorflow/stream_executor/launch_dim.h" #include "tensorflow/stream_executor/plugin.h" #include "tensorflow/stream_executor/rng.h" -#include "tensorflow/stream_executor/shared_memory_config.h" #include "tensorflow/stream_executor/stream.h" #include "tensorflow/stream_executor/stream_executor.h" #include "tensorflow/stream_executor/stream_executor_internal.h" @@ -182,15 +181,6 @@ class XlaInterpreterExecutor : public internal::StreamExecutorInterface { return true; } - SharedMemoryConfig GetDeviceSharedMemoryConfig() override { - return SharedMemoryConfig::kDefault; - } - - port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override { - return port::Status{port::error::UNIMPLEMENTED, - "Shared memory not supported"}; - } - std::unique_ptr CreateEventImplementation() override { return nullptr; diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD index 871576f6cef..22aa60a70a4 100644 --- a/tensorflow/stream_executor/BUILD +++ b/tensorflow/stream_executor/BUILD @@ -67,7 +67,6 @@ cc_library( "plugin.h", "plugin_registry.h", "rng.h", - "shared_memory_config.h", "stream_executor_pimpl.h", "temporary_device_memory.h", "temporary_memory_manager.h", @@ -123,7 +122,6 @@ cc_library( "multi_platform_manager.h", "platform.h", "plugin_registry.h", - "shared_memory_config.h", "stream_executor.h", "stream_executor_internal.h", "timer.h", @@ -173,11 +171,6 @@ cc_library( ], ) -cc_library( - name = "shared_memory_config", - hdrs = ["shared_memory_config.h"], -) - # Aliases for backwards compatibility. alias( name = "stream_header", @@ -343,7 +336,6 @@ cc_library( "kernel_cache_config.h", "kernel_spec.h", "platform.h", - "shared_memory_config.h", "stream.h", "stream_executor_internal.h", "trace_listener.h", @@ -455,7 +447,6 @@ cc_library( "stream_executor_internal.cc", ], hdrs = [ - "shared_memory_config.h", "stream_executor_internal.h", ], deps = [ @@ -484,7 +475,6 @@ cc_library( "dnn.h", "kernel.h", "kernel_cache_config.h", - "shared_memory_config.h", "stream_executor_pimpl.h", ], visibility = ["//visibility:public"], @@ -569,7 +559,6 @@ cc_library( "plugin.h", "plugin_registry.h", "rng.h", - "shared_memory_config.h", "stream.h", "stream_executor.h", "stream_executor_internal.h", @@ -619,7 +608,6 @@ cc_library( "plugin.h", "plugin_registry.h", "rng.h", - "shared_memory_config.h", "stream.h", "stream_executor.h", "stream_executor_internal.h", diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc index 79a027f1255..d649d00ded9 100644 --- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc +++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc @@ -101,12 +101,12 @@ static GpuTimer* AsGpuTimer(Timer* timer) { // N.B. we must lose constness in order to pass a suitable type to the existing // libcuda APIs, so the caller should take care to only pass the result of const // GPU memory conversions to libcuda functions which will honor constness. -static CUdeviceptr AsCudaDevicePtr(const DeviceMemoryBase &gpu_mem) { +static CUdeviceptr AsCudaDevicePtr(const DeviceMemoryBase& gpu_mem) { return reinterpret_cast(gpu_mem.opaque()); } // See description on const version above. -static CUdeviceptr AsCudaDevicePtr(DeviceMemoryBase *gpu_mem) { +static CUdeviceptr AsCudaDevicePtr(DeviceMemoryBase* gpu_mem) { return AsCudaDevicePtr(*gpu_mem); } @@ -225,11 +225,11 @@ port::Status GpuExecutor::LoadModuleFromCuBin(const char* cubin, if (*module == nullptr) { TF_RETURN_IF_ERROR(GpuDriver::LoadCubin(context_, cubin, module)); module_refcount = 1; - VLOG(3) << "Loaded CUBIN " << static_cast(cubin) + VLOG(3) << "Loaded CUBIN " << static_cast(cubin) << " as module " << *module; } else { ++module_refcount; - VLOG(3) << "CUBIN " << static_cast(cubin) + VLOG(3) << "CUBIN " << static_cast(cubin) << " is already loaded as module " << *module; } gpu_binary_to_module_[cubin] = {*module, module_refcount}; @@ -242,12 +242,12 @@ port::Status GpuExecutor::LoadModuleFromPtx(const char* ptx, CUmodule* module) { if (*module == nullptr) { TF_RETURN_IF_ERROR(GpuDriver::LoadPtx(context_, ptx, module)); - VLOG(3) << "Loaded PTX " << static_cast(ptx) << " as module " + VLOG(3) << "Loaded PTX " << static_cast(ptx) << " as module " << *module; module_refcount = 1; } else { ++module_refcount; - VLOG(3) << "PTX " << static_cast(ptx) + VLOG(3) << "PTX " << static_cast(ptx) << " is already loaded as module " << module; } gpu_binary_to_module_[ptx] = {*module, module_refcount}; @@ -271,7 +271,7 @@ port::Status GpuExecutor::GetKernel(const MultiKernelLoaderSpec& spec, if (spec.has_cuda_cubin_in_memory()) { absl::MutexLock lock{&in_memory_modules_mu_}; kernelname = &spec.cuda_cubin_in_memory().kernelname(); - const char *cubin = spec.cuda_cubin_in_memory().bytes(); + const char* cubin = spec.cuda_cubin_in_memory().bytes(); TF_RETURN_IF_ERROR(LoadModuleFromCuBin(cubin, &module)); kernel_to_gpu_binary_[kernel] = cubin; } else if (spec.has_cuda_ptx_in_memory()) { @@ -281,7 +281,7 @@ port::Status GpuExecutor::GetKernel(const MultiKernelLoaderSpec& spec, return port::InternalError("Compute capability not set"); } - const char *ptx = spec.cuda_ptx_in_memory().text(cc_major_, cc_minor_); + const char* ptx = spec.cuda_ptx_in_memory().text(cc_major_, cc_minor_); if (ptx == nullptr) { ptx = spec.cuda_ptx_in_memory().default_text(); } @@ -318,8 +318,8 @@ bool GpuExecutor::UnloadGpuBinary(const void* gpu_binary) { VLOG(3) << "No loaded CUDA module for " << gpu_binary; return false; } - auto &module = module_it->second.first; - auto &refcount = module_it->second.second; + auto& module = module_it->second.first; + auto& refcount = module_it->second.second; VLOG(3) << "Found CUDA module " << module << " with refcount " << refcount; if (--refcount == 0) { VLOG(3) << "Unloading CUDA module " << module; @@ -355,8 +355,8 @@ port::Status GpuExecutor::LoadModule(const MultiModuleLoaderSpec& spec, TF_RETURN_IF_ERROR(LoadModuleFromCuBin( reinterpret_cast(spec.cuda_cubin_in_memory().data()), &cu_module)); - *module_handle = ModuleHandle(const_cast( - static_cast(spec.cuda_cubin_in_memory().data()))); + *module_handle = ModuleHandle(const_cast( + static_cast(spec.cuda_cubin_in_memory().data()))); return port::Status::OK(); } else if (spec.has_cuda_ptx_in_memory()) { if (cc_major_ == 0 && cc_minor_ == 0) { @@ -370,15 +370,15 @@ port::Status GpuExecutor::LoadModule(const MultiModuleLoaderSpec& spec, absl::MutexLock lock{&in_memory_modules_mu_}; TF_RETURN_IF_ERROR( LoadModuleFromPtx(spec.cuda_ptx_in_memory(), &cu_module)); - *module_handle = ModuleHandle(const_cast( - static_cast(spec.cuda_ptx_in_memory()))); + *module_handle = ModuleHandle( + const_cast(static_cast(spec.cuda_ptx_in_memory()))); return port::Status::OK(); } return port::InternalError("No method of loading CUDA module provided"); } bool GpuExecutor::UnloadModule(ModuleHandle module_handle) { - const char *gpu_binary = reinterpret_cast(module_handle.id()); + const char* gpu_binary = reinterpret_cast(module_handle.id()); absl::MutexLock lock{&in_memory_modules_mu_}; return UnloadGpuBinary(gpu_binary); } @@ -425,7 +425,7 @@ port::Status GpuExecutor::Launch(Stream* stream, const ThreadDim& thread_dims, cufunc, cuda_kernel->GetGpuCacheConfig())); } - void **kernel_params = const_cast(args.argument_addresses().data()); + void** kernel_params = const_cast(args.argument_addresses().data()); return GpuDriver::LaunchKernel( context_, cufunc, block_dims.x, block_dims.y, block_dims.z, thread_dims.x, @@ -454,7 +454,7 @@ void GpuExecutor::VlogOccupancyInfo(const KernelBase& kernel, return; } - const DeviceDescription &device_description = + const DeviceDescription& device_description = kernel.parent()->GetDeviceDescription(); const GpuKernel* cuda_kernel = AsGpuKernel(&kernel); @@ -522,7 +522,7 @@ DeviceMemoryBase GpuExecutor::Allocate(uint64 size, int64 memory_space) { void* GpuExecutor::GetSubBuffer(DeviceMemoryBase* mem, uint64 offset_bytes, uint64 size_bytes) { // offset and size are in bytes, so char* works as the pointer type. - return reinterpret_cast(mem->opaque()) + offset_bytes; + return reinterpret_cast(mem->opaque()) + offset_bytes; } void GpuExecutor::Deallocate(DeviceMemoryBase* mem) { @@ -662,8 +662,8 @@ bool GpuExecutor::HostCallback(Stream* stream, /* static */ void GpuExecutor::InternalHostCallback(CUstream stream, CUresult status, void* data) { - std::function *callback = - reinterpret_cast *>(data); + std::function* callback = + reinterpret_cast*>(data); (*callback)(); delete callback; } @@ -744,7 +744,7 @@ port::Status GpuExecutor::BlockHostUntilDone(Stream* stream) { } blas::BlasSupport* GpuExecutor::CreateBlas() { - PluginRegistry *registry = PluginRegistry::Instance(); + PluginRegistry* registry = PluginRegistry::Instance(); port::StatusOr status = registry->GetFactory(cuda::kCudaPlatformId, plugin_config_.blas()); @@ -758,7 +758,7 @@ blas::BlasSupport* GpuExecutor::CreateBlas() { } dnn::DnnSupport* GpuExecutor::CreateDnn() { - PluginRegistry *registry = PluginRegistry::Instance(); + PluginRegistry* registry = PluginRegistry::Instance(); port::StatusOr status = registry->GetFactory(cuda::kCudaPlatformId, plugin_config_.dnn()); @@ -772,7 +772,7 @@ dnn::DnnSupport* GpuExecutor::CreateDnn() { } fft::FftSupport* GpuExecutor::CreateFft() { - PluginRegistry *registry = PluginRegistry::Instance(); + PluginRegistry* registry = PluginRegistry::Instance(); port::StatusOr status = registry->GetFactory(cuda::kCudaPlatformId, plugin_config_.fft()); @@ -786,7 +786,7 @@ fft::FftSupport* GpuExecutor::CreateFft() { } rng::RngSupport* GpuExecutor::CreateRng() { - PluginRegistry *registry = PluginRegistry::Instance(); + PluginRegistry* registry = PluginRegistry::Instance(); port::StatusOr status = registry->GetFactory(cuda::kCudaPlatformId, plugin_config_.rng()); @@ -812,47 +812,6 @@ port::Status GpuExecutor::EnablePeerAccessTo(StreamExecutorInterface* other) { return GpuDriver::EnablePeerAccess(context_, cuda_other->context_); } -SharedMemoryConfig GpuExecutor::GetDeviceSharedMemoryConfig() { - port::StatusOr cuda_config = - GpuDriver::ContextGetSharedMemConfig(context_); - if (!cuda_config.ok()) { - // Don't log; the failed call will log necessary output. - return SharedMemoryConfig::kDefault; - } - - switch (cuda_config.ValueOrDie()) { - case CU_SHARED_MEM_CONFIG_DEFAULT_BANK_SIZE: - return SharedMemoryConfig::kDefault; - case CU_SHARED_MEM_CONFIG_FOUR_BYTE_BANK_SIZE: - return SharedMemoryConfig::kFourByte; - case CU_SHARED_MEM_CONFIG_EIGHT_BYTE_BANK_SIZE: - return SharedMemoryConfig::kEightByte; - default: - LOG(FATAL) << "Invalid shared memory configuration returned: " - << cuda_config.ValueOrDie(); - } -} - -port::Status GpuExecutor::SetDeviceSharedMemoryConfig( - SharedMemoryConfig config) { - CUsharedconfig cuda_config; - switch (config) { - case SharedMemoryConfig::kDefault: - cuda_config = CU_SHARED_MEM_CONFIG_DEFAULT_BANK_SIZE; - break; - case SharedMemoryConfig::kFourByte: - cuda_config = CU_SHARED_MEM_CONFIG_FOUR_BYTE_BANK_SIZE; - break; - case SharedMemoryConfig::kEightByte: - cuda_config = CU_SHARED_MEM_CONFIG_EIGHT_BYTE_BANK_SIZE; - break; - default: - LOG(FATAL) << "Invalid shared memory configuration specified: " - << static_cast(config); - } - return GpuDriver::ContextSetSharedMemConfig(context_, cuda_config); -} - bool GpuExecutor::DeviceMemoryUsage(int64* free, int64* total) const { return GpuDriver::GetDeviceMemoryInfo(context_, free, total); } @@ -875,7 +834,7 @@ bool GpuExecutor::GetSymbol(const std::string& symbol_name, return lookup_in_module(it->second.first); } - for (auto &it : gpu_binary_to_module_) { + for (auto& it : gpu_binary_to_module_) { if (lookup_in_module(it.second.first)) { return true; } @@ -963,7 +922,7 @@ static int TryToReadNumaNode(const std::string& pci_bus_id, // We have to use fopen/fread here so that the device properties can be // populated before InitGoogle procedure has been completed (at which point we // could use the file::* utilities). - FILE *file = fopen(filename.c_str(), "r"); + FILE* file = fopen(filename.c_str(), "r"); if (file == nullptr) { LOG(ERROR) << "could not open file to read NUMA node: " << filename << "\nYour kernel may have been built without NUMA support."; @@ -980,8 +939,9 @@ static int TryToReadNumaNode(const std::string& pci_bus_id, if (port::safe_strto32(content, &value)) { if (value < 0) { // See http://b/18228951 for details on this path. LOG(INFO) << "successful NUMA node read from SysFS had negative value (" - << value << "), but there must be at least one NUMA node" - ", so returning NUMA node zero"; + << value + << "), but there must be at least one NUMA node" + ", so returning NUMA node zero"; fclose(file); return 0; } diff --git a/tensorflow/stream_executor/gpu/gpu_executor.h b/tensorflow/stream_executor/gpu/gpu_executor.h index fc4ea0e0ab2..edc015c6126 100644 --- a/tensorflow/stream_executor/gpu/gpu_executor.h +++ b/tensorflow/stream_executor/gpu/gpu_executor.h @@ -188,10 +188,6 @@ class GpuExecutor : public internal::StreamExecutorInterface { bool CanEnablePeerAccessTo(StreamExecutorInterface* other) override; - SharedMemoryConfig GetDeviceSharedMemoryConfig() override; - - port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override; - bool DeviceMemoryUsage(int64* free, int64* total) const override; // Search for the symbol and returns a device pointer and size. diff --git a/tensorflow/stream_executor/host/host_gpu_executor.h b/tensorflow/stream_executor/host/host_gpu_executor.h index 9b896fe06f8..953f8ced47f 100644 --- a/tensorflow/stream_executor/host/host_gpu_executor.h +++ b/tensorflow/stream_executor/host/host_gpu_executor.h @@ -148,20 +148,6 @@ class HostExecutor : public internal::StreamExecutorInterface { return true; } - SharedMemoryConfig GetDeviceSharedMemoryConfig() override { - LOG(INFO) << "Shared memory configuration is unsupported for host " - << "executors."; - return SharedMemoryConfig::kDefault; - } - - port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config) override { - std::string error_msg{ - "Shared memory configuration is unsupported for host " - "executors."}; - LOG(INFO) << error_msg; - return port::Status(port::error::UNIMPLEMENTED, error_msg); - } - bool SupportsBlas() const override; blas::BlasSupport *CreateBlas() override; diff --git a/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc b/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc index fd3b5f19913..2a85cb820ed 100644 --- a/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc +++ b/tensorflow/stream_executor/rocm/rocm_gpu_executor.cc @@ -720,47 +720,6 @@ port::Status GpuExecutor::EnablePeerAccessTo(StreamExecutorInterface* other) { return GpuDriver::EnablePeerAccess(context_, rocm_other->context_); } -SharedMemoryConfig GpuExecutor::GetDeviceSharedMemoryConfig() { - port::StatusOr rocm_config = - GpuDriver::ContextGetSharedMemConfig(context_); - if (!rocm_config.ok()) { - // Don't log; the failed call will log necessary output. - return SharedMemoryConfig::kDefault; - } - - switch (rocm_config.ValueOrDie()) { - case hipSharedMemBankSizeDefault: - return SharedMemoryConfig::kDefault; - case hipSharedMemBankSizeFourByte: - return SharedMemoryConfig::kFourByte; - case hipSharedMemBankSizeEightByte: - return SharedMemoryConfig::kEightByte; - default: - LOG(FATAL) << "Invalid shared memory configuration returned: " - << rocm_config.ValueOrDie(); - } -} - -port::Status GpuExecutor::SetDeviceSharedMemoryConfig( - SharedMemoryConfig config) { - hipSharedMemConfig rocm_config; - switch (config) { - case SharedMemoryConfig::kDefault: - rocm_config = hipSharedMemBankSizeDefault; - break; - case SharedMemoryConfig::kFourByte: - rocm_config = hipSharedMemBankSizeFourByte; - break; - case SharedMemoryConfig::kEightByte: - rocm_config = hipSharedMemBankSizeEightByte; - break; - default: - LOG(FATAL) << "Invalid shared memory configuration specified: " - << static_cast(config); - } - return GpuDriver::ContextSetSharedMemConfig(context_, rocm_config); -} - bool GpuExecutor::DeviceMemoryUsage(int64* free, int64* total) const { return GpuDriver::GetDeviceMemoryInfo(context_, free, total); } @@ -768,24 +727,24 @@ bool GpuExecutor::DeviceMemoryUsage(int64* free, int64* total) const { bool GpuExecutor::GetSymbol(const string& symbol_name, ModuleHandle module_handle, void** mem, size_t* bytes) { - absl::MutexLock lock{&in_memory_modules_mu_}; - if (static_cast(module_handle)) { - auto it = gpu_binary_to_module_.find(module_handle.id()); - CHECK(it != gpu_binary_to_module_.end()); - if (GpuDriver::GetModuleSymbol( - context_, it->second.first, symbol_name.c_str(), - reinterpret_cast(mem), bytes)) { - return true; - } + absl::MutexLock lock{&in_memory_modules_mu_}; + if (static_cast(module_handle)) { + auto it = gpu_binary_to_module_.find(module_handle.id()); + CHECK(it != gpu_binary_to_module_.end()); + if (GpuDriver::GetModuleSymbol( + context_, it->second.first, symbol_name.c_str(), + reinterpret_cast(mem), bytes)) { + return true; } + } - for (auto& it : gpu_binary_to_module_) { - if (GpuDriver::GetModuleSymbol( - context_, it.second.first, symbol_name.c_str(), - reinterpret_cast(mem), bytes)) { - return true; - } + for (auto& it : gpu_binary_to_module_) { + if (GpuDriver::GetModuleSymbol( + context_, it.second.first, symbol_name.c_str(), + reinterpret_cast(mem), bytes)) { + return true; } + } LOG(INFO) << "Falied to find symbol in any modules: " << symbol_name; return false; diff --git a/tensorflow/stream_executor/shared_memory_config.h b/tensorflow/stream_executor/shared_memory_config.h deleted file mode 100644 index 7cbeb3bcd91..00000000000 --- a/tensorflow/stream_executor/shared_memory_config.h +++ /dev/null @@ -1,34 +0,0 @@ -/* Copyright 2015 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// This file defines a uniform interface to configuration options for shared -// memory for supported devices. As with many StreamExecutor-supported features, -// support for the options defined herein is device-dependent. -#ifndef TENSORFLOW_STREAM_EXECUTOR_SHARED_MEMORY_CONFIG_H_ -#define TENSORFLOW_STREAM_EXECUTOR_SHARED_MEMORY_CONFIG_H_ - -namespace stream_executor { - -// SharedMemoryConfig enum describes potential widths of shared memory banks for -// a device or kernel. -enum class SharedMemoryConfig { - kDefault, // Use the device default configuration. - kFourByte, // Sets shared memory banks to be four bytes wide. - kEightByte, // Sets shared memory banks to be eight bytes wide. -}; - -} // namespace stream_executor - -#endif // TENSORFLOW_STREAM_EXECUTOR_SHARED_MEMORY_CONFIG_H_ diff --git a/tensorflow/stream_executor/stream_executor_internal.h b/tensorflow/stream_executor/stream_executor_internal.h index 408b4fc8207..437338085b3 100644 --- a/tensorflow/stream_executor/stream_executor_internal.h +++ b/tensorflow/stream_executor/stream_executor_internal.h @@ -44,7 +44,6 @@ limitations under the License. #include "tensorflow/stream_executor/platform.h" #include "tensorflow/stream_executor/platform/port.h" #include "tensorflow/stream_executor/plugin_registry.h" -#include "tensorflow/stream_executor/shared_memory_config.h" #include "tensorflow/stream_executor/trace_listener.h" namespace stream_executor { @@ -267,9 +266,6 @@ class StreamExecutorInterface { virtual int PlatformDeviceCount() = 0; virtual port::Status EnablePeerAccessTo(StreamExecutorInterface *other) = 0; virtual bool CanEnablePeerAccessTo(StreamExecutorInterface *other) = 0; - virtual SharedMemoryConfig GetDeviceSharedMemoryConfig() = 0; - virtual port::Status SetDeviceSharedMemoryConfig( - SharedMemoryConfig config) = 0; virtual int64 GetDeviceLoad() { return -1; } diff --git a/tensorflow/stream_executor/stream_executor_pimpl.cc b/tensorflow/stream_executor/stream_executor_pimpl.cc index d23f1472e33..db4e8f9b694 100644 --- a/tensorflow/stream_executor/stream_executor_pimpl.cc +++ b/tensorflow/stream_executor/stream_executor_pimpl.cc @@ -230,23 +230,6 @@ port::Status StreamExecutor::EnablePeerAccessTo(StreamExecutor *other) { return implementation_->EnablePeerAccessTo(other->implementation_.get()); } -SharedMemoryConfig StreamExecutor::GetDeviceSharedMemoryConfig() { - return implementation_->GetDeviceSharedMemoryConfig(); -} - -port::Status StreamExecutor::SetDeviceSharedMemoryConfig( - SharedMemoryConfig config) { - if (config != SharedMemoryConfig::kDefault && - config != SharedMemoryConfig::kFourByte && - config != SharedMemoryConfig::kEightByte) { - std::string error_msg = absl::StrFormat( - "Invalid shared memory config specified: %d", static_cast(config)); - LOG(ERROR) << error_msg; - return port::Status(port::error::INVALID_ARGUMENT, error_msg); - } - return implementation_->SetDeviceSharedMemoryConfig(config); -} - const DeviceDescription &StreamExecutor::GetDeviceDescription() const { absl::MutexLock lock(&mu_); if (device_description_ != nullptr) { @@ -858,7 +841,7 @@ absl::optional StreamExecutor::GetAllocatorStats() { } template -void StreamExecutor::SubmitTrace(TraceCallT trace_call, ArgsT &&... args) { +void StreamExecutor::SubmitTrace(TraceCallT trace_call, ArgsT &&...args) { if (tracing_enabled_) { { // instance tracers held in a block to limit the lock lifetime. diff --git a/tensorflow/stream_executor/stream_executor_pimpl.h b/tensorflow/stream_executor/stream_executor_pimpl.h index f7f69f78e89..b9b118ca42c 100644 --- a/tensorflow/stream_executor/stream_executor_pimpl.h +++ b/tensorflow/stream_executor/stream_executor_pimpl.h @@ -35,7 +35,6 @@ limitations under the License. #include "tensorflow/stream_executor/platform/logging.h" #include "tensorflow/stream_executor/platform/port.h" #include "tensorflow/stream_executor/rng.h" -#include "tensorflow/stream_executor/shared_memory_config.h" #include "tensorflow/stream_executor/stream.h" #include "tensorflow/stream_executor/stream_executor_internal.h" #include "tensorflow/stream_executor/trace_listener.h" @@ -54,8 +53,8 @@ struct AllocRecord { }; // Forward declaration of private friend class. -template +template class ScopedTracer; // A StreamExecutor manages a single device, in terms of executing work (kernel @@ -322,14 +321,6 @@ class StreamExecutor { // this is more an up-front test as to whether it's expressly forbidden. bool CanEnablePeerAccessTo(StreamExecutor *other); - // Gets the preferred shared memory configuration for the device to which this - // executor is bound. - SharedMemoryConfig GetDeviceSharedMemoryConfig(); - - // Sets the preferred shared memory configuration for the device to which this - // executor is bound. - port::Status SetDeviceSharedMemoryConfig(SharedMemoryConfig config); - // Obtains metadata about the underlying device. // The value is cached on first use. const DeviceDescription &GetDeviceDescription() const; @@ -507,12 +498,12 @@ class StreamExecutor { // To register a listener for all executors for a given platform, see // Platform::RegisterTraceListener(). // Does not take ownership of listener. - void RegisterTraceListener(TraceListener* listener); + void RegisterTraceListener(TraceListener *listener); // Removes a TraceListener from this StreamExecutor instance. // Returns false (and logs) in cases where the argument listener was not // previously registered. - bool UnregisterTraceListener(TraceListener* listener); + bool UnregisterTraceListener(TraceListener *listener); // Return allocator statistics. absl::optional GetAllocatorStats(); @@ -522,8 +513,8 @@ class StreamExecutor { StreamExecutorMemoryAllocator *GetAllocator() { return &allocator_; } private: - template + template friend class ScopedTracer; friend class Event; friend class Stream; @@ -648,7 +639,7 @@ class StreamExecutor { // Calls the relevant TraceListener routine to begin tracing for the specified // asynchronous method. template - void SubmitTrace(TraceCallT trace_call, ArgsT&&... args); + void SubmitTrace(TraceCallT trace_call, ArgsT &&...args); // Reader/writer lock for class-static StreamExecutor members. static absl::Mutex static_mu_; diff --git a/tensorflow/stream_executor/tpu/tpu_executor.h b/tensorflow/stream_executor/tpu/tpu_executor.h index faeae86da9b..2430a350463 100644 --- a/tensorflow/stream_executor/tpu/tpu_executor.h +++ b/tensorflow/stream_executor/tpu/tpu_executor.h @@ -96,8 +96,7 @@ class TpuExecutor : public tensorflow::tpu::TpuExecutorInterface { void DequeueOutfeed(int32 outfeed_queue_index, absl::Span bytes, StatusCallback done); - Status EnqueueInfeed(int32 infeed_queue_index, - absl::Span bytes); + Status EnqueueInfeed(int32 infeed_queue_index, absl::Span bytes); absl::optional GetAllocatorStats() override; @@ -175,10 +174,6 @@ class TpuExecutor : public tensorflow::tpu::TpuExecutorInterface { LOG(FATAL) << "Not yet implemented"; } - stream_executor::SharedMemoryConfig GetDeviceSharedMemoryConfig() override { - LOG(FATAL) << "not yet implemented"; - } - void* GetSubBuffer(DeviceMemoryBase* parent, uint64 offset, uint64 size) override { LOG(FATAL) << "not yet implemented"; @@ -197,10 +192,7 @@ class TpuExecutor : public tensorflow::tpu::TpuExecutorInterface { bool CanEnablePeerAccessTo(StreamExecutorInterface* other) override { LOG(FATAL) << "not yet implemented"; } - Status SetDeviceSharedMemoryConfig( - stream_executor::SharedMemoryConfig config) override { - LOG(FATAL) << "not yet implemented"; - } + void* HostMemoryAllocate(uint64 size) override { LOG(FATAL) << "not yet implemented"; } From f63356e8176c203eedbebf90665882fd8167b4c0 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Tue, 11 Aug 2020 20:46:52 -0700 Subject: [PATCH 0871/1017] Disable failed tests for the moment. PiperOrigin-RevId: 326159497 Change-Id: I48be692e03db171d7ff5cac3b016434118040211 --- tensorflow/lite/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD index 7007a847d83..5f8923bfd51 100644 --- a/tensorflow/lite/BUILD +++ b/tensorflow/lite/BUILD @@ -408,6 +408,7 @@ cc_test( ], features = ["-dynamic_link_test_srcs"], # see go/dynamic_link_test_srcs tags = [ + "noasan", # b/163674257 "tflite_not_portable_ios", # TODO(b/117786830) "tflite_smoke_test", ], From 38f0168da11ad93dc9351a03188844074a4fc323 Mon Sep 17 00:00:00 2001 From: Jaesung Chung Date: Tue, 11 Aug 2020 20:47:43 -0700 Subject: [PATCH 0872/1017] Move TF Broadcast op legalization process to the prepare_tf stage This change is to get benefits from the constant folding logic from TF dialect. PiperOrigin-RevId: 326159603 Change-Id: I482d698822ec0a1efb4347fbe149fda9058146ca --- tensorflow/compiler/mlir/lite/BUILD | 23 ++++ .../legalize-tf-no-runtime-verification.mlir | 7 +- .../compiler/mlir/lite/tests/legalize-tf.mlir | 22 ---- .../compiler/mlir/lite/tests/prepare-tf.mlir | 20 ++++ .../mlir/lite/transforms/legalize_tf.cc | 112 +----------------- .../mlir/lite/transforms/prepare_tf.cc | 45 ++++++- .../mlir/lite/utils/constant_utils.cc | 112 ++++++++++++++++++ .../compiler/mlir/lite/utils/constant_utils.h | 35 ++++++ 8 files changed, 239 insertions(+), 137 deletions(-) create mode 100644 tensorflow/compiler/mlir/lite/utils/constant_utils.cc create mode 100644 tensorflow/compiler/mlir/lite/utils/constant_utils.h diff --git a/tensorflow/compiler/mlir/lite/BUILD b/tensorflow/compiler/mlir/lite/BUILD index 555c11779f5..bd1dcdf06ea 100644 --- a/tensorflow/compiler/mlir/lite/BUILD +++ b/tensorflow/compiler/mlir/lite/BUILD @@ -237,6 +237,28 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "constant_utils", + srcs = [ + "utils/constant_utils.cc", + ], + hdrs = [ + "utils/constant_utils.h", + ], + copts = ["-std=c++14"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:mangling_util", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/platform:status", + "//tensorflow/stream_executor/lib", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:StandardOps", + "@llvm-project//mlir:Support", + ], +) + cc_library( name = "lstm_utils", srcs = [ @@ -347,6 +369,7 @@ cc_library( "transforms/passes.h", ], deps = [ + ":constant_utils", ":lstm_utils", ":stateful_ops_utils", ":tensorflow_lite", diff --git a/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir b/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir index 90266b4e78e..3c390df74b4 100644 --- a/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir +++ b/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir @@ -1,12 +1,11 @@ -// RUN: tf-opt %s -tfl-legalize-tf='run-tfl-runtime-verification=false' | FileCheck %s +// RUN: tf-opt %s -tfl-prepare-tf -tfl-legalize-tf='run-tfl-runtime-verification=false' | FileCheck %s func @broadcast_to_bf16(%arg0: tensor<3xbf16>, %arg1: tensor<2xi64>) -> tensor<3x3xbf16> { %0 = "tf.BroadcastTo"(%arg0, %arg1) : (tensor<3xbf16>, tensor<2xi64>) -> tensor<3x3xbf16> return %0: tensor<3x3xbf16> // CHECK-LABEL: broadcast_to_bf16 -// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor -// CHECK: [[FILL:%.*]] = "tfl.fill"(%arg1, [[CST]]) : (tensor<2xi64>, tensor) -> tensor<3x3xbf16> -// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[FILL]]) {fused_activation_function = "NONE"} : (tensor<3xbf16>, tensor<3x3xbf16>) -> tensor<3x3xbf16> +// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor<3x3xbf16> +// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[CST]]) {fused_activation_function = "NONE"} : (tensor<3xbf16>, tensor<3x3xbf16>) -> tensor<3x3xbf16> // CHECK: return [[MUL]] : tensor<3x3xbf16> } diff --git a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir index 7cb9c4dd22c..d02e4e705f4 100644 --- a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir @@ -1482,28 +1482,6 @@ func @UnidirectionalRnn(%arg: tensor<28x1x28xf32>) -> (tensor<28x1x28xf32>) { // CHECK: return [[VAL_4]] : tensor<28x1x28xf32> // CHECK: } -func @broadcast_to_f32(%arg0: tensor<3xf32>, %arg1: tensor<2xi32>) -> tensor<3x3xf32> { - %0 = "tf.BroadcastTo"(%arg0, %arg1) : (tensor<3xf32>, tensor<2xi32>) -> tensor<3x3xf32> - return %0: tensor<3x3xf32> - -// CHECK-LABEL: broadcast_to_f32 -// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor -// CHECK: [[FILL:%.*]] = "tfl.fill"(%arg1, [[CST]]) : (tensor<2xi32>, tensor) -> tensor<3x3xf32> -// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[FILL]]) {fused_activation_function = "NONE"} : (tensor<3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> -// CHECK: return [[MUL]] : tensor<3x3xf32> -} - -func @broadcast_to_i32(%input: tensor<3xi32>, %shape: tensor<2xi32>) -> tensor<3x3xi32> { - %0 = "tf.BroadcastTo"(%input, %shape) : (tensor<3xi32>, tensor<2xi32>) -> tensor<3x3xi32> - return %0: tensor<3x3xi32> - -// CHECK-LABEL: broadcast_to_i32 -// CHECK: [[CST:%.*]] = constant dense<1> : tensor -// CHECK: [[FILL:%.*]] = "tfl.fill"(%arg1, [[CST]]) : (tensor<2xi32>, tensor) -> tensor<3x3xi32> -// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[FILL]]) {fused_activation_function = "NONE"} : (tensor<3xi32>, tensor<3x3xi32>) -> tensor<3x3xi32> -// CHECK: return [[MUL]] : tensor<3x3xi32> -} - func @matmul_batch(%arg0: tensor<10x15xf32>, %arg1: tensor<15x17xf32>) -> tensor<10x17xf32> { %0 = "tf.BatchMatMul"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT", device = "/device:CPU:0", name = "MatMul", adj_x = false, adj_y = false} : (tensor<10x15xf32>, tensor<15x17xf32>) -> tensor<10x17xf32> diff --git a/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir b/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir index 066139e179b..6ee5b67d65e 100644 --- a/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir @@ -595,4 +595,24 @@ func @xla_conv(%arg0: tensor<4x8x8x16xf32>) -> tensor<4x8x8x16xf32> { // CHECK: return %[[RES]] } +func @broadcast_to_f32(%arg0: tensor<3xf32>, %arg1: tensor<2xi32>) -> tensor<3x3xf32> { + %0 = "tf.BroadcastTo"(%arg0, %arg1) : (tensor<3xf32>, tensor<2xi32>) -> tensor<3x3xf32> + return %0: tensor<3x3xf32> + +// CHECK-LABEL: broadcast_to_f32 +// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor<3x3xf32> +// CHECK: [[MUL:%.*]] = "tf.Mul"(%arg0, [[CST]]) : (tensor<3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK: return [[MUL]] : tensor<3x3xf32> +} + +func @broadcast_to_i32(%input: tensor<3xi32>, %shape: tensor<2xi32>) -> tensor<3x3xi32> { + %0 = "tf.BroadcastTo"(%input, %shape) : (tensor<3xi32>, tensor<2xi32>) -> tensor<3x3xi32> + return %0: tensor<3x3xi32> + +// CHECK-LABEL: broadcast_to_i32 +// CHECK: [[CST:%.*]] = constant dense<1> : tensor<3x3xi32> +// CHECK: [[MUL:%.*]] = "tf.Mul"(%arg0, [[CST]]) : (tensor<3xi32>, tensor<3x3xi32>) -> tensor<3x3xi32> +// CHECK: return [[MUL]] : tensor<3x3xi32> +} + } diff --git a/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc b/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc index 7a16e475ce3..297b1459fc5 100644 --- a/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc +++ b/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc @@ -45,6 +45,7 @@ limitations under the License. #include "tensorflow/compiler/mlir/lite/quantization/quantization_utils.h" #include "tensorflow/compiler/mlir/lite/transforms/passes.h" #include "tensorflow/compiler/mlir/lite/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/lite/utils/constant_utils.h" #include "tensorflow/compiler/mlir/lite/utils/validators.h" #include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" #include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" @@ -137,7 +138,6 @@ DECL_CONVERT_OP(StridedSlice); DECL_CONVERT_OP(Unpack); DECL_CONVERT_OP(Reciprocal); DECL_CONVERT_OP(RandomUniform); -DECL_CONVERT_OP(BroadcastTo); #undef DECL_CONVERT_OP @@ -483,89 +483,6 @@ LogicalResult ConvertTFAssertOp::matchAndRewrite( return success(); } -StatusOr CreateConstOpWithSingleValue(PatternRewriter* rewriter, - Location loc, - ShapedType shaped_type, - int value) { - Type element_type = shaped_type.getElementType(); - ShapedType scalar_type = RankedTensorType::get({}, element_type); - Attribute attr; - switch (element_type.getKind()) { - case mlir::StandardTypes::F16: { - auto floatType = mlir::FloatType::getF16(element_type.getContext()); - auto floatAttr = - mlir::FloatAttr::get(floatType, static_cast(value)); - std::vector floatValues({floatAttr}); - attr = DenseElementsAttr::get(scalar_type, floatValues); - break; - } - case mlir::StandardTypes::BF16: { - auto floatType = mlir::FloatType::getBF16(element_type.getContext()); - auto floatAttr = - mlir::FloatAttr::get(floatType, static_cast(value)); - std::vector floatValues({floatAttr}); - attr = DenseElementsAttr::get(scalar_type, floatValues); - break; - } - case mlir::StandardTypes::F32: { - attr = - DenseElementsAttr::get(scalar_type, static_cast(value)); - break; - } - case mlir::StandardTypes::Complex: { - auto etype = element_type.cast().getElementType(); - if (etype.isF32()) { - auto dialect = etype.getContext()->getRegisteredDialect("tf"); - tensorflow::TensorProto repr; - repr.set_dtype(tensorflow::DT_COMPLEX64); - - tensorflow::TensorShapeProto* shape = repr.mutable_tensor_shape(); - shape->set_unknown_rank(false); - shape->add_dim()->set_size(int64_t{1}); - std::string content; - auto complex_value = - std::complex(static_cast(value), 0.0f); - content.assign(reinterpret_cast(&complex_value), - sizeof(complex_value)); - repr.set_tensor_content(content); - std::string mangled = tensorflow::mangling_util::MangleTensor(repr); - - attr = mlir::OpaqueElementsAttr::get(dialect, scalar_type, mangled); - break; - } - return Status(tensorflow::error::INVALID_ARGUMENT, "Unsupported type"); - } - case mlir::StandardTypes::Integer: { - const auto& itype = element_type.cast(); - switch (itype.getWidth()) { - case 8: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - case 16: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - case 32: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - case 64: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - default: - return Status(tensorflow::error::INVALID_ARGUMENT, - "Unsupported type"); - } - break; - } - default: - return Status(tensorflow::error::INVALID_ARGUMENT, "Unsupported type"); - } - return rewriter->create(loc, scalar_type, attr); -} - LogicalResult ConvertTFReciprocalOp::matchAndRewrite( Operation* op, PatternRewriter& rewriter) const { auto tf_reciprocal_op = cast(op); @@ -586,31 +503,6 @@ LogicalResult ConvertTFReciprocalOp::matchAndRewrite( return success(); } -LogicalResult ConvertTFBroadcastToOp::matchAndRewrite( - Operation* op, PatternRewriter& rewriter) const { - auto tf_broadcast_to_op = cast(op); - auto element_type = tf_broadcast_to_op.input().getType().cast(); - auto output_type = tf_broadcast_to_op.output().getType(); - - auto status_or_const_op = - CreateConstOpWithSingleValue(&rewriter, op->getLoc(), element_type, 1); - if (!status_or_const_op.ok()) { - return failure(); - } - - auto tfl_fill_op = rewriter.create( - op->getLoc(), output_type, tf_broadcast_to_op.shape(), - status_or_const_op.ValueOrDie()); - - StringAttr fused_activation_function = - StringAttr::get("NONE", rewriter.getContext()); - - rewriter.replaceOpWithNewOp( - op, output_type, tf_broadcast_to_op.input(), tfl_fill_op, - fused_activation_function); - return success(); -} - // Legalize unidirectional sequence lstm. struct LegalizeUnidirectionalSequenceLstm : public RewritePattern { explicit LegalizeUnidirectionalSequenceLstm(MLIRContext* context) @@ -751,7 +643,7 @@ void LegalizeTF::runOnFunction() { ConvertTFMatrixDiagV3Op, ConvertTFPackOp, ConvertTFReshapeOp, ConvertTFSplitOp, ConvertTFSplitVOp, ConvertTFStridedSliceOp, ConvertTFUnpackOp, ConvertTFAssertOp, ConvertTFReciprocalOp, - ConvertTFRandomUniformOp, ConvertTFBroadcastToOp>(context); + ConvertTFRandomUniformOp>(context); // Ophint python converter converted tf node pattern. patterns.insert(op); + auto input_type = tf_broadcast_to_op.input().getType().cast(); + auto output_type = tf_broadcast_to_op.output().getType().cast(); + auto shape_type = tf_broadcast_to_op.shape().getType().cast(); + Type element_type = input_type.getElementType(); + + // Allow lowering when low dimension inputs are given and its type is F32 or + // I32. + if (!((output_type.hasRank() && output_type.getRank() <= 4) || + (shape_type.hasStaticShape() && shape_type.getRank() == 1 && + shape_type.getDimSize(0) <= 4))) + return failure(); + + if (!((element_type.getKind() == mlir::StandardTypes::F32) || + (element_type.getKind() == mlir::StandardTypes::BF16) || + (element_type.getKind() == mlir::StandardTypes::Integer && + element_type.cast().getWidth() == 32))) + return failure(); + + auto status_or_const_op = + CreateConstOpWithSingleValue(&rewriter, op->getLoc(), input_type, 1); + if (!status_or_const_op.ok()) { + return failure(); + } + + auto tf_fill_op = rewriter.create( + op->getLoc(), output_type, tf_broadcast_to_op.shape(), + status_or_const_op.ValueOrDie()); + + auto mul_op = rewriter.create( + op->getLoc(), output_type, tf_broadcast_to_op.input(), tf_fill_op); + rewriter.replaceOp(op, mul_op.getResult()); + return success(); + } +}; + #include "tensorflow/compiler/mlir/lite/transforms/generated_prepare_tf.inc" // Returns success if all the operations in the `op`'s regions including `op` @@ -767,7 +810,7 @@ void PrepareTFPass::runOnFunction() { patterns.insert, TF::ConvertTFBatchMatMulOp>(ctx); } - patterns.insert(ctx); applyPatternsAndFoldGreedily(func, patterns); } diff --git a/tensorflow/compiler/mlir/lite/utils/constant_utils.cc b/tensorflow/compiler/mlir/lite/utils/constant_utils.cc new file mode 100644 index 00000000000..8562f623258 --- /dev/null +++ b/tensorflow/compiler/mlir/lite/utils/constant_utils.cc @@ -0,0 +1,112 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/lite/utils/constant_utils.h" + +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/platform/status.h" + +namespace mlir { +namespace TFL { + +stream_executor::port::StatusOr CreateConstOpWithSingleValue( + PatternRewriter* rewriter, Location loc, ShapedType shaped_type, + int value) { + Type element_type = shaped_type.getElementType(); + ShapedType scalar_type = RankedTensorType::get({}, element_type); + Attribute attr; + switch (element_type.getKind()) { + case mlir::StandardTypes::F16: { + auto floatType = mlir::FloatType::getF16(element_type.getContext()); + auto floatAttr = + mlir::FloatAttr::get(floatType, static_cast(value)); + std::vector floatValues({floatAttr}); + attr = DenseElementsAttr::get(scalar_type, floatValues); + break; + } + case mlir::StandardTypes::BF16: { + auto floatType = mlir::FloatType::getBF16(element_type.getContext()); + auto floatAttr = + mlir::FloatAttr::get(floatType, static_cast(value)); + std::vector floatValues({floatAttr}); + attr = DenseElementsAttr::get(scalar_type, floatValues); + break; + } + case mlir::StandardTypes::F32: { + attr = + DenseElementsAttr::get(scalar_type, static_cast(value)); + break; + } + case mlir::StandardTypes::Complex: { + auto etype = element_type.cast().getElementType(); + if (etype.isF32()) { + auto dialect = etype.getContext()->getRegisteredDialect("tf"); + tensorflow::TensorProto repr; + repr.set_dtype(tensorflow::DT_COMPLEX64); + + tensorflow::TensorShapeProto* shape = repr.mutable_tensor_shape(); + shape->set_unknown_rank(false); + shape->add_dim()->set_size(int64_t{1}); + std::string content; + auto complex_value = + std::complex(static_cast(value), 0.0f); + content.assign(reinterpret_cast(&complex_value), + sizeof(complex_value)); + repr.set_tensor_content(content); + std::string mangled = tensorflow::mangling_util::MangleTensor(repr); + + attr = mlir::OpaqueElementsAttr::get(dialect, scalar_type, mangled); + break; + } + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Unsupported type"); + } + case mlir::StandardTypes::Integer: { + const auto& itype = element_type.cast(); + switch (itype.getWidth()) { + case 8: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + case 16: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + case 32: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + case 64: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + default: + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Unsupported type"); + } + break; + } + default: + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Unsupported type"); + } + return rewriter->create(loc, scalar_type, attr); +} + +} // namespace TFL +} // namespace mlir diff --git a/tensorflow/compiler/mlir/lite/utils/constant_utils.h b/tensorflow/compiler/mlir/lite/utils/constant_utils.h new file mode 100644 index 00000000000..5c348021b5e --- /dev/null +++ b/tensorflow/compiler/mlir/lite/utils/constant_utils.h @@ -0,0 +1,35 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_LITE_UTILS_CONSTANT_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_LITE_UTILS_CONSTANT_UTILS_H_ + +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/StandardTypes.h" // from @llvm-project +#include "tensorflow/stream_executor/lib/statusor.h" + +namespace mlir { +namespace TFL { + +// Returns a Constant op with a single value. +stream_executor::port::StatusOr CreateConstOpWithSingleValue( + PatternRewriter* rewriter, Location loc, ShapedType shaped_type, int value); + +} // namespace TFL +} // namespace mlir +#endif // TENSORFLOW_COMPILER_MLIR_LITE_UTILS_CONSTANT_UTILS_H_ From 916bd91c2bc1ded18ca460520e922a71c3033418 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 21:18:44 -0700 Subject: [PATCH 0873/1017] Fix a typo reported in https://github.com/tensorflow/tensorflow/issues/42217 PiperOrigin-RevId: 326163132 Change-Id: I2c093e562e457edf80df8cace2aee5671d6e17b6 --- tensorflow/lite/experimental/ios/BUILD.apple | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/lite/experimental/ios/BUILD.apple b/tensorflow/lite/experimental/ios/BUILD.apple index e1e3be2bcde..8524eab8fa4 100644 --- a/tensorflow/lite/experimental/ios/BUILD.apple +++ b/tensorflow/lite/experimental/ios/BUILD.apple @@ -79,7 +79,7 @@ ios_static_framework( # TensorFlow Lite runtime, it is intended to be linked along with the # TensorFlowLiteC framework above in a composable way. # -# bazel build -c opt --config=ios_fat //tensorflow/lite/experimental/ios:TensorFlowLiteCCoreMl_framework +# bazel build -c opt --config=ios_fat //tensorflow/lite/experimental/ios:TensorFlowLiteCCoreML_framework tflite_ios_static_framework( name = "TensorFlowLiteCCoreML_framework", hdrs = [ From 57f61ed3fd1b1d9aaf75351b08132cd553369795 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 22:15:24 -0700 Subject: [PATCH 0874/1017] Move TF Broadcast op legalization process to the prepare_tf stage This change is to get benefits from the constant folding logic from TF dialect. PiperOrigin-RevId: 326169849 Change-Id: I432eeb902b1ace379388cbd1870e2cf35b828b17 --- tensorflow/compiler/mlir/lite/BUILD | 23 ---- .../legalize-tf-no-runtime-verification.mlir | 7 +- .../compiler/mlir/lite/tests/legalize-tf.mlir | 22 ++++ .../compiler/mlir/lite/tests/prepare-tf.mlir | 20 ---- .../mlir/lite/transforms/legalize_tf.cc | 112 +++++++++++++++++- .../mlir/lite/transforms/prepare_tf.cc | 45 +------ .../mlir/lite/utils/constant_utils.cc | 112 ------------------ .../compiler/mlir/lite/utils/constant_utils.h | 35 ------ 8 files changed, 137 insertions(+), 239 deletions(-) delete mode 100644 tensorflow/compiler/mlir/lite/utils/constant_utils.cc delete mode 100644 tensorflow/compiler/mlir/lite/utils/constant_utils.h diff --git a/tensorflow/compiler/mlir/lite/BUILD b/tensorflow/compiler/mlir/lite/BUILD index bd1dcdf06ea..555c11779f5 100644 --- a/tensorflow/compiler/mlir/lite/BUILD +++ b/tensorflow/compiler/mlir/lite/BUILD @@ -237,28 +237,6 @@ cc_library( alwayslink = 1, ) -cc_library( - name = "constant_utils", - srcs = [ - "utils/constant_utils.cc", - ], - hdrs = [ - "utils/constant_utils.h", - ], - copts = ["-std=c++14"], - deps = [ - "//tensorflow/compiler/mlir/tensorflow", - "//tensorflow/compiler/mlir/tensorflow:mangling_util", - "//tensorflow/core:protos_all_cc", - "//tensorflow/core/platform:status", - "//tensorflow/stream_executor/lib", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:StandardOps", - "@llvm-project//mlir:Support", - ], -) - cc_library( name = "lstm_utils", srcs = [ @@ -369,7 +347,6 @@ cc_library( "transforms/passes.h", ], deps = [ - ":constant_utils", ":lstm_utils", ":stateful_ops_utils", ":tensorflow_lite", diff --git a/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir b/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir index 3c390df74b4..90266b4e78e 100644 --- a/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir +++ b/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir @@ -1,11 +1,12 @@ -// RUN: tf-opt %s -tfl-prepare-tf -tfl-legalize-tf='run-tfl-runtime-verification=false' | FileCheck %s +// RUN: tf-opt %s -tfl-legalize-tf='run-tfl-runtime-verification=false' | FileCheck %s func @broadcast_to_bf16(%arg0: tensor<3xbf16>, %arg1: tensor<2xi64>) -> tensor<3x3xbf16> { %0 = "tf.BroadcastTo"(%arg0, %arg1) : (tensor<3xbf16>, tensor<2xi64>) -> tensor<3x3xbf16> return %0: tensor<3x3xbf16> // CHECK-LABEL: broadcast_to_bf16 -// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor<3x3xbf16> -// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[CST]]) {fused_activation_function = "NONE"} : (tensor<3xbf16>, tensor<3x3xbf16>) -> tensor<3x3xbf16> +// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor +// CHECK: [[FILL:%.*]] = "tfl.fill"(%arg1, [[CST]]) : (tensor<2xi64>, tensor) -> tensor<3x3xbf16> +// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[FILL]]) {fused_activation_function = "NONE"} : (tensor<3xbf16>, tensor<3x3xbf16>) -> tensor<3x3xbf16> // CHECK: return [[MUL]] : tensor<3x3xbf16> } diff --git a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir index d02e4e705f4..7cb9c4dd22c 100644 --- a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir @@ -1482,6 +1482,28 @@ func @UnidirectionalRnn(%arg: tensor<28x1x28xf32>) -> (tensor<28x1x28xf32>) { // CHECK: return [[VAL_4]] : tensor<28x1x28xf32> // CHECK: } +func @broadcast_to_f32(%arg0: tensor<3xf32>, %arg1: tensor<2xi32>) -> tensor<3x3xf32> { + %0 = "tf.BroadcastTo"(%arg0, %arg1) : (tensor<3xf32>, tensor<2xi32>) -> tensor<3x3xf32> + return %0: tensor<3x3xf32> + +// CHECK-LABEL: broadcast_to_f32 +// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor +// CHECK: [[FILL:%.*]] = "tfl.fill"(%arg1, [[CST]]) : (tensor<2xi32>, tensor) -> tensor<3x3xf32> +// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[FILL]]) {fused_activation_function = "NONE"} : (tensor<3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK: return [[MUL]] : tensor<3x3xf32> +} + +func @broadcast_to_i32(%input: tensor<3xi32>, %shape: tensor<2xi32>) -> tensor<3x3xi32> { + %0 = "tf.BroadcastTo"(%input, %shape) : (tensor<3xi32>, tensor<2xi32>) -> tensor<3x3xi32> + return %0: tensor<3x3xi32> + +// CHECK-LABEL: broadcast_to_i32 +// CHECK: [[CST:%.*]] = constant dense<1> : tensor +// CHECK: [[FILL:%.*]] = "tfl.fill"(%arg1, [[CST]]) : (tensor<2xi32>, tensor) -> tensor<3x3xi32> +// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[FILL]]) {fused_activation_function = "NONE"} : (tensor<3xi32>, tensor<3x3xi32>) -> tensor<3x3xi32> +// CHECK: return [[MUL]] : tensor<3x3xi32> +} + func @matmul_batch(%arg0: tensor<10x15xf32>, %arg1: tensor<15x17xf32>) -> tensor<10x17xf32> { %0 = "tf.BatchMatMul"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT", device = "/device:CPU:0", name = "MatMul", adj_x = false, adj_y = false} : (tensor<10x15xf32>, tensor<15x17xf32>) -> tensor<10x17xf32> diff --git a/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir b/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir index 6ee5b67d65e..066139e179b 100644 --- a/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir @@ -595,24 +595,4 @@ func @xla_conv(%arg0: tensor<4x8x8x16xf32>) -> tensor<4x8x8x16xf32> { // CHECK: return %[[RES]] } -func @broadcast_to_f32(%arg0: tensor<3xf32>, %arg1: tensor<2xi32>) -> tensor<3x3xf32> { - %0 = "tf.BroadcastTo"(%arg0, %arg1) : (tensor<3xf32>, tensor<2xi32>) -> tensor<3x3xf32> - return %0: tensor<3x3xf32> - -// CHECK-LABEL: broadcast_to_f32 -// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor<3x3xf32> -// CHECK: [[MUL:%.*]] = "tf.Mul"(%arg0, [[CST]]) : (tensor<3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> -// CHECK: return [[MUL]] : tensor<3x3xf32> -} - -func @broadcast_to_i32(%input: tensor<3xi32>, %shape: tensor<2xi32>) -> tensor<3x3xi32> { - %0 = "tf.BroadcastTo"(%input, %shape) : (tensor<3xi32>, tensor<2xi32>) -> tensor<3x3xi32> - return %0: tensor<3x3xi32> - -// CHECK-LABEL: broadcast_to_i32 -// CHECK: [[CST:%.*]] = constant dense<1> : tensor<3x3xi32> -// CHECK: [[MUL:%.*]] = "tf.Mul"(%arg0, [[CST]]) : (tensor<3xi32>, tensor<3x3xi32>) -> tensor<3x3xi32> -// CHECK: return [[MUL]] : tensor<3x3xi32> -} - } diff --git a/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc b/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc index 297b1459fc5..7a16e475ce3 100644 --- a/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc +++ b/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc @@ -45,7 +45,6 @@ limitations under the License. #include "tensorflow/compiler/mlir/lite/quantization/quantization_utils.h" #include "tensorflow/compiler/mlir/lite/transforms/passes.h" #include "tensorflow/compiler/mlir/lite/utils/attribute_utils.h" -#include "tensorflow/compiler/mlir/lite/utils/constant_utils.h" #include "tensorflow/compiler/mlir/lite/utils/validators.h" #include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" #include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" @@ -138,6 +137,7 @@ DECL_CONVERT_OP(StridedSlice); DECL_CONVERT_OP(Unpack); DECL_CONVERT_OP(Reciprocal); DECL_CONVERT_OP(RandomUniform); +DECL_CONVERT_OP(BroadcastTo); #undef DECL_CONVERT_OP @@ -483,6 +483,89 @@ LogicalResult ConvertTFAssertOp::matchAndRewrite( return success(); } +StatusOr CreateConstOpWithSingleValue(PatternRewriter* rewriter, + Location loc, + ShapedType shaped_type, + int value) { + Type element_type = shaped_type.getElementType(); + ShapedType scalar_type = RankedTensorType::get({}, element_type); + Attribute attr; + switch (element_type.getKind()) { + case mlir::StandardTypes::F16: { + auto floatType = mlir::FloatType::getF16(element_type.getContext()); + auto floatAttr = + mlir::FloatAttr::get(floatType, static_cast(value)); + std::vector floatValues({floatAttr}); + attr = DenseElementsAttr::get(scalar_type, floatValues); + break; + } + case mlir::StandardTypes::BF16: { + auto floatType = mlir::FloatType::getBF16(element_type.getContext()); + auto floatAttr = + mlir::FloatAttr::get(floatType, static_cast(value)); + std::vector floatValues({floatAttr}); + attr = DenseElementsAttr::get(scalar_type, floatValues); + break; + } + case mlir::StandardTypes::F32: { + attr = + DenseElementsAttr::get(scalar_type, static_cast(value)); + break; + } + case mlir::StandardTypes::Complex: { + auto etype = element_type.cast().getElementType(); + if (etype.isF32()) { + auto dialect = etype.getContext()->getRegisteredDialect("tf"); + tensorflow::TensorProto repr; + repr.set_dtype(tensorflow::DT_COMPLEX64); + + tensorflow::TensorShapeProto* shape = repr.mutable_tensor_shape(); + shape->set_unknown_rank(false); + shape->add_dim()->set_size(int64_t{1}); + std::string content; + auto complex_value = + std::complex(static_cast(value), 0.0f); + content.assign(reinterpret_cast(&complex_value), + sizeof(complex_value)); + repr.set_tensor_content(content); + std::string mangled = tensorflow::mangling_util::MangleTensor(repr); + + attr = mlir::OpaqueElementsAttr::get(dialect, scalar_type, mangled); + break; + } + return Status(tensorflow::error::INVALID_ARGUMENT, "Unsupported type"); + } + case mlir::StandardTypes::Integer: { + const auto& itype = element_type.cast(); + switch (itype.getWidth()) { + case 8: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + case 16: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + case 32: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + case 64: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + default: + return Status(tensorflow::error::INVALID_ARGUMENT, + "Unsupported type"); + } + break; + } + default: + return Status(tensorflow::error::INVALID_ARGUMENT, "Unsupported type"); + } + return rewriter->create(loc, scalar_type, attr); +} + LogicalResult ConvertTFReciprocalOp::matchAndRewrite( Operation* op, PatternRewriter& rewriter) const { auto tf_reciprocal_op = cast(op); @@ -503,6 +586,31 @@ LogicalResult ConvertTFReciprocalOp::matchAndRewrite( return success(); } +LogicalResult ConvertTFBroadcastToOp::matchAndRewrite( + Operation* op, PatternRewriter& rewriter) const { + auto tf_broadcast_to_op = cast(op); + auto element_type = tf_broadcast_to_op.input().getType().cast(); + auto output_type = tf_broadcast_to_op.output().getType(); + + auto status_or_const_op = + CreateConstOpWithSingleValue(&rewriter, op->getLoc(), element_type, 1); + if (!status_or_const_op.ok()) { + return failure(); + } + + auto tfl_fill_op = rewriter.create( + op->getLoc(), output_type, tf_broadcast_to_op.shape(), + status_or_const_op.ValueOrDie()); + + StringAttr fused_activation_function = + StringAttr::get("NONE", rewriter.getContext()); + + rewriter.replaceOpWithNewOp( + op, output_type, tf_broadcast_to_op.input(), tfl_fill_op, + fused_activation_function); + return success(); +} + // Legalize unidirectional sequence lstm. struct LegalizeUnidirectionalSequenceLstm : public RewritePattern { explicit LegalizeUnidirectionalSequenceLstm(MLIRContext* context) @@ -643,7 +751,7 @@ void LegalizeTF::runOnFunction() { ConvertTFMatrixDiagV3Op, ConvertTFPackOp, ConvertTFReshapeOp, ConvertTFSplitOp, ConvertTFSplitVOp, ConvertTFStridedSliceOp, ConvertTFUnpackOp, ConvertTFAssertOp, ConvertTFReciprocalOp, - ConvertTFRandomUniformOp>(context); + ConvertTFRandomUniformOp, ConvertTFBroadcastToOp>(context); // Ophint python converter converted tf node pattern. patterns.insert(op); - auto input_type = tf_broadcast_to_op.input().getType().cast(); - auto output_type = tf_broadcast_to_op.output().getType().cast(); - auto shape_type = tf_broadcast_to_op.shape().getType().cast(); - Type element_type = input_type.getElementType(); - - // Allow lowering when low dimension inputs are given and its type is F32 or - // I32. - if (!((output_type.hasRank() && output_type.getRank() <= 4) || - (shape_type.hasStaticShape() && shape_type.getRank() == 1 && - shape_type.getDimSize(0) <= 4))) - return failure(); - - if (!((element_type.getKind() == mlir::StandardTypes::F32) || - (element_type.getKind() == mlir::StandardTypes::BF16) || - (element_type.getKind() == mlir::StandardTypes::Integer && - element_type.cast().getWidth() == 32))) - return failure(); - - auto status_or_const_op = - CreateConstOpWithSingleValue(&rewriter, op->getLoc(), input_type, 1); - if (!status_or_const_op.ok()) { - return failure(); - } - - auto tf_fill_op = rewriter.create( - op->getLoc(), output_type, tf_broadcast_to_op.shape(), - status_or_const_op.ValueOrDie()); - - auto mul_op = rewriter.create( - op->getLoc(), output_type, tf_broadcast_to_op.input(), tf_fill_op); - rewriter.replaceOp(op, mul_op.getResult()); - return success(); - } -}; - #include "tensorflow/compiler/mlir/lite/transforms/generated_prepare_tf.inc" // Returns success if all the operations in the `op`'s regions including `op` @@ -810,7 +767,7 @@ void PrepareTFPass::runOnFunction() { patterns.insert, TF::ConvertTFBatchMatMulOp>(ctx); } - patterns.insert(ctx); applyPatternsAndFoldGreedily(func, patterns); } diff --git a/tensorflow/compiler/mlir/lite/utils/constant_utils.cc b/tensorflow/compiler/mlir/lite/utils/constant_utils.cc deleted file mode 100644 index 8562f623258..00000000000 --- a/tensorflow/compiler/mlir/lite/utils/constant_utils.cc +++ /dev/null @@ -1,112 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow/compiler/mlir/lite/utils/constant_utils.h" - -#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" -#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" -#include "tensorflow/core/framework/tensor.pb.h" -#include "tensorflow/core/framework/tensor_shape.pb.h" -#include "tensorflow/core/platform/status.h" - -namespace mlir { -namespace TFL { - -stream_executor::port::StatusOr CreateConstOpWithSingleValue( - PatternRewriter* rewriter, Location loc, ShapedType shaped_type, - int value) { - Type element_type = shaped_type.getElementType(); - ShapedType scalar_type = RankedTensorType::get({}, element_type); - Attribute attr; - switch (element_type.getKind()) { - case mlir::StandardTypes::F16: { - auto floatType = mlir::FloatType::getF16(element_type.getContext()); - auto floatAttr = - mlir::FloatAttr::get(floatType, static_cast(value)); - std::vector floatValues({floatAttr}); - attr = DenseElementsAttr::get(scalar_type, floatValues); - break; - } - case mlir::StandardTypes::BF16: { - auto floatType = mlir::FloatType::getBF16(element_type.getContext()); - auto floatAttr = - mlir::FloatAttr::get(floatType, static_cast(value)); - std::vector floatValues({floatAttr}); - attr = DenseElementsAttr::get(scalar_type, floatValues); - break; - } - case mlir::StandardTypes::F32: { - attr = - DenseElementsAttr::get(scalar_type, static_cast(value)); - break; - } - case mlir::StandardTypes::Complex: { - auto etype = element_type.cast().getElementType(); - if (etype.isF32()) { - auto dialect = etype.getContext()->getRegisteredDialect("tf"); - tensorflow::TensorProto repr; - repr.set_dtype(tensorflow::DT_COMPLEX64); - - tensorflow::TensorShapeProto* shape = repr.mutable_tensor_shape(); - shape->set_unknown_rank(false); - shape->add_dim()->set_size(int64_t{1}); - std::string content; - auto complex_value = - std::complex(static_cast(value), 0.0f); - content.assign(reinterpret_cast(&complex_value), - sizeof(complex_value)); - repr.set_tensor_content(content); - std::string mangled = tensorflow::mangling_util::MangleTensor(repr); - - attr = mlir::OpaqueElementsAttr::get(dialect, scalar_type, mangled); - break; - } - return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, - "Unsupported type"); - } - case mlir::StandardTypes::Integer: { - const auto& itype = element_type.cast(); - switch (itype.getWidth()) { - case 8: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - case 16: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - case 32: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - case 64: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - default: - return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, - "Unsupported type"); - } - break; - } - default: - return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, - "Unsupported type"); - } - return rewriter->create(loc, scalar_type, attr); -} - -} // namespace TFL -} // namespace mlir diff --git a/tensorflow/compiler/mlir/lite/utils/constant_utils.h b/tensorflow/compiler/mlir/lite/utils/constant_utils.h deleted file mode 100644 index 5c348021b5e..00000000000 --- a/tensorflow/compiler/mlir/lite/utils/constant_utils.h +++ /dev/null @@ -1,35 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_COMPILER_MLIR_LITE_UTILS_CONSTANT_UTILS_H_ -#define TENSORFLOW_COMPILER_MLIR_LITE_UTILS_CONSTANT_UTILS_H_ - -#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project -#include "mlir/IR/Location.h" // from @llvm-project -#include "mlir/IR/Operation.h" // from @llvm-project -#include "mlir/IR/PatternMatch.h" // from @llvm-project -#include "mlir/IR/StandardTypes.h" // from @llvm-project -#include "tensorflow/stream_executor/lib/statusor.h" - -namespace mlir { -namespace TFL { - -// Returns a Constant op with a single value. -stream_executor::port::StatusOr CreateConstOpWithSingleValue( - PatternRewriter* rewriter, Location loc, ShapedType shaped_type, int value); - -} // namespace TFL -} // namespace mlir -#endif // TENSORFLOW_COMPILER_MLIR_LITE_UTILS_CONSTANT_UTILS_H_ From af9cb379b6728d11b4d929cb04dad90cf47fa408 Mon Sep 17 00:00:00 2001 From: Jaesung Chung Date: Tue, 11 Aug 2020 23:03:14 -0700 Subject: [PATCH 0875/1017] Move TF Broadcast op legalization process to the prepare_tf stage This change is to get benefits from the constant folding logic from TF dialect. PiperOrigin-RevId: 326174654 Change-Id: Icb25f11a6ac0df9904a94831f4969f5b259723a7 --- tensorflow/compiler/mlir/lite/BUILD | 23 ++++ .../legalize-tf-no-runtime-verification.mlir | 7 +- .../compiler/mlir/lite/tests/legalize-tf.mlir | 22 ---- .../compiler/mlir/lite/tests/prepare-tf.mlir | 20 ++++ .../mlir/lite/transforms/legalize_tf.cc | 112 +----------------- .../mlir/lite/transforms/prepare_tf.cc | 45 ++++++- .../mlir/lite/utils/constant_utils.cc | 112 ++++++++++++++++++ .../compiler/mlir/lite/utils/constant_utils.h | 35 ++++++ 8 files changed, 239 insertions(+), 137 deletions(-) create mode 100644 tensorflow/compiler/mlir/lite/utils/constant_utils.cc create mode 100644 tensorflow/compiler/mlir/lite/utils/constant_utils.h diff --git a/tensorflow/compiler/mlir/lite/BUILD b/tensorflow/compiler/mlir/lite/BUILD index 555c11779f5..bd1dcdf06ea 100644 --- a/tensorflow/compiler/mlir/lite/BUILD +++ b/tensorflow/compiler/mlir/lite/BUILD @@ -237,6 +237,28 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "constant_utils", + srcs = [ + "utils/constant_utils.cc", + ], + hdrs = [ + "utils/constant_utils.h", + ], + copts = ["-std=c++14"], + deps = [ + "//tensorflow/compiler/mlir/tensorflow", + "//tensorflow/compiler/mlir/tensorflow:mangling_util", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/platform:status", + "//tensorflow/stream_executor/lib", + "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:StandardOps", + "@llvm-project//mlir:Support", + ], +) + cc_library( name = "lstm_utils", srcs = [ @@ -347,6 +369,7 @@ cc_library( "transforms/passes.h", ], deps = [ + ":constant_utils", ":lstm_utils", ":stateful_ops_utils", ":tensorflow_lite", diff --git a/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir b/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir index 90266b4e78e..3c390df74b4 100644 --- a/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir +++ b/tensorflow/compiler/mlir/lite/tests/legalize-tf-no-runtime-verification.mlir @@ -1,12 +1,11 @@ -// RUN: tf-opt %s -tfl-legalize-tf='run-tfl-runtime-verification=false' | FileCheck %s +// RUN: tf-opt %s -tfl-prepare-tf -tfl-legalize-tf='run-tfl-runtime-verification=false' | FileCheck %s func @broadcast_to_bf16(%arg0: tensor<3xbf16>, %arg1: tensor<2xi64>) -> tensor<3x3xbf16> { %0 = "tf.BroadcastTo"(%arg0, %arg1) : (tensor<3xbf16>, tensor<2xi64>) -> tensor<3x3xbf16> return %0: tensor<3x3xbf16> // CHECK-LABEL: broadcast_to_bf16 -// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor -// CHECK: [[FILL:%.*]] = "tfl.fill"(%arg1, [[CST]]) : (tensor<2xi64>, tensor) -> tensor<3x3xbf16> -// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[FILL]]) {fused_activation_function = "NONE"} : (tensor<3xbf16>, tensor<3x3xbf16>) -> tensor<3x3xbf16> +// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor<3x3xbf16> +// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[CST]]) {fused_activation_function = "NONE"} : (tensor<3xbf16>, tensor<3x3xbf16>) -> tensor<3x3xbf16> // CHECK: return [[MUL]] : tensor<3x3xbf16> } diff --git a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir index 7cb9c4dd22c..d02e4e705f4 100644 --- a/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/legalize-tf.mlir @@ -1482,28 +1482,6 @@ func @UnidirectionalRnn(%arg: tensor<28x1x28xf32>) -> (tensor<28x1x28xf32>) { // CHECK: return [[VAL_4]] : tensor<28x1x28xf32> // CHECK: } -func @broadcast_to_f32(%arg0: tensor<3xf32>, %arg1: tensor<2xi32>) -> tensor<3x3xf32> { - %0 = "tf.BroadcastTo"(%arg0, %arg1) : (tensor<3xf32>, tensor<2xi32>) -> tensor<3x3xf32> - return %0: tensor<3x3xf32> - -// CHECK-LABEL: broadcast_to_f32 -// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor -// CHECK: [[FILL:%.*]] = "tfl.fill"(%arg1, [[CST]]) : (tensor<2xi32>, tensor) -> tensor<3x3xf32> -// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[FILL]]) {fused_activation_function = "NONE"} : (tensor<3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> -// CHECK: return [[MUL]] : tensor<3x3xf32> -} - -func @broadcast_to_i32(%input: tensor<3xi32>, %shape: tensor<2xi32>) -> tensor<3x3xi32> { - %0 = "tf.BroadcastTo"(%input, %shape) : (tensor<3xi32>, tensor<2xi32>) -> tensor<3x3xi32> - return %0: tensor<3x3xi32> - -// CHECK-LABEL: broadcast_to_i32 -// CHECK: [[CST:%.*]] = constant dense<1> : tensor -// CHECK: [[FILL:%.*]] = "tfl.fill"(%arg1, [[CST]]) : (tensor<2xi32>, tensor) -> tensor<3x3xi32> -// CHECK: [[MUL:%.*]] = "tfl.mul"(%arg0, [[FILL]]) {fused_activation_function = "NONE"} : (tensor<3xi32>, tensor<3x3xi32>) -> tensor<3x3xi32> -// CHECK: return [[MUL]] : tensor<3x3xi32> -} - func @matmul_batch(%arg0: tensor<10x15xf32>, %arg1: tensor<15x17xf32>) -> tensor<10x17xf32> { %0 = "tf.BatchMatMul"(%arg0, %arg1) {T = "tfdtype$DT_FLOAT", device = "/device:CPU:0", name = "MatMul", adj_x = false, adj_y = false} : (tensor<10x15xf32>, tensor<15x17xf32>) -> tensor<10x17xf32> diff --git a/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir b/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir index 066139e179b..6ee5b67d65e 100644 --- a/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/prepare-tf.mlir @@ -595,4 +595,24 @@ func @xla_conv(%arg0: tensor<4x8x8x16xf32>) -> tensor<4x8x8x16xf32> { // CHECK: return %[[RES]] } +func @broadcast_to_f32(%arg0: tensor<3xf32>, %arg1: tensor<2xi32>) -> tensor<3x3xf32> { + %0 = "tf.BroadcastTo"(%arg0, %arg1) : (tensor<3xf32>, tensor<2xi32>) -> tensor<3x3xf32> + return %0: tensor<3x3xf32> + +// CHECK-LABEL: broadcast_to_f32 +// CHECK: [[CST:%.*]] = constant dense<1.000000e+00> : tensor<3x3xf32> +// CHECK: [[MUL:%.*]] = "tf.Mul"(%arg0, [[CST]]) : (tensor<3xf32>, tensor<3x3xf32>) -> tensor<3x3xf32> +// CHECK: return [[MUL]] : tensor<3x3xf32> +} + +func @broadcast_to_i32(%input: tensor<3xi32>, %shape: tensor<2xi32>) -> tensor<3x3xi32> { + %0 = "tf.BroadcastTo"(%input, %shape) : (tensor<3xi32>, tensor<2xi32>) -> tensor<3x3xi32> + return %0: tensor<3x3xi32> + +// CHECK-LABEL: broadcast_to_i32 +// CHECK: [[CST:%.*]] = constant dense<1> : tensor<3x3xi32> +// CHECK: [[MUL:%.*]] = "tf.Mul"(%arg0, [[CST]]) : (tensor<3xi32>, tensor<3x3xi32>) -> tensor<3x3xi32> +// CHECK: return [[MUL]] : tensor<3x3xi32> +} + } diff --git a/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc b/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc index 7a16e475ce3..297b1459fc5 100644 --- a/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc +++ b/tensorflow/compiler/mlir/lite/transforms/legalize_tf.cc @@ -45,6 +45,7 @@ limitations under the License. #include "tensorflow/compiler/mlir/lite/quantization/quantization_utils.h" #include "tensorflow/compiler/mlir/lite/transforms/passes.h" #include "tensorflow/compiler/mlir/lite/utils/attribute_utils.h" +#include "tensorflow/compiler/mlir/lite/utils/constant_utils.h" #include "tensorflow/compiler/mlir/lite/utils/validators.h" #include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" #include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" @@ -137,7 +138,6 @@ DECL_CONVERT_OP(StridedSlice); DECL_CONVERT_OP(Unpack); DECL_CONVERT_OP(Reciprocal); DECL_CONVERT_OP(RandomUniform); -DECL_CONVERT_OP(BroadcastTo); #undef DECL_CONVERT_OP @@ -483,89 +483,6 @@ LogicalResult ConvertTFAssertOp::matchAndRewrite( return success(); } -StatusOr CreateConstOpWithSingleValue(PatternRewriter* rewriter, - Location loc, - ShapedType shaped_type, - int value) { - Type element_type = shaped_type.getElementType(); - ShapedType scalar_type = RankedTensorType::get({}, element_type); - Attribute attr; - switch (element_type.getKind()) { - case mlir::StandardTypes::F16: { - auto floatType = mlir::FloatType::getF16(element_type.getContext()); - auto floatAttr = - mlir::FloatAttr::get(floatType, static_cast(value)); - std::vector floatValues({floatAttr}); - attr = DenseElementsAttr::get(scalar_type, floatValues); - break; - } - case mlir::StandardTypes::BF16: { - auto floatType = mlir::FloatType::getBF16(element_type.getContext()); - auto floatAttr = - mlir::FloatAttr::get(floatType, static_cast(value)); - std::vector floatValues({floatAttr}); - attr = DenseElementsAttr::get(scalar_type, floatValues); - break; - } - case mlir::StandardTypes::F32: { - attr = - DenseElementsAttr::get(scalar_type, static_cast(value)); - break; - } - case mlir::StandardTypes::Complex: { - auto etype = element_type.cast().getElementType(); - if (etype.isF32()) { - auto dialect = etype.getContext()->getRegisteredDialect("tf"); - tensorflow::TensorProto repr; - repr.set_dtype(tensorflow::DT_COMPLEX64); - - tensorflow::TensorShapeProto* shape = repr.mutable_tensor_shape(); - shape->set_unknown_rank(false); - shape->add_dim()->set_size(int64_t{1}); - std::string content; - auto complex_value = - std::complex(static_cast(value), 0.0f); - content.assign(reinterpret_cast(&complex_value), - sizeof(complex_value)); - repr.set_tensor_content(content); - std::string mangled = tensorflow::mangling_util::MangleTensor(repr); - - attr = mlir::OpaqueElementsAttr::get(dialect, scalar_type, mangled); - break; - } - return Status(tensorflow::error::INVALID_ARGUMENT, "Unsupported type"); - } - case mlir::StandardTypes::Integer: { - const auto& itype = element_type.cast(); - switch (itype.getWidth()) { - case 8: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - case 16: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - case 32: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - case 64: - attr = DenseElementsAttr::get(scalar_type, - static_cast(value)); - break; - default: - return Status(tensorflow::error::INVALID_ARGUMENT, - "Unsupported type"); - } - break; - } - default: - return Status(tensorflow::error::INVALID_ARGUMENT, "Unsupported type"); - } - return rewriter->create(loc, scalar_type, attr); -} - LogicalResult ConvertTFReciprocalOp::matchAndRewrite( Operation* op, PatternRewriter& rewriter) const { auto tf_reciprocal_op = cast(op); @@ -586,31 +503,6 @@ LogicalResult ConvertTFReciprocalOp::matchAndRewrite( return success(); } -LogicalResult ConvertTFBroadcastToOp::matchAndRewrite( - Operation* op, PatternRewriter& rewriter) const { - auto tf_broadcast_to_op = cast(op); - auto element_type = tf_broadcast_to_op.input().getType().cast(); - auto output_type = tf_broadcast_to_op.output().getType(); - - auto status_or_const_op = - CreateConstOpWithSingleValue(&rewriter, op->getLoc(), element_type, 1); - if (!status_or_const_op.ok()) { - return failure(); - } - - auto tfl_fill_op = rewriter.create( - op->getLoc(), output_type, tf_broadcast_to_op.shape(), - status_or_const_op.ValueOrDie()); - - StringAttr fused_activation_function = - StringAttr::get("NONE", rewriter.getContext()); - - rewriter.replaceOpWithNewOp( - op, output_type, tf_broadcast_to_op.input(), tfl_fill_op, - fused_activation_function); - return success(); -} - // Legalize unidirectional sequence lstm. struct LegalizeUnidirectionalSequenceLstm : public RewritePattern { explicit LegalizeUnidirectionalSequenceLstm(MLIRContext* context) @@ -751,7 +643,7 @@ void LegalizeTF::runOnFunction() { ConvertTFMatrixDiagV3Op, ConvertTFPackOp, ConvertTFReshapeOp, ConvertTFSplitOp, ConvertTFSplitVOp, ConvertTFStridedSliceOp, ConvertTFUnpackOp, ConvertTFAssertOp, ConvertTFReciprocalOp, - ConvertTFRandomUniformOp, ConvertTFBroadcastToOp>(context); + ConvertTFRandomUniformOp>(context); // Ophint python converter converted tf node pattern. patterns.insert(op); + auto input_type = tf_broadcast_to_op.input().getType().cast(); + auto output_type = tf_broadcast_to_op.output().getType().cast(); + auto shape_type = tf_broadcast_to_op.shape().getType().cast(); + Type element_type = input_type.getElementType(); + + // Allow lowering when low dimension inputs are given and its type is F32 or + // I32. + if (!((output_type.hasRank() && output_type.getRank() <= 5) || + (shape_type.hasStaticShape() && shape_type.getRank() == 1 && + shape_type.getDimSize(0) <= 5))) + return failure(); + + if (!((element_type.getKind() == mlir::StandardTypes::F32) || + (element_type.getKind() == mlir::StandardTypes::BF16) || + (element_type.getKind() == mlir::StandardTypes::Integer && + element_type.cast().getWidth() == 32))) + return failure(); + + auto status_or_const_op = + CreateConstOpWithSingleValue(&rewriter, op->getLoc(), input_type, 1); + if (!status_or_const_op.ok()) { + return failure(); + } + + auto tf_fill_op = rewriter.create( + op->getLoc(), output_type, tf_broadcast_to_op.shape(), + status_or_const_op.ValueOrDie()); + + auto mul_op = rewriter.create( + op->getLoc(), output_type, tf_broadcast_to_op.input(), tf_fill_op); + rewriter.replaceOp(op, mul_op.getResult()); + return success(); + } +}; + #include "tensorflow/compiler/mlir/lite/transforms/generated_prepare_tf.inc" // Returns success if all the operations in the `op`'s regions including `op` @@ -767,7 +810,7 @@ void PrepareTFPass::runOnFunction() { patterns.insert, TF::ConvertTFBatchMatMulOp>(ctx); } - patterns.insert(ctx); applyPatternsAndFoldGreedily(func, patterns); } diff --git a/tensorflow/compiler/mlir/lite/utils/constant_utils.cc b/tensorflow/compiler/mlir/lite/utils/constant_utils.cc new file mode 100644 index 00000000000..8562f623258 --- /dev/null +++ b/tensorflow/compiler/mlir/lite/utils/constant_utils.cc @@ -0,0 +1,112 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/mlir/lite/utils/constant_utils.h" + +#include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" +#include "tensorflow/compiler/mlir/tensorflow/utils/mangling_util.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/platform/status.h" + +namespace mlir { +namespace TFL { + +stream_executor::port::StatusOr CreateConstOpWithSingleValue( + PatternRewriter* rewriter, Location loc, ShapedType shaped_type, + int value) { + Type element_type = shaped_type.getElementType(); + ShapedType scalar_type = RankedTensorType::get({}, element_type); + Attribute attr; + switch (element_type.getKind()) { + case mlir::StandardTypes::F16: { + auto floatType = mlir::FloatType::getF16(element_type.getContext()); + auto floatAttr = + mlir::FloatAttr::get(floatType, static_cast(value)); + std::vector floatValues({floatAttr}); + attr = DenseElementsAttr::get(scalar_type, floatValues); + break; + } + case mlir::StandardTypes::BF16: { + auto floatType = mlir::FloatType::getBF16(element_type.getContext()); + auto floatAttr = + mlir::FloatAttr::get(floatType, static_cast(value)); + std::vector floatValues({floatAttr}); + attr = DenseElementsAttr::get(scalar_type, floatValues); + break; + } + case mlir::StandardTypes::F32: { + attr = + DenseElementsAttr::get(scalar_type, static_cast(value)); + break; + } + case mlir::StandardTypes::Complex: { + auto etype = element_type.cast().getElementType(); + if (etype.isF32()) { + auto dialect = etype.getContext()->getRegisteredDialect("tf"); + tensorflow::TensorProto repr; + repr.set_dtype(tensorflow::DT_COMPLEX64); + + tensorflow::TensorShapeProto* shape = repr.mutable_tensor_shape(); + shape->set_unknown_rank(false); + shape->add_dim()->set_size(int64_t{1}); + std::string content; + auto complex_value = + std::complex(static_cast(value), 0.0f); + content.assign(reinterpret_cast(&complex_value), + sizeof(complex_value)); + repr.set_tensor_content(content); + std::string mangled = tensorflow::mangling_util::MangleTensor(repr); + + attr = mlir::OpaqueElementsAttr::get(dialect, scalar_type, mangled); + break; + } + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Unsupported type"); + } + case mlir::StandardTypes::Integer: { + const auto& itype = element_type.cast(); + switch (itype.getWidth()) { + case 8: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + case 16: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + case 32: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + case 64: + attr = DenseElementsAttr::get(scalar_type, + static_cast(value)); + break; + default: + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Unsupported type"); + } + break; + } + default: + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Unsupported type"); + } + return rewriter->create(loc, scalar_type, attr); +} + +} // namespace TFL +} // namespace mlir diff --git a/tensorflow/compiler/mlir/lite/utils/constant_utils.h b/tensorflow/compiler/mlir/lite/utils/constant_utils.h new file mode 100644 index 00000000000..5c348021b5e --- /dev/null +++ b/tensorflow/compiler/mlir/lite/utils/constant_utils.h @@ -0,0 +1,35 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_MLIR_LITE_UTILS_CONSTANT_UTILS_H_ +#define TENSORFLOW_COMPILER_MLIR_LITE_UTILS_CONSTANT_UTILS_H_ + +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/Location.h" // from @llvm-project +#include "mlir/IR/Operation.h" // from @llvm-project +#include "mlir/IR/PatternMatch.h" // from @llvm-project +#include "mlir/IR/StandardTypes.h" // from @llvm-project +#include "tensorflow/stream_executor/lib/statusor.h" + +namespace mlir { +namespace TFL { + +// Returns a Constant op with a single value. +stream_executor::port::StatusOr CreateConstOpWithSingleValue( + PatternRewriter* rewriter, Location loc, ShapedType shaped_type, int value); + +} // namespace TFL +} // namespace mlir +#endif // TENSORFLOW_COMPILER_MLIR_LITE_UTILS_CONSTANT_UTILS_H_ From 335b03c45c702612b9b552cc751cd99cfc0a4971 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Aug 2020 23:08:41 -0700 Subject: [PATCH 0876/1017] Add CompileTimeConstantInput to XlaGather. PiperOrigin-RevId: 326175243 Change-Id: I8540c0d09709be40ff6d5bfaa50a4eac918a6627 --- tensorflow/compiler/tests/xla_ops_test.py | 19 +++++++++++++++++++ .../tf2xla/kernels/gather_scatter_ops.cc | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/tests/xla_ops_test.py b/tensorflow/compiler/tests/xla_ops_test.py index 0d6ae81ef6e..3e9f5e8c5dd 100644 --- a/tensorflow/compiler/tests/xla_ops_test.py +++ b/tensorflow/compiler/tests/xla_ops_test.py @@ -79,6 +79,25 @@ class XlaOpsNumericalTest(xla_test.XLATestCase, parameterized.TestCase): args=(v,), expected=np.tile(v, (7, 42, 1, 1))) + @test_util.disable_mlir_bridge('Not supported yet') + def testGather(self): + operand = np.arange(10, dtype=np.int32).reshape([2, 5]) + start_indices = np.array([2], np.int32) + slice_sizes = np.array([1, 3], np.int32) + + def gather(operand, start_indices): + dimension_numbers = xla_data_pb2.GatherDimensionNumbers() + dimension_numbers.offset_dims.extend([1]) + dimension_numbers.collapsed_slice_dims.extend([0]) + dimension_numbers.start_index_map.extend([0]) + dimension_numbers.index_vector_dim = 1 + return xla.gather(operand, start_indices, dimension_numbers, slice_sizes) + + self._assertOpOutputMatchesExpected( + gather, + args=(operand, start_indices), + expected=np.array([[5, 6, 7]])) + @test_util.disable_mlir_bridge('Dynamic result types not supported') def testShiftRightLogical(self): self._assertOpOutputMatchesExpected( diff --git a/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc b/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc index 19aa85f9d42..b4b18dd2b36 100644 --- a/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/gather_scatter_ops.cc @@ -49,7 +49,8 @@ class GatherOp : public XlaOpKernel { bool indices_are_sorted_; }; -REGISTER_XLA_OP(Name("XlaGather"), GatherOp); +REGISTER_XLA_OP(Name("XlaGather").CompileTimeConstantInput("slice_sizes"), + GatherOp); class ScatterOp : public XlaOpKernel { public: From fd4de7629c5b90dc969e3ccfbf60566ff28e24a5 Mon Sep 17 00:00:00 2001 From: Yuanzhong Xu Date: Wed, 12 Aug 2020 00:31:27 -0700 Subject: [PATCH 0877/1017] [XLA:SPMD] Support partially partitioned dot PiperOrigin-RevId: 326182728 Change-Id: I1eeb94483265b167359a8c5605093b10a0b6f5cb --- .../xla/service/dot_as_convolution_util.cc | 49 ++ .../xla/service/dot_as_convolution_util.h | 5 + .../compiler/xla/service/hlo_sharding_util.cc | 51 ++ .../compiler/xla/service/hlo_sharding_util.h | 7 + .../xla/service/sharding_propagation.cc | 622 ++++++++---------- .../xla/service/sharding_propagation_test.cc | 130 +++- tensorflow/compiler/xla/service/spmd/BUILD | 1 + .../compiler/xla/service/spmd/dot_handler.cc | 347 ++++++++-- .../xla/service/spmd/spmd_partitioner.cc | 12 +- .../xla/service/spmd/spmd_partitioner_test.cc | 136 ++++ .../xla/service/spmd/spmd_partitioner_util.cc | 137 ++-- .../xla/service/spmd/spmd_partitioner_util.h | 12 +- 12 files changed, 1027 insertions(+), 482 deletions(-) diff --git a/tensorflow/compiler/xla/service/dot_as_convolution_util.cc b/tensorflow/compiler/xla/service/dot_as_convolution_util.cc index 4670ce6940a..b95636c7039 100644 --- a/tensorflow/compiler/xla/service/dot_as_convolution_util.cc +++ b/tensorflow/compiler/xla/service/dot_as_convolution_util.cc @@ -153,5 +153,54 @@ CreateShardedConvForDotGeneralConvolution( /*batch_group_count=*/1, window, conv_dnums, conv.precision_config()); } +DotGeneralAsConvolutionDimsInfo ParseDotGeneralFromDot( + const HloInstruction* dot) { + const auto& dot_dim_numbs = dot->dot_dimension_numbers(); + dot_as_convolution_util::DotGeneralAsConvolutionDimsInfo dnums; + for (int64 i = 0; i < dot_dim_numbs.lhs_batch_dimensions().size(); ++i) { + dnums.batch_dims.emplace_back(); + dnums.batch_dims.back().lhs = dot_dim_numbs.lhs_batch_dimensions(i); + dnums.batch_dims.back().rhs = dot_dim_numbs.rhs_batch_dimensions(i); + dnums.batch_dims.back().output = i; + dnums.batch_dims.back().spatial_dim = -1; + } + for (int64 i = 0; i < dot_dim_numbs.lhs_contracting_dimensions().size(); + ++i) { + dnums.contracting_dims.emplace_back(); + dnums.contracting_dims.back().lhs = + dot_dim_numbs.lhs_contracting_dimensions(i); + dnums.contracting_dims.back().rhs = + dot_dim_numbs.rhs_contracting_dimensions(i); + dnums.contracting_dims.back().output = -1; + dnums.contracting_dims.back().spatial_dim = -1; + } + for (int64 i = 0; i < dot->operand(0)->shape().rank(); ++i) { + if (!absl::c_linear_search(dot_dim_numbs.lhs_batch_dimensions(), i) && + !absl::c_linear_search(dot_dim_numbs.lhs_contracting_dimensions(), i)) { + dnums.lhs_non_contracting_dims.emplace_back(); + dnums.lhs_non_contracting_dims.back().lhs = i; + dnums.lhs_non_contracting_dims.back().rhs = -1; + dnums.lhs_non_contracting_dims.back().output = + dot_dim_numbs.lhs_batch_dimensions_size() + + dnums.lhs_non_contracting_dims.size() - 1; + dnums.lhs_non_contracting_dims.back().spatial_dim = -1; + } + } + for (int64 i = 0; i < dot->operand(1)->shape().rank(); ++i) { + if (!absl::c_linear_search(dot_dim_numbs.rhs_batch_dimensions(), i) && + !absl::c_linear_search(dot_dim_numbs.rhs_contracting_dimensions(), i)) { + dnums.rhs_non_contracting_dims.emplace_back(); + dnums.rhs_non_contracting_dims.back().lhs = -1; + dnums.rhs_non_contracting_dims.back().rhs = i; + dnums.rhs_non_contracting_dims.back().output = + dot_dim_numbs.lhs_batch_dimensions_size() + + dnums.lhs_non_contracting_dims.size() + + dnums.rhs_non_contracting_dims.size() - 1; + dnums.rhs_non_contracting_dims.back().spatial_dim = -1; + } + } + return dnums; +} + } // namespace dot_as_convolution_util } // namespace xla diff --git a/tensorflow/compiler/xla/service/dot_as_convolution_util.h b/tensorflow/compiler/xla/service/dot_as_convolution_util.h index 6a7cacf812d..81914b193a3 100644 --- a/tensorflow/compiler/xla/service/dot_as_convolution_util.h +++ b/tensorflow/compiler/xla/service/dot_as_convolution_util.h @@ -68,6 +68,11 @@ CreateShardedConvForDotGeneralConvolution( // dilation B. bool ConvSpatialDimensionIsParallel(const WindowDimension& wd, int64 lhs_size); +// Returns a DotGeneralAsConvolutionDimsInfo from a kDot instruction, where all +// the spatial_dim values are set to -1. +DotGeneralAsConvolutionDimsInfo ParseDotGeneralFromDot( + const HloInstruction* dot); + } // namespace dot_as_convolution_util } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_sharding_util.cc b/tensorflow/compiler/xla/service/hlo_sharding_util.cc index 65295a8e620..007b6158fc2 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding_util.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding_util.cc @@ -841,5 +841,56 @@ HloSharding RemoveShapeDimensions(const HloSharding& sharding, : HloSharding::Tile(new_tile); } +absl::optional TransposeShardingWithCollapsedDims( + const HloSharding& source, absl::Span src_to_tgt, + absl::Span tgt_to_src) { + if (source.IsTileMaximal()) { + return source; + } + if (source.ReplicateOnLastTileDim() && + src_to_tgt.size() < source.tile_assignment().num_dimensions()) { + std::vector new_src_to_tgt(src_to_tgt.begin(), src_to_tgt.end()); + new_src_to_tgt.push_back(tgt_to_src.size()); + std::vector new_tgt_to_src(tgt_to_src.begin(), tgt_to_src.end()); + new_tgt_to_src.push_back(src_to_tgt.size()); + return TransposeShardingWithCollapsedDims(source, new_src_to_tgt, + new_tgt_to_src); + } + std::vector tgt_dims_skipping_new(tgt_to_src.size(), -1); + int64 skipped_tgt_dims = 0; + for (int64 i = 0; i < tgt_to_src.size(); ++i) { + if (tgt_to_src[i] < 0) { + skipped_tgt_dims++; + } else { + tgt_dims_skipping_new[i] = i - skipped_tgt_dims; + } + } + int64 skipped_src_dims = absl::c_count(src_to_tgt, -1); + std::vector perm(src_to_tgt.size()); + for (int64 i = 0; i < src_to_tgt.size(); ++i) { + if (src_to_tgt[i] < 0) { + if (source.tile_assignment().dim(i) > 1) { + return absl::nullopt; + } + perm[src_to_tgt.size() - skipped_src_dims] = i; + skipped_src_dims--; + } else { + perm[tgt_dims_skipping_new[src_to_tgt[i]]] = i; + } + } + auto tgt_sharding = hlo_sharding_util::TransposeSharding(source, perm); + auto reshape_tiles = tgt_sharding.tile_assignment(); + std::vector tgt_tiles(tgt_to_src.size(), 1); + for (int64 i = 0; i < tgt_tiles.size(); ++i) { + if (tgt_to_src[i] >= 0) { + tgt_tiles[i] = reshape_tiles.dim(tgt_dims_skipping_new[i]); + } + } + reshape_tiles.Reshape(tgt_tiles); + return source.ReplicateOnLastTileDim() + ? HloSharding::PartialTile(reshape_tiles) + : HloSharding::Tile(reshape_tiles); +} + } // namespace hlo_sharding_util } // namespace xla diff --git a/tensorflow/compiler/xla/service/hlo_sharding_util.h b/tensorflow/compiler/xla/service/hlo_sharding_util.h index ce19d8c7a19..0de01fcab7e 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding_util.h +++ b/tensorflow/compiler/xla/service/hlo_sharding_util.h @@ -174,6 +174,13 @@ HloSharding PartiallyReplicateTiledShardingOnDims( HloSharding RemoveShapeDimensions(const HloSharding& sharding, const std::vector& dims_to_remove); +// Similar to TransposeSharding(), but allows removing/adding non-partitioned +// dimensions. In src_to_tgt and tgt_to_src, -1 represents a non-existing +// dimension. +absl::optional TransposeShardingWithCollapsedDims( + const HloSharding& source, absl::Span src_to_tgt, + absl::Span tgt_to_src); + } // namespace hlo_sharding_util } // namespace xla diff --git a/tensorflow/compiler/xla/service/sharding_propagation.cc b/tensorflow/compiler/xla/service/sharding_propagation.cc index 4ff492047a3..bcbebf3460f 100644 --- a/tensorflow/compiler/xla/service/sharding_propagation.cc +++ b/tensorflow/compiler/xla/service/sharding_propagation.cc @@ -120,6 +120,113 @@ HloSharding MergeForMoreSpecificSharding(const HloSharding& a, return IsShardingMoreSpecific(a, b) ? a : b; } +// Returns a sharding that is refined by merging old and to_merge. May combine +// partial sharding in addition to MergeForMoreSpecificSharding(). +HloSharding MergeSharding(const HloSharding& old, const HloSharding& to_merge, + bool may_combine_partial_sharding) { + if (old.IsTuple()) { + HloSharding result = old; + CHECK(to_merge.IsTuple()); + CHECK_EQ(old.tuple_elements().size(), to_merge.tuple_elements().size()); + for (int64 i = 0; i < result.tuple_elements().size(); ++i) { + result.tuple_elements()[i] = + MergeSharding(old.tuple_elements()[i], to_merge.tuple_elements()[i], + may_combine_partial_sharding); + } + return result; + } + if (!may_combine_partial_sharding || !old.ReplicateOnLastTileDim() || + !to_merge.ReplicateOnLastTileDim() || + old.tile_assignment().num_elements() != + to_merge.tile_assignment().num_elements()) { + return IsShardingMoreSpecific(to_merge, old) ? to_merge : old; + } + // Combine the tile dimension sizes from new and old. + int64 num_devices = old.tile_assignment().num_elements(); + std::vector new_tile_dims; + bool compatible = true; + new_tile_dims.reserve(to_merge.tile_assignment().num_dimensions()); + for (int64 i = 0; i < to_merge.tile_assignment().num_dimensions() - 1; ++i) { + int64 new_dim = to_merge.tile_assignment().dim(i); + int64 old_dim = old.tile_assignment().dim(i); + if (new_dim == 1) { + new_tile_dims.push_back(old_dim); + } else if (old_dim == 1) { + new_tile_dims.push_back(new_dim); + } else if (new_dim == old_dim) { + new_tile_dims.push_back(new_dim); + } else { + compatible = false; + break; + } + } + int64 replication = num_devices / Product(new_tile_dims); + if (!compatible || num_devices % Product(new_tile_dims) != 0 || + replication >= old.tile_assignment().dimensions().back()) { + return IsShardingMoreSpecific(to_merge, old) ? to_merge : old; + } + new_tile_dims.push_back(replication); + Array new_tile(new_tile_dims); + // Maps from replication group ID to sorted members. + absl::flat_hash_map> old_group_members; + absl::flat_hash_map> new_group_members; + auto get_group_index = [&](absl::Span tile_indices, + const HloSharding& sharding) { + int64 group_id = 0; + for (int64 i = 0; i < tile_indices.size() - 1; ++i) { + group_id *= to_merge.tile_assignment().dim(i); + group_id += tile_indices[i]; + } + return group_id; + }; + old.tile_assignment().Each( + [&](absl::Span indices, int64 device) { + old_group_members[get_group_index(indices, old)].insert(device); + }); + to_merge.tile_assignment().Each( + [&](absl::Span indices, int64 device) { + new_group_members[get_group_index(indices, to_merge)].insert(device); + }); + // Try to find the intersection of old and new replication groups, in + // order to determine the merged tile assignment. + new_tile.Each([&](absl::Span indices, int64* device) { + if (!compatible) { + return; + } + std::vector old_index(indices.begin(), indices.end()); + std::vector new_index = old_index; + for (int64 i = 0; i < indices.size() - 1; ++i) { + if (old.tile_assignment().dim(i) == 1) { + old_index[i] = 0; + } + if (to_merge.tile_assignment().dim(i) == 1) { + new_index[i] = 0; + } + } + int64 old_group_id = get_group_index(old_index, old); + int64 new_group_id = get_group_index(new_index, to_merge); + if (old_group_members[old_group_id].empty() || + new_group_members[new_group_id].empty() || + *old_group_members[old_group_id].begin() != + *new_group_members[new_group_id].begin()) { + compatible = false; + return; + } + *device = *old_group_members[old_group_id].begin(); + old_group_members[old_group_id].erase(*device); + new_group_members[new_group_id].erase(*device); + }); + if (compatible) { + if (replication == 1) { + new_tile_dims.pop_back(); + new_tile.Reshape(new_tile_dims); + return HloSharding::Tile(new_tile); + } + return HloSharding::PartialTile(new_tile); + } + return IsShardingMoreSpecific(to_merge, old) ? to_merge : old; +} + // Updates the sharding of the specified instruction with the specified sharding // if it is better than the current one and returns true if a new sharding have // been applied. If may_combine_partial_sharding is true, this may combine the @@ -137,103 +244,10 @@ bool MaybeImproveInstructionSharding(const HloSharding& sharding, instruction->set_sharding(sharding); return true; } - if (may_combine_partial_sharding && sharding.ReplicateOnLastTileDim() && - instruction->sharding().ReplicateOnLastTileDim()) { - if (sharding.tile_assignment().num_elements() == - instruction->sharding().tile_assignment().num_elements()) { - // Combine the tile dimension sizes from new and old. - int64 num_devices = sharding.tile_assignment().num_elements(); - std::vector new_tile_dims; - bool compatible = true; - new_tile_dims.reserve(sharding.tile_assignment().num_dimensions()); - for (int64 i = 0; i < sharding.tile_assignment().num_dimensions() - 1; - ++i) { - int64 new_dim = sharding.tile_assignment().dim(i); - int64 old_dim = instruction->sharding().tile_assignment().dim(i); - if (new_dim == 1) { - new_tile_dims.push_back(old_dim); - } else if (old_dim == 1) { - new_tile_dims.push_back(new_dim); - } else if (new_dim == old_dim) { - new_tile_dims.push_back(new_dim); - } else { - compatible = false; - break; - } - } - int64 replication = num_devices / Product(new_tile_dims); - if (compatible && num_devices % Product(new_tile_dims) == 0 && - replication < - instruction->sharding().tile_assignment().dimensions().back()) { - new_tile_dims.push_back(replication); - Array new_tile(new_tile_dims); - // Maps from replication group ID to sorted members. - absl::flat_hash_map> old_group_members; - absl::flat_hash_map> new_group_members; - auto get_group_index = [&](absl::Span tile_indices, - const HloSharding& sharding) { - int64 group_id = 0; - for (int64 i = 0; i < tile_indices.size() - 1; ++i) { - group_id *= sharding.tile_assignment().dim(i); - group_id += tile_indices[i]; - } - return group_id; - }; - instruction->sharding().tile_assignment().Each( - [&](absl::Span indices, int64 device) { - old_group_members[get_group_index(indices, - instruction->sharding())] - .insert(device); - }); - sharding.tile_assignment().Each([&](absl::Span indices, - int64 device) { - new_group_members[get_group_index(indices, sharding)].insert(device); - }); - // Try to find the intersection of old and new replication groups, in - // order to determine the merged tile assignment. - new_tile.Each([&](absl::Span indices, int64* device) { - if (!compatible) { - return; - } - std::vector old_index(indices.begin(), indices.end()); - std::vector new_index = old_index; - for (int64 i = 0; i < indices.size() - 1; ++i) { - if (instruction->sharding().tile_assignment().dim(i) == 1) { - old_index[i] = 0; - } - if (sharding.tile_assignment().dim(i) == 1) { - new_index[i] = 0; - } - } - int64 old_group_id = - get_group_index(old_index, instruction->sharding()); - int64 new_group_id = get_group_index(new_index, sharding); - if (old_group_members[old_group_id].empty() || - new_group_members[new_group_id].empty() || - *old_group_members[old_group_id].begin() != - *new_group_members[new_group_id].begin()) { - compatible = false; - return; - } - *device = *old_group_members[old_group_id].begin(); - old_group_members[old_group_id].erase(*device); - new_group_members[new_group_id].erase(*device); - }); - if (compatible) { - if (replication == 1) { - new_tile_dims.pop_back(); - new_tile.Reshape(new_tile_dims); - instruction->set_sharding(HloSharding::Tile(new_tile)); - } else { - instruction->set_sharding(HloSharding::PartialTile(new_tile)); - } - return true; - } - } - } - } - if (IsShardingMoreSpecific(sharding, instruction->sharding())) { - instruction->set_sharding(sharding); + auto merged = MergeSharding(instruction->sharding(), sharding, + may_combine_partial_sharding); + if (merged != instruction->sharding()) { + instruction->set_sharding(merged); return true; } return false; @@ -457,13 +471,82 @@ bool SupportSpatialPartitioning(const HloInstruction* instruction, } } +bool InferDotShardingFromOperands( + HloInstruction* instruction, + const dot_as_convolution_util::DotGeneralAsConvolutionDimsInfo& dnums, + bool may_combine_partial_sharding) { + auto from_operand = [&](int64 operand_index) { + auto operand = instruction->operand(operand_index); + const HloSharding& operand_sharding = operand->sharding(); + if (operand_sharding.IsTileMaximal()) { + return operand_sharding; + } + std::vector contracting_dims; + contracting_dims.reserve(dnums.contracting_dims.size()); + for (const auto& dim : dnums.contracting_dims) { + contracting_dims.push_back(operand_index == 0 ? dim.lhs : dim.rhs); + } + // It's possible that some size-1 spatial dims of convolutions are parsed as + // non-contracting dims. We might have tiled dimensions on them. + for (const auto& dim : operand_index == 0 + ? dnums.rhs_non_contracting_dims + : dnums.lhs_non_contracting_dims) { + int64 d = operand_index == 0 ? dim.lhs : dim.rhs; + if (d > 0) { + contracting_dims.push_back(d); + } + } + auto replicate_contracting_dims = + hlo_sharding_util::PartiallyReplicateTiledShardingOnDims( + operand_sharding, contracting_dims); + std::vector out_dims_to_op_perm(instruction->shape().rank(), -1); + std::vector op_dims_to_output_perm(operand->shape().rank(), -1); + for (const auto& dim : dnums.batch_dims) { + out_dims_to_op_perm[dim.output] = operand_index == 0 ? dim.lhs : dim.rhs; + op_dims_to_output_perm[operand_index == 0 ? dim.lhs : dim.rhs] = + dim.output; + } + for (const auto& dim : operand_index == 0 + ? dnums.lhs_non_contracting_dims + : dnums.rhs_non_contracting_dims) { + out_dims_to_op_perm[dim.output] = operand_index == 0 ? dim.lhs : dim.rhs; + op_dims_to_output_perm[operand_index == 0 ? dim.lhs : dim.rhs] = + dim.output; + } + return *hlo_sharding_util::TransposeShardingWithCollapsedDims( + replicate_contracting_dims, op_dims_to_output_perm, + out_dims_to_op_perm); + }; + bool changed = false; + int64 larger_operand = + ShapeUtil::ByteSizeOf(instruction->operand(0)->shape()) >= + ShapeUtil::ByteSizeOf(instruction->operand(1)->shape()) + ? 0 + : 1; + if (IsSpatiallyPartitioned(instruction->operand(larger_operand))) { + changed |= MaybeImproveInstructionSharding(from_operand(larger_operand), + instruction, + may_combine_partial_sharding); + } + if (IsSpatiallyPartitioned(instruction->operand(1 - larger_operand))) { + changed |= MaybeImproveInstructionSharding(from_operand(1 - larger_operand), + instruction, + may_combine_partial_sharding); + } + return changed; +} + // Convolution handling for InferShardingFromOperands(). bool InferConvolutionShardingFromOperands(HloInstruction* instruction, bool aggressive_prop, bool may_combine_partial_sharding) { + if (auto dot_dims = dot_as_convolution_util::ParseDotGeneralFromConvolution( + instruction)) { + return InferDotShardingFromOperands(instruction, *dot_dims, + may_combine_partial_sharding); + } const auto& dnums = instruction->convolution_dimension_numbers(); const HloInstruction* lhs = instruction->operand(0); - const HloInstruction* rhs = instruction->operand(1); auto get_tiled_sharding_based_on_lhs = [&] { CHECK(!lhs->sharding().IsTileMaximal()); std::vector output_to_lhs_indices(instruction->shape().rank()); @@ -478,103 +561,6 @@ bool InferConvolutionShardingFromOperands(HloInstruction* instruction, return hlo_sharding_util::TransposeSharding(lhs->sharding(), output_to_lhs_indices); }; - auto get_tiled_sharding_based_on_rhs = [&] { - CHECK(!rhs->sharding().IsTileMaximal()); - std::vector output_to_rhs_indices(instruction->shape().rank()); - output_to_rhs_indices[dnums.output_batch_dimension()] = - dnums.kernel_input_feature_dimension(); - output_to_rhs_indices[dnums.output_feature_dimension()] = - dnums.kernel_output_feature_dimension(); - for (int64 i = 0; i < dnums.input_spatial_dimensions_size(); ++i) { - output_to_rhs_indices[dnums.output_spatial_dimensions(i)] = - dnums.kernel_spatial_dimensions(i); - } - return hlo_sharding_util::TransposeSharding(rhs->sharding(), - output_to_rhs_indices); - }; - if (auto dot_dims = dot_as_convolution_util::ParseDotGeneralFromConvolution( - instruction)) { - // lhs_or_rhs: lhs is 0 and rhs is 1. Skips dimensions with size 1. - auto partitioned_only_along_non_trivial_dims = - [&](const HloSharding& sharding, - std::vector& dims, - int64 lhs_or_rhs) { - if (sharding.IsTileMaximal()) { - return false; - } - int64 partition_count = 1; - for (const auto& dim : dims) { - if (lhs_or_rhs == 0) { - if (lhs->shape().dimensions(dim.lhs) == 1) { - continue; - } - partition_count *= sharding.tile_assignment().dim(dim.lhs); - } else { - if (rhs->shape().dimensions(dim.rhs) == 1) { - continue; - } - CHECK_EQ(lhs_or_rhs, 1); - partition_count *= sharding.tile_assignment().dim(dim.rhs); - } - } - return partition_count == sharding.tile_assignment().num_elements(); - }; - // If LHS/RHS is partitioned only along the batch dimensions, propagate - // the sharding to the output, since batch dimensions are the easiest to - // partition. - if (IsSpatiallyPartitioned(lhs) && - partitioned_only_along_non_trivial_dims(lhs->sharding(), - dot_dims->batch_dims, 0)) { - return MaybeImproveInstructionSharding(get_tiled_sharding_based_on_lhs(), - instruction, - may_combine_partial_sharding); - } - if (IsSpatiallyPartitioned(rhs) && - partitioned_only_along_non_trivial_dims(rhs->sharding(), - dot_dims->batch_dims, 1)) { - return MaybeImproveInstructionSharding(get_tiled_sharding_based_on_rhs(), - instruction, - may_combine_partial_sharding); - } - if (aggressive_prop) { - // If LHS/RHS is partitioned only along the non-contracting - // dimensions, propagate the sharding to the output. - const bool can_propagate_from_lhs = - IsSpatiallyPartitioned(lhs) && - partitioned_only_along_non_trivial_dims( - lhs->sharding(), dot_dims->lhs_non_contracting_dims, 0); - const bool can_propagate_from_rhs = - IsSpatiallyPartitioned(rhs) && - partitioned_only_along_non_trivial_dims( - rhs->sharding(), dot_dims->rhs_non_contracting_dims, 1); - // If we can propagate from both operands, choose the larger one which - // should help us reduce communications. - if (can_propagate_from_lhs && can_propagate_from_rhs) { - if (Product(lhs->shape().dimensions()) >= - Product(rhs->shape().dimensions())) { - return MaybeImproveInstructionSharding( - get_tiled_sharding_based_on_lhs(), instruction, - may_combine_partial_sharding); - } else { - return MaybeImproveInstructionSharding( - get_tiled_sharding_based_on_rhs(), instruction, - may_combine_partial_sharding); - } - } - if (can_propagate_from_lhs) { - return MaybeImproveInstructionSharding( - get_tiled_sharding_based_on_lhs(), instruction, - may_combine_partial_sharding); - } - if (can_propagate_from_rhs) { - return MaybeImproveInstructionSharding( - get_tiled_sharding_based_on_rhs(), instruction, - may_combine_partial_sharding); - } - } - } - if (!IsSpatiallyPartitioned(lhs)) { return false; } @@ -859,83 +845,11 @@ bool InferShardingFromOperands(HloInstruction* instruction, instruction, /*may_combine_partial_sharding=*/is_spmd); } case HloOpcode::kDot: { - auto& dot_dim_numbs = instruction->dot_dimension_numbers(); - // Batch dimensions are the same for lhs and rhs on dot operations. - int64 num_batch_dims = dot_dim_numbs.lhs_batch_dimensions_size(); - std::vector contracting_dims(2); - contracting_dims[0] = dot_dim_numbs.lhs_contracting_dimensions(0); - contracting_dims[1] = dot_dim_numbs.rhs_contracting_dimensions(0); - std::vector ops_sharding(2, nullptr); - for (int64 op_num = 0; op_num < 2; ++op_num) { - const HloInstruction* op = instruction->operand(op_num); - if (IsSpatiallyPartitioned(op)) { - ops_sharding[op_num] = &op->sharding(); - } - } - if (ops_sharding[0] == nullptr && ops_sharding[1] == nullptr) { - return false; - } - - // Select representative operand. - int64 representative_op = -1; - if (ops_sharding[0] == nullptr) { - representative_op = 1; - } else if (ops_sharding[1] == nullptr) { - representative_op = 0; - } else if (ops_sharding[0]->IsReplicated() && - ops_sharding[1]->IsReplicated()) { - // Both replicated -> replicate - return MaybeImproveInstructionSharding( - HloSharding::Replicate(), instruction, - /*may_combine_partial_sharding=*/is_spmd); - } else if (!ops_sharding[0]->IsReplicated() && - !ops_sharding[1]->IsReplicated()) { - // Both tile sharded. The dot spatial partitioning implementation - // replicates the operand corresponding to the non-tiled dimension: - // dot(lhs, rhs), sharding={devices=[1, ..., n, 1]} replicates rhs - // dot(lhs, rhs), sharding={devices=[1, ..., 1, n]} replicates lhs - // so set sharding in order to replicate the smaller of lhs and rhs - representative_op = - ShapeUtil::ByteSizeOf(instruction->operand(0)->shape()) < - ShapeUtil::ByteSizeOf(instruction->operand(1)->shape()) - ? 1 - : 0; - } else { - // One is replicated and the other is tiled - pick the tiled one. - representative_op = ops_sharding[0]->IsReplicated() ? 1 : 0; - } - - if (ops_sharding[representative_op]->IsReplicated()) { - return MaybeImproveInstructionSharding( - HloSharding::Replicate(), instruction, - /*may_combine_partial_sharding=*/is_spmd); - } else { - // Tile-shard instruction according to representative op. - auto sharding = *ops_sharding[representative_op]; - if (instruction->shape().dimensions_size() != - sharding.tile_assignment().num_dimensions()) { - // It is necessarily the case of a matrix x vector, with - // representative_op being the matrix, because the vector op has the - // same shape as instruction. - CHECK_EQ(sharding.tile_assignment().num_dimensions(), - instruction->shape().dimensions_size() + 1); - // Reshape sharding so that last dimension is 1, and then remove - // last dimension. - std::vector non_batch_dims( - sharding.tile_assignment().num_dimensions() - num_batch_dims); - absl::c_iota(non_batch_dims, num_batch_dims); - sharding = hlo_sharding_util::ReshapeToTileDimension( - sharding, num_batch_dims, non_batch_dims); - auto tile_assignment = sharding.tile_assignment(); - auto dimensions = tile_assignment.dimensions(); - CHECK_EQ(dimensions.back(), 1); - dimensions.pop_back(); - tile_assignment.Reshape(dimensions); - sharding = HloSharding::Tile(tile_assignment); - } - return MaybeImproveInstructionSharding( - sharding, instruction, /*may_combine_partial_sharding=*/is_spmd); - } + const auto& dnums = + dot_as_convolution_util::ParseDotGeneralFromDot(instruction); + return InferDotShardingFromOperands( + instruction, dnums, + /*may_combine_partial_sharding=*/is_spmd); } case HloOpcode::kParameter: { auto parent_it = computation_map.find(instruction->parent()); @@ -1109,6 +1023,81 @@ bool InferShardingFromOperands(HloInstruction* instruction, return false; } +HloSharding InferDotOperandSharding( + const HloInstruction* instruction, + const dot_as_convolution_util::DotGeneralAsConvolutionDimsInfo& dnums, + int64 operand_index, bool may_combine_partial_sharding) { + auto operand = instruction->operand(operand_index); + auto other = instruction->operand(1 - operand_index); + std::vector output_dims_to_replicate; + std::vector other_operand_dims_to_replicate; + for (const auto& dim : operand_index == 0 ? dnums.rhs_non_contracting_dims + : dnums.lhs_non_contracting_dims) { + output_dims_to_replicate.push_back(dim.output); + other_operand_dims_to_replicate.push_back(operand_index == 0 ? dim.rhs + : dim.lhs); + } + // If this dot is interpreted from a conv, then contracting dims may have + // corresponding spatial dimensions in the output, and this operand's + // non-contracting dims may have corresponding spatial dims in the other + // operand. + for (const auto& dim : dnums.contracting_dims) { + if (dim.output >= 0) { + output_dims_to_replicate.push_back(dim.output); + } + } + for (const auto& dim : operand_index == 0 ? dnums.lhs_non_contracting_dims + : dnums.rhs_non_contracting_dims) { + int64 other_dim = operand_index == 0 ? dim.rhs : dim.lhs; + if (other_dim >= 0) { + other_operand_dims_to_replicate.push_back(other_dim); + } + } + auto output_other_dims_replicated = + hlo_sharding_util::PartiallyReplicateTiledShardingOnDims( + instruction->sharding(), output_dims_to_replicate); + std::vector output_to_operand_dims(instruction->shape().rank(), -1); + std::vector operand_to_output_dims(operand->shape().rank(), -1); + for (const auto& dim : dnums.batch_dims) { + output_to_operand_dims[dim.output] = operand_index == 0 ? dim.lhs : dim.rhs; + operand_to_output_dims[operand_index == 0 ? dim.lhs : dim.rhs] = dim.output; + } + for (const auto& dim : operand_index == 0 ? dnums.lhs_non_contracting_dims + : dnums.rhs_non_contracting_dims) { + output_to_operand_dims[dim.output] = operand_index == 0 ? dim.lhs : dim.rhs; + operand_to_output_dims[operand_index == 0 ? dim.lhs : dim.rhs] = dim.output; + } + auto sharding = *hlo_sharding_util::TransposeShardingWithCollapsedDims( + output_other_dims_replicated, output_to_operand_dims, + operand_to_output_dims); + if (IsSpatiallyPartitioned(other)) { + auto other_operand_dims_replicated = + hlo_sharding_util::PartiallyReplicateTiledShardingOnDims( + other->sharding(), other_operand_dims_to_replicate); + std::vector other_to_operand_dims(other->shape().rank(), -1); + std::vector operand_to_other_dims(operand->shape().rank(), -1); + for (const auto& dim : dnums.batch_dims) { + other_to_operand_dims[operand_index == 0 ? dim.rhs : dim.lhs] = + operand_index == 0 ? dim.lhs : dim.rhs; + operand_to_other_dims[operand_index == 0 ? dim.lhs : dim.rhs] = + operand_index == 0 ? dim.rhs : dim.lhs; + } + for (const auto& dim : dnums.contracting_dims) { + other_to_operand_dims[operand_index == 0 ? dim.rhs : dim.lhs] = + operand_index == 0 ? dim.lhs : dim.rhs; + operand_to_other_dims[operand_index == 0 ? dim.lhs : dim.rhs] = + operand_index == 0 ? dim.rhs : dim.lhs; + } + sharding = + MergeSharding(sharding, + *hlo_sharding_util::TransposeShardingWithCollapsedDims( + other_operand_dims_replicated, other_to_operand_dims, + operand_to_other_dims), + may_combine_partial_sharding); + } + return sharding; +} + // Return the sharding that should be propagated from user to instruction. absl::optional GetShardingFromUser( const HloInstruction& instruction, const HloInstruction& user, @@ -1186,62 +1175,10 @@ absl::optional GetShardingFromUser( case HloOpcode::kConvolution: { if (auto dot_dims = dot_as_convolution_util::ParseDotGeneralFromConvolution(&user)) { - const auto& dnums = user.convolution_dimension_numbers(); - auto partitioned_only_along_non_trivial_dims = - [&](const HloSharding& sharding, - std::vector& - dims) { - if (sharding.IsTileMaximal()) { - return false; - } - int64 partition_count = 1; - for (const auto& dim : dims) { - if (user.shape().dimensions(dim.output) == 1) { - continue; - } - partition_count *= sharding.tile_assignment().dim(dim.output); - } - return partition_count == - sharding.tile_assignment().num_elements(); - }; - // If output is partitioned only along the batch dimensions, or only - // along the non-contracting dimensions, propagate the sharding to the - // operand. - if (&instruction == user.operand(0) && - (partitioned_only_along_non_trivial_dims(user.sharding(), - dot_dims->batch_dims) || - partitioned_only_along_non_trivial_dims( - user.sharding(), dot_dims->lhs_non_contracting_dims))) { - std::vector lhs_to_output_indices(user.shape().rank()); - lhs_to_output_indices[dnums.input_batch_dimension()] = - dnums.output_batch_dimension(); - lhs_to_output_indices[dnums.input_feature_dimension()] = - dnums.output_feature_dimension(); - for (int64 i = 0; i < dnums.input_spatial_dimensions_size(); ++i) { - lhs_to_output_indices[dnums.input_spatial_dimensions(i)] = - dnums.output_spatial_dimensions(i); - } - return hlo_sharding_util::TransposeSharding(user.sharding(), - lhs_to_output_indices); - } - if (&instruction == user.operand(1) && - (partitioned_only_along_non_trivial_dims(user.sharding(), - dot_dims->batch_dims) || - partitioned_only_along_non_trivial_dims( - user.sharding(), dot_dims->rhs_non_contracting_dims))) { - std::vector rhs_to_output_indices(user.shape().rank()); - rhs_to_output_indices[dnums.kernel_input_feature_dimension()] = - dnums.output_batch_dimension(); - rhs_to_output_indices[dnums.kernel_output_feature_dimension()] = - dnums.output_feature_dimension(); - for (int64 i = 0; i < dnums.input_spatial_dimensions_size(); ++i) { - rhs_to_output_indices[dnums.kernel_spatial_dimensions(i)] = - dnums.output_spatial_dimensions(i); - } - return hlo_sharding_util::TransposeSharding(user.sharding(), - rhs_to_output_indices); - } + int64 op_idx = user.operand_index(&instruction); + return InferDotOperandSharding( + &user, *dot_dims, op_idx, + /*may_combine_partial_sharding=*/is_spmd); } return absl::nullopt; } @@ -1323,33 +1260,10 @@ absl::optional GetShardingFromUser( return new_sharding; } case HloOpcode::kDot: { - if (user.sharding().IsReplicated()) { - return user.sharding(); - } - auto& dim_numbers = user.dot_dimension_numbers(); int64 op_idx = user.operand_index(&instruction); - // Batch dimensions are the same on lhs and rhs for dot operations. - int64 num_batch_dims = dim_numbers.lhs_batch_dimensions_size(); - int64 num_spatial_dims = - instruction.shape().dimensions_size() - num_batch_dims; - if (num_spatial_dims == 1) { - // This is the vector of a matrix x vector operation -> replicate, - // since tiling on the vector would necessarily be on the contracting - // dimension, which we don't support. - CHECK_EQ(op_idx, 1); - return HloSharding::Replicate(); - } - // Instruction is necessarily a matrix because it is one of the operands - // of a matrix x matrix operation. - CHECK_EQ(num_spatial_dims, 2); - // Propagate tile sharding to the bigger operand, and replicate the other. - auto other_op = user.operand(op_idx ^ 1); - if (ShapeUtil::ByteSizeOf(instruction.shape()) > - ShapeUtil::ByteSizeOf(other_op->shape())) { - return user.sharding(); - } else { - return HloSharding::Replicate(); - } + auto dnums = dot_as_convolution_util::ParseDotGeneralFromDot(&user); + return InferDotOperandSharding(&user, dnums, op_idx, + /*may_combine_partial_sharding=*/is_spmd); } case HloOpcode::kReduce: { if (instruction.shape().rank() == 0) { diff --git a/tensorflow/compiler/xla/service/sharding_propagation_test.cc b/tensorflow/compiler/xla/service/sharding_propagation_test.cc index a182af001c2..5ed1398149b 100644 --- a/tensorflow/compiler/xla/service/sharding_propagation_test.cc +++ b/tensorflow/compiler/xla/service/sharding_propagation_test.cc @@ -675,13 +675,15 @@ ENTRY conv { %rhs = f32[2,2,1]{2,1,0} parameter(1) %conv = f32[3,2,3]{2,1,0} convolution(%lhs, %rhs), window={size=1}, dim_labels=bf0_oi0->bf0 - ROOT %tuple = f32[3,2,3]{2,1,0} tuple(%conv) + ROOT %tuple = (f32[3,2,3]{2,1,0}) tuple(%conv) })"; TF_ASSERT_OK_AND_ASSIGN(auto module, ParseAndReturnVerifiedModule(hlo_string)); TF_ASSERT_OK_AND_ASSIGN(bool changed, ShardingPropagation().Run(module.get())); - EXPECT_FALSE(changed); + EXPECT_TRUE(changed); + EXPECT_THAT(FindInstruction(module.get(), "conv"), + op::Sharding("{replicated}")); } TEST_F(ShardingPropagationTest, ConvolutionDifferentDimensionNumbers) { @@ -1047,7 +1049,7 @@ ENTRY %conv { %p0_copy_0 = f32[8,256,128] copy(%param.0), sharding={devices=[1,4,1]0,1,2,3} %p1_copy_0 = f32[8,128,512] copy(%param.1), - sharding={devices=[1,2,2]0,1,2,3} + sharding={devices=[1,1,4]0,1,2,3} %p2_copy = f32[8,128] copy(%param.2) %dot_prop_rhs = f32[8,256,512] dot(%p0_copy_0, %p1_copy_0), lhs_batch_dims={0}, rhs_batch_dims={0}, @@ -1076,16 +1078,18 @@ ENTRY %conv { ShardingPropagation().Run(module.get())); EXPECT_TRUE(changed); EXPECT_THAT(FindInstruction(module.get(), "dot_prop_rhs"), - op::Sharding("{devices=[1,2,2]0,1,2,3}")); + op::Sharding("{devices=[1,1,4]0,1,2,3}")); EXPECT_THAT(FindInstruction(module.get(), "dot_prop_lhs"), - op::Sharding("{devices=[1,2,2]0,1,2,3}")); + op::Sharding("{devices=[1,4,1]0,1,2,3}")); EXPECT_THAT(FindInstruction(module.get(), "dot_mat_vec"), op::Sharding("{devices=[1,4]0,1,2,3}")); - EXPECT_THAT(FindInstruction(module.get(), "p0_copy_1"), - op::Sharding("{replicated}")); - EXPECT_THAT(FindInstruction(module.get(), "p1_copy_1"), - op::Sharding("{devices=[1,2,2]0,1,2,3}")); + EXPECT_THAT( + FindInstruction(module.get(), "p0_copy_1"), + op::Sharding("{devices=[1,2,1,2]0,1,2,3 last_tile_dim_replicate}")); + EXPECT_THAT( + FindInstruction(module.get(), "p1_copy_1"), + op::Sharding("{devices=[1,1,2,2]0,2,1,3 last_tile_dim_replicate}")); EXPECT_THAT(FindInstruction(module.get(), "dot_back_prop_rhs"), op::Sharding("{devices=[1,2,2]0,1,2,3}")); } @@ -1114,6 +1118,114 @@ ENTRY %conv { op::Sharding("{devices=[2,2,1]0,1,2,3}")); } +TEST_F(ShardingPropagationTest, DotMergeOperands) { + const char* const hlo_string = R"( +HloModule module +ENTRY %conv { + %p0 = f32[8,256,512] parameter(0), + sharding={devices=[2,2,1,2]0,1,2,3,4,5,6,7 last_tile_dim_replicate} + %p1 = f32[8,128,512] parameter(1), + sharding={devices=[2,2,1,2]0,2,1,3,4,6,5,7 last_tile_dim_replicate} + %dot = f32[8,256,128] dot(%p0, %p1), + lhs_batch_dims={0}, rhs_batch_dims={0}, + lhs_contracting_dims={2}, rhs_contracting_dims={2} + ROOT %copy = f32[8,256,128] copy(%dot) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT(FindInstruction(module.get(), "dot"), + op::Sharding("{devices=[2,2,2]0,1,2,3,4,5,6,7}")); +} + +TEST_F(ShardingPropagationTest, DotMergeOperands2) { + const char* const hlo_string = R"( +HloModule module +ENTRY %conv { + %p0 = f32[8,256,512] parameter(0), sharding={devices=[2,2,2]0,1,2,3,4,5,6,7} + %p1 = f32[8,128,512] parameter(1), sharding={devices=[2,2,2]0,1,2,3,4,5,6,7} + %dot = f32[8,256,128] dot(%p0, %p1), + lhs_batch_dims={0}, rhs_batch_dims={0}, + lhs_contracting_dims={2}, rhs_contracting_dims={2} + ROOT %copy = f32[8,256,128] copy(%dot) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT( + FindInstruction(module.get(), "dot"), + op::Sharding( + "{devices=[2,2,1,2]0,1,2,3,4,5,6,7 last_tile_dim_replicate}")); +} + +TEST_F(ShardingPropagationTest, BackwardDotFromContracting) { + const char* const hlo_string = R"( +HloModule module +ENTRY %conv { + %p0 = f32[8,256,512] parameter(0), sharding={devices=[2,2,2]0,1,2,3,4,5,6,7} + %p1 = f32[8,128,512] parameter(1) + %copy1 = f32[8,128,512] copy(%p1) + %dot = f32[8,256,128] dot(%p0, %copy1), + lhs_batch_dims={0}, rhs_batch_dims={0}, + lhs_contracting_dims={2}, rhs_contracting_dims={2}, + sharding={devices=[2,1,2,2]0,1,2,3,4,5,6,7 last_tile_dim_replicate} + ROOT %copy = f32[8,256,128] copy(%dot) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT(FindInstruction(module.get(), "copy1"), + op::Sharding("{devices=[2,2,2]0,1,2,3,4,5,6,7}")); +} + +TEST_F(ShardingPropagationTest, ConvAsDotOnTrivialDims) { + const char* const hlo_string = R"( +HloModule module +ENTRY %conv { + %lhs = f32[128,1,1,1001] parameter(0), sharding={devices=[1,2,1,1]0,1} + %rhs = f32[1,1,1024,1001] parameter(1), sharding={devices=[1,2,1,1]0,1} + %convolution = f32[128,1,1,1024] convolution(%lhs, %rhs), + window={size=1x1 rhs_reversal=1x1}, dim_labels=b01f_01oi->b01f + ROOT %copy = f32[128,1,1,1024] copy(%convolution) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT(FindInstruction(module.get(), "convolution"), + op::Sharding("{devices=[1,1,2,1]0,1}")); +} + +TEST_F(ShardingPropagationTest, ConvAsDotOnTrivialDimsBackward) { + const char* const hlo_string = R"( +HloModule module +ENTRY %conv { + %p0 = f32[128,5,5,128] parameter(0) + %lhs = f32[128,5,5,128] copy(%p0) + %p1 = f32[5,5,128,768] parameter(1) + %rhs = f32[5,5,128,768] copy(%p1) + %convolution = f32[128,1,1,768] convolution(%lhs, %rhs), window={size=5x5}, + dim_labels=b01f_01io->b01f, sharding={devices=[1,2,1,1]0,1} + ROOT %copy = f32[128,1,1,768] copy(%convolution) +})"; + TF_ASSERT_OK_AND_ASSIGN(auto module, + ParseAndReturnVerifiedModule(hlo_string)); + TF_ASSERT_OK_AND_ASSIGN( + bool changed, ShardingPropagation(/*is_spmd=*/true).Run(module.get())); + EXPECT_TRUE(changed); + EXPECT_THAT(FindInstruction(module.get(), "lhs"), + op::Sharding("{replicated}")); + EXPECT_THAT(FindInstruction(module.get(), "rhs"), + op::Sharding("{replicated}")); +} + TEST_F(ShardingPropagationTest, ConcatFromUserUnshardedDim) { const char* const hlo_string = R"( HloModule module diff --git a/tensorflow/compiler/xla/service/spmd/BUILD b/tensorflow/compiler/xla/service/spmd/BUILD index ce19934bb88..dd3da796d61 100644 --- a/tensorflow/compiler/xla/service/spmd/BUILD +++ b/tensorflow/compiler/xla/service/spmd/BUILD @@ -48,6 +48,7 @@ cc_library( "//tensorflow/compiler/xla/service:pattern_matcher", "//tensorflow/compiler/xla/service:shape_inference", "//tensorflow/compiler/xla/service:tuple_simplifier", + "//tensorflow/core:lib", "//tensorflow/core/platform:numbers", "@com_google_absl//absl/algorithm:container", "@com_google_absl//absl/container:flat_hash_map", diff --git a/tensorflow/compiler/xla/service/spmd/dot_handler.cc b/tensorflow/compiler/xla/service/spmd/dot_handler.cc index 55ebe120d01..a24bafe26ce 100644 --- a/tensorflow/compiler/xla/service/spmd/dot_handler.cc +++ b/tensorflow/compiler/xla/service/spmd/dot_handler.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/compiler/xla/shape_util.h" #include "tensorflow/compiler/xla/util.h" #include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/lib/gtl/cleanup.h" #include "tensorflow/core/platform/numbers.h" namespace xla { @@ -102,6 +103,11 @@ StatusOr PartitionBaseCase( windowed_dot_general_loops) { const HloSharding& lhs_sharding = lhs.sharding(); const HloSharding& rhs_sharding = rhs.sharding(); + if (lhs_sharding.ReplicateOnLastTileDim() || + rhs_sharding.ReplicateOnLastTileDim() || + output_sharding.ReplicateOnLastTileDim()) { + return nullptr; + } std::vector lhs_to_rhs_indices(lhs.base_shape().rank(), -1); std::vector lhs_to_output_indices(lhs.base_shape().rank(), -1); std::vector rhs_to_lhs_indices(rhs.base_shape().rank(), -1); @@ -136,23 +142,23 @@ StatusOr PartitionBaseCase( populate_indices_mapping(mapping); } auto lhs_sharding_transposed_to_match_rhs = - TransposeShardingWithCollapsedDims(lhs_sharding, lhs_to_rhs_indices, - rhs_to_lhs_indices); + hlo_sharding_util::TransposeShardingWithCollapsedDims( + lhs_sharding, lhs_to_rhs_indices, rhs_to_lhs_indices); auto rhs_sharding_transposed_to_match_lhs = - TransposeShardingWithCollapsedDims(rhs_sharding, rhs_to_lhs_indices, - lhs_to_rhs_indices); + hlo_sharding_util::TransposeShardingWithCollapsedDims( + rhs_sharding, rhs_to_lhs_indices, lhs_to_rhs_indices); auto lhs_sharding_transposed_to_match_output = - TransposeShardingWithCollapsedDims(lhs_sharding, lhs_to_output_indices, - output_to_lhs_indices); + hlo_sharding_util::TransposeShardingWithCollapsedDims( + lhs_sharding, lhs_to_output_indices, output_to_lhs_indices); auto rhs_sharding_transposed_to_match_output = - TransposeShardingWithCollapsedDims(rhs_sharding, rhs_to_output_indices, - output_to_rhs_indices); + hlo_sharding_util::TransposeShardingWithCollapsedDims( + rhs_sharding, rhs_to_output_indices, output_to_rhs_indices); auto output_sharding_transposed_to_match_lhs = - TransposeShardingWithCollapsedDims(output_sharding, output_to_lhs_indices, - lhs_to_output_indices); + hlo_sharding_util::TransposeShardingWithCollapsedDims( + output_sharding, output_to_lhs_indices, lhs_to_output_indices); auto output_sharding_transposed_to_match_rhs = - TransposeShardingWithCollapsedDims(output_sharding, output_to_rhs_indices, - rhs_to_output_indices); + hlo_sharding_util::TransposeShardingWithCollapsedDims( + output_sharding, output_to_rhs_indices, rhs_to_output_indices); // LHS and RHS are partitioned the same way and only partitioned in batch // dimensions. @@ -576,9 +582,17 @@ StatusOr PartitionDotGroupOnBatch( const std::function( HloInstruction*, HloInstruction*, SpmdBuilder*)>& create_sharded_dot, HloModule* module, HloInstruction* original_hlo, + bool require_matching_devices_to_group, int64 threshold_for_windowed_einsum_mib, SpmdBuilder* b, std::vector* windowed_dot_general_loops) { + std::vector> + top_level_sharding_to_reset; + auto cleaner = tensorflow::gtl::MakeCleanup([&] { + for (auto& to_reset : top_level_sharding_to_reset) { + to_reset.first->set_sharding(to_reset.second); + } + }); std::vector lhs_dims; std::vector rhs_dims; std::vector output_dims; @@ -608,16 +622,20 @@ StatusOr PartitionDotGroupOnBatch( output_sharding_dims_adjusted_to_lhs[dim.output] = lhs.sharding().tile_assignment().dim(dim.lhs); } + if (require_matching_devices_to_group && lhs_rhs_dims_matching) { + lhs_rhs_dims_matching = + rhs.sharding() == UngroupSharding(AlignGroupsWith( + GroupShardingOnDims(rhs.sharding(), rhs_dims), + GroupShardingOnDims(lhs.sharding(), lhs_dims))); + } auto output_grouped = GroupShardingOnDims(output_sharding, output_dims); PartitionedHlo per_group_lhs = lhs; PartitionedHlo per_group_rhs = rhs; - auto lhs_sharding = lhs.sharding(); - auto rhs_sharding = rhs.sharding(); if (lhs_rhs_dims_matching) { auto lhs_grouped = GroupShardingOnDims(lhs.sharding(), lhs_dims); auto rhs_grouped = GroupShardingOnDims(rhs.sharding(), rhs_dims); - if (ShapeUtil::ByteSizeOf(lhs.base_shape()) > - ShapeUtil::ByteSizeOf(rhs.base_shape())) { + if (ShapeUtil::ByteSizeOf(lhs.hlo()->shape()) > + ShapeUtil::ByteSizeOf(rhs.hlo()->shape())) { rhs_grouped = AlignGroupsWith(std::move(rhs_grouped), lhs_grouped); rhs = rhs.Reshard(UngroupSharding(rhs_grouped)); } else { @@ -627,12 +645,17 @@ StatusOr PartitionDotGroupOnBatch( auto reshaped_output_tiling = output_sharding.tile_assignment(); reshaped_output_tiling.Reshape(output_sharding_dims_adjusted_to_lhs); output_grouped = AlignGroupsWith( - GroupShardingOnDims(HloSharding::Tile(reshaped_output_tiling), - output_dims), + GroupShardingOnDims( + output_sharding.ReplicateOnLastTileDim() + ? HloSharding::PartialTile(reshaped_output_tiling) + : HloSharding::Tile(reshaped_output_tiling), + output_dims), lhs_grouped); auto per_group_partitioner_state = CreatePerGroupPartitioningState( lhs.state(), lhs_grouped.device_groups, b); + top_level_sharding_to_reset.emplace_back(lhs.hlo(), lhs.sharding()); lhs.hlo()->set_sharding(lhs_grouped.sharding); + top_level_sharding_to_reset.emplace_back(rhs.hlo(), rhs.sharding()); rhs.hlo()->set_sharding(rhs_grouped.sharding); CHECK(lhs.hlo() != rhs.hlo() || lhs_grouped.sharding == rhs_grouped.sharding); @@ -654,9 +677,9 @@ StatusOr PartitionDotGroupOnBatch( int64 other_contracting_dim_partitions, std::vector* sharding_dims_adjusted_to_output) -> absl::optional { - if (operand.sharding().IsReplicated()) { + if (operand.sharding().IsTileMaximal()) { auto partially_sharded = PerGroupSliceFromReplicated( - operand.hlo(), operand.state().partition_id, + operand.Replicate().hlo(), operand.state().partition_id, output_grouped.device_groups, batch_dims, output_grouped.group_dim_sizes, b); partially_sharded->set_sharding(HloSharding::Replicate()); @@ -678,9 +701,16 @@ StatusOr PartitionDotGroupOnBatch( } int64 ratio = Product(*sharding_dims_adjusted_to_output) / reshaped_tiling.num_elements(); - if (ratio == non_contracting_dim_partitions && - (ratio != contracting_dim_partitions || - contracting_dim_partitions == other_contracting_dim_partitions)) { + if (operand.sharding().ReplicateOnLastTileDim() && + reshaped_tiling.dimensions().back() % ratio == 0) { + sharding_dims_adjusted_to_output->back() /= ratio; + if (sharding_dims_adjusted_to_output->back() == 1) { + sharding_dims_adjusted_to_output->pop_back(); + } + } else if (ratio == non_contracting_dim_partitions && + (ratio != contracting_dim_partitions || + contracting_dim_partitions == + other_contracting_dim_partitions)) { for (int64 dim : non_contracting_dims) { (*sharding_dims_adjusted_to_output)[dim] = 1; } @@ -688,6 +718,8 @@ StatusOr PartitionDotGroupOnBatch( for (int64 dim : contracting_dims) { (*sharding_dims_adjusted_to_output)[dim] = 1; } + } else { + return absl::nullopt; } } // If the operand is initially sharded more ways than the output in the @@ -699,9 +731,19 @@ StatusOr PartitionDotGroupOnBatch( } reshaped_tiling.Reshape(*sharding_dims_adjusted_to_output); auto grouped = AlignGroupsWith( - GroupShardingOnDims(HloSharding::Tile(reshaped_tiling), batch_dims), + GroupShardingOnDims(operand.base_shape().rank() < + sharding_dims_adjusted_to_output->size() + ? HloSharding::PartialTile(reshaped_tiling) + : HloSharding::Tile(reshaped_tiling), + batch_dims), output_grouped); + if (require_matching_devices_to_group && + operand.sharding() != UngroupSharding(grouped)) { + return absl::nullopt; + } auto resharded = operand.Reshard(UngroupSharding(grouped)); + top_level_sharding_to_reset.emplace_back(resharded.hlo(), + resharded.sharding()); resharded.hlo()->set_sharding(grouped.sharding); return PartitionedHlo(resharded.hlo(), GetPerGroupBaseShape(grouped, operand.base_shape()), @@ -757,9 +799,6 @@ StatusOr PartitionDotGroupOnBatch( create_sharded_dot, module, original_hlo, threshold_for_windowed_einsum_mib, b, windowed_dot_general_loops)); - // Make sure the operands' sharding are set to the ungrouped ones. - lhs.hlo()->set_sharding(lhs_sharding); - rhs.hlo()->set_sharding(rhs_sharding); dot->set_sharding(UngroupSharding(output_grouped)); return PartitionedHlo(dot, output_base_shape, lhs.state()) .Reshard(output_sharding) @@ -777,9 +816,18 @@ StatusOr PartitionDotGroupOnNonContracting( const std::function( HloInstruction*, HloInstruction*, SpmdBuilder*)>& create_sharded_dot, HloModule* module, HloInstruction* original_hlo, + bool require_matching_devices_to_group, int64 threshold_for_windowed_einsum_mib, SpmdBuilder* b, std::vector* windowed_dot_general_loops) { + std::vector> + top_level_sharding_to_reset; + auto cleaner = tensorflow::gtl::MakeCleanup([&] { + for (auto& to_reset : top_level_sharding_to_reset) { + to_reset.first->set_sharding(to_reset.second); + } + }); + const bool may_replicate_other_contracting_dims = (other_contracting_partitions == matching_non_contracting_partitions && other_non_contracting_partitions == @@ -790,8 +838,9 @@ StatusOr PartitionDotGroupOnNonContracting( std::vector other_group_dims; if (may_replicate_other_contracting_dims && (!may_replicate_other_non_contracting_dims || - ShapeUtil::ByteSizeOf(other.base_shape()) <= - ShapeUtil::ByteSizeOf(output_base_shape))) { + ShapeUtil::ByteSizeOf(other.hlo()->shape()) <= + ShapeUtil::ByteSizeOf( + MakePartitionedShape(output_base_shape, output_sharding)))) { for (const auto& dim : dims_mapping.contracting_dims) { other_group_dims.push_back(lhs_matching ? dim.rhs : dim.lhs); } @@ -801,7 +850,11 @@ StatusOr PartitionDotGroupOnNonContracting( : dims_mapping.lhs_non_contracting_dims) { other_group_dims.push_back(lhs_matching ? dim.rhs : dim.lhs); } - } else if (!other.sharding().IsReplicated()) { + } else if (!(other.sharding().ReplicateOnLastTileDim() && + other.sharding().tile_assignment().dimensions().back() % + matching_non_contracting_partitions == + 0) && + !other.sharding().IsReplicated()) { return nullptr; } auto matching_sharding_dims = @@ -822,12 +875,20 @@ StatusOr PartitionDotGroupOnNonContracting( auto reshaped_matching_tiling = matching.sharding().tile_assignment(); reshaped_matching_tiling.Reshape(matching_sharding_dims); auto matching_grouped = AlignGroupsWith( - GroupShardingOnDims(HloSharding::Tile(reshaped_matching_tiling), - matching_dims), + GroupShardingOnDims( + matching.sharding().ReplicateOnLastTileDim() + ? HloSharding::PartialTile(reshaped_matching_tiling) + : HloSharding::Tile(reshaped_matching_tiling), + matching_dims), output_grouped); + if (require_matching_devices_to_group && + matching.sharding() != UngroupSharding(matching_grouped)) { + return nullptr; + } matching = matching.Reshard(UngroupSharding(matching_grouped)); auto per_group_partitioner_state = CreatePerGroupPartitioningState( matching.state(), matching_grouped.device_groups, b); + top_level_sharding_to_reset.emplace_back(matching.hlo(), matching.sharding()); matching.hlo()->set_sharding(matching_grouped.sharding); auto matching_p = PartitionedHlo( matching.hlo(), @@ -835,11 +896,27 @@ StatusOr PartitionDotGroupOnNonContracting( per_group_partitioner_state); auto partially_replicated_other = other.hlo(); - if (!other.sharding().IsReplicated()) { + if (other.sharding().ReplicateOnLastTileDim() && + other.sharding().tile_assignment().dimensions().back() % + matching_non_contracting_partitions == + 0) { + auto grouped = AlignGroupsWith( + GroupShardingOnDims( + other.sharding(), + {other.sharding().tile_assignment().num_dimensions() - 1}, + {other.sharding().tile_assignment().dimensions().back() / + matching_non_contracting_partitions}), + output_grouped); + other = other.Reshard(UngroupSharding(grouped)); + partially_replicated_other = other.hlo(); + top_level_sharding_to_reset.emplace_back(other.hlo(), other.sharding()); + partially_replicated_other->set_sharding(grouped.sharding); + } else if (!other.sharding().IsReplicated()) { auto other_grouped = AlignGroupsWith(GroupShardingOnDims(other.sharding(), other_group_dims), output_grouped, /*ignore_group_order=*/true); other = other.Reshard(UngroupSharding(other_grouped)); + // TODO(yuanzx): Use reshard to replicate when ready. partially_replicated_other = other.ReplicatePartial(other_grouped.group_dims); partially_replicated_other->set_sharding(other_grouped.sharding); @@ -856,11 +933,161 @@ StatusOr PartitionDotGroupOnNonContracting( create_sharded_dot, module, original_hlo, threshold_for_windowed_einsum_mib, b, windowed_dot_general_loops)); - // Reset matching's sharding to the ungrouped one. - matching.hlo()->set_sharding(UngroupSharding(matching_grouped)); return dot; } +StatusOr PartitionDotGroupOnContracting( + PartitionedHlo lhs, PartitionedHlo rhs, int64 contracting_partitions, + int64 output_batch_partitions, int64 output_lhs_non_contracting_partitions, + int64 output_rhs_non_contracting_partitions, const Shape& output_base_shape, + const HloSharding& output_sharding, + const DotGeneralDimsMapping& dims_mapping, int64 num_partitions, + const std::function( + HloInstruction*, HloInstruction*, SpmdBuilder*)>& create_sharded_dot, + HloModule* module, HloInstruction* original_hlo, + bool require_matching_devices_to_group, + int64 threshold_for_windowed_einsum_mib, SpmdBuilder* b, + std::vector* + windowed_dot_general_loops) { + std::vector> + top_level_sharding_to_reset; + auto cleaner = tensorflow::gtl::MakeCleanup([&] { + for (auto& to_reset : top_level_sharding_to_reset) { + to_reset.first->set_sharding(to_reset.second); + } + }); + auto lhs_sharding = lhs.sharding(); + auto rhs_sharding = rhs.sharding(); + auto lhs_tile_shape = lhs_sharding.tile_assignment().dimensions(); + auto rhs_tile_shape = rhs_sharding.tile_assignment().dimensions(); + std::vector lhs_dims; + std::vector rhs_dims; + for (const auto& dim : dims_mapping.contracting_dims) { + lhs_dims.push_back(dim.lhs); + rhs_dims.push_back(dim.rhs); + } + if (ShapeUtil::ByteSizeOf(lhs.hlo()->shape()) > + ShapeUtil::ByteSizeOf(rhs.hlo()->shape())) { + for (const auto& dim : dims_mapping.contracting_dims) { + rhs_tile_shape[dim.rhs] = lhs_tile_shape[dim.lhs]; + } + auto new_tile = rhs.sharding().tile_assignment(); + new_tile.Reshape(rhs_tile_shape); + rhs_sharding = rhs_sharding.ReplicateOnLastTileDim() + ? HloSharding::PartialTile(new_tile) + : HloSharding::Tile(new_tile); + } else { + for (const auto& dim : dims_mapping.contracting_dims) { + lhs_tile_shape[dim.lhs] = rhs_tile_shape[dim.rhs]; + } + auto new_tile = lhs.sharding().tile_assignment(); + new_tile.Reshape(lhs_tile_shape); + lhs_sharding = lhs_sharding.ReplicateOnLastTileDim() + ? HloSharding::PartialTile(new_tile) + : HloSharding::Tile(new_tile); + } + auto lhs_grouped = GroupShardingOnDims(lhs_sharding, lhs_dims); + auto rhs_grouped = GroupShardingOnDims(rhs_sharding, rhs_dims); + if (ShapeUtil::ByteSizeOf(lhs.hlo()->shape()) > + ShapeUtil::ByteSizeOf(rhs.hlo()->shape())) { + rhs_grouped = AlignGroupsWith(rhs_grouped, lhs_grouped); + rhs_sharding = UngroupSharding(rhs_grouped); + if (require_matching_devices_to_group && rhs.sharding() != rhs_sharding) { + return nullptr; + } + rhs = rhs.Reshard(rhs_sharding); + } else { + lhs_grouped = AlignGroupsWith(lhs_grouped, rhs_grouped); + lhs_sharding = UngroupSharding(lhs_grouped); + if (require_matching_devices_to_group && lhs.sharding() != lhs_sharding) { + return nullptr; + } + lhs = lhs.Reshard(lhs_sharding); + } + top_level_sharding_to_reset.emplace_back(lhs.hlo(), lhs_sharding); + lhs.hlo()->set_sharding(lhs_grouped.sharding); + top_level_sharding_to_reset.emplace_back(rhs.hlo(), rhs_sharding); + rhs.hlo()->set_sharding(rhs_grouped.sharding); + + HloSharding inner_output_sharding = HloSharding::Replicate(); + HloSharding outer_output_tmp_sharding = HloSharding::Replicate(); + if (output_sharding.ReplicateOnLastTileDim() && + output_sharding.tile_assignment().dimensions().back() % + contracting_partitions == + 0) { + auto grouped = AlignGroupsWith( + GroupShardingOnDims( + output_sharding, + {output_sharding.tile_assignment().num_dimensions() - 1}, + {output_sharding.tile_assignment().dimensions().back() / + contracting_partitions}), + GroupShardingOnDims(lhs_sharding, lhs_dims)); + outer_output_tmp_sharding = UngroupSharding(grouped); + inner_output_sharding = std::move(grouped.sharding); + } else if (output_lhs_non_contracting_partitions == contracting_partitions || + output_rhs_non_contracting_partitions == contracting_partitions || + output_batch_partitions == contracting_partitions) { + std::vector group_dims; + if (output_lhs_non_contracting_partitions == contracting_partitions) { + for (const auto& dim : dims_mapping.lhs_non_contracting_dims) { + group_dims.push_back(dim.output); + } + } else if (output_rhs_non_contracting_partitions == + contracting_partitions) { + for (const auto& dim : dims_mapping.rhs_non_contracting_dims) { + group_dims.push_back(dim.output); + } + } else { + for (const auto& dim : dims_mapping.batch_dims) { + group_dims.push_back(dim.output); + } + } + auto grouped = + AlignGroupsWith(GroupShardingOnDims(output_sharding, group_dims), + GroupShardingOnDims(lhs_sharding, lhs_dims)); + inner_output_sharding = grouped.sharding; + outer_output_tmp_sharding = + hlo_sharding_util::PartiallyReplicateTiledShardingOnDims( + UngroupSharding(grouped), group_dims); + } + auto inner_state = CreatePerGroupPartitioningState( + lhs.state(), lhs_grouped.device_groups, b); + TF_ASSIGN_OR_RETURN( + auto dot, + PartitionDot( + PartitionedHlo(lhs.hlo(), + GetPerGroupBaseShape(lhs_grouped, lhs.base_shape()), + inner_state), + PartitionedHlo(rhs.hlo(), + GetPerGroupBaseShape(rhs_grouped, rhs.base_shape()), + inner_state), + MakePartitionedShape(output_base_shape, outer_output_tmp_sharding), + inner_output_sharding, dims_mapping, + num_partitions / contracting_partitions, create_sharded_dot, module, + original_hlo, threshold_for_windowed_einsum_mib, b, + windowed_dot_general_loops)); + if (!dot) { + return nullptr; + } + std::vector other_lhs_dims; + for (int64 i = 0; i < lhs_sharding.tile_assignment().num_dimensions(); ++i) { + if (!absl::c_linear_search(lhs_dims, i)) { + other_lhs_dims.push_back(i); + } + } + auto inverse_grouped = GroupShardingOnDims(lhs_sharding, other_lhs_dims); + auto ar = + CreatePerGroupPartitioningState(lhs.state(), + inverse_grouped.device_groups, b) + .collective_ops_creator.create_cross_partition_all_reduce( + b, dot, MakeBinaryAdd(output_base_shape.element_type(), module), + {}, (*lhs.state().next_channel_id)++); + ar->set_sharding(outer_output_tmp_sharding); + return PartitionedHlo(ar, output_base_shape, lhs.state()) + .Reshard(output_sharding) + .hlo(); +} + // Recursive partitioning function. If there are partial dimensions matching in // the operands and output, group the devices and recursively partition the // in-group dot. @@ -871,6 +1098,7 @@ StatusOr PartitionDot( const std::function( HloInstruction*, HloInstruction*, SpmdBuilder*)>& create_sharded_dot, HloModule* module, HloInstruction* original_hlo, + bool require_matching_devices_to_group, int64 threshold_for_windowed_einsum_mib, SpmdBuilder* b, std::vector* windowed_dot_general_loops) { @@ -941,8 +1169,8 @@ StatusOr PartitionDot( num_partitions, lhs_contracting_partitions, rhs_contracting_partitions, lhs_non_contracting_partitions, rhs_non_contracting_partitions, create_sharded_dot, module, - original_hlo, threshold_for_windowed_einsum_mib, b, - windowed_dot_general_loops)); + original_hlo, require_matching_devices_to_group, + threshold_for_windowed_einsum_mib, b, windowed_dot_general_loops)); if (dot) { return dot; } @@ -982,12 +1210,57 @@ StatusOr PartitionDot( : output_lhs_non_contracting_partitions, output_base_shape, output_sharding, dims_mapping, num_partitions, create_sharded_dot, module, original_hlo, + require_matching_devices_to_group, threshold_for_windowed_einsum_mib, b, windowed_dot_general_loops)); if (dot) { return dot; } } + // Case 3: Group partitions by contracting dimensions. + if (lhs_contracting_partitions == rhs_contracting_partitions && + lhs_contracting_partitions > 1) { + TF_ASSIGN_OR_RETURN( + auto dot, + PartitionDotGroupOnContracting( + lhs, rhs, lhs_contracting_partitions, output_batch_partitions, + output_lhs_non_contracting_partitions, + output_rhs_non_contracting_partitions, output_base_shape, + output_sharding, dims_mapping, num_partitions, create_sharded_dot, + module, original_hlo, require_matching_devices_to_group, + threshold_for_windowed_einsum_mib, b, windowed_dot_general_loops)); + if (dot) { + return dot; + } + } + return nullptr; +} + +StatusOr PartitionDot( + PartitionedHlo lhs, PartitionedHlo rhs, const Shape& output_base_shape, + const HloSharding& output_sharding, + const DotGeneralDimsMapping& dims_mapping, int64 num_partitions, + const std::function( + HloInstruction*, HloInstruction*, SpmdBuilder*)>& create_sharded_dot, + HloModule* module, HloInstruction* original_hlo, + int64 threshold_for_windowed_einsum_mib, SpmdBuilder* b, + std::vector* + windowed_dot_general_loops) { + // First try partitioning without resharding the groups, then try allow + // resharding the groups. + for (bool require_matching_devices_to_group : {true, false}) { + TF_ASSIGN_OR_RETURN( + auto try_partition, + PartitionDot(lhs, rhs, output_base_shape, output_sharding, dims_mapping, + num_partitions, create_sharded_dot, module, original_hlo, + require_matching_devices_to_group, + threshold_for_windowed_einsum_mib, b, + windowed_dot_general_loops)); + if (try_partition) { + return try_partition; + } + } + // Default action. TF_ASSIGN_OR_RETURN(auto dot, create_sharded_dot(lhs.Replicate().hlo(), rhs.Replicate().hlo(), b)); diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc index 343239983b0..a850c05600e 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner.cc @@ -1375,8 +1375,10 @@ Status SpmdPartitioningVisitor::HandleScatter(HloInstruction* hlo) { update_dim_to_index_dim[update_scatter_dims[i]] = indices_scatter_dim; index_dim_to_update_dim[indices_scatter_dim] = update_scatter_dims[i]; } - auto new_updates_sharding = TransposeShardingWithCollapsedDims( - indices.sharding(), index_dim_to_update_dim, update_dim_to_index_dim); + auto new_updates_sharding = + hlo_sharding_util::TransposeShardingWithCollapsedDims( + indices.sharding(), index_dim_to_update_dim, + update_dim_to_index_dim); CHECK(new_updates_sharding.has_value()); updates = updates.Reshard(*new_updates_sharding); // To avoid accumulating the initial operand multiple times during @@ -2243,8 +2245,10 @@ Status SpmdPartitioningVisitor::HandleGather(HloInstruction* hlo) { output_dim_to_index_dim[batch_dims[i]] = indices_batch_dim; index_dim_to_output_dim[indices_batch_dim] = batch_dims[i]; } - auto pgather_sharding = TransposeShardingWithCollapsedDims( - indices.sharding(), index_dim_to_output_dim, output_dim_to_index_dim); + auto pgather_sharding = + hlo_sharding_util::TransposeShardingWithCollapsedDims( + indices.sharding(), index_dim_to_output_dim, + output_dim_to_index_dim); CHECK(pgather_sharding.has_value()); pgather->set_sharding(*pgather_sharding); SetPartitionedHlo(hlo, [&]() { diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc index 3ffe2954d61..1dc4c474c49 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_test.cc @@ -4348,6 +4348,142 @@ ENTRY entry { EXPECT_THAT(root, AllOf(op::Shape("f32[4,4,12,32]"), op::Reshape(xpose))); } +TEST_F(SpmdPartitioningTest, SimpleDotPartial) { + const char* const hlo_string = R"( +HloModule module + +ENTRY entry { + %lhs = f32[2,24,100] parameter(0), + sharding={devices=[2,1,1,2]0,1,2,3 last_tile_dim_replicate} + %rhs = f32[2,32,100] parameter(1), + sharding={devices=[2,1,1,2]0,1,2,3 last_tile_dim_replicate} + ROOT %dot = f32[2,24,32] dot(%lhs, %rhs), + lhs_batch_dims={0}, rhs_batch_dims={0}, + lhs_contracting_dims={2}, rhs_contracting_dims={2}, + sharding={devices=[2,1,1,2]0,1,2,3 last_tile_dim_replicate} +})"; + + TF_ASSERT_OK_AND_ASSIGN(auto module, + PartitionComputation(hlo_string, /*num_devices=*/4)); + VLOG(1) << module->ToString(); + + auto lhs = AllOf(op::Shape("f32[1,24,100]"), op::Parameter(0)); + auto rhs = AllOf(op::Shape("f32[1,32,100]"), op::Parameter(1)); + auto dot = AllOf(op::Shape("f32[1,24,32]"), op::Dot(lhs, rhs)); + auto root = module->entry_computation()->root_instruction(); + EXPECT_THAT(root, dot); +} + +TEST_F(SpmdPartitioningTest, DotPartialContracting) { + const char* const hlo_string = R"( +HloModule module + +ENTRY entry { + %lhs = f32[24,100] parameter(0), + sharding={devices=[1,2,2]0,1,2,3 last_tile_dim_replicate} + %rhs = f32[32,100] parameter(1), + sharding={devices=[1,2,2]0,1,2,3 last_tile_dim_replicate} + ROOT %dot = f32[24,32] dot(%lhs, %rhs), + lhs_batch_dims={}, rhs_batch_dims={}, + lhs_contracting_dims={1}, rhs_contracting_dims={1}, + sharding={replicated} +})"; + + TF_ASSERT_OK_AND_ASSIGN(auto module, + PartitionComputation(hlo_string, /*num_devices=*/4)); + VLOG(1) << module->ToString(); + + auto lhs = AllOf(op::Shape("f32[24,50]"), op::Parameter(0)); + auto rhs = AllOf(op::Shape("f32[32,50]"), op::Parameter(1)); + auto dot = AllOf(op::Shape("f32[24,32]"), op::Dot(lhs, rhs)); + auto root = module->entry_computation()->root_instruction(); + EXPECT_THAT(root, op::AllReduce(dot)); +} + +TEST_F(SpmdPartitioningTest, DotPartialContracting2) { + const char* const hlo_string = R"( +HloModule module + +ENTRY entry { + %lhs = f32[24,100] parameter(0), + sharding={devices=[1,2,2]0,1,2,3 last_tile_dim_replicate} + %rhs = f32[32,100] parameter(1), + sharding={devices=[1,2,2]0,1,2,3 last_tile_dim_replicate} + ROOT %dot = f32[24,32] dot(%lhs, %rhs), + lhs_batch_dims={}, rhs_batch_dims={}, + lhs_contracting_dims={1}, rhs_contracting_dims={1}, + sharding={devices=[2,1,2]0,2,1,3 last_tile_dim_replicate} +})"; + + TF_ASSERT_OK_AND_ASSIGN(auto module, + PartitionComputation(hlo_string, /*num_devices=*/4)); + VLOG(1) << module->ToString(); + + auto lhs = AllOf(op::Shape("f32[24,50]"), op::Parameter(0)); + auto rhs = AllOf(op::Shape("f32[32,50]"), op::Parameter(1)); + auto dot = + AllOf(op::Shape("f32[12,32]"), + op::Dot(AllOf(op::Shape("f32[12,50]"), op::DynamicSlice(lhs, _, _)), + rhs)); + auto root = module->entry_computation()->root_instruction(); + EXPECT_THAT(root, op::AllReduce(dot)); +} + +TEST_F(SpmdPartitioningTest, DotBatchAndPartialContracting) { + const char* const hlo_string = R"( +HloModule module + +ENTRY entry { + %lhs = f32[4,24,100] parameter(0), + sharding={devices=[2,2,2]0,1,2,3,4,5,6,7} + %rhs = f32[4,32,100] parameter(1), + sharding={devices=[2,1,2,2]0,2,1,3,4,6,5,7 last_tile_dim_replicate} + ROOT %dot = f32[4,24,32] dot(%lhs, %rhs), + lhs_batch_dims={0}, rhs_batch_dims={0}, + lhs_contracting_dims={2}, rhs_contracting_dims={2}, + sharding={devices=[2,2,1,2]0,1,2,3,4,5,6,7 last_tile_dim_replicate} +})"; + + TF_ASSERT_OK_AND_ASSIGN(auto module, + PartitionComputation(hlo_string, /*num_devices=*/8)); + VLOG(1) << module->ToString(); + + auto lhs = AllOf(op::Shape("f32[2,12,50]"), op::Parameter(0)); + auto rhs = AllOf(op::Shape("f32[2,32,50]"), op::Parameter(1)); + auto dot = AllOf(op::Shape("f32[2,12,32]"), op::Dot(lhs, rhs)); + auto root = module->entry_computation()->root_instruction(); + EXPECT_THAT(root, op::AllReduce(dot)); +} + +TEST_F(SpmdPartitioningTest, DotPartialNonContracting) { + const char* const hlo_string = R"( +HloModule module + +ENTRY entry { + %lhs = f32[24,8,100] parameter(0), + sharding={devices=[2,1,1,2]0,1,2,3 last_tile_dim_replicate} + %rhs = f32[32,100] parameter(1), sharding={devices=[2,2]0,2,1,3} + ROOT %dot = f32[24,8,32] dot(%lhs, %rhs), + lhs_batch_dims={}, rhs_batch_dims={}, + lhs_contracting_dims={2}, rhs_contracting_dims={1}, + sharding={devices=[2,1,2]0,1,2,3} +})"; + + TF_ASSERT_OK_AND_ASSIGN(auto module, + PartitionComputation(hlo_string, /*num_devices=*/4)); + VLOG(1) << module->ToString(); + + auto lhs = AllOf(op::Shape("f32[12,8,100]"), op::Parameter(0)); + auto rhs = AllOf(op::Shape("f32[16,50]"), op::Parameter(1)); + auto partially_replicated_rhs = + AllOf(op::Shape("f32[16,100]"), + op::AllReduce(op::DynamicUpdateSlice(op::Broadcast(_), rhs, _, _))); + auto dot = + AllOf(op::Shape("f32[12,8,16]"), op::Dot(lhs, partially_replicated_rhs)); + auto root = module->entry_computation()->root_instruction(); + EXPECT_THAT(root, dot); +} + TEST_F(SpmdPartitioningTest, ElementwiseTest_PartialReplicateToTiledHaloExchange) { const char* const hlo_string = R"( diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc index 68e486afa83..da2a3a44405 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.cc @@ -1335,7 +1335,7 @@ GroupedSharding GroupShardingOnDims(const HloSharding& sharding, sharding.tile_assignment().dimensions(); std::vector group_dim_sizes(group_dims.size()); for (int64 i = 0; i < group_dims.size(); ++i) { - CHECK_GE(grouped_tiling_dims[group_dims[i]], group_dim_shards[i]); + CHECK_EQ(grouped_tiling_dims[group_dims[i]] % group_dim_shards[i], 0); group_dim_sizes[i] = grouped_tiling_dims[group_dims[i]] / group_dim_shards[i]; grouped_tiling_dims[group_dims[i]] = group_dim_shards[i]; @@ -1352,38 +1352,74 @@ GroupedSharding GroupShardingOnDims(const HloSharding& sharding, } device_groups[group_id].push_back(device); }); - Array grouped_tiling(grouped_tiling_dims); - grouped_tiling.FillIota(0); - return GroupedSharding( + auto grouped = GroupedSharding( std::move(device_groups), std::vector(group_dims.begin(), group_dims.end()), std::move(group_dim_sizes), sharding.tile_assignment().num_dimensions(), - HloSharding::Tile(grouped_tiling)); + HloSharding::Replicate()); + if (sharding.ReplicateOnLastTileDim()) { + grouped.data_rank--; + } + if (Product(grouped_tiling_dims) == 1 || + (sharding.ReplicateOnLastTileDim() && + Product(grouped_tiling_dims) == grouped_tiling_dims.back())) { + return grouped; + } + if (sharding.ReplicateOnLastTileDim() && grouped_tiling_dims.back() == 1) { + grouped_tiling_dims.pop_back(); + } + Array grouped_tiling(grouped_tiling_dims); + grouped_tiling.FillIota(0); + grouped.sharding = sharding.ReplicateOnLastTileDim() && + grouped_tiling_dims.size() == + sharding.tile_assignment().num_dimensions() + ? HloSharding::PartialTile(grouped_tiling) + : HloSharding::Tile(grouped_tiling); + return grouped; } HloSharding UngroupSharding(const GroupedSharding& grouped_sharding) { - CHECK(!grouped_sharding.sharding.IsTileMaximal()); - std::vector tiling_dims = - grouped_sharding.sharding.tile_assignment().dimensions(); + std::vector tiling_dims; + bool partial_sharding = false; + auto grouped_tiling = grouped_sharding.sharding.tile_assignment(); + if (grouped_sharding.sharding.IsTileMaximal()) { + tiling_dims = std::vector(grouped_sharding.data_rank, 1); + if (grouped_sharding.device_groups[0].size() != 1) { + // This is partial sharding. + tiling_dims.push_back(grouped_sharding.device_groups[0].size()); + partial_sharding = true; + } + grouped_tiling = Array(tiling_dims); + grouped_tiling.FillIota(0); + } else { + partial_sharding = grouped_sharding.sharding.ReplicateOnLastTileDim(); + tiling_dims = grouped_sharding.sharding.tile_assignment().dimensions(); + if (absl::c_linear_search(grouped_sharding.group_dims, + tiling_dims.size())) { + tiling_dims.push_back(1); + grouped_tiling.Reshape(tiling_dims); + partial_sharding = true; + } + } for (int64 i = 0; i < grouped_sharding.group_dims.size(); ++i) { - tiling_dims[grouped_sharding.group_dims[i]] = - grouped_sharding.group_dim_sizes[i]; + int64 dim = grouped_sharding.group_dims[i]; + tiling_dims[dim] = grouped_sharding.group_dim_sizes[i]; } Array tiling(tiling_dims); - grouped_sharding.sharding.tile_assignment().Each( - [&](absl::Span indices, int64 device) { - std::vector ungrouped_inds(indices.begin(), indices.end()); - for (int64 g = 0; g < grouped_sharding.device_groups.size(); ++g) { - int64 remaining_group_index = g; - for (int64 i = grouped_sharding.group_dims.size() - 1; i >= 0; --i) { - ungrouped_inds[grouped_sharding.group_dims[i]] = - remaining_group_index % grouped_sharding.group_dim_sizes[i]; - remaining_group_index /= grouped_sharding.group_dim_sizes[i]; - } - tiling(ungrouped_inds) = grouped_sharding.device_groups[g][device]; - } - }); - return HloSharding::Tile(tiling); + grouped_tiling.Each([&](absl::Span indices, int64 device) { + std::vector ungrouped_inds(indices.begin(), indices.end()); + for (int64 g = 0; g < grouped_sharding.device_groups.size(); ++g) { + int64 remaining_group_index = g; + for (int64 i = grouped_sharding.group_dims.size() - 1; i >= 0; --i) { + ungrouped_inds[grouped_sharding.group_dims[i]] = + remaining_group_index % grouped_sharding.group_dim_sizes[i]; + remaining_group_index /= grouped_sharding.group_dim_sizes[i]; + } + tiling(ungrouped_inds) = grouped_sharding.device_groups[g][device]; + } + }); + return partial_sharding ? HloSharding::PartialTile(tiling) + : HloSharding::Tile(tiling); } GroupedSharding AlignGroupsWith(GroupedSharding grouped_sharding, @@ -1437,12 +1473,15 @@ GroupedSharding AlignGroupsWith(GroupedSharding grouped_sharding, grouped_sharding.device_groups[g], reference.device_groups[ref_g]); } } - if (matching_groups) { + if (matching_groups && !grouped_sharding.sharding.IsTileMaximal()) { auto tiles = grouped_sharding.sharding.tile_assignment(); tiles.Each([&](absl::Span indices, int64* device) { *device = original_src_to_ref_permutation[*device]; }); - grouped_sharding.sharding = HloSharding::Tile(tiles); + grouped_sharding.sharding = + grouped_sharding.sharding.ReplicateOnLastTileDim() + ? HloSharding::PartialTile(tiles) + : HloSharding::Tile(tiles); } grouped_sharding.device_groups = std::move(reference.device_groups); return grouped_sharding; @@ -1453,6 +1492,9 @@ Shape GetPerGroupBaseShape(const GroupedSharding& grouped_sharding, auto result = original_base_shape; for (int64 i = 0; i < grouped_sharding.group_dims.size(); ++i) { int64 dim = grouped_sharding.group_dims[i]; + if (dim >= original_base_shape.rank()) { + continue; + } int64 groups = grouped_sharding.group_dim_sizes[i]; result.set_dimensions(dim, result.dimensions(dim) / groups); } @@ -1624,49 +1666,6 @@ HloInstruction* PerGroupSliceFromReplicated( shard_shape.dimensions())); } -absl::optional TransposeShardingWithCollapsedDims( - const HloSharding& source, absl::Span src_to_tgt, - absl::Span tgt_to_src) { - if (source.IsTileMaximal()) { - return source; - } - std::vector tgt_dims_skipping_new(tgt_to_src.size(), -1); - int64 skipped_tgt_dims = 0; - for (int64 i = 0; i < tgt_to_src.size(); ++i) { - if (tgt_to_src[i] < 0) { - skipped_tgt_dims++; - } else { - tgt_dims_skipping_new[i] = i - skipped_tgt_dims; - } - } - int64 skipped_src_dims = absl::c_count(src_to_tgt, -1); - std::vector perm(src_to_tgt.size()); - for (int64 i = 0; i < src_to_tgt.size(); ++i) { - if (src_to_tgt[i] < 0) { - if (source.tile_assignment().dim(i) > 1) { - return absl::nullopt; - } - perm[src_to_tgt.size() - skipped_src_dims] = i; - skipped_src_dims--; - } else { - perm[tgt_dims_skipping_new[src_to_tgt[i]]] = i; - } - } - auto tgt_sharding = hlo_sharding_util::TransposeSharding(source, perm); - if (skipped_tgt_dims == 0) { - return tgt_sharding; - } - auto reshape_tiles = tgt_sharding.tile_assignment(); - std::vector tgt_tiles(tgt_to_src.size(), 1); - for (int64 i = 0; i < tgt_tiles.size(); ++i) { - if (tgt_to_src[i] >= 0) { - tgt_tiles[i] = reshape_tiles.dim(tgt_dims_skipping_new[i]); - } - } - reshape_tiles.Reshape(tgt_tiles); - return HloSharding::Tile(reshape_tiles); -} - absl::optional ParseReductionComputation( const HloComputation* reduction_comp) { if (reduction_comp->num_parameters() != 2) { diff --git a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h index fab573c5111..69ed90a4b66 100644 --- a/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h +++ b/tensorflow/compiler/xla/service/spmd/spmd_partitioner_util.h @@ -291,16 +291,17 @@ bool CanReshardWithCollectivePermute(const HloSharding& source, struct GroupedSharding { GroupedSharding(std::vector> device_groups, std::vector group_dims, - std::vector group_dim_sizes, int64 rank, + std::vector group_dim_sizes, int64 data_rank, HloSharding grouped_sharding) : device_groups(std::move(device_groups)), group_dims(std::move(group_dims)), group_dim_sizes(std::move(group_dim_sizes)), + data_rank(data_rank), sharding(std::move(grouped_sharding)) {} std::vector> device_groups; std::vector group_dims; std::vector group_dim_sizes; - int64 rank; + int64 data_rank; HloSharding sharding; }; @@ -340,13 +341,6 @@ HloInstruction* PerGroupSliceFromReplicated( absl::Span group_dims, absl::Span group_dim_sizes, SpmdBuilder* b); -// Similar to hlo_sharding_util::TransposeSharding(), but allows removing/adding -// non-partitioned dimensions. In src_to_tgt and tgt_to_src, -1 represents a -// non-existing dimension. -absl::optional TransposeShardingWithCollapsedDims( - const HloSharding& source, absl::Span src_to_tgt, - absl::Span tgt_to_src); - // Returns the opcode if `reduction_comp` represents a simple binary elementwise // computation on the two operands. absl::optional ParseReductionComputation( From 1665e7a027d8827cc0f9ab1610fc642bcf8bf2ea Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 00:47:13 -0700 Subject: [PATCH 0878/1017] Integrate LLVM at llvm/llvm-project@30c1633386e7 Updates LLVM usage to match [30c1633386e7](https://github.com/llvm/llvm-project/commit/30c1633386e7) PiperOrigin-RevId: 326184631 Change-Id: I86a3aa009e684c4096f31b36bb0ac3f2852230cc --- tensorflow/workspace.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 95b9c91a62f..fbbeeecf182 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -712,8 +712,8 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): ) # Check out LLVM and MLIR from llvm-project. - LLVM_COMMIT = "950f1bf976b332eca60267b25bf759e2ad564e0c" - LLVM_SHA256 = "89b0e1e5d0cd56adfbe061fc42804088eaed6773e8ff9f1d597137b474055096" + LLVM_COMMIT = "30c1633386e7cfb01c0a54b31ccf4c3a3873e71b" + LLVM_SHA256 = "0cd17329d0981a86558beaafd2ae982af03fcebc71a659d8c134f39cb3988b3b" LLVM_URLS = [ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), "https://github.com/llvm/llvm-project/archive/{commit}.tar.gz".format(commit = LLVM_COMMIT), From 797eb7ea4651170adf483c58aed7352a10e4f40f Mon Sep 17 00:00:00 2001 From: Renjie Liu Date: Wed, 12 Aug 2020 01:43:39 -0700 Subject: [PATCH 0879/1017] Emit a warning for lstm fusion when batch size is not fixed. PiperOrigin-RevId: 326190548 Change-Id: I48a1921cbde14458327e4ce54d101de12023c111 --- .../lite/tests/prepare-composite-functions-tf.mlir | 1 + .../lite/transforms/prepare_composite_functions_tf.cc | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/mlir/lite/tests/prepare-composite-functions-tf.mlir b/tensorflow/compiler/mlir/lite/tests/prepare-composite-functions-tf.mlir index 6847cdd5874..9e8a957b34c 100644 --- a/tensorflow/compiler/mlir/lite/tests/prepare-composite-functions-tf.mlir +++ b/tensorflow/compiler/mlir/lite/tests/prepare-composite-functions-tf.mlir @@ -457,6 +457,7 @@ func @inference_standard_lstm_time_major_cannot_fuse(%arg0: tensor, % // ----- module { +// expected-warning @+1 {{we cannot fuse this lstm func because the batch size is not fixed, please consider setting fixed batch size}} func @dynamic_shape_non_fuse_standard_lstm(%arg0: tensor, %arg1: tensor, %arg2: tensor, %arg3: tensor<8x40xf32>, %arg4: tensor<10x40xf32>, %arg5: tensor<40xf32>) -> (tensor, tensor, tensor, tensor, tensor) attributes {tf._input_shapes = ["tfshape$dim { size: -1 } dim { size: 8 } dim { size: 8 }", "tfshape$dim { size: -1 } dim { size: 10 }", "tfshape$dim { size: -1 } dim { size: 10 }", "tfshape$unknown_rank: true", "tfshape$unknown_rank: true", "tfshape$unknown_rank: true"], tf.api_implements = "lstm_b4e9f0e7-ac55-42bc-8ef2-8496419a608c", tf.api_preferred_device = "CPU", tf.go_backwards = false, tf.time_major = true} { %0 = "tf.BatchMatMulV2"(%arg0, %arg3) {adj_x = false, adj_y = false} : (tensor, tensor<8x40xf32>) -> tensor %1 = "tf.Add"(%0, %arg5) : (tensor, tensor<40xf32>) -> tensor diff --git a/tensorflow/compiler/mlir/lite/transforms/prepare_composite_functions_tf.cc b/tensorflow/compiler/mlir/lite/transforms/prepare_composite_functions_tf.cc index 3be6246c0dd..0efd7187e16 100644 --- a/tensorflow/compiler/mlir/lite/transforms/prepare_composite_functions_tf.cc +++ b/tensorflow/compiler/mlir/lite/transforms/prepare_composite_functions_tf.cc @@ -261,7 +261,15 @@ LogicalResult CheckFusableKerasLstm(FuncOp lstm_func, ModuleOp module) { for (int i = 1; i < 3; ++i) { auto input = lstm_func.getArgument(i); auto input_type = input.getType().dyn_cast_or_null(); - if (!input_type || !input_type.hasStaticShape()) return failure(); + if (!input_type || !input_type.hasStaticShape()) { + lstm_func.emitWarning( + "we cannot fuse this lstm func because the batch size is not fixed, " + "please consider setting fixed batch size like " + "https://github.com/tensorflow/tensorflow/blob/master/tensorflow/" + "lite/examples/experimental_new_converter/" + "Keras_LSTM_fusion_Codelab.ipynb"); + return failure(); + } } return success(); From 2913133813721a5c13fbabe426d1dcd89c2ab0d3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 02:01:43 -0700 Subject: [PATCH 0880/1017] Update GraphDef version to 491. PiperOrigin-RevId: 326192248 Change-Id: I73fb57e909573fdbabd5fd65d867143eaa002216 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 34b358612ad..f60f3907c58 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 490 // Updated: 2020/8/11 +#define TF_GRAPH_DEF_VERSION 491 // Updated: 2020/8/12 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From 3ae4ccf38c9731343bd7a39db7bac8067335646a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 02:01:47 -0700 Subject: [PATCH 0881/1017] compat: Update forward compatibility horizon to 2020-08-12 PiperOrigin-RevId: 326192260 Change-Id: I6599f008cb6205929e6d054296478e22d72a6d1c --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index 3e91a8b3f45..b56c3fab967 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 11) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 12) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From 445fccd1b040d573d4722b7f4e2f9b09db1313d0 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Wed, 12 Aug 2020 06:03:59 -0700 Subject: [PATCH 0882/1017] Pass the inferred function types to the resolver's res_call - this simplifies the tfr_gen side. Update tfr_gen to correctly annotate more types. PiperOrigin-RevId: 326216963 Change-Id: I9242e86f02ec1d56614bd3c40247c107ed6af94e --- .../pyct/static_analysis/type_inference.py | 37 +++++++++++++----- .../static_analysis/type_inference_test.py | 38 +++++++++++++++---- 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_inference.py b/tensorflow/python/autograph/pyct/static_analysis/type_inference.py index a5ed40a1e53..9fc16480b32 100644 --- a/tensorflow/python/autograph/pyct/static_analysis/type_inference.py +++ b/tensorflow/python/autograph/pyct/static_analysis/type_inference.py @@ -79,13 +79,14 @@ class Resolver(object): """Resolves the type of a (possibly annotated) function argument.""" raise NotImplementedError('subclasses must implement') - def res_call(self, ns, types_ns, node, args, keywords): + def res_call(self, ns, types_ns, node, f_type, args, keywords): """Resolves the return type an external function or method call. Args: ns: namespace types_ns: types namespace node: str, the function name + f_type: types of the actual function being called, if known args: types of each respective argument in node.args keywords: types of each respective argument in node.keywords @@ -154,6 +155,9 @@ class _SymbolTable(object): return 'SymbolTable {}'.format(self.types) +NO_VALUE = object() + + class StmtInferrer(gast.NodeVisitor): """Runs type inference on a single AST statement. @@ -270,7 +274,17 @@ class StmtInferrer(gast.NodeVisitor): # Attempt to use the static value if known. parent_value = anno.Static.VALUE.of(node.value, None) if parent_value is not None: - static_value = getattr(parent_value, node.attr, None) + static_value = getattr(parent_value, node.attr, NO_VALUE) + + if static_value is NO_VALUE: + # Unexpected failure to resolve attribute. Ask the resolver about the + # full name instead. + types, static_value = self.resolver.res_name( + self.namespace, self.types_in, anno.Basic.QN.of(node)) + anno.setanno(node, anno.Static.VALUE, static_value) + if __debug__: + self._check_set(types) + return types else: # Fall back to the type if that is known. @@ -315,17 +329,17 @@ class StmtInferrer(gast.NodeVisitor): if ret_types is None: ret_types = {Any} - fn_types = set() + f_types = set() for rt in ret_types: - fn_types.add(Callable[[Any], rt]) + f_types.add(Callable[[Any], rt]) - self.new_symbols[f_name] = fn_types + self.new_symbols[f_name] = f_types # The definition of a function is an expression, hence has no return value. return None - def _resolve_typed_callable(self, fn_types, arg_types, keyword_types): + def _resolve_typed_callable(self, f_types, arg_types, keyword_types): ret_types = set() - for t in fn_types: + for t in f_types: if isinstance(t, Callable): # Note: these are undocummented - may be version-specific! @@ -351,8 +365,8 @@ class StmtInferrer(gast.NodeVisitor): if f_name in self.scope.bound: # Local function, use local type definitions, if available. - fn_type = self.types_in.types.get(f_name, None) - if fn_type is None: + f_type = self.types_in.types.get(f_name, None) + if f_type is None: # No static type info available, nothing more to do. ret_type, side_effects = None, None else: @@ -361,9 +375,12 @@ class StmtInferrer(gast.NodeVisitor): else: # Nonlocal function, resolve externally. + f_type = anno.Static.TYPES.of(node.func, None) ret_type, side_effects = self.resolver.res_call(self.namespace, self.types_in.types, node, - arg_types, keyword_types) + f_type, arg_types, + keyword_types) + if __debug__: self._check_set(ret_type) if side_effects: diff --git a/tensorflow/python/autograph/pyct/static_analysis/type_inference_test.py b/tensorflow/python/autograph/pyct/static_analysis/type_inference_test.py index ae54cd98b25..3a371588303 100644 --- a/tensorflow/python/autograph/pyct/static_analysis/type_inference_test.py +++ b/tensorflow/python/autograph/pyct/static_analysis/type_inference_test.py @@ -44,6 +44,8 @@ class BasicTestResolver(type_inference.Resolver): return {type(value)} def res_arg(self, ns, types_ns, f_name, name, type_anno): + if type_anno is None: + return None return {str(type_anno)} @@ -167,19 +169,20 @@ class TypeInferenceAnalyzerTest(test.TestCase): def test_expr(self): - self_test = self + test_self = self class Resolver(type_inference.Resolver): def res_value(self, ns, value): - self_test.assertEqual(value, tc.a) + test_self.assertEqual(value, tc.a) return {str} def res_name(self, ns, types_ns, name): - self_test.assertEqual(name, qual_names.QN('tc')) + test_self.assertEqual(name, qual_names.QN('tc')) return {TestClass}, tc - def res_call(self, ns, types_ns, node, args, keywords): + def res_call(self, ns, types_ns, node, f_type, args, keywords): + test_self.assertEqual(f_type, (str,)) return {int}, None class TestClass: @@ -401,7 +404,8 @@ class TypeInferenceAnalyzerTest(test.TestCase): test_self.assertEqual(name, qual_names.QN('g')) return {str}, g - def res_call(self, ns, types_ns, node, args, keywords): + def res_call(self, ns, types_ns, node, f_type, args, keywords): + test_self.assertEqual(f_type, (str,)) test_self.assertEqual( anno.getanno(node.func, anno.Basic.QN), qual_names.QN('g')) return {float}, None @@ -433,7 +437,8 @@ class TypeInferenceAnalyzerTest(test.TestCase): def res_arg(self, ns, types_ns, f_name, name, type_anno): return {str(type_anno)} - def res_call(self, ns, types_ns, node, args, keywords): + def res_call(self, ns, types_ns, node, f_type, args, keywords): + test_self.assertIsNone(f_type) return None, {qual_names.QN('x'): {str}} def g(): @@ -540,6 +545,22 @@ class TypeInferenceAnalyzerTest(test.TestCase): self.assertTypes(fn_body[1].targets[0], float) self.assertClosureTypes(fn_body[0], {'x': {float}}) + def test_local_function_hides_locals(self): + + def test_fn(a: int): # pylint:disable=unused-argument + + def local_fn(v): + a = v + return a + + local_fn(1) + + node, _ = TestTranspiler(BasicTestResolver).transform(test_fn, None) + fn_body = node.body + + self.assertFalse( + anno.hasanno(fn_body[0].body[0].targets[0], anno.Static.TYPES)) + def test_local_function_type(self): def test_fn(x: int): @@ -564,7 +585,7 @@ class TypeInferenceAnalyzerTest(test.TestCase): def res_name(self, ns, types_ns, name): test_self.assertEqual(name, qual_names.QN('g')) - return None, g + return {Callable[[Callable], None]}, g def res_value(self, ns, value): test_self.assertEqual(value, 1.0) @@ -573,8 +594,9 @@ class TypeInferenceAnalyzerTest(test.TestCase): def res_arg(self, ns, types_ns, f_name, name, type_anno): return {str(type_anno)} - def res_call(self, ns, types_ns, node, args, keywords): + def res_call(self, ns, types_ns, node, f_type, args, keywords): test_self.assertEqual(node.func.id, 'g') + test_self.assertEqual(f_type, (Callable[[Callable], None],)) return None, {qual_names.QN('x'): {str}} def g(foo): From 4e3283a891ff73078158059fe749db56f5447ac0 Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Wed, 12 Aug 2020 06:09:09 -0700 Subject: [PATCH 0883/1017] Restore the safeguard around maximum_iterations, correctly this time. PiperOrigin-RevId: 326217531 Change-Id: I313ca522ccfc7111d9977dbb3fc93de482debb0d --- .../python/autograph/operators/control_flow.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tensorflow/python/autograph/operators/control_flow.py b/tensorflow/python/autograph/operators/control_flow.py index f194c446dc0..9bd139c031f 100644 --- a/tensorflow/python/autograph/operators/control_flow.py +++ b/tensorflow/python/autograph/operators/control_flow.py @@ -81,6 +81,7 @@ from tensorflow.python.framework import ops from tensorflow.python.framework import tensor_util from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import control_flow_util from tensorflow.python.ops import math_ops from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops.ragged import ragged_tensor @@ -445,6 +446,12 @@ def _py_for_stmt(iter_, extra_test, body, get_state, set_state): body(target) +def _add_max_iterations_hint(opts, n): + # TODO(b/159186914): Remove the safeguard, and always set maximum_iterations. + if control_flow_util.GraphOrParentsInXlaContext(ops.get_default_graph()): + opts['maximum_iterations'] = n + + def _known_len_tf_for_stmt( iter_, extra_test, body, get_state, set_state, symbol_names, opts): """Overload of for_stmt that iterates over TF entities that admit a length.""" @@ -478,7 +485,7 @@ def _known_len_tf_for_stmt( return control_flow_ops.cond(main_test, extra_test, lambda: False) return main_test - opts['maximum_iterations'] = n + _add_max_iterations_hint(opts, n) _tf_while_stmt( aug_test, @@ -524,7 +531,7 @@ def _tf_ragged_for_stmt( return control_flow_ops.cond(main_test, extra_test, lambda: False) return main_test - opts['maximum_iterations'] = n + _add_max_iterations_hint(opts, n) _tf_while_stmt( aug_test, @@ -582,8 +589,9 @@ def _tf_range_for_stmt( main_test = control_flow_ops.cond(main_test, extra_test, lambda: False) return main_test - opts['maximum_iterations'] = math_ops.cast( - misc.get_range_len(start, limit, delta), dtypes.int32) + _add_max_iterations_hint( + opts, + math_ops.cast(misc.get_range_len(start, limit, delta), dtypes.int32)) _tf_while_stmt( aug_test, From 131a9225ed292e762e668cca72bb1ebe3625904c Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Wed, 12 Aug 2020 07:57:41 -0700 Subject: [PATCH 0884/1017] Disable the tests that is timing out for the moment. PiperOrigin-RevId: 326232317 Change-Id: I46eff3c24bc8d48801328c0ed06cc2155589f433 --- tensorflow/python/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index 745fed375b8..df853c98dff 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3447,10 +3447,10 @@ cuda_py_test( srcs = ["ops/collective_ops_gpu_test.py"], python_version = "PY3", tags = [ - "guitar", "multi_gpu", "no_rocm", "no_windows", + "noguitar", # b/163673583 ], tfrt_enabled = True, deps = [ From 01f75da549bc0b77a0926ed24f913491199ed7e7 Mon Sep 17 00:00:00 2001 From: Tzu-Wei Sung Date: Wed, 12 Aug 2020 08:20:29 -0700 Subject: [PATCH 0885/1017] Fix eager tensor --- tensorflow/python/ops/signal/shape_ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/ops/signal/shape_ops.py b/tensorflow/python/ops/signal/shape_ops.py index c059f6d58d6..6725af8fb06 100644 --- a/tensorflow/python/ops/signal/shape_ops.py +++ b/tensorflow/python/ops/signal/shape_ops.py @@ -169,7 +169,7 @@ def frame(signal, frame_length, frame_step, pad_end=False, pad_value=0, axis=-1, # Pad the inner dimension of signal by pad_samples. paddings = array_ops.concat( [array_ops.zeros([num_outer_dimensions, 2], dtype=pad_samples.dtype), - [[0, pad_samples]], + ops.convert_to_tensor([[0, pad_samples]]), array_ops.zeros([num_inner_dimensions, 2], dtype=pad_samples.dtype)], 0) signal = array_ops.pad(signal, paddings, constant_values=pad_value) From 8aa578ab85991b7916f6b9b1b22aac71d5b08d45 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Wed, 12 Aug 2020 09:17:57 -0700 Subject: [PATCH 0886/1017] Parallel device: Canonicalize device strings so things like "CPU:1" work In the future I think we should do this lower in the stack. For now canonicalizing in the language bindings is OK. PiperOrigin-RevId: 326245650 Change-Id: I91e65df558ad6f6c620cc2063e175a2e0fe6bb2b --- tensorflow/python/distribute/parallel_device/BUILD | 1 + .../distribute/parallel_device/parallel_device.py | 3 ++- .../parallel_device/parallel_device_test.py | 11 ++++------- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tensorflow/python/distribute/parallel_device/BUILD b/tensorflow/python/distribute/parallel_device/BUILD index 930816d4407..e0f76fa6652 100644 --- a/tensorflow/python/distribute/parallel_device/BUILD +++ b/tensorflow/python/distribute/parallel_device/BUILD @@ -20,6 +20,7 @@ py_library( ":parallel_device_ops", ":saving", "//tensorflow/python:_pywrap_parallel_device", + "//tensorflow/python/distribute:device_util", ], ) diff --git a/tensorflow/python/distribute/parallel_device/parallel_device.py b/tensorflow/python/distribute/parallel_device/parallel_device.py index 57942652af6..aa4bc695815 100644 --- a/tensorflow/python/distribute/parallel_device/parallel_device.py +++ b/tensorflow/python/distribute/parallel_device/parallel_device.py @@ -21,6 +21,7 @@ from __future__ import print_function import threading from tensorflow.python import _pywrap_parallel_device +from tensorflow.python.distribute import device_util from tensorflow.python.distribute.parallel_device import gen_parallel_device_ops from tensorflow.python.distribute.parallel_device import saving from tensorflow.python.eager import context @@ -52,7 +53,7 @@ class ParallelDevice(object): A string with the name of the newly created device. """ global _next_device_number, _next_device_number_lock - self.components = tuple(components) + self.components = tuple(device_util.canonicalize(d) for d in components) ctx = context.context() with _next_device_number_lock: # TODO(allenl): Better names for parallel devices (right now "CUSTOM" is diff --git a/tensorflow/python/distribute/parallel_device/parallel_device_test.py b/tensorflow/python/distribute/parallel_device/parallel_device_test.py index 7239dcb2b09..9a0f9505535 100644 --- a/tensorflow/python/distribute/parallel_device/parallel_device_test.py +++ b/tensorflow/python/distribute/parallel_device/parallel_device_test.py @@ -100,13 +100,10 @@ class _VirtualDeviceTestCase(test.TestCase): context.LogicalDeviceConfiguration() ]) - # TODO(allenl): Make CPU:0 and CPU:1 work (right now "CPU:1" soft-places - # onto CPU:0, which seems wrong). - components = [ - "/job:localhost/replica:0/task:0/device:CPU:0", - "/job:localhost/replica:0/task:0/device:CPU:1" - ] - self.device = parallel_device.ParallelDevice(components) + self.device = parallel_device.ParallelDevice( + components=["/job:localhost/device:CPU:0", "CPU:1"]) + self.assertIn("CPU:0", self.device.components[0]) + self.assertIn("CPU:1", self.device.components[1]) class ParallelDeviceTests(_VirtualDeviceTestCase): From 2b6a14093af52d06f8a8091a1238ff4ef4280b34 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Wed, 12 Aug 2020 09:30:38 -0700 Subject: [PATCH 0887/1017] Parallel device: Add an error for usage inside a tf.function Eventually we'll support it, but that depends on https://github.com/tensorflow/community/pull/275, and the plan is to make it experimental first. PiperOrigin-RevId: 326248003 Change-Id: I0727de39030c1156492d761783706cf1ab109ec0 --- .../distribute/parallel_device/parallel_device.py | 11 +++++++++++ .../parallel_device/parallel_device_test.py | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/tensorflow/python/distribute/parallel_device/parallel_device.py b/tensorflow/python/distribute/parallel_device/parallel_device.py index aa4bc695815..94d43561a30 100644 --- a/tensorflow/python/distribute/parallel_device/parallel_device.py +++ b/tensorflow/python/distribute/parallel_device/parallel_device.py @@ -78,6 +78,7 @@ class ParallelDevice(object): Returns: A single tensor placed on the ParallelDevice. """ + self._assert_eager() with ops.device(self._name): return tpu_ops.tpu_replicated_input(inputs=tensors) @@ -90,6 +91,7 @@ class ParallelDevice(object): Returns: A flat list of tensors, one per `self.components`. """ + self._assert_eager() with ops.device(self._name): return tpu_ops.tpu_replicated_output( parallel_tensor, num_replicas=len(self.components)) @@ -106,11 +108,20 @@ class ParallelDevice(object): """ return self._device_ids + def _assert_eager(self): + """Verifies that tracing is not active.""" + if not context.executing_eagerly(): + raise NotImplementedError( + "ParallelDevice is currently not supported inside `tf.function`. It " + "can however run calls to a `tf.function` in parallel:\n\n" + "with ParallelDevice() as p:\n f()") + def __enter__(self): """Runs ops in parallel, makes variables which save independent buffers.""" if (self._device_scope is not None or self._saving_scope is not None): raise AssertionError( "Re-entered a ParallelDevice scope without first exiting it.") + self._assert_eager() self._device_scope = ops.device(self._name) self._saving_scope = saving.independent_buffers(self) self._device_scope.__enter__() diff --git a/tensorflow/python/distribute/parallel_device/parallel_device_test.py b/tensorflow/python/distribute/parallel_device/parallel_device_test.py index 9a0f9505535..39c0d5ba5d8 100644 --- a/tensorflow/python/distribute/parallel_device/parallel_device_test.py +++ b/tensorflow/python/distribute/parallel_device/parallel_device_test.py @@ -196,6 +196,15 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): result = broadcast_send_recv(self.device.device_ids) self.assertAllClose([[2], [6]], self.device.unpack(result)) + def test_use_in_graph_error_is_informative(self): + @def_function.function + def uses_parallel(): + with self.device: + return self.device.unpack(array_ops.ones([])) + + with self.assertRaisesRegex(NotImplementedError, "inside `tf.function`"): + uses_parallel() + def test_checkpointing(self): self.skipTest( "Disable saving until SaveableObject's methods are traceable.") From 9f57b5b37f9ef2cd08c52e6c60906f47d5e332cb Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Wed, 12 Aug 2020 09:37:28 -0700 Subject: [PATCH 0888/1017] Update XNNPACK dependency Pull a fix for google/XNNPACK#958 PiperOrigin-RevId: 326249295 Change-Id: I89f86b735fddfb726e0e2fbb2aeae7200ed3f58d --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index fbbeeecf182..dad05a1dac7 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -164,11 +164,11 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "XNNPACK", - sha256 = "c6eae589a4af7785da467162acd339bae359842e14c93bddc8fbe84ffd361c70", - strip_prefix = "XNNPACK-aff24e26a760552ee98a036f2a6e95b123e1bc6d", + sha256 = "7d017f5f4bad9851fea8acaac0a24b5b1fa8105a24c5c05580356baffb0f1801", + strip_prefix = "XNNPACK-84324860cd20c3518f82597e1f9df343635a9426", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/XNNPACK/archive/aff24e26a760552ee98a036f2a6e95b123e1bc6d.zip", - "https://github.com/google/XNNPACK/archive/aff24e26a760552ee98a036f2a6e95b123e1bc6d.zip", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/XNNPACK/archive/84324860cd20c3518f82597e1f9df343635a9426.zip", + "https://github.com/google/XNNPACK/archive/84324860cd20c3518f82597e1f9df343635a9426.zip", ], ) From c2d1f967b9db24662b5b9dac518cb16e293076a9 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Wed, 12 Aug 2020 09:44:36 -0700 Subject: [PATCH 0889/1017] Replace empty SmallVector with ArrayRef in TPUExtractOutsideCompilation (NFC). PiperOrigin-RevId: 326250674 Change-Id: Ie126d55cbf5457a0db5ac905d22959817703f85f --- .../transforms/tpu_extract_outside_compilation.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc b/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc index 9365807663a..99be132ca59 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc @@ -154,13 +154,13 @@ TF::IfRegionOp CloneEmptyIfWithPredicate(Value predicate, bool is_stateless, auto& then_branch = host_side_if.then_branch(); builder->setInsertionPoint(&then_branch.front(), then_branch.front().begin()); builder->createBlock(&then_branch); - builder->create(loc, llvm::SmallVector({})); + builder->create(loc, /*operands=*/ArrayRef{}); // Create empty else branch region. auto& else_branch = host_side_if.else_branch(); builder->setInsertionPoint(&else_branch.front(), else_branch.front().begin()); builder->createBlock(&else_branch); - builder->create(loc, llvm::SmallVector({})); + builder->create(loc, /*operands=*/ArrayRef{}); return host_side_if; } @@ -300,10 +300,9 @@ tf_device::LaunchOp CreateLaunchOpForOutsideCluster( llvm::StringRef host_device) { // An empty string placeholder is used for the device as that will be later // populated with the device of the associated TPUReplicateMetadata op. - llvm::SmallVector result_types; auto launch_op = builder->create( last_cluster_op->getLoc(), builder->getStringAttr(host_device), - result_types); + /*result_types=*/ArrayRef{}); launch_op.body().push_back(new Block); From 723820dc7e0be49296a78467ff65e61cff418b91 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 09:53:20 -0700 Subject: [PATCH 0890/1017] Qualify uses of std::string PiperOrigin-RevId: 326252261 Change-Id: If813554d87312940018bfc48069f7825b78d366c --- tensorflow/core/util/events_writer.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/util/events_writer.h b/tensorflow/core/util/events_writer.h index d5952c3cbdf..249bf31a108 100644 --- a/tensorflow/core/util/events_writer.h +++ b/tensorflow/core/util/events_writer.h @@ -44,7 +44,7 @@ class EventsWriter { // to the ultimate filename once Init() is called. // Note that it is not recommended to simultaneously have two // EventWriters writing to the same file_prefix. - explicit EventsWriter(const string& file_prefix); + explicit EventsWriter(const std::string& file_prefix); ~EventsWriter(); // Sets the event file filename and opens file for writing. If not called by @@ -54,11 +54,11 @@ class EventsWriter { // but has since disappeared (e.g. deleted by another process), this will open // a new file with a new timestamp in its filename. Status Init(); - Status InitWithSuffix(const string& suffix); + Status InitWithSuffix(const std::string& suffix); // Returns the filename for the current events file: // filename_ = [file_prefix_].out.events.[timestamp].[hostname][suffix] - string FileName(); + std::string FileName(); // Append "event" to the file. The "tensorflow::" part is for swig happiness. void WriteEvent(const tensorflow::Event& event); @@ -84,9 +84,9 @@ class EventsWriter { Status InitIfNeeded(); Env* env_; - const string file_prefix_; - string file_suffix_; - string filename_; + const std::string file_prefix_; + std::string file_suffix_; + std::string filename_; std::unique_ptr recordio_file_; std::unique_ptr recordio_writer_; int num_outstanding_events_; From 076a03b5cf29d0f79276ff48ac92170d4e958ab5 Mon Sep 17 00:00:00 2001 From: Yilei Yang Date: Wed, 12 Aug 2020 10:09:24 -0700 Subject: [PATCH 0891/1017] Make these Python tools use exec instead of host mode. PiperOrigin-RevId: 326255951 Change-Id: I2e74d15f11ac40fa8a5da62790748866e17c8e93 --- tensorflow/lite/experimental/acceleration/compatibility/BUILD | 4 ++-- tensorflow/tools/compatibility/BUILD | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/lite/experimental/acceleration/compatibility/BUILD b/tensorflow/lite/experimental/acceleration/compatibility/BUILD index 97c903d561f..9aa5a4b5459 100644 --- a/tensorflow/lite/experimental/acceleration/compatibility/BUILD +++ b/tensorflow/lite/experimental/acceleration/compatibility/BUILD @@ -91,7 +91,7 @@ genrule( --output_source_file $(location :devicedb-sample.cc) \ --array_variable_name g_tflite_acceleration_devicedb_sample_binary """, - tools = [":convert_binary_to_cc_source"], + exec_tools = [":convert_binary_to_cc_source"], ) cc_library( @@ -132,7 +132,7 @@ genrule( --output_source_file $(location :gpu_compatibility_binary.cc) \ --array_variable_name g_tflite_acceleration_gpu_compatibility_binary """, - tools = [":convert_binary_to_cc_source"], + exec_tools = [":convert_binary_to_cc_source"], ) cc_library( diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD index f161796a886..18a8d72a0b0 100644 --- a/tensorflow/tools/compatibility/BUILD +++ b/tensorflow/tools/compatibility/BUILD @@ -216,7 +216,7 @@ genrule( " --infile $(location testdata/test_file_v0_11.py)" + " --outfile $(location test_file_v1_0.py)" + " --reportfile $(location report.txt)"), - tools = [":tf_upgrade"], + exec_tools = [":tf_upgrade"], ) py_test( @@ -243,7 +243,7 @@ genrule( " --outfile $(location test_file_v2_0.py)" + " --reportfile $(location report_v2.txt) && " + "sed -i'.original' 's/_TEST_VERSION = 1/_TEST_VERSION = 2/g' $(location test_file_v2_0.py)"), - tools = [":tf_upgrade_v2"], + exec_tools = [":tf_upgrade_v2"], ) py_test( From 5ce9dd0dddae0bc687cc3b341ec5f81a91a20621 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Wed, 12 Aug 2020 10:40:12 -0700 Subject: [PATCH 0892/1017] Extend TPUClusterFormation pass to use resource alias analysis when deciding which interleaved ops to moved after the cluster. Ops not in the cluster may be interleaved with ops in a cluster due to ops being in a topological sort order. This ordering though takes into account control dependencies prior. When working with resources, as ordering of such ops are not done via data dependencies, resource ops may be moved before a cluster, when in fact they should be after. Resource alias analysis is used then to decide whether a resource has been used in a cluster before deciding to move an op to after the cluster. PiperOrigin-RevId: 326263053 Change-Id: I625bdfdb9208703cd8918cd76ed01b1a151bbdba --- .../tests/tpu_cluster_formation.mlir | 49 +++++++++++ .../mlir/tensorflow/transforms/bridge.cc | 2 +- .../mlir/tensorflow/transforms/passes.h | 2 +- .../transforms/tpu_cluster_formation.cc | 87 +++++++++++++++---- 4 files changed, 120 insertions(+), 20 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/tpu_cluster_formation.mlir b/tensorflow/compiler/mlir/tensorflow/tests/tpu_cluster_formation.mlir index 37dfec5e6df..978f6e74aa8 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/tpu_cluster_formation.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/tpu_cluster_formation.mlir @@ -331,6 +331,55 @@ func @mirrored_variables(%arg0: tensor>>, %arg1: ten // CHECK-SAME: _replicated_input_indices = [0, 1, 2] +// Test resource usage after resource use in cluster is moved to after the +// cluster. +// CHECK-LABEL: func @resource_after_cluster +// CHECK-SAME: ([[USED_RESOURCE:%.*]]: tensor<*x!tf.resource>>, [[UNUSED_RESOURCE:%.*]]: tensor<*x!tf.resource>>) +func @resource_after_cluster(%arg0: tensor<*x!tf.resource>>, %arg1: tensor<*x!tf.resource>>) { + // CHECK-NEXT: [[CONST:%.*]] = "tf.Const" + %0 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + + // CHECK-NEXT: "tf.AssignSubVariableOp"([[UNUSED_RESOURCE]], [[CONST]]) + + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.ReadVariableOp"([[USED_RESOURCE]]) + // CHECK-NEXT: "tf.NoOp" + // CHECK-NEXT: tf_device.return + "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster_test_fn", allow_soft_placement = false, computation_shape = [], device_assignment = [], host_compute_core = [], num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, padding_map = [], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> () + %1 = "tf.ReadVariableOp"(%arg0) {_tpu_replicate = "cluster_test_fn"} : (tensor<*x!tf.resource>>) -> tensor + + "tf.AssignSubVariableOp"(%arg1, %0) : (tensor<*x!tf.resource>>, tensor) -> () + + // CHECK: "tf.AssignAddVariableOp"([[USED_RESOURCE]], [[CONST]]) + "tf.AssignAddVariableOp"(%arg0, %0) : (tensor<*x!tf.resource>>, tensor) -> () + + "tf.NoOp"() {_tpu_replicate = "cluster_test_fn"} : () -> () + return +} + + +// Test resource not used by cluster is moved to before the cluster. +// CHECK-LABEL: func @resource_before_cluster +func @resource_before_cluster() { + // CHECK-NEXT: [[CONST:%.*]] = "tf.Const" + %0 = "tf.Const"() {value = dense<1.000000e+00> : tensor} : () -> tensor + + // CHECK-NEXT: [[UNUSED_RESOURCE:%.*]] = "tf.VarHandleOp" + // CHECK-NEXT: "tf.AssignAddVariableOp"([[UNUSED_RESOURCE]], [[CONST]]) + + // CHECK: "tf_device.cluster" + // CHECK-NEXT: "tf.NoOp" + // CHECK-NEXT: tf_device.return + "tf.TPUReplicateMetadata"() {_tpu_replicate = "cluster_test_fn", allow_soft_placement = false, computation_shape = [], device_assignment = [], host_compute_core = [], num_cores_per_replica = 1 : i64, num_replicas = 1 : i64, padding_map = [], step_marker_location = "STEP_MARK_AT_ENTRY", topology = "", use_spmd_for_xla_partitioning = false, use_tpu = true} : () -> () + + %1 = "tf.VarHandleOp"() {container = "", shape = #tf.shape<>, shared_name = "x"} : () -> tensor<*x!tf.resource>> + "tf.AssignAddVariableOp"(%1, %0) : (tensor<*x!tf.resource>>, tensor) -> () + + "tf.NoOp"() {_tpu_replicate = "cluster_test_fn"} : () -> () + return +} + + // ----- diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc b/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc index 2a5c8a05ef3..5b0a4b4e619 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/bridge.cc @@ -86,8 +86,8 @@ void CreateTPUBridgePipeline(OpPassManager &pm) { // Encode this in its own scope so that func_pm is not mistakenly used // later on. { + pm.addPass(CreateTPUClusterFormationPass()); OpPassManager &func_pm = pm.nest(); - func_pm.addPass(CreateTPUClusterFormationPass()); // Place DecomposeResourceOpsPass before TFExecutorConstantSinking pass // because DecomposeResourceOpsPass uses pattern rewriter which hoists // changed constants out of tf_device.Launch. diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/passes.h b/tensorflow/compiler/mlir/tensorflow/transforms/passes.h index 3be6c9e1a70..fb2d6e39da3 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/passes.h +++ b/tensorflow/compiler/mlir/tensorflow/transforms/passes.h @@ -269,7 +269,7 @@ std::unique_ptr> CreateLaunchToDeviceAttributePass(); namespace TFTPU { // Creates a pass that forms clusters from operations of the same // `_tpu_replicate` attribute. -std::unique_ptr> CreateTPUClusterFormationPass(); +std::unique_ptr> CreateTPUClusterFormationPass(); // Creates a pass that allows TPU program inputs to have layouts determined at // run time. diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/tpu_cluster_formation.cc b/tensorflow/compiler/mlir/tensorflow/transforms/tpu_cluster_formation.cc index 162ecd77d4f..f5bdd08d980 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/tpu_cluster_formation.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/tpu_cluster_formation.cc @@ -48,6 +48,7 @@ limitations under the License. #include "mlir/Pass/PassRegistry.h" // from @llvm-project #include "mlir/Support/LogicalResult.h" // from @llvm-project #include "mlir/Transforms/RegionUtils.h" // from @llvm-project +#include "tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.h" #include "tensorflow/compiler/mlir/tensorflow/ir/tf_device.h" #include "tensorflow/compiler/mlir/tensorflow/ir/tf_ops.h" @@ -75,8 +76,11 @@ using ClusterMap = llvm::SmallDenseMap, 8>; struct TPUClusterFormation - : public PassWrapper { - void runOnFunction() override; + : public TF::PerFunctionAggregateAnalysisConsumerPass< + TPUClusterFormation, TF::ResourceAliasAnalysis> { + void runOnFunction( + FuncOp func, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis); }; // Creates a mapping from the TPUReplicateMetadata ops `_tpu_replicate` @@ -138,12 +142,33 @@ LogicalResult CollectAndGroupClusterOps(Block* block, ClusterMap* clusters) { return success(); } +// Collects all resource ids from an op. +void CollectResourceIdsFromOp( + Operation& op, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis, + llvm::SmallDenseSet& observed_resource_ids) { + op.walk([&](Operation* inner_op) { + for (Value operand : TF::filter_resources(inner_op->getOperands())) { + if (resource_alias_analysis.IsUnknownResource(operand)) continue; + auto ids = resource_alias_analysis.GetResourceUniqueIds(operand); + observed_resource_ids.insert(ids.begin(), ids.end()); + } + for (Value result : TF::filter_resources(inner_op->getResults())) { + if (resource_alias_analysis.IsUnknownResource(result)) continue; + auto ids = resource_alias_analysis.GetResourceUniqueIds(result); + observed_resource_ids.insert(ids.begin(), ids.end()); + } + }); +} + // Checks if an op should be moved after a cluster. There may be users of a // cluster interleaved among the cluster ops. bool ShouldMoveOpAfterCluster( Block* block, Operation* op, const llvm::SmallSetVector& cluster_ops, - const llvm::SmallSetVector& preceding_users) { + const llvm::SmallSetVector& preceding_users, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis, + const llvm::SmallDenseSet& observed_resource_ids) { auto result = op->walk([&](Operation* op) { for (Value operand : op->getOperands()) { Operation* def = operand.getDefiningOp(); @@ -157,6 +182,14 @@ bool ShouldMoveOpAfterCluster( return WalkResult::interrupt(); } } + + // Check for uses of any resource in or after cluster. + for (Value operand : TF::filter_resources(op->getOperands())) { + if (resource_alias_analysis.IsUnknownResource(operand)) continue; + auto ids = resource_alias_analysis.GetResourceUniqueIds(operand); + for (const auto& id : ids) + if (observed_resource_ids.contains(id)) return WalkResult::interrupt(); + } return WalkResult::advance(); }); @@ -165,16 +198,30 @@ bool ShouldMoveOpAfterCluster( // Collects ops that are before ops in the cluster but are users of other ops // in the cluster. This may happen because users of individual ops in the -// cluster may be interleaved with other ops in the cluster. +// cluster may be interleaved with other ops in the cluster. Resource id's are +// also captured, to keep track of resource usage before, in, or after the +// cluster. +// TODO(lyandy): Extend this to handle all side effecting ops while handling +// transitive data dependencies. llvm::SmallSetVector CollectClusterPrecedingUsers( - Block* block, const llvm::SmallSetVector& cluster_ops) { + Block* block, const llvm::SmallSetVector& cluster_ops, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis) { llvm::SmallSetVector preceding_users; + llvm::SmallDenseSet observed_resource_ids; for (Operation& op : llvm::make_range(Block::iterator(cluster_ops.front()), - Block::iterator(cluster_ops.back()))) - if (cluster_ops.count(&op) == 0 && - ShouldMoveOpAfterCluster(block, &op, cluster_ops, preceding_users)) + Block::iterator(cluster_ops.back()))) { + if (cluster_ops.contains(&op)) { + CollectResourceIdsFromOp(op, resource_alias_analysis, + observed_resource_ids); + } else if (ShouldMoveOpAfterCluster( + block, &op, cluster_ops, preceding_users, + resource_alias_analysis, observed_resource_ids)) { preceding_users.insert(&op); + CollectResourceIdsFromOp(op, resource_alias_analysis, + observed_resource_ids); + } + } return preceding_users; } @@ -442,8 +489,9 @@ LogicalResult ReplicateCluster(tf_device::ClusterOp cluster, int num_replicas) { // 8. Wrap cluster (`tf_device.cluster`) in a `tf_device.replicate` if // attribute `num_replicas` is greater than 1. // 9. Copy over TPUReplicateMetadata attributes to `tf_device.cluster`. -LogicalResult FormClustersInBlock(Block* block, - const MetadataMap& metadata_map) { +LogicalResult FormClustersInBlock( + Block* block, const MetadataMap& metadata_map, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis) { ClusterMap clusters; LogicalResult result = CollectAndGroupClusterOps(block, &clusters); if (failed(result)) return result; @@ -464,7 +512,8 @@ LogicalResult FormClustersInBlock(Block* block, } llvm::SmallSetVector preceding_users = - CollectClusterPrecedingUsers(block, cluster_ops); + CollectClusterPrecedingUsers(block, cluster_ops, + resource_alias_analysis); llvm::SmallVector results = CollectClusterResults(block, cluster_ops); @@ -496,17 +545,19 @@ LogicalResult FormClustersInBlock(Block* block, return success(); } -void TPUClusterFormation::runOnFunction() { +void TPUClusterFormation::runOnFunction( + FuncOp func, + const TF::ResourceAliasAnalysis::Info& resource_alias_analysis) { MetadataMap metadata_map; - if (failed(CollectMetadata(getFunction(), &metadata_map))) - return signalPassFailure(); + if (failed(CollectMetadata(func, &metadata_map))) return signalPassFailure(); - for (Block& block : getFunction()) - if (failed(FormClustersInBlock(&block, metadata_map))) + for (Block& block : func) + if (failed( + FormClustersInBlock(&block, metadata_map, resource_alias_analysis))) return signalPassFailure(); // Remove TPUReplicatedInput and TPUReplicatedOutput nodes. - auto remove_result = getFunction().walk([&](Operation* op) { + auto remove_result = func.walk([&](Operation* op) { if (!llvm::isa(op)) return WalkResult::advance(); @@ -533,7 +584,7 @@ void TPUClusterFormation::runOnFunction() { } } // anonymous namespace -std::unique_ptr> CreateTPUClusterFormationPass() { +std::unique_ptr> CreateTPUClusterFormationPass() { return std::make_unique(); } From 9220304a54a3d864a1e3fe69a8120b32207c3e4e Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 12 Aug 2020 17:44:55 +0000 Subject: [PATCH 0893/1017] minor fixes --- .bazelversion | 2 +- tensorflow/core/kernels/map_kernels.h | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.bazelversion b/.bazelversion index 4a36342fcab..fd2a01863fd 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -3.0.0 +3.1.0 diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 6b38fe7aee6..29b40eebcd3 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -152,8 +152,8 @@ class TensorMapErase : public OpKernel { OP_REQUIRES(c, m->tensors().find(key) != m->tensors().end(), errors::InvalidArgument("Trying to erase non-existent item.")); - const Tensor& t = m->tensors().find(key)->second; - c->set_output(1, t); + //const Tensor& t = m->tensors().find(key)->second; + //c->set_output(1, t); TensorMap* output_map = nullptr; OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); @@ -179,7 +179,8 @@ class TensorMapHasKey : public OpKernel { template Status TensorMapBinaryAdd(OpKernelContext* c, const TensorMap& a, const TensorMap& b, TensorMap* out) { - // binary add returns a union of keys, values with keys in the intersection are added + // Binary add returns a map containing the union of keys. + // Values with keys in the intersection are added. out->tensors() = a.tensors(); for (const std::pair& p : b.tensors()) { absl::flat_hash_map::iterator it = out->tensors().find(p.first); @@ -197,7 +198,7 @@ Status TensorMapBinaryAdd(OpKernelContext* c, const TensorMap& a, template Status TensorMapZerosLike(OpKernelContext* c, const TensorMap& x, TensorMap* y) { - // zeros like returns an empty map + // Zeros like returns an empty map. return Status::OK(); } From 42b2ba9b712750782f82ea1626a569c60661b6b3 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 12 Aug 2020 10:50:09 -0700 Subject: [PATCH 0894/1017] Introduce implementations for unimplemented TpuTransferManager functions PiperOrigin-RevId: 326265398 Change-Id: Ied20bbc9e0ff0b95fa561031de1236a1020184d8 --- tensorflow/core/tpu/tpu_library_init_fns.inc | 4 ++ .../stream_executor/tpu/tpu_executor_c_api.h | 22 ++++++ .../tpu/tpu_transfer_manager.cc | 72 +++++++++++++++++++ .../tpu/tpu_transfer_manager.h | 16 ++--- 4 files changed, 102 insertions(+), 12 deletions(-) diff --git a/tensorflow/core/tpu/tpu_library_init_fns.inc b/tensorflow/core/tpu/tpu_library_init_fns.inc index 9ac4fb9ec6d..64113a9c496 100644 --- a/tensorflow/core/tpu/tpu_library_init_fns.inc +++ b/tensorflow/core/tpu/tpu_library_init_fns.inc @@ -164,6 +164,10 @@ tensorflow::Status SetExecutorStructFn(void* library_handle) { TFTPU_SET_FN(executor_fn, TpuTransferManager_GetInfeedLayout); TFTPU_SET_FN(executor_fn, TpuTransferManager_LinearizeToBuffers); TFTPU_SET_FN(executor_fn, TpuTransferManager_FreeBuffers); + TFTPU_SET_FN(executor_fn, TpuTransferManager_TransferLiteralToInfeed); + TFTPU_SET_FN(executor_fn, TpuTransferManager_TransferBuffersToInfeed); + TFTPU_SET_FN(executor_fn, TpuTransferManager_TransferLiteralFromOutfeed); + TFTPU_SET_FN(executor_fn, TpuTransferManager_ResetDevices); TFTPU_SET_FN(executor_fn, TpuComputationPlacer_New); TFTPU_SET_FN(executor_fn, TpuComputationPlacer_Free); diff --git a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h index 149a00615a9..3280060e28b 100644 --- a/tensorflow/stream_executor/tpu/tpu_executor_c_api.h +++ b/tensorflow/stream_executor/tpu/tpu_executor_c_api.h @@ -189,6 +189,24 @@ void TpuTransferManager_LinearizeToBuffers( int64_t** buffers_size, int64_t* buffers_array_size, SE_Status* status); void TpuTransferManager_FreeBuffers(char** buffers_array, int64_t* buffers_size, int64_t buffers_array_size); +void TpuTransferManager_TransferLiteralToInfeed(XLA_TransferManager* manager, + SE_StreamExecutor* executor, + XLA_Literal* c_literal, + SE_Status* status); +void TpuTransferManager_TransferBuffersToInfeed(XLA_TransferManager* manager, + SE_StreamExecutor* executor, + uint32_t** buffers_array, + int64_t* buffers_size_in_uint32, + int64_t buffers_array_size, + SE_Status* status); +void TpuTransferManager_TransferLiteralFromOutfeed(XLA_TransferManager* manager, + SE_StreamExecutor* executor, + XLA_Shape* shape, + XLA_Literal* c_literal, + SE_Status* status); +void TpuTransferManager_ResetDevices(XLA_TransferManager* manager, + SE_StreamExecutor** executors, + int64_t num_executors, SE_Status* status); XLA_ComputationPlacer* TpuComputationPlacer_New(); void TpuComputationPlacer_Free(XLA_ComputationPlacer* placer); @@ -354,6 +372,10 @@ struct TfTpu_ExecutorApiFn { TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_GetInfeedLayout); TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_LinearizeToBuffers); TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_FreeBuffers); + TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_TransferLiteralToInfeed); + TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_TransferBuffersToInfeed); + TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_TransferLiteralFromOutfeed); + TFTPU_ADD_FN_IN_STRUCT(TpuTransferManager_ResetDevices); TFTPU_ADD_FN_IN_STRUCT(TpuComputationPlacer_New); TFTPU_ADD_FN_IN_STRUCT(TpuComputationPlacer_Free); diff --git a/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc b/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc index 9b268a6d8c9..e6b3460ac92 100644 --- a/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc +++ b/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/stream_executor/tpu/noncopyable_buffer.h" #include "tensorflow/stream_executor/tpu/proto_helper.h" #include "tensorflow/stream_executor/tpu/status_helper.h" +#include "tensorflow/stream_executor/tpu/tpu_executor.h" #include "tensorflow/stream_executor/tpu/tpu_executor_c_api.h" #include "tensorflow/stream_executor/tpu/tpu_platform.h" @@ -81,6 +82,77 @@ Status TpuTransferManager::TransferLiteralToDeviceAsync( return status.status(); } +Status TpuTransferManager::TransferLiteralToInfeed( + stream_executor::StreamExecutor* executor, + const xla::LiteralSlice& literal) { + StatusHelper status; + XLA_Literal c_literal; + ApiConverter::ToC(literal, &c_literal); + auto* tpu_executor = static_cast(executor->implementation()); + + tpu::ExecutorApiFn()->TpuTransferManager_TransferLiteralToInfeedFn( + manager_, tpu_executor->se_executor(), &c_literal, status.c_status); + + return status.status(); +} + +Status TpuTransferManager::TransferBuffersToInfeed( + se::StreamExecutor* executor, + const std::deque& buffers) { + StatusHelper status; + auto* tpu_executor = static_cast(executor->implementation()); + + std::vector buffers_size; + std::vector buffers_array; + + buffers_size.reserve(buffers.size()); + buffers_array.reserve(buffers.size()); + + for (int64_t i = 0; i < buffers.size(); ++i) { + buffers_array.push_back( + const_cast(buffers[i].const_data().data())); + buffers_size.push_back(buffers[i].const_data().size()); + } + + tpu::ExecutorApiFn()->TpuTransferManager_TransferBuffersToInfeedFn( + manager_, tpu_executor->se_executor(), buffers_array.data(), + buffers_size.data(), buffers_size.size(), status.c_status); + return status.status(); +} + +Status TpuTransferManager::TransferLiteralFromOutfeed( + stream_executor::StreamExecutor* executor, const xla::Shape& literal_shape, + xla::MutableBorrowingLiteral literal) { + StatusHelper status; + XLA_Shape c_shape; + XLA_Literal c_literal; + auto* tpu_executor = static_cast(executor->implementation()); + + ApiConverter::ToC(literal_shape, &c_shape); + ApiConverter::ToC(literal, &c_literal); + + tpu::ExecutorApiFn()->TpuTransferManager_TransferLiteralFromOutfeedFn( + manager_, tpu_executor->se_executor(), &c_shape, &c_literal, + status.c_status); + + return status.status(); +} + +Status TpuTransferManager::ResetDevices( + absl::Span executor) { + StatusHelper status; + std::vector se; + se.reserve(executor.size()); + for (int64_t i = 0; i < executor.size(); ++i) { + se.push_back(static_cast(executor[i]->implementation()) + ->se_executor()); + } + + tpu::ExecutorApiFn()->TpuTransferManager_ResetDevicesFn( + manager_, se.data(), executor.size(), status.c_status); + return status.status(); +} + struct TransferFromDeviceState { std::atomic remaining_transfers; SE_Status* overall_status = diff --git a/tensorflow/stream_executor/tpu/tpu_transfer_manager.h b/tensorflow/stream_executor/tpu/tpu_transfer_manager.h index 558a5106d86..53596e09be2 100644 --- a/tensorflow/stream_executor/tpu/tpu_transfer_manager.h +++ b/tensorflow/stream_executor/tpu/tpu_transfer_manager.h @@ -51,27 +51,19 @@ class TpuTransferManager : public xla::TpuTransferManagerInterface { const TransferMetadata* transfer_metadata) override; Status TransferLiteralToInfeed(stream_executor::StreamExecutor* executor, - const xla::LiteralSlice& literal) override { - LOG(FATAL) << "Not yet implemented"; - } + const xla::LiteralSlice& literal) override; Status TransferLiteralFromOutfeed( stream_executor::StreamExecutor* executor, const xla::Shape& literal_shape, - xla::MutableBorrowingLiteral literal) override { - LOG(FATAL) << "Not yet implemented"; - } + xla::MutableBorrowingLiteral literal) override; Status TransferBuffersToInfeed( se::StreamExecutor* executor, - const std::deque& buffers) override { - LOG(FATAL) << "Not yet implemented."; - } + const std::deque& buffers) override; Status ResetDevices( - absl::Span executor) override { - LOG(FATAL) << "Not yet implemented"; - } + absl::Span executor) override; int64 GetByteSizeRequirement(const xla::Shape& shape) const override; From 17a15df7e4512a2aaa9f02e1ceb0df27861d4db4 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Wed, 12 Aug 2020 10:51:02 -0700 Subject: [PATCH 0895/1017] Add Kokoro build ID for more analysis PiperOrigin-RevId: 326265653 Change-Id: I0f7ce0d1ce58c380389e969a4604e93e7f4fd90e --- tensorflow/tools/ci_build/sizetrack_helper.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tensorflow/tools/ci_build/sizetrack_helper.py b/tensorflow/tools/ci_build/sizetrack_helper.py index 03a04e36588..e56009df332 100755 --- a/tensorflow/tools/ci_build/sizetrack_helper.py +++ b/tensorflow/tools/ci_build/sizetrack_helper.py @@ -99,6 +99,10 @@ parser.add_argument( "--job", type=str, help="Name of job calling this script. Default: $KOKORO_JOB_NAME.") +parser.add_argument( + "--build_id", + type=str, + help="UUID of build calling this script. Default: $KOKORO_BUILD_ID.") parser.add_argument( "--print_schema", action="store_true", @@ -151,6 +155,7 @@ SCHEMA = ",".join([ "logged_date:timestamp", "uploaded_to:string", "job:string", + "build_id:string", ]) # Select the earliest recorded commit in the same table for the same artifact # and team. Used to determine the full range of tested commits for each @@ -328,6 +333,7 @@ def build_row(): current_time, get_upload_path(), FLAGS.job, + FLAGS.build_id, ] @@ -347,6 +353,8 @@ def main(): if not FLAGS.job: FLAGS.job = os.environ.get("KOKORO_JOB_NAME", "NO_JOB") + if not FLAGS.build_id: + FLAGS.build_id = os.environ.get("KOKORO_BUILD_ID", "NO_BUILD") # Generate data about this artifact into a Tab Separated Value file next_tsv_row = build_row() From 86efb18ca5812c76dd52c8536f336e6962b7f8ca Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Wed, 12 Aug 2020 10:53:50 -0700 Subject: [PATCH 0896/1017] Document the unified API unit-tests parameterization PiperOrigin-RevId: 326266337 Change-Id: Ib4502bd984190f2dcb66ea6b3397c01e9835c3f9 --- tensorflow/c/eager/c_api_unified_experimental_test.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow/c/eager/c_api_unified_experimental_test.cc b/tensorflow/c/eager/c_api_unified_experimental_test.cc index c56e8ab05fc..c669ff4cf96 100644 --- a/tensorflow/c/eager/c_api_unified_experimental_test.cc +++ b/tensorflow/c/eager/c_api_unified_experimental_test.cc @@ -30,6 +30,9 @@ using tensorflow::string; namespace tensorflow { namespace { +// The tests are parameterized on: +// - a string representing the tracing implementation: "mlir" or "graphdef". +// - a boolean that when true enables TFRT as the execution engine. class UnifiedCAPI : public ::testing::TestWithParam> { protected: @@ -983,6 +986,10 @@ TEST_P(UnifiedCAPI, TF_ExecutionContextGetTFEContextFromFunctionContextRaises) { TF_DeleteExecutionContext(graph_ctx); } + +// The above tests are run for a combination of: +// - graphdef and MLIR tracing engine +// - Using TFRT as an execution runtime (true == enable TFRT) #ifdef PLATFORM_GOOGLE INSTANTIATE_TEST_SUITE_P(Tracing, UnifiedCAPI, ::testing::Combine(::testing::Values("graphdef", From f7a9580ed34f1dcb59b524993d8df1a5c7b19b93 Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Wed, 12 Aug 2020 10:57:06 -0700 Subject: [PATCH 0897/1017] Add LegalizeTFCommunication pass to compile_mlir_util. PiperOrigin-RevId: 326267120 Change-Id: Ia475e12be459f00b7557f4b627c5a81f3a6fd3b9 --- tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc index eee2f0a560c..99a5e32adc2 100644 --- a/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc +++ b/tensorflow/compiler/mlir/tensorflow/utils/compile_mlir_util.cc @@ -318,6 +318,7 @@ Status ConvertMLIRToXlaComputation( for (auto& target_pass : custom_legalization_passes) { tf2xla.addNestedPass(std::move(target_pass)); } + tf2xla.addPass(mlir::mhlo::CreateLegalizeTFCommunicationPass()); tf2xla.addNestedPass(mlir::createCanonicalizerPass()); // Run shape inference pass to propagate shapes through tensor_cast operations // from static to dynamic shapes. This could be generated if the shape From 41472b07cb220d238287ff33a5e1e4fb5d092180 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Wed, 12 Aug 2020 10:58:19 -0700 Subject: [PATCH 0898/1017] Fix typo in build script PiperOrigin-RevId: 326267426 Change-Id: I92e88dd6c0e94b245e1abafee74413ebf3c4720b --- .../tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat | 2 +- .../tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat | 2 +- tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat index fc64acea607..e583c5eeabf 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_libtensorflow.bat @@ -13,7 +13,7 @@ :: limitations under the License. :: ============================================================================= -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\bazel\run_libtensorflow.bat || exit /b 1 diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat index 975de5bab54..0cca1e29703 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py35.bat @@ -15,6 +15,6 @@ SET PYTHON_DIRECTORY=Python35 -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat index 16f2df362f9..c5ffe4b4b02 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py36.bat @@ -15,6 +15,6 @@ SET PYTHON_DIRECTORY=Python36 -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat index 2050d2050b1..a7670ee49c6 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py37.bat @@ -15,6 +15,6 @@ SET PYTHON_DIRECTORY=Python37 -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat index 145f5a17c7b..9aa5013c6b9 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/cpu_py38.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python38 -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow_cpu" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat index c7e7376e63f..bd15e83c24c 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_libtensorflow.bat @@ -13,7 +13,7 @@ :: limitations under the License. :: ============================================================================= -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\bazel\run_libtensorflow.bat || exit /b diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat index c6ca873607e..207359b32e3 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_pip_on_cpu.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python36 -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\integration\gpu_pip_on_cpu\run.bat diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat index 3ac22d3e7da..f4d68954ea6 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py35.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python35 -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat index 6a590d1dd87..3e87cdb7e3f 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py36.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python36 -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat index 34ec6c26124..105258fa468 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py37.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python37 -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat index a7dd094c811..94916342c3f 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/gpu_py38.bat @@ -15,7 +15,7 @@ SET PYTHON_DIRECTORY=Python38 -CALL tensorflow\tools\ci_build\rel\windows\common_win_cuda11.bat +CALL tensorflow\tools\ci_build\rel\windows_cuda11\common_win_cuda11.bat call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_test_flags "--test_env=TF2_BEHAVIOR=1" --project_name "tensorflow" From 99720318dff80146ccfb18ca184bdb73c0785f5d Mon Sep 17 00:00:00 2001 From: Andy Ly Date: Wed, 12 Aug 2020 11:00:23 -0700 Subject: [PATCH 0899/1017] Fix bug in TPUExtractOutsideCompilation where a builder is setting an insertion position in a region block when a block has not been added to the region. PiperOrigin-RevId: 326267964 Change-Id: Ibc6dfcfbd656c1f6282153b571c4bf49d7804872 --- .../transforms/tpu_extract_outside_compilation.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc b/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc index 99be132ca59..8adafe05cd3 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/tpu_extract_outside_compilation.cc @@ -152,14 +152,14 @@ TF::IfRegionOp CloneEmptyIfWithPredicate(Value predicate, bool is_stateless, // Create empty then branch region. auto& then_branch = host_side_if.then_branch(); - builder->setInsertionPoint(&then_branch.front(), then_branch.front().begin()); - builder->createBlock(&then_branch); + then_branch.push_back(new Block); + builder->setInsertionPointToEnd(&then_branch.front()); builder->create(loc, /*operands=*/ArrayRef{}); // Create empty else branch region. auto& else_branch = host_side_if.else_branch(); - builder->setInsertionPoint(&else_branch.front(), else_branch.front().begin()); - builder->createBlock(&else_branch); + else_branch.push_back(new Block); + builder->setInsertionPointToEnd(&else_branch.front()); builder->create(loc, /*operands=*/ArrayRef{}); return host_side_if; } From ccdb409311c964dae895b04ffdd26bfa3ad60b0b Mon Sep 17 00:00:00 2001 From: Ran Chen Date: Wed, 12 Aug 2020 11:00:57 -0700 Subject: [PATCH 0900/1017] Remove distributed_coordinator from v2 keras distributed_coordinator is not needed for TF2 multi worker training. It's still needed for TF1. If I understand ModelVersionSelector correctly, training.Model is actually only for TF2. WorkerTrainingState is only used in BackupAndRestore callback, which is a TF2 only symbol. PiperOrigin-RevId: 326268093 Change-Id: Ie90df74c4cfc5336934770daacdeb08e7fc81482 --- .../keras/distribute/worker_training_state.py | 7 ------ tensorflow/python/keras/engine/training.py | 24 ------------------- 2 files changed, 31 deletions(-) diff --git a/tensorflow/python/keras/distribute/worker_training_state.py b/tensorflow/python/keras/distribute/worker_training_state.py index 06cf46b3333..29939edc8f9 100644 --- a/tensorflow/python/keras/distribute/worker_training_state.py +++ b/tensorflow/python/keras/distribute/worker_training_state.py @@ -20,7 +20,6 @@ from __future__ import print_function import os from tensorflow.python.distribute import distributed_file_utils -from tensorflow.python.distribute import multi_worker_util from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.keras import backend as K @@ -104,12 +103,6 @@ class WorkerTrainingState(object): True if the training state is successfully restored. False if the training state doesn't need to be restored, or error occurred so it can't. """ - # For multi-worker training, it should not restore a model in certain - # worker setting (e.g. non-chief worker in ParameterServerStrategy). - # pylint: disable=protected-access - if self._model._in_multi_worker_mode( - ) and not multi_worker_util.should_load_checkpoint(): - return self.read_checkpoint_manager.restore_or_initialize() def delete_backup(self): diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index bf542129e5c..5f91fcef231 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -25,8 +25,6 @@ import os import six from tensorflow.python.autograph.lang import directives -from tensorflow.python.distribute import distribute_coordinator as dc -from tensorflow.python.distribute import distribute_coordinator_context as dc_context from tensorflow.python.distribute import distribution_strategy_context as ds_context from tensorflow.python.distribute import parameter_server_strategy from tensorflow.python.distribute import values as ds_values @@ -95,26 +93,6 @@ except ImportError: # pylint: enable=g-import-not-at-top -def enable_multi_worker(method): - """Decorator that handles running `method` with multi-worker strategy.""" - - def _method_wrapper(self, *args, **kwargs): - if not self._in_multi_worker_mode(): # pylint: disable=protected-access - return method(self, *args, **kwargs) - - # Running inside `run_distribute_coordinator` already. - if dc_context.get_current_worker_context(): - return method(self, *args, **kwargs) - - return dc.run_distribute_coordinator( - lambda _: method(self, *args, **kwargs), - self.distribute_strategy, - mode=dc.CoordinatorMode.INDEPENDENT_WORKER) - - return tf_decorator.make_decorator( - target=method, decorator_func=_method_wrapper) - - def disable_multi_worker(method): """Decorator that disallows multi-worker use of `method`.""" @@ -824,7 +802,6 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): self.train_function = train_function return self.train_function - @enable_multi_worker def fit(self, x=None, y=None, @@ -1246,7 +1223,6 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): self.test_function = test_function return self.test_function - @enable_multi_worker def evaluate(self, x=None, y=None, From 6018bd0ec009da71180efd3f90552f8b0f5131c8 Mon Sep 17 00:00:00 2001 From: Jiri Simsa Date: Wed, 12 Aug 2020 11:02:40 -0700 Subject: [PATCH 0901/1017] [tf.data] Disable broken test. PiperOrigin-RevId: 326268579 Change-Id: Ib4a9f86f461eba29efea6606e2459385d466e3b7 --- tensorflow/core/grappler/optimizers/data/vectorization/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/grappler/optimizers/data/vectorization/BUILD b/tensorflow/core/grappler/optimizers/data/vectorization/BUILD index 7eae11a0c0c..57962163562 100644 --- a/tensorflow/core/grappler/optimizers/data/vectorization/BUILD +++ b/tensorflow/core/grappler/optimizers/data/vectorization/BUILD @@ -53,6 +53,7 @@ tf_cc_test( name = "vectorizer_registry_test", srcs = ["vectorizer_registry_test.cc"], tags = [ + "manual", # TODO(b/159771496) "no_oss", # TODO(b/159771496) "notap", # TODO(b/159771496) ], From cac50d2ee3f5766d46dd0d2da2b161198b937269 Mon Sep 17 00:00:00 2001 From: Mehdi Amini Date: Wed, 12 Aug 2020 11:20:29 -0700 Subject: [PATCH 0902/1017] Enable gradient testing on the MLIR tracing API Two issues preventing enabling this test: - the DType conversion was tring to match the tensor instead of the dtype. - attributes were not added on the created MLIR operations, preventing export to GraphDef PiperOrigin-RevId: 326272404 Change-Id: Ied7c03c63766a6375cc2fae73de6bbc2f71c3b56 --- tensorflow/c/eager/gradients_test.cc | 4 ++-- .../mlir/tensorflow/c/c_api_unified_experimental_mlir.cc | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow/c/eager/gradients_test.cc b/tensorflow/c/eager/gradients_test.cc index 41993b3e125..8392e71520f 100644 --- a/tensorflow/c/eager/gradients_test.cc +++ b/tensorflow/c/eager/gradients_test.cc @@ -365,13 +365,13 @@ TEST_P(CppGradients, TestExpGrad) { #ifdef PLATFORM_GOOGLE INSTANTIATE_TEST_SUITE_P( UnifiedCAPI, CppGradients, - ::testing::Combine(::testing::Values("graphdef"), + ::testing::Combine(::testing::Values("graphdef", "mlir"), /*tfrt*/ ::testing::Values(true, false), /*executing_eagerly*/ ::testing::Values(true, false))); #else INSTANTIATE_TEST_SUITE_P( UnifiedCAPI, CppGradients, - ::testing::Combine(::testing::Values("graphdef"), + ::testing::Combine(::testing::Values("graphdef", "mlir"), /*tfrt*/ ::testing::Values(false), /*executing_eagerly*/ ::testing::Values(true, false))); #endif diff --git a/tensorflow/compiler/mlir/tensorflow/c/c_api_unified_experimental_mlir.cc b/tensorflow/compiler/mlir/tensorflow/c/c_api_unified_experimental_mlir.cc index 66447995709..edf5d09b401 100644 --- a/tensorflow/compiler/mlir/tensorflow/c/c_api_unified_experimental_mlir.cc +++ b/tensorflow/compiler/mlir/tensorflow/c/c_api_unified_experimental_mlir.cc @@ -95,7 +95,7 @@ class MlirTensor : public TracingTensorHandle { tensorflow::DataType DataType() const override { tensorflow::DataType type; - Status s = ConvertScalarTypeToDataType(value_.getType(), &type); + Status s = ConvertToDataType(value_.getType(), &type); if (!s.ok()) { return tensorflow::DT_INVALID; } @@ -458,6 +458,7 @@ Status MlirAbstractOp::Create(ArrayRef operands, } } } + for (auto& it : attrs_) state_->addAttribute(it.first(), it.second); *state = state_.get(); return Status::OK(); } From 97e2627e62a39a65e04df7cb820da51d8f9fa48a Mon Sep 17 00:00:00 2001 From: Ayush Dubey Date: Wed, 12 Aug 2020 11:36:31 -0700 Subject: [PATCH 0903/1017] Disable tfrt and reenable guitar for multi-GPU collective ops test. PiperOrigin-RevId: 326275987 Change-Id: Iee1403b13fd29dcfd3ee73b5a285fd1959241530 --- tensorflow/python/BUILD | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index df853c98dff..5a4d1e3f68d 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -3447,12 +3447,11 @@ cuda_py_test( srcs = ["ops/collective_ops_gpu_test.py"], python_version = "PY3", tags = [ + "guitar", "multi_gpu", "no_rocm", "no_windows", - "noguitar", # b/163673583 ], - tfrt_enabled = True, deps = [ ":client_testlib", ":collective_ops", From c9333c74d9ae563b9cdf2377017136ed77fb12d6 Mon Sep 17 00:00:00 2001 From: Katherine Tian Date: Wed, 12 Aug 2020 18:44:40 +0000 Subject: [PATCH 0904/1017] minor fix --- tensorflow/core/kernels/map_kernels.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow/core/kernels/map_kernels.h b/tensorflow/core/kernels/map_kernels.h index 29b40eebcd3..cd33accb4e1 100644 --- a/tensorflow/core/kernels/map_kernels.h +++ b/tensorflow/core/kernels/map_kernels.h @@ -152,8 +152,8 @@ class TensorMapErase : public OpKernel { OP_REQUIRES(c, m->tensors().find(key) != m->tensors().end(), errors::InvalidArgument("Trying to erase non-existent item.")); - //const Tensor& t = m->tensors().find(key)->second; - //c->set_output(1, t); + const Tensor& t = m->tensors().find(key)->second; + c->set_output(1, t); TensorMap* output_map = nullptr; OP_REQUIRES_OK(c, ForwardInputOrCreateNewMap(c, 0, 0, *m, &output_map)); From fcba9e4c22738396d14d4fbacb9ce3741107c45c Mon Sep 17 00:00:00 2001 From: Akshay Modi Date: Wed, 12 Aug 2020 11:37:21 -0700 Subject: [PATCH 0905/1017] Remove keras sequential warning for tf numpy ndarray input PiperOrigin-RevId: 326276141 Change-Id: I684b86aae09e26a343b125eb2092adc7c581fd3f --- tensorflow/python/keras/engine/sequential.py | 4 +++- tensorflow/python/ops/numpy_ops/np_interop_test.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/keras/engine/sequential.py b/tensorflow/python/keras/engine/sequential.py index 3b50506370b..c4c6be11b13 100644 --- a/tensorflow/python/keras/engine/sequential.py +++ b/tensorflow/python/keras/engine/sequential.py @@ -33,6 +33,7 @@ from tensorflow.python.keras.saving.saved_model import model_serialization from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import layer_utils from tensorflow.python.keras.utils import tf_utils +from tensorflow.python.ops.numpy_ops import np_arrays from tensorflow.python.platform import tf_logging as logging from tensorflow.python.training.tracking import base as trackable from tensorflow.python.util import nest @@ -352,7 +353,8 @@ class Sequential(functional.Functional): def call(self, inputs, training=None, mask=None): # pylint: disable=redefined-outer-name # If applicable, update the static input shape of the model. if not self._has_explicit_input_shape: - if not tensor_util.is_tensor(inputs): + if not tensor_util.is_tensor(inputs) and not isinstance( + inputs, np_arrays.ndarray): # This is a Sequential with mutiple inputs. This is technically an # invalid use case of Sequential, but we tolerate it for backwards # compatibility. diff --git a/tensorflow/python/ops/numpy_ops/np_interop_test.py b/tensorflow/python/ops/numpy_ops/np_interop_test.py index 0b474035edd..3b52ae5bafc 100644 --- a/tensorflow/python/ops/numpy_ops/np_interop_test.py +++ b/tensorflow/python/ops/numpy_ops/np_interop_test.py @@ -305,12 +305,12 @@ class InteropTest(tf.test.TestCase): model = tf.keras.Sequential( [tf.keras.layers.Dense(100), ProjectionLayer(2)]) - output = model.call(np.random.randn(10, 100)) + output = model.call(np.random.randn(10, 100).astype(np.float32)) self.assertIsInstance(output, np.ndarray) dense_layer = tf.keras.layers.Dense(100) - output = dense_layer(np.random.randn(10, 100)) + output = dense_layer(np.random.randn(10, 100).astype(np.float32)) def testPForInterop(self): def outer_product(a): From f11f25daaa4f4ad50daf3db3678080846268af05 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 11:40:57 -0700 Subject: [PATCH 0906/1017] Extend MUL operation with constant HWC tensor. PiperOrigin-RevId: 326276881 Change-Id: Ib68142bdb9f69667543f13039ca0d3a699466ed6 --- .../delegates/gpu/common/model_builder.cc | 4 ++ .../lite/delegates/gpu/gl/kernels/BUILD | 1 + .../lite/delegates/gpu/gl/kernels/mul.cc | 54 +++++++++++++------ .../lite/delegates/gpu/gl/kernels/mul_test.cc | 26 +++++++++ 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/common/model_builder.cc b/tensorflow/lite/delegates/gpu/common/model_builder.cc index 426d4d2436a..9c1a5cde132 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder.cc +++ b/tensorflow/lite/delegates/gpu/common/model_builder.cc @@ -1175,6 +1175,10 @@ class MulOperationParser : public TFLiteOperationParser { Tensor tensor; RETURN_IF_ERROR(reader->ReadTensor(constant_tensor, &tensor)); attr.param = tensor.data[0]; + } else if (constant_dims->size == 3) { + Tensor tensor; + RETURN_IF_ERROR(reader->ReadTensor(constant_tensor, &tensor)); + attr.param = std::move(tensor); } else { Tensor tensor; RETURN_IF_ERROR(reader->ReadTensor(constant_tensor, &tensor)); diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/BUILD b/tensorflow/lite/delegates/gpu/gl/kernels/BUILD index 774b6755014..8e13b58051b 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/BUILD +++ b/tensorflow/lite/delegates/gpu/gl/kernels/BUILD @@ -349,6 +349,7 @@ cc_library( srcs = ["mul.cc"], hdrs = ["mul.h"], deps = [ + "//tensorflow/lite/delegates/gpu/common:convert", "//tensorflow/lite/delegates/gpu/common:operations", "//tensorflow/lite/delegates/gpu/common:status", "//tensorflow/lite/delegates/gpu/common:types", diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/mul.cc b/tensorflow/lite/delegates/gpu/gl/kernels/mul.cc index b66decc3ca3..b524def4bf0 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/mul.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/mul.cc @@ -23,6 +23,7 @@ limitations under the License. #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" +#include "tensorflow/lite/delegates/gpu/common/convert.h" #include "tensorflow/lite/delegates/gpu/common/status.h" #include "tensorflow/lite/delegates/gpu/common/types.h" @@ -81,18 +82,10 @@ absl::Status GenerateApplyMaskCode(const NodeShader::GenerationContext& ctx, absl::Status GenerateMultiplyScalarCode( const NodeShader::GenerationContext& ctx, GeneratedCode* generated_code) { const auto& attr = absl::any_cast(ctx.op_attr); - auto muls = absl::get_if>(&attr.param); - auto scalar = absl::get_if(&attr.param); - const auto* hwc_tensor = - absl::get_if>(&attr.param); - if (hwc_tensor) { - return absl::UnimplementedError("Mul does not support HWC constant tensor"); - } - - if (scalar) { + if (absl::holds_alternative(attr.param)) { *generated_code = { - /*parameters=*/{{"scalar", *scalar}}, + /*parameters=*/{{"scalar", absl::get(attr.param)}}, /*objects=*/{}, /*shared_variables=*/{}, /*workload=*/uint3(), @@ -101,13 +94,16 @@ absl::Status GenerateMultiplyScalarCode( /*input=*/IOStructure::AUTO, /*output=*/IOStructure::AUTO, }; - } else { - if (!muls) { - return absl::InvalidArgumentError("Empty parameters for Multiplication."); - } + return absl::OkStatus(); + } + + if (absl::holds_alternative>(attr.param)) { *generated_code = { /*parameters=*/{}, - /*objects=*/{{"mul_buffer", MakeReadonlyObject(muls->data)}}, + /*objects=*/ + {{"mul_buffer", + MakeReadonlyObject( + absl::get>(attr.param).data)}}, /*shared_variables=*/{}, // Declare workload explicitly because shader depends on gid.z. /*workload=*/ @@ -119,9 +115,35 @@ absl::Status GenerateMultiplyScalarCode( /*input=*/IOStructure::AUTO, /*output=*/IOStructure::AUTO, }; + return absl::OkStatus(); } - return absl::OkStatus(); + if (absl::holds_alternative>(attr.param)) { + *generated_code = { + /*parameters=*/{}, + /*objects=*/ + {{"hwc_buffer", + MakeReadonlyObject( + uint3(static_cast(ctx.input_shapes[0][2]), + static_cast(ctx.input_shapes[0][1]), + DivideRoundUp(static_cast(ctx.input_shapes[0][3]), 4)), + ConvertToPHWC4( + absl::get>(attr.param)))}}, + /*shared_variables=*/{}, + // Declare workload explicitly because shader depends on gid.z. + /*workload=*/ + uint3(static_cast(ctx.input_shapes[0][2]), + static_cast(ctx.input_shapes[0][1]), + DivideRoundUp(static_cast(ctx.input_shapes[0][3]), 4)), + /*workgroup=*/uint3(), + /*source_code=*/"value_0 *= $hwc_buffer[gid.x, gid.y, gid.z]$;", + /*input=*/IOStructure::AUTO, + /*output=*/IOStructure::AUTO, + }; + return absl::OkStatus(); + } + + return absl::InvalidArgumentError("Unsupported Multiplication case."); } class Multiply : public NodeShader { diff --git a/tensorflow/lite/delegates/gpu/gl/kernels/mul_test.cc b/tensorflow/lite/delegates/gpu/gl/kernels/mul_test.cc index e19f00f763e..04e3ae46ec8 100644 --- a/tensorflow/lite/delegates/gpu/gl/kernels/mul_test.cc +++ b/tensorflow/lite/delegates/gpu/gl/kernels/mul_test.cc @@ -74,6 +74,32 @@ TEST(MulTest, Linear) { EXPECT_THAT(model.GetOutput(0), Pointwise(FloatNear(1e-6), {2, 6, 6, 12})); } +TEST(MulTest, ConstTensor3D) { + TensorRef input; + input.type = DataType::FLOAT32; + input.ref = 0; + input.shape = BHWC(1, 1, 2, 2); + + TensorRef output; + output.type = DataType::FLOAT32; + output.ref = 1; + output.shape = BHWC(1, 1, 2, 2); + + ElementwiseAttributes attr; + Tensor tensor_3d; + tensor_3d.shape.h = 1; + tensor_3d.shape.w = 2; + tensor_3d.shape.c = 2; + tensor_3d.id = 2; + tensor_3d.data = {-2, 2, -3, 3}; + attr.param = std::move(tensor_3d); + + SingleOpModel model({ToString(OperationType::MUL), attr}, {input}, {output}); + ASSERT_TRUE(model.PopulateTensor(0, {1, 2, 3, 4})); + ASSERT_OK(model.Invoke(*NewMultiplyNodeShader())); + EXPECT_THAT(model.GetOutput(0), Pointwise(FloatNear(1e-6), {-2, 4, -9, 12})); +} + TEST(MulTest, MaskChannel1) { TensorRef input; input.type = DataType::FLOAT32; From 57cfc6e3c1d3e28bfec8f728a9cc3c8d66d3e399 Mon Sep 17 00:00:00 2001 From: Ruoxin Sang Date: Wed, 12 Aug 2020 11:54:07 -0700 Subject: [PATCH 0907/1017] Add a test of map_fn with dynamic inputs. PiperOrigin-RevId: 326279638 Change-Id: Ieb4f022c152c3f3c9a611fb01adec5fa60bdf313 --- .../custom_training_loop_input_test.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tensorflow/python/distribute/custom_training_loop_input_test.py b/tensorflow/python/distribute/custom_training_loop_input_test.py index 832dc061f65..6b68e4aadef 100644 --- a/tensorflow/python/distribute/custom_training_loop_input_test.py +++ b/tensorflow/python/distribute/custom_training_loop_input_test.py @@ -35,6 +35,7 @@ from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import map_fn from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.util import nest @@ -659,6 +660,39 @@ class InputIterationTest(test.TestCase, parameterized.TestCase, # This assumes that there are exactly 2 replicas self.assertAllEqual([1.5, 2.], run(next(input_iterator))) + @combinations.generate( + combinations.combine( + distribution=strategy_combinations.multidevice_strategies, + mode=["eager"])) + def testMapFnWithDynamicInputs(self, distribution): + + def dataset_fn(_): + data = array_ops.zeros((20, 300, 32), dtype=dtypes.int32) + dataset = get_dataset_from_tensor_slices(data) + dataset = dataset.batch(16) + return dataset + + input_iterator = iter( + distribution.experimental_distribute_datasets_from_function(dataset_fn)) + + def embedding_lookup(inputs): + embedding_weights = array_ops.zeros((1, 128)) + flat_inputs = array_ops.reshape(inputs, [-1]) + embeddings = array_ops.gather(embedding_weights, flat_inputs) + embeddings = array_ops.reshape(embeddings, inputs.shape.as_list() + [128]) + return embeddings + + @def_function.function + def step_fn(example): + return map_fn.map_fn( + embedding_lookup, example, fn_output_signature=dtypes.float32) + + # This assumes that there are exactly 2 replicas + outputs = distribution.experimental_local_results( + distribution.run(step_fn, args=(next(input_iterator),))) + self.assertAllEqual((16, 300, 32, 128), outputs[0].shape) + self.assertAllEqual((4, 300, 32, 128), outputs[1].shape) + @combinations.generate( combinations.combine( distribution=strategy_combinations.all_strategies, From 4e82f18c05951dc12d3aa4c6cb59c09c4d3f08f7 Mon Sep 17 00:00:00 2001 From: Haoyu Zhang Date: Wed, 12 Aug 2020 11:58:44 -0700 Subject: [PATCH 0908/1017] Fix c_api_distributed_test to use the correct function def when not injecting errors. PiperOrigin-RevId: 326280622 Change-Id: I79965bc7b43332af7ca1542b8d27cb1ac64408ff --- tensorflow/c/eager/c_api_distributed_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/c/eager/c_api_distributed_test.cc b/tensorflow/c/eager/c_api_distributed_test.cc index 3738768cf02..cf35c2d634d 100644 --- a/tensorflow/c/eager/c_api_distributed_test.cc +++ b/tensorflow/c/eager/c_api_distributed_test.cc @@ -518,7 +518,8 @@ void TestDistributedFunctionCancellation(bool inject_error) { TFE_TensorHandle* var_handle = TestVariable(ctx, 2.0, dev2_name); EXPECT_NE(var_handle, nullptr); - const string function_def = VariableAddFunctionWithGraphError(); + const string function_def = inject_error ? VariableAddFunctionWithGraphError() + : VariableAddFunction(); TFE_ContextAddFunctionDef(ctx, function_def.data(), function_def.size(), status); ASSERT_EQ(TF_GetCode(status), TF_OK) << TF_Message(status); From 6bc271c2a8f364dc3b8a8be9a061700e694db5e1 Mon Sep 17 00:00:00 2001 From: Sanjoy Das Date: Wed, 12 Aug 2020 12:09:49 -0700 Subject: [PATCH 0909/1017] CUDA version needs to be 11.0 not 11 since CUDA is installed at C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.0 PiperOrigin-RevId: 326283178 Change-Id: Iefa8fec1b73d06ebe32ee11069fa23cbf6c9e7a7 --- .../tools/ci_build/rel/windows_cuda11/common_win_cuda11.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/tools/ci_build/rel/windows_cuda11/common_win_cuda11.bat b/tensorflow/tools/ci_build/rel/windows_cuda11/common_win_cuda11.bat index dea3a791277..81f2c86fa12 100644 --- a/tensorflow/tools/ci_build/rel/windows_cuda11/common_win_cuda11.bat +++ b/tensorflow/tools/ci_build/rel/windows_cuda11/common_win_cuda11.bat @@ -15,7 +15,7 @@ echo on -SET TF_CUDA_VERSION=11 +SET TF_CUDA_VERSION=11.0 SET TF_CUDNN_VERSION=8 REM TODO(sanjoy): This script should be removed once common_win.bat From a675ffca8e71775f038d49b45f8678752fbdc465 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 12:15:21 -0700 Subject: [PATCH 0910/1017] Pass version to custom attributes parsing. PiperOrigin-RevId: 326284357 Change-Id: Id6e9f457a45e659da2fc0d6cb6d91192e61611b2 --- .../delegates/gpu/common/custom_parsers.h | 6 +- .../gpu/common/default/custom_parsers.cc | 8 +- .../delegates/gpu/common/model_builder.cc | 74 +++++++++---------- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tensorflow/lite/delegates/gpu/common/custom_parsers.h b/tensorflow/lite/delegates/gpu/common/custom_parsers.h index 707087e6fdb..d70e5849315 100644 --- a/tensorflow/lite/delegates/gpu/common/custom_parsers.h +++ b/tensorflow/lite/delegates/gpu/common/custom_parsers.h @@ -27,9 +27,9 @@ namespace gpu { // Matches the custom operation by the string name and parses attributes stored // as flexbuffers. -absl::Status ParseCustomAttributes(absl::string_view op_name, const void* data, - uint32_t data_size, absl::any* attr, - BHWC* output_shape); +absl::Status ParseCustomAttributes(absl::string_view op_name, int version, + const void* data, uint32_t data_size, + absl::any* attr, BHWC* output_shape); } // namespace gpu } // namespace tflite diff --git a/tensorflow/lite/delegates/gpu/common/default/custom_parsers.cc b/tensorflow/lite/delegates/gpu/common/default/custom_parsers.cc index 9844b8d8aee..5aa1303d55c 100644 --- a/tensorflow/lite/delegates/gpu/common/default/custom_parsers.cc +++ b/tensorflow/lite/delegates/gpu/common/default/custom_parsers.cc @@ -26,11 +26,11 @@ limitations under the License. namespace tflite { namespace gpu { -absl::Status ParseCustomAttributes(absl::string_view op_name, const void* data, - uint32_t data_size, absl::any* attr, - BHWC* output_shape) { +absl::Status ParseCustomAttributes(absl::string_view op_name, int version, + const void* data, uint32_t data_size, + absl::any* attr, BHWC* output_shape) { return absl::UnimplementedError(absl::StrCat( - "Attributes parsing is not enabled for ", op_name, " operation")); + "Attributes parsing is not enabled for ", op_name, " operation.")); } } // namespace gpu diff --git a/tensorflow/lite/delegates/gpu/common/model_builder.cc b/tensorflow/lite/delegates/gpu/common/model_builder.cc index 9c1a5cde132..efa75a244bf 100644 --- a/tensorflow/lite/delegates/gpu/common/model_builder.cc +++ b/tensorflow/lite/delegates/gpu/common/model_builder.cc @@ -2196,10 +2196,10 @@ class RoIToTransformMatrixOperationParser : public TFLiteOperationParser { std::string op_name = "roi_to_transform_matrix"; node->operation.type = op_name; BHWC output_shape; - RETURN_IF_ERROR( - ParseCustomAttributes(op_name, tflite_node->custom_initial_data, - tflite_node->custom_initial_data_size, - &(node->operation.attributes), &output_shape)); + RETURN_IF_ERROR(ParseCustomAttributes( + op_name, registration->version, tflite_node->custom_initial_data, + tflite_node->custom_initial_data_size, &(node->operation.attributes), + &output_shape)); auto output_value = graph->FindOutputs(node->id)[0]; output_value->tensor.shape = output_shape; @@ -2229,10 +2229,10 @@ class RoIToTransformMatrixV2OperationParser : public TFLiteOperationParser { std::string op_name = "roi_to_transform_matrix_v2"; node->operation.type = op_name; BHWC output_shape; - RETURN_IF_ERROR( - ParseCustomAttributes(op_name, tflite_node->custom_initial_data, - tflite_node->custom_initial_data_size, - &(node->operation.attributes), &output_shape)); + RETURN_IF_ERROR(ParseCustomAttributes( + op_name, registration->version, tflite_node->custom_initial_data, + tflite_node->custom_initial_data_size, &(node->operation.attributes), + &output_shape)); auto output_value = graph->FindOutputs(node->id)[0]; output_value->tensor.shape = output_shape; @@ -2263,10 +2263,10 @@ class TransformTensorOperationParser : public TFLiteOperationParser { std::string op_name = "transform_tensor"; node->operation.type = op_name; BHWC output_shape; - RETURN_IF_ERROR( - ParseCustomAttributes(op_name, tflite_node->custom_initial_data, - tflite_node->custom_initial_data_size, - &(node->operation.attributes), &output_shape)); + RETURN_IF_ERROR(ParseCustomAttributes( + op_name, registration->version, tflite_node->custom_initial_data, + tflite_node->custom_initial_data_size, &(node->operation.attributes), + &output_shape)); auto output_value = graph->FindOutputs(node->id)[0]; @@ -2300,10 +2300,10 @@ class TransformTensorBilinearV2OperationParser : public TFLiteOperationParser { std::string op_name = "transform_tensor_bilinear_v2"; node->operation.type = op_name; BHWC output_shape; - RETURN_IF_ERROR( - ParseCustomAttributes(op_name, tflite_node->custom_initial_data, - tflite_node->custom_initial_data_size, - &(node->operation.attributes), &output_shape)); + RETURN_IF_ERROR(ParseCustomAttributes( + op_name, registration->version, tflite_node->custom_initial_data, + tflite_node->custom_initial_data_size, &(node->operation.attributes), + &output_shape)); auto output_value = graph->FindOutputs(node->id)[0]; @@ -2335,11 +2335,11 @@ class TransformLandmarksOperationParser : public TFLiteOperationParser { RETURN_IF_ERROR(reader->AddOutputs(node)); std::string op_name = "transform_landmarks"; node->operation.type = op_name; - BHWC output_shape; - RETURN_IF_ERROR( - ParseCustomAttributes(op_name, tflite_node->custom_initial_data, - tflite_node->custom_initial_data_size, - &(node->operation.attributes), &output_shape)); + BHWC output_shape = graph->FindOutputs(node->id)[0]->tensor.shape; + RETURN_IF_ERROR(ParseCustomAttributes( + op_name, registration->version, tflite_node->custom_initial_data, + tflite_node->custom_initial_data_size, &(node->operation.attributes), + &output_shape)); auto output_value = graph->FindOutputs(node->id)[0]; @@ -2373,10 +2373,10 @@ class TransformLandmarksV2OperationParser : public TFLiteOperationParser { auto output_value = graph->FindOutputs(node->id)[0]; output_value->tensor.shape = graph->FindInputs(node->id)[0]->tensor.shape; BHWC output_shape = output_value->tensor.shape; - RETURN_IF_ERROR( - ParseCustomAttributes(op_name, tflite_node->custom_initial_data, - tflite_node->custom_initial_data_size, - &(node->operation.attributes), &output_shape)); + RETURN_IF_ERROR(ParseCustomAttributes( + op_name, registration->version, tflite_node->custom_initial_data, + tflite_node->custom_initial_data_size, &(node->operation.attributes), + &output_shape)); return absl::OkStatus(); } @@ -2403,10 +2403,10 @@ class Landmarks2TransformMatrixOperationParser : public TFLiteOperationParser { const std::string op_name = "landmarks_to_transform_matrix"; node->operation.type = op_name; BHWC output_shape; - RETURN_IF_ERROR( - ParseCustomAttributes(op_name, tflite_node->custom_initial_data, - tflite_node->custom_initial_data_size, - &(node->operation.attributes), &output_shape)); + RETURN_IF_ERROR(ParseCustomAttributes( + op_name, registration->version, tflite_node->custom_initial_data, + tflite_node->custom_initial_data_size, &(node->operation.attributes), + &output_shape)); auto output_value = graph->FindOutputs(node->id)[0]; output_value->tensor.shape = output_shape; @@ -2434,10 +2434,10 @@ class Landmarks2TransformMatrixV2OperationParser const std::string op_name = "landmarks_to_transform_matrix_v2"; node->operation.type = op_name; BHWC output_shape; - RETURN_IF_ERROR( - ParseCustomAttributes(op_name, tflite_node->custom_initial_data, - tflite_node->custom_initial_data_size, - &(node->operation.attributes), &output_shape)); + RETURN_IF_ERROR(ParseCustomAttributes( + op_name, registration->version, tflite_node->custom_initial_data, + tflite_node->custom_initial_data_size, &(node->operation.attributes), + &output_shape)); auto output_value = graph->FindOutputs(node->id)[0]; output_value->tensor.shape = output_shape; @@ -2465,10 +2465,10 @@ class AlignmentPointsToTransformMatrixOperationParser const std::string op_name = "alignment_points_to_transform_matrix"; node->operation.type = op_name; BHWC output_shape; - RETURN_IF_ERROR( - ParseCustomAttributes(op_name, tflite_node->custom_initial_data, - tflite_node->custom_initial_data_size, - &(node->operation.attributes), &output_shape)); + RETURN_IF_ERROR(ParseCustomAttributes( + op_name, registration->version, tflite_node->custom_initial_data, + tflite_node->custom_initial_data_size, &(node->operation.attributes), + &output_shape)); auto output_value = graph->FindOutputs(node->id)[0]; output_value->tensor.shape = output_shape; From f927469a6a7372aa16180177768c4433ae201e07 Mon Sep 17 00:00:00 2001 From: Austin Anderson Date: Wed, 12 Aug 2020 12:45:06 -0700 Subject: [PATCH 0911/1017] Add sizetracking for libtensorflow on Ubuntu PiperOrigin-RevId: 326290541 Change-Id: Ia64b59bfcf09b27280e950c44e65ac3ad2513056 --- .../ci_build/release/ubuntu_16/libtensorflow/cpu/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/libtensorflow/cpu/build.sh b/tensorflow/tools/ci_build/release/ubuntu_16/libtensorflow/cpu/build.sh index a0e3a7f4594..1504688dcbc 100644 --- a/tensorflow/tools/ci_build/release/ubuntu_16/libtensorflow/cpu/build.sh +++ b/tensorflow/tools/ci_build/release/ubuntu_16/libtensorflow/cpu/build.sh @@ -38,3 +38,9 @@ if [ -n "${IS_NIGHTLY_BUILD}" ]; then cp tensorflow/tools/ci_build/builds/libtensorflow_nightly_symlink.sh lib_package fi +# Upload to go/tf-sizetracker +python3 ./tensorflow/tools/ci_build/sizetrack_helper.py \ + --team tensorflow_libtensorflow \ + --artifact_id ubuntu_cpu_nightly \ + --upload \ + --artifact "$(find lib_package -iname "libtensorflow*.tar.gz" -not -iname "*jni*" | head -n 1)" From d1f0f29504bbae8e6450e39abc727f9d73cf4d4c Mon Sep 17 00:00:00 2001 From: Sachin Joglekar Date: Wed, 12 Aug 2020 12:53:24 -0700 Subject: [PATCH 0912/1017] Use dummy ptr instead of misaligned access in test PiperOrigin-RevId: 326292383 Change-Id: Ic5a81c465c2a216767027929aa99f8cee9ef9ed5 --- tensorflow/lite/BUILD | 1 - tensorflow/lite/interpreter_test.cc | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/lite/BUILD b/tensorflow/lite/BUILD index 5f8923bfd51..7007a847d83 100644 --- a/tensorflow/lite/BUILD +++ b/tensorflow/lite/BUILD @@ -408,7 +408,6 @@ cc_test( ], features = ["-dynamic_link_test_srcs"], # see go/dynamic_link_test_srcs tags = [ - "noasan", # b/163674257 "tflite_not_portable_ios", # TODO(b/117786830) "tflite_smoke_test", ], diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc index bf40843876c..067f2c54124 100644 --- a/tensorflow/lite/interpreter_test.cc +++ b/tensorflow/lite/interpreter_test.cc @@ -1578,8 +1578,9 @@ class TestCustomAllocation : public ::testing::Test { TEST_F(TestCustomAllocation, InvalidAlignment) { const TfLiteTensor* input_tensor = interpreter_->tensor(interpreter_->inputs()[0]); - auto input_alloc = - NewCustomAlloc(input_tensor->bytes, kDefaultTensorAlignment - 1); + intptr_t dummy_ptr = kDefaultTensorAlignment - 1; + TfLiteCustomAllocation input_alloc{reinterpret_cast(dummy_ptr), + input_tensor->bytes}; ASSERT_EQ(interpreter_->SetCustomAllocationForTensor( interpreter_->inputs()[0], input_alloc), kTfLiteError); From f5895bbe1775b3c44f90fd44af9843b902a259da Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 13:02:06 -0700 Subject: [PATCH 0913/1017] Internal change PiperOrigin-RevId: 326294425 Change-Id: Ieb61727e48dee0c6bd55f77ce3f1310c873a29b4 --- tensorflow/python/keras/callbacks.py | 2 - tensorflow/python/keras/saving/hdf5_format.py | 5 --- .../python/keras/saving/hdf5_format_test.py | 39 ------------------- 3 files changed, 46 deletions(-) diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py index 0d795a99d06..5a191263241 100644 --- a/tensorflow/python/keras/callbacks.py +++ b/tensorflow/python/keras/callbacks.py @@ -1358,8 +1358,6 @@ class ModelCheckpoint(Callback): raise IOError('Please specify a non-directory filepath for ' 'ModelCheckpoint. Filepath used is an existing ' 'directory: {}'.format(filepath)) - # Re-throw the error for any other causes. - raise e def _get_file_path(self, epoch, logs): """Returns the file path for checkpoint.""" diff --git a/tensorflow/python/keras/saving/hdf5_format.py b/tensorflow/python/keras/saving/hdf5_format.py index 4bc881ae5e2..31c9a6e14e0 100644 --- a/tensorflow/python/keras/saving/hdf5_format.py +++ b/tensorflow/python/keras/saving/hdf5_format.py @@ -35,7 +35,6 @@ from tensorflow.python.keras.utils.generic_utils import LazyLoader from tensorflow.python.keras.utils.io_utils import ask_to_proceed_with_overwrite from tensorflow.python.ops import variables as variables_module from tensorflow.python.platform import tf_logging as logging -from tensorflow.python.platform import gfile # pylint: disable=g-import-not-at-top try: @@ -100,10 +99,6 @@ def save_model_to_hdf5(model, filepath, overwrite=True, include_optimizer=True): if not proceed: return - # Try creating dir if not exist - dirpath = os.path.dirname(filepath) - gfile.MakeDirs(dirpath) - f = h5py.File(filepath, mode='w') opened_new_file = True else: diff --git a/tensorflow/python/keras/saving/hdf5_format_test.py b/tensorflow/python/keras/saving/hdf5_format_test.py index 92296b58023..dea492db4dc 100644 --- a/tensorflow/python/keras/saving/hdf5_format_test.py +++ b/tensorflow/python/keras/saving/hdf5_format_test.py @@ -730,45 +730,6 @@ class TestWholeModelSaving(keras_parameterized.TestCase): os.close(fd) os.remove(fname) - def test_model_saving_to_new_dir_path(self): - saved_model_dir = os.path.join(self._save_model_dir(), 'newdir', - 'saved_model') - save_format = testing_utils.get_save_format() - - with self.cached_session(): - model = keras.models.Sequential() - model.add(keras.layers.Dense(2, input_shape=(3,))) - model.add(keras.layers.RepeatVector(3)) - model.add(keras.layers.TimeDistributed(keras.layers.Dense(3))) - - x = np.random.random((1, 3)) - out = model.predict(x) - - keras.models.save_model(model, saved_model_dir, save_format=save_format) - - new_model = keras.models.load_model(saved_model_dir) - self._assert_same_weights_and_metrics(model, new_model) - - out2 = new_model.predict(x) - self.assertAllClose(out, out2, atol=1e-05) - - def test_model_raise_exception_with_failed_saving(self): - if h5py is None: - self.skipTest('h5py required to run this test') - - saved_model_dir = self._save_model_dir() - saved_model_path = os.path.join(saved_model_dir, 'saved_model.h5') - - with self.cached_session(): - model = keras.models.Sequential() - model.add(keras.layers.Dense(2, input_shape=(3,))) - model.add(keras.layers.RepeatVector(3)) - model.add(keras.layers.TimeDistributed(keras.layers.Dense(3))) - - with self.assertRaisesRegex(OSError, 'Unable to create file'): - with h5py.File(saved_model_path, 'w'): - keras.models.save_model(model, saved_model_path) - def test_saving_constant_initializer_with_numpy(self): saved_model_dir = self._save_model_dir() save_format = testing_utils.get_save_format() From c4985c2561b19dcec759f6671849c94b32f9367b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 13:10:35 -0700 Subject: [PATCH 0914/1017] add a check for NNAPI delegate to only accept constant size tensor for ResizeNearestNeighbor PiperOrigin-RevId: 326296272 Change-Id: I927d129e932d5e47a8298db159ecb4c4a6df1a10 --- tensorflow/lite/delegates/nnapi/nnapi_delegate.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc index 122ddc043b2..ad0d12763c1 100644 --- a/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc +++ b/tensorflow/lite/delegates/nnapi/nnapi_delegate.cc @@ -1797,6 +1797,15 @@ bool NNAPIDelegateKernel::Validate( ExpectMinAndroidSdkVersion(android_sdk_version, kMinSdkVersionForNNAPI12, &val_ctx); ExpectIsFloatOrQuant8Operator(context, node, &val_ctx); + Expect(node->inputs->size >= 2, + NNAPIValidationFailureType::kUnsupportedOperatorVariant, + "Expected at least 2 inputs", &val_ctx); + if (node->inputs->size >= 2) { + Expect(context->tensors[node->inputs->data[1]].allocation_type == + kTfLiteMmapRo, + NNAPIValidationFailureType::kInputTensorShouldHaveConstantShape, + "The size input tensor must be constant.", &val_ctx); + } auto builtin = reinterpret_cast( node->builtin_data); if (android_sdk_version <= kMinSdkVersionForNNAPI12) { From e3b8eb11454bdf4282808a912304e9e345ce6d98 Mon Sep 17 00:00:00 2001 From: Edward Loper Date: Wed, 12 Aug 2020 13:11:38 -0700 Subject: [PATCH 0915/1017] Update the validation logic for SavedModel signatures to complain if the output of a function stored in signatures is anything other than a Tensor, a sequence of Tensors, or a dictionary mapping strings to Tensors. PiperOrigin-RevId: 326296479 Change-Id: I3127a3b32ce080568b0f2e01f436b0e94d9d5af1 --- tensorflow/python/saved_model/save_test.py | 5 +- .../saved_model/signature_serialization.py | 54 ++++++++----------- 2 files changed, 23 insertions(+), 36 deletions(-) diff --git a/tensorflow/python/saved_model/save_test.py b/tensorflow/python/saved_model/save_test.py index 28b8fa907e0..c59b13144cd 100644 --- a/tensorflow/python/saved_model/save_test.py +++ b/tensorflow/python/saved_model/save_test.py @@ -247,7 +247,7 @@ class SaveTest(test.TestCase, parameterized.TestCase): root.f(constant_op.constant(1.)) to_save = root.f.get_concrete_function(constant_op.constant(1.)) save_dir = os.path.join(self.get_temp_dir(), "saved_model") - with self.assertRaisesRegex(ValueError, "non-flat outputs"): + with self.assertRaisesRegex(ValueError, "non-Tensor value"): save.save(root, save_dir, to_save) def test_nested_dict_outputs(self): @@ -259,8 +259,7 @@ class SaveTest(test.TestCase, parameterized.TestCase): root.f(constant_op.constant(1.)) to_save = root.f.get_concrete_function(constant_op.constant(1.)) save_dir = os.path.join(self.get_temp_dir(), "saved_model") - with self.assertRaisesRegex(ValueError, - "dictionary containing non-Tensor value"): + with self.assertRaisesRegex(ValueError, "non-Tensor value"): save.save(root, save_dir, to_save) def test_variable(self): diff --git a/tensorflow/python/saved_model/signature_serialization.py b/tensorflow/python/saved_model/signature_serialization.py index 55d0d70295e..74f76c690f2 100644 --- a/tensorflow/python/saved_model/signature_serialization.py +++ b/tensorflow/python/saved_model/signature_serialization.py @@ -150,44 +150,32 @@ def canonicalize_signatures(signatures): return concrete_signatures, wrapped_functions -def _is_flat(sequence): - sequence_flat = nest.flatten(sequence) - try: - nest.assert_same_structure(sequence_flat, sequence, check_types=False) - return True - except ValueError: - return False - except TypeError: - return False - - def _normalize_outputs(outputs, function_name, signature_key): """Construct an output dictionary from unnormalized function outputs.""" - if isinstance(outputs, collections_abc.Mapping): - for key, value in outputs.items(): - if not isinstance(value, ops.Tensor): - raise ValueError( - ("Got a dictionary containing non-Tensor value {} for key {} " - "in the output of the function {} used to generate a SavedModel " - "signature. Dictionaries outputs for functions used as signatures " - "should have one Tensor output per string key.") - .format(value, key, compat.as_str_any(function_name))) - return outputs - else: - original_outputs = outputs + # Convert `outputs` to a dictionary (if it's not one already). + if not isinstance(outputs, collections_abc.Mapping): if not isinstance(outputs, collections_abc.Sequence): outputs = [outputs] - if not _is_flat(outputs): + outputs = {("output_{}".format(output_index)): output + for output_index, output + in enumerate(outputs)} + + # Check that the keys of `outputs` are strings and the values are Tensors. + for key, value in outputs.items(): + if not isinstance(key, compat.bytes_or_text_types): raise ValueError( - ("Got non-flat outputs '{}' from '{}' for SavedModel " - "signature '{}'. Signatures have one Tensor per output, so " - "to have predictable names Python functions used to generate " - "these signatures should avoid outputting Tensors in nested " - "structures.") - .format(original_outputs, function_name, signature_key)) - return {("output_{}".format(output_index)): output - for output_index, output - in enumerate(outputs)} + ("Got a dictionary with a non-string key {!r} in the output of the " + "function {} used to generate the SavedModel signature {!r}.") + .format(key, compat.as_str_any(function_name), signature_key)) + if not isinstance(value, ops.Tensor): + raise ValueError( + ("Got a non-Tensor value {!r} for key {!r} in the output of the " + "function {} used to generate the SavedModel signature {!r}. " + "Outputs for functions used as signatures must be a single Tensor, " + "a sequence of Tensors, or a dictionary from string to Tensor.") + .format(value, key, compat.as_str_any(function_name), signature_key)) + + return outputs # _SignatureMap is immutable to ensure that users do not expect changes to be From d6062c061614a6bf495ec7e4c70b689af7341e32 Mon Sep 17 00:00:00 2001 From: Tim Shen Date: Wed, 12 Aug 2020 13:14:12 -0700 Subject: [PATCH 0916/1017] Roll back XLA/GPU LHLO sort emitter again It breaks an internal msan enabled test. PiperOrigin-RevId: 326297018 Change-Id: I89c905a6b1dd23ecc534e26658f98f6d1e816f23 --- tensorflow/compiler/mlir/xla/hlo_utils.cc | 3 + .../non_identity_layouts.hlotxt | 2 +- .../xla/transforms/mhlo_to_lhlo_with_xla.cc | 11 +- .../xla/transforms/mhlo_to_lhlo_with_xla.h | 3 +- tensorflow/compiler/xla/service/gpu/BUILD | 10 + .../compiler/xla/service/gpu/gpu_compiler.cc | 24 +- .../xla/service/gpu/hlo_to_ir_bindings.cc | 20 +- .../xla/service/gpu/hlo_to_ir_bindings.h | 4 + .../xla/service/gpu/ir_emitter_context.h | 7 +- .../xla/service/gpu/ir_emitter_unnested.cc | 416 +++++++++++---- .../xla/service/gpu/ir_emitter_unnested.h | 82 ++- .../compiler/xla/service/gpu/tests/BUILD | 29 + .../xla/service/gpu/tests/sorting.hlo | 504 +++++++++--------- .../xla/service/gpu/tests/sorting_test.cc | 71 +++ .../compiler/xla/service/llvm_ir/llvm_util.cc | 7 +- .../compiler/xla/service/llvm_ir/llvm_util.h | 2 +- 16 files changed, 792 insertions(+), 403 deletions(-) create mode 100644 tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc diff --git a/tensorflow/compiler/mlir/xla/hlo_utils.cc b/tensorflow/compiler/mlir/xla/hlo_utils.cc index cf78c81908d..18b4265d786 100644 --- a/tensorflow/compiler/mlir/xla/hlo_utils.cc +++ b/tensorflow/compiler/mlir/xla/hlo_utils.cc @@ -83,6 +83,9 @@ StatusOr> GetPermutationIfAvailable( strides[dim] = accumulated_stride; accumulated_stride *= shape.dimensions(dim); } + if (accumulated_stride == 0) { + return llvm::SmallVector{}; + } return llvm::SmallVector{ makeStridedLinearLayoutMap(strides, /*offset=*/0, builder.getContext())}; } diff --git a/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt b/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt index 3630d2d45e4..a83e36cff64 100644 --- a/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt +++ b/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt @@ -8,6 +8,6 @@ HloModule TestModule ENTRY TestComputation { x = f32[3, 2]{1,0} parameter(0) - // CHECK: "lmhlo.copy"(%{{.*}}, %{{.*}}) : (memref<3x2xf32>, memref<3x2xf32, #[[MAP]]>) -> () + // CHECK: "lmhlo.copy"(%{{.*}}, %{{.*}}) {name = "copy.1"} : (memref<3x2xf32>, memref<3x2xf32, #[[MAP]]>) -> () ROOT x.copy = f32[3, 2]{0,1} copy(x) } diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc index 832bad2dcc8..6ce91599fb1 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc @@ -34,7 +34,6 @@ limitations under the License. #include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Pass/PassOptions.h" // from @llvm-project #include "mlir/Translation.h" // from @llvm-project -#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" #include "tensorflow/compiler/mlir/xla/hlo_function_importer.h" #include "tensorflow/compiler/mlir/xla/hlo_utils.h" #include "tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.h" @@ -182,7 +181,10 @@ template StatusOr LhloDialectEmitter::CreateOpWithoutAttrs( HloInstruction* instr) { Location loc = getLocation(instr); - ArrayRef> attrs; + std::pair attrs[] = { + {Identifier::get("name", builder_.getContext()), + builder_.getStringAttr(instr->name())}, + }; ArrayRef rets{}; llvm::SmallVector operands; @@ -252,15 +254,14 @@ Status LhloDialectEmitter::DefaultAction(HloInstruction* instr) { return Status::OK(); } -StatusOr LhloDialectEmitter::EmitSortOp( - HloInstruction* instr) { +StatusOr LhloDialectEmitter::EmitSortOp(HloInstruction* instr) { TF_ASSIGN_OR_RETURN(auto sort, CreateOpWithoutAttrs(instr)); auto* sort_instr = ::xla::Cast<::xla::HloSortInstruction>(instr); sort.dimensionAttr(builder_.getI64IntegerAttr(sort_instr->sort_dimension())); sort.is_stableAttr(builder_.getBoolAttr(sort_instr->is_stable())); TF_RETURN_IF_ERROR(::xla::HloFunctionImporter::ImportAsRegion( *sort_instr->called_computations()[0], &sort.comparator(), &builder_)); - return sort.getOperation(); + return sort; } Status LhloDialectEmitter::HandleSort(HloInstruction* instr) { diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h index bdc977616b1..4000fa01970 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h @@ -19,6 +19,7 @@ limitations under the License. #include "mlir/IR/Builders.h" // from @llvm-project #include "mlir/IR/Module.h" // from @llvm-project #include "mlir/IR/StandardTypes.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/hlo_module.h" @@ -41,7 +42,7 @@ class LhloDialectEmitter : public ::xla::DfsHloVisitorWithDefault { builder_(module.getContext()), i8_type_(builder_.getIntegerType(8)) {} - ::xla::StatusOr EmitSortOp(::xla::HloInstruction* instr); + ::xla::StatusOr EmitSortOp(::xla::HloInstruction* instr); private: template diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index 074fbd92b27..a19f9965fc7 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -254,6 +254,11 @@ cc_library( ":target_util", ":thunk", ":thunk_emitter", + "//tensorflow/compiler/mlir/hlo:lhlo", + "//tensorflow/compiler/mlir/xla:hlo_utils", + "//tensorflow/compiler/mlir/xla:mhlo_to_lhlo_with_xla", + "//tensorflow/compiler/mlir/xla:mlir_hlo_to_hlo", + "//tensorflow/compiler/mlir/xla:type_to_shape", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", @@ -291,6 +296,8 @@ cc_library( "@com_google_absl//absl/types:span", "@llvm-project//llvm:Core", "@llvm-project//llvm:Support", + "@llvm-project//mlir:IR", + "@llvm-project//mlir:StandardOps", ], ) @@ -1159,6 +1166,7 @@ cc_library( ":target_constants", ":tree_reduction_rewriter", ":variadic_op_splitter", + "//tensorflow/compiler/mlir/xla:mhlo_to_lhlo_with_xla", "//tensorflow/compiler/xla:protobuf_util", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", @@ -1217,6 +1225,8 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@llvm-project//llvm:Core", + "@llvm-project//mlir:AllPassesAndDialectsNoRegistration", + "@llvm-project//mlir:IR", ], ) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index f5bf7476059..b796737e601 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -29,6 +29,8 @@ limitations under the License. #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" +#include "mlir/IR/Module.h" // from @llvm-project +#include "mlir/InitAllDialects.h" // from @llvm-project #include "tensorflow/compiler/xla/protobuf_util.h" #include "tensorflow/compiler/xla/service/algebraic_simplifier.h" #include "tensorflow/compiler/xla/service/all_reduce_combiner.h" @@ -516,15 +518,22 @@ static Status CompileModuleToLlvmIrImpl( DumpHloModuleIfEnabled(*hlo_module, **buffer_assignment, "after_optimizations"); + mlir::registerAllDialects(); + mlir::MLIRContext mlir_context; + IrEmitterContext ir_emitter_context( hlo_module, buffer_assignment->get(), platform_name, gpu_device_info, - cuda_compute_capability, profile_index_map, llvm_module->get()); + cuda_compute_capability, profile_index_map, &mlir_context, + llvm_module->get()); HloComputation* entry_computation = hlo_module->entry_computation(); - IrEmitterUnnested ir_emitter(hlo_module->config(), entry_computation, - &ir_emitter_context); - TF_RETURN_IF_ERROR(ir_emitter.EmitConstantGlobals()); + TF_ASSIGN_OR_RETURN( + auto ir_emitter, + IrEmitterUnnested::Create(hlo_module->config(), entry_computation, + &ir_emitter_context)); + + TF_RETURN_IF_ERROR(ir_emitter->EmitConstantGlobals()); { XLA_SCOPED_LOGGING_TIMER("GpuCompiler::RunBackend - IR emission"); @@ -533,9 +542,10 @@ static Status CompileModuleToLlvmIrImpl( ThunkSequence thunk_sequence; absl::Span order = hlo_schedule->ThunkLaunchOrder(); for (HloInstruction* instruction : order) { - TF_RETURN_IF_ERROR(instruction->Visit(&ir_emitter)); - TF_RETURN_IF_ERROR(ir_emitter.Postprocess(instruction)); - std::unique_ptr thunks = ir_emitter.ConsumeThunkSequence(); + TF_RETURN_IF_ERROR(instruction->Visit(ir_emitter.get())); + TF_RETURN_IF_ERROR(ir_emitter->Postprocess(instruction)); + std::unique_ptr thunks = + ir_emitter->ConsumeThunkSequence(); // The invariants between each input HloInstruction* and output Thunk* are // not all explicitly checked, but at least we can document them here: diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc index 5d38d1b727c..332db83b6ad 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc @@ -117,11 +117,11 @@ static bool HasMeaningfulName(llvm::Value* value) { return false; } -llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, - ShapeIndexView shape_index, - llvm::Value* ir_value) { - llvm::Type* pointee_type = llvm_ir::ShapeToIrType( - ShapeUtil::GetSubshape(hlo.shape(), shape_index), module_); +llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, + llvm::IRBuilder<>* b) { + llvm::Type* pointee_type = + llvm_ir::ShapeToIrType(shape, b->GetInsertBlock()->getModule()); + llvm::Type* dest_type = pointee_type->getPointerTo(); llvm::Value* typed_ir_value; @@ -129,9 +129,17 @@ llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, typed_ir_value = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast( llvm::cast(ir_value), dest_type); } else { - typed_ir_value = b_->CreatePointerBitCastOrAddrSpaceCast( + typed_ir_value = b->CreatePointerBitCastOrAddrSpaceCast( ir_value, pointee_type->getPointerTo()); } + return typed_ir_value; +} + +llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, + ShapeIndexView shape_index, + llvm::Value* ir_value) { + auto typed_ir_value = CastToTypedValue( + ShapeUtil::GetSubshape(hlo.shape(), shape_index), ir_value, b_); if (!HasMeaningfulName(ir_value)) { ir_value->setName(llvm_ir::IrName(&hlo, "raw")); } diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h index 5eef6727801..3813ec6c949 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h @@ -116,6 +116,10 @@ class HloToIrBindings { llvm::Value* temp_buffer_base_ = nullptr; }; +// Converts `ir_value` with type i8* to a typed LLVM Value* based on `shape`. +llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, + llvm::IRBuilder<>* b); + } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h index 9c43f80dc60..7d5a8d032e6 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_CONTEXT_H_ #include "llvm/IR/Module.h" +#include "mlir/IR/MLIRContext.h" // from @llvm-project #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/gpu/launch_dimensions.h" #include "tensorflow/compiler/xla/service/hlo_execution_profile.h" @@ -34,13 +35,15 @@ class IrEmitterContext { const HloModule* hlo_module, const BufferAssignment* buffer_assignment, std::string platform_name, GpuDeviceInfo gpu_device_info, absl::optional cuda_compute_capability, - const HloProfileIndexMap* profile_index_map, llvm::Module* llvm_module) + const HloProfileIndexMap* profile_index_map, + mlir::MLIRContext* mlir_context, llvm::Module* llvm_module) : hlo_module_(hlo_module), buffer_assignment_(buffer_assignment), platform_name_(std::move(platform_name)), gpu_device_info_(gpu_device_info), cuda_compute_capability_(cuda_compute_capability), profile_index_map_(profile_index_map), + mlir_context_(mlir_context), llvm_module_(llvm_module) {} // Disallow copy and assign. IrEmitterContext(const IrEmitterContext&) = delete; @@ -57,6 +60,7 @@ class IrEmitterContext { return cuda_compute_capability_; } const HloProfileIndexMap* profile_index_map() { return profile_index_map_; } + mlir::MLIRContext* mlir_context() { return mlir_context_; } llvm::Module* llvm_module() { return llvm_module_; } NameUniquer* name_uniquer() { return &name_uniquer_; } @@ -67,6 +71,7 @@ class IrEmitterContext { GpuDeviceInfo gpu_device_info_; absl::optional cuda_compute_capability_; const HloProfileIndexMap* profile_index_map_; + mlir::MLIRContext* mlir_context_; llvm::Module* llvm_module_; NameUniquer name_uniquer_; }; diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index 61b78b6004d..f88c70b1a33 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -37,6 +37,13 @@ limitations under the License. #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" +#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project +#include "mlir/IR/Builders.h" // from @llvm-project +#include "mlir/IR/Function.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" +#include "tensorflow/compiler/mlir/xla/hlo_utils.h" +#include "tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.h" +#include "tensorflow/compiler/mlir/xla/type_to_shape.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" @@ -144,13 +151,86 @@ void UpdateLaunchDimensions(const LaunchDimensions& launch_dims, Thunk* thunk, llvm::ConstantAsMetadata::get(threads_per_block_ir_value)})); } +const BufferAllocation* GetAllocation( + mlir::BlockArgument func_arg, const BufferAssignment& buffer_assignment) { + auto func_op = + mlir::cast(func_arg.getParentRegion()->getParentOp()); + int64 allocation_index = func_op + .getArgAttrOfType( + func_arg.getArgNumber(), "lmhlo.alloc") + .getValue() + .getSExtValue(); + return &buffer_assignment.GetAllocation(allocation_index); +} + +StatusOr GetAllocationSliceForMlir( + mlir::Value v, const BufferAssignment& buffer_assignment) { + int64 size = v.getType().cast().getSizeInBits() / 8; + + if (auto arg = v.dyn_cast()) { + return BufferAllocation::Slice(GetAllocation(arg, buffer_assignment), 0, + size); + } + + // We match two patterns here: + // * v = ViewOp(arg); + // * v = StaticMemRefCastOp(ViewOp(arg)); + if (mlir::Operation* op = v.getDefiningOp()) { + if (auto cast = mlir::dyn_cast(op)) { + mlir::Value source = cast.getViewSource(); + op = source.getDefiningOp(); + if (!op) { + return Unimplemented("StaticMemRefCastOp has to wrap an op"); + } + } + if (auto view = mlir::dyn_cast(op)) { + return BufferAllocation::Slice( + GetAllocation(view.source().cast(), + buffer_assignment), + mlir::cast(view.byte_shift().getDefiningOp()) + .value() + .cast() + .getValue() + .getSExtValue(), + size); + } + return Unimplemented("StaticMemRefCastOp has to wrap a ViewOp"); + } + + return Unimplemented( + "Operand has to be in the form of ViewOp(arg) or " + "StaticMemRefCastOp(ViewOp(arg))"); +} + +absl::string_view GetHloName(mlir::Operation* op) { + if (auto attr = op->getAttrOfType("name")) { + auto ref = attr.getValue(); + return absl::string_view(ref.data(), ref.size()); + } + return ""; +} + } // namespace IrEmitterUnnested::IrEmitterUnnested(const HloModuleConfig& hlo_module_config, const HloComputation* hlo_computation, IrEmitterContext* ir_emitter_context) : IrEmitter(hlo_module_config, ir_emitter_context, /*is_nested=*/false), - hlo_computation_(hlo_computation) {} + hlo_computation_(hlo_computation), + mlir_scratch_module_(mlir::ModuleOp::create( + mlir::Builder(ir_emitter_context->mlir_context()).getUnknownLoc())), + lhlo_scratch_emitter_(ir_emitter_context_->buffer_assignment(), + *hlo_computation, mlir_scratch_module_.get()) {} + +StatusOr> IrEmitterUnnested::Create( + const HloModuleConfig& hlo_module_config, + const HloComputation* hlo_computation, + IrEmitterContext* ir_emitter_context) { + auto emitter = std::unique_ptr(new IrEmitterUnnested( + hlo_module_config, hlo_computation, ir_emitter_context)); + TF_RETURN_IF_ERROR(emitter->lhlo_scratch_emitter_.Initialize()); + return std::move(emitter); +} Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { bindings_.UnbindAllLocalIrValues(); @@ -158,12 +238,11 @@ Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { } llvm::Function* IrEmitterUnnested::BuildKernelPrototype( - const HloInstruction& inst, - absl::Span args) { + absl::string_view name, absl::Span args) { // Compute the kernel name. The opcode string may contain "-" which cannot be // in a PTX function name, so sanitize the name before uniquifying it. string kernel_name = ir_emitter_context_->name_uniquer()->GetUniqueName( - llvm_ir::SanitizeFunctionName(inst.name())); + llvm_ir::SanitizeFunctionName(std::string(name))); // Create the kernel and add it to the module. llvm::Module* module = ir_emitter_context_->llvm_module(); @@ -359,7 +438,8 @@ Status IrEmitterUnnested::HandleDot(HloInstruction* dot) { } Status IrEmitterUnnested::HandleConditional(HloInstruction* conditional) { - AddThunkToThunkSequence(BuildConditionalThunk(conditional)); + TF_ASSIGN_OR_RETURN(auto thunk, BuildConditionalThunk(conditional)); + AddThunkToThunkSequence(std::move(thunk)); return Status::OK(); } @@ -1038,10 +1118,13 @@ Status IrEmitterUnnested::HandleWhile(HloInstruction* xla_while) { // Build ForThunk for conformant while loops, otherwise build WhileThunk. auto config = xla_while->backend_config(); if (config.ok() && config.ValueOrDie().has_known_trip_count()) { - AddThunkToThunkSequence( + TF_ASSIGN_OR_RETURN( + auto thunk, BuildForThunk(xla_while, config.ValueOrDie().known_trip_count().n())); + AddThunkToThunkSequence(std::move(thunk)); } else { - AddThunkToThunkSequence(BuildWhileThunk(xla_while)); + TF_ASSIGN_OR_RETURN(auto thunk, BuildWhileThunk(xla_while)); + AddThunkToThunkSequence(std::move(thunk)); } return Status::OK(); } @@ -1264,39 +1347,109 @@ Status IrEmitterUnnested::HandleSelect(HloInstruction* select) { return IrEmitter::HandleSelect(select); } +StatusOr +IrEmitterUnnested::GetOrCreateSubComputationFromRegion(mlir::Region* region) { + std::unique_ptr& module = scratch_nested_computations_[region]; + if (module == nullptr) { + xla::XlaComputation xla_computation; + TF_RETURN_IF_ERROR(ConvertRegionToComputation(region, &xla_computation)); + TF_ASSIGN_OR_RETURN(auto program_shape, xla_computation.GetProgramShape()); + TF_ASSIGN_OR_RETURN( + module, HloModule::CreateFromProto(xla_computation.proto(), + HloModuleConfig(program_shape))); + } + return module->entry_computation(); +} + Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { + MlirEmitterInput result; + + TF_ASSIGN_OR_RETURN(auto sort_op, lhlo_scratch_emitter_.EmitSortOp(sort)); + result.op = sort_op; + result.name = GetHloName(sort_op); + // The name in sort op has no semantics, and it's for debug only. If the name + // doesn't exist, we should use a namer (e.g. count-based). + // TODO(timshen): use a namer instead of relying on the HloInstruction names. + if (result.name.empty()) { + result.name = sort->name(); + } + const auto& buffer_assignment = ir_emitter_context_->buffer_assignment(); + auto& slice = result.extra_slice; + TF_ASSIGN_OR_RETURN(slice.buffer_slice, + buffer_assignment.GetUniqueSlice(sort, {})); + slice.written = true; + slice.shape = sort->shape(); + + result.thunk_info = GetThunkInfo(sort); + + return EmitMlirSort(result); +} + +Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { + const auto& buffer_assignment = ir_emitter_context_->buffer_assignment(); + auto sort_op = mlir::cast(input.op); + + int operand_count = sort_op.operands().size(); + std::vector operand_shapes(operand_count); + std::vector slices; + std::vector output_shapes(sort_op.output().size()); + + for (int i = 0; i < operand_count; i++) { + operand_shapes[i] = + TypeToShape(sort_op.operands()[i].getType().cast()); + } + + // Craft n + 1 slices, where the first n are output parameters, and the last + // is the on-device tuple storage. We don't need n operands because sorting + // kernels are always in-place. + for (int i = 0; i < operand_count; i++) { + output_shapes[i] = + TypeToShape(sort_op.output()[i].getType().cast()); + MlirBufferSlice slice; + TF_ASSIGN_OR_RETURN( + slice.buffer_slice, + GetAllocationSliceForMlir(sort_op.output()[i], buffer_assignment)); + slice.written = true; + slice.shape = operand_shapes[i]; + slices.push_back(slice); + } + slices.push_back(input.extra_slice); + std::vector> thunks; - Shape keys_shape = sort->operand(0)->shape(); - int64 dimension_to_sort = sort->dimensions(0); - for (int64 i = 0; i < sort->operand_count(); ++i) { - ShapeIndex shape_index = - sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({}); + + Shape keys_shape = operand_shapes[0]; + int64 dimension_to_sort = sort_op.dimension().getSExtValue(); + for (int64 i = 0; i < operand_count; ++i) { // We assume that the layout of all involved operands and outputs is the // same. - TF_RET_CHECK(LayoutUtil::LayoutsInShapesEqual(keys_shape, - sort->operand(i)->shape())); - TF_RET_CHECK(LayoutUtil::LayoutsInShapesEqual( - keys_shape, ShapeUtil::GetSubshape(sort->shape(), shape_index))); + TF_RET_CHECK( + LayoutUtil::LayoutsInShapesEqual(keys_shape, operand_shapes[i])); + TF_RET_CHECK( + LayoutUtil::LayoutsInShapesEqual(keys_shape, output_shapes[i])); // If possible, we share buffers. If that is not possible, we need to copy // the values, because the emitter does the sorting in-place. - auto destination_buffer = GetAllocationSlice(*sort, shape_index); - auto source_address = GetAllocationSlice(*sort->operand(i)); + TF_ASSIGN_OR_RETURN( + auto destination_buffer, + GetAllocationSliceForMlir(sort_op.output()[i], buffer_assignment)); + TF_ASSIGN_OR_RETURN( + auto source_address, + GetAllocationSliceForMlir(sort_op.operands()[i], buffer_assignment)); if (destination_buffer != source_address) { // TODO(b/26783907): Figure out why we never seem to share buffers for // key/value sort. - VLOG(2) << sort->name() << " requires initial D2D copy for operand " << i; + VLOG(2) << input.name << " requires initial D2D copy for operand " << i; thunks.push_back(absl::make_unique( Thunk::ThunkInfo(), /*source_address=*/source_address, /*destination_buffer=*/destination_buffer, - /*mem_size=*/ShapeUtil::ByteSizeOf(sort->operand(i)->shape()))); + /*mem_size=*/ShapeUtil::ByteSizeOf(operand_shapes[i]))); } } uint64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort); int64 num_stages = tensorflow::Log2Ceiling(dimension_to_sort_bound); - VLOG(2) << sort->name() << " requires " << num_stages << " stages."; + VLOG(2) << input.name << " requires " << num_stages << " stages."; CHECK_GE(1ULL << num_stages, dimension_to_sort_bound); CHECK_LT(1ULL << (num_stages - 1), dimension_to_sort_bound); @@ -1360,10 +1513,10 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { // we have not enough threads, or not enough shared memory. Also it does not // give a speedup if the tile size is < 128. int64 total_shared_memory_needed = 0; - for (int64 i = 0; i < sort->operand_count(); ++i) { + for (int64 i = 0; i < operand_count; ++i) { total_shared_memory_needed += - kTileSize * ShapeUtil::ByteSizeOfPrimitiveType( - sort->operand(i)->shape().element_type()); + kTileSize * + ShapeUtil::ByteSizeOfPrimitiveType(operand_shapes[i].element_type()); } bool no_tiling = kTileSize < 128 || @@ -1376,7 +1529,7 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { "kTileSize=%d < 128, " "kThreadsPerBlock=%d > threads_per_block_limit=%d, " "total_shared_memory_needed=%d > shared_memory_per_block=%d", - sort->name(), (no_tiling ? "won't" : "will"), kTileSize, kThreadsPerBlock, + input.name, (no_tiling ? "won't" : "will"), kTileSize, kThreadsPerBlock, ir_emitter_context_->gpu_device_info().threads_per_block_limit, total_shared_memory_needed, ir_emitter_context_->gpu_device_info().shared_memory_per_block); @@ -1384,37 +1537,38 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { uint64 num_blocks = CeilOfRatio(num_iterations, kThreadsPerBlock); LaunchDimensions tiled_launch_dimensions(num_blocks, kThreadsPerBlock); VLOG(2) << absl::StreamFormat("%s launch dims: %d blocks, %d threads/block", - sort->name(), num_blocks, kThreadsPerBlock); + input.name, num_blocks, kThreadsPerBlock); + std::vector ir_arrays; auto emit_kernel = [&](absl::Span xor_masks) { VLOG(2) << absl::StreamFormat( - "%s uses kernel for xor masks [%s]", sort->name(), + "%s uses kernel for xor masks [%s]", input.name, absl::StrJoin(xor_masks, ", ", [](std::string* out, int64 xor_mask) { absl::StrAppendFormat(out, "0x%x", xor_mask); })); - thunks.push_back( - BuildKernelThunk(sort, /*implements_whole_instruction=*/false)); + thunks.push_back(BuildKernelThunkForMlir(input.name, Thunk::ThunkInfo(), + slices, &ir_arrays)); LaunchDimensions launch_dimensions = xor_masks.size() > 1 ? tiled_launch_dimensions : standard_launch_dimensions; UpdateLaunchDimensions(launch_dimensions, thunks.back().get(), ir_emitter_context_->llvm_module()); std::vector values_arrays; - values_arrays.reserve(sort->operand_count()); - for (int64 i = 0; i < sort->operand_count(); ++i) { - ShapeIndex shape_index = - sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({}); - values_arrays.push_back(GetIrArray(*sort, *sort, shape_index)); + values_arrays.reserve(operand_count); + for (int64 i = 0; i < operand_count; ++i) { + values_arrays.push_back(ir_arrays[i]); } + TF_ASSIGN_OR_RETURN( + const HloComputation* comparator, + GetOrCreateSubComputationFromRegion(&sort_op.comparator())); return llvm_ir::EmitSortInPlace( - dimension_to_sort, values_arrays, IrName(sort), xor_masks, &b_, + dimension_to_sort, values_arrays, IrName(input.name), xor_masks, &b_, launch_dimensions, xor_masks.size() > 1 ? num_iterations_in_sort_dim : standard_num_iterations_in_sort_dim, kTileSize, [&](absl::Span operands, llvm::Value* output) { - return EmitCallToNestedComputation(*sort->to_apply(), operands, - output); + return EmitCallToNestedComputation(*comparator, operands, output); }); }; std::vector xor_masks; @@ -1441,17 +1595,18 @@ Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { TF_RETURN_IF_ERROR(emit_kernel(xor_masks)); } VLOG(2) << absl::StreamFormat( - "%s requires %d thunks (including any D2D copies)", sort->name(), + "%s requires %d thunks (including any D2D copies)", input.name, thunks.size()); - AddThunkToThunkSequence(absl::make_unique( - GetThunkInfo(sort), std::move(thunks))); - if (sort->operand_count() > 1) { + AddThunkToThunkSequence( + absl::make_unique(input.thunk_info, std::move(thunks))); + if (operand_count > 1) { // Emit the tuple as part of the last stage of sorting. // We are currently in the block sorted.in_bounds.after. b_.SetInsertPoint(b_.GetInsertBlock()->getTerminator()); - llvm_ir::EmitTuple(GetIrArray(*sort, *sort), - ConstructIrArrayForOutputs(*sort), &b_); + llvm_ir::EmitTuple( + ir_arrays[operand_count], + absl::MakeSpan(ir_arrays).subspan(0, ir_arrays.size() - 1), &b_); } return Status::OK(); } @@ -1589,24 +1744,6 @@ Status IrEmitterUnnested::HandleAfterAll(HloInstruction* after_all) { return Status::OK(); } -// Describes how to access a particular subshape for an HLO. For instance if -// `.hlo_index` is {1} and `.gte_index` is {3, 4} then buffer for `.instr` at -// ShapeIndex {1} (i.e. the buffer for the second tuple element of hlo) is found -// at `.buffer_slice`[3][4]. That is, `.slice` is a void***, which we -// dereference twice -- first at index 3, and then at index 4 -- to get the -// address of our buffer. -struct HloBufferSlice { - const HloInstruction* instr; - ShapeIndex hlo_index; - - // The root buffer to look at. - BufferAllocation::Slice buffer_slice; - - // Describes how to dereference starting at that buffer to get to the buffer - // in question. - ShapeIndex gte_index; -}; - // Figures out how to access the buffers for all subshapes of hlo's operands and // for hlo itself (i.e. all the buffers produced by HLO). // @@ -1715,22 +1852,22 @@ static std::vector GetHloBufferSlices( return result; } -std::unique_ptr IrEmitterUnnested::BuildKernelThunk( - const HloInstruction* inst, bool implements_whole_instruction) { - const BufferAssignment& buffer_assn = - ir_emitter_context_->buffer_assignment(); - - std::vector hlo_slices = - GetHloBufferSlices(inst, buffer_assn); +std::unique_ptr +IrEmitterUnnested::BuildKernelThunkFromBufferSlices( + absl::string_view name, Thunk::ThunkInfo thunk_info, + absl::Span slices, + std::function + bind_slice_to_ir_value) { + const auto& buffer_assn = ir_emitter_context_->buffer_assignment(); // Figure out which buffer allocations need to be passed as arguments to our - // kernel. This is simply all of the allocations referenced in hlo_slices, + // kernel. This is simply all of the allocations referenced in slices, // plus the XLA temp buffer (if we have it). We always include the temp // buffer because even if the kernel itself doesn't use it, a nested // subcomputation within the kernel (e.g. a kMap's computation) might. std::unordered_set buffers_needed; - for (const auto& hlo_buffer_slice : hlo_slices) { - buffers_needed.insert(hlo_buffer_slice.buffer_slice.allocation()); + for (auto* slice : slices) { + buffers_needed.insert(slice->buffer_slice.allocation()); } absl::optional temp_buffer; for (const BufferAllocation& alloc : buffer_assn.Allocations()) { @@ -1759,7 +1896,7 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( return a->index() < b->index(); }); - llvm::Function* kernel = BuildKernelPrototype(*inst, non_constant_buffers); + llvm::Function* kernel = BuildKernelPrototype(name, non_constant_buffers); // Build a map from a BufferAllocation to the corresponding argument in our // kernel. @@ -1793,24 +1930,19 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( // For each buffer our kernel might want to touch, bind it to a value derived // from our kernel args. - for (const auto& hlo_buffer_slice : hlo_slices) { - const HloInstruction* instr = hlo_buffer_slice.instr; - const ShapeIndex& index = hlo_buffer_slice.hlo_index; - const BufferAllocation::Slice& slice = hlo_buffer_slice.buffer_slice; - const ShapeIndex& gte_index = hlo_buffer_slice.gte_index; - - VLOG(3) << "Buffer for " << instr->ToString() << " at " << index.ToString() - << " is found in slice " << slice.ToString() << " at GTE index " - << gte_index.ToString(); + for (auto* slice : slices) { + const BufferAllocation::Slice& buffer_slice = slice->buffer_slice; + const ShapeIndex& gte_index = slice->gte_index; llvm::Value* loc; - if (slice.allocation()->is_constant()) { + if (buffer_slice.allocation()->is_constant()) { loc = ir_emitter_context_->llvm_module()->getGlobalVariable( - llvm_ir::ConstantBufferAllocationToGlobalName(*slice.allocation())); + llvm_ir::ConstantBufferAllocationToGlobalName( + *buffer_slice.allocation())); CHECK_NE(loc, nullptr); } else { - loc = InBoundsGEP(kernel_args.at(slice.allocation()), - {b_.getInt64(slice.offset())}); + loc = InBoundsGEP(kernel_args.at(buffer_slice.allocation()), + {b_.getInt64(buffer_slice.offset())}); } // If gte_index is nonempty, we have to dereference `loc` to get to the @@ -1822,7 +1954,7 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( loc = Load(InBoundsGEP(loc, {b_.getInt64(idx)})); } - bindings_.BindHloToIrValue(*instr, loc, index); + bind_slice_to_ir_value(slice, loc); } // Bind the temp buffer so that nested subcomputations can find it if they @@ -1834,9 +1966,66 @@ std::unique_ptr IrEmitterUnnested::BuildKernelThunk( llvm::ConstantPointerNull::get(b_.getInt8PtrTy())); } - return absl::make_unique( + return absl::make_unique(thunk_info, non_constant_buffers, + std::string(kernel->getName())); +} + +std::unique_ptr IrEmitterUnnested::BuildKernelThunk( + const HloInstruction* inst, bool implements_whole_instruction) { + std::vector hlo_slices = + GetHloBufferSlices(inst, ir_emitter_context_->buffer_assignment()); + + std::vector slice_ptrs; + slice_ptrs.reserve(hlo_slices.size()); + for (auto& slice : hlo_slices) { + slice_ptrs.push_back(&slice); + } + + return BuildKernelThunkFromBufferSlices( + inst->name(), implements_whole_instruction ? GetThunkInfo(inst) : Thunk::ThunkInfo(), - non_constant_buffers, std::string(kernel->getName())); + slice_ptrs, [this](const BufferSlice* slice, llvm::Value* value) { + const HloBufferSlice* hlo_buffer_slice = + static_cast(slice); + const HloInstruction* instr = hlo_buffer_slice->instr; + const ShapeIndex& index = hlo_buffer_slice->hlo_index; + VLOG(3) << "Buffer for " << instr->ToString() << " at " + << index.ToString() << " is found in slice " + << hlo_buffer_slice->buffer_slice.ToString() << " at GTE index " + << hlo_buffer_slice->gte_index.ToString(); + + bindings_.BindHloToIrValue(*instr, value, index); + }); +} + +std::unique_ptr IrEmitterUnnested::BuildKernelThunkForMlir( + absl::string_view name, Thunk::ThunkInfo thunk_info, + absl::Span slices, + std::vector* ir_arrays) { + absl::flat_hash_set buffers_written; + std::vector slice_ptrs; + slice_ptrs.reserve(slices.size()); + for (auto& slice : slices) { + slice_ptrs.push_back(&slice); + if (slice.written) { + buffers_written.insert(slice.buffer_slice); + } + } + + ir_arrays->clear(); + return BuildKernelThunkFromBufferSlices( + name, thunk_info, slice_ptrs, + [&](const BufferSlice* slice, llvm::Value* value) { + const auto& mlir_slice = static_cast(*slice); + + llvm_ir::IrArray ir_array( + CastToTypedValue(mlir_slice.shape, value, &b_), mlir_slice.shape); + if (!buffers_written.contains(slice->buffer_slice)) { + ir_array.MarkInvariantOverWholeProgram(&value->getContext()); + } + + ir_arrays->push_back(ir_array); + }); } StatusOr> IrEmitterUnnested::BuildInitializerThunk( @@ -2043,7 +2232,7 @@ Status CheckConditionalBuffersShareAllocation( } // namespace -std::unique_ptr IrEmitterUnnested::BuildWhileThunk( +StatusOr> IrEmitterUnnested::BuildWhileThunk( const HloInstruction* hlo) { // Check that all while-related buffers share an allocation. TF_CHECK_OK(CheckWhileBuffersShareAllocation( @@ -2051,24 +2240,26 @@ std::unique_ptr IrEmitterUnnested::BuildWhileThunk( // Generate thunk sequence for while 'condition'. HloComputation* condition = hlo->while_condition(); - IrEmitterUnnested ir_emitter_condition(hlo_module_config_, condition, - ir_emitter_context_); - TF_CHECK_OK(condition->Accept(&ir_emitter_condition)); + TF_ASSIGN_OR_RETURN(auto ir_emitter_condition, + IrEmitterUnnested::Create(hlo_module_config_, condition, + ir_emitter_context_)); + TF_RETURN_IF_ERROR(condition->Accept(ir_emitter_condition.get())); // Generate thunk sequence for while 'body'. HloComputation* body = hlo->while_body(); - IrEmitterUnnested ir_emitter_body(hlo_module_config_, body, - ir_emitter_context_); - TF_CHECK_OK(body->Accept(&ir_emitter_body)); + TF_ASSIGN_OR_RETURN( + auto ir_emitter_body, + IrEmitterUnnested::Create(hlo_module_config_, body, ir_emitter_context_)); + TF_RETURN_IF_ERROR(body->Accept(ir_emitter_body.get())); - return absl::make_unique( + return std::unique_ptr(new WhileThunk( GetThunkInfo(hlo), GetAllocationSlice(*condition->root_instruction()), // cond result - ir_emitter_condition.ConsumeThunkSequence(), - ir_emitter_body.ConsumeThunkSequence()); + ir_emitter_condition->ConsumeThunkSequence(), + ir_emitter_body->ConsumeThunkSequence())); } -std::unique_ptr IrEmitterUnnested::BuildForThunk( +StatusOr> IrEmitterUnnested::BuildForThunk( const HloInstruction* hlo, const int64 loop_limit) { // Check that all while-related buffers share an allocation. TF_CHECK_OK(CheckWhileBuffersShareAllocation( @@ -2076,15 +2267,16 @@ std::unique_ptr IrEmitterUnnested::BuildForThunk( // Generate thunk sequence for while 'body' (will be used a For loop body). HloComputation* body = hlo->while_body(); - IrEmitterUnnested ir_emitter_body(hlo_module_config_, body, - ir_emitter_context_); - TF_CHECK_OK(body->Accept(&ir_emitter_body)); + TF_ASSIGN_OR_RETURN( + auto ir_emitter_body, + IrEmitterUnnested::Create(hlo_module_config_, body, ir_emitter_context_)); + TF_RETURN_IF_ERROR(body->Accept(ir_emitter_body.get())); - return absl::make_unique(GetThunkInfo(hlo), loop_limit, - ir_emitter_body.ConsumeThunkSequence()); + return std::unique_ptr(new ForThunk( + GetThunkInfo(hlo), loop_limit, ir_emitter_body->ConsumeThunkSequence())); } -std::unique_ptr IrEmitterUnnested::BuildConditionalThunk( +StatusOr> IrEmitterUnnested::BuildConditionalThunk( const HloInstruction* hlo) { // Check that the buffers used in conditional are shared with the operands and // result appropriately. @@ -2096,15 +2288,17 @@ std::unique_ptr IrEmitterUnnested::BuildConditionalThunk( for (int j = 0; j < hlo->branch_count(); ++j) { branch_operands.emplace_back(GetAllocationSlice(*hlo->operand(j + 1))); HloComputation* branch_computation = hlo->branch_computation(j); - IrEmitterUnnested ir_emitter(hlo_module_config_, branch_computation, - ir_emitter_context_); - TF_CHECK_OK(branch_computation->Accept(&ir_emitter)); - branch_thunks.push_back(std::move(*ir_emitter.ConsumeThunkSequence())); + TF_ASSIGN_OR_RETURN( + auto ir_emitter, + IrEmitterUnnested::Create(hlo_module_config_, branch_computation, + ir_emitter_context_)); + TF_CHECK_OK(branch_computation->Accept(ir_emitter.get())); + branch_thunks.push_back(std::move(*ir_emitter->ConsumeThunkSequence())); } - return absl::make_unique( + return std::unique_ptr(new ConditionalThunk( GetThunkInfo(hlo), GetAllocationSlice(*hlo->operand(0)), branch_operands, - std::move(branch_thunks)); + std::move(branch_thunks))); } Status IrEmitterUnnested::EmitTargetElementLoopInThunk( diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h index 019fcdf21db..b9146dd8fae 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_ #include "absl/container/inlined_vector.h" +#include "tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h" #include "tensorflow/compiler/xla/service/gpu/ir_emitter.h" #include "tensorflow/compiler/xla/service/gpu/kernel_mapping_scheme.h" #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h" @@ -28,6 +29,40 @@ limitations under the License. namespace xla { namespace gpu { +struct BufferSlice { + // The root buffer to look at. + BufferAllocation::Slice buffer_slice; + + // Describes how to dereference starting at that buffer to get to the buffer + // in question. + ShapeIndex gte_index; +}; + +// Describes how to access a particular subshape for an HLO. For instance if +// `.hlo_index` is {1} and `.gte_index` is {3, 4} then buffer for `.instr` at +// ShapeIndex {1} (i.e. the buffer for the second tuple element of hlo) is +// found at `.buffer_slice`[3][4]. That is, `.slice` is a void***, which we +// dereference twice -- first at index 3, and then at index 4 -- to get the +// address of our buffer. +struct HloBufferSlice : public BufferSlice { + const HloInstruction* instr; + ShapeIndex hlo_index; +}; + +struct MlirBufferSlice : public BufferSlice { + // The buffer is modified by the kernel. + bool written; + + Shape shape; +}; + +struct MlirEmitterInput { + mlir::Operation* op; + absl::string_view name; + Thunk::ThunkInfo thunk_info; + MlirBufferSlice extra_slice; +}; + // Emits LLVM IR for an "unnested computation". // // An unnested computation is an HloComputation which you run by executing one @@ -89,12 +124,14 @@ class IrEmitterUnnested : public IrEmitter, const string& loop_name, llvm::Value* tile_height, llvm::Value* tile_width, KernelSupportLibrary* ksl)>; - IrEmitterUnnested(const HloModuleConfig& hlo_module_config, - const HloComputation* hlo_computation, - IrEmitterContext* ir_emitter_context); IrEmitterUnnested(const IrEmitterUnnested&) = delete; IrEmitterUnnested& operator=(const IrEmitterUnnested&) = delete; + static StatusOr> Create( + const HloModuleConfig& hlo_module_config, + const HloComputation* hlo_computation, + IrEmitterContext* ir_emitter_context); + // Transfers the ownship of thunk_sequence_ out. std::unique_ptr ConsumeThunkSequence() { return std::make_unique(std::move(thunk_sequence_)); @@ -124,6 +161,7 @@ class IrEmitterUnnested : public IrEmitter, Status HandleScatter(HloInstruction* scatter) override; Status HandleSelect(HloInstruction* select) override; Status HandleSort(HloInstruction* sort) override; + Status EmitMlirSort(MlirEmitterInput input); Status HandleTriangularSolve(HloInstruction* hlo) override; Status HandleTupleSelect(HloInstruction* tuple_select) override; Status HandleAllReduce(HloInstruction* crs) override; @@ -148,6 +186,10 @@ class IrEmitterUnnested : public IrEmitter, Status Postprocess(HloInstruction* hlo) override; private: + IrEmitterUnnested(const HloModuleConfig& hlo_module_config, + const HloComputation* hlo_computation, + IrEmitterContext* ir_emitter_context); + // Add a owning Thunk object to the thunk sequence. void AddThunkToThunkSequence(std::unique_ptr thunk) override { thunk_sequence_.emplace_back(std::move(thunk)); @@ -264,8 +306,7 @@ class IrEmitterUnnested : public IrEmitter, // Builds the prototype of the IR kernel for `inst` and adds it to the module. // This kernel takes as arguments pointers to the given buffer allocations. llvm::Function* BuildKernelPrototype( - const HloInstruction& inst, - absl::Span args); + absl::string_view name, absl::Span args); // Helper for writing extra outputs from inside a reduce kernel. Status EmitExtraOutputsForReduce( @@ -490,6 +531,12 @@ class IrEmitterUnnested : public IrEmitter, HloComputation* reducer, llvm::Type* element_type, llvm::Value* partial_result_address); + std::unique_ptr BuildKernelThunkFromBufferSlices( + absl::string_view name, Thunk::ThunkInfo thunk_info, + absl::Span slices, + std::function + bind_slice_to_ir_value); + // Returns a KernelThunk that invokes the kernel emitted for `inst`. The // caller needs to make sure `inst` outlives the lifetime of the returned // Thunk object. 'implements_whole_instruction' specifies whether this @@ -498,6 +545,11 @@ class IrEmitterUnnested : public IrEmitter, std::unique_ptr BuildKernelThunk( const HloInstruction* inst, bool implements_whole_instruction); + std::unique_ptr BuildKernelThunkForMlir( + absl::string_view name, Thunk::ThunkInfo thunk_info, + absl::Span slices, + std::vector* ir_arrays); + // Returns a thunk that, given a reduce or select-and-scatter op, // initializes its memory to the appropriate initial value. StatusOr> BuildInitializerThunk( @@ -505,17 +557,18 @@ class IrEmitterUnnested : public IrEmitter, // Returns a WhileThunk that invokes thunk sequences for 'condition' and // 'body' sub-computations of while instruction 'hlo'. - std::unique_ptr BuildWhileThunk(const HloInstruction* hlo); + StatusOr> BuildWhileThunk(const HloInstruction* hlo); // Returns a ForThunk which executes 'loop_limit' invocations of a thunk // sequence from the 'body' sub-computation of the while instruction 'hlo'. - std::unique_ptr BuildForThunk(const HloInstruction* hlo, - const int64 loop_limit); + StatusOr> BuildForThunk(const HloInstruction* hlo, + const int64 loop_limit); // Returns a ConditionalThunk which executes the thunk sequence for the // 'branch_computation' corresponding to the predicate/branch_index of the // given conditional instruction. - std::unique_ptr BuildConditionalThunk(const HloInstruction* hlo); + StatusOr> BuildConditionalThunk( + const HloInstruction* hlo); // Emits current thread id with the given type. // @@ -545,6 +598,9 @@ class IrEmitterUnnested : public IrEmitter, absl::optional thread_id_filter = absl::nullopt, absl::optional block_id_filter = absl::nullopt); + StatusOr GetOrCreateSubComputationFromRegion( + mlir::Region* region); + // Returns the last generated thunk. Thunk* LastThunk() const { return thunk_sequence_.back().get(); } @@ -555,6 +611,14 @@ class IrEmitterUnnested : public IrEmitter, // The HloComputation that this IrEmitter emits code for. const HloComputation* hlo_computation_; + + mlir::OwningModuleRef mlir_scratch_module_; + + // This is for cache-purpose only. It has no significant semantics. + mlir::LhloDialectEmitter lhlo_scratch_emitter_; + + absl::flat_hash_map> + scratch_nested_computations_; }; } // namespace gpu diff --git a/tensorflow/compiler/xla/service/gpu/tests/BUILD b/tensorflow/compiler/xla/service/gpu/tests/BUILD index a2bddd2d0d7..809b277317f 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/BUILD +++ b/tensorflow/compiler/xla/service/gpu/tests/BUILD @@ -458,6 +458,35 @@ xla_test( ], ) +tf_cc_test( + name = "sorting_test", + srcs = [ + "sorting_test.cc", + ], + tags = tf_cuda_tests_tags() + [ + "no_rocm", + ], + deps = [ + ":gpu_codegen_test", + "//tensorflow/compiler/xla:debug_options_flags", + "//tensorflow/compiler/xla:statusor", + "//tensorflow/compiler/xla:xla_proto_cc", + "//tensorflow/compiler/xla/service:gpu_plugin", + "//tensorflow/compiler/xla/service:hlo", + "//tensorflow/compiler/xla/service:hlo_module_config", + "//tensorflow/compiler/xla/service:hlo_parser", + "//tensorflow/compiler/xla/service/gpu:gpu_executable", + "//tensorflow/compiler/xla/tests:filecheck", + "//tensorflow/compiler/xla/tests:hlo_test_base", + "//tensorflow/compiler/xla/tests:llvm_irgen_test_base", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/stream_executor/lib", + "@com_google_absl//absl/memory", + ], +) + tf_cc_binary( name = "hlo_to_llvm_ir", srcs = ["hlo_to_llvm_ir.cc"], diff --git a/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo b/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo index 272c9a25769..4d29a8df116 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo +++ b/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo @@ -8,162 +8,162 @@ compare { ROOT lt = pred[] compare(p.0.lhs, p.0.rhs), direction=LT } -// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) +// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 +// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 +// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 -// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] -// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 -// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] -// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 +// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] +// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 +// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] +// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: call void @compare(float* [[TMP12]], float* [[TMP13]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP14:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP14]], 0 +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: call void @region_0_4(float* [[TMP16]], float* [[TMP17]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP18]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP16:%.*]] = load float, float* [[TMP13]], align 4 -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: store float [[TMP16]], float* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] +// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define internal void @compare(float* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) +// CHECK: define internal void @region_0_4(float* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) // CHECK-NEXT: entry: -// CHECK-NEXT: [[LT_TYPED:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[P_0_LHS_TYPED]], align 4 -// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[P_0_RHS_TYPED]], align 4 +// CHECK-NEXT: [[COMPARE_3_TYPED:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[ARG_0_1_TYPED:%.*]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[ARG_1_2_TYPED:%.*]], align 4 // CHECK-NEXT: [[TMP2:%.*]] = fcmp olt float [[TMP0]], [[TMP1]] // CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i8 -// CHECK-NEXT: store i8 [[TMP3]], i8* [[LT_TYPED]], align 1 -// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[LT_TYPED]], align 1 -// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG]], align 1 +// CHECK-NEXT: store i8 [[TMP3]], i8* [[COMPARE_3_TYPED]], align 1 +// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[COMPARE_3_TYPED]], align 1 +// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG:%.*]], align 1 // CHECK-NEXT: ret void -// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) { +// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) { // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 +// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 +// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = xor i64 [[TMP4]], 3 -// CHECK-NEXT: [[TMP8:%.*]] = icmp slt i64 [[TMP4]], [[TMP7]] -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], 3 -// CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP8]], [[TMP9]] -// CHECK-NEXT: br i1 [[TMP10]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP11:%.*]] = xor i64 [[TMP8]], 3 +// CHECK-NEXT: [[TMP12:%.*]] = icmp slt i64 [[TMP8]], [[TMP11]] +// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], 3 +// CHECK-NEXT: [[TMP14:%.*]] = and i1 [[TMP12]], [[TMP13]] +// CHECK-NEXT: br i1 [[TMP14]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: call void @compare(float* [[TMP11]], float* [[TMP12]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP13:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP13]], 0 +// CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP8]] +// CHECK-NEXT: call void @region_0_4(float* [[TMP15]], float* [[TMP16]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP17:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP17]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP14:%.*]] = load float, float* [[TMP11]], align 4 -// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: store float [[TMP14]], float* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = load float, float* [[TMP15]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP8]] +// CHECK-NEXT: store float [[TMP18]], float* [[TMP20]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) { +// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) { // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 +// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 +// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 -// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] -// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 -// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] -// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 +// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] +// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 +// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] +// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: call void @compare(float* [[TMP12]], float* [[TMP13]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP14:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP14]], 0 +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: call void @region_0_4(float* [[TMP16]], float* [[TMP17]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP18]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP16:%.*]] = load float, float* [[TMP13]], align 4 -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: store float [[TMP16]], float* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] +// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] +// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] ENTRY main { x = f32[2, 3] parameter(0) @@ -182,210 +182,198 @@ compare { ROOT lt = pred[] compare(p.1.lhs, p.1.rhs), direction=LT } -// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* -// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 -// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3]], i64 0 -// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* +// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 +// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 +// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 +// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 -// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] -// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 -// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] -// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP10]], 2 +// CHECK-NEXT: [[TMP14:%.*]] = xor i64 [[TMP13]], 1 +// CHECK-NEXT: [[TMP15:%.*]] = icmp slt i64 [[TMP13]], [[TMP14]] +// CHECK-NEXT: [[TMP16:%.*]] = icmp slt i64 [[TMP14]], 3 +// CHECK-NEXT: [[TMP17:%.*]] = and i1 [[TMP15]], [[TMP16]] +// CHECK-NEXT: br i1 [[TMP17]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: call void @compare(i32* [[TMP12]], i32* [[TMP13]], float* [[TMP14]], float* [[TMP15]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP16:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP16]], 0 +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP14]] +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP14]] +// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: call void @region_0_6(i32* [[TMP18]], i32* [[TMP19]], float* [[TMP20]], float* [[TMP21]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP22:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP22]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP13]], align 4 -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: store i32 [[TMP18]], i32* [[TMP20]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = load float, float* [[TMP14]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = load float, float* [[TMP15]], align 4 -// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP21]], float* [[TMP23]], align 4 -// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP8]] -// CHECK-NEXT: store float [[TMP22]], float* [[TMP24]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: store i32 [[TMP23]], i32* [[TMP25]], align 4 +// CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP14]] +// CHECK-NEXT: store i32 [[TMP24]], i32* [[TMP26]], align 4 +// CHECK-NEXT: [[TMP27:%.*]] = load float, float* [[TMP20]], align 4 +// CHECK-NEXT: [[TMP28:%.*]] = load float, float* [[TMP21]], align 4 +// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: store float [[TMP27]], float* [[TMP29]], align 4 +// CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP14]] +// CHECK-NEXT: store float [[TMP28]], float* [[TMP30]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define internal void @compare(i32* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], i32* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) +// CHECK: define internal void @region_0_6(i32* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], i32* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) // CHECK-NEXT: entry: -// CHECK-NEXT: [[LT_TYPED:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[P_1_LHS_TYPED]], align 4 -// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[P_1_RHS_TYPED]], align 4 +// CHECK-NEXT: [[COMPARE_5_TYPED:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[ARG_2_3_TYPED:%.*]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[ARG_3_4_TYPED:%.*]], align 4 // CHECK-NEXT: [[TMP2:%.*]] = fcmp olt float [[TMP0]], [[TMP1]] // CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i8 -// CHECK-NEXT: store i8 [[TMP3]], i8* [[LT_TYPED]], align 1 -// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[LT_TYPED]], align 1 -// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG]], align 1 +// CHECK-NEXT: store i8 [[TMP3]], i8* [[COMPARE_5_TYPED]], align 1 +// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[COMPARE_5_TYPED]], align 1 +// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG:%.*]], align 1 // CHECK-NEXT: ret void -// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* -// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2:%.*]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3:%.*]], i64 0 -// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* +// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 +// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 +// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 +// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP7:%.*]] = xor i64 [[TMP4]], 3 -// CHECK-NEXT: [[TMP8:%.*]] = icmp slt i64 [[TMP4]], [[TMP7]] -// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], 3 -// CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP8]], [[TMP9]] -// CHECK-NEXT: br i1 [[TMP10]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP13:%.*]] = xor i64 [[TMP10]], 3 +// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP10]], [[TMP13]] +// CHECK-NEXT: [[TMP15:%.*]] = icmp slt i64 [[TMP13]], 3 +// CHECK-NEXT: [[TMP16:%.*]] = and i1 [[TMP14]], [[TMP15]] +// CHECK-NEXT: br i1 [[TMP16]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: call void @compare(i32* [[TMP11]], i32* [[TMP12]], float* [[TMP13]], float* [[TMP14]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP15:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP15]], 0 +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP10]] +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP10]] +// CHECK-NEXT: call void @region_0_6(i32* [[TMP17]], i32* [[TMP18]], float* [[TMP19]], float* [[TMP20]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP21:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP21]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP11]], align 4 -// CHECK-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP12]], align 4 -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: store i32 [[TMP16]], i32* [[TMP18]], align 4 -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP13]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = load float, float* [[TMP14]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP4]] -// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 -// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] -// CHECK-NEXT: store float [[TMP21]], float* [[TMP23]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP10]] +// CHECK-NEXT: store i32 [[TMP22]], i32* [[TMP24]], align 4 +// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: store i32 [[TMP23]], i32* [[TMP25]], align 4 +// CHECK-NEXT: [[TMP26:%.*]] = load float, float* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP27:%.*]] = load float, float* [[TMP20]], align 4 +// CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP10]] +// CHECK-NEXT: store float [[TMP26]], float* [[TMP28]], align 4 +// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] +// CHECK-NEXT: store float [[TMP27]], float* [[TMP29]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* -// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* -// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2:%.*]], i64 0 -// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3:%.*]], i64 0 -// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 -// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 -// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] +// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* +// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 +// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 +// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 -// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 +// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: -// CHECK-NEXT: [[TMP7:%.*]] = bitcast [2 x [3 x i32]]* [[SORT_TYPED2]] to i8* -// CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[SORT_TYPED]], i64 0, i64 0 -// CHECK-NEXT: store i8* [[TMP7]], i8** [[TMP8]], align 8 -// CHECK-NEXT: [[TMP9:%.*]] = bitcast [2 x [3 x float]]* [[SORT_TYPED4]] to i8* -// CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[SORT_TYPED]], i64 0, i64 1 -// CHECK-NEXT: store i8* [[TMP9]], i8** [[TMP10]], align 8 +// CHECK-NEXT: [[TMP13:%.*]] = bitcast [2 x [3 x i32]]* [[TMP1]] to i8* +// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[TMP5]], i64 0, i64 0 +// CHECK-NEXT: store i8* [[TMP13]], i8** [[TMP14]], align 8 +// CHECK-NEXT: [[TMP15:%.*]] = bitcast [2 x [3 x float]]* [[TMP3]] to i8* +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[TMP5]], i64 0, i64 1 +// CHECK-NEXT: store i8* [[TMP15]], i8** [[TMP16]], align 8 // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP4]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 -// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] -// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 -// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] -// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP10]], 2 +// CHECK-NEXT: [[TMP18:%.*]] = xor i64 [[TMP17]], 1 +// CHECK-NEXT: [[TMP19:%.*]] = icmp slt i64 [[TMP17]], [[TMP18]] +// CHECK-NEXT: [[TMP20:%.*]] = icmp slt i64 [[TMP18]], 3 +// CHECK-NEXT: [[TMP21:%.*]] = and i1 [[TMP19]], [[TMP20]] +// CHECK-NEXT: br i1 [[TMP21]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP12]] -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP11]] -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP12]] -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP11]] -// CHECK-NEXT: call void @compare(i32* [[TMP16]], i32* [[TMP17]], float* [[TMP18]], float* [[TMP19]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP20:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP20]], 0 +// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP18]] +// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP17]] +// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP18]] +// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP17]] +// CHECK-NEXT: call void @region_0_6(i32* [[TMP22]], i32* [[TMP23]], float* [[TMP24]], float* [[TMP25]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP26:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP26]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP21:%.*]] = load i32, i32* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP11]] -// CHECK-NEXT: store i32 [[TMP21]], i32* [[TMP23]], align 4 -// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP12]] -// CHECK-NEXT: store i32 [[TMP22]], i32* [[TMP24]], align 4 -// CHECK-NEXT: [[TMP25:%.*]] = load float, float* [[TMP18]], align 4 -// CHECK-NEXT: [[TMP26:%.*]] = load float, float* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP11]] -// CHECK-NEXT: store float [[TMP25]], float* [[TMP27]], align 4 -// CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP12]] -// CHECK-NEXT: store float [[TMP26]], float* [[TMP28]], align 4 +// CHECK-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP22]], align 4 +// CHECK-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP23]], align 4 +// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP17]] +// CHECK-NEXT: store i32 [[TMP27]], i32* [[TMP29]], align 4 +// CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP18]] +// CHECK-NEXT: store i32 [[TMP28]], i32* [[TMP30]], align 4 +// CHECK-NEXT: [[TMP31:%.*]] = load float, float* [[TMP24]], align 4 +// CHECK-NEXT: [[TMP32:%.*]] = load float, float* [[TMP25]], align 4 +// CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP17]] +// CHECK-NEXT: store float [[TMP31]], float* [[TMP33]], align 4 +// CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP18]] +// CHECK-NEXT: store float [[TMP32]], float* [[TMP34]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] ENTRY main { x = s32[2, 3] parameter(0) diff --git a/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc b/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc new file mode 100644 index 00000000000..197a0c6cfeb --- /dev/null +++ b/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc @@ -0,0 +1,71 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/xla/service/gpu/gpu_executable.h" +#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h" +#include "tensorflow/compiler/xla/service/hlo_instruction.h" +#include "tensorflow/compiler/xla/service/hlo_module_config.h" +#include "tensorflow/compiler/xla/service/hlo_parser.h" +#include "tensorflow/compiler/xla/statusor.h" +#include "tensorflow/compiler/xla/tests/filecheck.h" +#include "tensorflow/compiler/xla/tests/hlo_test_base.h" +#include "tensorflow/compiler/xla/xla.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/stream_executor/lib/statusor.h" + +namespace xla { +namespace gpu { + +namespace { + +class SortingTest : public GpuCodegenTest { + protected: + HloModuleConfig ConfigWithoutLayoutAssignment() { + HloModuleConfig config; + auto debug_options = HloTestBase::GetDebugOptionsForTest(); + // Disable layout_assignment to use the preassigned layouts. + debug_options.add_xla_disable_hlo_passes("layout-assignment"); + config.set_debug_options(debug_options); + return config; + } +}; + +TEST_F(SortingTest, Regression1) { + const char* hlo_text = R"( +HloModule TestModule + +compare { + p.0.lhs = f32[] parameter(0) + p.0.rhs = f32[] parameter(1) + ROOT lt = pred[] compare(p.0.lhs, p.0.rhs), direction=LT +} + +ENTRY TestComputation { + x = f32[3, 2]{1, 0} parameter(0) + x.copy = f32[3, 2]{0, 1} copy(x) + ROOT sort = f32[3, 2]{0, 1} sort(x.copy), dimensions={1}, to_apply=compare +} + +)"; + + EXPECT_TRUE(RunAndCompareNoHloPasses(hlo_text, ErrorSpec{1e-5, 1e-5})); +} + +} // namespace +} // namespace gpu +} // namespace xla diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc index b01ae2efe43..2963d546380 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc @@ -415,9 +415,10 @@ llvm::Instruction* AddRangeMetadata(int64 lower, int64 upper, return inst; } -string IrName(string a) { - a.erase(std::remove(a.begin(), a.end(), '%'), a.end()); - return a; +string IrName(absl::string_view a) { + std::string s(a); + s.erase(std::remove(s.begin(), s.end(), '%'), s.end()); + return s; } string IrName(absl::string_view a, absl::string_view b) { diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h index 642965b6470..c0a55e4da33 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h @@ -87,7 +87,7 @@ string DumpModuleToString(const llvm::Module& module); // - joining all of the nonempty inputs by '.', and then // - removing all '%'s. // -string IrName(string a); +string IrName(absl::string_view a); string IrName(absl::string_view a, absl::string_view b); string IrName(const HloInstruction* a, absl::string_view b = ""); From 479fa040f6ba46378fed5fbf76a4a3c1124dc2ca Mon Sep 17 00:00:00 2001 From: Anna R Date: Wed, 12 Aug 2020 13:16:15 -0700 Subject: [PATCH 0917/1017] Change string.maketrans to str.maketrans since the former is deprecated in python 3. PiperOrigin-RevId: 326297477 Change-Id: Ib367ede472bab744ddfec30e51b65d9ef8cf58ce --- tensorflow/tools/test/run_and_gather_logs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow/tools/test/run_and_gather_logs.py b/tensorflow/tools/test/run_and_gather_logs.py index a1486826615..d8b706513ab 100644 --- a/tensorflow/tools/test/run_and_gather_logs.py +++ b/tensorflow/tools/test/run_and_gather_logs.py @@ -22,7 +22,6 @@ from __future__ import print_function import argparse import os import shlex -from string import maketrans import sys import time @@ -86,7 +85,7 @@ def main(unused_args): file_name = FLAGS.test_log_output_filename else: file_name = ( - six.ensure_str(name).strip("/").translate(maketrans("/:", "__")) + + six.ensure_str(name).strip("/").translate(str.maketrans("/:", "__")) + time.strftime("%Y%m%d%H%M%S", time.gmtime())) if FLAGS.test_log_output_use_tmpdir: tmpdir = test.get_temp_dir() From a0dbffedac9a540377fa1e98fd0da8f4d6cd5f90 Mon Sep 17 00:00:00 2001 From: Edward Loper Date: Wed, 12 Aug 2020 13:16:25 -0700 Subject: [PATCH 0918/1017] Add c++ utility function that converts AttrValue protobufs to corresponding PyObjects. PiperOrigin-RevId: 326297504 Change-Id: Id70b00d2708c44459b1b4038a273387b1dc54075 --- tensorflow/python/framework/op_def_util.cc | 109 ++++++++++++++++++ tensorflow/python/framework/op_def_util.h | 21 ++++ .../python/framework/op_def_util_pybind.cc | 20 +++- .../python/framework/op_def_util_test.py | 46 +++++++- 4 files changed, 190 insertions(+), 6 deletions(-) diff --git a/tensorflow/python/framework/op_def_util.cc b/tensorflow/python/framework/op_def_util.cc index c915c494be9..4e1569f190d 100644 --- a/tensorflow/python/framework/op_def_util.cc +++ b/tensorflow/python/framework/op_def_util.cc @@ -17,19 +17,28 @@ limitations under the License. #include #include "absl/strings/str_cat.h" +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" #include "tensorflow/core/framework/types.pb.h" +#include "tensorflow/python/lib/core/safe_ptr.h" #include "tensorflow/python/util/util.h" using ::tensorflow::swig::GetRegisteredPyObject; #if PY_MAJOR_VERSION < 3 +// Python 2.x: #define PY_STRING_CHECK(x) (PyString_Check(x) || PyUnicode_Check(x)) +#define PY_STRING_FROMSTRING(x) (PyString_FromString(x)) #define PY_INT_CHECK(x) (PyInt_Check(x)) #define PY_INT_TYPE PyInt_Type +#define PY_INT_FROM_LONG(x) (PyInt_FromLong(x)) #else +// Python 3.x: #define PY_STRING_CHECK(x) (PyBytes_Check(x) || PyUnicode_Check(x)) +#define PY_STRING_FROMSTRING(x) (PyUnicode_FromString(x)) #define PY_INT_CHECK(x) (PyLong_Check(x)) #define PY_INT_TYPE PyLong_Type +#define PY_INT_FROM_LONG(x) (PyLong_FromLong(x)) #endif namespace tensorflow { @@ -239,6 +248,64 @@ Safe_PyObjectPtr ConvertAttrOrNull(PyObject* value, AttributeType attr_type) { } } +// Returns a new reference to Py_True or Py_False depending on b. +PyObject* PyBool_FromBool(bool b) { + PyObject* result = b ? Py_True : Py_False; + Py_INCREF(result); + return result; +} + +Safe_PyObjectPtr AttrValueListToPyObject(AttrValue::ListValue list) { + if (list.s_size()) { + Safe_PyObjectPtr result(PyList_New(list.s_size())); + for (int i = 0; i < list.s_size(); ++i) { + PyList_SET_ITEM(result.get(), i, PY_STRING_FROMSTRING(list.s(i).c_str())); + } + return result; + } else if (list.i_size()) { + Safe_PyObjectPtr result(PyList_New(list.i_size())); + for (int i = 0; i < list.i_size(); ++i) { + PyList_SET_ITEM(result.get(), i, PY_INT_FROM_LONG(list.i(i))); + } + return result; + } else if (list.f_size()) { + Safe_PyObjectPtr result(PyList_New(list.f_size())); + for (int i = 0; i < list.f_size(); ++i) { + PyList_SET_ITEM(result.get(), i, PyFloat_FromDouble(list.f(i))); + } + return result; + } else if (list.b_size()) { + Safe_PyObjectPtr result(PyList_New(list.b_size())); + for (int i = 0; i < list.b_size(); ++i) { + PyList_SET_ITEM(result.get(), i, PyBool_FromBool(list.b(i))); + } + return result; + } else if (list.type_size()) { + Safe_PyObjectPtr result(PyList_New(list.type_size())); + for (int i = 0; i < list.type_size(); ++i) { + Safe_PyObjectPtr item(DataTypeToPyObject(list.type(i))); + Py_INCREF(item.get()); + PyList_SET_ITEM(result.get(), i, item.get()); + } + return result; + } else if (list.shape_size()) { + Safe_PyObjectPtr result(PyList_New(list.shape_size())); + for (int i = 0; i < list.shape_size(); ++i) { + Safe_PyObjectPtr item(TensorShapeProtoToPyObject(list.shape(i))); + Py_INCREF(item.get()); + PyList_SET_ITEM(result.get(), i, item.get()); + } + return result; + } else if (list.tensor_size() || list.func_size()) { + // TODO(edloper): Add support for tensorflow::AttrValue::kTensor. + PyErr_SetString(PyExc_TypeError, "Unsupported AttrValue type"); + return nullptr; + } else { + // Empty list + return Safe_PyObjectPtr(PyList_New(0)); + } +} + } // namespace AttributeType AttributeTypeFromName(const std::string& type_name) { @@ -269,4 +336,46 @@ Safe_PyObjectPtr ConvertPyObjectToAttributeType(PyObject* value, return result; } +Safe_PyObjectPtr AttrValueToPyObject(const AttrValue& attr_value) { + switch (attr_value.value_case()) { + case tensorflow::AttrValue::kS: + return Safe_PyObjectPtr(PY_STRING_FROMSTRING(attr_value.s().c_str())); + case tensorflow::AttrValue::kI: + return Safe_PyObjectPtr(PY_INT_FROM_LONG(attr_value.i())); + case tensorflow::AttrValue::kF: + return Safe_PyObjectPtr(PyFloat_FromDouble(attr_value.f())); + case tensorflow::AttrValue::kB: + return Safe_PyObjectPtr(PyBool_FromBool(attr_value.b())); + case tensorflow::AttrValue::kType: + return DataTypeToPyObject(attr_value.type()); + case tensorflow::AttrValue::kShape: + return TensorShapeProtoToPyObject(attr_value.shape()); + case tensorflow::AttrValue::kList: + return AttrValueListToPyObject(attr_value.list()); + default: + // TODO(edloper): Add support for tensorflow::AttrValue::kTensor. + PyErr_SetString(PyExc_ValueError, "Unsupported AttrValue type"); + return nullptr; + } +} + +Safe_PyObjectPtr DataTypeToPyObject(const DataType& data_type) { + Safe_PyObjectPtr enum_value(PY_INT_FROM_LONG(data_type)); + return ConvertDTypeFunctor()(enum_value.get()); +} + +Safe_PyObjectPtr TensorShapeProtoToPyObject( + const TensorShapeProto& tensor_shape) { + if (tensor_shape.unknown_rank()) { + return ConvertTensorShapeFunctor()(Py_None); + } else { + Safe_PyObjectPtr dims(PyTuple_New(tensor_shape.dim_size())); + for (int i = 0; i < tensor_shape.dim_size(); ++i) { + PyTuple_SET_ITEM(dims.get(), i, + PY_INT_FROM_LONG(tensor_shape.dim(i).size())); + } + return ConvertTensorShapeFunctor()(dims.get()); + } +} + } // namespace tensorflow diff --git a/tensorflow/python/framework/op_def_util.h b/tensorflow/python/framework/op_def_util.h index ef5e64e68fa..3b35c3ef7ad 100644 --- a/tensorflow/python/framework/op_def_util.h +++ b/tensorflow/python/framework/op_def_util.h @@ -15,8 +15,14 @@ limitations under the License. #ifndef TENSORFLOW_PYTHON_FRAMEWORK_OP_DEF_UTIL_H_ #define TENSORFLOW_PYTHON_FRAMEWORK_OP_DEF_UTIL_H_ +#include + #include +#include "tensorflow/core/framework/attr_value.pb.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/framework/types.pb.h" #include "tensorflow/python/lib/core/safe_ptr.h" namespace tensorflow { @@ -72,6 +78,21 @@ std::string AttributeTypeToName(AttributeType attr_type); Safe_PyObjectPtr ConvertPyObjectToAttributeType(PyObject* value, AttributeType type); +// Converts a c++ `AttrValue` protobuf message to a Python object; or sets a +// Python exception and returns nullptr if an error occurs. +Safe_PyObjectPtr AttrValueToPyObject(const AttrValue& attr_value); + +// Converts a c++ `DataType` protobuf enum to a Python object; or sets a +// Python exception and returns nullptr if an error occurs. +Safe_PyObjectPtr DataTypeToPyObject(const DataType& data_type); + +// Converts a c++ `TensorShapeProto` message to a Python object; or sets a +// Python exception and returns nullptr if an error occurs. +Safe_PyObjectPtr TensorShapeProtoToPyObject( + const TensorShapeProto& tensor_shape); + +// TODO(edloper): Define TensorProtoToPyObject? + } // namespace tensorflow #endif // TENSORFLOW_PYTHON_FRAMEWORK_OP_DEF_UTIL_H_ diff --git a/tensorflow/python/framework/op_def_util_pybind.cc b/tensorflow/python/framework/op_def_util_pybind.cc index d13f605b599..a7843322840 100644 --- a/tensorflow/python/framework/op_def_util_pybind.cc +++ b/tensorflow/python/framework/op_def_util_pybind.cc @@ -30,12 +30,26 @@ py::handle ConvertAttr(py::handle value, std::string attr_type) { return result.release(); } +py::handle SerializedAttrValueToPyObject(std::string attr_value_string) { + tensorflow::AttrValue attr_value; + attr_value.ParseFromString(attr_value_string); + tensorflow::Safe_PyObjectPtr result = + ::tensorflow::AttrValueToPyObject(attr_value); + if (!result) { + throw py::error_already_set(); + } + Py_INCREF(result.get()); + return result.release(); +} + } // namespace -// Expose ConvertPyObjectToAttributeType via Python. Note: this is done to -// simplify testing; ConvertPyObjectToAttributeType is expected to be called -// directly from c++. +// Expose op_def_util.h functions via Python. PYBIND11_MODULE(_op_def_util, m) { + // Note: the bindings below are added for testing purposes; but the functions + // are expected to be called from c++, not Python. m.def("ConvertPyObjectToAttributeType", ConvertAttr, py::arg("value"), py::arg("attr_type_enum")); + m.def("SerializedAttrValueToPyObject", SerializedAttrValueToPyObject, + py::arg("attr_value_string")); } diff --git a/tensorflow/python/framework/op_def_util_test.py b/tensorflow/python/framework/op_def_util_test.py index 69aaffbf19f..9f2ce61996f 100644 --- a/tensorflow/python/framework/op_def_util_test.py +++ b/tensorflow/python/framework/op_def_util_test.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== - """Tests for tensorflow.python.ops.op_def_library.""" from __future__ import absolute_import @@ -23,6 +22,8 @@ from absl.testing import parameterized import numpy as np +from google.protobuf import text_format +from tensorflow.core.framework import attr_value_pb2 from tensorflow.core.framework import tensor_pb2 from tensorflow.core.framework import types_pb2 from tensorflow.python import _op_def_util @@ -63,7 +64,7 @@ class OpDefUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): ("list(int)", (1, 2.3), [1, 2]), ("list(float)", (1, 2.3), [1.0, 2.3]), ("list(bool)", [True, False], [True, False]), - ]) + ]) # pyformat: disable def testConvert(self, attr_type, value, expected): result = _op_def_util.ConvertPyObjectToAttributeType(value, attr_type) @@ -93,6 +94,45 @@ class OpDefUtilTest(test_util.TensorFlowTestCase, parameterized.TestCase): with self.assertRaisesRegex(TypeError, "Failed to convert value"): _op_def_util.ConvertPyObjectToAttributeType(value, attr_type) + # Test AttrValueToPyObject(). Note: this test also exercises the code in + # DataTypeToPyObject() and TensorShapeToPyObject(), since those are used + # when the AttrValue contains a DataType or TensorShape. + @parameterized.parameters([ + ("s: 'foo'", "foo"), + ("i: 5", 5), + ("f: 8", 8.0), + ("b: True", True), + ("type: DT_INT32", dtypes.int32), + ("shape { dim: [{size: 3}, {size: 4}] }", + tensor_shape.TensorShape([3, 4])), + ("list { }", []), + ("list { s: [] }", []), + ("list { s: ['a', 'b', 'c'] }", ["a", "b", "c"]), + ("list { i: [1, 2, 3] }", [1, 2, 3]), + ("list { f: [2.0, 4.0] }", [2.0, 4.0]), + ]) # pyformat: disable + def testAttrValueToPyObject(self, pbtxt, expected): + proto = attr_value_pb2.AttrValue() + text_format.Parse(pbtxt, proto) + result = _op_def_util.SerializedAttrValueToPyObject( + proto.SerializeToString()) + + self.assertEqual(expected, result) + + @parameterized.parameters([ + "", # Empty value (oneof not set) + "tensor {}", # 'TensorProto' not supported (yet). + "func {}", # 'func' not supported. + "placeholder: ''", # 'placeholder' not supported. + "list { tensor [{}] }", # 'TensorProto' not supported (yet). + "list { func [{}] }", # 'func' not supported. + ]) # pyformat: disable + def testAttrValueToPyObjectError(self, pbtxt): + proto = attr_value_pb2.AttrValue() + text_format.Parse(pbtxt, proto) + with self.assertRaises((TypeError, ValueError)): + _op_def_util.SerializedAttrValueToPyObject(proto.SerializeToString()) + + if __name__ == "__main__": googletest.main() - From a014eda1feb7146ad177fe23c9da5be20e5053de Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 12 Aug 2020 13:17:01 -0700 Subject: [PATCH 0919/1017] Introduce a local cloud TPU client for backwards compatibility with existing APIs PiperOrigin-RevId: 326297624 Change-Id: I562ac757ff4b5ebaf962be5e94ad8cfb425a78ca --- .../cluster_resolver/tpu/tpu_cluster_resolver.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py b/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py index d400e7aeed4..393561a80aa 100644 --- a/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py +++ b/tensorflow/python/distribute/cluster_resolver/tpu/tpu_cluster_resolver.py @@ -42,6 +42,13 @@ def is_running_in_gce(): return True +class _LocalCloudTpuClient(object): + """Dummy local Cloud TPU client.""" + + def api_available(self): + return False + + _TPU_DEVICE_REGEX = re.compile( r'.*task:(?P\d+)/.*device:TPU:(?P\d+)$') _TPU_CONN_RETRIES = 120 @@ -201,7 +208,7 @@ class TPUClusterResolver(cluster_resolver.ClusterResolver): self._tpu = self._cloud_tpu_client.name() else: # Directly connected TPU environment - self._cloud_tpu_client = None + self._cloud_tpu_client = _LocalCloudTpuClient() self._tpu = 'local' # By default the task_type is 'worker` and the task_id is 0 (which is the From 46e340eb3efea1f97cf23788ad2a89b51232b432 Mon Sep 17 00:00:00 2001 From: Rick Chao Date: Wed, 12 Aug 2020 13:18:59 -0700 Subject: [PATCH 0920/1017] PSv2: Create tensorflow/python/distribute/client:remote_eager_lib py_library. PiperOrigin-RevId: 326298038 Change-Id: Ie969248ba449de1c129d8f20db08aaec9ee80515 --- tensorflow/python/distribute/client/BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/python/distribute/client/BUILD b/tensorflow/python/distribute/client/BUILD index d0d8d3af4ec..35cd8d06282 100644 --- a/tensorflow/python/distribute/client/BUILD +++ b/tensorflow/python/distribute/client/BUILD @@ -103,3 +103,9 @@ tf_py_test( "//tensorflow/python/eager:test", ], ) + +py_library( + name = "remote_eager_lib", + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) From 62b8669ae3484dce47f7b46fc2f035dcb5d40c99 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 13:45:58 -0700 Subject: [PATCH 0921/1017] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 326303912 Change-Id: I183cc7f66538bb4e1047f0a22aba352d31e50cb9 --- tensorflow/go/op/wrappers.go | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 4d39ab20deb..22223dc2b1a 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -10899,12 +10899,26 @@ func ExperimentalRandomDataset(scope *Scope, seed tf.Output, seed2 tf.Output, ou return op.Output(0) } +// ExperimentalIgnoreErrorsDatasetAttr is an optional argument to ExperimentalIgnoreErrorsDataset. +type ExperimentalIgnoreErrorsDatasetAttr func(optionalAttr) + +// ExperimentalIgnoreErrorsDatasetLogWarning sets the optional log_warning attribute to value. +// If not specified, defaults to false +func ExperimentalIgnoreErrorsDatasetLogWarning(value bool) ExperimentalIgnoreErrorsDatasetAttr { + return func(m optionalAttr) { + m["log_warning"] = value + } +} + // Creates a dataset that contains the elements of `input_dataset` ignoring errors. -func ExperimentalIgnoreErrorsDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { +func ExperimentalIgnoreErrorsDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape, optional ...ExperimentalIgnoreErrorsDatasetAttr) (handle tf.Output) { if scope.Err() != nil { return } attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ Type: "ExperimentalIgnoreErrorsDataset", Input: []tf.Input{ @@ -31035,12 +31049,26 @@ func MaxPool3DGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output return op.Output(0) } +// IgnoreErrorsDatasetAttr is an optional argument to IgnoreErrorsDataset. +type IgnoreErrorsDatasetAttr func(optionalAttr) + +// IgnoreErrorsDatasetLogWarning sets the optional log_warning attribute to value. +// If not specified, defaults to false +func IgnoreErrorsDatasetLogWarning(value bool) IgnoreErrorsDatasetAttr { + return func(m optionalAttr) { + m["log_warning"] = value + } +} + // Creates a dataset that contains the elements of `input_dataset` ignoring errors. -func IgnoreErrorsDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) { +func IgnoreErrorsDataset(scope *Scope, input_dataset tf.Output, output_types []tf.DataType, output_shapes []tf.Shape, optional ...IgnoreErrorsDatasetAttr) (handle tf.Output) { if scope.Err() != nil { return } attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes} + for _, a := range optional { + a(attrs) + } opspec := tf.OpSpec{ Type: "IgnoreErrorsDataset", Input: []tf.Input{ From 6bfbd0cd263bff88ab330a090339b8bf02232ebd Mon Sep 17 00:00:00 2001 From: "T.J. Alumbaugh" Date: Wed, 12 Aug 2020 14:02:55 -0700 Subject: [PATCH 0922/1017] Update Interpreter docs to warn about creating Interpreters outside an InterpreterBuilder PiperOrigin-RevId: 326307418 Change-Id: I87b75ff122fb0cce2b2f4ab84fa8dd9e8ad0a529 --- tensorflow/lite/examples/minimal/minimal.cc | 5 ++- tensorflow/lite/interpreter.h | 50 +++++++++++---------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/tensorflow/lite/examples/minimal/minimal.cc b/tensorflow/lite/examples/minimal/minimal.cc index ba142da799a..6e569f95a63 100644 --- a/tensorflow/lite/examples/minimal/minimal.cc +++ b/tensorflow/lite/examples/minimal/minimal.cc @@ -48,7 +48,10 @@ int main(int argc, char* argv[]) { tflite::FlatBufferModel::BuildFromFile(filename); TFLITE_MINIMAL_CHECK(model != nullptr); - // Build the interpreter + // Build the interpreter with the InterpreterBuilder. + // Note: all Interpreters should be built with the InterpreterBuilder, + // which allocates memory for the Intrepter and does various set up + // tasks so that the Interpreter can read the provided model. tflite::ops::builtin::BuiltinOpResolver resolver; InterpreterBuilder builder(*model, resolver); std::unique_ptr interpreter; diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h index 5c354d63dd5..5be48b793fa 100644 --- a/tensorflow/lite/interpreter.h +++ b/tensorflow/lite/interpreter.h @@ -56,35 +56,39 @@ namespace impl { /// Usage: /// /// 

-/// // Create basic model
-/// Interpreter foo(2, 1);
-/// foo.SetTensorParametersReadWrite(0, ...);
-/// foo.SetTensorParametersReadOnly(1, ...);
-/// foo.SetNodeParameters(0, ...)
-/// // Resize input array to 1 length.
-/// foo.ResizeInputTensor(0, 1);
-/// foo.AllocateTensors();
-/// // Install array data
-/// foo.typed_tensor(0)[0] = 3;
-/// foo.Invoke();
-/// foo.typed_tensor(0)[0] = 4;
-/// foo.Invoke();
-/// // Resize input array and set data.
-/// foo.ResizeInputTensor(0, 2);
-/// foo.AllocateTensors();
-/// foo.typed_tensor(0)[0] = 4;
-/// foo.typed_tensor(0)[1] = 8;
-/// foo.Invoke();
+/// // Create model from file. Note that the model instance must outlive the
+/// // interpreter instance.
+/// auto model = tflite::FlatBufferModel::BuildFromFile(...);
+/// if (model == nullptr) {
+///   // Return error.
+/// }
+/// // Create an Interpreter with an InterpreterBuilder.
+/// std::unique_ptr interpreter;
+/// tflite::ops::builtin::BuiltinOpResolver resolver;
+/// if (InterpreterBuilder(*model, resolver)(&interpreter) != kTfLiteOk) {
+///   // Return failure.
+/// }
+/// interpreter->AllocateTensors();
+///
+/// auto input = interpreter->typed_tensor(0);
+/// for (int i = 0; i < input_size; i++) {
+///   input[i] = ...;
+//  }
+/// interpreter.Invoke();
 ///
/// +/// Note: for nearly all practical use cases, one should not directly construct +/// an Interpreter object, but rather use the InterpreterBuilder. class Interpreter { public: - /// Instantiate an interpreter. All errors associated with reading and - /// processing this model will be forwarded to the error_reporter object. + // Instantiate an interpreter. All errors associated with reading and + // processing this model will be forwarded to the error_reporter object. // - /// Note, if error_reporter is nullptr, then a default StderrReporter is - /// used. Ownership of 'error_reporter' remains with the caller. + // Note, if error_reporter is nullptr, then a default StderrReporter is + // used. Ownership of 'error_reporter' remains with the caller. + // WARNING: Use of this constructor outside of an InterpreterBuilder is not + // recommended. explicit Interpreter(ErrorReporter* error_reporter = DefaultErrorReporter()); ~Interpreter(); From ab4662d10abbc475e196da8a5d9e65ba627223c2 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Wed, 12 Aug 2020 14:05:36 -0700 Subject: [PATCH 0923/1017] Update XNNPACK dependency Pull a work-around for missing _mm512_set_epi8 on older GCC PiperOrigin-RevId: 326308016 Change-Id: I17c32d18d0de35923085c878b59927ab949a17bc --- tensorflow/workspace.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index dad05a1dac7..4524ade5ba1 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -164,11 +164,11 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "XNNPACK", - sha256 = "7d017f5f4bad9851fea8acaac0a24b5b1fa8105a24c5c05580356baffb0f1801", - strip_prefix = "XNNPACK-84324860cd20c3518f82597e1f9df343635a9426", + sha256 = "1edb168b8eb1b48e4ed7f8d18640c381ab19745cb21ea4279f27884339b6f17e", + strip_prefix = "XNNPACK-2a18f7ea635f3c10a4d920113e07b2e6ce038ac8", urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/XNNPACK/archive/84324860cd20c3518f82597e1f9df343635a9426.zip", - "https://github.com/google/XNNPACK/archive/84324860cd20c3518f82597e1f9df343635a9426.zip", + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/XNNPACK/archive/2a18f7ea635f3c10a4d920113e07b2e6ce038ac8.zip", + "https://github.com/google/XNNPACK/archive/2a18f7ea635f3c10a4d920113e07b2e6ce038ac8.zip", ], ) From 05dd0d322931f481084cfccdffb3c1b702408d61 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Wed, 12 Aug 2020 14:15:23 -0700 Subject: [PATCH 0924/1017] [tf.data] Update auto_shard rewrite to rewrite RebatchDatasetV2 to RebatchDataset when sharding by data. This is to ensure that elements are produced in the right order when using DistributedDataset. PiperOrigin-RevId: 326310238 Change-Id: I24d3c49c4a8b9224dbfc6024acb23b1bd4774b54 --- .../grappler/optimizers/data/auto_shard.cc | 122 ++++++++++++++---- .../grappler/optimizers/data/auto_shard.h | 1 + .../experimental/auto_shard_dataset_op.cc | 40 +++--- .../data/experimental/auto_shard_dataset_op.h | 5 +- .../auto_shard_dataset_op_test.cc | 13 +- .../core/ops/experimental_dataset_ops.cc | 1 + .../kernel_tests/auto_shard_dataset_test.py | 104 +++++++++++---- .../data/experimental/ops/distribute.py | 25 +++- .../api/golden/v1/tensorflow.raw_ops.pbtxt | 2 +- .../api/golden/v2/tensorflow.raw_ops.pbtxt | 2 +- 10 files changed, 237 insertions(+), 78 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/data/auto_shard.cc b/tensorflow/core/grappler/optimizers/data/auto_shard.cc index 4ad9cec4fe4..c57bd2c0a5d 100644 --- a/tensorflow/core/grappler/optimizers/data/auto_shard.cc +++ b/tensorflow/core/grappler/optimizers/data/auto_shard.cc @@ -42,8 +42,11 @@ constexpr char kShardDatasetOpName[] = "ShardDataset"; constexpr char kShuffleDatasetOpName[] = "ShuffleDataset"; constexpr char kShuffleDatasetV2OpName[] = "ShuffleDatasetV2"; constexpr char kShuffleDatasetV3OpName[] = "ShuffleDatasetV3"; +constexpr char kRebatchDatasetOpName[] = "RebatchDataset"; +constexpr char kRebatchDatasetV2OpName[] = "RebatchDatasetV2"; constexpr char kNumWorkersAttrName[] = "num_workers"; +constexpr char kNumReplicasAttrName[] = "num_replicas"; constexpr char kIndexAttrName[] = "index"; constexpr char kAutoShardPolicyAttrName[] = "auto_shard_policy"; constexpr char kReshuffleEachIteration[] = "reshuffle_each_iteration"; @@ -111,7 +114,8 @@ constexpr std::array kUnshardableSourceDatasetOps = { // clang-format on Status OptimizeGraph(const GrapplerItem& item, int64 num_workers, int64 index, - AutoShardPolicy policy, GraphDef* output); + AutoShardPolicy policy, int64 num_replicas, + GraphDef* output); template bool IsDatasetNodeOfType(const NodeDef& node, @@ -125,6 +129,7 @@ bool IsDatasetNodeOfType(const NodeDef& node, return false; } +// Adds a ShardDataset node before `add_before`. Status AddShardNode(MutableGraphView* graph, const NodeDef& add_before, int64 num_workers, int64 index) { NodeDef new_node; @@ -472,8 +477,81 @@ Status RecursivelyHandleOp(const NodeDef& node, int64 num_workers, int64 index, nodes_to_delete); } +// Recursively walk the dataset graph from sink to source, searching for +// the first (i.e. closest to the sink) occurence of a ReaderDataset, such as +// CSVDataset, TFRecordDataset, etc. We then insert a ShardDataset op before +// that nodes input, so that each worker only reads a subset of files. +// Additionally, we remove sources of randomness (e.g. ShuffleDataset) that +// occur upstream of the ShardDataset transformation to ensure that sharding +// returns a sensible result. +Status ShardByFile(const NodeDef& sink_node, int64 num_workers, int64 index, + FunctionLibraryDefinition* flib, MutableGraphView* graph) { + absl::flat_hash_set nodes_to_delete; + TF_RETURN_IF_ERROR(RecursivelyHandleOp(sink_node, num_workers, index, flib, + graph, &nodes_to_delete)); + return graph->DeleteNodes(nodes_to_delete); +} + +Status RewriteRebatchV2ToV1(const NodeDef& sink_node, int64 num_replicas, + MutableGraphView* graph) { + // The final node before AutoShardDataset is RebatchDataset. + // This is always the case as RebatchDataset and AutoShardDataset are internal + // APIs used directly by tf.distribute's input_lib. As such, instead of + // walking the entire dataset graph, we can walk up directly from the + // sink_node to get the RebatchDataset. + NodeDef* input_node = graph_utils::GetInputNode(sink_node, *graph); + if (input_node->op() != kRebatchDatasetV2OpName) { + return Status::OK(); + } + + NodeDef* rebatch_node = input_node; + // Update RebatchDatasetV2 in place. Since Rebatch is an internal API, no + // other nodes should have it as an input. + rebatch_node->set_op(kRebatchDatasetOpName); + // Delete the `batch_sizes` and `drop_remainder` input. + rebatch_node->mutable_input()->DeleteSubrange(/*start=*/1, /*num=*/2); + // Add the `num_replicas` input. + if (num_replicas < 1) { + return errors::InvalidArgument( + "Cannot rewrite RebatchDatasetV2 to legacy RebatchDataset with invalid " + "num_replicas argument. `num_replicas` is ", + num_replicas, ", but expected to be >= 1."); + } + auto num_replicas_node = graph_utils::AddScalarConstNode(num_replicas, graph); + rebatch_node->add_input(num_replicas_node->name()); + + // Set `use_fallback` attr. This attr is not used anywhere, so its value + // does not matter + (*rebatch_node->mutable_attr())["use_fallback"].set_b(true); + + // Update the output_shapes attr to set all its batch dimensions to -1 + // (unknown). + auto* shapes_attr = + gtl::FindOrNull(*rebatch_node->mutable_attr(), "output_shapes"); + if (shapes_attr == nullptr) { + return errors::InvalidArgument( + "Cannot rewrite RebatchDatasetV2 with missing `output_shapes` attr."); + } + for (int i = 0; i < shapes_attr->list().shape_size(); ++i) { + auto* shape = shapes_attr->mutable_list()->mutable_shape(i); + if (shape->unknown_rank()) continue; + shape->mutable_dim(0)->set_size(-1); + } + + return Status::OK(); +} + +Status ShardByData(const NodeDef& sink_node, int64 num_workers, int64 index, + int64 num_replicas, MutableGraphView* graph) { + // Sharding by data only works with legacy RebatchDataset. As such, we rewrite + // all instances of RebatchDatasetV2 to RebatchDataset. + TF_RETURN_IF_ERROR(RewriteRebatchV2ToV1(sink_node, num_replicas, graph)); + return AddShardNode(graph, sink_node, num_workers, index); +} + Status OptimizeGraph(const GrapplerItem& item, int64 num_workers, int64 index, - AutoShardPolicy policy, GraphDef* output) { + AutoShardPolicy policy, int64 num_replicas, + GraphDef* output) { if (policy == AutoShardPolicy::OFF || (num_workers == 1 && index == 0)) { return Status::OK(); } @@ -482,49 +560,32 @@ Status OptimizeGraph(const GrapplerItem& item, int64 num_workers, int64 index, MutableGraphView graph(output); FunctionLibraryDefinition flib(OpRegistry::Global(), item.graph.library()); - absl::flat_hash_set nodes_to_delete; NodeDef* sink_node; TF_RETURN_IF_ERROR(graph_utils::GetFetchNode(graph, item, &sink_node)); - // The basic approach here is to walk the graph from sink to source, and find - // the latest occurrence of a ReaderDataset (e.g. CSVDataset, TFRecordDataset, - // etc...). We then add a shard after that dataset to shard the outputs of - // that dataset, in effect giving a piece to each worker. Finally, we remove - // occurrences from randomness from before that point in the graph (e.g. - // things like ShuffleDataset) to ensure that `shard` returns a sensible - // result. switch (policy) { case AutoShardPolicy::OFF: return Status::OK(); case AutoShardPolicy::FILE: - TF_RETURN_IF_ERROR(RecursivelyHandleOp(*sink_node, num_workers, index, - &flib, &graph, &nodes_to_delete)); - return graph.DeleteNodes(nodes_to_delete); - break; + return ShardByFile(*sink_node, num_workers, index, &flib, &graph); case AutoShardPolicy::DATA: - return AddShardNode(&graph, *sink_node, num_workers, index); - break; + return ShardByData(*sink_node, num_workers, index, num_replicas, &graph); case AutoShardPolicy::AUTO: default: - Status s = RecursivelyHandleOp(*sink_node, num_workers, index, &flib, - &graph, &nodes_to_delete); - if (!s.ok() && errors::IsNotFound(s)) { + Status s = ShardByFile(*sink_node, num_workers, index, &flib, &graph); + if (errors::IsNotFound(s)) { LOG(WARNING) << "In AUTO-mode, and switching to DATA-based sharding, " "instead of FILE-based sharding as we cannot find " "appropriate reader dataset op(s) to shard. Error: " << s.error_message(); - TF_RETURN_IF_ERROR( - AddShardNode(&graph, *sink_node, num_workers, index)); - } else if (!s.ok()) { - return s; + return ShardByData(*sink_node, num_workers, index, num_replicas, + &graph); } - - return graph.DeleteNodes(nodes_to_delete); - break; + return s; } } @@ -548,6 +609,7 @@ Status AutoShard::Init( index_ = config->parameter_map().at(kIndexAttrName).i(); auto_shard_policy_ = AutoShardPolicy(config->parameter_map().at(kAutoShardPolicyAttrName).i()); + num_replicas_ = config->parameter_map().at(kNumReplicasAttrName).i(); if (auto_shard_policy_ != AutoShardPolicy::OFF && auto_shard_policy_ != AutoShardPolicy::AUTO && @@ -566,6 +628,10 @@ Status AutoShard::Init( num_workers_, ", currently ", index_); } + if (num_replicas_ < 0) { + return errors::InvalidArgument(kNumReplicasAttrName, " should be >= 0"); + } + return Status::OK(); } @@ -574,8 +640,8 @@ Status AutoShard::OptimizeAndCollectStats(Cluster* /* cluster */, GraphDef* output, OptimizationStats* stats) { *output = item.graph; - TF_RETURN_IF_ERROR( - OptimizeGraph(item, num_workers_, index_, auto_shard_policy_, output)); + TF_RETURN_IF_ERROR(OptimizeGraph(item, num_workers_, index_, + auto_shard_policy_, num_replicas_, output)); stats->num_changes++; return Status::OK(); } diff --git a/tensorflow/core/grappler/optimizers/data/auto_shard.h b/tensorflow/core/grappler/optimizers/data/auto_shard.h index 0c73582890f..edb953ba48e 100644 --- a/tensorflow/core/grappler/optimizers/data/auto_shard.h +++ b/tensorflow/core/grappler/optimizers/data/auto_shard.h @@ -48,6 +48,7 @@ class AutoShard : public TFDataOptimizerBase { private: int64 num_workers_; + int64 num_replicas_; int64 index_; AutoShardPolicy auto_shard_policy_; }; diff --git a/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.cc b/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.cc index 821314740a2..343d6f36616 100644 --- a/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.cc @@ -33,14 +33,17 @@ constexpr char kOptimizerName[] = "tf_auto_shard"; AutoShardDatasetOp::AutoShardDatasetOp(OpKernelConstruction* ctx) : UnaryDatasetOpKernel(ctx), auto_shard_policy_(0) { - if (ctx->HasAttr("auto_shard_policy")) { - OP_REQUIRES_OK(ctx, ctx->GetAttr("auto_shard_policy", &auto_shard_policy_)); + if (ctx->HasAttr(kAutoShardPolicy)) { + OP_REQUIRES_OK(ctx, ctx->GetAttr(kAutoShardPolicy, &auto_shard_policy_)); + } + if (ctx->HasAttr(kNumReplicas)) { + OP_REQUIRES_OK(ctx, ctx->GetAttr(kNumReplicas, &num_replicas_)); } } void AutoShardDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input, DatasetBase** output) { - int64 index, num_workers, auto_shard_policy; + int64 index, num_workers, auto_shard_policy, num_replicas; OP_REQUIRES_OK(ctx, ParseScalarArgument(ctx, kNumWorkers, &num_workers)); OP_REQUIRES( ctx, num_workers > 0, @@ -51,9 +54,11 @@ void AutoShardDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input, ctx, index >= 0 && index < num_workers, errors::InvalidArgument("index must be between 0 and ", num_workers - 1)); auto_shard_policy = auto_shard_policy_; + num_replicas = num_replicas_; - auto config_factory = [num_workers, index, auto_shard_policy]() { - return CreateConfig(num_workers, index, auto_shard_policy); + auto config_factory = [num_workers, index, auto_shard_policy, + num_replicas]() { + return CreateConfig(num_workers, index, auto_shard_policy, num_replicas); }; // We only want to optimize functions for some particular datasets like @@ -65,7 +70,8 @@ void AutoShardDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input, } RewriterConfig AutoShardDatasetOp::CreateConfig(int64 num_workers, int64 index, - int64 auto_shard_policy) { + int64 auto_shard_policy, + int64 num_replicas) { RewriterConfig rewriter_config; rewriter_config.set_fail_on_optimizer_errors(true); rewriter_config.set_meta_optimizer_iterations(RewriterConfig::ONE); @@ -73,16 +79,18 @@ RewriterConfig AutoShardDatasetOp::CreateConfig(int64 num_workers, int64 index, rewriter_config.add_optimizers(kOptimizerName); auto custom_optimizer = rewriter_config.add_custom_optimizers(); custom_optimizer->set_name(kOptimizerName); - AttrValue num_workers_attr; - num_workers_attr.set_i(num_workers); - (*custom_optimizer->mutable_parameter_map())[kNumWorkers] = num_workers_attr; - AttrValue index_attr; - index_attr.set_i(index); - (*custom_optimizer->mutable_parameter_map())[kIndex] = index_attr; - AttrValue auto_shard_policy_attr; - auto_shard_policy_attr.set_i(auto_shard_policy); - (*custom_optimizer->mutable_parameter_map())[kAutoShardPolicy] = - auto_shard_policy_attr; + + const std::array, 4> attr_pairs = { + {{kNumWorkers, num_workers}, + {kIndex, index}, + {kAutoShardPolicy, auto_shard_policy}, + {kNumReplicas, num_replicas}}}; + + for (const auto& pair : attr_pairs) { + AttrValue attr; + attr.set_i(pair.second); + (*custom_optimizer->mutable_parameter_map())[pair.first] = attr; + } return rewriter_config; } diff --git a/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.h b/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.h index 174ab32de56..32afca491c6 100644 --- a/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.h +++ b/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.h @@ -28,6 +28,7 @@ class AutoShardDatasetOp : public UnaryDatasetOpKernel { static constexpr const char* const kNumWorkers = "num_workers"; static constexpr const char* const kIndex = "index"; static constexpr const char* const kAutoShardPolicy = "auto_shard_policy"; + static constexpr const char* const kNumReplicas = "num_replicas"; static constexpr const char* const kOutputTypes = "output_types"; static constexpr const char* const kOutputShapes = "output_shapes"; @@ -39,8 +40,10 @@ class AutoShardDatasetOp : public UnaryDatasetOpKernel { private: static RewriterConfig CreateConfig(int64 num_workers, int64 index, - int64 auto_shard_policy); + int64 auto_shard_policy, + int64 num_replicas); int64 auto_shard_policy_; + int64 num_replicas_; }; } // namespace experimental diff --git a/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op_test.cc b/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op_test.cc index 3c5c5188f19..727c6ba51fe 100644 --- a/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op_test.cc +++ b/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op_test.cc @@ -25,12 +25,14 @@ class AutoShardDatasetParams : public DatasetParams { public: template AutoShardDatasetParams(T input_dataset_params, int64 num_workers, int64 index, - int auto_shard_policy, DataTypeVector output_dtypes, + int auto_shard_policy, int64 num_replicas, + DataTypeVector output_dtypes, std::vector output_shapes, string node_name) : DatasetParams(std::move(output_dtypes), std::move(output_shapes), std::move(node_name)), num_workers_(num_workers), + num_replicas_(num_replicas), index_(index), auto_shard_policy_(auto_shard_policy) { input_dataset_params_.push_back(absl::make_unique(input_dataset_params)); @@ -55,6 +57,7 @@ class AutoShardDatasetParams : public DatasetParams { attr_vector->clear(); attr_vector->emplace_back(AutoShardDatasetOp::kAutoShardPolicy, auto_shard_policy_); + attr_vector->emplace_back(AutoShardDatasetOp::kNumReplicas, num_replicas_); attr_vector->emplace_back(AutoShardDatasetOp::kOutputTypes, output_dtypes_); attr_vector->emplace_back(AutoShardDatasetOp::kOutputShapes, output_shapes_); @@ -67,6 +70,7 @@ class AutoShardDatasetParams : public DatasetParams { private: int64 num_workers_; + int64 num_replicas_; int64 index_; int auto_shard_policy_; }; @@ -79,6 +83,7 @@ AutoShardDatasetParams AutoShardDatasetParams1() { /*num_workers=*/5, /*index=*/2, /*auto_shard_policy=*/0, + /*num_replicas=*/5, /*output_dtypes=*/{DT_INT64}, /*output_shapes=*/{PartialTensorShape({})}, /*node_name=*/kNodeName); @@ -90,6 +95,7 @@ AutoShardDatasetParams AutoShardDatasetParams2() { /*num_workers=*/5, /*index=*/2, /*auto_shard_policy=*/0, + /*num_replicas=*/5, /*output_dtypes=*/{DT_INT64}, /*output_shapes=*/{PartialTensorShape({})}, /*node_name=*/kNodeName); @@ -102,6 +108,7 @@ AutoShardDatasetParams AutoShardDatasetParams3() { /*num_workers=*/4, /*index=*/3, /*auto_shard_policy=*/0, + /*num_replicas=*/4, /*output_dtypes=*/{DT_INT64}, /*output_shapes=*/{PartialTensorShape({})}, /*node_name=*/kNodeName); @@ -116,6 +123,7 @@ AutoShardDatasetParams AutoShardDatasetParams4() { /*num_workers=*/5, /*index=*/7, /*auto_shard_policy=*/0, + /*num_replicas=*/5, /*output_dtypes=*/{DT_INT64}, /*output_shapes=*/{PartialTensorShape({})}, /*node_name=*/kNodeName); @@ -127,6 +135,7 @@ AutoShardDatasetParams AutoShardDatasetParams5() { /*num_workers=*/5, /*index=*/-3, /*auto_shard_policy=*/0, + /*num_replicas=*/5, /*output_dtypes=*/{DT_INT64}, /*output_shapes=*/{PartialTensorShape({})}, /*node_name=*/kNodeName); @@ -138,6 +147,7 @@ AutoShardDatasetParams AutoShardDatasetParams6() { /*num_workers=*/-3, /*index=*/1, /*auto_shard_policy=*/0, + /*num_replicas=*/5, /*output_dtypes=*/{DT_INT64}, /*output_shapes=*/{PartialTensorShape({})}, /*node_name=*/kNodeName); @@ -149,6 +159,7 @@ AutoShardDatasetParams AutoShardDatasetParams7() { /*num_workers=*/0, /*index=*/1, /*auto_shard_policy=*/0, + /*num_replicas=*/5, /*output_dtypes=*/{DT_INT64}, /*output_shapes=*/{PartialTensorShape({})}, /*node_name=*/kNodeName); diff --git a/tensorflow/core/ops/experimental_dataset_ops.cc b/tensorflow/core/ops/experimental_dataset_ops.cc index 443db5e9173..85cc7dc7e70 100644 --- a/tensorflow/core/ops/experimental_dataset_ops.cc +++ b/tensorflow/core/ops/experimental_dataset_ops.cc @@ -64,6 +64,7 @@ REGISTER_OP("AutoShardDataset") .Attr("auto_shard_policy: int = 0") .Attr("output_types: list(type) >= 1") .Attr("output_shapes: list(shape) >= 1") + .Attr("num_replicas: int = 0") .SetShapeFn(shape_inference::ScalarShape); REGISTER_OP("ExperimentalAutoShardDataset") diff --git a/tensorflow/python/data/experimental/kernel_tests/auto_shard_dataset_test.py b/tensorflow/python/data/experimental/kernel_tests/auto_shard_dataset_test.py index 36587d97ea0..1b1a72af8d3 100644 --- a/tensorflow/python/data/experimental/kernel_tests/auto_shard_dataset_test.py +++ b/tensorflow/python/data/experimental/kernel_tests/auto_shard_dataset_test.py @@ -393,32 +393,6 @@ class AutoShardDatasetTest(reader_dataset_ops_test_base.TFRecordDatasetTestBase, dataset = distribute._AutoShardDataset(dataset, 10, 0) self.evaluate(self.getNext(dataset)()) - @combinations.generate(test_base.default_test_combinations()) - def testShardWithLegacyRebatch(self): - # Tests that RebatchDatasetV1 is a passthrough op. - dataset = dataset_ops.Dataset.list_files(self.test_filenames, shuffle=False) - dataset = dataset.apply( - testing.assert_next(["Shard", "FlatMap", "Batch", "Rebatch"])) - dataset = dataset.flat_map(core_readers.TFRecordDataset) - dataset = dataset.batch(5) - dataset = distribute._LegacyRebatchDataset(dataset, num_replicas=1) - dataset = distribute._AutoShardDataset(dataset, 5, 3) - nxt = self.getNext(dataset) - self.evaluate(nxt()) - - @combinations.generate(test_base.default_test_combinations()) - def testShardWithRebatch(self): - # Tests that RebatchDatasetV2 is a passthrough op. - dataset = dataset_ops.Dataset.list_files(self.test_filenames, shuffle=False) - dataset = dataset.apply( - testing.assert_next(["Shard", "FlatMap", "Batch", "Rebatch"])) - dataset = dataset.flat_map(core_readers.TFRecordDataset) - dataset = dataset.batch(5) - dataset = distribute._RebatchDataset(dataset, batch_sizes=5) - dataset = distribute._AutoShardDataset(dataset, 5, 3) - nxt = self.getNext(dataset) - self.evaluate(nxt()) - @combinations.generate(test_base.default_test_combinations()) def testNoReaderPipelines(self): dataset = dataset_ops.Dataset.range(1024) @@ -529,5 +503,83 @@ class AutoShardTextLineDatasetTest( self.assertDatasetProduces(dataset, expected) +class AutoShardWithRebatchDatasetTest( + reader_dataset_ops_test_base.TFRecordDatasetTestBase, + parameterized.TestCase): + + def _setUpFiles(self, num_files, num_records_per_file): + self._num_files = num_files + self._num_records = num_records_per_file + self.test_filenames = self._createFiles() + + @combinations.generate(test_base.default_test_combinations()) + def testFileShardingWithLegacyRebatch(self): + # Tests that RebatchDatasetV1 is a passthrough op. + self._setUpFiles(num_files=5, num_records_per_file=10) + dataset = dataset_ops.Dataset.list_files(self.test_filenames, shuffle=False) + dataset = dataset.apply( + testing.assert_next(["Shard", "FlatMap", "Batch", "Rebatch"])) + dataset = dataset.flat_map(core_readers.TFRecordDataset) + dataset = dataset.batch(5) + dataset = distribute._LegacyRebatchDataset(dataset, num_replicas=5) + dataset = distribute._AutoShardDataset(dataset, 5, 3) + expected = [[self._record(3, i)] for i in range(10)] + self.assertDatasetProduces(dataset, expected) + + @combinations.generate(test_base.default_test_combinations()) + def testFileShardingWithRebatch(self): + # Tests that RebatchDatasetV2 is a passthrough op. + self._setUpFiles(num_files=3, num_records_per_file=5) + dataset = dataset_ops.Dataset.list_files(self.test_filenames, shuffle=False) + dataset = dataset.apply( + testing.assert_next(["Shard", "FlatMap", "Batch", "Rebatch"])) + dataset = dataset.flat_map(core_readers.TFRecordDataset) + dataset = dataset.batch(5) + dataset = distribute._RebatchDataset(dataset, batch_sizes=[2, 1, 2]) + dataset = distribute._AutoShardDataset(dataset, 3, 1) + expected = [[self._record(1, 0), self._record(1, 1)], [self._record(1, 2)], + [self._record(1, 3), self._record(1, 4)]] + self.assertDatasetProduces(dataset, expected) + + @combinations.generate( + combinations.times( + test_base.default_test_combinations(), + combinations.combine(sharding_policy=[ + distribute_options.AutoShardPolicy.DATA, + distribute_options.AutoShardPolicy.AUTO + ]))) + def testUseLegacyRebatchWithDataSharding(self, sharding_policy): + # This test simulates a distributed environment with 3 workers, each with + # 1 replica. + dataset = dataset_ops.Dataset.range(8) + dataset = dataset.batch(4) + options = dataset_ops.Options() + options.experimental_distribute.auto_shard_policy = sharding_policy + dataset = dataset.with_options(options) + # We expect the auto-shard rewrite to rewrite RebatchDatasetV2 to + # RebatchDataset(V1) for correctness reasons. This will modify the output + # of the dataset. + worker_a_dataset = distribute._RebatchDataset( + dataset, batch_sizes=[2, 1, 1]) + worker_a_dataset = distribute._AutoShardDataset( + worker_a_dataset, 3, 0, num_replicas=3) + expected = [[0, 1], [4, 5]] + self.assertDatasetProduces(worker_a_dataset, expected) + + worker_b_dataset = distribute._RebatchDataset( + dataset, batch_sizes=[1, 1, 2]) + worker_b_dataset = distribute._AutoShardDataset( + worker_b_dataset, 3, 1, num_replicas=3) + expected = [[2, 3], [6, 7]] + self.assertDatasetProduces(worker_b_dataset, expected) + + worker_c_dataset = distribute._RebatchDataset( + dataset, batch_sizes=[1, 2, 1]) + worker_c_dataset = distribute._AutoShardDataset( + worker_c_dataset, 3, 2, num_replicas=3) + expected = [[], []] + self.assertDatasetProduces(worker_c_dataset, expected) + + if __name__ == "__main__": test.main() diff --git a/tensorflow/python/data/experimental/ops/distribute.py b/tensorflow/python/data/experimental/ops/distribute.py index c5a9048630c..5105f30fd07 100644 --- a/tensorflow/python/data/experimental/ops/distribute.py +++ b/tensorflow/python/data/experimental/ops/distribute.py @@ -35,22 +35,38 @@ class _AutoShardDataset(dataset_ops.UnaryDataset): """A `Dataset` that shards the `Dataset` automatically. This dataset takes in an existing dataset and tries to automatically figure - out how to shard the dataset in a multi-worker scenario. Currently, it uses - Grappler to walk up the dataset graph until it finds a reader dataset (e.g. - CSVDataset, TFRecordDataset), then inserts a ShardDataset op before that node + out how to shard the dataset in a multi-worker scenario using graph rewrites. + + If the AutoShardPolicy is set to FILE, it walks up the dataset graph until + it finds a reader dataset, then inserts a ShardDataset op before that node so that each worker only sees some files. + If the AutoShardPolicy is set to DATA, it inserts a ShardDataset op at the + end of the input pipeline, before any terminal PrefetchDataset if there is + one. Additionally, if there is a RebatchDatasetV2 in the input pipeline, it + is written to legacy RebatchDataset for correctness reasons, since + RebatchDatasetV2 is incompatible with data sharding. + + If the AutoShardPolicy is set to AUTO, it tries to do file-based sharding. + If it cannot find a reader dataset, it falls back to doing data-based + sharding. + + If the AutoShardPolicy is set to OFF, it does nothing. + Args: num_workers: Total number of workers to shard this dataset across. index: The current worker index (out of the total number of workers) this dataset is for. + num_replicas: The total number of replicas across all workers. This is used + only when sharding by data (either DATA or AUTO) in order to rewrite + RebatchDatasetV2 to RebatchDataset. Raises: NotFoundError: If we cannot find a suitable reader dataset to begin automatically sharding the dataset. """ - def __init__(self, input_dataset, num_workers, index): + def __init__(self, input_dataset, num_workers, index, num_replicas=None): self._input_dataset = input_dataset self._element_spec = input_dataset.element_spec @@ -60,6 +76,7 @@ class _AutoShardDataset(dataset_ops.UnaryDataset): index=index, auto_shard_policy=int( input_dataset.options().experimental_distribute.auto_shard_policy), + num_replicas=num_replicas, **self._flat_structure) super(_AutoShardDataset, self).__init__(input_dataset, variant_tensor) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt index 0842adb478f..81ad89bf2ff 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.raw_ops.pbtxt @@ -266,7 +266,7 @@ tf_module { } member_method { name: "AutoShardDataset" - argspec: "args=[\'input_dataset\', \'num_workers\', \'index\', \'output_types\', \'output_shapes\', \'auto_shard_policy\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], " + argspec: "args=[\'input_dataset\', \'num_workers\', \'index\', \'output_types\', \'output_shapes\', \'auto_shard_policy\', \'num_replicas\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'0\', \'None\'], " } member_method { name: "AvgPool" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt index 0842adb478f..81ad89bf2ff 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.raw_ops.pbtxt @@ -266,7 +266,7 @@ tf_module { } member_method { name: "AutoShardDataset" - argspec: "args=[\'input_dataset\', \'num_workers\', \'index\', \'output_types\', \'output_shapes\', \'auto_shard_policy\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'None\'], " + argspec: "args=[\'input_dataset\', \'num_workers\', \'index\', \'output_types\', \'output_shapes\', \'auto_shard_policy\', \'num_replicas\', \'name\'], varargs=None, keywords=None, defaults=[\'0\', \'0\', \'None\'], " } member_method { name: "AvgPool" From f28b07dcfa597bb19d649e1975f0a2080fa8d457 Mon Sep 17 00:00:00 2001 From: Jay Shi Date: Wed, 12 Aug 2020 14:26:10 -0700 Subject: [PATCH 0925/1017] [tf.data] Add `OptimizeDatasetOp` to most input pipelines. Enable graph rewrites only when there are optimizations to be applied to the pipelines. PiperOrigin-RevId: 326312484 Change-Id: Ifdf1274db7fd8f07bb6c1815152f6f5938797de2 --- .../core/kernels/data/optimize_dataset_op.cc | 7 +++++++ tensorflow/python/data/ops/dataset_ops.py | 15 ++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tensorflow/core/kernels/data/optimize_dataset_op.cc b/tensorflow/core/kernels/data/optimize_dataset_op.cc index f1fa96d9ac3..f151ad5cdab 100644 --- a/tensorflow/core/kernels/data/optimize_dataset_op.cc +++ b/tensorflow/core/kernels/data/optimize_dataset_op.cc @@ -106,6 +106,13 @@ void OptimizeDatasetOp::MakeDataset(OpKernelContext* ctx, DatasetBase* input, } } + // If there are no optimizations to be applied, directly return the input. + if (optimizations.empty()) { + *output = input; + input->Ref(); + return; + } + auto config_factory = [this, &optimizations]() { return CreateConfig(optimizations, optimization_configs_); }; diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py index 19d9fe9d88b..270c65d0743 100644 --- a/tensorflow/python/data/ops/dataset_ops.py +++ b/tensorflow/python/data/ops/dataset_ops.py @@ -377,19 +377,20 @@ class DatasetV2(collections_abc.Iterable, tracking_base.Trackable, graph_rewrites = options._graph_rewrites() graph_rewrite_configs = options._graph_rewrite_configs() # pylint: enable=protected-access - if graph_rewrites.enabled or graph_rewrites.default: - if self._has_captured_ref(): + if self._has_captured_ref(): + if graph_rewrites.enabled or graph_rewrites.default: warnings.warn( "tf.data graph rewrites are not compatible with tf.Variable. " "The following rewrites will be disabled: %s. To enable " "rewrites, use resource variables instead by calling " "`tf.enable_resource_variables()` at the start of the program." % ", ".join(graph_rewrites.enabled + graph_rewrites.default)) - else: - dataset = _OptimizeDataset(dataset, graph_rewrites.enabled, - graph_rewrites.disabled, - graph_rewrites.default, - graph_rewrite_configs) + elif (graph_rewrites.enabled or graph_rewrites.default or + (options.experimental_optimization.apply_default_optimizations # pylint: disable=g-bool-id-comparison + is not False)): + dataset = _OptimizeDataset(dataset, graph_rewrites.enabled, + graph_rewrites.disabled, + graph_rewrites.default, graph_rewrite_configs) # (3) Apply autotune options autotune, algorithm, cpu_budget = options._autotune_settings() # pylint: disable=protected-access From 0e529eb70debc7224b83d1b654f22324cd1a7d53 Mon Sep 17 00:00:00 2001 From: Thomas O'Malley Date: Wed, 12 Aug 2020 14:34:48 -0700 Subject: [PATCH 0926/1017] Add loss and gradient transformation hooks to Optimizer. Switches Model.fit to use Optimizer.minimize. Also rolls forward previous changes to allow clipvalue and clipnorm to be used with tf.distribute PiperOrigin-RevId: 326314410 Change-Id: I2753a5d938c4fb0a5a50826d5f1d7b8664aa8847 --- RELEASE.md | 4 + .../distribute/distribute_strategy_test.py | 64 +++++ tensorflow/python/keras/engine/training.py | 68 +----- .../python/keras/engine/training_eager.py | 1 - .../experimental/loss_scale_optimizer.py | 12 +- tensorflow/python/keras/optimizer_v2/BUILD | 1 + .../python/keras/optimizer_v2/optimizer_v2.py | 219 +++++++++++------- tensorflow/python/keras/optimizer_v2/utils.py | 50 +++- tensorflow/python/keras/optimizers.py | 26 ++- ...n.experimental.-loss-scale-optimizer.pbtxt | 8 + ...ensorflow.keras.optimizers.-adadelta.pbtxt | 8 + ...tensorflow.keras.optimizers.-adagrad.pbtxt | 8 + .../tensorflow.keras.optimizers.-adam.pbtxt | 8 + .../tensorflow.keras.optimizers.-adamax.pbtxt | 8 + .../tensorflow.keras.optimizers.-ftrl.pbtxt | 8 + .../tensorflow.keras.optimizers.-nadam.pbtxt | 8 + ...nsorflow.keras.optimizers.-optimizer.pbtxt | 10 +- ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt | 8 + .../tensorflow.keras.optimizers.-s-g-d.pbtxt | 8 + ...n.experimental.-loss-scale-optimizer.pbtxt | 8 + ...ensorflow.keras.optimizers.-adadelta.pbtxt | 8 + ...tensorflow.keras.optimizers.-adagrad.pbtxt | 8 + .../tensorflow.keras.optimizers.-adam.pbtxt | 8 + .../tensorflow.keras.optimizers.-adamax.pbtxt | 8 + .../tensorflow.keras.optimizers.-ftrl.pbtxt | 8 + .../tensorflow.keras.optimizers.-nadam.pbtxt | 8 + ...nsorflow.keras.optimizers.-optimizer.pbtxt | 10 +- ...nsorflow.keras.optimizers.-r-m-sprop.pbtxt | 8 + .../tensorflow.keras.optimizers.-s-g-d.pbtxt | 8 + .../v2/tensorflow.optimizers.-adadelta.pbtxt | 8 + .../v2/tensorflow.optimizers.-adagrad.pbtxt | 8 + .../v2/tensorflow.optimizers.-adam.pbtxt | 8 + .../v2/tensorflow.optimizers.-adamax.pbtxt | 8 + .../v2/tensorflow.optimizers.-ftrl.pbtxt | 8 + .../v2/tensorflow.optimizers.-nadam.pbtxt | 8 + .../v2/tensorflow.optimizers.-optimizer.pbtxt | 10 +- .../v2/tensorflow.optimizers.-r-m-sprop.pbtxt | 8 + .../v2/tensorflow.optimizers.-s-g-d.pbtxt | 8 + 38 files changed, 517 insertions(+), 166 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 2f1a890e60b..d4b5b27630e 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -108,6 +108,10 @@ as an alternative to accepting a `callable` loss. * Added `beta` parameter to FTRL optimizer to match paper. * Added `mobilenet_v3` to keras application model. + * `Optimizer.__init__` now accepts a `gradient_aggregator` to allow for + customization of how gradients are aggregated across devices, as well as + `gradients_transformers` to allow for custom gradient transformations + (such as gradient clipping). * `tf.function` / AutoGraph: * Added `experimental_follow_type_hints` argument for `tf.function`. When True, the function may use type annotations to optimize the tracing diff --git a/tensorflow/python/keras/distribute/distribute_strategy_test.py b/tensorflow/python/keras/distribute/distribute_strategy_test.py index 4b6d3a80730..4ea53429195 100644 --- a/tensorflow/python/keras/distribute/distribute_strategy_test.py +++ b/tensorflow/python/keras/distribute/distribute_strategy_test.py @@ -22,6 +22,7 @@ import numpy as np from tensorflow.python import keras from tensorflow.python.data.experimental.ops import cardinality from tensorflow.python.data.ops import dataset_ops +from tensorflow.python.distribute import central_storage_strategy from tensorflow.python.distribute import combinations from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import mirrored_strategy @@ -1863,6 +1864,69 @@ class TestDistributionStrategyWithKerasModels(test.TestCase, self.assertEqual(bc.predict_begin_batches, [0]) self.assertEqual(bc.predict_end_batches, [24]) + @combinations.generate( + combinations.combine(distribution=all_strategies, mode=['eager'])) + def test_gradient_clipping(self, distribution): + + class MyLayer(keras.layers.Layer): + + def build(self, _): + self.v1 = variables.Variable(1.) + self.v2 = variables.Variable(1.) + + def call(self, x): + return 3 * self.v1 - 3 * self.v2 + + x, y = np.ones((10, 1)), np.ones((10, 1)) + + with distribution.scope(): + layer = MyLayer() + model = keras.Sequential([layer]) + optimizer = gradient_descent_keras.SGD(1., clipnorm=2., clipvalue=2.) + model.compile(optimizer, 'mae') + + if isinstance(distribution, + central_storage_strategy.CentralStorageStrategy): + with self.assertRaisesRegex(ValueError, 'not supported'): + model.fit(x, y, batch_size=10, epochs=1) + else: + model.fit(x, y, batch_size=10, epochs=1) + self.assertAllClose(self.evaluate(layer.v1), 3.) + self.assertAllClose(self.evaluate(layer.v2), -1.) + + @combinations.generate( + combinations.combine(distribution=all_strategies, mode=['eager'])) + def test_custom_gradient_transformation(self, distribution): + if isinstance(distribution, + central_storage_strategy.CentralStorageStrategy): + self.skipTest('Not supported with `CentralStorageStrategy`') + + class MyLayer(keras.layers.Layer): + + def build(self, _): + self.v1 = variables.Variable(1.) + self.v2 = variables.Variable(-1.) + + def call(self, x): + return x + self.v1 + self.v2 + + def custom_transform(grads_and_vars): + # Always set gradients to 1. + return [(array_ops.ones_like(g), v) for g, v in grads_and_vars] + + x, y = np.ones((10, 1)), np.ones((10, 1)) + + with distribution.scope(): + layer = MyLayer() + model = keras.Sequential([layer]) + optimizer = gradient_descent_keras.SGD( + 1., gradient_transformers=[custom_transform]) + model.compile(optimizer, 'mae') + + model.fit(x, y, batch_size=10, epochs=1) + self.assertAllClose(self.evaluate(layer.v1), 0.) + self.assertAllClose(self.evaluate(layer.v2), -2.) + @combinations.generate( combinations.times( all_strategy_combinations_minus_default())) diff --git a/tensorflow/python/keras/engine/training.py b/tensorflow/python/keras/engine/training.py index 5f91fcef231..6f479655f30 100644 --- a/tensorflow/python/keras/engine/training.py +++ b/tensorflow/python/keras/engine/training.py @@ -26,7 +26,6 @@ import six from tensorflow.python.autograph.lang import directives from tensorflow.python.distribute import distribution_strategy_context as ds_context -from tensorflow.python.distribute import parameter_server_strategy from tensorflow.python.distribute import values as ds_values from tensorflow.python.eager import backprop from tensorflow.python.eager import context @@ -719,8 +718,7 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): """ # These are the only transformations `Model.fit` applies to user-input - # data when a `tf.data.Dataset` is provided. These utilities will be exposed - # publicly. + # data when a `tf.data.Dataset` is provided. data = data_adapter.expand_1d(data) x, y, sample_weight = data_adapter.unpack_x_y_sample_weight(data) @@ -728,15 +726,7 @@ class Model(base_layer.Layer, version_utils.ModelVersionSelector): y_pred = self(x, training=True) loss = self.compiled_loss( y, y_pred, sample_weight, regularization_losses=self.losses) - # For custom training steps, users can just write: - # trainable_variables = self.trainable_variables - # gradients = tape.gradient(loss, trainable_variables) - # self.optimizer.apply_gradients(zip(gradients, trainable_variables)) - # The _minimize call does a few extra steps unnecessary in most cases, - # such as loss scaling and gradient clipping. - _minimize(self.distribute_strategy, tape, self.optimizer, loss, - self.trainable_variables) - + self.optimizer.minimize(loss, self.trainable_variables, tape=tape) self.compiled_metrics.update_state(y, y_pred, sample_weight) return {m.name: m.result() for m in self.metrics} @@ -2676,60 +2666,6 @@ def _tpu_multi_host_concat(v, strategy): return concat(ordered_replicas) -def _minimize(strategy, tape, optimizer, loss, trainable_variables): - """Minimizes loss for one step by updating `trainable_variables`. - - This is roughly equivalent to - - ```python - gradients = tape.gradient(loss, trainable_variables) - self.optimizer.apply_gradients(zip(gradients, trainable_variables)) - ``` - - However, this function also applies gradient clipping and loss scaling if the - optimizer is a LossScaleOptimizer. - - Args: - strategy: `tf.distribute.Strategy`. - tape: A gradient tape. The loss must have been computed under this tape. - optimizer: The optimizer used to minimize the loss. - loss: The loss tensor. - trainable_variables: The variables that will be updated in order to minimize - the loss. - """ - - with tape: - if isinstance(optimizer, lso.LossScaleOptimizer): - loss = optimizer.get_scaled_loss(loss) - - gradients = tape.gradient(loss, trainable_variables) - - # Whether to aggregate gradients outside of optimizer. This requires support - # of the optimizer and doesn't work with ParameterServerStrategy and - # CentralStorageStrategy. - aggregate_grads_outside_optimizer = ( - optimizer._HAS_AGGREGATE_GRAD and # pylint: disable=protected-access - not isinstance(strategy.extended, - parameter_server_strategy.ParameterServerStrategyExtended)) - - if aggregate_grads_outside_optimizer: - # We aggregate gradients before unscaling them, in case a subclass of - # LossScaleOptimizer all-reduces in fp16. All-reducing in fp16 can only be - # done on scaled gradients, not unscaled gradients, for numeric stability. - gradients = optimizer._aggregate_gradients(zip(gradients, # pylint: disable=protected-access - trainable_variables)) - if isinstance(optimizer, lso.LossScaleOptimizer): - gradients = optimizer.get_unscaled_gradients(gradients) - gradients = optimizer._clip_gradients(gradients) # pylint: disable=protected-access - if trainable_variables: - if aggregate_grads_outside_optimizer: - optimizer.apply_gradients( - zip(gradients, trainable_variables), - experimental_aggregate_gradients=False) - else: - optimizer.apply_gradients(zip(gradients, trainable_variables)) - - def _is_scalar(x): return isinstance(x, (ops.Tensor, variables.Variable)) and x.shape.rank == 0 diff --git a/tensorflow/python/keras/engine/training_eager.py b/tensorflow/python/keras/engine/training_eager.py index 8064bf2a7ab..b3ce3d13ed7 100644 --- a/tensorflow/python/keras/engine/training_eager.py +++ b/tensorflow/python/keras/engine/training_eager.py @@ -273,7 +273,6 @@ def _process_single_batch(model, if isinstance(model.optimizer, loss_scale_optimizer.LossScaleOptimizer): grads = model.optimizer.get_unscaled_gradients(grads) - grads = model.optimizer._clip_gradients(grads) model.optimizer.apply_gradients(zip(grads, trainable_weights)) else: logging.warning('The list of trainable weights is empty. Make sure that' diff --git a/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py b/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py index 4a3f459de80..69b39e3f989 100644 --- a/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py +++ b/tensorflow/python/keras/mixed_precision/experimental/loss_scale_optimizer.py @@ -22,6 +22,7 @@ from tensorflow.python.distribute import distribution_strategy_context from tensorflow.python.distribute import mirrored_strategy from tensorflow.python.distribute import one_device_strategy from tensorflow.python.distribute import tpu_strategy +from tensorflow.python.eager import backprop from tensorflow.python.framework import dtypes from tensorflow.python.framework import ops from tensorflow.python.framework import smart_cond @@ -349,9 +350,14 @@ class LossScaleOptimizer(_DelegatingTrackableMixin, optimizer_v2.OptimizerV2): ] def _compute_gradients(self, loss, var_list, grad_loss=None, tape=None): - loss = self.get_scaled_loss(loss) - grads_and_vars = self._optimizer._compute_gradients(loss, var_list, # pylint: disable=protected-access - grad_loss) + tape = backprop.GradientTape() if tape is None else tape + with tape: + loss = self.get_scaled_loss(loss) + grads_and_vars = self._optimizer._compute_gradients( # pylint: disable=protected-access + loss, + var_list, + grad_loss, + tape=tape) grads = [g for g, _ in grads_and_vars] variables = [v for _, v in grads_and_vars] unscaled_grads = self.get_unscaled_gradients(grads) diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD index b519ec7fb3d..9a317e5d114 100644 --- a/tensorflow/python/keras/optimizer_v2/BUILD +++ b/tensorflow/python/keras/optimizer_v2/BUILD @@ -40,6 +40,7 @@ py_library( "//tensorflow/python:state_ops", "//tensorflow/python:variable_scope", "//tensorflow/python:variables", + "//tensorflow/python/distribute:central_storage_strategy", "//tensorflow/python/distribute:distribute_lib", "//tensorflow/python/distribute:parameter_server_strategy", "//tensorflow/python/distribute:reduce_util", diff --git a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py index 18d94594542..c533b2c40c1 100644 --- a/tensorflow/python/keras/optimizer_v2/optimizer_v2.py +++ b/tensorflow/python/keras/optimizer_v2/optimizer_v2.py @@ -41,7 +41,6 @@ from tensorflow.python.keras.optimizer_v2 import utils as optimizer_utils from tensorflow.python.keras.utils import generic_utils from tensorflow.python.keras.utils import tf_utils from tensorflow.python.ops import array_ops -from tensorflow.python.ops import clip_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import gen_resource_variable_ops from tensorflow.python.ops import gradients @@ -278,7 +277,11 @@ class OptimizerV2(trackable.Trackable): # Note: This attribute will likely be removed in an upcoming release. _HAS_AGGREGATE_GRAD = False - def __init__(self, name, **kwargs): + def __init__(self, + name, + gradient_aggregator=None, + gradient_transformers=None, + **kwargs): """Create a new Optimizer. This must be called by the constructors of subclasses. @@ -287,11 +290,30 @@ class OptimizerV2(trackable.Trackable): you should be able to use the _set_hyper()/state.get_hyper() facility instead. - This class in stateful and thread-compatible. + This class is stateful and thread-compatible. + + Example of custom gradient transformations: + + ```python + def my_gradient_transformer(grads_and_vars): + # Simple example, double the gradients. + return [(2. * g, v) for g, v in grads_and_vars] + + optimizer = tf.keras.optimizers.SGD( + 1e-3, gradient_transformers=[my_gradient_transformer]) + ``` Args: name: A non-empty string. The name to use for accumulators created for the optimizer. + gradient_aggregator: The function to use to aggregate gradients across + devices (when using `tf.distribute.Strategy`). If `None`, defaults to + summing the gradients across devices. The function should accept and + return a list of `(gradient, variable)` tuples. + gradient_transformers: (Optional). List of functions to use to transform + gradients before applying updates to `Variable`s. The functions are + applied after `gradient_aggregator`. The functions should accept and + return a list of `(gradient, variable)` tuples. **kwargs: keyword arguments. Allowed to be {`clipnorm`, `clipvalue`, `lr`, `decay`}. `clipnorm` is clip gradients by norm; `clipvalue` is clip gradients by value, `decay` is included for backward compatibility to @@ -332,17 +354,7 @@ class OptimizerV2(trackable.Trackable): raise ValueError("decay cannot be less than 0: {}".format(decay)) self._initial_decay = decay - # Set the gradient clipping properties - self.clipnorm = kwargs.pop("clipnorm", None) - self.clipvalue = kwargs.pop("clipvalue", None) - if ((self.clipnorm is not None or self.clipvalue is not None) - and distribute_ctx.has_strategy()): - raise ValueError("Gradient clipping in the optimizer " - "(by setting clipnorm or clipvalue) is currently " - "unsupported when using a distribution strategy.") - self._hypers_created = False - # Store the distribution strategy object if the optimizer is created inside # strategy scope, so it could be used to create variables later. if distribute_ctx.has_strategy(): @@ -350,6 +362,74 @@ class OptimizerV2(trackable.Trackable): else: self._distribution_strategy = None + # Configure gradient transformations. + if gradient_aggregator is None: + gradient_aggregator = optimizer_utils.all_reduce_sum_gradients + self.gradient_aggregator = gradient_aggregator + if gradient_transformers is None: + gradient_transformers = [] + self.gradient_transformers = gradient_transformers + self.clipnorm = kwargs.pop("clipnorm", None) + self.clipvalue = kwargs.pop("clipvalue", None) + + @property + def clipnorm(self): + """`float` or `None`. If set, clips gradients to a maximum norm.""" + return self._clipnorm + + @clipnorm.setter + def clipnorm(self, val): + if val is not None and self.gradient_transformers: + raise ValueError("`clipnorm` cannot be set when `gradient_transformers` " + "is set. Instead, use the `gradient_transformers` to " + "specify clipping and other transformations.") + self._clipnorm = val + self._clipnorm_fn = optimizer_utils.make_gradient_clipnorm_fn( + self._clipnorm) + + @property + def clipvalue(self): + """`float` or `None`. If set, clips gradients to a maximum value.""" + return self._clipvalue + + @clipvalue.setter + def clipvalue(self, val): + if val is not None and self.gradient_transformers: + raise ValueError("`clipvalue` cannot be set when `gradient_transformers` " + "is set. Instead, use the `gradient_transformers` to " + "specify clipping and other transformations.") + self._clipvalue = val + self._clipvalue_fn = optimizer_utils.make_gradient_clipvalue_fn( + self._clipvalue) + + def _transform_loss(self, loss): + """Called in `.minimize` to transform loss before computing gradients.""" + return loss + + def _get_gradients(self, tape, loss, var_list, grad_loss=None): + """Called in `minimize` to compute gradients from loss.""" + grads = tape.gradient(loss, var_list, grad_loss) + return list(zip(grads, var_list)) + + def _transform_unaggregated_gradients(self, grads_and_vars): + """Called in `apply_gradients` before gradient aggregation.""" + return grads_and_vars + + def _aggregate_gradients(self, grads_and_vars): + """Called in `apply_gradients` to aggregate gradients across devices.""" + return self.gradient_aggregator(grads_and_vars) + + def _transform_gradients(self, grads_and_vars): + """Called in `apply_gradients` after aggregation.""" + if self._clipvalue is not None: + grads_and_vars = self._clipvalue_fn(grads_and_vars) + if self._clipnorm is not None: + grads_and_vars = self._clipnorm_fn(grads_and_vars) + + for fn in self.gradient_transformers: + grads_and_vars = fn(grads_and_vars) + return grads_and_vars + def minimize(self, loss, var_list, grad_loss=None, name=None, tape=None): """Minimize `loss` by updating `var_list`. @@ -385,26 +465,6 @@ class OptimizerV2(trackable.Trackable): loss, var_list=var_list, grad_loss=grad_loss, tape=tape) return self.apply_gradients(grads_and_vars, name=name) - def _clip_gradients(self, grads): - """Clip gradients according to the clipnorm and clipvalue attributes.""" - if self.clipnorm is not None: - if distribute_ctx.has_strategy(): - raise ValueError("Gradient clipping in the optimizer " - "(by setting clipnorm or clipvalue) is currently " - "unsupported when using a distribution strategy.") - grads = [None if g is None else clip_ops.clip_by_norm(g, self.clipnorm) - for g in grads] - if self.clipvalue is not None: - if distribute_ctx.has_strategy(): - raise ValueError("Gradient clipping in the optimizer " - "(by setting clipnorm or clipvalue) is currently " - "unsupported when using a distribution strategy.") - v = self.clipvalue - grads = [ - None if g is None else clip_ops.clip_by_value(g, -v, v) for g in grads - ] - return grads - def _compute_gradients(self, loss, var_list, grad_loss=None, tape=None): """Compute gradients of `loss` for the variables in `var_list`. @@ -444,19 +504,16 @@ class OptimizerV2(trackable.Trackable): with tape: if not callable(var_list): tape.watch(var_list) - - if callable(loss): - loss = loss() - + loss = loss() if callable(var_list): var_list = var_list() + with tape: + loss = self._transform_loss(loss) + var_list = nest.flatten(var_list) with ops.name_scope_v2(self._name + "/gradients"): - grads = tape.gradient(loss, var_list, grad_loss) - # TODO(omalleyt): Move to post-aggregation. - grads = self._clip_gradients(grads) - grads_and_vars = list(zip(grads, var_list)) + grads_and_vars = self._get_gradients(tape, loss, var_list, grad_loss) self._assert_valid_dtypes([ v for g, v in grads_and_vars @@ -465,34 +522,6 @@ class OptimizerV2(trackable.Trackable): return grads_and_vars - def get_gradients(self, loss, params): - """Returns gradients of `loss` with respect to `params`. - - Arguments: - loss: Loss tensor. - params: List of variables. - - Returns: - List of gradient tensors. - - Raises: - ValueError: In case any gradient cannot be computed (e.g. if gradient - function not implemented). - """ - params = nest.flatten(params) - with backend.get_graph().as_default(), backend.name_scope(self._name + - "/gradients"): - grads = gradients.gradients(loss, params) - for grad, param in zip(grads, params): - if grad is None: - raise ValueError("Variable {} has `None` for gradient. " - "Please make sure that all of your ops have a " - "gradient defined (i.e. are differentiable). " - "Common ops without gradient: " - "K.argmax, K.round, K.eval.".format(param)) - grads = self._clip_gradients(grads) - return grads - def apply_gradients(self, grads_and_vars, name=None, @@ -537,7 +566,7 @@ class OptimizerV2(trackable.Trackable): grads_and_vars = optimizer_utils.filter_empty_gradients(grads_and_vars) var_list = [v for (_, v) in grads_and_vars] - with backend.name_scope(self._name): + with ops.name_scope_v2(self._name): # Create iteration if necessary. with ops.init_scope(): self._create_all_weights(var_list) @@ -563,9 +592,10 @@ class OptimizerV2(trackable.Trackable): apply_state = self._prepare(var_list) if experimental_aggregate_gradients: - reduced_grads = self._aggregate_gradients(grads_and_vars) - var_list = [v for _, v in grads_and_vars] - grads_and_vars = list(zip(reduced_grads, var_list)) + grads_and_vars = self._transform_unaggregated_gradients(grads_and_vars) + grads_and_vars = self._aggregate_gradients(grads_and_vars) + grads_and_vars = self._transform_gradients(grads_and_vars) + return distribute_ctx.get_replica_context().merge_call( functools.partial(self._distributed_apply, apply_state=apply_state), args=(grads_and_vars,), @@ -573,20 +603,6 @@ class OptimizerV2(trackable.Trackable): "name": name, }) - def _aggregate_gradients(self, grads_and_vars): - """Returns aggregated gradients. - - This method must be preserved to maintain backwards compatibility with - Horovod aggregation. - - Args: - grads_and_vars: List of (gradient, variable) pairs. - - Returns: - A list of all-reduced gradients. - """ - return optimizer_utils.all_reduce_sum_gradients(grads_and_vars) - def _distributed_apply(self, distribution, grads_and_vars, name, apply_state): """`apply_gradients` using a `DistributionStrategy`.""" @@ -646,6 +662,35 @@ class OptimizerV2(trackable.Trackable): return self._iterations.assign_add(1) + def get_gradients(self, loss, params): + """Returns gradients of `loss` with respect to `params`. + + Should be used only in legacy v1 graph mode. + + Arguments: + loss: Loss tensor. + params: List of variables. + + Returns: + List of gradient tensors. + + Raises: + ValueError: In case any gradient cannot be computed (e.g. if gradient + function not implemented). + """ + params = nest.flatten(params) + with backend.get_graph().as_default(), backend.name_scope(self._name + + "/gradients"): + grads = gradients.gradients(loss, params) + for grad, param in zip(grads, params): + if grad is None: + raise ValueError("Variable {} has `None` for gradient. " + "Please make sure that all of your ops have a " + "gradient defined (i.e. are differentiable). " + "Common ops without gradient: " + "K.argmax, K.round, K.eval.".format(param)) + return grads + def get_updates(self, loss, params): grads = self.get_gradients(loss, params) grads_and_vars = list(zip(grads, params)) diff --git a/tensorflow/python/keras/optimizer_v2/utils.py b/tensorflow/python/keras/optimizer_v2/utils.py index 9f680e04dd6..909a25d4b15 100644 --- a/tensorflow/python/keras/optimizer_v2/utils.py +++ b/tensorflow/python/keras/optimizer_v2/utils.py @@ -18,8 +18,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.distribute import central_storage_strategy from tensorflow.python.distribute import distribution_strategy_context as distribute_ctx from tensorflow.python.distribute import reduce_util as ds_reduce_util +from tensorflow.python.ops import clip_ops from tensorflow.python.platform import tf_logging as logging @@ -30,7 +32,7 @@ def all_reduce_sum_gradients(grads_and_vars): grads_and_vars: List of (gradient, variable) pairs. Returns: - A list of all-reduced gradients. + List of (gradient, variable) pairs where gradients have been all-reduced. """ grads_and_vars = list(grads_and_vars) filtered_grads_and_vars = filter_empty_gradients(grads_and_vars) @@ -47,11 +49,11 @@ def all_reduce_sum_gradients(grads_and_vars): # Copy 'reduced' but add None gradients back in reduced_with_nones = [] reduced_pos = 0 - for g, _ in grads_and_vars: + for g, v in grads_and_vars: if g is None: - reduced_with_nones.append(None) + reduced_with_nones.append((None, v)) else: - reduced_with_nones.append(reduced[reduced_pos]) + reduced_with_nones.append((reduced[reduced_pos], v)) reduced_pos += 1 assert reduced_pos == len(reduced), "Failed to add all gradients" return reduced_with_nones @@ -82,6 +84,46 @@ def filter_empty_gradients(grads_and_vars): return filtered +def make_gradient_clipnorm_fn(clipnorm): + """Creates a gradient transformation function for clipping by norm.""" + if clipnorm is None: + return lambda grads_and_vars: grads_and_vars + + def gradient_clipnorm_fn(grads_and_vars): + + if isinstance(distribute_ctx.get_strategy(), + central_storage_strategy.CentralStorageStrategy): + raise ValueError( + "`clipnorm` is not supported with `CenteralStorageStrategy`") + + clipped_grads_and_vars = [ + (clip_ops.clip_by_norm(g, clipnorm), v) for g, v in grads_and_vars + ] + return clipped_grads_and_vars + + return gradient_clipnorm_fn + + +def make_gradient_clipvalue_fn(clipvalue): + """Creates a gradient transformation function for clipping by value.""" + if clipvalue is None: + return lambda grads_and_vars: grads_and_vars + + def gradient_clipvalue_fn(grads_and_vars): + + if isinstance(distribute_ctx.get_strategy(), + central_storage_strategy.CentralStorageStrategy): + raise ValueError( + "`clipvalue` is not supported with `CenteralStorageStrategy`") + + clipped_grads_and_vars = [(clip_ops.clip_by_value(g, -clipvalue, + clipvalue), v) + for g, v in grads_and_vars] + return clipped_grads_and_vars + + return gradient_clipvalue_fn + + def _all_reduce_sum_fn(distribution, grads_and_vars): return distribution.extended.batch_reduce_to(ds_reduce_util.ReduceOp.SUM, grads_and_vars) diff --git a/tensorflow/python/keras/optimizers.py b/tensorflow/python/keras/optimizers.py index 5d8e1351ae3..75d1e10242a 100644 --- a/tensorflow/python/keras/optimizers.py +++ b/tensorflow/python/keras/optimizers.py @@ -22,6 +22,7 @@ import six from six.moves import zip # pylint: disable=redefined-builtin from tensorflow.python.distribute import distribution_strategy_context +from tensorflow.python.eager import backprop from tensorflow.python.framework import ops from tensorflow.python.keras import backend as K from tensorflow.python.keras.optimizer_v2 import adadelta as adadelta_v2 @@ -41,6 +42,7 @@ from tensorflow.python.ops import state_ops from tensorflow.python.training import optimizer as tf_optimizer_module from tensorflow.python.training import training_util from tensorflow.python.training.tracking import base as trackable +from tensorflow.python.util import nest from tensorflow.python.util.tf_export import keras_export @@ -771,8 +773,28 @@ class TFOptimizer(Optimizer, trackable.Trackable): # TFOptimizer wrapper has no gradient clipping options. return grads - def apply_gradients(self, grads): - self.optimizer.apply_gradients(grads, global_step=self.iterations) + def minimize(self, loss, var_list, grad_loss=None, tape=None): + """Mimics the `OptimizerV2.minimize` API.""" + if not callable(loss) and tape is None: + raise ValueError('`tape` is required when a `Tensor` loss is passed.') + tape = tape if tape is not None else backprop.GradientTape() + + if callable(loss): + with tape: + if not callable(var_list): + tape.watch(var_list) + loss = loss() + if callable(var_list): + var_list = var_list() + + var_list = nest.flatten(var_list) + if var_list: + grads = tape.gradient(loss, var_list, grad_loss) + grads_and_vars = list(zip(grads, var_list)) + self.apply_gradients(grads_and_vars) + + def apply_gradients(self, grads_and_vars): + self.optimizer.apply_gradients(grads_and_vars, global_step=self.iterations) def get_grads(self, loss, params): return self.optimizer.compute_gradients(loss, params) diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt index dbab3abae8e..58f8cf24495 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt @@ -5,6 +5,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt index af854e98013..fb341cb24dd 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adadelta.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt index e89cc5cef75..d8039ed21ef 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adagrad.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt index 15414d7234f..912f92f83a6 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt index 8b3c429e6b5..3abc6d39b3f 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-adamax.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt index 51ab675db74..00880d3f73b 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-ftrl.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt index 342c0951bbe..2ce311d3504 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-nadam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt index f007b4b971a..4855395e020 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-optimizer.pbtxt @@ -3,6 +3,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" @@ -13,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'name\'], varargs=None, keywords=kwargs, defaults=None" + argspec: "args=[\'self\', \'name\', \'gradient_aggregator\', \'gradient_transformers\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\'], " } member_method { name: "add_slot" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt index d5bf6fa7f47..80a1449613c 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-r-m-sprop.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt index df904f72511..8acfe214256 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.optimizers.-s-g-d.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt index dbab3abae8e..58f8cf24495 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.mixed_precision.experimental.-loss-scale-optimizer.pbtxt @@ -5,6 +5,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt index af854e98013..fb341cb24dd 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adadelta.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt index e89cc5cef75..d8039ed21ef 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adagrad.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt index 15414d7234f..912f92f83a6 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt index 8b3c429e6b5..3abc6d39b3f 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-adamax.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt index 51ab675db74..00880d3f73b 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-ftrl.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt index 342c0951bbe..2ce311d3504 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-nadam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt index f007b4b971a..4855395e020 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-optimizer.pbtxt @@ -3,6 +3,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" @@ -13,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'name\'], varargs=None, keywords=kwargs, defaults=None" + argspec: "args=[\'self\', \'name\', \'gradient_aggregator\', \'gradient_transformers\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\'], " } member_method { name: "add_slot" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt index d5bf6fa7f47..80a1449613c 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-r-m-sprop.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt index df904f72511..8acfe214256 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.optimizers.-s-g-d.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt index cb3d38246a7..06212bdc95d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adadelta.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt index c7b2bca4b6b..09fff0514d8 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adagrad.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt index 209c9fe6620..195ba9e4f56 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt index 12bbb14fb71..9859da430bd 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-adamax.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt index 1482ed54eb9..a4ed911e39d 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-ftrl.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt index 2a422fa2340..128f223fdc7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-nadam.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt index e7021e02772..5633beb0916 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-optimizer.pbtxt @@ -3,6 +3,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" @@ -13,7 +21,7 @@ tf_class { } member_method { name: "__init__" - argspec: "args=[\'self\', \'name\'], varargs=None, keywords=kwargs, defaults=None" + argspec: "args=[\'self\', \'name\', \'gradient_aggregator\', \'gradient_transformers\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\'], " } member_method { name: "add_slot" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt index 6543f4023a4..db89ecbabe7 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-r-m-sprop.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt index 94ff8dfcdfc..0cb0205e65e 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.optimizers.-s-g-d.pbtxt @@ -4,6 +4,14 @@ tf_class { is_instance: "" is_instance: "" is_instance: "" + member { + name: "clipnorm" + mtype: "" + } + member { + name: "clipvalue" + mtype: "" + } member { name: "iterations" mtype: "" From c870489b518e64e585e0c8ce40e030bd44d7be91 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 14:51:05 -0700 Subject: [PATCH 0927/1017] grpc_impl namespace is gRPC internal and shouldn't be used by any application. Recommendation is to use grpc namespace. This CL makes the change. PiperOrigin-RevId: 326317917 Change-Id: Ia915db210cdb84f9218561190893ed75f074186c --- tensorflow/core/data/service/data_service.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow/core/data/service/data_service.cc b/tensorflow/core/data/service/data_service.cc index be09b10c1fc..31449f6f5ec 100644 --- a/tensorflow/core/data/service/data_service.cc +++ b/tensorflow/core/data/service/data_service.cc @@ -117,7 +117,7 @@ Status DataServiceDispatcherClient::GetTasks(int64 job_id, GetTasksRequest req; req.set_job_id(job_id); GetTasksResponse resp; - grpc_impl::ClientContext ctx; + grpc::ClientContext ctx; grpc::Status s = stub_->GetTasks(&ctx, req, &resp); if (!s.ok()) { return grpc_util::WrapError("Failed to get tasks", s); @@ -135,7 +135,7 @@ Status DataServiceDispatcherClient::GetWorkers( TF_RETURN_IF_ERROR(EnsureInitialized()); GetWorkersRequest req; GetWorkersResponse resp; - grpc_impl::ClientContext ctx; + grpc::ClientContext ctx; grpc::Status s = stub_->GetWorkers(&ctx, req, &resp); if (!s.ok()) { return grpc_util::WrapError("Failed to get workers", s); @@ -163,7 +163,7 @@ Status DataServiceWorkerClient::GetElement(int64 task_id, GetElementRequest req; req.set_task_id(task_id); GetElementResponse resp; - grpc_impl::ClientContext ctx; + grpc::ClientContext ctx; grpc::Status s = stub_->GetElement(&ctx, req, &resp); if (!s.ok()) { return grpc_util::WrapError("Failed to get element", s); From cdc891452ad428c0e78a8b127ace77a3a4fc03c2 Mon Sep 17 00:00:00 2001 From: Robert David Date: Wed, 12 Aug 2020 14:52:02 -0700 Subject: [PATCH 0928/1017] Change ModifyGraphWithDelegate to a template so it accepts unique_ptrs with any deleter type. PiperOrigin-RevId: 326318121 Change-Id: I92c4ad6937f3e7123cf5dde30165d4e888086bf3 --- tensorflow/lite/interpreter.cc | 7 ------- tensorflow/lite/interpreter.h | 30 +++++++++++++++++++++++++---- tensorflow/lite/interpreter_test.cc | 6 +----- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/tensorflow/lite/interpreter.cc b/tensorflow/lite/interpreter.cc index 1d702dd8397..4f81824d96f 100644 --- a/tensorflow/lite/interpreter.cc +++ b/tensorflow/lite/interpreter.cc @@ -376,13 +376,6 @@ TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) { return status; } -TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegatePtr delegate) { - // Note that we retain ownership of the delegate even if graph modification - // fails, as delegate use will be in an indeterminate state at that point. - owned_delegates_.push_back(std::move(delegate)); - return ModifyGraphWithDelegate(owned_delegates_.back().get()); -} - TfLiteStatus Interpreter::RemoveAllDelegates() { for (auto& subgraph : subgraphs_) { TF_LITE_ENSURE_STATUS(subgraph->RemoveAllDelegates()); diff --git a/tensorflow/lite/interpreter.h b/tensorflow/lite/interpreter.h index 5be48b793fa..d4bf3016810 100644 --- a/tensorflow/lite/interpreter.h +++ b/tensorflow/lite/interpreter.h @@ -21,6 +21,7 @@ limitations under the License. #include #include #include +#include #include #include @@ -422,10 +423,29 @@ class Interpreter { std::unique_ptr; /// Same as ModifyGraphWithDelegate except this interpreter takes - /// ownership of the provided delegate. Be sure to construct the unique_ptr - /// with a suitable destruction function. + /// ownership of the provided delegate. /// WARNING: This is an experimental API and subject to change. - TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegatePtr delegate); + template + inline TfLiteStatus ModifyGraphWithDelegate( + std::unique_ptr delegate) { + Deleter deleter = std::move(delegate.get_deleter()); + + // Note that we retain ownership of the delegate even if graph modification + // fails, as delegate use will be in an indeterminate state at that point. + owned_delegates_.emplace_back( + delegate.release(), [deleter](TfLiteDelegate* delegate_to_delete) { + deleter( + static_cast::pointer>( + delegate_to_delete)); + }); + return ModifyGraphWithDelegate(owned_delegates_.back().get()); + } + + /// This overload is *never* OK. TfLiteDelegate is a C structure, so it has no + /// virtual destructor. The default deleter of the unique_ptr does not know + /// how to delete C++ objects deriving from TfLiteDelegate. + TfLiteStatus ModifyGraphWithDelegate( + std::unique_ptr delegate) = delete; /// Ensure the data in `tensor.data` is readable. In case delegate is used, /// it might require to copy the data from delegate buffer to raw memory. @@ -604,7 +624,9 @@ class Interpreter { // interpreter instance. Useful if client delegate ownership is burdensome. // WARNING: This is an experimental API and subject to change. // TODO(b/116667551): Use TfLiteExternalContext for storing state. - std::vector owned_delegates_; + std::vector< + std::unique_ptr>> + owned_delegates_; // Profiler that has been installed and is owned by this interpreter instance. // Useful if client profiler ownership is burdensome. diff --git a/tensorflow/lite/interpreter_test.cc b/tensorflow/lite/interpreter_test.cc index 067f2c54124..a35799a8408 100644 --- a/tensorflow/lite/interpreter_test.cc +++ b/tensorflow/lite/interpreter_test.cc @@ -43,11 +43,7 @@ class InterpreterTest : public ::testing::Test { template static TfLiteStatus ModifyGraphWithDelegate( Interpreter* interpreter, std::unique_ptr delegate) { - Interpreter::TfLiteDelegatePtr tflite_delegate( - delegate.release(), [](TfLiteDelegate* delegate) { - delete reinterpret_cast(delegate); - }); - return interpreter->ModifyGraphWithDelegate(std::move(tflite_delegate)); + return interpreter->ModifyGraphWithDelegate(std::move(delegate)); } protected: From 9ec80ae8bd8e3b1753cda19973977578c7525dbf Mon Sep 17 00:00:00 2001 From: Dan Moldovan Date: Wed, 12 Aug 2020 14:58:17 -0700 Subject: [PATCH 0929/1017] Set the origin info more accurately on certain lines generated during control flow translation. This ensures the stack trace is correctly translated when errors occur in the generated code for the respective statements. PiperOrigin-RevId: 326319490 Change-Id: I1e421b1718eab537fddcc175d43e7565e00a0405 --- .../python/autograph/converters/control_flow.py | 14 +++++++++++--- tensorflow/python/autograph/pyct/origin_info.py | 12 ++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/tensorflow/python/autograph/converters/control_flow.py b/tensorflow/python/autograph/converters/control_flow.py index b54770cbd28..c3fc879ded5 100644 --- a/tensorflow/python/autograph/converters/control_flow.py +++ b/tensorflow/python/autograph/converters/control_flow.py @@ -24,6 +24,7 @@ from tensorflow.python.autograph.core import converter from tensorflow.python.autograph.lang import directives from tensorflow.python.autograph.pyct import anno from tensorflow.python.autograph.pyct import cfg +from tensorflow.python.autograph.pyct import origin_info from tensorflow.python.autograph.pyct import parser from tensorflow.python.autograph.pyct import qual_names from tensorflow.python.autograph.pyct import templates @@ -243,7 +244,7 @@ class ControlFlowTransformer(converter.Base): (symbol_names,), nouts) """ - return templates.replace( + new_nodes = templates.replace( template, body=node.body, body_name=self.ctx.namer.new_symbol('if_body', reserved), @@ -257,6 +258,8 @@ class ControlFlowTransformer(converter.Base): symbol_names=tuple(gast.Constant(str(s), kind=None) for s in cond_vars), test=node.test, undefined_assigns=undefined_assigns) + origin_info.copy_origin(node, new_nodes[-1]) + return new_nodes def visit_While(self, node): node = self.generic_visit(node) @@ -292,7 +295,7 @@ class ControlFlowTransformer(converter.Base): (symbol_names,), opts) """ - return templates.replace( + new_nodes = templates.replace( template, body=node.body, body_name=self.ctx.namer.new_symbol('loop_body', reserved), @@ -305,6 +308,8 @@ class ControlFlowTransformer(converter.Base): test=node.test, test_name=self.ctx.namer.new_symbol('loop_test', reserved), undefined_assigns=undefined_assigns) + origin_info.copy_origin(node, new_nodes[-1]) + return new_nodes def visit_For(self, node): node = self.generic_visit(node) @@ -356,6 +361,7 @@ class ControlFlowTransformer(converter.Base): """ iterate_expansion = templates.replace( template, iterate_arg_name=iterate_arg_name, iterates=node.target) + origin_info.copy_origin(node, iterate_expansion) template = """ state_functions @@ -374,7 +380,7 @@ class ControlFlowTransformer(converter.Base): (symbol_names,), opts) """ - return templates.replace( + new_nodes = templates.replace( template, body=node.body, body_name=self.ctx.namer.new_symbol('loop_body', reserved), @@ -390,6 +396,8 @@ class ControlFlowTransformer(converter.Base): state_getter_name=state_getter_name, state_setter_name=state_setter_name, undefined_assigns=undefined_assigns) + origin_info.copy_origin(node, new_nodes[-1]) + return new_nodes class AnnotatedDef(reaching_definitions.Definition): diff --git a/tensorflow/python/autograph/pyct/origin_info.py b/tensorflow/python/autograph/pyct/origin_info.py index ba25d96d2d6..cd909e16364 100644 --- a/tensorflow/python/autograph/pyct/origin_info.py +++ b/tensorflow/python/autograph/pyct/origin_info.py @@ -279,3 +279,15 @@ def resolve_entity(node, source, entity): col_offset = len(definition_line) - len(definition_line.lstrip()) resolve(node, source, filepath, lineno, col_offset) + + +def copy_origin(from_node, to_node): + """Copies the origin info from a node to another, recursively.""" + origin = anno.Basic.ORIGIN.of(from_node, default=None) + if origin is None: + return + if not isinstance(to_node, (list, tuple)): + to_node = (to_node,) + for node in to_node: + for n in gast.walk(node): + anno.setanno(n, anno.Basic.ORIGIN, origin) From 99cfa35b2475e845e7666c748cc3ec842d2a6549 Mon Sep 17 00:00:00 2001 From: George Karpenkov Date: Wed, 12 Aug 2020 15:12:05 -0700 Subject: [PATCH 0930/1017] [TF2XLA] Inject XLAControlFlowContext in ConcreteFunction, as it can be called directly, bypassing Function PiperOrigin-RevId: 326322743 Change-Id: I7df50c5c650b3cd317e360f127ac1b782a14fbaf --- .../python/eager/def_function_xla_jit_test.py | 2 ++ tensorflow/python/eager/function.py | 28 +++++++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/tensorflow/python/eager/def_function_xla_jit_test.py b/tensorflow/python/eager/def_function_xla_jit_test.py index ba75aed5f1c..f1e25c04fb2 100644 --- a/tensorflow/python/eager/def_function_xla_jit_test.py +++ b/tensorflow/python/eager/def_function_xla_jit_test.py @@ -220,6 +220,8 @@ class DefFunctionTest(xla_test.XLATestCase): self.assertAllClose(40.0, f(2.0)) self.assertAllClose([40.0, 28.0], g(2.0)) + self.assertAllClose(40.0, f.get_concrete_function(2.0)(2.0)) + self.assertAllClose([40.0, 28.0], g.get_concrete_function(2.0)(2.0)) def testMethodCompilation(self): diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py index f0004f0aa65..bb4449a3357 100644 --- a/tensorflow/python/eager/function.py +++ b/tensorflow/python/eager/function.py @@ -57,6 +57,7 @@ from tensorflow.python.framework import tensor_spec from tensorflow.python.framework import type_spec from tensorflow.python.ops import array_ops from tensorflow.python.ops import control_flow_ops +from tensorflow.python.ops import control_flow_util from tensorflow.python.ops import custom_gradient from tensorflow.python.ops import default_gradient from tensorflow.python.ops import functional_ops @@ -1939,15 +1940,24 @@ class ConcreteFunction(object): possible_gradient_type, executing_eagerly) forward_function, args_with_tangents = forward_backward.forward() - if executing_eagerly: - flat_outputs = forward_function.call( - ctx, args_with_tangents, - cancellation_manager=cancellation_manager) - else: - with default_graph._override_gradient_function( # pylint: disable=protected-access - {"PartitionedCall": self._get_gradient_function(), - "StatefulPartitionedCall": self._get_gradient_function()}): - flat_outputs = forward_function.call(ctx, args_with_tangents) + compiled_with_xla = self._attrs.get("_XlaMustCompile", False) and \ + not control_flow_util.GraphOrParentsInXlaContext(default_graph) + xla_context = control_flow_ops.XLAControlFlowContext() + try: + if compiled_with_xla: + xla_context.Enter() + if executing_eagerly: + flat_outputs = forward_function.call( + ctx, args_with_tangents, + cancellation_manager=cancellation_manager) + else: + with default_graph._override_gradient_function( # pylint: disable=protected-access + {"PartitionedCall": self._get_gradient_function(), + "StatefulPartitionedCall": self._get_gradient_function()}): + flat_outputs = forward_function.call(ctx, args_with_tangents) + finally: + if compiled_with_xla: + xla_context.Exit() forward_backward.record(flat_outputs) return self._build_call_outputs(flat_outputs) From 7aa06a93292acf92b4829b6df903cc68cb89b28f Mon Sep 17 00:00:00 2001 From: Ken Franko Date: Wed, 12 Aug 2020 15:13:17 -0700 Subject: [PATCH 0931/1017] Only mark unspported ops for outside compilation if allow_soft_placement attribute on cluster is true. PiperOrigin-RevId: 326322975 Change-Id: I888350d66bea7ec953d210a5ac291dbe61a1c690 --- .../mark_ops_for_outside_compilation.mlir | 50 +++++++++++++++---- .../mark_ops_for_outside_compilation.cc | 8 +++ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir b/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir index 2d86889e35b..9544a02dca4 100644 --- a/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir +++ b/tensorflow/compiler/mlir/tensorflow/tests/mark_ops_for_outside_compilation.mlir @@ -1,5 +1,33 @@ // RUN: tf-opt %s -tf-mark-ops-for-outside-compilation | FILECHECK_OPTS="" FileCheck %s +// CHECK-LABEL: func @unsupported_op_no_soft_placement +func @unsupported_op_no_soft_placement() -> tensor { + %0 = "tf_device.cluster"() ( { + // CHECK: "tf.UnsupportedOp" + // CHECK-NOT: _xla_outside_compilation + // CHECK: "tf.Identity" + // CHECK-NOT: _xla_outside_compilation + %1 = "tf.UnsupportedOp"() {value = dense<1> : tensor} : () -> tensor + %2 = "tf.Identity"(%1) : (tensor) -> tensor + tf_device.return %2 : tensor + }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + return %0 : tensor +} + +// CHECK-LABEL: func @unsupported_op_soft_placement_false +func @unsupported_op_soft_placement_false() -> tensor { + %0 = "tf_device.cluster"() ( { + // CHECK: "tf.UnsupportedOp" + // CHECK-NOT: _xla_outside_compilation + // CHECK: "tf.Identity" + // CHECK-NOT: _xla_outside_compilation + %1 = "tf.UnsupportedOp"() {value = dense<1> : tensor} : () -> tensor + %2 = "tf.Identity"(%1) : (tensor) -> tensor + tf_device.return %2 : tensor + }) {allow_soft_placement = false, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + return %0 : tensor +} + // CHECK-LABEL: func @unsupported_op func @unsupported_op() -> tensor { %0 = "tf_device.cluster"() ( { @@ -10,7 +38,7 @@ func @unsupported_op() -> tensor { %1 = "tf.UnsupportedOp"() {value = dense<1> : tensor} : () -> tensor %2 = "tf.Identity"(%1) : (tensor) -> tensor tf_device.return %2 : tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } @@ -28,7 +56,7 @@ func @tf2xla_fallback_op() -> tensor { %3 = "tf.Identity"(%1) : (tensor) -> tensor %4 = "tf.Sinh"(%2) : (tensor) -> tensor tf_device.return %4 : tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } @@ -42,7 +70,7 @@ func @ignore_embedding_ops() -> () { %2:2 = "tf.RecvTPUEmbeddingActivations"() {_tpu_embedding_layer = "call1", config = "\0A\0B\0C\0D"} : () -> (tensor<2x2xf32>, tensor<4x4xf32>) "tf.SendTPUEmbeddingGradients"(%2#0, %2#1) {_tpu_embedding_layer = "call1", config = "\0A\0B\0C\0D", operand_segment_sizes = dense<[2, 0]> : vector<2xi32>} : (tensor<2x2xf32>, tensor<4x4xf32>) -> () tf_device.return - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> () + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> () return } @@ -60,7 +88,7 @@ func @op_string_result() -> tensor { %2 = "tf.Const"() {value = dense<"x"> : tensor} : () -> tensor %3 = "tf.Identity"(%1) : (tensor) -> tensor tf_device.return %3 : tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } // CHECK-LABEL: func @op_string_operand @@ -77,7 +105,7 @@ func @op_string_operand(%arg0: tensor) -> tensor { %2 = "tf.StringToNumber"(%arg0) {out_type = f32} : (tensor) -> tensor %3 = "tf.Identity"(%1) : (tensor) -> tensor tf_device.return %3 : tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } @@ -95,7 +123,7 @@ func @op_string_operand_string_result(%arg0: tensor) -> tensor %2 = "tf.Identity"(%arg0) : (tensor) -> tensor %3 = "tf.Identity"(%1) : (tensor) -> tensor tf_device.return %3 : tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } @@ -119,7 +147,7 @@ func @if_region_captured_string(%arg0: tensor, %arg1: tensor) -> }) {is_stateless = true} : (tensor) -> (tensor) %5 = "tf.Identity"(%2) : (tensor) -> tensor tf_device.return %5 : tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } @@ -147,7 +175,7 @@ func @if_region_string_op(%arg0: tensor, %arg1: tensor) -> tensor) -> (tensor) %6 = "tf.Identity"(%2) : (tensor) -> tensor tf_device.return %6: tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } @@ -187,7 +215,7 @@ func @nested_if_region_string_op(%arg0: tensor, %arg1: tensor) -> ten }) {is_stateless = true} : (tensor) -> (tensor) %9 = "tf.Identity"(%2) : (tensor) -> tensor tf_device.return %9: tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } @@ -218,7 +246,7 @@ func @while_region_captured_string(%arg0: tensor, %arg1: tensor // CHECK-NOT: _xla_outside_compilation %5 = "tf.Identity"(%2#0) : (tensor) -> (tensor) tf_device.return %5 : tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } @@ -252,6 +280,6 @@ func @while_region_unsupported_op(%arg0: tensor, %arg1: tensor) // CHECK-NOT: _xla_outside_compilation %5 = "tf.Identity"(%2#0) : (tensor) -> (tensor) tf_device.return %5 : tensor - }) {num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor + }) {allow_soft_placement = true, num_cores_per_replica = 1, topology = "", device_assignment = []} : () -> tensor return %0 : tensor } diff --git a/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc b/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc index ece26dca416..e538491ae9d 100644 --- a/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc +++ b/tensorflow/compiler/mlir/tensorflow/transforms/mark_ops_for_outside_compilation.cc @@ -33,6 +33,7 @@ namespace TFDevice { namespace { constexpr char kXlaOutsideCompilationAttr[] = "_xla_outside_compilation"; +constexpr char kAllowSoftPlacementAttr[] = "allow_soft_placement"; // This pass marks unsupported ops in a device cluster with // `_xla_outside_compilation` attribute so the operations will run on the host @@ -152,6 +153,13 @@ void MarkOpsForOutsideCompilation::runOnOperation() { AddRewrittenEmbeddingOps(module.getContext(), &supported_ops); auto result = module.walk([&](tf_device::ClusterOp cluster) { + // Only if `allow_soft_placement` attribute is true should we mark ops + // for outside compilation. + auto soft_placement_attr = + cluster.getAttrOfType(kAllowSoftPlacementAttr); + if (!(soft_placement_attr && soft_placement_attr.getValue())) { + return WalkResult::advance(); + } if (failed( MarkUncompilableOps(tf_dialect, &cluster.GetBody(), supported_ops))) return WalkResult::interrupt(); From 896240212691bd8e3fe9bb7ae9ce4b0a650d9f3d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 15:18:28 -0700 Subject: [PATCH 0932/1017] Update ops-related pbtxt files. PiperOrigin-RevId: 326323972 Change-Id: I361868bd692c65f82ee07d8227990b3566d21390 --- .../ops_history_v2/AutoShardDataset.pbtxt | 45 +++++++++++++++++++ .../ExperimentalIgnoreErrorsDataset.pbtxt | 30 +++++++++++++ .../ops_history_v2/IgnoreErrorsDataset.pbtxt | 30 +++++++++++++ tensorflow/core/ops/ops.pbtxt | 21 +++++++++ 4 files changed, 126 insertions(+) diff --git a/tensorflow/core/ops/compat/ops_history_v2/AutoShardDataset.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/AutoShardDataset.pbtxt index 1ccd14b6627..af4ab4c03b5 100644 --- a/tensorflow/core/ops/compat/ops_history_v2/AutoShardDataset.pbtxt +++ b/tensorflow/core/ops/compat/ops_history_v2/AutoShardDataset.pbtxt @@ -36,3 +36,48 @@ op { minimum: 1 } } +op { + name: "AutoShardDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + input_arg { + name: "num_workers" + type: DT_INT64 + } + input_arg { + name: "index" + type: DT_INT64 + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "auto_shard_policy" + type: "int" + default_value { + i: 0 + } + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + attr { + name: "num_replicas" + type: "int" + default_value { + i: 0 + } + } +} diff --git a/tensorflow/core/ops/compat/ops_history_v2/ExperimentalIgnoreErrorsDataset.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/ExperimentalIgnoreErrorsDataset.pbtxt index e334e93046a..c9897f9f8f5 100644 --- a/tensorflow/core/ops/compat/ops_history_v2/ExperimentalIgnoreErrorsDataset.pbtxt +++ b/tensorflow/core/ops/compat/ops_history_v2/ExperimentalIgnoreErrorsDataset.pbtxt @@ -21,3 +21,33 @@ op { minimum: 1 } } +op { + name: "ExperimentalIgnoreErrorsDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + attr { + name: "log_warning" + type: "bool" + default_value { + b: false + } + } +} diff --git a/tensorflow/core/ops/compat/ops_history_v2/IgnoreErrorsDataset.pbtxt b/tensorflow/core/ops/compat/ops_history_v2/IgnoreErrorsDataset.pbtxt index 0670fd69e1b..e6ce7a0c94c 100644 --- a/tensorflow/core/ops/compat/ops_history_v2/IgnoreErrorsDataset.pbtxt +++ b/tensorflow/core/ops/compat/ops_history_v2/IgnoreErrorsDataset.pbtxt @@ -21,3 +21,33 @@ op { minimum: 1 } } +op { + name: "IgnoreErrorsDataset" + input_arg { + name: "input_dataset" + type: DT_VARIANT + } + output_arg { + name: "handle" + type: DT_VARIANT + } + attr { + name: "output_types" + type: "list(type)" + has_minimum: true + minimum: 1 + } + attr { + name: "output_shapes" + type: "list(shape)" + has_minimum: true + minimum: 1 + } + attr { + name: "log_warning" + type: "bool" + default_value { + b: false + } + } +} diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt index 7610619019d..f7b68c8c2a4 100644 --- a/tensorflow/core/ops/ops.pbtxt +++ b/tensorflow/core/ops/ops.pbtxt @@ -2835,6 +2835,13 @@ op { has_minimum: true minimum: 1 } + attr { + name: "num_replicas" + type: "int" + default_value { + i: 0 + } + } } op { name: "AvgPool" @@ -14735,6 +14742,13 @@ op { has_minimum: true minimum: 1 } + attr { + name: "log_warning" + type: "bool" + default_value { + b: false + } + } } op { name: "ExperimentalIteratorGetDevice" @@ -19020,6 +19034,13 @@ op { has_minimum: true minimum: 1 } + attr { + name: "log_warning" + type: "bool" + default_value { + b: false + } + } } op { name: "Imag" From 8c8f7b480e3f7741cdd58aa757e7b520ec8fc7da Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Wed, 12 Aug 2020 15:26:53 -0700 Subject: [PATCH 0933/1017] Parallel device: simplify saving to look like N separate variables Also bypasses recent infra changes which mean saving needs to be trace-safe There is some danger that we'll want eventually to run save ops on the parallel device, and that making that checkpoint-compatible with this will be hard. I think it's doable; essentially we'd be including device_id() in SaveSpec.name. Running save ops on the parallel device needs some changes to saving infra and/or some special-casing of ops in the parallel device. We'd basically want to pass device_id() to the ShardedFilename op, and for its output to get transposed (parallel scalar -> vector) when passed to MergeV2. I doubt people replicating many ways will want to save every buffer, so probably not worth optimizing right now. PiperOrigin-RevId: 326325670 Change-Id: I4c031ec0f3fc1e57f465e7aa3107f685e85e24f2 --- .../parallel_device/parallel_device_test.py | 4 - .../distribute/parallel_device/saving.py | 100 ++++++------------ 2 files changed, 31 insertions(+), 73 deletions(-) diff --git a/tensorflow/python/distribute/parallel_device/parallel_device_test.py b/tensorflow/python/distribute/parallel_device/parallel_device_test.py index 39c0d5ba5d8..e660086adef 100644 --- a/tensorflow/python/distribute/parallel_device/parallel_device_test.py +++ b/tensorflow/python/distribute/parallel_device/parallel_device_test.py @@ -206,8 +206,6 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): uses_parallel() def test_checkpointing(self): - self.skipTest( - "Disable saving until SaveableObject's methods are traceable.") prefix = os.path.join(self.get_temp_dir(), "ckpt") with self.device: different_values = self.device.pack( @@ -335,8 +333,6 @@ class LayerTests(_VirtualDeviceTestCase): self.assertIn(self.device.components[1], final_kernels[1].backing_device) def test_training_loop(self): - self.skipTest( - "Disable saving until SaveableObject's methods are traceable.") for _ in range(5): layer = _Dense(5) checkpoint = tracking.Checkpoint(layer=layer) diff --git a/tensorflow/python/distribute/parallel_device/saving.py b/tensorflow/python/distribute/parallel_device/saving.py index f2e7dadae41..ddc0aa51578 100644 --- a/tensorflow/python/distribute/parallel_device/saving.py +++ b/tensorflow/python/distribute/parallel_device/saving.py @@ -21,96 +21,58 @@ from __future__ import print_function import contextlib import functools -from tensorflow.python.framework import ops -from tensorflow.python.ops import array_ops from tensorflow.python.ops import gen_resource_variable_ops from tensorflow.python.ops import resource_variable_ops from tensorflow.python.ops import variable_scope -from tensorflow.python.ops import variables from tensorflow.python.training.saving import saveable_object -def _read_component(handle, dtype, replica_id, parallel_device): - """Read one component of a parallel variable and discard the rest.""" - with ops.device(handle.device): - read = gen_resource_variable_ops.read_variable_op( - resource=handle, dtype=dtype) - all_components = parallel_device.unpack(read) - # We're pretending that parallel variables have a first axis with length - # num_components, so we need to add a dummy first axis to the shape that gets - # saved. - return all_components[replica_id][None, ...] +class _ParallelComponentSaveable(saveable_object.SaveableObject): + """Saves and restores one component of a parallel variable.""" - -class _ParallelDeviceSaveable(saveable_object.SaveableObject): - """Saves and restores a parallel variable.""" - - def __init__(self, name, handle, dtype, component_shape, parallel_device): - # Each component device gets one spec with a tensor to save. - specs = [] - for replica_id, device_name in enumerate(parallel_device.components): - # TODO(b/151773535): SaveableObjects with SaveSpecs on different devices - # will cause extra copying at the moment. We should fix that before doing - # anything serious with this code. - specs.append( - saveable_object.SaveSpec( - tensor=functools.partial( - _read_component, - handle=handle, - dtype=dtype, - replica_id=replica_id, - parallel_device=parallel_device), - slice_spec=variables.Variable.SaveSliceInfo( - full_shape=([len(parallel_device.components)] + - component_shape), - var_offset=[replica_id] + [0] * len(component_shape), - var_shape=[1] + component_shape).spec, - device=device_name, - dtype=dtype, - name=name)) + def __init__(self, name, handle, dtype, shape): + specs = [saveable_object.SaveSpec( + tensor=functools.partial(gen_resource_variable_ops.read_variable_op, + resource=handle, dtype=dtype), + slice_spec="", + device=handle.device, + dtype=dtype, + name=name)] self._handle = handle - self._parallel_device = parallel_device - self._component_shape = component_shape - super(_ParallelDeviceSaveable, self).__init__(None, specs, name) + super(_ParallelComponentSaveable, self).__init__(handle, specs, name) def restore(self, tensors, restored_shapes=None): - with ops.device(self._handle.device): - # Combine the restored tensors into one parallel tensor to assign. - bundled = self._parallel_device.pack(tensors) - gen_resource_variable_ops.assign_variable_op( - resource=self._handle, - # Squeeze out the dummy first axis we added when saving. - value=array_ops.squeeze(bundled, axis=0)) + restored_tensor, = tensors + gen_resource_variable_ops.assign_variable_op( + resource=self._handle, value=restored_tensor) -class VariableWithFixedCheckpointing(resource_variable_ops.ResourceVariable): - """Overrides checkpointing behavior to save like a partitioned variable.""" +class ParallelVariable(resource_variable_ops.ResourceVariable): + """Overrides checkpointing behavior to save each component separately.""" def __init__(self, parallel_device, **kwargs): self._parallel_device = parallel_device - kwargs = {k: v for k, v in kwargs.items() - if k not in ["use_resource", "expected_shape"]} - super(VariableWithFixedCheckpointing, self).__init__(**kwargs) + super(ParallelVariable, self).__init__(**kwargs) + # TODO(allenl): Consider either adding a boolean argument for + # save-primary-only or looking at synchronization/aggregation properties. def _gather_saveables_for_checkpoint(self): - # Note VARIABLE_VALUE is the usual attribute name for variables. Using - # something different means (a) the checkpointing infrastructure won't try - # doing restore-on-create (which has shape issues), and (b) the saved - # variables won't be compatible with regular variables. Both of those are - # good in this case. - return dict( - PARALLEL_VARIABLE_VALUE=functools.partial( - _ParallelDeviceSaveable, - handle=self.handle, - dtype=self.dtype, - component_shape=self.shape, - parallel_device=self._parallel_device)) + component_saveables = {} + # Create one SaveableObject per device, each one of which looks like a + # regular ResourceVariable saveable. + for index, handle in enumerate(self._parallel_device.unpack(self.handle)): + component_saveables["parallel_component_{}".format(index)] = ( + functools.partial( + _ParallelComponentSaveable, + handle=handle, + dtype=self.dtype, + shape=self.shape)) + return component_saveables def _variable_creator(next_creator, parallel_device, **kwargs): del next_creator - return VariableWithFixedCheckpointing( - parallel_device=parallel_device, **kwargs) + return ParallelVariable(parallel_device=parallel_device, **kwargs) @contextlib.contextmanager From 1c98821ea402f98a564b012346e8377e90cea526 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 15:34:41 -0700 Subject: [PATCH 0934/1017] [XLA] Fix portability problem in representing NAN in total order comparison test PiperOrigin-RevId: 326327282 Change-Id: Ieb7dcb927edebe046c09817f3f63d845e002aaf7 --- tensorflow/compiler/xla/tests/BUILD | 1 - .../compiler/xla/tests/array_elementwise_ops_test.cc | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD index 3dac381ae7d..17444c042e7 100644 --- a/tensorflow/compiler/xla/tests/BUILD +++ b/tensorflow/compiler/xla/tests/BUILD @@ -728,7 +728,6 @@ xla_test( name = "array_elementwise_ops_test", srcs = ["array_elementwise_ops_test.cc"], shard_count = 25, - tags = ["no_oss"], # b/163416869 deps = [ ":test_macros_header", "//tensorflow/compiler/xla:array2d", diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc index fdc679a61c6..ef4ce24a839 100644 --- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc +++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc @@ -1235,9 +1235,13 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGeF32s) { XLA_TEST_F(ArrayElementwiseOpTest, CompareGeF32sTO) { SetFastMathDisabled(true); XlaBuilder builder(TestName()); + // For portability, need to represent NAN using the following call. + // The C++ standard does not specify if quiet_NaN() sets the sign bit of + // its result. The call to std::fabs will ensure that it is not set. + auto nan = std::fabs(std::numeric_limits::quiet_NaN()); auto lhs = - ConstantR1(&builder, {-2.5f, 25.5f, 2.25f, NAN, 6.0f, 6.0f}); - auto rhs = ConstantR1(&builder, {10.0f, 5.0f, 1.0f, 10.0f, NAN, -NAN}); + ConstantR1(&builder, {-2.5f, 25.5f, 2.25f, nan, 6.0f, 6.0f}); + auto rhs = ConstantR1(&builder, {10.0f, 5.0f, 1.0f, 10.0f, nan, -nan}); GeTotalOrder(lhs, rhs); ComputeAndCompareR1(&builder, {false, true, true, true, false, true}, From 7a512dd6257773095ff9eca288f56f3644346874 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 15:45:39 -0700 Subject: [PATCH 0935/1017] Go: Update generated wrapper functions for TensorFlow ops. PiperOrigin-RevId: 326329456 Change-Id: Ida61c94288641204e5e5f8faf358d72fd75f6477 --- tensorflow/go/op/wrappers.go | 62 ++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go index 22223dc2b1a..fa3949f6be0 100644 --- a/tensorflow/go/op/wrappers.go +++ b/tensorflow/go/op/wrappers.go @@ -15573,33 +15573,6 @@ func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate t return op.Output(0) } -// Outputs a `Summary` protocol buffer with a histogram. -// -// The generated -// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto) -// has one summary value containing a histogram for `values`. -// -// This op reports an `InvalidArgument` error if any value is not finite. -// -// Arguments: -// tag: Scalar. Tag to use for the `Summary.Value`. -// values: Any shape. Values to use to build the histogram. -// -// Returns Scalar. Serialized `Summary` protocol buffer. -func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) { - if scope.Err() != nil { - return - } - opspec := tf.OpSpec{ - Type: "HistogramSummary", - Input: []tf.Input{ - tag, values, - }, - } - op := scope.AddOperation(opspec) - return op.Output(0) -} - // StringLengthAttr is an optional argument to StringLength. type StringLengthAttr func(optionalAttr) @@ -28677,6 +28650,33 @@ func IteratorFromStringHandle(scope *Scope, string_handle tf.Output, optional .. return op.Output(0) } +// Outputs a `Summary` protocol buffer with a histogram. +// +// The generated +// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto) +// has one summary value containing a histogram for `values`. +// +// This op reports an `InvalidArgument` error if any value is not finite. +// +// Arguments: +// tag: Scalar. Tag to use for the `Summary.Value`. +// values: Any shape. Values to use to build the histogram. +// +// Returns Scalar. Serialized `Summary` protocol buffer. +func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) { + if scope.Err() != nil { + return + } + opspec := tf.OpSpec{ + Type: "HistogramSummary", + Input: []tf.Input{ + tag, values, + }, + } + op := scope.AddOperation(opspec) + return op.Output(0) +} + // Performs gradient updates of embedding tables. // // Arguments: @@ -48756,6 +48756,14 @@ func AutoShardDatasetAutoShardPolicy(value int64) AutoShardDatasetAttr { } } +// AutoShardDatasetNumReplicas sets the optional num_replicas attribute to value. +// If not specified, defaults to 0 +func AutoShardDatasetNumReplicas(value int64) AutoShardDatasetAttr { + return func(m optionalAttr) { + m["num_replicas"] = value + } +} + // Creates a dataset that shards the input dataset. // // Creates a dataset that shards the input dataset by num_workers, returning a From 01d29bf61ed89b728b71ccf0c0b2d4b60bd67714 Mon Sep 17 00:00:00 2001 From: Yuanzhong Xu Date: Wed, 12 Aug 2020 15:52:05 -0700 Subject: [PATCH 0936/1017] [XLA] Fix wrong rank check for partial sharding PiperOrigin-RevId: 326330773 Change-Id: I8b3d46ca6dd37fafe61b59d2a35dbbd3a6ee8249 --- tensorflow/compiler/xla/service/hlo_sharding.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc index 92270005ffd..4244cdaceea 100644 --- a/tensorflow/compiler/xla/service/hlo_sharding.cc +++ b/tensorflow/compiler/xla/service/hlo_sharding.cc @@ -242,7 +242,8 @@ std::vector HloSharding::TileLimitForDevice(const Shape& shape, shape.dimensions().end()); } - CHECK_EQ(shape.dimensions_size(), tile_assignment_.num_dimensions()); + CHECK_EQ(shape.dimensions_size() + (ReplicateOnLastTileDim() ? 1 : 0), + tile_assignment_.num_dimensions()); std::vector index = TileIndexForDevice(device); for (int64 i = 0; i < index.size(); ++i) { const int64 shape_dim = shape.dimensions(i); From 8e7f1aeb483963aad56fee6977599d117d8ba540 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 15:53:17 -0700 Subject: [PATCH 0937/1017] Enable profiler for tf.data service. PiperOrigin-RevId: 326331067 Change-Id: I704968182fb23d5241a071978c6c9afca265f8ab --- tensorflow/core/data/service/BUILD | 6 ++++-- .../core/data/service/credentials_factory.cc | 13 +++++++------ .../core/data/service/credentials_factory.h | 8 ++++---- .../core/data/service/grpc_dispatcher_impl.h | 10 +++++----- tensorflow/core/data/service/grpc_util.cc | 2 +- tensorflow/core/data/service/grpc_util.h | 2 +- .../core/data/service/grpc_worker_impl.cc | 10 +++++----- .../core/data/service/grpc_worker_impl.h | 10 +++++----- tensorflow/core/data/service/server_lib.cc | 19 ++++++++++++++----- tensorflow/core/data/service/server_lib.h | 12 ++++++++---- tensorflow/core/profiler/rpc/BUILD | 1 + 11 files changed, 55 insertions(+), 38 deletions(-) diff --git a/tensorflow/core/data/service/BUILD b/tensorflow/core/data/service/BUILD index 13034eb4354..c64748c6452 100644 --- a/tensorflow/core/data/service/BUILD +++ b/tensorflow/core/data/service/BUILD @@ -3,6 +3,7 @@ load( "//tensorflow/core/platform:build_config.bzl", "tf_additional_all_protos", "tf_proto_library", + "tf_protos_profiler_service", ) load("//tensorflow:tensorflow.bzl", "tf_grpc_cc_dependency") load( @@ -332,6 +333,7 @@ cc_library( "//tensorflow/core:lib", "//tensorflow/core:protos_all_cc", "//tensorflow/core:tensorflow", + "//tensorflow/core/profiler/rpc:profiler_service_impl", tf_grpc_cc_dependency(), ], alwayslink = 1, @@ -375,14 +377,14 @@ tf_cc_test( ":test_util", ":worker_cc_grpc_proto", ":worker_proto_cc", + "@com_google_absl//absl/strings", "//tensorflow/core:lib", "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core/data:compression_utils", "//tensorflow/core/kernels/data:dataset_test_base", - "@com_google_absl//absl/strings", tf_grpc_cc_dependency(), - ], + ] + tf_protos_profiler_service(), ) cc_grpc_library( diff --git a/tensorflow/core/data/service/credentials_factory.cc b/tensorflow/core/data/service/credentials_factory.cc index 88b0073ae26..43b56d54d2e 100644 --- a/tensorflow/core/data/service/credentials_factory.cc +++ b/tensorflow/core/data/service/credentials_factory.cc @@ -65,7 +65,8 @@ Status CredentialsFactory::Get(absl::string_view protocol, } Status CredentialsFactory::CreateServerCredentials( - absl::string_view protocol, std::shared_ptr* out) { + absl::string_view protocol, + std::shared_ptr<::grpc::ServerCredentials>* out) { CredentialsFactory* factory; TF_RETURN_IF_ERROR(CredentialsFactory::Get(protocol, &factory)); TF_RETURN_IF_ERROR(factory->CreateServerCredentials(out)); @@ -74,7 +75,7 @@ Status CredentialsFactory::CreateServerCredentials( Status CredentialsFactory::CreateClientCredentials( absl::string_view protocol, - std::shared_ptr* out) { + std::shared_ptr<::grpc::ChannelCredentials>* out) { CredentialsFactory* factory; TF_RETURN_IF_ERROR(CredentialsFactory::Get(protocol, &factory)); TF_RETURN_IF_ERROR(factory->CreateClientCredentials(out)); @@ -86,14 +87,14 @@ class InsecureCredentialsFactory : public CredentialsFactory { std::string Protocol() override { return "grpc"; } Status CreateServerCredentials( - std::shared_ptr* out) override { - *out = grpc::InsecureServerCredentials(); + std::shared_ptr<::grpc::ServerCredentials>* out) override { + *out = ::grpc::InsecureServerCredentials(); return Status::OK(); } Status CreateClientCredentials( - std::shared_ptr* out) override { - *out = grpc::InsecureChannelCredentials(); + std::shared_ptr<::grpc::ChannelCredentials>* out) override { + *out = ::grpc::InsecureChannelCredentials(); return Status::OK(); } }; diff --git a/tensorflow/core/data/service/credentials_factory.h b/tensorflow/core/data/service/credentials_factory.h index a93b9411ec0..2407f64ee7f 100644 --- a/tensorflow/core/data/service/credentials_factory.h +++ b/tensorflow/core/data/service/credentials_factory.h @@ -36,11 +36,11 @@ class CredentialsFactory { // Stores server credentials to `*out`. virtual Status CreateServerCredentials( - std::shared_ptr* out) = 0; + std::shared_ptr<::grpc::ServerCredentials>* out) = 0; // Stores client credentials to `*out`. virtual Status CreateClientCredentials( - std::shared_ptr* out) = 0; + std::shared_ptr<::grpc::ChannelCredentials>* out) = 0; // Registers a credentials factory. static void Register(CredentialsFactory* factory); @@ -49,13 +49,13 @@ class CredentialsFactory { // `protocol`, and stores them to `*out`. static Status CreateServerCredentials( absl::string_view protocol, - std::shared_ptr* out); + std::shared_ptr<::grpc::ServerCredentials>* out); // Creates client credentials using the credentials factory registered as // `protocol`, and stores them to `*out`. static Status CreateClientCredentials( absl::string_view protocol, - std::shared_ptr* out); + std::shared_ptr<::grpc::ChannelCredentials>* out); private: // Gets the credentials factory registered via `Register` for the specified diff --git a/tensorflow/core/data/service/grpc_dispatcher_impl.h b/tensorflow/core/data/service/grpc_dispatcher_impl.h index 1810c3fb6ac..7e8910b1680 100644 --- a/tensorflow/core/data/service/grpc_dispatcher_impl.h +++ b/tensorflow/core/data/service/grpc_dispatcher_impl.h @@ -35,16 +35,16 @@ namespace data { // class GrpcDispatcherImpl : public DispatcherService::Service { public: - explicit GrpcDispatcherImpl(grpc::ServerBuilder* server_builder, + explicit GrpcDispatcherImpl(::grpc::ServerBuilder* server_builder, const experimental::DispatcherConfig& config); ~GrpcDispatcherImpl() override {} Status Start(); -#define HANDLER(method) \ - grpc::Status method(grpc::ServerContext* context, \ - const method##Request* request, \ - method##Response* response) override; +#define HANDLER(method) \ + ::grpc::Status method(::grpc::ServerContext* context, \ + const method##Request* request, \ + method##Response* response) override; HANDLER(RegisterWorker); HANDLER(WorkerUpdate); HANDLER(GetOrRegisterDataset); diff --git a/tensorflow/core/data/service/grpc_util.cc b/tensorflow/core/data/service/grpc_util.cc index 7f9d2ac07e7..c86496c130a 100644 --- a/tensorflow/core/data/service/grpc_util.cc +++ b/tensorflow/core/data/service/grpc_util.cc @@ -26,7 +26,7 @@ namespace tensorflow { namespace data { namespace grpc_util { -Status WrapError(const std::string& message, const grpc::Status& status) { +Status WrapError(const std::string& message, const ::grpc::Status& status) { if (status.ok()) { return errors::Internal("Expected a non-ok grpc status. Wrapping message: ", message); diff --git a/tensorflow/core/data/service/grpc_util.h b/tensorflow/core/data/service/grpc_util.h index b0e39df79eb..0ae2a86d118 100644 --- a/tensorflow/core/data/service/grpc_util.h +++ b/tensorflow/core/data/service/grpc_util.h @@ -24,7 +24,7 @@ namespace data { namespace grpc_util { // Wraps a grpc::Status in a tensorflow::Status with the given message. -Status WrapError(const std::string& message, const grpc::Status& status); +Status WrapError(const std::string& message, const ::grpc::Status& status); // Retries the given function if the function produces UNAVAILABLE, ABORTED, or // CANCELLED status codes. We retry these codes because they can all indicate diff --git a/tensorflow/core/data/service/grpc_worker_impl.cc b/tensorflow/core/data/service/grpc_worker_impl.cc index 5e3183d61b8..b3a37fe0eec 100644 --- a/tensorflow/core/data/service/grpc_worker_impl.cc +++ b/tensorflow/core/data/service/grpc_worker_impl.cc @@ -35,11 +35,11 @@ Status GrpcWorkerImpl::Start(const std::string& worker_address) { return impl_.Start(worker_address); } -#define HANDLER(method) \ - grpc::Status GrpcWorkerImpl::method(ServerContext* context, \ - const method##Request* request, \ - method##Response* response) { \ - return ToGrpcStatus(impl_.method(request, response)); \ +#define HANDLER(method) \ + ::grpc::Status GrpcWorkerImpl::method(ServerContext* context, \ + const method##Request* request, \ + method##Response* response) { \ + return ToGrpcStatus(impl_.method(request, response)); \ } HANDLER(ProcessTask); HANDLER(GetElement); diff --git a/tensorflow/core/data/service/grpc_worker_impl.h b/tensorflow/core/data/service/grpc_worker_impl.h index 49caab246ac..c42e5639385 100644 --- a/tensorflow/core/data/service/grpc_worker_impl.h +++ b/tensorflow/core/data/service/grpc_worker_impl.h @@ -35,16 +35,16 @@ namespace data { // class GrpcWorkerImpl : public WorkerService::Service { public: - explicit GrpcWorkerImpl(grpc::ServerBuilder* server_builder, + explicit GrpcWorkerImpl(::grpc::ServerBuilder* server_builder, const experimental::WorkerConfig& config); ~GrpcWorkerImpl() override {} Status Start(const std::string& worker_address); -#define HANDLER(method) \ - grpc::Status method(grpc::ServerContext* context, \ - const method##Request* request, \ - method##Response* response) override; +#define HANDLER(method) \ + ::grpc::Status method(::grpc::ServerContext* context, \ + const method##Request* request, \ + method##Response* response) override; HANDLER(ProcessTask); HANDLER(GetElement); #undef HANDLER diff --git a/tensorflow/core/data/service/server_lib.cc b/tensorflow/core/data/service/server_lib.cc index fb33319db29..477f785dc84 100644 --- a/tensorflow/core/data/service/server_lib.cc +++ b/tensorflow/core/data/service/server_lib.cc @@ -51,7 +51,8 @@ Status GrpcDataServerBase::Start() { credentials, &bound_port_); builder.SetMaxReceiveMessageSize(-1); - AddServiceToBuilder(&builder); + AddDataServiceToBuilder(&builder); + AddProfilerServiceToBuilder(&builder); server_ = builder.BuildAndStart(); if (!server_) { return errors::Internal("Could not start gRPC server"); @@ -77,6 +78,12 @@ void GrpcDataServerBase::Join() { server_->Wait(); } int GrpcDataServerBase::BoundPort() { return bound_port(); } +void GrpcDataServerBase::AddProfilerServiceToBuilder( + ::grpc::ServerBuilder* builder) { + profiler_service_ = CreateProfilerService(); + builder->RegisterService(profiler_service_.get()); +} + DispatchGrpcDataServer::DispatchGrpcDataServer( const experimental::DispatcherConfig& config) : GrpcDataServerBase(config.port(), config.protocol(), "DispatchServer"), @@ -84,7 +91,8 @@ DispatchGrpcDataServer::DispatchGrpcDataServer( DispatchGrpcDataServer::~DispatchGrpcDataServer() { delete service_; } -void DispatchGrpcDataServer::AddServiceToBuilder(grpc::ServerBuilder* builder) { +void DispatchGrpcDataServer::AddDataServiceToBuilder( + ::grpc::ServerBuilder* builder) { service_ = absl::make_unique(builder, config_).release(); } @@ -95,8 +103,8 @@ Status DispatchGrpcDataServer::StartServiceInternal() { Status DispatchGrpcDataServer::NumWorkers(int* num_workers) { GetWorkersRequest req; GetWorkersResponse resp; - grpc::ServerContext ctx; - grpc::Status s = service_->GetWorkers(&ctx, &req, &resp); + ::grpc::ServerContext ctx; + ::grpc::Status s = service_->GetWorkers(&ctx, &req, &resp); if (!s.ok()) { return grpc_util::WrapError("Failed to get workers", s); } @@ -111,7 +119,8 @@ WorkerGrpcDataServer::WorkerGrpcDataServer( WorkerGrpcDataServer::~WorkerGrpcDataServer() { delete service_; } -void WorkerGrpcDataServer::AddServiceToBuilder(grpc::ServerBuilder* builder) { +void WorkerGrpcDataServer::AddDataServiceToBuilder( + ::grpc::ServerBuilder* builder) { service_ = absl::make_unique(builder, config_).release(); } diff --git a/tensorflow/core/data/service/server_lib.h b/tensorflow/core/data/service/server_lib.h index 62662e61c8a..0ddc80676c3 100644 --- a/tensorflow/core/data/service/server_lib.h +++ b/tensorflow/core/data/service/server_lib.h @@ -19,6 +19,7 @@ limitations under the License. #include "grpcpp/server.h" #include "grpcpp/server_builder.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/profiler/rpc/profiler_service_impl.h" #include "tensorflow/core/protobuf/data/experimental/service_config.pb.h" namespace tensorflow { @@ -52,7 +53,8 @@ class GrpcDataServerBase { int BoundPort(); protected: - virtual void AddServiceToBuilder(::grpc::ServerBuilder* builder) = 0; + virtual void AddDataServiceToBuilder(::grpc::ServerBuilder* builder) = 0; + void AddProfilerServiceToBuilder(::grpc::ServerBuilder* builder); // Starts the service. This will be called after building the service, so // bound_port() will return the actual bound port. virtual Status StartServiceInternal() = 0; @@ -68,7 +70,9 @@ class GrpcDataServerBase { bool started_ = false; bool stopped_ = false; - std::unique_ptr server_; + std::unique_ptr<::grpc::Server> server_; + // TensorFlow profiler service implementation. + std::unique_ptr profiler_service_ = nullptr; }; class DispatchGrpcDataServer : public GrpcDataServerBase { @@ -80,7 +84,7 @@ class DispatchGrpcDataServer : public GrpcDataServerBase { Status NumWorkers(int* num_workers); protected: - void AddServiceToBuilder(grpc::ServerBuilder* builder) override; + void AddDataServiceToBuilder(::grpc::ServerBuilder* builder) override; Status StartServiceInternal() override; private: @@ -95,7 +99,7 @@ class WorkerGrpcDataServer : public GrpcDataServerBase { ~WorkerGrpcDataServer() override; protected: - void AddServiceToBuilder(grpc::ServerBuilder* builder) override; + void AddDataServiceToBuilder(::grpc::ServerBuilder* builder) override; Status StartServiceInternal() override; private: diff --git a/tensorflow/core/profiler/rpc/BUILD b/tensorflow/core/profiler/rpc/BUILD index 06e5d2e4d2b..496e0c7d4d3 100644 --- a/tensorflow/core/profiler/rpc/BUILD +++ b/tensorflow/core/profiler/rpc/BUILD @@ -13,6 +13,7 @@ cc_library( features = ["-layering_check"], visibility = tf_external_workspace_visible( [ + "//tensorflow/core/data/service:__pkg__", "//tensorflow/core/distributed_runtime/rpc:__pkg__", "//tensorflow_serving/model_servers:__pkg__", ], From 143fe010ea3a2ef745160f7f41e95375ddfb0aad Mon Sep 17 00:00:00 2001 From: Robert Suderman Date: Wed, 12 Aug 2020 16:14:42 -0700 Subject: [PATCH 0938/1017] Lowering for mhlo.ceil to std.ceil PiperOrigin-RevId: 326335301 Change-Id: I4565f1d343ed38d2839f6dbf28d9d61c32740c35 --- .../mhlo/transforms/legalize_to_standard_patterns.td | 4 ++++ tensorflow/compiler/mlir/hlo/tests/legalize-to-std.mlir | 9 +++++++++ 2 files changed, 13 insertions(+) diff --git a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/legalize_to_standard_patterns.td b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/legalize_to_standard_patterns.td index ea67c052c5c..6ee6f124628 100644 --- a/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/legalize_to_standard_patterns.td +++ b/tensorflow/compiler/mlir/hlo/lib/Dialect/mhlo/transforms/legalize_to_standard_patterns.td @@ -36,6 +36,10 @@ def IsSameSizePred : CPred< def IsSameSizeConstraint : Constraint; +// Unary Lowering Patterns. +def : Pat<(HLO_CeilOp HLO_FpTensor:$i), (CeilFOp $i)>; + +// Binary Lowering Patterns. def : Pat<(HLO_AndOp HLO_PredTensor:$l, HLO_PredTensor:$r), (AndOp $l, $r), [(IsSameSizeConstraint $l, $r)]>; diff --git a/tensorflow/compiler/mlir/hlo/tests/legalize-to-std.mlir b/tensorflow/compiler/mlir/hlo/tests/legalize-to-std.mlir index 37a61498fbf..abe4e872b73 100644 --- a/tensorflow/compiler/mlir/hlo/tests/legalize-to-std.mlir +++ b/tensorflow/compiler/mlir/hlo/tests/legalize-to-std.mlir @@ -42,6 +42,15 @@ func @binary_ops_int(%arg0: tensor<4xi32>, %arg1: tensor<4xi32>) -> tensor<4xi32 return %4 : tensor<4xi32> } +// CHECK-LABEL: func @unary_ops_float +func @unary_ops_float(%arg0: tensor<4xf32>) -> tensor<4xf32> { + // CHECK-NEXT: %0 = ceilf %arg0 : tensor<4xf32> + %0 = "mhlo.ceil"(%arg0) : (tensor<4xf32>) -> tensor<4xf32> + + // CHECK-NEXT: return %0 : tensor<4xf32> + return %0 : tensor<4xf32> +} + // CHECK-LABEL: func @compare_int(%arg0: tensor<4xi32>) -> (tensor<4xi1>, tensor<4xi1>, tensor<4xi1>, tensor<4xi1>, tensor<4xi1>, tensor<4xi1>) { func @compare_int(%arg0: tensor<4xi32>) -> (tensor<4xi1>,tensor<4xi1>,tensor<4xi1>,tensor<4xi1>,tensor<4xi1>,tensor<4xi1>) { // CHECK-NEXT: %0 = cmpi "eq", %arg0, %arg0 : tensor<4xi32> From f08bbe942bf60ffa55f39aac5a44e21934716229 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 16:32:11 -0700 Subject: [PATCH 0939/1017] Roll back XLA/GPU LHLO sort emitter again It breaks an internal msan enabled test. PiperOrigin-RevId: 326338736 Change-Id: I98c836e382569d6737db2fccaf8c5def0ecf2be7 --- tensorflow/compiler/mlir/xla/hlo_utils.cc | 3 - .../non_identity_layouts.hlotxt | 2 +- .../xla/transforms/mhlo_to_lhlo_with_xla.cc | 11 +- .../xla/transforms/mhlo_to_lhlo_with_xla.h | 3 +- tensorflow/compiler/xla/service/gpu/BUILD | 10 - .../compiler/xla/service/gpu/gpu_compiler.cc | 24 +- .../xla/service/gpu/hlo_to_ir_bindings.cc | 20 +- .../xla/service/gpu/hlo_to_ir_bindings.h | 4 - .../xla/service/gpu/ir_emitter_context.h | 7 +- .../xla/service/gpu/ir_emitter_unnested.cc | 416 ++++----------- .../xla/service/gpu/ir_emitter_unnested.h | 82 +-- .../compiler/xla/service/gpu/tests/BUILD | 29 - .../xla/service/gpu/tests/sorting.hlo | 504 +++++++++--------- .../xla/service/gpu/tests/sorting_test.cc | 71 --- .../compiler/xla/service/llvm_ir/llvm_util.cc | 7 +- .../compiler/xla/service/llvm_ir/llvm_util.h | 2 +- 16 files changed, 403 insertions(+), 792 deletions(-) delete mode 100644 tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc diff --git a/tensorflow/compiler/mlir/xla/hlo_utils.cc b/tensorflow/compiler/mlir/xla/hlo_utils.cc index 18b4265d786..cf78c81908d 100644 --- a/tensorflow/compiler/mlir/xla/hlo_utils.cc +++ b/tensorflow/compiler/mlir/xla/hlo_utils.cc @@ -83,9 +83,6 @@ StatusOr> GetPermutationIfAvailable( strides[dim] = accumulated_stride; accumulated_stride *= shape.dimensions(dim); } - if (accumulated_stride == 0) { - return llvm::SmallVector{}; - } return llvm::SmallVector{ makeStridedLinearLayoutMap(strides, /*offset=*/0, builder.getContext())}; } diff --git a/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt b/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt index a83e36cff64..3630d2d45e4 100644 --- a/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt +++ b/tensorflow/compiler/mlir/xla/tests/hlo_to_lhlo_with_xla/non_identity_layouts.hlotxt @@ -8,6 +8,6 @@ HloModule TestModule ENTRY TestComputation { x = f32[3, 2]{1,0} parameter(0) - // CHECK: "lmhlo.copy"(%{{.*}}, %{{.*}}) {name = "copy.1"} : (memref<3x2xf32>, memref<3x2xf32, #[[MAP]]>) -> () + // CHECK: "lmhlo.copy"(%{{.*}}, %{{.*}}) : (memref<3x2xf32>, memref<3x2xf32, #[[MAP]]>) -> () ROOT x.copy = f32[3, 2]{0,1} copy(x) } diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc index 6ce91599fb1..832bad2dcc8 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc @@ -34,6 +34,7 @@ limitations under the License. #include "mlir/Pass/Pass.h" // from @llvm-project #include "mlir/Pass/PassOptions.h" // from @llvm-project #include "mlir/Translation.h" // from @llvm-project +#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" #include "tensorflow/compiler/mlir/xla/hlo_function_importer.h" #include "tensorflow/compiler/mlir/xla/hlo_utils.h" #include "tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.h" @@ -181,10 +182,7 @@ template StatusOr LhloDialectEmitter::CreateOpWithoutAttrs( HloInstruction* instr) { Location loc = getLocation(instr); - std::pair attrs[] = { - {Identifier::get("name", builder_.getContext()), - builder_.getStringAttr(instr->name())}, - }; + ArrayRef> attrs; ArrayRef rets{}; llvm::SmallVector operands; @@ -254,14 +252,15 @@ Status LhloDialectEmitter::DefaultAction(HloInstruction* instr) { return Status::OK(); } -StatusOr LhloDialectEmitter::EmitSortOp(HloInstruction* instr) { +StatusOr LhloDialectEmitter::EmitSortOp( + HloInstruction* instr) { TF_ASSIGN_OR_RETURN(auto sort, CreateOpWithoutAttrs(instr)); auto* sort_instr = ::xla::Cast<::xla::HloSortInstruction>(instr); sort.dimensionAttr(builder_.getI64IntegerAttr(sort_instr->sort_dimension())); sort.is_stableAttr(builder_.getBoolAttr(sort_instr->is_stable())); TF_RETURN_IF_ERROR(::xla::HloFunctionImporter::ImportAsRegion( *sort_instr->called_computations()[0], &sort.comparator(), &builder_)); - return sort; + return sort.getOperation(); } Status LhloDialectEmitter::HandleSort(HloInstruction* instr) { diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h index 4000fa01970..bdc977616b1 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h @@ -19,7 +19,6 @@ limitations under the License. #include "mlir/IR/Builders.h" // from @llvm-project #include "mlir/IR/Module.h" // from @llvm-project #include "mlir/IR/StandardTypes.h" // from @llvm-project -#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/hlo_module.h" @@ -42,7 +41,7 @@ class LhloDialectEmitter : public ::xla::DfsHloVisitorWithDefault { builder_(module.getContext()), i8_type_(builder_.getIntegerType(8)) {} - ::xla::StatusOr EmitSortOp(::xla::HloInstruction* instr); + ::xla::StatusOr EmitSortOp(::xla::HloInstruction* instr); private: template diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD index a19f9965fc7..074fbd92b27 100644 --- a/tensorflow/compiler/xla/service/gpu/BUILD +++ b/tensorflow/compiler/xla/service/gpu/BUILD @@ -254,11 +254,6 @@ cc_library( ":target_util", ":thunk", ":thunk_emitter", - "//tensorflow/compiler/mlir/hlo:lhlo", - "//tensorflow/compiler/mlir/xla:hlo_utils", - "//tensorflow/compiler/mlir/xla:mhlo_to_lhlo_with_xla", - "//tensorflow/compiler/mlir/xla:mlir_hlo_to_hlo", - "//tensorflow/compiler/mlir/xla:type_to_shape", "//tensorflow/compiler/xla:literal", "//tensorflow/compiler/xla:shape_util", "//tensorflow/compiler/xla:status_macros", @@ -296,8 +291,6 @@ cc_library( "@com_google_absl//absl/types:span", "@llvm-project//llvm:Core", "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:StandardOps", ], ) @@ -1166,7 +1159,6 @@ cc_library( ":target_constants", ":tree_reduction_rewriter", ":variadic_op_splitter", - "//tensorflow/compiler/mlir/xla:mhlo_to_lhlo_with_xla", "//tensorflow/compiler/xla:protobuf_util", "//tensorflow/compiler/xla:status_macros", "//tensorflow/compiler/xla:statusor", @@ -1225,8 +1217,6 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "@llvm-project//llvm:Core", - "@llvm-project//mlir:AllPassesAndDialectsNoRegistration", - "@llvm-project//mlir:IR", ], ) diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc index b796737e601..f5bf7476059 100644 --- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc +++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc @@ -29,8 +29,6 @@ limitations under the License. #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Verifier.h" -#include "mlir/IR/Module.h" // from @llvm-project -#include "mlir/InitAllDialects.h" // from @llvm-project #include "tensorflow/compiler/xla/protobuf_util.h" #include "tensorflow/compiler/xla/service/algebraic_simplifier.h" #include "tensorflow/compiler/xla/service/all_reduce_combiner.h" @@ -518,22 +516,15 @@ static Status CompileModuleToLlvmIrImpl( DumpHloModuleIfEnabled(*hlo_module, **buffer_assignment, "after_optimizations"); - mlir::registerAllDialects(); - mlir::MLIRContext mlir_context; - IrEmitterContext ir_emitter_context( hlo_module, buffer_assignment->get(), platform_name, gpu_device_info, - cuda_compute_capability, profile_index_map, &mlir_context, - llvm_module->get()); + cuda_compute_capability, profile_index_map, llvm_module->get()); HloComputation* entry_computation = hlo_module->entry_computation(); + IrEmitterUnnested ir_emitter(hlo_module->config(), entry_computation, + &ir_emitter_context); - TF_ASSIGN_OR_RETURN( - auto ir_emitter, - IrEmitterUnnested::Create(hlo_module->config(), entry_computation, - &ir_emitter_context)); - - TF_RETURN_IF_ERROR(ir_emitter->EmitConstantGlobals()); + TF_RETURN_IF_ERROR(ir_emitter.EmitConstantGlobals()); { XLA_SCOPED_LOGGING_TIMER("GpuCompiler::RunBackend - IR emission"); @@ -542,10 +533,9 @@ static Status CompileModuleToLlvmIrImpl( ThunkSequence thunk_sequence; absl::Span order = hlo_schedule->ThunkLaunchOrder(); for (HloInstruction* instruction : order) { - TF_RETURN_IF_ERROR(instruction->Visit(ir_emitter.get())); - TF_RETURN_IF_ERROR(ir_emitter->Postprocess(instruction)); - std::unique_ptr thunks = - ir_emitter->ConsumeThunkSequence(); + TF_RETURN_IF_ERROR(instruction->Visit(&ir_emitter)); + TF_RETURN_IF_ERROR(ir_emitter.Postprocess(instruction)); + std::unique_ptr thunks = ir_emitter.ConsumeThunkSequence(); // The invariants between each input HloInstruction* and output Thunk* are // not all explicitly checked, but at least we can document them here: diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc index 332db83b6ad..5d38d1b727c 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.cc @@ -117,11 +117,11 @@ static bool HasMeaningfulName(llvm::Value* value) { return false; } -llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, - llvm::IRBuilder<>* b) { - llvm::Type* pointee_type = - llvm_ir::ShapeToIrType(shape, b->GetInsertBlock()->getModule()); - +llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, + ShapeIndexView shape_index, + llvm::Value* ir_value) { + llvm::Type* pointee_type = llvm_ir::ShapeToIrType( + ShapeUtil::GetSubshape(hlo.shape(), shape_index), module_); llvm::Type* dest_type = pointee_type->getPointerTo(); llvm::Value* typed_ir_value; @@ -129,17 +129,9 @@ llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, typed_ir_value = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast( llvm::cast(ir_value), dest_type); } else { - typed_ir_value = b->CreatePointerBitCastOrAddrSpaceCast( + typed_ir_value = b_->CreatePointerBitCastOrAddrSpaceCast( ir_value, pointee_type->getPointerTo()); } - return typed_ir_value; -} - -llvm::Value* HloToIrBindings::GetTypedIrValue(const HloInstruction& hlo, - ShapeIndexView shape_index, - llvm::Value* ir_value) { - auto typed_ir_value = CastToTypedValue( - ShapeUtil::GetSubshape(hlo.shape(), shape_index), ir_value, b_); if (!HasMeaningfulName(ir_value)) { ir_value->setName(llvm_ir::IrName(&hlo, "raw")); } diff --git a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h index 3813ec6c949..5eef6727801 100644 --- a/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h +++ b/tensorflow/compiler/xla/service/gpu/hlo_to_ir_bindings.h @@ -116,10 +116,6 @@ class HloToIrBindings { llvm::Value* temp_buffer_base_ = nullptr; }; -// Converts `ir_value` with type i8* to a typed LLVM Value* based on `shape`. -llvm::Value* CastToTypedValue(const Shape& shape, llvm::Value* ir_value, - llvm::IRBuilder<>* b); - } // namespace gpu } // namespace xla diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h index 7d5a8d032e6..9c43f80dc60 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_context.h @@ -17,7 +17,6 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_CONTEXT_H_ #include "llvm/IR/Module.h" -#include "mlir/IR/MLIRContext.h" // from @llvm-project #include "tensorflow/compiler/xla/service/buffer_assignment.h" #include "tensorflow/compiler/xla/service/gpu/launch_dimensions.h" #include "tensorflow/compiler/xla/service/hlo_execution_profile.h" @@ -35,15 +34,13 @@ class IrEmitterContext { const HloModule* hlo_module, const BufferAssignment* buffer_assignment, std::string platform_name, GpuDeviceInfo gpu_device_info, absl::optional cuda_compute_capability, - const HloProfileIndexMap* profile_index_map, - mlir::MLIRContext* mlir_context, llvm::Module* llvm_module) + const HloProfileIndexMap* profile_index_map, llvm::Module* llvm_module) : hlo_module_(hlo_module), buffer_assignment_(buffer_assignment), platform_name_(std::move(platform_name)), gpu_device_info_(gpu_device_info), cuda_compute_capability_(cuda_compute_capability), profile_index_map_(profile_index_map), - mlir_context_(mlir_context), llvm_module_(llvm_module) {} // Disallow copy and assign. IrEmitterContext(const IrEmitterContext&) = delete; @@ -60,7 +57,6 @@ class IrEmitterContext { return cuda_compute_capability_; } const HloProfileIndexMap* profile_index_map() { return profile_index_map_; } - mlir::MLIRContext* mlir_context() { return mlir_context_; } llvm::Module* llvm_module() { return llvm_module_; } NameUniquer* name_uniquer() { return &name_uniquer_; } @@ -71,7 +67,6 @@ class IrEmitterContext { GpuDeviceInfo gpu_device_info_; absl::optional cuda_compute_capability_; const HloProfileIndexMap* profile_index_map_; - mlir::MLIRContext* mlir_context_; llvm::Module* llvm_module_; NameUniquer name_uniquer_; }; diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc index f88c70b1a33..61b78b6004d 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc @@ -37,13 +37,6 @@ limitations under the License. #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" -#include "mlir/Dialect/StandardOps/IR/Ops.h" // from @llvm-project -#include "mlir/IR/Builders.h" // from @llvm-project -#include "mlir/IR/Function.h" // from @llvm-project -#include "tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/lhlo_ops.h" -#include "tensorflow/compiler/mlir/xla/hlo_utils.h" -#include "tensorflow/compiler/mlir/xla/mlir_hlo_to_hlo.h" -#include "tensorflow/compiler/mlir/xla/type_to_shape.h" #include "tensorflow/compiler/xla/layout_util.h" #include "tensorflow/compiler/xla/literal.h" #include "tensorflow/compiler/xla/service/buffer_assignment.h" @@ -151,86 +144,13 @@ void UpdateLaunchDimensions(const LaunchDimensions& launch_dims, Thunk* thunk, llvm::ConstantAsMetadata::get(threads_per_block_ir_value)})); } -const BufferAllocation* GetAllocation( - mlir::BlockArgument func_arg, const BufferAssignment& buffer_assignment) { - auto func_op = - mlir::cast(func_arg.getParentRegion()->getParentOp()); - int64 allocation_index = func_op - .getArgAttrOfType( - func_arg.getArgNumber(), "lmhlo.alloc") - .getValue() - .getSExtValue(); - return &buffer_assignment.GetAllocation(allocation_index); -} - -StatusOr GetAllocationSliceForMlir( - mlir::Value v, const BufferAssignment& buffer_assignment) { - int64 size = v.getType().cast().getSizeInBits() / 8; - - if (auto arg = v.dyn_cast()) { - return BufferAllocation::Slice(GetAllocation(arg, buffer_assignment), 0, - size); - } - - // We match two patterns here: - // * v = ViewOp(arg); - // * v = StaticMemRefCastOp(ViewOp(arg)); - if (mlir::Operation* op = v.getDefiningOp()) { - if (auto cast = mlir::dyn_cast(op)) { - mlir::Value source = cast.getViewSource(); - op = source.getDefiningOp(); - if (!op) { - return Unimplemented("StaticMemRefCastOp has to wrap an op"); - } - } - if (auto view = mlir::dyn_cast(op)) { - return BufferAllocation::Slice( - GetAllocation(view.source().cast(), - buffer_assignment), - mlir::cast(view.byte_shift().getDefiningOp()) - .value() - .cast() - .getValue() - .getSExtValue(), - size); - } - return Unimplemented("StaticMemRefCastOp has to wrap a ViewOp"); - } - - return Unimplemented( - "Operand has to be in the form of ViewOp(arg) or " - "StaticMemRefCastOp(ViewOp(arg))"); -} - -absl::string_view GetHloName(mlir::Operation* op) { - if (auto attr = op->getAttrOfType("name")) { - auto ref = attr.getValue(); - return absl::string_view(ref.data(), ref.size()); - } - return ""; -} - } // namespace IrEmitterUnnested::IrEmitterUnnested(const HloModuleConfig& hlo_module_config, const HloComputation* hlo_computation, IrEmitterContext* ir_emitter_context) : IrEmitter(hlo_module_config, ir_emitter_context, /*is_nested=*/false), - hlo_computation_(hlo_computation), - mlir_scratch_module_(mlir::ModuleOp::create( - mlir::Builder(ir_emitter_context->mlir_context()).getUnknownLoc())), - lhlo_scratch_emitter_(ir_emitter_context_->buffer_assignment(), - *hlo_computation, mlir_scratch_module_.get()) {} - -StatusOr> IrEmitterUnnested::Create( - const HloModuleConfig& hlo_module_config, - const HloComputation* hlo_computation, - IrEmitterContext* ir_emitter_context) { - auto emitter = std::unique_ptr(new IrEmitterUnnested( - hlo_module_config, hlo_computation, ir_emitter_context)); - TF_RETURN_IF_ERROR(emitter->lhlo_scratch_emitter_.Initialize()); - return std::move(emitter); -} + hlo_computation_(hlo_computation) {} Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { bindings_.UnbindAllLocalIrValues(); @@ -238,11 +158,12 @@ Status IrEmitterUnnested::Postprocess(HloInstruction* hlo) { } llvm::Function* IrEmitterUnnested::BuildKernelPrototype( - absl::string_view name, absl::Span args) { + const HloInstruction& inst, + absl::Span args) { // Compute the kernel name. The opcode string may contain "-" which cannot be // in a PTX function name, so sanitize the name before uniquifying it. string kernel_name = ir_emitter_context_->name_uniquer()->GetUniqueName( - llvm_ir::SanitizeFunctionName(std::string(name))); + llvm_ir::SanitizeFunctionName(inst.name())); // Create the kernel and add it to the module. llvm::Module* module = ir_emitter_context_->llvm_module(); @@ -438,8 +359,7 @@ Status IrEmitterUnnested::HandleDot(HloInstruction* dot) { } Status IrEmitterUnnested::HandleConditional(HloInstruction* conditional) { - TF_ASSIGN_OR_RETURN(auto thunk, BuildConditionalThunk(conditional)); - AddThunkToThunkSequence(std::move(thunk)); + AddThunkToThunkSequence(BuildConditionalThunk(conditional)); return Status::OK(); } @@ -1118,13 +1038,10 @@ Status IrEmitterUnnested::HandleWhile(HloInstruction* xla_while) { // Build ForThunk for conformant while loops, otherwise build WhileThunk. auto config = xla_while->backend_config(); if (config.ok() && config.ValueOrDie().has_known_trip_count()) { - TF_ASSIGN_OR_RETURN( - auto thunk, + AddThunkToThunkSequence( BuildForThunk(xla_while, config.ValueOrDie().known_trip_count().n())); - AddThunkToThunkSequence(std::move(thunk)); } else { - TF_ASSIGN_OR_RETURN(auto thunk, BuildWhileThunk(xla_while)); - AddThunkToThunkSequence(std::move(thunk)); + AddThunkToThunkSequence(BuildWhileThunk(xla_while)); } return Status::OK(); } @@ -1347,109 +1264,39 @@ Status IrEmitterUnnested::HandleSelect(HloInstruction* select) { return IrEmitter::HandleSelect(select); } -StatusOr -IrEmitterUnnested::GetOrCreateSubComputationFromRegion(mlir::Region* region) { - std::unique_ptr& module = scratch_nested_computations_[region]; - if (module == nullptr) { - xla::XlaComputation xla_computation; - TF_RETURN_IF_ERROR(ConvertRegionToComputation(region, &xla_computation)); - TF_ASSIGN_OR_RETURN(auto program_shape, xla_computation.GetProgramShape()); - TF_ASSIGN_OR_RETURN( - module, HloModule::CreateFromProto(xla_computation.proto(), - HloModuleConfig(program_shape))); - } - return module->entry_computation(); -} - Status IrEmitterUnnested::HandleSort(HloInstruction* sort) { - MlirEmitterInput result; - - TF_ASSIGN_OR_RETURN(auto sort_op, lhlo_scratch_emitter_.EmitSortOp(sort)); - result.op = sort_op; - result.name = GetHloName(sort_op); - // The name in sort op has no semantics, and it's for debug only. If the name - // doesn't exist, we should use a namer (e.g. count-based). - // TODO(timshen): use a namer instead of relying on the HloInstruction names. - if (result.name.empty()) { - result.name = sort->name(); - } - const auto& buffer_assignment = ir_emitter_context_->buffer_assignment(); - auto& slice = result.extra_slice; - TF_ASSIGN_OR_RETURN(slice.buffer_slice, - buffer_assignment.GetUniqueSlice(sort, {})); - slice.written = true; - slice.shape = sort->shape(); - - result.thunk_info = GetThunkInfo(sort); - - return EmitMlirSort(result); -} - -Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { - const auto& buffer_assignment = ir_emitter_context_->buffer_assignment(); - auto sort_op = mlir::cast(input.op); - - int operand_count = sort_op.operands().size(); - std::vector operand_shapes(operand_count); - std::vector slices; - std::vector output_shapes(sort_op.output().size()); - - for (int i = 0; i < operand_count; i++) { - operand_shapes[i] = - TypeToShape(sort_op.operands()[i].getType().cast()); - } - - // Craft n + 1 slices, where the first n are output parameters, and the last - // is the on-device tuple storage. We don't need n operands because sorting - // kernels are always in-place. - for (int i = 0; i < operand_count; i++) { - output_shapes[i] = - TypeToShape(sort_op.output()[i].getType().cast()); - MlirBufferSlice slice; - TF_ASSIGN_OR_RETURN( - slice.buffer_slice, - GetAllocationSliceForMlir(sort_op.output()[i], buffer_assignment)); - slice.written = true; - slice.shape = operand_shapes[i]; - slices.push_back(slice); - } - slices.push_back(input.extra_slice); - std::vector> thunks; - - Shape keys_shape = operand_shapes[0]; - int64 dimension_to_sort = sort_op.dimension().getSExtValue(); - for (int64 i = 0; i < operand_count; ++i) { + Shape keys_shape = sort->operand(0)->shape(); + int64 dimension_to_sort = sort->dimensions(0); + for (int64 i = 0; i < sort->operand_count(); ++i) { + ShapeIndex shape_index = + sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({}); // We assume that the layout of all involved operands and outputs is the // same. - TF_RET_CHECK( - LayoutUtil::LayoutsInShapesEqual(keys_shape, operand_shapes[i])); - TF_RET_CHECK( - LayoutUtil::LayoutsInShapesEqual(keys_shape, output_shapes[i])); + TF_RET_CHECK(LayoutUtil::LayoutsInShapesEqual(keys_shape, + sort->operand(i)->shape())); + TF_RET_CHECK(LayoutUtil::LayoutsInShapesEqual( + keys_shape, ShapeUtil::GetSubshape(sort->shape(), shape_index))); // If possible, we share buffers. If that is not possible, we need to copy // the values, because the emitter does the sorting in-place. - TF_ASSIGN_OR_RETURN( - auto destination_buffer, - GetAllocationSliceForMlir(sort_op.output()[i], buffer_assignment)); - TF_ASSIGN_OR_RETURN( - auto source_address, - GetAllocationSliceForMlir(sort_op.operands()[i], buffer_assignment)); + auto destination_buffer = GetAllocationSlice(*sort, shape_index); + auto source_address = GetAllocationSlice(*sort->operand(i)); if (destination_buffer != source_address) { // TODO(b/26783907): Figure out why we never seem to share buffers for // key/value sort. - VLOG(2) << input.name << " requires initial D2D copy for operand " << i; + VLOG(2) << sort->name() << " requires initial D2D copy for operand " << i; thunks.push_back(absl::make_unique( Thunk::ThunkInfo(), /*source_address=*/source_address, /*destination_buffer=*/destination_buffer, - /*mem_size=*/ShapeUtil::ByteSizeOf(operand_shapes[i]))); + /*mem_size=*/ShapeUtil::ByteSizeOf(sort->operand(i)->shape()))); } } uint64 dimension_to_sort_bound = keys_shape.dimensions(dimension_to_sort); int64 num_stages = tensorflow::Log2Ceiling(dimension_to_sort_bound); - VLOG(2) << input.name << " requires " << num_stages << " stages."; + VLOG(2) << sort->name() << " requires " << num_stages << " stages."; CHECK_GE(1ULL << num_stages, dimension_to_sort_bound); CHECK_LT(1ULL << (num_stages - 1), dimension_to_sort_bound); @@ -1513,10 +1360,10 @@ Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { // we have not enough threads, or not enough shared memory. Also it does not // give a speedup if the tile size is < 128. int64 total_shared_memory_needed = 0; - for (int64 i = 0; i < operand_count; ++i) { + for (int64 i = 0; i < sort->operand_count(); ++i) { total_shared_memory_needed += - kTileSize * - ShapeUtil::ByteSizeOfPrimitiveType(operand_shapes[i].element_type()); + kTileSize * ShapeUtil::ByteSizeOfPrimitiveType( + sort->operand(i)->shape().element_type()); } bool no_tiling = kTileSize < 128 || @@ -1529,7 +1376,7 @@ Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { "kTileSize=%d < 128, " "kThreadsPerBlock=%d > threads_per_block_limit=%d, " "total_shared_memory_needed=%d > shared_memory_per_block=%d", - input.name, (no_tiling ? "won't" : "will"), kTileSize, kThreadsPerBlock, + sort->name(), (no_tiling ? "won't" : "will"), kTileSize, kThreadsPerBlock, ir_emitter_context_->gpu_device_info().threads_per_block_limit, total_shared_memory_needed, ir_emitter_context_->gpu_device_info().shared_memory_per_block); @@ -1537,38 +1384,37 @@ Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { uint64 num_blocks = CeilOfRatio(num_iterations, kThreadsPerBlock); LaunchDimensions tiled_launch_dimensions(num_blocks, kThreadsPerBlock); VLOG(2) << absl::StreamFormat("%s launch dims: %d blocks, %d threads/block", - input.name, num_blocks, kThreadsPerBlock); + sort->name(), num_blocks, kThreadsPerBlock); - std::vector ir_arrays; auto emit_kernel = [&](absl::Span xor_masks) { VLOG(2) << absl::StreamFormat( - "%s uses kernel for xor masks [%s]", input.name, + "%s uses kernel for xor masks [%s]", sort->name(), absl::StrJoin(xor_masks, ", ", [](std::string* out, int64 xor_mask) { absl::StrAppendFormat(out, "0x%x", xor_mask); })); - thunks.push_back(BuildKernelThunkForMlir(input.name, Thunk::ThunkInfo(), - slices, &ir_arrays)); + thunks.push_back( + BuildKernelThunk(sort, /*implements_whole_instruction=*/false)); LaunchDimensions launch_dimensions = xor_masks.size() > 1 ? tiled_launch_dimensions : standard_launch_dimensions; UpdateLaunchDimensions(launch_dimensions, thunks.back().get(), ir_emitter_context_->llvm_module()); std::vector values_arrays; - values_arrays.reserve(operand_count); - for (int64 i = 0; i < operand_count; ++i) { - values_arrays.push_back(ir_arrays[i]); + values_arrays.reserve(sort->operand_count()); + for (int64 i = 0; i < sort->operand_count(); ++i) { + ShapeIndex shape_index = + sort->operand_count() > 1 ? ShapeIndex({i}) : ShapeIndex({}); + values_arrays.push_back(GetIrArray(*sort, *sort, shape_index)); } - TF_ASSIGN_OR_RETURN( - const HloComputation* comparator, - GetOrCreateSubComputationFromRegion(&sort_op.comparator())); return llvm_ir::EmitSortInPlace( - dimension_to_sort, values_arrays, IrName(input.name), xor_masks, &b_, + dimension_to_sort, values_arrays, IrName(sort), xor_masks, &b_, launch_dimensions, xor_masks.size() > 1 ? num_iterations_in_sort_dim : standard_num_iterations_in_sort_dim, kTileSize, [&](absl::Span operands, llvm::Value* output) { - return EmitCallToNestedComputation(*comparator, operands, output); + return EmitCallToNestedComputation(*sort->to_apply(), operands, + output); }); }; std::vector xor_masks; @@ -1595,18 +1441,17 @@ Status IrEmitterUnnested::EmitMlirSort(MlirEmitterInput input) { TF_RETURN_IF_ERROR(emit_kernel(xor_masks)); } VLOG(2) << absl::StreamFormat( - "%s requires %d thunks (including any D2D copies)", input.name, + "%s requires %d thunks (including any D2D copies)", sort->name(), thunks.size()); - AddThunkToThunkSequence( - absl::make_unique(input.thunk_info, std::move(thunks))); - if (operand_count > 1) { + AddThunkToThunkSequence(absl::make_unique( + GetThunkInfo(sort), std::move(thunks))); + if (sort->operand_count() > 1) { // Emit the tuple as part of the last stage of sorting. // We are currently in the block sorted.in_bounds.after. b_.SetInsertPoint(b_.GetInsertBlock()->getTerminator()); - llvm_ir::EmitTuple( - ir_arrays[operand_count], - absl::MakeSpan(ir_arrays).subspan(0, ir_arrays.size() - 1), &b_); + llvm_ir::EmitTuple(GetIrArray(*sort, *sort), + ConstructIrArrayForOutputs(*sort), &b_); } return Status::OK(); } @@ -1744,6 +1589,24 @@ Status IrEmitterUnnested::HandleAfterAll(HloInstruction* after_all) { return Status::OK(); } +// Describes how to access a particular subshape for an HLO. For instance if +// `.hlo_index` is {1} and `.gte_index` is {3, 4} then buffer for `.instr` at +// ShapeIndex {1} (i.e. the buffer for the second tuple element of hlo) is found +// at `.buffer_slice`[3][4]. That is, `.slice` is a void***, which we +// dereference twice -- first at index 3, and then at index 4 -- to get the +// address of our buffer. +struct HloBufferSlice { + const HloInstruction* instr; + ShapeIndex hlo_index; + + // The root buffer to look at. + BufferAllocation::Slice buffer_slice; + + // Describes how to dereference starting at that buffer to get to the buffer + // in question. + ShapeIndex gte_index; +}; + // Figures out how to access the buffers for all subshapes of hlo's operands and // for hlo itself (i.e. all the buffers produced by HLO). // @@ -1852,22 +1715,22 @@ static std::vector GetHloBufferSlices( return result; } -std::unique_ptr -IrEmitterUnnested::BuildKernelThunkFromBufferSlices( - absl::string_view name, Thunk::ThunkInfo thunk_info, - absl::Span slices, - std::function - bind_slice_to_ir_value) { - const auto& buffer_assn = ir_emitter_context_->buffer_assignment(); +std::unique_ptr IrEmitterUnnested::BuildKernelThunk( + const HloInstruction* inst, bool implements_whole_instruction) { + const BufferAssignment& buffer_assn = + ir_emitter_context_->buffer_assignment(); + + std::vector hlo_slices = + GetHloBufferSlices(inst, buffer_assn); // Figure out which buffer allocations need to be passed as arguments to our - // kernel. This is simply all of the allocations referenced in slices, + // kernel. This is simply all of the allocations referenced in hlo_slices, // plus the XLA temp buffer (if we have it). We always include the temp // buffer because even if the kernel itself doesn't use it, a nested // subcomputation within the kernel (e.g. a kMap's computation) might. std::unordered_set buffers_needed; - for (auto* slice : slices) { - buffers_needed.insert(slice->buffer_slice.allocation()); + for (const auto& hlo_buffer_slice : hlo_slices) { + buffers_needed.insert(hlo_buffer_slice.buffer_slice.allocation()); } absl::optional temp_buffer; for (const BufferAllocation& alloc : buffer_assn.Allocations()) { @@ -1896,7 +1759,7 @@ IrEmitterUnnested::BuildKernelThunkFromBufferSlices( return a->index() < b->index(); }); - llvm::Function* kernel = BuildKernelPrototype(name, non_constant_buffers); + llvm::Function* kernel = BuildKernelPrototype(*inst, non_constant_buffers); // Build a map from a BufferAllocation to the corresponding argument in our // kernel. @@ -1930,19 +1793,24 @@ IrEmitterUnnested::BuildKernelThunkFromBufferSlices( // For each buffer our kernel might want to touch, bind it to a value derived // from our kernel args. - for (auto* slice : slices) { - const BufferAllocation::Slice& buffer_slice = slice->buffer_slice; - const ShapeIndex& gte_index = slice->gte_index; + for (const auto& hlo_buffer_slice : hlo_slices) { + const HloInstruction* instr = hlo_buffer_slice.instr; + const ShapeIndex& index = hlo_buffer_slice.hlo_index; + const BufferAllocation::Slice& slice = hlo_buffer_slice.buffer_slice; + const ShapeIndex& gte_index = hlo_buffer_slice.gte_index; + + VLOG(3) << "Buffer for " << instr->ToString() << " at " << index.ToString() + << " is found in slice " << slice.ToString() << " at GTE index " + << gte_index.ToString(); llvm::Value* loc; - if (buffer_slice.allocation()->is_constant()) { + if (slice.allocation()->is_constant()) { loc = ir_emitter_context_->llvm_module()->getGlobalVariable( - llvm_ir::ConstantBufferAllocationToGlobalName( - *buffer_slice.allocation())); + llvm_ir::ConstantBufferAllocationToGlobalName(*slice.allocation())); CHECK_NE(loc, nullptr); } else { - loc = InBoundsGEP(kernel_args.at(buffer_slice.allocation()), - {b_.getInt64(buffer_slice.offset())}); + loc = InBoundsGEP(kernel_args.at(slice.allocation()), + {b_.getInt64(slice.offset())}); } // If gte_index is nonempty, we have to dereference `loc` to get to the @@ -1954,7 +1822,7 @@ IrEmitterUnnested::BuildKernelThunkFromBufferSlices( loc = Load(InBoundsGEP(loc, {b_.getInt64(idx)})); } - bind_slice_to_ir_value(slice, loc); + bindings_.BindHloToIrValue(*instr, loc, index); } // Bind the temp buffer so that nested subcomputations can find it if they @@ -1966,66 +1834,9 @@ IrEmitterUnnested::BuildKernelThunkFromBufferSlices( llvm::ConstantPointerNull::get(b_.getInt8PtrTy())); } - return absl::make_unique(thunk_info, non_constant_buffers, - std::string(kernel->getName())); -} - -std::unique_ptr IrEmitterUnnested::BuildKernelThunk( - const HloInstruction* inst, bool implements_whole_instruction) { - std::vector hlo_slices = - GetHloBufferSlices(inst, ir_emitter_context_->buffer_assignment()); - - std::vector slice_ptrs; - slice_ptrs.reserve(hlo_slices.size()); - for (auto& slice : hlo_slices) { - slice_ptrs.push_back(&slice); - } - - return BuildKernelThunkFromBufferSlices( - inst->name(), + return absl::make_unique( implements_whole_instruction ? GetThunkInfo(inst) : Thunk::ThunkInfo(), - slice_ptrs, [this](const BufferSlice* slice, llvm::Value* value) { - const HloBufferSlice* hlo_buffer_slice = - static_cast(slice); - const HloInstruction* instr = hlo_buffer_slice->instr; - const ShapeIndex& index = hlo_buffer_slice->hlo_index; - VLOG(3) << "Buffer for " << instr->ToString() << " at " - << index.ToString() << " is found in slice " - << hlo_buffer_slice->buffer_slice.ToString() << " at GTE index " - << hlo_buffer_slice->gte_index.ToString(); - - bindings_.BindHloToIrValue(*instr, value, index); - }); -} - -std::unique_ptr IrEmitterUnnested::BuildKernelThunkForMlir( - absl::string_view name, Thunk::ThunkInfo thunk_info, - absl::Span slices, - std::vector* ir_arrays) { - absl::flat_hash_set buffers_written; - std::vector slice_ptrs; - slice_ptrs.reserve(slices.size()); - for (auto& slice : slices) { - slice_ptrs.push_back(&slice); - if (slice.written) { - buffers_written.insert(slice.buffer_slice); - } - } - - ir_arrays->clear(); - return BuildKernelThunkFromBufferSlices( - name, thunk_info, slice_ptrs, - [&](const BufferSlice* slice, llvm::Value* value) { - const auto& mlir_slice = static_cast(*slice); - - llvm_ir::IrArray ir_array( - CastToTypedValue(mlir_slice.shape, value, &b_), mlir_slice.shape); - if (!buffers_written.contains(slice->buffer_slice)) { - ir_array.MarkInvariantOverWholeProgram(&value->getContext()); - } - - ir_arrays->push_back(ir_array); - }); + non_constant_buffers, std::string(kernel->getName())); } StatusOr> IrEmitterUnnested::BuildInitializerThunk( @@ -2232,7 +2043,7 @@ Status CheckConditionalBuffersShareAllocation( } // namespace -StatusOr> IrEmitterUnnested::BuildWhileThunk( +std::unique_ptr IrEmitterUnnested::BuildWhileThunk( const HloInstruction* hlo) { // Check that all while-related buffers share an allocation. TF_CHECK_OK(CheckWhileBuffersShareAllocation( @@ -2240,26 +2051,24 @@ StatusOr> IrEmitterUnnested::BuildWhileThunk( // Generate thunk sequence for while 'condition'. HloComputation* condition = hlo->while_condition(); - TF_ASSIGN_OR_RETURN(auto ir_emitter_condition, - IrEmitterUnnested::Create(hlo_module_config_, condition, - ir_emitter_context_)); - TF_RETURN_IF_ERROR(condition->Accept(ir_emitter_condition.get())); + IrEmitterUnnested ir_emitter_condition(hlo_module_config_, condition, + ir_emitter_context_); + TF_CHECK_OK(condition->Accept(&ir_emitter_condition)); // Generate thunk sequence for while 'body'. HloComputation* body = hlo->while_body(); - TF_ASSIGN_OR_RETURN( - auto ir_emitter_body, - IrEmitterUnnested::Create(hlo_module_config_, body, ir_emitter_context_)); - TF_RETURN_IF_ERROR(body->Accept(ir_emitter_body.get())); + IrEmitterUnnested ir_emitter_body(hlo_module_config_, body, + ir_emitter_context_); + TF_CHECK_OK(body->Accept(&ir_emitter_body)); - return std::unique_ptr(new WhileThunk( + return absl::make_unique( GetThunkInfo(hlo), GetAllocationSlice(*condition->root_instruction()), // cond result - ir_emitter_condition->ConsumeThunkSequence(), - ir_emitter_body->ConsumeThunkSequence())); + ir_emitter_condition.ConsumeThunkSequence(), + ir_emitter_body.ConsumeThunkSequence()); } -StatusOr> IrEmitterUnnested::BuildForThunk( +std::unique_ptr IrEmitterUnnested::BuildForThunk( const HloInstruction* hlo, const int64 loop_limit) { // Check that all while-related buffers share an allocation. TF_CHECK_OK(CheckWhileBuffersShareAllocation( @@ -2267,16 +2076,15 @@ StatusOr> IrEmitterUnnested::BuildForThunk( // Generate thunk sequence for while 'body' (will be used a For loop body). HloComputation* body = hlo->while_body(); - TF_ASSIGN_OR_RETURN( - auto ir_emitter_body, - IrEmitterUnnested::Create(hlo_module_config_, body, ir_emitter_context_)); - TF_RETURN_IF_ERROR(body->Accept(ir_emitter_body.get())); + IrEmitterUnnested ir_emitter_body(hlo_module_config_, body, + ir_emitter_context_); + TF_CHECK_OK(body->Accept(&ir_emitter_body)); - return std::unique_ptr(new ForThunk( - GetThunkInfo(hlo), loop_limit, ir_emitter_body->ConsumeThunkSequence())); + return absl::make_unique(GetThunkInfo(hlo), loop_limit, + ir_emitter_body.ConsumeThunkSequence()); } -StatusOr> IrEmitterUnnested::BuildConditionalThunk( +std::unique_ptr IrEmitterUnnested::BuildConditionalThunk( const HloInstruction* hlo) { // Check that the buffers used in conditional are shared with the operands and // result appropriately. @@ -2288,17 +2096,15 @@ StatusOr> IrEmitterUnnested::BuildConditionalThunk( for (int j = 0; j < hlo->branch_count(); ++j) { branch_operands.emplace_back(GetAllocationSlice(*hlo->operand(j + 1))); HloComputation* branch_computation = hlo->branch_computation(j); - TF_ASSIGN_OR_RETURN( - auto ir_emitter, - IrEmitterUnnested::Create(hlo_module_config_, branch_computation, - ir_emitter_context_)); - TF_CHECK_OK(branch_computation->Accept(ir_emitter.get())); - branch_thunks.push_back(std::move(*ir_emitter->ConsumeThunkSequence())); + IrEmitterUnnested ir_emitter(hlo_module_config_, branch_computation, + ir_emitter_context_); + TF_CHECK_OK(branch_computation->Accept(&ir_emitter)); + branch_thunks.push_back(std::move(*ir_emitter.ConsumeThunkSequence())); } - return std::unique_ptr(new ConditionalThunk( + return absl::make_unique( GetThunkInfo(hlo), GetAllocationSlice(*hlo->operand(0)), branch_operands, - std::move(branch_thunks))); + std::move(branch_thunks)); } Status IrEmitterUnnested::EmitTargetElementLoopInThunk( diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h index b9146dd8fae..019fcdf21db 100644 --- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h +++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h @@ -17,7 +17,6 @@ limitations under the License. #define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_IR_EMITTER_UNNESTED_H_ #include "absl/container/inlined_vector.h" -#include "tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h" #include "tensorflow/compiler/xla/service/gpu/ir_emitter.h" #include "tensorflow/compiler/xla/service/gpu/kernel_mapping_scheme.h" #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h" @@ -29,40 +28,6 @@ limitations under the License. namespace xla { namespace gpu { -struct BufferSlice { - // The root buffer to look at. - BufferAllocation::Slice buffer_slice; - - // Describes how to dereference starting at that buffer to get to the buffer - // in question. - ShapeIndex gte_index; -}; - -// Describes how to access a particular subshape for an HLO. For instance if -// `.hlo_index` is {1} and `.gte_index` is {3, 4} then buffer for `.instr` at -// ShapeIndex {1} (i.e. the buffer for the second tuple element of hlo) is -// found at `.buffer_slice`[3][4]. That is, `.slice` is a void***, which we -// dereference twice -- first at index 3, and then at index 4 -- to get the -// address of our buffer. -struct HloBufferSlice : public BufferSlice { - const HloInstruction* instr; - ShapeIndex hlo_index; -}; - -struct MlirBufferSlice : public BufferSlice { - // The buffer is modified by the kernel. - bool written; - - Shape shape; -}; - -struct MlirEmitterInput { - mlir::Operation* op; - absl::string_view name; - Thunk::ThunkInfo thunk_info; - MlirBufferSlice extra_slice; -}; - // Emits LLVM IR for an "unnested computation". // // An unnested computation is an HloComputation which you run by executing one @@ -124,14 +89,12 @@ class IrEmitterUnnested : public IrEmitter, const string& loop_name, llvm::Value* tile_height, llvm::Value* tile_width, KernelSupportLibrary* ksl)>; + IrEmitterUnnested(const HloModuleConfig& hlo_module_config, + const HloComputation* hlo_computation, + IrEmitterContext* ir_emitter_context); IrEmitterUnnested(const IrEmitterUnnested&) = delete; IrEmitterUnnested& operator=(const IrEmitterUnnested&) = delete; - static StatusOr> Create( - const HloModuleConfig& hlo_module_config, - const HloComputation* hlo_computation, - IrEmitterContext* ir_emitter_context); - // Transfers the ownship of thunk_sequence_ out. std::unique_ptr ConsumeThunkSequence() { return std::make_unique(std::move(thunk_sequence_)); @@ -161,7 +124,6 @@ class IrEmitterUnnested : public IrEmitter, Status HandleScatter(HloInstruction* scatter) override; Status HandleSelect(HloInstruction* select) override; Status HandleSort(HloInstruction* sort) override; - Status EmitMlirSort(MlirEmitterInput input); Status HandleTriangularSolve(HloInstruction* hlo) override; Status HandleTupleSelect(HloInstruction* tuple_select) override; Status HandleAllReduce(HloInstruction* crs) override; @@ -186,10 +148,6 @@ class IrEmitterUnnested : public IrEmitter, Status Postprocess(HloInstruction* hlo) override; private: - IrEmitterUnnested(const HloModuleConfig& hlo_module_config, - const HloComputation* hlo_computation, - IrEmitterContext* ir_emitter_context); - // Add a owning Thunk object to the thunk sequence. void AddThunkToThunkSequence(std::unique_ptr thunk) override { thunk_sequence_.emplace_back(std::move(thunk)); @@ -306,7 +264,8 @@ class IrEmitterUnnested : public IrEmitter, // Builds the prototype of the IR kernel for `inst` and adds it to the module. // This kernel takes as arguments pointers to the given buffer allocations. llvm::Function* BuildKernelPrototype( - absl::string_view name, absl::Span args); + const HloInstruction& inst, + absl::Span args); // Helper for writing extra outputs from inside a reduce kernel. Status EmitExtraOutputsForReduce( @@ -531,12 +490,6 @@ class IrEmitterUnnested : public IrEmitter, HloComputation* reducer, llvm::Type* element_type, llvm::Value* partial_result_address); - std::unique_ptr BuildKernelThunkFromBufferSlices( - absl::string_view name, Thunk::ThunkInfo thunk_info, - absl::Span slices, - std::function - bind_slice_to_ir_value); - // Returns a KernelThunk that invokes the kernel emitted for `inst`. The // caller needs to make sure `inst` outlives the lifetime of the returned // Thunk object. 'implements_whole_instruction' specifies whether this @@ -545,11 +498,6 @@ class IrEmitterUnnested : public IrEmitter, std::unique_ptr BuildKernelThunk( const HloInstruction* inst, bool implements_whole_instruction); - std::unique_ptr BuildKernelThunkForMlir( - absl::string_view name, Thunk::ThunkInfo thunk_info, - absl::Span slices, - std::vector* ir_arrays); - // Returns a thunk that, given a reduce or select-and-scatter op, // initializes its memory to the appropriate initial value. StatusOr> BuildInitializerThunk( @@ -557,18 +505,17 @@ class IrEmitterUnnested : public IrEmitter, // Returns a WhileThunk that invokes thunk sequences for 'condition' and // 'body' sub-computations of while instruction 'hlo'. - StatusOr> BuildWhileThunk(const HloInstruction* hlo); + std::unique_ptr BuildWhileThunk(const HloInstruction* hlo); // Returns a ForThunk which executes 'loop_limit' invocations of a thunk // sequence from the 'body' sub-computation of the while instruction 'hlo'. - StatusOr> BuildForThunk(const HloInstruction* hlo, - const int64 loop_limit); + std::unique_ptr BuildForThunk(const HloInstruction* hlo, + const int64 loop_limit); // Returns a ConditionalThunk which executes the thunk sequence for the // 'branch_computation' corresponding to the predicate/branch_index of the // given conditional instruction. - StatusOr> BuildConditionalThunk( - const HloInstruction* hlo); + std::unique_ptr BuildConditionalThunk(const HloInstruction* hlo); // Emits current thread id with the given type. // @@ -598,9 +545,6 @@ class IrEmitterUnnested : public IrEmitter, absl::optional thread_id_filter = absl::nullopt, absl::optional block_id_filter = absl::nullopt); - StatusOr GetOrCreateSubComputationFromRegion( - mlir::Region* region); - // Returns the last generated thunk. Thunk* LastThunk() const { return thunk_sequence_.back().get(); } @@ -611,14 +555,6 @@ class IrEmitterUnnested : public IrEmitter, // The HloComputation that this IrEmitter emits code for. const HloComputation* hlo_computation_; - - mlir::OwningModuleRef mlir_scratch_module_; - - // This is for cache-purpose only. It has no significant semantics. - mlir::LhloDialectEmitter lhlo_scratch_emitter_; - - absl::flat_hash_map> - scratch_nested_computations_; }; } // namespace gpu diff --git a/tensorflow/compiler/xla/service/gpu/tests/BUILD b/tensorflow/compiler/xla/service/gpu/tests/BUILD index 809b277317f..a2bddd2d0d7 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/BUILD +++ b/tensorflow/compiler/xla/service/gpu/tests/BUILD @@ -458,35 +458,6 @@ xla_test( ], ) -tf_cc_test( - name = "sorting_test", - srcs = [ - "sorting_test.cc", - ], - tags = tf_cuda_tests_tags() + [ - "no_rocm", - ], - deps = [ - ":gpu_codegen_test", - "//tensorflow/compiler/xla:debug_options_flags", - "//tensorflow/compiler/xla:statusor", - "//tensorflow/compiler/xla:xla_proto_cc", - "//tensorflow/compiler/xla/service:gpu_plugin", - "//tensorflow/compiler/xla/service:hlo", - "//tensorflow/compiler/xla/service:hlo_module_config", - "//tensorflow/compiler/xla/service:hlo_parser", - "//tensorflow/compiler/xla/service/gpu:gpu_executable", - "//tensorflow/compiler/xla/tests:filecheck", - "//tensorflow/compiler/xla/tests:hlo_test_base", - "//tensorflow/compiler/xla/tests:llvm_irgen_test_base", - "//tensorflow/core:lib", - "//tensorflow/core:test", - "//tensorflow/core:test_main", - "//tensorflow/stream_executor/lib", - "@com_google_absl//absl/memory", - ], -) - tf_cc_binary( name = "hlo_to_llvm_ir", srcs = ["hlo_to_llvm_ir.cc"], diff --git a/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo b/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo index 4d29a8df116..272c9a25769 100644 --- a/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo +++ b/tensorflow/compiler/xla/service/gpu/tests/sorting.hlo @@ -8,162 +8,162 @@ compare { ROOT lt = pred[] compare(p.0.lhs, p.0.rhs), direction=LT } -// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) +// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 -// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 -// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 -// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] -// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 -// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] -// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 +// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] +// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 +// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] +// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: call void @region_0_4(float* [[TMP16]], float* [[TMP17]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP18]], 0 +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: call void @compare(float* [[TMP12]], float* [[TMP13]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP14:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP14]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] -// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 +// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = load float, float* [[TMP13]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: store float [[TMP16]], float* [[TMP18]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define internal void @region_0_4(float* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) +// CHECK: define internal void @compare(float* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) // CHECK-NEXT: entry: -// CHECK-NEXT: [[COMPARE_3_TYPED:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[ARG_0_1_TYPED:%.*]], align 4 -// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[ARG_1_2_TYPED:%.*]], align 4 +// CHECK-NEXT: [[LT_TYPED:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[P_0_LHS_TYPED]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[P_0_RHS_TYPED]], align 4 // CHECK-NEXT: [[TMP2:%.*]] = fcmp olt float [[TMP0]], [[TMP1]] // CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i8 -// CHECK-NEXT: store i8 [[TMP3]], i8* [[COMPARE_3_TYPED]], align 1 -// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[COMPARE_3_TYPED]], align 1 -// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG:%.*]], align 1 +// CHECK-NEXT: store i8 [[TMP3]], i8* [[LT_TYPED]], align 1 +// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[LT_TYPED]], align 1 +// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG]], align 1 // CHECK-NEXT: ret void -// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) { +// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) { // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 -// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 -// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP11:%.*]] = xor i64 [[TMP8]], 3 -// CHECK-NEXT: [[TMP12:%.*]] = icmp slt i64 [[TMP8]], [[TMP11]] -// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], 3 -// CHECK-NEXT: [[TMP14:%.*]] = and i1 [[TMP12]], [[TMP13]] -// CHECK-NEXT: br i1 [[TMP14]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = xor i64 [[TMP4]], 3 +// CHECK-NEXT: [[TMP8:%.*]] = icmp slt i64 [[TMP4]], [[TMP7]] +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], 3 +// CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP8]], [[TMP9]] +// CHECK-NEXT: br i1 [[TMP10]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP8]] -// CHECK-NEXT: call void @region_0_4(float* [[TMP15]], float* [[TMP16]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP17:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP17]], 0 +// CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: call void @compare(float* [[TMP11]], float* [[TMP12]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP13:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP13]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP18:%.*]] = load float, float* [[TMP15]], align 4 -// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP8]] -// CHECK-NEXT: store float [[TMP18]], float* [[TMP20]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 +// CHECK-NEXT: [[TMP14:%.*]] = load float, float* [[TMP11]], align 4 +// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: store float [[TMP14]], float* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]]) { +// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC1:%.*]]) { // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP4]] to i64 -// CHECK-NEXT: [[TMP5:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP5]] to i64 -// CHECK-NEXT: [[TMP6:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP6]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP7:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP8:%.*]] = urem i64 [[TMP7]], 2 -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP10:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP10]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP8]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 -// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] -// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 -// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] -// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 +// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] +// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 +// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] +// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: call void @region_0_4(float* [[TMP16]], float* [[TMP17]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP18:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP18]], 0 +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: call void @compare(float* [[TMP12]], float* [[TMP13]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP14:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP14]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP19:%.*]] = load float, float* [[TMP16]], align 4 -// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP11]] -// CHECK-NEXT: store float [[TMP19]], float* [[TMP21]], align 4 -// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP1]], i64 0, i64 [[TMP9]], i64 [[TMP12]] -// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 +// CHECK-NEXT: [[TMP15:%.*]] = load float, float* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = load float, float* [[TMP13]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP15]], float* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: store float [[TMP16]], float* [[TMP18]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] ENTRY main { x = f32[2, 3] parameter(0) @@ -182,198 +182,210 @@ compare { ROOT lt = pred[] compare(p.1.lhs, p.1.rhs), direction=LT } -// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* -// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 -// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 -// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* +// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0]], i64 0 +// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1]], i64 0 +// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3]], i64 0 +// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 -// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP10]], 2 -// CHECK-NEXT: [[TMP14:%.*]] = xor i64 [[TMP13]], 1 -// CHECK-NEXT: [[TMP15:%.*]] = icmp slt i64 [[TMP13]], [[TMP14]] -// CHECK-NEXT: [[TMP16:%.*]] = icmp slt i64 [[TMP14]], 3 -// CHECK-NEXT: [[TMP17:%.*]] = and i1 [[TMP15]], [[TMP16]] -// CHECK-NEXT: br i1 [[TMP17]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP4]], 2 +// CHECK-NEXT: [[TMP8:%.*]] = xor i64 [[TMP7]], 1 +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], [[TMP8]] +// CHECK-NEXT: [[TMP10:%.*]] = icmp slt i64 [[TMP8]], 3 +// CHECK-NEXT: [[TMP11:%.*]] = and i1 [[TMP9]], [[TMP10]] +// CHECK-NEXT: br i1 [[TMP11]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP14]] -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP14]] -// CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: call void @region_0_6(i32* [[TMP18]], i32* [[TMP19]], float* [[TMP20]], float* [[TMP21]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP22:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP22]], 0 +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: call void @compare(i32* [[TMP12]], i32* [[TMP13]], float* [[TMP14]], float* [[TMP15]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP16:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP16]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP18]], align 4 -// CHECK-NEXT: [[TMP24:%.*]] = load i32, i32* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: store i32 [[TMP23]], i32* [[TMP25]], align 4 -// CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP14]] -// CHECK-NEXT: store i32 [[TMP24]], i32* [[TMP26]], align 4 -// CHECK-NEXT: [[TMP27:%.*]] = load float, float* [[TMP20]], align 4 -// CHECK-NEXT: [[TMP28:%.*]] = load float, float* [[TMP21]], align 4 -// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: store float [[TMP27]], float* [[TMP29]], align 4 -// CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP14]] -// CHECK-NEXT: store float [[TMP28]], float* [[TMP30]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = load i32, i32* [[TMP13]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: store i32 [[TMP18]], i32* [[TMP20]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = load float, float* [[TMP14]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = load float, float* [[TMP15]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP21]], float* [[TMP23]], align 4 +// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP8]] +// CHECK-NEXT: store float [[TMP22]], float* [[TMP24]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define internal void @region_0_6(i32* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], i32* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) +// CHECK: define internal void @compare(i32* dereferenceable(4) [[P_0_LHS_TYPED:%.*]], i32* dereferenceable(4) [[P_0_RHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_LHS_TYPED:%.*]], float* dereferenceable(4) [[P_1_RHS_TYPED:%.*]], i8* dereferenceable(1) [[OUTPUT_ARG:%.*]]) // CHECK-NEXT: entry: -// CHECK-NEXT: [[COMPARE_5_TYPED:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[ARG_2_3_TYPED:%.*]], align 4 -// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[ARG_3_4_TYPED:%.*]], align 4 +// CHECK-NEXT: [[LT_TYPED:%.*]] = alloca i8, align 1 +// CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[P_1_LHS_TYPED]], align 4 +// CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[P_1_RHS_TYPED]], align 4 // CHECK-NEXT: [[TMP2:%.*]] = fcmp olt float [[TMP0]], [[TMP1]] // CHECK-NEXT: [[TMP3:%.*]] = zext i1 [[TMP2]] to i8 -// CHECK-NEXT: store i8 [[TMP3]], i8* [[COMPARE_5_TYPED]], align 1 -// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[COMPARE_5_TYPED]], align 1 -// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG:%.*]], align 1 +// CHECK-NEXT: store i8 [[TMP3]], i8* [[LT_TYPED]], align 1 +// CHECK-NEXT: [[LOAD_RET_VALUE:%.*]] = load i8, i8* [[LT_TYPED]], align 1 +// CHECK-NEXT: store i8 [[LOAD_RET_VALUE]], i8* [[OUTPUT_ARG]], align 1 // CHECK-NEXT: ret void -// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort__1(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* -// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 -// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 -// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* +// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2:%.*]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3:%.*]], i64 0 +// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 -// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP13:%.*]] = xor i64 [[TMP10]], 3 -// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP10]], [[TMP13]] -// CHECK-NEXT: [[TMP15:%.*]] = icmp slt i64 [[TMP13]], 3 -// CHECK-NEXT: [[TMP16:%.*]] = and i1 [[TMP14]], [[TMP15]] -// CHECK-NEXT: br i1 [[TMP16]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP7:%.*]] = xor i64 [[TMP4]], 3 +// CHECK-NEXT: [[TMP8:%.*]] = icmp slt i64 [[TMP4]], [[TMP7]] +// CHECK-NEXT: [[TMP9:%.*]] = icmp slt i64 [[TMP7]], 3 +// CHECK-NEXT: [[TMP10:%.*]] = and i1 [[TMP8]], [[TMP9]] +// CHECK-NEXT: br i1 [[TMP10]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP10]] -// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP10]] -// CHECK-NEXT: call void @region_0_6(i32* [[TMP17]], i32* [[TMP18]], float* [[TMP19]], float* [[TMP20]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP21:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP21]], 0 +// CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: call void @compare(i32* [[TMP11]], i32* [[TMP12]], float* [[TMP13]], float* [[TMP14]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP15:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP15]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP17]], align 4 -// CHECK-NEXT: [[TMP23:%.*]] = load i32, i32* [[TMP18]], align 4 -// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP10]] -// CHECK-NEXT: store i32 [[TMP22]], i32* [[TMP24]], align 4 -// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: store i32 [[TMP23]], i32* [[TMP25]], align 4 -// CHECK-NEXT: [[TMP26:%.*]] = load float, float* [[TMP19]], align 4 -// CHECK-NEXT: [[TMP27:%.*]] = load float, float* [[TMP20]], align 4 -// CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP10]] -// CHECK-NEXT: store float [[TMP26]], float* [[TMP28]], align 4 -// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP13]] -// CHECK-NEXT: store float [[TMP27]], float* [[TMP29]], align 4 +// CHECK-NEXT: [[TMP16:%.*]] = load i32, i32* [[TMP11]], align 4 +// CHECK-NEXT: [[TMP17:%.*]] = load i32, i32* [[TMP12]], align 4 +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: store i32 [[TMP16]], i32* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store i32 [[TMP17]], i32* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP20:%.*]] = load float, float* [[TMP13]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = load float, float* [[TMP14]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP4]] +// CHECK-NEXT: store float [[TMP20]], float* [[TMP22]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP7]] +// CHECK-NEXT: store float [[TMP21]], float* [[TMP23]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] -// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) +// CHECK: define void @sort__2(i8* noalias align 64 dereferenceable(24) [[ALLOC0:%.*]], i8* noalias align 64 dereferenceable(24) [[ALLOC1:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC2:%.*]], i8* noalias align 16 dereferenceable(24) [[ALLOC3:%.*]], i8* noalias align 64 dereferenceable(16) [[ALLOC4:%.*]]) // CHECK-NEXT: entry: // CHECK-NEXT: [[COMPARE_RETURN_BUFFER:%.*]] = alloca i8, align 1 -// CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 -// CHECK-NEXT: [[TMP1:%.*]] = bitcast i8* [[TMP0]] to [2 x [3 x i32]]* -// CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 -// CHECK-NEXT: [[TMP3:%.*]] = bitcast i8* [[TMP2]] to [2 x [3 x float]]* -// CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 -// CHECK-NEXT: [[TMP5:%.*]] = bitcast i8* [[TMP4]] to [2 x i8*]* -// CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 -// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP6]] to i64 -// CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 -// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP7]] to i64 -// CHECK-NEXT: [[TMP8:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 -// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP8]], [[THREAD_ID]] +// CHECK-NEXT: [[SORT_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC4:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED:%.*]] = bitcast i8* [[SORT_RAW]] to [2 x i8*]* +// CHECK-NEXT: [[SORT_RAW1:%.*]] = getelementptr inbounds i8, i8* [[ALLOC0:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED2:%.*]] = bitcast i8* [[SORT_RAW1]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[SORT_RAW3:%.*]] = getelementptr inbounds i8, i8* [[ALLOC1:%.*]], i64 0 +// CHECK-NEXT: [[SORT_TYPED4:%.*]] = bitcast i8* [[SORT_RAW3]] to [2 x [3 x float]]* +// CHECK-NEXT: [[X_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC2:%.*]], i64 0 +// CHECK-NEXT: [[X_TYPED:%.*]] = bitcast i8* [[X_RAW]] to [2 x [3 x i32]]* +// CHECK-NEXT: [[Y_RAW:%.*]] = getelementptr inbounds i8, i8* [[ALLOC3:%.*]], i64 0 +// CHECK-NEXT: [[Y_TYPED:%.*]] = bitcast i8* [[Y_RAW]] to [2 x [3 x float]]* +// CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.ctaid.x(), !range !6 +// CHECK-NEXT: [[BLOCK_ID:%.*]] = zext i32 [[TMP0]] to i64 +// CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !7 +// CHECK-NEXT: [[THREAD_ID:%.*]] = zext i32 [[TMP1]] to i64 +// CHECK-NEXT: [[TMP2:%.*]] = mul nuw nsw i64 [[BLOCK_ID]], 4 +// CHECK-NEXT: [[LINEAR_INDEX:%.*]] = add nuw nsw i64 [[TMP2]], [[THREAD_ID]] // CHECK-NEXT: [[LINEAR_INDEX_IN_RANGE:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 // CHECK-NEXT: call void @llvm.assume(i1 [[LINEAR_INDEX_IN_RANGE]]) -// CHECK-NEXT: [[TMP9:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 -// CHECK-NEXT: [[TMP10:%.*]] = urem i64 [[TMP9]], 2 -// CHECK-NEXT: [[TMP11:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 -// CHECK-NEXT: [[TMP12:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 -// CHECK-NEXT: br i1 [[TMP12]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] +// CHECK-NEXT: [[TMP3:%.*]] = udiv i64 [[LINEAR_INDEX]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = urem i64 [[TMP3]], 2 +// CHECK-NEXT: [[TMP5:%.*]] = udiv i64 [[LINEAR_INDEX]], 2 +// CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[LINEAR_INDEX]], 4 +// CHECK-NEXT: br i1 [[TMP6]], label [[SORT_IN_BOUNDS_TRUE:%.*]], label [[SORT_IN_BOUNDS_AFTER:%.*]] // CHECK: sort.in_bounds-after: -// CHECK-NEXT: [[TMP13:%.*]] = bitcast [2 x [3 x i32]]* [[TMP1]] to i8* -// CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[TMP5]], i64 0, i64 0 -// CHECK-NEXT: store i8* [[TMP13]], i8** [[TMP14]], align 8 -// CHECK-NEXT: [[TMP15:%.*]] = bitcast [2 x [3 x float]]* [[TMP3]] to i8* -// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[TMP5]], i64 0, i64 1 -// CHECK-NEXT: store i8* [[TMP15]], i8** [[TMP16]], align 8 +// CHECK-NEXT: [[TMP7:%.*]] = bitcast [2 x [3 x i32]]* [[SORT_TYPED2]] to i8* +// CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[SORT_TYPED]], i64 0, i64 0 +// CHECK-NEXT: store i8* [[TMP7]], i8** [[TMP8]], align 8 +// CHECK-NEXT: [[TMP9:%.*]] = bitcast [2 x [3 x float]]* [[SORT_TYPED4]] to i8* +// CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[SORT_TYPED]], i64 0, i64 1 +// CHECK-NEXT: store i8* [[TMP9]], i8** [[TMP10]], align 8 // CHECK-NEXT: ret void // CHECK: sort.in_bounds-true: -// CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP10]], 2 -// CHECK-NEXT: [[TMP18:%.*]] = xor i64 [[TMP17]], 1 -// CHECK-NEXT: [[TMP19:%.*]] = icmp slt i64 [[TMP17]], [[TMP18]] -// CHECK-NEXT: [[TMP20:%.*]] = icmp slt i64 [[TMP18]], 3 -// CHECK-NEXT: [[TMP21:%.*]] = and i1 [[TMP19]], [[TMP20]] -// CHECK-NEXT: br i1 [[TMP21]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] +// CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP4]], 2 +// CHECK-NEXT: [[TMP12:%.*]] = xor i64 [[TMP11]], 1 +// CHECK-NEXT: [[TMP13:%.*]] = icmp slt i64 [[TMP11]], [[TMP12]] +// CHECK-NEXT: [[TMP14:%.*]] = icmp slt i64 [[TMP12]], 3 +// CHECK-NEXT: [[TMP15:%.*]] = and i1 [[TMP13]], [[TMP14]] +// CHECK-NEXT: br i1 [[TMP15]], label [[SMALLER_COMPARISON_INDEX_TRUE:%.*]], label [[SMALLER_COMPARISON_INDEX_AFTER:%.*]] // CHECK: smaller_comparison_index-after: // CHECK-NEXT: br label [[SORT_IN_BOUNDS_AFTER]] // CHECK: smaller_comparison_index-true: -// CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP18]] -// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP17]] -// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP18]] -// CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP17]] -// CHECK-NEXT: call void @region_0_6(i32* [[TMP22]], i32* [[TMP23]], float* [[TMP24]], float* [[TMP25]], i8* [[COMPARE_RETURN_BUFFER]]) -// CHECK-NEXT: [[TMP26:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 -// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP26]], 0 +// CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP12]] +// CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP11]] +// CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP12]] +// CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP11]] +// CHECK-NEXT: call void @compare(i32* [[TMP16]], i32* [[TMP17]], float* [[TMP18]], float* [[TMP19]], i8* [[COMPARE_RETURN_BUFFER]]) +// CHECK-NEXT: [[TMP20:%.*]] = load i8, i8* [[COMPARE_RETURN_BUFFER]], align 1 +// CHECK-NEXT: [[BOOLEAN_PREDICATE:%.*]] = icmp ne i8 [[TMP20]], 0 // CHECK-NEXT: br i1 [[BOOLEAN_PREDICATE]], label [[IS_SMALLER_THAN_TRUE:%.*]], label [[IS_SMALLER_THAN_AFTER:%.*]] // CHECK: is_smaller_than-after: // CHECK-NEXT: br label [[SMALLER_COMPARISON_INDEX_AFTER]] // CHECK: is_smaller_than-true: -// CHECK-NEXT: [[TMP27:%.*]] = load i32, i32* [[TMP22]], align 4 -// CHECK-NEXT: [[TMP28:%.*]] = load i32, i32* [[TMP23]], align 4 -// CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP17]] -// CHECK-NEXT: store i32 [[TMP27]], i32* [[TMP29]], align 4 -// CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[TMP1]], i64 0, i64 [[TMP11]], i64 [[TMP18]] -// CHECK-NEXT: store i32 [[TMP28]], i32* [[TMP30]], align 4 -// CHECK-NEXT: [[TMP31:%.*]] = load float, float* [[TMP24]], align 4 -// CHECK-NEXT: [[TMP32:%.*]] = load float, float* [[TMP25]], align 4 -// CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP17]] -// CHECK-NEXT: store float [[TMP31]], float* [[TMP33]], align 4 -// CHECK-NEXT: [[TMP34:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[TMP3]], i64 0, i64 [[TMP11]], i64 [[TMP18]] -// CHECK-NEXT: store float [[TMP32]], float* [[TMP34]], align 4 +// CHECK-NEXT: [[TMP21:%.*]] = load i32, i32* [[TMP16]], align 4 +// CHECK-NEXT: [[TMP22:%.*]] = load i32, i32* [[TMP17]], align 4 +// CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP11]] +// CHECK-NEXT: store i32 [[TMP21]], i32* [[TMP23]], align 4 +// CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds [2 x [3 x i32]], [2 x [3 x i32]]* [[SORT_TYPED2]], i64 0, i64 [[TMP5]], i64 [[TMP12]] +// CHECK-NEXT: store i32 [[TMP22]], i32* [[TMP24]], align 4 +// CHECK-NEXT: [[TMP25:%.*]] = load float, float* [[TMP18]], align 4 +// CHECK-NEXT: [[TMP26:%.*]] = load float, float* [[TMP19]], align 4 +// CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP11]] +// CHECK-NEXT: store float [[TMP25]], float* [[TMP27]], align 4 +// CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds [2 x [3 x float]], [2 x [3 x float]]* [[SORT_TYPED4]], i64 0, i64 [[TMP5]], i64 [[TMP12]] +// CHECK-NEXT: store float [[TMP26]], float* [[TMP28]], align 4 // CHECK-NEXT: br label [[IS_SMALLER_THAN_AFTER]] ENTRY main { x = s32[2, 3] parameter(0) diff --git a/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc b/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc deleted file mode 100644 index 197a0c6cfeb..00000000000 --- a/tensorflow/compiler/xla/service/gpu/tests/sorting_test.cc +++ /dev/null @@ -1,71 +0,0 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include - -#include "tensorflow/compiler/xla/service/gpu/gpu_executable.h" -#include "tensorflow/compiler/xla/service/gpu/tests/gpu_codegen_test.h" -#include "tensorflow/compiler/xla/service/hlo_instruction.h" -#include "tensorflow/compiler/xla/service/hlo_module_config.h" -#include "tensorflow/compiler/xla/service/hlo_parser.h" -#include "tensorflow/compiler/xla/statusor.h" -#include "tensorflow/compiler/xla/tests/filecheck.h" -#include "tensorflow/compiler/xla/tests/hlo_test_base.h" -#include "tensorflow/compiler/xla/xla.pb.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/stream_executor/lib/statusor.h" - -namespace xla { -namespace gpu { - -namespace { - -class SortingTest : public GpuCodegenTest { - protected: - HloModuleConfig ConfigWithoutLayoutAssignment() { - HloModuleConfig config; - auto debug_options = HloTestBase::GetDebugOptionsForTest(); - // Disable layout_assignment to use the preassigned layouts. - debug_options.add_xla_disable_hlo_passes("layout-assignment"); - config.set_debug_options(debug_options); - return config; - } -}; - -TEST_F(SortingTest, Regression1) { - const char* hlo_text = R"( -HloModule TestModule - -compare { - p.0.lhs = f32[] parameter(0) - p.0.rhs = f32[] parameter(1) - ROOT lt = pred[] compare(p.0.lhs, p.0.rhs), direction=LT -} - -ENTRY TestComputation { - x = f32[3, 2]{1, 0} parameter(0) - x.copy = f32[3, 2]{0, 1} copy(x) - ROOT sort = f32[3, 2]{0, 1} sort(x.copy), dimensions={1}, to_apply=compare -} - -)"; - - EXPECT_TRUE(RunAndCompareNoHloPasses(hlo_text, ErrorSpec{1e-5, 1e-5})); -} - -} // namespace -} // namespace gpu -} // namespace xla diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc index 2963d546380..b01ae2efe43 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc @@ -415,10 +415,9 @@ llvm::Instruction* AddRangeMetadata(int64 lower, int64 upper, return inst; } -string IrName(absl::string_view a) { - std::string s(a); - s.erase(std::remove(s.begin(), s.end(), '%'), s.end()); - return s; +string IrName(string a) { + a.erase(std::remove(a.begin(), a.end(), '%'), a.end()); + return a; } string IrName(absl::string_view a, absl::string_view b) { diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h index c0a55e4da33..642965b6470 100644 --- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h +++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.h @@ -87,7 +87,7 @@ string DumpModuleToString(const llvm::Module& module); // - joining all of the nonempty inputs by '.', and then // - removing all '%'s. // -string IrName(absl::string_view a); +string IrName(string a); string IrName(absl::string_view a, absl::string_view b); string IrName(const HloInstruction* a, absl::string_view b = ""); From 2be895ce466fa3e3956e23452e191b909700d201 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Wed, 12 Aug 2020 16:51:20 -0700 Subject: [PATCH 0940/1017] Improve callback timing check by restricting it to custom callbacks and using the mean of 5 batches to get more accurate results. PiperOrigin-RevId: 326342300 Change-Id: I192001a64da11c5fa66de46a9a4e01b2b090a184 --- tensorflow/python/keras/BUILD | 2 +- tensorflow/python/keras/callbacks.py | 42 +++++++++++++++-------- tensorflow/python/keras/callbacks_test.py | 35 +++++++++++++++---- 3 files changed, 57 insertions(+), 22 deletions(-) diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD index 24c5b9de8ca..d8eff0f2260 100755 --- a/tensorflow/python/keras/BUILD +++ b/tensorflow/python/keras/BUILD @@ -522,7 +522,7 @@ tf_py_test( size = "medium", srcs = ["callbacks_test.py"], python_version = "PY3", - shard_count = 4, + shard_count = 6, tags = [ "no_oss", "notsan", diff --git a/tensorflow/python/keras/callbacks.py b/tensorflow/python/keras/callbacks.py index 5a191263241..ff3eef8b6e9 100644 --- a/tensorflow/python/keras/callbacks.py +++ b/tensorflow/python/keras/callbacks.py @@ -241,9 +241,12 @@ class CallbackList(object): # pylint: enable=protected-access # Performance check: Check batch hooks for slowness compared to batch time. - self._timing = {} - self._check_timing = False + # Only run check for custom callbacks (i.e. not present in this file). + self._check_timing = self.__class__ not in globals() + self._num_batches_for_timing_check = 5 + self._hook_times = {} self._batch_start_time = None + self._batch_times = [] def _add_default_callbacks(self, add_history, add_progbar): """Adds `Callback`s that are always present.""" @@ -294,7 +297,6 @@ class CallbackList(object): def _call_batch_begin_hook(self, mode, batch, logs): """Helper function for `on_*_batch_begin` methods.""" hook_name = 'on_{mode}_batch_begin'.format(mode=mode) - self._check_timing = batch == 1 and hook_name not in self._timing self._call_batch_hook_helper(hook_name, batch, logs) if self._check_timing: @@ -304,31 +306,39 @@ class CallbackList(object): """Helper function for `on_*_batch_end` methods.""" hook_name = 'on_{mode}_batch_end'.format(mode=mode) - if self._check_timing: + if self._check_timing and batch >= 1: batch_time = time.time() - self._batch_start_time + self._batch_times.append(batch_time) self._call_batch_hook_helper(hook_name, batch, logs) - if self._check_timing: + if len(self._batch_times) >= self._num_batches_for_timing_check: end_hook_name = hook_name begin_hook_name = 'on_{mode}_batch_begin'.format(mode=mode) + avg_batch_time = sum(self._batch_times) / len(self._batch_times) + avg_end_hook_time = sum(self._hook_times[end_hook_name]) / len( + self._hook_times[end_hook_name]) + avg_begin_hook_time = sum(self._hook_times[begin_hook_name]) / len( + self._hook_times[begin_hook_name]) - threshold_time = 1.5 * batch_time - warning_msg = ('Callbacks method `{hook}` is slow compared to ' + threshold_time = 1.5 * avg_batch_time + warning_msg = ('Callback method `{hook}` is slow compared to ' 'the batch time (batch time: {batch_time:.4f}s vs ' - '`{hook}` time: {cbk_time:.4f}s). Check your callbacks.') - if self._timing[begin_hook_name] > threshold_time: + '`{hook}` time: {hook_time:.4f}s). Check your callbacks.') + if avg_begin_hook_time > threshold_time: logging.warning(warning_msg.format( hook=begin_hook_name, - batch_time=batch_time, - cbk_time=self._timing[begin_hook_name])) - if self._timing[end_hook_name] > threshold_time: + batch_time=avg_batch_time, + hook_time=avg_begin_hook_time)) + if avg_end_hook_time > threshold_time: logging.warning(warning_msg.format( hook=end_hook_name, - batch_time=batch_time, - cbk_time=self._timing[end_hook_name])) + batch_time=avg_batch_time, + hook_time=avg_end_hook_time)) self._check_timing = False self._batch_start_time = None + self._batch_times = [] + self._hook_times = {} def _call_batch_hook_helper(self, hook_name, batch, logs): """Helper function for `on_*_batch_*` methods.""" @@ -347,7 +357,9 @@ class CallbackList(object): hook(batch, numpy_logs) if self._check_timing: - self._timing[hook_name] = time.time() - start_time + if hook_name not in self._hook_times: + self._hook_times[hook_name] = [] + self._hook_times[hook_name].append(time.time() - start_time) def _call_begin_hook(self, mode): """Helper function for on_{train|test|predict}_begin methods.""" diff --git a/tensorflow/python/keras/callbacks_test.py b/tensorflow/python/keras/callbacks_test.py index 1ac933135b9..828c78ebf15 100644 --- a/tensorflow/python/keras/callbacks_test.py +++ b/tensorflow/python/keras/callbacks_test.py @@ -285,10 +285,10 @@ class KerasCallbacksTest(keras_parameterized.TestCase): time.sleep(1) model = sequential.Sequential() - model.add(keras.layers.Dense(1, activation='sigmoid')) + model.add(keras.layers.Dense(1)) model.compile( 'sgd', - loss='binary_crossentropy', + loss='mse', run_eagerly=testing_utils.should_run_eagerly()) warning_messages = [] @@ -298,15 +298,38 @@ class KerasCallbacksTest(keras_parameterized.TestCase): with test.mock.patch.object(logging, 'warning', warning): model.fit( - np.ones((10, 10), 'float32'), - np.ones((10, 1), 'float32'), - batch_size=5, + np.ones((20, 1), 'float32'), + np.ones((20, 1), 'float32'), + batch_size=3, epochs=10, callbacks=[SleepCallback()]) - warning_msg = ('Callbacks method `on_train_batch_end` is slow compared ' + warning_msg = ('Callback method `on_train_batch_end` is slow compared ' 'to the batch time') self.assertIn(warning_msg, '\n'.join(warning_messages)) + @keras_parameterized.run_all_keras_modes + def test__default_callbacks_no_warning(self): + # Test that without the callback no warning is raised + model = sequential.Sequential() + model.add(keras.layers.Dense(1)) + model.compile( + 'sgd', + loss='mse', + run_eagerly=testing_utils.should_run_eagerly()) + + warning_messages = [] + + def warning(msg): + warning_messages.append(msg) + + with test.mock.patch.object(logging, 'warning', warning): + model.fit( + np.ones((20, 1), 'float32'), + np.ones((20, 1), 'float32'), + batch_size=3, + epochs=10) + self.assertListEqual(warning_messages, []) + @keras_parameterized.run_with_all_model_types(exclude_models='functional') @keras_parameterized.run_all_keras_modes def test_progbar_logging_deferred_model_build(self): From 8e01ae829bb88ff197c2e6b8c3ad1668fd2b9fa5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 16:56:05 -0700 Subject: [PATCH 0941/1017] PR #41916: [TF2XLA] Add EuclideanNorm kernel Imported from GitHub PR https://github.com/tensorflow/tensorflow/pull/41916 PiperOrigin-RevId: 326343156 Change-Id: I9810a3301570bf5a25e97b3004fe0043f8ee01db --- .../compiler/jit/mark_for_compilation_pass.cc | 1 - tensorflow/compiler/tests/reduce_ops_test.py | 22 +--------- .../compiler/tf2xla/kernels/reduction_ops.cc | 42 ------------------- .../compiler/tf2xla/kernels/reduction_ops.h | 4 -- .../tf2xla/kernels/reduction_ops_common.cc | 14 ++----- 5 files changed, 4 insertions(+), 79 deletions(-) diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc index 43619eca1fa..19eb61b6f72 100644 --- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc +++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc @@ -1892,7 +1892,6 @@ absl::flat_hash_set GetKnownXLAAllowlistOp() { "Einsum", "EmptyTensorList", "EnsureShape", - "EuclideanNorm", "ExtractImagePatches", "Igamma", "IgammaGradA", diff --git a/tensorflow/compiler/tests/reduce_ops_test.py b/tensorflow/compiler/tests/reduce_ops_test.py index a6844375c61..eb46c536e07 100644 --- a/tensorflow/compiler/tests/reduce_ops_test.py +++ b/tensorflow/compiler/tests/reduce_ops_test.py @@ -25,7 +25,6 @@ from absl.testing import parameterized import numpy as np from tensorflow.compiler.tests import xla_test -from tensorflow.python.eager import def_function from tensorflow.python.framework import dtypes from tensorflow.python.framework import errors_impl from tensorflow.python.ops import array_ops @@ -51,8 +50,7 @@ class ReduceOpsTest(xla_test.XLATestCase, parameterized.TestCase): with self.test_scope(): a = array_ops.placeholder(dtype) index = array_ops.placeholder(index_dtype) - out = def_function.function(experimental_compile=True)(tf_reduce_fn)( - a, index) + out = tf_reduce_fn(a, index) result = sess.run(out, {a: test_input, index: [0]}) self.assertAllClose( result, np_reduce_fn(test_input, axis=0), rtol=rtol, atol=atol) @@ -181,24 +179,6 @@ class ReduceOpsTest(xla_test.XLATestCase, parameterized.TestCase): 'Axes contains duplicate dimension'): sess.run(out, {a: [10, 20, 30], index: [0, 0]}) - def testReduceEuclideanNorm(self, index_dtype): - - def reference_euclidean_norm(dtype, inp, axis): - inp = inp.astype(dtype) - return np.sqrt(np.sum(inp * np.conj(inp), axis)).astype(dtype) - - for real_dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]: - self._testReduction( - math_ops.reduce_euclidean_norm, - functools.partial(reference_euclidean_norm, real_dtype), real_dtype, - self.REAL_DATA, index_dtype) - - for complex_dtype in [np.complex64]: - self._testReduction( - math_ops.reduce_euclidean_norm, - functools.partial(reference_euclidean_norm, complex_dtype), - complex_dtype, self.COMPLEX_DATA, index_dtype) - class ReduceOpPrecisionTest(xla_test.XLATestCase): diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc index f95d58fd96a..4f63c0d1b66 100644 --- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc +++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc @@ -16,15 +16,12 @@ limitations under the License. // XLA-specific reduction Ops. #include "tensorflow/compiler/tf2xla/kernels/reduction_ops.h" - #include "tensorflow/compiler/tf2xla/type_util.h" #include "tensorflow/compiler/tf2xla/xla_helpers.h" #include "tensorflow/compiler/tf2xla/xla_op_registry.h" #include "tensorflow/compiler/xla/client/lib/constants.h" -#include "tensorflow/compiler/xla/client/lib/math.h" #include "tensorflow/compiler/xla/client/xla_builder.h" #include "tensorflow/compiler/xla/literal.h" -#include "tensorflow/compiler/xla/primitive_util.h" #include "tensorflow/core/framework/kernel_def_builder.h" namespace tensorflow { @@ -187,44 +184,5 @@ class AnyOp : public XlaReductionOp { REGISTER_XLA_OP(Name("Any").CompileTimeConstantInput("reduction_indices"), AnyOp); -class EuclideanNormOp : public XlaReductionOp { - public: - explicit EuclideanNormOp(OpKernelConstruction* ctx) - : XlaReductionOp(ctx, - XlaHelpers::SumAccumulationType(ctx->input_type(0))) {} - xla::XlaOp InitialValue(xla::XlaBuilder* builder) override { - return xla::Zero(builder, xla_reduction_type_); - } - - xla::XlaOp PreprocessInput(xla::XlaBuilder* /*builder*/, - const xla::XlaOp& data) override { - return xla::Mul(data, MaybeConjugate(data, true)); - } - - void BuildReducer(xla::XlaBuilder* builder, const xla::XlaOp& scalar_lhs, - const xla::XlaOp& scalar_rhs) override { - xla::Add(scalar_lhs, scalar_rhs); - } - - xla::XlaOp BuildFinalizer( - xla::XlaBuilder* /*builder*/, const xla::XlaOp& input, - const xla::XlaOp& reduce_output, - const std::vector& dimensions_to_reduce) override { - if (xla::primitive_util::IsIntegralType(xla_reduction_type_)) { - // XLA only supports float and complex sqrt. - // Thus, cast integral type to F32 for computation. - return XlaHelpers::ConvertElementType( - xla::Sqrt(xla::ConvertElementType(reduce_output, xla::F32)), - input_type(0)); - } - return XlaHelpers::ConvertElementType(xla::Sqrt(reduce_output), - input_type(0)); - } -}; - -REGISTER_XLA_OP( - Name("EuclideanNorm").CompileTimeConstantInput("reduction_indices"), - EuclideanNormOp); - } // namespace } // namespace tensorflow diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h index 2091b496ddb..af716eab798 100644 --- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h +++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h @@ -39,10 +39,6 @@ class XlaReductionOp : public XlaOpKernel { // Return the base case for the reduction. virtual xla::XlaOp InitialValue(xla::XlaBuilder* builder) = 0; - // Preprocesses input before reduction. - virtual xla::XlaOp PreprocessInput(xla::XlaBuilder* builder, - const xla::XlaOp& data); - // Implement the (scalar,scalar)->scalar lambda that should be // applied to each pair of elements to be reduced. The desired // computation should be added to 'builder' and diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc index 58d53dfea58..b4284a5498c 100644 --- a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc +++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc @@ -35,12 +35,6 @@ XlaReductionOp::XlaReductionOp(OpKernelConstruction* ctx, ctx, DataTypeToPrimitiveType(reduction_type_, &xla_reduction_type_)); } -// The default pre-processor directly returns the data. This can be overridden. -xla::XlaOp XlaReductionOp::PreprocessInput(xla::XlaBuilder* /*builder*/, - const xla::XlaOp& data) { - return data; -} - // The default finalizer converts the results back into the input type. This can // be overridden. xla::XlaOp XlaReductionOp::BuildFinalizer( @@ -117,8 +111,7 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) { xla::PrimitiveType type; TF_CHECK_OK(DataTypeToPrimitiveType(reduction_type_, &type)); - auto converted_input = xla::ConvertElementType(ctx->Input(0), type); - auto processed_input = PreprocessInput(b, converted_input); + auto data = xla::ConvertElementType(ctx->Input(0), type); // Call virtual method to get the initial value. auto initial = xla::ConvertElementType(InitialValue(b), type); // Make two scalar parameters of the desired type for the lambda. @@ -128,9 +121,8 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) { BuildReducer(&r, rx, ry); xla::XlaComputation reduction_computation = r.Build().ConsumeValueOrDie(); - auto reduce = - xla::Reduce(processed_input, initial, reduction_computation, xla_axes); - auto finalized = BuildFinalizer(b, converted_input, reduce, xla_axes); + auto reduce = xla::Reduce(data, initial, reduction_computation, xla_axes); + auto finalized = BuildFinalizer(b, data, reduce, xla_axes); auto result = keep_dims_ ? xla::Reshape(finalized, final_shape) : finalized; ctx->SetOutput(0, result); } From 3ea5fc7f3f212531e12e54420d7ceba61715590f Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 12 Aug 2020 16:58:21 -0700 Subject: [PATCH 0942/1017] Introduce additional XLA TPU Ops to open source PiperOrigin-RevId: 326343558 Change-Id: I47da1dc0c96cdf8223ccebef012e2a5088a857a4 --- tensorflow/core/tpu/kernels/BUILD | 1 + tensorflow/core/tpu/kernels/xla/BUILD | 52 ++ .../core/tpu/kernels/xla/get_item_op.cc | 75 +++ .../core/tpu/kernels/xla/host_compute_ops.cc | 498 ++++++++++++++++++ tensorflow/core/tpu/kernels/xla/index_ops.cc | 34 ++ tensorflow/core/tpu/kernels/xla/infeed_op.cc | 162 ++++++ .../core/tpu/kernels/xla/inplace_ops.cc | 142 +++++ .../core/tpu/kernels/xla/outfeed_ops.cc | 91 ++++ .../tpu/kernels/xla/segment_reduction_ops.cc | 145 +++++ tensorflow/core/tpu/kernels/xla/where_op.cc | 91 ++++ 10 files changed, 1291 insertions(+) create mode 100644 tensorflow/core/tpu/kernels/xla/BUILD create mode 100644 tensorflow/core/tpu/kernels/xla/get_item_op.cc create mode 100644 tensorflow/core/tpu/kernels/xla/host_compute_ops.cc create mode 100644 tensorflow/core/tpu/kernels/xla/index_ops.cc create mode 100644 tensorflow/core/tpu/kernels/xla/infeed_op.cc create mode 100644 tensorflow/core/tpu/kernels/xla/inplace_ops.cc create mode 100644 tensorflow/core/tpu/kernels/xla/outfeed_ops.cc create mode 100644 tensorflow/core/tpu/kernels/xla/segment_reduction_ops.cc create mode 100644 tensorflow/core/tpu/kernels/xla/where_op.cc diff --git a/tensorflow/core/tpu/kernels/BUILD b/tensorflow/core/tpu/kernels/BUILD index 6d3369022ad..0c7ac668555 100644 --- a/tensorflow/core/tpu/kernels/BUILD +++ b/tensorflow/core/tpu/kernels/BUILD @@ -38,6 +38,7 @@ tf_kernel_library( ":tpu_execute_op", ":tpu_handle_to_key_op", ":transfer_ops", + "//tensorflow/core/tpu/kernels/xla:xla_ops", ], ) diff --git a/tensorflow/core/tpu/kernels/xla/BUILD b/tensorflow/core/tpu/kernels/xla/BUILD new file mode 100644 index 00000000000..f55583a570b --- /dev/null +++ b/tensorflow/core/tpu/kernels/xla/BUILD @@ -0,0 +1,52 @@ +# XLA Ops for TPUs + +package( + licenses = ["notice"], # Apache 2.0 +) + +cc_library( + name = "xla_ops", + srcs = [ + "get_item_op.cc", + "host_compute_ops.cc", + "index_ops.cc", + "infeed_op.cc", + "inplace_ops.cc", + "outfeed_ops.cc", + "segment_reduction_ops.cc", + "where_op.cc", + ], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow/compiler/tf2xla:common", + "//tensorflow/compiler/tf2xla:sharding_util", + "//tensorflow/compiler/tf2xla:side_effect_util", + "//tensorflow/compiler/tf2xla:xla_compiler", + "//tensorflow/compiler/tf2xla:xla_context", + "//tensorflow/compiler/tf2xla:xla_helpers", + "//tensorflow/compiler/tf2xla:xla_op_registry", + "//tensorflow/compiler/tf2xla/kernels:if_op", + "//tensorflow/compiler/tf2xla/kernels:while_op", + "//tensorflow/compiler/tf2xla/kernels:xla_ops", + "//tensorflow/compiler/tf2xla/lib:scatter", + "//tensorflow/compiler/xla:shape_util", + "//tensorflow/compiler/xla:util", + "//tensorflow/compiler/xla:xla_data_proto_cc", + "//tensorflow/compiler/xla/client:xla_builder", + "//tensorflow/compiler/xla/client/lib:arithmetic", + "//tensorflow/compiler/xla/client/lib:comparators", + "//tensorflow/compiler/xla/client/lib:constants", + "//tensorflow/core:core_cpu_internal", + "//tensorflow/core:framework", + "//tensorflow/core:graph", + "//tensorflow/core:lib", + "//tensorflow/core:protos_all_cc", + "//tensorflow/core/tpu:tpu_api", + "//tensorflow/core/tpu:tpu_defs", + "//tensorflow/core/tpu/kernels:cross_replica_ops", + "//tensorflow/stream_executor/tpu:c_api_conversions", + "//tensorflow/stream_executor/tpu:c_api_decl", + "@com_google_absl//absl/strings", + ], + alwayslink = 1, +) diff --git a/tensorflow/core/tpu/kernels/xla/get_item_op.cc b/tensorflow/core/tpu/kernels/xla/get_item_op.cc new file mode 100644 index 00000000000..094c6b87f64 --- /dev/null +++ b/tensorflow/core/tpu/kernels/xla/get_item_op.cc @@ -0,0 +1,75 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#define EIGEN_USE_THREADS + +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/tensor_util.h" + +namespace tensorflow { +namespace { + +// The Xla kernel to build up the computation for get_item(data, index). +class GetItemXlaOp : public XlaOpKernel { + public: + explicit GetItemXlaOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} + + void Compile(XlaOpKernelContext* ctx) override { + const TensorShape& data_shape = ctx->InputShape(0); + const TensorShape& index_shape = ctx->InputShape(1); + OP_REQUIRES( + ctx, TensorShapeUtils::IsVectorOrHigher(data_shape), + errors::InvalidArgument("data must be at least 1 dimensional.")); + OP_REQUIRES(ctx, index_shape.dims() == 1 && index_shape.dim_size(0) == 1, + errors::InvalidArgument("index must be a vector of size 1.")); + + // NOTE(pbar) Use Concat to extend the indices to match cl/142279605. + // This isn't the simplest way to emit the indices, but the code for + // dynamic slice needs to be able to see that minor dims are const zero. + auto const_zero = xla::ConstantR0(ctx->builder(), 0); + std::vector operands; + operands.push_back(xla::Reshape(ctx->Input(1), {})); + for (int i = 1; i < data_shape.dims(); i++) { + operands.push_back(const_zero); + } + + std::vector dims = {0}; + std::vector slice_sizes = {1}; + std::vector out_sizes = {}; + for (int i = 1; i < data_shape.dims(); i++) { + dims.push_back(i); + auto size = data_shape.dim_size(i); + slice_sizes.push_back(size); + out_sizes.push_back(size); + } + // NOTE: DynamicSlice here doesn't raise an error or wraps the index + // if its out-of-range. + auto slice = xla::DynamicSlice(ctx->Input(0), operands, slice_sizes); + // In-order collapse to remove the 1st dim. + auto reshape = xla::Reshape(slice, dims, out_sizes); + ctx->SetOutput(0, reshape); + } +}; + +REGISTER_XLA_OP(Name("GetItem"), GetItemXlaOp); + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/xla/host_compute_ops.cc b/tensorflow/core/tpu/kernels/xla/host_compute_ops.cc new file mode 100644 index 00000000000..be3ee1c9d24 --- /dev/null +++ b/tensorflow/core/tpu/kernels/xla/host_compute_ops.cc @@ -0,0 +1,498 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "absl/strings/str_cat.h" +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/sharding_util.h" +#include "tensorflow/compiler/tf2xla/side_effect_util.h" +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/compiler/tf2xla/xla_context.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/primitive_util.h" +#include "tensorflow/compiler/xla/xla_data.pb.h" +#include "tensorflow/core/common_runtime/function.h" +#include "tensorflow/core/common_runtime/graph_constructor.h" +#include "tensorflow/core/common_runtime/lower_function_call_op.h" +#include "tensorflow/core/common_runtime/lower_if_op.h" +#include "tensorflow/core/common_runtime/shape_refiner.h" +#include "tensorflow/core/framework/function.h" +#include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/versions.pb.h" +#include "tensorflow/core/graph/algorithm.h" +#include "tensorflow/core/tpu/tpu_defs.h" + +namespace tensorflow { + +namespace { + +// TODO(phawkins) add a canonical copy of these operator names and refactor +// everything to use it. +static const char* const kSendFromHostOp = "_XlaSendFromHost"; +static const char* const kRecvAtHostOp = "_XlaRecvAtHost"; + +Status MakeXlaShapes(gtl::ArraySlice shapes, + gtl::ArraySlice dtypes, + std::vector* xla_shapes, + xla::Shape* xla_shape) { + for (int i = 0; i < shapes.size(); i++) { + xla::Shape single_xla_shape; + TF_RETURN_IF_ERROR( + TensorShapeToXLAShape(dtypes[i], shapes[i], &single_xla_shape)); + VLOG(2) << "Shape " << single_xla_shape.DebugString(); + xla_shapes->push_back(single_xla_shape); + } + // Temporarily add a dummy output to the shape array before making the tuple: + // this output is used for control dependencies between host compute ops. + xla_shapes->push_back(xla::ShapeUtil::MakeShape(xla::PRED, {})); + *xla_shape = xla::ShapeUtil::MakeTupleShape(*xla_shapes); + // Remove the dummy output from the vector that will be used to copy real + // outputs from host to device. + xla_shapes->pop_back(); + return Status::OK(); +} + +// This TensorFlow pseudo-op is used to record host-side computation. +class HostComputeOp : public XlaOpKernel { + public: + explicit HostComputeOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("cost_estimate_ns", &cost_estimate_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("key", &key_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("tpu_core", &tpu_core_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("Tinputs", &input_dtypes_)); + OP_REQUIRES(ctx, ctx->num_inputs() == input_dtypes_.size(), + errors::InvalidArgument("Tinputs size=", input_dtypes_.size(), + " but expected ", ctx->num_inputs(), + " inputs.")); + OP_REQUIRES_OK(ctx, ctx->GetAttr("Toutputs", &output_dtypes_)); + OP_REQUIRES(ctx, ctx->num_outputs() == output_dtypes_.size(), + errors::InvalidArgument("Toutputs size=", output_dtypes_.size(), + " but expected ", ctx->num_outputs(), + " outputs.")); + OP_REQUIRES_OK(ctx, ctx->GetAttr("ancestors", &ancestors_)); + NameAttrList shape_inference_graph; + OP_REQUIRES_OK( + ctx, ctx->GetAttr("shape_inference_graph", &shape_inference_graph)); + if (shape_inference_graph.name().empty()) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &static_output_shapes_)); + OP_REQUIRES(ctx, static_output_shapes_.size() == output_dtypes_.size(), + errors::InvalidArgument( + "shapes attr list size ", static_output_shapes_.size(), + " differs from dtypes size ", output_dtypes_.size())); + OP_REQUIRES_OK(ctx, MakeXlaShapes(static_output_shapes_, output_dtypes_, + &static_xla_output_shapes_, + &static_xla_output_shape_)); + VLOG(2) << "Output Shape: " << static_xla_output_shape_.DebugString(); + } else { + FunctionLibraryRuntime* flib_runtime = ctx->function_library(); + OP_REQUIRES(ctx, flib_runtime != nullptr, + errors::Internal( + "No function library runtime at kernel construction")); + const FunctionLibraryDefinition* library = + flib_runtime->GetFunctionLibraryDefinition(); + const FunctionDef* fdef = library->Find(shape_inference_graph.name()); + OP_REQUIRES(ctx, fdef != nullptr, + errors::Internal("Failed to find function ", + shape_inference_graph.name(), + " in function library.")); + OP_REQUIRES_OK(ctx, FunctionDefToBodyHelper( + *fdef, AttrSlice(&shape_inference_graph.attr()), + library, &shape_inference_graph_function_)); + VLOG(2) << "Output Shape to be inferred at compile time"; + } + OP_REQUIRES_OK( + ctx, ctx->GetAttr(kXlaTokenInputNodesAttrName, &token_input_nodes_)); + OP_REQUIRES(ctx, !token_input_nodes_.empty(), + errors::InvalidArgument("XlaHostCompute node does not have ", + kXlaTokenInputNodesAttrName, " attr")); + OP_REQUIRES_OK(ctx, ctx->GetAttr(kXlaOriginalOutsideCompilationNodeName, + &original_node_name_)); + } + + ~HostComputeOp() override {} + + void Compile(XlaOpKernelContext* ctx) override { + xla::XlaBuilder* b = ctx->builder(); + XlaCompiler* compiler = ctx->compiler(); + + std::vector input_handles; + std::vector input_shapes; + auto inputs = ctx->InputList("inputs", &input_handles, &input_shapes); + const auto device_sharding = xla::sharding_builder::AssignDevice(tpu_core_); + xla::XlaScopedShardingAssignment assign_sharding(b, device_sharding); + + std::vector input_tokens; + for (auto& token_input_node : token_input_nodes_) { + auto token_or = compiler->GetNodeToken(token_input_node); + OP_REQUIRES_OK(ctx, token_or.status()); + input_tokens.push_back(token_or.ValueOrDie()); + } + xla::XlaOp token = xla::AfterAll(b, input_tokens); + + // Send values to the host. + std::vector send_to_host_tokens; + for (int i = 0; i < input_handles.size(); ++i) { + const string channel_name = absl::StrCat(key_, "_dtoh_", i); + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(input_dtypes_[i], + input_shapes[i], &xla_shape)); + // Specify frontend attributes. + xla::FrontendAttributes attrs; + (*attrs.mutable_map())[kXlaHostTransferRendezvousNameAttr] = channel_name; + (*attrs.mutable_map())[kXlaHostTransferOriginalTypeAttr] = + xla::primitive_util::LowercasePrimitiveTypeName( + xla_shape.element_type()); + b->SetFrontendAttributes(attrs); + xla::ChannelHandle channel; + OP_REQUIRES_OK( + ctx, compiler->GetDeviceToHostChannelHandle(channel_name, &channel)); + send_to_host_tokens.push_back( + xla::SendToHost(input_handles[i], token, xla_shape, channel)); + b->ClearOpMetadata(); + } + xla::XlaOp recv_from_host_token_input = + send_to_host_tokens.empty() ? token + : xla::AfterAll(b, send_to_host_tokens); + if (!input_handles.empty()) { + // Register the shapes used in this transfer. + OP_REQUIRES_OK(ctx, ctx->compiler()->SetDeviceToHostMetadata( + key_, input_dtypes_, input_shapes)); + } + // Compute the shapes of the values to copy to the device, if necessary. + std::vector* output_shapes; + std::vector* xla_output_shapes; + xla::Shape* xla_output_shape; + std::vector inferred_output_shapes; + std::vector inferred_xla_output_shapes; + xla::Shape inferred_xla_output_shape; + if (shape_inference_graph_function_) { + OP_REQUIRES_OK( + ctx, InferOutputShapes( + ctx, ctx->function_library()->GetFunctionLibraryDefinition(), + &inferred_output_shapes)); + OP_REQUIRES_OK(ctx, MakeXlaShapes(inferred_output_shapes, output_dtypes_, + &inferred_xla_output_shapes, + &inferred_xla_output_shape)); + output_shapes = &inferred_output_shapes; + xla_output_shapes = &inferred_xla_output_shapes; + xla_output_shape = &inferred_xla_output_shape; + } else { + output_shapes = &static_output_shapes_; + xla_output_shapes = &static_xla_output_shapes_; + xla_output_shape = &static_xla_output_shape_; + } + OP_REQUIRES( + ctx, output_shapes->size() == ctx->num_outputs(), + errors::InvalidArgument("Op has ", ctx->num_outputs(), " outputs ", + " but output shape vector of size ", + output_shapes->size())); + if (ctx->num_outputs() > 0) { + // Register the shapes used in this transfer. + OP_REQUIRES_OK(ctx, ctx->compiler()->SetHostToDeviceMetadata( + key_, output_dtypes_, *output_shapes)); + } + // Copy results to the device. + std::vector recv_from_host_tokens; + for (int i = 0; i < output_shapes->size(); ++i) { + const string channel_name = absl::StrCat(key_, "_htod_", i); + // Specify frontend attributes. + xla::FrontendAttributes attrs; + (*attrs.mutable_map())[kXlaHostTransferRendezvousNameAttr] = channel_name; + (*attrs.mutable_map())[kXlaHostTransferOriginalTypeAttr] = + xla::primitive_util::LowercasePrimitiveTypeName( + xla_output_shapes->at(i).element_type()); + b->SetFrontendAttributes(attrs); + xla::ChannelHandle channel; + OP_REQUIRES_OK( + ctx, compiler->GetHostToDeviceChannelHandle(channel_name, &channel)); + + const auto result_token_tuple = xla::RecvFromHost( + recv_from_host_token_input, xla_output_shapes->at(i), channel); + b->ClearOpMetadata(); + recv_from_host_tokens.push_back( + xla::GetTupleElement(result_token_tuple, /*index=*/1)); + ctx->SetOutput(i, xla::GetTupleElement(result_token_tuple, 0)); + } + + // Set token output. + xla::XlaOp token_output = recv_from_host_tokens.empty() + ? recv_from_host_token_input + : xla::AfterAll(b, recv_from_host_tokens); + OP_REQUIRES_OK( + ctx, ctx->compiler()->SetNodeToken(original_node_name_, token_output)); + } + + private: + Status LowerFunctionalOps(Graph* g, + const FunctionLibraryDefinition& flib_def) { + bool modified; + do { + modified = false; + + // Lower "If" nodes first. Their body functions will be expanded as + // function call nodes, which we will lower later. + // We do not need to lower "While" nodes because shape inference can + // handle them correctly (output shapes are input shapes). + std::vector if_nodes; + for (Node* n : g->op_nodes()) { + if (n->type_string() == "If") { + if_nodes.push_back(n); + } + } + for (Node* if_node : if_nodes) { + TF_RETURN_IF_ERROR( + RewriteIfNode(if_node, g, /*keep_node_fetchable=*/false)); + } + if (!if_nodes.empty()) { + modified = true; + } + + // Lower function call nodes. + std::vector call_nodes; + for (Node* n : g->op_nodes()) { + if (IsFunctionCall(flib_def, *n)) { + call_nodes.push_back(n); + } + } + for (Node* call_node : call_nodes) { + TF_RETURN_IF_ERROR(RewriteFunctionCallNode( + call_node, g, flib_def, /*keep_caller_fetchable=*/false)); + } + if (!call_nodes.empty()) { + modified = true; + } + } while (modified); + + return Status::OK(); + } + + Status InferOutputShapes(XlaOpKernelContext* ctx, + const FunctionLibraryDefinition* flib_def, + std::vector* output_shapes) { + // First unpack the inference graphdef from the attr into graph. Don't do + // any shape inference at this point. + Graph* graph = shape_inference_graph_function_->graph; + + // Lower functional ops, because they are not friendly to shape inference. + TF_RETURN_IF_ERROR(LowerFunctionalOps(graph, *flib_def)); + + // Now run shape inference, filling in the shapes of recvathost nodes. + bool got_output_shapes = false; + ShapeRefiner shape_refiner{graph->versions().producer(), + graph->op_registry()}; + std::vector nodes; + GetReversePostOrder(*graph, &nodes); + for (auto node : nodes) { + TF_RETURN_IF_ERROR(shape_refiner.AddNode(node)); + if (node->type_string() == kRecvAtHostOp) { + const AttrValue* key_attr = node->attrs().Find("key"); + if (key_attr == nullptr) { + return errors::InvalidArgument("Node ", node->name(), + " has no key attribute"); + } + std::vector dtoh_shapes; + if (!ctx->compiler() + ->GetDeviceToHostShapes(key_attr->s(), &dtoh_shapes) + .ok()) { + return errors::InvalidArgument( + "Shape inference for HostCompute ", ctx->op_kernel().name(), + " failed: host recv node ", node->name(), " with key '", + key_attr->s(), "' has unknown shapes."); + } + if (dtoh_shapes.size() != node->num_outputs()) { + return errors::InvalidArgument( + "Shape inference for HostCompute ", ctx->op_kernel().name(), + " failed: host recv node ", node->name(), " with key '", + key_attr->s(), "' has ", node->num_outputs(), + " outputs but inferred shapes expect ", dtoh_shapes.size()); + } + for (int i = 0; i < node->num_outputs(); ++i) { + shape_inference::InferenceContext* shape_ctx = + shape_refiner.GetContext(node); + shape_inference::ShapeHandle handle; + TF_RETURN_IF_ERROR( + shape_ctx->MakeShapeFromTensorShape(dtoh_shapes.at(i), &handle)); + shape_ctx->set_output(i, handle); + } + } else if (node->type_string() == kSendFromHostOp) { + if (got_output_shapes) { + return errors::InvalidArgument( + "Shape inference for HostCompute ", ctx->op_kernel().name(), + " failed: inference graph has multiple send from host nodes"); + } else { + got_output_shapes = true; + // The last input is the dynamic key so don't record its shape. + output_shapes->resize(node->num_inputs() - 1); + shape_inference::InferenceContext* shape_ctx = + shape_refiner.GetContext(node); + for (int i = 0; i < node->num_inputs() - 1; ++i) { + shape_inference::ShapeHandle handle = shape_ctx->input(i); + if (!shape_ctx->FullyDefined(handle)) { + return errors::InvalidArgument( + "Shape inference for HostCompute ", ctx->op_kernel().name(), + " failed: send from host node ", node->name(), + " has non-fully defined shape of input index ", i); + } + TensorShapeProto shape_proto; + shape_ctx->ShapeHandleToProto(handle, &shape_proto); + (*output_shapes)[i] = TensorShape(shape_proto); + VLOG(2) << "Inferred shape " << shape_proto.DebugString(); + } + } + } + } + if (!got_output_shapes) { + return errors::InvalidArgument( + "Shape inference for HostCompute ", ctx->op_kernel().name(), + " failed: inference graph has no send from host node"); + } + return Status::OK(); + } + + DataTypeVector input_dtypes_; + DataTypeVector output_dtypes_; + std::vector ancestors_; + std::vector static_output_shapes_; + std::vector static_xla_output_shapes_; + string original_node_name_; + // If static_xla_output_shapes_.size() == 1 then xla_output_shape_ is the + // unique output shape, otherwise it is a tuple of all the xla_output_shapes_. + xla::Shape static_xla_output_shape_; + string key_; + // If shape inference is performed at runtime, the graph needed to perform + // shape inference is stored in this function. + std::unique_ptr shape_inference_graph_function_; + int64 cost_estimate_; + int64 tpu_core_; + std::vector token_input_nodes_; + + TF_DISALLOW_COPY_AND_ASSIGN(HostComputeOp); +}; + +class SendToHostOp : public XlaOpKernel { + public: + explicit SendToHostOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("Tinput", &input_dtype_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("key", &key_)); + OP_REQUIRES_OK( + ctx, ctx->GetAttr(kXlaTokenInputNodesAttrName, &token_input_nodes_)); + OP_REQUIRES(ctx, !token_input_nodes_.empty(), + errors::InvalidArgument("XlaSendToHost node does not have ", + kXlaTokenInputNodesAttrName, " attr")); + OP_REQUIRES_OK(ctx, ctx->GetAttr(kXlaOriginalOutsideCompilationNodeName, + &original_node_name_)); + } + + ~SendToHostOp() override {} + + void Compile(XlaOpKernelContext* ctx) override { + xla::XlaBuilder* b = ctx->builder(); + + XlaCompiler* compiler = ctx->compiler(); + xla::XlaOp operand = ctx->Input(0); + std::vector input_tokens; + for (auto& token_input_node : token_input_nodes_) { + auto token_or = compiler->GetNodeToken(token_input_node); + OP_REQUIRES_OK(ctx, token_or.status()); + input_tokens.push_back(token_or.ValueOrDie()); + } + xla::XlaOp token = xla::AfterAll(b, input_tokens); + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(input_dtype_, ctx->InputShape(0), + &xla_shape)); + // Specify frontend attributes. + xla::FrontendAttributes attrs; + (*attrs.mutable_map())[kXlaHostTransferRendezvousNameAttr] = key_; + (*attrs.mutable_map())[kXlaHostTransferOriginalTypeAttr] = + xla::primitive_util::LowercasePrimitiveTypeName( + xla_shape.element_type()); + b->SetFrontendAttributes(attrs); + xla::ChannelHandle channel; + OP_REQUIRES_OK(ctx, compiler->GetDeviceToHostChannelHandle(key_, &channel)); + xla::XlaOp output_token = + xla::SendToHost(operand, token, xla_shape, channel); + OP_REQUIRES_OK(ctx, + compiler->SetNodeToken(original_node_name_, output_token)); + } + + private: + DataType input_dtype_; + string key_; + std::vector token_input_nodes_; + string original_node_name_; + TF_DISALLOW_COPY_AND_ASSIGN(SendToHostOp); +}; + +class RecvFromHostOp : public XlaOpKernel { + public: + explicit RecvFromHostOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("Toutput", &output_dtype_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &output_shape_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("key", &key_)); + OP_REQUIRES_OK( + ctx, ctx->GetAttr(kXlaTokenInputNodesAttrName, &token_input_nodes_)); + OP_REQUIRES(ctx, !token_input_nodes_.empty(), + errors::InvalidArgument("XlaRecvFromHost node does not have ", + kXlaTokenInputNodesAttrName, " attr")); + } + + ~RecvFromHostOp() override {} + + void Compile(XlaOpKernelContext* ctx) override { + xla::XlaBuilder* b = ctx->builder(); + + XlaCompiler* compiler = ctx->compiler(); + std::vector input_tokens; + for (auto& token_input_node : token_input_nodes_) { + auto token_or = compiler->GetNodeToken(token_input_node); + OP_REQUIRES_OK(ctx, token_or.status()); + input_tokens.push_back(token_or.ValueOrDie()); + } + xla::XlaOp token = xla::AfterAll(b, input_tokens); + xla::Shape xla_shape; + OP_REQUIRES_OK( + ctx, TensorShapeToXLAShape(output_dtype_, output_shape_, &xla_shape)); + // Specify frontend attributes. + xla::FrontendAttributes attrs; + (*attrs.mutable_map())[kXlaHostTransferRendezvousNameAttr] = key_; + (*attrs.mutable_map())[kXlaHostTransferOriginalTypeAttr] = + xla::primitive_util::LowercasePrimitiveTypeName( + xla_shape.element_type()); + b->SetFrontendAttributes(attrs); + xla::ChannelHandle channel; + OP_REQUIRES_OK(ctx, compiler->GetHostToDeviceChannelHandle(key_, &channel)); + xla::XlaOp result = xla::RecvFromHost(token, xla_shape, channel); + // xla::RecvFromHost returns a tuple of (received data, token). + ctx->SetOutput(0, xla::GetTupleElement(result, 0)); + OP_REQUIRES_OK( + ctx, compiler->SetNodeToken(name(), xla::GetTupleElement(result, 1))); + } + + private: + DataType output_dtype_; + TensorShape output_shape_; + string key_; + std::vector token_input_nodes_; + TF_DISALLOW_COPY_AND_ASSIGN(RecvFromHostOp); +}; + +REGISTER_XLA_OP(Name("XlaHostCompute"), HostComputeOp); +REGISTER_XLA_OP(Name("XlaSendToHost"), SendToHostOp); +REGISTER_XLA_OP(Name("XlaRecvFromHost"), RecvFromHostOp); + +} // anonymous namespace +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/xla/index_ops.cc b/tensorflow/core/tpu/kernels/xla/index_ops.cc new file mode 100644 index 00000000000..40148f31177 --- /dev/null +++ b/tensorflow/core/tpu/kernels/xla/index_ops.cc @@ -0,0 +1,34 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/tf2xla/kernels/index_ops.h" + +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/core/tpu/tpu_defs.h" + +namespace tensorflow { +namespace { + +// This registration is needed here because the ArgMax Op is defined in +// third_party where DEVICE_TPU_XLA_JIT is not visible. Most Ops don't need a +// specific TPU whitelist, but ArgMax does because it has a separate CustomCall +// implementation on CPU. +REGISTER_XLA_OP(Name("ArgMax") + .Device(DEVICE_TPU_XLA_JIT) + .CompileTimeConstantInput("dimension"), + XlaArgMaxOp); + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/xla/infeed_op.cc b/tensorflow/core/tpu/kernels/xla/infeed_op.cc new file mode 100644 index 00000000000..941a543e386 --- /dev/null +++ b/tensorflow/core/tpu/kernels/xla/infeed_op.cc @@ -0,0 +1,162 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/sharding_util.h" +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/compiler/xla/util.h" +#include "tensorflow/core/framework/kernel_def_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/tpu/tpu_api.h" +#include "tensorflow/core/tpu/tpu_defs.h" +#include "tensorflow/stream_executor/tpu/c_api_conversions.h" +#include "tensorflow/stream_executor/tpu/c_api_decl.h" + +namespace tensorflow { + +namespace { + +xla::Shape GetTPUInfeedLayout(const xla::Shape& shape) { + XLA_Shape c_shape; + XLA_Shape c_infeed_shape; + + ApiConverter::ToC(shape, &c_shape); + + tpu::ExecutorApiFn()->TpuTransferManager_GetInfeedLayoutFn(&c_shape, + &c_infeed_shape); + xla::Shape infeed_shape = ApiConverter::FromC(&c_infeed_shape); + ApiConverter::Free(&c_shape); + ApiConverter::Free(&c_infeed_shape); + return infeed_shape; +} + +// Updates the layout of the given infeed shape, optionally considering the +// sharding of the op. If the op has tile sharding, assign the layout based on +// the shard shape. +Status UpdateInfeedLayout(xla::Shape* shape, + absl::optional sharding) { + if (sharding && sharding->type() == xla::OpSharding::OTHER) { + TF_ASSIGN_OR_RETURN(auto hlo_sharding, + xla::HloSharding::FromProto(*sharding)); + for (int64 i = 0; i < sharding->tile_assignment_devices_size(); ++i) { + auto device = sharding->tile_assignment_devices(i); + auto shard_shape = + GetTPUInfeedLayout(hlo_sharding.TileShape(*shape, device)); + if (i == 0) { + *shape->mutable_layout() = shard_shape.layout(); + } + if (xla::ShapeUtil::ElementsIn(shard_shape) == 0) { + // Shapes with 0 dimensions may be assigned with a different layout, but + // it doesn't matter since we're not sending any data. + continue; + } + if (!xla::LayoutUtil::Equal(shard_shape.layout(), shape->layout())) { + return xla::Unimplemented( + "Sharded infeed with non-uniform layouts is not supported. Try " + "turning off the infeed layout optimization " + "(--transpose_tpu_infeed=false) and report to XLA team."); + } + } + return Status::OK(); + } + *shape = GetTPUInfeedLayout(*shape); + return Status::OK(); +} + +// TODO(pbar) Work out if we need to Infeed Tuples - if so then +// this op will need a way to provide a list of shapes +// since they can't be provided by the runtime JIT mechanism. +// (InfeedDequeue has no inputs!) +// Compare this op to tf.Queue operations which operate on N tensors. + +// This TensorFlow op supports the XLA Infeed primitve. +class InfeedDequeueOp : public XlaOpKernel { + public: + explicit InfeedDequeueOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shape", &shape_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); + OP_REQUIRES_OK(ctx, TensorShapeToXLAShape(dtype_, shape_, &xla_shape_)); + } + + void Compile(XlaOpKernelContext* ctx) override { + xla::XlaBuilder* b = ctx->builder(); + OP_REQUIRES_OK(ctx, UpdateInfeedLayout(&xla_shape_, b->sharding())); + ctx->SetOutput(0, xla::Infeed(b, xla_shape_)); + } + + private: + TensorShape shape_; + DataType dtype_; + xla::Shape xla_shape_; + + TF_DISALLOW_COPY_AND_ASSIGN(InfeedDequeueOp); +}; + +REGISTER_XLA_OP(Name("InfeedDequeue"), InfeedDequeueOp); + +// This TensorFlow op supports the XLA Infeed primitive for tuple types. +class InfeedDequeueTupleOp : public XlaOpKernel { + public: + explicit InfeedDequeueTupleOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("shapes", &shapes_)); + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); + for (int i = 0; i < shapes_.size(); i++) { + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, + TensorShapeToXLAShape(dtypes_[i], shapes_[i], &xla_shape)); + xla_shapes_.push_back(xla_shape); + } + } + + ~InfeedDequeueTupleOp() override {} + + void Compile(XlaOpKernelContext* ctx) override { + xla::XlaBuilder* b = ctx->builder(); + for (int64 i = 0; i < xla_shapes_.size(); ++i) { + absl::optional sharding; + if (b->sharding()) { + sharding = b->sharding()->type() == xla::OpSharding::TUPLE + ? b->sharding()->tuple_shardings(i) + : b->sharding(); + } + OP_REQUIRES_OK(ctx, UpdateInfeedLayout(&xla_shapes_[i], sharding)); + } + tuple_shape_ = xla::ShapeUtil::MakeTupleShape(xla_shapes_); + auto tuple = xla::Infeed(b, tuple_shape_); + + // Don't apply the infeed tuple sharding to the get-tuple-elements. They + // need non-tuple shardings. + xla::XlaScopedShardingAssignment clear_sharding(b, absl::nullopt); + for (int i = 0; i < shapes_.size(); ++i) { + ctx->SetOutput(i, xla::GetTupleElement(tuple, i)); + } + } + + private: + std::vector shapes_; + DataTypeVector dtypes_; + std::vector xla_shapes_; + xla::Shape tuple_shape_; + + TF_DISALLOW_COPY_AND_ASSIGN(InfeedDequeueTupleOp); +}; + +REGISTER_XLA_OP(Name("InfeedDequeueTuple"), InfeedDequeueTupleOp); + +} // anonymous namespace +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/xla/inplace_ops.cc b/tensorflow/core/tpu/kernels/xla/inplace_ops.cc new file mode 100644 index 00000000000..9baffd6bbf0 --- /dev/null +++ b/tensorflow/core/tpu/kernels/xla/inplace_ops.cc @@ -0,0 +1,142 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/core/framework/op_kernel.h" + +#include "tensorflow/compiler/tf2xla/lib/scatter.h" +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "tensorflow/core/framework/kernel_def_builder.h" + +namespace tensorflow { +namespace { + +class InplaceUpdateOp : public XlaOpKernel { + public: + explicit InplaceUpdateOp(OpKernelConstruction* context) + : XlaOpKernel(context) {} + + void Compile(XlaOpKernelContext* ctx) override { + VLOG(3) << "InplaceUpdateOp::Compile"; + + DataType index_type = input_type(1); + OP_REQUIRES(ctx, index_type == DT_INT32 || index_type == DT_INT64, + errors::InvalidArgument("index must be int32 or int64")); + + // TF Args are X, I, V + const TensorShape x_shape = ctx->InputShape(0); + const TensorShape i_shape = ctx->InputShape(1); + const TensorShape v_shape = ctx->InputShape(2); + + OP_REQUIRES(ctx, + TensorShapeUtils::IsScalar(i_shape) || + TensorShapeUtils::IsVector(i_shape), + errors::InvalidArgument("index must be Rank 0 or 1")); + OP_REQUIRES(ctx, (x_shape.dims() == v_shape.dims()), + errors::InvalidArgument("X and V must have the same Rank," + " X.shape=", + x_shape.DebugString(), + " V.shape=", v_shape.DebugString())); + + auto* builder = ctx->builder(); + auto const_zero = xla::ConstantR0(builder, 0); + auto current = ctx->Input(0); + + for (int64 i = 0; i < i_shape.num_elements(); i++) { + std::vector update_indices; + update_indices.push_back( + xla::Reshape(xla::SliceInDim(ctx->Input(1), i, i + 1, 1, 0), {})); + for (int xi = 1; xi < x_shape.dims(); xi++) { + update_indices.push_back(const_zero); + } + current = xla::DynamicUpdateSlice( + current, xla::SliceInDim(ctx->Input(2), i, i + 1, 1, 0), + update_indices); + } + ctx->SetOutput(0, current); + + // TODO(b/118122460): Uncomment+format this code to use XLA Scatter. + // auto* builder = ctx->builder(); + // const auto initial = ctx->Input(0); + // const auto indices = ctx->Input(1); + // const auto updates = ctx->Input(2); + // + // auto result = XlaScatter( + // initial, updates, indices, /*indices_are_vectors=*/false, + // [](xla::XlaOp, xla::XlaOp second, xla::XlaBuilder*) { return + // second; }, builder); + // OP_REQUIRES_OK(ctx, result.status()); + // ctx->SetOutput(0, result.ValueOrDie()); + } +}; + +REGISTER_XLA_OP(Name("InplaceUpdate"), InplaceUpdateOp); + +class InplaceAddOp : public XlaOpKernel { + public: + explicit InplaceAddOp(OpKernelConstruction* context) : XlaOpKernel(context) {} + + void Compile(XlaOpKernelContext* ctx) override { + VLOG(3) << "InplaceAddOp::Compile"; + + DataType index_type = input_type(1); + OP_REQUIRES(ctx, index_type == DT_INT32 || index_type == DT_INT64, + errors::InvalidArgument("index must be int32 or int64")); + + // TF Args are X, I, V + const TensorShape x_shape = ctx->InputShape(0); + const TensorShape i_shape = ctx->InputShape(1); + const TensorShape v_shape = ctx->InputShape(2); + OP_REQUIRES(ctx, + (TensorShapeUtils::IsScalar(i_shape) || + ((i_shape.dims() == 1) && (i_shape.num_elements() == 1))), + errors::InvalidArgument("index must be Rank 1 and size 1")); + OP_REQUIRES(ctx, (x_shape.dims() == v_shape.dims()), + errors::InvalidArgument("X and V must have the same Rank," + " X.shape=", + x_shape.DebugString(), + " V.shape=", v_shape.DebugString())); + // Pad the indices out to the match the rank of params. + auto* builder = ctx->builder(); + std::vector padded_indices; + padded_indices.push_back(xla::Reshape(ctx->Input(1), {})); + for (int i = 0; i < x_shape.dims() - 1; ++i) { + padded_indices.push_back(XlaHelpers::Zero(builder, index_type)); + } + + std::vector sizes; + sizes.push_back(1); + for (int i = 1; i < x_shape.dims(); i++) { + sizes.push_back(x_shape.dim_size(i)); + } + + auto prev = xla::DynamicSlice(ctx->Input(0), padded_indices, sizes); + auto updated = xla::Add(prev, ctx->Input(2)); + auto result = + xla::DynamicUpdateSlice(ctx->Input(0), updated, padded_indices); + ctx->SetOutput(0, result); + } +}; + +REGISTER_XLA_OP(Name("InplaceAdd"), InplaceAddOp); + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/xla/outfeed_ops.cc b/tensorflow/core/tpu/kernels/xla/outfeed_ops.cc new file mode 100644 index 00000000000..8abdd3d171f --- /dev/null +++ b/tensorflow/core/tpu/kernels/xla/outfeed_ops.cc @@ -0,0 +1,91 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/tf2xla/shape_util.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/core/framework/op_kernel.h" + +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/core/framework/kernel_def_builder.h" + +namespace tensorflow { + +namespace { + +// This TensorFlow op implements the XLA Outfeed primitive. +class OutfeedEnqueueOp : public XlaOpKernel { + public: + explicit OutfeedEnqueueOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtype", &dtype_)); + } + + void Compile(XlaOpKernelContext* ctx) override { + xla::Shape xla_shape; + OP_REQUIRES_OK( + ctx, TensorShapeToXLAShape(dtype_, ctx->InputShape(0), &xla_shape)); + // Outfeed configuration is only needed for embedding outfeed. + const string outfeed_config; + xla::Outfeed(ctx->Input(0), xla_shape, outfeed_config); + } + + private: + DataType dtype_; + + TF_DISALLOW_COPY_AND_ASSIGN(OutfeedEnqueueOp); +}; + +REGISTER_XLA_OP(Name("OutfeedEnqueue"), OutfeedEnqueueOp); + +// This TensorFlow op implements the XLA Outfeed primitive for tuple types. +class OutfeedEnqueueTupleOp : public XlaOpKernel { + public: + explicit OutfeedEnqueueTupleOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("dtypes", &dtypes_)); + } + + void Compile(XlaOpKernelContext* ctx) override { + std::vector handles; + std::vector shapes; + auto inputs = ctx->InputList("inputs", &handles, &shapes); + + std::vector xla_shapes; + for (int i = 0; i < shapes.size(); ++i) { + xla::Shape xla_shape; + OP_REQUIRES_OK(ctx, + TensorShapeToXLAShape(dtypes_[i], shapes[i], &xla_shape)); + xla_shapes.push_back(xla_shape); + } + xla::Shape tuple_shape = xla::ShapeUtil::MakeTupleShape(xla_shapes); + VLOG(1) << "OutfeedEnqueueTuple: " + << xla::ShapeUtil::HumanStringWithLayout(tuple_shape); + auto b = ctx->builder(); + auto tuple = xla::Tuple(b, handles); + // Outfeed configuration is only needed for embedding outfeed. + const string outfeed_config; + xla::Outfeed(tuple, tuple_shape, outfeed_config); + } + + private: + DataTypeVector dtypes_; + + TF_DISALLOW_COPY_AND_ASSIGN(OutfeedEnqueueTupleOp); +}; + +REGISTER_XLA_OP(Name("OutfeedEnqueueTuple"), OutfeedEnqueueTupleOp); + +} // anonymous namespace +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/xla/segment_reduction_ops.cc b/tensorflow/core/tpu/kernels/xla/segment_reduction_ops.cc new file mode 100644 index 00000000000..f7c33e57fa0 --- /dev/null +++ b/tensorflow/core/tpu/kernels/xla/segment_reduction_ops.cc @@ -0,0 +1,145 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/tf2xla/lib/scatter.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/core/tpu/tpu_defs.h" + +namespace tensorflow { +namespace { +// TODO(b/32945756): Add a scatter op in XLA and move this to a HLO optimization +// pass. Optimization for UnsortedSegmentSum on TPU: use k-hot matmul. This +// optimization requires: +// 1. data has dtype supported by TPU matmul and has rank of 1 or 2. +// 2. indices has rank of 1. +// 3. matmul op count is less than 800 billion. +// +// Example of calculating UnsortedSegmentSum by k-hot matmul: +// data shape [A, B] +// indices shape [A] +// num_segment N +// output shape [N, B] +// matmul op count N * A * B +// Step 1: create k-hot matrix +// k-hot matrix has shape of [A, N], where row i is responsible for +// collecting the sum of the i-th segment, concretely +// k-hot[i][j] = 1 if indices[i] = j +// Step 2: perform matmul +// the final result is obtained by multiplying k-hot matrix with data +// matrix, namely +// k-hot * data => result +// shape: [N, A] * [A, B] => [N, B] +xla::XlaOp KHotMatmul(XlaOpKernelContext* ctx, xla::XlaBuilder* builder, + const xla::XlaOp data, const xla::XlaOp indices, + int64 num_segments) { + DataType data_dtype = ctx->input_type(0); + xla::PrimitiveType indices_type = ctx->input_xla_type(1); + TensorShape data_shape = ctx->InputShape(0); + TensorShape indices_shape = ctx->InputShape(1); + xla::XlaOp linspace = xla::Iota(builder, indices_type, num_segments); + xla::XlaOp linspace_col = xla::Reshape(linspace, {num_segments, 1}); + TensorShape indices_row_shape = indices_shape; + indices_row_shape.InsertDim(0, 1); + xla::XlaOp indices_row = xla::Reshape(indices, indices_row_shape.dim_sizes()); + xla::XlaOp k_hot = xla::Eq(indices_row, linspace_col); + xla::XlaOp k_hot_with_data_dtype = + XlaHelpers::ConvertElementType(k_hot, data_dtype); + // F32 version of the KHotMatmul. It splits the F32 data into three + // BF16 partial data and run KHotMatmul for each of them. The final result + // is the summation of three BF16 results. + // Note that this still doesn't fully retain f32 precision. + // In particular, values smaller than 2^-111 may see loss of precision. + xla::PrecisionConfig precision_config; + if (data_dtype == DT_FLOAT) { + precision_config.add_operand_precision(xla::PrecisionConfig::HIGHEST); + } else { + CHECK_EQ(data_dtype, DT_BFLOAT16); + precision_config.add_operand_precision(xla::PrecisionConfig::DEFAULT); + } + precision_config.add_operand_precision(xla::PrecisionConfig::DEFAULT); + return xla::Dot(k_hot_with_data_dtype, data, &precision_config); +} + +class UnsortedSegmentSum : public XlaOpKernel { + public: + explicit UnsortedSegmentSum(OpKernelConstruction* ctx) : XlaOpKernel(ctx) { + OP_REQUIRES_OK(ctx, ctx->GetAttr("T", &dtype_)); + } + + void Compile(XlaOpKernelContext* ctx) override { + // output = unsorted_segment_sum(data, indices, num_segments) + // Compute a tensor such that: + // output[i] = sum over {j where indices[j] == i} of data[j] + // output[i] == 0 if i does not appear in indices + // + // Contrast with segment_sum(), which assumes indices are sorted and that + // max(indices)+1 is the desired size of the output. + // + // The returned output tensor has the same type as data, and the same shape + // as data with the first indices.rank dimensions are replaced + // by a single dimension with size num_segments. + xla::XlaOp data = ctx->Input(0); + TensorShape data_shape = ctx->InputShape(0); + + xla::XlaOp indices = ctx->Input(1); + TensorShape indices_shape = ctx->InputShape(1); + + int64 num_segments; + OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntScalar(2, &num_segments)); + + OP_REQUIRES(ctx, data_shape.dims() >= indices_shape.dims(), + errors::InvalidArgument( + "UnsortedSegmentSum requires that indices' rank be" + " less than or equal to data's rank.")); + // Validate that indices.shape is a prefix of data.shape. + for (int d = 0; d < indices_shape.dims(); ++d) { + OP_REQUIRES(ctx, (data_shape.dim_size(d) == indices_shape.dim_size(d)), + errors::InvalidArgument( + "UnsortedSegmentSum requires indices shape to be prefix" + " of data_shape, but dimension ", + d, " differs ", data_shape.dim_size(d), " vs. ", + indices_shape.dim_size(d))); + } + xla::XlaBuilder* builder = ctx->builder(); + TensorShape buffer_shape = data_shape; + buffer_shape.RemoveDimRange(0, indices_shape.dims()); + buffer_shape.InsertDim(0, num_segments); + auto buffer = xla::Broadcast(XlaHelpers::Zero(builder, dtype_), + buffer_shape.dim_sizes()); + + auto combiner = [](xla::XlaOp a, xla::XlaOp b, xla::XlaBuilder* builder) { + return a + b; + }; + + auto result = XlaScatter(buffer, /*updates=*/data, indices, + /*indices_are_vectors=*/false, combiner, builder); + OP_REQUIRES_OK(ctx, result.status()); + ctx->SetOutput(0, result.ValueOrDie()); + } + + private: + DataType dtype_; +}; + +REGISTER_XLA_OP(Name("UnsortedSegmentSum") + .Device(DEVICE_TPU_XLA_JIT) + .CompileTimeConstantInput("num_segments"), + UnsortedSegmentSum); + +} // namespace +} // namespace tensorflow diff --git a/tensorflow/core/tpu/kernels/xla/where_op.cc b/tensorflow/core/tpu/kernels/xla/where_op.cc new file mode 100644 index 00000000000..420d5bcb9c3 --- /dev/null +++ b/tensorflow/core/tpu/kernels/xla/where_op.cc @@ -0,0 +1,91 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/compiler/tf2xla/literal_util.h" +#include "tensorflow/compiler/tf2xla/type_util.h" +#include "tensorflow/compiler/tf2xla/xla_helpers.h" +#include "tensorflow/compiler/tf2xla/xla_op_kernel.h" +#include "tensorflow/compiler/tf2xla/xla_op_registry.h" +#include "tensorflow/compiler/xla/client/lib/arithmetic.h" +#include "tensorflow/compiler/xla/client/lib/comparators.h" +#include "tensorflow/compiler/xla/client/lib/constants.h" +#include "tensorflow/compiler/xla/client/xla_builder.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/ops_util.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/tpu/tpu_defs.h" + +namespace tensorflow { +namespace { + +class WhereOp : public XlaOpKernel { + public: + explicit WhereOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {} + + void Compile(XlaOpKernelContext* ctx) override { + xla::XlaOp condition = ctx->Input(0); + xla::StatusOr input_shape = ctx->builder()->GetShape(condition); + OP_REQUIRES_OK(ctx, input_shape.status()); + // Use S32 as indices first, then convert to S64 in the end if needed. + auto iota_shape = input_shape.ValueOrDie(); + iota_shape.set_element_type(xla::S32); + + int64 flattened_size = xla::Product(iota_shape.dimensions()); + xla::XlaOp reshaped_condition = xla::Reshape(condition, {flattened_size}); + xla::XlaOp zeros = xla::ZerosLike(reshaped_condition); + xla::XlaOp zeros_int = xla::ConvertElementType(zeros, xla::S32); + xla::XlaOp reshaped_condition_int = + xla::ConvertElementType(reshaped_condition, xla::S32); + xla::XlaOp compared = xla::ConvertElementType( + xla::Gt(reshaped_condition_int, zeros_int), xla::S32); + xla::XlaOp length = xla::ReduceAll( + compared, xla::Zero(ctx->builder(), xla::S32), + xla::CreateScalarAddComputation(xla::S32, ctx->builder())); + + std::vector to_sort = {reshaped_condition_int}; + std::vector types_to_sort = {xla::S32}; + // Generate iota for each dimension, which after combining becomes + // indices of each element. + for (int64 axis = 0; axis < iota_shape.rank(); ++axis) { + xla::XlaOp iota = xla::Iota(ctx->builder(), iota_shape, axis); + xla::XlaOp reshaped = xla::Reshape(iota, {flattened_size}); + to_sort.push_back(reshaped); + types_to_sort.push_back(xla::S32); + } + + xla::XlaOp sorted = xla::Sort( + to_sort, xla::CreateScalarGtComputation(types_to_sort, ctx->builder()), + /*dimension=*/0, + /*is_stable=*/true); + std::vector to_concat; + for (int64 i = 0; i < iota_shape.rank(); ++i) { + xla::XlaOp index_single_dim = xla::GetTupleElement(sorted, i + 1); + to_concat.push_back(xla::Reshape(index_single_dim, {flattened_size, 1})); + } + + xla::XlaOp result = xla::ConcatInDim(ctx->builder(), to_concat, 1); + result = xla::ConvertElementType(result, ctx->output_xla_type(0)); + // Dynamic padder will handle the dynamic dimension. + xla::XlaOp result_padded = xla::SetDimensionSize(result, length, 0); + ctx->SetOutput(0, result_padded); + } +}; + +REGISTER_XLA_OP(Name("Where").Device(DEVICE_TPU_XLA_JIT), WhereOp); + +} // namespace +} // namespace tensorflow From 730b802ee896dc410a1517b3ceaf7cc7183dabdb Mon Sep 17 00:00:00 2001 From: Ken Franko Date: Wed, 12 Aug 2020 17:09:55 -0700 Subject: [PATCH 0943/1017] Add unit test for outside compilation with summary inside control flow. This covers automatic outside compilation with If control flow and summary ops. PiperOrigin-RevId: 326345698 Change-Id: Id308670fd002036ffdc51e64d5995d057d157493 --- .../tpu/tpu_outside_compilation_test.py | 42 ++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/tpu/tpu_outside_compilation_test.py b/tensorflow/python/tpu/tpu_outside_compilation_test.py index 72e9f10d184..7e0278aa343 100644 --- a/tensorflow/python/tpu/tpu_outside_compilation_test.py +++ b/tensorflow/python/tpu/tpu_outside_compilation_test.py @@ -470,7 +470,8 @@ class TpuOutsideCompilationTest(test.TestCase, parameterized.TestCase): constant_op.constant(2916., shape=(strategy.num_replicas_in_sync))) -class OutsideCompilationOnUnsupportedOpTest(test.TestCase): +class OutsideCompilationOnUnsupportedOpTest(test.TestCase, + parameterized.TestCase): def setUp(self): super(OutsideCompilationOnUnsupportedOpTest, self).setUp() @@ -536,6 +537,45 @@ class OutsideCompilationOnUnsupportedOpTest(test.TestCase): self.assertEqual(events[1].summary.value[0].tag, "x") self.assertEqual(events[1].summary.value[0].simple_value, 3.0) + @parameterized.parameters((True), (False)) + def testSummaryControlFlowIfWithAutoOutsideCompilation( + self, take_true_branch): + strategy = get_tpu_strategy() + + @def_function.function + def step(): + + def computation(x): + x = x + 1.0 + if x < 5: + summary.scalar("x", x, step=0) + x = x * 2.0 + return x + 1.0 + + if take_true_branch: + return strategy.run(computation, args=(2.0,)) + else: + return strategy.run(computation, args=(10.0,)) + + logdir = tempfile.mkdtemp() + summary_writer = summary.create_file_writer(logdir, flush_millis=10000) + output_value = 12. + if take_true_branch: + output_value = 7. + with summary_writer.as_default(), summary.always_record_summaries(): + self.assertAllEqual( + strategy.experimental_local_results(step()), + constant_op.constant( + output_value, shape=(strategy.num_replicas_in_sync))) + if take_true_branch: + events = _events_from_logdir(self, logdir) + # There will be 2 entries: 1 summary file header entry, and 1 entry + # written by host. + # + self.assertLen(events, 2) + self.assertEqual(events[1].summary.value[0].tag, "cond/x") + self.assertEqual(events[1].summary.value[0].simple_value, 3.0) + def testAutoOutsideCompilationWithFunctionalNodes(self): strategy = get_tpu_strategy() From ac09152e882f7ba535707a35088410189b28733b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 17:21:53 -0700 Subject: [PATCH 0944/1017] Temporary change: added default value for beta parameter to allow it to be used in Estimator tests. PiperOrigin-RevId: 326347547 Change-Id: I34fc3daa8569156fb37284ffb81334c64bc670a9 --- tensorflow/python/keras/optimizer_v2/ftrl.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/python/keras/optimizer_v2/ftrl.py b/tensorflow/python/keras/optimizer_v2/ftrl.py index 0e96724a44d..512f55748f6 100644 --- a/tensorflow/python/keras/optimizer_v2/ftrl.py +++ b/tensorflow/python/keras/optimizer_v2/ftrl.py @@ -209,5 +209,7 @@ class Ftrl(optimizer_v2.OptimizerV2): self._serialize_hyperparameter('l2_regularization_strength'), 'l2_shrinkage_regularization_strength': self._l2_shrinkage_regularization_strength, + 'beta': + 0.0, }) return config From 8b03b9681e605bb55ac3605d3b4931190fe98add Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 12 Aug 2020 17:23:42 -0700 Subject: [PATCH 0945/1017] Fix memory leaks in TpuTransferManager by calling ApiConverter::Free PiperOrigin-RevId: 326347833 Change-Id: I4d6c3935cdcc56c16842e7985f10e44c2b0c34a1 --- tensorflow/stream_executor/tpu/tpu_transfer_manager.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc b/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc index e6b3460ac92..b49a0b56259 100644 --- a/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc +++ b/tensorflow/stream_executor/tpu/tpu_transfer_manager.cc @@ -93,6 +93,8 @@ Status TpuTransferManager::TransferLiteralToInfeed( tpu::ExecutorApiFn()->TpuTransferManager_TransferLiteralToInfeedFn( manager_, tpu_executor->se_executor(), &c_literal, status.c_status); + ApiConverter::Free(&c_literal); + return status.status(); } @@ -135,6 +137,9 @@ Status TpuTransferManager::TransferLiteralFromOutfeed( manager_, tpu_executor->se_executor(), &c_shape, &c_literal, status.c_status); + ApiConverter::Free(&c_shape); + ApiConverter::Free(&c_literal); + return status.status(); } @@ -265,6 +270,7 @@ Status TpuTransferManager::LinearizeToBuffers( tpu::ExecutorApiFn()->TpuTransferManager_FreeBuffersFn( buffers_array, buffers_size, buffers_array_size); + ApiConverter::Free(&c_literal); return status.status(); } From 97b414dd4658ee13d1e590319958a41d25743e90 Mon Sep 17 00:00:00 2001 From: Yuanzhong Xu Date: Wed, 12 Aug 2020 17:28:50 -0700 Subject: [PATCH 0946/1017] [XLA:SPMD] Add partial sharding API to SPMD and bridge support PiperOrigin-RevId: 326348623 Change-Id: I32e94f708ff7c13aa17fabac645100f178c2e1be --- .../experimental/xla_sharding/xla_sharding.py | 63 ++++++++++++++-- .../distributed_tpu_rewrite_pass.cc | 71 ++++++++++--------- 2 files changed, 96 insertions(+), 38 deletions(-) diff --git a/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py b/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py index 16563bab5bc..a926e8b3c88 100644 --- a/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py +++ b/tensorflow/compiler/xla/experimental/xla_sharding/xla_sharding.py @@ -89,6 +89,32 @@ class Sharding(object): tile_assignment_dimensions=dims, tile_assignment_devices=list(flattened_devices))) + @classmethod + def partial_tile(cls, tile_assignment): + """Returns a partially tiled sharding attribute. + + This is similar to tile(), but tile_assignment has one more dimension than + the tensor, and tiles in the last dimension of tile_assignment are + replicated. + + Args: + tile_assignment: An np.ndarray describing the topology of the tiling and + which device will compute which part of the topology. + + Raises: + TypeError: tile_assignment was not of np.array type. + """ + if not isinstance(tile_assignment, _np.ndarray): + raise TypeError('PartialTile assignment must be of type np.ndarray') + dims = list(tile_assignment.shape) + flattened_devices = tile_assignment.reshape(-1, order='C') + return Sharding( + proto=xla_data_pb2.OpSharding( + type=xla_data_pb2.OpSharding.OTHER, + tile_assignment_dimensions=dims, + tile_assignment_devices=list(flattened_devices), + replicate_on_last_tile_dim=True)) + @classmethod def split(cls, tensor, split_dimension, num_devices, input_shape=None): """Returns a Sharding that splits a tensor across a dimension. @@ -245,6 +271,23 @@ def split(tensor, return tensor +def partial_tile(tensor, tile_assignment, use_sharding_op=False): + """Returns a tensor that has tiled sharding. + + Args: + tensor: A tf.Tensor to shard. + tile_assignment: An np.ndarray describing the topology of the tiling and + which device will compute which part of the topology. It must have one + more dimension than tensor, and the last dimension represents partially + replicated tiles. + use_sharding_op: If true, adds a sharding op to set the sharding. + """ + if use_sharding_op: + tensor = tf2xla.sharding(tensor) + Sharding.partial_tile(tile_assignment).apply_to_tensor(tensor) + return tensor + + def get_op_sharding(op): """Returns sharding attribute of an op. @@ -313,20 +356,30 @@ def mesh_split(tensor, use_sharding_op: If true, adds a sharding op to set the sharding. Raises: - ValueError: The number of tensor split dimensions is different from device - mesh rank. + ValueError: The number of tensor split dimensions is larger than device mesh + rank. """ permutation = [d for d in tensor_split_dims_mapping if d >= 0] - if len(permutation) != len(device_mesh.shape): + if len(permutation) > len(device_mesh.shape): raise ValueError( - 'Number of tensor split dimensions (%r) is different from device mesh ' + 'Number of tensor split dimensions (%r) is larger than device mesh ' 'rank (%r). tensor_split_dims_mapping: %r, device_mesh.shape: %r' % (len(permutation), len( device_mesh.shape), tensor_split_dims_mapping, device_mesh.shape)) - tile_assignment = _np.transpose(device_mesh, permutation) + # Append replicated dimensions to the end. + transpose_permutation = permutation + [ + d for d in range(len(device_mesh.shape)) if d not in permutation + ] + tile_assignment = _np.transpose(device_mesh, transpose_permutation) tile_shape = [ 1 if d < 0 else device_mesh.shape[d] for d in tensor_split_dims_mapping ] + partial = len(permutation) < len(device_mesh.shape) + if partial: + tile_shape.append(_np.prod(device_mesh.shape) // _np.prod(tile_shape)) tile_assignment = _np.reshape(tile_assignment, tile_shape) + if partial: + return partial_tile( + tensor, tile_assignment, use_sharding_op=use_sharding_op) return tile(tensor, tile_assignment, use_sharding_op=use_sharding_op) diff --git a/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.cc b/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.cc index 73510319b0a..882947c1c65 100644 --- a/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.cc +++ b/tensorflow/core/tpu/graph_rewrite/distributed_tpu_rewrite_pass.cc @@ -599,8 +599,11 @@ Status GetStepMarkerLocation(const Node& replicate_node, // sharding attribute. Status GetDimensionIndicesAndNumSplitsFromSharding( const xla::OpSharding& sharding, std::map* split_dimension_map) { - for (int dim_index = 0; - dim_index < sharding.tile_assignment_dimensions_size(); dim_index++) { + int64 tensor_tile_rank = sharding.tile_assignment_dimensions_size(); + if (sharding.replicate_on_last_tile_dim()) { + tensor_tile_rank--; + } + for (int dim_index = 0; dim_index < tensor_tile_rank; dim_index++) { if (sharding.tile_assignment_dimensions(dim_index) > 1) { split_dimension_map->emplace( dim_index, sharding.tile_assignment_dimensions(dim_index)); @@ -777,8 +780,9 @@ xla::StatusOr CreateOrGetSplitNodesForInputSharding( // `split_nodes_for_dimension` now includes final split nodes // from which sharded data will be fed into TPUExcute nodes -- sorted by // row major order. - std::vector sharded_inputs_list; - sharded_inputs_list.reserve(split_nodes_for_dimension.size()); + std::vector sharded_inputs_list( + sharding.tile_assignment_devices_size()); + int64 next_core_tile_index = 0; while (!split_nodes_for_dimension.empty()) { Node* split_node = split_nodes_for_dimension.front(); split_nodes_for_dimension.pop(); @@ -786,7 +790,14 @@ xla::StatusOr CreateOrGetSplitNodesForInputSharding( TF_RETURN_IF_ERROR( GetNodeAttr(split_node->def(), "num_split", &num_splits)); for (int out_index = 0; out_index < num_splits; ++out_index) { - sharded_inputs_list.emplace_back(NodeOut{split_node, out_index}); + int64 repeat_count = sharding.replicate_on_last_tile_dim() + ? *sharding.tile_assignment_dimensions().rbegin() + : 1; + for (int64 i = 0; i < repeat_count; ++i) { + int64 next_core = + sharding.tile_assignment_devices(next_core_tile_index++); + sharded_inputs_list[next_core] = NodeOut{split_node, out_index}; + } } } @@ -889,19 +900,6 @@ xla::StatusOr CreateConcatNodesForRetval( return inputs_to_sharded_retval.at(0).node; } -absl::optional GetCoreIndexInSharding(const xla::OpSharding& sharding, - int64 core) { - absl::optional output_index; - for (int i = 0; i < sharding.tile_assignment_devices_size(); i++) { - int64 assigned_core = sharding.tile_assignment_devices(i); - if (assigned_core == core) { - output_index = i; - break; - } - } - return output_index; -} - // Set the padding ops the same devices as the original inputs. If the original // inputs are on TPUs, the padding ops will be placed on TPUs and XLA on demand // mode will be triggered, so we don't need to copy the data back to the host @@ -2763,14 +2761,8 @@ Status DistributedTPURewritePass::BuildExecuteNodes( sharding, orig_arg_num, dtype, replica, edge->src_output(), edge->src(), control_predecessor, graph, &input_index_to_sharded_inputs)); - - // Calculate which output we should receive from the Split node. - absl::optional output_index = - GetCoreIndexInSharding(sharding, core); - TF_RET_CHECK(output_index); - NodeOut split_node_and_index = - sharded_input_info.sharded_inputs.at(output_index.value()); + sharded_input_info.sharded_inputs.at(core); // Connect with Split node output. graph->AddEdge(split_node_and_index.node, split_node_and_index.index, node, i); @@ -2850,13 +2842,8 @@ Status DistributedTPURewritePass::BuildExecuteNodes( arg_shapes[orig_arg_num].handle_type, replica, var_data.index, var_data.node, control_predecessor, graph, &input_index_to_sharded_inputs)); - - // Calculate which output we should receive from the Split node. - absl::optional output_index = - GetCoreIndexInSharding(sharding, core); - TF_RET_CHECK(output_index); NodeOut split_node_and_index = - sharded_input_info.sharded_inputs[output_index.value()]; + sharded_input_info.sharded_inputs[core]; // Connect with Split node output. graph->AddEdge(split_node_and_index.node, split_node_and_index.index, node, i); @@ -2919,7 +2906,16 @@ Status DistributedTPURewritePass::BuildExecuteNodes( // Add a Concat node. std::vector orig_inputs; - for (int64 core_id : sharding.tile_assignment_devices()) { + for (int64 tile_index = 0; + tile_index < sharding.tile_assignment_devices_size(); + ++tile_index) { + int64 last_tile_dim_size = + *sharding.tile_assignment_dimensions().rbegin(); + if (sharding.replicate_on_last_tile_dim() && + tile_index % last_tile_dim_size != 0) { + continue; + } + int64 core_id = sharding.tile_assignment_devices(tile_index); int core_retval_index = retval_index_to_output_index_mapping[retval_index][core_id]; orig_inputs.push_back( @@ -2987,7 +2983,16 @@ Status DistributedTPURewritePass::BuildExecuteNodes( // Add a Concat node. std::vector orig_inputs; - for (int64 core_id : sharding.tile_assignment_devices()) { + for (int64 tile_index = 0; + tile_index < sharding.tile_assignment_devices_size(); + ++tile_index) { + int64 last_tile_dim_size = + *sharding.tile_assignment_dimensions().rbegin(); + if (sharding.replicate_on_last_tile_dim() && + tile_index % last_tile_dim_size != 0) { + continue; + } + int64 core_id = sharding.tile_assignment_devices(tile_index); int core_retval_num = orig_arg_num_to_output_index_mapping[orig_arg_num][core_id]; orig_inputs.push_back( From d4beef66937a2c4a8e942f00efb5f5ceee12802c Mon Sep 17 00:00:00 2001 From: Yanhui Liang Date: Wed, 12 Aug 2020 17:30:07 -0700 Subject: [PATCH 0947/1017] Internal change for Keras benchmarks. PiperOrigin-RevId: 326348822 Change-Id: I3c0e3002ca0d8ce0c1ef616180f7e2dc9881de50 --- tensorflow/python/data/experimental/service/BUILD | 1 - tensorflow/python/keras/benchmarks/BUILD | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow/python/data/experimental/service/BUILD b/tensorflow/python/data/experimental/service/BUILD index f072c5f2208..e8af29caa0e 100644 --- a/tensorflow/python/data/experimental/service/BUILD +++ b/tensorflow/python/data/experimental/service/BUILD @@ -18,7 +18,6 @@ tf_python_pybind_extension( "//tensorflow/python:pybind11_lib", "//tensorflow/python:pybind11_status", "//third_party/python_runtime:headers", - "@com_github_grpc_grpc//:grpc++_public_hdrs", "@pybind11", ], ) diff --git a/tensorflow/python/keras/benchmarks/BUILD b/tensorflow/python/keras/benchmarks/BUILD index 28a80820bc2..2252f888780 100644 --- a/tensorflow/python/keras/benchmarks/BUILD +++ b/tensorflow/python/keras/benchmarks/BUILD @@ -33,6 +33,12 @@ py_library( ], ) +# This lib is mainly for running benchmarks on mlcompass infra. +py_library( + name = "profiler_lib", + visibility = ["//tensorflow:internal"], +) + COMMON_TAGS = [ "no_pip", # b/161253163 "no_windows", # b/160628318 @@ -46,6 +52,7 @@ py_test( tags = COMMON_TAGS, deps = [ ":benchmark_util", + ":profiler_lib", "//tensorflow:tensorflow_py", "//third_party/py/numpy", ], @@ -90,6 +97,7 @@ cuda_py_test( tags = COMMON_TAGS, deps = [ ":benchmark_util", + ":profiler_lib", "//tensorflow:tensorflow_py", ], ) From 594e98e0243117c92be61160c2cc10dd15edd6db Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 17:30:29 -0700 Subject: [PATCH 0948/1017] Add GPU TensorCore utilization column to Tensorflow Stats page. PiperOrigin-RevId: 326348881 Change-Id: I256ef7306db6352b75b317b43b237c1b6c2582ea --- tensorflow/python/profiler/internal/profiler_wrapper.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/profiler/internal/profiler_wrapper.cc b/tensorflow/python/profiler/internal/profiler_wrapper.cc index 1ac294e720e..5956297c2e4 100644 --- a/tensorflow/python/profiler/internal/profiler_wrapper.cc +++ b/tensorflow/python/profiler/internal/profiler_wrapper.cc @@ -190,7 +190,7 @@ PYBIND11_MODULE(_pywrap_profiler, m) { xspace.ParseFromString(std::string(serialized_xspace_proto)); tensorflow::profiler::TfStatsDatabase tf_stats_db = tensorflow::profiler::ConvertOpStatsToTfStats( - ConvertXSpaceToOpStats(xspace, {OP_METRICS_DB})); + ConvertXSpaceToOpStats(xspace, {OP_METRICS_DB, KERNEL_STATS_DB})); return py::bytes(tf_stats_db.SerializeAsString()); }); From 8819d5b652592f19389738fa8aa9f7d14acfcee7 Mon Sep 17 00:00:00 2001 From: River Riddle Date: Wed, 12 Aug 2020 17:41:18 -0700 Subject: [PATCH 0949/1017] Remove kindof definitions on derived mlir Attribute and Type classes This method no longer needs to be explicitly defined by the user, so all of the existing definitions are dead and can be removed. PiperOrigin-RevId: 326350650 Change-Id: I1e8bc82c392bf400c2718bd85557924cf98a7ee9 --- .../mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.h | 3 --- tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h | 4 ---- tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h | 6 ------ tensorflow/compiler/mlir/tensorflow/ir/tf_types.h | 6 ------ .../compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h | 5 ----- 5 files changed, 24 deletions(-) diff --git a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.h b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.h index 0036cc0dc19..ad044e1d322 100644 --- a/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.h +++ b/tensorflow/compiler/mlir/hlo/include/mlir-hlo/Dialect/mhlo/IR/hlo_ops.h @@ -69,9 +69,6 @@ class TokenType : public Type::TypeBase { static TokenType get(MLIRContext *context) { return Base::get(context, HLOTypes::Token); } - - // Support method to enable LLVM-style type casting. - static bool kindof(unsigned kind) { return kind == HLOTypes::Token; } }; // Shape derivation function that computes the shape of the result based on diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h index 1edc7356ab4..e0fef228eb4 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_attributes.h @@ -70,8 +70,6 @@ class ShapeAttr : public Attribute::AttrBase= 0), it has static // shape. bool hasStaticShape() const; - - static bool kindof(unsigned kind) { return kind == AttrKind::SHAPE; } }; // Custom attribute to model AttrValue.value.func (NameAttrList type attribute). @@ -97,8 +95,6 @@ class FuncAttr SymbolRefAttr GetName() const; DictionaryAttr GetAttrs() const; - - static bool kindof(unsigned kind) { return kind == AttrKind::FUNC; } }; } // namespace TF diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h index 61358172d6d..da63826a6d4 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_executor.h @@ -61,9 +61,6 @@ class ControlType : public Type::TypeBase { static ControlType get(MLIRContext *context) { return Base::get(context, TFTypes::Control); } - - // Support method to enable LLVM-style type casting. - static bool kindof(unsigned kind) { return kind == TFTypes::Control; } }; class TokenType : public Type::TypeBase { @@ -73,9 +70,6 @@ class TokenType : public Type::TypeBase { static TokenType get(MLIRContext *context) { return Base::get(context, TFTypes::Token); } - - // Support method to enable LLVM-style type casting. - static bool kindof(unsigned kind) { return kind == TFTypes::Token; } }; // Declares the operations for this dialect using the generated header. diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_types.h b/tensorflow/compiler/mlir/tensorflow/ir/tf_types.h index 125f6bb31df..43d5f2fa476 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_types.h +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_types.h @@ -121,9 +121,6 @@ class TensorFlowTypeImpl static Derived get(MLIRContext* context) { return Base::get(context, Derived::getTypeKind()); } - - // Support method to enable LLVM-style type casting. - static bool kindof(unsigned kind) { return kind == Derived::getTypeKind(); } }; } // namespace detail @@ -243,9 +240,6 @@ class TypeWithSubtypeImpl static Derived get(MLIRContext* context) { return get({}, context); } - // Support method to enable LLVM-style type casting. - static bool kindof(unsigned kind) { return kind == Derived::getTypeKind(); } - static LogicalResult verifyConstructionInvariants( Location loc, ArrayRef subtypes) { // Each of the subtypes should be a valid TensorFlow type. diff --git a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h index 8d6e433d9b9..a4c588a41f5 100644 --- a/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h +++ b/tensorflow/compiler/mlir/tools/kernel_gen/ir/tf_framework_ops.h @@ -46,11 +46,6 @@ class OpKernelContextType static OpKernelContextType get(MLIRContext *context) { return Base::get(context, TFFrameworkTypes::Kind::OpKernelContextType); } - - /// Support method to enable LLVM-style type casting. - static bool kindof(unsigned kind) { - return kind == TFFrameworkTypes::Kind::OpKernelContextType; - } }; #define GET_OP_CLASSES From 6651e37dbf8585fdeff07a12c45f85ecf36aafc9 Mon Sep 17 00:00:00 2001 From: Andrew Audibert Date: Wed, 12 Aug 2020 18:20:08 -0700 Subject: [PATCH 0950/1017] [tf.data service] Store datasets in a "datasets" directory. This doesn't immediately change any functionality, but it will later enable us to transfer datasets to workers by sharing paths to the datasets instead of passing the full dataset definition, which may be many MB in size. PiperOrigin-RevId: 326356135 Change-Id: I7d89fa1eee53aa9f4f2f04ba37fc304a056aa68e --- tensorflow/core/data/service/BUILD | 29 +++++ tensorflow/core/data/service/dataset_store.cc | 92 ++++++++++++++ tensorflow/core/data/service/dataset_store.h | 77 ++++++++++++ .../core/data/service/dataset_store_test.cc | 115 ++++++++++++++++++ .../core/data/service/dispatcher_impl.cc | 34 +++++- .../core/data/service/dispatcher_impl.h | 3 + .../core/data/service/dispatcher_state.cc | 3 +- .../core/data/service/dispatcher_state.h | 8 +- .../data/service/dispatcher_state_test.cc | 29 +++-- tensorflow/core/data/service/journal.proto | 3 +- 10 files changed, 367 insertions(+), 26 deletions(-) create mode 100644 tensorflow/core/data/service/dataset_store.cc create mode 100644 tensorflow/core/data/service/dataset_store.h create mode 100644 tensorflow/core/data/service/dataset_store_test.cc diff --git a/tensorflow/core/data/service/BUILD b/tensorflow/core/data/service/BUILD index c64748c6452..da41c71b397 100644 --- a/tensorflow/core/data/service/BUILD +++ b/tensorflow/core/data/service/BUILD @@ -49,6 +49,34 @@ tf_proto_library( ], ) +cc_library( + name = "dataset_store", + srcs = ["dataset_store.cc"], + hdrs = ["dataset_store.h"], + deps = [ + ":common_proto_cc", + ":dispatcher_state", + "//tensorflow/core:lib", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + ], +) + +tf_cc_test( + name = "dataset_store_test", + srcs = ["dataset_store_test.cc"], + deps = [ + ":common_proto_cc", + ":dataset_store", + "//tensorflow/core:lib", + "//tensorflow/core:test", + "//tensorflow/core:test_main", + "//tensorflow/core:testlib", + "@com_google_absl//absl/memory", + ], +) + cc_library( name = "dispatcher_impl", srcs = ["dispatcher_impl.cc"], @@ -59,6 +87,7 @@ cc_library( ":common_proto_cc", ":credentials_factory", ":data_service", + ":dataset_store", ":dispatcher_proto_cc", ":dispatcher_state", ":grpc_util", diff --git a/tensorflow/core/data/service/dataset_store.cc b/tensorflow/core/data/service/dataset_store.cc new file mode 100644 index 00000000000..1cb10508555 --- /dev/null +++ b/tensorflow/core/data/service/dataset_store.cc @@ -0,0 +1,92 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow/core/data/service/dataset_store.h" + +#include + +#include "absl/memory/memory.h" +#include "tensorflow/core/data/service/common.pb.h" +#include "tensorflow/core/lib/io/record_reader.h" +#include "tensorflow/core/lib/io/record_writer.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/file_system.h" +#include "tensorflow/core/platform/path.h" + +namespace tensorflow { +namespace data { + +FileSystemDatasetStore::FileSystemDatasetStore(const std::string& datasets_dir) + : datasets_dir_(datasets_dir) {} + +Status FileSystemDatasetStore::Put(const std::string& key, + const DatasetDef& dataset) { + std::string path_to_write = io::JoinPath(datasets_dir_, key); + + if (Env::Default()->FileExists(path_to_write).ok()) { + return errors::AlreadyExists("File ", path_to_write, " already exists"); + } + std::unique_ptr file; + TF_RETURN_IF_ERROR(Env::Default()->NewWritableFile(path_to_write, &file)); + io::RecordWriter writer(file.get()); + TF_RETURN_IF_ERROR(writer.WriteRecord(dataset.SerializeAsString())); + return Status::OK(); +} + +Status FileSystemDatasetStore::Get( + const std::string& key, std::shared_ptr& dataset_def) { + std::string path = io::JoinPath(datasets_dir_, key); + TF_RETURN_IF_ERROR(Env::Default()->FileExists(path)); + std::unique_ptr file; + TF_RETURN_IF_ERROR(Env::Default()->NewRandomAccessFile(path, &file)); + io::RecordReader reader(file.get()); + uint64 offset = 0; + tstring record; + TF_RETURN_IF_ERROR(reader.ReadRecord(&offset, &record)); + dataset_def = std::make_shared(); + auto def = std::make_shared(); + if (!def->ParseFromString(record)) { + return errors::DataLoss("Failed to parse dataset definition"); + } + dataset_def = std::move(def); + return Status::OK(); +} + +MemoryDatasetStore::MemoryDatasetStore() {} + +Status MemoryDatasetStore::Put(const std::string& key, + const DatasetDef& dataset) { + auto& stored_dataset = datasets_[key]; + if (stored_dataset) { + return errors::AlreadyExists("Dataset with key ", key, + " is already stored."); + } + stored_dataset = std::make_shared(dataset); + return Status::OK(); +} + +Status MemoryDatasetStore::Get(const std::string& key, + std::shared_ptr& dataset_def) { + auto& stored_dataset = datasets_[key]; + if (!stored_dataset) { + return errors::NotFound("Dataset with key ", key, " not found"); + } + dataset_def = stored_dataset; + return Status::OK(); +} + +} // namespace data +} // namespace tensorflow diff --git a/tensorflow/core/data/service/dataset_store.h b/tensorflow/core/data/service/dataset_store.h new file mode 100644 index 00000000000..df2baf0ebda --- /dev/null +++ b/tensorflow/core/data/service/dataset_store.h @@ -0,0 +1,77 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_CORE_DATA_SERVICE_DATASET_STORE_H_ +#define TENSORFLOW_CORE_DATA_SERVICE_DATASET_STORE_H_ + +#include "absl/container/flat_hash_map.h" +#include "tensorflow/core/data/service/dispatcher_state.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/io/record_reader.h" +#include "tensorflow/core/lib/io/record_writer.h" +#include "tensorflow/core/platform/env.h" + +namespace tensorflow { +namespace data { + +// An interface for storing and getting dataset definitions. +class DatasetStore { + public: + virtual ~DatasetStore() = default; + + // Stores the given dataset under the given key. Returns ALREADY_EXISTS if the + // key already exists. + virtual Status Put(const std::string& key, const DatasetDef& dataset) = 0; + // Gets the dataset for the given key, storing the dataset in `dataset_def`. + virtual Status Get(const std::string& key, + std::shared_ptr& dataset_def) = 0; +}; + +// Dataset store which reads and writes datasets within a directory. +// The dataset with key `key` is stored at the path "datasets_dir/key". +class FileSystemDatasetStore : public DatasetStore { + public: + explicit FileSystemDatasetStore(const std::string& datasets_dir); + FileSystemDatasetStore(const FileSystemDatasetStore&) = delete; + FileSystemDatasetStore& operator=(const FileSystemDatasetStore&) = delete; + + Status Put(const std::string& key, const DatasetDef& dataset) override; + Status Get(const std::string& key, + std::shared_ptr& dataset_def) override; + + private: + const std::string datasets_dir_; +}; + +// DatasetStore which stores all datasets in memory. This is useful when the +// dispatcher doesn't have a work directory configured. +class MemoryDatasetStore : public DatasetStore { + public: + MemoryDatasetStore(); + MemoryDatasetStore(const MemoryDatasetStore&) = delete; + MemoryDatasetStore& operator=(const MemoryDatasetStore&) = delete; + + Status Put(const std::string& key, const DatasetDef& dataset) override; + Status Get(const std::string& key, + std::shared_ptr& dataset_def) override; + + private: + // Mapping from key to dataset definition. + absl::flat_hash_map> datasets_; +}; + +} // namespace data +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_DATA_SERVICE_DATASET_STORE_H_ diff --git a/tensorflow/core/data/service/dataset_store_test.cc b/tensorflow/core/data/service/dataset_store_test.cc new file mode 100644 index 00000000000..933ede679cb --- /dev/null +++ b/tensorflow/core/data/service/dataset_store_test.cc @@ -0,0 +1,115 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow/core/data/service/dataset_store.h" + +#include "absl/memory/memory.h" +#include "tensorflow/core/data/service/common.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/path.h" +#include "tensorflow/core/platform/test.h" + +namespace tensorflow { +namespace data { + +namespace { +const char kFileSystem[] = "file_system"; +const char kMemory[] = "memory"; + +std::string NewDatasetsDir() { + std::string dir = io::JoinPath(testing::TmpDir(), "datasets"); + if (Env::Default()->FileExists(dir).ok()) { + int64 undeleted_files; + int64 undeleted_dirs; + CHECK(Env::Default() + ->DeleteRecursively(dir, &undeleted_files, &undeleted_dirs) + .ok()); + } + CHECK(Env::Default()->RecursivelyCreateDir(dir).ok()); + return dir; +} + +std::unique_ptr MakeStore(const std::string& type) { + if (type == kFileSystem) { + return absl::make_unique(NewDatasetsDir()); + } else if (type == kMemory) { + return absl::make_unique(); + } else { + CHECK(false) << "unexpected type: " << type; + } +} + +DatasetDef DatasetDefWithVersion(int32 version) { + DatasetDef def; + def.mutable_graph()->set_version(version); + return def; +} + +} // namespace + +class DatasetStoreTest : public ::testing::Test, + public ::testing::WithParamInterface {}; + +TEST_P(DatasetStoreTest, StoreAndGet) { + std::unique_ptr store = MakeStore(GetParam()); + std::string key = "key"; + DatasetDef dataset_def = DatasetDefWithVersion(1); + TF_ASSERT_OK(store->Put(key, dataset_def)); + std::shared_ptr result; + TF_ASSERT_OK(store->Get(key, result)); + EXPECT_EQ(result->graph().version(), dataset_def.graph().version()); +} + +TEST_P(DatasetStoreTest, StoreAndGetMultiple) { + std::unique_ptr store = MakeStore(GetParam()); + int64 num_datasets = 10; + std::vector keys; + for (int i = 0; i < num_datasets; ++i) { + std::string key = absl::StrCat("key", i); + DatasetDef dataset_def = DatasetDefWithVersion(i); + TF_ASSERT_OK(store->Put(key, dataset_def)); + keys.push_back(key); + } + for (int i = 0; i < num_datasets; ++i) { + std::shared_ptr result; + TF_ASSERT_OK(store->Get(keys[i], result)); + EXPECT_EQ(result->graph().version(), i); + } +} + +TEST_P(DatasetStoreTest, StoreAlreadyExists) { + std::unique_ptr store = MakeStore(GetParam()); + int32 version = 1; + DatasetDef dataset_def = DatasetDefWithVersion(version); + std::string key = "key"; + TF_ASSERT_OK(store->Put(key, dataset_def)); + Status s = store->Put(key, dataset_def); + EXPECT_EQ(s.code(), error::ALREADY_EXISTS); + std::shared_ptr result; + TF_ASSERT_OK(store->Get(key, result)); + EXPECT_EQ(result->graph().version(), version); +} + +TEST_P(DatasetStoreTest, GetMissing) { + std::unique_ptr store = MakeStore(GetParam()); + std::shared_ptr result; + Status s = store->Get("missing", result); + EXPECT_EQ(s.code(), error::NOT_FOUND); +} + +INSTANTIATE_TEST_SUITE_P(DatasetStoreTests, DatasetStoreTest, + ::testing::Values(kFileSystem, kMemory)); +} // namespace data +} // namespace tensorflow diff --git a/tensorflow/core/data/service/dispatcher_impl.cc b/tensorflow/core/data/service/dispatcher_impl.cc index a30de89ccea..6d504ebca81 100644 --- a/tensorflow/core/data/service/dispatcher_impl.cc +++ b/tensorflow/core/data/service/dispatcher_impl.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/data/service/common.pb.h" #include "tensorflow/core/data/service/credentials_factory.h" #include "tensorflow/core/data/service/data_service.h" +#include "tensorflow/core/data/service/dataset_store.h" #include "tensorflow/core/data/service/dispatcher.pb.h" #include "tensorflow/core/data/service/grpc_util.h" #include "tensorflow/core/data/service/journal.h" @@ -45,6 +46,8 @@ namespace data { namespace { // The name of the journal directory inside the dispatcher's working directory. constexpr char kJournalDir[] = "tf_data_dispatcher_journal"; +// The name of the datasets directory inside the dispatcher's working directory. +constexpr char kDatasetsDir[] = "datasets"; using Dataset = DispatcherState::Dataset; using Worker = DispatcherState::Worker; @@ -56,6 +59,14 @@ std::string JournalDir(const std::string& work_dir) { return io::JoinPath(work_dir, kJournalDir); } +std::string DatasetsDir(const std::string& work_dir) { + return io::JoinPath(work_dir, kDatasetsDir); +} + +std::string DatasetKey(int64 id, uint64 fingerprint) { + return absl::StrCat("id_", id, "_fp_", fingerprint); +} + Status CreateWorkerStub(const std::string& address, const std::string& protocol, std::unique_ptr* stub) { ::grpc::ChannelArguments args; @@ -72,10 +83,20 @@ Status CreateWorkerStub(const std::string& address, const std::string& protocol, DataServiceDispatcherImpl::DataServiceDispatcherImpl( const experimental::DispatcherConfig& config) : config_(config) { + if (config_.work_dir().empty()) { + dataset_store_ = absl::make_unique(); + } else { + dataset_store_ = absl::make_unique( + DatasetsDir(config_.work_dir())); + } } Status DataServiceDispatcherImpl::Start() { mutex_lock l(mu_); + if (!config_.work_dir().empty()) { + TF_RETURN_IF_ERROR( + Env::Default()->RecursivelyCreateDir(DatasetsDir(config_.work_dir()))); + } if (!config_.fault_tolerant_mode()) { LOG(INFO) << "Running with fault_tolerant_mode=False. The dispatcher will " "not be able to recover its state on restart."; @@ -148,7 +169,10 @@ Status DataServiceDispatcherImpl::RegisterWorker( TaskDef* task_def = response->add_tasks(); std::shared_ptr dataset; TF_RETURN_IF_ERROR(state_.DatasetFromId(job->dataset_id, &dataset)); - *(task_def->mutable_dataset()) = dataset->dataset_def; + std::shared_ptr dataset_def; + TF_RETURN_IF_ERROR(dataset_store_->Get( + DatasetKey(dataset->dataset_id, dataset->fingerprint), dataset_def)); + *(task_def->mutable_dataset()) = *dataset_def; task_def->set_dataset_id(job->dataset_id); task_def->set_job_id(job->job_id); task_def->set_task_id(task->task_id); @@ -217,7 +241,8 @@ Status DataServiceDispatcherImpl::RegisterDataset(uint64 fingerprint, RegisterDatasetUpdate* register_dataset = update.mutable_register_dataset(); register_dataset->set_dataset_id(*dataset_id); register_dataset->set_fingerprint(fingerprint); - *register_dataset->mutable_dataset_def() = dataset; + TF_RETURN_IF_ERROR( + dataset_store_->Put(DatasetKey(*dataset_id, fingerprint), dataset)); return Apply(update); } @@ -406,7 +431,10 @@ Status DataServiceDispatcherImpl::AssignTask(std::shared_ptr task) mutex_lock l(mu_); std::shared_ptr dataset; TF_RETURN_IF_ERROR(state_.DatasetFromId(task->dataset_id, &dataset)); - *task_def->mutable_dataset() = dataset->dataset_def; + std::shared_ptr dataset_def; + TF_RETURN_IF_ERROR(dataset_store_->Get( + DatasetKey(dataset->dataset_id, dataset->fingerprint), dataset_def)); + *task_def->mutable_dataset() = *dataset_def; } task_def->set_task_id(task->task_id); ProcessTaskResponse resp; diff --git a/tensorflow/core/data/service/dispatcher_impl.h b/tensorflow/core/data/service/dispatcher_impl.h index f4cc6954fe8..2533e96d7ef 100644 --- a/tensorflow/core/data/service/dispatcher_impl.h +++ b/tensorflow/core/data/service/dispatcher_impl.h @@ -19,6 +19,7 @@ limitations under the License. #include "absl/container/flat_hash_map.h" #include "tensorflow/core/data/service/common.pb.h" #include "tensorflow/core/data/service/data_service.h" +#include "tensorflow/core/data/service/dataset_store.h" #include "tensorflow/core/data/service/dispatcher.pb.h" #include "tensorflow/core/data/service/dispatcher_state.h" #include "tensorflow/core/data/service/worker.grpc.pb.h" @@ -127,6 +128,8 @@ class DataServiceDispatcherImpl { // Cached worker stubs for communicating with workers. absl::flat_hash_map> worker_stubs_ TF_GUARDED_BY(mu_); + // Store of dataset definitions. + std::unique_ptr dataset_store_ TF_GUARDED_BY(mu_); absl::optional> journal_writer_ TF_GUARDED_BY(mu_); diff --git a/tensorflow/core/data/service/dispatcher_state.cc b/tensorflow/core/data/service/dispatcher_state.cc index aedfab7280b..19c1c1c9de5 100644 --- a/tensorflow/core/data/service/dispatcher_state.cc +++ b/tensorflow/core/data/service/dispatcher_state.cc @@ -53,8 +53,7 @@ void DispatcherState::RegisterDataset( const RegisterDatasetUpdate& register_dataset) { int64 id = register_dataset.dataset_id(); int64 fingerprint = register_dataset.fingerprint(); - auto dataset = std::make_shared(id, fingerprint, - register_dataset.dataset_def()); + auto dataset = std::make_shared(id, fingerprint); DCHECK(!datasets_by_id_.contains(id)); datasets_by_id_[id] = dataset; DCHECK(!datasets_by_fingerprint_.contains(fingerprint)); diff --git a/tensorflow/core/data/service/dispatcher_state.h b/tensorflow/core/data/service/dispatcher_state.h index 8db05064a40..70b91d634d8 100644 --- a/tensorflow/core/data/service/dispatcher_state.h +++ b/tensorflow/core/data/service/dispatcher_state.h @@ -60,15 +60,11 @@ class DispatcherState { // A dataset registered with the dispatcher. struct Dataset { - explicit Dataset(int64 dataset_id, int64 fingerprint, - const DatasetDef& dataset_def) - : dataset_id(dataset_id), - fingerprint(fingerprint), - dataset_def(dataset_def) {} + explicit Dataset(int64 dataset_id, int64 fingerprint) + : dataset_id(dataset_id), fingerprint(fingerprint) {} const int64 dataset_id; const int64 fingerprint; - const DatasetDef dataset_def; }; // A worker registered with the dispatcher. diff --git a/tensorflow/core/data/service/dispatcher_state_test.cc b/tensorflow/core/data/service/dispatcher_state_test.cc index 004890242c2..e0befb576a5 100644 --- a/tensorflow/core/data/service/dispatcher_state_test.cc +++ b/tensorflow/core/data/service/dispatcher_state_test.cc @@ -36,8 +36,7 @@ using Task = DispatcherState::Task; using ::testing::IsEmpty; using ::testing::SizeIs; -Status RegisterDatasetWithIdAndFingerprint(int64 id, uint64 fingerprint, - DispatcherState* state) { +Status RegisterDataset(int64 id, uint64 fingerprint, DispatcherState* state) { Update update; RegisterDatasetUpdate* register_dataset = update.mutable_register_dataset(); register_dataset->set_dataset_id(id); @@ -46,6 +45,10 @@ Status RegisterDatasetWithIdAndFingerprint(int64 id, uint64 fingerprint, return Status::OK(); } +Status RegisterDataset(int64 id, DispatcherState* state) { + return RegisterDataset(id, /*fingerprint=*/1, state); +} + Status RegisterWorker(std::string worker_address, DispatcherState* state) { Update update; update.mutable_register_worker()->set_worker_address(worker_address); @@ -103,7 +106,7 @@ TEST(DispatcherState, RegisterDataset) { int64 id = 10; uint64 fingerprint = 20; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(id, fingerprint, &state)); + TF_EXPECT_OK(RegisterDataset(id, fingerprint, &state)); EXPECT_EQ(state.NextAvailableDatasetId(), id + 1); { @@ -136,7 +139,7 @@ TEST(DispatcherState, NextAvailableDatasetId) { DispatcherState state; int64 id = state.NextAvailableDatasetId(); uint64 fingerprint = 20; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(id, fingerprint, &state)); + TF_EXPECT_OK(RegisterDataset(id, fingerprint, &state)); EXPECT_NE(state.NextAvailableDatasetId(), id); EXPECT_EQ(state.NextAvailableDatasetId(), state.NextAvailableDatasetId()); } @@ -188,7 +191,7 @@ TEST(DispatcherState, AnonymousJob) { int64 job_id = 3; int64 dataset_id = 10; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(RegisterDataset(dataset_id, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); std::shared_ptr job; TF_EXPECT_OK(state.JobFromId(job_id, &job)); @@ -205,7 +208,7 @@ TEST(DispatcherState, NamedJob) { int64 job_id = 3; int64 dataset_id = 10; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(RegisterDataset(dataset_id, &state)); NamedJobKey named_job_key("test", 1); TF_EXPECT_OK(CreateNamedJob(job_id, dataset_id, named_job_key, &state)); std::shared_ptr job; @@ -222,7 +225,7 @@ TEST(DispatcherState, CreateTask) { int64 task_id = 8; std::string worker_address = "test_worker_address"; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(RegisterDataset(dataset_id, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); TF_EXPECT_OK(CreateTask(task_id, job_id, dataset_id, worker_address, &state)); EXPECT_EQ(state.NextAvailableTaskId(), task_id + 1); @@ -253,7 +256,7 @@ TEST(DispatcherState, CreateTasksForSameJob) { int64 task_id_2 = 9; std::string worker_address = "test_worker_address"; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(RegisterDataset(dataset_id, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); TF_EXPECT_OK( CreateTask(task_id_1, job_id, dataset_id, worker_address, &state)); @@ -274,7 +277,7 @@ TEST(DispatcherState, CreateTasksForDifferentJobs) { int64 task_id_2 = 9; std::string worker_address = "test_worker_address"; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(RegisterDataset(dataset_id, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id_1, dataset_id, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id_2, dataset_id, &state)); TF_EXPECT_OK( @@ -300,7 +303,7 @@ TEST(DispatcherState, CreateTasksForSameWorker) { int64 task_id_2 = 9; std::string worker_address = "test_worker_address"; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(RegisterDataset(dataset_id, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); TF_EXPECT_OK( CreateTask(task_id_1, job_id, dataset_id, worker_address, &state)); @@ -321,7 +324,7 @@ TEST(DispatcherState, CreateTasksForDifferentWorkers) { std::string worker_address_1 = "test_worker_address_1"; std::string worker_address_2 = "test_worker_address_2"; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(RegisterDataset(dataset_id, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); TF_EXPECT_OK( CreateTask(task_id_1, job_id, dataset_id, worker_address_1, &state)); @@ -356,7 +359,7 @@ TEST(DispatcherState, FinishTask) { int64 task_id = 4; std::string worker_address = "test_worker_address"; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(RegisterDataset(dataset_id, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); TF_EXPECT_OK(CreateTask(task_id, job_id, dataset_id, worker_address, &state)); TF_EXPECT_OK(FinishTask(task_id, &state)); @@ -375,7 +378,7 @@ TEST(DispatcherState, FinishMultiTaskJob) { int64 task_id_2 = 5; std::string worker_address = "test_worker_address"; DispatcherState state; - TF_EXPECT_OK(RegisterDatasetWithIdAndFingerprint(dataset_id, 1, &state)); + TF_EXPECT_OK(RegisterDataset(dataset_id, &state)); TF_EXPECT_OK(CreateAnonymousJob(job_id, dataset_id, &state)); TF_EXPECT_OK( CreateTask(task_id_1, job_id, dataset_id, worker_address, &state)); diff --git a/tensorflow/core/data/service/journal.proto b/tensorflow/core/data/service/journal.proto index 725724a5cd5..5ad97ea6120 100644 --- a/tensorflow/core/data/service/journal.proto +++ b/tensorflow/core/data/service/journal.proto @@ -19,8 +19,7 @@ message Update { message RegisterDatasetUpdate { int64 dataset_id = 1; - DatasetDef dataset_def = 2; - uint64 fingerprint = 3; + uint64 fingerprint = 2; } message RegisterWorkerUpdate { From a3e153c8308266c5a8a8fba6d04bda9e6f9abc75 Mon Sep 17 00:00:00 2001 From: Mingming Liu Date: Wed, 12 Aug 2020 18:32:36 -0700 Subject: [PATCH 0951/1017] In shared batch scheduler, rename 'max_batch_size' to 'input_batch_size_limit'. PiperOrigin-RevId: 326357643 Change-Id: Iac19c5329175ae87ea1d4f35d930fa6a670e0f9e --- .../batching_util/basic_batch_scheduler.h | 3 +- .../batching_util/batch_resource_base.cc | 2 +- .../batching_util/shared_batch_scheduler.h | 47 ++++++++++--------- .../shared_batch_scheduler_test.cc | 18 +++---- 4 files changed, 36 insertions(+), 34 deletions(-) diff --git a/tensorflow/core/kernels/batching_util/basic_batch_scheduler.h b/tensorflow/core/kernels/batching_util/basic_batch_scheduler.h index bd44115db22..cdaf5331e9f 100644 --- a/tensorflow/core/kernels/batching_util/basic_batch_scheduler.h +++ b/tensorflow/core/kernels/batching_util/basic_batch_scheduler.h @@ -282,7 +282,8 @@ Status BasicBatchScheduler::Create( typename SharedBatchScheduler::QueueOptions shared_scheduler_queue_options; - shared_scheduler_queue_options.max_batch_size = options.max_batch_size; + shared_scheduler_queue_options.input_batch_size_limit = + options.max_batch_size; shared_scheduler_queue_options.batch_timeout_micros = options.batch_timeout_micros; shared_scheduler_queue_options.max_enqueued_batches = diff --git a/tensorflow/core/kernels/batching_util/batch_resource_base.cc b/tensorflow/core/kernels/batching_util/batch_resource_base.cc index 44e2879b9e4..98175b5b9d0 100644 --- a/tensorflow/core/kernels/batching_util/batch_resource_base.cc +++ b/tensorflow/core/kernels/batching_util/batch_resource_base.cc @@ -137,7 +137,7 @@ BatchResourceBase::GetBatcherQueueOptions( int32 max_enqueued_batches, const std::vector& allowed_batch_sizes, bool enable_large_batch_splitting) { BatcherT::QueueOptions batcher_queue_options; - batcher_queue_options.max_batch_size = max_batch_size; + batcher_queue_options.input_batch_size_limit = max_batch_size; batcher_queue_options.max_enqueued_batches = max_enqueued_batches; batcher_queue_options.batch_timeout_micros = batch_timeout_micros; // Support for splitting large batch is still in progress. diff --git a/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h b/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h index ce7823a7aef..c4cf111fe96 100644 --- a/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h +++ b/tensorflow/core/kernels/batching_util/shared_batch_scheduler.h @@ -136,17 +136,15 @@ class SharedBatchScheduler struct QueueOptions { // The size limit of an input batch to the queue. // - // If `enable_large_batch_splitting` is True, 'max_batch_size' should be - // greater or equal than `max_execution_batch_size`; otherwise - // `max_batch_size` should be equal to `max_execution_batch_size`. - // TODO(b/154140947): - // Rename it to 'input_batch_size_limit' here and in caller's code. - size_t max_batch_size = 1000; + // If `enable_large_batch_splitting` is True, 'input_batch_size_limit' + // should be greater or equal than `max_execution_batch_size`; otherwise + // `input_batch_size_limit` should be equal to `max_execution_batch_size`. + size_t input_batch_size_limit = 1000; // If a task has been enqueued for this amount of time (in microseconds), // and a thread is available, the scheduler will immediately form a batch // from enqueued tasks and assign the batch to the thread for processing, - // even if the batch's size is below 'max_batch_size'. + // even if the batch's size is below 'input_batch_size_limit'. // // This parameter offers a way to bound queue latency, so that a task isn't // stuck in the queue indefinitely waiting for enough tasks to arrive to @@ -173,7 +171,7 @@ class SharedBatchScheduler // `input_task`: a unit of task to be split. // `first_output_task_size`: task size of first output. - // `max_batch_size`: Maximum size of each batch. + // `max_execution_batch_size`: Maximum size of each batch. // `output_tasks`: A list of output tasks after split. // // REQUIRED: @@ -184,7 +182,7 @@ class SharedBatchScheduler // Instantiations of `TaskType` may vary, so it's up to caller to define // how (e.g., which members to access) to split input tasks. std::function* input_task, - int first_output_task_size, int max_batch_size, + int first_output_task_size, int input_batch_size_limit, std::vector>* output_tasks)> split_input_task_func; @@ -269,7 +267,7 @@ class Queue { using SchedulableBatchCallback = std::function; using SplitInputTaskIntoSubtasksCallback = std::function* input_task, int open_batch_remaining_slot, - int max_batch_size, + int max_execution_batch_size, std::vector>* output_tasks)>; Queue(const typename SharedBatchScheduler::QueueOptions& options, Env* env, ProcessBatchCallback process_batch_callback, @@ -297,7 +295,7 @@ class Queue { size_t SchedulingCapacity() const; // Returns the maximum allowed size of tasks submitted to the queue. - size_t max_task_size() const { return options_.max_batch_size; } + size_t max_task_size() const { return options_.input_batch_size_limit; } // Returns the maximum allowed size of tasks to be enqueued. // Returned value would be less than or equal to the maximum allowed input @@ -306,7 +304,7 @@ class Queue { if (options_.enable_large_batch_splitting) { return options_.max_execution_batch_size; } else { - return options_.max_batch_size; + return options_.input_batch_size_limit; } } @@ -459,9 +457,10 @@ Status SharedBatchScheduler::AddQueue( std::function>)> process_batch_callback, std::unique_ptr>* queue) { - if (options.max_batch_size == 0) { - return errors::InvalidArgument("max_batch_size must be positive; was ", - options.max_batch_size); + if (options.input_batch_size_limit == 0) { + return errors::InvalidArgument( + "input_batch_size_limit must be positive; was ", + options.input_batch_size_limit); } if (options.batch_timeout_micros < 0) { return errors::InvalidArgument( @@ -483,11 +482,12 @@ Status SharedBatchScheduler::AddQueue( } if (options.enable_large_batch_splitting && - (options.max_batch_size < options.max_execution_batch_size)) { + (options.input_batch_size_limit < options.max_execution_batch_size)) { return errors::InvalidArgument( - "When enable_large_batch_splitting is true, max_batch_size must be " + "When enable_large_batch_splitting is true, input_batch_size_limit " + "must be " "greater than or equal to max_execution_batch_size.", - options.enable_large_batch_splitting, options.max_batch_size, + options.enable_large_batch_splitting, options.input_batch_size_limit, options.max_execution_batch_size); } @@ -616,10 +616,10 @@ Status Queue::Schedule(std::unique_ptr* task) { template Status Queue::ScheduleWithoutSplit(std::unique_ptr* task) { - if ((*task)->size() > options_.max_batch_size) { + if ((*task)->size() > options_.input_batch_size_limit) { return errors::InvalidArgument("Task size ", (*task)->size(), " is larger than maximum input batch size ", - options_.max_batch_size); + options_.input_batch_size_limit); } bool notify_of_schedulable_batch = false; @@ -628,7 +628,8 @@ Status Queue::ScheduleWithoutSplit(std::unique_ptr* task) { DCHECK(!closed_); - if (batches_.back()->size() + (*task)->size() > options_.max_batch_size) { + if (batches_.back()->size() + (*task)->size() > + options_.input_batch_size_limit) { if (batches_.size() >= options_.max_enqueued_batches) { return errors::Unavailable( "The batch scheduling queue to which this task was submitted is " @@ -669,10 +670,10 @@ Status Queue::ScheduleWithSplit(std::unique_ptr* task) { profiler::TraceMe trace_me([task] { return strings::StrCat("ScheduleWithSplit:", (*task)->size()); }); - if ((*task)->size() > options_.max_batch_size) { + if ((*task)->size() > options_.input_batch_size_limit) { return errors::InvalidArgument("Task size ", (*task)->size(), " is larger than maximum input batch size ", - options_.max_batch_size); + options_.input_batch_size_limit); } // The max size to be enqueued. diff --git a/tensorflow/core/kernels/batching_util/shared_batch_scheduler_test.cc b/tensorflow/core/kernels/batching_util/shared_batch_scheduler_test.cc index a1958777a49..10f34cf829b 100644 --- a/tensorflow/core/kernels/batching_util/shared_batch_scheduler_test.cc +++ b/tensorflow/core/kernels/batching_util/shared_batch_scheduler_test.cc @@ -97,7 +97,7 @@ TEST(SharedBatchSchedulerTest, Basic) { // Create two queues. SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; + queue_options.input_batch_size_limit = 10; queue_options.batch_timeout_micros = 10 * 1000 * 1000; // 10 seconds queue_options.max_enqueued_batches = 2; std::unique_ptr> queue_0; @@ -155,7 +155,7 @@ TEST(SharedBatchSchedulerTest, ObeyBatchSizeConstraint) { std::shared_ptr> scheduler; TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; + queue_options.input_batch_size_limit = 10; queue_options.batch_timeout_micros = 10 * 1000 * 1000; // 10 seconds queue_options.max_enqueued_batches = 2; std::unique_ptr> queue; @@ -217,7 +217,7 @@ TEST(SharedBatchSchedulerTest, ObeysTimeout) { std::shared_ptr> scheduler; TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 4; + queue_options.input_batch_size_limit = 4; queue_options.batch_timeout_micros = 10; queue_options.max_enqueued_batches = 2; std::unique_ptr> queue; @@ -273,7 +273,7 @@ TEST(SharedBatchSchedulerTest, ObeysTimeoutWithRealClock) { std::shared_ptr> scheduler; TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; + queue_options.input_batch_size_limit = 10; queue_options.batch_timeout_micros = 100 * 1000; // 100 milliseconds queue_options.max_enqueued_batches = 2; std::unique_ptr> queue; @@ -318,7 +318,7 @@ TEST(SharedBatchSchedulerTest, TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); SharedBatchScheduler::QueueOptions queue_options; // Set a large batch size, so that we don't hit the batch size limit. - queue_options.max_batch_size = 100; + queue_options.input_batch_size_limit = 100; // Process a batch as soon as a thread is available. queue_options.batch_timeout_micros = 0; queue_options.max_enqueued_batches = 2; @@ -371,7 +371,7 @@ TEST(SharedBatchSchedulerTest, Fairness) { std::shared_ptr> scheduler; TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; + queue_options.input_batch_size_limit = 10; queue_options.batch_timeout_micros = 1; queue_options.max_enqueued_batches = 100 /* give plenty of room */; std::vector>> queues(2); @@ -423,7 +423,7 @@ TEST(SharedBatchSchedulerTest, ConstMethods) { std::shared_ptr> scheduler; TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 2; + queue_options.input_batch_size_limit = 2; queue_options.batch_timeout_micros = 0; queue_options.max_enqueued_batches = max_enqueued_batches; std::unique_ptr> queue; @@ -494,7 +494,7 @@ TEST(SharedBatchSchedulerTest, OneFullQueueDoesntBlockOtherQueues) { std::shared_ptr> scheduler; TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; + queue_options.input_batch_size_limit = 10; queue_options.batch_timeout_micros = 0; queue_options.max_enqueued_batches = 2; std::unique_ptr> queue_0; @@ -550,7 +550,7 @@ TEST(SharedBatchSchedulerTest, QueueDestructorBlocksUntilAllTasksProcessed) { std::shared_ptr> scheduler; TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; + queue_options.input_batch_size_limit = 10; queue_options.batch_timeout_micros = 0; queue_options.max_enqueued_batches = 2; std::unique_ptr> queue; From 4de1a5f5c2e00d4cb43d66daed0447c9e088bfcc Mon Sep 17 00:00:00 2001 From: Rahul Joshi Date: Wed, 12 Aug 2020 18:36:34 -0700 Subject: [PATCH 0952/1017] [MLIR] Add definition of ResourceAliasAnalysisInfo::kUnknownResourceId - Add definition to fix link failure. PiperOrigin-RevId: 326358168 Change-Id: Idc06f0eec1df38f0f7d42fc201953df2bdcdeb8e --- .../mlir/tensorflow/analysis/resource_alias_analysis.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc index 256217b6542..7ad2705263b 100644 --- a/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc +++ b/tensorflow/compiler/mlir/tensorflow/analysis/resource_alias_analysis.cc @@ -224,6 +224,8 @@ int64_t GetOrCreateIdForVarHandle(VarHandleOp handle, int64_t* next_id, // ResourceAliasAnalysisInfo //===----------------------------------------------------------------------===// +constexpr int64_t ResourceAliasAnalysisInfo::kUnknownResourceId; + // Constructs the analysis info by analyzing the given function. ResourceAliasAnalysisInfo::ResourceAliasAnalysisInfo( FuncOp func_op, const BacktrackAnalysis& backtrack_analysis) { From 2dc8f029bfcc51ccd5149570d64a41296e0ddf70 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Wed, 12 Aug 2020 18:39:47 -0700 Subject: [PATCH 0953/1017] [tf.data] Fix kokoro breakage PiperOrigin-RevId: 326358636 Change-Id: Iec0bfaf6c5569a885426f665283840bc3528220e --- .../core/kernels/data/experimental/auto_shard_dataset_op.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.cc b/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.cc index 343d6f36616..9637aae5d7c 100644 --- a/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.cc +++ b/tensorflow/core/kernels/data/experimental/auto_shard_dataset_op.cc @@ -25,6 +25,7 @@ namespace experimental { /* static */ constexpr const char* const AutoShardDatasetOp::kDatasetType; /* static */ constexpr const char* const AutoShardDatasetOp::kInputDataset; /* static */ constexpr const char* const AutoShardDatasetOp::kNumWorkers; +/* static */ constexpr const char* const AutoShardDatasetOp::kNumReplicas; /* static */ constexpr const char* const AutoShardDatasetOp::kIndex; /* static */ constexpr const char* const AutoShardDatasetOp::kOutputTypes; /* static */ constexpr const char* const AutoShardDatasetOp::kOutputShapes; From f32c80b3ed0d64eb0363f4196171467de79390d1 Mon Sep 17 00:00:00 2001 From: Zhenyu Tan Date: Wed, 12 Aug 2020 18:44:28 -0700 Subject: [PATCH 0954/1017] Add MultiHeadAttention Layer for Keras. PiperOrigin-RevId: 326359296 Change-Id: Iacdc310f66aa1848b068fa3f0fc8784ea7b80ef5 --- tensorflow/python/keras/layers/BUILD | 29 ++ tensorflow/python/keras/layers/__init__.py | 3 + .../keras/layers/advanced_activations.py | 55 ++- .../keras/layers/multi_head_attention.py | 460 ++++++++++++++++++ .../keras/layers/multi_head_attention_test.py | 230 +++++++++ .../python/keras/layers/serialization.py | 4 +- ...w.keras.layers.-multi-head-attention.pbtxt | 222 +++++++++ .../v1/tensorflow.keras.layers.-softmax.pbtxt | 2 +- .../golden/v1/tensorflow.keras.layers.pbtxt | 4 + ...w.keras.layers.-multi-head-attention.pbtxt | 222 +++++++++ .../v2/tensorflow.keras.layers.-softmax.pbtxt | 2 +- .../golden/v2/tensorflow.keras.layers.pbtxt | 4 + 12 files changed, 1232 insertions(+), 5 deletions(-) create mode 100644 tensorflow/python/keras/layers/multi_head_attention.py create mode 100644 tensorflow/python/keras/layers/multi_head_attention_test.py create mode 100644 tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multi-head-attention.pbtxt create mode 100644 tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multi-head-attention.pbtxt diff --git a/tensorflow/python/keras/layers/BUILD b/tensorflow/python/keras/layers/BUILD index fe46f580162..e3497c59061 100644 --- a/tensorflow/python/keras/layers/BUILD +++ b/tensorflow/python/keras/layers/BUILD @@ -39,6 +39,7 @@ py_library( ":kernelized", ":local", ":merge", + ":multi_head_attention", ":noise", ":normalization", ":normalization_v2", @@ -207,6 +208,22 @@ py_library( ], ) +py_library( + name = "multi_head_attention", + srcs = ["multi_head_attention.py"], + srcs_version = "PY2AND3", + deps = [ + "//tensorflow/python:special_math_ops", + "//tensorflow/python:tensor_shape", + "//tensorflow/python:util", + "//tensorflow/python/keras:activations", + "//tensorflow/python/keras:base_layer", + "//tensorflow/python/keras:constraints", + "//tensorflow/python/keras:initializers", + "//tensorflow/python/keras:regularizers", + ], +) + py_library( name = "embeddings", srcs = ["embeddings.py"], @@ -590,6 +607,18 @@ tf_py_test( ], ) +tf_py_test( + name = "multi_head_attention_test", + srcs = ["multi_head_attention_test.py"], + python_version = "PY3", + deps = [ + ":multi_head_attention", + "//tensorflow/python:client_testlib", + "//tensorflow/python/keras", + "@absl_py//absl/testing:parameterized", + ], +) + cuda_py_test( name = "embeddings_test", size = "medium", diff --git a/tensorflow/python/keras/layers/__init__.py b/tensorflow/python/keras/layers/__init__.py index 8ce1c7d8224..b07773ae03a 100644 --- a/tensorflow/python/keras/layers/__init__.py +++ b/tensorflow/python/keras/layers/__init__.py @@ -143,6 +143,9 @@ from tensorflow.python.keras.layers.embeddings import Embedding # Einsum-based dense layer/ from tensorflow.python.keras.layers.einsum_dense import EinsumDense +# Multi-head Attention layer. +from tensorflow.python.keras.layers.multi_head_attention import MultiHeadAttention + # Locally-connected layers. from tensorflow.python.keras.layers.local import LocallyConnected1D from tensorflow.python.keras.layers.local import LocallyConnected2D diff --git a/tensorflow/python/keras/layers/advanced_activations.py b/tensorflow/python/keras/layers/advanced_activations.py index 7cb40c172b7..e4323b45dc4 100644 --- a/tensorflow/python/keras/layers/advanced_activations.py +++ b/tensorflow/python/keras/layers/advanced_activations.py @@ -18,6 +18,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +from tensorflow.python.framework import dtypes from tensorflow.python.keras import backend as K from tensorflow.python.keras import constraints from tensorflow.python.keras import initializers @@ -259,10 +260,37 @@ class ThresholdedReLU(Layer): return input_shape +def _large_compatible_negative(tensor_type): + """Large negative number as Tensor. + + This function is necessary because the standard value for epsilon + in this module (-1e9) cannot be represented using tf.float16 + + Args: + tensor_type: a dtype to determine the type. + + Returns: + a large negative number. + """ + if tensor_type == dtypes.float16: + return dtypes.float16.min + return -1e9 + + @keras_export('keras.layers.Softmax') class Softmax(Layer): """Softmax activation function. + Example without mask: + + >>> inp = np.asarray([1., 2., 1.]) + >>> layer = tf.keras.layers.Softmax() + >>> layer(inp).numpy() + array([0.21194157, 0.5761169 , 0.21194157], dtype=float32) + >>> mask = np.asarray([True, False, True], dtype=bool) + >>> layer(inp, mask).numpy() + array([0.5, 0. , 0.5], dtype=float32) + Input shape: Arbitrary. Use the keyword argument `input_shape` (tuple of integers, does not include the samples axis) @@ -272,7 +300,14 @@ class Softmax(Layer): Same shape as the input. Arguments: - axis: Integer, axis along which the softmax normalization is applied. + axis: Integer, or list of Integers, axis along which the softmax + normalization is applied. + Call arguments: + inputs: The inputs, or logits to the softmax layer. + mask: A boolean mask of the same shape as `inputs`. Defaults to `None`. + + Returns: + softmaxed output with the same shape as `inputs`. """ def __init__(self, axis=-1, **kwargs): @@ -280,7 +315,23 @@ class Softmax(Layer): self.supports_masking = True self.axis = axis - def call(self, inputs): + def call(self, inputs, mask=None): + if mask is not None: + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -1e.9 for masked positions. + adder = (1.0 - math_ops.cast(mask, inputs.dtype)) * ( + _large_compatible_negative(inputs.dtype)) + + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + inputs += adder + if isinstance(self.axis, (tuple, list)): + if len(self.axis) > 1: + return math_ops.exp(inputs - math_ops.reduce_logsumexp( + inputs, axis=self.axis, keepdims=True)) + else: + return K.softmax(inputs, axis=self.axis[0]) return K.softmax(inputs, axis=self.axis) def get_config(self): diff --git a/tensorflow/python/keras/layers/multi_head_attention.py b/tensorflow/python/keras/layers/multi_head_attention.py new file mode 100644 index 00000000000..210d6133d58 --- /dev/null +++ b/tensorflow/python/keras/layers/multi_head_attention.py @@ -0,0 +1,460 @@ +# Lint as: python3 +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Keras-based attention layer.""" +# pylint: disable=g-classes-have-attributes +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import math +import string + +import numpy as np + +from tensorflow.python.framework import tensor_shape +from tensorflow.python.keras import constraints +from tensorflow.python.keras import initializers +from tensorflow.python.keras import regularizers +from tensorflow.python.keras.engine.base_layer import Layer +from tensorflow.python.keras.layers import advanced_activations +from tensorflow.python.keras.layers import core +from tensorflow.python.keras.layers import einsum_dense +from tensorflow.python.keras.utils import tf_utils +from tensorflow.python.ops import array_ops +from tensorflow.python.ops import math_ops +from tensorflow.python.ops import special_math_ops +from tensorflow.python.util.tf_export import keras_export + + +_CHR_IDX = string.ascii_lowercase + + +def _build_attention_equation(rank, attn_axes): + """Builds einsum equations for the attention computation. + + Query, key, value inputs after projection are expected to have the shape as: + (bs, , , num_heads, channels). + bs and are treated as . + The attention operations can be generalized: + (1) Query-key dot product: + (, , num_heads, channels), (, + , num_heads, channels) -> (, + num_heads, , ) + (2) Combination: + (, num_heads, , ), + (, , num_heads, channels) -> (, + , num_heads, channels) + + Args: + rank: the rank of query, key, value tensors. + attn_axes: a list/tuple of axes, [1, rank), that will do attention. + + Returns: + Einsum equations. + """ + target_notation = _CHR_IDX[:rank] + # `batch_dims` includes the head dim. + batch_dims = tuple(np.delete(range(rank), attn_axes + (rank - 1,))) + letter_offset = rank + source_notation = "" + for i in range(rank): + if i in batch_dims or i == rank - 1: + source_notation += target_notation[i] + else: + source_notation += _CHR_IDX[letter_offset] + letter_offset += 1 + + product_notation = "".join([target_notation[i] for i in batch_dims] + + [target_notation[i] for i in attn_axes] + + [source_notation[i] for i in attn_axes]) + dot_product_equation = "%s,%s->%s" % (source_notation, target_notation, + product_notation) + attn_scores_rank = len(product_notation) + combine_equation = "%s,%s->%s" % (product_notation, source_notation, + target_notation) + return dot_product_equation, combine_equation, attn_scores_rank + + +def _build_proj_equation(free_dims, bound_dims, output_dims): + """Builds an einsum equation for projections inside multi-head attention.""" + input_str = "" + kernel_str = "" + output_str = "" + bias_axes = "" + letter_offset = 0 + for i in range(free_dims): + char = _CHR_IDX[i + letter_offset] + input_str += char + output_str += char + + letter_offset += free_dims + for i in range(bound_dims): + char = _CHR_IDX[i + letter_offset] + input_str += char + kernel_str += char + + letter_offset += bound_dims + for i in range(output_dims): + char = _CHR_IDX[i + letter_offset] + kernel_str += char + output_str += char + bias_axes += char + equation = "%s,%s->%s" % (input_str, kernel_str, output_str) + + return equation, bias_axes, len(output_str) + + +def _get_output_shape(output_rank, known_last_dims): + return [None] * (output_rank - len(known_last_dims)) + list(known_last_dims) + + +@keras_export("keras.layers.MultiHeadAttention") +class MultiHeadAttention(Layer): + """MultiHeadAttention layer. + + This is an implementation of multi-headed attention based on "Attention + is all you Need". If `query`, `key,` `value` are the same, then + this is self-attention. Each timestep in `query` attends to the + corresponding sequence in `key`, and returns a fixed-width vector. + + This layer first projects `query`, `key` and `value`. These are + (effectively) a list of tensors of length `num_attention_heads`, where the + corresponding shapes are [batch_size, , key_dim], + [batch_size, , key_dim], + [batch_size, , value_dim]. + + Then, the query and key tensors are dot-producted and scaled. These are + softmaxed to obtain attention probabilities. The value tensors are then + interpolated by these probabilities, then concatenated back to a single + tensor. + + Finally, the result tensor with the last dimension as value_dim can take an + linear projection and return. + + Examples: + + Performs 1D cross-attention over two sequence inputs with an attention mask. + Returns the additional attention weights over heads. + + >>> layer = MultiHeadAttention(num_heads=2, key_dim=2) + >>> target = tf.keras.Input(shape=[8, 16]) + >>> source = tf.keras.Input(shape=[4, 16]) + >>> output_tensor, weights = layer(target, source, + ... return_attention_scores=True) + >>> print(output_tensor.shape) + (None, 8, 16) + >>> print(weights.shape) + (None, 2, 8, 4) + + Performs 2D self-attention over a 5D input tensor on axes 2 and 3. + + >>> layer = MultiHeadAttention(num_heads=2, key_dim=2, attention_axes=(2, 3)) + >>> input_tensor = tf.keras.Input(shape=[5, 3, 4, 16]) + >>> output_tensor = layer(input_tensor, input_tensor) + >>> print(output_tensor.shape) + (None, 5, 3, 4, 16) + + Arguments: + num_heads: Number of attention heads. + key_dim: Size of each attention head for query and key. + value_dim: Size of each attention head for value. + dropout: Dropout probability. + use_bias: Boolean, whether the dense layers use bias vectors/matrices. + output_shape: The expected shape of an output tensor, besides the batch and + sequence dims. If not specified, projects back to the key feature dim. + attention_axes: axes over which the attention is applied. `None` means + attention over all axes, but batch, heads, and features. + kernel_initializer: Initializer for dense layer kernels. + bias_initializer: Initializer for dense layer biases. + kernel_regularizer: Regularizer for dense layer kernels. + bias_regularizer: Regularizer for dense layer biases. + activity_regularizer: Regularizer for dense layer activity. + kernel_constraint: Constraint for dense layer kernels. + bias_constraint: Constraint for dense layer kernels. + + Call arguments: + query: Query `Tensor` of shape `[B, T, dim]`. + value: Value `Tensor` of shape `[B, S, dim]`. + key: Optional key `Tensor` of shape `[B, S, dim]`. If not given, will use + `value` for both `key` and `value`, which is the most common case. + attention_mask: a boolean mask of shape `[B, T, S]`, that prevents attention + to certain positions. + return_attention_scores: A boolean to indicate whether the output should + be attention output if True, or (attention_output, attention_scores) if + False. Defaults to False. + + Returns: + attention_output: The result of the computation, of shape [B, T, E], + where `T` is for target sequence shapes and `E` is the query input last + dimension if `output_shape` is `None`. Otherwise, the multi-head outputs + are project to the shape specified by `output_shape`. + attention_scores: [Optional] multi-head attention coeffients over + attention axes. + """ + + def __init__(self, + num_heads, + key_dim, + value_dim=None, + dropout=0.0, + use_bias=True, + output_shape=None, + attention_axes=None, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + **kwargs): + super(MultiHeadAttention, self).__init__(**kwargs) + self._num_heads = num_heads + self._key_dim = key_dim + self._value_dim = value_dim if value_dim else key_dim + self._dropout = dropout + self._use_bias = use_bias + self._output_shape = output_shape + self._kernel_initializer = initializers.get(kernel_initializer) + self._bias_initializer = initializers.get(bias_initializer) + self._kernel_regularizer = regularizers.get(kernel_regularizer) + self._bias_regularizer = regularizers.get(bias_regularizer) + self._kernel_constraint = constraints.get(kernel_constraint) + self._bias_constraint = constraints.get(bias_constraint) + if attention_axes is not None and not isinstance(attention_axes, + collections.abc.Sized): + self._attention_axes = (attention_axes,) + else: + self._attention_axes = attention_axes + self._built_from_signature = False + + def get_config(self): + config = { + "num_heads": + self._num_heads, + "key_dim": + self._key_dim, + "value_dim": + self._value_dim, + "dropout": + self._dropout, + "use_bias": + self._use_bias, + "output_shape": + self._output_shape, + "attention_axes": + self._attention_axes, + "kernel_initializer": + initializers.serialize(self._kernel_initializer), + "bias_initializer": + initializers.serialize(self._bias_initializer), + "kernel_regularizer": + regularizers.serialize(self._kernel_regularizer), + "bias_regularizer": + regularizers.serialize(self._bias_regularizer), + "activity_regularizer": + regularizers.serialize(self._activity_regularizer), + "kernel_constraint": + constraints.serialize(self._kernel_constraint), + "bias_constraint": + constraints.serialize(self._bias_constraint) + } + base_config = super(MultiHeadAttention, self).get_config() + return dict(list(base_config.items()) + list(config.items())) + + def _build_from_signature(self, query, value, key=None): + """Builds layers and variables. + + Once the method is called, self._built_from_signature will be set to True. + + Args: + query: query tensor or TensorShape. + value: value tensor or TensorShape. + key: key tensor or TensorShape. + """ + self._built_from_signature = True + if hasattr(query, "shape"): + query_shape = tensor_shape.TensorShape(query.shape) + else: + query_shape = query + if hasattr(value, "shape"): + value_shape = tensor_shape.TensorShape(value.shape) + else: + value_shape = value + if key is None: + key_shape = value_shape + elif hasattr(key, "shape"): + key_shape = tensor_shape.TensorShape(key.shape) + else: + key_shape = key + + common_kwargs = dict( + kernel_initializer=self._kernel_initializer, + bias_initializer=self._bias_initializer, + kernel_regularizer=self._kernel_regularizer, + bias_regularizer=self._bias_regularizer, + activity_regularizer=self._activity_regularizer, + kernel_constraint=self._kernel_constraint, + bias_constraint=self._bias_constraint) + # Any setup work performed only once should happen in an `init_scope` + # to avoid creating symbolic Tensors that will later pollute any eager + # operations. + with tf_utils.maybe_init_scope(self): + free_dims = query_shape.rank - 1 + einsum_equation, bias_axes, output_rank = _build_proj_equation( + free_dims, bound_dims=1, output_dims=2) + self._query_dense = einsum_dense.EinsumDense( + einsum_equation, + output_shape=_get_output_shape(output_rank - 1, + [self._num_heads, self._key_dim]), + bias_axes=bias_axes if self._use_bias else None, + name="query", + **common_kwargs) + einsum_equation, bias_axes, output_rank = _build_proj_equation( + key_shape.rank - 1, bound_dims=1, output_dims=2) + self._key_dense = einsum_dense.EinsumDense( + einsum_equation, + output_shape=_get_output_shape(output_rank - 1, + [self._num_heads, self._key_dim]), + bias_axes=bias_axes if self._use_bias else None, + name="key", + **common_kwargs) + einsum_equation, bias_axes, output_rank = _build_proj_equation( + value_shape.rank - 1, bound_dims=1, output_dims=2) + self._value_dense = einsum_dense.EinsumDense( + einsum_equation, + output_shape=_get_output_shape(output_rank - 1, + [self._num_heads, self._value_dim]), + bias_axes=bias_axes if self._use_bias else None, + name="value", + **common_kwargs) + + # Builds the attention computations for multi-head dot product attention. + # These computations could be wrapped into the keras attention layer once + # it support mult-head einsum computations. + self._build_attention(output_rank) + if self._output_shape: + if not isinstance(self._output_shape, collections.abc.Sized): + output_shape = [self._output_shape] + else: + output_shape = self._output_shape + else: + output_shape = [query_shape[-1]] + einsum_equation, bias_axes, output_rank = _build_proj_equation( + free_dims, bound_dims=2, output_dims=len(output_shape)) + self._output_dense = einsum_dense.EinsumDense( + einsum_equation, + output_shape=_get_output_shape(output_rank - 1, output_shape), + bias_axes=bias_axes if self._use_bias else None, + name="attention_output", + **common_kwargs) + + def _build_attention(self, rank): + """Builds multi-head dot-product attention computations. + + This function builds attributes necessary for `_compute_attention` to + costomize attention computation to replace the default dot-product + attention. + + Args: + rank: the rank of query, key, value tensors. + """ + if self._attention_axes is None: + self._attention_axes = tuple(range(1, rank - 2)) + else: + self._attention_axes = tuple(self._attention_axes) + self._dot_product_equation, self._combine_equation, attn_scores_rank = ( + _build_attention_equation(rank, attn_axes=self._attention_axes)) + norm_axes = tuple( + range(attn_scores_rank - len(self._attention_axes), attn_scores_rank)) + self._masked_softmax = advanced_activations.Softmax(axis=norm_axes) + self._dropout_layer = core.Dropout(rate=self._dropout) + + def _compute_attention(self, query, key, value, attention_mask=None): + """Applies Dot-product attention with query, key, value tensors. + + This function defines the computation inside `call` with projected + multi-head Q, K, V inputs. Users can override this function for customized + attention implementation. + + Args: + query: Projected query `Tensor` of shape `[B, T, N, key_dim]`. + key: Projected key `Tensor` of shape `[B, T, N, key_dim]`. + value: Projected value `Tensor` of shape `[B, T, N, value_dim]`. + attention_mask: a boolean mask of shape `[B, T, S]`, that prevents + attention to certain positions. + + Returns: + attention_output: Multi-headed outputs of attention computation. + attention_scores: Multi-headed attention weights. + """ + # Note: Applying scalar multiply at the smaller end of einsum improves + # XLA performance, but may introduce slight numeric differences in + # the Transformer attention head. + query = math_ops.multiply(query, 1.0 / math.sqrt(float(self._key_dim))) + + # Take the dot product between "query" and "key" to get the raw + # attention scores. + attention_scores = special_math_ops.einsum(self._dot_product_equation, key, + query) + + # Normalize the attention scores to probabilities. + # `attention_scores` = [B, N, T, S] + if attention_mask is not None: + # The expand dim happens starting from the `num_heads` dimension, + # (, num_heads, ) + mask_expansion_axes = [-len(self._attention_axes) * 2 - 1] + for _ in range(len(attention_scores.shape) - len(attention_mask.shape)): + attention_mask = array_ops.expand_dims( + attention_mask, axis=mask_expansion_axes) + attention_scores = self._masked_softmax(attention_scores, attention_mask) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_scores_dropout = self._dropout_layer(attention_scores) + + # `context_layer` = [B, T, N, H] + attention_output = special_math_ops.einsum(self._combine_equation, + attention_scores_dropout, value) + return attention_output, attention_scores + + def call(self, query, value, key=None, attention_mask=None, + return_attention_scores=False): + if not self._built_from_signature: + self._build_from_signature(query=query, value=value, key=key) + if key is None: + key = value + + # N = `num_attention_heads` + # H = `size_per_head` + # `query` = [B, T, N ,H] + query = self._query_dense(query) + + # `key` = [B, S, N, H] + key = self._key_dense(key) + + # `value` = [B, S, N, H] + value = self._value_dense(value) + + attention_output, attention_scores = self._compute_attention( + query, key, value, attention_mask) + attention_output = self._output_dense(attention_output) + + if return_attention_scores: + return attention_output, attention_scores + return attention_output + diff --git a/tensorflow/python/keras/layers/multi_head_attention_test.py b/tensorflow/python/keras/layers/multi_head_attention_test.py new file mode 100644 index 00000000000..7702a2898c4 --- /dev/null +++ b/tensorflow/python/keras/layers/multi_head_attention_test.py @@ -0,0 +1,230 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for the attention layer.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized + +import numpy as np + +from tensorflow.python import keras +from tensorflow.python.keras import keras_parameterized +from tensorflow.python.keras.layers import multi_head_attention +from tensorflow.python.platform import test + + +# This decorator runs the test in V1, V2-Eager, and V2-Functional mode. It +# guarantees forward compatibility of this code for the V2 switchover. +@keras_parameterized.run_all_keras_modes +class MultiHeadAttentionTest(keras_parameterized.TestCase): + + @parameterized.named_parameters( + ("key_value_same_proj", None, None, [40, 80]), + ("key_value_different_proj", 32, 60, [40, 60]), + ) + def test_non_masked_attention(self, value_dim, output_shape, output_dims): + """Test that the attention layer can be created without a mask tensor.""" + test_layer = multi_head_attention.MultiHeadAttention( + num_heads=12, + key_dim=64, + value_dim=value_dim, + output_shape=output_shape) + # Create a 3-dimensional input (the first dimension is implicit). + query = keras.Input(shape=(40, 80)) + value = keras.Input(shape=(20, 80)) + output = test_layer(query=query, value=value) + self.assertEqual(output.shape.as_list(), [None] + output_dims) + + def test_non_masked_self_attention(self): + """Test with one input (self-attenntion) and no mask tensor.""" + test_layer = multi_head_attention.MultiHeadAttention( + num_heads=12, key_dim=64) + # Create a 3-dimensional input (the first dimension is implicit). + query = keras.Input(shape=(40, 80)) + output = test_layer(query, query) + self.assertEqual(output.shape.as_list(), [None, 40, 80]) + + def test_attention_scores(self): + """Test attention outputs with coefficients.""" + test_layer = multi_head_attention.MultiHeadAttention( + num_heads=12, key_dim=64) + # Create a 3-dimensional input (the first dimension is implicit). + query = keras.Input(shape=(40, 80)) + output, coef = test_layer(query, query, return_attention_scores=True) + self.assertEqual(output.shape.as_list(), [None, 40, 80]) + self.assertEqual(coef.shape.as_list(), [None, 12, 40, 40]) + + def test_attention_scores_with_values(self): + """Test attention outputs with coefficients.""" + test_layer = multi_head_attention.MultiHeadAttention( + num_heads=12, key_dim=64) + # Create a 3-dimensional input (the first dimension is implicit). + query = keras.Input(shape=(40, 80)) + value = keras.Input(shape=(60, 80)) + output, coef = test_layer(query, value, return_attention_scores=True) + self.assertEqual(output.shape.as_list(), [None, 40, 80]) + self.assertEqual(coef.shape.as_list(), [None, 12, 40, 60]) + + @parameterized.named_parameters(("with_bias", True), ("no_bias", False)) + def test_masked_attention(self, use_bias): + """Test with a mask tensor.""" + test_layer = multi_head_attention.MultiHeadAttention( + num_heads=2, key_dim=2, use_bias=use_bias) + # Create a 3-dimensional input (the first dimension is implicit). + batch_size = 3 + query = keras.Input(shape=(4, 8)) + value = keras.Input(shape=(2, 8)) + mask_tensor = keras.Input(shape=(4, 2)) + output = test_layer(query=query, value=value, attention_mask=mask_tensor) + + # Create a model containing the test layer. + model = keras.Model([query, value, mask_tensor], output) + + # Generate data for the input (non-mask) tensors. + from_data = 10 * np.random.random_sample((batch_size, 4, 8)) + to_data = 10 * np.random.random_sample((batch_size, 2, 8)) + + # Invoke the data with a random set of mask data. This should mask at least + # one element. + mask_data = np.random.randint(2, size=(batch_size, 4, 2)) + masked_output_data = model.predict([from_data, to_data, mask_data]) + + # Invoke the same data, but with a null mask (where no elements are masked). + null_mask_data = np.ones((batch_size, 4, 2)) + unmasked_output_data = model.predict([from_data, to_data, null_mask_data]) + + # Because one data is masked and one is not, the outputs should not be the + # same. + self.assertNotAllClose(masked_output_data, unmasked_output_data) + + # Tests the layer with three inputs: Q, K, V. + key = keras.Input(shape=(2, 8)) + output = test_layer(query, value=value, key=key, attention_mask=mask_tensor) + model = keras.Model([query, value, key, mask_tensor], output) + + masked_output_data = model.predict([from_data, to_data, to_data, mask_data]) + unmasked_output_data = model.predict( + [from_data, to_data, to_data, null_mask_data]) + # Because one data is masked and one is not, the outputs should not be the + # same. + self.assertNotAllClose(masked_output_data, unmasked_output_data) + + if use_bias: + self.assertLen(test_layer._query_dense.trainable_variables, 2) + self.assertLen(test_layer._output_dense.trainable_variables, 2) + else: + self.assertLen(test_layer._query_dense.trainable_variables, 1) + self.assertLen(test_layer._output_dense.trainable_variables, 1) + + def test_initializer(self): + """Test with a specified initializer.""" + test_layer = multi_head_attention.MultiHeadAttention( + num_heads=12, + key_dim=64, + kernel_initializer=keras.initializers.TruncatedNormal(stddev=0.02)) + # Create a 3-dimensional input (the first dimension is implicit). + query = keras.Input(shape=(40, 80)) + output = test_layer(query, query) + self.assertEqual(output.shape.as_list(), [None, 40, 80]) + + def test_masked_attention_with_scores(self): + """Test with a mask tensor.""" + test_layer = multi_head_attention.MultiHeadAttention( + num_heads=2, key_dim=2) + # Create a 3-dimensional input (the first dimension is implicit). + batch_size = 3 + query = keras.Input(shape=(4, 8)) + value = keras.Input(shape=(2, 8)) + mask_tensor = keras.Input(shape=(4, 2)) + output = test_layer(query=query, value=value, attention_mask=mask_tensor) + + # Create a model containing the test layer. + model = keras.Model([query, value, mask_tensor], output) + + # Generate data for the input (non-mask) tensors. + from_data = 10 * np.random.random_sample((batch_size, 4, 8)) + to_data = 10 * np.random.random_sample((batch_size, 2, 8)) + + # Invoke the data with a random set of mask data. This should mask at least + # one element. + mask_data = np.random.randint(2, size=(batch_size, 4, 2)) + masked_output_data = model.predict([from_data, to_data, mask_data]) + + # Invoke the same data, but with a null mask (where no elements are masked). + null_mask_data = np.ones((batch_size, 4, 2)) + unmasked_output_data = model.predict([from_data, to_data, null_mask_data]) + + # Because one data is masked and one is not, the outputs should not be the + # same. + self.assertNotAllClose(masked_output_data, unmasked_output_data) + + # Create a model containing attention scores. + output, scores = test_layer( + query=query, value=value, attention_mask=mask_tensor, + return_attention_scores=True) + model = keras.Model([query, value, mask_tensor], [output, scores]) + masked_output_data_score, masked_score = model.predict( + [from_data, to_data, mask_data]) + unmasked_output_data_score, unmasked_score = model.predict( + [from_data, to_data, null_mask_data]) + self.assertNotAllClose(masked_output_data_score, unmasked_output_data_score) + self.assertAllClose(masked_output_data, masked_output_data_score) + self.assertAllClose(unmasked_output_data, unmasked_output_data_score) + self.assertNotAllClose(masked_score, unmasked_score) + + @parameterized.named_parameters( + ("4d_inputs_1freebatch_mask2", [3, 4], [3, 2], [4, 2], + (2,)), ("4d_inputs_1freebatch_mask3", [3, 4], [3, 2], [3, 4, 2], (2,)), + ("4d_inputs_1freebatch_mask4", [3, 4], [3, 2], [3, 2, 4, 2], + (2,)), ("4D_inputs_2D_attention", [3, 4], [3, 2], [3, 4, 3, 2], (1, 2)), + ("5D_inputs_2D_attention", [5, 3, 4], [5, 3, 2], [3, 4, 3, 2], (2, 3)), + ("5D_inputs_2D_attention_fullmask", [5, 3, 4], [5, 3, 2], [5, 3, 4, 3, 2], + (2, 3))) + def test_high_dim_attention(self, q_dims, v_dims, mask_dims, attention_axes): + """Test with a mask tensor.""" + test_layer = multi_head_attention.MultiHeadAttention( + num_heads=2, key_dim=2, attention_axes=attention_axes) + batch_size, hidden_size = 3, 8 + # Generate data for the input (non-mask) tensors. + query_shape = [batch_size] + q_dims + [hidden_size] + value_shape = [batch_size] + v_dims + [hidden_size] + mask_shape = [batch_size] + mask_dims + query = 10 * np.random.random_sample(query_shape) + value = 10 * np.random.random_sample(value_shape) + + # Invoke the data with a random set of mask data. This should mask at least + # one element. + mask_data = np.random.randint(2, size=mask_shape).astype("bool") + # Invoke the same data, but with a null mask (where no elements are masked). + null_mask_data = np.ones(mask_shape) + # Because one data is masked and one is not, the outputs should not be the + # same. + query_tensor = keras.Input(query_shape[1:], name="query") + value_tensor = keras.Input(value_shape[1:], name="value") + mask_tensor = keras.Input(mask_shape[1:], name="mask") + output = test_layer(query=query_tensor, value=value_tensor, + attention_mask=mask_tensor) + model = keras.Model([query_tensor, value_tensor, mask_tensor], output) + + self.assertNotAllClose( + model.predict([query, value, mask_data]), + model.predict([query, value, null_mask_data])) + + +if __name__ == "__main__": + test.main() diff --git a/tensorflow/python/keras/layers/serialization.py b/tensorflow/python/keras/layers/serialization.py index d990f2075c8..d1fa4c19e92 100644 --- a/tensorflow/python/keras/layers/serialization.py +++ b/tensorflow/python/keras/layers/serialization.py @@ -37,6 +37,7 @@ from tensorflow.python.keras.layers import einsum_dense from tensorflow.python.keras.layers import embeddings from tensorflow.python.keras.layers import local from tensorflow.python.keras.layers import merge +from tensorflow.python.keras.layers import multi_head_attention from tensorflow.python.keras.layers import noise from tensorflow.python.keras.layers import normalization from tensorflow.python.keras.layers import normalization_v2 @@ -70,7 +71,8 @@ ALL_MODULES = (base_layer, input_layer, advanced_activations, convolutional, pooling, image_preprocessing, preprocessing_integer_lookup_v1, preprocessing_normalization_v1, preprocessing_string_lookup_v1, preprocessing_text_vectorization_v1, recurrent, wrappers, - hashing, category_crossing, category_encoding_v1, discretization) + hashing, category_crossing, category_encoding_v1, discretization, + multi_head_attention) ALL_V2_MODULES = (rnn_cell_wrapper_v2, normalization_v2, recurrent_v2, preprocessing_integer_lookup, preprocessing_normalization, preprocessing_string_lookup, preprocessing_text_vectorization, diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multi-head-attention.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multi-head-attention.pbtxt new file mode 100644 index 00000000000..070ee20ab30 --- /dev/null +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-multi-head-attention.pbtxt @@ -0,0 +1,222 @@ +path: "tensorflow.keras.layers.MultiHeadAttention" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "dynamic" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "input_spec" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "metrics" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "name_scope" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "stateful" + mtype: "" + } + member { + name: "submodules" + mtype: "" + } + member { + name: "supports_masking" + mtype: "" + } + member { + name: "trainable" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'num_heads\', \'key_dim\', \'value_dim\', \'dropout\', \'use_bias\', \'output_shape\', \'attention_axes\', \'kernel_initializer\', \'bias_initializer\', \'kernel_regularizer\', \'bias_regularizer\', \'activity_regularizer\', \'kernel_constraint\', \'bias_constraint\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'0.0\', \'True\', \'None\', \'None\', \'glorot_uniform\', \'zeros\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'query\', \'value\', \'key\', \'attention_mask\', \'return_attention_scores\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\'], " + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_output_signature" + argspec: "args=[\'self\', \'input_signature\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "with_name_scope" + argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt index db272bdf782..97e4b91bfa3 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.-softmax.pbtxt @@ -149,7 +149,7 @@ tf_class { } member_method { name: "call" - argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "compute_mask" diff --git a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt index ea139297807..35714912b04 100644 --- a/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt +++ b/tensorflow/tools/api/golden/v1/tensorflow.keras.layers.pbtxt @@ -312,6 +312,10 @@ tf_module { name: "Minimum" mtype: "" } + member { + name: "MultiHeadAttention" + mtype: "" + } member { name: "Multiply" mtype: "" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multi-head-attention.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multi-head-attention.pbtxt new file mode 100644 index 00000000000..070ee20ab30 --- /dev/null +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-multi-head-attention.pbtxt @@ -0,0 +1,222 @@ +path: "tensorflow.keras.layers.MultiHeadAttention" +tf_class { + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + is_instance: "" + member { + name: "activity_regularizer" + mtype: "" + } + member { + name: "dtype" + mtype: "" + } + member { + name: "dynamic" + mtype: "" + } + member { + name: "inbound_nodes" + mtype: "" + } + member { + name: "input" + mtype: "" + } + member { + name: "input_mask" + mtype: "" + } + member { + name: "input_shape" + mtype: "" + } + member { + name: "input_spec" + mtype: "" + } + member { + name: "losses" + mtype: "" + } + member { + name: "metrics" + mtype: "" + } + member { + name: "name" + mtype: "" + } + member { + name: "name_scope" + mtype: "" + } + member { + name: "non_trainable_variables" + mtype: "" + } + member { + name: "non_trainable_weights" + mtype: "" + } + member { + name: "outbound_nodes" + mtype: "" + } + member { + name: "output" + mtype: "" + } + member { + name: "output_mask" + mtype: "" + } + member { + name: "output_shape" + mtype: "" + } + member { + name: "stateful" + mtype: "" + } + member { + name: "submodules" + mtype: "" + } + member { + name: "supports_masking" + mtype: "" + } + member { + name: "trainable" + mtype: "" + } + member { + name: "trainable_variables" + mtype: "" + } + member { + name: "trainable_weights" + mtype: "" + } + member { + name: "updates" + mtype: "" + } + member { + name: "variables" + mtype: "" + } + member { + name: "weights" + mtype: "" + } + member_method { + name: "__init__" + argspec: "args=[\'self\', \'num_heads\', \'key_dim\', \'value_dim\', \'dropout\', \'use_bias\', \'output_shape\', \'attention_axes\', \'kernel_initializer\', \'bias_initializer\', \'kernel_regularizer\', \'bias_regularizer\', \'activity_regularizer\', \'kernel_constraint\', \'bias_constraint\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'0.0\', \'True\', \'None\', \'None\', \'glorot_uniform\', \'zeros\', \'None\', \'None\', \'None\', \'None\', \'None\'], " + } + member_method { + name: "add_loss" + argspec: "args=[\'self\', \'losses\'], varargs=None, keywords=kwargs, defaults=None" + } + member_method { + name: "add_metric" + argspec: "args=[\'self\', \'value\', \'name\'], varargs=None, keywords=kwargs, defaults=[\'None\'], " + } + member_method { + name: "add_update" + argspec: "args=[\'self\', \'updates\', \'inputs\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "add_variable" + argspec: "args=[\'self\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "add_weight" + argspec: "args=[\'self\', \'name\', \'shape\', \'dtype\', \'initializer\', \'regularizer\', \'trainable\', \'constraint\', \'use_resource\', \'synchronization\', \'aggregation\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'None\', \'VariableSynchronization.AUTO\', \'VariableAggregation.NONE\'], " + } + member_method { + name: "apply" + argspec: "args=[\'self\', \'inputs\'], varargs=args, keywords=kwargs, defaults=None" + } + member_method { + name: "build" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "call" + argspec: "args=[\'self\', \'query\', \'value\', \'key\', \'attention_mask\', \'return_attention_scores\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'False\'], " + } + member_method { + name: "compute_mask" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " + } + member_method { + name: "compute_output_shape" + argspec: "args=[\'self\', \'input_shape\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "compute_output_signature" + argspec: "args=[\'self\', \'input_signature\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "count_params" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "from_config" + argspec: "args=[\'cls\', \'config\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_config" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_input_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_losses_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_mask_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_output_shape_at" + argspec: "args=[\'self\', \'node_index\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_updates_for" + argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "get_weights" + argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "set_weights" + argspec: "args=[\'self\', \'weights\'], varargs=None, keywords=None, defaults=None" + } + member_method { + name: "with_name_scope" + argspec: "args=[\'cls\', \'method\'], varargs=None, keywords=None, defaults=None" + } +} diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt index db272bdf782..97e4b91bfa3 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.-softmax.pbtxt @@ -149,7 +149,7 @@ tf_class { } member_method { name: "call" - argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=None, defaults=None" + argspec: "args=[\'self\', \'inputs\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\'], " } member_method { name: "compute_mask" diff --git a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt index 3706919341d..078c7ec8a67 100644 --- a/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt +++ b/tensorflow/tools/api/golden/v2/tensorflow.keras.layers.pbtxt @@ -304,6 +304,10 @@ tf_module { name: "Minimum" mtype: "" } + member { + name: "MultiHeadAttention" + mtype: "" + } member { name: "Multiply" mtype: "" From 73ded7cd0e3a60c6ac7c270dfbf3f2d2ce4161fb Mon Sep 17 00:00:00 2001 From: Peng Wang Date: Wed, 12 Aug 2020 18:49:29 -0700 Subject: [PATCH 0955/1017] [TF-numpy] Makes sure that data is copied to current device in `copy` and `array(copy=True)`. Also tests that it doesn't when copy=False. PiperOrigin-RevId: 326359881 Change-Id: Icd65a73377ddba6eb26588cd540fb23952f69b5f --- .../python/ops/numpy_ops/np_array_ops.py | 8 +- .../python/ops/numpy_ops/np_array_ops_test.py | 107 ++++++++++++------ 2 files changed, 77 insertions(+), 38 deletions(-) diff --git a/tensorflow/python/ops/numpy_ops/np_array_ops.py b/tensorflow/python/ops/numpy_ops/np_array_ops.py index 5f82bca0061..7217bae75e6 100644 --- a/tensorflow/python/ops/numpy_ops/np_array_ops.py +++ b/tensorflow/python/ops/numpy_ops/np_array_ops.py @@ -172,11 +172,6 @@ def _array_internal(val, dtype=None, copy=True, ndmin=0): # pylint: disable=red else: result_t = val - if copy and isinstance(result_t, ops.Tensor): - # Note: In eager mode, a copy of `result_t` is made only if it is not on - # the context device. - result_t = array_ops.identity(result_t) - if not isinstance(result_t, ops.Tensor): if not dtype: dtype = np_utils.result_type(result_t) @@ -203,6 +198,9 @@ def _array_internal(val, dtype=None, copy=True, ndmin=0): # pylint: disable=red elif dtype: result_t = math_ops.cast(result_t, dtype) + if copy: + result_t = array_ops.identity(result_t) + if ndmin == 0: return np_arrays.tensor_to_ndarray(result_t) diff --git a/tensorflow/python/ops/numpy_ops/np_array_ops_test.py b/tensorflow/python/ops/numpy_ops/np_array_ops_test.py index d52e0c4ea83..845db14935b 100644 --- a/tensorflow/python/ops/numpy_ops/np_array_ops_test.py +++ b/tensorflow/python/ops/numpy_ops/np_array_ops_test.py @@ -24,6 +24,8 @@ import numpy as np from six.moves import range from six.moves import zip +from tensorflow.python.eager import context +from tensorflow.python.framework import config from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes from tensorflow.python.framework import indexed_slices @@ -35,10 +37,27 @@ from tensorflow.python.ops.numpy_ops import np_arrays from tensorflow.python.platform import test +_virtual_devices_ready = False + + +def set_up_virtual_devices(): + global _virtual_devices_ready + if _virtual_devices_ready: + return + physical_devices = config.list_physical_devices('CPU') + config.set_logical_device_configuration( + physical_devices[0], [ + context.LogicalDeviceConfiguration(), + context.LogicalDeviceConfiguration() + ]) + _virtual_devices_ready = True + + class ArrayCreationTest(test.TestCase): def setUp(self): super(ArrayCreationTest, self).setUp() + set_up_virtual_devices() python_shapes = [ 0, 1, 2, (), (1,), (2,), (1, 2, 3), [], [1], [2], [1, 2, 3] ] @@ -282,42 +301,51 @@ class ArrayCreationTest(test.TestCase): zeros_list = np_array_ops.zeros(5) - # TODO(srbs): Test that copy=True when context.device is different from - # tensor device copies the tensor. + def test_copy_equal_false(): + # Backing tensor is the same if copy=False, other attributes being None. + self.assertIs( + np_array_ops.array(zeros_list, copy=False).data, zeros_list.data) + self.assertIs( + np_array_ops.array(zeros_list.data, copy=False).data, zeros_list.data) - # Backing tensor is the same if copy=False, other attributes being None. - self.assertIs( - np_array_ops.array(zeros_list, copy=False).data, zeros_list.data) - self.assertIs( - np_array_ops.array(zeros_list.data, copy=False).data, zeros_list.data) + # Backing tensor is different if ndmin is not satisfied. + self.assertIsNot( + np_array_ops.array(zeros_list, copy=False, ndmin=2).data, + zeros_list.data) + self.assertIsNot( + np_array_ops.array(zeros_list.data, copy=False, ndmin=2).data, + zeros_list.data) + self.assertIs( + np_array_ops.array(zeros_list, copy=False, ndmin=1).data, + zeros_list.data) + self.assertIs( + np_array_ops.array(zeros_list.data, copy=False, ndmin=1).data, + zeros_list.data) - # Backing tensor is different if ndmin is not satisfied. - self.assertIsNot( - np_array_ops.array(zeros_list, copy=False, ndmin=2).data, - zeros_list.data) - self.assertIsNot( - np_array_ops.array(zeros_list.data, copy=False, ndmin=2).data, - zeros_list.data) - self.assertIs( - np_array_ops.array(zeros_list, copy=False, ndmin=1).data, - zeros_list.data) - self.assertIs( - np_array_ops.array(zeros_list.data, copy=False, ndmin=1).data, - zeros_list.data) + # Backing tensor is different if dtype is not satisfied. + self.assertIsNot( + np_array_ops.array(zeros_list, copy=False, dtype=int).data, + zeros_list.data) + self.assertIsNot( + np_array_ops.array(zeros_list.data, copy=False, dtype=int).data, + zeros_list.data) + self.assertIs( + np_array_ops.array(zeros_list, copy=False, dtype=float).data, + zeros_list.data) + self.assertIs( + np_array_ops.array(zeros_list.data, copy=False, dtype=float).data, + zeros_list.data) - # Backing tensor is different if dtype is not satisfied. - self.assertIsNot( - np_array_ops.array(zeros_list, copy=False, dtype=int).data, - zeros_list.data) - self.assertIsNot( - np_array_ops.array(zeros_list.data, copy=False, dtype=int).data, - zeros_list.data) - self.assertIs( - np_array_ops.array(zeros_list, copy=False, dtype=float).data, - zeros_list.data) - self.assertIs( - np_array_ops.array(zeros_list.data, copy=False, dtype=float).data, - zeros_list.data) + test_copy_equal_false() + with ops.device('CPU:1'): + test_copy_equal_false() + + self.assertNotIn('CPU:1', zeros_list.data.backing_device) + with ops.device('CPU:1'): + self.assertIn('CPU:1', np_array_ops.array(zeros_list, copy=True).data + .backing_device) + self.assertIn('CPU:1', np_array_ops.array(np.array(0), copy=True).data + .backing_device) def testAsArray(self): for a, dtype in itertools.product(self.all_arrays, self.all_types): @@ -327,6 +355,8 @@ class ArrayCreationTest(test.TestCase): zeros_list = np_array_ops.zeros(5) # Same instance is returned if no dtype is specified and input is ndarray. self.assertIs(np_array_ops.asarray(zeros_list), zeros_list) + with ops.device('CPU:1'): + self.assertIs(np_array_ops.asarray(zeros_list), zeros_list) # Different instance is returned if dtype is specified and input is ndarray. self.assertIsNot(np_array_ops.asarray(zeros_list, dtype=int), zeros_list) @@ -338,6 +368,8 @@ class ArrayCreationTest(test.TestCase): zeros_list = np_array_ops.zeros(5) # Same instance is returned if no dtype is specified and input is ndarray. self.assertIs(np_array_ops.asanyarray(zeros_list), zeros_list) + with ops.device('CPU:1'): + self.assertIs(np_array_ops.asanyarray(zeros_list), zeros_list) # Different instance is returned if dtype is specified and input is ndarray. self.assertIsNot(np_array_ops.asanyarray(zeros_list, dtype=int), zeros_list) @@ -526,6 +558,7 @@ class ArrayMethodsTest(test.TestCase): def setUp(self): super(ArrayMethodsTest, self).setUp() + set_up_virtual_devices() self.array_transforms = [ lambda x: x, ops.convert_to_tensor, @@ -600,6 +633,14 @@ class ArrayMethodsTest(test.TestCase): run_test([True]) run_test(np.arange(9).reshape((3, 3)).tolist()) + a = np_array_ops.asarray(0) + self.assertNotIn('CPU:1', a.data.backing_device) + with ops.device('CPU:1'): + self.assertIn('CPU:1', np_array_ops.array(a, copy=True).data + .backing_device) + self.assertIn('CPU:1', np_array_ops.array(np.array(0), copy=True).data + .backing_device) + def testCumProdAndSum(self): def run_test(arr, *args, **kwargs): From f41e06142bd37fccde44759820d6da99687ae717 Mon Sep 17 00:00:00 2001 From: David Rim Date: Wed, 12 Aug 2020 20:17:31 -0700 Subject: [PATCH 0956/1017] Allow string tensors to be fed to calibration wrapper PiperOrigin-RevId: 326368907 Change-Id: I089c13b7ae50658dc752a7da75a044cd962b1651 --- .../lite/python/optimize/calibration_wrapper.cc | 8 ++++++++ .../lite/python/optimize/calibrator_test.py | 15 +++++++++++++++ .../test_data/string_input_flex_model.bin | Bin 0 -> 980 bytes .../tools/optimize/calibration/calibrator.cc | 7 +++---- 4 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 tensorflow/lite/python/optimize/test_data/string_input_flex_model.bin diff --git a/tensorflow/lite/python/optimize/calibration_wrapper.cc b/tensorflow/lite/python/optimize/calibration_wrapper.cc index b608d529c85..de3de413c1d 100644 --- a/tensorflow/lite/python/optimize/calibration_wrapper.cc +++ b/tensorflow/lite/python/optimize/calibration_wrapper.cc @@ -248,6 +248,14 @@ PyObject* CalibrationWrapper::SetTensor(int index, PyObject* value) { tensor = interpreter_->tensor(index); size_t size = PyArray_NBYTES(array); + + if (tensor->type == kTfLiteString) { + tflite::DynamicBuffer buffer; + buffer.AddString(reinterpret_cast(PyArray_BYTES(array)), size); + buffer.WriteToTensor(interpreter_->tensor(index), /*new_shape=*/nullptr); + Py_RETURN_NONE; + } + if (size != tensor->bytes) { PyErr_Format(PyExc_ValueError, "numpy array had %zu bytes but expected %zu bytes.", size, diff --git a/tensorflow/lite/python/optimize/calibrator_test.py b/tensorflow/lite/python/optimize/calibrator_test.py index 6590212e25e..371b3514ca3 100644 --- a/tensorflow/lite/python/optimize/calibrator_test.py +++ b/tensorflow/lite/python/optimize/calibrator_test.py @@ -91,6 +91,21 @@ class CalibratorTest(test_util.TensorFlowTestCase, parameterized.TestCase): input_gen, constants.FLOAT, constants.FLOAT, True, 'conv2d_8/BiasAdd') self.assertIsNotNone(quantized_model) + def test_calibration_with_string_input(self): + model_path = resource_loader.get_path_to_datafile( + 'test_data/string_input_flex_model.bin') + with open(model_path, 'rb') as fp: + model_with_string_input = fp.read() + quantizer = _calibrator.Calibrator(model_with_string_input) + # Input generator for the model. + def input_gen(): + for i in range(10): + yield [np.array(u'Test' + str(i))] + + quantized_model = quantizer.calibrate_and_quantize_single( + input_gen, constants.FLOAT, constants.FLOAT, True, 'Identity') + self.assertIsNotNone(quantized_model) + @parameterized.named_parameters( # Activation type Int8 ('UseActivationTypeInt8 - EnableMlirQuantizer', constants.INT8), diff --git a/tensorflow/lite/python/optimize/test_data/string_input_flex_model.bin b/tensorflow/lite/python/optimize/test_data/string_input_flex_model.bin new file mode 100644 index 0000000000000000000000000000000000000000..3ee08810c361d1b7512bc3bf3b34f15194f39b7a GIT binary patch literal 980 zcmb1PU|5=egGgZe>(fq_AXfq?;}ALIvC z1_p)&ObiSOObiTP7#SEoFfcGwK-~zk0pur;y`V4^lUfm6Qk0pO9+K~oSe)Thnw*_l;s&-UlYxOj42Pt<5DpnB z1}#A@?!406_$07V#X?LHY(^4XIFvImC3=a7uxNnXnFR@d7Ldje&!K1)O$4{sUnKh7(SpbRoh3;(execution_plan().size(), node_to_opinfo.size()); - for (const auto op_index : interpreter->execution_plan()) { - const auto* node_and_reg = interpreter->node_and_registration(op_index); - - auto op_info = node_to_opinfo.at(op_index); + for (const auto& entry : node_to_opinfo) { + auto op_info = entry.second; + const auto* node_and_reg = interpreter->node_and_registration(entry.first); op_info.registration = &node_and_reg->second; node_ptr_opinfo_map->insert({&node_and_reg->first, op_info}); } From 0e5300e6803bf7a9f71ddedbd827c0957d36f6ad Mon Sep 17 00:00:00 2001 From: Gaurav Jain Date: Wed, 12 Aug 2020 20:29:03 -0700 Subject: [PATCH 0957/1017] Fix unstack to not change the size of tensor list This ensures that v2 TensorList behaves the same as v1 TensorList. Also unify read error messages. PiperOrigin-RevId: 326369987 Change-Id: I65c86ec9ee07a3b9bc98d5c67965efeb179b7701 --- tensorflow/core/kernels/list_kernels.h | 8 ++-- .../kernel_tests/tensor_array_ops_test.py | 38 ++++++++++++++++--- tensorflow/python/ops/tensor_array_ops.py | 13 +++++++ 3 files changed, 49 insertions(+), 10 deletions(-) diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h index 37fc1b3ae08..2f667da341c 100644 --- a/tensorflow/core/kernels/list_kernels.h +++ b/tensorflow/core/kernels/list_kernels.h @@ -174,10 +174,10 @@ class TensorListGetItem : public OpKernel { " but list elements ", DataTypeString(l->element_dtype))); int32 index = c->input(1).scalar()(); - OP_REQUIRES(c, index < l->tensors().size(), - errors::InvalidArgument("Trying to access element ", index, - " in a list with ", l->tensors().size(), - " elements.")); + OP_REQUIRES( + c, index < l->tensors().size(), + errors::InvalidArgument("Tried to read from index ", index, + " but array size is: ", l->tensors().size())); if (l->tensors()[index].dtype() != DT_INVALID) { c->set_output(0, l->tensors()[index]); } else { diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py index 4d0f6507aef..11a6eff5ada 100644 --- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py +++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py @@ -493,18 +493,14 @@ class TensorArrayTest(test.TestCase): if (control_flow_util.ENABLE_CONTROL_FLOW_V2 and not context.executing_eagerly()): - error_msg = "Trying to access element -1 in a list with 3 elements." + error_msg = "Tried to read from index -1 but array size is: 3" else: error_msg = "index -1" # Test reading from a negative index, which is not allowed with self.assertRaisesOpError(error_msg): self.evaluate(ta.read(-1)) - if (control_flow_util.ENABLE_CONTROL_FLOW_V2 and - not context.executing_eagerly()): - error_msg = "Trying to access element 3 in a list with 3 elements." - else: - error_msg = "Tried to read from index 3 but array size is: 3" + error_msg = "Tried to read from index 3 but array size is: 3" # Test reading from too large an index with self.assertRaisesOpError(error_msg): self.evaluate(ta.read(3)) @@ -1794,6 +1790,36 @@ class TensorArrayTest(test.TestCase): ta = ta.write(0, [0]) self.assertEqual([42, 1], ta.stack().shape.as_list()) + def testUnstackShouldNotAffectSize(self): + with ops.Graph().as_default() as g: + with session_lib.Session(graph=g) as sess: + a = constant_op.constant([1., 2.]) + ta = tensor_array_ops.TensorArray( + dtypes.float32, size=10, dynamic_size=True, clear_after_read=False) + error_msg = "Tried to read from index 10 but array size is: 10" + with self.assertRaisesOpError(error_msg): + sess.run(ta.read(10)) + ua = ta.unstack(a) + self.assertAllEqual(sess.run(ua.read(9)), 0.) + with self.assertRaisesOpError(error_msg): + sess.run(ua.read(10)) + + @test_util.disable_control_flow_v2("b/122315734: Requires scatter in XLA") + @test_util.run_v1_only("b/122315734: Requires scatter in XLA") + def testUnstackShouldPreserveOldValues(self): + with ops.Graph().as_default() as g: + with session_lib.Session(graph=g) as sess: + ta = tensor_array_ops.TensorArray( + dtypes.float32, size=2, dynamic_size=True, clear_after_read=False) + error_msg = "Tried to read from index 2 but array size is: 2" + with self.assertRaisesOpError(error_msg): + sess.run(ta.read(2)) + ua = ta.scatter([2], [2.]) + self.assertAllEqual(sess.run(ua.read(2)), 2.) + b = constant_op.constant([0., 1.]) + ub = ua.unstack(b) + self.assertAllEqual(sess.run(ub.read(2)), 2.) + class TensorArrayBenchmark(test.Benchmark): diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py index 58dc92084a6..61a51f81158 100644 --- a/tensorflow/python/ops/tensor_array_ops.py +++ b/tensorflow/python/ops/tensor_array_ops.py @@ -591,6 +591,19 @@ class _GraphTensorArrayV2(object): value, preferred_dtype=self._dtype, name="value") _check_dtypes(value, self._dtype) self._check_element_shape(value.shape[1:]) + + # Pad the value with zeros in order to maintain the original list's size. + # As a result the behavior of TensorList in v2 vs v1 since the later uses + # scatter. Until b/122315734 is fixed users should call scatter directly + # if they require the same behavior. + if self._dtype != dtypes.variant: + num_elements = array_ops.shape(value)[0] + size = math_ops.maximum(num_elements, self.size()) + element_rank = array_ops.rank(value) - 1 + zeros = array_ops.zeros([element_rank, 2], dtype=dtypes.int32) + paddings = array_ops.concat([[[0, size - num_elements]], zeros], 0) + value = array_ops.pad(value, paddings) + flow_out = list_ops.tensor_list_from_tensor( tensor=value, element_shape=value.shape[1:]) return build_ta_with_new_flow(self, flow_out) From 7f5fdceaf05ceb1d2a75f3f899491c2e19da2b26 Mon Sep 17 00:00:00 2001 From: Tim Shen Date: Wed, 12 Aug 2020 21:14:38 -0700 Subject: [PATCH 0958/1017] [MLIR] Change XLA HLO -> LHLO view cache key to HloInstruction. The previous cache behavior is wrong, as buffers with the same (offset, size) range don't necessarily have the same shapes and element type, but the cache produces types with shapes and element type. PiperOrigin-RevId: 326375128 Change-Id: Ic9fce8751bbbadcefeb337b2f74fd4f4f49f8443 --- .../xla/transforms/mhlo_to_lhlo_with_xla.cc | 22 +++++++++---------- .../xla/transforms/mhlo_to_lhlo_with_xla.h | 10 ++++----- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc index 832bad2dcc8..cc74d82839b 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.cc @@ -327,19 +327,17 @@ Status LhloDialectEmitter::CreateView(const HloInstruction* instr, // create another view to adjust the slice for the shape of the instruction. Status LhloDialectEmitter::GetOrCreateView(const HloInstruction* instr, SmallVectorImpl* values) { - // In terms of cache key, we have several choices: - // * Use `instr`. It's the easiest, but it creates different cache entries for - // aliased buffers, which could have been deduplicated. - // * Use the actual content as the key, aka a tree of allocation slices. - // * Somewhere in the middle, use the allocation slice for the instruction. If - // `instr` is a tuple, the key is the allocated buffer for the tuple itself - // (an array of pointers). + // Cache generated ViewOp and StaticMemRefCastOp by instruction. We could have + // gone fancier to do the following cacheing: + // %range = ViewOp(%allocation, %offset) : memref + // %typed_range = ViewOp(%range) : memref // - // We choose the third approach for simplicity. - TF_ASSIGN_OR_RETURN(BufferAllocation::Slice slice, - assignment_.GetUniqueTopLevelSlice(instr)); - SliceKey slice_key(slice.allocation(), slice.offset(), slice.size()); - auto result = slices_.try_emplace(slice_key, llvm::SmallVector{}); + // where %range is cached. This in theory gives easier time for alias + // analysis, since the identity of %range defines alias. However, + // %typed_range can't be cached, as different buffers with different types and + // shapes may still alias. Creating two ViewOps doesn't seem to worth the + // effort for a slightly easier aliasing, so we don't over optimize here. + auto result = slices_.try_emplace(instr, llvm::SmallVector{}); llvm::SmallVectorImpl& new_values = result.first->second; if (result.second) { ::xla::ShapeIndex shape_index; diff --git a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h index bdc977616b1..b191d53840d 100644 --- a/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h +++ b/tensorflow/compiler/mlir/xla/transforms/mhlo_to_lhlo_with_xla.h @@ -86,9 +86,9 @@ class LhloDialectEmitter : public ::xla::DfsHloVisitorWithDefault { // (see below). llvm::DenseMap allocations_; - // This map provides access to MLIR buffers for each HLO instruction, keyed by - // its buffer slice. A slice is contained in a BufferAllocation, and has an - // offset and a size. + // This map provides access to MLIR buffers for each HLO instruction, keyed + // instruction identity. A slice is contained in a BufferAllocation, and has + // an offset and a size. // // As for why we don't use HloInstruction*, see GetOrCreateView(), but mostly // we want to leverage better of the aliased buffers. @@ -101,8 +101,8 @@ class LhloDialectEmitter : public ::xla::DfsHloVisitorWithDefault { // // `slices_` is populated lazily in the `GetOrCreateView()` helper as we // process every instruction. - using SliceKey = std::tuple; - llvm::DenseMap> slices_; + llvm::DenseMap> + slices_; // The BufferAssignment computed by XLA ahead of time. const ::xla::BufferAssignment& assignment_; From a01cf466aac96a2745e26e8626a45468c8d9516f Mon Sep 17 00:00:00 2001 From: Rohan Jain Date: Wed, 12 Aug 2020 21:42:55 -0700 Subject: [PATCH 0959/1017] Unifying the scatter_nd* type ops shape inference code. It was duplicated in two places. Also cleaned up the error messages a bit to remove references to inner and outer dimensions of tensors and directly reference which dimensions the error message is referring to. Also unifying eager and graph mode error messages and removing some run_deprecated_v1 annotations as a result Added some c++ shape inference tests as well. PiperOrigin-RevId: 326378038 Change-Id: I58ab87ef0c476049da79c5896838a3b92649b772 --- tensorflow/core/framework/common_shape_fns.cc | 61 ++++++-------- tensorflow/core/framework/common_shape_fns.h | 5 +- tensorflow/core/kernels/scatter_nd_op.cc | 71 +++++++++------- tensorflow/core/kernels/scatter_nd_op_test.cc | 11 ++- tensorflow/core/ops/array_ops.cc | 83 +++---------------- tensorflow/core/ops/array_ops_test.cc | 33 ++++++++ tensorflow/core/ops/state_ops.cc | 36 +++++--- tensorflow/core/ops/state_ops_test.cc | 22 +++++ .../kernel_tests/scatter_nd_ops_test.py | 22 ++--- 9 files changed, 180 insertions(+), 164 deletions(-) diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc index 36ae36e7b74..8157f4ee01d 100644 --- a/tensorflow/core/framework/common_shape_fns.cc +++ b/tensorflow/core/framework/common_shape_fns.cc @@ -2257,66 +2257,57 @@ Status GatherNdShape(InferenceContext* c) { return Status::OK(); } -Status ScatterNdUpdateShape(InferenceContext* c) { - ShapeHandle input_shape = c->input(0); - if (c->input_handle_shapes_and_types(0) != nullptr) { - // This is called for tf.scatter_nd_update; input is a Variable handle. - const auto& shape_and_type = *(c->input_handle_shapes_and_types(0)); - if (shape_and_type.size() == 1) { - input_shape = shape_and_type[0].shape; - } - } - ShapeHandle indices_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &indices_shape)); - ShapeHandle updates_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(2), 1, &updates_shape)); - +Status ScatterNdShapeHelper(InferenceContext* c, ShapeHandle indices_shape, + ShapeHandle updates_shape, + ShapeHandle input_shape) { if (c->Value(c->NumElements(input_shape)) == 0 && (c->Value(c->NumElements(indices_shape)) > 0 || c->Value(c->NumElements(updates_shape)) > 0)) { return errors::InvalidArgument( - "Indices and updates specified for empty output shape"); + "Indices and updates specified for empty input"); } if (c->RankKnown(indices_shape) && c->RankKnown(updates_shape)) { - const int64 num_outer_dims = c->Rank(indices_shape) - 1; - const DimensionHandle index_size = c->Dim(indices_shape, -1); + const int64 outer_dims = c->Rank(indices_shape) - 1; + const DimensionHandle ixdim = c->Dim(indices_shape, -1); // We can only do more validation if the last dimension of indices // is a known value. - if (c->ValueKnown(index_size)) { - const int64 ix = c->Value(index_size); + if (c->ValueKnown(ixdim)) { + int64 ix = c->Value(ixdim); ShapeHandle unused; ShapeHandle prefix_indices; TF_RETURN_IF_ERROR( - c->Subshape(indices_shape, 0, num_outer_dims, &prefix_indices)); + c->Subshape(indices_shape, 0, outer_dims, &prefix_indices)); ShapeHandle prefix_updates; TF_RETURN_IF_ERROR( - c->Subshape(updates_shape, 0, num_outer_dims, &prefix_updates)); + c->Subshape(updates_shape, 0, outer_dims, &prefix_updates)); Status s = c->Merge(prefix_indices, prefix_updates, &unused); if (!s.ok()) { return errors::InvalidArgument( - "The outer ", num_outer_dims, - " dimensions of indices.shape=", c->DebugString(indices_shape), - " must match the outer ", num_outer_dims, - " dimensions of updates.shape=", c->DebugString(updates_shape), - ": ", s.error_message()); + "Dimensions [0,", outer_dims, + ") of indices[shape=", c->DebugString(indices_shape), + "] = ", c->DebugString(prefix_indices), + " must match dimensions [0,", outer_dims, + ") of updates[shape=", c->DebugString(updates_shape), + "] = ", c->DebugString(prefix_updates), ": ", s.error_message()); } - ShapeHandle input_suffix; - TF_RETURN_IF_ERROR(c->Subshape(input_shape, ix, &input_suffix)); + ShapeHandle suffix_output; + TF_RETURN_IF_ERROR(c->Subshape(input_shape, ix, &suffix_output)); ShapeHandle suffix_updates; TF_RETURN_IF_ERROR( - c->Subshape(updates_shape, num_outer_dims, &suffix_updates)); - s = c->Merge(input_suffix, suffix_updates, &unused); + c->Subshape(updates_shape, outer_dims, &suffix_updates)); + s = c->Merge(suffix_output, suffix_updates, &unused); if (!s.ok()) { return errors::InvalidArgument( - "The inner ", c->Rank(input_shape) - ix, - " dimensions of input.shape=", c->DebugString(input_shape), - " must match the inner ", c->Rank(updates_shape) - num_outer_dims, - " dimensions of updates.shape=", c->DebugString(updates_shape), - ": ", s.error_message()); + "Dimensions [", ix, ",", c->Rank(input_shape), + ") of input[shape=", c->DebugString(input_shape), + "] = ", c->DebugString(suffix_output), " must match dimensions [", + outer_dims, ",", c->Rank(updates_shape), + ") of updates[shape=", c->DebugString(updates_shape), + "] = ", c->DebugString(suffix_updates), ": ", s.error_message()); } } } diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h index 218400c2435..f3e02638f54 100644 --- a/tensorflow/core/framework/common_shape_fns.h +++ b/tensorflow/core/framework/common_shape_fns.h @@ -241,8 +241,9 @@ Status ValidateVariableResourceHandle( // Shape function for GatherNd operations. Status GatherNdShape(InferenceContext* c); -// Shape function for ScatterNd update/add/sub/... operations. -Status ScatterNdUpdateShape(InferenceContext* c); +// Helper shape function for ScatterNd.../TensorScatter... operations. +Status ScatterNdShapeHelper(InferenceContext* c, ShapeHandle indices_shape, + ShapeHandle updates_shape, ShapeHandle input_shape); // Shape function for ops with an explicit "shape" attribute. Status ExplicitShape(InferenceContext* c); diff --git a/tensorflow/core/kernels/scatter_nd_op.cc b/tensorflow/core/kernels/scatter_nd_op.cc index 88bf16d974e..942740b9af3 100644 --- a/tensorflow/core/kernels/scatter_nd_op.cc +++ b/tensorflow/core/kernels/scatter_nd_op.cc @@ -100,29 +100,31 @@ class ScatterNdOp : public OpKernel { const int64 outer_dims = indices.shape().dims() - 1; for (int i = 0; i < outer_dims; ++i) { - OP_REQUIRES(c, indices.shape().dim_size(i) == updates.shape().dim_size(i), - errors::InvalidArgument( - "Outer dimensions of indices and update must match. " - "Indices shape: ", - indices.shape().DebugString(), - ", updates shape:", updates.shape().DebugString())); + OP_REQUIRES( + c, indices.shape().dim_size(i) == updates.shape().dim_size(i), + errors::InvalidArgument( + "Dimensions [0,", outer_dims, + ") of indices[shape=", indices.shape().DebugString(), + "] must match dimensions [0,", outer_dims, + ") of updates[shape=", updates.shape().DebugString(), "]")); } const int64 ix = indices.shape().dim_size(outer_dims); - OP_REQUIRES( - c, updates.shape().dims() - outer_dims == shape.dims() - ix, - errors::InvalidArgument("Inner dimensions of output shape must match " - "inner dimensions of updates shape. Output: ", - shape.DebugString(), - " updates: ", updates.shape().DebugString())); + OP_REQUIRES(c, updates.shape().dims() - outer_dims == shape.dims() - ix, + errors::InvalidArgument( + "Dimensions [", ix, ",", shape.dims(), ") of input[shape=", + shape.DebugString(), "] must match dimensions [", + outer_dims, ",", updates.shape().dims(), + ") of updates[shape=", updates.shape().DebugString(), "]")); + for (int i = 0; i + outer_dims < updates.shape().dims(); ++i) { OP_REQUIRES( c, updates.shape().dim_size(i + outer_dims) == shape.dim_size(ix + i), - errors::InvalidArgument( - "The inner ", shape.dims() - ix, - " dimensions of output.shape=", shape.DebugString(), - " must match the inner ", updates.shape().dims() - outer_dims, - " dimensions of updates.shape=", updates.shape().DebugString())); + errors::InvalidArgument("Dimensions [", ix, ",", shape.dims(), + ") of input[shape=", shape.DebugString(), + "] must match dimensions [", outer_dims, ",", + updates.shape().dims(), ") of updates[shape=", + updates.shape().DebugString(), "]")); } OP_REQUIRES(c, shape_input.dims() == 1, errors::InvalidArgument("Shape must be a vector")); @@ -602,30 +604,35 @@ Status ValidateUpdateShape(const TensorShape& params_shape, (indices.dims() > 1) ? indices.dim_size(indices.dims() - 1) : 1; const int64 batch_dim = (indices.dims() > 1) ? indices.dims() - 1 : 1; - auto shape_err = [&]() { + auto shape_err_prefix = [&]() { return errors::InvalidArgument( - "Must have updates.shape = indices.shape[:batch_dim] + ", - "params_shape[slice_dim:], got updates.shape: ", - updates.shape().DebugString(), - ", indices.shape: ", indices.shape().DebugString(), - ", params_shape: ", params_shape.DebugString(), - ", slice_dim: ", slice_dim, ", and batch_dim: ", batch_dim); + "Dimensions [0,", batch_dim, + ") of indices[shape=", indices.shape().DebugString(), + "] must match dimensions [0,", batch_dim, + ") of updates[shape=", updates.shape().DebugString(), "]"); + }; + auto shape_err_suffix = [&]() { + return errors::InvalidArgument( + "Dimensions [", slice_dim, ",", params_shape.dims(), + ") of input[shape=", params_shape.DebugString(), + "] must match dimensions [", slice_dim, ",", updates.dims(), + ") of updates[shape=", updates.shape().DebugString(), "]"); }; - if (updates.dims() < batch_dim) return shape_err(); + if (updates.dims() < batch_dim) return shape_err_prefix(); if (params_shape.dims() < slice_dim + (updates.dims() - batch_dim)) { - return shape_err(); + return shape_err_suffix(); } if (updates.dims() != batch_dim + params_shape.dims() - slice_dim) { - return shape_err(); + return shape_err_suffix(); } for (int d = 0; d < batch_dim; ++d) { - if (updates.dim_size(d) != indices.dim_size(d)) return shape_err(); + if (updates.dim_size(d) != indices.dim_size(d)) return shape_err_prefix(); } for (int d = 0; d < updates.dims() - batch_dim; ++d) { if (updates.dim_size(d + batch_dim) != params_shape.dim_size(d + slice_dim)) { - return shape_err(); + return shape_err_suffix(); } } return Status::OK(); @@ -654,9 +661,9 @@ Status PrepareAndValidateInputs(const TensorShape& params_shape, if (updates.dim_size(0) != indices.dim_size(0)) { return errors::InvalidArgument( - "The outermost dimension of updates and indices ", - "must match. Got indices.shape ", indices_shape.DebugString(), - ", updates.shape ", updates_shape.DebugString()); + "Dimensions [0,1) of indices[shape=", indices_shape.DebugString(), + "] = ", indices.dim_size(0), " must match dimensions [0,1) of updates[", + "shape=", updates_shape.DebugString(), "] = ", updates.dim_size(0)); } TF_RETURN_IF_ERROR(ValidateUpdateShape(params_shape, indices, updates)); diff --git a/tensorflow/core/kernels/scatter_nd_op_test.cc b/tensorflow/core/kernels/scatter_nd_op_test.cc index 1461831a1fb..9c31bed784f 100644 --- a/tensorflow/core/kernels/scatter_nd_op_test.cc +++ b/tensorflow/core/kernels/scatter_nd_op_test.cc @@ -200,8 +200,8 @@ TEST_F(ScatterNdUpdateOpTest, Error_WrongDimsIndices) { Status s = RunOpKernel(); EXPECT_TRUE(absl::StrContains( s.ToString(), - "The outermost dimension of updates and indices must match. Got " - "indices.shape [1,3,1], updates.shape [3,3]")) + "Dimensions [0,1) of indices[shape=[1,3,1]] = 1 must match dimensions " + "[0,1) of updates[shape=[3,3]] = 3")) << s; } @@ -217,7 +217,9 @@ TEST_F(ScatterNdUpdateOpTest, Error_MismatchedParamsAndUpdateDimensions) { {100, 101, 102, 103, 777, 778, 779, 780, 10000, 10001, 10002, 10004}); Status s = RunOpKernel(); EXPECT_TRUE(absl::StrContains( - s.ToString(), "Must have updates.shape = indices.shape[:batch_dim]")) + s.ToString(), + "Dimensions [1,2) of input[shape=[5,3]] must match dimensions [1,2) of " + "updates[shape=[3,4]]")) << s; } @@ -233,7 +235,8 @@ TEST_F(ScatterNdUpdateOpTest, Error_MismatchedIndicesAndUpdateDimensions) { Status s = RunOpKernel(); EXPECT_TRUE(absl::StrContains( s.ToString(), - "The outermost dimension of updates and indices must match.")) + "Dimensions [0,1) of indices[shape=[3,1]] = 3 must match dimensions [0,1)" + " of updates[shape=[2,3]] = 2")) << s; } diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc index 11bfb9a3346..b4dfe6187d5 100644 --- a/tensorflow/core/ops/array_ops.cc +++ b/tensorflow/core/ops/array_ops.cc @@ -2974,73 +2974,6 @@ REGISTER_OP("QuantizedInstanceNorm") namespace { -Status ScatterNdShapeHelper(InferenceContext* c, ShapeHandle indices_shape, - ShapeHandle updates_shape, - ShapeHandle output_shape) { - if (c->Value(c->NumElements(output_shape)) == 0 && - (c->Value(c->NumElements(indices_shape)) > 0 || - c->Value(c->NumElements(updates_shape)) > 0)) { - return errors::InvalidArgument( - "Indices and updates specified for empty output shape"); - } - - if (c->RankKnown(indices_shape) && c->RankKnown(updates_shape)) { - const int64 outer_dims = c->Rank(indices_shape) - 1; - const DimensionHandle ixdim = c->Dim(indices_shape, -1); - - // We can only do more validation if the last dimension of indices - // is a known value. - if (c->ValueKnown(ixdim)) { - int64 ix = c->Value(ixdim); - ShapeHandle unused; - ShapeHandle prefix_indices; - TF_RETURN_IF_ERROR( - c->Subshape(indices_shape, 0, outer_dims, &prefix_indices)); - ShapeHandle prefix_updates; - TF_RETURN_IF_ERROR( - c->Subshape(updates_shape, 0, outer_dims, &prefix_updates)); - - Status s = c->Merge(prefix_indices, prefix_updates, &unused); - if (!s.ok()) { - return errors::InvalidArgument( - "The outer ", outer_dims, - " dimensions of indices.shape=", c->DebugString(indices_shape), - " must match the outer ", outer_dims, - " dimensions of updates.shape=", c->DebugString(updates_shape), - ": ", s.error_message()); - } - - ShapeHandle suffix_output; - TF_RETURN_IF_ERROR(c->Subshape(output_shape, ix, &suffix_output)); - ShapeHandle suffix_updates; - TF_RETURN_IF_ERROR( - c->Subshape(updates_shape, outer_dims, &suffix_updates)); - s = c->Merge(suffix_output, suffix_updates, &unused); - if (!s.ok()) { - return errors::InvalidArgument( - "The inner ", c->Rank(output_shape) - ix, - " dimensions of output.shape=", c->DebugString(output_shape), - " must match the inner ", c->Rank(updates_shape) - outer_dims, - " dimensions of updates.shape=", c->DebugString(updates_shape), - ": ", s.error_message()); - } - } - } - - c->set_output(0, output_shape); - return Status::OK(); -} - -Status ScatterNdShape(InferenceContext* c) { - ShapeHandle indices_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &indices_shape)); - ShapeHandle updates_shape; - TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &updates_shape)); - ShapeHandle output_shape; - TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &output_shape)); - return ScatterNdShapeHelper(c, indices_shape, updates_shape, output_shape); -} - Status ScatterNdTensorShape(InferenceContext* c) { ShapeHandle output_shape; TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &output_shape)); @@ -3048,7 +2981,8 @@ Status ScatterNdTensorShape(InferenceContext* c) { TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &indices_shape)); ShapeHandle updates_shape; TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(2), 1, &updates_shape)); - return ScatterNdShapeHelper(c, indices_shape, updates_shape, output_shape); + return shape_inference::ScatterNdShapeHelper(c, indices_shape, updates_shape, + output_shape); } } // namespace @@ -3088,7 +3022,16 @@ REGISTER_OP("ScatterNd") .Output("output: T") .Attr("T: type") .Attr("Tindices: {int32, int64}") - .SetShapeFn(ScatterNdShape); + .SetShapeFn([](InferenceContext* c) { + ShapeHandle indices_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(0), 1, &indices_shape)); + ShapeHandle updates_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &updates_shape)); + ShapeHandle output_shape; + TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(2, &output_shape)); + return shape_inference::ScatterNdShapeHelper(c, indices_shape, + updates_shape, output_shape); + }); REGISTER_OP("TensorScatterUpdate") .Input("tensor: T") @@ -3142,7 +3085,7 @@ REGISTER_OP("ScatterNdNonAliasingAdd") .Output("output: T") .Attr("T: {numbertype, bool}") .Attr("Tindices: {int32, int64}") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdTensorShape); REGISTER_OP("FakeQuantWithMinMaxArgs") .Attr("min: float = -6.0") diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc index 1725bdbac39..412c926d386 100644 --- a/tensorflow/core/ops/array_ops_test.cc +++ b/tensorflow/core/ops/array_ops_test.cc @@ -27,6 +27,39 @@ limitations under the License. namespace tensorflow { +TEST(ArrayOpsTest, TensorScatterUpdate_ShapeFn) { + ShapeInferenceTestOp op("TensorScatterUpdate"); + + INFER_OK(op, "[4,3];[8,2];[8]", "in0"); + INFER_OK(op, "[?,?];[?,2];[?]", "in0"); + INFER_OK(op, "[?];[?];[?]", "in0"); + + INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, + "[];[?,2];[?]"); + INFER_ERROR("Indices and updates specified for empty input", op, + "[0,2,2];[8,2];[8]"); + INFER_ERROR( + "Dimensions [0,1) of indices[shape=[8,2]] = [8] must match " + "dimensions [0,1) of updates[shape=[9]] = [9]", + op, "[?,?];[8,2];[9]"); + INFER_ERROR( + "Dimensions [2,2) of input[shape=[?,?]] = [] must match " + "dimensions [1,2) of updates[shape=[?,1]] = [1]", + op, "[?,?];[?,2];[?,1]"); +} + +TEST(ArrayOpsTest, ScatterNd_ShapeFn) { + ShapeInferenceTestOp op("ScatterNd"); + + INFER_OK(op, "[8,2];[8];[2]", "[?,?]"); + + INFER_ERROR("Shape must be rank 1 but is rank 0", op, "[?,2];[?];[]"); + INFER_ERROR( + "Dimensions [0,1) of indices[shape=[8,2]] = [8] must match " + "dimensions [0,1) of updates[shape=[9]] = [9]", + op, "[8,2];[9];[?]"); +} + TEST(ArrayOpsTest, UnravelIndex_ShapeFn) { ShapeInferenceTestOp op("UnravelIndex"); diff --git a/tensorflow/core/ops/state_ops.cc b/tensorflow/core/ops/state_ops.cc index 500d5ec88b8..5d856396360 100644 --- a/tensorflow/core/ops/state_ops.cc +++ b/tensorflow/core/ops/state_ops.cc @@ -131,6 +131,22 @@ Status ScatterUpdateShape(InferenceContext* c) { return Status::OK(); } +Status ScatterNdUpdateShape(InferenceContext* c) { + ShapeHandle input_shape = c->input(0); + if (c->input_handle_shapes_and_types(0) != nullptr) { + const auto& shape_and_type = *(c->input_handle_shapes_and_types(0)); + if (!shape_and_type.empty()) { + input_shape = shape_and_type[0].shape; + } + } + ShapeHandle indices_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(1), 1, &indices_shape)); + ShapeHandle updates_shape; + TF_RETURN_IF_ERROR(c->WithRankAtLeast(c->input(2), 1, &updates_shape)); + return shape_inference::ScatterNdShapeHelper(c, indices_shape, updates_shape, + input_shape); +} + } // namespace REGISTER_OP("ScatterUpdate") @@ -211,7 +227,7 @@ REGISTER_OP("ScatterNdUpdate") .Attr("T: type") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = true") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("ResourceScatterNdUpdate") .Input("ref: resource") @@ -220,7 +236,7 @@ REGISTER_OP("ResourceScatterNdUpdate") .Attr("T: type") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = true") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("ResourceScatterNdAdd") .Input("ref: resource") @@ -229,7 +245,7 @@ REGISTER_OP("ResourceScatterNdAdd") .Attr("T: type") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = true") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("ResourceScatterNdSub") .Input("ref: resource") @@ -238,7 +254,7 @@ REGISTER_OP("ResourceScatterNdSub") .Attr("T: type") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = true") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("ResourceScatterNdMin") .Input("ref: resource") @@ -247,7 +263,7 @@ REGISTER_OP("ResourceScatterNdMin") .Attr("T: type") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = true") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("ResourceScatterNdMax") .Input("ref: resource") @@ -256,7 +272,7 @@ REGISTER_OP("ResourceScatterNdMax") .Attr("T: type") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = true") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("ScatterNdAdd") .Input("ref: Ref(T)") @@ -266,7 +282,7 @@ REGISTER_OP("ScatterNdAdd") .Attr("T: numbertype") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = false") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("ScatterNdSub") .Input("ref: Ref(T)") @@ -276,7 +292,7 @@ REGISTER_OP("ScatterNdSub") .Attr("T: numbertype") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = false") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("ScatterNdMax") .Input("ref: Ref(T)") @@ -286,7 +302,7 @@ REGISTER_OP("ScatterNdMax") .Attr("T: numbertype") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = false") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("ScatterNdMin") .Input("ref: Ref(T)") @@ -296,7 +312,7 @@ REGISTER_OP("ScatterNdMin") .Attr("T: numbertype") .Attr("Tindices: {int32, int64}") .Attr("use_locking: bool = false") - .SetShapeFn(shape_inference::ScatterNdUpdateShape); + .SetShapeFn(ScatterNdUpdateShape); REGISTER_OP("CountUpTo") .Input("ref: Ref(T)") diff --git a/tensorflow/core/ops/state_ops_test.cc b/tensorflow/core/ops/state_ops_test.cc index a0caad4a49f..bc68cf46f03 100644 --- a/tensorflow/core/ops/state_ops_test.cc +++ b/tensorflow/core/ops/state_ops_test.cc @@ -69,6 +69,28 @@ TEST(StateOpsTest, ScatterUpdate_ShapeFn) { INFER_ERROR("Shapes must be equal rank, but are 1 and 0", op, "[2];[];[2]"); } +TEST(StateOpsTest, ResourceScatterNdUpdate_ShapeFn) { + ShapeInferenceTestOp op("ResourceScatterNdUpdate"); + TF_ASSERT_OK(NodeDefBuilder("test", "ResourceScatterNdUpdate") + .Input("ref", 0, DT_RESOURCE) + .Input("indices", 0, DT_INT32) + .Input("updates", 1, DT_FLOAT) + .Finalize(&op.node_def)); + + std::vector shapes_and_types; + op.input_resource_handle_shapes_and_types.push_back(&shapes_and_types); + op.input_resource_handle_shapes_and_types.push_back(nullptr); + op.input_resource_handle_shapes_and_types.push_back(nullptr); + shapes_and_types.emplace_back("[?,?]", DT_FLOAT); + INFER_OK(op, "[?];[?,2];[?]", ""); + INFER_ERROR("Shape must be at least rank 1 but is rank 0", op, + "[?];[?,2];[]"); + INFER_ERROR( + "Dimensions [0,1) of indices[shape=[8,2]] = [8] must match " + "dimensions [0,1) of updates[shape=[9]] = [9]", + op, "[?];[8,2];[9]"); +} + TEST(StateOpsTest, TemporaryVariable_ShapeFn) { ShapeInferenceTestOp op("TemporaryVariable"); TensorShape shape({1, 2, 3}); diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py index c5e5e549ee7..d5843c1a766 100644 --- a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py +++ b/tensorflow/python/kernel_tests/scatter_nd_ops_test.py @@ -331,24 +331,24 @@ class StatefulScatterNdTest(test.TestCase): self.evaluate(ref.initializer) self.assertAllEqual(expected_result, self.evaluate(scatter_update)) - @test_util.run_deprecated_v1 def testRank3InvalidShape1(self): indices = array_ops.zeros([3, 2, 2], dtypes.int32) updates = array_ops.zeros([2, 2, 2], dtypes.int32) shape = np.array([2, 2, 2]) ref = variables.Variable(array_ops.zeros(shape, dtypes.int32)) with self.assertRaisesWithPredicateMatch( - ValueError, r"The outer \d+ dimensions of indices\.shape="): + (errors.InvalidArgumentError, ValueError), + r"Dimensions \[\d,\d\) of indices\[shape="): state_ops.scatter_nd_update(ref, indices, updates) - @test_util.run_deprecated_v1 def testRank3InvalidShape2(self): indices = array_ops.zeros([2, 2, 1], dtypes.int32) updates = array_ops.zeros([2, 2], dtypes.int32) shape = np.array([2, 2, 2]) ref = variables.Variable(array_ops.zeros(shape, dtypes.int32)) with self.assertRaisesWithPredicateMatch( - ValueError, r"The inner \d+ dimensions of input\.shape="): + (errors.InvalidArgumentError, ValueError), + r"Dimensions \[\d,\d\) of input\[shape="): state_ops.scatter_nd_update(ref, indices, updates) def testConcurrentUpdates(self): @@ -511,14 +511,14 @@ class ScatterNdTest(test.TestCase, parameterized.TestCase): shape = array_ops.placeholder(dtypes.int32, shape=[None]) self.scatter_nd(indices, updates, shape) - @test_util.run_deprecated_v1 def testEmptyOutputShape1(self): indices = array_ops.zeros([2, 2, 2], dtypes.int32) updates = array_ops.zeros([2, 2, 2], dtypes.int32) shape = constant_op.constant([0, 3, 2], dtypes.int32) with self.assertRaisesWithPredicateMatch( - ValueError, "Indices and updates specified for empty output shape"): + (errors.InvalidArgumentError, ValueError), + "Indices and updates specified for empty"): self.scatter_nd(indices, updates, shape) def testEmptyOutputShape2(self): @@ -529,7 +529,7 @@ class ScatterNdTest(test.TestCase, parameterized.TestCase): with self.cached_session(): with self.assertRaisesOpError( - "Indices and updates specified for empty output"): + "Indices and updates specified for empty (input|output)"): self.scatter_nd(indices, updates, shape).eval( feed_dict={ indices: np.zeros([2, 2, 2], dtype=np.int32), @@ -545,22 +545,22 @@ class ScatterNdTest(test.TestCase, parameterized.TestCase): with self.cached_session(): self.assertEqual(self.evaluate(scatter).size, 0) - @test_util.run_deprecated_v1 def testRank3InvalidShape1(self): indices = array_ops.zeros([3, 2, 2], dtypes.int32) updates = array_ops.zeros([2, 2, 2], dtypes.int32) shape = np.array([2, 2, 2]) with self.assertRaisesWithPredicateMatch( - ValueError, r"The outer \d+ dimensions of indices\.shape="): + (errors.InvalidArgumentError, ValueError), + r"Dimensions \[\d\,\d\) of indices\[shape="): self.scatter_nd(indices, updates, shape) - @test_util.run_deprecated_v1 def testRank3InvalidShape2(self): indices = array_ops.zeros([2, 2, 1], dtypes.int32) updates = array_ops.zeros([2, 2], dtypes.int32) shape = np.array([2, 2, 2]) with self.assertRaisesWithPredicateMatch( - ValueError, r"The inner \d+ dimensions of (input|output)\.shape="): + (errors.InvalidArgumentError, ValueError), + r"Dimensions \[\d\,\d\) of input\[shape="): self.scatter_nd(indices, updates, shape) @parameterized.parameters(set((True, context.executing_eagerly()))) From 9605dd80bdc143ea4e8a266081ebc7d0e80317cb Mon Sep 17 00:00:00 2001 From: Rohan Jain Date: Wed, 12 Aug 2020 22:15:54 -0700 Subject: [PATCH 0960/1017] Moving scatter_nd_ops_test to kernel_tests/array_ops/ PiperOrigin-RevId: 326381792 Change-Id: I81568f0a104607f2c423903f1faa4b592c0ee72a --- tensorflow/python/kernel_tests/BUILD | 18 ------------------ tensorflow/python/kernel_tests/array_ops/BUILD | 18 ++++++++++++++++++ .../{ => array_ops}/scatter_nd_ops_test.py | 0 3 files changed, 18 insertions(+), 18 deletions(-) rename tensorflow/python/kernel_tests/{ => array_ops}/scatter_nd_ops_test.py (100%) diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 2888730e2bb..5ce8f0935b8 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -979,24 +979,6 @@ tf_py_test( ], ) -cuda_py_test( - name = "scatter_nd_ops_test", - size = "medium", - srcs = ["scatter_nd_ops_test.py"], - tags = ["noasan"], # http://b/32635055 - deps = [ - "//tensorflow/python:array_ops", - "//tensorflow/python:client", - "//tensorflow/python:client_testlib", - "//tensorflow/python:framework_for_generated_wrappers", - "//tensorflow/python:gradients", - "//tensorflow/python:resource_variable_ops", - "//tensorflow/python:state_ops", - "//tensorflow/python:variables", - "//third_party/py/numpy", - ], -) - tf_py_test( name = "segment_reduction_ops_test", size = "medium", diff --git a/tensorflow/python/kernel_tests/array_ops/BUILD b/tensorflow/python/kernel_tests/array_ops/BUILD index bc448f3da05..42fa2fef81e 100644 --- a/tensorflow/python/kernel_tests/array_ops/BUILD +++ b/tensorflow/python/kernel_tests/array_ops/BUILD @@ -60,3 +60,21 @@ cuda_py_test( "@absl_py//absl/testing:parameterized", ], ) + +cuda_py_test( + name = "scatter_nd_ops_test", + size = "medium", + srcs = ["scatter_nd_ops_test.py"], + tags = ["noasan"], # http://b/32635055 + deps = [ + "//tensorflow/python:array_ops", + "//tensorflow/python:client", + "//tensorflow/python:client_testlib", + "//tensorflow/python:framework_for_generated_wrappers", + "//tensorflow/python:gradients", + "//tensorflow/python:resource_variable_ops", + "//tensorflow/python:state_ops", + "//tensorflow/python:variables", + "//third_party/py/numpy", + ], +) diff --git a/tensorflow/python/kernel_tests/scatter_nd_ops_test.py b/tensorflow/python/kernel_tests/array_ops/scatter_nd_ops_test.py similarity index 100% rename from tensorflow/python/kernel_tests/scatter_nd_ops_test.py rename to tensorflow/python/kernel_tests/array_ops/scatter_nd_ops_test.py From 2ddc7f60e89dde0c14b3f1b631d4a618f616018c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Aug 2020 22:24:44 -0700 Subject: [PATCH 0961/1017] Fix unstack to not change the size of tensor list This ensures that v2 TensorList behaves the same as v1 TensorList. Also unify read error messages. PiperOrigin-RevId: 326383033 Change-Id: If7eb236b0c6fcf87660c66defac21eebfebc2880 --- tensorflow/core/kernels/list_kernels.h | 8 ++-- .../kernel_tests/tensor_array_ops_test.py | 38 +++---------------- tensorflow/python/ops/tensor_array_ops.py | 13 ------- 3 files changed, 10 insertions(+), 49 deletions(-) diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h index 2f667da341c..37fc1b3ae08 100644 --- a/tensorflow/core/kernels/list_kernels.h +++ b/tensorflow/core/kernels/list_kernels.h @@ -174,10 +174,10 @@ class TensorListGetItem : public OpKernel { " but list elements ", DataTypeString(l->element_dtype))); int32 index = c->input(1).scalar()(); - OP_REQUIRES( - c, index < l->tensors().size(), - errors::InvalidArgument("Tried to read from index ", index, - " but array size is: ", l->tensors().size())); + OP_REQUIRES(c, index < l->tensors().size(), + errors::InvalidArgument("Trying to access element ", index, + " in a list with ", l->tensors().size(), + " elements.")); if (l->tensors()[index].dtype() != DT_INVALID) { c->set_output(0, l->tensors()[index]); } else { diff --git a/tensorflow/python/kernel_tests/tensor_array_ops_test.py b/tensorflow/python/kernel_tests/tensor_array_ops_test.py index 11a6eff5ada..4d0f6507aef 100644 --- a/tensorflow/python/kernel_tests/tensor_array_ops_test.py +++ b/tensorflow/python/kernel_tests/tensor_array_ops_test.py @@ -493,14 +493,18 @@ class TensorArrayTest(test.TestCase): if (control_flow_util.ENABLE_CONTROL_FLOW_V2 and not context.executing_eagerly()): - error_msg = "Tried to read from index -1 but array size is: 3" + error_msg = "Trying to access element -1 in a list with 3 elements." else: error_msg = "index -1" # Test reading from a negative index, which is not allowed with self.assertRaisesOpError(error_msg): self.evaluate(ta.read(-1)) - error_msg = "Tried to read from index 3 but array size is: 3" + if (control_flow_util.ENABLE_CONTROL_FLOW_V2 and + not context.executing_eagerly()): + error_msg = "Trying to access element 3 in a list with 3 elements." + else: + error_msg = "Tried to read from index 3 but array size is: 3" # Test reading from too large an index with self.assertRaisesOpError(error_msg): self.evaluate(ta.read(3)) @@ -1790,36 +1794,6 @@ class TensorArrayTest(test.TestCase): ta = ta.write(0, [0]) self.assertEqual([42, 1], ta.stack().shape.as_list()) - def testUnstackShouldNotAffectSize(self): - with ops.Graph().as_default() as g: - with session_lib.Session(graph=g) as sess: - a = constant_op.constant([1., 2.]) - ta = tensor_array_ops.TensorArray( - dtypes.float32, size=10, dynamic_size=True, clear_after_read=False) - error_msg = "Tried to read from index 10 but array size is: 10" - with self.assertRaisesOpError(error_msg): - sess.run(ta.read(10)) - ua = ta.unstack(a) - self.assertAllEqual(sess.run(ua.read(9)), 0.) - with self.assertRaisesOpError(error_msg): - sess.run(ua.read(10)) - - @test_util.disable_control_flow_v2("b/122315734: Requires scatter in XLA") - @test_util.run_v1_only("b/122315734: Requires scatter in XLA") - def testUnstackShouldPreserveOldValues(self): - with ops.Graph().as_default() as g: - with session_lib.Session(graph=g) as sess: - ta = tensor_array_ops.TensorArray( - dtypes.float32, size=2, dynamic_size=True, clear_after_read=False) - error_msg = "Tried to read from index 2 but array size is: 2" - with self.assertRaisesOpError(error_msg): - sess.run(ta.read(2)) - ua = ta.scatter([2], [2.]) - self.assertAllEqual(sess.run(ua.read(2)), 2.) - b = constant_op.constant([0., 1.]) - ub = ua.unstack(b) - self.assertAllEqual(sess.run(ub.read(2)), 2.) - class TensorArrayBenchmark(test.Benchmark): diff --git a/tensorflow/python/ops/tensor_array_ops.py b/tensorflow/python/ops/tensor_array_ops.py index 61a51f81158..58dc92084a6 100644 --- a/tensorflow/python/ops/tensor_array_ops.py +++ b/tensorflow/python/ops/tensor_array_ops.py @@ -591,19 +591,6 @@ class _GraphTensorArrayV2(object): value, preferred_dtype=self._dtype, name="value") _check_dtypes(value, self._dtype) self._check_element_shape(value.shape[1:]) - - # Pad the value with zeros in order to maintain the original list's size. - # As a result the behavior of TensorList in v2 vs v1 since the later uses - # scatter. Until b/122315734 is fixed users should call scatter directly - # if they require the same behavior. - if self._dtype != dtypes.variant: - num_elements = array_ops.shape(value)[0] - size = math_ops.maximum(num_elements, self.size()) - element_rank = array_ops.rank(value) - 1 - zeros = array_ops.zeros([element_rank, 2], dtype=dtypes.int32) - paddings = array_ops.concat([[[0, size - num_elements]], zeros], 0) - value = array_ops.pad(value, paddings) - flow_out = list_ops.tensor_list_from_tensor( tensor=value, element_shape=value.shape[1:]) return build_ta_with_new_flow(self, flow_out) From d38f51e12d77d9f613069ec68245238c93b1975f Mon Sep 17 00:00:00 2001 From: Thai Nguyen Date: Wed, 12 Aug 2020 22:43:24 -0700 Subject: [PATCH 0962/1017] Avoid passing --checkpoint to the build script PiperOrigin-RevId: 326385936 Change-Id: I660fcce3ba43beeea9f49a5357aeb3551f2b36b0 --- tensorflow/lite/tools/build_aar_with_docker.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorflow/lite/tools/build_aar_with_docker.sh b/tensorflow/lite/tools/build_aar_with_docker.sh index 2af4787c35c..094406e2076 100755 --- a/tensorflow/lite/tools/build_aar_with_docker.sh +++ b/tensorflow/lite/tools/build_aar_with_docker.sh @@ -35,6 +35,7 @@ function print_usage { # Check command line flags. ARGUMENTS=$@ +BUILD_FLAGS="" TARGET_ARCHS=x86,x86_64,arm64-v8a,armeabi-v7a FLAG_CHECKPOINT="master" @@ -48,9 +49,11 @@ do case $i in --input_models=*) FLAG_MODELS="${i#*=}" + BUILD_FLAGS="${BUILD_FLAGS} ${i}" shift;; --target_archs=*) TARGET_ARCHS="${i#*=}" + BUILD_FLAGS="${BUILD_FLAGS} ${i}" shift;; --checkpoint=*) FLAG_CHECKPOINT="${i#*=}" @@ -67,7 +70,7 @@ if [ ! -d /tensorflow_src ]; then do FLAG_DIR="${FLAG_DIR} -v ${model}:${model}" done - docker run --rm -it -v $PWD:/tmp -v ${SCRIPT_DIR}:/script_dir ${FLAG_DIR} \ + docker run --rm -it -v $PWD:/host_dir -v ${SCRIPT_DIR}:/script_dir ${FLAG_DIR} \ --entrypoint /script_dir/build_aar_with_docker.sh tflite-builder \ ${ARGUMENTS} exit 0 @@ -97,7 +100,7 @@ else git checkout ${FLAG_CHECKPOINT} # Building with bazel. - bash /tensorflow_src/tensorflow/lite/tools/build_aar.sh ${ARGUMENTS} + bash /tensorflow_src/tensorflow/lite/tools/build_aar.sh ${BUILD_FLAGS} # Copy the output files from docker container. clear @@ -107,7 +110,7 @@ else for i in ${OUT_FILES} do if [ -f $i ]; then - cp $i /tmp + cp $i /host_dir basename $i fi done From e58593b86a956e3e1c7df4ce775a791ec7295616 Mon Sep 17 00:00:00 2001 From: Chuanhao Zhuge Date: Wed, 12 Aug 2020 22:52:53 -0700 Subject: [PATCH 0963/1017] Implement CopyTensorHandleToDevice C_API for TFRT. Introduced a conversion function to handle transferring TFRuntimeFallback tensor to DenseGpu tensor. Enable a GPU micro-benchmark and several TF Python tests for TFRT. PiperOrigin-RevId: 326387094 Change-Id: I4e10f30c512f2a0a7060cb143318dff3119786c5 --- tensorflow/python/eager/BUILD | 1 + tensorflow/python/eager/benchmarks_test.py | 1 - tensorflow/python/eager/core_test.py | 39 ++++++++++++++++++---- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 358929dc870..9a54ee3d628 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -376,6 +376,7 @@ cuda_py_test( size = "small", srcs = ["core_test.py"], python_version = "PY3", + tfrt_enabled = True, deps = [ ":context", ":core", diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py index 22110e1ae71..667d3f1cff4 100644 --- a/tensorflow/python/eager/benchmarks_test.py +++ b/tensorflow/python/eager/benchmarks_test.py @@ -647,7 +647,6 @@ class MicroBenchmarks(benchmarks_test_base.MicroBenchmarksBase): num_iters=self._num_iters_2_by_2, execution_mode=context.ASYNC) - @test_util.disable_tfrt("copy to GPU not supported") def benchmark_tf_matmul_2_by_2_GPU(self): if not context.num_gpus(): return diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py index d756827f44f..d7f412b2408 100644 --- a/tensorflow/python/eager/core_test.py +++ b/tensorflow/python/eager/core_test.py @@ -324,6 +324,7 @@ class TFETest(test_util.TensorFlowTestCase): else: ops.disable_tensor_equality() + @test_util.disable_tfrt('Async execution mode not supported in TFRT.') def testContext(self): ctx = context.Context() self.assertTrue(ctx.executing_eagerly()) @@ -383,6 +384,7 @@ class TFETest(test_util.TensorFlowTestCase): with ctx.device(device_spec): self.assertEqual(device_name, ctx.device_name) + @test_util.disable_tfrt('Async execution mode not supported in TFRT.') def testAsyncBasic(self): ctx = context.Context(execution_mode=context.ASYNC) ctx.ensure_initialized() @@ -392,6 +394,7 @@ class TFETest(test_util.TensorFlowTestCase): self.assertTrue(has_cpu_device) del ctx + @test_util.disable_tfrt('Multi CPU placement not supported yet.') def testMultiCpuPlacement(self): with ops.device('cpu:1'): x = array_ops.identity(1.0) @@ -401,10 +404,12 @@ class TFETest(test_util.TensorFlowTestCase): self.assertEqual(y.device, '/job:localhost/replica:0/task:0/device:CPU:0') @test_util.run_gpu_only + @test_util.disable_tfrt('Device name incorrect (known issue for runtime ' + 'fallback).') def testShouldCopy(self): with ops.device('GPU:0'): x = array_ops.identity(1.0) - self.assertEqual(x.device, '/job:localhost/replica:0/task:0/device:GPU:0') + self.assertEndsWith(x.device, 'GPU:0') y = array_ops.identity(x) # The value we're testing y.device against will depend on what the behavior # of not explicitly specifying a device in the context is. This behavior is @@ -426,6 +431,7 @@ class TFETest(test_util.TensorFlowTestCase): self.assertFalse(switch.is_building_function) @test_util.run_gpu_only + @test_util.disable_tfrt('Resolve not implemented yet.') def testInt32GPU(self): with ops.device('gpu:0'): xent = nn_ops.sparse_softmax_cross_entropy_with_logits( @@ -461,6 +467,7 @@ class TFETest(test_util.TensorFlowTestCase): self.assertAllEqual(context_values, get_context_values(ctx)) @test_util.run_gpu_only + @test_util.disable_tfrt('Context config not supported in TFRT.') def testContextConfig(self): ctx = context.Context(config=config_pb2.ConfigProto( device_count={'GPU': 0})) @@ -478,6 +485,7 @@ class TFETest(test_util.TensorFlowTestCase): self.assertAllEqual(t.numpy(), 10.0) @test_util.run_gpu_only + @test_util.disable_tfrt('Resolve not implemented yet.') def testDevicePlacementEnforcesConsistency(self): cpu = context.device('cpu:0') gpu = context.device('gpu:0') @@ -494,6 +502,8 @@ class TFETest(test_util.TensorFlowTestCase): cpu.__exit__() @test_util.run_gpu_only + @test_util.disable_tfrt('Device name incorrect (known issue for runtime ' + 'fallback).') def testReEntrant(self): cpu = context.device('cpu:0') gpu = context.device('gpu:0') @@ -518,6 +528,7 @@ class TFETest(test_util.TensorFlowTestCase): self.assertEqual(3, result) @test_util.run_gpu_only + @test_util.disable_tfrt('Resolve not implemented yet.') def testResourceTensorPlacement(self): with context.device('gpu:0'): v = resource_variable_ops.ResourceVariable(1.0) @@ -540,6 +551,7 @@ class TFETest(test_util.TensorFlowTestCase): x.gpu(context.context().num_gpus() + 1) @test_util.run_gpu_only + @test_util.disable_tfrt('Async execution mode not supported in TFRT.') def testCopyBetweenDevicesAsync(self): with context.execution_mode(context.ASYNC): x = constant_op.constant([[1., 2.], [3., 4.]]) @@ -556,6 +568,7 @@ class TFETest(test_util.TensorFlowTestCase): context.context().executor.clear_error() @test_util.run_gpu_only + @test_util.disable_tfrt('TensorHandleInterface::Resolve() not implemented.') def testCopyScope(self): constant = constant_op.constant(1.0) with ops.device('gpu:0'): @@ -563,6 +576,7 @@ class TFETest(test_util.TensorFlowTestCase): c = constant + 1.0 self.assertAllEqual(c, 2.0) + @test_util.disable_tfrt('ContextFromInterface not implemented.') def testPyFunctionNullContext(self): def simple_fn(unused_handle): return 1. @@ -577,6 +591,7 @@ class TFETest(test_util.TensorFlowTestCase): self.assertAllEqual(test_fn(test_var), 1.0) + @test_util.disable_tfrt('Async execution mode not supported in TFRT.') def testPyFunctionAsync(self): def simple_fn(v): @@ -594,6 +609,7 @@ class TFETest(test_util.TensorFlowTestCase): async_executor.wait() @test_util.run_gpu_only + @test_util.disable_tfrt('Resolve not implemented yet.') def testNumpyForceCPU(self): cpu = constant_op.constant([[1., 2.], [3., 4.]]) c2g = cpu.gpu() @@ -622,6 +638,7 @@ class TFETest(test_util.TensorFlowTestCase): attrs=('T', three.dtype.as_datatype_enum))[0] self.assertAllEqual(15, product) + @test_util.disable_tfrt('Async execution mode not supported in TFRT.') def testExecuteBasicAsync(self): with context.execution_mode(context.ASYNC): three = constant_op.constant(3) @@ -650,6 +667,8 @@ class TFETest(test_util.TensorFlowTestCase): context.context().executor.clear_error() context.context().execution_mode = context.SYNC + @test_util.disable_tfrt('TFRT asserts correct number of outputs instead of ' + 'returning error status.') def testExecuteTooManyNumOutputs(self): # num_outputs provided is 50, but only one output is produced. product = execute( @@ -660,6 +679,8 @@ class TFETest(test_util.TensorFlowTestCase): attrs=('T', dtypes.int32.as_datatype_enum))[0] self.assertAllEqual(15, product) + @test_util.disable_tfrt('TFRT asserts correct number of outputs instead of ' + 'returning error status.') def testExecuteTooFewNumOutputs(self): # num_outputs provided is 0, but one output is produced. with self.assertRaises(errors.InvalidArgumentError): @@ -671,6 +692,7 @@ class TFETest(test_util.TensorFlowTestCase): attrs=('T', dtypes.int32.as_datatype_enum))[0] @test_util.run_gpu_only + @test_util.disable_tfrt('Resolve not implemented yet.') def testMatMulGPU(self): three = constant_op.constant([[3.]]).gpu() five = constant_op.constant([[5.]]).gpu() @@ -736,8 +758,8 @@ class TFETest(test_util.TensorFlowTestCase): product = execute( b'MatMul', num_outputs=1, - inputs=[constant_op.constant([[3]]), - constant_op.constant([[5]])], + inputs=[constant_op.constant([[3.]]), + constant_op.constant([[5.]])], attrs=('transpose_a', True, 'transpose_b', False, 'T', dtypes.int32.as_datatype_enum))[0] self.assertAllEqual([[15]], product) @@ -903,6 +925,7 @@ class TFETest(test_util.TensorFlowTestCase): inputs=[constant_op.constant(3.0)], attrs=('T', dtypes.float32.as_datatype_enum)) + @test_util.disable_tfrt('TFRT raises InternalError instead of NotFoundError') def testExecuteUnknownOp(self): with self.assertRaises(errors.NotFoundError): execute(b'BlahBlahBlah', num_outputs=1, inputs=[], attrs=None) @@ -973,7 +996,7 @@ class TFETest(test_util.TensorFlowTestCase): c = a + b # Op forced to CPU since all constants are integers and small. - self.assertEqual(c.device, '/job:localhost/replica:0/task:0/device:CPU:0') + self.assertEndsWith(c.device, 'CPU:0') a = array_ops.zeros((8, 10), dtype=dtypes.int64) b = array_ops.ones((8, 10), dtype=dtypes.int64) @@ -982,7 +1005,7 @@ class TFETest(test_util.TensorFlowTestCase): c = a + b # Op not forced to CPU since the tensors are larger than 64 elements. - self.assertEqual(c.device, '/job:localhost/replica:0/task:0/device:GPU:0') + self.assertEndsWith(c.device, 'GPU:0') a = constant_op.constant((1, 2, 3, 4, 5), dtype=dtypes.float32) b = constant_op.constant((2, 3, 4, 5, 6), dtype=dtypes.float32) @@ -990,7 +1013,7 @@ class TFETest(test_util.TensorFlowTestCase): c = a + b # Op not forced to CPU since the constants are not integers. - self.assertEqual(c.device, '/job:localhost/replica:0/task:0/device:GPU:0') + self.assertEndsWith(c.device, 'GPU:0') def testExecutionModeIsStoredThreadLocal(self): cv = threading.Condition() @@ -1024,6 +1047,8 @@ class TFETest(test_util.TensorFlowTestCase): for t in threads: t.join() + @test_util.disable_tfrt('Does not support converting DT_RESOURCE' + 'to op attr type yet.') def testEmptyResourceReturned(self): with ops.device('CPU:0'): v = variables.Variable(1.) @@ -1066,6 +1091,7 @@ class SendRecvTest(test_util.TensorFlowTestCase): context._reset_context() configure_virtual_cpus() + @test_util.disable_tfrt('Send/Receive not supported in TFRT yet.') def testBasic(self): t0 = constant_op.constant(1.0) t1 = constant_op.constant(2.0) @@ -1079,6 +1105,7 @@ class SendRecvTest(test_util.TensorFlowTestCase): 2.0) @test_util.run_gpu_only + @test_util.disable_tfrt('Send/Receive not supported in TFRT yet.') def testLocalCrossDevice(self): gpu_device_name = '/job:localhost/replica:0/task:0/device:GPU:0' with ops.device('GPU:0'): From cd47e8c149d96c5cf2d99fd76d9f28db5ea1da2c Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 12 Aug 2020 22:53:39 -0700 Subject: [PATCH 0964/1017] Remove extraneous kernel definitions to fix build issue PiperOrigin-RevId: 326387173 Change-Id: Ica3c59dfa1291498a33c70396f062deccee94fea --- tensorflow/core/tpu/kernels/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/core/tpu/kernels/BUILD b/tensorflow/core/tpu/kernels/BUILD index 0c7ac668555..6d3369022ad 100644 --- a/tensorflow/core/tpu/kernels/BUILD +++ b/tensorflow/core/tpu/kernels/BUILD @@ -38,7 +38,6 @@ tf_kernel_library( ":tpu_execute_op", ":tpu_handle_to_key_op", ":transfer_ops", - "//tensorflow/core/tpu/kernels/xla:xla_ops", ], ) From bbe632aac7592cad28a99e23d16b76c033928a01 Mon Sep 17 00:00:00 2001 From: Henry Tan Date: Wed, 12 Aug 2020 23:15:37 -0700 Subject: [PATCH 0965/1017] Refactor TpuProgramGroup supporting multi-programs adding tpu_program_group::set_hlo_metadatas() function. PiperOrigin-RevId: 326389324 Change-Id: Ifc91c3258a09d25648266648d0f8e0dfcb018c8d --- tensorflow/core/tpu/kernels/tpu_program_group.cc | 11 ++++++----- tensorflow/core/tpu/kernels/tpu_program_group.h | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tensorflow/core/tpu/kernels/tpu_program_group.cc b/tensorflow/core/tpu/kernels/tpu_program_group.cc index 39d1f38b104..ff7a526cd45 100644 --- a/tensorflow/core/tpu/kernels/tpu_program_group.cc +++ b/tensorflow/core/tpu/kernels/tpu_program_group.cc @@ -112,7 +112,7 @@ void TpuProgramGroup::Initialize( xla_tpu_programs.size()); std::vector hlo_metadatas(xla_tpu_programs.size()); for (size_t i = 0; i < xla_tpu_programs.size(); ++i) { - const XLA_TpuProgram* xla_tpu_program = xla_tpu_programs[i]; + const XLA_TpuProgram* xla_tpu_program = tpu_programs_[i]; bool may_modify_variables; TpuProgramApiFn()->TpuProgram_GetMayModifyVariablesFn( xla_tpu_program, &may_modify_variables); @@ -250,10 +250,11 @@ TpuProgramGroup::TpuProgramGroup(TpuProgramGroup&& other) RefreshHloMetadatasPtrs(); } -void TpuProgramGroup::set_hlo_metadata(const xla::HloProto& hlo_metadata) { - // TODO(henrytan): initialize hlo_metadatas_ for multi program support. - if (hlo_metadatas_.empty()) { - hlo_metadatas_.push_back(hlo_metadata); +void TpuProgramGroup::set_hlo_metadatas( + absl::Span hlo_metadatas) { + hlo_metadatas_.resize(hlo_metadatas.size()); + for (size_t i = 0; i < hlo_metadatas.size(); ++i) { + hlo_metadatas_[i] = hlo_metadatas[i]; } RefreshHloMetadatasPtrs(); } diff --git a/tensorflow/core/tpu/kernels/tpu_program_group.h b/tensorflow/core/tpu/kernels/tpu_program_group.h index b76ef3d507a..0fc8bff08de 100644 --- a/tensorflow/core/tpu/kernels/tpu_program_group.h +++ b/tensorflow/core/tpu/kernels/tpu_program_group.h @@ -133,7 +133,7 @@ class TpuProgramGroup : public TpuProgramGroupInterface { const TPUExecutableInfoProto& executable_info(int index) const; const TPUHostTransferInfoProto& host_transfer_info(int index) const; - void set_hlo_metadata(const xla::HloProto& hlo_metadata); + void set_hlo_metadatas(absl::Span hlo_metadatas); const xla::HloProto* hlo_metadata(int index) const; absl::Span hlo_metadatas() const override; From b26956c716bee48337c07a2bef4de25a4ed3544a Mon Sep 17 00:00:00 2001 From: Thai Nguyen Date: Thu, 13 Aug 2020 01:50:26 -0700 Subject: [PATCH 0966/1017] Use Json lib instead of absl::string to generate json string PiperOrigin-RevId: 326404382 Change-Id: Icc443330ecd69d07b8f510c7b8062f1ef5a84195 --- tensorflow/lite/tools/BUILD | 4 ++-- tensorflow/lite/tools/list_flex_ops.cc | 19 +++++++++---------- .../lite/tools/list_flex_ops_no_kernel.cc | 15 ++++++--------- tensorflow/lite/tools/list_flex_ops_test.cc | 16 ++++++++-------- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/tensorflow/lite/tools/BUILD b/tensorflow/lite/tools/BUILD index ad19cd2b519..c29d30e750e 100644 --- a/tensorflow/lite/tools/BUILD +++ b/tensorflow/lite/tools/BUILD @@ -267,8 +267,8 @@ cc_library( "//tensorflow/core:tensorflow", "//tensorflow/lite:framework", "//tensorflow/lite:util", - "@com_google_absl//absl/strings", "@flatbuffers", + "@jsoncpp_git//:jsoncpp", ], ) @@ -302,7 +302,7 @@ cc_library( hdrs = ["list_flex_ops.h"], deps = [ "//tensorflow/lite:framework", - "@com_google_absl//absl/strings", + "@jsoncpp_git//:jsoncpp", ], ) diff --git a/tensorflow/lite/tools/list_flex_ops.cc b/tensorflow/lite/tools/list_flex_ops.cc index 9c80afbc90e..6fdfd061134 100644 --- a/tensorflow/lite/tools/list_flex_ops.cc +++ b/tensorflow/lite/tools/list_flex_ops.cc @@ -19,9 +19,8 @@ limitations under the License. #include #include -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" #include "flatbuffers/flexbuffers.h" // from @flatbuffers +#include "include/json/json.h" #include "tensorflow/core/framework/node_def.pb.h" #include "tensorflow/core/framework/node_def_util.h" #include "tensorflow/core/framework/op.h" @@ -34,14 +33,14 @@ namespace tflite { namespace flex { std::string OpListToJSONString(const OpKernelSet& flex_ops) { - return absl::StrCat("[", - absl::StrJoin(flex_ops, ",\n", - [](std::string* out, const OpKernel& op) { - absl::StrAppend(out, "[\"", op.op_name, - "\", \"", op.kernel_name, - "\"]"); - }), - "]"); + Json::Value result(Json::arrayValue); + for (const OpKernel& op : flex_ops) { + Json::Value op_kernel(Json::arrayValue); + op_kernel.append(Json::Value(op.op_name)); + op_kernel.append(Json::Value(op.kernel_name)); + result.append(op_kernel); + } + return Json::FastWriter().write(result); } // Find the class name of the op kernel described in the node_def from the pool diff --git a/tensorflow/lite/tools/list_flex_ops_no_kernel.cc b/tensorflow/lite/tools/list_flex_ops_no_kernel.cc index 11a9f39dbfd..ea4d41c0de3 100644 --- a/tensorflow/lite/tools/list_flex_ops_no_kernel.cc +++ b/tensorflow/lite/tools/list_flex_ops_no_kernel.cc @@ -12,21 +12,18 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" +#include "include/json/json.h" #include "tensorflow/lite/tools/list_flex_ops.h" namespace tflite { namespace flex { std::string OpListToJSONString(const OpKernelSet& flex_ops) { - return absl::StrCat("[", - absl::StrJoin(flex_ops, ",\n", - [](std::string* out, const OpKernel& op) { - absl::StrAppend(out, "\"", op.op_name, - "\""); - }), - "]"); + Json::Value result(Json::arrayValue); + for (const OpKernel& op : flex_ops) { + result.append(Json::Value(op.op_name)); + } + return Json::FastWriter().write(result); } void AddFlexOpsFromModel(const tflite::Model* model, OpKernelSet* flex_ops) { diff --git a/tensorflow/lite/tools/list_flex_ops_test.cc b/tensorflow/lite/tools/list_flex_ops_test.cc index 872d7509d0c..7d81dda71e6 100644 --- a/tensorflow/lite/tools/list_flex_ops_test.cc +++ b/tensorflow/lite/tools/list_flex_ops_test.cc @@ -87,7 +87,7 @@ class FlexOpModel : public SingleOpModel { TEST_F(FlexOpsListTest, TestModelsNoFlex) { ReadOps("tensorflow/lite/testdata/test_model.bin"); - EXPECT_EQ(output_text_, "[]"); + EXPECT_EQ(output_text_, "[]\n"); } TEST_F(FlexOpsListTest, TestBrokenModel) { @@ -97,29 +97,29 @@ TEST_F(FlexOpsListTest, TestBrokenModel) { TEST_F(FlexOpsListTest, TestZeroSubgraphs) { ReadOps("tensorflow/lite/testdata/0_subgraphs.bin"); - EXPECT_EQ(output_text_, "[]"); + EXPECT_EQ(output_text_, "[]\n"); } TEST_F(FlexOpsListTest, TestFlexAdd) { ReadOps("tensorflow/lite/testdata/multi_add_flex.bin"); EXPECT_EQ(output_text_, - "[[\"Add\", \"BinaryOp>\"]]"); + "[[\"Add\",\"BinaryOp>\"]]\n"); } TEST_F(FlexOpsListTest, TestTwoModel) { ReadOps("tensorflow/lite/testdata/multi_add_flex.bin"); ReadOps("tensorflow/lite/testdata/softplus_flex.bin"); EXPECT_EQ(output_text_, - "[[\"Add\", \"BinaryOp>\"],\n[\"Softplus\", \"SoftplusOp\"]]"); + "[[\"Add\",\"BinaryOp>\"],[\"Softplus\",\"SoftplusOp\"]]\n"); } TEST_F(FlexOpsListTest, TestDuplicatedOp) { ReadOps("tensorflow/lite/testdata/multi_add_flex.bin"); ReadOps("tensorflow/lite/testdata/multi_add_flex.bin"); EXPECT_EQ(output_text_, - "[[\"Add\", \"BinaryOp>\"]]"); + "[[\"Add\",\"BinaryOp>\"]]\n"); } TEST_F(FlexOpsListTest, TestInvalidCustomOptions) { @@ -192,7 +192,7 @@ TEST_F(FlexOpsListTest, TestFlexAddWithSingleOpModel) { CreateFlexCustomOptions(nodedef_raw_str)); ReadOps(tflite::GetModel(max_model.GetModelBuffer())); EXPECT_EQ(output_text_, - "[[\"Add\", \"BinaryOp>\"]]"); + "[[\"Add\",\"BinaryOp>\"]]\n"); } } // namespace flex } // namespace tflite From 0e52e44a4d6a0237c03e7de1371994c0bd919fa4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 13 Aug 2020 02:01:21 -0700 Subject: [PATCH 0967/1017] compat: Update forward compatibility horizon to 2020-08-13 PiperOrigin-RevId: 326405448 Change-Id: I1e76680cc2747048c74181b53bff0b0d538eafd1 --- tensorflow/python/compat/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/python/compat/compat.py b/tensorflow/python/compat/compat.py index b56c3fab967..6d12e1071ed 100644 --- a/tensorflow/python/compat/compat.py +++ b/tensorflow/python/compat/compat.py @@ -33,7 +33,7 @@ from tensorflow.python.util.tf_export import tf_export # This value changes every day with an automatic CL. It can be modified in code # via `forward_compatibility_horizon()` or with the environment variable # TF_FORWARD_COMPATIBILITY_DELTA_DAYS, which is added to the compatibility date. -_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 12) +_FORWARD_COMPATIBILITY_HORIZON = datetime.date(2020, 8, 13) _FORWARD_COMPATIBILITY_DELTA_DAYS_VAR_NAME = "TF_FORWARD_COMPATIBILITY_DELTA_DAYS" _FORWARD_COMPATIBILITY_DATE_NUMBER = None From c60a66c4eec48011af907925409db466348b7da0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 13 Aug 2020 02:01:23 -0700 Subject: [PATCH 0968/1017] Update GraphDef version to 492. PiperOrigin-RevId: 326405451 Change-Id: Icd6cc73fd9d875fce99b890412aadc3a6cbb7e15 --- tensorflow/core/public/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index f60f3907c58..f804b8e14cb 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -108,7 +108,7 @@ limitations under the License. #define TF_GRAPH_DEF_VERSION_MIN_PRODUCER 0 #define TF_GRAPH_DEF_VERSION_MIN_CONSUMER 0 -#define TF_GRAPH_DEF_VERSION 491 // Updated: 2020/8/12 +#define TF_GRAPH_DEF_VERSION 492 // Updated: 2020/8/13 // Checkpoint compatibility versions (the versions field in SavedSliceMeta). // From e481c700c85396155e550402228a067ffbec7f82 Mon Sep 17 00:00:00 2001 From: Ben Barsdell Date: Tue, 4 Aug 2020 20:55:49 +1000 Subject: [PATCH 0969/1017] Enable depthwise convs in auto_mixed_precision - These are well-supported as of CUDNN v8. - Also adds a Python test. --- .../optimizers/auto_mixed_precision_lists.h | 10 ++--- .../grappler/auto_mixed_precision_test.py | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h b/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h index 805a7de9225..7902700fb0f 100644 --- a/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h +++ b/tensorflow/core/grappler/optimizers/auto_mixed_precision_lists.h @@ -127,11 +127,6 @@ class AutoMixedPrecisionListsCuda : public AutoMixedPrecisionLists { "GRUBlockCellGrad", "LSTMBlockCell", "LSTMBlockCellGrad", - // TODO(benbarsdell): Enable these when fast and safe fp16 kernels are - // available for depthwise convolutions. - // "DepthwiseConv2dNative", - // "DepthwiseConv2dNativeBackpropFilter", - // "DepthwiseConv2dNativeBackpropInput", "MatMul", }; if (cuda_version_ >= 9010) { @@ -147,6 +142,11 @@ class AutoMixedPrecisionListsCuda : public AutoMixedPrecisionLists { list.insert("Conv3DBackpropInput"); list.insert("Conv3DBackpropInputV2"); } + if (cudnn_version_ >= 8000) { + list.insert("DepthwiseConv2dNative"); + list.insert("DepthwiseConv2dNativeBackpropFilter"); + list.insert("DepthwiseConv2dNativeBackpropInput"); + } UpdateList("ALLOWLIST", &list); // For backwards compatibility, keeping the original env variable here. // TODO(reedwm): This should be removed if we don't have active users. diff --git a/tensorflow/python/grappler/auto_mixed_precision_test.py b/tensorflow/python/grappler/auto_mixed_precision_test.py index 567ff8c000d..0066fcb9712 100644 --- a/tensorflow/python/grappler/auto_mixed_precision_test.py +++ b/tensorflow/python/grappler/auto_mixed_precision_test.py @@ -46,6 +46,7 @@ from tensorflow.python.ops import random_ops from tensorflow.python.ops import tensor_array_ops from tensorflow.python.ops import variables from tensorflow.python.ops.losses import losses +from tensorflow.python.platform import sysconfig from tensorflow.python.platform import test from tensorflow.python.training import adam from tensorflow.python.training import gradient_descent @@ -138,6 +139,11 @@ def _conv_pool(x): return h_pool2 +def _depthwise_conv2d(x, w): + """Returns a 2d depthwise convolution layer with full stride.""" + return nn.depthwise_conv2d(x, w, strides=[1, 1, 1, 1], padding='SAME') + + def _simple_loop(x, functor): """Simple loop whose body is provided by the functor.""" init = (constant_op.constant(0), x) @@ -566,6 +572,42 @@ class AutoMixedPrecisionTest(test.TestCase, parameterized.TestCase): tol = 5e-3 if mode == 'mkl' else 1e-3 self.assertAllClose(output_val_ref, output_val, atol=tol, rtol=tol) + # TODO(benbarsdell): This test has not been tried with MKL. + @parameterized.parameters(['cuda']) + @test_util.run_deprecated_v1 + @test_util.disable_xla('This test does not pass with XLA') + def test_depthwise_conv2d(self, mode): + """Test grad ops with depthwise convolution2d graph.""" + self._maybe_skip(mode) + cudnn_version_str = sysconfig.get_build_info().get('cudnn_version', '0.0') + cudnn_version = tuple([int(x) for x in cudnn_version_str.split('.')]) + if cudnn_version < (8,): + # Depthwise conv2d ops are only enabled in auto_mixed_precision as of + # cuDNN v8. + self.skipTest('cuDNN version >= 8 required') + random_seed.set_random_seed(0) + x = _input([2, 8, 8, 1]) + f = _weight([3, 3, 1, 4]) + y = _depthwise_conv2d(x, f) + y = array_ops.identity(y) + optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.01) + g = optimizer.compute_gradients(y, [x, f]) + output = (y, g) + + output_val_ref, output_val, cost_graph = self._run(mode, output) + node_map = _build_node_map(cost_graph.node) + self._assert_output_f16(mode, node_map, 'depthwise') + self._assert_output_f16( + mode, node_map, + 'gradients/depthwise_grad/DepthwiseConv2dNativeBackpropInput') + self._assert_output_f16( + mode, node_map, + 'gradients/depthwise_grad/DepthwiseConv2dNativeBackpropFilter') + + output_val_ref, output_val, cost_graph = self._run(mode, output) + tol = 2e-3 + self.assertAllClose(output_val_ref, output_val, atol=tol, rtol=tol) + @parameterized.parameters(['cuda', 'mkl']) @test_util.run_v1_only('b/138749235') @test_util.disable_xla('This test does not pass with XLA') From 4b4c52406374fb4afc63dba3c63399f16a59b5cf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 13 Aug 2020 06:34:30 -0700 Subject: [PATCH 0970/1017] Make schema visible to internal targets. PiperOrigin-RevId: 326434621 Change-Id: I5be8a024c1d1ccc337464e5eea8ad5cda0bb630a --- tensorflow/lite/experimental/acceleration/compatibility/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow/lite/experimental/acceleration/compatibility/BUILD b/tensorflow/lite/experimental/acceleration/compatibility/BUILD index 9aa5a4b5459..559abc44a4d 100644 --- a/tensorflow/lite/experimental/acceleration/compatibility/BUILD +++ b/tensorflow/lite/experimental/acceleration/compatibility/BUILD @@ -28,6 +28,8 @@ flatbuffer_cc_library( srcs = ["database.fbs"], ) +exports_files(srcs = ["database.fbs"]) + cc_library( name = "devicedb", srcs = [ From e9bf0700e5aea3b368a11abfb1bbe105381f18c3 Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Thu, 13 Aug 2020 07:42:41 -0700 Subject: [PATCH 0971/1017] [JAX] [XLA:Python] Branch JAX pytree library into xla/python/. This is not an ideal location for this library, given it isn't really part of XLA, however it allows us to solve a library linking problem as part of work experimenting with faster C++ JIT dispatch for JAX. PiperOrigin-RevId: 326443769 Change-Id: I59a4cedcc3b22e1048922b37d12af5443496ec3b --- tensorflow/compiler/xla/python/BUILD | 22 + tensorflow/compiler/xla/python/pytree.cc | 648 +++++++++++++++++++++++ tensorflow/compiler/xla/python/pytree.h | 214 ++++++++ tensorflow/compiler/xla/python/xla.cc | 2 + 4 files changed, 886 insertions(+) create mode 100644 tensorflow/compiler/xla/python/pytree.cc create mode 100644 tensorflow/compiler/xla/python/pytree.h diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD index aa55a39218d..1330dca6402 100644 --- a/tensorflow/compiler/xla/python/BUILD +++ b/tensorflow/compiler/xla/python/BUILD @@ -327,6 +327,27 @@ cc_library( ], ) +# TODO(phawkins): this library is really part of JAX. Find a better home for it. +cc_library( + name = "pytree", + srcs = ["pytree.cc"], + hdrs = ["pytree.h"], + copts = [ + "-fexceptions", + "-fno-strict-aliasing", + ], + features = ["-use_header_modules"], + deps = [ + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/hash", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@pybind11", + ], +) + config_setting( name = "enable_gpu", values = {"define": "xla_python_enable_gpu=true"}, @@ -348,6 +369,7 @@ pybind_extension( ":dlpack", ":ops", ":py_client", + ":pytree", ":python_ref_manager", ":outfeed_receiver_py", ":traceback", diff --git a/tensorflow/compiler/xla/python/pytree.cc b/tensorflow/compiler/xla/python/pytree.cc new file mode 100644 index 00000000000..401b7bc66b9 --- /dev/null +++ b/tensorflow/compiler/xla/python/pytree.cc @@ -0,0 +1,648 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Caution: this code uses exceptions. The exception use is local to the +// binding code and the idiomatic way to emit Python exceptions. + +#include "tensorflow/compiler/xla/python/pytree.h" + +#include +#include +#include +#include + +#include "absl/algorithm/container.h" +#include "absl/container/flat_hash_map.h" +#include "absl/hash/hash.h" +#include "absl/memory/memory.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "pybind11/pybind11.h" +#include "pybind11/pytypes.h" +#include "pybind11/stl.h" + +namespace xla { + +namespace py = pybind11; + +/*static*/ CustomNodeRegistry* CustomNodeRegistry::Singleton() { + static auto* registry = new CustomNodeRegistry; + return registry; +} + +/*static*/ void CustomNodeRegistry::Register(py::object type, + py::function to_iterable, + py::function from_iterable) { + CustomNodeRegistry* registry = Singleton(); + auto registration = absl::make_unique(); + registration->type = type; + registration->to_iterable = std::move(to_iterable); + registration->from_iterable = std::move(from_iterable); + auto it = registry->registrations_.emplace(type, std::move(registration)); + if (!it.second) { + throw std::invalid_argument( + absl::StrFormat("Duplicate custom PyTreeDef type registration for %s.", + py::repr(type))); + } +} + +/*static*/ const CustomNodeRegistry::Registration* CustomNodeRegistry::Lookup( + py::handle type) { + CustomNodeRegistry* registry = Singleton(); + auto it = + registry->registrations_.find(py::reinterpret_borrow(type)); + return it == registry->registrations_.end() ? nullptr : it->second.get(); +} + +bool PyTreeDef::operator==(const PyTreeDef& other) const { + if (traversal_.size() != other.traversal_.size()) { + return false; + } + for (size_t i = 0; i < traversal_.size(); ++i) { + const Node& a = traversal_[i]; + const Node& b = other.traversal_[i]; + if (a.kind != b.kind || a.arity != b.arity || + (a.node_data.ptr() == nullptr) != (b.node_data.ptr() == nullptr) || + a.custom != b.custom) { + return false; + } + if (a.node_data && a.node_data.not_equal(b.node_data)) { + return false; + } + // We don't need to test equality of num_leaves and num_nodes since they + // are derivable from the other node data. + } + return true; +} + +/*static*/ PyTreeDef::Kind PyTreeDef::GetKind( + const py::handle& obj, CustomNodeRegistry::Registration const** custom) { + const PyObject* ptr = obj.ptr(); + if (PyTuple_CheckExact(ptr)) return Kind::kTuple; + if (PyList_CheckExact(ptr)) return Kind::kList; + if (PyDict_CheckExact(ptr)) return Kind::kDict; + if ((*custom = CustomNodeRegistry::Lookup(obj.get_type()))) { + return Kind::kCustom; + } else if (py::isinstance(obj)) { + return Kind::kNone; + } else if (py::isinstance(obj) && py::hasattr(obj, "_fields")) { + // We can only identify namedtuples heuristically, here by the presence of + // a _fields attribute. + return Kind::kNamedTuple; + } else { + return Kind::kLeaf; + } +} + +void PyTreeDef::FlattenInto(py::handle handle, + std::vector& leaves) { + Node node; + int start_num_nodes = traversal_.size(); + int start_num_leaves = leaves.size(); + node.kind = GetKind(handle, &node.custom); + if (node.kind == Kind::kNone) { + // Nothing to do. + } else if (node.kind == Kind::kTuple) { + py::tuple tuple = py::reinterpret_borrow(handle); + node.arity = tuple.size(); + for (py::handle entry : tuple) { + FlattenInto(entry, leaves); + } + } else if (node.kind == Kind::kList) { + py::list list = py::reinterpret_borrow(handle); + node.arity = list.size(); + for (py::handle entry : list) { + FlattenInto(entry, leaves); + } + } else if (node.kind == Kind::kDict) { + py::dict dict = py::reinterpret_borrow(handle); + py::list keys = py::reinterpret_steal(PyDict_Keys(dict.ptr())); + if (PyList_Sort(keys.ptr())) { + throw std::runtime_error("Dictionary key sort failed."); + } + for (py::handle key : keys) { + FlattenInto(dict[key], leaves); + } + node.arity = dict.size(); + node.node_data = std::move(keys); + } else if (node.kind == Kind::kCustom) { + py::tuple out = py::cast(node.custom->to_iterable(handle)); + if (out.size() != 2) { + throw std::runtime_error( + "PyTree custom to_iterable function should return a pair"); + } + node.node_data = out[1]; + node.arity = 0; + for (py::handle entry : py::cast(out[0])) { + ++node.arity; + FlattenInto(entry, leaves); + } + } else if (node.kind == Kind::kNamedTuple) { + py::tuple tuple = py::reinterpret_borrow(handle); + node.arity = tuple.size(); + node.node_data = py::reinterpret_borrow(tuple.get_type()); + for (py::handle entry : tuple) { + FlattenInto(entry, leaves); + } + } else { + assert(node.kind == Kind::kLeaf); + leaves.push_back(py::reinterpret_borrow(handle)); + } + node.num_nodes = traversal_.size() - start_num_nodes + 1; + node.num_leaves = leaves.size() - start_num_leaves; + traversal_.push_back(std::move(node)); +} + +/*static*/ std::pair, std::unique_ptr> +PyTreeDef::Flatten(py::handle x) { + std::vector leaves; + auto tree = absl::make_unique(); + tree->FlattenInto(x, leaves); + return std::make_pair(std::move(leaves), std::move(tree)); +} + +/*static*/ bool PyTreeDef::AllLeaves(const py::iterable& x) { + const CustomNodeRegistry::Registration* custom; + for (const py::handle& h : x) { + if (GetKind(h, &custom) != Kind::kLeaf) return false; + } + return true; +} + +py::object PyTreeDef::Unflatten(py::iterable leaves) const { + std::vector agenda; + auto it = leaves.begin(); + int leaf_count = 0; + for (const Node& node : traversal_) { + if (agenda.size() < node.arity) { + throw std::logic_error("Too few elements for TreeDef node."); + } + switch (node.kind) { + case Kind::kLeaf: + if (it == leaves.end()) { + throw std::invalid_argument(absl::StrFormat( + "Too few leaves for PyTreeDef; expected %d, got %d", num_leaves(), + leaf_count)); + } + agenda.push_back(py::reinterpret_borrow(*it)); + ++it; + ++leaf_count; + break; + + case Kind::kNone: + case Kind::kTuple: + case Kind::kNamedTuple: + case Kind::kList: + case Kind::kDict: + case Kind::kCustom: { + const int size = agenda.size(); + absl::Span span; + if (node.arity > 0) { + span = absl::Span(&agenda[size - node.arity], node.arity); + } + py::object o = MakeNode(node, span); + agenda.resize(size - node.arity); + agenda.push_back(o); + break; + } + } + } + if (it != leaves.end()) { + throw std::invalid_argument(absl::StrFormat( + "Too many leaves for PyTreeDef; expected %d.", num_leaves())); + } + if (agenda.size() != 1) { + throw std::logic_error("PyTreeDef traversal did not yield a singleton."); + } + return std::move(agenda.back()); +} + +/*static*/ py::object PyTreeDef::MakeNode(const PyTreeDef::Node& node, + absl::Span children) { + if (children.size() != node.arity) { + throw std::logic_error("Node arity mismatch."); + } + switch (node.kind) { + case Kind::kLeaf: + throw std::logic_error("MakeNode not implemented for leaves."); + + case Kind::kNone: + return py::none(); + + case Kind::kTuple: + case Kind::kNamedTuple: { + py::tuple tuple(node.arity); + for (int i = 0; i < node.arity; ++i) { + tuple[i] = std::move(children[i]); + } + if (node.kind == Kind::kNamedTuple) { + return node.node_data(*tuple); + } else { + return std::move(tuple); + } + } + + case Kind::kList: { + py::list list(node.arity); + for (int i = 0; i < node.arity; ++i) { + list[i] = std::move(children[i]); + } + return std::move(list); + } + + case Kind::kDict: { + py::dict dict; + py::list keys = py::reinterpret_borrow(node.node_data); + for (int i = 0; i < node.arity; ++i) { + dict[keys[i]] = std::move(children[i]); + } + return std::move(dict); + break; + } + case Kind::kCustom: { + py::tuple tuple(node.arity); + for (int i = 0; i < node.arity; ++i) { + tuple[i] = std::move(children[i]); + } + return node.custom->from_iterable(node.node_data, tuple); + } + } + throw std::logic_error("Unreachable code."); +} + +py::list PyTreeDef::FlattenUpTo(py::handle xs) const { + py::list leaves(num_leaves()); + std::vector agenda; + agenda.push_back(py::reinterpret_borrow(xs)); + auto it = traversal_.rbegin(); + int leaf = num_leaves() - 1; + while (!agenda.empty()) { + if (it == traversal_.rend()) { + throw std::invalid_argument(absl::StrFormat( + "Tree structures did not match: %s vs %s", py::repr(xs), ToString())); + } + const Node& node = *it; + py::object object = agenda.back(); + agenda.pop_back(); + ++it; + + switch (node.kind) { + case Kind::kLeaf: + if (leaf < 0) { + throw std::logic_error("Leaf count mismatch."); + } + leaves[leaf] = py::reinterpret_borrow(object); + --leaf; + break; + + case Kind::kNone: + break; + + case Kind::kTuple: { + if (!PyTuple_CheckExact(object.ptr())) { + throw std::invalid_argument( + absl::StrFormat("Expected tuple, got %s.", py::repr(object))); + } + py::tuple tuple = py::reinterpret_borrow(object); + if (tuple.size() != node.arity) { + throw std::invalid_argument( + absl::StrFormat("Tuple arity mismatch: %d != %d; tuple: %s.", + tuple.size(), node.arity, py::repr(object))); + } + for (py::handle entry : tuple) { + agenda.push_back(py::reinterpret_borrow(entry)); + } + break; + } + + case Kind::kList: { + if (!PyList_CheckExact(object.ptr())) { + throw std::invalid_argument( + absl::StrFormat("Expected list, got %s.", py::repr(object))); + } + py::list list = py::reinterpret_borrow(object); + if (list.size() != node.arity) { + throw std::invalid_argument( + absl::StrFormat("List arity mismatch: %d != %d; list: %s.", + list.size(), node.arity, py::repr(object))); + } + for (py::handle entry : list) { + agenda.push_back(py::reinterpret_borrow(entry)); + } + break; + } + + case Kind::kDict: { + if (!PyDict_CheckExact(object.ptr())) { + throw std::invalid_argument( + absl::StrFormat("Expected dict, got %s.", py::repr(object))); + } + py::dict dict = py::reinterpret_borrow(object); + py::list keys = + py::reinterpret_steal(PyDict_Keys(dict.ptr())); + if (PyList_Sort(keys.ptr())) { + throw std::runtime_error("Dictionary key sort failed."); + } + if (keys.not_equal(node.node_data)) { + throw std::invalid_argument( + absl::StrFormat("Dict key mismatch; expected keys: %s; dict: %s.", + py::repr(node.node_data), py::repr(object))); + } + for (py::handle key : keys) { + agenda.push_back(dict[key]); + } + break; + } + + case Kind::kNamedTuple: { + if (!py::isinstance(object) || + !py::hasattr(object, "_fields")) { + throw std::invalid_argument(absl::StrFormat( + "Expected named tuple, got %s.", py::repr(object))); + } + py::tuple tuple = py::reinterpret_borrow(object); + if (tuple.size() != node.arity) { + throw std::invalid_argument(absl::StrFormat( + "Named tuple arity mismatch: %d != %d; tuple: %s.", tuple.size(), + node.arity, py::repr(object))); + } + if (tuple.get_type().not_equal(node.node_data)) { + throw std::invalid_argument(absl::StrFormat( + "Named tuple type mismatch: expected type: %s, tuple: %s.", + py::repr(node.node_data), py::repr(object))); + } + for (py::handle entry : tuple) { + agenda.push_back(py::reinterpret_borrow(entry)); + } + break; + } + + case Kind::kCustom: { + auto* registration = CustomNodeRegistry::Lookup(object.get_type()); + if (registration != node.custom) { + throw std::invalid_argument(absl::StrFormat( + "Custom node type mismatch: expected type: %s, value: %s.", + py::repr(node.custom->type), py::repr(object))); + } + py::tuple out = py::cast(node.custom->to_iterable(object)); + if (out.size() != 2) { + throw std::runtime_error( + "PyTree custom to_iterable function should return a pair"); + } + if (node.node_data.not_equal(out[1])) { + throw std::invalid_argument(absl::StrFormat( + "Mismatch custom node data: %s != %s; value: %s.", + py::repr(node.node_data), py::repr(out[1]), py::repr(object))); + } + int arity = 0; + for (py::handle entry : py::cast(out[0])) { + ++arity; + agenda.push_back(py::reinterpret_borrow(entry)); + } + if (arity != node.arity) { + throw std::invalid_argument(absl::StrFormat( + "Custom type arity mismatch: %d != %d; value: %s.", arity, + node.arity, py::repr(object))); + } + break; + } + } + } + if (it != traversal_.rend() || leaf != -1) { + throw std::invalid_argument(absl::StrFormat( + "Tree structures did not match: %s vs %s", py::repr(xs), ToString())); + } + return leaves; +} + +py::object PyTreeDef::Walk(const py::function& f_node, py::handle f_leaf, + py::iterable leaves) const { + std::vector agenda; + auto it = leaves.begin(); + for (const Node& node : traversal_) { + switch (node.kind) { + case Kind::kLeaf: { + if (it == leaves.end()) { + throw std::invalid_argument("Too few leaves for PyTreeDef"); + } + + py::object leaf = py::reinterpret_borrow(*it); + agenda.push_back(f_leaf.is_none() ? std::move(leaf) + : f_leaf(std::move(leaf))); + ++it; + break; + } + + case Kind::kNone: + case Kind::kTuple: + case Kind::kNamedTuple: + case Kind::kList: + case Kind::kDict: + case Kind::kCustom: { + if (agenda.size() < node.arity) { + throw std::logic_error("Too few elements for custom type."); + } + py::tuple tuple(node.arity); + for (int i = node.arity - 1; i >= 0; --i) { + tuple[i] = agenda.back(); + agenda.pop_back(); + } + agenda.push_back(f_node(tuple)); + } + } + } + if (it != leaves.end()) { + throw std::invalid_argument("Too many leaves for PyTreeDef"); + } + if (agenda.size() != 1) { + throw std::logic_error("PyTreeDef traversal did not yield a singleton."); + } + return std::move(agenda.back()); +} + +py::object PyTreeDef::FromIterableTreeHelper( + py::handle xs, + std::vector::const_reverse_iterator* it) const { + if (*it == traversal_.rend()) { + throw std::invalid_argument("Tree structures did not match."); + } + const Node& node = **it; + ++*it; + if (node.kind == Kind::kLeaf) { + return py::reinterpret_borrow(xs); + } + py::iterable iterable = py::reinterpret_borrow(xs); + std::vector ys; + ys.reserve(node.arity); + for (py::handle x : iterable) { + ys.push_back(py::reinterpret_borrow(x)); + } + if (ys.size() != node.arity) { + throw std::invalid_argument("Arity mismatch between trees"); + } + for (int j = node.arity - 1; j >= 0; --j) { + ys[j] = FromIterableTreeHelper(ys[j], it); + } + + return MakeNode(node, absl::MakeSpan(ys)); +} + +py::object PyTreeDef::FromIterableTree(py::handle xs) const { + auto it = traversal_.rbegin(); + py::object out = FromIterableTreeHelper(xs, &it); + if (it != traversal_.rend()) { + throw std::invalid_argument("Tree structures did not match."); + } + return out; +} + +std::unique_ptr PyTreeDef::Compose(const PyTreeDef& inner) const { + auto out = absl::make_unique(); + for (const Node& n : traversal_) { + if (n.kind == Kind::kLeaf) { + absl::c_copy(inner.traversal_, std::back_inserter(out->traversal_)); + } else { + out->traversal_.push_back(n); + } + } + const auto& root = traversal_.back(); + const auto& inner_root = inner.traversal_.back(); + // TODO(tomhennigan): This should update all nodes in the traversal. + auto& out_root = out->traversal_.back(); + out_root.num_nodes = (root.num_nodes - root.num_leaves) + + (inner_root.num_nodes * root.num_leaves); + out_root.num_leaves *= inner_root.num_leaves; + return out; +} + +/*static*/ std::unique_ptr PyTreeDef::Tuple( + const std::vector& defs) { + auto out = absl::make_unique(); + for (const PyTreeDef& def : defs) { + absl::c_copy(def.traversal_, std::back_inserter(out->traversal_)); + } + Node node; + node.kind = Kind::kTuple; + node.arity = defs.size(); + out->traversal_.push_back(node); + return out; +} + +std::vector> PyTreeDef::Children() const { + std::vector> children; + if (traversal_.empty()) { + return children; + } + Node const& root = traversal_.back(); + children.resize(root.arity); + int pos = traversal_.size() - 1; + for (int i = root.arity - 1; i >= 0; --i) { + children[i] = absl::make_unique(); + const Node& node = traversal_.at(pos - 1); + if (pos < node.num_nodes) { + throw std::logic_error("children() walked off start of array"); + } + std::copy(traversal_.begin() + pos - node.num_nodes, + traversal_.begin() + pos, + std::back_inserter(children[i]->traversal_)); + pos -= node.num_nodes; + } + if (pos != 0) { + throw std::logic_error("pos != 0 at end of PyTreeDef::Children"); + } + return children; +} + +std::string PyTreeDef::ToString() const { + std::vector agenda; + for (const Node& node : traversal_) { + if (agenda.size() < node.arity) { + throw std::logic_error("Too few elements for container."); + } + + std::string kind; + switch (node.kind) { + case Kind::kLeaf: + agenda.push_back("*"); + continue; + case Kind::kNone: + kind = "None"; + break; + case Kind::kNamedTuple: + kind = "namedtuple"; + break; + case Kind::kTuple: + kind = "tuple"; + break; + case Kind::kList: + kind = "list"; + break; + case Kind::kDict: + kind = "dict"; + break; + case Kind::kCustom: + kind = static_cast(py::str(node.custom->type)); + break; + } + + std::string children = + absl::StrJoin(agenda.end() - node.arity, agenda.end(), ","); + agenda.erase(agenda.end() - node.arity, agenda.end()); + + std::string data; + if (node.node_data) { + data = absl::StrFormat("[%s]", py::str(node.node_data)); + } + + agenda.push_back( + absl::StrFormat("PyTreeDef(%s%s, [%s])", kind, data, children)); + } + + if (agenda.size() != 1) { + throw std::logic_error("PyTreeDef traversal did not yield a singleton."); + } + return std::move(agenda.back()); +} + +void BuildPytreeSubmodule(py::module& m) { + py::module pytree = m.def_submodule("pytree", "Python tree library"); + pytree.def("flatten", &PyTreeDef::Flatten); + pytree.def("tuple", &PyTreeDef::Tuple); + pytree.def("all_leaves", &PyTreeDef::AllLeaves); + + py::class_(m, "PyTreeDef") + .def("unflatten", &PyTreeDef::Unflatten) + .def("flatten_up_to", &PyTreeDef::FlattenUpTo) + .def("compose", &PyTreeDef::Compose) + .def("walk", &PyTreeDef::Walk) + .def("from_iterable_tree", &PyTreeDef::FromIterableTree) + .def("children", &PyTreeDef::Children) + .def_property_readonly("num_leaves", &PyTreeDef::num_leaves) + .def_property_readonly("num_nodes", &PyTreeDef::num_nodes) + .def("__repr__", &PyTreeDef::ToString) + .def("__eq__", + [](const PyTreeDef& a, const PyTreeDef& b) { return a == b; }) + .def("__ne__", + [](const PyTreeDef& a, const PyTreeDef& b) { return a != b; }) + .def("__hash__", + [](const PyTreeDef& t) { return absl::Hash()(t); }); + + pytree.def("register_node", [](py::object type, py::function to_iterable, + py::function from_iterable) { + return CustomNodeRegistry::Register(type, to_iterable, from_iterable); + }); +} + +} // namespace xla diff --git a/tensorflow/compiler/xla/python/pytree.h b/tensorflow/compiler/xla/python/pytree.h new file mode 100644 index 00000000000..69cd93a7d08 --- /dev/null +++ b/tensorflow/compiler/xla/python/pytree.h @@ -0,0 +1,214 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_COMPILER_XLA_PYTHON_PYTREE_H_ +#define TENSORFLOW_COMPILER_XLA_PYTHON_PYTREE_H_ + +// See https://jax.readthedocs.io/en/latest/pytrees.html for the documentation +// about pytree. + +// Caution: this code uses exceptions. The exception use is local to the +// binding code and the idiomatic way to emit Python exceptions. + +#include +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/hash/hash.h" +#include "absl/memory/memory.h" +#include "pybind11/pybind11.h" +#include "pybind11/pytypes.h" +#include "pybind11/stl.h" + +namespace xla { + +// Registry of custom node types. +class CustomNodeRegistry { + public: + struct Registration { + // The Python type object, used to identify the type. + pybind11::object type; + // A function with signature: object -> (iterable, aux_data) + pybind11::function to_iterable; + // A function with signature: (aux_data, iterable) -> object + pybind11::function from_iterable; + }; + + // Registers a new custom type. Objects of `type` will be treated as container + // node types in PyTrees. + static void Register(pybind11::object type, pybind11::function to_iterable, + pybind11::function from_iterable); + + // Finds the custom type registration for `type`. Returns nullptr if none + // exists. + static const Registration* Lookup(pybind11::handle type); + + private: + static CustomNodeRegistry* Singleton(); + + struct TypeHash { + size_t operator()(const pybind11::object& t) const { + return pybind11::hash(t); + } + }; + struct TypeEq { + bool operator()(const pybind11::object& a, + const pybind11::object& b) const { + return a.equal(b); + } + }; + absl::flat_hash_map, TypeHash, + TypeEq> + registrations_; +}; + +// A PyTreeDef describes the tree structure of a PyTree. A PyTree is a tree of +// Python values, where the interior nodes are tuples, lists, dictionaries, or +// user-defined containers, and the leaves are other objects. +class PyTreeDef { + public: + PyTreeDef() = default; + + // Flattens a Pytree into a list of leaves and a PyTreeDef. + static std::pair, std::unique_ptr> + Flatten(pybind11::handle x); + + // Recursive helper used to implement Flatten(). + void FlattenInto(pybind11::handle handle, + std::vector& leaves); + + // Tests whether the given list is a flat list of leaves. + static bool AllLeaves(const pybind11::iterable& x); + + // Flattens a Pytree up to this PyTreeDef. 'this' must be a tree prefix of + // the tree-structure of 'x'. For example, if we flatten a value + // [(1, (2, 3)), {"foo": 4}] with a treedef [(*, *), *], the result is the + // list of leaves [1, (2, 3), {"foo": 4}]. + pybind11::list FlattenUpTo(pybind11::handle x) const; + + // Returns an unflattened PyTree given an iterable of leaves and a PyTreeDef. + pybind11::object Unflatten(pybind11::iterable leaves) const; + + // Composes two PyTreeDefs, replacing the leaves of this tree with copies of + // `inner`. + std::unique_ptr Compose(const PyTreeDef& inner) const; + + // Makes a Tuple PyTreeDef out of a vector of PyTreeDefs. + static std::unique_ptr Tuple(const std::vector& defs); + + std::vector> Children() const; + + // Maps a function over a PyTree structure, applying f_leaf to each leaf, and + // f_node to each container node. + // TODO(phawkins): use flattening everywhere instead and delete this method. + pybind11::object Walk(const pybind11::function& f_node, + pybind11::handle f_leaf, + pybind11::iterable leaves) const; + + // Given a tree of iterables with the same node/leaf structure as this PyTree, + // build the corresponding PyTree. + // TODO(phawkins): use flattening everywhere instead and delete this method. + pybind11::object FromIterableTree(pybind11::handle xs) const; + + int num_leaves() const { + if (traversal_.empty()) { + return 0; + } + return traversal_.back().num_leaves; + } + + int num_nodes() const { return traversal_.size(); } + + size_t Hash() const; + + bool operator==(const PyTreeDef& other) const; + bool operator!=(const PyTreeDef& other) const { return !(*this == other); } + + std::string ToString() const; + + private: + enum class Kind { + kLeaf, // An opaque leaf node + kNone, // None. + kTuple, // A tuple + kNamedTuple, // A collections.namedtuple + kList, // A list + kDict, // A dict + kCustom, // A custom type. + }; + + struct Node { + Kind kind = Kind::kLeaf; + + // Arity for non-kLeaf types. + int arity = 0; + + // Kind-specific auxiliary data. For a kNamedTuple, contains the tuple type + // object. For a kDict, contains a sorted list of keys. For a kCustom type, + // contains the auxiliary data returned by the `to_iterable` function. + pybind11::object node_data; + + const CustomNodeRegistry::Registration* custom = nullptr; + + // Number of leaf nodes in the subtree rooted at this node. + int num_leaves = 0; + + // Number of leaf and interior nodes in the subtree rooted at this node. + int num_nodes = 0; + }; + template + friend H AbslHashValue(H h, const Node& n); + + template + friend H AbslHashValue(H h, const PyTreeDef& t); + + // Helper that manufactures an instance of a node given its children. + static pybind11::object MakeNode(const Node& node, + absl::Span children); + + // Recursive helper used to implement FromIterableTree() + pybind11::object FromIterableTreeHelper( + pybind11::handle xs, + std::vector::const_reverse_iterator* it) const; + + // Computes the node kind of a given Python object. + static Kind GetKind(const pybind11::handle& obj, + CustomNodeRegistry::Registration const** custom); + + // Nodes, in a post-order traversal. We use an ordered traversal to minimize + // allocations, and post-order corresponds to the order we need to rebuild the + // tree structure. + std::vector traversal_; +}; + +template +H AbslHashValue(H h, const PyTreeDef::Node& n) { + h = H::combine(std::move(h), n.kind, n.arity, n.custom); + return h; +} + +template +H AbslHashValue(H h, const PyTreeDef& t) { + return H::combine_contiguous(std::move(h), t.traversal_.data(), + t.traversal_.size()); +} + +void BuildPytreeSubmodule(pybind11::module& m); + +} // namespace xla + +#endif // TENSORFLOW_COMPILER_XLA_PYTHON_PYTREE_H_ diff --git a/tensorflow/compiler/xla/python/xla.cc b/tensorflow/compiler/xla/python/xla.cc index 510175cebf6..e3bbc49f85c 100644 --- a/tensorflow/compiler/xla/python/xla.cc +++ b/tensorflow/compiler/xla/python/xla.cc @@ -49,6 +49,7 @@ limitations under the License. #include "tensorflow/compiler/xla/python/py_buffer.h" #include "tensorflow/compiler/xla/python/py_executable.h" #include "tensorflow/compiler/xla/python/python_ref_manager.h" +#include "tensorflow/compiler/xla/python/pytree.h" #include "tensorflow/compiler/xla/python/traceback.h" #include "tensorflow/compiler/xla/python/types.h" #include "tensorflow/compiler/xla/service/custom_call_target_registry.h" @@ -897,6 +898,7 @@ PYBIND11_MODULE(xla_extension, m) { BuildOpsSubmodule(&m); BuildProfilerSubmodule(&m); BuildOutfeedReceiverSubmodule(&m); + BuildPytreeSubmodule(m); py::class_> From 6bcd7a22d588d48c2faa7ea495350b0cb5f32ba0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 13 Aug 2020 08:04:02 -0700 Subject: [PATCH 0972/1017] Make FlattenInto return py::handles not py::objects. PiperOrigin-RevId: 326446701 Change-Id: I3f35a3075dda8c4aac1db9be76ac22ba1218804d --- tensorflow/compiler/xla/python/pytree.cc | 16 ++++++++++------ tensorflow/compiler/xla/python/pytree.h | 6 +++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tensorflow/compiler/xla/python/pytree.cc b/tensorflow/compiler/xla/python/pytree.cc index 401b7bc66b9..58d6a585b08 100644 --- a/tensorflow/compiler/xla/python/pytree.cc +++ b/tensorflow/compiler/xla/python/pytree.cc @@ -107,7 +107,7 @@ bool PyTreeDef::operator==(const PyTreeDef& other) const { } void PyTreeDef::FlattenInto(py::handle handle, - std::vector& leaves) { + std::vector& leaves) { Node node; int start_num_nodes = traversal_.size(); int start_num_leaves = leaves.size(); @@ -158,19 +158,23 @@ void PyTreeDef::FlattenInto(py::handle handle, } } else { assert(node.kind == Kind::kLeaf); - leaves.push_back(py::reinterpret_borrow(handle)); + leaves.push_back(handle); } node.num_nodes = traversal_.size() - start_num_nodes + 1; node.num_leaves = leaves.size() - start_num_leaves; traversal_.push_back(std::move(node)); } -/*static*/ std::pair, std::unique_ptr> -PyTreeDef::Flatten(py::handle x) { - std::vector leaves; +/*static*/ std::pair> PyTreeDef::Flatten( + py::handle x) { + std::vector leaves; auto tree = absl::make_unique(); tree->FlattenInto(x, leaves); - return std::make_pair(std::move(leaves), std::move(tree)); + py::list outputs(leaves.size()); + for (int i = 0; i < leaves.size(); ++i) { + outputs[i] = py::reinterpret_borrow(leaves[i]); + } + return std::make_pair(std::move(outputs), std::move(tree)); } /*static*/ bool PyTreeDef::AllLeaves(const py::iterable& x) { diff --git a/tensorflow/compiler/xla/python/pytree.h b/tensorflow/compiler/xla/python/pytree.h index 69cd93a7d08..76fd76fad6a 100644 --- a/tensorflow/compiler/xla/python/pytree.h +++ b/tensorflow/compiler/xla/python/pytree.h @@ -84,12 +84,12 @@ class PyTreeDef { PyTreeDef() = default; // Flattens a Pytree into a list of leaves and a PyTreeDef. - static std::pair, std::unique_ptr> - Flatten(pybind11::handle x); + static std::pair> Flatten( + pybind11::handle x); // Recursive helper used to implement Flatten(). void FlattenInto(pybind11::handle handle, - std::vector& leaves); + std::vector& leaves); // Tests whether the given list is a flat list of leaves. static bool AllLeaves(const pybind11::iterable& x); From c26a79d3c7aa91c16aa8ac409b686ba9998df2fc Mon Sep 17 00:00:00 2001 From: Saurabh Saxena Date: Thu, 13 Aug 2020 09:08:03 -0700 Subject: [PATCH 0973/1017] In gradients_test, support wrapping models where an output could be a nullptr inside a function. We use the same trick as in python's tf.function by building the FunctionDef with the non-None outputs and then interleaving the Nones. PiperOrigin-RevId: 326457686 Change-Id: I98c329b47c99282a04ed2fb24a056bab29241200 --- tensorflow/c/eager/BUILD | 1 + tensorflow/c/eager/gradients_test.cc | 49 ++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD index d542eae8238..47452c245dc 100644 --- a/tensorflow/c/eager/BUILD +++ b/tensorflow/c/eager/BUILD @@ -249,6 +249,7 @@ tf_cuda_cc_test( "//tensorflow/core:test", "//tensorflow/core:test_main", "//tensorflow/core/lib/llvm_rtti", + "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/strings", "@com_google_absl//absl/types:span", ], diff --git a/tensorflow/c/eager/gradients_test.cc b/tensorflow/c/eager/gradients_test.cc index 8392e71520f..944b10c000b 100644 --- a/tensorflow/c/eager/gradients_test.cc +++ b/tensorflow/c/eager/gradients_test.cc @@ -16,6 +16,7 @@ limitations under the License. #include +#include "absl/container/flat_hash_set.h" #include "absl/types/span.h" #include "tensorflow/c/eager/abstract_tensor_handle.h" #include "tensorflow/c/eager/c_api_experimental.h" @@ -35,6 +36,8 @@ namespace tensorflow { namespace gradients { namespace internal { namespace { +using std::vector; +using tracing::TracingOperation; class CppGradients : public ::testing::TestWithParam> { @@ -60,9 +63,9 @@ Status Add(AbstractContext* ctx, Tape* tape, forward_op.ctx = ctx; TF_RETURN_IF_ERROR( Reset(add_op.get(), "Add", /*raw_device_name=*/nullptr, &forward_op)); - if (isa(add_op.get())) { + if (isa(add_op.get())) { TF_RETURN_IF_ERROR( - dyn_cast(add_op.get())->SetOpName("my_add")); + dyn_cast(add_op.get())->SetOpName("my_add")); } TF_RETURN_IF_ERROR(AddInput(add_op.get(), inputs[0], &forward_op)); TF_RETURN_IF_ERROR(AddInput(add_op.get(), inputs[1], &forward_op)); @@ -81,9 +84,9 @@ Status Exp(AbstractContext* ctx, Tape* tape, forward_op.ctx = ctx; TF_RETURN_IF_ERROR( Reset(exp_op.get(), "Exp", /*raw_device_name=*/nullptr, &forward_op)); - if (isa(exp_op.get())) { + if (isa(exp_op.get())) { TF_RETURN_IF_ERROR( - dyn_cast(exp_op.get())->SetOpName("my_exp")); + dyn_cast(exp_op.get())->SetOpName("my_exp")); } TF_RETURN_IF_ERROR(AddInput(exp_op.get(), inputs[0], &forward_op)); int num_retvals = 1; @@ -183,21 +186,36 @@ Status RunModel(Model model, AbstractContext* ctx, if (use_function) { const char* fn_name = "test_fn"; std::unique_ptr scoped_func; + // Returning null tensors from a tf.function is not supported, so we keep + // track of indices in the model's outputs are nullptr in this set. + // The FunctionDef only outputs the non-null tensors. We later pad the + // function op outputs to have nullptrs at the `null_indices`. + absl::flat_hash_set null_indices; { AbstractContextPtr func_ctx(BuildFunction(fn_name)); std::vector func_inputs; func_inputs.reserve(inputs.size()); TF_RETURN_IF_ERROR( CreateParamsForInputs(func_ctx.get(), inputs, &func_inputs)); - OutputList output_list; - output_list.expected_num_outputs = outputs.size(); - output_list.outputs.resize(outputs.size()); + vector model_outputs; + model_outputs.resize(outputs.size()); TF_RETURN_IF_ERROR(model(func_ctx.get(), absl::MakeSpan(func_inputs), - absl::MakeSpan(output_list.outputs), registry)); + absl::MakeSpan(model_outputs), registry)); for (auto func_input : func_inputs) { func_input->Unref(); } AbstractFunction* func = nullptr; + OutputList output_list; + output_list.expected_num_outputs = 0; + output_list.outputs.reserve(outputs.size()); + for (int i = 0; i < model_outputs.size(); i++) { + if (model_outputs[i]) { + output_list.outputs.emplace_back(model_outputs[i]); + output_list.expected_num_outputs += 1; + } else { + null_indices.insert(i); + } + } TF_RETURN_IF_ERROR(dyn_cast(func_ctx.get()) ->Finalize(&output_list, &func)); scoped_func.reset(func); @@ -212,8 +230,19 @@ Status RunModel(Model model, AbstractContext* ctx, for (auto input : inputs) { TF_RETURN_IF_ERROR(fn_op->AddInput(input)); } - int retvals = outputs.size(); - TF_RETURN_IF_ERROR(fn_op->Execute(outputs, &retvals)); + int retvals = outputs.size() - null_indices.size(); + vector fn_outputs(retvals); + TF_RETURN_IF_ERROR(fn_op->Execute( + absl::Span(fn_outputs.data(), fn_outputs.size()), + &retvals)); + int skipped_indices = 0; + for (int i = 0; i < outputs.size(); i++) { + if (!null_indices.contains(i)) { + outputs[i] = fn_outputs[i - skipped_indices]; + } else { + skipped_indices += 1; + } + } TF_RETURN_IF_ERROR(ctx->RemoveFunction(fn_name)); return Status::OK(); } else { From 357e80f364164da202354c67b764a425d3b97bbb Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 13 Aug 2020 09:08:33 -0700 Subject: [PATCH 0974/1017] Add "yes | configure" step to java-docs script. PiperOrigin-RevId: 326457785 Change-Id: I4e28fd30343ef386d3bfc748c70337a60c21143f --- tensorflow/tools/docs/build_java_api_docs.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow/tools/docs/build_java_api_docs.py b/tensorflow/tools/docs/build_java_api_docs.py index 343a561d225..31383f07fad 100644 --- a/tensorflow/tools/docs/build_java_api_docs.py +++ b/tensorflow/tools/docs/build_java_api_docs.py @@ -62,6 +62,13 @@ def main(unused_argv): shutil.copytree(SOURCE_PATH, merged_source / 'java') if FLAGS.gen_ops: + # `$ yes | configure` + yes = subprocess.Popen(['yes', ''], stdout=subprocess.PIPE) + configure = subprocess.Popen([TENSORFLOW_ROOT / 'configure'], + stdin=yes.stdout, + cwd=TENSORFLOW_ROOT) + configure.communicate() + subprocess.check_call( ['bazel', 'build', '//tensorflow/java:java_op_gen_sources'], cwd=TENSORFLOW_ROOT) From 46905310139597c18805e529d5c8fdfa78a275e9 Mon Sep 17 00:00:00 2001 From: Allen Lavoie Date: Thu, 13 Aug 2020 09:11:19 -0700 Subject: [PATCH 0975/1017] Parallel device: make saving compatible with non-parallel variables Allows matching based on the first N devices, with optional assertions indicating that not everything was restored. Fixes SavedModel loading; you can save the parallel version and load it either parallel or non-parallel. Still saves all of the buffers. I'll follow up making that optional. PiperOrigin-RevId: 326458344 Change-Id: Ibc020e96121701cb75d681d10b7d365102c699c2 --- .../parallel_device/parallel_device_test.py | 43 +++++++++++++++++++ .../distribute/parallel_device/saving.py | 41 +++++++++++++++--- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/tensorflow/python/distribute/parallel_device/parallel_device_test.py b/tensorflow/python/distribute/parallel_device/parallel_device_test.py index e660086adef..571a8ae4b6b 100644 --- a/tensorflow/python/distribute/parallel_device/parallel_device_test.py +++ b/tensorflow/python/distribute/parallel_device/parallel_device_test.py @@ -34,6 +34,8 @@ from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import math_ops from tensorflow.python.ops import variables from tensorflow.python.platform import test +from tensorflow.python.saved_model import load +from tensorflow.python.saved_model import save from tensorflow.python.training import checkpoint_management from tensorflow.python.training.tracking import util as tracking from tensorflow.python.util import nest @@ -221,6 +223,47 @@ class ParallelDeviceTests(_VirtualDeviceTestCase): outputs = self.device.unpack(v) self.assertAllClose([-1., 3.], outputs) + with self.device: + restore_on_create = tracking.Checkpoint() + restore_on_create.restore(save_path) + restore_on_create.v = variables.Variable(0.) + outputs = self.device.unpack(restore_on_create.v) + self.assertAllClose([-1., 3.], outputs) + + # Changing the number of devices / restoring into a single-device copy is OK + single_device = tracking.Checkpoint(v=variables.Variable(0.)) + status = single_device.restore(save_path) + status.assert_existing_objects_matched() + self.assertAllClose(-1., single_device.v) + with self.assertRaisesRegex(AssertionError, "parallel_component_1"): + # There are parts of the variable that aren't restored into a + # single-device copy. + status.assert_consumed() + + def test_saved_model(self): + with self.device: + different_values = self.device.pack( + [constant_op.constant(-1.), + constant_op.constant(3.)]) + m = module.Module() + m.v = variables.Variable(different_values) + m.f = def_function.function(lambda: m.v * 2.) + self.assertAllClose([-2., 6.], self.device.unpack(m.f())) + saved_model_path = os.path.join(self.get_temp_dir(), "saved_model") + save.save(m, saved_model_path) + + context._reset_context() + self.setUp() + + single_device_loaded = load.load(saved_model_path) + self.assertAllClose(-2., single_device_loaded.f()) + with self.device: + parallel_loaded = load.load(saved_model_path) + self.assertAllClose([-2., 6.], self.device.unpack(parallel_loaded.f())) + self.assertAllClose([-1., 3.], self.device.unpack(parallel_loaded.v)) + parallel_loaded.v.assign(self.device.pack([.1, .2])) + self.assertAllClose([.2, .4], self.device.unpack(parallel_loaded.f())) + def _assert_close_to_non_parallel(self, computation): """Asserts that replication of `computation` works and is equivalent.""" with self.device: diff --git a/tensorflow/python/distribute/parallel_device/saving.py b/tensorflow/python/distribute/parallel_device/saving.py index ddc0aa51578..f1539e49651 100644 --- a/tensorflow/python/distribute/parallel_device/saving.py +++ b/tensorflow/python/distribute/parallel_device/saving.py @@ -47,21 +47,31 @@ class _ParallelComponentSaveable(saveable_object.SaveableObject): resource=self._handle, value=restored_tensor) -class ParallelVariable(resource_variable_ops.ResourceVariable): - """Overrides checkpointing behavior to save each component separately.""" +class ParallelSavingMixin(resource_variable_ops.BaseResourceVariable): + """Mixin to to override variable checkpointing, saving each component.""" - def __init__(self, parallel_device, **kwargs): + def __init__(self, parallel_device, expected_shape=None, use_resource=None, + **kwargs): + del expected_shape, use_resource self._parallel_device = parallel_device - super(ParallelVariable, self).__init__(**kwargs) + super(ParallelSavingMixin, self).__init__(**kwargs) # TODO(allenl): Consider either adding a boolean argument for # save-primary-only or looking at synchronization/aggregation properties. def _gather_saveables_for_checkpoint(self): + """Generate SaveableObjects for each component device.""" component_saveables = {} # Create one SaveableObject per device, each one of which looks like a # regular ResourceVariable saveable. for index, handle in enumerate(self._parallel_device.unpack(self.handle)): - component_saveables["parallel_component_{}".format(index)] = ( + if index == 0: + # This is the name regular tf.Variables use to save. Using it for the + # component on the first device means non-parallel tf.Variable objects + # will use this value when pointed at a parallel checkpoint. + attribute = "VARIABLE_VALUE" + else: + attribute = "parallel_component_{}".format(index) + component_saveables[attribute] = ( functools.partial( _ParallelComponentSaveable, handle=handle, @@ -70,9 +80,26 @@ class ParallelVariable(resource_variable_ops.ResourceVariable): return component_saveables -def _variable_creator(next_creator, parallel_device, **kwargs): +class ParallelVariable( + ParallelSavingMixin, resource_variable_ops.ResourceVariable): + pass + + +class UninitializedParallelVariable( + ParallelSavingMixin, resource_variable_ops.UninitializedVariable): + pass + + +def _variable_creator(next_creator, parallel_device, initial_value=None, + **kwargs): del next_creator - return ParallelVariable(parallel_device=parallel_device, **kwargs) + if initial_value is not None: + return ParallelVariable( + parallel_device=parallel_device, initial_value=initial_value, **kwargs) + else: + # SavedModel loading does not pass an initial value. + return UninitializedParallelVariable( + parallel_device=parallel_device, **kwargs) @contextlib.contextmanager From bab410482e102bd84b7c0ff95637ecb5888de706 Mon Sep 17 00:00:00 2001 From: Scott Zhu Date: Thu, 13 Aug 2020 09:19:51 -0700 Subject: [PATCH 0976/1017] Remove the tags that has typo in their name. PiperOrigin-RevId: 326460020 Change-Id: Id135f471d29f00e647691cf5fdb26f3cc785f78c --- tensorflow/python/compiler/tensorrt/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow/python/compiler/tensorrt/BUILD b/tensorflow/python/compiler/tensorrt/BUILD index 2b26dd42818..a674feeb5a3 100644 --- a/tensorflow/python/compiler/tensorrt/BUILD +++ b/tensorflow/python/compiler/tensorrt/BUILD @@ -172,7 +172,6 @@ cuda_py_test( "no_oss", # TODO(b/125290478): allow running in at least some OSS configurations. "no_pip", "no_rocm", - "no_tap", # It is not able to download the mnist data. "no_windows", "nomac", ], From 0ca913e56e29ebc6f7227871f98ec8bc936939f3 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 13 Aug 2020 09:51:27 -0700 Subject: [PATCH 0977/1017] Return a matrix of all NaNs from tf.linalg.expm if the input contains non-finite values. PiperOrigin-RevId: 326466302 Change-Id: Icdf895e4515e64a38a65a78587135e3575419413 --- .../matrix_exponential_op_test.py | 8 +++----- tensorflow/python/ops/linalg/linalg_impl.py | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py index 4744d88c34d..61e2610e595 100644 --- a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py @@ -24,7 +24,6 @@ import numpy as np from tensorflow.python.client import session from tensorflow.python.framework import constant_op -from tensorflow.python.framework import errors_impl from tensorflow.python.framework import ops from tensorflow.python.framework import test_util from tensorflow.python.ops import array_ops @@ -140,10 +139,9 @@ class ExponentialOpTest(test.TestCase): def testInfinite(self): # Check that the op does not loop forever on infinite inputs. (b/158433036) - in_tensor = np.random.rand(100, 100).astype(np.float) - in_tensor[0][0] = np.inf - with self.assertRaises(errors_impl.InvalidArgumentError): - self.evaluate(linalg_impl.matrix_exponential(in_tensor)) + in_tensor = [[np.inf, 1.], [1., 1.]] + result = self.evaluate(linalg_impl.matrix_exponential(in_tensor)) + self.assertTrue(np.all(np.isnan(result))) def testEmpty(self): self._verifyExponentialReal(np.empty([0, 2, 2])) diff --git a/tensorflow/python/ops/linalg/linalg_impl.py b/tensorflow/python/ops/linalg/linalg_impl.py index cdef22695e9..8035a9901e6 100644 --- a/tensorflow/python/ops/linalg/linalg_impl.py +++ b/tensorflow/python/ops/linalg/linalg_impl.py @@ -31,7 +31,6 @@ from tensorflow.python.ops import gen_linalg_ops from tensorflow.python.ops import linalg_ops from tensorflow.python.ops import map_fn from tensorflow.python.ops import math_ops -from tensorflow.python.ops import numerics from tensorflow.python.ops import special_math_ops from tensorflow.python.util import dispatch from tensorflow.python.util.tf_export import tf_export @@ -277,8 +276,6 @@ def matrix_exponential(input, name=None): # pylint: disable=redefined-builtin math_ops.abs(matrix), axis=array_ops.size(array_ops.shape(matrix)) - 2), axis=-1)[..., array_ops.newaxis, array_ops.newaxis] - l1_norm = numerics.verify_tensor_all_finite( - l1_norm, 'l1 norm of matrix is Inf or NaN.') const = lambda x: constant_op.constant(x, l1_norm.dtype) @@ -324,13 +321,19 @@ def matrix_exponential(input, name=None): # pylint: disable=redefined-builtin else: raise ValueError('tf.linalg.expm does not support matrices of type %s' % matrix.dtype) - numer = u + v - denom = -u + v - result = linalg_ops.matrix_solve(denom, numer) - max_squarings = math_ops.reduce_max(squarings) + is_finite = math_ops.is_finite(math_ops.reduce_max(l1_norm)) + nan = constant_op.constant(np.nan, matrix.dtype) + result = control_flow_ops.cond( + is_finite, lambda: linalg_ops.matrix_solve(-u + v, u + v), + lambda: array_ops.fill(array_ops.shape(matrix), nan)) + max_squarings = math_ops.reduce_max(squarings) i = const(0.0) - c = lambda i, r: math_ops.less(i, max_squarings) + + def c(i, _): + return control_flow_ops.cond(is_finite, + lambda: math_ops.less(i, max_squarings), + lambda: constant_op.constant(False)) def b(i, r): return i + 1, array_ops.where_v2( From dc3f225de9b5d969ccb60871926f37dd7aaf7863 Mon Sep 17 00:00:00 2001 From: Ken Franko Date: Thu, 13 Aug 2020 09:56:40 -0700 Subject: [PATCH 0978/1017] Auto generate TensorFlow WriteSummary ops. Add summary and descriptions for these ops in api definition. PiperOrigin-RevId: 326467435 Change-Id: I8037543a925fd7ba254041aa270f45afc4553dc6 --- .../mlir/tensorflow/ir/tf_generated_ops.td | 134 ++++++++++++++++++ .../base_api/api_def_WriteAudioSummary.pbtxt | 5 + .../base_api/api_def_WriteGraphSummary.pbtxt | 4 + .../api_def_WriteHistogramSummary.pbtxt | 4 + .../base_api/api_def_WriteImageSummary.pbtxt | 5 + .../api_def_WriteRawProtoSummary.pbtxt | 4 + .../base_api/api_def_WriteScalarSummary.pbtxt | 4 + .../base_api/api_def_WriteSummary.pbtxt | 4 + 8 files changed, 164 insertions(+) diff --git a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td index cd8df635cba..cc07d50eee2 100644 --- a/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td +++ b/tensorflow/compiler/mlir/tensorflow/ir/tf_generated_ops.td @@ -11517,6 +11517,140 @@ where(input) ==> [[0, 0, 0], TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<0>; } +def TF_WriteAudioSummaryOp : TF_Op<"WriteAudioSummary", []> { + let summary = "Writes an audio summary."; + + let description = [{ +Writes encoded audio summary `tensor` at `step` with `tag` using summary `writer`. +`sample_rate` is the audio sample rate is Hz. + }]; + + let arguments = (ins + TF_ResourceTensor:$writer, + I64Tensor:$step, + TF_StrTensor:$tag, + F32Tensor:$tensor, + F32Tensor:$sample_rate, + + Confined, [IntMinValue<1>]>:$max_outputs + ); + + let results = (outs); +} + +def TF_WriteGraphSummaryOp : TF_Op<"WriteGraphSummary", []> { + let summary = "Writes a graph summary."; + + let description = [{ +Writes TensorFlow graph `tensor` at `step` using summary `writer`. + }]; + + let arguments = (ins + TF_ResourceTensor:$writer, + I64Tensor:$step, + TF_StrTensor:$tensor + ); + + let results = (outs); +} + +def TF_WriteHistogramSummaryOp : TF_Op<"WriteHistogramSummary", []> { + let summary = "Writes a histogram summary."; + + let description = [{ +Writes histogram `values` at `step` with `tag` using summary `writer`. + }]; + + let arguments = (ins + TF_ResourceTensor:$writer, + I64Tensor:$step, + TF_StrTensor:$tag, + TF_IntOrFpTensor:$values + ); + + let results = (outs); + + TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<3>; +} + +def TF_WriteImageSummaryOp : TF_Op<"WriteImageSummary", []> { + let summary = "Writes an image summary."; + + let description = [{ +Writes image `tensor` at `step` with `tag` using summary `writer`. +`tensor` is image with shape [height, width, channels]. + }]; + + let arguments = (ins + TF_ResourceTensor:$writer, + I64Tensor:$step, + TF_StrTensor:$tag, + TensorOf<[F16, F32, TF_Uint8]>:$tensor, + TF_Uint8Tensor:$bad_color, + + Confined, [IntMinValue<1>]>:$max_images + ); + + let results = (outs); + + TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<3>; +} + +def TF_WriteRawProtoSummaryOp : TF_Op<"WriteRawProtoSummary", []> { + let summary = "Writes a serialized proto summary."; + + let description = [{ +Writes `tensor`, a serialized proto at `step` using summary `writer`. + }]; + + let arguments = (ins + TF_ResourceTensor:$writer, + I64Tensor:$step, + TF_StrTensor:$tensor + ); + + let results = (outs); +} + +def TF_WriteScalarSummaryOp : TF_Op<"WriteScalarSummary", []> { + let summary = "Writes a scalar summary."; + + let description = [{ +Writes scalar `value` at `step` with `tag` using summary `writer`. + }]; + + let arguments = (ins + TF_ResourceTensor:$writer, + I64Tensor:$step, + TF_StrTensor:$tag, + TF_IntOrFpTensor:$value + ); + + let results = (outs); + + TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<3>; +} + +def TF_WriteSummaryOp : TF_Op<"WriteSummary", []> { + let summary = "Writes a tensor summary."; + + let description = [{ +Writes `tensor` at `step` with `tag` using summary `writer`. + }]; + + let arguments = (ins + TF_ResourceTensor:$writer, + I64Tensor:$step, + TF_Tensor:$tensor, + TF_StrTensor:$tag, + TF_StrTensor:$summary_metadata + ); + + let results = (outs); + + TF_DerivedOperandTypeAttr T = TF_DerivedOperandTypeAttr<2>; +} + def TF_XdivyOp : TF_Op<"Xdivy", [NoSideEffect, ResultsBroadcastableShape, TF_SameOperandsAndResultElementTypeResolveRef]>, WithBroadcastableBinOpBuilder { let summary = "Returns 0 if x == 0, and x / y otherwise, elementwise."; diff --git a/tensorflow/core/api_def/base_api/api_def_WriteAudioSummary.pbtxt b/tensorflow/core/api_def/base_api/api_def_WriteAudioSummary.pbtxt index 520952cd411..e7e0a95cabc 100644 --- a/tensorflow/core/api_def/base_api/api_def_WriteAudioSummary.pbtxt +++ b/tensorflow/core/api_def/base_api/api_def_WriteAudioSummary.pbtxt @@ -1,4 +1,9 @@ op { graph_op_name: "WriteAudioSummary" visibility: HIDDEN + summary: "Writes an audio summary." + description: <