From 9729442a35b73ffa0a221dc2d4b7e89e5f135700 Mon Sep 17 00:00:00 2001
From: Mihai Maruseac <mihaimaruseac@google.com>
Date: Mon, 25 Jan 2021 15:18:22 -0800
Subject: [PATCH] Update pylint to the same version as used internally.

Will update to newer version later, after testing that this works and only if needed. Might be rolled back if continuous builds fail after merging.

Added a lot of errors to allow-list, will organize fix-it to remove them.

Removes the `W9015:missing-param-doc` warning type added in #45427 as TF is not ready for that yet.

Part of the work for #46046, #43040. Unblocks #46075

PiperOrigin-RevId: 353741692
Change-Id: I542e3f28253afada12b595863e571c058f7228b9
---
 tensorflow/tools/ci_build/ci_sanity.sh        |   6 +-
 .../presubmit/ubuntu_16/sanity/build.sh       |   5 +-
 tensorflow/tools/ci_build/pylint_allowlist    | 100 ++++++++++++++++++
 .../tools/ci_build/rel/ubuntu/sanity.sh       |  14 +--
 .../release/ubuntu_16/sanity/build.sh         |  14 +--
 5 files changed, 115 insertions(+), 24 deletions(-)

diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index 6e021cc6be3..280ba2051ef 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -123,7 +123,7 @@ do_pylint() {
   fi
 
   # Now that we know we have to do work, check if `pylint` is installed
-  PYLINT_BIN="python3 -m pylint"
+  PYLINT_BIN="python3.8 -m pylint"
 
   echo ""
   echo "check whether pylint is available or not."
@@ -188,8 +188,7 @@ do_pylint() {
   # C0326 bad-whitespace
   # W0611 unused-import
   # W0622 redefined-builtin
-  # W9015 missing-param-doc
-  grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622|\[W9015)' ${OUTPUT_FILE} > ${ERRORS_FILE}
+  grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE}
 
   # Split the pylint reported errors into permitted ones and those we want to
   # block submit on until fixed.
@@ -224,6 +223,7 @@ do_pylint() {
     cat ${FORBID_FILE}
   fi
 
+  echo ""
   if [[ ${N_FORBID_ERRORS} != 0 ]]; then
     echo "FAIL: Found ${N_FORBID_ERRORS} non-allowlisted errors and ${N_PERMIT_ERRORS} allowlisted errors"
     return 1
diff --git a/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh b/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh
index bd2f71abc68..cdc6bc644c4 100644
--- a/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh
+++ b/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh
@@ -25,11 +25,10 @@ set +u
 set -x
 
 function install_pylint () {
-  # TODO(gunan): figure out why we get stuck with later versions of pylint.
   # TODO(mihaimaruseac): this is used in the release build in the same way,
   # maybe extract out to a common?
-  sudo python3 -m pip install setuptools --upgrade
-  sudo python3 -m pip install pylint==1.6.4
+  sudo python3.8 -m pip install setuptools --upgrade
+  sudo python3.8 -m pip install pylint==2.4.4
 }
 
 function run_sanity_checks () {
diff --git a/tensorflow/tools/ci_build/pylint_allowlist b/tensorflow/tools/ci_build/pylint_allowlist
index a7b13352694..27385c020ac 100644
--- a/tensorflow/tools/ci_build/pylint_allowlist
+++ b/tensorflow/tools/ci_build/pylint_allowlist
@@ -1,44 +1,144 @@
+^tensorflow/compiler/tests/binary_ops_test.py.*\[E1121.*too-many-function-args
+^tensorflow/compiler/tests/image_ops_test.py.*\[E1121.*too-many-function-args
 ^tensorflow/contrib/eager/python/evaluator\.py.*\[E0202.*method-hidden
 ^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden
 ^tensorflow/contrib/layers/python/layers/feature_column\.py.*\[E0110.*abstract-class-instantiated
 ^tensorflow/contrib/rate/rate\.py.*\[E0202.*method-hidden
+^tensorflow/lite/python/interpreter_test.py.*\[E1111.*assignment-from-no-return
 ^tensorflow/python/autograph/.*_py3_test\.py.*\[E0001.*syntax-error
+^tensorflow/python/autograph/converters/directives_test.py.*\[E1123.*unexpected-keyword-arg
 ^tensorflow/python/autograph/impl/api_test.py.*\[E0202.*method-hidden
+^tensorflow/python/autograph/pyct/static_analysis/activity_test.py.*\[W0611.*unused-import
+^tensorflow/python/client/session.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/compiler/tensorrt/test/base_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/biasadd_matmul_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/binary_tensor_weight_broadcast_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/dynamic_input_shapes_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/int32_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/testdata/gen_tftrt_model.py.*\[E1102.*not-callable
+^tensorflow/python/compiler/tensorrt/test/vgg_block_nchw_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/vgg_block_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/data/experimental/kernel_tests/data_service_ops_ft_test.py.*\[W0622.*redefined-builtin
+^tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/data/experimental/ops/optimization.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/data/kernel_tests/multi_device_iterator_test.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/debug/lib/grpc_debug_server.py.*\[W0622.*redefined-builtin
+^tensorflow/python/distribute/combinations_test.py.*\[E1124.*redundant-keyword-arg
 ^tensorflow/python/distribute/coordinator/cluster_coordinator.py.*\[E0702.*raising-bad-type
 ^tensorflow/python/distribute/parallel_device/saving.py.*\[E1120.*no-value-for-parameter
 ^tensorflow/python/distribute/parallel_device/saving.py.*\[E1123.*unexpected-keyword-arg
+^tensorflow/python/distribute/strategy_test_lib.py.*\[E1111.*assignment-from-no-return
 ^tensorflow/python/eager/function_test.py.*\[C0326.*bad-whitespace
+^tensorflow/python/eager/pywrap_tfe_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/eager/tensor_test.py.*\[E1120.*no-value-for-parameter
 ^tensorflow/python/feature_column/feature_column_test\.py.*\[E0110.*abstract-class-instantiated
+^tensorflow/python/framework/dtypes_test.py.*\[W0611.*unused-import
 ^tensorflow/python/framework/function_test\.py.*\[E1123.*noinline
+^tensorflow/python/framework/ops_test.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/framework/test_util.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/framework/type_spec_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/grappler/tf_optimizer_test.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/keras/callbacks\.py.*\[E1133.*not-an-iterable
+^tensorflow/python/keras/engine/base_layer.py.*\[E0202.*method-hidden
 ^tensorflow/python/keras/engine/base_layer.py.*\[E0203.*access-member-before-definition
+^tensorflow/python/keras/engine/base_layer.py.*\[E1003.*bad-super-call
 ^tensorflow/python/keras/engine/base_layer.py.*\[E1102.*not-callable
+^tensorflow/python/keras/engine/base_layer_v1.py.*\[E0202.*method-hidden
+^tensorflow/python/keras/engine/base_layer_v1.py.*\[E1003.*bad-super-call
+^tensorflow/python/keras/engine/base_layer_v1.py.*\[E1102.*not-callable
 ^tensorflow/python/keras/engine/base_preprocessing_layer.py.*\[E0202.*method-hidden
 ^tensorflow/python/keras/engine/base_preprocessing_layer_test.py.*\[E0202.*method-hidden
+^tensorflow/python/keras/layers/convolutional_transpose_test.py.*\[E1121.*too-many-function-args
 ^tensorflow/python/keras/layers/preprocessing/.*\[E1102.*not-callable
 ^tensorflow/python/keras/layers/recurrent\.py.*\[E0203.*access-member-before-definition
+^tensorflow/python/keras/legacy_tf_layers/base.py.*\[E0202.*method-hidden
+^tensorflow/python/keras/legacy_tf_layers/base.py.*\[E1003.*bad-super-call
+^tensorflow/python/keras/legacy_tf_layers/base.py.*\[E1102.*not-callable
 ^tensorflow/python/keras/legacy_tf_layers/base\.py.*\[E0203.*access-member-before-definition
+^tensorflow/python/keras/metrics_test.py.*\[E1111.*assignment-from-no-return
+^tensorflow/python/keras/mixed_precision/autocast_variable.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/keras/optimizer_v2/nadam.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/keras/preprocessing/image\.py.*\[E0240.*Inconsistent method resolution
 ^tensorflow/python/keras/saving/saved_model/json_utils.py.*\[E0202.*method-hidden
+^tensorflow/python/keras/utils/conv_utils_test.py.*\[E1133.*not-an-iterable
 ^tensorflow/python/keras/utils/data_utils.py.*\[E1102.*not-callable
+^tensorflow/python/keras/utils/generic_utils_test.py.*\[E0102.*function-redefined
 ^tensorflow/python/keras/utils/version_utils_test.py.*\[E0110.*abstract-class-instantiated
+^tensorflow/python/kernel_tests/basic_gpu_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/bias_op_base.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/clip_ops_test.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/kernel_tests/constant_op_eager_test.py.*\[E0303.*invalid-length-returned
+^tensorflow/python/kernel_tests/cwise_ops_binary_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/cwise_ops_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/distributions/special_math_test.py.*\[E1133.*not-an-iterable
+^tensorflow/python/kernel_tests/distributions/student_t_test.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/kernel_tests/losses_test.py.*\[E0633.*unpacking-non-sequence
+^tensorflow/python/kernel_tests/map_stage_op_test.py.*\[W0611.*unused-import
+^tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/parse_single_example_op_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/parsing_ops_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/partitioned_variables_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/unicode_encode_op_test.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/kernel_tests/unicode_encode_op_test.py.*\[E1123.*unexpected-keyword-arg
+^tensorflow/python/lib/core/bfloat16_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/modules_with_exports.py.*\[W0622.*redefined-builtin
+^tensorflow/python/ops/control_flow_grad.py.*\[W0622.*redefined-builtin
+^tensorflow/python/ops/distributions/bernoulli.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/beta.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/categorical.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/dirichlet.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/dirichlet_multinomial.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/multinomial.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/special_math.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/util.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/image_ops_test.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/linalg/linear_operator.py.*\[E0202.*method-hidden
+^tensorflow/python/ops/linalg/linear_operator.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg/linear_operator_block_diag.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg/linear_operator_block_lower_triangular.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg/linear_operator_circulant.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg/linear_operator_householder.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/linalg/linear_operator_identity.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg_grad.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/math_grad.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/math_ops.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/nn_grad.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/nn_impl.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/nn_ops.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/nn_test.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/numpy_ops/__init__.py.*\[W0622.*redefined-builtin
+^tensorflow/python/ops/numpy_ops/np_array_ops_test.py.*\[E1120.*no-value-for-parameter
 ^tensorflow/python/ops/parallel_for/control_flow_ops_test.py.*\[C0330.*bad-continuation
+^tensorflow/python/ops/ragged/ragged_getitem_test.py.*\[E1111.*assignment-from-no-return
+^tensorflow/python/ops/ragged/ragged_operators_test.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/ops/resource_variable_ops.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/ops/signal/fft_ops.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/signal/spectral_ops.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/standard_ops.py.*\[W0622.*redefined-builtin
 ^tensorflow/python/platform/default/_gfile\.py.*\[E0301.*non-iterator
 ^tensorflow/python/platform/default/_googletest\.py.*\[E0102.*function\salready\sdefined
 ^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator
+^tensorflow/python/platform/googletest.py.*\[E0001.*syntax-error
+^tensorflow/python/profiler/profile_context.py.*\[E1111.*assignment-from-no-return
+^tensorflow/python/pywrap_tensorflow.py.*\[W0622.*redefined-builtin
+^tensorflow/python/saved_model/model_utils/mode_keys_test.py.*\[E1137.*unsupported-assignment-operation
 ^tensorflow/python/tpu/async_checkpoint.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/bfloat16.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/datasets.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/device_assignment.py.*\[C0326.*bad-whitespace
+^tensorflow/python/tpu/feature_column.py.*\[E1124.*redundant-keyword-arg
+^tensorflow/python/tpu/feature_column_v2.py.*\[E1124.*redundant-keyword-arg
 ^tensorflow/python/tpu/tpu.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/tpu_embedding.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/tpu_embedding_v2.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/tpu_embedding_v2_utils.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/training_loop.py.*\[C0326.*bad-whitespace
+^tensorflow/python/training/tracking/data_structures.py.*\[E0402.*relative-beyond-top-level
+^tensorflow/python/training/tracking/data_structures_test.py.*\[E1102.*not-callable
+^tensorflow/python/training/tracking/data_structures_test.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/training/tracking/data_structures_test.py.*\[E1138.*unsupported-delete-operation
 ^tensorflow/python/training/tracking/tracking\.py.*\[E0202.*method-hidden
+^tensorflow/python/util/function_utils_test.py.*\[E1120.*no-value-for-parameter
 ^tensorflow/python/util/tf_stack_test.py.*\[E1121.*too-many-function-args
 ^tensorflow/security/fuzzing/raggedCountSparseOutput_fuzz.py.*\[C0330.*bad-continuation
+^tensorflow/tools/pip_package/setup.py.*\[E1111.*assignment-from-no-return
diff --git a/tensorflow/tools/ci_build/rel/ubuntu/sanity.sh b/tensorflow/tools/ci_build/rel/ubuntu/sanity.sh
index 4fc600de867..0dcd90ec827 100644
--- a/tensorflow/tools/ci_build/rel/ubuntu/sanity.sh
+++ b/tensorflow/tools/ci_build/rel/ubuntu/sanity.sh
@@ -21,16 +21,12 @@ install_bazelisk
 which bazel
 
 # We need py3 lint
-sudo pip3 install pep8
+sudo python3.8 -m pip install pep8
 
-# TODO(gunan): figure out why we get stuck with later versions of pylint.
-# Install pylint.
-sudo python3 -m pip install setuptools --upgrade
-sudo python2 -m pip install pylint==1.6.4
-sudo python3 -m pip install pylint==1.6.4
-
-# TODO(yifeif): print pylint version for debug. remove later.
-python3 -m pylint --version
+# Install pylint
+sudo python3.8 -m pip install setuptools --upgrade
+sudo python3.8 -m pip install pylint==2.4.4
+python3.8 -m pylint --version
 
 # Run tensorflow sanity checks.
 tensorflow/tools/ci_build/ci_sanity.sh
diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/sanity/build.sh b/tensorflow/tools/ci_build/release/ubuntu_16/sanity/build.sh
index 4fc600de867..0dcd90ec827 100644
--- a/tensorflow/tools/ci_build/release/ubuntu_16/sanity/build.sh
+++ b/tensorflow/tools/ci_build/release/ubuntu_16/sanity/build.sh
@@ -21,16 +21,12 @@ install_bazelisk
 which bazel
 
 # We need py3 lint
-sudo pip3 install pep8
+sudo python3.8 -m pip install pep8
 
-# TODO(gunan): figure out why we get stuck with later versions of pylint.
-# Install pylint.
-sudo python3 -m pip install setuptools --upgrade
-sudo python2 -m pip install pylint==1.6.4
-sudo python3 -m pip install pylint==1.6.4
-
-# TODO(yifeif): print pylint version for debug. remove later.
-python3 -m pylint --version
+# Install pylint
+sudo python3.8 -m pip install setuptools --upgrade
+sudo python3.8 -m pip install pylint==2.4.4
+python3.8 -m pylint --version
 
 # Run tensorflow sanity checks.
 tensorflow/tools/ci_build/ci_sanity.sh