Update pylint to the same version as used internally.

Will update to newer version later, after testing that this works and only if needed. Might be rolled back if continuous builds fail after merging.

Added a lot of errors to allow-list, will organize fix-it to remove them.

Removes the `W9015:missing-param-doc` warning type added in #45427 as TF is not ready for that yet.

Part of the work for #46046, #43040. Unblocks #46075

PiperOrigin-RevId: 353741692
Change-Id: I542e3f28253afada12b595863e571c058f7228b9

tensorflow/tools/ci_build/ci_sanity.sh

@@ -123,7 +123,7 @@ do_pylint() {
   fi
 
   # Now that we know we have to do work, check if `pylint` is installed
-  PYLINT_BIN="python3 -m pylint"
+  PYLINT_BIN="python3.8 -m pylint"
 
   echo ""
   echo "check whether pylint is available or not."
@@ -188,8 +188,7 @@ do_pylint() {
   # C0326 bad-whitespace
   # W0611 unused-import
   # W0622 redefined-builtin
-  # W9015 missing-param-doc
+  grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622)' ${OUTPUT_FILE} > ${ERRORS_FILE}
-  grep -E '(\[E|\[W0311|\[W0312|\[C0330|\[C0301|\[C0326|\[W0611|\[W0622|\[W9015)' ${OUTPUT_FILE} > ${ERRORS_FILE}
 
   # Split the pylint reported errors into permitted ones and those we want to
   # block submit on until fixed.
@@ -224,6 +223,7 @@ do_pylint() {
     cat ${FORBID_FILE}
   fi
 
+  echo ""
   if [[ ${N_FORBID_ERRORS} != 0 ]]; then
     echo "FAIL: Found ${N_FORBID_ERRORS} non-allowlisted errors and ${N_PERMIT_ERRORS} allowlisted errors"
     return 1

tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh

@@ -25,11 +25,10 @@ set +u
 set -x
 
 function install_pylint () {
-  # TODO(gunan): figure out why we get stuck with later versions of pylint.
   # TODO(mihaimaruseac): this is used in the release build in the same way,
   # maybe extract out to a common?
-  sudo python3 -m pip install setuptools --upgrade
+  sudo python3.8 -m pip install setuptools --upgrade
-  sudo python3 -m pip install pylint==1.6.4
+  sudo python3.8 -m pip install pylint==2.4.4
 }
 
 function run_sanity_checks () {

tensorflow/tools/ci_build/pylint_allowlist

@@ -1,44 +1,144 @@
+^tensorflow/compiler/tests/binary_ops_test.py.*\[E1121.*too-many-function-args
+^tensorflow/compiler/tests/image_ops_test.py.*\[E1121.*too-many-function-args
 ^tensorflow/contrib/eager/python/evaluator\.py.*\[E0202.*method-hidden
 ^tensorflow/contrib/eager/python/metrics_impl\.py.*\[E0202.*method-hidden
 ^tensorflow/contrib/layers/python/layers/feature_column\.py.*\[E0110.*abstract-class-instantiated
 ^tensorflow/contrib/rate/rate\.py.*\[E0202.*method-hidden
+^tensorflow/lite/python/interpreter_test.py.*\[E1111.*assignment-from-no-return
 ^tensorflow/python/autograph/.*_py3_test\.py.*\[E0001.*syntax-error
+^tensorflow/python/autograph/converters/directives_test.py.*\[E1123.*unexpected-keyword-arg
 ^tensorflow/python/autograph/impl/api_test.py.*\[E0202.*method-hidden
+^tensorflow/python/autograph/pyct/static_analysis/activity_test.py.*\[W0611.*unused-import
+^tensorflow/python/client/session.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/compiler/tensorrt/test/base_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/biasadd_matmul_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/binary_tensor_weight_broadcast_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/dynamic_input_shapes_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/int32_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/testdata/gen_tftrt_model.py.*\[E1102.*not-callable
+^tensorflow/python/compiler/tensorrt/test/vgg_block_nchw_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/compiler/tensorrt/test/vgg_block_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/data/experimental/kernel_tests/data_service_ops_ft_test.py.*\[W0622.*redefined-builtin
+^tensorflow/python/data/experimental/kernel_tests/parse_example_dataset_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/data/experimental/ops/optimization.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/data/kernel_tests/multi_device_iterator_test.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/debug/lib/grpc_debug_server.py.*\[W0622.*redefined-builtin
+^tensorflow/python/distribute/combinations_test.py.*\[E1124.*redundant-keyword-arg
 ^tensorflow/python/distribute/coordinator/cluster_coordinator.py.*\[E0702.*raising-bad-type
 ^tensorflow/python/distribute/parallel_device/saving.py.*\[E1120.*no-value-for-parameter
 ^tensorflow/python/distribute/parallel_device/saving.py.*\[E1123.*unexpected-keyword-arg
+^tensorflow/python/distribute/strategy_test_lib.py.*\[E1111.*assignment-from-no-return
 ^tensorflow/python/eager/function_test.py.*\[C0326.*bad-whitespace
+^tensorflow/python/eager/pywrap_tfe_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/eager/tensor_test.py.*\[E1120.*no-value-for-parameter
 ^tensorflow/python/feature_column/feature_column_test\.py.*\[E0110.*abstract-class-instantiated
+^tensorflow/python/framework/dtypes_test.py.*\[W0611.*unused-import
 ^tensorflow/python/framework/function_test\.py.*\[E1123.*noinline
+^tensorflow/python/framework/ops_test.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/framework/test_util.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/framework/type_spec_test.py.*\[E1003.*bad-super-call
+^tensorflow/python/grappler/tf_optimizer_test.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/keras/callbacks\.py.*\[E1133.*not-an-iterable
+^tensorflow/python/keras/engine/base_layer.py.*\[E0202.*method-hidden
 ^tensorflow/python/keras/engine/base_layer.py.*\[E0203.*access-member-before-definition
+^tensorflow/python/keras/engine/base_layer.py.*\[E1003.*bad-super-call
 ^tensorflow/python/keras/engine/base_layer.py.*\[E1102.*not-callable
+^tensorflow/python/keras/engine/base_layer_v1.py.*\[E0202.*method-hidden
+^tensorflow/python/keras/engine/base_layer_v1.py.*\[E1003.*bad-super-call
+^tensorflow/python/keras/engine/base_layer_v1.py.*\[E1102.*not-callable
 ^tensorflow/python/keras/engine/base_preprocessing_layer.py.*\[E0202.*method-hidden
 ^tensorflow/python/keras/engine/base_preprocessing_layer_test.py.*\[E0202.*method-hidden
+^tensorflow/python/keras/layers/convolutional_transpose_test.py.*\[E1121.*too-many-function-args
 ^tensorflow/python/keras/layers/preprocessing/.*\[E1102.*not-callable
 ^tensorflow/python/keras/layers/recurrent\.py.*\[E0203.*access-member-before-definition
+^tensorflow/python/keras/legacy_tf_layers/base.py.*\[E0202.*method-hidden
+^tensorflow/python/keras/legacy_tf_layers/base.py.*\[E1003.*bad-super-call
+^tensorflow/python/keras/legacy_tf_layers/base.py.*\[E1102.*not-callable
 ^tensorflow/python/keras/legacy_tf_layers/base\.py.*\[E0203.*access-member-before-definition
+^tensorflow/python/keras/metrics_test.py.*\[E1111.*assignment-from-no-return
+^tensorflow/python/keras/mixed_precision/autocast_variable.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/keras/optimizer_v2/nadam.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/keras/preprocessing/image\.py.*\[E0240.*Inconsistent method resolution
 ^tensorflow/python/keras/saving/saved_model/json_utils.py.*\[E0202.*method-hidden
+^tensorflow/python/keras/utils/conv_utils_test.py.*\[E1133.*not-an-iterable
 ^tensorflow/python/keras/utils/data_utils.py.*\[E1102.*not-callable
+^tensorflow/python/keras/utils/generic_utils_test.py.*\[E0102.*function-redefined
 ^tensorflow/python/keras/utils/version_utils_test.py.*\[E0110.*abstract-class-instantiated
+^tensorflow/python/kernel_tests/basic_gpu_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/bias_op_base.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/clip_ops_test.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/kernel_tests/constant_op_eager_test.py.*\[E0303.*invalid-length-returned
+^tensorflow/python/kernel_tests/cwise_ops_binary_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/cwise_ops_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/distributions/special_math_test.py.*\[E1133.*not-an-iterable
+^tensorflow/python/kernel_tests/distributions/student_t_test.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/kernel_tests/losses_test.py.*\[E0633.*unpacking-non-sequence
+^tensorflow/python/kernel_tests/map_stage_op_test.py.*\[W0611.*unused-import
+^tensorflow/python/kernel_tests/matrix_solve_ls_op_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/parse_single_example_op_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/parsing_ops_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/partitioned_variables_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/kernel_tests/unicode_encode_op_test.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/kernel_tests/unicode_encode_op_test.py.*\[E1123.*unexpected-keyword-arg
+^tensorflow/python/lib/core/bfloat16_test.py.*\[E1121.*too-many-function-args
+^tensorflow/python/modules_with_exports.py.*\[W0622.*redefined-builtin
+^tensorflow/python/ops/control_flow_grad.py.*\[W0622.*redefined-builtin
+^tensorflow/python/ops/distributions/bernoulli.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/beta.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/categorical.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/dirichlet.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/dirichlet_multinomial.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/multinomial.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/special_math.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/distributions/util.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/image_ops_test.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/linalg/linear_operator.py.*\[E0202.*method-hidden
+^tensorflow/python/ops/linalg/linear_operator.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg/linear_operator_block_diag.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg/linear_operator_block_lower_triangular.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg/linear_operator_circulant.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg/linear_operator_householder.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/linalg/linear_operator_identity.py.*\[E1102.*not-callable
+^tensorflow/python/ops/linalg_grad.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/math_grad.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/math_ops.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/nn_grad.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/nn_impl.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/nn_ops.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/nn_test.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/numpy_ops/__init__.py.*\[W0622.*redefined-builtin
+^tensorflow/python/ops/numpy_ops/np_array_ops_test.py.*\[E1120.*no-value-for-parameter
 ^tensorflow/python/ops/parallel_for/control_flow_ops_test.py.*\[C0330.*bad-continuation
+^tensorflow/python/ops/ragged/ragged_getitem_test.py.*\[E1111.*assignment-from-no-return
+^tensorflow/python/ops/ragged/ragged_operators_test.py.*\[E1130.*invalid-unary-operand-type
 ^tensorflow/python/ops/resource_variable_ops.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/ops/signal/fft_ops.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/signal/spectral_ops.py.*\[E1130.*invalid-unary-operand-type
+^tensorflow/python/ops/standard_ops.py.*\[W0622.*redefined-builtin
 ^tensorflow/python/platform/default/_gfile\.py.*\[E0301.*non-iterator
 ^tensorflow/python/platform/default/_googletest\.py.*\[E0102.*function\salready\sdefined
 ^tensorflow/python/platform/gfile\.py.*\[E0301.*non-iterator
+^tensorflow/python/platform/googletest.py.*\[E0001.*syntax-error
+^tensorflow/python/profiler/profile_context.py.*\[E1111.*assignment-from-no-return
+^tensorflow/python/pywrap_tensorflow.py.*\[W0622.*redefined-builtin
+^tensorflow/python/saved_model/model_utils/mode_keys_test.py.*\[E1137.*unsupported-assignment-operation
 ^tensorflow/python/tpu/async_checkpoint.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/bfloat16.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/datasets.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/device_assignment.py.*\[C0326.*bad-whitespace
+^tensorflow/python/tpu/feature_column.py.*\[E1124.*redundant-keyword-arg
+^tensorflow/python/tpu/feature_column_v2.py.*\[E1124.*redundant-keyword-arg
 ^tensorflow/python/tpu/tpu.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/tpu_embedding.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/tpu_embedding_v2.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/tpu_embedding_v2_utils.py.*\[C0326.*bad-whitespace
 ^tensorflow/python/tpu/training_loop.py.*\[C0326.*bad-whitespace
+^tensorflow/python/training/tracking/data_structures.py.*\[E0402.*relative-beyond-top-level
+^tensorflow/python/training/tracking/data_structures_test.py.*\[E1102.*not-callable
+^tensorflow/python/training/tracking/data_structures_test.py.*\[E1120.*no-value-for-parameter
+^tensorflow/python/training/tracking/data_structures_test.py.*\[E1138.*unsupported-delete-operation
 ^tensorflow/python/training/tracking/tracking\.py.*\[E0202.*method-hidden
+^tensorflow/python/util/function_utils_test.py.*\[E1120.*no-value-for-parameter
 ^tensorflow/python/util/tf_stack_test.py.*\[E1121.*too-many-function-args
 ^tensorflow/security/fuzzing/raggedCountSparseOutput_fuzz.py.*\[C0330.*bad-continuation
+^tensorflow/tools/pip_package/setup.py.*\[E1111.*assignment-from-no-return

tensorflow/tools/ci_build/rel/ubuntu/sanity.sh

@@ -21,16 +21,12 @@ install_bazelisk
 which bazel
 
 # We need py3 lint
-sudo pip3 install pep8
+sudo python3.8 -m pip install pep8
 
-# TODO(gunan): figure out why we get stuck with later versions of pylint.
+# Install pylint
-# Install pylint.
+sudo python3.8 -m pip install setuptools --upgrade
-sudo python3 -m pip install setuptools --upgrade
+sudo python3.8 -m pip install pylint==2.4.4
-sudo python2 -m pip install pylint==1.6.4
+python3.8 -m pylint --version
-sudo python3 -m pip install pylint==1.6.4
-
-# TODO(yifeif): print pylint version for debug. remove later.
-python3 -m pylint --version
 
 # Run tensorflow sanity checks.
 tensorflow/tools/ci_build/ci_sanity.sh

tensorflow/tools/ci_build/release/ubuntu_16/sanity/build.sh

@@ -21,16 +21,12 @@ install_bazelisk
 which bazel
 
 # We need py3 lint
-sudo pip3 install pep8
+sudo python3.8 -m pip install pep8
 
-# TODO(gunan): figure out why we get stuck with later versions of pylint.
+# Install pylint
-# Install pylint.
+sudo python3.8 -m pip install setuptools --upgrade
-sudo python3 -m pip install setuptools --upgrade
+sudo python3.8 -m pip install pylint==2.4.4
-sudo python2 -m pip install pylint==1.6.4
+python3.8 -m pylint --version
-sudo python3 -m pip install pylint==1.6.4
-
-# TODO(yifeif): print pylint version for debug. remove later.
-python3 -m pylint --version
 
 # Run tensorflow sanity checks.
 tensorflow/tools/ci_build/ci_sanity.sh