Source code for prospector.tools.pylint

import os
import re
import sys
from collections import defaultdict
from collections.abc import Iterable
from pathlib import Path
from typing import TYPE_CHECKING, Any, Optional, Union

from pylint.config import find_default_config_files
from pylint.exceptions import UnknownMessageError
from pylint.lint.run import _cpu_count

from prospector.finder import FileFinder
from prospector.message import Location, Message
from prospector.tools.base import ToolBase
from prospector.tools.pylint.collector import Collector
from prospector.tools.pylint.linter import ProspectorLinter

if TYPE_CHECKING:
    from prospector.config import ProspectorConfig

_UNUSED_WILDCARD_IMPORT_RE = re.compile(r"^Unused import(\(s\))? (.*) from wildcard import")

_IGNORE_RE = re.compile(r"#\s*pylint:\s*disable=([^#]*[^#\s])(\s*#.*)?$", re.IGNORECASE)
_IGNORE_NEXT_RE = re.compile(r"#\s*pylint:\s*disable-next=([^#]*[^#\s])(\s*#.*)?$", re.IGNORECASE)


def _is_in_dir(subpath: Path, path: Path) -> bool:
    return subpath.parent == path


[docs] class PylintTool(ToolBase): # There are several methods on this class which could technically # be functions (they don't use the 'self' argument) but that would # make this module/class a bit ugly. def __init__(self) -> None: self._args: Any = None self._collector: Optional[Collector] = None self._linter: Optional[ProspectorLinter] = None self._orig_sys_path: list[str] = [] def _prospector_configure(self, prospector_config: "ProspectorConfig", linter: ProspectorLinter) -> list[Message]: errors = [] if "django" in prospector_config.libraries: linter.load_plugin_modules(["pylint_django"]) if "celery" in prospector_config.libraries: linter.load_plugin_modules(["pylint_celery"]) profile_path = os.path.join(prospector_config.workdir, prospector_config.profile.name) for plugin in prospector_config.profile.pylint.get("load-plugins", []): # type: ignore[attr-defined] try: linter.load_plugin_modules([plugin]) except ImportError: errors.append(self._error_message(profile_path, f"Could not load plugin {plugin}")) for msg_id in prospector_config.get_disabled_messages("pylint"): try: # noqa: SIM105 linter.disable(msg_id) except UnknownMessageError: # If the msg_id doesn't exist in PyLint any more, # don't worry about it. pass options = prospector_config.tool_options("pylint") for checker in linter.get_checkers(): if not hasattr(checker, "options"): continue for option in checker.options: if option[0] in options: checker._arguments_manager.set_option( # pylint: disable=protected-access option[0], options[option[0]] ) # The warnings about disabling warnings are useful for figuring out # with other tools to suppress messages from. For example, an unused # import which is disabled with 'pylint disable=unused-import' will # still generate an 'FL0001' unused import warning from pyflakes. # Using the information from these messages, we can figure out what # was disabled. linter.disable("locally-disabled") # notification about disabling a message linter.enable("file-ignored") # notification about disabling an entire file linter.enable("suppressed-message") # notification about a message being suppressed linter.disable("deprecated-pragma") # notification about use of deprecated 'pragma' option max_line_length = prospector_config.max_line_length for checker in linter.get_checkers(): if not hasattr(checker, "options"): continue for option in checker.options: if max_line_length is not None and option[0] == "max-line-length": checker._arguments_manager.set_option( # pylint: disable=protected-access "max-line-length", max_line_length ) return errors def _error_message(self, filepath: Union[str, Path], message: str) -> Message: location = Location(filepath, None, None, 0, 0) return Message("prospector", "config-problem", location, message) def _pylintrc_configure(self, pylintrc: Union[str, Path], linter: ProspectorLinter) -> list[Message]: errors = [] are_plugins_loaded = linter.config_from_file(pylintrc) if not are_plugins_loaded and hasattr(linter.config, "load_plugins"): for plugin in linter.config.load_plugins: try: linter.load_plugin_modules([plugin]) except ImportError: errors.append(self._error_message(pylintrc, f"Could not load plugin {plugin}")) return errors
[docs] def configure( self, prospector_config: "ProspectorConfig", found_files: FileFinder ) -> Optional[tuple[Optional[Union[str, Path]], Optional[Iterable[Message]]]]: extra_sys_path = found_files.make_syspath() check_paths = self._get_pylint_check_paths(found_files) pylint_options = prospector_config.tool_options("pylint") self._set_path_finder(extra_sys_path, pylint_options) linter = ProspectorLinter(found_files) config_messages, configured_by = self._get_pylint_configuration( check_paths, linter, prospector_config, pylint_options ) # we don't want similarity reports right now linter.disable("similarities") # use the collector 'reporter' to simply gather the messages # given by PyLint self._collector = Collector(linter.msgs_store) linter.set_reporter(self._collector) if linter.config.jobs == 0: linter.config.jobs = _cpu_count() self._linter = linter return configured_by, config_messages
def _set_path_finder(self, extra_sys_path: list[Path], pylint_options: dict[str, Any]) -> None: # insert the target path into the system path to get correct behaviour self._orig_sys_path = sys.path if not pylint_options.get("use_pylint_default_path_finder"): sys.path = sys.path + [str(path.absolute()) for path in extra_sys_path] def _get_pylint_check_paths(self, found_files: FileFinder) -> list[Path]: # create a list of packages, but don't include packages which are # subpackages of others as checks will be duplicated check_paths = set() modules = found_files.python_modules packages = found_files.python_packages packages.sort(key=lambda p: len(str(p))) # don't add modules that are in known packages for module in modules: for package in packages: if _is_in_dir(module, package): break else: check_paths.add(module) # sort from earlier packages first... for idx, package in enumerate(packages): # yuck o(n2) but... temporary for prev_pkg in packages[:idx]: if _is_in_dir(package, prev_pkg): # this is a sub-package of a package we know about break else: # we should care about this one check_paths.add(package) # need to sort to make sure multiple runs are deterministic return sorted(check_paths) def _get_pylint_configuration( self, check_paths: list[Path], linter: ProspectorLinter, prospector_config: "ProspectorConfig", pylint_options: dict[str, Any], ) -> tuple[list[Message], Optional[Union[Path, str]]]: self._args = check_paths linter.load_default_plugins() config_messages: list[Message] = self._prospector_configure(prospector_config, linter) configured_by: Optional[Union[str, Path]] = None if prospector_config.use_external_config("pylint"): # Try to find a .pylintrc pylintrc: Optional[Union[str, Path]] = pylint_options.get("config_file") external_config = prospector_config.external_config_location("pylint") pylintrc = pylintrc or external_config if pylintrc is None: for p in find_default_config_files(): pylintrc = str(p) break if pylintrc is None: # nothing explicitly configured for possible in (".pylintrc", "pylintrc", "pyproject.toml", "setup.cfg"): pylintrc_path = os.path.join(prospector_config.workdir, possible) # TODO: pyproject and setup.cfg might not actually have any pylint config # in, they should be skipped in that case if os.path.exists(pylintrc_path): pylintrc = pylintrc_path break if pylintrc is not None: # load it! configured_by = pylintrc config_messages += self._pylintrc_configure(pylintrc, linter) return config_messages, configured_by def _combine_w0614(self, messages: list[Message]) -> list[Message]: """ For the "unused import from wildcard import" messages, we want to combine all warnings about the same line into a single message. """ by_loc = defaultdict(list) out = [] for message in messages: if message.code == "unused-wildcard-import": by_loc[message.location].append(message) else: out.append(message) for location, message_list in by_loc.items(): names = [] for msg in message_list: match_ = _UNUSED_WILDCARD_IMPORT_RE.match(msg.message) assert match_ is not None names.append(match_.group(1)) msgtxt = "Unused imports from wildcard import: {}".format(", ".join(names)) combined_message = Message("pylint", "unused-wildcard-import", location, msgtxt) out.append(combined_message) return out
[docs] def combine(self, messages: list[Message]) -> list[Message]: """ Combine repeated messages. Some error messages are repeated, causing many errors where only one is strictly necessary. For example, having a wildcard import will result in one 'Unused Import' warning for every unused import. This method will combine these into a single warning. """ combined = self._combine_w0614(messages) return sorted(combined)
[docs] def run(self, found_files: FileFinder) -> list[Message]: assert self._collector is not None assert self._linter is not None self._linter.check(self._args) sys.path = self._orig_sys_path messages = self._collector.get_messages() return self.combine(messages)
[docs] def get_ignored_codes(self, line: str) -> list[tuple[str, int]]: match = _IGNORE_RE.search(line) if match: return [(e.strip(), 0) for e in match.group(1).split(",")] match = _IGNORE_NEXT_RE.search(line) if match: return [(e.strip(), 1) for e in match.group(1).split(",")] return []