tests/TestDeepScan.py
Rama master · réplica de lectura del repositorio original.
# SPDX-License-Identifier: GPL-3.0-or-later
# Copyright (c) 2008-2026 Andrew Ziem.
#
# This work is licensed under the terms of the GNU GPL, version 3 or
# later. See the COPYING file in the top-level directory.
"""
Test case for module DeepScan
"""
# standard imports
import itertools
import os
import shutil
import subprocess
import unicodedata
import unittest
from unittest import mock
from tests.common import pytest
# first party imports
from tests import common
from tests import TestCleaner
from tests.common import SPECIAL_TEST_STRINGS
from bleachbit import IS_MAC, IS_WINDOWS, FS_CASE_SENSITIVE
from bleachbit.Options import options
from bleachbit.DeepScan import DeepScan, Search, normalized_walk
from bleachbit.FileUtilities import openfiles, whitelisted
if IS_WINDOWS:
# pylint: disable-next=ungrouped-imports
from tests.TestWindows import WindowsLinksMixIn
else:
WindowsLinksMixIn = object
class DeepScanTestCase(common.BleachbitTestCase, WindowsLinksMixIn):
"""Test Case for module DeepScan"""
def _test_encoding(self, fn):
"""Test encoding"""
fullpath = self.write_file(fn + ".bak")
# Add Unicode paths to encourage a crash.
subdir = os.path.join(self.tempdir, 'ɡælɪk.dir')
os.mkdir(subdir)
subfile = os.path.join(subdir, 'ɡælɪk.file')
common.touch_file(subfile)
searches = {self.tempdir: []}
searches[self.tempdir].append(
Search(command='delete', regex=r'\.[Bb][Aa][Kk]$'))
ds = DeepScan(searches)
found = False
for cmd in ds.scan():
if cmd is True:
# True is used to yield to GTK+, but it is not
# needed in this test.
continue
self.assertExists(cmd.path)
if cmd.path == fullpath:
found = True
self.assertTrue(found, "Did not find '%s'" % fullpath)
os.unlink(fullpath)
self.assertNotExists(fullpath)
shutil.rmtree(subdir)
@pytest.mark.no_xdist
def test_encoding(self):
"""Test encoding"""
for test in SPECIAL_TEST_STRINGS:
self._test_encoding(test)
def test_DeepScan(self):
"""Unit test for class DeepScan on the kinds of files a home has"""
# Not the real home: on CI it holds tool caches and the folders
# other workers write to, so its size and contents vary
root = self.mkdtemp()
# Other tests scan the whole class tempdir.
self.addCleanup(shutil.rmtree, root)
sub = os.path.join(root, 'sub')
deeper = os.path.join(sub, 'ɡælɪk.dir')
os.makedirs(deeper)
expected = {
self.write_file(os.path.join(root, 'Makefile')),
self.write_file(os.path.join(root, 'notes.txt~')),
self.write_file(os.path.join(root, 'backup.bak')),
self.write_file(os.path.join(sub, 'Thumbs.db')),
self.write_file(os.path.join(deeper, 'ɡælɪk.bak')),
}
for near_miss in ('Makefile.am', '~notes.txt', 'bak.txt'):
self.write_file(os.path.join(root, near_miss))
self.write_file(os.path.join(sub, 'Thumbs.db.old'))
if not IS_WINDOWS:
# Windows does not allow a colon in a file name
expected.add(self.write_file(
os.path.join(sub, 'Thumbs.db:encryptable')))
# Not followed, so the files under sub are reported once
os.symlink(sub, os.path.join(root, 'link_to_sub'))
# A dangling link still matches
broken = os.path.join(root, 'broken.bak')
os.symlink(os.path.join(root, 'missing'), broken)
expected.add(broken)
if os.geteuid() != 0:
# A folder that cannot be listed is skipped
locked = os.path.join(root, 'locked')
os.mkdir(locked)
self.write_file(os.path.join(locked, 'hidden.bak'))
os.chmod(locked, 0)
self.addCleanup(os.chmod, locked, 0o700)
searches = {root: [
Search(command='delete', regex=regex)
for regex in ('^Makefile$', '~$', 'bak$', '^Thumbs.db$',
'^Thumbs.db:encryptable$')]}
found = []
idle_yields = 0
with mock.patch('bleachbit.DeepScan.time') as mock_time:
# A clock that jumps on every read makes each folder look slow,
# so the scan also hands control to the GTK idle loop
mock_time.time.side_effect = itertools.count()
for cmd in DeepScan(searches).scan():
if cmd is True:
idle_yields += 1
continue
self.assertLExists(cmd.path)
found.append(cmd.path)
self.assertCountEqual(expected, found)
self.assertGreater(idle_yields, 0)
def test_compiled_search_path_prefix(self):
"""CompiledSearch.match should preserve root path semantics."""
from bleachbit.DeepScan import CompiledSearch, directory_prefix
compiled = CompiledSearch(
Search(command='delete', regex=r'\.txt$')
)
for dirpath in (self.tempdir, os.path.abspath(os.sep)):
with self.subTest(dirpath=dirpath):
path_prefix = directory_prefix(dirpath)
self.assertEqual(
compiled.match(dirpath, 'foo.txt', path_prefix),
os.path.join(dirpath, 'foo.txt'),
)
def test_directory_prefix_unc_share_root(self):
"""Bare UNC share roots must insert a separator before the filename."""
import ntpath
from bleachbit import DeepScan as deepscan_mod
from bleachbit.DeepScan import CompiledSearch
with mock.patch.object(deepscan_mod.os, 'path', ntpath):
unc_root = r'\\server\share'
prefix = deepscan_mod.directory_prefix(unc_root)
# join(unc_root, '') alone omits the separator; concatenation
# would otherwise produce \\server\sharefoo.txt.
self.assertNotEqual(prefix, unc_root)
self.assertTrue(prefix.endswith('\\'))
self.assertEqual(
prefix + 'foo.txt',
ntpath.join(unc_root, 'foo.txt'),
)
# Drive-relative paths must not gain a separator.
self.assertEqual(deepscan_mod.directory_prefix('C:'), 'C:')
self.assertEqual(
deepscan_mod.directory_prefix('C:') + 'foo.txt',
ntpath.join('C:', 'foo.txt'),
)
compiled = CompiledSearch(
Search(command='delete', regex=r'\.txt$')
)
self.assertEqual(
compiled.match(unc_root, 'foo.txt', prefix),
ntpath.join(unc_root, 'foo.txt'),
)
def test_skip_whitelisted_directory(self):
"""DeepScan should not search whitelisted directories."""
# Reminder: each test case gets a new options directory,
# so we do not need to restore the old keep list (whitelist).
keep_dir = self.mkdir('keep')
search_dir = self.mkdir('search')
keep_file = self.write_file(os.path.join(keep_dir, 'skip.bbtestbak'))
search_file = self.write_file(
os.path.join(search_dir, 'find.bbtestbak'))
options.set_whitelist_paths([('folder', keep_dir)])
searches = {
self.tempdir: [
Search(command='delete', regex=r'\.bbtestbak$')
]
}
paths = [
cmd.path
for cmd in DeepScan(searches).scan()
if cmd is not True
]
self.assertNotIn(keep_file, paths)
self.assertIn(search_file, paths)
@common.skipIfWindows
def test_scan_skip_open(self):
"""skip_open leaves files that a process holds open"""
root = self.mkdtemp()
open_fn = self.write_file(os.path.join(root, 'open.bbtestswp'), b'x')
closed_fn = self.write_file(
os.path.join(root, 'closed.bbtestswp'), b'x')
searches = {root: [
Search(command='delete', regex=r'\.bbtestswp$', skip_open=True)]}
# Open for reading so FreeBSD reports the path, as in test_open_files
with open(open_fn, 'rb'):
openfiles.scan()
paths = [cmd.path for cmd in DeepScan(
searches).scan() if cmd is not True]
self.assertEqual(paths, [closed_fn])
@common.skipIfWindows
def test_scan_skip_open_other_search(self):
"""An open file stays when a search without skip_open matches it too"""
top = self.mkdtemp()
# Other tests scan the whole class tempdir
self.addCleanup(shutil.rmtree, top)
subdir = self.mkdir(os.path.join(top, 'sub'))
open_fn = self.write_file(os.path.join(subdir, 'open.bbtestswp'), b'x')
closed_fn = self.write_file(
os.path.join(subdir, 'closed.bbtestswp'), b'x')
plain = Search(command='delete', regex=r'\.bbtestswp$')
guard = Search(command='delete', regex=r'\.bbtestswp$', skip_open=True)
cases = (
{top: [plain, guard]},
{top: [guard, plain]},
{subdir: [plain], top: [guard]},
{top: [guard], subdir: [plain]},
)
with open(open_fn, 'rb'):
openfiles.scan()
for searches in cases:
with self.subTest(searches=searches):
paths = [cmd.path for cmd in DeepScan(
searches).scan() if cmd is not True]
self.assertEqual(paths, [closed_fn])
@common.skipIfWindows
def test_scan_skip_open_unlisted(self):
"""skip_open keeps files when open files cannot be listed"""
root = self.mkdtemp()
self.write_file(os.path.join(root, 'a.bbtestswp'), b'x')
# A second match catches a failed listing cached as empty
self.write_file(os.path.join(root, 'b.bbtestswp'), b'x')
bak_fn = self.write_file(os.path.join(root, 'c.bbtestbak'), b'x')
searches = {root: [
Search(command='delete', regex=r'\.bbtestswp$', skip_open=True),
Search(command='delete', regex=r'\.bbtestbak$')]}
for exc in (FileNotFoundError('lsof'),
subprocess.CalledProcessError(1, 'lsof'),
UnicodeDecodeError('utf-8', b'\xff', 0, 1, 'invalid')):
with self.subTest(exc=exc), \
mock.patch('bleachbit.FileUtilities.open_files', side_effect=exc), \
mock.patch.object(openfiles, 'last_scan_time', None):
paths = [cmd.path for cmd in DeepScan(
searches).scan() if cmd is not True]
self.assertEqual(paths, [bak_fn])
@common.skipIfWindows
def test_keep_list_decomposed_name(self):
"""The macOS deep scan honors a kept file with a decomposed name"""
search_dir = self.mkdtemp()
# Other tests scan the whole class tempdir
self.addCleanup(shutil.rmtree, search_dir)
keep_file = self.write_file(os.path.join(
search_dir, unicodedata.normalize('NFD', 'r\u00e9sum\u00e9.bbtestbak')))
options.set_whitelist_paths([('file', keep_file)])
searches = {
search_dir: [
Search(command='delete', regex=r'\.bbtestbak$')
]
}
with mock.patch('bleachbit.DeepScan.IS_MAC', True), \
mock.patch('bleachbit.FileUtilities.IS_MAC', True):
paths = [
cmd.path
for cmd in DeepScan(searches).scan()
if cmd is not True
]
self.assertEqual(1, len(paths))
self.assertTrue(whitelisted(paths[0]))
def test_scan_yields_each_file_once(self):
"""Overlapping searches and nested tops yield a file only once"""
top = self.mkdtemp()
# Other tests scan the whole class tempdir
self.addCleanup(shutil.rmtree, top)
subdir = self.mkdir(os.path.join(top, 'sub'))
filename = self.write_file(os.path.join(subdir, 'foo.bbtestbak'))
searches = {
top: [
Search(command='delete', regex=r'\.bbtestbak$'),
Search(command='delete', regex=r'^foo\.'),
],
subdir: [Search(command='delete', regex=r'\.bbtestbak$')],
}
paths = [
cmd.path
for cmd in DeepScan(searches).scan()
if cmd is not True
]
self.assertEqual(paths, [filename])
@common.skipUnlessWindows
def test_scan_does_not_follow_junction(self):
"""DeepScan must not descend into a junction"""
scan_root = self.mkdir('scanroot')
victim = self.mkdir('victim')
self.write_file(os.path.join(victim, 'target.bbtestbak'))
self._create_win_junction(victim, os.path.join(scan_root, 'link'))
searches = {scan_root: [
Search(command='delete', regex=r'\.bbtestbak$')]}
paths = [cmd.path for cmd in DeepScan(
searches).scan() if cmd is not True]
self.assertEqual(paths, [], 'DeepScan followed a junction: %s' % paths)
def _get_compiled_search(self, wholeregex_substr):
"""Return the CompiledSearch for the deepscan action whose
wholeregex contains the given substring (e.g. 'node_modules',
'venv')."""
import xml.etree.ElementTree as ET
from bleachbit.DeepScan import CompiledSearch
xml_path = os.path.join(
os.path.dirname(__file__), '..', 'cleaners', 'deepscan.xml')
root = ET.parse(xml_path).getroot()
wholeregex = None
nwholeregex = None
for action in root.iter('action'):
if action.get('wholeregex') and wholeregex_substr in action.get('wholeregex'):
wholeregex = action.get('wholeregex')
nwholeregex = action.get('nwholeregex')
break
self.assertIsNotNone(wholeregex)
return CompiledSearch(Search(command='delete', wholeregex=wholeregex, nwholeregex=nwholeregex))
def test_angular_matches_only_cache_dir(self):
""".angular option should match the cache folder but not config files."""
cs = self._get_compiled_search('angular')
matched = [
('/home/u/proj/.angular/cache/19.0.0/babel-webpack', 'x.json'),
(r'C:\Users\u\proj\.angular\cache', 'x.json'),
]
excluded = [
('/home/u', '.angular-config.json'),
('/home/u/.config/angular', '.angular-config.json'),
('/home/u/proj', '.angular.json'),
('/home/u/proj', '.angular-cli.json'),
(r'C:\Users\u', '.angular-config.json'),
]
for dirpath, filename in matched:
with self.subTest(dirpath=dirpath, filename=filename):
self.assertIsNotNone(
cs.match(dirpath, filename),
f"Should match {dirpath}/{filename}")
for dirpath, filename in excluded:
with self.subTest(dirpath=dirpath, filename=filename):
self.assertIsNone(
cs.match(dirpath, filename),
f"Should exclude {dirpath}/{filename}")
def test_node_modules_excludes_dot_dirs(self):
"""node_modules option should exclude system/cache/editor directories."""
cs = self._get_compiled_search('node_modules')
excluded = [
('/home/neo/.nvm/versions/node/v22.22.3/lib/node_modules/corepack', 'README.md'),
('/home/trinity/.vscode/extensions/ms-python.python-2026.4.0-linux-x64/out/client/node_modules', 'sudo-prompt.js'),
('/home/morpheus/.npm/_npx/15b07286cbcc3329/node_modules',
'.package-lock.json'),
('/home/smith/.cache/typescript/5.9/node_modules', '.package-lock.json'),
# root's home when running as root, on Linux and macOS
('/root/.nvm/versions/node/v20.11.0/lib/node_modules/npm', 'index.js'),
('/root/.vscode-server/bin/abc/node_modules/foo', 'index.js'),
('/var/root/.nvm/versions/node/v20.11.0/lib/node_modules/npm', 'index.js'),
(r'C:\Users\glados\.windsurf\extensions\ms-python.python-2026.4.0-universal\out\client\node_modules', 'unicode.js'),
(r'C:\Users\glados\.cursor\projects\empty-window\canvases\node_modules', 'cursor.js'),
(r'C:\Users\glados\AppData\Local\Programs\myapp\node_modules', 'file.js'),
(r'C:\Users\glados\AppData\Local\ApertureScience\MainframeCore\node_modules\central-processing', 'index.js'),
(r'C:\Users\Alice\AppData\Local\Programs\BeeperTexts\resources\app.asar.unpacked\node_modules\@beeper',
'beeper_client_sdk_napi_binding.node'),
(r'C:\Users\bob\AppData\Local\Programs\Microsoft VS Code\7e7950df89\resources\app',
'node_modules.asar'),
(r'C:\Users\mallory\AppData\Local\Programs\Microsoft VS Code\fcf604774b\resources\app',
'node_modules.asar'),
('/home/trent/projects/important/node_modules_backup', 'README.md'),
('/home/trent/projects/important', 'node_modules.asar'),
(r'C:\Users\charlie\Documents\important', 'node_modules.asar'),
(r'C:\Users\charlie\Documents\important\node_modules_backup', 'aes.js'),
]
for dirpath, filename in excluded:
with self.subTest(dirpath=dirpath, filename=filename):
self.assertIsNone(
cs.match(dirpath, filename),
f"Should exclude {dirpath}/{filename}")
def test_node_modules_matches_ordinary_dirs(self):
"""node_modules option should match ordinary user project directories."""
cs = self._get_compiled_search('node_modules')
matched = [
('/home/sfalken/games/wopr/node_modules/joshua/ai', 'core.js'),
(r'C:\Users\esnowden\Documents\nsa\prism\node_modules\crypto-js', 'aes.js'),
(r'C:\Users\chunkylover53\Documents\work-from-home\node_modules\click-button-automator\node_modules\is-buffer', 'index.js'),
('/home/chunkylover53/Downloads/donut-radar/node_modules/react-dom', 'bar.js'),
('/root/projects/app/node_modules/react', 'index.js'),
]
for dirpath, filename in matched:
with self.subTest(dirpath=dirpath, filename=filename):
self.assertIsNotNone(
cs.match(dirpath, filename),
f"Should match {dirpath}/{filename}")
def test_venv_excludes_system_and_editor_dirs(self):
"""venv option should exclude system/cache/editor directories and non-venv paths."""
cs = self._get_compiled_search('venv')
excluded = [
# venv cannot be directly under home
('/home/username/.venv', 'pyvenv.cfg'),
# Python standard library venv module (not a virtual environment)
('/home/username/.local/share/uv/python/cpython-3.12.11-linux-x86_64-gnu/lib/python3.12/venv',
'__init__.py'),
('/home/username/.pyenv/versions/3.12.7/lib/python3.12/venv', '__init__.py'),
('/home/username/miniconda3/lib/python3.12/venv', '__init__.py'),
('/home/username/miniconda3/envs/foo/lib/python3.11/venv/scripts/common',
'activate'),
('/home/username/src/cpython/Lib/venv', '__init__.py'),
(r'C:\Users\charlie\scoop\apps\python\3.12.4\Lib\venv', '__init__.py'),
# tool-managed venvs in root's home
('/root/.local/share/uv/tools/ruff/.venv', 'pyvenv.cfg'),
# pipx-managed venvs (managed by pipx, like npm cache)
('/home/username/.local/share/pipx/venvs/duplicity', 'pyvenv.cfg'),
# typeshed stubs bundled with editor extensions
('/home/username/.vscode/extensions/ms-python.python-2026.4.0-linux-x64/python_files/lib/jedilsp/jedi/third_party/typeshed/stdlib/3/venv',
'__init__.pyi'),
# pyenv build patches
('/home/username/.pyenv/plugins/python-build/share/python-build/patches/3.5.2/Python-3.5.2',
'venv.patch'),
# Windows: editor extensions and AppData
(r'C:\Users\glados\.vscode\extensions\ms-python.python-2026.4.0-universal\python_files\lib\jedilsp\jedi\third_party\typeshed\stdlib\3\venv',
'__init__.pyi'),
(r'C:\Users\glados\AppData\Local\ApertureScience\MainframeCore\venv', 'pyvenv.cfg'),
# Files/directories that contain venv (but not a venv directory)
('/home/aturing/npl/ace_project/tools/', 'bootstrap_ace_venv.sh'),
('/home/vonneumann/edvac/logs/', 'venv_build.log'),
('/home/aturing/npl/ace_project/venv_tools/',
'requirements_numerics.txt'),
(r'C:\Users\charlie\Documents\important\venv_backup', 'pyvenv.cfg'),
# .venv bundled inside an AppData\Local application directory
(r'C:\Users\charlie\AppData\Local\AnkiProgramFiles\.venv\Lib\site-packages\PyQt6',
'QtCore.pyd'),
# Python standard library venv module under uv-managed Python (Windows)
(r'C:\Users\charlie\AppData\Roaming\uv\python\cpython-3.13.5-windows-x86_64-none\Lib\venv',
'__main__.py'),
]
if IS_WINDOWS:
# Windows is case insensitive
# venv cannot be directly under home
excluded.append((r'c:\users\username\.VENV', 'pyvenv.cfg'))
for dirpath, filename in excluded:
with self.subTest(dirpath=dirpath, filename=filename):
self.assertIsNone(
cs.match(dirpath, filename),
f"Should exclude {dirpath}/{filename}")
def test_venv_matches_ordinary_dirs(self):
"""venv option should match ordinary user project virtual environments."""
cs = self._get_compiled_search('venv')
matched = [
('/home/genisys/t800/.venv/yolo-v666', 'pyvenv.cfg'),
('/home/stark/jarvis/venv/yolo14', 'pyvenv.cfg'),
('/root/projects/app/.venv', 'pyvenv.cfg'),
('/home/stark/jarvis/venv/lib/python3.12/site-packages/numpy',
'__init__.py'),
(r'C:\Users\esnowden\Documents\nsa\prism\venv', 'pyvenv.cfg'),
]
if IS_WINDOWS:
# case insensitive
matched.append(
(r'C:\USERS\esnowden\Documents\nsa\prism\.VENV', 'pyvenv.cfg'))
matched.append(
(r'c:\USERS\pinky\world_domination\VENV\site-packages\hypnosisis', '__init__.py'))
for dirpath, filename in matched:
with self.subTest(dirpath=dirpath, filename=filename):
self.assertIsNotNone(
cs.match(dirpath, filename),
f"Should match {dirpath}/{filename}")
def _test_delete(self, command):
"""Delete files in a test environment"""
# make some files
base = self.mkdtemp()
f_del = self.write_file(os.path.join(base, 'foo.txt.bbtestbak'))
f_keep = self.write_file(os.path.join(base, 'foo.txt'))
subdir = os.path.join(base, 'sub')
os.mkdir(subdir)
f_del2 = self.write_file(os.path.join(subdir, 'bar.ini.bbtestbak'))
f_del3 = self.write_file(os.path.join(subdir, 'bar.ini.bbtestBAK'))
# sanity check
self.assertExists(f_del)
self.assertExists(f_keep)
self.assertExists(f_del2)
# run deep scan
def run_deep_scan(regex):
astr = '<action command="{}" search="deep" cache="false" path="{}" {}/>'.format(
command, base, regex)
cleaner = TestCleaner.action_to_cleaner(astr)
from bleachbit.Worker import backends, Worker
backends['test'] = cleaner
operations = {'test': ['option1']}
from bleachbit import CLI
ui = CLI.CliCallback()
worker = Worker(ui, True, operations)
list(worker.run())
del backends['test']
# validate results; 'sub' as a path component, as a random
# temporary directory name can contain it too
run_deep_scan(r'regex="\.bbtestbak$" '
r'wholeregex="[/\\]sub[/\\][^/\\]*\.bbtestbak$"')
self.assertExists(f_del)
self.assertExists(f_keep)
self.assertFalse(os.path.exists(f_del2))
if FS_CASE_SENSITIVE:
self.assertExists(f_del3)
else:
self.assertFalse(os.path.exists(f_del3))
# validate results
run_deep_scan(r'regex="\.bbtestbak$"')
self.assertFalse(os.path.exists(f_del))
self.assertExists(f_keep)
def test_delete(self):
self._test_delete('delete')
def test_shred(self):
self._test_delete('shred')
@unittest.skipUnless(IS_MAC, 'Not on Darwin')
def test_normalized_walk_darwin(self):
with mock.patch('os.walk') as mock_walk:
mock_walk.return_value = [
('/foo', ('bar',), ['ba\u0300z']),
('/foo/bar', (), ['spam', 'eggs']),
]
self.assertEqual(list(normalized_walk('.')), [
('/foo', ('bar',), ['b\u00e0z']),
('/foo/bar', (), ['spam', 'eggs']),
])
with mock.patch('os.walk') as mock_walk:
expected = [
('/foo', ('bar',), ['baz']),
('/foo/bar', (), ['spam', 'eggs']),
]
mock_walk.return_value = expected
self.assertEqual(list(normalized_walk('.')), expected)