Move rest of weblib stuff to grab.util. Remove weblib deps from setup/pyproject.

User · User · commit 06f8d2eaeccb · 2025-09-16T00:08:41.000+03:00
diff --git a/grab/__init__.py b/grab/__init__.py
@@ -1,7 +1,5 @@
 from __future__ import absolute_import
 
-from weblib.logs import default_logging  # noqa
-
 from grab.base import Grab  # noqa
 from grab.error import (  # noqa
     DataNotFound,
@@ -11,6 +9,7 @@
     GrabTimeoutError,
 )
 from grab.upload import UploadContent, UploadFile  # noqa
+from grab.util.log import default_logging  # noqa
 
 __version__ = "1.0.0"
 VERSION_NUMERIC = tuple(map(int, __version__.split(".")))
diff --git a/grab/deprecated.py b/grab/deprecated.py
@@ -3,7 +3,7 @@
 import six
 from selection import SelectionNotFoundError
 from selection.const import UNDEFINED
-from weblib.etree import get_node_text
+from grab.util.etree import get_node_text
 from grab.util.text import find_number
 
 from grab import error
diff --git a/grab/document.py b/grab/document.py
@@ -29,15 +29,15 @@
 from selection import SelectionNotFoundError, XpathSelector
 from six import BytesIO, StringIO
 from six.moves.urllib.parse import parse_qs, urljoin, urlsplit
-from weblib.files import hashed_path
-from weblib.rex import normalize_regexp
-from grab.util.text import normalize_space
 
 from grab.cookie import CookieManager
 from grab.error import DataNotFound, GrabMisuseError
 from grab.unset import UNSET, UnsetType
+from grab.util.files import hashed_path
 from grab.util.html import decode_entities, find_refresh_url
 from grab.util.http import smart_urlencode
+from grab.util.rex import normalize_regexp
+from grab.util.text import normalize_space
 from grab.util.warning import warn
 
 NULL_BYTE = chr(0)
diff --git a/grab/script/crawl.py b/grab/script/crawl.py
@@ -4,10 +4,10 @@
 from argparse import ArgumentParser
 
 import six
-from weblib.files import clear_directory
 
 from grab.util.config import build_root_config, build_spider_config
 from grab.util.encoding import make_str
+from grab.util.files import clear_directory
 from grab.util.log import default_logging
 from grab.util.module import load_spider_class
 
diff --git a/grab/spider/base.py b/grab/spider/base.py
@@ -10,7 +10,6 @@
 
 import six
 from six.moves.queue import Empty, Queue
-from weblib import metric
 
 from grab.base import Grab
 from grab.error import GrabInvalidUrl
@@ -24,6 +23,7 @@
 from grab.spider.task_generator_service import TaskGeneratorService
 from grab.stat import Stat
 from grab.unset import UNSET
+from grab.util.metric import format_traffic_value
 from grab.util.misc import camel_case_to_underscore
 from grab.util.warning import warn
 
@@ -425,7 +425,7 @@ def render_stats(self, timing=None):
         if "download-size" in self.stat.counters:
             out.append(
                 "Network download: %s"
-                % metric.format_traffic_value(self.stat.counters["download-size"])
+                % format_traffic_value(self.stat.counters["download-size"])
             )
         out.append(
             "Queue size: %d" % self.task_queue.size() if self.task_queue else "NA"
diff --git a/grab/util/etree.py b/grab/util/etree.py
@@ -0,0 +1,39 @@
+import six
+
+from grab.util.text import normalize_space as normalize_space_func
+
+
+def get_node_text(node, smart=False, normalize_space=True):
+    """
+    Extract text content of the `node` and all its descendants.
+
+    In smart mode `get_node_text` insert spaces between <tag><another tag>
+    and also ignores content of the script and style tags.
+
+    In non-smart mode this func just return text_content() of node
+    with normalized spaces
+    """
+
+    # If xpath return a attribute value, it value will be string not a node
+    if isinstance(node, six.string_types):
+        if normalize_space:
+            node = normalize_space_func(node)
+        return node
+
+    if smart:
+        value = " ".join(
+            node.xpath(
+                './descendant-or-self::*[name() != "script" and '
+                'name() != "style"]/text()[normalize-space()]'
+            )
+        )
+    else:
+        # If DOM tree was built with lxml.etree.fromstring
+        # then tree nodes do not have text_content() method
+        try:
+            value = node.text_content()
+        except AttributeError:
+            value = "".join(node.xpath(".//text()"))
+    if normalize_space:
+        value = normalize_space_func(value)
+    return value
diff --git a/grab/util/files.py b/grab/util/files.py
@@ -0,0 +1,40 @@
+import os
+import shutil
+from hashlib import sha1
+
+from six.moves.urllib.parse import urlsplit
+
+
+def hashed_path_details(url, ext="jpg", base_dir=None):
+    _hash = sha1(url).hexdigest()
+    a, b, tail = _hash[:2], _hash[2:4], _hash[4:]
+    directory = "%s/%s" % (a, b)
+    if base_dir is not None:
+        directory = "%s/%s" % (base_dir, directory)
+    if ext is not None:
+        filename = "%s.%s" % (tail, ext)
+    else:
+        filename = tail
+    full_path = "%s/%s" % (directory, filename)
+    return {
+        "directory": directory,
+        "filename": filename,
+        "full_path": full_path,
+    }
+
+
+def hashed_path(url, ext="jpg", base_dir=None):
+    return hashed_path_details(url, ext=ext, base_dir=base_dir)["full_path"]
+
+
+def clear_directory(path):
+    """
+    Delete recursively all directories and files in
+    specified directory.
+    """
+
+    for root, dirs, files in os.walk(path):
+        for fname in files:
+            os.unlink(os.path.join(root, fname))
+        for _dir in dirs:
+            shutil.rmtree(os.path.join(root, _dir))
diff --git a/grab/util/log.py b/grab/util/log.py
@@ -4,10 +4,13 @@
 from io import TextIOBase
 
 
-def default_logging(grab_log=None, # '/tmp/grab.log',
-                    network_log=None, # '/tmp/grab.network.log',
-                    level=logging.DEBUG, mode='a',
-                    propagate_network_logger=False):
+def default_logging(
+    grab_log=None,  # '/tmp/grab.log',
+    level=logging.DEBUG,
+    mode="a",
+    network_log=None,  # '/tmp/grab.network.log',
+    propagate_network_logger=False,
+):
     """
     Customize logging output to display all log messages
     except grab network logs.
@@ -17,14 +20,14 @@ def default_logging(grab_log=None, # '/tmp/grab.log',
 
     logging.basicConfig(level=level)
 
-    network_logger = logging.getLogger('grab.network')
+    network_logger = logging.getLogger("grab.network")
     network_logger.propagate = propagate_network_logger
     if network_log:
         hdl = logging.FileHandler(network_log, mode)
         network_logger.addHandler(hdl)
         network_logger.setLevel(level)
 
-    grab_logger = logging.getLogger('grab')
+    grab_logger = logging.getLogger("grab")
     if grab_log:
         hdl = logging.FileHandler(grab_log, mode)
         grab_logger.addHandler(hdl)
@@ -54,18 +57,18 @@ def write(self, data):
         self.buf.append(data)
 
     def get_output(self):
-        return ''.join(self.buf)
+        return "".join(self.buf)
 
     @contextmanager
     def handle_sigint(self):
         with self.record():
             try:
                 yield
-            except Exception: # pylint: disable=broad-except
-                if 'KeyboardInterrupt' in self.get_output():
+            except Exception:  # pylint: disable=broad-except
+                if "KeyboardInterrupt" in self.get_output():
                     raise KeyboardInterrupt
                 else:
                     raise
             else:
-                if 'KeyboardInterrupt' in self.get_output():
+                if "KeyboardInterrupt" in self.get_output():
                     raise KeyboardInterrupt
diff --git a/grab/util/metric.py b/grab/util/metric.py
@@ -0,0 +1,27 @@
+KB = 1024
+MB = 1024 * KB
+GB = MB * 1024
+
+
+def in_unit(num, unit):
+    if unit == "b":
+        return num
+    elif unit == "kb":
+        return round(num / float(KB), 2)
+    elif unit == "mb":
+        return round(num / float(MB), 2)
+    elif unit == "gb":
+        return round(num / float(GB), 2)
+    else:
+        return num
+
+
+def format_traffic_value(num):
+    if num < KB:
+        return "%s B" % in_unit(num, "b")
+    elif num < MB:
+        return "%s KB" % in_unit(num, "kb")
+    elif num < GB:
+        return "%s MB" % in_unit(num, "mb")
+    else:
+        return "%s GB" % in_unit(num, "gb")
diff --git a/grab/util/rex.py b/grab/util/rex.py
@@ -0,0 +1,29 @@
+import re
+
+import six
+
+REGEXP_CACHE = {}
+
+
+def cache_regexp(rex, flags=0):
+    key = (rex, flags)
+    try:
+        return REGEXP_CACHE[key]
+    except KeyError:
+        obj = re.compile(rex, flags)
+        # obj.source = rex
+        REGEXP_CACHE[key] = obj
+        return obj
+
+
+def normalize_regexp(regexp, flags=0):
+    """
+    Accept string or compiled regular expression object.
+
+    Compile string into regular expression object.
+    """
+
+    if isinstance(regexp, six.string_types):
+        return cache_regexp(regexp, flags)
+    else:
+        return regexp
diff --git a/pyproject.toml b/pyproject.toml
@@ -41,7 +41,6 @@ classifiers = [
     #"Typing :: Typed",
 ]
 dependencies = [
-        "weblib>=0.1.28",
         "six",
         "user_agent",
         "selection>=2.0.1",
diff --git a/setup.py b/setup.py
@@ -14,7 +14,6 @@
         "grab.util",
     ],
     install_requires=[
-        "weblib>=0.1.28",
         "six",
         "user_agent",
         "selection>=2.0.1",
