Initial commit.

.gitignore

@@ -0,0 +1,5 @@
+/.idea
+__pycache__/
+*.swp
+
+

README.md

@@ -0,0 +1,8 @@
+
+
+
+## Licences that need to be added to the project (todo)
+
+- MIT
+    - https://github.com/feathericons/feather
+    - halfmoon

fleabay/api_extractor.py

@@ -0,0 +1,5 @@
+
+class EbayApiExtractor:
+    def __init__(self, api_key):
+        pass
+

fleabay/definitions.py

@@ -0,0 +1,37 @@
+from typing import Optional, Dict
+
+import attr
+from immutabledict import immutabledict
+
+
+@attr.s(auto_attribs=True)
+class EbayItemSummary:
+    href: str
+    name: str
+    condition: str
+    min_price: int
+    max_price: int
+    # None if not an auction
+    bids: Optional[int]
+    country: str
+    # None if postage not offered (collection only etc)
+    postage: Optional[int]
+    image_src: str
+
+
+@attr.s(auto_attribs=True)
+class EbayMultiItemModelAxis:
+    code_name: str
+    # human_name: str
+    values: Dict[int, str]
+
+
+@attr.s(auto_attribs=True)
+class EbayItemDetails:
+    seller_name: str
+    seller_score: int
+    seller_percentage: str
+
+    axes: Dict[str, EbayMultiItemModelAxis]
+    # variants: Dict[immutabledict[str, int]]
+    variants: Dict[immutabledict, int]

fleabay/extractor.py

@@ -0,0 +1,190 @@
+import ast
+import json
+from pprint import pprint
+from re import Match
+from typing import List, Dict
+
+import bs4
+from bs4 import Tag, NavigableString
+from immutabledict import immutabledict
+from requests import Session
+import re
+
+from fleabay.definitions import EbayItemSummary, EbayItemDetails, EbayMultiItemModelAxis
+
+
+class EbayExtractor:
+    def __init__(self, domain: str = "https://www.ebay.co.uk"):
+        self.domain = domain
+        self.session = Session()
+
+    def search(self, term: str, items_per_page: int = 200, remote_lowest_first: bool = False) -> List[EbayItemSummary]:
+        params = {"_nkw": term, "_sacat": 0, "_ipg": items_per_page}
+
+        if remote_lowest_first:
+            params["_sop"] = 15
+
+        resp = self.session.get(self.domain + "/sch/i.html", params=params)
+
+        if resp.status_code != 200:
+            raise RuntimeError("not 200")
+
+        doc = bs4.BeautifulSoup(resp.content, "html.parser")
+
+        results = []
+
+        item_tag: Tag
+        for item_tag in doc.select(".s-item"):
+            title_tag = item_tag.select_one(".s-item__title")
+            if title_tag is None:
+                continue
+            for child in title_tag.children:
+                if isinstance(child, NavigableString):
+                    title = str(child)
+                    break
+            else:
+                continue
+
+            condition_tag = item_tag.select_one(".SECONDARY_INFO")
+            if condition_tag:
+                condition = condition_tag.text
+            else:
+                condition = "?"
+
+            price_tag = item_tag.select_one(".s-item__price")
+            if price_tag:
+                prices = re.findall(r"£([0-9]+)\.([0-9]+)", price_tag.text)
+                if not prices:
+                    print("No £ in ", price_tag.text)
+                    continue
+                min_price_m = prices[0]
+                max_price_m = prices[-1]
+                min_price = int(min_price_m[0]) * 100 + int(min_price_m[1])
+                max_price = int(max_price_m[0]) * 100 + int(max_price_m[1])
+            else:
+                min_price = -42
+                max_price = -42
+
+            bid_tag = item_tag.select_one(".s-item__bidCount")
+            if bid_tag:
+                bids = int(re.match("[0-9]+", bid_tag.text).group(0))
+            else:
+                bids = None
+
+            postage_tag = item_tag.select_one(".s-item__logisticsCost")
+            if postage_tag:
+                if "Free" in postage_tag.text:
+                    postage = 0
+                else:
+                    pmatch = re.search(r"£([0-9]+)\.([0-9]+)", postage_tag.text)
+                    if pmatch is None:
+                        print("no post ", postage_tag.text, item_tag)
+                    postage = int(pmatch.group(1)) * 100 + int(pmatch.group(2))
+            elif item_tag.select_one(".s-item__localDelivery") is not None:
+                # local collection only.
+                postage = None
+            else:
+                raise ValueError("Can't find postage in " + str(item_tag))
+
+            location_tag = item_tag.select_one(".s-item__itemLocation")
+            location = ""
+            if location_tag and location_tag.text.startswith("From "):
+                location = location_tag.text[5:]
+
+            link_tag = item_tag.find("a")
+            link_href = link_tag.attrs["href"]
+
+            image_tag = item_tag.find("img")
+            image_src = image_tag.attrs["src"]
+
+            results.append(
+                EbayItemSummary(
+                    link_href,
+                    title,
+                    condition,
+                    min_price,
+                    max_price,
+                    bids,
+                    location,
+                    postage,
+                    image_src,
+                )
+            )
+
+        return results
+
+    def details(self, url: str) -> EbayItemDetails:
+        page = self.session.get(url)
+        line: bytes
+        for line in page.iter_lines():
+            opening = line.find(b"$rwidgets([")
+            if opening == -1:
+                continue
+            ending = line.find(b"])", opening)
+            if ending == -1:
+                raise ValueError("Unable to find ending.")
+            break
+        else:
+            raise ValueError("Unable to find opening.")
+
+        pythonic_literal = (
+            line[opening + 10 : ending + 1]
+            .decode()
+            .replace(":null", ":None")
+            .replace(":false", ":False")
+            .replace(":true", ":True")
+        )
+        data = ast.literal_eval(pythonic_literal)
+
+        # pprint(data)
+        # print("-----")
+
+        for item in data:
+            if item[0] == "com.ebay.raptor.vi.lockedheader.LockedHeaderCore":
+                root_dict = item[2]
+                variations = root_dict["itemVariationsModel"]["itemVariationsMap"]
+                break
+        else:
+            raise ValueError("Variations not found")
+
+        # pprint(variations)
+
+        item_map: Dict[int, str] = {
+            int(k): v["displayName"]
+            for k, v in root_dict["itemVariationsModel"]["menuItemMap"].items()
+        }
+        models = root_dict["itemVariationsModel"]["menuModels"]
+        prices: Dict[immutabledict, int] = dict()  # id[str, int]
+
+        # pprint(models)
+
+        for variation in variations.values():
+            pmatch = re.search(r"£([0-9]+)\.([0-9]+)", variation["convertedPrice"])
+            if pmatch is None:
+                raise ValueError(
+                    "Unable to extract price from " + variation["convertedPrice"]
+                )
+            price = int(pmatch.group(1)) * 100 + int(pmatch.group(2))
+
+            trait_map = immutabledict(variation["traitValuesMap"])
+            prices[trait_map] = price
+
+        axes = dict()
+        for model in models:
+            value_map = dict()
+            for value in model["menuItemValueIds"]:
+                value_map[value] = item_map[value]
+
+            axes[model["name"]] = EbayMultiItemModelAxis(model["name"], value_map)
+
+        return EbayItemDetails("", -1, "-1%", axes, prices)
+
+
+if __name__ == "__main__":
+    # EbayExtractor().search("fish tank", items_per_page=10)
+
+    pprint(
+        EbayExtractor().details(
+            "https://www.ebay.co.uk/itm/Genuine-Perspex-Acrylic-Sheet-Black-White-Panel-Cut-To-Size-Plastic-3mm-5mm/124449453991?var=425326090110&hash=item1cf9c3f7a7:g:MPgAAOSw42Nft55q"
+        )
+    )

fleabay/static/fleabay.css

@@ -0,0 +1,12 @@
+.fleabay-table-compact {
+  font-size: 60%;
+}
+
+.fleabay-table-compact td, .fleabay-table-compact th {
+  padding: 0;
+  white-space: nowrap;
+}
+
+.fleabay-table-compact th {
+  padding-right: 0.5em;
+}

fleabay/static/menu.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-menu"><line x1="3" y1="12" x2="21" y2="12"></line><line x1="3" y1="6" x2="21" y2="6"></line><line x1="3" y1="18" x2="21" y2="18"></line></svg>

fleabay/static/moon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-moon"><path d="M21 12.79A9 9 0 1 1 11.21 3 7 7 0 0 0 21 12.79z"></path></svg>

fleabay/static/search.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-search"><circle cx="11" cy="11" r="8"></circle><line x1="21" y1="21" x2="16.65" y2="16.65"></line></svg>

fleabay/templates/base.html.j2

@@ -0,0 +1,72 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="utf-8" />
+  <meta content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0" name="viewport" />
+  <meta name="viewport" content="width=device-width" />
+
+  <!--- TODO <link rel="icon" href="path/to/fav.png"> -->
+  <title>{{ page_title or "Fleabay" }}</title>
+
+  <link href="/static/halfmoon.min.css" rel="stylesheet" />
+  <link href="/static/fleabay.css" rel="stylesheet" />
+</head>
+<body class="with-custom-webkit-scrollbars with-custom-css-scrollbars" data-dm-shortcut-enabled="true" data-sidebar-shortcut-enabled="true" data-set-preferred-mode-onload="true">
+  <!-- Modals go here -->
+  <!-- Reference: https://www.gethalfmoon.com/docs/modal -->
+
+  <!-- Page wrapper start TODO make the navbar not fixed -->
+  <!-- <div class="page-wrapper with-navbar with-sidebar" data-sidebar-type="full-height overlayed-sm-and-down"> -->
+  <div class="page-wrapper with-navbar">
+
+    <!-- Navbar start -->
+    <nav class="navbar">
+      <!-- Reference: https://www.gethalfmoon.com/docs/navbar -->
+      <div class="navbar-content">
+        <!-- <button class="btn btn-action" type="button" onclick="halfmoon.toggleSidebar()">
+          M
+          <span class="sr-only">Toggle sidebar</span>
+        </button> -->
+      </div>
+
+      <a href="/" class="navbar-brand">
+        Fleabay
+      </a>
+
+      <div class="navbar-content">
+        <button class="btn btn-action" type="button" onclick="halfmoon.toggleDarkMode()" aria-label="Toggle dark mode">
+          L/D
+        </button>
+      </div>
+    </nav>
+    <!-- Navbar end -->
+
+    <!-- Sidebar overlay -->
+    <!--- <div class="sidebar-overlay" onclick="halfmoon.toggleSidebar()"></div> -->
+
+    <!-- Sidebar start -->
+    <!-- <div class="sidebar"> -->
+      <!-- Reference: https://www.gethalfmoon.com/docs/sidebar -->
+    <!-- </div> -->
+    <!-- Sidebar end -->
+
+    <!-- Content wrapper start -->
+    <div class="content-wrapper">
+      {% block content %}
+      <!--
+        Add your page's main content here
+        Examples:
+        1. https://www.gethalfmoon.com/docs/content-and-cards/#building-a-page
+        2. https://www.gethalfmoon.com/docs/grid-system/#building-a-dashboard
+      -->
+      {% endblock content %}
+    </div>
+    <!-- Content wrapper end -->
+
+  </div>
+  <!-- Page wrapper end -->
+
+  <!-- Halfmoon JS -->
+  <script src="/static/halfmoon.min.js"></script>
+</body>
+</html>

fleabay/templates/index.html.j2

@@ -0,0 +1,7 @@
+{% extends "base.html.j2" %}
+
+{% set page_title = "Fleabay - Home" %}
+
+{% block content %}
+  {% include "search_form.f.html.j2" %}
+{% endblock content %}

fleabay/templates/search.html.j2

@@ -0,0 +1,97 @@
+{% extends "base.html.j2" %}
+
+{% set page_title = "Fleabay - Home" %}
+
+{% macro variant_table(axis, value_items) -%}
+  <div class="flex-grow-1 d-flex align-items-center">
+    <div class="p-10 m-auto">
+      <table class="table w-full fleabay-table-compact">
+        <thead>
+          <tr>
+            <th>{{ axis }}</th>
+            <th>Price(s)</th>
+          </tr>
+        </thead>
+        <tbody>
+          {% for variant, (min_price, max_price) in value_items %}
+          <tr>
+            <th>{{ variant }}</th>
+            <td>
+              £{{ min_price // 100 }}.{{ "{:02d}".format(min_price % 100) }}
+              {% if min_price != max_price %}
+                — £{{ max_price // 100 }}.{{ "{:02d}".format(max_price % 100) }}
+              {% endif %}
+            </td>
+          </tr>
+          {% endfor %}
+        </tbody>
+      </table>
+    </div>
+  </div>
+{%- endmacro %}
+
+{% block content %}
+  {% include "search_form.f.html.j2" %}
+
+  <div id="results" class="row">
+    {% for result in results %}
+      {% if result.href in axes_ranges and (axes_ranges[result.href][1] | length) > 25 %}
+      <div class="card mw-full m-0 p-0 d-flex col-6" target="_blank" rel="noopener">
+      {% else %}
+      <div class="card mw-full m-0 p-0 d-flex col-3" target="_blank" rel="noopener">
+      {% endif %}
+        <div class="w-60 h-60 m-10 align-self-center">
+            <div class="w-60 h-60 rounded d-flex align-items-center justify-content-center bg-light-lm bg-dark-light-dm text-dark-lm text-light-dm">
+                <img src="{{ result.image_src }}" width=200>
+            </div>
+
+            <div class="p-10 flex-grow-1 m-auto">
+              <a class="m-0 font-weight-medium text-dark-lm text-light-dm font-size-14" href="{{ result.href }}">
+                {{ result.name }}
+              </a>
+              <p class="m-0 mt-5 font-size-12">
+                <span class="font-size-14">
+                  £{{ result.min_price // 100 }}.{{ "{:02d}".format(result.min_price % 100) }}
+                  {% if result.min_price != result.max_price %}
+                    — £{{ result.max_price // 100 }}.{{ "{:02d}".format(result.max_price % 100) }}
+                  {% endif %}
+                </span><br>
+                {{ result.condition }}.<br>
+                {% if result.bids is not none %}
+                  {{ result.bids }} bids.
+                {% else %}
+                  'Buy it now'.
+                {% endif %}<br>
+                {% if result.postage is none %}
+                  Local collection only.
+                {% elif result.postage %}
+                  P&amp;P: £{{ result.postage // 100 }}.{{ "{:02d}".format(result.postage % 100) }} (included in price).
+                {% else %}
+                  P&amp;P included.
+                {% endif %}
+                {% if result.country %}
+                  From <strong>{{ result.country }}</strong>
+                {% endif %}
+              </p>
+            </div>
+        </div>
+        {% if result.href in axes_ranges %}
+          {% set axis, values = axes_ranges[result.href] %}
+          {% set value_items = values.items() | list %}
+          {% set value_len = value_items | length %}
+          {% if values | length > 25 %}
+            {{ variant_table(axis, value_items[0:value_len // 2]) }}
+            {{ variant_table(axis, value_items[value_len // 2:]) }}
+          {% else %}
+            {{ variant_table(axis, value_items) }}
+          {% endif %}
+        {% elif result.href in axes_errors %}
+          <div class="p-10 m-auto">
+            {{ axes_errors[result.href] }}
+          </div>
+      {% endif %}
+      </div>
+    {% endfor %}
+  </div>
+
+{% endblock content %}

fleabay/templates/search_form.f.html.j2

@@ -0,0 +1,19 @@
+<div class="content">
+  <form action="/search" method="GET"> <!-- w-400 = width: 40rem (400px), mw-full = max-width: 100% -->
+    <!-- Input -->
+    <div class="input-group mb-20">
+      {{ search_form.search_term(class="form-control") }}
+      <div class="input-group-append">
+        <button class="btn btn-primary" type="submit">Search</button>
+      </div>
+    </div>
+
+    <div class="custom-checkbox d-inline-block mr-10">
+      {{ search_form.remote_lowest_first }} {{ search_form.remote_lowest_first.label() }}
+    </div>
+
+    <div class="custom-checkbox d-inline-block mr-10">
+      {{ search_form.include_local_collection }} {{ search_form.include_local_collection.label() }}
+    </div>
+  </form>
+</div>

fleabay/webapp.py

@@ -0,0 +1,115 @@
+import asyncio
+from concurrent.futures.thread import ThreadPoolExecutor
+from typing import Dict
+
+import quart
+from quart import Quart, render_template, request
+from wtforms_async import Form, StringField, BooleanField
+from wtforms_async.widgets import CheckboxInput
+
+from fleabay.definitions import EbayItemSummary, EbayItemDetails
+from fleabay.extractor import EbayExtractor
+
+app = Quart(__name__)
+
+pool = ThreadPoolExecutor(8)
+
+
+class SearchForm(Form):
+    search_term = StringField("Search Term")
+    include_local_collection = BooleanField("Include local collection")
+    remote_lowest_first = BooleanField("Ask remote for 'Lowest Price+P&P First'")
+
+
+@app.route("/", methods=["GET"])
+async def index():
+    return await render_template("index.html.j2", search_form=SearchForm())
+
+
+@app.route("/search", methods=["GET"])
+async def search():
+    search_form = SearchForm(request.args)
+
+    if not await search_form.validate():
+        return await render_template("index.html.j2", search_form=SearchForm())
+
+    ex = EbayExtractor()
+
+    search_term = search_form.search_term.data
+
+    loop = asyncio.get_running_loop()
+
+    results = await loop.run_in_executor(pool, ex.search, search_term, 50)
+
+    if not search_form.include_local_collection.data:
+        results = [r for r in results if r.postage is not None]
+
+    item_details_fut = dict()
+
+    for result in results:
+        if result.min_price != result.max_price:
+            fut = loop.run_in_executor(pool, ex.details, result.href)
+            item_details_fut[fut] = result.href
+
+    futs = tuple(item_details_fut.keys())
+    detail_results = await asyncio.gather(*futs, return_exceptions=True)
+
+    item_details: Dict[str, EbayItemDetails] = {
+        item_details_fut[fut]: res
+        for res, fut in zip(detail_results, futs)
+        if isinstance(res, EbayItemDetails)
+    }
+    axes_errors = {
+        item_details_fut[fut]: str(res)
+        for res, fut in zip(detail_results, futs)
+        if isinstance(res, Exception)
+    }
+
+    print(f"{len(axes_errors)} errors")
+
+    # want to find the most insightful axis to display over
+    # Insightful suggests that the individual values in the axis have a tight range
+    axes_ranges = dict()
+    for href, details in item_details.items():
+        best_axis = None
+        best_axis_tightness = 1.0e9
+        for axis in details.axes.values():
+            tightnesses = []
+            for value, _v_name in axis.values.items():
+                min_vp = 1e9
+                max_vp = 0
+                for variant, price in details.variants.items():
+                    if variant[axis.code_name] != value:
+                        continue
+                    max_vp = max(max_vp, price)
+                    min_vp = min(min_vp, price)
+
+                tightnesses.append(max_vp - min_vp)
+            avg_tightness = sum(tightnesses) / len(tightnesses)
+            if avg_tightness < best_axis_tightness:
+                best_axis = axis
+                best_axis_tightness = avg_tightness
+
+        if best_axis is not None:
+            axis_value_ranges = dict()
+
+            for value, value_name in best_axis.values.items():
+                min_vp = 1e9
+                max_vp = 0
+                for variant, price in details.variants.items():
+                    if variant[best_axis.code_name] != value:
+                        continue
+                    max_vp = max(max_vp, price)
+                    min_vp = min(min_vp, price)
+
+                axis_value_ranges[value_name] = (min_vp, max_vp)
+
+            axes_ranges[href] = (best_axis.code_name, axis_value_ranges)
+
+    return await render_template(
+        "search.html.j2",
+        search_form=search_form,
+        results=results,
+        axes_ranges=axes_ranges,
+        axes_errors=axes_errors,
+    )

requirements.txt

@@ -0,0 +1,4 @@
+requests~=2.25.0
+beautifulsoup4~=4.9.3
+immutabledict
+git+https://gitlab.com/reivilibre/wtforms_async.git

setup.py

@@ -0,0 +1,12 @@
+from setuptools import setup
+
+setup(
+    name="fleabay",
+    version="0.0.1",
+    packages=["fleabay"],
+    url="",
+    license="",
+    author="reivilibre",
+    author_email="",
+    description="Flexible eBay front-end",
+)