Allow using the eBay API

Fixes #1.

Need to set an eBay API key using EBAY_KEY env var.

fleabay/api_extractor.py

@@ -1,5 +1,211 @@
+import itertools
+import os
+import pprint
+from typing import List, Dict
+
+from immutabledict import immutabledict
+from requests import Session
+
+from fleabay.definitions import EbayItemSummary, EbayItemDetails, EbayMultiItemModelAxis
+from fleabay.utils import chunker, decimal_price_to_pence
+
 
 class EbayApiExtractor:
-    def __init__(self, api_key):
+    def __init__(self, api_key: str):
-        pass
+        self.api_key = api_key
+        self.session = Session()
 
+    def search(self, term: str, items_per_page: int = 200, remote_lowest_first: bool = False) -> List[EbayItemSummary]:
+        url = "https://svcs.ebay.com/services/search/FindingService/v1"
+        params = {
+            "SECURITY-APPNAME": self.api_key,
+            "OPERATION-NAME": "findItemsByKeywords",
+            "SERVICE-VERSION": "1.0.0",
+            "RESPONSE-DATA-FORMAT": "JSON",
+            "REST-PAYLOAD": "",
+            "keywords": term,
+            "paginationInput.entriesPerPage": items_per_page,
+            "GLOBAL-ID": "EBAY-GB",
+            "siteid": 3,
+            "buyerPostalCode": "M11 2GX"
+        }
+        if remote_lowest_first:
+            params["sortOrder"] = "PricePlusShippingLowest"
+
+        resp = self.session.get(url, params=params).json()
+        resp_k = resp["findItemsByKeywordsResponse"][0]
+        if resp_k["ack"][0] != "Success":
+            raise RuntimeError("search by API not successful.")
+
+        items = []
+
+        found_item_ids = dict()
+
+        if resp_k["searchResult"][0]["@count"] == "0":
+            return []
+
+        try:
+            item_results = resp_k["searchResult"][0]["item"]
+        except KeyError as e:
+            raise RuntimeError("%r" % resp_k["searchResult"][0].items()) from e
+
+        for result in item_results:
+            item_id = result["itemId"][0]
+
+            if item_id in found_item_ids:
+                continue
+
+            if item_id == "254817515397":
+                pprint.pprint(result)
+
+            is_buy_it_now = result["listingInfo"][0]["listingType"][0] != "Auction"
+
+            postage_price = result["shippingInfo"][0]["shippingServiceCost"][0]
+            postage_pence = -42
+            if postage_price["@currencyId"] == "GBP":
+                postage_pence = int(100 * float(postage_price["__value__"]))
+
+            if result["shippingInfo"][0]["shippingType"][0] == "FreePickup":
+                postage_pence = None
+
+            item_price = result["sellingStatus"][0]["currentPrice"][0]
+            item_pence = -42
+
+            bid_count = result["sellingStatus"][0].get("bidCount")
+            if bid_count is not None:
+                bid_count = bid_count[0]
+
+            if item_price["@currencyId"] == "GBP":
+                item_pence = int(100 * float(item_price["__value__"]))
+
+            view_url = f"https://www.ebay.co.uk/itm/{item_id}"
+
+            try:
+                condition = result["condition"][0]["conditionDisplayName"][0]
+            except KeyError:
+                condition = "???"
+
+            if item_id == "154280943581":
+                pprint.pprint(result)
+
+            item = EbayItemSummary(
+                item_id, view_url, result["title"][0], condition,
+                item_pence, -42, None if is_buy_it_now else bid_count, result["country"][0], postage_pence, result["galleryURL"][0]
+            )
+
+            found_item_ids[item_id] = item
+
+            items.append(item)
+
+        for item_id, details in self.details_batch(list(found_item_ids.keys())).items():
+            summary = found_item_ids[item_id]
+            summary.extras["details"] = details
+            summary.min_price = int(details.extra["min_price"])
+            summary.max_price = int(details.extra["max_price"])
+
+        return items
+
+    @staticmethod
+    def details(summary: EbayItemSummary) -> EbayItemDetails:
+        return summary.extras["details"]
+
+    def details_batch(self, ids: List[str]) -> Dict[str, EbayItemDetails]:
+        # eBay can do up to 20 items at once.
+        result = dict()
+        for id_chunk in chunker(ids, 20):
+            url = "https://open.api.ebay.com/shopping"
+            params = {
+                "ItemID": ",".join(id_chunk),
+                "callname": "GetMultipleItems",
+                "IncludeSelector": "Details,Variations",
+                "appid": self.api_key,
+                "siteid": 3,
+                "version": "967",
+                "responseencoding": "JSON"
+            }
+            response = self.session.get(url, params=params)
+            # print(response.status_code)
+
+            resp = response.json()
+
+            for item_dict in resp["Item"]:
+                item_id = item_dict["ItemID"]
+                seller_dict = item_dict["Seller"]
+                seller = {
+                    "name": seller_dict["UserID"],
+                    "feedback": f"{seller_dict['PositiveFeedbackPercent']}% ({seller_dict['FeedbackScore']})"
+                }
+
+                extras = {
+                    "variant_extras": dict()
+                }
+
+                base_pence = -42
+                if item_dict["CurrentPrice"]["CurrencyID"] == "GBP":
+                    base_pence = decimal_price_to_pence("%.2f" % item_dict["CurrentPrice"]["Value"])
+
+                min_pence = base_pence
+                max_pence = base_pence
+
+                variants_dict = dict()
+                axes: Dict[str, EbayMultiItemModelAxis] = dict()
+                axes_inverse: Dict[str, Dict[str, int]] = dict()
+
+                variations = item_dict.get("Variations")
+                if variations is not None:
+                    min_pence = 1e9
+                    max_pence = -42
+
+                    for axis in variations["VariationSpecificsSet"]["NameValueList"]:
+                        axes_inverse[axis["Name"]] = {
+                            val: i for i, val in enumerate(axis["Value"])
+                        }
+                        axes[axis["Name"]] = EbayMultiItemModelAxis(axis["Name"], dict(enumerate(axis["Value"])))
+
+                    for variation in variations["Variation"]:
+                        quantity = variation["Quantity"] - variation["SellingStatus"].get("QuantitySold", 0)
+
+                        if quantity <= 0:
+                            continue
+
+                        price = -42
+                        if variation["StartPrice"]["CurrencyID"] == "GBP":
+                            price = decimal_price_to_pence("%.2f" % variation["StartPrice"]["Value"])
+
+                            min_pence = min(min_pence, price)
+                            max_pence = max(max_pence, price)
+
+                        sold = variation["SellingStatus"]["QuantitySold"]
+
+                        variant_immdict = immutabledict({
+                            pair["Name"]: axes_inverse[pair["Name"]][pair["Value"][0]]
+                            for pair in variation["VariationSpecifics"]["NameValueList"]
+                        })
+
+                        extras["variant_extras"][variant_immdict] = {
+                            "quantity": quantity,
+                            "sold": sold
+                        }
+
+                        assert isinstance(price, int)
+                        variants_dict[variant_immdict] = price
+
+                        if item_id == "223749865106":
+                            pprint.pprint(variation)
+                            print("vi", "%r" % variant_immdict)
+                            print("vp", "%r" % price)
+                            print("qt", "%r" % quantity)
+
+                extras["min_price"] = int(min_pence)
+                extras["max_price"] = int(max_pence)
+
+                result[item_id] = EbayItemDetails(
+                    seller_dict["UserID"],
+                    int(seller_dict["FeedbackScore"]),
+                    seller_dict["PositiveFeedbackPercent"],
+                    axes,
+                    variants_dict,
+                    extras
+                )
+
+        return result

fleabay/definitions.py

@@ -1,4 +1,4 @@
-from typing import Optional, Dict
+from typing import Optional, Dict, Any
 
 import attr
 from immutabledict import immutabledict
@@ -6,6 +6,8 @@ from immutabledict import immutabledict
 
 @attr.s(auto_attribs=True)
 class EbayItemSummary:
+    # arbitrary ID
+    id: str
     href: str
     name: str
     condition: str
@@ -18,6 +20,9 @@ class EbayItemSummary:
     postage: Optional[int]
     image_src: str
 
+    # extra data about the item.
+    extras: Dict[str, Any] = attr.attrib(factory=dict)
+
 
 @attr.s(auto_attribs=True)
 class EbayMultiItemModelAxis:
@@ -35,3 +40,19 @@ class EbayItemDetails:
     axes: Dict[str, EbayMultiItemModelAxis]
     # variants: Dict[immutabledict[str, int]]
     variants: Dict[immutabledict, int]
+
+    extra: Dict[str, Any] = attr.attrib(factory=dict)
+
+
+codes_to_symbols = {
+    "GBP": "£",
+    "EUR": "€",
+    "USD": "$",
+    "JPY": "¥"
+}
+
+
+@attr.s(auto_attribs=True)
+class Price:
+    currency_code: str
+    amount: int

fleabay/extractor.py

@@ -107,6 +107,7 @@ class EbayExtractor:
 
             results.append(
                 EbayItemSummary(
+                    link_href,
                     link_href,
                     title,
                     condition,
@@ -121,8 +122,8 @@ class EbayExtractor:
 
         return results
 
-    def details(self, url: str) -> EbayItemDetails:
+    def details(self, summary: EbayItemSummary) -> EbayItemDetails:
-        page = self.session.get(url)
+        page = self.session.get(summary.href)
         line: bytes
         for line in page.iter_lines():
             opening = line.find(b"$rwidgets([")

fleabay/templates/search_form.f.html.j2

@@ -8,6 +8,11 @@
       </div>
     </div>
 
+    <div class="form-group">
+      {{ search_form.extractor.label }}
+      {{ search_form.extractor(class="form-control") }}
+    </div>
+
     <div class="custom-checkbox d-inline-block mr-10">
       {{ search_form.remote_lowest_first }} {{ search_form.remote_lowest_first.label() }}
     </div>

fleabay/utils.py

@@ -0,0 +1,15 @@
+import re
+
+
+def decimal_price_to_pence(price: str) -> int:
+    pmatch = re.search(r"£?([0-9]+)\.([0-9]+)", price)
+    if pmatch is None:
+        raise ValueError(
+            "Unable to extract price from " + price
+        )
+    return int(pmatch.group(1)) * 100 + int(pmatch.group(2))
+
+
+# https://stackoverflow.com/a/434328 CC BY-SA 4.0
+def chunker(seq, size):
+    return (seq[pos:pos + size] for pos in range(0, len(seq), size))

fleabay/webapp.py

@@ -1,12 +1,15 @@
 import asyncio
+import os
 from concurrent.futures.thread import ThreadPoolExecutor
-from typing import Dict
+from pprint import pprint
+from typing import Dict, Any, Tuple
 
 import quart
 from quart import Quart, render_template, request
-from wtforms_async import Form, StringField, BooleanField
+from wtforms_async import Form, StringField, BooleanField, SelectField
 from wtforms_async.widgets import CheckboxInput
 
+from fleabay.api_extractor import EbayApiExtractor
 from fleabay.definitions import EbayItemSummary, EbayItemDetails
 from fleabay.extractor import EbayExtractor
 
@@ -15,7 +18,32 @@ app = Quart(__name__)
 pool = ThreadPoolExecutor(8)
 
 
+def make_extractors() -> Tuple[Dict[str, Any], Dict[str, str]]:
+    available_extractors = dict()
+    extractor_names = dict()
+
+    ebay_key = os.environ.get("EBAY_KEY")
+    if ebay_key:
+        available_extractors["eBay_api"] = EbayApiExtractor(ebay_key)
+        extractor_names["eBay_api"] = "eBay (by API — best)"
+
+    available_extractors["eBay_scrape"] = EbayExtractor()
+    extractor_names["eBay_scrape"] = "eBay (by scraper — slow, error-prone)"
+
+    zapiex_key = os.environ.get("ZAPIEX_KEY")
+    if zapiex_key:
+        available_extractors["AE_zapiex"] = None
+        extractor_names["AE_zapiex"] = "AliExpress (by Zapiex API — limited)"
+
+    return available_extractors, extractor_names
+
+
+# we need only initialise the extractors once.
+available_extractors, extractor_names = make_extractors()
+
+
 class SearchForm(Form):
+    extractor = SelectField("Site/Extractor", choices=list(extractor_names.items()))
     search_term = StringField("Search Term")
     include_local_collection = BooleanField("Include local collection")
     remote_lowest_first = BooleanField("Ask remote for 'Lowest Price+P&P First'")
@@ -33,13 +61,13 @@ async def search():
     if not await search_form.validate():
         return await render_template("index.html.j2", search_form=SearchForm())
 
-    ex = EbayExtractor()
+    ex = available_extractors[search_form.extractor.data]
 
     search_term = search_form.search_term.data
 
     loop = asyncio.get_running_loop()
 
-    results = await loop.run_in_executor(pool, ex.search, search_term, 50)
+    results = await loop.run_in_executor(pool, ex.search, search_term, 100, search_form.remote_lowest_first.data)
 
     if not search_form.include_local_collection.data:
         results = [r for r in results if r.postage is not None]
@@ -47,9 +75,9 @@ async def search():
     item_details_fut = dict()
 
     for result in results:
-        if result.min_price != result.max_price:
+        if not isinstance(ex, EbayExtractor) or result.min_price != result.max_price:
-            fut = loop.run_in_executor(pool, ex.details, result.href)
+            fut = loop.run_in_executor(pool, ex.details, result)
-            item_details_fut[fut] = result.href
+            item_details_fut[fut] = result.id
 
     futs = tuple(item_details_fut.keys())
     detail_results = await asyncio.gather(*futs, return_exceptions=True)
@@ -70,7 +98,7 @@ async def search():
     # want to find the most insightful axis to display over
     # Insightful suggests that the individual values in the axis have a tight range
     axes_ranges = dict()
-    for href, details in item_details.items():
+    for item_id, details in item_details.items():
         best_axis = None
         best_axis_tightness = 1.0e9
         for axis in details.axes.values():
@@ -96,15 +124,18 @@ async def search():
             for value, value_name in best_axis.values.items():
                 min_vp = 1e9
                 max_vp = 0
+                found_any = False
                 for variant, price in details.variants.items():
                     if variant[best_axis.code_name] != value:
                         continue
                     max_vp = max(max_vp, price)
                     min_vp = min(min_vp, price)
+                    found_any = True
 
-                axis_value_ranges[value_name] = (min_vp, max_vp)
+                if found_any:
+                    axis_value_ranges[value_name] = (int(min_vp), int(max_vp))
 
-            axes_ranges[href] = (best_axis.code_name, axis_value_ranges)
+            axes_ranges[item_id] = (best_axis.code_name, axis_value_ranges)
 
     return await render_template(
         "search.html.j2",