#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from urllib.request import urlopen, Request
import urllib.parse
from http.client import HTTPResponse
from random_user_agent.user_agent import UserAgent
from random_user_agent.params import SoftwareName, OperatingSystem, HardwareType
import copy
import re
import sys
from bs4 import BeautifulSoup, element

def remove_html_comments(html: str) -> str:
    """
    Strips HTML comments. See https://stackoverflow.com/a/57996414
    :param html: html string to process
    :return: html string with comments stripped
    """
    result = re.sub(r'(<!--.*?-->)|(<!--[\S\s]+?-->)|(<!--[\S\s]*?$)', "", html)
    return result

def extract_russian_soup(response: HTTPResponse) -> BeautifulSoup:
   new_soup = BeautifulSoup('', 'html.parser')
   # remove HTML comments before processing
   html_str = response.read().decode('UTF-8')
   cleaner_html = remove_html_comments(html_str)
   soup = BeautifulSoup(cleaner_html, 'html.parser')
   # get rid of certain tags to make it lighter
   # to work with
   [s.extract() for s in soup(['head', 'script', 'footer'])]
   for h2 in soup.find_all('h2'):
      for span in h2.children:
         try:
            if 'Russian' in span['id']:
               new_soup.append(copy.copy(h2))
               # capture everything in the Russian section
               for curr_sibling in h2.next_siblings:
                  if curr_sibling.name == "h2":
                     break
                  else:
                     new_soup.append(copy.copy(curr_sibling))
               break
         except:
            pass
   return new_soup
   
def check_excluded_ids(span_id: str) -> bool:
   excluded = ['Pronunciation', 'Alternative_forms', 'Etymology']
   for ex in excluded:
      if re.search(ex, span_id, re.IGNORECASE):
         return True
   return False

def remove_dl_ul(li: element.Tag) -> element.Tag:
   try:
      dl_extract = li.dl.extract()
   except AttributeError:
      pass
      # sometimes citations are presented in <ul> so remove
   try:
      ul_extract = li.ul.extract()
   except AttributeError:
      pass
   return li

def url_from_ru_word(raw_word:str) -> str:
   # strip syllabic stress diacritical marks
   raw_word = re.sub(r'\u0301|\u0300', "", raw_word)
   raw_word = raw_word.replace(" ", "_").strip()
   word = urllib.parse.quote(raw_word)
   return f'https://en.wiktionary.org/wiki/{word}#Russian'

def request_headers() -> dict:
   hn = [HardwareType.COMPUTER.value]
   user_agent_rotator = UserAgent(hardware_types=hn,limit=20)
   user_agent = user_agent_rotator.get_random_user_agent()
   return {'user-agent': user_agent}

if __name__ == "__main__":
   __version__ = 1.0
   
   # accept word as either argument or on stdin
   try:
      raw_word = sys.argv[1]
   except IndexError:
      raw_word = sys.stdin.read()
      
   url = url_from_ru_word(raw_word)
   headers = request_headers()
   
   try:
      response = urlopen(Request(url, headers = headers))
   except urllib.error.HTTPError as e:
      if e.code == 404:
         print("Error - no such word")
      else:
         print(f"Error: status {e.code}")
      sys.exit(1)
   
   # first extract the Russian content because
   # we may have other languages. This just
   # simplifies the parsing for the headword
   new_soup = extract_russian_soup(response)
            
   # use the derived soup to pick out the headword from
   # the Russian-specific content
   definitions = []
   
   # there are cases (as with the word 'бухта' where there are
   # multiple etymologies. In these cases, the page structure is
   # different. We will try both structures.
   
   for tag in ['h3', 'h4']:
      for h3_or_h4 in new_soup.find_all(tag):
         found = False
         for h3_or_h4_child in h3_or_h4.children:
            if h3_or_h4_child.name == 'span':
               if h3_or_h4_child.get('class'):
                  span_classes = h3_or_h4_child.get('class')
                  if 'mw-headline' in span_classes:
                     span_id = h3_or_h4_child.get('id')
                     # exclude any h3 whose span is not a part of speech
                     if not check_excluded_ids(span_id):
                        found = True
                     break
         if found:
            ol = h3_or_h4.find_next_sibling('ol')
            if ol is None:
               continue
            lis = ol.children
            for li in lis:
               # skip '\n' children
               if li.name != 'li':
                  continue
               # remove any extraneous detail tags + children, etc.
               li = remove_dl_ul(li)
               li_def = li.text.strip()
               definitions.append(li_def)
   definition_list = '; '.join(definitions)
   # if a definition has a single line, remove the ;\s
   definition_list = re.sub(r'^(?:;\s)+(.*)$', '\\1', definition_list)
   # remove "see also" links
   definition_list = re.sub(r'\(see also[^\)]*\)+', "", definition_list)
   print(definition_list)

echo "собака" | ruendef # or
ruendef "собака"