
    jfM-                         d dl Z d dlZd dlmZ d dlZd dlZd dlZd dlZd dlZdedefdZ	d Z
d Zd Z	 dd	Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zy)    N)datetimeall_sessionsreturnc                 8   g }t        j                  d      }|j                  j                  d      }|D ]b  }|j	                  dd      }|j                  d      }|d   dk7  s/d|d   z   dz   |d	   z   d
z   |d   z   dz   |d   z   }|j                  |       d |S )Nzthttps://proxy.webshare.io/api/v2/proxy/list/download/hdaovifqwgapnzijunmiptygnyrtyqaeyvvvqgdo/-/any/username/direct/
 :r   zhttp://      @   )requestsgettextrsplitreplacesplitappend)r   ip_addressesresponsesplittedipclean_ipsplitted_ipfull_ips           -/var/www/highfloat_scraper/HelperFunctions.pygetWebshareProxiesr   
   s    L||~H }}##D)H )::dB'nnS)q>Ra.! a.! 	
 a.!  a.!  ()      c                    	 t        j                  ddddd      }|j                         }|j                  d| |f       |j	                          |j                          |j                          y y # t        t         j                  f$ r!}t        dt        |      z          Y d }~Id }~ww xY w# |j                          w w xY w)NpostgresBerufsorientierung1!23.88.122.576432databaseuserpasswordhostportz5INSERT INTO floats(inspect_link, rank) VALUES(%s, %s)zinsertShittyRankSupaBase(): )
psycopg2connectcursorexecutecommitclose	ExceptionDatabaseErrorwriteErrorToLogstr)inspect_linkrankpostgresql_connpostgresql_curerrors        r   insertShittyRankSupaBaser:   #   s    $"**+
 )//1C	
 	  &!!# ' x--. E6UCDDE &!!# 's*   AA4 4B.B)$B1 )B..B1 1Cc                    d}d}d}|s5t        j                  d       	 |j                  |       j                         }|dz  }d|v rd|d   v rd}|d   S t	        dt        |       z          g|d	kD  rt        j                  d
d
ddd      }|j                         }d|v r|j                  d|d   f       n|j                  d|d   f       	 |j                         d   }	|j                         }|j                  d|	f       |j                          |dz  }|d	kD  rd}t	        dt        |       z          3|s4y y #  Y >xY w# t        $ r d}t	        dt        |       z          Y fw xY w)NFr   r   r   iteminfofloatidTz7iteminfo not in r_json with single_inspect_server_url:    r!   r"   r#   r$   r%   itemidz(SELECT floatid FROM history WHERE a = %saz-FLOATID NOT IN AFTER 5 TRIES, GIVING UP WITH z$DELETE FROM items WHERE floatid = %sz-EXECUTED DB QUERY MULTIPLE TIMES, SKIPPING - )timesleepgetRequestNakedjsonr3   r4   r+   r,   r-   r.   fetchone	TypeErrorr/   )
single_inspect_server_url	bulk_at_jrequest_managerend_floatidcounter_floatidcounter_execr_jsonr7   r8   r=   s
             r   floatidNotInBulkrN   =   s   KOL

1	$445NOTTVF 	1F:.."j)) I/01 Q&..#/#O -335N9$&&>8AT@V &&>3@Q(113A6 -335N""#IG:V""$ALa"C345 s 	B  "C345 s   E  E  E"E.-E.c                    d| d   vr$t        | d         }t        | d   d         }d}d}n$t        | d         }| d   d   }| d   d   d   d   }t        |      j                  d	      }t        |d
         dd  }dt        |      z   dz   t        |      z   dz   t        |      z   }d| v r#t        | d         dz  }t        | d         dz  }	n"t        | d         dz  }t        | d         dz  }	||	z   }
t        |
d      }
|||||
|dS )Nmarket_actionsasset	listingididr	   zZsteam://rungame/730/76561202255233023/+csgo_econ_action_preview%20M%listingid%A%assetid%D0r   link%r   Csteam://rungame/730/76561202255233023/+csgo_econ_action_preview%20MADconverted_priced   converted_feepricefeer   )rT   mr@   dr]   high_lowr4   r   floatround)listing_infora   m_parama_paramd_paramfull_inspect_linkspliced_inspect_linkfinal_inspect_linkr]   r^   
full_prices              r   buildListingInfoListrm   }   sx   |G44 l;/0l7+D12xl;/0w'-(12BCAFvN0177<&r*+AB/GM
g,	
	 g,	 		
 g,	  L(l#456<L12S8 l7+,s2L'(3.Jz1%J # r   c                 <   t        ||         }| d   |   d   d   }d| d   |   d   vry | d   |   d   d   d   d   }t        |      j                  d      }t        |d         d	d  }d
t        |      z   dz   t        |      z   dz   t        |      z   }	d| d   |   v r/t        | d   |   d         dz  }
t        | d   |   d         dz  }n.t        | d   |   d         dz  }
t        | d   |   d         dz  }|
|z   }t        |d      }|du rd|	iS |	||||dS )NlistinginforQ   rS   rP   r   rT   rU   rV   r   rW   rX   rY   rZ   r[   r\   r]   r^   r   T)rT   r_   r@   r`   r]   rb   )listings_page_json	temp_listtemp_list_entryinspect_link_onlyrf   rg   ri   rj   rh   rk   r]   r^   rl   s                r   buildListParamsrt      s    )O,-G /8A$GG1-@I'RR*=9'B7K		 0177<&r*+AB/GM
g,	
	 g,	 		
 g,	  .}=gFF$]3G<=NOPSVV 	 &}5g>OPSVV (7@IJSP&}5g>uEFLJz1%JD *++ '
 	
r   c           
         dg ig}| D ]9  }	 |d   d   j                  dt        |j                               d   d   i       ; |d   S # t        $ r}	 t	        t        |              t	        t        t        |j                                            t	        t        |             t	        dt        |      z          Y d }~# t        $ r%}t	        dt        |      z          Y d }~Y d }~d }~ww xY wd }~ww xY w)Nlinksr   rT   zgetSplittedBulkList():zEXCEPT HELL: )r   listvaluesr1   r3   r4   )splitted_bulkbulk_inspect_links_listies       r   getSplittedBulkListr}      s     '}o 	#A&w/66ahhj)!,V45  #1%%  		M 23D$4 56$; <= 83q6 AB #a& 89		s6   6A			C8A*C	C0C+"C3+C00C33C8c                     t         j                  j                  |       }|j                  dd      }|j                  dd      }|S )N%28(%29)urllibparsequoter   full_item_nameitem_encodeds     r   encodeItemNamer      s@    <<%%n5L''s3L''s3Lr   c                     t         j                  j                  |       }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|S )	Nr   r   r   r   |z%7C&z%26r   r   s     r   encodeAidsItemNamer      sd    <<%%n5L''s3L''s3L''U3L''U3Lr   c                 x    d }| D ]  }|d   |d   k(  s|d   }|c S  |d k(  rt        dt        |      z          yy )Nr_   r]   zNO PRICE WITH: r   r3   r4   )listings_info_listbulk_response_json_on_jr]   r{   s       r   getPricer      s[    E S6,S11gJEL })C0G,HHI r   c                 x    d }| D ]  }|d   |d   k(  s|d   }|c S  |d k(  rt        dt        |      z          y y )Nr_   rT   zNO INSPECT LINK WITH: r   )r   r   r5   r{   s       r   getRealInspectLinkr   	  s]    L  S6,S11V9L  t037N3OOP r   c                 ^    dt        |       z   dz   t        |      z   dz   t        |      z   }|S )Nz/https://steamcommunity.com/market/listings/730/z#buylisting|z|730|2|r4   )r   r_   r@   market_links       r   generateMarketLinkr     sM    9
l
	
	 a&	 		
 a&	  r   c                     t        |       dk7  r.dt        |       z   dz   t        |      z   dz   t        |      z   }|S dt        |      z   dz   t        |      z   dz   t        |      z   }|S )N0rW   rX   rY   r   )r_   sr@   r`   r5   s        r   generateInspectLinkr      s    
1v}Q!f !f 	
 !f 	$  R!f !f 	
 !f 	 r   c                 d    t        |       j                  d      d   j                  d      d   }|S )Nz%20r   rX   r   )r4   r   )r5   m_ss     r   returnMorSfromInspectLinkr   8  s1    
l

!
!%
(
+
1
1#
6q
9CJr   c                     t        j                  d|       }|8|j                         }t        j                  ddt	        |            }|y d|z   S y )Nz"Market_LoadOrderSpread\( [0-9]+ \)z\Dr	   zhhttps://steamcommunity.com/market/itemordershistogram?country=FR&language=german&currency=3&item_nameid=)researchgroupsubr4   )listing_source_pagexaidsnameids       r   manuallyGetNameIDURLr   =  sZ    
		68KLA}wwyb#d),> { r   c                     t        dd      5 }| |j                         v r
	 d d d        y	 d d d        y# 1 sw Y   y xY w)Nzexcluded_skins.txtrTF)openread)skin_name_with_conditionfs     r   checkIfSkinExcludedr   N  sD    	"C	( A#qvvx/  	  s   44=c                     t        ddd      }|j                  t        j                  | d             |j                  d       |j	                          y )Nzcheap_top_skins.txtr@   utf-8encodingr>   indent,)r   writerD   dumpsr0   )single_cheap_top_dictf_cheaps     r   writeSingleCheapSkinToFiler   V  s>    (#@GMM$**21=>MM#MMOr   c                     t        t        |      dd      }|j                  t        j                  | d             |j                  d       |j                          y )Nr@   r   r   r>   r   r   )r   r4   r   rD   r   r0   )single_dictfilenamer   s      r   writeSingleAnyToFiler   ]  s@    3x=#8GMM$**[34MM#MMOr   c                 (   t        j                         }|j                  d      }t        ddd      }|j	                  t        |      dz   t        |       z          |j	                  d       |j                          t        j                  d       y )	Nz%d/%m/%Y %H:%M:%Sz
errors.txtr@   r   r   z: r   g      ?)	r   nowstrftimer   r   r4   r0   rA   rB   )r9   r   	dt_stringr   s       r   r3   r3   d  sg    
,,.C01I\31AGGC	NT!CJ./GGDMGGIJJsOr   c                     t        t        |       dd      }|j                  t        |      dz          |j                          y )Nr@   r   r   r   )r   r4   r   r0   )r   logr   s      r   writeAnythingToAnyFiler   n  s1    S]C'2AGGCHtOGGIr   )T)r   rD   r   rA   urllib.parser   r+   r   rw   r   r:   rN   rm   rt   r}   r   r   r   r   r   r   r   r   r   r   r   r3   r    r   r   <module>r      s    	       T d 2$4=@*\ GK+
\&*	0
"r   