+
    حi              
       x   R t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RI	H
t
 ^ RIHt ^ RIt^ RIHtHtHtHtHtHt RRRR	R
^/t0 R%mt. R&Ot. R'Ot. R(Ot. R)Ot. R*OtR tR tR tR t ]! ]!4      PD                  t#RR]R,           R]R,           2RR]R,           R]R,           2RR]R,           R]R,           2/t$RRRRRR/t%R t&R t'R t(R]R]/t)R ]R!] R"](/t*]+! ])PY                  4       4      ]+! ]*PY                  4       4      ,           t-R# t.]/R$8X  d   ]P`                  ! ].! 4       4       R# R# )+a  
Search property platforms for new listings matching Cyber Prairie criteria.

Discovers new properties and adds them to the store for pipeline processing.
Supports: Immobiliare.it, Green-Acres, Leggett, Properstar.

Usage:
    python3 search_properties.py                           # Search all platforms
    python3 search_properties.py --platform immobiliare    # Single platform
    python3 search_properties.py --dry-run                 # Show URLs without saving
    python3 search_properties.py --limit 20                # Max new properties per platform
    python3 search_properties.py --pages 3                 # Max pages per region search
N)datetime)unescape)Path)loadpersistupsertbrowser_pagewait_for_cloudflare	CHROME_UA	price_mini 	price_maxi bedrooms_minitcomptc           !        . p\        V P                  4       4      p\        P                  ! 4       pVP                  P                  R\        RR/4       \         EFB  w  rgV'       d   \        V4      V8  d    V# \        RV R24       \        ^V^,           4       EF  pR\        R,           R\        R	,           R
\        R,           R2p	V^8  d   V	RV 2,          p	RV RV	 2p
 VP                  V
^R7      pVP                  ^8w  d   \        RV RVP                   24        EMZ\        P                  ! RVP                   \        P"                  4      pV'       g   \        RV R24        EM\$        P&                  ! VP)                  ^4      4      pVP                  R/ 4      P                  R/ 4      P                  R/ 4      P                  R. 4      pV'       g    EMV^ ,          P                  R/ 4      P                  R/ 4      P                  R. 4      pV'       g   \        RV R24        EMK^ pV EF  pVP                  R/ 4      pVP                  R/ 4      pVP                  RR 4      pV'       d   VV9   d   KK  VP                  R!/ .4      ^ ,          pVP                  R"/ 4      pVP                  R#/ 4      pVP                  R$R 4      pR%pV'       dJ   \        P                  ! R&VP+                  R'R 4      4      pV'       d   \-        VP)                  ^4      4      pR%pVP                  R(4      pV'       d    \-        V4      pR%pVP                  R)4      pV'       d    \-        V4      pVP                  R*/ 4      P                  R+R 4      pVP3                  4       \4        9   d   EKz  VP                  R,/ 4      p V P                  R-4      ;'       g    / P                  R.R 4      p!VP7                  RVR/R0R1VP                  R1R 4      R#VP                  R24      R3VP                  R34      R4VP                  R5/ 4      P                  R6R74      R8VP                  R94      R:VP                  R;4      R<VR=VR)VR>VP                  R*/ 4      P                  R+4      R?V!R@V/4       VP9                  V4       V^,          pV'       g   EK  \        V4      V8  g   EK   M	  \        RV RA\        V4       RBV RC24       V'       d   \        V4      V8  d    MdV^ ,          P                  R/ 4      P                  R/ 4      P                  RD^ 4      p"V^,          V"8  d    M\:        P<                  ! ^4       EK  	  \:        P<                  ! ^4       EKE  	  V#   \.        \0        3 d     ELci ; i  \.        \0        3 d     ELTi ; i  \>         d.   p#\        RET RA\A        T#4      RF,           24        R%p#?# K  R%p#?#ii ; i)Gz8Search immobiliare.it via __NEXT_DATA__ JSON extraction.
User-AgentAccept-Languageenz  Searching Immobiliare.it: ...zprezzoMinimo=r   z&prezzoMassimo=r   z&localiMinimo=r   z &idTipologia=7,23,24,28,29,31,33z&pag=z+https://www.immobiliare.it/en/vendita-case/z/?timeoutz	    Page z: HTTP z.<script id="__NEXT_DATA__"[^>]*>(.*?)</script>z: no __NEXT_DATA__ foundprops	pagePropsdehydratedStatequeriesstatedataresultsz: no results
realEstateseourl 
propertieslocationpricesurfaceNz(\d+),bedRoomsNumberroomstypologynamephotourlssmallsourceimmobiliaretitlevaluecitycountrynationidITlatlatitudelon	longitudebuilding_sizebedroomsproperty_type	thumbnailsearch_region:  listings,  newcountz    Error on page NP   N)!setkeysreqSessionheadersupdater
   REGIONS_IMMOBILIARElenprintrangeCRITERIAgetstatus_coderesearchtextDOTALLjsonloadsgroupreplaceint
ValueError	TypeErrorlower
SKIP_TYPESappendaddtimesleep	Exceptionstr)$storelimit	max_pagesnew_urlsexistingsessionregion_slugregion_namepage_numparamsr!   respmatch	next_datar   r   	batch_newrre_datar    prop_urlr   loc
price_datar&   building_m2mr=   beds_strr)   	rooms_strr*   r,   thumbtotales$   &&&                                 Y/Users/jonathan/Documents/Zakelijk/ClaudeOS/03_Lab/farmmatch/scraper/search_properties.pysearch_immobiliarer      sd   H5::< HkkmGOOL)5FMN$7$7 S]e+r Oo 	,[M=>aQ/H &h{&;%<OHU`LaKb&x'?&@8:F !|E(,,?}BvhWCf{{3{3##s*IhZwt7G7G6HIJ 		"SUYU^U^`b`i`ijIhZ/GHI JJu{{1~6	#--488bIMMN_acdhhirtvw!!*.."599&"EII)UWXIhZ|<=	 AeeL"5G%%r*C"wwub1H#x8'; #KKrd;A>E))J3C!(Wb!9J $ii	26G"&KIIhR0HI*-aggaj/K#H$yy)9:H!'*8}H !E %		' 2I !$'	NE
  '{{:r:>>vrJH~~':5 !IIgr2E"YYv.44"99'2FEOOx -Wb!9!8!3778R#8#<#<T4#Hswwz2sww{3'"H'Z)D)H)H)P#U'%   LL*NIuX%!7 !B 	(2c'l^;ykQUVWS]e3  
w377CGGQRSb=E)

1[ 0f 	

1s %8v Ow !+I6 ! ! !+I6 ! !Z  *8*Bs1vc{mDEs   *?W,7W$W6A.W(A W)W;B(W$.W4W	V#W0V;;A'W#C*WW&:W"AW+W#V8	4W7V8	8W;W	WW	WX!XXc                  a* . p\        V P                  4       4      p\        P                  ! 4       pVP                  P                  R\        RR/4       \         EF-  w  rgpV'       d   \        V4      V8  d    V# \        RV RV R24       RV RV 2p	 VP                  V	^R	7      p
V
P                  ^8w  d   \        R
V
P                   24       K{  V
P                  p\        P                  ! R\        P                  4      pVP!                  V4      pV'       g   \        R4       K  ^ pV EF!  w  pp \"        P$                  ! V4      P'                  R4      o*S*'       d   S*T9   d   K>  \*        ;QJ d    T*3R lR> 4       F  '       g   K   RM	  RM! T*3R lR> 4       4      '       d   K  Rp\        P,                  ! RS*4      pT'       d0   TP/                  ^4      P1                  RR4      P3                  4       pTP5                  RT R24      pT^ 8  d   K  TP5                  RT4      pT^ 8  g   TTR,           8  d   RpM\7        TTTR,            4      pRpT'       dZ   \        P,                  ! RT4      pT'       d;   \        P8                  ! RRTP/                  ^4      4      pT'       d   \;        T4      pT'       d(   T\<        R,          8  g   T\<        R,          8  d   EK  T'       dI   \        P,                  ! RT4      pT'       d*   TP/                  ^4      P?                  4       pT'       d   TpRpRpRpRpT'       EdO   \        P,                  ! R T4      pT'       d1   \;        \        P8                  ! RRTP/                  ^4      4      4      p\        P,                  ! R!T4      p T '       do   T P/                  ^4      P1                  R"R4      P?                  4       p! \A        T!4      p"R#T P/                  ^4      9   d   \;        T"R$,          4      M
\;        T"4      p\        P,                  ! R%T4      p#T#'       d   \;        T#P/                  ^4      4      p\        P,                  ! R&T4      p$T$'       d   \;        T$P/                  ^4      4      pRp%TTTR',            p&\        P,                  ! R(T&4      p'T''       d   T'P/                  ^4      p%TR)8X  d   R*MR+p(TPE                  R,S*R-R.R/T;'       g    TR0TR1TR2T(R3TR4TR5TR6TR7T%R8T/4       TPG                  S*4       T^,          pT'       g   EK  \        T4      T8  g   EK"   M	  \        R9\        V4       R:V R;24       \J        PL                  ! ^4       EK0  	  V#   \(         d     EKm  i ; i  \B         d     ELi ; i  \(         d)   p)\        R<\I        T)4      R=,           24        Rp)?)LpRp)?)ii ; i)?z2Search green-acres.fr/it via server-rendered HTML.r   r   r   z  Searching Green-Acres.rA   r   zhttps://www.green-acres.z/property-for-sale/r   z	    HTTP zT<div[^>]*class="announce-card[^"]*"[^>]*data-advertid="([^"]*)"[^>]*data-o="([^"]*)"z    No listing cards foundzutf-8c              3   H   <"   T F  qSP                  4       9   x  K  	  R # 5i)N)r_   ).0skiprw   s   & r   	<genexpr>$search_greenacres.<locals>.<genexpr>\  s     t<sDx~~//<ss   "TFNz$/properties/[^/]+/([^/]+)/[^/]+\.htm- zdata-advertid=""zannounce-infoi:  r"   i  zinfo-price[^>]*>([\d,. ]+)z[^\d]r   r   z%announce-localisation[^>]*>\s*([^<]+)z(title="Living area"[^>]*>\s*(\d[\d,. ]*)z-title="Land"[^>]*>\s*([\d,. ]+)\s*(m|hectare)r'   hectare'  ztitle="Rooms"[^>]*>\s*(\d+)ztitle="Bedrooms"[^>]*>\s*(\d+)  z#announce-card-img[^>]*src="([^"]+)"frFRr7   r!   r/   
greenacresr1   r%   r3   r4   r<   	land_sizer)   r=   r?   r@   
    Found rB   rC       Error: rE   )business-premises
commercialofficegarage)'rG   rH   rI   rJ   rK   rL   r
   REGIONS_GREENACRESrN   rO   rR   rS   rV   rT   compilerW   findallbase64	b64decodedecodere   anyrU   rZ   r[   r1   findr   subr\   rQ   stripfloatr]   ra   rb   rf   rc   rd   )+rg   rh   ri   rj   rk   rl   country_tldrm   rn   r!   rq   htmlcard_patterncardsrt   	advert_iddata_or3   	url_match
card_start
info_start	info_htmlr%   price_matchdigits	loc_matchloc_textrz   land_m2r)   r=   
area_match
land_matchval_strvalrooms_match
beds_matchr~   	card_html	img_matchcountry_coder   rw   s+   &&&                                       @r   search_greenacresr   /  s3   H5::< HkkmGOOL)5FMN1C1C-+S]e+Z OW 	(R}CHI(5HVC	/;;sB;/D3&	$"2"2!34599D ::g		L
 !((.E24I%*!	6%//7>>wGH  8x#7 3t<st333t<sttt II&MxX	$??1-55c3?EEGD "YY1'EF
>!YY
C
>Z*u2D%D "I (jd9J)K LI "$)),I9"UK"!#"k6G6G6J!K!$'KE eh{&;;uxP[G\?\  "		*RT] ^I #,??1#5#;#;#=##+D #9!#+VXa!bJ!&)"&&2z?O?OPQ?R*S&T!#+[]f!gJ!","2"21"5"="=c2"F"L"L"N!"'.C:CzGWGWXYGZ:Zc#+&6`cdg`hG #%)),JI"VK" #K$5$5a$8 9!#+Li!XJ!#&z'7'7':#;  J,=>	II&LiX	%OOA.E'2d':t8lT00[UD|#[U#[!  X&Q	5S]e3U &+X Js5zl+i[EF
 	

1[ 2D^ Oa ! B  * ! !T  	/KAs}-..	/s   <V4A	V4 V4-V49%VV4&V4:V4V4/"V4BV4(V4.V46V4	.V48V4&V4>V4V40AV4?/V4/>V"-V48V4AV44V49V4 V4V4VV4VV4"V1-V40V11V44W'?W""W'c                  "   . p\        VP                  4       4      p\         EF  w  rgV'       d   \        V4      V8  d    V# RV R\        R,           R\        R,           R\        R,           R2	p\        R	V R
24        V P                  VRRR7      G Rj  xL
  V P                  R4      G Rj  xL
  \        V RR7      G Rj  xL
 '       g   \        RV R24       K  V P                  R4      G Rj  xL
 p	^ p
V	 F  pVP                  RR4      pV'       d   W9   d   K%  VP                  R4      pV'       d'   V\        R,          8  g   V\        R,          8  d   Kd  V
^,          p
VP                  RVRRRVRVP                  R4      RVRR/4       VP                  V4       V'       g   K  \        V4      V8  g   K   M	  \        R\        V	4       R V
 R!24       V P                  R$4      G Rj  xL
  EK  	  V#  ELl ELV ELD EL  \         d)   p\        R"\        T4      R#,           24        Rp?LZRp?ii ; i LM5i)%z6Search frenchestateagents.com for matching properties.z?https://www.frenchestateagents.com/french-property-for-sale/in/z/?min_price=r   z&max_price=r   z&min_bedrooms=r   z&type=house,farmz  Searching Leggett: r   domcontentloaded0u  
wait_untilr   Ni  r   )
timeout_msz    Cloudflare blocked. Skip .a  () => {
                const results = [];
                // Try multiple selector patterns for robustness
                const anchors = document.querySelectorAll('a[href*="/french-property-for-sale/view/"]');
                const seen = new Set();
                for (const a of anchors) {
                    const url = a.href;
                    if (seen.has(url)) continue;
                    seen.add(url);

                    // Walk up to find the card container for price/thumb
                    let card = a.closest('article, .card, .property-card, .listing-card, li');
                    let price = null;
                    let thumb = null;

                    if (card) {
                        const priceEl = card.querySelector('.price .new-price, .price, [class*="price"]');
                        if (priceEl) price = parseInt(priceEl.textContent.replace(/[^0-9]/g, '')) || null;
                        const img = card.querySelector('img[src*="http"]');
                        if (img) thumb = img.src;
                    }

                    results.push({ url, price, thumb });
                }
                return results;
            }r!   r"   r%   r/   leggettr?   r~   r@   r4   r   r   rB   rC   r   rE     )rG   rH   REGIONS_LEGGETTrN   rQ   rO   gotowait_for_timeoutr	   evaluaterR   ra   rb   re   rf   )pagerg   rh   ri   rj   rk   rm   rn   r!   itemsrt   itemrw   r%   r   s   &&&&           r   search_leggettr     s>    H5::< H$3O S]e+X OU M "%k23;x?T>U 89!	# 	%k]#67>	/))C,>)NNN''---,TeDDD5k]!DE-- )  E6 I88E2.8#7)eh{&;;uxP[G\?\Q	8iU'!2#[t!  X&5S]e3+ . Js5zl+i[EF
 ##D)))Y %4\ OC O-Dj  	/KAs}-..	/ 	*s   BIHHH5H6H
H	HH$I&H:H;AHA1H7H	H(I<I
=IHHHHII=IIIc                  "   . p\        VP                  4       4      p\         EFn  w  rgV'       d   \        V4      V8  d    V# RV R\        R,           R\        R,           R\        R,           R2	p\        RV R	24        V P                  VR
RR7      G Rj  xL
  V P                  R4      G Rj  xL
  V P                  R4      G Rj  xL
 p	^ p
V	 F  pVP                  RR4      pV'       d   W9   d   K%  V
^,          p
VP                  RVRRRVP                  R4      RV/4       VP                  V4       V'       g   Kq  \        V4      V8  g   K   M	  \        R\        V	4       RV
 R24       V P                  R4      G Rj  xL
  EKq  	  V#  L L L  \         d)   p\        R\        T4      R,           24        Rp?LTRp?ii ; i LG5i)z-Search properstar.nl for matching properties.zhttps://www.properstar.nl/z/kopen/huis?budget=r   r   r   z
&bedrooms=r   z  Searching Properstar: r   r   r   r   Nr   a  () => {
                const results = [];
                const seen = new Set();
                // Properstar listing links
                const anchors = document.querySelectorAll('a[href*="/listing/"]');
                for (const a of anchors) {
                    const url = a.href;
                    if (seen.has(url) || !url.includes('/listing/')) continue;
                    seen.add(url);

                    let card = a.closest('article, [class*="card"], [class*="listing"]');
                    let price = null;

                    if (card) {
                        const priceEl = card.querySelector('[class*="price"], [itemprop="price"]');
                        if (priceEl) price = parseInt(priceEl.textContent.replace(/[^0-9]/g, '')) || null;
                    }

                    results.push({ url, price });
                }
                return results;
            }r!   r"   r/   
properstarr%   r@   r   rB   rC   r   rE   r   )rG   rH   REGIONS_PROPERSTARrN   rQ   rO   r   r   r   rR   ra   rb   re   rf   )r   rg   rh   ri   rj   rk   rm   rn   r!   r   rt   r   rw   r   s   &&&&          r   search_properstarr   "  s    H5::< H$6$6 S]e+x Ou ,K= 9$$,[$9#:!H[<Q;RH^45Q8 	(S9:0	/))C,>)NNN''----- )  E. I88E2.8#7Q	8lTXXg.#[	!  X&5S]e3 " Js5zl+i[EF
 ##D)))y %7| Og O-V  	/KAs}-..	/ 	*s   BGFFF5F6FFA4FFF9GGGFFFG*GGGGzprezzo-min_z,prezzo-max_zprecio-min_z,precio-max_z
preco-min_z,preco-max_z
/immobile/z
/inmueble/z/imovel/c                J  "    V P                  R4      G Rj  xL
 pRT9   ;'       g    RTP                  4       9   pT P                  R4      G Rj  xL
 ^ 8  pT'       d   T'       g   \        RT 24       \        R4       \	        ^(4       F  pT P                  R	4      G Rj  xL
   T P                  R4      G Rj  xL
 pRT9  dd   \        R
4        TP                  \        T4      R7      G Rj  xL
  \        RTP                   24       T P                  R4      G Rj  xL
   R#  T^,          ^8X  g   K  \        R4       K  	  \        RT 24       R# R#  ELG  \         d     R# i ; i EL# L L L  \         d     Lyi ; i Lh  \         d     EK  i ; i5i)zTDetect DataDome CAPTCHA and wait for manual solving. Returns True if page is usable.z;() => document.documentElement.outerHTML.substring(0, 3000)NFzcaptcha-delivery.comdatadomez1() => document.querySelectorAll("article").lengthz.      DataDome CAPTCHA detected for idealista.z7      Please solve the CAPTCHA in the browser window...r   z      CAPTCHA solved!pathz      Cookies saved to r   Tz(      Still waiting for CAPTCHA solve...z*      CAPTCHA timeout, skipping idealista.)	r   re   r_   rO   rP   r   storage_staterf   r+   )	r   contextdomain	auth_filer   has_captchahas_contentattemptnew_htmls	   &&&&     r   _idealista_check_captchar   {  s    ]]#`aa )D0NNJ$**,4NK&YZZ]^^K;>vhGHGIRyG''---!%/l!mm)913%33Y3HHH 7	7GHI //555 : {a@B# !$ 	:6(CDA b  [ .m
 I$ 5 s   F#E" EE" F#(F#E4F#!F#);F#$E7%F#*F>E9?FE=4E;5E=F%F&F*F#>!F#E" "E1-F#0E11F#7F#9F;E==FF
FFF F#F  F#c                  "   VP                   pRVP                  4       9   g+   RVP                  4       9   g   RVP                  4       9   d   \        RV 24       \        R4       \        ^<4       F  pVP	                  R4      G Rj  xL
  VP                   pRVP                  4       9  d-   RVP                  4       9  d   RVP                  4       9  d    M\        RVR	,           R
24       K  	   V P                  \        V4      R7      G Rj  xL
  \        RVP                   24       R# R#  L L"  \         d     R# i ; i5i)zACheck if logged in, prompt for manual login if needed, save auth.loginaccedientrarz!    Login required for idealista.z*    Please log in in the browser window...r   Nz     Still waiting for login... (:N2   N)r   z    Auth saved to )	r!   r_   rO   rP   r   r   rf   r+   re   )r   r   r   r   current_url_s   &&&&  r   _idealista_ensure_authr     s1    ((K+##%%[5F5F5H)HHXcXiXiXkLk1&:;:<rA''---((Kk//11hkFWFWFY6Y^fnynn  oB  _B4[5E4FaHI 	''S^'<<<&y~~&678 Ml
 . = 		sI   BE	D5
A+E6D9 D7D9 1E7D9 9EEEEc                  "   ^ RI Hp . p\        VP                  4       4      p/ p\         F)  w  rrVP                  V. 4      P                  WV34       K+  	  V! 4       ;_uu_4       GRj  xL
 pVP                  4        EF  w  rV'       d   \        V4      V8  d    EMz\        RV R2,          p\        V,          p\        V,          p\        RV 2,          pVP                  RR7       \        RV R	\        V4       R
24       VP                  P                  \!        V4      RRRR7      G Rj  xL
 pVP"                  '       d   VP"                  ^ ,          MVP%                  4       G Rj  xL
 p V EF  w  rpV'       d   \        V4      V8  d    EMcRV RV	 2pV RV R2p\        RV
 R24       \'        ^V^,           4       EF  pV^8X  d   TMV RV R2p VP)                  VRRR7      G Rj  xL
  VP+                  R4      G Rj  xL
  \-        VVW4      G Rj  xL
 '       g   \        RV R24        EM VP/                  R4      G Rj  xL
 pRVP3                  4       9   g+   RVP3                  4       9   g   RVP3                  4       9   d   \        R V R!24        EM0R"VP4                  P3                  4       9   g    R#VP4                  P3                  4       9   dK   \7        VVW4      G Rj  xL
  VP)                  VRRR7      G Rj  xL
  VP+                  R$4      G Rj  xL
  VP9                  RR%4      pVP/                  R&V4      G Rj  xL
 p^ pV EF  pVP;                  R'R4      pV'       d   VV9   d   K'  VP;                  R(4      pV'       d'   V\<        R),          8  g   V\<        R*,          8  d   Kf  V^,          pVP                  R'VR+R,R(VR-VP;                  R.4      R/VP;                  R/4      R0VP;                  R14      R2VP;                  R34      R4VP;                  R44      R5V
R6V/
4       VP?                  V4       V'       g   K  \        V4      V8  g   EK   M	  \        R7V R8\        V4       R9V R:24       V'       d   V^ 8X  d    M8V'       d   \        V4      V8  d    MTP+                  R$4      G Rj  xL
  EK  	  VP+                  R=4      G Rj  xL
  EK  	  VPA                  4       G Rj  xL
  EK  	  RRR4      GRj  xL
  V#  EL EL EL EL< EL& EL EL  \0         d    Rp ELi ; i ELX EL? EL) EL   \0         d.   p\        R;T R8\!        T4      R<,           24        Rp? K  Rp?ii ; i L L L  TPA                  4       G Rj  xL 
  i ; i L  + GRj  xL 
 '       g   i     T# ; i5i)>zSearch idealista.it/.com/.pt for matching rural properties.

Uses patchright (undetected Playwright fork) with persistent context
per domain to bypass DataDome bot detection. The shared `page` argument
is ignored, we launch our own patchright browser.
)async_playwrightNidealista_auth_z.jsonz.idealista_profile_T)exist_okz
  idealista.z (z	 regions)chromeF)user_data_dirchannelheadlessno_viewportzhttps://www.idealista./r'   z    Searching: r   zpagina-z.htmr   r   r   i  z-      Blocked by CAPTCHA, skipping remaining z regionszD() => document.body ? document.body.innerText.substring(0, 200) : ""r"   zuso improprioblockedbloccatoz      IP blocked by idealista.z
, skippingr   r   r   z\/a  (pattern) => {
                                const results = [];
                                const seen = new Set();
                                const re = new RegExp(pattern);
                                const articles = document.querySelectorAll('article.item, article[data-adid], .item-info-container');
                                for (const art of articles) {
                                    const links = art.querySelectorAll('a[href]');
                                    for (const a of links) {
                                        if (!re.test(a.href) || seen.has(a.href)) continue;
                                        seen.add(a.href);
                                        const priceEl = art.querySelector('.item-price, .price-row, [class*="price"] h3, [class*="price"]');
                                        const price = priceEl ? parseInt(priceEl.textContent.replace(/[^0-9]/g, '')) || null : null;
                                        const locEl = art.querySelector('.item-detail-char .item-detail, .item-description');
                                        const location = locEl ? locEl.textContent.trim().substring(0, 80) : null;
                                        const titleEl = art.querySelector('.item-link, a.item-link');
                                        const title = titleEl ? titleEl.textContent.trim() : null;
                                        const img = art.querySelector('img[src*="http"], picture img');
                                        const thumb = img ? img.src : null;
                                        const detailEls = art.querySelectorAll('.item-detail');
                                        let sqm = null, rooms = null;
                                        for (const d of detailEls) {
                                            const t = d.textContent.trim();
                                            const sqmMatch = t.match(/(\d[\d.,]*)\s*m/);
                                            if (sqmMatch) sqm = parseInt(sqmMatch[1].replace(/[.,]/g, ''));
                                            const roomMatch = t.match(/(\d+)\s*(room|habitaci|local|vano)/i);
                                            if (roomMatch) rooms = parseInt(roomMatch[1]);
                                        }
                                        results.push({ url: a.href, price, location, title, thumb, sqm, rooms });
                                        break;
                                    }
                                }
                                if (results.length === 0) {
                                    const allLinks = document.querySelectorAll('a[href]');
                                    for (const a of allLinks) {
                                        if (!re.test(a.href) || seen.has(a.href)) continue;
                                        seen.add(a.href);
                                        results.push({ url: a.href, price: null, location: null, title: a.textContent.trim().substring(0, 80), thumb: null, sqm: null, rooms: null });
                                    }
                                }
                                return results;
                            }r!   r%   r   r   r/   	idealistar3   r$   r1   r?   r~   r<   sqmr)   r@   r4   z      Page rA   rB   rC   z      Error on page rE   r   )!patchright.async_apir   rG   rH   REGIONS_IDEALISTA
setdefaultra   r   rN   IDEALISTA_AUTH_DIRIDEALISTA_LISTING_PATTERNSIDEALISTA_PRICE_FILTERmkdirrO   chromiumlaunch_persistent_contextrf   pagesnew_pagerP   r   r   r   r   re   r_   r!   r   r[   rR   rQ   rb   close) r   rg   rh   ri   patchright_playwrightrj   rk   	by_domainr   url_pathrn   r   pwregionsr   listing_patternprice_fragmentr   r   domain_pagebase_url
search_urlro   page_url	body_textescaped_patternr   rt   r   rw   r%   r   s    &&&&                            r   search_idealistar    s     OH5::< HI7H3+VR(//0UV 8I %&&&"(0OFX%/*vhe-LLI8@O3F;N.3Fvh1OOM.N6("S\N)DEKKAA!-0  	 B  G /6mmm'--*wGWGWGYAYKt&;B7H<X%!7!7xq
KH$,:Q~.>a!@JOK=<=$)!Y]$;19Q:zlRYZbYccgDha""-"2"28HZdi"2"jjj"-">">t"DDD)A+wX^)j#j#j %(UV\U]]e&f g %/2=2F2F  HN  3O  -O	  /)//2CCyT]TcTcTeGeis  xA  xG  xG  xI  jI %(Fvhj&Y Z %&+//*?*?*AAXQ\Q`Q`QfQfQhEh&<WkSY&e e e&1&6&6xL^hm&6&n n n&1&B&B4&H H H.=.E.Ec5.QO*5*>*> (@!P #2Q(+3 (%3ET )*I(-+/88E2+>'/8x3G$,(,(9#(eh{6K.KuW_`kWlOl$, )Q	 ($)8$,k$+U$*DHHZ,@$+TXXg->$/'1B$3TXXe_$+TXXg->$3[$-|1" !# !)X 6#(5S]e-C$)- ).0 "KzCJ<{S\R]]a"bc#(IN %$X%)? % *::4@@@M %<P &66t<<<a <Cf mmo%%Q  1 '&V OW ' BZ kD#j
 -O#, /,.	/ !f n H(%3R  ) "!$8
"SVC[M"RS!" A< &gmmo%%S '&&&V Os  A*W!,T.-W!0CW5T1
6W&W2T4
3W8A+V$$U#
;T7<U#
T:U#
)T=*	U#
4U#
V$U
U U
 AU#
.V$1AU#
?U U#
UU#
4U5+U#
 U !AU#
)B5U#
#U#
61U#
'V$)U#
1U#
V$VV$5V 6	V$?WV"
	WW!'W(W!1W4W7U#
:U#
=U#
 U
UU#
UU#
U#
U#
 U#
#V.!VV$VV$ V$"W$V?	8V;9V?	?WW!W	
W
W	W		W!r0   r   r   r   r   c                    "   \         P                  ! R R7      p V P                  R\        RR7       V P                  R\        ^2RR7       V P                  R\        ^R	R7       V P                  R
RRR7       V P                  RRRR7       V P                  4       p\        4       pVP                  '       d   VP                  .M\        p\        R4       \        R\        V4       R24       \        RRP                  V4       24       \        R\        R,          R R\        R,          R R\        R,           R24       \        RVP                   24       \        4        . pR pV Uu. uF  qf\        9   g   K  VNK  	  ppV F  p\        RB 4       \        RVP                  4        R24       \        RB 4       \        V,          p	V	! W!P                  VP                  4      p
VP!                  V
4       V! W*VP"                  4       \        R \        V
4       R!V R"24       K  	  V Uu. uF  qf\$        9   g   K  VNK  	  ppR#0pV Uu. uF  qfV9  g   K  VNK  	  ppV Uu. uF  qfV9   g   K  VNK  	  ppV'       d   \'        VP(                  R$7      ;_uu_4       GR%j  xL
 pV F  p\        RB 4       \        RVP                  4        R&24       \        RB 4       \$        V,          p	V	! WVP                  VP                  4      G R%j  xL
 p
VP!                  V
4       V! W*VP"                  4       \        R \        V
4       R!V R"24       K  	  R%R%R%4      GR%j  xL
  V F  p\        RB 4       \        RVP                  4        R'24       \        RB 4       \$        V,          p	V	! R%W!P                  VP                  4      G R%j  xL
 p
VP!                  V
4       V! W*VP"                  4       \        R \        V
4       R!V R"24       K  	  \        RB 4       \        R(4       \        RB 4       / p/ pV FV  pVR),          pVP+                  R*R+4      pVP+                  V^ 4      ^,           VV&   VP+                  V^ 4      ^,           VV&   KX  	  \        R,\        V4       24       \-        VP/                  4       R- R.7       F  w  pp\        R/V R0V 24       K  	  \        4        \-        VP/                  4       R1 R.7       F  w  pp\        R/V R0V 24       K  	  VP"                  '       d   \        R24       VR3,           F  pVP+                  R44      '       d   R5VR4,          R 2MR+pVP+                  R64      ;'       g    VP+                  R*R74      p\        RVR),          R8 R9VR: R9VR; RVR<,          R=,           24       K  	  R%# V'       d'   \        R>\        V4       R?\        V4       R@24       R%# \        RA4       R%# u upi u upi u upi u upi  EL EL EL  + GR%j  xL 
 '       g   i     EL; i EL~5i)Cz#Search platforms for new properties)descriptionz
--platformzSearch single platform)choiceshelpz--limitz-Max new properties per platform (default: 50))typedefaultr  z--pagesz(Max result pages per region (default: 3)z	--dry-run
store_truez'Show found URLs without saving to store)actionr  z
--headlessz-Run browser headless (may fail on Cloudflare)u%   SEARCH — Discovering new propertiesz	  Store: z existing propertiesz  Platforms: z, z  Criteria: EUR r   r'   r   r   r   z+ bedsz  Max pages per region: c           	        V'       g	   V'       g   R# \         P                  ! 4       P                  4       pV F\  pRVR,          RVRVP                  R4      /pR	 F"  pVP                  V4      f   K  WF,          WV&   K$  	  \	        WR,          V4       K^  	  \        V 4       \        R\        V4       R\        V 4       R24       R# )
z?Persist batch immediately so partial results survive IP blocks.Nr/   discovered_atr@   r!   z	  (saved z to store,  total))r1   r%   r3   r4   r8   r:   r<   r   r=   r)   r>   r?   )r   now	isoformatrR   r   r   rO   rN   )stbatchdryr  r   fieldskeys   &&&    r   _save_batchmain.<locals>._save_batch  s    elln&&(DX%txx'@BF6 88C=,"&)FK	6
 2E{F+  		#e*[R	AB    z  z (API)z  -> z
 new from 
r   )r   Nz
 (Browser)z (Standalone browser)z  SEARCH COMPLETEr/   r@   ?z  Total new: c                     V ^,          ) #     xs   &r   <lambda>main.<locals>.<lambda>  s
    1Q4%r$  )r!  z    rA   c                     V ^,          ) # r(  r*  r+  s   &r   r-  r.    s
    1r$  u   
DRY RUN — showing first 30::N   Nr%   zEUR r3   r"   13sr   25sz>15sr!   :N<   Nz
Added z new properties to store (r  z
No new properties foundz2==================================================)argparseArgumentParseradd_argumentALL_PLATFORMSr\   
parse_argsr   platformrO   rN   joinrQ   r   API_PLATFORMSupperrh   extenddry_runBROWSER_PLATFORMSr   r   rR   sortedr   )parserargsrg   	platformsall_newr"  p
api_to_runr9  	search_fnrj   browser_to_runstandalone_browsersshared_browser_platformsstandalone_platform_listr   by_platform	by_regionr   ru   c	price_strr3   s                          r   mainrP  i  s    $$1VWF
m5  7
	RL  N
	QG  I
LF  H
\L  NDFE#'===mI	13	Ic%j\!5
67	M$))I./
01	Xk215Qx7LQ6Orn%&f. /	$TZZL
12	GGC$ '=Y}*<!!YJ=
8>>#$F+,
!(+	UJJ

;x ET\\2c(m_Jxj;<  "+EA3D.DaaNE&-+9Z>aFY=Y>Z+9V>aBU=U>V77744
$8>>+,J78
$-h7	!*4

DJJ!OOx(ET\\:c(m_JxjCD 5 87 -
8>>#$$9:;
%h/	"4

DJJGGx ET\\2c(m_Jxj;< - 
XJ		XJKINHH_c*$A.2A }}Q*Q.	!	  
M#g,
(){((*@1QCr!o A	Gy(o>1QCr!o ? |||/1CLLD48HHW4E4E$tG}Q/03I88F#DDtxx'DDBtH~c*!D:Qy6FbeUXIYHZ[\ ! 	W&@UGTU)*U > F  [V 8 P 8777$ Hs   E,Y.X?XB-Y2X X 		YX%X%%Y+X*8X*>,Y*X/+Y.A)X8X2
AX8Y*X5+A,YYE*YAYAY(AY2X85Y8Y	>Y?
Y	
Y	
Y__main__>   flatlandplotshopr   r   studioparkingprojectstorageterrainterraced	apartment	penthouse	townhouser   semi-detachedr   ))marche	Le Marche)umbriaUmbria)toscanaTuscany)liguriaLiguria)piemontePiedmont)abruzzoAbruzzo)pugliaPuglia)calabriaCalabria))r   drome   Drôme)r   ardeche   Ardèche)r   herault   Hérault)r   cotes-d-armor   Côtes-d'Armor)r   morbihanMorbihan)r   	finistere
   Finistère)r   charenteCharente)r   charente-maritimeCharente-Maritime)r   dordogneDordogne)r   gardGard)r   lotLot)r   correze   Corrèze)r   creuseCreuse)r   gersGers)r   lot-et-garonneLot-et-Garonne)r   mayenneMayenne)r   orneOrne)r   indreIndre)r   deux-sevres   Deux-Sèvres)r   sartheSarthe)r   r`  ra  )r   rb  rc  )r   rd  re  )r   rf  rg  ))rz  r{  )rt  ru  )rv  rw  )rx  ry  )rp  rq  )r|  r}  )r~  r  )rr  rs  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  )r  r  ))zfrance/finisterer{  )zfrance/heraultru  )zfrance/charenter}  )zfrance/cotes-d-armorrw  )zfrance/dromerq  )zfrance/ardechers  )zfrance/morbihanry  )zfrance/dordogner  )zitaly/marchera  )zitaly/umbriarc  )zitaly/tuscanyre  )zportugal/alentejoAlentejo)zspain/asturiasAsturias)zspain/galiciaGalicia))r   z'en/geo/vendita-case/toscana/con-rusticire  r7   )r   z&en/geo/vendita-case/umbria/con-rusticirc  r7   )r   z&en/geo/vendita-case/marche/con-rusticira  r7   )r   z'en/geo/vendita-case/liguria/con-rusticirg  r7   )r   z(en/geo/vendita-case/piemonte/con-rusticiri  r7   )r   z'en/geo/vendita-case/abruzzo/con-rusticirk  r7   )r   z&en/geo/vendita-case/puglia/con-rusticirm  r7   )r   z3en/geo/venta-viviendas/asturias/con-casas-de-pueblor  ES)r   z2en/geo/venta-viviendas/galicia/con-casas-de-pueblor  r  )r   z1en/geo/venta-viviendas/aragon/con-casas-de-puebloAragonr  )r   z:en/geo/venta-viviendas/castilla-y-leon/con-casas-de-pueblozCastilla y Leonr  )r   z3en/geo/venta-viviendas/cataluna/con-casas-de-pueblo	Cataloniar  )r   z/en/geo/comprar-casas/alentejo/com-casa-rusticasr  PT)r   z-en/geo/comprar-casas/centro/com-casa-rusticasCentror  )r   z,en/geo/comprar-casas/norte/com-casa-rusticasNorter  )1__doc__r4  asyncior   rX   mathrT   rc   r   r   r   pathlibr   requestsrI   rg   r   r   r   r   r	   r
   rQ   r`   rM   r   r   r   r   r   r   r   r   __file__parentr   r   r   r   r   r  r;  r?  listrH   r7  rP  __name__runr*  r$  r   <module>r     s        	      U U
 A)
	  B8 . 0BNVvSpCP (^**  	K-.l8K;P:Q
R	[+./|H[<Q;RSJx,-[+9N8O
P  	,	<* #L,[D %# ~#!  ]'')*T2C2H2H2J-KK
w+t zKK r$  