import string import socket import memhub import stdio import stdlib import zlib.pyzlib as pyzlib # ============================================================ # requests/__init__.vp - HTTP(S) client library package entry point (Python requests style) # # Package structure: # requests HTTP client (pure socket implementation, this file) # requests.https HTTPS client (openssl SSL/TLS + certificate verification, see https.vp) # # Usage: # requests._mbuddy = my_mbuddy # resp = requests.get("http://example.com/path") # if resp is not None: # print(resp.status_code) # print(resp.text) # requests.release(resp) # ============================================================ # Module-level global MBuddy pointer (assigned by the user in main) _mbuddy: memhub.MemBuddy* # Socket initialization flag _socket_ready: bool = False define HTTP_PORT = 80 define HTTP_BUFSIZ = 4096 define GZIP_BUFSIZ = 65536 # Module-internal redirect / failure diagnostics (static buffer) _http_err_store: [512]char def last_error() -> str: """Last HTTP-layer failure reason (empty string if none).""" return _http_err_store def _SetHttpError(text: str): if text is None: return _http_err_store[0] = 0 stdio.snprintf(_http_err_store, 512, "%s", text) # ============================================================ # Response — HTTP response object # ============================================================ class Response: """HTTP response object, similar to how Python's requests.Response works. Allocated from _mbuddy, call release(resp) to free it after use. """ status_code: int # HTTP status code (e.g., 200, 404) text: str # Response body text body_len: int # Response body length _raw_buf: str # raw response buffer (headers+body, freed by release) _body_owned: str # separately decoded body buffer (chunked etc., freed by release) # ============================================================ # internal auxiliary functions # ============================================================ def _EnsureSocketInit() -> int: """Ensure Socket library is initialized (idempotent).""" global _socket_ready if _socket_ready: return 0 result: int = socket.SocketInit() if result == 0: _socket_ready = True return result def _AllocResponse() -> Response*: """Allocate Response struct from _mbuddy and initialize it in-place.""" if _mbuddy is None: return None resp: Response* = _mbuddy.alloc(Response.__sizeof__()) if resp is None: return None resp.status_code = 0 resp.text = None resp.body_len = 0 resp._raw_buf = None resp._body_owned = None return resp def _ParseUrl(url: str, host_buf: str, host_size: int, path_buf: str, path_size: int, port: int*) -> int: """Parse URL and extract host, port, and path.""" if url is None: return -1 if host_buf is None or path_buf is None or port is None: return -1 p: str = url # Skip "http://" if string.strncmp(p, "http://", 7) == 0: p += 7 # Parse host host_start: str = p host_len: int = 0 while p[0] != '\0' and p[0] != ':' and p[0] != '/': host_len += 1 p += 1 if host_len == 0 or host_len >= host_size: return -1 string.strncpy(host_buf, host_start, host_len) host_buf[host_len] = 0 # Parse port if p[0] == ':': p += 1 port_val: int = 0 while p[0] >= '0' and p[0] <= '9': port_val = port_val * 10 + int(p[0] - '0') p += 1 *port = port_val else: *port = HTTP_PORT # Parse path if p[0] == '/': path_start: str = p path_len: int = 0 while p[0] != '\0': path_len += 1 p += 1 if path_len >= path_size: return -1 string.strncpy(path_buf, path_start, path_len) path_buf[path_len] = 0 else: if path_size < 2: return -1 path_buf[0] = '/' path_buf[1] = 0 return 0 def _ParseStatusLine(resp_text: str, out_code: int*) -> int: """Parse HTTP status line and extract status code.""" if resp_text is None or out_code is None: return -1 p: str = resp_text # Skip "HTTP/x.x " while p[0] != '\0' and p[0] != ' ': p += 1 if p[0] == ' ': p += 1 # Parse status code code: int = 0 while p[0] >= '0' and p[0] <= '9': code = code * 10 + int(p[0] - '0') p += 1 *out_code = code # Find \n while p[0] != '\0' and p[0] != '\n': p += 1 if p[0] == '\n': p += 1 return int(uint64(p) - uint64(resp_text)) def _FindBodyStart(resp_text: str, resp_len: int) -> int: """Find the start of the HTTP response body (after rnrn).""" if resp_text is None: return -1 end_of_headers: str = string.strstr(resp_text, "\r\n\r\n") if end_of_headers is None: end_of_headers = string.strstr(resp_text, "\n\n") if end_of_headers is None: return -1 return int(uint64(end_of_headers) - uint64(resp_text)) + 2 return int(uint64(end_of_headers) - uint64(resp_text)) + 4 # ============================================================ # Public helpers (reused by requests.https and other submodules) # ============================================================ def create_response() -> Response*: """Create and initialize an empty HTTP Response (allocated from _mbuddy). Call release(resp) to free it after use. """ return _AllocResponse() def parse_status_line(resp_text: str, out_code: int*) -> int: """Parse HTTP status line, write status code into out_code.""" return _ParseStatusLine(resp_text, out_code) def find_body_start(resp_text: str, resp_len: int) -> int: """Locate the start of the HTTP response body.""" return _FindBodyStart(resp_text, resp_len) # ============================================================ # Response parsing (header lookup / chunked decode / unified parse_response) # ============================================================ def _CIEq(a: char, b: char) -> bool: """Case-insensitive char equality (ASCII only).""" if a >= 'A' and a <= 'Z': a = char(int(a) - int('A') + int('a')) if b >= 'A' and b <= 'Z': b = char(int(b) - int('A') + int('a')) return a == b def header_value(resp: Response*, name: str, out: str, out_size: int) -> int: """Look up a response header by name in the raw response (case-insensitive, first value). Returns 0 on found+copied; -1 if missing or bad args. """ if resp is None or resp._raw_buf is None or name is None or out is None: return -1 p: str = resp._raw_buf # Skip status line while p[0] != '\0' and p[0] != '\n': p += 1 if p[0] == '\n': p += 1 # Match each header line while p[0] != '\0' and p[0] != '\r' and p[0] != '\n': h: str = p q: str = name matched: bool = True while q[0] != '\0': if h[0] != q[0] and not _CIEq(h[0], q[0]): matched = False break h += 1 q += 1 # Allow [spaces]* ':' after name (RFC has none; lenient) while h[0] == ' ': h += 1 if matched and h[0] == ':': h += 1 while h[0] == ' ' or h[0] == '\t': h += 1 i: int = 0 while h[0] != '\0' and h[0] != '\r' and h[0] != '\n' and i < out_size - 1: out[i] = h[0] i += 1 h += 1 out[i] = 0 return 0 # Skip to next line while p[0] != '\0' and p[0] != '\n': p += 1 if p[0] == '\n': p += 1 return -1 def _HexVal(ch: char) -> int: if ch >= '0' and ch <= '9': return int(ch - '0') if ch >= 'a' and ch <= 'f': return int(ch - 'a') + 10 if ch >= 'A' and ch <= 'F': return int(ch - 'A') + 10 return -1 def decode_chunked(src: str, src_len: int, dst: str, dst_cap: int) -> int: """Decode chunked body per RFC 7230 (chunk-ext ignored, trailer ignored). Returns decoded byte count; -1 on failure. """ if src is None or dst is None: return -1 r: int = 0 w: int = 0 while r < src_len: # 1) chunk size (hex, optional ;ext) size: int = 0 digit_seen: bool = False while r < src_len: ch: char = src[r] if ch == '\r' or ch == '\n' or ch == ';': break hv: int = _HexVal(ch) if hv < 0: return -1 digit_seen = True if size > 0x7fffffff: return -1 size = size * 16 + hv r += 1 if not digit_seen: return -1 # 2) Skip extension and line end while r < src_len and src[r] != '\n': r += 1 if r < src_len and src[r] == '\n': r += 1 # 3) Terminator chunk if size == 0: return w # 4) Copy data + skip CRLF if w + size > dst_cap: return -1 for k in range(int(size)): dst[w + k] = src[r + k] w += size r += size if r < src_len and src[r] == '\r': r += 1 if r < src_len and src[r] == '\n': r += 1 return w def parse_response(resp: Response*, total: int) -> int: """Parse the received raw response into Response fields; chunked Transfer-Encoding decodes into _body_owned while _raw_buf is kept. Returns 0 ok; -1 fail (text=None, body_len=0). """ if resp is None or resp._raw_buf is None: return -1 buf: str = resp._raw_buf buf[total] = 0 _ParseStatusLine(buf, &resp.status_code) body_offset: int = _FindBodyStart(buf, total) if body_offset < 0 or body_offset >= total: resp.text = None resp.body_len = 0 return 0 # Detect Transfer-Encoding: chunked te_buf: [64]char chunked: bool = False if header_value(resp, "Transfer-Encoding", te_buf, 64) == 0: if string.strstr(te_buf, "chunked") is not None: chunked = True if chunked: new_buf: str = _mbuddy.alloc(HTTP_BUFSIZ) if new_buf is None: resp.text = None resp.body_len = 0 return -1 dec_len: int = decode_chunked(buf + body_offset, total - body_offset, new_buf, HTTP_BUFSIZ) if dec_len < 0: _mbuddy.free(new_buf) resp.text = None resp.body_len = 0 return -1 new_buf[dec_len] = 0 resp._body_owned = new_buf resp.text = new_buf resp.body_len = dec_len else: resp.text = buf + body_offset resp.body_len = total - body_offset # ---- Content-Encoding: gzip decompress (zlib.pyzlib, pure Viper) ---- ce_buf: [64]char if header_value(resp, "Content-Encoding", ce_buf, 64) == 0 \ and string.strstr(ce_buf, "gzip") is not None \ and resp.text is not None and resp.body_len > 0: src_len2: sizet = sizet(resp.body_len) olen2: sizet = 0 gz_out: bytes = pyzlib.decompress(_mbuddy, bytes(resp.text), src_len2, 15, GZIP_BUFSIZ, &olen2) if gz_out is not None: old_owned: str = resp._body_owned if old_owned is not None: _mbuddy.free(old_owned) gz_str: str = str(gz_out) gz_str[olen2] = 0 resp._body_owned = gz_str resp.text = gz_str resp.body_len = int(olen2) else: # Decompress failed: keep raw body, record diagnostics stdio.snprintf(_http_err_store, 512, "gzip decode failed: %s", pyzlib.get_error()) return 0 # ============================================================ # Public API # ============================================================ def _HttpGetOnce(url: str) -> Response*: """HTTP GET single request (no redirect). Sends Accept-Encoding: gzip.""" if _mbuddy is None: return None resp: Response* = _AllocResponse() if resp is None: return None if _EnsureSocketInit() != 0: release(resp) return None # Parse URL host_buf: [256]char path_buf: [1024]char port: int = 0 if _ParseUrl(url, host_buf, 256, path_buf, 1024, &port) != 0: release(resp) return None # Create Socket sock = socket.Socket(socket.AF_INET, socket.SOCK_STREAM) if sock.fd == socket.INVALID_SOCKET: release(resp) return None if sock.connect(host_buf, port) != 0: sock.close() release(resp) return None # Build HTTP request req_buf: str = _mbuddy.alloc(HTTP_BUFSIZ) if req_buf is None: sock.close() release(resp) return None stdio.snprintf(req_buf, HTTP_BUFSIZ, "GET %s HTTP/1.1\r\nHost: %s\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n", path_buf, host_buf) req_len: int = int(string.strlen(req_buf)) sent: int = sock.send(req_buf, req_len) _mbuddy.free(req_buf) if sent <= 0: sock.close() release(resp) return None # Receive response body resp_buf: str = _mbuddy.alloc(HTTP_BUFSIZ) if resp_buf is None: sock.close() release(resp) return None total: int = 0 while total < HTTP_BUFSIZ - 1: n: int = sock.recv(resp_buf + total, HTTP_BUFSIZ - 1 - total) if n <= 0: break total += n resp_buf[total] = 0 sock.close() resp._raw_buf = resp_buf parse_response(resp, total) return resp def _HttpPostOnce(url: str, data: str = None, content_type: str = "application/x-www-form-urlencoded") -> Response*: """HTTP POST single request (no redirect). Sends Accept-Encoding: gzip.""" if _mbuddy is None: return None resp: Response* = _AllocResponse() if resp is None: return None if _EnsureSocketInit() != 0: release(resp) return None # Parse URL host_buf: [256]char path_buf: [1024]char port: int = 0 if _ParseUrl(url, host_buf, 256, path_buf, 1024, &port) != 0: release(resp) return None body_len: int = int(string.strlen(data)) if data is not None else 0 ct: str = content_type if content_type is not None else "application/x-www-form-urlencoded" # Create Socket sock = socket.Socket(socket.AF_INET, socket.SOCK_STREAM) if sock.fd == socket.INVALID_SOCKET: release(resp) return None if sock.connect(host_buf, port) != 0: sock.close() release(resp) return None # Build HTTP request req_buf: str = _mbuddy.alloc(HTTP_BUFSIZ) if req_buf is None: sock.close() release(resp) return None stdio.snprintf(req_buf, HTTP_BUFSIZ, "POST %s HTTP/1.1\r\nHost: %s\r\nContent-Type: %s\r\nContent-Length: %d\r\nAccept-Encoding: gzip\r\nConnection: close\r\n\r\n", path_buf, host_buf, ct, body_len) hdr_len: int = int(string.strlen(req_buf)) # Send headers + body sent: int = sock.send(req_buf, hdr_len) if sent > 0 and body_len > 0 and data is not None: sock.send(data, body_len) _mbuddy.free(req_buf) # Receive response body resp_buf: str = _mbuddy.alloc(HTTP_BUFSIZ) if resp_buf is None: sock.close() release(resp) return None total: int = 0 while total < HTTP_BUFSIZ - 1: n: int = sock.recv(resp_buf + total, HTTP_BUFSIZ - 1 - total) if n <= 0: break total += n resp_buf[total] = 0 sock.close() resp._raw_buf = resp_buf parse_response(resp, total) return resp # ============================================================ # Redirect following # ============================================================ def _ResolveRedirectUrl(base: str, loc: str, out: str, cap: int) -> int: """Merge Location into a full URL (absolute / protocol-relative / root-relative), written to out. Returns 0 on success; -1 on failure. """ if base is None or loc is None or out is None: return -1 lb: int = int(string.strlen(loc)) if string.strncmp(loc, "http://", 7) == 0 or string.strncmp(loc, "https://", 8) == 0: if lb >= cap: return -1 string.strcpy(out, loc) return 0 # Take scheme://authority prefix of base q: str = base while q[0] != '\0' and q[0] != ':': q += 1 if q[0] != ':': return -1 q += 1 if q[0] != '/': return -1 q += 1 if q[0] != '/': return -1 q += 1 while q[0] != '\0' and q[0] != '/': q += 1 prefix_len: int = int(uint64(q) - uint64(base)) need: int = prefix_len + lb + 2 if need >= cap: return -1 for i in range(int(prefix_len)): out[i] = base[i] if loc[0] == '/': j: int = 0 while loc[j] != '\0': out[prefix_len + j] = loc[j] j += 1 out[prefix_len + j] = 0 else: out[prefix_len] = '/' j: int = 0 while loc[j] != '\0': out[prefix_len + 1 + j] = loc[j] j += 1 out[prefix_len + 1 + j] = 0 return 0 def _HttpRedirectLoop(method: str, url: str, data: str, content_type: str) -> Response*: """GET/POST with redirect following (http:// chain, max 5 hops).""" if _mbuddy is None: return None cur: str = _mbuddy.alloc(2048) if cur is None: return None if int(string.strlen(url)) >= 2047: _SetHttpError("url too long") _mbuddy.free(cur) return None string.strcpy(cur, url) for hop in range(5): resp: Response* = None if string.strcmp(method, "POST") == 0: resp = _HttpPostOnce(cur, data, content_type) else: resp = _HttpGetOnce(cur) if resp is None: _mbuddy.free(cur) return None code: int = resp.status_code if not _IsHttpRedirect(code): _mbuddy.free(cur) return resp loc: str = _mbuddy.alloc(2048) if loc is None: _mbuddy.free(cur) return resp if header_value(resp, "Location", loc, 2048) != 0: _mbuddy.free(loc) _mbuddy.free(cur) return resp nxt: str = _mbuddy.alloc(2048) if nxt is None: _mbuddy.free(loc) _mbuddy.free(cur) return resp if _ResolveRedirectUrl(cur, loc, nxt, 2048) != 0: _SetHttpError("unresolvable redirect Location") release(resp) _mbuddy.free(nxt) _mbuddy.free(loc) _mbuddy.free(cur) return None if string.strncmp(nxt, "https://", 8) == 0: _SetHttpError("redirect to https:// needs requests.https") release(resp) _mbuddy.free(nxt) _mbuddy.free(loc) _mbuddy.free(cur) return None release(resp) _mbuddy.free(loc) _mbuddy.free(cur) cur = nxt _SetHttpError("too many redirects") _mbuddy.free(cur) return None def _IsHttpRedirect(code: int) -> bool: return code >= 300 and code < 400 def get(url: str) -> Response*: """HTTP GET (follows 3xx redirects, max 5 hops). Returns Response* or None on failure (see last_error()). """ return _HttpRedirectLoop("GET", url, None, None) def post(url: str, data: str = None, content_type: str = "application/x-www-form-urlencoded") -> Response*: """HTTP POST (follows 3xx redirects, max 5 hops).""" return _HttpRedirectLoop("POST", url, data, content_type) def release(resp: Response*): """Release Response object and its internal buffers. Args: resp: The Response object to release """ if resp is None: return if _mbuddy is None: return if resp._body_owned is not None: _mbuddy.free(resp._body_owned) if resp._raw_buf is not None: _mbuddy.free(resp._raw_buf) _mbuddy.free(rawptr(resp)) # ============================================================ # Large-file Range download primitive for requests.download: _DlOnce issues a "Range: bytes= # start-end" GET (no Accept-Encoding) into dst. Thread-safe (no _mbuddy, own socket). >=0 bytes; -1/-2/-3. # ============================================================ def _RawHeaderValue(buf: str, total: int, name: str, out: str, out_size: int) -> int: """Case-insensitive lookup on raw response header area (0=hit+copied; -1=miss).""" if buf is None or name is None or out is None or out_size <= 0: return -1 body: int = _FindBodyStart(buf, total) if body < 0: body = total nl: int = 0 while nl < body: eol: int = nl while eol < body and buf[eol] != '\r' and buf[eol] != '\n': eol += 1 if eol > nl: k: int = 0 ok: bool = True while name[k] != 0 and nl + k < eol: if not _CIEq(buf[nl + k], name[k]): ok = False break k += 1 if ok and name[k] == 0 and nl + k < eol and buf[nl + k] == ':': v: int = nl + k + 1 while v < eol and (buf[v] == ' ' or buf[v] == '\t'): v += 1 vlen: int = eol - v if vlen > out_size - 1: vlen = out_size - 1 for i in range(int(vlen)): out[i] = buf[v + i] out[vlen] = 0 return 0 nl = eol while nl < body and (buf[nl] == '\r' or buf[nl] == '\n'): nl += 1 return -1 def _StatusRaw(buf: str) -> int: """Parse status code from the first line ("HTTP/1.1 206 ...").""" if buf is None: return 0 i: int = 0 while buf[i] == ' ': i += 1 while buf[i] != 0 and buf[i] != ' ': i += 1 while buf[i] == ' ': i += 1 code: int = 0 while buf[i] >= '0' and buf[i] <= '9': code = code * 10 + int(buf[i] - '0') i += 1 return code def _CLenOf(buf: str, total: int) -> int: """Read Content-Length (-1 if absent).""" hbuf: [32]char if _RawHeaderValue(buf, total, "Content-Length", hbuf, 32) != 0: return -1 v: int = 0 i: int = 0 while hbuf[i] >= '0' and hbuf[i] <= '9': v = v * 10 + int(hbuf[i] - '0') i += 1 return v def _CRTotalOf(buf: str, total: int) -> sizet: """Parse TOTAL from Content-Range: bytes s-e/TOTAL (0 if not 206 / header absent).""" hbuf: [128]char if _RawHeaderValue(buf, total, "Content-Range", hbuf, 128) != 0: return 0 p: int = 0 while hbuf[p] != 0 and hbuf[p] != '/': p += 1 if hbuf[p] == 0: return 0 p += 1 v: sizet = 0 while hbuf[p] != 0 and hbuf[p] >= '0' and hbuf[p] <= '9': v = v * 10 + sizet(hbuf[p] - '0') p += 1 return v def _DlOnce(url: str, start: sizet, count: int, dst: str, dst_cap: int, out_status: int*, out_loc: str, out_loccap: int, out_total: sizet*) -> int: """HTTP single Range fetch (thread-safe; see module comment).""" if url is None or dst is None or dst_cap <= 0 \ or out_status is None or out_total is None: return -1 *out_status = 0 *out_total = 0 if out_loc is not None and out_loccap > 0: out_loc[0] = 0 if _EnsureSocketInit() != 0: _SetHttpError("socket init failed") return -3 host_buf: [256]char path_buf: [1024]char port: int = 0 if _ParseUrl(url, host_buf, 256, path_buf, 1024, &port) != 0: _SetHttpError("bad http url") return -3 sock = socket.Socket(socket.AF_INET, socket.SOCK_STREAM) if sock.fd == socket.INVALID_SOCKET: _SetHttpError("socket() failed") return -3 if sock.connect(host_buf, port) != 0: _SetHttpError("connect failed") sock.close() return -3 req: str = stdlib.malloc(1024) if req is None: sock.close() _SetHttpError("out of memory") return -3 cnt: int = count if cnt < 0: cnt = 0 end: sizet = start + sizet(cnt) - 1 if cnt <= 0: end = start stdio.snprintf(req, 1024, "GET %s HTTP/1.1\r\nHost: %s\r\nRange: bytes=%zu-%zu\r\nConnection: close\r\n\r\n", path_buf, host_buf, start, end) sent: int = sock.send(req, int(string.strlen(req))) stdlib.free(req) if sent <= 0: sock.close() _SetHttpError("send failed") return -3 total: int = 0 tgt: int = -1 while total < dst_cap - 1: room: int = dst_cap - 1 - total if room > 4096: room = 4096 n: int = sock.recv(dst + total, room) if n <= 0: break total += n if tgt < 0: bo: int = _FindBodyStart(dst, total) if bo >= 0: cl: int = _CLenOf(dst, total) if cl >= 0: tgt = bo + cl if tgt >= 0 and total >= tgt: break sock.close() dst[total] = 0 st: int = _StatusRaw(dst) *out_status = st if st == 0: _SetHttpError("empty response") return -3 if _IsHttpRedirect(st): if out_loc is not None and out_loccap > 0: if _RawHeaderValue(dst, total, "Location", out_loc, out_loccap) != 0: out_loc[0] = 0 return -2 if st != 200 and st != 206: stdio.snprintf(_http_err_store, 512, "http status %d", st) return -3 if st == 206: *out_total = _CRTotalOf(dst, total) else: cl2: int = _CLenOf(dst, total) if cl2 > 0: *out_total = sizet(cl2) bo: int = _FindBodyStart(dst, total) if bo < 0 or bo >= total: return 0 blen: int = total - bo if blen > cnt and cnt > 0: blen = cnt return blen