WvStreams
wvhttppool.h
1/* -*- Mode: C++ -*-
2 * Worldvisions Weaver Software:
3 * Copyright (C) 1997-2002 Net Integration Technologies, Inc.
4 *
5 * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
6 *
7 * Just create a WvHttpPool object, add it to your list, and use pool.addurl()
8 * to get a WvStream* that gives you the file you requested.
9 */
10#ifndef __WVHTTPPOOL_H
11#define __WVHTTPPOOL_H
12
13#include "ftpparse.h"
14#include "wvurl.h"
15#include "wvistreamlist.h"
16#include "wvstreamclone.h"
17#include "wvlog.h"
18#include "wvhashtable.h"
19#include "wvbufstream.h"
20#include "wvbuf.h"
21#include "wvcont.h"
22#include "wvtcp.h"
23
24class WvBufUrlStream;
25class WvUrlStream;
26class WvHttpStream;
27
28static const WvString DEFAULT_ANON_PW("weasels@");
29
31{
32 WvString name, value;
33
35 : name(_name), value(_value)
36 {}
37};
38
39
40DeclareWvDict(WvHTTPHeader, WvString, name);
41
42
44{
45public:
46 WvUrl url;
47 WvString headers;
48 WvUrlStream *instream;
49 WvBufUrlStream *outstream;
50 WvStream *putstream;
51
52 bool pipeline_test;
53 bool inuse;
54 bool is_dir;
55 bool create_dirs;
56 WvString method;
57
58 WvUrlRequest(WvStringParm _url, WvStringParm _method, WvStringParm _headers,
59 WvStream *content_source, bool _create_dirs, bool _pipeline_test);
61
62 void done();
63};
64
65DeclareWvList(WvUrlRequest);
66
67
69{
70 WvString linkname;
71 WvUrl url;
72
73 WvUrlLink(WvStringParm _linkname, WvStringParm _url)
74 : linkname(_linkname), url(_url)
75 {}
76};
77DeclareWvList(WvUrlLink);
78
79
81{
82public:
83 WvString url;
84 WvString proto;
85 WvUrlLinkList links; // HTML links or FTP directory listing
86
87 // HTTP stuff...
88 WvString version;
89 int status;
90 WvHTTPHeaderDict headers;
91
92 WvBufUrlStream() : status(0), headers(10)
93 {}
94 virtual ~WvBufUrlStream()
95 {}
96
97public:
98 const char *wstype() const { return "WvBufUrlStream"; }
99};
100
101DeclareWvTable(WvIPPortAddr);
102
103
105{
106public:
107 class Target
108 {
109 public:
110 WvIPPortAddr remaddr;
111 WvString username;
112
113 Target(const WvIPPortAddr &_remaddr, WvStringParm _username)
114 : remaddr(_remaddr), username(_username) {}
115
116 ~Target() {}
117
118 bool operator== (const Target &n2) const
119 { return (username == n2.username && remaddr == n2.remaddr); }
120 };
121 Target target;
122 static int max_requests;
123
124protected:
125 WvLog log;
126 WvUrlRequestList urls, waiting_urls;
127 int request_count;
128 WvUrlRequest *curl; // current url
129 virtual void doneurl() = 0;
130 virtual void request_next() = 0;
131
132public:
133 WvUrlStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
134 WvStringParm logname)
135 : WvStreamClone(new WvTCPConn(_remaddr)), target(_remaddr, _username),
136 log(logname, WvLog::Debug)
137 {
138 request_count = 0;
139 curl = NULL;
140 }
141
142 virtual ~WvUrlStream() {};
143
144 virtual void close() = 0;
145 void addurl(WvUrlRequest *url);
146 void delurl(WvUrlRequest *url);
147 // only implemented in WvHttpStream
148 virtual size_t remaining()
149 { return 0; }
150
151 virtual void execute() = 0;
152
153public:
154 const char *wstype() const { return "WvUrlStream"; }
155};
156
157unsigned WvHash(const WvUrlStream::Target &n);
158
159DeclareWvDict(WvUrlStream, WvUrlStream::Target, target);
160
161
163{
164public:
165 static bool global_enable_pipelining;
166 bool enable_pipelining;
167
168private:
169 int pipeline_test_count;
170 bool ssl;
171 bool sent_url_request; // Have we sent a request to the server yet?
172 WvIPPortAddrTable &pipeline_incompatible;
173 WvString http_response, pipeline_test_response;
174 WvDynBuf putstream_data;
175
176 enum { Unknown, Chunked, ContentLength, Infinity,
177 PostHeadInfinity, PostHeadChunked, PostHeadStream,
178 ChuckInfinity, ChuckChunked, ChuckStream } encoding;
179 size_t bytes_remaining;
180 bool in_chunk_trailer, last_was_pipeline_test, in_doneurl;
181
182 virtual void doneurl();
183 virtual void request_next();
184 void start_pipeline_test(WvUrl *url);
185 WvString request_str(WvUrlRequest *url, bool keep_alive);
186 void send_request(WvUrlRequest *url);
187 void pipelining_is_broken(int why);
188
189public:
190 WvHttpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
191 bool ssl, WvIPPortAddrTable &_pipeline_incompatible);
192 virtual ~WvHttpStream();
193
194 virtual void close();
195 virtual void pre_select(SelectInfo &si);
196 virtual bool post_select(SelectInfo &si);
197 virtual void execute();
198 virtual size_t remaining()
199 { return bytes_remaining; }
200
201public:
202 const char *wstype() const { return "WvHttpStream"; }
203};
204
205
207{
208 bool logged_in, pasv_acked;
209 WvString password;
210 WvTCPConn *data;
211 time_t last_request_time;
212 bool sure;
213
214 virtual void doneurl();
215 virtual void request_next();
216
217 // Disregard all lines that are of the form "xxx-", meaning that another
218 // line follows. Only the last line is important for us.
219 char *get_important_line();
220
221 // Parse response to "PASV" command and returns a pointer to the address
222 // of the data port (or NULL if it can't parse the response)..
223 // This mucks about with line.
224 WvIPPortAddr *parse_pasv_response(char *line);
225
226 WvString parse_for_links(char *line);
227
228 WvCont cont;
229 void* real_execute(void*);
230
231public:
232 WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
233 WvStringParm _password);
234
235 virtual void pre_select(SelectInfo &si);
236 virtual bool post_select(SelectInfo &si);
237 virtual void close();
238 virtual void execute();
239
240public:
241 const char *wstype() const { return "WvFtpStream"; }
242};
243
244
245// FIXME: Rename this to WvUrlPool someday.
247{
248 WvLog log;
249 WvResolver dns;
250 WvUrlStreamDict conns;
251 WvUrlRequestList urls;
252 int num_streams_created;
253 bool sure;
254
255 WvIPPortAddrTable pipeline_incompatible;
256
257public:
258 WvHttpPool();
259 virtual ~WvHttpPool();
260
261 virtual void pre_select(SelectInfo &si);
262 virtual bool post_select(SelectInfo &si);
263 virtual void execute();
264
265 WvBufUrlStream *addurl(WvStringParm _url, WvStringParm _method = "GET",
266 WvStringParm _headers = "",
267 WvStream *content_source = NULL,
268 bool create_dirs = false);
269
270 // For URL uploads. create_dirs should be true if you want all
271 // non-existent directories in _url to be created.
272// WvBufUrlStream *addputurl(WvStringParm _url, WvStringParm _headers,
273// WvStream *s, bool create_dirs = false);
274private:
275 void unconnect(WvUrlStream *s);
276
277public:
278 bool idle() const
279 { return !urls.count(); }
280
281public:
282 const char *wstype() const { return "WvHttpPool"; }
283};
284
285
286#endif // __WVHTTPPOOL_H
WvBufStream stores data written by write(), and returns it later on in read().
Definition: wvbufstream.h:32
WvCont provides "continuations", which are apparently also known as semi-coroutines.
Definition: wvcont.h:30
A WvFastString acts exactly like a WvString, but can take (const char *) strings without needing to a...
Definition: wvstring.h:94
virtual void close()
Close this stream.
Definition: wvftpstream.cc:93
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
Definition: wvftpstream.cc:465
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvftpstream.cc:133
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvftpstream.cc:149
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
Definition: wvhttppool.cc:202
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
Definition: wvhttppool.cc:121
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
Definition: wvhttppool.cc:150
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling ::select().
virtual bool post_select(SelectInfo &si)
post_select() is called after ::select(), and returns true if this object is now ready.
virtual void close()
Close this stream.
Definition: wvhttpstream.cc:73
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
An IP+Port address also includes a port number, with the resulting form www.xxx.yyy....
Definition: wvaddr.h:394
WvStreamList holds a list of WvStream objects – and its select() and callback() functions know how to...
Definition: wvistreamlist.h:21
A WvLog stream accepts log messages from applications and forwards them to all registered WvLogRcv's.
Definition: wvlog.h:57
ASynchronous DNS resolver functions, so that we can do non-blocking lookups.
Definition: wvresolver.h:25
WvStreamClone simply forwards all requests to the "cloned" stream.
Definition: wvstreamclone.h:24
Unified support for streams, that is, sequences of bytes that may or may not be ready for read/write ...
Definition: wvstream.h:25
WvString is an implementation of a simple and efficient printable-string class.
Definition: wvstring.h:330
WvTCPConn tries to make all outgoing connections asynchronously (in the background).
Definition: wvtcp.h:40
virtual void close()=0
Close this stream.
virtual void execute()=0
The callback() function calls execute(), and then calls the user- specified callback if one is define...
Definition: wvurl.h:17
the data structure used by pre_select()/post_select() and internally by select().
Definition: iwvstream.h:50