服务器之家

服务器之家 > 正文

利用stream实现一个简单的http下载器

时间:2021-02-23 11:51     来源/作者:C语言教程网

其实这个http下载器的功能已经相当完善了,支持:限速、post投递和上传、自定义http header、设置user agent、设置range和超时

而且它还不单纯只能下载http,由于使用了stream,所以也支持其他协议,你也可以用它来进行文件之间的copy、纯tcp下载等等。。

完整demo请参考:https://github.com/waruqi/tbox/wiki

stream.c

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
/* //////////////////////////////////////////////////////////////////////////////////////
 * includes
 */
#include "../demo.h"
 
/* //////////////////////////////////////////////////////////////////////////////////////
 * types
 */
typedef struct __tb_demo_context_t
{
  // verbose
  tb_bool_t      verbose;
 
}tb_demo_context_t;
 
/* //////////////////////////////////////////////////////////////////////////////////////
 * func
 */
#ifdef TB_CONFIG_MODULE_HAVE_OBJECT
static tb_bool_t tb_demo_http_post_func(tb_size_t state, tb_hize_t offset, tb_hong_t size, tb_hize_t save, tb_size_t rate, tb_cpointer_t priv)
{
  // percent
  tb_size_t percent = 0;
  if (size > 0) percent = (tb_size_t)((offset * 100) / size);
  else if (state == TB_STATE_CLOSED) percent = 100;
 
  // trace
  tb_trace_i("post: %llu, rate: %lu bytes/s, percent: %lu%%, state: %s", save, rate, percent, tb_state_cstr(state));
 
  // ok
  return tb_true;
}
static tb_bool_t tb_demo_stream_head_func(tb_char_t const* line, tb_cpointer_t priv)
{
  tb_printf("response: %s\n", line);
  return tb_true;
}
static tb_bool_t tb_demo_stream_save_func(tb_size_t state, tb_hize_t offset, tb_hong_t size, tb_hize_t save, tb_size_t rate, tb_cpointer_t priv)
{
  // check
  tb_demo_context_t* context = (tb_demo_context_t*)priv;
  tb_assert_and_check_return_val(context, tb_false);
 
  // print verbose info
  if (context->verbose)
  {
    // percent
    tb_size_t percent = 0;
    if (size > 0) percent = (tb_size_t)((offset * 100) / size);
    else if (state == TB_STATE_CLOSED) percent = 100;
 
    // trace
    tb_printf("save: %llu bytes, rate: %lu bytes/s, percent: %lu%%, state: %s\n", save, rate, percent, tb_state_cstr(state));
  }
 
  // ok
  return tb_true;
}
 
/* //////////////////////////////////////////////////////////////////////////////////////
 * globals
 */
static tb_option_item_t g_options[] =
{
  {'-'"gzip",     TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "enable gzip"        }
,  {'-'"no-verbose",  TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "disable verbose info"   }
,  {'d'"debug",    TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "enable debug info"     }
,  {'k'"keep-alive",  TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "keep alive"        }
,  {'h'"header",    TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_CSTR,    "the custem http header"  }
,  {'-'"post-data",  TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_CSTR,    "set the post data"     }
,  {'-'"post-file",  TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_CSTR,    "set the post file"     }
,  {'-'"range",    TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_CSTR,    "set the range"       }
,  {'-'"timeout",   TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_INTEGER,   "set the timeout"      }
,  {'-'"limitrate",  TB_OPTION_MODE_KEY_VAL,   TB_OPTION_TYPE_INTEGER,   "set the limitrate"     }
,  {'h'"help",     TB_OPTION_MODE_KEY,     TB_OPTION_TYPE_BOOL,    "display this help and exit"}
,  {'-'"url",     TB_OPTION_MODE_VAL,     TB_OPTION_TYPE_CSTR,    "the url"          }
,  {'-',  tb_null,    TB_OPTION_MODE_MORE,    TB_OPTION_TYPE_NONE,    tb_null           }
 
};
 
/* //////////////////////////////////////////////////////////////////////////////////////
 * main
 */
tb_int_t tb_demo_stream_main(tb_int_t argc, tb_char_t** argv)
{
  // done
  tb_option_ref_t   option = tb_null;
  tb_stream_ref_t   istream = tb_null;
  tb_stream_ref_t   ostream = tb_null;
  tb_stream_ref_t   pstream = tb_null;
  do
  {
    // init option
    option = tb_option_init("stream", "the stream demo", g_options);
    tb_assert_and_check_break(option);
   
    // done option
    if (tb_option_done(option, argc - 1, &argv[1]))
    {
      // debug & verbose
      tb_bool_t debug = tb_option_find(option, "debug");
      tb_bool_t verbose = tb_option_find(option, "no-verbose")? tb_false : tb_true;
     
      // done url
      if (tb_option_find(option, "url"))
      {
        // init istream
        istream = tb_stream_init_from_url(tb_option_item_cstr(option, "url"));
        tb_assert_and_check_break(istream);
   
        // ctrl http
        if (tb_stream_type(istream) == TB_STREAM_TYPE_HTTP)
        {
          // enable gzip?
          if (tb_option_find(option, "gzip"))
          {
            // auto unzip
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_AUTO_UNZIP, 1)) break;
 
            // need gzip
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Accept-Encoding", "gzip,deflate")) break;
          }
 
          // enable debug?
          if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD_FUNC, debug? tb_demo_stream_head_func : tb_null)) break;
 
          // custem header?
          if (tb_option_find(option, "header"))
          {
            // init
            tb_string_t key;
            tb_string_t val;
            tb_string_init(&key);
            tb_string_init(&val);
 
            // done
            tb_bool_t      k = tb_true;
            tb_char_t const*  p = tb_option_item_cstr(option, "header");
            while (*p)
            {
              // is key?
              if (k)
              {
                if (*p != ':' && !tb_isspace(*p)) tb_string_chrcat(&key, *p++);
                else if (*p == ':')
                {
                  // skip ':'
                  p++;
 
                  // skip space
                  while (*p && tb_isspace(*p)) p++;
 
                  // is val now
                  k = tb_false;
                }
                else p++;
              }
              // is val?
              else
              {
                if (*p != ';') tb_string_chrcat(&val, *p++);
                else
                {
                  // skip ';'
                  p++;
 
                  // skip space
                  while (*p && tb_isspace(*p)) p++;
 
                  // set header
                  if (tb_string_size(&key) && tb_string_size(&val))
                  {
                    if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
                    if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
                  }
 
                  // is key now
                  k = tb_true;
 
                  // clear key & val
                  tb_string_clear(&key);
                  tb_string_clear(&val);
                }
              }
            }
 
            // set header
            if (tb_string_size(&key) && tb_string_size(&val))
            {
              if (debug) tb_printf("header: %s: %s\n", tb_string_cstr(&key), tb_string_cstr(&val));
              if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, tb_string_cstr(&key), tb_string_cstr(&val))) break;
            }
 
            // exit
            tb_string_exit(&key);
            tb_string_exit(&val);
          }
 
          // keep alive?
          if (tb_option_find(option, "keep-alive"))
          {
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_HEAD, "Connection", "keep-alive")) break;
          }
 
          // post-data?
          if (tb_option_find(option, "post-data"))
          {
            tb_char_t const*  post_data = tb_option_item_cstr(option, "post-data");
            tb_hize_t      post_size = tb_strlen(post_data);
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_DATA, post_data, post_size)) break;
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
            if (debug) tb_printf("post: %llu\n", post_size);
          }
          // post-file?
          else if (tb_option_find(option, "post-file"))
          {
            tb_char_t const* url = tb_option_item_cstr(option, "post-file");
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_METHOD, TB_HTTP_METHOD_POST)) break;
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_URL, url)) break;
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_POST_FUNC, tb_demo_http_post_func)) break;
            if (debug) tb_printf("post: %s\n", url);
          }
        }
 
        // set range
        if (tb_option_find(option, "range"))
        {
          tb_char_t const* p = tb_option_item_cstr(option, "range");
          if (p)
          {
            // the bof
            tb_hize_t eof = 0;
            tb_hize_t bof = tb_atoll(p);
            while (*p && tb_isdigit(*p)) p++;
            if (*p == '-')
            {
              p++;
              eof = tb_atoll(p);
            }
            if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_HTTP_SET_RANGE, bof, eof)) break;
          }
        }
 
        // set timeout
        if (tb_option_find(option, "timeout"))
        {
          tb_size_t timeout = tb_option_item_uint32(option, "timeout");
          if (!tb_stream_ctrl(istream, TB_STREAM_CTRL_SET_TIMEOUT, timeout)) break;
        }
 
        // print verbose info
        if (verbose) tb_printf("open: %s: ..\n", tb_option_item_cstr(option, "url"));
 
        // open istream
        if (!tb_stream_open(istream))
        {
          // print verbose info
          if (verbose) tb_printf("open: %s\n", tb_state_cstr(tb_stream_state(istream)));
          break;
        }
 
        // print verbose info
        if (verbose) tb_printf("open: ok\n");
 
        // init ostream
        if (tb_option_find(option, "more0"))
        {
          // the path
          tb_char_t const* path = tb_option_item_cstr(option, "more0");
 
          // init
          ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
 
          // print verbose info
          if (verbose) tb_printf("save: %s\n", path);
        }
        else
        {
          // the name
          tb_char_t const* name = tb_strrchr(tb_option_item_cstr(option, "url"), '/');
          if (!name) name = tb_strrchr(tb_option_item_cstr(option, "url"), '\\');
          if (!name) name = "/stream.file";
 
          // the path
          tb_char_t path[TB_PATH_MAXN] = {0};
          if (tb_directory_curt(path, TB_PATH_MAXN))
            tb_strcat(path, name);
          else break;
 
          // init file
          ostream = tb_stream_init_from_file(path, TB_FILE_MODE_RW | TB_FILE_MODE_CREAT | TB_FILE_MODE_BINARY | TB_FILE_MODE_TRUNC);
 
          // print verbose info
          if (verbose) tb_printf("save: %s\n", path);
        }
        tb_assert_and_check_break(ostream);
 
        // the limit rate
        tb_size_t limitrate = 0;
        if (tb_option_find(option, "limitrate"))
          limitrate = tb_option_item_uint32(option, "limitrate");
 
        // save it
        tb_hong_t      save = 0;
        tb_demo_context_t  context = {0};
        context.verbose   = verbose;
        if ((save = tb_transfer_done(istream, ostream, limitrate, tb_demo_stream_save_func, &context)) < 0) break;
      }
      else tb_option_help(option);
    }
    else tb_option_help(option);
 
  } while (0);
 
  // exit pstream
  if (pstream) tb_stream_exit(pstream);
  pstream = tb_null;
 
  // exit istream
  if (istream) tb_stream_exit(istream);
  istream = tb_null;
 
  // exit ostream
  if (ostream) tb_stream_exit(ostream);
  ostream = tb_null;
 
  // exit option
  if (option) tb_option_exit(option);
  option = tb_null;
 
  return 0;
}
#else
tb_int_t tb_demo_stream_main(tb_int_t argc, tb_char_t** argv)
{
  return 0;
}
#endif

以上所述就是本文的全部内容了,希望大家能够喜欢。

标签:

相关文章

热门资讯

2020微信伤感网名听哭了 让对方看到心疼的伤感网名大全
2020微信伤感网名听哭了 让对方看到心疼的伤感网名大全 2019-12-26
yue是什么意思 网络流行语yue了是什么梗
yue是什么意思 网络流行语yue了是什么梗 2020-10-11
Intellij idea2020永久破解,亲测可用!!!
Intellij idea2020永久破解,亲测可用!!! 2020-07-29
背刺什么意思 网络词语背刺是什么梗
背刺什么意思 网络词语背刺是什么梗 2020-05-22
苹果12mini价格表官网报价 iPhone12mini全版本价格汇总
苹果12mini价格表官网报价 iPhone12mini全版本价格汇总 2020-11-13
返回顶部