libcurl小记-简单http封装使用-源码分析

欢迎关注技术公众号：汉客儿
img

0x00. 前面

以前用Wininet api包了一个简单易用的http请求的lib，但是居然会遇到系统不支持的情况，难道要我自己用socket写吗？no way！

以前知道liburl，第一次使用，啥都不知道，反正感觉挺强大的

< libcurl is a free and easy-to-use client-side URL transfer library, supporting DICT, FILE, FTP, FTPS, Gopher, HTTP, HTTPS, IMAP, IMAPS, LDAP, LDAPS, POP3, POP3S, RTMP, RTSP, SCP, SFTP, SMTP, SMTPS, Telnet and TFTP.

须知，我这里只用到了HTTP

home: https://curl.haxx.se/

document: https://curl.haxx.se/libcurl/

其实使用比较简单，但对我没有认真看过文档，并且没有找到好资料的情况下，我遇到了很多弯路，并且想吐槽实例代码，搞那么复杂干嘛，还没有我想要的代码。

下面开始坑。

0x01. 就这么简单

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
/* curl stuff */ 
#include <curl/curl.h>
//#pragma comment(lib, "liburl.lib")

int main(void)
{
  CURL *curl;
  CURLcode res;
 
  /* In windows, this will init the winsock stuff */ 
  curl_global_init(CURL_GLOBAL_ALL);
 
  /* get a curl handle */ 
  curl = curl_easy_init();
  if(curl) {
    /* First set the URL that is about to receive our POST. This URL can
       just as well be a https:// URL if that is what should receive the
       data. */ 
    curl_easy_setopt(curl, CURLOPT_URL, "http://postit.example.com/moo.cgi");
    /* Now specify the POST data */ 
    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");
 
    /* Perform the request, res will get the return code */ 
    res = curl_easy_perform(curl);
    /* Check for errors */ 
    if(res != CURLE_OK)
      fprintf(stderr, "curl_easy_perform() failed: %s\n",
              curl_easy_strerror(res));
 
    /* always cleanup */ 
    curl_easy_cleanup(curl);
  }
  curl_global_cleanup();
  return 0;
}

简单应用就是这样子，关键在这两句

1
2
3
curl_easy_setopt(curl, CURLOPT_URL, "http://postit.example.com/moo.cgi");
    /* Now specify the POST data */ 
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");

liburl通过设置各种回调函数来完成各种功能。

0x02. http请求

CURLOPT_URL ： 访问的目标url路径，如果是GET方式请求，需要将请求数据加到URL后面

1
www.baidu.com/login.asp?name=111&password=111

CURLOPT_POSTFIELDS ： POST请求中发送的数据

1
curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "name=daniel&project=curl");

并且POST请求中还需要设置CURLOPT_POST为1

1
2
3
/* size of the POST data */
curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.length());
curl_easy_setopt(curl, CURLOPT_POST, 1);

另外，奇葩的

1
curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); //url在release可正常使用，debug去不行，只能使用url.c_str();

0x03. 接收数据

接收数据需要注册CURLOPT_WRITEFUNCTION回调函数，在回调函数中进行数据处理

1
curl_easy_setopt(*curl, CURLOPT_WRITEFUNCTION, write_callback);

如果数据不能一次接收完成，需要利用回调中的参数来缓存数据，也就是通过CURLOPT_WRITEDATA设置

1
curl_easy_setopt(*curl, CURLOPT_WRITEDATA, data);

回调函数处理中，最后一次参数就是设置的用于缓存的变量，需要注意的是数据长度，不是size，而是size*nmemb。 并且如果返回值不等于size*nmemb，libcurl会认为处理失败

1
2
3
4
5
6
7
8
9
10
size_t write_callback(char *ptr, size_t size, size_t nmemb, void *userdata)
{
	size_t all_size = size*nmemb;
	PWRITE_CALLBACK_DATA data = (PWRITE_CALLBACK_DATA)userdata;

	data->data.append(ptr);
	data->size += all_size;

	return all_size;
}

0x04. cookie

通过cookie文件保存，读取cookie

1
2
curl_easy_setopt(*curl, CURLOPT_COOKIEJAR, "cookie.txt");        //把服务器发过来的cookie保存到cookie.txt
curl_easy_setopt(*curl, CURLOPT_COOKIEFILE, "cookie.txt");        //读取本地存储的cookie

直接设置cookie信息

1
//curl_easy_setopt(curl, CURLOPT_COOKIE, m_cookies.c_str());

0x05. 一点点分析

下面是遇到坑时的一小点点源码翻阅，觉得有用的可以看看

1. curl_easy_setopt调用中

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
//curl_easy_setopt调用中
CURLcode curl_easy_setopt(struct Curl_easy *data, CURLoption tag, ...)//lib/easy.c
-->
CURLcode Curl_setopt(struct Curl_easy *data, CURLoption option,
                     va_list param)//lib/url.c
{
    //根据option类型，设置不同回调
    //保存在data->set的不同字段中
    case CURLOPT_URL:
    if(data->change.url_alloc) {
      /* the already set URL is allocated, free it first! */
      Curl_safefree(data->change.url);
      data->change.url_alloc = FALSE;
    }
    result = setstropt(&data->set.str[STRING_SET_URL],
                       va_arg(param, char *));
    data->change.url = data->set.str[STRING_SET_URL];
    break;
    case CURLOPT_PORT:
    data->set.use_port = va_arg(param, long);
    break;
    case CURLOPT_WRITEFUNCTION:
    data->set.fwrite_func = va_arg(param, curl_write_callback);
    if(!data->set.fwrite_func) {
      data->set.is_fwrite_set = 0;
      /* When set to NULL, reset to our internal default function */
      data->set.fwrite_func = (curl_write_callback)fwrite;
    }
    else
      data->set.is_fwrite_set = 1;
    break;
    case CURLOPT_WRITEDATA:
    data->set.out = va_arg(param, void *);
    break;
    case CURLOPT_HTTPHEADER:
    data->set.headers = va_arg(param, struct curl_slist *);
    break;
    case CURLOPT_COOKIEJAR:
    {
    struct CookieInfo *newcookies;
    result = setstropt(&data->set.str[STRING_COOKIEJAR],
                       va_arg(param, char *));
    newcookies = Curl_cookie_init(data, NULL, data->cookies,
                                  data->set.cookiesession);
    if(!newcookies)
      result = CURLE_OUT_OF_MEMORY;
    data->cookies = newcookies;
    }
    break;

}

2. 请求中

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
CURLcode curl_easy_perform(struct Curl_easy *data)//lib/easy.c
->
static CURLcode easy_perform(struct Curl_easy *data, bool events)//lib/easy.c
->
static CURLcode easy_transfer(struct Curl_multi *multi)//lib/easy.c
->
CURLMcode curl_multi_perform(struct Curl_multi *multi, int *running_handles)//\lib\multi.c
{
  data=multi->easyp;//就是Curl_easy *data
  while(data) {
    CURLMcode result;
    SIGPIPE_VARIABLE(pipe_st);

    sigpipe_ignore(data, &pipe_st);
    result = multi_runsingle(multi, now, data);//一次请求
    sigpipe_restore(&pipe_st);

    if(result)
      returncode = result;

    data = data->next; /* operate on next handle */
  }
}
->
static CURLMcode multi_runsingle(struct Curl_multi *multi,
                                 struct timeval now,
                                 struct Curl_easy *data)//\lib\multi.c
{
    //这里面有个重要的字段data->mstate，表示当前curl的状态
    //通过multistate(data, CURLM_STATE_PERFORM);=>static void mstate(struct Curl_easy *data, CURLMstate state)赋值
    //CURLMcode curl_multi_add_handle(struct Curl_multi *multi, struct Curl_easy *data) => multistate(data, CURLM_STATE_INIT);
    //CURLMcode Curl_multi_add_perform(struct Curl_multi *multi, struct Curl_easy *data, struct connectdata *conn)=>multistate(data, CURLM_STATE_PERFORM);
    //等等
    
    //...
    do{
        //本函数中，循环各种状态判断，处理不同逻辑
        switch(data->mstate) {
        //初始化
        case CURLM_STATE_INIT:
          /* init this transfer. */
          result=Curl_pretransfer(data);//各种信息初始化，ssl，cookie
    
          if(!result) {
            /* after init, go CONNECT */
            multistate(data, CURLM_STATE_CONNECT);//状态更改
            Curl_pgrsTime(data, TIMER_STARTOP);
            rc = CURLM_CALL_MULTI_PERFORM;
          }
          break;
         case CURLM_STATE_CONNECT:
          /* Connect. We want to get a connection identifier filled in. */
          Curl_pgrsTime(data, TIMER_STARTSINGLE);
          result = Curl_connect(data, &data->easy_conn,
                                &async, &protocol_connect);
          if(CURLE_NO_CONNECTION_AVAILABLE == result) {
            /* There was no connection available. We will go to the pending
               state and wait for an available connection. */
            multistate(data, CURLM_STATE_CONNECT_PEND);
    
            /* add this handle to the list of connect-pending handles */
            if(!Curl_llist_insert_next(multi->pending, multi->pending->tail, data))
              result = CURLE_OUT_OF_MEMORY;
            else
              result = CURLE_OK;
            break;
          }
    
          if(!result) {
            /* Add this handle to the send or pend pipeline */
            result = Curl_add_handle_to_pipeline(data, data->easy_conn);
            if(result)
              disconnect_conn = TRUE;
            else {
              if(async)
                /* We're now waiting for an asynchronous name lookup */
                multistate(data, CURLM_STATE_WAITRESOLVE);
              else {
                /* after the connect has been sent off, go WAITCONNECT unless the
                   protocol connect is already done and we can go directly to
                   WAITDO or DO! */
                rc = CURLM_CALL_MULTI_PERFORM;
    
                if(protocol_connect)
                  multistate(data, Curl_pipeline_wanted(multi, CURLPIPE_HTTP1)?
                             CURLM_STATE_WAITDO:CURLM_STATE_DO);
                else {
    #ifndef CURL_DISABLE_HTTP
                  if(data->easy_conn->tunnel_state[FIRSTSOCKET] == TUNNEL_CONNECT)
                    multistate(data, CURLM_STATE_WAITPROXYCONNECT);
                  else
    #endif
                    multistate(data, CURLM_STATE_WAITCONNECT);
                }
              }
            }
          }
          break;
        case CURLM_STATE_DO://开始发送
            /* Perform the protocol's DO action */
            result = multi_do(&data->easy_conn, &dophase_done);
            //->
            //详细的http请求封装，可以看看这个Curl_http
            //CURLcode Curl_http(struct connectdata *conn, bool *done)//\lib\http.c
            
        case CURLM_STATE_DONE:
            /* post-transfer command */
            res = multi_done(&data->easy_conn, result, FALSE);
            
        case CURLM_STATE_PERFORM:
            /* read/write data if it is ready to do so */
            result = Curl_readwrite(data->easy_conn, data, &done);//接受数据中

        //...
        
    }while((rc == CURLM_CALL_MULTI_PERFORM) || multi_ischanged(multi, FALSE));

  data->result = result;
}

//所有状态
/* NOTE: if you add a state here, add the name to the statename[] array as
   well!
*/
typedef enum {
  CURLM_STATE_INIT,         /* 0 - start in this state */
  CURLM_STATE_CONNECT_PEND, /* 1 - no connections, waiting for one */
  CURLM_STATE_CONNECT,      /* 2 - resolve/connect has been sent off */
  CURLM_STATE_WAITRESOLVE,  /* 3 - awaiting the resolve to finalize */
  CURLM_STATE_WAITCONNECT,  /* 4 - awaiting the TCP connect to finalize */
  CURLM_STATE_WAITPROXYCONNECT, /* 5 - awaiting proxy CONNECT to finalize */
  CURLM_STATE_SENDPROTOCONNECT, /* 6 - initiate protocol connect procedure */
  CURLM_STATE_PROTOCONNECT, /* 7 - completing the protocol-specific connect
                                   phase */
  CURLM_STATE_WAITDO,       /* 8 - wait for our turn to send the request */
  CURLM_STATE_DO,           /* 9 - start send off the request (part 1) */
  CURLM_STATE_DOING,        /* 10 - sending off the request (part 1) */
  CURLM_STATE_DO_MORE,      /* 11 - send off the request (part 2) */
  CURLM_STATE_DO_DONE,      /* 12 - done sending off request */
  CURLM_STATE_WAITPERFORM,  /* 13 - wait for our turn to read the response */
  CURLM_STATE_PERFORM,      /* 14 - transfer data */
  CURLM_STATE_TOOFAST,      /* 15 - wait because limit-rate exceeded */
  CURLM_STATE_DONE,         /* 16 - post data transfer operation */
  CURLM_STATE_COMPLETED,    /* 17 - operation complete */
  CURLM_STATE_MSGSENT,      /* 18 - the operation complete message is sent */
  CURLM_STATE_LAST          /* 19 - not a true state, never use this */
} CURLMstate;


//单独看connect
CURLcode Curl_connect(struct Curl_easy *data,
                      struct connectdata **in_connect,
                      bool *asyncp,
                      bool *protocol_done)
{
    static CURLcode create_conn(struct Curl_easy *data,
                            struct connectdata **in_connect,
                            bool *async)
    ->
    static CURLcode resolve_server(struct Curl_easy *data,
                               struct connectdata *conn,
                               bool *async)
                               
}  

result = Curl_async_resolved(data->easy_conn, &protocol_connect);->
result = Curl_setup_conn(conn, protocol_done);->
result = Curl_connecthost(conn, conn->dns_entry);->
result = singleipconnect(conn, conn->tempaddr[0], &(conn->tempsock[0]));->
static CURLcode singleipconnect(struct connectdata *conn,
                                const Curl_addrinfo *ai,
                                curl_socket_t *sockp)
->
CURLcode Curl_socket(struct connectdata *conn,
                     const Curl_addrinfo *ai,
                     struct Curl_sockaddr_ex *addr,
                     curl_socket_t *sockfd)
                     
//Crul对socket等的封装
//\lib\connect.c
CURLcode Curl_socket(struct connectdata *conn,
                     const Curl_addrinfo *ai,
                     struct Curl_sockaddr_ex *addr,
                     curl_socket_t *sockfd)
CURLcode Curl_connecthost(struct connectdata *conn,  /* context */
                          const struct Curl_dns_entry *remotehost)           int Curl_closesocket(struct connectdata *conn,
                      curl_socket_t sock)
curl_socket_t Curl_getconnectinfo(struct Curl_easy *data,
                                  struct connectdata **connp)

3. 接受数据时

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
static CURLMcode multi_runsingle(struct Curl_multi *multi,
                                 struct timeval now,
                                 struct Curl_easy *data)//\lib\multi.c
->
CURLcode Curl_readwrite(struct connectdata *conn,
                        struct Curl_easy *data,
                        bool *done)//\lib\transfer.c
{
 /* We go ahead and do a read if we have a readable socket or if
     the stream was rewound (in which case we have data in a
     buffer) */
  if((k->keepon & KEEP_RECV) &&
     ((select_res & CURL_CSELECT_IN) || conn->bits.stream_was_rewound)) {

    result = readwrite_data(data, conn, k, &didwhat, done);//
    if(result || *done)
      return result;
  }
}
->
static CURLcode readwrite_data(struct Curl_easy *data,
                               struct connectdata *conn,
                               struct SingleRequest *k,
                               int *didwhat, bool *done)//\lib\transfer.c
{
    //接受头部
    result = Curl_http_readwrite_headers(data, conn, &nread, &stop_reading);
    //->
    //CURLcode Curl_client_write(struct connectdata *conn,
    //                       int type,
    //                       char *ptr,
    //                       size_t len)//lib\sendf.c
    
    //接受数据
    result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str,
                                           nread);
    //-》
    //Curl_client_write->Curl_client_chop_write-> 调用回调函数
    //
    
}

//调用回调函数
CURLcode Curl_client_chop_write(struct connectdata *conn,
                                int type,
                                char * ptr,
                                size_t len)
{               

    curl_write_callback writeheader = NULL;
    curl_write_callback writebody = NULL;
    //...
    
    /* Determine the callback(s) to use. */
  if(type & CLIENTWRITE_BODY)
    writebody = data->set.fwrite_func;
    
    if(writebody) {
        //调用回调函数
      size_t wrote = writebody(ptr, 1, chunklen, data->set.out);

}

4. cookie相关

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
//解析指定的cookie文件
//支持# Netscape HTTP Cookie File和Mozilla cookie
CURLcode Curl_pretransfer(struct Curl_easy *data)//lib\transfer.c
{
 /* If there is a list of cookie files to read, do it now! */
  if(data->change.cookielist)//cookie文件列表
    Curl_cookie_loadfiles(data);
}
->
void Curl_cookie_loadfiles(struct Curl_easy *data)//lib\cookie.c
->
//读取cookie文件，初始化cookie结构
struct CookieInfo *Curl_cookie_init(struct Curl_easy *data,
                                    const char *file,
                                    struct CookieInfo *inc,
                                    bool newsession)//lib\cookie.c
->
//lineptr是从cookie.txt中读取的每行数据，解析数据，插入CookieInfo链
struct Cookie *
Curl_cookie_add(struct Curl_easy *data,
                /* The 'data' pointer here may be NULL at times, and thus
                   must only be used very carefully for things that can deal
                   with data being NULL. Such as infof() and similar */

                struct CookieInfo *c,
                bool httpheader, /* TRUE if HTTP header-style line */
                char *lineptr,   /* first character of the line */
                const char *domain, /* default domain */
                const char *path)   /* full path used when this cookie is set,
                                       used to get default path for the cookie
                                       unless set */
{
//...
clist = c->cookies;
  replace_old = FALSE;
  while(clist) {
    if(Curl_raw_equal(clist->name, co->name)) {
      /* the names are identical */

      if(clist->domain && co->domain) {
        if(Curl_raw_equal(clist->domain, co->domain))
          /* The domains are identical */
          replace_old=TRUE;
      }
      else if(!clist->domain && !co->domain)
        replace_old = TRUE;

      if(replace_old) {
        /* the domains were identical */

        if(clist->spath && co->spath) {
          if(Curl_raw_equal(clist->spath, co->spath)) {
            replace_old = TRUE;
          }
          else
            replace_old = FALSE;
        }
        else if(!clist->spath && !co->spath)
          replace_old = TRUE;
        else
          replace_old = FALSE;

      }
      //。。。

      if(replace_old) {
        co->next = clist->next; /* get the next-pointer first */

        *clist = *co;  /* then store all the new data */

        free(co);   /* free the newly alloced memory */
        co = clist; /* point to the previous struct instead */

        /* We have replaced a cookie, now skip the rest of the list but
           make sure the 'lastc' pointer is properly set */
        do {
          lastc = clist;
          clist = clist->next;
        } while(clist);
        break;
      }
    }
    lastc = clist;
    clist = clist->next;
  }

  if(c->running)
    /* Only show this when NOT reading the cookies from a file */
    infof(data, "%s cookie %s=\"%s\" for domain %s, path %s, "
          "expire %" CURL_FORMAT_CURL_OFF_T "\n",
          replace_old?"Replaced":"Added", co->name, co->value,
          co->domain, co->path, co->expires);

  if(!replace_old) {
    /* then make the last item point on this new one */
    if(lastc)
      lastc->next = co;
    else
      c->cookies = co;
    c->numcookies++; /* one more cookie in the jar */
  }
 //...
}

0x06. 其他

封装了一份简单的http类，支持GET、POST、ajax，代码比较简单，有需要的可以拿来用，后续可能会更新

源码：https://github.com/anhkgg/minihttp

https://curl.haxx.se/libcurl/

https://curl.haxx.se/libcurl/3

http://blog.csdn.net/breaksoftware/article/details/45874197

转载请注明出处：http://anhkgg.github.io/liburl-use-minihttp/

赏

谢谢你请我吃糖果

微信

扫一扫，分享到微信

微信分享二维码

tag:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

Windows Kernel/Rootkit/Reverse Engineer/Expolit<br/>内核研究/逆向分析/漏洞分析挖掘