To: vim_dev@googlegroups.com Subject: Patch 7.4.1666 Fcc: outbox From: Bram Moolenaar Mime-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ------------ Patch 7.4.1666 Problem: When reading JSON from a channel all readahead is used. Solution: Use the fill function to reduce overhead. Files: src/channel.c, src/json.c, src/structs.h *** ../vim-7.4.1665/src/channel.c 2016-03-27 19:13:29.618002618 +0200 --- src/channel.c 2016-03-28 13:58:41.814253501 +0200 *************** *** 1184,1190 **** int len = (int)STRLEN(line); char_u *p; - /* TODO: check if channel can be written to, do not block on write */ if ((p = alloc(len + 2)) == NULL) return; STRCPY(p, line); --- 1184,1189 ---- *************** *** 1213,1225 **** in_part->ch_buffer = NULL; return; } - if (in_part->ch_fd == INVALID_FD) - /* pipe was closed */ - return; for (lnum = in_part->ch_buf_top; lnum <= in_part->ch_buf_bot && lnum <= buf->b_ml.ml_line_count; ++lnum) { write_buf_line(buf, lnum, channel); ++written; } --- 1212,1225 ---- in_part->ch_buffer = NULL; return; } for (lnum = in_part->ch_buf_top; lnum <= in_part->ch_buf_bot && lnum <= buf->b_ml.ml_line_count; ++lnum) { + if (in_part->ch_fd == INVALID_FD) + /* pipe was closed */ + return; + /* TODO: check if channel can be written to, do not block on write */ write_buf_line(buf, lnum, channel); ++written; } *************** *** 1365,1374 **** /* * Store "buf[len]" on "channel"/"part". * Returns OK or FAIL. */ static int ! channel_save(channel_T *channel, int part, char_u *buf, int len, char *lead) { readq_T *node; readq_T *head = &channel->ch_part[part].ch_head; --- 1365,1376 ---- /* * Store "buf[len]" on "channel"/"part". + * When "prepend" is TRUE put in front, otherwise append at the end. * Returns OK or FAIL. */ static int ! channel_save(channel_T *channel, int part, char_u *buf, int len, ! int prepend, char *lead) { readq_T *node; readq_T *head = &channel->ch_part[part].ch_head; *************** *** 1400,1413 **** node->rq_buffer[len] = NUL; } ! /* append node to the tail of the queue */ ! node->rq_next = NULL; ! node->rq_prev = head->rq_prev; ! if (head->rq_prev == NULL) head->rq_next = node; else ! head->rq_prev->rq_next = node; ! head->rq_prev = node; if (log_fd != NULL && lead != NULL) { --- 1402,1429 ---- node->rq_buffer[len] = NUL; } ! if (prepend) ! { ! /* preend node to the head of the queue */ ! node->rq_next = head->rq_next; ! node->rq_prev = NULL; ! if (head->rq_next == NULL) ! head->rq_prev = node; ! else ! head->rq_next->rq_prev = node; head->rq_next = node; + } else ! { ! /* append node to the tail of the queue */ ! node->rq_next = NULL; ! node->rq_prev = head->rq_prev; ! if (head->rq_prev == NULL) ! head->rq_next = node; ! else ! head->rq_prev->rq_next = node; ! head->rq_prev = node; ! } if (log_fd != NULL && lead != NULL) { *************** *** 1420,1425 **** --- 1436,1477 ---- return OK; } + static int + channel_fill(js_read_T *reader) + { + channel_T *channel = (channel_T *)reader->js_cookie; + int part = reader->js_cookie_arg; + char_u *next = channel_get(channel, part); + int unused; + int len; + char_u *p; + + if (next == NULL) + return FALSE; + + unused = reader->js_end - reader->js_buf - reader->js_used; + if (unused > 0) + { + /* Prepend unused text. */ + len = (int)STRLEN(next); + p = alloc(unused + len + 1); + if (p == NULL) + { + vim_free(next); + return FALSE; + } + mch_memmove(p, reader->js_buf + reader->js_used, unused); + mch_memmove(p + unused, next, len + 1); + vim_free(next); + next = p; + } + + vim_free(reader->js_buf); + reader->js_buf = next; + reader->js_used = 0; + return TRUE; + } + /* * Use the read buffer of "channel"/"part" and parse a JSON message that is * complete. The messages are added to the queue. *************** *** 1439,1457 **** if (channel_peek(channel, part) == NULL) return FALSE; ! /* TODO: make reader work properly */ ! /* reader.js_buf = channel_peek(channel, part); */ ! reader.js_buf = channel_get_all(channel, part); reader.js_used = 0; ! reader.js_fill = NULL; ! /* reader.js_fill = channel_fill; */ reader.js_cookie = channel; /* When a message is incomplete we wait for a short while for more to * arrive. After the delay drop the input, otherwise a truncated string * or list will make us hang. */ status = json_decode(&reader, &listtv, ! chanpart->ch_mode == MODE_JS ? JSON_JS : 0); if (status == OK) { /* Only accept the response when it is a list with at least two --- 1491,1507 ---- if (channel_peek(channel, part) == NULL) return FALSE; ! reader.js_buf = channel_get(channel, part); reader.js_used = 0; ! reader.js_fill = channel_fill; reader.js_cookie = channel; + reader.js_cookie_arg = part; /* When a message is incomplete we wait for a short while for more to * arrive. After the delay drop the input, otherwise a truncated string * or list will make us hang. */ status = json_decode(&reader, &listtv, ! chanpart->ch_mode == MODE_JS ? JSON_JS : 0); if (status == OK) { /* Only accept the response when it is a list with at least two *************** *** 1552,1561 **** } else if (reader.js_buf[reader.js_used] != NUL) { ! /* Put the unread part back into the channel. ! * TODO: insert in front */ channel_save(channel, part, reader.js_buf + reader.js_used, ! (int)(reader.js_end - reader.js_buf) - reader.js_used, NULL); ret = status == MAYBE ? FALSE: TRUE; } else --- 1602,1611 ---- } else if (reader.js_buf[reader.js_used] != NUL) { ! /* Put the unread part back into the channel. */ channel_save(channel, part, reader.js_buf + reader.js_used, ! (int)(reader.js_end - reader.js_buf) - reader.js_used, ! TRUE, NULL); ret = status == MAYBE ? FALSE: TRUE; } else *************** *** 2419,2425 **** break; /* error or nothing more to read */ /* Store the read message in the queue. */ ! channel_save(channel, part, buf, len, "RECV "); readlen += len; if (len < MAXMSGSIZE) break; /* did read everything that's available */ --- 2469,2475 ---- break; /* error or nothing more to read */ /* Store the read message in the queue. */ ! channel_save(channel, part, buf, len, FALSE, "RECV "); readlen += len; if (len < MAXMSGSIZE) break; /* did read everything that's available */ *************** *** 2446,2452 **** if (channel->ch_part[part].ch_mode == MODE_RAW || channel->ch_part[part].ch_mode == MODE_NL) channel_save(channel, part, (char_u *)DETACH_MSG_RAW, ! (int)STRLEN(DETACH_MSG_RAW), "PUT "); /* TODO: When reading from stdout is not possible, should we try to * keep stdin and stderr open? Probably not, assume the other side --- 2496,2502 ---- if (channel->ch_part[part].ch_mode == MODE_RAW || channel->ch_part[part].ch_mode == MODE_NL) channel_save(channel, part, (char_u *)DETACH_MSG_RAW, ! (int)STRLEN(DETACH_MSG_RAW), FALSE, "PUT "); /* TODO: When reading from stdout is not possible, should we try to * keep stdin and stderr open? Probably not, assume the other side *** ../vim-7.4.1665/src/json.c 2016-03-20 16:40:33.214484441 +0100 --- src/json.c 2016-03-28 14:07:48.244647427 +0200 *************** *** 350,357 **** if (reader->js_fill != NULL && c == NUL) { if (reader->js_fill(reader)) reader->js_end = reader->js_buf + STRLEN(reader->js_buf); ! continue; } if (c == NUL || c > ' ') break; --- 350,359 ---- if (reader->js_fill != NULL && c == NUL) { if (reader->js_fill(reader)) + { reader->js_end = reader->js_buf + STRLEN(reader->js_buf); ! continue; ! } } if (c == NUL || c > ' ') break; *** ../vim-7.4.1665/src/structs.h 2016-03-25 15:40:45.924372980 +0100 --- src/structs.h 2016-03-28 13:59:02.018046345 +0200 *************** *** 2971,2976 **** --- 2971,2977 ---- /* function to fill the buffer or NULL; * return TRUE when the buffer was filled */ void *js_cookie; /* can be used by js_fill */ + int js_cookie_arg; /* can be used by js_fill */ }; typedef struct js_reader js_read_T; *** ../vim-7.4.1665/src/version.c 2016-03-27 19:13:29.618002618 +0200 --- src/version.c 2016-03-28 14:10:42.082865003 +0200 *************** *** 750,751 **** --- 750,753 ---- { /* Add new patch number below this line */ + /**/ + 1666, /**/ -- Drink wet cement and get really stoned. /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net \\\ /// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\ \\\ an exciting new programming language -- http://www.Zimbu.org /// \\\ help me help AIDS victims -- http://ICCF-Holland.org ///