blob: 8ee92086ac566baad3d8b7181845bc3837f9e0d3 [file] [log] [blame]
bellard24236862006-04-30 21:28:36 +00001#define CONCAT_I(a, b) a ## b
2#define CONCAT(a, b) CONCAT_I(a, b)
3#define pixel_t CONCAT(uint, CONCAT(BPP, _t))
bellard35127792006-05-14 18:11:49 +00004#ifdef GENERIC
aliguori7eac3a82008-09-15 16:03:41 +00005#define NAME CONCAT(generic_, BPP)
bellard35127792006-05-14 18:11:49 +00006#else
7#define NAME BPP
8#endif
bellard24236862006-04-30 21:28:36 +00009
Peter Maydelle12acaf2023-08-18 16:10:56 +010010#define MAX_BYTES_PER_PIXEL 4
11
bellard35127792006-05-14 18:11:49 +000012static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
13 int x, int y, int w, int h,
aliguori7eac3a82008-09-15 16:03:41 +000014 void *last_bg_,
15 void *last_fg_,
bellard35127792006-05-14 18:11:49 +000016 int *has_bg, int *has_fg)
bellard24236862006-04-30 21:28:36 +000017{
Stefano Stabellini1fc62412009-08-03 10:54:32 +010018 VncDisplay *vd = vs->vd;
Gerd Hoffmann9f649162012-10-10 13:29:43 +020019 uint8_t *row = vnc_server_fb_ptr(vd, x, y);
bellard24236862006-04-30 21:28:36 +000020 pixel_t *irow = (pixel_t *)row;
21 int j, i;
aliguori7eac3a82008-09-15 16:03:41 +000022 pixel_t *last_bg = (pixel_t *)last_bg_;
23 pixel_t *last_fg = (pixel_t *)last_fg_;
bellard24236862006-04-30 21:28:36 +000024 pixel_t bg = 0;
25 pixel_t fg = 0;
26 int n_colors = 0;
27 int bg_count = 0;
28 int fg_count = 0;
29 int flags = 0;
Peter Maydelle12acaf2023-08-18 16:10:56 +010030 uint8_t data[(MAX_BYTES_PER_PIXEL + 2) * 16 * 16];
bellard24236862006-04-30 21:28:36 +000031 int n_data = 0;
32 int n_subtiles = 0;
33
Peter Maydelle12acaf2023-08-18 16:10:56 +010034 /* Enforced by set_pixel_format() */
35 assert(vs->client_pf.bytes_per_pixel <= MAX_BYTES_PER_PIXEL);
36
bellard24236862006-04-30 21:28:36 +000037 for (j = 0; j < h; j++) {
Paolo Bonzinib86d01b2018-12-14 12:38:28 +010038 for (i = 0; i < w; i++) {
39 switch (n_colors) {
40 case 0:
41 bg = irow[i];
42 n_colors = 1;
43 break;
44 case 1:
45 if (irow[i] != bg) {
46 fg = irow[i];
47 n_colors = 2;
48 }
49 break;
50 case 2:
51 if (irow[i] != bg && irow[i] != fg) {
52 n_colors = 3;
53 } else {
54 if (irow[i] == bg)
55 bg_count++;
56 else if (irow[i] == fg)
57 fg_count++;
58 }
59 break;
60 default:
61 break;
62 }
63 }
64 if (n_colors > 2)
65 break;
66 irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
bellard24236862006-04-30 21:28:36 +000067 }
68
69 if (n_colors > 1 && fg_count > bg_count) {
Paolo Bonzinib86d01b2018-12-14 12:38:28 +010070 pixel_t tmp = fg;
71 fg = bg;
72 bg = tmp;
bellard24236862006-04-30 21:28:36 +000073 }
74
75 if (!*has_bg || *last_bg != bg) {
Paolo Bonzinib86d01b2018-12-14 12:38:28 +010076 flags |= 0x02;
77 *has_bg = 1;
78 *last_bg = bg;
bellard24236862006-04-30 21:28:36 +000079 }
80
Anthony Liguori02c2b872009-12-14 14:11:56 -060081 if (n_colors < 3 && (!*has_fg || *last_fg != fg)) {
Paolo Bonzinib86d01b2018-12-14 12:38:28 +010082 flags |= 0x04;
83 *has_fg = 1;
84 *last_fg = fg;
bellard24236862006-04-30 21:28:36 +000085 }
86
87 switch (n_colors) {
88 case 1:
Paolo Bonzinib86d01b2018-12-14 12:38:28 +010089 n_data = 0;
90 break;
bellard24236862006-04-30 21:28:36 +000091 case 2:
Paolo Bonzinib86d01b2018-12-14 12:38:28 +010092 flags |= 0x08;
bellard24236862006-04-30 21:28:36 +000093
Paolo Bonzinib86d01b2018-12-14 12:38:28 +010094 irow = (pixel_t *)row;
ths5fafdf22007-09-16 21:08:06 +000095
Paolo Bonzinib86d01b2018-12-14 12:38:28 +010096 for (j = 0; j < h; j++) {
97 int min_x = -1;
98 for (i = 0; i < w; i++) {
99 if (irow[i] == fg) {
100 if (min_x == -1)
101 min_x = i;
102 } else if (min_x != -1) {
103 hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
104 n_data += 2;
105 n_subtiles++;
106 min_x = -1;
107 }
108 }
109 if (min_x != -1) {
110 hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
111 n_data += 2;
112 n_subtiles++;
113 }
114 irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
115 }
116 break;
bellard24236862006-04-30 21:28:36 +0000117 case 3:
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100118 flags |= 0x18;
bellard24236862006-04-30 21:28:36 +0000119
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100120 irow = (pixel_t *)row;
bellard24236862006-04-30 21:28:36 +0000121
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100122 if (!*has_bg || *last_bg != bg)
123 flags |= 0x02;
bellard24236862006-04-30 21:28:36 +0000124
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100125 for (j = 0; j < h; j++) {
126 int has_color = 0;
127 int min_x = -1;
128 pixel_t color = 0; /* shut up gcc */
bellard24236862006-04-30 21:28:36 +0000129
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100130 for (i = 0; i < w; i++) {
131 if (!has_color) {
132 if (irow[i] == bg)
133 continue;
134 color = irow[i];
135 min_x = i;
136 has_color = 1;
137 } else if (irow[i] != color) {
138 has_color = 0;
bellard35127792006-05-14 18:11:49 +0000139#ifdef GENERIC
140 vnc_convert_pixel(vs, data + n_data, color);
Gerd Hoffmann9f649162012-10-10 13:29:43 +0200141 n_data += vs->client_pf.bytes_per_pixel;
bellard35127792006-05-14 18:11:49 +0000142#else
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100143 memcpy(data + n_data, &color, sizeof(color));
bellard35127792006-05-14 18:11:49 +0000144 n_data += sizeof(pixel_t);
145#endif
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100146 hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
147 n_data += 2;
148 n_subtiles++;
bellard24236862006-04-30 21:28:36 +0000149
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100150 min_x = -1;
151 if (irow[i] != bg) {
152 color = irow[i];
153 min_x = i;
154 has_color = 1;
155 }
156 }
157 }
158 if (has_color) {
bellard35127792006-05-14 18:11:49 +0000159#ifdef GENERIC
160 vnc_convert_pixel(vs, data + n_data, color);
Gerd Hoffmann9f649162012-10-10 13:29:43 +0200161 n_data += vs->client_pf.bytes_per_pixel;
bellard35127792006-05-14 18:11:49 +0000162#else
163 memcpy(data + n_data, &color, sizeof(color));
164 n_data += sizeof(pixel_t);
165#endif
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100166 hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
167 n_data += 2;
168 n_subtiles++;
169 }
170 irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
171 }
bellard24236862006-04-30 21:28:36 +0000172
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100173 /* A SubrectsColoured subtile invalidates the foreground color */
174 *has_fg = 0;
175 if (n_data > (w * h * sizeof(pixel_t))) {
176 n_colors = 4;
177 flags = 0x01;
178 *has_bg = 0;
bellard24236862006-04-30 21:28:36 +0000179
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100180 /* we really don't have to invalidate either the bg or fg
181 but we've lost the old values. oh well. */
182 }
Stefan Weil0ea5c0c2012-02-25 14:57:03 +0100183 break;
bellard24236862006-04-30 21:28:36 +0000184 default:
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100185 break;
bellard24236862006-04-30 21:28:36 +0000186 }
187
188 if (n_colors > 3) {
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100189 flags = 0x01;
190 *has_fg = 0;
191 *has_bg = 0;
192 n_colors = 4;
bellard24236862006-04-30 21:28:36 +0000193 }
194
195 vnc_write_u8(vs, flags);
196 if (n_colors < 4) {
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100197 if (flags & 0x02)
198 vs->write_pixels(vs, last_bg, sizeof(pixel_t));
199 if (flags & 0x04)
200 vs->write_pixels(vs, last_fg, sizeof(pixel_t));
201 if (n_subtiles) {
202 vnc_write_u8(vs, n_subtiles);
203 vnc_write(vs, data, n_data);
204 }
bellard24236862006-04-30 21:28:36 +0000205 } else {
Paolo Bonzinib86d01b2018-12-14 12:38:28 +0100206 for (j = 0; j < h; j++) {
207 vs->write_pixels(vs, row, w * 4);
208 row += vnc_server_fb_stride(vd);
209 }
bellard24236862006-04-30 21:28:36 +0000210 }
211}
212
Peter Maydelle12acaf2023-08-18 16:10:56 +0100213#undef MAX_BYTES_PER_PIXEL
bellard35127792006-05-14 18:11:49 +0000214#undef NAME
bellard24236862006-04-30 21:28:36 +0000215#undef pixel_t
216#undef CONCAT_I
217#undef CONCAT