[OTR-dev] otrl_base64_otr_decode() function...

[Ian Goldberg]

Good catches, thanks!

How's this for a proposed patch?  [Paul, I think this addresses your
"magic numbers" concern as well.]

   - Ian

diff --git a/ChangeLog b/ChangeLog
index 7f6e9ed..4684415 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+2012-07-19
+
+	* src/b64.[ch], src/proto.c: Clean up the previous b64 patch
+	and apply it to all places where otrl_base64_decode() is called.
+
 2012-07-17
 
 	* src/b64.c: Use ceil instead of floor to compute the size
diff --git a/src/b64.c b/src/b64.c
index 9ed3feb..8ab0b80 100644
--- a/src/b64.c
+++ b/src/b64.c
@@ -56,7 +56,7 @@ VERSION HISTORY:
 \******************************************************************* */
 
 /* system headers */
-#include <stdlib.h>
+#include <stdio.h>
 #include <string.h>
 
 /* libotr headers */
@@ -148,8 +148,9 @@ static size_t decode(unsigned char *out, const char *in, size_t b64len)
  * base64 decode data.  Skip non-base64 chars, and terminate at the
  * first '=', or the end of the buffer.
  *
- * The buffer data must contain at least (base64len / 4) * 3 bytes of
- * space.  This function will return the number of bytes actually used.
+ * The buffer data must contain at least ((base64len+3) / 4) * 3 bytes
+ * of space.  This function will return the number of bytes actually
+ * used.
  */
 size_t otrl_base64_decode(unsigned char *data, const char *base64data,
 	size_t base64len)
@@ -236,14 +237,18 @@ int otrl_base64_otr_decode(const char *msg, unsigned char **bufp,
 	return -2;
     }
 
+    /* Skip over the "?OTR:" */
+    otrtag += 5;
+    msglen -= 5;
+
     /* Base64-decode the message */
-    rawlen = ((msglen-5+3) / 4) * 3;   /* maximum possible */
+    rawlen = OTRL_B64_MAX_DECODED_SIZE(msglen);   /* maximum possible */
     rawmsg = malloc(rawlen);
     if (!rawmsg && rawlen > 0) {
 	return -1;
     }
 
-    rawlen = otrl_base64_decode(rawmsg, otrtag+5, msglen-5);  /* actual size */
+    rawlen = otrl_base64_decode(rawmsg, otrtag, msglen);  /* actual size */
 
     *bufp = rawmsg;
     *lenp = rawlen;
diff --git a/src/b64.h b/src/b64.h
index 6502cdc..bdd584a 100644
--- a/src/b64.h
+++ b/src/b64.h
@@ -21,6 +21,19 @@
 #ifndef __B64_H__
 #define __B64_H__
 
+#include <stdlib.h>
+
+/* Base64 encodes blocks of this many bytes: */
+#define OTRL_B64_DECODED_LEN 3
+/* into blocks of this many bytes: */
+#define OTRL_B64_ENCODED_LEN 4
+
+/* An encoded block of length encoded_len can turn into a maximum of
+ * this many decoded bytes: */
+#define OTRL_B64_MAX_DECODED_SIZE(encoded_len) \
+    (((encoded_len + OTRL_B64_ENCODED_LEN - 1) / OTRL_B64_ENCODED_LEN) \
+	* OTRL_B64_DECODED_LEN)
+
 /*
  * base64 encode data.  Insert no linebreaks or whitespace.
  *
@@ -34,8 +47,9 @@ size_t otrl_base64_encode(char *base64data, const unsigned char *data,
  * base64 decode data.  Skip non-base64 chars, and terminate at the
  * first '=', or the end of the buffer.
  *
- * The buffer data must contain at least (base64len / 4) * 3 bytes of
- * space.  This function will return the number of bytes actually used.
+ * The buffer data must contain at least ((base64len+3) / 4) * 3 bytes
+ * of space.  This function will return the number of bytes actually
+ * used.
  */
 size_t otrl_base64_decode(unsigned char *data, const char *base64data,
 	size_t base64len);
diff --git a/src/proto.c b/src/proto.c
index aed1b85..3c96e05 100644
--- a/src/proto.c
+++ b/src/proto.c
@@ -443,7 +443,7 @@ gcry_error_t otrl_proto_instance(const char *otrmsg,
     if (strlen(otrtag) < 21 ) goto invval;
 
     /* Decode and extract instance tag */
-    bufp = malloc(9);
+    bufp = malloc(OTRL_B64_MAX_DECODED_SIZE(12));
     bufp_head = bufp;
     lenp = otrl_base64_decode(bufp, otrtag+9, 12);
     read_int(*instance_from);
@@ -649,13 +649,17 @@ gcry_error_t otrl_proto_data_read_flags(const char *datamsg,
 	msglen = strlen(otrtag);
     }
 
+    /* Skip over the "?OTR:" */
+    otrtag += 5;
+    msglen -= 5;
+
     /* Base64-decode the message */
-    rawlen = ((msglen-5) / 4) * 3;   /* maximum possible */
+    rawlen = OTRL_B64_MAX_DECODED_SIZE(msglen);   /* maximum possible */
     rawmsg = malloc(rawlen);
     if (!rawmsg && rawlen > 0) {
 	return gcry_error(GPG_ERR_ENOMEM);
     }
-    rawlen = otrl_base64_decode(rawmsg, otrtag+5, msglen-5);  /* actual size */
+    rawlen = otrl_base64_decode(rawmsg, otrtag, msglen);  /* actual size */
 
     bufp = rawmsg;
     lenp = rawlen;
@@ -721,14 +725,18 @@ gcry_error_t otrl_proto_accept_data(char **plaintextp, OtrlTLV **tlvsp,
 	msglen = strlen(otrtag);
     }
 
+    /* Skip over the "?OTR:" */
+    otrtag += 5;
+    msglen -= 5;
+
     /* Base64-decode the message */
-    rawlen = ((msglen-5) / 4) * 3;   /* maximum possible */
+    rawlen = OTRL_B64_MAX_DECODED_SIZE(msglen);   /* maximum possible */
     rawmsg = malloc(rawlen);
     if (!rawmsg && rawlen > 0) {
 	err = gcry_error(GPG_ERR_ENOMEM);
 	goto err;
     }
-    rawlen = otrl_base64_decode(rawmsg, otrtag+5, msglen-5);  /* actual size */
+    rawlen = otrl_base64_decode(rawmsg, otrtag, msglen);  /* actual size */
 
     bufp = rawmsg;
     lenp = rawlen;